diff --git a/.travis.yml b/.travis.yml
index 282739874d10599bc7363afb668ad4ef54d0363d..efbff23c009b58cda56dc3dc284944ac62e682e1 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,5 +1,6 @@
# TODO: test condor
language: python
+dist: trusty
matrix:
include:
- python: 2.7
diff --git a/docker/coexecutor/Dockerfile b/docker/coexecutor/Dockerfile
index 7d55e69a58bce36a1379b3d089153b4d15434ba7..8ffc70ddc9f65f89953faabe94839916f454d328 100644
--- a/docker/coexecutor/Dockerfile
+++ b/docker/coexecutor/Dockerfile
@@ -4,13 +4,14 @@ ENV PYTHONUNBUFFERED 1
ENV DEBIAN_FRONTEND noninteractive
ENV PULSAR_CONFIG_CONDA_PREFIX /usr/local
+# wget, gcc, pip - to build and install Pulsar.
+# bzip2 for Miniconda.
# TODO: pycurl stuff...
-
RUN apt-get update \
- # Install CVMFS client
&& apt-get install -y --no-install-recommends lsb-release wget \
gcc python-setuptools \
python-dev python-pip \
+ bzip2 \
&& apt-get -y autoremove \
&& apt-get autoclean \
&& rm -rf /var/lib/apt/lists/* /var/log/dpkg.log
@@ -19,4 +20,5 @@ RUN pip install -U pip && pip install wheel kombu pykube poster
ADD pulsar_app-*.dev0-py2.py3-none-any.whl /pulsar_app-*.dev0-py2.py3-none-any.whl
-RUN pip install /pulsar_app-*.dev0-py2.py3-none-any.whl
+RUN pip install /pulsar_app-*.dev0-py2.py3-none-any.whl[galaxy_extended_metadata]
+RUN _pulsar-conda-init --conda_prefix=/pulsar_dependencies/conda
diff --git a/docker/coexecutor/Makefile b/docker/coexecutor/Makefile
index ec1ab0afaf286936b31387535a39aa764081d720..26ede1af46244c922ebd6b89c9f6bab71f519345 100644
--- a/docker/coexecutor/Makefile
+++ b/docker/coexecutor/Makefile
@@ -1,7 +1,9 @@
-docker-image:
+dist:
cd ../..; make dist; cp dist/pulsar*whl docker/coexecutor
-all: docker-image
- docker build -t 'galaxy/pulsar-pod-staging:0.12.0' .
+docker-image:
+ docker build -t 'galaxy/pulsar-pod-staging:0.13.0' .
+
+all: dist docker-image
diff --git a/pulsar/client/client.py b/pulsar/client/client.py
index 81b4cc134d4c29e233fbc975bf3d7c355238635c..298c0458a06e6ad0495fcd3ad6a78752a5f64390 100644
--- a/pulsar/client/client.py
+++ b/pulsar/client/client.py
@@ -397,9 +397,6 @@ class MessageCoexecutionPodJobClient(BaseMessageJobClient):
volume_mounts = [
{"mountPath": "/pulsar_staging", "name": "staging-directory"},
]
- tool_container_image = container # TODO: this isn't right at all...
- if not container:
- raise Exception("Must declare a container for kubernetes job execution.")
pulsar_container_dict = {
"name": "pulsar-container",
"image": pulsar_container_image,
@@ -408,17 +405,19 @@ class MessageCoexecutionPodJobClient(BaseMessageJobClient):
"workingDir": "/",
"volumeMounts": volume_mounts,
}
- command = TOOL_EXECUTION_CONTAINER_COMMAND_TEMPLATE % job_directory.job_directory
- tool_container_spec = {
- "name": "tool-container",
- "image": tool_container_image,
- "command": ["sh"],
- "args": ["-c", command],
- "workingDir": "/",
- "volumeMounts": volume_mounts,
- }
-
- container_dicts = [pulsar_container_dict, tool_container_spec]
+ tool_container_image = container
+ container_dicts = [pulsar_container_dict]
+ if container:
+ command = TOOL_EXECUTION_CONTAINER_COMMAND_TEMPLATE % job_directory.job_directory
+ tool_container_spec = {
+ "name": "tool-container",
+ "image": tool_container_image,
+ "command": ["sh"],
+ "args": ["-c", command],
+ "workingDir": "/",
+ "volumeMounts": volume_mounts,
+ }
+ container_dicts.append(tool_container_spec)
for container_dict in container_dicts:
if self._default_pull_policy:
container_dict["imagePullPolicy"] = self._default_pull_policy
diff --git a/pulsar/client/staging/__init__.py b/pulsar/client/staging/__init__.py
index 86cb2158ae011d1bb3d611838baf197ac93b694c..00e16c5d19ac66f239e4115d2333970a202026ec 100644
--- a/pulsar/client/staging/__init__.py
+++ b/pulsar/client/staging/__init__.py
@@ -14,7 +14,7 @@ from galaxy.util.bunch import Bunch
from ..util import PathHelper
COMMAND_VERSION_FILENAME = "COMMAND_VERSION"
-DEFAULT_DYNAMIC_COLLECTION_PATTERN = [r"primary_.*|galaxy.json|metadata_.*|dataset_\d+\.dat|__instrument_.*|dataset_\d+_files.+"]
+DEFAULT_DYNAMIC_COLLECTION_PATTERN = [r"primary_.*|galaxy.json|metadata_.*|dataset_\d+\.dat|__instrument_.*|dataset_\d+_files.+|outputs_populated/.*"]
class ClientJobDescription(object):
diff --git a/pulsar/core.py b/pulsar/core.py
index 2f392d492560ac4f73e88df7e30dd5684b1b2040..f3e497aac3e94a39f887ac54a4e2bf4a89adbdf1 100644
--- a/pulsar/core.py
+++ b/pulsar/core.py
@@ -10,11 +10,13 @@ from pulsar.tools.authorization import get_authorizer
from pulsar import messaging
from galaxy.objectstore import build_object_store_from_config
try:
- # If galaxy-lib or Galaxy <19.05 present.
- from galaxy.tools.deps import DependencyManager
-except ImportError:
# If galaxy-tool-util or Galaxy >=19.09 present.
+ from galaxy.tools.deps import build_dependency_manager
+ DependencyManager = None
+except ImportError:
+ # If galaxy-lib or Galaxy <19.05 present.
from galaxy.tool_util.deps import DependencyManager
+ build_dependency_manager = None
try:
# If galaxy-lib or Galaxy <19.05 present.
from galaxy.jobs.metrics import JobMetrics
@@ -140,10 +142,13 @@ class PulsarApp(object):
self.object_store = build_object_store_from_config(object_store_config, config_dict=config_dict)
def __setup_dependency_manager(self, conf):
- dependencies_dir = conf.get("tool_dependency_dir", "dependencies")
- resolvers_config_file = conf.get("dependency_resolvers_config_file", "dependency_resolvers_conf.xml")
- conda_config = {k: v for k, v in conf.items() if k.startswith("conda_")}
- self.dependency_manager = DependencyManager(dependencies_dir, resolvers_config_file, app_config=conda_config)
+ if build_dependency_manager is not None:
+ self.dependency_manager = build_dependency_manager(app_config_dict=conf)
+ else:
+ dependencies_dir = conf.get("tool_dependency_dir", "dependencies")
+ resolvers_config_file = conf.get("dependency_resolvers_config_file", "dependency_resolvers_conf.xml")
+ conda_config = {k: v for k, v in conf.items() if k.startswith("conda_")}
+ self.dependency_manager = DependencyManager(dependencies_dir, resolvers_config_file, app_config=conda_config)
def __setup_job_metrics(self, conf):
job_metrics = conf.get("job_metrics", None)
diff --git a/pulsar/scripts/_conda_init.py b/pulsar/scripts/_conda_init.py
new file mode 100644
index 0000000000000000000000000000000000000000..dad2b8b7ca0dd525cc0d326100283384972cf909
--- /dev/null
+++ b/pulsar/scripts/_conda_init.py
@@ -0,0 +1,28 @@
+"""Small utility for bootstrapping a Conda environment for Pulsar.
+
+This should probably be moved into galaxy-tool-util.
+"""
+
+import os.path
+import sys
+from argparse import ArgumentParser
+
+from galaxy.tool_util.deps.conda_util import CondaContext, install_conda
+from galaxy.util import safe_makedirs
+
+
+def main(argv=None):
+ mod_docstring = sys.modules[__name__].__doc__
+ arg_parser = ArgumentParser(description=mod_docstring)
+ arg_parser.add_argument("--conda_prefix", required=True)
+ args = arg_parser.parse_args(argv)
+ conda_prefix = args.conda_prefix
+ safe_makedirs(os.path.dirname(conda_prefix))
+ conda_context = CondaContext(
+ conda_prefix=conda_prefix,
+ )
+ install_conda(conda_context)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/setup.py b/setup.py
index d9cd7db624063e00910beb3b099fe6422205f348..2064548f7b0b7f2d81837406b2b16950ffa34587 100644
--- a/setup.py
+++ b/setup.py
@@ -100,6 +100,7 @@ setup(
pulsar-chown-working-directory=pulsar.scripts.chown_working_directory:main
pulsar-submit=pulsar.scripts.submit:main
pulsar-run=pulsar.scripts.run:main
+ _pulsar-conda-init=pulsar.scripts._conda_init:main
''',
scripts=scripts,
package_data={'pulsar': [
@@ -111,6 +112,7 @@ setup(
install_requires=requirements,
extras_require={
':python_version=="2.7"': py27_requirements,
+ 'galaxy_extended_metadata': ['galaxy-job-execution>=19.9.0.dev0'],
},
license="Apache License 2.0",
zip_safe=False,