From 11ce744c449409ba7371944bd55a4a80a0d0f300 Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Mon, 12 Dec 2016 14:50:31 -0500
Subject: [PATCH] Finish hooking the pieces together for Conda resolution.
With a test case.
---
Makefile | 2 +-
install_test/common_functions.bash | 12 ++++++++++++
install_test/test_install_conda.bash | 24 ++++++++++++++++++++++++
pulsar/core.py | 6 +++++-
pulsar/managers/base/__init__.py | 5 +++--
pulsar/managers/base/directory.py | 2 +-
pulsar/managers/queued_cli.py | 2 +-
pulsar/managers/unqueued.py | 2 +-
pulsar/scripts/config.py | 13 +++++++++++++
9 files changed, 61 insertions(+), 7 deletions(-)
create mode 100755 install_test/test_install_conda.bash
diff --git a/Makefile b/Makefile
index 316a4e4b..26bc3790 100644
--- a/Makefile
+++ b/Makefile
@@ -83,7 +83,7 @@ test-install-pypi:
bash install_test/test_install.bash
test-install-wheel: dist
- PULSAR_INSTALL_TARGET=$(shell pwd)/dist/pulsar_app*.whl bash install_test/test_install.bash
+ PULSAR_INSTALL_TARGET=$(shell pwd)/dist/pulsar_app*.whl bash install_test/test_install_conda.bash
coverage:
coverage run --source $(SOURCE_DIR) setup.py $(TEST_DIR)
diff --git a/install_test/common_functions.bash b/install_test/common_functions.bash
index f19745a4..2d0e8a4c 100644
--- a/install_test/common_functions.bash
+++ b/install_test/common_functions.bash
@@ -2,6 +2,7 @@ set -e
shopt -s nullglob
+PULSAR_TARGET_POORT="${PULSAR_TARGET_POORT:-8913}"
PULSAR_INSTALL_TARGET="${PULSAR_INSTALL_TARGET:-pulsar-app}"
PLANEMO_INSTALL_TARGET="${PLANEMO_INSTALL_TARGET:-planemo==0.36.1}"
@@ -38,9 +39,20 @@ stop_pulsar() {
check_pulsar() {
cd pulsar
+ if curl -s "http://localhost:$PULSAR_TARGET_POORT"
+ then
+ echo "Port $PULSAR_TARGET_POORT already bound, Pulsar will fail to start."
+ exit 1;
+ fi
+
echo "Starting Pulsar in daemon mode."
pulsar --daemon
echo "Waiting for Pulsar to start."
+ while ! curl -s "http://localhost:$PULSAR_TARGET_POORT";
+ do
+ printf "."
+ sleep 1;
+ done
sleep 2
echo "Running a standalone Pulsar job."
pulsar-check # runs a test job
diff --git a/install_test/test_install_conda.bash b/install_test/test_install_conda.bash
new file mode 100755
index 00000000..959e277f
--- /dev/null
+++ b/install_test/test_install_conda.bash
@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -e
+
+SCRIPT_DIR="$( cd "$(dirname "$0")" ; pwd )"
+. "$SCRIPT_DIR/common_functions.bash"
+
+init_temp_dir
+
+init_pulsar
+
+cd pulsar
+echo "Running pulsar-config with --auto_conda"
+pulsar-config --auto_conda
+
+cd ..
+
+check_pulsar
+
+init_planemo "conda_testing"
+
+run_planemo --job_config_file "$SCRIPT_DIR/galaxy_job_conf.xml" test_tools/bwa.xml
+
+stop_pulsar
diff --git a/pulsar/core.py b/pulsar/core.py
index e4547d27..346dcc38 100644
--- a/pulsar/core.py
+++ b/pulsar/core.py
@@ -124,7 +124,11 @@ class PulsarApp(object):
def __setup_dependency_manager(self, conf):
dependencies_dir = conf.get("tool_dependency_dir", "dependencies")
resolvers_config_file = conf.get("dependency_resolvers_config_file", "dependency_resolvers_conf.xml")
- self.dependency_manager = DependencyManager(dependencies_dir, resolvers_config_file)
+ conda_config = {}
+ for key, value in conf.items():
+ if key.startswith("conda_"):
+ conda_config[key] = value
+ self.dependency_manager = DependencyManager(dependencies_dir, resolvers_config_file, **conda_config)
def __setup_job_metrics(self, conf):
job_metrics = conf.get("job_metrics", None)
diff --git a/pulsar/managers/base/__init__.py b/pulsar/managers/base/__init__.py
index 513e9f6a..91d27570 100644
--- a/pulsar/managers/base/__init__.py
+++ b/pulsar/managers/base/__init__.py
@@ -176,7 +176,7 @@ class BaseManager(ManagerInterface):
else:
return listdir(directory_or_none)
- def _expand_command_line(self, command_line, dependencies_description):
+ def _expand_command_line(self, command_line, dependencies_description, job_directory=None):
if dependencies_description is None:
return command_line
@@ -184,7 +184,8 @@ class BaseManager(ManagerInterface):
installed_tool_dependencies = dependencies_description.installed_tool_dependencies
dependency_commands = self.dependency_manager.dependency_shell_commands(
requirements=requirements,
- installed_tool_dependencies=installed_tool_dependencies
+ installed_tool_dependencies=installed_tool_dependencies,
+ job_directory=job_directory,
)
if dependency_commands:
command_line = "%s; %s" % ("; ".join(dependency_commands), command_line)
diff --git a/pulsar/managers/base/directory.py b/pulsar/managers/base/directory.py
index 90308bef..0463b84c 100644
--- a/pulsar/managers/base/directory.py
+++ b/pulsar/managers/base/directory.py
@@ -105,7 +105,7 @@ class DirectoryBaseManager(BaseManager):
# Helpers methods related to setting up job script files.
def _setup_job_file(self, job_id, command_line, dependencies_description=None, env=[]):
- command_line = self._expand_command_line(command_line, dependencies_description)
+ command_line = self._expand_command_line(command_line, dependencies_description, job_directory=self.job_directory(job_id).job_directory)
script_env = self._job_template_env(job_id, command_line=command_line, env=env)
script = job_script(**script_env)
return self._write_job_script(job_id, script)
diff --git a/pulsar/managers/queued_cli.py b/pulsar/managers/queued_cli.py
index 65fc442c..016cd6c6 100644
--- a/pulsar/managers/queued_cli.py
+++ b/pulsar/managers/queued_cli.py
@@ -26,7 +26,7 @@ class CliQueueManager(ExternalBaseManager):
stdout_path = self._stdout_path(job_id)
stderr_path = self._stderr_path(job_id)
job_name = self._job_name(job_id)
- command_line = self._expand_command_line(command_line, dependencies_description)
+ command_line = self._expand_command_line(command_line, dependencies_description, job_directory=self.job_directory(job_id).job_directory)
job_script_kwargs = self._job_template_env(job_id, command_line=command_line, env=env)
extra_kwargs = job_interface.job_script_kwargs(stdout_path, stderr_path, job_name)
job_script_kwargs.update(extra_kwargs)
diff --git a/pulsar/managers/unqueued.py b/pulsar/managers/unqueued.py
index 705b72e1..525215a5 100644
--- a/pulsar/managers/unqueued.py
+++ b/pulsar/managers/unqueued.py
@@ -152,7 +152,7 @@ class Manager(DirectoryBaseManager):
if platform.system().lower() == "windows":
# TODO: Don't ignore requirements and env without warning. Ideally
# process them or at least warn about them being ignored.
- command_line = self._expand_command_line(command_line, dependencies_description)
+ command_line = self._expand_command_line(command_line, dependencies_description, job_directory=self.job_directory(job_id).job_directory)
else:
command_line = self._setup_job_file(job_id, command_line, dependencies_description=dependencies_description, env=env)
return command_line
diff --git a/pulsar/scripts/config.py b/pulsar/scripts/config.py
index efb7a738..abde0d74 100644
--- a/pulsar/scripts/config.py
+++ b/pulsar/scripts/config.py
@@ -29,6 +29,8 @@ DESCRIPTION = "Initialize a directory with a minimal pulsar config."
HELP_DIRECTORY = "Directory containing the configuration files for Pulsar."
HELP_MQ = ("Write configuration files for message queue server deployment "
"instead of more traditional RESTful web based pulsar.")
+HELP_AUTO_CONDA = ("Auto initialize Conda for tool resolution and auto install "
+ "dependencies on demand.")
HELP_NO_LOGGING = ("Do not write Pulsar's default logging configuration to server.ini "
"and if uwsgi is configured do not configure its logging either.")
HELP_SUPERVISOR = ("Write a supervisord configuration file for "
@@ -145,6 +147,10 @@ def main(argv=None):
arg_parser.add_argument("--directory",
default=".",
help=HELP_DIRECTORY)
+ arg_parser.add_argument("--auto_conda",
+ action="store_true",
+ default=False,
+ help=HELP_AUTO_CONDA)
arg_parser.add_argument("--mq",
action="store_true",
default=False,
@@ -198,6 +204,10 @@ def main(argv=None):
if not os.path.exists(directory):
os.makedirs(directory)
+ default_dependencies_dir = os.path.join(directory, "dependencies")
+ if not os.path.exists(default_dependencies_dir):
+ os.makedirs(default_dependencies_dir)
+
print("Bootstrapping pulsar configuration into directory %s" % relative_directory)
_handle_app_yaml(args, directory)
_handle_server_ini(args, directory)
@@ -298,6 +308,9 @@ def _handle_app_yaml(args, directory):
contents += 'private_token: %s\n' % args.private_token
if args.mq:
contents += 'message_queue_url: "amqp://guest:guest@localhost:5672//"\n'
+ if args.auto_conda:
+ contents += 'conda_auto_init: true\n'
+ contents += 'conda_auto_install: true\n'
else:
if not IS_WINDOWS and args.libdrmaa_path:
contents += 'manager:\n type: queued_drmaa\n'
--
GitLab