From 5d8cda6233ed48028f58238a69993a7123afe11a Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Thu, 5 Mar 2015 23:09:45 -0500
Subject: [PATCH] Fill out pulsar-config script.
- Add many new options.
- Add tests.
---
pulsar/scripts/config.py | 263 +++++++++++++++++++++++++++---------
test/scripts_config_test.py | 75 ++++++++++
2 files changed, 273 insertions(+), 65 deletions(-)
create mode 100644 test/scripts_config_test.py
diff --git a/pulsar/scripts/config.py b/pulsar/scripts/config.py
index 293758a4..66710b67 100644
--- a/pulsar/scripts/config.py
+++ b/pulsar/scripts/config.py
@@ -2,6 +2,7 @@
from __future__ import print_function
import os
+import string
import sys
from pulsar.main import (
@@ -11,42 +12,66 @@ from pulsar.main import (
)
DESCRIPTION = "Initialize a directory with a minimal pulsar config."
+HELP_DIRECTORY = "Directory containing the configuration files for Pulsar."
+HELP_MQ = ("Write configuration files for message queue server deployment "
+ "instead of more traditional RESTful web based pulsar.")
+HELP_SUPERVISOR = ("Write a supervisord configuration file for "
+ "managing pulsar out as well.")
+HELP_FORCE = "Overwrite existing files if they already exist."
+HELP_WSGI_SERVER = ("Web server stack used to host Pulsar wsgi application.")
+HELP_LIBDRMAA = ("Configure Pulsar to submit jobs to a cluster via DRMAA by "
+ "supplying the path to a libdrmaa .so file using this argument.")
+HELP_INSTALL = ("Install optional dependencies required by specified configuration "
+ "(e.g. drmaa, supervisor, uwsgi, etc...).")
+HELP_HOST = ("Host to bind Pulsar to - defaults to localhost. Set to 0.0.0.0 "
+ "to listen on all interfaces.")
+HELP_PORT = ("Port to bind Pulsar to (ignored if --mq is specified).")
-def main():
- arg_parser = ArgumentParser(description=DESCRIPTION)
- arg_parser.add_argument("--directory", default=".")
- arg_parser.add_argument("--mq",
- action="store_true",
- default=False,
- help=("Write configuration files for message queue "
- "instead of web based pulsar."))
- arg_parser.add_argument("--force",
- action="store_true",
- default=False,
- help="Overwrite existing files.")
- args = arg_parser.parse_args()
- directory = args.directory
- force = args.force
- directory = os.path.abspath(directory)
+LOGGING_CONFIG_SECTIONS = """## Configure Python loggers.
+[loggers]
+keys = root,pulsar
- if not os.path.exists(directory):
- os.makedirs(directory)
+[handlers]
+keys = console
- yaml_file = os.path.join(directory, DEFAULT_APP_YAML)
- check_file(yaml_file, force)
- if args.mq:
- open(yaml_file, "w").write("""---
-message_queue_url: "amqp://guest:guest@localhost:5672//"
+[formatters]
+keys = generic
+
+[logger_root]
+level = INFO
+handlers = console
+
+[logger_pulsar]
+level = DEBUG
+handlers = console
+qualname = pulsar
+propagate = 0
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = DEBUG
+formatter = generic
+
+[formatter_generic]
+format = %(asctime)s %(levelname)-5.5s [%(name)s][%(threadName)s] %(message)s
+"""
+
+SUPERVISOR_CONFIG_TEMPLATE = string.Template("""[program:pulsar]
+user = ${user}
+directory = ${directory}
+command = pulsar --mode '${mode}' --config '${directory}'
+redirect_stderr = true
+autorestart = true
""")
- else:
- open(yaml_file, "w").write("""---""")
- ini_file = os.path.join(directory, DEFAULT_INI)
- if not args.mq:
- check_file(ini_file, force)
- open(ini_file, "w").write("""[server:main]
+SERVER_CONFIG_TEMPLATE = string.Template("""[server:main]
use = egg:Paste#http
+port = ${port}
+host = ${host}
+## pem file to use to enable SSL.
+# ssl_pem = host.pem
[app:main]
paste.app_factory = pulsar.web.wsgi:app_factory
@@ -55,8 +80,8 @@ app_config = %(here)s/app.yml
## Configure uWSGI (if used).
[uwsgi]
master = True
-paste-logger = True
-socket = 127.0.0.1:3031
+paste-logger = ${use_logging}
+socket = ${host}:3031
processes = 1
enable-threads = True
@@ -74,45 +99,17 @@ use_sockets = True
numprocesses = 1
[socket:web]
-host = localhost
-port = 8913
+host = ${host}
+port = ${port}
-## Configure Python loggers.
-[loggers]
-keys = root,pulsar
-
-[handlers]
-keys = console
-
-[formatters]
-keys = generic
-
-[logger_root]
-level = INFO
-handlers = console
-
-[logger_pulsar]
-level = DEBUG
-handlers = console
-qualname = pulsar
-propagate = 0
-
-[handler_console]
-class = StreamHandler
-args = (sys.stderr,)
-level = DEBUG
-formatter = generic
-
-[formatter_generic]
-format = %(asctime)s %(levelname)-5.5s [%(name)s][%(threadName)s] %(message)s
+${logging_sections}
""")
- local_env_file = os.path.join(directory, "local_env.sh")
- open(local_env_file, "w").write("""## Place local configuration variables used by Pulsar and run.sh in here. For example
+LOCAL_ENV_TEMPLATE = string.Template("""## Place local configuration variables used by Pulsar and run.sh in here. For example
## If using the drmaa queue manager, you will need to set the DRMAA_LIBRARY_PATH variable,
## you may also need to update LD_LIBRARY_PATH for underlying library as well.
-#export DRMAA_LIBRARY_PATH=/path/to/libdrmaa.so
+$libdrmaa_line
## If you wish to use a variety of Galaxy tools that depend on galaxy.eggs being defined,
@@ -121,7 +118,143 @@ format = %(asctime)s %(levelname)-5.5s [%(name)s][%(threadName)s] %(message)s
""")
-def check_file(path, force):
+def main(argv=None):
+ if argv is None:
+ argv = sys.argv
+ dependencies = []
+ arg_parser = ArgumentParser(description=DESCRIPTION)
+ arg_parser.add_argument("--directory",
+ default=".",
+ help=HELP_DIRECTORY)
+ arg_parser.add_argument("--mq",
+ action="store_true",
+ default=False,
+ help=HELP_MQ)
+ arg_parser.add_argument("--no_logging",
+ dest="logging",
+ action="store_false",
+ default=True,
+ help=HELP_MQ)
+ arg_parser.add_argument("--supervisor",
+ action="store_true",
+ default=False,
+ help=HELP_SUPERVISOR)
+ arg_parser.add_argument("--wsgi_server",
+ choices=["paste", "uwsgi"],
+ default=None,
+ help=HELP_WSGI_SERVER)
+ arg_parser.add_argument("--libdrmaa_path",
+ help=HELP_LIBDRMAA)
+ arg_parser.add_argument("--host",
+ default="localhost",
+ help=HELP_HOST)
+ arg_parser.add_argument("--port",
+ default="8913",
+ help=HELP_PORT)
+ arg_parser.add_argument("--install",
+ help=HELP_INSTALL)
+ arg_parser.add_argument("--force",
+ action="store_true",
+ default=False,
+ help=HELP_FORCE)
+ args = arg_parser.parse_args(argv)
+ directory = args.directory
+ directory = os.path.abspath(directory)
+
+ mode = _determine_mode(args)
+ if mode == "uwsgi":
+ dependencies.append("uwsgi")
+
+ if not os.path.exists(directory):
+ os.makedirs(directory)
+
+ _handle_app_yaml(args, directory)
+ _handle_server_ini(args, directory)
+ _handle_local_env(args, directory, dependencies)
+ _handle_supervisor(args, mode, directory, dependencies)
+ _handle_install(args, dependencies)
+
+
+def _determine_mode(args):
+ if args.wsgi_server:
+ mode = args.wsgi_server
+ elif args.mq:
+ mode = "webless"
+ else:
+ mode = "paster"
+ return mode
+
+
+def _handle_server_ini(args, directory):
+ force = args.force
+ ini_file = os.path.join(directory, DEFAULT_INI)
+ if not args.mq:
+ _check_file(ini_file, force)
+ config_dict = dict(
+ port=args.port,
+ host=args.host,
+ )
+ if args.logging:
+ config_dict["logging_sections"] = LOGGING_CONFIG_SECTIONS
+ config_dict["use_logging"] = "true"
+ else:
+ config_dict["logging_sections"] = ""
+ config_dict["use_logging"] = "false"
+
+ server_config = SERVER_CONFIG_TEMPLATE.safe_substitute(
+ **config_dict
+ )
+ open(ini_file, "w").write(server_config)
+
+
+def _handle_app_yaml(args, directory):
+ force = args.force
+ yaml_file = os.path.join(directory, DEFAULT_APP_YAML)
+ _check_file(yaml_file, force)
+ contents = "---\n"
+ if args.mq:
+ contents += 'message_queue_url: "amqp://guest:guest@localhost:5672//"\n'
+ else:
+ if args.libdrmaa_path:
+ contents += 'manager:\n type: queued_drmaa\n'
+ open(yaml_file, "w").write(contents)
+
+
+def _handle_local_env(args, directory, dependencies):
+ local_env_file = os.path.join(directory, "local_env.sh")
+ if args.libdrmaa_path:
+ libdrmaa_line = 'export DRMAA_LIBRARY_PATH=%s' % args.libdrmaa_path
+ os.environ["DRMAA_LIBRARY_PATH"] = args.libdrmaa_path
+ dependencies.append("drmaa")
+ else:
+ libdrmaa_line = '#export DRMAA_LIBRARY_PATH=/path/to/libdrmaa.so'
+
+ local_env_contents = LOCAL_ENV_TEMPLATE.safe_substitute(
+ libdrmaa_line=libdrmaa_line,
+ )
+ open(local_env_file, "w").write(local_env_contents)
+
+
+def _handle_supervisor(args, mode, directory, dependencies):
+ if args.supervisor:
+ template = SUPERVISOR_CONFIG_TEMPLATE
+ config = template.safe_substitute(
+ user=os.environ["USER"],
+ directory=directory,
+ mode=mode,
+ )
+ conf_path = os.path.join(directory, "supervisor.conf")
+ open(conf_path, "w").write(config)
+ dependencies.append("supervisor")
+
+
+def _handle_install(args, dependencies):
+ if args.install and dependencies:
+ import pip
+ pip.main("install", *dependencies)
+
+
+def _check_file(path, force):
if os.path.exists(path) and not force:
print("File %s exists, exiting." % path, file=sys.stderr)
sys.exit(1)
diff --git a/test/scripts_config_test.py b/test/scripts_config_test.py
new file mode 100644
index 00000000..33036e80
--- /dev/null
+++ b/test/scripts_config_test.py
@@ -0,0 +1,75 @@
+import collections
+import os
+import subprocess
+import yaml
+
+from six.moves import configparser
+
+from pulsar.scripts.config import main
+
+from test_utils import temp_directory
+
+
+def test_default_web_config():
+ with temp_directory() as project_dir:
+ main(["--directory", project_dir])
+ project = _check_project_directory(project_dir)
+ assert project.ini_config is not None
+
+ local_env = os.path.join(project_dir, "local_env.sh")
+ assert os.path.exists(local_env)
+ exit_code = subprocess.check_call(['/bin/bash', '-c', '. %s' % local_env])
+ assert exit_code == 0
+
+
+def test_mq_config():
+ with temp_directory() as project_dir:
+ main(["--directory", project_dir, "--mq"])
+ project = _check_project_directory(project_dir)
+ assert project.ini_config is None
+ assert "message_queue_url" in project.app_config
+
+
+def test_with_supervisor():
+ with temp_directory() as project_dir:
+ main(["--directory", project_dir, "--supervisor"])
+ project = _check_project_directory(project_dir)
+ assert project.ini_config is not None
+
+ supervisor_conf = os.path.join(project_dir, "supervisor.conf")
+ assert os.path.exists(supervisor_conf)
+
+
+def test_libdrmaa_config():
+ with temp_directory() as project_dir:
+ main(["--directory", project_dir, "--libdrmaa_path", "/path/to/test/libdrmaa.so"])
+
+ local_env = os.path.join(project_dir, "local_env.sh")
+ assert os.path.exists(local_env)
+ exit_code = subprocess.check_call(['/bin/bash', '-c', '. %s' % local_env])
+ assert exit_code == 0
+
+
+def _check_project_directory(project_dir):
+ def path_if_exists(name):
+ path = os.path.join(project_dir, name)
+ if os.path.exists(path):
+ return path
+ else:
+ return None
+
+ app_config = None
+ app_config_path = path_if_exists("app.yml")
+ if app_config_path:
+ app_config = yaml.load(open(app_config_path, "r"))
+ assert isinstance(app_config, dict) or (app_config is None)
+
+ ini_config = None
+ ini_path = path_if_exists("server.ini")
+ if ini_path:
+ ini_config = configparser.ConfigParser()
+ ini_config.read([ini_path])
+
+ return Project(ini_config, app_config)
+
+Project = collections.namedtuple('Project', ['ini_config', 'app_config'])
--
GitLab