From 951a3f680e25f25efca676671acadcf76906e962 Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Tue, 3 Jun 2014 20:47:45 -0500
Subject: [PATCH] Move all web stuffs into lwr.web. Refactor core app into
lwr/core.py.
Deprecating old wsgi app_factory module in lieu of lwr.web.wsgi.
---
lwr/app.py | 166 +-----------------------------------
lwr/core.py | 119 ++++++++++++++++++++++++++
lwr/lwr_client/interface.py | 4 +-
lwr/web/__init__.py | 0
lwr/{ => web}/framework.py | 0
lwr/{ => web}/routes.py | 4 +-
lwr/web/wsgi.py | 45 ++++++++++
server.ini.sample | 2 +-
test/routes_test.py | 2 +-
test/test_utils.py | 2 +-
10 files changed, 175 insertions(+), 169 deletions(-)
create mode 100644 lwr/core.py
create mode 100644 lwr/web/__init__.py
rename lwr/{ => web}/framework.py (100%)
rename lwr/{ => web}/routes.py (99%)
create mode 100644 lwr/web/wsgi.py
diff --git a/lwr/app.py b/lwr/app.py
index 13e62c46..d9b6da7a 100644
--- a/lwr/app.py
+++ b/lwr/app.py
@@ -1,164 +1,6 @@
+""" Deprecated module for wsgi app factory. LWR servers should transition to
+``lwr.web.wsgi:app_factory``.
"""
-"""
-import atexit
-import inspect
-import os
-from tempfile import tempdir
-
-from lwr.manager_factory import build_managers
-from lwr.cache import Cache
-from lwr.framework import RoutingApp
-from lwr.tools import ToolBox
-from lwr.tools.authorization import get_authorizer
-from lwr import messaging
-import lwr.routes
-from galaxy.objectstore import build_object_store_from_config
-from galaxy.tools.deps import DependencyManager
-from galaxy.jobs.metrics import JobMetrics
-from galaxy.util.bunch import Bunch
-
-from logging import getLogger
-log = getLogger(__name__)
-
-DEFAULT_PRIVATE_KEY = None
-DEFAULT_STAGING_DIRECTORY = "lwr_staging"
-DEFAULT_PERSISTENCE_DIRECTORY = "persisted_data"
-
-
-NOT_WHITELIST_WARNING = "Starting the LWR without a toolbox to white-list." + \
- "Ensure this application is protected by firewall or a configured private token."
-
-
-class LwrApp(object):
-
- def __init__(self, **conf):
- if conf is None:
- conf = {}
- self.__setup_staging_directory(conf.get("staging_directory", DEFAULT_STAGING_DIRECTORY))
- self.__setup_private_key(conf.get("private_key", DEFAULT_PRIVATE_KEY))
- self.__setup_persistence_directory(conf.get("persistence_directory", None))
- self.__setup_tool_config(conf)
- self.__setup_object_store(conf)
- self.__setup_dependency_manager(conf)
- self.__setup_job_metrics(conf)
- self.__setup_managers(conf)
- self.__setup_file_cache(conf)
- self.__setup_bind_to_message_queue(conf)
-
- def shutdown(self):
- for manager in self.managers.values():
- try:
- manager.shutdown()
- except Exception:
- pass
-
- if self.__queue_state:
- self.__queue_state.deactivate()
-
- def __setup_bind_to_message_queue(self, conf):
- message_queue_url = conf.get("message_queue_url", None)
- queue_state = None
- if message_queue_url:
- queue_state = messaging.bind_app(self, message_queue_url, conf)
- self.__queue_state = queue_state
-
- def __setup_tool_config(self, conf):
- """
- Setups toolbox object and authorization mechanism based
- on supplied toolbox_path.
- """
- tool_config_files = conf.get("tool_config_files", None)
- if not tool_config_files:
- # For compatibity with Galaxy, allow tool_config_file
- # option name.
- tool_config_files = conf.get("tool_config_file", None)
- toolbox = None
- if tool_config_files:
- toolbox = ToolBox(tool_config_files)
- else:
- log.info(NOT_WHITELIST_WARNING)
- self.toolbox = toolbox
- self.authorizer = get_authorizer(toolbox)
-
- def __setup_staging_directory(self, staging_directory):
- self.staging_directory = os.path.abspath(staging_directory)
-
- def __setup_managers(self, conf):
- self.managers = build_managers(self, conf)
-
- def __setup_private_key(self, private_key):
- self.private_key = private_key
- if private_key:
- log.info("Securing LWR web app with private key, please verify you are using HTTPS so key cannot be obtained by monitoring traffic.")
-
- def __setup_routes(self):
- for func_name, func in inspect.getmembers(lwr.routes, lambda x: getattr(x, '__controller__', False)):
- self.__add_route_for_function(func)
-
- def __setup_persistence_directory(self, persistence_directory):
- self.persistence_directory = persistence_directory or DEFAULT_PERSISTENCE_DIRECTORY
-
- def __setup_file_cache(self, conf):
- file_cache_dir = conf.get('file_cache_dir', None)
- self.file_cache = Cache(file_cache_dir) if file_cache_dir else None
-
- def __setup_object_store(self, conf):
- if "object_store_config_file" not in conf:
- self.object_store = None
- return
- object_store_config = Bunch(
- object_store_config_file=conf['object_store_config_file'],
- file_path=conf.get("object_store_file_path", None),
- object_store_check_old_style=False,
- job_working_directory=conf.get("object_store_job_working_directory", None),
- new_file_path=conf.get("object_store_new_file_path", tempdir),
- umask=int(conf.get("object_store_umask", "0000")),
- )
- self.object_store = build_object_store_from_config(object_store_config)
-
- def __setup_dependency_manager(self, conf):
- dependencies_dir = conf.get("tool_dependency_dir", "dependencies")
- resolvers_config_file = conf.get("dependency_resolvers_config_file", "dependency_resolvers_conf.xml")
- self.dependency_manager = DependencyManager(dependencies_dir, resolvers_config_file)
-
- def __setup_job_metrics(self, conf):
- job_metrics_config_file = conf.get("job_metrics_config_file", "job_metrics_conf.xml")
- self.job_metrics = JobMetrics(job_metrics_config_file)
-
-
-def app_factory(global_conf, **local_conf):
- """
- Returns the LWR WSGI application.
- """
- lwr_app = LwrApp(global_conf=global_conf, **local_conf)
- webapp = LwrWebApp(lwr_app=lwr_app)
- atexit.register(webapp.shutdown)
- return webapp
-
-
-class LwrWebApp(RoutingApp):
- """
- Web application for LWR web server.
- """
-
- def __init__(self, lwr_app):
- super(LwrWebApp, self).__init__()
- self.lwr_app = lwr_app
- self.__setup_routes()
-
- def __setup_routes(self):
- for func_name, func in inspect.getmembers(lwr.routes, lambda x: getattr(x, '__controller__', False)):
- self.__add_route_for_function(func)
-
- def __add_route_for_function(self, function):
- route_suffix = '/%s' % function.__name__
- # Default or old-style route without explicit manager specified,
- # will be routed to manager '_default_'.
- default_manager_route = route_suffix
- self.add_route(default_manager_route, function)
- # Add route for named manager as well.
- named_manager_route = '/managers/{manager_name}%s' % route_suffix
- self.add_route(named_manager_route, function)
+from lwr.web.wsgi import app_factory
- def __getattr__(self, name):
- return getattr(self.lwr_app, name)
+__all__ = ['app_factory']
diff --git a/lwr/core.py b/lwr/core.py
new file mode 100644
index 00000000..7b9d3772
--- /dev/null
+++ b/lwr/core.py
@@ -0,0 +1,119 @@
+"""
+"""
+import inspect
+import os
+from tempfile import tempdir
+
+from lwr.manager_factory import build_managers
+from lwr.cache import Cache
+from lwr.tools import ToolBox
+from lwr.tools.authorization import get_authorizer
+from lwr import messaging
+from galaxy.objectstore import build_object_store_from_config
+from galaxy.tools.deps import DependencyManager
+from galaxy.jobs.metrics import JobMetrics
+from galaxy.util.bunch import Bunch
+
+from logging import getLogger
+log = getLogger(__name__)
+
+DEFAULT_PRIVATE_KEY = None
+DEFAULT_STAGING_DIRECTORY = "lwr_staging"
+DEFAULT_PERSISTENCE_DIRECTORY = "persisted_data"
+
+
+NOT_WHITELIST_WARNING = "Starting the LWR without a toolbox to white-list." + \
+ "Ensure this application is protected by firewall or a configured private token."
+
+
+class LwrApp(object):
+
+ def __init__(self, **conf):
+ if conf is None:
+ conf = {}
+ self.__setup_staging_directory(conf.get("staging_directory", DEFAULT_STAGING_DIRECTORY))
+ self.__setup_private_key(conf.get("private_key", DEFAULT_PRIVATE_KEY))
+ self.__setup_persistence_directory(conf.get("persistence_directory", None))
+ self.__setup_tool_config(conf)
+ self.__setup_object_store(conf)
+ self.__setup_dependency_manager(conf)
+ self.__setup_job_metrics(conf)
+ self.__setup_managers(conf)
+ self.__setup_file_cache(conf)
+ self.__setup_bind_to_message_queue(conf)
+
+ def shutdown(self):
+ for manager in self.managers.values():
+ try:
+ manager.shutdown()
+ except Exception:
+ pass
+
+ if self.__queue_state:
+ self.__queue_state.deactivate()
+
+ def __setup_bind_to_message_queue(self, conf):
+ message_queue_url = conf.get("message_queue_url", None)
+ queue_state = None
+ if message_queue_url:
+ queue_state = messaging.bind_app(self, message_queue_url, conf)
+ self.__queue_state = queue_state
+
+ def __setup_tool_config(self, conf):
+ """
+ Setups toolbox object and authorization mechanism based
+ on supplied toolbox_path.
+ """
+ tool_config_files = conf.get("tool_config_files", None)
+ if not tool_config_files:
+ # For compatibity with Galaxy, allow tool_config_file
+ # option name.
+ tool_config_files = conf.get("tool_config_file", None)
+ toolbox = None
+ if tool_config_files:
+ toolbox = ToolBox(tool_config_files)
+ else:
+ log.info(NOT_WHITELIST_WARNING)
+ self.toolbox = toolbox
+ self.authorizer = get_authorizer(toolbox)
+
+ def __setup_staging_directory(self, staging_directory):
+ self.staging_directory = os.path.abspath(staging_directory)
+
+ def __setup_managers(self, conf):
+ self.managers = build_managers(self, conf)
+
+ def __setup_private_key(self, private_key):
+ self.private_key = private_key
+ if private_key:
+ log.info("Securing LWR web app with private key, please verify you are using HTTPS so key cannot be obtained by monitoring traffic.")
+
+ def __setup_persistence_directory(self, persistence_directory):
+ self.persistence_directory = persistence_directory or DEFAULT_PERSISTENCE_DIRECTORY
+
+ def __setup_file_cache(self, conf):
+ file_cache_dir = conf.get('file_cache_dir', None)
+ self.file_cache = Cache(file_cache_dir) if file_cache_dir else None
+
+ def __setup_object_store(self, conf):
+ if "object_store_config_file" not in conf:
+ self.object_store = None
+ return
+ object_store_config = Bunch(
+ object_store_config_file=conf['object_store_config_file'],
+ file_path=conf.get("object_store_file_path", None),
+ object_store_check_old_style=False,
+ job_working_directory=conf.get("object_store_job_working_directory", None),
+ new_file_path=conf.get("object_store_new_file_path", tempdir),
+ umask=int(conf.get("object_store_umask", "0000")),
+ )
+ self.object_store = build_object_store_from_config(object_store_config)
+
+ def __setup_dependency_manager(self, conf):
+ dependencies_dir = conf.get("tool_dependency_dir", "dependencies")
+ resolvers_config_file = conf.get("dependency_resolvers_config_file", "dependency_resolvers_conf.xml")
+ self.dependency_manager = DependencyManager(dependencies_dir, resolvers_config_file)
+
+ def __setup_job_metrics(self, conf):
+ job_metrics_config_file = conf.get("job_metrics_config_file", "job_metrics_conf.xml")
+ self.job_metrics = JobMetrics(job_metrics_config_file)
diff --git a/lwr/lwr_client/interface.py b/lwr/lwr_client/interface.py
index fbacc567..95538d22 100644
--- a/lwr/lwr_client/interface.py
+++ b/lwr/lwr_client/interface.py
@@ -78,8 +78,8 @@ class LocalLwrInterface(LwrInteface):
def execute(self, command, args={}, data=None, input_path=None, output_path=None):
# If data set, should be unicode (on Python 2) or str (on Python 3).
- from lwr import routes
- from lwr.framework import build_func_args
+ from lwr.web import routes
+ from lwr.web.framework import build_func_args
controller = getattr(routes, command)
action = controller.func
body_args = dict(body=self.__build_body(data, input_path))
diff --git a/lwr/web/__init__.py b/lwr/web/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/lwr/framework.py b/lwr/web/framework.py
similarity index 100%
rename from lwr/framework.py
rename to lwr/web/framework.py
diff --git a/lwr/routes.py b/lwr/web/routes.py
similarity index 99%
rename from lwr/routes.py
rename to lwr/web/routes.py
index 3919c9f1..1d491f8b 100644
--- a/lwr/routes.py
+++ b/lwr/web/routes.py
@@ -7,9 +7,9 @@ from galaxy.util import (
copy_to_temp,
)
from lwr.lwr_client.job_directory import verify_is_in_directory
-from lwr.framework import Controller
+from lwr.web.framework import Controller
from lwr.manager_factory import DEFAULT_MANAGER_NAME
-from .manager_endpoint_util import (
+from lwr.manager_endpoint_util import (
submit_job,
setup_job,
full_status,
diff --git a/lwr/web/wsgi.py b/lwr/web/wsgi.py
new file mode 100644
index 00000000..334a9a76
--- /dev/null
+++ b/lwr/web/wsgi.py
@@ -0,0 +1,45 @@
+import atexit
+import inspect
+
+from lwr.core import LwrApp
+from lwr.web.framework import RoutingApp
+
+import lwr.web.routes
+
+
+def app_factory(global_conf, **local_conf):
+ """
+ Returns the LWR WSGI application.
+ """
+ lwr_app = LwrApp(global_conf=global_conf, **local_conf)
+ webapp = LwrWebApp(lwr_app=lwr_app)
+ atexit.register(webapp.shutdown)
+ return webapp
+
+
+class LwrWebApp(RoutingApp):
+ """
+ Web application for LWR web server.
+ """
+
+ def __init__(self, lwr_app):
+ super(LwrWebApp, self).__init__()
+ self.lwr_app = lwr_app
+ self.__setup_routes()
+
+ def __setup_routes(self):
+ for func_name, func in inspect.getmembers(lwr.web.routes, lambda x: getattr(x, '__controller__', False)):
+ self.__add_route_for_function(func)
+
+ def __add_route_for_function(self, function):
+ route_suffix = '/%s' % function.__name__
+ # Default or old-style route without explicit manager specified,
+ # will be routed to manager '_default_'.
+ default_manager_route = route_suffix
+ self.add_route(default_manager_route, function)
+ # Add route for named manager as well.
+ named_manager_route = '/managers/{manager_name}%s' % route_suffix
+ self.add_route(named_manager_route, function)
+
+ def __getattr__(self, name):
+ return getattr(self.lwr_app, name)
diff --git a/server.ini.sample b/server.ini.sample
index 9cc58970..73507e7d 100644
--- a/server.ini.sample
+++ b/server.ini.sample
@@ -17,7 +17,7 @@ host = localhost
# ssl_pem = host.pem
[app:main]
-paste.app_factory = lwr.app:app_factory
+paste.app_factory = lwr.web.wsgi:app_factory
## Directory to stage files to. This should likely be updated to point
## to an absolute path, such as /tmp/lwr_staging or C:\\lwr_staging
diff --git a/test/routes_test.py b/test/routes_test.py
index ebb8b68a..7f5b5295 100644
--- a/test/routes_test.py
+++ b/test/routes_test.py
@@ -1,6 +1,6 @@
from os.path import join
-from lwr.routes import _output_path
+from lwr.web.routes import _output_path
from .test_utils import test_manager
diff --git a/test/test_utils.py b/test/test_utils.py
index 60ec50e6..6ee20d9f 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -25,7 +25,7 @@ from webtest.http import StopableWSGIServer
import galaxy.util
from lwr.tools import ToolBox
from lwr.managers.base import JobDirectory
-from lwr.framework import file_response
+from lwr.web.framework import file_response
TEST_DIR = dirname(__file__)
ROOT_DIR = join(TEST_DIR, pardir)
--
GitLab