diff --git a/docker/coexecutor/Makefile b/docker/coexecutor/Makefile index ab4ce34c6e4ba1bc000727e4196dd57c59aa7478..ec1ab0afaf286936b31387535a39aa764081d720 100644 --- a/docker/coexecutor/Makefile +++ b/docker/coexecutor/Makefile @@ -4,4 +4,4 @@ docker-image: cd ../..; make dist; cp dist/pulsar*whl docker/coexecutor all: docker-image - docker build -t 'galaxy/pulsar-pod-staging:0.1' . + docker build -t 'galaxy/pulsar-pod-staging:0.12.0' . diff --git a/pulsar/client/client.py b/pulsar/client/client.py index e8e6184ee1f8b854bbb33b6209dcb0d2743de7de..81b4cc134d4c29e233fbc975bf3d7c355238635c 100644 --- a/pulsar/client/client.py +++ b/pulsar/client/client.py @@ -4,6 +4,7 @@ import os from six import string_types from pulsar.managers.util.pykube_util import ( + ensure_pykube, Job, job_object_dict, produce_unique_k8s_job_name, @@ -364,6 +365,7 @@ class MessageCLIJobClient(BaseMessageJobClient): class MessageCoexecutionPodJobClient(BaseMessageJobClient): def __init__(self, destination_params, job_id, client_manager): + ensure_pykube() super(MessageCoexecutionPodJobClient, self).__init__(destination_params, job_id, client_manager) self.pulsar_container_image = destination_params.get("pulsar_container_image", "galaxy/pulsar-pod-staging:0.10.0") self._default_pull_policy = pull_policy(destination_params) diff --git a/pulsar/main.py b/pulsar/main.py index 5f81dd853b560d29cc60840f1e7a63d6a92b7c07..83744b5f07dcc840dfda8fcb7399791aedb0c1ff 100644 --- a/pulsar/main.py +++ b/pulsar/main.py @@ -64,6 +64,42 @@ HELP_DAEMONIZE = "Daemonzie process (requires daemonize library)." CONFIG_PREFIX = "PULSAR_CONFIG_" +LOGGING_CONFIG_DEFAULT = { + 'version': 1, + 'root': { + 'handlers': ['console'], + 'level': 'INFO', + }, + 'loggers': { + 'pulsar': { + 'handlers': ['console'], + 'level': 'DEBUG', + 'propagate': 0, + 'qualname': 'pulsar', + }, + 'galaxy': { + 'handlers': ['console'], + 'level': 'DEBUG', + 'propagate': 0, + 'qualname': 'pulsar', + }, + }, + 'handlers': { + 'console': { + 'class': 'logging.StreamHandler', + 'formatter': 'default', + 'level': 'DEBUG', + 'stream': 'ext://sys.stderr', + }, + }, + 'formatters': { + 'default': { + 'format': '%(asctime)s %(levelname)-5.5s [%(name)s][%(threadName)s] %(message)s' + }, + }, +} + + def load_pulsar_app( config_builder, config_env=False, @@ -88,7 +124,7 @@ def load_pulsar_app( log.exception("Failed to add Pulsar to sys.path") raise - config_builder.setup_logging() + config_builder.setup_file_logging() config = config_builder.load() config.update(kwds) @@ -240,6 +276,7 @@ class PulsarConfigBuilder(object): if self.app_conf_base64: from pulsar.client.util import from_base64_json local_conf = from_base64_json(self.app_conf_base64) + self.setup_dict_logging(local_conf) load_kwds["local_conf"] = local_conf else: load_kwds.update(dict( @@ -249,19 +286,24 @@ class PulsarConfigBuilder(object): )) return load_app_configuration(**load_kwds) - def setup_logging(self): - if not self.ini_path: - # TODO: should be possible can configure using dict. - return - raw_config = configparser.ConfigParser() - raw_config.read([self.ini_path]) - # https://github.com/mozilla-services/chaussette/pull/32/files - if raw_config.has_section('loggers'): - config_file = os.path.abspath(self.ini_path) - fileConfig( - config_file, - dict(__file__=config_file, here=os.path.dirname(config_file)) - ) + def setup_file_logging(self): + if self.ini_path: + raw_config = configparser.ConfigParser() + raw_config.read([self.ini_path]) + # https://github.com/mozilla-services/chaussette/pull/32/files + if raw_config.has_section('loggers'): + config_file = os.path.abspath(self.ini_path) + fileConfig( + config_file, + dict(__file__=config_file, here=os.path.dirname(config_file)) + ) + + def setup_dict_logging(self, config): + logging_conf = config.get('logging', None) + if logging_conf is None: + # if using the default logging config, honor the log_level setting + logging_conf = LOGGING_CONFIG_DEFAULT + logging.config.dictConfig(logging_conf) def to_dict(self): return dict( diff --git a/pulsar/scripts/mesos_framework.py b/pulsar/scripts/mesos_framework.py index 5da99dcc20c542dd866d79a300aaef344730957a..15d859e88e31a0ebc6acd0e69d08a064fbb83d6f 100644 --- a/pulsar/scripts/mesos_framework.py +++ b/pulsar/scripts/mesos_framework.py @@ -21,7 +21,7 @@ def main(argv=None): ensure_mesos_libs() config_builder = PulsarManagerConfigBuilder(args) - config_builder.setup_logging() + config_builder.setup_file_logging() config = config_builder.load() run(