diff --git a/pulsar/cache/__init__.py b/pulsar/cache/__init__.py
index 7a97262b38b32fd89252660931e9f38ad4c702d9..57cbffe835db0eaa62c2a38cf773a2af37759950 100644
--- a/pulsar/cache/__init__.py
+++ b/pulsar/cache/__init__.py
@@ -1,6 +1,6 @@
-from os.path import join, exists
from hashlib import sha256
+from os.path import exists, join
from .persistence import PersistenceStore
from .util import atomicish_move
diff --git a/pulsar/cache/persistence.py b/pulsar/cache/persistence.py
index 2bc667d9fd43a3ec83ea8a91590b9255337d483f..67262850c45ce3c17c3cf02b48238e6d1b4d2a05 100644
--- a/pulsar/cache/persistence.py
+++ b/pulsar/cache/persistence.py
@@ -1,7 +1,9 @@
+
import shelve
-from threading import Lock
import traceback
+from threading import Lock
+
class PersistenceStore(object):
diff --git a/pulsar/cache/util.py b/pulsar/cache/util.py
index a8926e3edd7c4a248c342c355bff8320afadce9a..dba36897e6c908ec86bf53eadea891a3c1ba8a44 100644
--- a/pulsar/cache/util.py
+++ b/pulsar/cache/util.py
@@ -1,12 +1,12 @@
+
import os
import shutil
+
from datetime import datetime
def atomicish_move(source, destination, tmp_suffix="_TMP"):
- """
- Move source to destination without copying to directly to destination
- there is never a partial file.
+ """Move source to destination without risk of partial moves.
> from tempfile import mkdtemp
> from os.path import join, exists
@@ -28,8 +28,9 @@ def atomicish_move(source, destination, tmp_suffix="_TMP"):
class Time:
- """ Time utilities of now that can be instrumented for testing."""
+ """Time utilities of now that can be instrumented for testing."""
@classmethod
def now(cls):
+ """Return the current datetime."""
return datetime.utcnow()
diff --git a/pulsar/client/__init__.py b/pulsar/client/__init__.py
index b28ab87cd93d2b8c915c847127c9e8583741f4cb..a42c60173cef27dd2005ffaa0ff5bd10f2cb401d 100644
--- a/pulsar/client/__init__.py
+++ b/pulsar/client/__init__.py
@@ -39,16 +39,16 @@ in the ``galaxy.ini`` file.
"""
-from .staging.down import finish_job
-from .staging.up import submit_job
-from .staging import ClientJobDescription
-from .staging import PulsarOutputs
-from .staging import ClientOutputs
from .client import OutputNotFoundException
-from .manager import build_client_manager
from .destination import url_to_destination_params
-from .path_mapper import PathMapper
from .exceptions import PulsarClientTransportError
+from .manager import build_client_manager
+from .path_mapper import PathMapper
+from .staging import ClientJobDescription
+from .staging import PulsarOutputs
+from .staging import ClientOutputs
+from .staging.down import finish_job
+from .staging.up import submit_job
__all__ = [
'build_client_manager',
diff --git a/pulsar/client/action_mapper.py b/pulsar/client/action_mapper.py
index 34e40b753f91d4fd385bd1b7ce4b9e0735066e76..6d6985926dbc01cee6246ad482f64dcec262f8a5 100644
--- a/pulsar/client/action_mapper.py
+++ b/pulsar/client/action_mapper.py
@@ -1,26 +1,34 @@
+import fnmatch
+import tempfile
+
from contextlib import contextmanager
from json import load
from os import makedirs
from os import unlink
-from os.path import exists
-from os.path import abspath
-from os.path import dirname
-from os.path import join
-from os.path import basename
-from os.path import sep
-import fnmatch
-from re import compile
-from re import escape
+from os.path import (
+ abspath,
+ basename,
+ dirname,
+ exists,
+ join,
+ sep,
+)
+from re import compile, escape
+
from galaxy.util.bunch import Bunch
+
from .config_util import read_file
-from .util import directory_files
-from .util import unique_path_prefix
-from .util import copy_to_path
from .transport import get_file
from .transport import post_file
-from .transport import rsync_get_file, scp_get_file
-from .transport import rsync_post_file, scp_post_file
-import tempfile
+from .transport import (
+ rsync_get_file,
+ rsync_post_file,
+ scp_get_file,
+ scp_post_file,
+)
+from .util import copy_to_path
+from .util import directory_files
+from .util import unique_path_prefix
DEFAULT_MAPPED_ACTION = 'transfer' # Not really clear to me what this should be, exception?
diff --git a/pulsar/client/amqp_exchange.py b/pulsar/client/amqp_exchange.py
index 03ad85ffa95be5fef37dd6adc9fdb5641e6e0b83..9c47ee0803a7441189daf913d763f13ed4441808 100644
--- a/pulsar/client/amqp_exchange.py
+++ b/pulsar/client/amqp_exchange.py
@@ -1,8 +1,9 @@
import copy
-import uuid
-import socket
import logging
+import socket
import threading
+import uuid
+
from time import sleep, time
try:
diff --git a/pulsar/client/client.py b/pulsar/client/client.py
index 4b9c4df07d289538880c552de3d347c130861889..e3af9783fd475c78eb42362bad96cad93af8a52b 100644
--- a/pulsar/client/client.py
+++ b/pulsar/client/client.py
@@ -1,23 +1,23 @@
+import logging
import os
from six import string_types
-from .destination import submit_params
-from .setup_handler import build as build_setup_handler
-from .job_directory import RemoteJobDirectory
+from .action_mapper import (
+ actions,
+ path_type,
+)
from .decorators import parseJson
from .decorators import retry
-from .util import json_dumps
-from .util import json_loads
+from .destination import submit_params
+from .job_directory import RemoteJobDirectory
+from .setup_handler import build as build_setup_handler
from .util import copy
from .util import ensure_directory
+from .util import json_dumps
+from .util import json_loads
from .util import to_base64_json
-from .action_mapper import (
- path_type,
- actions,
-)
-import logging
log = logging.getLogger(__name__)
CACHE_WAIT_SECONDS = 3
diff --git a/pulsar/client/destination.py b/pulsar/client/destination.py
index cee7a40f63025b8cefdce00ca039d5861ec4a094..7197da976a2d323c025564d75be0d0d2517de3c6 100644
--- a/pulsar/client/destination.py
+++ b/pulsar/client/destination.py
@@ -1,5 +1,6 @@
from re import match
+
from .util import filter_destination_params
SUBMIT_PREFIX = "submit_"
diff --git a/pulsar/client/interface.py b/pulsar/client/interface.py
index d0f5b117ddbfae1992af0c9a4550e6cdaa3eb9bb..00962d3ca71efdae852f66bd26395f1332bd78b7 100644
--- a/pulsar/client/interface.py
+++ b/pulsar/client/interface.py
@@ -1,18 +1,19 @@
+import logging
+
from abc import ABCMeta
from abc import abstractmethod
from string import Template
from six import BytesIO
-from six.moves.urllib.parse import urlencode
-from six.moves.urllib.parse import urljoin
try:
from six import text_type
except ImportError:
from galaxy.util import unicodify as text_type
+from six.moves.urllib.parse import urlencode
+from six.moves.urllib.parse import urljoin
from .util import copy_to_path
-import logging
log = logging.getLogger(__name__)
diff --git a/pulsar/client/job_directory.py b/pulsar/client/job_directory.py
index f8d49b1c12151cfb5b0928a881626dd307d3cf50..cd00e96b5ac59ad2514b2c6167ce03c57c6ecd5f 100644
--- a/pulsar/client/job_directory.py
+++ b/pulsar/client/job_directory.py
@@ -1,13 +1,14 @@
"""
"""
import os.path
-from collections import deque
import posixpath
-from .util import PathHelper
+from collections import deque
+from logging import getLogger
+
from galaxy.util import in_directory
-from logging import getLogger
+from .util import PathHelper
log = getLogger(__name__)
diff --git a/pulsar/client/manager.py b/pulsar/client/manager.py
index c43db0896c2db45738a00ff6b7d47ce458056f33..459efa0905ab33a0c9e97469b17e65c28527d02c 100644
--- a/pulsar/client/manager.py
+++ b/pulsar/client/manager.py
@@ -5,28 +5,29 @@ to create a ``ClientManager``, that in return can create Pulsar clients for
specific actions.
"""
-import threading
import functools
+import threading
+
+from logging import getLogger
+from os import getenv
try:
from Queue import Queue
except ImportError:
from queue import Queue
-from os import getenv
-from .client import JobClient
+from .amqp_exchange_factory import get_exchange
from .client import InputCachingJobClient
+from .client import JobClient
from .client import MessageJobClient
from .client import MessageCLIJobClient
+from .destination import url_to_destination_params
from .interface import HttpPulsarInterface
from .interface import LocalPulsarInterface
from .object_client import ObjectStoreClient
from .transport import get_transport
from .util import TransferEventManager
-from .destination import url_to_destination_params
-from .amqp_exchange_factory import get_exchange
-from logging import getLogger
log = getLogger(__name__)
DEFAULT_TRANSFER_THREADS = 2
diff --git a/pulsar/client/path_mapper.py b/pulsar/client/path_mapper.py
index 8e963387bb171fe37eb641ae9b764a5a6bde4d81..4ab8805b62a1eddeadb6fdebee898f9b581c841f 100644
--- a/pulsar/client/path_mapper.py
+++ b/pulsar/client/path_mapper.py
@@ -1,10 +1,11 @@
import os.path
+
+from galaxy.util import in_directory
+
from .action_mapper import FileActionMapper
from .action_mapper import path_type
from .util import PathHelper
-from galaxy.util import in_directory
-
class PathMapper(object):
""" Ties together a FileActionMapper and remote job configuration returned
diff --git a/pulsar/client/setup_handler.py b/pulsar/client/setup_handler.py
index 368deb8b9d00f73965a20f8085ebe3822453871c..8147579499c37cfb22d0d9692131959c3ef0a350 100644
--- a/pulsar/client/setup_handler.py
+++ b/pulsar/client/setup_handler.py
@@ -1,8 +1,9 @@
import os
-from .util import filter_destination_params
from pulsar import __version__ as pulsar_version
+from .util import filter_destination_params
+
REMOTE_SYSTEM_PROPERTY_PREFIX = "remote_property_"
diff --git a/pulsar/client/staging/__init__.py b/pulsar/client/staging/__init__.py
index 029e08bc9c2bd60d04580933fdad641a1aeed6dd..de2718ade6edc4b507af2db608cd8e4ad86defcf 100644
--- a/pulsar/client/staging/__init__.py
+++ b/pulsar/client/staging/__init__.py
@@ -1,8 +1,12 @@
+
import re
-from os.path import basename
-from os.path import join
-from os.path import dirname
+
from os import sep
+from os.path import (
+ basename,
+ dirname,
+ join,
+)
from ..util import PathHelper
diff --git a/pulsar/client/staging/down.py b/pulsar/client/staging/down.py
index 10a3423bf4599a39fe9627dec5a31877a64b9838..afdd1cec5c98cc14722dffa279ac625b40be4d2d 100644
--- a/pulsar/client/staging/down.py
+++ b/pulsar/client/staging/down.py
@@ -1,13 +1,12 @@
"""Code run on the client side for unstaging complete Pulsar jobs."""
-from os.path import join
-from os.path import relpath
from contextlib import contextmanager
+from logging import getLogger
+from os.path import join, relpath
-from ..staging import COMMAND_VERSION_FILENAME
from ..action_mapper import FileActionMapper
+from ..staging import COMMAND_VERSION_FILENAME
-from logging import getLogger
log = getLogger(__name__)
diff --git a/pulsar/client/staging/up.py b/pulsar/client/staging/up.py
index 1db85b0bdd89fbec3c72b9185704efaba4d4f0ab..92b93cd2faa3e271e20db5215a47ed8ed5a69669 100644
--- a/pulsar/client/staging/up.py
+++ b/pulsar/client/staging/up.py
@@ -1,19 +1,24 @@
-from os.path import abspath, basename, join, exists
-from os.path import isfile
-from os.path import dirname
-from os.path import relpath
+from io import open
+from logging import getLogger
from os import listdir, sep
+from os.path import (
+ abspath,
+ basename,
+ dirname,
+ exists,
+ isfile,
+ join,
+ relpath,
+)
from re import findall
-from io import open
-from ..staging import COMMAND_VERSION_FILENAME
from ..action_mapper import FileActionMapper
-from ..action_mapper import path_type
from ..action_mapper import MessageAction
-from ..util import PathHelper
+from ..action_mapper import path_type
+from ..staging import COMMAND_VERSION_FILENAME
from ..util import directory_files
+from ..util import PathHelper
-from logging import getLogger
log = getLogger(__name__)
diff --git a/pulsar/client/test/check.py b/pulsar/client/test/check.py
index 599cf01bc018b87b48556cd3d5ac8c3602b8f050..c9b2f8ab1f86e4c43cc165abc99be67964877364 100644
--- a/pulsar/client/test/check.py
+++ b/pulsar/client/test/check.py
@@ -3,6 +3,7 @@
Exercises various features both the Pulsar client and server.
"""
+
import optparse
import os
import re
@@ -12,20 +13,23 @@ import tempfile
import threading
import time
import traceback
-from collections import namedtuple
+from collections import namedtuple
from io import open
-from six import binary_type
-
-from pulsar.client import submit_job
-from pulsar.client import finish_job
-from pulsar.client import PulsarOutputs
-from pulsar.client import ClientOutputs
-from pulsar.client import build_client_manager
-from pulsar.client import ClientJobDescription
from galaxy.tools.deps.dependencies import DependenciesDescription
from galaxy.tools.deps.requirements import ToolRequirement
+from six import binary_type
+
+from pulsar.client import (
+ build_client_manager,
+ ClientJobDescription,
+ ClientOutputs,
+ finish_job,
+ PulsarOutputs,
+ submit_job,
+)
+
from .test_common import write_config
TEST_SCRIPT = b"""# -*- coding: utf-8 -*-
diff --git a/pulsar/client/transport/__init__.py b/pulsar/client/transport/__init__.py
index db2541baabb55c83c1a861ed612873b7e29972c0..a1a0f7de74e96a872a5b8f94c163462264618e0d 100644
--- a/pulsar/client/transport/__init__.py
+++ b/pulsar/client/transport/__init__.py
@@ -1,12 +1,12 @@
-from .standard import Urllib2Transport
-from .curl import PycurlTransport
import os
+from .curl import curl_available
+from .curl import PycurlTransport
+from .requests import requests_multipart_post_available
from .ssh import rsync_get_file, scp_get_file
from .ssh import rsync_post_file, scp_post_file
+from .standard import Urllib2Transport
-from .curl import curl_available
-from .requests import requests_multipart_post_available
if curl_available:
from .curl import get_file
from .curl import post_file
diff --git a/pulsar/client/transport/curl.py b/pulsar/client/transport/curl.py
index 02ff12c547c0a3fb9934c46ea4dcaf727b9e2a28..f59d9364dc834bf1fba0d5c2279fa4fdc1d4ab14 100644
--- a/pulsar/client/transport/curl.py
+++ b/pulsar/client/transport/curl.py
@@ -1,8 +1,5 @@
-import os.path
import logging
-
-from six import string_types
-from six import BytesIO
+import os.path
try:
import pycurl
@@ -10,6 +7,8 @@ try:
curl_available = True
except ImportError:
curl_available = False
+from six import string_types
+from six import BytesIO
from ..exceptions import PulsarClientTransportError
@@ -52,9 +51,9 @@ class PycurlTransport(object):
c.perform()
except error as exc:
raise PulsarClientTransportError(
- _error_curl_to_pulsar(exc.args[0]),
- transport_code=exc.args[0],
- transport_message=exc.args[1])
+ _error_curl_to_pulsar(exc.args[0]),
+ transport_code=exc.args[0],
+ transport_message=exc.args[1])
if not output_path:
return buf.getvalue()
finally:
diff --git a/pulsar/client/transport/ssh.py b/pulsar/client/transport/ssh.py
index b0c7a3b13fd1acbdb777bdd250466690100c2f3e..f0aff6f925081cb00b3213a6ede55aa231fc0467 100644
--- a/pulsar/client/transport/ssh.py
+++ b/pulsar/client/transport/ssh.py
@@ -1,5 +1,5 @@
-import subprocess
import os
+import subprocess
SSH_OPTIONS = ['-o', 'StrictHostKeyChecking=no', '-o', 'PreferredAuthentications=publickey', '-o', 'PubkeyAuthentication=yes']
diff --git a/pulsar/client/transport/standard.py b/pulsar/client/transport/standard.py
index 01df9c71b80e07fb4d19da5f808e7e86f02d0efe..25ce84c490a2f4caf210300763dcd62d2f9ebf02 100644
--- a/pulsar/client/transport/standard.py
+++ b/pulsar/client/transport/standard.py
@@ -1,10 +1,13 @@
"""
Pulsar HTTP Client layer based on Python Standard Library (urllib2)
"""
+
from __future__ import with_statement
-from os.path import getsize
+
import mmap
import socket
+
+from os.path import getsize
try:
from urllib2 import urlopen, URLError
except ImportError:
diff --git a/pulsar/client/util.py b/pulsar/client/util.py
index 9edda1a0b40273c786156c3bc199355d27ba93f2..1a2497eba190a59991ace136c7cad21d4b9592ad 100644
--- a/pulsar/client/util.py
+++ b/pulsar/client/util.py
@@ -1,22 +1,27 @@
-from functools import wraps
-from threading import Lock, Event
-from weakref import WeakValueDictionary
-from os import walk
-from os import curdir
-from os import listdir
-from os import makedirs
-from os import unlink
-from os.path import relpath
-from os.path import join
-from os.path import abspath
-from os.path import exists
-from errno import ENOENT, EEXIST
-import os.path
import hashlib
-import shutil
import json
+import os.path
+import shutil
import sys
+from errno import EEXIST, ENOENT
+from functools import wraps
+from os import (
+ curdir,
+ listdir,
+ makedirs,
+ unlink,
+ walk,
+)
+from os.path import (
+ abspath,
+ exists,
+ join,
+ relpath,
+)
+from threading import Event, Lock
+from weakref import WeakValueDictionary
+
from six import binary_type
# TODO: move to galaxy.util so it doesn't have to be duplicated
diff --git a/pulsar/mesos/framework.py b/pulsar/mesos/framework.py
index 636c25cc41ce689c64d7d16115031e8421b77168..dc774247222582001006ade424c93f44594e3419 100644
--- a/pulsar/mesos/framework.py
+++ b/pulsar/mesos/framework.py
@@ -2,18 +2,18 @@ import collections
import logging
import os
-from pulsar.mesos import (
- Scheduler,
- MesosSchedulerDriver,
- mesos_pb2,
-)
from pulsar.client.util import to_base64_json
-
from pulsar.main import (
PULSAR_ROOT_DIR,
)
+from pulsar.mesos import (
+ mesos_pb2,
+ MesosSchedulerDriver,
+ Scheduler,
+)
from pulsar.messaging import bind_amqp
+
log = logging.getLogger(__name__)
diff --git a/pulsar/messaging/__init__.py b/pulsar/messaging/__init__.py
index 720486a5cfce1c94fb68445fec3b0b85796cc289..67d1a6d36de69e92ab8005e9fba0a98cfbf58ca5 100644
--- a/pulsar/messaging/__init__.py
+++ b/pulsar/messaging/__init__.py
@@ -5,9 +5,10 @@ submodules of ``pulsar.client``.
import logging
-from ..messaging import bind_amqp
from six import itervalues
+from ..messaging import bind_amqp
+
log = logging.getLogger(__name__)
diff --git a/pulsar/messaging/bind_amqp.py b/pulsar/messaging/bind_amqp.py
index 54c54cc2231317c987bc0c50693c26cb2066d95d..b17c7e2441385768e7cef59a849dab64311c029d 100644
--- a/pulsar/messaging/bind_amqp.py
+++ b/pulsar/messaging/bind_amqp.py
@@ -1,9 +1,11 @@
-from galaxy.util import asbool, mask_password_from_url
-from pulsar.client import amqp_exchange_factory
-from pulsar import manager_endpoint_util
import functools
-import threading
import logging
+import threading
+
+from galaxy.util import asbool, mask_password_from_url
+
+from pulsar import manager_endpoint_util
+from pulsar.client import amqp_exchange_factory
log = logging.getLogger(__name__)
diff --git a/pulsar/scripts/submit_util.py b/pulsar/scripts/submit_util.py
index 87ce6df17f25482385e9ce7b86b7b8548626be48..5de1334a09e0c92cd849e96e93e502fd1db361a6 100644
--- a/pulsar/scripts/submit_util.py
+++ b/pulsar/scripts/submit_util.py
@@ -1,7 +1,8 @@
""" CLI related utilities for submitting Pulsar jobs.
"""
-import time
import json
+import logging
+import time
from pulsar.client.util import from_base64_json
from pulsar.main import (
@@ -11,7 +12,6 @@ from pulsar.main import (
from pulsar.manager_endpoint_util import submit_job
from pulsar.managers.status import is_job_done
-import logging
log = logging.getLogger(__name__)
DEFAULT_POLL_TIME = 2
diff --git a/pulsar/tools/toolbox.py b/pulsar/tools/toolbox.py
index 6386d578e3015d6d769bfc8b55326e20fe353377..0d191ab17eee6aaf2fb2d708a78c11fe3343e3be 100644
--- a/pulsar/tools/toolbox.py
+++ b/pulsar/tools/toolbox.py
@@ -1,10 +1,11 @@
-from pulsar.tools.validator import ExpressionValidator
+from io import open
+from logging import getLogger
+from os.path import abspath, dirname, join
from xml.etree import ElementTree
-from os.path import join, abspath, dirname
-from io import open
+from pulsar.tools.validator import ExpressionValidator
+
-from logging import getLogger
log = getLogger(__name__)
diff --git a/pulsar/tools/validator.py b/pulsar/tools/validator.py
index 9cb77e95bc526dd15ba8f6fede5fb6359da26276..b181afc4b839fb80e1fba8826639b4fa25decfba 100644
--- a/pulsar/tools/validator.py
+++ b/pulsar/tools/validator.py
@@ -1,7 +1,7 @@
+from os.path import join
+from re import compile, escape
from xml.etree.ElementTree import fromstring
-from re import escape, compile
-from os.path import join
from galaxy.util import in_directory
diff --git a/pulsar/web/routes.py b/pulsar/web/routes.py
index 1f529608b86957c7f020f65f385a20f3bbabf290..f460ef439cb488f030fe04e585a5306fd6a00b2a 100644
--- a/pulsar/web/routes.py
+++ b/pulsar/web/routes.py
@@ -1,22 +1,24 @@
+import logging
import os
-from webob import exc
+
from json import loads
+from webob import exc
+
from pulsar.util import (
copy_to_path,
copy_to_temp,
)
from pulsar.client.job_directory import verify_is_in_directory
-from pulsar.web.framework import Controller
-from pulsar.manager_factory import DEFAULT_MANAGER_NAME
+from pulsar.client.action_mapper import path_type
from pulsar.manager_endpoint_util import (
- submit_job,
setup_job,
status_dict,
+ submit_job,
)
-from pulsar.client.action_mapper import path_type
+from pulsar.manager_factory import DEFAULT_MANAGER_NAME
+from pulsar.web.framework import Controller
-import logging
log = logging.getLogger(__name__)