From a238a724c33f91bb35a58db0c51f55babb801840 Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Mon, 29 Aug 2016 23:53:48 -0400
Subject: [PATCH] Slew of import order fixes to bring in line with gx standard.
---
pulsar/cache/__init__.py | 2 +-
pulsar/cache/persistence.py | 4 ++-
pulsar/cache/util.py | 9 ++++---
pulsar/client/__init__.py | 14 +++++------
pulsar/client/action_mapper.py | 38 +++++++++++++++++------------
pulsar/client/amqp_exchange.py | 5 ++--
pulsar/client/client.py | 20 +++++++--------
pulsar/client/destination.py | 1 +
pulsar/client/interface.py | 7 +++---
pulsar/client/job_directory.py | 7 +++---
pulsar/client/manager.py | 13 +++++-----
pulsar/client/path_mapper.py | 5 ++--
pulsar/client/setup_handler.py | 3 ++-
pulsar/client/staging/__init__.py | 10 +++++---
pulsar/client/staging/down.py | 7 +++---
pulsar/client/staging/up.py | 23 ++++++++++-------
pulsar/client/test/check.py | 22 ++++++++++-------
pulsar/client/transport/__init__.py | 8 +++---
pulsar/client/transport/curl.py | 13 +++++-----
pulsar/client/transport/ssh.py | 2 +-
pulsar/client/transport/standard.py | 5 +++-
pulsar/client/util.py | 35 ++++++++++++++------------
pulsar/mesos/framework.py | 12 ++++-----
pulsar/messaging/__init__.py | 3 ++-
pulsar/messaging/bind_amqp.py | 10 +++++---
pulsar/scripts/submit_util.py | 4 +--
pulsar/tools/toolbox.py | 9 ++++---
pulsar/tools/validator.py | 4 +--
pulsar/web/routes.py | 14 ++++++-----
29 files changed, 176 insertions(+), 133 deletions(-)
diff --git a/pulsar/cache/__init__.py b/pulsar/cache/__init__.py
index 7a97262b..57cbffe8 100644
--- a/pulsar/cache/__init__.py
+++ b/pulsar/cache/__init__.py
@@ -1,6 +1,6 @@
-from os.path import join, exists
from hashlib import sha256
+from os.path import exists, join
from .persistence import PersistenceStore
from .util import atomicish_move
diff --git a/pulsar/cache/persistence.py b/pulsar/cache/persistence.py
index 2bc667d9..67262850 100644
--- a/pulsar/cache/persistence.py
+++ b/pulsar/cache/persistence.py
@@ -1,7 +1,9 @@
+
import shelve
-from threading import Lock
import traceback
+from threading import Lock
+
class PersistenceStore(object):
diff --git a/pulsar/cache/util.py b/pulsar/cache/util.py
index a8926e3e..dba36897 100644
--- a/pulsar/cache/util.py
+++ b/pulsar/cache/util.py
@@ -1,12 +1,12 @@
+
import os
import shutil
+
from datetime import datetime
def atomicish_move(source, destination, tmp_suffix="_TMP"):
- """
- Move source to destination without copying to directly to destination
- there is never a partial file.
+ """Move source to destination without risk of partial moves.
> from tempfile import mkdtemp
> from os.path import join, exists
@@ -28,8 +28,9 @@ def atomicish_move(source, destination, tmp_suffix="_TMP"):
class Time:
- """ Time utilities of now that can be instrumented for testing."""
+ """Time utilities of now that can be instrumented for testing."""
@classmethod
def now(cls):
+ """Return the current datetime."""
return datetime.utcnow()
diff --git a/pulsar/client/__init__.py b/pulsar/client/__init__.py
index b28ab87c..a42c6017 100644
--- a/pulsar/client/__init__.py
+++ b/pulsar/client/__init__.py
@@ -39,16 +39,16 @@ in the ``galaxy.ini`` file.
"""
-from .staging.down import finish_job
-from .staging.up import submit_job
-from .staging import ClientJobDescription
-from .staging import PulsarOutputs
-from .staging import ClientOutputs
from .client import OutputNotFoundException
-from .manager import build_client_manager
from .destination import url_to_destination_params
-from .path_mapper import PathMapper
from .exceptions import PulsarClientTransportError
+from .manager import build_client_manager
+from .path_mapper import PathMapper
+from .staging import ClientJobDescription
+from .staging import PulsarOutputs
+from .staging import ClientOutputs
+from .staging.down import finish_job
+from .staging.up import submit_job
__all__ = [
'build_client_manager',
diff --git a/pulsar/client/action_mapper.py b/pulsar/client/action_mapper.py
index 34e40b75..6d698592 100644
--- a/pulsar/client/action_mapper.py
+++ b/pulsar/client/action_mapper.py
@@ -1,26 +1,34 @@
+import fnmatch
+import tempfile
+
from contextlib import contextmanager
from json import load
from os import makedirs
from os import unlink
-from os.path import exists
-from os.path import abspath
-from os.path import dirname
-from os.path import join
-from os.path import basename
-from os.path import sep
-import fnmatch
-from re import compile
-from re import escape
+from os.path import (
+ abspath,
+ basename,
+ dirname,
+ exists,
+ join,
+ sep,
+)
+from re import compile, escape
+
from galaxy.util.bunch import Bunch
+
from .config_util import read_file
-from .util import directory_files
-from .util import unique_path_prefix
-from .util import copy_to_path
from .transport import get_file
from .transport import post_file
-from .transport import rsync_get_file, scp_get_file
-from .transport import rsync_post_file, scp_post_file
-import tempfile
+from .transport import (
+ rsync_get_file,
+ rsync_post_file,
+ scp_get_file,
+ scp_post_file,
+)
+from .util import copy_to_path
+from .util import directory_files
+from .util import unique_path_prefix
DEFAULT_MAPPED_ACTION = 'transfer' # Not really clear to me what this should be, exception?
diff --git a/pulsar/client/amqp_exchange.py b/pulsar/client/amqp_exchange.py
index 03ad85ff..9c47ee08 100644
--- a/pulsar/client/amqp_exchange.py
+++ b/pulsar/client/amqp_exchange.py
@@ -1,8 +1,9 @@
import copy
-import uuid
-import socket
import logging
+import socket
import threading
+import uuid
+
from time import sleep, time
try:
diff --git a/pulsar/client/client.py b/pulsar/client/client.py
index 4b9c4df0..e3af9783 100644
--- a/pulsar/client/client.py
+++ b/pulsar/client/client.py
@@ -1,23 +1,23 @@
+import logging
import os
from six import string_types
-from .destination import submit_params
-from .setup_handler import build as build_setup_handler
-from .job_directory import RemoteJobDirectory
+from .action_mapper import (
+ actions,
+ path_type,
+)
from .decorators import parseJson
from .decorators import retry
-from .util import json_dumps
-from .util import json_loads
+from .destination import submit_params
+from .job_directory import RemoteJobDirectory
+from .setup_handler import build as build_setup_handler
from .util import copy
from .util import ensure_directory
+from .util import json_dumps
+from .util import json_loads
from .util import to_base64_json
-from .action_mapper import (
- path_type,
- actions,
-)
-import logging
log = logging.getLogger(__name__)
CACHE_WAIT_SECONDS = 3
diff --git a/pulsar/client/destination.py b/pulsar/client/destination.py
index cee7a40f..7197da97 100644
--- a/pulsar/client/destination.py
+++ b/pulsar/client/destination.py
@@ -1,5 +1,6 @@
from re import match
+
from .util import filter_destination_params
SUBMIT_PREFIX = "submit_"
diff --git a/pulsar/client/interface.py b/pulsar/client/interface.py
index d0f5b117..00962d3c 100644
--- a/pulsar/client/interface.py
+++ b/pulsar/client/interface.py
@@ -1,18 +1,19 @@
+import logging
+
from abc import ABCMeta
from abc import abstractmethod
from string import Template
from six import BytesIO
-from six.moves.urllib.parse import urlencode
-from six.moves.urllib.parse import urljoin
try:
from six import text_type
except ImportError:
from galaxy.util import unicodify as text_type
+from six.moves.urllib.parse import urlencode
+from six.moves.urllib.parse import urljoin
from .util import copy_to_path
-import logging
log = logging.getLogger(__name__)
diff --git a/pulsar/client/job_directory.py b/pulsar/client/job_directory.py
index f8d49b1c..cd00e96b 100644
--- a/pulsar/client/job_directory.py
+++ b/pulsar/client/job_directory.py
@@ -1,13 +1,14 @@
"""
"""
import os.path
-from collections import deque
import posixpath
-from .util import PathHelper
+from collections import deque
+from logging import getLogger
+
from galaxy.util import in_directory
-from logging import getLogger
+from .util import PathHelper
log = getLogger(__name__)
diff --git a/pulsar/client/manager.py b/pulsar/client/manager.py
index c43db089..459efa09 100644
--- a/pulsar/client/manager.py
+++ b/pulsar/client/manager.py
@@ -5,28 +5,29 @@ to create a ``ClientManager``, that in return can create Pulsar clients for
specific actions.
"""
-import threading
import functools
+import threading
+
+from logging import getLogger
+from os import getenv
try:
from Queue import Queue
except ImportError:
from queue import Queue
-from os import getenv
-from .client import JobClient
+from .amqp_exchange_factory import get_exchange
from .client import InputCachingJobClient
+from .client import JobClient
from .client import MessageJobClient
from .client import MessageCLIJobClient
+from .destination import url_to_destination_params
from .interface import HttpPulsarInterface
from .interface import LocalPulsarInterface
from .object_client import ObjectStoreClient
from .transport import get_transport
from .util import TransferEventManager
-from .destination import url_to_destination_params
-from .amqp_exchange_factory import get_exchange
-from logging import getLogger
log = getLogger(__name__)
DEFAULT_TRANSFER_THREADS = 2
diff --git a/pulsar/client/path_mapper.py b/pulsar/client/path_mapper.py
index 8e963387..4ab8805b 100644
--- a/pulsar/client/path_mapper.py
+++ b/pulsar/client/path_mapper.py
@@ -1,10 +1,11 @@
import os.path
+
+from galaxy.util import in_directory
+
from .action_mapper import FileActionMapper
from .action_mapper import path_type
from .util import PathHelper
-from galaxy.util import in_directory
-
class PathMapper(object):
""" Ties together a FileActionMapper and remote job configuration returned
diff --git a/pulsar/client/setup_handler.py b/pulsar/client/setup_handler.py
index 368deb8b..81475794 100644
--- a/pulsar/client/setup_handler.py
+++ b/pulsar/client/setup_handler.py
@@ -1,8 +1,9 @@
import os
-from .util import filter_destination_params
from pulsar import __version__ as pulsar_version
+from .util import filter_destination_params
+
REMOTE_SYSTEM_PROPERTY_PREFIX = "remote_property_"
diff --git a/pulsar/client/staging/__init__.py b/pulsar/client/staging/__init__.py
index 029e08bc..de2718ad 100644
--- a/pulsar/client/staging/__init__.py
+++ b/pulsar/client/staging/__init__.py
@@ -1,8 +1,12 @@
+
import re
-from os.path import basename
-from os.path import join
-from os.path import dirname
+
from os import sep
+from os.path import (
+ basename,
+ dirname,
+ join,
+)
from ..util import PathHelper
diff --git a/pulsar/client/staging/down.py b/pulsar/client/staging/down.py
index 10a3423b..afdd1cec 100644
--- a/pulsar/client/staging/down.py
+++ b/pulsar/client/staging/down.py
@@ -1,13 +1,12 @@
"""Code run on the client side for unstaging complete Pulsar jobs."""
-from os.path import join
-from os.path import relpath
from contextlib import contextmanager
+from logging import getLogger
+from os.path import join, relpath
-from ..staging import COMMAND_VERSION_FILENAME
from ..action_mapper import FileActionMapper
+from ..staging import COMMAND_VERSION_FILENAME
-from logging import getLogger
log = getLogger(__name__)
diff --git a/pulsar/client/staging/up.py b/pulsar/client/staging/up.py
index 1db85b0b..92b93cd2 100644
--- a/pulsar/client/staging/up.py
+++ b/pulsar/client/staging/up.py
@@ -1,19 +1,24 @@
-from os.path import abspath, basename, join, exists
-from os.path import isfile
-from os.path import dirname
-from os.path import relpath
+from io import open
+from logging import getLogger
from os import listdir, sep
+from os.path import (
+ abspath,
+ basename,
+ dirname,
+ exists,
+ isfile,
+ join,
+ relpath,
+)
from re import findall
-from io import open
-from ..staging import COMMAND_VERSION_FILENAME
from ..action_mapper import FileActionMapper
-from ..action_mapper import path_type
from ..action_mapper import MessageAction
-from ..util import PathHelper
+from ..action_mapper import path_type
+from ..staging import COMMAND_VERSION_FILENAME
from ..util import directory_files
+from ..util import PathHelper
-from logging import getLogger
log = getLogger(__name__)
diff --git a/pulsar/client/test/check.py b/pulsar/client/test/check.py
index 599cf01b..c9b2f8ab 100644
--- a/pulsar/client/test/check.py
+++ b/pulsar/client/test/check.py
@@ -3,6 +3,7 @@
Exercises various features both the Pulsar client and server.
"""
+
import optparse
import os
import re
@@ -12,20 +13,23 @@ import tempfile
import threading
import time
import traceback
-from collections import namedtuple
+from collections import namedtuple
from io import open
-from six import binary_type
-
-from pulsar.client import submit_job
-from pulsar.client import finish_job
-from pulsar.client import PulsarOutputs
-from pulsar.client import ClientOutputs
-from pulsar.client import build_client_manager
-from pulsar.client import ClientJobDescription
from galaxy.tools.deps.dependencies import DependenciesDescription
from galaxy.tools.deps.requirements import ToolRequirement
+from six import binary_type
+
+from pulsar.client import (
+ build_client_manager,
+ ClientJobDescription,
+ ClientOutputs,
+ finish_job,
+ PulsarOutputs,
+ submit_job,
+)
+
from .test_common import write_config
TEST_SCRIPT = b"""# -*- coding: utf-8 -*-
diff --git a/pulsar/client/transport/__init__.py b/pulsar/client/transport/__init__.py
index db2541ba..a1a0f7de 100644
--- a/pulsar/client/transport/__init__.py
+++ b/pulsar/client/transport/__init__.py
@@ -1,12 +1,12 @@
-from .standard import Urllib2Transport
-from .curl import PycurlTransport
import os
+from .curl import curl_available
+from .curl import PycurlTransport
+from .requests import requests_multipart_post_available
from .ssh import rsync_get_file, scp_get_file
from .ssh import rsync_post_file, scp_post_file
+from .standard import Urllib2Transport
-from .curl import curl_available
-from .requests import requests_multipart_post_available
if curl_available:
from .curl import get_file
from .curl import post_file
diff --git a/pulsar/client/transport/curl.py b/pulsar/client/transport/curl.py
index 02ff12c5..f59d9364 100644
--- a/pulsar/client/transport/curl.py
+++ b/pulsar/client/transport/curl.py
@@ -1,8 +1,5 @@
-import os.path
import logging
-
-from six import string_types
-from six import BytesIO
+import os.path
try:
import pycurl
@@ -10,6 +7,8 @@ try:
curl_available = True
except ImportError:
curl_available = False
+from six import string_types
+from six import BytesIO
from ..exceptions import PulsarClientTransportError
@@ -52,9 +51,9 @@ class PycurlTransport(object):
c.perform()
except error as exc:
raise PulsarClientTransportError(
- _error_curl_to_pulsar(exc.args[0]),
- transport_code=exc.args[0],
- transport_message=exc.args[1])
+ _error_curl_to_pulsar(exc.args[0]),
+ transport_code=exc.args[0],
+ transport_message=exc.args[1])
if not output_path:
return buf.getvalue()
finally:
diff --git a/pulsar/client/transport/ssh.py b/pulsar/client/transport/ssh.py
index b0c7a3b1..f0aff6f9 100644
--- a/pulsar/client/transport/ssh.py
+++ b/pulsar/client/transport/ssh.py
@@ -1,5 +1,5 @@
-import subprocess
import os
+import subprocess
SSH_OPTIONS = ['-o', 'StrictHostKeyChecking=no', '-o', 'PreferredAuthentications=publickey', '-o', 'PubkeyAuthentication=yes']
diff --git a/pulsar/client/transport/standard.py b/pulsar/client/transport/standard.py
index 01df9c71..25ce84c4 100644
--- a/pulsar/client/transport/standard.py
+++ b/pulsar/client/transport/standard.py
@@ -1,10 +1,13 @@
"""
Pulsar HTTP Client layer based on Python Standard Library (urllib2)
"""
+
from __future__ import with_statement
-from os.path import getsize
+
import mmap
import socket
+
+from os.path import getsize
try:
from urllib2 import urlopen, URLError
except ImportError:
diff --git a/pulsar/client/util.py b/pulsar/client/util.py
index 9edda1a0..1a2497eb 100644
--- a/pulsar/client/util.py
+++ b/pulsar/client/util.py
@@ -1,22 +1,27 @@
-from functools import wraps
-from threading import Lock, Event
-from weakref import WeakValueDictionary
-from os import walk
-from os import curdir
-from os import listdir
-from os import makedirs
-from os import unlink
-from os.path import relpath
-from os.path import join
-from os.path import abspath
-from os.path import exists
-from errno import ENOENT, EEXIST
-import os.path
import hashlib
-import shutil
import json
+import os.path
+import shutil
import sys
+from errno import EEXIST, ENOENT
+from functools import wraps
+from os import (
+ curdir,
+ listdir,
+ makedirs,
+ unlink,
+ walk,
+)
+from os.path import (
+ abspath,
+ exists,
+ join,
+ relpath,
+)
+from threading import Event, Lock
+from weakref import WeakValueDictionary
+
from six import binary_type
# TODO: move to galaxy.util so it doesn't have to be duplicated
diff --git a/pulsar/mesos/framework.py b/pulsar/mesos/framework.py
index 636c25cc..dc774247 100644
--- a/pulsar/mesos/framework.py
+++ b/pulsar/mesos/framework.py
@@ -2,18 +2,18 @@ import collections
import logging
import os
-from pulsar.mesos import (
- Scheduler,
- MesosSchedulerDriver,
- mesos_pb2,
-)
from pulsar.client.util import to_base64_json
-
from pulsar.main import (
PULSAR_ROOT_DIR,
)
+from pulsar.mesos import (
+ mesos_pb2,
+ MesosSchedulerDriver,
+ Scheduler,
+)
from pulsar.messaging import bind_amqp
+
log = logging.getLogger(__name__)
diff --git a/pulsar/messaging/__init__.py b/pulsar/messaging/__init__.py
index 720486a5..67d1a6d3 100644
--- a/pulsar/messaging/__init__.py
+++ b/pulsar/messaging/__init__.py
@@ -5,9 +5,10 @@ submodules of ``pulsar.client``.
import logging
-from ..messaging import bind_amqp
from six import itervalues
+from ..messaging import bind_amqp
+
log = logging.getLogger(__name__)
diff --git a/pulsar/messaging/bind_amqp.py b/pulsar/messaging/bind_amqp.py
index 54c54cc2..b17c7e24 100644
--- a/pulsar/messaging/bind_amqp.py
+++ b/pulsar/messaging/bind_amqp.py
@@ -1,9 +1,11 @@
-from galaxy.util import asbool, mask_password_from_url
-from pulsar.client import amqp_exchange_factory
-from pulsar import manager_endpoint_util
import functools
-import threading
import logging
+import threading
+
+from galaxy.util import asbool, mask_password_from_url
+
+from pulsar import manager_endpoint_util
+from pulsar.client import amqp_exchange_factory
log = logging.getLogger(__name__)
diff --git a/pulsar/scripts/submit_util.py b/pulsar/scripts/submit_util.py
index 87ce6df1..5de1334a 100644
--- a/pulsar/scripts/submit_util.py
+++ b/pulsar/scripts/submit_util.py
@@ -1,7 +1,8 @@
""" CLI related utilities for submitting Pulsar jobs.
"""
-import time
import json
+import logging
+import time
from pulsar.client.util import from_base64_json
from pulsar.main import (
@@ -11,7 +12,6 @@ from pulsar.main import (
from pulsar.manager_endpoint_util import submit_job
from pulsar.managers.status import is_job_done
-import logging
log = logging.getLogger(__name__)
DEFAULT_POLL_TIME = 2
diff --git a/pulsar/tools/toolbox.py b/pulsar/tools/toolbox.py
index 6386d578..0d191ab1 100644
--- a/pulsar/tools/toolbox.py
+++ b/pulsar/tools/toolbox.py
@@ -1,10 +1,11 @@
-from pulsar.tools.validator import ExpressionValidator
+from io import open
+from logging import getLogger
+from os.path import abspath, dirname, join
from xml.etree import ElementTree
-from os.path import join, abspath, dirname
-from io import open
+from pulsar.tools.validator import ExpressionValidator
+
-from logging import getLogger
log = getLogger(__name__)
diff --git a/pulsar/tools/validator.py b/pulsar/tools/validator.py
index 9cb77e95..b181afc4 100644
--- a/pulsar/tools/validator.py
+++ b/pulsar/tools/validator.py
@@ -1,7 +1,7 @@
+from os.path import join
+from re import compile, escape
from xml.etree.ElementTree import fromstring
-from re import escape, compile
-from os.path import join
from galaxy.util import in_directory
diff --git a/pulsar/web/routes.py b/pulsar/web/routes.py
index 1f529608..f460ef43 100644
--- a/pulsar/web/routes.py
+++ b/pulsar/web/routes.py
@@ -1,22 +1,24 @@
+import logging
import os
-from webob import exc
+
from json import loads
+from webob import exc
+
from pulsar.util import (
copy_to_path,
copy_to_temp,
)
from pulsar.client.job_directory import verify_is_in_directory
-from pulsar.web.framework import Controller
-from pulsar.manager_factory import DEFAULT_MANAGER_NAME
+from pulsar.client.action_mapper import path_type
from pulsar.manager_endpoint_util import (
- submit_job,
setup_job,
status_dict,
+ submit_job,
)
-from pulsar.client.action_mapper import path_type
+from pulsar.manager_factory import DEFAULT_MANAGER_NAME
+from pulsar.web.framework import Controller
-import logging
log = logging.getLogger(__name__)
--
GitLab