From fe2cd68443339af2bfba32c4cb73ccb8b25cf0d3 Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Fri, 10 May 2019 17:06:43 -0400
Subject: [PATCH] ActionMapperRefactor: path -> source dict.
---
pulsar/client/action_mapper.py | 23 ++++++++++++-----------
pulsar/client/path_mapper.py | 4 ++--
pulsar/client/staging/down.py | 2 +-
pulsar/client/staging/up.py | 4 ++--
test/action_mapper_test.py | 6 +++---
test/path_mapper_test.py | 4 ++--
6 files changed, 22 insertions(+), 21 deletions(-)
diff --git a/pulsar/client/action_mapper.py b/pulsar/client/action_mapper.py
index 8f1be3c0..b940ea8f 100644
--- a/pulsar/client/action_mapper.py
+++ b/pulsar/client/action_mapper.py
@@ -104,46 +104,46 @@ class FileActionMapper(object):
... return mapper
>>> mapper = mapper_for(default_action='none', config_contents=json_string)
>>> # Test first config line above, implicit path prefix mapper
- >>> action = mapper.action('/opt/galaxy/tools/filters/catWrapper.py', 'input')
+ >>> action = mapper.action({'path': '/opt/galaxy/tools/filters/catWrapper.py'}, 'input')
>>> action.action_type == u'none'
True
>>> action.staging_needed
False
>>> # Test another (2nd) mapper, this one with a different action
- >>> action = mapper.action('/galaxy/data/files/000/dataset_1.dat', 'input')
+ >>> action = mapper.action({'path': '/galaxy/data/files/000/dataset_1.dat'}, 'input')
>>> action.action_type == u'transfer'
True
>>> action.staging_needed
True
>>> # Always at least copy work_dir outputs.
- >>> action = mapper.action('/opt/galaxy/database/working_directory/45.sh', 'workdir')
+ >>> action = mapper.action({'path': '/opt/galaxy/database/working_directory/45.sh'}, 'workdir')
>>> action.action_type == u'copy'
True
>>> action.staging_needed
True
>>> # Test glob mapper (matching test)
- >>> mapper.action('/cool/bamfiles/projectABC/study1/patient3.bam', 'input').action_type == u'copy'
+ >>> mapper.action({'path': '/cool/bamfiles/projectABC/study1/patient3.bam'}, 'input').action_type == u'copy'
True
>>> # Test glob mapper (non-matching test)
- >>> mapper.action('/cool/bamfiles/projectABC/study1/patient3.bam.bai', 'input').action_type == u'none'
+ >>> mapper.action({'path': '/cool/bamfiles/projectABC/study1/patient3.bam.bai'}, 'input').action_type == u'none'
True
>>> # Regex mapper test.
- >>> mapper.action('/old/galaxy/data/dataset_10245.dat', 'input').action_type == u'copy'
+ >>> mapper.action({'path': '/old/galaxy/data/dataset_10245.dat'}, 'input').action_type == u'copy'
True
>>> # Doesn't map unstructured paths by default
- >>> mapper.action('/old/galaxy/data/dataset_10245.dat', 'unstructured').action_type == u'none'
+ >>> mapper.action({'path': '/old/galaxy/data/dataset_10245.dat'}, 'unstructured').action_type == u'none'
True
>>> input_only_mapper = mapper_for(default_action="none", config_contents=r'''{"paths": [ \
{"path": "/", "action": "transfer", "path_types": "input"} \
] }''')
- >>> input_only_mapper.action('/dataset_1.dat', 'input').action_type == u'transfer'
+ >>> input_only_mapper.action({'path': '/dataset_1.dat'}, 'input').action_type == u'transfer'
True
- >>> input_only_mapper.action('/dataset_1.dat', 'output').action_type == u'none'
+ >>> input_only_mapper.action({'path': '/dataset_1.dat'}, 'output').action_type == u'none'
True
>>> unstructured_mapper = mapper_for(default_action="none", config_contents=r'''{"paths": [ \
{"path": "/", "action": "transfer", "path_types": "*any*"} \
] }''')
- >>> unstructured_mapper.action('/old/galaxy/data/dataset_10245.dat', 'unstructured').action_type == u'transfer'
+ >>> unstructured_mapper.action({'path': '/old/galaxy/data/dataset_10245.dat'}, 'unstructured').action_type == u'transfer'
True
"""
@@ -161,7 +161,8 @@ class FileActionMapper(object):
self.mappers = mappers_from_dicts(config.get("paths", []))
self.files_endpoint = config.get("files_endpoint", None)
- def action(self, path, type, mapper=None):
+ def action(self, source, type, mapper=None):
+ path = source["path"]
mapper = self.__find_mapper(path, type, mapper)
action_class = self.__action_class(path, type, mapper)
file_lister = DEFAULT_FILE_LISTER
diff --git a/pulsar/client/path_mapper.py b/pulsar/client/path_mapper.py
index b7478103..eb7ed09d 100644
--- a/pulsar/client/path_mapper.py
+++ b/pulsar/client/path_mapper.py
@@ -60,7 +60,7 @@ class PathMapper(object):
def check_for_arbitrary_rewrite(self, local_path):
path = str(local_path) # Use false_path if needed.
- action = self.action_mapper.action(path, path_type.UNSTRUCTURED)
+ action = self.action_mapper.action({"path": path}, path_type.UNSTRUCTURED)
if not action.staging_needed:
return action.path_rewrite(self.path_helper), []
unique_names = action.unstructured_map()
@@ -72,7 +72,7 @@ class PathMapper(object):
""" Return remote path of this file (if staging is required) else None.
"""
path = str(dataset_path) # Use false_path if needed.
- action = self.action_mapper.action(path, dataset_path_type)
+ action = self.action_mapper.action({"path": path}, dataset_path_type)
if action.staging_needed:
if name is None:
name = os.path.basename(path)
diff --git a/pulsar/client/staging/down.py b/pulsar/client/staging/down.py
index d6aad82b..8bd94001 100644
--- a/pulsar/client/staging/down.py
+++ b/pulsar/client/staging/down.py
@@ -135,7 +135,7 @@ class ResultsCollector(object):
# path.
collected = False
with self.exception_tracker():
- action = self.action_mapper.action(path, output_type)
+ action = self.action_mapper.action({"path": path}, output_type)
if self._collect_output(output_type, action, name):
collected = True
diff --git a/pulsar/client/staging/up.py b/pulsar/client/staging/up.py
index f88d29b4..119fe719 100644
--- a/pulsar/client/staging/up.py
+++ b/pulsar/client/staging/up.py
@@ -197,7 +197,7 @@ class FileStager(object):
if path not in referenced_arbitrary_path_mappers:
referenced_arbitrary_path_mappers[path] = mapper
for path, mapper in referenced_arbitrary_path_mappers.items():
- action = self.action_mapper.action(path, path_type.UNSTRUCTURED, mapper)
+ action = self.action_mapper.action({"path": path}, path_type.UNSTRUCTURED, mapper)
unstructured_map = action.unstructured_map(self.path_helper)
self.arbitrary_files.update(unstructured_map)
@@ -501,7 +501,7 @@ class TransferTracker(object):
self.job_inputs.rewrite_paths(local_path, remote_path)
def __action(self, path, type):
- return self.action_mapper.action(path, type)
+ return self.action_mapper.action({"path": path}, type)
def _read(path):
diff --git a/test/action_mapper_test.py b/test/action_mapper_test.py
index 696f87f7..2131cb65 100644
--- a/test/action_mapper_test.py
+++ b/test/action_mapper_test.py
@@ -9,7 +9,7 @@ def test_endpoint_validation():
mapper = FileActionMapper(client)
exception_found = False
try:
- mapper.action('/opt/galaxy/tools/filters/catWrapper.py', 'input')
+ mapper.action({'path': '/opt/galaxy/tools/filters/catWrapper.py'}, 'input')
except Exception as e:
exception_found = True
assert "files_endpoint" in str(e)
@@ -21,7 +21,7 @@ def test_ssh_key_validation():
mapper = FileActionMapper(client)
exception_found = False
try:
- mapper.action('/opt/galaxy/tools/filters/catWrapper.py', 'input')
+ mapper.action({'path': '/opt/galaxy/tools/filters/catWrapper.py'}, 'input')
except Exception as e:
exception_found = True
assert "ssh_key" in str(e)
@@ -31,7 +31,7 @@ def test_ssh_key_validation():
def test_ssh_key_defaults():
client = _client("remote_rsync_transfer")
mapper = FileActionMapper(client)
- action = mapper.action('/opt/galaxy/tools/filters/catWrapper.py', 'input')
+ action = mapper.action({'path': '/opt/galaxy/tools/filters/catWrapper.py'}, 'input')
action.to_dict()
diff --git a/test/path_mapper_test.py b/test/path_mapper_test.py
index 58e5fb86..e668f585 100644
--- a/test/path_mapper_test.py
+++ b/test/path_mapper_test.py
@@ -75,7 +75,7 @@ class TestActionMapper(object):
if not staging_needed:
self._action.path_rewrite = lambda path: None
- def action(self, path, type):
- assert self.expected_path == path
+ def action(self, source, type):
+ assert self.expected_path == source["path"]
assert self.expected_type == type
return self._action
--
GitLab