From c46010eb898bb59a842377ff2864f545a2bba558 Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Fri, 29 Nov 2013 09:10:05 -0600
Subject: [PATCH] More bug fixes discovered through Python 3 testing.
Also improved unicode handling and testing.
---
lwr/framework.py | 1 +
lwr/lwr_client/manager.py | 3 ++-
lwr/lwr_client/stager.py | 25 ++++++++++++++-----------
lwr/managers/queued.py | 3 +--
lwr/persistence.py | 3 +--
test/check.py | 16 ++++++++--------
test/integration_test.py | 4 ++--
7 files changed, 29 insertions(+), 26 deletions(-)
diff --git a/lwr/framework.py b/lwr/framework.py
index 37ae5cef..17a72db7 100644
--- a/lwr/framework.py
+++ b/lwr/framework.py
@@ -146,6 +146,7 @@ class Controller(object):
resp = self.__build_response(result)
return resp(environ, start_response)
+
controller_replacement.func = func
controller_replacement.response_type = self.response_type
controller_replacement.body = self.body
diff --git a/lwr/lwr_client/manager.py b/lwr/lwr_client/manager.py
index 2f134d11..d077e8f7 100644
--- a/lwr/lwr_client/manager.py
+++ b/lwr/lwr_client/manager.py
@@ -123,6 +123,7 @@ class LocalJobManagerInterface(object):
}
def execute(self, command, args={}, data=None, input_path=None, output_path=None):
+ # If data set, should be unicode (on Python 2) or str (on Python 3).
from lwr import routes
from lwr.framework import build_func_args
controller = getattr(routes, command)
@@ -139,7 +140,7 @@ class LocalJobManagerInterface(object):
def __build_body(self, data, input_path):
if data is not None:
- return BytesIO(data)
+ return BytesIO(data.encode('utf-8'))
elif input_path is not None:
return open(input_path, 'rb')
else:
diff --git a/lwr/lwr_client/stager.py b/lwr/lwr_client/stager.py
index 83e244ae..1c00ec29 100644
--- a/lwr/lwr_client/stager.py
+++ b/lwr/lwr_client/stager.py
@@ -1,6 +1,7 @@
from os.path import abspath, basename, join, exists
from os import listdir, sep
from re import findall
+from io import open
from .action_mapper import FileActionMapper
@@ -24,22 +25,24 @@ class JobInputs(object):
>>> import tempfile
>>> tf = tempfile.NamedTemporaryFile()
>>> def setup_inputs(tf):
- ... open(tf.name, "w").write("world /path/to/input the rest")
- ... inputs = JobInputs("hello /path/to/input", [tf.name])
+ ... open(tf.name, "w").write(u"world /path/to/input the rest")
+ ... inputs = JobInputs(u"hello /path/to/input", [tf.name])
... return inputs
>>> inputs = setup_inputs(tf)
- >>> inputs.rewrite_paths("/path/to/input", 'C:\\input')
- >>> inputs.rewritten_command_line
- 'hello C:\\\\input'
- >>> inputs.rewritten_config_files[tf.name]
- 'world C:\\\\input the rest'
+ >>> inputs.rewrite_paths(u"/path/to/input", u'C:\\input')
+ >>> inputs.rewritten_command_line == u'hello C:\\\\input'
+ True
+ >>> inputs.rewritten_config_files[tf.name] == u'world C:\\\\input the rest'
+ True
>>> tf.close()
>>> tf = tempfile.NamedTemporaryFile()
>>> inputs = setup_inputs(tf)
- >>> inputs.find_referenced_subfiles('/path/to')
- ['/path/to/input']
+ >>> inputs.find_referenced_subfiles('/path/to') == [u'/path/to/input']
+ True
>>> inputs.path_referenced('/path/to')
True
+ >>> inputs.path_referenced(u'/path/to')
+ True
>>> inputs.path_referenced('/path/to/input')
True
>>> inputs.path_referenced('/path/to/notinput')
@@ -340,9 +343,9 @@ def submit_job(client, tool, command_line, config_files, input_files, output_fil
def _read(path):
"""
Utility method to quickly read small files (config files and tool
- wrappers) into memory as strings.
+ wrappers) into memory as bytes.
"""
- input = open(path, "r")
+ input = open(path, "r", encoding="utf-8")
try:
return input.read()
finally:
diff --git a/lwr/managers/queued.py b/lwr/managers/queued.py
index c09ccffd..d926dc0c 100644
--- a/lwr/managers/queued.py
+++ b/lwr/managers/queued.py
@@ -3,7 +3,6 @@ try:
from Queue import Queue
except ImportError:
from queue import Queue
-import sys
import threading
import traceback
@@ -79,7 +78,7 @@ class QueueManager(Manager):
self._run(job_id, command_line, async=False)
except:
log.warn("Uncaught exception running job with job_id %s" % job_id)
- traceback.print_exc(file=sys.stdout)
+ traceback.print_exc()
class PersistedJobStore(JobMetadataStore):
diff --git a/lwr/persistence.py b/lwr/persistence.py
index 14b7d8de..3f24c545 100644
--- a/lwr/persistence.py
+++ b/lwr/persistence.py
@@ -1,7 +1,6 @@
import shelve
from threading import Lock
import traceback
-import sys
class PersistenceStore(object):
@@ -31,7 +30,7 @@ class PersistenceStore(object):
try:
return func()
except:
- traceback.print_exc(file=sys.stdout)
+ traceback.print_exc()
if not suppress_exception:
raise
diff --git a/test/check.py b/test/check.py
index 3ec1e2f2..9cc282c1 100644
--- a/test/check.py
+++ b/test/check.py
@@ -49,8 +49,8 @@ def run(options):
__write_to_file(temp_config_path, EXPECTED_OUTPUT)
__write_to_file(temp_tool_path, TEST_SCRIPT)
- empty_input = "/foo/bar/x"
- command_line = 'python %s "%s" "%s" "%s" "%s"' % (temp_tool_path, temp_config_path, temp_input_path, temp_output_path, empty_input)
+ empty_input = u"/foo/bar/x"
+ command_line = u'python %s "%s" "%s" "%s" "%s"' % (temp_tool_path, temp_config_path, temp_input_path, temp_output_path, empty_input)
config_files = [temp_config_path]
input_files = [temp_input_path, empty_input]
output_files = [temp_output_path]
@@ -69,10 +69,10 @@ def run(options):
__check_output(temp_output_path)
__exercise_errors(options, client, temp_output_path, temp_directory)
- except BaseException as e:
+ except BaseException:
if not options.suppress_output:
- traceback.print_exc(e)
- raise e
+ traceback.print_exc()
+ raise
finally:
shutil.rmtree(temp_directory)
@@ -81,7 +81,7 @@ def __check_output(temp_output_path):
"""
Verified the correct outputs were written (i.e. job ran properly).
"""
- output_file = open(temp_output_path, 'r')
+ output_file = open(temp_output_path, 'rb')
try:
output_contents = output_file.read()
assert output_contents == EXPECTED_OUTPUT, "Invalid output_contents: %s" % output_contents
@@ -98,9 +98,9 @@ def __exercise_errors(options, client, temp_output_path, temp_directory):
if getattr(options, 'test_errors', False):
try:
client.fetch_output(temp_output_path + "x", temp_directory)
- except BaseException as e:
+ except BaseException:
if not options.suppress_output:
- traceback.print_exc(e)
+ traceback.print_exc()
def __client(options):
diff --git a/test/integration_test.py b/test/integration_test.py
index cd64d54d..b1e83b10 100644
--- a/test/integration_test.py
+++ b/test/integration_test.py
@@ -66,10 +66,10 @@ class IntegrationTests(BaseIntegrationTest):
self._run(private_token=None, transport="curl", **self.default_kwargs)
def test_integration_token(self):
- self._run(app_conf={"private_key": "testtoken"}, private_token="testtoken", transport="curl", **self.default_kwargs)
+ self._run(app_conf={"private_key": "testtoken"}, private_token="testtoken", **self.default_kwargs)
def test_integration_errors(self):
- self._run(app_conf={"private_key": "testtoken"}, private_token="testtoken", transport="curl", test_errors=True, **self.default_kwargs)
+ self._run(app_conf={"private_key": "testtoken"}, private_token="testtoken", test_errors=True, **self.default_kwargs)
@skipUnlessModule("drmaa")
def test_integration_drmaa(self):
--
GitLab