More bug fixes discovered through Python 3 testing.

Also improved unicode handling and testing.

lwr/framework.py

@@ -146,6 +146,7 @@ class Controller(object):
             resp = self.__build_response(result)
 
             return resp(environ, start_response)
+
         controller_replacement.func = func
         controller_replacement.response_type = self.response_type
         controller_replacement.body = self.body

lwr/lwr_client/manager.py

@@ -123,6 +123,7 @@ class LocalJobManagerInterface(object):
         }
 
     def execute(self, command, args={}, data=None, input_path=None, output_path=None):
+        # If data set, should be unicode (on Python 2) or str (on Python 3).
         from lwr import routes
         from lwr.framework import build_func_args
         controller = getattr(routes, command)
@@ -139,7 +140,7 @@ class LocalJobManagerInterface(object):
 
     def __build_body(self, data, input_path):
         if data is not None:
-            return BytesIO(data)
+            return BytesIO(data.encode('utf-8'))
         elif input_path is not None:
             return open(input_path, 'rb')
         else:

lwr/lwr_client/stager.py

 from os.path import abspath, basename, join, exists
 from os import listdir, sep
 from re import findall
+from io import open
 
 from .action_mapper import FileActionMapper
 
@@ -24,22 +25,24 @@ class JobInputs(object):
     >>> import tempfile
     >>> tf = tempfile.NamedTemporaryFile()
     >>> def setup_inputs(tf):
-    ...     open(tf.name, "w").write("world /path/to/input the rest")
-    ...     inputs = JobInputs("hello /path/to/input", [tf.name])
+    ...     open(tf.name, "w").write(u"world /path/to/input the rest")
+    ...     inputs = JobInputs(u"hello /path/to/input", [tf.name])
     ...     return inputs
     >>> inputs = setup_inputs(tf)
-    >>> inputs.rewrite_paths("/path/to/input", 'C:\\input')
-    >>> inputs.rewritten_command_line
-    'hello C:\\\\input'
-    >>> inputs.rewritten_config_files[tf.name]
-    'world C:\\\\input the rest'
+    >>> inputs.rewrite_paths(u"/path/to/input", u'C:\\input')
+    >>> inputs.rewritten_command_line == u'hello C:\\\\input'
+    True
+    >>> inputs.rewritten_config_files[tf.name] == u'world C:\\\\input the rest'
+    True
     >>> tf.close()
     >>> tf = tempfile.NamedTemporaryFile()
     >>> inputs = setup_inputs(tf)
-    >>> inputs.find_referenced_subfiles('/path/to')
-    ['/path/to/input']
+    >>> inputs.find_referenced_subfiles('/path/to') == [u'/path/to/input']
+    True
     >>> inputs.path_referenced('/path/to')
     True
+    >>> inputs.path_referenced(u'/path/to') 
+    True
     >>> inputs.path_referenced('/path/to/input')
     True
     >>> inputs.path_referenced('/path/to/notinput')
@@ -340,9 +343,9 @@ def submit_job(client, tool, command_line, config_files, input_files, output_fil
 def _read(path):
     """
     Utility method to quickly read small files (config files and tool
-    wrappers) into memory as strings.
+    wrappers) into memory as bytes.
     """
-    input = open(path, "r")
+    input = open(path, "r", encoding="utf-8")
     try:
         return input.read()
     finally:

lwr/managers/queued.py

@@ -3,7 +3,6 @@ try:
     from Queue import Queue
 except ImportError:
     from queue import Queue
-import sys
 import threading
 import traceback
 
@@ -79,7 +78,7 @@ class QueueManager(Manager):
                 self._run(job_id, command_line, async=False)
             except:
                 log.warn("Uncaught exception running job with job_id %s" % job_id)
-                traceback.print_exc(file=sys.stdout)
+                traceback.print_exc()
 
 
 class PersistedJobStore(JobMetadataStore):

lwr/persistence.py

 import shelve
 from threading import Lock
 import traceback
-import sys
 
 
 class PersistenceStore(object):
@@ -31,7 +30,7 @@ class PersistenceStore(object):
                 try:
                     return func()
                 except:
-                    traceback.print_exc(file=sys.stdout)
+                    traceback.print_exc()
                     if not suppress_exception:
                         raise
 

test/check.py

@@ -49,8 +49,8 @@ def run(options):
         __write_to_file(temp_config_path, EXPECTED_OUTPUT)
         __write_to_file(temp_tool_path, TEST_SCRIPT)
 
-        empty_input = "/foo/bar/x"
-        command_line = 'python %s "%s" "%s" "%s" "%s"' % (temp_tool_path, temp_config_path, temp_input_path, temp_output_path, empty_input)
+        empty_input = u"/foo/bar/x"
+        command_line = u'python %s "%s" "%s" "%s" "%s"' % (temp_tool_path, temp_config_path, temp_input_path, temp_output_path, empty_input)
         config_files = [temp_config_path]
         input_files = [temp_input_path, empty_input]
         output_files = [temp_output_path]
@@ -69,10 +69,10 @@ def run(options):
         __check_output(temp_output_path)
 
         __exercise_errors(options, client, temp_output_path, temp_directory)
-    except BaseException as e:
+    except BaseException:
         if not options.suppress_output:
-            traceback.print_exc(e)
-        raise e
+            traceback.print_exc()
+        raise
     finally:
         shutil.rmtree(temp_directory)
 
@@ -81,7 +81,7 @@ def __check_output(temp_output_path):
     """
     Verified the correct outputs were written (i.e. job ran properly).
     """
-    output_file = open(temp_output_path, 'r')
+    output_file = open(temp_output_path, 'rb')
     try:
         output_contents = output_file.read()
         assert output_contents == EXPECTED_OUTPUT, "Invalid output_contents: %s" % output_contents
@@ -98,9 +98,9 @@ def __exercise_errors(options, client, temp_output_path, temp_directory):
     if getattr(options, 'test_errors', False):
         try:
             client.fetch_output(temp_output_path + "x", temp_directory)
-        except BaseException as e:
+        except BaseException:
             if not options.suppress_output:
-                traceback.print_exc(e)
+                traceback.print_exc()
 
 
 def __client(options):

test/integration_test.py

@@ -66,10 +66,10 @@ class IntegrationTests(BaseIntegrationTest):
         self._run(private_token=None, transport="curl", **self.default_kwargs)
 
     def test_integration_token(self):
-        self._run(app_conf={"private_key": "testtoken"}, private_token="testtoken", transport="curl", **self.default_kwargs)
+        self._run(app_conf={"private_key": "testtoken"}, private_token="testtoken", **self.default_kwargs)
 
     def test_integration_errors(self):
-        self._run(app_conf={"private_key": "testtoken"}, private_token="testtoken", transport="curl", test_errors=True, **self.default_kwargs)
+        self._run(app_conf={"private_key": "testtoken"}, private_token="testtoken", test_errors=True, **self.default_kwargs)
 
     @skipUnlessModule("drmaa")
     def test_integration_drmaa(self):