From be363295d9b601826a5026161c63822b0768b716 Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Wed, 28 Sep 2022 09:34:03 -0400
Subject: [PATCH] Fix Pulsar for exit code handling changes in
https://github.com/galaxyproject/galaxy/pull/13557/files
---
pulsar/managers/base/__init__.py | 7 ++++++-
pulsar/managers/base/directory.py | 16 +++++++++++++---
pulsar/managers/queued_cli.py | 4 +++-
pulsar/managers/unqueued.py | 6 ++++--
pulsar/managers/util/job_script/__init__.py | 2 +-
5 files changed, 27 insertions(+), 8 deletions(-)
diff --git a/pulsar/managers/base/__init__.py b/pulsar/managers/base/__init__.py
index 92670473..2bbe0e8e 100644
--- a/pulsar/managers/base/__init__.py
+++ b/pulsar/managers/base/__init__.py
@@ -6,6 +6,7 @@ import errno
import json
import logging
import os
+import platform
from os import (
curdir,
getenv,
@@ -78,6 +79,10 @@ class BaseManager(ManagerInterface):
self.job_metrics = app.job_metrics
self.object_store = app.object_store
+ @property
+ def _is_windows(self) -> bool:
+ return platform.system().lower() == "windows"
+
def clean(self, job_id):
if self.debug:
# In debug mode skip cleaning job directories.
@@ -193,7 +198,7 @@ class BaseManager(ManagerInterface):
else:
return listdir(directory_or_none)
- def _expand_command_line(self, command_line, dependencies_description, job_directory=None):
+ def _expand_command_line(self, job_id, command_line: str, dependencies_description, job_directory=None) -> str:
if dependencies_description is None:
return command_line
diff --git a/pulsar/managers/base/directory.py b/pulsar/managers/base/directory.py
index 9390758a..98e133e5 100644
--- a/pulsar/managers/base/directory.py
+++ b/pulsar/managers/base/directory.py
@@ -117,9 +117,21 @@ class DirectoryBaseManager(BaseManager):
tool_id = job_directory.load_metadata(JOB_FILE_TOOL_ID)
return tool_id
+ def _expand_command_line(self, job_id, command_line: str, dependencies_description, job_directory=None) -> str:
+ command_line = super()._expand_command_line(
+ job_id, command_line, dependencies_description, job_directory=job_directory
+ )
+ if not self._is_windows:
+ rc_path = self._return_code_path(job_id)
+ CAPTURE_RETURN_CODE = "return_code=$?"
+ command_line = f"{command_line}; {CAPTURE_RETURN_CODE}; echo $return_code > {rc_path};"
+ return command_line
+
# Helpers methods related to setting up job script files.
def _setup_job_file(self, job_id, command_line, dependencies_description=None, env=[], setup_params=None):
- command_line = self._expand_command_line(command_line, dependencies_description, job_directory=self.job_directory(job_id).job_directory)
+ command_line = self._expand_command_line(
+ job_id, command_line, dependencies_description, job_directory=self.job_directory(job_id).job_directory
+ )
script_env = self._job_template_env(job_id, command_line=command_line, env=env, setup_params=setup_params)
script = job_script(**script_env)
return self._write_job_script(job_id, script)
@@ -138,7 +150,6 @@ class DirectoryBaseManager(BaseManager):
return tmp_dir
def _job_template_env(self, job_id, command_line=None, env=[], setup_params=None):
- return_code_path = self._return_code_path(job_id)
# TODO: Add option to ignore remote env.
env = env + self.env_vars
setup_params = setup_params or {}
@@ -149,7 +160,6 @@ class DirectoryBaseManager(BaseManager):
'galaxy_lib': self._galaxy_lib(),
'preserve_python_environment': setup_params.get('preserve_galaxy_python_environment', False),
'env_setup_commands': env_setup_commands,
- 'exit_code_path': return_code_path,
'job_directory': self.job_directory(job_id).job_directory,
'working_directory': self.job_directory(job_id).working_directory(),
'metadata_directory': self.job_directory(job_id).metadata_directory(),
diff --git a/pulsar/managers/queued_cli.py b/pulsar/managers/queued_cli.py
index 5216e7ef..bc24dd2f 100644
--- a/pulsar/managers/queued_cli.py
+++ b/pulsar/managers/queued_cli.py
@@ -30,7 +30,9 @@ class CliQueueManager(ExternalBaseManager):
stdout_path = self._stdout_path(job_id)
stderr_path = self._stderr_path(job_id)
job_name = self._job_name(job_id)
- command_line = self._expand_command_line(command_line, dependencies_description, job_directory=self.job_directory(job_id).job_directory)
+ command_line = self._expand_command_line(
+ job_id, command_line, dependencies_description, job_directory=self.job_directory(job_id).job_directory
+ )
job_script_kwargs = self._job_template_env(
job_id,
command_line=command_line,
diff --git a/pulsar/managers/unqueued.py b/pulsar/managers/unqueued.py
index 75e282e7..9a791148 100644
--- a/pulsar/managers/unqueued.py
+++ b/pulsar/managers/unqueued.py
@@ -60,10 +60,12 @@ class BaseUnqueuedManager(DirectoryBaseManager):
def _prepare_run(self, job_id, command_line, dependencies_description, env, setup_params=None):
self._check_execution_with_tool_file(job_id, command_line)
self._record_submission(job_id)
- if platform.system().lower() == "windows":
+ if self._is_windows:
# TODO: Don't ignore requirements and env without warning. Ideally
# process them or at least warn about them being ignored.
- command_line = self._expand_command_line(command_line, dependencies_description, job_directory=self.job_directory(job_id).job_directory)
+ command_line = self._expand_command_line(
+ job_id, command_line, dependencies_description, job_directory=self.job_directory(job_id).job_directory
+ )
else:
command_line = self._setup_job_file(
job_id,
diff --git a/pulsar/managers/util/job_script/__init__.py b/pulsar/managers/util/job_script/__init__.py
index e89eb285..3d662811 100644
--- a/pulsar/managers/util/job_script/__init__.py
+++ b/pulsar/managers/util/job_script/__init__.py
@@ -42,7 +42,7 @@ INTEGRITY_SYNC_COMMAND = "/bin/sync"
DEFAULT_INTEGRITY_CHECK = True
DEFAULT_INTEGRITY_COUNT = 35
DEFAULT_INTEGRITY_SLEEP = 0.25
-REQUIRED_TEMPLATE_PARAMS = ["working_directory", "command", "exit_code_path"]
+REQUIRED_TEMPLATE_PARAMS = ["working_directory", "command"]
OPTIONAL_TEMPLATE_PARAMS: Dict[str, Any] = {
"galaxy_lib": None,
"galaxy_virtual_env": None,
--
GitLab