From ce66a09c6a3acab8d5b9f15e05b3416fbad8edb7 Mon Sep 17 00:00:00 2001 From: "Yakubov, Sergey" Date: Wed, 4 Jun 2025 11:11:15 -0400 Subject: [PATCH 1/3] use real file for locks --- lib/galaxy/authnz/managers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/authnz/managers.py b/lib/galaxy/authnz/managers.py index 98d3bc3ad4..0a08b56326 100644 --- a/lib/galaxy/authnz/managers.py +++ b/lib/galaxy/authnz/managers.py @@ -290,7 +290,7 @@ class AuthnzManager: raise exceptions.ItemAccessibilityException(msg) def refresh_expiring_oidc_tokens_for_provider(self, trans, auth): - with open("/dev/null", "w") as lock: + with open("/tmp/galaxy_refresh_lock", "w") as lock: try: fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) success, message, backend = self._get_authnz_backend(auth.provider) -- GitLab From 517c5fbce32ec8a063660f2b37be036f5d21a12c Mon Sep 17 00:00:00 2001 From: "Yakubov, Sergey" Date: Fri, 6 Jun 2025 08:45:46 -0400 Subject: [PATCH 2/3] add detailed job status for pulsar jobs --- lib/galaxy/jobs/runners/pulsar.py | 40 ++++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 11 deletions(-) diff --git a/lib/galaxy/jobs/runners/pulsar.py b/lib/galaxy/jobs/runners/pulsar.py index 3317da5f2f..8ad91dc9fc 100644 --- a/lib/galaxy/jobs/runners/pulsar.py +++ b/lib/galaxy/jobs/runners/pulsar.py @@ -9,7 +9,6 @@ import logging import os import re import subprocess -from pathlib import Path from time import sleep from typing import ( Any, @@ -35,10 +34,10 @@ from pulsar.client import ( submit_job as pulsar_submit_job, url_to_destination_params, ) - # TODO: Perform pulsar release with this included in the client package from pulsar.client.staging import DEFAULT_DYNAMIC_COLLECTION_PATTERN from sqlalchemy import select +from sqlalchemy.orm.attributes import flag_modified from galaxy import model from galaxy.job_execution.compute_environment import ( @@ -53,6 +52,7 @@ from galaxy.jobs.runners import ( JobState, ) from galaxy.model.base import check_database_connection +from galaxy.model.base import transaction from galaxy.tool_util.deps import dependencies from galaxy.util import ( galaxy_directory, @@ -193,7 +193,6 @@ PULSAR_PARAM_SPECS = dict( ), ) - PARAMETER_SPECIFICATION_REQUIRED = object() PARAMETER_SPECIFICATION_IGNORED = object() @@ -322,6 +321,21 @@ class PulsarJobRunner(AsynchronousJobRunner): job_state = self._update_job_state_for_status(job_state, status) return job_state + def _update_job_for_status_details(self, job, job_wrapper, status_details): + if not job.job_messages: + job.job_messages = [status_details] + else: + for message in job.job_messages: + if "status_details" in message: + message.update(status_details) + break + else: + job.job_messages.append(status_details) + flag_modified(job, "job_messages") + job_wrapper.sa_session.add(job) + with transaction(self.sa_session): + self.sa_session.commit() + def _update_job_state_for_status(self, job_state, pulsar_status, full_status=None): log.debug("(%s) Received status update: %s", job_state.job_id, pulsar_status) if pulsar_status in ["complete", "cancelled"]: @@ -372,7 +386,7 @@ class PulsarJobRunner(AsynchronousJobRunner): client_inputs_list = [] for input_dataset_wrapper in job_wrapper.job_io.get_input_paths( - compute_environment.materialized_objects + compute_environment.materialized_objects ): # str here to resolve false_path if set on a DatasetPath object. path = str(input_dataset_wrapper) @@ -721,7 +735,7 @@ class PulsarJobRunner(AsynchronousJobRunner): if not PulsarJobRunner.__remote_metadata(client): # we need an actual exit code file in the job working directory to detect job errors in the metadata script with open( - os.path.join(job_wrapper.working_directory, f"galaxy_{job_wrapper.job_id}.ec"), "w" + os.path.join(job_wrapper.working_directory, f"galaxy_{job_wrapper.job_id}.ec"), "w" ) as exit_code_file: exit_code_file.write(str(exit_code)) self._handle_metadata_externally(job_wrapper, resolve_requirements=True) @@ -950,12 +964,12 @@ class PulsarJobRunner(AsynchronousJobRunner): return string_as_bool_or_none(pulsar_client.destination_params.get("rewrite_parameters", False)) or False def __build_metadata_configuration( - self, - client, - job_wrapper, - remote_metadata, - remote_job_config, - compute_environment: Optional["PulsarComputeEnvironment"] = None, + self, + client, + job_wrapper, + remote_metadata, + remote_job_config, + compute_environment: Optional["PulsarComputeEnvironment"] = None, ): metadata_kwds: Dict[str, Any] = {} if remote_metadata: @@ -1028,7 +1042,11 @@ class PulsarJobRunner(AsynchronousJobRunner): galaxy_job_id = remote_job_id job, job_wrapper = self.app.job_manager.job_handler.job_queue.job_pair_for_id(galaxy_job_id) job_state = self._job_state(job, job_wrapper) + self._update_job_state_for_status(job_state, full_status["status"], full_status=full_status) + if "status_details" in full_status: + self._update_job_for_status_details(job, job_wrapper, full_status["status_details"]) + except Exception: log.exception(f"Failed to update Pulsar job status for job_id ({galaxy_job_id}/{remote_job_id})") raise -- GitLab From 83d2e2ca7f3aeaea52fabd9fcf0033f6b29a0e92 Mon Sep 17 00:00:00 2001 From: "Yakubov, Sergey" Date: Fri, 6 Jun 2025 08:46:36 -0400 Subject: [PATCH 3/3] bump version --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index dbed95c565..72cc0c838b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -9,8 +9,8 @@ variables: CONTAINER_GALAXY_URL: "${NDIP_DOCKER_REPOSITORY}/${CI_PROJECT_PATH}" CONTAINER_GALAXY_BASE_URL: "${CONTAINER_GALAXY_URL}/base" CONTAINER_GALAXY_COMMIT_URL: "${CONTAINER_GALAXY_URL}/commit" - GALAXY_VERSION_PYTHON: 24.2.dev1+ornl - GALAXY_VERSION_DOCKER: 24.2.dev1.ornl + GALAXY_VERSION_PYTHON: 24.2.dev2+ornl + GALAXY_VERSION_DOCKER: 24.2.dev2.ornl # This import is for the func_rse_docker_* functions before_script: -- GitLab