From ac6fbdf0f93cd88090b0cc2709d2ed6a45ed826c Mon Sep 17 00:00:00 2001 From: Gregory Cage Date: Thu, 1 May 2025 15:53:59 -0400 Subject: [PATCH 1/5] Raise exception if exit code is not 0 in container monitoring port parsing --- lib/galaxy/job_execution/container_monitor.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/job_execution/container_monitor.py b/lib/galaxy/job_execution/container_monitor.py index 0590ce715d..307b254705 100644 --- a/lib/galaxy/job_execution/container_monitor.py +++ b/lib/galaxy/job_execution/container_monitor.py @@ -30,7 +30,8 @@ def parse_ports(container_name, connection_configuration): stdout_file.seek(0) ports_raw = stdout_file.read().decode("utf-8") return ports_raw - + else: + raise Exception("Could not find port.") def get_ip_command(cmd) -> str: return subprocess.check_output(shlex.split(cmd), text=True).strip() -- GitLab From e4e36e220528de5e7490eab1234650e483ae15a7 Mon Sep 17 00:00:00 2001 From: "Yakubov, Sergey" Date: Fri, 2 May 2025 08:33:00 -0400 Subject: [PATCH 2/5] fix user groups in docker run --- lib/galaxy/tool_util/deps/container_classes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/tool_util/deps/container_classes.py b/lib/galaxy/tool_util/deps/container_classes.py index 4a4f9551ba..2391d7fc7b 100644 --- a/lib/galaxy/tool_util/deps/container_classes.py +++ b/lib/galaxy/tool_util/deps/container_classes.py @@ -501,7 +501,7 @@ class DockerContainer(Container, HasDockerLikeVolumes): auto_rm=asbool(self.prop("auto_rm", docker_util.DEFAULT_AUTO_REMOVE)), set_user=self.prop("set_user", docker_util.DEFAULT_SET_USER), set_user_from_host=host_user, - run_extra_arguments=self.prop("run_extra_arguments", docker_util.DEFAULT_RUN_EXTRA_ARGUMENTS), + run_extra_arguments=run_extra_arguments, guest_ports=self.tool_info.guest_ports, container_name=self.container_name, **docker_host_props, -- GitLab From 8f5c7a6d1602ce4f0108939075158d9853568a3e Mon Sep 17 00:00:00 2001 From: Gregory Cage Date: Thu, 8 May 2025 19:40:51 -0400 Subject: [PATCH 3/5] Increase timeout for container monitor --- lib/galaxy/job_execution/container_monitor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/job_execution/container_monitor.py b/lib/galaxy/job_execution/container_monitor.py index 307b254705..95d59b4d15 100644 --- a/lib/galaxy/job_execution/container_monitor.py +++ b/lib/galaxy/job_execution/container_monitor.py @@ -65,7 +65,7 @@ def main(): ports_raw = None exc_traceback = "" - for i in range(10): + for i in range(60): try: ports_raw = parse_ports(container_name, connection_configuration) if ports_raw is not None: -- GitLab From 03b9a67ecfbb1130effbacca02411e02e70ffa70 Mon Sep 17 00:00:00 2001 From: "Yakubov, Sergey" Date: Thu, 15 May 2025 11:47:32 -0400 Subject: [PATCH 4/5] add an option to disable checksum by file ingest --- .gitlab-ci.yml | 4 +- lib/galaxy/objectstore/rucio.py | 6 ++- lib/galaxy/objectstore/rucio_extra_clients.py | 38 ++++++++++++++++++- 3 files changed, 44 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e39e58df9b..956153143a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -9,8 +9,8 @@ variables: CONTAINER_GALAXY_URL: "${NDIP_DOCKER_REPOSITORY}/${CI_PROJECT_PATH}" CONTAINER_GALAXY_BASE_URL: "${CONTAINER_GALAXY_URL}/base" CONTAINER_GALAXY_COMMIT_URL: "${CONTAINER_GALAXY_URL}/commit" - GALAXY_VERSION_PYTHON: 24.2.dev0+ornl - GALAXY_VERSION_DOCKER: 24.2.dev0.ornl + GALAXY_VERSION_PYTHON: 24.2.dev1+ornl + GALAXY_VERSION_DOCKER: 24.2.dev1.ornl # This import is for the func_rse_docker_* functions before_script: diff --git a/lib/galaxy/objectstore/rucio.py b/lib/galaxy/objectstore/rucio.py index 97d1bd9f8e..9e6763573b 100644 --- a/lib/galaxy/objectstore/rucio.py +++ b/lib/galaxy/objectstore/rucio.py @@ -93,11 +93,13 @@ def parse_config_xml(config_xml): rucio_upload_scheme = e_xml[0].get("scheme", None) rucio_scope = e_xml[0].get("scope", None) rucio_register_only = string_as_bool(e_xml[0].get("register_only", "False")) + rucio_register_with_checksum = string_as_bool(e_xml[0].get("rucio_register_with_checksum", "True")) else: rucio_upload_rse_name = None rucio_upload_scheme = None rucio_scope = None rucio_register_only = False + rucio_register_with_checksum = True e_xml = config_xml.findall("rucio_auth") if not e_xml: @@ -118,6 +120,7 @@ def parse_config_xml(config_xml): "upload_scheme": rucio_upload_scheme, "scope": rucio_scope, "register_only": rucio_register_only, + "rucio_register_with_checksum": rucio_register_with_checksum, "download_schemes": rucio_download_schemes, "account": rucio_account, "auth_host": rucio_auth_host, @@ -148,6 +151,7 @@ class RucioBroker: self.upload_rse_name = rucio_config["upload_rse_name"] self.scope = rucio_config["scope"] self.register_only = rucio_config["register_only"] + self.register_with_checksum = rucio_config.get("register_with_checksum", True) self.download_schemes = rucio_config["download_schemes"] if Client is None: raise Exception(NO_RUCIO_ERROR_MESSAGE) @@ -177,7 +181,7 @@ class RucioBroker: def get_rucio_ingest_client(self, auth_token=None): client = self.get_rucio_client() - ic = InPlaceIngestClient(_client=client) + ic = InPlaceIngestClient(client, self.register_with_checksum) ic.auth_token = auth_token return ic diff --git a/lib/galaxy/objectstore/rucio_extra_clients.py b/lib/galaxy/objectstore/rucio_extra_clients.py index 500366ee36..51fa996590 100644 --- a/lib/galaxy/objectstore/rucio_extra_clients.py +++ b/lib/galaxy/objectstore/rucio_extra_clients.py @@ -1,5 +1,6 @@ import copy import logging +import os import time try: @@ -10,7 +11,7 @@ try: NotAllFilesUploaded, RSEWriteBlocked, ) - from rucio.common.utils import generate_uuid + from rucio.common.utils import generate_uuid, adler32, md5 from rucio.rse import rsemanager as rsemgr except ImportError: UploadClient = object @@ -57,6 +58,9 @@ class DeleteClient(UploadClient): class InPlaceIngestClient(UploadClient): + def __init__(self, client, register_with_checksum): + super().__init__(client) + self.register_with_checksum = register_with_checksum def ingest(self, items, summary_file_path=None, traces_copy_out=None, ignore_availability=False, activity=None): """ :param items: List of dictionaries. Each dictionary describing a file to upload. Keys: @@ -195,3 +199,35 @@ class InPlaceIngestClient(UploadClient): elif num_succeeded != len(files): raise NotAllFilesUploaded() return 0 + def _collect_file_info(self, filepath, item): + """ + Collects infos (e.g. size, checksums, etc.) about the file and + returns them as a dictionary + (This function is meant to be used as class internal only) + + :param filepath: path where the file is stored + :param item: input options for the given file + + :returns: a dictionary containing all collected info and the input options + """ + new_item = copy.deepcopy(item) + new_item['path'] = filepath + new_item['dirname'] = os.path.dirname(filepath) + new_item['basename'] = os.path.basename(filepath) + + new_item['bytes'] = os.stat(filepath).st_size + if self.register_with_checksum: + new_item['adler32'] = adler32(filepath) + new_item['md5'] = md5(filepath) + else: + new_item['adler32'] = "00000001" # empty file + new_item['md5'] = "d41d8cd98f00b204e9800998ecf8427e" # empty file + + new_item['meta'] = {'guid': self._get_file_guid(new_item)} + new_item['state'] = 'C' + if not new_item.get('did_scope'): + new_item['did_scope'] = self.default_file_scope + if not new_item.get('did_name'): + new_item['did_name'] = new_item['basename'] + + return new_item \ No newline at end of file -- GitLab From bbaa4e801025993a94be194cb101c569d6218949 Mon Sep 17 00:00:00 2001 From: "Yakubov, Sergey" Date: Thu, 15 May 2025 12:34:51 -0400 Subject: [PATCH 5/5] switch client tests to manual --- .gitlab-ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 956153143a..dbed95c565 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -52,6 +52,9 @@ client-tests: - rse-multi-builder except: - docker-base + when: manual + allow_failure: true + #api-tests: # stage: test -- GitLab