diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e39e58df9b4cd62cc75613a91caba6b4e8473bf0..dbed95c565d5c0edba9ce92f035cdedcb8004e16 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -9,8 +9,8 @@ variables: CONTAINER_GALAXY_URL: "${NDIP_DOCKER_REPOSITORY}/${CI_PROJECT_PATH}" CONTAINER_GALAXY_BASE_URL: "${CONTAINER_GALAXY_URL}/base" CONTAINER_GALAXY_COMMIT_URL: "${CONTAINER_GALAXY_URL}/commit" - GALAXY_VERSION_PYTHON: 24.2.dev0+ornl - GALAXY_VERSION_DOCKER: 24.2.dev0.ornl + GALAXY_VERSION_PYTHON: 24.2.dev1+ornl + GALAXY_VERSION_DOCKER: 24.2.dev1.ornl # This import is for the func_rse_docker_* functions before_script: @@ -52,6 +52,9 @@ client-tests: - rse-multi-builder except: - docker-base + when: manual + allow_failure: true + #api-tests: # stage: test diff --git a/lib/galaxy/objectstore/rucio.py b/lib/galaxy/objectstore/rucio.py index 97d1bd9f8e143b7850aea61188631607845cfd30..9e6763573b74f572ea4d1814898b201de001b654 100644 --- a/lib/galaxy/objectstore/rucio.py +++ b/lib/galaxy/objectstore/rucio.py @@ -93,11 +93,13 @@ def parse_config_xml(config_xml): rucio_upload_scheme = e_xml[0].get("scheme", None) rucio_scope = e_xml[0].get("scope", None) rucio_register_only = string_as_bool(e_xml[0].get("register_only", "False")) + rucio_register_with_checksum = string_as_bool(e_xml[0].get("rucio_register_with_checksum", "True")) else: rucio_upload_rse_name = None rucio_upload_scheme = None rucio_scope = None rucio_register_only = False + rucio_register_with_checksum = True e_xml = config_xml.findall("rucio_auth") if not e_xml: @@ -118,6 +120,7 @@ def parse_config_xml(config_xml): "upload_scheme": rucio_upload_scheme, "scope": rucio_scope, "register_only": rucio_register_only, + "rucio_register_with_checksum": rucio_register_with_checksum, "download_schemes": rucio_download_schemes, "account": rucio_account, "auth_host": rucio_auth_host, @@ -148,6 +151,7 @@ class RucioBroker: self.upload_rse_name = rucio_config["upload_rse_name"] self.scope = rucio_config["scope"] self.register_only = rucio_config["register_only"] + self.register_with_checksum = rucio_config.get("register_with_checksum", True) self.download_schemes = rucio_config["download_schemes"] if Client is None: raise Exception(NO_RUCIO_ERROR_MESSAGE) @@ -177,7 +181,7 @@ class RucioBroker: def get_rucio_ingest_client(self, auth_token=None): client = self.get_rucio_client() - ic = InPlaceIngestClient(_client=client) + ic = InPlaceIngestClient(client, self.register_with_checksum) ic.auth_token = auth_token return ic diff --git a/lib/galaxy/objectstore/rucio_extra_clients.py b/lib/galaxy/objectstore/rucio_extra_clients.py index 500366ee3618e2f83bd5fb609610edeca4fb10b7..51fa996590550294d6ef0ddd1db20beadf42d55f 100644 --- a/lib/galaxy/objectstore/rucio_extra_clients.py +++ b/lib/galaxy/objectstore/rucio_extra_clients.py @@ -1,5 +1,6 @@ import copy import logging +import os import time try: @@ -10,7 +11,7 @@ try: NotAllFilesUploaded, RSEWriteBlocked, ) - from rucio.common.utils import generate_uuid + from rucio.common.utils import generate_uuid, adler32, md5 from rucio.rse import rsemanager as rsemgr except ImportError: UploadClient = object @@ -57,6 +58,9 @@ class DeleteClient(UploadClient): class InPlaceIngestClient(UploadClient): + def __init__(self, client, register_with_checksum): + super().__init__(client) + self.register_with_checksum = register_with_checksum def ingest(self, items, summary_file_path=None, traces_copy_out=None, ignore_availability=False, activity=None): """ :param items: List of dictionaries. Each dictionary describing a file to upload. Keys: @@ -195,3 +199,35 @@ class InPlaceIngestClient(UploadClient): elif num_succeeded != len(files): raise NotAllFilesUploaded() return 0 + def _collect_file_info(self, filepath, item): + """ + Collects infos (e.g. size, checksums, etc.) about the file and + returns them as a dictionary + (This function is meant to be used as class internal only) + + :param filepath: path where the file is stored + :param item: input options for the given file + + :returns: a dictionary containing all collected info and the input options + """ + new_item = copy.deepcopy(item) + new_item['path'] = filepath + new_item['dirname'] = os.path.dirname(filepath) + new_item['basename'] = os.path.basename(filepath) + + new_item['bytes'] = os.stat(filepath).st_size + if self.register_with_checksum: + new_item['adler32'] = adler32(filepath) + new_item['md5'] = md5(filepath) + else: + new_item['adler32'] = "00000001" # empty file + new_item['md5'] = "d41d8cd98f00b204e9800998ecf8427e" # empty file + + new_item['meta'] = {'guid': self._get_file_guid(new_item)} + new_item['state'] = 'C' + if not new_item.get('did_scope'): + new_item['did_scope'] = self.default_file_scope + if not new_item.get('did_name'): + new_item['did_name'] = new_item['basename'] + + return new_item \ No newline at end of file