Commit 408136de authored by Yakubov, Sergey's avatar Yakubov, Sergey
Browse files

Merge branch '136-add-an-option-to-disable-checksums' into 'dev' 

Resolve "Add an option to disable checksums"

Closes #136

See merge request !117
parents f6099947 bbaa4e80

.gitlab-ci.yml

+5 −2
Original line number Diff line number Diff line
@@ -9,8 +9,8 @@ variables: 
  CONTAINER_GALAXY_URL: "${NDIP_DOCKER_REPOSITORY}/${CI_PROJECT_PATH}"

  CONTAINER_GALAXY_BASE_URL: "${CONTAINER_GALAXY_URL}/base"

  CONTAINER_GALAXY_COMMIT_URL: "${CONTAINER_GALAXY_URL}/commit"

  GALAXY_VERSION_PYTHON: 24.2.dev0+ornl

  GALAXY_VERSION_DOCKER: 24.2.dev0.ornl

  GALAXY_VERSION_PYTHON: 24.2.dev1+ornl

  GALAXY_VERSION_DOCKER: 24.2.dev1.ornl



# This import is for the func_rse_docker_* functions

before_script:
@@ -52,6 +52,9 @@ client-tests: 
    - rse-multi-builder

  except:

      - docker-base

  when: manual

  allow_failure: true





#api-tests:

#  stage: test

lib/galaxy/objectstore/rucio.py

+5 −1
Original line number Diff line number Diff line
@@ -93,11 +93,13 @@ def parse_config_xml(config_xml): 
            rucio_upload_scheme = e_xml[0].get("scheme", None)

            rucio_scope = e_xml[0].get("scope", None)

            rucio_register_only = string_as_bool(e_xml[0].get("register_only", "False"))

            rucio_register_with_checksum = string_as_bool(e_xml[0].get("rucio_register_with_checksum", "True"))

        else:

            rucio_upload_rse_name = None

            rucio_upload_scheme = None

            rucio_scope = None

            rucio_register_only = False

            rucio_register_with_checksum = True



        e_xml = config_xml.findall("rucio_auth")

        if not e_xml:
@@ -118,6 +120,7 @@ def parse_config_xml(config_xml): 
            "upload_scheme": rucio_upload_scheme,

            "scope": rucio_scope,

            "register_only": rucio_register_only,

            "rucio_register_with_checksum": rucio_register_with_checksum,

            "download_schemes": rucio_download_schemes,

            "account": rucio_account,

            "auth_host": rucio_auth_host,
@@ -148,6 +151,7 @@ class RucioBroker: 
        self.upload_rse_name = rucio_config["upload_rse_name"]

        self.scope = rucio_config["scope"]

        self.register_only = rucio_config["register_only"]

        self.register_with_checksum = rucio_config.get("register_with_checksum", True)

        self.download_schemes = rucio_config["download_schemes"]

        if Client is None:

            raise Exception(NO_RUCIO_ERROR_MESSAGE)
@@ -177,7 +181,7 @@ class RucioBroker: 


    def get_rucio_ingest_client(self, auth_token=None):

        client = self.get_rucio_client()

        ic = InPlaceIngestClient(_client=client)

        ic = InPlaceIngestClient(client, self.register_with_checksum)

        ic.auth_token = auth_token

        return ic

lib/galaxy/objectstore/rucio_extra_clients.py

+37 −1
Original line number Diff line number Diff line 
import copy

import logging

import os

import time



try:
@@ -10,7 +11,7 @@ try: 
        NotAllFilesUploaded,

        RSEWriteBlocked,

    )

    from rucio.common.utils import generate_uuid

    from rucio.common.utils import generate_uuid, adler32, md5

    from rucio.rse import rsemanager as rsemgr

except ImportError:

    UploadClient = object
@@ -57,6 +58,9 @@ class DeleteClient(UploadClient): 




class InPlaceIngestClient(UploadClient):

    def __init__(self, client, register_with_checksum):

        super().__init__(client)

        self.register_with_checksum = register_with_checksum

    def ingest(self, items, summary_file_path=None, traces_copy_out=None, ignore_availability=False, activity=None):

        """

        :param items: List of dictionaries. Each dictionary describing a file to upload. Keys:
@@ -195,3 +199,35 @@ class InPlaceIngestClient(UploadClient): 
        elif num_succeeded != len(files):

            raise NotAllFilesUploaded()

        return 0

    def _collect_file_info(self, filepath, item):

        """

        Collects infos (e.g. size, checksums, etc.) about the file and

        returns them as a dictionary

        (This function is meant to be used as class internal only)



        :param filepath: path where the file is stored

        :param item: input options for the given file



        :returns: a dictionary containing all collected info and the input options

        """

        new_item = copy.deepcopy(item)

        new_item['path'] = filepath

        new_item['dirname'] = os.path.dirname(filepath)

        new_item['basename'] = os.path.basename(filepath)



        new_item['bytes'] = os.stat(filepath).st_size

        if self.register_with_checksum:

            new_item['adler32'] = adler32(filepath)

            new_item['md5'] = md5(filepath)

        else:

            new_item['adler32'] = "00000001"  # empty file

            new_item['md5'] = "d41d8cd98f00b204e9800998ecf8427e"  # empty file



        new_item['meta'] = {'guid': self._get_file_guid(new_item)}

        new_item['state'] = 'C'

        if not new_item.get('did_scope'):

            new_item['did_scope'] = self.default_file_scope

        if not new_item.get('did_name'):

            new_item['did_name'] = new_item['basename']



        return new_item
 
 No newline at end of file