Merge branch 'dev' into 'main' (8a47610d) · Commits · NDIP / Galaxy

.gitlab-ci.yml

+5 −2

Original line number	Diff line number	Diff line
		@@ -9,8 +9,8 @@ variables:
		CONTAINER_GALAXY_URL: "${NDIP_DOCKER_REPOSITORY}/${CI_PROJECT_PATH}"
		CONTAINER_GALAXY_BASE_URL: "${CONTAINER_GALAXY_URL}/base"
		CONTAINER_GALAXY_COMMIT_URL: "${CONTAINER_GALAXY_URL}/commit"
		GALAXY_VERSION_PYTHON: 24.2.dev0+ornl
		GALAXY_VERSION_DOCKER: 24.2.dev0.ornl
		GALAXY_VERSION_PYTHON: 24.2.dev1+ornl
		GALAXY_VERSION_DOCKER: 24.2.dev1.ornl

		# This import is for the func_rse_docker_* functions
		before_script:
		@@ -52,6 +52,9 @@ client-tests:
		- rse-multi-builder
		except:
		- docker-base
		when: manual
		allow_failure: true


		#api-tests:
		# stage: test

lib/galaxy/job_execution/container_monitor.py

+3 −2

Original line number	Diff line number	Diff line
		@@ -30,7 +30,8 @@ def parse_ports(container_name, connection_configuration):
		stdout_file.seek(0)
		ports_raw = stdout_file.read().decode("utf-8")
		return ports_raw

		else:
		raise Exception("Could not find port.")

		def get_ip_command(cmd) -> str:
		return subprocess.check_output(shlex.split(cmd), text=True).strip()
		@@ -64,7 +65,7 @@ def main():

		ports_raw = None
		exc_traceback = ""
		for i in range(10):
		for i in range(60):
		try:
		ports_raw = parse_ports(container_name, connection_configuration)
		if ports_raw is not None:

lib/galaxy/objectstore/rucio.py

+5 −1

Original line number	Diff line number	Diff line
		@@ -93,11 +93,13 @@ def parse_config_xml(config_xml):
		rucio_upload_scheme = e_xml[0].get("scheme", None)
		rucio_scope = e_xml[0].get("scope", None)
		rucio_register_only = string_as_bool(e_xml[0].get("register_only", "False"))
		rucio_register_with_checksum = string_as_bool(e_xml[0].get("rucio_register_with_checksum", "True"))
		else:
		rucio_upload_rse_name = None
		rucio_upload_scheme = None
		rucio_scope = None
		rucio_register_only = False
		rucio_register_with_checksum = True

		e_xml = config_xml.findall("rucio_auth")
		if not e_xml:
		@@ -118,6 +120,7 @@ def parse_config_xml(config_xml):
		"upload_scheme": rucio_upload_scheme,
		"scope": rucio_scope,
		"register_only": rucio_register_only,
		"rucio_register_with_checksum": rucio_register_with_checksum,
		"download_schemes": rucio_download_schemes,
		"account": rucio_account,
		"auth_host": rucio_auth_host,
		@@ -148,6 +151,7 @@ class RucioBroker:
		self.upload_rse_name = rucio_config["upload_rse_name"]
		self.scope = rucio_config["scope"]
		self.register_only = rucio_config["register_only"]
		self.register_with_checksum = rucio_config.get("register_with_checksum", True)
		self.download_schemes = rucio_config["download_schemes"]
		if Client is None:
		raise Exception(NO_RUCIO_ERROR_MESSAGE)
		@@ -177,7 +181,7 @@ class RucioBroker:

		def get_rucio_ingest_client(self, auth_token=None):
		client = self.get_rucio_client()
		ic = InPlaceIngestClient(_client=client)
		ic = InPlaceIngestClient(client, self.register_with_checksum)
		ic.auth_token = auth_token
		return ic

lib/galaxy/objectstore/rucio_extra_clients.py

+37 −1

Original line number	Diff line number	Diff line
		import copy
		import logging
		import os
		import time

		try:
		@@ -10,7 +11,7 @@ try:
		NotAllFilesUploaded,
		RSEWriteBlocked,
		)
		from rucio.common.utils import generate_uuid
		from rucio.common.utils import generate_uuid, adler32, md5
		from rucio.rse import rsemanager as rsemgr
		except ImportError:
		UploadClient = object
		@@ -57,6 +58,9 @@ class DeleteClient(UploadClient):


		class InPlaceIngestClient(UploadClient):
		def __init__(self, client, register_with_checksum):
		super().__init__(client)
		self.register_with_checksum = register_with_checksum
		def ingest(self, items, summary_file_path=None, traces_copy_out=None, ignore_availability=False, activity=None):
		"""
		:param items: List of dictionaries. Each dictionary describing a file to upload. Keys:
		@@ -195,3 +199,35 @@ class InPlaceIngestClient(UploadClient):
		elif num_succeeded != len(files):
		raise NotAllFilesUploaded()
		return 0
		def _collect_file_info(self, filepath, item):
		"""
		Collects infos (e.g. size, checksums, etc.) about the file and
		returns them as a dictionary
		(This function is meant to be used as class internal only)

		:param filepath: path where the file is stored
		:param item: input options for the given file

		:returns: a dictionary containing all collected info and the input options
		"""
		new_item = copy.deepcopy(item)
		new_item['path'] = filepath
		new_item['dirname'] = os.path.dirname(filepath)
		new_item['basename'] = os.path.basename(filepath)

		new_item['bytes'] = os.stat(filepath).st_size
		if self.register_with_checksum:
		new_item['adler32'] = adler32(filepath)
		new_item['md5'] = md5(filepath)
		else:
		new_item['adler32'] = "00000001" # empty file
		new_item['md5'] = "d41d8cd98f00b204e9800998ecf8427e" # empty file

		new_item['meta'] = {'guid': self._get_file_guid(new_item)}
		new_item['state'] = 'C'
		if not new_item.get('did_scope'):
		new_item['did_scope'] = self.default_file_scope
		if not new_item.get('did_name'):
		new_item['did_name'] = new_item['basename']

		return new_item
		No newline at end of file

lib/galaxy/tool_util/deps/container_classes.py

+1 −1

Original line number	Diff line number	Diff line
		@@ -501,7 +501,7 @@ class DockerContainer(Container, HasDockerLikeVolumes):
		auto_rm=asbool(self.prop("auto_rm", docker_util.DEFAULT_AUTO_REMOVE)),
		set_user=self.prop("set_user", docker_util.DEFAULT_SET_USER),
		set_user_from_host=host_user,
		run_extra_arguments=self.prop("run_extra_arguments", docker_util.DEFAULT_RUN_EXTRA_ARGUMENTS),
		run_extra_arguments=run_extra_arguments,
		guest_ports=self.tool_info.guest_ports,
		container_name=self.container_name,
		**docker_host_props,