Merge pull request #14528 from mvdbeek/tool_util_packaging_fixes (afebb89c) · Commits · NDIP / Galaxy

lib/galaxy/tool_util/deps/container_resolvers/init.py

+8 −1

Original line number	Diff line number	Diff line
		@@ -4,11 +4,18 @@ from abc import (
		abstractmethod,
		abstractproperty,
		)
		from typing import (
		Optional,
		TYPE_CHECKING,
		)


		from galaxy.util.bunch import Bunch
		from galaxy.util.dictifiable import Dictifiable

		if TYPE_CHECKING:
		from beaker.cache import Cache


		class ResolutionCache(Bunch):
		"""Simple cache for duplicated computation created once per set of requests (likely web request in Galaxy context).
		@@ -17,7 +24,7 @@ class ResolutionCache(Bunch):
		one resolution at a time in a single thread.
		"""

		mulled_resolution_cache = None
		mulled_resolution_cache: Optional["Cache"] = None


		class ContainerResolver(Dictifiable, metaclass=ABCMeta):

lib/galaxy/tool_util/deps/mulled/util.py

+30 −14

Original line number	Diff line number	Diff line
		@@ -8,10 +8,17 @@ import sys
		import tarfile
		import threading
		from io import BytesIO
		from typing import (
		List,
		TYPE_CHECKING,
		)

		import packaging.version
		import requests

		if TYPE_CHECKING:
		from galaxy.tool_util.deps.container_resolvers import ResolutionCache

		log = logging.getLogger(__name__)

		QUAY_REPOSITORY_API_ENDPOINT = 'https://quay.io/api/v1/repository'
		@@ -59,18 +66,7 @@ def quay_repository(namespace, pkg_name, session=None):
		return data


		def _namespace_has_repo_name(namespace, repo_name, resolution_cache):
		"""
		Get all quay containers in the biocontainers repo
		"""
		# resolution_cache.mulled_resolution_cache is the persistent variant of the resolution cache
		resolution_cache = resolution_cache.mulled_resolution_cache or resolution_cache
		cache_key = NAMESPACE_HAS_REPO_NAME_KEY
		if resolution_cache is not None:
		try:
		return repo_name in resolution_cache.get(cache_key)
		except KeyError:
		pass
		def _get_namespace(namespace: str) -> List[str]:
		next_page = None
		repo_names = []
		repos_headers = {"Accept-encoding": "gzip", "Accept": "application/json"}
		@@ -85,8 +81,28 @@ def _namespace_has_repo_name(namespace, repo_name, resolution_cache):
		next_page = repos_response_json.get("next_page")
		if not next_page:
		break
		if resolution_cache is not None:
		resolution_cache[cache_key] = repo_names
		return repo_names


		def _namespace_has_repo_name(namespace: str, repo_name: str, resolution_cache: "ResolutionCache") -> bool:
		"""
		Get all quay containers in the biocontainers repo
		"""
		# resolution_cache.mulled_resolution_cache is the persistent variant of the resolution cache
		preferred_resolution_cache = resolution_cache.mulled_resolution_cache or resolution_cache
		cache_key = NAMESPACE_HAS_REPO_NAME_KEY
		if preferred_resolution_cache is not None:
		try:
		cached_namespace = preferred_resolution_cache.get(cache_key)
		if cached_namespace:
		return repo_name in cached_namespace
		except KeyError:
		# preferred_resolution_cache may be a beaker Cache instance, which
		# raises KeyError if key is not present on `.get`
		pass
		repo_names = _get_namespace(namespace)
		if preferred_resolution_cache is not None:
		preferred_resolution_cache[cache_key] = repo_names
		return repo_name in repo_names

packages/tool_util/HISTORY.rst

+12 −0

Original line number	Diff line number	Diff line
		@@ -5,6 +5,18 @@ History

		.. to_doc

		22.1.2 (2022-08-29)
		-------------------

		* Fix lint context error level
		* Pin galaxy-util to >= 22.1
		* Fix biocontainer resolution without beaker cache

		22.1.1 (2022-08-22)
		--------------------

		* First release from the 22.01 branch of Galaxy

		---------------------
		21.9.2 (2021-11-23)
		---------------------

packages/tool_util/requirements.txt

+2 −2

Original line number	Diff line number	Diff line
		galaxy-util>=20.1.0.dev0
		galaxy-containers
		galaxy-util>=22.1
		galaxy-containers>=22.1
		lxml
		pydantic
		pyyaml

test/unit/tool_util/test_container_resolution.py

+30 −1

Original line number	Diff line number	Diff line
		from subprocess import CalledProcessError

		import pytest

		from galaxy.tool_util.deps.container_classes import DOCKER_CONTAINER_TYPE
		from galaxy.tool_util.deps.container_resolvers.mulled import (
		CachedMulledDockerContainerResolver,
		CachedMulledSingularityContainerResolver,
		MulledDockerContainerResolver,
		)
		from galaxy.tool_util.deps.dependencies import ToolInfo
		from galaxy.tool_util.deps.containers import ContainerRegistry
		from galaxy.tool_util.deps.dependencies import (
		AppInfo,
		ToolInfo,
		)
		from galaxy.tool_util.deps.requirements import ToolRequirement


		SINGULARITY_IMAGES = ('foo:1.0--bar', 'baz:2.22', 'mulled-v2-fe8a3b846bc50d24e5df78fa0b562c43477fe9ce:9f946d13f673ab2903cb0da849ad42916d619d18-0')


		@pytest.fixture
		def container_registry():
		app_info = AppInfo(
		involucro_auto_init=True,
		enable_mulled_containers=True,
		container_image_cache_path=".",
		)
		return ContainerRegistry(app_info)


		def test_container_registry(container_registry, mocker):
		mocker.patch('galaxy.tool_util.deps.mulled.util._get_namespace', return_value=["samtools"])
		tool_info = ToolInfo(requirements=[ToolRequirement(name="samtools", version="1.10", type="package")])
		container_description = container_registry.find_best_container_description(
		[DOCKER_CONTAINER_TYPE],
		tool_info,
		install=False,
		)
		assert container_description.type == "docker"
		assert "samtools:1.10" in container_description.identifier


		def test_docker_container_resolver_detects_docker_cli_absent(mocker):
		mocker.patch('galaxy.tool_util.deps.container_resolvers.mulled.which', return_value=None)
		resolver = CachedMulledDockerContainerResolver()

Admin message