Unverified Commit afebb89c authored by Marius van den Beek's avatar Marius van den Beek Committed by GitHub
Browse files

Merge pull request #14528 from mvdbeek/tool_util_packaging_fixes

[22.01] Fix tool-util packaging
parents 7a9e83c4 609c90fa
Loading
Loading
Loading
Loading
+8 −1
Original line number Diff line number Diff line
@@ -4,11 +4,18 @@ from abc import (
    abstractmethod,
    abstractproperty,
)
from typing import (
    Optional,
    TYPE_CHECKING,
)


from galaxy.util.bunch import Bunch
from galaxy.util.dictifiable import Dictifiable

if TYPE_CHECKING:
    from beaker.cache import Cache


class ResolutionCache(Bunch):
    """Simple cache for duplicated computation created once per set of requests (likely web request in Galaxy context).
@@ -17,7 +24,7 @@ class ResolutionCache(Bunch):
    one resolution at a time in a single thread.
    """

    mulled_resolution_cache = None
    mulled_resolution_cache: Optional["Cache"] = None


class ContainerResolver(Dictifiable, metaclass=ABCMeta):
+30 −14
Original line number Diff line number Diff line
@@ -8,10 +8,17 @@ import sys
import tarfile
import threading
from io import BytesIO
from typing import (
    List,
    TYPE_CHECKING,
)

import packaging.version
import requests

if TYPE_CHECKING:
    from galaxy.tool_util.deps.container_resolvers import ResolutionCache

log = logging.getLogger(__name__)

QUAY_REPOSITORY_API_ENDPOINT = 'https://quay.io/api/v1/repository'
@@ -59,18 +66,7 @@ def quay_repository(namespace, pkg_name, session=None):
    return data


def _namespace_has_repo_name(namespace, repo_name, resolution_cache):
    """
    Get all quay containers in the biocontainers repo
    """
    # resolution_cache.mulled_resolution_cache is the persistent variant of the resolution cache
    resolution_cache = resolution_cache.mulled_resolution_cache or resolution_cache
    cache_key = NAMESPACE_HAS_REPO_NAME_KEY
    if resolution_cache is not None:
        try:
            return repo_name in resolution_cache.get(cache_key)
        except KeyError:
            pass
def _get_namespace(namespace: str) -> List[str]:
    next_page = None
    repo_names = []
    repos_headers = {"Accept-encoding": "gzip", "Accept": "application/json"}
@@ -85,8 +81,28 @@ def _namespace_has_repo_name(namespace, repo_name, resolution_cache):
        next_page = repos_response_json.get("next_page")
        if not next_page:
            break
    if resolution_cache is not None:
        resolution_cache[cache_key] = repo_names
    return repo_names


def _namespace_has_repo_name(namespace: str, repo_name: str, resolution_cache: "ResolutionCache") -> bool:
    """
    Get all quay containers in the biocontainers repo
    """
    # resolution_cache.mulled_resolution_cache is the persistent variant of the resolution cache
    preferred_resolution_cache = resolution_cache.mulled_resolution_cache or resolution_cache
    cache_key = NAMESPACE_HAS_REPO_NAME_KEY
    if preferred_resolution_cache is not None:
        try:
            cached_namespace = preferred_resolution_cache.get(cache_key)
            if cached_namespace:
                return repo_name in cached_namespace
        except KeyError:
            # preferred_resolution_cache may be a beaker Cache instance, which
            # raises KeyError if key is not present on `.get`
            pass
    repo_names = _get_namespace(namespace)
    if preferred_resolution_cache is not None:
        preferred_resolution_cache[cache_key] = repo_names
    return repo_name in repo_names


+12 −0
Original line number Diff line number Diff line
@@ -5,6 +5,18 @@ History

.. to_doc

22.1.2 (2022-08-29)
-------------------

* Fix lint context error level
* Pin galaxy-util to >= 22.1
* Fix biocontainer resolution without beaker cache

22.1.1 (2022-08-22)
--------------------

* First release from the 22.01 branch of Galaxy

---------------------
21.9.2 (2021-11-23)
---------------------
+2 −2
Original line number Diff line number Diff line
galaxy-util>=20.1.0.dev0
galaxy-containers
galaxy-util>=22.1
galaxy-containers>=22.1
lxml
pydantic
pyyaml
+30 −1
Original line number Diff line number Diff line
from subprocess import CalledProcessError

import pytest

from galaxy.tool_util.deps.container_classes import DOCKER_CONTAINER_TYPE
from galaxy.tool_util.deps.container_resolvers.mulled import (
    CachedMulledDockerContainerResolver,
    CachedMulledSingularityContainerResolver,
    MulledDockerContainerResolver,
)
from galaxy.tool_util.deps.dependencies import ToolInfo
from galaxy.tool_util.deps.containers import ContainerRegistry
from galaxy.tool_util.deps.dependencies import (
    AppInfo,
    ToolInfo,
)
from galaxy.tool_util.deps.requirements import ToolRequirement


SINGULARITY_IMAGES = ('foo:1.0--bar', 'baz:2.22', 'mulled-v2-fe8a3b846bc50d24e5df78fa0b562c43477fe9ce:9f946d13f673ab2903cb0da849ad42916d619d18-0')


@pytest.fixture
def container_registry():
    app_info = AppInfo(
        involucro_auto_init=True,
        enable_mulled_containers=True,
        container_image_cache_path=".",
    )
    return ContainerRegistry(app_info)


def test_container_registry(container_registry, mocker):
    mocker.patch('galaxy.tool_util.deps.mulled.util._get_namespace', return_value=["samtools"])
    tool_info = ToolInfo(requirements=[ToolRequirement(name="samtools", version="1.10", type="package")])
    container_description = container_registry.find_best_container_description(
        [DOCKER_CONTAINER_TYPE],
        tool_info,
        install=False,
    )
    assert container_description.type == "docker"
    assert "samtools:1.10" in container_description.identifier


def test_docker_container_resolver_detects_docker_cli_absent(mocker):
    mocker.patch('galaxy.tool_util.deps.container_resolvers.mulled.which', return_value=None)
    resolver = CachedMulledDockerContainerResolver()
Loading