Unverified Commit dd2468a6 authored by Marius van den Beek's avatar Marius van den Beek Committed by GitHub
Browse files

Merge pull request #15682 from bernt-matthias/22.05-conda-package-fmt-change

[22.05] backport #15446: Fix for new style conda packages 
parents 5f9a982e ba4f6de8
Loading
Loading
Loading
Loading
+1 −0
Original line number Diff line number Diff line
@@ -46,6 +46,7 @@ cloudbridge==3.0.0
colorama==0.4.5; sys_platform == "win32" and python_version >= "3.7" and python_full_version >= "3.6.2" and platform_system == "Windows" and (python_version >= "2.7" and python_full_version < "3.0.0" and platform_system == "Windows" or python_full_version >= "3.5.0" and platform_system == "Windows") and (python_version >= "3.7" and python_full_version < "3.0.0" and sys_platform == "win32" or sys_platform == "win32" and python_version >= "3.7" and python_full_version >= "3.5.0") and (python_version >= "3.6" and python_full_version < "3.0.0" and sys_platform == "win32" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6") or sys_platform == "win32" and python_version >= "3.6" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6") and python_full_version >= "3.5.0")
coloredlogs==15.0.1; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4" or python_version >= "3.6" and python_version < "4" and python_full_version >= "3.5.0"
commonmark==0.9.1; python_full_version >= "3.6.3" and python_full_version < "4.0.0"
conda-package-streaming==0.7.0; python_version >= "3.7"
coverage==6.4.1; python_version >= "3.7"
cryptography==38.0.3; python_version >= "3.7" and python_full_version < "3.0.0" and python_version < "4" and (python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4") or python_full_version >= "3.5.0" and python_version < "4" and python_version >= "3.7" and (python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4")
cwltest==2.2.20210901154959; python_version >= "3.6" and python_version < "4"
+1 −0
Original line number Diff line number Diff line
@@ -41,6 +41,7 @@ cloudbridge==3.0.0
colorama==0.4.5; python_version >= "3.7" and python_full_version < "3.0.0" and platform_system == "Windows" or platform_system == "Windows" and python_version >= "3.7" and python_full_version >= "3.5.0"
coloredlogs==15.0.1; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4" or python_version >= "3.6" and python_version < "4" and python_full_version >= "3.5.0"
commonmark==0.9.1; python_full_version >= "3.6.3" and python_full_version < "4.0.0"
conda-package-streaming==0.7.0; python_version >= "3.7"
cryptography==38.0.3; python_version >= "3.7" and python_full_version < "3.0.0" and python_version < "4" and (python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4") or python_full_version >= "3.5.0" and python_version < "4" and python_version >= "3.7" and (python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4")
cwltool==3.1.20211107152837; python_version >= "3.6" and python_version < "4"
decorator==5.1.1; python_version >= "3.6" and python_version < "4"
+18 −34
Original line number Diff line number Diff line
@@ -8,8 +8,12 @@ A shallow search (default for singularity and conda generation scripts) just che
# import doctest
import json
import logging
import tarfile
from glob import glob
from typing import (
    Any,
    Dict,
    Optional,
)

import requests
import yaml
@@ -21,8 +25,9 @@ except ImportError:
    Template = None  # type: ignore[assignment,misc]
    UndefinedError = Exception  # type: ignore[assignment,misc]

from galaxy.util import unicodify
from .util import (
    get_file_from_recipe_url,
    get_file_from_conda_package,
    MULLED_SOCKET_TIMEOUT,
    split_container_name,
)
@@ -32,7 +37,7 @@ INSTALL_JINJA_EXCEPTION = (
)


def get_commands_from_yaml(yaml_content):
def get_commands_from_yaml(yaml_content: bytes) -> Optional[Dict[str, Any]]:
    """
    Parse tests from Conda's meta.yaml file contents
    """
@@ -77,7 +82,7 @@ def get_commands_from_yaml(yaml_content):
    return package_tests


def get_run_test(file):
def get_run_test(file: str) -> Dict[str, Any]:
    r"""
    Get tests from a run_test.sh file
    """
@@ -101,40 +106,19 @@ def prepend_anaconda_url(url):
    return f"https://anaconda.org{url}"


def get_test_from_anaconda(url):
def get_test_from_anaconda(url: str) -> Optional[Dict[str, Any]]:
    """
    Given the URL of an anaconda tarball, return tests
    """
    try:
        tarball = get_file_from_recipe_url(url)
    except tarfile.ReadError:
        return None

    try:
        metafile = tarball.extractfile("info/recipe/meta.yaml")
    except (tarfile.ReadError, KeyError, TypeError):
        pass
    else:
        package_tests = get_commands_from_yaml(metafile.read())
        if package_tests:
            return package_tests

    # this part is perhaps unnecessary, but some of the older tarballs have a testfile with .yaml.template ext
    try:
        metafile = tarball.extractfile("info/recipe/meta.yaml.template")
    except (tarfile.ReadError, KeyError, TypeError):
        pass
    else:
        package_tests = get_commands_from_yaml(metafile)
    name, content = get_file_from_conda_package(
        url, ["info/recipe/meta.yaml", "info/recipe/meta.yaml.template", "info/recipe/run_test.sh"]
    )
    if name and content and name.startswith("info/recipe/meta.yaml"):
        package_tests = get_commands_from_yaml(content)
        if package_tests:
            return package_tests

    # if meta.yaml was not present or there were no tests in it, try and get run_test.sh instead
    try:
        run_test = tarball.extractfile("info/recipe/run_test.sh")
        return get_run_test(run_test)
    except KeyError:
        logging.info("run_test.sh file not present.")
    if name and content and name == "info/recipe/run_test.sh":
        return get_run_test(unicodify(content))
    return None


+21 −10
Original line number Diff line number Diff line
@@ -18,6 +18,10 @@ import string
import subprocess
import sys
from sys import platform as _platform
from typing import (
    List,
    TYPE_CHECKING,
)

import yaml

@@ -39,7 +43,7 @@ from .util import (
    conda_build_target_str,
    create_repository,
    default_mulled_conda_channels_from_env,
    get_file_from_recipe_url,
    get_file_from_conda_package,
    PrintProgress,
    quay_repository,
    v1_image_name,
@@ -47,6 +51,9 @@ from .util import (
)
from ..conda_compat import MetaData

if TYPE_CHECKING:
    from .util import Target

log = logging.getLogger(__name__)

DIRNAME = os.path.dirname(__file__)
@@ -154,19 +161,23 @@ def get_conda_hits_for_targets(targets, conda_context):
    return [r for r in search_results if r]


def base_image_for_targets(targets, conda_context):
def base_image_for_targets(targets: List["Target"], conda_context: CondaContext) -> str:
    """
    determine base image (DEFAULT_BASE_IMAGE/DEFAULT_EXTENDED_BASE_IMAGE) for a
    list of targets by inspecting the conda package (i.e. if the use of an
    extended image is indicated in info/about.json or info/recipe/meta.yaml
    """
    hits = get_conda_hits_for_targets(targets, conda_context)
    for hit in hits:
        try:
            tarball = get_file_from_recipe_url(hit["url"])
            meta_content = unicodify(tarball.extractfile("info/about.json").read())
            if json.loads(meta_content).get("extra", {}).get("container", {}).get("extended-base", False):
            name, content = get_file_from_conda_package(hit["url"], ["info/about.json", "info/recipe/meta.yaml"])
            strcontent = unicodify(content)
            if name == "info/about.json" and json.loads(strcontent).get("extra", {}).get("container", {}).get(
                "extended-base", False
            ):
                return DEFAULT_EXTENDED_BASE_IMAGE
            elif (
                yaml.safe_load(unicodify(tarball.extractfile("info/recipe/meta.yaml").read()))
                .get("extra", {})
                .get("container", {})
                .get("extended-base", False)
            elif name == "info/recipe/meta.yaml" and (
                yaml.safe_load(strcontent).get("extra", {}).get("container", {}).get("extended-base", False)
            ):
                return DEFAULT_EXTENDED_BASE_IMAGE
        except Exception:
+41 −8
Original line number Diff line number Diff line
@@ -6,16 +6,20 @@ import logging
import os
import re
import sys
import tarfile
import threading
from io import BytesIO
from typing import (
    Iterable,
    List,
    Optional,
    Tuple,
    TYPE_CHECKING,
    Union,
)

import packaging.version
import requests
from conda_package_streaming.package_streaming import stream_conda_info
from conda_package_streaming.url import stream_conda_info as stream_conda_info_from_url

if TYPE_CHECKING:
    from galaxy.tool_util.deps.container_resolvers import ResolutionCache
@@ -346,13 +350,42 @@ def v2_image_name(targets, image_build=None, name_override=None):
        return f"mulled-v2-{package_hash.hexdigest()}{suffix}"


def get_file_from_recipe_url(url):
    """Downloads file at url and returns tarball"""
    if url.startswith("file://"):
        return tarfile.open(mode="r:bz2", name=url[7:])
def get_file_from_conda_package(
    url: str, checklist: Union[str, Iterable[str]]
) -> Tuple[Optional[str], Optional[bytes]]:
    """
    Get file content for an element in a conda package.
    The url can be a path to a local file or an url.
    The checklist can be the name of the element to etract
    or a Iterable of potentially desired elements the content
    of the first that is contained in the conda package
    is returned.
    Return the name and content (bytes) of the first found element
    and None, None otherwise

    >>> name, content = get_file_from_conda_package("https://anaconda.org/conda-forge/chopin2/1.0.6/download/noarch/chopin2-1.0.6-pyhd8ed1ab_0.tar.bz2", "info/recipe/meta.yaml")
    >>> assert name == "info/recipe/meta.yaml"
    >>> assert isinstance(content, bytes)
    >>> name, content = get_file_from_conda_package("https://anaconda.org/conda-forge/chopin2/1.0.7/download/noarch/chopin2-1.0.7-pyhd8ed1ab_1.conda", ["info/about.json", "info/recipe/meta.yaml"])
    >>> assert name == "info/recipe/meta.yaml"
    >>> assert isinstance(content, bytes)
    """

    if isinstance(checklist, str):
        checklist = set([checklist])
    else:
        r = requests.get(url, timeout=MULLED_SOCKET_TIMEOUT)
        return tarfile.open(mode="r:bz2", fileobj=BytesIO(r.content))
        checklist = set(checklist)
    # print(checklist)
    try:
        stream = stream_conda_info(url)
    except FileNotFoundError:
        stream = stream_conda_info_from_url(url)
    for tar, member in stream:
        # print(member.name)
        if member.name in checklist:
            return member.name, tar.extractfile(member).read()
    # print("None")
    return None, None


def split_container_name(name):
Loading