Unverified Commit dd2468a6 authored by Marius van den Beek's avatar Marius van den Beek Committed by GitHub
Browse files

Merge pull request #15682 from bernt-matthias/22.05-conda-package-fmt-change 

[22.05] backport #15446: Fix for new style conda packages
parents 5f9a982e ba4f6de8

lib/galaxy/dependencies/dev-requirements.txt

+1 −0
Original line number Diff line number Diff line
@@ -46,6 +46,7 @@ cloudbridge==3.0.0 
colorama==0.4.5; sys_platform == "win32" and python_version >= "3.7" and python_full_version >= "3.6.2" and platform_system == "Windows" and (python_version >= "2.7" and python_full_version < "3.0.0" and platform_system == "Windows" or python_full_version >= "3.5.0" and platform_system == "Windows") and (python_version >= "3.7" and python_full_version < "3.0.0" and sys_platform == "win32" or sys_platform == "win32" and python_version >= "3.7" and python_full_version >= "3.5.0") and (python_version >= "3.6" and python_full_version < "3.0.0" and sys_platform == "win32" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6") or sys_platform == "win32" and python_version >= "3.6" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6") and python_full_version >= "3.5.0")

coloredlogs==15.0.1; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4" or python_version >= "3.6" and python_version < "4" and python_full_version >= "3.5.0"

commonmark==0.9.1; python_full_version >= "3.6.3" and python_full_version < "4.0.0"

conda-package-streaming==0.7.0; python_version >= "3.7"

coverage==6.4.1; python_version >= "3.7"

cryptography==38.0.3; python_version >= "3.7" and python_full_version < "3.0.0" and python_version < "4" and (python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4") or python_full_version >= "3.5.0" and python_version < "4" and python_version >= "3.7" and (python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4")

cwltest==2.2.20210901154959; python_version >= "3.6" and python_version < "4"

lib/galaxy/dependencies/pinned-requirements.txt

+1 −0
Original line number Diff line number Diff line
@@ -41,6 +41,7 @@ cloudbridge==3.0.0 
colorama==0.4.5; python_version >= "3.7" and python_full_version < "3.0.0" and platform_system == "Windows" or platform_system == "Windows" and python_version >= "3.7" and python_full_version >= "3.5.0"

coloredlogs==15.0.1; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4" or python_version >= "3.6" and python_version < "4" and python_full_version >= "3.5.0"

commonmark==0.9.1; python_full_version >= "3.6.3" and python_full_version < "4.0.0"

conda-package-streaming==0.7.0; python_version >= "3.7"

cryptography==38.0.3; python_version >= "3.7" and python_full_version < "3.0.0" and python_version < "4" and (python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4") or python_full_version >= "3.5.0" and python_version < "4" and python_version >= "3.7" and (python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4")

cwltool==3.1.20211107152837; python_version >= "3.6" and python_version < "4"

decorator==5.1.1; python_version >= "3.6" and python_version < "4"

lib/galaxy/tool_util/deps/mulled/get_tests.py

+18 −34
Original line number Diff line number Diff line
@@ -8,8 +8,12 @@ A shallow search (default for singularity and conda generation scripts) just che 
# import doctest

import json

import logging

import tarfile

from glob import glob

from typing import (

    Any,

    Dict,

    Optional,

)



import requests

import yaml
@@ -21,8 +25,9 @@ except ImportError: 
    Template = None  # type: ignore[assignment,misc]

    UndefinedError = Exception  # type: ignore[assignment,misc]



from galaxy.util import unicodify

from .util import (

    get_file_from_recipe_url,

    get_file_from_conda_package,

    MULLED_SOCKET_TIMEOUT,

    split_container_name,

)
@@ -32,7 +37,7 @@ INSTALL_JINJA_EXCEPTION = ( 
)





def get_commands_from_yaml(yaml_content):

def get_commands_from_yaml(yaml_content: bytes) -> Optional[Dict[str, Any]]:

    """

    Parse tests from Conda's meta.yaml file contents

    """
@@ -77,7 +82,7 @@ def get_commands_from_yaml(yaml_content): 
    return package_tests





def get_run_test(file):

def get_run_test(file: str) -> Dict[str, Any]:

    r"""

    Get tests from a run_test.sh file

    """
@@ -101,40 +106,19 @@ def prepend_anaconda_url(url): 
    return f"https://anaconda.org{url}"





def get_test_from_anaconda(url):

def get_test_from_anaconda(url: str) -> Optional[Dict[str, Any]]:

    """

    Given the URL of an anaconda tarball, return tests

    """

    try:

        tarball = get_file_from_recipe_url(url)

    except tarfile.ReadError:

        return None



    try:

        metafile = tarball.extractfile("info/recipe/meta.yaml")

    except (tarfile.ReadError, KeyError, TypeError):

        pass

    else:

        package_tests = get_commands_from_yaml(metafile.read())

        if package_tests:

            return package_tests



    # this part is perhaps unnecessary, but some of the older tarballs have a testfile with .yaml.template ext

    try:

        metafile = tarball.extractfile("info/recipe/meta.yaml.template")

    except (tarfile.ReadError, KeyError, TypeError):

        pass

    else:

        package_tests = get_commands_from_yaml(metafile)

    name, content = get_file_from_conda_package(

        url, ["info/recipe/meta.yaml", "info/recipe/meta.yaml.template", "info/recipe/run_test.sh"]

    )

    if name and content and name.startswith("info/recipe/meta.yaml"):

        package_tests = get_commands_from_yaml(content)

        if package_tests:

            return package_tests



    # if meta.yaml was not present or there were no tests in it, try and get run_test.sh instead

    try:

        run_test = tarball.extractfile("info/recipe/run_test.sh")

        return get_run_test(run_test)

    except KeyError:

        logging.info("run_test.sh file not present.")

    if name and content and name == "info/recipe/run_test.sh":

        return get_run_test(unicodify(content))

    return None

lib/galaxy/tool_util/deps/mulled/mulled_build.py

+21 −10
Original line number Diff line number Diff line
@@ -18,6 +18,10 @@ import string 
import subprocess

import sys

from sys import platform as _platform

from typing import (

    List,

    TYPE_CHECKING,

)



import yaml
@@ -39,7 +43,7 @@ from .util import ( 
    conda_build_target_str,

    create_repository,

    default_mulled_conda_channels_from_env,

    get_file_from_recipe_url,

    get_file_from_conda_package,

    PrintProgress,

    quay_repository,

    v1_image_name,
@@ -47,6 +51,9 @@ from .util import ( 
)

from ..conda_compat import MetaData



if TYPE_CHECKING:

    from .util import Target



log = logging.getLogger(__name__)



DIRNAME = os.path.dirname(__file__)
@@ -154,19 +161,23 @@ def get_conda_hits_for_targets(targets, conda_context): 
    return [r for r in search_results if r]





def base_image_for_targets(targets, conda_context):

def base_image_for_targets(targets: List["Target"], conda_context: CondaContext) -> str:

    """

    determine base image (DEFAULT_BASE_IMAGE/DEFAULT_EXTENDED_BASE_IMAGE) for a

    list of targets by inspecting the conda package (i.e. if the use of an

    extended image is indicated in info/about.json or info/recipe/meta.yaml

    """

    hits = get_conda_hits_for_targets(targets, conda_context)

    for hit in hits:

        try:

            tarball = get_file_from_recipe_url(hit["url"])

            meta_content = unicodify(tarball.extractfile("info/about.json").read())

            if json.loads(meta_content).get("extra", {}).get("container", {}).get("extended-base", False):

            name, content = get_file_from_conda_package(hit["url"], ["info/about.json", "info/recipe/meta.yaml"])

            strcontent = unicodify(content)

            if name == "info/about.json" and json.loads(strcontent).get("extra", {}).get("container", {}).get(

                "extended-base", False

            ):

                return DEFAULT_EXTENDED_BASE_IMAGE

            elif (

                yaml.safe_load(unicodify(tarball.extractfile("info/recipe/meta.yaml").read()))

                .get("extra", {})

                .get("container", {})

                .get("extended-base", False)

            elif name == "info/recipe/meta.yaml" and (

                yaml.safe_load(strcontent).get("extra", {}).get("container", {}).get("extended-base", False)

            ):

                return DEFAULT_EXTENDED_BASE_IMAGE

        except Exception:

lib/galaxy/tool_util/deps/mulled/util.py

+41 −8
Original line number Diff line number Diff line
@@ -6,16 +6,20 @@ import logging 
import os

import re

import sys

import tarfile

import threading

from io import BytesIO

from typing import (

    Iterable,

    List,

    Optional,

    Tuple,

    TYPE_CHECKING,

    Union,

)



import packaging.version

import requests

from conda_package_streaming.package_streaming import stream_conda_info

from conda_package_streaming.url import stream_conda_info as stream_conda_info_from_url



if TYPE_CHECKING:

    from galaxy.tool_util.deps.container_resolvers import ResolutionCache
@@ -346,13 +350,42 @@ def v2_image_name(targets, image_build=None, name_override=None): 
        return f"mulled-v2-{package_hash.hexdigest()}{suffix}"





def get_file_from_recipe_url(url):

    """Downloads file at url and returns tarball"""

    if url.startswith("file://"):

        return tarfile.open(mode="r:bz2", name=url[7:])

def get_file_from_conda_package(

    url: str, checklist: Union[str, Iterable[str]]

) -> Tuple[Optional[str], Optional[bytes]]:

    """

    Get file content for an element in a conda package.

    The url can be a path to a local file or an url.

    The checklist can be the name of the element to etract

    or a Iterable of potentially desired elements the content

    of the first that is contained in the conda package

    is returned.

    Return the name and content (bytes) of the first found element

    and None, None otherwise



    >>> name, content = get_file_from_conda_package("https://anaconda.org/conda-forge/chopin2/1.0.6/download/noarch/chopin2-1.0.6-pyhd8ed1ab_0.tar.bz2", "info/recipe/meta.yaml")

    >>> assert name == "info/recipe/meta.yaml"

    >>> assert isinstance(content, bytes)

    >>> name, content = get_file_from_conda_package("https://anaconda.org/conda-forge/chopin2/1.0.7/download/noarch/chopin2-1.0.7-pyhd8ed1ab_1.conda", ["info/about.json", "info/recipe/meta.yaml"])

    >>> assert name == "info/recipe/meta.yaml"

    >>> assert isinstance(content, bytes)

    """



    if isinstance(checklist, str):

        checklist = set([checklist])

    else:

        r = requests.get(url, timeout=MULLED_SOCKET_TIMEOUT)

        return tarfile.open(mode="r:bz2", fileobj=BytesIO(r.content))

        checklist = set(checklist)

    # print(checklist)

    try:

        stream = stream_conda_info(url)

    except FileNotFoundError:

        stream = stream_conda_info_from_url(url)

    for tar, member in stream:

        # print(member.name)

        if member.name in checklist:

            return member.name, tar.extractfile(member).read()

    # print("None")

    return None, None





def split_container_name(name):