Unverified Commit 95c6e4b7 authored by Nicola Soranzo's avatar Nicola Soranzo
Browse files

Fix ``get_test_from_anaconda()`` and ``base_image_for_targets()`` functions 

by extracting all specified files in ``get_files_from_conda_package()``
and processing them in order of precedence.

Fix broken mulled unit test:

```
FAILED test/unit/tool_util/mulled/test_get_tests.py::test_hashed_test_search - AssertionError: assert ['bamtools --help'] == ['bamtools --help', 'samtools --help']
  Right contains one more item: 'samtools --help'
  Full diff:
  - ['bamtools --help', 'samtools --help']
  + ['bamtools --help']
```

which failed because, after a new build of samtools 1.3.1 was released,
``get_file_from_conda_package()`` started returning the content of
``info/recipe/meta.yaml.template`` instead of ``info/recipe/meta.yaml``
when called inside ``get_test_from_anaconda()``. Then parsing
the content of ``info/recipe/meta.yaml.template`` in
``get_commands_from_yaml()`` failed with

```
jinja2.exceptions.UndefinedError: 'compiler' is undefined
```

because that file contains ``{{ compiler('c') }}`` .

Broken in https://github.com/galaxyproject/galaxy/pull/15682 .
parent 5889ae0f

lib/galaxy/tool_util/deps/mulled/get_tests.py

+6 −5
Original line number Diff line number Diff line
@@ -27,7 +27,7 @@ except ImportError: 


from galaxy.util import unicodify

from .util import (

    get_file_from_conda_package,

    get_files_from_conda_package,

    MULLED_SOCKET_TIMEOUT,

    split_container_name,

)
@@ -110,15 +110,16 @@ def get_test_from_anaconda(url: str) -> Optional[Dict[str, Any]]: 
    """

    Given the URL of an anaconda tarball, return tests

    """

    name, content = get_file_from_conda_package(

    content_dict = get_files_from_conda_package(

        url, ["info/recipe/meta.yaml", "info/recipe/meta.yaml.template", "info/recipe/run_test.sh"]

    )

    if name and content and name.startswith("info/recipe/meta.yaml"):

    content = content_dict.get("info/recipe/meta.yaml", content_dict.get("info/recipe/meta.yaml.template"))

    if content:

        package_tests = get_commands_from_yaml(content)

        if package_tests:

            return package_tests

    if name and content and name == "info/recipe/run_test.sh":

        return get_run_test(unicodify(content))

    if "info/recipe/run_test.sh" in content_dict:

        return get_run_test(unicodify(content_dict["info/recipe/run_test.sh"]))

    return None

lib/galaxy/tool_util/deps/mulled/mulled_build.py

+10 −8
Original line number Diff line number Diff line
@@ -43,7 +43,7 @@ from .util import ( 
    conda_build_target_str,

    create_repository,

    default_mulled_conda_channels_from_env,

    get_file_from_conda_package,

    get_files_from_conda_package,

    PrintProgress,

    quay_repository,

    v1_image_name,
@@ -170,14 +170,16 @@ def base_image_for_targets(targets: List["Target"], conda_context: CondaContext) 
    hits = get_conda_hits_for_targets(targets, conda_context)

    for hit in hits:

        try:

            name, content = get_file_from_conda_package(hit["url"], ["info/about.json", "info/recipe/meta.yaml"])

            strcontent = unicodify(content)

            if name == "info/about.json" and json.loads(strcontent).get("extra", {}).get("container", {}).get(

                "extended-base", False

            ):

            content_dict = get_files_from_conda_package(hit["url"], ["info/about.json", "info/recipe/meta.yaml"])

            if "info/about.json" in content_dict and json.loads(unicodify(content_dict["info/about.json"])).get(

                "extra", {}

            ).get("container", {}).get("extended-base", False):

                return DEFAULT_EXTENDED_BASE_IMAGE

            elif name == "info/recipe/meta.yaml" and (

                yaml.safe_load(strcontent).get("extra", {}).get("container", {}).get("extended-base", False)

            elif "info/recipe/meta.yaml" in content_dict and (

                yaml.safe_load(unicodify(content_dict["info/recipe/meta.yaml"]))

                .get("extra", {})

                .get("container", {})

                .get("extended-base", False)

            ):

                return DEFAULT_EXTENDED_BASE_IMAGE

        except Exception:

lib/galaxy/tool_util/deps/mulled/util.py

+18 −30
Original line number Diff line number Diff line
@@ -8,12 +8,10 @@ import re 
import sys

import threading

from typing import (

    Dict,

    Iterable,

    List,

    Optional,

    Tuple,

    TYPE_CHECKING,

    Union,

)



import packaging.version
@@ -350,42 +348,31 @@ def v2_image_name(targets, image_build=None, name_override=None): 
        return f"mulled-v2-{package_hash.hexdigest()}{suffix}"





def get_file_from_conda_package(

    url: str, checklist: Union[str, Iterable[str]]

) -> Tuple[Optional[str], Optional[bytes]]:

def get_files_from_conda_package(url: str, filepaths: Iterable[str]) -> Dict[str, bytes]:

    """

    Get file content for an element in a conda package.

    Get content of specified files in a conda package.

    The url can be a path to a local file or an url.

    The checklist can be the name of the element to etract

    or a Iterable of potentially desired elements the content

    of the first that is contained in the conda package

    is returned.

    Return the name and content (bytes) of the first found element

    and None, None otherwise



    >>> name, content = get_file_from_conda_package("https://anaconda.org/conda-forge/chopin2/1.0.6/download/noarch/chopin2-1.0.6-pyhd8ed1ab_0.tar.bz2", "info/recipe/meta.yaml")

    >>> assert name == "info/recipe/meta.yaml"

    >>> assert isinstance(content, bytes)

    >>> name, content = get_file_from_conda_package("https://anaconda.org/conda-forge/chopin2/1.0.7/download/noarch/chopin2-1.0.7-pyhd8ed1ab_1.conda", ["info/about.json", "info/recipe/meta.yaml"])

    >>> assert name == "info/recipe/meta.yaml"

    >>> assert isinstance(content, bytes)

    The filepaths is an iterable of paths to extract from the conda package, if

    found in it.

    Return a dictionary mapping each found filepath to the corresponding content

    (as bytes).



    >>> content_dict = get_files_from_conda_package("https://anaconda.org/conda-forge/chopin2/1.0.6/download/noarch/chopin2-1.0.6-pyhd8ed1ab_0.tar.bz2", ["info/recipe/meta.yaml"])

    >>> assert "info/recipe/meta.yaml" in content_dict, content_dict

    >>> assert isinstance(content_dict["info/recipe/meta.yaml"], bytes)

    >>> content_dict = get_files_from_conda_package("https://anaconda.org/conda-forge/chopin2/1.0.7/download/noarch/chopin2-1.0.7-pyhd8ed1ab_1.conda", ["info/about.json", "info/recipe/meta.yaml", "foo/bar"])

    >>> assert sorted(content_dict.keys()) == ["info/about.json", "info/recipe/meta.yaml"], content_dict

    """



    if isinstance(checklist, str):

        checklist = set([checklist])

    else:

        checklist = set(checklist)

    # print(checklist)

    try:

        stream = stream_conda_info(url)

    except FileNotFoundError:

        stream = stream_conda_info_from_url(url)

    ret = {}

    for tar, member in stream:

        # print(member.name)

        if member.name in checklist:

            return member.name, tar.extractfile(member).read()

    # print("None")

    return None, None

        if member.name in filepaths:

            ret[member.name] = tar.extractfile(member).read()

    return ret





def split_container_name(name):
@@ -423,6 +410,7 @@ image_name = v1_image_name # deprecated 
__all__ = (

    "build_target",

    "conda_build_target_str",

    "get_files_from_conda_package",

    "image_name",

    "mulled_tags_for",

    "quay_versions",

test/unit/tool_util/mulled/test_get_tests.py

+4 −3
Original line number Diff line number Diff line
@@ -42,7 +42,6 @@ def test_get_run_test(): 
    assert commands["commands"] == [' #!/bin/bash && pslScore 2> /dev/null || [[ "$?" == 255 ]]']





@external_dependency_management

def test_get_anaconda_url():

    url = get_anaconda_url("samtools:1.7--1")

    assert url == "https://anaconda.org/bioconda/samtools/1.7/download/linux-64/samtools-1.7-1.tar.bz2"
@@ -72,9 +71,11 @@ def test_get_test_from_anaconda(): 


    # test for package defining tests in info/recipe/run_test.sh

    tests = get_test_from_anaconda(

        "https://anaconda.org/bioconda/ucsc-pslmap/366/download/linux-64/ucsc-pslmap-366-hdd26221_0.tar.bz2"

        "https://anaconda.org/bioconda/mugsy/1.2.3/download/noarch/mugsy-1.2.3-hdfd78af_4.tar.bz2"

    )

    assert tests and tests["commands"] == ['#!/bin/bash && pslMap 2> /dev/null || [[ "$?" == 255 ]] && ']

    assert tests and tests["commands"] == [

        "#!/bin/bash &&  && export MUGSY_INSTALL=${PREFIX}/bin && mugsy -h | grep mugsy > /dev/null && mugsyWGA  --version && synchain-mugsy 2>&1 | grep mugsy > /dev/null && "

    ]  # This script is clearly broken, but the whole get_tests module is currently far from usable





@external_dependency_management