Commit 7e31d3bb authored by Matthias Bernt's avatar Matthias Bernt
Browse files

attempt to fix

parent b51af0a1

lib/galaxy/dependencies/pinned-requirements.txt

+1 −0
Original line number Diff line number Diff line
@@ -41,6 +41,7 @@ cloudbridge==3.0.0 
colorama==0.4.5; python_version >= "3.7" and python_full_version < "3.0.0" and platform_system == "Windows" or platform_system == "Windows" and python_version >= "3.7" and python_full_version >= "3.5.0"

coloredlogs==15.0.1; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4" or python_version >= "3.6" and python_version < "4" and python_full_version >= "3.5.0"

commonmark==0.9.1; python_full_version >= "3.6.3" and python_full_version < "4.0.0"

conda_package_streaming==0.7.0 ; python_version >= "3.6" and python_version < "4.0.0"

cryptography==38.0.3; python_version >= "3.7" and python_full_version < "3.0.0" and python_version < "4" and (python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4") or python_full_version >= "3.5.0" and python_version < "4" and python_version >= "3.7" and (python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4")

cwltool==3.1.20211107152837; python_version >= "3.6" and python_version < "4"

decorator==5.1.1; python_version >= "3.6" and python_version < "4"

lib/galaxy/tool_util/deps/mulled/get_tests.py

+56 −32
Original line number Diff line number Diff line
@@ -10,6 +10,10 @@ import json 
import logging

import tarfile

from glob import glob

from typing import (

    Dict,

    Optional,

)



import requests

import yaml
@@ -21,8 +25,9 @@ except ImportError: 
    Template = None  # type: ignore[assignment,misc]

    UndefinedError = Exception  # type: ignore[assignment,misc]



from galaxy.util import unicodify

from .util import (

    get_file_from_recipe_url,

    get_file_from_conda_package,

    MULLED_SOCKET_TIMEOUT,

    split_container_name,

)
@@ -77,7 +82,7 @@ def get_commands_from_yaml(yaml_content): 
    return package_tests





def get_run_test(file):

def get_run_test(file: str) -> Dict:

    r"""

    Get tests from a run_test.sh file

    """
@@ -101,41 +106,60 @@ def prepend_anaconda_url(url): 
    return f"https://anaconda.org{url}"





def get_test_from_anaconda(url):

def get_test_from_anaconda(url: str) -> Optional[Dict]:

    """

    Given the URL of an anaconda tarball, return tests

    """

    try:

        tarball = get_file_from_recipe_url(url)

    except tarfile.ReadError:

        return None



    try:

        metafile = tarball.extractfile("info/recipe/meta.yaml")

    except (tarfile.ReadError, KeyError, TypeError):

        pass

    else:

        package_tests = get_commands_from_yaml(metafile.read())

        if package_tests:

            return package_tests



    # this part is perhaps unnecessary, but some of the older tarballs have a testfile with .yaml.template ext

    try:

        metafile = tarball.extractfile("info/recipe/meta.yaml.template")

    except (tarfile.ReadError, KeyError, TypeError):

        pass

    else:

        package_tests = get_commands_from_yaml(metafile)

    >>> # test for info/recipe/meta.yaml

    >>> tests = get_test_from_anaconda("https://anaconda.org/conda-forge/chopin2/1.0.6/download/noarch/chopin2-1.0.6-pyhd8ed1ab_0.tar.bz2")

    >>> assert tests == {'commands': ['pip check', 'chopin2 --version'], 'imports': ['chopin2'], 'import_lang': 'python -c'}, tests

    >>> tests = get_test_from_anaconda("https://anaconda.org/conda-forge/chopin2/1.0.7/download/noarch/chopin2-1.0.7-pyhd8ed1ab_1.conda")

    >>> assert tests == {'commands': ['pip check', 'chopin2 --version'], 'imports': ['chopin2'], 'import_lang': 'python -c'}, tests

    >>> # test for info/recipe/run_test.sh

    >>> tests = get_test_from_anaconda("https://anaconda.org/bioconda/ucsc-pslmap/366/download/linux-64/ucsc-pslmap-366-hdd26221_0.tar.bz2")

    >>> assert tests == {'commands': ['#!/bin/bash && pslMap 2> /dev/null || [[ "$?" == 255 ]] && ']}, tests

    """

    name, content = get_file_from_conda_package(

        url, ["info/recipe/meta.yaml", "info/recipe/meta.yaml.template", "info/recipe/run_test.sh"]

    )

    if name.startswith("info/recipe/meta.yaml"):

        package_tests = get_commands_from_yaml(content)

        if package_tests:

            return package_tests



    # if meta.yaml was not present or there were no tests in it, try and get run_test.sh instead

    try:

        run_test = tarball.extractfile("info/recipe/run_test.sh")

        return get_run_test(run_test)

    except KeyError:

        logging.info("run_test.sh file not present.")

    if name == "info/recipe/run_test.sh":

        return get_run_test(unicodify(content))

    return None

    # try:

    #     tarball = get_file_from_recipe_url(url)

    # except tarfile.ReadError:

    #     return None



    # try:

    #     metafile = tarball.extractfile("info/recipe/meta.yaml")

    # except (tarfile.ReadError, KeyError, TypeError):

    #     pass

    # else:

    #     package_tests = get_commands_from_yaml(metafile.read())

    #     if package_tests:

    #         return package_tests



    # # this part is perhaps unnecessary, but some of the older tarballs have a testfile with .yaml.template ext

    # try:

    #     metafile = tarball.extractfile("info/recipe/meta.yaml.template")

    # except (tarfile.ReadError, KeyError, TypeError):

    #     pass

    # else:

    #     package_tests = get_commands_from_yaml(metafile)

    #     if package_tests:

    #         return package_tests



    # # if meta.yaml was not present or there were no tests in it, try and get run_test.sh instead

    # try:

    #     run_test = tarball.extractfile("info/recipe/run_test.sh")

    #     return get_run_test(run_test)

    # except KeyError:

    #     logging.info("run_test.sh file not present.")

    #     return None





def find_anaconda_versions(name, anaconda_channel="bioconda"):

lib/galaxy/tool_util/deps/mulled/mulled_build.py

+18 −10
Original line number Diff line number Diff line
@@ -18,6 +18,7 @@ import string 
import subprocess

import sys

from sys import platform as _platform

from typing import List, TYPE_CHECKING



import yaml
@@ -39,7 +40,7 @@ from .util import ( 
    conda_build_target_str,

    create_repository,

    default_mulled_conda_channels_from_env,

    get_file_from_recipe_url,

    get_file_from_conda_package,

    PrintProgress,

    quay_repository,

    v1_image_name,
@@ -47,6 +48,9 @@ from .util import ( 
)

from ..conda_compat import MetaData



if TYPE_CHECKING:

    from .util import Target



log = logging.getLogger(__name__)



DIRNAME = os.path.dirname(__file__)
@@ -154,19 +158,23 @@ def get_conda_hits_for_targets(targets, conda_context): 
    return [r for r in search_results if r]





def base_image_for_targets(targets, conda_context):

def base_image_for_targets(targets: List["Target"], conda_context: CondaContext) -> str:

    """

    determine base image (DEFAULT_BASE_IMAGE/DEFAULT_EXTENDED_BASE_IMAGE) for a

    list of targets by inspecting the conda package (i.e. if the use of an

    extended image is indicated in info/about.json or info/recipe/meta.yaml

    """

    hits = get_conda_hits_for_targets(targets, conda_context)

    for hit in hits:

        try:

            tarball = get_file_from_recipe_url(hit["url"])

            meta_content = unicodify(tarball.extractfile("info/about.json").read())

            if json.loads(meta_content).get("extra", {}).get("container", {}).get("extended-base", False):

            name, content = get_file_from_conda_package(hit["url"], ["info/about.json", "info/recipe/meta.yaml"])

            strcontent = unicodify(content)

            if name == "info/about.json" and json.loads(strcontent).get("extra", {}).get("container", {}).get(

                "extended-base", False

            ):

                return DEFAULT_EXTENDED_BASE_IMAGE

            elif (

                yaml.safe_load(unicodify(tarball.extractfile("info/recipe/meta.yaml").read()))

                .get("extra", {})

                .get("container", {})

                .get("extended-base", False)

            elif name == "info/recipe/meta.yaml" and (

                yaml.safe_load(strcontent).get("extra", {}).get("container", {}).get("extended-base", False)

            ):

                return DEFAULT_EXTENDED_BASE_IMAGE

        except Exception:

lib/galaxy/tool_util/deps/mulled/util.py

+40 −12
Original line number Diff line number Diff line
@@ -10,12 +10,19 @@ import tarfile 
import threading

from io import BytesIO

from typing import (

    Iterable,

    List,

    NamedTuple,

    Optional,

    Tuple,

    TYPE_CHECKING,

    Union,

)



import packaging.version

import requests

from conda_package_streaming.package_streaming import stream_conda_info

from conda_package_streaming.url import stream_conda_info as stream_conda_info_from_url



if TYPE_CHECKING:

    from galaxy.tool_util.deps.container_resolvers import ResolutionCache
@@ -346,21 +353,42 @@ def v2_image_name(targets, image_build=None, name_override=None): 
        return f"mulled-v2-{package_hash.hexdigest()}{suffix}"





def get_file_from_recipe_url(url):

def get_file_from_conda_package(

    url: str, checklist: Union[str, Iterable[str]]

) -> Tuple[Optional[str], Optional[bytes]]:

    """

    Downloads file at url and returns tarball

    

    >>> f = get_file_from_recipe_url("https://anaconda.org/conda-forge/chopin2/1.0.6/download/noarch/chopin2-1.0.6-pyhd8ed1ab_0.tar.bz2")

    >>> assert isinstance(f, tarfile.TarFile)

    >>> f = get_file_from_recipe_url("https://anaconda.org/conda-forge/chopin2/1.0.7/download/noarch/chopin2-1.0.7-pyhd8ed1ab_1.conda")

    >>> assert isinstance(f, tarfile.TarFile)

    Get file content for an element in a conda package.

    The url can be a path to a local file or an url.

    The checklist can be the name of the element to etract

    or a Iterable of potentially desired elements the content

    of the first that is contained in the conda package

    is returned.

    Return the name and content (bytes) of the first found element

    and None, None otherwise



    >>> name, content = get_file_from_conda_package("https://anaconda.org/conda-forge/chopin2/1.0.6/download/noarch/chopin2-1.0.6-pyhd8ed1ab_0.tar.bz2", "info/recipe/meta.yaml")

    >>> assert name == "info/recipe/meta.yaml"

    >>> assert isinstance(content, bytes)

    >>> name, content = get_file_from_conda_package("https://anaconda.org/conda-forge/chopin2/1.0.7/download/noarch/chopin2-1.0.7-pyhd8ed1ab_1.conda", ["info/about.json", "info/recipe/meta.yaml"])

    >>> assert name == "info/recipe/meta.yaml"

    >>> assert isinstance(content, bytes)

    """

    log.error(f"{url}")

    if url.startswith("file://"):

        return tarfile.open(mode="r:bz2", name=url[7:])



    if isinstance(checklist, str):

        checklist = set([checklist])

    else:

        r = requests.get(url, timeout=MULLED_SOCKET_TIMEOUT)

        return tarfile.open(mode="r:bz2", fileobj=BytesIO(r.content))

        checklist = set(checklist)

    # print(checklist)

    try:

        stream = stream_conda_info(url)

    except FileNotFoundError:

        stream = stream_conda_info_from_url(url)

    for tar, member in stream:

        # print(member.name)

        if member.name in checklist:

            return member.name, tar.extractfile(member).read()

    # print("None")

    return None, None





def split_container_name(name):