Unverified Commit 766f093e authored by Nicola Soranzo's avatar Nicola Soranzo Committed by GitHub
Browse files

Merge pull request #16125 from nsoranzo/release_22.05_fix_get_file_from_conda_package 

[22.05] Fix ``get_test_from_anaconda()`` and ``base_image_for_targets()`` functions
parents 012102dc ffb52910

.github/workflows/mulled.yaml

+2 −0
Original line number Diff line number Diff line
@@ -42,6 +42,8 @@ jobs: 
        with:

          path: .tox

          key: tox-cache-${{ runner.os }}-${{ steps.full-python-version.outputs.version }}-${{ hashFiles('galaxy root/requirements.txt') }}-mulled

      - name: Install Apptainer's singularity

        uses: eWaterCycle/setup-apptainer@v2

      - name: Install tox

        run: pip install tox

      - name: run tests

.github/workflows/toolshed.yaml

+1 −1
Original line number Diff line number Diff line
@@ -17,7 +17,7 @@ concurrency: 
jobs:

  test:

    name: Test

    runs-on: ubuntu-18.04

    runs-on: ubuntu-latest

    strategy:

      matrix:

        python-version: ['3.7']

doc/source/admin/special_topics/mulled_containers.rst

+7 −5
Original line number Diff line number Diff line
@@ -153,8 +153,6 @@ Multiple containers can be installed simultaneously by giving ``--containers`` m 


   $ mulled-update-singularity-containers --containers samtools:1.6--0 bamtools:2.4.1--0 --filepath /tmp/sing/ --installation /usr/local/bin/singularity



.. code-block:: bash



For a large number of containers, it may be more convenient to employ the ``--container-list`` option:



.. code-block:: bash
@@ -163,15 +161,19 @@ For a large number of containers, it may be more convenient to employ the ``--co 


Here ``list.txt`` should contain a list of containers, each on a new line.



In order to generate the list file the ``mulled-list`` command may be useful. The following command returns a list of all Docker containers available on the quay.io biocontainers organization, excluding those already available as Singularity containers via https://depot.galaxyproject.org/singularity/.:: bash

In order to generate the list file the ``mulled-list`` command may be useful. The following command returns a list of all Docker containers available on the quay.io biocontainers organization, excluding those already available as Singularity containers on https://depot.galaxyproject.org/singularity/ .



.. code-block:: bash



   $ mulled-list --source docker --not-singularity --blacklist blacklist.txt --file output.txt



The list of containers will be saved as ``output.txt``. The (optional) ``--blacklist`` option may be used to exclude containers which should not included in the output; ``blacklist.txt`` should contain a list of the 'blacklisted' containers, each on a new line.



Containers, once generated, should be tested. This can be achieved by affixing ``--testing test-output.log`` to the command, or alternatively, by use of the dedicated ``mulled-singularity-testing`` tool.:: bash

The generated containers should also be tested. This can be achieved by affixing ``--testing test-output.log`` to the ``mulled-update-singularity-containers`` command:



.. code-block:: bash



   $ mulled-singularity-testing --container-list list.txt --filepath /tmp/sing/ --installation /usr/local/bin/singularity --logfile test-output.txt

   $ mulled-update-singularity-containers --container-list list.txt --filepath /tmp/sing/ --installation /usr/local/bin/singularity --testing test-output.log



.. _IUC: https://galaxyproject.org/iuc/

.. _container annotation:  https://github.com/galaxyproject/galaxy/blob/dev/test/functional/tools/catDocker.xml#L4

lib/galaxy/tool_util/deps/conda_util.py

+2 −0
Original line number Diff line number Diff line
@@ -517,6 +517,8 @@ def best_search_result( 


    Return ``None`` if no results match.

    """

    # Cannot specify the version here (i.e. conda_target.package_specifier)

    # because if the version is not found, the exec_search() call would fail.

    search_args = [conda_target.package]

    try:

        res = conda_context.exec_search(search_args, json=True, offline=offline, platform=platform)

lib/galaxy/tool_util/deps/mulled/get_tests.py

+32 −18
Original line number Diff line number Diff line
@@ -12,6 +12,7 @@ from glob import glob 
from typing import (

    Any,

    Dict,

    List,

    Optional,

)
@@ -26,8 +27,9 @@ except ImportError: 
    UndefinedError = Exception  # type: ignore[assignment,misc]



from galaxy.util import unicodify

from galaxy.util.commands import argv_to_str

from .util import (

    get_file_from_conda_package,

    get_files_from_conda_package,

    MULLED_SOCKET_TIMEOUT,

    split_container_name,

)
@@ -66,6 +68,7 @@ def get_commands_from_yaml(yaml_content: bytes) -> Optional[Dict[str, Any]]: 
        return None



    # need to know what scripting languages are needed to run the container

    package_tests["import_lang"] = "python -c"  # python by default

    try:

        requirements = list(meta_yaml["requirements"]["run"])

    except (KeyError, TypeError):
@@ -77,8 +80,6 @@ def get_commands_from_yaml(yaml_content: bytes) -> Optional[Dict[str, Any]]: 
                break

            # elif ... :

            # other languages if necessary ... hopefully python and perl should suffice though

        else:  # python by default

            package_tests["import_lang"] = "python -c"

    return package_tests
@@ -110,15 +111,16 @@ def get_test_from_anaconda(url: str) -> Optional[Dict[str, Any]]: 
    """

    Given the URL of an anaconda tarball, return tests

    """

    name, content = get_file_from_conda_package(

    content_dict = get_files_from_conda_package(

        url, ["info/recipe/meta.yaml", "info/recipe/meta.yaml.template", "info/recipe/run_test.sh"]

    )

    if name and content and name.startswith("info/recipe/meta.yaml"):

    content = content_dict.get("info/recipe/meta.yaml", content_dict.get("info/recipe/meta.yaml.template"))

    if content:

        package_tests = get_commands_from_yaml(content)

        if package_tests:

            return package_tests

    if name and content and name == "info/recipe/run_test.sh":

        return get_run_test(unicodify(content))

    if "info/recipe/run_test.sh" in content_dict:

        return get_run_test(unicodify(content_dict["info/recipe/run_test.sh"]))

    return None
@@ -128,7 +130,7 @@ def find_anaconda_versions(name, anaconda_channel="bioconda"): 
    """

    r = requests.get(f"https://anaconda.org/{anaconda_channel}/{name}/files", timeout=MULLED_SOCKET_TIMEOUT)

    urls = []

    for line in r.text.split("\n"):

    for line in r.text.splitlines():

        if "download/linux" in line:

            urls.append(line.split('"')[1])

    return urls
@@ -259,23 +261,34 @@ def main_test_search( 
    return {"container": container}





def import_test_to_command_list(import_lang: str, import_: str) -> List[str]:

    if import_lang == "python -c":

        return ["python", "-c", f"import {import_}"]

    elif import_lang == "perl -e":

        return ["perl", "-e", f"use {import_}"]

    else:

        raise ValueError(f"Unsupported import_lang '{import_lang}'")





def hashed_test_search(

    container, recipes_path=None, deep=False, anaconda_channel="bioconda", github_repo="bioconda/bioconda-recipes"

):

    container: str, recipes_path=None, deep=False, anaconda_channel="bioconda", github_repo="bioconda/bioconda-recipes"

) -> Dict[str, Any]:

    """

    Get test for hashed containers

    """

    package_tests = {"commands": [], "imports": [], "container": container, "import_lang": "python -c"}

    package_tests: Dict[str, Any] = {"commands": [], "imports": [], "container": container, "import_lang": "python -c"}



    githubpage = requests.get(

    response = requests.get(

        f"https://raw.githubusercontent.com/BioContainers/multi-package-containers/master/combinations/{container}.tsv",

        timeout=MULLED_SOCKET_TIMEOUT,

    )

    if githubpage.status_code == 200:

        packages = githubpage.text.split(",")  # get names of packages from github

        packages = [package.split("=") for package in packages]

    else:

        packages = []

    response.raise_for_status()

    for line in response.text.splitlines():

        if not line.startswith("#"):

            break

    concatenated_targets = line.split("\t")[0]

    targets = concatenated_targets.split(",")

    packages = [target.split("=") for target in targets]



    containers = []

    for package in packages:
@@ -293,7 +306,8 @@ def hashed_test_search( 
    for container in containers:

        tests = main_test_search(container, recipes_path, deep, anaconda_channel, github_repo)

        package_tests["commands"] += tests.get("commands", [])  # not a very nice solution but probably the simplest

        # Given that this could be a mix of Python and Perl packages, translate imports to commands

        for imp in tests.get("imports", []):

            package_tests["imports"].append(f"{tests['import_lang']} 'import {imp}'")

            package_tests["commands"].append(argv_to_str(import_test_to_command_list(tests["import_lang"], imp)))



    return package_tests