Unverified Commit a0d6e87f authored by Nicola Soranzo's avatar Nicola Soranzo
Browse files

Fix testing importing modules in containers

parent 1fedac75

lib/galaxy/tool_util/deps/mulled/get_tests.py

+14 −3
Original line number Diff line number Diff line
@@ -12,6 +12,7 @@ from glob import glob 
from typing import (

    Any,

    Dict,

    List,

    Optional,

)
@@ -26,6 +27,7 @@ except ImportError: 
    UndefinedError = Exception  # type: ignore[assignment,misc]



from galaxy.util import unicodify

from galaxy.util.commands import argv_to_str

from .util import (

    get_files_from_conda_package,

    MULLED_SOCKET_TIMEOUT,
@@ -66,6 +68,7 @@ def get_commands_from_yaml(yaml_content: bytes) -> Optional[Dict[str, Any]]: 
        return None



    # need to know what scripting languages are needed to run the container

    package_tests["import_lang"] = "python -c"  # python by default

    try:

        requirements = list(meta_yaml["requirements"]["run"])

    except (KeyError, TypeError):
@@ -77,8 +80,6 @@ def get_commands_from_yaml(yaml_content: bytes) -> Optional[Dict[str, Any]]: 
                break

            # elif ... :

            # other languages if necessary ... hopefully python and perl should suffice though

        else:  # python by default

            package_tests["import_lang"] = "python -c"

    return package_tests
@@ -260,6 +261,15 @@ def main_test_search( 
    return {"container": container}





def import_test_to_command_list(import_lang: str, import_: str) -> List[str]:

    if import_lang == "python -c":

        return ["python", "-c", f"import {import_}"]

    elif import_lang == "perl -e":

        return ["perl", "-e", f"use {import_}"]

    else:

        raise ValueError(f"Unsupported import_lang '{import_lang}'")





def hashed_test_search(

    container, recipes_path=None, deep=False, anaconda_channel="bioconda", github_repo="bioconda/bioconda-recipes"

):
@@ -294,7 +304,8 @@ def hashed_test_search( 
    for container in containers:

        tests = main_test_search(container, recipes_path, deep, anaconda_channel, github_repo)

        package_tests["commands"] += tests.get("commands", [])  # not a very nice solution but probably the simplest

        # Given that this could be a mix of Python and Perl packages, translate imports to commands

        for imp in tests.get("imports", []):

            package_tests["imports"].append(f"{tests['import_lang']} 'import {imp}'")

            package_tests["commands"].append(argv_to_str(import_test_to_command_list(tests["import_lang"], imp)))



    return package_tests

lib/galaxy/tool_util/deps/mulled/mulled_update_singularity_containers.py

+5 −1
Original line number Diff line number Diff line
@@ -17,6 +17,7 @@ from typing import ( 
from galaxy.util import unicodify

from .get_tests import (

    hashed_test_search,

    import_test_to_command_list,

    main_test_search,

)
@@ -77,7 +78,10 @@ def singularity_container_test( 


                for imp in test.get("imports", []):

                    try:

                        check_output(exec_command + [test["import_lang"], f"import {imp}"], stderr=subprocess.STDOUT)

                        check_output(

                            exec_command + import_test_to_command_list(test["import_lang"], imp),

                            stderr=subprocess.STDOUT,

                        )

                    except subprocess.CalledProcessError as e:

                        errors.append({"import": imp, "output": unicodify(e.output)})

                        test_passed = False

test/unit/tool_util/mulled/test_mulled_update_singularity_containers.py

+8 −18
Original line number Diff line number Diff line 
import os

from typing import (

    Any,

    Dict,

)



import pytest



from galaxy.tool_util.deps.mulled.get_tests import main_test_search

from galaxy.tool_util.deps.mulled.mulled_update_singularity_containers import (

    docker_to_singularity,

    get_list_from_file,
@@ -38,19 +35,12 @@ def test_docker_to_singularity(tmp_path) -> None: 
@external_dependency_management

@pytest.mark.skipif(not which("singularity"), reason="requires singularity but singularity not on PATH")

def test_singularity_container_test(tmp_path) -> None:

    tests: Dict[str, Dict[str, Any]] = {

        "pybigwig:0.1.11--py36_0": {

            "imports": ["pyBigWig"],

            "commands": ['python -c "import pyBigWig; assert(pyBigWig.numpy == 1); assert(pyBigWig.remote == 1)"'],

            "import_lang": "python -c",

        },

        "samtools:1.0--1": {

            "commands": ["samtools --help"],

            "import_lang": "python -c",

            "container": "samtools:1.0--1",

        },

        "yasm:1.3.0--0": {},

    }

    containers = [

        "pybigwig:0.3.22--py36h54a71a5_0",  # test Python imports

        "samtools:1.0--1",

        "yasm:1.3.0--0",  # test missing tests

    ]

    tests = {container: main_test_search(container) for container in containers}

    for n in tests.keys():

        docker_to_singularity(n, "singularity", tmp_path, no_sudo=True)

    results = singularity_container_test(
@@ -59,5 +49,5 @@ def test_singularity_container_test(tmp_path) -> None: 
        tmp_path,

    )

    assert "samtools:1.0--1" in results["passed"]

    assert results["failed"][0]["imports"] == ["pyBigWig"]

    assert "pybigwig:0.3.22--py36h54a71a5_0" in results["passed"]

    assert "yasm:1.3.0--0" in results["notest"]