Unverified Commit 1d59dcb4 authored by Nicola Soranzo's avatar Nicola Soranzo
Browse files

Fix the ``singularity_container_test()`` function 

Fix ``test_singularity_container_test`` unit test, which failed with:

```
test/unit/tool_util/mulled/test_mulled_update_singularity_containers.py:48:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
lib/galaxy/tool_util/deps/mulled/mulled_update_singularity_containers.py:62: in singularity_container_test
    check_output(exec_command.extend(["bash", "-c", test_command]), stderr=subprocess.STDOUT)
/usr/lib/python3.8/subprocess.py:415: in check_output
    return run(*popenargs, stdout=PIPE, timeout=timeout, check=True,
/usr/lib/python3.8/subprocess.py:493: in run
    with Popen(*popenargs, **kwargs) as process:
/usr/lib/python3.8/subprocess.py:858: in __init__
    self._execute_child(args, executable, preexec_fn, close_fds,

>           args = list(args)
E           TypeError: 'NoneType' object is not iterable

/usr/lib/python3.8/subprocess.py:1569: TypeError
```

Broken in commit 1dc1e5f2 .

Also:
- Use `tmp_path` pytest fixture.
- Add type annotations.
- Code refactorings.
- Install singularity in the mulled GitHub workflow so that the 2 singularity
  unit tests are not skipped any more.
parent 591ace57

.github/workflows/mulled.yaml

+2 −0
Original line number Diff line number Diff line
@@ -42,6 +42,8 @@ jobs: 
        with:

          path: .tox

          key: tox-cache-${{ runner.os }}-${{ steps.full-python-version.outputs.version }}-${{ hashFiles('galaxy root/requirements.txt') }}-mulled

      - name: Install Apptainer's singularity

        uses: eWaterCycle/setup-apptainer@v2

      - name: Install tox

        run: pip install tox

      - name: run tests

lib/galaxy/tool_util/deps/mulled/mulled_update_singularity_containers.py

+31 −26
Original line number Diff line number Diff line 
#!/usr/bin/env python



import argparse

import os

import os.path

import subprocess

import tempfile

from glob import glob

from subprocess import check_output

from typing import (

    Any,

    Dict,

    List,

    Union,

)



from galaxy.util import unicodify

from .get_tests import (
@@ -25,7 +33,7 @@ def docker_to_singularity(container, installation, filepath, no_sudo=False): 
    """

    Convert docker to singularity container.

    """

    cmd = [installation, "build", "/".join((filepath, container)), f"docker://quay.io/biocontainers/{container}"]

    cmd = [installation, "build", os.path.join(filepath, container), f"docker://quay.io/biocontainers/{container}"]

    try:

        if no_sudo:

            check_output(cmd, stderr=subprocess.STDOUT)
@@ -36,43 +44,40 @@ def docker_to_singularity(container, installation, filepath, no_sudo=False): 
        raise Exception(f"Docker to Singularity conversion failed.\nOutput was:\n{unicodify(e.output)}")





def singularity_container_test(tests, installation, filepath):

def singularity_container_test(

    tests: Dict[str, Dict[str, Any]], installation: str, filepath: Union[str, os.PathLike]

) -> Dict[str, List]:

    """

    Run tests, record if they pass or fail

    """

    test_results = {"passed": [], "failed": [], "notest": []}

    test_results: Dict[str, List] = {"passed": [], "failed": [], "notest": []}



    # create a 'sanitised home' directory in which the containers may be mounted - see http://singularity.lbl.gov/faq#solution-1-specify-the-home-to-mount

    with tempfile.TemporaryDirectory() as tmpdirname:

        for container, test in tests.items():

            if "commands" not in test and "imports" not in test:

                test_results["notest"].append(container)



            else:

                exec_command = [installation, "exec", "-H", tmpdirname, "/".join((filepath, container))]

                exec_command = [installation, "exec", "-H", tmpdirname, os.path.join(filepath, container)]

                test_passed = True

                errors = []

                if test.get("commands", False):

                    for test_command in test["commands"]:

                for test_command in test.get("commands", []):

                    test_command = test_command.replace("$PREFIX", "/usr/local/")

                    test_command = test_command.replace("${PREFIX}", "/usr/local/")

                    test_command = test_command.replace("$R ", "Rscript ")



                    try:

                            check_output(exec_command.extend(["bash", "-c", test_command]), stderr=subprocess.STDOUT)

                        check_output(exec_command + ["bash", "-c", test_command], stderr=subprocess.STDOUT)

                    except subprocess.CalledProcessError:

                        try:

                                check_output(exec_command.append(test_command), stderr=subprocess.STDOUT)

                            check_output(exec_command + [test_command], stderr=subprocess.STDOUT)

                        except subprocess.CalledProcessError as e:

                            errors.append({"command": test_command, "output": unicodify(e.output)})

                            test_passed = False



                if test.get("imports", False):

                    for imp in test["imports"]:

                for imp in test.get("imports", []):

                    try:

                            check_output(

                                exec_command.extend([test["import_lang"], f"import {imp}"]), stderr=subprocess.STDOUT

                            )

                        check_output(exec_command + [test["import_lang"], f"import {imp}"], stderr=subprocess.STDOUT)

                    except subprocess.CalledProcessError as e:

                        errors.append({"import": imp, "output": unicodify(e.output)})

                        test_passed = False

test/unit/tool_util/mulled/test_mulled_update_singularity_containers.py

+41 −49
Original line number Diff line number Diff line 
import os

import shutil

import tempfile

from typing import (

    Any,

    Dict,

)



import pytest
@@ -13,10 +15,8 @@ from galaxy.util import which 
from ..util import external_dependency_management





def test_get_list_from_file():

    test_dir = tempfile.mkdtemp()

    try:

        list_file = os.path.join(test_dir, "list_file.txt")

def test_get_list_from_file(tmp_path) -> None:

    list_file = os.path.join(tmp_path, "list_file.txt")

    with open(list_file, "w") as f:

        f.write("bbmap:36.84--0\nbiobambam:2.0.42--0\nconnor:0.5.1--py35_0\ndiamond:0.8.26--0\nedd:1.1.18--py27_0")

    assert get_list_from_file(list_file) == [
@@ -26,32 +26,22 @@ def test_get_list_from_file(): 
        "diamond:0.8.26--0",

        "edd:1.1.18--py27_0",

    ]

    finally:

        shutil.rmtree(test_dir)





@external_dependency_management

@pytest.mark.skipif(not which("singularity"), reason="requires singularity but singularity not on PATH")

def test_docker_to_singularity(tmp_path):

    tmp_dir = str(tmp_path)

    docker_to_singularity("abundancebin:1.0.1--0", "singularity", tmp_dir, no_sudo=True)

def test_docker_to_singularity(tmp_path) -> None:

    docker_to_singularity("abundancebin:1.0.1--0", "singularity", tmp_path, no_sudo=True)

    assert tmp_path.joinpath("abundancebin:1.0.1--0").exists()





@external_dependency_management

@pytest.mark.skipif(not which("singularity"), reason="requires singularity but singularity not on PATH")

def test_singularity_container_test(tmp_path):

    test_dir = tempfile.mkdtemp()

    try:

        for n in ["pybigwig:0.1.11--py36_0", "samtools:1.0--1", "yasm:1.3.0--0"]:

            docker_to_singularity(n, "singularity", test_dir, no_sudo=True)

        results = singularity_container_test(

            {

def test_singularity_container_test(tmp_path) -> None:

    tests: Dict[str, Dict[str, Any]] = {

        "pybigwig:0.1.11--py36_0": {

            "imports": ["pyBigWig"],

                    "commands": [

                        'python -c "import pyBigWig; assert(pyBigWig.numpy == 1); assert(pyBigWig.remote == 1)"'

                    ],

            "commands": ['python -c "import pyBigWig; assert(pyBigWig.numpy == 1); assert(pyBigWig.remote == 1)"'],

            "import_lang": "python -c",

        },

        "samtools:1.0--1": {
@@ -60,12 +50,14 @@ def test_singularity_container_test(tmp_path): 
            "container": "samtools:1.0--1",

        },

        "yasm:1.3.0--0": {},

            },

    }

    for n in tests.keys():

        docker_to_singularity(n, "singularity", tmp_path, no_sudo=True)

    results = singularity_container_test(

        tests,

        "singularity",

            test_dir,

        tmp_path,

    )

    assert "samtools:1.0--1" in results["passed"]

    assert results["failed"][0]["imports"] == ["pyBigWig"]

    assert "yasm:1.3.0--0" in results["notest"]
 
    finally:

        shutil.rmtree(test_dir)