Unverified Commit 155c1c6e authored by Dannon's avatar Dannon Committed by GitHub
Browse files

Merge pull request #5378 from jmchilton/resource_requirements 

Allow specification of resource requirements in tools.
parents 69b2bce5 2026b8c2

lib/galaxy/tool_util/cwl/parser.py

+3 −0
Original line number Diff line number Diff line
@@ -459,6 +459,9 @@ class CommandLineToolProxy(ToolProxy): 
                    requirements.append((package["package"], first_version))

        return requirements



    def resource_requirements(self):

        return [r for r in self.requirements if r["class"] == "ResourceRequirement"]



    @property

    def requirements(self):

        return getattr(self._tool, "requirements", [])

lib/galaxy/tool_util/deps/mulled/mulled_build_tool.py

+1 −1
Original line number Diff line number Diff line
@@ -29,7 +29,7 @@ def main(argv=None): 
    parser.add_argument("tool", metavar="TOOL", default=None, help="Path to tool to build mulled image for.")

    args = parser.parse_args()

    tool_source = get_tool_source(args.tool)

    requirements, _ = tool_source.parse_requirements_and_containers()

    requirements, *_ = tool_source.parse_requirements_and_containers()

    targets = requirements_to_mulled_targets(requirements)

    kwds = args_to_mull_targets_kwds(args)

    mull_targets(targets, **kwds)

lib/galaxy/tool_util/deps/requirements.py

+98 −4
Original line number Diff line number Diff line 
import copy

from typing import (

    Callable,

    cast,

    Dict,

    List,

    Optional,

    Union,

)



from typing_extensions import (

    get_args,

    Literal,

)



from galaxy.util import (

    asbool,
@@ -204,20 +217,91 @@ class ContainerDescription: 
        return f"ContainerDescription[identifier={self.identifier},type={self.type}]"





ResourceType = Literal[

    "cores_min",

    "cores_max",

    "ram_min",

    "ram_max",

    "tmpdir_min",

    "tmpdir_max",

]

VALID_RESOURCE_TYPES = get_args(ResourceType)





class ResourceRequirement:

    def __init__(self, value_or_expression: Union[int, float, str], resource_type: ResourceType):

        self.value_or_expression = value_or_expression

        if not resource_type:

            raise ValueError("Missing resource requirement type")

        if resource_type not in VALID_RESOURCE_TYPES:

            raise ValueError(f"Invalid resource requirement type '{resource_type}'")

        self.resource_type = resource_type

        try:

            float(self.value_or_expression)

            self.runtime_required = False

        except ValueError:

            self.runtime_required = True



    @staticmethod

    def from_dict(resource_dict):

        resource_type = next(iter(resource_dict.keys()))

        value_or_expression = resource_dict[resource_type]

        return ResourceRequirement(value_or_expression=value_or_expression, resource_type=resource_type)



    def get_value(self, runtime: Optional[Dict] = None, js_evaluator: Optional[Callable] = None):

        if self.runtime_required:

            # TODO: hook up evaluator

            # return js_evaluator(self.value_or_expression, runtime)

            raise NotImplementedError(

                f"{self.value_or_expression} is not an integer or float value, expressions currently not implemented"

            )

        return float(self.value_or_expression)





def resource_requirements_from_list(requirements) -> List[ResourceRequirement]:

    cwl_to_galaxy = {

        "coresMin": "cores_min",

        "coresMax": "cores_max",

        "ramMin": "ram_min",

        "ramMax": "ram_max",

        "tmpdirMin": "tmpdir_min",

        "tmpdirMax": "tmpdir_max",

    }

    rr = []

    for r in requirements:

        if r.get("class") == "ResourceRequirement":

            valid_key_set = set(cwl_to_galaxy.keys())

        elif r.get("type") == "resource":

            valid_key_set = set(cwl_to_galaxy.values())

        else:

            continue

        for key in valid_key_set.intersection(set(r.keys())):

            value = r[key]

            key = cast(ResourceType, cwl_to_galaxy.get(key, key))

            rr.append(ResourceRequirement(value_or_expression=value, resource_type=key))

    return rr





def parse_requirements_from_dict(root_dict):

    requirements = root_dict.get("requirements", [])

    resource_requirements = resource_requirements_from_list(requirements)

    containers = root_dict.get("containers", [])

    return ToolRequirements.from_list(requirements), [ContainerDescription.from_dict(c) for c in containers]

    return (

        ToolRequirements.from_list(requirements),

        [ContainerDescription.from_dict(c) for c in containers],

        resource_requirements,

    )





def parse_requirements_from_xml(xml_root):

def parse_requirements_from_xml(xml_root, parse_resources=False):

    """

    Parses requirements, containers and optionally resource requirements from Xml tree.



    >>> from galaxy.util import parse_xml_string

    >>> def load_requirements(contents):

    >>> def load_requirements(contents, parse_resources=False):

    ...     contents_document = '''<tool><requirements>%s</requirements></tool>'''

    ...     root = parse_xml_string(contents_document % contents)

    ...     return parse_requirements_from_xml(root)

    ...     return parse_requirements_from_xml(root, parse_resources=parse_resources)

    >>> reqs, containers = load_requirements('''<requirement>bwa</requirement>''')

    >>> reqs[0].name

    'bwa'
@@ -252,10 +336,20 @@ def parse_requirements_from_xml(xml_root): 
        container_elems = requirements_elem.findall("container")



    containers = [container_from_element(c) for c in container_elems]

    if parse_resources:

        resource_elems = requirements_elem.findall("resource") if requirements_elem else []

        resources = [resource_from_element(r) for r in resource_elems]

        return requirements, containers, resources



    return requirements, containers





def resource_from_element(resource_elem):

    value_or_expression = xml_text(resource_elem)

    resource_type = resource_elem.get("type")

    return ResourceRequirement(value_or_expression=value_or_expression, resource_type=resource_type)





def container_from_element(container_elem):

    identifier = xml_text(container_elem)

    type = container_elem.get("type", DEFAULT_CONTAINER_TYPE)

lib/galaxy/tool_util/linters/cwl.py

+1 −1
Original line number Diff line number Diff line
@@ -32,7 +32,7 @@ def lint_new_draft(tool_source, lint_ctx): 




def lint_docker_image(tool_source, lint_ctx):

    _, containers = tool_source.parse_requirements_and_containers()

    _, containers, *_ = tool_source.parse_requirements_and_containers()

    if len(containers) == 0:

        lint_ctx.warn("Tool does not specify a DockerPull source.")

    else:

lib/galaxy/tool_util/linters/general.py

+4 −1
Original line number Diff line number Diff line
@@ -69,7 +69,7 @@ def lint_general(tool_source, lint_ctx): 
    else:

        lint_ctx.valid(PROFILE_INFO_SPECIFIED_MSG % profile, node=tool_node)



    requirements, containers = tool_source.parse_requirements_and_containers()

    requirements, containers, resource_requirements = tool_source.parse_requirements_and_containers()

    for r in requirements:

        if r.type == "package":

            if not r.name:
@@ -79,3 +79,6 @@ def lint_general(tool_source, lint_ctx): 
            # Warn requirement attributes with leading/trailing whitespace:

            elif r.version != r.version.strip():

                lint_ctx.warn(WARN_WHITESPACE_MSG % ("Requirement version", r.version))

    for rr in resource_requirements:

        if rr.runtime_required:

            lint_ctx.warn("Expressions in resource requirement not supported yet")