Unverified Commit 50d339e2 authored by Nicola Soranzo's avatar Nicola Soranzo
Browse files

Fix jobs API not allowing dates for ``date_range_min`` and ``date_range_max``

Dates (without time) are specifically mentioned in our parameters' documentation:
`Limit listing of jobs to those that are updated after specified date (e.g. '2014-01-01')`

Follow-up on https://github.com/galaxyproject/galaxy/pull/14281 , which
broke backward compatibility as shown in this traceback:

```
_________________________ TestGalaxyJobs.test_get_jobs _________________________
has_gi = <bioblend._tests.TestGalaxyJobs.TestGalaxyJobs testMethod=test_get_jobs>
args = (), kwargs = {}
tools = [{'config_file': '/home/runner/work/bioblend/bioblend/galaxy-release_22.05/tools/data_source/upload.xml', 'description...ata_source/ebi_sra.xml', 'description': 'ENA SRA', 'edam_operations': ['operation_0224'], 'edam_topics': [], ...}, ...]
tool_ids = ['upload1', 'ucsc_table_direct1', 'ucsc_table_direct_archaea1', 'ncbi_datasets_source', 'sra_source', 'ebi_sra_main', ...]
    def wrapped_method(has_gi, *args, **kwargs):
        tools = has_gi.gi.tools.get_tools()
        # In panels by default, so flatten out sections...
        tool_ids = [_["id"] for _ in tools]
        if tool_id not in tool_ids:
            raise unittest.SkipTest(MISSING_TOOL_MESSAGE % tool_id)
>       return method(has_gi, *args, **kwargs)
bioblend/_tests/test_util.py:113:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
bioblend/_tests/TestGalaxyJobs.py:46: in test_get_jobs
    jobs = self.gi.jobs.get_jobs(date_range_max=yesterday.strftime("%Y-%m-%d"), history_id=self.history_id)
bioblend/galaxy/jobs/__init__.py:136: in get_jobs
    return self._get(params=params)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <bioblend.galaxy.jobs.JobsClient object at 0x7f36a493a250>, id = None
deleted = False, contents = False, url = 'http://localhost:8080/api/jobs'
params = {'date_range_max': '2022-07-11', 'history_id': '915ae9a80309f157', 'limit': 500, 'offset': 0}
json = True
    def _get(
        self,
        id: Optional[str] = None,
        deleted: bool = False,
        contents: bool = False,
        url: Optional[str] = None,
        params=None,
        json: bool = True,
    ):
        """
        Do a GET request, composing the URL from ``id``, ``deleted`` and
        ``contents``.  Alternatively, an explicit ``url`` can be provided.
        If ``json`` is set to ``True``, return a decoded JSON object
        (and treat an empty or undecodable response as an error).
        The request will optionally be retried as configured by
        ``max_get_retries`` and ``get_retry_delay``: this offers some
        resilience in the presence of temporary failures.
        :return: The decoded response if ``json`` is set to ``True``, otherwise
          the response object
        """
        if not url:
            url = self._make_url(module_id=id, deleted=deleted, contents=contents)
        attempts_left = self.max_get_retries()
        retry_delay = self.get_retry_delay()
        bioblend.log.debug("GET - attempts left: %s; retry delay: %s", attempts_left, retry_delay)
        msg = ""
        while attempts_left > 0:
            attempts_left -= 1
            try:
                r = self.gi.make_get_request(url, params=params)
            except requests.exceptions.ConnectionError as e:
                msg = str(e)
                r = requests.Response()  # empty Response object used when raising ConnectionError
            else:
                if r.status_code == 200:
                    if not json:
                        return r
                    elif not r.content:
                        msg = "GET: empty response"
                    else:
                        try:
                            return r.json()
                        except ValueError:
                            msg = f"GET: invalid JSON : {r.content!r}"
                else:
                    msg = f"GET: error {r.status_code}: {r.content!r}"
            msg = f"{msg}, {attempts_left} attempts left"
            if attempts_left <= 0:
                bioblend.log.error(msg)
                raise ConnectionError(
                    msg,
                    body=r.text,
>                   status_code=r.status_code,
                )
E               bioblend.ConnectionError: GET: error 400: b'{"err_msg":"1 validation error for Request\\nquery -> date_range_max\\n  invalid datetime format (type=value_error.datetime)","err_code":400008,"validation_errors":[{"loc":["query","date_range_max"],"msg":"invalid datetime format","type":"value_error.datetime"}]}', 0 attempts left: {"err_msg":"1 validation error for Request\nquery -> date_range_max\n  invalid datetime format (type=value_error.datetime)","err_code":400008,"validation_errors":[{"loc":["query","date_range_max"],"msg":"invalid datetime format","type":"value_error.datetime"}]}
bioblend/galaxy/client.py:166: ConnectionError
```

from BioBlend CI tests:

https://github.com/galaxyproject/bioblend/runs/7292858519?check_suite_focus=true
parent c811c17c
Loading
Loading
Loading
Loading
+5 −2
Original line number Diff line number Diff line
import json
import logging
import typing
from datetime import datetime
from datetime import (
    date,
    datetime,
)

from boltons.iterutils import remap
from pydantic import (
@@ -108,7 +111,7 @@ class JobManager:

        def build_and_apply_filters(query, objects, filter_func):
            if objects is not None:
                if isinstance(objects, str) or isinstance(objects, datetime):
                if isinstance(objects, (str, date, datetime)):
                    query = query.filter(filter_func(objects))
                elif isinstance(objects, list):
                    t = []
+6 −3
Original line number Diff line number Diff line
@@ -2,7 +2,10 @@

import json
import re
from datetime import datetime
from datetime import (
    date,
    datetime,
)
from enum import Enum
from typing import (
    Any,
@@ -1109,8 +1112,8 @@ class JobIndexQueryPayload(Model):
    user_id: Optional[DecodedDatabaseIdField] = None
    tool_ids: Optional[List[str]] = None
    tool_ids_like: Optional[List[str]] = None
    date_range_min: Optional[datetime] = None
    date_range_max: Optional[datetime] = None
    date_range_min: Optional[Union[datetime, date]] = None
    date_range_max: Optional[Union[datetime, date]] = None
    history_id: Optional[DecodedDatabaseIdField] = None
    workflow_id: Optional[DecodedDatabaseIdField] = None
    invocation_id: Optional[DecodedDatabaseIdField] = None
+9 −5
Original line number Diff line number Diff line
@@ -5,12 +5,16 @@ API operations on a jobs.
"""

import logging
from datetime import datetime
from datetime import (
    date,
    datetime,
)
from typing import (
    Any,
    Dict,
    List,
    Optional,
    Union,
)

from fastapi import Query
@@ -99,13 +103,13 @@ ToolIdLikeQueryParam: Optional[str] = Query(
    description="Limit listing of jobs to those that match one of the included tool ID sql-like patterns. If none, all are returned",
)

DateRangeMinQueryParam: Optional[datetime] = Query(
DateRangeMinQueryParam: Optional[Union[datetime, date]] = Query(
    default=None,
    title="Date Range Minimum",
    description="Limit listing of jobs to those that are updated after specified date (e.g. '2014-01-01')",
)

DateRangeMaxQueryParam: Optional[datetime] = Query(
DateRangeMaxQueryParam: Optional[Union[datetime, date]] = Query(
    default=None,
    title="Date Range Maximum",
    description="Limit listing of jobs to those that are updated before specified date (e.g. '2014-01-01')",
@@ -187,8 +191,8 @@ class FastAPIJobs:
        view: JobIndexViewEnum = ViewQueryParam,
        tool_id: Optional[str] = ToolIdQueryParam,
        tool_id_like: Optional[str] = ToolIdLikeQueryParam,
        date_range_min: Optional[datetime] = DateRangeMinQueryParam,
        date_range_max: Optional[datetime] = DateRangeMaxQueryParam,
        date_range_min: Optional[Union[datetime, date]] = DateRangeMinQueryParam,
        date_range_max: Optional[Union[datetime, date]] = DateRangeMaxQueryParam,
        history_id: Optional[EncodedDatabaseIdField] = HistoryIdQueryParam,
        workflow_id: Optional[EncodedDatabaseIdField] = WorkflowIdQueryParam,
        invocation_id: Optional[EncodedDatabaseIdField] = InvocationIdQueryParam,
+11 −7
Original line number Diff line number Diff line
@@ -84,21 +84,25 @@ class JobsApiTestCase(ApiTestCase, TestsTools):
    def test_index_date_filter(self, history_id):
        two_weeks_ago = (datetime.datetime.utcnow() - datetime.timedelta(14)).isoformat()
        last_week = (datetime.datetime.utcnow() - datetime.timedelta(7)).isoformat()
        next_week = (datetime.datetime.utcnow() + datetime.timedelta(7)).isoformat()
        today = datetime.datetime.utcnow().isoformat()
        tomorrow = (datetime.datetime.utcnow() + datetime.timedelta(1)).isoformat()
        before = datetime.datetime.utcnow().isoformat()
        today = before[:10]
        tomorrow = (datetime.datetime.utcnow() + datetime.timedelta(1)).isoformat()[:10]
        self.__history_with_new_dataset(history_id)
        after = datetime.datetime.utcnow().isoformat()

        # Test using dates
        jobs = self.__jobs_index(data={"date_range_min": today, "date_range_max": tomorrow})
        assert len(jobs) > 0
        today_job_id = jobs[0]["id"]
        today_job = jobs[0]
        today_job_id = today_job["id"]

        # Test using datetimes
        jobs = self.__jobs_index(data={"date_range_min": before, "date_range_max": after})
        assert today_job_id in map(itemgetter("id"), jobs), f"before: {before}, after: {after}, job: {today_job}"

        jobs = self.__jobs_index(data={"date_range_min": two_weeks_ago, "date_range_max": last_week})
        assert today_job_id not in map(itemgetter("id"), jobs)

        jobs = self.__jobs_index(data={"date_range_min": last_week, "date_range_max": next_week})
        assert today_job_id in map(itemgetter("id"), jobs)

    @uses_test_history(require_new=True)
    def test_index_history(self, history_id):
        self.__history_with_new_dataset(history_id)