Unverified Commit cdba4a09 authored by Marius van den Beek's avatar Marius van den Beek Committed by GitHub
Browse files

Merge pull request #16640 from mvdbeek/lazy_tool_data_tables 

[23.0] Lazy load tool data tables in celery worker
parents 1e969700 5684562f

lib/galaxy/app.py

+2 −2
Original line number Diff line number Diff line
@@ -576,8 +576,6 @@ class GalaxyManagerApplication(MinimalManagerApp, MinimalGalaxyApplication): 


        self._configure_tool_shed_registry()

        self._register_singleton(tool_shed_registry.Registry, self.tool_shed_registry)

        # Tool Data Tables

        self._configure_tool_data_tables(from_shed_config=False)



    def _configure_tool_shed_registry(self) -> None:

        # Set up the tool sheds registry
@@ -630,6 +628,8 @@ class UniverseApplication(StructuredApp, GalaxyManagerApplication): 
        )

        self.api_keys_manager = self._register_singleton(ApiKeyManager)



        # Tool Data Tables

        self._configure_tool_data_tables(from_shed_config=False)

        # Load dbkey / genome build manager

        self._configure_genome_builds(data_table_name="__dbkeys__", load_old_style=True)

lib/galaxy/celery/tasks.py

+10 −0
Original line number Diff line number Diff line
@@ -34,6 +34,7 @@ from galaxy.managers.model_stores import ModelStoreManager 
from galaxy.managers.tool_data import ToolDataImportManager

from galaxy.metadata.set_metadata import set_metadata_portable

from galaxy.model.scoped_session import galaxy_scoped_session

from galaxy.queue_worker import GalaxyQueueWorker

from galaxy.schema.tasks import (

    ComputeDatasetHashTaskRequest,

    GenerateHistoryContentDownload,
@@ -58,6 +59,11 @@ from galaxy.web.short_term_storage import ShortTermStorageMonitor 
log = get_logger(__name__)





@lru_cache()

def setup_data_table_manager(app):

    app._configure_tool_data_tables(from_shed_config=False)





@lru_cache()

def cached_create_tool_from_representation(app, raw_tool_source):

    return create_tool_from_representation(
@@ -369,6 +375,7 @@ def compute_dataset_hash( 


@galaxy_task(action="import a data bundle")

def import_data_bundle(

    app: MinimalManagerApp,

    hda_manager: HDAManager,

    ldda_manager: LDDAManager,

    tool_data_import_manager: ToolDataImportManager,
@@ -378,6 +385,7 @@ def import_data_bundle( 
    id: Optional[int] = None,

    tool_data_file_path: Optional[str] = None,

):

    setup_data_table_manager(app)

    if src == "uri":

        assert uri

        tool_data_import_manager.import_data_bundle_by_uri(config, uri, tool_data_file_path=tool_data_file_path)
@@ -389,6 +397,8 @@ def import_data_bundle( 
        else:

            dataset = ldda_manager.by_id(id)

        tool_data_import_manager.import_data_bundle_by_dataset(config, dataset, tool_data_file_path=tool_data_file_path)

    queue_worker = GalaxyQueueWorker(app)

    queue_worker.send_control_task("reload_tool_data_tables")





@galaxy_task(action="pruning history audit table")

test/integration/test_tool_data_delete.py

+2 −0
Original line number Diff line number Diff line
@@ -45,6 +45,8 @@ class DataManagerIntegrationTestCase(integration_util.IntegrationTestCase): 
        return show_response.json()["fields"]



    def _testbeta_field_count(self) -> int:

        # We need to wait for the reload message to reach the control message consumer

        time.sleep(1)

        return len(self._testbase_fields())