Unverified Commit 49e6f75b authored by John Chilton's avatar John Chilton Committed by GitHub
Browse files

Merge pull request #16620 from mvdbeek/reload_toolbox_postfork 

[23.1] Reload toolbox after forking when using `--preload`
parents 9d308f5b 5ab925b6

lib/galaxy/app.py

+5 −0
Original line number Diff line number Diff line
@@ -84,6 +84,7 @@ from galaxy.objectstore import ( 
)

from galaxy.queue_worker import (

    GalaxyQueueWorker,

    reload_toolbox,

    send_local_control_task,

)

from galaxy.quota import (
@@ -746,6 +747,10 @@ class UniverseApplication(StructuredApp, GalaxyManagerApplication): 
        # Start web stack message handling

        self.application_stack.register_postfork_function(self.application_stack.start)

        self.application_stack.register_postfork_function(self.queue_worker.bind_and_start)

        # Reload toolbox to pick up changes to toolbox made after master was ready

        self.application_stack.register_postfork_function(

            lambda: reload_toolbox(self, save_integrated_tool_panel=False), post_fork_only=True

        )

        # Delay toolbox index until after startup

        self.application_stack.register_postfork_function(

            lambda: send_local_control_task(self, "rebuild_toolbox_search_index")

lib/galaxy/tools/cache.py

+36 −5
Original line number Diff line number Diff line
@@ -6,6 +6,10 @@ import sqlite3 
import tempfile

import zlib

from threading import Lock

from typing import (

    Dict,

    Optional,

)



from sqlitedict import SqliteDict
@@ -134,7 +138,7 @@ class ToolCache: 


    def __init__(self):

        self._lock = Lock()

        self._hash_by_tool_paths = {}

        self._hash_by_tool_paths: Dict[str, ToolHash] = {}

        self._tools_by_path = {}

        self._tool_paths_by_id = {}

        self._macro_paths_by_id = {}
@@ -195,9 +199,12 @@ class ToolCache: 
        try:

            new_mtime = os.path.getmtime(config_filename)

            tool_hash = self._hash_by_tool_paths.get(config_filename)

            if tool_hash.modtime < new_mtime:

                if md5_hash_file(config_filename) != tool_hash.hash:

            if tool_hash and tool_hash.modtime_less_than(new_mtime):

                if not tool_hash.hash_equals(md5_hash_file(config_filename)):

                    return True

                else:

                    # No change of content, so not necessary to calculate the md5 checksum every time

                    tool_hash.modtime = new_mtime

            tool = self._tools_by_path[config_filename]

            for macro_path in tool._macro_paths:

                new_mtime = os.path.getmtime(macro_path)
@@ -256,13 +263,37 @@ class ToolCache: 




class ToolHash:

    def __init__(self, path, modtime=None, lazy_hash=False):

    def __init__(self, path: str, modtime: Optional[float] = None, lazy_hash: bool = False):

        self.path = path

        self.modtime = modtime or os.path.getmtime(path)

        self._modtime = modtime or os.path.getmtime(path)

        self._tool_hash = None

        if not lazy_hash:

            self.hash  # noqa: B018



    def modtime_less_than(self, other_modtime: float):

        if self._modtime is None:

            # For the purposes of the tool cache,

            # if we haven't seen the modtime we consider it not equal

            return True

        return self._modtime < other_modtime



    def hash_equals(self, other_hash: Optional[str]):

        if self._tool_hash is None or other_hash is None:

            # For the purposes of the tool cache,

            # if we haven't seen the hash yet we consider it not equal

            return False

        return self.hash == other_hash



    @property

    def modtime(self) -> float:

        if self._modtime is None:

            self._modtime = os.path.getmtime(self.path)

        return self._modtime



    @modtime.setter

    def modtime(self, new_value: float):

        self._modtime = new_value



    @property

    def hash(self):

        if self._tool_hash is None:

lib/galaxy/web_stack/__init__.py

+7 −6
Original line number Diff line number Diff line
@@ -43,7 +43,8 @@ class ApplicationStack: 
        return {}



    @classmethod

    def register_postfork_function(cls, f, *args, **kwargs):

    def register_postfork_function(cls, f, *args, post_fork_only=False, **kwargs):

        if not post_fork_only:

            f(*args, **kwargs)



    def __init__(self, app=None, config=None):
@@ -191,7 +192,7 @@ class GunicornApplicationStack(ApplicationStack): 
    late_postfork_thread: threading.Thread



    @classmethod

    def register_postfork_function(cls, f, *args, **kwargs):

    def register_postfork_function(cls, f, *args, post_fork_only=False, **kwargs):

        # do_post_fork determines if we need to run postfork functions

        if cls.do_post_fork:

            # if so, we call ApplicationStack.late_postfork once after forking ...
@@ -199,7 +200,7 @@ class GunicornApplicationStack(ApplicationStack): 
                os.register_at_fork(after_in_child=cls.late_postfork)

            # ... and store everything we need to run in ApplicationStack.postfork_functions

            cls.postfork_functions.append(lambda: f(*args, **kwargs))

        else:

        elif not post_fork_only:

            f(*args, **kwargs)



    @classmethod
@@ -300,8 +301,8 @@ def application_stack_log_formatter(): 
    return logging.Formatter(fmt=application_stack_class().log_format)





def register_postfork_function(f, *args, **kwargs):

    application_stack_class().register_postfork_function(f, *args, **kwargs)

def register_postfork_function(f, *args, post_fork_only=False, **kwargs):

    application_stack_class().register_postfork_function(f, *args, post_fork_only=post_fork_only**kwargs)





def get_app_kwds(config_section, app_name=None):