Unverified Commit a83af33f authored by mvdbeek's avatar mvdbeek
Browse files

Reload toolbox after forking 

When adding gunicorn workers or restarting workers the new workers need
to load up the current toolbox, as it may have changed from when the
master process became ready.

To test this, start gunicorn with `--preload` and `--max-requests 50`.
After the instance has booted change the tool version in a single tool.
The tool watcher will pick this up and you get the new tool version.

Now trigger the worker replacement by making 50 requests. Without
this commit you would get the old version.
parent 27d37db6

lib/galaxy/app.py

+5 −0
Original line number Diff line number Diff line
@@ -84,6 +84,7 @@ from galaxy.objectstore import ( 
)

from galaxy.queue_worker import (

    GalaxyQueueWorker,

    reload_toolbox,

    send_local_control_task,

)

from galaxy.quota import (
@@ -746,6 +747,10 @@ class UniverseApplication(StructuredApp, GalaxyManagerApplication): 
        # Start web stack message handling

        self.application_stack.register_postfork_function(self.application_stack.start)

        self.application_stack.register_postfork_function(self.queue_worker.bind_and_start)

        # Reload toolbox to pick up changes to toolbox made after master was ready

        self.application_stack.register_postfork_function(

            lambda: reload_toolbox(self, save_integrated_tool_panel=False), post_fork_only=True

        )

        # Delay toolbox index until after startup

        self.application_stack.register_postfork_function(

            lambda: send_local_control_task(self, "rebuild_toolbox_search_index")

lib/galaxy/web_stack/__init__.py

+7 −6
Original line number Diff line number Diff line
@@ -43,7 +43,8 @@ class ApplicationStack: 
        return {}



    @classmethod

    def register_postfork_function(cls, f, *args, **kwargs):

    def register_postfork_function(cls, f, *args, post_fork_only=False, **kwargs):

        if not post_fork_only:

            f(*args, **kwargs)



    def __init__(self, app=None, config=None):
@@ -191,7 +192,7 @@ class GunicornApplicationStack(ApplicationStack): 
    late_postfork_thread: threading.Thread



    @classmethod

    def register_postfork_function(cls, f, *args, **kwargs):

    def register_postfork_function(cls, f, *args, post_fork_only=False, **kwargs):

        # do_post_fork determines if we need to run postfork functions

        if cls.do_post_fork:

            # if so, we call ApplicationStack.late_postfork once after forking ...
@@ -199,7 +200,7 @@ class GunicornApplicationStack(ApplicationStack): 
                os.register_at_fork(after_in_child=cls.late_postfork)

            # ... and store everything we need to run in ApplicationStack.postfork_functions

            cls.postfork_functions.append(lambda: f(*args, **kwargs))

        else:

        elif not post_fork_only:

            f(*args, **kwargs)



    @classmethod
@@ -300,8 +301,8 @@ def application_stack_log_formatter(): 
    return logging.Formatter(fmt=application_stack_class().log_format)





def register_postfork_function(f, *args, **kwargs):

    application_stack_class().register_postfork_function(f, *args, **kwargs)

def register_postfork_function(f, *args, post_fork_only=False, **kwargs):

    application_stack_class().register_postfork_function(f, *args, post_fork_only=post_fork_only**kwargs)





def get_app_kwds(config_section, app_name=None):