Unverified Commit 53b56a87 authored by Marius van den Beek's avatar Marius van den Beek Committed by GitHub
Browse files

Merge pull request #18279 from mvdbeek/server_external_hgweb 

[23.1] More fixes for running the TS with external hgweb
parents aeaafc0c 992777ed

lib/galaxy/web/framework/middleware/error.py

+33 −34
Original line number Diff line number Diff line
@@ -9,6 +9,7 @@ Error handler middleware 
When an exception is thrown from the wrapper application, this logs

the exception and displays an error page.

"""

import logging

import sys

import traceback

from io import StringIO
@@ -25,6 +26,8 @@ from paste.exceptions import ( 
    reporter,

)



log = logging.getLogger(__name__)



__all__ = ("ErrorMiddleware", "handle_exception")
@@ -170,10 +173,11 @@ class ErrorMiddleware: 
                for expect in environ.get("paste.expected_exceptions", []):

                    if isinstance(exc_info[1], expect):

                        raise

                log.exception("Uncaught Exception")

                start_response("500 Internal Server Error", [("content-type", "text/html")], exc_info)

                # @@: it would be nice to deal with bad content types here

                response = self.exception_handler(exc_info, environ)

                return [response]

                return [response.encode(errors="ignore")]

            finally:

                # clean up locals...

                exc_info = None
@@ -184,7 +188,7 @@ class ErrorMiddleware: 
            return app_iter

        return CatchingIter(app_iter, environ, sr_checker, self)



    def exception_handler(self, exc_info, environ):

    def exception_handler(self, exc_info, environ) -> str:

        simple_html_error = False

        if self.xmlhttp_key:

            get_vars = wsgilib.parse_querystring(environ)
@@ -193,7 +197,6 @@ class ErrorMiddleware: 
        return handle_exception(

            exc_info,

            environ["wsgi.errors"],

            html=True,

            debug_mode=self.debug_mode,

            error_email=self.error_email,

            error_log=self.error_log,
@@ -344,7 +347,6 @@ class Supplement: 
def handle_exception(

    exc_info,

    error_stream,

    html=True,

    debug_mode=False,

    error_email=None,

    error_log=None,
@@ -358,7 +360,7 @@ def handle_exception( 
    error_message=None,

    simple_html_error=False,

    environ=None,

):

) -> str:

    """

    For exception handling outside of a web context
@@ -389,28 +391,27 @@ def handle_exception( 
            smtp_use_tls=smtp_use_tls,

            subject_prefix=error_subject_prefix,

        )

        rep_err = send_report(rep, exc_data, html=html)

        rep_err = send_report(rep, exc_data, html=True)

        if rep_err:

            extra_data += rep_err

        else:

            reported = True

    if error_log:

        rep = reporter.LogReporter(filename=error_log)

        rep_err = send_report(rep, exc_data, html=html)

        rep_err = send_report(rep, exc_data, html=True)

        if rep_err:

            extra_data += rep_err

        else:

            reported = True

    if show_exceptions_in_wsgi_errors:

        rep = reporter.FileReporter(file=error_stream)

        rep_err = send_report(rep, exc_data, html=html)

        rep_err = send_report(rep, exc_data, html=True)

        if rep_err:

            extra_data += rep_err

        else:

            reported = True

    else:

        error_stream.write(f"Error - {exc_data.exception_type}: {exc_data.exception_value}\n")

    if html:

    if debug_mode and simple_html_error:

        return_error = formatter.format_html(

            exc_data, include_hidden_frames=False, include_reusable=False, show_extra_data=False
@@ -435,8 +436,6 @@ def handle_exception( 
            extra += f"<b><large>GURU MEDITATION: #{environ['sentry_event_id']}</large></b>"

        extra += "</p>"

        return_error = error_template("", msg, extra)

    else:

        return_error = None

    if not reported and error_stream:

        err_report = formatter.format_text(exc_data, show_hidden_frames=True)

        err_report += f"\n{'-' * 60}\n"

lib/tool_shed/util/hg_util.py

+1 −2
Original line number Diff line number Diff line
@@ -71,14 +71,13 @@ def get_hgrc_path(repo_path): 
    return os.path.join(repo_path, ".hg", "hgrc")





def create_hgrc_file(app, repository):

def create_hgrc_file(app, repository, repo_path):

    # Since we support both http and https, we set `push_ssl` to False to

    # override the default (which is True) in the Mercurial API.

    # The hg purge extension purges all files and directories not being tracked

    # by Mercurial in the current repository. It will remove unknown files and

    # empty directories. This is not currently used because it is not supported

    # in the Mercurial API.

    repo_path = repository.repo_path(app)

    hgrc_path = get_hgrc_path(repo_path)

    with open(hgrc_path, "w") as fp:

        fp.write("[paths]\n")

lib/tool_shed/util/hgweb_config.py

+5 −6
Original line number Diff line number Diff line
@@ -5,8 +5,6 @@ import shutil 
import threading

from datetime import date



from galaxy.util import unicodify



log = logging.getLogger(__name__)



new_hgweb_config_template = """
@@ -20,6 +18,7 @@ class HgWebConfigManager: 
        self.hgweb_config_dir = None

        self.in_memory_config = None

        self.lock = threading.Lock()

        self.hgweb_repo_prefix = None



    def add_entry(self, lhs, rhs):

        """Add an entry in the hgweb.config file for a new repository."""
@@ -35,8 +34,8 @@ class HgWebConfigManager: 
            self.in_memory_config.set("paths", lhs, rhs)

            # Persist our in-memory configuration.

            self.write_config()

        except Exception as e:

            log.debug("Exception in HgWebConfigManager.add_entry(): %s", unicodify(e))

        except Exception:

            log.exception("Exception in HgWebConfigManager.add_entry()")

        finally:

            self.lock.release()
@@ -51,8 +50,8 @@ class HgWebConfigManager: 
            self.in_memory_config.set("paths", new_lhs, new_rhs)

            # Persist our in-memory configuration.

            self.write_config()

        except Exception as e:

            log.debug("Exception in HgWebConfigManager.change_entry(): %s", unicodify(e))

        except Exception:

            log.exception("Exception in HgWebConfigManager.change_entry()")

        finally:

            self.lock.release()

lib/tool_shed/util/repository_util.py

+31 −38
Original line number Diff line number Diff line
@@ -2,6 +2,7 @@ import configparser 
import logging

import os

import re

import tempfile

from typing import (

    Optional,

    Tuple,
@@ -154,23 +155,18 @@ def create_repo_info_dict( 
def create_repository_admin_role(app: "ToolShedApp", repository: "Repository"):

    """

    Create a new role with name-spaced name based on the repository name and its owner's public user

    name.  This will ensure that the tole name is unique.

    name.  This will ensure that the role name is unique.

    """

    sa_session = app.model.session

    name = get_repository_admin_role_name(str(repository.name), str(repository.user.username))

    description = "A user or group member with this role can administer this repository."

    role = app.model.Role(name=name, description=description, type=app.model.Role.types.SYSTEM)

    sa_session.add(role)

    session = sa_session()

    with transaction(session):

        session.commit()

    # Associate the role with the repository owner.

    app.model.UserRoleAssociation(repository.user, role)

    # Associate the role with the repository.

    rra = app.model.RepositoryRoleAssociation(repository, role)

    sa_session.add(rra)

    with transaction(session):

        session.commit()

    return role
@@ -180,7 +176,7 @@ def create_repository( 
    type: str,

    description,

    long_description,

    user_id,

    user,

    category_ids=None,

    remote_repository_url=None,

    homepage_url=None,
@@ -196,43 +192,40 @@ def create_repository( 
        homepage_url=homepage_url,

        description=description,

        long_description=long_description,

        user_id=user_id,

        user=user,

    )

    # Flush to get the id.

    sa_session.add(repository)

    session = sa_session()

    with transaction(session):

        session.commit()

    # Create an admin role for the repository.

    create_repository_admin_role(app, repository)

    # Determine the repository's repo_path on disk.

    dir = os.path.join(app.config.file_path, *util.directory_hash_id(repository.id))

    # Create directory if it does not exist.

    if not os.path.exists(dir):

        os.makedirs(dir)

    # Define repo name inside hashed directory.

    repository_path = os.path.join(dir, "repo_%d" % repository.id)

    # Create local repository directory.

    if not os.path.exists(repository_path):

        os.makedirs(repository_path)

    # Create the local repository.

    init_repository(repo_path=repository_path)

    # Add an entry in the hgweb.config file for the local repository.

    lhs = f"{app.config.hgweb_repo_prefix}{repository.user.username}/{repository.name}"

    app.hgweb_config_manager.add_entry(lhs, repository_path)

    # Create a .hg/hgrc file for the local repository.

    create_hgrc_file(app, repository)

    flush_needed = False

    if category_ids:

        # Create category associations

        for category_id in category_ids:

            category = sa_session.query(app.model.Category).get(app.security.decode_id(category_id))

            rca = app.model.RepositoryCategoryAssociation(repository, category)

            sa_session.add(rca)

            flush_needed = True

    if flush_needed:

    # Create an admin role for the repository.

    create_repository_admin_role(app, repository)

    # Create a temporary repo_path on disk.

    repository_path = tempfile.mkdtemp(

        dir=app.config.file_path,

        prefix=f"{repository.user.username}-{repository.name}",

    )

    # Create the local repository.

    init_repository(repo_path=repository_path)

    # Create a .hg/hgrc file for the local repository.

    create_hgrc_file(app, repository, repo_path=repository_path)

    # Add an entry in the hgweb.config file for the local repository.

    lhs = f"{app.config.hgweb_repo_prefix}{repository.user.username}/{repository.name}"

    # Flush to get the id.

    session = sa_session()

    with transaction(session):

        session.commit()

    dir = os.path.join(app.config.file_path, *util.directory_hash_id(repository.id))

    # Define repo name inside hashed directory.

    final_repository_path = os.path.join(dir, "repo_%d" % repository.id)

    # Create final repository directory.

    if not os.path.exists(final_repository_path):

        os.makedirs(final_repository_path)

    os.rename(repository_path, final_repository_path)

    app.hgweb_config_manager.add_entry(lhs, final_repository_path)

    # Update the repository registry.

    app.repository_registry.add_entry(repository)

    message = f"Repository <b>{escape(str(repository.name))}</b> has been created."
@@ -505,8 +498,8 @@ def update_repository(trans: "ProvidesUserContext", id: str, **kwds) -> Tuple[Op 


        repo_dir = repository.repo_path(app)

        # Change the entry in the hgweb.config file for the repository.

        old_lhs = f"repos/{repository.user.username}/{repository.name}"

        new_lhs = f"repos/{repository.user.username}/{kwds['name']}"

        old_lhs = f"{trans.app.config.hgweb_repo_prefix}{repository.user.username}/{repository.name}"

        new_lhs = f"{trans.app.config.hgweb_repo_prefix}{repository.user.username}/{kwds['name']}"

        trans.app.hgweb_config_manager.change_entry(old_lhs, new_lhs, repo_dir)



        # Change the entry in the repository's hgrc file.

lib/tool_shed/util/shed_index.py

+4 −4
Original line number Diff line number Diff line
@@ -33,7 +33,7 @@ def _get_or_create_index(whoosh_index_dir): 
    return get_or_create_index(whoosh_index_dir, repo_schema), get_or_create_index(tool_index_dir, tool_schema)





def build_index(whoosh_index_dir, file_path, hgweb_config_dir, dburi, **kwargs):

def build_index(whoosh_index_dir, file_path, hgweb_config_dir, hgweb_repo_prefix, dburi, **kwargs):

    """

    Build two search indexes simultaneously

    One is for repositories and the other for tools.
@@ -51,7 +51,7 @@ def build_index(whoosh_index_dir, file_path, hgweb_config_dir, dburi, **kwargs): 


    execution_timer = ExecutionTimer()

    with repo_index.searcher() as searcher:

        for repo in get_repos(sa_session, file_path, hgweb_config_dir, **kwargs):

        for repo in get_repos(sa_session, file_path, hgweb_config_dir, hgweb_repo_prefix, **kwargs):

            tools_list = repo.pop("tools_list")

            repo_id = repo["id"]

            indexed_document = searcher.document(id=repo_id)
@@ -85,7 +85,7 @@ def build_index(whoosh_index_dir, file_path, hgweb_config_dir, dburi, **kwargs): 
    return repos_indexed, tools_indexed





def get_repos(sa_session, file_path, hgweb_config_dir, **kwargs):

def get_repos(sa_session, file_path, hgweb_config_dir, hgweb_repo_prefix, **kwargs):

    """

    Load repos from DB and included tools from .xml configs.

    """
@@ -126,7 +126,7 @@ def get_repos(sa_session, file_path, hgweb_config_dir, **kwargs): 


        # Load all changesets of the repo for lineage.

        repo_path = os.path.join(

            hgweb_config_dir, hgwcm.get_entry(os.path.join("repos", repo.user.username, repo.name))

            hgweb_config_dir, hgwcm.get_entry(os.path.join(hgweb_repo_prefix, repo.user.username, repo.name))

        )

        hg_repo = hg.repository(ui.ui(), repo_path.encode("utf-8"))

        lineage = []