Commit 400332ca authored by davelopez's avatar davelopez
Browse files

Fix ftp import export using tasks 

The FTP file source needs the user's context to access the ftp folder. It will also be required in case there are access restrictions on the target file source.
parent cf56a70a

lib/galaxy/files/uris.py

+2 −1
Original line number Diff line number Diff line
@@ -47,13 +47,14 @@ def stream_url_to_file( 
    file_sources: Optional["ConfiguredFileSources"] = None,

    prefix: str = "gx_file_stream",

    dir: Optional[str] = None,

    user_context=None,

) -> str:

    temp_name: str

    if file_sources and file_sources.looks_like_uri(path):

        file_source_path = file_sources.get_file_source_path(path)

        with tempfile.NamedTemporaryFile(prefix=prefix, delete=False, dir=dir) as temp:

            temp_name = temp.name

        file_source_path.file_source.realize_to(file_source_path.path, temp_name)

        file_source_path.file_source.realize_to(file_source_path.path, temp_name, user_context=user_context)

    elif path.startswith("base64://"):

        with tempfile.NamedTemporaryFile(prefix=prefix, delete=False, dir=dir) as temp:

            temp_name = temp.name

lib/galaxy/managers/model_stores.py

+50 −16
Original line number Diff line number Diff line
@@ -3,7 +3,9 @@ from typing import Optional 
from galaxy import model

from galaxy.exceptions import RequestParameterInvalidException

from galaxy.jobs.manager import JobManager

from galaxy.managers.context import ProvidesUserContext

from galaxy.managers.histories import HistoryManager

from galaxy.managers.users import UserManager

from galaxy.model.scoped_session import galaxy_scoped_session

from galaxy.model.store import (

    ImportDiscardedDataType,
@@ -29,6 +31,28 @@ from galaxy.web.short_term_storage import ( 
)





class ModelStoreUserContext(ProvidesUserContext):

    def __init__(self, app: MinimalManagerApp, user: model.User) -> None:

        self._app = app

        self._user = user



    @property

    def app(self):

        return self._app



    @property

    def url_builder(self):

        raise NotImplementedError("URL builder not available in ModelStore context.")



    def get_user(self):

        return self._user



    def set_user(self, user):

        raise NotImplementedError("Cannot change user from ModelStore context.")



    user = property(get_user, set_user)





class ModelStoreManager:

    def __init__(

        self,
@@ -37,12 +61,14 @@ class ModelStoreManager: 
        sa_session: galaxy_scoped_session,

        job_manager: JobManager,

        short_term_storage_monitor: ShortTermStorageMonitor,

        user_manager: UserManager,

    ):

        self._app = app

        self._sa_session = sa_session

        self._job_manager = job_manager

        self._history_manager = history_manager

        self._short_term_storage_monitor = short_term_storage_monitor

        self._user_manager = user_manager



    def setup_history_export_job(self, request: SetupHistoryExportJob):

        history_id = request.history_id
@@ -112,9 +138,13 @@ class ModelStoreManager: 
        model_store_format = request.model_store_format

        export_files = "symlink" if request.include_files else None

        target_uri = request.target_uri

        with model.store.get_export_store_factory(self._app, model_store_format, export_files=export_files)(

            target_uri

        ) as export_store:

        user_context = self._build_user_context(request.user.user_id)

        with model.store.get_export_store_factory(

            self._app,

            model_store_format,

            export_files=export_files,

            user_context=user_context,

        )(target_uri) as export_store:

            invocation = self._sa_session.query(model.WorkflowInvocation).get(request.invocation_id)

            export_store.export_workflow_invocation(

                invocation, include_hidden=request.include_hidden, include_deleted=request.include_deleted
@@ -124,9 +154,10 @@ class ModelStoreManager: 
        model_store_format = request.model_store_format

        export_files = "symlink" if request.include_files else None

        target_uri = request.target_uri

        with model.store.get_export_store_factory(self._app, model_store_format, export_files=export_files)(

            target_uri

        ) as export_store:

        user_context = self._build_user_context(request.user.user_id)

        with model.store.get_export_store_factory(

            self._app, model_store_format, export_files=export_files, user_context=user_context

        )(target_uri) as export_store:

            if request.content_type == HistoryContentType.dataset:

                hda = self._sa_session.query(model.HistoryDatasetAssociation).get(request.content_id)

                export_store.add_dataset(hda)
@@ -140,9 +171,10 @@ class ModelStoreManager: 
        model_store_format = request.model_store_format

        export_files = "symlink" if request.include_files else None

        target_uri = request.target_uri

        with model.store.get_export_store_factory(self._app, model_store_format, export_files=export_files)(

            target_uri

        ) as export_store:

        user_context = self._build_user_context(request.user.user_id)

        with model.store.get_export_store_factory(

            self._app, model_store_format, export_files=export_files, user_context=user_context

        )(target_uri) as export_store:

            history = self._history_manager.by_id(request.history_id)

            export_store.export_history(

                history, include_hidden=request.include_hidden, include_deleted=request.include_deleted
@@ -157,17 +189,13 @@ class ModelStoreManager: 
            history = self._sa_session.query(model.History).get(history_id)

        else:

            history = None

        user_id = request.user.user_id

        if user_id:

            galaxy_user = self._sa_session.query(model.User).get(user_id)

        else:

            galaxy_user = None

        user_context = self._build_user_context(request.user.user_id)

        model_import_store = source_to_import_store(

            request.source_uri,

            self._app,

            galaxy_user,

            import_options,

            model_store_format=request.model_store_format,

            user_context=user_context,

        )

        new_history = history is None and not request.for_library

        if new_history:
@@ -183,6 +211,11 @@ class ModelStoreManager: 
            )

        return object_tracker



    def _build_user_context(self, user_id: int):

        user = self._user_manager.by_id(user_id)

        user_context = ModelStoreUserContext(self._app, user)

        return user_context





def create_objects_from_store(

    app: MinimalManagerApp,
@@ -195,12 +228,13 @@ def create_objects_from_store( 
        discarded_data=ImportDiscardedDataType.FORCE,

        allow_library_creation=for_library,

    )

    user_context = ModelStoreUserContext(app, galaxy_user) if galaxy_user is not None else None

    model_import_store = source_to_import_store(

        payload.store_content_uri or payload.store_dict,

        app=app,

        galaxy_user=galaxy_user,

        import_options=import_options,

        model_store_format=payload.model_store_format,

        user_context=user_context,

    )

    new_history = history is None and not for_library

    if new_history:

lib/galaxy/model/store/__init__.py

+23 −7
Original line number Diff line number Diff line
@@ -39,7 +39,10 @@ from galaxy.exceptions import ( 
    ObjectNotFound,

    RequestParameterInvalidException,

)

from galaxy.files import ConfiguredFileSources

from galaxy.files import (

    ConfiguredFileSources,

    ProvidesUserFileSourcesUserContext,

)

from galaxy.files.uris import stream_url_to_file

from galaxy.model.mapping import GalaxyModelMapping

from galaxy.model.metadata import MetadataCollection
@@ -50,6 +53,7 @@ from galaxy.model.orm.util import ( 
)

from galaxy.model.tags import GalaxyTagHandler

from galaxy.objectstore import ObjectStore

from galaxy.schema.schema import ModelStoreFormat

from galaxy.security.idencoding import IdEncodingHelper

from galaxy.util import (

    FILENAME_VALID_CHARS,
@@ -1603,7 +1607,8 @@ class DirectoryModelExportStore(ModelExportStore): 
        export_files: Optional[str] = None,

        strip_metadata_files: bool = True,

        serialize_jobs: bool = True,

    ):

        user_context=None,

    ) -> None:

        """

        :param export_directory: path to export directory. Will be created if it does not exist.

        :param app: Galaxy App or app-like object. Must be provided if `for_edit` and/or `serialize_dataset_objects` are True
@@ -1627,6 +1632,7 @@ class DirectoryModelExportStore(ModelExportStore): 
            sessionless = True

            security = IdEncodingHelper(id_secret="randomdoesntmatter")



        self.user_context = ProvidesUserFileSourcesUserContext(user_context)

        self.file_sources = file_sources

        self.serialize_jobs = serialize_jobs

        self.sessionless = sessionless
@@ -2154,7 +2160,7 @@ class TarModelExportStore(DirectoryModelExportStore): 
            file_source_path = self.file_sources.get_file_source_path(self.file_source_uri)

            file_source = file_source_path.file_source

            assert os.path.exists(self.out_file)

            file_source.write_from(file_source_path.path, self.out_file)

            file_source.write_from(file_source_path.path, self.out_file, user_context=self.user_context)

        shutil.rmtree(self.temp_output_dir)
@@ -2193,16 +2199,22 @@ class BagArchiveModelExportStore(BagDirectoryModelExportStore): 
            file_source_path = self.file_sources.get_file_source_path(self.file_source_uri)

            file_source = file_source_path.file_source

            assert os.path.exists(rval)

            file_source.write_from(file_source_path.path, rval)

            file_source.write_from(file_source_path.path, rval, user_context=self.user_context)

        shutil.rmtree(self.temp_output_dir)





def get_export_store_factory(app, download_format: str, export_files=None) -> Callable[[str], ModelExportStore]:

def get_export_store_factory(

    app,

    download_format: str,

    export_files=None,

    user_context=None,

) -> Callable[[str], ModelExportStore]:

    export_store_class: Union[Type[TarModelExportStore], Type[BagArchiveModelExportStore]]

    export_store_class_kwds = {

        "app": app,

        "export_files": export_files,

        "serialize_dataset_objects": False,

        "user_context": user_context,

    }

    if download_format in ["tar.gz", "tgz"]:

        export_store_class = TarModelExportStore
@@ -2251,10 +2263,11 @@ def imported_store_for_metadata(directory, object_store=None): 
def source_to_import_store(

    source: Union[str, dict],

    app: StoreAppProtocol,

    galaxy_user: Optional[model.User],

    import_options: Optional[ImportOptions],

    model_store_format: Optional[ModelStoreFormat] = None,

    user_context=None,

) -> ModelImportStore:

    galaxy_user = user_context.user if user_context else None

    if isinstance(source, dict):

        if model_store_format is not None:

            raise Exception(
@@ -2273,7 +2286,10 @@ def source_to_import_store( 
        if source_uri.startswith("file://"):

            source_uri = source_uri[len("file://") :]

        if "://" in source_uri:

            source_uri = stream_url_to_file(source_uri, app.file_sources, prefix="gx_import_model_store")

            user_context = ProvidesUserFileSourcesUserContext(user_context)

            source_uri = stream_url_to_file(

                source_uri, app.file_sources, prefix="gx_import_model_store", user_context=user_context

            )

            delete = True

        target_path = source_uri

        if target_path.endswith(".json"):