Merge pull request #15140 from davelopez/22.05_backport_#14989_and_#15090 (0775ba27) · Commits · NDIP / Galaxy

lib/galaxy/files/uris.py

+2 −1

Original line number	Diff line number	Diff line
		@@ -47,13 +47,14 @@ def stream_url_to_file(
		file_sources: Optional["ConfiguredFileSources"] = None,
		prefix: str = "gx_file_stream",
		dir: Optional[str] = None,
		user_context=None,
		) -> str:
		temp_name: str
		if file_sources and file_sources.looks_like_uri(path):
		file_source_path = file_sources.get_file_source_path(path)
		with tempfile.NamedTemporaryFile(prefix=prefix, delete=False, dir=dir) as temp:
		temp_name = temp.name
		file_source_path.file_source.realize_to(file_source_path.path, temp_name)
		file_source_path.file_source.realize_to(file_source_path.path, temp_name, user_context=user_context)
		elif path.startswith("base64://"):
		with tempfile.NamedTemporaryFile(prefix=prefix, delete=False, dir=dir) as temp:
		temp_name = temp.name

lib/galaxy/managers/model_stores.py

+51 −17

Original line number	Diff line number	Diff line
		@@ -3,7 +3,9 @@ from typing import Optional
		from galaxy import model
		from galaxy.exceptions import RequestParameterInvalidException
		from galaxy.jobs.manager import JobManager
		from galaxy.managers.context import ProvidesUserContext
		from galaxy.managers.histories import HistoryManager
		from galaxy.managers.users import UserManager
		from galaxy.model.scoped_session import galaxy_scoped_session
		from galaxy.model.store import (
		ImportDiscardedDataType,
		@@ -29,6 +31,28 @@ from galaxy.web.short_term_storage import (
		)


		class ModelStoreUserContext(ProvidesUserContext):
		def __init__(self, app: MinimalManagerApp, user: model.User) -> None:
		self._app = app
		self._user = user

		@property
		def app(self):
		return self._app

		@property
		def url_builder(self):
		raise NotImplementedError("URL builder not available in ModelStore context.")

		def get_user(self):
		return self._user

		def set_user(self, user):
		raise NotImplementedError("Cannot change user from ModelStore context.")

		user = property(get_user, set_user)


		class ModelStoreManager:
		def __init__(
		self,
		@@ -37,12 +61,14 @@ class ModelStoreManager:
		sa_session: galaxy_scoped_session,
		job_manager: JobManager,
		short_term_storage_monitor: ShortTermStorageMonitor,
		user_manager: UserManager,
		):
		self._app = app
		self._sa_session = sa_session
		self._job_manager = job_manager
		self._history_manager = history_manager
		self._short_term_storage_monitor = short_term_storage_monitor
		self._user_manager = user_manager

		def setup_history_export_job(self, request: SetupHistoryExportJob):
		history_id = request.history_id
		@@ -112,9 +138,13 @@ class ModelStoreManager:
		model_store_format = request.model_store_format
		export_files = "symlink" if request.include_files else None
		target_uri = request.target_uri
		with model.store.get_export_store_factory(self._app, model_store_format, export_files=export_files)(
		target_uri
		) as export_store:
		user_context = self._build_user_context(request.user.user_id)
		with model.store.get_export_store_factory(
		self._app,
		model_store_format,
		export_files=export_files,
		user_context=user_context,
		)(target_uri) as export_store:
		invocation = self._sa_session.query(model.WorkflowInvocation).get(request.invocation_id)
		export_store.export_workflow_invocation(
		invocation, include_hidden=request.include_hidden, include_deleted=request.include_deleted
		@@ -124,9 +154,10 @@ class ModelStoreManager:
		model_store_format = request.model_store_format
		export_files = "symlink" if request.include_files else None
		target_uri = request.target_uri
		with model.store.get_export_store_factory(self._app, model_store_format, export_files=export_files)(
		target_uri
		) as export_store:
		user_context = self._build_user_context(request.user.user_id)
		with model.store.get_export_store_factory(
		self._app, model_store_format, export_files=export_files, user_context=user_context
		)(target_uri) as export_store:
		if request.content_type == HistoryContentType.dataset:
		hda = self._sa_session.query(model.HistoryDatasetAssociation).get(request.content_id)
		export_store.add_dataset(hda)
		@@ -140,9 +171,10 @@ class ModelStoreManager:
		model_store_format = request.model_store_format
		export_files = "symlink" if request.include_files else None
		target_uri = request.target_uri
		with model.store.get_export_store_factory(self._app, model_store_format, export_files=export_files)(
		target_uri
		) as export_store:
		user_context = self._build_user_context(request.user.user_id)
		with model.store.get_export_store_factory(
		self._app, model_store_format, export_files=export_files, user_context=user_context
		)(target_uri) as export_store:
		history = self._history_manager.by_id(request.history_id)
		export_store.export_history(
		history, include_hidden=request.include_hidden, include_deleted=request.include_deleted
		@@ -150,7 +182,6 @@ class ModelStoreManager:

		def import_model_store(self, request: ImportModelStoreTaskRequest):
		import_options = ImportOptions(
		discarded_data=ImportDiscardedDataType.FORCE,
		allow_library_creation=request.for_library,
		)
		history_id = request.history_id
		@@ -158,16 +189,13 @@ class ModelStoreManager:
		history = self._sa_session.query(model.History).get(history_id)
		else:
		history = None
		user_id = request.user.user_id
		if user_id:
		galaxy_user = self._sa_session.query(model.User).get(user_id)
		else:
		galaxy_user = None
		user_context = self._build_user_context(request.user.user_id)
		model_import_store = source_to_import_store(
		request.source_uri,
		self._app,
		galaxy_user,
		import_options,
		model_store_format=request.model_store_format,
		user_context=user_context,
		)
		new_history = history is None and not request.for_library
		if new_history:
		@@ -183,6 +211,11 @@ class ModelStoreManager:
		)
		return object_tracker

		def _build_user_context(self, user_id: int):
		user = self._user_manager.by_id(user_id)
		user_context = ModelStoreUserContext(self._app, user)
		return user_context


		def create_objects_from_store(
		app: MinimalManagerApp,
		@@ -195,12 +228,13 @@ def create_objects_from_store(
		discarded_data=ImportDiscardedDataType.FORCE,
		allow_library_creation=for_library,
		)
		user_context = ModelStoreUserContext(app, galaxy_user) if galaxy_user is not None else None
		model_import_store = source_to_import_store(
		payload.store_content_uri or payload.store_dict,
		app=app,
		galaxy_user=galaxy_user,
		import_options=import_options,
		model_store_format=payload.model_store_format,
		user_context=user_context,
		)
		new_history = history is None and not for_library
		if new_history:

lib/galaxy/model/store/init.py

+47 −16

Original line number	Diff line number	Diff line
		import abc
		import contextlib
		import datetime
		import logging
		import os
		import shutil
		import tarfile
		@@ -38,7 +39,10 @@ from galaxy.exceptions import (
		ObjectNotFound,
		RequestParameterInvalidException,
		)
		from galaxy.files import ConfiguredFileSources
		from galaxy.files import (
		ConfiguredFileSources,
		ProvidesUserFileSourcesUserContext,
		)
		from galaxy.files.uris import stream_url_to_file
		from galaxy.model.mapping import GalaxyModelMapping
		from galaxy.model.metadata import MetadataCollection
		@@ -47,7 +51,9 @@ from galaxy.model.orm.util import (
		add_object_to_session,
		get_object_session,
		)
		from galaxy.model.tags import GalaxyTagHandler
		from galaxy.objectstore import ObjectStore
		from galaxy.schema.schema import ModelStoreFormat
		from galaxy.security.idencoding import IdEncodingHelper
		from galaxy.util import (
		FILENAME_VALID_CHARS,
		@@ -64,6 +70,8 @@ from ..item_attrs import (
		)
		from ... import model

		log = logging.getLogger(__name__)

		ObjectKeyType = Union[str, int]

		ATTRS_FILENAME_HISTORY = "history_attrs.txt"
		@@ -96,6 +104,7 @@ class StoreAppProtocol(Protocol):
		datatypes_registry: Registry
		object_store: ObjectStore
		security: IdEncodingHelper
		tag_handler: GalaxyTagHandler
		model: GalaxyModelMapping
		file_sources: ConfiguredFileSources

		@@ -582,9 +591,18 @@ class ModelImportStore(metaclass=abc.ABCMeta):
		pass
		if not self.import_options.allow_edit:
		# external import, metadata files need to be regenerated (as opposed to extended metadata dataset import)
		if self.app.datatypes_registry.set_external_metadata_tool:
		self.app.datatypes_registry.set_external_metadata_tool.regenerate_imported_metadata_if_needed(
		dataset_instance, history, **regenerate_kwds
		)
		else:
		# Try to set metadata directly. @mvdbeek thinks we should only record the datasets
		try:
		if dataset_instance.has_metadata_files:
		dataset_instance.datatype.set_meta(dataset_instance)
		except Exception:
		log.debug(f"Metadata setting failed on {dataset_instance}", exc_info=True)
		dataset_instance.dataset.state = dataset_instance.dataset.states.FAILED_METADATA

		if model_class == "HistoryDatasetAssociation":
		if object_key in dataset_attrs:
		@@ -1589,7 +1607,8 @@ class DirectoryModelExportStore(ModelExportStore):
		export_files: Optional[str] = None,
		strip_metadata_files: bool = True,
		serialize_jobs: bool = True,
		):
		user_context=None,
		) -> None:
		"""
		:param export_directory: path to export directory. Will be created if it does not exist.
		:param app: Galaxy App or app-like object. Must be provided if `for_edit` and/or `serialize_dataset_objects` are True
		@@ -1613,6 +1632,7 @@ class DirectoryModelExportStore(ModelExportStore):
		sessionless = True
		security = IdEncodingHelper(id_secret="randomdoesntmatter")

		self.user_context = ProvidesUserFileSourcesUserContext(user_context)
		self.file_sources = file_sources
		self.serialize_jobs = serialize_jobs
		self.sessionless = sessionless
		@@ -2140,7 +2160,7 @@ class TarModelExportStore(DirectoryModelExportStore):
		file_source_path = self.file_sources.get_file_source_path(self.file_source_uri)
		file_source = file_source_path.file_source
		assert os.path.exists(self.out_file)
		file_source.write_from(file_source_path.path, self.out_file)
		file_source.write_from(file_source_path.path, self.out_file, user_context=self.user_context)
		shutil.rmtree(self.temp_output_dir)


		@@ -2179,16 +2199,22 @@ class BagArchiveModelExportStore(BagDirectoryModelExportStore):
		file_source_path = self.file_sources.get_file_source_path(self.file_source_uri)
		file_source = file_source_path.file_source
		assert os.path.exists(rval)
		file_source.write_from(file_source_path.path, rval)
		file_source.write_from(file_source_path.path, rval, user_context=self.user_context)
		shutil.rmtree(self.temp_output_dir)


		def get_export_store_factory(app, download_format: str, export_files=None) -> Callable[[str], ModelExportStore]:
		def get_export_store_factory(
		app,
		download_format: str,
		export_files=None,
		user_context=None,
		) -> Callable[[str], ModelExportStore]:
		export_store_class: Union[Type[TarModelExportStore], Type[BagArchiveModelExportStore]]
		export_store_class_kwds = {
		"app": app,
		"export_files": export_files,
		"serialize_dataset_objects": False,
		"user_context": user_context,
		}
		if download_format in ["tar.gz", "tgz"]:
		export_store_class = TarModelExportStore
		@@ -2237,10 +2263,11 @@ def imported_store_for_metadata(directory, object_store=None):
		def source_to_import_store(
		source: Union[str, dict],
		app: StoreAppProtocol,
		galaxy_user: Optional[model.User],
		import_options: Optional[ImportOptions],
		model_store_format: Optional[str] = None,
		model_store_format: Optional[ModelStoreFormat] = None,
		user_context=None,
		) -> ModelImportStore:
		galaxy_user = user_context.user if user_context else None
		if isinstance(source, dict):
		if model_store_format is not None:
		raise Exception(
		@@ -2255,10 +2282,14 @@ def source_to_import_store(
		else:
		source_uri: str = str(source)
		delete = False
		tag_handler = app.tag_handler.create_tag_handler_session()
		if source_uri.startswith("file://"):
		source_uri = source_uri[len("file://") :]
		if "://" in source_uri:
		source_uri = stream_url_to_file(source_uri, app.file_sources, prefix="gx_import_model_store")
		user_context = ProvidesUserFileSourcesUserContext(user_context)
		source_uri = stream_url_to_file(
		source_uri, app.file_sources, prefix="gx_import_model_store", user_context=user_context
		)
		delete = True
		target_path = source_uri
		if target_path.endswith(".json"):
		@@ -2273,11 +2304,11 @@ def source_to_import_store(
		)
		elif os.path.isdir(target_path):
		model_import_store = get_import_model_store_for_directory(
		target_path, import_options=import_options, app=app, user=galaxy_user
		target_path, import_options=import_options, app=app, user=galaxy_user, tag_handler=tag_handler
		)
		else:
		model_store_format = model_store_format or "tgz"
		if model_store_format in ["tar.gz", "tgz", "tar"]:
		model_store_format = model_store_format or ModelStoreFormat.TGZ
		if ModelStoreFormat.is_compressed(model_store_format):
		try:
		temp_dir = mkdtemp()
		target_dir = CompressedFile(target_path).extract(temp_dir)
		@@ -2285,9 +2316,9 @@ def source_to_import_store(
		if delete:
		os.remove(target_path)
		model_import_store = get_import_model_store_for_directory(
		target_dir, import_options=import_options, app=app, user=galaxy_user
		target_dir, import_options=import_options, app=app, user=galaxy_user, tag_handler=tag_handler
		)
		elif model_store_format in ["bag.gz", "bag.tar", "bag.zip"]:
		elif ModelStoreFormat.is_bag(model_store_format):
		model_import_store = BagArchiveImportModelStore(
		target_path, import_options=import_options, app=app, user=galaxy_user
		)

lib/galaxy/schema/schema.py

+8 −0

Original line number	Diff line number	Diff line
		@@ -1291,6 +1291,14 @@ class ModelStoreFormat(str, Enum):
		BAG_DOT_TAR = "bag.tar"
		BAG_DOT_TGZ = "bag.tgz"

		@classmethod
		def is_compressed(cls, value: "ModelStoreFormat"):
		return value in [cls.TAR_DOT_GZ, cls.TGZ, cls.TAR]

		@classmethod
		def is_bag(cls, value: "ModelStoreFormat"):
		return value in [cls.BAG_DOT_TAR, cls.BAG_DOT_TGZ, cls.BAG_DOT_ZIP]


		class StoreContentSource(Model):
		store_content_uri: Optional[str]

lib/galaxy/schema/tasks.py

+2 −0

Original line number	Diff line number	Diff line
		@@ -8,6 +8,7 @@ from pydantic import (
		from .schema import (
		DatasetSourceType,
		HistoryContentType,
		ModelStoreFormat,
		StoreExportPayload,
		WriteStoreToPayload,
		)
		@@ -82,6 +83,7 @@ class ImportModelStoreTaskRequest(BaseModel):
		history_id: Optional[int]
		source_uri: str
		for_library: bool
		model_store_format: Optional[ModelStoreFormat]


		class MaterializeDatasetInstanceTaskRequest(BaseModel):