Merge branch '48-delete-datasets-in-object-store' into 'dev' (5304480e) · Commits · NDIP / Galaxy

lib/galaxy/celery/tasks.py

+2 −2

Original line number	Diff line number	Diff line
		@@ -91,8 +91,8 @@ def purge_hda(hda_manager: HDAManager, hda_id: int):


		@galaxy_task(ignore_result=True, action="completely removes a set of datasets from the object_store")
		def purge_datasets(dataset_manager: DatasetManager, request: PurgeDatasetsTaskRequest):
		dataset_manager.purge_datasets(request)
		def purge_datasets(dataset_manager: DatasetManager, request: PurgeDatasetsTaskRequest, user: model.User):
		dataset_manager.purge_datasets(request, user)


		@galaxy_task(ignore_result=True, action="materializing dataset instance")

lib/galaxy/jobs/init.py

+3 −2

Original line number	Diff line number	Diff line
		@@ -1857,7 +1857,8 @@ class MinimalJobWrapper(HasResourceParameters):
		# the tasks failed. So include the stderr, stdout, and exit code:
		return fail()

		extended_metadata = self.external_output_metadata.extended and not self.tool.tool_type == "interactive"

		extended_metadata = self.external_output_metadata.extended # why was that here? and not self.tool.tool_type == "interactive"

		# We collect the stderr from tools that write their stderr to galaxy.json
		tool_provided_metadata = self.get_tool_provided_job_metadata()
		@@ -2428,7 +2429,7 @@ class MinimalJobWrapper(HasResourceParameters):
		# is a bit of hack - our object store abstractions would be stronger
		# and more consistent if tools weren't writing there directly.
		try:
		dataset.full_delete()
		dataset.full_delete(user=hda.user)
		except ObjectNotFound:
		pass

lib/galaxy/managers/datasets.py

+5 −5

Original line number	Diff line number	Diff line
		@@ -75,7 +75,7 @@ class DatasetManager(base.ModelManager[model.Dataset], secured.AccessibleManager
		def copy(self, dataset, **kwargs):
		raise exceptions.NotImplemented("Datasets cannot be copied")

		def purge(self, dataset, flush=True):
		def purge(self, dataset, flush=True, user=None):
		"""
		Remove the object_store/file for this dataset from storage and mark
		as purged.
		@@ -85,7 +85,7 @@ class DatasetManager(base.ModelManager[model.Dataset], secured.AccessibleManager
		self.error_unless_dataset_purge_allowed(dataset)

		# the following also marks dataset as purged and deleted
		dataset.full_delete()
		dataset.full_delete(user=user)
		self.session().add(dataset)
		if flush:
		session = self.session()
		@@ -93,7 +93,7 @@ class DatasetManager(base.ModelManager[model.Dataset], secured.AccessibleManager
		session.commit()
		return dataset

		def purge_datasets(self, request: PurgeDatasetsTaskRequest):
		def purge_datasets(self, request: PurgeDatasetsTaskRequest, user: model.User):
		"""
		Caution: any additional security checks must be done before executing this action.

		@@ -106,7 +106,7 @@ class DatasetManager(base.ModelManager[model.Dataset], secured.AccessibleManager
		dataset: model.Dataset = self.session().query(model.Dataset).get(dataset_id)
		if dataset.user_can_purge:
		try:
		dataset.full_delete()
		dataset.full_delete(user)
		except Exception:
		log.exception(f"Unable to purge dataset ({dataset.id})")

		@@ -365,7 +365,7 @@ class DatasetAssociationManager(

		# more importantly, purge underlying dataset as well
		if dataset_assoc.dataset.user_can_purge:
		self.dataset_manager.purge(dataset_assoc.dataset)
		self.dataset_manager.purge(dataset_assoc.dataset, user=dataset_assoc.user)
		return dataset_assoc

		def by_user(self, user):

lib/galaxy/managers/hdas.py

+4 −4

Original line number	Diff line number	Diff line
		@@ -439,7 +439,7 @@ class HDAStorageCleanerManager(base.StorageCleanerManager):
		with transaction(session):
		session.commit()

		self._request_full_delete_all(dataset_ids_to_remove)
		self._request_full_delete_all(dataset_ids_to_remove, user=user)

		return StorageItemsCleanupResult(
		total_item_count=len(item_ids),
		@@ -448,15 +448,15 @@ class HDAStorageCleanerManager(base.StorageCleanerManager):
		errors=errors,
		)

		def _request_full_delete_all(self, dataset_ids_to_remove: Set[int]):
		def _request_full_delete_all(self, dataset_ids_to_remove: Set[int], user: model.User):
		use_tasks = self.dataset_manager.app.config.enable_celery_tasks
		request = PurgeDatasetsTaskRequest(dataset_ids=list(dataset_ids_to_remove))
		if use_tasks:
		from galaxy.celery.tasks import purge_datasets

		purge_datasets.delay(request=request)
		purge_datasets.delay(request=request, user=user)
		else:
		self.dataset_manager.purge_datasets(request)
		self.dataset_manager.purge_datasets(request, user=user)


		class HDASerializer( # datasets._UnflattenedMetadataDatasetAssociationSerializer,

lib/galaxy/model/init.py

+3 −3

Original line number	Diff line number	Diff line
		@@ -4145,11 +4145,11 @@ class Dataset(Base, StorableObject, Serializable):
		and len(self.history_associations) == len(self.purged_history_associations)
		)

		def full_delete(self):
		def full_delete(self, user=None):
		"""Remove the file and extra files, marks deleted and purged"""
		# os.unlink( self.file_name )
		try:
		self.object_store.delete(self)
		self.object_store.delete(self, user=user)
		except galaxy.exceptions.ObjectNotFound:
		pass
		rel_path = self._extra_files_rel_path
		@@ -5000,7 +5000,7 @@ class HistoryDatasetAssociation(DatasetInstance, HasTags, Dictifiable, UsesAnnot
		self.copy_tags_from(self.user, other_hda)
		self.dataset = new_dataset or other_hda.dataset
		if old_dataset:
		old_dataset.full_delete()
		old_dataset.full_delete(user=self.user)

		def copy(self, parent_id=None, copy_tags=None, flush=True, copy_hid=True, new_name=None):
		"""