Unverified Commit c71f9440 authored by mvdbeek's avatar mvdbeek
Browse files

Make sure we set file size also for purged outputs 

Fixes one of the integration tests:
```
Exception: HistoryDatasetAssociation <galaxy.model.HistoryDatasetAssociation(None) at 0x7f50ddcf56a0> in state ok with null file size, this is not valid
ERROR    galaxy.jobs.runners:__init__.py:177 (1) Unhandled exception calling finish_job
Traceback (most recent call last):
  File "/home/runner/work/galaxy/galaxy/galaxy root/lib/galaxy/jobs/runners/pulsar.py", line 702, in finish_job
    job_wrapper.finish(
  File "/home/runner/work/galaxy/galaxy/galaxy root/lib/galaxy/jobs/__init__.py", line 2054, in finish
    self.sa_session.commit()
  File "/home/runner/work/galaxy/galaxy/galaxy root/.venv/lib/python3.8/site-packages/sqlalchemy/orm/scoping.py", line 597, in commit
    return self._proxied.commit()
  File "/home/runner/work/galaxy/galaxy/galaxy root/.venv/lib/python3.8/site-packages/sqlalchemy/orm/session.py", line 2017, in commit
    trans.commit(_to_root=True)
  File "<string>", line 2, in commit
  File "/home/runner/work/galaxy/galaxy/galaxy root/.venv/lib/python3.8/site-packages/sqlalchemy/orm/state_changes.py", line 139, in _go
    ret_value = fn(self, *arg, **kw)
  File "/home/runner/work/galaxy/galaxy/galaxy root/.venv/lib/python3.8/site-packages/sqlalchemy/orm/session.py", line 1302, in commit
    self._prepare_impl()
  File "<string>", line 2, in _prepare_impl
  File "/home/runner/work/galaxy/galaxy/galaxy root/.venv/lib/python3.8/site-packages/sqlalchemy/orm/state_changes.py", line 139, in _go
    ret_value = fn(self, *arg, **kw)
  File "/home/runner/work/galaxy/galaxy/galaxy root/.venv/lib/python3.8/site-packages/sqlalchemy/orm/session.py", line 1277, in _prepare_impl
    self.session.flush()
  File "/home/runner/work/galaxy/galaxy/galaxy root/.venv/lib/python3.8/site-packages/sqlalchemy/orm/session.py", line 4341, in flush
    self._flush(objects)
  File "/home/runner/work/galaxy/galaxy/galaxy root/.venv/lib/python3.8/site-packages/sqlalchemy/orm/session.py", line 4369, in _flush
    self.dispatch.before_flush(self, flush_context, objects)
  File "/home/runner/work/galaxy/galaxy/galaxy root/.venv/lib/python3.8/site-packages/sqlalchemy/event/attr.py", line 378, in __call__
    fn(*args, **kw)
  File "/home/runner/work/galaxy/galaxy/galaxy root/lib/galaxy/model/base.py", line 184, in before_flush
    obj.__strict_check_before_flush__()
  File "/home/runner/work/galaxy/galaxy/galaxy root/lib/galaxy/model/__init__.py", line 5223, in __strict_check_before_flush__
    raise Exception(
Exception: HistoryDatasetAssociation <galaxy.model.HistoryDatasetAssociation(None) at 0x7f50ddcf56a0> in state ok with null file size, this is not valid

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/runner/work/galaxy/galaxy/galaxy root/lib/galaxy/jobs/runners/__init__.py", line 174, in run_next
    method(arg)
  File "/home/runner/work/galaxy/galaxy/galaxy root/lib/galaxy/jobs/runners/pulsar.py", line 713, in finish_job
    job_wrapper.fail("Unable to finish job", exception=True)
  File "/home/runner/work/galaxy/galaxy/galaxy root/lib/galaxy/jobs/__init__.py", line 1468, in fail
    self.sa_session.commit()
  File "/home/runner/work/galaxy/galaxy/galaxy root/.venv/lib/python3.8/site-packages/sqlalchemy/orm/scoping.py", line 597, in commit
    return self._proxied.commit()
  File "/home/runner/work/galaxy/galaxy/galaxy root/.venv/lib/python3.8/site-packages/sqlalchemy/orm/session.py", line 2017, in commit
    trans.commit(_to_root=True)
  File "<string>", line 2, in commit
  File "/home/runner/work/galaxy/galaxy/galaxy root/.venv/lib/python3.8/site-packages/sqlalchemy/orm/state_changes.py", line 139, in _go
    ret_value = fn(self, *arg, **kw)
  File "/home/runner/work/galaxy/galaxy/galaxy root/.venv/lib/python3.8/site-packages/sqlalchemy/orm/session.py", line 1302, in commit
    self._prepare_impl()
  File "<string>", line 2, in _prepare_impl
  File "/home/runner/work/galaxy/galaxy/galaxy root/.venv/lib/python3.8/site-packages/sqlalchemy/orm/state_changes.py", line 139, in _go
    ret_value = fn(self, *arg, **kw)
  File "/home/runner/work/galaxy/galaxy/galaxy root/.venv/lib/python3.8/site-packages/sqlalchemy/orm/session.py", line 1277, in _prepare_impl
    self.session.flush()
  File "/home/runner/work/galaxy/galaxy/galaxy root/.venv/lib/python3.8/site-packages/sqlalchemy/orm/session.py", line 4341, in flush
    self._flush(objects)
  File "/home/runner/work/galaxy/galaxy/galaxy root/.venv/lib/python3.8/site-packages/sqlalchemy/orm/session.py", line 4369, in _flush
    self.dispatch.before_flush(self, flush_context, objects)
  File "/home/runner/work/galaxy/galaxy/galaxy root/.venv/lib/python3.8/site-packages/sqlalchemy/event/attr.py", line 378, in __call__
    fn(*args, **kw)
  File "/home/runner/work/galaxy/galaxy/galaxy root/lib/galaxy/model/base.py", line 184, in before_flush
    obj.__strict_check_before_flush__()
  File "/home/runner/work/galaxy/galaxy/galaxy root/lib/galaxy/model/__init__.py", line 5223, in __strict_check_before_flush__
    raise Exception(
Exception: HistoryDatasetAssociation <galaxy.model.HistoryDatasetAssociation(None) at 0x7f50ddcf56a0> in state ok with null file size, this is not valid
```

More fallout from https://github.com/galaxyproject/galaxy/pull/18653
parent 591f36c4

lib/galaxy/jobs/__init__.py

+5 −5
Original line number Diff line number Diff line
@@ -2001,13 +2001,13 @@ class MinimalJobWrapper(HasResourceParameters): 
        # Once datasets are collected, set the total dataset size (includes extra files)

        for dataset_assoc in job.output_datasets:

            dataset = dataset_assoc.dataset.dataset

            if not dataset.purged:

            # assume all datasets in a job get written to the same objectstore

            quota_source_info = dataset.quota_source_info

            collected_bytes += dataset.set_total_size()

            else:

            if dataset.purged:

                # Purge, in case job wrote directly to object store

                dataset.full_delete()

                collected_bytes = 0



        user = job.user

        if user and collected_bytes > 0 and quota_source_info is not None and quota_source_info.use:

lib/galaxy/model/store/__init__.py

+3 −3
Original line number Diff line number Diff line
@@ -665,7 +665,7 @@ class ModelImportStore(metaclass=abc.ABCMeta): 
                                assert file_source_root

                                dataset_extra_files_path = os.path.join(file_source_root, dataset_extra_files_path)

                                persist_extra_files(self.object_store, dataset_extra_files_path, dataset_instance)

                            # Don't trust serialized file size

                            # Only trust file size if the dataset is purged. If we keep the data we should check the file size.

                            dataset_instance.dataset.file_size = None

                            dataset_instance.dataset.set_total_size()  # update the filesize record in the database