Unverified Commit 56ee745e authored by mvdbeek's avatar mvdbeek
Browse files

"Use ``object_store.construct_path`` to build paths" 

This allows us to figure out what the extra)path for tools to consume should
be without having to create the directory up front ... which would fail
many tools that start with `mkdir`
parent 1c32d95e

lib/galaxy/model/__init__.py

+3 −1
Original line number Diff line number Diff line
@@ -3574,7 +3574,9 @@ class Dataset(Base, StorableObject, Serializable): 
        if not getattr(self, "external_extra_files_path", None):

            if self.object_store.exists(self, dir_only=True, extra_dir=self._extra_files_rel_path):

                return self.object_store.get_filename(self, dir_only=True, extra_dir=self._extra_files_rel_path)

            return ""

            return self.object_store.construct_path(

                self, dir_only=True, extra_dir=self._extra_files_rel_path, in_cache=True

            )

        else:

            return os.path.abspath(self.external_extra_files_path)

lib/galaxy/objectstore/__init__.py

+15 −0
Original line number Diff line number Diff line
@@ -96,6 +96,12 @@ class ObjectStore(metaclass=abc.ABCMeta): 
        """Return True if the object identified by `obj` exists, False otherwise."""

        raise NotImplementedError()



    @abc.abstractmethod

    def construct_path(

        self, obj, base_dir=None, dir_only=False, extra_dir=None, extra_dir_at_root=False, alt_name=None

    ):

        raise NotImplementedError()



    @abc.abstractmethod

    def create(

        self, obj, base_dir=None, dir_only=False, extra_dir=None, extra_dir_at_root=False, alt_name=None, obj_dir=False
@@ -353,6 +359,9 @@ class BaseObjectStore(ObjectStore): 
    def exists(self, obj, **kwargs):

        return self._invoke("exists", obj, **kwargs)



    def construct_path(self, obj, **kwargs):

        return self._invoke("construct_path", obj, **kwargs)



    def create(self, obj, **kwargs):

        return self._invoke("create", obj, **kwargs)
@@ -978,6 +987,9 @@ class DistributedObjectStore(NestedObjectStore): 
            self.weighted_backend_ids = new_weighted_backend_ids

            sleeper.sleep(120)  # Test free space every 2 minutes



    def _construct_path(self, obj, **kwargs):

        return self.backends[obj.object_store_id].construct_path(obj, **kwargs)



    def _create(self, obj, **kwargs):

        """The only method in which obj.object_store_id may be None."""

        if obj.object_store_id is None or not self._exists(obj, **kwargs):
@@ -1084,6 +1096,9 @@ class HierarchicalObjectStore(NestedObjectStore): 
                return True

        return False



    def _construct_path(self, obj, **kwargs):

        return self.backends[0].construct_path(obj, **kwargs)



    def _create(self, obj, **kwargs):

        """Call the primary object store."""

        self.backends[0].create(obj, **kwargs)

lib/galaxy/objectstore/azure_blob.py

+4 −0
Original line number Diff line number Diff line
@@ -172,6 +172,7 @@ class AzureBlobObjectStore(ConcreteObjectStore): 
        extra_dir_at_root=False,

        alt_name=None,

        obj_dir=False,

        in_cache=False,

        **kwargs,

    ):

        # extra_dir should never be constructed from provided data but just
@@ -210,6 +211,9 @@ class AzureBlobObjectStore(ConcreteObjectStore): 
        if not dir_only:

            rel_path = os.path.join(rel_path, alt_name if alt_name else f"dataset_{self._get_object_id(obj)}.dat")



        if in_cache:

            return self._get_cache_path(rel_path)



        return rel_path



    def _fix_permissions(self, rel_path):

lib/galaxy/objectstore/cloud.py

+5 −0
Original line number Diff line number Diff line
@@ -354,6 +354,7 @@ class Cloud(ConcreteObjectStore, CloudConfigMixin): 
        extra_dir_at_root=False,

        alt_name=None,

        obj_dir=False,

        in_cache=False,

        **kwargs,

    ):

        # extra_dir should never be constructed from provided data but just
@@ -389,6 +390,10 @@ class Cloud(ConcreteObjectStore, CloudConfigMixin): 


        if not dir_only:

            rel_path = os.path.join(rel_path, alt_name if alt_name else f"dataset_{self._get_object_id(obj)}.dat")



        if in_cache:

            return self._get_cache_path(rel_path)



        return rel_path



    def _get_cache_path(self, rel_path):

lib/galaxy/objectstore/irods.py

+5 −0
Original line number Diff line number Diff line
@@ -329,6 +329,7 @@ class IRODSObjectStore(DiskObjectStore, CloudConfigMixin): 
        extra_dir_at_root=False,

        alt_name=None,

        obj_dir=False,

        in_cache=False,

        **kwargs,

    ):

        ipt_timer = ExecutionTimer()
@@ -364,6 +365,10 @@ class IRODSObjectStore(DiskObjectStore, CloudConfigMixin): 
        if not dir_only:

            rel_path = os.path.join(rel_path, alt_name if alt_name else f"dataset_{self._get_object_id(obj)}.dat")

        log.debug("irods_pt _construct_path: %s", ipt_timer)



        if in_cache:

            return self._get_cache_path(rel_path)



        return rel_path



    def _get_cache_path(self, rel_path):