Unverified Commit 1db61fc5 authored by Marius van den Beek's avatar Marius van den Beek Committed by GitHub
Browse files

Merge pull request #13773 from kxk302/main_irods_fixes 

Added exist_ok flag to other object stores
parents e612a38c d99f26de

lib/galaxy/objectstore/azure_blob.py

+3 −3
Original line number Diff line number Diff line
@@ -246,7 +246,7 @@ class AzureBlobObjectStore(ConcreteObjectStore): 
        # Ensure the cache directory structure exists (e.g., dataset_#_files/)

        rel_path_dir = os.path.dirname(rel_path)

        if not os.path.exists(self._get_cache_path(rel_path_dir)):

            os.makedirs(self._get_cache_path(rel_path_dir))

            os.makedirs(self._get_cache_path(rel_path_dir), exist_ok=True)

        # Now pull in the file

        file_ok = self._download(rel_path)

        self._fix_permissions(self._get_cache_path(rel_path_dir))
@@ -327,7 +327,7 @@ class AzureBlobObjectStore(ConcreteObjectStore): 
            # for JOB_WORK directory

            elif base_dir:

                if not os.path.exists(rel_path):

                    os.makedirs(rel_path)

                    os.makedirs(rel_path, exist_ok=True)

                return True

            else:

                return False
@@ -381,7 +381,7 @@ class AzureBlobObjectStore(ConcreteObjectStore): 
            # Create given directory in cache

            cache_dir = os.path.join(self.staging_path, rel_path)

            if not os.path.exists(cache_dir):

                os.makedirs(cache_dir)

                os.makedirs(cache_dir, exist_ok=True)



            # Although not really necessary to create S3 folders (because S3 has

            # flat namespace), do so for consistency with the regular file system

lib/galaxy/objectstore/cloud.py

+3 −3
Original line number Diff line number Diff line
@@ -417,7 +417,7 @@ class Cloud(ConcreteObjectStore, CloudConfigMixin): 
        # Ensure the cache directory structure exists (e.g., dataset_#_files/)

        rel_path_dir = os.path.dirname(rel_path)

        if not os.path.exists(self._get_cache_path(rel_path_dir)):

            os.makedirs(self._get_cache_path(rel_path_dir))

            os.makedirs(self._get_cache_path(rel_path_dir), exist_ok=True)

        # Now pull in the file

        file_ok = self._download(rel_path)

        self._fix_permissions(self._get_cache_path(rel_path_dir))
@@ -529,7 +529,7 @@ class Cloud(ConcreteObjectStore, CloudConfigMixin): 
            # for JOB_WORK directory

            elif base_dir:

                if not os.path.exists(rel_path):

                    os.makedirs(rel_path)

                    os.makedirs(rel_path, exist_ok=True)

                return True

            else:

                return False
@@ -565,7 +565,7 @@ class Cloud(ConcreteObjectStore, CloudConfigMixin): 
            # Create given directory in cache

            cache_dir = os.path.join(self.staging_path, rel_path)

            if not os.path.exists(cache_dir):

                os.makedirs(cache_dir)

                os.makedirs(cache_dir, exist_ok=True)



            if not dir_only:

                rel_path = os.path.join(rel_path, alt_name if alt_name else f"dataset_{self._get_object_id(obj)}.dat")

lib/galaxy/objectstore/pithos.py

+4 −4
Original line number Diff line number Diff line
@@ -210,7 +210,7 @@ class PithosObjectStore(ConcreteObjectStore): 
        rel_path_dir = os.path.dirname(rel_path)

        rel_cache_path_dir = self._get_cache_path(rel_path_dir)

        if not os.path.exists(rel_cache_path_dir):

            os.makedirs(self._get_cache_path(rel_path_dir))

            os.makedirs(self._get_cache_path(rel_path_dir), exist_ok=True)

        # Now pull in the file

        cache_path = self._get_cache_path(rel_path_dir)

        self.pithos.download_object(rel_path, cache_path)
@@ -239,7 +239,7 @@ class PithosObjectStore(ConcreteObjectStore): 
                return True

            elif base_dir:  # for JOB_WORK directory

                if not os.path.exists(path):

                    os.makedirs(path)

                    os.makedirs(path, exist_ok=True)

                return True

            return False
@@ -273,7 +273,7 @@ class PithosObjectStore(ConcreteObjectStore): 
            # Create given directory in cache

            cache_dir = os.path.join(self.staging_path, rel_path)

            if not os.path.exists(cache_dir):

                os.makedirs(cache_dir)

                os.makedirs(cache_dir, exist_ok=True)



            if dir_only:

                self.pithos.upload_from_string(
@@ -378,7 +378,7 @@ class PithosObjectStore(ConcreteObjectStore): 
        cache_path = self._get_cache_path(path)

        if dir_only:

            if not os.path.exists(cache_path):

                os.makedirs(cache_path)

                os.makedirs(cache_path, exist_ok=True)

            return cache_path

        if self._in_cache(path):

            return cache_path

lib/galaxy/objectstore/s3.py

+3 −3
Original line number Diff line number Diff line
@@ -420,7 +420,7 @@ class S3ObjectStore(ConcreteObjectStore, CloudConfigMixin): 
        # Ensure the cache directory structure exists (e.g., dataset_#_files/)

        rel_path_dir = os.path.dirname(rel_path)

        if not os.path.exists(self._get_cache_path(rel_path_dir)):

            os.makedirs(self._get_cache_path(rel_path_dir))

            os.makedirs(self._get_cache_path(rel_path_dir), exist_ok=True)

        # Now pull in the file

        file_ok = self._download(rel_path)

        self._fix_permissions(self._get_cache_path(rel_path_dir))
@@ -529,7 +529,7 @@ class S3ObjectStore(ConcreteObjectStore, CloudConfigMixin): 
            # for JOB_WORK directory

            elif base_dir:

                if not os.path.exists(rel_path):

                    os.makedirs(rel_path)

                    os.makedirs(rel_path, exist_ok=True)

                return True

            else:

                return False
@@ -565,7 +565,7 @@ class S3ObjectStore(ConcreteObjectStore, CloudConfigMixin): 
            # Create given directory in cache

            cache_dir = os.path.join(self.staging_path, rel_path)

            if not os.path.exists(cache_dir):

                os.makedirs(cache_dir)

                os.makedirs(cache_dir, exist_ok=True)



            # Although not really necessary to create S3 folders (because S3 has

            # flat namespace), do so for consistency with the regular file system

lib/galaxy/objectstore/unittest_utils/__init__.py

+1 −1
Original line number Diff line number Diff line
@@ -57,7 +57,7 @@ class Config: 
        path = os.path.join(self.temp_directory, name)

        directory = os.path.dirname(path)

        if not os.path.exists(directory):

            os.makedirs(directory)

            os.makedirs(directory, exist_ok=True)

        contents_template = Template(contents)

        expanded_contents = contents_template.safe_substitute(temp_directory=self.temp_directory)

        open(path, "w").write(expanded_contents)