Merge branch '3-adjust-api-to-work-with-galaxy-object-store' into 'main' (45fde874) · Commits · NDIP / Remote Data Brokers

.gitlab-ci.yml

+1 −1

Original line number	Diff line number	Diff line
		@@ -35,7 +35,7 @@ lint:
		- >
		docker run --rm remote-data-broker bash -c
		"set -e;
		poetry run isort --filter-files --check-only . ;
		poetry run isort --filter-files --check-only --profile black . ;
		poetry run black --check --diff .;
		poetry run flake8 .;
		poetry run pylint app tests;

.pre-commit-config.yaml

+1 −1

Original line number	Diff line number	Diff line
		@@ -28,7 +28,7 @@ repos:
		- id: system
		name: Isort
		description: use Isort to sort imports
		entry: poetry run isort --filter-files .
		entry: poetry run isort --filter-files --profile black .
		pass_filenames: false
		language: system
		- repo: local

app/brokers/abstract.py

+16 −2

Original line number	Diff line number	Diff line
		from abc import ABC, abstractmethod

		from app.common import DownloadData, UploadData
		from app.common import (
		DownloadData,
		ObjectData,
		ObjectMetaResponse,
		UploadData,
		UploadResponse,
		)


		class RemoteDataBroker(ABC):
		"""an abstract class for remote data broker"""

		@abstractmethod
		def upload(self, data: UploadData) -> str:
		def upload(self, data: UploadData) -> UploadResponse:
		pass

		@abstractmethod
		def meta(self, data: ObjectData) -> ObjectMetaResponse:
		pass

		@abstractmethod
		def delete(self, data: ObjectData) -> None:
		pass

		@abstractmethod

app/brokers/filesys/broker.py

+32 −14

Original line number	Diff line number	Diff line
		import base64
		import os
		import shutil
		from pathlib import Path

		from app.auth import AuthBroker
		from app.brokers import RemoteDataBroker
		from app.common import DownloadData, UploadData, UserInfo
		from app.common import (
		DownloadData,
		ObjectData,
		ObjectMetaResponse,
		UploadData,
		UploadResponse,
		UserInfo,
		b64d,
		b64e,
		uid_or_key,
		)
		from app.settings import settings


		def b64e(value: str) -> str:
		return base64.b64encode(value.encode()).decode()


		def b64d(value: str) -> str:
		return base64.b64decode(value).decode()


		def encode_filename(user_info: UserInfo, data: UploadData) -> str:
		if data.key:
		return b64e(str(data.key))
		encoded = (
		b64e(user_info.user_name)
		+ "%"
		+ b64e(data.dataset)
		+ b64e(str(data.dataset))
		+ "%"
		+ b64e(os.path.basename(data.input_path))
		)
		@@ -50,16 +54,30 @@ class FilesysBroker(RemoteDataBroker):
		def _authorize(self, token: str) -> UserInfo:
		return self._auth_broker.process_user_token(token)

		def upload(self, data: UploadData) -> str:
		def upload(self, data: UploadData) -> UploadResponse:
		user_info = self._authorize(data.token)
		uid = encode_filename(user_info, data)
		copy_complete(user_info, data.input_path, settings.rdb_storage_path + "/" + uid)
		return uid
		return UploadResponse(uid=uid)

		def download(self, data: DownloadData) -> None:
		user_info = self._authorize(data.token)
		copy_complete(
		user_info,
		settings.rdb_storage_path + "/" + data.uid,
		settings.rdb_storage_path + "/" + uid_or_key(data),
		data.output_path,
		)

		def meta(self, data: ObjectData) -> ObjectMetaResponse:
		self._authorize(data.token)
		fname = settings.rdb_storage_path + "/" + uid_or_key(data)
		try:
		size = Path(fname).stat().st_size
		return ObjectMetaResponse(exists=True, size=size)
		except FileNotFoundError:
		return ObjectMetaResponse(exists=False, size=0)

		def delete(self, data: ObjectData) -> None:
		self._authorize(data.token)
		fname = settings.rdb_storage_path + "/" + uid_or_key(data)
		Path(fname).unlink()

app/common.py

+63 −3

Original line number	Diff line number	Diff line
		"""This module contains common definitions"""

		from pydantic import BaseModel
		import base64
		from typing import Any, Optional

		from pydantic import BaseModel, validator

		# pylint: disable=E0213


		class ObjectMetaResponse(BaseModel):
		"""This is a model for object meta response"""

		exists: bool
		size: int


		class ObjectData(BaseModel):
		"""This is a model for object"""

		uid: Optional[str]
		key: Optional[str]
		token: str

		@validator("key", always=True)
		def uid_or_key(cls: Any, key: Any, values: Any) -> Any:
		if not values.get("uid") and not key:
		raise ValueError("Either uid or key is required")
		if values.get("uid") and key:
		raise ValueError("Only one uid or key should be set")
		return key


		class UploadResponse(BaseModel):
		@@ -14,16 +42,34 @@ class UploadData(BaseModel):

		input_path: str
		token: str
		dataset: str
		dataset: Optional[str]
		key: Optional[str]

		@validator("key", always=True)
		def dataset_or_key(cls: Any, key: Any, values: Any) -> Any:
		if not values.get("dataset") and not key:
		raise ValueError("Either dataset or key is required")
		if values.get("dataset") and key:
		raise ValueError("Only one dataset or key should be set")
		return key


		class DownloadData(BaseModel):
		"""This is a model for file download"""

		uid: str
		uid: Optional[str]
		key: Optional[str]
		output_path: str
		token: str

		@validator("key", always=True)
		def uid_or_key(cls: Any, key: Any, values: Any) -> Any:
		if not values.get("uid") and not key:
		raise ValueError("Either uid or key is required")
		if values.get("uid") and key:
		raise ValueError("Only one uid or key should be set")
		return key


		class UserInfo(BaseModel):
		"""This is a model for user information extracted from an oidc token"""
		@@ -37,3 +83,17 @@ class ServiceError(Exception):

		class AuthError(Exception):
		"""An authorization error"""


		def b64e(value: str) -> str:
		return base64.b64encode(value.encode()).decode()


		def b64d(value: str) -> str:
		return base64.b64decode(value).decode()


		def uid_or_key(data: Any) -> str:
		if data.key:
		return b64e(str(data.key))
		return str(data.uid)