Configure fetching via globus-url-copy (f20283ec) · Commits · ARM / k8s / ADL

.gitlab-ci.yml

+13 −24

Original line number	Diff line number	Diff line
		@@ -14,19 +14,10 @@ variables:
		PROJECT_NAME: adl
		FF_GITLAB_REGISTRY_HELPER_IMAGE: 'true'

		chart_lint:
		stage: test-chart
		image: camden.ornl.gov/hub_proxy/alpine/helm:latest
		tags:
		- arm-k8s
		script:
		- helm lint helm/adl

		build-image:
		"Build API Image":
		stage: build
		image:
		name: gcr.io/kaniko-project/executor:debug
		entrypoint: [""]
		tags:
		- arm-k8s
		script:
		@@ -35,17 +26,15 @@ build-image:
		/kaniko/executor
		--cache=true
		--context "$CI_PROJECT_DIR"
		--dockerfile Dockerfile
		--dockerfile Dockerfile-api
		--destination "$IMAGE_REPO:$CI_COMMIT_SHORT_SHA"
		--destination "$IMAGE_REPO:latest"

		build_chart:
		"Build Chart":
		stage: build
		image: camden.ornl.gov/hub_proxy/alpine/helm:latest
		image: camden.ornl.gov/arm/toolbox:latest
		tags:
		- arm-k8s
		needs:
		- job: chart_lint
		artifacts:
		reports:
		dotenv: chart-build.env
		@@ -53,13 +42,13 @@ build_chart:
		- apk add --update --no-cache git
		script:
		- \|
		helm lint helm/adl
		CHART_VERSION=$(helm show chart helm/adl \| sed -n 's/^version: $.*$$/\1/p')
		echo "Repo chart version is ${CHART_VERSION}"
		CHART_VERSION="${CHART_VERSION}-snapshot+${CI_COMMIT_SHORT_SHA}"
		echo "Augmented chart version is ${CHART_VERSION}"
		echo "CHART_VERSION=${CHART_VERSION}" >> chart-build.env
		helm plugin install https://github.com/chartmuseum/helm-push
		helm push helm/adl --username "${REPO_ROBOT}" --password "${REPO_TOKEN}" --version="${CHART_VERSION}" https://camden.ornl.gov/chartrepo/arm --debug
		helm cm-push helm/adl --username "${REPO_ROBOT}" --password "${REPO_TOKEN}" --version="${CHART_VERSION}" https://camden.ornl.gov/chartrepo/arm --debug

		deploy:
		stage: deploy
		@@ -67,13 +56,13 @@ deploy:
		name: review/$CI_BUILD_REF_NAME
		url: https://api-$CI_BUILD_REF_SLUG.k8s.arm.gov/docs
		on_stop: stop_review
		image: camden.ornl.gov/hub_proxy/alpine/helm:latest
		image: camden.ornl.gov/arm/toolbox:latest
		tags:
		- arm-k8s
		needs:
		- job: build_chart
		- job: "Build Chart"
		artifacts: true
		- job: build-image
		- job: "Build API Image"
		rules:
		- if: $CI_MERGE_REQUEST_IID
		before_script:
		@@ -99,7 +88,7 @@ stop_review:
		name: review/$CI_BUILD_REF_NAME
		url: https://$PROJECT_NAME-$CI_BUILD_REF_SLUG.k8s.arm.gov/docs
		action: stop
		image: camden.ornl.gov/hub_proxy/alpine/helm:latest
		image: camden.ornl.gov/arm/toolbox:latest
		tags:
		- arm-k8s
		rules:
		@@ -112,13 +101,13 @@ deploy:prod:
		environment:
		name: production
		url: https://api.k8s.arm.gov
		image: camden.ornl.gov/hub_proxy/alpine/helm:latest
		image: camden.ornl.gov/arm/toolbox:latest
		tags:
		- arm-k8s
		needs:
		- job: build_chart
		- job: "Build Chart"
		artifacts: true
		- job: build-image
		- job: "Build API Image"
		rules:
		- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
		before_script:

Dockerfile

deleted100644 → 0

+0 −6

Original line number	Diff line number	Diff line
		FROM camden.ornl.gov/hub_proxy/centos/python-38-centos7:latest
		COPY bin/* /usr/local/bin/
		COPY HPSS.conf /usr/local/etc/HPSS.conf
		COPY app app
		RUN pip install --no-cache-dir ./app
		CMD ["uvicorn", "ADL.api.app:app", "--host", "0.0.0.0"]

Dockerfile-api

0 → 100644

+25 −0

Original line number	Diff line number	Diff line
		FROM camden.ornl.gov/hub_proxy/centos/python-38-centos7:latest
		USER 0
		RUN yum install -y epel-release && \
		yum update -y && \
		yum install -y globus-gass-copy-progs myproxy && \
		yum clean all

		# Fetch the certs from HPSS globus connector. Globus barfs on the ' in Let's Encrypt
		# so, grab them and let it fail, edit to escape the ' and run it again
		RUN myproxy-get-trustroots -bvs myproxy1.ccs.ornl.gov \|\| true
		RUN sed -i 's/Let\x27s/Let\\\x27s/' /etc/grid-security/certificates/*.signing_policy
		RUN myproxy-get-trustroots -bvs myproxy1.ccs.ornl.gov

		# and once again for the Open DTN
		# ...except this is broken...
		# RUN myproxy-get-trustroots -bvs myproxy1.op.ccs.ornl.gov \|\| true
		# RUN sed -i 's/Let\x27s/Let\\\x27s/' /etc/grid-security/certificates/*.signing_policy
		# RUN myproxy-get-trustroots -bvs myproxy1.op.ccs.ornl.gov

		USER 1001
		COPY bin/ /usr/local/bin/
		COPY HPSS.conf /usr/local/etc/HPSS.conf
		COPY app app
		RUN pip install --no-cache-dir ./app
		CMD ["uvicorn", "ADL.api.app:app", "--host", "0.0.0.0"]

app/ADL/api/config.py

+3 −0

Original line number	Diff line number	Diff line
		@@ -13,6 +13,9 @@ class Settings(BaseSettings):
		hsi_user: str = "johnny"
		hsi_keytab: FilePath = "/opt/johnny.keytab"
		hsi_bin: FilePath = "/usr/local/bin/hsi"
		guc_cert: FilePath = "/opt/globus.pem"
		guc_bin: str = "/usr/bin/globus-url-copy"
		guc_endpoint: str= "gsiftp://hpss-gridftp.ccs.ornl.gov"
		cache_location: DirectoryPath = "/data/cache"
		stream_bytes_per_response: int = 100000
		db_drivername: str = "postgresql"

app/ADL/api/fs.py

+78 −0

Original line number	Diff line number	Diff line
		@@ -11,6 +11,8 @@ from fsspec.asyn import AsyncFileSystem
		from fsspec.registry import get_filesystem_class
		from fsspec.utils import infer_storage_options

		from . import config

		log = getLogger(__name__)


		@@ -174,3 +176,79 @@ class HPSSFileSystem(AsyncFileSystem):
		May require FS-specific handling, e.g., for relative paths or links.
		"""
		return infer_storage_options(path)["path"]


		class GUCFileSystem(AsyncFileSystem):

		protocol = 'guc'

		async def globus_url_copy(self, *args):
		cmd_args = [
		'-create-dest',
		'-concurrency', '5',
		'-sync-level', '1',
		'-fast',
		'-sync',
		'-continue-on-error',
		'-cred', self.guc_cert.as_posix(),
		]
		if config.debug:
		cmd_args.append('-dbg')
		cmd_args += args
		log.debug('Executing %s %s', self.guc_bin, cmd_args)

		p = await create_subprocess_exec(self.guc_bin, *cmd_args, stdout=PIPE, stderr=STDOUT)
		stdout, stderr = await p.communicate()

		if stdout:
		log.debug(stdout)

		if stderr:
		log.error(stderr)

		rc = p.returncode
		if not rc == 0:
		raise HPSSError(f'globus_url_copy returned {rc}')

		return rc, stdout, stderr

		def __init__(self, args, guc_bin='/bin/globus-url-copy', guc_cert='/opt/globus/cert', guc_endpoint='opendtn.ccs.ornl.gov', root='/', asynchronous=False, loop=None, *storage_options):
		super().__init__(self, args, asynchronous=asynchronous, loop=loop, *storage_options)
		self.guc_cert = guc_cert
		self.guc_bin = guc_bin
		self.guc_endpoint = guc_endpoint
		self.root = root

		async def _info(self, path, **kwargs):
		path = self._strip_protocol(path)
		return await self._ls(path, detail=True, **kwargs)

		async def _ls(self, path, detail=True, **kwargs):
		rc, stdout, stderr = self.globus_url_copy('ls', '-la', path)

		description = self.ls_regex.match(stdout)
		if description is None:
		raise HPSSError('Unable to parse line: %s', stdout)

		yield HPSSEntry(**description.groupdict()).to_fsspec_dict()

		async def _get_file(self, rpath, lpath, **storage_options):
		# rpath = os.path.join(self.root, self._strip_protocol(rpath))
		rpath = f'{self.guc_endpoint}{self._strip_protocol(rpath)}'
		lpath = PosixPath(lpath)
		os.makedirs(lpath.parent, exist_ok=True)
		rc, stdout, stderr = await self.globus_url_copy(rpath, lpath.as_posix())
		if stdout:
		log.info(stdout)
		if stderr:
		log.error(stderr)
		log.info(f'globus-url-copy return code is {rc}')
		if not rc == 0:
		raise RuntimeError(f'globus-url-copy exited {rc}')

		@classmethod
		def _strip_protocol(cls, path):
		"""Turn path from fully-qualified to file-system-specific
		May require FS-specific handling, e.g., for relative paths or links.
		"""
		return infer_storage_options(path)["path"]