Commit e33ca4ac authored by Price, Zach's avatar Price, Zach
Browse files

Merge branch 'stage_via_guc' into 'master' 

Configure fetching via globus-url-copy

See merge request !3
parents 705ab900 f20283ec

.gitlab-ci.yml

+13 −24
Original line number Diff line number Diff line
@@ -14,19 +14,10 @@ variables: 
  PROJECT_NAME: adl

  FF_GITLAB_REGISTRY_HELPER_IMAGE: 'true'



chart_lint:

  stage: test-chart

  image: camden.ornl.gov/hub_proxy/alpine/helm:latest

  tags:

    - arm-k8s

  script:

    - helm lint helm/adl



build-image:

"Build API Image":

  stage: build

  image:

    name: gcr.io/kaniko-project/executor:debug

    entrypoint: [""]

  tags:

    - arm-k8s

  script:
@@ -35,17 +26,15 @@ build-image: 
    /kaniko/executor

    --cache=true

    --context "$CI_PROJECT_DIR"

    --dockerfile Dockerfile

    --dockerfile Dockerfile-api

    --destination "$IMAGE_REPO:$CI_COMMIT_SHORT_SHA"

    --destination "$IMAGE_REPO:latest"



build_chart:

"Build Chart":

  stage: build

  image: camden.ornl.gov/hub_proxy/alpine/helm:latest

  image: camden.ornl.gov/arm/toolbox:latest

  tags:

    - arm-k8s

  needs:

  - job: chart_lint

  artifacts:

    reports:

      dotenv: chart-build.env
@@ -53,13 +42,13 @@ build_chart: 
    - apk add --update --no-cache git

  script:

  - |

    helm lint helm/adl

    CHART_VERSION=$(helm show chart helm/adl | sed -n 's/^version: \(.*\)$/\1/p')

    echo "Repo chart version is ${CHART_VERSION}"

    CHART_VERSION="${CHART_VERSION}-snapshot+${CI_COMMIT_SHORT_SHA}"

    echo "Augmented chart version is ${CHART_VERSION}"

    echo "CHART_VERSION=${CHART_VERSION}" >> chart-build.env

    helm plugin install https://github.com/chartmuseum/helm-push

    helm push helm/adl --username "${REPO_ROBOT}" --password "${REPO_TOKEN}" --version="${CHART_VERSION}" https://camden.ornl.gov/chartrepo/arm --debug

    helm cm-push helm/adl --username "${REPO_ROBOT}" --password "${REPO_TOKEN}" --version="${CHART_VERSION}" https://camden.ornl.gov/chartrepo/arm --debug



deploy:

  stage: deploy
@@ -67,13 +56,13 @@ deploy: 
    name: review/$CI_BUILD_REF_NAME

    url: https://api-$CI_BUILD_REF_SLUG.k8s.arm.gov/docs

    on_stop: stop_review

  image: camden.ornl.gov/hub_proxy/alpine/helm:latest

  image: camden.ornl.gov/arm/toolbox:latest

  tags:

    - arm-k8s

  needs:

  - job: build_chart

  - job: "Build Chart"

    artifacts: true

  - job: build-image

  - job: "Build API Image"

  rules:

  - if: $CI_MERGE_REQUEST_IID

  before_script:
@@ -99,7 +88,7 @@ stop_review: 
    name: review/$CI_BUILD_REF_NAME

    url: https://$PROJECT_NAME-$CI_BUILD_REF_SLUG.k8s.arm.gov/docs

    action: stop

  image: camden.ornl.gov/hub_proxy/alpine/helm:latest

  image: camden.ornl.gov/arm/toolbox:latest

  tags:

    - arm-k8s

  rules:
@@ -112,13 +101,13 @@ deploy:prod: 
  environment:

    name: production

    url: https://api.k8s.arm.gov

  image: camden.ornl.gov/hub_proxy/alpine/helm:latest

  image: camden.ornl.gov/arm/toolbox:latest

  tags:

    - arm-k8s

  needs:

  - job: build_chart

  - job: "Build Chart"

    artifacts: true

  - job: build-image

  - job: "Build API Image"

  rules:

  - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH

  before_script:

Dockerfile

deleted100644 → 0
+0 −6
Original line number Diff line number Diff line 
FROM camden.ornl.gov/hub_proxy/centos/python-38-centos7:latest

COPY bin/* /usr/local/bin/

COPY HPSS.conf /usr/local/etc/HPSS.conf

COPY app app

RUN pip install --no-cache-dir ./app

CMD ["uvicorn", "ADL.api.app:app", "--host", "0.0.0.0"]

Dockerfile-api

0 → 100644
+25 −0
Original line number Diff line number Diff line 
FROM camden.ornl.gov/hub_proxy/centos/python-38-centos7:latest

USER 0

RUN yum install -y epel-release && \

    yum update -y && \

    yum install -y globus-gass-copy-progs myproxy && \

    yum clean all



# Fetch the certs from HPSS globus connector. Globus barfs on the ' in Let's Encrypt

# so, grab them and let it fail, edit to escape the ' and run it again

RUN myproxy-get-trustroots -bvs myproxy1.ccs.ornl.gov || true

RUN sed -i 's/Let\x27s/Let\\\x27s/' /etc/grid-security/certificates/*.signing_policy

RUN myproxy-get-trustroots -bvs myproxy1.ccs.ornl.gov



# and once again for the Open DTN

# ...except this is broken...

# RUN myproxy-get-trustroots -bvs myproxy1.op.ccs.ornl.gov || true

# RUN sed -i 's/Let\x27s/Let\\\x27s/' /etc/grid-security/certificates/*.signing_policy

# RUN myproxy-get-trustroots -bvs myproxy1.op.ccs.ornl.gov



USER 1001

COPY bin/ /usr/local/bin/

COPY HPSS.conf /usr/local/etc/HPSS.conf

COPY app app

RUN pip install --no-cache-dir ./app

CMD ["uvicorn", "ADL.api.app:app", "--host", "0.0.0.0"]

app/ADL/api/config.py

+3 −0
Original line number Diff line number Diff line
@@ -13,6 +13,9 @@ class Settings(BaseSettings): 
    hsi_user: str = "johnny"

    hsi_keytab: FilePath = "/opt/johnny.keytab"

    hsi_bin: FilePath = "/usr/local/bin/hsi"

    guc_cert: FilePath = "/opt/globus.pem"

    guc_bin: str = "/usr/bin/globus-url-copy"

    guc_endpoint: str= "gsiftp://hpss-gridftp.ccs.ornl.gov"

    cache_location: DirectoryPath = "/data/cache"

    stream_bytes_per_response: int = 100000

    db_drivername: str = "postgresql"

app/ADL/api/fs.py

+78 −0
Original line number Diff line number Diff line
@@ -11,6 +11,8 @@ from fsspec.asyn import AsyncFileSystem 
from fsspec.registry import get_filesystem_class

from fsspec.utils import infer_storage_options



from . import config



log = getLogger(__name__)
@@ -174,3 +176,79 @@ class HPSSFileSystem(AsyncFileSystem): 
        May require FS-specific handling, e.g., for relative paths or links.

        """

        return infer_storage_options(path)["path"]





class GUCFileSystem(AsyncFileSystem):



    protocol = 'guc'



    async def globus_url_copy(self, *args):

        cmd_args = [

            '-create-dest',

            '-concurrency', '5',

            '-sync-level', '1',

            '-fast',

            '-sync',

            '-continue-on-error',

            '-cred', self.guc_cert.as_posix(),

        ]

        if config.debug:

            cmd_args.append('-dbg')

        cmd_args += args

        log.debug('Executing %s %s', self.guc_bin, cmd_args)



        p = await create_subprocess_exec(self.guc_bin, *cmd_args, stdout=PIPE, stderr=STDOUT)

        stdout, stderr = await p.communicate()



        if stdout:

            log.debug(stdout)



        if stderr:

            log.error(stderr)



        rc = p.returncode

        if not rc == 0:

            raise HPSSError(f'globus_url_copy returned {rc}')



        return rc, stdout, stderr



    def __init__(self, *args, guc_bin='/bin/globus-url-copy', guc_cert='/opt/globus/cert', guc_endpoint='opendtn.ccs.ornl.gov', root='/', asynchronous=False, loop=None, **storage_options):

        super().__init__(self, *args, asynchronous=asynchronous, loop=loop, **storage_options)

        self.guc_cert = guc_cert

        self.guc_bin = guc_bin

        self.guc_endpoint = guc_endpoint

        self.root = root



    async def _info(self, path, **kwargs):

        path = self._strip_protocol(path)

        return await self._ls(path, detail=True, **kwargs)



    async def _ls(self, path, detail=True, **kwargs):

        rc, stdout, stderr = self.globus_url_copy('ls', '-la', path)



        description = self.ls_regex.match(stdout)

        if description is None:

            raise HPSSError('Unable to parse line: %s', stdout)



        yield HPSSEntry(**description.groupdict()).to_fsspec_dict()



    async def _get_file(self, rpath, lpath, **storage_options):

        # rpath = os.path.join(self.root, self._strip_protocol(rpath))

        rpath = f'{self.guc_endpoint}{self._strip_protocol(rpath)}'

        lpath = PosixPath(lpath)

        os.makedirs(lpath.parent, exist_ok=True)

        rc, stdout, stderr = await self.globus_url_copy(rpath, lpath.as_posix())

        if stdout:

            log.info(stdout)

        if stderr:

            log.error(stderr)

        log.info(f'globus-url-copy return code is {rc}')

        if not rc == 0:

            raise RuntimeError(f'globus-url-copy exited {rc}')



    @classmethod

    def _strip_protocol(cls, path):

        """Turn path from fully-qualified to file-system-specific

        May require FS-specific handling, e.g., for relative paths or links.

        """

        return infer_storage_options(path)["path"]