Configure fetching via HSI

  stage: deploy

  environment:

    name: review/$CI_BUILD_REF_NAME

    url: https://api-$PROJECT_NAME-$CI_BUILD_REF_SLUG.k8s.arm.gov/docs

    url: https://api-$CI_BUILD_REF_SLUG.k8s.arm.gov/docs

    on_stop: stop_review

  image: camden.ornl.gov/hub_proxy/alpine/helm:latest

  tags:

FROM camden.ornl.gov/hub_proxy/centos/python-38-centos7:latest

COPY bin/* /usr/local/bin/

COPY HPSS.conf /usr/local/etc/HPSS.conf

COPY app app

RUN pip install --no-cache-dir ./app

CMD ["uvicorn", "ADL.api.app:app", "--host", "0.0.0.0"]

; HPSS.conf

;

; NOTE:  This file has been streamlined and customized for ORNL/CCS.  An

; original template can be found in /opt/hpss/config/templates.

;

;

; This stanza is used by PFTP clients to set some runtime parameters.  For

; more documentation, see page 163 of the HPSS 4.5 Installation Guide.

;

PFTP Client = {

    ; The Default COS for PFTP puts.  We probably don't want to use this.

    ;

    # Default COS = 100

    ; Default Mover protocol.  Must be PDATA_AND_MOVER or PDATA_ONLY.

    ; The usual protocol is PDATA_AND_MOVER.  The PDATA_ONLY protocol can

    ; improve performance over WAN connections.

    ;

    Protocol = PDATA_AND_MOVER

    ; Files smaller than this size will not be considered for parallel puts.

    ; This improves performance for small files.

    ;

    Auto Parallel Size = 4MB

    ; Port range for firewalls.  This is identical to RPC_RESTRICTED_PORTS

    ; on most of our DCE systems.

    ;

    PortRange = ncadg_ip_udp[10100-12100]:ncadg_ip_tcp[10100-12100]

    ; PDATA Options.

    ;

    Parallel Block Size = 1MB

    Transfer Buffer Size = 16MB

    Socket Buffer Size = 1MB

    ; Ptran size is the amount of data transferred before PFTP closes its

    ; sockets and reopens them again.  A value which is too small can degrade

    ; performance with excessive socket rebuilding.  There is a hard-coded

    ; maximum of 250GB.

    ;

    MAX Ptran Size = 10GB

}

; This stanza tells the PFTP client which network interface(s) to use to

; talk to remote PFTP daemons and Movers.  This is most useful for hosts

; with multiple network adapters.  Multiple client hosts can be specified

; here, so the file can be shared among them.

;

;PFTP Client Interfaces = {

    ; As an example, assume the client is eagle163s, which uses all of these

    ; IP addresses:

    ;

    ;    160.91.195.163

    ;    160.91.199.163

    ;    160.91.199.248

    ;    160.91.196.163

    ;

    ; For PFTP transfers, it needs to talk to the PFTP daemon host and the

    ; HPSS Mover hosts; i.e., arm16, bobcat, and mooch.  All these have IP

    ; addresses in the 199 subnet.  Presumably, it will be most efficient

    ; for eagle163s to use its 199 subnet addresses to talk to these hosts.

    ; So the entry would look like this:

    ;

    # eagle163s eagle163s.ccs.ornl.gov = {

	; The Default section can be specified to tell eagle163s which of its

	; IP addresses to use if its target host is not found below.

	# Default = {

 	    # 160.91.199.163

 	# }

	; Now specify the IP addresses to use when talking to the HPSS hosts.

	# arm16 arm16.ccs.ornl.gov = {

	    # 160.91.199.163

	    # 160.91.199.248

	# }

    # }

    ; Now another client section could be added for a different host.

    # cheetah0033 cheetah0033.ccs.ornl.gov = {

        # blah blah as before...

    # }

;}

; Network transfer options used by all components (PFTP, Movers, HSI, and

; NDAPI daemons).  This is by far the most important section for getting

; decent performance out of the Movers and HSI!

;

Network Options = {

    ; Default network write size if not specified as WriteSize below.  Its

    ; usefulness seems dubious.

    ;

    ; Default Write Size = 512KB

    ; Specify per-host network options.

    ;

    ; NOTE 1: HSI does NOT do differentiate between the various clients -

    ;       it essentially ignores the first level, and builds its tables

    ;       from all the various client substanzas.  When searching for

    ;       an IP address, it stops at the first one it finds that matches

    ;       based upon IP address and Netmask.  This simplifies setting

    ;       values for an entire subnet, with the caveat that if multiple

    ;       client-level stanzas reference the same destination IP address,

    ;       the first one in the list will be used.

    ; 

    ; NOTE 2: The sb_max setting on AIX, and the system max setting on

    ;       other systems, must be set high enough to allow the settings herein

    ;       to be used.

    ;

    ;     160.91.199 jumbo gige net

    Default = {

        ; 202 subnet hosts

        160.91.202.192 = {

            NetMask = 255.255.255.192

            RFC1323 = 1

            SendSpace = 128KB

            RecvSpace = 128KB

            WriteSize = 32KB

            TcpNoDelay = 1

        }

	; 195/192 subnet hosts

	160.91.195.192 = {

            NetMask = 255.255.255.192

            RFC1323 = 1

            SendSpace = 128KB

            RecvSpace = 128KB

            WriteSize = 32KB

            TcpNoDelay = 1

	}

        ; The default for everybody else

        Default = {

            NetMask = 255.255.255.255

            RFC1323 = 1

            SendSpace = 128KB

            RecvSpace = 128KB

            WriteSize = 32KB

            TcpNoDelay = 1

         }  # Default dest. host 

    } # end of Default client stanza

} # end of Network Options section

PFTP Client Interfaces = {

    ret1 ret1.ornl.gov = {

        hpss-nccs hpss-nccs.ccs.ornl.gov = {

            eth2

        }

    }

}

Transfer Agent = {

        # The optional "Umask" setting is the value that the Transfer Agent

        # should set when it starts up.  It is specified as a 3-digit

        #  octal value.

        Umask = 022

}

app = VersionedFastAPI(app)

class EndpointFilter(logging.Filter):

    '''

    Don't log acess to the healthcheck to same log spam.

    '''

    def filter(self, record: logging.LogRecord) -> bool:

        return record.getMessage().find("/healthz") == -1

def setup_logging():

    logger = logging.getLogger()

    logging.config.dictConfig(config.logging)

    if not config.debug:

        logging.getLogger("uvicorn.access").addFilter(EndpointFilter())

@app.on_event("startup")

import sys

from logging import getLogger

from pydantic import BaseSettings, PostgresDsn, SecretStr

from pydantic import (BaseSettings, DirectoryPath, FilePath, PostgresDsn,

                      SecretStr)

from sqlalchemy.engine.url import URL

logger = getLogger(__name__)

class Settings(BaseSettings):

    debug: bool = True

    hsi_user: str = "johnny"

    hsi_keytab: FilePath = "/opt/johnny.keytab"

    hsi_bin: FilePath = "/usr/local/bin/hsi"

    cache_location: DirectoryPath = "/data/cache"

    stream_bytes_per_response: int = 100000

    db_drivername: str = "postgresql"

    db_username: str = "db_user"

    db_password: SecretStr = "S00perS3kr1t"

            "uvicorn.access": {"propagate": True},

            "uvicorn.asgi": {"propagate": True},

            "uvicorn.error": {"propagate": True},

            "ADL": {"propogate": True},

        },

    }