Commit 3d30fd67 authored by Price, Zach's avatar Price, Zach
Browse files

Switch to LoadBalance for HSI data movers

parent 705ab900

.gitlab-ci.yml

+9 −20
Original line number Diff line number Diff line
@@ -4,25 +4,15 @@ workflow: 
  - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH



stages:

- test-chart

- build

- deploy





variables:

  IMAGE_REPO: camden.ornl.gov/arm/adl

  PROJECT_NAME: adl

  FF_GITLAB_REGISTRY_HELPER_IMAGE: 'true'



chart_lint:

  stage: test-chart

  image: camden.ornl.gov/hub_proxy/alpine/helm:latest

  tags:

    - arm-k8s

  script:

    - helm lint helm/adl



build-image:

"Build Image":

  stage: build

  image:

    name: gcr.io/kaniko-project/executor:debug
@@ -39,13 +29,11 @@ build-image: 
    --destination "$IMAGE_REPO:$CI_COMMIT_SHORT_SHA"

    --destination "$IMAGE_REPO:latest"



build_chart:

"Build Chart":

  stage: build

  image: camden.ornl.gov/hub_proxy/alpine/helm:latest

  tags:

    - arm-k8s

  needs:

  - job: chart_lint

  artifacts:

    reports:

      dotenv: chart-build.env
@@ -53,6 +41,7 @@ build_chart: 
    - apk add --update --no-cache git

  script:

  - |

    helm lint helm/adl

    CHART_VERSION=$(helm show chart helm/adl | sed -n 's/^version: \(.*\)$/\1/p')

    echo "Repo chart version is ${CHART_VERSION}"

    CHART_VERSION="${CHART_VERSION}-snapshot+${CI_COMMIT_SHORT_SHA}"
@@ -61,7 +50,7 @@ build_chart: 
    helm plugin install https://github.com/chartmuseum/helm-push

    helm push helm/adl --username "${REPO_ROBOT}" --password "${REPO_TOKEN}" --version="${CHART_VERSION}" https://camden.ornl.gov/chartrepo/arm --debug



deploy:

"Deploy for review":

  stage: deploy

  environment:

    name: review/$CI_BUILD_REF_NAME
@@ -71,9 +60,9 @@ deploy: 
  tags:

    - arm-k8s

  needs:

  - job: build_chart

  - job: "Build Chart"

    artifacts: true

  - job: build-image

  - job: "Build Image"

  rules:

  - if: $CI_MERGE_REQUEST_IID

  before_script:
@@ -107,7 +96,7 @@ stop_review: 
    when: manual

  script: helm delete $PROJECT_NAME-$CI_BUILD_REF_SLUG -n adl-review



deploy:prod:

"Production Deployment":

  stage: deploy

  environment:

    name: production
@@ -116,9 +105,9 @@ deploy:prod: 
  tags:

    - arm-k8s

  needs:

  - job: build_chart

  - job: "Build Chart"

    artifacts: true

  - job: build-image

  - job: "Build Image"

  rules:

  - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH

  before_script:

helm/adl/templates/deployment.yaml

+2 −37
Original line number Diff line number Diff line 
{{ $data_ports := split "-" .Values.dataService.portRange }}

{{ $data_port_start := $data_ports._0}}

{{ $data_port_end := $data_ports._1}}

{{ if not (eq .Values.dataService.backend "hsi") }}

apiVersion: apps/v1

kind: Deployment

metadata:
@@ -30,20 +28,6 @@ spec: 
      serviceAccountName: {{ include "adl.serviceAccountName" . }}

      securityContext:

        {{- toYaml .Values.podSecurityContext | nindent 8 }}

      initContainers:

      - name: chown-hsi-keytab

        image: alpine:latest

        command: ["/bin/sh", "-c"]

        args:

        - cp -L /opt/hsi/johnny.keytab /opt/hsi-target/johnny.keytab;

          chmod 0400 /opt/hsi-target/johnny.keytab;

        volumeMounts:

        - name: hsi-keytab-from-secret

          mountPath: /opt/hsi

        - name: hsi-keytab-fixed

          mountPath: /opt/hsi-target

        securityContext:

          runAsUser: 1001

      containers:

        - name: {{ .Chart.Name }}

          securityContext:
@@ -67,27 +51,13 @@ spec: 
            value: {{ .Values.config.hsi.mountPath }}/johnny.keytab

          - name: ADL_CACHE_LOCATION

            value: /data/cache

          - name: HPSS_PORT_RANGE

            value: {{ .Values.dataService.portRange }}

          - name: HPSS_HOSTNAME

            value: {{ .Values.ingress.url }}

          - name: HPSS_PFTPC_PORT_RANGE

            value: ncacn_ip_tcp[{{ .Values.dataService.portRange }}]

          volumeMounts:

          - name: hsi-keytab-fixed

            mountPath: {{ .Values.config.hsi.mountPath }}

            readOnly: True

          - mountPath: /data/cache

            name: cache-storage

          ports:

            - name: http

              containerPort: 8000

              protocol: TCP

            {{- range untilStep ($data_port_start | int ) ($data_port_end | int ) 1 }}

            - name: hsi-data-{{ . }}

              containerPort: {{ . }}

              protocol: TCP

            {{- end }}

          livenessProbe:

            httpGet:

              path: /v1_0/healthz
@@ -99,12 +69,6 @@ spec: 
          resources:

            {{- toYaml .Values.resources | nindent 12 }}

      volumes:

      - name: hsi-keytab-from-secret

        secret:

          secretName: {{ .Values.config.hsi.secretName }}

          defaultMode: 0444

      - name: hsi-keytab-fixed

        emptyDir: {}

      - name: cache-storage

        {{- if .Values.persistence.enabled }}

        persistentVolumeClaim:
@@ -124,3 +88,4 @@ spec: 
      tolerations:

        {{- toYaml . | nindent 8 }}

      {{- end }}

{{ end }}

helm/adl/templates/service.yaml→helm/adl/templates/service-api.yaml

+15 −0
Original line number Diff line number Diff line 
{{ $data_ports := split "-" .Values.dataService.portRange }}

{{ $data_port_start := $data_ports._0}}

{{ $data_port_end := $data_ports._1}}

apiVersion: v1

kind: Service

metadata:
@@ -8,17 +5,11 @@ metadata: 
  labels:

    {{- include "adl.labels" . | nindent 4 }}

spec:

  type: {{ .Values.service.type }}

  type: ClusterIP

  ports:

    - port: {{ .Values.service.port }}

    - port: 8080

      targetPort: http

      protocol: TCP

      name: http

    {{- range untilStep ($data_port_start | int ) ($data_port_end | int ) 1 }}

    - port: {{ . }}

      targetPort: {{ . }}

      protocol: TCP

      name: hsi-data-{{ . }}

    {{- end }}

  selector:

    {{- include "adl.selectorLabels" . | nindent 4 }}

helm/adl/templates/service-data-movers.yaml

0 → 100644
+24 −0
Original line number Diff line number Diff line 
{{ if eq .Values.dataService.backend "hsi" }}

{{ $data_ports := split "-" .Values.dataService.hsi.portRange }}



{{ range $replica, $e := until (int .Values.replicaCount) }}

apiVersion: v1

kind: Service

metadata:

  name: {{ include "adl.fullname" $ }}-data-{{ $replica }}

  labels:

    {{- include "adl.labels" $ | nindent 4 }}

spec:

  type: LoadBalancer

  loadBalancerIP: {{ index $.Values.dataService.hsi.loadBalancerIPs $replica }}

  ports:

    {{- range untilStep (int $data_ports._0) (int $data_ports._1) 1 }}

    - port: {{ . }}

      targetPort: {{ . }}

      protocol: TCP

      name: hsi-data-{{ . }}

    {{- end }}

  selector:

    statefulset.kubernetes.io/pod-name: {{ include "adl.fullname" $ }}-{{ $replica }}

{{ end }}

{{ end }}

helm/adl/templates/statefulset.yaml

0 → 100644
+125 −0
Original line number Diff line number Diff line 
{{ if eq .Values.dataService.backend "hsi" }}

{{ $data_ports := split "-" .Values.dataService.hsi.portRange }}

apiVersion: apps/v1

kind: StatefulSet

metadata:

  name: {{ include "adl.fullname" . }}

  labels:

    {{- include "adl.labels" . | nindent 4 }}

spec:

  selector:

    matchLabels:

      {{- include "adl.selectorLabels" . | nindent 6 }}

  serviceName: {{ include "adl.fullname" $ }}-data-0

  replicas: {{ .Values.replicaCount }}

  template:

    metadata:

      {{- with .Values.podAnnotations }}

      annotations:

        {{- toYaml . | nindent 8 }}

      {{- end }}

      labels:

        {{- include "adl.selectorLabels" . | nindent 8 }}

    spec:

      {{- with .Values.imagePullSecrets }}

      imagePullSecrets:

        {{- toYaml . | nindent 8 }}

      {{- end }}

      serviceAccountName: {{ include "adl.serviceAccountName" . }}

      securityContext:

        {{- toYaml .Values.podSecurityContext | nindent 8 }}

      initContainers:

      - name: chown-hsi-keytab

        image: alpine:latest

        command: ["/bin/sh", "-c"]

        args:

        - cp -L /opt/hsi/johnny.keytab /opt/hsi-target/johnny.keytab;

          chmod 0400 /opt/hsi-target/johnny.keytab;

        volumeMounts:

        - name: hsi-keytab-from-secret

          mountPath: /opt/hsi

        - name: hsi-keytab-fixed

          mountPath: /opt/hsi-target

        securityContext:

          runAsUser: 1001

      containers:

        - name: {{ .Chart.Name }}

          securityContext:

            {{- toYaml .Values.securityContext | nindent 12 }}

          image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"

          imagePullPolicy: {{ .Values.image.pullPolicy }}

          env:

          {{- range $key, $val := .Values.config.db }}

          - name: ADL_DB_{{ upper $key }}

            valueFrom:

              secretKeyRef:

                name: {{ include "adl.fullname" $ }}-dbsecret

                key: ADL_DB_{{ upper $key }}

          {{- end}}

          - name: ADL_HSI_USER

            valueFrom:

              secretKeyRef:

                name: {{ .Values.config.hsi.secretName }}

                key: user

          - name: ADL_HSI_KEYTAB

            value: {{ .Values.config.hsi.mountPath }}/johnny.keytab

          - name: ADL_CACHE_LOCATION

            value: /data/cache

          - name: HPSS_PORT_RANGE

            value: {{ .Values.dataService.hsi.portRange }}

          - name: HPSS_HOSTNAME

            value: {{ index .Values.dataService.hsi.loadBalancerIPs 0 }}

          - name: HPSS_PFTPC_PORT_RANGE

            value: ncacn_ip_tcp[{{ .Values.dataService.hsi.portRange }}]

          volumeMounts:

          - mountPath: /data/cache

            name: cache-storage

          - name: hsi-keytab-fixed

            mountPath: {{ .Values.config.hsi.mountPath }}

            readOnly: True

          ports:

            - name: http

              containerPort: 8000

              protocol: TCP

            {{- range untilStep (int $data_ports._0) (int $data_ports._1) 1 }}

            - containerPort: {{ . }}

              protocol: TCP

              name: hsi-data-{{ . }}

            {{- end }}

          livenessProbe:

            httpGet:

              path: /v1_0/healthz

              port: http

          readinessProbe:

            httpGet:

              path: /v1_0/healthz

              port: http

          resources:

            {{- toYaml .Values.resources | nindent 12 }}

      volumes:

      - name: cache-storage

        {{- if .Values.persistence.enabled }}

        persistentVolumeClaim:

          claimName: {{ template "adl.fullname" . }}-cache

        {{- else }}

        emptyDir: {}

        {{- end }}

      - name: hsi-keytab-from-secret

        secret:

          secretName: {{ .Values.config.hsi.secretName }}

          defaultMode: 0444

      - name: hsi-keytab-fixed

        emptyDir: {}

      {{- with .Values.nodeSelector }}

      nodeSelector:

        {{- toYaml . | nindent 8 }}

      {{- end }}

      {{- with .Values.affinity }}

      affinity:

        {{- toYaml . | nindent 8 }}

      {{- end }}

      {{- with .Values.tolerations }}

      tolerations:

        {{- toYaml . | nindent 8 }}

      {{- end }}

{{ end }}