Commit 0a86c74c authored by Matt Pryor's avatar Matt Pryor
Browse files

Add utilities for compose-file generation

parent f0b5220b

Jenkinsfile

+48 −45
Original line number Diff line number Diff line
@@ -83,7 +83,6 @@ pipeline 
    ESGF_HUB='esgfhub'

    ESGF_PREFIX=''

    ESGF_DOCKER_REPO_PATH="${env.WORKSPACE}"

    ESGF_HOSTNAME=sh(returnStdout: true, script: 'hostname')

    ESGF_CONFIG="${env.WORKSPACE}/config"

    ESGF_DATA="${env.WORKSPACE}/data"
@@ -240,7 +239,7 @@ pipeline 
            dir(ESGF_DOCKER_REPO_PATH)

            {

              info("building esgf-docker images with the tag '${env.ESGF_VERSION}' and hub '${ESGF_HUB}'")

              sh('docker-compose build')

              sh('docker-compose -f docker-compose.build.yml build')

            }



            end_block('build')
@@ -254,15 +253,21 @@ pipeline 
            start_block('config containers')



            info('delete the previous configuration files of ESGF docker')

            sh 'rm -fr "${ESGF_CONFIG}" ; mkdir "${ESGF_CONFIG}"; mkdir -p "${ESGF_DATA}"'

            sh 'rm -fr "${ESGF_CONFIG}" ; mkdir -p "${ESGF_CONFIG}"; mkdir -p "${ESGF_DATA}"'

            // Write $ESGF_CONFIG/environment config file

            sh '''cat > "${ESGF_CONFIG}/environment" <<EOF

            ESGF_HOSTNAME=$(hostname)

            ESGF_DATA=${ESGF_DATA}

            EOF

            '''

            dir(ESGF_DOCKER_REPO_PATH)

            {

              info('generating esgf secrets')

              sh 'docker-compose run -u $UID esgf-setup generate-secrets'

              sh './bin/esgf-setup generate-secrets'

              info('generating certificates')

              sh 'docker-compose run -u $UID esgf-setup generate-test-certificates'

              sh './bin/esgf-setup generate-test-certificates'

              info('creating trust bundle')

              sh 'docker-compose run -u $UID esgf-setup create-trust-bundle'

              sh './bin/esgf-setup create-trust-bundle'



              // Enable containers to read the private keys.

              sh 'chmod +r "${ESGF_CONFIG}/certificates/hostcert/hostcert.key"'
@@ -294,15 +299,13 @@ pipeline 
              sh(script: """

                   set +x

                   export ESGF_CONFIG=${ESGF_CONFIG}

                   export ESGF_DATA=${ESGF_DATA}

                   export ESGF_HOSTNAME=${ESGF_HOSTNAME}

                   docker-compose up -d

                   ./bin/esgf-compose up -d

                   """)



              info("waiting ${WAITING_TIME} seconds for the containers")

              sleep(time:WAITING_TIME, unit: 'SECONDS')

              info('container status:')

              sh 'docker ps'

              sh './bin/esgf-compose ps'

            }

          }

          post
@@ -355,7 +358,7 @@ pipeline 
            failure

            {

              info('log of the containers:')

              dir(ESGF_DOCKER_REPO_PATH) {sh 'docker-compose logs'}          

              dir(ESGF_DOCKER_REPO_PATH) {sh './bin/esgf-compose logs'}

            }



            // Cleanup is run after all post condition statements.
@@ -428,7 +431,7 @@ pipeline 
                  // the result of the job will be a failure even if the

                  // instructions retried are successful.

                  retry(3)

                  {sh(script: 'docker-compose push')}

                  {sh(script: 'docker-compose -f docker-compose.build.yml push')}

                }

              }

            }
@@ -462,7 +465,7 @@ pipeline 
                info("retagging images with ${env.ESGF_VERSION}")

                dir(ESGF_DOCKER_REPO_PATH)

                {

                 sh('docker-compose build') // Quickly retag the images

                 sh('docker-compose -f docker-compose.build.yml build') // Quickly retag the images

                }

              }

            }
@@ -485,7 +488,7 @@ pipeline 
                  // the result of the job will be a failure even if the

                  // instructions retried are successful.

                  retry(3)

                  {sh(script: 'docker-compose push')}

                  {sh(script: 'docker-compose -f docker-compose.build.yml push')}

                }

              }

            }
@@ -603,7 +606,7 @@ def shutdown() 
  dir(ESGF_DOCKER_REPO_PATH)

  {

    info('shutting down the containers')

    sh 'docker-compose down -v'

    sh './bin/esgf-compose down -v'

  }

}

bin/esgf-compose

0 → 100755
+19 −0
Original line number Diff line number Diff line 
#!/usr/bin/env bash



#####

# This script runs docker-compose using a Compose file generated by esgf-setup

#####



set -e



BIN_DIR="$(realpath "$(dirname "$BASH_SOURCE")")"

REPO_DIR="$(dirname "$BIN_DIR")"



# In order for interactive commands like "run esgf-publisher bash" to work,

# we can't use stdin for the compose file

# So write a tempfile

ESGF_COMPOSE_FILE="$(mktemp)"

"$BIN_DIR/esgf-setup" compose-file > "$ESGF_COMPOSE_FILE"

# By default, specifying the compose file uses the directory containing that

# file as the project directory. So we also need to correct that.

exec docker-compose -f "$ESGF_COMPOSE_FILE" --project-directory "$REPO_DIR" "$@"

bin/esgf-setup

0 → 100755
+26 −0
Original line number Diff line number Diff line 
#!/usr/bin/env bash



#####

# This script just executes the esgf/setup container

#####



if [ -z "$ESGF_CONFIG" ]; then

    echo "ESGF_CONFIG is not set"

fi



# Set the same defaults for these as used by the compose files

: ${ESGF_HUB:=esgfhub}

: ${ESGF_PREFIX:=}

: ${ESGF_VERSION:=latest}



# Run the given command in the docker container with the required environment variables and volumes

# It requires:

#   * The actual value of the ESGF_CONFIG variable for interpolation

#   * The $ESGF_CONFIG directory mounted at /esg

#   * The esgf-docker installation mounted at /opt/esgf-docker

exec docker run --rm -it -u $UID \

  -e ESGF_CONFIG \

  -v "$ESGF_CONFIG:/esg" \

  -v "$(realpath "$(dirname "$BASH_SOURCE")/.."):/opt/esgf-docker" \

  "${ESGF_HUB}/${ESGF_PREFIX}setup:${ESGF_VERSION}" \

  "$@"

docker-compose.build.yml

0 → 100644
+149 −0
Original line number Diff line number Diff line 
#####

# Docker Compose file to build ESGF containers

#####



version: '3.4'





services:

  esgf-configure:

    image: "${ESGF_HUB}/${ESGF_PREFIX}configure:${ESGF_VERSION}"

    build: ./configure

    entrypoint: ["true"]



  esgf-postgres:

    image: "${ESGF_HUB}/${ESGF_PREFIX}postgres:${ESGF_VERSION}"

    build: ./postgres

    entrypoint: ["true"]



  esgf-solr:

    image: "${ESGF_HUB}/${ESGF_PREFIX}solr:${ESGF_VERSION}"

    build: ./solr

    entrypoint: ["true"]



  esgf-tomcat:

    image: "${ESGF_HUB}/${ESGF_PREFIX}tomcat:${ESGF_VERSION}"

    build: ./tomcat

    entrypoint: ["true"]



  esgf-django:

    image: "${ESGF_HUB}/${ESGF_PREFIX}django:${ESGF_VERSION}"

    build: ./django

    entrypoint: ["true"]



  esgf-setup:

    image: "${ESGF_HUB}/${ESGF_PREFIX}setup:${ESGF_VERSION}"

    build: ./setup

    entrypoint: ["true"]



  esgf-publisher:

    image: "${ESGF_HUB}/${ESGF_PREFIX}publisher:${ESGF_VERSION}"

    build: ./publisher

    entrypoint: ["true"]



  esgf-proxy:

    image: "${ESGF_HUB}/${ESGF_PREFIX}proxy:${ESGF_VERSION}"

    build: ./proxy

    entrypoint: ["true"]



  esgf-postgres-security:

    image: "${ESGF_HUB}/${ESGF_PREFIX}postgres-security:${ESGF_VERSION}"

    build:

      context: ./postgres-security

      args:

        ESGF_HUB: $ESGF_HUB

        ESGF_PREFIX: $ESGF_PREFIX

        ESGF_VERSION: $ESGF_VERSION

    entrypoint: ["true"]

    depends_on:

      - esgf-postgres



  esgf-orp:

    image: "${ESGF_HUB}/${ESGF_PREFIX}orp:${ESGF_VERSION}"

    build:

      context: ./orp

      args:

        ESGF_HUB: $ESGF_HUB

        ESGF_PREFIX: $ESGF_PREFIX

        ESGF_VERSION: $ESGF_VERSION

    entrypoint: ["true"]

    depends_on:

      - esgf-configure

      - esgf-tomcat



  esgf-index-node:

    image: "${ESGF_HUB}/${ESGF_PREFIX}index-node:${ESGF_VERSION}"

    build:

      context: ./index-node

      args:

        ESGF_HUB: $ESGF_HUB

        ESGF_PREFIX: $ESGF_PREFIX

        ESGF_VERSION: $ESGF_VERSION

    entrypoint: ["true"]

    depends_on:

      - esgf-configure

      - esgf-tomcat



  esgf-idp-node:

    image: "${ESGF_HUB}/${ESGF_PREFIX}idp-node:${ESGF_VERSION}"

    build:

      context: ./idp-node

      args:

        ESGF_HUB: $ESGF_HUB

        ESGF_PREFIX: $ESGF_PREFIX

        ESGF_VERSION: $ESGF_VERSION

    entrypoint: ["true"]

    depends_on:

      - esgf-configure

      - esgf-tomcat



  esgf-tds:

    image: "${ESGF_HUB}/${ESGF_PREFIX}tds:${ESGF_VERSION}"

    build:

      context: ./tds

      args:

        ESGF_HUB: $ESGF_HUB

        ESGF_PREFIX: $ESGF_PREFIX

        ESGF_VERSION: $ESGF_VERSION

    entrypoint: ["true"]

    depends_on:

      - esgf-configure

      - esgf-tomcat



  esgf-cog:

    image: "${ESGF_HUB}/${ESGF_PREFIX}cog:${ESGF_VERSION}"

    build:

      context: ./cog

      args:

        ESGF_HUB: $ESGF_HUB

        ESGF_PREFIX: $ESGF_PREFIX

        ESGF_VERSION: $ESGF_VERSION

    entrypoint: ["true"]

    depends_on:

      - esgf-configure

      - esgf-django



  esgf-auth:

    image: "${ESGF_HUB}/${ESGF_PREFIX}auth:${ESGF_VERSION}"

    build:

      context: ./auth

      args:

        ESGF_HUB: $ESGF_HUB

        ESGF_PREFIX: $ESGF_PREFIX

        ESGF_VERSION: $ESGF_VERSION

    entrypoint: ["true"]

    depends_on:

      - esgf-configure

      - esgf-django



  esgf-slcs:

    image: "${ESGF_HUB}/${ESGF_PREFIX}slcs:${ESGF_VERSION}"

    build:

      context: ./slcs

      args:

        ESGF_HUB: $ESGF_HUB

        ESGF_PREFIX: $ESGF_PREFIX

        ESGF_VERSION: $ESGF_VERSION

    entrypoint: ["true"]

    depends_on:

      - esgf-django

docker-compose.yml

+60 −185

File changed.

Preview size limit exceeded, changes collapsed.