Commit e611752f authored by Matt Pryor's avatar Matt Pryor
Browse files

Set up CI for ESGF-based repository

parent 7aa5c8fd

.env

+2 −3
Original line number Diff line number Diff line 
# Default environment values used by ESGF/Docker

# Note that any value defined in the shell takes precedence over values defined in this file.



ZOOKEEPER_VERSION=3.4



ESGF_HUB=cedadev

ESGF_HUB=esgfhub

ESGF_PREFIX=

ESGF_VERSION=latest

.travis.yml

+38 −33
Original line number Diff line number Diff line
@@ -11,44 +11,49 @@ addons: 
branches:

  only:

    - master

    - devel



script:

  - |

    export ESGF_VERSION="$(git describe --always --tags)"

    # Define a bash function that just echos something every 60s

    periodic_echo() {

    function periodic_echo {

        while true; do

            sleep 60

            echo "Ensuring execution continues..."

        done

    }

    # Start the build in the background

    docker-compose build &

    build_pid=$!

    # Start the periodic echo in the background

    function keep_alive {

        periodic_echo &

    echo_pid=$!

    # Wait for the build to exit

    wait $build_pid

    # The final result should be the exit status of the build

    build_status=$?

    # Kill the echo

        local echo_pid=$!

        $@ &

        local command_pid=$!

        wait $command_pid

        local command_status=$?

        kill -9 $echo_pid

    # If the build failed, exit now

    [ "$build_status" -ne "0" ] && exit $build_status

    # Only if the branch is master and not a pull request, push to Docker Hub

    if [ "$TRAVIS_BRANCH" = "master" ] && [ "$TRAVIS_PULL_REQUEST" = "false" ]; then

      # Log in to Docker Hub

        [ "$command_status" -ne "0" ] && exit $command_status

    }

    # If the build is for a tag, use that as the image tag

    # If not, use the output of git-describe

    export ESGF_VERSION="${TRAVIS_TAG:-"$(git describe --always --tags)"}"

    keep_alive docker-compose build

    # If this is a pull request, we are done - all we wanted to do is check it builds

    [ "$TRAVIS_PULL_REQUEST" = "false" ] || exit

    # If it is not a pull request, then push the tag to Docker Hub

    docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD"

      # Push with the git describe tag

      docker-compose push

      # Retag with latest and push

      export ESGF_VERSION=latest

      docker-compose build

      docker-compose push

    keep_alive docker-compose push

    # If building master, retag with "latest" and push to Docker Hub

    # If building devel, retag with "devel" and push to Docker Hub

    # Any other branch (which shouldn't happen due to the whitelist above), we are done

    if [ "$TRAVIS_BRANCH" = "master" ]; then

        export ESGF_VERSION="latest"

    elif [ "$TRAVIS_BRANCH" = "devel" ]; then

        export ESGF_VERSION="devel"

    else

        exit

    fi

    keep_alive docker-compose build

    keep_alive docker-compose push



notifications:

  email: false

  slack:

    secure: b1x8LxhKWKHS+cID+9XkFm3JLwGPviTVnPu91L7K9asBEsjYssYFudS/LCSxHdwVC9lJomoKe0ojy8fKx10NAAKacCb7h2TmVh/niaum38f12VeObhm6IPxUz3kytzcbCPR6OIs5fackkDSTuztR2vT2LlLKphs5mlvIMqkUg0wOPNxbl6KN188YSR9ozHfqt4uALitvYNdhkk5NEDmLZOVS//1cwYYHmDGw2uX6xQUf/nudLx14H7ea7eb0y3CyeQeAg6a7rE2iMEbwxqS35B231WYFD5rEkLPIEz/9h9syQRcK6eLObymbu7Ol9kNHa3GcepgI1RYfDJ71rKjzna1Y0rmqxAlf4M7ZvUGhiwqSasY9xgeWFzNn5dp+PsxhryVRUzHZ9W3JLq6NcVFp1evqPOQx94AE5uXwA8WWF5o/65yi4lQQ2uqXeRlYkYsXot8TPowZWV3HZZdQP859az0bivXKGif+N6Q3Ls56zFwQWJbtRFmVq1vDpa7f4wQ/zZysjxmJsYWfyAXo3hfcY59IGFtddoC96NIVSAaozjlKWOjWpG9GPjSNjZHF+GMNNYb/lBw6rddx5NDpvEvCDl3uNiX3CDjR/vZTg3xNkSAX3Ps5HDg+i7rXJ0Y/fe0Lf9dfx3jRxOLsBWgjE8OyZelr+mcgmakPr/aEhaL/pSc=
 
    secure: v3c0zFQbSa5D+tE6CpMZwztWCt7jYeMZeQlgwCrGHCTWduCQhbeeHjzjWLyqqEag/ecvm2k40zWOSFF9HBDwStnXDaW0LmRRhtAQgapNlCvtAxa5IQhTcPtMfki0CrA8xj58SCfHdGbVDRquwNUuvahlhg45ddJBO1mw8hrMS1KRutE3AAZUVTdVJ2N7D6x7qA7i2hKVKjGfGlmHN+eA5ja/nW0J28XRKRXX6ztfelWIy7RNMWnhy/bsjkHNeiaIV6UztTKRi0mIzZSu/DXGM9VvrUleNUTqn/w/CumNnIh+Oboktl1jarGojnW0njRPqZbQ0Xm9mxc/vAcA0vFjLUVdae/fTNKojW2j8GZ2dOf9bFabdLflup6KAlF1+8PoSW9vDtSquYt8Ez1KpTm5YxkwGtf74KklEzDNqVoCrOK8uZWYOd05g0A32FbRZDlPGdouk1oz8Tw0ShOTtUd96pd69l8PsViXN7PkLh8UQ3yQv8E/A3Y7SHYUnFbQPN0uANwhqE9S0m9X/FQHRL2KGNXdnBTUKXaa7nRGxol57lXwWTIQbRwyBfYQf174yfMroZZO12lhmdBDzau5gtKBnPt33lbbjoGQBgRrF2zWukRKuMWOg8H8UiHMgJhcK6MFfMQeB/B0SOw/3t3sLGNm/1grbkiS/8lOOs1d2ZBdK5s=

auth/Dockerfile

+4 −3
Original line number Diff line number Diff line
@@ -5,12 +5,13 @@ 
## a builder image to check out the code and build wheels for all our dependencies

#####



ARG ESGF_HUB=cedadev

ARG ESGF_HUB=esgfhub

ARG ESGF_PREFIX=

ARG ESGF_VERSION=latest



# This build stage is required because COPY --from=$ARG is not supported

# https://github.com/moby/moby/issues/34482

FROM ${ESGF_HUB}/esgf-configure:${ESGF_VERSION} as configuration

FROM ${ESGF_HUB}/${ESGF_PREFIX}configure:${ESGF_VERSION} as configuration



FROM python:2.7-slim AS esgf-auth-build
@@ -38,7 +39,7 @@ RUN mkdir /pip-wheels && \ 
## Stage 2 ##

#############



FROM ${ESGF_HUB}/esgf-django:${ESGF_VERSION}

FROM ${ESGF_HUB}/${ESGF_PREFIX}django:${ESGF_VERSION}



MAINTAINER Earth System Grid Federation <esgf-devel@lists.llnl.gov>

cog/Dockerfile

+4 −3
Original line number Diff line number Diff line
@@ -5,12 +5,13 @@ 
## a builder image to check out the code and build wheels for all our dependencies

#####



ARG ESGF_HUB=cedadev

ARG ESGF_HUB=esgfhub

ARG ESGF_PREFIX=

ARG ESGF_VERSION=latest



# This build stage is required because COPY --from=$ARG is not supported

# https://github.com/moby/moby/issues/34482

FROM ${ESGF_HUB}/esgf-configure:${ESGF_VERSION} as configuration

FROM ${ESGF_HUB}/${ESGF_PREFIX}configure:${ESGF_VERSION} as configuration



FROM python:2.7-slim AS esgf-cog-build
@@ -47,7 +48,7 @@ RUN mkdir /pip-wheels && \ 
## Stage 2 ##

#############



FROM ${ESGF_HUB}/esgf-django:${ESGF_VERSION}

FROM ${ESGF_HUB}/${ESGF_PREFIX}django:${ESGF_VERSION}



MAINTAINER Earth System Grid Federation <esgf-devel@lists.llnl.gov>

docker-compose.yml

+34 −23
Original line number Diff line number Diff line
@@ -17,7 +17,7 @@ version: '3.4' 
services:

  # Build dependency only

  esgf-configure:

    image: "${ESGF_HUB}/esgf-configure:${ESGF_VERSION}"

    image: "${ESGF_HUB}/${ESGF_PREFIX}configure:${ESGF_VERSION}"

    build: ./configure

    # Use an entrypoint that exits immediately with success

    entrypoint: ["true"]
@@ -29,28 +29,28 @@ services: 


  # Build dependency only

  esgf-postgres:

    image: "${ESGF_HUB}/esgf-postgres:${ESGF_VERSION}"

    image: "${ESGF_HUB}/${ESGF_PREFIX}postgres:${ESGF_VERSION}"

    build: ./postgres

    # Use an entrypoint that exits immediately with success

    entrypoint: ["true"]



  # Build dependency only

  esgf-solr:

    image: "${ESGF_HUB}/esgf-solr:${ESGF_VERSION}"

    image: "${ESGF_HUB}/${ESGF_PREFIX}solr:${ESGF_VERSION}"

    build: ./solr

    # Use an entrypoint that exits immediately with success

    entrypoint: ["true"]



  # Build dependency only

  esgf-tomcat:

    image: "${ESGF_HUB}/esgf-tomcat:${ESGF_VERSION}"

    image: "${ESGF_HUB}/${ESGF_PREFIX}tomcat:${ESGF_VERSION}"

    build: ./tomcat

    # Use an entrypoint that exits immediately with success

    entrypoint: ["true"]



  # Build dependency only

  esgf-django:

    image: "${ESGF_HUB}/esgf-django:${ESGF_VERSION}"

    image: "${ESGF_HUB}/${ESGF_PREFIX}django:${ESGF_VERSION}"

    build: ./django

    # Use an entrypoint that exits immediately with success

    entrypoint: ["true"]
@@ -58,7 +58,7 @@ services: 
  # The setup is included here so it can be used with docker-compose run

  # The command is modified to a simple quit with 0 exit status

  esgf-setup:

    image: "${ESGF_HUB}/esgf-setup:${ESGF_VERSION}"

    image: "${ESGF_HUB}/${ESGF_PREFIX}setup:${ESGF_VERSION}"

    build: ./setup

    environment:

      ESGF_HOSTNAME:
@@ -70,7 +70,7 @@ services: 
  # The publisher is included here so it can be used with docker-compose run

  # The command is modified to a simple quit with 0 exit status

  esgf-publisher:

    image: "${ESGF_HUB}/esgf-publisher:${ESGF_VERSION}"

    image: "${ESGF_HUB}/${ESGF_PREFIX}publisher:${ESGF_VERSION}"

    build: ./publisher

    environment:

      <<: *config_env
@@ -96,7 +96,7 @@ services: 
      - esgf-tds



  esgf-proxy:

    image: "${ESGF_HUB}/esgf-proxy:${ESGF_VERSION}"

    image: "${ESGF_HUB}/${ESGF_PREFIX}proxy:${ESGF_VERSION}"

    build: ./proxy

    restart: always

    ports:
@@ -127,16 +127,18 @@ services: 
      - esgf-cog



  esgf-solr-master:

    image: "${ESGF_HUB}/esgf-solr:${ESGF_VERSION}"

    image: "${ESGF_HUB}/${ESGF_PREFIX}solr:${ESGF_VERSION}"

    restart: always

    environment:

      SOLR_HOME: /esg/solr-home

      SOLR_HEAP: 1g

    volumes:

      - "solr-master-home:/esg/solr-home"

    depends_on:

      - esgf-solr



  esgf-solr-slave:

    image: "${ESGF_HUB}/esgf-solr:${ESGF_VERSION}"

    image: "${ESGF_HUB}/${ESGF_PREFIX}solr:${ESGF_VERSION}"

    restart: always

    environment:

      SOLR_HOME: /esg/solr-home
@@ -146,14 +148,16 @@ services: 
    volumes:

      - "solr-slave-home:/esg/solr-home"

    depends_on:

      - esgf-solr

      - esgf-solr-master



  esgf-postgres-esgcet:

    image: "${ESGF_HUB}/esgf-postgres-esgcet:${ESGF_VERSION}"

    image: "${ESGF_HUB}/${ESGF_PREFIX}postgres-esgcet:${ESGF_VERSION}"

    build:

      context: ./postgres-esgcet

      args:

        ESGF_HUB: $ESGF_HUB

        ESGF_PREFIX: $ESGF_PREFIX

        ESGF_VERSION: $ESGF_VERSION

    restart: always

    environment:
@@ -171,13 +175,16 @@ services: 
      - "$ESGF_CONFIG/secrets/database-publisher-password:/esg/secrets/database-publisher-password:ro"

      - "$ESGF_CONFIG/secrets/rootadmin-password:/esg/secrets/rootadmin-password:ro"

      - "postgres-esgcet-data:/var/lib/pgsql/data"

    depends_on:

      - esgf-postgres



  esgf-orp:

    image: "${ESGF_HUB}/esgf-orp:${ESGF_VERSION}"

    image: "${ESGF_HUB}/${ESGF_PREFIX}orp:${ESGF_VERSION}"

    build:

      context: ./orp

      args:

        ESGF_HUB: $ESGF_HUB

        ESGF_PREFIX: $ESGF_PREFIX

        ESGF_VERSION: $ESGF_VERSION

    restart: always

    environment:
@@ -194,11 +201,12 @@ services: 
      - esgf-tomcat



  esgf-index-node:

    image: "${ESGF_HUB}/esgf-index-node:${ESGF_VERSION}"

    image: "${ESGF_HUB}/${ESGF_PREFIX}index-node:${ESGF_VERSION}"

    build:

      context: ./index-node

      args:

        ESGF_HUB: $ESGF_HUB

        ESGF_PREFIX: $ESGF_PREFIX

        ESGF_VERSION: $ESGF_VERSION

    restart: always

    environment:
@@ -216,11 +224,12 @@ services: 
      - esgf-orp



  esgf-idp-node:

    image: "${ESGF_HUB}/esgf-idp-node:${ESGF_VERSION}"

    image: "${ESGF_HUB}/${ESGF_PREFIX}idp-node:${ESGF_VERSION}"

    build:

      context: ./idp-node

      args:

        ESGF_HUB: $ESGF_HUB

        ESGF_PREFIX: $ESGF_PREFIX

        ESGF_VERSION: $ESGF_VERSION

    restart: always

    environment:
@@ -235,11 +244,12 @@ services: 
      - esgf-postgres-esgcet



  esgf-tds:

    image: "${ESGF_HUB}/esgf-tds:${ESGF_VERSION}"

    image: "${ESGF_HUB}/${ESGF_PREFIX}tds:${ESGF_VERSION}"

    build:

      context: ./tds

      args:

        ESGF_HUB: $ESGF_HUB

        ESGF_PREFIX: $ESGF_PREFIX

        ESGF_VERSION: $ESGF_VERSION

    environment:

      <<: *config_env
@@ -259,7 +269,7 @@ services: 
      - esgf-postgres-esgcet



  esgf-postgres-cog:

    image: "${ESGF_HUB}/esgf-postgres:${ESGF_VERSION}"

    image: "${ESGF_HUB}/${ESGF_PREFIX}postgres:${ESGF_VERSION}"

    restart: always

    environment:

      POSTGRESQL_DATABASE: cogdb
@@ -272,11 +282,12 @@ services: 
      - esgf-postgres



  esgf-cog:

    image: "${ESGF_HUB}/esgf-cog:${ESGF_VERSION}"

    image: "${ESGF_HUB}/${ESGF_PREFIX}cog:${ESGF_VERSION}"

    build:

      context: ./cog

      args:

        ESGF_HUB: $ESGF_HUB

        ESGF_PREFIX: $ESGF_PREFIX

        ESGF_VERSION: $ESGF_VERSION

    restart: always

    environment:
@@ -301,7 +312,7 @@ services: 
      - esgf-postgres-cog



  esgf-postgres-auth:

    image: "${ESGF_HUB}/esgf-postgres:${ESGF_VERSION}"

    image: "${ESGF_HUB}/${ESGF_PREFIX}postgres:${ESGF_VERSION}"

    restart: always

    environment:

      POSTGRESQL_DATABASE: auth
@@ -314,11 +325,12 @@ services: 
      - esgf-postgres



  esgf-auth:

    image: "${ESGF_HUB}/esgf-auth:${ESGF_VERSION}"

    image: "${ESGF_HUB}/${ESGF_PREFIX}auth:${ESGF_VERSION}"

    build:

      context: ./auth

      args:

        ESGF_HUB: $ESGF_HUB

        ESGF_PREFIX: $ESGF_PREFIX

        ESGF_VERSION: $ESGF_VERSION

    restart: always

    environment:
@@ -345,7 +357,7 @@ services: 
      - esgf-postgres-auth



  esgf-postgres-slcs:

    image: "${ESGF_HUB}/esgf-postgres:${ESGF_VERSION}"

    image: "${ESGF_HUB}/${ESGF_PREFIX}postgres:${ESGF_VERSION}"

    restart: always

    environment:

      POSTGRESQL_DATABASE: slcs
@@ -358,11 +370,12 @@ services: 
      - esgf-postgres



  esgf-slcs:

    image: "${ESGF_HUB}/esgf-slcs:${ESGF_VERSION}"

    image: "${ESGF_HUB}/${ESGF_PREFIX}slcs:${ESGF_VERSION}"

    build:

      context: ./slcs

      args:

        ESGF_HUB: $ESGF_HUB

        ESGF_PREFIX: $ESGF_PREFIX

        ESGF_VERSION: $ESGF_VERSION

    restart: always

    environment:
@@ -413,8 +426,6 @@ services: 


# Use Docker volumes to store persistent data

volumes:

  zk-data:

  zk-datalog:

  solr-master-home:

  solr-slave-home:

  postgres-auth-data: