Unverified Commit 7aa5c8fd authored by Matt Pryor's avatar Matt Pryor Committed by GitHub
Browse files

Changes from cedadev fork (#43)

* Postgres container working

* Improvements to solr, tomcat and index-node

* Improved postgres container

* SLCS is working

* idp-node seems to be working

* Changes to support URL prefix

* Proxy -> CoG working

* Solr + Zookeeper working

* Index node working + certificates

* Revamped configuration + index node

* Use a common config container

* ORP seems to be working

* Setup container + fixes to compose

* SLCS working

* Auth app seems to run, but isn't working properly

* Should have been in last commit

* Set theme jekyll-theme-cayman

* Testing GitHub pages for project

* Testing Github pages site

* Create architecture.md

* Create comparison.md

* Quick start done

* Comparison with previous impl

* Container architecture docs

* Add version and hub to docker-compose

* Docs on configuration

* Changes while doing Helm integration

* Changes to support OpenShift deployment

* Fix problem with ORP trusting IDP

* Changes to make hub and version flexible

* Fix certificates, move generation of Java certs into containers

* Publishing seems to be working apart from some issues with facets

* Fix description of init

* Get index node to query Solr for facets correctly

* Add Travis support

* Extend Travis timeout

* Fix timeout but still get output

* Fixed omission of while loop

* Bring docker push into actual command

* Bring back-to-top link to top

* Small modifications to configs in prep for future work

* Added helm-values command + pinned conda version in publisher

* Make HTTPS everywhere the default

* Fix compose file for newly required variable

* Make containers work as non-root

* Improved postgres containers for permissions and separated cogdb

* Revert to master + slave for Solr
parent 1f41ee5f
Loading
Loading
Loading
Loading
+4 −5
Original line number Diff line number Diff line
# Default environment values used by ESGF/Docker
# Note that any value defined in the shell takes precedence over values defined in this file.

ESGF_HOSTNAME=my-node.esgf.org
ESGF_CONFIG=~/esgf_config
ESGF_DATA_DIR=/tmp
ESGF_VERSION=1.0
ESGF_IMAGES_HUB=esgfhub
 No newline at end of file
ZOOKEEPER_VERSION=3.4

ESGF_HUB=cedadev
ESGF_VERSION=latest

.travis.yml

0 → 100644
+54 −0
Original line number Diff line number Diff line
sudo: required

services:
  - docker

addons:
  apt:
    packages:
    - docker-ce

branches:
  only:
    - master

script:
  - |
    export ESGF_VERSION="$(git describe --always --tags)"
    # Define a bash function that just echos something every 60s
    periodic_echo() {
      while true; do
        sleep 60
        echo "Ensuring execution continues..."
      done
    }
    # Start the build in the background
    docker-compose build &
    build_pid=$!
    # Start the periodic echo in the background
    periodic_echo &
    echo_pid=$!
    # Wait for the build to exit
    wait $build_pid
    # The final result should be the exit status of the build
    build_status=$?
    # Kill the echo
    kill -9 $echo_pid
    # If the build failed, exit now
    [ "$build_status" -ne "0" ] && exit $build_status
    # Only if the branch is master and not a pull request, push to Docker Hub
    if [ "$TRAVIS_BRANCH" = "master" ] && [ "$TRAVIS_PULL_REQUEST" = "false" ]; then
      # Log in to Docker Hub
      docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD"
      # Push with the git describe tag
      docker-compose push
      # Retag with latest and push
      export ESGF_VERSION=latest
      docker-compose build
      docker-compose push
    fi

notifications:
  email: false
  slack:
    secure: b1x8LxhKWKHS+cID+9XkFm3JLwGPviTVnPu91L7K9asBEsjYssYFudS/LCSxHdwVC9lJomoKe0ojy8fKx10NAAKacCb7h2TmVh/niaum38f12VeObhm6IPxUz3kytzcbCPR6OIs5fackkDSTuztR2vT2LlLKphs5mlvIMqkUg0wOPNxbl6KN188YSR9ozHfqt4uALitvYNdhkk5NEDmLZOVS//1cwYYHmDGw2uX6xQUf/nudLx14H7ea7eb0y3CyeQeAg6a7rE2iMEbwxqS35B231WYFD5rEkLPIEz/9h9syQRcK6eLObymbu7Ol9kNHa3GcepgI1RYfDJ71rKjzna1Y0rmqxAlf4M7ZvUGhiwqSasY9xgeWFzNn5dp+PsxhryVRUzHZ9W3JLq6NcVFp1evqPOQx94AE5uXwA8WWF5o/65yi4lQQ2uqXeRlYkYsXot8TPowZWV3HZZdQP859az0bivXKGif+N6Q3Ls56zFwQWJbtRFmVq1vDpa7f4wQ/zZysjxmJsYWfyAXo3hfcY59IGFtddoC96NIVSAaozjlKWOjWpG9GPjSNjZHF+GMNNYb/lBw6rddx5NDpvEvCDl3uNiX3CDjR/vZTg3xNkSAX3Ps5HDg+i7rXJ0Y/fe0Lf9dfx3jRxOLsBWgjE8OyZelr+mcgmakPr/aEhaL/pSc=
+6 −1
Original line number Diff line number Diff line
# esgf-docker
ESGF software stack as Docker images

ESGF software stack as Docker images.

## Documentation

For documentation, please visit [cedadev.github.io/esgf-docker](https://cedadev.github.io/esgf-docker).
+68 −38
Original line number Diff line number Diff line
# ESGF-Auth client application.
#####
## Image for running the ESGF Auth client Django application
##
## To avoid including unnecessary build dependencies in the final image, we use
## a builder image to check out the code and build wheels for all our dependencies
#####

ARG ESGF_IMAGES_HUB=esgfhub
ARG ESGF_VERSION=devel
ARG ESGF_REPO=http://distrib-coffee.ipsl.jussieu.fr/pub/esgf
ARG ESGF_HUB=cedadev
ARG ESGF_VERSION=latest

FROM $ESGF_IMAGES_HUB/esgf-node:$ESGF_VERSION
MAINTAINER Earth System Grid Federation <esgf-devel@lists.llnl.gov>
# This build stage is required because COPY --from=$ARG is not supported
# https://github.com/moby/moby/issues/34482
FROM ${ESGF_HUB}/esgf-configure:${ESGF_VERSION} as configuration

FROM python:2.7-slim AS esgf-auth-build

# Install git for checking out the source code and pulling dependencies
RUN apt-get update && \
    apt-get install -y git && \
    rm -rf /var/lib/apt/lists/*

# Clone the github repo and checkout the specified version
ARG ESGF_AUTH_GIT_VERSION=master
RUN git clone https://github.com/ESGF/esgf-auth.git /application && \
    cd /application && \
    git checkout $ESGF_AUTH_GIT_VERSION && \
    rm -rf .git && \
    cd -

# installation directory
ENV ESGF_AUTH_INSTALL_DIR=/usr/local/esgf-auth
# Create wheels from the requirements
RUN mkdir /pip-wheels && \
    pip wheel --wheel-dir /pip-wheels git+https://github.com/philipkershaw/crypto-cookie.git && \
    pip wheel --wheel-dir /pip-wheels psycopg2 && \
    pip wheel --wheel-dir /pip-wheels -r /application/requirements.txt

# library dependencies
RUN yum -y update; yum -y install libffi-devel; yum clean all

# create python virtual environment
RUN mkdir -p $ESGF_AUTH_INSTALL_DIR && \
    cd $ESGF_AUTH_INSTALL_DIR && \
    virtualenv venv
#############
## Stage 2 ##
#############

FROM ${ESGF_HUB}/esgf-django:${ESGF_VERSION}

MAINTAINER Earth System Grid Federation <esgf-devel@lists.llnl.gov>

# install source code
RUN mkdir -p $ESGF_AUTH_INSTALL_DIR/src
# Copy the application and wheels from the build stage
COPY --from=esgf-auth-build /application /application
COPY --from=esgf-auth-build /pip-wheels  /pip-wheels

# install crypto cookie
RUN cd $ESGF_AUTH_INSTALL_DIR && \
    source venv/bin/activate && \
    cd src && \
    git clone https://github.com/philipkershaw/crypto-cookie.git && \
    cd crypto-cookie && \
    python setup.py install
# Install the dependency wheels (remove the wheels when we are done)
RUN pip install --no-index --find-links=/pip-wheels -r /application/requirements.txt && \
    pip install --no-index --find-links=/pip-wheels crypto-cookie && \
    pip install --no-index --find-links=/pip-wheels psycopg2 && \
    rm -rf /pip-wheels

# copy mock configuration file which is read by settings.py during the database creation
COPY conf/esgf_auth_config.json /esg/config/esgf_auth_config.json
# Because esgf-auth has no setup.py, we can't use pip install -e /application
# Instead, we add /application to the PYTHONPATH
ENV PYTHONPATH /application:$PYTHONPATH

# install esgf-auth
RUN cd $ESGF_AUTH_INSTALL_DIR && \
    source venv/bin/activate && \
    cd src && \
    git clone https://github.com/lukaszlacinski/esgf-auth.git && \
    cd esgf-auth && \
    pip install -r requirements.txt && \
    python ./manage.py migrate
# Install ESGF Auth settings wrappers
COPY python/*.py /usr/local/lib/python2.7/site-packages/

# change permission on parent directory and database file so user apache can write
RUN chmod a+w /usr/local/esgf-auth/src/esgf-auth && \
    chmod a+w /usr/local/esgf-auth/src/esgf-auth/db.sqlite3
# Install configuration files
# Ensure the config directory is owned by the Django user and root group
RUN mkdir -p /esg/config && chown 1001:0 /esg/config && chmod 775 /esg/config
COPY --chown=1001:0 --from=configuration /esg/config /esg/config
# Make sure the config interpolation runs first by giving it a low ordinal
COPY --from=configuration /opt/esgf-docker/scripts/interpolate-configs.sh /django-init.d/01-interpolate.sh
# Install the template auth config file
ENV ESGF_AUTH_CONFIG_FILE /esg/auth/esgf_auth_config.json
# Ensure /esg/auth exists and is owned by the Django user and root group
RUN mkdir -p /esg/auth && chown 1001:0 /esg/auth && chmod 775 /esg/auth
COPY conf/esgf_auth_config.json.template "$ESGF_AUTH_CONFIG_FILE.template"
# This script interpolates the auth config files with additional environment variables
COPY scripts/auth-configure.sh /django-init.d/02-auth-configure.sh

VOLUME /usr/local/esgf-auth
# Run as the Django user unless otherwise specified
USER 1001

CMD ["supervisord", "--nodaemon", "-c", "/etc/supervisord.conf"]
# The "command" is just the settings module to use
CMD ["esgf_auth_settings"]

auth/conf/esgf_auth_config.json

deleted100644 → 0
+0 −5
Original line number Diff line number Diff line
{
"ESGF_HOSTNAME":"localhost",
"ESGF_SECRET_KEY":"A7JiFckYWpqezusdOfJoH[==",
"DJANGO_SECRET_KEY":"5_7$jn1-c8h5@(e$x8fhk=&!mk=ea6=irwm+b22nu)h=rhox46"
}
Loading