Commit 4c849d87 authored by Matt Pryor's avatar Matt Pryor
Browse files

Add ability to override configs + download static lists

parent b9552074

django/scripts/django-serve.sh

+3 −1
Original line number Diff line number Diff line
@@ -29,6 +29,8 @@ info "Updating trusted certificates" 
cat /etc/ssl/certs/ca-certificates.crt > /var/run/django/conf/trust-bundle.pem

cat /esg/certificates/esg-trust-bundle.pem >> /var/run/django/conf/trust-bundle.pem

export SSL_CERT_FILE=/var/run/django/conf/trust-bundle.pem

# Also set the requests-specific environment variable, as it doesn't respect SSL_CERT_FILE

export REQUESTS_CA_BUNDLE="${SSL_CERT_FILE}"



# Run database migrations

info "Running database migrations"
@@ -122,4 +124,4 @@ exec gunicorn \ 
    --access-logfile '-' \

    --error-logfile '-' \

    --log-level ${GUNICORN_LOG_LEVEL:-info} \

    --workers ${GUNICORN_WORKERS:-1}
 
    --workers ${GUNICORN_WORKERS:-4}

docker-compose.yml

+18 −0
Original line number Diff line number Diff line
@@ -225,6 +225,8 @@ services: 
      - "$ESGF_CONFIG/certificates/hostcert:/esg/certificates/hostcert:ro"

      # Make sure the trusted certificate bundle is available

      - "$ESGF_CONFIG/certificates/esg-trust-bundle.pem:/esg/certificates/esg-trust-bundle.pem:ro"

      # Allow the main ESGF configuration to be overridden

      - "$ESGF_CONFIG/config:/esg/config/.overrides:ro"

    depends_on:

      - esgf-configure

      - esgf-tomcat
@@ -244,6 +246,8 @@ services: 
    volumes:

      # Make sure the trusted certificate bundle is available

      - "$ESGF_CONFIG/certificates/esg-trust-bundle.pem:/esg/certificates/esg-trust-bundle.pem:ro"

      # Allow the main ESGF configuration to be overridden

      - "$ESGF_CONFIG/config:/esg/config/.overrides:ro"

    depends_on:

      - esgf-configure

      - esgf-tomcat
@@ -268,6 +272,8 @@ services: 
      - "$ESGF_CONFIG/secrets/rootadmin-password:/esg/config/.esgf_pass:ro"

      # Make sure the trusted certificate bundle is available

      - "$ESGF_CONFIG/certificates/esg-trust-bundle.pem:/esg/certificates/esg-trust-bundle.pem:ro"

      # Allow the main ESGF configuration to be overridden

      - "$ESGF_CONFIG/config:/esg/config/.overrides:ro"

    depends_on:

      - esgf-configure

      - esgf-tomcat
@@ -297,6 +303,11 @@ services: 
      - "$ESGF_CONFIG/secrets/rootadmin-password:/esg/config/.esgf_pass:ro"

      # Make sure the trusted certificate bundle is available

      - "$ESGF_CONFIG/certificates/esg-trust-bundle.pem:/esg/certificates/esg-trust-bundle.pem:ro"

      # Allow the main ESGF configuration to be overridden

      - "$ESGF_CONFIG/config:/esg/config/.overrides:ro"

      # Allow the THREDDS config XML and root catalog to be overridden

      # Note that this is separate to the esgcet catalog being generated by the publisher

      - "$ESGF_CONFIG/thredds:/esg/content/thredds/.overrides:ro"

    depends_on:

      - esgf-configure

      - esgf-tomcat
@@ -341,6 +352,8 @@ services: 
      - "$ESGF_CONFIG/secrets/cog-secret-key:/esg/secrets/cog-secret-key:ro"

      # Make sure the trusted certificate bundle is available

      - "$ESGF_CONFIG/certificates/esg-trust-bundle.pem:/esg/certificates/esg-trust-bundle.pem:ro"

      # Allow the main ESGF configuration to be overridden

      - "$ESGF_CONFIG/config:/esg/config/.overrides:ro"

    depends_on:

      - esgf-configure

      - esgf-django
@@ -387,6 +400,11 @@ services: 
      - "$ESGF_CONFIG/secrets/auth-database-password:/esg/secrets/auth-database-password:ro"

      # Make sure the trusted certificate bundle is available

      - "$ESGF_CONFIG/certificates/esg-trust-bundle.pem:/esg/certificates/esg-trust-bundle.pem:ro"

      # Allow the main ESGF configuration to be overridden

      - "$ESGF_CONFIG/config:/esg/config/.overrides:ro"

      # Allow the ESGF auth configuration to be overridden

      # In particular, this allows an esgf_oauth2.json to be dropped in with OAuth2 credentials

      - "$ESGF_CONFIG/auth:/esg/auth/.overrides:ro"

    depends_on:

      - esgf-configure

      - esgf-django

setup/scripts/available-commands

+24 −20
Original line number Diff line number Diff line 
#!/bin/bash



set -eo pipefail



echo "****************************"

echo "** esgf-docker setup tool **"

echo "****************************"

echo ""

echo "The following commands are available:"

echo ""

echo "    generate-test-certificates"

echo "        Generates self-signed certificates for a test installation"

echo ""

echo "    generate-secrets"

echo "        Generates random secret keys and password required for deployment"

echo ""

echo "    create-trust-bundle"

echo "        Creates a PEM-encoded trust bundle from the available certificates"

echo ""

echo "    helm-values"

echo "        Outputs YAML configuration for use with esgf-helm Helm chart"

echo ""
 
set -e



cat <<EOF

****************************

** esgf-docker setup tool **

****************************



The following commands are available:



    generate-test-certificates

        Generates self-signed certificates for a test installation



    generate-secrets

        Generates random secret keys and password required for deployment



    fetch-static-configs

        Fetchs static configuration files from a distribution site



    create-trust-bundle

        Creates a PEM-encoded trust bundle from the available certificates



    helm-values

        Outputs YAML configuration for use with esgf-helm Helm chart

EOF

setup/scripts/create-trust-bundle

+5 −3
Original line number Diff line number Diff line
@@ -2,7 +2,7 @@ 


set -eo pipefail



source "$(dirname "$(realpath "$0")")/functions.sh"

. "$(dirname $BASH_SOURCE)/functions.sh"



#####

## This script creates a PEM-encoded trust bundle containing the following certificates:
@@ -26,8 +26,10 @@ touch "$BUNDLE" 
if [ -f "$CERTS/esg_trusted_certificates.tar" ]; then

    info "Unpacking existing trustroots"

    tar -xf "$CERTS/esg_trusted_certificates.tar" -C "$CERTS"

    for certfile in $(grep -lr -- "-----BEGIN CERTIFICATE-----" "$CERTS/esg_trusted_certificates"); do

        info "  Appending $certfile to bundle"

    certfiles=($(grep -lr -- "-----BEGIN CERTIFICATE-----" "$CERTS/esg_trusted_certificates"))

    for i in "${!certfiles[@]}"; do

        certfile="${certfiles[$i]}"

        info "[$((i + 1))/${#certfiles[@]}] Appending $certfile to bundle"

        cat "$certfile" >> "$BUNDLE"

    done

    rm -rf "$CERTS/esg_trusted_certificates"

setup/scripts/fetch-static-configs

0 → 100755
+35 −0
Original line number Diff line number Diff line 
#!/bin/bash



set -eo pipefail



. "$(dirname $BASH_SOURCE)/functions.sh"





#####

## This script downloads static configurations from the esgf-config Github repo

#####



profile="${1:-"esgf-prod"}"

config_files=(

    "esgf_ats_static.xml"

    "esgf_cogs.xml"

    "esgf_endpoints.xml"

    "esgf_idp_static.xml"

    "esgf_known_providers.xml"

    "esgf_search_aliases.xml"

)



info "Ensuring config directory exists"

mkdir -p /esg/config



info "Downloading config files"

for i in "${!config_files[@]}"; do

    config_file="${config_files[$i]}"

    info "[$((i + 1))/${#config_files[@]}] $config_file"

    curl -o "/esg/config/$config_file" -fsSL "https://raw.githubusercontent.com/ESGF/esgf-config/master/${profile}/xml/${config_file}"

done



info "Downloading trusted certificates tarball"

curl -o "/esg/certificates/esg_trusted_certificates.tar" -fsSL "${ESGF_DIST:-"http://dist.ceda.ac.uk/esgf/dist"}/certs/esg_trusted_certificates.tar"



info "Done"