Commit e74a48cb authored by Matt Pryor's avatar Matt Pryor
Browse files

Changes for Solr shard configuration

parent b8ba92eb

docker-compose.build.yml

+8 −1
Original line number Diff line number Diff line
@@ -33,8 +33,15 @@ services: 


  esgf-setup:

    image: "${ESGF_HUB}/${ESGF_PREFIX}setup:${ESGF_VERSION}"

    build: ./setup

    build:

      context: ./setup

      args:

        ESGF_HUB: $ESGF_HUB

        ESGF_PREFIX: $ESGF_PREFIX

        ESGF_VERSION: $ESGF_VERSION

    entrypoint: ["true"]

    depends_on:

      - esgf-configure



  esgf-publisher:

    image: "${ESGF_HUB}/${ESGF_PREFIX}publisher:${ESGF_VERSION}"

docker-compose.yml

+1 −11
Original line number Diff line number Diff line
@@ -112,7 +112,7 @@ services: 
    image: "${ESGF_HUB}/${ESGF_PREFIX}solr:${ESGF_VERSION}"

    restart: always

    environment:

      ESGF_SOLR_MASTER_URL: http://esgf-solr-master:8983

      ESGF_SOLR_MASTER_URL: http://esgf-solr-master:8983/solr

      # Use a shorter replication interval for the local slave than the default

      ESGF_SOLR_REPLICATION_INTERVAL: "00:00:60"

    volumes:
@@ -120,14 +120,6 @@ services: 
    depends_on:

      - esgf-solr-master



  # Example remote-replica configuration

  # The replica would also need to be added to the ESGF_SOLR_SHARDS environment variable for esgf-index-node

#  esgf-solr-nasa-jpl:

#    image: "${ESGF_HUB}/${ESGF_PREFIX}solr:${ESGF_VERSION}"

#    restart: always

#    environment:

#      ESGF_SOLR_MASTER_URL: http://esgf-node.jpl.nasa.gov:80



  esgf-postgres-esgcet:

    image: "${ESGF_HUB}/${ESGF_PREFIX}postgres:${ESGF_VERSION}"

    restart: always
@@ -178,8 +170,6 @@ services: 
    environment:

      <<: *esgf_properties

      ESGF_SOLR_SHARDS: "esgf-solr-slave:8983/solr"

      # Example of configuring additional shards

#     ESGF_SOLR_SHARDS: "esgf-solr-slave:8983/solr,esgf-solr-nasa-jpl:8983/solr"

    volumes:

      # Make sure the trusted certificate bundle is available

      - "$ESGF_CONFIG/certificates/esg-trust-bundle.pem:/esg/certificates/esg-trust-bundle.pem:ro"

docs/_docs/usage/configuration.md

+40 −1
Original line number Diff line number Diff line
@@ -32,6 +32,10 @@ $ESGF_CONFIG 
├── environment

|

|   # OPTIONAL

|   # YAML file containing information about the remote Solr shards to deploy

├── solr_shards.yaml

|

|   # OPTIONAL

|   # Configuration overrides for esgf-auth

├── auth

|   |   # OPTIONAL
@@ -157,7 +161,7 @@ By default, an ESGF Docker deployment will be self-contained, i.e. it does not 
attempt to participate in a federation. In order to participate in a federation,

details of other federation members need to be specified via a set of XML files,

and the certificates for the federation need to be trusted (configuration of

Solr shards is more complex, and is addressed later).

Solr shards is more complex, and is [addressed later](#solr-replica-shard-configuration)).



ESGF Docker allows individual configuration files to be overridden in containers

by dropping files into the optional directories under `$ESGF_CONFIG`. Any files
@@ -205,3 +209,38 @@ The command to create the trust bundle is: 
```sh

./bin/esgf-setup create-trust-bundle

```





## Solr replica shard configuration



Rather than running all the Solr replica shards in one place using separate ports, as in a

"traditional" ESGF deployment, ESGF Docker uses a separate Docker container for each

replica shard.



The deployed Solr replica shards are configured by placing a `solr_shards.yaml` file at

`$ESGF_CONFIG/solr_shards.yaml`. This file has the following structure:



```yaml

# YAML list of the shards

shards:

  - url: http://esgf.remote.site/solr  # REQUIRED - The URL to replicate

    name: remote-site  # OPTIONAL - The service name to use, derived from URL if not given

    replicationInterval: "06:00:00"  # OPTIONAL - The replication interval, default 01:00:00



  - url: "..."

```



The `./bin/esgf-compose` command ensures that the Docker Compose configuration for the

replica shards is correctly passed to `docker-compose`.



<div class="note note-warning" markdown="1">

Remember to use `./bin/esgf-compose` whenever you would normally use `docker-compose`.

This ensures that the shard configurations are included in the Docker Compose configuration

that is used.

</div>



<div class="note note-info" markdown="1">

To see the generated Docker Compose configuration, you can use the command `./bin/esgf-setup compose-file`.

`./bin/esgf-compose` uses this command to generate the Docker Compose configuration before passing it

to `docker-compose` automatically, so it is not normally necessary to use it directly.

</div>

setup/Dockerfile

+12 −0
Original line number Diff line number Diff line
@@ -8,6 +8,15 @@ 
## on the host

#####



ARG ESGF_HUB=esgfhub

ARG ESGF_PREFIX=

ARG ESGF_VERSION=latest



# This build stage is required because COPY --from=$ARG is not supported

# https://github.com/moby/moby/issues/34482

FROM ${ESGF_HUB}/${ESGF_PREFIX}configure:${ESGF_VERSION} as configuration





# Base container is the JDK, which has openssl and keytool

FROM openjdk:8-jdk
@@ -16,6 +25,9 @@ ARG COMPOSE_VERSION=1.22.0 
RUN curl -fsSL https://github.com/docker/compose/releases/download/${COMPOSE_VERSION}/docker-compose-$(uname -s)-$(uname -m) -o /usr/local/bin/docker-compose && \

    chmod +x /usr/local/bin/docker-compose



# Copy the gomplate binary from esgf-configure

COPY --from=configuration /esg/bin/gomplate /usr/local/bin/gomplate



# Copy the scripts into the container

COPY scripts/* /usr/local/bin/

setup/scripts/compose-file

+43 −2
Original line number Diff line number Diff line
@@ -2,9 +2,50 @@ 


#####

## This script is a wrapper for docker-compose that makes sure the

## environment for the deployment has been imported

## environment for the deployment has been imported and injects

## Solr shard configuration

#####



set -eu



exec docker-compose config
 
COMPOSE_OPTS="-f docker-compose.yml"



# First, we need to generate the Solr shard configuration

# To do this, we use a gomplate template with the shard configuration as a datasource

if [ -f /esg/solr_shards.yaml ]; then

    COMPOSE_SHARDS_FILE="$(mktemp)"

    gomplate --out "$COMPOSE_SHARDS_FILE" --datasource shards=/esg/solr_shards.yaml <<"EOF"

{{- define "name" }}{{ if has . "name" }}{{ .name }}{{ else }}{{ (urlParse .url).Host | replaceAll "." "-" }}{{ end }}{{ end }}

{{- define "service-name" }}esgf-solr-{{ template "name" . }}{{ end }}

{{- define "volume-name" }}solr-{{ template "name" . }}-home{{ end }}



{{- $shards := (datasource "shards").shards }}



version: '3.4'



services:

{{- range $shards }}

  {{ template "service-name" . }}:

    image: "${ESGF_HUB}/${ESGF_PREFIX}solr:${ESGF_VERSION}"

    restart: always

    environment:

      ESGF_SOLR_MASTER_URL: "{{ .url }}"

      {{- if has . "replicationInterval" }}

      ESGF_SOLR_REPLICATION_INTERVAL: "{{ .replicationInterval }}"

      {{- end }}

    volumes:

      - "{{ template "volume-name" . }}:/esg/solr-home"

{{ end }}

  esgf-index-node:

    environment:

      ESGF_SOLR_SHARDS: "esgf-solr-slave:8983/solr{{ range $shards }},{{ template "service-name" . }}:8983/solr{{ end }}"



volumes:

{{- range $shards }}

  {{ template "volume-name" . }}:

{{- end }}

EOF

    COMPOSE_OPTS="$COMPOSE_OPTS -f $COMPOSE_SHARDS_FILE"

fi



exec docker-compose $COMPOSE_OPTS config