Modify Solr configs to make replica definition easier (8ef719b0) · Commits · ESGF / mirrors / Esgf Docker

docker-compose.yml

+15 −5

Original line number	Diff line number	Diff line
		@@ -151,8 +151,7 @@ services:
		image: "${ESGF_HUB}/${ESGF_PREFIX}solr:${ESGF_VERSION}"
		restart: always
		environment:
		SOLR_HOME: /esg/solr-home
		SOLR_HEAP: 1g
		ESGF_SOLR_ROLE: master
		volumes:
		- "solr-master-home:/esg/solr-home"
		depends_on:
		@@ -162,16 +161,25 @@ services:
		image: "${ESGF_HUB}/${ESGF_PREFIX}solr:${ESGF_VERSION}"
		restart: always
		environment:
		SOLR_HOME: /esg/solr-home
		SOLR_HEAP: 1g
		ESGF_SOLR_ENABLE_SLAVE: "1"
		ESGF_SOLR_MASTER_URL: http://esgf-solr-master:8983
		# Use a shorter replication interval for the local slave than the default
		ESGF_SOLR_REPLICATION_INTERVAL: "00:00:60"
		volumes:
		- "solr-slave-home:/esg/solr-home"
		depends_on:
		- esgf-solr
		- esgf-solr-master

		# Example remote-replica configuration
		# The replica would also need to be added to the ESGF_SOLR_SHARDS environment variable for esgf-index-node
		# esgf-solr-nasa-jpl:
		# image: "${ESGF_HUB}/${ESGF_PREFIX}solr:${ESGF_VERSION}"
		# restart: always
		# environment:
		# ESGF_SOLR_MASTER_URL: http://esgf-node.jpl.nasa.gov:80
		# depends_on:
		# - esgf-solr

		esgf-postgres-esgcet:
		image: "${ESGF_HUB}/${ESGF_PREFIX}postgres:${ESGF_VERSION}"
		restart: always
		@@ -247,6 +255,8 @@ services:
		environment:
		<<: *esgf_properties
		ESGF_SOLR_SHARDS: "esgf-solr-slave:8983/solr"
		# Example of configuring additional shards
		# ESGF_SOLR_SHARDS: "esgf-solr-slave:8983/solr,esgf-solr-nasa-jpl:8983/solr"
		volumes:
		# Make sure the trusted certificate bundle is available
		- "$ESGF_CONFIG/certificates/esg-trust-bundle.pem:/esg/certificates/esg-trust-bundle.pem:ro"

setup/scripts/fetch-static-configs

+0 −6

Original line number	Diff line number	Diff line
		@@ -30,12 +30,6 @@ for i in "${!config_files[@]}"; do
		curl -o "/esg/config/$config_file" -fsSL "https://raw.githubusercontent.com/ESGF/esgf-config/master/${profile}/xml/${config_file}"
		done

		# Download esgf_shards_static.xml, removing the entry for localhost
		info "Downloading esgf_shards_static.xml"
		curl -fsSL "https://raw.githubusercontent.com/ESGF/esgf-config/master/${profile}/xml/esgf_shards_static.xml" \| \
		grep -v "localhost/solr" \
		> "/esg/config/esgf_shards_static.xml"

		# Download the trusted certs
		info "Downloading trusted certificates tarball"
		curl -o "/esg/certificates/esg_trusted_certificates.tar" -fsSL "${ESGF_DIST:-"http://dist.ceda.ac.uk/esgf/dist"}/certs/esg_trusted_certificates.tar"

solr/Dockerfile

+15 −11

Original line number	Diff line number	Diff line
		#####
		## Standard Solr container with setup of ESGF cores built-in
		## Standard Solr container with setup of ESGF cores built-in
		#####

		FROM solr:6.6

		MAINTAINER Earth System Grid Federation <esgf-devel@lists.llnl.gov>

		# We need to be root to make our modifications
		# We need to be root to make our modifications
		USER root

		# Make sure solr home initialises correctly
		ENV INIT_SOLR_HOME="yes"
		# Make sure solr home initialises correctly
		ENV INIT_SOLR_HOME "yes"
		ENV SOLR_HOME /esg/solr-home

		# Install the Solr home template from the distribution site
		# Install the Solr home template from the distribution site
		# Make customisations to the solrconfig.xml to enable extra Java properties
		ARG ESGF_REPO=http://dist.ceda.ac.uk/esgf
		RUN mkdir -p /esg/solr-core-config && \
		curl -fsSL $ESGF_REPO/dist/esg-search/solr-home.tar \| \
		tar -x -C /esg/solr-core-config --strip-components=2 solr-home/mycore && \
		rm -rf /esg/solr-core-config/core.properties && \
		sed -i \
		"s/http:\/\/localhost:8984/\${esgf.solr.master.url}/g" \
		"s/http:\/\/localhost:8984/\${esgf.solr.master.url:}/g" \
		/esg/solr-core-config/conf/solrconfig.xml && \
		sed -i \
		"s/00:00:60/\${esgf.solr.replication.interval:}/g" \
		/esg/solr-core-config/conf/solrconfig.xml

		# Correct the permissions on /opt/solr so that any user in the root group can use it
		# Correct the permissions on /opt/solr so that any user in the root group can use it
		RUN chown -R 8983:0 /opt/solr && chmod -R g+w /opt/solr

		# Create /esg/solr-home with the correct permissions
		RUN mkdir -p /esg/solr-home && chown 8983:0 /esg/solr-home && chmod 775 /esg/solr-home
		VOLUME /esg/solr-home
		# Create /esg/solr-home with the correct permissions
		RUN mkdir -p $SOLR_HOME && chown 8983:0 $SOLR_HOME && chmod 775 $SOLR_HOME

		# Install the initialisation scripts
		# Install the initialisation scripts
		COPY initdb.d/* /docker-entrypoint-initdb.d/

		# Revert to the solr user

solr/initdb.d/update-solr-in.sh

100644 → 100755

+30 −11

Original line number	Diff line number	Diff line
		#!/bin/bash

		set -eu

		#####
		# This script updates solr.in.sh with the required variables
		#####

		SOLR_IN_SH=/opt/solr/bin/solr.in.sh

		# If SOLR_HEAP is set, use it to update the solr.in.sh
		[ -n "$SOLR_HEAP" ] && {
		sed -i -e "s/SOLR_HEAP=\".*\"/SOLR_HEAP=\"$SOLR_HEAP\"/" "$SOLR_IN_SH"
		}
		echo "[INFO] Writing Solr configuration to ${SOLR_IN_SH}"

		# Set the master and slave flags
		echo "SOLR_OPTS=\"\$SOLR_OPTS -Denable.master=true\"" >> "$SOLR_IN_SH"
		[ -n "${ESGF_SOLR_ENABLE_SLAVE:-}" ] && {
		echo "SOLR_OPTS=\"\$SOLR_OPTS -Denable.slave=true\"" >> "$SOLR_IN_SH"
		}
		# If SOLR_HEAP is not set, use a 1GB heap
		echo "SOLR_HEAP=\"${SOLR_HEAP:-"1g"}\"" \| tee -a "$SOLR_IN_SH"

		# Set the master URL property
		echo "SOLR_OPTS=\"\$SOLR_OPTS -Desgf.solr.master.url=$ESGF_SOLR_MASTER_URL\"" >> "$SOLR_IN_SH"
		# Set Java properties
		# Role is either master or replica, default replica
		: ${ESGF_SOLR_ROLE:="replica"}
		case "$ESGF_SOLR_ROLE" in
		"master")
		# Enable master only
		echo 'SOLR_OPTS="$SOLR_OPTS -Denable.master=true"' \| tee -a "$SOLR_IN_SH"
		;;
		"replica")
		# Replica nodes have master and slave enabled
		# They also need a master URL (required) and a replication interval (default 1 hour)
		cat <<EOF \| tee -a "$SOLR_IN_SH"
		SOLR_OPTS="\$SOLR_OPTS -Denable.master=true -Denable.slave=true"
		SOLR_OPTS="\$SOLR_OPTS -Desgf.solr.master.url=$ESGF_SOLR_MASTER_URL"
		SOLR_OPTS="\$SOLR_OPTS -Desgf.solr.replication.interval=${ESGF_SOLR_REPLICATION_INTERVAL:-"01:00:00"}"
		EOF
		;;
		*)
		echo "[ERROR] Unknown ESGF_SOLR_ROLE: $ESGF_SOLR_ROLE" 1>&2
		exit 1
		esac