Commit 1dcb2e55 authored by Luca Cinquini's avatar Luca Cinquini
Browse files

Using docker compose to start solr copud cluster.

parent 8f6376fa

solr-cloud/Dockerfile

+5 −5
Original line number Diff line number Diff line
@@ -17,7 +17,7 @@ VOLUME /root/.m2 
ENV SOLR_CLOUD_HOME /usr/local/solr-cloud

RUN mkdir -p $SOLR_CLOUD_HOME



# mvn: Solr installation + ESGF configuration

# mvn: install 3 Solr instances configured for ESGF

ENV SRC_DIR /usr/local/src

RUN mkdir  -p $SRC_DIR

RUN cd $SRC_DIR && \
@@ -27,13 +27,13 @@ RUN cd $SRC_DIR && \ 
    cd etc/solr-cloud && \    

    mvn install



# start/stop scripts

RUN cp $SRC_DIR/esg-search/etc/solr-cloud/scripts/* /usr/local/bin/.



# Python software for harvesting/migrating Solr catalogs

# and dependencies

RUN pip install solrpy

RUN cd $SRC_DIR && \

    git clone https://github.com/EarthSystemCoG/esgfpy-publish.git



# scripts

COPY scripts/ /usr/local/bin/



EXPOSE 8983 8984 8985

solr-cloud/docker-compose.yml

0 → 100644
+34 −0
Original line number Diff line number Diff line 
# docker-compose file for setting up an ESGF Solr Cloud



version: '2'



services:



  # node with 3 Solr instances + embedded ZooKeeper

  esgf-solr-cloud1:

    image: esgfhub/esgf-solr-cloud

    container_name: esgf-solr-cloud1

    ports:

      - "8983:8983"

      - "9983:9983"

      - "8984:8984"

      - "8985:8985"

    networks:

      - default

    entrypoint: /usr/local/bin/docker-entrypoint.sh



  # other node with 3 Solr instances

  esgf-solr-cloud2:

    image: esgfhub/esgf-solr-cloud

    container_name: esgf-solr-cloud2

    expose:

      - "8983"

      - "8984"

      - "8985"

    networks:

      - default

    entrypoint: /usr/local/bin/docker-entrypoint.sh esgf-solr-cloud1

    #command: tail -f /dev/null

    # dependency garantees that this node does not start untill ZK is up

    depends_on:

      - esgf-solr-cloud1

solr-cloud/scripts/docker-entrypoint.sh

0 → 100755
+10 −0
Original line number Diff line number Diff line 
#!/bin/bash

# script to start all Solr instances om this node

# and keep the Docker container running



# optional FQDN for ZooKeeper host

zkhost=$1



/usr/local/bin/solr_cloud_start.sh $zkhost



tail -f $SOLR_CLOUD_HOME/solr/server/logs/solr.log

solr-cloud/scripts/solr_cloud_add_shard.sh

0 → 100755
+23 −0
Original line number Diff line number Diff line 
#!/bin/sh

# Script to add a new shard to replicate a remote index node, 

# for all ESGF collections (datsets, files, aggregations)

# This script MUST be executed onto the node that runs the Solr with the embedded ZooKeeper.

# The shard replicas will be spread across all currently running Solr instances, on all nodes.

#

# Usage: ./solr_cloud_add_shard.sh <index_node_fqdn>



collections=('datasets' 'files' 'aggregations')

shard=$1

if [ "$shard" = "" ]; then

  echo "Invalid <index_node_fqdn> supplied"

  exit

fi



for ((i=0; i < ${#collections[@]}; i++)); do



  collection=${collections[i]}

  echo "Creating shard: $shard for collection: $collection"

  curl "http://localhost:8983/solr/admin/collections?action=CREATESHARD&shard=${shard}&collection=${collection}"



done

solr-cloud/scripts/solr_cloud_crontab

0 → 100644
+15 −0
Original line number Diff line number Diff line 
#!/bin/sh

# Example crontab script to periodically harvest metadata from Solr repeaters

# Install as: 

# 0 0,4,8,12,16,20 * * * /usr/local/bin/solr_cloud_crontab.sh > /tmp/solr_cloud_crontab.log 2>&1



cd /usr/local/src/esgfpy-publish

export PYTHONPATH=.

python esgfpy/harvest/harvester.py 'http://esgf-node.jpl.nasa.gov:8983/solr' 'http://esgf-cloud.jpl.nasa.gov:8983/solr' --query 'index_node:esgf-node.jpl.nasa.gov'

python esgfpy/harvest/harvester.py 'http://esgf-node.jpl.nasa.gov:8985/solr' 'http://esgf-cloud.jpl.nasa.gov:8983/solr' --query 'index_node:esgdata.gfdl.noaa.gov'

python esgfpy/harvest/harvester.py 'http://esgf-node.jpl.nasa.gov:8986/solr' 'http://esgf-cloud.jpl.nasa.gov:8983/solr' --query 'index_node:esgf-data.dkrz.de'

python esgfpy/harvest/harvester.py 'http://esgf-node.jpl.nasa.gov:8987/solr' 'http://esgf-cloud.jpl.nasa.gov:8983/solr' --query 'index_node:esg-dn1.nsc.liu.se'

python esgfpy/harvest/harvester.py 'http://esgf-node.jpl.nasa.gov:8988/solr' 'http://esgf-cloud.jpl.nasa.gov:8983/solr' --query 'index_node:esgf-node.ipsl.upmc.fr'

python esgfpy/harvest/harvester.py 'http://esgf-node.jpl.nasa.gov:8989/solr' 'http://esgf-cloud.jpl.nasa.gov:8983/solr' --query 'index_node:esgf-index1.ceda.ac.uk'

python esgfpy/harvest/harvester.py 'http://esgf-node.jpl.nasa.gov:8990/solr' 'http://esgf-cloud.jpl.nasa.gov:8983/solr' --query 'index_node:pcmdi.llnl.gov'

python esgfpy/harvest/harvester.py 'http://esgf-node.jpl.nasa.gov:8991/solr' 'http://esgf-cloud.jpl.nasa.gov:8983/solr' --query 'index_node:esgf.nccs.nasa.gov'