Commit 4c995963 authored by Matt Pryor's avatar Matt Pryor
Browse files

Solr shards working

parent 6e115aea

Vagrantfile

+9 −0
Original line number Diff line number Diff line
@@ -55,6 +55,12 @@ Vagrant.configure(2) do |config| 
        location: "/test_data/group_workspaces/jasmin2/cp4cds1/data/c3s-cordex"

      }

    ]

    solr_replicas = [

      {

        name: "llnl",

        master_url: "https://esgf-node.llnl.gov/solr"

      }

    ]

    ansible.groups = {

      "data" => ["default"],

      "index" => ["default"],
@@ -65,6 +71,9 @@ Vagrant.configure(2) do |config| 
      "data:vars" => {

        "data_mounts" => "#{data_mounts.to_json}",

        "data_datasets" => "#{data_datasets.to_json}"

      },

      "index:vars" => {

        "solr_replicas" => "#{solr_replicas.to_json}"

      }

    }

  end

deploy/ansible/roles/index/tasks/main.yml

+3 −0
Original line number Diff line number Diff line
@@ -12,6 +12,9 @@ 
  include: solr_uninstall.yml

  when: "'index' not in group_names or not solr_enabled"



- name: Reconcile Solr replicas

  include: solr_replica_reconcile.yml



- name: Install search application

  include: search_install.yml

  when: "'index' in group_names and search_enabled"

deploy/ansible/roles/index/tasks/solr_install.yml

+13 −1
Original line number Diff line number Diff line
@@ -28,6 +28,18 @@ 
    state: started

    restart: yes



- name: Set solr_shards_whitelist variable

  set_fact:

    solr_shards_whitelist: >-

      [

        # First the slave

        "solr-slave:8983/solr",

        # Then each replica shard

        {% for replica in solr_replicas %}

        "solr-replica-{{ replica.name }}:8983/solr",

        {% endfor %}

      ]



- name: Create Solr slave home volume

  docker_volume:

    name: solr-slave-home
@@ -46,7 +58,7 @@ 
      - "-Dslave.pollInterval={{ solr_slave_poll_interval }}"

      # This is the Solr instance used for queries, so we need to enable shards

      # However we want to use a whitelist so that only queries for configured shards are allowed

#      - "-Dsolr.shardsWhitelist={{ include "esgf.solr.shardsWhitelist" . }}"

      - "-Dsolr.shardsWhitelist={{ solr_shards_whitelist | join(',') }}"

    detach: yes

    restart_policy: unless-stopped

    exposed_ports:

deploy/ansible/roles/index/tasks/solr_replica_reconcile.yml

0 → 100644
+79 −0
Original line number Diff line number Diff line 
---



# Use the created volumes to detect the names of running replicas

- name: Get existing volumes

  docker_host_info:

    volumes: yes

  register: docker_host_info



- name: Set replica information facts

  set_fact:

    running_replicas: >-

      [

        {% for volume in docker_host_info.volumes %}

        {% if volume.Name is match('solr-replica-') %}

        "{{ volume.Name | regex_replace('^solr-replica-', '') | regex_replace('-home$', '') }}",

        {% endif %}

        {% endfor %}

      ]

    # If the index node or Solr is disabled, the number of desired replicas is zero

    # whatever solr_replicas is set to

    desired_replicas: >-

      [

        {% if 'index' in group_names and solr_enabled %}

        {% for replica in solr_replicas %}

        "{{ replica.name }}",

        {% endfor %}

        {% endif %}

      ]



# Ensure volumes exist for all the desired replicas

- name: Create Solr replica home volume

  docker_volume:

    name: "solr-replica-{{ item.name }}-home"

  loop: "{{ solr_replicas }}"

  when: "item.name in desired_replicas"

  loop_control:

    label: "{{ item.name }}"



# Ensure containers exist for all the desired replicas

- name: Start Solr replica container

  docker_container:

    name: "solr-replica-{{ item.name }}"

    image: "{{ solr_image_prefix }}/{{ solr_image_repository }}:{{ solr_image_tag }}"

    pull: "{{ solr_image_pull }}"

    command:

      - solr

      - -f

      - "-Dmaster.enable=true"

      - "-Dslave.enable=true"

      - "-Dslave.masterUrl={{ item.master_url }}"

      - "-Dslave.pollInterval={{ item.poll_interval | default(solr_replica_poll_interval) }}"

    detach: yes

    restart_policy: unless-stopped

    exposed_ports:

      - "8983"

    networks:

      - name: esgf

    networks_cli_compatible: yes

    volumes: ["solr-replica-{{ item.name }}-home:/var/solr/data:rw"]

    state: started

    restart: yes

  loop: "{{ solr_replicas }}"

  when: "item.name in desired_replicas"

  loop_control:

    label: "{{ item.name }}"



# Stop any running replicas that are no longer desired

- name: Stop Solr replica container

  docker_container:

    name: "solr-replica-{{ item }}"

    state: absent

  loop: "{{ running_replicas | difference(desired_replicas) }}"



# Remove the volumes for any running replicas that are no longer desired

- name: Remove Solr replica home volume

  docker_volume:

    name: "solr-replica-{{ item }}-home"

    state: absent

  loop: "{{ running_replicas | difference(desired_replicas) }}"

deploy/ansible/roles/index/templates/esgf_shards_static.xml.j2

+10 −1
Original line number Diff line number Diff line
@@ -4,6 +4,15 @@ 
    {% if solr_enabled %}

    <value>solr-slave:8983/solr</value>

    {% else %}

    <value>{{ solr_slave_external_url }}</value>

    <value>{{ solr_slave_external_url | regex_replace('^http(s)?://', '') }}</value>

    {% endif %}



    <!-- Replicas -->

    {% for replica in solr_replicas %}

    {% if solr_enabled %}

    <value>solr-replica-{{ replica.name }}:8983/solr</value>

    {% else %}

    <value>{{ replica.master_url | regex_replace('^http(s)?://', '') }}</value>

    {% endif %}

    {% endfor %}

</shards>