First pass at Ansible installer - no data mounts yet

# -*- mode: ruby -*-

# vi: set ft=ruby :

# Vagrant configuration

Vagrant.configure(2) do |config|

  config.vm.box = "cedadev/centos7"

  # Use a fixed IP on the local network

  config.vm.network :private_network, ip: "192.168.100.100"

  # Set some virtualbox flags to improve time synchronisation between host and guest

  config.vm.provider :virtualbox do |virtualbox|

    # sync time every 10 seconds

    virtualbox.customize [ "guestproperty", "set", :id, "/VirtualBox/GuestAdd/VBoxService/--timesync-interval", 10000 ]

    # adjustments if drift > 100 ms

    virtualbox.customize [ "guestproperty", "set", :id, "/VirtualBox/GuestAdd/VBoxService/--timesync-min-adjust", 100 ]

    # sync time on restore

    virtualbox.customize [ "guestproperty", "set", :id, "/VirtualBox/GuestAdd/VBoxService/--timesync-set-on-restore", 1 ]

    # sync time on start

    virtualbox.customize [ "guestproperty", "set", :id, "/VirtualBox/GuestAdd/VBoxService/--timesync-set-start", 1 ]

    # at 1 second drift, the time will be set and not "smoothly" adjusted

    virtualbox.customize [ "guestproperty", "set", :id, "/VirtualBox/GuestAdd/VBoxService/--timesync-set-threshold", 1000 ]

  end

  # Provision the VM with our Ansible playbook

  config.vm.provision :ansible do |ansible|

    ansible.playbook = "deploy/ansible/playbook.yml"

    ansible.groups = { "data" => ["default"] }

  end

end

---

# The hostname to use - by default, use the FQDN of the host

hostname: "{{ ansible_fqdn }}"

# Configuration for custom registries that require authentication

registries: []

  # - registry: my.registry.org

  #   username: registryuser

  #   password: registrypassword

# Defaults for Docker images

image: &imageDefaults

  prefix: esgfhub

  tag: latest

data:

  # The mounts that are required to serve data, as defined by the given datasets

  #

  # Each specified mount should include the following keys:

  #

  #   hostPath: The path on the host that should be mounted into the container

  #   mountPath (optional): The mount path to use inside the container - if not given, hostPath is used

  mounts: []

    # - hostPath: /datacentre/archiveroots/archive/badc

    #   mountPath: /badc

  # The datasets to expose - for each dataset, a THREDDS datasetScan element and an Nginx location are generated

  #

  # Each specified dataset should have the following keys:

  #

  #   name: The name of the dataset

  #   path: The URL path for the dataset

  #   location: The location of the dataset *inside the container* (see mounts above)

  #   filters (optional): A list of filter specifications for the datasetScan element

  datasets:

    # - name: CMIP5

    #   path: esg_dataroot

    #   location: /badc/cmip5/data

  # Configuration for the THREDDS container

  thredds:

    # Indicates if THREDDS should be deployed or not

    enabled: true

    # Image overrides for the THREDDS image

    image:

      <<: *imageDefaults

      repository: thredds

    # Start the THREDDS container with the given user and groups

    securityContext: &securityContext

      # By default, use the Nginx user and group

      user: "1000"

      groups: ["1000"]

  # Configuration for the Nginx file server container

  fileserver:

    # Indicates if the Nginx file server should be deployed or not

    enabled: true

    # Image overrides for the Nginx image

    image:

      <<: *imageDefaults

      repository: nginx

    # Start the Nginx container with the given user and groups

    securityContext: *securityContext

proxy:

  image:

    <<: *imageDefaults

    repository: nginx

---

#####

## This playbook deploys the configured hosts as ESGF nodes using Docker containers

#####

- hosts: all

  become: true

  roles:

    - docker

    - { name: data, when: "'data' in group_names", tags: [data] }

    - proxy

---

#####

## Tasks to configure and deploy containers for a data node

#####

- name: Make config directories

  file:

    path: /esg/config/{{ item }}

    state: directory

  loop: [fileserver, thredds]

- name: Create Docker network

  docker_network:

    name: esgf

- name: Configure THREDDS

  block:

    - name: Write THREDDS configuration

      template:

        src: catalog.xml.j2

        dest: /esg/config/thredds/catalog.xml

    - name: Start THREDDS container

      docker_container:

        name: thredds

        image: "{{ data.thredds.image.prefix }}/{{ data.thredds.image.repository }}:{{ data.thredds.image.tag }}"

        detach: yes

        restart_policy: on-failure

        exposed_ports:

          - "8080"

        networks:

          - name: esgf

        networks_cli_compatible: yes

        user: "{{ data.thredds.securityContext.user }}"

        groups: "{{ data.thredds.securityContext.groups }}"

        volumes:

          # Start with the catalog volume

          - "/esg/config/thredds:/opt/tomcat/content/thredds/esgcet:ro"

        state: started

        restart: yes

  when: "data.thredds.enabled | bool"

- name: Configure Nginx fileserver

  block:

    - name: Write fileserver configuration

      template:

        src: fileserver.conf.j2

        dest: /esg/config/fileserver/fileserver.conf

      register: thredds_config

    - name: Start fileserver container

      docker_container:

        name: fileserver

        image: "{{ data.fileserver.image.prefix }}/{{ data.fileserver.image.repository }}:{{ data.fileserver.image.tag }}"

        detach: yes

        restart_policy: on-failure

        exposed_ports:

          - "8080"

        networks:

          - name: esgf

        networks_cli_compatible: yes

        user: "{{ data.fileserver.securityContext.user }}"

        groups: "{{ data.fileserver.securityContext.groups }}"

        volumes:

          # Start with the Nginx config

          - "/esg/config/fileserver:/etc/nginx/conf.d:ro"

        state: started

        restart: yes

  when: "data.fileserver.enabled | bool"

<?xml version="1.0" encoding="UTF-8"?>

<catalog name="Earth System Root Catalog"

         xmlns="http://www.unidata.ucar.edu/namespaces/thredds/InvCatalog/v1.0"

         xmlns:xlink="http://www.w3.org/1999/xlink"

         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"

         xsi:schemaLocation="http://www.unidata.ucar.edu/namespaces/thredds/InvCatalog/v1.0

                             http://www.unidata.ucar.edu/schemas/thredds/InvCatalog.1.0.6.xsd">

    <!-- Configure the available services -->

    <service name="all" serviceType="Compound" base="">

        <service name="fileserver" serviceType="HTTPServer" base="/thredds/fileServer/" />

        <service name="opendap" serviceType="OPeNDAP" base="/thredds/dodsC/" />

    </service>

    <!-- Produce a scan element for each of the given datasets -->

    {% for dataset in data.datasets %}

    <datasetScan name="{{ dataset.name }}" ID="{{ dataset.path }}" path="{{ dataset.path }}" location="{{ dataset.location }}">

        <metadata inherited="true">

            <serviceName>all</serviceName>

        </metadata>

        <filter>

            <!-- Exclude files or directories beginning with a dot -->

            <exclude wildcard=".*" collection="true" />

            {% for filter in dataset.filters | default([]) %}

            {% set filter_type = filter.pop('type') %}

            <{{ filter_type }} {% for k, v in filter.items() %}{{ k }}="{{ v }}" {% endfor %}/>

            {% endfor %}

        </filter>

    </datasetScan>

    {% endfor %}

</catalog>