Commit a031a2ca authored by Matt Pryor's avatar Matt Pryor
Browse files

Enable data volumes for Ansible + other small tweaks + remove old docs

parent d65ca162

Vagrantfile

+26 −1
Original line number Diff line number Diff line
@@ -22,9 +22,34 @@ Vagrant.configure(2) do |config| 
    virtualbox.customize [ "guestproperty", "set", :id, "/VirtualBox/GuestAdd/VBoxService/--timesync-set-threshold", 1000 ]

  end



  config.vm.provision :shell, inline: <<-SHELL

    set -euo pipefail

    echo "Installing roocs/mini-esgf-data into /test_data..."

    curl -fsSL https://github.com/roocs/mini-esgf-data/tarball/master | tar -xz --strip-components=1 -C / */test_data

  SHELL



  # Provision the VM with our Ansible playbook

  config.vm.provision :ansible do |ansible|

    ansible.playbook = "deploy/ansible/playbook.yml"

    ansible.groups = { "data" => ["default"] }

    ansible.config_file = "deploy/ansible/ansible.cfg"

    ansible.groups = { data: ["default"] }

    # Configure the datasets from mini-esgf-data

    ansible.extra_vars = {

      data: {

        mounts: [{ hostPath: "/test_data", mountPath: "/test_data" }],

        datasets: [

          {

            name: "CMIP5",

            path: "esg_cmip5",

            location: "/test_data/badc/cmip5/data"

          },

          {

            name: "CORDEX",

            path: "esg_cordex",

            location: "/test_data/group_workspaces/jasmin2/cp4cds1/data/c3s-cordex"

          }

        ]

      }

    }

  end

end

ansible.cfg

0 → 120000
+1 −0
Original line number Diff line number Diff line 
deploy/ansible/ansible.cfg
 
 No newline at end of file

deploy/ansible/ansible.cfg

0 → 100644
+5 −0
Original line number Diff line number Diff line 
[defaults]

# Use the merge behaviour when combining values files

# This allows overriding parts of dictionaries, like in Helm, allowing

# for less clutter in the global namespace

hash_behaviour = merge

deploy/ansible/group_vars/all.yml

+11 −14
Original line number Diff line number Diff line
@@ -9,10 +9,10 @@ registries: [] 
  #   username: registryuser

  #   password: registrypassword



# Defaults for Docker images

image: &imageDefaults

  prefix: esgfhub

image_defaults:

  prefix: esgfdeploy

  tag: latest

  pull: true



data:

  # The mounts that are required to serve data, as defined by the given datasets
@@ -33,24 +33,25 @@ data: 
  #   path: The URL path for the dataset

  #   location: The location of the dataset *inside the container* (see mounts above)

  #   filters (optional): A list of filter specifications for the datasetScan element

  datasets:

  datasets: []

    # - name: CMIP5

    #   path: esg_dataroot

    #   location: /badc/cmip5/data



  # Security parameters required to run Nginx and serve data

  # By default, use the ESGF user and group

  security_context:

    user: "1000"

    # In order to run THREDDS/Nginx inside the container, the user must have group 1000

    groups: ["1000"]



  # Configuration for the THREDDS container

  thredds:

    # Indicates if THREDDS should be deployed or not

    enabled: true

    # Image overrides for the THREDDS image

    image:

      <<: *imageDefaults

      repository: thredds

    # Start the THREDDS container with the given user and groups

    securityContext: &securityContext

      # By default, use the Nginx user and group

      user: "1000"

      groups: ["1000"]



  # Configuration for the Nginx file server container

  fileserver:
@@ -58,12 +59,8 @@ data: 
    enabled: true

    # Image overrides for the Nginx image

    image:

      <<: *imageDefaults

      repository: nginx

    # Start the Nginx container with the given user and groups

    securityContext: *securityContext



proxy:

  image:

    <<: *imageDefaults

    repository: nginx

deploy/ansible/roles/data/tasks/main.yml

+29 −13
Original line number Diff line number Diff line
@@ -14,6 +14,15 @@ 
  docker_network:

    name: esgf



- name: Configure data volumes

  set_fact:

    data_docker_mounts: >-

      [

        {% for mount in data.mounts %}

        "{{ mount.hostPath }}:{{ mount.mountPath }}:ro",

        {% endfor %}

      ]



- name: Configure THREDDS

  block:

    - name: Write THREDDS configuration
@@ -21,10 +30,15 @@ 
        src: catalog.xml.j2

        dest: /esg/config/thredds/catalog.xml



    - name: Set THREDDS image fact

      set_fact:

        thredds_image: "{{ image_defaults | combine(data.thredds.image) }}"



    - name: Start THREDDS container

      docker_container:

        name: thredds

        image: "{{ data.thredds.image.prefix }}/{{ data.thredds.image.repository }}:{{ data.thredds.image.tag }}"

        image: "{{ thredds_image.prefix }}/{{ thredds_image.repository }}:{{ thredds_image.tag }}"

        pull: "{{ thredds_image.pull }}"

        detach: yes

        restart_policy: on-failure

        exposed_ports:
@@ -32,11 +46,10 @@ 
        networks:

          - name: esgf

        networks_cli_compatible: yes

        user: "{{ data.thredds.securityContext.user }}"

        groups: "{{ data.thredds.securityContext.groups }}"

        volumes:

          # Start with the catalog volume

          - "/esg/config/thredds:/opt/tomcat/content/thredds/esgcet:ro"

        user: "{{ data.security_context.user }}"

        groups: "{{ data.security_context.groups }}"

        # Append the catalog volume to the data mounts

        volumes: "{{ ['/esg/config/thredds:/opt/tomcat/content/thredds/esgcet:ro'] + data_docker_mounts }}"

        state: started

        restart: yes

  when: "data.thredds.enabled | bool"
@@ -47,12 +60,16 @@ 
      template:

        src: fileserver.conf.j2

        dest: /esg/config/fileserver/fileserver.conf

      register: thredds_config



    - name: Set fileserver image fact

      set_fact:

        fileserver_image: "{{ image_defaults | combine(data.fileserver.image) }}"



    - name: Start fileserver container

      docker_container:

        name: fileserver

        image: "{{ data.fileserver.image.prefix }}/{{ data.fileserver.image.repository }}:{{ data.fileserver.image.tag }}"

        image: "{{ fileserver_image.prefix }}/{{ fileserver_image.repository }}:{{ fileserver_image.tag }}"

        pull: "{{ fileserver_image.pull }}"

        detach: yes

        restart_policy: on-failure

        exposed_ports:
@@ -60,11 +77,10 @@ 
        networks:

          - name: esgf

        networks_cli_compatible: yes

        user: "{{ data.fileserver.securityContext.user }}"

        groups: "{{ data.fileserver.securityContext.groups }}"

        volumes:

          # Start with the Nginx config

          - "/esg/config/fileserver:/etc/nginx/conf.d:ro"

        user: "{{ data.security_context.user }}"

        groups: "{{ data.security_context.groups }}"

        # Append the Nginx config volume to the data mounts

        volumes: "{{ ['/esg/config/fileserver:/etc/nginx/conf.d:ro'] + data_docker_mounts }}"

        state: started

        restart: yes

  when: "data.fileserver.enabled | bool"