Commit d8f7fa93 authored by William Tucker's avatar William Tucker
Browse files

Added S3 path templating for the Ansible deployment

parent 540c9266

deploy/ansible/group_vars/data.yml

+11 −0
Original line number Diff line number Diff line
@@ -30,6 +30,17 @@ data_datasets: 
    path: esg_cordex

    location: /data/cordex



# An S3 location for a dataset can be specified using `s3Location` instead of `location` in the yaml config.

# We don't currently support adding security parameters for accessing secured S3 buckets.

data_datasets:

  - name: CMIP6

    path: esg_cmip6

    s3Location:

      host: example.com

      port: 443

      bucket: bucket_name

      dataPath: path/to/files



# https://github.com/ESGF/esgf-docker/blob/master/docs/deploy-ansible.md#configuring-the-available-datasets

deploy/ansible/roles/data/templates/catalog.xml.j2

+4 −0
Original line number Diff line number Diff line
@@ -13,7 +13,11 @@ 


    <!-- Produce a scan element for each of the given datasets -->

    {% for dataset in data_datasets %}

    {% if dataset.s3Location %}

    <datasetScan name="{{ dataset.s3Location.name }}" ID="{{ dataset.s3Location.path }}" path="{{ dataset.s3Location.path }}" location="cdms3://{{ dataset.s3Location.host }}:{{ dataset.s3Location.port }}/{{ dataset.s3Location.bucket }}?{{ dataset.s3Location.dataPath.strip("/") }}/#delimiter=/">

    {% else %}

    <datasetScan name="{{ dataset.name }}" ID="{{ dataset.path }}" path="{{ dataset.path }}" location="{{ dataset.location }}">

    {% endif %}

        <metadata inherited="true">

            <serviceName>all</serviceName>

        </metadata>

deploy/ansible/roles/data/templates/fileserver.conf.j2

+4 −0
Original line number Diff line number Diff line
@@ -17,8 +17,12 @@ server { 
    # Create a location block for each dataset

    {% for dataset in data_datasets %}

    location /thredds/fileServer/{{ dataset.path }}/ {

        {% if dataset.s3Location %}

        proxy_pass https://{{ dataset.s3Location.host }}:{{ dataset.s3Location.port }}/{{ dataset.s3Location.bucket }}/{{ dataset.s3Location.dataPath.strip("/") }}/;

        {% else %}

        alias {{ dataset.location }}/;

        try_files $uri =404;

        {% endif %}

    }

    {% endfor %}

}