Unverified Commit b37849b4 authored by Matt Pryor's avatar Matt Pryor Committed by GitHub
Browse files

Merge pull request #126 from ESGF/issue/123/existing-catalogs 

Use pre-existing THREDDS catalogs
parents 19d09633 6b90be82

.gitlab-ci.yml

+14 −7
Original line number Diff line number Diff line
@@ -74,25 +74,32 @@ build:base: 
  variables:

    CONTEXT_DIR: $CI_PROJECT_DIR/images/base



build:nginx:

build:jdk:

  extends: .docker-build

  stage: build-2

  variables:

    CONTEXT_DIR: $CI_PROJECT_DIR/images/nginx

    CONTEXT_DIR: $CI_PROJECT_DIR/images/jdk

  needs: ["build:base"]



build:jdk:

build:jre:

  extends: .docker-build

  stage: build-2

  variables:

    CONTEXT_DIR: $CI_PROJECT_DIR/images/jdk

    CONTEXT_DIR: $CI_PROJECT_DIR/images/jre

  needs: ["build:base"]



build:jre:

build:rsync:

  extends: .docker-build

  stage: build-2

  variables:

    CONTEXT_DIR: $CI_PROJECT_DIR/images/jre

    CONTEXT_DIR: $CI_PROJECT_DIR/images/rsync

  needs: ["build:base"]



build:nginx:

  extends: .docker-build

  stage: build-2

  variables:

    CONTEXT_DIR: $CI_PROJECT_DIR/images/nginx

  needs: ["build:base"]



build:tomcat:
@@ -107,4 +114,4 @@ build:thredds: 
  stage: build-4

  variables:

    CONTEXT_DIR: $CI_PROJECT_DIR/images/thredds

  needs: ["build:tomcat"]
 
  needs: ["build:jdk", "build:tomcat"]

README.md

+9 −5
Original line number Diff line number Diff line
@@ -26,10 +26,14 @@ This project is under heavy active development, with the implementation dependin 
Future Architecture discussions.



Currently, only an unauthenticated data node is implemented. The data node uses THREDDS to serve

catalog and OPeNDAP endpoints and Nginx to serve files, using

[datasetScan elements](https://www.unidata.ucar.edu/software/tds/current/reference/DatasetScan.html)

for a catalog-free configuration. As such, it is designed to work with the next-generation publisher

being developed at LLNL that does not rely on THREDDS catalogs for publishing metadata.

catalog and OPeNDAP endpoints, but uses Nginx to do direct file serving which should by more

performant than THREDDS.



The data node is capable of using existing catalogs from the current publisher to specify the

available data, however it can also use a catalog-free configuration which utilises

[datasetScan elements](https://www.unidata.ucar.edu/software/tds/current/reference/DatasetScan.html),

to serve all files under a given dataset root. This is designed to work with the next-generation

publisher being developed at LLNL that does not rely on THREDDS catalogs for publishing metadata.



## Image tags
@@ -70,7 +74,7 @@ cd esgf-docker 
These changes have not yet been committed to `master`, so you will need to check out the development branch:



```sh

git checkout issue/112/nginx-data-node

git checkout future-architecture

```



Then follow the deployment guide for your chosen deployment method:

Vagrantfile

+1 −1
Original line number Diff line number Diff line
@@ -59,7 +59,7 @@ Vagrant.configure(2) do |config| 
      "data" => ["default"],

      "data:vars" => {

        "hostname" => "192.168.100.100.nip.io",

        "image_tag" => "issue-112-nginx-data-node",

        "image_tag" => "issue-123-existing-catalogs",

        "data_mounts" => "#{data_mounts.to_json}",

        "data_datasets" => "#{data_datasets.to_json}"

      }

deploy/ansible/roles/data/defaults/main.yml

+3 −0
Original line number Diff line number Diff line
@@ -43,6 +43,9 @@ data_datasets: [] 
  #   path: esg_dataroot

  #   location: /badc/cmip5/data



# If specified, catalogs from this directory are used instead of the generated catalogs

# thredds_catalog_host_path:



# Security parameters required to run Nginx and serve data

#

# If a special user or groups are required to access data, they can be set here, which

deploy/ansible/roles/data/tasks/main.yml

+20 −6
Original line number Diff line number Diff line
@@ -16,11 +16,22 @@ 


- name: Configure THREDDS

  block:

    - name: Write THREDDS configuration

    - name: Generate THREDDS catalogs

      block:

        - name: Write THREDDS catalog

          template:

            src: catalog.xml.j2

            dest: /esg/config/thredds/catalog.xml



        - name: Set thredds_catalog_host_path for generated catalogs

          set_fact:

            thredds_catalog_host_path: /esg/config/thredds

      when: thredds_catalog_host_path is not defined



    - name: Create THREDDS cache volume

      docker_volume:

        name: thredds-cache



    - name: Start THREDDS container

      docker_container:

        name: thredds
@@ -35,10 +46,13 @@ 
        networks_cli_compatible: yes

        user: "{{ data_security_context_user }}"

        groups: "{{ data_security_context_groups }}"

        # Append the catalog volume to the data mounts

        volumes: >-

          [

            "/esg/config/thredds:/opt/tomcat/content/thredds/esgcet:ro",

            # First the catalog volume

            "{{ thredds_catalog_host_path }}:/opt/tomcat/content/thredds/esgcet:ro",

            # Then the cache volume

            "thredds-cache:/opt/tomcat/content/thredds/cache:rw",

            # Then finally the data mounts

            {% for mount in data_mounts %}

            "{{ mount.host_path }}:{{ mount.mount_path }}:ro",

            {% endfor %}