Unverified Commit 66c9e29a authored by William Tucker's avatar William Tucker Committed by GitHub
Browse files

Merge pull request #210 from ESGF/s3-storage-compatibility 

S3 storage compatibility
parents 43b33592 f15a8bc1

deploy/kubernetes/chart/files/fileServer/datasets.conf

+5 −0
Original line number Diff line number Diff line
@@ -17,8 +17,13 @@ server { 
    # Create a location block for each dataset

    {{- range .Values.data.datasets }}

    location /thredds/fileServer/{{ .path }}/ {

        {{- if .s3Location }}

        {{- $s3 := .s3Location }}

        proxy_pass https://{{ $s3.host }}:{{ $s3.port }}/{{ $s3.bucket }}/{{ trimAll "/" $s3.dataPath }}/;

        {{- else }}

        alias {{ trimSuffix "/" .location }}/;

        try_files $uri =404;

        {{- end }}

    }

    {{- end }}

}

deploy/kubernetes/chart/files/thredds/catalog.xml

+5 −0
Original line number Diff line number Diff line
@@ -13,7 +13,12 @@ 


    <!-- Produce a scan element for each of the given datasets -->

    {{- range .Values.data.datasets }}

    {{- if .s3Location }}

    {{- $s3 := .s3Location }}

    <datasetScan name="{{ .name }}" ID="{{ .path }}" path="{{ .path }}" location="cdms3://{{ $s3.host }}:{{ $s3.port }}/{{ $s3.bucket }}?{{ trimAll "/" $s3.dataPath }}/#delimiter=/">

    {{- else }}

    <datasetScan name="{{ .name }}" ID="{{ .path }}" path="{{ .path }}" location="{{ .location }}">

    {{- end }}

        <metadata inherited="true">

            <serviceName>all</serviceName>

        </metadata>

docs/kubernetes/configure-datanode.md

+17 −0
Original line number Diff line number Diff line
@@ -69,6 +69,23 @@ data: 
      location: /data/cordex

```



## Using S3 buckets for data



An S3 location for a dataset can be specified using `s3Location` instead of `location` in the yaml config, e.g.:



```yaml

  datasets:

    - name: CMIP6

      path: esg_cmip6

      s3Location:

        host: example.com

        port: 443

        bucket: bucket_name

        dataPath: path/to/files

```



We don't currently support adding security parameters for accessing secured S3 buckets.



## Fowarding access logs



The THREDDS and Nginx file server components can be configured to forward access logs to