Commit 16d4b523 authored by William Tucker's avatar William Tucker
Browse files

Merge branch 'master' of github.com:ESGF/esgf-docker into refactor

parents 6069cdde a84c0ba2

deploy/ansible/roles/data/templates/stats.conf.j2

+8 −0
Original line number Diff line number Diff line
@@ -18,10 +18,18 @@ filter { 
        }

        remove_field => ["message"]

    }

    # Drop HEAD requests

    if [method] == "HEAD" {

        drop { }

    }

    # Only consider file downloads and OPeNDAP accesses

    if [request] !~ "^/thredds/(fileServer|dodsC)" {

        drop { }

    }

    # Drop extraneous OPeNDAP requests

    if [request] =~ "^/thredds/dodsC/([\?].*)\.(html|nc|dds|das)" {

        drop { }

    }

    # Replace the logstash timestamp with the timestamp from the request

    date {

        match => ["request_time", "dd/MMM/yyyy:HH:mm:ss Z"]

deploy/kubernetes/chart/files/logstash/10-stats.conf

+8 −0
Original line number Diff line number Diff line
@@ -12,10 +12,18 @@ filter { 
    if [user_agent] =~ "^kube-probe" {

        drop { }

    }

    # Drop HEAD requests

    if [method] == "HEAD" {

        drop { }

    }

    # Only consider file downloads and OPeNDAP accesses

    if [request] !~ "^/thredds/(fileServer|dodsC)" {

        drop { }

    }

    # Drop extraneous OPeNDAP requests

    if [request] =~ "^/thredds/dodsC/([\?].*)\.(html|nc|dds|das)" {

        drop { }

    }

    # Replace the logstash timestamp with the timestamp from the request

    date {

        match => ["request_time", "dd/MMM/yyyy:HH:mm:ss Z"]

deploy/kubernetes/chart/templates/ingress/ingress-publish.yaml

+1 −0
Original line number Diff line number Diff line
@@ -6,6 +6,7 @@ metadata: 
  labels: {{ include "esgf.component.labels" (list . "ingressPublish" .Values.ingress.labels) | nindent 4 }}

  annotations:

    nginx.ingress.kubernetes.io/whitelist-source-range: {{ .Values.index.search.publishIpRange }}

    haproxy-ingress.github.io/allowlist-source-range: {{ .Values.index.search.publishIpRange }}

    {{ include "esgf.ingress.auth" . | indent 4 }}

    {{- with .Values.ingress.annotations }}

    {{ toYaml . | nindent 4 }}

docs/resource-requirements.md

0 → 100644
+19 −0
Original line number Diff line number Diff line 
# Resource Requirements



Below are some recommendations for server resources when running a data node using this deployment.



## For a VM-based data node



**THREDDS:**

* Memory: 4GB minimum, 16GB recommended, 32GB if expecting a lot of usage

* CPU: 1 should be enough



**Nginx File Server:**

* Memory: Depends on expected usage. 1GB should be enough, but with high usage you may need to increase this as far as 32GB

* CPU: 1 should be enoughFor a combined THREDDS / Nginx data node, you can add up the memory needs.

_Keep in mind that THREDDS may eat up the available memory on a node under heavy use. Especially when subsetting larger files._



For Kubernetes nodes, follow the same guidance as above when configuring

[data.thredds.resources](https://github.com/ESGF/esgf-docker/blob/ace917dc33098a06bc853e1d65be6a9fe0844486/deploy/kubernetes/chart/values.yaml#L316)

and [data.fileServer.resources](https://github.com/ESGF/esgf-docker/blob/ace917dc33098a06bc853e1d65be6a9fe0844486/deploy/kubernetes/chart/values.yaml#L366),

but keep in mind that smaller memory limit may be needed if autoscaling is enabled, or using multiple replicas

images/logstash/Dockerfile

+1 −1
Original line number Diff line number Diff line
@@ -25,7 +25,7 @@ COPY logstash.repo /etc/yum.repos.d/logstash.repo 
# So SHA1 needs to be enabled to install it.

RUN update-crypto-policies --set DEFAULT:SHA1

RUN dnf makecache && \

    dnf install -y logstash-7.10.0 && \

    dnf install -y logstash && \

    dnf clean all && \

    chown -R $ESGF_UID:$ESGF_GID /usr/share/logstash