Helm chart with logging sidecar working for stdout (7292eec5) · Commits · ESGF / mirrors / Esgf Docker

deploy/kubernetes/chart/files/logstash/access-log.conf

0 → 100644

+52 −0

Original line number	Diff line number	Diff line
		input {
		pipe {
		command => "cat ${ACCESS_LOG_FILE}"
		}
		}
		filter {
		# Extract components from the access log messages
		grok {
		match => {
		"message" => "%{IPORHOST:client_ip} - %{USER:user} \[%{HTTPDATE:request_time}\] \"%{WORD:method} %{URIPATHPARAM:request} HTTP/%{NUMBER:http_version}\" %{NUMBER:status_code} %{NUMBER:bytes_sent} \"%{DATA:referrer}\" \"%{DATA:user_agent}\" \"%{DATA:forwarded_for}\" \[%{NUMBER:request_duration}\]"
		}
		remove_field => ["message"]
		}
		# Drop any messages from the kube-probe
		if [user_agent] =~ "^kube-probe" {
		drop { }
		}
		# Only consider file downloads and OPeNDAP accesses
		if [request] !~ "^/thredds/(fileServer\|dodsC)" {
		drop { }
		}
		# Replace the logstash timestamp with the timestamp from the request
		date {
		match => ["request_time", "dd/MMM/yyyy:HH:mm:ss Z"]
		remove_field => ["request_time"]
		}
		# Replace the client IP with the forwarded for IP if present
		if [forwarded_for] != "-" {
		mutate {
		rename => { "forwarded_for" => "client_ip" }
		}
		}
		# Get the country for the IP, dropping the IP in the process
		geoip {
		source => "client_ip"
		fields => ["country_code2"]
		remove_field => ["client_ip"]
		}
		mutate {
		# Replace the host with the specified external hostname
		replace => { "host" => "${EXTERNAL_HOSTNAME}" }
		# Move the country code to the top-level
		add_field => { "country_code" => "%{[geoip][country_code2]}" }
		# Remove any unrequired fields
		remove_field => ["command", "geoip", "user"]
		}
		}
		output {
		stdout {
		codec => rubydebug
		}
		}

deploy/kubernetes/chart/templates/fileServer/deployment.yaml

+24 −6

Original line number	Diff line number	Diff line
		{{- $fileServer := .Values.data.fileServer -}}
		{{- $accessLogSidecar := .Values.data.accessLogSidecar -}}
		{{- if (and .Values.data.enabled $fileServer.enabled) -}}
		apiVersion: apps/v1
		kind: Deployment
		@@ -18,6 +19,7 @@ spec:
		# https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
		annotations:
		checksum/configmap: {{ include (print $.Template.BasePath "/fileServer/configmap.yaml") . \| sha256sum }}
		checksum/logstash-configmap: {{ include (print $.Template.BasePath "/logstash/configmap.yaml") . \| sha256sum }}
		spec:
		{{- with (default .Values.image.pullSecrets $fileServer.image.pullSecrets) }}
		imagePullSecrets: {{ toYaml . \| nindent 8 }}
		@@ -89,18 +91,25 @@ spec:
		{{- end }}
		# Tail the access log separately
		- name: file-server-access-log
		{{ include "esgf.deployment.image" (list . $fileServer.image) }}
		args:
		# Just cat the access log
		- cat
		- /var/log/nginx/access.log
		resources: {{ toYaml .Values.data.logTailResources \| nindent 12 }}
		{{ include "esgf.deployment.image" (list . $accessLogSidecar.image) }}
		env:
		- name: ACCESS_LOG_FILE
		value: /var/log/nginx/access.log
		- name: EXTERNAL_HOSTNAME
		value: "{{ .Values.hostname }}"
		resources: {{ toYaml $accessLogSidecar.resources \| nindent 12 }}
		{{- with .Values.data.securityContext }}
		securityContext: {{ toYaml . \| nindent 12 }}
		{{- end }}
		volumeMounts:
		- name: logstash-conf
		mountPath: /etc/logstash/conf.d
		- name: nginx-logs
		mountPath: /var/log/nginx
		- name: logstash-data
		mountPath: /usr/share/logstash/data
		- name: logstash-tmp
		mountPath: /tmp/logstash
		{{- with $fileServer.nodeSelector }}
		nodeSelector: {{ toYaml . \| nindent 8 }}
		{{- end }}
		@@ -123,6 +132,15 @@ spec:
		emptyDir: {}
		- name: nginx-run
		emptyDir: {}
		# Logstash config is in a configmap
		- name: logstash-conf
		configMap:
		name: {{ include "esgf.component.fullname" (list . "logstash") }}
		# Logstash also needs a tmp dir and a data dir
		- name: logstash-data
		emptyDir: {}
		- name: logstash-tmp
		emptyDir: {}
		{{- include "esgf.data.volumes" . \| nindent 8 }}
		{{- with $fileServer.extraVolumes }}
		{{- toYaml . \| nindent 8 }}

deploy/kubernetes/chart/templates/logstash/configmap.yaml

0 → 100644

+9 −0

Original line number	Diff line number	Diff line
		{{- if .Values.data.enabled -}}
		apiVersion: v1
		kind: ConfigMap
		metadata:
		name: {{ include "esgf.component.fullname" (list . "logstash") }}
		labels: {{ include "esgf.component.labels" (list . "logstash" dict) \| nindent 4 }}
		data:
		{{ (.Files.Glob "files/logstash/*").AsConfig \| indent 2 }}
		{{- end -}}

deploy/kubernetes/chart/templates/thredds/deployment.yaml

+25 −7

Original line number	Diff line number	Diff line
		{{- $thredds := .Values.data.thredds -}}
		{{- $accessLogSidecar := .Values.data.accessLogSidecar -}}
		{{- if (and .Values.data.enabled $thredds.enabled) -}}
		apiVersion: apps/v1
		kind: Deployment
		@@ -22,6 +23,7 @@ spec:
		# If using templated catalogs, we only need to roll if the configmap changes
		checksum/configmap: {{ include (print $.Template.BasePath "/thredds/configmap.yaml") . \| sha256sum }}
		{{- end }}
		checksum/logstash-configmap: {{ include (print $.Template.BasePath "/logstash/configmap.yaml") . \| sha256sum }}
		spec:
		{{- with (default .Values.image.pullSecrets $thredds.image.pullSecrets) }}
		imagePullSecrets: {{ toYaml . \| nindent 8 }}
		@@ -107,7 +109,7 @@ spec:
		env: {{ toYaml $thredds.extraEnv \| nindent 12 }}
		readinessProbe: &probe
		httpGet:
		path: /thredds/
		path: /thredds/catalog/catalog.html
		port: 8080
		httpHeaders:
		- name: Host
		@@ -150,18 +152,25 @@ spec:
		{{- end }}
		# Tail the access log separately
		- name: thredds-log-localhost-access-log
		{{ include "esgf.deployment.image" (list . $thredds.image) }}
		args:
		# Just cat the access log
		- cat
		- /thredds/logs/localhost_access_log.txt
		resources: {{ toYaml .Values.data.logTailResources \| nindent 12 }}
		{{ include "esgf.deployment.image" (list . $accessLogSidecar.image) }}
		env:
		- name: ACCESS_LOG_FILE
		value: /thredds/logs/localhost_access_log.txt
		- name: EXTERNAL_HOSTNAME
		value: "{{ .Values.hostname }}"
		resources: {{ toYaml $accessLogSidecar.resources \| nindent 12 }}
		{{- with .Values.data.securityContext }}
		securityContext: {{ toYaml . \| nindent 12 }}
		{{- end }}
		volumeMounts:
		- name: logstash-conf
		mountPath: /etc/logstash/conf.d
		- name: tomcat-logs
		mountPath: /thredds/logs
		- name: logstash-data
		mountPath: /usr/share/logstash/data
		- name: logstash-tmp
		mountPath: /tmp/logstash
		{{- with $thredds.nodeSelector }}
		nodeSelector: {{ toYaml . \| nindent 8 }}
		{{- end }}
		@@ -200,6 +209,15 @@ spec:
		emptyDir: {}
		- name: tmp-thredds
		emptyDir: {}
		# Logstash config is in a configmap
		- name: logstash-conf
		configMap:
		name: {{ include "esgf.component.fullname" (list . "logstash") }}
		# Logstash also needs a tmp dir and a data dir
		- name: logstash-data
		emptyDir: {}
		- name: logstash-tmp
		emptyDir: {}
		{{- include "esgf.data.volumes" . \| nindent 8 }}
		{{- with $thredds.extraVolumes }}
		{{- toYaml . \| nindent 8 }}

deploy/kubernetes/chart/values.yaml

+8 −5

Original line number	Diff line number	Diff line
		@@ -97,11 +97,14 @@ data:
		# Run with a read-only root filesystem by default
		readOnlyRootFilesystem: true

		# The resources for log-tailing containers
		logTailResources:
		requests:
		cpu: 1m
		memory: 1Mi
		# Configuration for the access log sidecar
		accessLogSidecar:
		# Image overrides for the access logging sidecar
		image:
		repository: logstash
		# The resource allocations for the access log container
		# See https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
		resources: {}

		# Configuration for the THREDDS pod
		thredds: