add fluentd log service (becec53b) · Commits · NDIP / Charts / Pulsar

pulsar/Chart.yaml

+2 −2

Original line number	Diff line number	Diff line
		@@ -15,10 +15,10 @@ type: application
		# This is the chart version. This version number should be incremented each time you make changes
		# to the chart and its templates, including the app version.
		# Versions are expected to follow Semantic Versioning (https://semver.org/)
		version: 0.7.0
		version: 0.8.0

		# This is the version number of the application being deployed. This version number should be
		# incremented each time you make changes to the application. Versions are not expected to
		# follow Semantic Versioning. They should reflect the version the application is using.
		# It is recommended to use it with quotes.
		appVersion: "0.15.3.dev5.ornl.0"
		appVersion: "0.15.3.dev6.ornl.0"

pulsar/templates/config_fluentd.yaml

0 → 100644

+53 −0

Original line number	Diff line number	Diff line
		apiVersion: v1
		kind: ConfigMap
		metadata:
		name: {{ include "pulsar.fullname" . }}-fluentd
		labels:
		{{- include "pulsar.labels" . \| nindent 4 }}
		data:
		fluent.conf: \|-
		<source>
		@type tail
		path {{ .Values.pulsar.log_folder }}/pulsar.log*
		pos_file {{ .Values.pulsar.log_folder }}/pulsar.pos
		tag pulsar
		multiline_flush_interval 5s
		<parse>
		@type multiline
		format_firstline /\d{4}-\d{1,2}-\d{1,2}/
		format1 /^(?<timestamp>\S+ \S+)\s+(?<log_level>\S+)\s+\[(?<class>\S+?)\]\[(?<thread>\S+)\]\s+(?<message>.*)/
		time_key timestamp
		time_format %Y-%m-%d %H:%M:%S,%L
		</parse>
		</source>
		<filter pulsar>
		@type record_transformer
		enable_ruby
		<record>
		hostname onyx
		service pulsar
		log_level ${record["log_level"].downcase}
		</record>
		</filter>
		<match pulsar>
		@type elasticsearch
		logstash_format true
		host {{ .Values.elastic.host }}
		path {{ .Values.elastic.path }}
		port {{ .Values.elastic.port }}
		scheme https
		ssl_verify false
		user {{ .Values.elastic.user }}
		password {{ .Values.elastic.password }}
		index_name ndip
		<buffer>
		@type memory
		total_limit_size 10MB
		flush_mode interval
		flush_interval 3s
		flush_thread_count 3
		chunk_limit_size 500KB
		overflow_action drop_oldest_chunk
		</buffer>
		</match>

pulsar/templates/config_server.yaml

+9 −3

Original line number	Diff line number	Diff line
		@@ -32,18 +32,18 @@ data:
		keys = root,pulsar

		[handlers]
		keys = console
		keys = console,file

		[formatters]
		keys = generic

		[logger_root]
		level = INFO
		handlers = console
		handlers = console,file

		[logger_pulsar]
		level = DEBUG
		handlers = console
		handlers = console,file
		qualname = pulsar
		propagate = 0

		@@ -53,6 +53,12 @@ data:
		level = DEBUG
		formatter = generic

		[handler_file]
		class=handlers.RotatingFileHandler
		level=DEBUG
		formatter=generic
		args=('{{ .Values.pulsar.log_folder }}/pulsar.log', 'a', {{ .Values.pulsar.log_file_size }}, {{ .Values.pulsar.n_rotated_logs }})

		[formatter_generic]
		format = %(asctime)s %(levelname)-5.5s [%(name)s][%(threadName)s] %(message)s

pulsar/templates/deployment.yaml

+23 −1

Original line number	Diff line number	Diff line
		@@ -8,6 +8,8 @@ spec:
		selector:
		matchLabels:
		{{- include "pulsar.selectorLabels" . \| nindent 6 }}
		strategy:
		type: Recreate
		template:
		metadata:
		annotations:
		@@ -16,6 +18,8 @@ spec:
		{{- end }}
		checksum/configa: {{ include (print $.Template.BasePath "/config_app.yaml") . \| sha256sum }}
		checksum/configs: {{ include (print $.Template.BasePath "/config_server.yaml") . \| sha256sum }}
		checksum/configf: {{ include (print $.Template.BasePath "/config_fluentd.yaml") . \| sha256sum }}
		checksum/configr: {{ include (print $.Template.BasePath "/config_rucio.yaml") . \| sha256sum }}
		labels:
		{{- include "pulsar.selectorLabels" . \| nindent 8 }}
		spec:
		@@ -29,6 +33,13 @@ spec:
		- name: {{ include "pulsar.fullname" . }}-rucio
		configMap:
		name: {{ include "pulsar.fullname" . }}-rucio
		- name: {{ include "pulsar.fullname" . }}-fluentd
		configMap:
		name: {{ include "pulsar.fullname" . }}-fluentd
		- name: {{ include "pulsar.fullname" . }}-logs
		persistentVolumeClaim:
		claimName: {{ include "pulsar.fullname" . }}-logs-pvc

		containers:
		- name: {{ .Chart.Name }}
		image: "{{ .Values.image.repository }}:{{ .Values.image.tag \| default .Chart.AppVersion }}"
		@@ -43,3 +54,14 @@ spec:
		- mountPath: "/opt/rucio/etc/rucio.cfg"
		name: {{ include "pulsar.fullname" . }}-rucio
		subPath: rucio.cfg
		- mountPath: {{ .Values.pulsar.log_folder }}
		name: {{ include "pulsar.fullname" . }}-logs
		- name: {{ .Chart.Name }}-sidecar-fluentd
		image: "{{ .Values.fluentd.image.repository }}:{{ .Values.fluentd.image.tag \| default "latest" }}"
		imagePullPolicy: {{ .Values.image.pullPolicy }}
		volumeMounts:
		- mountPath: "/fluentd/etc/fluent.conf"
		name: {{ include "pulsar.fullname" . }}-fluentd
		subPath: fluent.conf
		- mountPath: {{ .Values.pulsar.log_folder }}
		name: {{ include "pulsar.fullname" . }}-logs

pulsar/templates/logs_volume.yaml

0 → 100644

+10 −0

Original line number	Diff line number	Diff line
		apiVersion: v1
		kind: PersistentVolumeClaim
		metadata:
		name: {{ include "pulsar.fullname" . }}-logs-pvc
		spec:
		accessModes:
		- ReadWriteOnce
		resources:
		requests:
		storage: 20Mi
		No newline at end of file