Unverified Commit 16d21adb authored by William Tucker's avatar William Tucker Committed by GitHub
Browse files

Merge pull request #270 from ESGF/logstash-update 

Logstash update
parents 154c0532 ac3218b6

deploy/ansible/roles/data/defaults/main.yml

+2 −36
Original line number Diff line number Diff line
@@ -83,39 +83,5 @@ logstash_image_tag: "{{ image_tag }}" 
logstash_image_pull: "{{ image_pull }}"

logstash_image_repository: logstash

# Settings for the stats server

logstash_stats_server: ophidialab.cmcc.it

logstash_stats_port: 5045

logstash_stats_certificate: |

  -----BEGIN CERTIFICATE-----

  MIIFpDCCA4wCCQDwfiQImQrAHjANBgkqhkiG9w0BAQsFADCBkzELMAkGA1UEBhMC

  SVQxDjAMBgNVBAgMBUl0YWx5MQ4wDAYDVQQHDAVMZWNjZTENMAsGA1UECgwERVNH

  RjENMAsGA1UECwwERVNHRjEbMBkGA1UEAwwSb3BoaWRpYWxhYi5jbWNjLml0MSkw

  JwYJKoZIhvcNAQkBFhpzdGF0aXN0aWNzLXN1cHBvcnRAY21jYy5pdDAeFw0yMzA5

  MjUxNDAwMzRaFw0yNDA5MjQxNDAwMzRaMIGTMQswCQYDVQQGEwJJVDEOMAwGA1UE

  CAwFSXRhbHkxDjAMBgNVBAcMBUxlY2NlMQ0wCwYDVQQKDARFU0dGMQ0wCwYDVQQL

  DARFU0dGMRswGQYDVQQDDBJvcGhpZGlhbGFiLmNtY2MuaXQxKTAnBgkqhkiG9w0B

  CQEWGnN0YXRpc3RpY3Mtc3VwcG9ydEBjbWNjLml0MIICIjANBgkqhkiG9w0BAQEF

  AAOCAg8AMIICCgKCAgEA0ink9ncyjBGU9207PiK/26Ue1lx+fCVjKUVDl569t7Kg

  n5XELdAdgyWQRuTGG1w7H03OFVKPsUGFb80agZBb5T6eP8yRXLz5Ejp5DQHlo0nh

  9ia4bMNJcx0jE1WdYfCP7GCKohTKlFpuThvuu3IhXGHLzgR06ygvDI7Hf554P59D

  OxQ6JO3fTUutO9z78xvyxtY4MBZyMSDRmiJJLc4hzJ1AaPvFjNgU7AZBZ/X8Rl3Y

  69k/sWQtGZlJGZ4ZR66+OMJZEk5NiqrnShBRsczJuUwehpLv8kWVfQ/0nfRmI1aZ

  BGjq1pUH6611G2A42PqseaCMXGEbidPVUZIRwMbMPXud0bdf4+wVAJftehnCHg5G

  LizNgK2/jQWTo69BPbJyAMgI4qjK+NAfNwknRgpamvikQ66GuV4YCOKUV8/GUtLO

  uYMXfYlg2wNhnXJ2YDiEKJAY1/NnAkPxvYpS+r9pi6kd+6uTH8H11G+kjDzK+29y

  /6O1V2LSZL902KF7gdTA104r3HCYOf9BfRWA3C/C5P2vf376fLwskcUy5iQSX0Tc

  IORRkjUKsoKN/FZ2IsPS9I05o5uUW1075Lf6aS/kyg0cEVfTSPyiWug0ozS1oIP4

  dvfbvJv/R7Sz6nz04rcEq6CYaqlw+65WMDlBoUIVu65Gsg05gbnLO3fcNDPlbBsC

  AwEAATANBgkqhkiG9w0BAQsFAAOCAgEAm6iITRmACwLZqSRMC0X9mb0/dwo75n3q

  H6bXHHSU9tozJkrq+OywTNCnWQV9oqUZvh9Iect2tQHmp00LxKrRKgPLEJ3PCq1n

  adQifN10CeKt9b/657X8DgfitaaPYdDu4bVbITRLyeiMJRXfGAzBbEO1hIGapyuI

  5FmuwicCIxxwHU3Ej9fc7ZJr7zQFbmLuLLr6ntsIddNwoMlOrzgj9U3VpZeKoyXo

  rdn2Nnl0WkVowic+Q7mSkQ2dYU4pflVkDHKuFHRZxc8OBEr2QOSzEgx/57BDDsd+

  KTJ6n951f4w0IdviRV7U23H+YojKPIrXsAyaT+axd3C71eE5ZidggAgk0u4dE6Hq

  W/sEIlzlY+JMPQj8ynLpJUeoSO3osHoKn7S4SxCwQQLdaWeUWcLq8t1tCQOYf6CX

  B+6Hvjq5tAH7DVmDcqdXgFQYb/1rdOWiZsuChD2jNEENw7YoJAq0yQPH4BcpkmPU

  qxT26GCw9q8dHas7LkL4oXovUCozKyS02FEud/fan+5jSaiHvbQ1Ge7pHQeclhqr

  3k9JO5ZfOvZad3FRBEVnsEYb0VX+JuEqZarnZhqrzxc/764StngT7zd+YT3O1P+f

  ZOSy8kG7ZaRItC+KjVwOma4rEF6/kKCBUu/LeOgbHnXmwVPBoFiNUQA1FHMRwGPH

  HWB6z/apQUM=

  -----END CERTIFICATE-----
 
logstash_stats_url: https://esgf-logstash.cmcc.it/api/exchanges/%2f/log_exchange/publish

logstash_stats_authorization_key: ""

deploy/ansible/roles/data/tasks/logstash_install.yml

+2 −12
Original line number Diff line number Diff line
@@ -12,18 +12,10 @@ 
  docker_network:

    name: esgf



- name: Ensure logstash config directories exist

- name: Ensure logstash config directory exist

  file:

    path: /esg/config/logstash/{{ item }}

    path: /esg/config/logstash/pipelines

    state: directory

  loop:

    - certs

    - pipelines



- name: Install certificate for stats server

  copy:

    content: "{{ logstash_stats_certificate }}"

    dest: /esg/config/logstash/certs/lumberjack.cert



- name: Install stats logstash configuration

  template:
@@ -44,8 +36,6 @@ 
    user: "{{ data_security_context_user }}"

    groups: "{{ data_security_context_groups }}"

    volumes:

      # Logstash certificates

      - "/esg/config/logstash/certs:/etc/logstash/certs:ro"

      # Logstash configuration

      - "/esg/config/logstash/pipelines:/etc/logstash/conf.d:ro"

      # Log directory

deploy/ansible/roles/data/templates/stats.conf.j2

+24 −6
Original line number Diff line number Diff line
@@ -63,16 +63,34 @@ filter { 
        replace => { "host" => "{{ hostname }}" }

        # Move the country code to the top-level

        add_field => { "country_code" => "%{[geoip][country_code2]}" }

        # Construct a new message string

        add_field => {

            "rabbitmq_message" => '%{host} %{country_code} %{request_time} %{method} %{request} HTTP/%{http_version} %{status_code} %{bytes_sent} %{request_duration} \"%{user_agent}\"'

        }

    }

    mutate {

        # For Rabbit queue compatibility

        add_field => {

            "rabbitmq_payload" => '{ "properties": { "delivery_mode": 2 }, "routing_key": "log_queue", "payload": "%{rabbitmq_message}", "payload_encoding": "string" }'

        }

        # Remove any unrequired fields

        remove_field => ["client_ip", "command", "geoip", "tags", "user"]

        remove_field => ["rabbitmq_message", "command", "geoip", "tags", "user", "host", "country_code", "request_time", "method", "request", "http_version", "status_code", "bytes_sent", "referrer", "user_agent", "client_ip", "request_duration"]

    }

}

output {

    lumberjack {

        codec => json

        hosts => ["{{ logstash_stats_server }}"]

        port => {{ logstash_stats_port }}

        ssl_certificate => "/etc/logstash/certs/lumberjack.cert"

    http {

        url => "{{ logstash_stats_url }}"

        http_method => "post"

        headers => {

            "Authorization" => "Basic {{ logstash_stats_authorization_key }}"

            "Content-Type" => "application/json"

        }

        

        format => "message" 

        

        message => "%{rabbitmq_payload}"

        

        http_compression => false

    }

    stdout {

        codec => rubydebug

deploy/kubernetes/chart/files/logstash/10-stats.conf

+9 −3
Original line number Diff line number Diff line
@@ -57,12 +57,18 @@ filter { 
        replace => { "host" => "${EXTERNAL_HOSTNAME}" }

        # Move the country code to the top-level

        add_field => { "country_code" => "%{[geoip][country_code2]}" }

        # Remove any unrequired fields

        remove_field => ["client_ip", "command", "geoip", "tags", "user"]

        # Construct a new message string

        add_field => {

            "rabbitmq_message" => '%{host} %{country_code} %{request_time} %{method} %{request} HTTP/%{http_version} %{status_code} %{bytes_sent} %{request_duration} \"%{user_agent}\"'

        }

    }

    mutate {

        # For Rabbit queue compatibility

        add_field => {

            "rabbitmq_payload" => '{ "properties": { "delivery_mode": 2 }, "routing_key": "log_queue", "payload": "%{message}", "payload_encoding": "string" }'

            "rabbitmq_payload" => '{ "properties": { "delivery_mode": 2 }, "routing_key": "log_queue", "payload": "%{rabbitmq_message}", "payload_encoding": "string" }'

        }

        # Remove any unrequired fields

        remove_field => ["rabbitmq_message", "command", "geoip", "tags", "user", "host", "country_code", "request_time", "method", "request", "http_version", "status_code", "bytes_sent", "referrer", "user_agent", "client_ip", "request_duration"]

    }

}

output {

deploy/kubernetes/chart/templates/fileServer/deployment.yaml

+0 −7
Original line number Diff line number Diff line
@@ -25,7 +25,6 @@ spec: 
        {{- if $accessLogSidecar.enabled }}

        # When the access log sidecar is enabled, roll the deployment if the logstash pipelines or certificates change

        checksum/logstash-pipelines: {{ include (print $.Template.BasePath "/logstash/pipelines.yaml") . | sha256sum }}

        checksum/logstash-certs: {{ include (print $.Template.BasePath "/logstash/certs.yaml") . | sha256sum }}

        {{- end }}

        {{- with $fileServer.annotations }}

        {{- toYaml . | nindent 8 }}
@@ -127,8 +126,6 @@ spec: 
          volumeMounts:

            - name: logstash-conf

              mountPath: /etc/logstash/conf.d

            - name: logstash-certs

              mountPath: /etc/logstash/certs

            - name: nginx-logs

              mountPath: /var/log/nginx

            - name: logstash-data
@@ -200,10 +197,6 @@ spec: 
        - name: logstash-conf

          secret:

            secretName: {{ include "esgf.component.fullname" (list . "logstash-pipelines") }}

        # As are any certificates that are required

        - name: logstash-certs

          secret:

            secretName: {{ include "esgf.component.fullname" (list . "logstash-certs") }}

        # Logstash also needs a tmp dir and a data dir

        - name: logstash-data

          emptyDir: {}