Logstash working for Ansible installation

  config.vm.network :private_network, ip: "192.168.100.100"

  # Set some virtualbox flags to improve time synchronisation between host and guest

  config.vm.provider :virtualbox do |virtualbox|

  config.vm.provider :virtualbox do |v|

    # 512MB RAM is not really enough

    v.memory = 4096

    # sync time every 10 seconds

    virtualbox.customize [ "guestproperty", "set", :id, "/VirtualBox/GuestAdd/VBoxService/--timesync-interval", 10000 ]

    v.customize [ "guestproperty", "set", :id, "/VirtualBox/GuestAdd/VBoxService/--timesync-interval", 10000 ]

    # adjustments if drift > 100 ms

    virtualbox.customize [ "guestproperty", "set", :id, "/VirtualBox/GuestAdd/VBoxService/--timesync-min-adjust", 100 ]

    v.customize [ "guestproperty", "set", :id, "/VirtualBox/GuestAdd/VBoxService/--timesync-min-adjust", 100 ]

    # sync time on restore

    virtualbox.customize [ "guestproperty", "set", :id, "/VirtualBox/GuestAdd/VBoxService/--timesync-set-on-restore", 1 ]

    v.customize [ "guestproperty", "set", :id, "/VirtualBox/GuestAdd/VBoxService/--timesync-set-on-restore", 1 ]

    # sync time on start

    virtualbox.customize [ "guestproperty", "set", :id, "/VirtualBox/GuestAdd/VBoxService/--timesync-set-start", 1 ]

    v.customize [ "guestproperty", "set", :id, "/VirtualBox/GuestAdd/VBoxService/--timesync-set-start", 1 ]

    # at 1 second drift, the time will be set and not "smoothly" adjusted

    virtualbox.customize [ "guestproperty", "set", :id, "/VirtualBox/GuestAdd/VBoxService/--timesync-set-threshold", 1000 ]

    v.customize [ "guestproperty", "set", :id, "/VirtualBox/GuestAdd/VBoxService/--timesync-set-threshold", 1000 ]

  end

  config.vm.provision :shell, inline: <<-SHELL

      {

        name: "CORDEX",

        path: "esg_cordex",

        location: "/test_data/group_workspaces/jasmin2/cp4cds1/data/c3s-cordex"

        location: "/test_data/group_workspaces/jasmin2/cp4cds1/vol1/data/c3s-cordex"

      }

    ]

    solr_replicas = [

      "index" => ["default"],

      "all:vars" => {

        "hostname" => "192.168.100.100.nip.io",

        "image_tag" => "issue-115-esg-search",

        "image_tag" => "future-architecture",

      },

      "data:vars" => {

        "data_mounts" => "#{data_mounts.to_json}",

#

# If a special user or groups are required to access data, they can be set here, which

# will result in the data node containers launching with the specified UID and GIDs

#

# By default, use the ESGF user

data_security_context_user: "1000"

# Due to permissions set inside the container, the user *must* belong to group 1000

fileserver_image_tag: "{{ image_tag }}"

fileserver_image_pull: "{{ image_pull }}"

fileserver_image_repository: nginx

# Configuration for the logstash access log exporter

logstash_enabled: true

# Settings for the logstash access log exporter image

logstash_image_prefix: "{{ image_prefix }}"

logstash_image_tag: "{{ image_tag }}"

logstash_image_pull: "{{ image_pull }}"

logstash_image_repository: logstash

    src: fileserver.conf.j2

    dest: /esg/config/fileserver/fileserver.conf

# If the access log exporter is enabled, we use a FIFO pipe for the Nginx access log

# This will be followed by the exporter to get the access logs

# Unfortunately, in order to get the container to pick this up we have to splat

# the whole logs directory, so we also need to set up the other logs as symlinks to stdout

- name: Set up fileserver logs directory

  block:

    - name: Ensure fileserver logs directory exists

      file:

        path: /esg/logs/fileserver

        state: directory

    - name: Link error log to stderr

      file:

        path: /esg/logs/fileserver/error.log

        state: link

        src: /dev/stderr

        force: yes

    - name: Make the access log pipe

      command: mkfifo /esg/logs/fileserver/access.log

      args:

        creates: /esg/logs/fileserver/access.log

    - name: Transfer ownership of logs to security context user

      file:

        path: /esg/logs/fileserver

        recurse: yes

        owner: "{{ data_security_context_user }}"

  when: logstash_enabled | bool

- name: Start fileserver container

  docker_container:

    name: fileserver

    volumes: >-

      [

        "/esg/config/fileserver:/etc/nginx/conf.d:ro",

        # If logstash is enabled, mount the log directory

        {% if logstash_enabled %}

        "/esg/logs/fileserver:/var/log/nginx",

        {% endif %}

        # Then the data mounts

        {% for mount in data_mounts %}

        "{{ mount.host_path }}:{{ mount.mount_path }}:ro",

        {% endfor %}

      ]

    state: started

    restart: yes

    recreate: yes

---

- name: Create Docker network

  docker_network:

    name: esgf

- name: Ensure logstash config directory exists

  file:

    path: /esg/config/logstash

    state: directory

- name: Install logstash configuration

  template:

    src: access-log.conf.j2

    dest: /esg/config/logstash/access-log.conf

# Configure logstash to follow the THREDDS and fileserver logs

- name: Start logstash container

  docker_container:

    name: logstash

    image: "{{ logstash_image_prefix }}/{{ logstash_image_repository }}:{{ logstash_image_tag }}"

    pull: "{{ logstash_image_pull }}"

    detach: yes

    restart_policy: unless-stopped

    networks:

      - name: esgf

    networks_cli_compatible: yes

    user: "{{ data_security_context_user }}"

    groups: "{{ data_security_context_groups }}"

    volumes:

      # Logstash configuration

      - "/esg/config/logstash:/etc/logstash/conf.d:ro"

      # Log directory

      - "/esg/logs:/esg/logs:ro"

    state: started

    recreate: yes

---

- name: Stop logstash container

  docker_container:

    name: logstash

    state: absent

- name: Remove logstash config directory

  file:

    path: /esg/config/logstash

    state: absent