From 3f8716f96c765056a881b718ea7dfa2b4f550648 Mon Sep 17 00:00:00 2001
From: Ivan Shumkov <ivan@shumkov.ru>
Date: Thu, 8 Aug 2024 17:05:11 +0700
Subject: [PATCH 01/12] refactor: split elastic_beat into core_filebeat,
 metricbeat and dashmate

---
 ansible/deploy.yml                            | 32 ++++-----
 ansible/group_vars/all                        |  1 +
 ansible/roles/core_filebeat/defaults/main.yml |  3 +
 ansible/roles/core_filebeat/tasks/main.yml    | 32 +++++++++
 ansible/roles/core_filebeat/vars/common.yml   | 12 ++++
 ansible/roles/core_filebeat/vars/core.yml     | 17 +++++
 ansible/roles/dashmate/defaults/filebeat.yml  | 72 +++++++++++++++++++
 ansible/roles/dashmate/tasks/logs.yml         | 49 +++++++++++++
 ansible/roles/dashmate/tasks/main.yml         | 35 +--------
 .../tasks/main.yml                            |  6 ++
 .../vars/common.yml                           |  0
 .../vars/core.yml                             |  0
 .../vars/drive.yml                            |  0
 .../vars/tenderdash.yml                       |  0
 ansible/roles/metricbeat/tasks/main.yml       | 59 +++++++++++++++
 15 files changed, 269 insertions(+), 49 deletions(-)
 create mode 100644 ansible/roles/core_filebeat/defaults/main.yml
 create mode 100644 ansible/roles/core_filebeat/tasks/main.yml
 create mode 100644 ansible/roles/core_filebeat/vars/common.yml
 create mode 100644 ansible/roles/core_filebeat/vars/core.yml
 create mode 100644 ansible/roles/dashmate/defaults/filebeat.yml
 create mode 100644 ansible/roles/dashmate/tasks/logs.yml
 rename ansible/roles/{elastic_beats => dashmate_elastic_beats}/tasks/main.yml (97%)
 rename ansible/roles/{elastic_beats => dashmate_elastic_beats}/vars/common.yml (100%)
 rename ansible/roles/{elastic_beats => dashmate_elastic_beats}/vars/core.yml (100%)
 rename ansible/roles/{elastic_beats => dashmate_elastic_beats}/vars/drive.yml (100%)
 rename ansible/roles/{elastic_beats => dashmate_elastic_beats}/vars/tenderdash.yml (100%)
 create mode 100644 ansible/roles/metricbeat/tasks/main.yml

diff --git a/ansible/deploy.yml b/ansible/deploy.yml
index 9bba5a1c..7f5aa1e2 100644
--- a/ansible/deploy.yml
+++ b/ansible/deploy.yml
@@ -78,6 +78,7 @@
   become: true
   roles:
     - elastic_stack
+    - metricbeat
 
 - name: Setup load tester
   hosts: load_test
@@ -90,7 +91,8 @@
   hosts: metrics
   become: true
   roles:
-    - role: metrics
+    - metrics
+    - metricbeat
 
 - name: Set up miners
   hosts: miners
@@ -100,8 +102,8 @@
     - role: dashd
       tags:
         - dashd
-    - role: elastic_beats
-      core_container_name: dashd
+    - core_filebeat
+    - metricbeat
 
 - name: Set up mixers
   hosts: mixer_nodes
@@ -111,8 +113,8 @@
     - role: dashd
       tags:
         - dashd
-    - role: elastic_beats
-      core_container_name: dashd
+    - core_filebeat
+    - metricbeat
 
 - name: Set up core and tenderdash on seed nodes
   hosts: seed_nodes
@@ -133,8 +135,8 @@
       tags:
         - dashd
     - role: tenderdash
-    - role: elastic_beats
-      core_container_name: dashd
+    - core_filebeat
+    - metricbeat
 
 - name: Set up core on masternodes
   hosts: masternodes
@@ -158,8 +160,8 @@
       tags:
         - dashd
     - mn_status_report
-    - role: elastic_beats
-      core_container_name: dashd
+    - core_filebeat
+    - metricbeat
 
 
 # Start network
@@ -189,8 +191,8 @@
       dashd_zmq: true
       dashd_listen: true
     - insight
-    - role: elastic_beats
-      core_container_name: dashd
+    - core_filebeat
+    - metricbeat
   tags:
     - web
 
@@ -205,8 +207,8 @@
       enable_wallet: true
       tags:
         - dashd
-    - role: elastic_beats
-      core_container_name: dashd
+    - core_filebeat
+    - metricbeat
 
 # Register masternodes and set sporks
 
@@ -283,9 +285,7 @@
     - role: dash_cli
     - role: dashmate
     - role: mn_status_report
-    - role: elastic_beats
-      core_container_name: core
-      abci_logs_path: "{{ dashmate_logs_dir }}"
+    - role: metricbeat
 
 - name: Set up protx diff script
   hosts: masternodes
diff --git a/ansible/group_vars/all b/ansible/group_vars/all
index 3f932252..774ce1f7 100644
--- a/ansible/group_vars/all
+++ b/ansible/group_vars/all
@@ -157,6 +157,7 @@ kibana_encryptionkey:
 # Set to 50% of instance memory
 # https://www.elastic.co/guide/en/elasticsearch/guide/current/heap-sizing.html
 elastic_heap_size: 8g
+metricbeat_enabled: true
 
 elastic_compose_project_name: elastic
 elastic_path: '{{ dashd_home }}/{{ elastic_compose_project_name }}'
diff --git a/ansible/roles/core_filebeat/defaults/main.yml b/ansible/roles/core_filebeat/defaults/main.yml
new file mode 100644
index 00000000..7da5862f
--- /dev/null
+++ b/ansible/roles/core_filebeat/defaults/main.yml
@@ -0,0 +1,3 @@
+---
+
+core_container_name: "dashd"
diff --git a/ansible/roles/core_filebeat/tasks/main.yml b/ansible/roles/core_filebeat/tasks/main.yml
new file mode 100644
index 00000000..11cd8d77
--- /dev/null
+++ b/ansible/roles/core_filebeat/tasks/main.yml
@@ -0,0 +1,32 @@
+---
+
+- name: Load common filebeat config
+  ansible.builtin.include_vars:
+    file: common.yml
+
+- name: Get core container host info
+  community.docker.docker_host_info:
+    containers: true
+    containers_filters:
+      name: '{{ core_container_name }}'
+  register: core_host_info
+
+- name: Set container ids for core if core is running
+  ansible.builtin.set_fact:
+    core_container_id: '{{ core_host_info.containers[0].Id }}'
+  when: core_host_info.containers | length > 0
+
+- name: Load core input config if core is running
+  ansible.builtin.include_vars:
+    file: core.yml
+  when: core_container_id is defined
+
+- name: Set up filebeat log monitoring
+  ansible.builtin.include_role:
+    name: geerlingguy.filebeat
+
+- name: Make sure filebeat is restarted
+  service:
+    name: filebeat
+    state: restarted
+    enabled: true
diff --git a/ansible/roles/core_filebeat/vars/common.yml b/ansible/roles/core_filebeat/vars/common.yml
new file mode 100644
index 00000000..98c12315
--- /dev/null
+++ b/ansible/roles/core_filebeat/vars/common.yml
@@ -0,0 +1,12 @@
+---
+
+filebeat_version: 8.x
+filebeat_package: "filebeat={{ elastic_version }}"
+filebeat_output_logstash_enabled: false
+filebeat_output_elasticsearch_enabled: true
+filebeat_output_elasticsearch_hosts: "{{ query('inventory_hostnames', 'logs') | map('extract', hostvars, ['private_ip']) | map('~(item) => item + \":9200\"') | list }}"
+filebeat_output_elasticsearch_auth:
+  username: "{{ elastic_username }}"
+  password: "{{ elastic_password }}"
+filebeat_enable_logging: true
+filebeat_log_level: info
diff --git a/ansible/roles/core_filebeat/vars/core.yml b/ansible/roles/core_filebeat/vars/core.yml
new file mode 100644
index 00000000..12810d32
--- /dev/null
+++ b/ansible/roles/core_filebeat/vars/core.yml
@@ -0,0 +1,17 @@
+---
+
+filebeat_inputs:
+  - type: container
+    enabled: true
+    index: "logs-core-{{ dash_network_name }}-%{[agent.version]}"
+    paths:
+      - '/var/lib/docker/containers/{{ core_container_id }}/*.log'
+    processors:
+      - add_fields:
+          target: event
+          fields:
+            dataset: "core-{{ dash_network_name }}"
+      - dissect:
+          tokenizer: "%{?timestamp} %{message}"
+          overwrite_keys: true
+          target_prefix: ""
diff --git a/ansible/roles/dashmate/defaults/filebeat.yml b/ansible/roles/dashmate/defaults/filebeat.yml
new file mode 100644
index 00000000..34488bba
--- /dev/null
+++ b/ansible/roles/dashmate/defaults/filebeat.yml
@@ -0,0 +1,72 @@
+---
+
+filebeat_version: 8.x
+filebeat_package: "filebeat={{ elastic_version }}"
+filebeat_output_logstash_enabled: false
+filebeat_output_elasticsearch_enabled: true
+filebeat_output_elasticsearch_hosts: "{{ query('inventory_hostnames', 'logs') | map('extract', hostvars, ['private_ip']) | map('~(item) => item + \":9200\"') | list }}"
+filebeat_output_elasticsearch_auth:
+  username: "{{ elastic_username }}"
+  password: "{{ elastic_password }}"
+filebeat_enable_logging: true
+filebeat_log_level: info
+filebeat_inputs:
+  - type: log
+    enabled: true
+    index: "logs-core-{{ dash_network_name }}-%{[agent.version]}"
+    paths:
+      - "{{ dashmate_logs_dir }}/core.log*"
+    processors:
+      - add_fields:
+          target: event
+          fields:
+            dataset: "core-{{ dash_network_name }}"
+      - dissect:
+          tokenizer: "%{?timestamp} %{message}"
+          overwrite_keys: true
+          target_prefix: ""
+  - type: log
+    enabled: "{{ dashmate_platform_enable }}"
+    json.message_key: message
+    exclude_files: ['\.gz$']
+    index: "logs-drive.abci-{{ dash_network_name }}-%{[agent.version]}"
+    paths:
+      - "{{ dashmate_logs_dir }}/drive-json.log*"
+    processors:
+      - timestamp:
+          field: json.timestamp
+          layouts:
+            - UNIX_MS
+      - add_fields:
+          target: event
+          fields:
+            dataset: "drive.abci-{{ dash_network_name }}"
+      - rename:
+          fields:
+            - from: "json.fields.message"
+              to: "message"
+            - from: "json.level"
+              to: "log.level"
+          ignore_missing: true
+          fail_on_error: true
+  - type: log
+    enabled: "{{ dashmate_platform_enable }}"
+    json.message_key: message
+    index: "logs-drive.tenderdash-{{ dash_network_name }}-%{[agent.version]}"
+    paths:
+      - "{{ dashmate_logs_dir }}/tenderdash.log*"
+    processors:
+      - add_fields:
+          target: event
+          fields:
+            dataset: "drive.tenderdash-{{ dash_network_name }}"
+      - rename:
+          fields:
+            - from: "json.message"
+              to: "message"
+          ignore_missing: true
+          fail_on_error: true
+      - rename:
+          fields:
+            - from: "json.level"
+              to: "log.level"
diff --git a/ansible/roles/dashmate/tasks/logs.yml b/ansible/roles/dashmate/tasks/logs.yml
new file mode 100644
index 00000000..b886c6d2
--- /dev/null
+++ b/ansible/roles/dashmate/tasks/logs.yml
@@ -0,0 +1,49 @@
+---
+
+- name: Create logs dir
+  ansible.builtin.file:
+    path: '{{ dashmate_logs_dir }}'
+    state: directory
+    owner: '{{ dashmate_user }}'
+    group: '{{ dashmate_group }}'
+    recurse: true
+
+- name: Configure log rotation
+  ansible.builtin.include_role:
+    name: arillso.logrotate
+  vars:
+    logrotate_applications:
+      - name: platform-logs
+        definitions:
+          - logs:
+              - '{{ dashmate_logs_dir }}/*.log'
+            options:
+              - rotate 7
+              - daily
+              - maxsize 1G
+              - missingok
+              - notifempty
+              - copytruncate
+              - compress
+              - delaycompress
+
+- name: Ensure logrotate runs hourly under systemd timer
+  ansible.builtin.lineinfile:
+    path: /lib/systemd/system/logrotate.timer
+    regexp: '^OnCalendar=hourly'
+    insertafter: '^OnCalendar=daily'
+    line: OnCalendar=hourly
+
+- name: Load common filebeat config
+  ansible.builtin.include_vars:
+    file: filebeat.yml
+
+- name: Set up filebeat log monitoring
+  ansible.builtin.include_role:
+    name: geerlingguy.filebeat
+
+- name: Make sure filebeat is restarted
+  service:
+    name: filebeat
+    state: restarted
+    enabled: true
diff --git a/ansible/roles/dashmate/tasks/main.yml b/ansible/roles/dashmate/tasks/main.yml
index 25e6ba6e..44b0aadd 100644
--- a/ansible/roles/dashmate/tasks/main.yml
+++ b/ansible/roles/dashmate/tasks/main.yml
@@ -42,39 +42,8 @@
       dir: '{{ dashmate_home }}'
     users: '{{ system_users + [dashmate_user_dict] }}'
 
-- name: Create logs dir
-  ansible.builtin.file:
-    path: '{{ dashmate_logs_dir }}'
-    state: directory
-    owner: '{{ dashmate_user }}'
-    group: '{{ dashmate_group }}'
-    recurse: true
-
-- name: Configure log rotation
-  ansible.builtin.include_role:
-    name: arillso.logrotate
-  vars:
-    logrotate_applications:
-      - name: platform-logs
-        definitions:
-          - logs:
-              - '{{ dashmate_logs_dir }}/*.log'
-            options:
-              - rotate 7
-              - daily
-              - maxsize 1G
-              - missingok
-              - notifempty
-              - copytruncate
-              - compress
-              - delaycompress
-
-- name: Ensure logrotate runs hourly under systemd timer
-  ansible.builtin.lineinfile:
-    path: /lib/systemd/system/logrotate.timer
-    regexp: '^OnCalendar=hourly'
-    insertafter: '^OnCalendar=daily'
-    line: OnCalendar=hourly
+- name: Configure logs
+  ansible.builtin.import_tasks: ./logs.yml
 
 - name: Create dashmate config dir
   ansible.builtin.file:
diff --git a/ansible/roles/elastic_beats/tasks/main.yml b/ansible/roles/dashmate_elastic_beats/tasks/main.yml
similarity index 97%
rename from ansible/roles/elastic_beats/tasks/main.yml
rename to ansible/roles/dashmate_elastic_beats/tasks/main.yml
index 2646f6e0..88b4df77 100644
--- a/ansible/roles/elastic_beats/tasks/main.yml
+++ b/ansible/roles/dashmate_elastic_beats/tasks/main.yml
@@ -58,6 +58,12 @@
   ansible.builtin.include_role:
     name: geerlingguy.filebeat
 
+- name: Make sure Filebeat is restarted
+  service:
+    name: filebeat
+    state: restarted
+    enabled: true
+
 # TODO: Make sure we have retention policy for metrics
 - name: Set up metricbeat
   ansible.builtin.include_role:
diff --git a/ansible/roles/elastic_beats/vars/common.yml b/ansible/roles/dashmate_elastic_beats/vars/common.yml
similarity index 100%
rename from ansible/roles/elastic_beats/vars/common.yml
rename to ansible/roles/dashmate_elastic_beats/vars/common.yml
diff --git a/ansible/roles/elastic_beats/vars/core.yml b/ansible/roles/dashmate_elastic_beats/vars/core.yml
similarity index 100%
rename from ansible/roles/elastic_beats/vars/core.yml
rename to ansible/roles/dashmate_elastic_beats/vars/core.yml
diff --git a/ansible/roles/elastic_beats/vars/drive.yml b/ansible/roles/dashmate_elastic_beats/vars/drive.yml
similarity index 100%
rename from ansible/roles/elastic_beats/vars/drive.yml
rename to ansible/roles/dashmate_elastic_beats/vars/drive.yml
diff --git a/ansible/roles/elastic_beats/vars/tenderdash.yml b/ansible/roles/dashmate_elastic_beats/vars/tenderdash.yml
similarity index 100%
rename from ansible/roles/elastic_beats/vars/tenderdash.yml
rename to ansible/roles/dashmate_elastic_beats/vars/tenderdash.yml
diff --git a/ansible/roles/metricbeat/tasks/main.yml b/ansible/roles/metricbeat/tasks/main.yml
new file mode 100644
index 00000000..9d8979b4
--- /dev/null
+++ b/ansible/roles/metricbeat/tasks/main.yml
@@ -0,0 +1,59 @@
+---
+
+- name: Set up metricbeat
+  ansible.builtin.include_role:
+    name: elastic.beats
+  vars:
+    beats_version: "{{ elastic_version }}"
+    beat: metricbeat
+    beat_conf:
+      setup:
+        dashboards:
+          enabled: true
+        kibana:
+          host: "{{ hostvars['logs-1'].private_ip }}:5601"
+          username: "{{ elastic_username }}"
+          password: "{{ elastic_password }}"
+      metricbeat:
+        modules:
+          - module: system
+            metricsets:
+              - cpu             # CPU usage
+              - load            # CPU load averages
+              - memory          # Memory usage
+              - network         # Network IO
+              - process         # Per process metrics
+              - process_summary # Process summary
+              - uptime          # System Uptime
+              - socket_summary  # Socket summary
+              - core           # Per CPU core usage
+              - diskio         # Disk IO
+              - fsstat         # File system summary metrics
+              - socket         # Sockets and connection info (linux only)
+            enabled: true
+            period: 10s
+            processes: ['.*']
+
+            # Configure the metric types that are included by these metricsets.
+            cpu.metrics: ["percentages", "normalized_percentages"]  # The other available option is ticks.
+            core.metrics: ["percentages"]  # The other available option is ticks.
+          - module: docker
+            metricsets:
+              - "container"
+              - "cpu"
+              - "diskio"
+              - "event"
+              - "healthcheck"
+              - "info"
+              - "memory"
+              - "network"
+              # - "network_summary"
+            hosts: ["unix:///var/run/docker.sock"]
+            period: 10s
+            enabled: true
+    output_conf:
+      elasticsearch:
+        hosts: "{{ query('inventory_hostnames', 'logs') | map('extract', hostvars, ['private_ip']) | map('~(item) => item + \":9200\"') | list }}"
+        username: "{{ elastic_username }}"
+        password: "{{ elastic_password }}"
+  when: metricbeat_enabled

From 38fb6f62e9fb3268fd1a7434574e297ff7729a27 Mon Sep 17 00:00:00 2001
From: Ivan Shumkov <ivan@shumkov.ru>
Date: Thu, 8 Aug 2024 17:16:35 +0700
Subject: [PATCH 02/12] refactor: use FQN and remove temp role

---
 ansible/roles/core_filebeat/tasks/main.yml    |   2 +-
 ansible/roles/dashmate/tasks/logs.yml         |   2 +-
 .../dashmate_elastic_beats/tasks/main.yml     | 124 ------------------
 .../dashmate_elastic_beats/vars/common.yml    |  13 --
 .../dashmate_elastic_beats/vars/core.yml      |  17 ---
 .../dashmate_elastic_beats/vars/drive.yml     |  27 ----
 .../vars/tenderdash.yml                       |  24 ----
 7 files changed, 2 insertions(+), 207 deletions(-)
 delete mode 100644 ansible/roles/dashmate_elastic_beats/tasks/main.yml
 delete mode 100644 ansible/roles/dashmate_elastic_beats/vars/common.yml
 delete mode 100644 ansible/roles/dashmate_elastic_beats/vars/core.yml
 delete mode 100644 ansible/roles/dashmate_elastic_beats/vars/drive.yml
 delete mode 100644 ansible/roles/dashmate_elastic_beats/vars/tenderdash.yml

diff --git a/ansible/roles/core_filebeat/tasks/main.yml b/ansible/roles/core_filebeat/tasks/main.yml
index 11cd8d77..001cf43e 100644
--- a/ansible/roles/core_filebeat/tasks/main.yml
+++ b/ansible/roles/core_filebeat/tasks/main.yml
@@ -26,7 +26,7 @@
     name: geerlingguy.filebeat
 
 - name: Make sure filebeat is restarted
-  service:
+  ansible.builtin.service:
     name: filebeat
     state: restarted
     enabled: true
diff --git a/ansible/roles/dashmate/tasks/logs.yml b/ansible/roles/dashmate/tasks/logs.yml
index b886c6d2..7025e9b8 100644
--- a/ansible/roles/dashmate/tasks/logs.yml
+++ b/ansible/roles/dashmate/tasks/logs.yml
@@ -43,7 +43,7 @@
     name: geerlingguy.filebeat
 
 - name: Make sure filebeat is restarted
-  service:
+  ansible.builtin.service:
     name: filebeat
     state: restarted
     enabled: true
diff --git a/ansible/roles/dashmate_elastic_beats/tasks/main.yml b/ansible/roles/dashmate_elastic_beats/tasks/main.yml
deleted file mode 100644
index 88b4df77..00000000
--- a/ansible/roles/dashmate_elastic_beats/tasks/main.yml
+++ /dev/null
@@ -1,124 +0,0 @@
----
-
-- name: Get core container host info
-  community.docker.docker_host_info:
-    containers: true
-    containers_filters:
-      name: '{{ core_container_name }}'
-  register: core_host_info
-
-- name: Get tenderdash container host info
-  community.docker.docker_host_info:
-    containers: true
-    containers_filters:
-      name: tender
-  register: tenderdash_host_info
-
-- name: Get drive container host info
-  community.docker.docker_host_info:
-    containers: true
-    containers_filters:
-      name: abci
-  register: drive_host_info
-
-- name: Set container ids for core and tenderdash
-  ansible.builtin.set_fact:
-    core_container_id: '{{ core_host_info.containers[0].Id if (core_host_info.containers | length > 0) else "null" }}'
-    tenderdash_container_id: '{{ tenderdash_host_info.containers[0].Id if (tenderdash_host_info.containers | length > 0) else "null" }}'
-
-- name: Load common filebeat config
-  ansible.builtin.include_vars:
-    file: common.yml
-
-- name: Load core input config
-  ansible.builtin.include_vars:
-    file: core.yml
-
-- name: Load tenderdash input config
-  ansible.builtin.include_vars:
-    file: tenderdash.yml
-  when: tenderdash_host_info.containers | length > 0
-
-- name: Load drive input config
-  ansible.builtin.include_vars:
-    file: drive.yml
-  when: drive_host_info.containers | length > 0
-
-- name: Merge drive and tenderdash input configs
-  ansible.builtin.set_fact:
-    platform_filebeat_inputs: "{{ [platform_filebeat_inputs, drive_filebeat_inputs] | community.general.lists_mergeby('index') }}"
-  when: platform_filebeat_inputs is defined and drive_filebeat_inputs is defined
-
-- name: Merge platform and core input configs
-  ansible.builtin.set_fact:
-    filebeat_inputs: "{{ [filebeat_inputs, platform_filebeat_inputs] | community.general.lists_mergeby('index') }}"
-  when: platform_filebeat_inputs is defined
-
-- name: Set up filebeat log monitoring
-  ansible.builtin.include_role:
-    name: geerlingguy.filebeat
-
-- name: Make sure Filebeat is restarted
-  service:
-    name: filebeat
-    state: restarted
-    enabled: true
-
-# TODO: Make sure we have retention policy for metrics
-- name: Set up metricbeat
-  ansible.builtin.include_role:
-    name: elastic.beats
-  vars:
-    beats_version: "{{ elastic_version }}"
-    beat: metricbeat
-    beat_conf:
-      setup:
-        dashboards:
-          enabled: true
-        kibana:
-          host: "{{ hostvars['logs-1'].private_ip }}:5601"
-          username: "{{ elastic_username }}"
-          password: "{{ elastic_password }}"
-      metricbeat:
-        modules:
-          - module: system
-            metricsets:
-              - cpu             # CPU usage
-              - load            # CPU load averages
-              - memory          # Memory usage
-              - network         # Network IO
-              - process         # Per process metrics
-              - process_summary # Process summary
-              - uptime          # System Uptime
-              - socket_summary  # Socket summary
-              - core           # Per CPU core usage
-              - diskio         # Disk IO
-              - fsstat         # File system summary metrics
-              - socket         # Sockets and connection info (linux only)
-            enabled: true
-            period: 10s
-            processes: ['.*']
-
-            # Configure the metric types that are included by these metricsets.
-            cpu.metrics: ["percentages", "normalized_percentages"]  # The other available option is ticks.
-            core.metrics: ["percentages"]  # The other available option is ticks.
-          - module: docker
-            metricsets:
-              - "container"
-              - "cpu"
-              - "diskio"
-              - "event"
-              - "healthcheck"
-              - "info"
-              - "memory"
-              - "network"
-              # - "network_summary"
-            hosts: ["unix:///var/run/docker.sock"]
-            period: 10s
-            enabled: true
-    output_conf:
-      elasticsearch:
-        hosts:
-          - "{{ hostvars['logs-1'].private_ip }}:9200"
-        username: "{{ elastic_username }}"
-        password: "{{ elastic_password }}"
diff --git a/ansible/roles/dashmate_elastic_beats/vars/common.yml b/ansible/roles/dashmate_elastic_beats/vars/common.yml
deleted file mode 100644
index d2b0f728..00000000
--- a/ansible/roles/dashmate_elastic_beats/vars/common.yml
+++ /dev/null
@@ -1,13 +0,0 @@
----
-
-filebeat_version: 8.x
-filebeat_package: "filebeat={{ elastic_version }}"
-filebeat_output_logstash_enabled: false
-filebeat_output_elasticsearch_enabled: true
-filebeat_output_elasticsearch_hosts:
-  - "{{ hostvars['logs-1'].private_ip }}:9200"
-filebeat_output_elasticsearch_auth:
-  username: "{{ elastic_username }}"
-  password: "{{ elastic_password }}"
-filebeat_enable_logging: true
-filebeat_log_level: info
diff --git a/ansible/roles/dashmate_elastic_beats/vars/core.yml b/ansible/roles/dashmate_elastic_beats/vars/core.yml
deleted file mode 100644
index ed7e161b..00000000
--- a/ansible/roles/dashmate_elastic_beats/vars/core.yml
+++ /dev/null
@@ -1,17 +0,0 @@
----
-
-filebeat_inputs:
-  - type: container
-    enabled: "{{ core_host_info.containers | length > 0 }}"
-    index: "logs-core-{{ dash_network_name }}-%{[agent.version]}"
-    paths:
-      - '/var/lib/docker/containers/{{ core_container_id }}/*.log'
-    processors:
-      - add_fields:
-          target: event
-          fields:
-            dataset: "core-{{ dash_network_name }}"
-      - dissect:
-          tokenizer: "%{?timestamp} %{message}"
-          overwrite_keys: true
-          target_prefix: ""
diff --git a/ansible/roles/dashmate_elastic_beats/vars/drive.yml b/ansible/roles/dashmate_elastic_beats/vars/drive.yml
deleted file mode 100644
index 6cc5efc9..00000000
--- a/ansible/roles/dashmate_elastic_beats/vars/drive.yml
+++ /dev/null
@@ -1,27 +0,0 @@
----
-
-drive_filebeat_inputs:
-  - type: log
-    enabled: "{{ drive_host_info.containers | length > 0 }}"
-    json.message_key: message
-    exclude_files: ['\.gz$']
-    index: "logs-drive.abci-{{ dash_network_name }}-%{[agent.version]}"
-    paths:
-      - "{{ abci_logs_path }}/drive-json*.log*"
-    processors:
-      - timestamp:
-          field: json.timestamp
-          layouts:
-            - UNIX_MS
-      - add_fields:
-          target: event
-          fields:
-            dataset: "drive.abci-{{ dash_network_name }}"
-      - rename:
-          fields:
-            - from: "json.fields.message"
-              to: "message"
-            - from: "json.level"
-              to: "log.level"
-          ignore_missing: true
-          fail_on_error: true
diff --git a/ansible/roles/dashmate_elastic_beats/vars/tenderdash.yml b/ansible/roles/dashmate_elastic_beats/vars/tenderdash.yml
deleted file mode 100644
index f958ec10..00000000
--- a/ansible/roles/dashmate_elastic_beats/vars/tenderdash.yml
+++ /dev/null
@@ -1,24 +0,0 @@
----
-
-platform_filebeat_inputs:
-  - type: container
-    enabled: "{{ tenderdash_host_info.containers | length > 0 }}"
-    json.message_key: message
-    index: "logs-drive.tenderdash-{{ dash_network_name }}-%{[agent.version]}"
-    paths:
-      - '/var/lib/docker/containers/{{ tenderdash_container_id }}/*.log'
-    processors:
-      - add_fields:
-          target: event
-          fields:
-            dataset: "drive.tenderdash-{{ dash_network_name }}"
-      - rename:
-          fields:
-            - from: "json.message"
-              to: "message"
-          ignore_missing: true
-          fail_on_error: true
-      - rename:
-          fields:
-            - from: "json.level"
-              to: "log.level"

From 1966976d40cff248940320913c13d6e0fd8df532 Mon Sep 17 00:00:00 2001
From: Ivan Shumkov <ivan@shumkov.ru>
Date: Thu, 8 Aug 2024 17:24:46 +0700
Subject: [PATCH 03/12] refactor: reuse filebeat config

---
 ansible/roles/core_filebeat/vars/common.yml            |  8 +++++++-
 .../defaults/{filebeat.yml => filebeat_inputs.yml}     | 10 ----------
 ansible/roles/dashmate/tasks/logs.yml                  |  6 +++++-
 3 files changed, 12 insertions(+), 12 deletions(-)
 rename ansible/roles/dashmate/defaults/{filebeat.yml => filebeat_inputs.yml} (78%)

diff --git a/ansible/roles/core_filebeat/vars/common.yml b/ansible/roles/core_filebeat/vars/common.yml
index 98c12315..2421c59f 100644
--- a/ansible/roles/core_filebeat/vars/common.yml
+++ b/ansible/roles/core_filebeat/vars/common.yml
@@ -4,7 +4,13 @@ filebeat_version: 8.x
 filebeat_package: "filebeat={{ elastic_version }}"
 filebeat_output_logstash_enabled: false
 filebeat_output_elasticsearch_enabled: true
-filebeat_output_elasticsearch_hosts: "{{ query('inventory_hostnames', 'logs') | map('extract', hostvars, ['private_ip']) | map('~(item) => item + \":9200\"') | list }}"
+filebeat_output_elasticsearch_hosts: >-
+  {{
+    query('inventory_hostnames', 'logs') |
+    map('extract', hostvars, ['private_ip']) |
+    map('~(item) => item + \":9200\"') |
+    list
+  }}
 filebeat_output_elasticsearch_auth:
   username: "{{ elastic_username }}"
   password: "{{ elastic_password }}"
diff --git a/ansible/roles/dashmate/defaults/filebeat.yml b/ansible/roles/dashmate/defaults/filebeat_inputs.yml
similarity index 78%
rename from ansible/roles/dashmate/defaults/filebeat.yml
rename to ansible/roles/dashmate/defaults/filebeat_inputs.yml
index 34488bba..80896735 100644
--- a/ansible/roles/dashmate/defaults/filebeat.yml
+++ b/ansible/roles/dashmate/defaults/filebeat_inputs.yml
@@ -1,15 +1,5 @@
 ---
 
-filebeat_version: 8.x
-filebeat_package: "filebeat={{ elastic_version }}"
-filebeat_output_logstash_enabled: false
-filebeat_output_elasticsearch_enabled: true
-filebeat_output_elasticsearch_hosts: "{{ query('inventory_hostnames', 'logs') | map('extract', hostvars, ['private_ip']) | map('~(item) => item + \":9200\"') | list }}"
-filebeat_output_elasticsearch_auth:
-  username: "{{ elastic_username }}"
-  password: "{{ elastic_password }}"
-filebeat_enable_logging: true
-filebeat_log_level: info
 filebeat_inputs:
   - type: log
     enabled: true
diff --git a/ansible/roles/dashmate/tasks/logs.yml b/ansible/roles/dashmate/tasks/logs.yml
index 7025e9b8..72037105 100644
--- a/ansible/roles/dashmate/tasks/logs.yml
+++ b/ansible/roles/dashmate/tasks/logs.yml
@@ -36,7 +36,11 @@
 
 - name: Load common filebeat config
   ansible.builtin.include_vars:
-    file: filebeat.yml
+    file: "{{ role_path }}/../core_filebeat/vars/common.yml"
+
+- name: Load filebeat inputs
+  ansible.builtin.include_vars:
+    file: filebeat_inputs.yml
 
 - name: Set up filebeat log monitoring
   ansible.builtin.include_role:

From f8925b232d3a1e2bb410e7f44fc6ecb7c8f683c7 Mon Sep 17 00:00:00 2001
From: Ivan Shumkov <ivan@shumkov.ru>
Date: Thu, 8 Aug 2024 20:49:43 +0700
Subject: [PATCH 04/12] fix: hosts

---
 ansible/roles/core_filebeat/vars/common.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/ansible/roles/core_filebeat/vars/common.yml b/ansible/roles/core_filebeat/vars/common.yml
index 2421c59f..125f4a1b 100644
--- a/ansible/roles/core_filebeat/vars/common.yml
+++ b/ansible/roles/core_filebeat/vars/common.yml
@@ -6,9 +6,10 @@ filebeat_output_logstash_enabled: false
 filebeat_output_elasticsearch_enabled: true
 filebeat_output_elasticsearch_hosts: >-
   {{
-    query('inventory_hostnames', 'logs') |
+    query('inventory_hostnames', 'logs_nodes') |
     map('extract', hostvars, ['private_ip']) |
-    map('~(item) => item + \":9200\"') |
+    map('string') |
+    map('format', '{}:9200') |
     list
   }}
 filebeat_output_elasticsearch_auth:

From b914f39baff51e409f40d776e3dddd7ea08cf5aa Mon Sep 17 00:00:00 2001
From: Ivan Shumkov <ivan@shumkov.ru>
Date: Thu, 8 Aug 2024 20:53:26 +0700
Subject: [PATCH 05/12] fix: hosts 2

---
 ansible/roles/metricbeat/tasks/main.yml | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/ansible/roles/metricbeat/tasks/main.yml b/ansible/roles/metricbeat/tasks/main.yml
index 9d8979b4..7b378c9b 100644
--- a/ansible/roles/metricbeat/tasks/main.yml
+++ b/ansible/roles/metricbeat/tasks/main.yml
@@ -53,7 +53,14 @@
             enabled: true
     output_conf:
       elasticsearch:
-        hosts: "{{ query('inventory_hostnames', 'logs') | map('extract', hostvars, ['private_ip']) | map('~(item) => item + \":9200\"') | list }}"
+        hosts: >-
+          {{
+            query('inventory_hostnames', 'logs_nodes') |
+            map('extract', hostvars, ['private_ip']) |
+            map('string') |
+            map('format', '{}:9200') |
+            list
+          }}
         username: "{{ elastic_username }}"
         password: "{{ elastic_password }}"
   when: metricbeat_enabled

From 2eb1a43013a1c83d85829e86020d65d587e94d41 Mon Sep 17 00:00:00 2001
From: Ivan Shumkov <ivan@shumkov.ru>
Date: Thu, 8 Aug 2024 21:48:01 +0700
Subject: [PATCH 06/12] fix: start kibana only on logs-1

---
 .../elastic_stack/tasks/configure_cluster.yml  | 15 +--------------
 ansible/roles/elastic_stack/tasks/main.yml     | 18 +++++++++++++++++-
 2 files changed, 18 insertions(+), 15 deletions(-)

diff --git a/ansible/roles/elastic_stack/tasks/configure_cluster.yml b/ansible/roles/elastic_stack/tasks/configure_cluster.yml
index 9cea11cb..899afd0f 100644
--- a/ansible/roles/elastic_stack/tasks/configure_cluster.yml
+++ b/ansible/roles/elastic_stack/tasks/configure_cluster.yml
@@ -20,7 +20,7 @@
     body_format: json
     body: "{{ lookup('file', 'files/index-templates/logs-core.json') }}"
 
-- name: Set up abci index template
+- name: Set up drive abci index template
   ansible.builtin.uri:
     url: http://localhost:9200/_index_template/logs-drive.abci
     method: PUT
@@ -52,16 +52,3 @@
     body:
       index:
         number_of_replicas: 0
-
-- name: Set up Kibana UI
-  ansible.builtin.uri:
-    url: http://localhost:5601/api/saved_objects/_bulk_create
-    method: POST
-    status_code: 200
-    user: '{{ elastic_username }}'
-    password: '{{ elastic_password }}'
-    body_format: json
-    force_basic_auth: true
-    headers:
-      kbn-xsrf: true
-    body: "{{ lookup('file', 'files/infrastructure-ui-source.json') }}"
diff --git a/ansible/roles/elastic_stack/tasks/main.yml b/ansible/roles/elastic_stack/tasks/main.yml
index 07015c25..f6e8ec6c 100644
--- a/ansible/roles/elastic_stack/tasks/main.yml
+++ b/ansible/roles/elastic_stack/tasks/main.yml
@@ -103,6 +103,7 @@
     recreate: always
     services:
       - kibana
+  when: inventory_hostname == "logs-1"
 
 - name: Wait for Kibana to be available
   ansible.builtin.uri:
@@ -114,8 +115,23 @@
   until: response.status == 200 and response.json.status.overall.level == "available"
   retries: 10
   delay: 10
+  when: inventory_hostname == "logs-1"
 
-- name: Configure Elasticsearch and Kibana
+- name: Set up Kibana UI
+  ansible.builtin.uri:
+    url: http://localhost:5601/api/saved_objects/_bulk_create
+    method: POST
+    status_code: 200
+    user: '{{ elastic_username }}'
+    password: '{{ elastic_password }}'
+    body_format: json
+    force_basic_auth: true
+    headers:
+      kbn-xsrf: true
+    body: "{{ lookup('file', 'files/infrastructure-ui-source.json') }}"
+  when: inventory_hostname == "logs-1"
+
+- name: Configure Elasticsearch cluster
   ansible.builtin.import_tasks: configure_cluster.yml
   run_once: true
   delegate_to: '{{ play_hosts | first }}'

From 00947f3469f21c7352ebc0a699b8d06c4c49abd4 Mon Sep 17 00:00:00 2001
From: Ivan Shumkov <ivan@shumkov.ru>
Date: Thu, 8 Aug 2024 21:48:26 +0700
Subject: [PATCH 07/12] fix: select host nodes

---
 ansible/roles/core_filebeat/vars/common.yml | 3 ++-
 ansible/roles/metricbeat/tasks/main.yml     | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/ansible/roles/core_filebeat/vars/common.yml b/ansible/roles/core_filebeat/vars/common.yml
index 125f4a1b..4567afcd 100644
--- a/ansible/roles/core_filebeat/vars/common.yml
+++ b/ansible/roles/core_filebeat/vars/common.yml
@@ -9,7 +9,8 @@ filebeat_output_elasticsearch_hosts: >-
     query('inventory_hostnames', 'logs_nodes') |
     map('extract', hostvars, ['private_ip']) |
     map('string') |
-    map('format', '{}:9200') |
+    product([':9200']) |
+    map('join') |
     list
   }}
 filebeat_output_elasticsearch_auth:
diff --git a/ansible/roles/metricbeat/tasks/main.yml b/ansible/roles/metricbeat/tasks/main.yml
index 7b378c9b..20212bf5 100644
--- a/ansible/roles/metricbeat/tasks/main.yml
+++ b/ansible/roles/metricbeat/tasks/main.yml
@@ -58,7 +58,8 @@
             query('inventory_hostnames', 'logs_nodes') |
             map('extract', hostvars, ['private_ip']) |
             map('string') |
-            map('format', '{}:9200') |
+            product([':9200']) |
+            map('join') |
             list
           }}
         username: "{{ elastic_username }}"

From 2efdbc1109abf67ae45b3860917fee527a30b7e1 Mon Sep 17 00:00:00 2001
From: Ivan Shumkov <ivan@shumkov.ru>
Date: Thu, 8 Aug 2024 21:48:44 +0700
Subject: [PATCH 08/12] fix: filebeat_inputs.yml not found

---
 ansible/roles/dashmate/{defaults => vars}/filebeat_inputs.yml | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename ansible/roles/dashmate/{defaults => vars}/filebeat_inputs.yml (100%)

diff --git a/ansible/roles/dashmate/defaults/filebeat_inputs.yml b/ansible/roles/dashmate/vars/filebeat_inputs.yml
similarity index 100%
rename from ansible/roles/dashmate/defaults/filebeat_inputs.yml
rename to ansible/roles/dashmate/vars/filebeat_inputs.yml

From 79381ae59382a4364b230d64dc7e8f0cdad48308 Mon Sep 17 00:00:00 2001
From: Ivan Shumkov <ivan@shumkov.ru>
Date: Thu, 8 Aug 2024 22:45:02 +0700
Subject: [PATCH 09/12] fix: reading logs from gzip

---
 ansible/roles/dashmate/vars/filebeat_inputs.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/ansible/roles/dashmate/vars/filebeat_inputs.yml b/ansible/roles/dashmate/vars/filebeat_inputs.yml
index 80896735..d0d21f40 100644
--- a/ansible/roles/dashmate/vars/filebeat_inputs.yml
+++ b/ansible/roles/dashmate/vars/filebeat_inputs.yml
@@ -5,7 +5,7 @@ filebeat_inputs:
     enabled: true
     index: "logs-core-{{ dash_network_name }}-%{[agent.version]}"
     paths:
-      - "{{ dashmate_logs_dir }}/core.log*"
+      - "{{ dashmate_logs_dir }}/core.log"
     processors:
       - add_fields:
           target: event
@@ -21,7 +21,7 @@ filebeat_inputs:
     exclude_files: ['\.gz$']
     index: "logs-drive.abci-{{ dash_network_name }}-%{[agent.version]}"
     paths:
-      - "{{ dashmate_logs_dir }}/drive-json.log*"
+      - "{{ dashmate_logs_dir }}/drive-json.log"
     processors:
       - timestamp:
           field: json.timestamp
@@ -44,7 +44,7 @@ filebeat_inputs:
     json.message_key: message
     index: "logs-drive.tenderdash-{{ dash_network_name }}-%{[agent.version]}"
     paths:
-      - "{{ dashmate_logs_dir }}/tenderdash.log*"
+      - "{{ dashmate_logs_dir }}/tenderdash.log"
     processors:
       - add_fields:
           target: event

From 54a4e8915bc6eb5abf6e688c9cf6a5e6d5a1ef73 Mon Sep 17 00:00:00 2001
From: Ivan Shumkov <ivan@shumkov.ru>
Date: Thu, 8 Aug 2024 22:46:45 +0700
Subject: [PATCH 10/12] fix: tenderdash indexing gz

---
 ansible/roles/dashmate/vars/filebeat_inputs.yml | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/ansible/roles/dashmate/vars/filebeat_inputs.yml b/ansible/roles/dashmate/vars/filebeat_inputs.yml
index d0d21f40..1cb69208 100644
--- a/ansible/roles/dashmate/vars/filebeat_inputs.yml
+++ b/ansible/roles/dashmate/vars/filebeat_inputs.yml
@@ -5,7 +5,8 @@ filebeat_inputs:
     enabled: true
     index: "logs-core-{{ dash_network_name }}-%{[agent.version]}"
     paths:
-      - "{{ dashmate_logs_dir }}/core.log"
+      - "{{ dashmate_logs_dir }}/core.log*"
+    exclude_files: ['\.gz$']
     processors:
       - add_fields:
           target: event
@@ -18,10 +19,10 @@ filebeat_inputs:
   - type: log
     enabled: "{{ dashmate_platform_enable }}"
     json.message_key: message
-    exclude_files: ['\.gz$']
     index: "logs-drive.abci-{{ dash_network_name }}-%{[agent.version]}"
     paths:
-      - "{{ dashmate_logs_dir }}/drive-json.log"
+      - "{{ dashmate_logs_dir }}/drive-json.log*"
+    exclude_files: ['\.gz$']
     processors:
       - timestamp:
           field: json.timestamp
@@ -44,7 +45,8 @@ filebeat_inputs:
     json.message_key: message
     index: "logs-drive.tenderdash-{{ dash_network_name }}-%{[agent.version]}"
     paths:
-      - "{{ dashmate_logs_dir }}/tenderdash.log"
+      - "{{ dashmate_logs_dir }}/tenderdash.log*"
+    exclude_files: ['\.gz$']
     processors:
       - add_fields:
           target: event

From 8ab72f660c1fa575ecd33dd324e80346515d6b24 Mon Sep 17 00:00:00 2001
From: Ivan Shumkov <ivan@shumkov.ru>
Date: Fri, 9 Aug 2024 12:35:41 +0700
Subject: [PATCH 11/12] fix: indexing old data

---
 ansible/roles/dashmate/vars/filebeat_inputs.yml | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/ansible/roles/dashmate/vars/filebeat_inputs.yml b/ansible/roles/dashmate/vars/filebeat_inputs.yml
index 1cb69208..e6886cbb 100644
--- a/ansible/roles/dashmate/vars/filebeat_inputs.yml
+++ b/ansible/roles/dashmate/vars/filebeat_inputs.yml
@@ -5,7 +5,7 @@ filebeat_inputs:
     enabled: true
     index: "logs-core-{{ dash_network_name }}-%{[agent.version]}"
     paths:
-      - "{{ dashmate_logs_dir }}/core.log*"
+      - "{{ dashmate_logs_dir }}/core.log"
     exclude_files: ['\.gz$']
     processors:
       - add_fields:
@@ -21,8 +21,7 @@ filebeat_inputs:
     json.message_key: message
     index: "logs-drive.abci-{{ dash_network_name }}-%{[agent.version]}"
     paths:
-      - "{{ dashmate_logs_dir }}/drive-json.log*"
-    exclude_files: ['\.gz$']
+      - "{{ dashmate_logs_dir }}/drive-json.log"
     processors:
       - timestamp:
           field: json.timestamp
@@ -45,8 +44,7 @@ filebeat_inputs:
     json.message_key: message
     index: "logs-drive.tenderdash-{{ dash_network_name }}-%{[agent.version]}"
     paths:
-      - "{{ dashmate_logs_dir }}/tenderdash.log*"
-    exclude_files: ['\.gz$']
+      - "{{ dashmate_logs_dir }}/tenderdash.log"
     processors:
       - add_fields:
           target: event

From 9b3b16375ea3a3f07e3b99172561a879005ee12c Mon Sep 17 00:00:00 2001
From: Ivan Shumkov <ivan@shumkov.ru>
Date: Fri, 9 Aug 2024 14:15:28 +0700
Subject: [PATCH 12/12] refactor: remove unused exclude

---
 ansible/roles/dashmate/vars/filebeat_inputs.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/ansible/roles/dashmate/vars/filebeat_inputs.yml b/ansible/roles/dashmate/vars/filebeat_inputs.yml
index e6886cbb..552254e5 100644
--- a/ansible/roles/dashmate/vars/filebeat_inputs.yml
+++ b/ansible/roles/dashmate/vars/filebeat_inputs.yml
@@ -6,7 +6,6 @@ filebeat_inputs:
     index: "logs-core-{{ dash_network_name }}-%{[agent.version]}"
     paths:
       - "{{ dashmate_logs_dir }}/core.log"
-    exclude_files: ['\.gz$']
     processors:
       - add_fields:
           target: event