Convert Filebeat icinga.* to ECS (#9294)

- Map Icinga module fields to ECS: - icinga.debug.message => message - icinga.debug.severity => log.level - icinga.main.message => message - icinga.main.severity => log.level - icinga.startup.message => message - icinga.startup.severity => log.level
elastic · Dec 20, 2018 · 08fc351 · 08fc351
1 parent 837929e
commit 08fc351
Show file tree

Hide file tree

Showing 13 changed files with 100 additions and 93 deletions.
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
@@ -180,6 +180,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha1...v7.0.0-alpha2[Check the
 - Fix saved objects in filebeat haproxy dashboard. {pull}9417[9417]
 - Use `log.source.address` instead of `log.source.ip` for network input sources. {pull}9487[9487]
 - Rename many `redis.log.*` fields to map to ECS. {pull}9315[9315]
+- Rename many `icinga.*` fields to map to ECS. {pull}9294[9294]
 
 *Metricbeat*
 

diff --git a/dev-tools/ecs-migration.yml b/dev-tools/ecs-migration.yml
@@ -97,6 +97,8 @@
   alias6: true
 
 
+# Filebeat modules
+
 # Suricata module
 
 - from: source_ecs.ip
@@ -131,6 +133,8 @@
   to: source.geo.region_iso_code
   alias: true
 
+## System module
+
 - from: system.syslog.hostname
   to: host.hostname
   alias: true
@@ -188,8 +192,6 @@
   to: source.geo.*
   alias: true
 
-# Filebeat modules
-
 ## Apache module
 
 - from: apache2.access.user_name
@@ -254,7 +256,7 @@
   to: process.thread.id
   alias: true
 
-# IIS module
+## IIS module
 
 - from: iis.access.server_ip
   to: destination.ip
@@ -312,8 +314,6 @@
   to: source.geo.region_iso_code
   alias: true
 
-# Note: `http` is not officially in ECS yet
-
 - from: iis.access.method
   to: http.request.method
   alias: true
@@ -326,7 +326,8 @@
   to: http.request.referrer
   alias: true
 
-# HAProxy module
+## HAProxy module
+
 - from: haproxy.client.port
   to: source.port
   alias: true
@@ -375,6 +376,8 @@
   to: network.forwarded_ip
   alias: true
 
+## NGINX module
+
 - from: nginx.access.user_name
   to: user.name
   alias: true
@@ -387,8 +390,6 @@
   to: user_agent.original
   alias: true
 
-# Note: `http` is not officially in ECS yet
-
 - from: nginx.access.response_code
   to: http.response.status_code
   alias: true
@@ -447,12 +448,39 @@
   to: message
   alias: true
 
-# From Auditbeat's auditd module.
+## Icinga module
+
+- from: icinga.debug.message
+  to: message
+  alias: true
+- from: icinga.debug.severity
+  to: log.level
+  alias: true
+
+- from: icinga.main.message
+  to: message
+  alias: true
+- from: icinga.main.severity
+  to: log.level
+  alias: true
+
+- from: icinga.startup.message
+  to: message
+  alias: true
+- from: icinga.startup.severity
+  to: log.level
+  alias: true
+
+# Auditbeat
+
+## From Auditbeat's auditd module.
 - from: source.hostname
   to: source.domain
   alias: true
 
-# Metricbeat base fields
+# Metricbeat
+
+## Metricbeat base fields
 - from: metricset.name
   to: event.dataset
   alias: false
@@ -477,4 +505,3 @@
   to: event.dataset
   alias: false
   comment: No alias mapping as field did not always exist
-
diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc
@@ -4813,20 +4813,18 @@ Specifies what component of Icinga logged the message.
 *`icinga.debug.severity`*::
 +
 --
-type: keyword
-
-Possible values are "debug", "notice", "information", "warning" or "critical".
+type: alias
 
+alias to: log.level
 
 --
 
 *`icinga.debug.message`*::
 +
 --
-type: text
-
-The logged message.
+type: alias
 
+alias to: message
 
 --
 
@@ -4850,20 +4848,18 @@ Specifies what component of Icinga logged the message.
 *`icinga.main.severity`*::
 +
 --
-type: keyword
-
-Possible values are "debug", "notice", "information", "warning" or "critical".
+type: alias
 
+alias to: log.level
 
 --
 
 *`icinga.main.message`*::
 +
 --
-type: text
-
-The logged message.
+type: alias
 
+alias to: message
 
 --
 
@@ -4887,20 +4883,18 @@ Specifies what component of Icinga logged the message.
 *`icinga.startup.severity`*::
 +
 --
-type: keyword
-
-Possible values are "debug", "notice", "information", "warning" or "critical".
+type: alias
 
+alias to: log.level
 
 --
 
 *`icinga.startup.message`*::
 +
 --
-type: text
-
-The logged message.
+type: alias
 
+alias to: message
 
 --
 

diff --git a/filebeat/module/icinga/debug/_meta/fields.yml b/filebeat/module/icinga/debug/_meta/fields.yml
@@ -7,12 +7,12 @@
       type: keyword
       description: >
         Specifies what component of Icinga logged the message.
+
     - name: severity
-      type: keyword
-      description: >
-        Possible values are "debug", "notice", "information", "warning" or
-        "critical".
+      type: alias
+      path: log.level
+      migration: true
     - name: message
-      type: text
-      description: >
-        The logged message.
+      type: alias
+      path: message
+      migration: true
diff --git a/filebeat/module/icinga/debug/ingest/pipeline.json b/filebeat/module/icinga/debug/ingest/pipeline.json
@@ -4,7 +4,7 @@
     "grok": {
       "field": "message",
       "patterns":[
-        "\\[%{TIMESTAMP:icinga.debug.timestamp}\\] %{WORD:icinga.debug.severity}/%{WORD:icinga.debug.facility}: %{GREEDYMULTILINE:icinga.debug.message}"
+        "\\[%{TIMESTAMP:icinga.debug.timestamp}\\] %{WORD:log.level}/%{WORD:icinga.debug.facility}: %{GREEDYMULTILINE:message}"
       ],
       "ignore_missing": true,
       "pattern_definitions": {
@@ -13,11 +13,6 @@
       }
     }
   },
-  {
-    "remove": {
-      "field": "message"
-    }
-  },
   {
     "date": {
       "field": "icinga.debug.timestamp",

diff --git a/filebeat/module/icinga/debug/test/test.log-expected.json b/filebeat/module/icinga/debug/test/test.log-expected.json
@@ -4,29 +4,29 @@
         "event.dataset": "debug", 
         "event.module": "icinga", 
         "icinga.debug.facility": "GraphiteWriter", 
-        "icinga.debug.message": "Add to metric list:'icinga2.demo.services.procs.procs.perfdata.procs.warn 250 1491306189'.", 
-        "icinga.debug.severity": "debug", 
         "input.type": "log", 
-        "log.offset": 0
+        "log.level": "debug", 
+        "log.offset": 0, 
+        "message": "Add to metric list:'icinga2.demo.services.procs.procs.perfdata.procs.warn 250 1491306189'."
     }, 
     {
         "@timestamp": "2017-04-04T11:43:09.000Z", 
         "event.dataset": "debug", 
         "event.module": "icinga", 
         "icinga.debug.facility": "IdoMysqlConnection", 
-        "icinga.debug.message": "Query: UPDATE icinga_servicestatus SET acknowledgement_type = '0',  active_checks_enabled = '1',  check_command = 'mysql_health',  check_source = 'demo',  check_type = '0',  current_check_attempt = '1',  current_notification_number = '180',  current_state = '2',  endpoint_object_id = 242,  event_handler = '',  event_handler_enabled = '1',  execution_time = '0.355594',  flap_detection_enabled = '0',  has_been_checked = '1',  instance_id = 1,  is_flapping = '0',  is_reachable = '1',  last_check = FROM_UNIXTIME(1491306189),  last_hard_state = '2',  last_hard_state_change = FROM_UNIXTIME(1491290599),  last_notification = FROM_UNIXTIME(1491304989),  last_state_change = FROM_UNIXTIME(1491290599),  last_time_critical = FROM_UNIXTIME(1491306189),  last_time_unknown = FROM_UNIXTIME(1491290589),  latency = '0.001466',  long_output = '',  max_check_attempts = '5',  next_check = FROM_UNIXTIME(1491306198),  next_notification = FROM_UNIXTIME(1491306789),  normal_check_interval = '0.166667',  notifications_enabled = '1',  original_attributes = 'null',  output = 'CRITICAL - cannot connect to information_schema. Access denied for user \\'test1\\'@\\'blerims-mbp.int.netways.de\\' (using password: YES)',  passive_checks_enabled = '1',  percent_state_change = '0',  perfdata = '',  problem_has_been_acknowledged = '0',  process_performance_data = '1',  retry_check_interval = '0.166667',  scheduled_downtime_depth = '0',  service_object_id = 333,  should_be_scheduled = '1',  state_type = '1',  status_update_time = FROM_UNIXTIME(1491306189) WHERE service_object_id = 333", 
-        "icinga.debug.severity": "debug", 
         "input.type": "log", 
-        "log.offset": 141
+        "log.level": "debug", 
+        "log.offset": 141, 
+        "message": "Query: UPDATE icinga_servicestatus SET acknowledgement_type = '0',  active_checks_enabled = '1',  check_command = 'mysql_health',  check_source = 'demo',  check_type = '0',  current_check_attempt = '1',  current_notification_number = '180',  current_state = '2',  endpoint_object_id = 242,  event_handler = '',  event_handler_enabled = '1',  execution_time = '0.355594',  flap_detection_enabled = '0',  has_been_checked = '1',  instance_id = 1,  is_flapping = '0',  is_reachable = '1',  last_check = FROM_UNIXTIME(1491306189),  last_hard_state = '2',  last_hard_state_change = FROM_UNIXTIME(1491290599),  last_notification = FROM_UNIXTIME(1491304989),  last_state_change = FROM_UNIXTIME(1491290599),  last_time_critical = FROM_UNIXTIME(1491306189),  last_time_unknown = FROM_UNIXTIME(1491290589),  latency = '0.001466',  long_output = '',  max_check_attempts = '5',  next_check = FROM_UNIXTIME(1491306198),  next_notification = FROM_UNIXTIME(1491306789),  normal_check_interval = '0.166667',  notifications_enabled = '1',  original_attributes = 'null',  output = 'CRITICAL - cannot connect to information_schema. Access denied for user \\'test1\\'@\\'blerims-mbp.int.netways.de\\' (using password: YES)',  passive_checks_enabled = '1',  percent_state_change = '0',  perfdata = '',  problem_has_been_acknowledged = '0',  process_performance_data = '1',  retry_check_interval = '0.166667',  scheduled_downtime_depth = '0',  service_object_id = 333,  should_be_scheduled = '1',  state_type = '1',  status_update_time = FROM_UNIXTIME(1491306189) WHERE service_object_id = 333"
     }, 
     {
         "@timestamp": "2017-04-04T11:43:11.000Z", 
         "event.dataset": "debug", 
         "event.module": "icinga", 
         "icinga.debug.facility": "Process", 
-        "icinga.debug.message": "Running command '/usr/lib/nagios/plugins/check_ping' '-H' 'mysql.icinga.com' '-c' '5000,100%' '-w' '3000,80%': PID 8288", 
-        "icinga.debug.severity": "notice", 
         "input.type": "log", 
-        "log.offset": 1763
+        "log.level": "notice", 
+        "log.offset": 1763, 
+        "message": "Running command '/usr/lib/nagios/plugins/check_ping' '-H' 'mysql.icinga.com' '-c' '5000,100%' '-w' '3000,80%': PID 8288"
     }
 ]
diff --git a/filebeat/module/icinga/fields.go b/filebeat/module/icinga/fields.go
diff --git a/filebeat/module/icinga/main/_meta/fields.yml b/filebeat/module/icinga/main/_meta/fields.yml
@@ -7,12 +7,12 @@
       type: keyword
       description: >
         Specifies what component of Icinga logged the message.
+
     - name: severity
-      type: keyword
-      description: >
-        Possible values are "debug", "notice", "information", "warning" or
-        "critical".
+      type: alias
+      path: log.level
+      migration: true
     - name: message
-      type: text
-      description: >
-        The logged message.
+      type: alias
+      path: message
+      migration: true
diff --git a/filebeat/module/icinga/main/ingest/pipeline.json b/filebeat/module/icinga/main/ingest/pipeline.json
@@ -4,7 +4,7 @@
     "grok": {
       "field": "message",
       "patterns":[
-        "\\[%{TIMESTAMP:icinga.main.timestamp}\\] %{WORD:icinga.main.severity}/%{WORD:icinga.main.facility}: %{GREEDYMULTILINE:icinga.main.message}"
+        "\\[%{TIMESTAMP:icinga.main.timestamp}\\] %{WORD:log.level}/%{WORD:icinga.main.facility}: %{GREEDYMULTILINE:message}"
       ],
       "ignore_missing": true,
       "pattern_definitions": {
@@ -13,11 +13,6 @@
       }
     }
   },
-  {
-    "remove": {
-      "field": "message"
-    }
-  },
   {
     "date": {
       "field": "icinga.main.timestamp",

diff --git a/filebeat/module/icinga/main/test/test.log-expected.json b/filebeat/module/icinga/main/test/test.log-expected.json
@@ -4,32 +4,32 @@
         "event.dataset": "main", 
         "event.module": "icinga", 
         "icinga.main.facility": "Notification", 
-        "icinga.main.message": "Sending 'Recovery' notification 'demo!load!mail-icingaadmin for user 'on-call'", 
-        "icinga.main.severity": "information", 
         "input.type": "log", 
-        "log.offset": 0
+        "log.level": "information", 
+        "log.offset": 0, 
+        "message": "Sending 'Recovery' notification 'demo!load!mail-icingaadmin for user 'on-call'"
     }, 
     {
         "@timestamp": "2017-04-04T09:16:34.000Z", 
         "event.dataset": "main", 
         "event.module": "icinga", 
         "icinga.main.facility": "PluginNotificationTask", 
-        "icinga.main.message": "Notification command for object 'demo!load' (PID: 19401, arguments: '/etc/icinga2/scripts/mail-service-notification.sh') terminated with exit code 127, output: /etc/icinga2/scripts/mail-service-notification.sh: 20: /etc/icinga2/scripts/mail-service-notification.sh: mail: not found\n/usr/bin/printf: write error: Broken pipe\n", 
-        "icinga.main.severity": "warning", 
         "input.type": "log", 
         "log.flags": [
             "multiline"
         ], 
-        "log.offset": 133
+        "log.level": "warning", 
+        "log.offset": 133, 
+        "message": "Notification command for object 'demo!load' (PID: 19401, arguments: '/etc/icinga2/scripts/mail-service-notification.sh') terminated with exit code 127, output: /etc/icinga2/scripts/mail-service-notification.sh: 20: /etc/icinga2/scripts/mail-service-notification.sh: mail: not found\n/usr/bin/printf: write error: Broken pipe\n"
     }, 
     {
         "@timestamp": "2017-04-04T09:16:48.000Z", 
         "event.dataset": "main", 
         "event.module": "icinga", 
         "icinga.main.facility": "IdoMysqlConnection", 
-        "icinga.main.message": "Query queue items: 0, query rate: 5.38333/s (323/min 1610/5min 4778/15min);", 
-        "icinga.main.severity": "information", 
         "input.type": "log", 
-        "log.offset": 518
+        "log.level": "information", 
+        "log.offset": 518, 
+        "message": "Query queue items: 0, query rate: 5.38333/s (323/min 1610/5min 4778/15min);"
     }
 ]
diff --git a/filebeat/module/icinga/startup/_meta/fields.yml b/filebeat/module/icinga/startup/_meta/fields.yml
@@ -7,12 +7,12 @@
       type: keyword
       description: >
         Specifies what component of Icinga logged the message.
+
     - name: severity
-      type: keyword
-      description: >
-        Possible values are "debug", "notice", "information", "warning" or
-        "critical".
+      type: alias
+      path: log.level
+      migration: true.
     - name: message
-      type: text
-      description: >
-        The logged message.
+      type: alias
+      path: message
+      migration: true