Convert Filebeat icinga.* to ECS

CHANGELOG.asciidoc

@@ -180,6 +180,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha1...v7.0.0-alpha2[Check the
 - Fix saved objects in filebeat haproxy dashboard. {pull}9417[9417]
 - Use `log.source.address` instead of `log.source.ip` for network input sources. {pull}9487[9487]
 - Rename many `redis.log.*` fields to map to ECS. {pull}9315[9315]
+- Rename many `icinga.*` fields to map to ECS. {pull}9294[9294]
 
 *Metricbeat*
 

dev-tools/ecs-migration.yml

@@ -97,6 +97,8 @@
  alias6: true
 
 
+# Filebeat modules
+
 # Suricata module
 
 - from: source_ecs.ip
@@ -131,6 +133,8 @@
  to: source.geo.region_iso_code
  alias: true
 
+## System module
+
 - from: system.syslog.hostname
  to: host.hostname
  alias: true
@@ -188,8 +192,6 @@
  to: source.geo.*
  alias: true
 
-# Filebeat modules
-
 ## Apache module
 
 - from: apache2.access.user_name
@@ -254,7 +256,7 @@
  to: process.thread.id
  alias: true
 
-# IIS module
+## IIS module
 
 - from: iis.access.server_ip
  to: destination.ip
@@ -312,8 +314,6 @@
  to: source.geo.region_iso_code
  alias: true
 
-# Note: `http` is not officially in ECS yet
-
 - from: iis.access.method
  to: http.request.method
  alias: true
@@ -326,7 +326,8 @@
  to: http.request.referrer
  alias: true
 
-# HAProxy module
+## HAProxy module
+
 - from: haproxy.client.port
  to: source.port
  alias: true
@@ -375,6 +376,8 @@
  to: network.forwarded_ip
  alias: true
 
+## NGINX module
+
 - from: nginx.access.user_name
  to: user.name
  alias: true
@@ -387,8 +390,6 @@
  to: user_agent.original
  alias: true
 
-# Note: `http` is not officially in ECS yet
-
 - from: nginx.access.response_code
  to: http.response.status_code
  alias: true
@@ -447,12 +448,39 @@
  to: message
  alias: true
 
-# From Auditbeat's auditd module.
+## Icinga module
+
+- from: icinga.debug.message
+ to: message
+ alias: true
+- from: icinga.debug.severity
+ to: log.level
+ alias: true
+
+- from: icinga.main.message
+ to: message
+ alias: true
+- from: icinga.main.severity
+ to: log.level
+ alias: true
+
+- from: icinga.startup.message
+ to: message
+ alias: true
+- from: icinga.startup.severity
+ to: log.level
+ alias: true
+
+# Auditbeat
+
+## From Auditbeat's auditd module.
 - from: source.hostname
  to: source.domain
  alias: true
 
-# Metricbeat base fields
+# Metricbeat
+
+## Metricbeat base fields
 - from: metricset.name
  to: event.dataset
  alias: false
@@ -477,4 +505,3 @@
  to: event.dataset
  alias: false
  comment: No alias mapping as field did not always exist
-

filebeat/docs/fields.asciidoc

@@ -4813,20 +4813,18 @@ Specifies what component of Icinga logged the message.
 *`icinga.debug.severity`*::
 +
 --
-type: keyword
-
-Possible values are "debug", "notice", "information", "warning" or "critical".
+type: alias
 
+alias to: log.level
 
 --
 
 *`icinga.debug.message`*::
 +
 --
-type: text
-
-The logged message.
+type: alias
 
+alias to: message
 
 --
 
@@ -4850,20 +4848,18 @@ Specifies what component of Icinga logged the message.
 *`icinga.main.severity`*::
 +
 --
-type: keyword
-
-Possible values are "debug", "notice", "information", "warning" or "critical".
+type: alias
 
+alias to: log.level
 
 --
 
 *`icinga.main.message`*::
 +
 --
-type: text
-
-The logged message.
+type: alias
 
+alias to: message
 
 --
 
@@ -4887,20 +4883,18 @@ Specifies what component of Icinga logged the message.
 *`icinga.startup.severity`*::
 +
 --
-type: keyword
-
-Possible values are "debug", "notice", "information", "warning" or "critical".
+type: alias
 
+alias to: log.level
 
 --
 
 *`icinga.startup.message`*::
 +
 --
-type: text
-
-The logged message.
+type: alias
 
+alias to: message
 
 --
 

filebeat/module/icinga/debug/_meta/fields.yml

@@ -7,12 +7,12 @@
  type: keyword
  description: >
  Specifies what component of Icinga logged the message.
+
  - name: severity
- type: keyword
- description: >
- Possible values are "debug", "notice", "information", "warning" or
- "critical".
+ type: alias
+ path: log.level
+ migration: true
  - name: message
- type: text
- description: >
-  The logged message.
+ type: alias
+ path: message
+ migration: true

filebeat/module/icinga/debug/ingest/pipeline.json

@@ -4,7 +4,7 @@
  "grok": {
  "field": "message",
  "patterns":[
- "\\[%{TIMESTAMP:icinga.debug.timestamp}\\] %{WORD:icinga.debug.severity}/%{WORD:icinga.debug.facility}: %{GREEDYMULTILINE:icinga.debug.message}"
+ "\\[%{TIMESTAMP:icinga.debug.timestamp}\\] %{WORD:log.level}/%{WORD:icinga.debug.facility}: %{GREEDYMULTILINE:message}"
  ],
  "ignore_missing": true,
  "pattern_definitions": {
@@ -13,11 +13,6 @@
  }
  }
  },
- {
- "remove": {
- "field": "message"
- }
- },
  {
  "date": {
  "field": "icinga.debug.timestamp",

filebeat/module/icinga/debug/test/test.log-expected.json

@@ -4,29 +4,29 @@
  "event.dataset": "debug", 
  "event.module": "icinga", 
  "icinga.debug.facility": "GraphiteWriter", 
- "icinga.debug.message": "Add to metric list:'icinga2.demo.services.procs.procs.perfdata.procs.warn 250 1491306189'.", 
- "icinga.debug.severity": "debug", 
  "input.type": "log", 
- "log.offset": 0
+ "log.level": "debug", 
+ "log.offset": 0, 
+ "message": "Add to metric list:'icinga2.demo.services.procs.procs.perfdata.procs.warn 250 1491306189'."
  }, 
  {
  "@timestamp": "2017-04-04T11:43:09.000Z", 
  "event.dataset": "debug", 
  "event.module": "icinga", 
  "icinga.debug.facility": "IdoMysqlConnection", 
- "icinga.debug.message": "Query: UPDATE icinga_servicestatus SET acknowledgement_type = '0', active_checks_enabled = '1', check_command = 'mysql_health', check_source = 'demo', check_type = '0', current_check_attempt = '1', current_notification_number = '180', current_state = '2', endpoint_object_id = 242, event_handler = '', event_handler_enabled = '1', execution_time = '0.355594', flap_detection_enabled = '0', has_been_checked = '1', instance_id = 1, is_flapping = '0', is_reachable = '1', last_check = FROM_UNIXTIME(1491306189), last_hard_state = '2', last_hard_state_change = FROM_UNIXTIME(1491290599), last_notification = FROM_UNIXTIME(1491304989), last_state_change = FROM_UNIXTIME(1491290599), last_time_critical = FROM_UNIXTIME(1491306189), last_time_unknown = FROM_UNIXTIME(1491290589), latency = '0.001466', long_output = '', max_check_attempts = '5', next_check = FROM_UNIXTIME(1491306198), next_notification = FROM_UNIXTIME(1491306789), normal_check_interval = '0.166667', notifications_enabled = '1', original_attributes = 'null', output = 'CRITICAL - cannot connect to information_schema. Access denied for user \\'test1\\'@\\'blerims-mbp.int.netways.de\\' (using password: YES)', passive_checks_enabled = '1', percent_state_change = '0', perfdata = '', problem_has_been_acknowledged = '0', process_performance_data = '1', retry_check_interval = '0.166667', scheduled_downtime_depth = '0', service_object_id = 333, should_be_scheduled = '1', state_type = '1', status_update_time = FROM_UNIXTIME(1491306189) WHERE service_object_id = 333", 
- "icinga.debug.severity": "debug", 
  "input.type": "log", 
- "log.offset": 141
+ "log.level": "debug", 
+ "log.offset": 141, 
+ "message": "Query: UPDATE icinga_servicestatus SET acknowledgement_type = '0', active_checks_enabled = '1', check_command = 'mysql_health', check_source = 'demo', check_type = '0', current_check_attempt = '1', current_notification_number = '180', current_state = '2', endpoint_object_id = 242, event_handler = '', event_handler_enabled = '1', execution_time = '0.355594', flap_detection_enabled = '0', has_been_checked = '1', instance_id = 1, is_flapping = '0', is_reachable = '1', last_check = FROM_UNIXTIME(1491306189), last_hard_state = '2', last_hard_state_change = FROM_UNIXTIME(1491290599), last_notification = FROM_UNIXTIME(1491304989), last_state_change = FROM_UNIXTIME(1491290599), last_time_critical = FROM_UNIXTIME(1491306189), last_time_unknown = FROM_UNIXTIME(1491290589), latency = '0.001466', long_output = '', max_check_attempts = '5', next_check = FROM_UNIXTIME(1491306198), next_notification = FROM_UNIXTIME(1491306789), normal_check_interval = '0.166667', notifications_enabled = '1', original_attributes = 'null', output = 'CRITICAL - cannot connect to information_schema. Access denied for user \\'test1\\'@\\'blerims-mbp.int.netways.de\\' (using password: YES)', passive_checks_enabled = '1', percent_state_change = '0', perfdata = '', problem_has_been_acknowledged = '0', process_performance_data = '1', retry_check_interval = '0.166667', scheduled_downtime_depth = '0', service_object_id = 333, should_be_scheduled = '1', state_type = '1', status_update_time = FROM_UNIXTIME(1491306189) WHERE service_object_id = 333"
  }, 
  {
  "@timestamp": "2017-04-04T11:43:11.000Z", 
  "event.dataset": "debug", 
  "event.module": "icinga", 
  "icinga.debug.facility": "Process", 
- "icinga.debug.message": "Running command '/usr/lib/nagios/plugins/check_ping' '-H' 'mysql.icinga.com' '-c' '5000,100%' '-w' '3000,80%': PID 8288", 
- "icinga.debug.severity": "notice", 
  "input.type": "log", 
- "log.offset": 1763
+ "log.level": "notice", 
+ "log.offset": 1763, 
+ "message": "Running command '/usr/lib/nagios/plugins/check_ping' '-H' 'mysql.icinga.com' '-c' '5000,100%' '-w' '3000,80%': PID 8288"
  }
 ]

filebeat/module/icinga/main/_meta/fields.yml

@@ -7,12 +7,12 @@
  type: keyword
  description: >
  Specifies what component of Icinga logged the message.
+
  - name: severity
- type: keyword
- description: >
- Possible values are "debug", "notice", "information", "warning" or
- "critical".
+ type: alias
+ path: log.level
+ migration: true
  - name: message
- type: text
- description: >
-  The logged message.
+ type: alias
+ path: message
+ migration: true

filebeat/module/icinga/main/ingest/pipeline.json

@@ -4,7 +4,7 @@
  "grok": {
  "field": "message",
  "patterns":[
- "\\[%{TIMESTAMP:icinga.main.timestamp}\\] %{WORD:icinga.main.severity}/%{WORD:icinga.main.facility}: %{GREEDYMULTILINE:icinga.main.message}"
+ "\\[%{TIMESTAMP:icinga.main.timestamp}\\] %{WORD:log.level}/%{WORD:icinga.main.facility}: %{GREEDYMULTILINE:message}"
  ],
  "ignore_missing": true,
  "pattern_definitions": {
@@ -13,11 +13,6 @@
  }
  }
  },
- {
- "remove": {
- "field": "message"
- }
- },
  {
  "date": {
  "field": "icinga.main.timestamp",

filebeat/module/icinga/main/test/test.log-expected.json

@@ -4,32 +4,32 @@
  "event.dataset": "main", 
  "event.module": "icinga", 
  "icinga.main.facility": "Notification", 
- "icinga.main.message": "Sending 'Recovery' notification 'demo!load!mail-icingaadmin for user 'on-call'", 
- "icinga.main.severity": "information", 
  "input.type": "log", 
- "log.offset": 0
+ "log.level": "information", 
+ "log.offset": 0, 
+ "message": "Sending 'Recovery' notification 'demo!load!mail-icingaadmin for user 'on-call'"
  }, 
  {
  "@timestamp": "2017-04-04T09:16:34.000Z", 
  "event.dataset": "main", 
  "event.module": "icinga", 
  "icinga.main.facility": "PluginNotificationTask", 
- "icinga.main.message": "Notification command for object 'demo!load' (PID: 19401, arguments: '/etc/icinga2/scripts/mail-service-notification.sh') terminated with exit code 127, output: /etc/icinga2/scripts/mail-service-notification.sh: 20: /etc/icinga2/scripts/mail-service-notification.sh: mail: not found\n/usr/bin/printf: write error: Broken pipe\n", 
- "icinga.main.severity": "warning", 
  "input.type": "log", 
  "log.flags": [
  "multiline"
  ], 
- "log.offset": 133
+ "log.level": "warning", 
+ "log.offset": 133, 
+ "message": "Notification command for object 'demo!load' (PID: 19401, arguments: '/etc/icinga2/scripts/mail-service-notification.sh') terminated with exit code 127, output: /etc/icinga2/scripts/mail-service-notification.sh: 20: /etc/icinga2/scripts/mail-service-notification.sh: mail: not found\n/usr/bin/printf: write error: Broken pipe\n"
  }, 
  {
  "@timestamp": "2017-04-04T09:16:48.000Z", 
  "event.dataset": "main", 
  "event.module": "icinga", 
  "icinga.main.facility": "IdoMysqlConnection", 
- "icinga.main.message": "Query queue items: 0, query rate: 5.38333/s (323/min 1610/5min 4778/15min);", 
- "icinga.main.severity": "information", 
  "input.type": "log", 
- "log.offset": 518
+ "log.level": "information", 
+ "log.offset": 518, 
+ "message": "Query queue items: 0, query rate: 5.38333/s (323/min 1610/5min 4778/15min);"
  }
 ]

filebeat/module/icinga/startup/_meta/fields.yml

@@ -7,12 +7,12 @@
  type: keyword
  description: >
  Specifies what component of Icinga logged the message.
+
  - name: severity
- type: keyword
- description: >
- Possible values are "debug", "notice", "information", "warning" or
- "critical".
+ type: alias
+ path: log.level
+ migration: true.
  - name: message
- type: text
- description: >
-  The logged message.
+ type: alias
+ path: message
+ migration: true