From 39cbece2c6f37e9e4291cf5a59bf59fe0e973ef4 Mon Sep 17 00:00:00 2001 From: "Lee E. Hinman" Date: Mon, 6 Apr 2020 14:48:46 -0500 Subject: [PATCH] Improve ECS categorization field mappings for nats module - event.kind - event.type - related.ip Closes #16173 --- CHANGELOG.next.asciidoc | 3 +- filebeat/module/nats/log/ingest/pipeline.yml | 14 +++ .../nats/log/test/test.log-expected.json | 114 ++++++++++++++++++ 3 files changed, 130 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index bae89e6716df..13eb635ee382 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -239,7 +239,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Improve ECS categorization field mappings for mssql module. {issue}16171[16171] {pull}17376[17376] - Added access_key_id, secret_access_key and session_token into aws module config. {pull}17456[17456] - Add dashboard for Google Cloud Audit and AWS CloudTrail. {pull}17379[17379] -- Improve ECS categorization field mappings for mysql module. {issue}16172[16172] {pull}XXXXX[XXXXX] +- Improve ECS categorization field mappings for mysql module. {issue}16172[16172] {pull}17491[17491] +- Improve ECS categorization field mappings for nats module. {issue}16173[16173] {pull}17550[17550] *Heartbeat* diff --git a/filebeat/module/nats/log/ingest/pipeline.yml b/filebeat/module/nats/log/ingest/pipeline.yml index 58558638ad91..53c4f774b5ea 100644 --- a/filebeat/module/nats/log/ingest/pipeline.yml +++ b/filebeat/module/nats/log/ingest/pipeline.yml @@ -161,6 +161,20 @@ processors: - yyyy/MM/dd HH:mm:ss.SSSSSS - remove: field: nats.log.timestamp +- set: + field: event.kind + value: event +- append: + field: event.type + value: info +- append: + field: event.type + value: error + if: "ctx?.log?.level != null && (ctx.log.level == 'error' || ctx.log.level == 'fatal')" +- append: + field: related.ip + value: "{{client.ip}}" + if: "ctx?.client?.ip != null" on_failure: - set: field: error.message diff --git a/filebeat/module/nats/log/test/test.log-expected.json b/filebeat/module/nats/log/test/test.log-expected.json index 34dd0c2f17f4..efa4fba75090 100644 --- a/filebeat/module/nats/log/test/test.log-expected.json +++ b/filebeat/module/nats/log/test/test.log-expected.json @@ -2,7 +2,11 @@ { "@timestamp": "2019-02-06T07:19:40.624Z", "event.dataset": "nats.log", + "event.kind": "event", "event.module": "nats", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "info", @@ -14,7 +18,11 @@ { "@timestamp": "2019-02-06T07:19:40.624Z", "event.dataset": "nats.log", + "event.kind": "event", "event.module": "nats", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "info", @@ -26,7 +34,11 @@ { "@timestamp": "2019-02-06T07:19:40.624Z", "event.dataset": "nats.log", + "event.kind": "event", "event.module": "nats", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "info", @@ -38,7 +50,11 @@ { "@timestamp": "2019-02-06T07:19:40.624Z", "event.dataset": "nats.log", + "event.kind": "event", "event.module": "nats", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "info", @@ -52,7 +68,11 @@ "client.ip": "172.18.0.1", "client.port": "38630", "event.dataset": "nats.log", + "event.kind": "event", "event.module": "nats", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "debug", @@ -60,6 +80,9 @@ "message": "Client connection created", "nats.log.client.id": "1", "process.pid": "1", + "related.ip": [ + "172.18.0.1" + ], "service.type": "nats" }, { @@ -67,7 +90,11 @@ "client.ip": "172.18.0.1", "client.port": "38630", "event.dataset": "nats.log", + "event.kind": "event", "event.module": "nats", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "trace", @@ -77,6 +104,9 @@ "nats.log.msg.type": "connection", "network.direction": "outbound", "process.pid": "1", + "related.ip": [ + "172.18.0.1" + ], "service.type": "nats" }, { @@ -84,7 +114,11 @@ "client.ip": "172.18.0.1", "client.port": "38630", "event.dataset": "nats.log", + "event.kind": "event", "event.module": "nats", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "trace", @@ -95,6 +129,9 @@ "nats.log.msg.type": "subscribe", "network.direction": "outbound", "process.pid": "1", + "related.ip": [ + "172.18.0.1" + ], "service.type": "nats" }, { @@ -102,7 +139,11 @@ "client.ip": "172.18.0.1", "client.port": "38630", "event.dataset": "nats.log", + "event.kind": "event", "event.module": "nats", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "trace", @@ -111,6 +152,9 @@ "nats.log.msg.type": "ping", "network.direction": "outbound", "process.pid": "1", + "related.ip": [ + "172.18.0.1" + ], "service.type": "nats" }, { @@ -118,7 +162,11 @@ "client.ip": "172.18.0.1", "client.port": "38630", "event.dataset": "nats.log", + "event.kind": "event", "event.module": "nats", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "trace", @@ -127,6 +175,9 @@ "nats.log.msg.type": "pong", "network.direction": "inbound", "process.pid": "1", + "related.ip": [ + "172.18.0.1" + ], "service.type": "nats" }, { @@ -134,7 +185,11 @@ "client.ip": "50.39.246.116", "client.port": "62388", "event.dataset": "nats.log", + "event.kind": "event", "event.module": "nats", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "trace", @@ -146,6 +201,9 @@ "nats.log.msg.type": "publish", "network.direction": "outbound", "process.pid": "1", + "related.ip": [ + "50.39.246.116" + ], "service.type": "nats" }, { @@ -153,7 +211,11 @@ "client.ip": "50.39.246.116", "client.port": "62388", "event.dataset": "nats.log", + "event.kind": "event", "event.module": "nats", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "trace", @@ -162,6 +224,9 @@ "nats.log.msg.type": "payload", "network.direction": "outbound", "process.pid": "1", + "related.ip": [ + "50.39.246.116" + ], "service.type": "nats" }, { @@ -169,7 +234,11 @@ "client.ip": "192.168.176.11", "client.port": "36262", "event.dataset": "nats.log", + "event.kind": "event", "event.module": "nats", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "trace", @@ -182,6 +251,9 @@ "nats.log.msg.type": "message", "network.direction": "inbound", "process.pid": "1", + "related.ip": [ + "192.168.176.11" + ], "service.type": "nats" }, { @@ -189,7 +261,11 @@ "client.ip": "192.168.176.11", "client.port": "36262", "event.dataset": "nats.log", + "event.kind": "event", "event.module": "nats", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "trace", @@ -200,6 +276,9 @@ "nats.log.msg.type": "publish", "network.direction": "outbound", "process.pid": "1", + "related.ip": [ + "192.168.176.11" + ], "service.type": "nats" }, { @@ -207,7 +286,11 @@ "client.ip": "192.168.176.11", "client.port": "36262", "event.dataset": "nats.log", + "event.kind": "event", "event.module": "nats", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "trace", @@ -216,6 +299,9 @@ "nats.log.msg.type": "payload", "network.direction": "outbound", "process.pid": "1", + "related.ip": [ + "192.168.176.11" + ], "service.type": "nats" }, { @@ -223,7 +309,11 @@ "client.ip": "50.39.246.116", "client.port": "62388", "event.dataset": "nats.log", + "event.kind": "event", "event.module": "nats", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "trace", @@ -235,6 +325,9 @@ "nats.log.msg.type": "message", "network.direction": "inbound", "process.pid": "1", + "related.ip": [ + "50.39.246.116" + ], "service.type": "nats" }, { @@ -242,7 +335,11 @@ "client.ip": "50.39.246.116", "client.port": "62388", "event.dataset": "nats.log", + "event.kind": "event", "event.module": "nats", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "trace", @@ -253,6 +350,9 @@ "nats.log.msg.type": "publish", "network.direction": "outbound", "process.pid": "1", + "related.ip": [ + "50.39.246.116" + ], "service.type": "nats" }, { @@ -260,7 +360,11 @@ "client.ip": "192.168.176.11", "client.port": "36262", "event.dataset": "nats.log", + "event.kind": "event", "event.module": "nats", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "trace", @@ -272,6 +376,9 @@ "nats.log.msg.type": "message", "network.direction": "inbound", "process.pid": "1", + "related.ip": [ + "192.168.176.11" + ], "service.type": "nats" }, { @@ -279,7 +386,11 @@ "client.ip": "172.18.0.1", "client.port": "38630", "event.dataset": "nats.log", + "event.kind": "event", "event.module": "nats", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "trace", @@ -288,6 +399,9 @@ "nats.log.msg.type": "acknowledge", "network.direction": "inbound", "process.pid": "1", + "related.ip": [ + "172.18.0.1" + ], "service.type": "nats" } ] \ No newline at end of file