From bb0151fcea0ee0be7a11291169796477f5169807 Mon Sep 17 00:00:00 2001 From: "Lee E. Hinman" Date: Mon, 11 May 2020 13:40:06 -0500 Subject: [PATCH 1/2] Improve ECS categorization field mappings in coredns module. - event.kind - event.category - event.type - event.outcome - network.protocol - related.ip Closes #16159 --- CHANGELOG.next.asciidoc | 1 + .../coredns/log/ingest/pipeline-entry.yml | 28 +++++++++++++++ .../log/test/coredns-json.log-expected.json | 36 +++++++++++++++++++ .../log/test/coredns.log-expected.json | 24 +++++++++++++ 4 files changed, 89 insertions(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index dc03b3f45e6..704bfcc2c6f 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -311,6 +311,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Improve ECS categorization field mappings in system module. {issue}16031[16031] {pull}18065[18065] - Change the `json.*` input settings implementation to merge parsed json objects with existing objects in the event instead of fully replacing them. {pull}17958[17958] - Improve ECS categorization field mappings in osquery module. {issue}16176[16176] {pull}17881[17881] +- Improve ECS categorization field mappings in coredns module. {issue}16159[16159] {pull}18424[18424] *Heartbeat* diff --git a/x-pack/filebeat/module/coredns/log/ingest/pipeline-entry.yml b/x-pack/filebeat/module/coredns/log/ingest/pipeline-entry.yml index 8f55838b21e..44733495399 100644 --- a/x-pack/filebeat/module/coredns/log/ingest/pipeline-entry.yml +++ b/x-pack/filebeat/module/coredns/log/ingest/pipeline-entry.yml @@ -107,6 +107,34 @@ processors: if (q.charAt(end) == (char) '.') { ctx.dns.question.name = q.substring(0, end); } + - set: + field: event.kind + value: event + - append: + field: event.category + value: network + - append: + field: event.type + value: protocol + - set: + field: event.outcome + value: success + if: "ctx?.dns?.response_code == 'NOERROR'" + - set: + field: event.outcome + value: failure + if: "ctx?.dns?.response_code != null && ctx.dns.response_code != 'NOERROR'" + - set: + field: network.protocol + value: dns + - append: + field: related.ip + value: "{{source.ip}}" + if: "ctx?.source?.ip != null" + - append: + field: related.ip + value: "{{destination.ip}}" + if: "ctx?.destination?.ip != null" on_failure: - set: field: error.message diff --git a/x-pack/filebeat/module/coredns/log/test/coredns-json.log-expected.json b/x-pack/filebeat/module/coredns/log/test/coredns-json.log-expected.json index 637f8cb0b4d..23515a993ce 100644 --- a/x-pack/filebeat/module/coredns/log/test/coredns-json.log-expected.json +++ b/x-pack/filebeat/module/coredns/log/test/coredns-json.log-expected.json @@ -24,10 +24,18 @@ "dns.question.name": "httpbin.org.cluster.local", "dns.question.type": "A", "dns.response_code": "NXDOMAIN", + "event.category": [ + "network" + ], "event.dataset": "coredns.log", "event.duration": 102078, + "event.kind": "event", "event.module": "coredns", "event.original": "{\"message\":\"2019-02-12T00:27:28.903Z [INFO] 172.17.0.4:36413 - 21583 \\\"A IN httpbin.org.cluster.local. udp 43 false 512\\\" NXDOMAIN qr,rd,ra 136 0.000102078s\", \"stream\": \"stdout\", \"time\": \"2019-02-12T00:27:28.903433597Z\", \"kubernetes\": { \"container\": { \"name\": \"coredns\" }, \"node\": { \"name\": \"minikube\" }, \"pod\": { \"uid\": \"d57d545e-2a9d-11e9-995f-08002730e0dc\", \"name\": \"coredns-86c58d9df4-jwhsg\" }, \"namespace\": \"kube-system\", \"replicaset\": { \"name\": \"coredns-86c58d9df4\" }, \"labels\": { \"pod-template-hash\": \"86c58d9df4\", \"k8s-app\": \"kube-dns\" } } }", + "event.outcome": "failure", + "event.type": [ + "protocol" + ], "fileset.name": "log", "input.type": "log", "kubernetes.container.name": "coredns", @@ -41,7 +49,11 @@ "log.level": "INFO", "log.offset": 0, "message": "2019-02-12T00:27:28.903Z [INFO] 172.17.0.4:36413 - 21583 \"A IN httpbin.org.cluster.local. udp 43 false 512\" NXDOMAIN qr,rd,ra 136 0.000102078s", + "network.protocol": "dns", "network.transport": "udp", + "related.ip": [ + "172.17.0.4" + ], "service.type": "coredns", "source.address": "172.17.0.4", "source.ip": "172.17.0.4", @@ -75,10 +87,18 @@ "dns.question.name": "httpbin.org", "dns.question.type": "A", "dns.response_code": "NOERROR", + "event.category": [ + "network" + ], "event.dataset": "coredns.log", "event.duration": 82083, + "event.kind": "event", "event.module": "coredns", "event.original": "{\"message\":\"2019-03-19T02:57:23.213Z [INFO] 172.17.0.9:37723 - 6966 \\\"A IN httpbin.org. udp 29 false 512\\\" NOERROR qr,rd,ra 83 0.000082083s\\n\",\"stream\":\"stdout\",\"time\":\"2019-03-19T02:57:23.214583742Z\", \"kubernetes\": { \"container\": { \"name\": \"coredns\" }, \"node\": { \"name\": \"minikube\" }, \"pod\": { \"uid\": \"d57d545e-2a9d-11e9-995f-08002730e0dc\", \"name\": \"coredns-86c58d9df4-jwhsg\" }, \"namespace\": \"kube-system\", \"replicaset\": { \"name\": \"coredns-86c58d9df4\" }, \"labels\": { \"pod-template-hash\": \"86c58d9df4\", \"k8s-app\": \"kube-dns\" } } }", + "event.outcome": "success", + "event.type": [ + "protocol" + ], "fileset.name": "log", "input.type": "log", "kubernetes.container.name": "coredns", @@ -92,7 +112,11 @@ "log.level": "INFO", "log.offset": 550, "message": "2019-03-19T02:57:23.213Z [INFO] 172.17.0.9:37723 - 6966 \"A IN httpbin.org. udp 29 false 512\" NOERROR qr,rd,ra 83 0.000082083s\n", + "network.protocol": "dns", "network.transport": "udp", + "related.ip": [ + "172.17.0.9" + ], "service.type": "coredns", "source.address": "172.17.0.9", "source.ip": "172.17.0.9", @@ -126,10 +150,18 @@ "dns.question.name": "czbaoyu.com", "dns.question.type": "AAAA", "dns.response_code": "NOERROR", + "event.category": [ + "network" + ], "event.dataset": "coredns.log", "event.duration": 62860, + "event.kind": "event", "event.module": "coredns", "event.original": "{\"message\":\"2019-03-11T07:16:34.013Z [INFO] [::1]:37915 - 62762 \\\"AAAA IN czbaoyu.com. udp 29 false 512\\\" NOERROR qr,rd,ra 100 0.00006286s\\n\",\"stream\":\"stdout\",\"time\":\"2019-03-11T07:16:34.013970788Z\", \"kubernetes\": { \"container\": { \"name\": \"coredns\" }, \"node\": { \"name\": \"minikube\" }, \"pod\": { \"uid\": \"d57d545e-2a9d-11e9-995f-08002730e0dc\", \"name\": \"coredns-86c58d9df4-jwhsg\" }, \"namespace\": \"kube-system\", \"replicaset\": { \"name\": \"coredns-86c58d9df4\" }, \"labels\": { \"pod-template-hash\": \"86c58d9df4\", \"k8s-app\": \"kube-dns\" } } }", + "event.outcome": "success", + "event.type": [ + "protocol" + ], "fileset.name": "log", "input.type": "log", "kubernetes.container.name": "coredns", @@ -143,7 +175,11 @@ "log.level": "INFO", "log.offset": 1081, "message": "2019-03-11T07:16:34.013Z [INFO] [::1]:37915 - 62762 \"AAAA IN czbaoyu.com. udp 29 false 512\" NOERROR qr,rd,ra 100 0.00006286s\n", + "network.protocol": "dns", "network.transport": "udp", + "related.ip": [ + "::1" + ], "service.type": "coredns", "source.address": "::1", "source.ip": "::1", diff --git a/x-pack/filebeat/module/coredns/log/test/coredns.log-expected.json b/x-pack/filebeat/module/coredns/log/test/coredns.log-expected.json index ba3191a9e17..2d573602c17 100644 --- a/x-pack/filebeat/module/coredns/log/test/coredns.log-expected.json +++ b/x-pack/filebeat/module/coredns/log/test/coredns.log-expected.json @@ -24,15 +24,27 @@ "dns.question.name": "httpbin.org.cluster.local", "dns.question.type": "A", "dns.response_code": "NXDOMAIN", + "event.category": [ + "network" + ], "event.dataset": "coredns.log", "event.duration": 102078, + "event.kind": "event", "event.module": "coredns", + "event.outcome": "failure", + "event.type": [ + "protocol" + ], "fileset.name": "log", "input.type": "log", "log.level": "INFO", "log.offset": 0, "message": "2019-03-06T08:55:28.903Z [INFO] 172.17.0.4:36413 - 21583 \"A IN httpbin.org.cluster.local. udp 43 false 512\" NXDOMAIN qr,rd,ra 136 0.000102078s", + "network.protocol": "dns", "network.transport": "udp", + "related.ip": [ + "172.17.0.4" + ], "service.type": "coredns", "source.address": "172.17.0.4", "source.ip": "172.17.0.4", @@ -66,15 +78,27 @@ "dns.question.name": "www.yahoo.com", "dns.question.type": "A", "dns.response_code": "NOERROR", + "event.category": [ + "network" + ], "event.dataset": "coredns.log", "event.duration": 20948545, + "event.kind": "event", "event.module": "coredns", + "event.outcome": "success", + "event.type": [ + "protocol" + ], "fileset.name": "log", "input.type": "log", "log.level": "INFO", "log.offset": 143, "message": "2019-03-18T22:13:36.289-07:00 [INFO] [::1]:57413 - 14639 \"A IN www.yahoo.com. udp 42 false 4096\" NOERROR qr,rd,ra 188 0.020948545s", + "network.protocol": "dns", "network.transport": "udp", + "related.ip": [ + "::1" + ], "service.type": "coredns", "source.address": "::1", "source.ip": "::1", From 16ec63f95e999d9add4669da1feddbd84445648e Mon Sep 17 00:00:00 2001 From: "Lee E. Hinman" Date: Mon, 11 May 2020 14:26:12 -0500 Subject: [PATCH 2/2] Explicitly set ECS version --- x-pack/filebeat/module/coredns/log/config/coredns.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/x-pack/filebeat/module/coredns/log/config/coredns.yml b/x-pack/filebeat/module/coredns/log/config/coredns.yml index c085c9e3aab..b2f0ebe4519 100644 --- a/x-pack/filebeat/module/coredns/log/config/coredns.yml +++ b/x-pack/filebeat/module/coredns/log/config/coredns.yml @@ -5,3 +5,7 @@ paths: {{ end }} tags: {{.tags}} processors: + - add_fields: + target: '' + fields: + ecs.version: 1.5.0