From 3c7c65053d922f010b7d7c7861a4faaf02c3ea81 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Mon, 7 Jan 2019 20:00:14 +0100 Subject: [PATCH 1/2] Handle IPv6 zone id in IIS filebeat ingest pipeline (#9869) IIS logs can include zone ids when using IPv6, this is correctly parsed but geoip processor doesn't accept these addresses. Create a temporary field without the zone id to be used by geoip processor. (cherry picked from commit d59ae8ce7ae21d84d49b92c0c9905fd1184b5c3b) --- CHANGELOG.next.asciidoc | 2 ++ filebeat/module/iis/error/ingest/default.json | 16 +++++++++++++++- filebeat/module/iis/error/test/ipv6_zone_id.log | 5 +++++ .../error/test/ipv6_zone_id.log-expected.json | 16 ++++++++++++++++ 4 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 filebeat/module/iis/error/test/ipv6_zone_id.log create mode 100644 filebeat/module/iis/error/test/ipv6_zone_id.log-expected.json diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 276040c04fe..0f684a47b58 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -56,6 +56,7 @@ https://github.com/elastic/beats/compare/v6.6.0...6.x[Check the HEAD diff] *Auditbeat* *Filebeat* + - Correctly parse `December` or `Dec` in the Syslog input. {pull}9349[9349] - Fix improperly set config for CRI Flag in Docker Input {pull}8899[8899] - Just enabling the `elasticsearch` fileset and starting Filebeat no longer causes an error. {pull}8891[8891] @@ -66,6 +67,7 @@ https://github.com/elastic/beats/compare/v6.6.0...6.x[Check the HEAD diff] - Stop runners disabled by hints after previously being started. {pull}9305[9305] - Fix saved objects in filebeat haproxy dashboard. {pull}9417[9417] - Fixed a memory leak when harvesters are closed. {pull}7820[7820] +- Support IPv6 addresses with zone id in IIS ingest pipeline. {issue}9836[9836] {pull}9869[9869] *Heartbeat* diff --git a/filebeat/module/iis/error/ingest/default.json b/filebeat/module/iis/error/ingest/default.json index 632e31d717f..af3c470afe7 100644 --- a/filebeat/module/iis/error/ingest/default.json +++ b/filebeat/module/iis/error/ingest/default.json @@ -28,10 +28,24 @@ "field": "iis.error.time" } }, { - "geoip": { + "grok": { "field": "iis.error.remote_ip", + "patterns": [ + "%{NOZONEIP:iis.error.remote_ip_geoip}" + ], + "pattern_definitions": { + "NOZONEIP": "[^%]*" + } + } + }, { + "geoip": { + "field": "iis.error.remote_ip_geoip", "target_field": "iis.error.geoip" } + }, { + "remove": { + "field": "iis.error.remote_ip_geoip" + } }], "on_failure" : [{ "set" : { diff --git a/filebeat/module/iis/error/test/ipv6_zone_id.log b/filebeat/module/iis/error/test/ipv6_zone_id.log new file mode 100644 index 00000000000..436e133e344 --- /dev/null +++ b/filebeat/module/iis/error/test/ipv6_zone_id.log @@ -0,0 +1,5 @@ +#Software: Microsoft HTTP API 2.0 +#Version: 1.0 +#Date: 2018-12-30 13:48:36 +#Fields: date time c-ip c-port s-ip s-port cs-version cs-method cs-uri streamid sc-status s-siteid s-reason s-queuename +2018-12-30 14:22:07 ::1%0 49958 ::1%0 80 - - - - - - Timer_ConnectionIdle - diff --git a/filebeat/module/iis/error/test/ipv6_zone_id.log-expected.json b/filebeat/module/iis/error/test/ipv6_zone_id.log-expected.json new file mode 100644 index 00000000000..99c1a3bd093 --- /dev/null +++ b/filebeat/module/iis/error/test/ipv6_zone_id.log-expected.json @@ -0,0 +1,16 @@ +[ + { + "@timestamp": "2018-12-30T14:22:07.000Z", + "ecs.version": "1.0.0-beta2", + "event.dataset": "error", + "event.module": "iis", + "iis.error.queue_name": "-", + "iis.error.reason_phrase": "Timer_ConnectionIdle", + "iis.error.remote_ip": "::1%0", + "iis.error.remote_port": "49958", + "iis.error.server_ip": "::1%0", + "iis.error.server_port": "80", + "input.type": "log", + "log.offset": 195 + } +] \ No newline at end of file From 41d9e304756234039e71d397544d20702078186e Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Tue, 8 Jan 2019 16:15:58 +0100 Subject: [PATCH 2/2] Regenerate expected json --- .../module/iis/error/test/ipv6_zone_id.log-expected.json | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/filebeat/module/iis/error/test/ipv6_zone_id.log-expected.json b/filebeat/module/iis/error/test/ipv6_zone_id.log-expected.json index 99c1a3bd093..e83699c91cb 100644 --- a/filebeat/module/iis/error/test/ipv6_zone_id.log-expected.json +++ b/filebeat/module/iis/error/test/ipv6_zone_id.log-expected.json @@ -1,9 +1,9 @@ [ { "@timestamp": "2018-12-30T14:22:07.000Z", - "ecs.version": "1.0.0-beta2", - "event.dataset": "error", - "event.module": "iis", + "event.dataset": "iis.error", + "fileset.module": "iis", + "fileset.name": "error", "iis.error.queue_name": "-", "iis.error.reason_phrase": "Timer_ConnectionIdle", "iis.error.remote_ip": "::1%0", @@ -11,6 +11,7 @@ "iis.error.server_ip": "::1%0", "iis.error.server_port": "80", "input.type": "log", - "log.offset": 195 + "offset": 195, + "prospector.type": "log" } ] \ No newline at end of file