From eaf196d10e94ad9668febadcc84610fee28e7b79 Mon Sep 17 00:00:00 2001 From: Adrian Serrano Date: Sat, 16 May 2020 17:00:58 +0200 Subject: [PATCH] o365: Support IPv4 enclosed in square brackets (#18591) The O365 Management API is generating some events that contain a `ClientIP` / `ClientIPAddress` field consisting of an IPv4 address enclosed in square brackets. This is breaking ingestion of those events as the brackets are only stripped for IPv6 addresses. > "ClientIP": "[10.11.12.13]:12345" This patch adds support for IPv4 enclosed in brackets and a few other edge cases. Fixes #18587 --- CHANGELOG.next.asciidoc | 1 + .../module/o365/audit/config/pipeline.js | 24 +- .../module/o365/audit/test/ip-formats.log | 15 + .../audit/test/ip-formats.log-expected.json | 389 ++++++++++++++++++ 4 files changed, 419 insertions(+), 10 deletions(-) create mode 100644 x-pack/filebeat/module/o365/audit/test/ip-formats.log create mode 100644 x-pack/filebeat/module/o365/audit/test/ip-formats.log-expected.json diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 8f0e7aaaef2..7471b1b99f8 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -145,6 +145,7 @@ field. You can revert this change by configuring tags for the module and omittin - Fixed ingestion of some Cisco ASA and FTD messages when a hostname was used instead of an IP for NAT fields. {issue}14034[14034] {pull}18376[18376] - Fix a rate limit related issue in httpjson input for Okta module. {issue}18530[18530] {pull}18534[18534] - Fix `googlecloud.audit` pipeline to only take in fields that are explicitly defined by the dataset. {issue}18465[18465] {pull}18472[18472] +- Fix `o365.audit` failing to ingest events when ip address is surrounded by square brackets. {issue}18587[18587] {pull}18591[18591] *Heartbeat* diff --git a/x-pack/filebeat/module/o365/audit/config/pipeline.js b/x-pack/filebeat/module/o365/audit/config/pipeline.js index 679330a494b..ae8e1a7afe6 100644 --- a/x-pack/filebeat/module/o365/audit/config/pipeline.js +++ b/x-pack/filebeat/module/o365/audit/config/pipeline.js @@ -725,22 +725,23 @@ function AuditProcessor(tenant_names, debug) { 'Yammer': yammerSchema(debug).Run, })); - builder.Add("extractClientIPv4Port", new processor.Dissect({ - tokenizer: '%{ip}:%{port}', + builder.Add("extractClientIPPortBrackets", new processor.Dissect({ + tokenizer: '[%{_ip}]:%{port}', field: 'client.address', target_prefix: 'client', - 'when.and': [ - {'contains.client.address': '.'}, - {'contains.client.address': ':'}, - ], + 'when.contains.client.address': ']:', })); - builder.Add("extractClientIPv6Port", new processor.Dissect({ - tokenizer: '[%{ip}]:%{port}', + builder.Add("extractClientIPv4Port", new processor.Dissect({ + tokenizer: '%{_ip}:%{port}', field: 'client.address', target_prefix: 'client', 'when.and': [ - {'contains.client.address': '['}, + {'not.has_fields': ['client._ip', 'client.port']}, + {'contains.client.address': '.'}, {'contains.client.address': ':'}, + // Best effort to avoid parsing IPv6-mapped IPv4 as ip:port. + // Won't succeed if IPv6 address is not shortened. + {'not.contains.client.address': '::'}, ], })); @@ -749,11 +750,14 @@ function AuditProcessor(tenant_names, debug) { fields: [ {from: "client.address", to: "client.ip", type: "ip"}, {from: "server.address", to: "server.ip", type: "ip"}, + {from: "client._ip", to: "client.ip", type: "ip"}, ], ignore_missing: true, fail_on_error: false })); - + builder.Add("removeTempIP", function (evt) { + evt.Delete("client._ip"); + }); builder.Add("setSrcDstFields", new processor.Convert({ fields: [ {from: "client.ip", to: "source.ip"}, diff --git a/x-pack/filebeat/module/o365/audit/test/ip-formats.log b/x-pack/filebeat/module/o365/audit/test/ip-formats.log new file mode 100644 index 00000000000..bcfd62f2409 --- /dev/null +++ b/x-pack/filebeat/module/o365/audit/test/ip-formats.log @@ -0,0 +1,15 @@ +{"ClientIP":"[10.11.12.13]:12345","RecordType":-1,"CreationTime":"2020-02-17T17:12:03","Id":"3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226"} +{"ClientIP":"10.11.12.13:12345","RecordType":-1,"CreationTime":"2020-02-17T17:12:03","Id":"3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226"} +{"ClientIP":"10.11.12.13","RecordType":-1,"CreationTime":"2020-02-17T17:12:03","Id":"3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226"} +{"ClientIP":"::ffff:10.11.12.13","RecordType":-1,"CreationTime":"2020-02-17T17:12:03","Id":"3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226"} +{"ClientIP":"[::ffff:10.11.12.13]:12345","RecordType":-1,"CreationTime":"2020-02-17T17:12:03","Id":"3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226"} +{"ClientIP":"[2001:db8::abcd]:12345","RecordType":-1,"CreationTime":"2020-02-17T17:12:03","Id":"3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226"} +{"ClientIP":"2001:db8::abcd","RecordType":-1,"CreationTime":"2020-02-17T17:12:03","Id":"3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226"} +{"ClientIP":"[2001:db8::abcd]","RecordType":-1,"CreationTime":"2020-02-17T17:12:03","Id":"3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226"} +{"ClientIP":"[10.11.12.13]","RecordType":-1,"CreationTime":"2020-02-17T17:12:03","Id":"3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226"} +{"ClientIP":"localhost","RecordType":-1,"CreationTime":"2020-02-17T17:12:03","Id":"3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226"} +{"ClientIP":"[localhost]:12345","RecordType":-1,"CreationTime":"2020-02-17T17:12:03","Id":"3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226"} +{"ClientIP":"localhost:12345","RecordType":-1,"CreationTime":"2020-02-17T17:12:03","Id":"3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226"} +{"ClientIP":"[cool.client.local]:12345","RecordType":-1,"CreationTime":"2020-02-17T17:12:03","Id":"3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226"} +{"ClientIP":"cool.client.local","RecordType":-1,"CreationTime":"2020-02-17T17:12:03","Id":"3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226"} +{"ClientIP":"cool.client.local:12345","RecordType":-1,"CreationTime":"2020-02-17T17:12:03","Id":"3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226"} diff --git a/x-pack/filebeat/module/o365/audit/test/ip-formats.log-expected.json b/x-pack/filebeat/module/o365/audit/test/ip-formats.log-expected.json new file mode 100644 index 00000000000..8083856828b --- /dev/null +++ b/x-pack/filebeat/module/o365/audit/test/ip-formats.log-expected.json @@ -0,0 +1,389 @@ +[ + { + "@timestamp": "2020-02-17T17:12:03.000Z", + "client.address": "[10.11.12.13]:12345", + "client.ip": "10.11.12.13", + "client.port": "12345", + "event.category": "web", + "event.code": -1, + "event.dataset": "o365.audit", + "event.id": "3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226", + "event.kind": "event", + "event.module": "o365", + "event.outcome": "success", + "event.type": "info", + "fileset.name": "audit", + "input.type": "log", + "log.offset": 0, + "network.type": "ipv4", + "o365.audit.ClientIP": "[10.11.12.13]:12345", + "o365.audit.CreationTime": "2020-02-17T17:12:03", + "o365.audit.Id": "3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226", + "o365.audit.RecordType": -1, + "related.ip": "10.11.12.13", + "service.type": "o365", + "source.ip": "10.11.12.13", + "source.port": "12345", + "tags": [ + "forwarded" + ] + }, + { + "@timestamp": "2020-02-17T17:12:03.000Z", + "client.address": "10.11.12.13:12345", + "client.ip": "10.11.12.13", + "client.port": "12345", + "event.category": "web", + "event.code": -1, + "event.dataset": "o365.audit", + "event.id": "3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226", + "event.kind": "event", + "event.module": "o365", + "event.outcome": "success", + "event.type": "info", + "fileset.name": "audit", + "input.type": "log", + "log.offset": 132, + "network.type": "ipv4", + "o365.audit.ClientIP": "10.11.12.13:12345", + "o365.audit.CreationTime": "2020-02-17T17:12:03", + "o365.audit.Id": "3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226", + "o365.audit.RecordType": -1, + "related.ip": "10.11.12.13", + "service.type": "o365", + "source.ip": "10.11.12.13", + "source.port": "12345", + "tags": [ + "forwarded" + ] + }, + { + "@timestamp": "2020-02-17T17:12:03.000Z", + "client.address": "10.11.12.13", + "client.ip": "10.11.12.13", + "event.category": "web", + "event.code": -1, + "event.dataset": "o365.audit", + "event.id": "3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226", + "event.kind": "event", + "event.module": "o365", + "event.outcome": "success", + "event.type": "info", + "fileset.name": "audit", + "input.type": "log", + "log.offset": 262, + "network.type": "ipv4", + "o365.audit.ClientIP": "10.11.12.13", + "o365.audit.CreationTime": "2020-02-17T17:12:03", + "o365.audit.Id": "3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226", + "o365.audit.RecordType": -1, + "related.ip": "10.11.12.13", + "service.type": "o365", + "source.ip": "10.11.12.13", + "tags": [ + "forwarded" + ] + }, + { + "@timestamp": "2020-02-17T17:12:03.000Z", + "client.address": "::ffff:10.11.12.13", + "client.ip": "::ffff:10.11.12.13", + "event.category": "web", + "event.code": -1, + "event.dataset": "o365.audit", + "event.id": "3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226", + "event.kind": "event", + "event.module": "o365", + "event.outcome": "success", + "event.type": "info", + "fileset.name": "audit", + "input.type": "log", + "log.offset": 386, + "network.type": "ipv4", + "o365.audit.ClientIP": "::ffff:10.11.12.13", + "o365.audit.CreationTime": "2020-02-17T17:12:03", + "o365.audit.Id": "3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226", + "o365.audit.RecordType": -1, + "related.ip": "::ffff:10.11.12.13", + "service.type": "o365", + "source.ip": "::ffff:10.11.12.13", + "tags": [ + "forwarded" + ] + }, + { + "@timestamp": "2020-02-17T17:12:03.000Z", + "client.address": "[::ffff:10.11.12.13]:12345", + "client.ip": "::ffff:10.11.12.13", + "client.port": "12345", + "event.category": "web", + "event.code": -1, + "event.dataset": "o365.audit", + "event.id": "3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226", + "event.kind": "event", + "event.module": "o365", + "event.outcome": "success", + "event.type": "info", + "fileset.name": "audit", + "input.type": "log", + "log.offset": 517, + "network.type": "ipv4", + "o365.audit.ClientIP": "[::ffff:10.11.12.13]:12345", + "o365.audit.CreationTime": "2020-02-17T17:12:03", + "o365.audit.Id": "3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226", + "o365.audit.RecordType": -1, + "related.ip": "::ffff:10.11.12.13", + "service.type": "o365", + "source.ip": "::ffff:10.11.12.13", + "source.port": "12345", + "tags": [ + "forwarded" + ] + }, + { + "@timestamp": "2020-02-17T17:12:03.000Z", + "client.address": "[2001:db8::abcd]:12345", + "client.ip": "2001:db8::abcd", + "client.port": "12345", + "event.category": "web", + "event.code": -1, + "event.dataset": "o365.audit", + "event.id": "3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226", + "event.kind": "event", + "event.module": "o365", + "event.outcome": "success", + "event.type": "info", + "fileset.name": "audit", + "input.type": "log", + "log.offset": 656, + "network.type": "ipv6", + "o365.audit.ClientIP": "[2001:db8::abcd]:12345", + "o365.audit.CreationTime": "2020-02-17T17:12:03", + "o365.audit.Id": "3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226", + "o365.audit.RecordType": -1, + "related.ip": "2001:db8::abcd", + "service.type": "o365", + "source.ip": "2001:db8::abcd", + "source.port": "12345", + "tags": [ + "forwarded" + ] + }, + { + "@timestamp": "2020-02-17T17:12:03.000Z", + "client.address": "2001:db8::abcd", + "client.ip": "2001:db8::abcd", + "event.category": "web", + "event.code": -1, + "event.dataset": "o365.audit", + "event.id": "3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226", + "event.kind": "event", + "event.module": "o365", + "event.outcome": "success", + "event.type": "info", + "fileset.name": "audit", + "input.type": "log", + "log.offset": 791, + "network.type": "ipv6", + "o365.audit.ClientIP": "2001:db8::abcd", + "o365.audit.CreationTime": "2020-02-17T17:12:03", + "o365.audit.Id": "3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226", + "o365.audit.RecordType": -1, + "related.ip": "2001:db8::abcd", + "service.type": "o365", + "source.ip": "2001:db8::abcd", + "tags": [ + "forwarded" + ] + }, + { + "@timestamp": "2020-02-17T17:12:03.000Z", + "client.address": "[2001:db8::abcd]", + "event.category": "web", + "event.code": -1, + "event.dataset": "o365.audit", + "event.id": "3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226", + "event.kind": "event", + "event.module": "o365", + "event.outcome": "success", + "event.type": "info", + "fileset.name": "audit", + "input.type": "log", + "log.offset": 918, + "o365.audit.ClientIP": "[2001:db8::abcd]", + "o365.audit.CreationTime": "2020-02-17T17:12:03", + "o365.audit.Id": "3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226", + "o365.audit.RecordType": -1, + "service.type": "o365", + "tags": [ + "forwarded" + ] + }, + { + "@timestamp": "2020-02-17T17:12:03.000Z", + "client.address": "[10.11.12.13]", + "event.category": "web", + "event.code": -1, + "event.dataset": "o365.audit", + "event.id": "3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226", + "event.kind": "event", + "event.module": "o365", + "event.outcome": "success", + "event.type": "info", + "fileset.name": "audit", + "input.type": "log", + "log.offset": 1047, + "o365.audit.ClientIP": "[10.11.12.13]", + "o365.audit.CreationTime": "2020-02-17T17:12:03", + "o365.audit.Id": "3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226", + "o365.audit.RecordType": -1, + "service.type": "o365", + "tags": [ + "forwarded" + ] + }, + { + "@timestamp": "2020-02-17T17:12:03.000Z", + "client.address": "localhost", + "event.category": "web", + "event.code": -1, + "event.dataset": "o365.audit", + "event.id": "3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226", + "event.kind": "event", + "event.module": "o365", + "event.outcome": "success", + "event.type": "info", + "fileset.name": "audit", + "input.type": "log", + "log.offset": 1173, + "o365.audit.ClientIP": "localhost", + "o365.audit.CreationTime": "2020-02-17T17:12:03", + "o365.audit.Id": "3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226", + "o365.audit.RecordType": -1, + "service.type": "o365", + "tags": [ + "forwarded" + ] + }, + { + "@timestamp": "2020-02-17T17:12:03.000Z", + "client.address": "[localhost]:12345", + "client.port": "12345", + "event.category": "web", + "event.code": -1, + "event.dataset": "o365.audit", + "event.id": "3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226", + "event.kind": "event", + "event.module": "o365", + "event.outcome": "success", + "event.type": "info", + "fileset.name": "audit", + "input.type": "log", + "log.offset": 1295, + "o365.audit.ClientIP": "[localhost]:12345", + "o365.audit.CreationTime": "2020-02-17T17:12:03", + "o365.audit.Id": "3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226", + "o365.audit.RecordType": -1, + "service.type": "o365", + "source.port": "12345", + "tags": [ + "forwarded" + ] + }, + { + "@timestamp": "2020-02-17T17:12:03.000Z", + "client.address": "localhost:12345", + "event.category": "web", + "event.code": -1, + "event.dataset": "o365.audit", + "event.id": "3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226", + "event.kind": "event", + "event.module": "o365", + "event.outcome": "success", + "event.type": "info", + "fileset.name": "audit", + "input.type": "log", + "log.offset": 1425, + "o365.audit.ClientIP": "localhost:12345", + "o365.audit.CreationTime": "2020-02-17T17:12:03", + "o365.audit.Id": "3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226", + "o365.audit.RecordType": -1, + "service.type": "o365", + "tags": [ + "forwarded" + ] + }, + { + "@timestamp": "2020-02-17T17:12:03.000Z", + "client.address": "[cool.client.local]:12345", + "client.port": "12345", + "event.category": "web", + "event.code": -1, + "event.dataset": "o365.audit", + "event.id": "3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226", + "event.kind": "event", + "event.module": "o365", + "event.outcome": "success", + "event.type": "info", + "fileset.name": "audit", + "input.type": "log", + "log.offset": 1553, + "o365.audit.ClientIP": "[cool.client.local]:12345", + "o365.audit.CreationTime": "2020-02-17T17:12:03", + "o365.audit.Id": "3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226", + "o365.audit.RecordType": -1, + "service.type": "o365", + "source.port": "12345", + "tags": [ + "forwarded" + ] + }, + { + "@timestamp": "2020-02-17T17:12:03.000Z", + "client.address": "cool.client.local", + "event.category": "web", + "event.code": -1, + "event.dataset": "o365.audit", + "event.id": "3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226", + "event.kind": "event", + "event.module": "o365", + "event.outcome": "success", + "event.type": "info", + "fileset.name": "audit", + "input.type": "log", + "log.offset": 1691, + "o365.audit.ClientIP": "cool.client.local", + "o365.audit.CreationTime": "2020-02-17T17:12:03", + "o365.audit.Id": "3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226", + "o365.audit.RecordType": -1, + "service.type": "o365", + "tags": [ + "forwarded" + ] + }, + { + "@timestamp": "2020-02-17T17:12:03.000Z", + "client.address": "cool.client.local:12345", + "client.port": "12345", + "event.category": "web", + "event.code": -1, + "event.dataset": "o365.audit", + "event.id": "3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226", + "event.kind": "event", + "event.module": "o365", + "event.outcome": "success", + "event.type": "info", + "fileset.name": "audit", + "input.type": "log", + "log.offset": 1821, + "o365.audit.ClientIP": "cool.client.local:12345", + "o365.audit.CreationTime": "2020-02-17T17:12:03", + "o365.audit.Id": "3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226", + "o365.audit.RecordType": -1, + "service.type": "o365", + "source.port": "12345", + "tags": [ + "forwarded" + ] + } +] \ No newline at end of file