diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 23b6f1a874e..b4a4697ada4 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -132,6 +132,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix a rate limit related issue in httpjson input for Okta module. {issue}18530[18530] {pull}18534[18534] - Fixed ingestion of some Cisco ASA and FTD messages when a hostname was used instead of an IP for NAT fields. {issue}14034[14034] {pull}18376[18376] - Fix PANW module wrong mappings for bytes and packets counters. {issue}18522[18522] {pull}18525[18525] +- Fix `o365.audit` failing to ingest events when ip address is surrounded by square brackets. {issue}18587[18587] {pull}18591[18591] *Heartbeat* diff --git a/x-pack/filebeat/module/o365/audit/config/pipeline.js b/x-pack/filebeat/module/o365/audit/config/pipeline.js index 679330a494b..ae8e1a7afe6 100644 --- a/x-pack/filebeat/module/o365/audit/config/pipeline.js +++ b/x-pack/filebeat/module/o365/audit/config/pipeline.js @@ -725,22 +725,23 @@ function AuditProcessor(tenant_names, debug) { 'Yammer': yammerSchema(debug).Run, })); - builder.Add("extractClientIPv4Port", new processor.Dissect({ - tokenizer: '%{ip}:%{port}', + builder.Add("extractClientIPPortBrackets", new processor.Dissect({ + tokenizer: '[%{_ip}]:%{port}', field: 'client.address', target_prefix: 'client', - 'when.and': [ - {'contains.client.address': '.'}, - {'contains.client.address': ':'}, - ], + 'when.contains.client.address': ']:', })); - builder.Add("extractClientIPv6Port", new processor.Dissect({ - tokenizer: '[%{ip}]:%{port}', + builder.Add("extractClientIPv4Port", new processor.Dissect({ + tokenizer: '%{_ip}:%{port}', field: 'client.address', target_prefix: 'client', 'when.and': [ - {'contains.client.address': '['}, + {'not.has_fields': ['client._ip', 'client.port']}, + {'contains.client.address': '.'}, {'contains.client.address': ':'}, + // Best effort to avoid parsing IPv6-mapped IPv4 as ip:port. + // Won't succeed if IPv6 address is not shortened. + {'not.contains.client.address': '::'}, ], })); @@ -749,11 +750,14 @@ function AuditProcessor(tenant_names, debug) { fields: [ {from: "client.address", to: "client.ip", type: "ip"}, {from: "server.address", to: "server.ip", type: "ip"}, + {from: "client._ip", to: "client.ip", type: "ip"}, ], ignore_missing: true, fail_on_error: false })); - + builder.Add("removeTempIP", function (evt) { + evt.Delete("client._ip"); + }); builder.Add("setSrcDstFields", new processor.Convert({ fields: [ {from: "client.ip", to: "source.ip"}, diff --git a/x-pack/filebeat/module/o365/audit/test/ip-formats.log b/x-pack/filebeat/module/o365/audit/test/ip-formats.log new file mode 100644 index 00000000000..bcfd62f2409 --- /dev/null +++ b/x-pack/filebeat/module/o365/audit/test/ip-formats.log @@ -0,0 +1,15 @@ +{"ClientIP":"[10.11.12.13]:12345","RecordType":-1,"CreationTime":"2020-02-17T17:12:03","Id":"3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226"} +{"ClientIP":"10.11.12.13:12345","RecordType":-1,"CreationTime":"2020-02-17T17:12:03","Id":"3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226"} +{"ClientIP":"10.11.12.13","RecordType":-1,"CreationTime":"2020-02-17T17:12:03","Id":"3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226"} +{"ClientIP":"::ffff:10.11.12.13","RecordType":-1,"CreationTime":"2020-02-17T17:12:03","Id":"3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226"} +{"ClientIP":"[::ffff:10.11.12.13]:12345","RecordType":-1,"CreationTime":"2020-02-17T17:12:03","Id":"3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226"} +{"ClientIP":"[2001:db8::abcd]:12345","RecordType":-1,"CreationTime":"2020-02-17T17:12:03","Id":"3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226"} +{"ClientIP":"2001:db8::abcd","RecordType":-1,"CreationTime":"2020-02-17T17:12:03","Id":"3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226"} +{"ClientIP":"[2001:db8::abcd]","RecordType":-1,"CreationTime":"2020-02-17T17:12:03","Id":"3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226"} +{"ClientIP":"[10.11.12.13]","RecordType":-1,"CreationTime":"2020-02-17T17:12:03","Id":"3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226"} +{"ClientIP":"localhost","RecordType":-1,"CreationTime":"2020-02-17T17:12:03","Id":"3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226"} +{"ClientIP":"[localhost]:12345","RecordType":-1,"CreationTime":"2020-02-17T17:12:03","Id":"3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226"} +{"ClientIP":"localhost:12345","RecordType":-1,"CreationTime":"2020-02-17T17:12:03","Id":"3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226"} +{"ClientIP":"[cool.client.local]:12345","RecordType":-1,"CreationTime":"2020-02-17T17:12:03","Id":"3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226"} +{"ClientIP":"cool.client.local","RecordType":-1,"CreationTime":"2020-02-17T17:12:03","Id":"3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226"} +{"ClientIP":"cool.client.local:12345","RecordType":-1,"CreationTime":"2020-02-17T17:12:03","Id":"3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226"} diff --git a/x-pack/filebeat/module/o365/audit/test/ip-formats.log-expected.json b/x-pack/filebeat/module/o365/audit/test/ip-formats.log-expected.json new file mode 100644 index 00000000000..943456ae944 --- /dev/null +++ b/x-pack/filebeat/module/o365/audit/test/ip-formats.log-expected.json @@ -0,0 +1,344 @@ +[ + { + "@timestamp": "2020-02-17T17:12:03.000Z", + "client.address": "[10.11.12.13]:12345", + "client.ip": "10.11.12.13", + "client.port": "12345", + "event.category": "web", + "event.code": -1, + "event.dataset": "o365.audit", + "event.id": "3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226", + "event.kind": "event", + "event.module": "o365", + "event.outcome": "success", + "event.type": "info", + "fileset.name": "audit", + "input.type": "log", + "log.offset": 0, + "network.type": "ipv4", + "o365.audit.ClientIP": "[10.11.12.13]:12345", + "o365.audit.CreationTime": "2020-02-17T17:12:03", + "o365.audit.Id": "3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226", + "o365.audit.RecordType": -1, + "related.ip": "10.11.12.13", + "service.type": "o365", + "source.ip": "10.11.12.13", + "source.port": "12345" + }, + { + "@timestamp": "2020-02-17T17:12:03.000Z", + "client.address": "10.11.12.13:12345", + "client.ip": "10.11.12.13", + "client.port": "12345", + "event.category": "web", + "event.code": -1, + "event.dataset": "o365.audit", + "event.id": "3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226", + "event.kind": "event", + "event.module": "o365", + "event.outcome": "success", + "event.type": "info", + "fileset.name": "audit", + "input.type": "log", + "log.offset": 132, + "network.type": "ipv4", + "o365.audit.ClientIP": "10.11.12.13:12345", + "o365.audit.CreationTime": "2020-02-17T17:12:03", + "o365.audit.Id": "3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226", + "o365.audit.RecordType": -1, + "related.ip": "10.11.12.13", + "service.type": "o365", + "source.ip": "10.11.12.13", + "source.port": "12345" + }, + { + "@timestamp": "2020-02-17T17:12:03.000Z", + "client.address": "10.11.12.13", + "client.ip": "10.11.12.13", + "event.category": "web", + "event.code": -1, + "event.dataset": "o365.audit", + "event.id": "3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226", + "event.kind": "event", + "event.module": "o365", + "event.outcome": "success", + "event.type": "info", + "fileset.name": "audit", + "input.type": "log", + "log.offset": 262, + "network.type": "ipv4", + "o365.audit.ClientIP": "10.11.12.13", + "o365.audit.CreationTime": "2020-02-17T17:12:03", + "o365.audit.Id": "3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226", + "o365.audit.RecordType": -1, + "related.ip": "10.11.12.13", + "service.type": "o365", + "source.ip": "10.11.12.13" + }, + { + "@timestamp": "2020-02-17T17:12:03.000Z", + "client.address": "::ffff:10.11.12.13", + "client.ip": "::ffff:10.11.12.13", + "event.category": "web", + "event.code": -1, + "event.dataset": "o365.audit", + "event.id": "3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226", + "event.kind": "event", + "event.module": "o365", + "event.outcome": "success", + "event.type": "info", + "fileset.name": "audit", + "input.type": "log", + "log.offset": 386, + "network.type": "ipv4", + "o365.audit.ClientIP": "::ffff:10.11.12.13", + "o365.audit.CreationTime": "2020-02-17T17:12:03", + "o365.audit.Id": "3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226", + "o365.audit.RecordType": -1, + "related.ip": "::ffff:10.11.12.13", + "service.type": "o365", + "source.ip": "::ffff:10.11.12.13" + }, + { + "@timestamp": "2020-02-17T17:12:03.000Z", + "client.address": "[::ffff:10.11.12.13]:12345", + "client.ip": "::ffff:10.11.12.13", + "client.port": "12345", + "event.category": "web", + "event.code": -1, + "event.dataset": "o365.audit", + "event.id": "3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226", + "event.kind": "event", + "event.module": "o365", + "event.outcome": "success", + "event.type": "info", + "fileset.name": "audit", + "input.type": "log", + "log.offset": 517, + "network.type": "ipv4", + "o365.audit.ClientIP": "[::ffff:10.11.12.13]:12345", + "o365.audit.CreationTime": "2020-02-17T17:12:03", + "o365.audit.Id": "3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226", + "o365.audit.RecordType": -1, + "related.ip": "::ffff:10.11.12.13", + "service.type": "o365", + "source.ip": "::ffff:10.11.12.13", + "source.port": "12345" + }, + { + "@timestamp": "2020-02-17T17:12:03.000Z", + "client.address": "[2001:db8::abcd]:12345", + "client.ip": "2001:db8::abcd", + "client.port": "12345", + "event.category": "web", + "event.code": -1, + "event.dataset": "o365.audit", + "event.id": "3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226", + "event.kind": "event", + "event.module": "o365", + "event.outcome": "success", + "event.type": "info", + "fileset.name": "audit", + "input.type": "log", + "log.offset": 656, + "network.type": "ipv6", + "o365.audit.ClientIP": "[2001:db8::abcd]:12345", + "o365.audit.CreationTime": "2020-02-17T17:12:03", + "o365.audit.Id": "3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226", + "o365.audit.RecordType": -1, + "related.ip": "2001:db8::abcd", + "service.type": "o365", + "source.ip": "2001:db8::abcd", + "source.port": "12345" + }, + { + "@timestamp": "2020-02-17T17:12:03.000Z", + "client.address": "2001:db8::abcd", + "client.ip": "2001:db8::abcd", + "event.category": "web", + "event.code": -1, + "event.dataset": "o365.audit", + "event.id": "3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226", + "event.kind": "event", + "event.module": "o365", + "event.outcome": "success", + "event.type": "info", + "fileset.name": "audit", + "input.type": "log", + "log.offset": 791, + "network.type": "ipv6", + "o365.audit.ClientIP": "2001:db8::abcd", + "o365.audit.CreationTime": "2020-02-17T17:12:03", + "o365.audit.Id": "3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226", + "o365.audit.RecordType": -1, + "related.ip": "2001:db8::abcd", + "service.type": "o365", + "source.ip": "2001:db8::abcd" + }, + { + "@timestamp": "2020-02-17T17:12:03.000Z", + "client.address": "[2001:db8::abcd]", + "event.category": "web", + "event.code": -1, + "event.dataset": "o365.audit", + "event.id": "3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226", + "event.kind": "event", + "event.module": "o365", + "event.outcome": "success", + "event.type": "info", + "fileset.name": "audit", + "input.type": "log", + "log.offset": 918, + "o365.audit.ClientIP": "[2001:db8::abcd]", + "o365.audit.CreationTime": "2020-02-17T17:12:03", + "o365.audit.Id": "3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226", + "o365.audit.RecordType": -1, + "service.type": "o365" + }, + { + "@timestamp": "2020-02-17T17:12:03.000Z", + "client.address": "[10.11.12.13]", + "event.category": "web", + "event.code": -1, + "event.dataset": "o365.audit", + "event.id": "3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226", + "event.kind": "event", + "event.module": "o365", + "event.outcome": "success", + "event.type": "info", + "fileset.name": "audit", + "input.type": "log", + "log.offset": 1047, + "o365.audit.ClientIP": "[10.11.12.13]", + "o365.audit.CreationTime": "2020-02-17T17:12:03", + "o365.audit.Id": "3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226", + "o365.audit.RecordType": -1, + "service.type": "o365" + }, + { + "@timestamp": "2020-02-17T17:12:03.000Z", + "client.address": "localhost", + "event.category": "web", + "event.code": -1, + "event.dataset": "o365.audit", + "event.id": "3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226", + "event.kind": "event", + "event.module": "o365", + "event.outcome": "success", + "event.type": "info", + "fileset.name": "audit", + "input.type": "log", + "log.offset": 1173, + "o365.audit.ClientIP": "localhost", + "o365.audit.CreationTime": "2020-02-17T17:12:03", + "o365.audit.Id": "3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226", + "o365.audit.RecordType": -1, + "service.type": "o365" + }, + { + "@timestamp": "2020-02-17T17:12:03.000Z", + "client.address": "[localhost]:12345", + "client.port": "12345", + "event.category": "web", + "event.code": -1, + "event.dataset": "o365.audit", + "event.id": "3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226", + "event.kind": "event", + "event.module": "o365", + "event.outcome": "success", + "event.type": "info", + "fileset.name": "audit", + "input.type": "log", + "log.offset": 1295, + "o365.audit.ClientIP": "[localhost]:12345", + "o365.audit.CreationTime": "2020-02-17T17:12:03", + "o365.audit.Id": "3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226", + "o365.audit.RecordType": -1, + "service.type": "o365", + "source.port": "12345" + }, + { + "@timestamp": "2020-02-17T17:12:03.000Z", + "client.address": "localhost:12345", + "event.category": "web", + "event.code": -1, + "event.dataset": "o365.audit", + "event.id": "3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226", + "event.kind": "event", + "event.module": "o365", + "event.outcome": "success", + "event.type": "info", + "fileset.name": "audit", + "input.type": "log", + "log.offset": 1425, + "o365.audit.ClientIP": "localhost:12345", + "o365.audit.CreationTime": "2020-02-17T17:12:03", + "o365.audit.Id": "3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226", + "o365.audit.RecordType": -1, + "service.type": "o365" + }, + { + "@timestamp": "2020-02-17T17:12:03.000Z", + "client.address": "[cool.client.local]:12345", + "client.port": "12345", + "event.category": "web", + "event.code": -1, + "event.dataset": "o365.audit", + "event.id": "3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226", + "event.kind": "event", + "event.module": "o365", + "event.outcome": "success", + "event.type": "info", + "fileset.name": "audit", + "input.type": "log", + "log.offset": 1553, + "o365.audit.ClientIP": "[cool.client.local]:12345", + "o365.audit.CreationTime": "2020-02-17T17:12:03", + "o365.audit.Id": "3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226", + "o365.audit.RecordType": -1, + "service.type": "o365", + "source.port": "12345" + }, + { + "@timestamp": "2020-02-17T17:12:03.000Z", + "client.address": "cool.client.local", + "event.category": "web", + "event.code": -1, + "event.dataset": "o365.audit", + "event.id": "3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226", + "event.kind": "event", + "event.module": "o365", + "event.outcome": "success", + "event.type": "info", + "fileset.name": "audit", + "input.type": "log", + "log.offset": 1691, + "o365.audit.ClientIP": "cool.client.local", + "o365.audit.CreationTime": "2020-02-17T17:12:03", + "o365.audit.Id": "3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226", + "o365.audit.RecordType": -1, + "service.type": "o365" + }, + { + "@timestamp": "2020-02-17T17:12:03.000Z", + "client.address": "cool.client.local:12345", + "client.port": "12345", + "event.category": "web", + "event.code": -1, + "event.dataset": "o365.audit", + "event.id": "3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226", + "event.kind": "event", + "event.module": "o365", + "event.outcome": "success", + "event.type": "info", + "fileset.name": "audit", + "input.type": "log", + "log.offset": 1821, + "o365.audit.ClientIP": "cool.client.local:12345", + "o365.audit.CreationTime": "2020-02-17T17:12:03", + "o365.audit.Id": "3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226", + "o365.audit.RecordType": -1, + "service.type": "o365", + "source.port": "12345" + } +] \ No newline at end of file