From 3e6edf2812130a7e83e8c65768c9b5f3d65d3873 Mon Sep 17 00:00:00 2001 From: Lee Hinman <57081003+leehinman@users.noreply.github.com> Date: Thu, 12 Mar 2020 18:01:55 -0500 Subject: [PATCH] Improve ECS field mappings for CEF module (#16338) - related.hash - related.ip - related.user - fix description Closes #16157 Closes #16289 --- CHANGELOG.next.asciidoc | 1 + .../module/cef/log/ingest/pipeline.yml | 34 ++++++++- x-pack/filebeat/module/cef/log/test/cef.log | 1 + .../module/cef/log/test/cef.log-expected.json | 71 +++++++++++++++++++ .../log/test/fp-ngfw-smc.log-expected.json | 33 +++++++++ 5 files changed, 139 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 45b3e5979d6..a9d92bcb3c4 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -178,6 +178,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Improve ECS categorization field mapping in icinga module. {issue}16164[16164] {pull}16533[16533] - Improve ECS categorization field mappings in ibmmq module. {issue}16163[16163] {pull}16532[16532] - Improve ECS categorization, host field mappings in elasticsearch module. {issue}16160[16160] {pull}16469[16469] +- Add ECS related fields to CEF module {issue}16157[16157] {pull}16338[16338] *Heartbeat* diff --git a/x-pack/filebeat/module/cef/log/ingest/pipeline.yml b/x-pack/filebeat/module/cef/log/ingest/pipeline.yml index 2600dbeec08..2d14926e7f8 100644 --- a/x-pack/filebeat/module/cef/log/ingest/pipeline.yml +++ b/x-pack/filebeat/module/cef/log/ingest/pipeline.yml @@ -1,5 +1,5 @@ --- -description: Pipeline for Filebeat NetFlow +description: Pipeline for Filebeat CEF processors: # IP Geolocation Lookup @@ -45,6 +45,38 @@ processors: field: destination.as.organization_name target_field: destination.as.organization.name ignore_missing: true + - append: + field: related.hash + value: "{{cef.extensions.fileHash}}" + if: "ctx?.cef?.extensions?.fileHash != null" + - append: + field: related.hash + value: "{{cef.extensions.oldFileHash}}" + if: "ctx?.cef?.extensions?.oldFileHash != null" + - append: + field: related.ip + value: "{{destination.ip}}" + if: "ctx?.destination?.ip != null" + - append: + field: related.ip + value: "{{destination.nat.ip}}" + if: "ctx?.destination?.nat?.ip != null" + - append: + field: related.ip + value: "{{source.ip}}" + if: "ctx?.source?.ip != null" + - append: + field: related.ip + value: "{{source.nat.ip}}" + if: "ctx?.source?.nat?.ip != null" + - append: + field: related.user + value: "{{destination.user.name}}" + if: "ctx?.destination?.user?.name != null" + - append: + field: related.user + value: "{{source.user.name}}" + if: "ctx?.source?.user?.name != null" - pipeline: name: '{< IngestPipeline "fp-pipeline" >}' if: "ctx.cef?.device?.vendor == 'FORCEPOINT'" diff --git a/x-pack/filebeat/module/cef/log/test/cef.log b/x-pack/filebeat/module/cef/log/test/cef.log index 812cc45a78b..90e80c16e15 100644 --- a/x-pack/filebeat/module/cef/log/test/cef.log +++ b/x-pack/filebeat/module/cef/log/test/cef.log @@ -1 +1,2 @@ CEF:0|Elastic|Vaporware|1.0.0-alpha|18|Web request|low|eventId=3457 requestMethod=POST slat=38.915 slong=-77.511 proto=TCP sourceServiceName=httpd requestContext=https://www.google.com src=6.7.8.9 spt=33876 dst=192.168.10.1 dpt=443 request=https://www.example.com/cart +CEF:0|Elastic|Vaporware|1.0.0-alpha|18|Authentication|low|eventId=123 src=6.7.8.9 spt=33876 dst=1.2.3.4 dpt=443 duser=alice suser=bob destinationTranslatedAddress=10.10.10.10 fileHash=bc8bbe52f041fd17318f08a0f73762ce oldFileHash=a9796280592f86b74b27e370662d41eb diff --git a/x-pack/filebeat/module/cef/log/test/cef.log-expected.json b/x-pack/filebeat/module/cef/log/test/cef.log-expected.json index 4127423e588..5446c79c727 100644 --- a/x-pack/filebeat/module/cef/log/test/cef.log-expected.json +++ b/x-pack/filebeat/module/cef/log/test/cef.log-expected.json @@ -38,6 +38,10 @@ "observer.product": "Vaporware", "observer.vendor": "Elastic", "observer.version": "1.0.0-alpha", + "related.ip": [ + "192.168.10.1", + "6.7.8.9" + ], "service.type": "cef", "source.geo.continent_name": "North America", "source.geo.country_iso_code": "US", @@ -50,5 +54,72 @@ "cef" ], "url.original": "https://www.example.com/cart" + }, + { + "cef.device.event_class_id": "18", + "cef.device.product": "Vaporware", + "cef.device.vendor": "Elastic", + "cef.device.version": "1.0.0-alpha", + "cef.extensions.destinationAddress": "1.2.3.4", + "cef.extensions.destinationPort": 443, + "cef.extensions.destinationTranslatedAddress": "10.10.10.10", + "cef.extensions.destinationUserName": "alice", + "cef.extensions.eventId": 123, + "cef.extensions.fileHash": "bc8bbe52f041fd17318f08a0f73762ce", + "cef.extensions.oldFileHash": "a9796280592f86b74b27e370662d41eb", + "cef.extensions.sourceAddress": "6.7.8.9", + "cef.extensions.sourcePort": 33876, + "cef.extensions.sourceUserName": "bob", + "cef.name": "Authentication", + "cef.severity": "low", + "cef.version": "0", + "destination.geo.city_name": "Moscow", + "destination.geo.continent_name": "Europe", + "destination.geo.country_iso_code": "RU", + "destination.geo.location.lat": 55.7527, + "destination.geo.location.lon": 37.6172, + "destination.geo.region_iso_code": "RU-MOW", + "destination.geo.region_name": "Moscow", + "destination.ip": "1.2.3.4", + "destination.nat.ip": "10.10.10.10", + "destination.port": 443, + "destination.user.name": "alice", + "event.code": "18", + "event.dataset": "cef.log", + "event.id": 123, + "event.module": "cef", + "event.original": "CEF:0|Elastic|Vaporware|1.0.0-alpha|18|Authentication|low|eventId=123 src=6.7.8.9 spt=33876 dst=1.2.3.4 dpt=443 duser=alice suser=bob destinationTranslatedAddress=10.10.10.10 fileHash=bc8bbe52f041fd17318f08a0f73762ce oldFileHash=a9796280592f86b74b27e370662d41eb", + "event.severity": 0, + "fileset.name": "log", + "input.type": "log", + "log.offset": 269, + "message": "Authentication", + "observer.product": "Vaporware", + "observer.vendor": "Elastic", + "observer.version": "1.0.0-alpha", + "related.hash": [ + "bc8bbe52f041fd17318f08a0f73762ce", + "a9796280592f86b74b27e370662d41eb" + ], + "related.ip": [ + "1.2.3.4", + "10.10.10.10", + "6.7.8.9" + ], + "related.user": [ + "alice", + "bob" + ], + "service.type": "cef", + "source.geo.continent_name": "North America", + "source.geo.country_iso_code": "US", + "source.geo.location.lat": 37.751, + "source.geo.location.lon": -97.822, + "source.ip": "6.7.8.9", + "source.port": 33876, + "source.user.name": "bob", + "tags": [ + "cef" + ] } ] \ No newline at end of file diff --git a/x-pack/filebeat/module/cef/log/test/fp-ngfw-smc.log-expected.json b/x-pack/filebeat/module/cef/log/test/fp-ngfw-smc.log-expected.json index b421822914d..be322967983 100644 --- a/x-pack/filebeat/module/cef/log/test/fp-ngfw-smc.log-expected.json +++ b/x-pack/filebeat/module/cef/log/test/fp-ngfw-smc.log-expected.json @@ -106,6 +106,10 @@ "observer.product": "Firewall", "observer.vendor": "FORCEPOINT", "observer.version": "6.6.1", + "related.ip": [ + "10.1.1.40", + "10.37.205.252" + ], "rule.id": "2097157.1", "service.type": "cef", "source.ip": "10.37.205.252", @@ -154,6 +158,10 @@ "observer.product": "Firewall", "observer.vendor": "FORCEPOINT", "observer.version": "unknown", + "related.ip": [ + "255.255.255.255", + "172.16.1.1" + ], "rule.id": "605.0", "service.type": "cef", "source.ip": "172.16.1.1", @@ -202,6 +210,10 @@ "observer.product": "Firewall", "observer.vendor": "FORCEPOINT", "observer.version": "unknown", + "related.ip": [ + "192.168.1.1", + "172.16.1.1" + ], "rule.id": "601.0", "service.type": "cef", "source.ip": "172.16.1.1", @@ -247,6 +259,9 @@ "observer.product": "Firewall", "observer.vendor": "FORCEPOINT", "observer.version": "unknown", + "related.user": [ + "alice" + ], "service.type": "cef", "source.bytes": 32526, "source.user.name": "alice", @@ -283,6 +298,12 @@ "observer.product": "Firewall", "observer.vendor": "FORCEPOINT", "observer.version": "unknown", + "related.ip": [ + "192.168.1.1" + ], + "related.user": [ + "bob" + ], "service.type": "cef", "source.ip": "192.168.1.1", "source.user.name": "bob", @@ -319,6 +340,12 @@ "observer.product": "Firewall", "observer.vendor": "FORCEPOINT", "observer.version": "unknown", + "related.ip": [ + "192.168.1.1" + ], + "related.user": [ + "bob" + ], "service.type": "cef", "source.ip": "192.168.1.1", "source.user.name": "bob", @@ -355,6 +382,12 @@ "observer.product": "Firewall", "observer.vendor": "FORCEPOINT", "observer.version": "unknown", + "related.ip": [ + "172.16.2.1" + ], + "related.user": [ + "alice" + ], "service.type": "cef", "source.ip": "172.16.2.1", "source.user.name": "alice",