diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 2b5b15edc52..d6ec4de1db1 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -834,6 +834,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Updated microsoft defender_atp and m365_defender to ECS 1.8. {pull}23897[23897] {issue}23118[23118] - Updated o365 module to ECS 1.8. {issue}23118[23118] {pull}23896[23896] - Upgrade CEF module to ECS 1.8.0. {pull}23832[23832] +- Upgrade fortinet/firewall to ECS 1.8 {issue}23118[23118] {pull}23902[23902] *Heartbeat* diff --git a/x-pack/filebeat/module/fortinet/firewall/config/firewall.yml b/x-pack/filebeat/module/fortinet/firewall/config/firewall.yml index cddd13573a4..e455019cfdc 100644 --- a/x-pack/filebeat/module/fortinet/firewall/config/firewall.yml +++ b/x-pack/filebeat/module/fortinet/firewall/config/firewall.yml @@ -27,7 +27,7 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.7.0 + ecs.version: 1.8.0 {{ if .external_interfaces }} - add_fields: diff --git a/x-pack/filebeat/module/fortinet/firewall/ingest/event.yml b/x-pack/filebeat/module/fortinet/firewall/ingest/event.yml index 8278c538c26..4e299f4be08 100644 --- a/x-pack/filebeat/module/fortinet/firewall/ingest/event.yml +++ b/x-pack/filebeat/module/fortinet/firewall/ingest/event.yml @@ -242,93 +242,6 @@ processors: type: integer ignore_failure: true ignore_missing: true -- geoip: - field: source.ip - target_field: source.geo - ignore_missing: true - if: "ctx.source?.geo == null" -- geoip: - field: destination.ip - target_field: destination.geo - ignore_missing: true - if: "ctx.destination?.geo == null" -- geoip: - database_file: GeoLite2-ASN.mmdb - field: source.ip - target_field: source.as - properties: - - asn - - organization_name - ignore_missing: true -- geoip: - database_file: GeoLite2-ASN.mmdb - field: destination.ip - target_field: destination.as - properties: - - asn - - organization_name - ignore_missing: true -- geoip: - field: source.nat.ip - target_field: source.geo - ignore_missing: true - if: "ctx.source?.geo == null" -- geoip: - field: destination.nat.ip - target_field: destination.geo - ignore_missing: true - if: "ctx.destination?.geo == null" -- geoip: - database_file: GeoLite2-ASN.mmdb - field: source.nat.ip - target_field: source.as - properties: - - asn - - organization_name - ignore_missing: true - if: "ctx.source?.as == null" -- geoip: - database_file: GeoLite2-ASN.mmdb - field: destination.nat.ip - target_field: destination.as - properties: - - asn - - organization_name - ignore_missing: true - if: "ctx.destination?.as == null" -- rename: - field: source.as.asn - target_field: source.as.number - ignore_missing: true -- rename: - field: source.as.organization_name - target_field: source.as.organization.name - ignore_missing: true -- rename: - field: destination.as.asn - target_field: destination.as.number - ignore_missing: true -- rename: - field: destination.as.organization_name - target_field: destination.as.organization.name - ignore_missing: true -- script: - lang: painless - source: ctx.network.bytes = ctx.source.bytes + ctx.destination.bytes - if: "ctx?.source?.bytes != null && ctx?.destination?.bytes != null" - ignore_failure: true -- append: - field: related.ip - value: "{{source.ip}}" - if: "ctx.source?.ip != null" -- append: - field: related.ip - value: "{{destination.ip}}" - if: "ctx.destination?.ip != null" -- append: - field: related.user - value: "{{source.user.name}}" - if: "ctx.source?.user?.name != null" - remove: field: - fortinet.firewall.dstport diff --git a/x-pack/filebeat/module/fortinet/firewall/ingest/pipeline.yml b/x-pack/filebeat/module/fortinet/firewall/ingest/pipeline.yml index a227d770082..c103fd14700 100644 --- a/x-pack/filebeat/module/fortinet/firewall/ingest/pipeline.yml +++ b/x-pack/filebeat/module/fortinet/firewall/ingest/pipeline.yml @@ -15,14 +15,17 @@ processors: ignore_missing: true ignore_failure: false trim_value: "\"" -- remove: - field: fortinet.tmp.assignip - if: "ctx.fortinet?.tmp?.assignip == 'N/A'" - ignore_missing: true - rename: field: fortinet.tmp target_field: fortinet.firewall ignore_missing: true +- script: + lang: painless + source: | + def fw = ctx?.fortinet?.firewall; + if (fw != null) { + fw.entrySet().removeIf(entry -> entry.getValue() == "N/A"); + } - set: field: observer.vendor value: Fortinet @@ -134,36 +137,6 @@ processors: field: fortinet.firewall.level target_field: log.level ignore_missing: true -- remove: - field: fortinet.firewall.assignip - if: "ctx.fortinet?.firewall?.assignip == 'N/A'" -- remove: - field: fortinet.firewall.dstip - if: "ctx.fortinet?.firewall?.dstip == 'N/A'" -- remove: - field: fortinet.firewall.srcip - if: "ctx.fortinet?.firewall?.srcip == 'N/A'" -- remove: - field: fortinet.firewall.remip - if: "ctx.fortinet?.firewall?.remip == 'N/A'" -- remove: - field: fortinet.firewall.locip - if: "ctx.fortinet?.firewall?.locip == 'N/A'" -- remove: - field: fortinet.firewall.group - if: "ctx.fortinet?.firewall?.group == 'N/A'" -- remove: - field: fortinet.firewall.user - if: "ctx.fortinet?.firewall?.user == 'N/A'" -- remove: - field: fortinet.firewall.tranip - if: "ctx.fortinet?.firewall?.tranip == 'N/A'" -- remove: - field: fortinet.firewall.transip - if: "ctx.fortinet?.firewall?.transip == 'N/A'" -- remove: - field: fortinet.firewall.tunnelip - if: "ctx.fortinet?.firewall?.tunnelip == 'N/A'" # Handle interface-based network directionality - set: field: network.direction @@ -259,6 +232,128 @@ processors: field: fortinet.firewall.size type: long ignore_missing: true +- geoip: + field: source.ip + target_field: source.geo + ignore_missing: true + if: "ctx.source?.geo == null" +- geoip: + field: destination.ip + target_field: destination.geo + ignore_missing: true + if: "ctx.destination?.geo == null" +- geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true +- geoip: + database_file: GeoLite2-ASN.mmdb + field: destination.ip + target_field: destination.as + properties: + - asn + - organization_name + ignore_missing: true +- geoip: + field: source.nat.ip + target_field: source.geo + ignore_missing: true + if: "ctx.source?.geo == null" +- geoip: + field: destination.nat.ip + target_field: destination.geo + ignore_missing: true + if: "ctx.destination?.geo == null" +- geoip: + database_file: GeoLite2-ASN.mmdb + field: source.nat.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true + if: "ctx.source?.as == null" +- geoip: + database_file: GeoLite2-ASN.mmdb + field: destination.nat.ip + target_field: destination.as + properties: + - asn + - organization_name + ignore_missing: true + if: "ctx.destination?.as == null" +- rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true +- rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true +- rename: + field: destination.as.asn + target_field: destination.as.number + ignore_missing: true +- rename: + field: destination.as.organization_name + target_field: destination.as.organization.name + ignore_missing: true +- script: + lang: painless + source: "ctx.network.bytes = ctx.source.bytes + ctx.destination.bytes" + if: "ctx?.source?.bytes != null && ctx?.destination?.bytes != null" + ignore_failure: true +- script: + lang: painless + source: "ctx.network.packets = ctx.source.packets + ctx.destination.packets" + if: "ctx?.source?.packets != null && ctx?.destination?.packets != null" + ignore_failure: true +- append: + field: related.ip + value: "{{source.ip}}" + if: "ctx.source?.ip != null" +- append: + field: related.ip + value: "{{destination.ip}}" + if: "ctx.destination?.ip != null" +- append: + field: related.user + value: "{{source.user.name}}" + if: "ctx.source?.user?.name != null" +- append: + field: related.user + value: "{{destination.user.name}}" + if: "ctx.destination?.user?.name != null" +- append: + field: related.hosts + value: "{{destination.address}}" + if: "ctx.destination?.address != null" +- append: + field: related.hosts + value: "{{source.address}}" + if: "ctx.source?.address != null" +- append: + field: related.hosts + value: "{{dns.question.name}}" + if: "ctx.dns?.question?.name != null" +- script: + lang: painless + source: | + def dnsIPs = ctx?.dns?.resolved_ip; + if (dnsIPs != null && dnsIPs instanceof List) { + if (ctx?.related?.ip == null) { + ctx.related.ip = []; + } + for (ip in dnsIPs) { + if (!ctx.related.ip.contains(ip)) { + ctx.related.ip.add(ip); + } + } + } on_failure: - set: field: error.message diff --git a/x-pack/filebeat/module/fortinet/firewall/ingest/traffic.yml b/x-pack/filebeat/module/fortinet/firewall/ingest/traffic.yml index 051a3eca2f8..5166332e2a1 100644 --- a/x-pack/filebeat/module/fortinet/firewall/ingest/traffic.yml +++ b/x-pack/filebeat/module/fortinet/firewall/ingest/traffic.yml @@ -200,102 +200,6 @@ processors: field: fortinet.firewall.url target_field: url.path ignore_missing: true -- geoip: - field: source.ip - target_field: source.geo - ignore_missing: true - if: "ctx.source?.geo == null" -- geoip: - field: destination.ip - target_field: destination.geo - ignore_missing: true - if: "ctx.destination?.geo == null" -- geoip: - database_file: GeoLite2-ASN.mmdb - field: source.ip - target_field: source.as - properties: - - asn - - organization_name - ignore_missing: true -- geoip: - database_file: GeoLite2-ASN.mmdb - field: destination.ip - target_field: destination.as - properties: - - asn - - organization_name - ignore_missing: true -- geoip: - field: source.nat.ip - target_field: source.geo - ignore_missing: true - if: "ctx.source?.geo == null" -- geoip: - field: destination.nat.ip - target_field: destination.geo - ignore_missing: true - if: "ctx.destination?.geo == null" -- geoip: - database_file: GeoLite2-ASN.mmdb - field: source.nat.ip - target_field: source.as - properties: - - asn - - organization_name - ignore_missing: true - if: "ctx.source?.as == null" -- geoip: - database_file: GeoLite2-ASN.mmdb - field: destination.nat.ip - target_field: destination.as - properties: - - asn - - organization_name - ignore_missing: true - if: "ctx.destination?.as == null" -- rename: - field: source.as.asn - target_field: source.as.number - ignore_missing: true -- rename: - field: source.as.organization_name - target_field: source.as.organization.name - ignore_missing: true -- rename: - field: destination.as.asn - target_field: destination.as.number - ignore_missing: true -- rename: - field: destination.as.organization_name - target_field: destination.as.organization.name - ignore_missing: true -- script: - lang: painless - source: "ctx.network.bytes = ctx.source.bytes + ctx.destination.bytes" - if: "ctx?.source?.bytes != null && ctx?.destination?.bytes != null" - ignore_failure: true -- script: - lang: painless - source: "ctx.network.packets = ctx.source.packets + ctx.destination.packets" - if: "ctx?.source?.packets != null && ctx?.destination?.packets != null" - ignore_failure: true -- append: - field: related.ip - value: "{{source.ip}}" - if: "ctx.source?.ip != null" -- append: - field: related.ip - value: "{{destination.ip}}" - if: "ctx.destination?.ip != null" -- append: - field: related.user - value: "{{source.user.name}}" - if: "ctx.source?.user?.name != null" -- append: - field: related.user - value: "{{destination.user.name}}" - if: "ctx.destination?.user?.name != null" - remove: field: - fortinet.firewall.dstport @@ -310,4 +214,4 @@ processors: on_failure: - set: field: error.message - value: '{{ _ingest.on_failure_message }}' \ No newline at end of file + value: '{{ _ingest.on_failure_message }}' diff --git a/x-pack/filebeat/module/fortinet/firewall/ingest/utm.yml b/x-pack/filebeat/module/fortinet/firewall/ingest/utm.yml index e3df460546c..a788aa4c8bc 100644 --- a/x-pack/filebeat/module/fortinet/firewall/ingest/utm.yml +++ b/x-pack/filebeat/module/fortinet/firewall/ingest/utm.yml @@ -348,93 +348,6 @@ processors: field: fortinet.firewall.filehash target_field: fortinet.file.hash.crc32 ignore_missing: true -- geoip: - field: source.ip - target_field: source.geo - ignore_missing: true - if: "ctx.source?.geo == null" -- geoip: - field: destination.ip - target_field: destination.geo - ignore_missing: true - if: "ctx.destination?.geo == null" -- geoip: - database_file: GeoLite2-ASN.mmdb - field: source.ip - target_field: source.as - properties: - - asn - - organization_name - ignore_missing: true -- geoip: - database_file: GeoLite2-ASN.mmdb - field: destination.ip - target_field: destination.as - properties: - - asn - - organization_name - ignore_missing: true -- geoip: - field: source.nat.ip - target_field: source.geo - ignore_missing: true - if: "ctx.source?.geo == null" -- geoip: - field: destination.nat.ip - target_field: destination.geo - ignore_missing: true - if: "ctx.destination?.geo == null" -- geoip: - database_file: GeoLite2-ASN.mmdb - field: source.nat.ip - target_field: source.as - properties: - - asn - - organization_name - ignore_missing: true - if: "ctx.source?.as == null" -- geoip: - database_file: GeoLite2-ASN.mmdb - field: destination.nat.ip - target_field: destination.as - properties: - - asn - - organization_name - ignore_missing: true - if: "ctx.destination?.as == null" -- rename: - field: source.as.asn - target_field: source.as.number - ignore_missing: true -- rename: - field: source.as.organization_name - target_field: source.as.organization.name - ignore_missing: true -- rename: - field: destination.as.asn - target_field: destination.as.number - ignore_missing: true -- rename: - field: destination.as.organization_name - target_field: destination.as.organization.name - ignore_missing: true -- script: - lang: painless - source: "ctx.network.bytes = ctx.source.bytes + ctx.destination.bytes" - if: "ctx?.source?.bytes != null && ctx?.destination?.bytes != null" - ignore_failure: true -- append: - field: related.ip - value: "{{source.ip}}" - if: "ctx.source?.ip != null" -- append: - field: related.ip - value: "{{destination.ip}}" - if: "ctx.destination?.ip != null" -- append: - field: related.user - value: "{{source.user.name}}" - if: "ctx.source?.user?.name != null" - append: field: related.hash value: "{{fortinet.file.hash.crc32}}" diff --git a/x-pack/filebeat/module/fortinet/firewall/test/fortinet.log-expected.json b/x-pack/filebeat/module/fortinet/firewall/test/fortinet.log-expected.json index 2a485f787f4..172748796d1 100644 --- a/x-pack/filebeat/module/fortinet/firewall/test/fortinet.log-expected.json +++ b/x-pack/filebeat/module/fortinet/firewall/test/fortinet.log-expected.json @@ -427,6 +427,9 @@ "observer.serial_number": "somerouterid", "observer.type": "firewall", "observer.vendor": "Fortinet", + "related.hosts": [ + "elastic.example.com" + ], "related.ip": [ "192.168.2.1", "8.8.8.8" @@ -498,9 +501,13 @@ "observer.serial_number": "somerouterid", "observer.type": "firewall", "observer.vendor": "Fortinet", + "related.hosts": [ + "elastic.example.com" + ], "related.ip": [ "192.168.2.1", - "8.8.8.8" + "8.8.8.8", + "8.8.4.4" ], "rule.category": "Web-based Email", "rule.id": "26", @@ -642,6 +649,9 @@ "observer.serial_number": "somerouterid", "observer.type": "firewall", "observer.vendor": "Fortinet", + "related.hosts": [ + "elastic.co" + ], "related.ip": [ "192.168.2.1", "8.8.8.8" @@ -704,6 +714,9 @@ "observer.serial_number": "somerouterid", "observer.type": "firewall", "observer.vendor": "Fortinet", + "related.hosts": [ + "elastic.co" + ], "related.ip": [ "192.168.2.1", "8.8.8.8" @@ -864,9 +877,6 @@ "fortinet.firewall.subtype": "vpn", "fortinet.firewall.type": "event", "fortinet.firewall.vd": "root", - "fortinet.firewall.vpntunnel": "N/A", - "fortinet.firewall.xauthgroup": "N/A", - "fortinet.firewall.xauthuser": "N/A", "input.type": "log", "log.level": "error", "log.offset": 7112, @@ -934,8 +944,6 @@ "fortinet.firewall.type": "event", "fortinet.firewall.vd": "root", "fortinet.firewall.vpntunnel": "elasticvpn", - "fortinet.firewall.xauthgroup": "N/A", - "fortinet.firewall.xauthuser": "N/A", "input.type": "log", "log.level": "notice", "log.offset": 7680, @@ -1096,8 +1104,6 @@ "fortinet.firewall.type": "event", "fortinet.firewall.vd": "root", "fortinet.firewall.vpntunnel": "testvpn", - "fortinet.firewall.xauthgroup": "N/A", - "fortinet.firewall.xauthuser": "N/A", "input.type": "log", "log.level": "notice", "log.offset": 9122, @@ -1198,7 +1204,6 @@ }, { "@timestamp": "2020-04-23T12:23:47.000-05:00", - "destination.address": "N/A", "destination.as.number": 15169, "destination.as.organization.name": "Google LLC", "destination.geo.continent_name": "North America", @@ -1221,7 +1226,6 @@ ], "fileset.name": "firewall", "fortinet.firewall.action": "ssl-new-con", - "fortinet.firewall.reason": "N/A", "fortinet.firewall.subtype": "vpn", "fortinet.firewall.tunnelid": "2", "fortinet.firewall.tunneltype": "ssl", @@ -1248,7 +1252,6 @@ }, { "@timestamp": "2020-04-23T12:23:47.000-05:00", - "destination.address": "N/A", "destination.as.number": 3356, "destination.as.organization.name": "Level 3 Parent, LLC", "destination.geo.continent_name": "North America", @@ -2005,8 +2008,6 @@ "fortinet.firewall.type": "event", "fortinet.firewall.vd": "root", "fortinet.firewall.vpntunnel": "P1_Test", - "fortinet.firewall.xauthgroup": "N/A", - "fortinet.firewall.xauthuser": "N/A", "input.type": "log", "log.level": "notice", "log.offset": 17123,