diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 1e083caa4223..418f5a6b198c 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -827,6 +827,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add `include_s3_metadata` config option to the `aws-s3` input for including object metadata in events. {pull}26267[26267] - RFC 5424 and UNIX socket support in the Syslog input are now GA {pull}26293[26293] - Update grok patterns for HA Proxy module {issue}25827[25827] {pull}25835[25835] +- Update PanOS module's date processor formats to parse `strict_date_optional_time_nanos`. {issue}26033[26033] {pull}26158[26158] - Update Okta module to parse additional fields to `okta.debug_context.debug_data`. {issue}25689[25689] {pull}25818[25818] - Added dataset `anomalithreatstream` to the `threatintel` module to ingest indicators from Anomali ThreatStream {pull}26350[26350] - Add support for `copytruncate` method when rotating input logs with an external tool in `filestream` input. {pull}23457[23457] diff --git a/x-pack/filebeat/module/panw/panos/ingest/pipeline.yml b/x-pack/filebeat/module/panw/panos/ingest/pipeline.yml index cbb7dad91b36..10cbe1d3511d 100644 --- a/x-pack/filebeat/module/panw/panos/ingest/pipeline.yml +++ b/x-pack/filebeat/module/panw/panos/ingest/pipeline.yml @@ -23,12 +23,14 @@ processors: field: "_temp_.generated_time" formats: - "yyyy/MM/dd HH:mm:ss" + - "strict_date_optional_time_nanos" on_failure: [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}] - date: if: "ctx.event.timezone != null" field: "_temp_.generated_time" formats: - "yyyy/MM/dd HH:mm:ss" + - "strict_date_optional_time_nanos" timezone: "{{ event.timezone }}" on_failure: [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}] @@ -39,6 +41,7 @@ processors: target_field: "event.created" formats: - "yyyy/MM/dd HH:mm:ss" + - "strict_date_optional_time_nanos" on_failure: [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}] - date: if: "ctx.event.timezone != null && ctx.event.created != null " @@ -46,6 +49,7 @@ processors: target_field: "event.created" formats: - "yyyy/MM/dd HH:mm:ss" + - "strict_date_optional_time_nanos" timezone: "{{ event.timezone }}" on_failure: [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}] @@ -56,6 +60,7 @@ processors: target_field: "event.start" formats: - "yyyy/MM/dd HH:mm:ss" + - "strict_date_optional_time_nanos" on_failure: [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}] - date: if: "ctx.event.timezone != null && ctx.event.start != null" @@ -64,6 +69,7 @@ processors: timezone: "{{ event.timezone }}" formats: - "yyyy/MM/dd HH:mm:ss" + - "strict_date_optional_time_nanos" on_failure: [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}] # convert integer fields as the output of the CSV processor is always a string. diff --git a/x-pack/filebeat/module/panw/panos/test/global_protect.log-expected.json b/x-pack/filebeat/module/panw/panos/test/global_protect.log-expected.json index df1158db6990..02583d8ae595 100644 --- a/x-pack/filebeat/module/panw/panos/test/global_protect.log-expected.json +++ b/x-pack/filebeat/module/panw/panos/test/global_protect.log-expected.json @@ -89,8 +89,8 @@ "panw.panos.type": "GLOBALPROTECT", "panw.panos.virtual_sys": "vsys1", "related.hosts": [ - "GlobalProtect_GW", - "CP935" + "CP935", + "GlobalProtect_GW" ], "related.ip": [ "10.20.13.217", @@ -368,8 +368,8 @@ "source.nat.ip": "10.20.30.40", "source.user.name": "maxmustermann", "tags": [ - "pan-os", - "forwarded" + "forwarded", + "pan-os" ], "user.name": "maxmustermann" }, @@ -432,8 +432,8 @@ "source.user.domain": "domain", "source.user.name": "musterman", "tags": [ - "pan-os", - "forwarded" + "forwarded", + "pan-os" ], "user.domain": "domain", "user.name": "musterman" @@ -493,8 +493,8 @@ "source.user.domain": "domain.de", "source.user.name": "Max.Mustermann", "tags": [ - "pan-os", - "forwarded" + "forwarded", + "pan-os" ], "user.domain": "domain.de", "user.name": "Max.Mustermann" @@ -559,8 +559,8 @@ "source.user.domain": "domain", "source.user.name": "maxmustermann", "tags": [ - "pan-os", - "forwarded" + "forwarded", + "pan-os" ], "user.domain": "domain", "user.name": "maxmustermann" diff --git a/x-pack/filebeat/module/panw/panos/test/hipmatch.log-expected.json b/x-pack/filebeat/module/panw/panos/test/hipmatch.log-expected.json index 5f12cd767668..3423326e91c1 100644 --- a/x-pack/filebeat/module/panw/panos/test/hipmatch.log-expected.json +++ b/x-pack/filebeat/module/panw/panos/test/hipmatch.log-expected.json @@ -33,8 +33,8 @@ "panw.panos.virtual_sys": "vsys1", "panw.panos.vsys_id": "1", "related.hosts": [ - "de-firewall", - "PC12345" + "PC12345", + "de-firewall" ], "related.ip": [ "10.20.30.40" @@ -48,8 +48,8 @@ "source.user.domain": "domain", "source.user.name": "mustermanm", "tags": [ - "pan-os", - "forwarded" + "forwarded", + "pan-os" ], "user.domain": "domain", "user.name": "mustermanm" @@ -113,8 +113,8 @@ "source.ip": "67.240.185.235", "source.user.name": "ira", "tags": [ - "pan-os", - "forwarded" + "forwarded", + "pan-os" ], "user.name": "ira" } diff --git a/x-pack/filebeat/module/panw/panos/test/traffic_nanos_time.log b/x-pack/filebeat/module/panw/panos/test/traffic_nanos_time.log new file mode 100644 index 000000000000..92ed8cf89472 --- /dev/null +++ b/x-pack/filebeat/module/panw/panos/test/traffic_nanos_time.log @@ -0,0 +1 @@ +Oct 30 09:46:42 1,2021-05-26T16:27:07.000000Z,no-serial,TRAFFIC,end,9.1,2021-05-26T16:26:47.000000Z,127.0.0.0,127.0.0.1,0.0.0.0,0.0.0.0,intrazone-default,,,web-browsing,vsys1,untrust,untrust,ethernet1/1,ethernet1/1,Cortex Data Lake,,688290,1,35834,443,35834,20077,0x1400070,tcp,allow,7291,1696,5595,21,2021-05-26T16:26:30.000000Z,1,medium-risk,,620386,0x8800000000000000,US,SG,,14,7,tcp-fin,22,18,0,0,,GP cloud service,from-policy,,,0,,0,1970-01-01T00:00:00.000000Z,N/A,0,0,0,0,6a2f6161-88f2-4afc-8dd5-256bc4505a64,0,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, diff --git a/x-pack/filebeat/module/panw/panos/test/traffic_nanos_time.log-expected.json b/x-pack/filebeat/module/panw/panos/test/traffic_nanos_time.log-expected.json new file mode 100644 index 000000000000..fc646d76dc00 --- /dev/null +++ b/x-pack/filebeat/module/panw/panos/test/traffic_nanos_time.log-expected.json @@ -0,0 +1,106 @@ +[ + { + "@timestamp": "2021-05-26T16:26:47.000Z", + "client.bytes": 1696, + "client.ip": "127.0.0.0", + "client.nat.ip": "0.0.0.0", + "client.nat.port": 35834, + "client.packets": 14, + "client.port": 35834, + "destination.address": "127.0.0.1", + "destination.bytes": 5595, + "destination.ip": "127.0.0.1", + "destination.nat.ip": "0.0.0.0", + "destination.nat.port": 20077, + "destination.packets": 7, + "destination.port": 443, + "event.action": "flow_terminated", + "event.category": [ + "network", + "network_traffic" + ], + "event.dataset": "panw.panos", + "event.duration": 1000000000, + "event.end": "2021-05-26T16:26:31.000Z", + "event.kind": "event", + "event.module": "panw", + "event.outcome": "success", + "event.start": "2021-05-26T16:26:30.000Z", + "event.timezone": "-02:00", + "event.type": [ + "allowed", + "connection", + "end" + ], + "fileset.name": "panos", + "input.type": "log", + "labels.nat_translated": true, + "labels.ssl_decrypted": true, + "log.offset": 0, + "log.original": "Oct 30 09:46:42 1,2021-05-26T16:27:07.000000Z,no-serial,TRAFFIC,end,9.1,2021-05-26T16:26:47.000000Z,127.0.0.0,127.0.0.1,0.0.0.0,0.0.0.0,intrazone-default,,,web-browsing,vsys1,untrust,untrust,ethernet1/1,ethernet1/1,Cortex Data Lake,,688290,1,35834,443,35834,20077,0x1400070,tcp,allow,7291,1696,5595,21,2021-05-26T16:26:30.000000Z,1,medium-risk,,620386,0x8800000000000000,US,SG,,14,7,tcp-fin,22,18,0,0,,GP cloud service,from-policy,,,0,,0,1970-01-01T00:00:00.000000Z,N/A,0,0,0,0,6a2f6161-88f2-4afc-8dd5-256bc4505a64,0,0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,", + "network.application": "web-browsing", + "network.bytes": 7291, + "network.community_id": [ + "1:lME0D6scndGsx6dABDTbtWkIb3E=", + "1:q5HHCKGDtoHfI//AqHbOlmLMsRQ=" + ], + "network.direction": "external", + "network.packets": 21, + "network.transport": "tcp", + "network.type": "ipv4", + "observer.egress.interface.name": "ethernet1/1", + "observer.egress.zone": "untrust", + "observer.hostname": "GP cloud service", + "observer.ingress.interface.name": "ethernet1/1", + "observer.ingress.zone": "untrust", + "observer.product": "PAN-OS", + "observer.serial_number": "no-serial", + "observer.type": "firewall", + "observer.vendor": "Palo Alto Networks", + "panw.panos.action": "allow", + "panw.panos.destination.interface": "ethernet1/1", + "panw.panos.destination.nat.ip": "0.0.0.0", + "panw.panos.destination.nat.port": 20077, + "panw.panos.destination.zone": "untrust", + "panw.panos.endreason": "tcp-fin", + "panw.panos.flow_id": "688290", + "panw.panos.network.nat.community_id": "1:lME0D6scndGsx6dABDTbtWkIb3E=", + "panw.panos.ruleset": "intrazone-default", + "panw.panos.sequence_number": 620386, + "panw.panos.source.interface": "ethernet1/1", + "panw.panos.source.nat.ip": "0.0.0.0", + "panw.panos.source.nat.port": 35834, + "panw.panos.source.zone": "untrust", + "panw.panos.sub_type": "end", + "panw.panos.type": "TRAFFIC", + "panw.panos.url.category": "medium-risk", + "panw.panos.virtual_sys": "vsys1", + "related.hosts": [ + "GP cloud service" + ], + "related.ip": [ + "0.0.0.0", + "127.0.0.0", + "127.0.0.1" + ], + "rule.name": "intrazone-default", + "server.bytes": 5595, + "server.ip": "127.0.0.1", + "server.nat.ip": "0.0.0.0", + "server.nat.port": 20077, + "server.packets": 7, + "server.port": 443, + "service.type": "panw", + "source.address": "127.0.0.0", + "source.bytes": 1696, + "source.ip": "127.0.0.0", + "source.nat.ip": "0.0.0.0", + "source.nat.port": 35834, + "source.packets": 14, + "source.port": 35834, + "tags": [ + "forwarded", + "pan-os" + ] + } +] \ No newline at end of file