[Filebeat][Fortinet Module] Small bugfixes for time formats and IP ad…

…dress arrays (elastic#19316)

* added split for when dns is array
* only fix eventtime if the time is not in seconds
* added new timezone format option
* new testlogs has been added
* updated changelog
* fixing format set for events with non timezones

(cherry picked from commit 39961ad)

CHANGELOG.next.asciidoc

@@ -119,6 +119,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix `o365` module ignoring `var.api` settings. {pull}18948[18948]
 - Fix `netflow` module to support 7 bytepad for IPFIX template. {issue}18098[18098]
 - Fix Cisco ASA ASA 3020** and 106023 messages {pull}17964[17964]
+- Fix date and timestamp formats for fortigate module {pull}19316[19316]
 
 *Heartbeat*
 

x-pack/filebeat/module/fortinet/firewall/ingest/pipeline.yml

@@ -37,14 +37,15 @@ processors:
     if: "ctx.fortinet?.firewall?.tz != null"
 - set:
     field: _temp.time
-    value: "{{fortinet.firewall.date}} {{fortinet.firewall.time}}Z"
+    value: "{{fortinet.firewall.date}} {{fortinet.firewall.time}}"
     if: "ctx.fortinet?.firewall?.tz == null"
 - date:
     field: _temp.time
     target_field: "@timestamp"
     formats:
     - yyyy-MM-dd HH:mm:ss
     - yyyy-MM-dd HH:mm:ss Z
+    - yyyy-MM-dd HH:mm:ss z
     - ISO8601
     timezone: "{{fortinet.firewall.tz}}"
     if: "ctx.fortinet?.firewall?.tz != null"
@@ -54,18 +55,27 @@ processors:
     formats:
     - yyyy-MM-dd HH:mm:ss
     - yyyy-MM-dd HH:mm:ss Z
+    - yyyy-MM-dd HH:mm:ss z
     - ISO8601
     if: "ctx.fortinet?.firewall?.tz == null"
 - gsub:
     field: fortinet.firewall.eventtime
     pattern: "\\d{6}$"
     replacement: ""
+    if: "(ctx.fortinet?.firewall?.eventtime).length() > 18"
 - date:
     field: fortinet.firewall.eventtime
     target_field: event.start
     formats:
     - UNIX_MS
     timezone: "{{fortinet.firewall.tz}}"
+    if: "ctx.fortinet?.firewall?.tz != null"
+- date:
+    field: fortinet.firewall.eventtime
+    target_field: event.start
+    formats:
+    - UNIX_MS
+    if: "ctx.fortinet?.firewall?.tz == null"
 - rename:
     field: fortinet.firewall.devname
     target_field: observer.name

x-pack/filebeat/module/fortinet/firewall/ingest/utm.yml

@@ -230,6 +230,10 @@ processors:
     field: fortinet.firewall.ipaddr
     target_field: dns.resolved_ip
     ignore_missing: true
+- split:
+    field: dns.resolved_ip
+    separator: ', '
+    ignore_missing: true
 - rename:
     field: fortinet.firewall.level
     target_field: log.level

x-pack/filebeat/module/fortinet/firewall/test/fortinet.log

@@ -1,7 +1,10 @@
 <188>date=2020-04-23 time=12:17:48 devname="testswitch1" devid="somerouterid" logid="0316013056" type="utm" subtype="webfilter" eventtype="ftgd_blk" level="warning" vd="root" eventtime=1587230269052907555 tz="-0500" policyid=100602 sessionid=1234 user="elasticuser" group="elasticgroup" authserver="elasticauth" srcip=192.168.2.1 srcport=61930 srcintf="port1" srcintfrole="lan" dstip=8.8.8.8 dstport=443 dstintf="wan1" dstintfrole="wan" proto=6 service="HTTPS" hostname="elastic.co" profile="elasticruleset" action="blocked" reqtype="direct" url="/config/" sentbyte=1152 rcvdbyte=1130 direction="outgoing" msg="URL belongs to a denied category in policy" method="domain" cat=76 catdesc="Internet Telephony"
+<189>date=2020-04-23 time=01:16:08 devname="testswitch1" devid="somerouterid" logid="0000000013" type="traffic" subtype="forward" level="notice" vd="OPERATIONAL" eventtime=1592961368 srcip=10.10.10.10 srcport=60899 srcintf="srcintfname" srcintfrole="lan" dstip=8.8.8.8 dstport=161 dstintf="dstintfname" dstintfrole="lan" sessionid=155313 proto=17 action="deny" policyid=0 policytype="policy" service="SNMP" dstcountry="Reserved" srccountry="Reserved" trandisp="noop" duration=0 sentbyte=0 rcvdbyte=0 sentpkt=0 appcat="unscanned" crscore=30 craction=131072 crlevel="high"
 <189>date=2020-04-23 time=12:17:45 devname="testswitch1" devid="somerouterid" logid="0317013312" type="utm" subtype="webfilter" eventtype="ftgd_allow" level="notice" vd="root" eventtime=1587230266314799756 tz="-0500" policyid=38 sessionid=543234 user="elasticuser" group="elasticgroup" authserver="elasticauth" srcip=192.168.2.1 srcport=65236 srcintf="port1" srcintfrole="lan" dstip=8.8.8.8 dstport=443 dstintf="wan1" dstintfrole="wan" proto=6 service="HTTPS" hostname="elastic.co" profile="elasticruleset" action="passthrough" reqtype="direct" url="/" sentbyte=3545 rcvdbyte=6812 direction="outgoing" msg="URL belongs to an allowed category in policy" method="domain" cat=23 catdesc="Web-based Email"
 <190>date=2020-04-23 time=13:17:35 devname="testswitch1" devid="somerouterid" logid="1059028704" type="utm" subtype="app-ctrl" eventtype="signature" level="information" vd="root" eventtime=1587230255061492894 tz="-0400" appid=40568 user="elasticuser" group="elasticgroup" authserver="elasticauth" srcip=192.168.2.1 dstip=8.8.8.8 srcport=59790 dstport=443 srcintf="LAN" srcintfrole="lan" dstintf="wan1" dstintfrole="wan" proto=6 service="SSL" direction="outgoing" policyid=12 sessionid=453234 applist="elasticruleset" action="pass" appcat="Web.Client" app="HTTPS.BROWSER" hostname="elastic.co" incidentserialno=23465 url="/" msg="Web.Client: HTTPS.BROWSER," apprisk="medium" scertcname="test.elastic.co"
+<190>date=2020-04-23 time=13:17:35 devname="testswitch1" devid="somerouterid" logid="1059028704" type="utm" subtype="app-ctrl" eventtype="signature" level="information" vd="root" eventtime=1591788391 tz="-0400" appid=40568 user="elasticuser" group="elasticgroup" authserver="elasticauth" srcip=192.168.2.1 dstip=8.8.8.8 srcport=59790 dstport=443 srcintf="LAN" srcintfrole="lan" dstintf="wan1" dstintfrole="wan" proto=6 service="SSL" direction="outgoing" policyid=12 sessionid=453234 applist="elasticruleset" action="pass" appcat="Web.Client" app="HTTPS.BROWSER" hostname="elastic.co" incidentserialno=23465 url="/" msg="Web.Client: HTTPS.BROWSER," apprisk="medium" scertcname="test.elastic.co"
 <189>date=2020-04-23 time=12:17:29 devname="testswitch1" devid="somerouterid" logid="1501054802" type="utm" subtype="dns" eventtype="dns-response" level="notice" vd="root" eventtime=1587230249360109339 tz="-0500" policyid=26 sessionid=543234 srcip=192.168.2.1 srcport=53430 srcintf="port1" srcintfrole="lan" dstip=8.8.8.8 dstport=53 dstintf="wan1" dstintfrole="wan" proto=17 profile="test" xid=2234 qname="elastic.example.com" qtype="A" qtypeval=1 qclass="IN" ipaddr="8.8.8.8" msg="Domain is monitored" action="pass" cat=23 catdesc="Web-based Email"
+<189>date=2020-04-23 time=12:17:29 devname="testswitch1" devid="somerouterid" logid="1501054802" type="utm" subtype="dns" eventtype="dns-response" level="notice" vd="root" eventtime=1587230249360109339 tz="-0500" policyid=26 sessionid=543234 srcip=192.168.2.1 srcport=53430 srcintf="port1" srcintfrole="lan" dstip=8.8.8.8 dstport=53 dstintf="wan1" dstintfrole="wan" proto=17 profile="test" xid=2234 qname="elastic.example.com" qtype="A" qtypeval=1 qclass="IN" ipaddr="8.8.8.8, 8.8.4.4" msg="Domain is monitored" action="pass" cat=23 catdesc="Web-based Email"
 <190>date=2020-04-23 time=12:17:11 devname="testswitch1" devid="somerouterid" logid="1059028704" type="utm" subtype="app-ctrl" eventtype="signature" level="information" vd="root" eventtime=1587230232148674303 tz="-0500" appid=40568 user="elasticuser" group="elasticgroup" authserver="elasticauth" srcip=192.168.2.1 dstip=8.8.8.8 srcport=63012 dstport=443 srcintf="port1" srcintfrole="lan" dstintf="wan1" dstintfrole="wan" proto=6 service="SSL" direction="outgoing" policyid=100602 sessionid=543234 applist="elasticruleset" action="pass" appcat="Web.Client" app="HTTPS.BROWSER" hostname="elastic.no" incidentserialno=54323 url="/" msg="Web.Client: HTTPS.BROWSER," apprisk="medium"
 <189>date=2020-04-23 time=12:17:04 devname="testswitch1" devid="somerouterid" logid="1501054802" type="utm" subtype="dns" eventtype="dns-response" level="notice" vd="root" eventtime=1587230224712900694 tz="-0500" policyid=26 sessionid=5432 srcip=192.168.2.1 srcport=54438 srcintf="port1" srcintfrole="lan" dstip=8.8.8.8 dstport=53 dstintf="wan1" dstintfrole="wan" proto=17 profile="elastictest" xid=2352 qname="elastic.co" qtype="A" qtypeval=1 qclass="IN" ipaddr="8.8.8.8" msg="Domain is monitored" action="pass" cat=93 catdesc="Remote Access"
 <190>date=2020-04-23 time=12:17:12 devname="testswitch1" devid="somerouterid" logid="1500054000" type="utm" subtype="dns" eventtype="dns-query" level="information" vd="root" eventtime=1587230232658642672 tz="-0500" policyid=26 sessionid=543234 srcip=192.168.2.1 srcport=54788 srcintf="port1" srcintfrole="lan" dstip=8.8.8.8 dstport=53 dstintf="wan1" dstintfrole="wan" proto=17 profile="elastictest" xid=235 qname="elastic.co" qtype="A" qtypeval=1 qclass="IN"