[ECS] Zeek upgrade to ecs 1.8.0 (#23847)

* Change ecs version to 1.8.0 * Add ecs mappings to http and mysql filesets
elastic · Feb 9, 2021 · 8cb2be2 · 8cb2be2
1 parent fa2980d
commit 8cb2be2
Show file tree

Hide file tree

Showing 41 changed files with 119 additions and 39 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -835,6 +835,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Updated o365 module to ECS 1.8. {issue}23118[23118] {pull}23896[23896]
 - Upgrade CEF module to ECS 1.8.0. {pull}23832[23832]
 - Upgrade fortinet/firewall to ECS 1.8 {issue}23118[23118] {pull}23902[23902]
+- Upgrade Zeek to ECS 1.8.0. {issue}23118[23118] {pull}23847[23847]
 
 *Heartbeat*
 

diff --git a/x-pack/filebeat/module/zeek/capture_loss/config/capture_loss.yml b/x-pack/filebeat/module/zeek/capture_loss/config/capture_loss.yml
@@ -22,4 +22,4 @@ processors:
   - add_fields:
       target: ''
       fields:
-        ecs.version: 1.7.0
+        ecs.version: 1.8.0
diff --git a/x-pack/filebeat/module/zeek/connection/config/connection.yml b/x-pack/filebeat/module/zeek/connection/config/connection.yml
@@ -102,4 +102,4 @@ processors:
   - add_fields:
       target: ''
       fields:
-        ecs.version: 1.7.0
+        ecs.version: 1.8.0
diff --git a/x-pack/filebeat/module/zeek/dce_rpc/config/dce_rpc.yml b/x-pack/filebeat/module/zeek/dce_rpc/config/dce_rpc.yml
@@ -58,4 +58,4 @@ processors:
   - add_fields:
       target: ''
       fields:
-        ecs.version: 1.7.0
+        ecs.version: 1.8.0
diff --git a/x-pack/filebeat/module/zeek/dhcp/config/dhcp.yml b/x-pack/filebeat/module/zeek/dhcp/config/dhcp.yml
@@ -120,4 +120,4 @@ processors:
   - add_fields:
       target: ''
       fields:
-        ecs.version: 1.7.0
+        ecs.version: 1.8.0
diff --git a/x-pack/filebeat/module/zeek/dnp3/config/dnp3.yml b/x-pack/filebeat/module/zeek/dnp3/config/dnp3.yml
@@ -68,4 +68,4 @@ processors:
   - add_fields:
       target: ''
       fields:
-        ecs.version: 1.7.0
+        ecs.version: 1.8.0
diff --git a/x-pack/filebeat/module/zeek/dns/config/dns.yml b/x-pack/filebeat/module/zeek/dns/config/dns.yml
@@ -210,4 +210,4 @@ processors:
   - add_fields:
       target: ''
       fields:
-        ecs.version: 1.7.0
+        ecs.version: 1.8.0
diff --git a/x-pack/filebeat/module/zeek/dpd/config/dpd.yml b/x-pack/filebeat/module/zeek/dpd/config/dpd.yml
@@ -57,4 +57,4 @@ processors:
   - add_fields:
       target: ''
       fields:
-        ecs.version: 1.7.0
+        ecs.version: 1.8.0
diff --git a/x-pack/filebeat/module/zeek/files/config/files.yml b/x-pack/filebeat/module/zeek/files/config/files.yml
@@ -42,4 +42,4 @@ processors:
   - add_fields:
       target: ''
       fields:
-        ecs.version: 1.7.0
+        ecs.version: 1.8.0
diff --git a/x-pack/filebeat/module/zeek/ftp/config/ftp.yml b/x-pack/filebeat/module/zeek/ftp/config/ftp.yml
@@ -86,4 +86,4 @@ processors:
   - add_fields:
       target: ''
       fields:
-        ecs.version: 1.7.0
+        ecs.version: 1.8.0
diff --git a/x-pack/filebeat/module/zeek/http/config/http.yml b/x-pack/filebeat/module/zeek/http/config/http.yml
@@ -76,6 +76,7 @@ processors:
         - {from: "destination.address", to: "destination.ip", type: "ip"}
         - {from: "destination.port", to: "url.port"}
         - {from: "http.request.method", to: "event.action"}
+        - {from: "url.username", to: "user.name"}
       ignore_missing: true
       fail_on_error: false
   - add_fields:
@@ -93,4 +94,4 @@ processors:
   - add_fields:
       target: ''
       fields:
-        ecs.version: 1.7.0
+        ecs.version: 1.8.0
diff --git a/x-pack/filebeat/module/zeek/http/test/http-json.log b/x-pack/filebeat/module/zeek/http/test/http-json.log
@@ -1,2 +1,2 @@
-{"ts":1547687130.172944,"uid":"CCNp8v1SNzY7v9d1Ih","id.orig_h":"10.178.98.102","id.orig_p":62995,"id.resp_h":"17.253.5.203","id.resp_p":80,"trans_depth":1,"method":"GET","host":"ocsp.apple.com","uri":"/ocsp04-aaica02/ME4wTKADAgEAMEUwQzBBMAkGBSsOAwIaBQAEFNqvF+Za6oA4ceFRLsAWwEInjUhJBBQx6napI3Sl39T97qDBpp7GEQ4R7AIIUP1IOZZ86ns=","version":"1.1","user_agent":"com.apple.trustd/2.0","request_body_len":0,"response_body_len":3735,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["F5zuip1tSwASjNAHy7"],"resp_mime_types":["application/ocsp-response"]}
-{"ts":1547707019.757479,"uid":"CMnIaR2V8VXyu7EPs","id.orig_h":"10.20.8.197","id.orig_p":35684,"id.resp_h":"34.206.130.40","id.resp_p":80,"trans_depth":1,"method":"GET","host":"httpbin.org","uri":"/ip","version":"1.1","user_agent":"curl/7.58.0","request_body_len":0,"response_body_len":32,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FwGPlr1GcKUWWdkXoi"],"resp_mime_types":["text/json"]}
+{"ts":1547687130.172944,"uid":"CCNp8v1SNzY7v9d1Ih","id.orig_h":"10.178.98.102","id.orig_p":62995,"id.resp_h":"17.253.5.203","username":"user","id.resp_p":80,"trans_depth":1,"method":"GET","host":"ocsp.apple.com","uri":"/ocsp04-aaica02/ME4wTKADAgEAMEUwQzBBMAkGBSsOAwIaBQAEFNqvF+Za6oA4ceFRLsAWwEInjUhJBBQx6napI3Sl39T97qDBpp7GEQ4R7AIIUP1IOZZ86ns=","version":"1.1","user_agent":"com.apple.trustd/2.0","request_body_len":0,"response_body_len":3735,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["F5zuip1tSwASjNAHy7"],"resp_mime_types":["application/ocsp-response"]}
+{"ts":1547707019.757479,"uid":"CMnIaR2V8VXyu7EPs","id.orig_h":"10.20.8.197","id.orig_p":35684,"id.resp_h":"34.206.130.40","id.resp_p":80,"trans_depth":1,"method":"GET","host":"httpbin.org","uri":"/ip","version":"1.1","user_agent":"curl/7.58.0","request_body_len":0,"response_body_len":32,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FwGPlr1GcKUWWdkXoi"],"resp_mime_types":["text/json"]}
diff --git a/x-pack/filebeat/module/zeek/http/test/http-json.log-expected.json b/x-pack/filebeat/module/zeek/http/test/http-json.log-expected.json
@@ -43,6 +43,9 @@
             "10.178.98.102",
             "17.253.5.203"
         ],
+        "related.user": [
+            "user"
+        ],
         "service.type": "zeek",
         "source.address": "10.178.98.102",
         "source.ip": "10.178.98.102",
@@ -53,6 +56,8 @@
         "url.domain": "ocsp.apple.com",
         "url.original": "/ocsp04-aaica02/ME4wTKADAgEAMEUwQzBBMAkGBSsOAwIaBQAEFNqvF+Za6oA4ceFRLsAWwEInjUhJBBQx6napI3Sl39T97qDBpp7GEQ4R7AIIUP1IOZZ86ns=",
         "url.port": 80,
+        "url.username": "user",
+        "user.name": "user",
         "user_agent.device.name": "Other",
         "user_agent.name": "Other",
         "user_agent.original": "com.apple.trustd/2.0",
@@ -66,5 +71,74 @@
         "zeek.http.tags": [],
         "zeek.http.trans_depth": 1,
         "zeek.session_id": "CCNp8v1SNzY7v9d1Ih"
+    },
+    {
+        "@timestamp": "2019-01-17T06:36:59.757Z",
+        "destination.address": "34.206.130.40",
+        "destination.as.number": 14618,
+        "destination.as.organization.name": "Amazon.com, Inc.",
+        "destination.geo.city_name": "Ashburn",
+        "destination.geo.continent_name": "North America",
+        "destination.geo.country_iso_code": "US",
+        "destination.geo.country_name": "United States",
+        "destination.geo.location.lat": 39.0481,
+        "destination.geo.location.lon": -77.4728,
+        "destination.geo.region_iso_code": "US-VA",
+        "destination.geo.region_name": "Virginia",
+        "destination.ip": "34.206.130.40",
+        "destination.port": 80,
+        "event.action": "get",
+        "event.category": [
+            "network",
+            "web"
+        ],
+        "event.dataset": "zeek.http",
+        "event.id": "CMnIaR2V8VXyu7EPs",
+        "event.kind": "event",
+        "event.module": "zeek",
+        "event.outcome": "success",
+        "event.type": [
+            "connection",
+            "info",
+            "protocol"
+        ],
+        "fileset.name": "http",
+        "http.request.body.bytes": 0,
+        "http.request.method": "GET",
+        "http.response.body.bytes": 32,
+        "http.response.status_code": 200,
+        "http.version": "1.1",
+        "input.type": "log",
+        "log.offset": 574,
+        "network.community_id": "1:Ol0Btm49e1mxnu/BXm1GM8w5ixY=",
+        "network.transport": "tcp",
+        "related.ip": [
+            "10.20.8.197",
+            "34.206.130.40"
+        ],
+        "service.type": "zeek",
+        "source.address": "10.20.8.197",
+        "source.ip": "10.20.8.197",
+        "source.port": 35684,
+        "tags": [
+            "zeek.http"
+        ],
+        "url.domain": "httpbin.org",
+        "url.original": "/ip",
+        "url.port": 80,
+        "user_agent.device.name": "Other",
+        "user_agent.name": "curl",
+        "user_agent.original": "curl/7.58.0",
+        "user_agent.version": "7.58.0",
+        "zeek.http.resp_fuids": [
+            "FwGPlr1GcKUWWdkXoi"
+        ],
+        "zeek.http.resp_mime_types": [
+            "text/json"
+        ],
+        "zeek.http.status_msg": "OK",
+        "zeek.http.tags": [],
+        "zeek.http.trans_depth": 1,
+        "zeek.session_id": "CMnIaR2V8VXyu7EPs"
     }
 ]
diff --git a/x-pack/filebeat/module/zeek/intel/config/intel.yml b/x-pack/filebeat/module/zeek/intel/config/intel.yml
@@ -67,4 +67,4 @@ processors:
   - add_fields:
       target: ''
       fields:
-        ecs.version: 1.7.0
+        ecs.version: 1.8.0
diff --git a/x-pack/filebeat/module/zeek/irc/config/irc.yml b/x-pack/filebeat/module/zeek/irc/config/irc.yml
@@ -72,4 +72,4 @@ processors:
   - add_fields:
       target: ''
       fields:
-        ecs.version: 1.7.0
+        ecs.version: 1.8.0
diff --git a/x-pack/filebeat/module/zeek/kerberos/config/kerberos.yml b/x-pack/filebeat/module/zeek/kerberos/config/kerberos.yml
@@ -104,4 +104,4 @@ processors:
   - add_fields:
       target: ''
       fields:
-        ecs.version: 1.7.0
+        ecs.version: 1.8.0
diff --git a/x-pack/filebeat/module/zeek/modbus/config/modbus.yml b/x-pack/filebeat/module/zeek/modbus/config/modbus.yml
@@ -73,4 +73,4 @@ processors:
   - add_fields:
       target: ''
       fields:
-        ecs.version: 1.7.0
+        ecs.version: 1.8.0
diff --git a/x-pack/filebeat/module/zeek/mysql/config/mysql.yml b/x-pack/filebeat/module/zeek/mysql/config/mysql.yml
@@ -72,4 +72,4 @@ processors:
   - add_fields:
       target: ''
       fields:
-        ecs.version: 1.7.0
+        ecs.version: 1.8.0
diff --git a/x-pack/filebeat/module/zeek/mysql/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/mysql/ingest/pipeline.yml
@@ -80,6 +80,10 @@ processors:
     field: event.type
     value: end
     if: "ctx?.zeek?.mysql?.cmd != null && ctx.zeek.mysql.cmd == 'connect_out'"
+- append:
+    field: event.category
+    value: session
+    if: "ctx?.zeek?.mysql?.cmd != null && (ctx.zeek.mysql.cmd == 'connect' || ctx.zeek.mysql.cmd == 'connect_out')"
 on_failure:
 - set:
     field: error.message

diff --git a/x-pack/filebeat/module/zeek/notice/config/notice.yml b/x-pack/filebeat/module/zeek/notice/config/notice.yml
@@ -104,4 +104,4 @@ processors:
   - add_fields:
       target: ''
       fields:
-        ecs.version: 1.7.0
+        ecs.version: 1.8.0
diff --git a/x-pack/filebeat/module/zeek/ntlm/config/ntlm.yml b/x-pack/filebeat/module/zeek/ntlm/config/ntlm.yml
@@ -86,4 +86,4 @@ processors:
   - add_fields:
       target: ''
       fields:
-        ecs.version: 1.7.0
+        ecs.version: 1.8.0
diff --git a/x-pack/filebeat/module/zeek/ocsp/config/ocsp.yml b/x-pack/filebeat/module/zeek/ocsp/config/ocsp.yml
@@ -64,4 +64,4 @@ processors:
   - add_fields:
       target: ''
       fields:
-        ecs.version: 1.7.0
+        ecs.version: 1.8.0
diff --git a/x-pack/filebeat/module/zeek/pe/config/pe.yml b/x-pack/filebeat/module/zeek/pe/config/pe.yml
@@ -33,4 +33,4 @@ processors:
   - add_fields:
       target: ''
       fields:
-        ecs.version: 1.7.0
+        ecs.version: 1.8.0
diff --git a/x-pack/filebeat/module/zeek/radius/config/radius.yml b/x-pack/filebeat/module/zeek/radius/config/radius.yml
@@ -58,4 +58,4 @@ processors:
   - add_fields:
       target: ''
       fields:
-        ecs.version: 1.7.0
+        ecs.version: 1.8.0
diff --git a/x-pack/filebeat/module/zeek/rdp/config/rdp.yml b/x-pack/filebeat/module/zeek/rdp/config/rdp.yml
@@ -88,4 +88,4 @@ processors:
   - add_fields:
       target: ''
       fields:
-        ecs.version: 1.7.0
+        ecs.version: 1.8.0
diff --git a/x-pack/filebeat/module/zeek/rfb/config/rfb.yml b/x-pack/filebeat/module/zeek/rfb/config/rfb.yml
@@ -73,4 +73,4 @@ processors:
   - add_fields:
       target: ''
       fields:
-        ecs.version: 1.7.0
+        ecs.version: 1.8.0
diff --git a/x-pack/filebeat/module/zeek/sip/config/sip.yml b/x-pack/filebeat/module/zeek/sip/config/sip.yml
@@ -95,4 +95,4 @@ processors:
   - add_fields:
       target: ''
       fields:
-        ecs.version: 1.7.0
+        ecs.version: 1.8.0
diff --git a/x-pack/filebeat/module/zeek/smb_cmd/config/smb_cmd.yml b/x-pack/filebeat/module/zeek/smb_cmd/config/smb_cmd.yml
@@ -101,4 +101,4 @@ processors:
   - add_fields:
       target: ''
       fields:
-        ecs.version: 1.7.0
+        ecs.version: 1.8.0
diff --git a/x-pack/filebeat/module/zeek/smb_files/config/smb_files.yml b/x-pack/filebeat/module/zeek/smb_files/config/smb_files.yml
@@ -61,4 +61,4 @@ processors:
   - add_fields:
       target: ''
       fields:
-        ecs.version: 1.7.0
+        ecs.version: 1.8.0
diff --git a/x-pack/filebeat/module/zeek/smb_mapping/config/smb_mapping.yml b/x-pack/filebeat/module/zeek/smb_mapping/config/smb_mapping.yml
@@ -57,4 +57,4 @@ processors:
   - add_fields:
       target: ''
       fields:
-        ecs.version: 1.7.0
+        ecs.version: 1.8.0
diff --git a/x-pack/filebeat/module/zeek/smtp/config/smtp.yml b/x-pack/filebeat/module/zeek/smtp/config/smtp.yml
@@ -67,4 +67,4 @@ processors:
   - add_fields:
       target: ''
       fields:
-        ecs.version: 1.7.0
+        ecs.version: 1.8.0
diff --git a/x-pack/filebeat/module/zeek/snmp/config/snmp.yml b/x-pack/filebeat/module/zeek/snmp/config/snmp.yml
@@ -69,4 +69,4 @@ processors:
   - add_fields:
       target: ''
       fields:
-        ecs.version: 1.7.0
+        ecs.version: 1.8.0
diff --git a/x-pack/filebeat/module/zeek/socks/config/socks.yml b/x-pack/filebeat/module/zeek/socks/config/socks.yml
@@ -67,4 +67,4 @@ processors:
   - add_fields:
       target: ''
       fields:
-        ecs.version: 1.7.0
+        ecs.version: 1.8.0
diff --git a/x-pack/filebeat/module/zeek/ssh/config/ssh.yml b/x-pack/filebeat/module/zeek/ssh/config/ssh.yml
@@ -76,4 +76,4 @@ processors:
   - add_fields:
       target: ''
       fields:
-        ecs.version: 1.7.0
+        ecs.version: 1.8.0
diff --git a/x-pack/filebeat/module/zeek/ssl/config/ssl.yml b/x-pack/filebeat/module/zeek/ssl/config/ssl.yml
@@ -94,4 +94,4 @@ processors:
   - add_fields:
       target: ''
       fields:
-        ecs.version: 1.7.0
+        ecs.version: 1.8.0
diff --git a/x-pack/filebeat/module/zeek/stats/config/stats.yml b/x-pack/filebeat/module/zeek/stats/config/stats.yml
@@ -97,4 +97,4 @@ processors:
   - add_fields:
       target: ''
       fields:
-        ecs.version: 1.7.0
+        ecs.version: 1.8.0
diff --git a/x-pack/filebeat/module/zeek/syslog/config/syslog.yml b/x-pack/filebeat/module/zeek/syslog/config/syslog.yml
@@ -57,4 +57,4 @@ processors:
   - add_fields:
       target: ''
       fields:
-        ecs.version: 1.7.0
+        ecs.version: 1.8.0
diff --git a/x-pack/filebeat/module/zeek/traceroute/config/traceroute.yml b/x-pack/filebeat/module/zeek/traceroute/config/traceroute.yml
@@ -45,4 +45,4 @@ processors:
   - add_fields:
       target: ''
       fields:
-        ecs.version: 1.7.0
+        ecs.version: 1.8.0
diff --git a/x-pack/filebeat/module/zeek/tunnel/config/tunnel.yml b/x-pack/filebeat/module/zeek/tunnel/config/tunnel.yml
@@ -56,4 +56,4 @@ processors:
   - add_fields:
       target: ''
       fields:
-        ecs.version: 1.7.0
+        ecs.version: 1.8.0
diff --git a/x-pack/filebeat/module/zeek/weird/config/weird.yml b/x-pack/filebeat/module/zeek/weird/config/weird.yml
@@ -56,4 +56,4 @@ processors:
   - add_fields:
       target: ''
       fields:
-        ecs.version: 1.7.0
+        ecs.version: 1.8.0
diff --git a/x-pack/filebeat/module/zeek/x509/config/x509.yml b/x-pack/filebeat/module/zeek/x509/config/x509.yml
@@ -67,4 +67,4 @@ processors:
   - add_fields:
       target: ''
       fields:
-        ecs.version: 1.7.0
+        ecs.version: 1.8.0