Merge branch 'feature-ecs-1.8' into fb-aws-ct-ecs-1.8

elastic · Feb 10, 2021 · cba6ab3 · cba6ab3
2 parents eef321a + 80123fb
commit cba6ab3
Show file tree

Hide file tree

Showing 81 changed files with 838 additions and 1,157 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -379,6 +379,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix handling of ModifiedProperties field in Office 365. {pull}23777[23777]
 - Fix goroutines leak with some inputs in autodiscover. {pull}23722[23722]
 - Fix various processing errors in the Suricata module. {pull}23236[23236]
+- aws/s3access dataset was populating event.duration using the wrong unit. {pull}23920[23920]
 
 *Heartbeat*
 
@@ -833,6 +834,12 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Update Filebeat auditd dataset to ECS 1.8.0. {pull}23723[23723] {issue}23118[23118]
 - Updated microsoft defender_atp and m365_defender to ECS 1.8. {pull}23897[23897] {issue}23118[23118]
 - Updated o365 module to ECS 1.8. {issue}23118[23118] {pull}23896[23896]
+- Upgrade CEF module to ECS 1.8.0. {pull}23832[23832]
+- Upgrade fortinet/firewall to ECS 1.8 {issue}23118[23118] {pull}23902[23902]
+- Upgrade Zeek to ECS 1.8.0. {issue}23118[23118] {pull}23847[23847]
+- Updated azure module to ECS 1.8. {issue}23118[23118] {pull}23927[23927]
+- Update aws/s3access to ECS 1.8. {issue}23118[23118] {pull}23920[23920]
+- Upgrade panw module to ecs 1.8 {issue}23118[23118] {pull}23931[23931]
 - Updated aws/cloudtrail fileset to ECS 1.8. {issue}23118[23118] {pull}23911[23911]
 
 *Heartbeat*

diff --git a/x-pack/filebeat/module/aws/s3access/config/aws-s3.yml b/x-pack/filebeat/module/aws/s3access/config/aws-s3.yml
@@ -52,4 +52,4 @@ processors:
   - add_fields:
       target: ''
       fields:
-        ecs.version: 1.7.0
+        ecs.version: 1.8.0
diff --git a/x-pack/filebeat/module/aws/s3access/config/file.yml b/x-pack/filebeat/module/aws/s3access/config/file.yml
@@ -11,4 +11,4 @@ processors:
   - add_fields:
       target: ''
       fields:
-        ecs.version: 1.7.0
+        ecs.version: 1.8.0
diff --git a/x-pack/filebeat/module/aws/s3access/ingest/pipeline.yml b/x-pack/filebeat/module/aws/s3access/ingest/pipeline.yml
@@ -1,6 +1,12 @@
 description: "Pipeline for s3 server access logs"
 
 processors:
+  - set:
+      field: event.category
+      value: web
+  - append:
+      field: event.type
+      value: access
   - set:
       field: event.ingested
       value: '{{_ingest.timestamp}}'
@@ -24,6 +30,22 @@ processors:
         S3ID: "[a-zA-Z0-9\\/_\\.\\-%+=]+"
         S3VERSION: "[a-zA-Z0-9.]+"
 
+  - grok:
+      field: aws.s3access.request_uri
+      ignore_failure: true
+      patterns:
+        - '%{NOTSPACE:http.request.method} %{NOTSPACE:url.original} [hH][tT][tT][pP]/%{NOTSPACE:http.version}'
+
+  #
+  # Best-effort parse of url.original in the form /path?query"
+  #
+  - grok:
+      field: url.original
+      ignore_failure: true
+      patterns:
+        - '^%{ABS_PATH:url.path}(?:\?%{DATA:url.query})?$'
+      pattern_definitions:
+        ABS_PATH: '/[^?]*'
   - append:
       if: "ctx?.aws?.s3access?.bucket_owner != null"
       field: related.user
@@ -99,10 +121,25 @@ processors:
       field: event.outcome
       value: success
 
-  - set:
-      field: event.duration
-      value: "{{aws.s3access.total_time}}"
-      ignore_empty_value: true
+  - convert:
+      field: aws.s3access.bytes_sent
+      target_field: http.response.body.bytes
+      type: long
+      ignore_failure: true
+
+  - convert:
+      field: aws.s3access.total_time
+      target_field: event.duration
+      type: long
+      ignore_failure: true
+
+  - script:
+      lang: painless
+      if: ctx.event?.duration != null
+      params:
+        MS_TO_NS: 1000000
+      source: >-
+        ctx.event.duration *= params.MS_TO_NS;
 
   - set:
       field: http.request.referrer
@@ -137,13 +174,18 @@ processors:
       field: event.kind
       value: event
 
+  #
+  # Save original message into event.original
+  #
+  - rename:
+      field: "message"
+      target_field: "event.original"
+
   #
   # Remove temporary fields
   #
   - remove:
-      field:
-        - message
-        - _temp_
+      field: _temp_
       ignore_missing: true
 
 on_failure:

diff --git a/x-pack/filebeat/module/aws/s3access/test/s3_server_access.log-expected.json b/x-pack/filebeat/module/aws/s3access/test/s3_server_access.log-expected.json
@@ -23,12 +23,17 @@
         "client.user.id": "arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9",
         "cloud.provider": "aws",
         "event.action": "REST.GET.LOCATION",
+        "event.category": "web",
         "event.dataset": "aws.s3access",
-        "event.duration": "17",
+        "event.duration": 17000000,
         "event.id": "44EE8651683CB4DA",
         "event.kind": "event",
         "event.module": "aws",
+        "event.original": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [01/Aug/2019:00:24:41 +0000] 72.21.217.31 arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9 44EE8651683CB4DA REST.GET.LOCATION - \"GET /test-s3-ks/?location&aws-account=627959692251 HTTP/1.1\" 200 - 142 - 17 - \"-\" \"AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation\" - BsCfJedfuSnds2QFoxi+E/O7M6OEWzJnw4dUaes/2hyA363sONRJKzB7EOY+Bt9DTHYUn+HoHxI= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.ap-southeast-1.amazonaws.com TLSv1.2",
         "event.outcome": "success",
+        "event.type": [
+            "access"
+        ],
         "fileset.name": "s3access",
         "geo.city_name": "Ashburn",
         "geo.continent_name": "North America",
@@ -38,7 +43,10 @@
         "geo.location.lon": -77.4728,
         "geo.region_iso_code": "US-VA",
         "geo.region_name": "Virginia",
+        "http.request.method": "GET",
+        "http.response.body.bytes": 142,
         "http.response.status_code": 200,
+        "http.version": "1.1",
         "input.type": "log",
         "log.offset": 0,
         "related.ip": [
@@ -54,6 +62,9 @@
         "tls.cipher": "ECDHE-RSA-AES128-SHA",
         "tls.version": "1.2",
         "tls.version_protocol": "tls",
+        "url.original": "/test-s3-ks/?location&aws-account=627959692251",
+        "url.path": "/test-s3-ks/",
+        "url.query": "location&aws-account=627959692251",
         "user_agent.device.name": "Other",
         "user_agent.name": "aws-sdk-java",
         "user_agent.original": "AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation",
@@ -86,12 +97,17 @@
         "client.user.id": "arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9",
         "cloud.provider": "aws",
         "event.action": "REST.GET.LOCATION",
+        "event.category": "web",
         "event.dataset": "aws.s3access",
-        "event.duration": "3",
+        "event.duration": 3000000,
         "event.id": "E26222010BCC32B6",
         "event.kind": "event",
         "event.module": "aws",
+        "event.original": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [01/Aug/2019:00:24:42 +0000] 72.21.217.31 arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9 E26222010BCC32B6 REST.GET.LOCATION - \"GET /test-s3-ks/?location&aws-account=627959692251 HTTP/1.1\" 200 - 142 - 3 - \"-\" \"AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation\" - gNl/Q1IzY6nGTBygqI3rnMz/ZFOFwOTDpSMrNca+IcEmMAd6sCIs1ZRLYDekD8LB9lrj9UdQLWE= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.ap-southeast-1.amazonaws.com TLSv1.2",
         "event.outcome": "success",
+        "event.type": [
+            "access"
+        ],
         "fileset.name": "s3access",
         "geo.city_name": "Ashburn",
         "geo.continent_name": "North America",
@@ -101,7 +117,10 @@
         "geo.location.lon": -77.4728,
         "geo.region_iso_code": "US-VA",
         "geo.region_name": "Virginia",
+        "http.request.method": "GET",
+        "http.response.body.bytes": 142,
         "http.response.status_code": 200,
+        "http.version": "1.1",
         "input.type": "log",
         "log.offset": 715,
         "related.ip": [
@@ -117,6 +136,9 @@
         "tls.cipher": "ECDHE-RSA-AES128-SHA",
         "tls.version": "1.2",
         "tls.version_protocol": "tls",
+        "url.original": "/test-s3-ks/?location&aws-account=627959692251",
+        "url.path": "/test-s3-ks/",
+        "url.query": "location&aws-account=627959692251",
         "user_agent.device.name": "Other",
         "user_agent.name": "aws-sdk-java",
         "user_agent.original": "AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation",
@@ -150,12 +172,17 @@
         "client.user.id": "arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9",
         "cloud.provider": "aws",
         "event.action": "REST.GET.BUCKET",
+        "event.category": "web",
         "event.dataset": "aws.s3access",
-        "event.duration": "2",
+        "event.duration": 2000000,
         "event.id": "4DD6D17D1C5C401C",
         "event.kind": "event",
         "event.module": "aws",
+        "event.original": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [01/Aug/2019:00:24:43 +0000] 72.21.217.31 arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9 4DD6D17D1C5C401C REST.GET.BUCKET - \"GET /test-s3-ks/?max-keys=0&encoding-type=url&aws-account=627959692251 HTTP/1.1\" 200 - 265 - 2 1 \"-\" \"AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation\" - KzvchfojYQnuFC4PABYVJVxIlv/f6r17LRaTSvw7x+bxj4PkkPKT1kX9x8wbqtq40iD4PC881iE= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.ap-southeast-1.amazonaws.com TLSv1.2",
         "event.outcome": "success",
+        "event.type": [
+            "access"
+        ],
         "fileset.name": "s3access",
         "geo.city_name": "Ashburn",
         "geo.continent_name": "North America",
@@ -165,7 +192,10 @@
         "geo.location.lon": -77.4728,
         "geo.region_iso_code": "US-VA",
         "geo.region_name": "Virginia",
+        "http.request.method": "GET",
+        "http.response.body.bytes": 265,
         "http.response.status_code": 200,
+        "http.version": "1.1",
         "input.type": "log",
         "log.offset": 1429,
         "related.ip": [
@@ -181,6 +211,9 @@
         "tls.cipher": "ECDHE-RSA-AES128-SHA",
         "tls.version": "1.2",
         "tls.version_protocol": "tls",
+        "url.original": "/test-s3-ks/?max-keys=0&encoding-type=url&aws-account=627959692251",
+        "url.path": "/test-s3-ks/",
+        "url.query": "max-keys=0&encoding-type=url&aws-account=627959692251",
         "user_agent.device.name": "Other",
         "user_agent.name": "aws-sdk-java",
         "user_agent.original": "AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation",
@@ -213,12 +246,17 @@
         "client.user.id": "arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9",
         "cloud.provider": "aws",
         "event.action": "REST.GET.LOCATION",
+        "event.category": "web",
         "event.dataset": "aws.s3access",
-        "event.duration": "4",
+        "event.duration": 4000000,
         "event.id": "706992E2F3CC3C3D",
         "event.kind": "event",
         "event.module": "aws",
+        "event.original": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [01/Aug/2019:00:24:43 +0000] 72.21.217.31 arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9 706992E2F3CC3C3D REST.GET.LOCATION - \"GET /test-s3-ks/?location&aws-account=627959692251 HTTP/1.1\" 200 - 142 - 4 - \"-\" \"AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation\" - cIN12KTrJwx+uTBZD+opZUPE4iGypi8oG/oXGPzFk9CMuHQGuEpmAeNELdtYKDxf2TDor25Nikg= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.ap-southeast-1.amazonaws.com TLSv1.2",
         "event.outcome": "success",
+        "event.type": [
+            "access"
+        ],
         "fileset.name": "s3access",
         "geo.city_name": "Ashburn",
         "geo.continent_name": "North America",
@@ -228,7 +266,10 @@
         "geo.location.lon": -77.4728,
         "geo.region_iso_code": "US-VA",
         "geo.region_name": "Virginia",
+        "http.request.method": "GET",
+        "http.response.body.bytes": 142,
         "http.response.status_code": 200,
+        "http.version": "1.1",
         "input.type": "log",
         "log.offset": 2161,
         "related.ip": [
@@ -244,6 +285,9 @@
         "tls.cipher": "ECDHE-RSA-AES128-SHA",
         "tls.version": "1.2",
         "tls.version_protocol": "tls",
+        "url.original": "/test-s3-ks/?location&aws-account=627959692251",
+        "url.path": "/test-s3-ks/",
+        "url.query": "location&aws-account=627959692251",
         "user_agent.device.name": "Other",
         "user_agent.name": "aws-sdk-java",
         "user_agent.original": "AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation",
@@ -274,11 +318,16 @@
         "client.user.id": "arn:aws:iam::123456:user/test@elastic.co",
         "cloud.provider": "aws",
         "event.action": "BATCH.DELETE.OBJECT",
+        "event.category": "web",
         "event.dataset": "aws.s3access",
         "event.id": "8CD7A4A71E2E5C9E",
         "event.kind": "event",
         "event.module": "aws",
+        "event.original": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 jsoriano-s3-test [10/Sep/2019:15:11:07 +0000] 77.227.156.41 arn:aws:iam::123456:user/test@elastic.co 8CD7A4A71E2E5C9E BATCH.DELETE.OBJECT jolokia-war-1.5.0.war - 204 - - 344017 - - - - - IeDW5I3wefFxU8iHOcAzi5qr+O+1bdRlcQ0AO2WGjFh7JwYM6qCoKq+1TrUshrXMlBxPFtg97Vk= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.eu-central-1.amazonaws.com TLSv1.2",
         "event.outcome": "success",
+        "event.type": [
+            "access"
+        ],
         "fileset.name": "s3access",
         "geo.city_name": "Teruel",
         "geo.continent_name": "Europe",
@@ -327,11 +376,16 @@
         "client.user.id": "arn:aws:iam::123456:user/test@elastic.co",
         "cloud.provider": "aws",
         "event.action": "BATCH.DELETE.OBJECT",
+        "event.category": "web",
         "event.dataset": "aws.s3access",
         "event.id": "6CE38F1312D32BDD",
         "event.kind": "event",
         "event.module": "aws",
+        "event.original": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [19/Sep/2019:17:06:39 +0000] 174.29.206.152 arn:aws:iam::123456:user/test@elastic.co 6CE38F1312D32BDD BATCH.DELETE.OBJECT Screen+Shot+2019-09-09+at+9.08.44+AM.png - 204 - - 57138 - - - - - LwRa4w6DbuU48GKQiH3jDbjfTyLCbwasFBsdttugRQ+9lH4jK8lT91+HhGZKMYI3sPyKuQ9LvU0= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3-ap-southeast-1.amazonaws.com TLSv1.2",
         "event.outcome": "success",
+        "event.type": [
+            "access"
+        ],
         "fileset.name": "s3access",
         "geo.city_name": "Denver",
         "geo.continent_name": "North America",

diff --git a/x-pack/filebeat/module/aws/s3access/test/test.log b/x-pack/filebeat/module/aws/s3access/test/test.log
@@ -3,3 +3,4 @@
 79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be awsexamplebucket [06/Feb/2019:00:00:38 +0000] 192.0.2.3 79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be A1206F460EXAMPLE REST.GET.BUCKETPOLICY - "GET /awsexamplebucket?policy HTTP/1.1" 404 NoSuchBucketPolicy 297 - 38 - "-" "S3Console/0.4" - BNaBsXZQQDbssi6xMBdBU2sLt+Yf5kZDmeBUP35sFoKa3sLLeMC78iwEIWxs99CRUrbS4n11234= SigV2 ECDHE-RSA-AES128-GCM-SHA256 AuthHeader awsexamplebucket.s3.amazonaws.com TLSV1.1
 79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be awsexamplebucket [06/Feb/2019:00:01:00 +0000] 192.0.2.3 79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be 7B4A0FABBEXAMPLE REST.GET.VERSIONING - "GET /awsexamplebucket?versioning HTTP/1.1" 200 - 113 - 33 - "-" "S3Console/0.4" - Ke1bUcazaN1jWuUlPJaxF64cQVpUEhoZKEG/hmy/gijN/I1DeWqDfFvnpybfEseEME/u7ME1234= SigV2 ECDHE-RSA-AES128-GCM-SHA256 AuthHeader awsexamplebucket.s3.amazonaws.com TLSV1.1
 79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be awsexamplebucket [06/Feb/2019:00:01:57 +0000] 192.0.2.3 79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be DD6CC733AEXAMPLE REST.PUT.OBJECT s3-dg.pdf "PUT /awsexamplebucket/s3-dg.pdf HTTP/1.1" 200 - - 4406583 41754 28 "-" "S3Console/0.4" - 10S62Zv81kBW7BB6SX4XJ48o6kpcl6LPwEoizZQQxJd5qDSCTLX0TgS37kYUBKQW3+bPdrg1234= SigV4 ECDHE-RSA-AES128-SHA AuthHeader awsexamplebucket.s3.amazonaws.com TLSV1.1
+79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be faketest [09/Feb/2021:14:48:42 +0200] 192.0.2.3 79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be DD6CC733AEXAMPLE REST.OPTIONS.FAKE s3-dg.pdf "OPTIONS * HTTP/1.0" 200 - - 4406583 41754 28 "-" "S3Console/0.4" - 10S62Zv81kBW7BB6SX4XJ48o6kpcl6LPwEoizZQQxJd5qDSCTLX0TgS37kYUBKQW3+bPdrg1234= SigV4 ECDHE-RSA-AES128-SHA AuthHeader awsexamplebucket.s3.amazonaws.com TLSV1.1