[Filebeat] Update AWS ELB ingest pipeline (#26441)

elastic · Jun 24, 2021 · d7724cd · d7724cd
1 parent 7742fda
commit d7724cd
Show file tree

Hide file tree

Showing 7 changed files with 297 additions and 61 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -277,7 +277,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix bug in aws-s3 input where the end of gzipped log files might have been discarded. {pull}26260[26260]
 - Fix bug in `httpjson` that prevented `first_event` getting updated. {pull}26407[26407]
 - Fix bug in the Syslog input that misparsed rfc5424 days starting with 0. {pull}26419[26419]
-- Do not close filestream harvester if an unexpected error is returned when close.on_state_change.* is enabled. {pull}26411[26411] 
+- Do not close filestream harvester if an unexpected error is returned when close.on_state_change.* is enabled. {pull}26411[26411]
 
 *Filebeat*
 
@@ -386,6 +386,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix incorrect field name appending to `related.hash` in `threatintel.abusechmalware` ingest pipeline. {issue}25151[25151] {pull}25674[25674]
 - Add improvements to the azure activitylogs and platformlogs ingest pipelines. {pull}26148[26148]
 - Fix `kibana.log` pipeline when `event.duration` calculation becomes a Long. {issue}24556[24556] {pull}25675[25675]
+- Removed incorrect `http.request.referrer` field from `aws.elb` module. {issue}26435[26435] {pull}26441[26441]
 
 *Heartbeat*
 
@@ -831,6 +832,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Update Okta module to parse additional fields to `okta.debug_context.debug_data`. {issue}25689[25689] {pull}25818[25818]
 - Added dataset `anomalithreatstream` to the `threatintel` module to ingest indicators from Anomali ThreatStream {pull}26350[26350]
 - Add support for `copytruncate` method when rotating input logs with an external tool in `filestream` input. {pull}23457[23457]
+- Add `uri_parts` and `user_agent` ingest processors to `aws.elb` module. {issue}26435[26435] {pull}26441[26441]
 
 *Heartbeat*
 

diff --git a/x-pack/filebeat/module/aws/elb/ingest/pipeline.yml b/x-pack/filebeat/module/aws/elb/ingest/pipeline.yml
@@ -77,8 +77,8 @@ processors:
           (?:-|%{NUMBER:aws.elb.backend.http.response.status_code:long})
           %{NUMBER:http.request.body.bytes:long}
           %{NUMBER:http.response.body.bytes:long}
-          \"(?:-|%{WORD:http.request.method}) (?:-|%{NOTSPACE:http.request.referrer}) (?:-|HTTP/%{NOTSPACE:http.version})\"
-          \"%{DATA:user_agent.original}\"
+          \"(?:-|%{WORD:http.request.method}) (?:-|%{NOTSPACE:_tmp.uri_orig}) (?:-|HTTP/%{NOTSPACE:http.version})\"
+          \"%{DATA:_tmp.user_agent}\"
           %{ELBSSL}
         ELBTCPLOG: >-
           %{ELBCOMMON}
@@ -105,6 +105,16 @@ processors:
       field: 'aws.elb.protocol'
       value: 'http'
 
+  - uri_parts:
+      if: 'ctx?._tmp?.uri_orig != null'
+      field: _tmp.uri_orig
+      ignore_failure: true
+
+  - user_agent:
+      if: 'ctx?._tmp?.user_agent != null'
+      field: _tmp.user_agent
+      ignore_missing: true
+
   - set:
       if: 'ctx.http != null'
       field: event.category

diff --git a/x-pack/filebeat/module/aws/elb/test/application-lb-http.log-expected.json b/x-pack/filebeat/module/aws/elb/test/application-lb-http.log-expected.json
@@ -23,7 +23,6 @@
         "fileset.name": "elb",
         "http.request.body.bytes": 125,
         "http.request.method": "GET",
-        "http.request.referrer": "http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/",
         "http.response.body.bytes": 0,
         "http.response.status_code": 460,
         "http.version": "1.1",
@@ -46,7 +45,15 @@
             "forwarded"
         ],
         "trace.id": "Root=1-5da09932-2c342a443bfb96249aa50ed7",
-        "user_agent.original": "curl/7.58.0"
+        "url.domain": "filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com",
+        "url.original": "http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/",
+        "url.path": "/",
+        "url.port": 80,
+        "url.scheme": "http",
+        "user_agent.device.name": "Other",
+        "user_agent.name": "curl",
+        "user_agent.original": "curl/7.58.0",
+        "user_agent.version": "7.58.0"
     },
     {
         "@timestamp": "2019-10-11T15:01:50.492Z",
@@ -72,7 +79,6 @@
         "fileset.name": "elb",
         "http.request.body.bytes": 125,
         "http.request.method": "GET",
-        "http.request.referrer": "http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/",
         "http.response.body.bytes": 308,
         "http.response.status_code": 504,
         "http.version": "1.1",
@@ -95,7 +101,15 @@
             "forwarded"
         ],
         "trace.id": "Root=1-5da09954-2c342a443bfb96249aa50ed7",
-        "user_agent.original": "curl/7.58.0"
+        "url.domain": "filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com",
+        "url.original": "http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/",
+        "url.path": "/",
+        "url.port": 80,
+        "url.scheme": "http",
+        "user_agent.device.name": "Other",
+        "user_agent.name": "curl",
+        "user_agent.original": "curl/7.58.0",
+        "user_agent.version": "7.58.0"
     },
     {
         "@timestamp": "2019-10-11T15:01:22.915Z",
@@ -121,7 +135,6 @@
         "fileset.name": "elb",
         "http.request.body.bytes": 125,
         "http.request.method": "GET",
-        "http.request.referrer": "http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/",
         "http.response.body.bytes": 308,
         "http.response.status_code": 504,
         "http.version": "1.1",
@@ -144,7 +157,15 @@
             "forwarded"
         ],
         "trace.id": "Root=1-5da09938-d9c72660e247c36070017828",
-        "user_agent.original": "curl/7.58.0"
+        "url.domain": "filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com",
+        "url.original": "http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/",
+        "url.path": "/",
+        "url.port": 80,
+        "url.scheme": "http",
+        "user_agent.device.name": "Other",
+        "user_agent.name": "curl",
+        "user_agent.original": "curl/7.58.0",
+        "user_agent.version": "7.58.0"
     },
     {
         "@timestamp": "2019-10-11T15:01:35.190Z",
@@ -170,7 +191,6 @@
         "fileset.name": "elb",
         "http.request.body.bytes": 125,
         "http.request.method": "GET",
-        "http.request.referrer": "http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/",
         "http.response.body.bytes": 308,
         "http.response.status_code": 504,
         "http.version": "1.1",
@@ -193,7 +213,15 @@
             "forwarded"
         ],
         "trace.id": "Root=1-5da09945-0eaa8050df7d96f84806ded0",
-        "user_agent.original": "curl/7.58.0"
+        "url.domain": "filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com",
+        "url.original": "http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/",
+        "url.path": "/",
+        "url.port": 80,
+        "url.scheme": "http",
+        "user_agent.device.name": "Other",
+        "user_agent.name": "curl",
+        "user_agent.original": "curl/7.58.0",
+        "user_agent.version": "7.58.0"
     },
     {
         "@timestamp": "2019-10-11T15:02:28.837Z",
@@ -219,7 +247,6 @@
         "fileset.name": "elb",
         "http.request.body.bytes": 125,
         "http.request.method": "GET",
-        "http.request.referrer": "http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/",
         "http.response.body.bytes": 308,
         "http.response.status_code": 504,
         "http.version": "1.1",
@@ -242,7 +269,15 @@
             "forwarded"
         ],
         "trace.id": "Root=1-5da0997a-5add00b04bc8ae20ae96d9f0",
-        "user_agent.original": "curl/7.58.0"
+        "url.domain": "filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com",
+        "url.original": "http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/",
+        "url.path": "/",
+        "url.port": 80,
+        "url.scheme": "http",
+        "user_agent.device.name": "Other",
+        "user_agent.name": "curl",
+        "user_agent.original": "curl/7.58.0",
+        "user_agent.version": "7.58.0"
     },
     {
         "@timestamp": "2019-10-11T15:02:41.203Z",
@@ -268,7 +303,6 @@
         "fileset.name": "elb",
         "http.request.body.bytes": 125,
         "http.request.method": "GET",
-        "http.request.referrer": "http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/",
         "http.response.body.bytes": 308,
         "http.response.status_code": 504,
         "http.version": "1.1",
@@ -291,7 +325,15 @@
             "forwarded"
         ],
         "trace.id": "Root=1-5da09987-cc391940b332434860dfa848",
-        "user_agent.original": "curl/7.58.0"
+        "url.domain": "filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com",
+        "url.original": "http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/",
+        "url.path": "/",
+        "url.port": 80,
+        "url.scheme": "http",
+        "user_agent.device.name": "Other",
+        "user_agent.name": "curl",
+        "user_agent.original": "curl/7.58.0",
+        "user_agent.version": "7.58.0"
     },
     {
         "@timestamp": "2019-10-11T15:03:49.331Z",
@@ -317,7 +359,6 @@
         "fileset.name": "elb",
         "http.request.body.bytes": 125,
         "http.request.method": "GET",
-        "http.request.referrer": "http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/",
         "http.response.body.bytes": 308,
         "http.response.status_code": 504,
         "http.version": "1.1",
@@ -340,7 +381,15 @@
             "forwarded"
         ],
         "trace.id": "Root=1-5da099cb-3d3b17eb2b75373f4c0c36c5",
-        "user_agent.original": "curl/7.58.0"
+        "url.domain": "filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com",
+        "url.original": "http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/",
+        "url.path": "/",
+        "url.port": 80,
+        "url.scheme": "http",
+        "user_agent.device.name": "Other",
+        "user_agent.name": "curl",
+        "user_agent.original": "curl/7.58.0",
+        "user_agent.version": "7.58.0"
     },
     {
         "@timestamp": "2019-10-11T15:55:09.308Z",
@@ -370,7 +419,6 @@
         "fileset.name": "elb",
         "http.request.body.bytes": 125,
         "http.request.method": "GET",
-        "http.request.referrer": "http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/",
         "http.response.body.bytes": 859,
         "http.response.status_code": 200,
         "http.version": "1.1",
@@ -393,7 +441,15 @@
             "forwarded"
         ],
         "trace.id": "Root=1-5da0a5dd-4d9a423a0e9a782fe2f390af",
-        "user_agent.original": "curl/7.58.0"
+        "url.domain": "filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com",
+        "url.original": "http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/",
+        "url.path": "/",
+        "url.port": 80,
+        "url.scheme": "http",
+        "user_agent.device.name": "Other",
+        "user_agent.name": "curl",
+        "user_agent.original": "curl/7.58.0",
+        "user_agent.version": "7.58.0"
     },
     {
         "@timestamp": "2019-10-11T15:55:11.354Z",
@@ -423,7 +479,6 @@
         "fileset.name": "elb",
         "http.request.body.bytes": 125,
         "http.request.method": "GET",
-        "http.request.referrer": "http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/",
         "http.response.body.bytes": 859,
         "http.response.status_code": 200,
         "http.version": "1.1",
@@ -446,7 +501,15 @@
             "forwarded"
         ],
         "trace.id": "Root=1-5da0a5df-7d64cabe9955b4df9acc800a",
-        "user_agent.original": "curl/7.58.0"
+        "url.domain": "filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com",
+        "url.original": "http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/",
+        "url.path": "/",
+        "url.port": 80,
+        "url.scheme": "http",
+        "user_agent.device.name": "Other",
+        "user_agent.name": "curl",
+        "user_agent.original": "curl/7.58.0",
+        "user_agent.version": "7.58.0"
     },
     {
         "@timestamp": "2019-10-11T15:55:11.987Z",
@@ -476,7 +539,6 @@
         "fileset.name": "elb",
         "http.request.body.bytes": 125,
         "http.request.method": "GET",
-        "http.request.referrer": "http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/",
         "http.response.body.bytes": 859,
         "http.response.status_code": 200,
         "http.version": "1.1",
@@ -499,7 +561,15 @@
             "forwarded"
         ],
         "trace.id": "Root=1-5da0a5df-7c958e828ff43b63d0e0fac4",
-        "user_agent.original": "curl/7.58.0"
+        "url.domain": "filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com",
+        "url.original": "http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/",
+        "url.path": "/",
+        "url.port": 80,
+        "url.scheme": "http",
+        "user_agent.device.name": "Other",
+        "user_agent.name": "curl",
+        "user_agent.original": "curl/7.58.0",
+        "user_agent.version": "7.58.0"
     },
     {
         "@timestamp": "2018-07-02T22:23:00.186Z",
@@ -536,7 +606,6 @@
         "fileset.name": "elb",
         "http.request.body.bytes": 34,
         "http.request.method": "GET",
-        "http.request.referrer": "http://www.example.com:80/",
         "http.response.body.bytes": 366,
         "http.response.status_code": 200,
         "http.version": "1.1",
@@ -549,6 +618,14 @@
             "forwarded"
         ],
         "trace.id": "Root=1-58337262-36d228ad5d99923122bbe354",
-        "user_agent.original": "curl/7.46.0"
+        "url.domain": "www.example.com",
+        "url.original": "http://www.example.com:80/",
+        "url.path": "/",
+        "url.port": 80,
+        "url.scheme": "http",
+        "user_agent.device.name": "Other",
+        "user_agent.name": "curl",
+        "user_agent.original": "curl/7.46.0",
+        "user_agent.version": "7.46.0"
     }
 ]