From d7724cdd18e3b2ca2be44f442740b7d34703ed26 Mon Sep 17 00:00:00 2001 From: Alex Resnick Date: Thu, 24 Jun 2021 17:26:49 -0500 Subject: [PATCH] [Filebeat] Update AWS ELB ingest pipeline (#26441) --- CHANGELOG.next.asciidoc | 4 +- .../module/aws/elb/ingest/pipeline.yml | 14 +- .../application-lb-http.log-expected.json | 121 ++++++++++++++---- .../aws/elb/test/elb-http.log-expected.json | 61 +++++++-- .../test/example-alb-http.log-expected.json | 118 ++++++++++++++--- .../elb/test/example-http.log-expected.json | 29 ++++- .../elb/test/example-https.log-expected.json | 11 +- 7 files changed, 297 insertions(+), 61 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 418f5a6b198c..f64b74a31f05 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -277,7 +277,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix bug in aws-s3 input where the end of gzipped log files might have been discarded. {pull}26260[26260] - Fix bug in `httpjson` that prevented `first_event` getting updated. {pull}26407[26407] - Fix bug in the Syslog input that misparsed rfc5424 days starting with 0. {pull}26419[26419] -- Do not close filestream harvester if an unexpected error is returned when close.on_state_change.* is enabled. {pull}26411[26411] +- Do not close filestream harvester if an unexpected error is returned when close.on_state_change.* is enabled. {pull}26411[26411] *Filebeat* @@ -386,6 +386,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix incorrect field name appending to `related.hash` in `threatintel.abusechmalware` ingest pipeline. {issue}25151[25151] {pull}25674[25674] - Add improvements to the azure activitylogs and platformlogs ingest pipelines. {pull}26148[26148] - Fix `kibana.log` pipeline when `event.duration` calculation becomes a Long. {issue}24556[24556] {pull}25675[25675] +- Removed incorrect `http.request.referrer` field from `aws.elb` module. {issue}26435[26435] {pull}26441[26441] *Heartbeat* @@ -831,6 +832,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Update Okta module to parse additional fields to `okta.debug_context.debug_data`. {issue}25689[25689] {pull}25818[25818] - Added dataset `anomalithreatstream` to the `threatintel` module to ingest indicators from Anomali ThreatStream {pull}26350[26350] - Add support for `copytruncate` method when rotating input logs with an external tool in `filestream` input. {pull}23457[23457] +- Add `uri_parts` and `user_agent` ingest processors to `aws.elb` module. {issue}26435[26435] {pull}26441[26441] *Heartbeat* diff --git a/x-pack/filebeat/module/aws/elb/ingest/pipeline.yml b/x-pack/filebeat/module/aws/elb/ingest/pipeline.yml index 31f5e9932489..9bbba8efc9ce 100644 --- a/x-pack/filebeat/module/aws/elb/ingest/pipeline.yml +++ b/x-pack/filebeat/module/aws/elb/ingest/pipeline.yml @@ -77,8 +77,8 @@ processors: (?:-|%{NUMBER:aws.elb.backend.http.response.status_code:long}) %{NUMBER:http.request.body.bytes:long} %{NUMBER:http.response.body.bytes:long} - \"(?:-|%{WORD:http.request.method}) (?:-|%{NOTSPACE:http.request.referrer}) (?:-|HTTP/%{NOTSPACE:http.version})\" - \"%{DATA:user_agent.original}\" + \"(?:-|%{WORD:http.request.method}) (?:-|%{NOTSPACE:_tmp.uri_orig}) (?:-|HTTP/%{NOTSPACE:http.version})\" + \"%{DATA:_tmp.user_agent}\" %{ELBSSL} ELBTCPLOG: >- %{ELBCOMMON} @@ -105,6 +105,16 @@ processors: field: 'aws.elb.protocol' value: 'http' + - uri_parts: + if: 'ctx?._tmp?.uri_orig != null' + field: _tmp.uri_orig + ignore_failure: true + + - user_agent: + if: 'ctx?._tmp?.user_agent != null' + field: _tmp.user_agent + ignore_missing: true + - set: if: 'ctx.http != null' field: event.category diff --git a/x-pack/filebeat/module/aws/elb/test/application-lb-http.log-expected.json b/x-pack/filebeat/module/aws/elb/test/application-lb-http.log-expected.json index 2e82bd8d32bc..6b1533adb578 100644 --- a/x-pack/filebeat/module/aws/elb/test/application-lb-http.log-expected.json +++ b/x-pack/filebeat/module/aws/elb/test/application-lb-http.log-expected.json @@ -23,7 +23,6 @@ "fileset.name": "elb", "http.request.body.bytes": 125, "http.request.method": "GET", - "http.request.referrer": "http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/", "http.response.body.bytes": 0, "http.response.status_code": 460, "http.version": "1.1", @@ -46,7 +45,15 @@ "forwarded" ], "trace.id": "Root=1-5da09932-2c342a443bfb96249aa50ed7", - "user_agent.original": "curl/7.58.0" + "url.domain": "filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com", + "url.original": "http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/", + "url.path": "/", + "url.port": 80, + "url.scheme": "http", + "user_agent.device.name": "Other", + "user_agent.name": "curl", + "user_agent.original": "curl/7.58.0", + "user_agent.version": "7.58.0" }, { "@timestamp": "2019-10-11T15:01:50.492Z", @@ -72,7 +79,6 @@ "fileset.name": "elb", "http.request.body.bytes": 125, "http.request.method": "GET", - "http.request.referrer": "http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/", "http.response.body.bytes": 308, "http.response.status_code": 504, "http.version": "1.1", @@ -95,7 +101,15 @@ "forwarded" ], "trace.id": "Root=1-5da09954-2c342a443bfb96249aa50ed7", - "user_agent.original": "curl/7.58.0" + "url.domain": "filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com", + "url.original": "http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/", + "url.path": "/", + "url.port": 80, + "url.scheme": "http", + "user_agent.device.name": "Other", + "user_agent.name": "curl", + "user_agent.original": "curl/7.58.0", + "user_agent.version": "7.58.0" }, { "@timestamp": "2019-10-11T15:01:22.915Z", @@ -121,7 +135,6 @@ "fileset.name": "elb", "http.request.body.bytes": 125, "http.request.method": "GET", - "http.request.referrer": "http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/", "http.response.body.bytes": 308, "http.response.status_code": 504, "http.version": "1.1", @@ -144,7 +157,15 @@ "forwarded" ], "trace.id": "Root=1-5da09938-d9c72660e247c36070017828", - "user_agent.original": "curl/7.58.0" + "url.domain": "filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com", + "url.original": "http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/", + "url.path": "/", + "url.port": 80, + "url.scheme": "http", + "user_agent.device.name": "Other", + "user_agent.name": "curl", + "user_agent.original": "curl/7.58.0", + "user_agent.version": "7.58.0" }, { "@timestamp": "2019-10-11T15:01:35.190Z", @@ -170,7 +191,6 @@ "fileset.name": "elb", "http.request.body.bytes": 125, "http.request.method": "GET", - "http.request.referrer": "http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/", "http.response.body.bytes": 308, "http.response.status_code": 504, "http.version": "1.1", @@ -193,7 +213,15 @@ "forwarded" ], "trace.id": "Root=1-5da09945-0eaa8050df7d96f84806ded0", - "user_agent.original": "curl/7.58.0" + "url.domain": "filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com", + "url.original": "http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/", + "url.path": "/", + "url.port": 80, + "url.scheme": "http", + "user_agent.device.name": "Other", + "user_agent.name": "curl", + "user_agent.original": "curl/7.58.0", + "user_agent.version": "7.58.0" }, { "@timestamp": "2019-10-11T15:02:28.837Z", @@ -219,7 +247,6 @@ "fileset.name": "elb", "http.request.body.bytes": 125, "http.request.method": "GET", - "http.request.referrer": "http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/", "http.response.body.bytes": 308, "http.response.status_code": 504, "http.version": "1.1", @@ -242,7 +269,15 @@ "forwarded" ], "trace.id": "Root=1-5da0997a-5add00b04bc8ae20ae96d9f0", - "user_agent.original": "curl/7.58.0" + "url.domain": "filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com", + "url.original": "http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/", + "url.path": "/", + "url.port": 80, + "url.scheme": "http", + "user_agent.device.name": "Other", + "user_agent.name": "curl", + "user_agent.original": "curl/7.58.0", + "user_agent.version": "7.58.0" }, { "@timestamp": "2019-10-11T15:02:41.203Z", @@ -268,7 +303,6 @@ "fileset.name": "elb", "http.request.body.bytes": 125, "http.request.method": "GET", - "http.request.referrer": "http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/", "http.response.body.bytes": 308, "http.response.status_code": 504, "http.version": "1.1", @@ -291,7 +325,15 @@ "forwarded" ], "trace.id": "Root=1-5da09987-cc391940b332434860dfa848", - "user_agent.original": "curl/7.58.0" + "url.domain": "filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com", + "url.original": "http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/", + "url.path": "/", + "url.port": 80, + "url.scheme": "http", + "user_agent.device.name": "Other", + "user_agent.name": "curl", + "user_agent.original": "curl/7.58.0", + "user_agent.version": "7.58.0" }, { "@timestamp": "2019-10-11T15:03:49.331Z", @@ -317,7 +359,6 @@ "fileset.name": "elb", "http.request.body.bytes": 125, "http.request.method": "GET", - "http.request.referrer": "http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/", "http.response.body.bytes": 308, "http.response.status_code": 504, "http.version": "1.1", @@ -340,7 +381,15 @@ "forwarded" ], "trace.id": "Root=1-5da099cb-3d3b17eb2b75373f4c0c36c5", - "user_agent.original": "curl/7.58.0" + "url.domain": "filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com", + "url.original": "http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/", + "url.path": "/", + "url.port": 80, + "url.scheme": "http", + "user_agent.device.name": "Other", + "user_agent.name": "curl", + "user_agent.original": "curl/7.58.0", + "user_agent.version": "7.58.0" }, { "@timestamp": "2019-10-11T15:55:09.308Z", @@ -370,7 +419,6 @@ "fileset.name": "elb", "http.request.body.bytes": 125, "http.request.method": "GET", - "http.request.referrer": "http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/", "http.response.body.bytes": 859, "http.response.status_code": 200, "http.version": "1.1", @@ -393,7 +441,15 @@ "forwarded" ], "trace.id": "Root=1-5da0a5dd-4d9a423a0e9a782fe2f390af", - "user_agent.original": "curl/7.58.0" + "url.domain": "filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com", + "url.original": "http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/", + "url.path": "/", + "url.port": 80, + "url.scheme": "http", + "user_agent.device.name": "Other", + "user_agent.name": "curl", + "user_agent.original": "curl/7.58.0", + "user_agent.version": "7.58.0" }, { "@timestamp": "2019-10-11T15:55:11.354Z", @@ -423,7 +479,6 @@ "fileset.name": "elb", "http.request.body.bytes": 125, "http.request.method": "GET", - "http.request.referrer": "http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/", "http.response.body.bytes": 859, "http.response.status_code": 200, "http.version": "1.1", @@ -446,7 +501,15 @@ "forwarded" ], "trace.id": "Root=1-5da0a5df-7d64cabe9955b4df9acc800a", - "user_agent.original": "curl/7.58.0" + "url.domain": "filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com", + "url.original": "http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/", + "url.path": "/", + "url.port": 80, + "url.scheme": "http", + "user_agent.device.name": "Other", + "user_agent.name": "curl", + "user_agent.original": "curl/7.58.0", + "user_agent.version": "7.58.0" }, { "@timestamp": "2019-10-11T15:55:11.987Z", @@ -476,7 +539,6 @@ "fileset.name": "elb", "http.request.body.bytes": 125, "http.request.method": "GET", - "http.request.referrer": "http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/", "http.response.body.bytes": 859, "http.response.status_code": 200, "http.version": "1.1", @@ -499,7 +561,15 @@ "forwarded" ], "trace.id": "Root=1-5da0a5df-7c958e828ff43b63d0e0fac4", - "user_agent.original": "curl/7.58.0" + "url.domain": "filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com", + "url.original": "http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/", + "url.path": "/", + "url.port": 80, + "url.scheme": "http", + "user_agent.device.name": "Other", + "user_agent.name": "curl", + "user_agent.original": "curl/7.58.0", + "user_agent.version": "7.58.0" }, { "@timestamp": "2018-07-02T22:23:00.186Z", @@ -536,7 +606,6 @@ "fileset.name": "elb", "http.request.body.bytes": 34, "http.request.method": "GET", - "http.request.referrer": "http://www.example.com:80/", "http.response.body.bytes": 366, "http.response.status_code": 200, "http.version": "1.1", @@ -549,6 +618,14 @@ "forwarded" ], "trace.id": "Root=1-58337262-36d228ad5d99923122bbe354", - "user_agent.original": "curl/7.46.0" + "url.domain": "www.example.com", + "url.original": "http://www.example.com:80/", + "url.path": "/", + "url.port": 80, + "url.scheme": "http", + "user_agent.device.name": "Other", + "user_agent.name": "curl", + "user_agent.original": "curl/7.46.0", + "user_agent.version": "7.46.0" } ] \ No newline at end of file diff --git a/x-pack/filebeat/module/aws/elb/test/elb-http.log-expected.json b/x-pack/filebeat/module/aws/elb/test/elb-http.log-expected.json index 48701c8a9853..6f47911622a9 100644 --- a/x-pack/filebeat/module/aws/elb/test/elb-http.log-expected.json +++ b/x-pack/filebeat/module/aws/elb/test/elb-http.log-expected.json @@ -19,7 +19,6 @@ "fileset.name": "elb", "http.request.body.bytes": 0, "http.request.method": "GET", - "http.request.referrer": "http://18.194.223.56:80/", "http.response.body.bytes": 612, "http.response.status_code": 200, "http.version": "1.1", @@ -41,7 +40,18 @@ "tags": [ "forwarded" ], - "user_agent.original": "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36" + "url.domain": "18.194.223.56", + "url.original": "http://18.194.223.56:80/", + "url.path": "/", + "url.port": 80, + "url.scheme": "http", + "user_agent.device.name": "Other", + "user_agent.name": "Chrome", + "user_agent.original": "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36", + "user_agent.os.full": "Windows 7", + "user_agent.os.name": "Windows", + "user_agent.os.version": "7", + "user_agent.version": "41.0.2228.0" }, { "@timestamp": "2019-10-14T12:01:41.918Z", @@ -63,7 +73,6 @@ "fileset.name": "elb", "http.request.body.bytes": 0, "http.request.method": "GET", - "http.request.referrer": "http://18.194.223.56:80/", "http.response.body.bytes": 612, "http.response.status_code": 200, "http.version": "1.1", @@ -85,7 +94,18 @@ "tags": [ "forwarded" ], - "user_agent.original": "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36" + "url.domain": "18.194.223.56", + "url.original": "http://18.194.223.56:80/", + "url.path": "/", + "url.port": 80, + "url.scheme": "http", + "user_agent.device.name": "Other", + "user_agent.name": "Chrome", + "user_agent.original": "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36", + "user_agent.os.full": "Windows 7", + "user_agent.os.name": "Windows", + "user_agent.os.version": "7", + "user_agent.version": "41.0.2228.0" }, { "@timestamp": "2019-10-14T12:01:49.543Z", @@ -107,7 +127,6 @@ "fileset.name": "elb", "http.request.body.bytes": 0, "http.request.method": "GET", - "http.request.referrer": "http://filebeat-aws-elb-test-1703142762.eu-central-1.elb.amazonaws.com:80/", "http.response.body.bytes": 612, "http.response.status_code": 200, "http.version": "1.1", @@ -129,7 +148,15 @@ "tags": [ "forwarded" ], - "user_agent.original": "curl/7.58.0" + "url.domain": "filebeat-aws-elb-test-1703142762.eu-central-1.elb.amazonaws.com", + "url.original": "http://filebeat-aws-elb-test-1703142762.eu-central-1.elb.amazonaws.com:80/", + "url.path": "/", + "url.port": 80, + "url.scheme": "http", + "user_agent.device.name": "Other", + "user_agent.name": "curl", + "user_agent.original": "curl/7.58.0", + "user_agent.version": "7.58.0" }, { "@timestamp": "2019-10-14T12:01:50.199Z", @@ -151,7 +178,6 @@ "fileset.name": "elb", "http.request.body.bytes": 0, "http.request.method": "GET", - "http.request.referrer": "http://filebeat-aws-elb-test-1703142762.eu-central-1.elb.amazonaws.com:80/", "http.response.body.bytes": 612, "http.response.status_code": 200, "http.version": "1.1", @@ -173,7 +199,15 @@ "tags": [ "forwarded" ], - "user_agent.original": "curl/7.58.0" + "url.domain": "filebeat-aws-elb-test-1703142762.eu-central-1.elb.amazonaws.com", + "url.original": "http://filebeat-aws-elb-test-1703142762.eu-central-1.elb.amazonaws.com:80/", + "url.path": "/", + "url.port": 80, + "url.scheme": "http", + "user_agent.device.name": "Other", + "user_agent.name": "curl", + "user_agent.original": "curl/7.58.0", + "user_agent.version": "7.58.0" }, { "@timestamp": "2019-10-14T12:01:50.831Z", @@ -195,7 +229,6 @@ "fileset.name": "elb", "http.request.body.bytes": 0, "http.request.method": "GET", - "http.request.referrer": "http://filebeat-aws-elb-test-1703142762.eu-central-1.elb.amazonaws.com:80/", "http.response.body.bytes": 612, "http.response.status_code": 200, "http.version": "1.1", @@ -217,6 +250,14 @@ "tags": [ "forwarded" ], - "user_agent.original": "curl/7.58.0" + "url.domain": "filebeat-aws-elb-test-1703142762.eu-central-1.elb.amazonaws.com", + "url.original": "http://filebeat-aws-elb-test-1703142762.eu-central-1.elb.amazonaws.com:80/", + "url.path": "/", + "url.port": 80, + "url.scheme": "http", + "user_agent.device.name": "Other", + "user_agent.name": "curl", + "user_agent.original": "curl/7.58.0", + "user_agent.version": "7.58.0" } ] \ No newline at end of file diff --git a/x-pack/filebeat/module/aws/elb/test/example-alb-http.log-expected.json b/x-pack/filebeat/module/aws/elb/test/example-alb-http.log-expected.json index 66d06e583271..2c12c3b61b84 100644 --- a/x-pack/filebeat/module/aws/elb/test/example-alb-http.log-expected.json +++ b/x-pack/filebeat/module/aws/elb/test/example-alb-http.log-expected.json @@ -27,7 +27,6 @@ "fileset.name": "elb", "http.request.body.bytes": 34, "http.request.method": "GET", - "http.request.referrer": "http://www.example.com:80/", "http.response.body.bytes": 366, "http.response.status_code": 200, "http.version": "1.1", @@ -40,7 +39,15 @@ "forwarded" ], "trace.id": "Root=1-58337262-36d228ad5d99923122bbe354", - "user_agent.original": "curl/7.46.0" + "url.domain": "www.example.com", + "url.original": "http://www.example.com:80/", + "url.path": "/", + "url.port": 80, + "url.scheme": "http", + "user_agent.device.name": "Other", + "user_agent.name": "curl", + "user_agent.original": "curl/7.46.0", + "user_agent.version": "7.46.0" }, { "@timestamp": "2018-07-02T22:23:00.186Z", @@ -75,7 +82,6 @@ "fileset.name": "elb", "http.request.body.bytes": 0, "http.request.method": "GET", - "http.request.referrer": "https://www.example.com:443/", "http.response.body.bytes": 57, "http.response.status_code": 200, "http.version": "1.1", @@ -91,7 +97,15 @@ "tls.version": "1.2", "tls.version_protocol": "tls", "trace.id": "Root=1-58337281-1d84f3d73c47ec4e58577259", - "user_agent.original": "curl/7.46.0" + "url.domain": "www.example.com", + "url.original": "https://www.example.com:443/", + "url.path": "/", + "url.port": 443, + "url.scheme": "https", + "user_agent.device.name": "Other", + "user_agent.name": "curl", + "user_agent.original": "curl/7.46.0", + "user_agent.version": "7.46.0" }, { "@timestamp": "2018-07-02T22:23:00.186Z", @@ -124,7 +138,6 @@ "fileset.name": "elb", "http.request.body.bytes": 5, "http.request.method": "GET", - "http.request.referrer": "https://10.0.2.105:773/", "http.response.body.bytes": 257, "http.response.status_code": 200, "http.version": "2.0", @@ -140,7 +153,15 @@ "tls.version": "1.2", "tls.version_protocol": "tls", "trace.id": "Root=1-58337327-72bd00b0343d75b906739c42", - "user_agent.original": "curl/7.46.0" + "url.domain": "10.0.2.105", + "url.original": "https://10.0.2.105:773/", + "url.path": "/", + "url.port": 773, + "url.scheme": "https", + "user_agent.device.name": "Other", + "user_agent.name": "curl", + "user_agent.original": "curl/7.46.0", + "user_agent.version": "7.46.0" }, { "@timestamp": "2018-07-02T22:23:00.186Z", @@ -170,7 +191,6 @@ "fileset.name": "elb", "http.request.body.bytes": 218, "http.request.method": "GET", - "http.request.referrer": "http://10.0.0.30:80/", "http.response.body.bytes": 587, "http.response.status_code": 101, "http.version": "1.1", @@ -183,6 +203,13 @@ "forwarded" ], "trace.id": "Root=1-58337364-23a8c76965a2ef7629b185e3", + "url.domain": "10.0.0.30", + "url.original": "http://10.0.0.30:80/", + "url.path": "/", + "url.port": 80, + "url.scheme": "http", + "user_agent.device.name": "Other", + "user_agent.name": "Other", "user_agent.original": "-" }, { @@ -207,7 +234,6 @@ "fileset.name": "elb", "http.request.body.bytes": 218, "http.request.method": "GET", - "http.request.referrer": "https://10.0.0.30:443/", "http.response.body.bytes": 786, "http.response.status_code": 101, "http.version": "1.1", @@ -222,6 +248,13 @@ "tls.cipher": "ECDHE-RSA-AES128-GCM-SHA256", "tls.version": "1.2", "tls.version_protocol": "tls", + "url.domain": "10.0.0.30", + "url.original": "https://10.0.0.30:443/", + "url.path": "/", + "url.port": 443, + "url.scheme": "https", + "user_agent.device.name": "Other", + "user_agent.name": "Other", "user_agent.original": "-" }, { @@ -250,7 +283,6 @@ "fileset.name": "elb", "http.request.body.bytes": 34, "http.request.method": "GET", - "http.request.referrer": "http://www.example.com:80/", "http.response.body.bytes": 366, "http.response.status_code": 200, "http.version": "1.1", @@ -263,7 +295,15 @@ "forwarded" ], "trace.id": "Root=1-58337364-23a8c76965a2ef7629b185e3", - "user_agent.original": "curl/7.46.0" + "url.domain": "www.example.com", + "url.original": "http://www.example.com:80/", + "url.path": "/", + "url.port": 80, + "url.scheme": "http", + "user_agent.device.name": "Other", + "user_agent.name": "curl", + "user_agent.original": "curl/7.46.0", + "user_agent.version": "7.46.0" }, { "@timestamp": "2018-11-30T22:23:00.186Z", @@ -291,7 +331,6 @@ "fileset.name": "elb", "http.request.body.bytes": 34, "http.request.method": "GET", - "http.request.referrer": "http://www.example.com:80/", "http.response.body.bytes": 366, "http.response.status_code": 502, "http.version": "1.1", @@ -304,7 +343,15 @@ "forwarded" ], "trace.id": "Root=1-58337364-23a8c76965a2ef7629b185e3", - "user_agent.original": "curl/7.46.0" + "url.domain": "www.example.com", + "url.original": "http://www.example.com:80/", + "url.path": "/", + "url.port": 80, + "url.scheme": "http", + "user_agent.device.name": "Other", + "user_agent.name": "curl", + "user_agent.original": "curl/7.46.0", + "user_agent.version": "7.46.0" }, { "@timestamp": "2018-11-30T22:23:00.186Z", @@ -324,7 +371,6 @@ "event.start": "2018-11-30T22:22:48.364000Z", "fileset.name": "elb", "http.request.body.bytes": 0, - "http.request.referrer": "http://www.example.com:80-", "http.response.body.bytes": 0, "http.response.status_code": 400, "input.type": "log", @@ -336,6 +382,12 @@ "forwarded" ], "trace.id": "-", + "url.domain": null, + "url.original": "http://www.example.com:80-", + "url.path": "", + "url.scheme": "http", + "user_agent.device.name": "Other", + "user_agent.name": "Other", "user_agent.original": "-" }, { @@ -367,6 +419,8 @@ "forwarded" ], "trace.id": "-", + "user_agent.device.name": "Other", + "user_agent.name": "Other", "user_agent.original": "-" }, { @@ -406,7 +460,6 @@ "fileset.name": "elb", "http.request.body.bytes": 5, "http.request.method": "GET", - "http.request.referrer": "https://10.0.2.105:773/", "http.response.body.bytes": 257, "http.response.status_code": 200, "http.version": "2.0", @@ -422,7 +475,15 @@ "tls.version": "1.2", "tls.version_protocol": "tls", "trace.id": "Root=1-58337327-72bd00b0343d75b906739c42", - "user_agent.original": "curl/7.46.0" + "url.domain": "10.0.2.105", + "url.original": "https://10.0.2.105:773/", + "url.path": "/", + "url.port": 773, + "url.scheme": "https", + "user_agent.device.name": "Other", + "user_agent.name": "curl", + "user_agent.original": "curl/7.46.0", + "user_agent.version": "7.46.0" }, { "@timestamp": "2018-07-02T22:23:00.186Z", @@ -463,7 +524,6 @@ "fileset.name": "elb", "http.request.body.bytes": 0, "http.request.method": "GET", - "http.request.referrer": "https://www.example.com:443/", "http.response.body.bytes": 57, "http.response.status_code": 200, "http.version": "1.1", @@ -479,7 +539,15 @@ "tls.version": "1.2", "tls.version_protocol": "tls", "trace.id": "Root=1-58337281-1d84f3d73c47ec4e58577259", - "user_agent.original": "curl/7.46.0" + "url.domain": "www.example.com", + "url.original": "https://www.example.com:443/", + "url.path": "/", + "url.port": 443, + "url.scheme": "https", + "user_agent.device.name": "Other", + "user_agent.name": "curl", + "user_agent.original": "curl/7.46.0", + "user_agent.version": "7.46.0" }, { "@timestamp": "2018-07-02T22:23:00.186Z", @@ -515,7 +583,6 @@ "fileset.name": "elb", "http.request.body.bytes": 218, "http.request.method": "GET", - "http.request.referrer": "http://10.0.0.30:80/", "http.response.body.bytes": 587, "http.response.status_code": 101, "http.version": "1.1", @@ -528,6 +595,13 @@ "forwarded" ], "trace.id": "Root=1-58337364-23a8c76965a2ef7629b185e3", + "url.domain": "10.0.0.30", + "url.original": "http://10.0.0.30:80/", + "url.path": "/", + "url.port": 80, + "url.scheme": "http", + "user_agent.device.name": "Other", + "user_agent.name": "Other", "user_agent.original": "-" }, { @@ -566,7 +640,6 @@ "fileset.name": "elb", "http.request.body.bytes": 218, "http.request.method": "GET", - "http.request.referrer": "https://10.0.0.30:443/", "http.response.body.bytes": 786, "http.response.status_code": 101, "http.version": "1.1", @@ -582,6 +655,13 @@ "tls.version": "1.2", "tls.version_protocol": "tls", "trace.id": "Root=1-58337364-23a8c76965a2ef7629b185e3", + "url.domain": "10.0.0.30", + "url.original": "https://10.0.0.30:443/", + "url.path": "/", + "url.port": 443, + "url.scheme": "https", + "user_agent.device.name": "Other", + "user_agent.name": "Other", "user_agent.original": "-" } ] \ No newline at end of file diff --git a/x-pack/filebeat/module/aws/elb/test/example-http.log-expected.json b/x-pack/filebeat/module/aws/elb/test/example-http.log-expected.json index 8a5c542f5a05..6cee3bb663d0 100644 --- a/x-pack/filebeat/module/aws/elb/test/example-http.log-expected.json +++ b/x-pack/filebeat/module/aws/elb/test/example-http.log-expected.json @@ -19,7 +19,6 @@ "fileset.name": "elb", "http.request.body.bytes": 0, "http.request.method": "GET", - "http.request.referrer": "http://www.example.com:80/", "http.response.body.bytes": 29, "http.response.status_code": 200, "http.version": "1.1", @@ -31,7 +30,15 @@ "tags": [ "forwarded" ], - "user_agent.original": "curl/7.38.0" + "url.domain": "www.example.com", + "url.original": "http://www.example.com:80/", + "url.path": "/", + "url.port": 80, + "url.scheme": "http", + "user_agent.device.name": "Other", + "user_agent.name": "curl", + "user_agent.original": "curl/7.38.0", + "user_agent.version": "7.38.0" }, { "@timestamp": "2015-05-13T23:39:43.945Z", @@ -47,7 +54,6 @@ "fileset.name": "elb", "http.request.body.bytes": 0, "http.request.method": "GET", - "http.request.referrer": "http://www.example.com:80/", "http.response.body.bytes": 0, "http.response.status_code": 503, "http.version": "1.1", @@ -59,7 +65,15 @@ "tags": [ "forwarded" ], - "user_agent.original": "curl/7.38.0" + "url.domain": "www.example.com", + "url.original": "http://www.example.com:80/", + "url.path": "/", + "url.port": 80, + "url.scheme": "http", + "user_agent.device.name": "Other", + "user_agent.name": "curl", + "user_agent.original": "curl/7.38.0", + "user_agent.version": "7.38.0" }, { "@timestamp": "2015-05-13T23:39:43.945Z", @@ -75,7 +89,6 @@ "fileset.name": "elb", "http.request.body.bytes": 0, "http.request.method": "GET", - "http.request.referrer": "http://www.example.com:80-", "http.response.body.bytes": 0, "http.response.status_code": 400, "input.type": "log", @@ -86,6 +99,12 @@ "tags": [ "forwarded" ], + "url.domain": null, + "url.original": "http://www.example.com:80-", + "url.path": "", + "url.scheme": "http", + "user_agent.device.name": "Other", + "user_agent.name": "Other", "user_agent.original": "-" } ] \ No newline at end of file diff --git a/x-pack/filebeat/module/aws/elb/test/example-https.log-expected.json b/x-pack/filebeat/module/aws/elb/test/example-https.log-expected.json index 56baf18563af..330ea15f0102 100644 --- a/x-pack/filebeat/module/aws/elb/test/example-https.log-expected.json +++ b/x-pack/filebeat/module/aws/elb/test/example-https.log-expected.json @@ -21,7 +21,6 @@ "fileset.name": "elb", "http.request.body.bytes": 0, "http.request.method": "GET", - "http.request.referrer": "https://www.example.com:443/", "http.response.body.bytes": 57, "http.response.status_code": 200, "http.version": "1.1", @@ -36,6 +35,14 @@ "tls.cipher": "DHE-RSA-AES128-SHA", "tls.version": "1.2", "tls.version_protocol": "tls", - "user_agent.original": "curl/7.38.0" + "url.domain": "www.example.com", + "url.original": "https://www.example.com:443/", + "url.path": "/", + "url.port": 443, + "url.scheme": "https", + "user_agent.device.name": "Other", + "user_agent.name": "curl", + "user_agent.original": "curl/7.38.0", + "user_agent.version": "7.38.0" } ] \ No newline at end of file