From 595cc41fa67071f656f6da51f8525528e023a87e Mon Sep 17 00:00:00 2001 From: Lee Hinman <57081003+leehinman@users.noreply.github.com> Date: Fri, 21 Feb 2020 08:38:48 -0600 Subject: [PATCH] [Filebeat] Improve ECS field mappings in aws module (#16307) (#16478) * Improve ECS field mappings in aws module - elb fileset + cloud.provider + event.category + event.kind + event.outcome + http.response.status_code, convert to long + http.request.method, lowercase + tracing.trace.id - s3access fileset + client.address + client.ip + geo + client.user.id + cloud.provider + event.action + event.code + event.duration + event.id + event.kind + event.outcome + http.request.referrer + http.response.status_code + related.ip + related.user + user_agent - vpcflow fileset + cloud.provider + cloud.account.id + cloud.instance.id + event.kind Closes #16154 (cherry picked from commit 913f7eeaa76dcd072854b2696b32cde78763599e) --- CHANGELOG.next.asciidoc | 1 + .../module/aws/elb/ingest/pipeline.yml | 40 +++- .../application-lb-http.log-expected.json | 96 +++++++--- .../aws/elb/test/elb-http.log-expected.json | 50 +++-- .../aws/elb/test/elb-tcp.log-expected.json | 18 ++ .../test/example-alb-http.log-expected.json | 88 ++++++--- .../elb/test/example-http.log-expected.json | 26 ++- .../elb/test/example-https.log-expected.json | 10 +- .../test/example-nlb-tcp.log-expected.json | 3 + .../elb/test/example-ssl.log-expected.json | 3 + .../elb/test/example-tcp.log-expected.json | 6 + .../module/aws/s3access/ingest/pipeline.yml | 90 ++++++++- .../test/s3_server_access.log-expected.json | 176 +++++++++++++++++- .../aws/s3access/test/test.log-expected.json | 111 ++++++++++- .../module/aws/vpcflow/ingest/pipeline.yml | 18 ++ .../accept-reject-traffic.log-expected.json | 12 ++ .../test/custom-nat-gateway.log-expected.json | 5 + .../custom-transit-gateway.log-expected.json | 4 + .../aws/vpcflow/test/ipv6.log-expected.json | 3 + .../test/no-data-skip-data.log-expected.json | 6 + .../test/tcp-flag-sequence.log-expected.json | 4 + 21 files changed, 679 insertions(+), 91 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index a9c4870a574..1723bd0249e 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -204,6 +204,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add an SSL config example in config.yml for filebeat MISP module. {pull}16320[16320] - Improve ECS categorization, container & process field mappings in auditd module. {issue}16153[16153] {pull}16280[16280] - Add ECS categorization fields to activemq module. {issue}16151[16151] {pull}16201[16201] +- Improve ECS field mappings in aws module. {issue}16154[16154] {pull}16307[16307] *Heartbeat* diff --git a/x-pack/filebeat/module/aws/elb/ingest/pipeline.yml b/x-pack/filebeat/module/aws/elb/ingest/pipeline.yml index 802ea112f37..a206ccf314a 100644 --- a/x-pack/filebeat/module/aws/elb/ingest/pipeline.yml +++ b/x-pack/filebeat/module/aws/elb/ingest/pipeline.yml @@ -70,8 +70,8 @@ processors: %{ELBPROCESSINGTIME} ELBHTTPLOG: >- %{ELBCOMMON} - %{NUMBER:http.response.status_code} - (?:-|%{NUMBER:aws.elb.backend.http.response.status_code}) + %{NUMBER:http.response.status_code:long} + (?:-|%{NUMBER:aws.elb.backend.http.response.status_code:long}) %{NUMBER:http.request.body.bytes:long} %{NUMBER:http.response.body.bytes:long} \"(?:-|%{WORD:http.request.method}) (?:-|%{NOTSPACE:http.request.referrer}) (?:-|HTTP/%{NOTSPACE:http.version})\" @@ -89,17 +89,53 @@ processors: ELBV2TYPE: '%{WORD:aws.elb.type}' ELBV2LOGVERSION: '%{NOTSPACE}' # Could be used to support different log versions, only 1.0 exists now + - set: + field: event.kind + value: event + + - set: + field: cloud.provider + value: aws - set: if: 'ctx.http != null' field: 'aws.elb.protocol' value: 'http' + - set: + if: 'ctx.http != null' + field: event.category + value: web + - set: if: 'ctx.http == null' field: 'aws.elb.protocol' value: 'tcp' + - set: + if: 'ctx.http == null' + field: event.category + value: network + + - set: + if: 'ctx?.http?.response?.status_code != null && ctx.http.response.status_code < 400' + field: event.outcome + value: success + + - set: + if: 'ctx?.http?.response?.status_code != null && ctx.http.response.status_code >= 400' + field: event.outcome + value: failure + + - lowercase: + field: http.request.method + ignore_missing: true + + - set: + if: "ctx?.aws?.elb?.trace_id != null" + field: tracing.trace.id + value: "{{aws.elb.trace_id}}" + - split: field: '_tmp.actions_executed' target_field: 'aws.elb.action_executed' diff --git a/x-pack/filebeat/module/aws/elb/test/application-lb-http.log-expected.json b/x-pack/filebeat/module/aws/elb/test/application-lb-http.log-expected.json index 5d461f641c2..093cc1fc2e7 100644 --- a/x-pack/filebeat/module/aws/elb/test/application-lb-http.log-expected.json +++ b/x-pack/filebeat/module/aws/elb/test/application-lb-http.log-expected.json @@ -12,16 +12,20 @@ "aws.elb.target_group.arn": "arn:aws:elasticloadbalancing:eu-central-1:627959692251:targetgroup/test-lb-instances/8f04c4fe71f5f794", "aws.elb.trace_id": "Root=1-5da09932-2c342a443bfb96249aa50ed7", "aws.elb.type": "http", + "cloud.provider": "aws", + "event.category": "web", "event.dataset": "aws.elb", "event.end": "2019-10-11T15:01:12.376Z", + "event.kind": "event", "event.module": "aws", + "event.outcome": "failure", "event.start": "2019-10-11T15:01:06.657000Z", "fileset.name": "elb", "http.request.body.bytes": 125, - "http.request.method": "GET", + "http.request.method": "get", "http.request.referrer": "http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/", "http.response.body.bytes": 0, - "http.response.status_code": "460", + "http.response.status_code": 460, "http.version": "1.1", "input.type": "log", "log.offset": 0, @@ -37,6 +41,7 @@ "source.geo.region_name": "Teruel", "source.ip": "77.227.156.41", "source.port": "56398", + "tracing.trace.id": "Root=1-5da09932-2c342a443bfb96249aa50ed7", "user_agent.original": "curl/7.58.0" }, { @@ -52,16 +57,20 @@ "aws.elb.target_group.arn": "arn:aws:elasticloadbalancing:eu-central-1:627959692251:targetgroup/test-lb-instances/8f04c4fe71f5f794", "aws.elb.trace_id": "Root=1-5da09954-2c342a443bfb96249aa50ed7", "aws.elb.type": "http", + "cloud.provider": "aws", + "event.category": "web", "event.dataset": "aws.elb", "event.end": "2019-10-11T15:01:50.492Z", + "event.kind": "event", "event.module": "aws", + "event.outcome": "failure", "event.start": "2019-10-11T15:01:40.491000Z", "fileset.name": "elb", "http.request.body.bytes": 125, - "http.request.method": "GET", + "http.request.method": "get", "http.request.referrer": "http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/", "http.response.body.bytes": 308, - "http.response.status_code": "504", + "http.response.status_code": 504, "http.version": "1.1", "input.type": "log", "log.offset": 438, @@ -77,6 +86,7 @@ "source.geo.region_name": "Teruel", "source.ip": "77.227.156.41", "source.port": "56488", + "tracing.trace.id": "Root=1-5da09954-2c342a443bfb96249aa50ed7", "user_agent.original": "curl/7.58.0" }, { @@ -92,16 +102,20 @@ "aws.elb.target_group.arn": "arn:aws:elasticloadbalancing:eu-central-1:627959692251:targetgroup/test-lb-instances/8f04c4fe71f5f794", "aws.elb.trace_id": "Root=1-5da09938-d9c72660e247c36070017828", "aws.elb.type": "http", + "cloud.provider": "aws", + "event.category": "web", "event.dataset": "aws.elb", "event.end": "2019-10-11T15:01:22.915Z", + "event.kind": "event", "event.module": "aws", + "event.outcome": "failure", "event.start": "2019-10-11T15:01:12.914000Z", "fileset.name": "elb", "http.request.body.bytes": 125, - "http.request.method": "GET", + "http.request.method": "get", "http.request.referrer": "http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/", "http.response.body.bytes": 308, - "http.response.status_code": "504", + "http.response.status_code": 504, "http.version": "1.1", "input.type": "log", "log.offset": 878, @@ -117,6 +131,7 @@ "source.geo.region_name": "Teruel", "source.ip": "77.227.156.41", "source.port": "56416", + "tracing.trace.id": "Root=1-5da09938-d9c72660e247c36070017828", "user_agent.original": "curl/7.58.0" }, { @@ -132,16 +147,20 @@ "aws.elb.target_group.arn": "arn:aws:elasticloadbalancing:eu-central-1:627959692251:targetgroup/test-lb-instances/8f04c4fe71f5f794", "aws.elb.trace_id": "Root=1-5da09945-0eaa8050df7d96f84806ded0", "aws.elb.type": "http", + "cloud.provider": "aws", + "event.category": "web", "event.dataset": "aws.elb", "event.end": "2019-10-11T15:01:35.190Z", + "event.kind": "event", "event.module": "aws", + "event.outcome": "failure", "event.start": "2019-10-11T15:01:25.189000Z", "fileset.name": "elb", "http.request.body.bytes": 125, - "http.request.method": "GET", + "http.request.method": "get", "http.request.referrer": "http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/", "http.response.body.bytes": 308, - "http.response.status_code": "504", + "http.response.status_code": 504, "http.version": "1.1", "input.type": "log", "log.offset": 1318, @@ -157,6 +176,7 @@ "source.geo.region_name": "Teruel", "source.ip": "77.227.156.41", "source.port": "56448", + "tracing.trace.id": "Root=1-5da09945-0eaa8050df7d96f84806ded0", "user_agent.original": "curl/7.58.0" }, { @@ -172,16 +192,20 @@ "aws.elb.target_group.arn": "arn:aws:elasticloadbalancing:eu-central-1:627959692251:targetgroup/test-lb-instances/8f04c4fe71f5f794", "aws.elb.trace_id": "Root=1-5da0997a-5add00b04bc8ae20ae96d9f0", "aws.elb.type": "http", + "cloud.provider": "aws", + "event.category": "web", "event.dataset": "aws.elb", "event.end": "2019-10-11T15:02:28.837Z", + "event.kind": "event", "event.module": "aws", + "event.outcome": "failure", "event.start": "2019-10-11T15:02:18.836000Z", "fileset.name": "elb", "http.request.body.bytes": 125, - "http.request.method": "GET", + "http.request.method": "get", "http.request.referrer": "http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/", "http.response.body.bytes": 308, - "http.response.status_code": "504", + "http.response.status_code": 504, "http.version": "1.1", "input.type": "log", "log.offset": 1758, @@ -197,6 +221,7 @@ "source.geo.region_name": "Teruel", "source.ip": "77.227.156.41", "source.port": "56602", + "tracing.trace.id": "Root=1-5da0997a-5add00b04bc8ae20ae96d9f0", "user_agent.original": "curl/7.58.0" }, { @@ -212,16 +237,20 @@ "aws.elb.target_group.arn": "arn:aws:elasticloadbalancing:eu-central-1:627959692251:targetgroup/test-lb-instances/8f04c4fe71f5f794", "aws.elb.trace_id": "Root=1-5da09987-cc391940b332434860dfa848", "aws.elb.type": "http", + "cloud.provider": "aws", + "event.category": "web", "event.dataset": "aws.elb", "event.end": "2019-10-11T15:02:41.203Z", + "event.kind": "event", "event.module": "aws", + "event.outcome": "failure", "event.start": "2019-10-11T15:02:31.202000Z", "fileset.name": "elb", "http.request.body.bytes": 125, - "http.request.method": "GET", + "http.request.method": "get", "http.request.referrer": "http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/", "http.response.body.bytes": 308, - "http.response.status_code": "504", + "http.response.status_code": 504, "http.version": "1.1", "input.type": "log", "log.offset": 2198, @@ -237,6 +266,7 @@ "source.geo.region_name": "Teruel", "source.ip": "77.227.156.41", "source.port": "56638", + "tracing.trace.id": "Root=1-5da09987-cc391940b332434860dfa848", "user_agent.original": "curl/7.58.0" }, { @@ -252,16 +282,20 @@ "aws.elb.target_group.arn": "arn:aws:elasticloadbalancing:eu-central-1:627959692251:targetgroup/test-lb-instances/8f04c4fe71f5f794", "aws.elb.trace_id": "Root=1-5da099cb-3d3b17eb2b75373f4c0c36c5", "aws.elb.type": "http", + "cloud.provider": "aws", + "event.category": "web", "event.dataset": "aws.elb", "event.end": "2019-10-11T15:03:49.331Z", + "event.kind": "event", "event.module": "aws", + "event.outcome": "failure", "event.start": "2019-10-11T15:03:39.331000Z", "fileset.name": "elb", "http.request.body.bytes": 125, - "http.request.method": "GET", + "http.request.method": "get", "http.request.referrer": "http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/", "http.response.body.bytes": 308, - "http.response.status_code": "504", + "http.response.status_code": 504, "http.version": "1.1", "input.type": "log", "log.offset": 2638, @@ -277,6 +311,7 @@ "source.geo.region_name": "Teruel", "source.ip": "77.227.156.41", "source.port": "37632", + "tracing.trace.id": "Root=1-5da099cb-3d3b17eb2b75373f4c0c36c5", "user_agent.original": "curl/7.58.0" }, { @@ -284,7 +319,7 @@ "aws.elb.action_executed": [ "forward" ], - "aws.elb.backend.http.response.status_code": "200", + "aws.elb.backend.http.response.status_code": 200, "aws.elb.backend.ip": "10.0.0.192", "aws.elb.backend.port": "80", "aws.elb.backend_processing_time.sec": 0.0, @@ -296,16 +331,20 @@ "aws.elb.target_group.arn": "arn:aws:elasticloadbalancing:eu-central-1:627959692251:targetgroup/test-lb-instances/8f04c4fe71f5f794", "aws.elb.trace_id": "Root=1-5da0a5dd-4d9a423a0e9a782fe2f390af", "aws.elb.type": "http", + "cloud.provider": "aws", + "event.category": "web", "event.dataset": "aws.elb", "event.end": "2019-10-11T15:55:09.308Z", + "event.kind": "event", "event.module": "aws", + "event.outcome": "success", "event.start": "2019-10-11T15:55:09.307000Z", "fileset.name": "elb", "http.request.body.bytes": 125, - "http.request.method": "GET", + "http.request.method": "get", "http.request.referrer": "http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/", "http.response.body.bytes": 859, - "http.response.status_code": "200", + "http.response.status_code": 200, "http.version": "1.1", "input.type": "log", "log.offset": 3078, @@ -321,6 +360,7 @@ "source.geo.region_name": "Teruel", "source.ip": "77.227.156.41", "source.port": "37838", + "tracing.trace.id": "Root=1-5da0a5dd-4d9a423a0e9a782fe2f390af", "user_agent.original": "curl/7.58.0" }, { @@ -328,7 +368,7 @@ "aws.elb.action_executed": [ "forward" ], - "aws.elb.backend.http.response.status_code": "200", + "aws.elb.backend.http.response.status_code": 200, "aws.elb.backend.ip": "10.0.1.107", "aws.elb.backend.port": "80", "aws.elb.backend_processing_time.sec": 0.001, @@ -340,16 +380,20 @@ "aws.elb.target_group.arn": "arn:aws:elasticloadbalancing:eu-central-1:627959692251:targetgroup/test-lb-instances/8f04c4fe71f5f794", "aws.elb.trace_id": "Root=1-5da0a5df-7d64cabe9955b4df9acc800a", "aws.elb.type": "http", + "cloud.provider": "aws", + "event.category": "web", "event.dataset": "aws.elb", "event.end": "2019-10-11T15:55:11.354Z", + "event.kind": "event", "event.module": "aws", + "event.outcome": "success", "event.start": "2019-10-11T15:55:11.352000Z", "fileset.name": "elb", "http.request.body.bytes": 125, - "http.request.method": "GET", + "http.request.method": "get", "http.request.referrer": "http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/", "http.response.body.bytes": 859, - "http.response.status_code": "200", + "http.response.status_code": 200, "http.version": "1.1", "input.type": "log", "log.offset": 3529, @@ -365,6 +409,7 @@ "source.geo.region_name": "Teruel", "source.ip": "77.227.156.41", "source.port": "37850", + "tracing.trace.id": "Root=1-5da0a5df-7d64cabe9955b4df9acc800a", "user_agent.original": "curl/7.58.0" }, { @@ -372,7 +417,7 @@ "aws.elb.action_executed": [ "forward" ], - "aws.elb.backend.http.response.status_code": "200", + "aws.elb.backend.http.response.status_code": 200, "aws.elb.backend.ip": "10.0.0.192", "aws.elb.backend.port": "80", "aws.elb.backend_processing_time.sec": 0.001, @@ -384,16 +429,20 @@ "aws.elb.target_group.arn": "arn:aws:elasticloadbalancing:eu-central-1:627959692251:targetgroup/test-lb-instances/8f04c4fe71f5f794", "aws.elb.trace_id": "Root=1-5da0a5df-7c958e828ff43b63d0e0fac4", "aws.elb.type": "http", + "cloud.provider": "aws", + "event.category": "web", "event.dataset": "aws.elb", "event.end": "2019-10-11T15:55:11.987Z", + "event.kind": "event", "event.module": "aws", + "event.outcome": "success", "event.start": "2019-10-11T15:55:11.987000Z", "fileset.name": "elb", "http.request.body.bytes": 125, - "http.request.method": "GET", + "http.request.method": "get", "http.request.referrer": "http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/", "http.response.body.bytes": 859, - "http.response.status_code": "200", + "http.response.status_code": 200, "http.version": "1.1", "input.type": "log", "log.offset": 3980, @@ -409,6 +458,7 @@ "source.geo.region_name": "Teruel", "source.ip": "77.227.156.41", "source.port": "37856", + "tracing.trace.id": "Root=1-5da0a5df-7c958e828ff43b63d0e0fac4", "user_agent.original": "curl/7.58.0" } ] \ No newline at end of file diff --git a/x-pack/filebeat/module/aws/elb/test/elb-http.log-expected.json b/x-pack/filebeat/module/aws/elb/test/elb-http.log-expected.json index bd3627c8d73..f8b0d751e75 100644 --- a/x-pack/filebeat/module/aws/elb/test/elb-http.log-expected.json +++ b/x-pack/filebeat/module/aws/elb/test/elb-http.log-expected.json @@ -1,7 +1,7 @@ [ { "@timestamp": "2019-10-14T12:00:20.694Z", - "aws.elb.backend.http.response.status_code": "200", + "aws.elb.backend.http.response.status_code": 200, "aws.elb.backend.ip": "10.0.1.185", "aws.elb.backend.port": "80", "aws.elb.backend_processing_time.sec": 0.000785, @@ -9,15 +9,19 @@ "aws.elb.protocol": "http", "aws.elb.request_processing_time.sec": 4.3e-05, "aws.elb.response_processing_time.sec": 2.3e-05, + "cloud.provider": "aws", + "event.category": "web", "event.dataset": "aws.elb", "event.end": "2019-10-14T12:00:20.694Z", + "event.kind": "event", "event.module": "aws", + "event.outcome": "success", "fileset.name": "elb", "http.request.body.bytes": 0, - "http.request.method": "GET", + "http.request.method": "get", "http.request.referrer": "http://18.194.223.56:80/", "http.response.body.bytes": 612, - "http.response.status_code": "200", + "http.response.status_code": 200, "http.version": "1.1", "input.type": "log", "log.offset": 0, @@ -37,7 +41,7 @@ }, { "@timestamp": "2019-10-14T12:01:41.918Z", - "aws.elb.backend.http.response.status_code": "200", + "aws.elb.backend.http.response.status_code": 200, "aws.elb.backend.ip": "10.0.0.169", "aws.elb.backend.port": "80", "aws.elb.backend_processing_time.sec": 0.00491, @@ -45,15 +49,19 @@ "aws.elb.protocol": "http", "aws.elb.request_processing_time.sec": 4.1e-05, "aws.elb.response_processing_time.sec": 2.7e-05, + "cloud.provider": "aws", + "event.category": "web", "event.dataset": "aws.elb", "event.end": "2019-10-14T12:01:41.918Z", + "event.kind": "event", "event.module": "aws", + "event.outcome": "success", "fileset.name": "elb", "http.request.body.bytes": 0, - "http.request.method": "GET", + "http.request.method": "get", "http.request.referrer": "http://18.194.223.56:80/", "http.response.body.bytes": 612, - "http.response.status_code": "200", + "http.response.status_code": 200, "http.version": "1.1", "input.type": "log", "log.offset": 271, @@ -73,7 +81,7 @@ }, { "@timestamp": "2019-10-14T12:01:49.543Z", - "aws.elb.backend.http.response.status_code": "200", + "aws.elb.backend.http.response.status_code": 200, "aws.elb.backend.ip": "10.0.1.185", "aws.elb.backend.port": "80", "aws.elb.backend_processing_time.sec": 0.00079, @@ -81,15 +89,19 @@ "aws.elb.protocol": "http", "aws.elb.request_processing_time.sec": 4.1e-05, "aws.elb.response_processing_time.sec": 2.4e-05, + "cloud.provider": "aws", + "event.category": "web", "event.dataset": "aws.elb", "event.end": "2019-10-14T12:01:49.543Z", + "event.kind": "event", "event.module": "aws", + "event.outcome": "success", "fileset.name": "elb", "http.request.body.bytes": 0, - "http.request.method": "GET", + "http.request.method": "get", "http.request.referrer": "http://filebeat-aws-elb-test-1703142762.eu-central-1.elb.amazonaws.com:80/", "http.response.body.bytes": 612, - "http.response.status_code": "200", + "http.response.status_code": 200, "http.version": "1.1", "input.type": "log", "log.offset": 540, @@ -109,7 +121,7 @@ }, { "@timestamp": "2019-10-14T12:01:50.199Z", - "aws.elb.backend.http.response.status_code": "200", + "aws.elb.backend.http.response.status_code": 200, "aws.elb.backend.ip": "10.0.0.169", "aws.elb.backend.port": "80", "aws.elb.backend_processing_time.sec": 0.001184, @@ -117,15 +129,19 @@ "aws.elb.protocol": "http", "aws.elb.request_processing_time.sec": 3.9e-05, "aws.elb.response_processing_time.sec": 2.8e-05, + "cloud.provider": "aws", + "event.category": "web", "event.dataset": "aws.elb", "event.end": "2019-10-14T12:01:50.199Z", + "event.kind": "event", "event.module": "aws", + "event.outcome": "success", "fileset.name": "elb", "http.request.body.bytes": 0, - "http.request.method": "GET", + "http.request.method": "get", "http.request.referrer": "http://filebeat-aws-elb-test-1703142762.eu-central-1.elb.amazonaws.com:80/", "http.response.body.bytes": 612, - "http.response.status_code": "200", + "http.response.status_code": 200, "http.version": "1.1", "input.type": "log", "log.offset": 772, @@ -145,7 +161,7 @@ }, { "@timestamp": "2019-10-14T12:01:50.831Z", - "aws.elb.backend.http.response.status_code": "200", + "aws.elb.backend.http.response.status_code": 200, "aws.elb.backend.ip": "10.0.1.185", "aws.elb.backend.port": "80", "aws.elb.backend_processing_time.sec": 0.000787, @@ -153,15 +169,19 @@ "aws.elb.protocol": "http", "aws.elb.request_processing_time.sec": 3.8e-05, "aws.elb.response_processing_time.sec": 2.4e-05, + "cloud.provider": "aws", + "event.category": "web", "event.dataset": "aws.elb", "event.end": "2019-10-14T12:01:50.831Z", + "event.kind": "event", "event.module": "aws", + "event.outcome": "success", "fileset.name": "elb", "http.request.body.bytes": 0, - "http.request.method": "GET", + "http.request.method": "get", "http.request.referrer": "http://filebeat-aws-elb-test-1703142762.eu-central-1.elb.amazonaws.com:80/", "http.response.body.bytes": 612, - "http.response.status_code": "200", + "http.response.status_code": 200, "http.version": "1.1", "input.type": "log", "log.offset": 1005, diff --git a/x-pack/filebeat/module/aws/elb/test/elb-tcp.log-expected.json b/x-pack/filebeat/module/aws/elb/test/elb-tcp.log-expected.json index b0a8b26a99e..c587af8defb 100644 --- a/x-pack/filebeat/module/aws/elb/test/elb-tcp.log-expected.json +++ b/x-pack/filebeat/module/aws/elb/test/elb-tcp.log-expected.json @@ -8,9 +8,12 @@ "aws.elb.protocol": "tcp", "aws.elb.request_processing_time.sec": 0.000943, "aws.elb.response_processing_time.sec": 1.5e-05, + "cloud.provider": "aws", "destination.bytes": 859, + "event.category": "network", "event.dataset": "aws.elb", "event.end": "2019-10-17T13:22:51.758Z", + "event.kind": "event", "event.module": "aws", "fileset.name": "elb", "input.type": "log", @@ -38,9 +41,12 @@ "aws.elb.protocol": "tcp", "aws.elb.request_processing_time.sec": 0.000501, "aws.elb.response_processing_time.sec": 1.5e-05, + "cloud.provider": "aws", "destination.bytes": 859, + "event.category": "network", "event.dataset": "aws.elb", "event.end": "2019-10-17T13:23:07.523Z", + "event.kind": "event", "event.module": "aws", "fileset.name": "elb", "input.type": "log", @@ -68,9 +74,12 @@ "aws.elb.protocol": "tcp", "aws.elb.request_processing_time.sec": 0.001105, "aws.elb.response_processing_time.sec": 1.5e-05, + "cloud.provider": "aws", "destination.bytes": 859, + "event.category": "network", "event.dataset": "aws.elb", "event.end": "2019-10-17T13:23:08.477Z", + "event.kind": "event", "event.module": "aws", "fileset.name": "elb", "input.type": "log", @@ -98,9 +107,12 @@ "aws.elb.protocol": "tcp", "aws.elb.request_processing_time.sec": 0.000422, "aws.elb.response_processing_time.sec": 1.3e-05, + "cloud.provider": "aws", "destination.bytes": 859, + "event.category": "network", "event.dataset": "aws.elb", "event.end": "2019-10-17T13:23:09.174Z", + "event.kind": "event", "event.module": "aws", "fileset.name": "elb", "input.type": "log", @@ -128,9 +140,12 @@ "aws.elb.protocol": "tcp", "aws.elb.request_processing_time.sec": 0.000534, "aws.elb.response_processing_time.sec": 1.6e-05, + "cloud.provider": "aws", "destination.bytes": 343, + "event.category": "network", "event.dataset": "aws.elb", "event.end": "2019-10-17T13:26:14.308Z", + "event.kind": "event", "event.module": "aws", "fileset.name": "elb", "input.type": "log", @@ -158,9 +173,12 @@ "aws.elb.protocol": "tcp", "aws.elb.request_processing_time.sec": 0.001004, "aws.elb.response_processing_time.sec": 1.5e-05, + "cloud.provider": "aws", "destination.bytes": 343, + "event.category": "network", "event.dataset": "aws.elb", "event.end": "2019-10-17T13:26:19.318Z", + "event.kind": "event", "event.module": "aws", "fileset.name": "elb", "input.type": "log", diff --git a/x-pack/filebeat/module/aws/elb/test/example-alb-http.log-expected.json b/x-pack/filebeat/module/aws/elb/test/example-alb-http.log-expected.json index 65824aab220..1a46cee8d85 100644 --- a/x-pack/filebeat/module/aws/elb/test/example-alb-http.log-expected.json +++ b/x-pack/filebeat/module/aws/elb/test/example-alb-http.log-expected.json @@ -4,7 +4,7 @@ "aws.elb.action_executed": [ "forward" ], - "aws.elb.backend.http.response.status_code": "200", + "aws.elb.backend.http.response.status_code": 200, "aws.elb.backend.ip": "10.0.0.1", "aws.elb.backend.port": "80", "aws.elb.backend_processing_time.sec": 0.001, @@ -16,22 +16,27 @@ "aws.elb.target_group.arn": "arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067", "aws.elb.trace_id": "Root=1-58337262-36d228ad5d99923122bbe354", "aws.elb.type": "http", + "cloud.provider": "aws", + "event.category": "web", "event.dataset": "aws.elb", "event.end": "2018-07-02T22:23:00.186Z", + "event.kind": "event", "event.module": "aws", + "event.outcome": "success", "event.start": "2018-07-02T22:22:48.364000Z", "fileset.name": "elb", "http.request.body.bytes": 34, - "http.request.method": "GET", + "http.request.method": "get", "http.request.referrer": "http://www.example.com:80/", "http.response.body.bytes": 366, - "http.response.status_code": "200", + "http.response.status_code": 200, "http.version": "1.1", "input.type": "log", "log.offset": 0, "service.type": "aws", "source.ip": "192.168.131.39", "source.port": "2817", + "tracing.trace.id": "Root=1-58337262-36d228ad5d99923122bbe354", "user_agent.original": "curl/7.46.0" }, { @@ -40,7 +45,7 @@ "authenticate", "forward" ], - "aws.elb.backend.http.response.status_code": "200", + "aws.elb.backend.http.response.status_code": 200, "aws.elb.backend.ip": "10.0.0.1", "aws.elb.backend.port": "80", "aws.elb.backend_processing_time.sec": 0.048, @@ -55,17 +60,21 @@ "aws.elb.target_group.arn": "arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067", "aws.elb.trace_id": "Root=1-58337281-1d84f3d73c47ec4e58577259", "aws.elb.type": "https", + "cloud.provider": "aws", "destination.domain": "www.example.com", + "event.category": "web", "event.dataset": "aws.elb", "event.end": "2018-07-02T22:23:00.186Z", + "event.kind": "event", "event.module": "aws", + "event.outcome": "success", "event.start": "2018-07-02T22:22:48.364000Z", "fileset.name": "elb", "http.request.body.bytes": 0, - "http.request.method": "GET", + "http.request.method": "get", "http.request.referrer": "https://www.example.com:443/", "http.response.body.bytes": 57, - "http.response.status_code": "200", + "http.response.status_code": 200, "http.version": "1.1", "input.type": "log", "log.offset": 386, @@ -75,6 +84,7 @@ "tls.cipher": "ECDHE-RSA-AES128-GCM-SHA256", "tls.version": "1.2", "tls.version_protocol": "tls", + "tracing.trace.id": "Root=1-58337281-1d84f3d73c47ec4e58577259", "user_agent.original": "curl/7.46.0" }, { @@ -82,7 +92,7 @@ "aws.elb.action_executed": [ "redirect" ], - "aws.elb.backend.http.response.status_code": "200", + "aws.elb.backend.http.response.status_code": 200, "aws.elb.backend.ip": "10.0.0.66", "aws.elb.backend.port": "9000", "aws.elb.backend_processing_time.sec": 0.002, @@ -97,16 +107,20 @@ "aws.elb.target_group.arn": "arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067", "aws.elb.trace_id": "Root=1-58337327-72bd00b0343d75b906739c42", "aws.elb.type": "h2", + "cloud.provider": "aws", + "event.category": "web", "event.dataset": "aws.elb", "event.end": "2018-07-02T22:23:00.186Z", + "event.kind": "event", "event.module": "aws", + "event.outcome": "success", "event.start": "2018-07-02T22:22:48.364000Z", "fileset.name": "elb", "http.request.body.bytes": 5, - "http.request.method": "GET", + "http.request.method": "get", "http.request.referrer": "https://10.0.2.105:773/", "http.response.body.bytes": 257, - "http.response.status_code": "200", + "http.response.status_code": 200, "http.version": "2.0", "input.type": "log", "log.offset": 914, @@ -116,6 +130,7 @@ "tls.cipher": "ECDHE-RSA-AES128-GCM-SHA256", "tls.version": "1.2", "tls.version_protocol": "tls", + "tracing.trace.id": "Root=1-58337327-72bd00b0343d75b906739c42", "user_agent.original": "curl/7.46.0" }, { @@ -123,7 +138,7 @@ "aws.elb.action_executed": [ "forward" ], - "aws.elb.backend.http.response.status_code": "101", + "aws.elb.backend.http.response.status_code": 101, "aws.elb.backend.ip": "10.0.1.192", "aws.elb.backend.port": "8010", "aws.elb.backend_processing_time.sec": 0.003, @@ -135,27 +150,32 @@ "aws.elb.target_group.arn": "arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067", "aws.elb.trace_id": "Root=1-58337364-23a8c76965a2ef7629b185e3", "aws.elb.type": "ws", + "cloud.provider": "aws", + "event.category": "web", "event.dataset": "aws.elb", "event.end": "2018-07-02T22:23:00.186Z", + "event.kind": "event", "event.module": "aws", + "event.outcome": "success", "event.start": "2018-07-02T22:22:48.364000Z", "fileset.name": "elb", "http.request.body.bytes": 218, - "http.request.method": "GET", + "http.request.method": "get", "http.request.referrer": "http://10.0.0.30:80/", "http.response.body.bytes": 587, - "http.response.status_code": "101", + "http.response.status_code": 101, "http.version": "1.1", "input.type": "log", "log.offset": 1349, "service.type": "aws", "source.ip": "10.0.0.140", "source.port": "40914", + "tracing.trace.id": "Root=1-58337364-23a8c76965a2ef7629b185e3", "user_agent.original": "-" }, { "@timestamp": "2018-07-02T22:23:00.186Z", - "aws.elb.backend.http.response.status_code": "101", + "aws.elb.backend.http.response.status_code": 101, "aws.elb.backend.ip": "10.0.0.171", "aws.elb.backend.port": "8010", "aws.elb.backend_processing_time.sec": 0.001, @@ -165,15 +185,19 @@ "aws.elb.response_processing_time.sec": 0.0, "aws.elb.ssl_cipher": "ECDHE-RSA-AES128-GCM-SHA256", "aws.elb.ssl_protocol": "TLSv1.2", + "cloud.provider": "aws", + "event.category": "web", "event.dataset": "aws.elb", "event.end": "2018-07-02T22:23:00.186Z", + "event.kind": "event", "event.module": "aws", + "event.outcome": "success", "fileset.name": "elb", "http.request.body.bytes": 218, - "http.request.method": "GET", + "http.request.method": "get", "http.request.referrer": "https://10.0.0.30:443/", "http.response.body.bytes": 786, - "http.response.status_code": "101", + "http.response.status_code": 101, "http.version": "1.1", "input.type": "log", "log.offset": 1719, @@ -190,7 +214,7 @@ "aws.elb.action_executed": [ "forward" ], - "aws.elb.backend.http.response.status_code": "200", + "aws.elb.backend.http.response.status_code": 200, "aws.elb.backend_processing_time.sec": 0.001, "aws.elb.matched_rule_priority": "0", "aws.elb.name": "app/my-loadbalancer/50dc6c495c0c9188", @@ -200,22 +224,27 @@ "aws.elb.target_group.arn": "arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067", "aws.elb.trace_id": "Root=1-58337364-23a8c76965a2ef7629b185e3", "aws.elb.type": "http", + "cloud.provider": "aws", + "event.category": "web", "event.dataset": "aws.elb", "event.end": "2018-11-30T22:23:00.186Z", + "event.kind": "event", "event.module": "aws", + "event.outcome": "success", "event.start": "2018-11-30T22:22:48.364000Z", "fileset.name": "elb", "http.request.body.bytes": 34, - "http.request.method": "GET", + "http.request.method": "get", "http.request.referrer": "http://www.example.com:80/", "http.response.body.bytes": 366, - "http.response.status_code": "200", + "http.response.status_code": 200, "http.version": "1.1", "input.type": "log", "log.offset": 2123, "service.type": "aws", "source.ip": "192.168.131.39", "source.port": "2817", + "tracing.trace.id": "Root=1-58337364-23a8c76965a2ef7629b185e3", "user_agent.original": "curl/7.46.0" }, { @@ -233,22 +262,27 @@ "aws.elb.target_group.arn": "arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067", "aws.elb.trace_id": "Root=1-58337364-23a8c76965a2ef7629b185e3", "aws.elb.type": "http", + "cloud.provider": "aws", + "event.category": "web", "event.dataset": "aws.elb", "event.end": "2018-11-30T22:23:00.186Z", + "event.kind": "event", "event.module": "aws", + "event.outcome": "failure", "event.start": "2018-11-30T22:22:48.364000Z", "fileset.name": "elb", "http.request.body.bytes": 34, - "http.request.method": "GET", + "http.request.method": "get", "http.request.referrer": "http://www.example.com:80/", "http.response.body.bytes": 366, - "http.response.status_code": "502", + "http.response.status_code": 502, "http.version": "1.1", "input.type": "log", "log.offset": 2499, "service.type": "aws", "source.ip": "192.168.131.39", "source.port": "2817", + "tracing.trace.id": "Root=1-58337364-23a8c76965a2ef7629b185e3", "user_agent.original": "curl/7.46.0" }, { @@ -259,20 +293,25 @@ "aws.elb.target_group.arn": "-", "aws.elb.trace_id": "-", "aws.elb.type": "http", + "cloud.provider": "aws", + "event.category": "web", "event.dataset": "aws.elb", "event.end": "2018-11-30T22:23:00.186Z", + "event.kind": "event", "event.module": "aws", + "event.outcome": "failure", "event.start": "2018-11-30T22:22:48.364000Z", "fileset.name": "elb", "http.request.body.bytes": 0, "http.request.referrer": "http://www.example.com:80-", "http.response.body.bytes": 0, - "http.response.status_code": "400", + "http.response.status_code": 400, "input.type": "log", "log.offset": 2893, "service.type": "aws", "source.ip": "192.168.131.39", "source.port": "2817", + "tracing.trace.id": "-", "user_agent.original": "-" }, { @@ -283,19 +322,24 @@ "aws.elb.target_group.arn": "-", "aws.elb.trace_id": "-", "aws.elb.type": "http", + "cloud.provider": "aws", + "event.category": "web", "event.dataset": "aws.elb", "event.end": "2018-11-30T22:23:00.186Z", + "event.kind": "event", "event.module": "aws", + "event.outcome": "failure", "event.start": "2018-11-30T22:22:48.364000Z", "fileset.name": "elb", "http.request.body.bytes": 0, "http.response.body.bytes": 0, - "http.response.status_code": "400", + "http.response.status_code": 400, "input.type": "log", "log.offset": 3101, "service.type": "aws", "source.ip": "192.168.131.39", "source.port": "2817", + "tracing.trace.id": "-", "user_agent.original": "-" } ] \ No newline at end of file diff --git a/x-pack/filebeat/module/aws/elb/test/example-http.log-expected.json b/x-pack/filebeat/module/aws/elb/test/example-http.log-expected.json index 137891aae8c..72f9a57f6e3 100644 --- a/x-pack/filebeat/module/aws/elb/test/example-http.log-expected.json +++ b/x-pack/filebeat/module/aws/elb/test/example-http.log-expected.json @@ -1,7 +1,7 @@ [ { "@timestamp": "2015-05-13T23:39:43.945Z", - "aws.elb.backend.http.response.status_code": "200", + "aws.elb.backend.http.response.status_code": 200, "aws.elb.backend.ip": "10.0.0.1", "aws.elb.backend.port": "80", "aws.elb.backend_processing_time.sec": 0.001048, @@ -9,15 +9,19 @@ "aws.elb.protocol": "http", "aws.elb.request_processing_time.sec": 7.3e-05, "aws.elb.response_processing_time.sec": 5.7e-05, + "cloud.provider": "aws", + "event.category": "web", "event.dataset": "aws.elb", "event.end": "2015-05-13T23:39:43.945Z", + "event.kind": "event", "event.module": "aws", + "event.outcome": "success", "fileset.name": "elb", "http.request.body.bytes": 0, - "http.request.method": "GET", + "http.request.method": "get", "http.request.referrer": "http://www.example.com:80/", "http.response.body.bytes": 29, - "http.response.status_code": "200", + "http.response.status_code": 200, "http.version": "1.1", "input.type": "log", "log.offset": 0, @@ -30,15 +34,19 @@ "@timestamp": "2015-05-13T23:39:43.945Z", "aws.elb.name": "my-loadbalancer", "aws.elb.protocol": "http", + "cloud.provider": "aws", + "event.category": "web", "event.dataset": "aws.elb", "event.end": "2015-05-13T23:39:43.945Z", + "event.kind": "event", "event.module": "aws", + "event.outcome": "failure", "fileset.name": "elb", "http.request.body.bytes": 0, - "http.request.method": "GET", + "http.request.method": "get", "http.request.referrer": "http://www.example.com:80/", "http.response.body.bytes": 0, - "http.response.status_code": "503", + "http.response.status_code": 503, "http.version": "1.1", "input.type": "log", "log.offset": 176, @@ -51,15 +59,19 @@ "@timestamp": "2015-05-13T23:39:43.945Z", "aws.elb.name": "my-loadbalancer", "aws.elb.protocol": "http", + "cloud.provider": "aws", + "event.category": "web", "event.dataset": "aws.elb", "event.end": "2015-05-13T23:39:43.945Z", + "event.kind": "event", "event.module": "aws", + "event.outcome": "failure", "fileset.name": "elb", "http.request.body.bytes": 0, - "http.request.method": "GET", + "http.request.method": "get", "http.request.referrer": "http://www.example.com:80-", "http.response.body.bytes": 0, - "http.response.status_code": "400", + "http.response.status_code": 400, "input.type": "log", "log.offset": 321, "service.type": "aws", diff --git a/x-pack/filebeat/module/aws/elb/test/example-https.log-expected.json b/x-pack/filebeat/module/aws/elb/test/example-https.log-expected.json index acd0407d4c3..ef09a37d579 100644 --- a/x-pack/filebeat/module/aws/elb/test/example-https.log-expected.json +++ b/x-pack/filebeat/module/aws/elb/test/example-https.log-expected.json @@ -1,7 +1,7 @@ [ { "@timestamp": "2015-05-13T23:39:43.945Z", - "aws.elb.backend.http.response.status_code": "200", + "aws.elb.backend.http.response.status_code": 200, "aws.elb.backend.ip": "10.0.0.1", "aws.elb.backend.port": "80", "aws.elb.backend_processing_time.sec": 0.001048, @@ -11,15 +11,19 @@ "aws.elb.response_processing_time.sec": 0.001337, "aws.elb.ssl_cipher": "DHE-RSA-AES128-SHA", "aws.elb.ssl_protocol": "TLSv1.2", + "cloud.provider": "aws", + "event.category": "web", "event.dataset": "aws.elb", "event.end": "2015-05-13T23:39:43.945Z", + "event.kind": "event", "event.module": "aws", + "event.outcome": "success", "fileset.name": "elb", "http.request.body.bytes": 0, - "http.request.method": "GET", + "http.request.method": "get", "http.request.referrer": "https://www.example.com:443/", "http.response.body.bytes": 57, - "http.response.status_code": "200", + "http.response.status_code": 200, "http.version": "1.1", "input.type": "log", "log.offset": 0, diff --git a/x-pack/filebeat/module/aws/elb/test/example-nlb-tcp.log-expected.json b/x-pack/filebeat/module/aws/elb/test/example-nlb-tcp.log-expected.json index 950dc276a5d..74c1c0e8cc7 100644 --- a/x-pack/filebeat/module/aws/elb/test/example-nlb-tcp.log-expected.json +++ b/x-pack/filebeat/module/aws/elb/test/example-nlb-tcp.log-expected.json @@ -12,10 +12,13 @@ "aws.elb.ssl_protocol": "tlsv12", "aws.elb.tls_handshake_time.ms": 2.0, "aws.elb.type": "tls", + "cloud.provider": "aws", "destination.bytes": 246, "destination.domain": "my-network-loadbalancer-c6e77e28c25b2234.elb.us-east-2.amazonaws.com", + "event.category": "network", "event.dataset": "aws.elb", "event.end": "2018-12-20T02:59:40.000Z", + "event.kind": "event", "event.module": "aws", "fileset.name": "elb", "input.type": "log", diff --git a/x-pack/filebeat/module/aws/elb/test/example-ssl.log-expected.json b/x-pack/filebeat/module/aws/elb/test/example-ssl.log-expected.json index c19bbbccdac..84f2748861c 100644 --- a/x-pack/filebeat/module/aws/elb/test/example-ssl.log-expected.json +++ b/x-pack/filebeat/module/aws/elb/test/example-ssl.log-expected.json @@ -10,9 +10,12 @@ "aws.elb.response_processing_time.sec": 2.3e-05, "aws.elb.ssl_cipher": "ECDHE-ECDSA-AES128-GCM-SHA256", "aws.elb.ssl_protocol": "TLSv1.2", + "cloud.provider": "aws", "destination.bytes": 502, + "event.category": "network", "event.dataset": "aws.elb", "event.end": "2015-05-13T23:39:43.945Z", + "event.kind": "event", "event.module": "aws", "fileset.name": "elb", "input.type": "log", diff --git a/x-pack/filebeat/module/aws/elb/test/example-tcp.log-expected.json b/x-pack/filebeat/module/aws/elb/test/example-tcp.log-expected.json index 3a15b257191..af89134a830 100644 --- a/x-pack/filebeat/module/aws/elb/test/example-tcp.log-expected.json +++ b/x-pack/filebeat/module/aws/elb/test/example-tcp.log-expected.json @@ -8,9 +8,12 @@ "aws.elb.protocol": "tcp", "aws.elb.request_processing_time.sec": 0.001069, "aws.elb.response_processing_time.sec": 4.1e-05, + "cloud.provider": "aws", "destination.bytes": 305, + "event.category": "network", "event.dataset": "aws.elb", "event.end": "2015-05-13T23:39:43.945Z", + "event.kind": "event", "event.module": "aws", "fileset.name": "elb", "input.type": "log", @@ -24,9 +27,12 @@ "@timestamp": "2015-05-13T23:39:43.945Z", "aws.elb.name": "my-loadbalancer", "aws.elb.protocol": "tcp", + "cloud.provider": "aws", "destination.bytes": 0, + "event.category": "network", "event.dataset": "aws.elb", "event.end": "2015-05-13T23:39:43.945Z", + "event.kind": "event", "event.module": "aws", "fileset.name": "elb", "input.type": "log", diff --git a/x-pack/filebeat/module/aws/s3access/ingest/pipeline.yml b/x-pack/filebeat/module/aws/s3access/ingest/pipeline.yml index 8c54475ade9..5cae87aa0f5 100644 --- a/x-pack/filebeat/module/aws/s3access/ingest/pipeline.yml +++ b/x-pack/filebeat/module/aws/s3access/ingest/pipeline.yml @@ -10,7 +10,7 @@ processors: %{S3OPERATION:aws.s3access.operation} (?:-|%{S3KEY:aws.s3access.key}) (?:-|\"%{DATA:aws.s3access.request_uri}\") %{NUMBER:aws.s3access.http_status:long} (?:-|%{WORD:aws.s3access.error_code}) (?:-|%{NUMBER:aws.s3access.bytes_sent:long}) (?:-|%{NUMBER:aws.s3access.object_size:long}) (?:-|%{NUMBER:aws.s3access.total_time:long}) (?:-|%{NUMBER:aws.s3access.turn_around_time:long}) - (?:-|\"%{DATA:aws.s3access.referrer}\") (?:-|\"(-|%{DATA:aws.s3access.user_agent})\") (?:-|%{S3KEY:aws.s3access.version_id}) + (?:-|\"-\"|\"%{DATA:aws.s3access.referrer}\") (?:-|\"(-|%{DATA:aws.s3access.user_agent})\") (?:-|%{S3KEY:aws.s3access.version_id}) (?:-|%{S3ID:aws.s3access.host_id}) (?:-|%{S3VERSION:aws.s3access.signature_version}) (?:-|%{S3KEY:aws.s3access.cipher_suite}) (?:-|%{WORD:aws.s3access.authentication_type}) (?:-|%{S3ID:aws.s3access.host_header}) (?:-|%{S3VERSION:aws.s3access.tls_version}) pattern_definitions: @@ -21,6 +21,11 @@ processors: S3ID: "[a-zA-Z0-9\\/_\\.\\-%+=]+" S3VERSION: "[a-zA-Z0-9.]+" + - append: + if: "ctx?.aws?.s3access?.bucket_owner != null" + field: related.user + value: "{{aws.s3access.bucket_owner}}" + # # Parse the date included in s3 access logs # @@ -30,6 +35,81 @@ processors: ignore_failure: true formats: - "dd/MMM/yyyy:H:m:s Z" + + - set: + if: "ctx?.aws?.s3access?.remote_ip != null" + field: client.ip + value: "{{aws.s3access.remote_ip}}" + + - append: + if: "ctx?.aws?.s3access?.remote_ip != null" + field: related.ip + value: "{{aws.s3access.remote_ip}}" + + - set: + if: "ctx?.aws?.s3access?.remote_ip != null" + field: client.address + value: "{{aws.s3access.remote_ip}}" + + - geoip: + if: "ctx?.aws?.s3access?.remote_ip != null" + field: aws.s3access.remote_ip + target_field: geo + + - set: + if: "ctx?.aws?.s3access?.requester != null" + field: client.user.id + value: "{{aws.s3access.requester}}" + + - set: + if: "ctx?.aws?.s3access?.request_id != null" + field: event.id + value: "{{aws.s3access.request_id}}" + + - set: + if: "ctx?.aws?.s3access?.operation != null" + field: event.action + value: "{{aws.s3access.operation}}" + + - set: + if: "ctx?.aws?.s3access?.http_status != null" + field: http.response.status_code + value: "{{aws.s3access.http_status}}" + + - convert: + if: "ctx?.http?.response?.status_code != null" + field: http.response.status_code + type: long + + - set: + if: "ctx?.aws?.s3access?.error_code != null" + field: event.outcome + value: failure + + - set: + if: "ctx?.aws?.s3access?.error_code != null" + field: event.code + value: "{{aws.s3access.error_code}}" + + - set: + if: "ctx?.aws?.s3access?.error_code == null" + field: event.outcome + value: success + + - set: + if: "ctx?.aws?.s3access?.total_time != null" + field: event.duration + value: "{{aws.s3access.total_time}}" + + - set: + if: "ctx?.aws?.s3access?.referrer != null" + field: http.request.referrer + value: "{{aws.s3access.referrer}}" + + - user_agent: + if: "ctx?.aws?.s3access?.user_agent != null" + field: aws.s3access.user_agent + - set: field: tls.cipher value: '{{aws.s3access.cipher_suite}}' @@ -46,6 +126,14 @@ processors: ctx.tls.version = parts[1]; ctx.tls.version_protocol = parts[0] + - set: + field: cloud.provider + value: aws + + - set: + field: event.kind + value: event + # # Remove temporary fields # diff --git a/x-pack/filebeat/module/aws/s3access/test/s3_server_access.log-expected.json b/x-pack/filebeat/module/aws/s3access/test/s3_server_access.log-expected.json index 53a2055b6ca..b312118a644 100644 --- a/x-pack/filebeat/module/aws/s3access/test/s3_server_access.log-expected.json +++ b/x-pack/filebeat/module/aws/s3access/test/s3_server_access.log-expected.json @@ -10,7 +10,6 @@ "aws.s3access.host_id": "BsCfJedfuSnds2QFoxi+E/O7M6OEWzJnw4dUaes/2hyA363sONRJKzB7EOY+Bt9DTHYUn+HoHxI=", "aws.s3access.http_status": 200, "aws.s3access.operation": "REST.GET.LOCATION", - "aws.s3access.referrer": "-", "aws.s3access.remote_ip": "72.21.217.31", "aws.s3access.request_id": "44EE8651683CB4DA", "aws.s3access.request_uri": "GET /test-s3-ks/?location&aws-account=627959692251 HTTP/1.1", @@ -19,15 +18,45 @@ "aws.s3access.tls_version": "TLSv1.2", "aws.s3access.total_time": 17, "aws.s3access.user_agent": "AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation", + "client.address": "72.21.217.31", + "client.ip": "72.21.217.31", + "client.user.id": "arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9", + "cloud.provider": "aws", + "event.action": "REST.GET.LOCATION", "event.dataset": "aws.s3access", + "event.duration": "17", + "event.id": "44EE8651683CB4DA", + "event.kind": "event", "event.module": "aws", + "event.outcome": "success", "fileset.name": "s3access", + "geo.city_name": "Ashburn", + "geo.continent_name": "North America", + "geo.country_iso_code": "US", + "geo.location.lat": 39.0481, + "geo.location.lon": -77.4728, + "geo.region_iso_code": "US-VA", + "geo.region_name": "Virginia", + "http.response.status_code": 200, "input.type": "log", "log.offset": 0, + "related.ip": [ + "72.21.217.31" + ], + "related.user": [ + "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2" + ], "service.type": "aws", "tls.cipher": "ECDHE-RSA-AES128-SHA", "tls.version": "1.2", - "tls.version_protocol": "tls" + "tls.version_protocol": "tls", + "user_agent.device.name": "Other", + "user_agent.name": "aws-sdk-java", + "user_agent.original": "AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation", + "user_agent.os.full": "Linux 4.9.137", + "user_agent.os.name": "Linux", + "user_agent.os.version": "4.9.137", + "user_agent.version": "1.11.590" }, { "@timestamp": "2019-08-01T00:24:42.000Z", @@ -40,7 +69,6 @@ "aws.s3access.host_id": "gNl/Q1IzY6nGTBygqI3rnMz/ZFOFwOTDpSMrNca+IcEmMAd6sCIs1ZRLYDekD8LB9lrj9UdQLWE=", "aws.s3access.http_status": 200, "aws.s3access.operation": "REST.GET.LOCATION", - "aws.s3access.referrer": "-", "aws.s3access.remote_ip": "72.21.217.31", "aws.s3access.request_id": "E26222010BCC32B6", "aws.s3access.request_uri": "GET /test-s3-ks/?location&aws-account=627959692251 HTTP/1.1", @@ -49,15 +77,45 @@ "aws.s3access.tls_version": "TLSv1.2", "aws.s3access.total_time": 3, "aws.s3access.user_agent": "AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation", + "client.address": "72.21.217.31", + "client.ip": "72.21.217.31", + "client.user.id": "arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9", + "cloud.provider": "aws", + "event.action": "REST.GET.LOCATION", "event.dataset": "aws.s3access", + "event.duration": "3", + "event.id": "E26222010BCC32B6", + "event.kind": "event", "event.module": "aws", + "event.outcome": "success", "fileset.name": "s3access", + "geo.city_name": "Ashburn", + "geo.continent_name": "North America", + "geo.country_iso_code": "US", + "geo.location.lat": 39.0481, + "geo.location.lon": -77.4728, + "geo.region_iso_code": "US-VA", + "geo.region_name": "Virginia", + "http.response.status_code": 200, "input.type": "log", "log.offset": 715, + "related.ip": [ + "72.21.217.31" + ], + "related.user": [ + "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2" + ], "service.type": "aws", "tls.cipher": "ECDHE-RSA-AES128-SHA", "tls.version": "1.2", - "tls.version_protocol": "tls" + "tls.version_protocol": "tls", + "user_agent.device.name": "Other", + "user_agent.name": "aws-sdk-java", + "user_agent.original": "AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation", + "user_agent.os.full": "Linux 4.9.137", + "user_agent.os.name": "Linux", + "user_agent.os.version": "4.9.137", + "user_agent.version": "1.11.590" }, { "@timestamp": "2019-08-01T00:24:43.000Z", @@ -70,7 +128,6 @@ "aws.s3access.host_id": "KzvchfojYQnuFC4PABYVJVxIlv/f6r17LRaTSvw7x+bxj4PkkPKT1kX9x8wbqtq40iD4PC881iE=", "aws.s3access.http_status": 200, "aws.s3access.operation": "REST.GET.BUCKET", - "aws.s3access.referrer": "-", "aws.s3access.remote_ip": "72.21.217.31", "aws.s3access.request_id": "4DD6D17D1C5C401C", "aws.s3access.request_uri": "GET /test-s3-ks/?max-keys=0&encoding-type=url&aws-account=627959692251 HTTP/1.1", @@ -80,15 +137,45 @@ "aws.s3access.total_time": 2, "aws.s3access.turn_around_time": 1, "aws.s3access.user_agent": "AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation", + "client.address": "72.21.217.31", + "client.ip": "72.21.217.31", + "client.user.id": "arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9", + "cloud.provider": "aws", + "event.action": "REST.GET.BUCKET", "event.dataset": "aws.s3access", + "event.duration": "2", + "event.id": "4DD6D17D1C5C401C", + "event.kind": "event", "event.module": "aws", + "event.outcome": "success", "fileset.name": "s3access", + "geo.city_name": "Ashburn", + "geo.continent_name": "North America", + "geo.country_iso_code": "US", + "geo.location.lat": 39.0481, + "geo.location.lon": -77.4728, + "geo.region_iso_code": "US-VA", + "geo.region_name": "Virginia", + "http.response.status_code": 200, "input.type": "log", "log.offset": 1429, + "related.ip": [ + "72.21.217.31" + ], + "related.user": [ + "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2" + ], "service.type": "aws", "tls.cipher": "ECDHE-RSA-AES128-SHA", "tls.version": "1.2", - "tls.version_protocol": "tls" + "tls.version_protocol": "tls", + "user_agent.device.name": "Other", + "user_agent.name": "aws-sdk-java", + "user_agent.original": "AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation", + "user_agent.os.full": "Linux 4.9.137", + "user_agent.os.name": "Linux", + "user_agent.os.version": "4.9.137", + "user_agent.version": "1.11.590" }, { "@timestamp": "2019-08-01T00:24:43.000Z", @@ -101,7 +188,6 @@ "aws.s3access.host_id": "cIN12KTrJwx+uTBZD+opZUPE4iGypi8oG/oXGPzFk9CMuHQGuEpmAeNELdtYKDxf2TDor25Nikg=", "aws.s3access.http_status": 200, "aws.s3access.operation": "REST.GET.LOCATION", - "aws.s3access.referrer": "-", "aws.s3access.remote_ip": "72.21.217.31", "aws.s3access.request_id": "706992E2F3CC3C3D", "aws.s3access.request_uri": "GET /test-s3-ks/?location&aws-account=627959692251 HTTP/1.1", @@ -110,15 +196,45 @@ "aws.s3access.tls_version": "TLSv1.2", "aws.s3access.total_time": 4, "aws.s3access.user_agent": "AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation", + "client.address": "72.21.217.31", + "client.ip": "72.21.217.31", + "client.user.id": "arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9", + "cloud.provider": "aws", + "event.action": "REST.GET.LOCATION", "event.dataset": "aws.s3access", + "event.duration": "4", + "event.id": "706992E2F3CC3C3D", + "event.kind": "event", "event.module": "aws", + "event.outcome": "success", "fileset.name": "s3access", + "geo.city_name": "Ashburn", + "geo.continent_name": "North America", + "geo.country_iso_code": "US", + "geo.location.lat": 39.0481, + "geo.location.lon": -77.4728, + "geo.region_iso_code": "US-VA", + "geo.region_name": "Virginia", + "http.response.status_code": 200, "input.type": "log", "log.offset": 2161, + "related.ip": [ + "72.21.217.31" + ], + "related.user": [ + "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2" + ], "service.type": "aws", "tls.cipher": "ECDHE-RSA-AES128-SHA", "tls.version": "1.2", - "tls.version_protocol": "tls" + "tls.version_protocol": "tls", + "user_agent.device.name": "Other", + "user_agent.name": "aws-sdk-java", + "user_agent.original": "AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation", + "user_agent.os.full": "Linux 4.9.137", + "user_agent.os.name": "Linux", + "user_agent.os.version": "4.9.137", + "user_agent.version": "1.11.590" }, { "@timestamp": "2019-09-10T15:11:07.000Z", @@ -137,11 +253,33 @@ "aws.s3access.requester": "arn:aws:iam::123456:user/test@elastic.co", "aws.s3access.signature_version": "SigV4", "aws.s3access.tls_version": "TLSv1.2", + "client.address": "77.227.156.41", + "client.ip": "77.227.156.41", + "client.user.id": "arn:aws:iam::123456:user/test@elastic.co", + "cloud.provider": "aws", + "event.action": "BATCH.DELETE.OBJECT", "event.dataset": "aws.s3access", + "event.id": "8CD7A4A71E2E5C9E", + "event.kind": "event", "event.module": "aws", + "event.outcome": "success", "fileset.name": "s3access", + "geo.city_name": "Teruel", + "geo.continent_name": "Europe", + "geo.country_iso_code": "ES", + "geo.location.lat": 40.3456, + "geo.location.lon": -1.1065, + "geo.region_iso_code": "ES-TE", + "geo.region_name": "Teruel", + "http.response.status_code": 204, "input.type": "log", "log.offset": 2875, + "related.ip": [ + "77.227.156.41" + ], + "related.user": [ + "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2" + ], "service.type": "aws", "tls.cipher": "ECDHE-RSA-AES128-SHA", "tls.version": "1.2", @@ -164,11 +302,33 @@ "aws.s3access.requester": "arn:aws:iam::123456:user/test@elastic.co", "aws.s3access.signature_version": "SigV4", "aws.s3access.tls_version": "TLSv1.2", + "client.address": "174.29.206.152", + "client.ip": "174.29.206.152", + "client.user.id": "arn:aws:iam::123456:user/test@elastic.co", + "cloud.provider": "aws", + "event.action": "BATCH.DELETE.OBJECT", "event.dataset": "aws.s3access", + "event.id": "6CE38F1312D32BDD", + "event.kind": "event", "event.module": "aws", + "event.outcome": "success", "fileset.name": "s3access", + "geo.city_name": "Denver", + "geo.continent_name": "North America", + "geo.country_iso_code": "US", + "geo.location.lat": 39.7044, + "geo.location.lon": -105.0023, + "geo.region_iso_code": "US-CO", + "geo.region_name": "Colorado", + "http.response.status_code": 204, "input.type": "log", "log.offset": 3280, + "related.ip": [ + "174.29.206.152" + ], + "related.user": [ + "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2" + ], "service.type": "aws", "tls.cipher": "ECDHE-RSA-AES128-SHA", "tls.version": "1.2", diff --git a/x-pack/filebeat/module/aws/s3access/test/test.log-expected.json b/x-pack/filebeat/module/aws/s3access/test/test.log-expected.json index 9dfe82bcd5a..61baec94c6c 100644 --- a/x-pack/filebeat/module/aws/s3access/test/test.log-expected.json +++ b/x-pack/filebeat/module/aws/s3access/test/test.log-expected.json @@ -10,7 +10,6 @@ "aws.s3access.host_id": "s9lzHYrFp76ZVxRcpX9+5cjAnEH2ROuNkd2BHfIa6UkFVdtjf5mKR3/eTPFvsiP/XV/VLi31234=", "aws.s3access.http_status": 200, "aws.s3access.operation": "REST.GET.VERSIONING", - "aws.s3access.referrer": "-", "aws.s3access.remote_ip": "192.0.2.3", "aws.s3access.request_id": "3E57427F3EXAMPLE", "aws.s3access.request_uri": "GET /awsexamplebucket?versioning HTTP/1.1", @@ -19,15 +18,34 @@ "aws.s3access.tls_version": "TLSV1.1", "aws.s3access.total_time": 7, "aws.s3access.user_agent": "S3Console/0.4", + "client.address": "192.0.2.3", + "client.ip": "192.0.2.3", + "client.user.id": "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be", + "cloud.provider": "aws", + "event.action": "REST.GET.VERSIONING", "event.dataset": "aws.s3access", + "event.duration": "7", + "event.id": "3E57427F3EXAMPLE", + "event.kind": "event", "event.module": "aws", + "event.outcome": "success", "fileset.name": "s3access", + "http.response.status_code": 200, "input.type": "log", "log.offset": 0, + "related.ip": [ + "192.0.2.3" + ], + "related.user": [ + "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be" + ], "service.type": "aws", "tls.cipher": "ECDHE-RSA-AES128-GCM-SHA256", "tls.version": "1.1", - "tls.version_protocol": "tls" + "tls.version_protocol": "tls", + "user_agent.device.name": "Other", + "user_agent.name": "Other", + "user_agent.original": "S3Console/0.4" }, { "@timestamp": "2019-02-06T00:00:38.000Z", @@ -40,7 +58,6 @@ "aws.s3access.host_id": "9vKBE6vMhrNiWHZmb2L0mXOcqPGzQOI5XLnCtZNPxev+Hf+7tpT6sxDwDty4LHBUOZJG96N1234=", "aws.s3access.http_status": 200, "aws.s3access.operation": "REST.GET.LOGGING_STATUS", - "aws.s3access.referrer": "-", "aws.s3access.remote_ip": "192.0.2.3", "aws.s3access.request_id": "891CE47D2EXAMPLE", "aws.s3access.request_uri": "GET /awsexamplebucket?logging HTTP/1.1", @@ -49,15 +66,34 @@ "aws.s3access.tls_version": "TLSV1.1", "aws.s3access.total_time": 11, "aws.s3access.user_agent": "S3Console/0.4", + "client.address": "192.0.2.3", + "client.ip": "192.0.2.3", + "client.user.id": "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be", + "cloud.provider": "aws", + "event.action": "REST.GET.LOGGING_STATUS", "event.dataset": "aws.s3access", + "event.duration": "11", + "event.id": "891CE47D2EXAMPLE", + "event.kind": "event", "event.module": "aws", + "event.outcome": "success", "fileset.name": "s3access", + "http.response.status_code": 200, "input.type": "log", "log.offset": 471, + "related.ip": [ + "192.0.2.3" + ], + "related.user": [ + "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be" + ], "service.type": "aws", "tls.cipher": "ECDHE-RSA-AES128-GCM-SHA256", "tls.version": "1.1", - "tls.version_protocol": "tls" + "tls.version_protocol": "tls", + "user_agent.device.name": "Other", + "user_agent.name": "Other", + "user_agent.original": "S3Console/0.4" }, { "@timestamp": "2019-02-06T00:00:38.000Z", @@ -71,7 +107,6 @@ "aws.s3access.host_id": "BNaBsXZQQDbssi6xMBdBU2sLt+Yf5kZDmeBUP35sFoKa3sLLeMC78iwEIWxs99CRUrbS4n11234=", "aws.s3access.http_status": 404, "aws.s3access.operation": "REST.GET.BUCKETPOLICY", - "aws.s3access.referrer": "-", "aws.s3access.remote_ip": "192.0.2.3", "aws.s3access.request_id": "A1206F460EXAMPLE", "aws.s3access.request_uri": "GET /awsexamplebucket?policy HTTP/1.1", @@ -80,15 +115,35 @@ "aws.s3access.tls_version": "TLSV1.1", "aws.s3access.total_time": 38, "aws.s3access.user_agent": "S3Console/0.4", + "client.address": "192.0.2.3", + "client.ip": "192.0.2.3", + "client.user.id": "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be", + "cloud.provider": "aws", + "event.action": "REST.GET.BUCKETPOLICY", + "event.code": "NoSuchBucketPolicy", "event.dataset": "aws.s3access", + "event.duration": "38", + "event.id": "A1206F460EXAMPLE", + "event.kind": "event", "event.module": "aws", + "event.outcome": "failure", "fileset.name": "s3access", + "http.response.status_code": 404, "input.type": "log", "log.offset": 944, + "related.ip": [ + "192.0.2.3" + ], + "related.user": [ + "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be" + ], "service.type": "aws", "tls.cipher": "ECDHE-RSA-AES128-GCM-SHA256", "tls.version": "1.1", - "tls.version_protocol": "tls" + "tls.version_protocol": "tls", + "user_agent.device.name": "Other", + "user_agent.name": "Other", + "user_agent.original": "S3Console/0.4" }, { "@timestamp": "2019-02-06T00:01:00.000Z", @@ -101,7 +156,6 @@ "aws.s3access.host_id": "Ke1bUcazaN1jWuUlPJaxF64cQVpUEhoZKEG/hmy/gijN/I1DeWqDfFvnpybfEseEME/u7ME1234=", "aws.s3access.http_status": 200, "aws.s3access.operation": "REST.GET.VERSIONING", - "aws.s3access.referrer": "-", "aws.s3access.remote_ip": "192.0.2.3", "aws.s3access.request_id": "7B4A0FABBEXAMPLE", "aws.s3access.request_uri": "GET /awsexamplebucket?versioning HTTP/1.1", @@ -110,15 +164,34 @@ "aws.s3access.tls_version": "TLSV1.1", "aws.s3access.total_time": 33, "aws.s3access.user_agent": "S3Console/0.4", + "client.address": "192.0.2.3", + "client.ip": "192.0.2.3", + "client.user.id": "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be", + "cloud.provider": "aws", + "event.action": "REST.GET.VERSIONING", "event.dataset": "aws.s3access", + "event.duration": "33", + "event.id": "7B4A0FABBEXAMPLE", + "event.kind": "event", "event.module": "aws", + "event.outcome": "success", "fileset.name": "s3access", + "http.response.status_code": 200, "input.type": "log", "log.offset": 1431, + "related.ip": [ + "192.0.2.3" + ], + "related.user": [ + "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be" + ], "service.type": "aws", "tls.cipher": "ECDHE-RSA-AES128-GCM-SHA256", "tls.version": "1.1", - "tls.version_protocol": "tls" + "tls.version_protocol": "tls", + "user_agent.device.name": "Other", + "user_agent.name": "Other", + "user_agent.original": "S3Console/0.4" }, { "@timestamp": "2019-02-06T00:01:57.000Z", @@ -132,7 +205,6 @@ "aws.s3access.key": "s3-dg.pdf", "aws.s3access.object_size": 4406583, "aws.s3access.operation": "REST.PUT.OBJECT", - "aws.s3access.referrer": "-", "aws.s3access.remote_ip": "192.0.2.3", "aws.s3access.request_id": "DD6CC733AEXAMPLE", "aws.s3access.request_uri": "PUT /awsexamplebucket/s3-dg.pdf HTTP/1.1", @@ -142,14 +214,33 @@ "aws.s3access.total_time": 41754, "aws.s3access.turn_around_time": 28, "aws.s3access.user_agent": "S3Console/0.4", + "client.address": "192.0.2.3", + "client.ip": "192.0.2.3", + "client.user.id": "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be", + "cloud.provider": "aws", + "event.action": "REST.PUT.OBJECT", "event.dataset": "aws.s3access", + "event.duration": "41754", + "event.id": "DD6CC733AEXAMPLE", + "event.kind": "event", "event.module": "aws", + "event.outcome": "success", "fileset.name": "s3access", + "http.response.status_code": 200, "input.type": "log", "log.offset": 1903, + "related.ip": [ + "192.0.2.3" + ], + "related.user": [ + "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be" + ], "service.type": "aws", "tls.cipher": "ECDHE-RSA-AES128-SHA", "tls.version": "1.1", - "tls.version_protocol": "tls" + "tls.version_protocol": "tls", + "user_agent.device.name": "Other", + "user_agent.name": "Other", + "user_agent.original": "S3Console/0.4" } ] \ No newline at end of file diff --git a/x-pack/filebeat/module/aws/vpcflow/ingest/pipeline.yml b/x-pack/filebeat/module/aws/vpcflow/ingest/pipeline.yml index c099ca19039..4ff3ed383fa 100644 --- a/x-pack/filebeat/module/aws/vpcflow/ingest/pipeline.yml +++ b/x-pack/filebeat/module/aws/vpcflow/ingest/pipeline.yml @@ -79,6 +79,24 @@ processors: field: related.ip value: ["{{source.ip}}", "{{destination.ip}}"] + - set: + field: cloud.provider + value: aws + + - set: + if: "ctx?.aws?.vpcflow?.account_id != null" + field: cloud.account.id + value: "{{aws.vpcflow.account_id}}" + + - set: + if: "ctx?.aws?.vpcflow?.instance_id != null && ctx.aws.vpcflow.instance_id != '-'" + field: cloud.instance.id + value: "{{aws.vpcflow.instance_id}}" + + - set: + field: event.kind + value: event + on_failure: - set: field: "error.message" diff --git a/x-pack/filebeat/module/aws/vpcflow/test/accept-reject-traffic.log-expected.json b/x-pack/filebeat/module/aws/vpcflow/test/accept-reject-traffic.log-expected.json index 1218a61da07..f31e0bf9931 100644 --- a/x-pack/filebeat/module/aws/vpcflow/test/accept-reject-traffic.log-expected.json +++ b/x-pack/filebeat/module/aws/vpcflow/test/accept-reject-traffic.log-expected.json @@ -6,6 +6,8 @@ "aws.vpcflow.interface_id": "eni-1235b8ca123456789", "aws.vpcflow.log_status": "OK", "aws.vpcflow.version": "2", + "cloud.account.id": "123456789010", + "cloud.provider": "aws", "destination.address": "158.109.0.1", "destination.as.number": 13041, "destination.as.organization.name": "Consorci de Serveis Universitaris de Catalunya", @@ -18,6 +20,7 @@ "event.category": "network_traffic", "event.dataset": "aws.vpcflow", "event.end": "2014-12-14T04:07:50.000Z", + "event.kind": "event", "event.module": "aws", "event.original": "2 123456789010 eni-1235b8ca123456789 78.24.182.42 158.109.0.1 20641 22 6 20 4249 1418530010 1418530070 ACCEPT OK", "event.outcome": "allow", @@ -59,6 +62,8 @@ "aws.vpcflow.interface_id": "eni-1235b8ca123456789", "aws.vpcflow.log_status": "OK", "aws.vpcflow.version": "2", + "cloud.account.id": "123456789010", + "cloud.provider": "aws", "destination.address": "158.109.0.1", "destination.as.number": 13041, "destination.as.organization.name": "Consorci de Serveis Universitaris de Catalunya", @@ -71,6 +76,7 @@ "event.category": "network_traffic", "event.dataset": "aws.vpcflow", "event.end": "2014-12-14T04:07:50.000Z", + "event.kind": "event", "event.module": "aws", "event.original": "2 123456789010 eni-1235b8ca123456789 78.24.182.42 158.109.0.1 49761 3389 6 20 4249 1418530010 1418530070 REJECT OK", "event.outcome": "deny", @@ -112,12 +118,15 @@ "aws.vpcflow.interface_id": "eni-1235b8ca123456789", "aws.vpcflow.log_status": "OK", "aws.vpcflow.version": "2", + "cloud.account.id": "123456789010", + "cloud.provider": "aws", "destination.address": "172.31.16.139", "destination.ip": "172.31.16.139", "destination.port": 0, "event.category": "network_traffic", "event.dataset": "aws.vpcflow", "event.end": "2015-05-29T16:32:22.000Z", + "event.kind": "event", "event.module": "aws", "event.original": "2 123456789010 eni-1235b8ca123456789 203.0.113.12 172.31.16.139 0 0 1 4 336 1432917027 1432917142 ACCEPT OK", "event.outcome": "allow", @@ -149,12 +158,15 @@ "aws.vpcflow.interface_id": "eni-1235b8ca123456789", "aws.vpcflow.log_status": "OK", "aws.vpcflow.version": "2", + "cloud.account.id": "123456789010", + "cloud.provider": "aws", "destination.address": "203.0.113.12", "destination.ip": "203.0.113.12", "destination.port": 0, "event.category": "network_traffic", "event.dataset": "aws.vpcflow", "event.end": "2015-05-29T16:32:22.000Z", + "event.kind": "event", "event.module": "aws", "event.original": "2 123456789010 eni-1235b8ca123456789 172.31.16.139 203.0.113.12 0 0 1 4 336 1432917094 1432917142 REJECT OK", "event.outcome": "deny", diff --git a/x-pack/filebeat/module/aws/vpcflow/test/custom-nat-gateway.log-expected.json b/x-pack/filebeat/module/aws/vpcflow/test/custom-nat-gateway.log-expected.json index 155d5007fb7..a1e34b59b5c 100644 --- a/x-pack/filebeat/module/aws/vpcflow/test/custom-nat-gateway.log-expected.json +++ b/x-pack/filebeat/module/aws/vpcflow/test/custom-nat-gateway.log-expected.json @@ -4,10 +4,12 @@ "aws.vpcflow.interface_id": "eni-1235b8ca123456789", "aws.vpcflow.pkt_dstaddr": "203.0.113.5", "aws.vpcflow.pkt_srcaddr": "10.0.1.5", + "cloud.provider": "aws", "destination.address": "10.0.0.220", "destination.ip": "10.0.0.220", "event.category": "network_traffic", "event.dataset": "aws.vpcflow", + "event.kind": "event", "event.module": "aws", "event.original": "- eni-1235b8ca123456789 10.0.1.5 10.0.0.220 10.0.1.5 203.0.113.5", "event.type": "flow", @@ -28,10 +30,13 @@ "aws.vpcflow.interface_id": "eni-1111aaaa2222bbbb3", "aws.vpcflow.pkt_dstaddr": "203.0.113.5", "aws.vpcflow.pkt_srcaddr": "10.0.1.5", + "cloud.instance.id": "i-01234567890123456", + "cloud.provider": "aws", "destination.address": "203.0.113.5", "destination.ip": "203.0.113.5", "event.category": "network_traffic", "event.dataset": "aws.vpcflow", + "event.kind": "event", "event.module": "aws", "event.original": "i-01234567890123456 eni-1111aaaa2222bbbb3 10.0.1.5 203.0.113.5 10.0.1.5 203.0.113.5", "event.type": "flow", diff --git a/x-pack/filebeat/module/aws/vpcflow/test/custom-transit-gateway.log-expected.json b/x-pack/filebeat/module/aws/vpcflow/test/custom-transit-gateway.log-expected.json index 87ff5dc60a3..d288b8b06db 100644 --- a/x-pack/filebeat/module/aws/vpcflow/test/custom-transit-gateway.log-expected.json +++ b/x-pack/filebeat/module/aws/vpcflow/test/custom-transit-gateway.log-expected.json @@ -12,11 +12,15 @@ "aws.vpcflow.type": "IPv4", "aws.vpcflow.version": "3", "aws.vpcflow.vpc_id": "vpc-abcdefab012345678", + "cloud.account.id": "123456789010", + "cloud.instance.id": "i-01234567890123456", + "cloud.provider": "aws", "destination.address": "10.40.2.236", "destination.ip": "10.40.2.236", "destination.port": 80, "event.category": "network_traffic", "event.dataset": "aws.vpcflow", + "event.kind": "event", "event.module": "aws", "event.original": "3 eni-33333333333333333 123456789010 vpc-abcdefab012345678 subnet-22222222bbbbbbbbb i-01234567890123456 10.20.33.164 10.40.2.236 39812 80 6 3 IPv4 10.20.33.164 10.40.2.236 ACCEPT OK", "event.outcome": "allow", diff --git a/x-pack/filebeat/module/aws/vpcflow/test/ipv6.log-expected.json b/x-pack/filebeat/module/aws/vpcflow/test/ipv6.log-expected.json index c8c95ee428e..12899b7b728 100644 --- a/x-pack/filebeat/module/aws/vpcflow/test/ipv6.log-expected.json +++ b/x-pack/filebeat/module/aws/vpcflow/test/ipv6.log-expected.json @@ -6,12 +6,15 @@ "aws.vpcflow.interface_id": "eni-1235b8ca123456789", "aws.vpcflow.log_status": "OK", "aws.vpcflow.version": "2", + "cloud.account.id": "123456789010", + "cloud.provider": "aws", "destination.address": "2001:db8:1234:a102:3304:8879:34cf:4071", "destination.ip": "2001:db8:1234:a102:3304:8879:34cf:4071", "destination.port": 22, "event.category": "network_traffic", "event.dataset": "aws.vpcflow", "event.end": "2016-10-31T11:37:00.000Z", + "event.kind": "event", "event.module": "aws", "event.original": "2 123456789010 eni-1235b8ca123456789 2001:db8:1234:a100:8d6e:3477:df66:f105 2001:db8:1234:a102:3304:8879:34cf:4071 34892 22 6 54 8855 1477913708 1477913820 ACCEPT OK", "event.outcome": "allow", diff --git a/x-pack/filebeat/module/aws/vpcflow/test/no-data-skip-data.log-expected.json b/x-pack/filebeat/module/aws/vpcflow/test/no-data-skip-data.log-expected.json index c934d68d96b..456b3efca62 100644 --- a/x-pack/filebeat/module/aws/vpcflow/test/no-data-skip-data.log-expected.json +++ b/x-pack/filebeat/module/aws/vpcflow/test/no-data-skip-data.log-expected.json @@ -6,9 +6,12 @@ "aws.vpcflow.interface_id": "eni-1235b8ca123456789", "aws.vpcflow.log_status": "NODATA", "aws.vpcflow.version": "2", + "cloud.account.id": "123456789010", + "cloud.provider": "aws", "event.category": "network_traffic", "event.dataset": "aws.vpcflow", "event.end": "2015-05-10T18:02:14.000Z", + "event.kind": "event", "event.module": "aws", "event.original": "2 123456789010 eni-1235b8ca123456789 - - - - - - - 1431280876 1431280934 - NODATA", "event.start": "2015-05-10T18:01:16.000Z", @@ -25,9 +28,12 @@ "aws.vpcflow.interface_id": "eni-11111111aaaaaaaaa", "aws.vpcflow.log_status": "SKIPDATA", "aws.vpcflow.version": "2", + "cloud.account.id": "123456789010", + "cloud.provider": "aws", "event.category": "network_traffic", "event.dataset": "aws.vpcflow", "event.end": "2015-05-10T18:02:14.000Z", + "event.kind": "event", "event.module": "aws", "event.original": "2 123456789010 eni-11111111aaaaaaaaa - - - - - - - 1431280876 1431280934 - SKIPDATA", "event.start": "2015-05-10T18:01:16.000Z", diff --git a/x-pack/filebeat/module/aws/vpcflow/test/tcp-flag-sequence.log-expected.json b/x-pack/filebeat/module/aws/vpcflow/test/tcp-flag-sequence.log-expected.json index c36a3141346..cb24fd34183 100644 --- a/x-pack/filebeat/module/aws/vpcflow/test/tcp-flag-sequence.log-expected.json +++ b/x-pack/filebeat/module/aws/vpcflow/test/tcp-flag-sequence.log-expected.json @@ -13,12 +13,16 @@ "aws.vpcflow.type": "IPv4", "aws.vpcflow.version": "3", "aws.vpcflow.vpc_id": "vpc-abcdefab012345678", + "cloud.account.id": "123456789010", + "cloud.instance.id": "i-01234567890123456", + "cloud.provider": "aws", "destination.address": "10.0.0.62", "destination.ip": "10.0.0.62", "destination.port": 5001, "event.category": "network_traffic", "event.dataset": "aws.vpcflow", "event.end": "2019-08-26T19:48:53.000Z", + "event.kind": "event", "event.module": "aws", "event.original": "3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 IPv4 52.213.180.42 10.0.0.62 43416 5001 52.213.180.42 10.0.0.62 6 568 8 1566848875 1566848933 ACCEPT 2 OK", "event.outcome": "allow",