[Filebeat] Improve ECS field mappings in aws module (#16307) (#16478)

* Improve ECS field mappings in aws module

- elb fileset
  + cloud.provider
  + event.category
  + event.kind
  + event.outcome
  + http.response.status_code, convert to long
  + http.request.method, lowercase
  + tracing.trace.id

- s3access fileset
  + client.address
  + client.ip
  + geo
  + client.user.id
  + cloud.provider
  + event.action
  + event.code
  + event.duration
  + event.id
  + event.kind
  + event.outcome
  + http.request.referrer
  + http.response.status_code
  + related.ip
  + related.user
  + user_agent

- vpcflow fileset
  + cloud.provider
  + cloud.account.id
  + cloud.instance.id
  + event.kind

Closes #16154

(cherry picked from commit 913f7ee)

CHANGELOG.next.asciidoc

@@ -204,6 +204,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add an SSL config example in config.yml for filebeat MISP module. {pull}16320[16320]
 - Improve ECS categorization, container & process field mappings in auditd module. {issue}16153[16153] {pull}16280[16280]
 - Add ECS categorization fields to activemq module. {issue}16151[16151] {pull}16201[16201]
+- Improve ECS field mappings in aws module. {issue}16154[16154] {pull}16307[16307]
 
 *Heartbeat*
 

x-pack/filebeat/module/aws/elb/ingest/pipeline.yml

@@ -70,8 +70,8 @@ processors:
  %{ELBPROCESSINGTIME}
  ELBHTTPLOG: >-
  %{ELBCOMMON}
- %{NUMBER:http.response.status_code}
- (?:-|%{NUMBER:aws.elb.backend.http.response.status_code})
+ %{NUMBER:http.response.status_code:long}
+ (?:-|%{NUMBER:aws.elb.backend.http.response.status_code:long})
  %{NUMBER:http.request.body.bytes:long}
  %{NUMBER:http.response.body.bytes:long}
  \"(?:-|%{WORD:http.request.method}) (?:-|%{NOTSPACE:http.request.referrer}) (?:-|HTTP/%{NOTSPACE:http.version})\"
@@ -89,17 +89,53 @@ processors:
  ELBV2TYPE: '%{WORD:aws.elb.type}'
  ELBV2LOGVERSION: '%{NOTSPACE}' # Could be used to support different log versions, only 1.0 exists now
 
+ - set:
+ field: event.kind
+ value: event
+
+ - set:
+ field: cloud.provider
+ value: aws
 
  - set:
  if: 'ctx.http != null'
  field: 'aws.elb.protocol'
  value: 'http'
 
+ - set:
+ if: 'ctx.http != null'
+ field: event.category
+ value: web
+
  - set:
  if: 'ctx.http == null'
  field: 'aws.elb.protocol'
  value: 'tcp'
 
+ - set:
+ if: 'ctx.http == null'
+ field: event.category
+ value: network
+
+ - set:
+ if: 'ctx?.http?.response?.status_code != null && ctx.http.response.status_code < 400'
+ field: event.outcome
+ value: success
+
+ - set:
+ if: 'ctx?.http?.response?.status_code != null && ctx.http.response.status_code >= 400'
+ field: event.outcome
+ value: failure
+
+ - lowercase:
+ field: http.request.method
+ ignore_missing: true
+
+ - set:
+ if: "ctx?.aws?.elb?.trace_id != null"
+ field: tracing.trace.id
+ value: "{{aws.elb.trace_id}}"
+
  - split:
  field: '_tmp.actions_executed'
  target_field: 'aws.elb.action_executed'