Skip to content

Commit

Permalink
Improve ECS field mappings in aws module
Browse files Browse the repository at this point in the history
- elb fileset
  + cloud.provider
  + event.category
  + event.kind
  + event.outcome
  + http.response.status_code, convert to long
  + http.request.method, lowercase
  + tracing.trace.id

- s3access fileset
  + client.address
  + client.ip
  + geo
  + client.user.id
  + cloud.provider
  + event.action
  + event.code
  + event.duration
  + event.id
  + event.kind
  + event.outcome
  + http.request.referrer
  + http.response.status_code
  + related.user
  + user_agent

- vpcflow fileset
  + cloud.provider
  + cloud.account.id
  + cloud.instance.id
  + event.kind

Closes elastic#16154
  • Loading branch information
leehinman committed Feb 13, 2020
1 parent 69cd8d8 commit 0539c3e
Show file tree
Hide file tree
Showing 21 changed files with 628 additions and 73 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.next.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
- Add ingress nginx controller fileset {pull}16197[16197]
- move create-[module,fileset,fields] to mage and enable in x-pack/filebeat {pull}15836[15836]
- Add ECS tls and categorization fields to apache module. {issue}16032[16032] {pull}16121[16121]
- Improve ECS field mappings in aws module. {issue}16154[16154] {pull}16307[16307]

*Heartbeat*

Expand Down
41 changes: 41 additions & 0 deletions x-pack/filebeat/module/aws/elb/ingest/pipeline.yml
Original file line number Diff line number Diff line change
Expand Up @@ -89,17 +89,58 @@ processors:
ELBV2TYPE: '%{WORD:aws.elb.type}'
ELBV2LOGVERSION: '%{NOTSPACE}' # Could be used to support different log versions, only 1.0 exists now

- set:
field: event.kind
value: event

- set:
field: cloud.provider
value: aws

- set:
if: 'ctx.http != null'
field: 'aws.elb.protocol'
value: 'http'

- set:
if: 'ctx.http != null'
field: event.category
value: web

- set:
if: 'ctx.http == null'
field: 'aws.elb.protocol'
value: 'tcp'

- set:
if: 'ctx.http == null'
field: event.category
value: network

- convert:
field: http.response.status_code
type: long
ignore_failure: true

- set:
if: 'ctx?.http?.response?.status_code != null && ctx.http.response.status_code < 400'
field: event.outcome
value: success

- set:
if: 'ctx?.http?.response?.status_code != null && ctx.http.response.status_code > 399'
field: event.outcome
value: failure

- lowercase:
field: http.request.method
ignore_missing: true

- set:
if: "ctx?.aws?.elb?.trace_id != null"
field: tracing.trace.id
value: "{{aws.elb.trace_id}}"

- split:
field: '_tmp.actions_executed'
target_field: 'aws.elb.action_executed'
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,16 +12,20 @@
"aws.elb.target_group.arn": "arn:aws:elasticloadbalancing:eu-central-1:627959692251:targetgroup/test-lb-instances/8f04c4fe71f5f794",
"aws.elb.trace_id": "Root=1-5da09932-2c342a443bfb96249aa50ed7",
"aws.elb.type": "http",
"cloud.provider": "aws",
"event.category": "web",
"event.dataset": "aws.elb",
"event.end": "2019-10-11T15:01:12.376Z",
"event.kind": "event",
"event.module": "aws",
"event.outcome": "failure",
"event.start": "2019-10-11T15:01:06.657000Z",
"fileset.name": "elb",
"http.request.body.bytes": 125,
"http.request.method": "GET",
"http.request.method": "get",
"http.request.referrer": "http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/",
"http.response.body.bytes": 0,
"http.response.status_code": "460",
"http.response.status_code": 460,
"http.version": "1.1",
"input.type": "log",
"log.offset": 0,
Expand All @@ -37,6 +41,7 @@
"source.geo.region_name": "Teruel",
"source.ip": "77.227.156.41",
"source.port": "56398",
"tracing.trace.id": "Root=1-5da09932-2c342a443bfb96249aa50ed7",
"user_agent.original": "curl/7.58.0"
},
{
Expand All @@ -52,16 +57,20 @@
"aws.elb.target_group.arn": "arn:aws:elasticloadbalancing:eu-central-1:627959692251:targetgroup/test-lb-instances/8f04c4fe71f5f794",
"aws.elb.trace_id": "Root=1-5da09954-2c342a443bfb96249aa50ed7",
"aws.elb.type": "http",
"cloud.provider": "aws",
"event.category": "web",
"event.dataset": "aws.elb",
"event.end": "2019-10-11T15:01:50.492Z",
"event.kind": "event",
"event.module": "aws",
"event.outcome": "failure",
"event.start": "2019-10-11T15:01:40.491000Z",
"fileset.name": "elb",
"http.request.body.bytes": 125,
"http.request.method": "GET",
"http.request.method": "get",
"http.request.referrer": "http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/",
"http.response.body.bytes": 308,
"http.response.status_code": "504",
"http.response.status_code": 504,
"http.version": "1.1",
"input.type": "log",
"log.offset": 438,
Expand All @@ -77,6 +86,7 @@
"source.geo.region_name": "Teruel",
"source.ip": "77.227.156.41",
"source.port": "56488",
"tracing.trace.id": "Root=1-5da09954-2c342a443bfb96249aa50ed7",
"user_agent.original": "curl/7.58.0"
},
{
Expand All @@ -92,16 +102,20 @@
"aws.elb.target_group.arn": "arn:aws:elasticloadbalancing:eu-central-1:627959692251:targetgroup/test-lb-instances/8f04c4fe71f5f794",
"aws.elb.trace_id": "Root=1-5da09938-d9c72660e247c36070017828",
"aws.elb.type": "http",
"cloud.provider": "aws",
"event.category": "web",
"event.dataset": "aws.elb",
"event.end": "2019-10-11T15:01:22.915Z",
"event.kind": "event",
"event.module": "aws",
"event.outcome": "failure",
"event.start": "2019-10-11T15:01:12.914000Z",
"fileset.name": "elb",
"http.request.body.bytes": 125,
"http.request.method": "GET",
"http.request.method": "get",
"http.request.referrer": "http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/",
"http.response.body.bytes": 308,
"http.response.status_code": "504",
"http.response.status_code": 504,
"http.version": "1.1",
"input.type": "log",
"log.offset": 878,
Expand All @@ -117,6 +131,7 @@
"source.geo.region_name": "Teruel",
"source.ip": "77.227.156.41",
"source.port": "56416",
"tracing.trace.id": "Root=1-5da09938-d9c72660e247c36070017828",
"user_agent.original": "curl/7.58.0"
},
{
Expand All @@ -132,16 +147,20 @@
"aws.elb.target_group.arn": "arn:aws:elasticloadbalancing:eu-central-1:627959692251:targetgroup/test-lb-instances/8f04c4fe71f5f794",
"aws.elb.trace_id": "Root=1-5da09945-0eaa8050df7d96f84806ded0",
"aws.elb.type": "http",
"cloud.provider": "aws",
"event.category": "web",
"event.dataset": "aws.elb",
"event.end": "2019-10-11T15:01:35.190Z",
"event.kind": "event",
"event.module": "aws",
"event.outcome": "failure",
"event.start": "2019-10-11T15:01:25.189000Z",
"fileset.name": "elb",
"http.request.body.bytes": 125,
"http.request.method": "GET",
"http.request.method": "get",
"http.request.referrer": "http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/",
"http.response.body.bytes": 308,
"http.response.status_code": "504",
"http.response.status_code": 504,
"http.version": "1.1",
"input.type": "log",
"log.offset": 1318,
Expand All @@ -157,6 +176,7 @@
"source.geo.region_name": "Teruel",
"source.ip": "77.227.156.41",
"source.port": "56448",
"tracing.trace.id": "Root=1-5da09945-0eaa8050df7d96f84806ded0",
"user_agent.original": "curl/7.58.0"
},
{
Expand All @@ -172,16 +192,20 @@
"aws.elb.target_group.arn": "arn:aws:elasticloadbalancing:eu-central-1:627959692251:targetgroup/test-lb-instances/8f04c4fe71f5f794",
"aws.elb.trace_id": "Root=1-5da0997a-5add00b04bc8ae20ae96d9f0",
"aws.elb.type": "http",
"cloud.provider": "aws",
"event.category": "web",
"event.dataset": "aws.elb",
"event.end": "2019-10-11T15:02:28.837Z",
"event.kind": "event",
"event.module": "aws",
"event.outcome": "failure",
"event.start": "2019-10-11T15:02:18.836000Z",
"fileset.name": "elb",
"http.request.body.bytes": 125,
"http.request.method": "GET",
"http.request.method": "get",
"http.request.referrer": "http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/",
"http.response.body.bytes": 308,
"http.response.status_code": "504",
"http.response.status_code": 504,
"http.version": "1.1",
"input.type": "log",
"log.offset": 1758,
Expand All @@ -197,6 +221,7 @@
"source.geo.region_name": "Teruel",
"source.ip": "77.227.156.41",
"source.port": "56602",
"tracing.trace.id": "Root=1-5da0997a-5add00b04bc8ae20ae96d9f0",
"user_agent.original": "curl/7.58.0"
},
{
Expand All @@ -212,16 +237,20 @@
"aws.elb.target_group.arn": "arn:aws:elasticloadbalancing:eu-central-1:627959692251:targetgroup/test-lb-instances/8f04c4fe71f5f794",
"aws.elb.trace_id": "Root=1-5da09987-cc391940b332434860dfa848",
"aws.elb.type": "http",
"cloud.provider": "aws",
"event.category": "web",
"event.dataset": "aws.elb",
"event.end": "2019-10-11T15:02:41.203Z",
"event.kind": "event",
"event.module": "aws",
"event.outcome": "failure",
"event.start": "2019-10-11T15:02:31.202000Z",
"fileset.name": "elb",
"http.request.body.bytes": 125,
"http.request.method": "GET",
"http.request.method": "get",
"http.request.referrer": "http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/",
"http.response.body.bytes": 308,
"http.response.status_code": "504",
"http.response.status_code": 504,
"http.version": "1.1",
"input.type": "log",
"log.offset": 2198,
Expand All @@ -237,6 +266,7 @@
"source.geo.region_name": "Teruel",
"source.ip": "77.227.156.41",
"source.port": "56638",
"tracing.trace.id": "Root=1-5da09987-cc391940b332434860dfa848",
"user_agent.original": "curl/7.58.0"
},
{
Expand All @@ -252,16 +282,20 @@
"aws.elb.target_group.arn": "arn:aws:elasticloadbalancing:eu-central-1:627959692251:targetgroup/test-lb-instances/8f04c4fe71f5f794",
"aws.elb.trace_id": "Root=1-5da099cb-3d3b17eb2b75373f4c0c36c5",
"aws.elb.type": "http",
"cloud.provider": "aws",
"event.category": "web",
"event.dataset": "aws.elb",
"event.end": "2019-10-11T15:03:49.331Z",
"event.kind": "event",
"event.module": "aws",
"event.outcome": "failure",
"event.start": "2019-10-11T15:03:39.331000Z",
"fileset.name": "elb",
"http.request.body.bytes": 125,
"http.request.method": "GET",
"http.request.method": "get",
"http.request.referrer": "http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/",
"http.response.body.bytes": 308,
"http.response.status_code": "504",
"http.response.status_code": 504,
"http.version": "1.1",
"input.type": "log",
"log.offset": 2638,
Expand All @@ -277,6 +311,7 @@
"source.geo.region_name": "Teruel",
"source.ip": "77.227.156.41",
"source.port": "37632",
"tracing.trace.id": "Root=1-5da099cb-3d3b17eb2b75373f4c0c36c5",
"user_agent.original": "curl/7.58.0"
},
{
Expand All @@ -296,16 +331,20 @@
"aws.elb.target_group.arn": "arn:aws:elasticloadbalancing:eu-central-1:627959692251:targetgroup/test-lb-instances/8f04c4fe71f5f794",
"aws.elb.trace_id": "Root=1-5da0a5dd-4d9a423a0e9a782fe2f390af",
"aws.elb.type": "http",
"cloud.provider": "aws",
"event.category": "web",
"event.dataset": "aws.elb",
"event.end": "2019-10-11T15:55:09.308Z",
"event.kind": "event",
"event.module": "aws",
"event.outcome": "success",
"event.start": "2019-10-11T15:55:09.307000Z",
"fileset.name": "elb",
"http.request.body.bytes": 125,
"http.request.method": "GET",
"http.request.method": "get",
"http.request.referrer": "http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/",
"http.response.body.bytes": 859,
"http.response.status_code": "200",
"http.response.status_code": 200,
"http.version": "1.1",
"input.type": "log",
"log.offset": 3078,
Expand All @@ -321,6 +360,7 @@
"source.geo.region_name": "Teruel",
"source.ip": "77.227.156.41",
"source.port": "37838",
"tracing.trace.id": "Root=1-5da0a5dd-4d9a423a0e9a782fe2f390af",
"user_agent.original": "curl/7.58.0"
},
{
Expand All @@ -340,16 +380,20 @@
"aws.elb.target_group.arn": "arn:aws:elasticloadbalancing:eu-central-1:627959692251:targetgroup/test-lb-instances/8f04c4fe71f5f794",
"aws.elb.trace_id": "Root=1-5da0a5df-7d64cabe9955b4df9acc800a",
"aws.elb.type": "http",
"cloud.provider": "aws",
"event.category": "web",
"event.dataset": "aws.elb",
"event.end": "2019-10-11T15:55:11.354Z",
"event.kind": "event",
"event.module": "aws",
"event.outcome": "success",
"event.start": "2019-10-11T15:55:11.352000Z",
"fileset.name": "elb",
"http.request.body.bytes": 125,
"http.request.method": "GET",
"http.request.method": "get",
"http.request.referrer": "http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/",
"http.response.body.bytes": 859,
"http.response.status_code": "200",
"http.response.status_code": 200,
"http.version": "1.1",
"input.type": "log",
"log.offset": 3529,
Expand All @@ -365,6 +409,7 @@
"source.geo.region_name": "Teruel",
"source.ip": "77.227.156.41",
"source.port": "37850",
"tracing.trace.id": "Root=1-5da0a5df-7d64cabe9955b4df9acc800a",
"user_agent.original": "curl/7.58.0"
},
{
Expand All @@ -384,16 +429,20 @@
"aws.elb.target_group.arn": "arn:aws:elasticloadbalancing:eu-central-1:627959692251:targetgroup/test-lb-instances/8f04c4fe71f5f794",
"aws.elb.trace_id": "Root=1-5da0a5df-7c958e828ff43b63d0e0fac4",
"aws.elb.type": "http",
"cloud.provider": "aws",
"event.category": "web",
"event.dataset": "aws.elb",
"event.end": "2019-10-11T15:55:11.987Z",
"event.kind": "event",
"event.module": "aws",
"event.outcome": "success",
"event.start": "2019-10-11T15:55:11.987000Z",
"fileset.name": "elb",
"http.request.body.bytes": 125,
"http.request.method": "GET",
"http.request.method": "get",
"http.request.referrer": "http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/",
"http.response.body.bytes": 859,
"http.response.status_code": "200",
"http.response.status_code": 200,
"http.version": "1.1",
"input.type": "log",
"log.offset": 3980,
Expand All @@ -409,6 +458,7 @@
"source.geo.region_name": "Teruel",
"source.ip": "77.227.156.41",
"source.port": "37856",
"tracing.trace.id": "Root=1-5da0a5df-7c958e828ff43b63d0e0fac4",
"user_agent.original": "curl/7.58.0"
}
]
Loading

0 comments on commit 0539c3e

Please sign in to comment.