Skip to content

Commit

Permalink
Filebeat: Update aws/cloudtrail dataset to ECS 1.8 (elastic#23911)
Browse files Browse the repository at this point in the history
Updates aws/cloudtrail to map multiuser events to ECS 1.8.
  • Loading branch information
adriansr authored Feb 10, 2021
1 parent 25aeea3 commit eb15834
Show file tree
Hide file tree
Showing 33 changed files with 146 additions and 24 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.next.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -840,6 +840,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
- Updated azure module to ECS 1.8. {issue}23118[23118] {pull}23927[23927]
- Update aws/s3access to ECS 1.8. {issue}23118[23118] {pull}23920[23920]
- Upgrade panw module to ecs 1.8 {issue}23118[23118] {pull}23931[23931]
- Updated aws/cloudtrail fileset to ECS 1.8. {issue}23118[23118] {pull}23911[23911]

*Heartbeat*

Expand Down
2 changes: 1 addition & 1 deletion x-pack/filebeat/module/aws/cloudtrail/config/aws-s3.yml
Original file line number Diff line number Diff line change
Expand Up @@ -66,4 +66,4 @@ processors:
- add_fields:
target: ''
fields:
ecs.version: 1.7.0
ecs.version: 1.8.0
2 changes: 1 addition & 1 deletion x-pack/filebeat/module/aws/cloudtrail/config/file.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,4 +11,4 @@ processors:
- add_fields:
target: ''
fields:
ecs.version: 1.7.0
ecs.version: 1.8.0
63 changes: 41 additions & 22 deletions x-pack/filebeat/module/aws/cloudtrail/ingest/pipeline.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,11 @@ processors:
field: "json.userIdentity.type"
target_field: "aws.cloudtrail.user_identity.type"
ignore_failure: true
- append:
field: related.user
value: '{{json.userIdentity.userName}}'
allow_duplicates: false
if: 'ctx.json?.userIdentity?.userName != null'
- rename:
field: "json.userIdentity.userName"
target_field: "user.name"
Expand Down Expand Up @@ -225,28 +230,16 @@ processors:
field: "json.vpcEndpointId"
target_field: "aws.cloudtrail.vpc_endpoint_id"
ignore_failure: true
- script:
lang: painless
ignore_failure: true
source: >-
void addRelatedUser(def ctx, String userName) {
if (ctx.related == null) {
Map map = new HashMap();
ctx.put("related", map);
}
if (ctx.related.user == null) {
ArrayList al = new ArrayList();
ctx.related.put("user", al);
}
ctx.related.user.add(userName);
}
if (ctx?.aws?.cloudtrail?.flattened?.request_parameters?.userName != null) {
addRelatedUser(ctx, ctx.aws.cloudtrail.flattened.request_parameters.userName);
}
if (ctx?.aws?.cloudtrail?.flattened?.request_parameters?.newUserName != null) {
addRelatedUser(ctx, ctx.aws.cloudtrail.flattened.request_parameters.newUserName);
}
- append:
field: related.user
value: '{{aws.cloudtrail.flattened.request_parameters.userName}}'
allow_duplicates: false
if: 'ctx.aws?.cloudtrail?.flattened?.request_parameters?.userName != null'
- append:
field: related.user
value: '{{aws.cloudtrail.flattened.request_parameters.newUserName}}'
allow_duplicates: false
if: 'ctx.aws?.cloudtrail?.flattened?.request_parameters?.newUserName != null'
- script:
lang: painless
ignore_failure: true
Expand Down Expand Up @@ -685,6 +678,32 @@ processors:
field: "json.insightDetails"
target_field: "aws.cloudtrail.insight_details"
ignore_failure: true
- set:
field: group.id
value: '{{aws.cloudtrail.flattened.response_elements.group.groupId}}'
ignore_empty_value: true
ignore_failure: true
- set:
field: user.target.id
value: '{{aws.cloudtrail.flattened.response_elements.user.userId}}'
ignore_empty_value: true
ignore_failure: true
- set:
field: user.changes.name
value: '{{aws.cloudtrail.flattened.request_parameters.newUserName}}'
ignore_empty_value: true
ignore_failure: true
- set:
field: group.name
value: '{{aws.cloudtrail.flattened.request_parameters.groupName}}'
ignore_empty_value: true
ignore_failure: true
- set:
field: user.target.name
value: '{{aws.cloudtrail.flattened.request_parameters.userName}}'
ignore_empty_value: true
ignore_failure: true

- remove:
field:
- "json"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,11 @@
"change"
],
"fileset.name": "cloudtrail",
"group.name": "admin",
"input.type": "log",
"log.offset": 0,
"related.user": [
"Alice",
"Bob"
],
"service.type": "aws",
Expand All @@ -40,6 +42,7 @@
],
"user.id": "EX_PRINCIPAL_ID",
"user.name": "Alice",
"user.target.name": "Bob",
"user_agent.device.name": "Other",
"user_agent.name": "Other",
"user_agent.original": "AWSConsole"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,9 @@
"fileset.name": "cloudtrail",
"input.type": "log",
"log.offset": 0,
"related.user": [
"Alice"
],
"service.type": "aws",
"source.address": "127.0.0.1",
"source.ip": "127.0.0.1",
Expand Down Expand Up @@ -70,6 +73,9 @@
"fileset.name": "cloudtrail",
"input.type": "log",
"log.offset": 720,
"related.user": [
"Alice"
],
"service.type": "aws",
"source.address": "127.0.0.1",
"source.ip": "127.0.0.1",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,9 @@
"fileset.name": "cloudtrail",
"input.type": "log",
"log.offset": 0,
"related.user": [
"JohnDoe"
],
"service.type": "aws",
"source.address": "192.0.2.110",
"source.ip": "192.0.2.110",
Expand Down Expand Up @@ -82,6 +85,9 @@
"fileset.name": "cloudtrail",
"input.type": "log",
"log.offset": 658,
"related.user": [
"JaneDoe"
],
"service.type": "aws",
"source.address": "192.0.2.100",
"source.ip": "192.0.2.100",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
"input.type": "log",
"log.offset": 0,
"related.user": [
"Alice",
"Bob"
],
"service.type": "aws",
Expand All @@ -48,6 +49,7 @@
],
"user.id": "EXAMPLE_ID",
"user.name": "Alice",
"user.target.name": "Bob",
"user_agent.device.name": "Other",
"user_agent.name": "Other",
"user_agent.original": "signin.amazonaws.com"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,13 @@
"creation"
],
"fileset.name": "cloudtrail",
"group.id": "EXAMPLE_ID",
"group.name": "TEST-GROUP",
"input.type": "log",
"log.offset": 0,
"related.user": [
"Alice"
],
"service.type": "aws",
"source.address": "127.0.0.1",
"source.ip": "127.0.0.1",
Expand Down Expand Up @@ -80,8 +85,12 @@
"creation"
],
"fileset.name": "cloudtrail",
"group.name": "TEST-GROUP",
"input.type": "log",
"log.offset": 903,
"related.user": [
"Alice"
],
"service.type": "aws",
"source.address": "127.0.0.1",
"source.ip": "127.0.0.1",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,9 @@
"fileset.name": "cloudtrail",
"input.type": "log",
"log.offset": 0,
"related.user": [
"Alice"
],
"service.type": "aws",
"source.address": "72.21.198.64",
"source.as.number": 16509,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,9 @@
"fileset.name": "cloudtrail",
"input.type": "log",
"log.offset": 0,
"related.user": [
"Alice"
],
"service.type": "aws",
"source.address": "127.0.0.1",
"source.ip": "127.0.0.1",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
"input.type": "log",
"log.offset": 0,
"related.user": [
"Alice",
"Bob"
],
"service.type": "aws",
Expand All @@ -43,6 +44,8 @@
],
"user.id": "EX_PRINCIPAL_ID",
"user.name": "Alice",
"user.target.id": "EXAMPLEUSERID",
"user.target.name": "Bob",
"user_agent.device.name": "Other",
"user_agent.name": "aws-cli",
"user_agent.original": "aws-cli/1.3.2 Python/2.7.5 Windows/7",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,9 @@
"fileset.name": "cloudtrail",
"input.type": "log",
"log.offset": 0,
"related.user": [
"Alice"
],
"service.type": "aws",
"source.address": "127.0.0.1",
"source.ip": "127.0.0.1",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@
],
"user.id": "EXAMPLE_ID",
"user.name": "Alice",
"user.target.name": "Alice",
"user_agent.device.name": "Other",
"user_agent.name": "Other",
"user_agent.original": "signin.amazonaws.com"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
"input.type": "log",
"log.offset": 0,
"related.user": [
"Alice",
"Bob"
],
"service.type": "aws",
Expand All @@ -44,6 +45,7 @@
],
"user.id": "EXAMPLE_ID",
"user.name": "Alice",
"user.target.name": "Bob",
"user_agent.device.name": "Other",
"user_agent.name": "Other",
"user_agent.original": "signin.amazonaws.com"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,12 @@
"deletion"
],
"fileset.name": "cloudtrail",
"group.name": "TEST-GROUP",
"input.type": "log",
"log.offset": 0,
"related.user": [
"Alice"
],
"service.type": "aws",
"source.address": "127.0.0.1",
"source.ip": "127.0.0.1",
Expand Down Expand Up @@ -74,8 +78,12 @@
"deletion"
],
"fileset.name": "cloudtrail",
"group.name": "TEST-GROUP",
"input.type": "log",
"log.offset": 747,
"related.user": [
"Alice"
],
"service.type": "aws",
"source.address": "127.0.0.1",
"source.ip": "127.0.0.1",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
"input.type": "log",
"log.offset": 0,
"related.user": [
"Alice",
"Bob"
],
"service.type": "aws",
Expand All @@ -44,6 +45,7 @@
],
"user.id": "EXAMPLE_ID",
"user.name": "Alice",
"user.target.name": "Bob",
"user_agent.device.name": "Other",
"user_agent.name": "Other",
"user_agent.original": "signin.amazonaws.com"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,9 @@
"fileset.name": "cloudtrail",
"input.type": "log",
"log.offset": 0,
"related.user": [
"Alice"
],
"service.type": "aws",
"source.address": "127.0.0.1",
"source.ip": "127.0.0.1",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
"input.type": "log",
"log.offset": 0,
"related.user": [
"Alice",
"Bob"
],
"service.type": "aws",
Expand All @@ -43,6 +44,7 @@
],
"user.id": "EX_PRINCIPAL_ID",
"user.name": "Alice",
"user.target.name": "Bob",
"user_agent.device.name": "Other",
"user_agent.name": "Other",
"user_agent.original": "signin.amazonaws.com"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,9 @@
"fileset.name": "cloudtrail",
"input.type": "log",
"log.offset": 0,
"related.user": [
"Alice"
],
"service.type": "aws",
"source.address": "127.0.0.1",
"source.ip": "127.0.0.1",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,9 @@
"fileset.name": "cloudtrail",
"input.type": "log",
"log.offset": 0,
"related.user": [
"REDACTED"
],
"service.type": "aws",
"source.address": "REDACTED",
"tags": [
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
"input.type": "log",
"log.offset": 0,
"related.user": [
"Alice",
"Bob"
],
"service.type": "aws",
Expand All @@ -43,6 +44,7 @@
],
"user.id": "EXAMPLE_ID",
"user.name": "Alice",
"user.target.name": "Bob",
"user_agent.device.name": "Other",
"user_agent.name": "Other",
"user_agent.original": "console.amazonaws.com"
Expand Down
Loading

0 comments on commit eb15834

Please sign in to comment.