[Filebeat] Improve AWS cloudtrail field mappings (elastic#17155)

* Improve AWS cloudtrail field mappings - sessionIssuer.type -> aws.cloudtrail.user_identity.session_issuer.type - sessionIssuer.principalId -> aws.cloudtrail.user_identity.session_issuer.principal_id - sessionIssuer.userName -> user.name - sessionIssuer.arn -> aws.cloudtrail.user_identity.session_issuer.arn - sessionIssuer.accountId -> aws.cloudtrail.user_identity.session_issuer.account_id - add aws.cloudtrail.console_login.additional_eventdata.mobile_version - add aws.cloudtrail.console_login.additional_eventdata.login_to - add aws.cloudtrail.console_login.additional_eventdata.mfa_used - copy source.address to source.ip if value is an IP address Closes elastic#16086 Closes elastic#16110 (cherry picked from commit 57e194b)
leehinman · Mar 23, 2020 · 0edd90d · 0edd90d
1 parent a5d8979
commit 0edd90d
Show file tree

Hide file tree

Showing 36 changed files with 286 additions and 5 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -274,6 +274,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add `o365audit` input type for consuming events from Office 365 Management Activity API. {issue}16196[16196] {pull}16244[16244]
 - Add custom string mapping to CEF module to support Check Point devices. {issue}16041[16041] {pull}16907[16907]
 - Added new module `o365` for ingesting Office 365 management activity API events. {issue}16196[16196] {pull}16386[16386]
+- Improve AWS cloudtrail field mappings {issue}16086[16086] {issue}16110[16110] {pull}17155[17155]
 
 *Heartbeat*
 

diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc
@@ -1155,6 +1155,48 @@ type: keyword
 
 --
 
+[float]
+=== session_issuer
+
+If the request was made with temporary security credentials, an element that provides information about how the credentials were obtained.
+
+
+*`aws.cloudtrail.user_identity.session_issuer.type`*::
++
+--
+The source of the temporary security credentials, such as Root, IAMUser, or Role.
+
+type: keyword
+
+--
+
+*`aws.cloudtrail.user_identity.session_issuer.principal_id`*::
++
+--
+The internal ID of the entity that was used to get credentials.
+
+type: keyword
+
+--
+
+*`aws.cloudtrail.user_identity.session_issuer.arn`*::
++
+--
+The ARN of the source (account, IAM user, or role) that was used to get temporary security credentials.
+
+type: keyword
+
+--
+
+*`aws.cloudtrail.user_identity.session_issuer.account_id`*::
++
+--
+The account that owns the entity that was used to get credentials.
+
+type: keyword
+
+--
+
 *`aws.cloudtrail.error_code`*::
 +
 --
@@ -1314,6 +1356,46 @@ type: keyword
 
 --
 
+[float]
+=== console_login
+
+Fields specific to ConsoleLogin events
+
+
+[float]
+=== additional_eventdata
+
+Additional Event Data for ConsoleLogin events
+
+
+
+*`aws.cloudtrail.console_login.additional_eventdata.mobile_version`*::
++
+--
+Identifies whether ConsoleLogin was from mobile version
+
+type: boolean
+
+--
+
+*`aws.cloudtrail.console_login.additional_eventdata.login_to`*::
++
+--
+URL for ConsoleLogin
+
+type: keyword
+
+--
+
+*`aws.cloudtrail.console_login.additional_eventdata.mfa_used`*::
++
+--
+Identifies whether multi factor authentication was used during ConsoleLogin
+
+type: boolean
+
+--
+
 [float]
 === cloudwatch
 

diff --git a/x-pack/filebeat/module/aws/cloudtrail/_meta/fields.yml b/x-pack/filebeat/module/aws/cloudtrail/_meta/fields.yml
@@ -51,6 +51,33 @@
           description: >-
             The name of the AWS service that made the request, such as
             Amazon EC2 Auto Scaling or AWS Elastic Beanstalk.
+        - name: session_issuer
+          type: group
+          description: >-
+            If the request was made with temporary security
+            credentials, an element that provides information about
+            how the credentials were obtained.
+          fields:
+            - name: type
+              type: keyword
+              description: >-
+                The source of the temporary security credentials, such
+                as Root, IAMUser, or Role.
+            - name: principal_id
+              type: keyword
+              description: >-
+                The internal ID of the entity that was used to get
+                credentials.
+            - name: arn
+              type: keyword
+              description: >-
+                The ARN of the source (account, IAM user, or role)
+                that was used to get temporary security credentials.
+            - name: account_id
+              type: keyword
+              description: >-
+                The account that owns the entity that was used to get
+                credentials.
     - name: error_code
       type: keyword
       description: >-
@@ -133,3 +160,26 @@
       description: >-
         Identifies the VPC endpoint in which requests were made from a
         VPC to another AWS service, such as Amazon S3.
+    - name: console_login
+      type: group
+      description: >-
+        Fields specific to ConsoleLogin events
+      fields:
+        - name: additional_eventdata
+          type: group
+          description: >
+            Additional Event Data for ConsoleLogin events
+          fields:
+            - name: mobile_version
+              type: boolean
+              description: >-
+                Identifies whether ConsoleLogin was from mobile version
+            - name: login_to
+              type: keyword
+              description: >-
+                URL for ConsoleLogin
+            - name: mfa_used
+              type: boolean
+              description: >-
+                Identifies whether multi factor authentication was
+                used during ConsoleLogin
diff --git a/x-pack/filebeat/module/aws/cloudtrail/ingest/pipeline.yml b/x-pack/filebeat/module/aws/cloudtrail/ingest/pipeline.yml
@@ -55,6 +55,27 @@ processors:
       field: "json.userIdentity.invokedBy"
       target_field: "aws.cloudtrail.user_identity.invoked_by"
       ignore_failure: true
+  - rename:
+      field: "json.userIdentity.sessionIssuer.type"
+      target_field: "aws.cloudtrail.user_identity.session_issuer.type"
+      ignore_failure: true
+# userIdentity.sessionIssuer.userName is only set with assumed roles.
+  - rename:
+      field: "json.userIdentity.sessionIssuer.userName"
+      target_field: "user.name"
+      ignore_failure: true
+  - rename:
+      field: "json.userIdentity.sessionIssuer.principalId"
+      target_field: "aws.cloudtrail.user_identity.session_issuer.principal_id"
+      ignore_failure: true
+  - rename:
+      field: "json.userIdentity.sessionIssuer.arn"
+      target_field: "aws.cloudtrail.user_identity.session_issuer.arn"
+      ignore_failure: true
+  - rename:
+      field: "json.userIdentity.sessionIssuer.accountId"
+      target_field: "aws.cloudtrail.user_identity.session_issuer.account_id"
+      ignore_failure: true
   - rename:
       field: "json.eventSource"
       target_field: "event.provider"
@@ -67,14 +88,20 @@ processors:
       field: "json.awsRegion"
       target_field: "cloud.region"
       ignore_failure: true
-  - geoip:
-      field: "json.sourceIPAddress"
-      target_field: "source.geo"
-      ignore_failure: true
   - rename:
       field: "json.sourceIPAddress"
       target_field: "source.address"
       ignore_failure: true
+  - grok:
+      field: source.address
+      ignore_failure: true
+      patterns:
+        - ^%{IP:source.ip}$
+  - geoip:
+      field: "source.ip"
+      target_field: "source.geo"
+      ignore_failure: true
+      ignore_missing: true
   - user_agent:
       field: "json.userAgent"
       target_field: "user_agent"
@@ -204,6 +231,38 @@ processors:
         if (ctx.json?.requestParameters.newUserName != null) {
           addRelatedUser(ctx, ctx.json.requestParameters.newUserName);
         }
+
+  - script:
+      lang: painless
+      ignore_failure: true
+      source: >-
+        if (ctx.json?.eventName != 'ConsoleLogin') {
+          return;
+        }
+        Map aed_map = new HashMap();
+        if (ctx.json?.additionalEventData?.MobileVersion != null) {
+          if (ctx.json.additionalEventData.MobileVersion == 'No') {
+            aed_map.put("mobile_version", false);
+          } else {
+            aed_map.put("mobile_version", true);
+          }
+        }
+        if (ctx.json?.additionalEventData?.LoginTo != null) {
+          aed_map.put("login_to", ctx.json.additionalEventData.LoginTo);
+        }
+        if (ctx.json?.additionalEventData?.MFAUsed != null) {
+          if (ctx.json.additionalEventData.MFAUsed == 'No') {
+            aed_map.put("mfa_used", false);
+          } else {
+            aed_map.put("mfa_used", true);
+          }
+        }
+        if (aed_map.size() > 0) {
+          Map cl_map = new HashMap();
+          cl_map.put("additional_eventdata", aed_map);
+          ctx.aws.cloudtrail.put("console_login", cl_map);
+        }
+
   - remove:
       field:
         - "json"

diff --git a/x-pack/filebeat/module/aws/cloudtrail/test/add-user-to-group-json.log-expected.json b/x-pack/filebeat/module/aws/cloudtrail/test/add-user-to-group-json.log-expected.json
@@ -26,6 +26,7 @@
         ],
         "service.type": "aws",
         "source.address": "127.0.0.1",
+        "source.ip": "127.0.0.1",
         "user.id": "EX_PRINCIPAL_ID",
         "user.name": "Alice",
         "user_agent.device.name": "Other",

diff --git a/x-pack/filebeat/module/aws/cloudtrail/test/assume-role-json.log-expected.json b/x-pack/filebeat/module/aws/cloudtrail/test/assume-role-json.log-expected.json
@@ -33,6 +33,7 @@
         "source.geo.location.lon": 106.5531,
         "source.geo.region_iso_code": "CN-CQ",
         "source.geo.region_name": "Chongqing",
+        "source.ip": "123.145.67.89",
         "user.id": "AROAIN5ATK5U7KEXAMPLE:JohnRole1",
         "user_agent.device.name": "Spider",
         "user_agent.name": "aws-cli",

diff --git a/x-pack/filebeat/module/aws/cloudtrail/test/change-password-json.log-expected.json b/x-pack/filebeat/module/aws/cloudtrail/test/change-password-json.log-expected.json
@@ -25,6 +25,7 @@
         "log.offset": 0,
         "service.type": "aws",
         "source.address": "127.0.0.1",
+        "source.ip": "127.0.0.1",
         "user.id": "0123456789012",
         "user.name": "Alice",
         "user_agent.device.name": "Spider",
@@ -56,6 +57,7 @@
         "log.offset": 720,
         "service.type": "aws",
         "source.address": "127.0.0.1",
+        "source.ip": "127.0.0.1",
         "user.id": "0123456789012",
         "user.name": "Alice",
         "user_agent.device.name": "Spider",

diff --git a/x-pack/filebeat/module/aws/cloudtrail/test/console-login-json.log b/x-pack/filebeat/module/aws/cloudtrail/test/console-login-json.log
@@ -1,2 +1,3 @@
 {"eventVersion":"1.05","userIdentity":{"type":"IAMUser","principalId":"AIDACKCEVSQ6C2EXAMPLE","arn":"arn:aws:iam::111122223333:user/JohnDoe","accountId":"111122223333","userName":"JohnDoe"},"eventTime":"2014-07-16T15:49:27Z","eventSource":"signin.amazonaws.com","eventName":"ConsoleLogin","awsRegion":"us-east-2","sourceIPAddress":"192.0.2.110","userAgent":"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:24.0) Gecko/20100101 Firefox/24.0","requestParameters":null,"responseElements":{"ConsoleLogin":"Success"},"additionalEventData":{"MobileVersion":"No","LoginTo":"https://console.aws.amazon.com/s3/","MFAUsed":"No"},"eventID":"3fcfb182-98f8-4744-bd45-10aEXAMPLE"}
 {"eventVersion":"1.05","userIdentity":{"type":"IAMUser","principalId":"AIDACKCEVSQ6C2EXAMPLE","arn":"arn:aws:iam::111122223333:user/JaneDoe","accountId":"111122223333","userName":"JaneDoe"},"eventTime":"2014-07-08T17:35:27Z","eventSource":"signin.amazonaws.com","eventName":"ConsoleLogin","awsRegion":"us-east-2","sourceIPAddress":"192.0.2.100","userAgent":"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:24.0) Gecko/20100101 Firefox/24.0","errorMessage":"Failed authentication","requestParameters":null,"responseElements":{"ConsoleLogin":"Failure"},"additionalEventData":{"MobileVersion":"No","LoginTo":"https://console.aws.amazon.com/sns","MFAUsed":"No"},"eventID":"11ea990b-4678-4bcd-8fbe-625EXAMPLE"}
+{"eventVersion":"1.05","userIdentity":{"type":"AssumedRole","principalId":"AROAIDPPEZS35WEXAMPLE:AssumedRoleSessionName","arn":"arn:aws:sts::123456789012:assumed-role/RoleToBeAssumed/MySessionName","accountId":"123456789012","accessKeyId":"AKIAIOSFODNN7EXAMPLE","sessionContext":{"attributes":{"mfaAuthenticated":"false","creationDate":"20131102T010628Z"}},"sessionIssuer":{"type":"Role","principalId":"AROAIDPPEZS35WEXAMPLE","arn":"arn:aws:iam::123456789012:role/RoleToBeAssumed","accountId":"123456789012","userName":"RoleToBeAssumed"}},"eventTime":"2014-07-08T17:35:27Z","eventSource":"signin.amazonaws.com","eventName":"ConsoleLogin","awsRegion":"us-east-2","sourceIPAddress":"192.0.2.100","userAgent":"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:24.0) Gecko/20100101 Firefox/24.0","errorMessage":"Failed authentication","requestParameters":null,"responseElements":{"ConsoleLogin":"Failure"},"additionalEventData":{"MobileVersion":"No","LoginTo":"https://console.aws.amazon.com/sns","MFAUsed":"No"},"eventID":"11ea990b-4678-4bcd-8fbe-625EXAMPLE"}
diff --git a/x-pack/filebeat/module/aws/cloudtrail/test/console-login-json.log-expected.json b/x-pack/filebeat/module/aws/cloudtrail/test/console-login-json.log-expected.json
@@ -2,6 +2,9 @@
     {
         "@timestamp": "2014-07-16T15:49:27.000Z",
         "aws.cloudtrail.additional_eventdata": "{LoginTo=https://console.aws.amazon.com/s3/, MobileVersion=No, MFAUsed=No}",
+        "aws.cloudtrail.console_login.additional_eventdata.login_to": "https://console.aws.amazon.com/s3/",
+        "aws.cloudtrail.console_login.additional_eventdata.mfa_used": false,
+        "aws.cloudtrail.console_login.additional_eventdata.mobile_version": false,
         "aws.cloudtrail.event_version": "1.05",
         "aws.cloudtrail.response_elements": "{ConsoleLogin=Success}",
         "aws.cloudtrail.user_identity.arn": "arn:aws:iam::111122223333:user/JohnDoe",
@@ -23,6 +26,7 @@
         "log.offset": 0,
         "service.type": "aws",
         "source.address": "192.0.2.110",
+        "source.ip": "192.0.2.110",
         "user.id": "AIDACKCEVSQ6C2EXAMPLE",
         "user.name": "JohnDoe",
         "user_agent.device.name": "Other",
@@ -36,6 +40,9 @@
     {
         "@timestamp": "2014-07-08T17:35:27.000Z",
         "aws.cloudtrail.additional_eventdata": "{LoginTo=https://console.aws.amazon.com/sns, MobileVersion=No, MFAUsed=No}",
+        "aws.cloudtrail.console_login.additional_eventdata.login_to": "https://console.aws.amazon.com/sns",
+        "aws.cloudtrail.console_login.additional_eventdata.mfa_used": false,
+        "aws.cloudtrail.console_login.additional_eventdata.mobile_version": false,
         "aws.cloudtrail.error_message": "Failed authentication",
         "aws.cloudtrail.event_version": "1.05",
         "aws.cloudtrail.response_elements": "{ConsoleLogin=Failure}",
@@ -58,6 +65,7 @@
         "log.offset": 658,
         "service.type": "aws",
         "source.address": "192.0.2.100",
+        "source.ip": "192.0.2.100",
         "user.id": "AIDACKCEVSQ6C2EXAMPLE",
         "user.name": "JaneDoe",
         "user_agent.device.name": "Other",
@@ -67,5 +75,50 @@
         "user_agent.os.name": "Windows",
         "user_agent.os.version": "7",
         "user_agent.version": "24.0."
+    },
+    {
+        "@timestamp": "2014-07-08T17:35:27.000Z",
+        "aws.cloudtrail.additional_eventdata": "{LoginTo=https://console.aws.amazon.com/sns, MobileVersion=No, MFAUsed=No}",
+        "aws.cloudtrail.console_login.additional_eventdata.login_to": "https://console.aws.amazon.com/sns",
+        "aws.cloudtrail.console_login.additional_eventdata.mfa_used": false,
+        "aws.cloudtrail.console_login.additional_eventdata.mobile_version": false,
+        "aws.cloudtrail.error_message": "Failed authentication",
+        "aws.cloudtrail.event_version": "1.05",
+        "aws.cloudtrail.response_elements": "{ConsoleLogin=Failure}",
+        "aws.cloudtrail.user_identity.access_key_id": "AKIAIOSFODNN7EXAMPLE",
+        "aws.cloudtrail.user_identity.arn": "arn:aws:sts::123456789012:assumed-role/RoleToBeAssumed/MySessionName",
+        "aws.cloudtrail.user_identity.session_context.mfa_authenticated": "false",
+        "aws.cloudtrail.user_identity.session_issuer.account_id": "123456789012",
+        "aws.cloudtrail.user_identity.session_issuer.arn": "arn:aws:iam::123456789012:role/RoleToBeAssumed",
+        "aws.cloudtrail.user_identity.session_issuer.principal_id": "AROAIDPPEZS35WEXAMPLE",
+        "aws.cloudtrail.user_identity.session_issuer.type": "Role",
+        "aws.cloudtrail.user_identity.type": "AssumedRole",
+        "cloud.account.id": "123456789012",
+        "cloud.region": "us-east-2",
+        "event.action": "ConsoleLogin",
+        "event.category": "authentication",
+        "event.dataset": "aws.cloudtrail",
+        "event.id": "11ea990b-4678-4bcd-8fbe-625EXAMPLE",
+        "event.kind": "event",
+        "event.module": "aws",
+        "event.original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"AssumedRole\",\"principalId\":\"AROAIDPPEZS35WEXAMPLE:AssumedRoleSessionName\",\"arn\":\"arn:aws:sts::123456789012:assumed-role/RoleToBeAssumed/MySessionName\",\"accountId\":\"123456789012\",\"accessKeyId\":\"AKIAIOSFODNN7EXAMPLE\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"false\",\"creationDate\":\"20131102T010628Z\"}},\"sessionIssuer\":{\"type\":\"Role\",\"principalId\":\"AROAIDPPEZS35WEXAMPLE\",\"arn\":\"arn:aws:iam::123456789012:role/RoleToBeAssumed\",\"accountId\":\"123456789012\",\"userName\":\"RoleToBeAssumed\"}},\"eventTime\":\"2014-07-08T17:35:27Z\",\"eventSource\":\"signin.amazonaws.com\",\"eventName\":\"ConsoleLogin\",\"awsRegion\":\"us-east-2\",\"sourceIPAddress\":\"192.0.2.100\",\"userAgent\":\"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:24.0) Gecko/20100101 Firefox/24.0\",\"errorMessage\":\"Failed authentication\",\"requestParameters\":null,\"responseElements\":{\"ConsoleLogin\":\"Failure\"},\"additionalEventData\":{\"MobileVersion\":\"No\",\"LoginTo\":\"https://console.aws.amazon.com/sns\",\"MFAUsed\":\"No\"},\"eventID\":\"11ea990b-4678-4bcd-8fbe-625EXAMPLE\"}",
+        "event.outcome": "failure",
+        "event.provider": "signin.amazonaws.com",
+        "event.type": "info",
+        "fileset.name": "cloudtrail",
+        "input.type": "log",
+        "log.offset": 1355,
+        "service.type": "aws",
+        "source.address": "192.0.2.100",
+        "source.ip": "192.0.2.100",
+        "user.id": "AROAIDPPEZS35WEXAMPLE:AssumedRoleSessionName",
+        "user.name": "RoleToBeAssumed",
+        "user_agent.device.name": "Other",
+        "user_agent.name": "Firefox",
+        "user_agent.original": "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:24.0) Gecko/20100101 Firefox/24.0",
+        "user_agent.os.full": "Windows 7",
+        "user_agent.os.name": "Windows",
+        "user_agent.os.version": "7",
+        "user_agent.version": "24.0."
     }
 ]
diff --git a/x-pack/filebeat/module/aws/cloudtrail/test/create-access-key-json.log-expected.json b/x-pack/filebeat/module/aws/cloudtrail/test/create-access-key-json.log-expected.json
@@ -31,6 +31,7 @@
         ],
         "service.type": "aws",
         "source.address": "127.0.0.1",
+        "source.ip": "127.0.0.1",
         "user.id": "EXAMPLE_ID",
         "user.name": "Alice",
         "user_agent.device.name": "Other",