Add field limit check for AWS Cloudtrail flattened fields (elastic#21388

) add 32k length check for - aws.cloudtrail.flattened.request_parameters - aws.cloudtrail.flattened.response_elements - aws.cloudtrail.flattened.additional_eventdata - aws.cloudtrail.flattened.service_event_details Closes elastic#21382
leehinman · Sep 30, 2020 · eae9f5c · eae9f5c
1 parent fd52ca5
commit eae9f5c
Show file tree

Hide file tree

Showing 2 changed files with 26 additions and 37 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -275,6 +275,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Handle multiple upstreams in ingress-controller. {pull}21215[21215]
 - Provide backwards compatibility for the `append` processor when Elasticsearch is less than 7.10.0. {pull}21159[21159]
 - Fix checkpoint module when logs contain time field. {pull}20567[20567]
+- Add field limit check for AWS Cloudtrail flattened fields. {pull}21388[21388] {issue}21382[21382]
 
 *Heartbeat*
 

diff --git a/x-pack/filebeat/module/aws/cloudtrail/ingest/pipeline.yml b/x-pack/filebeat/module/aws/cloudtrail/ingest/pipeline.yml
@@ -146,37 +146,36 @@ processors:
       field: "json.errorMessage"
       target_field: "aws.cloudtrail.error_message"
       ignore_failure: true
-  - rename:
-      field: json.requestParameters
-      target_field: "aws.cloudtrail.flattened.request_parameters"
-      if: ctx?.json?.requestParameters != null
   - script:
       lang: painless
       source: |
-        if (ctx.aws.cloudtrail.flattened.request_parameters != null) {
-          ctx.aws.cloudtrail.request_parameters = ctx.aws.cloudtrail.flattened.request_parameters.toString();
+        if (ctx.aws.cloudtrail?.flattened == null) {
+            Map map = new HashMap();
+            ctx.aws.cloudtrail.put("flattened", map);
+          }
+        if (ctx.json.requestParameters != null) {
+          ctx.aws.cloudtrail.request_parameters = ctx.json.requestParameters.toString();
+          if (ctx.aws.cloudtrail.request_parameters.length() < 32766) {
+            ctx.aws.cloudtrail.flattened.put("request_parameters", ctx.json.requestParameters);
+          }
         }
-      ignore_failure: true
-  - rename:
-      field: json.responseElements
-      target_field: "aws.cloudtrail.flattened.response_elements"
-      if: ctx?.json?.responseElements != null
-  - script:
-      lang: painless
-      source: |
-        if (ctx.aws.cloudtrail.flattened.response_elements != null) {
-          ctx.aws.cloudtrail.response_elements = ctx.aws.cloudtrail.flattened.response_elements.toString();
+        if (ctx.json.responseElements != null) {
+          ctx.aws.cloudtrail.response_elements = ctx.json.responseElements.toString();
+          if (ctx.aws.cloudtrail.response_elements.length() < 32766) {
+            ctx.aws.cloudtrail.flattened.put("response_elements", ctx.json.responseElements);
+          }
         }
-      ignore_failure: true
-  - rename:
-      field: json.additionalEventData
-      target_field: "aws.cloudtrail.flattened.additional_eventdata"
-      if: ctx?.json?.additionalEventData != null
-  - script:
-      lang: painless
-      source: |
-        if (ctx.aws.cloudtrail.flattened.additional_eventdata != null) {
-          ctx.aws.cloudtrail.additional_eventdata = ctx.aws.cloudtrail.flattened.additional_eventdata.toString();
+        if (ctx.json.additionalEventData != null) {
+          ctx.aws.cloudtrail.additional_eventdata = ctx.json.additionalEventData.toString();
+          if (ctx.aws.cloudtrail.additional_eventdata.length() < 32766) {
+            ctx.aws.cloudtrail.flattened.put("additional_eventdata", ctx.json.additionalEventData);
+          }
+        }
+        if (ctx.json.serviceEventDetails != null) {
+          ctx.aws.cloudtrail.service_event_details = ctx.json.serviceEventDetails.toString();
+          if (ctx.aws.cloudtrail.service_event_details.length() < 32766) {
+            ctx.aws.cloudtrail.flattened.put("service_event_details", ctx.json.serviceEventDetails);
+          }
         }
       ignore_failure: true
   - rename:
@@ -219,17 +218,6 @@ processors:
       field: "json.recipientAccountId"
       target_field: "aws.cloudtrail.recipient_account_id"
       ignore_failure: true
-  - rename:
-      field: json.serviceEventDetails
-      target_field: "aws.cloudtrail.flattened.service_event_details"
-      if: ctx?.json?.serviceEventDetails != null
-  - script:
-      lang: painless
-      source: |
-        if (ctx.aws.cloudtrail.flattened.service_event_details != null) {
-          ctx.aws.cloudtrail.service_event_details = ctx.aws.cloudtrail.flattened.service_event_details.toString();
-        }
-      ignore_failure: true
   - rename:
       field: "json.sharedEventId"
       target_field: "aws.cloudtrail.shared_event_id"