From 6a7c8641e81f8ad491a2ae6e6bfdd135d34708f0 Mon Sep 17 00:00:00 2001 From: "Lee E. Hinman" Date: Mon, 14 Sep 2020 08:46:53 -0500 Subject: [PATCH 1/3] Add support for Cloudtrail digest files - allow file matching with file_selectors in s3 input - update cloudtrail pipeline - update cloudtrail config to use file_selectors - add cloudtrail digest fields Closes #20943 --- filebeat/docs/fields.asciidoc | 105 +++++++++++++++ .../docs/inputs/input-aws-s3.asciidoc | 22 +++ x-pack/filebeat/filebeat.reference.yml | 12 ++ x-pack/filebeat/input/s3/config.go | 16 +++ x-pack/filebeat/input/s3/input.go | 70 ++++++---- x-pack/filebeat/input/s3/input_test.go | 5 +- x-pack/filebeat/module/aws/_meta/config.yml | 12 ++ .../module/aws/cloudtrail/_meta/fields.yml | 60 +++++++++ .../module/aws/cloudtrail/config/s3.yml | 15 ++- .../module/aws/cloudtrail/ingest/pipeline.yml | 93 ++++++++++++- .../module/aws/cloudtrail/manifest.yml | 6 + .../test/cloudtrail-digest-json.log | 1 + .../cloudtrail-digest-json.log-expected.json | 126 ++++++++++++++++++ .../aws/cloudtrail/test/insight-json.log | 1 + .../test/insight-json.log-expected.json | 24 ++++ x-pack/filebeat/module/aws/fields.go | 2 +- x-pack/filebeat/modules.d/aws.yml.disabled | 12 ++ 17 files changed, 546 insertions(+), 36 deletions(-) create mode 100644 x-pack/filebeat/module/aws/cloudtrail/test/cloudtrail-digest-json.log create mode 100644 x-pack/filebeat/module/aws/cloudtrail/test/cloudtrail-digest-json.log-expected.json create mode 100644 x-pack/filebeat/module/aws/cloudtrail/test/insight-json.log create mode 100644 x-pack/filebeat/module/aws/cloudtrail/test/insight-json.log-expected.json diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index 67e84f08f35..8d7784c5e9e 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -1487,6 +1487,111 @@ type: flattened -- +[float] +=== digest + +Fields from Cloudtrail Digest Logs + + +*`aws.cloudtrail.digest.log_files`*:: ++ +-- +A list of Logfiles contained in the digest + +type: nested + +-- + +*`aws.cloudtrail.digest.start_time`*:: ++ +-- +The starting UTC time range that the digest file covers, taking as a reference the time in which log files have been delivered by CloudTrail. + +type: date + +-- + +*`aws.cloudtrail.digest.end_time`*:: ++ +-- +The ending UTC time range that the digest file covers, taking as a reference the time in which log files have been delivered by CloudTrail. + +type: date + +-- + +*`aws.cloudtrail.digest.s3_bucket`*:: ++ +-- +The name of the Amazon S3 bucket to which the current digest file has been delivered. + +type: keyword + +-- + +*`aws.cloudtrail.digest.s3_object`*:: ++ +-- +The Amazon S3 object key (that is, the Amazon S3 bucket location) of the current digest file. + +type: keyword + +-- + +*`aws.cloudtrail.digest.newest_event_time`*:: ++ +-- +The UTC time of the most recent event among all of the events in the log files in the digest. + +type: date + +-- + +*`aws.cloudtrail.digest.oldest_event_time`*:: ++ +-- +The UTC time of the oldest event among all of the events in the log files in the digest. + +type: date + +-- + +*`aws.cloudtrail.digest.previous_s3_bucket`*:: ++ +-- +The Amazon S3 bucket to which the previous digest file was delivered. + +type: keyword + +-- + +*`aws.cloudtrail.digest.previous_hash_algorithm`*:: ++ +-- +The name of the hash algorithm that was used to hash the previous digest file. + +type: keyword + +-- + +*`aws.cloudtrail.digest.public_key_fingerprint`*:: ++ +-- +The hexadecimal encoded fingerprint of the public key that matches the private key used to sign this digest file. + +type: keyword + +-- + +*`aws.cloudtrail.digest.signature_algorithm`*:: ++ +-- +The algorithm used to sign the digest file. + +type: keyword + +-- + [float] === cloudwatch diff --git a/x-pack/filebeat/docs/inputs/input-aws-s3.asciidoc b/x-pack/filebeat/docs/inputs/input-aws-s3.asciidoc index 1024656f082..9b8731c52c8 100644 --- a/x-pack/filebeat/docs/inputs/input-aws-s3.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-aws-s3.asciidoc @@ -67,6 +67,28 @@ type will not be checked. If a file has "application/json" content-type, `expand_event_list_from_field` becomes required to read the json file. +[float] +==== `file_selectors` + +If the SQS queue will have events that correspond to files that +{beatname_uc} shouldn't process `file_selectors` can be used to limit +the files that are downloaded. This is a list of selectors which are +made up of `regex` and `expand_event_list_from_field` options. The +`regex` should match the S3 object key in the SQS message, and the +optional `expand_event_list_from_field` is the same as the global +setting. Regex syntax is the same as the Go language. Files that +don't match one of the regexes won't be processed. + +["source", "yml"] +---- +file_selectors: + - regex: '^AWSLogs/\d+/CloudTrail/' + expand_event_list_from_field: 'Records' + - regex: '^AWSLogs/\d+/CloudTrail-Digest' +``` +---- + + [float] ==== `api_timeout` diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index d58cc94ae4d..df27a2ca93b 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -102,6 +102,18 @@ filebeat.modules: # AWS SQS queue url #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue + # Process CloudTrail logs + # default is true, set to false to skip Cloudtrail logs + # var.process_cloudtrail_logs: false + + # Process CloudTrail Digest logs + # default true, set to false to skip CloudTrail Digest logs + # var.process_digest_logs: false + + # Process CloudTrail Insight logs + # default true, set to false to skip CloudTrail Insight logs + # var.process_insight_logs: false + # Filename of AWS credential file # If not set "$HOME/.aws/credentials" is used on Linux/Mac # "%UserProfile%\.aws\credentials" is used on Windows diff --git a/x-pack/filebeat/input/s3/config.go b/x-pack/filebeat/input/s3/config.go index 72960ad9ade..f9780d82277 100644 --- a/x-pack/filebeat/input/s3/config.go +++ b/x-pack/filebeat/input/s3/config.go @@ -6,6 +6,7 @@ package s3 import ( "fmt" + "regexp" "time" "github.com/elastic/beats/v7/filebeat/harvester" @@ -19,6 +20,14 @@ type config struct { AwsConfig awscommon.ConfigAWS `config:",inline"` ExpandEventListFromField string `config:"expand_event_list_from_field"` APITimeout time.Duration `config:"api_timeout"` + FileSelectors []FileSelectorCfg `config:"file_selectors"` +} + +// FileSelectorCfg defines type and configuration of FileSelectors +type FileSelectorCfg struct { + RegexString string `config:"regex"` + Regex *regexp.Regexp `config:",ignore"` + ExpandEventListFromField string `config:"expand_event_list_from_field"` } func defaultConfig() config { @@ -40,5 +49,12 @@ func (c *config) Validate() error { return fmt.Errorf("api timeout %v needs to be larger than"+ " 0s and smaller than half of the visibility timeout", c.APITimeout) } + for i := range c.FileSelectors { + r, err := regexp.Compile(c.FileSelectors[i].RegexString) + if err != nil { + return err + } + c.FileSelectors[i].Regex = r + } return nil } diff --git a/x-pack/filebeat/input/s3/input.go b/x-pack/filebeat/input/s3/input.go index 65984dace45..1085c9dccbd 100644 --- a/x-pack/filebeat/input/s3/input.go +++ b/x-pack/filebeat/input/s3/input.go @@ -74,10 +74,11 @@ type s3Input struct { } type s3Info struct { - name string - key string - region string - arn string + name string + key string + region string + arn string + expandEventListFromField string } type bucket struct { @@ -252,7 +253,7 @@ func (p *s3Input) processor(queueURL string, messages []sqs.Message, visibilityT func (p *s3Input) processMessage(svcS3 s3iface.ClientAPI, message sqs.Message, wg *sync.WaitGroup, errC chan error) { defer wg.Done() - s3Infos, err := handleSQSMessage(message) + s3Infos, err := p.handleSQSMessage(message) if err != nil { p.logger.Error(errors.Wrap(err, "handleSQSMessage failed")) return @@ -352,7 +353,7 @@ func getRegionFromQueueURL(queueURL string) (string, error) { } // handle message -func handleSQSMessage(m sqs.Message) ([]s3Info, error) { +func (p *s3Input) handleSQSMessage(m sqs.Message) ([]s3Info, error) { msg := sqsMessage{} err := json.Unmarshal([]byte(*m.Body), &msg) if err != nil { @@ -361,21 +362,40 @@ func handleSQSMessage(m sqs.Message) ([]s3Info, error) { var s3Infos []s3Info for _, record := range msg.Records { - if record.EventSource == "aws:s3" && strings.HasPrefix(record.EventName, "ObjectCreated:") { - // Unescape substrings from s3 log name. For example, convert "%3D" back to "=" - filename, err := url.QueryUnescape(record.S3.object.Key) - if err != nil { - return nil, errors.Wrapf(err, "url.QueryUnescape failed for '%s'", record.S3.object.Key) - } + if record.EventSource != "aws:s3" || !strings.HasPrefix(record.EventName, "ObjectCreated:") { + return nil, errors.New("this SQS queue should be dedicated to s3 ObjectCreated event notifications") + } + // Unescape substrings from s3 log name. For example, convert "%3D" back to "=" + filename, err := url.QueryUnescape(record.S3.object.Key) + if err != nil { + return nil, errors.Wrapf(err, "url.QueryUnescape failed for '%s'", record.S3.object.Key) + } + if len(p.config.FileSelectors) == 0 { s3Infos = append(s3Infos, s3Info{ - region: record.AwsRegion, - name: record.S3.bucket.Name, - key: filename, - arn: record.S3.bucket.Arn, + region: record.AwsRegion, + name: record.S3.bucket.Name, + key: filename, + arn: record.S3.bucket.Arn, + expandEventListFromField: p.config.ExpandEventListFromField, }) - } else { - return nil, errors.New("this SQS queue should be dedicated to s3 ObjectCreated event notifications") + continue + } + + for _, fs := range p.config.FileSelectors { + if fs.Regex == nil { + continue + } + if fs.Regex.MatchString(filename) { + s3Infos = append(s3Infos, s3Info{ + region: record.AwsRegion, + name: record.S3.bucket.Name, + key: filename, + arn: record.S3.bucket.Arn, + expandEventListFromField: fs.ExpandEventListFromField, + }) + break + } } } return s3Infos, nil @@ -456,7 +476,7 @@ func (p *s3Input) createEventsFromS3Info(svc s3iface.ClientAPI, info s3Info, s3C } // Decode JSON documents when content-type is "application/json" or expand_event_list_from_field is given in config - if resp.ContentType != nil && *resp.ContentType == "application/json" || p.config.ExpandEventListFromField != "" { + if resp.ContentType != nil && *resp.ContentType == "application/json" || info.expandEventListFromField != "" { decoder := json.NewDecoder(reader) err := p.decodeJSON(decoder, objectHash, info, s3Ctx) if err != nil { @@ -537,10 +557,10 @@ func (p *s3Input) decodeJSON(decoder *json.Decoder, objectHash string, s3Info s3 func (p *s3Input) jsonFieldsType(jsonFields interface{}, offset int, objectHash string, s3Info s3Info, s3Ctx *s3Context) (int, error) { switch f := jsonFields.(type) { case map[string][]interface{}: - if p.config.ExpandEventListFromField != "" { - textValues, ok := f[p.config.ExpandEventListFromField] + if s3Info.expandEventListFromField != "" { + textValues, ok := f[s3Info.expandEventListFromField] if !ok { - err := errors.Errorf("key '%s' not found", p.config.ExpandEventListFromField) + err := errors.Errorf("key '%s' not found", s3Info.expandEventListFromField) p.logger.Error(err) return offset, err } @@ -555,10 +575,10 @@ func (p *s3Input) jsonFieldsType(jsonFields interface{}, offset int, objectHash return offset, nil } case map[string]interface{}: - if p.config.ExpandEventListFromField != "" { - textValues, ok := f[p.config.ExpandEventListFromField] + if s3Info.expandEventListFromField != "" { + textValues, ok := f[s3Info.expandEventListFromField] if !ok { - err := errors.Errorf("key '%s' not found", p.config.ExpandEventListFromField) + err := errors.Errorf("key '%s' not found", s3Info.expandEventListFromField) p.logger.Error(err) return offset, err } diff --git a/x-pack/filebeat/input/s3/input_test.go b/x-pack/filebeat/input/s3/input_test.go index c9ff86780de..d1fab05cb3c 100644 --- a/x-pack/filebeat/input/s3/input_test.go +++ b/x-pack/filebeat/input/s3/input_test.go @@ -120,9 +120,10 @@ func TestHandleMessage(t *testing.T) { }, } + p := &s3Input{context: &channelContext{}} for _, c := range casesPositive { t.Run(c.title, func(t *testing.T) { - s3Info, err := handleSQSMessage(c.message) + s3Info, err := p.handleSQSMessage(c.message) assert.NoError(t, err) assert.Equal(t, len(c.expectedS3Infos), len(s3Info)) if len(s3Info) > 0 { @@ -155,7 +156,7 @@ func TestHandleMessage(t *testing.T) { for _, c := range casesNegative { t.Run(c.title, func(t *testing.T) { - s3Info, err := handleSQSMessage(c.message) + s3Info, err := p.handleSQSMessage(c.message) assert.Error(t, err) assert.Nil(t, s3Info) }) diff --git a/x-pack/filebeat/module/aws/_meta/config.yml b/x-pack/filebeat/module/aws/_meta/config.yml index 7c9c9f0e7b0..b7e0c25b674 100644 --- a/x-pack/filebeat/module/aws/_meta/config.yml +++ b/x-pack/filebeat/module/aws/_meta/config.yml @@ -5,6 +5,18 @@ # AWS SQS queue url #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue + # Process CloudTrail logs + # default is true, set to false to skip Cloudtrail logs + # var.process_cloudtrail_logs: false + + # Process CloudTrail Digest logs + # default true, set to false to skip CloudTrail Digest logs + # var.process_digest_logs: false + + # Process CloudTrail Insight logs + # default true, set to false to skip CloudTrail Insight logs + # var.process_insight_logs: false + # Filename of AWS credential file # If not set "$HOME/.aws/credentials" is used on Linux/Mac # "%UserProfile%\.aws\credentials" is used on Windows diff --git a/x-pack/filebeat/module/aws/cloudtrail/_meta/fields.yml b/x-pack/filebeat/module/aws/cloudtrail/_meta/fields.yml index d4bd04b548b..29fff28719e 100644 --- a/x-pack/filebeat/module/aws/cloudtrail/_meta/fields.yml +++ b/x-pack/filebeat/module/aws/cloudtrail/_meta/fields.yml @@ -223,3 +223,63 @@ description: >- Identifies the service event, including what triggered the event and the result. + - name: digest + type: group + description: >- + Fields from Cloudtrail Digest Logs + fields: + - name: log_files + type: nested + description: >- + A list of Logfiles contained in the digest + - name: start_time + type: date + description: >- + The starting UTC time range that the digest file covers, + taking as a reference the time in which log files have + been delivered by CloudTrail. + - name: end_time + type: date + description: >- + The ending UTC time range that the digest file covers, + taking as a reference the time in which log files have + been delivered by CloudTrail. + - name: s3_bucket + type: keyword + description: >- + The name of the Amazon S3 bucket to which the current + digest file has been delivered. + - name: s3_object + type: keyword + description: >- + The Amazon S3 object key (that is, the Amazon S3 bucket + location) of the current digest file. + - name: newest_event_time + type: date + description: >- + The UTC time of the most recent event among all of the + events in the log files in the digest. + - name: oldest_event_time + type: date + description: >- + The UTC time of the oldest event among all of the events + in the log files in the digest. + - name: previous_s3_bucket + type: keyword + description: >- + The Amazon S3 bucket to which the previous digest file was + delivered. + - name: previous_hash_algorithm + type: keyword + description: >- + The name of the hash algorithm that was used to hash the + previous digest file. + - name: public_key_fingerprint + type: keyword + description: >- + The hexadecimal encoded fingerprint of the public key that + matches the private key used to sign this digest file. + - name: signature_algorithm + type: keyword + description: >- + The algorithm used to sign the digest file. diff --git a/x-pack/filebeat/module/aws/cloudtrail/config/s3.yml b/x-pack/filebeat/module/aws/cloudtrail/config/s3.yml index 2a6f38d1fad..2fd6b7f1a00 100644 --- a/x-pack/filebeat/module/aws/cloudtrail/config/s3.yml +++ b/x-pack/filebeat/module/aws/cloudtrail/config/s3.yml @@ -1,6 +1,19 @@ type: s3 queue_url: {{ .queue_url }} -expand_event_list_from_field: Records +file_selectors: +{{ if .process_cloudtrail_logs }} + - regex: '^AWSLogs/\d+/CloudTrail/' + expand_event_list_from_field: 'Records' +{{ end }} + +{{ if .process_digest_logs }} + - regex: '^AWSLogs/\d+/CloudTrail-Digest/' +{{ end }} + +{{ if .process_insight_logs } + - regex: '^AWSLogs/\d+/CloudTrail-Insight/' + expand_event_list_from_field: 'Records' +{{ end }} {{ if .credential_profile_name }} credential_profile_name: {{ .credential_profile_name }} diff --git a/x-pack/filebeat/module/aws/cloudtrail/ingest/pipeline.yml b/x-pack/filebeat/module/aws/cloudtrail/ingest/pipeline.yml index e22c986be6f..cf976147d75 100644 --- a/x-pack/filebeat/module/aws/cloudtrail/ingest/pipeline.yml +++ b/x-pack/filebeat/module/aws/cloudtrail/ingest/pipeline.yml @@ -4,6 +4,9 @@ processors: - set: field: event.ingested value: '{{_ingest.timestamp}}' + - set: + field: event.created + value: '{{@timestamp}}' - rename: field: "message" target_field: "event.original" @@ -58,7 +61,7 @@ processors: field: "json.userIdentity.sessionContext.sessionIssuer.type" target_field: "aws.cloudtrail.user_identity.session_context.session_issuer.type" ignore_failure: true -# userIdentity.sessionIssuer.userName is only set with assumed roles. + # userIdentity.sessionIssuer.userName is only set with assumed roles. - rename: field: "json.userIdentity.sessionContext.sessionIssuer.userName" target_field: "user.name" @@ -88,6 +91,11 @@ processors: value: "{{json.eventName}}" ignore_failure: true ignore_empty_value: true + - set: + field: "event.action" + value: "{{json.eventCategory}}" + ignore_failure: true + ignore_empty_value: true - rename: field: "json.awsRegion" target_field: "cloud.region" @@ -119,9 +127,9 @@ processors: target_field: source.as.number ignore_missing: true - rename: - field: source.as.organization_name - target_field: source.as.organization.name - ignore_missing: true + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true - user_agent: field: "json.userAgent" target_field: "user_agent" @@ -141,7 +149,7 @@ processors: - rename: field: json.requestParameters target_field: "aws.cloudtrail.flattened.request_parameters" - if: ctx.json.requestParameters != null + if: ctx?.json?.requestParameters != null - script: lang: painless source: | @@ -152,7 +160,7 @@ processors: - rename: field: json.responseElements target_field: "aws.cloudtrail.flattened.response_elements" - if: ctx.json.responseElements != null + if: ctx?.json?.responseElements != null - script: lang: painless source: | @@ -214,7 +222,7 @@ processors: - rename: field: json.serviceEventDetails target_field: "aws.cloudtrail.flattened.service_event_details" - if: ctx.json.serviceEventDetails != null + if: ctx?.json?.serviceEventDetails != null - script: lang: painless source: | @@ -615,6 +623,77 @@ processors: def hm = new HashMap(params.get(ctx.event.action)); hm.forEach((k, v) -> ctx.event[k] = v); + - rename: + field: "json.awsAccountId" + target_field: "cloud.account.id" + ignore_failure: true + - rename: + field: "json.previousDigestS3Object" + target_field: "file.path" + ignore_failure: true + - rename: + field: "json.previousDigestSignature" + target_field: "file.hash.sha256" + if: >- + ctx?.json?.previousDigestHashAlgorithm != null && ctx.json.previousDigestHashAlgorithm == 'SHA-256' + - append: + field: "related.hash" + value: "{{json.previousDigestSignature}}" + if: "ctx?.json?.previousDigestSignature != null" + - rename: + field: "json.logFiles" + target_field: "aws.cloudtrail.digest.log_files" + ignore_failure: true + - date: + field: "json.digestStartTime" + target_field: "aws.cloudtrail.digest.start_time" + ignore_failure: true + formats: + - ISO8601 + - date: + field: "json.digestEndTime" + target_field: "@timestamp" + ignore_failure: true + formats: + - ISO8601 + - date: + field: "json.digestEndTime" + target_field: "aws.cloudtrail.digest.end_time" + ignore_failure: true + formats: + - ISO8601 + - rename: + field: "json.digestS3Bucket" + target_field: "aws.cloudtrail.digest.s3_bucket" + ignore_failure: true + - date: + field: "json.newestEventTime" + target_field: "aws.cloudtrail.digest.newest_event_time" + ignore_failure: true + formats: + - ISO8601 + - date: + field: "json.oldestEventTime" + target_field: "aws.cloudtrail.digest.oldest_event_time" + ignore_failure: true + formats: + - ISO8601 + - rename: + field: "json.previousDigestS3Bucket" + target_field: "aws.cloudtrail.digest.previous_s3_bucket" + ignore_failure: true + - rename: + field: "json.previousDigestHashAlgorithm" + target_field: "aws.cloudtrail.digest.previous_hash_algorithm" + ignore_failure: true + - rename: + field: "json.publicKeyFingerprint" + target_field: "aws.cloudtrail.digest.public_key_fingerprint" + ignore_failure: true + - rename: + field: "json.digestSignatureAlgorithm" + target_field: "aws.cloudtrail.digest.signature_algorithm" + ignore_failure: true - remove: field: - "json" diff --git a/x-pack/filebeat/module/aws/cloudtrail/manifest.yml b/x-pack/filebeat/module/aws/cloudtrail/manifest.yml index 2878c79936d..732967ff0b0 100644 --- a/x-pack/filebeat/module/aws/cloudtrail/manifest.yml +++ b/x-pack/filebeat/module/aws/cloudtrail/manifest.yml @@ -15,6 +15,12 @@ var: - name: role_arn - name: tags default: [forwarded] + - name: process_cloudtrail_logs + default: true + - name: process_digest_logs + default: true + - name: process_insight_logs + default: true ingest_pipeline: ingest/pipeline.yml input: config/{{.input}}.yml diff --git a/x-pack/filebeat/module/aws/cloudtrail/test/cloudtrail-digest-json.log b/x-pack/filebeat/module/aws/cloudtrail/test/cloudtrail-digest-json.log new file mode 100644 index 00000000000..54182991959 --- /dev/null +++ b/x-pack/filebeat/module/aws/cloudtrail/test/cloudtrail-digest-json.log @@ -0,0 +1 @@ +{"awsAccountId":"144492464627","digestStartTime":"2020-09-11T18:36:49Z","digestEndTime":"2020-09-11T19:36:49Z","digestS3Bucket":"leh-ct-test-bucket","digestS3Object":"AWSLogs/144492464627/CloudTrail-Digest/us-west-2/2020/09/11/144492464627_CloudTrail-Digest_us-west-2_leh-ct-test_us-west-2_20200911T193649Z.json.gz","digestPublicKeyFingerprint":"4a7aa197feec25e9bd0b5e98fcad8ecb","digestSignatureAlgorithm":"SHA256withRSA","newestEventTime":"2020-09-11T19:26:24Z","oldestEventTime":"2020-09-11T18:32:04Z","previousDigestS3Bucket":"leh-ct-test-bucket","previousDigestS3Object":"AWSLogs/144492464627/CloudTrail-Digest/us-west-2/2020/09/11/144492464627_CloudTrail-Digest_us-west-2_leh-ct-test_us-west-2_20200911T183649Z.json.gz","previousDigestHashValue":"531914fcfa0dbacf09cdd4175a1fdcb5eda6e852911209f3c3ecb0a39036c860","previousDigestHashAlgorithm":"SHA-256","previousDigestSignature":"10e8072f3f2a1d2990dcc98e9d44c88a6ae2ada9df9c3ae6d35dfe8d5c47caf807702f1e1eec47edfeecfcc2248388f78fddfc945ae587fb4a3e7676b5547f342fa872b2a1c5baa62b33bcb528ce7c10e3748aab8f82c16c024de79d8a20128b4c9e5ec65e98a9c4e613687ecc89c224a11bb3df06ce414ff740e4acf9bd41159e77f568355011828412f1193e357866fbd0436faffb56e149af196e35a7675c3bbd0a398f43159343c35f9129d6339a2818afdb3192f4fffea9bd2d1bb0a70e5bfae1291f2133aab0ad92522aea6d0f828c1780d3f3de6b8270ab3ba42459916b0fbb8e2fba6ff9677bafe730e60f5edcc0f1580cdb4e36f3e3b03e653e9798","logFiles":[{"s3Bucket":"leh-ct-test-bucket","s3Object":"AWSLogs/144492464627/CloudTrail/us-west-2/2020/09/11/144492464627_CloudTrail_us-west-2_20200911T1930Z_l2pGqVS53QcGdAkp.json.gz","hashValue":"24078a45bb1c2e9ca444251ee8c1357644fdaebf4fee02d22250d8b6d48413c9","hashAlgorithm":"SHA-256","newestEventTime":"2020-09-11T19:26:24Z","oldestEventTime":"2020-09-11T19:26:24Z"},{"s3Bucket":"leh-ct-test-bucket","s3Object":"AWSLogs/144492464627/CloudTrail/us-west-2/2020/09/11/144492464627_CloudTrail_us-west-2_20200911T1915Z_TIKlbLnJ6IwUxqxw.json.gz","hashValue":"41eeb2ab841d03c2bb16e4549fc8fe3ac304ed7430173a91b37c878c77336169","hashAlgorithm":"SHA-256","newestEventTime":"2020-09-11T19:11:18Z","oldestEventTime":"2020-09-11T19:11:18Z"},{"s3Bucket":"leh-ct-test-bucket","s3Object":"AWSLogs/144492464627/CloudTrail/us-west-2/2020/09/11/144492464627_CloudTrail_us-west-2_20200911T1835Z_OPJhVNodH1gY760s.json.gz","hashValue":"6295aeb3b41cf021fe67e0b36f5ac51e557c41c85af6eef282d77ef056210d70","hashAlgorithm":"SHA-256","newestEventTime":"2020-09-11T18:32:04Z","oldestEventTime":"2020-09-11T18:32:04Z"},{"s3Bucket":"leh-ct-test-bucket","s3Object":"AWSLogs/144492464627/CloudTrail/us-west-2/2020/09/11/144492464627_CloudTrail_us-west-2_20200911T1925Z_zJNGzQovyNAImZV9.json.gz","hashValue":"54a29065f5cbfc192584e9245f8d3d86dfabf5714a5a5ec7dd2a0f4312b86198","hashAlgorithm":"SHA-256","newestEventTime":"2020-09-11T19:21:28Z","oldestEventTime":"2020-09-11T19:21:28Z"},{"s3Bucket":"leh-ct-test-bucket","s3Object":"AWSLogs/144492464627/CloudTrail/us-west-2/2020/09/11/144492464627_CloudTrail_us-west-2_20200911T1855Z_RqN9YzoKAJCKbejj.json.gz","hashValue":"551c8cbe75d081526b64fc799c67007652f2502d2f95bb9d5c8699ffc5ba7079","hashAlgorithm":"SHA-256","newestEventTime":"2020-09-11T18:51:21Z","oldestEventTime":"2020-09-11T18:51:21Z"},{"s3Bucket":"leh-ct-test-bucket","s3Object":"AWSLogs/144492464627/CloudTrail/us-west-2/2020/09/11/144492464627_CloudTrail_us-west-2_20200911T1850Z_jLldN7U8XrspES8p.json.gz","hashValue":"81605414e790e84dff20da66235f07134f77bb5c4863729bafe7672a98f07cb0","hashAlgorithm":"SHA-256","newestEventTime":"2020-09-11T18:46:45Z","oldestEventTime":"2020-09-11T18:46:45Z"},{"s3Bucket":"leh-ct-test-bucket","s3Object":"AWSLogs/144492464627/CloudTrail/us-west-2/2020/09/11/144492464627_CloudTrail_us-west-2_20200911T1905Z_jBNdmg4bSGxZ3wC8.json.gz","hashValue":"45050ec665636f9185f5b5a1e43c745a8282cb2e504092a452f0a4dc1fb8a6d5","hashAlgorithm":"SHA-256","newestEventTime":"2020-09-11T19:01:06Z","oldestEventTime":"2020-09-11T19:01:06Z"},{"s3Bucket":"leh-ct-test-bucket","s3Object":"AWSLogs/144492464627/CloudTrail/us-west-2/2020/09/11/144492464627_CloudTrail_us-west-2_20200911T1920Z_bj5DRrmILF6jK23a.json.gz","hashValue":"60ed8fcbd172d3f6d1ca4fa872681f429b005ed8a81fc9450aa06db92ccabd47","hashAlgorithm":"SHA-256","newestEventTime":"2020-09-11T19:16:12Z","oldestEventTime":"2020-09-11T19:16:12Z"},{"s3Bucket":"leh-ct-test-bucket","s3Object":"AWSLogs/144492464627/CloudTrail/us-west-2/2020/09/11/144492464627_CloudTrail_us-west-2_20200911T1900Z_6LjrkrhsLQMzCiSN.json.gz","hashValue":"b2b0e2804d1c6b92d76eee203d7eba32d30d03e696f7175723a83cec2d7ad4ba","hashAlgorithm":"SHA-256","newestEventTime":"2020-09-11T18:56:05Z","oldestEventTime":"2020-09-11T18:56:05Z"},{"s3Bucket":"leh-ct-test-bucket","s3Object":"AWSLogs/144492464627/CloudTrail/us-west-2/2020/09/11/144492464627_CloudTrail_us-west-2_20200911T1910Z_DLyqye8LaeoD204N.json.gz","hashValue":"4397a13565a67d9e6de577379b8eb7e61ca5b2b191c9b5da0243136dfc5581c7","hashAlgorithm":"SHA-256","newestEventTime":"2020-09-11T19:06:31Z","oldestEventTime":"2020-09-11T19:06:31Z"},{"s3Bucket":"leh-ct-test-bucket","s3Object":"AWSLogs/144492464627/CloudTrail/us-west-2/2020/09/11/144492464627_CloudTrail_us-west-2_20200911T1845Z_TSDKyASOn2ejOq5n.json.gz","hashValue":"49f09d2389632c70bc0066ed5d65768632dd2e60ed9c80af9d0c2c5f59bd60b6","hashAlgorithm":"SHA-256","newestEventTime":"2020-09-11T18:41:58Z","oldestEventTime":"2020-09-11T18:41:58Z"},{"s3Bucket":"leh-ct-test-bucket","s3Object":"AWSLogs/144492464627/CloudTrail/us-west-2/2020/09/11/144492464627_CloudTrail_us-west-2_20200911T1840Z_btJydJ2t7hCRnjsN.json.gz","hashValue":"0944f9a05d70868bc6f6048d58fd00764ab65e123b8ffefb193b22ca4394d771","hashAlgorithm":"SHA-256","newestEventTime":"2020-09-11T18:37:10Z","oldestEventTime":"2020-09-11T18:37:10Z"}]} diff --git a/x-pack/filebeat/module/aws/cloudtrail/test/cloudtrail-digest-json.log-expected.json b/x-pack/filebeat/module/aws/cloudtrail/test/cloudtrail-digest-json.log-expected.json new file mode 100644 index 00000000000..9a628cb3488 --- /dev/null +++ b/x-pack/filebeat/module/aws/cloudtrail/test/cloudtrail-digest-json.log-expected.json @@ -0,0 +1,126 @@ +[ + { + "@timestamp": "2020-09-11T19:36:49.000Z", + "aws.cloudtrail.digest.end_time": "2020-09-11T19:36:49.000Z", + "aws.cloudtrail.digest.log_files": [ + { + "hashAlgorithm": "SHA-256", + "hashValue": "24078a45bb1c2e9ca444251ee8c1357644fdaebf4fee02d22250d8b6d48413c9", + "newestEventTime": "2020-09-11T19:26:24Z", + "oldestEventTime": "2020-09-11T19:26:24Z", + "s3Bucket": "leh-ct-test-bucket", + "s3Object": "AWSLogs/144492464627/CloudTrail/us-west-2/2020/09/11/144492464627_CloudTrail_us-west-2_20200911T1930Z_l2pGqVS53QcGdAkp.json.gz" + }, + { + "hashAlgorithm": "SHA-256", + "hashValue": "41eeb2ab841d03c2bb16e4549fc8fe3ac304ed7430173a91b37c878c77336169", + "newestEventTime": "2020-09-11T19:11:18Z", + "oldestEventTime": "2020-09-11T19:11:18Z", + "s3Bucket": "leh-ct-test-bucket", + "s3Object": "AWSLogs/144492464627/CloudTrail/us-west-2/2020/09/11/144492464627_CloudTrail_us-west-2_20200911T1915Z_TIKlbLnJ6IwUxqxw.json.gz" + }, + { + "hashAlgorithm": "SHA-256", + "hashValue": "6295aeb3b41cf021fe67e0b36f5ac51e557c41c85af6eef282d77ef056210d70", + "newestEventTime": "2020-09-11T18:32:04Z", + "oldestEventTime": "2020-09-11T18:32:04Z", + "s3Bucket": "leh-ct-test-bucket", + "s3Object": "AWSLogs/144492464627/CloudTrail/us-west-2/2020/09/11/144492464627_CloudTrail_us-west-2_20200911T1835Z_OPJhVNodH1gY760s.json.gz" + }, + { + "hashAlgorithm": "SHA-256", + "hashValue": "54a29065f5cbfc192584e9245f8d3d86dfabf5714a5a5ec7dd2a0f4312b86198", + "newestEventTime": "2020-09-11T19:21:28Z", + "oldestEventTime": "2020-09-11T19:21:28Z", + "s3Bucket": "leh-ct-test-bucket", + "s3Object": "AWSLogs/144492464627/CloudTrail/us-west-2/2020/09/11/144492464627_CloudTrail_us-west-2_20200911T1925Z_zJNGzQovyNAImZV9.json.gz" + }, + { + "hashAlgorithm": "SHA-256", + "hashValue": "551c8cbe75d081526b64fc799c67007652f2502d2f95bb9d5c8699ffc5ba7079", + "newestEventTime": "2020-09-11T18:51:21Z", + "oldestEventTime": "2020-09-11T18:51:21Z", + "s3Bucket": "leh-ct-test-bucket", + "s3Object": "AWSLogs/144492464627/CloudTrail/us-west-2/2020/09/11/144492464627_CloudTrail_us-west-2_20200911T1855Z_RqN9YzoKAJCKbejj.json.gz" + }, + { + "hashAlgorithm": "SHA-256", + "hashValue": "81605414e790e84dff20da66235f07134f77bb5c4863729bafe7672a98f07cb0", + "newestEventTime": "2020-09-11T18:46:45Z", + "oldestEventTime": "2020-09-11T18:46:45Z", + "s3Bucket": "leh-ct-test-bucket", + "s3Object": "AWSLogs/144492464627/CloudTrail/us-west-2/2020/09/11/144492464627_CloudTrail_us-west-2_20200911T1850Z_jLldN7U8XrspES8p.json.gz" + }, + { + "hashAlgorithm": "SHA-256", + "hashValue": "45050ec665636f9185f5b5a1e43c745a8282cb2e504092a452f0a4dc1fb8a6d5", + "newestEventTime": "2020-09-11T19:01:06Z", + "oldestEventTime": "2020-09-11T19:01:06Z", + "s3Bucket": "leh-ct-test-bucket", + "s3Object": "AWSLogs/144492464627/CloudTrail/us-west-2/2020/09/11/144492464627_CloudTrail_us-west-2_20200911T1905Z_jBNdmg4bSGxZ3wC8.json.gz" + }, + { + "hashAlgorithm": "SHA-256", + "hashValue": "60ed8fcbd172d3f6d1ca4fa872681f429b005ed8a81fc9450aa06db92ccabd47", + "newestEventTime": "2020-09-11T19:16:12Z", + "oldestEventTime": "2020-09-11T19:16:12Z", + "s3Bucket": "leh-ct-test-bucket", + "s3Object": "AWSLogs/144492464627/CloudTrail/us-west-2/2020/09/11/144492464627_CloudTrail_us-west-2_20200911T1920Z_bj5DRrmILF6jK23a.json.gz" + }, + { + "hashAlgorithm": "SHA-256", + "hashValue": "b2b0e2804d1c6b92d76eee203d7eba32d30d03e696f7175723a83cec2d7ad4ba", + "newestEventTime": "2020-09-11T18:56:05Z", + "oldestEventTime": "2020-09-11T18:56:05Z", + "s3Bucket": "leh-ct-test-bucket", + "s3Object": "AWSLogs/144492464627/CloudTrail/us-west-2/2020/09/11/144492464627_CloudTrail_us-west-2_20200911T1900Z_6LjrkrhsLQMzCiSN.json.gz" + }, + { + "hashAlgorithm": "SHA-256", + "hashValue": "4397a13565a67d9e6de577379b8eb7e61ca5b2b191c9b5da0243136dfc5581c7", + "newestEventTime": "2020-09-11T19:06:31Z", + "oldestEventTime": "2020-09-11T19:06:31Z", + "s3Bucket": "leh-ct-test-bucket", + "s3Object": "AWSLogs/144492464627/CloudTrail/us-west-2/2020/09/11/144492464627_CloudTrail_us-west-2_20200911T1910Z_DLyqye8LaeoD204N.json.gz" + }, + { + "hashAlgorithm": "SHA-256", + "hashValue": "49f09d2389632c70bc0066ed5d65768632dd2e60ed9c80af9d0c2c5f59bd60b6", + "newestEventTime": "2020-09-11T18:41:58Z", + "oldestEventTime": "2020-09-11T18:41:58Z", + "s3Bucket": "leh-ct-test-bucket", + "s3Object": "AWSLogs/144492464627/CloudTrail/us-west-2/2020/09/11/144492464627_CloudTrail_us-west-2_20200911T1845Z_TSDKyASOn2ejOq5n.json.gz" + }, + { + "hashAlgorithm": "SHA-256", + "hashValue": "0944f9a05d70868bc6f6048d58fd00764ab65e123b8ffefb193b22ca4394d771", + "newestEventTime": "2020-09-11T18:37:10Z", + "oldestEventTime": "2020-09-11T18:37:10Z", + "s3Bucket": "leh-ct-test-bucket", + "s3Object": "AWSLogs/144492464627/CloudTrail/us-west-2/2020/09/11/144492464627_CloudTrail_us-west-2_20200911T1840Z_btJydJ2t7hCRnjsN.json.gz" + } + ], + "aws.cloudtrail.digest.newest_event_time": "2020-09-11T19:26:24.000Z", + "aws.cloudtrail.digest.oldest_event_time": "2020-09-11T18:32:04.000Z", + "aws.cloudtrail.digest.previous_hash_algorithm": "SHA-256", + "aws.cloudtrail.digest.previous_s3_bucket": "leh-ct-test-bucket", + "aws.cloudtrail.digest.s3_bucket": "leh-ct-test-bucket", + "aws.cloudtrail.digest.signature_algorithm": "SHA256withRSA", + "aws.cloudtrail.digest.start_time": "2020-09-11T18:36:49.000Z", + "cloud.account.id": "144492464627", + "event.dataset": "aws.cloudtrail", + "event.kind": "event", + "event.module": "aws", + "event.original": "{\"awsAccountId\":\"144492464627\",\"digestStartTime\":\"2020-09-11T18:36:49Z\",\"digestEndTime\":\"2020-09-11T19:36:49Z\",\"digestS3Bucket\":\"leh-ct-test-bucket\",\"digestS3Object\":\"AWSLogs/144492464627/CloudTrail-Digest/us-west-2/2020/09/11/144492464627_CloudTrail-Digest_us-west-2_leh-ct-test_us-west-2_20200911T193649Z.json.gz\",\"digestPublicKeyFingerprint\":\"4a7aa197feec25e9bd0b5e98fcad8ecb\",\"digestSignatureAlgorithm\":\"SHA256withRSA\",\"newestEventTime\":\"2020-09-11T19:26:24Z\",\"oldestEventTime\":\"2020-09-11T18:32:04Z\",\"previousDigestS3Bucket\":\"leh-ct-test-bucket\",\"previousDigestS3Object\":\"AWSLogs/144492464627/CloudTrail-Digest/us-west-2/2020/09/11/144492464627_CloudTrail-Digest_us-west-2_leh-ct-test_us-west-2_20200911T183649Z.json.gz\",\"previousDigestHashValue\":\"531914fcfa0dbacf09cdd4175a1fdcb5eda6e852911209f3c3ecb0a39036c860\",\"previousDigestHashAlgorithm\":\"SHA-256\",\"previousDigestSignature\":\"10e8072f3f2a1d2990dcc98e9d44c88a6ae2ada9df9c3ae6d35dfe8d5c47caf807702f1e1eec47edfeecfcc2248388f78fddfc945ae587fb4a3e7676b5547f342fa872b2a1c5baa62b33bcb528ce7c10e3748aab8f82c16c024de79d8a20128b4c9e5ec65e98a9c4e613687ecc89c224a11bb3df06ce414ff740e4acf9bd41159e77f568355011828412f1193e357866fbd0436faffb56e149af196e35a7675c3bbd0a398f43159343c35f9129d6339a2818afdb3192f4fffea9bd2d1bb0a70e5bfae1291f2133aab0ad92522aea6d0f828c1780d3f3de6b8270ab3ba42459916b0fbb8e2fba6ff9677bafe730e60f5edcc0f1580cdb4e36f3e3b03e653e9798\",\"logFiles\":[{\"s3Bucket\":\"leh-ct-test-bucket\",\"s3Object\":\"AWSLogs/144492464627/CloudTrail/us-west-2/2020/09/11/144492464627_CloudTrail_us-west-2_20200911T1930Z_l2pGqVS53QcGdAkp.json.gz\",\"hashValue\":\"24078a45bb1c2e9ca444251ee8c1357644fdaebf4fee02d22250d8b6d48413c9\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T19:26:24Z\",\"oldestEventTime\":\"2020-09-11T19:26:24Z\"},{\"s3Bucket\":\"leh-ct-test-bucket\",\"s3Object\":\"AWSLogs/144492464627/CloudTrail/us-west-2/2020/09/11/144492464627_CloudTrail_us-west-2_20200911T1915Z_TIKlbLnJ6IwUxqxw.json.gz\",\"hashValue\":\"41eeb2ab841d03c2bb16e4549fc8fe3ac304ed7430173a91b37c878c77336169\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T19:11:18Z\",\"oldestEventTime\":\"2020-09-11T19:11:18Z\"},{\"s3Bucket\":\"leh-ct-test-bucket\",\"s3Object\":\"AWSLogs/144492464627/CloudTrail/us-west-2/2020/09/11/144492464627_CloudTrail_us-west-2_20200911T1835Z_OPJhVNodH1gY760s.json.gz\",\"hashValue\":\"6295aeb3b41cf021fe67e0b36f5ac51e557c41c85af6eef282d77ef056210d70\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T18:32:04Z\",\"oldestEventTime\":\"2020-09-11T18:32:04Z\"},{\"s3Bucket\":\"leh-ct-test-bucket\",\"s3Object\":\"AWSLogs/144492464627/CloudTrail/us-west-2/2020/09/11/144492464627_CloudTrail_us-west-2_20200911T1925Z_zJNGzQovyNAImZV9.json.gz\",\"hashValue\":\"54a29065f5cbfc192584e9245f8d3d86dfabf5714a5a5ec7dd2a0f4312b86198\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T19:21:28Z\",\"oldestEventTime\":\"2020-09-11T19:21:28Z\"},{\"s3Bucket\":\"leh-ct-test-bucket\",\"s3Object\":\"AWSLogs/144492464627/CloudTrail/us-west-2/2020/09/11/144492464627_CloudTrail_us-west-2_20200911T1855Z_RqN9YzoKAJCKbejj.json.gz\",\"hashValue\":\"551c8cbe75d081526b64fc799c67007652f2502d2f95bb9d5c8699ffc5ba7079\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T18:51:21Z\",\"oldestEventTime\":\"2020-09-11T18:51:21Z\"},{\"s3Bucket\":\"leh-ct-test-bucket\",\"s3Object\":\"AWSLogs/144492464627/CloudTrail/us-west-2/2020/09/11/144492464627_CloudTrail_us-west-2_20200911T1850Z_jLldN7U8XrspES8p.json.gz\",\"hashValue\":\"81605414e790e84dff20da66235f07134f77bb5c4863729bafe7672a98f07cb0\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T18:46:45Z\",\"oldestEventTime\":\"2020-09-11T18:46:45Z\"},{\"s3Bucket\":\"leh-ct-test-bucket\",\"s3Object\":\"AWSLogs/144492464627/CloudTrail/us-west-2/2020/09/11/144492464627_CloudTrail_us-west-2_20200911T1905Z_jBNdmg4bSGxZ3wC8.json.gz\",\"hashValue\":\"45050ec665636f9185f5b5a1e43c745a8282cb2e504092a452f0a4dc1fb8a6d5\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T19:01:06Z\",\"oldestEventTime\":\"2020-09-11T19:01:06Z\"},{\"s3Bucket\":\"leh-ct-test-bucket\",\"s3Object\":\"AWSLogs/144492464627/CloudTrail/us-west-2/2020/09/11/144492464627_CloudTrail_us-west-2_20200911T1920Z_bj5DRrmILF6jK23a.json.gz\",\"hashValue\":\"60ed8fcbd172d3f6d1ca4fa872681f429b005ed8a81fc9450aa06db92ccabd47\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T19:16:12Z\",\"oldestEventTime\":\"2020-09-11T19:16:12Z\"},{\"s3Bucket\":\"leh-ct-test-bucket\",\"s3Object\":\"AWSLogs/144492464627/CloudTrail/us-west-2/2020/09/11/144492464627_CloudTrail_us-west-2_20200911T1900Z_6LjrkrhsLQMzCiSN.json.gz\",\"hashValue\":\"b2b0e2804d1c6b92d76eee203d7eba32d30d03e696f7175723a83cec2d7ad4ba\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T18:56:05Z\",\"oldestEventTime\":\"2020-09-11T18:56:05Z\"},{\"s3Bucket\":\"leh-ct-test-bucket\",\"s3Object\":\"AWSLogs/144492464627/CloudTrail/us-west-2/2020/09/11/144492464627_CloudTrail_us-west-2_20200911T1910Z_DLyqye8LaeoD204N.json.gz\",\"hashValue\":\"4397a13565a67d9e6de577379b8eb7e61ca5b2b191c9b5da0243136dfc5581c7\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T19:06:31Z\",\"oldestEventTime\":\"2020-09-11T19:06:31Z\"},{\"s3Bucket\":\"leh-ct-test-bucket\",\"s3Object\":\"AWSLogs/144492464627/CloudTrail/us-west-2/2020/09/11/144492464627_CloudTrail_us-west-2_20200911T1845Z_TSDKyASOn2ejOq5n.json.gz\",\"hashValue\":\"49f09d2389632c70bc0066ed5d65768632dd2e60ed9c80af9d0c2c5f59bd60b6\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T18:41:58Z\",\"oldestEventTime\":\"2020-09-11T18:41:58Z\"},{\"s3Bucket\":\"leh-ct-test-bucket\",\"s3Object\":\"AWSLogs/144492464627/CloudTrail/us-west-2/2020/09/11/144492464627_CloudTrail_us-west-2_20200911T1840Z_btJydJ2t7hCRnjsN.json.gz\",\"hashValue\":\"0944f9a05d70868bc6f6048d58fd00764ab65e123b8ffefb193b22ca4394d771\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T18:37:10Z\",\"oldestEventTime\":\"2020-09-11T18:37:10Z\"}]}", + "event.type": "info", + "file.hash.sha256": "10e8072f3f2a1d2990dcc98e9d44c88a6ae2ada9df9c3ae6d35dfe8d5c47caf807702f1e1eec47edfeecfcc2248388f78fddfc945ae587fb4a3e7676b5547f342fa872b2a1c5baa62b33bcb528ce7c10e3748aab8f82c16c024de79d8a20128b4c9e5ec65e98a9c4e613687ecc89c224a11bb3df06ce414ff740e4acf9bd41159e77f568355011828412f1193e357866fbd0436faffb56e149af196e35a7675c3bbd0a398f43159343c35f9129d6339a2818afdb3192f4fffea9bd2d1bb0a70e5bfae1291f2133aab0ad92522aea6d0f828c1780d3f3de6b8270ab3ba42459916b0fbb8e2fba6ff9677bafe730e60f5edcc0f1580cdb4e36f3e3b03e653e9798", + "file.path": "AWSLogs/144492464627/CloudTrail-Digest/us-west-2/2020/09/11/144492464627_CloudTrail-Digest_us-west-2_leh-ct-test_us-west-2_20200911T183649Z.json.gz", + "fileset.name": "cloudtrail", + "input.type": "log", + "log.offset": 0, + "service.type": "aws", + "tags": [ + "forwarded" + ] + } +] \ No newline at end of file diff --git a/x-pack/filebeat/module/aws/cloudtrail/test/insight-json.log b/x-pack/filebeat/module/aws/cloudtrail/test/insight-json.log new file mode 100644 index 00000000000..c5c536fe7a6 --- /dev/null +++ b/x-pack/filebeat/module/aws/cloudtrail/test/insight-json.log @@ -0,0 +1 @@ +{"eventVersion":"1.07","eventTime":"2020-09-09T23:00:00Z","awsRegion":"us-east-1","eventID":"41ed77ca-d659-b45a-8e9a-74e504300007","eventType":"AwsCloudTrailInsight","recipientAccountId":"123456789012","sharedEventID":"e672c2b1-e71a-4779-f96c-02da7bb30d2e","insightDetails":{"state":"End","eventSource":"iam.amazonaws.com","eventName":"AttachUserPolicy","insightType":"ApiCallRateInsight","insffightContext":{"statistics":{"baseline":{"average":0.0},"insight":{"average":2.0},"insightDuration":1,"baselineDuration":11459},"attributions":[{"attribute":"userIdentityArn","insight":[{"value":"arn:aws:iam::123456789012:user/Alice","average":2.0}],"baseline":[]},{"attribute":"userAgent","insight":[{"value":"console.amazonaws.com","average":2.0}],"baseline":[]},{"attribute":"errorCode","insight":[{"value":"null","average":2.0}],"baseline":[]}]}},"eventCategory":"Insight"} diff --git a/x-pack/filebeat/module/aws/cloudtrail/test/insight-json.log-expected.json b/x-pack/filebeat/module/aws/cloudtrail/test/insight-json.log-expected.json new file mode 100644 index 00000000000..ec3dde12e81 --- /dev/null +++ b/x-pack/filebeat/module/aws/cloudtrail/test/insight-json.log-expected.json @@ -0,0 +1,24 @@ +[ + { + "@timestamp": "2020-09-09T23:00:00.000Z", + "aws.cloudtrail.event_type": "AwsCloudTrailInsight", + "aws.cloudtrail.event_version": "1.07", + "aws.cloudtrail.recipient_account_id": "123456789012", + "cloud.region": "us-east-1", + "event.action": "Insight", + "event.dataset": "aws.cloudtrail", + "event.id": "41ed77ca-d659-b45a-8e9a-74e504300007", + "event.kind": "event", + "event.module": "aws", + "event.original": "{\"eventVersion\":\"1.07\",\"eventTime\":\"2020-09-09T23:00:00Z\",\"awsRegion\":\"us-east-1\",\"eventID\":\"41ed77ca-d659-b45a-8e9a-74e504300007\",\"eventType\":\"AwsCloudTrailInsight\",\"recipientAccountId\":\"123456789012\",\"sharedEventID\":\"e672c2b1-e71a-4779-f96c-02da7bb30d2e\",\"insightDetails\":{\"state\":\"End\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"AttachUserPolicy\",\"insightType\":\"ApiCallRateInsight\",\"insffightContext\":{\"statistics\":{\"baseline\":{\"average\":0.0},\"insight\":{\"average\":2.0},\"insightDuration\":1,\"baselineDuration\":11459},\"attributions\":[{\"attribute\":\"userIdentityArn\",\"insight\":[{\"value\":\"arn:aws:iam::123456789012:user/Alice\",\"average\":2.0}],\"baseline\":[]},{\"attribute\":\"userAgent\",\"insight\":[{\"value\":\"console.amazonaws.com\",\"average\":2.0}],\"baseline\":[]},{\"attribute\":\"errorCode\",\"insight\":[{\"value\":\"null\",\"average\":2.0}],\"baseline\":[]}]}},\"eventCategory\":\"Insight\"}", + "event.outcome": "success", + "event.type": "info", + "fileset.name": "cloudtrail", + "input.type": "log", + "log.offset": 0, + "service.type": "aws", + "tags": [ + "forwarded" + ] + } +] \ No newline at end of file diff --git a/x-pack/filebeat/module/aws/fields.go b/x-pack/filebeat/module/aws/fields.go index 5b8a3d61898..5ce70304dba 100644 --- a/x-pack/filebeat/module/aws/fields.go +++ b/x-pack/filebeat/module/aws/fields.go @@ -19,5 +19,5 @@ func init() { // AssetAws returns asset data. // This is the base64 encoded gzipped contents of module/aws. func AssetAws() string { - return "eJzMXF9z4zZwf79PsZOX2DOSOrlkOh130hmdz9eocS6upUvaJwYCVxJqCGAA0Drdp+8sAFKkSEqyReVOD4nPpBa/Xex/LDyEJ9zeANvYNwBOOIk3MP5z+gbAoERm8Qbm6NgbgBQtNyJzQqsb+I83AAC/6TSXCAttYMVUKoVagtRLCwuj10Rm9AZgIVCm9sZ/YQiKrbFYjj5um+ENLI3Os/iblnXo88GTKSn7dUbxaXWJ6jJc6jx1hglZPmpbkT773BafFBcsly7xS9zAgkmLtcetYKuAtfF4bwnLjLDUoLfBr7KAz6hc8ozGCq1qbxScPOF2o0269+wAMPrMVlhFFOmDXoBbIQEMCxP6NXOjVmi5RZOIFJUTbtsKbV/ITWDDVmREeRIJA0pcExSulWNCWUjRMSEtsLnOncdLq4FeNGhNxr9BARDcijlYsxT9Vwz+naN1A2Aqhc1K8BVwg/5dJi1s0GCDXG4xHcFkAQ7XmTbMbBvf8e8M/AoFbrvSGwsrvaHfNmg2COg5cYnpaO/VNiWp7gbJoPHwsI40t6PlhbAjUcKesY4tr1i32dfUlyNpKkYBZbxmX7SCR7Q6NxzhI1sjXI0fP14XADMjFBcZk3t7zpmU+2KtoOYcrU2ecJuINnx94Q/rECGYvA8IN8x6xQGnwYqlqmpoN2CLlow2IcPAz64TcpsVngp4sqhi8UC9ODfCrSpmYJHnpk0loK7iZG6lYXjWM6OfRYoWhAq+htzQzrIjj610S9Fxg8xh6l2tW2mL1SVbvtplSlXhrhcsYblbERVO1FvfPq4VpwoaonY8M5kjCAvO0P+j+LV23imCNt6p+Z83xGonsVbPFEW021AmrfYyrPEatpe1i50+v30YQ4rPguO/g3YrNBthcRCiY1Nhq3L1e0VamzLXBT7I9MALLxEokfFO3ok1wmaFwbqautuUmLA2bzriOj+FEfp3zUGGuuzwJRz1YY/wepvspBfD2+nhrPgcskU4Ht6KzylmCC+QM0TtiTEmBpbDSjMAm/PVQZLMwqPWbkBG/MmiGZBBP2rZYTRVAZRBrT06XVoQQjk0ikmKWVEa1byqGsGW2K0nUNe942y35xKX5nb8+LHgMmrAFeNc5ypsnfe/fu+Mlnh9kFybeI4o0glSCWC+jirExQNneqPs5bSh4FeoZ/2EaTJv82h9JWa0VLHrVLFZNBThugoHMnZgbfkFFDnq3e1bGOdOw5QzXxzHWvBOMusEh3fIlHVMPrUXWGiMNgnX6b7nO73wa6+vqtz5RcpEI8YVgy43yvrIQM8P4VujtWzZJ8TJYTChvKoQKR1SN9RIK8mYYWt0aPb37VyR7ggPSJhMbQfRFigKWoqtIUZ3Z/brXDqRHK3zDqX6HQ+PNDGqYrKZVhaTmA/0LaWCfplvUD7KOH3HFnb2hMBXTC3RwlXI7AfNSjyjtM574BQlUoYXiFx/g0JlaSoIHJOJb6qkrNZhOleu45I85bqsUjyFFk7pkZV2pKYumkuDUGFvFNbiPn2D0iwsuRH6ztXNUHl5aYUOx0KgrdlryAij35yjUMtmO4dJiSksUaFhzn9f2EC6w4f6/l5LdnuWA63jL5o3FX0oAKYVRTHItUnbYbJMnN2EPIpz/DApO5HMWs3Frhj1zzd2nIlbJmWDkudgRnwekPWaKbb0ficYYp9GCO+0lshUhxptVkhlckXawsK+F4AKwvBWVzBjaaKVbG+6nr0VO6zCgs5IT2hHCLBfekhL7x50YQyJc3sEeU1jeAxSWO+8Stqxl4YpCLUT7Us7p5drV5Y9yvHjx2aieFIy3weMcUzYd6VbIUFK31u6Fr32lI+Ixjun0lJMsZOh8eDPom5u2MYOo98demQ3FOmG9FX/7w4N5CITZOydAj7HYB4xM0h5XfBdbCdjb/sGOYpn71+FPWTMka/gkZJ4tnE5H1um/bTcAITiMk+pNNkQamfEcokmhIV2Jxt6aUGHcvktJrF2xQymUaC97vl/fpq8r4TO+bZ6huY05Er8naPcFvpcfd4uzXig6XeGyk+qzEIqG0OIDbWD05CKxQIN/SOcz9Y/Uf9su5I9ZzxBlWZa9C2SPfX64+EWioXIlMPJWkygYkvQl9Ke7WYApO87DUz5rnK1UC0L7qK4nv7YzivXymqJidRL0Z6svCb6xNNcmyEXC8EJ5G1Y6J7Wibv50shzvDI4jLqJvD0A7EqEO2/D76lQoNrrOA+H+KjystZzIbEjSaxzMg+pUsc7J7fCmxlLjRsqebyWBWRwCFnBhNeZxOmD8Ps57Pn0eN/YgcMCXrAkt0cOoi4mWu/XYcG4o4p9d1hEnmrT0QWDeGYOaW4oxHSyWrC4kMw5VA0eX2+2d9MdUV8e+3SD5K7n/4fceQZN6OzZfB4UHZhB9b2DJ6U3irwYS5+Z4o1quFfz7uK9yeUxEz+vCwCndQJe0Fp7DY/dXdoeGm2n97v6hd5X9wte1AF7WarZH8c9Jp5wNPmsTXxtmKudw33Nia8/CcuLJr4O9dNbUuMjvqEOoiDeFBzyt19fYne3b8P8oFAV4KcKTmQJS1OD9vXlU0N8u6NPdBCp74po713QtEhTzk+X5vIysrx/9yK1a9TRZ8utep4mNUthziSF0Y4jmrN6oK0AqkNrNQBeSM9v4Z5++S7+sqN0csws0SV+90bNVtGZECsHzWGhoCa7cdrOMFamrMKSl95v4pyJy2tPpNxoaCDVWAp5d/cvM9pprveL3jNBFVTb9/Rq5VxG8dDx7PrIKaDRHK0Vapk4scaRRd6KdCE1e6G/9XqnHZNh3kgosMi1otJRKI570gsZVki2hN3JOFdOSBC1+p+BwSVtCUXJOeNPqDqa9fHhN8RmhQ16EgGCE1LWfmEdM87GRImyg87mckjbvjKHZUJX3buyoVPjsr6XXjxSdDYFd6ILfK3bQ5rUjSOol3EVDam6UwrWQkoRmR1EbgN8nfk6qsIQl9ruD3eVnlPahPyYXbEnvCwfxVjj7H4K5ZIkaK7Xmc+O9/gC3aKlK2ZhjqgArWNzKeyqi7XC/MR+PXqmh5s87CcZhRLtND10lI954AJhps3rz7navbA2rujSn4uOXPaoLDGtYy635w28tEIOhIEIN83zqvq0aaDhCxQPyX6dNuG06a9O6H+1cmytTLjIVn0H6un0HgLd0GoRiozgX/yvy03oSG0I02UiNS1fRusX4+IrbVElHI27aMYV1gFaRyz85DPEM50wulZRgtfCt2gE61m4gSaofD1Hc2FehOJ67cOrtAmT2LczodpqiSaemuuF9+B+nUpEnW/bUvjCjgNXvhkUOe4OR/RTmrQ1Es/kg2B74iF/70BgGMdzjl9al/bXP5QrI8Zf/zMcr7+o4YxWG07Sv2CFLO2qutZUYWOamFxSOiV0y9j42Wl7oLrbZJ875bIcrfQQ6tOVYkEJL70TH3fMpYQkCT8jz5uXNM4EHg/iCuLh8kAt1O2GFK8W2myYSQewEJ8xHRaRYVC7WTEaja5HMHHAmSoutIHFZzRMBvF02KHBVBjkLslNz97k0+N99NBe4nEdP4LDi+5+KYIDc6Ajg8z2fUUxDKcGysWMarkdEd+CCVlFVka3H8N0yNduxEx/9L1PNMXNr5d0ZeY5f0KXtI1pnOs2mNJKcCbDZaLdbIhfa2/6PcDoyOX8s8u1jgL94Cr2775EYyRlWDKhunokBtfaYdKRrTd+fYpryDJmQjA9uTkIzU7EP7ap5Xr+xIDB0KtnrurXvopz+sPdkz4D2Bis82eDtamK8oS/fagC2W6moB1qOZzWr3h3w3C+N5bCKlwTgxS5ZFQZMAvT38cPo/LNATzeTWejX2azh2SNbqXTUTF85ac+B/Dn3bvpZHZ36BVt4N14dvvL6P3d/d3sbvT7u/+6u521s/6EPYfv755w+131yHAXpCl2oKKiJ/Ugvxt+V3jpnahSjeHU0VFNzvzF1/IQ77Cm5Ub0y8tjIDz89DipcUSyLx1L44ykCo3KviRUez22MlS+RiN4wFGtR3fzei2nsD3eE2kvlUozvPNx+FanWN1npWOA1pznxnQ2S7YObWK7Bm5fLbFY/ZRNOL+Or+AHgJ+L40Uv0l3n+BkNpcNVNr6g0R1uxM8IJFZ8aZfsOX0rIlqGXL8OFWZeAzvbJv6bvpN2EUnWGmT7oVYoWEixXLnKxJpPa763kKGxGWWNzx0a6nKjEmZ0rtJ/DD5zFQW2GQXrSu6+1bk5HEMMLtCYviN0rQbyuvkY14ll2vFC1v/dDbY8Z4D9RGifLJrhmFY6WEPGyareK9viLsDkfdFvLCPPqcEmkpikx0LOSvec2RADn4ds/WUo0uFbf/Ov1Eb87FClu4QLJu87OnRiqZjLTddc3bmdpIJ8IacBTMXyD4+Wfvhp0LzPWc0Ya07i1XllaF8mNheNC//ntiGRE3NTTRWEhXu2RQNX0+n9ddEz3Y1F4VI7Uf4ZCVL/aRtr9KCjFVEbijvvSk973IjH2eXfbqhP4YW/PDPO3eoXb6thuqj+TrBiO4D/ztFspyH1pvf+pn8XufhVZnBIuoEppXjXr99ab1Vh0Z4bA8WMcaGWsZNJPx6ZD3bSXsaaZoYp609HgqJNi6vdV7P76XXpzSqaFhub+yeBlanthdSb0xsYl5rL+ePhFgjJi1oXF5ExIflASO710hZL+L/JtNU5KUO8G+wZj5dIwn3hQvzCwtvyC2GcewsMeG6dXnd9o0OVerhY0p54+1sA5YWS4nSz2IKuVr1Ds7hEh3nXRlDoNto87dby2MKQv59hMWyxEDyeh2uTHu7b9gtz795E2w3CiG8A49vbu4eZ/+sbd921tNTLQ7Xeq5FKvVySo42VXhRusb0D+P3XAXz8/f14NvaR+NfJA/3cte3WMXXRXS+W8KL9vinZV2jFoEjdStrC+s6jd4pbnXfMFT25xBrO0rQ9nrymlZcxyg6GEp9RwpU2YikUk9dF67N5JB/Z6UaYWvePIEypVlQhsldgFu7iIM7njF9QY/w1HrLD8o/29eo9bD5X2L/b3eEPC1ySBcezZCHZsmfPMhduzexTrOXKwKGl1BvyOLPbB/DL3sDbn6f/+3Hww7/R/4bj218HP/z8YfJx8NPPj9NZO+TLDWgGqd3A5OH5pwH99199iXf3YTx68/8BAAD//624A44=" + return "eJzcXN9z27aTf89fsdOX2jOSbr5p5+bGN70ZxXGuurqpz1LauycWIlcSzhDAAqAU56+/WQD8IRGUZItKOl89JLJIAp9d7G8sOIQnfL4BtjVvACy3Am9g/Mf0DYBGgczgDczRsjcAGZpU89xyJW/gP94AAPyqskIgLJSGFZOZ4HIJQi0NLLRa0zCjNwALjiIzN+6BIUi2xnI6+tjnHG9gqVWRh18i89DngxumGtnNMwpXm1M0p0mFKjKrGRfVpdiM9NmntvxkuGCFsImb4gYWTBjcuRwF2wSstMN7S1hmhGUHegx+kwTcoLTJBrXhSu7cUVLyhM9bpbO9aweA0We2wiaiMD6oBdgVEkA/MaFfMzuKQisM6oRnKC23z1Fo+0xuAxtGkdHIkzAwoMA1QUmVtIxLAxlaxoUBNleFdXhpNlCL1liT8a9QAgS7YhbWLEP3iMa/CjR2AExmsF3xdAWpRncvEwa2qLE1XGEwG8FkARbXudJMP7eecfcM3AwlbrNSWwMrtaVfW2O2BlBzohKz0d6tMSFprgbxoHXxsIy0lyNyg1+RwGFHWMeSN7Rb70vqy5G0BaOEMl6zL0rCIxpV6BThI1sjXI0fP16XAHPNZcpzJvbWPGVC7LO1gTpN0ZjkCZ8THsPXF34/Dw0Ek/ce4ZYZJzhgFRi+lE0J7QZs0JDSJqQY+Nl2Qo5p4amAJ4smFgfUsXPL7aqhBgbTQsdEAnZFnNStUgxHeq7VhmdogEtva8gM1ZodaIyOW7Eu1cgsZs7U2pUy2Jwy8miXKjWZu16whBV2RaOkNHr07uNScSqjIUjHhokCgRuwmv4P7FfKOqMISjuj5r5vidTOwaKWKbCoXlAmjHI83KHVLy+Ls50+v34YQ4YbnuK/g7Ir1FtucOC9Y1tgm3x1a0VSmzHbBd7z9MANL2EoDeOMvOVrhO0KvXa1ZbfNMW5M0TbEu/SUSuju1QcJ6tLDl1DUhz7C63Wyc7zg3k53Z+XnkC7CcfdWfk5RQ3gBnyFIT/AxwbEcFpoBmCJdHRySGXhUyg5IiT8Z1ANS6EclOpSmyYDKqcW906UZwaVFLZkgnxW40Yyrmh5sid1yAruyd5zseCxxaWrHjx9LKoMEXLE0VYX0S+fsr1s7rQReHxwuxp4jgnQCVzyYbyMKYXJPmdpKczlpKOnlcqOeMEvmMYvWV2BGU5WrThmbQU0eritxIGUHFosvoIxR727fwriwCqYpc8lxyAXvBDOWp/AOmTSWiad4goVaK52kKtu3fKcnfvH8qkmdm6QKNIJf0WgLLY3zDHT9EL41GsOWfUKcHAbj06vGIJVB6oYaxkpyptkaLer9dTuXpfXAA2Imk8+DoAvkBQ35Vu+juyP7dSEsT47meYdC/Y6LR4oYTTaZXEmDSYgH+uZSOX4Vb1A8ylJ6xpR69oSQrphcooErH9kP2pl4TmGds8AZCqQIzw9y/TdkKssyTuCYSFxRJWM7FaZz+TquhqdYlzWSJ1/CqSyyVJbE1AZ1aQ1U6hu5tbBOf0Nulprccn3nyqbPvBy3fIVjwdHs6KuPCIPdnCOXy3Y5hwmBGSxRombWPc+NH7rDhrr6XiS6PcuA7uIvizcNeSgBZg1B0ZgqncVhspyfXYQ8inP8MKkqkcwYlfI6GXXXt2ac81smRGskR8GM6DzA6zWTbOnsjlfEPpUQ3iklkMkOMdqukNLkBre5gX0rAA2E/q4uZ8ayREkRL7qevRQ1Vm5A5SQntCIE2E09pKnrC10YfeAc9yCvKQyPQXDjjFc1dqilYQZc1qx9aeX0cuXKqkY5fvzYDhRPCub7gDEOAXudupUcpPA9UrXotaZ8hDXOOFWaosuV9IUHtxd1c8O2Zhjs7tAhuyFPN6RH3d8dEpjynJOydzL4HIV5xFwjxXXedrGax073NabIN86+cnNImQNd3iIlYW/jcja2CvtpugFwmYoio9RkS6it5sslau8W4kbW19K8DBXi7xjEmhXTmAWG9rrm//lp8r7hOufPzT00q6CQ/K8CxXMpz83rcW6GDU23MpR+UmbmQ9ngQozPHayCjC8WqOkPvz+7+wnyZ+JCtsnTBGWWK943S/bE6/eHWygnIlX2O2shgAolQZdKO7LbDpCetwqYdFXlZqJaJdxlcj39IU5rqqRRAhOhljwerLzG+4TdXJNjyhc8JZC3fqJ7mies5ks9z/HM4DDqNvK4A6hThDunw+8pUaDc6zgNh+ho0rJWcy6wI0jcpWTuQ6WOe04uhbcjlh1qKOVxUuaRwSFkJRFOZhKrDsLvZ7Pn0+N9awUOM3jBksIc2Yi6GGudXYcFSy1l7PVmEVmqbUcVDMKeOWSFJhfTSWpJ4kIwa1G2aHy92t5N60FdeuzCDeK7mv8fptYRqH1lzxRzL+jANMrvLTxJtZVkxVi2YTJtZcO9qncX7W0qj6n4eVUAOK0S8ILS2mto7K7S9lBoO73e1S/0vqpf8KIK2MtCzf4o7jHwhJOCz5LMjC/RxJP7Mzy/cyW3VUMZvHezwL1avtTrC7VMFly0UuMapkQT7zg4Kdur8uR7tXTzlK1TdZ4cZVFDUCzTNrF83Z39dezOn6oKbgZa/U+zW78zr0novRbUCIHwQ6rId8f1wLInGob54oQLkFNvzt2oVQgq1BI8M1ZsE5ewOaIkZeIbJ4g70X23UqHMLsoplNk/BZ/MD8m8SJ+iu4EX2cErswTw01LE7kl0fQqF1u0KYJitwdIVM3v0HqTQBxUXpLCmyk/l2siuQqo4iBIeHUsoH7tVPXOBIU3iuymVuCWPH2rXFxT+SuoDyrUyvrRSFkiBrRWJtRCH4pmQagfjV4v4jjXsJleJ7NuQ6yfuoLQ7XYNXU5pr3HBVmORraOthDS2h7KhjV6ZxgnZWpK2YWSVMLJXmdrX+StaIJoVq0naXhLveJb4xVhwgtJgLnro+1gWXS9S55lFL1xedK/zMMkz5mglAmaoMM2jMXHXlOlzOYBH50eHWzKarEDDmmm+YRffAXnMsP5EVdDuzhcavst718rZ6eaNod04pbInyN/ugvs0phT8Iy4tOKRzqAYmUc4/ks7sgysHbjMP07bfn2N3tW3/mhcsG8FMZx/OEZZlG8/qSf4t9dbseWgij1xs/LiNGHeGmmJ/OzeVleHn/7kVi19r7OZtvTZstFMtgzgSTKXa0FZ21bx8F0DxosQPAMWnzFu7px3fhx45yv2V6iTZxqzdqb2+eCbHRHOkn8mJSHwHrLL1UGTA3Ftsbj2fictITRm5twlF2JCWm3TvWuVZWpWp/o+ZMUOWo8TW9Wlmbg9Jg0/z6SOeaVikaw+XShaAjg2kU6UKolnM9Re6UZaLKBQ2mSmYGDC+zxJp7viroC4Tc1DwupOUC+M6eFeWaS1oSlyiy9AllR4NJuPg3IrNBBl0JAMFyIXZ+cKUME4p7lKt3NkT4UuM3prAqQjbXrtqE3KFydy0dewTv3MiuWefpWsddmlCttqmXUVUmrI2VkrDmQvBA7CBQ6+Gr3NX+GwSlQpn9TKGynIISBJmZFXvCy9JRHsWZ3U+hmpIYnap17iq6e3SBikhpVaBAY9lccLPqIq1UP75fAD3Twk0e9oOMUohqSfcx+zELXCLMlX59b1bcCittyxz4XHRkskfVtoixzBbmvCbtKGQ/MNDAbfW8al5tK6h/gPwh6a9V2ld+/+yE/meUYmNEkvJ81bejnk7vwY/rMyUuSQn+xf1cLUJHaEOYLuOpafrKW78YV7pSBmWSorYXjbj8PEDz8IU7rQehD8mnnA0heC18g5qznpnrxwRZrOeoL0wLl6laO/cqTMIE9m1MKLdaog6dnmrhLLibp+FR58+xEL7UY0+V28AMFHe7I/qWJbGdqzPpINhucB+/dyDQLMVzWoaiU7sjy9JWHuPP/xmO11/kcEazDSfZn7BClnVlXb5KlCW6EBROcRU56nh22O5HrRfZxU6FqI4DOQi7J4L4ggJeuidc7uil9kESfsa0aG/znQk8NI+Vg/sDrzuurj5Yc7VQest0NoAF/4zZsPQMg53TwKPR6HoEEwspk+VOIhjcoGbCs6dDDzVmXGNqk0L3bE0+Pd4HC+04HuZxbeNp2ZFSseDA2aWRRmb6fq2GP1DlRy7PVVXLEfAtGBdNZPUWku9o/taFmOkPbr8edfm2gpdUZXw1P4m1Fp9rNphUkqdM+APwdT+zm2vvxKaH0RHLxfY2eiwdlRsa5S5t87x2UEYShiXjsqtGonGtLCYd0Xrr51NMQ54z7Z3pycVBaFcivtqiVvO5LhcGQyeehdx9VUHZW3q4etKnAxuDsa6fbacTuN7NijYCI6v7YONQqwMV/bK3PsDhamMZrPyrDSDDVDDKDJiB6W/jh1F15wAe76az0c+z2UOyRrtS2ag8MOBOKg3gj7t308ns7tAtSsO78ez259H7u/u72d3ot3f/dXc7i5P+hD277++e8Pm7Zptb7aTJd4RdKgfyu+F3pZWuWZUp9J1ylnJy5vagqsazw5JWaN4vLY9+4OGnx8kORcT7yrC09kia0CjtS3y212MpQxZr1Dz1OJr5aH3GJNI52OPZ5niqVKnhnfPDtyrD5jpLFRy0Sl3LQ1ex5NmiSUzXIbFXcyxkP1URzs3jMvgB4OeyJc6xtK4cb1BTONwk4wtq1WFGXF9IYviXOGfPqVvRoJXL9f0nXIKTwM6yiXsy1jXRDyd3CmT7rpZLWAi+XNnGKQsX1nxvIEdtcooaNx0SagstE6ZVEW/wugh8ZhsCbHJy1o3Y/VkV+rAPcZ1dum8PvZMDOdl8DPOENO14IuveFceW5xy6PBHaJ4N6OKaZDuaQ4TRA75lteX518r6sN1ae51RnE4aYZMdczkr1HNkQAZ+HbP1lyLPhW/e2ikoa8bNFmdUBF0zed1ToqqaLi7y1sBq+5NMApnz5u0NLX34ctLtrmhHjjpF4dVzpy5eJKXir4evcMiSmRNxUUQZh4J49o4ar6fT+uqyZ1q38uFSWV68+I/GfxkijCx2liJ2DHOcdQ4/7jbCdXb1vbPfkiH9b4riwq5+drvqO+N17vBabAfx3gfp56kNvuu8v+ruMxa9yjUOSDcwoxLt+/dI6rfKT9lwYKM/FlWIZKpn09ciZNivMZbRpppk0bnfEC9q0fB3R1ex+el1Zs4akhcLm/k5g46ThQqjt6QWMS/Xl/P5wC4TkRaWLi/CYkHwgJPdqacop3HtEn1VBwhDeZ+MIDweffStvyX5u4G31gD+C+AwM0sJYte56okOUejgMHQ+83cnV6hB0ubtZLkFXqd6iXlyiwlyXESTardJP9VwOW91oajVbLHga9sOVzg7XbfuFuXfWN/bWi4BvAOPb27uHmXtj3F13Li3U8lCu92qkQi2XZGhDpheYWy7vAH77ZQAff3s/no2dJ/5l8kDfu5bdWCYvuurlFI6137c5+wqpGJShWzU2N67y6Izisyo6+oqebGJ0yrIs7k9eU8rLGUUHQ4EbFHClNF9yycR1Wfpsb8kHcroRZsZ+FYQZ5YrSe/YGzNJcHMS5ydMLSow7ek56WL1oulfrYYq5xP7Nbo3fT3BJEmyaJwvBWgfeziRhzu2amaeQy1WOQwmhtmRxZrcP4Ka9gbc/Tf/34+Af/0b/Dce3vwz+8dOHycfBjz89TmdxyJdr0PRcu4HJw+bHAf37ry7Fu/swHr35/wAAAP//r4rS6w==" } diff --git a/x-pack/filebeat/modules.d/aws.yml.disabled b/x-pack/filebeat/modules.d/aws.yml.disabled index 7a0da775c8d..0fe8465211b 100644 --- a/x-pack/filebeat/modules.d/aws.yml.disabled +++ b/x-pack/filebeat/modules.d/aws.yml.disabled @@ -8,6 +8,18 @@ # AWS SQS queue url #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue + # Process CloudTrail logs + # default is true, set to false to skip Cloudtrail logs + # var.process_cloudtrail_logs: false + + # Process CloudTrail Digest logs + # default true, set to false to skip CloudTrail Digest logs + # var.process_digest_logs: false + + # Process CloudTrail Insight logs + # default true, set to false to skip CloudTrail Insight logs + # var.process_insight_logs: false + # Filename of AWS credential file # If not set "$HOME/.aws/credentials" is used on Linux/Mac # "%UserProfile%\.aws\credentials" is used on Windows From ffca9e07374ca2c54e8d27992d3cf44dd2131bbb Mon Sep 17 00:00:00 2001 From: "Lee E. Hinman" Date: Mon, 14 Sep 2020 17:15:34 -0500 Subject: [PATCH 2/3] incorporate feedback - Add changelog - update docs - add insight section --- CHANGELOG.next.asciidoc | 1 + filebeat/docs/fields.asciidoc | 11 ++- .../docs/inputs/input-aws-s3.asciidoc | 7 +- .../module/aws/cloudtrail/_meta/fields.yml | 9 +- .../module/aws/cloudtrail/ingest/pipeline.yml | 4 + .../test/cloudtrail-digest-json.log | 2 +- .../cloudtrail-digest-json.log-expected.json | 84 +++++++++---------- .../test/insight-json.log-expected.json | 40 +++++++++ x-pack/filebeat/module/aws/fields.go | 2 +- 9 files changed, 112 insertions(+), 48 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index dc6a588dc1b..b4f1c06c8b8 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -261,6 +261,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Provide backwards compatibility for the `set` processor when Elasticsearch is less than 7.9.0. {pull}20908[20908] - Remove wrongly mapped `tls.client.server_name` from `fortinet/firewall` fileset. {pull}20983[20983] - Fix an error updating file size being logged when EOF is reached. {pull}21048[21048] +- Fix error when processing AWS Cloudtrail Digest logs. {pull}21086[21086] {issue}20943[20943] *Heartbeat* diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index 8d7784c5e9e..3268e1a2931 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -1496,7 +1496,7 @@ Fields from Cloudtrail Digest Logs *`aws.cloudtrail.digest.log_files`*:: + -- -A list of Logfiles contained in the digest +A list of Logfiles contained in the digest. type: nested @@ -1592,6 +1592,15 @@ type: keyword -- +*`aws.cloudtrail.insight_details`*:: ++ +-- +Shows information about the underlying triggers of an Insights event, such as event source, user agent, statistics, API name, and whether the event is the start or end of the Insights event. + +type: flattened + +-- + [float] === cloudwatch diff --git a/x-pack/filebeat/docs/inputs/input-aws-s3.asciidoc b/x-pack/filebeat/docs/inputs/input-aws-s3.asciidoc index 9b8731c52c8..8891e38fcc4 100644 --- a/x-pack/filebeat/docs/inputs/input-aws-s3.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-aws-s3.asciidoc @@ -76,8 +76,11 @@ the files that are downloaded. This is a list of selectors which are made up of `regex` and `expand_event_list_from_field` options. The `regex` should match the S3 object key in the SQS message, and the optional `expand_event_list_from_field` is the same as the global -setting. Regex syntax is the same as the Go language. Files that -don't match one of the regexes won't be processed. +setting. If `file_selectors` is given, then any global +`expand_event_list_from_field` value is ignored in favor of the ones +specified in the `file_selectors`. Regex syntax is the same as the Go +language. Files that don't match one of the regexes won't be +processed. ["source", "yml"] ---- diff --git a/x-pack/filebeat/module/aws/cloudtrail/_meta/fields.yml b/x-pack/filebeat/module/aws/cloudtrail/_meta/fields.yml index 29fff28719e..155908315e9 100644 --- a/x-pack/filebeat/module/aws/cloudtrail/_meta/fields.yml +++ b/x-pack/filebeat/module/aws/cloudtrail/_meta/fields.yml @@ -231,7 +231,7 @@ - name: log_files type: nested description: >- - A list of Logfiles contained in the digest + A list of Logfiles contained in the digest. - name: start_time type: date description: >- @@ -283,3 +283,10 @@ type: keyword description: >- The algorithm used to sign the digest file. + - name: insight_details + type: flattened + description: >- + Shows information about the underlying triggers of an Insights + event, such as event source, user agent, statistics, API name, + and whether the event is the start or end of the Insights + event. diff --git a/x-pack/filebeat/module/aws/cloudtrail/ingest/pipeline.yml b/x-pack/filebeat/module/aws/cloudtrail/ingest/pipeline.yml index cf976147d75..8421e12d7f0 100644 --- a/x-pack/filebeat/module/aws/cloudtrail/ingest/pipeline.yml +++ b/x-pack/filebeat/module/aws/cloudtrail/ingest/pipeline.yml @@ -694,6 +694,10 @@ processors: field: "json.digestSignatureAlgorithm" target_field: "aws.cloudtrail.digest.signature_algorithm" ignore_failure: true + - rename: + field: "json.insightDetails" + target_field: "aws.cloudtrail.insight_details" + ignore_failure: true - remove: field: - "json" diff --git a/x-pack/filebeat/module/aws/cloudtrail/test/cloudtrail-digest-json.log b/x-pack/filebeat/module/aws/cloudtrail/test/cloudtrail-digest-json.log index 54182991959..f3393babceb 100644 --- a/x-pack/filebeat/module/aws/cloudtrail/test/cloudtrail-digest-json.log +++ b/x-pack/filebeat/module/aws/cloudtrail/test/cloudtrail-digest-json.log @@ -1 +1 @@ -{"awsAccountId":"144492464627","digestStartTime":"2020-09-11T18:36:49Z","digestEndTime":"2020-09-11T19:36:49Z","digestS3Bucket":"leh-ct-test-bucket","digestS3Object":"AWSLogs/144492464627/CloudTrail-Digest/us-west-2/2020/09/11/144492464627_CloudTrail-Digest_us-west-2_leh-ct-test_us-west-2_20200911T193649Z.json.gz","digestPublicKeyFingerprint":"4a7aa197feec25e9bd0b5e98fcad8ecb","digestSignatureAlgorithm":"SHA256withRSA","newestEventTime":"2020-09-11T19:26:24Z","oldestEventTime":"2020-09-11T18:32:04Z","previousDigestS3Bucket":"leh-ct-test-bucket","previousDigestS3Object":"AWSLogs/144492464627/CloudTrail-Digest/us-west-2/2020/09/11/144492464627_CloudTrail-Digest_us-west-2_leh-ct-test_us-west-2_20200911T183649Z.json.gz","previousDigestHashValue":"531914fcfa0dbacf09cdd4175a1fdcb5eda6e852911209f3c3ecb0a39036c860","previousDigestHashAlgorithm":"SHA-256","previousDigestSignature":"10e8072f3f2a1d2990dcc98e9d44c88a6ae2ada9df9c3ae6d35dfe8d5c47caf807702f1e1eec47edfeecfcc2248388f78fddfc945ae587fb4a3e7676b5547f342fa872b2a1c5baa62b33bcb528ce7c10e3748aab8f82c16c024de79d8a20128b4c9e5ec65e98a9c4e613687ecc89c224a11bb3df06ce414ff740e4acf9bd41159e77f568355011828412f1193e357866fbd0436faffb56e149af196e35a7675c3bbd0a398f43159343c35f9129d6339a2818afdb3192f4fffea9bd2d1bb0a70e5bfae1291f2133aab0ad92522aea6d0f828c1780d3f3de6b8270ab3ba42459916b0fbb8e2fba6ff9677bafe730e60f5edcc0f1580cdb4e36f3e3b03e653e9798","logFiles":[{"s3Bucket":"leh-ct-test-bucket","s3Object":"AWSLogs/144492464627/CloudTrail/us-west-2/2020/09/11/144492464627_CloudTrail_us-west-2_20200911T1930Z_l2pGqVS53QcGdAkp.json.gz","hashValue":"24078a45bb1c2e9ca444251ee8c1357644fdaebf4fee02d22250d8b6d48413c9","hashAlgorithm":"SHA-256","newestEventTime":"2020-09-11T19:26:24Z","oldestEventTime":"2020-09-11T19:26:24Z"},{"s3Bucket":"leh-ct-test-bucket","s3Object":"AWSLogs/144492464627/CloudTrail/us-west-2/2020/09/11/144492464627_CloudTrail_us-west-2_20200911T1915Z_TIKlbLnJ6IwUxqxw.json.gz","hashValue":"41eeb2ab841d03c2bb16e4549fc8fe3ac304ed7430173a91b37c878c77336169","hashAlgorithm":"SHA-256","newestEventTime":"2020-09-11T19:11:18Z","oldestEventTime":"2020-09-11T19:11:18Z"},{"s3Bucket":"leh-ct-test-bucket","s3Object":"AWSLogs/144492464627/CloudTrail/us-west-2/2020/09/11/144492464627_CloudTrail_us-west-2_20200911T1835Z_OPJhVNodH1gY760s.json.gz","hashValue":"6295aeb3b41cf021fe67e0b36f5ac51e557c41c85af6eef282d77ef056210d70","hashAlgorithm":"SHA-256","newestEventTime":"2020-09-11T18:32:04Z","oldestEventTime":"2020-09-11T18:32:04Z"},{"s3Bucket":"leh-ct-test-bucket","s3Object":"AWSLogs/144492464627/CloudTrail/us-west-2/2020/09/11/144492464627_CloudTrail_us-west-2_20200911T1925Z_zJNGzQovyNAImZV9.json.gz","hashValue":"54a29065f5cbfc192584e9245f8d3d86dfabf5714a5a5ec7dd2a0f4312b86198","hashAlgorithm":"SHA-256","newestEventTime":"2020-09-11T19:21:28Z","oldestEventTime":"2020-09-11T19:21:28Z"},{"s3Bucket":"leh-ct-test-bucket","s3Object":"AWSLogs/144492464627/CloudTrail/us-west-2/2020/09/11/144492464627_CloudTrail_us-west-2_20200911T1855Z_RqN9YzoKAJCKbejj.json.gz","hashValue":"551c8cbe75d081526b64fc799c67007652f2502d2f95bb9d5c8699ffc5ba7079","hashAlgorithm":"SHA-256","newestEventTime":"2020-09-11T18:51:21Z","oldestEventTime":"2020-09-11T18:51:21Z"},{"s3Bucket":"leh-ct-test-bucket","s3Object":"AWSLogs/144492464627/CloudTrail/us-west-2/2020/09/11/144492464627_CloudTrail_us-west-2_20200911T1850Z_jLldN7U8XrspES8p.json.gz","hashValue":"81605414e790e84dff20da66235f07134f77bb5c4863729bafe7672a98f07cb0","hashAlgorithm":"SHA-256","newestEventTime":"2020-09-11T18:46:45Z","oldestEventTime":"2020-09-11T18:46:45Z"},{"s3Bucket":"leh-ct-test-bucket","s3Object":"AWSLogs/144492464627/CloudTrail/us-west-2/2020/09/11/144492464627_CloudTrail_us-west-2_20200911T1905Z_jBNdmg4bSGxZ3wC8.json.gz","hashValue":"45050ec665636f9185f5b5a1e43c745a8282cb2e504092a452f0a4dc1fb8a6d5","hashAlgorithm":"SHA-256","newestEventTime":"2020-09-11T19:01:06Z","oldestEventTime":"2020-09-11T19:01:06Z"},{"s3Bucket":"leh-ct-test-bucket","s3Object":"AWSLogs/144492464627/CloudTrail/us-west-2/2020/09/11/144492464627_CloudTrail_us-west-2_20200911T1920Z_bj5DRrmILF6jK23a.json.gz","hashValue":"60ed8fcbd172d3f6d1ca4fa872681f429b005ed8a81fc9450aa06db92ccabd47","hashAlgorithm":"SHA-256","newestEventTime":"2020-09-11T19:16:12Z","oldestEventTime":"2020-09-11T19:16:12Z"},{"s3Bucket":"leh-ct-test-bucket","s3Object":"AWSLogs/144492464627/CloudTrail/us-west-2/2020/09/11/144492464627_CloudTrail_us-west-2_20200911T1900Z_6LjrkrhsLQMzCiSN.json.gz","hashValue":"b2b0e2804d1c6b92d76eee203d7eba32d30d03e696f7175723a83cec2d7ad4ba","hashAlgorithm":"SHA-256","newestEventTime":"2020-09-11T18:56:05Z","oldestEventTime":"2020-09-11T18:56:05Z"},{"s3Bucket":"leh-ct-test-bucket","s3Object":"AWSLogs/144492464627/CloudTrail/us-west-2/2020/09/11/144492464627_CloudTrail_us-west-2_20200911T1910Z_DLyqye8LaeoD204N.json.gz","hashValue":"4397a13565a67d9e6de577379b8eb7e61ca5b2b191c9b5da0243136dfc5581c7","hashAlgorithm":"SHA-256","newestEventTime":"2020-09-11T19:06:31Z","oldestEventTime":"2020-09-11T19:06:31Z"},{"s3Bucket":"leh-ct-test-bucket","s3Object":"AWSLogs/144492464627/CloudTrail/us-west-2/2020/09/11/144492464627_CloudTrail_us-west-2_20200911T1845Z_TSDKyASOn2ejOq5n.json.gz","hashValue":"49f09d2389632c70bc0066ed5d65768632dd2e60ed9c80af9d0c2c5f59bd60b6","hashAlgorithm":"SHA-256","newestEventTime":"2020-09-11T18:41:58Z","oldestEventTime":"2020-09-11T18:41:58Z"},{"s3Bucket":"leh-ct-test-bucket","s3Object":"AWSLogs/144492464627/CloudTrail/us-west-2/2020/09/11/144492464627_CloudTrail_us-west-2_20200911T1840Z_btJydJ2t7hCRnjsN.json.gz","hashValue":"0944f9a05d70868bc6f6048d58fd00764ab65e123b8ffefb193b22ca4394d771","hashAlgorithm":"SHA-256","newestEventTime":"2020-09-11T18:37:10Z","oldestEventTime":"2020-09-11T18:37:10Z"}]} +{"awsAccountId":"123456789123","digestStartTime":"2020-09-11T18:36:49Z","digestEndTime":"2020-09-11T19:36:49Z","digestS3Bucket":"alice-bucket","digestS3Object":"AWSLogs/123456789123/CloudTrail-Digest/us-west-2/2020/09/11/123456789123_CloudTrail-Digest_us-west-2_leh-ct-test_us-west-2_20200911T193649Z.json.gz","digestPublicKeyFingerprint":"47aaa19f7eec22e9bd0b5e58cfade8cb","digestSignatureAlgorithm":"SHA256withRSA","newestEventTime":"2020-09-11T19:26:24Z","oldestEventTime":"2020-09-11T18:32:04Z","previousDigestS3Bucket":"alice-bucket","previousDigestS3Object":"AWSLogs/123456789123/CloudTrail-Digest/us-west-2/2020/09/11/123456789123_CloudTrail-Digest_us-west-2_leh-ct-test_us-west-2_20200911T183649Z.json.gz","previousDigestHashValue":"531914fcfa0dbacf0c9dd1475a1fdcb5dea6e85921409f3c3ec0ba39063c860","previousDigestHashAlgorithm":"SHA-256","previousDigestSignature":"10e0872f32fa1d299d0cc98e94d4c88a6a2eada9d9fc3ae6d53dfe8d54c7caf807072f1e1eec47efdeecfcc22483887f8fddfc954ae587fba43e7676b5547f432fa8722ba1c5baa6b233bcb528ce7c01e3748aab8f28c16c024de79da820128b4c9e5ce65e98a9c4e631687ecc89c224a11bb3df06ce441ff740e4ac9fbd41159e77f5863550118284121f193e357866fbd0463faffb56e194af196e35a7675c3bbd0a398f43159343c3f59129d6339a281a8fdb3192f3fffea9bd21dbb0a705ebfae1921f2133aab0ad29522aea6df0828c1780d3f3ed6b8270ab3ba24459916b0fbbe82fba6ff9677bafe7306e0f5edcc0f1508cdb4e36f3e3b30e653e9987","logFiles":[{"s3Bucket":"alice-bucket","s3Object":"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1930Z_l2pGqVS53QcGdAkp.json.gz","hashValue":"420784a5bbc12e9ac442451e8ec1356744fdeabf4fee0d2222508db6d448139c","hashAlgorithm":"SHA-256","newestEventTime":"2020-09-11T19:26:24Z","oldestEventTime":"2020-09-11T19:26:24Z"},{"s3Bucket":"alice-bucket","s3Object":"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1915Z_TIKlbLnJ6IwUxqxw.json.gz","hashValue":"4e1eb2a8b41d032cbb16e5449fc8f3eac304e7d43017a391b37c788c77336196","hashAlgorithm":"SHA-256","newestEventTime":"2020-09-11T19:11:18Z","oldestEventTime":"2020-09-11T19:11:18Z"},{"s3Bucket":"alice-bucket","s3Object":"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1835Z_OPJhVNodH1gY760s.json.gz","hashValue":"2695aeb3b4c1f021fe76e0b36f5ac15e557c41c58af6eef282d77ef056210d70","hashAlgorithm":"SHA-256","newestEventTime":"2020-09-11T18:32:04Z","oldestEventTime":"2020-09-11T18:32:04Z"},{"s3Bucket":"alice-bucket","s3Object":"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1925Z_zJNGzQovyNAImZV9.json.gz","hashValue":"45a2906f55cbfc912584e9425f8d3d8d6fabf571a45a5ecd7d2a0f4132b81689","hashAlgorithm":"SHA-256","newestEventTime":"2020-09-11T19:21:28Z","oldestEventTime":"2020-09-11T19:21:28Z"},{"s3Bucket":"alice-bucket","s3Object":"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1855Z_RqN9YzoKAJCKbejj.json.gz","hashValue":"515cc8be750d815266b4fc799c7600765f22502d29f5bb9d5c8969ffc5ab7097","hashAlgorithm":"SHA-256","newestEventTime":"2020-09-11T18:51:21Z","oldestEventTime":"2020-09-11T18:51:21Z"},{"s3Bucket":"alice-bucket","s3Object":"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1850Z_jLldN7U8XrspES8p.json.gz","hashValue":"18650414e79e084dff02da66253f071347f7bb5c4863279bafe7762a980f7c0b","hashAlgorithm":"SHA-256","newestEventTime":"2020-09-11T18:46:45Z","oldestEventTime":"2020-09-11T18:46:45Z"},{"s3Bucket":"alice-bucket","s3Object":"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1905Z_jBNdmg4bSGxZ3wC8.json.gz","hashValue":"54050ec665636f1985f5b51ae43c74a58282cb2e500492a45f20a4dc1bf8a6d5","hashAlgorithm":"SHA-256","newestEventTime":"2020-09-11T19:01:06Z","oldestEventTime":"2020-09-11T19:01:06Z"},{"s3Bucket":"alice-bucket","s3Object":"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1920Z_bj5DRrmILF6jK23a.json.gz","hashValue":"6e0d8fcbd712d3f6d1caf4a872681f4290b05ed8a8f1c9450a0a6db92ccab4d7","hashAlgorithm":"SHA-256","newestEventTime":"2020-09-11T19:16:12Z","oldestEventTime":"2020-09-11T19:16:12Z"},{"s3Bucket":"alice-bucket","s3Object":"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1900Z_6LjrkrhsLQMzCiSN.json.gz","hashValue":"b2b0e2804d1c6b92d76eee203d7eba32d3d003e6967f175723a83ecc2d7ad4ba","hashAlgorithm":"SHA-256","newestEventTime":"2020-09-11T18:56:05Z","oldestEventTime":"2020-09-11T18:56:05Z"},{"s3Bucket":"alice-bucket","s3Object":"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1910Z_DLyqye8LaeoD204N.json.gz","hashValue":"4397a13565a67d9ed6e57737b98eb7e61ca52bb191c9b5da0423136dfc5581c7","hashAlgorithm":"SHA-256","newestEventTime":"2020-09-11T19:06:31Z","oldestEventTime":"2020-09-11T19:06:31Z"},{"s3Bucket":"alice-bucket","s3Object":"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1845Z_TSDKyASOn2ejOq5n.json.gz","hashValue":"94f09d2398632c7b0c0066ed5d56768632dd2e06ed9c80af9d0c2c5f59bd60b6","hashAlgorithm":"SHA-256","newestEventTime":"2020-09-11T18:41:58Z","oldestEventTime":"2020-09-11T18:41:58Z"},{"s3Bucket":"alice-bucket","s3Object":"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1840Z_btJydJ2t7hCRnjsN.json.gz","hashValue":"9044f9a05d70688bc6f6048d5f8d00764ab65e132b8ffefb193b22ca4394d771","hashAlgorithm":"SHA-256","newestEventTime":"2020-09-11T18:37:10Z","oldestEventTime":"2020-09-11T18:37:10Z"}]} diff --git a/x-pack/filebeat/module/aws/cloudtrail/test/cloudtrail-digest-json.log-expected.json b/x-pack/filebeat/module/aws/cloudtrail/test/cloudtrail-digest-json.log-expected.json index 9a628cb3488..e4cf4e32a06 100644 --- a/x-pack/filebeat/module/aws/cloudtrail/test/cloudtrail-digest-json.log-expected.json +++ b/x-pack/filebeat/module/aws/cloudtrail/test/cloudtrail-digest-json.log-expected.json @@ -5,116 +5,116 @@ "aws.cloudtrail.digest.log_files": [ { "hashAlgorithm": "SHA-256", - "hashValue": "24078a45bb1c2e9ca444251ee8c1357644fdaebf4fee02d22250d8b6d48413c9", + "hashValue": "420784a5bbc12e9ac442451e8ec1356744fdeabf4fee0d2222508db6d448139c", "newestEventTime": "2020-09-11T19:26:24Z", "oldestEventTime": "2020-09-11T19:26:24Z", - "s3Bucket": "leh-ct-test-bucket", - "s3Object": "AWSLogs/144492464627/CloudTrail/us-west-2/2020/09/11/144492464627_CloudTrail_us-west-2_20200911T1930Z_l2pGqVS53QcGdAkp.json.gz" + "s3Bucket": "alice-bucket", + "s3Object": "AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1930Z_l2pGqVS53QcGdAkp.json.gz" }, { "hashAlgorithm": "SHA-256", - "hashValue": "41eeb2ab841d03c2bb16e4549fc8fe3ac304ed7430173a91b37c878c77336169", + "hashValue": "4e1eb2a8b41d032cbb16e5449fc8f3eac304e7d43017a391b37c788c77336196", "newestEventTime": "2020-09-11T19:11:18Z", "oldestEventTime": "2020-09-11T19:11:18Z", - "s3Bucket": "leh-ct-test-bucket", - "s3Object": "AWSLogs/144492464627/CloudTrail/us-west-2/2020/09/11/144492464627_CloudTrail_us-west-2_20200911T1915Z_TIKlbLnJ6IwUxqxw.json.gz" + "s3Bucket": "alice-bucket", + "s3Object": "AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1915Z_TIKlbLnJ6IwUxqxw.json.gz" }, { "hashAlgorithm": "SHA-256", - "hashValue": "6295aeb3b41cf021fe67e0b36f5ac51e557c41c85af6eef282d77ef056210d70", + "hashValue": "2695aeb3b4c1f021fe76e0b36f5ac15e557c41c58af6eef282d77ef056210d70", "newestEventTime": "2020-09-11T18:32:04Z", "oldestEventTime": "2020-09-11T18:32:04Z", - "s3Bucket": "leh-ct-test-bucket", - "s3Object": "AWSLogs/144492464627/CloudTrail/us-west-2/2020/09/11/144492464627_CloudTrail_us-west-2_20200911T1835Z_OPJhVNodH1gY760s.json.gz" + "s3Bucket": "alice-bucket", + "s3Object": "AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1835Z_OPJhVNodH1gY760s.json.gz" }, { "hashAlgorithm": "SHA-256", - "hashValue": "54a29065f5cbfc192584e9245f8d3d86dfabf5714a5a5ec7dd2a0f4312b86198", + "hashValue": "45a2906f55cbfc912584e9425f8d3d8d6fabf571a45a5ecd7d2a0f4132b81689", "newestEventTime": "2020-09-11T19:21:28Z", "oldestEventTime": "2020-09-11T19:21:28Z", - "s3Bucket": "leh-ct-test-bucket", - "s3Object": "AWSLogs/144492464627/CloudTrail/us-west-2/2020/09/11/144492464627_CloudTrail_us-west-2_20200911T1925Z_zJNGzQovyNAImZV9.json.gz" + "s3Bucket": "alice-bucket", + "s3Object": "AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1925Z_zJNGzQovyNAImZV9.json.gz" }, { "hashAlgorithm": "SHA-256", - "hashValue": "551c8cbe75d081526b64fc799c67007652f2502d2f95bb9d5c8699ffc5ba7079", + "hashValue": "515cc8be750d815266b4fc799c7600765f22502d29f5bb9d5c8969ffc5ab7097", "newestEventTime": "2020-09-11T18:51:21Z", "oldestEventTime": "2020-09-11T18:51:21Z", - "s3Bucket": "leh-ct-test-bucket", - "s3Object": "AWSLogs/144492464627/CloudTrail/us-west-2/2020/09/11/144492464627_CloudTrail_us-west-2_20200911T1855Z_RqN9YzoKAJCKbejj.json.gz" + "s3Bucket": "alice-bucket", + "s3Object": "AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1855Z_RqN9YzoKAJCKbejj.json.gz" }, { "hashAlgorithm": "SHA-256", - "hashValue": "81605414e790e84dff20da66235f07134f77bb5c4863729bafe7672a98f07cb0", + "hashValue": "18650414e79e084dff02da66253f071347f7bb5c4863279bafe7762a980f7c0b", "newestEventTime": "2020-09-11T18:46:45Z", "oldestEventTime": "2020-09-11T18:46:45Z", - "s3Bucket": "leh-ct-test-bucket", - "s3Object": "AWSLogs/144492464627/CloudTrail/us-west-2/2020/09/11/144492464627_CloudTrail_us-west-2_20200911T1850Z_jLldN7U8XrspES8p.json.gz" + "s3Bucket": "alice-bucket", + "s3Object": "AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1850Z_jLldN7U8XrspES8p.json.gz" }, { "hashAlgorithm": "SHA-256", - "hashValue": "45050ec665636f9185f5b5a1e43c745a8282cb2e504092a452f0a4dc1fb8a6d5", + "hashValue": "54050ec665636f1985f5b51ae43c74a58282cb2e500492a45f20a4dc1bf8a6d5", "newestEventTime": "2020-09-11T19:01:06Z", "oldestEventTime": "2020-09-11T19:01:06Z", - "s3Bucket": "leh-ct-test-bucket", - "s3Object": "AWSLogs/144492464627/CloudTrail/us-west-2/2020/09/11/144492464627_CloudTrail_us-west-2_20200911T1905Z_jBNdmg4bSGxZ3wC8.json.gz" + "s3Bucket": "alice-bucket", + "s3Object": "AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1905Z_jBNdmg4bSGxZ3wC8.json.gz" }, { "hashAlgorithm": "SHA-256", - "hashValue": "60ed8fcbd172d3f6d1ca4fa872681f429b005ed8a81fc9450aa06db92ccabd47", + "hashValue": "6e0d8fcbd712d3f6d1caf4a872681f4290b05ed8a8f1c9450a0a6db92ccab4d7", "newestEventTime": "2020-09-11T19:16:12Z", "oldestEventTime": "2020-09-11T19:16:12Z", - "s3Bucket": "leh-ct-test-bucket", - "s3Object": "AWSLogs/144492464627/CloudTrail/us-west-2/2020/09/11/144492464627_CloudTrail_us-west-2_20200911T1920Z_bj5DRrmILF6jK23a.json.gz" + "s3Bucket": "alice-bucket", + "s3Object": "AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1920Z_bj5DRrmILF6jK23a.json.gz" }, { "hashAlgorithm": "SHA-256", - "hashValue": "b2b0e2804d1c6b92d76eee203d7eba32d30d03e696f7175723a83cec2d7ad4ba", + "hashValue": "b2b0e2804d1c6b92d76eee203d7eba32d3d003e6967f175723a83ecc2d7ad4ba", "newestEventTime": "2020-09-11T18:56:05Z", "oldestEventTime": "2020-09-11T18:56:05Z", - "s3Bucket": "leh-ct-test-bucket", - "s3Object": "AWSLogs/144492464627/CloudTrail/us-west-2/2020/09/11/144492464627_CloudTrail_us-west-2_20200911T1900Z_6LjrkrhsLQMzCiSN.json.gz" + "s3Bucket": "alice-bucket", + "s3Object": "AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1900Z_6LjrkrhsLQMzCiSN.json.gz" }, { "hashAlgorithm": "SHA-256", - "hashValue": "4397a13565a67d9e6de577379b8eb7e61ca5b2b191c9b5da0243136dfc5581c7", + "hashValue": "4397a13565a67d9ed6e57737b98eb7e61ca52bb191c9b5da0423136dfc5581c7", "newestEventTime": "2020-09-11T19:06:31Z", "oldestEventTime": "2020-09-11T19:06:31Z", - "s3Bucket": "leh-ct-test-bucket", - "s3Object": "AWSLogs/144492464627/CloudTrail/us-west-2/2020/09/11/144492464627_CloudTrail_us-west-2_20200911T1910Z_DLyqye8LaeoD204N.json.gz" + "s3Bucket": "alice-bucket", + "s3Object": "AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1910Z_DLyqye8LaeoD204N.json.gz" }, { "hashAlgorithm": "SHA-256", - "hashValue": "49f09d2389632c70bc0066ed5d65768632dd2e60ed9c80af9d0c2c5f59bd60b6", + "hashValue": "94f09d2398632c7b0c0066ed5d56768632dd2e06ed9c80af9d0c2c5f59bd60b6", "newestEventTime": "2020-09-11T18:41:58Z", "oldestEventTime": "2020-09-11T18:41:58Z", - "s3Bucket": "leh-ct-test-bucket", - "s3Object": "AWSLogs/144492464627/CloudTrail/us-west-2/2020/09/11/144492464627_CloudTrail_us-west-2_20200911T1845Z_TSDKyASOn2ejOq5n.json.gz" + "s3Bucket": "alice-bucket", + "s3Object": "AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1845Z_TSDKyASOn2ejOq5n.json.gz" }, { "hashAlgorithm": "SHA-256", - "hashValue": "0944f9a05d70868bc6f6048d58fd00764ab65e123b8ffefb193b22ca4394d771", + "hashValue": "9044f9a05d70688bc6f6048d5f8d00764ab65e132b8ffefb193b22ca4394d771", "newestEventTime": "2020-09-11T18:37:10Z", "oldestEventTime": "2020-09-11T18:37:10Z", - "s3Bucket": "leh-ct-test-bucket", - "s3Object": "AWSLogs/144492464627/CloudTrail/us-west-2/2020/09/11/144492464627_CloudTrail_us-west-2_20200911T1840Z_btJydJ2t7hCRnjsN.json.gz" + "s3Bucket": "alice-bucket", + "s3Object": "AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1840Z_btJydJ2t7hCRnjsN.json.gz" } ], "aws.cloudtrail.digest.newest_event_time": "2020-09-11T19:26:24.000Z", "aws.cloudtrail.digest.oldest_event_time": "2020-09-11T18:32:04.000Z", "aws.cloudtrail.digest.previous_hash_algorithm": "SHA-256", - "aws.cloudtrail.digest.previous_s3_bucket": "leh-ct-test-bucket", - "aws.cloudtrail.digest.s3_bucket": "leh-ct-test-bucket", + "aws.cloudtrail.digest.previous_s3_bucket": "alice-bucket", + "aws.cloudtrail.digest.s3_bucket": "alice-bucket", "aws.cloudtrail.digest.signature_algorithm": "SHA256withRSA", "aws.cloudtrail.digest.start_time": "2020-09-11T18:36:49.000Z", - "cloud.account.id": "144492464627", + "cloud.account.id": "123456789123", "event.dataset": "aws.cloudtrail", "event.kind": "event", "event.module": "aws", - "event.original": "{\"awsAccountId\":\"144492464627\",\"digestStartTime\":\"2020-09-11T18:36:49Z\",\"digestEndTime\":\"2020-09-11T19:36:49Z\",\"digestS3Bucket\":\"leh-ct-test-bucket\",\"digestS3Object\":\"AWSLogs/144492464627/CloudTrail-Digest/us-west-2/2020/09/11/144492464627_CloudTrail-Digest_us-west-2_leh-ct-test_us-west-2_20200911T193649Z.json.gz\",\"digestPublicKeyFingerprint\":\"4a7aa197feec25e9bd0b5e98fcad8ecb\",\"digestSignatureAlgorithm\":\"SHA256withRSA\",\"newestEventTime\":\"2020-09-11T19:26:24Z\",\"oldestEventTime\":\"2020-09-11T18:32:04Z\",\"previousDigestS3Bucket\":\"leh-ct-test-bucket\",\"previousDigestS3Object\":\"AWSLogs/144492464627/CloudTrail-Digest/us-west-2/2020/09/11/144492464627_CloudTrail-Digest_us-west-2_leh-ct-test_us-west-2_20200911T183649Z.json.gz\",\"previousDigestHashValue\":\"531914fcfa0dbacf09cdd4175a1fdcb5eda6e852911209f3c3ecb0a39036c860\",\"previousDigestHashAlgorithm\":\"SHA-256\",\"previousDigestSignature\":\"10e8072f3f2a1d2990dcc98e9d44c88a6ae2ada9df9c3ae6d35dfe8d5c47caf807702f1e1eec47edfeecfcc2248388f78fddfc945ae587fb4a3e7676b5547f342fa872b2a1c5baa62b33bcb528ce7c10e3748aab8f82c16c024de79d8a20128b4c9e5ec65e98a9c4e613687ecc89c224a11bb3df06ce414ff740e4acf9bd41159e77f568355011828412f1193e357866fbd0436faffb56e149af196e35a7675c3bbd0a398f43159343c35f9129d6339a2818afdb3192f4fffea9bd2d1bb0a70e5bfae1291f2133aab0ad92522aea6d0f828c1780d3f3de6b8270ab3ba42459916b0fbb8e2fba6ff9677bafe730e60f5edcc0f1580cdb4e36f3e3b03e653e9798\",\"logFiles\":[{\"s3Bucket\":\"leh-ct-test-bucket\",\"s3Object\":\"AWSLogs/144492464627/CloudTrail/us-west-2/2020/09/11/144492464627_CloudTrail_us-west-2_20200911T1930Z_l2pGqVS53QcGdAkp.json.gz\",\"hashValue\":\"24078a45bb1c2e9ca444251ee8c1357644fdaebf4fee02d22250d8b6d48413c9\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T19:26:24Z\",\"oldestEventTime\":\"2020-09-11T19:26:24Z\"},{\"s3Bucket\":\"leh-ct-test-bucket\",\"s3Object\":\"AWSLogs/144492464627/CloudTrail/us-west-2/2020/09/11/144492464627_CloudTrail_us-west-2_20200911T1915Z_TIKlbLnJ6IwUxqxw.json.gz\",\"hashValue\":\"41eeb2ab841d03c2bb16e4549fc8fe3ac304ed7430173a91b37c878c77336169\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T19:11:18Z\",\"oldestEventTime\":\"2020-09-11T19:11:18Z\"},{\"s3Bucket\":\"leh-ct-test-bucket\",\"s3Object\":\"AWSLogs/144492464627/CloudTrail/us-west-2/2020/09/11/144492464627_CloudTrail_us-west-2_20200911T1835Z_OPJhVNodH1gY760s.json.gz\",\"hashValue\":\"6295aeb3b41cf021fe67e0b36f5ac51e557c41c85af6eef282d77ef056210d70\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T18:32:04Z\",\"oldestEventTime\":\"2020-09-11T18:32:04Z\"},{\"s3Bucket\":\"leh-ct-test-bucket\",\"s3Object\":\"AWSLogs/144492464627/CloudTrail/us-west-2/2020/09/11/144492464627_CloudTrail_us-west-2_20200911T1925Z_zJNGzQovyNAImZV9.json.gz\",\"hashValue\":\"54a29065f5cbfc192584e9245f8d3d86dfabf5714a5a5ec7dd2a0f4312b86198\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T19:21:28Z\",\"oldestEventTime\":\"2020-09-11T19:21:28Z\"},{\"s3Bucket\":\"leh-ct-test-bucket\",\"s3Object\":\"AWSLogs/144492464627/CloudTrail/us-west-2/2020/09/11/144492464627_CloudTrail_us-west-2_20200911T1855Z_RqN9YzoKAJCKbejj.json.gz\",\"hashValue\":\"551c8cbe75d081526b64fc799c67007652f2502d2f95bb9d5c8699ffc5ba7079\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T18:51:21Z\",\"oldestEventTime\":\"2020-09-11T18:51:21Z\"},{\"s3Bucket\":\"leh-ct-test-bucket\",\"s3Object\":\"AWSLogs/144492464627/CloudTrail/us-west-2/2020/09/11/144492464627_CloudTrail_us-west-2_20200911T1850Z_jLldN7U8XrspES8p.json.gz\",\"hashValue\":\"81605414e790e84dff20da66235f07134f77bb5c4863729bafe7672a98f07cb0\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T18:46:45Z\",\"oldestEventTime\":\"2020-09-11T18:46:45Z\"},{\"s3Bucket\":\"leh-ct-test-bucket\",\"s3Object\":\"AWSLogs/144492464627/CloudTrail/us-west-2/2020/09/11/144492464627_CloudTrail_us-west-2_20200911T1905Z_jBNdmg4bSGxZ3wC8.json.gz\",\"hashValue\":\"45050ec665636f9185f5b5a1e43c745a8282cb2e504092a452f0a4dc1fb8a6d5\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T19:01:06Z\",\"oldestEventTime\":\"2020-09-11T19:01:06Z\"},{\"s3Bucket\":\"leh-ct-test-bucket\",\"s3Object\":\"AWSLogs/144492464627/CloudTrail/us-west-2/2020/09/11/144492464627_CloudTrail_us-west-2_20200911T1920Z_bj5DRrmILF6jK23a.json.gz\",\"hashValue\":\"60ed8fcbd172d3f6d1ca4fa872681f429b005ed8a81fc9450aa06db92ccabd47\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T19:16:12Z\",\"oldestEventTime\":\"2020-09-11T19:16:12Z\"},{\"s3Bucket\":\"leh-ct-test-bucket\",\"s3Object\":\"AWSLogs/144492464627/CloudTrail/us-west-2/2020/09/11/144492464627_CloudTrail_us-west-2_20200911T1900Z_6LjrkrhsLQMzCiSN.json.gz\",\"hashValue\":\"b2b0e2804d1c6b92d76eee203d7eba32d30d03e696f7175723a83cec2d7ad4ba\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T18:56:05Z\",\"oldestEventTime\":\"2020-09-11T18:56:05Z\"},{\"s3Bucket\":\"leh-ct-test-bucket\",\"s3Object\":\"AWSLogs/144492464627/CloudTrail/us-west-2/2020/09/11/144492464627_CloudTrail_us-west-2_20200911T1910Z_DLyqye8LaeoD204N.json.gz\",\"hashValue\":\"4397a13565a67d9e6de577379b8eb7e61ca5b2b191c9b5da0243136dfc5581c7\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T19:06:31Z\",\"oldestEventTime\":\"2020-09-11T19:06:31Z\"},{\"s3Bucket\":\"leh-ct-test-bucket\",\"s3Object\":\"AWSLogs/144492464627/CloudTrail/us-west-2/2020/09/11/144492464627_CloudTrail_us-west-2_20200911T1845Z_TSDKyASOn2ejOq5n.json.gz\",\"hashValue\":\"49f09d2389632c70bc0066ed5d65768632dd2e60ed9c80af9d0c2c5f59bd60b6\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T18:41:58Z\",\"oldestEventTime\":\"2020-09-11T18:41:58Z\"},{\"s3Bucket\":\"leh-ct-test-bucket\",\"s3Object\":\"AWSLogs/144492464627/CloudTrail/us-west-2/2020/09/11/144492464627_CloudTrail_us-west-2_20200911T1840Z_btJydJ2t7hCRnjsN.json.gz\",\"hashValue\":\"0944f9a05d70868bc6f6048d58fd00764ab65e123b8ffefb193b22ca4394d771\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T18:37:10Z\",\"oldestEventTime\":\"2020-09-11T18:37:10Z\"}]}", + "event.original": "{\"awsAccountId\":\"123456789123\",\"digestStartTime\":\"2020-09-11T18:36:49Z\",\"digestEndTime\":\"2020-09-11T19:36:49Z\",\"digestS3Bucket\":\"alice-bucket\",\"digestS3Object\":\"AWSLogs/123456789123/CloudTrail-Digest/us-west-2/2020/09/11/123456789123_CloudTrail-Digest_us-west-2_leh-ct-test_us-west-2_20200911T193649Z.json.gz\",\"digestPublicKeyFingerprint\":\"47aaa19f7eec22e9bd0b5e58cfade8cb\",\"digestSignatureAlgorithm\":\"SHA256withRSA\",\"newestEventTime\":\"2020-09-11T19:26:24Z\",\"oldestEventTime\":\"2020-09-11T18:32:04Z\",\"previousDigestS3Bucket\":\"alice-bucket\",\"previousDigestS3Object\":\"AWSLogs/123456789123/CloudTrail-Digest/us-west-2/2020/09/11/123456789123_CloudTrail-Digest_us-west-2_leh-ct-test_us-west-2_20200911T183649Z.json.gz\",\"previousDigestHashValue\":\"531914fcfa0dbacf0c9dd1475a1fdcb5dea6e85921409f3c3ec0ba39063c860\",\"previousDigestHashAlgorithm\":\"SHA-256\",\"previousDigestSignature\":\"10e0872f32fa1d299d0cc98e94d4c88a6a2eada9d9fc3ae6d53dfe8d54c7caf807072f1e1eec47efdeecfcc22483887f8fddfc954ae587fba43e7676b5547f432fa8722ba1c5baa6b233bcb528ce7c01e3748aab8f28c16c024de79da820128b4c9e5ce65e98a9c4e631687ecc89c224a11bb3df06ce441ff740e4ac9fbd41159e77f5863550118284121f193e357866fbd0463faffb56e194af196e35a7675c3bbd0a398f43159343c3f59129d6339a281a8fdb3192f3fffea9bd21dbb0a705ebfae1921f2133aab0ad29522aea6df0828c1780d3f3ed6b8270ab3ba24459916b0fbbe82fba6ff9677bafe7306e0f5edcc0f1508cdb4e36f3e3b30e653e9987\",\"logFiles\":[{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1930Z_l2pGqVS53QcGdAkp.json.gz\",\"hashValue\":\"420784a5bbc12e9ac442451e8ec1356744fdeabf4fee0d2222508db6d448139c\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T19:26:24Z\",\"oldestEventTime\":\"2020-09-11T19:26:24Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1915Z_TIKlbLnJ6IwUxqxw.json.gz\",\"hashValue\":\"4e1eb2a8b41d032cbb16e5449fc8f3eac304e7d43017a391b37c788c77336196\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T19:11:18Z\",\"oldestEventTime\":\"2020-09-11T19:11:18Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1835Z_OPJhVNodH1gY760s.json.gz\",\"hashValue\":\"2695aeb3b4c1f021fe76e0b36f5ac15e557c41c58af6eef282d77ef056210d70\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T18:32:04Z\",\"oldestEventTime\":\"2020-09-11T18:32:04Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1925Z_zJNGzQovyNAImZV9.json.gz\",\"hashValue\":\"45a2906f55cbfc912584e9425f8d3d8d6fabf571a45a5ecd7d2a0f4132b81689\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T19:21:28Z\",\"oldestEventTime\":\"2020-09-11T19:21:28Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1855Z_RqN9YzoKAJCKbejj.json.gz\",\"hashValue\":\"515cc8be750d815266b4fc799c7600765f22502d29f5bb9d5c8969ffc5ab7097\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T18:51:21Z\",\"oldestEventTime\":\"2020-09-11T18:51:21Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1850Z_jLldN7U8XrspES8p.json.gz\",\"hashValue\":\"18650414e79e084dff02da66253f071347f7bb5c4863279bafe7762a980f7c0b\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T18:46:45Z\",\"oldestEventTime\":\"2020-09-11T18:46:45Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1905Z_jBNdmg4bSGxZ3wC8.json.gz\",\"hashValue\":\"54050ec665636f1985f5b51ae43c74a58282cb2e500492a45f20a4dc1bf8a6d5\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T19:01:06Z\",\"oldestEventTime\":\"2020-09-11T19:01:06Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1920Z_bj5DRrmILF6jK23a.json.gz\",\"hashValue\":\"6e0d8fcbd712d3f6d1caf4a872681f4290b05ed8a8f1c9450a0a6db92ccab4d7\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T19:16:12Z\",\"oldestEventTime\":\"2020-09-11T19:16:12Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1900Z_6LjrkrhsLQMzCiSN.json.gz\",\"hashValue\":\"b2b0e2804d1c6b92d76eee203d7eba32d3d003e6967f175723a83ecc2d7ad4ba\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T18:56:05Z\",\"oldestEventTime\":\"2020-09-11T18:56:05Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1910Z_DLyqye8LaeoD204N.json.gz\",\"hashValue\":\"4397a13565a67d9ed6e57737b98eb7e61ca52bb191c9b5da0423136dfc5581c7\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T19:06:31Z\",\"oldestEventTime\":\"2020-09-11T19:06:31Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1845Z_TSDKyASOn2ejOq5n.json.gz\",\"hashValue\":\"94f09d2398632c7b0c0066ed5d56768632dd2e06ed9c80af9d0c2c5f59bd60b6\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T18:41:58Z\",\"oldestEventTime\":\"2020-09-11T18:41:58Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1840Z_btJydJ2t7hCRnjsN.json.gz\",\"hashValue\":\"9044f9a05d70688bc6f6048d5f8d00764ab65e132b8ffefb193b22ca4394d771\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T18:37:10Z\",\"oldestEventTime\":\"2020-09-11T18:37:10Z\"}]}", "event.type": "info", - "file.hash.sha256": "10e8072f3f2a1d2990dcc98e9d44c88a6ae2ada9df9c3ae6d35dfe8d5c47caf807702f1e1eec47edfeecfcc2248388f78fddfc945ae587fb4a3e7676b5547f342fa872b2a1c5baa62b33bcb528ce7c10e3748aab8f82c16c024de79d8a20128b4c9e5ec65e98a9c4e613687ecc89c224a11bb3df06ce414ff740e4acf9bd41159e77f568355011828412f1193e357866fbd0436faffb56e149af196e35a7675c3bbd0a398f43159343c35f9129d6339a2818afdb3192f4fffea9bd2d1bb0a70e5bfae1291f2133aab0ad92522aea6d0f828c1780d3f3de6b8270ab3ba42459916b0fbb8e2fba6ff9677bafe730e60f5edcc0f1580cdb4e36f3e3b03e653e9798", - "file.path": "AWSLogs/144492464627/CloudTrail-Digest/us-west-2/2020/09/11/144492464627_CloudTrail-Digest_us-west-2_leh-ct-test_us-west-2_20200911T183649Z.json.gz", + "file.hash.sha256": "10e0872f32fa1d299d0cc98e94d4c88a6a2eada9d9fc3ae6d53dfe8d54c7caf807072f1e1eec47efdeecfcc22483887f8fddfc954ae587fba43e7676b5547f432fa8722ba1c5baa6b233bcb528ce7c01e3748aab8f28c16c024de79da820128b4c9e5ce65e98a9c4e631687ecc89c224a11bb3df06ce441ff740e4ac9fbd41159e77f5863550118284121f193e357866fbd0463faffb56e194af196e35a7675c3bbd0a398f43159343c3f59129d6339a281a8fdb3192f3fffea9bd21dbb0a705ebfae1921f2133aab0ad29522aea6df0828c1780d3f3ed6b8270ab3ba24459916b0fbbe82fba6ff9677bafe7306e0f5edcc0f1508cdb4e36f3e3b30e653e9987", + "file.path": "AWSLogs/123456789123/CloudTrail-Digest/us-west-2/2020/09/11/123456789123_CloudTrail-Digest_us-west-2_leh-ct-test_us-west-2_20200911T183649Z.json.gz", "fileset.name": "cloudtrail", "input.type": "log", "log.offset": 0, diff --git a/x-pack/filebeat/module/aws/cloudtrail/test/insight-json.log-expected.json b/x-pack/filebeat/module/aws/cloudtrail/test/insight-json.log-expected.json index ec3dde12e81..2bfe4bedd30 100644 --- a/x-pack/filebeat/module/aws/cloudtrail/test/insight-json.log-expected.json +++ b/x-pack/filebeat/module/aws/cloudtrail/test/insight-json.log-expected.json @@ -3,6 +3,46 @@ "@timestamp": "2020-09-09T23:00:00.000Z", "aws.cloudtrail.event_type": "AwsCloudTrailInsight", "aws.cloudtrail.event_version": "1.07", + "aws.cloudtrail.insight_details.eventName": "AttachUserPolicy", + "aws.cloudtrail.insight_details.eventSource": "iam.amazonaws.com", + "aws.cloudtrail.insight_details.insffightContext.attributions": [ + { + "attribute": "userIdentityArn", + "baseline": [], + "insight": [ + { + "average": 2.0, + "value": "arn:aws:iam::123456789012:user/Alice" + } + ] + }, + { + "attribute": "userAgent", + "baseline": [], + "insight": [ + { + "average": 2.0, + "value": "console.amazonaws.com" + } + ] + }, + { + "attribute": "errorCode", + "baseline": [], + "insight": [ + { + "average": 2.0, + "value": "null" + } + ] + } + ], + "aws.cloudtrail.insight_details.insffightContext.statistics.baseline.average": 0.0, + "aws.cloudtrail.insight_details.insffightContext.statistics.baselineDuration": 11459, + "aws.cloudtrail.insight_details.insffightContext.statistics.insight.average": 2.0, + "aws.cloudtrail.insight_details.insffightContext.statistics.insightDuration": 1, + "aws.cloudtrail.insight_details.insightType": "ApiCallRateInsight", + "aws.cloudtrail.insight_details.state": "End", "aws.cloudtrail.recipient_account_id": "123456789012", "cloud.region": "us-east-1", "event.action": "Insight", diff --git a/x-pack/filebeat/module/aws/fields.go b/x-pack/filebeat/module/aws/fields.go index 5ce70304dba..352932f1b1c 100644 --- a/x-pack/filebeat/module/aws/fields.go +++ b/x-pack/filebeat/module/aws/fields.go @@ -19,5 +19,5 @@ func init() { // AssetAws returns asset data. // This is the base64 encoded gzipped contents of module/aws. func AssetAws() string { - return "eJzcXN9z27aTf89fsdOX2jOSbr5p5+bGN70ZxXGuurqpz1LauycWIlcSzhDAAqAU56+/WQD8IRGUZItKOl89JLJIAp9d7G8sOIQnfL4BtjVvACy3Am9g/Mf0DYBGgczgDczRsjcAGZpU89xyJW/gP94AAPyqskIgLJSGFZOZ4HIJQi0NLLRa0zCjNwALjiIzN+6BIUi2xnI6+tjnHG9gqVWRh18i89DngxumGtnNMwpXm1M0p0mFKjKrGRfVpdiM9NmntvxkuGCFsImb4gYWTBjcuRwF2wSstMN7S1hmhGUHegx+kwTcoLTJBrXhSu7cUVLyhM9bpbO9aweA0We2wiaiMD6oBdgVEkA/MaFfMzuKQisM6oRnKC23z1Fo+0xuAxtGkdHIkzAwoMA1QUmVtIxLAxlaxoUBNleFdXhpNlCL1liT8a9QAgS7YhbWLEP3iMa/CjR2AExmsF3xdAWpRncvEwa2qLE1XGEwG8FkARbXudJMP7eecfcM3AwlbrNSWwMrtaVfW2O2BlBzohKz0d6tMSFprgbxoHXxsIy0lyNyg1+RwGFHWMeSN7Rb70vqy5G0BaOEMl6zL0rCIxpV6BThI1sjXI0fP16XAHPNZcpzJvbWPGVC7LO1gTpN0ZjkCZ8THsPXF34/Dw0Ek/ce4ZYZJzhgFRi+lE0J7QZs0JDSJqQY+Nl2Qo5p4amAJ4smFgfUsXPL7aqhBgbTQsdEAnZFnNStUgxHeq7VhmdogEtva8gM1ZodaIyOW7Eu1cgsZs7U2pUy2Jwy8miXKjWZu16whBV2RaOkNHr07uNScSqjIUjHhokCgRuwmv4P7FfKOqMISjuj5r5vidTOwaKWKbCoXlAmjHI83KHVLy+Ls50+v34YQ4YbnuK/g7Ir1FtucOC9Y1tgm3x1a0VSmzHbBd7z9MANL2EoDeOMvOVrhO0KvXa1ZbfNMW5M0TbEu/SUSuju1QcJ6tLDl1DUhz7C63Wyc7zg3k53Z+XnkC7CcfdWfk5RQ3gBnyFIT/AxwbEcFpoBmCJdHRySGXhUyg5IiT8Z1ANS6EclOpSmyYDKqcW906UZwaVFLZkgnxW40Yyrmh5sid1yAruyd5zseCxxaWrHjx9LKoMEXLE0VYX0S+fsr1s7rQReHxwuxp4jgnQCVzyYbyMKYXJPmdpKczlpKOnlcqOeMEvmMYvWV2BGU5WrThmbQU0eritxIGUHFosvoIxR727fwriwCqYpc8lxyAXvBDOWp/AOmTSWiad4goVaK52kKtu3fKcnfvH8qkmdm6QKNIJf0WgLLY3zDHT9EL41GsOWfUKcHAbj06vGIJVB6oYaxkpyptkaLer9dTuXpfXAA2Imk8+DoAvkBQ35Vu+juyP7dSEsT47meYdC/Y6LR4oYTTaZXEmDSYgH+uZSOX4Vb1A8ylJ6xpR69oSQrphcooErH9kP2pl4TmGds8AZCqQIzw9y/TdkKssyTuCYSFxRJWM7FaZz+TquhqdYlzWSJ1/CqSyyVJbE1AZ1aQ1U6hu5tbBOf0Nulprccn3nyqbPvBy3fIVjwdHs6KuPCIPdnCOXy3Y5hwmBGSxRombWPc+NH7rDhrr6XiS6PcuA7uIvizcNeSgBZg1B0ZgqncVhspyfXYQ8inP8MKkqkcwYlfI6GXXXt2ac81smRGskR8GM6DzA6zWTbOnsjlfEPpUQ3iklkMkOMdqukNLkBre5gX0rAA2E/q4uZ8ayREkRL7qevRQ1Vm5A5SQntCIE2E09pKnrC10YfeAc9yCvKQyPQXDjjFc1dqilYQZc1qx9aeX0cuXKqkY5fvzYDhRPCub7gDEOAXudupUcpPA9UrXotaZ8hDXOOFWaosuV9IUHtxd1c8O2Zhjs7tAhuyFPN6RH3d8dEpjynJOydzL4HIV5xFwjxXXedrGax073NabIN86+cnNImQNd3iIlYW/jcja2CvtpugFwmYoio9RkS6it5sslau8W4kbW19K8DBXi7xjEmhXTmAWG9rrm//lp8r7hOufPzT00q6CQ/K8CxXMpz83rcW6GDU23MpR+UmbmQ9ngQozPHayCjC8WqOkPvz+7+wnyZ+JCtsnTBGWWK943S/bE6/eHWygnIlX2O2shgAolQZdKO7LbDpCetwqYdFXlZqJaJdxlcj39IU5rqqRRAhOhljwerLzG+4TdXJNjyhc8JZC3fqJ7mies5ks9z/HM4DDqNvK4A6hThDunw+8pUaDc6zgNh+ho0rJWcy6wI0jcpWTuQ6WOe04uhbcjlh1qKOVxUuaRwSFkJRFOZhKrDsLvZ7Pn0+N9awUOM3jBksIc2Yi6GGudXYcFSy1l7PVmEVmqbUcVDMKeOWSFJhfTSWpJ4kIwa1G2aHy92t5N60FdeuzCDeK7mv8fptYRqH1lzxRzL+jANMrvLTxJtZVkxVi2YTJtZcO9qncX7W0qj6n4eVUAOK0S8ILS2mto7K7S9lBoO73e1S/0vqpf8KIK2MtCzf4o7jHwhJOCz5LMjC/RxJP7Mzy/cyW3VUMZvHezwL1avtTrC7VMFly0UuMapkQT7zg4Kdur8uR7tXTzlK1TdZ4cZVFDUCzTNrF83Z39dezOn6oKbgZa/U+zW78zr0novRbUCIHwQ6rId8f1wLInGob54oQLkFNvzt2oVQgq1BI8M1ZsE5ewOaIkZeIbJ4g70X23UqHMLsoplNk/BZ/MD8m8SJ+iu4EX2cErswTw01LE7kl0fQqF1u0KYJitwdIVM3v0HqTQBxUXpLCmyk/l2siuQqo4iBIeHUsoH7tVPXOBIU3iuymVuCWPH2rXFxT+SuoDyrUyvrRSFkiBrRWJtRCH4pmQagfjV4v4jjXsJleJ7NuQ6yfuoLQ7XYNXU5pr3HBVmORraOthDS2h7KhjV6ZxgnZWpK2YWSVMLJXmdrX+StaIJoVq0naXhLveJb4xVhwgtJgLnro+1gWXS9S55lFL1xedK/zMMkz5mglAmaoMM2jMXHXlOlzOYBH50eHWzKarEDDmmm+YRffAXnMsP5EVdDuzhcavst718rZ6eaNod04pbInyN/ugvs0phT8Iy4tOKRzqAYmUc4/ks7sgysHbjMP07bfn2N3tW3/mhcsG8FMZx/OEZZlG8/qSf4t9dbseWgij1xs/LiNGHeGmmJ/OzeVleHn/7kVi19r7OZtvTZstFMtgzgSTKXa0FZ21bx8F0DxosQPAMWnzFu7px3fhx45yv2V6iTZxqzdqb2+eCbHRHOkn8mJSHwHrLL1UGTA3Ftsbj2fictITRm5twlF2JCWm3TvWuVZWpWp/o+ZMUOWo8TW9Wlmbg9Jg0/z6SOeaVikaw+XShaAjg2kU6UKolnM9Re6UZaLKBQ2mSmYGDC+zxJp7viroC4Tc1DwupOUC+M6eFeWaS1oSlyiy9AllR4NJuPg3IrNBBl0JAMFyIXZ+cKUME4p7lKt3NkT4UuM3prAqQjbXrtqE3KFydy0dewTv3MiuWefpWsddmlCttqmXUVUmrI2VkrDmQvBA7CBQ6+Gr3NX+GwSlQpn9TKGynIISBJmZFXvCy9JRHsWZ3U+hmpIYnap17iq6e3SBikhpVaBAY9lccLPqIq1UP75fAD3Twk0e9oOMUohqSfcx+zELXCLMlX59b1bcCittyxz4XHRkskfVtoixzBbmvCbtKGQ/MNDAbfW8al5tK6h/gPwh6a9V2ld+/+yE/meUYmNEkvJ81bejnk7vwY/rMyUuSQn+xf1cLUJHaEOYLuOpafrKW78YV7pSBmWSorYXjbj8PEDz8IU7rQehD8mnnA0heC18g5qznpnrxwRZrOeoL0wLl6laO/cqTMIE9m1MKLdaog6dnmrhLLibp+FR58+xEL7UY0+V28AMFHe7I/qWJbGdqzPpINhucB+/dyDQLMVzWoaiU7sjy9JWHuPP/xmO11/kcEazDSfZn7BClnVlXb5KlCW6EBROcRU56nh22O5HrRfZxU6FqI4DOQi7J4L4ggJeuidc7uil9kESfsa0aG/znQk8NI+Vg/sDrzuurj5Yc7VQest0NoAF/4zZsPQMg53TwKPR6HoEEwspk+VOIhjcoGbCs6dDDzVmXGNqk0L3bE0+Pd4HC+04HuZxbeNp2ZFSseDA2aWRRmb6fq2GP1DlRy7PVVXLEfAtGBdNZPUWku9o/taFmOkPbr8edfm2gpdUZXw1P4m1Fp9rNphUkqdM+APwdT+zm2vvxKaH0RHLxfY2eiwdlRsa5S5t87x2UEYShiXjsqtGonGtLCYd0Xrr51NMQ54z7Z3pycVBaFcivtqiVvO5LhcGQyeehdx9VUHZW3q4etKnAxuDsa6fbacTuN7NijYCI6v7YONQqwMV/bK3PsDhamMZrPyrDSDDVDDKDJiB6W/jh1F15wAe76az0c+z2UOyRrtS2ag8MOBOKg3gj7t308ns7tAtSsO78ez259H7u/u72d3ot3f/dXc7i5P+hD277++e8Pm7Zptb7aTJd4RdKgfyu+F3pZWuWZUp9J1ylnJy5vagqsazw5JWaN4vLY9+4OGnx8kORcT7yrC09kia0CjtS3y212MpQxZr1Dz1OJr5aH3GJNI52OPZ5niqVKnhnfPDtyrD5jpLFRy0Sl3LQ1ex5NmiSUzXIbFXcyxkP1URzs3jMvgB4OeyJc6xtK4cb1BTONwk4wtq1WFGXF9IYviXOGfPqVvRoJXL9f0nXIKTwM6yiXsy1jXRDyd3CmT7rpZLWAi+XNnGKQsX1nxvIEdtcooaNx0SagstE6ZVEW/wugh8ZhsCbHJy1o3Y/VkV+rAPcZ1dum8PvZMDOdl8DPOENO14IuveFceW5xy6PBHaJ4N6OKaZDuaQ4TRA75lteX518r6sN1ae51RnE4aYZMdczkr1HNkQAZ+HbP1lyLPhW/e2ikoa8bNFmdUBF0zed1ToqqaLi7y1sBq+5NMApnz5u0NLX34ctLtrmhHjjpF4dVzpy5eJKXir4evcMiSmRNxUUQZh4J49o4ar6fT+uqyZ1q38uFSWV68+I/GfxkijCx2liJ2DHOcdQ4/7jbCdXb1vbPfkiH9b4riwq5+drvqO+N17vBabAfx3gfp56kNvuu8v+ruMxa9yjUOSDcwoxLt+/dI6rfKT9lwYKM/FlWIZKpn09ciZNivMZbRpppk0bnfEC9q0fB3R1ex+el1Zs4akhcLm/k5g46ThQqjt6QWMS/Xl/P5wC4TkRaWLi/CYkHwgJPdqacop3HtEn1VBwhDeZ+MIDweffStvyX5u4G31gD+C+AwM0sJYte56okOUejgMHQ+83cnV6hB0ubtZLkFXqd6iXlyiwlyXESTardJP9VwOW91oajVbLHga9sOVzg7XbfuFuXfWN/bWi4BvAOPb27uHmXtj3F13Li3U8lCu92qkQi2XZGhDpheYWy7vAH77ZQAff3s/no2dJ/5l8kDfu5bdWCYvuurlFI6137c5+wqpGJShWzU2N67y6Izisyo6+oqebGJ0yrIs7k9eU8rLGUUHQ4EbFHClNF9yycR1Wfpsb8kHcroRZsZ+FYQZ5YrSe/YGzNJcHMS5ydMLSow7ek56WL1oulfrYYq5xP7Nbo3fT3BJEmyaJwvBWgfeziRhzu2amaeQy1WOQwmhtmRxZrcP4Ka9gbc/Tf/34+Af/0b/Dce3vwz+8dOHycfBjz89TmdxyJdr0PRcu4HJw+bHAf37ry7Fu/swHr35/wAAAP//r4rS6w==" + return "eJzcXN+T4raTf9+/oisvma0Crr6b1NXVXOWq2NnJhctkMzewyd2TI+wGdCMkR5JhyV9/1ZL8AywDM5jd1JeHXQbb0qdb/VstD+EZd7fAtuYNgOVW4C2Mf5++AdAokBm8hTla9gYgQ5Nqnluu5C38xxsAgF9UVgiEhdKwYjITXC5BqKWBhVZrGmb0BmDBUWTm1j0wBMnWWE5HH7vL8RaWWhV5+CUyD31+dMNUI7t5RuFqc4rmNKlQRWY146K6FJuRPofUlp8MF6wQNnFT3MKCCYN7l6Ngm4CVdnjvCMuMsOxBj8FvkoAblDbZoDZcyb07SkqecbdVOju4dgQYfWYrbCIK44NagF0hAfQTE/o1s6MotMKgTniG0nK7i0I7ZHIb2DCKjEaehIEBBa4JSqqkZVwayNAyLgywuSqsw0uzgVq0xpqMf4ESINgVs7BmGbpHNP5ZoLEDYDKD7YqnK0g1unuZMLBFja3hCoPZCCYLsLjOlWZ613rG3TNwM5S4zUptDazUln5tjdkaQM2JSsxGB7fGhKS5GsSD1sXjMtJejsgNfkUChx1hHUve0G59KKkvR9IWjBLKeM3+UhKe0KhCpwgf2RrhZvz08W0JMNdcpjxn4mDNUybEIVsbqNMUjUmecZfwGL6+8Pt5aCCYfPAIt8w4wQGrwPClbEpoN2CDhpQ2IcXAz7YTckwLzwU8WTSxOKCOnVtuVw01MJgWOiYSsC/ipG6VYjjSc602PEMDXHpbQ2ao1uxAY3TcinWpRmYxc6bWrpTB5pSRR7tUqcnc9YIlrLArGiWl0aN3n5aKcxkNQTo2TBQI3IDV9H9gv1LWGUVQ2hk1931LpHYOFrVMgUX1gjJhlOPhHq1+eVmc7fT55ccxZLjhKf47KLtCveUGB947tgW2yVe3ViS1GbNd4D1Pj9zwEobSMM7IW75G2K7Qa1dbdtsc48YUbUO8T0+phO5efZSgLj18CUV96CO8Xic7xwvu7Xx3Vn6O6SKcdm/l5xw1hBfwGYL0BB8THMtxoRmAKdLV0SGZgSel7ICU+JNBPSCFflKiQ2maDKicWtw7XZsRXFrUkgnyWYEbzbiq6cGW2C0nsC97p8mOxxLXpnb89LGkMkjADUtTVUi/dM7+urXTSuDbo8PF2HNCkM7gigfzdUQhTO4pU1tpricNJb1cbtQzZsk8ZtH6CsxoqnLVKWMzqMnDdSUOpOzAYvEFlDHq/d07GBdWwTRlLjkOueC9YMbyFN4jk8Yy8RxPsFBrpZNUZYeW7/zEL55fNalzk1SBRvArGm2hpXGega4fw7dGY9iyT4iT42B8etUYpDJI3VDDWEnONFujRX24bpeytB54QMxkcjcIukBe0JBv9T66O7JfF8Ly5GSedyzU77h4oojRZJPJlTSYhHigby6V41fxBsWjLKVnTKlnzwjpisklGrjxkf2gnYnnFNY5C5yhQIrw/CBv/4ZMZVnGCRwTiSuqZGyvwnQpX8fV8BTrskby5Es4lUWWypKY2qAurYFKfSO3Ftbpb8jNUpNbru9S2fSZl+OWr3AsOJo9ffURYbCbc+Ry2S7nMCEwgyVK1My657nxQ3fYUFffi0S3FxnQffxl8aYhDyXArCEoGlOlszhMlvOLi5AncY4fJ1UlkhmjUl4no+761oxzfseEaI3kKJgRnUd4vWaSLZ3d8YrYpxLCe6UEMtkhRtsVUprc4DY3cGgFoIHQ39XlzFiWKCniRdeLl6LGyg2onOSEVoQAu6mHNHV9oQujD5zjHuQ1heExCG6c8arGDrU0zIDLmrUvrZxer1xZ1SjHTx/bgeJZwXwfMMYhYK9Tt5KDFL5Hqha91pRPsMYZp0pTdLmSvvDg9qJub9nWDIPdHTpkt+TphvSo+7tDAlOec1L2TgZfojBPmGukuM7bLlbz2Om+xhT5xtlXbo4pc6DLW6Qk7G1cz8ZWYT9NNwAuU1FklJpsCbXVfLlE7d1C3Mj6WpqXoUL8HYNYs2Ias8DQXtf8Pz9NPjRc53zX3EOzCgrJ/yxQ7Ep5bl6PczNsaLqVofSTMjMfygYXYnzuYBVkfLFATX/4/dn9T5A/ExeyTZ4mKLNc8b5ZciBevz3eQTkRqbLfWQsBVCgJulTakd12gPS8VcCkqyo3E9Uq4S6T6+l3cVpTJY0SmAi15PFg5TXeJ+zmmhxTvuApgbzzEz3QPGE1X+p5TmcGx1G3kccdQJ0i3Dsd/kCJAuVep2k4RkeTlrWac4EdQeI+JXMfKnXcc3YpvB2x7FFDKY+TMo8MjiEriXAyk1h1FH4/mz2fnh5aK3CcwQuWFObERtTVWOvsOixYailjrzeLyFJtO6pgEPbMISs0uZhOUksSF4JZi7JF4+vV9n5aD+rSYxduEN/V/P8wtY5A7St7pph7QQemUX5r4VmqrSQrxrINk2krG+5Vvbtob1N5SsUvqwLAeZWAF5TWXkNjd5W2h0Lb+fWufqH3Vf2CF1XAXhZq9kdxj4EnnBV8lmRmfIkmntxf4PmdK7mrGsrgg5sFHtTypV5fqGWy4KKVGtcwJZp4x8FZ2V6VJz+opZunbJ2q82TPoiOSYpm2ieXr7vSvY3v+XF1wM9Dyf5rd+a15TVLv1aCGCEQApIqcd1wRLHumYZivTrgIOfX23I1axaBCLcFzY8U2cRGbI0rSJr5xkrgX3nfzCmV2VU6hzP4p+GS+S+ZF+hzdDrzKFl6ZJoCflkJ2T6JrVCi0bpcAw2wNlq6YOaD3KIU+qrgihTVVfirXR3YTcsVBlPDoWEL54K1qmgsMaRLfTanELbn8ULy+ovBXUh9QrpXxtZWyQgpsrUishTgW0IRcO1i/WsTPNIdKZF+HXD9xB6Xd+Rq8mtJc44arwiRfQluPa2gJZU8du1KNM7SzIm3FzCphYqk0t6v1F7JGNClUk7bbJNz1LvGNseIIocVc8NQ1si64XKLONY9aur7oXOFnlmHK10wAylRlmEFj5qot1+FyBovIjw63ZjZdhYgx13zDLLoHDrpj+ZmsoNuZLTR+kfWul7fVzNuJtu5wMXy5Ol4A7orIT+Gbug70eINtITPUYkehQYjBDa0XkzDxiNr6FiL4shznjZOv5g98Zypb+jsss9xYnpqB29sjStvBiW/Aj2yP2TJMpCQHZVbK0XFgNXv3ToFsSbDeHHL065wC+Z2wvOgUyLEem0i5/ES9YB9EOXibcZi++/ocu797588UcdkAfi7jeJ6wLNNoXr+l0mJf3Q6JFsLo9caaqzigjnBTzM/n5vI6vHx4/yKxa+2tXcy3pksUimUwZ4LJFDvati7qi4gCaB5k2QPgmLR5Bw/04/vwY8d2imV6iTZxqzdqbx9fCLHRfOon8mJSH7HrLG1VFQZuyE8cbuxeiMtJTxi5tclJyaeUmHZ3BORaWZWqw42wC0GVo8bX9GZlbU7uw6b52xOdgVqlaAyXSxfhjwymHU5YtWKXc+ROWSaqVNtgqmRmwPAyCa+556uuvgDLTc3jQlougO/tCVIqv6QlcXk4S59RdjTwhIt/IzIbZNCVABAsF2LvBxcCmFA8zbhcdjac+FLuV6awKvI2167a5N2jcn8tHXsE72wUqFnn6VrHXZpQrba0l1FV1gMaKyVhzYXggdhBoNbDV7nbW2kQlAplDhOxynIKyr9kZlbsGa9LR3nUafYwhWpKYnSq1rmrmB/QBSoipVX9B41lc8HNqou0Uv34YYH5Qgs3eTwMMkohqiXdp0SnLHCJMFf69b1vcSustC1LDJeiI5M9qradKIcozGVN8FHIfmCggdvqedO82lZQ/wD5Q9Jfq7SvrP/RCf2PKMXGiCTl+apvRz2dPoAf1yeiXJIS/Iv7uVqEjtCGMF3HU9P0lbd+Ma50pQzKJEVtrxpx+XmA5uELdxoSQp+Xz+gbQvBa+AY1Zz0z148JsljPUV+ZFi5TtXbuVZiECezbmFButUQdOmnVwllwN0/Do853sRC+1GNPldsgDhR3uyP6liWxncEL6SDYbnAfv3cg0CzFS1qyolO7I+HSVh7jj/8Zjtd/yeGMZhtOsj9ghSzryrp8ES5LdCEonOIqcpT04rDdj1ovsoudClEdt3IQ9k9c8QUFvHRPuNzRq+6DJPyMadHeRr0QeGjOKwf3B4r3XF19cOlmofSW6WwAC/4Zs2HpGQZ7p61Ho9HbEUwspEyWO7VgcIOaCc+eDj3UmHGNqU0K3bM1+fT0ECy043iYxxUO07Ljp2LBkbNhI43M9P3aEn9gzY9cnlurliPgWzAumsjqHTrfMf61CzHT71w/BOrybRAvqcr4zZIk1rp9qdlgUkmeMuHLuHW/uJvr4ESsh9ERy8W2jnosHZX7ReUmePM8fFBGEoYl47KrRqJxrSwmHdF66+dzTEOeM+2d6dnFQWhXIr7YolbzuS4iBkMnnoXcfxVE2bt7vHrSpwMbg7GuX3Cv07reLIw2WiOr+4zjUKsDK/2ytz4g42pjGaz8qyMgw1QwygyYgemv48dRdecAnu6ns9FPs9ljska7UtmoPJDhToIN4Pf799PJ7P7YLUrD+/Hs7qfRh/uH+9n96Nf3/3V/N4uT/ow9u+9vnnH3TbONsHbS5DvCJqAD+c3wm9JK16zKFPpOREs5OXNbfFVj33FJKzTvl5YnP/Dw09NkjyLifWVYWnskTWiU9iU+2+uxlCGLNWqeehzNfLQ+wxPpzOzx7Hg8VarU8N754TuVYXOdpQoOWqWuo6SrWLKzaBLTdQjv1RwL2U9VhHPzuAx+APi5bDl0LK0rxxvUFA43yfgLteowI67tJjH8rzhnL6lb0aCVy/XtPVyCk8DOsol7MtaU0g8n9wpkh66WS1gIvlzZxikWF9Z8ayBHbXKKGjcdEmoLLROmVRHvn7sKfGYbAmxyctaN2H2nCn3ch7jGOd23h97LgZxsPoV5Qpp2OpF17+Jz2+7XhvbJoB6OaaajOWQ4bdF7ZlueD558KOuNlec519mEISbZKZezUj1HNkTA5yFb/zXk2fCdextIJY342aLM6oALJh86KnRVT8tV3gpZDV/yaQBTvvzNoaUv3w/azUvNiHHPSLw6rvTly8QUvNVPd2kZElMibqoogzDwwHao4WY6fXhb1kzroxK4VJZXr5Yj8Z/GSKMLHaWIvYMylx3zj/uNsJ1dvc9t/2SOfxvluLCrn5yu+hMH+/d4LTYD+O8C9W7qQ2+670/6u4zFb3KNQ5INzCjEe/v6pXVa5SftuTBQnjssxTJUMunriTODVpjraNNMM2nc7ogXtGn5uqeb2cP0bWXNGpIWCpuHO4GNk5wLobbnFzCu1Zfz2+MdEJIXlS6uwmNC8iMheVBLU07h3tO6UwUJQ3hfkCM8HCz3ndIl+7mBd9UD/ojnDhikhbFq3fVEhyj1cNg8Hni7k8HVIfNyd7Ncgq5SvUW9uEaFuS4jSLRbpZ/ruRy2uo/XarZY8DTshyudHa/b9gvz4Cx17K0iAd8Axnd3948z90a+++5cWqjlsVzv1UiFWi7J0IZMLzC3XN4B/PrzAD7++mE8GztP/PPkkb53NpNaJq+66uUUjrXftjn7CqkYlKFbNTY3rvLojOJOFR19Rc82MTplWRb3J68p5eWMooOhwA0KuFGaL7lk4m1Z+mxvyQdyuhFmxn4RhBnlitJ79gbM0lwcxbnJ0ytKjDvaT3pYvci7V+thirnE/s1ujd9PcE0SbJonC8FaBwovJGHO7ZqZ55DLVY5DCaG2ZHFmd4/gpr2Fdz9M//fj4B//Rv8Nx3c/D/7xw4+Tj4Pvf3iazuKQr9eg6bl2C5PHzfcD+vdfXYp3/+N49Ob/AwAA//+8xDjD" } From c82ffaacf79efe6b5a75226e4bcb8ec98ac31642 Mon Sep 17 00:00:00 2001 From: "Lee E. Hinman" Date: Mon, 14 Sep 2020 18:36:01 -0500 Subject: [PATCH 3/3] Fix missing } --- x-pack/filebeat/module/aws/cloudtrail/config/s3.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/filebeat/module/aws/cloudtrail/config/s3.yml b/x-pack/filebeat/module/aws/cloudtrail/config/s3.yml index 2fd6b7f1a00..2094f77c712 100644 --- a/x-pack/filebeat/module/aws/cloudtrail/config/s3.yml +++ b/x-pack/filebeat/module/aws/cloudtrail/config/s3.yml @@ -10,7 +10,7 @@ file_selectors: - regex: '^AWSLogs/\d+/CloudTrail-Digest/' {{ end }} -{{ if .process_insight_logs } +{{ if .process_insight_logs }} - regex: '^AWSLogs/\d+/CloudTrail-Insight/' expand_event_list_from_field: 'Records' {{ end }}