From 48bd77b74ad0e7c69895e00a6487ef6ba8411471 Mon Sep 17 00:00:00 2001 From: Lee Hinman <57081003+leehinman@users.noreply.github.com> Date: Mon, 14 Sep 2020 23:15:17 -0500 Subject: [PATCH] [Filebeat] Add support for Cloudtrail digest files (#21086) (#21089) * Add support for Cloudtrail digest files - allow file matching with file_selectors in s3 input - update cloudtrail pipeline - update cloudtrail config to use file_selectors - add cloudtrail digest fields - add cloudtrail insight fields Closes #20943 (cherry picked from commit c9f7a99e1736853955c07230c78649867d4e8c40) --- CHANGELOG.next.asciidoc | 1 + filebeat/docs/fields.asciidoc | 114 ++++++++++++++++ filebeat/tests/system/test_modules.py | 2 +- .../docs/inputs/input-aws-s3.asciidoc | 25 ++++ x-pack/filebeat/filebeat.reference.yml | 12 ++ x-pack/filebeat/input/s3/config.go | 16 +++ x-pack/filebeat/input/s3/input.go | 70 ++++++---- x-pack/filebeat/input/s3/input_test.go | 5 +- x-pack/filebeat/module/aws/_meta/config.yml | 12 ++ .../module/aws/cloudtrail/_meta/fields.yml | 67 ++++++++++ .../module/aws/cloudtrail/config/s3.yml | 15 ++- .../module/aws/cloudtrail/ingest/pipeline.yml | 100 +++++++++++++- .../module/aws/cloudtrail/manifest.yml | 6 + .../test/cloudtrail-digest-json.log | 1 + .../cloudtrail-digest-json.log-expected.json | 126 ++++++++++++++++++ .../aws/cloudtrail/test/insight-json.log | 1 + .../test/insight-json.log-expected.json | 64 +++++++++ x-pack/filebeat/module/aws/fields.go | 2 +- x-pack/filebeat/modules.d/aws.yml.disabled | 12 ++ 19 files changed, 614 insertions(+), 37 deletions(-) create mode 100644 x-pack/filebeat/module/aws/cloudtrail/test/cloudtrail-digest-json.log create mode 100644 x-pack/filebeat/module/aws/cloudtrail/test/cloudtrail-digest-json.log-expected.json create mode 100644 x-pack/filebeat/module/aws/cloudtrail/test/insight-json.log create mode 100644 x-pack/filebeat/module/aws/cloudtrail/test/insight-json.log-expected.json diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 3d6874efc00..680cfb74490 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -73,6 +73,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fixing `ingress_controller.` fields to be of type keyword instead of text. {issue}17834[17834] - Fixed typo in log message. {pull}17897[17897] - Fix an error updating file size being logged when EOF is reached. {pull}21048[21048] +- Fix error when processing AWS Cloudtrail Digest logs. {pull}21086[21086] {issue}20943[20943] *Heartbeat* diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index 19a3bcb4604..c503aca03ea 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -1487,6 +1487,120 @@ type: flattened -- +[float] +=== digest + +Fields from Cloudtrail Digest Logs + + +*`aws.cloudtrail.digest.log_files`*:: ++ +-- +A list of Logfiles contained in the digest. + +type: nested + +-- + +*`aws.cloudtrail.digest.start_time`*:: ++ +-- +The starting UTC time range that the digest file covers, taking as a reference the time in which log files have been delivered by CloudTrail. + +type: date + +-- + +*`aws.cloudtrail.digest.end_time`*:: ++ +-- +The ending UTC time range that the digest file covers, taking as a reference the time in which log files have been delivered by CloudTrail. + +type: date + +-- + +*`aws.cloudtrail.digest.s3_bucket`*:: ++ +-- +The name of the Amazon S3 bucket to which the current digest file has been delivered. + +type: keyword + +-- + +*`aws.cloudtrail.digest.s3_object`*:: ++ +-- +The Amazon S3 object key (that is, the Amazon S3 bucket location) of the current digest file. + +type: keyword + +-- + +*`aws.cloudtrail.digest.newest_event_time`*:: ++ +-- +The UTC time of the most recent event among all of the events in the log files in the digest. + +type: date + +-- + +*`aws.cloudtrail.digest.oldest_event_time`*:: ++ +-- +The UTC time of the oldest event among all of the events in the log files in the digest. + +type: date + +-- + +*`aws.cloudtrail.digest.previous_s3_bucket`*:: ++ +-- +The Amazon S3 bucket to which the previous digest file was delivered. + +type: keyword + +-- + +*`aws.cloudtrail.digest.previous_hash_algorithm`*:: ++ +-- +The name of the hash algorithm that was used to hash the previous digest file. + +type: keyword + +-- + +*`aws.cloudtrail.digest.public_key_fingerprint`*:: ++ +-- +The hexadecimal encoded fingerprint of the public key that matches the private key used to sign this digest file. + +type: keyword + +-- + +*`aws.cloudtrail.digest.signature_algorithm`*:: ++ +-- +The algorithm used to sign the digest file. + +type: keyword + +-- + +*`aws.cloudtrail.insight_details`*:: ++ +-- +Shows information about the underlying triggers of an Insights event, such as event source, user agent, statistics, API name, and whether the event is the start or end of the Insights event. + +type: flattened + +-- + [float] === cloudwatch diff --git a/filebeat/tests/system/test_modules.py b/filebeat/tests/system/test_modules.py index e37ee2bf76d..2aee6f82395 100644 --- a/filebeat/tests/system/test_modules.py +++ b/filebeat/tests/system/test_modules.py @@ -222,7 +222,7 @@ def clean_keys(obj): host_keys.append("host.name") # The create timestamps area always new - time_keys = ["event.created"] + time_keys = ["event.created", "event.ingested"] # source path and agent.version can be different for each run other_keys = ["log.file.path", "agent.version"] # ECS versions change for any ECS release, large or small diff --git a/x-pack/filebeat/docs/inputs/input-aws-s3.asciidoc b/x-pack/filebeat/docs/inputs/input-aws-s3.asciidoc index 1024656f082..8891e38fcc4 100644 --- a/x-pack/filebeat/docs/inputs/input-aws-s3.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-aws-s3.asciidoc @@ -67,6 +67,31 @@ type will not be checked. If a file has "application/json" content-type, `expand_event_list_from_field` becomes required to read the json file. +[float] +==== `file_selectors` + +If the SQS queue will have events that correspond to files that +{beatname_uc} shouldn't process `file_selectors` can be used to limit +the files that are downloaded. This is a list of selectors which are +made up of `regex` and `expand_event_list_from_field` options. The +`regex` should match the S3 object key in the SQS message, and the +optional `expand_event_list_from_field` is the same as the global +setting. If `file_selectors` is given, then any global +`expand_event_list_from_field` value is ignored in favor of the ones +specified in the `file_selectors`. Regex syntax is the same as the Go +language. Files that don't match one of the regexes won't be +processed. + +["source", "yml"] +---- +file_selectors: + - regex: '^AWSLogs/\d+/CloudTrail/' + expand_event_list_from_field: 'Records' + - regex: '^AWSLogs/\d+/CloudTrail-Digest' +``` +---- + + [float] ==== `api_timeout` diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index 980053bda81..8c736967ea5 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -102,6 +102,18 @@ filebeat.modules: # AWS SQS queue url #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue + # Process CloudTrail logs + # default is true, set to false to skip Cloudtrail logs + # var.process_cloudtrail_logs: false + + # Process CloudTrail Digest logs + # default true, set to false to skip CloudTrail Digest logs + # var.process_digest_logs: false + + # Process CloudTrail Insight logs + # default true, set to false to skip CloudTrail Insight logs + # var.process_insight_logs: false + # Filename of AWS credential file # If not set "$HOME/.aws/credentials" is used on Linux/Mac # "%UserProfile%\.aws\credentials" is used on Windows diff --git a/x-pack/filebeat/input/s3/config.go b/x-pack/filebeat/input/s3/config.go index 72960ad9ade..f9780d82277 100644 --- a/x-pack/filebeat/input/s3/config.go +++ b/x-pack/filebeat/input/s3/config.go @@ -6,6 +6,7 @@ package s3 import ( "fmt" + "regexp" "time" "github.com/elastic/beats/v7/filebeat/harvester" @@ -19,6 +20,14 @@ type config struct { AwsConfig awscommon.ConfigAWS `config:",inline"` ExpandEventListFromField string `config:"expand_event_list_from_field"` APITimeout time.Duration `config:"api_timeout"` + FileSelectors []FileSelectorCfg `config:"file_selectors"` +} + +// FileSelectorCfg defines type and configuration of FileSelectors +type FileSelectorCfg struct { + RegexString string `config:"regex"` + Regex *regexp.Regexp `config:",ignore"` + ExpandEventListFromField string `config:"expand_event_list_from_field"` } func defaultConfig() config { @@ -40,5 +49,12 @@ func (c *config) Validate() error { return fmt.Errorf("api timeout %v needs to be larger than"+ " 0s and smaller than half of the visibility timeout", c.APITimeout) } + for i := range c.FileSelectors { + r, err := regexp.Compile(c.FileSelectors[i].RegexString) + if err != nil { + return err + } + c.FileSelectors[i].Regex = r + } return nil } diff --git a/x-pack/filebeat/input/s3/input.go b/x-pack/filebeat/input/s3/input.go index 65984dace45..1085c9dccbd 100644 --- a/x-pack/filebeat/input/s3/input.go +++ b/x-pack/filebeat/input/s3/input.go @@ -74,10 +74,11 @@ type s3Input struct { } type s3Info struct { - name string - key string - region string - arn string + name string + key string + region string + arn string + expandEventListFromField string } type bucket struct { @@ -252,7 +253,7 @@ func (p *s3Input) processor(queueURL string, messages []sqs.Message, visibilityT func (p *s3Input) processMessage(svcS3 s3iface.ClientAPI, message sqs.Message, wg *sync.WaitGroup, errC chan error) { defer wg.Done() - s3Infos, err := handleSQSMessage(message) + s3Infos, err := p.handleSQSMessage(message) if err != nil { p.logger.Error(errors.Wrap(err, "handleSQSMessage failed")) return @@ -352,7 +353,7 @@ func getRegionFromQueueURL(queueURL string) (string, error) { } // handle message -func handleSQSMessage(m sqs.Message) ([]s3Info, error) { +func (p *s3Input) handleSQSMessage(m sqs.Message) ([]s3Info, error) { msg := sqsMessage{} err := json.Unmarshal([]byte(*m.Body), &msg) if err != nil { @@ -361,21 +362,40 @@ func handleSQSMessage(m sqs.Message) ([]s3Info, error) { var s3Infos []s3Info for _, record := range msg.Records { - if record.EventSource == "aws:s3" && strings.HasPrefix(record.EventName, "ObjectCreated:") { - // Unescape substrings from s3 log name. For example, convert "%3D" back to "=" - filename, err := url.QueryUnescape(record.S3.object.Key) - if err != nil { - return nil, errors.Wrapf(err, "url.QueryUnescape failed for '%s'", record.S3.object.Key) - } + if record.EventSource != "aws:s3" || !strings.HasPrefix(record.EventName, "ObjectCreated:") { + return nil, errors.New("this SQS queue should be dedicated to s3 ObjectCreated event notifications") + } + // Unescape substrings from s3 log name. For example, convert "%3D" back to "=" + filename, err := url.QueryUnescape(record.S3.object.Key) + if err != nil { + return nil, errors.Wrapf(err, "url.QueryUnescape failed for '%s'", record.S3.object.Key) + } + if len(p.config.FileSelectors) == 0 { s3Infos = append(s3Infos, s3Info{ - region: record.AwsRegion, - name: record.S3.bucket.Name, - key: filename, - arn: record.S3.bucket.Arn, + region: record.AwsRegion, + name: record.S3.bucket.Name, + key: filename, + arn: record.S3.bucket.Arn, + expandEventListFromField: p.config.ExpandEventListFromField, }) - } else { - return nil, errors.New("this SQS queue should be dedicated to s3 ObjectCreated event notifications") + continue + } + + for _, fs := range p.config.FileSelectors { + if fs.Regex == nil { + continue + } + if fs.Regex.MatchString(filename) { + s3Infos = append(s3Infos, s3Info{ + region: record.AwsRegion, + name: record.S3.bucket.Name, + key: filename, + arn: record.S3.bucket.Arn, + expandEventListFromField: fs.ExpandEventListFromField, + }) + break + } } } return s3Infos, nil @@ -456,7 +476,7 @@ func (p *s3Input) createEventsFromS3Info(svc s3iface.ClientAPI, info s3Info, s3C } // Decode JSON documents when content-type is "application/json" or expand_event_list_from_field is given in config - if resp.ContentType != nil && *resp.ContentType == "application/json" || p.config.ExpandEventListFromField != "" { + if resp.ContentType != nil && *resp.ContentType == "application/json" || info.expandEventListFromField != "" { decoder := json.NewDecoder(reader) err := p.decodeJSON(decoder, objectHash, info, s3Ctx) if err != nil { @@ -537,10 +557,10 @@ func (p *s3Input) decodeJSON(decoder *json.Decoder, objectHash string, s3Info s3 func (p *s3Input) jsonFieldsType(jsonFields interface{}, offset int, objectHash string, s3Info s3Info, s3Ctx *s3Context) (int, error) { switch f := jsonFields.(type) { case map[string][]interface{}: - if p.config.ExpandEventListFromField != "" { - textValues, ok := f[p.config.ExpandEventListFromField] + if s3Info.expandEventListFromField != "" { + textValues, ok := f[s3Info.expandEventListFromField] if !ok { - err := errors.Errorf("key '%s' not found", p.config.ExpandEventListFromField) + err := errors.Errorf("key '%s' not found", s3Info.expandEventListFromField) p.logger.Error(err) return offset, err } @@ -555,10 +575,10 @@ func (p *s3Input) jsonFieldsType(jsonFields interface{}, offset int, objectHash return offset, nil } case map[string]interface{}: - if p.config.ExpandEventListFromField != "" { - textValues, ok := f[p.config.ExpandEventListFromField] + if s3Info.expandEventListFromField != "" { + textValues, ok := f[s3Info.expandEventListFromField] if !ok { - err := errors.Errorf("key '%s' not found", p.config.ExpandEventListFromField) + err := errors.Errorf("key '%s' not found", s3Info.expandEventListFromField) p.logger.Error(err) return offset, err } diff --git a/x-pack/filebeat/input/s3/input_test.go b/x-pack/filebeat/input/s3/input_test.go index c9ff86780de..d1fab05cb3c 100644 --- a/x-pack/filebeat/input/s3/input_test.go +++ b/x-pack/filebeat/input/s3/input_test.go @@ -120,9 +120,10 @@ func TestHandleMessage(t *testing.T) { }, } + p := &s3Input{context: &channelContext{}} for _, c := range casesPositive { t.Run(c.title, func(t *testing.T) { - s3Info, err := handleSQSMessage(c.message) + s3Info, err := p.handleSQSMessage(c.message) assert.NoError(t, err) assert.Equal(t, len(c.expectedS3Infos), len(s3Info)) if len(s3Info) > 0 { @@ -155,7 +156,7 @@ func TestHandleMessage(t *testing.T) { for _, c := range casesNegative { t.Run(c.title, func(t *testing.T) { - s3Info, err := handleSQSMessage(c.message) + s3Info, err := p.handleSQSMessage(c.message) assert.Error(t, err) assert.Nil(t, s3Info) }) diff --git a/x-pack/filebeat/module/aws/_meta/config.yml b/x-pack/filebeat/module/aws/_meta/config.yml index 7c9c9f0e7b0..b7e0c25b674 100644 --- a/x-pack/filebeat/module/aws/_meta/config.yml +++ b/x-pack/filebeat/module/aws/_meta/config.yml @@ -5,6 +5,18 @@ # AWS SQS queue url #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue + # Process CloudTrail logs + # default is true, set to false to skip Cloudtrail logs + # var.process_cloudtrail_logs: false + + # Process CloudTrail Digest logs + # default true, set to false to skip CloudTrail Digest logs + # var.process_digest_logs: false + + # Process CloudTrail Insight logs + # default true, set to false to skip CloudTrail Insight logs + # var.process_insight_logs: false + # Filename of AWS credential file # If not set "$HOME/.aws/credentials" is used on Linux/Mac # "%UserProfile%\.aws\credentials" is used on Windows diff --git a/x-pack/filebeat/module/aws/cloudtrail/_meta/fields.yml b/x-pack/filebeat/module/aws/cloudtrail/_meta/fields.yml index d4bd04b548b..155908315e9 100644 --- a/x-pack/filebeat/module/aws/cloudtrail/_meta/fields.yml +++ b/x-pack/filebeat/module/aws/cloudtrail/_meta/fields.yml @@ -223,3 +223,70 @@ description: >- Identifies the service event, including what triggered the event and the result. + - name: digest + type: group + description: >- + Fields from Cloudtrail Digest Logs + fields: + - name: log_files + type: nested + description: >- + A list of Logfiles contained in the digest. + - name: start_time + type: date + description: >- + The starting UTC time range that the digest file covers, + taking as a reference the time in which log files have + been delivered by CloudTrail. + - name: end_time + type: date + description: >- + The ending UTC time range that the digest file covers, + taking as a reference the time in which log files have + been delivered by CloudTrail. + - name: s3_bucket + type: keyword + description: >- + The name of the Amazon S3 bucket to which the current + digest file has been delivered. + - name: s3_object + type: keyword + description: >- + The Amazon S3 object key (that is, the Amazon S3 bucket + location) of the current digest file. + - name: newest_event_time + type: date + description: >- + The UTC time of the most recent event among all of the + events in the log files in the digest. + - name: oldest_event_time + type: date + description: >- + The UTC time of the oldest event among all of the events + in the log files in the digest. + - name: previous_s3_bucket + type: keyword + description: >- + The Amazon S3 bucket to which the previous digest file was + delivered. + - name: previous_hash_algorithm + type: keyword + description: >- + The name of the hash algorithm that was used to hash the + previous digest file. + - name: public_key_fingerprint + type: keyword + description: >- + The hexadecimal encoded fingerprint of the public key that + matches the private key used to sign this digest file. + - name: signature_algorithm + type: keyword + description: >- + The algorithm used to sign the digest file. + - name: insight_details + type: flattened + description: >- + Shows information about the underlying triggers of an Insights + event, such as event source, user agent, statistics, API name, + and whether the event is the start or end of the Insights + event. diff --git a/x-pack/filebeat/module/aws/cloudtrail/config/s3.yml b/x-pack/filebeat/module/aws/cloudtrail/config/s3.yml index 2a6f38d1fad..2094f77c712 100644 --- a/x-pack/filebeat/module/aws/cloudtrail/config/s3.yml +++ b/x-pack/filebeat/module/aws/cloudtrail/config/s3.yml @@ -1,6 +1,19 @@ type: s3 queue_url: {{ .queue_url }} -expand_event_list_from_field: Records +file_selectors: +{{ if .process_cloudtrail_logs }} + - regex: '^AWSLogs/\d+/CloudTrail/' + expand_event_list_from_field: 'Records' +{{ end }} + +{{ if .process_digest_logs }} + - regex: '^AWSLogs/\d+/CloudTrail-Digest/' +{{ end }} + +{{ if .process_insight_logs }} + - regex: '^AWSLogs/\d+/CloudTrail-Insight/' + expand_event_list_from_field: 'Records' +{{ end }} {{ if .credential_profile_name }} credential_profile_name: {{ .credential_profile_name }} diff --git a/x-pack/filebeat/module/aws/cloudtrail/ingest/pipeline.yml b/x-pack/filebeat/module/aws/cloudtrail/ingest/pipeline.yml index 36773124439..3fd015d0252 100644 --- a/x-pack/filebeat/module/aws/cloudtrail/ingest/pipeline.yml +++ b/x-pack/filebeat/module/aws/cloudtrail/ingest/pipeline.yml @@ -1,6 +1,12 @@ --- description: Pipeline for AWS CloudTrail Logs processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' + - set: + field: event.created + value: '{{@timestamp}}' - rename: field: "message" target_field: "event.original" @@ -55,7 +61,7 @@ processors: field: "json.userIdentity.sessionContext.sessionIssuer.type" target_field: "aws.cloudtrail.user_identity.session_context.session_issuer.type" ignore_failure: true -# userIdentity.sessionIssuer.userName is only set with assumed roles. + # userIdentity.sessionIssuer.userName is only set with assumed roles. - rename: field: "json.userIdentity.sessionContext.sessionIssuer.userName" target_field: "user.name" @@ -85,6 +91,11 @@ processors: value: "{{json.eventName}}" ignore_failure: true ignore_empty_value: true + - set: + field: "event.action" + value: "{{json.eventCategory}}" + ignore_failure: true + ignore_empty_value: true - rename: field: "json.awsRegion" target_field: "cloud.region" @@ -116,9 +127,9 @@ processors: target_field: source.as.number ignore_missing: true - rename: - field: source.as.organization_name - target_field: source.as.organization.name - ignore_missing: true + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true - user_agent: field: "json.userAgent" target_field: "user_agent" @@ -138,7 +149,7 @@ processors: - rename: field: json.requestParameters target_field: "aws.cloudtrail.flattened.request_parameters" - if: ctx.json.requestParameters != null + if: ctx?.json?.requestParameters != null - script: lang: painless source: | @@ -149,7 +160,7 @@ processors: - rename: field: json.responseElements target_field: "aws.cloudtrail.flattened.response_elements" - if: ctx.json.responseElements != null + if: ctx?.json?.responseElements != null - script: lang: painless source: | @@ -211,7 +222,7 @@ processors: - rename: field: json.serviceEventDetails target_field: "aws.cloudtrail.flattened.service_event_details" - if: ctx.json.serviceEventDetails != null + if: ctx?.json?.serviceEventDetails != null - script: lang: painless source: | @@ -618,6 +629,81 @@ processors: def hm = new HashMap(params.get(ctx.event.action)); hm.forEach((k, v) -> ctx.event[k] = v); + - rename: + field: "json.awsAccountId" + target_field: "cloud.account.id" + ignore_failure: true + - rename: + field: "json.previousDigestS3Object" + target_field: "file.path" + ignore_failure: true + - rename: + field: "json.previousDigestSignature" + target_field: "file.hash.sha256" + if: >- + ctx?.json?.previousDigestHashAlgorithm != null && ctx.json.previousDigestHashAlgorithm == 'SHA-256' + - append: + field: "related.hash" + value: "{{json.previousDigestSignature}}" + if: "ctx?.json?.previousDigestSignature != null" + - rename: + field: "json.logFiles" + target_field: "aws.cloudtrail.digest.log_files" + ignore_failure: true + - date: + field: "json.digestStartTime" + target_field: "aws.cloudtrail.digest.start_time" + ignore_failure: true + formats: + - ISO8601 + - date: + field: "json.digestEndTime" + target_field: "@timestamp" + ignore_failure: true + formats: + - ISO8601 + - date: + field: "json.digestEndTime" + target_field: "aws.cloudtrail.digest.end_time" + ignore_failure: true + formats: + - ISO8601 + - rename: + field: "json.digestS3Bucket" + target_field: "aws.cloudtrail.digest.s3_bucket" + ignore_failure: true + - date: + field: "json.newestEventTime" + target_field: "aws.cloudtrail.digest.newest_event_time" + ignore_failure: true + formats: + - ISO8601 + - date: + field: "json.oldestEventTime" + target_field: "aws.cloudtrail.digest.oldest_event_time" + ignore_failure: true + formats: + - ISO8601 + - rename: + field: "json.previousDigestS3Bucket" + target_field: "aws.cloudtrail.digest.previous_s3_bucket" + ignore_failure: true + - rename: + field: "json.previousDigestHashAlgorithm" + target_field: "aws.cloudtrail.digest.previous_hash_algorithm" + ignore_failure: true + - rename: + field: "json.publicKeyFingerprint" + target_field: "aws.cloudtrail.digest.public_key_fingerprint" + ignore_failure: true + - rename: + field: "json.digestSignatureAlgorithm" + target_field: "aws.cloudtrail.digest.signature_algorithm" + ignore_failure: true + - rename: + field: "json.insightDetails" + target_field: "aws.cloudtrail.insight_details" + ignore_failure: true - remove: field: - "json" diff --git a/x-pack/filebeat/module/aws/cloudtrail/manifest.yml b/x-pack/filebeat/module/aws/cloudtrail/manifest.yml index 2878c79936d..732967ff0b0 100644 --- a/x-pack/filebeat/module/aws/cloudtrail/manifest.yml +++ b/x-pack/filebeat/module/aws/cloudtrail/manifest.yml @@ -15,6 +15,12 @@ var: - name: role_arn - name: tags default: [forwarded] + - name: process_cloudtrail_logs + default: true + - name: process_digest_logs + default: true + - name: process_insight_logs + default: true ingest_pipeline: ingest/pipeline.yml input: config/{{.input}}.yml diff --git a/x-pack/filebeat/module/aws/cloudtrail/test/cloudtrail-digest-json.log b/x-pack/filebeat/module/aws/cloudtrail/test/cloudtrail-digest-json.log new file mode 100644 index 00000000000..f3393babceb --- /dev/null +++ b/x-pack/filebeat/module/aws/cloudtrail/test/cloudtrail-digest-json.log @@ -0,0 +1 @@ +{"awsAccountId":"123456789123","digestStartTime":"2020-09-11T18:36:49Z","digestEndTime":"2020-09-11T19:36:49Z","digestS3Bucket":"alice-bucket","digestS3Object":"AWSLogs/123456789123/CloudTrail-Digest/us-west-2/2020/09/11/123456789123_CloudTrail-Digest_us-west-2_leh-ct-test_us-west-2_20200911T193649Z.json.gz","digestPublicKeyFingerprint":"47aaa19f7eec22e9bd0b5e58cfade8cb","digestSignatureAlgorithm":"SHA256withRSA","newestEventTime":"2020-09-11T19:26:24Z","oldestEventTime":"2020-09-11T18:32:04Z","previousDigestS3Bucket":"alice-bucket","previousDigestS3Object":"AWSLogs/123456789123/CloudTrail-Digest/us-west-2/2020/09/11/123456789123_CloudTrail-Digest_us-west-2_leh-ct-test_us-west-2_20200911T183649Z.json.gz","previousDigestHashValue":"531914fcfa0dbacf0c9dd1475a1fdcb5dea6e85921409f3c3ec0ba39063c860","previousDigestHashAlgorithm":"SHA-256","previousDigestSignature":"10e0872f32fa1d299d0cc98e94d4c88a6a2eada9d9fc3ae6d53dfe8d54c7caf807072f1e1eec47efdeecfcc22483887f8fddfc954ae587fba43e7676b5547f432fa8722ba1c5baa6b233bcb528ce7c01e3748aab8f28c16c024de79da820128b4c9e5ce65e98a9c4e631687ecc89c224a11bb3df06ce441ff740e4ac9fbd41159e77f5863550118284121f193e357866fbd0463faffb56e194af196e35a7675c3bbd0a398f43159343c3f59129d6339a281a8fdb3192f3fffea9bd21dbb0a705ebfae1921f2133aab0ad29522aea6df0828c1780d3f3ed6b8270ab3ba24459916b0fbbe82fba6ff9677bafe7306e0f5edcc0f1508cdb4e36f3e3b30e653e9987","logFiles":[{"s3Bucket":"alice-bucket","s3Object":"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1930Z_l2pGqVS53QcGdAkp.json.gz","hashValue":"420784a5bbc12e9ac442451e8ec1356744fdeabf4fee0d2222508db6d448139c","hashAlgorithm":"SHA-256","newestEventTime":"2020-09-11T19:26:24Z","oldestEventTime":"2020-09-11T19:26:24Z"},{"s3Bucket":"alice-bucket","s3Object":"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1915Z_TIKlbLnJ6IwUxqxw.json.gz","hashValue":"4e1eb2a8b41d032cbb16e5449fc8f3eac304e7d43017a391b37c788c77336196","hashAlgorithm":"SHA-256","newestEventTime":"2020-09-11T19:11:18Z","oldestEventTime":"2020-09-11T19:11:18Z"},{"s3Bucket":"alice-bucket","s3Object":"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1835Z_OPJhVNodH1gY760s.json.gz","hashValue":"2695aeb3b4c1f021fe76e0b36f5ac15e557c41c58af6eef282d77ef056210d70","hashAlgorithm":"SHA-256","newestEventTime":"2020-09-11T18:32:04Z","oldestEventTime":"2020-09-11T18:32:04Z"},{"s3Bucket":"alice-bucket","s3Object":"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1925Z_zJNGzQovyNAImZV9.json.gz","hashValue":"45a2906f55cbfc912584e9425f8d3d8d6fabf571a45a5ecd7d2a0f4132b81689","hashAlgorithm":"SHA-256","newestEventTime":"2020-09-11T19:21:28Z","oldestEventTime":"2020-09-11T19:21:28Z"},{"s3Bucket":"alice-bucket","s3Object":"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1855Z_RqN9YzoKAJCKbejj.json.gz","hashValue":"515cc8be750d815266b4fc799c7600765f22502d29f5bb9d5c8969ffc5ab7097","hashAlgorithm":"SHA-256","newestEventTime":"2020-09-11T18:51:21Z","oldestEventTime":"2020-09-11T18:51:21Z"},{"s3Bucket":"alice-bucket","s3Object":"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1850Z_jLldN7U8XrspES8p.json.gz","hashValue":"18650414e79e084dff02da66253f071347f7bb5c4863279bafe7762a980f7c0b","hashAlgorithm":"SHA-256","newestEventTime":"2020-09-11T18:46:45Z","oldestEventTime":"2020-09-11T18:46:45Z"},{"s3Bucket":"alice-bucket","s3Object":"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1905Z_jBNdmg4bSGxZ3wC8.json.gz","hashValue":"54050ec665636f1985f5b51ae43c74a58282cb2e500492a45f20a4dc1bf8a6d5","hashAlgorithm":"SHA-256","newestEventTime":"2020-09-11T19:01:06Z","oldestEventTime":"2020-09-11T19:01:06Z"},{"s3Bucket":"alice-bucket","s3Object":"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1920Z_bj5DRrmILF6jK23a.json.gz","hashValue":"6e0d8fcbd712d3f6d1caf4a872681f4290b05ed8a8f1c9450a0a6db92ccab4d7","hashAlgorithm":"SHA-256","newestEventTime":"2020-09-11T19:16:12Z","oldestEventTime":"2020-09-11T19:16:12Z"},{"s3Bucket":"alice-bucket","s3Object":"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1900Z_6LjrkrhsLQMzCiSN.json.gz","hashValue":"b2b0e2804d1c6b92d76eee203d7eba32d3d003e6967f175723a83ecc2d7ad4ba","hashAlgorithm":"SHA-256","newestEventTime":"2020-09-11T18:56:05Z","oldestEventTime":"2020-09-11T18:56:05Z"},{"s3Bucket":"alice-bucket","s3Object":"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1910Z_DLyqye8LaeoD204N.json.gz","hashValue":"4397a13565a67d9ed6e57737b98eb7e61ca52bb191c9b5da0423136dfc5581c7","hashAlgorithm":"SHA-256","newestEventTime":"2020-09-11T19:06:31Z","oldestEventTime":"2020-09-11T19:06:31Z"},{"s3Bucket":"alice-bucket","s3Object":"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1845Z_TSDKyASOn2ejOq5n.json.gz","hashValue":"94f09d2398632c7b0c0066ed5d56768632dd2e06ed9c80af9d0c2c5f59bd60b6","hashAlgorithm":"SHA-256","newestEventTime":"2020-09-11T18:41:58Z","oldestEventTime":"2020-09-11T18:41:58Z"},{"s3Bucket":"alice-bucket","s3Object":"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1840Z_btJydJ2t7hCRnjsN.json.gz","hashValue":"9044f9a05d70688bc6f6048d5f8d00764ab65e132b8ffefb193b22ca4394d771","hashAlgorithm":"SHA-256","newestEventTime":"2020-09-11T18:37:10Z","oldestEventTime":"2020-09-11T18:37:10Z"}]} diff --git a/x-pack/filebeat/module/aws/cloudtrail/test/cloudtrail-digest-json.log-expected.json b/x-pack/filebeat/module/aws/cloudtrail/test/cloudtrail-digest-json.log-expected.json new file mode 100644 index 00000000000..e4cf4e32a06 --- /dev/null +++ b/x-pack/filebeat/module/aws/cloudtrail/test/cloudtrail-digest-json.log-expected.json @@ -0,0 +1,126 @@ +[ + { + "@timestamp": "2020-09-11T19:36:49.000Z", + "aws.cloudtrail.digest.end_time": "2020-09-11T19:36:49.000Z", + "aws.cloudtrail.digest.log_files": [ + { + "hashAlgorithm": "SHA-256", + "hashValue": "420784a5bbc12e9ac442451e8ec1356744fdeabf4fee0d2222508db6d448139c", + "newestEventTime": "2020-09-11T19:26:24Z", + "oldestEventTime": "2020-09-11T19:26:24Z", + "s3Bucket": "alice-bucket", + "s3Object": "AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1930Z_l2pGqVS53QcGdAkp.json.gz" + }, + { + "hashAlgorithm": "SHA-256", + "hashValue": "4e1eb2a8b41d032cbb16e5449fc8f3eac304e7d43017a391b37c788c77336196", + "newestEventTime": "2020-09-11T19:11:18Z", + "oldestEventTime": "2020-09-11T19:11:18Z", + "s3Bucket": "alice-bucket", + "s3Object": "AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1915Z_TIKlbLnJ6IwUxqxw.json.gz" + }, + { + "hashAlgorithm": "SHA-256", + "hashValue": "2695aeb3b4c1f021fe76e0b36f5ac15e557c41c58af6eef282d77ef056210d70", + "newestEventTime": "2020-09-11T18:32:04Z", + "oldestEventTime": "2020-09-11T18:32:04Z", + "s3Bucket": "alice-bucket", + "s3Object": "AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1835Z_OPJhVNodH1gY760s.json.gz" + }, + { + "hashAlgorithm": "SHA-256", + "hashValue": "45a2906f55cbfc912584e9425f8d3d8d6fabf571a45a5ecd7d2a0f4132b81689", + "newestEventTime": "2020-09-11T19:21:28Z", + "oldestEventTime": "2020-09-11T19:21:28Z", + "s3Bucket": "alice-bucket", + "s3Object": "AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1925Z_zJNGzQovyNAImZV9.json.gz" + }, + { + "hashAlgorithm": "SHA-256", + "hashValue": "515cc8be750d815266b4fc799c7600765f22502d29f5bb9d5c8969ffc5ab7097", + "newestEventTime": "2020-09-11T18:51:21Z", + "oldestEventTime": "2020-09-11T18:51:21Z", + "s3Bucket": "alice-bucket", + "s3Object": "AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1855Z_RqN9YzoKAJCKbejj.json.gz" + }, + { + "hashAlgorithm": "SHA-256", + "hashValue": "18650414e79e084dff02da66253f071347f7bb5c4863279bafe7762a980f7c0b", + "newestEventTime": "2020-09-11T18:46:45Z", + "oldestEventTime": "2020-09-11T18:46:45Z", + "s3Bucket": "alice-bucket", + "s3Object": "AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1850Z_jLldN7U8XrspES8p.json.gz" + }, + { + "hashAlgorithm": "SHA-256", + "hashValue": "54050ec665636f1985f5b51ae43c74a58282cb2e500492a45f20a4dc1bf8a6d5", + "newestEventTime": "2020-09-11T19:01:06Z", + "oldestEventTime": "2020-09-11T19:01:06Z", + "s3Bucket": "alice-bucket", + "s3Object": "AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1905Z_jBNdmg4bSGxZ3wC8.json.gz" + }, + { + "hashAlgorithm": "SHA-256", + "hashValue": "6e0d8fcbd712d3f6d1caf4a872681f4290b05ed8a8f1c9450a0a6db92ccab4d7", + "newestEventTime": "2020-09-11T19:16:12Z", + "oldestEventTime": "2020-09-11T19:16:12Z", + "s3Bucket": "alice-bucket", + "s3Object": "AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1920Z_bj5DRrmILF6jK23a.json.gz" + }, + { + "hashAlgorithm": "SHA-256", + "hashValue": "b2b0e2804d1c6b92d76eee203d7eba32d3d003e6967f175723a83ecc2d7ad4ba", + "newestEventTime": "2020-09-11T18:56:05Z", + "oldestEventTime": "2020-09-11T18:56:05Z", + "s3Bucket": "alice-bucket", + "s3Object": "AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1900Z_6LjrkrhsLQMzCiSN.json.gz" + }, + { + "hashAlgorithm": "SHA-256", + "hashValue": "4397a13565a67d9ed6e57737b98eb7e61ca52bb191c9b5da0423136dfc5581c7", + "newestEventTime": "2020-09-11T19:06:31Z", + "oldestEventTime": "2020-09-11T19:06:31Z", + "s3Bucket": "alice-bucket", + "s3Object": "AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1910Z_DLyqye8LaeoD204N.json.gz" + }, + { + "hashAlgorithm": "SHA-256", + "hashValue": "94f09d2398632c7b0c0066ed5d56768632dd2e06ed9c80af9d0c2c5f59bd60b6", + "newestEventTime": "2020-09-11T18:41:58Z", + "oldestEventTime": "2020-09-11T18:41:58Z", + "s3Bucket": "alice-bucket", + "s3Object": "AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1845Z_TSDKyASOn2ejOq5n.json.gz" + }, + { + "hashAlgorithm": "SHA-256", + "hashValue": "9044f9a05d70688bc6f6048d5f8d00764ab65e132b8ffefb193b22ca4394d771", + "newestEventTime": "2020-09-11T18:37:10Z", + "oldestEventTime": "2020-09-11T18:37:10Z", + "s3Bucket": "alice-bucket", + "s3Object": "AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1840Z_btJydJ2t7hCRnjsN.json.gz" + } + ], + "aws.cloudtrail.digest.newest_event_time": "2020-09-11T19:26:24.000Z", + "aws.cloudtrail.digest.oldest_event_time": "2020-09-11T18:32:04.000Z", + "aws.cloudtrail.digest.previous_hash_algorithm": "SHA-256", + "aws.cloudtrail.digest.previous_s3_bucket": "alice-bucket", + "aws.cloudtrail.digest.s3_bucket": "alice-bucket", + "aws.cloudtrail.digest.signature_algorithm": "SHA256withRSA", + "aws.cloudtrail.digest.start_time": "2020-09-11T18:36:49.000Z", + "cloud.account.id": "123456789123", + "event.dataset": "aws.cloudtrail", + "event.kind": "event", + "event.module": "aws", + "event.original": "{\"awsAccountId\":\"123456789123\",\"digestStartTime\":\"2020-09-11T18:36:49Z\",\"digestEndTime\":\"2020-09-11T19:36:49Z\",\"digestS3Bucket\":\"alice-bucket\",\"digestS3Object\":\"AWSLogs/123456789123/CloudTrail-Digest/us-west-2/2020/09/11/123456789123_CloudTrail-Digest_us-west-2_leh-ct-test_us-west-2_20200911T193649Z.json.gz\",\"digestPublicKeyFingerprint\":\"47aaa19f7eec22e9bd0b5e58cfade8cb\",\"digestSignatureAlgorithm\":\"SHA256withRSA\",\"newestEventTime\":\"2020-09-11T19:26:24Z\",\"oldestEventTime\":\"2020-09-11T18:32:04Z\",\"previousDigestS3Bucket\":\"alice-bucket\",\"previousDigestS3Object\":\"AWSLogs/123456789123/CloudTrail-Digest/us-west-2/2020/09/11/123456789123_CloudTrail-Digest_us-west-2_leh-ct-test_us-west-2_20200911T183649Z.json.gz\",\"previousDigestHashValue\":\"531914fcfa0dbacf0c9dd1475a1fdcb5dea6e85921409f3c3ec0ba39063c860\",\"previousDigestHashAlgorithm\":\"SHA-256\",\"previousDigestSignature\":\"10e0872f32fa1d299d0cc98e94d4c88a6a2eada9d9fc3ae6d53dfe8d54c7caf807072f1e1eec47efdeecfcc22483887f8fddfc954ae587fba43e7676b5547f432fa8722ba1c5baa6b233bcb528ce7c01e3748aab8f28c16c024de79da820128b4c9e5ce65e98a9c4e631687ecc89c224a11bb3df06ce441ff740e4ac9fbd41159e77f5863550118284121f193e357866fbd0463faffb56e194af196e35a7675c3bbd0a398f43159343c3f59129d6339a281a8fdb3192f3fffea9bd21dbb0a705ebfae1921f2133aab0ad29522aea6df0828c1780d3f3ed6b8270ab3ba24459916b0fbbe82fba6ff9677bafe7306e0f5edcc0f1508cdb4e36f3e3b30e653e9987\",\"logFiles\":[{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1930Z_l2pGqVS53QcGdAkp.json.gz\",\"hashValue\":\"420784a5bbc12e9ac442451e8ec1356744fdeabf4fee0d2222508db6d448139c\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T19:26:24Z\",\"oldestEventTime\":\"2020-09-11T19:26:24Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1915Z_TIKlbLnJ6IwUxqxw.json.gz\",\"hashValue\":\"4e1eb2a8b41d032cbb16e5449fc8f3eac304e7d43017a391b37c788c77336196\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T19:11:18Z\",\"oldestEventTime\":\"2020-09-11T19:11:18Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1835Z_OPJhVNodH1gY760s.json.gz\",\"hashValue\":\"2695aeb3b4c1f021fe76e0b36f5ac15e557c41c58af6eef282d77ef056210d70\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T18:32:04Z\",\"oldestEventTime\":\"2020-09-11T18:32:04Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1925Z_zJNGzQovyNAImZV9.json.gz\",\"hashValue\":\"45a2906f55cbfc912584e9425f8d3d8d6fabf571a45a5ecd7d2a0f4132b81689\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T19:21:28Z\",\"oldestEventTime\":\"2020-09-11T19:21:28Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1855Z_RqN9YzoKAJCKbejj.json.gz\",\"hashValue\":\"515cc8be750d815266b4fc799c7600765f22502d29f5bb9d5c8969ffc5ab7097\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T18:51:21Z\",\"oldestEventTime\":\"2020-09-11T18:51:21Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1850Z_jLldN7U8XrspES8p.json.gz\",\"hashValue\":\"18650414e79e084dff02da66253f071347f7bb5c4863279bafe7762a980f7c0b\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T18:46:45Z\",\"oldestEventTime\":\"2020-09-11T18:46:45Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1905Z_jBNdmg4bSGxZ3wC8.json.gz\",\"hashValue\":\"54050ec665636f1985f5b51ae43c74a58282cb2e500492a45f20a4dc1bf8a6d5\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T19:01:06Z\",\"oldestEventTime\":\"2020-09-11T19:01:06Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1920Z_bj5DRrmILF6jK23a.json.gz\",\"hashValue\":\"6e0d8fcbd712d3f6d1caf4a872681f4290b05ed8a8f1c9450a0a6db92ccab4d7\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T19:16:12Z\",\"oldestEventTime\":\"2020-09-11T19:16:12Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1900Z_6LjrkrhsLQMzCiSN.json.gz\",\"hashValue\":\"b2b0e2804d1c6b92d76eee203d7eba32d3d003e6967f175723a83ecc2d7ad4ba\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T18:56:05Z\",\"oldestEventTime\":\"2020-09-11T18:56:05Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1910Z_DLyqye8LaeoD204N.json.gz\",\"hashValue\":\"4397a13565a67d9ed6e57737b98eb7e61ca52bb191c9b5da0423136dfc5581c7\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T19:06:31Z\",\"oldestEventTime\":\"2020-09-11T19:06:31Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1845Z_TSDKyASOn2ejOq5n.json.gz\",\"hashValue\":\"94f09d2398632c7b0c0066ed5d56768632dd2e06ed9c80af9d0c2c5f59bd60b6\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T18:41:58Z\",\"oldestEventTime\":\"2020-09-11T18:41:58Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1840Z_btJydJ2t7hCRnjsN.json.gz\",\"hashValue\":\"9044f9a05d70688bc6f6048d5f8d00764ab65e132b8ffefb193b22ca4394d771\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T18:37:10Z\",\"oldestEventTime\":\"2020-09-11T18:37:10Z\"}]}", + "event.type": "info", + "file.hash.sha256": "10e0872f32fa1d299d0cc98e94d4c88a6a2eada9d9fc3ae6d53dfe8d54c7caf807072f1e1eec47efdeecfcc22483887f8fddfc954ae587fba43e7676b5547f432fa8722ba1c5baa6b233bcb528ce7c01e3748aab8f28c16c024de79da820128b4c9e5ce65e98a9c4e631687ecc89c224a11bb3df06ce441ff740e4ac9fbd41159e77f5863550118284121f193e357866fbd0463faffb56e194af196e35a7675c3bbd0a398f43159343c3f59129d6339a281a8fdb3192f3fffea9bd21dbb0a705ebfae1921f2133aab0ad29522aea6df0828c1780d3f3ed6b8270ab3ba24459916b0fbbe82fba6ff9677bafe7306e0f5edcc0f1508cdb4e36f3e3b30e653e9987", + "file.path": "AWSLogs/123456789123/CloudTrail-Digest/us-west-2/2020/09/11/123456789123_CloudTrail-Digest_us-west-2_leh-ct-test_us-west-2_20200911T183649Z.json.gz", + "fileset.name": "cloudtrail", + "input.type": "log", + "log.offset": 0, + "service.type": "aws", + "tags": [ + "forwarded" + ] + } +] \ No newline at end of file diff --git a/x-pack/filebeat/module/aws/cloudtrail/test/insight-json.log b/x-pack/filebeat/module/aws/cloudtrail/test/insight-json.log new file mode 100644 index 00000000000..c5c536fe7a6 --- /dev/null +++ b/x-pack/filebeat/module/aws/cloudtrail/test/insight-json.log @@ -0,0 +1 @@ +{"eventVersion":"1.07","eventTime":"2020-09-09T23:00:00Z","awsRegion":"us-east-1","eventID":"41ed77ca-d659-b45a-8e9a-74e504300007","eventType":"AwsCloudTrailInsight","recipientAccountId":"123456789012","sharedEventID":"e672c2b1-e71a-4779-f96c-02da7bb30d2e","insightDetails":{"state":"End","eventSource":"iam.amazonaws.com","eventName":"AttachUserPolicy","insightType":"ApiCallRateInsight","insffightContext":{"statistics":{"baseline":{"average":0.0},"insight":{"average":2.0},"insightDuration":1,"baselineDuration":11459},"attributions":[{"attribute":"userIdentityArn","insight":[{"value":"arn:aws:iam::123456789012:user/Alice","average":2.0}],"baseline":[]},{"attribute":"userAgent","insight":[{"value":"console.amazonaws.com","average":2.0}],"baseline":[]},{"attribute":"errorCode","insight":[{"value":"null","average":2.0}],"baseline":[]}]}},"eventCategory":"Insight"} diff --git a/x-pack/filebeat/module/aws/cloudtrail/test/insight-json.log-expected.json b/x-pack/filebeat/module/aws/cloudtrail/test/insight-json.log-expected.json new file mode 100644 index 00000000000..2bfe4bedd30 --- /dev/null +++ b/x-pack/filebeat/module/aws/cloudtrail/test/insight-json.log-expected.json @@ -0,0 +1,64 @@ +[ + { + "@timestamp": "2020-09-09T23:00:00.000Z", + "aws.cloudtrail.event_type": "AwsCloudTrailInsight", + "aws.cloudtrail.event_version": "1.07", + "aws.cloudtrail.insight_details.eventName": "AttachUserPolicy", + "aws.cloudtrail.insight_details.eventSource": "iam.amazonaws.com", + "aws.cloudtrail.insight_details.insffightContext.attributions": [ + { + "attribute": "userIdentityArn", + "baseline": [], + "insight": [ + { + "average": 2.0, + "value": "arn:aws:iam::123456789012:user/Alice" + } + ] + }, + { + "attribute": "userAgent", + "baseline": [], + "insight": [ + { + "average": 2.0, + "value": "console.amazonaws.com" + } + ] + }, + { + "attribute": "errorCode", + "baseline": [], + "insight": [ + { + "average": 2.0, + "value": "null" + } + ] + } + ], + "aws.cloudtrail.insight_details.insffightContext.statistics.baseline.average": 0.0, + "aws.cloudtrail.insight_details.insffightContext.statistics.baselineDuration": 11459, + "aws.cloudtrail.insight_details.insffightContext.statistics.insight.average": 2.0, + "aws.cloudtrail.insight_details.insffightContext.statistics.insightDuration": 1, + "aws.cloudtrail.insight_details.insightType": "ApiCallRateInsight", + "aws.cloudtrail.insight_details.state": "End", + "aws.cloudtrail.recipient_account_id": "123456789012", + "cloud.region": "us-east-1", + "event.action": "Insight", + "event.dataset": "aws.cloudtrail", + "event.id": "41ed77ca-d659-b45a-8e9a-74e504300007", + "event.kind": "event", + "event.module": "aws", + "event.original": "{\"eventVersion\":\"1.07\",\"eventTime\":\"2020-09-09T23:00:00Z\",\"awsRegion\":\"us-east-1\",\"eventID\":\"41ed77ca-d659-b45a-8e9a-74e504300007\",\"eventType\":\"AwsCloudTrailInsight\",\"recipientAccountId\":\"123456789012\",\"sharedEventID\":\"e672c2b1-e71a-4779-f96c-02da7bb30d2e\",\"insightDetails\":{\"state\":\"End\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"AttachUserPolicy\",\"insightType\":\"ApiCallRateInsight\",\"insffightContext\":{\"statistics\":{\"baseline\":{\"average\":0.0},\"insight\":{\"average\":2.0},\"insightDuration\":1,\"baselineDuration\":11459},\"attributions\":[{\"attribute\":\"userIdentityArn\",\"insight\":[{\"value\":\"arn:aws:iam::123456789012:user/Alice\",\"average\":2.0}],\"baseline\":[]},{\"attribute\":\"userAgent\",\"insight\":[{\"value\":\"console.amazonaws.com\",\"average\":2.0}],\"baseline\":[]},{\"attribute\":\"errorCode\",\"insight\":[{\"value\":\"null\",\"average\":2.0}],\"baseline\":[]}]}},\"eventCategory\":\"Insight\"}", + "event.outcome": "success", + "event.type": "info", + "fileset.name": "cloudtrail", + "input.type": "log", + "log.offset": 0, + "service.type": "aws", + "tags": [ + "forwarded" + ] + } +] \ No newline at end of file diff --git a/x-pack/filebeat/module/aws/fields.go b/x-pack/filebeat/module/aws/fields.go index 5b8a3d61898..352932f1b1c 100644 --- a/x-pack/filebeat/module/aws/fields.go +++ b/x-pack/filebeat/module/aws/fields.go @@ -19,5 +19,5 @@ func init() { // AssetAws returns asset data. // This is the base64 encoded gzipped contents of module/aws. func AssetAws() string { - return "eJzMXF9z4zZwf79PsZOX2DOSOrlkOh130hmdz9eocS6upUvaJwYCVxJqCGAA0Drdp+8sAFKkSEqyReVOD4nPpBa/Xex/LDyEJ9zeANvYNwBOOIk3MP5z+gbAoERm8Qbm6NgbgBQtNyJzQqsb+I83AAC/6TSXCAttYMVUKoVagtRLCwuj10Rm9AZgIVCm9sZ/YQiKrbFYjj5um+ENLI3Os/iblnXo88GTKSn7dUbxaXWJ6jJc6jx1hglZPmpbkT773BafFBcsly7xS9zAgkmLtcetYKuAtfF4bwnLjLDUoLfBr7KAz6hc8ozGCq1qbxScPOF2o0269+wAMPrMVlhFFOmDXoBbIQEMCxP6NXOjVmi5RZOIFJUTbtsKbV/ITWDDVmREeRIJA0pcExSulWNCWUjRMSEtsLnOncdLq4FeNGhNxr9BARDcijlYsxT9Vwz+naN1A2Aqhc1K8BVwg/5dJi1s0GCDXG4xHcFkAQ7XmTbMbBvf8e8M/AoFbrvSGwsrvaHfNmg2COg5cYnpaO/VNiWp7gbJoPHwsI40t6PlhbAjUcKesY4tr1i32dfUlyNpKkYBZbxmX7SCR7Q6NxzhI1sjXI0fP14XADMjFBcZk3t7zpmU+2KtoOYcrU2ecJuINnx94Q/rECGYvA8IN8x6xQGnwYqlqmpoN2CLlow2IcPAz64TcpsVngp4sqhi8UC9ODfCrSpmYJHnpk0loK7iZG6lYXjWM6OfRYoWhAq+htzQzrIjj610S9Fxg8xh6l2tW2mL1SVbvtplSlXhrhcsYblbERVO1FvfPq4VpwoaonY8M5kjCAvO0P+j+LV23imCNt6p+Z83xGonsVbPFEW021AmrfYyrPEatpe1i50+v30YQ4rPguO/g3YrNBthcRCiY1Nhq3L1e0VamzLXBT7I9MALLxEokfFO3ok1wmaFwbqautuUmLA2bzriOj+FEfp3zUGGuuzwJRz1YY/wepvspBfD2+nhrPgcskU4Ht6KzylmCC+QM0TtiTEmBpbDSjMAm/PVQZLMwqPWbkBG/MmiGZBBP2rZYTRVAZRBrT06XVoQQjk0ikmKWVEa1byqGsGW2K0nUNe942y35xKX5nb8+LHgMmrAFeNc5ypsnfe/fu+Mlnh9kFybeI4o0glSCWC+jirExQNneqPs5bSh4FeoZ/2EaTJv82h9JWa0VLHrVLFZNBThugoHMnZgbfkFFDnq3e1bGOdOw5QzXxzHWvBOMusEh3fIlHVMPrUXWGiMNgnX6b7nO73wa6+vqtz5RcpEI8YVgy43yvrIQM8P4VujtWzZJ8TJYTChvKoQKR1SN9RIK8mYYWt0aPb37VyR7ggPSJhMbQfRFigKWoqtIUZ3Z/brXDqRHK3zDqX6HQ+PNDGqYrKZVhaTmA/0LaWCfplvUD7KOH3HFnb2hMBXTC3RwlXI7AfNSjyjtM574BQlUoYXiFx/g0JlaSoIHJOJb6qkrNZhOleu45I85bqsUjyFFk7pkZV2pKYumkuDUGFvFNbiPn2D0iwsuRH6ztXNUHl5aYUOx0KgrdlryAij35yjUMtmO4dJiSksUaFhzn9f2EC6w4f6/l5LdnuWA63jL5o3FX0oAKYVRTHItUnbYbJMnN2EPIpz/DApO5HMWs3Frhj1zzd2nIlbJmWDkudgRnwekPWaKbb0ficYYp9GCO+0lshUhxptVkhlckXawsK+F4AKwvBWVzBjaaKVbG+6nr0VO6zCgs5IT2hHCLBfekhL7x50YQyJc3sEeU1jeAxSWO+8Stqxl4YpCLUT7Us7p5drV5Y9yvHjx2aieFIy3weMcUzYd6VbIUFK31u6Fr32lI+Ixjun0lJMsZOh8eDPom5u2MYOo98demQ3FOmG9FX/7w4N5CITZOydAj7HYB4xM0h5XfBdbCdjb/sGOYpn71+FPWTMka/gkZJ4tnE5H1um/bTcAITiMk+pNNkQamfEcokmhIV2Jxt6aUGHcvktJrF2xQymUaC97vl/fpq8r4TO+bZ6huY05Er8naPcFvpcfd4uzXig6XeGyk+qzEIqG0OIDbWD05CKxQIN/SOcz9Y/Uf9su5I9ZzxBlWZa9C2SPfX64+EWioXIlMPJWkygYkvQl9Ke7WYApO87DUz5rnK1UC0L7qK4nv7YzivXymqJidRL0Z6svCb6xNNcmyEXC8EJ5G1Y6J7Wibv50shzvDI4jLqJvD0A7EqEO2/D76lQoNrrOA+H+KjystZzIbEjSaxzMg+pUsc7J7fCmxlLjRsqebyWBWRwCFnBhNeZxOmD8Ps57Pn0eN/YgcMCXrAkt0cOoi4mWu/XYcG4o4p9d1hEnmrT0QWDeGYOaW4oxHSyWrC4kMw5VA0eX2+2d9MdUV8e+3SD5K7n/4fceQZN6OzZfB4UHZhB9b2DJ6U3irwYS5+Z4o1quFfz7uK9yeUxEz+vCwCndQJe0Fp7DY/dXdoeGm2n97v6hd5X9wte1AF7WarZH8c9Jp5wNPmsTXxtmKudw33Nia8/CcuLJr4O9dNbUuMjvqEOoiDeFBzyt19fYne3b8P8oFAV4KcKTmQJS1OD9vXlU0N8u6NPdBCp74po713QtEhTzk+X5vIysrx/9yK1a9TRZ8utep4mNUthziSF0Y4jmrN6oK0AqkNrNQBeSM9v4Z5++S7+sqN0csws0SV+90bNVtGZECsHzWGhoCa7cdrOMFamrMKSl95v4pyJy2tPpNxoaCDVWAp5d/cvM9pprveL3jNBFVTb9/Rq5VxG8dDx7PrIKaDRHK0Vapk4scaRRd6KdCE1e6G/9XqnHZNh3kgosMi1otJRKI570gsZVki2hN3JOFdOSBC1+p+BwSVtCUXJOeNPqDqa9fHhN8RmhQ16EgGCE1LWfmEdM87GRImyg87mckjbvjKHZUJX3buyoVPjsr6XXjxSdDYFd6ILfK3bQ5rUjSOol3EVDam6UwrWQkoRmR1EbgN8nfk6qsIQl9ruD3eVnlPahPyYXbEnvCwfxVjj7H4K5ZIkaK7Xmc+O9/gC3aKlK2ZhjqgArWNzKeyqi7XC/MR+PXqmh5s87CcZhRLtND10lI954AJhps3rz7navbA2rujSn4uOXPaoLDGtYy635w28tEIOhIEIN83zqvq0aaDhCxQPyX6dNuG06a9O6H+1cmytTLjIVn0H6un0HgLd0GoRiozgX/yvy03oSG0I02UiNS1fRusX4+IrbVElHI27aMYV1gFaRyz85DPEM50wulZRgtfCt2gE61m4gSaofD1Hc2FehOJ67cOrtAmT2LczodpqiSaemuuF9+B+nUpEnW/bUvjCjgNXvhkUOe4OR/RTmrQ1Es/kg2B74iF/70BgGMdzjl9al/bXP5QrI8Zf/zMcr7+o4YxWG07Sv2CFLO2qutZUYWOamFxSOiV0y9j42Wl7oLrbZJ875bIcrfQQ6tOVYkEJL70TH3fMpYQkCT8jz5uXNM4EHg/iCuLh8kAt1O2GFK8W2myYSQewEJ8xHRaRYVC7WTEaja5HMHHAmSoutIHFZzRMBvF02KHBVBjkLslNz97k0+N99NBe4nEdP4LDi+5+KYIDc6Ajg8z2fUUxDKcGysWMarkdEd+CCVlFVka3H8N0yNduxEx/9L1PNMXNr5d0ZeY5f0KXtI1pnOs2mNJKcCbDZaLdbIhfa2/6PcDoyOX8s8u1jgL94Cr2775EYyRlWDKhunokBtfaYdKRrTd+fYpryDJmQjA9uTkIzU7EP7ap5Xr+xIDB0KtnrurXvopz+sPdkz4D2Bis82eDtamK8oS/fagC2W6moB1qOZzWr3h3w3C+N5bCKlwTgxS5ZFQZMAvT38cPo/LNATzeTWejX2azh2SNbqXTUTF85ac+B/Dn3bvpZHZ36BVt4N14dvvL6P3d/d3sbvT7u/+6u521s/6EPYfv755w+131yHAXpCl2oKKiJ/Ugvxt+V3jpnahSjeHU0VFNzvzF1/IQ77Cm5Ub0y8tjIDz89DipcUSyLx1L44ykCo3KviRUez22MlS+RiN4wFGtR3fzei2nsD3eE2kvlUozvPNx+FanWN1npWOA1pznxnQ2S7YObWK7Bm5fLbFY/ZRNOL+Or+AHgJ+L40Uv0l3n+BkNpcNVNr6g0R1uxM8IJFZ8aZfsOX0rIlqGXL8OFWZeAzvbJv6bvpN2EUnWGmT7oVYoWEixXLnKxJpPa763kKGxGWWNzx0a6nKjEmZ0rtJ/DD5zFQW2GQXrSu6+1bk5HEMMLtCYviN0rQbyuvkY14ll2vFC1v/dDbY8Z4D9RGifLJrhmFY6WEPGyareK9viLsDkfdFvLCPPqcEmkpikx0LOSvec2RADn4ds/WUo0uFbf/Ov1Eb87FClu4QLJu87OnRiqZjLTddc3bmdpIJ8IacBTMXyD4+Wfvhp0LzPWc0Ya07i1XllaF8mNheNC//ntiGRE3NTTRWEhXu2RQNX0+n9ddEz3Y1F4VI7Uf4ZCVL/aRtr9KCjFVEbijvvSk973IjH2eXfbqhP4YW/PDPO3eoXb6thuqj+TrBiO4D/ztFspyH1pvf+pn8XufhVZnBIuoEppXjXr99ab1Vh0Z4bA8WMcaGWsZNJPx6ZD3bSXsaaZoYp609HgqJNi6vdV7P76XXpzSqaFhub+yeBlanthdSb0xsYl5rL+ePhFgjJi1oXF5ExIflASO710hZL+L/JtNU5KUO8G+wZj5dIwn3hQvzCwtvyC2GcewsMeG6dXnd9o0OVerhY0p54+1sA5YWS4nSz2IKuVr1Ds7hEh3nXRlDoNto87dby2MKQv59hMWyxEDyeh2uTHu7b9gtz795E2w3CiG8A49vbu4eZ/+sbd921tNTLQ7Xeq5FKvVySo42VXhRusb0D+P3XAXz8/f14NvaR+NfJA/3cte3WMXXRXS+W8KL9vinZV2jFoEjdStrC+s6jd4pbnXfMFT25xBrO0rQ9nrymlZcxyg6GEp9RwpU2YikUk9dF67N5JB/Z6UaYWvePIEypVlQhsldgFu7iIM7njF9QY/w1HrLD8o/29eo9bD5X2L/b3eEPC1ySBcezZCHZsmfPMhduzexTrOXKwKGl1BvyOLPbB/DL3sDbn6f/+3Hww7/R/4bj218HP/z8YfJx8NPPj9NZO+TLDWgGqd3A5OH5pwH99199iXf3YTx68/8BAAD//624A44=" + return "eJzcXN+T4raTf9+/oisvma0Crr6b1NXVXOWq2NnJhctkMzewyd2TI+wGdCMkR5JhyV9/1ZL8AywDM5jd1JeHXQbb0qdb/VstD+EZd7fAtuYNgOVW4C2Mf5++AdAokBm8hTla9gYgQ5Nqnluu5C38xxsAgF9UVgiEhdKwYjITXC5BqKWBhVZrGmb0BmDBUWTm1j0wBMnWWE5HH7vL8RaWWhV5+CUyD31+dMNUI7t5RuFqc4rmNKlQRWY146K6FJuRPofUlp8MF6wQNnFT3MKCCYN7l6Ngm4CVdnjvCMuMsOxBj8FvkoAblDbZoDZcyb07SkqecbdVOju4dgQYfWYrbCIK44NagF0hAfQTE/o1s6MotMKgTniG0nK7i0I7ZHIb2DCKjEaehIEBBa4JSqqkZVwayNAyLgywuSqsw0uzgVq0xpqMf4ESINgVs7BmGbpHNP5ZoLEDYDKD7YqnK0g1unuZMLBFja3hCoPZCCYLsLjOlWZ613rG3TNwM5S4zUptDazUln5tjdkaQM2JSsxGB7fGhKS5GsSD1sXjMtJejsgNfkUChx1hHUve0G59KKkvR9IWjBLKeM3+UhKe0KhCpwgf2RrhZvz08W0JMNdcpjxn4mDNUybEIVsbqNMUjUmecZfwGL6+8Pt5aCCYfPAIt8w4wQGrwPClbEpoN2CDhpQ2IcXAz7YTckwLzwU8WTSxOKCOnVtuVw01MJgWOiYSsC/ipG6VYjjSc602PEMDXHpbQ2ao1uxAY3TcinWpRmYxc6bWrpTB5pSRR7tUqcnc9YIlrLArGiWl0aN3n5aKcxkNQTo2TBQI3IDV9H9gv1LWGUVQ2hk1931LpHYOFrVMgUX1gjJhlOPhHq1+eVmc7fT55ccxZLjhKf47KLtCveUGB947tgW2yVe3ViS1GbNd4D1Pj9zwEobSMM7IW75G2K7Qa1dbdtsc48YUbUO8T0+phO5efZSgLj18CUV96CO8Xic7xwvu7Xx3Vn6O6SKcdm/l5xw1hBfwGYL0BB8THMtxoRmAKdLV0SGZgSel7ICU+JNBPSCFflKiQ2maDKicWtw7XZsRXFrUkgnyWYEbzbiq6cGW2C0nsC97p8mOxxLXpnb89LGkMkjADUtTVUi/dM7+urXTSuDbo8PF2HNCkM7gigfzdUQhTO4pU1tpricNJb1cbtQzZsk8ZtH6CsxoqnLVKWMzqMnDdSUOpOzAYvEFlDHq/d07GBdWwTRlLjkOueC9YMbyFN4jk8Yy8RxPsFBrpZNUZYeW7/zEL55fNalzk1SBRvArGm2hpXGega4fw7dGY9iyT4iT42B8etUYpDJI3VDDWEnONFujRX24bpeytB54QMxkcjcIukBe0JBv9T66O7JfF8Ly5GSedyzU77h4oojRZJPJlTSYhHigby6V41fxBsWjLKVnTKlnzwjpisklGrjxkf2gnYnnFNY5C5yhQIrw/CBv/4ZMZVnGCRwTiSuqZGyvwnQpX8fV8BTrskby5Es4lUWWypKY2qAurYFKfSO3Ftbpb8jNUpNbru9S2fSZl+OWr3AsOJo9ffURYbCbc+Ry2S7nMCEwgyVK1My657nxQ3fYUFffi0S3FxnQffxl8aYhDyXArCEoGlOlszhMlvOLi5AncY4fJ1UlkhmjUl4no+761oxzfseEaI3kKJgRnUd4vWaSLZ3d8YrYpxLCe6UEMtkhRtsVUprc4DY3cGgFoIHQ39XlzFiWKCniRdeLl6LGyg2onOSEVoQAu6mHNHV9oQujD5zjHuQ1heExCG6c8arGDrU0zIDLmrUvrZxer1xZ1SjHTx/bgeJZwXwfMMYhYK9Tt5KDFL5Hqha91pRPsMYZp0pTdLmSvvDg9qJub9nWDIPdHTpkt+TphvSo+7tDAlOec1L2TgZfojBPmGukuM7bLlbz2Om+xhT5xtlXbo4pc6DLW6Qk7G1cz8ZWYT9NNwAuU1FklJpsCbXVfLlE7d1C3Mj6WpqXoUL8HYNYs2Ias8DQXtf8Pz9NPjRc53zX3EOzCgrJ/yxQ7Ep5bl6PczNsaLqVofSTMjMfygYXYnzuYBVkfLFATX/4/dn9T5A/ExeyTZ4mKLNc8b5ZciBevz3eQTkRqbLfWQsBVCgJulTakd12gPS8VcCkqyo3E9Uq4S6T6+l3cVpTJY0SmAi15PFg5TXeJ+zmmhxTvuApgbzzEz3QPGE1X+p5TmcGx1G3kccdQJ0i3Dsd/kCJAuVep2k4RkeTlrWac4EdQeI+JXMfKnXcc3YpvB2x7FFDKY+TMo8MjiEriXAyk1h1FH4/mz2fnh5aK3CcwQuWFObERtTVWOvsOixYailjrzeLyFJtO6pgEPbMISs0uZhOUksSF4JZi7JF4+vV9n5aD+rSYxduEN/V/P8wtY5A7St7pph7QQemUX5r4VmqrSQrxrINk2krG+5Vvbtob1N5SsUvqwLAeZWAF5TWXkNjd5W2h0Lb+fWufqH3Vf2CF1XAXhZq9kdxj4EnnBV8lmRmfIkmntxf4PmdK7mrGsrgg5sFHtTypV5fqGWy4KKVGtcwJZp4x8FZ2V6VJz+opZunbJ2q82TPoiOSYpm2ieXr7vSvY3v+XF1wM9Dyf5rd+a15TVLv1aCGCEQApIqcd1wRLHumYZivTrgIOfX23I1axaBCLcFzY8U2cRGbI0rSJr5xkrgX3nfzCmV2VU6hzP4p+GS+S+ZF+hzdDrzKFl6ZJoCflkJ2T6JrVCi0bpcAw2wNlq6YOaD3KIU+qrgihTVVfirXR3YTcsVBlPDoWEL54K1qmgsMaRLfTanELbn8ULy+ovBXUh9QrpXxtZWyQgpsrUishTgW0IRcO1i/WsTPNIdKZF+HXD9xB6Xd+Rq8mtJc44arwiRfQluPa2gJZU8du1KNM7SzIm3FzCphYqk0t6v1F7JGNClUk7bbJNz1LvGNseIIocVc8NQ1si64XKLONY9aur7oXOFnlmHK10wAylRlmEFj5qot1+FyBovIjw63ZjZdhYgx13zDLLoHDrpj+ZmsoNuZLTR+kfWul7fVzNuJtu5wMXy5Ol4A7orIT+Gbug70eINtITPUYkehQYjBDa0XkzDxiNr6FiL4shznjZOv5g98Zypb+jsss9xYnpqB29sjStvBiW/Aj2yP2TJMpCQHZVbK0XFgNXv3ToFsSbDeHHL065wC+Z2wvOgUyLEem0i5/ES9YB9EOXibcZi++/ocu797588UcdkAfi7jeJ6wLNNoXr+l0mJf3Q6JFsLo9caaqzigjnBTzM/n5vI6vHx4/yKxa+2tXcy3pksUimUwZ4LJFDvati7qi4gCaB5k2QPgmLR5Bw/04/vwY8d2imV6iTZxqzdqbx9fCLHRfOon8mJSH7HrLG1VFQZuyE8cbuxeiMtJTxi5tclJyaeUmHZ3BORaWZWqw42wC0GVo8bX9GZlbU7uw6b52xOdgVqlaAyXSxfhjwymHU5YtWKXc+ROWSaqVNtgqmRmwPAyCa+556uuvgDLTc3jQlougO/tCVIqv6QlcXk4S59RdjTwhIt/IzIbZNCVABAsF2LvBxcCmFA8zbhcdjac+FLuV6awKvI2167a5N2jcn8tHXsE72wUqFnn6VrHXZpQrba0l1FV1gMaKyVhzYXggdhBoNbDV7nbW2kQlAplDhOxynIKyr9kZlbsGa9LR3nUafYwhWpKYnSq1rmrmB/QBSoipVX9B41lc8HNqou0Uv34YYH5Qgs3eTwMMkohqiXdp0SnLHCJMFf69b1vcSustC1LDJeiI5M9qradKIcozGVN8FHIfmCggdvqedO82lZQ/wD5Q9Jfq7SvrP/RCf2PKMXGiCTl+apvRz2dPoAf1yeiXJIS/Iv7uVqEjtCGMF3HU9P0lbd+Ma50pQzKJEVtrxpx+XmA5uELdxoSQp+Xz+gbQvBa+AY1Zz0z148JsljPUV+ZFi5TtXbuVZiECezbmFButUQdOmnVwllwN0/Do853sRC+1GNPldsgDhR3uyP6liWxncEL6SDYbnAfv3cg0CzFS1qyolO7I+HSVh7jj/8Zjtd/yeGMZhtOsj9ghSzryrp8ES5LdCEonOIqcpT04rDdj1ovsoudClEdt3IQ9k9c8QUFvHRPuNzRq+6DJPyMadHeRr0QeGjOKwf3B4r3XF19cOlmofSW6WwAC/4Zs2HpGQZ7p61Ho9HbEUwspEyWO7VgcIOaCc+eDj3UmHGNqU0K3bM1+fT0ECy043iYxxUO07Ljp2LBkbNhI43M9P3aEn9gzY9cnlurliPgWzAumsjqHTrfMf61CzHT71w/BOrybRAvqcr4zZIk1rp9qdlgUkmeMuHLuHW/uJvr4ESsh9ERy8W2jnosHZX7ReUmePM8fFBGEoYl47KrRqJxrSwmHdF66+dzTEOeM+2d6dnFQWhXIr7YolbzuS4iBkMnnoXcfxVE2bt7vHrSpwMbg7GuX3Cv07reLIw2WiOr+4zjUKsDK/2ytz4g42pjGaz8qyMgw1QwygyYgemv48dRdecAnu6ns9FPs9ljska7UtmoPJDhToIN4Pf799PJ7P7YLUrD+/Hs7qfRh/uH+9n96Nf3/3V/N4uT/ow9u+9vnnH3TbONsHbS5DvCJqAD+c3wm9JK16zKFPpOREs5OXNbfFVj33FJKzTvl5YnP/Dw09NkjyLifWVYWnskTWiU9iU+2+uxlCGLNWqeehzNfLQ+wxPpzOzx7Hg8VarU8N754TuVYXOdpQoOWqWuo6SrWLKzaBLTdQjv1RwL2U9VhHPzuAx+APi5bDl0LK0rxxvUFA43yfgLteowI67tJjH8rzhnL6lb0aCVy/XtPVyCk8DOsol7MtaU0g8n9wpkh66WS1gIvlzZxikWF9Z8ayBHbXKKGjcdEmoLLROmVRHvn7sKfGYbAmxyctaN2H2nCn3ch7jGOd23h97LgZxsPoV5Qpp2OpF17+Jz2+7XhvbJoB6OaaajOWQ4bdF7ZlueD558KOuNlec519mEISbZKZezUj1HNkTA5yFb/zXk2fCdextIJY342aLM6oALJh86KnRVT8tV3gpZDV/yaQBTvvzNoaUv3w/azUvNiHHPSLw6rvTly8QUvNVPd2kZElMibqoogzDwwHao4WY6fXhb1kzroxK4VJZXr5Yj8Z/GSKMLHaWIvYMylx3zj/uNsJ1dvc9t/2SOfxvluLCrn5yu+hMH+/d4LTYD+O8C9W7qQ2+670/6u4zFb3KNQ5INzCjEe/v6pXVa5SftuTBQnjssxTJUMunriTODVpjraNNMM2nc7ogXtGn5uqeb2cP0bWXNGpIWCpuHO4GNk5wLobbnFzCu1Zfz2+MdEJIXlS6uwmNC8iMheVBLU07h3tO6UwUJQ3hfkCM8HCz3ndIl+7mBd9UD/ojnDhikhbFq3fVEhyj1cNg8Hni7k8HVIfNyd7Ncgq5SvUW9uEaFuS4jSLRbpZ/ruRy2uo/XarZY8DTshyudHa/b9gvz4Cx17K0iAd8Axnd3948z90a+++5cWqjlsVzv1UiFWi7J0IZMLzC3XN4B/PrzAD7++mE8GztP/PPkkb53NpNaJq+66uUUjrXftjn7CqkYlKFbNTY3rvLojOJOFR19Rc82MTplWRb3J68p5eWMooOhwA0KuFGaL7lk4m1Z+mxvyQdyuhFmxn4RhBnlitJ79gbM0lwcxbnJ0ytKjDvaT3pYvci7V+thirnE/s1ujd9PcE0SbJonC8FaBwovJGHO7ZqZ55DLVY5DCaG2ZHFmd4/gpr2Fdz9M//fj4B//Rv8Nx3c/D/7xw4+Tj4Pvf3iazuKQr9eg6bl2C5PHzfcD+vdfXYp3/+N49Ob/AwAA//+8xDjD" } diff --git a/x-pack/filebeat/modules.d/aws.yml.disabled b/x-pack/filebeat/modules.d/aws.yml.disabled index 7e4baf005ef..f68ac889d73 100644 --- a/x-pack/filebeat/modules.d/aws.yml.disabled +++ b/x-pack/filebeat/modules.d/aws.yml.disabled @@ -8,6 +8,18 @@ # AWS SQS queue url #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue + # Process CloudTrail logs + # default is true, set to false to skip Cloudtrail logs + # var.process_cloudtrail_logs: false + + # Process CloudTrail Digest logs + # default true, set to false to skip CloudTrail Digest logs + # var.process_digest_logs: false + + # Process CloudTrail Insight logs + # default true, set to false to skip CloudTrail Insight logs + # var.process_insight_logs: false + # Filename of AWS credential file # If not set "$HOME/.aws/credentials" is used on Linux/Mac # "%UserProfile%\.aws\credentials" is used on Windows