From 53e3d8bb405a8dbcea99985cc633ab7a54ce63fd Mon Sep 17 00:00:00 2001 From: Alex Resnick Date: Thu, 22 Sep 2022 14:04:54 +0000 Subject: [PATCH 01/11] [Metricbeat] Add Data Granularity config option for AWS Cloudwatch metrics --- CHANGELOG.next.asciidoc | 2 + .../metricbeat/module/aws/_meta/docs.asciidoc | 9 +++- x-pack/metricbeat/module/aws/aws.go | 49 ++++++++++++------- .../module/aws/cloudwatch/cloudwatch.go | 8 +-- 4 files changed, 44 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 71cd6d695123..f48fc6adb5e7 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -144,6 +144,8 @@ https://github.com/elastic/beats/compare/v8.2.0\...main[Check the HEAD diff] *Metricbeat* +- Add Data Granularity option to AWS module to allow for for fewer API calls of longer periods and keep small intervals. {issue}33133[33133] {pull}[] + *Packetbeat* diff --git a/x-pack/metricbeat/module/aws/_meta/docs.asciidoc b/x-pack/metricbeat/module/aws/_meta/docs.asciidoc index c446aa957586..2b0b70d7f7f2 100644 --- a/x-pack/metricbeat/module/aws/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/aws/_meta/docs.asciidoc @@ -37,6 +37,13 @@ or none get collected by Metricbeat. In this case, please specify a `latency` parameter so collection start time and end time will be shifted by the given latency amount. +* *data_granularity* + +This module accepts an optional `data_granularity` setting which sets the granularity of the returned data points. +For example the period (how often the cloudwatch API is called) can be set to 1h with a `data_granularity` of 5m and the result will contain 12 metrics. +This enables less API calls which lowers the cost of the module while maintaining the same level of details in the metrics. +The only difference is the delay in ingesting the metrics data due to the larger period. If no value is specified, the value is set to the `period`. + * *endpoint* Most AWS services offer a regional endpoint that can be used to make requests. @@ -57,7 +64,7 @@ For example, if tags parameter is given as `Organization=Engineering` under `Organization` and tag value equals to `Engineering`. In order to filter for different values for the same key, add the values to the value array (see example) -Note: tag filtering only works for metricsets with `resource_type` specified in the +Note: tag filtering only works for metricsets with `resource_type` specified in the metricset-specific configuration. [source,yaml] diff --git a/x-pack/metricbeat/module/aws/aws.go b/x-pack/metricbeat/module/aws/aws.go index 3e286f66560a..ea4d24e02403 100644 --- a/x-pack/metricbeat/module/aws/aws.go +++ b/x-pack/metricbeat/module/aws/aws.go @@ -29,24 +29,26 @@ type describeRegionsClient interface { // Config defines all required and optional parameters for aws metricsets type Config struct { - Period time.Duration `config:"period" validate:"nonzero,required"` - Regions []string `config:"regions"` - Latency time.Duration `config:"latency"` - AWSConfig awscommon.ConfigAWS `config:",inline"` - TagsFilter []Tag `config:"tags_filter"` + Period time.Duration `config:"period" validate:"nonzero,required"` + DataGranularity time.Duration `config:"data_granularity"` + Regions []string `config:"regions"` + Latency time.Duration `config:"latency"` + AWSConfig awscommon.ConfigAWS `config:",inline"` + TagsFilter []Tag `config:"tags_filter"` } // MetricSet is the base metricset for all aws metricsets type MetricSet struct { mb.BaseMetricSet - RegionsList []string - Endpoint string - Period time.Duration - Latency time.Duration - AwsConfig *awssdk.Config - AccountName string - AccountID string - TagsFilter []Tag + RegionsList []string + Endpoint string + Period time.Duration + DataGranularity time.Duration + Latency time.Duration + AwsConfig *awssdk.Config + AccountName string + AccountID string + TagsFilter []Tag } // Tag holds a configuration specific for ec2 and cloudwatch metricset. @@ -91,16 +93,25 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { } base.Logger().Debug("aws config endpoint = ", config.AWSConfig.Endpoint) + if config.DataGranularity > config.Period { + return nil, fmt.Errorf("Data Granularity cannot be larger than the period") + } + + if config.DataGranularity == 0 { + config.DataGranularity = config.Period + } metricSet := MetricSet{ - BaseMetricSet: base, - Period: config.Period, - Latency: config.Latency, - AwsConfig: &awsConfig, - TagsFilter: config.TagsFilter, - Endpoint: config.AWSConfig.Endpoint, + BaseMetricSet: base, + Period: config.Period, + DataGranularity: config.DataGranularity, + Latency: config.Latency, + AwsConfig: &awsConfig, + TagsFilter: config.TagsFilter, + Endpoint: config.AWSConfig.Endpoint, } base.Logger().Debug("Metricset level config for period: ", metricSet.Period) + base.Logger().Debug("Metricset level config for data granularity: ", metricSet.DataGranularity) base.Logger().Debug("Metricset level config for tags filter: ", metricSet.TagsFilter) base.Logger().Warn("extra charges on AWS API requests will be generated by this metricset") diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go index 3ed35ff65765..2a831dae9181 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go @@ -378,20 +378,20 @@ func (m *MetricSet) readCloudwatchConfig() (listMetricWithDetail, map[string][]n return listMetricDetailTotal, namespaceDetailTotal } -func createMetricDataQueries(listMetricsTotal []metricsWithStatistics, period time.Duration) []types.MetricDataQuery { +func createMetricDataQueries(listMetricsTotal []metricsWithStatistics, dataGranularity time.Duration) []types.MetricDataQuery { var metricDataQueries []types.MetricDataQuery for i, listMetric := range listMetricsTotal { for j, statistic := range listMetric.statistic { stat := statistic metric := listMetric.cloudwatchMetric label := constructLabel(listMetric.cloudwatchMetric, statistic) - periodInSec := int32(period.Seconds()) + dataGranularityInSec := int32(dataGranularity.Seconds()) id := "cw" + strconv.Itoa(i) + "stats" + strconv.Itoa(j) metricDataQueries = append(metricDataQueries, types.MetricDataQuery{ Id: &id, MetricStat: &types.MetricStat{ - Period: &periodInSec, + Period: &dataGranularityInSec, Stat: &stat, Metric: &metric, }, @@ -476,7 +476,7 @@ func (m *MetricSet) createEvents(svcCloudwatch cloudwatch.GetMetricDataAPIClient events := map[string]mb.Event{} // Construct metricDataQueries - metricDataQueries := createMetricDataQueries(listMetricWithStatsTotal, m.Period) + metricDataQueries := createMetricDataQueries(listMetricWithStatsTotal, m.DataGranularity) m.logger.Debugf("Number of MetricDataQueries = %d", len(metricDataQueries)) if len(metricDataQueries) == 0 { return events, nil From 5646580cab7df95f2cda5c4c5fb7bba76981486f Mon Sep 17 00:00:00 2001 From: Alex Resnick Date: Thu, 22 Sep 2022 08:37:13 -0600 Subject: [PATCH 02/11] Update CHANGELOG.next.asciidoc Co-authored-by: kaiyan-sheng --- CHANGELOG.next.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index f48fc6adb5e7..6aa8d6f26921 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -144,7 +144,7 @@ https://github.com/elastic/beats/compare/v8.2.0\...main[Check the HEAD diff] *Metricbeat* -- Add Data Granularity option to AWS module to allow for for fewer API calls of longer periods and keep small intervals. {issue}33133[33133] {pull}[] +- Add Data Granularity option to AWS module to allow for for fewer API calls of longer periods and keep small intervals. {issue}33133[33133] {pull}33166[33166] *Packetbeat* From 4c0135c8e571f8002ce9d78b1f0f4a22b5236cd1 Mon Sep 17 00:00:00 2001 From: Alex Resnick Date: Thu, 22 Sep 2022 15:06:07 +0000 Subject: [PATCH 03/11] loop through multiple metrics per result. Update billing code. --- .../metricbeat/module/aws/billing/billing.go | 25 ++++++--------- .../module/aws/cloudwatch/cloudwatch.go | 31 +++++++------------ 2 files changed, 20 insertions(+), 36 deletions(-) diff --git a/x-pack/metricbeat/module/aws/billing/billing.go b/x-pack/metricbeat/module/aws/billing/billing.go index e8510c4de7ca..ce110cb32cc9 100644 --- a/x-pack/metricbeat/module/aws/billing/billing.go +++ b/x-pack/metricbeat/module/aws/billing/billing.go @@ -181,7 +181,7 @@ func (m *MetricSet) getCloudWatchBillingMetrics( return events } - metricDataQueriesTotal := constructMetricQueries(listMetricsOutput, m.Period) + metricDataQueriesTotal := constructMetricQueries(listMetricsOutput, m.DataGranularity) metricDataOutput, err := aws.GetMetricDataResults(metricDataQueriesTotal, svcCloudwatch, startTime, endTime) if err != nil { err = fmt.Errorf("aws GetMetricDataResults failed with %w, skipping region %s", err, regionName) @@ -189,22 +189,15 @@ func (m *MetricSet) getCloudWatchBillingMetrics( return nil } - // Find a timestamp for all metrics in output - timestamp := aws.FindTimestamp(metricDataOutput) - if timestamp.IsZero() { - return nil - } - for _, output := range metricDataOutput { if len(output.Values) == 0 { continue } - exists, timestampIdx := aws.CheckTimestampInArray(timestamp, output.Timestamps) - if exists { + for valI, metricDataResultValue := range output.Values { labels := strings.Split(*output.Label, labelSeparator) - event := aws.InitEvent("", m.AccountName, m.AccountID, timestamp) - _, _ = event.MetricSetFields.Put(labels[0], output.Values[timestampIdx]) + event := aws.InitEvent("", m.AccountName, m.AccountID, output.Timestamps[valI]) + _, _ = event.MetricSetFields.Put(labels[0], metricDataResultValue) i := 1 for i < len(labels)-1 { @@ -345,11 +338,11 @@ func (m *MetricSet) addCostMetrics(metrics map[string]costexplorertypes.MetricVa return event } -func constructMetricQueries(listMetricsOutput []types.Metric, period time.Duration) []types.MetricDataQuery { +func constructMetricQueries(listMetricsOutput []types.Metric, dataGranularity time.Duration) []types.MetricDataQuery { var metricDataQueries []types.MetricDataQuery metricDataQueryEmpty := types.MetricDataQuery{} for i, listMetric := range listMetricsOutput { - metricDataQuery := createMetricDataQuery(listMetric, i, period) + metricDataQuery := createMetricDataQuery(listMetric, i, dataGranularity) if metricDataQuery == metricDataQueryEmpty { continue } @@ -358,9 +351,9 @@ func constructMetricQueries(listMetricsOutput []types.Metric, period time.Durati return metricDataQueries } -func createMetricDataQuery(metric types.Metric, index int, period time.Duration) types.MetricDataQuery { +func createMetricDataQuery(metric types.Metric, index int, dataGranularity time.Duration) types.MetricDataQuery { statistic := "Maximum" - periodInSeconds := int32(period.Seconds()) + dataGranularityInSeconds := int32(dataGranularity.Seconds()) id := metricsetName + strconv.Itoa(index) metricDims := metric.Dimensions metricName := *metric.MetricName @@ -373,7 +366,7 @@ func createMetricDataQuery(metric types.Metric, index int, period time.Duration) return types.MetricDataQuery{ Id: &id, MetricStat: &types.MetricStat{ - Period: &periodInSeconds, + Period: &dataGranularityInSeconds, Stat: &statistic, Metric: &metric, }, diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go index 2a831dae9181..070d3766e1b2 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go @@ -489,37 +489,29 @@ func (m *MetricSet) createEvents(svcCloudwatch cloudwatch.GetMetricDataAPIClient return events, fmt.Errorf("getMetricDataResults failed: %w", err) } - // Find a timestamp for all metrics in output - timestamp := aws.FindTimestamp(metricDataResults) - if timestamp.IsZero() { - return nil, nil - } - // Create events when there is no tags_filter or resource_type specified. if len(resourceTypeTagFilters) == 0 { for _, metricDataResult := range metricDataResults { if len(metricDataResult.Values) == 0 { continue } - - exists, timestampIdx := aws.CheckTimestampInArray(timestamp, metricDataResult.Timestamps) - if exists { + for valI, metricDataResultValue := range metricDataResult.Values { labels := strings.Split(*metricDataResult.Label, labelSeparator) if len(labels) != 5 { // when there is no identifier value in label, use region+accountID+namespace instead identifier := regionName + m.AccountID + labels[namespaceIdx] if _, ok := events[identifier]; !ok { - events[identifier] = aws.InitEvent(regionName, m.AccountName, m.AccountID, timestamp) + events[identifier] = aws.InitEvent(regionName, m.AccountName, m.AccountID, metricDataResult.Timestamps[valI]) } - events[identifier] = insertRootFields(events[identifier], metricDataResult.Values[timestampIdx], labels) + events[identifier] = insertRootFields(events[identifier], metricDataResultValue, labels) continue } identifierValue := labels[identifierValueIdx] if _, ok := events[identifierValue]; !ok { - events[identifierValue] = aws.InitEvent(regionName, m.AccountName, m.AccountID, timestamp) + events[identifierValue+fmt.Sprint(valI)] = aws.InitEvent(regionName, m.AccountName, m.AccountID, metricDataResult.Timestamps[valI]) } - events[identifierValue] = insertRootFields(events[identifierValue], metricDataResult.Values[timestampIdx], labels) + events[identifierValue+fmt.Sprint(valI)] = insertRootFields(events[identifierValue], metricDataResultValue, labels) } } return events, nil @@ -554,8 +546,7 @@ func (m *MetricSet) createEvents(svcCloudwatch cloudwatch.GetMetricDataAPIClient continue } - exists, timestampIdx := aws.CheckTimestampInArray(timestamp, output.Timestamps) - if exists { + for valI, metricDataResultValue := range output.Values { labels := strings.Split(*output.Label, labelSeparator) if len(labels) != 5 { // if there is no tag in labels but there is a tagsFilter, then no event should be reported. @@ -566,23 +557,23 @@ func (m *MetricSet) createEvents(svcCloudwatch cloudwatch.GetMetricDataAPIClient // when there is no identifier value in label, use region+accountID+namespace instead identifier := regionName + m.AccountID + labels[namespaceIdx] if _, ok := events[identifier]; !ok { - events[identifier] = aws.InitEvent(regionName, m.AccountName, m.AccountID, timestamp) + events[identifier+fmt.Sprint(valI)] = aws.InitEvent(regionName, m.AccountName, m.AccountID, output.Timestamps[valI]) } - events[identifier] = insertRootFields(events[identifier], output.Values[timestampIdx], labels) + events[identifier+fmt.Sprint(valI)] = insertRootFields(events[identifier], metricDataResultValue, labels) continue } identifierValue := labels[identifierValueIdx] - if _, ok := events[identifierValue]; !ok { + if _, ok := events[identifierValue+fmt.Sprint(valI)]; !ok { // when tagsFilter is not empty but no entry in // resourceTagMap for this identifier, do not initialize // an event for this identifier. if len(tagsFilter) != 0 && resourceTagMap[identifierValue] == nil { continue } - events[identifierValue] = aws.InitEvent(regionName, m.AccountName, m.AccountID, timestamp) + events[identifierValue+fmt.Sprint(valI)] = aws.InitEvent(regionName, m.AccountName, m.AccountID, output.Timestamps[valI]) } - events[identifierValue] = insertRootFields(events[identifierValue], output.Values[timestampIdx], labels) + events[identifierValue+fmt.Sprint(valI)] = insertRootFields(events[identifierValue], metricDataResultValue, labels) // add tags to event based on identifierValue insertTags(events, identifierValue, resourceTagMap) From f6f2cbdb5a9dde10c4bf1c3f021c891ac0c13a58 Mon Sep 17 00:00:00 2001 From: Alex Resnick Date: Fri, 23 Sep 2022 21:25:30 +0000 Subject: [PATCH 04/11] update per comments --- x-pack/metricbeat/module/aws/_meta/docs.asciidoc | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/x-pack/metricbeat/module/aws/_meta/docs.asciidoc b/x-pack/metricbeat/module/aws/_meta/docs.asciidoc index 2b0b70d7f7f2..01e3264e8393 100644 --- a/x-pack/metricbeat/module/aws/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/aws/_meta/docs.asciidoc @@ -39,10 +39,12 @@ latency amount. * *data_granularity* -This module accepts an optional `data_granularity` setting which sets the granularity of the returned data points. -For example the period (how often the cloudwatch API is called) can be set to 1h with a `data_granularity` of 5m and the result will contain 12 metrics. -This enables less API calls which lowers the cost of the module while maintaining the same level of details in the metrics. -The only difference is the delay in ingesting the metrics data due to the larger period. If no value is specified, the value is set to the `period`. +This module accepts an optional `data_granularity` parameter which sets the granularity of the returned data points. +This parameter can be used to decouple how frequently metricbeat would collect data from AWS, from how granular this data should be. +This would helpful to reduce the cost incurred in calling AWS Cloudwatch, in cases where highly-granular data is required, +but a delay in collecting and ingesting the metrics can be tolerated. + +If no value is specified, the value is set to the `period`. * *endpoint* From bd8350a4366324449844cca9f48408003a5329fc Mon Sep 17 00:00:00 2001 From: Alex Resnick Date: Sat, 1 Oct 2022 15:09:43 +0000 Subject: [PATCH 05/11] update to pass tests --- CHANGELOG.next.asciidoc | 1 - .../module/aws/cloudwatch/cloudwatch.go | 35 ++++---- .../module/aws/cloudwatch/cloudwatch_test.go | 84 +++++++++++++++++-- 3 files changed, 96 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 6aa8d6f26921..dd1197b8fc8f 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -146,7 +146,6 @@ https://github.com/elastic/beats/compare/v8.2.0\...main[Check the HEAD diff] - Add Data Granularity option to AWS module to allow for for fewer API calls of longer periods and keep small intervals. {issue}33133[33133] {pull}33166[33166] - *Packetbeat* - Add option to allow sniffer to change device when default route changes. {issue}31905[31905] {pull}32681[32681] diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go index 070d3766e1b2..74cf07a485f6 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go @@ -473,7 +473,7 @@ func insertRootFields(event mb.Event, metricValue float64, labels []string) mb.E func (m *MetricSet) createEvents(svcCloudwatch cloudwatch.GetMetricDataAPIClient, svcResourceAPI resourcegroupstaggingapi.GetResourcesAPIClient, listMetricWithStatsTotal []metricsWithStatistics, resourceTypeTagFilters map[string][]aws.Tag, regionName string, startTime time.Time, endTime time.Time) (map[string]mb.Event, error) { // Initialize events for each identifier. - events := map[string]mb.Event{} + events := make(map[string]mb.Event) // Construct metricDataQueries metricDataQueries := createMetricDataQueries(listMetricWithStatsTotal, m.DataGranularity) @@ -495,11 +495,11 @@ func (m *MetricSet) createEvents(svcCloudwatch cloudwatch.GetMetricDataAPIClient if len(metricDataResult.Values) == 0 { continue } + labels := strings.Split(*metricDataResult.Label, labelSeparator) for valI, metricDataResultValue := range metricDataResult.Values { - labels := strings.Split(*metricDataResult.Label, labelSeparator) if len(labels) != 5 { // when there is no identifier value in label, use region+accountID+namespace instead - identifier := regionName + m.AccountID + labels[namespaceIdx] + identifier := regionName + m.AccountID + *metricDataResult.Label + fmt.Sprint("-", valI) if _, ok := events[identifier]; !ok { events[identifier] = aws.InitEvent(regionName, m.AccountName, m.AccountID, metricDataResult.Timestamps[valI]) } @@ -507,11 +507,11 @@ func (m *MetricSet) createEvents(svcCloudwatch cloudwatch.GetMetricDataAPIClient continue } - identifierValue := labels[identifierValueIdx] + identifierValue := *metricDataResult.Label + fmt.Sprint("-", valI) if _, ok := events[identifierValue]; !ok { - events[identifierValue+fmt.Sprint(valI)] = aws.InitEvent(regionName, m.AccountName, m.AccountID, metricDataResult.Timestamps[valI]) + events[identifierValue] = aws.InitEvent(regionName, m.AccountName, m.AccountID, metricDataResult.Timestamps[valI]) } - events[identifierValue+fmt.Sprint(valI)] = insertRootFields(events[identifierValue], metricDataResultValue, labels) + events[identifierValue] = insertRootFields(events[identifierValue], metricDataResultValue, labels) } } return events, nil @@ -546,37 +546,38 @@ func (m *MetricSet) createEvents(svcCloudwatch cloudwatch.GetMetricDataAPIClient continue } + labels := strings.Split(*output.Label, labelSeparator) for valI, metricDataResultValue := range output.Values { - labels := strings.Split(*output.Label, labelSeparator) if len(labels) != 5 { // if there is no tag in labels but there is a tagsFilter, then no event should be reported. if len(tagsFilter) != 0 { continue } - // when there is no identifier value in label, use region+accountID+namespace instead - identifier := regionName + m.AccountID + labels[namespaceIdx] + // when there is no identifier value in label, use region+accountID+labels instead + identifier := regionName + m.AccountID + *output.Label + fmt.Sprint("-", valI) if _, ok := events[identifier]; !ok { - events[identifier+fmt.Sprint(valI)] = aws.InitEvent(regionName, m.AccountName, m.AccountID, output.Timestamps[valI]) + events[identifier] = aws.InitEvent(regionName, m.AccountName, m.AccountID, output.Timestamps[valI]) } - events[identifier+fmt.Sprint(valI)] = insertRootFields(events[identifier], metricDataResultValue, labels) + events[identifier] = insertRootFields(events[identifier], metricDataResultValue, labels) continue } identifierValue := labels[identifierValueIdx] - if _, ok := events[identifierValue+fmt.Sprint(valI)]; !ok { + uniqueIdentifierValue := *output.Label + fmt.Sprint("-", valI) + if _, ok := events[uniqueIdentifierValue]; !ok { // when tagsFilter is not empty but no entry in // resourceTagMap for this identifier, do not initialize // an event for this identifier. if len(tagsFilter) != 0 && resourceTagMap[identifierValue] == nil { continue } - events[identifierValue+fmt.Sprint(valI)] = aws.InitEvent(regionName, m.AccountName, m.AccountID, output.Timestamps[valI]) + events[uniqueIdentifierValue] = aws.InitEvent(regionName, m.AccountName, m.AccountID, output.Timestamps[valI]) } - events[identifierValue+fmt.Sprint(valI)] = insertRootFields(events[identifierValue], metricDataResultValue, labels) + events[uniqueIdentifierValue] = insertRootFields(events[uniqueIdentifierValue], metricDataResultValue, labels) // add tags to event based on identifierValue - insertTags(events, identifierValue, resourceTagMap) + insertTags(events, uniqueIdentifierValue, identifierValue, resourceTagMap) } } } @@ -616,7 +617,7 @@ func compareAWSDimensions(dim1 []types.Dimension, dim2 []types.Dimension) bool { return reflect.DeepEqual(dim1NameToValue, dim2NameToValue) } -func insertTags(events map[string]mb.Event, identifier string, resourceTagMap map[string][]resourcegroupstaggingapitypes.Tag) { +func insertTags(events map[string]mb.Event, uniqueIdentifierValue string, identifier string, resourceTagMap map[string][]resourcegroupstaggingapitypes.Tag) { // Check if identifier includes dimensionSeparator (comma in this case), // split the identifier and check for each sub-identifier. // For example, identifier might be [storageType, s3BucketName]. @@ -635,7 +636,7 @@ func insertTags(events map[string]mb.Event, identifier string, resourceTagMap ma // By default, replace dot "." using underscore "_" for tag keys. // Note: tag values are not dedotted. for _, tag := range tags { - _, _ = events[identifier].RootFields.Put("aws.tags."+common.DeDot(*tag.Key), *tag.Value) + _, _ = events[uniqueIdentifierValue].RootFields.Put("aws.tags."+common.DeDot(*tag.Key), *tag.Value) } continue } diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go index 5417b8323ccf..14ef8b386af3 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go @@ -1344,6 +1344,33 @@ func (m *MockCloudWatchClientWithoutDim) GetMetricData(context.Context, *cloudwa }, nil } +// MockCloudWatchClientWithDataGranularity struct is used for unit tests. +type MockCloudWatchClientWithDataGranularity struct{} + +// GetMetricData implements cloudwatch.GetMetricDataAPIClient. +func (m *MockCloudWatchClientWithDataGranularity) GetMetricData(context.Context, *cloudwatch.GetMetricDataInput, ...func(*cloudwatch.Options)) (*cloudwatch.GetMetricDataOutput, error) { + emptyString := "" + return &cloudwatch.GetMetricDataOutput{ + Messages: nil, + MetricDataResults: []cloudwatchtypes.MetricDataResult{ + { + Id: &id1, + Label: &label3, + Values: []float64{value1, value1}, + Timestamps: []time.Time{timestamp, timestamp}, + }, + { + Id: &id2, + Label: &label4, + Values: []float64{value2, value2}, + Timestamps: []time.Time{timestamp, timestamp}, + }, + }, + NextToken: &emptyString, + ResultMetadata: middleware.Metadata{}, + }, nil +} + // MockResourceGroupsTaggingClient is used for unit tests. type MockResourceGroupsTaggingClient struct{} @@ -1396,12 +1423,13 @@ func TestCreateEventsWithIdentifier(t *testing.T) { events, err := m.createEvents(mockCloudwatchSvc, mockTaggingSvc, listMetricWithStatsTotal, resourceTypeTagFilters, regionName, startTime, endTime) assert.NoError(t, err) + assert.Equal(t, 2, len(events)) - metricValue, err := events["i-1"].RootFields.GetValue("aws.ec2.metrics.CPUUtilization.avg") + metricValue, err := events[label1+"-0"].RootFields.GetValue("aws.ec2.metrics.CPUUtilization.avg") assert.NoError(t, err) assert.Equal(t, value1, metricValue) - dimension, err := events["i-1"].RootFields.GetValue("aws.dimensions.InstanceId") + dimension, err := events[label2+"-0"].RootFields.GetValue("aws.dimensions.InstanceId") assert.NoError(t, err) assert.Equal(t, instanceID1, dimension) } @@ -1437,16 +1465,60 @@ func TestCreateEventsWithoutIdentifier(t *testing.T) { events, err := m.createEvents(mockCloudwatchSvc, mockTaggingSvc, listMetricWithStatsTotal, resourceTypeTagFilters, regionName, startTime, endTime) assert.NoError(t, err) - expectedID := regionName + accountID + namespace - metricValue, err := events[expectedID].RootFields.GetValue("aws.ec2.metrics.CPUUtilization.avg") + expectedID := regionName + accountID + metricValue, err := events[expectedID+label3+"-0"].RootFields.GetValue("aws.ec2.metrics.CPUUtilization.avg") assert.NoError(t, err) assert.Equal(t, value1, metricValue) - dimension, err := events[expectedID].RootFields.GetValue("aws.ec2.metrics.DiskReadOps.avg") + dimension, err := events[expectedID+label4+"-0"].RootFields.GetValue("aws.ec2.metrics.DiskReadOps.avg") assert.NoError(t, err) assert.Equal(t, value2, dimension) } +func TestCreateEventsWithDataGranularity(t *testing.T) { + m := MetricSet{} + m.CloudwatchConfigs = []Config{{Statistic: []string{"Average"}}} + m.MetricSet = &aws.MetricSet{Period: 10, AccountID: accountID, DataGranularity: 5} + m.logger = logp.NewLogger("test") + + mockTaggingSvc := &MockResourceGroupsTaggingClient{} + mockCloudwatchSvc := &MockCloudWatchClientWithDataGranularity{} + listMetricWithStatsTotal := []metricsWithStatistics{ + { + cloudwatchtypes.Metric{ + MetricName: awssdk.String("CPUUtilization"), + Namespace: awssdk.String("AWS/EC2"), + }, + []string{"Average"}, + }, + { + cloudwatchtypes.Metric{ + MetricName: awssdk.String("DiskReadOps"), + Namespace: awssdk.String("AWS/EC2"), + }, + []string{"Average"}, + }, + } + + resourceTypeTagFilters := map[string][]aws.Tag{} + startTime, endTime := aws.GetStartTimeEndTime(time.Now(), m.MetricSet.Period, m.MetricSet.Latency) + + events, err := m.createEvents(mockCloudwatchSvc, mockTaggingSvc, listMetricWithStatsTotal, resourceTypeTagFilters, regionName, startTime, endTime) + assert.NoError(t, err) + + expectedID := regionName + accountID + metricValue, err := events[expectedID+label3+"-0"].RootFields.GetValue("aws.ec2.metrics.CPUUtilization.avg") + metricValue1, err := events[expectedID+label3+"-1"].RootFields.GetValue("aws.ec2.metrics.CPUUtilization.avg") + metricValue2, err := events[expectedID+label4+"-0"].RootFields.GetValue("aws.ec2.metrics.DiskReadOps.avg") + metricValue3, err := events[expectedID+label4+"-1"].RootFields.GetValue("aws.ec2.metrics.DiskReadOps.avg") + assert.NoError(t, err) + assert.Equal(t, value1, metricValue) + assert.Equal(t, value1, metricValue1) + assert.Equal(t, value2, metricValue2) + assert.Equal(t, value2, metricValue3) + assert.Equal(t, 4, len(events)) +} + func TestCreateEventsWithTagsFilter(t *testing.T) { m := MetricSet{} m.CloudwatchConfigs = []Config{{Statistic: []string{"Average"}}} @@ -1560,7 +1632,7 @@ func TestInsertTags(t *testing.T) { for _, c := range cases { t.Run(c.title, func(t *testing.T) { - insertTags(events, c.identifier, resourceTagMap) + insertTags(events, c.identifier, c.identifier, resourceTagMap) value, err := events[c.identifier].RootFields.GetValue(c.expectedTagKey) assert.NoError(t, err) assert.Equal(t, c.expectedTagValue, value) From 2d87b4b3a0a952d9aa217bc65d0ee5207d81bfee Mon Sep 17 00:00:00 2001 From: girodav <1390902+girodav@users.noreply.github.com> Date: Wed, 12 Oct 2022 11:28:21 +0100 Subject: [PATCH 06/11] Add aws.asciidoc to fix CI --- metricbeat/docs/modules/aws.asciidoc | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/metricbeat/docs/modules/aws.asciidoc b/metricbeat/docs/modules/aws.asciidoc index 868ba92a184c..c7a8d9a3c2c3 100644 --- a/metricbeat/docs/modules/aws.asciidoc +++ b/metricbeat/docs/modules/aws.asciidoc @@ -49,6 +49,15 @@ or none get collected by Metricbeat. In this case, please specify a `latency` parameter so collection start time and end time will be shifted by the given latency amount. +* *data_granularity* + +This module accepts an optional `data_granularity` parameter which sets the granularity of the returned data points. +This parameter can be used to decouple how frequently metricbeat would collect data from AWS, from how granular this data should be. +This would helpful to reduce the cost incurred in calling AWS Cloudwatch, in cases where highly-granular data is required, +but a delay in collecting and ingesting the metrics can be tolerated. + +If no value is specified, the value is set to the `period`. + * *endpoint* Most AWS services offer a regional endpoint that can be used to make requests. @@ -69,7 +78,7 @@ For example, if tags parameter is given as `Organization=Engineering` under `Organization` and tag value equals to `Engineering`. In order to filter for different values for the same key, add the values to the value array (see example) -Note: tag filtering only works for metricsets with `resource_type` specified in the +Note: tag filtering only works for metricsets with `resource_type` specified in the metricset-specific configuration. [source,yaml] From 9bcd8eeb70e8bff6699a8c3ce2882160b776d89c Mon Sep 17 00:00:00 2001 From: girodav <1390902+girodav@users.noreply.github.com> Date: Wed, 12 Oct 2022 12:08:57 +0100 Subject: [PATCH 07/11] Fix "ineffectual assignment to err" returned by golint --- x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go index 14ef8b386af3..4d0c763e1310 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go @@ -1508,8 +1508,11 @@ func TestCreateEventsWithDataGranularity(t *testing.T) { expectedID := regionName + accountID metricValue, err := events[expectedID+label3+"-0"].RootFields.GetValue("aws.ec2.metrics.CPUUtilization.avg") + assert.NoError(t, err) metricValue1, err := events[expectedID+label3+"-1"].RootFields.GetValue("aws.ec2.metrics.CPUUtilization.avg") + assert.NoError(t, err) metricValue2, err := events[expectedID+label4+"-0"].RootFields.GetValue("aws.ec2.metrics.DiskReadOps.avg") + assert.NoError(t, err) metricValue3, err := events[expectedID+label4+"-1"].RootFields.GetValue("aws.ec2.metrics.DiskReadOps.avg") assert.NoError(t, err) assert.Equal(t, value1, metricValue) From 6b674a0de1ef001a191091ae834f3296d0a137dc Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Wed, 19 Oct 2022 09:59:26 -0600 Subject: [PATCH 08/11] remove FindTimestamp function and its unit test --- x-pack/metricbeat/module/aws/utils.go | 45 ------------- x-pack/metricbeat/module/aws/utils_test.go | 75 ---------------------- 2 files changed, 120 deletions(-) diff --git a/x-pack/metricbeat/module/aws/utils.go b/x-pack/metricbeat/module/aws/utils.go index d9810a8cae65..37ef6803ea06 100644 --- a/x-pack/metricbeat/module/aws/utils.go +++ b/x-pack/metricbeat/module/aws/utils.go @@ -118,51 +118,6 @@ func CheckTimestampInArray(timestamp time.Time, timestampArray []time.Time) (boo return false, -1 } -// FindTimestamp function checks MetricDataResults and find the timestamp to collect metrics from. -// For example, MetricDataResults might look like: -// -// metricDataResults = [{ -// Id: "sqs0", -// Label: "testName SentMessageSize", -// StatusCode: Complete, -// Timestamps: [2019-03-11 17:45:00 +0000 UTC], -// Values: [981] -// } { -// -// Id: "sqs1", -// Label: "testName NumberOfMessagesSent", -// StatusCode: Complete, -// Timestamps: [2019-03-11 17:45:00 +0000 UTC,2019-03-11 17:40:00 +0000 UTC], -// Values: [0.5,0] -// }] -// -// This case, we are collecting values for both metrics from timestamp 2019-03-11 17:45:00 +0000 UTC. -func FindTimestamp(getMetricDataResults []types.MetricDataResult) time.Time { - timestamp := time.Time{} - for _, output := range getMetricDataResults { - // When there are outputs with one timestamp, use this timestamp. - if output.Timestamps != nil && len(output.Timestamps) == 1 { - // Use the first timestamp from Timestamps field to collect the latest data. - timestamp = output.Timestamps[0] - return timestamp - } - } - - // When there is no output with one timestamp, use the latest timestamp from timestamp list. - if timestamp.IsZero() { - for _, output := range getMetricDataResults { - // When there are outputs with one timestamp, use this timestamp - if output.Timestamps != nil && len(output.Timestamps) > 1 { - // Example Timestamps: [2019-03-11 17:36:00 +0000 UTC,2019-03-11 17:31:00 +0000 UTC] - timestamp = output.Timestamps[0] - return timestamp - } - } - } - - return timestamp -} - // GetResourcesTags function queries AWS resource groupings tagging API // to get a resource tag mapping with specific resource type filters func GetResourcesTags(svc resourcegroupstaggingapi.GetResourcesAPIClient, resourceTypeFilters []string) (map[string][]resourcegroupstaggingapitypes.Tag, error) { diff --git a/x-pack/metricbeat/module/aws/utils_test.go b/x-pack/metricbeat/module/aws/utils_test.go index d1dad2c756b3..ebaa3121429f 100644 --- a/x-pack/metricbeat/module/aws/utils_test.go +++ b/x-pack/metricbeat/module/aws/utils_test.go @@ -260,81 +260,6 @@ func TestCheckTimestampInArray(t *testing.T) { } } -func TestFindTimestamp(t *testing.T) { - timestamp1 := time.Now() - timestamp2 := timestamp1.Add(5 * time.Minute) - cases := []struct { - getMetricDataResults []cloudwatchtypes.MetricDataResult - expectedTimestamp time.Time - }{ - { - getMetricDataResults: []cloudwatchtypes.MetricDataResult{ - { - Id: &id1, - Label: &label1, - StatusCode: cloudwatchtypes.StatusCodeComplete, - Timestamps: []time.Time{timestamp1, timestamp2}, - Values: []float64{0, 1}, - }, - { - Id: &id2, - Label: &label2, - StatusCode: cloudwatchtypes.StatusCodeComplete, - Timestamps: []time.Time{timestamp1}, - Values: []float64{2, 3}, - }, - }, - expectedTimestamp: timestamp1, - }, - { - getMetricDataResults: []cloudwatchtypes.MetricDataResult{ - { - Id: &id1, - Label: &label1, - StatusCode: cloudwatchtypes.StatusCodeComplete, - Timestamps: []time.Time{timestamp1, timestamp2}, - Values: []float64{0, 1}, - }, - { - Id: &id2, - Label: &label2, - StatusCode: cloudwatchtypes.StatusCodeComplete, - }, - }, - expectedTimestamp: timestamp1, - }, - { - getMetricDataResults: []cloudwatchtypes.MetricDataResult{ - { - Id: &id1, - Label: &label1, - StatusCode: cloudwatchtypes.StatusCodeComplete, - Timestamps: []time.Time{timestamp1, timestamp2}, - Values: []float64{0, 1}, - }, - { - Id: &id2, - Label: &label2, - StatusCode: cloudwatchtypes.StatusCodeComplete, - }, - { - Id: &id3, - Label: &label2, - StatusCode: cloudwatchtypes.StatusCodeComplete, - Timestamps: []time.Time{timestamp2}, - Values: []float64{2, 3}, - }, - }, - expectedTimestamp: timestamp2, - }, - } - - for _, c := range cases { - outputTimestamp := FindTimestamp(c.getMetricDataResults) - assert.Equal(t, c.expectedTimestamp, outputTimestamp) - } -} - func TestFindIdentifierFromARN(t *testing.T) { cases := []struct { resourceARN string From aedb26db8e048e6d55cac934f392080b6296a515 Mon Sep 17 00:00:00 2001 From: girodav <1390902+girodav@users.noreply.github.com> Date: Wed, 19 Oct 2022 17:33:36 +0100 Subject: [PATCH 09/11] Improved docs --- metricbeat/docs/modules/aws.asciidoc | 12 +++++++----- x-pack/metricbeat/module/aws/_meta/docs.asciidoc | 12 +++++++----- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/metricbeat/docs/modules/aws.asciidoc b/metricbeat/docs/modules/aws.asciidoc index c7a8d9a3c2c3..0dd31bf2543d 100644 --- a/metricbeat/docs/modules/aws.asciidoc +++ b/metricbeat/docs/modules/aws.asciidoc @@ -51,12 +51,14 @@ latency amount. * *data_granularity* -This module accepts an optional `data_granularity` parameter which sets the granularity of the returned data points. -This parameter can be used to decouple how frequently metricbeat would collect data from AWS, from how granular this data should be. -This would helpful to reduce the cost incurred in calling AWS Cloudwatch, in cases where highly-granular data is required, -but a delay in collecting and ingesting the metrics can be tolerated. +AWS CloudWatch allows to define the granularity of the returned datapoints, by setting "Period" while querying metrics. +Please see https://docs.aws.amazon.com/AmazonCloudWatch/latest/APIReference/API_MetricDataQuery.html[MetricDataQuery parameters] for more information. -If no value is specified, the value is set to the `period`. +By default, metricbeat will query CloudWatch setting "Period" to Metricbeat collection period. If you wish to set a custom value for "Period", please specify a `data_granularity` parameter. +By setting `period` and `data_granularity` together, you can control, respectively, how frequently you want your metrics to be collected and how granular they have to be. + +If you are concerned about reducing the cost derived by CloudWatch API calls made by Metricbeat with an extra delay in retrieving metrics as trade off, you may consider setting `data_granularity` and increase Metricbeat collection period. For example, +setting `data_granularity` to your current value for `period`, and doubling the value of `period`, may lead to a 50% savings in terms of GetMetricData API calls cost. * *endpoint* diff --git a/x-pack/metricbeat/module/aws/_meta/docs.asciidoc b/x-pack/metricbeat/module/aws/_meta/docs.asciidoc index 01e3264e8393..c7db56229531 100644 --- a/x-pack/metricbeat/module/aws/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/aws/_meta/docs.asciidoc @@ -39,12 +39,14 @@ latency amount. * *data_granularity* -This module accepts an optional `data_granularity` parameter which sets the granularity of the returned data points. -This parameter can be used to decouple how frequently metricbeat would collect data from AWS, from how granular this data should be. -This would helpful to reduce the cost incurred in calling AWS Cloudwatch, in cases where highly-granular data is required, -but a delay in collecting and ingesting the metrics can be tolerated. +AWS CloudWatch allows to define the granularity of the returned datapoints, by setting "Period" while querying metrics. +Please see https://docs.aws.amazon.com/AmazonCloudWatch/latest/APIReference/API_MetricDataQuery.html[MetricDataQuery parameters] for more information. -If no value is specified, the value is set to the `period`. +By default, metricbeat will query CloudWatch setting "Period" to Metricbeat collection period. If you wish to set a custom value for "Period", please specify a `data_granularity` parameter. +By setting `period` and `data_granularity` together, you can control, respectively, how frequently you want your metrics to be collected and how granular they have to be. + +If you are concerned about reducing the cost derived by CloudWatch API calls made by Metricbeat with an extra delay in retrieving metrics as trade off, you may consider setting `data_granularity` and increase Metricbeat collection period. For example, +setting `data_granularity` to your current value for `period`, and doubling the value of `period`, may lead to a 50% savings in terms of GetMetricData API calls cost. * *endpoint* From a36f07ceb99b3368187cdb1a420b90a44c79025a Mon Sep 17 00:00:00 2001 From: girodav <1390902+girodav@users.noreply.github.com> Date: Mon, 24 Oct 2022 17:51:37 +0100 Subject: [PATCH 10/11] Fixed unit tests --- .../module/aws/cloudwatch/cloudwatch_test.go | 25 ++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go index 4d0c763e1310..3a4f0104affe 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go @@ -1525,7 +1525,7 @@ func TestCreateEventsWithDataGranularity(t *testing.T) { func TestCreateEventsWithTagsFilter(t *testing.T) { m := MetricSet{} m.CloudwatchConfigs = []Config{{Statistic: []string{"Average"}}} - m.MetricSet = &aws.MetricSet{Period: 5} + m.MetricSet = &aws.MetricSet{Period: 5, AccountID: accountID} m.logger = logp.NewLogger("test") mockTaggingSvc := &MockResourceGroupsTaggingClient{} @@ -1542,6 +1542,17 @@ func TestCreateEventsWithTagsFilter(t *testing.T) { }, []string{"Average"}, }, + { + cloudwatchtypes.Metric{ + Dimensions: []cloudwatchtypes.Dimension{{ + Name: awssdk.String("InstanceId"), + Value: awssdk.String("i-1"), + }}, + MetricName: awssdk.String("DiskReadOps"), + Namespace: awssdk.String("AWS/EC2"), + }, + []string{"Average"}, + }, } // Check that the event is created when the tag filter matches @@ -1556,7 +1567,7 @@ func TestCreateEventsWithTagsFilter(t *testing.T) { startTime, endTime := aws.GetStartTimeEndTime(time.Now(), m.MetricSet.Period, m.MetricSet.Latency) events, err := m.createEvents(mockCloudwatchSvc, mockTaggingSvc, listMetricWithStatsTotal, resourceTypeTagFilters, regionName, startTime, endTime) assert.NoError(t, err) - assert.Equal(t, 1, len(events)) + assert.Equal(t, 2, len(events)) // Specify a tag filter that does not match the tag for i-1 resourceTypeTagFilters["ec2:instance"] = []aws.Tag{ @@ -1702,6 +1713,13 @@ func TestCreateEventsTimestamp(t *testing.T) { }, []string{"Average"}, }, + { + cloudwatchtypes.Metric{ + MetricName: awssdk.String("DiskReadOps"), + Namespace: awssdk.String("AWS/EC2"), + }, + []string{"Average"}, + }, } resourceTypeTagFilters := map[string][]aws.Tag{} @@ -1711,7 +1729,8 @@ func TestCreateEventsTimestamp(t *testing.T) { resGroupTaggingClientMock := &MockResourceGroupsTaggingClient{} events, err := m.createEvents(cloudwatchMock, resGroupTaggingClientMock, listMetricWithStatsTotal, resourceTypeTagFilters, regionName, startTime, endTime) assert.NoError(t, err) - assert.Equal(t, timestamp, events[regionName+accountID+namespace].Timestamp) + assert.Equal(t, timestamp, events[regionName+accountID+label3+"-0"].Timestamp) + assert.Equal(t, timestamp, events[regionName+accountID+label4+"-0"].Timestamp) } func TestGetStartTimeEndTime(t *testing.T) { From d52d84da3f16ae35da1101e00340ea7870fbe5e7 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Mon, 24 Oct 2022 11:24:17 -0600 Subject: [PATCH 11/11] update comment --- x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go index e8816f5603de..8209f5a6f5f8 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go @@ -498,7 +498,7 @@ func (m *MetricSet) createEvents(svcCloudwatch cloudwatch.GetMetricDataAPIClient labels := strings.Split(*metricDataResult.Label, labelSeparator) for valI, metricDataResultValue := range metricDataResult.Values { if len(labels) != 5 { - // when there is no identifier value in label, use region+accountID+namespace instead + // when there is no identifier value in label, use region+accountID+label+index instead identifier := regionName + m.AccountID + *metricDataResult.Label + fmt.Sprint("-", valI) if _, ok := events[identifier]; !ok { events[identifier] = aws.InitEvent(regionName, m.AccountName, m.AccountID, metricDataResult.Timestamps[valI])