[MetricBeat] support wildcard * dimension value in AWS CloudWatch m…

…odule (#19660)

* dimension value could be wildcard `*`, it is more flexible

CHANGELOG.next.asciidoc

@@ -535,6 +535,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add support for v1 consumer API in Cloud Foundry module, use it by default. {pull}19268[19268]
 - Add support for named ports in autodiscover. {pull}19398[19398]
 - Add param `aws_partition` to support aws-cn, aws-us-gov regions. {issue}18850[18850] {pull}19423[19423]
+- Add support for wildcard `*` in dimension value of AWS CloudWatch metrics config. {issue}18050[18050] {pull}19660[19660]
 - The `elasticsearch/index` metricset now collects metrics for hidden indices as well. {issue}18639[18639] {pull}18703[18703]
 
 *Packetbeat*

x-pack/metricbeat/module/aws/cloudwatch/_meta/docs.asciidoc

@@ -20,7 +20,8 @@ iam:ListAccountAliases
 For example, AWS/EC2, AWS/S3. If wildcard * is given for namespace, metrics
 from all namespaces will be collected automatically.
 * *name*: The name of the metric to filter against. For example, CPUUtilization for EC2 instance.
-* *dimensions*: The dimensions to filter against. For example, InstanceId=i-123.
+* *dimensions*: The dimensions to filter against. For example, InstanceId=i-123. Dimension value
+could be wildcard `*` to match any value.
 * *tags.resource_type_filter*: The constraints on the resources that you want returned.
 The format of each resource type is service[:resourceType].
 For example, specifying a resource type of ec2 returns all Amazon EC2 resources
@@ -158,3 +159,26 @@ metric(average) from EC2 instance i-456.
           value: i-456
       statistic: ["Average"]
 ----
+
+
+With the configuration below, user can filter out only `LoadBalacer` and `TargetGroup` dimension
+metircs with the metric name `UnHealthyHostCount`, `LoadBalacer` and `TargetGroup` value could
+be any.
+
+[source,yaml]
+----
+- module: aws
+  period: 300s
+  metricsets:
+    - cloudwatch
+  metrics:
+    - namespace: AWS/ApplicationELB
+      statistic: ['Maximum']
+      name: ['UnHealthyHostCount']
+      dimensions:
+        - name: LoadBalancer
+          value: "*"
+        - name: TargetGroup
+          value: "*"
+      tags.resource_type_filter: elasticloadbalancing
+----

x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go

@@ -6,7 +6,6 @@ package cloudwatch
 
 import (
 	"reflect"
-	"sort"
 	"strconv"
 	"strings"
 	"time"
@@ -26,15 +25,16 @@ import (
 )
 
 var (
-	metricsetName      = "cloudwatch"
-	metricNameIdx      = 0
-	namespaceIdx       = 1
-	statisticIdx       = 2
-	identifierNameIdx  = 3
-	identifierValueIdx = 4
-	defaultStatistics  = []string{"Average", "Maximum", "Minimum", "Sum", "SampleCount"}
-	labelSeparator     = "|"
-	dimensionSeparator = ","
+	metricsetName          = "cloudwatch"
+	metricNameIdx          = 0
+	namespaceIdx           = 1
+	statisticIdx           = 2
+	identifierNameIdx      = 3
+	identifierValueIdx     = 4
+	defaultStatistics      = []string{"Average", "Maximum", "Minimum", "Sum", "SampleCount"}
+	labelSeparator         = "|"
+	dimensionSeparator     = ","
+	dimensionValueWildcard = "*"
 )
 
 // init registers the MetricSet with the central registry as soon as the program
@@ -252,8 +252,21 @@ func filterListMetricsOutput(listMetricsOutput []cloudwatch.Metric, namespaceDet
 						statistic:        configPerNamespace.statistics,
 						tags:             configPerNamespace.tags,
 					})
+			} else if configPerNamespace.names != nil && configPerNamespace.dimensions != nil {
+				if exists, _ := aws.StringInSlice(*listMetric.MetricName, configPerNamespace.names); !exists {
+					continue
+				}
+				if !compareAWSDimensions(listMetric.Dimensions, configPerNamespace.dimensions) {
+					continue
+				}
+				filteredMetricWithStatsTotal = append(filteredMetricWithStatsTotal,
+					metricsWithStatistics{
+						cloudwatchMetric: listMetric,
+						statistic:        configPerNamespace.statistics,
+						tags:             configPerNamespace.tags,
+					})
 			} else {
-				// if no metric name or dimensions given, then keep all listMetricsOutput
+				// if no metric name and no dimensions given, then keep all listMetricsOutput
 				filteredMetricWithStatsTotal = append(filteredMetricWithStatsTotal,
 					metricsWithStatistics{
 						cloudwatchMetric: listMetric,
@@ -320,8 +333,10 @@ func (m *MetricSet) readCloudwatchConfig() (listMetricWithDetail, map[string][]n
 				Value: &value,
 			})
 		}
-
-		if config.MetricName != nil && config.Dimensions != nil {
+		// if any Dimension value contains wildcard, then compare dimensions with
+		// listMetrics result in filterListMetricsOutput
+		if config.MetricName != nil && config.Dimensions != nil &&
+			!configDimensionValueContainsWildcard(config.Dimensions) {
 			namespace := config.Namespace
 			for i := range config.MetricName {
 				metricsWithStats := metricsWithStatistics{
@@ -589,22 +604,37 @@ func reportEvents(eventsWithIdentifier map[string]mb.Event, report mb.ReporterV2
 	return nil
 }
 
+func configDimensionValueContainsWildcard(dim []Dimension) bool {
+	for i := range dim {
+		if dim[i].Value == dimensionValueWildcard {
+			return true
+		}
+	}
+	return false
+}
+
 func compareAWSDimensions(dim1 []cloudwatch.Dimension, dim2 []cloudwatch.Dimension) bool {
 	if len(dim1) != len(dim2) {
 		return false
 	}
-	var dim1String []string
-	var dim2String []string
-	for i := range dim1 {
-		dim1String = append(dim1String, dim1[i].String())
-	}
+
+	var dim1NameToValue = make(map[string]string, len(dim1))
+	var dim2NameToValue = make(map[string]string, len(dim1))
+
 	for i := range dim2 {
-		dim2String = append(dim2String, dim2[i].String())
+		dim1NameToValue[*dim1[i].Name] = *dim1[i].Value
+		dim2NameToValue[*dim2[i].Name] = *dim2[i].Value
 	}
-
-	sort.Strings(dim1String)
-	sort.Strings(dim2String)
-	return reflect.DeepEqual(dim1String, dim2String)
+	for name, v1 := range dim1NameToValue {
+		v2, exists := dim2NameToValue[name]
+		if exists && v2 == dimensionValueWildcard {
+			// wildcard can represent any value, so we set the
+			// dimension name with value in CloudWatch ListMetircs result,
+			// then the compare result is true
+			dim2NameToValue[name] = v1
+		}
+	}
+	return reflect.DeepEqual(dim1NameToValue, dim2NameToValue)
 }
 
 func insertTags(events map[string]mb.Event, identifier string, resourceTagMap map[string][]resourcegroupstaggingapi.Tag) {

x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go

@@ -834,6 +834,60 @@ func TestCompareAWSDimensions(t *testing.T) {
 			[]cloudwatch.Dimension{},
 			false,
 		},
+		{
+			"compare with wildcard dimension value, one same name dimension",
+			[]cloudwatch.Dimension{
+				{Name: awssdk.String("ID1"), Value: awssdk.String("111")},
+			},
+			[]cloudwatch.Dimension{
+				{Name: awssdk.String("ID1"), Value: awssdk.String(dimensionValueWildcard)},
+			},
+			true,
+		},
+		{
+			"compare with wildcard dimension value, one different name dimension",
+			[]cloudwatch.Dimension{
+				{Name: awssdk.String("IDx"), Value: awssdk.String("111")},
+			},
+			[]cloudwatch.Dimension{
+				{Name: awssdk.String("ID1"), Value: awssdk.String(dimensionValueWildcard)},
+			},
+			false,
+		},
+		{
+			"compare with wildcard dimension value, two same name dimensions",
+			[]cloudwatch.Dimension{
+				{Name: awssdk.String("ID1"), Value: awssdk.String("111")},
+				{Name: awssdk.String("ID2"), Value: awssdk.String("222")},
+			},
+			[]cloudwatch.Dimension{
+				{Name: awssdk.String("ID1"), Value: awssdk.String("111")},
+				{Name: awssdk.String("ID2"), Value: awssdk.String(dimensionValueWildcard)},
+			},
+			true,
+		},
+		{
+			"compare with wildcard dimension value, different length, case1",
+			[]cloudwatch.Dimension{
+				{Name: awssdk.String("ID1"), Value: awssdk.String("111")},
+				{Name: awssdk.String("ID2"), Value: awssdk.String("222")},
+			},
+			[]cloudwatch.Dimension{
+				{Name: awssdk.String("ID2"), Value: awssdk.String(dimensionValueWildcard)},
+			},
+			false,
+		},
+		{
+			"compare with wildcard dimension value, different length, case2",
+			[]cloudwatch.Dimension{
+				{Name: awssdk.String("ID1"), Value: awssdk.String("111")},
+			},
+			[]cloudwatch.Dimension{
+				{Name: awssdk.String("ID1"), Value: awssdk.String("111")},
+				{Name: awssdk.String("ID2"), Value: awssdk.String(dimensionValueWildcard)},
+			},
+			false,
+		},
 	}
 
 	for _, c := range cases {
@@ -1471,3 +1525,47 @@ func TestInsertTags(t *testing.T) {
 		})
 	}
 }
+
+func TestConfigDimensionValueContainsWildcard(t *testing.T) {
+	cases := []struct {
+		title          string
+		dimensions     []Dimension
+		expectedResult bool
+	}{
+		{
+			"test dimensions without wolidcard value",
+			[]Dimension{
+				{
+					Name:  "InstanceId",
+					Value: "i-111111",
+				},
+				{
+					Name:  "InstanceId",
+					Value: "i-2222",
+				},
+			},
+			false,
+		},
+		{
+			"test dimensions without wolidcard value",
+			[]Dimension{
+				{
+					Name:  "InstanceId",
+					Value: "i-111111",
+				},
+				{
+					Name:  "InstanceId",
+					Value: dimensionValueWildcard,
+				},
+			},
+			true,
+		},
+	}
+
+	for _, c := range cases {
+		t.Run(c.title, func(t *testing.T) {
+			result := configDimensionValueContainsWildcard(c.dimensions)
+			assert.Equal(t, c.expectedResult, result)
+		})
+	}
+}