Add container azure metricset (elastic#16421)

* add container * update changelog * fix test * separate container * separate * regenerate * fixes * tests * fix test (cherry picked from commit d89675f)
narph · Mar 3, 2020 · 89525e9 · 89525e9
1 parent 949fb14
commit 89525e9
Show file tree

Hide file tree

Showing 39 changed files with 984 additions and 52 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -267,6 +267,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add database_account azure metricset. {issue}15758[15758]
 - Add support for Dropwizard metrics 4.1. {pull}16332[16332]
 - Add support for NATS 2.1. {pull}16317[16317]
+- Add azure container metricset in order to monitor containers. {issue}15751[15751] {pull}16421[16421]
 - Improve the `haproxy` module to support metrics exposed via HTTPS. {issue}14579[14579] {pull}16333[16333]
 - Add filtering option for prometheus collector. {pull}16420[16420]
 - Add metricsets based on Ceph Manager Daemon to the `ceph` module. {issue}7723[7723] {pull}16254[16254]

diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc
@@ -3207,6 +3207,36 @@ type: object
 compute_vm_scaleset
 
 
+type: object
+
+--
+
+*`azure.container_instance.*.*`*::
++
+--
+container instance
+
+
+type: object
+
+--
+
+*`azure.container_registry.*.*`*::
++
+--
+container registry
+
+
+type: object
+
+--
+
+*`azure.container_service.*.*`*::
++
+--
+container service
+
+
 type: object
 
 --

diff --git a/metricbeat/docs/modules/azure.asciidoc b/metricbeat/docs/modules/azure.asciidoc
@@ -150,6 +150,37 @@ metricbeat.modules:
   tenant_id: '${AZURE_TENANT_ID:""}'
   subscription_id: '${AZURE_SUBSCRIPTION_ID:""}'
 
+- module: azure
+  metricsets:
+  - container_instance
+  enabled: true
+  period: 300s
+  client_id: '${AZURE_CLIENT_ID:""}'
+  client_secret: '${AZURE_CLIENT_SECRET:""}'
+  tenant_id: '${AZURE_TENANT_ID:""}'
+  subscription_id: '${AZURE_SUBSCRIPTION_ID:""}'
+
+- module: azure
+  metricsets:
+  - container_service
+  enabled: true
+  period: 300s
+  client_id: '${AZURE_CLIENT_ID:""}'
+  client_secret: '${AZURE_CLIENT_SECRET:""}'
+  tenant_id: '${AZURE_TENANT_ID:""}'
+  subscription_id: '${AZURE_SUBSCRIPTION_ID:""}'
+
+- module: azure
+  metricsets:
+  - container_registry
+  enabled: true
+  period: 300s
+  client_id: '${AZURE_CLIENT_ID:""}'
+  client_secret: '${AZURE_CLIENT_SECRET:""}'
+  tenant_id: '${AZURE_TENANT_ID:""}'
+  subscription_id: '${AZURE_SUBSCRIPTION_ID:""}'
+
+
 - module: azure
   metricsets:
   - database_account
@@ -171,6 +202,12 @@ The following metricsets are available:
 
 * <<metricbeat-metricset-azure-compute_vm_scaleset,compute_vm_scaleset>>
 
+* <<metricbeat-metricset-azure-container_instance,container_instance>>
+
+* <<metricbeat-metricset-azure-container_registry,container_registry>>
+
+* <<metricbeat-metricset-azure-container_service,container_service>>
+
 * <<metricbeat-metricset-azure-database_account,database_account>>
 
 * <<metricbeat-metricset-azure-monitor,monitor>>
@@ -181,6 +218,12 @@ include::azure/compute_vm.asciidoc[]
 
 include::azure/compute_vm_scaleset.asciidoc[]
 
+include::azure/container_instance.asciidoc[]
+
+include::azure/container_registry.asciidoc[]
+
+include::azure/container_service.asciidoc[]
+
 include::azure/database_account.asciidoc[]
 
 include::azure/monitor.asciidoc[]

diff --git a/metricbeat/docs/modules/azure/container_instance.asciidoc b/metricbeat/docs/modules/azure/container_instance.asciidoc
@@ -0,0 +1,24 @@
+////
+This file is generated! See scripts/mage/docs_collector.go
+////
+
+[[metricbeat-metricset-azure-container_instance]]
+=== azure container_instance metricset
+
+beta[]
+
+include::../../../../x-pack/metricbeat/module/azure/container_instance/_meta/docs.asciidoc[]
+
+This is a default metricset. If the host module is unconfigured, this metricset is enabled by default.
+
+==== Fields
+
+For a description of each field in the metricset, see the
+<<exported-fields-azure,exported fields>> section.
+
+Here is an example document generated by this metricset:
+
+[source,json]
+----
+include::../../../../x-pack/metricbeat/module/azure/container_instance/_meta/data.json[]
+----
diff --git a/metricbeat/docs/modules/azure/container_registry.asciidoc b/metricbeat/docs/modules/azure/container_registry.asciidoc
@@ -0,0 +1,24 @@
+////
+This file is generated! See scripts/mage/docs_collector.go
+////
+
+[[metricbeat-metricset-azure-container_registry]]
+=== azure container_registry metricset
+
+beta[]
+
+include::../../../../x-pack/metricbeat/module/azure/container_registry/_meta/docs.asciidoc[]
+
+This is a default metricset. If the host module is unconfigured, this metricset is enabled by default.
+
+==== Fields
+
+For a description of each field in the metricset, see the
+<<exported-fields-azure,exported fields>> section.
+
+Here is an example document generated by this metricset:
+
+[source,json]
+----
+include::../../../../x-pack/metricbeat/module/azure/container_registry/_meta/data.json[]
+----
diff --git a/metricbeat/docs/modules/azure/container_service.asciidoc b/metricbeat/docs/modules/azure/container_service.asciidoc
@@ -0,0 +1,24 @@
+////
+This file is generated! See scripts/mage/docs_collector.go
+////
+
+[[metricbeat-metricset-azure-container_service]]
+=== azure container_service metricset
+
+beta[]
+
+include::../../../../x-pack/metricbeat/module/azure/container_service/_meta/docs.asciidoc[]
+
+This is a default metricset. If the host module is unconfigured, this metricset is enabled by default.
+
+==== Fields
+
+For a description of each field in the metricset, see the
+<<exported-fields-azure,exported fields>> section.
+
+Here is an example document generated by this metricset:
+
+[source,json]
+----
+include::../../../../x-pack/metricbeat/module/azure/container_service/_meta/data.json[]
+----
diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc
@@ -31,8 +31,11 @@ This file is generated! See scripts/mage/docs_collector.go
 |<<metricbeat-metricset-aws-usage,usage>> beta[]  
 |<<metricbeat-metricset-aws-vpc,vpc>> beta[]  
 |<<metricbeat-module-azure,azure>>  beta[]   |image:./images/icon-yes.png[Prebuilt dashboards are available]    |  
-.5+| .5+|  |<<metricbeat-metricset-azure-compute_vm,compute_vm>> beta[]  
+.8+| .8+|  |<<metricbeat-metricset-azure-compute_vm,compute_vm>> beta[]  
 |<<metricbeat-metricset-azure-compute_vm_scaleset,compute_vm_scaleset>> beta[]  
+|<<metricbeat-metricset-azure-container_instance,container_instance>> beta[]  
+|<<metricbeat-metricset-azure-container_registry,container_registry>> beta[]  
+|<<metricbeat-metricset-azure-container_service,container_service>> beta[]  
 |<<metricbeat-metricset-azure-database_account,database_account>> beta[]  
 |<<metricbeat-metricset-azure-monitor,monitor>> beta[]  
 |<<metricbeat-metricset-azure-storage,storage>> beta[]  

diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml
@@ -265,6 +265,37 @@ metricbeat.modules:
   tenant_id: '${AZURE_TENANT_ID:""}'
   subscription_id: '${AZURE_SUBSCRIPTION_ID:""}'
 
+- module: azure
+  metricsets:
+  - container_instance
+  enabled: true
+  period: 300s
+  client_id: '${AZURE_CLIENT_ID:""}'
+  client_secret: '${AZURE_CLIENT_SECRET:""}'
+  tenant_id: '${AZURE_TENANT_ID:""}'
+  subscription_id: '${AZURE_SUBSCRIPTION_ID:""}'
+
+- module: azure
+  metricsets:
+  - container_service
+  enabled: true
+  period: 300s
+  client_id: '${AZURE_CLIENT_ID:""}'
+  client_secret: '${AZURE_CLIENT_SECRET:""}'
+  tenant_id: '${AZURE_TENANT_ID:""}'
+  subscription_id: '${AZURE_SUBSCRIPTION_ID:""}'
+
+- module: azure
+  metricsets:
+  - container_registry
+  enabled: true
+  period: 300s
+  client_id: '${AZURE_CLIENT_ID:""}'
+  client_secret: '${AZURE_CLIENT_SECRET:""}'
+  tenant_id: '${AZURE_TENANT_ID:""}'
+  subscription_id: '${AZURE_SUBSCRIPTION_ID:""}'
+
+
 - module: azure
   metricsets:
   - database_account

diff --git a/x-pack/metricbeat/module/azure/_meta/config.reference.yml b/x-pack/metricbeat/module/azure/_meta/config.reference.yml
@@ -43,6 +43,37 @@
   tenant_id: '${AZURE_TENANT_ID:""}'
   subscription_id: '${AZURE_SUBSCRIPTION_ID:""}'
 
+- module: azure
+  metricsets:
+  - container_instance
+  enabled: true
+  period: 300s
+  client_id: '${AZURE_CLIENT_ID:""}'
+  client_secret: '${AZURE_CLIENT_SECRET:""}'
+  tenant_id: '${AZURE_TENANT_ID:""}'
+  subscription_id: '${AZURE_SUBSCRIPTION_ID:""}'
+
+- module: azure
+  metricsets:
+  - container_service
+  enabled: true
+  period: 300s
+  client_id: '${AZURE_CLIENT_ID:""}'
+  client_secret: '${AZURE_CLIENT_SECRET:""}'
+  tenant_id: '${AZURE_TENANT_ID:""}'
+  subscription_id: '${AZURE_SUBSCRIPTION_ID:""}'
+
+- module: azure
+  metricsets:
+  - container_registry
+  enabled: true
+  period: 300s
+  client_id: '${AZURE_CLIENT_ID:""}'
+  client_secret: '${AZURE_CLIENT_SECRET:""}'
+  tenant_id: '${AZURE_TENANT_ID:""}'
+  subscription_id: '${AZURE_SUBSCRIPTION_ID:""}'
+
+
 - module: azure
   metricsets:
   - database_account

diff --git a/x-pack/metricbeat/module/azure/_meta/config.yml b/x-pack/metricbeat/module/azure/_meta/config.yml
@@ -47,6 +47,39 @@
 #  subscription_id: '${AZURE_SUBSCRIPTION_ID:""}'
 #  refresh_list_interval: 600s
 
+#- module: azure
+#  metricsets:
+#  - container_instance
+#  enabled: true
+#  period: 300s
+#  client_id: '${AZURE_CLIENT_ID:""}'
+#  client_secret: '${AZURE_CLIENT_SECRET:""}'
+#  tenant_id: '${AZURE_TENANT_ID:""}'
+#  subscription_id: '${AZURE_SUBSCRIPTION_ID:""}'
+#  refresh_list_interval: 600s
+
+#- module: azure
+#  metricsets:
+#  - container_service
+#  enabled: true
+#  period: 300s
+#  client_id: '${AZURE_CLIENT_ID:""}'
+#  client_secret: '${AZURE_CLIENT_SECRET:""}'
+#  tenant_id: '${AZURE_TENANT_ID:""}'
+#  subscription_id: '${AZURE_SUBSCRIPTION_ID:""}'
+#  refresh_list_interval: 600s
+
+#- module: azure
+#  metricsets:
+#  - container_registry
+#  enabled: true
+#  period: 300s
+#  client_id: '${AZURE_CLIENT_ID:""}'
+#  client_secret: '${AZURE_CLIENT_SECRET:""}'
+#  tenant_id: '${AZURE_TENANT_ID:""}'
+#  subscription_id: '${AZURE_SUBSCRIPTION_ID:""}'
+#  refresh_list_interval: 600s
+
 #- module: azure
 #  metricsets:
 #  - database_account

diff --git a/x-pack/metricbeat/module/azure/azure.go b/x-pack/metricbeat/module/azure/azure.go
@@ -106,14 +106,16 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) {
 		// check for lightweight resources if no groups or ids have been entered, if not a new resource is created to check the entire subscription
 		var resources []ResourceConfig
 		for _, resource := range config.Resources {
-			if len(resource.Group) != 0 || len(resource.ID) != 0 {
+			if hasConfigOptions(resource.Group) || hasConfigOptions(resource.ID) {
 				resources = append(resources, resource)
 			}
 		}
+		// check if this is a light metricset or not and no resources have been configured
 		if len(resources) == 0 && len(config.Resources) != 0 {
-			res := config.Resources[0]
-			res.Query = fmt.Sprintf("resourceType eq '%s'", config.DefaultResourceType)
-			resources = append(resources, res)
+			resources = append(resources, ResourceConfig{
+				Query:   fmt.Sprintf("resourceType eq '%s'", config.DefaultResourceType),
+				Metrics: config.Resources[0].Metrics,
+			})
 		}
 		config.Resources = resources
 	}
@@ -152,3 +154,16 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error {
 	}
 	return nil
 }
+
+// hasConfigOptions func will check if any resource id or resource group options have been entered in the light metricsets
+func hasConfigOptions(config []string) bool {
+	if config == nil {
+		return false
+	}
+	for _, group := range config {
+		if group == "" {
+			return false
+		}
+	}
+	return true
+}
diff --git a/x-pack/metricbeat/module/azure/client.go b/x-pack/metricbeat/module/azure/client.go
@@ -114,13 +114,13 @@ func (client *Client) GetMetricValues(metrics []Metric, report mb.ReporterV2) []
 			client.Log.Error(err)
 			report.Error(err)
 		} else {
-			if metric.TimeGrain == "" {
-				metric.TimeGrain = timegrain
-			}
 			for i, currentMetric := range client.Resources.Metrics {
 				if matchMetrics(currentMetric, metric) {
 					current := mapMetricValues(resp, currentMetric.Values, endTime.Truncate(time.Minute).Add(interval*(-1)), endTime.Truncate(time.Minute))
 					client.Resources.Metrics[i].Values = current
+					if client.Resources.Metrics[i].TimeGrain == "" {
+						client.Resources.Metrics[i].TimeGrain = timegrain
+					}
 					resultedMetrics = append(resultedMetrics, client.Resources.Metrics[i])
 				}
 			}

diff --git a/x-pack/metricbeat/module/azure/client_utils.go b/x-pack/metricbeat/module/azure/client_utils.go
@@ -188,3 +188,13 @@ func groupMetricsByResource(metrics []Metric) map[string][]Metric {
 	}
 	return grouped
 }
+
+// ContainsDimension will check if the dimension value is found in the list
+func ContainsDimension(dimension string, dimensions []insights.LocalizableString) bool {
+	for _, dim := range dimensions {
+		if *dim.Value == dimension {
+			return true
+		}
+	}
+	return false
+}