diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 4d254fe0a89..632de702330 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -277,6 +277,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Release `statsd` module as GA. {pull}16447[16447] {issue}14280[14280] - Add `redisenterprise` module. {pull}16482[16482] {issue}15269[15269] - Add `cloudfoundry` module to send events from Cloud Foundry. {pull}16671[16671] +- Align fields to ECS and add more tests for the azure module. {issue}16024[16024] {pull}16754[16754] *Packetbeat* diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index e4cabfcb168..5858935bbb3 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -3123,16 +3123,6 @@ The resource specified -*`azure.resource.name`*:: -+ --- -The name of the resource - - -type: keyword - --- - *`azure.resource.type`*:: + -- diff --git a/x-pack/metricbeat/module/azure/_meta/fields.yml b/x-pack/metricbeat/module/azure/_meta/fields.yml index 4d51e824087..61913ce0019 100644 --- a/x-pack/metricbeat/module/azure/_meta/fields.yml +++ b/x-pack/metricbeat/module/azure/_meta/fields.yml @@ -17,10 +17,6 @@ description: > The resource specified fields: - - name: name - type: keyword - description: > - The name of the resource - name: type type: keyword description: > diff --git a/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-blob-storage-overview.json b/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-blob-storage-overview.json index b29fd135f81..aad73bdc922 100644 --- a/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-blob-storage-overview.json +++ b/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-blob-storage-overview.json @@ -483,7 +483,7 @@ "index_pattern": "metricbeat-*", "interval": "60m", "isModelInvalid": false, - "pivot_id": "azure.resource.name", + "pivot_id": "cloud.instance.name", "pivot_label": "Resource Name", "pivot_rows": "30", "pivot_type": "string", @@ -513,7 +513,7 @@ "split_color_mode": "gradient", "split_mode": "everything", "stacked": "none", - "terms_field": "azure.resource.name", + "terms_field": "cloud.instance.name", "terms_order_by": "61fb4192-32e4-11ea-b9f8-4d0b340ad993", "type": "timeseries", "value_template": "{{value}}" @@ -616,7 +616,7 @@ "split_color_mode": "gradient", "split_mode": "everything", "stacked": "none", - "terms_field": "azure.resource.name", + "terms_field": "cloud.instance.name", "terms_order_by": "61fb4192-32e4-11ea-b9f8-4d0b340ad993", "type": "timeseries", "value_template": "{{value}}" @@ -767,7 +767,7 @@ "type": "list" }, { - "fieldName": "azure.resource.name", + "fieldName": "cloud.instance.name", "id": "1578578146383", "indexPatternRefName": "control_2_index_pattern", "label": "Resource Name", @@ -1048,7 +1048,7 @@ "split_color_mode": "gradient", "split_mode": "everything", "stacked": "none", - "terms_field": "azure.resource.name", + "terms_field": "cloud.instance.name", "terms_order_by": "61fb4192-32e4-11ea-b9f8-4d0b340ad993", "type": "timeseries", "value_template": "{{value}}" diff --git a/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-file-storage-overview.json b/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-file-storage-overview.json index a2c911a036c..b0ea29b3c66 100644 --- a/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-file-storage-overview.json +++ b/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-file-storage-overview.json @@ -523,7 +523,7 @@ "index_pattern": "metricbeat-*", "interval": "60m", "isModelInvalid": false, - "pivot_id": "azure.resource.name", + "pivot_id": "cloud.instance.name", "pivot_label": "Resource Name", "pivot_rows": "30", "pivot_type": "string", @@ -553,7 +553,7 @@ "split_color_mode": "gradient", "split_mode": "everything", "stacked": "none", - "terms_field": "azure.resource.name", + "terms_field": "cloud.instance.name", "terms_order_by": "61fb4192-32e4-11ea-b9f8-4d0b340ad993", "type": "timeseries", "value_template": "{{value}}" @@ -656,7 +656,7 @@ "split_color_mode": "gradient", "split_mode": "everything", "stacked": "none", - "terms_field": "azure.resource.name", + "terms_field": "cloud.instance.name", "terms_order_by": "61fb4192-32e4-11ea-b9f8-4d0b340ad993", "type": "timeseries", "value_template": "{{value}}" @@ -759,7 +759,7 @@ "split_color_mode": "gradient", "split_mode": "everything", "stacked": "none", - "terms_field": "azure.resource.name", + "terms_field": "cloud.instance.name", "terms_order_by": "61fb4192-32e4-11ea-b9f8-4d0b340ad993", "type": "timeseries", "value_template": "{{value}}" @@ -910,7 +910,7 @@ "type": "list" }, { - "fieldName": "azure.resource.name", + "fieldName": "cloud.instance.name", "id": "1578578146383", "indexPatternRefName": "control_2_index_pattern", "label": "Resource Name", @@ -1165,7 +1165,7 @@ "index_pattern": "metricbeat-*", "interval": "60m", "isModelInvalid": false, - "pivot_id": "azure.resource.name", + "pivot_id": "cloud.instance.name", "pivot_label": "Resource Name", "pivot_rows": "30", "pivot_type": "string", @@ -1195,7 +1195,7 @@ "split_color_mode": "gradient", "split_mode": "everything", "stacked": "none", - "terms_field": "azure.resource.name", + "terms_field": "cloud.instance.name", "terms_order_by": "61fb4192-32e4-11ea-b9f8-4d0b340ad993", "type": "timeseries", "value_template": "{{value}}" @@ -1272,7 +1272,7 @@ "index_pattern": "metricbeat-*", "interval": "60m", "isModelInvalid": false, - "pivot_id": "azure.resource.name", + "pivot_id": "cloud.instance.name", "pivot_label": "Resource Name", "pivot_rows": "30", "pivot_type": "string", @@ -1302,7 +1302,7 @@ "split_color_mode": "gradient", "split_mode": "everything", "stacked": "none", - "terms_field": "azure.resource.name", + "terms_field": "cloud.instance.name", "terms_order_by": "61fb4192-32e4-11ea-b9f8-4d0b340ad993", "type": "timeseries", "value_template": "{{value}}" diff --git a/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-queue-storage-overview.json b/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-queue-storage-overview.json index e447d41aa23..8783bbaa8a6 100644 --- a/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-queue-storage-overview.json +++ b/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-queue-storage-overview.json @@ -483,7 +483,7 @@ "index_pattern": "metricbeat-*", "interval": "60m", "isModelInvalid": false, - "pivot_id": "azure.resource.name", + "pivot_id": "cloud.instance.name", "pivot_label": "Resource Name", "pivot_rows": "30", "pivot_type": "string", @@ -513,7 +513,7 @@ "split_color_mode": "gradient", "split_mode": "everything", "stacked": "none", - "terms_field": "azure.resource.name", + "terms_field": "cloud.instance.name", "terms_order_by": "61fb4192-32e4-11ea-b9f8-4d0b340ad993", "type": "timeseries", "value_template": "{{value}}" @@ -616,7 +616,7 @@ "split_color_mode": "gradient", "split_mode": "everything", "stacked": "none", - "terms_field": "azure.resource.name", + "terms_field": "cloud.instance.name", "terms_order_by": "61fb4192-32e4-11ea-b9f8-4d0b340ad993", "type": "timeseries", "value_template": "{{value}}" @@ -719,7 +719,7 @@ "split_color_mode": "gradient", "split_mode": "everything", "stacked": "none", - "terms_field": "azure.resource.name", + "terms_field": "cloud.instance.name", "terms_order_by": "61fb4192-32e4-11ea-b9f8-4d0b340ad993", "type": "timeseries", "value_template": "{{value}}" @@ -870,7 +870,7 @@ "type": "list" }, { - "fieldName": "azure.resource.name", + "fieldName": "cloud.instance.name", "id": "1578578146383", "indexPatternRefName": "control_2_index_pattern", "label": "Resource Name", diff --git a/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-storage-overview.json b/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-storage-overview.json index 4dcabeb10e1..4de70977eeb 100644 --- a/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-storage-overview.json +++ b/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-storage-overview.json @@ -474,7 +474,7 @@ "index_pattern": "metricbeat-*", "interval": "60m", "isModelInvalid": false, - "pivot_id": "azure.resource.name", + "pivot_id": "cloud.instance.name", "pivot_label": "Resource Name", "pivot_rows": "30", "pivot_type": "string", @@ -504,7 +504,7 @@ "split_color_mode": "gradient", "split_mode": "everything", "stacked": "none", - "terms_field": "azure.resource.name", + "terms_field": "cloud.instance.name", "terms_order_by": "61fb4192-32e4-11ea-b9f8-4d0b340ad993", "type": "timeseries", "value_template": "{{value}}" @@ -607,7 +607,7 @@ "split_color_mode": "gradient", "split_mode": "everything", "stacked": "none", - "terms_field": "azure.resource.name", + "terms_field": "cloud.instance.name", "terms_order_by": "61fb4192-32e4-11ea-b9f8-4d0b340ad993", "type": "timeseries", "value_template": "{{value}}" @@ -710,7 +710,7 @@ "split_color_mode": "gradient", "split_mode": "everything", "stacked": "none", - "terms_field": "azure.resource.name", + "terms_field": "cloud.instance.name", "terms_order_by": "61fb4192-32e4-11ea-b9f8-4d0b340ad993", "type": "timeseries", "value_template": "{{value}}" @@ -861,7 +861,7 @@ "type": "list" }, { - "fieldName": "azure.resource.name", + "fieldName": "cloud.instance.name", "id": "1578578146383", "indexPatternRefName": "control_2_index_pattern", "label": "Resource Name", @@ -1116,7 +1116,7 @@ "index_pattern": "metricbeat-*", "interval": "60m", "isModelInvalid": false, - "pivot_id": "azure.resource.name", + "pivot_id": "cloud.instance.name", "pivot_label": "Resource Name", "pivot_rows": "30", "pivot_type": "string", @@ -1146,7 +1146,7 @@ "split_color_mode": "gradient", "split_mode": "terms", "stacked": "none", - "terms_field": "azure.resource.name", + "terms_field": "cloud.instance.name", "terms_order_by": "61fb4192-32e4-11ea-b9f8-4d0b340ad993", "type": "timeseries", "value_template": "{{value}}" diff --git a/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-table-storage-overview.json b/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-table-storage-overview.json index 6b2fcd65dc0..ad7bbfe8970 100644 --- a/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-table-storage-overview.json +++ b/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-table-storage-overview.json @@ -483,7 +483,7 @@ "index_pattern": "metricbeat-*", "interval": "60m", "isModelInvalid": false, - "pivot_id": "azure.resource.name", + "pivot_id": "cloud.instance.name", "pivot_label": "Resource Name", "pivot_rows": "30", "pivot_type": "string", @@ -513,7 +513,7 @@ "split_color_mode": "gradient", "split_mode": "everything", "stacked": "none", - "terms_field": "azure.resource.name", + "terms_field": "cloud.instance.name", "terms_order_by": "61fb4192-32e4-11ea-b9f8-4d0b340ad993", "type": "timeseries", "value_template": "{{value}}" @@ -616,7 +616,7 @@ "split_color_mode": "gradient", "split_mode": "everything", "stacked": "none", - "terms_field": "azure.resource.name", + "terms_field": "cloud.instance.name", "terms_order_by": "61fb4192-32e4-11ea-b9f8-4d0b340ad993", "type": "timeseries", "value_template": "{{value}}" @@ -719,7 +719,7 @@ "split_color_mode": "gradient", "split_mode": "everything", "stacked": "none", - "terms_field": "azure.resource.name", + "terms_field": "cloud.instance.name", "terms_order_by": "61fb4192-32e4-11ea-b9f8-4d0b340ad993", "type": "timeseries", "value_template": "{{value}}" @@ -870,7 +870,7 @@ "type": "list" }, { - "fieldName": "azure.resource.name", + "fieldName": "cloud.instance.name", "id": "1578578146383", "indexPatternRefName": "control_2_index_pattern", "label": "Resource Name", diff --git a/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-vm-guestmetrics-overview.json b/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-vm-guestmetrics-overview.json index 873616c609c..0613a7e4a8e 100644 --- a/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-vm-guestmetrics-overview.json +++ b/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-vm-guestmetrics-overview.json @@ -251,7 +251,7 @@ "type": "list" }, { - "fieldName": "azure.resource.name", + "fieldName": "cloud.instance.name", "id": "1570774891724", "indexPatternRefName": "control_2_index_pattern", "label": "VM Name", diff --git a/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-vm-overview.json b/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-vm-overview.json index e73bc6a1e5e..08b5b044633 100644 --- a/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-vm-overview.json +++ b/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-vm-overview.json @@ -266,7 +266,7 @@ "type": "list" }, { - "fieldName": "azure.resource.name", + "fieldName": "cloud.instance.name", "id": "1570774891724", "indexPatternRefName": "control_2_index_pattern", "label": "VM Name", @@ -372,7 +372,7 @@ "split_color_mode": "rainbow", "split_mode": "terms", "stacked": "none", - "terms_field": "azure.resource.name", + "terms_field": "cloud.instance.name", "value_template": "{{value}}%" } ], @@ -447,7 +447,7 @@ "separate_axis": 0, "split_mode": "terms", "stacked": "none", - "terms_field": "azure.resource.name", + "terms_field": "cloud.instance.name", "value_template": "{{value}}" } ], @@ -522,7 +522,7 @@ "separate_axis": 0, "split_mode": "terms", "stacked": "none", - "terms_field": "azure.resource.name", + "terms_field": "cloud.instance.name", "type": "timeseries", "value_template": "{{value}}" } @@ -598,7 +598,7 @@ "separate_axis": 0, "split_mode": "terms", "stacked": "none", - "terms_field": "azure.resource.name", + "terms_field": "cloud.instance.name", "value_template": "{{value}}" } ], @@ -673,7 +673,7 @@ "separate_axis": 0, "split_mode": "terms", "stacked": "none", - "terms_field": "azure.resource.name", + "terms_field": "cloud.instance.name", "type": "timeseries", "value_template": "{{value}}" } @@ -749,7 +749,7 @@ "separate_axis": 0, "split_mode": "terms", "stacked": "none", - "terms_field": "azure.resource.name", + "terms_field": "cloud.instance.name", "type": "timeseries", "value_template": "{{value}}" } @@ -825,7 +825,7 @@ "separate_axis": 0, "split_mode": "terms", "stacked": "none", - "terms_field": "azure.resource.name", + "terms_field": "cloud.instance.name", "type": "timeseries", "value_template": "{{value}}" } @@ -901,7 +901,7 @@ "separate_axis": 0, "split_mode": "terms", "stacked": "none", - "terms_field": "azure.resource.name", + "terms_field": "cloud.instance.name", "type": "timeseries", "value_template": "{{value}}" } diff --git a/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-vmss-overview.json b/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-vmss-overview.json index fe615da131f..7e8ed312b79 100644 --- a/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-vmss-overview.json +++ b/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-vmss-overview.json @@ -374,7 +374,7 @@ "split_color_mode": "rainbow", "split_mode": "terms", "stacked": "none", - "terms_field": "azure.resource.name", + "terms_field": "cloud.instance.name", "value_template": "{{value}}" } ], @@ -449,7 +449,7 @@ "separate_axis": 0, "split_mode": "terms", "stacked": "none", - "terms_field": "azure.resource.name", + "terms_field": "cloud.instance.name", "value_template": "{{value}}" } ], @@ -524,7 +524,7 @@ "separate_axis": 0, "split_mode": "terms", "stacked": "none", - "terms_field": "azure.resource.name", + "terms_field": "cloud.instance.name", "type": "timeseries", "value_template": "{{value}}" } @@ -600,7 +600,7 @@ "separate_axis": 0, "split_mode": "terms", "stacked": "none", - "terms_field": "azure.resource.name", + "terms_field": "cloud.instance.name", "type": "timeseries", "value_template": "{{value}}" } @@ -676,7 +676,7 @@ "separate_axis": 0, "split_mode": "terms", "stacked": "none", - "terms_field": "azure.resource.name", + "terms_field": "cloud.instance.name", "type": "timeseries", "value_template": "{{value}}" } @@ -752,7 +752,7 @@ "separate_axis": 0, "split_mode": "terms", "stacked": "none", - "terms_field": "azure.resource.name", + "terms_field": "cloud.instance.name", "type": "timeseries", "value_template": "{{value}}" } @@ -828,7 +828,7 @@ "separate_axis": 0, "split_mode": "terms", "stacked": "none", - "terms_field": "azure.resource.name", + "terms_field": "cloud.instance.name", "type": "timeseries", "value_template": "{{value}}" } @@ -904,7 +904,7 @@ "separate_axis": 0, "split_mode": "terms", "stacked": "none", - "terms_field": "azure.resource.name", + "terms_field": "cloud.instance.name", "type": "timeseries", "value_template": "{{value}}" } diff --git a/x-pack/metricbeat/module/azure/azure.go b/x-pack/metricbeat/module/azure/azure.go index 675e5c35d3c..b321ac5f8ae 100644 --- a/x-pack/metricbeat/module/azure/azure.go +++ b/x-pack/metricbeat/module/azure/azure.go @@ -17,10 +17,10 @@ import ( // Config options type Config struct { - ClientID string `config:"client_id" validate:"required"` + ClientId string `config:"client_id" validate:"required"` ClientSecret string `config:"client_secret" validate:"required"` - TenantID string `config:"tenant_id" validate:"required"` - SubscriptionID string `config:"subscription_id" validate:"required"` + TenantId string `config:"tenant_id" validate:"required"` + SubscriptionId string `config:"subscription_id" validate:"required"` Period time.Duration `config:"period" validate:"nonzero,required"` Resources []ResourceConfig `config:"resources"` RefreshListInterval time.Duration `config:"refresh_list_interval"` @@ -29,7 +29,7 @@ type Config struct { // ResourceConfig contains resource and metric list specific configuration. type ResourceConfig struct { - ID []string `config:"resource_id"` + Id []string `config:"resource_id"` Group []string `config:"resource_group"` Metrics []MetricConfig `config:"metrics"` Type string `config:"resource_type"` @@ -106,7 +106,7 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { // check for lightweight resources if no groups or ids have been entered, if not a new resource is created to check the entire subscription var resources []ResourceConfig for _, resource := range config.Resources { - if hasConfigOptions(resource.Group) || hasConfigOptions(resource.ID) { + if hasConfigOptions(resource.Group) || hasConfigOptions(resource.Id) { resources = append(resources, resource) } } diff --git a/x-pack/metricbeat/module/azure/client.go b/x-pack/metricbeat/module/azure/client.go index 1d0064785ee..57da9c3b23d 100644 --- a/x-pack/metricbeat/module/azure/client.go +++ b/x-pack/metricbeat/module/azure/client.go @@ -31,7 +31,7 @@ type mapResourceMetrics func(client *Client, resources []resources.GenericResour // NewClient instantiates the an Azure monitoring client func NewClient(config Config) (*Client, error) { - azureMonitorService, err := NewService(config.ClientID, config.ClientSecret, config.TenantID, config.SubscriptionID) + azureMonitorService, err := NewService(config.ClientId, config.ClientSecret, config.TenantId, config.SubscriptionId) if err != nil { return nil, err } @@ -57,14 +57,14 @@ func (client *Client) InitResources(fn mapResourceMetrics, report mb.ReporterV2) var metrics []Metric for _, resource := range client.Config.Resources { // retrieve azure resources information - resourceList, err := client.AzureMonitorService.GetResourceDefinitions(resource.ID, resource.Group, resource.Type, resource.Query) + resourceList, err := client.AzureMonitorService.GetResourceDefinitions(resource.Id, resource.Group, resource.Type, resource.Query) if err != nil { err = errors.Wrap(err, "failed to retrieve resources") return err } if len(resourceList.Values()) == 0 { err = errors.Errorf("failed to retrieve resources: No resources returned using the configuration options resource ID %s, resource group %s, resource type %s, resource query %s", - resource.ID, resource.Group, resource.Type, resource.Query) + resource.Id, resource.Group, resource.Type, resource.Query) client.Log.Error(err) continue } @@ -107,10 +107,10 @@ func (client *Client) GetMetricValues(metrics []Metric, report mb.ReporterV2) [] } filter = strings.Join(filterList, " AND ") } - resp, timegrain, err := client.AzureMonitorService.GetMetricValues(metric.Resource.SubID, metric.Namespace, metric.TimeGrain, timespan, metric.Names, + resp, timegrain, err := client.AzureMonitorService.GetMetricValues(metric.Resource.SubId, metric.Namespace, metric.TimeGrain, timespan, metric.Names, metric.Aggregations, filter) if err != nil { - err = errors.Wrapf(err, "error while listing metric values by resource ID %s and namespace %s", metric.Resource.SubID, metric.Namespace) + err = errors.Wrapf(err, "error while listing metric values by resource ID %s and namespace %s", metric.Resource.SubId, metric.Namespace) client.Log.Error(err) report.Error(err) } else { @@ -130,17 +130,19 @@ func (client *Client) GetMetricValues(metrics []Metric, report mb.ReporterV2) [] } // CreateMetric function will create a client metric based on the resource and metrics configured -func (client *Client) CreateMetric(selectedResourceID string, resource resources.GenericResource, namespace string, metrics []string, aggregations string, dimensions []Dimension, timegrain string) Metric { +func (client *Client) CreateMetric(selectedResourceID string, resource resources.GenericResource, resourceSize string, namespace string, metrics []string, aggregations string, dimensions []Dimension, timegrain string) Metric { met := Metric{ Resource: Resource{ - SubID: selectedResourceID, - ID: *resource.ID, + SubId: selectedResourceID, + Id: *resource.ID, Name: *resource.Name, Location: *resource.Location, Type: *resource.Type, - Group: getResourceGroupFromID(*resource.ID), + Group: getResourceGroupFromId(*resource.ID), Tags: mapTags(resource.Tags), - Subscription: client.Config.SubscriptionID}, + Subscription: client.Config.SubscriptionId, + Size: resourceSize, + }, Namespace: namespace, Names: metrics, Dimensions: dimensions, @@ -156,7 +158,7 @@ func (client *Client) CreateMetric(selectedResourceID string, resource resources } // MapMetricByPrimaryAggregation will map the primary aggregation of the metric definition to the client metric -func MapMetricByPrimaryAggregation(client *Client, metrics []insights.MetricDefinition, resource resources.GenericResource, selectedResourceID string, namespace string, dim []Dimension, timegrain string) []Metric { +func (client *Client) MapMetricByPrimaryAggregation(metrics []insights.MetricDefinition, resource resources.GenericResource, selectedResourceID string, resourceSize string, namespace string, dim []Dimension, timegrain string) []Metric { var clientMetrics []Metric metricGroups := make(map[string][]insights.MetricDefinition) @@ -171,7 +173,7 @@ func MapMetricByPrimaryAggregation(client *Client, metrics []insights.MetricDefi if selectedResourceID == "" { selectedResourceID = *resource.ID } - clientMetrics = append(clientMetrics, client.CreateMetric(selectedResourceID, resource, namespace, metricNames, key, dim, timegrain)) + clientMetrics = append(clientMetrics, client.CreateMetric(selectedResourceID, resource, resourceSize, namespace, metricNames, key, dim, timegrain)) } return clientMetrics } diff --git a/x-pack/metricbeat/module/azure/client_test.go b/x-pack/metricbeat/module/azure/client_test.go index 559b4efa7eb..970a3d6e794 100644 --- a/x-pack/metricbeat/module/azure/client_test.go +++ b/x-pack/metricbeat/module/azure/client_test.go @@ -17,7 +17,7 @@ import ( var ( resourceIDConfig = Config{ Resources: []ResourceConfig{ - {ID: []string{"123"}, + {Id: []string{"123"}, Metrics: []MetricConfig{ { Name: []string{"hello", "test"}, diff --git a/x-pack/metricbeat/module/azure/client_utils.go b/x-pack/metricbeat/module/azure/client_utils.go index ce55a7a1459..ebf2f7fc812 100644 --- a/x-pack/metricbeat/module/azure/client_utils.go +++ b/x-pack/metricbeat/module/azure/client_utils.go @@ -87,15 +87,15 @@ func metricIsEmpty(metric insights.MetricValue) bool { // matchMetrics will compare current metrics func matchMetrics(prevMet Metric, met Metric) bool { - if prevMet.Namespace == met.Namespace && reflect.DeepEqual(prevMet.Names, met.Names) && prevMet.Resource.ID == met.Resource.ID && + if prevMet.Namespace == met.Namespace && reflect.DeepEqual(prevMet.Names, met.Names) && prevMet.Resource.Id == met.Resource.Id && prevMet.Aggregations == met.Aggregations && prevMet.TimeGrain == met.TimeGrain { return true } return false } -// getResourceGroupFormID maps resource group from resource ID -func getResourceGroupFromID(path string) string { +// getResourceGroupFromId maps resource group from resource ID +func getResourceGroupFromId(path string) string { params := strings.Split(path, "/") for i, param := range params { if param == "resourceGroups" { @@ -105,8 +105,8 @@ func getResourceGroupFromID(path string) string { return "" } -// getResourceNameFormID maps resource group from resource ID -func getResourceTypeFromID(path string) string { +// getResourceTypeFromId maps resource group from resource ID +func getResourceTypeFromId(path string) string { params := strings.Split(path, "/") for i, param := range params { if param == "providers" { @@ -116,8 +116,8 @@ func getResourceTypeFromID(path string) string { return "" } -// getResourceNameFormID maps resource group from resource ID -func getResourceNameFromID(path string) string { +// getResourceNameFormId maps resource group from resource ID +func getResourceNameFromId(path string) string { params := strings.Split(path, "/") if strings.HasSuffix(path, "/") { return params[len(params)-2] @@ -181,10 +181,10 @@ func convertTimegrainToDuration(timegrain string) time.Duration { func groupMetricsByResource(metrics []Metric) map[string][]Metric { grouped := make(map[string][]Metric) for _, metric := range metrics { - if _, ok := grouped[metric.Resource.ID]; !ok { - grouped[metric.Resource.ID] = make([]Metric, 0) + if _, ok := grouped[metric.Resource.Id]; !ok { + grouped[metric.Resource.Id] = make([]Metric, 0) } - grouped[metric.Resource.ID] = append(grouped[metric.Resource.ID], metric) + grouped[metric.Resource.Id] = append(grouped[metric.Resource.Id], metric) } return grouped } diff --git a/x-pack/metricbeat/module/azure/client_utils_test.go b/x-pack/metricbeat/module/azure/client_utils_test.go index a21ebf026e4..323175c468e 100644 --- a/x-pack/metricbeat/module/azure/client_utils_test.go +++ b/x-pack/metricbeat/module/azure/client_utils_test.go @@ -60,7 +60,7 @@ func TestMetricExists(t *testing.T) { func TestMatchMetrics(t *testing.T) { prev := Metric{ - Resource: Resource{Name: "vm", Group: "group", ID: "id"}, + Resource: Resource{Name: "vm", Group: "group", Id: "id"}, Namespace: "namespace", Names: []string{"TotalRequests,Capacity"}, Aggregations: "Average,Total", @@ -69,7 +69,7 @@ func TestMatchMetrics(t *testing.T) { TimeGrain: "1PM", } current := Metric{ - Resource: Resource{Name: "vm", Group: "group", ID: "id"}, + Resource: Resource{Name: "vm", Group: "group", Id: "id"}, Namespace: "namespace", Names: []string{"TotalRequests,Capacity"}, Aggregations: "Average,Total", @@ -79,7 +79,7 @@ func TestMatchMetrics(t *testing.T) { } result := matchMetrics(prev, current) assert.True(t, result) - current.Resource.ID = "id1" + current.Resource.Id = "id1" result = matchMetrics(prev, current) assert.False(t, result) } @@ -107,19 +107,19 @@ func TestMetricIsEmpty(t *testing.T) { func TestGetResourceGroupFromID(t *testing.T) { path := "subscriptions/qw3e45r6t-23ws-1234-6587-1234ed4532/resourceGroups/obs-infrastructure/providers/Microsoft.Compute/virtualMachines/obstestmemleak" - group := getResourceGroupFromID(path) + group := getResourceGroupFromId(path) assert.Equal(t, group, "obs-infrastructure") } func TestGetResourceTypeFromID(t *testing.T) { path := "subscriptions/qw3e45r6t-23ws-1234-6587-1234ed4532/resourceGroups/obs-infrastructure/providers/Microsoft.Compute/virtualMachines/obstestmemleak" - rType := getResourceTypeFromID(path) + rType := getResourceTypeFromId(path) assert.Equal(t, rType, "Microsoft.Compute/virtualMachines") } func TestGetResourceNameFromID(t *testing.T) { path := "subscriptions/qw3e45r6t-23ws-1234-6587-1234ed4532/resourceGroups/obs-infrastructure/providers/Microsoft.Compute/virtualMachines/obstestmemleak" - name := getResourceNameFromID(path) + name := getResourceNameFromId(path) assert.Equal(t, name, "obstestmemleak") } diff --git a/x-pack/metricbeat/module/azure/compute_vm/_meta/data.json b/x-pack/metricbeat/module/azure/compute_vm/_meta/data.json index f0692fae3b6..5858cf0407e 100644 --- a/x-pack/metricbeat/module/azure/compute_vm/_meta/data.json +++ b/x-pack/metricbeat/module/azure/compute_vm/_meta/data.json @@ -1,154 +1,569 @@ { - "@timestamp":"2019-09-20T11:02:00.000Z", - "azure":{ - "timegrain" : "PT5M", - "resource":{ - "name":"obslinux", - "type":"Microsoft.Compute/virtualMachines", - "group":"resource-group" - }, - "subscription_id":"123456-qwer-1234-5678-12345678", - "namespace":"Microsoft.Compute/virtualMachines", - "compute_vm":{ - "inbound_flows_maximum_creation_rate":{ - "avg":14.6 + "@timestamp": "2017-10-12T08:05:34.853Z", + "azure": { + "compute_vm": { + "asp_net_application_restarts": { + "avg": 0 + }, + "asp_net_applications_anonymous_requests": { + "avg": 0 + }, + "asp_net_applications_anonymous_requests_per_sec": { + "avg": 0 + }, + "asp_net_applications_cache_api_entries": { + "avg": 0 + }, + "asp_net_applications_cache_api_hit_ratio": { + "avg": 0 + }, + "asp_net_applications_cache_api_hits": { + "avg": 0 + }, + "asp_net_applications_cache_api_misses": { + "avg": 0 + }, + "asp_net_applications_cache_api_turnover_rate": { + "avg": 0 + }, + "asp_net_applications_cache_total_entries": { + "avg": 0 + }, + "asp_net_applications_cache_total_hit_ratio": { + "avg": 0 + }, + "asp_net_applications_cache_total_hits": { + "avg": 0 + }, + "asp_net_applications_cache_total_misses": { + "avg": 0 + }, + "asp_net_applications_cache_total_turnover_rate": { + "avg": 0 + }, + "asp_net_applications_compilations_total": { + "avg": 0 + }, + "asp_net_applications_debugging_requests": { + "avg": 0 + }, + "asp_net_applications_errors_during_compilation": { + "avg": 0 + }, + "asp_net_applications_errors_during_execution": { + "avg": 0 + }, + "asp_net_applications_errors_during_preprocessing": { + "avg": 0 + }, + "asp_net_applications_errors_total": { + "avg": 0 + }, + "asp_net_applications_errors_total_per_sec": { + "avg": 0 + }, + "asp_net_applications_errors_unhandled_during_execution": { + "avg": 0 + }, + "asp_net_applications_errors_unhandled_during_execution_per_sec": { + "avg": 0 + }, + "asp_net_applications_output_cache_entries": { + "avg": 0 + }, + "asp_net_applications_output_cache_hit_ratio": { + "avg": 0 + }, + "asp_net_applications_output_cache_hits": { + "avg": 0 + }, + "asp_net_applications_output_cache_misses": { + "avg": 0 + }, + "asp_net_applications_output_cache_turnover_rate": { + "avg": 0 + }, + "asp_net_applications_pipeline_instance_count": { + "avg": 0 + }, + "asp_net_applications_request_bytes_in_total": { + "avg": 0 + }, + "asp_net_applications_request_bytes_out_total": { + "avg": 0 + }, + "asp_net_applications_requests_executing": { + "avg": 0 + }, + "asp_net_applications_requests_failed": { + "avg": 0 + }, + "asp_net_applications_requests_in_application_queue": { + "avg": 0 + }, + "asp_net_applications_requests_not_authorized": { + "avg": 0 + }, + "asp_net_applications_requests_not_found": { + "avg": 0 + }, + "asp_net_applications_requests_per_sec": { + "avg": 0 + }, + "asp_net_applications_requests_succeeded": { + "avg": 0 + }, + "asp_net_applications_requests_timed_out": { + "avg": 0 + }, + "asp_net_applications_requests_total": { + "avg": 0 + }, + "asp_net_applications_running": { + "avg": 0 + }, + "asp_net_applications_sessions_abandoned": { + "avg": 0 + }, + "asp_net_applications_sessions_active": { + "avg": 0 + }, + "asp_net_applications_sessions_timed_out": { + "avg": 0 + }, + "asp_net_applications_sessions_total": { + "avg": 0 + }, + "asp_net_applications_transactions_aborted": { + "avg": 0 + }, + "asp_net_applications_transactions_committed": { + "avg": 0 + }, + "asp_net_applications_transactions_pending": { + "avg": 0 + }, + "asp_net_applications_transactions_per_sec": { + "avg": 0 + }, + "asp_net_applications_transactions_total": { + "avg": 0 + }, + "asp_net_apps_v4_0_30319_anonymous_requests": { + "avg": 0 + }, + "asp_net_apps_v4_0_30319_anonymous_requests_per_sec": { + "avg": 0 + }, + "asp_net_apps_v4_0_30319_cache_api_entries": { + "avg": 0 + }, + "asp_net_apps_v4_0_30319_cache_api_hit_ratio": { + "avg": 0 + }, + "asp_net_apps_v4_0_30319_cache_api_hits": { + "avg": 0 + }, + "asp_net_apps_v4_0_30319_cache_api_misses": { + "avg": 0 + }, + "asp_net_apps_v4_0_30319_cache_api_turnover_rate": { + "avg": 0 + }, + "asp_net_apps_v4_0_30319_cache_total_entries": { + "avg": 0 + }, + "asp_net_apps_v4_0_30319_cache_total_hit_ratio": { + "avg": 0 + }, + "asp_net_apps_v4_0_30319_cache_total_hits": { + "avg": 0 + }, + "asp_net_apps_v4_0_30319_cache_total_misses": { + "avg": 0 + }, + "asp_net_apps_v4_0_30319_cache_total_turnover_rate": { + "avg": 0 + }, + "asp_net_apps_v4_0_30319_compilations_total": { + "avg": 0 + }, + "asp_net_apps_v4_0_30319_debugging_requests": { + "avg": 0 + }, + "asp_net_apps_v4_0_30319_errors_during_compilation": { + "avg": 0 + }, + "asp_net_apps_v4_0_30319_errors_during_execution": { + "avg": 0 + }, + "asp_net_apps_v4_0_30319_errors_during_preprocessing": { + "avg": 0 + }, + "asp_net_apps_v4_0_30319_errors_total": { + "avg": 0 + }, + "asp_net_apps_v4_0_30319_errors_total_per_sec": { + "avg": 0 + }, + "asp_net_apps_v4_0_30319_errors_unhandled_during_execution": { + "avg": 0 + }, + "asp_net_apps_v4_0_30319_errors_unhandled_during_execution_per_sec": { + "avg": 0 + }, + "asp_net_apps_v4_0_30319_output_cache_entries": { + "avg": 0 + }, + "asp_net_apps_v4_0_30319_output_cache_hit_ratio": { + "avg": 0 + }, + "asp_net_apps_v4_0_30319_output_cache_hits": { + "avg": 0 + }, + "asp_net_apps_v4_0_30319_output_cache_misses": { + "avg": 0 + }, + "asp_net_apps_v4_0_30319_output_cache_turnover_rate": { + "avg": 0 }, - "os_per_disk_write_bytes_per_sec":{ - "avg":15261.86 + "asp_net_apps_v4_0_30319_pipeline_instance_count": { + "avg": 0 }, - "outbound_flows_maximum_creation_rate":{ - "avg":18.6 + "asp_net_apps_v4_0_30319_request_bytes_in_total": { + "avg": 0 }, - "os_per_disk_read_operations_per_sec":{ - "avg":0 + "asp_net_apps_v4_0_30319_request_bytes_out_total": { + "avg": 0 }, - "data_disk_queue_depth":{ - "avg":0 + "asp_net_apps_v4_0_30319_requests_executing": { + "avg": 0 }, - "per_disk_read_operations_per_sec":{ - "avg":0 + "asp_net_apps_v4_0_30319_requests_failed": { + "avg": 0 }, - "inbound_flows":{ - "avg":60.4 + "asp_net_apps_v4_0_30319_requests_in_application_queue": { + "avg": 0 }, - "os_disk_write_operations_per_sec":{ - "avg":1.0175 + "asp_net_apps_v4_0_30319_requests_not_authorized": { + "avg": 0 }, - "per_disk_read_bytes_per_sec":{ - "avg":0 + "asp_net_apps_v4_0_30319_requests_not_found": { + "avg": 0 }, - "disk_write_operations_per_sec":{ - "avg":1.03 + "asp_net_apps_v4_0_30319_requests_per_sec": { + "avg": 0 }, - "outbound_flows":{ - "avg":60.4 + "asp_net_apps_v4_0_30319_requests_succeeded": { + "avg": 0 }, - "network_in":{ - "total":3624009.0 + "asp_net_apps_v4_0_30319_requests_timed_out": { + "avg": 0 }, - "os_per_disk_qd":{ - "avg":0 + "asp_net_apps_v4_0_30319_requests_total": { + "avg": 0 }, - "data_disk_read_bytes_per_sec":{ - "avg":0 + "asp_net_apps_v4_0_30319_sessions_abandoned": { + "avg": 0 }, - "disk_read_operations_per_sec":{ - "avg":0 + "asp_net_apps_v4_0_30319_sessions_active": { + "avg": 0 }, - "os_disk_read_operations_per_sec":{ - "avg":0 + "asp_net_apps_v4_0_30319_sessions_timed_out": { + "avg": 0 }, - "per_disk_qd":{ - "avg":0 + "asp_net_apps_v4_0_30319_sessions_total": { + "avg": 0 }, - "os_disk_write_bytes_per_sec":{ - "avg":15261.86 + "asp_net_apps_v4_0_30319_transactions_aborted": { + "avg": 0 }, - "disk_write_bytes":{ - "total":2806782.57 + "asp_net_apps_v4_0_30319_transactions_committed": { + "avg": 0 }, - "per_disk_write_operations_per_sec":{ - "avg":0 + "asp_net_apps_v4_0_30319_transactions_pending": { + "avg": 0 }, - "network_out":{ - "total":704862 + "asp_net_apps_v4_0_30319_transactions_per_sec": { + "avg": 0 }, - "os_disk_read_bytes_per_sec":{ - "avg":0 + "asp_net_apps_v4_0_30319_transactions_total": { + "avg": 0 }, - "os_per_disk_read_bytes_per_sec":{ - "avg":0 + "asp_net_request_execution_time": { + "avg": 0 }, - "os_per_disk_write_operations_per_sec":{ - "avg":1.0175 + "asp_net_request_wait_time": { + "avg": 0 }, - "percentage_cpu":{ - "avg":7.683333333333334 + "asp_net_requests_current": { + "avg": 0 }, - "data_disk_write_operations_per_sec":{ - "avg":0 + "asp_net_requests_disconnected": { + "avg": 0 }, - "network_in_total":{ - "total":6787492.0 + "asp_net_requests_queued": { + "avg": 0 }, - "disk_read_bytes":{ - "total":0 + "asp_net_requests_rejected": { + "avg": 0 }, - "data_disk_write_bytes_per_sec":{ - "avg":0 + "asp_net_v4_0_30319_application_restarts": { + "avg": 0 }, - "per_disk_write_bytes_per_sec":{ - "avg":0 + "asp_net_v4_0_30319_applications_running": { + "avg": 0 }, - "data_disk_read_operations_per_sec":{ - "avg":0 + "asp_net_v4_0_30319_request_execution_time": { + "avg": 0 }, - "os_disk_queue_depth":{ - "avg":0 + "asp_net_v4_0_30319_request_wait_time": { + "avg": 0 }, - "network_out_total":{ - "total":1889767.0 + "asp_net_v4_0_30319_requests_current": { + "avg": 0 + }, + "asp_net_v4_0_30319_requests_disconnected": { + "avg": 0 + }, + "asp_net_v4_0_30319_requests_queued": { + "avg": 0 + }, + "asp_net_v4_0_30319_requests_rejected": { + "avg": 0 + }, + "asp_net_v4_0_30319_worker_process_restarts": { + "avg": 0 + }, + "asp_net_v4_0_30319_worker_processes_running": { + "avg": 0 + }, + "asp_net_worker_process_restarts": { + "avg": 0 + }, + "asp_net_worker_processes_running": { + "avg": 0 + }, + "logical_disk_avg_disk_queue_length": { + "avg": 0 + }, + "logical_disk_avg_disk_read_queue_length": { + "avg": 0 + }, + "logical_disk_avg_disk_sec_per_read": { + "avg": 0 + }, + "logical_disk_avg_disk_sec_per_transfer": { + "avg": 0 + }, + "logical_disk_avg_disk_sec_per_write": { + "avg": 0 + }, + "logical_disk_avg_disk_write_queue_length": { + "avg": 0 + }, + "logical_disk_disk_bytes_per_sec": { + "avg": 449380 + }, + "logical_disk_disk_read_bytes_per_sec": { + "avg": 4783 + }, + "logical_disk_disk_reads_per_sec": { + "avg": 0 + }, + "logical_disk_disk_transfers_per_sec": { + "avg": 6.6 + }, + "logical_disk_disk_write_bytes_per_sec": { + "avg": 444596.8 + }, + "logical_disk_disk_writes_per_sec": { + "avg": 6.4 + }, + "logical_disk_free_megabytes": { + "avg": 10433 + }, + "logical_disk_pct_disk_read_time": { + "avg": 0 + }, + "logical_disk_pct_disk_time": { + "avg": 1.6 + }, + "logical_disk_pct_disk_write_time": { + "avg": 1.6 + }, + "logical_disk_pct_free_space": { + "avg": 22 + }, + "logical_disk_pct_idle_time": { + "avg": 98.6 + }, + "memory_available_bytes": { + "avg": 2951829094.4 + }, + "memory_cache_bytes": { + "avg": 60126003.2 + }, + "memory_committed_bytes": { + "avg": 8922399539.2 + }, + "memory_page_faults_per_sec": { + "avg": 1484.2 + }, + "memory_pages_per_sec": { + "avg": 6 + }, + "memory_pct_committed_bytes_in_use": { + "avg": 82.4 + }, + "memory_pool_nonpaged_bytes": { + "avg": 204028313.6 + }, + "memory_pool_paged_bytes": { + "avg": 359613235.2 + }, + "network_interface_bytes_received_per_sec": { + "avg": 7922.333333333333 + }, + "network_interface_bytes_sent_per_sec": { + "avg": 60510.666666666664 + }, + "network_interface_bytes_total_per_sec": { + "avg": 68433.26666666666 + }, + "network_interface_packets_outbound_errors": { + "avg": 0 + }, + "network_interface_packets_per_sec": { + "avg": 43.4 + }, + "network_interface_packets_received_errors": { + "avg": 0 + }, + "network_interface_packets_received_per_sec": { + "avg": 27.533333333333335 + }, + "network_interface_packets_sent_per_sec": { + "avg": 15.733333333333333 + }, + "process_handle_count": { + "avg": 119553 + }, + "process_thread_count": { + "avg": 1705.8 + }, + "process_working_set": { + "avg": 5566961254.4 + }, + "process_working_set_private": { + "avg": 4168472985.6 + }, + "processor_information_pct_privileged_time": { + "avg": 2.6 + }, + "processor_information_pct_processor_time": { + "avg": 6.4 + }, + "processor_information_pct_user_time": { + "avg": 3.4 + }, + "processor_information_processor_frequency": { + "avg": 2295 + }, + "sqlserver_buffer_manager_buffer_cache_hit_ratio": { + "avg": 100 + }, + "sqlserver_buffer_manager_checkpoint_pages_per_sec": { + "avg": 0 + }, + "sqlserver_buffer_manager_database_pages": { + "avg": 1483 + }, + "sqlserver_buffer_manager_lazy_writes_per_sec": { + "avg": 0 + }, + "sqlserver_buffer_manager_page_reads_per_sec": { + "avg": 0 + }, + "sqlserver_buffer_manager_page_writes_per_sec": { + "avg": 0 + }, + "sqlserver_general_statistics_user_connections": { + "avg": 1 + }, + "sqlserver_memory_manager_memory_grants_pending": { + "avg": 0 + }, + "sqlserver_memory_manager_total_server_memory": { + "avg": 317976 + }, + "sqlserver_sql_statistics_batch_requests_per_sec": { + "avg": 0 + }, + "sqlserver_sql_statistics_sql_compilations_per_sec": { + "avg": 0 + }, + "sqlserver_sql_statistics_sql_re-compilations_per_sec": { + "avg": 0 + }, + "system_context_switches_per_sec": { + "avg": 7782.6 + }, + "system_processes": { + "avg": 114 + }, + "system_processor_queue_length": { + "avg": 0.2 + }, + "system_system_up_time": { + "avg": 12530764 + }, + "web_service_bytes_total_per_sec": { + "avg": 0 + }, + "web_service_current_connections": { + "avg": 0 + }, + "web_service_isapi_extension_requests_per_sec": { + "avg": 0 + }, + "web_service_total_method_requests_per_sec": { + "avg": 0 } - } - }, - "service":{ - "type":"azure" - }, - "cloud":{ - "provider":"azure", - "region":"westeurope" - }, - "ecs":{ - "version":"1.1.0" + }, + "namespace": "Azure.VM.Windows.GuestMetrics", + "resource": { + "group": "obs-infrastructure", + "tags": { + "vmtest": "value1, value 2", + "vmtest1": "value3" + }, + "type": "Microsoft.Compute/virtualMachines" + }, + "subscription_id": "fd675b6f-b5e5-426e-ac45-d1f876d0ffa6", + "timegrain": "PT5M" }, - "host":{ - "architecture":"x86_64", - "os":{ - "build":"17134.1006", - "platform":"windows", - "version":"10.0", - "family":"windows", - "name":"Windows 10 Pro", - "kernel":"10.0.17134.1006 (WinBuild.160101.0800)" + "cloud": { + "instance": { + "id": "/subscriptions/fd675b6f-b5e5-426e-ac45-d1f876d0ffa6/resourceGroups/obs-infrastructure/providers/Microsoft.Compute/virtualMachines/obstestmemleak", + "name": "obstestmemleak" + }, + "machine": { + "type": "Standard_B2ms" }, - "id":"123456-qwer-1234-5678-12345678", - "name":"DESKTOP-RFOOE09", - "hostname":"DESKTOP-RFOOE09" + "provider": "azure", + "region": "westeurope" }, - "agent":{ - "hostname":"DESKTOP-RFOOE09", - "id":"123456-qwer-1234-5678-12345678", - "version":"8.0.0", - "type":"metricbeat", - "ephemeral_id":"123456-qwer-1234-5678-12345678" + "event": { + "dataset": "azure.compute_vm", + "duration": 115000, + "module": "azure" }, - "event":{ - "duration":4611928900, - "dataset":"azure.compute_vm", - "module":"azure" + "metricset": { + "name": "compute_vm", + "period": 10000 }, - "metricset":{ - "name":"compute_vm", - "period":300000 + "service": { + "type": "azure" } } diff --git a/x-pack/metricbeat/module/azure/compute_vm/client_helper.go b/x-pack/metricbeat/module/azure/compute_vm/client_helper.go index 87690ef0bd5..dee4755d02b 100644 --- a/x-pack/metricbeat/module/azure/compute_vm/client_helper.go +++ b/x-pack/metricbeat/module/azure/compute_vm/client_helper.go @@ -19,6 +19,8 @@ func mapMetrics(client *azure.Client, resources []resources.GenericResource, res return nil, nil } for _, resource := range resources { + // return resource size + resourceSize := mapResourceSize(resource, client) // return all namespaces supported for this resource namespaces, err := client.AzureMonitorService.GetMetricNamespaces(*resource.ID) if err != nil { @@ -38,8 +40,32 @@ func mapMetrics(client *azure.Client, resources []resources.GenericResource, res filteredMetricDefinitions = append(filteredMetricDefinitions, metricDefinition) } // map azure metric definitions to client metrics - metrics = append(metrics, azure.MapMetricByPrimaryAggregation(client, filteredMetricDefinitions, resource, "", *namespace.Properties.MetricNamespaceName, nil, azure.DefaultTimeGrain)...) + metrics = append(metrics, client.MapMetricByPrimaryAggregation(filteredMetricDefinitions, resource, "", resourceSize, *namespace.Properties.MetricNamespaceName, nil, azure.DefaultTimeGrain)...) } } return metrics, nil } + +// mapResourceSize func will try to map if existing the resource size +func mapResourceSize(resource resources.GenericResource, client *azure.Client) string { + if resource.Sku != nil && resource.Sku.Name != nil { + return *resource.Sku.Name + } + if resource.Sku == nil && resource.Properties == nil { + expandedResource, err := client.AzureMonitorService.GetResourceDefinitionById(*resource.ID) + if err != nil { + client.Log.Error(err, "could not retrieve the resource details by resource ID %s", *resource.ID) + return "" + } + if expandedResource.Properties != nil { + if properties, ok := expandedResource.Properties.(map[string]interface{}); ok { + if hardware, ok := properties["hardwareProfile"]; ok { + if vmSize, ok := hardware.(map[string]interface{})["vmSize"]; ok { + return vmSize.(string) + } + } + } + } + } + return "" +} diff --git a/x-pack/metricbeat/module/azure/compute_vm/client_helper_test.go b/x-pack/metricbeat/module/azure/compute_vm/client_helper_test.go index e5c82279498..39ffbbc5091 100644 --- a/x-pack/metricbeat/module/azure/compute_vm/client_helper_test.go +++ b/x-pack/metricbeat/module/azure/compute_vm/client_helper_test.go @@ -81,8 +81,11 @@ func TestMapMetric(t *testing.T) { metricConfig := azure.MetricConfig{Name: []string{"*"}} var resourceConfig = azure.ResourceConfig{Metrics: []azure.MetricConfig{metricConfig}} client := azure.NewMockClient() + res := resource + res.Properties = map[string]interface{}{"hardwareProfile": map[string]interface{}{"vmSize": "A1Basic"}} t.Run("return error when the metric namespaces api call returns an error", func(t *testing.T) { m := &azure.MockService{} + m.On("GetResourceDefinitionById", mock.Anything).Return(res, nil) m.On("GetMetricNamespaces", mock.Anything).Return(insights.MetricNamespaceCollection{}, errors.New("invalid resource ID")) client.AzureMonitorService = m metric, err := mapMetrics(client, []resources.GenericResource{resource}, resourceConfig) @@ -93,6 +96,7 @@ func TestMapMetric(t *testing.T) { }) t.Run("return error when no metric definitions were found", func(t *testing.T) { m := &azure.MockService{} + m.On("GetResourceDefinitionById", mock.Anything).Return(res, nil) m.On("GetMetricNamespaces", mock.Anything).Return(namespace, nil) m.On("GetMetricDefinitions", mock.Anything, mock.Anything).Return(emptyMetricDefinitions, nil) client.AzureMonitorService = m @@ -104,20 +108,22 @@ func TestMapMetric(t *testing.T) { }) t.Run("return mapped metrics correctly", func(t *testing.T) { m := &azure.MockService{} + m.On("GetResourceDefinitionById", mock.Anything).Return(res, nil) m.On("GetMetricNamespaces", mock.Anything).Return(namespace, nil) m.On("GetMetricDefinitions", mock.Anything, mock.Anything).Return(metricDefinitions, nil) client.AzureMonitorService = m metrics, err := mapMetrics(client, []resources.GenericResource{resource}, resourceConfig) assert.Nil(t, err) - assert.Equal(t, metrics[0].Resource.ID, "123") + assert.Equal(t, metrics[0].Resource.Id, "123") assert.Equal(t, metrics[0].Resource.Name, "resourceName") assert.Equal(t, metrics[0].Resource.Type, "resourceType") assert.Equal(t, metrics[0].Resource.Location, "resourceLocation") assert.Equal(t, metrics[0].Namespace, "namespace") - assert.Equal(t, metrics[1].Resource.ID, "123") + assert.Equal(t, metrics[1].Resource.Id, "123") assert.Equal(t, metrics[1].Resource.Name, "resourceName") assert.Equal(t, metrics[1].Resource.Type, "resourceType") assert.Equal(t, metrics[1].Resource.Location, "resourceLocation") + assert.Equal(t, metrics[1].Resource.Size, "A1Basic") assert.Equal(t, metrics[1].Namespace, "namespace") assert.Equal(t, metrics[0].Dimensions, []azure.Dimension(nil)) assert.Equal(t, metrics[1].Dimensions, []azure.Dimension(nil)) diff --git a/x-pack/metricbeat/module/azure/compute_vm/compute_vm_integration_test.go b/x-pack/metricbeat/module/azure/compute_vm/compute_vm_integration_test.go new file mode 100644 index 00000000000..04319405207 --- /dev/null +++ b/x-pack/metricbeat/module/azure/compute_vm/compute_vm_integration_test.go @@ -0,0 +1,39 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build integration + +package compute_vm + +import ( + "testing" + + "github.com/elastic/beats/v7/x-pack/metricbeat/module/azure/test" + + "github.com/stretchr/testify/assert" + + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" +) + +func TestFetchMetricset(t *testing.T) { + config, err := test.GetConfig("compute_vm") + if err != nil { + t.Skip("Skipping TestFetch: " + err.Error()) + } + metricSet := mbtest.NewReportingMetricSetV2Error(t, config) + events, errs := mbtest.ReportingFetchV2Error(metricSet) + if len(errs) > 0 { + t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) + } + assert.NotEmpty(t, events) +} + +func TestData(t *testing.T) { + config, err := test.GetConfig("compute_vm") + if err != nil { + t.Skip("Skipping TestFetch: " + err.Error()) + } + metricSet := mbtest.NewFetcher(t, config) + metricSet.WriteEvents(t, "/") +} diff --git a/x-pack/metricbeat/module/azure/compute_vm_scaleset/_meta/data.json b/x-pack/metricbeat/module/azure/compute_vm_scaleset/_meta/data.json index ac7687bd539..8edb9c5b69a 100644 --- a/x-pack/metricbeat/module/azure/compute_vm_scaleset/_meta/data.json +++ b/x-pack/metricbeat/module/azure/compute_vm_scaleset/_meta/data.json @@ -1,121 +1,58 @@ { - "@timestamp" : "2019-09-20T11:13:00.000Z", - "azure" : { - "timegrain" : "PT5M", - "resource" : { - "name" : "winvmss", - "type" : "Microsoft.Compute/virtualMachineScaleSets", - "group" : "resource-group" - }, - "subscription_id" : "123456-qwer-1234-5678-12345678", - "namespace" : "Microsoft.Compute/virtualMachineScaleSets", - "dimensions" : { - "vmname" : "winvmss_3" - }, - "compute_vm_scaleset" : { - "outbound_flows" : { - "avg" : 10.6 - }, - "disk_write_bytes" : { - "total" : 2.838777151E7 - }, - "disk_read_operations_per_sec" : { - "avg" : 3.53 - }, - "outbound_flows_maximum_creation_rate" : { - "avg" : 0.2 - }, - "data_disk_read_bytes_per_sec" : { - "avg" : 5051.38125 - }, - "os_disk_read_bytes_per_sec" : { - "avg" : 122285.5025 - }, - "data_disk_write_bytes_per_sec" : { - "avg" : 0 - }, - "data_disk_read_operations_per_sec" : { - "avg" : 0.495 - }, - "disk_read_bytes" : { - "total" : 4.081481671E7 - }, - "disk_write_operations_per_sec" : { - "avg" : 3.132 + "@timestamp": "2017-10-12T08:05:34.853Z", + "azure": { + "compute_vm_scaleset": { + "cpu_credits_consumed": { + "avg": 0.019 }, - "network_out_total" : { - "total" : 1509069.0 + "cpu_credits_remaining": { + "avg": 97.99 }, - "inbound_flows" : { - "avg" : 10.6 + "os_per_disk_qd": { + "avg": 0 }, - "os_disk_read_operations_per_sec" : { - "avg" : 1.805 + "os_per_disk_read_bytes_per_sec": { + "avg": 0 }, - "data_disk_queue_depth" : { - "avg" : 0 + "os_per_disk_read_operations_per_sec": { + "avg": 0 }, - "inbound_flows_maximum_creation_rate" : { - "avg" : 0.2 + "os_per_disk_write_bytes_per_sec": { + "avg": 19201.653 }, - "data_disk_write_operations_per_sec" : { - "avg" : 0 - }, - "os_disk_write_bytes_per_sec" : { - "avg" : 95619.85625 - }, - "percentage_cpu" : { - "avg" : 1.592 - }, - "os_disk_write_operations_per_sec" : { - "avg" : 3.045 - }, - "network_in_total" : { - "total" : 794397 - }, - "os_disk_queue_depth" : { - "avg" : 0.005 + "os_per_disk_write_operations_per_sec": { + "avg": 1.0366666666666666 } - } - }, - "service" : { - "type" : "azure" - }, - "ecs" : { - "version" : "1.1.0" - }, - "host" : { - "architecture" : "x86_64", - "os" : { - "build" : "17134.1006", - "platform" : "windows", - "version" : "10.0", - "family" : "windows", - "name" : "Windows 10 Pro", - "kernel" : "10.0.17134.1006 (WinBuild.160101.0800)" }, - "id" : "123456-qwer-1234-5678-12345678", - "hostname" : "DESKTOP-RFOOE09", - "name" : "DESKTOP-RFOOE09" + "namespace": "Microsoft.Compute/virtualMachineScaleSets", + "resource": { + "group": "obs-infrastructure", + "type": "Microsoft.Compute/virtualMachineScaleSets" + }, + "subscription_id": "fd675b6f-b5e5-426e-ac45-d1f876d0ffa6", + "timegrain": "PT5M" }, - "agent" : { - "version" : "8.0.0", - "type" : "metricbeat", - "ephemeral_id" : "123456-qwer-1234-5678-12345678", - "hostname" : "DESKTOP-RFOOE09", - "id" : "123456-qwer-1234-5678-12345678" + "cloud": { + "instance": { + "id": "/subscriptions/fd675b6f-b5e5-426e-ac45-d1f876d0ffa6/resourceGroups/obs-infrastructure/providers/Microsoft.Compute/virtualMachineScaleSets/obslinuxvmss", + "name": "obslinuxvmss" + }, + "machine": { + "type": "Standard_B1ls" + }, + "provider": "azure", + "region": "westeurope" }, - "cloud" : { - "provider" : "azure", - "region" : "westeurope" + "event": { + "dataset": "azure.compute_vm_scaleset", + "duration": 115000, + "module": "azure" }, - "event" : { - "dataset" : "azure.compute_vm_scaleset", - "module" : "azure", - "duration" : 69629063400 + "metricset": { + "name": "compute_vm_scaleset", + "period": 10000 }, - "metricset" : { - "name" : "compute_vm_scaleset", - "period" : 300000 + "service": { + "type": "azure" } } diff --git a/x-pack/metricbeat/module/azure/compute_vm_scaleset/client_helper.go b/x-pack/metricbeat/module/azure/compute_vm_scaleset/client_helper.go index e39c591bdc2..f45896dce8e 100644 --- a/x-pack/metricbeat/module/azure/compute_vm_scaleset/client_helper.go +++ b/x-pack/metricbeat/module/azure/compute_vm_scaleset/client_helper.go @@ -24,6 +24,8 @@ const ( func mapMetrics(client *azure.Client, resources []resources.GenericResource, resourceConfig azure.ResourceConfig) ([]azure.Metric, error) { var metrics []azure.Metric for _, resource := range resources { + // return resource size + resourceSize := mapResourceSize(resource) for _, metric := range resourceConfig.Metrics { metricDefinitions, err := client.AzureMonitorService.GetMetricDefinitions(*resource.ID, metric.Namespace) if err != nil { @@ -75,9 +77,17 @@ func mapMetrics(client *azure.Client, resources []resources.GenericResource, res if key != azure.NoDimension { dimensions = []azure.Dimension{{Name: key, Value: "*"}} } - metrics = append(metrics, azure.MapMetricByPrimaryAggregation(client, metricGroup, resource, "", metric.Namespace, dimensions, azure.DefaultTimeGrain)...) + metrics = append(metrics, client.MapMetricByPrimaryAggregation(metricGroup, resource, "", resourceSize, metric.Namespace, dimensions, azure.DefaultTimeGrain)...) } } } return metrics, nil } + +// mapResourceSize func will try to map if existing the resource size, for the vmss it seems that SKU is populated and resource size is mapped in the name +func mapResourceSize(resource resources.GenericResource) string { + if resource.Sku != nil && resource.Sku.Name != nil { + return *resource.Sku.Name + } + return "" +} diff --git a/x-pack/metricbeat/module/azure/compute_vm_scaleset/client_helper_test.go b/x-pack/metricbeat/module/azure/compute_vm_scaleset/client_helper_test.go index e81582d48b3..6fba1dab50f 100644 --- a/x-pack/metricbeat/module/azure/compute_vm_scaleset/client_helper_test.go +++ b/x-pack/metricbeat/module/azure/compute_vm_scaleset/client_helper_test.go @@ -21,11 +21,16 @@ func MockResource() resources.GenericResource { name := "resourceName" location := "resourceLocation" rType := "resourceType" + skuName := "standard" + sku := resources.Sku{ + Name: &skuName, + } return resources.GenericResource{ ID: &id, Name: &name, Location: &location, Type: &rType, + Sku: &sku, } } @@ -93,15 +98,16 @@ func TestMapMetric(t *testing.T) { assert.Nil(t, err) assert.Equal(t, len(metrics), 2) - assert.Equal(t, metrics[0].Resource.ID, "123") + assert.Equal(t, metrics[0].Resource.Id, "123") assert.Equal(t, metrics[0].Resource.Name, "resourceName") assert.Equal(t, metrics[0].Resource.Type, "resourceType") assert.Equal(t, metrics[0].Resource.Location, "resourceLocation") assert.Equal(t, metrics[0].Namespace, "namespace") - assert.Equal(t, metrics[1].Resource.ID, "123") + assert.Equal(t, metrics[1].Resource.Id, "123") assert.Equal(t, metrics[1].Resource.Name, "resourceName") assert.Equal(t, metrics[1].Resource.Type, "resourceType") assert.Equal(t, metrics[1].Resource.Location, "resourceLocation") + assert.Equal(t, metrics[1].Resource.Size, "standard") assert.Equal(t, metrics[1].Namespace, "namespace") assert.Equal(t, metrics[0].Dimensions, []azure.Dimension(nil)) assert.Equal(t, metrics[1].Dimensions, []azure.Dimension(nil)) diff --git a/x-pack/metricbeat/module/azure/compute_vm_scaleset/compute_vm_scaleset_integration_test.go b/x-pack/metricbeat/module/azure/compute_vm_scaleset/compute_vm_scaleset_integration_test.go new file mode 100644 index 00000000000..acd6450a97b --- /dev/null +++ b/x-pack/metricbeat/module/azure/compute_vm_scaleset/compute_vm_scaleset_integration_test.go @@ -0,0 +1,39 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build integration + +package compute_vm_scaleset + +import ( + "testing" + + "github.com/elastic/beats/v7/x-pack/metricbeat/module/azure/test" + + "github.com/stretchr/testify/assert" + + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" +) + +func TestFetchMetricset(t *testing.T) { + config, err := test.GetConfig("compute_vm_scaleset") + if err != nil { + t.Skip("Skipping TestFetch: " + err.Error()) + } + metricSet := mbtest.NewReportingMetricSetV2Error(t, config) + events, errs := mbtest.ReportingFetchV2Error(metricSet) + if len(errs) > 0 { + t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) + } + assert.NotEmpty(t, events) +} + +func TestData(t *testing.T) { + config, err := test.GetConfig("compute_vm_scaleset") + if err != nil { + t.Skip("Skipping TestFetch: " + err.Error()) + } + metricSet := mbtest.NewFetcher(t, config) + metricSet.WriteEvents(t, "/") +} diff --git a/x-pack/metricbeat/module/azure/container_instance/_meta/data.json b/x-pack/metricbeat/module/azure/container_instance/_meta/data.json index 163b491021e..aec7b3bdffb 100644 --- a/x-pack/metricbeat/module/azure/container_instance/_meta/data.json +++ b/x-pack/metricbeat/module/azure/container_instance/_meta/data.json @@ -1,46 +1,46 @@ { - "@timestamp" : "2020-02-28T14:20:00.000Z", - "cloud" : { - "provider" : "azure", - "region" : "westeurope" - }, - "ecs" : { - "version" : "1.4.0" - }, - "metricset" : { - "period" : 300000, - "name" : "container_instance" - }, - "event" : { - "duration" : 2383878000, - "dataset" : "azure.container_instance", - "module" : "azure" - }, - "azure" : { - "dimensions" : { - "container_name" : "testcontainergroup" - }, - "container_instance" : { - "cpu_usage" : { - "avg" : 0 + "@timestamp": "2017-10-12T08:05:34.853Z", + "azure": { + "container_instance": { + "cpu_usage": { + "avg": 0 }, - "memory_usage" : { - "avg" : 0 + "memory_usage": { + "avg": 0 } }, - "timegrain" : "PT5M", - "resource" : { - "group" : "obs-infrastructure", - "tags" : { - "tag1" : "value1" + "dimensions": { + "container_name": "testcontainergroup" + }, + "namespace": "Microsoft.ContainerInstance/containerGroups", + "resource": { + "group": "obs-infrastructure", + "tags": { + "tag1": "value1" }, - "name" : "testcontainergroup", - "type" : "Microsoft.ContainerInstance/containerGroups" + "type": "Microsoft.ContainerInstance/containerGroups" + }, + "subscription_id": "fd675b6f-b5e5-426e-ac45-d1f876d0ffa6", + "timegrain": "PT5M" + }, + "cloud": { + "instance": { + "id": "/subscriptions/fd675b6f-b5e5-426e-ac45-d1f876d0ffa6/resourceGroups/obs-infrastructure/providers/Microsoft.ContainerInstance/containerGroups/testcontainergroup", + "name": "testcontainergroup" }, - "subscription_id" : "123456-qwer-1234-5678-12345678", - "namespace" : "Microsoft.ContainerInstance/containerGroups" + "provider": "azure", + "region": "westeurope" + }, + "event": { + "dataset": "azure.container_instance", + "duration": 115000, + "module": "azure" + }, + "metricset": { + "name": "container_instance", + "period": 10000 }, - "service" : { - "type" : "azure" + "service": { + "type": "azure" } } diff --git a/x-pack/metricbeat/module/azure/container_instance/container_instance_integration_test.go b/x-pack/metricbeat/module/azure/container_instance/container_instance_integration_test.go index 9a7188f7067..7d251e93396 100644 --- a/x-pack/metricbeat/module/azure/container_instance/container_instance_integration_test.go +++ b/x-pack/metricbeat/module/azure/container_instance/container_instance_integration_test.go @@ -2,50 +2,41 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. +// +build integration + package container_instance import ( - "errors" - "os" "testing" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/azure/test" + + "github.com/stretchr/testify/assert" + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" + + // Register input module and metricset + _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/azure/monitor" ) -func TestData(t *testing.T) { - config, err := getConfig() +func TestFetchMetricset(t *testing.T) { + config, err := test.GetConfig("container_instance") if err != nil { - t.Skip("Skipping TestData: " + err.Error()) + t.Skip("Skipping TestFetch: " + err.Error()) } - - metricSet := mbtest.NewFetcher(t, config) - metricSet.WriteEvents(t, "/") + metricSet := mbtest.NewReportingMetricSetV2Error(t, config) + events, errs := mbtest.ReportingFetchV2Error(metricSet) + if len(errs) > 0 { + t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) + } + assert.NotEmpty(t, events) } -func getConfig() (map[string]interface{}, error) { - clientId, ok := os.LookupEnv("AZURE_CLIENT_ID") - if !ok { - return nil, errors.New("missing AZURE_CLIENT_ID key") - } - clientSecret, ok := os.LookupEnv("AZURE_CLIENT_SECRET") - if !ok { - return nil, errors.New("missing AZURE_CLIENT_SECRET key") - } - tenantId, ok := os.LookupEnv("AZURE_TENANT_ID") - if !ok { - return nil, errors.New("missing AZURE_TENANT_ID key") - } - subscriptionId, ok := os.LookupEnv("AZURE_SUBSCRIPTION_ID") - if !ok { - return nil, errors.New("missing AZURE_SUBSCRIPTION_ID key") - } - config := map[string]interface{}{ - "module": "azure", - "metricsets": []string{"container_instance"}, - "client_id": clientId, - "client_secret": clientSecret, - "tenant_id": tenantId, - "subscription_id": subscriptionId, +func TestData(t *testing.T) { + config, err := test.GetConfig("container_instance") + if err != nil { + t.Skip("Skipping TestFetch: " + err.Error()) } - return config, nil + metricSet := mbtest.NewFetcher(t, config) + metricSet.WriteEvents(t, "/") } diff --git a/x-pack/metricbeat/module/azure/container_registry/_meta/data.json b/x-pack/metricbeat/module/azure/container_registry/_meta/data.json index 0167132328b..0ae5b5f3e25 100644 --- a/x-pack/metricbeat/module/azure/container_registry/_meta/data.json +++ b/x-pack/metricbeat/module/azure/container_registry/_meta/data.json @@ -1,37 +1,46 @@ { - "@timestamp" : "2020-02-28T14:47:00.000Z", - "ecs" : { - "version" : "1.4.0" - }, - "cloud" : { - "region" : "westeurope", - "provider" : "azure" + "@timestamp": "2017-10-12T08:05:34.853Z", + "azure": { + "container_registry": { + "successful_pull_count": { + "avg": 1 + }, + "successful_push_count": { + "avg": 1 + }, + "total_pull_count": { + "avg": 1 + }, + "total_push_count": { + "avg": 1 + } + }, + "namespace": "Microsoft.ContainerRegistry/registries", + "resource": { + "group": "obs-infrastructure", + "type": "Microsoft.ContainerRegistry/registries" + }, + "subscription_id": "fd675b6f-b5e5-426e-ac45-d1f876d0ffa6", + "timegrain": "PT5M" }, - "event" : { - "dataset" : "azure.container_registry", - "module" : "azure", - "duration" : 2730896000 + "cloud": { + "instance": { + "id": "/subscriptions/fd675b6f-b5e5-426e-ac45-d1f876d0ffa6/resourceGroups/obs-infrastructure/providers/Microsoft.ContainerRegistry/registries/obstest", + "name": "obstest" + }, + "provider": "azure", + "region": "westeurope" }, - "metricset" : { - "name" : "container_registry", - "period" : 300000 + "event": { + "dataset": "azure.container_registry", + "duration": 115000, + "module": "azure" }, - "azure" : { - "subscription_id" : "123456-qwer-1234-5678-12345678", - "namespace" : "Microsoft.ContainerRegistry/registries", - "container_registry" : { - "successful_push_count" : { - "avg" : 1 - } - }, - "timegrain" : "PT5M", - "resource" : { - "type" : "Microsoft.ContainerRegistry/registries", - "group" : "obs-infrastructure", - "name" : "obstest" - } + "metricset": { + "name": "container_registry", + "period": 10000 }, - "service" : { - "type" : "azure" + "service": { + "type": "azure" } } diff --git a/x-pack/metricbeat/module/azure/container_registry/container_registry_integration_test.go b/x-pack/metricbeat/module/azure/container_registry/container_registry_integration_test.go index ef4929fc1b1..53fce487a7a 100644 --- a/x-pack/metricbeat/module/azure/container_registry/container_registry_integration_test.go +++ b/x-pack/metricbeat/module/azure/container_registry/container_registry_integration_test.go @@ -2,50 +2,41 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. +// +build integration + package container_registry import ( - "errors" - "os" "testing" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/azure/test" + + "github.com/stretchr/testify/assert" + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" + + // Register input module and metricset + _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/azure/monitor" ) -func TestData(t *testing.T) { - config, err := getConfig() +func TestFetchMetricset(t *testing.T) { + config, err := test.GetConfig("container_registry") if err != nil { - t.Skip("Skipping TestData: " + err.Error()) + t.Skip("Skipping TestFetch: " + err.Error()) } - - metricSet := mbtest.NewFetcher(t, config) - metricSet.WriteEvents(t, "/") + metricSet := mbtest.NewReportingMetricSetV2Error(t, config) + events, errs := mbtest.ReportingFetchV2Error(metricSet) + if len(errs) > 0 { + t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) + } + assert.NotEmpty(t, events) } -func getConfig() (map[string]interface{}, error) { - clientId, ok := os.LookupEnv("AZURE_CLIENT_ID") - if !ok { - return nil, errors.New("missing AZURE_CLIENT_ID key") - } - clientSecret, ok := os.LookupEnv("AZURE_CLIENT_SECRET") - if !ok { - return nil, errors.New("missing AZURE_CLIENT_SECRET key") - } - tenantId, ok := os.LookupEnv("AZURE_TENANT_ID") - if !ok { - return nil, errors.New("missing AZURE_TENANT_ID key") - } - subscriptionId, ok := os.LookupEnv("AZURE_SUBSCRIPTION_ID") - if !ok { - return nil, errors.New("missing AZURE_SUBSCRIPTION_ID key") - } - config := map[string]interface{}{ - "module": "azure", - "metricsets": []string{"container_registry"}, - "client_id": clientId, - "client_secret": clientSecret, - "tenant_id": tenantId, - "subscription_id": subscriptionId, +func TestData(t *testing.T) { + config, err := test.GetConfig("container_registry") + if err != nil { + t.Skip("Skipping TestFetch: " + err.Error()) } - return config, nil + metricSet := mbtest.NewFetcher(t, config) + metricSet.WriteEvents(t, "/") } diff --git a/x-pack/metricbeat/module/azure/container_service/_meta/data.json b/x-pack/metricbeat/module/azure/container_service/_meta/data.json index fe6623057a3..07d8025c93d 100644 --- a/x-pack/metricbeat/module/azure/container_service/_meta/data.json +++ b/x-pack/metricbeat/module/azure/container_service/_meta/data.json @@ -1,46 +1,40 @@ { - "@timestamp" : "2020-02-28T14:47:00.000Z", - "ecs" : { - "version" : "1.4.0" - }, - "cloud" : { - "region" : "westeurope", - "provider" : "azure" + "@timestamp": "2017-10-12T08:05:34.853Z", + "azure": { + "container_service": { + "kube_node_status_condition": { + "avg": 1 + } + }, + "dimensions": { + "status": "true" + }, + "namespace": "Microsoft.ContainerService/managedClusters", + "resource": { + "group": "obs-infrastructure", + "type": "Microsoft.ContainerService/managedClusters" + }, + "subscription_id": "fd675b6f-b5e5-426e-ac45-d1f876d0ffa6", + "timegrain": "PT5M" }, - "event" : { - "dataset" : "azure.container_service", - "module" : "azure", - "duration" : 2730896000 + "cloud": { + "instance": { + "id": "/subscriptions/fd675b6f-b5e5-426e-ac45-d1f876d0ffa6/resourceGroups/obs-infrastructure/providers/Microsoft.ContainerService/managedClusters/obskube", + "name": "obskube" + }, + "provider": "azure", + "region": "westeurope" }, - "metricset" : { - "name" : "container_service", - "period" : 300000 + "event": { + "dataset": "azure.container_service", + "duration": 115000, + "module": "azure" }, - "azure" : { - "subscription_id" : "123456-qwer-1234-5678-12345678", - "namespace" : "Microsoft.ContainerService/managedClusters", - "container_service" : { - "kube_pod_status_ready" : { - "avg" : 9 - }, - "kube_pod_status_phase" : { - "avg" : 9 - }, - "kube_node_status_allocatable_cpu_cores" : { - "avg" : 2 - }, - "kube_node_status_allocatable_memory_bytes" : { - "avg" : 2.25349632E9 - } - }, - "timegrain" : "PT5M", - "resource" : { - "type" : "Microsoft.ContainerService/managedClusters", - "group" : "obs-infrastructure", - "name" : "obskube" - } + "metricset": { + "name": "container_service", + "period": 10000 }, - "service" : { - "type" : "azure" + "service": { + "type": "azure" } } diff --git a/x-pack/metricbeat/module/azure/container_service/container_service_integration_test.go b/x-pack/metricbeat/module/azure/container_service/container_service_integration_test.go index 8423cb06668..93acb39da51 100644 --- a/x-pack/metricbeat/module/azure/container_service/container_service_integration_test.go +++ b/x-pack/metricbeat/module/azure/container_service/container_service_integration_test.go @@ -2,50 +2,41 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. +// +build integration + package container_service import ( - "errors" - "os" "testing" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/azure/test" + + "github.com/stretchr/testify/assert" + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" + + // Register input module and metricset + _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/azure/monitor" ) -func TestData(t *testing.T) { - config, err := getConfig() +func TestFetchMetricset(t *testing.T) { + config, err := test.GetConfig("container_service") if err != nil { - t.Skip("Skipping TestData: " + err.Error()) + t.Skip("Skipping TestFetch: " + err.Error()) } - - metricSet := mbtest.NewFetcher(t, config) - metricSet.WriteEvents(t, "/") + metricSet := mbtest.NewReportingMetricSetV2Error(t, config) + events, errs := mbtest.ReportingFetchV2Error(metricSet) + if len(errs) > 0 { + t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) + } + assert.NotEmpty(t, events) } -func getConfig() (map[string]interface{}, error) { - clientId, ok := os.LookupEnv("AZURE_CLIENT_ID") - if !ok { - return nil, errors.New("missing AZURE_CLIENT_ID key") - } - clientSecret, ok := os.LookupEnv("AZURE_CLIENT_SECRET") - if !ok { - return nil, errors.New("missing AZURE_CLIENT_SECRET key") - } - tenantId, ok := os.LookupEnv("AZURE_TENANT_ID") - if !ok { - return nil, errors.New("missing AZURE_TENANT_ID key") - } - subscriptionId, ok := os.LookupEnv("AZURE_SUBSCRIPTION_ID") - if !ok { - return nil, errors.New("missing AZURE_SUBSCRIPTION_ID key") - } - config := map[string]interface{}{ - "module": "azure", - "metricsets": []string{"container_registry"}, - "client_id": clientId, - "client_secret": clientSecret, - "tenant_id": tenantId, - "subscription_id": subscriptionId, +func TestData(t *testing.T) { + config, err := test.GetConfig("container_service") + if err != nil { + t.Skip("Skipping TestFetch: " + err.Error()) } - return config, nil + metricSet := mbtest.NewFetcher(t, config) + metricSet.WriteEvents(t, "/") } diff --git a/x-pack/metricbeat/module/azure/data.go b/x-pack/metricbeat/module/azure/data.go index 64920f1eb64..23a043ae6f9 100644 --- a/x-pack/metricbeat/module/azure/data.go +++ b/x-pack/metricbeat/module/azure/data.go @@ -35,7 +35,7 @@ func EventsMapping(metrics []Metric, metricset string, report mb.ReporterV2) err continue } // build a resource key with unique resource namespace combination - resNamkey := fmt.Sprintf("%s,%s", metric.Resource.ID, metric.Namespace) + resNamkey := fmt.Sprintf("%s,%s", metric.Resource.Id, metric.Namespace) groupByResourceNamespace[resNamkey] = append(groupByResourceNamespace[resNamkey], metric) } // grouping metrics by the dimensions configured @@ -147,7 +147,6 @@ func createEvent(timestamp time.Time, metric Metric, metricValues []MetricValue) ModuleFields: common.MapStr{ "timegrain": metric.TimeGrain, "resource": common.MapStr{ - "name": metric.Resource.Name, "type": metric.Resource.Type, "group": metric.Resource.Group, }, @@ -173,7 +172,15 @@ func createEvent(timestamp time.Time, metric Metric, metricValues []MetricValue) event.RootFields = common.MapStr{} event.RootFields.Put("cloud.provider", "azure") event.RootFields.Put("cloud.region", metric.Resource.Location) - + event.RootFields.Put("cloud.instance.name", metric.Resource.Name) + if metric.Resource.SubId != "" { + event.RootFields.Put("cloud.instance.id", metric.Resource.SubId) + } else { + event.RootFields.Put("cloud.instance.id", metric.Resource.Id) + } + if metric.Resource.Size != "" { + event.RootFields.Put("cloud.machine.type", metric.Resource.Size) + } metricList := common.MapStr{} for _, value := range metricValues { metricNameString := fmt.Sprintf("%s", managePropertyName(value.name)) diff --git a/x-pack/metricbeat/module/azure/data_test.go b/x-pack/metricbeat/module/azure/data_test.go index d26f9270f0f..1ef042b12c2 100644 --- a/x-pack/metricbeat/module/azure/data_test.go +++ b/x-pack/metricbeat/module/azure/data_test.go @@ -75,7 +75,7 @@ func TestCreateEvent(t *testing.T) { } metric := Metric{ Resource: Resource{ - ID: "resId", + Id: "resId", Name: "res", Location: "west_europe", Type: "resType", diff --git a/x-pack/metricbeat/module/azure/database_account/database_account_integration_test.go b/x-pack/metricbeat/module/azure/database_account/database_account_integration_test.go index 16b8cd0d809..8099e729687 100644 --- a/x-pack/metricbeat/module/azure/database_account/database_account_integration_test.go +++ b/x-pack/metricbeat/module/azure/database_account/database_account_integration_test.go @@ -2,50 +2,38 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. +// +build integration + package database_account import ( - "errors" - "os" "testing" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/azure/test" + + "github.com/stretchr/testify/assert" + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" ) -func TestData(t *testing.T) { - config, err := getConfig() +func TestFetchMetricset(t *testing.T) { + config, err := test.GetConfig("database_account") if err != nil { - t.Skip("Skipping TestData: " + err.Error()) + t.Skip("Skipping TestFetch: " + err.Error()) } - - metricSet := mbtest.NewFetcher(t, config) - metricSet.WriteEvents(t, "/") + metricSet := mbtest.NewReportingMetricSetV2Error(t, config) + events, errs := mbtest.ReportingFetchV2Error(metricSet) + if len(errs) > 0 { + t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) + } + assert.NotEmpty(t, events) } -func getConfig() (map[string]interface{}, error) { - clientId, ok := os.LookupEnv("AZURE_CLIENT_ID") - if !ok { - return nil, errors.New("missing AZURE_CLIENT_ID key") - } - clientSecret, ok := os.LookupEnv("AZURE_CLIENT_SECRET") - if !ok { - return nil, errors.New("missing AZURE_CLIENT_SECRET key") - } - tenantId, ok := os.LookupEnv("AZURE_TENANT_ID") - if !ok { - return nil, errors.New("missing AZURE_TENANT_ID key") - } - subscriptionId, ok := os.LookupEnv("AZURE_SUBSCRIPTION_ID") - if !ok { - return nil, errors.New("missing AZURE_SUBSCRIPTION_ID key") - } - config := map[string]interface{}{ - "module": "azure", - "metricsets": []string{"database_account"}, - "client_id": clientId, - "client_secret": clientSecret, - "tenant_id": tenantId, - "subscription_id": subscriptionId, +func TestData(t *testing.T) { + config, err := test.GetConfig("database_account") + if err != nil { + t.Skip("Skipping TestFetch: " + err.Error()) } - return config, nil + metricSet := mbtest.NewFetcher(t, config) + metricSet.WriteEvents(t, "/") } diff --git a/x-pack/metricbeat/module/azure/fields.go b/x-pack/metricbeat/module/azure/fields.go index 25f8d1b6fcc..478d649d8a2 100644 --- a/x-pack/metricbeat/module/azure/fields.go +++ b/x-pack/metricbeat/module/azure/fields.go @@ -19,5 +19,5 @@ func init() { // AssetAzure returns asset data. // This is the base64 encoded gzipped contents of module/azure. func AssetAzure() string { - return "eJzUlr+O2zAMxvc8BZExwOUBPBQo0KVDt+6GIjMue7ZkkPQV6dMXjm2d/D/p3ZBoCBBJ/r7fJ9KCX+AVLwmYvzXjDkBJC0xgf/2/3wFkKJapUvIugS87AGj3QumzumgeYSzQCCZwQjU7gDNhkUly3foCzpT4Lt8MvVSYQM6+rrqZGY+hTCylVGLOhlxY6SVf8fLHcxbNzwq34+cvhK9tEFQmO6PbOzKKr9nixDDOcINdrwNSoaUzYYw6jhsDNL+DheXEGxg9SqMI/gwaYc1aNz6fa908fpP1+HQ/wTtUYKodAptcjodZW3/6jVZHS+1kugYWbUlLU1Xk8m7//rC/L0TbsSHGFXbSr82vVGamYe9+Q4IUCBZoNWrZ3k3qU5BIKfu4ZywI379NDDMq0Ql5N6zTQo026nNrbVaYB7dIBDcBt76sasX0rTweBujjG/SeSOfCm4XF/w30zrkSIRVrChTUZ8kSgMP6TDqnhhxySk7UOIuPHq7jhZ53JRJjTqJ8eZZIPe9KJEF+o+cpUoc7vdCMmpMRTI21vnYP/kb1tNDRTuKU3pF63vxaWsq14j2VXvtsaq9kGR3nytFtHt89R7gRpRk/WkBg1JodZsflq0nUs8kfvNU7yNAZ/wIAAP//ilZVQA==" + return "eJzUls+OmzAQxu95ilGOkTYPwKFSpV566K135JgJnS7YaGbYij59RQCv+b9pe0h8iBTbfN/38wwWL/CKTQLmd814AFDSAhM43v4fDwAZimWqlLxL4NMBALq9UPqsLtpHGAs0gglcUM0B4EpYZJLctr6AMyW+y7dDmwoTyNnXVT+z4DGWiaWUSszZkAsrg+QrNr88Z9H8onA3vv9A+NyBoDLZBd3BkVF8zRZnhjHDB+wGHZAKLV0J46hT3BFyU+FoYZ14J8YQpX0c/BU0irVoPUX8D97hGObaAdjkcj4t2vrLT7Q6Weom061g0Za0NFVFLu/3H0/H+yC6tgkYt7Czpml/pTILXXN3mwYpECzQatQ3g5vUlyCRUvbvnrEgfP0yM8yoRCfk3bhOKzXaqc9Ha7ORefQqR+Fmwa0vq1oxfSvPp1H06TV2D9K18GZl8W+B3nNuIKRiTYGC+iwsIXBYX6Bzasghp+REjbP46HB9XhjybiAx5iTKzbMgDXk3kAT5jZ6nSH3c+YVm1FyMYGqs9bV78DdqSAt92hlO6R2p591PljWuDe+59Na3S3cly+Q4N45u9/juOcIdlHZ86wICo9bsMDuvX02ink3+4K3ehwyd8ScAAP//n2w3Eg==" } diff --git a/x-pack/metricbeat/module/azure/mock_service.go b/x-pack/metricbeat/module/azure/mock_service.go index 05ee83ddad9..4d2dcd73f6e 100644 --- a/x-pack/metricbeat/module/azure/mock_service.go +++ b/x-pack/metricbeat/module/azure/mock_service.go @@ -18,27 +18,33 @@ type MockService struct { mock.Mock } +// GetResourceDefinitionById is a mock function for the azure service +func (client *MockService) GetResourceDefinitionById(id string) (resources.GenericResource, error) { + args := client.Called(id) + return args.Get(0).(resources.GenericResource), args.Error(1) +} + // GetResourceDefinitions is a mock function for the azure service -func (client *MockService) GetResourceDefinitions(ID []string, group []string, rType string, query string) (resources.ListResultPage, error) { - args := client.Called(ID, group, rType, query) +func (client *MockService) GetResourceDefinitions(id []string, group []string, rType string, query string) (resources.ListResultPage, error) { + args := client.Called(id, group, rType, query) return args.Get(0).(resources.ListResultPage), args.Error(1) } // GetMetricDefinitions is a mock function for the azure service -func (client *MockService) GetMetricDefinitions(resourceID string, namespace string) (insights.MetricDefinitionCollection, error) { - args := client.Called(resourceID, namespace) +func (client *MockService) GetMetricDefinitions(resourceId string, namespace string) (insights.MetricDefinitionCollection, error) { + args := client.Called(resourceId, namespace) return args.Get(0).(insights.MetricDefinitionCollection), args.Error(1) } // GetMetricNamespaces is a mock function for the azure service -func (client *MockService) GetMetricNamespaces(resourceID string) (insights.MetricNamespaceCollection, error) { - args := client.Called(resourceID) +func (client *MockService) GetMetricNamespaces(resourceId string) (insights.MetricNamespaceCollection, error) { + args := client.Called(resourceId) return args.Get(0).(insights.MetricNamespaceCollection), args.Error(1) } // GetMetricValues is a mock function for the azure service -func (client *MockService) GetMetricValues(resourceID string, namespace string, timegrain string, timespan string, metricNames []string, aggregations string, filter string) ([]insights.Metric, string, error) { - args := client.Called(resourceID, namespace) +func (client *MockService) GetMetricValues(resourceId string, namespace string, timegrain string, timespan string, metricNames []string, aggregations string, filter string) ([]insights.Metric, string, error) { + args := client.Called(resourceId, namespace) return args.Get(0).([]insights.Metric), args.String(1), args.Error(2) } diff --git a/x-pack/metricbeat/module/azure/monitor/_meta/data.json b/x-pack/metricbeat/module/azure/monitor/_meta/data.json index d267245ce1e..67f5b91b62b 100644 --- a/x-pack/metricbeat/module/azure/monitor/_meta/data.json +++ b/x-pack/metricbeat/module/azure/monitor/_meta/data.json @@ -1,58 +1,46 @@ { - "@timestamp":"2019-08-20T13:52:00.000Z", - "service":{ - "type":"azure" - }, - "azure":{ - "timegrain" : "PT1M", - "subscriptionID":"123456-qwer-1234-5678-12345678", - "metrics":{ - "percentage_cpu":{ - "min":1.86, - "max":2.8, - "avg":2.535 + "@timestamp": "2017-10-12T08:05:34.853Z", + "azure": { + "metrics": { + "data_usage": { + "total": 0 + }, + "document_count": { + "total": 0 + }, + "document_quota": { + "total": 53687091200 } }, - "resource":{ - "name":"vmwindows", - "type":"Microsoft.Compute/virtualMachines" + "namespace": "Microsoft.DocumentDb/databaseAccounts", + "resource": { + "group": "obs-infrastructure", + "tags": { + "defaultExperience": "Core (SQL)" + }, + "type": "Microsoft.DocumentDb/databaseAccounts" }, - "namespace":"Microsoft.Compute/virtualMachines" - }, - "cloud":{ - "provider":"azure", - "region":"westeurope" - }, - "metricset":{ - "name":"monitor" + "subscription_id": "fd675b6f-b5e5-426e-ac45-d1f876d0ffa6", + "timegrain": "PT5M" }, - "event":{ - "module":"azure", - "duration":40189255100, - "dataset":"azure.monitor" + "cloud": { + "instance": { + "id": "/subscriptions/fd675b6f-b5e5-426e-ac45-d1f876d0ffa6/resourceGroups/obs-infrastructure/providers/Microsoft.DocumentDb/databaseAccounts/obsaccount", + "name": "obsaccount" + }, + "provider": "azure", + "region": "westeurope" }, - "ecs":{ - "version":"1.0.1" + "event": { + "dataset": "azure.monitor", + "duration": 115000, + "module": "azure" }, - "host":{ - "name":"DESKTOP-RFOOE09", - "architecture":"x86_64", - "os":{ - "family":"windows", - "name":"Windows 10 Pro", - "kernel":"10.0.17134.950 (WinBuild.160101.0800)", - "build":"17134.950", - "platform":"windows", - "version":"10.0" - }, - "id":"123456-qwer-1234-5678-12345678", - "hostname":"DESKTOP-RFOOE09" + "metricset": { + "name": "monitor", + "period": 10000 }, - "agent":{ - "ephemeral_id":"123456-qwer-1234-5678-12345678", - "hostname":"DESKTOP-RFOOE09", - "id":"123456-qwer-1234-5678-12345678", - "version":"8.0.0", - "type":"metricbeat" + "service": { + "type": "azure" } } diff --git a/x-pack/metricbeat/module/azure/monitor/client_helper.go b/x-pack/metricbeat/module/azure/monitor/client_helper.go index 55a805f76e6..4d00bcd64de 100644 --- a/x-pack/metricbeat/module/azure/monitor/client_helper.go +++ b/x-pack/metricbeat/module/azure/monitor/client_helper.go @@ -54,7 +54,7 @@ func mapMetrics(client *azure.Client, resources []resources.GenericResource, res for _, metricName := range metricGroup { metricNames = append(metricNames, *metricName.Name.Value) } - metrics = append(metrics, client.CreateMetric(*resource.ID, resource, metric.Namespace, metricNames, key, dim, metric.Timegrain)) + metrics = append(metrics, client.CreateMetric(*resource.ID, resource, "", metric.Namespace, metricNames, key, dim, metric.Timegrain)) } } } @@ -62,7 +62,7 @@ func mapMetrics(client *azure.Client, resources []resources.GenericResource, res } // filterMetricNames func will verify if the metric names entered are valid and will also return the corresponding list of metrics -func filterMetricNames(resourceID string, metricConfig azure.MetricConfig, metricDefinitions []insights.MetricDefinition) ([]string, error) { +func filterMetricNames(resourceId string, metricConfig azure.MetricConfig, metricDefinitions []insights.MetricDefinition) ([]string, error) { var supportedMetricNames []string var unsupportedMetricNames []string // check if all metric names are selected (*) @@ -75,11 +75,11 @@ func filterMetricNames(resourceID string, metricConfig azure.MetricConfig, metri supportedMetricNames, unsupportedMetricNames = filterSConfiguredMetrics(metricConfig.Name, metricDefinitions) if len(unsupportedMetricNames) > 0 { return nil, errors.Errorf("the metric names configured %s are not supported for the resource %s and namespace %s", - strings.Join(unsupportedMetricNames, ","), resourceID, metricConfig.Namespace) + strings.Join(unsupportedMetricNames, ","), resourceId, metricConfig.Namespace) } } if len(supportedMetricNames) == 0 { - return nil, errors.Errorf("the metric names configured : %s are not supported for the resource %s and namespace %s ", strings.Join(metricConfig.Name, ","), resourceID, metricConfig.Namespace) + return nil, errors.Errorf("the metric names configured : %s are not supported for the resource %s and namespace %s ", strings.Join(metricConfig.Name, ","), resourceId, metricConfig.Namespace) } return supportedMetricNames, nil } diff --git a/x-pack/metricbeat/module/azure/monitor/client_helper_test.go b/x-pack/metricbeat/module/azure/monitor/client_helper_test.go index 03cc0a07eb8..8765d589e84 100644 --- a/x-pack/metricbeat/module/azure/monitor/client_helper_test.go +++ b/x-pack/metricbeat/module/azure/monitor/client_helper_test.go @@ -79,7 +79,7 @@ func TestMapMetric(t *testing.T) { resourceConfig.Metrics = []azure.MetricConfig{metricConfig} metrics, err := mapMetrics(client, []resources.GenericResource{resource}, resourceConfig) assert.Nil(t, err) - assert.Equal(t, metrics[0].Resource.ID, "123") + assert.Equal(t, metrics[0].Resource.Id, "123") assert.Equal(t, metrics[0].Resource.Name, "resourceName") assert.Equal(t, metrics[0].Resource.Type, "resourceType") assert.Equal(t, metrics[0].Resource.Location, "resourceLocation") @@ -100,7 +100,7 @@ func TestMapMetric(t *testing.T) { assert.Nil(t, err) assert.True(t, len(metrics) > 0) - assert.Equal(t, metrics[0].Resource.ID, "123") + assert.Equal(t, metrics[0].Resource.Id, "123") assert.Equal(t, metrics[0].Resource.Name, "resourceName") assert.Equal(t, metrics[0].Resource.Type, "resourceType") assert.Equal(t, metrics[0].Resource.Location, "resourceLocation") diff --git a/x-pack/metricbeat/module/azure/monitor/monitor_integration_test.go b/x-pack/metricbeat/module/azure/monitor/monitor_integration_test.go new file mode 100644 index 00000000000..87de0e38bce --- /dev/null +++ b/x-pack/metricbeat/module/azure/monitor/monitor_integration_test.go @@ -0,0 +1,43 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build integration + +package monitor + +import ( + "testing" + + "github.com/elastic/beats/v7/x-pack/metricbeat/module/azure/test" + + "github.com/stretchr/testify/assert" + + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" +) + +func TestFetchMetricset(t *testing.T) { + config, err := test.GetConfig("monitor") + if err != nil { + t.Skip("Skipping TestFetch: " + err.Error()) + } + metricSet := mbtest.NewReportingMetricSetV2Error(t, config) + events, errs := mbtest.ReportingFetchV2Error(metricSet) + if len(errs) > 0 { + t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) + } + assert.NotEmpty(t, events) +} + +func TestData(t *testing.T) { + config, err := test.GetConfig("monitor") + if err != nil { + t.Skip("Skipping TestFetch: " + err.Error()) + } + config["resources"] = []map[string]interface{}{{ + "resource_query": "resourceType eq 'Microsoft.DocumentDb/databaseAccounts'", + "metrics": []map[string]interface{}{{"namespace": "Microsoft.DocumentDb/databaseAccounts", + "name": []string{"DataUsage", "DocumentCount", "DocumentQuota"}}}}} + metricSet := mbtest.NewFetcher(t, config) + metricSet.WriteEvents(t, "/") +} diff --git a/x-pack/metricbeat/module/azure/monitor_service.go b/x-pack/metricbeat/module/azure/monitor_service.go index cae605d6364..4657e3bc43d 100644 --- a/x-pack/metricbeat/module/azure/monitor_service.go +++ b/x-pack/metricbeat/module/azure/monitor_service.go @@ -29,16 +29,16 @@ type MonitorService struct { const metricNameLimit = 20 // NewService instantiates the Azure monitoring service -func NewService(clientID string, clientSecret string, tenantID string, subscriptionID string) (*MonitorService, error) { - clientConfig := auth.NewClientCredentialsConfig(clientID, clientSecret, tenantID) +func NewService(clientId string, clientSecret string, tenantId string, subscriptionId string) (*MonitorService, error) { + clientConfig := auth.NewClientCredentialsConfig(clientId, clientSecret, tenantId) authorizer, err := clientConfig.Authorizer() if err != nil { return nil, err } - metricsClient := insights.NewMetricsClient(subscriptionID) - metricsDefinitionClient := insights.NewMetricDefinitionsClient(subscriptionID) - resourceClient := resources.NewClient(subscriptionID) - metricNamespaceClient := insights.NewMetricNamespacesClient(subscriptionID) + metricsClient := insights.NewMetricsClient(subscriptionId) + metricsDefinitionClient := insights.NewMetricDefinitionsClient(subscriptionId) + resourceClient := resources.NewClient(subscriptionId) + metricNamespaceClient := insights.NewMetricNamespacesClient(subscriptionId) metricsClient.Authorizer = authorizer metricsDefinitionClient.Authorizer = authorizer resourceClient.Authorizer = authorizer @@ -55,15 +55,15 @@ func NewService(clientID string, clientSecret string, tenantID string, subscript } // GetResourceDefinitions will retrieve the azure resources based on the options entered -func (service MonitorService) GetResourceDefinitions(ID []string, group []string, rType string, query string) (resources.ListResultPage, error) { +func (service MonitorService) GetResourceDefinitions(id []string, group []string, rType string, query string) (resources.ListResultPage, error) { var resourceQuery string - if len(ID) > 0 { + if len(id) > 0 { var filterList []string // listing resourceID conditions does not seem to work with the API but querying by name or resource types will work - for _, id := range ID { - filterList = append(filterList, fmt.Sprintf("name eq '%s'", getResourceNameFromID(id))) + for _, id := range id { + filterList = append(filterList, fmt.Sprintf("name eq '%s'", getResourceNameFromId(id))) } - resourceQuery = fmt.Sprintf("(%s) AND resourceType eq '%s'", strings.Join(filterList, " OR "), getResourceTypeFromID(ID[0])) + resourceQuery = fmt.Sprintf("(%s) AND resourceType eq '%s'", strings.Join(filterList, " OR "), getResourceTypeFromId(id[0])) } else if len(group) > 0 { var filterList []string for _, gr := range group { @@ -76,21 +76,26 @@ func (service MonitorService) GetResourceDefinitions(ID []string, group []string } else if query != "" { resourceQuery = query } - return service.resourceClient.List(service.context, resourceQuery, "resourceID", nil) + return service.resourceClient.List(service.context, resourceQuery, "", nil) +} + +// GetResourceDefinitionById will retrieve the azure resource based on the resource Id +func (service MonitorService) GetResourceDefinitionById(id string) (resources.GenericResource, error) { + return service.resourceClient.GetByID(service.context, id) } // GetMetricNamespaces will return all supported namespaces based on the resource id and namespace -func (service *MonitorService) GetMetricNamespaces(resourceID string) (insights.MetricNamespaceCollection, error) { - return service.metricNamespaceClient.List(service.context, resourceID, "") +func (service *MonitorService) GetMetricNamespaces(resourceId string) (insights.MetricNamespaceCollection, error) { + return service.metricNamespaceClient.List(service.context, resourceId, "") } // GetMetricDefinitions will return all supported metrics based on the resource id and namespace -func (service *MonitorService) GetMetricDefinitions(resourceID string, namespace string) (insights.MetricDefinitionCollection, error) { - return service.metricDefinitionClient.List(service.context, resourceID, namespace) +func (service *MonitorService) GetMetricDefinitions(resourceId string, namespace string) (insights.MetricDefinitionCollection, error) { + return service.metricDefinitionClient.List(service.context, resourceId, namespace) } // GetMetricValues will return the metric values based on the resource and metric details -func (service *MonitorService) GetMetricValues(resourceID string, namespace string, timegrain string, timespan string, metricNames []string, aggregations string, filter string) ([]insights.Metric, string, error) { +func (service *MonitorService) GetMetricValues(resourceId string, namespace string, timegrain string, timespan string, metricNames []string, aggregations string, filter string) ([]insights.Metric, string, error) { var tg *string var interval string if timegrain != "" { @@ -103,12 +108,12 @@ func (service *MonitorService) GetMetricValues(resourceID string, namespace stri if end > len(metricNames) { end = len(metricNames) } - resp, err := service.metricsClient.List(service.context, resourceID, timespan, tg, strings.Join(metricNames[i:end], ","), + resp, err := service.metricsClient.List(service.context, resourceId, timespan, tg, strings.Join(metricNames[i:end], ","), aggregations, nil, "", filter, insights.Data, namespace) // check for applied charges before returning any errors if resp.Cost != nil && *resp.Cost != 0 { - service.log.Warnf("Charges amounted to %v are being applied while retrieving the metric values from the resource %s ", *resp.Cost, resourceID) + service.log.Warnf("Charges amounted to %v are being applied while retrieving the metric values from the resource %s ", *resp.Cost, resourceId) } if err != nil { return metrics, "", err diff --git a/x-pack/metricbeat/module/azure/resources.go b/x-pack/metricbeat/module/azure/resources.go index da51accc502..c55757d24ac 100644 --- a/x-pack/metricbeat/module/azure/resources.go +++ b/x-pack/metricbeat/module/azure/resources.go @@ -11,15 +11,16 @@ import ( // Resource will contain the main azure resource details type Resource struct { - // SubID is used for the metric values api as namespaces can apply to sub resrouces ex. storage account: container, blob, vm scaleset: vms - SubID string - ID string + // SubId is used for the metric values api as namespaces can apply to sub resrouces ex. storage account: container, blob, vm scaleset: vms + SubId string + Id string Name string Location string Type string Group string Tags map[string]string Subscription string + Size string } // Metric will contain the main azure metric details diff --git a/x-pack/metricbeat/module/azure/service_interface.go b/x-pack/metricbeat/module/azure/service_interface.go index f8310b6ca7b..e8985a7eedd 100644 --- a/x-pack/metricbeat/module/azure/service_interface.go +++ b/x-pack/metricbeat/module/azure/service_interface.go @@ -11,8 +11,9 @@ import ( // Service interface for the azure monitor service and mock for testing type Service interface { - GetResourceDefinitions(ID []string, group []string, rType string, query string) (resources.ListResultPage, error) - GetMetricDefinitions(resourceID string, namespace string) (insights.MetricDefinitionCollection, error) - GetMetricNamespaces(resourceID string) (insights.MetricNamespaceCollection, error) - GetMetricValues(resourceID string, namespace string, timegrain string, timespan string, metricNames []string, aggregations string, filter string) ([]insights.Metric, string, error) + GetResourceDefinitionById(id string) (resources.GenericResource, error) + GetResourceDefinitions(id []string, group []string, rType string, query string) (resources.ListResultPage, error) + GetMetricDefinitions(resourceId string, namespace string) (insights.MetricDefinitionCollection, error) + GetMetricNamespaces(resourceId string) (insights.MetricNamespaceCollection, error) + GetMetricValues(resourceId string, namespace string, timegrain string, timespan string, metricNames []string, aggregations string, filter string) ([]insights.Metric, string, error) } diff --git a/x-pack/metricbeat/module/azure/storage/_meta/data.json b/x-pack/metricbeat/module/azure/storage/_meta/data.json index 6ede71f6edc..76700ecb8d5 100644 --- a/x-pack/metricbeat/module/azure/storage/_meta/data.json +++ b/x-pack/metricbeat/module/azure/storage/_meta/data.json @@ -1,85 +1,43 @@ { - "_index" : "metricbeat-8.0.0-2020.01.09-000001", - "_type" : "_doc", - "_id" : "XxYjjG8BQe78yNOEwING", - "_score" : null, - "_source" : { - "@timestamp" : "2020-01-09T21:04:00.000Z", - "event" : { - "duration" : 122304420600, - "dataset" : "azure.storage", - "module" : "azure" + "@timestamp": "2017-10-12T08:05:34.853Z", + "azure": { + "namespace": "Microsoft.Storage/storageAccounts/queueServices", + "resource": { + "group": "obs-infrastructure", + "type": "Microsoft.Storage/storageAccounts" }, - "metricset" : { - "name" : "storage", - "period" : 300000 - }, - "azure" : { - "timegrain" : "PT5M", - "namespace" : "Microsoft.Storage/storageAccounts/blobServices", - "dimensions" : { - "apiname" : "DeleteBlob" - }, - "storage" : { - "successe2elatency" : { - "avg" : 9.1 - }, - "availability" : { - "avg" : 100 - }, - "transactions" : { - "total" : 10 - }, - "ingress" : { - "total" : 5330 - }, - "egress" : { - "total" : 1010 - }, - "successserverlatency" : { - "avg" : 9.1 - } + "storage": { + "queue_capacity": { + "avg": 0 }, - "resource" : { - "name" : "obsdiaglinux", - "type" : "Microsoft.Storage/storageAccounts", - "group" : "obs-infrastructure" + "queue_count": { + "avg": 0 }, - "subscription_id" : "12345678-qwer-1234-5678-12345678" - }, - "service" : { - "type" : "azure" + "queue_message_count": { + "avg": 0 + } }, - "ecs" : { - "version" : "1.2.0" - }, - "host" : { - "architecture" : "x86_64", - "name" : "DESKTOP-RFOOE09", - "os" : { - "kernel" : "10.0.18362.535 (WinBuild.160101.0800)", - "build" : "18362.535", - "platform" : "windows", - "version" : "10.0", - "family" : "windows", - "name" : "Windows 10 Pro" - }, - "id" : "12345678-qwer-1234-5678-12345678", - "hostname" : "DESKTOP-RFOOE09" - }, - "agent" : { - "hostname" : "DESKTOP-RFOOE09", - "id" : "12345678-qwer-1234-5678-12345678", - "version" : "8.0.0", - "type" : "metricbeat", - "ephemeral_id" : "12345678-qwer-1234-5678-12345678" + "subscription_id": "fd675b6f-b5e5-426e-ac45-d1f876d0ffa6", + "timegrain": "PT1H" + }, + "cloud": { + "instance": { + "id": "/subscriptions/fd675b6f-b5e5-426e-ac45-d1f876d0ffa6/resourceGroups/obs-infrastructure/providers/Microsoft.Storage/storageAccounts/urcbyscmrkbygsawinvm/queueServices/default", + "name": "urcbyscmrkbygsawinvm" }, - "cloud" : { - "provider" : "azure", - "region" : "westeurope" - } + "provider": "azure", + "region": "westeurope" + }, + "event": { + "dataset": "azure.storage", + "duration": 115000, + "module": "azure" + }, + "metricset": { + "name": "storage", + "period": 10000 }, - "sort" : [ - 1578603840000 - ] + "service": { + "type": "azure" + } } diff --git a/x-pack/metricbeat/module/azure/storage/client_helper.go b/x-pack/metricbeat/module/azure/storage/client_helper.go index 4103649c2c9..3fa8bb7e4b9 100644 --- a/x-pack/metricbeat/module/azure/storage/client_helper.go +++ b/x-pack/metricbeat/module/azure/storage/client_helper.go @@ -61,7 +61,7 @@ func mapMetrics(client *azure.Client, resources []resources.GenericResource, res if dimension != azure.NoDimension { dimensions = []azure.Dimension{{Name: dimension, Value: "*"}} } - metrics = append(metrics, azure.MapMetricByPrimaryAggregation(client, mets, resource, resourceID, namespace, dimensions, time)...) + metrics = append(metrics, client.MapMetricByPrimaryAggregation(mets, resource, resourceID, "", namespace, dimensions, time)...) } } } diff --git a/x-pack/metricbeat/module/azure/storage/client_helper_test.go b/x-pack/metricbeat/module/azure/storage/client_helper_test.go index 6221c97a2fb..24fea2facf6 100644 --- a/x-pack/metricbeat/module/azure/storage/client_helper_test.go +++ b/x-pack/metricbeat/module/azure/storage/client_helper_test.go @@ -109,12 +109,12 @@ func TestMapMetric(t *testing.T) { client.AzureMonitorService = m metrics, err := mapMetrics(client, []resources.GenericResource{resource}, resourceConfig) assert.Nil(t, err) - assert.Equal(t, metrics[0].Resource.ID, "123") + assert.Equal(t, metrics[0].Resource.Id, "123") assert.Equal(t, metrics[0].Resource.Name, "resourceName") assert.Equal(t, metrics[0].Resource.Type, "resourceType") assert.Equal(t, metrics[0].Resource.Location, "resourceLocation") assert.Equal(t, metrics[0].Namespace, "Microsoft.Storage/storageAccounts") - assert.Equal(t, metrics[1].Resource.ID, "123") + assert.Equal(t, metrics[1].Resource.Id, "123") assert.Equal(t, metrics[1].Resource.Name, "resourceName") assert.Equal(t, metrics[1].Resource.Type, "resourceType") assert.Equal(t, metrics[1].Resource.Location, "resourceLocation") diff --git a/x-pack/metricbeat/module/azure/storage/storage_integration_test.go b/x-pack/metricbeat/module/azure/storage/storage_integration_test.go new file mode 100644 index 00000000000..920f21393c1 --- /dev/null +++ b/x-pack/metricbeat/module/azure/storage/storage_integration_test.go @@ -0,0 +1,39 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build integration + +package storage + +import ( + "testing" + + "github.com/elastic/beats/v7/x-pack/metricbeat/module/azure/test" + + "github.com/stretchr/testify/assert" + + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" +) + +func TestFetchMetricset(t *testing.T) { + config, err := test.GetConfig("storage") + if err != nil { + t.Skip("Skipping TestFetch: " + err.Error()) + } + metricSet := mbtest.NewReportingMetricSetV2Error(t, config) + events, errs := mbtest.ReportingFetchV2Error(metricSet) + if len(errs) > 0 { + t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) + } + assert.NotEmpty(t, events) +} + +func TestData(t *testing.T) { + config, err := test.GetConfig("storage") + if err != nil { + t.Skip("Skipping TestFetch: " + err.Error()) + } + metricSet := mbtest.NewFetcher(t, config) + metricSet.WriteEvents(t, "/") +} diff --git a/x-pack/metricbeat/module/azure/test/integration.go b/x-pack/metricbeat/module/azure/test/integration.go new file mode 100644 index 00000000000..9e8e011a217 --- /dev/null +++ b/x-pack/metricbeat/module/azure/test/integration.go @@ -0,0 +1,40 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package test + +import ( + "errors" + "os" +) + +// GetConfig function gets azure credentials for integration tests. +func GetConfig(metricSetName string) (map[string]interface{}, error) { + clientId, ok := os.LookupEnv("AZURE_CLIENT_ID") + if !ok { + return nil, errors.New("could not find var AZURE_CLIENT_ID") + } + clientSecret, ok := os.LookupEnv("AZURE_CLIENT_SECRET") + if !ok { + return nil, errors.New("could not find var AZURE_CLIENT_SECRET") + } + tenantId, ok := os.LookupEnv("AZURE_TENANT_ID") + if !ok { + return nil, errors.New("could not find var AZURE_TENANT_ID") + } + subId, ok := os.LookupEnv("AZURE_SUBSCRIPTION_ID") + if !ok { + return nil, errors.New("could not find var AZURE_SUBSCRIPTION_ID") + } + return map[string]interface{}{ + "module": "azure", + "period": "300s", + "refresh_list_interval": "600s", + "metricsets": []string{metricSetName}, + "client_id": clientId, + "client_secret": clientSecret, + "tenant_id": tenantId, + "subscription_id": subId, + }, nil +}