Map cloud.account.id to azure sub id (#21483)

* mofidy doc * tet * add cloud * changelog
elastic · Oct 5, 2020 · 9aab0c0 · 9aab0c0
1 parent 67a2d38
commit 9aab0c0
Show file tree

Hide file tree

Showing 7 changed files with 120 additions and 47 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -731,6 +731,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add overview and platform health dashboards to Cloud Foundry module. {pull}21124[21124]
 - Release lambda metricset in aws module as GA. {issue}21251[21251] {pull}21255[21255]
 - Add dashboard for pubsub metricset in googlecloud module. {pull}21326[21326] {issue}17137[17137]
+- Map cloud data filed `cloud.account.id` to azure subscription. {pull}21483[21483] {issue}21381[21381]
 
 *Packetbeat*
 

diff --git a/libbeat/processors/add_cloud_metadata/provider_azure_vm.go b/libbeat/processors/add_cloud_metadata/provider_azure_vm.go
@@ -34,6 +34,9 @@ var azureVMMetadataFetcher = provider{
  azHeaders := map[string]string{"Metadata": "true"}
  azSchema := func(m map[string]interface{}) common.MapStr {
  out, _ := s.Schema{
+ "account": s.Object{
+ "id": c.Str("subscriptionId"),
+ },
  "instance": s.Object{
  "id": c.Str("vmId"),
  "name": c.Str("name"),

diff --git a/libbeat/processors/add_cloud_metadata/provider_azure_vm_test.go b/libbeat/processors/add_cloud_metadata/provider_azure_vm_test.go
@@ -40,7 +40,8 @@ const azInstanceIdentityDocument = `{
  "sku": "14.04.4-LTS",
  "version": "14.04.201605091",
  "vmId": "04ab04c3-63de-4709-a9f9-9ab8c0411d5e",
- "vmSize": "Standard_D3_v2"
+ "vmSize": "Standard_D3_v2",
+ "subscriptionId": "5tfb04c3-63de-4709-a9f9-9ab8c0411d5e"
 }`
 
 func initAzureTestServer() *httptest.Server {
@@ -87,6 +88,9 @@ func TestRetrieveAzureMetadata(t *testing.T) {
  "machine": common.MapStr{
  "type": "Standard_D3_v2",
  },
+ "account": common.MapStr{
+ "id": "5tfb04c3-63de-4709-a9f9-9ab8c0411d5e",
+ },
  "region": "eastus2",
  },
  }

diff --git a/x-pack/metricbeat/module/azure/add_metadata.go b/x-pack/metricbeat/module/azure/add_metadata.go
@@ -35,7 +35,7 @@ func addHostMetadata(event *mb.Event, metricList common.MapStr) {
  }
 }
 
-func addCloudVMMetadata(event *mb.Event, vm VmResource) {
+func addCloudVMMetadata(event *mb.Event, vm VmResource, subscriptionId string) {
  if vm.Name != "" {
  event.RootFields.Put("cloud.instance.name", vm.Name)
  event.RootFields.Put("host.name", vm.Name)
@@ -47,4 +47,7 @@ func addCloudVMMetadata(event *mb.Event, vm VmResource) {
  if vm.Size != "" {
  event.RootFields.Put("cloud.machine.type", vm.Size)
  }
+ if subscriptionId != "" {
+ event.RootFields.Put("cloud.account.id", subscriptionId)
+ }
 }
diff --git a/x-pack/metricbeat/module/azure/compute_vm/_meta/data.json b/x-pack/metricbeat/module/azure/compute_vm/_meta/data.json
@@ -2,62 +2,86 @@
  "@timestamp": "2017-10-12T08:05:34.853Z",
  "azure": {
  "compute_vm": {
+ "disk_read_bytes": {
+ "total": 0
+ },
  "disk_read_operations_per_sec": {
- "avg": 3.3875
+ "avg": 0
+ },
+ "disk_write_bytes": {
+ "total": 2969709.4
  },
  "disk_write_operations_per_sec": {
- "avg": 0.6705
+ "avg": 0.7809999999999999
  },
  "inbound_flows": {
- "avg": 28.4
+ "avg": 10
  },
  "inbound_flows_maximum_creation_rate": {
- "avg": 10.4
+ "avg": 10.6
+ },
+ "network_in": {
+ "total": 1478232
+ },
+ "network_in_total": {
+ "total": 1569665
+ },
+ "network_out": {
+ "total": 793344
+ },
+ "network_out_total": {
+ "total": 1074624
+ },
+ "os_disk_bandwidth_consumed_percentage": {
+ "avg": 0
+ },
+ "os_disk_iops_consumed_percentage": {
+ "avg": 0
  },
  "os_disk_queue_depth": {
- "avg": 0.00125
+ "avg": 0.002
  },
  "os_disk_read_bytes_per_sec": {
- "avg": 602589.1825
+ "avg": 0
  },
  "os_disk_read_operations_per_sec": {
- "avg": 5.28375
+ "avg": 0
  },
  "os_disk_write_bytes_per_sec": {
- "avg": 14137.59375
+ "avg": 9899.025
  },
  "os_disk_write_operations_per_sec": {
- "avg": 1.46875
+ "avg": 1.5619999999999998
  },
  "os_per_disk_qd": {
- "avg": 0.00125
+ "avg": 0.002
  },
  "os_per_disk_read_bytes_per_sec": {
- "avg": 602589.1825
+ "avg": 0
  },
  "os_per_disk_read_operations_per_sec": {
- "avg": 5.28375
+ "avg": 0
  },
  "os_per_disk_write_bytes_per_sec": {
- "avg": 14137.59375
+ "avg": 9899.025
  },
  "os_per_disk_write_operations_per_sec": {
- "avg": 1.46875
+ "avg": 1.5619999999999998
  },
  "outbound_flows": {
- "avg": 28.4
+ "avg": 10
  },
  "outbound_flows_maximum_creation_rate": {
- "avg": 10.4
+ "avg": 10.6
  },
  "per_disk_qd": {
- "avg": 0.0025
+ "avg": 0
  },
  "per_disk_read_bytes_per_sec": {
- "avg": 51985.035
+ "avg": 0
  },
  "per_disk_read_operations_per_sec": {
- "avg": 2.92875
+ "avg": 0
  },
  "per_disk_write_bytes_per_sec": {
  "avg": 0
@@ -66,26 +90,41 @@
  "avg": 0
  },
  "percentage_cpu": {
- "avg": 9.747
+ "avg": 1.8719999999999999
+ },
+ "vm_cached_bandwidth_consumed_percentage": {
+ "avg": 0
+ },
+ "vm_cached_iops_consumed_percentage": {
+ "avg": 0
+ },
+ "vm_uncached_bandwidth_consumed_percentage": {
+ "avg": 0
+ },
+ "vm_uncached_iops_consumed_percentage": {
+ "avg": 0
  }
  },
  "namespace": "Microsoft.Compute/virtualMachines",
  "resource": {
  "group": "obs-infrastructure",
- "id": "/subscriptions/70bd6e64-4b1e-4835-8896-db77b8eef364/resourceGroups/obs-infrastructure/providers/Microsoft.Compute/virtualMachines/obslinux",
- "name": "obslinux",
+ "id": "/subscriptions/fd675b6f-b5e5-426e-ac45-d1f876d0ffa6/resourceGroups/obs-infrastructure/providers/Microsoft.Compute/virtualMachines/testaz",
+ "name": "testaz",
  "type": "Microsoft.Compute/virtualMachines"
  },
- "subscription_id": "70bd6e64-4b1e-4835-8896-db77b8eef364",
+ "subscription_id": "fd675b6f-b5e5-426e-ac45-d1f876d0ffa6",
  "timegrain": "PT5M"
  },
  "cloud": {
+ "account": {
+ "id": "fd675b6f-b5e5-426e-ac45-d1f876d0ffa6"
+ },
  "instance": {
- "id": "d5d9444a-1964-4d23-9c62-5463ecb16fe0",
- "name": "obslinux"
+ "id": "490fe4cf-2b33-4ead-a016-7e614c2f48ad",
+ "name": "testaz"
  },
  "machine": {
- "type": "Basic_A0"
+ "type": "Standard_A1_v2"
  },
  "provider": "azure",
  "region": "westeurope"
@@ -97,10 +136,28 @@
  },
  "host": {
  "cpu": {
- "pct": 0.09747
+ "pct": 0.01872
  },
- "id": "d5d9444a-1964-4d23-9c62-5463ecb16fe0",
- "name": "obslinux"
+ "disk": {
+ "read": {
+ "bytes": 0
+ },
+ "write": {
+ "bytes": 2969709.4
+ }
+ },
+ "id": "490fe4cf-2b33-4ead-a016-7e614c2f48ad",
+ "name": "testaz",
+ "network": {
+ "in": {
+ "bytes": 1569665,
+ "packets": 1478232
+ },
+ "out": {
+ "bytes": 1074624,
+ "packets": 793344
+ }
+ }
  },
  "metricset": {
  "name": "compute_vm",

diff --git a/x-pack/metricbeat/module/azure/compute_vm_scaleset/_meta/data.json b/x-pack/metricbeat/module/azure/compute_vm_scaleset/_meta/data.json
@@ -2,6 +2,12 @@
  "@timestamp": "2017-10-12T08:05:34.853Z",
  "azure": {
  "compute_vm_scaleset": {
+ "cpu_credits_consumed": {
+ "avg": 0.006999999999999999
+ },
+ "cpu_credits_remaining": {
+ "avg": 84.72
+ },
  "os_per_disk_qd": {
  "avg": 0
  },
@@ -12,43 +18,42 @@
  "avg": 0
  },
  "os_per_disk_write_bytes_per_sec": {
- "avg": 1872.1200000000001
+ "avg": 3846.531
  },
  "os_per_disk_write_operations_per_sec": {
- "avg": 0.296
+ "avg": 0.5519999999999999
  }
  },
  "namespace": "Microsoft.Compute/virtualMachineScaleSets",
  "resource": {
- "group": "testgroup",
- "id": "/subscriptions/70bd6e23-e3er3-4835-6785-db77b8eef364/resourceGroups/testgroup/providers/Microsoft.Compute/virtualMachineScaleSets/vmscaleset",
- "name": "vmscaleset",
- "tags": {
- "environment": "staging",
- "role": "allocator"
- },
+ "group": "obs-infrastructure",
+ "id": "/subscriptions/fd675b6f-b5e5-426e-ac45-d1f876d0ffa6/resourceGroups/obs-infrastructure/providers/Microsoft.Compute/virtualMachineScaleSets/obslinuxvmss",
+ "name": "obslinuxvmss",
  "type": "Microsoft.Compute/virtualMachineScaleSets"
  },
- "subscription_id": "70bd6e23-e3er3-4835-6785-db77b8eef364",
+ "subscription_id": "fd675b6f-b5e5-426e-ac45-d1f876d0ffa6",
  "timegrain": "PT5M"
  },
  "cloud": {
+ "account": {
+ "id": "fd675b6f-b5e5-426e-ac45-d1f876d0ffa6"
+ },
  "instance": {
- "name": "vmscaleset"
+ "name": "obslinuxvmss"
  },
  "machine": {
- "type": "Standard_D4s_v3"
+ "type": "Standard_B1ls"
  },
  "provider": "azure",
- "region": "eastus2"
+ "region": "westeurope"
  },
  "event": {
  "dataset": "azure.compute_vm_scaleset",
  "duration": 115000,
  "module": "azure"
  },
  "host": {
- "name": "vmscaleset"
+ "name": "obslinuxvmss"
  },
  "metricset": {
  "name": "compute_vm_scaleset",

diff --git a/x-pack/metricbeat/module/azure/data.go b/x-pack/metricbeat/module/azure/data.go
@@ -82,15 +82,15 @@ func EventsMapping(metrics []Metric, client *Client, report mb.ReporterV2) error
  event, metricList = createEvent(timestamp, defaultMetric, resource, groupDimValues)
  if client.Config.AddCloudMetadata {
  vm = client.GetVMForMetaData(&resource, groupDimValues)
- addCloudVMMetadata(&event, vm)
+ addCloudVMMetadata(&event, vm, resource.Subscription)
  }
  }
  }
  } else {
  event, metricList = createEvent(timestamp, defaultMetric, resource, groupTimeValues)
  if client.Config.AddCloudMetadata {
  vm = client.GetVMForMetaData(&resource, groupTimeValues)
- addCloudVMMetadata(&event, vm)
+ addCloudVMMetadata(&event, vm, resource.Subscription)
  }
  }
  if client.Config.DefaultResourceType == "" {