Add cloud.service.name to add_cloud_metadata (#24993) (#25008)

* Add cloud.service.name to add_cloud_metadata * update changelog (cherry picked from commit 5bde9d0) Co-authored-by: Andrew Stucki <andrew.stucki@elastic.co>
elastic · Apr 9, 2021 · ddf6ccd · ddf6ccd
1 parent 663448b
commit ddf6ccd
Show file tree

Hide file tree

Showing 15 changed files with 63 additions and 4 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -355,6 +355,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add replace_fields config option in add_host_metadata for replacing host fields. {pull}20490[20490] {issue}20464[20464]
 - Add option to select the type of index template to load: legacy, component, index. {pull}21212[21212]
 - Add `wineventlog` schema to `decode_xml` processor. {issue}23910[23910] {pull}24726[24726]
+- Add new ECS 1.9 field `cloud.service.name` to `add_cloud_metadata` processor. {pull}24993[24993]
 
 *Auditbeat*
 

diff --git a/libbeat/processors/add_cloud_metadata/provider_alibaba_cloud.go b/libbeat/processors/add_cloud_metadata/provider_alibaba_cloud.go
@@ -33,6 +33,9 @@ var alibabaCloudMetadataFetcher = provider{
  ecsMetadataZoneURI := "/latest/meta-data/zone-id"
 
  ecsSchema := func(m map[string]interface{}) common.MapStr {
+ m["service"] = common.MapStr{
+ "name": "ECS",
+ }
  return common.MapStr(m)
  }
 

diff --git a/libbeat/processors/add_cloud_metadata/provider_alibaba_cloud_test.go b/libbeat/processors/add_cloud_metadata/provider_alibaba_cloud_test.go
@@ -81,6 +81,9 @@ func TestRetrieveAlibabaCloudMetadata(t *testing.T) {
  },
  "region": "cn-shenzhen",
  "availability_zone": "cn-shenzhen-a",
+ "service": common.MapStr{
+ "name": "ECS",
+ },
  },
  }
  assert.Equal(t, expected, actual.Fields)

diff --git a/libbeat/processors/add_cloud_metadata/provider_aws_ec2.go b/libbeat/processors/add_cloud_metadata/provider_aws_ec2.go
@@ -33,13 +33,17 @@ var ec2MetadataFetcher = provider{
 
  Create: func(_ string, config *common.Config) (metadataFetcher, error) {
  ec2Schema := func(m map[string]interface{}) common.MapStr {
+ m["serviceName"] = "EC2"
  out, _ := s.Schema{
  "instance": s.Object{"id": c.Str("instanceId")},
  "machine": s.Object{"type": c.Str("instanceType")},
  "region": c.Str("region"),
  "availability_zone": c.Str("availabilityZone"),
- "account": s.Object{"id": c.Str("accountId")},
- "image": s.Object{"id": c.Str("imageId")},
+ "service": s.Object{
+ "name": c.Str("serviceName"),
+ },
+ "account": s.Object{"id": c.Str("accountId")},
+ "image": s.Object{"id": c.Str("imageId")},
  }.Apply(m)
  return out
  }

diff --git a/libbeat/processors/add_cloud_metadata/provider_aws_ec2_test.go b/libbeat/processors/add_cloud_metadata/provider_aws_ec2_test.go
@@ -72,7 +72,7 @@ func TestRetrieveAWSMetadataEC2(t *testing.T) {
  "imageId" : "%s",
  "instanceType" : "%s",
  "devpayProductCodes" : null,
- "privateIp" : "10.0.0.1", 
+ "privateIp" : "10.0.0.1",
  "version" : "2010-08-31",
  "billingProducts" : null,
  "pendingTime" : "2016-09-20T15:43:02Z",
@@ -114,6 +114,9 @@ func TestRetrieveAWSMetadataEC2(t *testing.T) {
  "image": common.MapStr{"id": imageIDDoc1},
  "region": regionDoc1,
  "availability_zone": availabilityZoneDoc1,
+ "service": common.MapStr{
+ "name": "EC2",
+ },
  },
  },
  },
@@ -154,6 +157,9 @@ func TestRetrieveAWSMetadataEC2(t *testing.T) {
  "image": common.MapStr{"id": imageIDDoc1},
  "region": regionDoc1,
  "availability_zone": availabilityZoneDoc1,
+ "service": common.MapStr{
+ "name": "EC2",
+ },
  },
  },
  },
@@ -172,6 +178,9 @@ func TestRetrieveAWSMetadataEC2(t *testing.T) {
  "image": common.MapStr{"id": imageIDDoc1},
  "region": regionDoc1,
  "availability_zone": availabilityZoneDoc1,
+ "service": common.MapStr{
+ "name": "EC2",
+ },
  },
  },
  },
@@ -194,6 +203,9 @@ func TestRetrieveAWSMetadataEC2(t *testing.T) {
  "image": common.MapStr{"id": imageIDDoc1},
  "region": regionDoc1,
  "availability_zone": availabilityZoneDoc1,
+ "service": common.MapStr{
+ "name": "EC2",
+ },
  },
  },
  },
@@ -215,6 +227,9 @@ func TestRetrieveAWSMetadataEC2(t *testing.T) {
  "image": common.MapStr{"id": imageIDDoc1},
  "region": regionDoc1,
  "availability_zone": availabilityZoneDoc1,
+ "service": common.MapStr{
+ "name": "EC2",
+ },
  },
  },
  },

diff --git a/libbeat/processors/add_cloud_metadata/provider_azure_vm.go b/libbeat/processors/add_cloud_metadata/provider_azure_vm.go
@@ -33,6 +33,7 @@ var azureVMMetadataFetcher = provider{
  azMetadataURI := "/metadata/instance/compute?api-version=2017-04-02"
  azHeaders := map[string]string{"Metadata": "true"}
  azSchema := func(m map[string]interface{}) common.MapStr {
+ m["serviceName"] = "Virtual Machines"
  out, _ := s.Schema{
  "account": s.Object{
  "id": c.Str("subscriptionId"),
@@ -44,6 +45,9 @@ var azureVMMetadataFetcher = provider{
  "machine": s.Object{
  "type": c.Str("vmSize"),
  },
+ "service": s.Object{
+ "name": c.Str("serviceName"),
+ },
  "region": c.Str("location"),
  }.Apply(m)
  return out

diff --git a/libbeat/processors/add_cloud_metadata/provider_azure_vm_test.go b/libbeat/processors/add_cloud_metadata/provider_azure_vm_test.go
@@ -91,6 +91,9 @@ func TestRetrieveAzureMetadata(t *testing.T) {
  "account": common.MapStr{
  "id": "5tfb04c3-63de-4709-a9f9-9ab8c0411d5e",
  },
+ "service": common.MapStr{
+ "name": "Virtual Machines",
+ },
  "region": "eastus2",
  },
  }

diff --git a/libbeat/processors/add_cloud_metadata/provider_digital_ocean.go b/libbeat/processors/add_cloud_metadata/provider_digital_ocean.go
@@ -31,11 +31,15 @@ var doMetadataFetcher = provider{
 
  Create: func(provider string, config *common.Config) (metadataFetcher, error) {
  doSchema := func(m map[string]interface{}) common.MapStr {
+ m["serviceName"] = "Droplets"
  out, _ := s.Schema{
  "instance": s.Object{
  "id": c.StrFromNum("droplet_id"),
  },
  "region": c.Str("region"),
+ "service": s.Object{
+ "name": c.Str("serviceName"),
+ },
  }.Apply(m)
  return out
  }

diff --git a/libbeat/processors/add_cloud_metadata/provider_digital_ocean_test.go b/libbeat/processors/add_cloud_metadata/provider_digital_ocean_test.go
@@ -117,6 +117,9 @@ func TestRetrieveDigitalOceanMetadata(t *testing.T) {
  "instance": common.MapStr{
  "id": "1111111",
  },
+ "service": common.MapStr{
+ "name": "Droplets",
+ },
  "region": "nyc3",
  },
  }

diff --git a/libbeat/processors/add_cloud_metadata/provider_google_gce.go b/libbeat/processors/add_cloud_metadata/provider_google_gce.go
@@ -35,7 +35,11 @@ var gceMetadataFetcher = provider{
  gceMetadataURI := "/computeMetadata/v1/?recursive=true&alt=json"
  gceHeaders := map[string]string{"Metadata-Flavor": "Google"}
  gceSchema := func(m map[string]interface{}) common.MapStr {
- out := common.MapStr{}
+ out := common.MapStr{
+ "service": common.MapStr{
+ "name": "GCE",
+ },
+ }
 
  trimLeadingPath := func(key string) {
  v, err := out.GetValue(key)

diff --git a/libbeat/processors/add_cloud_metadata/provider_google_gce_test.go b/libbeat/processors/add_cloud_metadata/provider_google_gce_test.go
@@ -167,6 +167,9 @@ func TestRetrieveGCEMetadata(t *testing.T) {
  "project": common.MapStr{
  "id": "test-dev",
  },
+ "service": common.MapStr{
+ "name": "GCE",
+ },
  },
  }
  assert.Equal(t, expected, actual.Fields)

diff --git a/libbeat/processors/add_cloud_metadata/provider_openstack_nova.go b/libbeat/processors/add_cloud_metadata/provider_openstack_nova.go
@@ -46,6 +46,9 @@ var openstackNovaSSLMetadataFetcher = provider{
 func buildOpenstackNovaCreate(scheme string) func(provider string, c *common.Config) (metadataFetcher, error) {
  return func(provider string, c *common.Config) (metadataFetcher, error) {
  osSchema := func(m map[string]interface{}) common.MapStr {
+ m["service"] = common.MapStr{
+ "name": "Nova",
+ }
  return common.MapStr(m)
  }
 

diff --git a/libbeat/processors/add_cloud_metadata/provider_openstack_nova_test.go b/libbeat/processors/add_cloud_metadata/provider_openstack_nova_test.go
@@ -109,6 +109,9 @@ func assertOpenstackNova(t *testing.T, config *common.Config) {
  "type": "m1.xlarge",
  },
  "availability_zone": "az-test-2",
+ "service": common.MapStr{
+ "name": "Nova",
+ },
  },
  }
  assert.Equal(t, expected, actual.Fields)

diff --git a/libbeat/processors/add_cloud_metadata/provider_tencent_cloud.go b/libbeat/processors/add_cloud_metadata/provider_tencent_cloud.go
@@ -33,6 +33,9 @@ var qcloudMetadataFetcher = provider{
  qcloudMetadataZoneURI := "/meta-data/placement/zone"
 
  qcloudSchema := func(m map[string]interface{}) common.MapStr {
+ m["service"] = common.MapStr{
+ "name": "CVM",
+ }
  return common.MapStr(m)
  }
 

diff --git a/libbeat/processors/add_cloud_metadata/provider_tencent_cloud_test.go b/libbeat/processors/add_cloud_metadata/provider_tencent_cloud_test.go
@@ -81,6 +81,9 @@ func TestRetrieveQCloudMetadata(t *testing.T) {
  },
  "region": "china-south-gz",
  "availability_zone": "gz-azone2",
+ "service": common.MapStr{
+ "name": "CVM",
+ },
  },
  }
  assert.Equal(t, expected, actual.Fields)