Cherry-pick #18398 to 7.x: [Metricbeat] Remove checking region/zone a…

…nd service name from googlecloud (#18433) * [Metricbeat] Remove checking region/zone and service name from googlecloud (#18398) * remove validate region/zone * add stackdriver into googlecloud default config (cherry picked from commit 2d7c9d5) * add changelog
elastic · May 12, 2020 · 2d7f1a9 · 2d7f1a9
1 parent 6d0b831
commit 2d7f1a9
Show file tree

Hide file tree

Showing 10 changed files with 103 additions and 27 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -222,6 +222,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix storage metricset to allow config without region/zone. {issue}17623[17623] {pull}17624[17624]
 - Fix overflow on Prometheus rates when new buckets are added on the go. {pull}17753[17753]
 - Add a switch to the driver definition on SQL module to use pretty names {pull}17378[17378]
+- Remove required for region/zone and make stackdriver a metricset in googlecloud. {issue}16785[16785] {pull}18398[18398]
 
 *Packetbeat*
 

diff --git a/metricbeat/docs/modules/googlecloud.asciidoc b/metricbeat/docs/modules/googlecloud.asciidoc
@@ -29,7 +29,8 @@ a partial region name like `us-east` or `us-east*`, which will monitor all regio
 `us-east`: `us-east1` and `us-east4`. If both region and zone are configured,
 only region will be used.
 Please see https://cloud.google.com/compute/docs/regions-zones#available[GCP regions]
-for regions that are available in GCP.
+for regions that are available in GCP. If both `region` and `zone` are not
+specified, metrics will be collected from all regions/zones.
 
 * *project_id*: A single string with your GCP Project ID
 
@@ -236,6 +237,23 @@ metricbeat.modules:
   credentials_file_path: "your JSON credentials file path"
   exclude_labels: false
   period: 5m
+
+- module: googlecloud
+  metricsets:
+    - stackdriver
+  project_id: "your project id"
+  credentials_file_path: "your JSON credentials file path"
+  exclude_labels: false
+  period: 1m
+  stackdriver:
+    service: compute
+    metrics:
+      - aligner: ALIGN_NONE
+        metric_types:
+          - "compute.googleapis.com/instance/cpu/reserved_cores"
+          - "compute.googleapis.com/instance/cpu/usage_time"
+          - "compute.googleapis.com/instance/cpu/utilization"
+          - "compute.googleapis.com/instance/uptime"
 ----
 
 [float]

diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml
@@ -524,6 +524,23 @@ metricbeat.modules:
   exclude_labels: false
   period: 5m
 
+- module: googlecloud
+  metricsets:
+    - stackdriver
+  project_id: "your project id"
+  credentials_file_path: "your JSON credentials file path"
+  exclude_labels: false
+  period: 1m
+  stackdriver:
+    service: compute
+    metrics:
+      - aligner: ALIGN_NONE
+        metric_types:
+          - "compute.googleapis.com/instance/cpu/reserved_cores"
+          - "compute.googleapis.com/instance/cpu/usage_time"
+          - "compute.googleapis.com/instance/cpu/utilization"
+          - "compute.googleapis.com/instance/uptime"
+
 #------------------------------- Graphite Module -------------------------------
 - module: graphite
   metricsets: ["server"]

diff --git a/x-pack/metricbeat/module/googlecloud/_meta/config.yml b/x-pack/metricbeat/module/googlecloud/_meta/config.yml
@@ -25,3 +25,20 @@
   credentials_file_path: "your JSON credentials file path"
   exclude_labels: false
   period: 5m
+
+- module: googlecloud
+  metricsets:
+    - stackdriver
+  project_id: "your project id"
+  credentials_file_path: "your JSON credentials file path"
+  exclude_labels: false
+  period: 1m
+  stackdriver:
+    service: compute
+    metrics:
+      - aligner: ALIGN_NONE
+        metric_types:
+          - "compute.googleapis.com/instance/cpu/reserved_cores"
+          - "compute.googleapis.com/instance/cpu/usage_time"
+          - "compute.googleapis.com/instance/cpu/utilization"
+          - "compute.googleapis.com/instance/uptime"
diff --git a/x-pack/metricbeat/module/googlecloud/_meta/docs.asciidoc b/x-pack/metricbeat/module/googlecloud/_meta/docs.asciidoc
@@ -19,7 +19,8 @@ a partial region name like `us-east` or `us-east*`, which will monitor all regio
 `us-east`: `us-east1` and `us-east4`. If both region and zone are configured,
 only region will be used.
 Please see https://cloud.google.com/compute/docs/regions-zones#available[GCP regions]
-for regions that are available in GCP.
+for regions that are available in GCP. If both `region` and `zone` are not
+specified, metrics will be collected from all regions/zones.
 
 * *project_id*: A single string with your GCP Project ID
 

diff --git a/x-pack/metricbeat/module/googlecloud/stackdriver/metadata_services.go b/x-pack/metricbeat/module/googlecloud/stackdriver/metadata_services.go
@@ -5,8 +5,6 @@
 package stackdriver
 
 import (
-	"github.com/pkg/errors"
-
 	"github.com/elastic/beats/v7/x-pack/metricbeat/module/googlecloud"
 	"github.com/elastic/beats/v7/x-pack/metricbeat/module/googlecloud/stackdriver/compute"
 )
@@ -17,9 +15,7 @@ func NewMetadataServiceForConfig(c config) (googlecloud.MetadataService, error)
 	switch c.ServiceName {
 	case googlecloud.ServiceCompute:
 		return compute.NewMetadataService(c.ProjectID, c.Zone, c.Region, c.opt...)
-	case googlecloud.ServicePubsub, googlecloud.ServiceLoadBalancing, googlecloud.ServiceStorage:
-		return nil, nil
 	default:
-		return nil, errors.Errorf("service '%s' not supported", c.ServiceName)
+		return nil, nil
 	}
 }
diff --git a/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester.go b/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester.go
@@ -105,6 +105,9 @@ var serviceRegexp = regexp.MustCompile(`^(?P<service>[a-z]+)\.googleapis.com.*`)
 // if they have a region specified.
 func (r *stackdriverMetricsRequester) getFilterForMetric(m string) (f string) {
 	f = fmt.Sprintf(`metric.type="%s"`, m)
+	if r.config.Zone == "" && r.config.Region == "" {
+		return
+	}
 
 	service := serviceRegexp.ReplaceAllString(m, "${service}")
 

diff --git a/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester_test.go b/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester_test.go
@@ -88,6 +88,12 @@ func TestGetFilterForMetric(t *testing.T) {
 			stackdriverMetricsRequester{config: config{Zone: "us-west1-*"}, logger: logger},
 			"metric.type=\"compute.googleapis.com/instance/uptime\" AND resource.labels.zone = starts_with(\"us-west1-\")",
 		},
+		{
+			"compute service with no region/zone in config",
+			"compute.googleapis.com/firewall/dropped_bytes_count",
+			stackdriverMetricsRequester{config: config{}},
+			"metric.type=\"compute.googleapis.com/firewall/dropped_bytes_count\"",
+		},
 	}
 
 	for _, c := range cases {

diff --git a/x-pack/metricbeat/module/googlecloud/stackdriver/metricset.go b/x-pack/metricbeat/module/googlecloud/stackdriver/metricset.go
@@ -106,7 +106,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
 		return nil, errors.Wrap(err, "error creating Stackdriver client")
 	}
 
-	m.metricsMeta, err = metricDescriptor(ctx, client, m.config.ProjectID, m.stackDriverConfig)
+	m.metricsMeta, err = m.metricDescriptor(ctx, client)
 	if err != nil {
 		return nil, errors.Wrap(err, "error calling metricDescriptor function")
 	}
@@ -188,19 +188,6 @@ func validatePeriodForGCP(d time.Duration) (err error) {
 	return nil
 }
 
-// Validate googlecloud module config
-func (c *config) Validate() error {
-	// storage metricset does not require region or zone config parameter.
-	if c.ServiceName == "storage" {
-		return nil
-	}
-
-	if c.Region == "" && c.Zone == "" {
-		return errors.New("region and zone in Google Cloud config file cannot both be empty")
-	}
-	return nil
-}
-
 // Validate stackdriver related config
 func (mc *stackDriverConfig) Validate() error {
 	gcpAlignerNames := make([]string, 0)
@@ -218,13 +205,13 @@ func (mc *stackDriverConfig) Validate() error {
 
 // metricDescriptor calls ListMetricDescriptorsRequest API to get metric metadata
 // (sample period and ingest delay) of each given metric type
-func metricDescriptor(ctx context.Context, client *monitoring.MetricClient, projectID string, stackDriverConfigs []stackDriverConfig) (map[string]metricMeta, error) {
+func (m *MetricSet) metricDescriptor(ctx context.Context, client *monitoring.MetricClient) (map[string]metricMeta, error) {
 	metricsWithMeta := make(map[string]metricMeta, 0)
 
-	for _, sdc := range stackDriverConfigs {
+	for _, sdc := range m.stackDriverConfig {
 		for _, mt := range sdc.MetricTypes {
 			req := &monitoringpb.ListMetricDescriptorsRequest{
-				Name:   "projects/" + projectID,
+				Name:   "projects/" + m.config.ProjectID,
 				Filter: fmt.Sprintf(`metric.type = "%s"`, mt),
 			}
 
@@ -234,10 +221,23 @@ func metricDescriptor(ctx context.Context, client *monitoring.MetricClient, proj
 				return metricsWithMeta, errors.Errorf("Could not make ListMetricDescriptors request: %s: %v", mt, err)
 			}
 
-			metricsWithMeta[mt] = metricMeta{
-				samplePeriod: time.Duration(out.Metadata.SamplePeriod.Seconds) * time.Second,
-				ingestDelay:  time.Duration(out.Metadata.IngestDelay.Seconds) * time.Second,
+			// Set samplePeriod default to 60 seconds and ingestDelay default to 0.
+			meta := metricMeta{
+				samplePeriod: 60 * time.Second,
+				ingestDelay:  0 * time.Second,
 			}
+
+			if out.Metadata.SamplePeriod != nil {
+				m.Logger().Debugf("For metric type %s: sample period = %s", mt, out.Metadata.SamplePeriod)
+				meta.samplePeriod = time.Duration(out.Metadata.SamplePeriod.Seconds) * time.Second
+			}
+
+			if out.Metadata.IngestDelay != nil {
+				m.Logger().Debugf("For metric type %s: ingest delay = %s", mt, out.Metadata.IngestDelay)
+				meta.ingestDelay = time.Duration(out.Metadata.IngestDelay.Seconds) * time.Second
+			}
+
+			metricsWithMeta[mt] = meta
 		}
 	}
 

diff --git a/x-pack/metricbeat/modules.d/googlecloud.yml.disabled b/x-pack/metricbeat/modules.d/googlecloud.yml.disabled
@@ -28,3 +28,20 @@
   credentials_file_path: "your JSON credentials file path"
   exclude_labels: false
   period: 5m
+
+- module: googlecloud
+  metricsets:
+    - stackdriver
+  project_id: "your project id"
+  credentials_file_path: "your JSON credentials file path"
+  exclude_labels: false
+  period: 1m
+  stackdriver:
+    service: compute
+    metrics:
+      - aligner: ALIGN_NONE
+        metric_types:
+          - "compute.googleapis.com/instance/cpu/reserved_cores"
+          - "compute.googleapis.com/instance/cpu/usage_time"
+          - "compute.googleapis.com/instance/cpu/utilization"
+          - "compute.googleapis.com/instance/uptime"