diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 896b4b878c1..059a98225a8 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -479,6 +479,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Collect new `bulk` indexing metrics from Elasticsearch when `xpack.enabled:true` is set. {issue} {pull}17992[17992] - Remove requirement to connect as sysdba in Oracle module {issue}15846[15846] {pull}18182[18182] - Update MSSQL module to fix some SSPI authentication and add brackets to USE statements {pull}17862[17862]] +- Remove required for region/zone and make stackdriver a metricset in googlecloud. {issue}16785[16785] {pull}18398[18398] *Packetbeat* diff --git a/metricbeat/docs/modules/googlecloud.asciidoc b/metricbeat/docs/modules/googlecloud.asciidoc index 5eaf4d8b575..f6617b81101 100644 --- a/metricbeat/docs/modules/googlecloud.asciidoc +++ b/metricbeat/docs/modules/googlecloud.asciidoc @@ -29,7 +29,8 @@ a partial region name like `us-east` or `us-east*`, which will monitor all regio `us-east`: `us-east1` and `us-east4`. If both region and zone are configured, only region will be used. Please see https://cloud.google.com/compute/docs/regions-zones#available[GCP regions] -for regions that are available in GCP. +for regions that are available in GCP. If both `region` and `zone` are not +specified, metrics will be collected from all regions/zones. * *project_id*: A single string with your GCP Project ID @@ -236,6 +237,23 @@ metricbeat.modules: credentials_file_path: "your JSON credentials file path" exclude_labels: false period: 5m + +- module: googlecloud + metricsets: + - stackdriver + project_id: "your project id" + credentials_file_path: "your JSON credentials file path" + exclude_labels: false + period: 1m + stackdriver: + service: compute + metrics: + - aligner: ALIGN_NONE + metric_types: + - "compute.googleapis.com/instance/cpu/reserved_cores" + - "compute.googleapis.com/instance/cpu/usage_time" + - "compute.googleapis.com/instance/cpu/utilization" + - "compute.googleapis.com/instance/uptime" ---- [float] diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 67bd927aa36..535ec3b9eff 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -524,6 +524,23 @@ metricbeat.modules: exclude_labels: false period: 5m +- module: googlecloud + metricsets: + - stackdriver + project_id: "your project id" + credentials_file_path: "your JSON credentials file path" + exclude_labels: false + period: 1m + stackdriver: + service: compute + metrics: + - aligner: ALIGN_NONE + metric_types: + - "compute.googleapis.com/instance/cpu/reserved_cores" + - "compute.googleapis.com/instance/cpu/usage_time" + - "compute.googleapis.com/instance/cpu/utilization" + - "compute.googleapis.com/instance/uptime" + #------------------------------- Graphite Module ------------------------------- - module: graphite metricsets: ["server"] diff --git a/x-pack/metricbeat/module/googlecloud/_meta/config.yml b/x-pack/metricbeat/module/googlecloud/_meta/config.yml index e316785b894..231bb968ab1 100644 --- a/x-pack/metricbeat/module/googlecloud/_meta/config.yml +++ b/x-pack/metricbeat/module/googlecloud/_meta/config.yml @@ -25,3 +25,20 @@ credentials_file_path: "your JSON credentials file path" exclude_labels: false period: 5m + +- module: googlecloud + metricsets: + - stackdriver + project_id: "your project id" + credentials_file_path: "your JSON credentials file path" + exclude_labels: false + period: 1m + stackdriver: + service: compute + metrics: + - aligner: ALIGN_NONE + metric_types: + - "compute.googleapis.com/instance/cpu/reserved_cores" + - "compute.googleapis.com/instance/cpu/usage_time" + - "compute.googleapis.com/instance/cpu/utilization" + - "compute.googleapis.com/instance/uptime" diff --git a/x-pack/metricbeat/module/googlecloud/_meta/docs.asciidoc b/x-pack/metricbeat/module/googlecloud/_meta/docs.asciidoc index 692db78bb55..c1d05a7e5f5 100644 --- a/x-pack/metricbeat/module/googlecloud/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/googlecloud/_meta/docs.asciidoc @@ -19,7 +19,8 @@ a partial region name like `us-east` or `us-east*`, which will monitor all regio `us-east`: `us-east1` and `us-east4`. If both region and zone are configured, only region will be used. Please see https://cloud.google.com/compute/docs/regions-zones#available[GCP regions] -for regions that are available in GCP. +for regions that are available in GCP. If both `region` and `zone` are not +specified, metrics will be collected from all regions/zones. * *project_id*: A single string with your GCP Project ID diff --git a/x-pack/metricbeat/module/googlecloud/stackdriver/metadata_services.go b/x-pack/metricbeat/module/googlecloud/stackdriver/metadata_services.go index d628ee4b09d..bee0a2d834a 100644 --- a/x-pack/metricbeat/module/googlecloud/stackdriver/metadata_services.go +++ b/x-pack/metricbeat/module/googlecloud/stackdriver/metadata_services.go @@ -5,8 +5,6 @@ package stackdriver import ( - "github.com/pkg/errors" - "github.com/elastic/beats/v7/x-pack/metricbeat/module/googlecloud" "github.com/elastic/beats/v7/x-pack/metricbeat/module/googlecloud/stackdriver/compute" ) @@ -17,9 +15,7 @@ func NewMetadataServiceForConfig(c config) (googlecloud.MetadataService, error) switch c.ServiceName { case googlecloud.ServiceCompute: return compute.NewMetadataService(c.ProjectID, c.Zone, c.Region, c.opt...) - case googlecloud.ServicePubsub, googlecloud.ServiceLoadBalancing, googlecloud.ServiceStorage: - return nil, nil default: - return nil, errors.Errorf("service '%s' not supported", c.ServiceName) + return nil, nil } } diff --git a/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester.go b/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester.go index 11b04e5a1b2..a0560769dbe 100644 --- a/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester.go +++ b/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester.go @@ -105,6 +105,9 @@ var serviceRegexp = regexp.MustCompile(`^(?P[a-z]+)\.googleapis.com.*`) // if they have a region specified. func (r *stackdriverMetricsRequester) getFilterForMetric(m string) (f string) { f = fmt.Sprintf(`metric.type="%s"`, m) + if r.config.Zone == "" && r.config.Region == "" { + return + } service := serviceRegexp.ReplaceAllString(m, "${service}") diff --git a/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester_test.go b/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester_test.go index f7aff666c0f..42a7fd01d0a 100644 --- a/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester_test.go +++ b/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester_test.go @@ -88,6 +88,12 @@ func TestGetFilterForMetric(t *testing.T) { stackdriverMetricsRequester{config: config{Zone: "us-west1-*"}, logger: logger}, "metric.type=\"compute.googleapis.com/instance/uptime\" AND resource.labels.zone = starts_with(\"us-west1-\")", }, + { + "compute service with no region/zone in config", + "compute.googleapis.com/firewall/dropped_bytes_count", + stackdriverMetricsRequester{config: config{}}, + "metric.type=\"compute.googleapis.com/firewall/dropped_bytes_count\"", + }, } for _, c := range cases { diff --git a/x-pack/metricbeat/module/googlecloud/stackdriver/metricset.go b/x-pack/metricbeat/module/googlecloud/stackdriver/metricset.go index 81fa98751aa..e31aa96202c 100644 --- a/x-pack/metricbeat/module/googlecloud/stackdriver/metricset.go +++ b/x-pack/metricbeat/module/googlecloud/stackdriver/metricset.go @@ -106,7 +106,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { return nil, errors.Wrap(err, "error creating Stackdriver client") } - m.metricsMeta, err = metricDescriptor(ctx, client, m.config.ProjectID, m.stackDriverConfig) + m.metricsMeta, err = m.metricDescriptor(ctx, client) if err != nil { return nil, errors.Wrap(err, "error calling metricDescriptor function") } @@ -188,19 +188,6 @@ func validatePeriodForGCP(d time.Duration) (err error) { return nil } -// Validate googlecloud module config -func (c *config) Validate() error { - // storage metricset does not require region or zone config parameter. - if c.ServiceName == "storage" { - return nil - } - - if c.Region == "" && c.Zone == "" { - return errors.New("region and zone in Google Cloud config file cannot both be empty") - } - return nil -} - // Validate stackdriver related config func (mc *stackDriverConfig) Validate() error { gcpAlignerNames := make([]string, 0) @@ -218,13 +205,13 @@ func (mc *stackDriverConfig) Validate() error { // metricDescriptor calls ListMetricDescriptorsRequest API to get metric metadata // (sample period and ingest delay) of each given metric type -func metricDescriptor(ctx context.Context, client *monitoring.MetricClient, projectID string, stackDriverConfigs []stackDriverConfig) (map[string]metricMeta, error) { +func (m *MetricSet) metricDescriptor(ctx context.Context, client *monitoring.MetricClient) (map[string]metricMeta, error) { metricsWithMeta := make(map[string]metricMeta, 0) - for _, sdc := range stackDriverConfigs { + for _, sdc := range m.stackDriverConfig { for _, mt := range sdc.MetricTypes { req := &monitoringpb.ListMetricDescriptorsRequest{ - Name: "projects/" + projectID, + Name: "projects/" + m.config.ProjectID, Filter: fmt.Sprintf(`metric.type = "%s"`, mt), } @@ -234,10 +221,23 @@ func metricDescriptor(ctx context.Context, client *monitoring.MetricClient, proj return metricsWithMeta, errors.Errorf("Could not make ListMetricDescriptors request: %s: %v", mt, err) } - metricsWithMeta[mt] = metricMeta{ - samplePeriod: time.Duration(out.Metadata.SamplePeriod.Seconds) * time.Second, - ingestDelay: time.Duration(out.Metadata.IngestDelay.Seconds) * time.Second, + // Set samplePeriod default to 60 seconds and ingestDelay default to 0. + meta := metricMeta{ + samplePeriod: 60 * time.Second, + ingestDelay: 0 * time.Second, } + + if out.Metadata.SamplePeriod != nil { + m.Logger().Debugf("For metric type %s: sample period = %s", mt, out.Metadata.SamplePeriod) + meta.samplePeriod = time.Duration(out.Metadata.SamplePeriod.Seconds) * time.Second + } + + if out.Metadata.IngestDelay != nil { + m.Logger().Debugf("For metric type %s: ingest delay = %s", mt, out.Metadata.IngestDelay) + meta.ingestDelay = time.Duration(out.Metadata.IngestDelay.Seconds) * time.Second + } + + metricsWithMeta[mt] = meta } } diff --git a/x-pack/metricbeat/modules.d/googlecloud.yml.disabled b/x-pack/metricbeat/modules.d/googlecloud.yml.disabled index 5f1e9fb8a35..421b609c0e7 100644 --- a/x-pack/metricbeat/modules.d/googlecloud.yml.disabled +++ b/x-pack/metricbeat/modules.d/googlecloud.yml.disabled @@ -28,3 +28,20 @@ credentials_file_path: "your JSON credentials file path" exclude_labels: false period: 5m + +- module: googlecloud + metricsets: + - stackdriver + project_id: "your project id" + credentials_file_path: "your JSON credentials file path" + exclude_labels: false + period: 1m + stackdriver: + service: compute + metrics: + - aligner: ALIGN_NONE + metric_types: + - "compute.googleapis.com/instance/cpu/reserved_cores" + - "compute.googleapis.com/instance/cpu/usage_time" + - "compute.googleapis.com/instance/cpu/utilization" + - "compute.googleapis.com/instance/uptime"