From d5266030a63da42b7739af0f764dbe0902f472cd Mon Sep 17 00:00:00 2001 From: Mariana Date: Tue, 31 Mar 2020 15:57:40 +0200 Subject: [PATCH 01/15] config --- metricbeat/module/windows/perfmon/config.go | 114 +++++++++++++++++++ metricbeat/module/windows/perfmon/perfmon.go | 34 +----- 2 files changed, 117 insertions(+), 31 deletions(-) create mode 100644 metricbeat/module/windows/perfmon/config.go diff --git a/metricbeat/module/windows/perfmon/config.go b/metricbeat/module/windows/perfmon/config.go new file mode 100644 index 000000000000..f9ccdaca6de6 --- /dev/null +++ b/metricbeat/module/windows/perfmon/config.go @@ -0,0 +1,114 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build windows + +package perfmon + +import ( + "github.com/pkg/errors" + "strings" +) + +// Config for the windows perfmon metricset. +type Config struct { + IgnoreNECounters bool `config:"perfmon.ignore_non_existent_counters"` + GroupMeasurements bool `config:"perfmon.group_measurements_by_instance"` + CounterConfig []CounterConfig `config:"perfmon.counters" validate:"required"` + QueryConfig []QueryConfig `config:"perfmon.queries" validate:"required"` + GroupAllCountersTo string `config:"perfmon.group_all_counter"` + MetricFormat bool +} + +// CounterConfig for perfmon counters. +type CounterConfig struct { + InstanceLabel string `config:"instance_label"` + InstanceName string `config:"instance_name"` + MeasurementLabel string `config:"measurement_label" validate:"required"` + Query string `config:"query" validate:"required"` + Format string `config:"format"` +} + +// QueryConfig for perfmon queries. This will be used as the new configuration format +type QueryConfig struct { + Name string `config:"object" validate:"required"` + Field string `config:"field"` + Instance string `config:"instance"` + Counters []QueryConfigCounter `config:"counters" validate:"required"` +} + +// QueryConfigCounter for perfmon queries. This will be used as the new configuration format +type QueryConfigCounter struct { + Name string `config:"counters" validate:"required"` + Field string `config:"field"` + Format string `config:"format"` +} + +func (conf *Config) ValidateConfig() error { + if len(conf.CounterConfig) == 0 && len(conf.QueryConfig) == 0 { + return errors.New("no perfmon counters or queries have been configured") + } + if len(conf.QueryConfig) > 0 { + conf.MetricFormat = true + conf.CounterConfig = []CounterConfig{} + for _, query := range conf.QueryConfig { + for _, counter := range query.Counters { + var counterConf = CounterConfig{ + InstanceLabel: "instance", + InstanceName: query.Instance, + MeasurementLabel: counter.Field, + Query: query.Name, + Format: counter.Format, + } + conf.CounterConfig = append(conf.CounterConfig, counterConf) + } + + } + + } + + // add default format in the config + for _, value := range conf.CounterConfig { + form := strings.ToLower(value.Format) + switch form { + case "", "float": + value.Format = "float" + case "long", "large": + default: + return errors.Errorf("initialization failed: format '%s' "+ + "for counter '%s' is invalid (must be float, large or long)", + value.Format, value.InstanceLabel) + } + } + for _, value := range conf.QueryConfig { + for _, path := range value.Counters { + form := strings.ToLower(path.Format) + switch form { + case "", "float": + path.Format = "float" + case "long", "large": + default: + return errors.Errorf("initialization failed: format '%s' "+ + "for counter '%s' is invalid (must be float, large or long)", + path.Format, value.Name) + } + } + + } + + return nil +} diff --git a/metricbeat/module/windows/perfmon/perfmon.go b/metricbeat/module/windows/perfmon/perfmon.go index 76f8833b3f04..a469293c36ea 100644 --- a/metricbeat/module/windows/perfmon/perfmon.go +++ b/metricbeat/module/windows/perfmon/perfmon.go @@ -20,8 +20,6 @@ package perfmon import ( - "strings" - "github.com/elastic/beats/v7/metricbeat/mb/parse" "github.com/pkg/errors" @@ -31,23 +29,6 @@ import ( "github.com/elastic/beats/v7/metricbeat/mb" ) -// CounterConfig for perfmon counters. -type CounterConfig struct { - InstanceLabel string `config:"instance_label"` - InstanceName string `config:"instance_name"` - MeasurementLabel string `config:"measurement_label" validate:"required"` - Query string `config:"query" validate:"required"` - Format string `config:"format"` -} - -// Config for the windows perfmon metricset. -type Config struct { - IgnoreNECounters bool `config:"perfmon.ignore_non_existent_counters"` - GroupMeasurements bool `config:"perfmon.group_measurements_by_instance"` - CounterConfig []CounterConfig `config:"perfmon.counters" validate:"required"` - GroupAllCountersTo string `config:"perfmon.group_all_counter"` -} - const metricsetName = "perfmon" func init() { @@ -68,19 +49,10 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { if err := base.Module().UnpackConfig(&config); err != nil { return nil, err } - for _, value := range config.CounterConfig { - form := strings.ToLower(value.Format) - switch form { - case "", "float": - value.Format = "float" - case "long", "large": - default: - return nil, errors.Errorf("initialization failed: format '%s' "+ - "for counter '%s' is invalid (must be float, large or long)", - value.Format, value.InstanceLabel) - } - + if err := config.ValidateConfig(); err != nil { + return nil, err } + reader, err := NewReader(config) if err != nil { return nil, errors.Wrap(err, "initialization of reader failed") From 22f51cf06cf8b6e139c97febf94d083df27b5f2c Mon Sep 17 00:00:00 2001 From: Mariana Date: Tue, 7 Apr 2020 21:39:32 +0200 Subject: [PATCH 02/15] work on perfmon --- metricbeat/module/windows/perfmon/config.go | 168 ++++++++++++++---- .../module/windows/perfmon/config_test.go | 20 +++ metricbeat/module/windows/perfmon/data.go | 147 +++++++++++++++ .../module/windows/perfmon/data_test.go | 20 +++ metricbeat/module/windows/perfmon/perfmon.go | 2 +- .../module/windows/perfmon/perfmon_test.go | 92 +++++----- metricbeat/module/windows/perfmon/reader.go | 143 ++------------- .../module/windows/perfmon/reader_test.go | 12 +- .../module/iis/website/manifest.yml | 129 ++++++++------ 9 files changed, 461 insertions(+), 272 deletions(-) create mode 100644 metricbeat/module/windows/perfmon/config_test.go create mode 100644 metricbeat/module/windows/perfmon/data.go create mode 100644 metricbeat/module/windows/perfmon/data_test.go diff --git a/metricbeat/module/windows/perfmon/config.go b/metricbeat/module/windows/perfmon/config.go index f9ccdaca6de6..76888d25952d 100644 --- a/metricbeat/module/windows/perfmon/config.go +++ b/metricbeat/module/windows/perfmon/config.go @@ -20,61 +20,86 @@ package perfmon import ( - "github.com/pkg/errors" + "fmt" + "regexp" "strings" + "unicode" + + "github.com/pkg/errors" ) +const replaceUpperCaseRegex = `(?:[^A-Z_\W])([A-Z])[^A-Z]` + // Config for the windows perfmon metricset. type Config struct { - IgnoreNECounters bool `config:"perfmon.ignore_non_existent_counters"` - GroupMeasurements bool `config:"perfmon.group_measurements_by_instance"` - CounterConfig []CounterConfig `config:"perfmon.counters" validate:"required"` - QueryConfig []QueryConfig `config:"perfmon.queries" validate:"required"` - GroupAllCountersTo string `config:"perfmon.group_all_counter"` + IgnoreNECounters bool `config:"perfmon.ignore_non_existent_counters"` + GroupMeasurements bool `config:"perfmon.group_measurements_by_instance"` + Counters []Counter `config:"perfmon.counters"` + Queries []Query `config:"perfmon.queries"` + GroupAllCountersTo string `config:"perfmon.group_all_counter"` MetricFormat bool } -// CounterConfig for perfmon counters. -type CounterConfig struct { +// Counter for perfmon counters. +type Counter struct { InstanceLabel string `config:"instance_label"` InstanceName string `config:"instance_name"` MeasurementLabel string `config:"measurement_label" validate:"required"` Query string `config:"query" validate:"required"` Format string `config:"format"` + Object string + ChildQueries []string } // QueryConfig for perfmon queries. This will be used as the new configuration format -type QueryConfig struct { - Name string `config:"object" validate:"required"` - Field string `config:"field"` - Instance string `config:"instance"` - Counters []QueryConfigCounter `config:"counters" validate:"required"` +type Query struct { + Name string `config:"object" validate:"required"` + Field string `config:"field"` + Instance []string `config:"instance"` + Counters []QueryCounter `config:"counters" validate:"required"` } // QueryConfigCounter for perfmon queries. This will be used as the new configuration format -type QueryConfigCounter struct { - Name string `config:"counters" validate:"required"` +type QueryCounter struct { + Name string `config:"name" validate:"required"` Field string `config:"field"` Format string `config:"format"` } func (conf *Config) ValidateConfig() error { - if len(conf.CounterConfig) == 0 && len(conf.QueryConfig) == 0 { + if len(conf.Counters) == 0 && len(conf.Queries) == 0 { return errors.New("no perfmon counters or queries have been configured") } - if len(conf.QueryConfig) > 0 { + + if len(conf.Queries) > 0 { conf.MetricFormat = true - conf.CounterConfig = []CounterConfig{} - for _, query := range conf.QueryConfig { + conf.Counters = []Counter{} + for _, query := range conf.Queries { for _, counter := range query.Counters { - var counterConf = CounterConfig{ - InstanceLabel: "instance", - InstanceName: query.Instance, - MeasurementLabel: counter.Field, - Query: query.Name, - Format: counter.Format, + if len(query.Instance) == 0 { + var counterConf = Counter{ + InstanceLabel: "instance", + InstanceName: "", + MeasurementLabel: "metrics." + mapCounterPathLabel(counter.Field, counter.Name), + Query: mapQuery(query.Name, "", counter.Name), + Format: counter.Format, + Object: query.Name, + } + conf.Counters = append(conf.Counters, counterConf) + } else { + for _, instance := range query.Instance { + var counterConf = Counter{ + InstanceLabel: "instance", + InstanceName: instance, + MeasurementLabel: "metrics." + mapCounterPathLabel(counter.Field, counter.Name), + Query: mapQuery(query.Name, instance, counter.Name), + Format: counter.Format, + Object: query.Name, + } + conf.Counters = append(conf.Counters, counterConf) + } } - conf.CounterConfig = append(conf.CounterConfig, counterConf) + } } @@ -82,7 +107,7 @@ func (conf *Config) ValidateConfig() error { } // add default format in the config - for _, value := range conf.CounterConfig { + for _, value := range conf.Counters { form := strings.ToLower(value.Format) switch form { case "", "float": @@ -94,21 +119,86 @@ func (conf *Config) ValidateConfig() error { value.Format, value.InstanceLabel) } } - for _, value := range conf.QueryConfig { - for _, path := range value.Counters { - form := strings.ToLower(path.Format) - switch form { - case "", "float": - path.Format = "float" - case "long", "large": - default: - return errors.Errorf("initialization failed: format '%s' "+ - "for counter '%s' is invalid (must be float, large or long)", - path.Format, value.Name) + + return nil +} + +func (re *Reader) getCounter(query string) (bool, Counter) { + for _, counter := range re.config.Counters { + for _, childQuery := range counter.ChildQueries { + if childQuery == query { + return true, counter } } + } + return false, Counter{} +} +func mapQuery(obj string, instance string, path string) string { + var query string + if strings.HasPrefix(obj, "\\") { + query = obj + } else { + query = fmt.Sprintf("\\%s", obj) } + if instance != "" { + query += fmt.Sprintf("(%s)", instance) + } + if strings.HasPrefix(path, "\\") { + query += path + } else { + query += fmt.Sprintf("\\%s", path) + } + return query +} - return nil +func mapCounterPathLabel(label string, path string) string { + var resultMetricName string + if label != "" { + resultMetricName = label + } else { + resultMetricName = path + } + // replace spaces with underscores + resultMetricName = strings.Replace(resultMetricName, " ", "_", -1) + // replace backslashes with "per" + resultMetricName = strings.Replace(resultMetricName, "/sec", "_per_sec", -1) + resultMetricName = strings.Replace(resultMetricName, "/_sec", "_per_sec", -1) + resultMetricName = strings.Replace(resultMetricName, "\\", "_", -1) + // replace actual percentage symbol with the smbol "pct" + resultMetricName = strings.Replace(resultMetricName, "_%_", "_pct_", -1) + // create an object in case of ":" + resultMetricName = strings.Replace(resultMetricName, ":", "_", -1) + // create an object in case of ":" + resultMetricName = strings.Replace(resultMetricName, "_-_", "_", -1) + // replace uppercases with underscores + resultMetricName = replaceUpperCase(resultMetricName) + + // avoid cases as this "logicaldisk_avg._disk_sec_per_transfer" + obj := strings.Split(resultMetricName, ".") + for index := range obj { + // in some cases a trailing "_" is found + obj[index] = strings.TrimPrefix(obj[index], "_") + obj[index] = strings.TrimSuffix(obj[index], "_") + } + resultMetricName = strings.ToLower(strings.Join(obj, "_")) + + return resultMetricName +} + +// replaceUpperCase func will replace upper case with '_' +func replaceUpperCase(src string) string { + replaceUpperCaseRegexp := regexp.MustCompile(replaceUpperCaseRegex) + return replaceUpperCaseRegexp.ReplaceAllStringFunc(src, func(str string) string { + var newStr string + for _, r := range str { + // split into fields based on class of unicode character + if unicode.IsUpper(r) { + newStr += "_" + strings.ToLower(string(r)) + } else { + newStr += string(r) + } + } + return newStr + }) } diff --git a/metricbeat/module/windows/perfmon/config_test.go b/metricbeat/module/windows/perfmon/config_test.go new file mode 100644 index 000000000000..f99cee71d6ce --- /dev/null +++ b/metricbeat/module/windows/perfmon/config_test.go @@ -0,0 +1,20 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build windows + +package perfmon diff --git a/metricbeat/module/windows/perfmon/data.go b/metricbeat/module/windows/perfmon/data.go new file mode 100644 index 000000000000..7b7024510d0d --- /dev/null +++ b/metricbeat/module/windows/perfmon/data.go @@ -0,0 +1,147 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build windows + +package perfmon + +import ( + "fmt" + "regexp" + "strconv" + "strings" + + "github.com/pkg/errors" + + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/logp" + "github.com/elastic/beats/v7/metricbeat/helper/windows/pdh" + "github.com/elastic/beats/v7/metricbeat/mb" +) + +var processRegexp = regexp.MustCompile(`(.+?)#[1-9]+`) + +func (re *Reader) groupToEvents(counters map[string][]pdh.CounterValue) []mb.Event { + eventMap := make(map[string]*mb.Event) + for counterPath, values := range counters { + hasCounter, counter := re.getCounter(counterPath) + for ind, val := range values { + // Some counters, such as rate counters, require two counter values in order to compute a displayable value. In this case we must call PdhCollectQueryData twice before calling PdhGetFormattedCounterValue. + // For more information, see Collecting Performance Data (https://docs.microsoft.com/en-us/windows/desktop/PerfCtrs/collecting-performance-data). + if val.Err != nil && !re.executed { + re.log.Debugw("Ignoring the first measurement because the data isn't ready", + "error", val.Err, logp.Namespace("perfmon"), "query", counterPath) + continue + } + + var eventKey string + if re.config.GroupMeasurements && val.Err == nil { + // Send measurements with the same instance label as part of the same event + eventKey = val.Instance + } else { + // Send every measurement as an individual event + // If a counter contains an error, it will always be sent as an individual event + eventKey = counterPath + strconv.Itoa(ind) + } + + // Create a new event if the key doesn't exist in the map + if _, ok := eventMap[eventKey]; !ok { + eventMap[eventKey] = &mb.Event{ + MetricSetFields: common.MapStr{}, + Error: errors.Wrapf(val.Err, "failed on query=%v", counterPath), + } + if val.Instance != "" && hasCounter { + //will ignore instance counter + if ok, match := matchesParentProcess(val.Instance); ok { + eventMap[eventKey].MetricSetFields.Put(counter.InstanceLabel, match) + } else { + eventMap[eventKey].MetricSetFields.Put(counter.InstanceLabel, val.Instance) + } + } + } + event := eventMap[eventKey] + if val.Measurement != nil { + event.MetricSetFields.Put(counter.MeasurementLabel, val.Measurement) + } else { + event.MetricSetFields.Put(counter.MeasurementLabel, 0) + } + if re.config.MetricFormat { + event.MetricSetFields.Put("object", counter.Object) + } + } + } + // Write the values into the map. + events := make([]mb.Event, 0, len(eventMap)) + for _, val := range eventMap { + events = append(events, *val) + } + return events +} + +func (re *Reader) groupToEvent(counters map[string][]pdh.CounterValue) mb.Event { + event := mb.Event{ + MetricSetFields: common.MapStr{}, + } + measurements := make(map[string]float64, 0) + for counterPath, values := range counters { + _, readerCounter := re.getCounter(counterPath) + for _, val := range values { + // Some counters, such as rate counters, require two counter values in order to compute a displayable value. In this case we must call PdhCollectQueryData twice before calling PdhGetFormattedCounterValue. + // For more information, see Collecting Performance Data (https://docs.microsoft.com/en-us/windows/desktop/PerfCtrs/collecting-performance-data). + if val.Err != nil && !re.executed { + re.log.Debugw("Ignoring the first measurement because the data isn't ready", + "error", val.Err, logp.Namespace("perfmon"), "query", counterPath) + continue + } + var counterVal float64 + switch val.Measurement.(type) { + case int64: + counterVal = float64(val.Measurement.(int64)) + default: + counterVal = val.Measurement.(float64) + } + if _, ok := measurements[readerCounter.MeasurementLabel]; !ok { + measurements[readerCounter.MeasurementLabel] = counterVal + measurements[readerCounter.MeasurementLabel+instanceCountLabel] = 1 + } else { + measurements[readerCounter.MeasurementLabel+instanceCountLabel] = measurements[readerCounter.MeasurementLabel+instanceCountLabel] + 1 + measurements[readerCounter.MeasurementLabel] = measurements[readerCounter.MeasurementLabel] + counterVal + } + } + } + for key, val := range measurements { + if strings.Contains(key, instanceCountLabel) { + if val == 1 { + continue + } else { + event.MetricSetFields.Put(fmt.Sprintf("%s.%s", strings.Split(key, ".")[0], re.config.GroupAllCountersTo), val) + } + } else { + event.MetricSetFields.Put(key, val) + } + } + return event +} + +// matchParentProcess will try to get the parent process name +func matchesParentProcess(instanceName string) (bool, string) { + matches := processRegexp.FindStringSubmatch(instanceName) + if len(matches) == 2 { + return true, matches[1] + } + return false, instanceName +} diff --git a/metricbeat/module/windows/perfmon/data_test.go b/metricbeat/module/windows/perfmon/data_test.go new file mode 100644 index 000000000000..f99cee71d6ce --- /dev/null +++ b/metricbeat/module/windows/perfmon/data_test.go @@ -0,0 +1,20 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build windows + +package perfmon diff --git a/metricbeat/module/windows/perfmon/perfmon.go b/metricbeat/module/windows/perfmon/perfmon.go index a469293c36ea..e5c11f3220be 100644 --- a/metricbeat/module/windows/perfmon/perfmon.go +++ b/metricbeat/module/windows/perfmon/perfmon.go @@ -49,7 +49,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { if err := base.Module().UnpackConfig(&config); err != nil { return nil, err } - if err := config.ValidateConfig(); err != nil { + if err := config.ValidateConfig(); err != nil { return nil, err } diff --git a/metricbeat/module/windows/perfmon/perfmon_test.go b/metricbeat/module/windows/perfmon/perfmon_test.go index cd882131b541..274d83b8b7f3 100644 --- a/metricbeat/module/windows/perfmon/perfmon_test.go +++ b/metricbeat/module/windows/perfmon/perfmon_test.go @@ -115,7 +115,7 @@ func TestQuery(t *testing.T) { t.Fatal(err) } defer q.Close() - counter := CounterConfig{Format: "float", InstanceName: "TestInstanceName"} + counter := Counter{Format: "float", InstanceName: "TestInstanceName"} path, err := q.GetCounterPaths(processorTimeCounter) if err != nil { t.Fatal(err) @@ -150,12 +150,12 @@ func TestQuery(t *testing.T) { func TestExistingCounter(t *testing.T) { config := Config{ - CounterConfig: make([]CounterConfig, 1), + Counters: make([]Counter, 1), } - config.CounterConfig[0].InstanceLabel = "processor.name" - config.CounterConfig[0].MeasurementLabel = "processor.time.total.pct" - config.CounterConfig[0].Query = processorTimeCounter - config.CounterConfig[0].Format = "float" + config.Counters[0].InstanceLabel = "processor.name" + config.Counters[0].MeasurementLabel = "processor.time.total.pct" + config.Counters[0].Query = processorTimeCounter + config.Counters[0].Format = "float" handle, err := NewReader(config) if err != nil { t.Fatal(err) @@ -172,12 +172,12 @@ func TestExistingCounter(t *testing.T) { func TestNonExistingCounter(t *testing.T) { config := Config{ - CounterConfig: make([]CounterConfig, 1), + Counters: make([]Counter, 1), } - config.CounterConfig[0].InstanceLabel = "processor.name" - config.CounterConfig[0].MeasurementLabel = "processor.time.total.pct" - config.CounterConfig[0].Query = "\\Processor Information(_Total)\\not existing counter" - config.CounterConfig[0].Format = "float" + config.Counters[0].InstanceLabel = "processor.name" + config.Counters[0].MeasurementLabel = "processor.time.total.pct" + config.Counters[0].Query = "\\Processor Information(_Total)\\not existing counter" + config.Counters[0].Format = "float" handle, err := NewReader(config) if assert.Error(t, err) { assert.EqualValues(t, pdh.PDH_CSTATUS_NO_COUNTER, errors.Cause(err)) @@ -191,13 +191,13 @@ func TestNonExistingCounter(t *testing.T) { func TestIgnoreNonExistentCounter(t *testing.T) { config := Config{ - CounterConfig: make([]CounterConfig, 1), + Counters: make([]Counter, 1), IgnoreNECounters: true, } - config.CounterConfig[0].InstanceLabel = "processor.name" - config.CounterConfig[0].MeasurementLabel = "processor.time.total.pct" - config.CounterConfig[0].Query = "\\Processor Information(_Total)\\not existing counter" - config.CounterConfig[0].Format = "float" + config.Counters[0].InstanceLabel = "processor.name" + config.Counters[0].MeasurementLabel = "processor.time.total.pct" + config.Counters[0].Query = "\\Processor Information(_Total)\\not existing counter" + config.Counters[0].Format = "float" handle, err := NewReader(config) values, err := handle.Read() @@ -216,12 +216,12 @@ func TestIgnoreNonExistentCounter(t *testing.T) { func TestNonExistingObject(t *testing.T) { config := Config{ - CounterConfig: make([]CounterConfig, 1), + Counters: make([]Counter, 1), } - config.CounterConfig[0].InstanceLabel = "processor.name" - config.CounterConfig[0].MeasurementLabel = "processor.time.total.pct" - config.CounterConfig[0].Query = "\\non existing object\\% Processor Performance" - config.CounterConfig[0].Format = "float" + config.Counters[0].InstanceLabel = "processor.name" + config.Counters[0].MeasurementLabel = "processor.time.total.pct" + config.Counters[0].Query = "\\non existing object\\% Processor Performance" + config.Counters[0].Format = "float" handle, err := NewReader(config) if assert.Error(t, err) { assert.EqualValues(t, pdh.PDH_CSTATUS_NO_OBJECT, errors.Cause(err)) @@ -240,7 +240,7 @@ func TestLongOutputFormat(t *testing.T) { t.Fatal(err) } defer query.Close() - counter := CounterConfig{Format: "long"} + counter := Counter{Format: "long"} path, err := query.GetCounterPaths(processorTimeCounter) if err != nil { t.Fatal(err) @@ -280,7 +280,7 @@ func TestFloatOutputFormat(t *testing.T) { t.Fatal(err) } defer query.Close() - counter := CounterConfig{Format: "float"} + counter := Counter{Format: "float"} path, err := query.GetCounterPaths(processorTimeCounter) if err != nil { t.Fatal(err) @@ -315,13 +315,13 @@ func TestFloatOutputFormat(t *testing.T) { func TestWildcardQuery(t *testing.T) { config := Config{ - CounterConfig: make([]CounterConfig, 1), + Counters: make([]Counter, 1), } - config.CounterConfig[0].InstanceLabel = "processor.name" - config.CounterConfig[0].InstanceName = "TestInstanceName" - config.CounterConfig[0].MeasurementLabel = "processor.time.pct" - config.CounterConfig[0].Query = `\Processor Information(*)\% Processor Time` - config.CounterConfig[0].Format = "float" + config.Counters[0].InstanceLabel = "processor.name" + config.Counters[0].InstanceName = "TestInstanceName" + config.Counters[0].MeasurementLabel = "processor.time.pct" + config.Counters[0].Query = `\Processor Information(*)\% Processor Time` + config.Counters[0].Format = "float" handle, err := NewReader(config) if err != nil { t.Fatal(err) @@ -354,11 +354,11 @@ func TestWildcardQuery(t *testing.T) { func TestWildcardQueryNoInstanceName(t *testing.T) { config := Config{ - CounterConfig: make([]CounterConfig, 1), + Counters: make([]Counter, 1), } - config.CounterConfig[0].InstanceLabel = "process_private_bytes" - config.CounterConfig[0].MeasurementLabel = "process.private.bytes" - config.CounterConfig[0].Query = `\Process(*)\Private Bytes` + config.Counters[0].InstanceLabel = "process_private_bytes" + config.Counters[0].MeasurementLabel = "process.private.bytes" + config.Counters[0].Query = `\Process(*)\Private Bytes` handle, err := NewReader(config) if err != nil { t.Fatal(err) @@ -393,23 +393,23 @@ func TestWildcardQueryNoInstanceName(t *testing.T) { func TestGroupByInstance(t *testing.T) { config := Config{ - CounterConfig: make([]CounterConfig, 3), + Counters: make([]Counter, 3), GroupMeasurements: true, } - config.CounterConfig[0].InstanceLabel = "processor.name" - config.CounterConfig[0].MeasurementLabel = "processor.time.pct" - config.CounterConfig[0].Query = `\Processor Information(_Total)\% Processor Time` - config.CounterConfig[0].Format = "float" + config.Counters[0].InstanceLabel = "processor.name" + config.Counters[0].MeasurementLabel = "processor.time.pct" + config.Counters[0].Query = `\Processor Information(_Total)\% Processor Time` + config.Counters[0].Format = "float" - config.CounterConfig[1].InstanceLabel = "processor.name" - config.CounterConfig[1].MeasurementLabel = "processor.time.user.pct" - config.CounterConfig[1].Query = `\Processor Information(_Total)\% User Time` - config.CounterConfig[1].Format = "float" + config.Counters[1].InstanceLabel = "processor.name" + config.Counters[1].MeasurementLabel = "processor.time.user.pct" + config.Counters[1].Query = `\Processor Information(_Total)\% User Time` + config.Counters[1].Format = "float" - config.CounterConfig[2].InstanceLabel = "processor.name" - config.CounterConfig[2].MeasurementLabel = "processor.time.privileged.ns" - config.CounterConfig[2].Query = `\Processor Information(_Total)\% Privileged Time` - config.CounterConfig[2].Format = "float" + config.Counters[2].InstanceLabel = "processor.name" + config.Counters[2].MeasurementLabel = "processor.time.privileged.ns" + config.Counters[2].Query = `\Processor Information(_Total)\% Privileged Time` + config.Counters[2].Format = "float" handle, err := NewReader(config) if err != nil { diff --git a/metricbeat/module/windows/perfmon/reader.go b/metricbeat/module/windows/perfmon/reader.go index a7ecd6663621..1fc13d7004d1 100644 --- a/metricbeat/module/windows/perfmon/reader.go +++ b/metricbeat/module/windows/perfmon/reader.go @@ -20,32 +20,24 @@ package perfmon import ( - "fmt" - "regexp" - "strconv" "strings" "github.com/elastic/beats/v7/metricbeat/helper/windows/pdh" "github.com/pkg/errors" - "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/metricbeat/mb" ) -var processRegexp = regexp.MustCompile(`(.+?)#[1-9]+`) - const instanceCountLabel = ":count" // Reader will contain the config options type Reader struct { - query pdh.Query // PDH Query - instanceLabel map[string]string // Mapping of counter path to key used for the label (e.g. processor.name) - measurement map[string]string // Mapping of counter path to key used for the value (e.g. processor.cpu_time). - executed bool // Indicates if the query has been executed. - log *logp.Logger // - config Config // Metricset configuration + query pdh.Query // PDH Query + executed bool // Indicates if the query has been executed. + log *logp.Logger // + config Config // Metricset configuration } // NewReader creates a new instance of Reader. @@ -55,13 +47,11 @@ func NewReader(config Config) (*Reader, error) { return nil, err } r := &Reader{ - query: query, - instanceLabel: map[string]string{}, - measurement: map[string]string{}, - log: logp.NewLogger("perfmon"), - config: config, + query: query, + log: logp.NewLogger("perfmon"), } - for _, counter := range config.CounterConfig { + for i, counter := range config.Counters { + config.Counters[i].ChildQueries = []string{} childQueries, err := query.GetCounterPaths(counter.Query) if err != nil { if config.IgnoreNECounters { @@ -91,17 +81,18 @@ func NewReader(config Config) (*Reader, error) { if err := query.AddCounter(v, counter.InstanceName, counter.Format, len(childQueries) > 1); err != nil { return nil, errors.Wrapf(err, `failed to add counter (query="%v")`, counter.Query) } - r.instanceLabel[v] = counter.InstanceLabel - r.measurement[v] = counter.MeasurementLabel + config.Counters[i].ChildQueries = append(config.Counters[i].ChildQueries, v) } } + r.config = config return r, nil } // RefreshCounterPaths will recheck for any new instances and add them to the counter list func (re *Reader) RefreshCounterPaths() error { var newCounters []string - for _, counter := range re.config.CounterConfig { + for i, counter := range re.config.Counters { + re.config.Counters[i].ChildQueries = []string{} childQueries, err := re.query.GetCounterPaths(counter.Query) if err != nil { if re.config.IgnoreNECounters { @@ -123,8 +114,7 @@ func (re *Reader) RefreshCounterPaths() error { if err := re.query.AddCounter(v, counter.InstanceName, counter.Format, len(childQueries) > 1); err != nil { return errors.Wrapf(err, "failed to add counter (query='%v')", counter.Query) } - re.instanceLabel[v] = counter.InstanceLabel - re.measurement[v] = counter.MeasurementLabel + re.config.Counters[i].ChildQueries = append(re.config.Counters[i].ChildQueries, v) } } } @@ -161,114 +151,7 @@ func (re *Reader) Read() ([]mb.Event, error) { return events, nil } -func (re *Reader) groupToEvents(counters map[string][]pdh.CounterValue) []mb.Event { - eventMap := make(map[string]*mb.Event) - - for counterPath, values := range counters { - for ind, val := range values { - // Some counters, such as rate counters, require two counter values in order to compute a displayable value. In this case we must call PdhCollectQueryData twice before calling PdhGetFormattedCounterValue. - // For more information, see Collecting Performance Data (https://docs.microsoft.com/en-us/windows/desktop/PerfCtrs/collecting-performance-data). - if val.Err != nil && !re.executed { - re.log.Debugw("Ignoring the first measurement because the data isn't ready", - "error", val.Err, logp.Namespace("perfmon"), "query", counterPath) - continue - } - - var eventKey string - if re.config.GroupMeasurements && val.Err == nil { - // Send measurements with the same instance label as part of the same event - eventKey = val.Instance - } else { - // Send every measurement as an individual event - // If a counter contains an error, it will always be sent as an individual event - eventKey = counterPath + strconv.Itoa(ind) - } - - // Create a new event if the key doesn't exist in the map - if _, ok := eventMap[eventKey]; !ok { - eventMap[eventKey] = &mb.Event{ - MetricSetFields: common.MapStr{}, - Error: errors.Wrapf(val.Err, "failed on query=%v", counterPath), - } - if val.Instance != "" && re.instanceLabel[counterPath] != "" { - //will ignore instance counter - if ok, match := matchesParentProcess(val.Instance); ok { - eventMap[eventKey].MetricSetFields.Put(re.instanceLabel[counterPath], match) - } else { - eventMap[eventKey].MetricSetFields.Put(re.instanceLabel[counterPath], val.Instance) - } - } - } - event := eventMap[eventKey] - if val.Measurement != nil { - event.MetricSetFields.Put(re.measurement[counterPath], val.Measurement) - } else { - event.MetricSetFields.Put(re.measurement[counterPath], 0) - } - } - } - // Write the values into the map. - events := make([]mb.Event, 0, len(eventMap)) - for _, val := range eventMap { - events = append(events, *val) - } - return events -} - -func (re *Reader) groupToEvent(counters map[string][]pdh.CounterValue) mb.Event { - event := mb.Event{ - MetricSetFields: common.MapStr{}, - } - measurements := make(map[string]float64, 0) - for counterPath, values := range counters { - for _, val := range values { - // Some counters, such as rate counters, require two counter values in order to compute a displayable value. In this case we must call PdhCollectQueryData twice before calling PdhGetFormattedCounterValue. - // For more information, see Collecting Performance Data (https://docs.microsoft.com/en-us/windows/desktop/PerfCtrs/collecting-performance-data). - if val.Err != nil && !re.executed { - re.log.Debugw("Ignoring the first measurement because the data isn't ready", - "error", val.Err, logp.Namespace("perfmon"), "query", counterPath) - continue - } - var counterVal float64 - switch val.Measurement.(type) { - case int64: - counterVal = float64(val.Measurement.(int64)) - default: - counterVal = val.Measurement.(float64) - } - if _, ok := measurements[re.measurement[counterPath]]; !ok { - measurements[re.measurement[counterPath]] = counterVal - measurements[re.measurement[counterPath]+instanceCountLabel] = 1 - } else { - measurements[re.measurement[counterPath]+instanceCountLabel] = measurements[re.measurement[counterPath]+instanceCountLabel] + 1 - measurements[re.measurement[counterPath]] = measurements[re.measurement[counterPath]] + counterVal - } - } - } - for key, val := range measurements { - if strings.Contains(key, instanceCountLabel) { - if val == 1 { - continue - } else { - event.MetricSetFields.Put(fmt.Sprintf("%s.%s", strings.Split(key, ".")[0], re.config.GroupAllCountersTo), val) - } - } else { - event.MetricSetFields.Put(key, val) - } - } - return event -} - // Close will close the PDH query for now. func (re *Reader) Close() error { return re.query.Close() } - -// matchParentProcess will try to get the parent process name -func matchesParentProcess(instanceName string) (bool, string) { - matches := processRegexp.FindStringSubmatch(instanceName) - if len(matches) == 2 { - return true, matches[1] - } - return false, instanceName -} diff --git a/metricbeat/module/windows/perfmon/reader_test.go b/metricbeat/module/windows/perfmon/reader_test.go index 7f925d21a623..f316f1b8ae95 100644 --- a/metricbeat/module/windows/perfmon/reader_test.go +++ b/metricbeat/module/windows/perfmon/reader_test.go @@ -29,11 +29,11 @@ var validQuery = `\Processor Information(_Total)\% Processor Time` // TestNewReaderWhenQueryPathNotProvided will check for invalid/no query. func TestNewReaderWhenQueryPathNotProvided(t *testing.T) { - counter := CounterConfig{Format: "float", InstanceName: "TestInstanceName"} + counter := Counter{Format: "float", InstanceName: "TestInstanceName"} config := Config{ IgnoreNECounters: false, GroupMeasurements: false, - CounterConfig: []CounterConfig{counter}, + Counters: []Counter{counter}, } reader, err := NewReader(config) assert.NotNil(t, err) @@ -43,11 +43,11 @@ func TestNewReaderWhenQueryPathNotProvided(t *testing.T) { // TestNewReaderWithValidQueryPath should successfully instantiate the reader. func TestNewReaderWithValidQueryPath(t *testing.T) { - counter := CounterConfig{Format: "float", InstanceName: "TestInstanceName", Query: validQuery} + counter := Counter{Format: "float", InstanceName: "TestInstanceName", Query: validQuery} config := Config{ IgnoreNECounters: false, GroupMeasurements: false, - CounterConfig: []CounterConfig{counter}, + Counters: []Counter{counter}, } reader, err := NewReader(config) defer reader.Close() @@ -62,11 +62,11 @@ func TestNewReaderWithValidQueryPath(t *testing.T) { // TestReadSuccessfully will test the func read when it first retrieves no events (and ignored) and then starts retrieving events. func TestReadSuccessfully(t *testing.T) { - counter := CounterConfig{Format: "float", InstanceName: "TestInstanceName", Query: validQuery} + counter := Counter{Format: "float", InstanceName: "TestInstanceName", Query: validQuery} config := Config{ IgnoreNECounters: false, GroupMeasurements: false, - CounterConfig: []CounterConfig{counter}, + Counters: []Counter{counter}, } reader, err := NewReader(config) if err != nil { diff --git a/x-pack/metricbeat/module/iis/website/manifest.yml b/x-pack/metricbeat/module/iis/website/manifest.yml index 564ad6db86ce..3e41b942fd5e 100644 --- a/x-pack/metricbeat/module/iis/website/manifest.yml +++ b/x-pack/metricbeat/module/iis/website/manifest.yml @@ -5,57 +5,86 @@ input: defaults: perfmon.group_measurements_by_instance: true perfmon.ignore_non_existent_counters: true - perfmon.counters: - #network - - instance_label: 'name' - measurement_label: total_bytes_received - query: '\Web Service(*)\Total Bytes Received' - - instance_label: 'name' - measurement_label: total_bytes_sent - query: '\Web Service(*)\Total Bytes Sent' - - instance_label: 'name' - measurement_label: bytes_sent_per_sec - query: '\Web Service(*)\Bytes Sent/sec' - - instance_label: 'name' - measurement_label: bytes_received_per_sec - query: '\Web Service(*)\Bytes Received/sec' - - instance_label: 'name' - measurement_label: current_connections - query: '\Web Service(*)\Current Connections' - - instance_label: 'name' - measurement_label: maximum_connections - query: '\Web Service(*)\Maximum Connections' - - instance_label: 'name' - measurement_label: total_connection_attempts - query: '\Web Service(*)\Total Connection Attempts (all instances)' - - instance_label: 'name' - measurement_label: total_get_requests - query: '\Web Service(*)\Total Get Requests' - - instance_label: 'name' - measurement_label: get_requests_per_sec - query: '\Web Service(*)\Get Requests/sec' - - instance_label: 'name' - measurement_label: total_post_requests - query: '\Web Service(*)\Total Post Requests' - - instance_label: 'name' - measurement_label: post_requests_per_sec - query: '\Web Service(*)\Post Requests/sec' - - instance_label: 'name' - measurement_label: total_delete_requests - query: '\Web Service(*)\Total Delete Requests' - - instance_label: 'name' - measurement_label: delete_requests_per_sec - query: '\Web Service(*)\Delete Requests/sec' - - instance_label: 'name' - measurement_label: service_uptime - query: '\Web Service(*)\Service Uptime' - - instance_label: 'name' - measurement_label: total_put_requests - query: '\Web Service(*)\Total PUT Requests' - - instance_label: 'name' - measurement_label: put_requests_per_sec - query: '\Web Service(*)\PUT Requests/sec' + perfmon.queries: + - object: 'Web Service' + instance: "*" + counters: + - name: 'Total Bytes Received' + field: total_bytes_received + - name: 'Total Bytes Sent' + field: total_bytes_sent + - name: "Bytes Sent/sec" + field: bytes_sent_per_sec + - name: "Bytes Received/sec" + field: bytes_received_per_sec + - name: "Current Connections" + field: current_connections + - name: "Maximum Connections" + field: maximum_connections + - name: "Total Connection Attempts (all instances)" + field: total_connection_attempts + - name: "Total Get Requests" + field: total_get_requests + - name: "Get Requests/sec" + field: get_requests_per_sec + - name: "Total Post Requests" + field: total_post_requests + - name: "Post Requests/sec" + field: post_requests_per_sec + - name: "Total Delete Requests" + field: total_delete_requests + - name: "Delete Requests/sec" + field: delete_requests_per_sec + - name: "Service Uptime" + field: service_uptime + - name: "Total PUT Requests" + field: total_put_requests + - name: "PUT Requests/sec" + field: put_requests_per_sec processors: - drop_event.when.equals: iis.website.name: '_Total' +- rename: + ignore_missing: true + fields: + - from: "iis.website.metrics.total_bytes_received" + to: "iis.website.total_bytes_received" + - from: "iis.website.metrics.total_bytes_sent" + to: "iis.website.total_bytes_sent" + - from: "iis.website.metrics.bytes_sent_per_sec" + to: "iis.website.bytes_sent_per_sec" + - from: "iis.website.metrics.bytes_received_per_sec" + to: "iis.website.bytes_received_per_sec" + - from: "iis.website.metrics.current_connections" + to: "iis.website.current_connections" + - from: "iis.website.metrics.maximum_connections" + to: "iis.website.maximum_connections" + - from: "iis.website.metrics.total_connection_attempts" + to: "iis.website.total_connection_attempts" + - from: "iis.website.metrics.total_get_requests" + to: "iis.website.total_get_requests" + - from: "iis.website.metrics.get_requests_per_sec" + to: "iis.website.get_requests_per_sec" + - from: "iis.website.metrics.total_post_requests" + to: "iis.website.total_post_requests" + - from: "iis.website.metrics.post_requests_per_sec" + to: "iis.website.post_requests_per_sec" + - from: "iis.website.metrics.total_delete_requests" + to: "iis.website.total_delete_requests" + - from: "iis.website.metrics.delete_requests_per_sec" + to: "iis.website.delete_requests_per_sec" + - from: "iis.website.metrics.service_uptime" + to: "iis.website.service_uptime" + - from: "iis.website.metrics.total_put_requests" + to: "iis.website.total_put_requests" + - from: "iis.website.metrics.put_requests_per_sec" + to: "iis.website.put_requests_per_sec" + - from: "iis.website.instance" + to: "iis.website.name" +- drop_fields: + fields: ["iis.website.object", "iis.website.metrics"] + + + + From ad90e0096993776d578bf961b7bfa67070ddcd6b Mon Sep 17 00:00:00 2001 From: Mariana Date: Thu, 9 Apr 2020 10:21:11 +0200 Subject: [PATCH 03/15] progress --- metricbeat/module/windows/perfmon/config.go | 139 +++--------------- metricbeat/module/windows/perfmon/data.go | 20 +-- metricbeat/module/windows/perfmon/reader.go | 154 +++++++++++++++++++- 3 files changed, 178 insertions(+), 135 deletions(-) diff --git a/metricbeat/module/windows/perfmon/config.go b/metricbeat/module/windows/perfmon/config.go index 76888d25952d..9ae3bc79ea01 100644 --- a/metricbeat/module/windows/perfmon/config.go +++ b/metricbeat/module/windows/perfmon/config.go @@ -20,12 +20,9 @@ package perfmon import ( - "fmt" - "regexp" - "strings" - "unicode" - + "github.com/elastic/beats/v7/libbeat/common/cfgwarn" "github.com/pkg/errors" + "strings" ) const replaceUpperCaseRegex = `(?:[^A-Z_\W])([A-Z])[^A-Z]` @@ -40,15 +37,13 @@ type Config struct { MetricFormat bool } -// Counter for perfmon counters. +// Counter for the perfmon counters (old implementation deprecated). type Counter struct { InstanceLabel string `config:"instance_label"` InstanceName string `config:"instance_name"` MeasurementLabel string `config:"measurement_label" validate:"required"` Query string `config:"query" validate:"required"` Format string `config:"format"` - Object string - ChildQueries []string } // QueryConfig for perfmon queries. This will be used as the new configuration format @@ -70,48 +65,20 @@ func (conf *Config) ValidateConfig() error { if len(conf.Counters) == 0 && len(conf.Queries) == 0 { return errors.New("no perfmon counters or queries have been configured") } + if len(conf.Counters) > 0 { + cfgwarn.Deprecate("8.0", "perfmon.counters configuration option is deprecated and will be remove in the future major version, we advise using the perfmon.queries configuration option instead") + } if len(conf.Queries) > 0 { conf.MetricFormat = true - conf.Counters = []Counter{} - for _, query := range conf.Queries { - for _, counter := range query.Counters { - if len(query.Instance) == 0 { - var counterConf = Counter{ - InstanceLabel: "instance", - InstanceName: "", - MeasurementLabel: "metrics." + mapCounterPathLabel(counter.Field, counter.Name), - Query: mapQuery(query.Name, "", counter.Name), - Format: counter.Format, - Object: query.Name, - } - conf.Counters = append(conf.Counters, counterConf) - } else { - for _, instance := range query.Instance { - var counterConf = Counter{ - InstanceLabel: "instance", - InstanceName: instance, - MeasurementLabel: "metrics." + mapCounterPathLabel(counter.Field, counter.Name), - Query: mapQuery(query.Name, instance, counter.Name), - Format: counter.Format, - Object: query.Name, - } - conf.Counters = append(conf.Counters, counterConf) - } - } - - } - - } - } // add default format in the config - for _, value := range conf.Counters { + for i, value := range conf.Counters { form := strings.ToLower(value.Format) switch form { case "", "float": - value.Format = "float" + conf.Counters[i].Format = "float" case "long", "large": default: return errors.Errorf("initialization failed: format '%s' "+ @@ -119,86 +86,20 @@ func (conf *Config) ValidateConfig() error { value.Format, value.InstanceLabel) } } - - return nil -} - -func (re *Reader) getCounter(query string) (bool, Counter) { - for _, counter := range re.config.Counters { - for _, childQuery := range counter.ChildQueries { - if childQuery == query { - return true, counter + for _, value := range conf.Queries { + for i, q := range value.Counters { + form := strings.ToLower(q.Format) + switch form { + case "", "float": + value.Counters[i].Format = "float" + case "long", "large": + default: + return errors.Errorf("initialization failed: format '%s' "+ + "for counter '%s' is invalid (must be float, large or long)", + q.Format, q.Field) } } - } - return false, Counter{} -} - -func mapQuery(obj string, instance string, path string) string { - var query string - if strings.HasPrefix(obj, "\\") { - query = obj - } else { - query = fmt.Sprintf("\\%s", obj) - } - if instance != "" { - query += fmt.Sprintf("(%s)", instance) - } - if strings.HasPrefix(path, "\\") { - query += path - } else { - query += fmt.Sprintf("\\%s", path) - } - return query -} - -func mapCounterPathLabel(label string, path string) string { - var resultMetricName string - if label != "" { - resultMetricName = label - } else { - resultMetricName = path - } - // replace spaces with underscores - resultMetricName = strings.Replace(resultMetricName, " ", "_", -1) - // replace backslashes with "per" - resultMetricName = strings.Replace(resultMetricName, "/sec", "_per_sec", -1) - resultMetricName = strings.Replace(resultMetricName, "/_sec", "_per_sec", -1) - resultMetricName = strings.Replace(resultMetricName, "\\", "_", -1) - // replace actual percentage symbol with the smbol "pct" - resultMetricName = strings.Replace(resultMetricName, "_%_", "_pct_", -1) - // create an object in case of ":" - resultMetricName = strings.Replace(resultMetricName, ":", "_", -1) - // create an object in case of ":" - resultMetricName = strings.Replace(resultMetricName, "_-_", "_", -1) - // replace uppercases with underscores - resultMetricName = replaceUpperCase(resultMetricName) - // avoid cases as this "logicaldisk_avg._disk_sec_per_transfer" - obj := strings.Split(resultMetricName, ".") - for index := range obj { - // in some cases a trailing "_" is found - obj[index] = strings.TrimPrefix(obj[index], "_") - obj[index] = strings.TrimSuffix(obj[index], "_") } - resultMetricName = strings.ToLower(strings.Join(obj, "_")) - - return resultMetricName -} - -// replaceUpperCase func will replace upper case with '_' -func replaceUpperCase(src string) string { - replaceUpperCaseRegexp := regexp.MustCompile(replaceUpperCaseRegex) - return replaceUpperCaseRegexp.ReplaceAllStringFunc(src, func(str string) string { - var newStr string - for _, r := range str { - // split into fields based on class of unicode character - if unicode.IsUpper(r) { - newStr += "_" + strings.ToLower(string(r)) - } else { - newStr += string(r) - } - } - return newStr - }) + return nil } diff --git a/metricbeat/module/windows/perfmon/data.go b/metricbeat/module/windows/perfmon/data.go index 7b7024510d0d..d4284dec427c 100644 --- a/metricbeat/module/windows/perfmon/data.go +++ b/metricbeat/module/windows/perfmon/data.go @@ -67,20 +67,20 @@ func (re *Reader) groupToEvents(counters map[string][]pdh.CounterValue) []mb.Eve if val.Instance != "" && hasCounter { //will ignore instance counter if ok, match := matchesParentProcess(val.Instance); ok { - eventMap[eventKey].MetricSetFields.Put(counter.InstanceLabel, match) + eventMap[eventKey].MetricSetFields.Put(counter.InstanceField, match) } else { - eventMap[eventKey].MetricSetFields.Put(counter.InstanceLabel, val.Instance) + eventMap[eventKey].MetricSetFields.Put(counter.InstanceField, val.Instance) } } } event := eventMap[eventKey] if val.Measurement != nil { - event.MetricSetFields.Put(counter.MeasurementLabel, val.Measurement) + event.MetricSetFields.Put(counter.QueryField, val.Measurement) } else { - event.MetricSetFields.Put(counter.MeasurementLabel, 0) + event.MetricSetFields.Put(counter.QueryField, 0) } if re.config.MetricFormat { - event.MetricSetFields.Put("object", counter.Object) + event.MetricSetFields.Put(counter.ObjectField, counter.ObjectName) } } } @@ -114,12 +114,12 @@ func (re *Reader) groupToEvent(counters map[string][]pdh.CounterValue) mb.Event default: counterVal = val.Measurement.(float64) } - if _, ok := measurements[readerCounter.MeasurementLabel]; !ok { - measurements[readerCounter.MeasurementLabel] = counterVal - measurements[readerCounter.MeasurementLabel+instanceCountLabel] = 1 + if _, ok := measurements[readerCounter.QueryField]; !ok { + measurements[readerCounter.QueryField] = counterVal + measurements[readerCounter.QueryField+instanceCountLabel] = 1 } else { - measurements[readerCounter.MeasurementLabel+instanceCountLabel] = measurements[readerCounter.MeasurementLabel+instanceCountLabel] + 1 - measurements[readerCounter.MeasurementLabel] = measurements[readerCounter.MeasurementLabel] + counterVal + measurements[readerCounter.QueryField+instanceCountLabel] = measurements[readerCounter.QueryField+instanceCountLabel] + 1 + measurements[readerCounter.QueryField] = measurements[readerCounter.QueryField] + counterVal } } } diff --git a/metricbeat/module/windows/perfmon/reader.go b/metricbeat/module/windows/perfmon/reader.go index 1fc13d7004d1..4601706b4be0 100644 --- a/metricbeat/module/windows/perfmon/reader.go +++ b/metricbeat/module/windows/perfmon/reader.go @@ -20,7 +20,10 @@ package perfmon import ( + "fmt" + "regexp" "strings" + "unicode" "github.com/elastic/beats/v7/metricbeat/helper/windows/pdh" @@ -38,6 +41,18 @@ type Reader struct { executed bool // Indicates if the query has been executed. log *logp.Logger // config Config // Metricset configuration + counters []PerfCounter +} + +type PerfCounter struct { + InstanceField string + InstanceName string + QueryField string + Query string + Format string + ObjectName string + ObjectField string + ChildQueries []string } // NewReader creates a new instance of Reader. @@ -50,8 +65,9 @@ func NewReader(config Config) (*Reader, error) { query: query, log: logp.NewLogger("perfmon"), } - for i, counter := range config.Counters { - config.Counters[i].ChildQueries = []string{} + r.mapCounters(config) + for i, counter := range r.counters { + r.counters[i].ChildQueries = []string{} childQueries, err := query.GetCounterPaths(counter.Query) if err != nil { if config.IgnoreNECounters { @@ -81,7 +97,7 @@ func NewReader(config Config) (*Reader, error) { if err := query.AddCounter(v, counter.InstanceName, counter.Format, len(childQueries) > 1); err != nil { return nil, errors.Wrapf(err, `failed to add counter (query="%v")`, counter.Query) } - config.Counters[i].ChildQueries = append(config.Counters[i].ChildQueries, v) + r.counters[i].ChildQueries = append(r.counters[i].ChildQueries, v) } } r.config = config @@ -91,8 +107,8 @@ func NewReader(config Config) (*Reader, error) { // RefreshCounterPaths will recheck for any new instances and add them to the counter list func (re *Reader) RefreshCounterPaths() error { var newCounters []string - for i, counter := range re.config.Counters { - re.config.Counters[i].ChildQueries = []string{} + for i, counter := range re.counters { + re.counters[i].ChildQueries = []string{} childQueries, err := re.query.GetCounterPaths(counter.Query) if err != nil { if re.config.IgnoreNECounters { @@ -114,7 +130,7 @@ func (re *Reader) RefreshCounterPaths() error { if err := re.query.AddCounter(v, counter.InstanceName, counter.Format, len(childQueries) > 1); err != nil { return errors.Wrapf(err, "failed to add counter (query='%v')", counter.Query) } - re.config.Counters[i].ChildQueries = append(re.config.Counters[i].ChildQueries, v) + re.counters[i].ChildQueries = append(re.counters[i].ChildQueries, v) } } } @@ -155,3 +171,129 @@ func (re *Reader) Read() ([]mb.Event, error) { func (re *Reader) Close() error { return re.query.Close() } + +func (re *Reader) getCounter(query string) (bool, PerfCounter) { + for _, counter := range re.counters { + for _, childQuery := range counter.ChildQueries { + if childQuery == query { + return true, counter + } + } + } + return false, PerfCounter{} +} + +func (re *Reader) mapCounters(config Config) { + re.counters = []PerfCounter{} + if len(config.Counters) > 0 { + for _, counter := range config.Counters { + re.counters = append(re.counters, PerfCounter{ + InstanceField: counter.InstanceLabel, + InstanceName: counter.InstanceName, + QueryField: counter.MeasurementLabel, + Query: counter.Query, + Format: counter.Format, + ChildQueries: nil, + }) + } + } + if len(config.Queries) > 0 { + for _, query := range config.Queries { + for _, counter := range query.Counters { + if len(query.Instance) == 0 { + re.counters = append(re.counters, PerfCounter{ + InstanceField: "instance", + InstanceName: "", + QueryField: "metrics." + mapCounterPathLabel(counter.Field, counter.Name), + Query: mapQuery(query.Name, "", counter.Name), + Format: counter.Format, + ObjectName: query.Name, + }) + } else { + for _, instance := range query.Instance { + re.counters = append(re.counters, PerfCounter{ + InstanceField: "instance", + InstanceName: instance, + QueryField: "metrics." + mapCounterPathLabel(counter.Field, counter.Name), + Query: mapQuery(query.Name, instance, counter.Name), + Format: counter.Format, + ObjectName: query.Name, + ObjectField: "object", + }) + } + } + + } + + } + } +} + +func mapQuery(obj string, instance string, path string) string { + var query string + if strings.HasPrefix(obj, "\\") { + query = obj + } else { + query = fmt.Sprintf("\\%s", obj) + } + if instance != "" { + query += fmt.Sprintf("(%s)", instance) + } + if strings.HasPrefix(path, "\\") { + query += path + } else { + query += fmt.Sprintf("\\%s", path) + } + return query +} + +func mapCounterPathLabel(label string, path string) string { + var resultMetricName string + if label != "" { + resultMetricName = label + } else { + resultMetricName = path + } + // replace spaces with underscores + resultMetricName = strings.Replace(resultMetricName, " ", "_", -1) + // replace backslashes with "per" + resultMetricName = strings.Replace(resultMetricName, "/sec", "_per_sec", -1) + resultMetricName = strings.Replace(resultMetricName, "/_sec", "_per_sec", -1) + resultMetricName = strings.Replace(resultMetricName, "\\", "_", -1) + // replace actual percentage symbol with the smbol "pct" + resultMetricName = strings.Replace(resultMetricName, "_%_", "_pct_", -1) + // create an object in case of ":" + resultMetricName = strings.Replace(resultMetricName, ":", "_", -1) + // create an object in case of ":" + resultMetricName = strings.Replace(resultMetricName, "_-_", "_", -1) + // replace uppercases with underscores + resultMetricName = replaceUpperCase(resultMetricName) + + // avoid cases as this "logicaldisk_avg._disk_sec_per_transfer" + obj := strings.Split(resultMetricName, ".") + for index := range obj { + // in some cases a trailing "_" is found + obj[index] = strings.TrimPrefix(obj[index], "_") + obj[index] = strings.TrimSuffix(obj[index], "_") + } + resultMetricName = strings.ToLower(strings.Join(obj, "_")) + + return resultMetricName +} + +// replaceUpperCase func will replace upper case with '_' +func replaceUpperCase(src string) string { + replaceUpperCaseRegexp := regexp.MustCompile(replaceUpperCaseRegex) + return replaceUpperCaseRegexp.ReplaceAllStringFunc(src, func(str string) string { + var newStr string + for _, r := range str { + // split into fields based on class of unicode character + if unicode.IsUpper(r) { + newStr += "_" + strings.ToLower(string(r)) + } else { + newStr += string(r) + } + } + return newStr + }) +} From a782840cbe580e0ef83f87bf4396fd97a96eb94b Mon Sep 17 00:00:00 2001 From: Mariana Date: Fri, 10 Apr 2020 13:59:09 +0200 Subject: [PATCH 04/15] work on perf --- metricbeat/module/windows/perfmon/config.go | 12 ++--- metricbeat/module/windows/perfmon/data.go | 2 +- metricbeat/module/windows/perfmon/reader.go | 54 ++++++++++--------- .../module/iis/website/manifest.yml | 15 ------ 4 files changed, 33 insertions(+), 50 deletions(-) diff --git a/metricbeat/module/windows/perfmon/config.go b/metricbeat/module/windows/perfmon/config.go index 9ae3bc79ea01..d2090983545b 100644 --- a/metricbeat/module/windows/perfmon/config.go +++ b/metricbeat/module/windows/perfmon/config.go @@ -20,9 +20,11 @@ package perfmon import ( - "github.com/elastic/beats/v7/libbeat/common/cfgwarn" - "github.com/pkg/errors" "strings" + + "github.com/pkg/errors" + + "github.com/elastic/beats/v7/libbeat/common/cfgwarn" ) const replaceUpperCaseRegex = `(?:[^A-Z_\W])([A-Z])[^A-Z]` @@ -34,7 +36,6 @@ type Config struct { Counters []Counter `config:"perfmon.counters"` Queries []Query `config:"perfmon.queries"` GroupAllCountersTo string `config:"perfmon.group_all_counter"` - MetricFormat bool } // Counter for the perfmon counters (old implementation deprecated). @@ -69,10 +70,6 @@ func (conf *Config) ValidateConfig() error { cfgwarn.Deprecate("8.0", "perfmon.counters configuration option is deprecated and will be remove in the future major version, we advise using the perfmon.queries configuration option instead") } - if len(conf.Queries) > 0 { - conf.MetricFormat = true - } - // add default format in the config for i, value := range conf.Counters { form := strings.ToLower(value.Format) @@ -99,7 +96,6 @@ func (conf *Config) ValidateConfig() error { q.Format, q.Field) } } - } return nil } diff --git a/metricbeat/module/windows/perfmon/data.go b/metricbeat/module/windows/perfmon/data.go index d4284dec427c..bcbd0cfad79f 100644 --- a/metricbeat/module/windows/perfmon/data.go +++ b/metricbeat/module/windows/perfmon/data.go @@ -79,7 +79,7 @@ func (re *Reader) groupToEvents(counters map[string][]pdh.CounterValue) []mb.Eve } else { event.MetricSetFields.Put(counter.QueryField, 0) } - if re.config.MetricFormat { + if counter.ObjectField != "" { event.MetricSetFields.Put(counter.ObjectField, counter.ObjectName) } } diff --git a/metricbeat/module/windows/perfmon/reader.go b/metricbeat/module/windows/perfmon/reader.go index 4601706b4be0..eb6d3f9d9eae 100644 --- a/metricbeat/module/windows/perfmon/reader.go +++ b/metricbeat/module/windows/perfmon/reader.go @@ -33,7 +33,11 @@ import ( "github.com/elastic/beats/v7/metricbeat/mb" ) -const instanceCountLabel = ":count" +const ( + instanceCountLabel = ":count" + defaultInstanceField = "instance" + defaultObjectField = "object" +) // Reader will contain the config options type Reader struct { @@ -62,8 +66,9 @@ func NewReader(config Config) (*Reader, error) { return nil, err } r := &Reader{ - query: query, - log: logp.NewLogger("perfmon"), + query: query, + log: logp.NewLogger("perfmon"), + config: config, } r.mapCounters(config) for i, counter := range r.counters { @@ -100,7 +105,6 @@ func NewReader(config Config) (*Reader, error) { r.counters[i].ChildQueries = append(r.counters[i].ChildQueries, v) } } - r.config = config return r, nil } @@ -200,31 +204,31 @@ func (re *Reader) mapCounters(config Config) { if len(config.Queries) > 0 { for _, query := range config.Queries { for _, counter := range query.Counters { + // counter paths can also not contain any instances if len(query.Instance) == 0 { re.counters = append(re.counters, PerfCounter{ - InstanceField: "instance", + InstanceField: defaultInstanceField, InstanceName: "", - QueryField: "metrics." + mapCounterPathLabel(counter.Field, counter.Name), + QueryField: mapCounterPathLabel(counter.Field, counter.Name), Query: mapQuery(query.Name, "", counter.Name), Format: counter.Format, ObjectName: query.Name, + ObjectField: defaultObjectField, }) } else { for _, instance := range query.Instance { re.counters = append(re.counters, PerfCounter{ - InstanceField: "instance", + InstanceField: defaultInstanceField, InstanceName: instance, - QueryField: "metrics." + mapCounterPathLabel(counter.Field, counter.Name), + QueryField: mapCounterPathLabel(counter.Field, counter.Name), Query: mapQuery(query.Name, instance, counter.Name), Format: counter.Format, ObjectName: query.Name, - ObjectField: "object", + ObjectField: defaultObjectField, }) } } - } - } } } @@ -248,37 +252,35 @@ func mapQuery(obj string, instance string, path string) string { } func mapCounterPathLabel(label string, path string) string { - var resultMetricName string + resultMetricName := "metrics." if label != "" { - resultMetricName = label - } else { - resultMetricName = path + return resultMetricName + label } // replace spaces with underscores - resultMetricName = strings.Replace(resultMetricName, " ", "_", -1) + path = strings.Replace(path, " ", "_", -1) // replace backslashes with "per" - resultMetricName = strings.Replace(resultMetricName, "/sec", "_per_sec", -1) - resultMetricName = strings.Replace(resultMetricName, "/_sec", "_per_sec", -1) - resultMetricName = strings.Replace(resultMetricName, "\\", "_", -1) + path = strings.Replace(path, "/sec", "_per_sec", -1) + path = strings.Replace(path, "/_sec", "_per_sec", -1) + path = strings.Replace(path, "\\", "_", -1) // replace actual percentage symbol with the smbol "pct" - resultMetricName = strings.Replace(resultMetricName, "_%_", "_pct_", -1) + path = strings.Replace(path, "_%_", "_pct_", -1) // create an object in case of ":" - resultMetricName = strings.Replace(resultMetricName, ":", "_", -1) + path = strings.Replace(path, ":", "_", -1) // create an object in case of ":" - resultMetricName = strings.Replace(resultMetricName, "_-_", "_", -1) + path = strings.Replace(path, "_-_", "_", -1) // replace uppercases with underscores - resultMetricName = replaceUpperCase(resultMetricName) + path = replaceUpperCase(path) // avoid cases as this "logicaldisk_avg._disk_sec_per_transfer" - obj := strings.Split(resultMetricName, ".") + obj := strings.Split(path, ".") for index := range obj { // in some cases a trailing "_" is found obj[index] = strings.TrimPrefix(obj[index], "_") obj[index] = strings.TrimSuffix(obj[index], "_") } - resultMetricName = strings.ToLower(strings.Join(obj, "_")) + path = strings.ToLower(strings.Join(obj, "_")) - return resultMetricName + return resultMetricName + path } // replaceUpperCase func will replace upper case with '_' diff --git a/x-pack/metricbeat/module/iis/website/manifest.yml b/x-pack/metricbeat/module/iis/website/manifest.yml index 3e41b942fd5e..fd73b1b85938 100644 --- a/x-pack/metricbeat/module/iis/website/manifest.yml +++ b/x-pack/metricbeat/module/iis/website/manifest.yml @@ -10,37 +10,22 @@ input: instance: "*" counters: - name: 'Total Bytes Received' - field: total_bytes_received - name: 'Total Bytes Sent' - field: total_bytes_sent - name: "Bytes Sent/sec" - field: bytes_sent_per_sec - name: "Bytes Received/sec" - field: bytes_received_per_sec - name: "Current Connections" - field: current_connections - name: "Maximum Connections" - field: maximum_connections - name: "Total Connection Attempts (all instances)" field: total_connection_attempts - name: "Total Get Requests" - field: total_get_requests - name: "Get Requests/sec" - field: get_requests_per_sec - name: "Total Post Requests" - field: total_post_requests - name: "Post Requests/sec" - field: post_requests_per_sec - name: "Total Delete Requests" - field: total_delete_requests - name: "Delete Requests/sec" - field: delete_requests_per_sec - name: "Service Uptime" - field: service_uptime - name: "Total PUT Requests" - field: total_put_requests - name: "PUT Requests/sec" - field: put_requests_per_sec processors: - drop_event.when.equals: From af5219c30df2a4a6f6071cd0cc7c3b0b0811ff12 Mon Sep 17 00:00:00 2001 From: Mariana Date: Tue, 14 Apr 2020 14:02:11 +0200 Subject: [PATCH 05/15] Create tests --- .../module/windows/perfmon/config_test.go | 33 ++++ metricbeat/module/windows/perfmon/data.go | 4 +- .../module/windows/perfmon/data_test.go | 147 ++++++++++++++++ metricbeat/module/windows/perfmon/reader.go | 57 ++++--- .../perfmon/reader_integration_test.go | 86 ++++++++++ .../module/windows/perfmon/reader_test.go | 159 ++++++++++++------ 6 files changed, 416 insertions(+), 70 deletions(-) create mode 100644 metricbeat/module/windows/perfmon/reader_integration_test.go diff --git a/metricbeat/module/windows/perfmon/config_test.go b/metricbeat/module/windows/perfmon/config_test.go index f99cee71d6ce..61791ea2d1c7 100644 --- a/metricbeat/module/windows/perfmon/config_test.go +++ b/metricbeat/module/windows/perfmon/config_test.go @@ -18,3 +18,36 @@ // +build windows package perfmon + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestValidateConfig(t *testing.T) { + config := Config{} + err := config.ValidateConfig() + assert.Error(t, err, "no perfmon counters or queries have been configured") + config.Counters = []Counter{ + { + MeasurementLabel: "processor.time.total.pct", + Query: `UDPv4\Datagrams Sent/sec`, + }, + } + config.Queries = []Query{ + { + Name: "UDPv4", + Counters: []QueryCounter{ + { + Name: "Datagrams Sent/sec", + }, + }, + }, + } + err = config.ValidateConfig() + assert.NoError(t, err) + assert.Equal(t, config.Counters[0].Format, "float") + assert.Equal(t, config.Queries[0].Counters[0].Format, "float") + +} diff --git a/metricbeat/module/windows/perfmon/data.go b/metricbeat/module/windows/perfmon/data.go index bcbd0cfad79f..7db0a338de22 100644 --- a/metricbeat/module/windows/perfmon/data.go +++ b/metricbeat/module/windows/perfmon/data.go @@ -92,7 +92,7 @@ func (re *Reader) groupToEvents(counters map[string][]pdh.CounterValue) []mb.Eve return events } -func (re *Reader) groupToEvent(counters map[string][]pdh.CounterValue) mb.Event { +func (re *Reader) groupToSingleEvent(counters map[string][]pdh.CounterValue) mb.Event { event := mb.Event{ MetricSetFields: common.MapStr{}, } @@ -111,6 +111,8 @@ func (re *Reader) groupToEvent(counters map[string][]pdh.CounterValue) mb.Event switch val.Measurement.(type) { case int64: counterVal = float64(val.Measurement.(int64)) + case int: + counterVal = float64(val.Measurement.(int)) default: counterVal = val.Measurement.(float64) } diff --git a/metricbeat/module/windows/perfmon/data_test.go b/metricbeat/module/windows/perfmon/data_test.go index f99cee71d6ce..616d87d686f5 100644 --- a/metricbeat/module/windows/perfmon/data_test.go +++ b/metricbeat/module/windows/perfmon/data_test.go @@ -18,3 +18,150 @@ // +build windows package perfmon + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/metricbeat/helper/windows/pdh" +) + +func TestGroupToEvents(t *testing.T) { + reader := Reader{ + query: pdh.Query{}, + executed: true, + log: nil, + counters: []PerfCounter{ + { + QueryField: "datagrams_sent_per_sec", + QueryName: `\UDPv4\Datagrams Sent/sec`, + Format: "float", + ObjectName: "UDPv4", + ObjectField: "object", + ChildQueries: []string{`\UDPv4\Datagrams Sent/sec`}, + }, + }, + } + counters := map[string][]pdh.CounterValue{ + `\UDPv4\Datagrams Sent/sec`: { + { + Instance: "", + Measurement: 23, + Err: nil, + }, + }, + } + events := reader.groupToEvents(counters) + assert.NotNil(t, events) + assert.Equal(t, len(events), 1) + ok, err := events[0].MetricSetFields.HasKey("datagrams_sent_per_sec") + assert.NoError(t, err) + assert.True(t, ok) + ok, err = events[0].MetricSetFields.HasKey("object") + assert.NoError(t, err) + assert.True(t, ok) + val, err := events[0].MetricSetFields.GetValue("datagrams_sent_per_sec") + assert.NoError(t, err) + assert.Equal(t, val, 23) + val, err = events[0].MetricSetFields.GetValue("object") + assert.NoError(t, err) + assert.Equal(t, val, "UDPv4") + +} + +func TestGroupToSingleEvent(t *testing.T) { + reader := Reader{ + query: pdh.Query{}, + executed: true, + log: nil, + config: Config{ + GroupAllCountersTo: "processor_count", + }, + counters: []PerfCounter{ + { + QueryField: "%_processor_time", + QueryName: `\Processor Information(*)\% Processor Time`, + Format: "float", + ObjectName: "Processor Information", + ObjectField: "object", + InstanceName: "*", + InstanceField: "instance", + ChildQueries: []string{`\Processor Information(processor0)\% Processor Time`, `\Processor Information(processor1)\% Processor Time`}, + }, + { + QueryField: "%_user_time", + QueryName: `\Processor Information(*)\% User Time`, + Format: "float", + ObjectName: "Processor Information", + ObjectField: "object", + InstanceName: "*", + InstanceField: "instance", + ChildQueries: []string{`\Processor Information(processor0)\% User Time`, `\Processor Information(processor1)\% User Time`}, + }, + }, + } + + counters := map[string][]pdh.CounterValue{ + `\Processor Information(processor0)\% Processor Time`: { + { + Instance: "processor0", + Measurement: 23, + }, + }, + `\Processor Information(processor1)\% Processor Time`: { + { + Instance: "processor1", + Measurement: 21, + }, + }, + `\Processor Information(processor0)\% User Time`: { + { + Instance: "processor0", + Measurement: 10, + }, + }, + `\Processor Information(processor1)\% User Time`: { + { + Instance: "processor1", + Measurement: 11, + }, + }, + } + event := reader.groupToSingleEvent(counters) + assert.NotNil(t, event) + ok, err := event.MetricSetFields.HasKey("%_processor_time") + assert.NoError(t, err) + assert.True(t, ok) + ok, err = event.MetricSetFields.HasKey("%_processor_time:count") + assert.NoError(t, err) + assert.True(t, ok) + val, err := event.MetricSetFields.GetValue("%_processor_time") + assert.NoError(t, err) + assert.Equal(t, val, float64(44)) + val, err = event.MetricSetFields.GetValue("%_processor_time:count") + assert.NoError(t, err) + assert.Equal(t, val, common.MapStr{"processor_count": float64(2)}) + ok, err = event.MetricSetFields.HasKey("%_user_time") + assert.NoError(t, err) + assert.True(t, ok) + ok, err = event.MetricSetFields.HasKey("%_user_time:count") + assert.NoError(t, err) + assert.True(t, ok) + val, err = event.MetricSetFields.GetValue("%_user_time") + assert.NoError(t, err) + assert.Equal(t, val, float64(21)) + val, err = event.MetricSetFields.GetValue("%_user_time:count") + assert.NoError(t, err) + assert.Equal(t, val, common.MapStr{"processor_count": float64(2)}) +} + +func TestMatchesParentProcess(t *testing.T) { + ok, val := matchesParentProcess("svchost") + assert.False(t, ok) + assert.Equal(t, val, "svchost") + ok, val = matchesParentProcess("svchost#54") + assert.True(t, ok) + assert.Equal(t, val, "svchost") +} diff --git a/metricbeat/module/windows/perfmon/reader.go b/metricbeat/module/windows/perfmon/reader.go index eb6d3f9d9eae..5a44a956f759 100644 --- a/metricbeat/module/windows/perfmon/reader.go +++ b/metricbeat/module/windows/perfmon/reader.go @@ -52,7 +52,7 @@ type PerfCounter struct { InstanceField string InstanceName string QueryField string - Query string + QueryName string Format string ObjectName string ObjectField string @@ -73,34 +73,34 @@ func NewReader(config Config) (*Reader, error) { r.mapCounters(config) for i, counter := range r.counters { r.counters[i].ChildQueries = []string{} - childQueries, err := query.GetCounterPaths(counter.Query) + childQueries, err := query.GetCounterPaths(counter.QueryName) if err != nil { if config.IgnoreNECounters { switch err { case pdh.PDH_CSTATUS_NO_COUNTER, pdh.PDH_CSTATUS_NO_COUNTERNAME, pdh.PDH_CSTATUS_NO_INSTANCE, pdh.PDH_CSTATUS_NO_OBJECT: r.log.Infow("Ignoring non existent counter", "error", err, - logp.Namespace("perfmon"), "query", counter.Query) + logp.Namespace("perfmon"), "query", counter.QueryName) continue } } else { query.Close() - return nil, errors.Wrapf(err, `failed to expand counter (query="%v")`, counter.Query) + return nil, errors.Wrapf(err, `failed to expand counter (query="%v")`, counter.QueryName) } } // check if the pdhexpandcounterpath/pdhexpandwildcardpath functions have expanded the counter successfully. if len(childQueries) == 0 || (len(childQueries) == 1 && strings.Contains(childQueries[0], "*")) { // covering cases when PdhExpandWildCardPathW returns no counter paths or is unable to expand and the ignore_non_existent_counters flag is set if config.IgnoreNECounters { - r.log.Infow("Ignoring non existent counter", "initial query", counter.Query, + r.log.Infow("Ignoring non existent counter", "initial query", counter.QueryName, logp.Namespace("perfmon"), "expanded query", childQueries) continue } - return nil, errors.Errorf(`failed to expand counter (query="%v")`, counter.Query) + return nil, errors.Errorf(`failed to expand counter (query="%v")`, counter.QueryName) } for _, v := range childQueries { if err := query.AddCounter(v, counter.InstanceName, counter.Format, len(childQueries) > 1); err != nil { - return nil, errors.Wrapf(err, `failed to add counter (query="%v")`, counter.Query) + return nil, errors.Wrapf(err, `failed to add counter (query="%v")`, counter.QueryName) } r.counters[i].ChildQueries = append(r.counters[i].ChildQueries, v) } @@ -113,18 +113,18 @@ func (re *Reader) RefreshCounterPaths() error { var newCounters []string for i, counter := range re.counters { re.counters[i].ChildQueries = []string{} - childQueries, err := re.query.GetCounterPaths(counter.Query) + childQueries, err := re.query.GetCounterPaths(counter.QueryName) if err != nil { if re.config.IgnoreNECounters { switch err { case pdh.PDH_CSTATUS_NO_COUNTER, pdh.PDH_CSTATUS_NO_COUNTERNAME, pdh.PDH_CSTATUS_NO_INSTANCE, pdh.PDH_CSTATUS_NO_OBJECT: re.log.Infow("Ignoring non existent counter", "error", err, - logp.Namespace("perfmon"), "query", counter.Query) + logp.Namespace("perfmon"), "query", counter.QueryName) continue } } else { - return errors.Wrapf(err, `failed to expand counter (query="%v")`, counter.Query) + return errors.Wrapf(err, `failed to expand counter (query="%v")`, counter.QueryName) } } newCounters = append(newCounters, childQueries...) @@ -132,7 +132,7 @@ func (re *Reader) RefreshCounterPaths() error { if err == nil && len(childQueries) >= 1 && !strings.Contains(childQueries[0], "*") { for _, v := range childQueries { if err := re.query.AddCounter(v, counter.InstanceName, counter.Format, len(childQueries) > 1); err != nil { - return errors.Wrapf(err, "failed to add counter (query='%v')", counter.Query) + return errors.Wrapf(err, "failed to add counter (query='%v')", counter.QueryName) } re.counters[i].ChildQueries = append(re.counters[i].ChildQueries, v) } @@ -162,7 +162,7 @@ func (re *Reader) Read() ([]mb.Event, error) { var events []mb.Event // GroupAllCountersTo config option where counters for all instances are aggregated and instance count is added in the event under the string value provided by this option. if re.config.GroupAllCountersTo != "" { - event := re.groupToEvent(values) + event := re.groupToSingleEvent(values) events = append(events, event) } else { events = re.groupToEvents(values) @@ -195,7 +195,7 @@ func (re *Reader) mapCounters(config Config) { InstanceField: counter.InstanceLabel, InstanceName: counter.InstanceName, QueryField: counter.MeasurementLabel, - Query: counter.Query, + QueryName: counter.Query, Format: counter.Format, ChildQueries: nil, }) @@ -210,10 +210,10 @@ func (re *Reader) mapCounters(config Config) { InstanceField: defaultInstanceField, InstanceName: "", QueryField: mapCounterPathLabel(counter.Field, counter.Name), - Query: mapQuery(query.Name, "", counter.Name), + QueryName: mapQuery(query.Name, "", counter.Name), Format: counter.Format, ObjectName: query.Name, - ObjectField: defaultObjectField, + ObjectField: mapObjectName(query.Field), }) } else { for _, instance := range query.Instance { @@ -221,10 +221,10 @@ func (re *Reader) mapCounters(config Config) { InstanceField: defaultInstanceField, InstanceName: instance, QueryField: mapCounterPathLabel(counter.Field, counter.Name), - Query: mapQuery(query.Name, instance, counter.Name), + QueryName: mapQuery(query.Name, instance, counter.Name), Format: counter.Format, ObjectName: query.Name, - ObjectField: defaultObjectField, + ObjectField: mapObjectName(query.Field), }) } } @@ -233,16 +233,27 @@ func (re *Reader) mapCounters(config Config) { } } +func mapObjectName(objectField string) string { + if objectField != "" { + return objectField + } + return defaultObjectField +} + func mapQuery(obj string, instance string, path string) string { var query string - if strings.HasPrefix(obj, "\\") { - query = obj - } else { - query = fmt.Sprintf("\\%s", obj) - } + // trim object + obj = strings.TrimPrefix(obj, "\\") + obj = strings.TrimSuffix(obj, "\\") + query = fmt.Sprintf("\\%s", obj) + if instance != "" { + // trim instance + instance = strings.TrimPrefix(instance, "(") + instance = strings.TrimSuffix(instance, ")") query += fmt.Sprintf("(%s)", instance) } + if strings.HasPrefix(path, "\\") { query += path } else { @@ -280,6 +291,8 @@ func mapCounterPathLabel(label string, path string) string { } path = strings.ToLower(strings.Join(obj, "_")) + // avoid cases as this "logicaldisk_avg__disk_sec_per_transfer" + path = strings.Replace(path, "__", "_", -1) return resultMetricName + path } diff --git a/metricbeat/module/windows/perfmon/reader_integration_test.go b/metricbeat/module/windows/perfmon/reader_integration_test.go new file mode 100644 index 000000000000..fd19b1e5c091 --- /dev/null +++ b/metricbeat/module/windows/perfmon/reader_integration_test.go @@ -0,0 +1,86 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build integration +// +build windows + +package perfmon + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +var validQuery = `\Processor Information(_Total)\% Processor Time` + +// TestNewReaderWhenQueryPathNotProvided will check for invalid/no query. +func TestNewReaderWhenQueryPathNotProvided(t *testing.T) { + counter := Counter{Format: "float", InstanceName: "TestInstanceName"} + config := Config{ + IgnoreNECounters: false, + GroupMeasurements: false, + Counters: []Counter{counter}, + } + reader, err := NewReader(config) + assert.NotNil(t, err) + assert.Nil(t, reader) + assert.EqualValues(t, err.Error(), `failed to expand counter (query=""): no query path given`) +} + +// TestNewReaderWithValidQueryPath should successfully instantiate the reader. +func TestNewReaderWithValidQueryPath(t *testing.T) { + counter := Counter{Format: "float", InstanceName: "TestInstanceName", Query: validQuery} + config := Config{ + IgnoreNECounters: false, + GroupMeasurements: false, + Counters: []Counter{counter}, + } + reader, err := NewReader(config) + defer reader.Close() + assert.Nil(t, err) + assert.NotNil(t, reader) + assert.NotNil(t, reader.query) + assert.NotNil(t, reader.query.Handle) + assert.NotNil(t, reader.query.Counters) + assert.NotZero(t, len(reader.query.Counters)) + +} + +// TestReadSuccessfully will test the func read when it first retrieves no events (and ignored) and then starts retrieving events. +func TestReadSuccessfully(t *testing.T) { + counter := Counter{Format: "float", InstanceName: "TestInstanceName", Query: validQuery} + config := Config{ + IgnoreNECounters: false, + GroupMeasurements: false, + Counters: []Counter{counter}, + } + reader, err := NewReader(config) + if err != nil { + t.Fatal(err) + } + //Some counters, such as rate counters, require two counter values in order to compute a displayable value. In this case we call reader.Read() twice. + // For more information, see Collecting Performance Data (https://docs.microsoft.com/en-us/windows/desktop/PerfCtrs/collecting-performance-data). + events, err := reader.Read() + assert.Nil(t, err) + assert.NotNil(t, events) + assert.Zero(t, len(events)) + events, err = reader.Read() + assert.Nil(t, err) + assert.NotNil(t, events) + assert.NotZero(t, len(events)) +} diff --git a/metricbeat/module/windows/perfmon/reader_test.go b/metricbeat/module/windows/perfmon/reader_test.go index f316f1b8ae95..0d1d8e560291 100644 --- a/metricbeat/module/windows/perfmon/reader_test.go +++ b/metricbeat/module/windows/perfmon/reader_test.go @@ -23,63 +23,128 @@ import ( "testing" "github.com/stretchr/testify/assert" -) -var validQuery = `\Processor Information(_Total)\% Processor Time` + "github.com/elastic/beats/v7/metricbeat/helper/windows/pdh" +) -// TestNewReaderWhenQueryPathNotProvided will check for invalid/no query. -func TestNewReaderWhenQueryPathNotProvided(t *testing.T) { - counter := Counter{Format: "float", InstanceName: "TestInstanceName"} - config := Config{ - IgnoreNECounters: false, - GroupMeasurements: false, - Counters: []Counter{counter}, +func TestGetCounter(t *testing.T) { + reader := Reader{ + query: pdh.Query{}, + executed: true, + log: nil, + counters: []PerfCounter{ + { + QueryField: "datagrams_sent_per_sec", + QueryName: `\UDPv4\Datagrams Sent/sec`, + Format: "float", + ObjectName: "UDPv4", + ObjectField: "object", + ChildQueries: []string{`\UDPv4\Datagrams Sent/sec`}, + }, + }, } - reader, err := NewReader(config) - assert.NotNil(t, err) - assert.Nil(t, reader) - assert.EqualValues(t, err.Error(), `failed to expand counter (query=""): no query path given`) + ok, val := reader.getCounter(`\UDPv4\Datagrams Sent/sec`) + assert.True(t, ok) + assert.Equal(t, val.QueryField, "datagrams_sent_per_sec") + assert.Equal(t, val.ObjectName, "UDPv4") + } -// TestNewReaderWithValidQueryPath should successfully instantiate the reader. -func TestNewReaderWithValidQueryPath(t *testing.T) { - counter := Counter{Format: "float", InstanceName: "TestInstanceName", Query: validQuery} +func TestMapCounters(t *testing.T) { config := Config{ IgnoreNECounters: false, GroupMeasurements: false, - Counters: []Counter{counter}, + Counters: []Counter{ + { + InstanceLabel: "physical_disk.name", + InstanceName: "total", + MeasurementLabel: "physical_disk.write.time.pct", + Query: `\PhysicalDisk(*)\% Disk Write Time`, + Format: "float", + }, + }, + Queries: []Query{ + { + Name: "Process", + Instance: []string{"svchost*"}, + Counters: []QueryCounter{ + { + Name: "% Processor Time", + Format: "float", + }, + }, + }, + { + Name: "Process", + Field: "disk", + Instance: []string{"conhost*"}, + Counters: []QueryCounter{ + { + Name: "IO Read Operations/sec", + Field: "read_ops", + Format: "double", + }, + }, + }, + }, } - reader, err := NewReader(config) - defer reader.Close() - assert.Nil(t, err) - assert.NotNil(t, reader) - assert.NotNil(t, reader.query) - assert.NotNil(t, reader.query.Handle) - assert.NotNil(t, reader.query.Counters) - assert.NotZero(t, len(reader.query.Counters)) + reader := Reader{} + reader.mapCounters(config) + assert.Equal(t, len(reader.counters), 3) + for _, readerCounter := range reader.counters { + if readerCounter.InstanceField == "physical_disk.name" { + assert.Equal(t, readerCounter.InstanceName, "total") + assert.Equal(t, readerCounter.ObjectName, "") + assert.Equal(t, readerCounter.ObjectField, "") + assert.Equal(t, readerCounter.QueryField, "physical_disk.write.time.pct") + assert.Equal(t, readerCounter.QueryName, `\PhysicalDisk(*)\% Disk Write Time`) + assert.Equal(t, len(readerCounter.ChildQueries), 0) + assert.Equal(t, readerCounter.Format, "float") + } else if readerCounter.InstanceName == "svchost*" { + assert.Equal(t, readerCounter.ObjectName, "Process") + assert.Equal(t, readerCounter.ObjectField, "object") + assert.Equal(t, readerCounter.QueryField, "metrics.%_processor_time") + assert.Equal(t, readerCounter.QueryName, `\Process(svchost*)\% Processor Time`) + assert.Equal(t, len(readerCounter.ChildQueries), 0) + assert.Equal(t, readerCounter.Format, "float") + } else { + assert.Equal(t, readerCounter.InstanceName, "conhost*") + assert.Equal(t, readerCounter.ObjectName, "Process") + assert.Equal(t, readerCounter.ObjectField, "disk") + assert.Equal(t, readerCounter.QueryField, "metrics.read_ops") + assert.Equal(t, readerCounter.QueryName, `\Process(conhost*)\IO Read Operations/sec`) + assert.Equal(t, len(readerCounter.ChildQueries), 0) + assert.Equal(t, readerCounter.Format, "double") + } + } } -// TestReadSuccessfully will test the func read when it first retrieves no events (and ignored) and then starts retrieving events. -func TestReadSuccessfully(t *testing.T) { - counter := Counter{Format: "float", InstanceName: "TestInstanceName", Query: validQuery} - config := Config{ - IgnoreNECounters: false, - GroupMeasurements: false, - Counters: []Counter{counter}, - } - reader, err := NewReader(config) - if err != nil { - t.Fatal(err) - } - //Some counters, such as rate counters, require two counter values in order to compute a displayable value. In this case we call reader.Read() twice. - // For more information, see Collecting Performance Data (https://docs.microsoft.com/en-us/windows/desktop/PerfCtrs/collecting-performance-data). - events, err := reader.Read() - assert.Nil(t, err) - assert.NotNil(t, events) - assert.Zero(t, len(events)) - events, err = reader.Read() - assert.Nil(t, err) - assert.NotNil(t, events) - assert.NotZero(t, len(events)) +func TestMapQuery(t *testing.T) { + //mapQuery(obj string, instance string, path string) string { + obj := "Process" + instance := "*" + path := "% Processor Time" + result := mapQuery(obj, instance, path) + assert.Equal(t, result, `\Process(*)\% Processor Time`) + + obj = `\Process\` + instance = "(*" + result = mapQuery(obj, instance, path) + assert.Equal(t, result, `\Process(*)\% Processor Time`) +} + +func TestMapCounterPathLabel(t *testing.T) { + result := mapCounterPathLabel("", `WININET: Bytes from server`) + assert.Equal(t, result, "metrics.wininet_bytes_from_server") + result = mapCounterPathLabel("", `RSC Coalesced Packet Bucket 5 (16To31)`) + assert.Equal(t, result, "metrics.rsc_coalesced_packet_bucket_5_(16_to31)") + result = mapCounterPathLabel("", `Total Memory Usage --- Non-Paged Pool`) + assert.Equal(t, result, "metrics.total_memory_usage_---_non-paged_pool") + result = mapCounterPathLabel("", `IPv6 NBLs/sec indicated with low-resource flag`) + assert.Equal(t, result, "metrics.ipv6_nbls_per_sec_indicated_with_low-resource_flag") + result = mapCounterPathLabel("", `Queued Poison Messages Per Second`) + assert.Equal(t, result, "metrics.queued_poison_messages_per_second") + result = mapCounterPathLabel("", `I/O Log Writes Average Latency`) + assert.Equal(t, result, "metrics.i/o_log_writes_average_latency") } From d54f74a9a402583d1967d1c04dc8c042482059f8 Mon Sep 17 00:00:00 2001 From: Mariana Date: Tue, 14 Apr 2020 16:30:30 +0200 Subject: [PATCH 06/15] work on documentation --- metricbeat/docs/modules/windows.asciidoc | 13 +++-- metricbeat/metricbeat.reference.yml | 13 +++-- .../module/windows/_meta/config.reference.yml | 13 +++-- metricbeat/module/windows/_meta/config.yml | 23 ++++---- .../module/windows/perfmon/_meta/data.json | 28 ++++++---- .../windows/perfmon/_meta/docs.asciidoc | 50 ++++++++++++++++- .../module/windows/perfmon/perfmon_test.go | 54 ++++++++++++++----- metricbeat/modules.d/windows.yml.disabled | 23 ++++---- 8 files changed, 150 insertions(+), 67 deletions(-) diff --git a/metricbeat/docs/modules/windows.asciidoc b/metricbeat/docs/modules/windows.asciidoc index e343a359dccd..e2c11a344fce 100644 --- a/metricbeat/docs/modules/windows.asciidoc +++ b/metricbeat/docs/modules/windows.asciidoc @@ -24,11 +24,14 @@ metricbeat.modules: period: 10s perfmon.ignore_non_existent_counters: false perfmon.group_measurements_by_instance: false - perfmon.counters: - # - instance_label: processor.name - # instance_name: total - # measurement_label: processor.time.total.pct - # query: '\Processor Information(_Total)\% Processor Time' + perfmon.queries: +# - object: 'Process' +# instance: ["*"] +# counters: +# - name: 'Disk Writes/sec' +# field: physical_disk.write.per_sec +# format: "float" +# - name: "% Disk Write Time" - module: windows metricsets: ["service"] diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 7d0b9219e9a0..46e24d32bcf5 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -834,11 +834,14 @@ metricbeat.modules: period: 10s perfmon.ignore_non_existent_counters: false perfmon.group_measurements_by_instance: false - perfmon.counters: - # - instance_label: processor.name - # instance_name: total - # measurement_label: processor.time.total.pct - # query: '\Processor Information(_Total)\% Processor Time' + perfmon.queries: +# - object: 'Process' +# instance: ["*"] +# counters: +# - name: 'Disk Writes/sec' +# field: physical_disk.write.per_sec +# format: "float" +# - name: "% Disk Write Time" - module: windows metricsets: ["service"] diff --git a/metricbeat/module/windows/_meta/config.reference.yml b/metricbeat/module/windows/_meta/config.reference.yml index dc9b27a9c3d1..49656feb554a 100644 --- a/metricbeat/module/windows/_meta/config.reference.yml +++ b/metricbeat/module/windows/_meta/config.reference.yml @@ -4,11 +4,14 @@ period: 10s perfmon.ignore_non_existent_counters: false perfmon.group_measurements_by_instance: false - perfmon.counters: - # - instance_label: processor.name - # instance_name: total - # measurement_label: processor.time.total.pct - # query: '\Processor Information(_Total)\% Processor Time' + perfmon.queries: +# - object: 'Process' +# instance: ["*"] +# counters: +# - name: 'Disk Writes/sec' +# field: physical_disk.write.per_sec +# format: "float" +# - name: "% Disk Write Time" - module: windows metricsets: ["service"] diff --git a/metricbeat/module/windows/_meta/config.yml b/metricbeat/module/windows/_meta/config.yml index 3d3269a3b079..82de4c51e171 100644 --- a/metricbeat/module/windows/_meta/config.yml +++ b/metricbeat/module/windows/_meta/config.yml @@ -5,18 +5,13 @@ #- module: windows # metricsets: -# - perfmon +# - perfmon # period: 10s -# perfmon.counters: -# - instance_label: processor.name -# instance_name: total -# measurement_label: processor.time.total.pct -# query: '\Processor Information(_Total)\% Processor Time' -# -# - instance_label: physical_disk.name -# measurement_label: physical_disk.write.per_sec -# query: '\PhysicalDisk(*)\Disk Writes/sec' -# -# - instance_label: physical_disk.name -# measurement_label: physical_disk.write.time.pct -# query: '\PhysicalDisk(*)\% Disk Write Time' +# perfmon.queries: +# - object: 'Process' +# instance: ["*"] +# counters: +# - name: 'Disk Writes/sec' +# field: physical_disk.write.per_sec +# format: "float" +# - name: "% Disk Write Time" diff --git a/metricbeat/module/windows/perfmon/_meta/data.json b/metricbeat/module/windows/perfmon/_meta/data.json index e6b39dd85880..4e4d87240783 100644 --- a/metricbeat/module/windows/perfmon/_meta/data.json +++ b/metricbeat/module/windows/perfmon/_meta/data.json @@ -1,24 +1,30 @@ { "@timestamp": "2017-10-12T08:05:34.853Z", - "beat": { - "hostname": "host.example.com", - "name": "host.example.com" + "event": { + "dataset": "windows.perfmon", + "duration": 115000, + "module": "windows" }, "metricset": { - "module": "windows", "name": "perfmon", - "rtt": 115 + "period": 10000 + }, + "service": { + "type": "windows" }, "windows": { "perfmon": { - "processor": { - "name": "_Total", - "time": { - "total": { - "pct": 1.4663385364361736 + "instance": "_Total", + "metrics": { + "processor": { + "time": { + "total": { + "pct": 6.310940413107646 + } } } - } + }, + "object": "Processor Information" } } } \ No newline at end of file diff --git a/metricbeat/module/windows/perfmon/_meta/docs.asciidoc b/metricbeat/module/windows/perfmon/_meta/docs.asciidoc index 4c90de92fdd8..e573b1f8824e 100644 --- a/metricbeat/module/windows/perfmon/_meta/docs.asciidoc +++ b/metricbeat/module/windows/perfmon/_meta/docs.asciidoc @@ -15,6 +15,28 @@ to collect. The example below collects processor time and disk writes every period: 10s perfmon.ignore_non_existent_counters: true perfmon.group_measurements_by_instance: true + perfmon.queries: + - object: "Process" + instance: ["svchost*", "conhost*"] + counters: + - name: "% Processor Time" + field: time.processor.pct + format: "float" + - name: "Thread Count" + field: thread_count + - name: "IO Read Operations/sec" + - object: "PhysicalDisk" + field : "disk" + instance: "*" + counters: + - name: "Disk Writes/sec" + - name: "% Disk Write Time" + field: "write_time" + format: "float" + + + // deprecated, will be removed in 8.0 + perfmon.counters: - instance_label: processor.name instance_name: total @@ -46,7 +68,33 @@ counter requires three config options - `instance_label`, `measurement_label`, and `query`. [float] -==== Counter Configuration +==== Query Configuration + +Each item in the `query` list specifies multiple perfmon queries to perform. In the +events generated by the metricset these configuration options map to the field +values as shown below. + +*`object`*:: The performance object to query. A performance object can be a physical component, such as processors, disks, and memory, or a system object, such as processes and threads. Required + +*`field`*:: The object field/label. Not required, if not entered, it will be `object`. + +*`instance`*:: Matches the ParentInstance, ObjectInstance, and InstanceIndex are included in the path if multiple instances of the object can exist. Not required for performance counters which do not contain one. + +*`counters`*:: List of the partial counter paths (At least one partial counter path is required). + +*`name`*:: The counter name. Required. This is the counter specified in Performance Data Helper (PDH) syntax. For example in case of the counter path `\Processor Information(_Total)\% Processor Time`, +the value for this configuration option will be `% Processor Time`. + +*`field`*:: The counter path value field/label. Not required, if not entered, it will be generated based on the counter path. + +*`format`*:: Format of the measurement value. The value can be either `float` or +`long`. The default is `float`. + + + + +[float] +==== Deprecated Counter Configuration Each item in the `counters` list specifies a perfmon query to perform. In the events generated by the metricset these configuration options map to the field diff --git a/metricbeat/module/windows/perfmon/perfmon_test.go b/metricbeat/module/windows/perfmon/perfmon_test.go index 274d83b8b7f3..772d8b0a9ab0 100644 --- a/metricbeat/module/windows/perfmon/perfmon_test.go +++ b/metricbeat/module/windows/perfmon/perfmon_test.go @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. -// +build integration // +build windows package perfmon @@ -27,8 +26,6 @@ import ( "github.com/elastic/beats/v7/metricbeat/helper/windows/pdh" - "github.com/elastic/beats/v7/libbeat/common" - "github.com/pkg/errors" "github.com/stretchr/testify/assert" @@ -39,6 +36,37 @@ import ( const processorTimeCounter = `\Processor Information(_Total)\% Processor Time` func TestData(t *testing.T) { + config := map[string]interface{}{ + "module": "windows", + "metricsets": []string{"perfmon"}, + "perfmon.queries": []map[string]interface{}{ + { + "object": "Processor Information", + "instance": []string{"_Total"}, + "counters": []map[string]interface{}{ + { + "name": "% Processor Time", + "field": "processor.time.total.pct", + }, + { + "name": "% User Time", + }, + }, + }, + }, + } + + ms := mbtest.NewReportingMetricSetV2Error(t, config) + mbtest.ReportingFetchV2Error(ms) + time.Sleep(60 * time.Millisecond) + + if err := mbtest.WriteEventsReporterV2Error(ms, t, "/"); err != nil { + t.Fatal("write", err) + } + +} + +func TestDataOld(t *testing.T) { config := map[string]interface{}{ "module": "windows", "metricsets": []string{"perfmon"}, @@ -81,12 +109,14 @@ func TestCounterWithNoInstanceName(t *testing.T) { config := map[string]interface{}{ "module": "windows", "metricsets": []string{"perfmon"}, - "perfmon.counters": []map[string]string{ + "perfmon.queries": []map[string]interface{}{ { - "instance_label": "processor.name", - "measurement_label": "processor.time.total.pct", - "query": `\UDPv4\Datagrams Sent/sec`, - //"query": `\UDPv4\Verzonden datagrammen per seconde`, + "object": "UDPv4", + "counters": []map[string]interface{}{ + { + "name": "Datagrams Sent/sec", + }, + }, }, }, } @@ -102,9 +132,10 @@ func TestCounterWithNoInstanceName(t *testing.T) { if len(events) == 0 { t.Fatal("no events received") } - process := events[0].MetricSetFields["processor"].(common.MapStr) + val, err := events[0].MetricSetFields.GetValue("object") + assert.NoError(t, err) // Check values - assert.EqualValues(t, "UDPv4", process["name"]) + assert.EqualValues(t, "UDPv4", val) } @@ -115,12 +146,11 @@ func TestQuery(t *testing.T) { t.Fatal(err) } defer q.Close() - counter := Counter{Format: "float", InstanceName: "TestInstanceName"} path, err := q.GetCounterPaths(processorTimeCounter) if err != nil { t.Fatal(err) } - err = q.AddCounter(path[0], counter.InstanceName, counter.Format, false) + err = q.AddCounter(path[0], "TestInstanceName", "float", false) if err != nil { t.Fatal(err) } diff --git a/metricbeat/modules.d/windows.yml.disabled b/metricbeat/modules.d/windows.yml.disabled index 059667d82f38..7d5804b55537 100644 --- a/metricbeat/modules.d/windows.yml.disabled +++ b/metricbeat/modules.d/windows.yml.disabled @@ -8,18 +8,13 @@ #- module: windows # metricsets: -# - perfmon +# - perfmon # period: 10s -# perfmon.counters: -# - instance_label: processor.name -# instance_name: total -# measurement_label: processor.time.total.pct -# query: '\Processor Information(_Total)\% Processor Time' -# -# - instance_label: physical_disk.name -# measurement_label: physical_disk.write.per_sec -# query: '\PhysicalDisk(*)\Disk Writes/sec' -# -# - instance_label: physical_disk.name -# measurement_label: physical_disk.write.time.pct -# query: '\PhysicalDisk(*)\% Disk Write Time' +# perfmon.queries: +# - object: 'Process' +# instance: ["*"] +# counters: +# - name: 'Disk Writes/sec' +# field: physical_disk.write.per_sec +# format: "float" +# - name: "% Disk Write Time" From c360b9b6d6e1d43ebff1a54d7ce06e693c799255 Mon Sep 17 00:00:00 2001 From: Mariana Date: Tue, 14 Apr 2020 16:35:03 +0200 Subject: [PATCH 07/15] fix test --- metricbeat/module/windows/perfmon/perfmon_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/metricbeat/module/windows/perfmon/perfmon_test.go b/metricbeat/module/windows/perfmon/perfmon_test.go index 772d8b0a9ab0..df6757f08492 100644 --- a/metricbeat/module/windows/perfmon/perfmon_test.go +++ b/metricbeat/module/windows/perfmon/perfmon_test.go @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +// +build integration // +build windows package perfmon From aec06157a3007e3054ecc236b402eeeb63afd97e Mon Sep 17 00:00:00 2001 From: Mariana Date: Wed, 15 Apr 2020 13:20:17 +0200 Subject: [PATCH 08/15] changelog --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index c3356cbc880c..e194192ec728 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -282,6 +282,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add Storage metricsets to GCP module {pull}15598[15598] - Added documentation for running Metricbeat in Cloud Foundry. {pull}17275[17275] - Add final tests and move label to GA for the azure module in metricbeat. {pull}17319[17319] +- Refactor windows/perfmon metricset configuration options and event output. {pull}17596[17596] *Packetbeat* From 3d5101e5a811eab9edd2814e7188d09e1ab53bd9 Mon Sep 17 00:00:00 2001 From: Mariana Date: Wed, 15 Apr 2020 17:56:53 +0200 Subject: [PATCH 09/15] work on webserver --- metricbeat/docs/fields.asciidoc | 13 +- metricbeat/docs/modules/windows.asciidoc | 9 +- metricbeat/module/windows/_meta/config.yml | 4 +- metricbeat/module/windows/_meta/docs.asciidoc | 9 +- metricbeat/module/windows/perfmon/config.go | 18 +- .../module/windows/perfmon/perfmon_test.go | 203 ++++++++++++++-- metricbeat/module/windows/perfmon/reader.go | 13 +- metricbeat/modules.d/windows.yml.disabled | 4 +- x-pack/metricbeat/metricbeat.reference.yml | 13 +- x-pack/metricbeat/module/iis/fields.go | 2 +- .../module/iis/webserver/manifest.yml | 225 ++++++------------ .../module/iis/website/_meta/data.json | 31 ++- .../module/iis/website/_meta/fields.yml | 6 +- .../module/iis/website/manifest.yml | 38 +-- 14 files changed, 325 insertions(+), 263 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 55e3f8e772da..a98ea628b697 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -18292,20 +18292,13 @@ type: object -- -[float] -=== website - -website - - - -*`iis.website.name`*:: +*`iis.website.*.*`*:: + -- -website name +website -type: keyword +type: object -- diff --git a/metricbeat/docs/modules/windows.asciidoc b/metricbeat/docs/modules/windows.asciidoc index e2c11a344fce..09bf17b6947e 100644 --- a/metricbeat/docs/modules/windows.asciidoc +++ b/metricbeat/docs/modules/windows.asciidoc @@ -5,8 +5,13 @@ This file is generated! See scripts/mage/docs_collector.go [[metricbeat-module-windows]] == Windows module -This is the Windows module. It collects metrics from Windows systems, -by default metricset `service` is enabled. +This is the Windows module which collects metrics from Windows systems. +The module contains the `service` metricset, which is set up by default when the windows module is enabled. +The `service` metricset will retrieve status information of the services on the Windows machines. The other windows +metricset is `perfmon` which collects windows performance counter values. + + + [float] diff --git a/metricbeat/module/windows/_meta/config.yml b/metricbeat/module/windows/_meta/config.yml index 82de4c51e171..c9ddb73e5492 100644 --- a/metricbeat/module/windows/_meta/config.yml +++ b/metricbeat/module/windows/_meta/config.yml @@ -1,6 +1,6 @@ - module: windows - #metricsets: - # - service + metricsets: + - service period: 1m #- module: windows diff --git a/metricbeat/module/windows/_meta/docs.asciidoc b/metricbeat/module/windows/_meta/docs.asciidoc index 3a14b2eb03f7..f9c1aaf0a049 100644 --- a/metricbeat/module/windows/_meta/docs.asciidoc +++ b/metricbeat/module/windows/_meta/docs.asciidoc @@ -1,2 +1,7 @@ -This is the Windows module. It collects metrics from Windows systems, -by default metricset `service` is enabled. +This is the Windows module which collects metrics from Windows systems. +The module contains the `service` metricset, which is set up by default when the windows module is enabled. +The `service` metricset will retrieve status information of the services on the Windows machines. The other windows +metricset is `perfmon` which collects windows performance counter values. + + + diff --git a/metricbeat/module/windows/perfmon/config.go b/metricbeat/module/windows/perfmon/config.go index d2090983545b..2b61dcbbe425 100644 --- a/metricbeat/module/windows/perfmon/config.go +++ b/metricbeat/module/windows/perfmon/config.go @@ -49,10 +49,11 @@ type Counter struct { // QueryConfig for perfmon queries. This will be used as the new configuration format type Query struct { - Name string `config:"object" validate:"required"` - Field string `config:"field"` - Instance []string `config:"instance"` - Counters []QueryCounter `config:"counters" validate:"required"` + Name string `config:"object" validate:"required"` + Field string `config:"field"` + Instance []string `config:"instance"` + Counters []QueryCounter `config:"counters" validate:"required"` + Namespace string `config:"namespace"` } // QueryConfigCounter for perfmon queries. This will be used as the new configuration format @@ -83,12 +84,15 @@ func (conf *Config) ValidateConfig() error { value.Format, value.InstanceLabel) } } - for _, value := range conf.Queries { - for i, q := range value.Counters { + for i, value := range conf.Queries { + if value.Namespace == "" { + conf.Queries[i].Namespace = "metrics" + } + for j, q := range value.Counters { form := strings.ToLower(q.Format) switch form { case "", "float": - value.Counters[i].Format = "float" + value.Counters[j].Format = "float" case "long", "large": default: return errors.Errorf("initialization failed: format '%s' "+ diff --git a/metricbeat/module/windows/perfmon/perfmon_test.go b/metricbeat/module/windows/perfmon/perfmon_test.go index df6757f08492..069138a4226d 100644 --- a/metricbeat/module/windows/perfmon/perfmon_test.go +++ b/metricbeat/module/windows/perfmon/perfmon_test.go @@ -67,7 +67,7 @@ func TestData(t *testing.T) { } -func TestDataOld(t *testing.T) { +func TestDataDeprecated(t *testing.T) { config := map[string]interface{}{ "module": "windows", "metricsets": []string{"perfmon"}, @@ -180,6 +180,30 @@ func TestQuery(t *testing.T) { } func TestExistingCounter(t *testing.T) { + config := Config{ + Queries: make([]Query, 1), + } + config.Queries[0].Name = "Processor Information" + config.Queries[0].Instance = []string{"_Total"} + config.Queries[0].Counters = []QueryCounter{ + { + Name: "% Processor Time", + }, + } + handle, err := NewReader(config) + if err != nil { + t.Fatal(err) + } + defer handle.query.Close() + + values, err := handle.Read() + if err != nil { + t.Fatal(err) + } + t.Log(values) +} + +func TestExistingCounterDeprecated(t *testing.T) { config := Config{ Counters: make([]Counter, 1), } @@ -202,6 +226,28 @@ func TestExistingCounter(t *testing.T) { } func TestNonExistingCounter(t *testing.T) { + config := Config{ + Queries: make([]Query, 1), + } + config.Queries[0].Name = "Processor Information" + config.Queries[0].Instance = []string{"_Total"} + config.Queries[0].Counters = []QueryCounter{ + { + Name: "% Processor Time time", + }, + } + handle, err := NewReader(config) + if assert.Error(t, err) { + assert.EqualValues(t, pdh.PDH_CSTATUS_NO_COUNTER, errors.Cause(err)) + } + + if handle != nil { + err = handle.query.Close() + assert.NoError(t, err) + } +} + +func TestNonExistingCounterDeprecated(t *testing.T) { config := Config{ Counters: make([]Counter, 1), } @@ -221,6 +267,34 @@ func TestNonExistingCounter(t *testing.T) { } func TestIgnoreNonExistentCounter(t *testing.T) { + config := Config{ + Queries: make([]Query, 1), + IgnoreNECounters: true, + } + config.Queries[0].Name = "Processor Information" + config.Queries[0].Instance = []string{"_Total"} + config.Queries[0].Counters = []QueryCounter{ + { + Name: "% Processor Time time", + }, + } + handle, err := NewReader(config) + + values, err := handle.Read() + + if assert.Error(t, err) { + assert.EqualValues(t, pdh.PDH_NO_DATA, errors.Cause(err)) + } + + if handle != nil { + err = handle.query.Close() + assert.NoError(t, err) + } + + t.Log(values) +} + +func TestIgnoreNonExistentCounterDeprecated(t *testing.T) { config := Config{ Counters: make([]Counter, 1), IgnoreNECounters: true, @@ -246,6 +320,28 @@ func TestIgnoreNonExistentCounter(t *testing.T) { } func TestNonExistingObject(t *testing.T) { + config := Config{ + Queries: make([]Query, 1), + } + config.Queries[0].Name = "Processor MisInformation" + config.Queries[0].Instance = []string{"_Total"} + config.Queries[0].Counters = []QueryCounter{ + { + Name: "% Processor Time", + }, + } + handle, err := NewReader(config) + if assert.Error(t, err) { + assert.EqualValues(t, pdh.PDH_CSTATUS_NO_OBJECT, errors.Cause(err)) + } + + if handle != nil { + err = handle.query.Close() + assert.NoError(t, err) + } +} + +func TestNonExistingObjectDeprecated(t *testing.T) { config := Config{ Counters: make([]Counter, 1), } @@ -271,13 +367,12 @@ func TestLongOutputFormat(t *testing.T) { t.Fatal(err) } defer query.Close() - counter := Counter{Format: "long"} path, err := query.GetCounterPaths(processorTimeCounter) if err != nil { t.Fatal(err) } assert.NotZero(t, len(path)) - err = query.AddCounter(path[0], counter.InstanceName, counter.Format, false) + err = query.AddCounter(path[0], "", "long", false) if err != nil && err != pdh.PDH_NO_MORE_DATA { t.Fatal(err) } @@ -311,13 +406,12 @@ func TestFloatOutputFormat(t *testing.T) { t.Fatal(err) } defer query.Close() - counter := Counter{Format: "float"} path, err := query.GetCounterPaths(processorTimeCounter) if err != nil { t.Fatal(err) } assert.NotZero(t, len(path)) - err = query.AddCounter(path[0], counter.InstanceName, counter.Format, false) + err = query.AddCounter(path[0], "", "float", false) if err != nil && err != pdh.PDH_NO_MORE_DATA { t.Fatal(err) } @@ -346,13 +440,15 @@ func TestFloatOutputFormat(t *testing.T) { func TestWildcardQuery(t *testing.T) { config := Config{ - Counters: make([]Counter, 1), + Queries: make([]Query, 1), + } + config.Queries[0].Name = "Processor Information" + config.Queries[0].Instance = []string{"*"} + config.Queries[0].Counters = []QueryCounter{ + { + Name: "% Processor Time", + }, } - config.Counters[0].InstanceLabel = "processor.name" - config.Counters[0].InstanceName = "TestInstanceName" - config.Counters[0].MeasurementLabel = "processor.time.pct" - config.Counters[0].Query = `\Processor Information(*)\% Processor Time` - config.Counters[0].Format = "float" handle, err := NewReader(config) if err != nil { t.Fatal(err) @@ -368,28 +464,26 @@ func TestWildcardQuery(t *testing.T) { t.Fatal(err) } assert.NotZero(t, len(values)) - pctKey, err := values[0].MetricSetFields.HasKey("processor.time.pct") + pctKey, err := values[0].MetricSetFields.HasKey("metrics.%_processor_time") if err != nil { t.Fatal(err) } assert.True(t, pctKey) - - pct, err := values[0].MetricSetFields.GetValue("processor.name") - if err != nil { - t.Fatal(err) - } - assert.NotEqual(t, "TestInstanceName", pct) - t.Log(values) } func TestWildcardQueryNoInstanceName(t *testing.T) { config := Config{ - Counters: make([]Counter, 1), + Queries: make([]Query, 1), + } + config.Queries[0].Name = "Process" + config.Queries[0].Instance = []string{"*"} + config.Queries[0].Counters = []QueryCounter{ + { + Name: "Private Bytes", + }, } - config.Counters[0].InstanceLabel = "process_private_bytes" - config.Counters[0].MeasurementLabel = "process.private.bytes" - config.Counters[0].Query = `\Process(*)\Private Bytes` + handle, err := NewReader(config) if err != nil { t.Fatal(err) @@ -405,24 +499,81 @@ func TestWildcardQueryNoInstanceName(t *testing.T) { t.Fatal(err) } assert.NotZero(t, len(values)) - pctKey, err := values[0].MetricSetFields.HasKey("process.private.bytes") + pctKey, err := values[0].MetricSetFields.HasKey("metrics.private_bytes") if err != nil { t.Fatal(err) } assert.True(t, pctKey) for _, s := range values { - pct, err := s.MetricSetFields.GetValue("process_private_bytes") + instance, err := s.MetricSetFields.GetValue("instance") if err != nil { t.Fatal(err) } - assert.False(t, strings.Contains(pct.(string), "*")) + assert.False(t, strings.Contains(instance.(string), "*")) } t.Log(values) } func TestGroupByInstance(t *testing.T) { + config := Config{ + Queries: make([]Query, 1), + GroupMeasurements: true, + } + config.Queries[0].Name = "Processor Information" + config.Queries[0].Instance = []string{"_Total"} + config.Queries[0].Counters = []QueryCounter{ + { + Name: "% Processor Time", + }, + { + Name: "% User Time", + }, + { + Name: "% Privileged Time", + }, + } + handle, err := NewReader(config) + if err != nil { + t.Fatal(err) + } + defer handle.query.Close() + + values, _ := handle.Read() + + time.Sleep(time.Millisecond * 1000) + + values, err = handle.Read() + if err != nil { + t.Fatal(err) + } + + assert.EqualValues(t, 1, len(values)) // Assert all metrics have been grouped into a single event + + // Test all keys exist in the event + pctKey, err := values[0].MetricSetFields.HasKey("metrics.%_processor_time") + if err != nil { + t.Fatal(err) + } + assert.True(t, pctKey) + + pctKey, err = values[0].MetricSetFields.HasKey("metrics.%_user_time") + if err != nil { + t.Fatal(err) + } + assert.True(t, pctKey) + + pctKey, err = values[0].MetricSetFields.HasKey("metrics.%_privileged_time") + if err != nil { + t.Fatal(err) + } + assert.True(t, pctKey) + + t.Log(values) +} + +func TestGroupByInstanceDeprecated(t *testing.T) { config := Config{ Counters: make([]Counter, 3), GroupMeasurements: true, diff --git a/metricbeat/module/windows/perfmon/reader.go b/metricbeat/module/windows/perfmon/reader.go index 5a44a956f759..04aa897ffe4f 100644 --- a/metricbeat/module/windows/perfmon/reader.go +++ b/metricbeat/module/windows/perfmon/reader.go @@ -96,7 +96,7 @@ func NewReader(config Config) (*Reader, error) { logp.Namespace("perfmon"), "expanded query", childQueries) continue } - return nil, errors.Errorf(`failed to expand counter (query="%v")`, counter.QueryName) + return nil, errors.Errorf(`failed to expand counter (query="%v"), no error returned`, counter.QueryName) } for _, v := range childQueries { if err := query.AddCounter(v, counter.InstanceName, counter.Format, len(childQueries) > 1); err != nil { @@ -209,7 +209,7 @@ func (re *Reader) mapCounters(config Config) { re.counters = append(re.counters, PerfCounter{ InstanceField: defaultInstanceField, InstanceName: "", - QueryField: mapCounterPathLabel(counter.Field, counter.Name), + QueryField: mapCounterPathLabel(query.Namespace, counter.Field, counter.Name), QueryName: mapQuery(query.Name, "", counter.Name), Format: counter.Format, ObjectName: query.Name, @@ -220,7 +220,7 @@ func (re *Reader) mapCounters(config Config) { re.counters = append(re.counters, PerfCounter{ InstanceField: defaultInstanceField, InstanceName: instance, - QueryField: mapCounterPathLabel(counter.Field, counter.Name), + QueryField: mapCounterPathLabel(query.Namespace, counter.Field, counter.Name), QueryName: mapQuery(query.Name, instance, counter.Name), Format: counter.Format, ObjectName: query.Name, @@ -262,10 +262,9 @@ func mapQuery(obj string, instance string, path string) string { return query } -func mapCounterPathLabel(label string, path string) string { - resultMetricName := "metrics." +func mapCounterPathLabel(namespace string, label string, path string) string { if label != "" { - return resultMetricName + label + return namespace + "." + label } // replace spaces with underscores path = strings.Replace(path, " ", "_", -1) @@ -293,7 +292,7 @@ func mapCounterPathLabel(label string, path string) string { // avoid cases as this "logicaldisk_avg__disk_sec_per_transfer" path = strings.Replace(path, "__", "_", -1) - return resultMetricName + path + return namespace + "." + path } // replaceUpperCase func will replace upper case with '_' diff --git a/metricbeat/modules.d/windows.yml.disabled b/metricbeat/modules.d/windows.yml.disabled index 7d5804b55537..c2d8c0f8f6d3 100644 --- a/metricbeat/modules.d/windows.yml.disabled +++ b/metricbeat/modules.d/windows.yml.disabled @@ -2,8 +2,8 @@ # Docs: https://www.elastic.co/guide/en/beats/metricbeat/master/metricbeat-module-windows.html - module: windows - #metricsets: - # - service + metricsets: + - service period: 1m #- module: windows diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 462338ad3995..80ea98747cd3 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -1242,11 +1242,14 @@ metricbeat.modules: period: 10s perfmon.ignore_non_existent_counters: false perfmon.group_measurements_by_instance: false - perfmon.counters: - # - instance_label: processor.name - # instance_name: total - # measurement_label: processor.time.total.pct - # query: '\Processor Information(_Total)\% Processor Time' + perfmon.queries: +# - object: 'Process' +# instance: ["*"] +# counters: +# - name: 'Disk Writes/sec' +# field: physical_disk.write.per_sec +# format: "float" +# - name: "% Disk Write Time" - module: windows metricsets: ["service"] diff --git a/x-pack/metricbeat/module/iis/fields.go b/x-pack/metricbeat/module/iis/fields.go index 8a124fdbef0d..18bad877cf93 100644 --- a/x-pack/metricbeat/module/iis/fields.go +++ b/x-pack/metricbeat/module/iis/fields.go @@ -19,5 +19,5 @@ func init() { // AssetIis returns asset data. // This is the base64 encoded gzipped contents of module/iis. func AssetIis() string { - return "eJy0kbGOgzAQRHt/xYgSKfkAF/cryMAm2ovBlu1cxN+ffAYOiEE02YJidzTzzFzwoEGC2QsgcNAkUTD7QgAt+caxDWx6iS8BIOrQmfapSQCONClPEjUFJQRwY9Ktl3/KC3rV0eQcJwyWJO7OPO24yQSsTZZGylrNjYriyhqjZ0HOOc4a73+fTU1zkLHFWqLF7+owIT1oeBnXbm4HABsIRIi1/ZT5otqT+yF3La/liVcnIFN/UxMW67So0vWmjdo5Vp2ylvv7qCzK4twfnTHnbfYlHOgTfb5bv9e4W+J+hYcFjpnJ7zcAAP//navhqA==" + return "eJzMkrGOgzAQRHt/xYgSKfkAF/cryMAm2ovBlr25iL8/+QwcIEAp44JidjTzFu0FDxo0mKMChMWSRsEcCwW0FJvAXtj1Gl8KQPKhc+3TkgICWTKRNGoSoxRwY7Jt1H/OC3rT0ZScngyeNO7BPf2o7BSsQ5ZBxnvLjUnmyjtnZ8NecnprvH99tzW/k44t1hItfVeDCelBw8uFdjM7AdhAIEGs46fOF9WRwg+Fa3kt39g6A7n6mxpZyFmo8vRmnTkYVp3xnvv76CzK4r0/OmPO6u4mLPT5e7DQ6UEcnsPxMZyewtiZ834DAAD///e79wo=" } diff --git a/x-pack/metricbeat/module/iis/webserver/manifest.yml b/x-pack/metricbeat/module/iis/webserver/manifest.yml index c8d148a21c13..32da4b384b60 100644 --- a/x-pack/metricbeat/module/iis/webserver/manifest.yml +++ b/x-pack/metricbeat/module/iis/webserver/manifest.yml @@ -6,158 +6,81 @@ input: perfmon.group_measurements_by_instance: true perfmon.ignore_non_existent_counters: true perfmon.group_all_counter: "worker_process_count" - perfmon.counters: - #network - - instance_label: '' - measurement_label: network.total_bytes_received - query: '\Web Service(_Total)\Total Bytes Received' - - instance_label: '' - measurement_label: network.total_bytes_sent - query: '\Web Service(_Total)\Total Bytes Sent' - - instance_label: '' - measurement_label: network.bytes_sent_per_sec - query: '\Web Service(_Total)\Bytes Sent/sec' - - instance_label: '' - measurement_label: network.bytes_received_per_sec - query: '\Web Service(_Total)\Bytes Received/sec' - - instance_label: '' - measurement_label: network.current_connections - query: '\Web Service(_Total)\Current Connections' - - instance_label: '' - measurement_label: network.maximum_connections - query: '\Web Service(_Total)\Maximum Connections' - - instance_label: '' - measurement_label: network.total_connection_attempts - query: '\Web Service(_Total)\Total Connection Attempts (all instances)' - - instance_label: '' - measurement_label: network.total_get_requests - query: '\Web Service(_Total)\Total Get Requests' - - instance_label: '' - measurement_label: network.get_requests_per_sec - query: '\Web Service(_Total)\Get Requests/sec' - - instance_label: '' - measurement_label: network.total_post_requests - query: '\Web Service(_Total)\Total Post Requests' - - instance_label: '' - measurement_label: network.post_requests_per_sec - query: '\Web Service(_Total)\Post Requests/sec' - - instance_label: '' - measurement_label: network.total_delete_requests - query: '\Web Service(_Total)\Total Delete Requests' - - instance_label: '' - measurement_label: network.delete_requests_per_sec - query: '\Web Service(_Total)\Delete Requests/sec' - - instance_label: '' - measurement_label: network.service_uptime - query: '\Web Service(_Total)\Service Uptime' - - instance_label: '' - measurement_label: network.current_anonymous_users - query: '\Web Service(_Total)\Current Anonymous Users' - - instance_label: '' - measurement_label: network.current_nonanonymous_users - query: '\Web Service(_Total)\Current NonAnonymous Users' - - instance_label: '' - measurement_label: network.total_anonymous_users - query: '\Web Service(_Total)\Total Anonymous Users' - - instance_label: '' - measurement_label: network.anonymous_users_per_sec - query: '\Web Service(_Total)\Anonymous Users/sec' - - instance_label: '' - measurement_label: network.total_nonanonymous_users - query: '\Web Service(_Total)\Total NonAnonymous Users' - + perfmon.queries: + - object: "Web Service" + instance: "_Total" + namespace : "network" + counters: + # network + - name: "Total Bytes Received" + - name: "Total Bytes Sent" + - name: "Bytes Sent/sec" + - name: "Bytes Received/sec" + - name: "Current Connections" + - name: "Maximum Connections" + - name: "Total Connection Attempts (all instances)" + field: "total_connection_attempts" + - name: "Total Get Requests" + - name: "Get Requests/sec" + - name: "Total Post Requests" + - name: "Post Requests/sec" + - name: "Total Delete Requests" + - name: "Delete Requests/sec" + - name: "Service Uptime" + - name: "Current Anonymous Users" + - name: "Current NonAnonymous Users" + - name: "Total Anonymous Users" + - name: "Anonymous Users/sec" + - name: "Total NonAnonymous Users" #asp.net - - instance_label: '' - measurement_label: asp_net_application.errors_per_sec - query: '\ASP.NET Applications(__Total__)\Errors Total/Sec' - - instance_label: '' - measurement_label: asp_net_application.pipeline_instance_count - query: '\ASP.NET Applications(__Total__)\Pipeline Instance Count' - - instance_label: '' - measurement_label: asp_net_application.requests_executing - query: '\ASP.NET Applications(__Total__)\Requests Executing' - - instance_label: '' - measurement_label: asp_net_application.requests_in_application_queue - query: '\ASP.NET Applications(__Total__)\Requests in Application Queue' - format: 'large' - - instance_label: '' - measurement_label: asp_net_application.requests_per_sec - query: '\ASP.NET Applications(__Total__)\Requests/Sec' - - instance_label: '' - measurement_label: asp_net.application_restarts - query: '\ASP.NET\Application Restarts' - - instance_label: '' - measurement_label: asp_net.request_wait_time - query: '\ASP.NET\Request Wait Time' + - object: "ASP.NET Applications" + instance: "__Total__" + namespace: "asp_net" + counters: + - name: "Application Restarts" + - name: "Request Wait Time" + #asp_net_application + - object: "ASP.NET Applications" + instance: "__Total__" + namespace: "asp_net_application" + counters: + - name: "Errors Total/Sec" + - name: "Pipeline Instance Count" + - name: "Requests Executing" + - name: "Requests in Application Queue" + format: 'large' + - name: "Requests/Sec" #cache - - instance_label: '' - measurement_label: cache.current_files_cached - query: '\Web Service Cache\Current Files Cached' - - instance_label: '' - measurement_label: cache.total_files_cached - query: '\Web Service Cache\Total Files Cached' - - instance_label: '' - measurement_label: cache.file_cache_hits - query: '\Web Service Cache\File Cache Hits' - - instance_label: '' - measurement_label: cache.file_cache_misses - query: '\Web Service Cache\File Cache Misses' - - instance_label: '' - measurement_label: cache.current_file_cache_memory_usage - query: '\Web Service Cache\Current File Cache Memory Usage' - - instance_label: '' - measurement_label: cache.maximum_file_cache_memory_usage - query: '\Web Service Cache\Maximum File Cache Memory Usage' - - instance_label: '' - measurement_label: cache.current_uris_cached - query: '\Web Service Cache\Current URIs Cached' - - instance_label: '' - measurement_label: cache.total_uris_cached - query: '\Web Service Cache\Total URIs Cached' - - instance_label: '' - measurement_label: cache.uri_cache_hits - query: '\Web Service Cache\URI Cache Hits' - - instance_label: '' - measurement_label: cache.uri_cache_misses - query: '\Web Service Cache\URI Cache Misses' - - instance_label: '' - measurement_label: cache.output_cache_current_memory_usage - query: '\Web Service Cache\Output Cache Current Memory Usage' - - instance_label: '' - measurement_label: cache.output_cache_current_items - query: '\Web Service Cache\Output Cache Current Items' - - instance_label: '' - measurement_label: cache.output_cache_total_hits - query: '\Web Service Cache\Output Cache Total Hits' - - instance_label: '' - measurement_label: cache.output_cache_total_misses - query: '\Web Service Cache\Output Cache Total Misses' + - object: "Web Service Cache" + namespace: "cache" + counters: + - name: "Current Files Cached" + - name: "Total Files Cached" + - name: "File Cache Hits" + - name: "File Cache Misses" + - name: "Current File Cache Memory Usage" + - name: "Maximum File Cache Memory Usage" + - name: "Current URIs Cached" + - name: "Total URIs Cached" + - name: "URI Cache Hits" + - name: "URI Cache Misses" + - name: "Output Cache Current Memory Usage" + - name: "Output Cache Current Items" + - name: "Output Cache Total Hits" + - name: "Output Cache Total Misses" #process - - instance_label: '' - measurement_label: process.cpu_usage_perc - query: '\Process(w3wp*)\% Processor Time' - - instance_label: '' - measurement_label: process.handle_count - query: '\Process(w3wp*)\Handle Count' - - instance_label: '' - measurement_label: process.thread_count - query: '\Process(w3wp*)\Thread Count' - - instance_label: '' - measurement_label: process.working_set - query: '\Process(w3wp*)\Working Set' - - instance_label: '' - measurement_label: process.private_byte - query: '\Process(w3wp*)\Private Bytes' - - instance_label: '' - measurement_label: process.virtual_bytes - query: '\Process(w3wp*)\Virtual Bytes' - - instance_label: '' - measurement_label: process.page_faults_per_Sec - query: '\Process(w3wp*)\Page Faults/sec' - - instance_label: '' - measurement_label: process.io_read_operations_per_sec - query: '\Process(w3wp*)\IO Read Operations/sec' - - instance_label: '' - measurement_label: process.io_write_operations_per_sec - query: '\Process(w3wp*)\IO Write Operations/sec' + - object: "Process" + field: "process" + namespace: "process" + instance: "w3wp*" + counters: + - name: "% Processor Time" + - name: "Handle Count" + - name: "Thread Count" + - name: "Working Set" + - name: "Private Bytes" + - name: "Virtual Bytes" + - name: "Page Faults/sec" + - name: "IO Read Operations/sec" + - name: "IO Write Operations/sec" diff --git a/x-pack/metricbeat/module/iis/website/_meta/data.json b/x-pack/metricbeat/module/iis/website/_meta/data.json index d9804730f977..e2a6ed9905c2 100644 --- a/x-pack/metricbeat/module/iis/website/_meta/data.json +++ b/x-pack/metricbeat/module/iis/website/_meta/data.json @@ -7,17 +7,24 @@ }, "iis": { "website": { - "current_connections": 0, - "maximum_connections": 1, - "name": "test2.local", - "service_uptime": 346586, - "total_bytes_received": 1666, - "total_bytes_sent": 84224, - "total_connection_attempts": 2, - "total_delete_requests": 0, - "total_get_requests": 4, - "total_post_requests": 0, - "total_put_requests": 0 + "network": { + "bytes_sent_per_sec": 0, + "maximum_connections": 1, + "total_post_requests": 0, + "post_requests_per_sec": 0, + "total_connection_attempts": 1, + "service_uptime": 114161, + "get_requests_per_sec": 0, + "total_put_requests": 0, + "current_connections": 0, + "total_delete_requests": 0, + "bytes_received_per_sec": 0, + "put_requests_per_sec": 0, + "total_bytes_sent": 944, + "total_get_requests": 1, + "delete_requests_per_sec": 0 + }, + "name": "Default Web Site" } }, "metricset": { @@ -27,4 +34,4 @@ "service": { "type": "iis" } -} \ No newline at end of file +} diff --git a/x-pack/metricbeat/module/iis/website/_meta/fields.yml b/x-pack/metricbeat/module/iis/website/_meta/fields.yml index 74be20459303..1bd4ebc3accc 100644 --- a/x-pack/metricbeat/module/iis/website/_meta/fields.yml +++ b/x-pack/metricbeat/module/iis/website/_meta/fields.yml @@ -1,6 +1,8 @@ -- name: website - type: group +- name: website.*.* release: beta + type: object + object_type: float + object_type_mapping_type: "*" description: > website fields: diff --git a/x-pack/metricbeat/module/iis/website/manifest.yml b/x-pack/metricbeat/module/iis/website/manifest.yml index fd73b1b85938..62ebfd27ed85 100644 --- a/x-pack/metricbeat/module/iis/website/manifest.yml +++ b/x-pack/metricbeat/module/iis/website/manifest.yml @@ -8,6 +8,7 @@ input: perfmon.queries: - object: 'Web Service' instance: "*" + namespace : "network" counters: - name: 'Total Bytes Received' - name: 'Total Bytes Sent' @@ -30,45 +31,14 @@ input: processors: - drop_event.when.equals: iis.website.name: '_Total' +- drop_fields: + fields: "iis.website.object" - rename: ignore_missing: true fields: - - from: "iis.website.metrics.total_bytes_received" - to: "iis.website.total_bytes_received" - - from: "iis.website.metrics.total_bytes_sent" - to: "iis.website.total_bytes_sent" - - from: "iis.website.metrics.bytes_sent_per_sec" - to: "iis.website.bytes_sent_per_sec" - - from: "iis.website.metrics.bytes_received_per_sec" - to: "iis.website.bytes_received_per_sec" - - from: "iis.website.metrics.current_connections" - to: "iis.website.current_connections" - - from: "iis.website.metrics.maximum_connections" - to: "iis.website.maximum_connections" - - from: "iis.website.metrics.total_connection_attempts" - to: "iis.website.total_connection_attempts" - - from: "iis.website.metrics.total_get_requests" - to: "iis.website.total_get_requests" - - from: "iis.website.metrics.get_requests_per_sec" - to: "iis.website.get_requests_per_sec" - - from: "iis.website.metrics.total_post_requests" - to: "iis.website.total_post_requests" - - from: "iis.website.metrics.post_requests_per_sec" - to: "iis.website.post_requests_per_sec" - - from: "iis.website.metrics.total_delete_requests" - to: "iis.website.total_delete_requests" - - from: "iis.website.metrics.delete_requests_per_sec" - to: "iis.website.delete_requests_per_sec" - - from: "iis.website.metrics.service_uptime" - to: "iis.website.service_uptime" - - from: "iis.website.metrics.total_put_requests" - to: "iis.website.total_put_requests" - - from: "iis.website.metrics.put_requests_per_sec" - to: "iis.website.put_requests_per_sec" - from: "iis.website.instance" to: "iis.website.name" -- drop_fields: - fields: ["iis.website.object", "iis.website.metrics"] + From 644217af60d8ed55a22a0a05bc801771b001553f Mon Sep 17 00:00:00 2001 From: Mariana Date: Thu, 16 Apr 2020 12:00:34 +0200 Subject: [PATCH 10/15] fix test --- metricbeat/module/windows/perfmon/reader_test.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/metricbeat/module/windows/perfmon/reader_test.go b/metricbeat/module/windows/perfmon/reader_test.go index 0d1d8e560291..8fe252eea171 100644 --- a/metricbeat/module/windows/perfmon/reader_test.go +++ b/metricbeat/module/windows/perfmon/reader_test.go @@ -135,16 +135,16 @@ func TestMapQuery(t *testing.T) { } func TestMapCounterPathLabel(t *testing.T) { - result := mapCounterPathLabel("", `WININET: Bytes from server`) + result := mapCounterPathLabel("metrics", "", `WININET: Bytes from server`) assert.Equal(t, result, "metrics.wininet_bytes_from_server") - result = mapCounterPathLabel("", `RSC Coalesced Packet Bucket 5 (16To31)`) + result = mapCounterPathLabel("metrics", "", `RSC Coalesced Packet Bucket 5 (16To31)`) assert.Equal(t, result, "metrics.rsc_coalesced_packet_bucket_5_(16_to31)") - result = mapCounterPathLabel("", `Total Memory Usage --- Non-Paged Pool`) + result = mapCounterPathLabel("metrics", "", `Total Memory Usage --- Non-Paged Pool`) assert.Equal(t, result, "metrics.total_memory_usage_---_non-paged_pool") - result = mapCounterPathLabel("", `IPv6 NBLs/sec indicated with low-resource flag`) + result = mapCounterPathLabel("metrics", "", `IPv6 NBLs/sec indicated with low-resource flag`) assert.Equal(t, result, "metrics.ipv6_nbls_per_sec_indicated_with_low-resource_flag") - result = mapCounterPathLabel("", `Queued Poison Messages Per Second`) + result = mapCounterPathLabel("metrics", "", `Queued Poison Messages Per Second`) assert.Equal(t, result, "metrics.queued_poison_messages_per_second") - result = mapCounterPathLabel("", `I/O Log Writes Average Latency`) + result = mapCounterPathLabel("metrics", "", `I/O Log Writes Average Latency`) assert.Equal(t, result, "metrics.i/o_log_writes_average_latency") } From 6416739793fc9e7a8647ae177ddcebfab2e157a4 Mon Sep 17 00:00:00 2001 From: Mariana Date: Mon, 20 Apr 2020 13:07:20 +0200 Subject: [PATCH 11/15] add test --- metricbeat/docs/modules/windows.asciidoc | 8 +-- metricbeat/module/windows/_meta/docs.asciidoc | 8 +-- metricbeat/module/windows/perfmon/config.go | 71 +++++++++++++------ .../module/windows/perfmon/config_test.go | 40 +++++++---- metricbeat/module/windows/perfmon/perfmon.go | 4 -- metricbeat/module/windows/perfmon/reader.go | 28 ++++---- .../module/windows/perfmon/reader_test.go | 13 ++-- 7 files changed, 104 insertions(+), 68 deletions(-) diff --git a/metricbeat/docs/modules/windows.asciidoc b/metricbeat/docs/modules/windows.asciidoc index 09bf17b6947e..60faf2cf6421 100644 --- a/metricbeat/docs/modules/windows.asciidoc +++ b/metricbeat/docs/modules/windows.asciidoc @@ -5,10 +5,10 @@ This file is generated! See scripts/mage/docs_collector.go [[metricbeat-module-windows]] == Windows module -This is the Windows module which collects metrics from Windows systems. -The module contains the `service` metricset, which is set up by default when the windows module is enabled. -The `service` metricset will retrieve status information of the services on the Windows machines. The other windows -metricset is `perfmon` which collects windows performance counter values. +This is the `windows` module which collects metrics from Windows systems. +The module contains the `service` metricset, which is set up by default when the `windows` module is enabled. +The `service` metricset will retrieve status information of the services on the Windows machines. The second `windows` +metricset is `perfmon` which collects Windows performance counter values. diff --git a/metricbeat/module/windows/_meta/docs.asciidoc b/metricbeat/module/windows/_meta/docs.asciidoc index f9c1aaf0a049..b7ed8584d22c 100644 --- a/metricbeat/module/windows/_meta/docs.asciidoc +++ b/metricbeat/module/windows/_meta/docs.asciidoc @@ -1,7 +1,7 @@ -This is the Windows module which collects metrics from Windows systems. -The module contains the `service` metricset, which is set up by default when the windows module is enabled. -The `service` metricset will retrieve status information of the services on the Windows machines. The other windows -metricset is `perfmon` which collects windows performance counter values. +This is the `windows` module which collects metrics from Windows systems. +The module contains the `service` metricset, which is set up by default when the `windows` module is enabled. +The `service` metricset will retrieve status information of the services on the Windows machines. The second `windows` +metricset is `perfmon` which collects Windows performance counter values. diff --git a/metricbeat/module/windows/perfmon/config.go b/metricbeat/module/windows/perfmon/config.go index 2b61dcbbe425..c6075cf4a750 100644 --- a/metricbeat/module/windows/perfmon/config.go +++ b/metricbeat/module/windows/perfmon/config.go @@ -29,6 +29,8 @@ import ( const replaceUpperCaseRegex = `(?:[^A-Z_\W])([A-Z])[^A-Z]` +var allowedFormats = []string{"float", "large", "long"} + // Config for the windows perfmon metricset. type Config struct { IgnoreNECounters bool `config:"perfmon.ignore_non_existent_counters"` @@ -63,43 +65,70 @@ type QueryCounter struct { Format string `config:"format"` } -func (conf *Config) ValidateConfig() error { +func (query *Query) InitDefaults() { + query.Namespace = "metrics" +} + +func (counter *QueryCounter) InitDefaults() { + form := strings.ToLower(counter.Format) + switch form { + case "", "float": + counter.Format = "float" + case "long", "large": + } +} + +func (counter *Counter) InitDefaults() { + form := strings.ToLower(counter.Format) + switch form { + case "", "float": + counter.Format = "float" + case "long", "large": + } +} + +func (conf *Config) Validate() error { if len(conf.Counters) == 0 && len(conf.Queries) == 0 { return errors.New("no perfmon counters or queries have been configured") } if len(conf.Counters) > 0 { - cfgwarn.Deprecate("8.0", "perfmon.counters configuration option is deprecated and will be remove in the future major version, we advise using the perfmon.queries configuration option instead") + cfgwarn.Deprecate("8.0", "perfmon.counters configuration option is deprecated and will be removed in the future major version, "+ + "we advise using the perfmon.queries configuration option instead.") + } + if len(conf.Queries) > 0 { + for _, query := range conf.Queries { + if len(query.Counters) == 0 { + return errors.Errorf("no perfmon counters have been configured for object %s", query.Name) + } + } } - // add default format in the config - for i, value := range conf.Counters { - form := strings.ToLower(value.Format) - switch form { - case "", "float": - conf.Counters[i].Format = "float" - case "long", "large": - default: + for _, value := range conf.Counters { + if !isValidFormat(value.Format) { return errors.Errorf("initialization failed: format '%s' "+ "for counter '%s' is invalid (must be float, large or long)", value.Format, value.InstanceLabel) } + } - for i, value := range conf.Queries { - if value.Namespace == "" { - conf.Queries[i].Namespace = "metrics" - } - for j, q := range value.Counters { - form := strings.ToLower(q.Format) - switch form { - case "", "float": - value.Counters[j].Format = "float" - case "long", "large": - default: + for _, value := range conf.Queries { + for _, q := range value.Counters { + if !isValidFormat(q.Format) { return errors.Errorf("initialization failed: format '%s' "+ "for counter '%s' is invalid (must be float, large or long)", q.Format, q.Field) } + } } return nil } + +func isValidFormat(format string) bool { + for _, form := range allowedFormats { + if form == format { + return true + } + } + return false +} diff --git a/metricbeat/module/windows/perfmon/config_test.go b/metricbeat/module/windows/perfmon/config_test.go index 61791ea2d1c7..cb4957f608b8 100644 --- a/metricbeat/module/windows/perfmon/config_test.go +++ b/metricbeat/module/windows/perfmon/config_test.go @@ -22,32 +22,42 @@ package perfmon import ( "testing" + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/go-ucfg" + "github.com/stretchr/testify/assert" ) -func TestValidateConfig(t *testing.T) { - config := Config{} - err := config.ValidateConfig() - assert.Error(t, err, "no perfmon counters or queries have been configured") - config.Counters = []Counter{ - { - MeasurementLabel: "processor.time.total.pct", - Query: `UDPv4\Datagrams Sent/sec`, - }, +func TestValidate(t *testing.T) { + conf := common.MapStr{ + "module": "windows", + "period": "10s", + "metricsets": []string{"perfmon"}, + "perfmon.group_measurements_by_instance": true, } - config.Queries = []Query{ + c, err := ucfg.NewFrom(conf) + assert.NoError(t, err) + var config Config + err = c.Unpack(&config) + assert.Error(t, err, "no perfmon counters or queries have been configured") + conf["perfmon.queries"] = []common.MapStr{ { - Name: "UDPv4", - Counters: []QueryCounter{ + "object": "Process", + "counters": []common.MapStr{ { - Name: "Datagrams Sent/sec", + "name": "Thread Count", }, }, }, } - err = config.ValidateConfig() + c, err = ucfg.NewFrom(conf) + assert.NoError(t, err) + err = c.Unpack(&config) assert.NoError(t, err) - assert.Equal(t, config.Counters[0].Format, "float") assert.Equal(t, config.Queries[0].Counters[0].Format, "float") + assert.Equal(t, config.Queries[0].Namespace, "metrics") + assert.Equal(t, config.Queries[0].Name, "Process") + assert.Equal(t, config.Queries[0].Counters[0].Name, "Thread Count") + assert.True(t, config.GroupMeasurements) } diff --git a/metricbeat/module/windows/perfmon/perfmon.go b/metricbeat/module/windows/perfmon/perfmon.go index e5c11f3220be..c0490a344309 100644 --- a/metricbeat/module/windows/perfmon/perfmon.go +++ b/metricbeat/module/windows/perfmon/perfmon.go @@ -49,10 +49,6 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { if err := base.Module().UnpackConfig(&config); err != nil { return nil, err } - if err := config.ValidateConfig(); err != nil { - return nil, err - } - reader, err := NewReader(config) if err != nil { return nil, errors.Wrap(err, "initialization of reader failed") diff --git a/metricbeat/module/windows/perfmon/reader.go b/metricbeat/module/windows/perfmon/reader.go index 04aa897ffe4f..7000f25d3466 100644 --- a/metricbeat/module/windows/perfmon/reader.go +++ b/metricbeat/module/windows/perfmon/reader.go @@ -263,36 +263,36 @@ func mapQuery(obj string, instance string, path string) string { } func mapCounterPathLabel(namespace string, label string, path string) string { - if label != "" { - return namespace + "." + label + if label == "" { + label = path } // replace spaces with underscores - path = strings.Replace(path, " ", "_", -1) + label = strings.Replace(label, " ", "_", -1) // replace backslashes with "per" - path = strings.Replace(path, "/sec", "_per_sec", -1) - path = strings.Replace(path, "/_sec", "_per_sec", -1) - path = strings.Replace(path, "\\", "_", -1) + label = strings.Replace(label, "/sec", "_per_sec", -1) + label = strings.Replace(label, "/_sec", "_per_sec", -1) + label = strings.Replace(label, "\\", "_", -1) // replace actual percentage symbol with the smbol "pct" - path = strings.Replace(path, "_%_", "_pct_", -1) + label = strings.Replace(label, "_%_", "_pct_", -1) // create an object in case of ":" - path = strings.Replace(path, ":", "_", -1) + label = strings.Replace(label, ":", "_", -1) // create an object in case of ":" - path = strings.Replace(path, "_-_", "_", -1) + label = strings.Replace(label, "_-_", "_", -1) // replace uppercases with underscores - path = replaceUpperCase(path) + label = replaceUpperCase(label) // avoid cases as this "logicaldisk_avg._disk_sec_per_transfer" - obj := strings.Split(path, ".") + obj := strings.Split(label, ".") for index := range obj { // in some cases a trailing "_" is found obj[index] = strings.TrimPrefix(obj[index], "_") obj[index] = strings.TrimSuffix(obj[index], "_") } - path = strings.ToLower(strings.Join(obj, "_")) + label = strings.ToLower(strings.Join(obj, "_")) // avoid cases as this "logicaldisk_avg__disk_sec_per_transfer" - path = strings.Replace(path, "__", "_", -1) - return namespace + "." + path + label = strings.Replace(label, "__", "_", -1) + return namespace + "." + label } // replaceUpperCase func will replace upper case with '_' diff --git a/metricbeat/module/windows/perfmon/reader_test.go b/metricbeat/module/windows/perfmon/reader_test.go index 8fe252eea171..0cfb1a2edf64 100644 --- a/metricbeat/module/windows/perfmon/reader_test.go +++ b/metricbeat/module/windows/perfmon/reader_test.go @@ -65,8 +65,9 @@ func TestMapCounters(t *testing.T) { }, Queries: []Query{ { - Name: "Process", - Instance: []string{"svchost*"}, + Name: "Process", + Namespace: "metrics", + Instance: []string{"svchost*"}, Counters: []QueryCounter{ { Name: "% Processor Time", @@ -75,9 +76,10 @@ func TestMapCounters(t *testing.T) { }, }, { - Name: "Process", - Field: "disk", - Instance: []string{"conhost*"}, + Name: "Process", + Field: "disk", + Namespace: "metrics", + Instance: []string{"conhost*"}, Counters: []QueryCounter{ { Name: "IO Read Operations/sec", @@ -88,7 +90,6 @@ func TestMapCounters(t *testing.T) { }, }, } - reader := Reader{} reader.mapCounters(config) assert.Equal(t, len(reader.counters), 3) From ddc124d5f7e2380a6ab63846724e89a1e845f1a9 Mon Sep 17 00:00:00 2001 From: Mariana Date: Mon, 20 Apr 2020 14:56:28 +0200 Subject: [PATCH 12/15] validate --- .../windows/perfmon/_meta/docs.asciidoc | 4 +- metricbeat/module/windows/perfmon/config.go | 37 +++++++++---------- 2 files changed, 20 insertions(+), 21 deletions(-) diff --git a/metricbeat/module/windows/perfmon/_meta/docs.asciidoc b/metricbeat/module/windows/perfmon/_meta/docs.asciidoc index e573b1f8824e..f04c9247c049 100644 --- a/metricbeat/module/windows/perfmon/_meta/docs.asciidoc +++ b/metricbeat/module/windows/perfmon/_meta/docs.asciidoc @@ -87,7 +87,7 @@ the value for this configuration option will be `% Processor Time`. *`field`*:: The counter path value field/label. Not required, if not entered, it will be generated based on the counter path. -*`format`*:: Format of the measurement value. The value can be either `float` or +*`format`*:: Format of the measurement value. The value can be either `float`, `large` or `long`. The default is `float`. @@ -125,6 +125,6 @@ Performance Data Helper (PDH) syntax. This field is required. For example place of an instance name to perform a wildcard query that generates an event for each counter instance (e.g. `\PhysicalDisk(*)\Disk Writes/sec`). -*`format`*:: Format of the measurement value. The value can be either `float` or +*`format`*:: Format of the measurement value. The value can be either `float`, `large` or `long`. The default is `float`. diff --git a/metricbeat/module/windows/perfmon/config.go b/metricbeat/module/windows/perfmon/config.go index c6075cf4a750..1541348b88ac 100644 --- a/metricbeat/module/windows/perfmon/config.go +++ b/metricbeat/module/windows/perfmon/config.go @@ -87,6 +87,24 @@ func (counter *Counter) InitDefaults() { } } +func (counter *Counter) Validate() error { + if !isValidFormat(counter.Format) { + return errors.Errorf("initialization failed: format '%s' "+ + "for counter '%s' is invalid (must be float, large or long)", + counter.Format, counter.InstanceLabel) + } + return nil +} + +func (counter *QueryCounter) Validate() error { + if !isValidFormat(counter.Format) { + return errors.Errorf("initialization failed: format '%s' "+ + "for counter '%s' is invalid (must be float, large or long)", + counter.Format, counter.Name) + } + return nil +} + func (conf *Config) Validate() error { if len(conf.Counters) == 0 && len(conf.Queries) == 0 { return errors.New("no perfmon counters or queries have been configured") @@ -102,25 +120,6 @@ func (conf *Config) Validate() error { } } } - - for _, value := range conf.Counters { - if !isValidFormat(value.Format) { - return errors.Errorf("initialization failed: format '%s' "+ - "for counter '%s' is invalid (must be float, large or long)", - value.Format, value.InstanceLabel) - } - - } - for _, value := range conf.Queries { - for _, q := range value.Counters { - if !isValidFormat(q.Format) { - return errors.Errorf("initialization failed: format '%s' "+ - "for counter '%s' is invalid (must be float, large or long)", - q.Format, q.Field) - } - - } - } return nil } From 4dbd811916836eda83ded28db24bda9c2860a07a Mon Sep 17 00:00:00 2001 From: Mariana Date: Mon, 20 Apr 2020 16:30:45 +0200 Subject: [PATCH 13/15] dynamic mapping --- metricbeat/docs/fields.asciidoc | 27 +++++++++++++++++++ metricbeat/module/windows/fields.go | 2 +- .../module/windows/perfmon/_meta/fields.yml | 18 ++++++++++++- metricbeat/module/windows/perfmon/config.go | 9 +------ .../module/windows/perfmon/config_test.go | 10 +++++++ 5 files changed, 56 insertions(+), 10 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 02cd01398d3c..81e2d066557f 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -38965,6 +38965,33 @@ Module for Windows +[float] +=== perfmon + +perfmon + + + +*`windows.perfmon.instance`*:: ++ +-- +Instance value. + + +type: keyword + +-- + +*`windows.perfmon.metrics.*.*`*:: ++ +-- +Metric values returned. + + +type: object + +-- + [float] === service diff --git a/metricbeat/module/windows/fields.go b/metricbeat/module/windows/fields.go index 34f9f8b78612..1087083cc130 100644 --- a/metricbeat/module/windows/fields.go +++ b/metricbeat/module/windows/fields.go @@ -32,5 +32,5 @@ func init() { // AssetWindows returns asset data. // This is the base64 encoded gzipped contents of ../metricbeat/module/windows. func AssetWindows() string { - return "eJysVcFu2zgQvfsrBr30Ehvb9rQ+LOCNN7tebNOiSWEsECBiyJE1CEWqnKEd/31BSXZk2akdIzwY9JB8897MoziER1yPYUXO+BUPAITE4hjezZvIuwGAQdaBKiHvxvDHAADgszfRIuQ+wHx7lAsf5F57l9NiDLmyjAOAgBYV4xgWagCQE1rD4xpkCE6V2E2ehqyrtDn4WLWRA/l3gRqwbaIHFNWJN0kYw5I0buOHEr2YrBlZi5GB9k4UOQYpEFiURO7WYpOMR53zu3XYjL6MLmUyO+EN40dcr3zor+GTKqvUuWL+6frfK/1hUvV2/EJZGhOIjn5EhNm01lJLa3SMYCZADAoKxQX4vF4slS7I4XuGv7/PpqCcSeE93Baj1vRcj4OC0+85kueoealfJ/f2Wd4p1AxxZdX6/myKrTP+WqITuPTWohYfXs+5JVLT2nRi06ZfS2BRQe4T21cIOKWKCTdWNUyfUb2h8sz0YBGWykZkUAEhm0TxpRLS2cUeavan95JdQDYlVg8WTZp/Vi4qm13URstu1ixYZidJPrdn17cw+X77z5dvs9v/7/7zWtmbvY/ICTWaaO2jk6Zj0RkMsCpIF6C2BgzR8REplZLibCWX47v57Hr6ZX5zx3XhPn2846UuPMsInxCGj9DVB8NXfjuuorVr+BGVpZzQ1GRBfG2FnCyCFEqAEpkSnXDXI/vtJ6dtNOQWoMIi1geO91ne2NVKS1S2QT7d1ZfeCblIbnHI1l9V5HqpmTa+/hada4M3ya7bua+qZt4YPv1Hc8zx+ETpCTZvWI0rH57Tb582kII4PQqpMBiCD5DSNp3e+horH4T3IFcFuuZypi6LB27VptISw4qshQessRfoMJDuv617mB0O0VnkHZNBFfySTGrTJjTkCjXlpDsnj93BF15k693ipav34bffP55R740rDtdbMXtNStJVC14nsV9n0yPsYyVU4qjst+NFDbkPpZIxmBhUIttbJldFud9sKslaYtTemX6C01/i99yyhLY5aIDcDvZo8DMAAP//0yHt8w==" + return "eJysVlFv2zYQfvevOOSlQJEYa/s0PxTw4mXzsKRFk8IYEMCiqZN1C0WqvKMdA/vxAyXZkW05dozqoVB51Hffd99HM1fwhKsBLMmmbsk9ACExOICLSb1y0QNIkbWnUsjZAXzuAQDcujQYhMx5mGw+5dx5mWpnM5oPIFOGsQfg0aBiHMBc9QAyQpPyoAK5AqsKbDePj6zKuNm7UDYrHf23gdpgJfqscHaz3gUYnw2tGYpqrXc2q5996F0SLzTIsiircQugZvKEq6Xz6VZlq+t/WyWAcYMFC2UC9ju6FSieNPff9993NHSzf1HLVqFemtb1zDh1sDwtVFmSnTd7L95fHCb+eYf4bUWrps3gUYK3mL4I2DOP0S9oa2jd5r3SNWkwEtDOiiLLIDkCi5LA7cCum3G/KxVz9arNLaPTHdGHTQbAZ1WU8Xjlk093f93oD8NyZ8er8wQYQrD0IyCMR5WWSlqtow9jAWJQkCvOwWVVsVA6J4vvGP74Ph6Bsmlc3sNtMCpNHf60Bcd/z5E8Qc0L/Ta5Dy/yTqGWEpdGraZnU2yS8fsCrcC1Mwa1OP92zg2RitbaibVNr0tgUb4+eG8QcMoUI24oK5hdRtWG0jHTzOD6tCqPkAyDuEIJ6eRyDzX5zTlJLiEZEauZwTS+3yoblEkuq6Al9ysWLJKTJJ/r2d0DDL8//Pnl2/jhn8e/nVbmfu9H5IQZDbV2wUrtWLApeljmpHNQmwD6YPmIlFJJfraS68HjZHw3+jK5f+RqcJ8+PvJC546lj88IV0/Q1gdXb/ztuAnGrOBHUIYywrQiC+KqKGRkECRXAhTJFGiF2xnZt5+sNiElOwfl56H64LjP8pNTrbQEZWrk01N97ayQDWTnXbH+qgJXpfq1zvW3YG2zeB/junl31eW4CXz8P6bHEo/PFP9OSn/iNG6cf2m/udpAcuJ4KcTBoPfOQ2xbO73JNZbOC+9BLnO09eGMLosDbtTG0RLDkoyBGVbYc7QYr/qdu3UPs8UhWIO8FTIovVtQGm1aL11xiZoy0q0vj53BAzeycXZ+6Oh9+OXXj2fMe52K7nkrZqdJSTxq3uko9ut4dIR9KIUK7Be7dhzUkDlfKBlAGryKZHfKZMsg0/WmgowhRu1sutvg9Jv4HTcsoTEHUyC7hd3v/R8AAP//KJpZdg==" } diff --git a/metricbeat/module/windows/perfmon/_meta/fields.yml b/metricbeat/module/windows/perfmon/_meta/fields.yml index 8033a27f5ac5..9e07225e17d8 100644 --- a/metricbeat/module/windows/perfmon/_meta/fields.yml +++ b/metricbeat/module/windows/perfmon/_meta/fields.yml @@ -1 +1,17 @@ -- release: beta +- name: perfmon + type: group + release: beta + description: > + perfmon + fields: + - name: instance + type: keyword + description: | + Instance value. + - name: metrics.*.* + type: object + object_type: float + object_type_mapping_type: "*" + description: > + Metric values returned. + diff --git a/metricbeat/module/windows/perfmon/config.go b/metricbeat/module/windows/perfmon/config.go index 1541348b88ac..1868d67b62fc 100644 --- a/metricbeat/module/windows/perfmon/config.go +++ b/metricbeat/module/windows/perfmon/config.go @@ -54,7 +54,7 @@ type Query struct { Name string `config:"object" validate:"required"` Field string `config:"field"` Instance []string `config:"instance"` - Counters []QueryCounter `config:"counters" validate:"required"` + Counters []QueryCounter `config:"counters" validate:"required,nonzero"` Namespace string `config:"namespace"` } @@ -113,13 +113,6 @@ func (conf *Config) Validate() error { cfgwarn.Deprecate("8.0", "perfmon.counters configuration option is deprecated and will be removed in the future major version, "+ "we advise using the perfmon.queries configuration option instead.") } - if len(conf.Queries) > 0 { - for _, query := range conf.Queries { - if len(query.Counters) == 0 { - return errors.Errorf("no perfmon counters have been configured for object %s", query.Name) - } - } - } return nil } diff --git a/metricbeat/module/windows/perfmon/config_test.go b/metricbeat/module/windows/perfmon/config_test.go index cb4957f608b8..8f96a9ca1ed4 100644 --- a/metricbeat/module/windows/perfmon/config_test.go +++ b/metricbeat/module/windows/perfmon/config_test.go @@ -40,6 +40,16 @@ func TestValidate(t *testing.T) { var config Config err = c.Unpack(&config) assert.Error(t, err, "no perfmon counters or queries have been configured") + conf["perfmon.queries"] = []common.MapStr{ + { + "object": "Process", + }, + } + c, err = ucfg.NewFrom(conf) + assert.NoError(t, err) + err = c.Unpack(&config) + assert.Error(t, err, "missing required field accessing 'perfmon.queries.0.counters'") + conf["perfmon.queries"] = []common.MapStr{ { "object": "Process", From 561be46190585b086b8046abd99928efd1d39794 Mon Sep 17 00:00:00 2001 From: Mariana Date: Mon, 20 Apr 2020 16:49:07 +0200 Subject: [PATCH 14/15] init format --- metricbeat/module/windows/perfmon/config.go | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/metricbeat/module/windows/perfmon/config.go b/metricbeat/module/windows/perfmon/config.go index 1868d67b62fc..0e7c50296c12 100644 --- a/metricbeat/module/windows/perfmon/config.go +++ b/metricbeat/module/windows/perfmon/config.go @@ -20,8 +20,6 @@ package perfmon import ( - "strings" - "github.com/pkg/errors" "github.com/elastic/beats/v7/libbeat/common/cfgwarn" @@ -70,21 +68,11 @@ func (query *Query) InitDefaults() { } func (counter *QueryCounter) InitDefaults() { - form := strings.ToLower(counter.Format) - switch form { - case "", "float": - counter.Format = "float" - case "long", "large": - } + counter.Format = "float" } func (counter *Counter) InitDefaults() { - form := strings.ToLower(counter.Format) - switch form { - case "", "float": - counter.Format = "float" - case "long", "large": - } + counter.Format = "float" } func (counter *Counter) Validate() error { From ee0418c5966fbcdf51fc58f5d82fcd7cce286955 Mon Sep 17 00:00:00 2001 From: Mariana Date: Tue, 21 Apr 2020 09:17:02 +0200 Subject: [PATCH 15/15] work on test --- metricbeat/module/windows/_meta/config.yml | 14 ++++++------ metricbeat/module/windows/perfmon/config.go | 2 -- metricbeat/module/windows/perfmon/reader.go | 22 ++++++------------- .../module/windows/perfmon/reader_test.go | 9 ++++++++ metricbeat/modules.d/windows.yml.disabled | 14 ++++++------ 5 files changed, 30 insertions(+), 31 deletions(-) diff --git a/metricbeat/module/windows/_meta/config.yml b/metricbeat/module/windows/_meta/config.yml index c9ddb73e5492..57ab272ef237 100644 --- a/metricbeat/module/windows/_meta/config.yml +++ b/metricbeat/module/windows/_meta/config.yml @@ -8,10 +8,10 @@ # - perfmon # period: 10s # perfmon.queries: -# - object: 'Process' -# instance: ["*"] -# counters: -# - name: 'Disk Writes/sec' -# field: physical_disk.write.per_sec -# format: "float" -# - name: "% Disk Write Time" +# - object: 'Process' +# instance: ["*"] +# counters: +# - name: 'Disk Writes/sec' +# field: physical_disk.write.per_sec +# format: "float" +# - name: "% Disk Write Time" diff --git a/metricbeat/module/windows/perfmon/config.go b/metricbeat/module/windows/perfmon/config.go index 0e7c50296c12..971d4629b273 100644 --- a/metricbeat/module/windows/perfmon/config.go +++ b/metricbeat/module/windows/perfmon/config.go @@ -25,8 +25,6 @@ import ( "github.com/elastic/beats/v7/libbeat/common/cfgwarn" ) -const replaceUpperCaseRegex = `(?:[^A-Z_\W])([A-Z])[^A-Z]` - var allowedFormats = []string{"float", "large", "long"} // Config for the windows perfmon metricset. diff --git a/metricbeat/module/windows/perfmon/reader.go b/metricbeat/module/windows/perfmon/reader.go index 7000f25d3466..c65c4a8118ae 100644 --- a/metricbeat/module/windows/perfmon/reader.go +++ b/metricbeat/module/windows/perfmon/reader.go @@ -34,9 +34,10 @@ import ( ) const ( - instanceCountLabel = ":count" - defaultInstanceField = "instance" - defaultObjectField = "object" + instanceCountLabel = ":count" + defaultInstanceField = "instance" + defaultObjectField = "object" + replaceUpperCaseRegex = `(?:[^A-Z_\W])([A-Z])[^A-Z]` ) // Reader will contain the config options @@ -267,17 +268,10 @@ func mapCounterPathLabel(namespace string, label string, path string) string { label = path } // replace spaces with underscores - label = strings.Replace(label, " ", "_", -1) // replace backslashes with "per" - label = strings.Replace(label, "/sec", "_per_sec", -1) - label = strings.Replace(label, "/_sec", "_per_sec", -1) - label = strings.Replace(label, "\\", "_", -1) - // replace actual percentage symbol with the smbol "pct" - label = strings.Replace(label, "_%_", "_pct_", -1) - // create an object in case of ":" - label = strings.Replace(label, ":", "_", -1) - // create an object in case of ":" - label = strings.Replace(label, "_-_", "_", -1) + // replace actual percentage symbol with the symbol "pct" + r := strings.NewReplacer(" ", "_", "/sec", "_per_sec", "/_sec", "_per_sec", "\\", "_", "_%_", "_pct_", ":", "_", "_-_", "_") + label = r.Replace(label) // replace uppercases with underscores label = replaceUpperCase(label) @@ -289,8 +283,6 @@ func mapCounterPathLabel(namespace string, label string, path string) string { obj[index] = strings.TrimSuffix(obj[index], "_") } label = strings.ToLower(strings.Join(obj, "_")) - - // avoid cases as this "logicaldisk_avg__disk_sec_per_transfer" label = strings.Replace(label, "__", "_", -1) return namespace + "." + label } diff --git a/metricbeat/module/windows/perfmon/reader_test.go b/metricbeat/module/windows/perfmon/reader_test.go index 0cfb1a2edf64..697281b64e65 100644 --- a/metricbeat/module/windows/perfmon/reader_test.go +++ b/metricbeat/module/windows/perfmon/reader_test.go @@ -148,4 +148,13 @@ func TestMapCounterPathLabel(t *testing.T) { assert.Equal(t, result, "metrics.queued_poison_messages_per_second") result = mapCounterPathLabel("metrics", "", `I/O Log Writes Average Latency`) assert.Equal(t, result, "metrics.i/o_log_writes_average_latency") + result = mapCounterPathLabel("metrics", "io.logwrites.average latency", `I/O Log Writes Average Latency`) + assert.Equal(t, result, "metrics.io_logwrites_average_latency") + + result = mapCounterPathLabel("metrics", "this.is__exceptional-test:case/sec", `RSC Coalesced Packet Bucket 5 (16To31)`) + assert.Equal(t, result, "metrics.this_is_exceptional-test_case_per_sec") + + result = mapCounterPathLabel("metrics", "logicaldisk_avg._disk_sec_per_transfer", `RSC Coalesced Packet Bucket 5 (16To31)`) + assert.Equal(t, result, "metrics.logicaldisk_avg_disk_sec_per_transfer") + } diff --git a/metricbeat/modules.d/windows.yml.disabled b/metricbeat/modules.d/windows.yml.disabled index c2d8c0f8f6d3..270e5a7f64cc 100644 --- a/metricbeat/modules.d/windows.yml.disabled +++ b/metricbeat/modules.d/windows.yml.disabled @@ -11,10 +11,10 @@ # - perfmon # period: 10s # perfmon.queries: -# - object: 'Process' -# instance: ["*"] -# counters: -# - name: 'Disk Writes/sec' -# field: physical_disk.write.per_sec -# format: "float" -# - name: "% Disk Write Time" +# - object: 'Process' +# instance: ["*"] +# counters: +# - name: 'Disk Writes/sec' +# field: physical_disk.write.per_sec +# format: "float" +# - name: "% Disk Write Time"