Add improved config/event output options to windows/perfmon metricset

CHANGELOG.next.asciidoc

@@ -323,6 +323,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Added documentation for running Metricbeat in Cloud Foundry. {pull}17275[17275]
 - Add test for documented fields check for metricsets without a http input. {issue}17315[17315] {pull}17334[17334]
 - Add final tests and move label to GA for the azure module in metricbeat. {pull}17319[17319]
+- Refactor windows/perfmon metricset configuration options and event output. {pull}17596[17596]
 - Reference kubernetes manifests mount data directory from the host when running metricbeat as daemonset, so data persist between executions in the same node. {pull}17429[17429]
 
 *Packetbeat*

metricbeat/docs/fields.asciidoc

@@ -18569,20 +18569,13 @@ type: object
 
 --
 
-[float]
-=== website
-
-website
-
-
-
-*`iis.website.name`*::
+*`iis.website.*.*`*::
 +
 --
-website name
+website
 
 
-type: keyword
+type: object
 
 --
 

metricbeat/docs/modules/windows.asciidoc

@@ -5,8 +5,13 @@ This file is generated! See scripts/mage/docs_collector.go
 [[metricbeat-module-windows]]
 == Windows module
 
-This is the Windows module. It collects metrics from Windows systems,
-by default metricset `service` is enabled.
+This is the Windows module which collects metrics from Windows systems.
+The module contains the `service` metricset, which is set up by default when the windows module is enabled.
+The `service` metricset will retrieve status information of the services on the Windows machines. The other windows
+metricset is `perfmon` which collects windows performance counter values.
+
+
+
 
 
 [float]
@@ -24,11 +29,14 @@ metricbeat.modules:
   period: 10s
   perfmon.ignore_non_existent_counters: false
   perfmon.group_measurements_by_instance: false
-  perfmon.counters:
-  #  - instance_label: processor.name
-  #    instance_name: total
-  #    measurement_label: processor.time.total.pct
-  #    query: '\Processor Information(_Total)\% Processor Time'
+  perfmon.queries:
+#  - object: 'Process'
+#    instance: ["*"]
+#    counters:
+#    - name: 'Disk Writes/sec'
+#      field: physical_disk.write.per_sec
+#      format: "float"
+#    - name: "% Disk Write Time"
 
 - module: windows
   metricsets: ["service"]

metricbeat/metricbeat.reference.yml

@@ -848,11 +848,14 @@ metricbeat.modules:
   period: 10s
   perfmon.ignore_non_existent_counters: false
   perfmon.group_measurements_by_instance: false
-  perfmon.counters:
-  #  - instance_label: processor.name
-  #    instance_name: total
-  #    measurement_label: processor.time.total.pct
-  #    query: '\Processor Information(_Total)\% Processor Time'
+  perfmon.queries:
+#  - object: 'Process'
+#    instance: ["*"]
+#    counters:
+#    - name: 'Disk Writes/sec'
+#      field: physical_disk.write.per_sec
+#      format: "float"
+#    - name: "% Disk Write Time"
 
 - module: windows
   metricsets: ["service"]

metricbeat/module/windows/_meta/config.reference.yml

@@ -4,11 +4,14 @@
   period: 10s
   perfmon.ignore_non_existent_counters: false
   perfmon.group_measurements_by_instance: false
-  perfmon.counters:
-  #  - instance_label: processor.name
-  #    instance_name: total
-  #    measurement_label: processor.time.total.pct
-  #    query: '\Processor Information(_Total)\% Processor Time'
+  perfmon.queries:
+#  - object: 'Process'
+#    instance: ["*"]
+#    counters:
+#    - name: 'Disk Writes/sec'
+#      field: physical_disk.write.per_sec
+#      format: "float"
+#    - name: "% Disk Write Time"
 
 - module: windows
   metricsets: ["service"]

metricbeat/module/windows/_meta/config.yml

@@ -1,22 +1,17 @@
 - module: windows
-  #metricsets:
-  #  - service
+  metricsets:
+    - service
   period: 1m
 
 #- module: windows
 #  metricsets:
-#    - perfmon
+#  - perfmon
 #  period: 10s
-#  perfmon.counters:
-#    - instance_label: processor.name
-#      instance_name: total
-#      measurement_label: processor.time.total.pct
-#      query: '\Processor Information(_Total)\% Processor Time'
-#
-#    - instance_label: physical_disk.name
-#      measurement_label: physical_disk.write.per_sec
-#      query: '\PhysicalDisk(*)\Disk Writes/sec'
-#
-#    - instance_label: physical_disk.name
-#      measurement_label: physical_disk.write.time.pct
-#      query: '\PhysicalDisk(*)\% Disk Write Time'
+#  perfmon.queries:
+#  - object: 'Process'
+#    instance: ["*"]
+#    counters:
+#    - name: 'Disk Writes/sec'
+#      field: physical_disk.write.per_sec
+#      format: "float"
+#    - name: "% Disk Write Time"

metricbeat/module/windows/_meta/docs.asciidoc

@@ -1,2 +1,7 @@
-This is the Windows module. It collects metrics from Windows systems,
-by default metricset `service` is enabled.
+This is the Windows module which collects metrics from Windows systems.
+The module contains the `service` metricset, which is set up by default when the windows module is enabled.
+The `service` metricset will retrieve status information of the services on the Windows machines. The other windows
+metricset is `perfmon` which collects windows performance counter values.
+
+
+

metricbeat/module/windows/perfmon/_meta/data.json

@@ -1,24 +1,30 @@
 {
     "@timestamp": "2017-10-12T08:05:34.853Z",
-    "beat": {
-        "hostname": "host.example.com",
-        "name": "host.example.com"
+    "event": {
+        "dataset": "windows.perfmon",
+        "duration": 115000,
+        "module": "windows"
     },
     "metricset": {
-        "module": "windows",
         "name": "perfmon",
-        "rtt": 115
+        "period": 10000
+    },
+    "service": {
+        "type": "windows"
     },
     "windows": {
         "perfmon": {
-            "processor": {
-                "name": "_Total",
-                "time": {
-                    "total": {
-                        "pct": 1.4663385364361736
+            "instance": "_Total",
+            "metrics": {
+                "processor": {
+                    "time": {
+                        "total": {
+                            "pct": 6.310940413107646
+                        }
                     }
                 }
-            }
+            },
+            "object": "Processor Information"
         }
     }
 }

metricbeat/module/windows/perfmon/_meta/docs.asciidoc

@@ -15,6 +15,28 @@ to collect. The example below collects processor time and disk writes every
   period: 10s
   perfmon.ignore_non_existent_counters: true
   perfmon.group_measurements_by_instance: true
+  perfmon.queries:
+  - object: "Process"
+    instance: ["svchost*", "conhost*"]
+    counters:
+    - name: "% Processor Time"
+      field: time.processor.pct
+      format: "float"
+    - name: "Thread Count"
+      field: thread_count
+    - name: "IO Read Operations/sec"
+  - object: "PhysicalDisk"
+    field : "disk"
+    instance: "*"
+    counters:
+    - name: "Disk Writes/sec"
+    - name: "% Disk Write Time"
+      field: "write_time"
+      format: "float"
+
+
+  // deprecated, will be removed in 8.0
+
   perfmon.counters:
     - instance_label: processor.name
       instance_name: total
@@ -46,7 +68,33 @@ counter requires three config options - `instance_label`, `measurement_label`,
 and `query`.
 
 [float]
-==== Counter Configuration
+==== Query Configuration
+
+Each item in the `query` list specifies multiple perfmon queries to perform. In the
+events generated by the metricset these configuration options map to the field
+values as shown below.
+
+*`object`*:: The performance object to query. A performance object can be a physical component, such as processors, disks, and memory, or a system object, such as processes and threads. Required
+
+*`field`*:: The object field/label. Not required, if not entered, it will be `object`.
+
+*`instance`*:: Matches the ParentInstance, ObjectInstance, and InstanceIndex are included in the path if multiple instances of the object can exist. Not required for performance counters which do not contain one.
+
+*`counters`*:: List of the partial counter paths (At least one partial counter path is required).
+
+*`name`*:: The counter name. Required. This is the counter specified in Performance Data Helper (PDH) syntax. For example in case of the counter path `\Processor Information(_Total)\% Processor Time`,
+the value for this configuration option will be `% Processor Time`.
+
+*`field`*:: The counter path value field/label. Not required, if not entered, it will be generated based on the counter path.
+
+*`format`*:: Format of the measurement value. The value can be either `float` or
+`long`. The default is `float`.
+
+
+
+
+[float]
+==== Deprecated Counter Configuration
 
 Each item in the `counters` list specifies a perfmon query to perform. In the
 events generated by the metricset these configuration options map to the field

metricbeat/module/windows/perfmon/config.go

@@ -0,0 +1,105 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// +build windows
+
+package perfmon
+
+import (
+	"strings"
+
+	"github.com/pkg/errors"
+
+	"github.com/elastic/beats/v7/libbeat/common/cfgwarn"
+)
+
+const replaceUpperCaseRegex = `(?:[^A-Z_\W])([A-Z])[^A-Z]`
+
+// Config for the windows perfmon metricset.
+type Config struct {
+	IgnoreNECounters   bool      `config:"perfmon.ignore_non_existent_counters"`
+	GroupMeasurements  bool      `config:"perfmon.group_measurements_by_instance"`
+	Counters           []Counter `config:"perfmon.counters"`
+	Queries            []Query   `config:"perfmon.queries"`
+	GroupAllCountersTo string    `config:"perfmon.group_all_counter"`
+}
+
+// Counter for the perfmon counters (old implementation deprecated).
+type Counter struct {
+	InstanceLabel    string `config:"instance_label"`
+	InstanceName     string `config:"instance_name"`
+	MeasurementLabel string `config:"measurement_label" validate:"required"`
+	Query            string `config:"query"             validate:"required"`
+	Format           string `config:"format"`
+}
+
+// QueryConfig for perfmon queries. This will be used as the new configuration format
+type Query struct {
+	Name      string         `config:"object" validate:"required"`
+	Field     string         `config:"field"`
+	Instance  []string       `config:"instance"`
+	Counters  []QueryCounter `config:"counters" validate:"required"`
+	Namespace string         `config:"namespace"`
+}
+
+// QueryConfigCounter for perfmon queries. This will be used as the new configuration format
+type QueryCounter struct {
+	Name   string `config:"name" validate:"required"`
+	Field  string `config:"field"`
+	Format string `config:"format"`
+}
+
+func (conf *Config) ValidateConfig() error {
+	if len(conf.Counters) == 0 && len(conf.Queries) == 0 {
+		return errors.New("no perfmon counters or queries have been configured")
+	}
+	if len(conf.Counters) > 0 {
+		cfgwarn.Deprecate("8.0", "perfmon.counters configuration option is deprecated and will be remove in the future major version, we advise using the perfmon.queries configuration option instead")
+	}
+
+	// add default format in the config
+	for i, value := range conf.Counters {
+		form := strings.ToLower(value.Format)
+		switch form {
+		case "", "float":
+			conf.Counters[i].Format = "float"
+		case "long", "large":
+		default:
+			return errors.Errorf("initialization failed: format '%s' "+
+				"for counter '%s' is invalid (must be float, large or long)",
+				value.Format, value.InstanceLabel)
+		}
+	}
+	for i, value := range conf.Queries {
+		if value.Namespace == "" {
+			conf.Queries[i].Namespace = "metrics"
+		}
+		for j, q := range value.Counters {
+			form := strings.ToLower(q.Format)
+			switch form {
+			case "", "float":
+				value.Counters[j].Format = "float"
+			case "long", "large":
+			default:
+				return errors.Errorf("initialization failed: format '%s' "+
+					"for counter '%s' is invalid (must be float, large or long)",
+					q.Format, q.Field)
+			}
+		}
+	}
+	return nil
+}