Use common host parser in vsphere module (#26904)

Use the common host parser builder to parse hosts defined in vsphere module configuration. Since #21022, sanitized URIs included in modules host data are used as `service.address`. vsphere did a custom parsing that didn't fill the sanitized URI and then `service.address` was not filled. (cherry picked from commit 68e9909)
elastic · Jul 15, 2021 · 4e99ee8 · 4e99ee8
1 parent 463f1a2
commit 4e99ee8
Show file tree

Hide file tree

Showing 9 changed files with 107 additions and 86 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -111,6 +111,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Added `statsd.mappings` configuration for Statsd module {pull}26220[26220]
 - Added Airflow lightweight module {pull}26220[26220]
 - Add state_job metricset to Kubernetes module{pull}26479[26479]
+- Recover service.address field in vsphere module {issue}26902[26902] {pull}26904[26904]
 
 *Packetbeat*
 

diff --git a/metricbeat/module/vsphere/datastore/_meta/data.json b/metricbeat/module/vsphere/datastore/_meta/data.json
@@ -10,21 +10,21 @@
         "period": 10000
     },
     "service": {
-        "address": "http://127.0.0.1:51027/sdk",
+        "address": "127.0.0.1:33365",
         "type": "vsphere"
     },
     "vsphere": {
         "datastore": {
             "capacity": {
                 "free": {
-                    "bytes": 267958476800
+                    "bytes": 37120094208
                 },
                 "total": {
-                    "bytes": 484736266240
+                    "bytes": 74686664704
                 },
                 "used": {
-                    "bytes": 216777789440,
-                    "pct": 0.44720769733509097
+                    "bytes": 37566570496,
+                    "pct": 0.502988996026061
                 }
             },
             "fstype": "local",

diff --git a/metricbeat/module/vsphere/datastore/datastore.go b/metricbeat/module/vsphere/datastore/datastore.go
@@ -19,12 +19,12 @@ package datastore
 
 import (
 	"context"
-	"net/url"
 
 	"github.com/pkg/errors"
 
 	"github.com/elastic/beats/v7/libbeat/common"
 	"github.com/elastic/beats/v7/metricbeat/mb"
+	"github.com/elastic/beats/v7/metricbeat/module/vsphere"
 
 	"github.com/vmware/govmomi"
 	"github.com/vmware/govmomi/view"
@@ -33,41 +33,23 @@ import (
 
 func init() {
 	mb.Registry.MustAddMetricSet("vsphere", "datastore", New,
+		mb.WithHostParser(vsphere.HostParser),
 		mb.DefaultMetricSet(),
 	)
 }
 
-// MetricSet type defines all fields of the MetricSet
+// MetricSet type defines all fields of the MetricSet.
 type MetricSet struct {
-	mb.BaseMetricSet
-	HostURL  *url.URL
-	Insecure bool
+	*vsphere.MetricSet
 }
 
-// New create a new instance of the MetricSet
+// New creates a new instance of the MetricSet.
 func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
-	config := struct {
-		Username string `config:"username"`
-		Password string `config:"password"`
-		Insecure bool   `config:"insecure"`
-	}{}
-
-	if err := base.Module().UnpackConfig(&config); err != nil {
-		return nil, err
-	}
-
-	u, err := url.Parse(base.HostData().URI)
+	ms, err := vsphere.NewMetricSet(base)
 	if err != nil {
 		return nil, err
 	}
-
-	u.User = url.UserPassword(config.Username, config.Password)
-
-	return &MetricSet{
-		BaseMetricSet: base,
-		HostURL:       u,
-		Insecure:      config.Insecure,
-	}, nil
+	return &MetricSet{ms}, nil
 }
 
 // Fetch methods implements the data gathering and data conversion to the right

diff --git a/metricbeat/module/vsphere/host/_meta/data.json b/metricbeat/module/vsphere/host/_meta/data.json
@@ -1,14 +1,17 @@
 {
     "@timestamp": "2017-10-12T08:05:34.853Z",
-    "beat": {
-        "hostname": "host.example.com",
-        "name": "host.example.com"
+    "event": {
+        "dataset": "vsphere.host",
+        "duration": 115000,
+        "module": "vsphere"
     },
     "metricset": {
-        "host": "http://127.0.0.1:43843/sdk",
-        "module": "vsphere",
         "name": "host",
-        "rtt": 115
+        "period": 10000
+    },
+    "service": {
+        "address": "127.0.0.1:38517",
+        "type": "vsphere"
     },
     "vsphere": {
         "host": {

diff --git a/metricbeat/module/vsphere/host/host.go b/metricbeat/module/vsphere/host/host.go
@@ -20,13 +20,13 @@ package host
 import (
 	"context"
 	"fmt"
-	"net/url"
 	"strings"
 
 	"github.com/pkg/errors"
 
 	"github.com/elastic/beats/v7/libbeat/common"
 	"github.com/elastic/beats/v7/metricbeat/mb"
+	"github.com/elastic/beats/v7/metricbeat/module/vsphere"
 
 	"github.com/vmware/govmomi"
 	"github.com/vmware/govmomi/property"
@@ -38,41 +38,23 @@ import (
 
 func init() {
 	mb.Registry.MustAddMetricSet("vsphere", "host", New,
+		mb.WithHostParser(vsphere.HostParser),
 		mb.DefaultMetricSet(),
 	)
 }
 
-// MetricSet type defines all fields of the MetricSet
+// MetricSet type defines all fields of the MetricSet.
 type MetricSet struct {
-	mb.BaseMetricSet
-	HostURL  *url.URL
-	Insecure bool
+	*vsphere.MetricSet
 }
 
-// New create a new instance of the MetricSet
+// New creates a new instance of the MetricSet.
 func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
-	config := struct {
-		Username string `config:"username"`
-		Password string `config:"password"`
-		Insecure bool   `config:"insecure"`
-	}{}
-
-	if err := base.Module().UnpackConfig(&config); err != nil {
-		return nil, err
-	}
-
-	u, err := url.Parse(base.HostData().URI)
+	ms, err := vsphere.NewMetricSet(base)
 	if err != nil {
 		return nil, err
 	}
-
-	u.User = url.UserPassword(config.Username, config.Password)
-
-	return &MetricSet{
-		BaseMetricSet: base,
-		HostURL:       u,
-		Insecure:      config.Insecure,
-	}, nil
+	return &MetricSet{ms}, nil
 }
 
 // Fetch methods implements the data gathering and data conversion to the right

diff --git a/metricbeat/module/vsphere/metricset.go b/metricbeat/module/vsphere/metricset.go
@@ -0,0 +1,59 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package vsphere
+
+import (
+	"net/url"
+
+	"github.com/elastic/beats/v7/metricbeat/mb"
+	"github.com/elastic/beats/v7/metricbeat/mb/parse"
+)
+
+var HostParser = parse.URLHostParserBuilder{
+	DefaultScheme: "https",
+	DefaultPath:   "/sdk",
+}.Build()
+
+// MetricSet type defines all fields of the MetricSet.
+type MetricSet struct {
+	mb.BaseMetricSet
+	Insecure bool
+	HostURL  *url.URL
+}
+
+// NewMetricSet creates a new instance of the MetricSet.
+func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) {
+	config := struct {
+		Insecure bool `config:"insecure"`
+	}{}
+
+	if err := base.Module().UnpackConfig(&config); err != nil {
+		return nil, err
+	}
+
+	u, err := url.Parse(base.HostData().URI)
+	if err != nil {
+		return nil, err
+	}
+
+	return &MetricSet{
+		BaseMetricSet: base,
+		HostURL:       u,
+		Insecure:      config.Insecure,
+	}, nil
+}
diff --git a/metricbeat/module/vsphere/test_vsphere.py b/metricbeat/module/vsphere/test_vsphere.py
@@ -42,6 +42,8 @@ def test_datastore(self):
 
         self.assertCountEqual(self.de_dot(VSPHERE_FIELDS), evt.keys(), evt)
 
+        self.assertEqual(evt["service"]["address"], self.get_hosts()[0])
+
         self.assert_fields_are_documented(evt)
 
     @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test")
@@ -71,6 +73,8 @@ def test_host(self):
 
         self.assertCountEqual(self.de_dot(VSPHERE_FIELDS), evt.keys(), evt)
 
+        self.assertEqual(evt["service"]["address"], self.get_hosts()[0])
+
         self.assert_fields_are_documented(evt)
 
     @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test")
@@ -100,4 +104,6 @@ def test_virtualmachine(self):
 
         self.assertCountEqual(self.de_dot(VSPHERE_FIELDS), evt.keys(), evt)
 
+        self.assertEqual(evt["service"]["address"], self.get_hosts()[0])
+
         self.assert_fields_are_documented(evt)
diff --git a/metricbeat/module/vsphere/virtualmachine/_meta/data.json b/metricbeat/module/vsphere/virtualmachine/_meta/data.json
@@ -10,7 +10,7 @@
         "period": 10000
     },
     "service": {
-        "address": "http://127.0.0.1:37231/sdk",
+        "address": "127.0.0.1:39149",
         "type": "vsphere"
     },
     "vsphere": {
@@ -43,9 +43,6 @@
                 }
             },
             "name": "ha-host_VM0",
-            "network_names": [
-                "VM Network"
-            ],
             "os": "otherGuest"
         }
     }

diff --git a/metricbeat/module/vsphere/virtualmachine/virtualmachine.go b/metricbeat/module/vsphere/virtualmachine/virtualmachine.go
@@ -20,11 +20,11 @@ package virtualmachine
 import (
 	"context"
 	"fmt"
-	"net/url"
 	"strings"
 
 	"github.com/elastic/beats/v7/libbeat/common"
 	"github.com/elastic/beats/v7/metricbeat/mb"
+	"github.com/elastic/beats/v7/metricbeat/module/vsphere"
 
 	"github.com/pkg/errors"
 	"github.com/vmware/govmomi"
@@ -38,44 +38,35 @@ import (
 
 func init() {
 	mb.Registry.MustAddMetricSet("vsphere", "virtualmachine", New,
+		mb.WithHostParser(vsphere.HostParser),
 		mb.DefaultMetricSet(),
 	)
 }
 
-// MetricSet type defines all fields of the MetricSet
+// MetricSet type defines all fields of the MetricSet.
 type MetricSet struct {
-	mb.BaseMetricSet
-	HostURL         *url.URL
-	Insecure        bool
+	*vsphere.MetricSet
 	GetCustomFields bool
 }
 
-// New create a new instance of the MetricSet
+// New creates a new instance of the MetricSet.
 func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
+	ms, err := vsphere.NewMetricSet(base)
+	if err != nil {
+		return nil, err
+	}
+
 	config := struct {
-		Username        string `config:"username"`
-		Password        string `config:"password"`
-		Insecure        bool   `config:"insecure"`
-		GetCustomFields bool   `config:"get_custom_fields"`
+		GetCustomFields bool `config:"get_custom_fields"`
 	}{
 		GetCustomFields: false,
 	}
 
 	if err := base.Module().UnpackConfig(&config); err != nil {
 		return nil, err
 	}
-
-	u, err := url.Parse(base.HostData().URI)
-	if err != nil {
-		return nil, err
-	}
-
-	u.User = url.UserPassword(config.Username, config.Password)
-
 	return &MetricSet{
-		BaseMetricSet:   base,
-		HostURL:         u,
-		Insecure:        config.Insecure,
+		MetricSet:       ms,
 		GetCustomFields: config.GetCustomFields,
 	}, nil
 }