Fix Metricbeat/vsphere - Error "datastore '*' not found" #4879 (#4883)

* Fix #4879 * Fix #4673
elastic · Nov 28, 2017 · 550b81e · 550b81e
1 parent d61c015
commit 550b81e
Show file tree

Hide file tree

Showing 102 changed files with 6,162 additions and 7,179 deletions.
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
@@ -64,6 +64,8 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di
 - Fix the fetching of process information when some data is missing under MacOS X. {issue}5337[5337]
 - Change `MySQL active connections` visualization title to `MySQL total connections`. {issue}4812[4812]
 - Fix map overwrite in docker diskio module. {issue}5582[5582]
+- Fix error `datastore '*' not found` in Vsphere module. {issue}4879[4879]
+- Fix error `NotAuthenticated` in Vsphere module. {issue}4673[4673]
 - Fix connection leak in mongodb module. {issue}5688[5688]
 
 *Packetbeat*

diff --git a/NOTICE.txt b/NOTICE.txt
@@ -1134,7 +1134,7 @@ Apache License 2.0
 
 --------------------------------------------------------------------
 Dependency: github.com/google/uuid
-Revision: 6a5e28554805e78ea6141142aba763936c4761c0
+Revision: 281f560d28af7174109514e936f94c2ab2cb2823
 License type (autodetected): BSD 3-clause license
 ./metricbeat/module/vsphere/vendor/github.com/google/uuid/LICENSE:
 --------------------------------------------------------------------
@@ -3501,7 +3501,7 @@ Apache License 2.0
 
 --------------------------------------------------------------------
 Dependency: github.com/vmware/govmomi
-Revision: 9bfdc5ce62c0585b48b154cc460f8664dcd124c3
+Revision: 2cad15190b417804d82edb4981e7b3e62907c4ee
 License type (autodetected): Apache License 2.0
 ./metricbeat/module/vsphere/vendor/github.com/vmware/govmomi/LICENSE.txt:
 --------------------------------------------------------------------
@@ -3510,7 +3510,7 @@ Apache License 2.0
 
 --------------------------------------------------------------------
 Dependency: github.com/vmware/govmomi/vim25/xml
-Revision: 5072cda664c79ada30834d171d2ed1f76317d3b2
+Revision: 2cad15190b417804d82edb4981e7b3e62907c4ee
 License type (autodetected): BSD 3-clause license
 ./metricbeat/module/vsphere/vendor/github.com/vmware/govmomi/vim25/xml/LICENSE:
 --------------------------------------------------------------------
@@ -3542,15 +3542,6 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
---------------------------------------------------------------------
-Dependency: github.com/vmware/vic
-Revision: a2b2afb419d70009cd4d0b58f37b1a095c58b526
-License type (autodetected): Apache License 2.0
-./metricbeat/module/vsphere/vendor/github.com/vmware/vic/LICENSE:
---------------------------------------------------------------------
-Apache License 2.0
-
-
 --------------------------------------------------------------------
 Dependency: github.com/yuin/gopher-lua
 Revision: b402f3114ec730d8bddb074a6c137309f561aa78

diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc
@@ -11287,14 +11287,6 @@ datastore
 
 
 
-[float]
-=== `vsphere.datastore.datacenter`
-
-type: keyword
-
-Datacenter name
-
-
 [float]
 === `vsphere.datastore.name`
 
@@ -11358,14 +11350,6 @@ host
 
 
 
-[float]
-=== `vsphere.host.datacenter`
-
-type: keyword
-
-Datacenter name
-
-
 [float]
 === `vsphere.host.name`
 
@@ -11436,11 +11420,11 @@ virtualmachine
 
 
 [float]
-=== `vsphere.virtualmachine.datacenter`
+=== `vsphere.virtualmachine.host`
 
 type: keyword
 
-Datacenter name
+Host name
 
 
 [float]

diff --git a/metricbeat/docs/modules/vsphere.asciidoc b/metricbeat/docs/modules/vsphere.asciidoc
@@ -31,6 +31,7 @@ metricbeat.modules:
   insecure: false
   # Get custom fields when using virtualmachine metric set. Default false.
   # get_custom_fields: false
+  
 ----
 
 [float]

diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml
@@ -461,6 +461,7 @@ metricbeat.modules:
   insecure: false
   # Get custom fields when using virtualmachine metric set. Default false.
   # get_custom_fields: false
+
 
 #------------------------------- Windows Module ------------------------------
 - module: windows

diff --git a/metricbeat/module/vsphere/_meta/config.yml b/metricbeat/module/vsphere/_meta/config.yml
@@ -9,3 +9,4 @@
   insecure: false
   # Get custom fields when using virtualmachine metric set. Default false.
   # get_custom_fields: false
+
diff --git a/metricbeat/module/vsphere/datastore/_meta/data.json b/metricbeat/module/vsphere/datastore/_meta/data.json
@@ -25,9 +25,8 @@
                     "pct": 88
                 }
             },
-            "datacenter": "ha-datacenter",
             "fstype": "local",
             "name": "LocalDS_0"
         }
     }
-}
+}
diff --git a/metricbeat/module/vsphere/datastore/_meta/fields.yml b/metricbeat/module/vsphere/datastore/_meta/fields.yml
@@ -4,10 +4,6 @@
     datastore
   release: beta
   fields:
-    - name: datacenter
-      type: keyword
-      description: >
-        Datacenter name
     - name: name
       type: keyword
       description: >

diff --git a/metricbeat/module/vsphere/datastore/datastore.go b/metricbeat/module/vsphere/datastore/datastore.go
@@ -8,13 +8,9 @@ import (
 	"github.com/elastic/beats/libbeat/common/cfgwarn"
 	"github.com/elastic/beats/metricbeat/mb"
 
-	"github.com/pkg/errors"
 	"github.com/vmware/govmomi"
-	"github.com/vmware/govmomi/find"
-	"github.com/vmware/govmomi/property"
-	"github.com/vmware/govmomi/vim25"
+	"github.com/vmware/govmomi/view"
 	"github.com/vmware/govmomi/vim25/mo"
-	"github.com/vmware/govmomi/vim25/types"
 )
 
 func init() {
@@ -25,7 +21,8 @@ func init() {
 
 type MetricSet struct {
 	mb.BaseMetricSet
-	Client *vim25.Client
+	HostURL  *url.URL
+	Insecure bool
 }
 
 func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
@@ -48,80 +45,71 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
 
 	u.User = url.UserPassword(config.Username, config.Password)
 
-	c, err := govmomi.NewClient(context.TODO(), u, config.Insecure)
-	if err != nil {
-		return nil, err
-	}
-
 	return &MetricSet{
 		BaseMetricSet: base,
-		Client:        c.Client,
+		HostURL:       u,
+		Insecure:      config.Insecure,
 	}, nil
 }
 
 func (m *MetricSet) Fetch() ([]common.MapStr, error) {
-	f := find.NewFinder(m.Client, true)
 
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
-	// Get all data centers.
-	dcs, err := f.DatacenterList(ctx, "*")
+	events := []common.MapStr{}
+
+	client, err := govmomi.NewClient(ctx, m.HostURL, m.Insecure)
 	if err != nil {
 		return nil, err
 	}
 
-	var events []common.MapStr
-	for _, dc := range dcs {
-		f.SetDatacenter(dc)
+	defer client.Logout(ctx)
 
-		dss, err := f.DatastoreList(ctx, "*")
-		if err != nil {
-			return nil, errors.Wrap(err, "failed to get datastore list")
-		}
+	c := client.Client
 
-		pc := property.DefaultCollector(m.Client)
+	// Create a view of Datastore objects
+	mgr := view.NewManager(c)
 
-		// Convert datastores into list of references.
-		var refs []types.ManagedObjectReference
-		for _, ds := range dss {
-			refs = append(refs, ds.Reference())
-		}
+	v, err := mgr.CreateContainerView(ctx, c.ServiceContent.RootFolder, []string{"Datastore"}, true)
+	if err != nil {
+		return nil, err
+	}
 
-		// Retrieve summary property.
-		var dst []mo.Datastore
-		err = pc.Retrieve(ctx, refs, []string{"summary"}, &dst)
-		if err != nil {
-			return nil, err
-		}
+	defer v.Destroy(ctx)
 
-		for _, ds := range dst {
-			var usedSpacePercent int64
-			if ds.Summary.Capacity > 0 {
-				usedSpacePercent = 100 * (ds.Summary.Capacity - ds.Summary.FreeSpace) / ds.Summary.Capacity
-			}
-			usedSpaceBytes := ds.Summary.Capacity - ds.Summary.FreeSpace
-
-			event := common.MapStr{
-				"datacenter": dc.Name(),
-				"name":       ds.Summary.Name,
-				"fstype":     ds.Summary.Type,
-				"capacity": common.MapStr{
-					"total": common.MapStr{
-						"bytes": ds.Summary.Capacity,
-					},
-					"free": common.MapStr{
-						"bytes": ds.Summary.FreeSpace,
-					},
-					"used": common.MapStr{
-						"bytes": usedSpaceBytes,
-						"pct":   usedSpacePercent,
-					},
-				},
-			}
+	// Retrieve summary property for all datastores
+	var dst []mo.Datastore
+	err = v.Retrieve(ctx, []string{"Datastore"}, []string{"summary"}, &dst)
+	if err != nil {
+		return nil, err
+	}
 
-			events = append(events, event)
+	for _, ds := range dst {
+		var usedSpacePercent int64
+		if ds.Summary.Capacity > 0 {
+			usedSpacePercent = 100 * (ds.Summary.Capacity - ds.Summary.FreeSpace) / ds.Summary.Capacity
 		}
+		usedSpaceBytes := ds.Summary.Capacity - ds.Summary.FreeSpace
+
+		event := common.MapStr{
+			"name":   ds.Summary.Name,
+			"fstype": ds.Summary.Type,
+			"capacity": common.MapStr{
+				"total": common.MapStr{
+					"bytes": ds.Summary.Capacity,
+				},
+				"free": common.MapStr{
+					"bytes": ds.Summary.FreeSpace,
+				},
+				"used": common.MapStr{
+					"bytes": usedSpaceBytes,
+					"pct":   usedSpacePercent,
+				},
+			},
+		}
+
+		events = append(events, event)
 	}
 
 	return events, nil

diff --git a/metricbeat/module/vsphere/datastore/datastore_test.go b/metricbeat/module/vsphere/datastore/datastore_test.go
@@ -6,7 +6,7 @@ import (
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
 
 	"github.com/stretchr/testify/assert"
-	"github.com/vmware/vic/pkg/vsphere/simulator"
+	"github.com/vmware/govmomi/simulator"
 )
 
 func TestFetchEventContents(t *testing.T) {
@@ -28,7 +28,6 @@ func TestFetchEventContents(t *testing.T) {
 
 	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event.StringToPrint())
 
-	assert.EqualValues(t, "ha-datacenter", event["datacenter"])
 	assert.EqualValues(t, "LocalDS_0", event["name"])
 	assert.EqualValues(t, "local", event["fstype"])
 

diff --git a/metricbeat/module/vsphere/host/_meta/data.json b/metricbeat/module/vsphere/host/_meta/data.json
@@ -24,7 +24,6 @@
                     "mhz": 67
                 }
             },
-            "datacenter": "ha-datacenter",
             "memory": {
                 "free": {
                     "bytes": 2822230016
@@ -39,4 +38,4 @@
             "name": "localhost.localdomain"
         }
     }
-}
+}
diff --git a/metricbeat/module/vsphere/host/_meta/fields.yml b/metricbeat/module/vsphere/host/_meta/fields.yml
@@ -4,10 +4,6 @@
     host
   release: beta
   fields:
-    - name: datacenter
-      type: keyword
-      description: >
-        Datacenter name
     - name: name
       type: keyword
       description: >

diff --git a/metricbeat/module/vsphere/host/data.go b/metricbeat/module/vsphere/host/data.go
@@ -6,15 +6,14 @@ import (
 	"github.com/vmware/govmomi/vim25/mo"
 )
 
-func eventMapping(hs mo.HostSystem, datacenterName string) common.MapStr {
+func eventMapping(hs mo.HostSystem) common.MapStr {
 	totalCpu := int64(hs.Summary.Hardware.CpuMhz) * int64(hs.Summary.Hardware.NumCpuCores)
 	freeCpu := int64(totalCpu) - int64(hs.Summary.QuickStats.OverallCpuUsage)
 	usedMemory := int64(hs.Summary.QuickStats.OverallMemoryUsage) * 1024 * 1024
 	freeMemory := int64(hs.Summary.Hardware.MemorySize) - usedMemory
 
 	event := common.MapStr{
-		"datacenter": datacenterName,
-		"name":       hs.Summary.Config.Name,
+		"name": hs.Summary.Config.Name,
 		"cpu": common.MapStr{
 			"used": common.MapStr{
 				"mhz": hs.Summary.QuickStats.OverallCpuUsage,

diff --git a/metricbeat/module/vsphere/host/data_test.go b/metricbeat/module/vsphere/host/data_test.go
@@ -28,10 +28,7 @@ func TestEventMapping(t *testing.T) {
 		},
 	}
 
-	event := eventMapping(HostSystemTest, "test")
-
-	datacenter, _ := event.GetValue("datacenter")
-	assert.EqualValues(t, "test", datacenter)
+	event := eventMapping(HostSystemTest)
 
 	cpuUsed, _ := event.GetValue("cpu.used.mhz")
 	assert.EqualValues(t, 67, cpuUsed)