Refactor of system/memory metricset (elastic#26334)

* init commit

* start on linux implementation

* finish linux, start work on darwin

* fix build platform issues

* fix metrics on darwin

* add openbsd

* add freebsd

* add windows, aix

* fix aix build

* finish memory

* fix up opt changes

* cleanup metricset code

* fix up folder methods

* fix calculations, Opt API, gomod

* make notice

* go mod tidy, mage fmt

* fix up linux/memory

* update fields

* update system tests

* fix system tests, again

* fix extra print statements

* fix if block in fillPercentages

* vix Value API

* fix up tests, opt

(cherry picked from commit e008024)

go.mod

@@ -73,7 +73,7 @@ require (
 	github.com/elastic/go-sysinfo v1.7.0
 	github.com/elastic/go-txfile v0.0.7
 	github.com/elastic/go-ucfg v0.8.3
-	github.com/elastic/go-windows v1.0.1 // indirect
+	github.com/elastic/go-windows v1.0.1
 	github.com/elastic/gosigar v0.14.1
 	github.com/fatih/color v1.9.0
 	github.com/fsnotify/fsevents v0.1.1

libbeat/metric/system/process/process.go

@@ -32,7 +32,7 @@ import (
 	"github.com/elastic/beats/v7/libbeat/common"
 	"github.com/elastic/beats/v7/libbeat/common/match"
 	"github.com/elastic/beats/v7/libbeat/logp"
-	"github.com/elastic/beats/v7/libbeat/metric/system/memory"
+	sysinfo "github.com/elastic/go-sysinfo"
 	sigar "github.com/elastic/gosigar"
 	"github.com/elastic/gosigar/cgroup"
 )
@@ -290,6 +290,22 @@ func GetOwnResourceUsageTimeInMillis() (int64, int64, error) {
 }
 
 func (procStats *Stats) getProcessEvent(process *Process) common.MapStr {
+	// This is a holdover until we migrate this library to metricbeat/internal
+	// At which point we'll use the memory code there.
+	var totalPhyMem uint64
+	host, err := sysinfo.Host()
+	if err != nil {
+		procStats.logger.Warnf("Getting host details: %v", err)
+	} else {
+		memStats, err := host.Memory()
+		if err != nil {
+			procStats.logger.Warnf("Getting memory details: %v", err)
+		} else {
+			totalPhyMem = memStats.Total
+		}
+
+	}
+
 	proc := common.MapStr{
 		"pid":      process.Pid,
 		"ppid":     process.Ppid,

metricbeat/docs/fields.asciidoc

@@ -44241,6 +44241,18 @@ format: bytes
 The total amount of free memory in bytes. This value does not include memory consumed by system caches and buffers (see system.memory.actual.free).
 
 
+type: long
+
+format: bytes
+
+--
+
+*`system.memory.cached`*::
++
+--
+Total Cached memory on system.
+
+
 type: long
 
 format: bytes

metricbeat/internal/metrics/memory/memory.go

@@ -0,0 +1,102 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package memory
+
+import (
+	"github.com/pkg/errors"
+
+	"github.com/elastic/beats/v7/libbeat/common"
+	"github.com/elastic/beats/v7/metricbeat/internal/metrics"
+)
+
+// Memory holds os-specifc memory usage data
+// The vast majority of these values are cross-platform
+// However, we're wrapping all them for the sake of safety, and for the more variable swap metrics
+type Memory struct {
+	Total metrics.OptUint `struct:"total,omitempty"`
+	Used  UsedMemStats    `struct:"used,omitempty"`
+
+	Free   metrics.OptUint `struct:"free,omitempty"`
+	Cached metrics.OptUint `struct:"cached,omitempty"`
+	// "Actual" values are, technically, a linux-only concept
+	// For better or worse we've expanded it to include "derived"
+	// Memory values on other platforms, which we should
+	// probably keep for the sake of backwards compatibility
+	// However, because the derived value varies from platform to platform,
+	// We may want to more precisely document what these mean.
+	Actual ActualMemoryMetrics `struct:"actual,omitempty"`
+
+	// Swap metrics
+	Swap SwapMetrics `struct:"swap,omitempty"`
+}
+
+// UsedMemStats wraps used.* memory metrics
+type UsedMemStats struct {
+	Pct   metrics.OptFloat `struct:"pct,omitempty"`
+	Bytes metrics.OptUint  `struct:"bytes,omitempty"`
+}
+
+// ActualMemoryMetrics wraps the actual.* memory metrics
+type ActualMemoryMetrics struct {
+	Free metrics.OptUint `struct:"free,omitempty"`
+	Used UsedMemStats    `struct:"used,omitempty"`
+}
+
+// SwapMetrics wraps swap.* memory metrics
+type SwapMetrics struct {
+	Total metrics.OptUint `struct:"total,omitempty"`
+	Used  UsedMemStats    `struct:"used,omitempty"`
+	Free  metrics.OptUint `struct:"free,omitempty"`
+}
+
+// Get returns platform-independent memory metrics.
+func Get(procfs string) (Memory, error) {
+	base, err := get(procfs)
+	if err != nil {
+		return Memory{}, errors.Wrap(err, "error getting system memory info")
+	}
+	base.fillPercentages()
+	return base, nil
+}
+
+// IsZero implements the zeroer interface for structform's folders
+func (used UsedMemStats) IsZero() bool {
+	return used.Pct.IsZero() && used.Bytes.IsZero()
+}
+
+// IsZero implements the zeroer interface for structform's folders
+func (swap SwapMetrics) IsZero() bool {
+	return swap.Free.IsZero() && swap.Used.IsZero() && swap.Total.IsZero()
+}
+
+func (base *Memory) fillPercentages() {
+	// Add percentages
+	// In theory, `Used` and `Total` are available everywhere, so assume values are good.
+	if base.Total.Exists() && base.Total.ValueOr(0) != 0 {
+		percUsed := float64(base.Used.Bytes.ValueOr(0)) / float64(base.Total.ValueOr(1))
+		base.Used.Pct = metrics.OptFloatWith(common.Round(percUsed, common.DefaultDecimalPlacesCount))
+
+		actualPercUsed := float64(base.Actual.Used.Bytes.ValueOr(0)) / float64(base.Total.ValueOr(0))
+		base.Actual.Used.Pct = metrics.OptFloatWith(common.Round(actualPercUsed, common.DefaultDecimalPlacesCount))
+	}
+
+	if base.Swap.Total.ValueOr(0) != 0 && base.Swap.Used.Bytes.Exists() {
+		perc := float64(base.Swap.Used.Bytes.ValueOr(0)) / float64(base.Swap.Total.ValueOr(0))
+		base.Swap.Used.Pct = metrics.OptFloatWith(common.Round(perc, common.DefaultDecimalPlacesCount))
+	}
+}

metricbeat/internal/metrics/memory/memory_aix.go

@@ -0,0 +1,75 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package memory
+
+/*
+#cgo LDFLAGS: -L/usr/lib -lperfstat
+
+#include <libperfstat.h>
+#include <procinfo.h>
+#include <unistd.h>
+#include <utmp.h>
+#include <sys/mntctl.h>
+#include <sys/proc.h>
+#include <sys/types.h>
+#include <sys/vmount.h>
+
+*/
+import "C"
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/elastic/beats/v7/metricbeat/internal/metrics"
+)
+
+var system struct {
+	ticks    uint64
+	btime    uint64
+	pagesize uint64
+}
+
+func init() {
+	// sysconf(_SC_CLK_TCK) returns the number of ticks by second.
+	system.ticks = uint64(C.sysconf(C._SC_CLK_TCK))
+	system.pagesize = uint64(os.Getpagesize())
+}
+
+func get(_ string) (Memory, error) {
+	memData := Memory{}
+	meminfo := C.perfstat_memory_total_t{}
+	_, err := C.perfstat_memory_total(nil, &meminfo, C.sizeof_perfstat_memory_total_t, 1)
+	if err != nil {
+		return memData, fmt.Errorf("perfstat_memory_total: %s", err)
+	}
+
+	totalMem := uint64(meminfo.real_total) * system.pagesize
+	freeMem := uint64(meminfo.real_free) * system.pagesize
+
+	memData.Total = metrics.OptUintWith(totalMem)
+	memData.Free = metrics.OptUintWith(freeMem)
+
+	kern := uint64(meminfo.numperm) * system.pagesize // number of pages in file cache
+
+	memData.Used.Bytes = metrics.OptUintWith(totalMem - freeMem)
+	memData.Actual.Free = metrics.OptUintWith(freeMem + kern)
+	memData.Actual.Used.Bytes = metrics.OptUintWith(memData.Used.Bytes.ValueOr(0) - kern)
+
+	return memData, nil
+}

metricbeat/internal/metrics/memory/memory_darwin.go

@@ -0,0 +1,133 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package memory
+
+/*
+#include <stdlib.h>
+#include <sys/sysctl.h>
+#include <sys/mount.h>
+#include <mach/mach_init.h>
+#include <mach/mach_host.h>
+#include <mach/host_info.h>
+#include <libproc.h>
+#include <mach/processor_info.h>
+#include <mach/vm_map.h>
+*/
+import "C"
+
+import (
+	"bytes"
+	"encoding/binary"
+	"fmt"
+	"syscall"
+	"unsafe"
+
+	"github.com/pkg/errors"
+
+	"github.com/elastic/beats/v7/metricbeat/internal/metrics"
+)
+
+type xswUsage struct {
+	Total, Avail, Used uint64
+}
+
+// get is the darwin implementation for fetching Memory data
+func get(_ string) (Memory, error) {
+	var vmstat C.vm_statistics_data_t
+
+	mem := Memory{}
+
+	var total uint64
+
+	if err := sysctlbyname("hw.memsize", &total); err != nil {
+		return Memory{}, errors.Wrap(err, "error getting memsize")
+	}
+	mem.Total = metrics.OptUintWith(total)
+
+	if err := vmInfo(&vmstat); err != nil {
+		return Memory{}, errors.Wrap(err, "error getting VM info")
+	}
+
+	kern := uint64(vmstat.inactive_count) << 12
+	free := uint64(vmstat.free_count) << 12
+
+	mem.Free = metrics.OptUintWith(free)
+	mem.Used.Bytes = metrics.OptUintWith(total - free)
+
+	mem.Actual.Free = metrics.OptUintWith(free + kern)
+	mem.Actual.Used.Bytes = metrics.OptUintWith((total - free) - kern)
+
+	var err error
+	mem.Swap, err = getSwap()
+	if err != nil {
+		return mem, errors.Wrap(err, "error getting swap memory")
+	}
+
+	return mem, nil
+}
+
+// Get fetches swap data
+func getSwap() (SwapMetrics, error) {
+	swUsage := xswUsage{}
+
+	swap := SwapMetrics{}
+	if err := sysctlbyname("vm.swapusage", &swUsage); err != nil {
+		return swap, errors.Wrap(err, "error getting swap usage")
+	}
+
+	swap.Total = metrics.OptUintWith(swUsage.Total)
+	swap.Used.Bytes = metrics.OptUintWith(swUsage.Used)
+	swap.Free = metrics.OptUintWith(swUsage.Avail)
+
+	return swap, nil
+}
+
+// generic Sysctl buffer unmarshalling
+func sysctlbyname(name string, data interface{}) (err error) {
+	val, err := syscall.Sysctl(name)
+	if err != nil {
+		return err
+	}
+
+	buf := []byte(val)
+
+	switch v := data.(type) {
+	case *uint64:
+		*v = *(*uint64)(unsafe.Pointer(&buf[0]))
+		return
+	}
+
+	bbuf := bytes.NewBuffer([]byte(val))
+	return binary.Read(bbuf, binary.LittleEndian, data)
+}
+
+func vmInfo(vmstat *C.vm_statistics_data_t) error {
+	var count C.mach_msg_type_number_t = C.HOST_VM_INFO_COUNT
+
+	status := C.host_statistics(
+		C.host_t(C.mach_host_self()),
+		C.HOST_VM_INFO,
+		C.host_info_t(unsafe.Pointer(vmstat)),
+		&count)
+
+	if status != C.KERN_SUCCESS {
+		return fmt.Errorf("host_statistics=%d", status)
+	}
+
+	return nil
+}