[7.17](backport #31001) Filesystem refactor (#33022)

* Filesystem refactor (#31001)

* first pass at filesystem refactoring

* cleanup code

* final cleanup

* cleanup, docs

* license headers

* make linter happy

* try to hack around linter platform issues

* fix function sig

* minor fixes, formatting

* try to make linter happy

* more linter fixes

* hopefully the last linter cleanup

* try to fix unit tests

* fix tests

* refactor after merges from elastic-agent libs

* make linter happy

* try to update deps

* okay, try that

* fix merge

* add changelog, revert opt changes

* fix broken merge

* mage update

(cherry picked from commit 1d4a7ca)

# Conflicts:
#	go.sum
#	metricbeat/module/system/fields.go
#	metricbeat/module/system/filesystem/_meta/fields.yml
#	metricbeat/module/system/filesystem/filesystem.go
#	metricbeat/module/system/filesystem/filesystem_test.go
#	metricbeat/module/system/filesystem/helper.go
#	metricbeat/module/system/fsstat/fsstat.go
#	metricbeat/module/system/test_system.py

* attempt to clean up backport

* update notice

* try to make linter happy

* still fixing up backport

* still un-breaking merge

* fix changelog

Co-authored-by: Alex K <8418476+fearful-symmetry@users.noreply.github.com>
Co-authored-by: Alex Kristiansen <alex.kristiansen@elastic.co>

CHANGELOG.next.asciidoc

@@ -52,6 +52,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 - in module/windows/perfmon, changed collection method of the second counter value required to create a displayable value {pull}32305[32305]
 - Fix and improve AWS metric period calculation to avoid zero-length intervals {pull}32724[32724]
+- make `system/filesystem` code sensitive to `hostfs` and migrate libraries to `elastic-agent-opts` {pull}31001[31001]
 
 *Packetbeat*
 

go.mod

@@ -80,15 +80,15 @@ require (
 	github.com/fatih/color v1.13.0
 	github.com/fsnotify/fsevents v0.1.1
 	github.com/fsnotify/fsnotify v1.5.1
-	github.com/go-ole/go-ole v1.2.5-0.20190920104607-14974a1cf647 // indirect
+	github.com/go-ole/go-ole v1.2.6 // indirect
 	github.com/go-sourcemap/sourcemap v2.1.2+incompatible // indirect
 	github.com/go-sql-driver/mysql v1.5.0
 	github.com/go-test/deep v1.0.7
 	github.com/gocarina/gocsv v0.0.0-20170324095351-ffef3ffc77be
 	github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e
 	github.com/godror/godror v0.10.4
 	github.com/gofrs/flock v0.7.2-0.20190320160742-5135e617513b
-	github.com/gofrs/uuid v3.3.0+incompatible
+	github.com/gofrs/uuid v4.2.0+incompatible
 	github.com/gogo/protobuf v1.3.2
 	github.com/golang/mock v1.6.0
 	github.com/golang/protobuf v1.5.2
@@ -159,7 +159,7 @@ require (
 	go.elastic.co/apm v1.11.0
 	go.elastic.co/apm/module/apmelasticsearch v1.7.2
 	go.elastic.co/apm/module/apmhttp v1.7.2
-	go.elastic.co/ecszap v1.0.0
+	go.elastic.co/ecszap v1.0.1
 	go.elastic.co/go-licence-detector v0.5.0
 	go.etcd.io/bbolt v1.3.6
 	go.uber.org/atomic v1.9.0
@@ -195,6 +195,11 @@ require (
 	kernel.org/pub/linux/libs/security/libcap/cap v1.2.57
 )
 
+require (
+	github.com/elastic/elastic-agent-libs v0.2.11
+	github.com/elastic/elastic-agent-system-metrics v0.4.4
+)
+
 require (
 	cloud.google.com/go v0.99.0 // indirect
 	code.cloudfoundry.org/gofileutils v0.0.0-20170111115228-4d0c80011a0f // indirect
@@ -238,7 +243,7 @@ require (
 	github.com/googleapis/gax-go/v2 v2.1.1 // indirect
 	github.com/googleapis/gnostic v0.4.1 // indirect
 	github.com/gorilla/websocket v1.4.2 // indirect
-	github.com/hashicorp/errwrap v1.0.0 // indirect
+	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/hashicorp/go-rootcerts v1.0.2 // indirect
 	github.com/hashicorp/go-uuid v1.0.2 // indirect
@@ -252,7 +257,7 @@ require (
 	github.com/jcmturner/rpc/v2 v2.0.3 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/karrick/godirwalk v1.15.6 // indirect
+	github.com/karrick/godirwalk v1.15.8 // indirect
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
 	github.com/klauspost/compress v1.13.6 // indirect
 	github.com/markbates/pkger v0.17.0 // indirect

metricbeat/docs/fields.asciidoc

@@ -52125,13 +52125,23 @@ type: keyword
 *`system.filesystem.files`*::
 +
 --
-The total number of file nodes in the file system.
+Total number of inodes on the system, which will be a combination of files, folders, symlinks, and devices.
 
 
 type: long
 
 --
 
+*`system.filesystem.options`*::
++
+--
+The options present on the filesystem mount.
+
+
+type: keyword
+
+--
+
 *`system.filesystem.free`*::
 +
 --

metricbeat/module/system/filesystem/_meta/docs.asciidoc

@@ -16,7 +16,8 @@ This metricset is available on:
 not be collected from filesystems matching these types. This setting also
 affects the `fsstats` metricset. If this option is not set, metricbeat ignores
 all types for virtual devices in systems where this information is available (e.g.
-all types marked as `nodev` in `/proc/filesystems` in Linux systems).
+all types marked as `nodev` in `/proc/filesystems` in Linux systems). This can be set to an empty list (`[]`)
+to make filebeat report all filesystems, regardless of type.
 
 [float]
 === Filtering

metricbeat/module/system/filesystem/_meta/fields.yml

@@ -24,7 +24,11 @@
     - name: files
       type: long
       description: >
-        The total number of file nodes in the file system.
+         Total number of inodes on the system, which will be a combination of files, folders, symlinks, and devices.
+    - name: options
+      type: keyword
+      description: >
+        The options present on the filesystem mount.
     - name: free
       type: long
       format: bytes

metricbeat/module/system/filesystem/filesystem.go

@@ -21,17 +21,20 @@
 package filesystem
 
 import (
+	"fmt"
 	"strings"
 
+	"github.com/elastic/beats/v7/libbeat/common"
 	"github.com/elastic/beats/v7/libbeat/logp"
 	"github.com/elastic/beats/v7/metricbeat/mb"
 	"github.com/elastic/beats/v7/metricbeat/mb/parse"
+	"github.com/elastic/beats/v7/metricbeat/module/system"
 
-	"github.com/pkg/errors"
+	"github.com/elastic/beats/v7/libbeat/common/transform/typeconv"
+	fs "github.com/elastic/elastic-agent-system-metrics/metric/system/filesystem"
+	"github.com/elastic/elastic-agent-system-metrics/metric/system/resolve"
 )
 
-var debugf = logp.MakeDebug("system.filesystem")
-
 func init() {
 	mb.Registry.MustAddMetricSet("system", "filesystem", New,
 		mb.WithHostParser(parse.EmptyHostParser),
@@ -42,6 +45,12 @@ func init() {
 type MetricSet struct {
 	mb.BaseMetricSet
 	config Config
+	sys    resolve.Resolver
+}
+
+// Config stores the metricset-local config
+type Config struct {
+	IgnoreTypes []string `config:"filesystem.ignore_types"`
 }
 
 // New creates and returns a new instance of MetricSet.
@@ -50,51 +59,46 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
 	if err := base.Module().UnpackConfig(&config); err != nil {
 		return nil, err
 	}
-
+	sys, ok := base.Module().(system.SystemModule)
+	if !ok {
+		return nil, fmt.Errorf("resolver cannot be cast from the module")
+	}
+	wrapper := resolve.NewTestResolver(sys.GetHostFS())
 	if config.IgnoreTypes == nil {
-		config.IgnoreTypes = DefaultIgnoredTypes()
+		config.IgnoreTypes = fs.DefaultIgnoredTypes(wrapper)
 	}
 	if len(config.IgnoreTypes) > 0 {
 		logp.Info("Ignoring filesystem types: %s", strings.Join(config.IgnoreTypes, ", "))
 	}
-
 	return &MetricSet{
 		BaseMetricSet: base,
 		config:        config,
+		sys:           wrapper,
 	}, nil
 }
 
 // Fetch fetches filesystem metrics for all mounted filesystems and returns
 // an event for each mount point.
 func (m *MetricSet) Fetch(r mb.ReporterV2) error {
-	fss, err := GetFileSystemList()
-	if err != nil {
-		return errors.Wrap(err, "error getting filesystem list")
-
-	}
 
-	if len(m.config.IgnoreTypes) > 0 {
-		fss = Filter(fss, BuildTypeFilter(m.config.IgnoreTypes...))
+	fsList, err := fs.GetFilesystems(m.sys, fs.BuildFilterWithList(m.config.IgnoreTypes))
+	if err != nil {
+		return fmt.Errorf("error fetching filesystem list: %w", err)
 	}
 
-	for _, fs := range fss {
-		stat, err := GetFileSystemStat(fs)
-		addStats := true
+	for _, fs := range fsList {
+		err := fs.GetUsage()
 		if err != nil {
-			addStats = false
-			m.Logger().Debugf("error fetching filesystem stats for '%s': %v", fs.DirName, err)
+			return fmt.Errorf("error getting filesystem usage for %s: %w", fs.Directory, err)
 		}
-		fsStat := FSStat{
-			FileSystemUsage: stat,
-			DevName:         fs.DevName,
-			Mount:           fs.DirName,
-			SysTypeName:     fs.SysTypeName,
+		out := common.MapStr{}
+		err = typeconv.Convert(&out, fs)
+		if err != nil {
+			return fmt.Errorf("error converting event %s: %w", fs.Device, err)
 		}
 
-		AddFileSystemUsedPercentage(&fsStat)
-
 		event := mb.Event{
-			MetricSetFields: GetFilesystemEvent(&fsStat, addStats),
+			MetricSetFields: out,
 		}
 		if !r.Event(event) {
 			return nil

metricbeat/module/system/filesystem/filesystem_test.go

@@ -26,18 +26,26 @@ import (
 	"github.com/stretchr/testify/assert"
 
 	mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing"
+	_ "github.com/elastic/beats/v7/metricbeat/module/system"
+	"github.com/elastic/elastic-agent-libs/logp"
+	fs "github.com/elastic/elastic-agent-system-metrics/metric/system/filesystem"
+	"github.com/elastic/elastic-agent-system-metrics/metric/system/resolve"
 )
 
 func TestFetch(t *testing.T) {
 	f := mbtest.NewReportingMetricSetV2Error(t, getConfig())
 	events, errs := mbtest.ReportingFetchV2Error(f)
-
+	err := logp.DevelopmentSetup()
+	assert.NoError(t, err)
 	assert.Empty(t, errs)
 	if !assert.NotEmpty(t, events) {
 		t.FailNow()
 	}
-	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(),
-		events[0].BeatEvent("system", "filesystem").Fields.StringToPrint())
+	for _, event := range events {
+		t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(),
+			event.BeatEvent("system", "filesystem").Fields.StringToPrint())
+	}
+
 }
 
 func TestData(t *testing.T) {
@@ -49,7 +57,7 @@ func TestData(t *testing.T) {
 }
 
 func getConfig() map[string]interface{} {
-	ignoreTypes := append(DefaultIgnoredTypes(), "fuse.lxcfs", "fuse.gvfsd-fuse", "nsfs", "squashfs")
+	ignoreTypes := append(fs.DefaultIgnoredTypes(resolve.NewTestResolver("")), "fuse.lxcfs", "fuse.gvfsd-fuse", "nsfs", "squashfs")
 	return map[string]interface{}{
 		"module":                  "system",
 		"metricsets":              []string{"filesystem"},