Filesystem refactor (#31001)

* first pass at filesystem refactoring

* cleanup code

* final cleanup

* cleanup, docs

* license headers

* make linter happy

* try to hack around linter platform issues

* fix function sig

* minor fixes, formatting

* try to make linter happy

* more linter fixes

* hopefully the last linter cleanup

* try to fix unit tests

* fix tests

* refactor after merges from elastic-agent libs

* make linter happy

* try to update deps

* okay, try that

* fix merge

* add changelog, revert opt changes

* fix broken merge

* mage update

CHANGELOG.next.asciidoc

@@ -70,6 +70,7 @@ https://github.com/elastic/beats/compare/v8.2.0\...main[Check the HEAD diff]
 *Metricbeat*
 
 - Improve handling of disabled commands in Zookeeper Metricbeat module. {pull}31013[#31013]
+- make `system/filesystem` code sensitive to `hostfs` and migrate libraries to `elastic-agent-opts` {pull}31001[31001]
 
 *Packetbeat*
 

go.sum

@@ -538,11 +538,11 @@ github.com/elastic/elastic-agent-client/v7 v7.0.0-20210727140539-f0905d9377f6/go
 github.com/elastic/elastic-agent-libs v0.0.0-20220303160015-5b4e674da3dd/go.mod h1://82M1l73IHx0wDbS2Tzkq6Fx9fkmytS1KgkIyzvNTM=
 github.com/elastic/elastic-agent-libs v0.2.2/go.mod h1:1xDLBhIqBIjhJ7lr2s+xRFFkQHpitSp8q2zzv1Dqg+s=
 github.com/elastic/elastic-agent-libs v0.2.4 h1:TOy+vild5MSkn/eTOwrnffAeAntq4GiLpkvWe+uNVms=
+github.com/elastic/elastic-agent-libs v0.2.4 h1:TOy+vild5MSkn/eTOwrnffAeAntq4GiLpkvWe+uNVms=
+github.com/elastic/elastic-agent-libs v0.2.4/go.mod h1:eUiaofWIVdxVOAR4ICGxn2wbFByhrnmR6/kppwYq3qI=
 github.com/elastic/elastic-agent-libs v0.2.4/go.mod h1:eUiaofWIVdxVOAR4ICGxn2wbFByhrnmR6/kppwYq3qI=
 github.com/elastic/elastic-agent-system-metrics v0.3.1 h1:WXdDyIaBr9zIJoyvFNzgZbcFiEnmtaKfazHPJR4DJOM=
 github.com/elastic/elastic-agent-system-metrics v0.3.1/go.mod h1:RIYhJOS7mUeyIthfOSqmmbEILYSzaDWLi5zQ70bQo+o=
-github.com/elastic/elastic-agent-libs v0.2.4 h1:TOy+vild5MSkn/eTOwrnffAeAntq4GiLpkvWe+uNVms=
-github.com/elastic/elastic-agent-libs v0.2.4/go.mod h1:eUiaofWIVdxVOAR4ICGxn2wbFByhrnmR6/kppwYq3qI=
 github.com/elastic/fsevents v0.0.0-20181029231046-e1d381a4d270 h1:cWPqxlPtir4RoQVCpGSRXmLqjEHpJKbR60rxh1nQZY4=
 github.com/elastic/fsevents v0.0.0-20181029231046-e1d381a4d270/go.mod h1:Msl1pdboCbArMF/nSCDUXgQuWTeoMmE/z8607X+k7ng=
 github.com/elastic/glog v1.0.1-0.20210831205241-7d8b5c89dfc4 h1:ViJxdtOsHeO+SWVekzM82fYHH1xnvZ8CvGPXZj+G4YI=

metricbeat/docs/fields.asciidoc

@@ -60259,6 +60259,16 @@ type: long
 
 --
 
+*`system.filesystem.options`*::
++
+--
+The options present on the filesystem mount.
+
+
+type: keyword
+
+--
+
 *`system.filesystem.free`*::
 +
 --

metricbeat/module/system/filesystem/_meta/docs.asciidoc

@@ -16,7 +16,8 @@ This metricset is available on:
 not be collected from filesystems matching these types. This setting also
 affects the `fsstats` metricset. If this option is not set, metricbeat ignores
 all types for virtual devices in systems where this information is available (e.g.
-all types marked as `nodev` in `/proc/filesystems` in Linux systems).
+all types marked as `nodev` in `/proc/filesystems` in Linux systems). This can be set to an empty list (`[]`)
+to make filebeat report all filesystems, regardless of type.
 
 [float]
 === Filtering

metricbeat/module/system/filesystem/_meta/fields.yml

@@ -25,6 +25,10 @@
       type: long
       description: >
         Total number of inodes on the system, which will be a combination of files, folders, symlinks, and devices.
+    - name: options
+      type: keyword
+      description: >
+        The options present on the filesystem mount.
     - name: free
       type: long
       format: bytes

metricbeat/module/system/filesystem/filesystem.go

@@ -21,28 +21,35 @@
 package filesystem
 
 import (
+	"fmt"
 	"strings"
 
+	"github.com/elastic/beats/v7/libbeat/common/transform/typeconv"
+
 	"github.com/elastic/beats/v7/metricbeat/mb"
 	"github.com/elastic/beats/v7/metricbeat/mb/parse"
 	"github.com/elastic/elastic-agent-libs/logp"
+	"github.com/elastic/elastic-agent-libs/mapstr"
+	fs "github.com/elastic/elastic-agent-system-metrics/metric/system/filesystem"
 	"github.com/elastic/elastic-agent-system-metrics/metric/system/resolve"
-
-	"github.com/pkg/errors"
 )
 
-var debugf = logp.MakeDebug("system.filesystem")
-
 func init() {
 	mb.Registry.MustAddMetricSet("system", "filesystem", New,
 		mb.WithHostParser(parse.EmptyHostParser),
 	)
 }
 
+// Config stores the metricset-local config
+type Config struct {
+	IgnoreTypes []string `config:"filesystem.ignore_types"`
+}
+
 // MetricSet for fetching filesystem metrics.
 type MetricSet struct {
 	mb.BaseMetricSet
 	config Config
+	sys    resolve.Resolver
 }
 
 // New creates and returns a new instance of MetricSet.
@@ -51,51 +58,46 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
 	if err := base.Module().UnpackConfig(&config); err != nil {
 		return nil, err
 	}
-	sys := base.Module().(resolve.Resolver)
+	sys, ok := base.Module().(resolve.Resolver)
+	if !ok {
+		return nil, fmt.Errorf("resolver cannot be cast from the module")
+	}
+
 	if config.IgnoreTypes == nil {
-		config.IgnoreTypes = DefaultIgnoredTypes(sys)
+		config.IgnoreTypes = fs.DefaultIgnoredTypes(sys)
 	}
 	if len(config.IgnoreTypes) > 0 {
 		logp.Info("Ignoring filesystem types: %s", strings.Join(config.IgnoreTypes, ", "))
 	}
-
 	return &MetricSet{
 		BaseMetricSet: base,
 		config:        config,
+		sys:           sys,
 	}, nil
 }
 
 // Fetch fetches filesystem metrics for all mounted filesystems and returns
 // an event for each mount point.
 func (m *MetricSet) Fetch(r mb.ReporterV2) error {
-	fss, err := GetFileSystemList()
-	if err != nil {
-		return errors.Wrap(err, "error getting filesystem list")
-
-	}
 
-	if len(m.config.IgnoreTypes) > 0 {
-		fss = Filter(fss, BuildTypeFilter(m.config.IgnoreTypes...))
+	fsList, err := fs.GetFilesystems(m.sys, fs.BuildFilterWithList(m.config.IgnoreTypes))
+	if err != nil {
+		return fmt.Errorf("error fetching filesystem list: %w", err)
 	}
 
-	for _, fs := range fss {
-		stat, err := GetFileSystemStat(fs)
-		addStats := true
+	for _, fs := range fsList {
+		err := fs.GetUsage()
 		if err != nil {
-			addStats = false
-			m.Logger().Debugf("error fetching filesystem stats for '%s': %v", fs.DirName, err)
+			return fmt.Errorf("error getting filesystem usage for %s: %w", fs.Directory, err)
 		}
-		fsStat := FSStat{
-			FileSystemUsage: stat,
-			DevName:         fs.DevName,
-			Mount:           fs.DirName,
-			SysTypeName:     fs.SysTypeName,
+		out := mapstr.M{}
+		err = typeconv.Convert(&out, fs)
+		if err != nil {
+			return fmt.Errorf("error converting event %s: %w", fs.Device, err)
 		}
 
-		AddFileSystemUsedPercentage(&fsStat)
-
 		event := mb.Event{
-			MetricSetFields: GetFilesystemEvent(&fsStat, addStats),
+			MetricSetFields: out,
 		}
 		if !r.Event(event) {
 			return nil

metricbeat/module/system/filesystem/filesystem_test.go

@@ -27,19 +27,25 @@ import (
 
 	mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing"
 	_ "github.com/elastic/beats/v7/metricbeat/module/system"
+	"github.com/elastic/elastic-agent-libs/logp"
+	fs "github.com/elastic/elastic-agent-system-metrics/metric/system/filesystem"
 	"github.com/elastic/elastic-agent-system-metrics/metric/system/resolve"
 )
 
 func TestFetch(t *testing.T) {
 	f := mbtest.NewReportingMetricSetV2Error(t, getConfig())
 	events, errs := mbtest.ReportingFetchV2Error(f)
-
+	err := logp.DevelopmentSetup()
+	assert.NoError(t, err)
 	assert.Empty(t, errs)
 	if !assert.NotEmpty(t, events) {
 		t.FailNow()
 	}
-	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(),
-		events[0].BeatEvent("system", "filesystem").Fields.StringToPrint())
+	for _, event := range events {
+		t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(),
+			event.BeatEvent("system", "filesystem").Fields.StringToPrint())
+	}
+
 }
 
 func TestData(t *testing.T) {
@@ -51,7 +57,7 @@ func TestData(t *testing.T) {
 }
 
 func getConfig() map[string]interface{} {
-	ignoreTypes := append(DefaultIgnoredTypes(resolve.NewTestResolver("")), "fuse.lxcfs", "fuse.gvfsd-fuse", "nsfs", "squashfs")
+	ignoreTypes := append(fs.DefaultIgnoredTypes(resolve.NewTestResolver("")), "fuse.lxcfs", "fuse.gvfsd-fuse", "nsfs", "squashfs")
 	return map[string]interface{}{
 		"module":                  "system",
 		"metricsets":              []string{"filesystem"},