From 887aa12e72d3c6ece69b925f33cce440750d5f34 Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Tue, 1 Dec 2020 15:15:05 +0100 Subject: [PATCH] Auditbeat: stop monitoring excluded paths (#21282) (#22486) If the files match the excluded pattern, recursiveWatcher does not emit events for them, as well as errors if any. (cherry picked from commit af8eedd9e93b59edef6e8e1df161d739032b67e6) Co-authored-by: Peter Deng --- CHANGELOG.next.asciidoc | 1 + .../file_integrity/eventreader_fsnotify.go | 18 +-- .../module/file_integrity/monitor/monitor.go | 10 +- .../file_integrity/monitor/monitor_test.go | 110 +++++++++++++++++- .../file_integrity/monitor/recursive.go | 25 ++-- 5 files changed, 140 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index d34b883a6e2c..91aa59c4bef8 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -170,6 +170,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - system/socket: Fixed tracking of long-running connections. {pull}19033[19033] - auditd: Fix an error condition causing a lot of `audit_send_reply` kernel threads being created. {pull}22673[22673] - system/socket: Fixed start failure when run under config reloader. {issue}20851[20851] {pull}21693[21693] +- file_integrity: stop monitoring excluded paths {issue}21278[21278] {pull}21282[21282] *Filebeat* diff --git a/auditbeat/module/file_integrity/eventreader_fsnotify.go b/auditbeat/module/file_integrity/eventreader_fsnotify.go index 4c59191dfe71..b80085f4b2dc 100644 --- a/auditbeat/module/file_integrity/eventreader_fsnotify.go +++ b/auditbeat/module/file_integrity/eventreader_fsnotify.go @@ -39,20 +39,22 @@ type reader struct { // NewEventReader creates a new EventProducer backed by fsnotify. func NewEventReader(c Config) (EventProducer, error) { - watcher, err := monitor.New(c.Recursive) - if err != nil { - return nil, err - } - return &reader{ - watcher: watcher, - config: c, - log: logp.NewLogger(moduleName), + config: c, + log: logp.NewLogger(moduleName), }, nil } func (r *reader) Start(done <-chan struct{}) (<-chan Event, error) { + watcher, err := monitor.New(r.config.Recursive, r.config.IsExcludedPath) + if err != nil { + return nil, err + } + + r.watcher = watcher if err := r.watcher.Start(); err != nil { + // Ensure that watcher is closed so that we don't leak watchers + r.watcher.Close() return nil, errors.Wrap(err, "unable to start watcher") } diff --git a/auditbeat/module/file_integrity/monitor/monitor.go b/auditbeat/module/file_integrity/monitor/monitor.go index d4da03378486..8485da65234f 100644 --- a/auditbeat/module/file_integrity/monitor/monitor.go +++ b/auditbeat/module/file_integrity/monitor/monitor.go @@ -37,15 +37,15 @@ type Watcher interface { // New creates a new Watcher backed by fsnotify with optional recursive // logic. -func New(recursive bool) (Watcher, error) { - fsnotify, err := fsnotify.NewWatcher() +func New(recursive bool, IsExcludedPath func(path string) bool) (Watcher, error) { + watcher, err := fsnotify.NewWatcher() if err != nil { return nil, err } // Use our simulated recursive watches unless the fsnotify implementation // supports OS-provided recursive watches - if recursive && fsnotify.SetRecursive() != nil { - return newRecursiveWatcher(fsnotify), nil + if recursive && watcher.SetRecursive() != nil { + return newRecursiveWatcher(watcher, IsExcludedPath), nil } - return (*nonRecursiveWatcher)(fsnotify), nil + return (*nonRecursiveWatcher)(watcher), nil } diff --git a/auditbeat/module/file_integrity/monitor/monitor_test.go b/auditbeat/module/file_integrity/monitor/monitor_test.go index 9b028bae83ab..117d25cb1b96 100644 --- a/auditbeat/module/file_integrity/monitor/monitor_test.go +++ b/auditbeat/module/file_integrity/monitor/monitor_test.go @@ -32,6 +32,10 @@ import ( "github.com/stretchr/testify/assert" ) +func alwaysInclude(path string) bool { + return false +} + func TestNonRecursive(t *testing.T) { dir, err := ioutil.TempDir("", "monitor") assertNoError(t, err) @@ -44,7 +48,7 @@ func TestNonRecursive(t *testing.T) { } defer os.RemoveAll(dir) - watcher, err := New(false) + watcher, err := New(false, alwaysInclude) assertNoError(t, err) assertNoError(t, watcher.Add(dir)) @@ -92,7 +96,7 @@ func TestRecursive(t *testing.T) { } defer os.RemoveAll(dir) - watcher, err := New(true) + watcher, err := New(true, alwaysInclude) assertNoError(t, err) assertNoError(t, watcher.Add(dir)) @@ -147,7 +151,7 @@ func TestRecursiveNoFollowSymlink(t *testing.T) { // Start the watcher - watcher, err := New(true) + watcher, err := New(true, alwaysInclude) assertNoError(t, err) assertNoError(t, watcher.Add(dir)) @@ -208,7 +212,7 @@ func TestRecursiveSubdirPermissions(t *testing.T) { // Setup watches on watched dir - watcher, err := New(true) + watcher, err := New(true, alwaysInclude) assertNoError(t, err) assertNoError(t, watcher.Start()) @@ -263,6 +267,104 @@ func TestRecursiveSubdirPermissions(t *testing.T) { } } +func TestRecursiveExcludedPaths(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Skipping permissions test on Windows") + } + + // Create dir to be watched + + dir, err := ioutil.TempDir("", "monitor") + assertNoError(t, err) + if runtime.GOOS == "darwin" { + if dirAlt, err := filepath.EvalSymlinks(dir); err == nil { + dir = dirAlt + } + } + defer os.RemoveAll(dir) + + // Create not watched dir + + outDir, err := ioutil.TempDir("", "non-watched") + assertNoError(t, err) + if runtime.GOOS == "darwin" { + if dirAlt, err := filepath.EvalSymlinks(outDir); err == nil { + outDir = dirAlt + } + } + defer os.RemoveAll(outDir) + + // Populate not watched subdir + + for _, name := range []string{"a", "b", "c"} { + path := filepath.Join(outDir, name) + assertNoError(t, os.Mkdir(path, 0755)) + assertNoError(t, ioutil.WriteFile(filepath.Join(path, name), []byte("Hello"), 0644)) + } + + // excludes file/dir named "b" + selectiveExclude := func(path string) bool { + r := filepath.Base(path) == "b" + t.Logf("path: %v, excluded: %v\n", path, r) + return r + } + + // Setup watches on watched dir + + watcher, err := New(true, selectiveExclude) + assertNoError(t, err) + + assertNoError(t, watcher.Start()) + assertNoError(t, watcher.Add(dir)) + + defer func() { + assertNoError(t, watcher.Close()) + }() + + // No event is received + + ev, err := readTimeout(t, watcher) + assert.Equal(t, errReadTimeout, err) + if err != errReadTimeout { + t.Fatalf("Expected timeout, got event %+v", ev) + } + + // Move the outside directory into the watched + + dest := filepath.Join(dir, "subdir") + assertNoError(t, os.Rename(outDir, dest)) + + // Receive all events + + var evs []fsnotify.Event + for { + // No event is received + ev, err := readTimeout(t, watcher) + if err == errReadTimeout { + break + } + assertNoError(t, err) + evs = append(evs, ev) + } + + // Verify that events for all accessible files are received + // "b" and "b/b" are missing as they are excluded + + expected := map[string]fsnotify.Op{ + dest: fsnotify.Create, + filepath.Join(dest, "a"): fsnotify.Create, + filepath.Join(dest, "a/a"): fsnotify.Create, + filepath.Join(dest, "c"): fsnotify.Create, + filepath.Join(dest, "c/c"): fsnotify.Create, + } + assert.Len(t, evs, len(expected)) + for _, ev := range evs { + op, found := expected[ev.Name] + assert.True(t, found, ev.Name) + assert.Equal(t, op, ev.Op) + } +} + func testDirOps(t *testing.T, dir string, watcher Watcher) { fpath := filepath.Join(dir, "file.txt") fpath2 := filepath.Join(dir, "file2.txt") diff --git a/auditbeat/module/file_integrity/monitor/recursive.go b/auditbeat/module/file_integrity/monitor/recursive.go index 8999ba5f8a0f..14cc99379d54 100644 --- a/auditbeat/module/file_integrity/monitor/recursive.go +++ b/auditbeat/module/file_integrity/monitor/recursive.go @@ -36,16 +36,19 @@ type recursiveWatcher struct { addC chan string addErrC chan error log *logp.Logger + + isExcludedPath func(path string) bool } -func newRecursiveWatcher(inner *fsnotify.Watcher) *recursiveWatcher { +func newRecursiveWatcher(inner *fsnotify.Watcher, IsExcludedPath func(path string) bool) *recursiveWatcher { return &recursiveWatcher{ - inner: inner, - tree: FileTree{}, - eventC: make(chan fsnotify.Event, 1), - addC: make(chan string), - addErrC: make(chan error), - log: logp.NewLogger(moduleName), + inner: inner, + tree: FileTree{}, + eventC: make(chan fsnotify.Event, 1), + addC: make(chan string), + addErrC: make(chan error), + log: logp.NewLogger(moduleName), + isExcludedPath: IsExcludedPath, } } @@ -82,8 +85,16 @@ func (watcher *recursiveWatcher) ErrorChannel() <-chan error { } func (watcher *recursiveWatcher) addRecursive(path string) error { + if watcher.isExcludedPath(path) { + return nil + } + var errs multierror.Errors err := filepath.Walk(path, func(path string, info os.FileInfo, fnErr error) error { + if watcher.isExcludedPath(path) { + return nil + } + if fnErr != nil { errs = append(errs, errors.Wrapf(fnErr, "error walking path '%s'", path)) // If FileInfo is not nil, the directory entry can be processed