-
Notifications
You must be signed in to change notification settings - Fork 4.9k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Enhancements to auditbeat file metricset (#4796)
Modify `file.paths` to be a list instead of a dictionary. Added `file.hash_types` config option for controlling the hash types. Added new hash types. Fixed `file.max_file_size` config option. It was missing the "file." prefix when reading the option. Added the ability to specify byte unit suffixes to `file.max_file_size`. Added documentation for the audit file metricset.
- Loading branch information
1 parent
eb92974
commit 07170c6
Showing
16 changed files
with
335 additions
and
173 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,33 +1,35 @@ | ||
| { | ||
| "@timestamp": "2017-07-10T18:13:08.197Z", | ||
| "audit": { | ||
| "file": { | ||
| "action": "created", | ||
| "atime": "2017-07-10T18:13:08Z", | ||
| "ctime": "2017-07-10T18:13:08Z", | ||
| "gid": 20, | ||
| "group": "staff", | ||
| "hashed": true, | ||
| "inode": "11706872", | ||
| "md5": "5eb63bbbe01eeed093cb22bb8f5acdc3", | ||
| "mode": "0600", | ||
| "mtime": "2017-07-10T18:13:08Z", | ||
| "owner": "akroh", | ||
| "path": "/var/folders/8x/rnyk6yxn6w97lddn3bs02gf00000gn/T/audit-file232876074/file.data", | ||
| "sha1": "2aae6c35c94fcfb415dbe95f408b9ce91ee846ed", | ||
| "sha256": "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9", | ||
| "size": 11, | ||
| "type": "file", | ||
| "uid": 501 | ||
| } | ||
| }, | ||
| "beat": { | ||
| "hostname": "host.example.com", | ||
| "name": "host.example.com" | ||
| }, | ||
| "metricset": { | ||
| "module": "audit", | ||
| "name": "file", | ||
| "rtt": 115 | ||
| "@timestamp": "2017-07-31T14:40:28.173Z", | ||
| "@metadata": { | ||
| "beat": "noindex", | ||
| "type": "doc" | ||
| }, | ||
| "audit": { | ||
| "file": { | ||
| "hashed": true, | ||
| "group": "staff", | ||
| "path": "/var/folders/8x/rnyk6yxn6w97lddn3bs02gf00000gn/T/audit-file927187403/file.data", | ||
| "inode": "12512120", | ||
| "size": 11, | ||
| "ctime": "2017-07-31T14:40:28.000Z", | ||
| "mode": "0600", | ||
| "owner": "akroh", | ||
| "atime": "2017-07-31T14:40:28.000Z", | ||
| "sha1": "2aae6c35c94fcfb415dbe95f408b9ce91ee846ed", | ||
| "action": "created", | ||
| "mtime": "2017-07-31T14:40:28.000Z", | ||
| "type": "file", | ||
| "uid": 501, | ||
| "gid": 20 | ||
| } | ||
| }, | ||
| "metricset": { | ||
| "module": "audit", | ||
| "name": "file", | ||
| "rtt": 115 | ||
| }, | ||
| "beat": { | ||
| "name": "host.example.com", | ||
| "hostname": "host.example.com" | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,36 +1,67 @@ | ||
| === Audit file metricset | ||
|
|
||
| The `file` metricset sends events when a file is changed (created, updated, or | ||
| deleted) on disk. The events contain file metadata and hashes (MD5, SHA1, and | ||
| SHA256). | ||
| deleted) on disk. The events contain file metadata and hashes. | ||
|
|
||
| The metricset is implemented for Linux, macOS (Darwin), and Windows. | ||
|
|
||
| //// | ||
| [float] | ||
| === How it works | ||
|
|
||
| This metricset... | ||
| //// | ||
| This metricset uses features of the operating system to monitor file changes in | ||
| realtime. When the metricset starts it creates a subscription with the OS to | ||
| receive notifications of changes to the specified files or directories. Upon | ||
| receiving notification of a change the metricset will read the file's metadata | ||
| and the compute a hash of the file's contents. | ||
|
|
||
| This metricset does not perform an initial scan of the given paths to generate | ||
| events for the files as they existed at start time. It is strictly driven by | ||
| change notifications from the OS. | ||
|
|
||
| The operating system features that power this feature are as follows. | ||
|
|
||
| * Linux - `inotify` is used, and therefore the kernel must have inotify support. | ||
| Inotify was initially merged into the 2.6.13 Linux kernel. | ||
| * macOS (Darwin) - `kqueue` is used. It requires one file descriptor for each | ||
| file so please check the `ulimit` values used with {beatname_uc}. The FSEvents | ||
| API was considered for the implementation, but FSEvents coalesces multiple | ||
| notifications into a single event which is inconsistent with the metricset's | ||
| behavior on other operating systems. | ||
| * Windows - `ReadDirectoryChangesW` is used. | ||
|
|
||
| The file metricset should not be used to monitor paths on network file systems. | ||
|
|
||
| [float] | ||
| === Configuration options | ||
|
|
||
| This metricset has some configuration options for tuning its behavior. The | ||
| following example shows all configuration options with their default values. | ||
| following example shows all configuration options with their default values for | ||
| Linux. | ||
|
|
||
| [source,yaml] | ||
| ---- | ||
| - module: audit | ||
| metricsets: [file] | ||
| file.paths: | ||
| binaries: | ||
| - /bin | ||
| - /usr/bin | ||
| - /sbin | ||
| - /usr/sbin | ||
| conf: | ||
| - /etc | ||
| - /bin | ||
| - /usr/bin | ||
| - /sbin | ||
| - /usr/sbin | ||
| - /etc | ||
| file.max_file_size: 100 MiB | ||
| file.hash_types: [sha1] | ||
| ---- | ||
|
|
||
| *`file.paths`*:: A list of paths (directories or files) to watch. The watches | ||
| are non-recursive and globs are not supported. The specified paths should exist | ||
| when the metricset is started. | ||
|
|
||
| *`file.max_file_size`*:: The maximum size of a file in bytes for which | ||
| {beatname_uc} will compute hashes. Files larger than this size will not be | ||
| hashed. The default value is 100 MiB. For convenience units can be specified as | ||
| a suffix to the value. The supported units are `b`, `kib`, `kb`, `mib`, `mb`, | ||
| `gib`, `gb`, `tib`, `tb`, `pib`, `pb`, `eib`, and `eb`. | ||
|
|
||
| *`file.paths`*:: A dictionary of arbitrary names (strings) to a list of paths | ||
| (directories or files). | ||
| *`file.hash_types`*:: A list of hash types to compute when the file changes. | ||
| The supported hash types are md5, sha1, sha224, sha256, sha384, sha512, | ||
| sha512_224, and sha512_256. The default value is sha1. |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,42 @@ | ||
| package file | ||
|
|
||
| import ( | ||
| "strings" | ||
|
|
||
| "github.com/dustin/go-humanize" | ||
| "github.com/joeshaw/multierror" | ||
| "github.com/pkg/errors" | ||
| ) | ||
|
|
||
| type Config struct { | ||
| Paths []string `config:"file.paths" validate:"required"` | ||
| HashTypes []string `config:"file.hash_types"` | ||
| MaxFileSize string `config:"file.max_file_size"` | ||
| MaxFileSizeBytes uint64 `config:",ignore"` | ||
| } | ||
|
|
||
| func (c *Config) Validate() error { | ||
| var errs multierror.Errors | ||
| var err error | ||
|
|
||
| c.MaxFileSizeBytes, err = humanize.ParseBytes(c.MaxFileSize) | ||
| if err != nil { | ||
| errs = append(errs, errors.Wrap(err, "invalid file.max_file_size value")) | ||
| } | ||
|
|
||
| for _, ht := range c.HashTypes { | ||
| switch strings.ToLower(ht) { | ||
| case "md5", "sha1", "sha224", "sha256", "sha384", "sha512", "sha512_224", "sha512_256": | ||
| default: | ||
| errs = append(errs, errors.Errorf("invalid hash type '%v'", ht)) | ||
| } | ||
| } | ||
|
|
||
| return errs.Err() | ||
| } | ||
|
|
||
| var defaultConfig = Config{ | ||
| MaxFileSize: "100 MiB", | ||
| MaxFileSizeBytes: 100 * 1024 * 1024, | ||
| HashTypes: []string{"sha1"}, | ||
| } |
Oops, something went wrong.