feat: Add ignore_older option for local.file_match (#2245)

* feat: Add ignore_older option for local.file_match

* Review comments

* Change the config name

* Push the check for each file instead of dir

Doc update

* Add default value to docs

* Update docs/sources/reference/components/local/local.file_match.md

Co-authored-by: Clayton Cornell <131809008+clayton-cornell@users.noreply.github.com>

* Update docs/sources/reference/components/local/local.file_match.md

---------

Co-authored-by: William Dumont <william.dumont@grafana.com>
Co-authored-by: Clayton Cornell <131809008+clayton-cornell@users.noreply.github.com>

CHANGELOG.md

@@ -49,6 +49,7 @@ Main (unreleased)
   - Add perf_schema quantile columns to collector
 
 - Add three new stdlib functions to_base64, from_URLbase64 and to_URLbase64 (@ravishankar15)
+- Add `ignore_older_than` option for local.file_match (@ravishankar15)
 
 - Use a forked `github.com/goccy/go-json` module which reduces the memory consumption of an Alloy instance by 20MB.
   If Alloy is running certain otelcol components, this reduction will not apply. (@ptodev)
@@ -103,7 +104,7 @@ v1.5.1
 
 - Fixed a crash when updating the configuration of `remote.http`. (@kinolaev)
 
-- Fixed an issue in the `otelcol.processor.attribute` component where the actions `delete` and `hash` could not be used with the `pattern` argument. (@wildum) 
+- Fixed an issue in the `otelcol.processor.attribute` component where the actions `delete` and `hash` could not be used with the `pattern` argument. (@wildum)
 
 - Fixed an issue in the `prometheus.exporter.postgres` component that would leak goroutines when the target was not reachable (@dehaansa)
 
@@ -303,7 +304,7 @@ v1.4.0
 
 - Add the label `alloy_cluster` in the metric `alloy_config_hash` when the flag `cluster.name` is set to help differentiate between
   configs from the same alloy cluster or different alloy clusters. (@wildum)
-  
+
 - Add support for discovering the cgroup path(s) of a process in `process.discovery`. (@mahendrapaipuri)
 
 ### Bugfixes

docs/sources/reference/components/local/local.file_match.md

@@ -24,16 +24,19 @@ local.file_match "LABEL" {
 
 The following arguments are supported:
 
-Name            | Type                | Description                                                                                | Default | Required
---------------- | ------------------- | ------------------------------------------------------------------------------------------ |---------| --------
-`path_targets`  | `list(map(string))` | Targets to expand; looks for glob patterns on the  `__path__` and `__path_exclude__` keys. |         | yes
-`sync_period`   | `duration`          | How often to sync filesystem and targets.                                                  | `"10s"` | no
+Name                | Type                | Description                                                                                | Default | Required
+---------------     | ------------------- | ------------------------------------------------------------------------------------------ |---------| --------
+`path_targets`      | `list(map(string))` | Targets to expand; looks for glob patterns on the  `__path__` and `__path_exclude__` keys. |         | yes
+`sync_period`       | `duration`          | How often to sync filesystem and targets.                                                  | `"10s"` | no
+`ignore_older_than` | `duration`          | Ignores files which are modified before this duration.                                     |  `"0s"` | no
 
 `path_targets` uses [doublestar][] style paths.
 * `/tmp/**/*.log` will match all subfolders of `tmp` and include any files that end in `*.log`.
 * `/tmp/apache/*.log` will match only files in `/tmp/apache/` that end in `*.log`.
 * `/tmp/**` will match all subfolders of `tmp`, `tmp` itself, and all files.
 
+`local.file_match` doesn't ignore files when `ignore_older_than` is set to the default, `0s`.
+
 
 ## Exported fields
 

internal/component/local/file_match/file.go

@@ -26,8 +26,9 @@ func init() {
 // Arguments holds values which are used to configure the local.file_match
 // component.
 type Arguments struct {
-	PathTargets []discovery.Target `alloy:"path_targets,attr"`
-	SyncPeriod  time.Duration      `alloy:"sync_period,attr,optional"`
+	PathTargets     []discovery.Target `alloy:"path_targets,attr"`
+	SyncPeriod      time.Duration      `alloy:"sync_period,attr,optional"`
+	IgnoreOlderThan time.Duration      `alloy:"ignore_older_than,attr,optional"`
 }
 
 var _ component.Component = (*Component)(nil)
@@ -80,8 +81,9 @@ func (c *Component) Update(args component.Arguments) error {
 	c.watches = c.watches[:0]
 	for _, v := range c.args.PathTargets {
 		c.watches = append(c.watches, watch{
-			target: v,
-			log:    c.opts.Logger,
+			target:          v,
+			log:             c.opts.Logger,
+			ignoreOlderThan: c.args.IgnoreOlderThan,
 		})
 	}
 

internal/component/local/file_match/file_test.go

@@ -63,6 +63,35 @@ func TestDirectoryFile(t *testing.T) {
 	require.True(t, contains(foundFiles, "t1.txt"))
 }
 
+func TestFileIgnoreOlder(t *testing.T) {
+	dir := path.Join(os.TempDir(), "alloy_testing", "t1")
+	err := os.MkdirAll(dir, 0755)
+	require.NoError(t, err)
+	writeFile(t, dir, "t1.txt")
+	t.Cleanup(func() {
+		os.RemoveAll(dir)
+	})
+	c := createComponent(t, dir, []string{path.Join(dir, "*.txt")}, nil)
+	ct := context.Background()
+	ct, ccl := context.WithTimeout(ct, 5*time.Second)
+	defer ccl()
+	c.args.SyncPeriod = 10 * time.Millisecond
+	c.args.IgnoreOlderThan = 100 * time.Millisecond
+	c.Update(c.args)
+	go c.Run(ct)
+
+	foundFiles := c.getWatchedFiles()
+	require.Len(t, foundFiles, 1)
+	require.True(t, contains(foundFiles, "t1.txt"))
+	time.Sleep(150 * time.Millisecond)
+
+	writeFile(t, dir, "t2.txt")
+	ct.Done()
+	foundFiles = c.getWatchedFiles()
+	require.Len(t, foundFiles, 1)
+	require.True(t, contains(foundFiles, "t2.txt"))
+}
+
 func TestAddingFile(t *testing.T) {
 	dir := path.Join(os.TempDir(), "alloy_testing", "t2")
 	err := os.MkdirAll(dir, 0755)

internal/component/local/file_match/watch.go

@@ -3,6 +3,7 @@ package file_match
 import (
 	"os"
 	"path/filepath"
+	"time"
 
 	"github.com/bmatcuk/doublestar"
 	"github.com/go-kit/log"
@@ -14,8 +15,9 @@ import (
 
 // watch handles a single discovery.target for file watching.
 type watch struct {
-	target discovery.Target
-	log    log.Logger
+	target          discovery.Target
+	log             log.Logger
+	ignoreOlderThan time.Duration
 }
 
 func (w *watch) getPaths() ([]discovery.Target, error) {
@@ -48,9 +50,15 @@ func (w *watch) getPaths() ([]discovery.Target, error) {
 			}
 			continue
 		}
+
 		if fi.IsDir() {
 			continue
 		}
+
+		if w.ignoreOlderThan != 0 && fi.ModTime().Before(time.Now().Add(-w.ignoreOlderThan)) {
+			continue
+		}
+
 		dt := discovery.Target{}
 		for dk, v := range w.target {
 			dt[dk] = v