Fix relative paths in prospectors definitions

CHANGELOG.asciidoc

@@ -52,6 +52,8 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di
 - Fix Filebeat not starting if command line and modules configs are used together. {issue}5376[5376]
 - Add support for adding string tags {pull}5395{5395}
 - Fix race condition when limiting the number of harvesters running in parallel {issue}5458[5458]
+- Fix relative paths in the prospector definitions. {pull}5443[5433]
+- Fix `recursive_globe.enabled` option. {pull}5443[5443]
 
 *Heartbeat*
 

filebeat/docs/filebeat-options.asciidoc

@@ -63,13 +63,15 @@ paths. You can specify one path per line. Each line begins with a dash (-).
 
 [float]
 [[recursive_glob]]
-==== `recursive_glob`
+==== `recursive_glob.enabled`
 
-*`enabled`*:: Enable expanding `**` into recursive glob patterns. With this feature enabled,
-the rightmost `**` in each path is expanded into a fixed
-number of glob patterns. For example: `/foo/**` expands to `/foo`, `/foo/*`,
-`/foo/*/*`, and so on. The feature is disabled by default, and if enabled it expands a single `**`
-into a 8-level deep `*` pattern.
+Enable expanding `**` into recursive glob patterns. With this feature enabled,
+the rightmost `**` in each path is expanded into a fixed number of glob
+patterns. For example: `/foo/**` expands to `/foo`, `/foo/*`, `/foo/*/*`, and so
+on. If enabled it expands a single `**` into a 8-level deep `*` pattern.
+
+This feature is enabled by default, set to `recursive_glob.enabled` to false to
+disable it.
 
 [float]
 ==== `encoding`

filebeat/prospector/log/config.go

@@ -2,6 +2,7 @@ package log
 
 import (
 	"fmt"
+	"path/filepath"
 	"time"
 
 	"github.com/dustin/go-humanize"
@@ -33,6 +34,7 @@ var (
 		TailFiles:      false,
 		ScanSort:       "",
 		ScanOrder:      "asc",
+		RecursiveGlob:  true,
 
 		// Harvester
 		BufferSize: 16 * humanize.KiByte,
@@ -168,14 +170,15 @@ func (c *config) Validate() error {
 	return nil
 }
 
-func (c *config) resolvePaths() error {
-	var paths []string
+// resolveRecursiveGlobs expands `**` from the globs in multiple patterns
+func (c *config) resolveRecursiveGlobs() error {
 	if !c.RecursiveGlob {
 		logp.Debug("prospector", "recursive glob disabled")
-		paths = c.Paths
-	} else {
-		logp.Debug("prospector", "recursive glob enabled")
+		return nil
 	}
+
+	logp.Debug("prospector", "recursive glob enabled")
+	var paths []string
 	for _, path := range c.Paths {
 		patterns, err := file.GlobPatterns(path, recursiveGlobDepth)
 		if err != nil {
@@ -189,3 +192,17 @@ func (c *config) resolvePaths() error {
 	c.Paths = paths
 	return nil
 }
+
+// normalizeGlobPatterns calls `filepath.Abs` on all the globs from config
+func (c *config) normalizeGlobPatterns() error {
+	var paths []string
+	for _, path := range c.Paths {
+		pathAbs, err := filepath.Abs(path)
+		if err != nil {
+			return fmt.Errorf("Failed to get the absolute path for %s: %v", path, err)
+		}
+		paths = append(paths, pathAbs)
+	}
+	c.Paths = paths
+	return nil
+}

filebeat/prospector/log/prospector.go

@@ -85,9 +85,11 @@ func NewProspector(
 	if err := cfg.Unpack(&p.config); err != nil {
 		return nil, err
 	}
-	if err := p.config.resolvePaths(); err != nil {
-		logp.Err("Failed to resolve paths in config: %+v", err)
-		return nil, err
+	if err := p.config.resolveRecursiveGlobs(); err != nil {
+		return nil, fmt.Errorf("Failed to resolve recursive globs in config: %v", err)
+	}
+	if err := p.config.normalizeGlobPatterns(); err != nil {
+		return nil, fmt.Errorf("Failed to normalize globs patterns: %v", err)
 	}
 
 	// Create empty harvester to check if configs are fine
@@ -115,7 +117,7 @@ func NewProspector(
 // It goes through all states coming from the registry. Only the states which match the glob patterns of
 // the prospector will be loaded and updated. All other states will not be touched.
 func (p *Prospector) loadStates(states []file.State) error {
-	logp.Debug("prospector", "exclude_files: %s", p.config.ExcludeFiles)
+	logp.Debug("prospector", "exclude_files: %s. Number of stats: %d", p.config.ExcludeFiles, len(states))
 
 	for _, state := range states {
 		// Check if state source belongs to this prospector. If yes, update the state.

filebeat/tests/system/config/filebeat.yml.j2

@@ -10,7 +10,7 @@ filebeat.prospectors:
   # Paths that should be crawled and fetched
   {% if path %}paths:
     - {{ path }}{% endif %}
-  {% if recursive_glob %}recursive_glob.enabled: true
+  {% if disable_recursive_glob %}recursive_glob.enabled: false
   {% endif %}
   # Type of the files. Annotated in every documented
   scan_frequency: {{scan_frequency | default("0.1s") }}

filebeat/tests/system/test_prospector.py

@@ -683,8 +683,7 @@ def test_restart_recursive_glob(self):
         """
         self.render_config_template(
             path=os.path.abspath(self.working_dir) + "/log/**",
-            scan_frequency="1s",
-            recursive_glob=True,
+            scan_frequency="1s"
         )
 
         testfile_dir = os.path.join(self.working_dir, "log", "some", "other", "subdir")
@@ -715,3 +714,23 @@ def test_restart_recursive_glob(self):
             name="output contains 'entry2'")
 
         filebeat.check_kill_and_wait()
+
+    def test_disable_recursive_glob(self):
+        """
+        Check that the recursive glob can be disabled from the config.
+        """
+        self.render_config_template(
+            path=os.path.abspath(self.working_dir) + "/log/**",
+            scan_frequency="1s",
+            disable_recursive_glob=True,
+        )
+
+        testfile_dir = os.path.join(self.working_dir, "log", "some", "other", "subdir")
+        os.makedirs(testfile_dir)
+        testfile_path = os.path.join(testfile_dir, "input")
+        filebeat = self.start_beat()
+        self.wait_until(
+            lambda: self.log_contains(
+                "recursive glob disabled"),
+            max_timeout=10)
+        filebeat.check_kill_and_wait()