Stop following symlinks. (#1767)

* Stop following symlinks.

Previously symlinks were followed. This had the consequence if a symlink and the file itself existed, the file was read twice. Now symlinks are not followed anymore.

This closes #1686

* Add symlink for windows

* Turn around params

* Remove symlink comment

* Fix for windows symlink

CHANGELOG.asciidoc

@@ -22,6 +22,8 @@ https://github.com/elastic/beats/compare/v5.0.0-alpha3...master[Check the HEAD d
 
 *Filebeat*
 
+- Stop following symlink. Symlinks are now ignored: {pull}1686[1686]
+
 *Winlogbeat*
 
 
@@ -79,7 +81,7 @@ https://github.com/elastic/beats/compare/v5.0.0alpha2...v5.0.0-alpha3[View commi
 *Affecting all Beats*
 
 - All configuration settings under `shipper:` are moved to be top level configuration settings. I.e.
-  `shipper.name:` becomes `name:` in the configuration file. #1570
+  `shipper.name:` becomes `name:` in the configuration file. {pull}1570[1570]
 
 *Topbeat*
 

filebeat/crawler/prospector_log.go

@@ -78,12 +78,16 @@ func (p *ProspectorLog) getFiles() map[string]os.FileInfo {
 				continue
 			}
 
-			// Stat the file, following any symlinks.
-			fileinfo, err := os.Stat(file)
+			fileinfo, err := os.Lstat(file)
 			if err != nil {
 				logp.Debug("prospector", "stat(%s) failed: %s", file, err)
 				continue
 			}
+			// Check if file is symlink
+			if fileinfo.Mode()&os.ModeSymlink != 0 {
+				logp.Debug("prospector", "File %s skipped as it is a symlink.", file)
+				continue
+			}
 
 			if fileinfo.IsDir() {
 				logp.Debug("prospector", "Skipping directory: %s", file)

filebeat/tests/system/test_prospector.py

@@ -514,3 +514,45 @@ def test_close_older_file_rotation_and_removal(self):
                 max_timeout=10)
 
         filebeat.check_kill_and_wait()
+
+    def test_skip_symlinks(self):
+        """
+        Test that symlinks are skipped
+        """
+        self.render_config_template(
+            path=os.path.abspath(self.working_dir) + "/log/*",
+        )
+
+        os.mkdir(self.working_dir + "/log/")
+        testfile = self.working_dir + "/log/test-2016.log"
+        symlink_file = self.working_dir + "/log/test.log"
+
+        # write first line
+        with open(testfile, 'a') as file:
+            file.write("Hello world\n")
+
+        if os.name == "nt":
+            import win32file
+            win32file.CreateSymbolicLink(symlink_file, testfile, 0)
+        else:
+            os.symlink(testfile, symlink_file)
+
+        filebeat = self.start_beat()
+
+        # wait for file to be skipped
+        self.wait_until(
+            lambda: self.log_contains("skipped as it is a symlink"),
+            max_timeout=10)
+
+        # wait for log to be read
+        self.wait_until(
+            lambda: self.output_has(lines=1),
+            max_timeout=15)
+
+        time.sleep(5)
+        filebeat.check_kill_and_wait()
+
+        data = self.read_output()
+
+        # Make sure there is only one entry, means it didn't follow the symlink
+        assert len(data) == 1