From 3573cf344c11349ec57dad5c96b9e1e801a03331 Mon Sep 17 00:00:00 2001
From: Andreas Ntaflos <daff@pseudoterminal.org>
Date: Thu, 4 Jan 2024 17:58:05 +0100
Subject: [PATCH] Add support for setting `take_over` (#341)

The new optional Boolean parameter `take_over` for `filebeat::input`
defaults to `undef` but can be set to `true` to enable the "take over"
functionality when switching from the deprecated `log` input type to the
new `filestream` type. This allows Filebeat to take over files from the
`log` input without completely re-ingesting the files, which leads to
duplicated log entries. This feature is available in Filebeat 8.x.

This change adds very basic spec tests for the `filestream` input type
and the `take_over` logic, and updates the docs where appropriate.
---
 README.md                  |  3 ++-
 manifests/config.pp        |  1 -
 manifests/input.pp         |  1 +
 spec/defines/input_spec.rb | 52 ++++++++++++++++++++++++++++++++++++++
 templates/input.yml.erb    | 11 +++++---
 5 files changed, 62 insertions(+), 6 deletions(-)

diff --git a/README.md b/README.md
index f08a85b..28d3c79 100644
--- a/README.md
+++ b/README.md
@@ -398,7 +398,8 @@ to fully understand what these parameters do.
   - `syslog_host`: [String] Host to listen for syslog messages (default: localhost:5140)
   - `exclude_files`: [Array] Files that match any regex in the list are excluded from filebeat (default: [])
   - `encoding`: [String] The file encoding. (default: plain)
-  - `input_type`: [String] where filebeat reads the log from (default:log)
+  - `input_type`: [String] where filebeat reads the log from (default: filestream)
+  - `take_over` : [Boolean] Optionally enable [`take_over`](https://www.elastic.co/guide/en/beats/filebeat/8.11/filebeat-input-filestream.html#filebeat-input-filestream-take-over) when switchting from the deprecated input type `log` to the new input type `filestream`. This avoids re-ingesting already logfiles Filebeat already read when switching to `filestream`. This feature requires Filebeat 8.x.
   - `fields`: [Hash] Optional fields to add information to the output (default: {})
   - `fields_under_root`: [Boolean] Should the `fields` parameter fields be stored at the top level of indexed documents.
   - `ignore_older`: [String] Files older than this field will be ignored by filebeat (default: ignore nothing)
diff --git a/manifests/config.pp b/manifests/config.pp
index f9d5391..c3561cd 100644
--- a/manifests/config.pp
+++ b/manifests/config.pp
@@ -68,7 +68,6 @@
     else {
       $filebeat_config = $filebeat_config_xpack
     }
-
   } else {
     $filebeat_config_temp = delete_undef_values({
         'shutdown_timeout'  => $filebeat::shutdown_timeout,
diff --git a/manifests/input.pp b/manifests/input.pp
index 0a4483f..f56a702 100644
--- a/manifests/input.pp
+++ b/manifests/input.pp
@@ -19,6 +19,7 @@
   Boolean $cri_parse_flags                 = false,
   String $encoding                         = 'plain',
   String $input_type                       = $filebeat::params::default_input_type,
+  Optional[Boolean] $take_over             = undef,
   Hash $fields                             = {},
   Boolean $fields_under_root               = $filebeat::fields_under_root,
   Hash $ssl                                = {},
diff --git a/spec/defines/input_spec.rb b/spec/defines/input_spec.rb
index 934fad0..27f8f28 100644
--- a/spec/defines/input_spec.rb
+++ b/spec/defines/input_spec.rb
@@ -78,6 +78,58 @@
         }
       end
     end
+
+    context "with filestream input support on #{os}" do
+      let(:facts) { os_facts }
+
+      # Filestream
+      let(:title) { 'some-filestream' }
+
+      context "with take_over unset" do
+        let(:params) do
+          {
+            'input_type' => 'filestream',
+            'paths' => ['/var/log/foo.log'],
+          }
+        end
+
+        if os_facts[:kernel] == 'Linux'
+          it { is_expected.to compile }
+
+          it {
+            is_expected.to contain_file('filebeat-some-filestream').with(
+              notify: 'Service[filebeat]',
+            )
+            is_expected.to contain_file('filebeat-some-filestream').with_content(
+              %r{- type: filestream\n\s{2}id: some-filestream\n\s{2}paths:\n\s{2}- /var/log/foo.log},
+            )
+          }
+        end
+      end
+
+      context "with take_over => true" do
+        let(:params) do
+          {
+            'input_type' => 'filestream',
+            'paths' => ['/var/log/foo.log'],
+            'take_over' => true,
+          }
+        end
+
+        if os_facts[:kernel] == 'Linux'
+          it { is_expected.to compile }
+
+          it {
+            is_expected.to contain_file('filebeat-some-filestream').with(
+              notify: 'Service[filebeat]',
+            )
+            is_expected.to contain_file('filebeat-some-filestream').with_content(
+              %r{- type: filestream\n\s{2}id: some-filestream\n\s{2}take_over: true\n\s{2}paths:\n\s{2}- /var/log/foo.log},
+            )
+          }
+        end
+      end
+    end
   end
 
   on_supported_os.each do |os, os_facts|
diff --git a/templates/input.yml.erb b/templates/input.yml.erb
index 599ac84..87126ae 100644
--- a/templates/input.yml.erb
+++ b/templates/input.yml.erb
@@ -6,6 +6,9 @@
   <%- if @input_type =~ /(filestream|journald)/ -%>
   id: <%= @name %>
   <%- end -%>
+  <%- if @input_type == 'filestream' && @take_over != nil && @take_over == true -%>
+  take_over: true
+  <%- end -%>
   <%- if @input_type =~ /(tcp|udp)/ -%>
   host: <%= @host %>
     <%- if @ssl.length > 0 -%>
@@ -84,14 +87,14 @@
   <%- if @scan_frequency or @exclude_files -%>
     <%- if @input_type == 'filestream' -%>
   prospector:
-     scanner:
+    scanner:
       <%- if @scan_frequency -%>
-        check_interval: <%= @scan_frequency %>
+      check_interval: <%= @scan_frequency %>
       <%- end -%>
       <%- if @exclude_files.length > 0 -%>
-        exclude_files:
+      exclude_files:
         <%- @exclude_files.each do |exclude_file| -%>
-        - <%= exclude_file %>
+      - <%= exclude_file %>
         <%- end -%>
       <%- end -%>
     <%- else -%>