Add support for setting take_over (#341)

The new optional Boolean parameter `take_over` for `filebeat::input`
defaults to `undef` but can be set to `true` to enable the "take over"
functionality when switching from the deprecated `log` input type to the
new `filestream` type. This allows Filebeat to take over files from the
`log` input without completely re-ingesting the files, which leads to
duplicated log entries. This feature is available in Filebeat 8.x.

This change adds very basic spec tests for the `filestream` input type
and the `take_over` logic, and updates the docs where appropriate.

README.md

@@ -398,7 +398,8 @@ to fully understand what these parameters do.
  - `syslog_host`: [String] Host to listen for syslog messages (default: localhost:5140)
  - `exclude_files`: [Array] Files that match any regex in the list are excluded from filebeat (default: [])
  - `encoding`: [String] The file encoding. (default: plain)
- - `input_type`: [String] where filebeat reads the log from (default:log)
+ - `input_type`: [String] where filebeat reads the log from (default: filestream)
+ - `take_over` : [Boolean] Optionally enable [`take_over`](https://www.elastic.co/guide/en/beats/filebeat/8.11/filebeat-input-filestream.html#filebeat-input-filestream-take-over) when switchting from the deprecated input type `log` to the new input type `filestream`. This avoids re-ingesting already logfiles Filebeat already read when switching to `filestream`. This feature requires Filebeat 8.x.
  - `fields`: [Hash] Optional fields to add information to the output (default: {})
  - `fields_under_root`: [Boolean] Should the `fields` parameter fields be stored at the top level of indexed documents.
  - `ignore_older`: [String] Files older than this field will be ignored by filebeat (default: ignore nothing)

manifests/config.pp

@@ -68,7 +68,6 @@
  else {
  $filebeat_config = $filebeat_config_xpack
  }
-
  } else {
  $filebeat_config_temp = delete_undef_values({
  'shutdown_timeout' => $filebeat::shutdown_timeout,

manifests/input.pp

@@ -19,6 +19,7 @@
  Boolean $cri_parse_flags = false,
  String $encoding = 'plain',
  String $input_type = $filebeat::params::default_input_type,
+ Optional[Boolean] $take_over = undef,
  Hash $fields = {},
  Boolean $fields_under_root = $filebeat::fields_under_root,
  Hash $ssl = {},

spec/defines/input_spec.rb

@@ -78,6 +78,58 @@
  }
  end
  end
+
+ context "with filestream input support on #{os}" do
+ let(:facts) { os_facts }
+
+ # Filestream
+ let(:title) { 'some-filestream' }
+
+ context "with take_over unset" do
+ let(:params) do
+ {
+ 'input_type' => 'filestream',
+ 'paths' => ['/var/log/foo.log'],
+ }
+ end
+
+ if os_facts[:kernel] == 'Linux'
+ it { is_expected.to compile }
+
+ it {
+ is_expected.to contain_file('filebeat-some-filestream').with(
+ notify: 'Service[filebeat]',
+ )
+ is_expected.to contain_file('filebeat-some-filestream').with_content(
+ %r{- type: filestream\n\s{2}id: some-filestream\n\s{2}paths:\n\s{2}- /var/log/foo.log},
+ )
+ }
+ end
+ end
+
+ context "with take_over => true" do
+ let(:params) do
+ {
+ 'input_type' => 'filestream',
+ 'paths' => ['/var/log/foo.log'],
+ 'take_over' => true,
+ }
+ end
+
+ if os_facts[:kernel] == 'Linux'
+ it { is_expected.to compile }
+
+ it {
+ is_expected.to contain_file('filebeat-some-filestream').with(
+ notify: 'Service[filebeat]',
+ )
+ is_expected.to contain_file('filebeat-some-filestream').with_content(
+ %r{- type: filestream\n\s{2}id: some-filestream\n\s{2}take_over: true\n\s{2}paths:\n\s{2}- /var/log/foo.log},
+ )
+ }
+ end
+ end
+ end
  end
 
  on_supported_os.each do |os, os_facts|

templates/input.yml.erb

@@ -6,6 +6,9 @@
  <%- if @input_type =~ /(filestream|journald)/ -%>
  id: <%= @name %>
  <%- end -%>
+ <%- if @input_type == 'filestream' && @take_over != nil && @take_over == true -%>
+ take_over: true
+ <%- end -%>
  <%- if @input_type =~ /(tcp|udp)/ -%>
  host: <%= @host %>
  <%- if @ssl.length > 0 -%>
@@ -84,14 +87,14 @@
  <%- if @scan_frequency or @exclude_files -%>
  <%- if @input_type == 'filestream' -%>
  prospector:
-  scanner:
+ scanner:
  <%- if @scan_frequency -%>
-  check_interval: <%= @scan_frequency %>
+ check_interval: <%= @scan_frequency %>
  <%- end -%>
  <%- if @exclude_files.length > 0 -%>
-  exclude_files:
+ exclude_files:
  <%- @exclude_files.each do |exclude_file| -%>
-  - <%= exclude_file %>
+ - <%= exclude_file %>
  <%- end -%>
  <%- end -%>
  <%- else -%>