Implement filebeat modules

README.md

@@ -17,6 +17,7 @@
  - [Multiline Logs](#multiline-logs)
  - [JSON Logs](#json-logs)
  - [Inputs in Hiera](#inputs-in-hiera)
+ - [Usage of filebeat modules](#usage-of-filebeat-modules)
  - [Usage on Windows](#usage-on-windows)
  - [Processors](#processors)
  - [Processors in Hiera](#processors-in-hiera)
@@ -34,6 +35,7 @@
  - [Class: `filebeat::install::windows`](#class-filebeatinstallwindows)
  - [Public Defines](#public-defines)
  - [Define: `filebeat::input`](#define-filebeatinput)
+ - [Define: `filebeat::module`](#define-filebeatmodule)
  - [Limitations](#limitations)
  - [Generic template](#generic-template)
  - [Debian Systems](#debian-systems)
@@ -159,6 +161,35 @@ flag.
 `inputs` can be a Hash that will follow all the parameters listed on this documentation or an
 Array that will output as is to the input config file.
 
+### Usage of filebeat modules
+
+Filebeat ships with modules which contain pipelines and dashboards for common software. Filebeat needs to be setup to ship directly into elasticsearch that
+it's possible that filebeat will setup pipelines and dashboards automatically.
+
+If your setup includes logstash or some other service between filebeat and elasticsearch the following settings might not work as expected.
+
+The following should be a minimal example to get `filebeat::module::*` to create the required config and push pipeline and dashboards into your elasticsearch & kibana.
+
+```puppet
+class { 'filebeat::module::system':
+ syslog_enabled => true,
+ auth_enabled => true,
+}
+
+class { 'filebeat':
+ enable_conf_modules => true,
+ overwrite_pipelines => true,
+ setup => {
+ dashboards => {
+ enabled => true
+ },
+ kibana => {
+ host => 'http://kibana.example.com:5601',
+ }
+ }
+}
+```
+
 ### Usage on Windows
 
 When installing on Windows, this module will download the windows version of Filebeat from
@@ -278,6 +309,7 @@ filebeat::setup:
  - [Class: `filebeat::install::windows`](#class-filebeatinstallwindows)
  - [Public Defines](#public-defines)
  - [Define: `filebeat::input`](#define-filebeatinput)
+ - [Define: `filebeat::module`](#define-filebeatmodule)
  - [Limitations](#limitations)
  - [Generic template](#generic-template)
  - [Debian Systems](#debian-systems)
@@ -435,6 +467,14 @@ to fully understand what these parameters do.
  - `index`: [String] If present, this formatted string overrides the index for events from this input (for elasticsearch outputs), or sets the raw_index field of the event’s metadata (for other outputs) (default: undef)
  - `publisher_pipeline_disable_host`: [Boolean] This disables the "host.name" attribute being added to events. See [filebeat input configuration reference](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-log.html#_publisher_pipeline_disable_host_13) (default: false)
 
+#### Define: `filebeat::module`
+
+Base resource used to implement filebeat module support in this puppet module and can be useful if you have custom filebeat modules.
+
+**Parameters for `filebeat::module`**
+ - `ensure`: The ensure parameter on the module configuration file. (default: present)
+ - `config`: [Hash] Full hash representation of the module configuration
+
 ## Limitations
 This module doesn't load the [elasticsearch index template](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-getting-started.html#filebeat-template) into elasticsearch (required when shipping
 directly to elasticsearch).

manifests/config.pp

@@ -143,6 +143,19 @@
  force => true,
  notify => Service['filebeat'],
  }
+
+ file { 'filebeat-modules-dir':
+ ensure => $filebeat::directory_ensure,
+ path => $filebeat::modules_dir,
+ owner => $filebeat::config_dir_owner,
+ group => $filebeat::config_dir_group,
+ mode => $filebeat::config_dir_mode,
+ recurse => $filebeat::purge_conf_dir,
+ purge => $filebeat::purge_conf_dir,
+ force => true,
+ notify => Service['filebeat'],
+ require => File['filebeat-config-dir'],
+ }
  } # end Linux
 
  'SunOS' : {
@@ -173,6 +186,18 @@
  purge => $filebeat::purge_conf_dir,
  force => true,
  }
+
+ file { 'filebeat-modules-dir':
+ ensure => $filebeat::directory_ensure,
+ path => $filebeat::modules_dir,
+ owner => $filebeat::config_dir_owner,
+ group => $filebeat::config_dir_group,
+ mode => $filebeat::config_dir_mode,
+ recurse => $filebeat::purge_conf_dir,
+ purge => $filebeat::purge_conf_dir,
+ force => true,
+ require => File['filebeat-config-dir'],
+ }
  } # end SunOS
 
  'FreeBSD' : {
@@ -207,6 +232,19 @@
  force => true,
  notify => Service['filebeat'],
  }
+
+ file { 'filebeat-modules-dir':
+ ensure => $filebeat::directory_ensure,
+ path => $filebeat::modules_dir,
+ owner => $filebeat::config_dir_owner,
+ group => $filebeat::config_dir_group,
+ mode => $filebeat::config_dir_mode,
+ recurse => $filebeat::purge_conf_dir,
+ purge => $filebeat::purge_conf_dir,
+ force => true,
+ notify => Service['filebeat'],
+ require => File['filebeat-config-dir'],
+ }
  } # end FreeBSD
 
  'OpenBSD' : {
@@ -241,6 +279,19 @@
  force => true,
  notify => Service['filebeat'],
  }
+
+ file { 'filebeat-modules-dir':
+ ensure => $filebeat::directory_ensure,
+ path => $filebeat::modules_dir,
+ owner => $filebeat::config_dir_owner,
+ group => $filebeat::config_dir_group,
+ mode => $filebeat::config_dir_mode,
+ recurse => $filebeat::purge_conf_dir,
+ purge => $filebeat::purge_conf_dir,
+ force => true,
+ notify => Service['filebeat'],
+ require => File['filebeat-config-dir'],
+ }
  } # end OpenBSD
 
  'Windows' : {
@@ -271,6 +322,15 @@
  purge => $filebeat::purge_conf_dir,
  force => true,
  }
+
+ file { 'filebeat-modules-dir':
+ ensure => $filebeat::directory_ensure,
+ path => $filebeat::modules_dir,
+ recurse => $filebeat::purge_conf_dir,
+ purge => $filebeat::purge_conf_dir,
+ force => true,
+ require => File['filebeat-config-dir'],
+ }
  } # end Windows
 
  default : {

manifests/module.pp

@@ -0,0 +1,65 @@
+# filebeat::module
+#
+# @summary Base resource to manage Filebeat modules. Check filebeat::module::* for specific implementations.
+#
+# @example
+# filebeat::module { 'namevar': 
+# config => {
+# 'log' => {
+# 'enabled' => true,
+# 'var.paths' => [ '/var/log/*.log' ],
+# },
+# },
+# }
+#
+# @param ensure Present or absent. Default: present.
+# @param config Hash with the module configuration.
+#
+define filebeat::module (
+ Enum['absent', 'present'] $ensure = present,
+ Hash $config = {},
+) {
+ $filebeat_config = [{ 'module' => $name } + $config]
+
+ case $facts['kernel'] {
+ 'Linux', 'OpenBSD' : {
+ file { "filebeat-module-${name}":
+ ensure => $ensure,
+ path => "${filebeat::modules_dir}/${name}.yml",
+ owner => 'root',
+ group => '0',
+ mode => $filebeat::config_file_mode,
+ content => template("${module_name}/pure_hash.yml.erb"),
+ notify => Service['filebeat'],
+ before => File['filebeat.yml'],
+ }
+ }
+
+ 'FreeBSD' : {
+ file { "filebeat-module-${name}":
+ ensure => $ensure,
+ path => "${filebeat::modules_dir}/${name}.yml",
+ owner => 'root',
+ group => 'wheel',
+ mode => $filebeat::config_file_mode,
+ content => template("${module_name}/pure_hash.yml.erb"),
+ notify => Service['filebeat'],
+ before => File['filebeat.yml'],
+ }
+ }
+
+ 'Windows' : {
+ file { "filebeat-module-${name}":
+ ensure => $ensure,
+ path => "${filebeat::modules_dir}/${name}.yml",
+ content => template("${module_name}/pure_hash.yml.erb"),
+ notify => Service['filebeat'],
+ before => File['filebeat.yml'],
+ }
+ }
+
+ default : {
+ fail($filebeat::kernel_fail_message)
+ }
+ }
+}

manifests/module/apache.pp

@@ -0,0 +1,49 @@
+# filebeat::module::apache
+#
+# @summary 
+# This class manages the Filebeat module for Apache HTTP Server.
+#
+# @example
+# class { 'filebeat::module::apache':
+# access_enabled => true,
+# access_paths => [
+# '/var/log/apache2/access.log',
+# ],
+# error_enabled => true,
+# error_paths => [
+# '/var/log/apache2/error.log',
+# ],
+# }
+#
+# @param access_enabled
+# Whether to enable the Apache access log module. Defaults to `false`.
+# @param access_paths
+# An array of absolute paths to Apache access log files. Defaults to `undef`.
+# @param error_enabled
+# Whether to enable the Apache error log module. Defaults to `false`.
+# @param error_paths
+# An array of absolute paths to Apache error log files. Defaults to `undef`.
+#
+class filebeat::module::apache (
+ Boolean $access_enabled = false,
+ Optional[Array[Stdlib::Absolutepath]] $access_paths = undef,
+ Boolean $error_enabled = false,
+ Optional[Array[Stdlib::Absolutepath]] $error_paths = undef,
+) {
+ filebeat::module { 'apache':
+ config => {
+ 'access' => delete_undef_values(
+ {
+ 'enabled' => $access_enabled,
+ 'var.paths' => $access_paths,
+ }
+ ),
+ 'error' => delete_undef_values(
+ {
+ 'enabled' => $error_enabled,
+ 'var.paths' => $error_paths,
+ }
+ ),
+ },
+ }
+}

manifests/module/auditd.pp

@@ -0,0 +1,33 @@
+# filebeat::module::auditd
+#
+# @summary
+# This class manages the Filebeat module for auditd.
+#
+# @example
+# class { 'filebeat::module::auditd':
+# log_enabled => true,
+# log_paths => [
+# '/var/log/audit/audit.log',
+# ],
+# }
+#
+# @param log_enabled
+# Whether to enable the auditd module.
+# @param log_paths
+# An array of absolute paths to the auditd log files.
+#
+class filebeat::module::auditd (
+ Boolean $log_enabled = false,
+ Optional[Array[Stdlib::Absolutepath]] $log_paths = undef,
+) {
+ filebeat::module { 'auditd':
+ config => {
+ 'log' => delete_undef_values(
+ {
+ 'enabled' => $log_enabled,
+ 'var.paths' => $log_paths,
+ }
+ ),
+ },
+ }
+}