[Filebeat] Add RabbitMQ module (#12032)

* Parses single and multiline messages from [RabbitMQ's unified log file](https://www.rabbitmq.com/logging.html#log-file-location)
* `var.paths` uses the `RABBITMQ_LOGS` env variable if present
* `var.convert_timezone` (enabled by default since the [timestamps are localtime by default](https://github.com/erlang-lager/lager#universal-time)) 
* No dashboard
* Tested with
  ```
  ~/go/src/github.com/elastic/beats/x-pack/filebeat [master @ 092a3f5] ✓ ‣ MODULES_PATH=./module GENERATE=1 INTEGRATION_TESTS=1 TESTING_FILEBEAT_MODULES=rabbitmq nosetests tests/system/test_xpack_modules.py -v
  test_fileset_file_0_rabbitmq (test_xpack_modules.XPackTest) ... ok

  ----------------------------------------------------------------------
  Ran 1 test in 3.320s

  OK
  ```

CHANGELOG.next.asciidoc

@@ -20,6 +20,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Socket dataset: Exclude localhost by default {pull}11993[11993]
 
 *Filebeat*
+
 - Modify apache/error dataset to follow ECS. {pull}8963[8963]
 - Rename many `traefik.access.*` fields to map to ECS. {pull}9005[9005]
 - Fix parsing of GC entries in elasticsearch server log. {issue}9513[9513] {pull}9810[9810]
@@ -176,6 +177,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add apache2(httpd) log path (`/var/log/httpd`) to make apache2 module work out of the box on Redhat-family OSes. {issue}11887[11887] {pull}11888[11888]
 - Add support to new MongoDB additional diagnostic information {pull}11952[11952]
 - New module `palo_alto` for Palo Alto Networks PAN-OS logs. {pull}11999[11999]
+- Add RabbitMQ module. {pull}12032[12032]
 
 *Heartbeat*
 

filebeat/docs/fields.asciidoc

@@ -43,6 +43,7 @@ grouped in the following categories:
 * <<exported-fields-palo_alto>>
 * <<exported-fields-postgresql>>
 * <<exported-fields-process>>
+* <<exported-fields-rabbitmq>>
 * <<exported-fields-redis>>
 * <<exported-fields-santa>>
 * <<exported-fields-suricata>>
@@ -12195,6 +12196,37 @@ alias to: process.executable
 
 --
 
+[[exported-fields-rabbitmq]]
+== RabbitMQ fields
+
+RabbitMQ Module
+
+
+
+[float]
+== rabbitmq fields
+
+
+
+
+[float]
+== log fields
+
+RabbitMQ log files
+
+
+
+*`rabbitmq.log.pid`*::
++
+--
+type: keyword
+
+example: <0.222.0>
+
+The Erlang process id
+
+--
+
 [[exported-fields-redis]]
 == Redis fields
 

filebeat/docs/include/var-convert-timezone.asciidoc

@@ -5,4 +5,6 @@ parsing time to convert the timestamp to UTC. The local timezone is also added
 in each event in a dedicated field (`beat.timezone`). The conversion is only
 possible in Elasticsearch >= 6.1. If the Elasticsearch version is less than 6.1,
 the `beat.timezone` field is added, but the conversion to UTC is not made.  The
-default is `false`.
+default is
+ifdef::default_convert_timezone[`true`]
+ifndef::default_convert_timezone[`false`]

filebeat/docs/modules/rabbitmq.asciidoc

@@ -0,0 +1,72 @@
+////
+This file is generated! See scripts/docs_collector.py
+////
+
+[[filebeat-module-rabbitmq]]
+:modulename: rabbitmq
+:has-dashboards: false
+
+== RabbitMQ module
+
+This is the module for parsing https://www.rabbitmq.com/logging.html[RabbitMQ log files]
+
+include::../include/what-happens.asciidoc[]
+
+[float]
+=== Compatibility
+
+Parses https://www.rabbitmq.com/logging.html[single file format] introduced in 3.7.0.
+
+Tested with version 3.7.14.
+
+include::../include/running-modules.asciidoc[]
+
+include::../include/configuring-intro.asciidoc[]
+
+The following example shows how to set paths in the +modules.d/{modulename}.yml+
+file to override the default paths for RabbitMQ logs:
+
+
+["source","yaml",subs="attributes"]
+-----
+- module: rabbitmq
+  log:
+    enabled: true
+    var.paths: ["/path/to/log/rabbitmq/*.log*"]
+-----
+
+
+To specify the same settings at the command line, you use:
+
+["source","sh",subs="attributes"]
+-----
+-M "rabbitmq.log.var.paths=[/path/to/log/rabbitmq/*.log*]"
+-----
+
+:fileset_ex: log
+
+include::../include/config-option-intro.asciidoc[]
+
+[float]
+==== `log` fileset settings
+
+include::../include/var-paths.asciidoc[]
+
+:default_convert_timezone: true
+
+include::../include/var-convert-timezone.asciidoc[]
+
+:has-dashboards!:
+
+:fileset_ex!:
+:default_convert_timezone!:
+
+:modulename!:
+
+
+[float]
+=== Fields
+
+For a description of each field in the module, see the
+<<exported-fields-rabbitmq,exported fields>> section.
+

filebeat/docs/modules_list.asciidoc

@@ -24,6 +24,7 @@ This file is generated! See scripts/docs_collector.py
   * <<filebeat-module-osquery>>
   * <<filebeat-module-palo_alto>>
   * <<filebeat-module-postgresql>>
+  * <<filebeat-module-rabbitmq>>
   * <<filebeat-module-redis>>
   * <<filebeat-module-santa>>
   * <<filebeat-module-suricata>>
@@ -56,6 +57,7 @@ include::modules/nginx.asciidoc[]
 include::modules/osquery.asciidoc[]
 include::modules/palo_alto.asciidoc[]
 include::modules/postgresql.asciidoc[]
+include::modules/rabbitmq.asciidoc[]
 include::modules/redis.asciidoc[]
 include::modules/santa.asciidoc[]
 include::modules/suricata.asciidoc[]

x-pack/filebeat/filebeat.reference.yml

@@ -429,6 +429,19 @@ filebeat.modules:
     # can be added under this section.
     #input:
 
+#------------------------------- RabbitMQ Module -------------------------------
+#- module: rabbitmq
+  # All logs
+  #log:
+    #enabled: true
+
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths: ["/var/log/rabbitmq/rabbit@localhost.log*"]
+
+    # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1.
+    #var.convert_timezone: false
+
 #-------------------------------- Redis Module --------------------------------
 #- module: redis
   # Main logs

x-pack/filebeat/module/rabbitmq/_meta/config.yml

@@ -0,0 +1,11 @@
+#- module: rabbitmq
+  # All logs
+  #log:
+    #enabled: true
+
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths: ["/var/log/rabbitmq/rabbit@localhost.log*"]
+
+    # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1.
+    #var.convert_timezone: false

x-pack/filebeat/module/rabbitmq/_meta/docs.asciidoc

@@ -0,0 +1,59 @@
+:modulename: rabbitmq
+:has-dashboards: false
+
+== RabbitMQ module
+
+This is the module for parsing https://www.rabbitmq.com/logging.html[RabbitMQ log files]
+
+include::../include/what-happens.asciidoc[]
+
+[float]
+=== Compatibility
+
+Parses https://www.rabbitmq.com/logging.html[single file format] introduced in 3.7.0.
+
+Tested with version 3.7.14.
+
+include::../include/running-modules.asciidoc[]
+
+include::../include/configuring-intro.asciidoc[]
+
+The following example shows how to set paths in the +modules.d/{modulename}.yml+
+file to override the default paths for RabbitMQ logs:
+
+
+["source","yaml",subs="attributes"]
+-----
+- module: rabbitmq
+  log:
+    enabled: true
+    var.paths: ["/path/to/log/rabbitmq/*.log*"]
+-----
+
+
+To specify the same settings at the command line, you use:
+
+["source","sh",subs="attributes"]
+-----
+-M "rabbitmq.log.var.paths=[/path/to/log/rabbitmq/*.log*]"
+-----
+
+:fileset_ex: log
+
+include::../include/config-option-intro.asciidoc[]
+
+[float]
+==== `log` fileset settings
+
+include::../include/var-paths.asciidoc[]
+
+:default_convert_timezone: true
+
+include::../include/var-convert-timezone.asciidoc[]
+
+:has-dashboards!:
+
+:fileset_ex!:
+:default_convert_timezone!:
+
+:modulename!:

x-pack/filebeat/module/rabbitmq/_meta/fields.yml

@@ -0,0 +1,9 @@
+- key: rabbitmq
+  title: "RabbitMQ"
+  description: >
+    RabbitMQ Module
+  fields:
+    - name: rabbitmq
+      type: group
+      description: >
+      fields:

x-pack/filebeat/module/rabbitmq/log/_meta/fields.yml

@@ -0,0 +1,9 @@
+- name: log
+  type: group
+  description: >
+    RabbitMQ log files
+  fields:
+    - name: pid
+      type: keyword
+      description: The Erlang process id
+      example: <0.222.0>

x-pack/filebeat/module/rabbitmq/log/config/log.yml

@@ -0,0 +1,19 @@
+type: log
+paths:
+{{ range $i, $path := .paths }}
+ - {{$path}}
+{{ end }}
+exclude_files: [".gz$"]
+
+# If the line doesn't start with a timestamp, consider it a continuation of the previous line
+# From https://www.elastic.co/guide/en/beats/filebeat/current/_examples_of_multiline_configuration.html#_timestamps
+# ideally, this would be the same pattern (`DATESTAMP`) used in `processors.grok.patterns`
+multiline:
+  pattern: '[0-9]{4}-[0-9]{2}-[0-9]{2}'
+  negate: true
+  match: after
+
+processors:
+{{ if .convert_timezone }}
+  - add_locale: ~
+{{ end }}

x-pack/filebeat/module/rabbitmq/log/ingest/pipeline.yml

@@ -0,0 +1,28 @@
+---
+description: Pipeline for parsing RabbitMQ logs
+processors:
+- grok:
+    field: message
+    pattern_definitions:
+      GREEDYMULTILINE: "(.|\n)*"
+      ERL_PID: "\\<%{INT}+\\.%{INT}+\\.%{INT}+\\>"
+    patterns:
+    - "%{DATESTAMP:timestamp} \\[%{WORD:log.level}\\] %{ERL_PID:rabbitmq.log.pid}
+      %{GREEDYMULTILINE:message}"
+    ignore_missing: true
+- date:
+    field: timestamp
+    target_field: "@timestamp"
+    formats:
+    - yy-MM-dd HH:mm:ss.SSS
+    ignore_failure: true
+    #{< if .convert_timezone >}
+    timezone: "{{ event.timezone }}"
+    #{< end >}
+- remove:
+    field:
+    - timestamp
+on_failure:
+- set:
+    field: error.message
+    value: "{{ _ingest.on_failure_message }}"

x-pack/filebeat/module/rabbitmq/log/manifest.yml

@@ -0,0 +1,19 @@
+module_version: 1.0
+
+var:
+  - name: paths
+    default:
+      - ${RABBITMQ_LOGS:/var/log/rabbitmq/rabbit@localhost.log*}
+    os.darwin:
+      - ${RABBITMQ_LOGS:/usr/local/var/log/rabbitmq/rabbit@localhost.log*}
+    os.windows:
+      #- '%APPDATA%\RabbitMQ\log\rabbit@localhost.log*'
+  - name: convert_timezone
+    default: true
+    # if ES < 6.1.0, this flag switches to false automatically when evaluating the
+    # pipeline
+    min_elasticsearch_version:
+      version: 6.1.0
+      value: false
+ingest_pipeline: ingest/pipeline.yml
+input: config/log.yml