Filebeat syslog module (#3191)

* Basic parsing of syslog fields
* Supports multiline messages if the lines after the first one start
  with a space.
* Contains a simple Kibana dashboard

filebeat/_meta/kibana/dashboard/Filebeat-syslog-dashboard.json

@@ -0,0 +1,13 @@
+{
+  "hits": 0, 
+  "timeRestore": false, 
+  "description": "", 
+  "title": "Filebeat syslog dashboard", 
+  "uiStateJSON": "{}", 
+  "panelsJSON": "[{\"id\":\"Syslog-events-by-hostname\",\"type\":\"visualization\",\"panelIndex\":1,\"size_x\":8,\"size_y\":4,\"col\":1,\"row\":1},{\"id\":\"Syslog-hostnames-and-processes\",\"type\":\"visualization\",\"panelIndex\":2,\"size_x\":4,\"size_y\":4,\"col\":9,\"row\":1},{\"id\":\"Syslog-system-logs\",\"type\":\"search\",\"panelIndex\":3,\"size_x\":12,\"size_y\":7,\"col\":1,\"row\":5,\"columns\":[\"syslog.system.hostname\",\"syslog.system.program\",\"syslog.system.message\"],\"sort\":[\"@timestamp\",\"desc\"]}]", 
+  "optionsJSON": "{\"darkTheme\":false}", 
+  "version": 1, 
+  "kibanaSavedObjectMeta": {
+    "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}}}]}"
+  }
+}

filebeat/_meta/kibana/search/Syslog-system-logs.json

@@ -0,0 +1,18 @@
+{
+  "sort": [
+    "@timestamp", 
+    "desc"
+  ], 
+  "hits": 0, 
+  "description": "", 
+  "title": "Syslog system logs", 
+  "version": 1, 
+  "kibanaSavedObjectMeta": {
+    "searchSourceJSON": "{\"index\":\"filebeat-*\",\"filter\":[],\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"require_field_match\":false,\"fragment_size\":2147483647},\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}}}"
+  }, 
+  "columns": [
+    "syslog.system.hostname", 
+    "syslog.system.program", 
+    "syslog.system.message"
+  ]
+}

filebeat/_meta/kibana/visualization/Syslog-events-by-hostname.json

@@ -0,0 +1,11 @@
+{
+  "visState": "{\"title\":\"Syslog events by hostname\",\"type\":\"histogram\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"scale\":\"linear\",\"mode\":\"stacked\",\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"syslog.system.hostname\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}", 
+  "description": "", 
+  "title": "Syslog events by hostname", 
+  "uiStateJSON": "{}", 
+  "version": 1, 
+  "savedSearchId": "Syslog-system-logs", 
+  "kibanaSavedObjectMeta": {
+    "searchSourceJSON": "{\"filter\":[]}"
+  }
+}

filebeat/_meta/kibana/visualization/Syslog-hostnames-and-processes.json

@@ -0,0 +1,11 @@
+{
+  "visState": "{\"title\":\"Syslog hostnames and processes\",\"type\":\"pie\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"bottom\",\"isDonut\":true},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"syslog.system.hostname\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"syslog.system.program\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}", 
+  "description": "", 
+  "title": "Syslog hostnames and processes", 
+  "uiStateJSON": "{}", 
+  "version": 1, 
+  "savedSearchId": "Syslog-system-logs", 
+  "kibanaSavedObjectMeta": {
+    "searchSourceJSON": "{\"filter\":[]}"
+  }
+}

filebeat/docs/fields.asciidoc

@@ -17,6 +17,7 @@ grouped in the following categories:
 * <<exported-fields-log>>
 * <<exported-fields-mysql>>
 * <<exported-fields-nginx>>
+* <<exported-fields-syslog>>
 
 --
 [[exported-fields-beat]]
@@ -567,3 +568,54 @@ type: text
 The error message
 
 
+[[exported-fields-syslog]]
+== Syslog Fields
+
+Module for parsing syslog files.
+
+
+
+[float]
+== syslog Fields
+
+Fields from the syslog files.
+
+
+
+[float]
+== system Fields
+
+Contains fields from the syslog system logs.
+
+
+
+[float]
+=== syslog.system.timestamp
+
+The timestamp as read from the syslog message.
+
+
+[float]
+=== syslog.system.hostname
+
+The hostname as read from the syslog message.
+
+
+[float]
+=== syslog.system.program
+
+The process name as read from the syslog message.
+
+
+[float]
+=== syslog.system.pid
+
+The PID of the process that sent the syslog message.
+
+
+[float]
+=== syslog.system.message
+
+The message in the log line.
+
+

filebeat/filebeat.template-es2x.json

@@ -321,6 +321,39 @@
           "index": "not_analyzed",
           "type": "string"
         },
+        "syslog": {
+          "properties": {
+            "system": {
+              "properties": {
+                "hostname": {
+                  "ignore_above": 1024,
+                  "index": "not_analyzed",
+                  "type": "string"
+                },
+                "message": {
+                  "ignore_above": 1024,
+                  "index": "not_analyzed",
+                  "type": "string"
+                },
+                "pid": {
+                  "ignore_above": 1024,
+                  "index": "not_analyzed",
+                  "type": "string"
+                },
+                "program": {
+                  "ignore_above": 1024,
+                  "index": "not_analyzed",
+                  "type": "string"
+                },
+                "timestamp": {
+                  "ignore_above": 1024,
+                  "index": "not_analyzed",
+                  "type": "string"
+                }
+              }
+            }
+          }
+        },
         "tags": {
           "ignore_above": 1024,
           "index": "not_analyzed",

filebeat/filebeat.template.json

@@ -275,6 +275,34 @@
           "ignore_above": 1024,
           "type": "keyword"
         },
+        "syslog": {
+          "properties": {
+            "system": {
+              "properties": {
+                "hostname": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "message": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "pid": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "program": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "timestamp": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                }
+              }
+            }
+          }
+        },
         "tags": {
           "ignore_above": 1024,
           "type": "keyword"

filebeat/module/syslog/_meta/fields.yml

@@ -0,0 +1,10 @@
+- key: syslog
+  title: "Syslog"
+  description: >
+    Module for parsing syslog files.
+  fields:
+    - name: syslog
+      type: group
+      description: >
+        Fields from the syslog files.
+      fields:

filebeat/module/syslog/system/_meta/fields.yml

@@ -0,0 +1,20 @@
+- name: system
+  type: group
+  description: >
+    Contains fields from the syslog system logs.
+  fields:
+    - name: timestamp
+      description: >
+        The timestamp as read from the syslog message.
+    - name: hostname
+      description: >
+        The hostname as read from the syslog message.
+    - name: program
+      description: >
+        The process name as read from the syslog message.
+    - name: pid
+      description: >
+        The PID of the process that sent the syslog message.
+    - name: message
+      description: >
+        The message in the log line.

filebeat/module/syslog/system/config/system.yml

@@ -0,0 +1,9 @@
+- input_type: log
+  paths:
+    - {{path}}
+  multiline:
+    pattern: "^\\s"
+    match: after
+  fields:
+    source_type: syslog-system
+    pipeline_id: {{beat.pipeline_id}}

filebeat/module/syslog/system/ingest/pipeline.json

@@ -0,0 +1,39 @@
+{
+	"description": "Pipeline for parsing Syslog messages.",
+	"processors": [
+		{
+			"grok": {
+				"field": "message",
+				"patterns": [
+					"%{SYSLOGTIMESTAMP:syslog.system.timestamp} %{SYSLOGHOST:syslog.system.hostname} %{DATA:syslog.system.program}(?:\\[%{POSINT:syslog.system.pid}\\])?: %{GREEDYMULTILINE:syslog.system.message}"
+				],
+        "pattern_definitions" : {
+          "GREEDYMULTILINE" : "(.|\n)*"
+        },
+        "ignore_missing": true
+			}
+    },
+		{
+      "remove": {
+        "field": "message"
+      }
+    },
+		{
+      "date": {
+        "field": "syslog.system.timestamp",
+        "target_field": "@timestamp",
+        "formats": [
+					"MMM  d HH:mm:ss",
+					"MMM dd HH:mm:ss"
+        ],
+        "ignore_failure": true
+      }
+		}
+	],
+  "on_failure" : [{
+    "set" : {
+      "field" : "error",
+      "value" : "{{ _ingest.on_failure_message }}"
+    }
+  }]
+}

filebeat/module/syslog/system/manifest.yml

@@ -0,0 +1,10 @@
+module_version: 1.0
+
+vars:
+  path:
+    default: /var/log/messages
+    os.darwin: /var/log/system.log
+
+ingest_pipeline: ingest/pipeline.json
+prospectors:
+  - config/system.yml