Add fileset for parsing linux auditd logs (#3750)

The PR adds and audit fileset to the system module for parsing logs from the auditd daemon. There is a sample dashboard for viewing this data.

Features

- Parses audit event type, unix epoch time, audit event counter, and the arbitrary key/value pairs that follow.
- Applies geoip lookup to the `system.audit.addr` field which is present for some remote login events.
- Decodes hex encoded ascii values that are sometimes used for the `system.audit.exe` and `system.audit.cmd` fields.
- Remove key/value pairs where the value is `?`.

Missing Features

- Decoder for `system.audit.saddr` field present in SOCKADDR audit events. A recipe is described [here](https://unix.stackexchange.com/questions/102926/how-to-interpret-the-saddr-field-of-an-audit-log) and could probably be ported to painless. Sample value:
  `type=SOCKADDR msg=audit(1481078693.491:873): saddr=01002F7661722F72756E2F6E7363642F736F636B657400008983B330BE7F0000000000000000000070830130BE7F000004000000000000001B00000000000000D128B7ED000000008B8BB330BE7F00003B00000000000000403E4BF7FD7F0000447F0130BE7F000080150230BE7F`

* Apply fileset overrides before checking enabled

Disable other filesets in a module while testing.

* Addressing review comments

- Renamed system.audit.type to system.audit.record_type
- Renamed system.audit.counter to system.audit.sequence
- Dropped system.audit.epoch after parsing it into @timestamp
- Dropped message field.
- Removed “and others” from the supported OSes listed in docs.
- Updated dashboards after renaming and deleting fields.
- Updated script to drop fields where value was `(null)`.

* Document old_auid, new_auid, old_ses, new_ses

filebeat/docs/fields.asciidoc

@@ -865,6 +865,130 @@ Fields from the system log files.
 
 
 
+[float]
+== audit Fields
+
+Fields from the Linux audit logs. Not all fields are documented here because they are dynamic and vary by audit event type.
+
+
+
+[float]
+=== system.audit.record_type
+
+The audit event type.
+
+
+[float]
+=== system.audit.old_auid
+
+For login events this is the old audit ID used for the user prior to this login.
+
+
+[float]
+=== system.audit.new_auid
+
+For login events this is the new audit ID. The audit ID can be used to trace future events to the user even if their identity changes (like becoming root).
+
+
+[float]
+=== system.audit.old_ses
+
+For login events this is the old session ID used for the user prior to this login.
+
+
+[float]
+=== system.audit.new_ses
+
+For login events this is the new session ID. It can be used to tie a user to future events by session ID.
+
+
+[float]
+=== system.audit.sequence
+
+type: long
+
+The audit event sequence number.
+
+
+[float]
+=== system.audit.pid
+
+type: long
+
+The ID of the process.
+
+
+[float]
+=== system.audit.ppid
+
+type: long
+
+The ID of the process.
+
+
+[float]
+=== system.audit.items
+
+type: long
+
+The number of items in an event.
+
+
+[float]
+=== system.audit.item
+
+type: long
+
+The item field indicates which item out of the total number of items. This number is zero-based; a value of 0 means it is the first item.
+
+
+[float]
+== geoip Fields
+
+Contains GeoIP information gathered based on the `system.audit.addr` field. Only present if the GeoIP Elasticsearch plugin is available and used.
+
+
+
+[float]
+=== system.audit.geoip.continent_name
+
+type: keyword
+
+The name of the continent.
+
+
+[float]
+=== system.audit.geoip.city_name
+
+type: keyword
+
+The name of the city.
+
+
+[float]
+=== system.audit.geoip.region_name
+
+type: keyword
+
+The name of the region.
+
+
+[float]
+=== system.audit.geoip.country_iso_code
+
+type: keyword
+
+Country ISO code.
+
+
+[float]
+=== system.audit.geoip.location
+
+type: geo_point
+
+The longitude and latitude.
+
+
 [float]
 == auth Fields
 

filebeat/docs/modules/system.asciidoc

@@ -5,21 +5,29 @@ This file is generated! See scripts/docs_collector.py
 [[filebeat-module-system]]
 == System module
 
-This module collects and parses logs created by system logging server of common Unix/Linux based
-distributions.
+This module collects and parses logs created by system logging server of common
+Unix/Linux based distributions.
 
+[float]
 === Compatibility
 
-This module was tested with logs from OSes like Ubuntu 12.04, Centos 7, macOS Sierra, and others.
+This module was tested with logs from OSes like Ubuntu 12.04, Centos 7, and
+macOS Sierra.
 
 This module is not available for Windows.
 
+[float]
 === Dashboard
 
-This module comes with a sample dashboard.
+This module comes with sample dashboards. The first is a more generic syslog
+dashboard that shows syslog data.
 
 image::./images/kibana-system.png[]
 
+And second is the audit dashboard that shows audit log data.
+
+image::./images/kibana-system-audit.png[]
+
 [float]
 === Syslog fileset settings
 

filebeat/fileset/modules.go

@@ -44,15 +44,15 @@ func newModuleRegistry(modulesPath string,
 				fcfg = &defaultFilesetConfig
 			}
 
-			if fcfg.Enabled != nil && (*fcfg.Enabled) == false {
-				continue
-			}
-
 			fcfg, err = applyOverrides(fcfg, mcfg.Module, filesetName, overrides)
 			if err != nil {
 				return nil, fmt.Errorf("Error applying overrides on fileset %s/%s: %v", mcfg.Module, filesetName, err)
 			}
 
+			if fcfg.Enabled != nil && (*fcfg.Enabled) == false {
+				continue
+			}
+
 			fileset, err := New(modulesPath, filesetName, &mcfg, fcfg)
 			if err != nil {
 				return nil, err

filebeat/fileset/modules_test.go

@@ -38,7 +38,7 @@ func TestNewModuleRegistry(t *testing.T) {
 	expectedModules := map[string][]string{
 		"nginx":  {"access", "error"},
 		"mysql":  {"slowlog", "error"},
-		"system": {"syslog", "auth"},
+		"system": {"syslog", "audit", "auth"},
 	}
 
 	assert.Equal(t, len(expectedModules), len(reg.registry))
@@ -98,7 +98,7 @@ func TestNewModuleRegistryConfig(t *testing.T) {
 	assert.NotContains(t, reg.registry["nginx"], "error")
 }
 
-func TestAppplyOverrides(t *testing.T) {
+func TestApplyOverrides(t *testing.T) {
 
 	falseVar := false
 	trueVar := true

filebeat/module/system/_meta/docs.asciidoc

@@ -1,20 +1,28 @@
 == System module
 
-This module collects and parses logs created by system logging server of common Unix/Linux based
-distributions.
+This module collects and parses logs created by system logging server of common
+Unix/Linux based distributions.
 
+[float]
 === Compatibility
 
-This module was tested with logs from OSes like Ubuntu 12.04, Centos 7, macOS Sierra, and others.
+This module was tested with logs from OSes like Ubuntu 12.04, Centos 7, and
+macOS Sierra.
 
 This module is not available for Windows.
 
+[float]
 === Dashboard
 
-This module comes with a sample dashboard.
+This module comes with sample dashboards. The first is a more generic syslog
+dashboard that shows syslog data.
 
 image::./images/kibana-system.png[]
 
+And second is the audit dashboard that shows audit log data.
+
+image::./images/kibana-system-audit.png[]
+
 [float]
 === Syslog fileset settings
 

filebeat/module/system/audit/_meta/fields.yml

@@ -0,0 +1,74 @@
+- name: audit
+  type: group
+  description: >
+    Fields from the Linux audit logs. Not all fields are documented here because
+    they are dynamic and vary by audit event type.
+  fields:
+    - name: record_type
+      description: >
+        The audit event type.
+    - name: old_auid
+      description: >
+        For login events this is the old audit ID used for the user prior to
+        this login.
+    - name: new_auid
+      description: >
+        For login events this is the new audit ID. The audit ID can be used to
+        trace future events to the user even if their identity changes (like
+        becoming root).
+    - name: old_ses
+      description: >
+        For login events this is the old session ID used for the user prior to
+        this login.
+    - name: new_ses
+      description: >
+        For login events this is the new session ID. It can be used to tie a
+        user to future events by session ID.
+    - name: sequence
+      type: long
+      description: >
+        The audit event sequence number.
+    - name: pid
+      type: long
+      description: >
+        The ID of the process.
+    - name: ppid
+      type: long
+      description: >
+        The ID of the process.
+    - name: items
+      type: long
+      description: >
+        The number of items in an event.
+    - name: item
+      type: long
+      description: >
+        The item field indicates which item out of the total number of items.
+        This number is zero-based; a value of 0 means it is the first item.
+    - name: geoip
+      type: group
+      description: >
+        Contains GeoIP information gathered based on the `system.audit.addr`
+        field. Only present if the GeoIP Elasticsearch plugin is available and
+        used.
+      fields:
+        - name: continent_name
+          type: keyword
+          description: >
+            The name of the continent.
+        - name: city_name
+          type: keyword
+          description: >
+            The name of the city.
+        - name: region_name
+          type: keyword
+          description: >
+            The name of the region.
+        - name: country_iso_code
+          type: keyword
+          description: >
+            Country ISO code.
+        - name: location
+          type: geo_point
+          description: >
+            The longitude and latitude.

filebeat/module/system/audit/_meta/kibana/dashboard/dfbb49f0-0a0f-11e7-8a62-2d05eaaac5cb.json

@@ -0,0 +1,13 @@
+{
+  "hits": 0, 
+  "timeRestore": false, 
+  "description": "", 
+  "title": "Filebeat System Audit", 
+  "uiStateJSON": "{\"P-2\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}}", 
+  "panelsJSON": "[{\"col\":1,\"id\":\"6295bdd0-0a0e-11e7-825f-6748cda7d858\",\"panelIndex\":1,\"row\":1,\"size_x\":4,\"size_y\":4,\"type\":\"visualization\"},{\"col\":9,\"id\":\"5ebdbe50-0a0f-11e7-825f-6748cda7d858\",\"panelIndex\":2,\"row\":1,\"size_x\":4,\"size_y\":4,\"type\":\"visualization\"},{\"col\":1,\"id\":\"2bb0fa70-0a11-11e7-9e84-43da493ad0c7\",\"panelIndex\":3,\"row\":5,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":7,\"id\":\"d1726930-0a7f-11e7-8b04-eb22a5669f27\",\"panelIndex\":5,\"row\":5,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":5,\"id\":\"c5411910-0a87-11e7-8b04-eb22a5669f27\",\"panelIndex\":6,\"row\":1,\"size_x\":4,\"size_y\":4,\"type\":\"visualization\"},{\"size_x\":12,\"size_y\":3,\"panelIndex\":7,\"type\":\"search\",\"id\":\"4ac0a370-0a11-11e7-8b04-eb22a5669f27\",\"col\":1,\"row\":8,\"columns\":[\"system.audit.record_type\",\"system.audit.sequence\",\"system.audit.acct\"],\"sort\":[\"@timestamp\",\"desc\"]}]", 
+  "optionsJSON": "{\"darkTheme\":false}", 
+  "version": 1, 
+  "kibanaSavedObjectMeta": {
+    "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}}}],\"highlightAll\":true,\"version\":true}"
+  }
+}

filebeat/module/system/audit/_meta/kibana/search/4ac0a370-0a11-11e7-8b04-eb22a5669f27.json

@@ -0,0 +1,18 @@
+{
+  "sort": [
+    "@timestamp", 
+    "desc"
+  ], 
+  "hits": 0, 
+  "description": "", 
+  "title": "Audit Events", 
+  "version": 1, 
+  "kibanaSavedObjectMeta": {
+    "searchSourceJSON": "{\"index\":\"filebeat-*\",\"highlightAll\":true,\"version\":true,\"query\":{\"query_string\":{\"query\":\"_exists_:system.audit\",\"analyze_wildcard\":true}},\"filter\":[]}"
+  }, 
+  "columns": [
+    "system.audit.record_type", 
+    "system.audit.sequence", 
+    "system.audit.acct"
+  ]
+}

filebeat/module/system/audit/_meta/kibana/visualization/2bb0fa70-0a11-11e7-9e84-43da493ad0c7.json

@@ -0,0 +1,10 @@
+{
+  "visState": "{\"type\":\"timelion\",\"title\":\"Audit Event Results\",\"params\":{\"expression\":\".es(q=\\\"_exists_:system.audit NOT system.audit.res:failure\\\").label(\\\"Success\\\") .es(q=\\\"system.audit.res:failed\\\").label(\\\"Failure\\\").title(\\\"Audit Event Results\\\")\",\"interval\":\"auto\"}}", 
+  "description": "", 
+  "title": "Audit Event Results", 
+  "uiStateJSON": "{}", 
+  "version": 1, 
+  "kibanaSavedObjectMeta": {
+    "searchSourceJSON": "{}"
+  }
+}

filebeat/module/system/audit/_meta/kibana/visualization/5ebdbe50-0a0f-11e7-825f-6748cda7d858.json

@@ -0,0 +1,10 @@
+{
+  "visState": "{\"title\":\"Audit Top Exec Commands\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"system.audit.a0\",\"size\":30,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Command (arg 0)\"}}],\"listeners\":{}}", 
+  "description": "", 
+  "title": "Audit Top Exec Commands", 
+  "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", 
+  "version": 1, 
+  "kibanaSavedObjectMeta": {
+    "searchSourceJSON": "{\"index\":\"filebeat-*\",\"query\":{\"query_string\":{\"query\":\"system.audit.record_type:EXECVE\",\"analyze_wildcard\":true}},\"filter\":[]}"
+  }
+}

filebeat/module/system/audit/_meta/kibana/visualization/6295bdd0-0a0e-11e7-825f-6748cda7d858.json

@@ -0,0 +1,10 @@
+{
+  "visState": "{\n  \"title\": \"Audit Event Types\",\n  \"type\": \"pie\",\n  \"params\": {\n    \"addTooltip\": true,\n    \"addLegend\": true,\n    \"legendPosition\": \"right\",\n    \"isDonut\": true\n  },\n  \"aggs\": [\n    {\n      \"id\": \"1\",\n      \"enabled\": true,\n      \"type\": \"count\",\n      \"schema\": \"metric\",\n      \"params\": {}\n    },\n    {\n      \"id\": \"2\",\n      \"enabled\": true,\n      \"type\": \"terms\",\n      \"schema\": \"segment\",\n      \"params\": {\n        \"field\": \"system.audit.record_type\",\n        \"size\": 50,\n        \"order\": \"desc\",\n        \"orderBy\": \"1\"\n      }\n    }\n  ],\n  \"listeners\": {}\n}", 
+  "description": "", 
+  "title": "Audit Event Types", 
+  "uiStateJSON": "{}", 
+  "version": 1, 
+  "kibanaSavedObjectMeta": {
+    "searchSourceJSON": "{\n  \"index\": \"filebeat-*\",\n  \"query\": {\n    \"query_string\": {\n      \"query\": \"*\",\n      \"analyze_wildcard\": true\n    }\n  },\n  \"filter\": []\n}"
+  }
+}

filebeat/module/system/audit/_meta/kibana/visualization/c5411910-0a87-11e7-8b04-eb22a5669f27.json

@@ -0,0 +1,10 @@
+{
+  "visState": "{\"title\":\"Audit Event Account Tag Cloud\",\"type\":\"tagcloud\",\"params\":{\"scale\":\"linear\",\"orientation\":\"single\",\"minFontSize\":15,\"maxFontSize\":42,\"hideLabel\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"system.audit.acct\",\"size\":15,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}", 
+  "description": "", 
+  "title": "Audit Event Account Tag Cloud", 
+  "uiStateJSON": "{}", 
+  "version": 1, 
+  "kibanaSavedObjectMeta": {
+    "searchSourceJSON": "{\"index\":\"filebeat-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
+  }
+}