Filebeat: Elasticsearch module: Audit log (#7365)

This is initial PR for Audit logs in Elasticsearch. It successfully index basic events. Following PR's will improve grok patterns and add files for testing. Dashboards will come as bonus.
elastic · Jun 28, 2018 · 7b4e4cb · 7b4e4cb
1 parent c1e0d06
commit 7b4e4cb
Show file tree

Hide file tree

Showing 12 changed files with 355 additions and 1 deletion.
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
@@ -202,6 +202,7 @@ https://github.com/elastic/beats/compare/v6.2.3...master[Check the HEAD diff]
 - Add support for TLS with client authentication to the TCP input {pull}7056[7056]
 - Converted part of pipeline from treafik/access metricSet to dissect to improve efficeny. {pull}7209[7209]
 - Add GC fileset to the Elasticsearch module. {pull}7305[7305]
+- Add Audit log fileset to the Elasticsearch module. {pull}7365[7365]
 
 *Heartbeat*
 

diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc
@@ -800,6 +800,122 @@ elasticsearch Module
 
 
 
+[float]
+== audit fields
+
+
+
+
+*`elasticsearch.audit.node_name`*::
++
+--
+type: keyword
+
+example: v_VJhjV
+
+The name of the node
+
+--
+
+*`elasticsearch.audit.layer`*::
++
+--
+type: keyword
+
+example: rest
+
+The layer from which this event originated: rest, transport or ip_filter
+
+--
+
+*`elasticsearch.audit.event_type`*::
++
+--
+type: keyword
+
+example: access_granted
+
+The type of event that occurred: anonymous_access_denied, authentication_failed, access_denied, access_granted, connection_granted, connection_denied, tampered_request, run_as_granted, run_as_denied
+
+--
+
+*`elasticsearch.audit.origin_type`*::
++
+--
+type: keyword
+
+example: local_node
+
+Where the request originated: rest (request originated from a REST API request), transport (request was received on the transport channel), local_node (the local node issued the request)
+
+--
+
+*`elasticsearch.audit.origin_address`*::
++
+--
+type: ip
+
+example: 192.168.1.42
+
+The IP address from which the request originated
+
+--
+
+*`elasticsearch.audit.principal`*::
++
+--
+type: keyword
+
+example: _anonymous
+
+The principal (username) that failed authentication
+
+--
+
+*`elasticsearch.audit.action`*::
++
+--
+type: keyword
+
+example: cluster:monitor/main
+
+The name of the action that was executed
+
+--
+
+*`elasticsearch.audit.uri`*::
++
+--
+type: keyword
+
+example: /_xpack/security/_authenticate
+
+The REST endpoint URI
+
+--
+
+*`elasticsearch.audit.request`*::
++
+--
+type: keyword
+
+example: ClearScrollRequest
+
+The type of request that was executed
+
+--
+
+*`elasticsearch.audit.request_body`*::
++
+--
+type: text
+
+example: body
+
+The body of the request, if enabled
+
+--
+
 [float]
 == gc fields
 

diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml
@@ -98,6 +98,12 @@ filebeat.modules:
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
 
+  audit:
+    enabled: true
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths:
+
 #------------------------------- Icinga Module -------------------------------
 #- module: icinga
   # Main logs

diff --git a/filebeat/include/fields.go b/filebeat/include/fields.go
diff --git a/filebeat/module/elasticsearch/_meta/config.yml b/filebeat/module/elasticsearch/_meta/config.yml
@@ -12,3 +12,9 @@
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
+
+  audit:
+    enabled: true
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths:
diff --git a/filebeat/module/elasticsearch/audit/_meta/fields.yml b/filebeat/module/elasticsearch/audit/_meta/fields.yml
@@ -0,0 +1,44 @@
+- name: audit
+  type: group
+  description: >
+  fields:
+    - name: node_name
+      description: "The name of the node"
+      example: "v_VJhjV"
+      type: keyword
+    - name: layer
+      description: "The layer from which this event originated: rest, transport or ip_filter"
+      example: "rest"
+      type: keyword
+    - name: event_type
+      description: "The type of event that occurred: anonymous_access_denied, authentication_failed, access_denied, access_granted, connection_granted, connection_denied, tampered_request, run_as_granted, run_as_denied"
+      example: "access_granted"
+      type: keyword
+    - name: origin_type
+      description: "Where the request originated: rest (request originated from a REST API request), transport (request was received on the transport channel), local_node (the local node issued the request)"
+      example: "local_node"
+      type: keyword
+    - name: origin_address
+      description: "The IP address from which the request originated"
+      example: "192.168.1.42"
+      type: ip
+    - name: principal
+      description: "The principal (username) that failed authentication"
+      example: "_anonymous"
+      type: keyword
+    - name: action
+      description: "The name of the action that was executed"
+      example: "cluster:monitor/main"
+      type: keyword
+    - name: uri
+      description: "The REST endpoint URI"
+      example: /_xpack/security/_authenticate
+      type: keyword
+    - name: request
+      description: "The type of request that was executed"
+      example: "ClearScrollRequest"
+      type: keyword
+    - name: request_body
+      description: "The body of the request, if enabled"
+      example: "body"
+      type: text
diff --git a/filebeat/module/elasticsearch/audit/config/audit.yml b/filebeat/module/elasticsearch/audit/config/audit.yml
@@ -0,0 +1,10 @@
+type: log
+paths:
+{{ range $i, $path := .paths }}
+ - {{$path}}
+{{ end }}
+exclude_files: [".gz$"]
+
+fields:
+  service.name: "elasticsearch"
+fields_under_root: true
diff --git a/filebeat/module/elasticsearch/audit/ingest/pipeline.json b/filebeat/module/elasticsearch/audit/ingest/pipeline.json
@@ -0,0 +1,31 @@
+{
+  "description": "Pipeline for parsing elasticsearch audit logs",
+  "processors": [
+      {
+          "rename": {
+              "field": "@timestamp",
+              "target_field": "event.created"
+          }
+      },
+      {
+          "grok": {
+              "field": "message",
+              "patterns": [
+                  "\\[%{TIMESTAMP_ISO8601:elasticsearch.audit.timestamp}\\]\\s*(\\[%{WORD:elasticsearch.audit.node_name}\\])?\\s*\\[%{WORD:elasticsearch.audit.layer}\\]\\s*\\[%{WORD:elasticsearch.audit.event_type}\\]\\s*(origin_type\\=\\[%{WORD:elasticsearch.audit.origin_type}\\])?,?\\s*(origin_address\\=\\[%{IPORHOST:elasticsearch.audit.origin_address}\\])?,?\\s*(principal\\=\\[%{WORD:elasticsearch.audit.principal}\\])?,?\\s*(action\\=\\[%{DATA:elasticsearch.audit.action}\\])?,?\\s*?(uri=\\[%{DATA:elasticsearch.audit.uri}\\])?,?\\s*(request\\=\\[%{WORD:elasticsearch.audit.request}\\])?,?\\s*(request_body\\=\\[%{DATA:elasticsearch.audit.request_body}\\])?,?"
+              ]
+          }
+      },
+      {
+          "rename": {
+              "field": "elasticsearch.audit.timestamp",
+              "target_field": "@timestamp"
+          }
+      }
+    ],
+  "on_failure" : [{
+    "set" : {
+      "field" : "error.message",
+      "value" : "{{ _ingest.on_failure_message }}"
+    }
+  }]
+}
diff --git a/filebeat/module/elasticsearch/audit/manifest.yml b/filebeat/module/elasticsearch/audit/manifest.yml
@@ -0,0 +1,13 @@
+module_version: 1.0
+
+var:
+  - name: paths
+    default:
+      - /var/log/elasticsearch/*_access.log
+    os.darwin:
+      - /usr/local/elasticsearch/*_access.log
+    os.windows:
+      - c:/ProgramData/Elastic/Elasticsearch/logs/*_access.log
+
+ingest_pipeline: ingest/pipeline.json
+input: config/audit.yml
diff --git a/filebeat/module/elasticsearch/audit/test/test.log b/filebeat/module/elasticsearch/audit/test/test.log
@@ -0,0 +1,7 @@
+[2018-06-19T05:16:15,549] [rest] [authentication_failed]        origin_address=[147.107.128.77], principal=[i030648], uri=[/_xpack/security/_authenticate]
+[2018-06-19T05:07:52,304] [v_VJhjV] [rest] [authentication_failed]	origin_address=[172.22.0.3], principal=[rado], uri=[/_xpack/security/_authenticate]
+[2018-06-19T05:00:15,778] [transport] [access_granted]  origin_type=[local_node], origin_address=[192.168.1.165], principal=[_xpack_security], action=[indices:data/read/scroll/clear], request=[ClearScrollRequest]
+[2018-06-19T05:07:45,544] [v_VJhjV] [rest] [anonymous_access_denied]	origin_address=[172.22.0.3], uri=[/_xpack/security/_authenticate]
+[2018-06-19T05:26:27,268] [rest] [authentication_failed]	origin_address=[147.107.128.77], principal=[N078801], uri=[/_xpack/security/_authenticate]
+[2018-06-19T05:55:26,898] [transport] [access_denied]	origin_type=[rest], origin_address=[147.107.128.77], principal=[_anonymous], action=[cluster:monitor/main], request=[MainRequest]
+[2018-06-19T05:24:15,190] [v_VJhjV] [rest] [authentication_failed]	origin_address=[172.18.0.3], principal=[elastic], uri=[/_nodes?filter_path=nodes.*.version%2Cnodes.*.http.publish_address%2Cnodes.*.ip], request_body=[body]
diff --git a/filebeat/module/elasticsearch/audit/test/test.log-expected.json b/filebeat/module/elasticsearch/audit/test/test.log-expected.json
@@ -0,0 +1,114 @@
+[
+    {
+        "@timestamp": "2018-06-19T05:16:15,549", 
+        "elasticsearch.audit.event_type": "authentication_failed", 
+        "elasticsearch.audit.layer": "rest", 
+        "elasticsearch.audit.origin_address": "147.107.128.77", 
+        "elasticsearch.audit.principal": "i030648", 
+        "elasticsearch.audit.uri": "/_xpack/security/_authenticate", 
+        "fileset.module": "elasticsearch", 
+        "fileset.name": "audit", 
+        "input.type": "log", 
+        "message": "[2018-06-19T05:16:15,549] [rest] [authentication_failed]        origin_address=[147.107.128.77], principal=[i030648], uri=[/_xpack/security/_authenticate]", 
+        "offset": 0, 
+        "prospector.type": "log", 
+        "service.name": "elasticsearch"
+    }, 
+    {
+        "@timestamp": "2018-06-19T05:07:52,304", 
+        "elasticsearch.audit.event_type": "authentication_failed", 
+        "elasticsearch.audit.layer": "rest", 
+        "elasticsearch.audit.node_name": "v_VJhjV", 
+        "elasticsearch.audit.origin_address": "172.22.0.3", 
+        "elasticsearch.audit.principal": "rado", 
+        "elasticsearch.audit.uri": "/_xpack/security/_authenticate", 
+        "fileset.module": "elasticsearch", 
+        "fileset.name": "audit", 
+        "input.type": "log", 
+        "message": "[2018-06-19T05:07:52,304] [v_VJhjV] [rest] [authentication_failed]\torigin_address=[172.22.0.3], principal=[rado], uri=[/_xpack/security/_authenticate]", 
+        "offset": 155, 
+        "prospector.type": "log", 
+        "service.name": "elasticsearch"
+    }, 
+    {
+        "@timestamp": "2018-06-19T05:00:15,778", 
+        "elasticsearch.audit.action": "indices:data/read/scroll/clear", 
+        "elasticsearch.audit.event_type": "access_granted", 
+        "elasticsearch.audit.layer": "transport", 
+        "elasticsearch.audit.origin_address": "192.168.1.165", 
+        "elasticsearch.audit.origin_type": "local_node", 
+        "elasticsearch.audit.principal": "_xpack_security", 
+        "elasticsearch.audit.request": "ClearScrollRequest", 
+        "fileset.module": "elasticsearch", 
+        "fileset.name": "audit", 
+        "input.type": "log", 
+        "message": "[2018-06-19T05:00:15,778] [transport] [access_granted]  origin_type=[local_node], origin_address=[192.168.1.165], principal=[_xpack_security], action=[indices:data/read/scroll/clear], request=[ClearScrollRequest]", 
+        "offset": 306, 
+        "prospector.type": "log", 
+        "service.name": "elasticsearch"
+    }, 
+    {
+        "@timestamp": "2018-06-19T05:07:45,544", 
+        "elasticsearch.audit.event_type": "anonymous_access_denied", 
+        "elasticsearch.audit.layer": "rest", 
+        "elasticsearch.audit.node_name": "v_VJhjV", 
+        "elasticsearch.audit.origin_address": "172.22.0.3", 
+        "elasticsearch.audit.uri": "/_xpack/security/_authenticate", 
+        "fileset.module": "elasticsearch", 
+        "fileset.name": "audit", 
+        "input.type": "log", 
+        "message": "[2018-06-19T05:07:45,544] [v_VJhjV] [rest] [anonymous_access_denied]\torigin_address=[172.22.0.3], uri=[/_xpack/security/_authenticate]", 
+        "offset": 519, 
+        "prospector.type": "log", 
+        "service.name": "elasticsearch"
+    }, 
+    {
+        "@timestamp": "2018-06-19T05:26:27,268", 
+        "elasticsearch.audit.event_type": "authentication_failed", 
+        "elasticsearch.audit.layer": "rest", 
+        "elasticsearch.audit.origin_address": "147.107.128.77", 
+        "elasticsearch.audit.principal": "N078801", 
+        "elasticsearch.audit.uri": "/_xpack/security/_authenticate", 
+        "fileset.module": "elasticsearch", 
+        "fileset.name": "audit", 
+        "input.type": "log", 
+        "message": "[2018-06-19T05:26:27,268] [rest] [authentication_failed]\torigin_address=[147.107.128.77], principal=[N078801], uri=[/_xpack/security/_authenticate]", 
+        "offset": 654, 
+        "prospector.type": "log", 
+        "service.name": "elasticsearch"
+    }, 
+    {
+        "@timestamp": "2018-06-19T05:55:26,898", 
+        "elasticsearch.audit.action": "cluster:monitor/main", 
+        "elasticsearch.audit.event_type": "access_denied", 
+        "elasticsearch.audit.layer": "transport", 
+        "elasticsearch.audit.origin_address": "147.107.128.77", 
+        "elasticsearch.audit.origin_type": "rest", 
+        "elasticsearch.audit.principal": "_anonymous", 
+        "elasticsearch.audit.request": "MainRequest", 
+        "fileset.module": "elasticsearch", 
+        "fileset.name": "audit", 
+        "input.type": "log", 
+        "message": "[2018-06-19T05:55:26,898] [transport] [access_denied]\torigin_type=[rest], origin_address=[147.107.128.77], principal=[_anonymous], action=[cluster:monitor/main], request=[MainRequest]", 
+        "offset": 802, 
+        "prospector.type": "log", 
+        "service.name": "elasticsearch"
+    }, 
+    {
+        "@timestamp": "2018-06-19T05:24:15,190", 
+        "elasticsearch.audit.event_type": "authentication_failed", 
+        "elasticsearch.audit.layer": "rest", 
+        "elasticsearch.audit.node_name": "v_VJhjV", 
+        "elasticsearch.audit.origin_address": "172.18.0.3", 
+        "elasticsearch.audit.principal": "elastic", 
+        "elasticsearch.audit.request_body": "body", 
+        "elasticsearch.audit.uri": "/_nodes?filter_path=nodes.*.version%2Cnodes.*.http.publish_address%2Cnodes.*.ip", 
+        "fileset.module": "elasticsearch", 
+        "fileset.name": "audit", 
+        "input.type": "log", 
+        "message": "[2018-06-19T05:24:15,190] [v_VJhjV] [rest] [authentication_failed]\torigin_address=[172.18.0.3], principal=[elastic], uri=[/_nodes?filter_path=nodes.*.version%2Cnodes.*.http.publish_address%2Cnodes.*.ip], request_body=[body]", 
+        "offset": 986, 
+        "prospector.type": "log", 
+        "service.name": "elasticsearch"
+    }
+]
diff --git a/filebeat/modules.d/elasticsearch.yml.disabled b/filebeat/modules.d/elasticsearch.yml.disabled
@@ -12,3 +12,9 @@
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
+
+  audit:
+    enabled: true
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths: