Cherry-pick #10137 to 6.6: Teach elasticsearch/audit fileset to parse out some more fields

CHANGELOG.next.asciidoc

@@ -60,6 +60,8 @@ https://github.com/elastic/beats/compare/1035569addc4a3b29ffa14f8a08c27c1ace16ef
 
 *Filebeat*
 
+- Teach elasticsearch/audit fileset to parse out some more fields. {issue}10134[10134] {pull}10137[10137]
+
 *Heartbeat*
 
 *Journalbeat*

filebeat/docs/fields.asciidoc

@@ -1054,6 +1054,26 @@ The principal (username) that failed authentication
 
 --
 
+*`elasticsearch.audit.realm`*::
++
+--
+type: keyword
+
+The authentication realm
+
+--
+
+*`elasticsearch.audit.roles`*::
++
+--
+type: array
+
+example: ['kibana_user', 'beats_admin']
+
+Roles to which the principal belongs
+
+--
+
 *`elasticsearch.audit.action`*::
 +
 --
@@ -1076,6 +1096,17 @@ The REST endpoint URI
 
 --
 
+*`elasticsearch.audit.indices`*::
++
+--
+type: array
+
+example: ['foo-2019.01.04', 'foo-2019.01.03', 'foo-2019.01.06']
+
+Indices accessed by action
+
+--
+
 *`elasticsearch.audit.request`*::
 +
 --

filebeat/module/elasticsearch/audit/_meta/fields.yml

@@ -22,6 +22,14 @@
       description: "The principal (username) that failed authentication"
       example: "_anonymous"
       type: keyword
+    - name: realm
+      description: "The authentication realm"
+      example": "active_directory"
+      type: keyword
+    - name: roles
+      description: "Roles to which the principal belongs"
+      example: [ "kibana_user", "beats_admin" ]
+      type: array
     - name: action
       description: "The name of the action that was executed"
       example: "cluster:monitor/main"
@@ -30,6 +38,10 @@
       description: "The REST endpoint URI"
       example: /_xpack/security/_authenticate
       type: keyword
+    - name: indices
+      description: "Indices accessed by action"
+      example: [ "foo-2019.01.04", "foo-2019.01.03", "foo-2019.01.06" ]
+      type: array
     - name: request
       description: "The type of request that was executed"
       example: "ClearScrollRequest"

filebeat/module/elasticsearch/audit/ingest/pipeline.json

@@ -1,26 +1,78 @@
 {
-  "description": "Pipeline for parsing elasticsearch audit logs",
-  "processors": [
-      {
-          "rename": {
-              "field": "@timestamp",
-              "target_field": "event.created"
-          }
-      },
-      {
-          "grok": {
-              "field": "message",
-              "patterns": [
-                  "\\[%{TIMESTAMP_ISO8601:elasticsearch.audit.timestamp}\\]\\s*(\\[%{WORD:elasticsearch.node.name}\\])?\\s*\\[%{WORD:elasticsearch.audit.layer}\\]\\s*\\[%{WORD:elasticsearch.audit.event_type}\\]\\s*(origin_type\\=\\[%{WORD:elasticsearch.audit.origin_type}\\])?,?\\s*(origin_address\\=\\[%{IPORHOST:elasticsearch.audit.origin_address}\\])?,?\\s*(principal\\=\\[%{WORD:elasticsearch.audit.principal}\\])?,?\\s*(action\\=\\[%{DATA:elasticsearch.audit.action}\\])?,?\\s*?(uri=\\[%{DATA:elasticsearch.audit.uri}\\])?,?\\s*(request\\=\\[%{WORD:elasticsearch.audit.request}\\])?,?\\s*(request_body\\=\\[%{DATA:elasticsearch.audit.request_body}\\])?,?"
-              ]
-          }
-      },
-      {
-          "rename": {
-              "field": "elasticsearch.audit.timestamp",
-              "target_field": "@timestamp"
-          }
-      }
+    "description": "Pipeline for parsing elasticsearch audit logs",
+    "processors": [
+        {
+            "rename": {
+                "field": "@timestamp",
+                "target_field": "event.created"
+            }
+        },
+        {
+            "grok": {
+                "field": "message",
+                "pattern_definitions": {
+                    "ES_TIMESTAMP": "\\[%{TIMESTAMP_ISO8601:elasticsearch.audit.@timestamp}\\]",
+                    "ES_NODE_NAME": "(\\[%{DATA:elasticsearch.node.name}\\])?",
+                    "ES_AUDIT_LAYER": "\\[%{WORD:elasticsearch.audit.layer}\\]",
+                    "ES_AUDIT_EVENT_TYPE": "\\[%{WORD:elasticsearch.audit.event_type}\\]",
+                    "ES_AUDIT_ORIGIN_TYPE": "(origin_type\\=\\[%{WORD:elasticsearch.audit.origin_type}\\])?",
+                    "ES_AUDIT_ORIGIN_ADDRESS": "(origin_address\\=\\[%{IPORHOST:elasticsearch.audit.origin_address}\\])?",
+                    "ES_AUDIT_PRINCIPAL": "(principal\\=\\[%{DATA:elasticsearch.audit.principal}\\])?",
+                    "ES_AUDIT_REALM": "(realm\\=\\[%{WORD:elasticsearch.audit.realm}\\])?",
+                    "ES_AUDIT_ROLES": "(roles\\=\\[%{DATA:elasticsearch.audit.roles}\\])?",
+                    "ES_AUDIT_ACTION": "(action\\=\\[%{DATA:elasticsearch.audit.action}(\\[%{DATA:elasticsearch.audit.sub_action}\\])?\\])?",
+                    "ES_AUDIT_URI": "(uri=\\[%{DATA:elasticsearch.audit.uri}\\])?",
+                    "ES_AUDIT_INDICES": "(indices\\=\\[%{DATA:elasticsearch.audit.indices}\\])?",
+                    "ES_AUDIT_REQUEST": "(request\\=\\[%{WORD:elasticsearch.audit.request}\\])?",
+                    "ES_AUDIT_REQUEST_BODY": "(request_body\\=\\[%{DATA:elasticsearch.audit.request_body}\\])?"
+                },
+                "patterns": [
+                    "%{ES_TIMESTAMP}\\s*%{ES_NODE_NAME}\\s*%{ES_AUDIT_LAYER}\\s*%{ES_AUDIT_EVENT_TYPE}\\s*%{ES_AUDIT_ORIGIN_TYPE},?\\s*%{ES_AUDIT_ORIGIN_ADDRESS},?\\s*%{ES_AUDIT_PRINCIPAL},?\\s*%{ES_AUDIT_REALM},?\\s*%{ES_AUDIT_ROLES},?\\s*%{ES_AUDIT_ACTION},?\\s*%{ES_AUDIT_INDICES},?\\s*%{ES_AUDIT_URI},?\\s*%{ES_AUDIT_REQUEST},?\\s*%{ES_AUDIT_REQUEST_BODY},?"
+                ]
+            }
+        },
+        {
+            "split": {
+                "field": "elasticsearch.audit.roles",
+                "separator": ",",
+                "ignore_missing": true
+            }
+        },
+        {
+            "split": {
+                "field": "elasticsearch.audit.indices",
+                "separator": ",",
+                "ignore_missing": true
+            }
+        },
+        {
+            "script": {
+                "lang": "painless",
+                "source": "if (ctx.elasticsearch.audit.sub_action != null) { ctx.elasticsearch.audit.action += '[' + ctx.elasticsearch.audit.sub_action + ']' }"
+            }
+        },
+        {
+            "remove": {
+                "field": "elasticsearch.audit.sub_action",
+                "ignore_missing": true
+            }
+        },
+        {
+            "date": {
+                "field": "elasticsearch.audit.timestamp",
+                "target_field": "@timestamp",
+                "formats": [
+                    "ISO8601"
+                ],
+                {< if .convert_timezone >}"timezone": "{{ event.timezone }}",{< end >}
+                "ignore_failure": true
+            }
+        },
+        {
+            "remove": {
+                "field": "elasticsearch.audit.timestamp"
+            }
+        }
     ],
   "on_failure" : [{
     "set" : {