Teach elasticsearch/audit fileset to parse out some more fields (#10137)

* Be more lenient in parsing node name * Parse out elasticsearch.audit.realm * Adding CHANGELOG entry * Parse out elasticsearch.audit.roles * Parse out elasticsearch.audit.indices * Parse out optional sub-action * Regenerating generated files * Regenerating generated files
elastic · Jan 18, 2019 · 210460e · 210460e
1 parent 99926ea
commit 210460e
Show file tree

Hide file tree

Showing 6 changed files with 93 additions and 5 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -141,6 +141,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add support for MariaDB in the `slowlog` fileset of `mysql` module. {pull}9731[9731]
 - Elasticsearch module's slowlog now populates `event.duration` (ECS). {pull}9293[9293]
 - HAProxy module now populates `event.duration` and `http.response.bytes` (ECS). {pull}10143[10143]
+- Teach elasticsearch/audit fileset to parse out some more fields. {issue}10134[10134] {pull}10137[10137]
 
 *Heartbeat*
 

diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc
@@ -3965,6 +3965,26 @@ Where the request originated: rest (request originated from a REST API request),
 
 --
 
+*`elasticsearch.audit.realm`*::
++
+--
+type: keyword
+
+The authentication realm
+
+--
+
+*`elasticsearch.audit.roles`*::
++
+--
+type: array
+
+example: ['kibana_user', 'beats_admin']
+
+Roles to which the principal belongs
+
+--
+
 *`elasticsearch.audit.action`*::
 +
 --
@@ -3976,6 +3996,17 @@ The name of the action that was executed
 
 --
 
+*`elasticsearch.audit.indices`*::
++
+--
+type: array
+
+example: ['foo-2019.01.04', 'foo-2019.01.03', 'foo-2019.01.06']
+
+Indices accessed by action
+
+--
+
 *`elasticsearch.audit.request`*::
 +
 --

diff --git a/filebeat/module/elasticsearch/audit/_meta/fields.yml b/filebeat/module/elasticsearch/audit/_meta/fields.yml
@@ -10,15 +10,26 @@
       description: "Where the request originated: rest (request originated from a REST API request), transport (request was received on the transport channel), local_node (the local node issued the request)"
       example: "local_node"
       type: keyword
+    - name: realm
+      description: "The authentication realm"
+      example": "active_directory"
+      type: keyword
+    - name: roles
+      description: "Roles to which the principal belongs"
+      example: [ "kibana_user", "beats_admin" ]
+      type: array
     - name: action
       description: "The name of the action that was executed"
       example: "cluster:monitor/main"
       type: keyword
+    - name: indices
+      description: "Indices accessed by action"
+      example: [ "foo-2019.01.04", "foo-2019.01.03", "foo-2019.01.06" ]
+      type: array
     - name: request
       description: "The type of request that was executed"
       example: "ClearScrollRequest"
       type: keyword
-
     - name: event_type
       type: alias
       path: event.type

diff --git a/filebeat/module/elasticsearch/audit/ingest/pipeline.json b/filebeat/module/elasticsearch/audit/ingest/pipeline.json
@@ -18,16 +18,45 @@
                     "ES_AUDIT_ORIGIN_TYPE": "(origin_type\\=\\[%{WORD:elasticsearch.audit.origin_type}\\])?",
                     "ES_AUDIT_ORIGIN_ADDRESS": "(origin_address\\=\\[%{IPORHOST:source.ip}\\])?",
                     "ES_AUDIT_PRINCIPAL": "(principal\\=\\[%{WORD:user.name}\\])?",
-                    "ES_AUDIT_ACTION": "(action\\=\\[%{DATA:elasticsearch.audit.action}\\])?",
+                    "ES_AUDIT_REALM": "(realm\\=\\[%{WORD:elasticsearch.audit.realm}\\])?",
+                    "ES_AUDIT_ROLES": "(roles\\=\\[%{DATA:elasticsearch.audit.roles}\\])?",
+                    "ES_AUDIT_ACTION": "(action\\=\\[%{DATA:elasticsearch.audit.action}(\\[%{DATA:elasticsearch.audit.sub_action}\\])?\\])?",
                     "ES_AUDIT_URI": "(uri=\\[%{DATA:url.original}\\])?",
+                    "ES_AUDIT_INDICES": "(indices\\=\\[%{DATA:elasticsearch.audit.indices}\\])?",
                     "ES_AUDIT_REQUEST": "(request\\=\\[%{WORD:elasticsearch.audit.request}\\])?",
                     "ES_AUDIT_REQUEST_BODY": "(request_body\\=\\[%{DATA:http.request.body.content}\\])?"
                 },
                 "patterns": [
-                    "%{ES_TIMESTAMP}\\s*%{ES_NODE_NAME}\\s*%{ES_AUDIT_LAYER}\\s*%{ES_AUDIT_EVENT_TYPE}\\s*%{ES_AUDIT_ORIGIN_TYPE},?\\s*%{ES_AUDIT_ORIGIN_ADDRESS},?\\s*%{ES_AUDIT_PRINCIPAL},?\\s*%{ES_AUDIT_ACTION},?\\s*%{ES_AUDIT_URI},?\\s*%{ES_AUDIT_REQUEST},?\\s*%{ES_AUDIT_REQUEST_BODY},?"
+                    "%{ES_TIMESTAMP}\\s*%{ES_NODE_NAME}\\s*%{ES_AUDIT_LAYER}\\s*%{ES_AUDIT_EVENT_TYPE}\\s*%{ES_AUDIT_ORIGIN_TYPE},?\\s*%{ES_AUDIT_ORIGIN_ADDRESS},?\\s*%{ES_AUDIT_PRINCIPAL},?\\s*%{ES_AUDIT_REALM},?\\s*%{ES_AUDIT_ROLES},?\\s*%{ES_AUDIT_ACTION},?\\s*%{ES_AUDIT_INDICES},?\\s*%{ES_AUDIT_URI},?\\s*%{ES_AUDIT_REQUEST},?\\s*%{ES_AUDIT_REQUEST_BODY},?"
                 ]
             }
         },
+        {
+            "split": {
+                "field": "elasticsearch.audit.roles",
+                "separator": ",",
+                "ignore_missing": true
+            }
+        },
+        {
+            "split": {
+                "field": "elasticsearch.audit.indices",
+                "separator": ",",
+                "ignore_missing": true
+            }
+        },
+        {
+            "script": {
+                "lang": "painless",
+                "source": "if (ctx.elasticsearch.audit.sub_action != null) { ctx.elasticsearch.audit.action += '[' + ctx.elasticsearch.audit.sub_action + ']' }"
+            }
+        },
+        {
+            "remove": {
+                "field": "elasticsearch.audit.sub_action",
+                "ignore_missing": true
+            }
+        },
         {
             "date": {
                 "field": "elasticsearch.audit.timestamp",

diff --git a/filebeat/module/elasticsearch/audit/test/test.log-expected.json b/filebeat/module/elasticsearch/audit/test/test.log-expected.json
@@ -121,8 +121,24 @@
     {
         "@timestamp": "2019-01-08T14:15:02.011Z",
         "ecs.version": "1.0.0-beta2",
+        "elasticsearch.audit.action": "indices:data/read/search[free_context]",
+        "elasticsearch.audit.indices": [
+            "foo-2019.01.04",
+            "foo-2019.01.03",
+            "foo-2019.01.06",
+            "foo-2019.01.05",
+            "foo-2019.01.08",
+            "servicelog-2019.01.07"
+        ],
         "elasticsearch.audit.layer": "transport",
         "elasticsearch.audit.origin_type": "transport",
+        "elasticsearch.audit.realm": "active_directory",
+        "elasticsearch.audit.request": "SearchFreeContextRequest",
+        "elasticsearch.audit.roles": [
+            "kibana_user",
+            "my_custom_role_1",
+            "foo_reader"
+        ],
         "elasticsearch.node.name": "NodeName-0",
         "event.dataset": "elasticsearch.audit",
         "event.module": "elasticsearch",
@@ -135,4 +151,4 @@
         "source.ip": "192.168.2.1",
         "user.name": "username"
     }
-]
+]
diff --git a/filebeat/module/elasticsearch/fields.go b/filebeat/module/elasticsearch/fields.go