elastic · ycombinator · Jan 21, 2019 · Jan 18, 2019
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -75,6 +75,7 @@ https://github.com/elastic/beats/compare/v6.6.0...6.x[Check the HEAD diff]
 - Add `convert_timezone` option to Elasticsearch module to convert dates to UTC. {issue}9756[9756] {pull}9761[9761]
 - Support IPv6 addresses with zone id in IIS ingest pipeline. {issue}9836[9836] error log: {pull}9869[9869] access log: {pull}10029[10029]
 - Support haproxy log lines without captured headers. {issue}9463[9463] {pull}9958[9958]
+- Make elasticsearch/audit fileset be more lenient in parsing node name. {issue}10035[10035] {pull}10135[10135]
 
 *Heartbeat*
 

diff --git a/filebeat/module/elasticsearch/audit/ingest/pipeline.json b/filebeat/module/elasticsearch/audit/ingest/pipeline.json
@@ -1,41 +1,56 @@
 {
-  "description": "Pipeline for parsing elasticsearch audit logs",
-  "processors": [
-      {
-          "rename": {
-              "field": "@timestamp",
-              "target_field": "event.created"
-          }
-      },
-      {
-          "grok": {
-              "field": "message",
-              "patterns": [
-                  "\\[%{TIMESTAMP_ISO8601:elasticsearch.audit.timestamp}\\]\\s*(\\[%{WORD:elasticsearch.node.name}\\])?\\s*\\[%{WORD:elasticsearch.audit.layer}\\]\\s*\\[%{WORD:elasticsearch.audit.event_type}\\]\\s*(origin_type\\=\\[%{WORD:elasticsearch.audit.origin_type}\\])?,?\\s*(origin_address\\=\\[%{IPORHOST:elasticsearch.audit.origin_address}\\])?,?\\s*(principal\\=\\[%{WORD:elasticsearch.audit.principal}\\])?,?\\s*(action\\=\\[%{DATA:elasticsearch.audit.action}\\])?,?\\s*?(uri=\\[%{DATA:elasticsearch.audit.uri}\\])?,?\\s*(request\\=\\[%{WORD:elasticsearch.audit.request}\\])?,?\\s*(request_body\\=\\[%{DATA:elasticsearch.audit.request_body}\\])?,?"
-              ]
-          }
-      },
-      {
-          "date": {
-              "field": "elasticsearch.audit.timestamp",
-              "target_field": "@timestamp",
-              "formats": [
-                "ISO8601"
-            ],
-            {< if .convert_timezone >}"timezone": "{{ event.timezone }}",{< end >}
-            "ignore_failure": true
-          }
-      },
-      {
-        "remove": {
-            "field": "elasticsearch.audit.timestamp"
-          }
-      }
+    "description": "Pipeline for parsing elasticsearch audit logs",
+    "processors": [
+        {
+            "rename": {
+                "field": "@timestamp",
+                "target_field": "event.created"
+            }
+        },
+        {
+            "grok": {
+                "field": "message",
+                "pattern_definitions": {
+                    "ES_TIMESTAMP": "\\[%{TIMESTAMP_ISO8601:elasticsearch.audit.timestamp}\\]",
+                    "ES_NODE_NAME": "(\\[%{DATA:elasticsearch.node.name}\\])?",
+                    "ES_AUDIT_LAYER": "\\[%{WORD:elasticsearch.audit.layer}\\]",
+                    "ES_AUDIT_EVENT_TYPE": "\\[%{WORD:event.type}\\]",
+                    "ES_AUDIT_ORIGIN_TYPE": "(origin_type\\=\\[%{WORD:elasticsearch.audit.origin_type}\\])?",
+                    "ES_AUDIT_ORIGIN_ADDRESS": "(origin_address\\=\\[%{IPORHOST:elasticsearch.audit.origin_address}\\])?",
+                    "ES_AUDIT_PRINCIPAL": "(principal\\=\\[%{WORD:elasticsearch.audit.principal}\\])?",
+                    "ES_AUDIT_ACTION": "(action\\=\\[%{DATA:elasticsearch.audit.action}\\])?",
+                    "ES_AUDIT_URI": "(uri=\\[%{DATA:elasticsearch.audit.uri\\])?",
+                    "ES_AUDIT_REQUEST": "(request\\=\\[%{WORD:elasticsearch.audit.request}\\])?",
+                    "ES_AUDIT_REQUEST_BODY": "(request_body\\=\\[%{DATA:elasticsearch.audit.request_body}\\])?"
+                },
+                "patterns": [
+                    "%{ES_TIMESTAMP}\\s*%{ES_NODE_NAME}\\s*%{ES_AUDIT_LAYER}\\s*%{ES_AUDIT_EVENT_TYPE}\\s*%{ES_AUDIT_ORIGIN_TYPE},?\\s*%{ES_AUDIT_ORIGIN_ADDRESS},?\\s*%{ES_AUDIT_PRINCIPAL},?\\s*%{ES_AUDIT_ACTION},?\\s*%{ES_AUDIT_URI},?\\s*%{ES_AUDIT_REQUEST},?\\s*%{ES_AUDIT_REQUEST_BODY},?"
+                ]
+            }
+        },
+        {
+            "date": {
+                "field": "elasticsearch.audit.timestamp",
+                "target_field": "@timestamp",
+                "formats": [
+                    "ISO8601"
+                ],
+                {< if .convert_timezone >}"timezone": "{{ event.timezone }}",{< end >}
+                "ignore_failure": true
+            }
+        },
+        {
+            "remove": {
+                "field": "elasticsearch.audit.timestamp"
+            }
+        }
     ],
-  "on_failure" : [{
-    "set" : {
-      "field" : "error.message",
-      "value" : "{{ _ingest.on_failure_message }}"
-    }
-  }]
+    "on_failure": [
+        {
+            "set": {
+                "field": "error.message",
+                "value": "{{ _ingest.on_failure_message }}"
+            }
+        }
+    ]
 }
diff --git a/filebeat/module/elasticsearch/audit/test/test.log b/filebeat/module/elasticsearch/audit/test/test.log
@@ -5,3 +5,4 @@
 [2018-06-19T05:26:27,268] [rest] [authentication_failed]	origin_address=[147.107.128.77], principal=[N078801], uri=[/_xpack/security/_authenticate]
 [2018-06-19T05:55:26,898] [transport] [access_denied]	origin_type=[rest], origin_address=[147.107.128.77], principal=[_anonymous], action=[cluster:monitor/main], request=[MainRequest]
 [2018-06-19T05:24:15,190] [v_VJhjV] [rest] [authentication_failed]	origin_address=[172.18.0.3], principal=[elastic], uri=[/_nodes?filter_path=nodes.*.version%2Cnodes.*.http.publish_address%2Cnodes.*.ip], request_body=[body]
+[2019-01-08T14:15:02,011] [NodeName-0] [transport] [access_granted]     origin_type=[transport], origin_address=[192.168.2.1], principal=[username], realm=[active_directory], roles=[kibana_user,my_custom_role_1,foo_reader], action=[indices:data/read/search[free_context]], indices=[foo-2019.01.04,foo-2019.01.03,foo-2019.01.06,foo-2019.01.05,foo-2019.01.08,servicelog-2019.01.07], request=[SearchFreeContextRequest]
diff --git a/filebeat/module/elasticsearch/audit/test/test.log-expected.json b/filebeat/module/elasticsearch/audit/test/test.log-expected.json
@@ -1,12 +1,11 @@
 [
     {
         "@timestamp": "2018-06-19T05:16:15.549Z",
-        "elasticsearch.audit.event_type": "authentication_failed",
         "elasticsearch.audit.layer": "rest",
         "elasticsearch.audit.origin_address": "147.107.128.77",
         "elasticsearch.audit.principal": "i030648",
-        "elasticsearch.audit.uri": "/_xpack/security/_authenticate",
         "event.dataset": "elasticsearch.audit",
+        "event.type": "authentication_failed",
         "fileset.module": "elasticsearch",
         "fileset.name": "audit",
         "input.type": "log",
@@ -17,13 +16,12 @@
     },
     {
         "@timestamp": "2018-06-19T05:07:52.304Z",
-        "elasticsearch.audit.event_type": "authentication_failed",
         "elasticsearch.audit.layer": "rest",
         "elasticsearch.audit.origin_address": "172.22.0.3",
         "elasticsearch.audit.principal": "rado",
-        "elasticsearch.audit.uri": "/_xpack/security/_authenticate",
         "elasticsearch.node.name": "v_VJhjV",
         "event.dataset": "elasticsearch.audit",
+        "event.type": "authentication_failed",
         "fileset.module": "elasticsearch",
         "fileset.name": "audit",
         "input.type": "log",
@@ -35,13 +33,13 @@
     {
         "@timestamp": "2018-06-19T05:00:15.778Z",
         "elasticsearch.audit.action": "indices:data/read/scroll/clear",
-        "elasticsearch.audit.event_type": "access_granted",
         "elasticsearch.audit.layer": "transport",
         "elasticsearch.audit.origin_address": "192.168.1.165",
         "elasticsearch.audit.origin_type": "local_node",
         "elasticsearch.audit.principal": "_xpack_security",
         "elasticsearch.audit.request": "ClearScrollRequest",
         "event.dataset": "elasticsearch.audit",
+        "event.type": "access_granted",
         "fileset.module": "elasticsearch",
         "fileset.name": "audit",
         "input.type": "log",
@@ -52,12 +50,11 @@
     },
     {
         "@timestamp": "2018-06-19T05:07:45.544Z",
-        "elasticsearch.audit.event_type": "anonymous_access_denied",
         "elasticsearch.audit.layer": "rest",
         "elasticsearch.audit.origin_address": "172.22.0.3",
-        "elasticsearch.audit.uri": "/_xpack/security/_authenticate",
         "elasticsearch.node.name": "v_VJhjV",
         "event.dataset": "elasticsearch.audit",
+        "event.type": "anonymous_access_denied",
         "fileset.module": "elasticsearch",
         "fileset.name": "audit",
         "input.type": "log",
@@ -68,12 +65,11 @@
     },
     {
         "@timestamp": "2018-06-19T05:26:27.268Z",
-        "elasticsearch.audit.event_type": "authentication_failed",
         "elasticsearch.audit.layer": "rest",
         "elasticsearch.audit.origin_address": "147.107.128.77",
         "elasticsearch.audit.principal": "N078801",
-        "elasticsearch.audit.uri": "/_xpack/security/_authenticate",
         "event.dataset": "elasticsearch.audit",
+        "event.type": "authentication_failed",
         "fileset.module": "elasticsearch",
         "fileset.name": "audit",
         "input.type": "log",
@@ -85,13 +81,13 @@
     {
         "@timestamp": "2018-06-19T05:55:26.898Z",
         "elasticsearch.audit.action": "cluster:monitor/main",
-        "elasticsearch.audit.event_type": "access_denied",
         "elasticsearch.audit.layer": "transport",
         "elasticsearch.audit.origin_address": "147.107.128.77",
         "elasticsearch.audit.origin_type": "rest",
         "elasticsearch.audit.principal": "_anonymous",
         "elasticsearch.audit.request": "MainRequest",
         "event.dataset": "elasticsearch.audit",
+        "event.type": "access_denied",
         "fileset.module": "elasticsearch",
         "fileset.name": "audit",
         "input.type": "log",
@@ -102,20 +98,35 @@
     },
     {
         "@timestamp": "2018-06-19T05:24:15.190Z",
-        "elasticsearch.audit.event_type": "authentication_failed",
         "elasticsearch.audit.layer": "rest",
         "elasticsearch.audit.origin_address": "172.18.0.3",
         "elasticsearch.audit.principal": "elastic",
-        "elasticsearch.audit.request_body": "body",
-        "elasticsearch.audit.uri": "/_nodes?filter_path=nodes.*.version%2Cnodes.*.http.publish_address%2Cnodes.*.ip",
         "elasticsearch.node.name": "v_VJhjV",
         "event.dataset": "elasticsearch.audit",
+        "event.type": "authentication_failed",
         "fileset.module": "elasticsearch",
         "fileset.name": "audit",
         "input.type": "log",
         "message": "[2018-06-19T05:24:15,190] [v_VJhjV] [rest] [authentication_failed]\torigin_address=[172.18.0.3], principal=[elastic], uri=[/_nodes?filter_path=nodes.*.version%2Cnodes.*.http.publish_address%2Cnodes.*.ip], request_body=[body]",
         "offset": 986,
         "prospector.type": "log",
         "service.name": "elasticsearch"
+    },
+    {
+        "@timestamp": "2019-01-08T14:15:02.011Z",
+        "elasticsearch.audit.layer": "transport",
+        "elasticsearch.audit.origin_address": "192.168.2.1",
+        "elasticsearch.audit.origin_type": "transport",
+        "elasticsearch.audit.principal": "username",
+        "elasticsearch.node.name": "NodeName-0",
+        "event.dataset": "elasticsearch.audit",
+        "event.type": "access_granted",
+        "fileset.module": "elasticsearch",
+        "fileset.name": "audit",
+        "input.type": "log",
+        "message": "[2019-01-08T14:15:02,011] [NodeName-0] [transport] [access_granted]     origin_type=[transport], origin_address=[192.168.2.1], principal=[username], realm=[active_directory], roles=[kibana_user,my_custom_role_1,foo_reader], action=[indices:data/read/search[free_context]], indices=[foo-2019.01.04,foo-2019.01.03,foo-2019.01.06,foo-2019.01.05,foo-2019.01.08,servicelog-2019.01.07], request=[SearchFreeContextRequest]",
+        "offset": 1210,
+        "prospector.type": "log",
+        "service.name": "elasticsearch"
     }
 ]