diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 16e9502ef7e..13990683dec 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -41,6 +41,7 @@ https://github.com/elastic/beats/compare/1035569addc4a3b29ffa14f8a08c27c1ace16ef - Fix bad bytes count in `docker` input when filtering by stream. {pull}10211[10211] - Add `convert_timezone` option to Logstash module to convert dates to UTC. {issue}9756[9756] {pull}9797[9797] - Add `convert_timezone` option to Elasticsearch module to convert dates to UTC. {issue}9756[9756] {pull}9761[9761] +- Make elasticsearch/audit fileset be more lenient in parsing node name. {issue}10035[10035] {pull}10135[10135] *Heartbeat* diff --git a/filebeat/module/elasticsearch/audit/ingest/pipeline.json b/filebeat/module/elasticsearch/audit/ingest/pipeline.json index d686ba846b2..d704f3a3ecd 100644 --- a/filebeat/module/elasticsearch/audit/ingest/pipeline.json +++ b/filebeat/module/elasticsearch/audit/ingest/pipeline.json @@ -1,41 +1,56 @@ { - "description": "Pipeline for parsing elasticsearch audit logs", - "processors": [ - { - "rename": { - "field": "@timestamp", - "target_field": "event.created" - } - }, - { - "grok": { - "field": "message", - "patterns": [ - "\\[%{TIMESTAMP_ISO8601:elasticsearch.audit.timestamp}\\]\\s*(\\[%{WORD:elasticsearch.node.name}\\])?\\s*\\[%{WORD:elasticsearch.audit.layer}\\]\\s*\\[%{WORD:elasticsearch.audit.event_type}\\]\\s*(origin_type\\=\\[%{WORD:elasticsearch.audit.origin_type}\\])?,?\\s*(origin_address\\=\\[%{IPORHOST:elasticsearch.audit.origin_address}\\])?,?\\s*(principal\\=\\[%{WORD:elasticsearch.audit.principal}\\])?,?\\s*(action\\=\\[%{DATA:elasticsearch.audit.action}\\])?,?\\s*?(uri=\\[%{DATA:elasticsearch.audit.uri}\\])?,?\\s*(request\\=\\[%{WORD:elasticsearch.audit.request}\\])?,?\\s*(request_body\\=\\[%{DATA:elasticsearch.audit.request_body}\\])?,?" - ] - } - }, - { - "date": { - "field": "elasticsearch.audit.timestamp", - "target_field": "@timestamp", - "formats": [ - "ISO8601" - ], - {< if .convert_timezone >}"timezone": "{{ event.timezone }}",{< end >} - "ignore_failure": true - } - }, - { - "remove": { - "field": "elasticsearch.audit.timestamp" - } - } + "description": "Pipeline for parsing elasticsearch audit logs", + "processors": [ + { + "rename": { + "field": "@timestamp", + "target_field": "event.created" + } + }, + { + "grok": { + "field": "message", + "pattern_definitions": { + "ES_TIMESTAMP": "\\[%{TIMESTAMP_ISO8601:elasticsearch.audit.timestamp}\\]", + "ES_NODE_NAME": "(\\[%{DATA:elasticsearch.node.name}\\])?", + "ES_AUDIT_LAYER": "\\[%{WORD:elasticsearch.audit.layer}\\]", + "ES_AUDIT_EVENT_TYPE": "\\[%{WORD:elasticsearch.audit.event_type}\\]", + "ES_AUDIT_ORIGIN_TYPE": "(origin_type\\=\\[%{WORD:elasticsearch.audit.origin_type}\\])?", + "ES_AUDIT_ORIGIN_ADDRESS": "(origin_address\\=\\[%{IPORHOST:elasticsearch.audit.origin_address}\\])?", + "ES_AUDIT_PRINCIPAL": "(principal\\=\\[%{WORD:elasticsearch.audit.principal}\\])?", + "ES_AUDIT_ACTION": "(action\\=\\[%{DATA:elasticsearch.audit.action}\\])?", + "ES_AUDIT_URI": "(uri=\\[%{DATA:elasticsearch.audit.uri}\\])?", + "ES_AUDIT_REQUEST": "(request\\=\\[%{WORD:elasticsearch.audit.request}\\])?", + "ES_AUDIT_REQUEST_BODY": "(request_body\\=\\[%{DATA:elasticsearch.audit.request_body}\\])?" + }, + "patterns": [ + "%{ES_TIMESTAMP}\\s*%{ES_NODE_NAME}\\s*%{ES_AUDIT_LAYER}\\s*%{ES_AUDIT_EVENT_TYPE}\\s*%{ES_AUDIT_ORIGIN_TYPE},?\\s*%{ES_AUDIT_ORIGIN_ADDRESS},?\\s*%{ES_AUDIT_PRINCIPAL},?\\s*%{ES_AUDIT_ACTION},?\\s*%{ES_AUDIT_URI},?\\s*%{ES_AUDIT_REQUEST},?\\s*%{ES_AUDIT_REQUEST_BODY},?" + ] + } + }, + { + "date": { + "field": "elasticsearch.audit.timestamp", + "target_field": "@timestamp", + "formats": [ + "ISO8601" + ], + {< if .convert_timezone >}"timezone": "{{ event.timezone }}",{< end >} + "ignore_failure": true + } + }, + { + "remove": { + "field": "elasticsearch.audit.timestamp" + } + } ], - "on_failure" : [{ - "set" : { - "field" : "error.message", - "value" : "{{ _ingest.on_failure_message }}" - } - }] + "on_failure": [ + { + "set": { + "field": "error.message", + "value": "{{ _ingest.on_failure_message }}" + } + } + ] } diff --git a/filebeat/module/elasticsearch/audit/test/test.log b/filebeat/module/elasticsearch/audit/test/test.log index cabc1ee67d9..c631cc62837 100644 --- a/filebeat/module/elasticsearch/audit/test/test.log +++ b/filebeat/module/elasticsearch/audit/test/test.log @@ -5,3 +5,4 @@ [2018-06-19T05:26:27,268] [rest] [authentication_failed] origin_address=[147.107.128.77], principal=[N078801], uri=[/_xpack/security/_authenticate] [2018-06-19T05:55:26,898] [transport] [access_denied] origin_type=[rest], origin_address=[147.107.128.77], principal=[_anonymous], action=[cluster:monitor/main], request=[MainRequest] [2018-06-19T05:24:15,190] [v_VJhjV] [rest] [authentication_failed] origin_address=[172.18.0.3], principal=[elastic], uri=[/_nodes?filter_path=nodes.*.version%2Cnodes.*.http.publish_address%2Cnodes.*.ip], request_body=[body] +[2019-01-08T14:15:02,011] [NodeName-0] [transport] [access_granted] origin_type=[transport], origin_address=[192.168.2.1], principal=[username], realm=[active_directory], roles=[kibana_user,my_custom_role_1,foo_reader], action=[indices:data/read/search[free_context]], indices=[foo-2019.01.04,foo-2019.01.03,foo-2019.01.06,foo-2019.01.05,foo-2019.01.08,servicelog-2019.01.07], request=[SearchFreeContextRequest] diff --git a/filebeat/module/elasticsearch/audit/test/test.log-expected.json b/filebeat/module/elasticsearch/audit/test/test.log-expected.json index 91a7f16d3ea..6da7c2e1436 100644 --- a/filebeat/module/elasticsearch/audit/test/test.log-expected.json +++ b/filebeat/module/elasticsearch/audit/test/test.log-expected.json @@ -117,5 +117,22 @@ "offset": 986, "prospector.type": "log", "service.name": "elasticsearch" + }, + { + "@timestamp": "2019-01-08T14:15:02.011Z", + "elasticsearch.audit.event_type": "access_granted", + "elasticsearch.audit.layer": "transport", + "elasticsearch.audit.origin_address": "192.168.2.1", + "elasticsearch.audit.origin_type": "transport", + "elasticsearch.audit.principal": "username", + "elasticsearch.node.name": "NodeName-0", + "event.dataset": "elasticsearch.audit", + "fileset.module": "elasticsearch", + "fileset.name": "audit", + "input.type": "log", + "message": "[2019-01-08T14:15:02,011] [NodeName-0] [transport] [access_granted] origin_type=[transport], origin_address=[192.168.2.1], principal=[username], realm=[active_directory], roles=[kibana_user,my_custom_role_1,foo_reader], action=[indices:data/read/search[free_context]], indices=[foo-2019.01.04,foo-2019.01.03,foo-2019.01.06,foo-2019.01.05,foo-2019.01.08,servicelog-2019.01.07], request=[SearchFreeContextRequest]", + "offset": 1210, + "prospector.type": "log", + "service.name": "elasticsearch" } ] \ No newline at end of file