From dc17df230de9f340d56407213bb48b59a7f4ad01 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Fri, 18 Jan 2019 07:03:22 -0800 Subject: [PATCH 1/3] Elasticsearch/audit fileset should be more lenient in parsing node name (#10135) Resolves https://github.com/elastic/beats/issues/10035. This PR: * Uses `DATA` instead of `WORD` in the grok pattern for parsing out `elasticsearch.node.name`, * Breaks out the grok pattern into pattern definitions to increase readability * Removes a redundant `?` after a `*` in the grok pattern (between `elasticsearch.audit.action` and `elasticsearch.audit.uri`), and * Properly reindents the pipeline JSON (so you might want to view the diff with `?w=1` appended to the URL) (cherry picked from commit 93851c22c394c98f8f3571b3cf29a29c3bb6b769) --- CHANGELOG.next.asciidoc | 4 + .../elasticsearch/audit/ingest/pipeline.json | 91 ++++---- .../module/elasticsearch/audit/test/test.log | 1 + .../audit/test/test.log-expected.json | 210 +++++++++--------- 4 files changed, 163 insertions(+), 143 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 16e9502ef7e..2c09659629a 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -41,6 +41,10 @@ https://github.com/elastic/beats/compare/1035569addc4a3b29ffa14f8a08c27c1ace16ef - Fix bad bytes count in `docker` input when filtering by stream. {pull}10211[10211] - Add `convert_timezone` option to Logstash module to convert dates to UTC. {issue}9756[9756] {pull}9797[9797] - Add `convert_timezone` option to Elasticsearch module to convert dates to UTC. {issue}9756[9756] {pull}9761[9761] +- Support IPv6 addresses with zone id in IIS ingest pipeline. + {issue}9836[9836] error log: {pull}9869[9869], access log: {pull}9955[9955]. +- Support haproxy log lines without captured headers. {issue}9463[9463] {pull}9958[9958] +- Make elasticsearch/audit fileset be more lenient in parsing node name. {issue}10035[10035] {pull}10135[10135] *Heartbeat* diff --git a/filebeat/module/elasticsearch/audit/ingest/pipeline.json b/filebeat/module/elasticsearch/audit/ingest/pipeline.json index d686ba846b2..d704f3a3ecd 100644 --- a/filebeat/module/elasticsearch/audit/ingest/pipeline.json +++ b/filebeat/module/elasticsearch/audit/ingest/pipeline.json @@ -1,41 +1,56 @@ { - "description": "Pipeline for parsing elasticsearch audit logs", - "processors": [ - { - "rename": { - "field": "@timestamp", - "target_field": "event.created" - } - }, - { - "grok": { - "field": "message", - "patterns": [ - "\\[%{TIMESTAMP_ISO8601:elasticsearch.audit.timestamp}\\]\\s*(\\[%{WORD:elasticsearch.node.name}\\])?\\s*\\[%{WORD:elasticsearch.audit.layer}\\]\\s*\\[%{WORD:elasticsearch.audit.event_type}\\]\\s*(origin_type\\=\\[%{WORD:elasticsearch.audit.origin_type}\\])?,?\\s*(origin_address\\=\\[%{IPORHOST:elasticsearch.audit.origin_address}\\])?,?\\s*(principal\\=\\[%{WORD:elasticsearch.audit.principal}\\])?,?\\s*(action\\=\\[%{DATA:elasticsearch.audit.action}\\])?,?\\s*?(uri=\\[%{DATA:elasticsearch.audit.uri}\\])?,?\\s*(request\\=\\[%{WORD:elasticsearch.audit.request}\\])?,?\\s*(request_body\\=\\[%{DATA:elasticsearch.audit.request_body}\\])?,?" - ] - } - }, - { - "date": { - "field": "elasticsearch.audit.timestamp", - "target_field": "@timestamp", - "formats": [ - "ISO8601" - ], - {< if .convert_timezone >}"timezone": "{{ event.timezone }}",{< end >} - "ignore_failure": true - } - }, - { - "remove": { - "field": "elasticsearch.audit.timestamp" - } - } + "description": "Pipeline for parsing elasticsearch audit logs", + "processors": [ + { + "rename": { + "field": "@timestamp", + "target_field": "event.created" + } + }, + { + "grok": { + "field": "message", + "pattern_definitions": { + "ES_TIMESTAMP": "\\[%{TIMESTAMP_ISO8601:elasticsearch.audit.timestamp}\\]", + "ES_NODE_NAME": "(\\[%{DATA:elasticsearch.node.name}\\])?", + "ES_AUDIT_LAYER": "\\[%{WORD:elasticsearch.audit.layer}\\]", + "ES_AUDIT_EVENT_TYPE": "\\[%{WORD:elasticsearch.audit.event_type}\\]", + "ES_AUDIT_ORIGIN_TYPE": "(origin_type\\=\\[%{WORD:elasticsearch.audit.origin_type}\\])?", + "ES_AUDIT_ORIGIN_ADDRESS": "(origin_address\\=\\[%{IPORHOST:elasticsearch.audit.origin_address}\\])?", + "ES_AUDIT_PRINCIPAL": "(principal\\=\\[%{WORD:elasticsearch.audit.principal}\\])?", + "ES_AUDIT_ACTION": "(action\\=\\[%{DATA:elasticsearch.audit.action}\\])?", + "ES_AUDIT_URI": "(uri=\\[%{DATA:elasticsearch.audit.uri}\\])?", + "ES_AUDIT_REQUEST": "(request\\=\\[%{WORD:elasticsearch.audit.request}\\])?", + "ES_AUDIT_REQUEST_BODY": "(request_body\\=\\[%{DATA:elasticsearch.audit.request_body}\\])?" + }, + "patterns": [ + "%{ES_TIMESTAMP}\\s*%{ES_NODE_NAME}\\s*%{ES_AUDIT_LAYER}\\s*%{ES_AUDIT_EVENT_TYPE}\\s*%{ES_AUDIT_ORIGIN_TYPE},?\\s*%{ES_AUDIT_ORIGIN_ADDRESS},?\\s*%{ES_AUDIT_PRINCIPAL},?\\s*%{ES_AUDIT_ACTION},?\\s*%{ES_AUDIT_URI},?\\s*%{ES_AUDIT_REQUEST},?\\s*%{ES_AUDIT_REQUEST_BODY},?" + ] + } + }, + { + "date": { + "field": "elasticsearch.audit.timestamp", + "target_field": "@timestamp", + "formats": [ + "ISO8601" + ], + {< if .convert_timezone >}"timezone": "{{ event.timezone }}",{< end >} + "ignore_failure": true + } + }, + { + "remove": { + "field": "elasticsearch.audit.timestamp" + } + } ], - "on_failure" : [{ - "set" : { - "field" : "error.message", - "value" : "{{ _ingest.on_failure_message }}" - } - }] + "on_failure": [ + { + "set": { + "field": "error.message", + "value": "{{ _ingest.on_failure_message }}" + } + } + ] } diff --git a/filebeat/module/elasticsearch/audit/test/test.log b/filebeat/module/elasticsearch/audit/test/test.log index cabc1ee67d9..c631cc62837 100644 --- a/filebeat/module/elasticsearch/audit/test/test.log +++ b/filebeat/module/elasticsearch/audit/test/test.log @@ -5,3 +5,4 @@ [2018-06-19T05:26:27,268] [rest] [authentication_failed] origin_address=[147.107.128.77], principal=[N078801], uri=[/_xpack/security/_authenticate] [2018-06-19T05:55:26,898] [transport] [access_denied] origin_type=[rest], origin_address=[147.107.128.77], principal=[_anonymous], action=[cluster:monitor/main], request=[MainRequest] [2018-06-19T05:24:15,190] [v_VJhjV] [rest] [authentication_failed] origin_address=[172.18.0.3], principal=[elastic], uri=[/_nodes?filter_path=nodes.*.version%2Cnodes.*.http.publish_address%2Cnodes.*.ip], request_body=[body] +[2019-01-08T14:15:02,011] [NodeName-0] [transport] [access_granted] origin_type=[transport], origin_address=[192.168.2.1], principal=[username], realm=[active_directory], roles=[kibana_user,my_custom_role_1,foo_reader], action=[indices:data/read/search[free_context]], indices=[foo-2019.01.04,foo-2019.01.03,foo-2019.01.06,foo-2019.01.05,foo-2019.01.08,servicelog-2019.01.07], request=[SearchFreeContextRequest] diff --git a/filebeat/module/elasticsearch/audit/test/test.log-expected.json b/filebeat/module/elasticsearch/audit/test/test.log-expected.json index 91a7f16d3ea..c928ebf9390 100644 --- a/filebeat/module/elasticsearch/audit/test/test.log-expected.json +++ b/filebeat/module/elasticsearch/audit/test/test.log-expected.json @@ -1,121 +1,121 @@ [ { - "@timestamp": "2018-06-19T05:16:15.549Z", - "elasticsearch.audit.event_type": "authentication_failed", - "elasticsearch.audit.layer": "rest", - "elasticsearch.audit.origin_address": "147.107.128.77", - "elasticsearch.audit.principal": "i030648", - "elasticsearch.audit.uri": "/_xpack/security/_authenticate", - "event.dataset": "elasticsearch.audit", - "fileset.module": "elasticsearch", - "fileset.name": "audit", - "input.type": "log", - "message": "[2018-06-19T05:16:15,549] [rest] [authentication_failed] origin_address=[147.107.128.77], principal=[i030648], uri=[/_xpack/security/_authenticate]", - "offset": 0, - "prospector.type": "log", + "@timestamp": "2018-06-19T05:16:15.549Z", + "elasticsearch.audit.event_type": "authentication_failed", + "elasticsearch.audit.layer": "rest", + "elasticsearch.audit.origin_address": "147.107.128.77", + "elasticsearch.audit.principal": "i030648", + "elasticsearch.audit.uri": "/_xpack/security/_authenticate", + "event.dataset": "elasticsearch.audit", + "fileset.module": "elasticsearch", + "fileset.name": "audit", + "input.type": "log", + "message": "[2018-06-19T05:16:15,549] [rest] [authentication_failed] origin_address=[147.107.128.77], principal=[i030648], uri=[/_xpack/security/_authenticate]", + "offset": 0, + "prospector.type": "log", "service.name": "elasticsearch" - }, + }, { - "@timestamp": "2018-06-19T05:07:52.304Z", - "elasticsearch.audit.event_type": "authentication_failed", - "elasticsearch.audit.layer": "rest", - "elasticsearch.audit.origin_address": "172.22.0.3", - "elasticsearch.audit.principal": "rado", - "elasticsearch.audit.uri": "/_xpack/security/_authenticate", - "elasticsearch.node.name": "v_VJhjV", - "event.dataset": "elasticsearch.audit", - "fileset.module": "elasticsearch", - "fileset.name": "audit", - "input.type": "log", - "message": "[2018-06-19T05:07:52,304] [v_VJhjV] [rest] [authentication_failed]\torigin_address=[172.22.0.3], principal=[rado], uri=[/_xpack/security/_authenticate]", - "offset": 155, - "prospector.type": "log", + "@timestamp": "2018-06-19T05:07:52.304Z", + "elasticsearch.audit.event_type": "authentication_failed", + "elasticsearch.audit.layer": "rest", + "elasticsearch.audit.origin_address": "172.22.0.3", + "elasticsearch.audit.principal": "rado", + "elasticsearch.audit.uri": "/_xpack/security/_authenticate", + "elasticsearch.node.name": "v_VJhjV", + "event.dataset": "elasticsearch.audit", + "fileset.module": "elasticsearch", + "fileset.name": "audit", + "input.type": "log", + "message": "[2018-06-19T05:07:52,304] [v_VJhjV] [rest] [authentication_failed]\torigin_address=[172.22.0.3], principal=[rado], uri=[/_xpack/security/_authenticate]", + "offset": 155, + "prospector.type": "log", "service.name": "elasticsearch" - }, + }, { - "@timestamp": "2018-06-19T05:00:15.778Z", - "elasticsearch.audit.action": "indices:data/read/scroll/clear", - "elasticsearch.audit.event_type": "access_granted", - "elasticsearch.audit.layer": "transport", - "elasticsearch.audit.origin_address": "192.168.1.165", - "elasticsearch.audit.origin_type": "local_node", - "elasticsearch.audit.principal": "_xpack_security", - "elasticsearch.audit.request": "ClearScrollRequest", - "event.dataset": "elasticsearch.audit", - "fileset.module": "elasticsearch", - "fileset.name": "audit", - "input.type": "log", - "message": "[2018-06-19T05:00:15,778] [transport] [access_granted] origin_type=[local_node], origin_address=[192.168.1.165], principal=[_xpack_security], action=[indices:data/read/scroll/clear], request=[ClearScrollRequest]", - "offset": 306, - "prospector.type": "log", + "@timestamp": "2018-06-19T05:00:15.778Z", + "elasticsearch.audit.action": "indices:data/read/scroll/clear", + "elasticsearch.audit.event_type": "access_granted", + "elasticsearch.audit.layer": "transport", + "elasticsearch.audit.origin_address": "192.168.1.165", + "elasticsearch.audit.origin_type": "local_node", + "elasticsearch.audit.principal": "_xpack_security", + "elasticsearch.audit.request": "ClearScrollRequest", + "event.dataset": "elasticsearch.audit", + "fileset.module": "elasticsearch", + "fileset.name": "audit", + "input.type": "log", + "message": "[2018-06-19T05:00:15,778] [transport] [access_granted] origin_type=[local_node], origin_address=[192.168.1.165], principal=[_xpack_security], action=[indices:data/read/scroll/clear], request=[ClearScrollRequest]", + "offset": 306, + "prospector.type": "log", "service.name": "elasticsearch" - }, + }, { - "@timestamp": "2018-06-19T05:07:45.544Z", - "elasticsearch.audit.event_type": "anonymous_access_denied", - "elasticsearch.audit.layer": "rest", - "elasticsearch.audit.origin_address": "172.22.0.3", - "elasticsearch.audit.uri": "/_xpack/security/_authenticate", - "elasticsearch.node.name": "v_VJhjV", - "event.dataset": "elasticsearch.audit", - "fileset.module": "elasticsearch", - "fileset.name": "audit", - "input.type": "log", - "message": "[2018-06-19T05:07:45,544] [v_VJhjV] [rest] [anonymous_access_denied]\torigin_address=[172.22.0.3], uri=[/_xpack/security/_authenticate]", - "offset": 519, - "prospector.type": "log", + "@timestamp": "2018-06-19T05:07:45.544Z", + "elasticsearch.audit.event_type": "anonymous_access_denied", + "elasticsearch.audit.layer": "rest", + "elasticsearch.audit.origin_address": "172.22.0.3", + "elasticsearch.audit.uri": "/_xpack/security/_authenticate", + "elasticsearch.node.name": "v_VJhjV", + "event.dataset": "elasticsearch.audit", + "fileset.module": "elasticsearch", + "fileset.name": "audit", + "input.type": "log", + "message": "[2018-06-19T05:07:45,544] [v_VJhjV] [rest] [anonymous_access_denied]\torigin_address=[172.22.0.3], uri=[/_xpack/security/_authenticate]", + "offset": 519, + "prospector.type": "log", "service.name": "elasticsearch" - }, + }, { - "@timestamp": "2018-06-19T05:26:27.268Z", - "elasticsearch.audit.event_type": "authentication_failed", - "elasticsearch.audit.layer": "rest", - "elasticsearch.audit.origin_address": "147.107.128.77", - "elasticsearch.audit.principal": "N078801", - "elasticsearch.audit.uri": "/_xpack/security/_authenticate", - "event.dataset": "elasticsearch.audit", - "fileset.module": "elasticsearch", - "fileset.name": "audit", - "input.type": "log", - "message": "[2018-06-19T05:26:27,268] [rest] [authentication_failed]\torigin_address=[147.107.128.77], principal=[N078801], uri=[/_xpack/security/_authenticate]", - "offset": 654, - "prospector.type": "log", + "@timestamp": "2018-06-19T05:26:27.268Z", + "elasticsearch.audit.event_type": "authentication_failed", + "elasticsearch.audit.layer": "rest", + "elasticsearch.audit.origin_address": "147.107.128.77", + "elasticsearch.audit.principal": "N078801", + "elasticsearch.audit.uri": "/_xpack/security/_authenticate", + "event.dataset": "elasticsearch.audit", + "fileset.module": "elasticsearch", + "fileset.name": "audit", + "input.type": "log", + "message": "[2018-06-19T05:26:27,268] [rest] [authentication_failed]\torigin_address=[147.107.128.77], principal=[N078801], uri=[/_xpack/security/_authenticate]", + "offset": 654, + "prospector.type": "log", "service.name": "elasticsearch" - }, + }, { - "@timestamp": "2018-06-19T05:55:26.898Z", - "elasticsearch.audit.action": "cluster:monitor/main", - "elasticsearch.audit.event_type": "access_denied", - "elasticsearch.audit.layer": "transport", - "elasticsearch.audit.origin_address": "147.107.128.77", - "elasticsearch.audit.origin_type": "rest", - "elasticsearch.audit.principal": "_anonymous", - "elasticsearch.audit.request": "MainRequest", - "event.dataset": "elasticsearch.audit", - "fileset.module": "elasticsearch", - "fileset.name": "audit", - "input.type": "log", - "message": "[2018-06-19T05:55:26,898] [transport] [access_denied]\torigin_type=[rest], origin_address=[147.107.128.77], principal=[_anonymous], action=[cluster:monitor/main], request=[MainRequest]", - "offset": 802, - "prospector.type": "log", + "@timestamp": "2018-06-19T05:55:26.898Z", + "elasticsearch.audit.action": "cluster:monitor/main", + "elasticsearch.audit.event_type": "access_denied", + "elasticsearch.audit.layer": "transport", + "elasticsearch.audit.origin_address": "147.107.128.77", + "elasticsearch.audit.origin_type": "rest", + "elasticsearch.audit.principal": "_anonymous", + "elasticsearch.audit.request": "MainRequest", + "event.dataset": "elasticsearch.audit", + "fileset.module": "elasticsearch", + "fileset.name": "audit", + "input.type": "log", + "message": "[2018-06-19T05:55:26,898] [transport] [access_denied]\torigin_type=[rest], origin_address=[147.107.128.77], principal=[_anonymous], action=[cluster:monitor/main], request=[MainRequest]", + "offset": 802, + "prospector.type": "log", "service.name": "elasticsearch" - }, + }, { - "@timestamp": "2018-06-19T05:24:15.190Z", - "elasticsearch.audit.event_type": "authentication_failed", - "elasticsearch.audit.layer": "rest", - "elasticsearch.audit.origin_address": "172.18.0.3", - "elasticsearch.audit.principal": "elastic", - "elasticsearch.audit.request_body": "body", - "elasticsearch.audit.uri": "/_nodes?filter_path=nodes.*.version%2Cnodes.*.http.publish_address%2Cnodes.*.ip", - "elasticsearch.node.name": "v_VJhjV", - "event.dataset": "elasticsearch.audit", - "fileset.module": "elasticsearch", - "fileset.name": "audit", - "input.type": "log", - "message": "[2018-06-19T05:24:15,190] [v_VJhjV] [rest] [authentication_failed]\torigin_address=[172.18.0.3], principal=[elastic], uri=[/_nodes?filter_path=nodes.*.version%2Cnodes.*.http.publish_address%2Cnodes.*.ip], request_body=[body]", - "offset": 986, - "prospector.type": "log", + "@timestamp": "2018-06-19T05:24:15.190Z", + "elasticsearch.audit.event_type": "authentication_failed", + "elasticsearch.audit.layer": "rest", + "elasticsearch.audit.origin_address": "172.18.0.3", + "elasticsearch.audit.principal": "elastic", + "elasticsearch.audit.request_body": "body", + "elasticsearch.audit.uri": "/_nodes?filter_path=nodes.*.version%2Cnodes.*.http.publish_address%2Cnodes.*.ip", + "elasticsearch.node.name": "v_VJhjV", + "event.dataset": "elasticsearch.audit", + "fileset.module": "elasticsearch", + "fileset.name": "audit", + "input.type": "log", + "message": "[2018-06-19T05:24:15,190] [v_VJhjV] [rest] [authentication_failed]\torigin_address=[172.18.0.3], principal=[elastic], uri=[/_nodes?filter_path=nodes.*.version%2Cnodes.*.http.publish_address%2Cnodes.*.ip], request_body=[body]", + "offset": 986, + "prospector.type": "log", "service.name": "elasticsearch" } -] \ No newline at end of file +] From 4a40a1ff39360a2e523f8f13a9ca690e96ae17f4 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Thu, 31 Jan 2019 08:05:24 -0800 Subject: [PATCH 2/3] Regenerating golden file --- .../audit/test/test.log-expected.json | 227 ++++++++++-------- 1 file changed, 122 insertions(+), 105 deletions(-) diff --git a/filebeat/module/elasticsearch/audit/test/test.log-expected.json b/filebeat/module/elasticsearch/audit/test/test.log-expected.json index c928ebf9390..6da7c2e1436 100644 --- a/filebeat/module/elasticsearch/audit/test/test.log-expected.json +++ b/filebeat/module/elasticsearch/audit/test/test.log-expected.json @@ -1,121 +1,138 @@ [ { - "@timestamp": "2018-06-19T05:16:15.549Z", - "elasticsearch.audit.event_type": "authentication_failed", - "elasticsearch.audit.layer": "rest", - "elasticsearch.audit.origin_address": "147.107.128.77", - "elasticsearch.audit.principal": "i030648", - "elasticsearch.audit.uri": "/_xpack/security/_authenticate", - "event.dataset": "elasticsearch.audit", - "fileset.module": "elasticsearch", - "fileset.name": "audit", - "input.type": "log", - "message": "[2018-06-19T05:16:15,549] [rest] [authentication_failed] origin_address=[147.107.128.77], principal=[i030648], uri=[/_xpack/security/_authenticate]", - "offset": 0, - "prospector.type": "log", + "@timestamp": "2018-06-19T05:16:15.549Z", + "elasticsearch.audit.event_type": "authentication_failed", + "elasticsearch.audit.layer": "rest", + "elasticsearch.audit.origin_address": "147.107.128.77", + "elasticsearch.audit.principal": "i030648", + "elasticsearch.audit.uri": "/_xpack/security/_authenticate", + "event.dataset": "elasticsearch.audit", + "fileset.module": "elasticsearch", + "fileset.name": "audit", + "input.type": "log", + "message": "[2018-06-19T05:16:15,549] [rest] [authentication_failed] origin_address=[147.107.128.77], principal=[i030648], uri=[/_xpack/security/_authenticate]", + "offset": 0, + "prospector.type": "log", "service.name": "elasticsearch" - }, + }, { - "@timestamp": "2018-06-19T05:07:52.304Z", - "elasticsearch.audit.event_type": "authentication_failed", - "elasticsearch.audit.layer": "rest", - "elasticsearch.audit.origin_address": "172.22.0.3", - "elasticsearch.audit.principal": "rado", - "elasticsearch.audit.uri": "/_xpack/security/_authenticate", - "elasticsearch.node.name": "v_VJhjV", - "event.dataset": "elasticsearch.audit", - "fileset.module": "elasticsearch", - "fileset.name": "audit", - "input.type": "log", - "message": "[2018-06-19T05:07:52,304] [v_VJhjV] [rest] [authentication_failed]\torigin_address=[172.22.0.3], principal=[rado], uri=[/_xpack/security/_authenticate]", - "offset": 155, - "prospector.type": "log", + "@timestamp": "2018-06-19T05:07:52.304Z", + "elasticsearch.audit.event_type": "authentication_failed", + "elasticsearch.audit.layer": "rest", + "elasticsearch.audit.origin_address": "172.22.0.3", + "elasticsearch.audit.principal": "rado", + "elasticsearch.audit.uri": "/_xpack/security/_authenticate", + "elasticsearch.node.name": "v_VJhjV", + "event.dataset": "elasticsearch.audit", + "fileset.module": "elasticsearch", + "fileset.name": "audit", + "input.type": "log", + "message": "[2018-06-19T05:07:52,304] [v_VJhjV] [rest] [authentication_failed]\torigin_address=[172.22.0.3], principal=[rado], uri=[/_xpack/security/_authenticate]", + "offset": 155, + "prospector.type": "log", "service.name": "elasticsearch" - }, + }, { - "@timestamp": "2018-06-19T05:00:15.778Z", - "elasticsearch.audit.action": "indices:data/read/scroll/clear", - "elasticsearch.audit.event_type": "access_granted", - "elasticsearch.audit.layer": "transport", - "elasticsearch.audit.origin_address": "192.168.1.165", - "elasticsearch.audit.origin_type": "local_node", - "elasticsearch.audit.principal": "_xpack_security", - "elasticsearch.audit.request": "ClearScrollRequest", - "event.dataset": "elasticsearch.audit", - "fileset.module": "elasticsearch", - "fileset.name": "audit", - "input.type": "log", - "message": "[2018-06-19T05:00:15,778] [transport] [access_granted] origin_type=[local_node], origin_address=[192.168.1.165], principal=[_xpack_security], action=[indices:data/read/scroll/clear], request=[ClearScrollRequest]", - "offset": 306, - "prospector.type": "log", + "@timestamp": "2018-06-19T05:00:15.778Z", + "elasticsearch.audit.action": "indices:data/read/scroll/clear", + "elasticsearch.audit.event_type": "access_granted", + "elasticsearch.audit.layer": "transport", + "elasticsearch.audit.origin_address": "192.168.1.165", + "elasticsearch.audit.origin_type": "local_node", + "elasticsearch.audit.principal": "_xpack_security", + "elasticsearch.audit.request": "ClearScrollRequest", + "event.dataset": "elasticsearch.audit", + "fileset.module": "elasticsearch", + "fileset.name": "audit", + "input.type": "log", + "message": "[2018-06-19T05:00:15,778] [transport] [access_granted] origin_type=[local_node], origin_address=[192.168.1.165], principal=[_xpack_security], action=[indices:data/read/scroll/clear], request=[ClearScrollRequest]", + "offset": 306, + "prospector.type": "log", "service.name": "elasticsearch" - }, + }, { - "@timestamp": "2018-06-19T05:07:45.544Z", - "elasticsearch.audit.event_type": "anonymous_access_denied", - "elasticsearch.audit.layer": "rest", - "elasticsearch.audit.origin_address": "172.22.0.3", - "elasticsearch.audit.uri": "/_xpack/security/_authenticate", - "elasticsearch.node.name": "v_VJhjV", - "event.dataset": "elasticsearch.audit", - "fileset.module": "elasticsearch", - "fileset.name": "audit", - "input.type": "log", - "message": "[2018-06-19T05:07:45,544] [v_VJhjV] [rest] [anonymous_access_denied]\torigin_address=[172.22.0.3], uri=[/_xpack/security/_authenticate]", - "offset": 519, - "prospector.type": "log", + "@timestamp": "2018-06-19T05:07:45.544Z", + "elasticsearch.audit.event_type": "anonymous_access_denied", + "elasticsearch.audit.layer": "rest", + "elasticsearch.audit.origin_address": "172.22.0.3", + "elasticsearch.audit.uri": "/_xpack/security/_authenticate", + "elasticsearch.node.name": "v_VJhjV", + "event.dataset": "elasticsearch.audit", + "fileset.module": "elasticsearch", + "fileset.name": "audit", + "input.type": "log", + "message": "[2018-06-19T05:07:45,544] [v_VJhjV] [rest] [anonymous_access_denied]\torigin_address=[172.22.0.3], uri=[/_xpack/security/_authenticate]", + "offset": 519, + "prospector.type": "log", "service.name": "elasticsearch" - }, + }, { - "@timestamp": "2018-06-19T05:26:27.268Z", - "elasticsearch.audit.event_type": "authentication_failed", - "elasticsearch.audit.layer": "rest", - "elasticsearch.audit.origin_address": "147.107.128.77", - "elasticsearch.audit.principal": "N078801", - "elasticsearch.audit.uri": "/_xpack/security/_authenticate", - "event.dataset": "elasticsearch.audit", - "fileset.module": "elasticsearch", - "fileset.name": "audit", - "input.type": "log", - "message": "[2018-06-19T05:26:27,268] [rest] [authentication_failed]\torigin_address=[147.107.128.77], principal=[N078801], uri=[/_xpack/security/_authenticate]", - "offset": 654, - "prospector.type": "log", + "@timestamp": "2018-06-19T05:26:27.268Z", + "elasticsearch.audit.event_type": "authentication_failed", + "elasticsearch.audit.layer": "rest", + "elasticsearch.audit.origin_address": "147.107.128.77", + "elasticsearch.audit.principal": "N078801", + "elasticsearch.audit.uri": "/_xpack/security/_authenticate", + "event.dataset": "elasticsearch.audit", + "fileset.module": "elasticsearch", + "fileset.name": "audit", + "input.type": "log", + "message": "[2018-06-19T05:26:27,268] [rest] [authentication_failed]\torigin_address=[147.107.128.77], principal=[N078801], uri=[/_xpack/security/_authenticate]", + "offset": 654, + "prospector.type": "log", "service.name": "elasticsearch" - }, + }, { - "@timestamp": "2018-06-19T05:55:26.898Z", - "elasticsearch.audit.action": "cluster:monitor/main", - "elasticsearch.audit.event_type": "access_denied", - "elasticsearch.audit.layer": "transport", - "elasticsearch.audit.origin_address": "147.107.128.77", - "elasticsearch.audit.origin_type": "rest", - "elasticsearch.audit.principal": "_anonymous", - "elasticsearch.audit.request": "MainRequest", - "event.dataset": "elasticsearch.audit", - "fileset.module": "elasticsearch", - "fileset.name": "audit", - "input.type": "log", - "message": "[2018-06-19T05:55:26,898] [transport] [access_denied]\torigin_type=[rest], origin_address=[147.107.128.77], principal=[_anonymous], action=[cluster:monitor/main], request=[MainRequest]", - "offset": 802, - "prospector.type": "log", + "@timestamp": "2018-06-19T05:55:26.898Z", + "elasticsearch.audit.action": "cluster:monitor/main", + "elasticsearch.audit.event_type": "access_denied", + "elasticsearch.audit.layer": "transport", + "elasticsearch.audit.origin_address": "147.107.128.77", + "elasticsearch.audit.origin_type": "rest", + "elasticsearch.audit.principal": "_anonymous", + "elasticsearch.audit.request": "MainRequest", + "event.dataset": "elasticsearch.audit", + "fileset.module": "elasticsearch", + "fileset.name": "audit", + "input.type": "log", + "message": "[2018-06-19T05:55:26,898] [transport] [access_denied]\torigin_type=[rest], origin_address=[147.107.128.77], principal=[_anonymous], action=[cluster:monitor/main], request=[MainRequest]", + "offset": 802, + "prospector.type": "log", "service.name": "elasticsearch" - }, + }, { - "@timestamp": "2018-06-19T05:24:15.190Z", - "elasticsearch.audit.event_type": "authentication_failed", - "elasticsearch.audit.layer": "rest", - "elasticsearch.audit.origin_address": "172.18.0.3", - "elasticsearch.audit.principal": "elastic", - "elasticsearch.audit.request_body": "body", - "elasticsearch.audit.uri": "/_nodes?filter_path=nodes.*.version%2Cnodes.*.http.publish_address%2Cnodes.*.ip", - "elasticsearch.node.name": "v_VJhjV", - "event.dataset": "elasticsearch.audit", - "fileset.module": "elasticsearch", - "fileset.name": "audit", - "input.type": "log", - "message": "[2018-06-19T05:24:15,190] [v_VJhjV] [rest] [authentication_failed]\torigin_address=[172.18.0.3], principal=[elastic], uri=[/_nodes?filter_path=nodes.*.version%2Cnodes.*.http.publish_address%2Cnodes.*.ip], request_body=[body]", - "offset": 986, - "prospector.type": "log", + "@timestamp": "2018-06-19T05:24:15.190Z", + "elasticsearch.audit.event_type": "authentication_failed", + "elasticsearch.audit.layer": "rest", + "elasticsearch.audit.origin_address": "172.18.0.3", + "elasticsearch.audit.principal": "elastic", + "elasticsearch.audit.request_body": "body", + "elasticsearch.audit.uri": "/_nodes?filter_path=nodes.*.version%2Cnodes.*.http.publish_address%2Cnodes.*.ip", + "elasticsearch.node.name": "v_VJhjV", + "event.dataset": "elasticsearch.audit", + "fileset.module": "elasticsearch", + "fileset.name": "audit", + "input.type": "log", + "message": "[2018-06-19T05:24:15,190] [v_VJhjV] [rest] [authentication_failed]\torigin_address=[172.18.0.3], principal=[elastic], uri=[/_nodes?filter_path=nodes.*.version%2Cnodes.*.http.publish_address%2Cnodes.*.ip], request_body=[body]", + "offset": 986, + "prospector.type": "log", + "service.name": "elasticsearch" + }, + { + "@timestamp": "2019-01-08T14:15:02.011Z", + "elasticsearch.audit.event_type": "access_granted", + "elasticsearch.audit.layer": "transport", + "elasticsearch.audit.origin_address": "192.168.2.1", + "elasticsearch.audit.origin_type": "transport", + "elasticsearch.audit.principal": "username", + "elasticsearch.node.name": "NodeName-0", + "event.dataset": "elasticsearch.audit", + "fileset.module": "elasticsearch", + "fileset.name": "audit", + "input.type": "log", + "message": "[2019-01-08T14:15:02,011] [NodeName-0] [transport] [access_granted] origin_type=[transport], origin_address=[192.168.2.1], principal=[username], realm=[active_directory], roles=[kibana_user,my_custom_role_1,foo_reader], action=[indices:data/read/search[free_context]], indices=[foo-2019.01.04,foo-2019.01.03,foo-2019.01.06,foo-2019.01.05,foo-2019.01.08,servicelog-2019.01.07], request=[SearchFreeContextRequest]", + "offset": 1210, + "prospector.type": "log", "service.name": "elasticsearch" } -] +] \ No newline at end of file From 745beb706a6e225d52b6bd5fb0f1351bd8a1ece1 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Thu, 31 Jan 2019 08:07:11 -0800 Subject: [PATCH 3/3] Remove CHANGELOG entries not from this PR --- CHANGELOG.next.asciidoc | 3 --- 1 file changed, 3 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 2c09659629a..13990683dec 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -41,9 +41,6 @@ https://github.com/elastic/beats/compare/1035569addc4a3b29ffa14f8a08c27c1ace16ef - Fix bad bytes count in `docker` input when filtering by stream. {pull}10211[10211] - Add `convert_timezone` option to Logstash module to convert dates to UTC. {issue}9756[9756] {pull}9797[9797] - Add `convert_timezone` option to Elasticsearch module to convert dates to UTC. {issue}9756[9756] {pull}9761[9761] -- Support IPv6 addresses with zone id in IIS ingest pipeline. - {issue}9836[9836] error log: {pull}9869[9869], access log: {pull}9955[9955]. -- Support haproxy log lines without captured headers. {issue}9463[9463] {pull}9958[9958] - Make elasticsearch/audit fileset be more lenient in parsing node name. {issue}10035[10035] {pull}10135[10135] *Heartbeat*