diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 628ce5512a9..5236c247f7c 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -60,6 +60,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix s3 input hanging with GetObjectRequest API call by adding context_timeout config. {issue}15502[15502] {pull}15590[15590] - Add shared_credential_file to cloudtrail config {issue}15652[15652] {pull}15656[15656] - Fix typos in zeek notice fileset config file. {issue}15764[15764] {pull}15765[15765] +- Improve `elasticsearch/audit` fileset to handle timestamps correctly. {pull}15942[15942] *Heartbeat* diff --git a/filebeat/module/elasticsearch/audit/config/audit.yml b/filebeat/module/elasticsearch/audit/config/audit.yml index e8c035e32cc..d96242ac040 100644 --- a/filebeat/module/elasticsearch/audit/config/audit.yml +++ b/filebeat/module/elasticsearch/audit/config/audit.yml @@ -6,5 +6,4 @@ paths: exclude_files: [".gz$"] processors: -# Locale for timezone is only needed in non-json logs -- add_locale.when.not.regexp.message: "^{" +- add_locale: ~ diff --git a/filebeat/module/elasticsearch/audit/ingest/pipeline-json.json b/filebeat/module/elasticsearch/audit/ingest/pipeline-json.json index f0cf48a19ff..1a1dcdcb794 100644 --- a/filebeat/module/elasticsearch/audit/ingest/pipeline-json.json +++ b/filebeat/module/elasticsearch/audit/ingest/pipeline-json.json @@ -16,7 +16,23 @@ "remove": { "field": "elasticsearch.audit.type", "ignore_missing": true - + } + }, + { + "date": { + "if": "ctx.elasticsearch.audit['@timestamp'] != null && ctx.event.timezone != null", + "field": "elasticsearch.audit.@timestamp", + "target_field": "elasticsearch.audit.@timestamp", + "formats": [ + "yyyy-MM-dd'T'HH:mm:ss,SSS" + ], + "timezone": "{{ event.timezone }}" + } + }, + { + "remove": { + "if": "ctx.elasticsearch.audit['@timestamp'] == null && ctx.event.timezone != null", + "field": "event.timezone" } }, { diff --git a/filebeat/module/elasticsearch/audit/test/test-audit-711.log b/filebeat/module/elasticsearch/audit/test/test-audit-711.log new file mode 100644 index 00000000000..bfc437d3fe4 --- /dev/null +++ b/filebeat/module/elasticsearch/audit/test/test-audit-711.log @@ -0,0 +1,3 @@ +{"@timestamp":"2019-09-05T14:02:37,921", "node.id":"UwRu4mReRtyJO1-FWAPvIQ", "event.type":"transport", "event.action":"authentication_success", "user.name":"_system", "origin.type":"local_node", "origin.address":"127.0.0.1:9300", "realm":"__fallback", "request.id":"474ZciqtQteOhjLO3OdZIw", "action":"indices:monitor/stats", "request.name":"IndicesStatsRequest"} +{"@timestamp":"2020-01-29T09:41:10,856", "node.id":"DJKjhISiTzy-JY5nCU8h3Q", "event.type":"transport", "event.action":"access_granted", "user.name":"_xpack_security", "user.realm":"__attach", "user.roles":["superuser"], "origin.type":"local_node", "origin.address":"127.0.0.1:9300", "request.id":"I9bQCw28Qfe4HWtIJHgoAg", "action":"cluster:admin/xpack/security/realm/cache/clear", "request.name":"ClearRealmCacheRequest"} +{"@timestamp":"2020-01-29T09:41:10,859", "node.id":"DJKjhISiTzy-JY5nCU8h3Q", "event.type":"transport", "event.action":"access_granted", "user.name":"_xpack_security", "user.realm":"__attach", "user.roles":["superuser"], "origin.type":"local_node", "origin.address":"127.0.0.1:9300", "request.id":"I9bQCw28Qfe4HWtIJHgoAg", "action":"cluster:admin/xpack/security/realm/cache/clear[n]", "request.name":"Node"} diff --git a/filebeat/module/elasticsearch/audit/test/test-audit-711.log-expected.json b/filebeat/module/elasticsearch/audit/test/test-audit-711.log-expected.json new file mode 100644 index 00000000000..01cf1d9b0d9 --- /dev/null +++ b/filebeat/module/elasticsearch/audit/test/test-audit-711.log-expected.json @@ -0,0 +1,77 @@ +[ + { + "@timestamp": "2019-09-05T16:02:37.921Z", + "elasticsearch.audit.action": "indices:monitor/stats", + "elasticsearch.audit.layer": "transport", + "elasticsearch.audit.origin.type": "local_node", + "elasticsearch.audit.realm": "__fallback", + "elasticsearch.audit.request.id": "474ZciqtQteOhjLO3OdZIw", + "elasticsearch.audit.request.name": "IndicesStatsRequest", + "elasticsearch.node.id": "UwRu4mReRtyJO1-FWAPvIQ", + "event.action": "authentication_success", + "event.dataset": "elasticsearch.audit", + "event.module": "elasticsearch", + "event.timezone": "-02:00", + "fileset.name": "audit", + "input.type": "log", + "log.offset": 0, + "message": "{\"@timestamp\":\"2019-09-05T14:02:37,921\", \"node.id\":\"UwRu4mReRtyJO1-FWAPvIQ\", \"event.type\":\"transport\", \"event.action\":\"authentication_success\", \"user.name\":\"_system\", \"origin.type\":\"local_node\", \"origin.address\":\"127.0.0.1:9300\", \"realm\":\"__fallback\", \"request.id\":\"474ZciqtQteOhjLO3OdZIw\", \"action\":\"indices:monitor/stats\", \"request.name\":\"IndicesStatsRequest\"}", + "service.type": "elasticsearch", + "source.address": "127.0.0.1:9300", + "source.ip": "127.0.0.1", + "source.port": 9300, + "user.name": "_system" + }, + { + "@timestamp": "2020-01-29T11:41:10.856Z", + "elasticsearch.audit.action": "cluster:admin/xpack/security/realm/cache/clear", + "elasticsearch.audit.layer": "transport", + "elasticsearch.audit.origin.type": "local_node", + "elasticsearch.audit.request.id": "I9bQCw28Qfe4HWtIJHgoAg", + "elasticsearch.audit.request.name": "ClearRealmCacheRequest", + "elasticsearch.audit.user.realm": "__attach", + "elasticsearch.audit.user.roles": [ + "superuser" + ], + "elasticsearch.node.id": "DJKjhISiTzy-JY5nCU8h3Q", + "event.action": "access_granted", + "event.dataset": "elasticsearch.audit", + "event.module": "elasticsearch", + "event.timezone": "-02:00", + "fileset.name": "audit", + "input.type": "log", + "log.offset": 363, + "message": "{\"@timestamp\":\"2020-01-29T09:41:10,856\", \"node.id\":\"DJKjhISiTzy-JY5nCU8h3Q\", \"event.type\":\"transport\", \"event.action\":\"access_granted\", \"user.name\":\"_xpack_security\", \"user.realm\":\"__attach\", \"user.roles\":[\"superuser\"], \"origin.type\":\"local_node\", \"origin.address\":\"127.0.0.1:9300\", \"request.id\":\"I9bQCw28Qfe4HWtIJHgoAg\", \"action\":\"cluster:admin/xpack/security/realm/cache/clear\", \"request.name\":\"ClearRealmCacheRequest\"}", + "service.type": "elasticsearch", + "source.address": "127.0.0.1:9300", + "source.ip": "127.0.0.1", + "source.port": 9300, + "user.name": "_xpack_security" + }, + { + "@timestamp": "2020-01-29T11:41:10.859Z", + "elasticsearch.audit.action": "cluster:admin/xpack/security/realm/cache/clear[n]", + "elasticsearch.audit.layer": "transport", + "elasticsearch.audit.origin.type": "local_node", + "elasticsearch.audit.request.id": "I9bQCw28Qfe4HWtIJHgoAg", + "elasticsearch.audit.request.name": "Node", + "elasticsearch.audit.user.realm": "__attach", + "elasticsearch.audit.user.roles": [ + "superuser" + ], + "elasticsearch.node.id": "DJKjhISiTzy-JY5nCU8h3Q", + "event.action": "access_granted", + "event.dataset": "elasticsearch.audit", + "event.module": "elasticsearch", + "event.timezone": "-02:00", + "fileset.name": "audit", + "input.type": "log", + "log.offset": 785, + "message": "{\"@timestamp\":\"2020-01-29T09:41:10,859\", \"node.id\":\"DJKjhISiTzy-JY5nCU8h3Q\", \"event.type\":\"transport\", \"event.action\":\"access_granted\", \"user.name\":\"_xpack_security\", \"user.realm\":\"__attach\", \"user.roles\":[\"superuser\"], \"origin.type\":\"local_node\", \"origin.address\":\"127.0.0.1:9300\", \"request.id\":\"I9bQCw28Qfe4HWtIJHgoAg\", \"action\":\"cluster:admin/xpack/security/realm/cache/clear[n]\", \"request.name\":\"Node\"}", + "service.type": "elasticsearch", + "source.address": "127.0.0.1:9300", + "source.ip": "127.0.0.1", + "source.port": 9300, + "user.name": "_xpack_security" + } +] \ No newline at end of file diff --git a/filebeat/module/elasticsearch/audit/test/test-audit.log-expected.json b/filebeat/module/elasticsearch/audit/test/test-audit.log-expected.json index 4155cfd829b..07667974cd5 100644 --- a/filebeat/module/elasticsearch/audit/test/test-audit.log-expected.json +++ b/filebeat/module/elasticsearch/audit/test/test-audit.log-expected.json @@ -1,12 +1,13 @@ [ { - "@timestamp": "2018-10-31T09:34:25.109Z", + "@timestamp": "2018-10-31T11:34:25.109Z", "elasticsearch.audit.layer": "rest", "elasticsearch.audit.origin.type": "rest", "elasticsearch.node.id": "DSiWcTyeThWtUXLB9J0BMw", "event.action": "authentication_failed", "event.dataset": "elasticsearch.audit", "event.module": "elasticsearch", + "event.timezone": "-02:00", "fileset.name": "audit", "input.type": "log", "log.offset": 0, @@ -19,13 +20,14 @@ "user.name": "elastic" }, { - "@timestamp": "2018-10-31T09:34:25.207Z", + "@timestamp": "2018-10-31T11:34:25.207Z", "elasticsearch.audit.layer": "rest", "elasticsearch.audit.origin.type": "rest", "elasticsearch.node.id": "DSiWcTyeThWtUXLB9J0BMw", "event.action": "authentication_failed", "event.dataset": "elasticsearch.audit", "event.module": "elasticsearch", + "event.timezone": "-02:00", "fileset.name": "audit", "input.type": "log", "log.offset": 274, @@ -38,7 +40,7 @@ "user.name": "elastic" }, { - "@timestamp": "2018-10-31T09:35:11.428Z", + "@timestamp": "2018-10-31T11:35:11.428Z", "elasticsearch.audit.action": "cluster:admin/xpack/security/realm/cache/clear", "elasticsearch.audit.layer": "transport", "elasticsearch.audit.origin.type": "local_node", @@ -51,6 +53,7 @@ "event.action": "access_granted", "event.dataset": "elasticsearch.audit", "event.module": "elasticsearch", + "event.timezone": "-02:00", "fileset.name": "audit", "input.type": "log", "log.offset": 558, @@ -62,7 +65,7 @@ "user.name": "_xpack_security" }, { - "@timestamp": "2018-10-31T09:35:11.430Z", + "@timestamp": "2018-10-31T11:35:11.430Z", "elasticsearch.audit.action": "cluster:admin/xpack/security/realm/cache/clear[n]", "elasticsearch.audit.layer": "transport", "elasticsearch.audit.origin.type": "local_node", @@ -75,6 +78,7 @@ "event.action": "access_granted", "event.dataset": "elasticsearch.audit", "event.module": "elasticsearch", + "event.timezone": "-02:00", "fileset.name": "audit", "input.type": "log", "log.offset": 941, @@ -86,7 +90,7 @@ "user.name": "_xpack_security" }, { - "@timestamp": "2018-10-31T09:35:12.303Z", + "@timestamp": "2018-10-31T11:35:12.303Z", "elasticsearch.audit.action": "cluster:admin/xpack/security/user/change_password", "elasticsearch.audit.layer": "transport", "elasticsearch.audit.origin.type": "rest", @@ -99,6 +103,7 @@ "event.action": "access_granted", "event.dataset": "elasticsearch.audit", "event.module": "elasticsearch", + "event.timezone": "-02:00", "fileset.name": "audit", "input.type": "log", "log.offset": 1309, @@ -110,7 +115,7 @@ "user.name": "elastic" }, { - "@timestamp": "2018-10-31T09:35:12.314Z", + "@timestamp": "2018-10-31T11:35:12.314Z", "elasticsearch.audit.action": "indices:admin/create", "elasticsearch.audit.indices": [ ".security-6" @@ -126,6 +131,7 @@ "event.action": "access_granted", "event.dataset": "elasticsearch.audit", "event.module": "elasticsearch", + "event.timezone": "-02:00", "fileset.name": "audit", "input.type": "log", "log.offset": 1676, @@ -137,7 +143,7 @@ "user.name": "_xpack_security" }, { - "@timestamp": "2019-01-27T20:15:10.380Z", + "@timestamp": "2019-01-27T22:15:10.380Z", "elasticsearch.audit.layer": "rest", "elasticsearch.audit.origin.type": "rest", "elasticsearch.audit.realm": "default_file", @@ -147,6 +153,7 @@ "event.action": "authentication_success", "event.dataset": "elasticsearch.audit", "event.module": "elasticsearch", + "event.timezone": "-02:00", "fileset.name": "audit", "http.request.body.content": "\n{\n \"query\" : {\n \"term\" : { \"user\" : \"kimchy\" }\n }\n}\n", "http.request.method": "GET",