Teach elasticsearch/audit fileset to handle timestamps correctly (#…

…15942) * Add samples from ES 7.1.1 audit log * When JSON logs contain @timestamp and TZ is detected, use detected TZ * Add CHANGELOG entry * Remove redundant guard * Revert unrelated change * s/teach/improve/ in changelog entry
elastic · Feb 7, 2020 · 356100f · 356100f
1 parent 1446025
commit 356100f
Show file tree

Hide file tree

Showing 6 changed files with 113 additions and 10 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -96,6 +96,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Prevent Elasticsearch from spewing log warnings about redundant wildcards when setting up ingest pipelines for the `elasticsearch` module. {issue}15840[15840] {pull}15900[15900]
 - Fix mapping error for cloudtrail additionalEventData field {pull}16088[16088]
 - Fix a connection error in httpjson input. {pull}16123[16123]
+- Improve `elasticsearch/audit` fileset to handle timestamps correctly. {pull}15942[15942]
 
 *Heartbeat*
 

diff --git a/filebeat/module/elasticsearch/audit/config/audit.yml b/filebeat/module/elasticsearch/audit/config/audit.yml
@@ -6,5 +6,4 @@ paths:
 exclude_files: [".gz$"]
 
 processors:
-# Locale for timezone is only needed in non-json logs
-- add_locale.when.not.regexp.message: "^{"
+- add_locale: ~
diff --git a/filebeat/module/elasticsearch/audit/ingest/pipeline-json.json b/filebeat/module/elasticsearch/audit/ingest/pipeline-json.json
@@ -16,7 +16,23 @@
  "remove": {
  "field": "elasticsearch.audit.type",
  "ignore_missing": true
-
+ }
+ },
+ {
+ "date": {
+ "if": "ctx.elasticsearch.audit['@timestamp'] != null && ctx.event.timezone != null",
+ "field": "elasticsearch.audit.@timestamp",
+ "target_field": "elasticsearch.audit.@timestamp",
+ "formats": [
+ "yyyy-MM-dd'T'HH:mm:ss,SSS"
+ ],
+ "timezone": "{{ event.timezone }}"
+ }
+ },
+ {
+ "remove": {
+ "if": "ctx.elasticsearch.audit['@timestamp'] == null && ctx.event.timezone != null",
+ "field": "event.timezone"
  }
  },
  {

diff --git a/filebeat/module/elasticsearch/audit/test/test-audit-711.log b/filebeat/module/elasticsearch/audit/test/test-audit-711.log
@@ -0,0 +1,3 @@
+{"@timestamp":"2019-09-05T14:02:37,921", "node.id":"UwRu4mReRtyJO1-FWAPvIQ", "event.type":"transport", "event.action":"authentication_success", "user.name":"_system", "origin.type":"local_node", "origin.address":"127.0.0.1:9300", "realm":"__fallback", "request.id":"474ZciqtQteOhjLO3OdZIw", "action":"indices:monitor/stats", "request.name":"IndicesStatsRequest"}
+{"@timestamp":"2020-01-29T09:41:10,856", "node.id":"DJKjhISiTzy-JY5nCU8h3Q", "event.type":"transport", "event.action":"access_granted", "user.name":"_xpack_security", "user.realm":"__attach", "user.roles":["superuser"], "origin.type":"local_node", "origin.address":"127.0.0.1:9300", "request.id":"I9bQCw28Qfe4HWtIJHgoAg", "action":"cluster:admin/xpack/security/realm/cache/clear", "request.name":"ClearRealmCacheRequest"}
+{"@timestamp":"2020-01-29T09:41:10,859", "node.id":"DJKjhISiTzy-JY5nCU8h3Q", "event.type":"transport", "event.action":"access_granted", "user.name":"_xpack_security", "user.realm":"__attach", "user.roles":["superuser"], "origin.type":"local_node", "origin.address":"127.0.0.1:9300", "request.id":"I9bQCw28Qfe4HWtIJHgoAg", "action":"cluster:admin/xpack/security/realm/cache/clear[n]", "request.name":"Node"}
diff --git a/filebeat/module/elasticsearch/audit/test/test-audit-711.log-expected.json b/filebeat/module/elasticsearch/audit/test/test-audit-711.log-expected.json
@@ -0,0 +1,77 @@
+[
+ {
+ "@timestamp": "2019-09-05T16:02:37.921Z",
+ "elasticsearch.audit.action": "indices:monitor/stats",
+ "elasticsearch.audit.layer": "transport",
+ "elasticsearch.audit.origin.type": "local_node",
+ "elasticsearch.audit.realm": "__fallback",
+ "elasticsearch.audit.request.id": "474ZciqtQteOhjLO3OdZIw",
+ "elasticsearch.audit.request.name": "IndicesStatsRequest",
+ "elasticsearch.node.id": "UwRu4mReRtyJO1-FWAPvIQ",
+ "event.action": "authentication_success",
+ "event.dataset": "elasticsearch.audit",
+ "event.module": "elasticsearch",
+ "event.timezone": "-02:00",
+ "fileset.name": "audit",
+ "input.type": "log",
+ "log.offset": 0,
+ "message": "{\"@timestamp\":\"2019-09-05T14:02:37,921\", \"node.id\":\"UwRu4mReRtyJO1-FWAPvIQ\", \"event.type\":\"transport\", \"event.action\":\"authentication_success\", \"user.name\":\"_system\", \"origin.type\":\"local_node\", \"origin.address\":\"127.0.0.1:9300\", \"realm\":\"__fallback\", \"request.id\":\"474ZciqtQteOhjLO3OdZIw\", \"action\":\"indices:monitor/stats\", \"request.name\":\"IndicesStatsRequest\"}",
+ "service.type": "elasticsearch",
+ "source.address": "127.0.0.1:9300",
+ "source.ip": "127.0.0.1",
+ "source.port": 9300,
+ "user.name": "_system"
+ },
+ {
+ "@timestamp": "2020-01-29T11:41:10.856Z",
+ "elasticsearch.audit.action": "cluster:admin/xpack/security/realm/cache/clear",
+ "elasticsearch.audit.layer": "transport",
+ "elasticsearch.audit.origin.type": "local_node",
+ "elasticsearch.audit.request.id": "I9bQCw28Qfe4HWtIJHgoAg",
+ "elasticsearch.audit.request.name": "ClearRealmCacheRequest",
+ "elasticsearch.audit.user.realm": "__attach",
+ "elasticsearch.audit.user.roles": [
+ "superuser"
+ ],
+ "elasticsearch.node.id": "DJKjhISiTzy-JY5nCU8h3Q",
+ "event.action": "access_granted",
+ "event.dataset": "elasticsearch.audit",
+ "event.module": "elasticsearch",
+ "event.timezone": "-02:00",
+ "fileset.name": "audit",
+ "input.type": "log",
+ "log.offset": 363,
+ "message": "{\"@timestamp\":\"2020-01-29T09:41:10,856\", \"node.id\":\"DJKjhISiTzy-JY5nCU8h3Q\", \"event.type\":\"transport\", \"event.action\":\"access_granted\", \"user.name\":\"_xpack_security\", \"user.realm\":\"__attach\", \"user.roles\":[\"superuser\"], \"origin.type\":\"local_node\", \"origin.address\":\"127.0.0.1:9300\", \"request.id\":\"I9bQCw28Qfe4HWtIJHgoAg\", \"action\":\"cluster:admin/xpack/security/realm/cache/clear\", \"request.name\":\"ClearRealmCacheRequest\"}",
+ "service.type": "elasticsearch",
+ "source.address": "127.0.0.1:9300",
+ "source.ip": "127.0.0.1",
+ "source.port": 9300,
+ "user.name": "_xpack_security"
+ },
+ {
+ "@timestamp": "2020-01-29T11:41:10.859Z",
+ "elasticsearch.audit.action": "cluster:admin/xpack/security/realm/cache/clear[n]",
+ "elasticsearch.audit.layer": "transport",
+ "elasticsearch.audit.origin.type": "local_node",
+ "elasticsearch.audit.request.id": "I9bQCw28Qfe4HWtIJHgoAg",
+ "elasticsearch.audit.request.name": "Node",
+ "elasticsearch.audit.user.realm": "__attach",
+ "elasticsearch.audit.user.roles": [
+ "superuser"
+ ],
+ "elasticsearch.node.id": "DJKjhISiTzy-JY5nCU8h3Q",
+ "event.action": "access_granted",
+ "event.dataset": "elasticsearch.audit",
+ "event.module": "elasticsearch",
+ "event.timezone": "-02:00",
+ "fileset.name": "audit",
+ "input.type": "log",
+ "log.offset": 785,
+ "message": "{\"@timestamp\":\"2020-01-29T09:41:10,859\", \"node.id\":\"DJKjhISiTzy-JY5nCU8h3Q\", \"event.type\":\"transport\", \"event.action\":\"access_granted\", \"user.name\":\"_xpack_security\", \"user.realm\":\"__attach\", \"user.roles\":[\"superuser\"], \"origin.type\":\"local_node\", \"origin.address\":\"127.0.0.1:9300\", \"request.id\":\"I9bQCw28Qfe4HWtIJHgoAg\", \"action\":\"cluster:admin/xpack/security/realm/cache/clear[n]\", \"request.name\":\"Node\"}",
+ "service.type": "elasticsearch",
+ "source.address": "127.0.0.1:9300",
+ "source.ip": "127.0.0.1",
+ "source.port": 9300,
+ "user.name": "_xpack_security"
+ }
+]
diff --git a/filebeat/module/elasticsearch/audit/test/test-audit.log-expected.json b/filebeat/module/elasticsearch/audit/test/test-audit.log-expected.json
@@ -1,12 +1,13 @@
 [
  {
- "@timestamp": "2018-10-31T09:34:25.109Z",
+ "@timestamp": "2018-10-31T11:34:25.109Z",
  "elasticsearch.audit.layer": "rest",
  "elasticsearch.audit.origin.type": "rest",
  "elasticsearch.node.id": "DSiWcTyeThWtUXLB9J0BMw",
  "event.action": "authentication_failed",
  "event.dataset": "elasticsearch.audit",
  "event.module": "elasticsearch",
+ "event.timezone": "-02:00",
  "fileset.name": "audit",
  "input.type": "log",
  "log.offset": 0,
@@ -19,13 +20,14 @@
  "user.name": "elastic"
  },
  {
- "@timestamp": "2018-10-31T09:34:25.207Z",
+ "@timestamp": "2018-10-31T11:34:25.207Z",
  "elasticsearch.audit.layer": "rest",
  "elasticsearch.audit.origin.type": "rest",
  "elasticsearch.node.id": "DSiWcTyeThWtUXLB9J0BMw",
  "event.action": "authentication_failed",
  "event.dataset": "elasticsearch.audit",
  "event.module": "elasticsearch",
+ "event.timezone": "-02:00",
  "fileset.name": "audit",
  "input.type": "log",
  "log.offset": 274,
@@ -38,7 +40,7 @@
  "user.name": "elastic"
  },
  {
- "@timestamp": "2018-10-31T09:35:11.428Z",
+ "@timestamp": "2018-10-31T11:35:11.428Z",
  "elasticsearch.audit.action": "cluster:admin/xpack/security/realm/cache/clear",
  "elasticsearch.audit.layer": "transport",
  "elasticsearch.audit.origin.type": "local_node",
@@ -51,6 +53,7 @@
  "event.action": "access_granted",
  "event.dataset": "elasticsearch.audit",
  "event.module": "elasticsearch",
+ "event.timezone": "-02:00",
  "fileset.name": "audit",
  "input.type": "log",
  "log.offset": 558,
@@ -62,7 +65,7 @@
  "user.name": "_xpack_security"
  },
  {
- "@timestamp": "2018-10-31T09:35:11.430Z",
+ "@timestamp": "2018-10-31T11:35:11.430Z",
  "elasticsearch.audit.action": "cluster:admin/xpack/security/realm/cache/clear[n]",
  "elasticsearch.audit.layer": "transport",
  "elasticsearch.audit.origin.type": "local_node",
@@ -75,6 +78,7 @@
  "event.action": "access_granted",
  "event.dataset": "elasticsearch.audit",
  "event.module": "elasticsearch",
+ "event.timezone": "-02:00",
  "fileset.name": "audit",
  "input.type": "log",
  "log.offset": 941,
@@ -86,7 +90,7 @@
  "user.name": "_xpack_security"
  },
  {
- "@timestamp": "2018-10-31T09:35:12.303Z",
+ "@timestamp": "2018-10-31T11:35:12.303Z",
  "elasticsearch.audit.action": "cluster:admin/xpack/security/user/change_password",
  "elasticsearch.audit.layer": "transport",
  "elasticsearch.audit.origin.type": "rest",
@@ -99,6 +103,7 @@
  "event.action": "access_granted",
  "event.dataset": "elasticsearch.audit",
  "event.module": "elasticsearch",
+ "event.timezone": "-02:00",
  "fileset.name": "audit",
  "input.type": "log",
  "log.offset": 1309,
@@ -110,7 +115,7 @@
  "user.name": "elastic"
  },
  {
- "@timestamp": "2018-10-31T09:35:12.314Z",
+ "@timestamp": "2018-10-31T11:35:12.314Z",
  "elasticsearch.audit.action": "indices:admin/create",
  "elasticsearch.audit.indices": [
  ".security-6"
@@ -126,6 +131,7 @@
  "event.action": "access_granted",
  "event.dataset": "elasticsearch.audit",
  "event.module": "elasticsearch",
+ "event.timezone": "-02:00",
  "fileset.name": "audit",
  "input.type": "log",
  "log.offset": 1676,
@@ -137,7 +143,7 @@
  "user.name": "_xpack_security"
  },
  {
- "@timestamp": "2019-01-27T20:15:10.380Z",
+ "@timestamp": "2019-01-27T22:15:10.380Z",
  "elasticsearch.audit.layer": "rest",
  "elasticsearch.audit.origin.type": "rest",
  "elasticsearch.audit.realm": "default_file",
@@ -147,6 +153,7 @@
  "event.action": "authentication_success",
  "event.dataset": "elasticsearch.audit",
  "event.module": "elasticsearch",
+ "event.timezone": "-02:00",
  "fileset.name": "audit",
  "http.request.body.content": "\n{\n \"query\" : {\n \"term\" : { \"user\" : \"kimchy\" }\n }\n}\n",
  "http.request.method": "GET",