diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index d913c8b29caf..7a5f1937f27a 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -51,6 +51,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d *Filebeat* +- Add `convert_timezone` option to Elasticsearch module to convert dates to UTC. {issue}9756[9756] {pull}9761[9761] + *Heartbeat* - Made monitors.d configuration part of the default config. {pull}9004[9004] diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml index ad23069eb165..51c8881e5ed9 100644 --- a/filebeat/filebeat.reference.yml +++ b/filebeat/filebeat.reference.yml @@ -92,6 +92,9 @@ filebeat.modules: # Filebeat will choose the paths depending on your OS. #var.paths: + # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1. + #var.convert_timezone: false + gc: enabled: true # Set custom paths for the log files. If left empty, @@ -104,18 +107,28 @@ filebeat.modules: # Filebeat will choose the paths depending on your OS. #var.paths: + # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1. + #var.convert_timezone: false + slowlog: enabled: true # Set custom paths for the log files. If left empty, # Filebeat will choose the paths depending on your OS. #var.paths: + # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1. + #var.convert_timezone: false + deprecation: enabled: true # Set custom paths for the log files. If left empty, # Filebeat will choose the paths depending on your OS. #var.paths: + # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1. + #var.convert_timezone: false + + #------------------------------- Haproxy Module ------------------------------- - module: haproxy # All logs diff --git a/filebeat/module/elasticsearch/_meta/config.yml b/filebeat/module/elasticsearch/_meta/config.yml index 0c2562f27969..3105d5803792 100644 --- a/filebeat/module/elasticsearch/_meta/config.yml +++ b/filebeat/module/elasticsearch/_meta/config.yml @@ -7,6 +7,9 @@ # Filebeat will choose the paths depending on your OS. #var.paths: + # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1. + #var.convert_timezone: false + gc: enabled: true # Set custom paths for the log files. If left empty, @@ -19,14 +22,24 @@ # Filebeat will choose the paths depending on your OS. #var.paths: + # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1. + #var.convert_timezone: false + slowlog: enabled: true # Set custom paths for the log files. If left empty, # Filebeat will choose the paths depending on your OS. #var.paths: + # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1. + #var.convert_timezone: false + deprecation: enabled: true # Set custom paths for the log files. If left empty, # Filebeat will choose the paths depending on your OS. #var.paths: + + # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1. + #var.convert_timezone: false + diff --git a/filebeat/module/elasticsearch/audit/config/audit.yml b/filebeat/module/elasticsearch/audit/config/audit.yml index 8d9139a7115f..a6dab3dc2a9e 100644 --- a/filebeat/module/elasticsearch/audit/config/audit.yml +++ b/filebeat/module/elasticsearch/audit/config/audit.yml @@ -5,6 +5,11 @@ paths: {{ end }} exclude_files: [".gz$"] +{{ if .convert_timezone }} +processors: +- add_locale: ~ +{{ end }} + fields: service.name: "elasticsearch" fields_under_root: true diff --git a/filebeat/module/elasticsearch/audit/ingest/pipeline.json b/filebeat/module/elasticsearch/audit/ingest/pipeline.json index 9c8323624015..d686ba846b2e 100644 --- a/filebeat/module/elasticsearch/audit/ingest/pipeline.json +++ b/filebeat/module/elasticsearch/audit/ingest/pipeline.json @@ -16,9 +16,19 @@ } }, { - "rename": { + "date": { "field": "elasticsearch.audit.timestamp", - "target_field": "@timestamp" + "target_field": "@timestamp", + "formats": [ + "ISO8601" + ], + {< if .convert_timezone >}"timezone": "{{ event.timezone }}",{< end >} + "ignore_failure": true + } + }, + { + "remove": { + "field": "elasticsearch.audit.timestamp" } } ], diff --git a/filebeat/module/elasticsearch/audit/manifest.yml b/filebeat/module/elasticsearch/audit/manifest.yml index 00298cdf9812..7ccb68c7e434 100644 --- a/filebeat/module/elasticsearch/audit/manifest.yml +++ b/filebeat/module/elasticsearch/audit/manifest.yml @@ -8,6 +8,13 @@ var: - /usr/local/var/lib/elasticsearch/*_access.log os.windows: - c:/ProgramData/Elastic/Elasticsearch/logs/*_access.log + - name: convert_timezone + default: false + # if ES < 6.1.0, this flag switches to false automatically when evaluating the + # pipeline + min_elasticsearch_version: + version: 6.1.0 + value: false ingest_pipeline: ingest/pipeline.json input: config/audit.yml diff --git a/filebeat/module/elasticsearch/audit/test/test.log-expected.json b/filebeat/module/elasticsearch/audit/test/test.log-expected.json index 45a7d55ac4a8..f4b880a84cf5 100644 --- a/filebeat/module/elasticsearch/audit/test/test.log-expected.json +++ b/filebeat/module/elasticsearch/audit/test/test.log-expected.json @@ -1,6 +1,6 @@ [ { - "@timestamp": "2018-06-19T05:16:15,549", + "@timestamp": "2018-06-19T05:16:15.549Z", "ecs.version": "1.0.0-beta2", "elasticsearch.audit.event_type": "authentication_failed", "elasticsearch.audit.layer": "rest", @@ -15,7 +15,7 @@ "service.name": "elasticsearch" }, { - "@timestamp": "2018-06-19T05:07:52,304", + "@timestamp": "2018-06-19T05:07:52.304Z", "ecs.version": "1.0.0-beta2", "elasticsearch.audit.event_type": "authentication_failed", "elasticsearch.audit.layer": "rest", @@ -31,7 +31,7 @@ "service.name": "elasticsearch" }, { - "@timestamp": "2018-06-19T05:00:15,778", + "@timestamp": "2018-06-19T05:00:15.778Z", "ecs.version": "1.0.0-beta2", "elasticsearch.audit.action": "indices:data/read/scroll/clear", "elasticsearch.audit.event_type": "access_granted", @@ -48,7 +48,7 @@ "service.name": "elasticsearch" }, { - "@timestamp": "2018-06-19T05:07:45,544", + "@timestamp": "2018-06-19T05:07:45.544Z", "ecs.version": "1.0.0-beta2", "elasticsearch.audit.event_type": "anonymous_access_denied", "elasticsearch.audit.layer": "rest", @@ -63,7 +63,7 @@ "service.name": "elasticsearch" }, { - "@timestamp": "2018-06-19T05:26:27,268", + "@timestamp": "2018-06-19T05:26:27.268Z", "ecs.version": "1.0.0-beta2", "elasticsearch.audit.event_type": "authentication_failed", "elasticsearch.audit.layer": "rest", @@ -78,7 +78,7 @@ "service.name": "elasticsearch" }, { - "@timestamp": "2018-06-19T05:55:26,898", + "@timestamp": "2018-06-19T05:55:26.898Z", "ecs.version": "1.0.0-beta2", "elasticsearch.audit.action": "cluster:monitor/main", "elasticsearch.audit.event_type": "access_denied", @@ -95,7 +95,7 @@ "service.name": "elasticsearch" }, { - "@timestamp": "2018-06-19T05:24:15,190", + "@timestamp": "2018-06-19T05:24:15.190Z", "ecs.version": "1.0.0-beta2", "elasticsearch.audit.event_type": "authentication_failed", "elasticsearch.audit.layer": "rest", diff --git a/filebeat/module/elasticsearch/deprecation/config/log.yml b/filebeat/module/elasticsearch/deprecation/config/log.yml index e49d0c86542c..aa265c0b1566 100644 --- a/filebeat/module/elasticsearch/deprecation/config/log.yml +++ b/filebeat/module/elasticsearch/deprecation/config/log.yml @@ -9,6 +9,11 @@ multiline: negate: true match: after +{{ if .convert_timezone }} +processors: +- add_locale: ~ +{{ end }} + fields: service.name: "elasticsearch" diff --git a/filebeat/module/elasticsearch/deprecation/ingest/pipeline.json b/filebeat/module/elasticsearch/deprecation/ingest/pipeline.json index 17bb9ddd409a..8f126f11e2f0 100755 --- a/filebeat/module/elasticsearch/deprecation/ingest/pipeline.json +++ b/filebeat/module/elasticsearch/deprecation/ingest/pipeline.json @@ -27,9 +27,19 @@ } }, { - "rename": { + "date": { "field": "timestamp", - "target_field": "@timestamp" + "target_field": "@timestamp", + "formats": [ + "ISO8601" + ], + {< if .convert_timezone >}"timezone": "{{ event.timezone }}",{< end >} + "ignore_failure": true + } + }, + { + "remove": { + "field": "timestamp" } } ] diff --git a/filebeat/module/elasticsearch/deprecation/manifest.yml b/filebeat/module/elasticsearch/deprecation/manifest.yml index f156c4c6e31b..28525f4f3da3 100644 --- a/filebeat/module/elasticsearch/deprecation/manifest.yml +++ b/filebeat/module/elasticsearch/deprecation/manifest.yml @@ -8,6 +8,13 @@ var: - /usr/local/var/lib/elasticsearch/*_deprecation.log os.windows: - c:/ProgramData/Elastic/Elasticsearch/logs/*_deprecation.log + - name: convert_timezone + default: false + # if ES < 6.1.0, this flag switches to false automatically when evaluating the + # pipeline + min_elasticsearch_version: + version: 6.1.0 + value: false ingest_pipeline: ingest/pipeline.json input: config/log.yml diff --git a/filebeat/module/elasticsearch/deprecation/test/elasticsearch_deprecation.log-expected.json b/filebeat/module/elasticsearch/deprecation/test/elasticsearch_deprecation.log-expected.json index b6bdb785ffd1..f209a7618e25 100644 --- a/filebeat/module/elasticsearch/deprecation/test/elasticsearch_deprecation.log-expected.json +++ b/filebeat/module/elasticsearch/deprecation/test/elasticsearch_deprecation.log-expected.json @@ -1,6 +1,6 @@ [ { - "@timestamp": "2018-04-23T16:40:13,737", + "@timestamp": "2018-04-23T16:40:13.737Z", "ecs.version": "1.0.0-beta2", "elasticsearch.server.component": "o.e.d.a.a.i.t.p.PutIndexTemplateRequest", "event.dataset": "deprecation", @@ -12,7 +12,7 @@ "service.name": "elasticsearch" }, { - "@timestamp": "2018-04-23T16:40:13,862", + "@timestamp": "2018-04-23T16:40:13.862Z", "ecs.version": "1.0.0-beta2", "elasticsearch.server.component": "o.e.d.a.a.i.t.p.PutIndexTemplateRequest", "event.dataset": "deprecation", @@ -24,7 +24,7 @@ "service.name": "elasticsearch" }, { - "@timestamp": "2018-04-23T16:40:14,792", + "@timestamp": "2018-04-23T16:40:14.792Z", "ecs.version": "1.0.0-beta2", "elasticsearch.server.component": "o.e.d.a.a.i.t.p.PutIndexTemplateRequest", "event.dataset": "deprecation", @@ -36,7 +36,7 @@ "service.name": "elasticsearch" }, { - "@timestamp": "2018-04-23T16:40:15,127", + "@timestamp": "2018-04-23T16:40:15.127Z", "ecs.version": "1.0.0-beta2", "elasticsearch.server.component": "o.e.d.a.a.i.t.p.PutIndexTemplateRequest", "event.dataset": "deprecation", diff --git a/filebeat/module/elasticsearch/deprecation/test/other_elasticsearch_deprecation.log-expected.json b/filebeat/module/elasticsearch/deprecation/test/other_elasticsearch_deprecation.log-expected.json index a36decc6f817..15c97d314159 100644 --- a/filebeat/module/elasticsearch/deprecation/test/other_elasticsearch_deprecation.log-expected.json +++ b/filebeat/module/elasticsearch/deprecation/test/other_elasticsearch_deprecation.log-expected.json @@ -1,6 +1,6 @@ [ { - "@timestamp": "2017-11-30T13:38:16,911", + "@timestamp": "2017-11-30T13:38:16.911Z", "ecs.version": "1.0.0-beta2", "elasticsearch.server.component": "o.e.d.c.ParseField", "event.dataset": "deprecation", @@ -12,7 +12,7 @@ "service.name": "elasticsearch" }, { - "@timestamp": "2017-11-30T13:38:16,941", + "@timestamp": "2017-11-30T13:38:16.941Z", "ecs.version": "1.0.0-beta2", "elasticsearch.server.component": "o.e.d.c.ParseField", "event.dataset": "deprecation", @@ -24,7 +24,7 @@ "service.name": "elasticsearch" }, { - "@timestamp": "2017-11-30T13:39:28,986", + "@timestamp": "2017-11-30T13:39:28.986Z", "ecs.version": "1.0.0-beta2", "elasticsearch.server.component": "o.e.d.i.m.UidFieldMapper", "event.dataset": "deprecation", @@ -36,7 +36,7 @@ "service.name": "elasticsearch" }, { - "@timestamp": "2017-11-30T13:39:36,339", + "@timestamp": "2017-11-30T13:39:36.339Z", "ecs.version": "1.0.0-beta2", "elasticsearch.server.component": "o.e.d.i.m.UidFieldMapper", "event.dataset": "deprecation", @@ -48,7 +48,7 @@ "service.name": "elasticsearch" }, { - "@timestamp": "2017-11-30T13:40:49,540", + "@timestamp": "2017-11-30T13:40:49.540Z", "ecs.version": "1.0.0-beta2", "elasticsearch.server.component": "o.e.d.i.m.UidFieldMapper", "event.dataset": "deprecation", @@ -60,7 +60,7 @@ "service.name": "elasticsearch" }, { - "@timestamp": "2017-11-30T14:08:37,413", + "@timestamp": "2017-11-30T14:08:37.413Z", "ecs.version": "1.0.0-beta2", "elasticsearch.server.component": "o.e.d.i.m.UidFieldMapper", "event.dataset": "deprecation", @@ -72,7 +72,7 @@ "service.name": "elasticsearch" }, { - "@timestamp": "2017-11-30T14:08:37,413", + "@timestamp": "2017-11-30T14:08:37.413Z", "ecs.version": "1.0.0-beta2", "elasticsearch.server.component": "o.e.d.i.m.UidFieldMapper", "event.dataset": "deprecation", @@ -84,7 +84,7 @@ "service.name": "elasticsearch" }, { - "@timestamp": "2017-11-30T14:08:46,006", + "@timestamp": "2017-11-30T14:08:46.006Z", "ecs.version": "1.0.0-beta2", "elasticsearch.server.component": "o.e.d.i.m.UidFieldMapper", "event.dataset": "deprecation", @@ -96,7 +96,7 @@ "service.name": "elasticsearch" }, { - "@timestamp": "2017-11-30T14:08:46,006", + "@timestamp": "2017-11-30T14:08:46.006Z", "ecs.version": "1.0.0-beta2", "elasticsearch.server.component": "o.e.d.i.m.UidFieldMapper", "event.dataset": "deprecation", @@ -108,7 +108,7 @@ "service.name": "elasticsearch" }, { - "@timestamp": "2017-12-01T14:05:54,017", + "@timestamp": "2017-12-01T14:05:54.017Z", "ecs.version": "1.0.0-beta2", "elasticsearch.server.component": "o.e.d.i.m.AllFieldMapper", "event.dataset": "deprecation", @@ -120,7 +120,7 @@ "service.name": "elasticsearch" }, { - "@timestamp": "2017-12-01T14:05:54,019", + "@timestamp": "2017-12-01T14:05:54.019Z", "ecs.version": "1.0.0-beta2", "elasticsearch.server.component": "o.e.d.i.m.AllFieldMapper", "event.dataset": "deprecation", @@ -132,7 +132,7 @@ "service.name": "elasticsearch" }, { - "@timestamp": "2017-12-01T14:06:52,059", + "@timestamp": "2017-12-01T14:06:52.059Z", "ecs.version": "1.0.0-beta2", "elasticsearch.server.component": "o.e.d.i.m.AllFieldMapper", "event.dataset": "deprecation", @@ -144,7 +144,7 @@ "service.name": "elasticsearch" }, { - "@timestamp": "2017-12-01T14:46:10,428", + "@timestamp": "2017-12-01T14:46:10.428Z", "ecs.version": "1.0.0-beta2", "elasticsearch.server.component": "o.e.d.s.a.InternalOrder$Parser", "event.dataset": "deprecation", @@ -156,7 +156,7 @@ "service.name": "elasticsearch" }, { - "@timestamp": "2017-12-04T16:17:18,271", + "@timestamp": "2017-12-04T16:17:18.271Z", "ecs.version": "1.0.0-beta2", "elasticsearch.server.component": "o.e.d.a.a.i.t.p.PutIndexTemplateRequest", "event.dataset": "deprecation", @@ -168,7 +168,7 @@ "service.name": "elasticsearch" }, { - "@timestamp": "2017-12-04T16:17:18,282", + "@timestamp": "2017-12-04T16:17:18.282Z", "ecs.version": "1.0.0-beta2", "elasticsearch.server.component": "o.e.d.i.m.MapperService", "event.dataset": "deprecation", @@ -180,7 +180,7 @@ "service.name": "elasticsearch" }, { - "@timestamp": "2017-12-04T16:20:43,248", + "@timestamp": "2017-12-04T16:20:43.248Z", "ecs.version": "1.0.0-beta2", "elasticsearch.server.component": "o.e.d.i.m.MapperService", "event.dataset": "deprecation", diff --git a/filebeat/module/elasticsearch/server/config/log.yml b/filebeat/module/elasticsearch/server/config/log.yml index 19a0839f107d..e65a0a0f787e 100644 --- a/filebeat/module/elasticsearch/server/config/log.yml +++ b/filebeat/module/elasticsearch/server/config/log.yml @@ -9,6 +9,11 @@ multiline: negate: true match: after +{{ if .convert_timezone }} +processors: +- add_locale: ~ +{{ end }} + fields: service.name: "elasticsearch" diff --git a/filebeat/module/elasticsearch/server/ingest/pipeline.json b/filebeat/module/elasticsearch/server/ingest/pipeline.json index b1c149b7ebda..d78707866fd9 100755 --- a/filebeat/module/elasticsearch/server/ingest/pipeline.json +++ b/filebeat/module/elasticsearch/server/ingest/pipeline.json @@ -57,9 +57,19 @@ } }, { - "rename": { + "date": { "field": "elasticsearch.server.timestamp", - "target_field": "@timestamp" + "target_field": "@timestamp", + "formats": [ + "ISO8601" + ], + {< if .convert_timezone >}"timezone": "{{ event.timezone }}",{< end >} + "ignore_failure": true + } + }, + { + "remove": { + "field": "elasticsearch.server.timestamp" } } ] diff --git a/filebeat/module/elasticsearch/server/manifest.yml b/filebeat/module/elasticsearch/server/manifest.yml index 8300280d8b11..cd9f3a7f1186 100644 --- a/filebeat/module/elasticsearch/server/manifest.yml +++ b/filebeat/module/elasticsearch/server/manifest.yml @@ -8,6 +8,13 @@ var: - /usr/local/var/lib/elasticsearch/*.log os.windows: - c:/ProgramData/Elastic/Elasticsearch/logs/*.log + - name: convert_timezone + default: false + # if ES < 6.1.0, this flag switches to false automatically when evaluating the + # pipeline + min_elasticsearch_version: + version: 6.1.0 + value: false ingest_pipeline: ingest/pipeline.json input: config/log.yml diff --git a/filebeat/module/elasticsearch/server/test/test.log-expected.json b/filebeat/module/elasticsearch/server/test/test.log-expected.json index a2e51a1069dd..3c603b8332f9 100644 --- a/filebeat/module/elasticsearch/server/test/test.log-expected.json +++ b/filebeat/module/elasticsearch/server/test/test.log-expected.json @@ -1,6 +1,6 @@ [ { - "@timestamp": "2018-05-17T08:29:12,177", + "@timestamp": "2018-05-17T08:29:12.177Z", "ecs.version": "1.0.0-beta2", "elasticsearch.index.name": "test-filebeat-modules", "elasticsearch.node.name": "vWNJsZ3", @@ -14,7 +14,7 @@ "service.name": "elasticsearch" }, { - "@timestamp": "2018-05-17T08:19:35,939", + "@timestamp": "2018-05-17T08:19:35.939Z", "ecs.version": "1.0.0-beta2", "elasticsearch.node.name": "", "elasticsearch.server.component": "o.e.n.Node", @@ -27,7 +27,7 @@ "service.name": "elasticsearch" }, { - "@timestamp": "2018-05-17T08:19:36,089", + "@timestamp": "2018-05-17T08:19:36.089Z", "ecs.version": "1.0.0-beta2", "elasticsearch.node.name": "vWNJsZ3", "elasticsearch.server.component": "o.e.e.NodeEnvironment", @@ -40,7 +40,7 @@ "service.name": "elasticsearch" }, { - "@timestamp": "2018-05-17T08:19:36,090", + "@timestamp": "2018-05-17T08:19:36.090Z", "ecs.version": "1.0.0-beta2", "elasticsearch.node.name": "vWNJsZ3", "elasticsearch.server.component": "o.e.e.NodeEnvironment", @@ -53,7 +53,7 @@ "service.name": "elasticsearch" }, { - "@timestamp": "2018-05-17T08:19:36,116", + "@timestamp": "2018-05-17T08:19:36.116Z", "ecs.version": "1.0.0-beta2", "elasticsearch.server.component": "o.e.n.Node", "event.dataset": "server", @@ -65,7 +65,7 @@ "service.name": "elasticsearch" }, { - "@timestamp": "2018-05-17T08:23:48,941", + "@timestamp": "2018-05-17T08:23:48.941Z", "ecs.version": "1.0.0-beta2", "elasticsearch.node.name": "vWNJsZ3", "elasticsearch.server.component": "o.e.c.r.a.DiskThresholdMonitor", @@ -78,7 +78,7 @@ "service.name": "elasticsearch" }, { - "@timestamp": "2018-05-17T08:29:09,245", + "@timestamp": "2018-05-17T08:29:09.245Z", "ecs.version": "1.0.0-beta2", "elasticsearch.index.name": "filebeat-test-input", "elasticsearch.node.name": "vWNJsZ3", @@ -92,7 +92,7 @@ "service.name": "elasticsearch" }, { - "@timestamp": "2018-05-17T08:29:09,576", + "@timestamp": "2018-05-17T08:29:09.576Z", "ecs.version": "1.0.0-beta2", "elasticsearch.index.id": "aOGgDwbURfCV57AScqbCgw", "elasticsearch.index.name": "filebeat-test-input", @@ -107,7 +107,7 @@ "service.name": "elasticsearch" }, { - "@timestamp": "2018-07-09T12:47:33,959", + "@timestamp": "2018-07-09T12:47:33.959Z", "ecs.version": "1.0.0-beta2", "elasticsearch.index.id": "3tWftqb4RLKdyCAga9syGA", "elasticsearch.index.name": ".kibana", @@ -122,7 +122,7 @@ "service.name": "elasticsearch" }, { - "@timestamp": "2018-05-17T08:29:25,598", + "@timestamp": "2018-05-17T08:29:25.598Z", "ecs.version": "1.0.0-beta2", "elasticsearch.node.name": "vWNJsZ3", "elasticsearch.server.component": "o.e.n.Node", @@ -135,7 +135,7 @@ "service.name": "elasticsearch" }, { - "@timestamp": "2018-05-17T08:29:25,612", + "@timestamp": "2018-05-17T08:29:25.612Z", "ecs.version": "1.0.0-beta2", "elasticsearch.node.name": "vWNJsZ3", "elasticsearch.server.component": "o.e.n.Node", @@ -148,7 +148,7 @@ "service.name": "elasticsearch" }, { - "@timestamp": "2018-07-03T11:45:48,548", + "@timestamp": "2018-07-03T11:45:48.548Z", "ecs.version": "1.0.0-beta2", "elasticsearch.node.name": "srvmulpvlsk252_md", "elasticsearch.server.component": "o.e.d.z.ZenDiscovery", @@ -161,7 +161,7 @@ "service.name": "elasticsearch" }, { - "@timestamp": "2018-07-03T11:45:48,548", + "@timestamp": "2018-07-03T11:45:48.548Z", "ecs.version": "1.0.0-beta2", "elasticsearch.node.name": "srvmulpvlsk252_md", "elasticsearch.server.component": "o.e.d.z.ZenDiscovery", @@ -177,7 +177,7 @@ "service.name": "elasticsearch" }, { - "@timestamp": "2018-07-03T11:45:52,666", + "@timestamp": "2018-07-03T11:45:52.666Z", "ecs.version": "1.0.0-beta2", "elasticsearch.server.component": "r.suppressed", "event.dataset": "server", @@ -192,7 +192,7 @@ "service.name": "elasticsearch" }, { - "@timestamp": "2018-07-03T11:48:02,552", + "@timestamp": "2018-07-03T11:48:02.552Z", "ecs.version": "1.0.0-beta2", "elasticsearch.server.component": "r.suppressed", "event.dataset": "server", @@ -207,7 +207,7 @@ "service.name": "elasticsearch" }, { - "@timestamp": "2018-07-03T11:45:27,896", + "@timestamp": "2018-07-03T11:45:27.896Z", "ecs.version": "1.0.0-beta2", "elasticsearch.node.name": "srvmulpvlsk252_md", "elasticsearch.server.component": "o.e.m.j.JvmGcMonitorService", @@ -225,7 +225,7 @@ "service.name": "elasticsearch" }, { - "@timestamp": "2018-07-03T11:45:45,604", + "@timestamp": "2018-07-03T11:45:45.604Z", "ecs.version": "1.0.0-beta2", "elasticsearch.node.name": "srvmulpvlsk252_md", "elasticsearch.server.component": "o.e.m.j.JvmGcMonitorService", @@ -241,7 +241,7 @@ "service.name": "elasticsearch" }, { - "@timestamp": "2018-07-03T11:48:02,541", + "@timestamp": "2018-07-03T11:48:02.541Z", "ecs.version": "1.0.0-beta2", "elasticsearch.node.name": "srvmulpvlsk252_md", "elasticsearch.server.component": "o.e.a.b.TransportShardBulkAction", @@ -254,7 +254,7 @@ "service.name": "elasticsearch" }, { - "@timestamp": "2018-07-03T20:10:07,376", + "@timestamp": "2018-07-03T20:10:07.376Z", "ecs.version": "1.0.0-beta2", "elasticsearch.node.name": "srvmulpvlsk252_md", "elasticsearch.server.component": "o.e.x.m.MonitoringService", diff --git a/filebeat/module/elasticsearch/slowlog/config/slowlog.yml b/filebeat/module/elasticsearch/slowlog/config/slowlog.yml index c65f00f5ae0b..afb6b902a010 100644 --- a/filebeat/module/elasticsearch/slowlog/config/slowlog.yml +++ b/filebeat/module/elasticsearch/slowlog/config/slowlog.yml @@ -10,6 +10,11 @@ multiline: negate: true match: after +{{ if .convert_timezone }} +processors: +- add_locale: ~ +{{ end }} + fields: service.name: "elasticsearch" fields_under_root: true diff --git a/filebeat/module/elasticsearch/slowlog/ingest/pipeline.json b/filebeat/module/elasticsearch/slowlog/ingest/pipeline.json index 0fb4b7ab7560..1d2f4d788704 100644 --- a/filebeat/module/elasticsearch/slowlog/ingest/pipeline.json +++ b/filebeat/module/elasticsearch/slowlog/ingest/pipeline.json @@ -20,12 +20,22 @@ } }, { - "rename": { + "date": { "field": "elasticsearch.slowlog.timestamp", - "target_field": "@timestamp" + "target_field": "@timestamp", + "formats": [ + "ISO8601" + ], + {< if .convert_timezone >}"timezone": "{{ event.timezone }}",{< end >} + "ignore_failure": true + } + }, + { + "remove": { + "field": "elasticsearch.slowlog.timestamp" } } - ], + ], "on_failure" : [{ "set" : { "field" : "error.message", diff --git a/filebeat/module/elasticsearch/slowlog/manifest.yml b/filebeat/module/elasticsearch/slowlog/manifest.yml index 5fc6306b9fe2..904b5db5ef63 100644 --- a/filebeat/module/elasticsearch/slowlog/manifest.yml +++ b/filebeat/module/elasticsearch/slowlog/manifest.yml @@ -11,6 +11,13 @@ var: os.windows: - c:/ProgramData/Elastic/Elasticsearch/logs/*_index_search_slowlog.log - c:/ProgramData/Elastic/Elasticsearch/logs/*_index_indexing_slowlog.log + - name: convert_timezone + default: false + # if ES < 6.1.0, this flag switches to false automatically when evaluating the + # pipeline + min_elasticsearch_version: + version: 6.1.0 + value: false ingest_pipeline: ingest/pipeline.json input: config/slowlog.yml diff --git a/filebeat/module/elasticsearch/slowlog/test/test.log-expected.json b/filebeat/module/elasticsearch/slowlog/test/test.log-expected.json index a58b3acdadc0..efe533256be8 100644 --- a/filebeat/module/elasticsearch/slowlog/test/test.log-expected.json +++ b/filebeat/module/elasticsearch/slowlog/test/test.log-expected.json @@ -1,6 +1,6 @@ [ { - "@timestamp": "2018-06-29T10:06:14,933", + "@timestamp": "2018-06-29T10:06:14.933Z", "ecs.version": "1.0.0-beta2", "elasticsearch.index.name": "metricbeat-6.3.0-2018.06.26", "elasticsearch.node.name": "v_VJhjV", @@ -23,7 +23,7 @@ "service.name": "elasticsearch" }, { - "@timestamp": "2018-06-29T10:06:14,943", + "@timestamp": "2018-06-29T10:06:14.943Z", "ecs.version": "1.0.0-beta2", "elasticsearch.index.name": "metricbeat-6.3.0-2018.06.26", "elasticsearch.node.name": "v_VJhjV", @@ -46,7 +46,7 @@ "service.name": "elasticsearch" }, { - "@timestamp": "2018-06-29T09:01:01,821", + "@timestamp": "2018-06-29T09:01:01.821Z", "ecs.version": "1.0.0-beta2", "elasticsearch.index.name": "metricbeat-6.3.0-2018.06.26", "elasticsearch.node.name": "v_VJhjV", @@ -69,7 +69,7 @@ "service.name": "elasticsearch" }, { - "@timestamp": "2018-06-29T09:01:01,827", + "@timestamp": "2018-06-29T09:01:01.827Z", "ecs.version": "1.0.0-beta2", "elasticsearch.index.name": "metricbeat-6.3.0-2018.06.26", "elasticsearch.node.name": "v_VJhjV", @@ -92,7 +92,7 @@ "service.name": "elasticsearch" }, { - "@timestamp": "2018-07-04T13:48:07,452", + "@timestamp": "2018-07-04T13:48:07.452Z", "ecs.version": "1.0.0-beta2", "elasticsearch.index.id": "VLKxBLvUSYuIMKzpacGjRg", "elasticsearch.index.name": "metricbeat-6.3.0-2018.07.04", @@ -113,7 +113,7 @@ "service.name": "elasticsearch" }, { - "@timestamp": "2018-07-04T21:51:30,411", + "@timestamp": "2018-07-04T21:51:30.411Z", "ecs.version": "1.0.0-beta2", "elasticsearch.index.id": "VLKxBLvUSYuIMKzpacGjRg", "elasticsearch.index.name": "metricbeat-6.3.0-2018.07.04", diff --git a/filebeat/modules.d/elasticsearch.yml.disabled b/filebeat/modules.d/elasticsearch.yml.disabled index 4db2df4eaea3..986a7a881371 100644 --- a/filebeat/modules.d/elasticsearch.yml.disabled +++ b/filebeat/modules.d/elasticsearch.yml.disabled @@ -10,6 +10,9 @@ # Filebeat will choose the paths depending on your OS. #var.paths: + # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1. + #var.convert_timezone: false + gc: enabled: true # Set custom paths for the log files. If left empty, @@ -22,14 +25,24 @@ # Filebeat will choose the paths depending on your OS. #var.paths: + # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1. + #var.convert_timezone: false + slowlog: enabled: true # Set custom paths for the log files. If left empty, # Filebeat will choose the paths depending on your OS. #var.paths: + # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1. + #var.convert_timezone: false + deprecation: enabled: true # Set custom paths for the log files. If left empty, # Filebeat will choose the paths depending on your OS. #var.paths: + + # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1. + #var.convert_timezone: false + diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index 2bbbe8711fd7..b32717a9e2b1 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -92,6 +92,9 @@ filebeat.modules: # Filebeat will choose the paths depending on your OS. #var.paths: + # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1. + #var.convert_timezone: false + gc: enabled: true # Set custom paths for the log files. If left empty, @@ -104,18 +107,28 @@ filebeat.modules: # Filebeat will choose the paths depending on your OS. #var.paths: + # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1. + #var.convert_timezone: false + slowlog: enabled: true # Set custom paths for the log files. If left empty, # Filebeat will choose the paths depending on your OS. #var.paths: + # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1. + #var.convert_timezone: false + deprecation: enabled: true # Set custom paths for the log files. If left empty, # Filebeat will choose the paths depending on your OS. #var.paths: + # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1. + #var.convert_timezone: false + + #------------------------------- Haproxy Module ------------------------------- - module: haproxy # All logs