diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index eaaf196880ef..40584019faa3 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -216,6 +216,7 @@ https://github.com/elastic/beats/compare/v6.2.3...master[Check the HEAD diff] - Add Audit log fileset to the Elasticsearch module. {pull}7365[7365] - Add Slow log fileset to the Elasticsearch module. {pull}7473[7473] - Add deprecation fileset to the Elasticsearch module. {pull}7474[7474] +- Add `config_timezone` option to Kafka module to convert dates to UTC. {issue}7546[7546] {pull}7578[7578] *Heartbeat* diff --git a/filebeat/docs/modules/kafka.asciidoc b/filebeat/docs/modules/kafka.asciidoc index a6b903d4c95f..b6d33ff95311 100644 --- a/filebeat/docs/modules/kafka.asciidoc +++ b/filebeat/docs/modules/kafka.asciidoc @@ -63,6 +63,8 @@ include::../include/config-option-intro.asciidoc[] include::../include/var-paths.asciidoc[] +include::../include/var-convert-timezone.asciidoc[] + [float] === Fields diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml index e78587bbd173..8d787d0f8afc 100644 --- a/filebeat/filebeat.reference.yml +++ b/filebeat/filebeat.reference.yml @@ -194,6 +194,8 @@ filebeat.modules: # Filebeat will choose the paths depending on your OS. #var.paths: + # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1. + #var.convert_timezone: false #------------------------------- kibana Module ------------------------------- - module: kibana diff --git a/filebeat/module/kafka/_meta/config.yml b/filebeat/module/kafka/_meta/config.yml index fe568e8e5621..9193a87a5fc9 100644 --- a/filebeat/module/kafka/_meta/config.yml +++ b/filebeat/module/kafka/_meta/config.yml @@ -11,3 +11,5 @@ # Filebeat will choose the paths depending on your OS. #var.paths: + # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1. + #var.convert_timezone: false diff --git a/filebeat/module/kafka/_meta/docs.asciidoc b/filebeat/module/kafka/_meta/docs.asciidoc index bdce41e96262..737a381be698 100644 --- a/filebeat/module/kafka/_meta/docs.asciidoc +++ b/filebeat/module/kafka/_meta/docs.asciidoc @@ -57,3 +57,5 @@ include::../include/config-option-intro.asciidoc[] ==== `log` fileset settings include::../include/var-paths.asciidoc[] + +include::../include/var-convert-timezone.asciidoc[] diff --git a/filebeat/module/kafka/log/config/log.yml b/filebeat/module/kafka/log/config/log.yml index 98c684ef52f6..2aec4b7b218e 100644 --- a/filebeat/module/kafka/log/config/log.yml +++ b/filebeat/module/kafka/log/config/log.yml @@ -8,3 +8,7 @@ multiline: pattern: '^\[' negate: true match: after +{{ if .convert_timezone }} +processors: +- add_locale: ~ +{{ end }} diff --git a/filebeat/module/kafka/log/ingest/pipeline.json b/filebeat/module/kafka/log/ingest/pipeline.json index 9ba9a2480640..6e9407e62a9f 100644 --- a/filebeat/module/kafka/log/ingest/pipeline.json +++ b/filebeat/module/kafka/log/ingest/pipeline.json @@ -52,7 +52,9 @@ "date": { "field": "kafka.log.timestamp", "target_field": "@timestamp", - "formats": ["yyyy-MM-dd HH:mm:ss,SSS"] + "formats": ["yyyy-MM-dd HH:mm:ss,SSS"], + {< if .convert_timezone >}"timezone": "{{ beat.timezone }}",{< end >} + "ignore_failure": true } }, {"remove": {"field": "kafka.log.timestamp" }} diff --git a/filebeat/module/kafka/log/manifest.yml b/filebeat/module/kafka/log/manifest.yml index 97ea201ec89d..bdf24847b008 100644 --- a/filebeat/module/kafka/log/manifest.yml +++ b/filebeat/module/kafka/log/manifest.yml @@ -9,6 +9,13 @@ var: - "{{.kafka_home}}/logs/server.log*" - "{{.kafka_home}}/logs/state-change.log*" - "{{.kafka_home}}/logs/kafka-*.log*" + - name: convert_timezone + default: false + # if ES < 6.1.0, this flag switches to false automatically when evaluating the + # pipeline + min_elasticsearch_version: + version: 6.1.0 + value: false ingest_pipeline: ingest/pipeline.json input: config/log.yml diff --git a/filebeat/modules.d/kafka.yml.disabled b/filebeat/modules.d/kafka.yml.disabled index fe568e8e5621..9193a87a5fc9 100644 --- a/filebeat/modules.d/kafka.yml.disabled +++ b/filebeat/modules.d/kafka.yml.disabled @@ -11,3 +11,5 @@ # Filebeat will choose the paths depending on your OS. #var.paths: + # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1. + #var.convert_timezone: false