Filebeat: Add option to convert kafka module timezones to UTC (#7578)

elastic · Jul 16, 2018 · fda0ec0 · fda0ec0
1 parent fa3c844
commit fda0ec0
Show file tree

Hide file tree

Showing 9 changed files with 25 additions and 1 deletion.
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
@@ -216,6 +216,7 @@ https://github.com/elastic/beats/compare/v6.2.3...master[Check the HEAD diff]
 - Add Audit log fileset to the Elasticsearch module. {pull}7365[7365]
 - Add Slow log fileset to the Elasticsearch module. {pull}7473[7473]
 - Add deprecation fileset to the Elasticsearch module. {pull}7474[7474]
+- Add `config_timezone` option to Kafka module to convert dates to UTC. {issue}7546[7546] {pull}7578[7578]
 
 *Heartbeat*
 

diff --git a/filebeat/docs/modules/kafka.asciidoc b/filebeat/docs/modules/kafka.asciidoc
@@ -63,6 +63,8 @@ include::../include/config-option-intro.asciidoc[]
 
 include::../include/var-paths.asciidoc[]
 
+include::../include/var-convert-timezone.asciidoc[]
+
 
 [float]
 === Fields

diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml
@@ -194,6 +194,8 @@ filebeat.modules:
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
 
+    # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1.
+    #var.convert_timezone: false
 
 #------------------------------- kibana Module -------------------------------
 - module: kibana

diff --git a/filebeat/module/kafka/_meta/config.yml b/filebeat/module/kafka/_meta/config.yml
@@ -11,3 +11,5 @@
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
 
+    # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1.
+    #var.convert_timezone: false
diff --git a/filebeat/module/kafka/_meta/docs.asciidoc b/filebeat/module/kafka/_meta/docs.asciidoc
@@ -57,3 +57,5 @@ include::../include/config-option-intro.asciidoc[]
 ==== `log` fileset settings
 
 include::../include/var-paths.asciidoc[]
+
+include::../include/var-convert-timezone.asciidoc[]
diff --git a/filebeat/module/kafka/log/config/log.yml b/filebeat/module/kafka/log/config/log.yml
@@ -8,3 +8,7 @@ multiline:
   pattern: '^\['
   negate: true
   match: after
+{{ if .convert_timezone }}
+processors:
+- add_locale: ~
+{{ end }}
diff --git a/filebeat/module/kafka/log/ingest/pipeline.json b/filebeat/module/kafka/log/ingest/pipeline.json
@@ -52,7 +52,9 @@
       "date": {
         "field": "kafka.log.timestamp",
         "target_field": "@timestamp",
-        "formats": ["yyyy-MM-dd HH:mm:ss,SSS"]
+        "formats": ["yyyy-MM-dd HH:mm:ss,SSS"],
+        {< if .convert_timezone >}"timezone": "{{ beat.timezone }}",{< end >}
+        "ignore_failure": true
       }
     },
     {"remove": {"field": "kafka.log.timestamp" }}

diff --git a/filebeat/module/kafka/log/manifest.yml b/filebeat/module/kafka/log/manifest.yml
@@ -9,6 +9,13 @@ var:
       - "{{.kafka_home}}/logs/server.log*"
       - "{{.kafka_home}}/logs/state-change.log*"
       - "{{.kafka_home}}/logs/kafka-*.log*"
+  - name: convert_timezone
+    default: false
+    # if ES < 6.1.0, this flag switches to false automatically when evaluating the
+    # pipeline
+    min_elasticsearch_version:
+      version: 6.1.0
+      value: false
 
 ingest_pipeline: ingest/pipeline.json
 input: config/log.yml
diff --git a/filebeat/modules.d/kafka.yml.disabled b/filebeat/modules.d/kafka.yml.disabled
@@ -11,3 +11,5 @@
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
 
+    # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1.
+    #var.convert_timezone: false
-Original file line number
+Diff line change
@@ Expand Up / @@ -63,6 +63,8 @@ include::../include/config-option-intro.asciidoc[] @@
     include::../include/var-paths.asciidoc[]
+    include::../include/var-convert-timezone.asciidoc[]
     [float]
     === Fields
@@ Expand Down @@
Original file line number	Diff line number	Diff line change
Expand Up		@@ -57,3 +57,5 @@ include::../include/config-option-intro.asciidoc[]
		==== `log` fileset settings

		include::../include/var-paths.asciidoc[]

		include::../include/var-convert-timezone.asciidoc[]