Remove convert_timezone option from filebeat system module (elastic…

…#12410) Timezone is now obtained from the system where filebeat is running, and timezone conversion when parsing the timestamp is always done. This timezone information is included in the `event.timezone` field, it can be overriden with the `add_fields` processor, or removed with the `remove_fields` processor.
andrewvc · Jun 12, 2019 · 338fabb · 338fabb
1 parent a23a8b7
commit 338fabb
Show file tree

Hide file tree

Showing 13 changed files with 20 additions and 54 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -26,6 +26,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Rename many `traefik.access.*` fields to map to ECS. {pull}9005[9005]
 - Fix parsing of GC entries in elasticsearch server log. {issue}9513[9513] {pull}9810[9810]
 - Add read_buffer configuration option. {pull}11739[11739]
+- `convert_timezone` option is removed and locale is always added to the event so timezone is used when parsing the timestamp, this behaviour can be overriden with processors. {pull}12410[12410]
 
 *Heartbeat*
 

diff --git a/filebeat/docs/include/timezone-support.asciidoc b/filebeat/docs/include/timezone-support.asciidoc
@@ -0,0 +1,17 @@
+[float]
+==== Timezone support
+
+This module parses logs that don't contain timezone information. For these logs,
+Filebeat reads the local timezone and uses it when parsing to convert the
+timestamp to UTC. The timezone to be used for parsing is included in the event
+in the `event.timezone` field.
+
+To disable this conversion, the `event.timezone` field can be removed with
+the `drop_fields` processor.
+
+If logs are originated from systems or applications with a different timezone to
+the local one, the `event.timezone` field can be overwritten with the original
+timezone using the `add_fields` processor.
+
+See <<filtering-and-enhancing-data>> for information about specifying
+processors in your config.
diff --git a/filebeat/docs/modules/system.asciidoc b/filebeat/docs/modules/system.asciidoc
@@ -69,14 +69,12 @@ include::../include/config-option-intro.asciidoc[]
 
 include::../include/var-paths.asciidoc[]
 
-include::../include/var-convert-timezone.asciidoc[]
-
 [float]
 ==== `auth` fileset settings
 
 include::../include/var-paths.asciidoc[]
 
-include::../include/var-convert-timezone.asciidoc[]
+include::../include/timezone-support.asciidoc[]
 
 :has-dashboards!:
 

diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml
@@ -21,9 +21,6 @@ filebeat.modules:
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
 
-    # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1.
-    #var.convert_timezone: false
-
     # Input configuration (advanced). Any input configuration option
     # can be added under this section.
     #input:
@@ -36,9 +33,6 @@ filebeat.modules:
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
 
-    # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1.
-    #var.convert_timezone: false
-
     # Input configuration (advanced). Any input configuration option
     # can be added under this section.
     #input:

diff --git a/filebeat/module/system/_meta/config.reference.yml b/filebeat/module/system/_meta/config.reference.yml
@@ -7,9 +7,6 @@
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
 
-    # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1.
-    #var.convert_timezone: false
-
     # Input configuration (advanced). Any input configuration option
     # can be added under this section.
     #input:
@@ -22,9 +19,6 @@
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
 
-    # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1.
-    #var.convert_timezone: false
-
     # Input configuration (advanced). Any input configuration option
     # can be added under this section.
     #input:
diff --git a/filebeat/module/system/_meta/config.yml b/filebeat/module/system/_meta/config.yml
@@ -7,16 +7,10 @@
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
 
-    # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1.
-    #var.convert_timezone: false
-
   # Authorization logs
   auth:
     enabled: true
 
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
-
-    # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1.
-    #var.convert_timezone: false
diff --git a/filebeat/module/system/_meta/docs.asciidoc b/filebeat/module/system/_meta/docs.asciidoc
@@ -64,14 +64,12 @@ include::../include/config-option-intro.asciidoc[]
 
 include::../include/var-paths.asciidoc[]
 
-include::../include/var-convert-timezone.asciidoc[]
-
 [float]
 ==== `auth` fileset settings
 
 include::../include/var-paths.asciidoc[]
 
-include::../include/var-convert-timezone.asciidoc[]
+include::../include/timezone-support.asciidoc[]
 
 :has-dashboards!:
 

diff --git a/filebeat/module/system/auth/config/auth.yml b/filebeat/module/system/auth/config/auth.yml
@@ -7,7 +7,5 @@ exclude_files: [".gz$"]
 multiline:
   pattern: "^\\s"
   match: after
-{{ if .convert_timezone }}
 processors:
 - add_locale: ~
-{{ end }}
diff --git a/filebeat/module/system/auth/manifest.yml b/filebeat/module/system/auth/manifest.yml
@@ -10,13 +10,6 @@ var:
       # ssh logs to files
       - /var/log/secure.log*
     os.windows: []
-  - name: convert_timezone
-    default: false
-    # if ES < 6.1.0, this flag switches to false automatically when evaluating the
-    # pipeline
-    min_elasticsearch_version:
-      version: 6.1.0
-      value: false
 
 ingest_pipeline: ingest/pipeline.json
 input: config/auth.yml
diff --git a/filebeat/module/system/syslog/config/syslog.yml b/filebeat/module/system/syslog/config/syslog.yml
@@ -7,7 +7,5 @@ exclude_files: [".gz$"]
 multiline:
   pattern: "^\\s"
   match: after
-{{ if .convert_timezone }}
 processors:
 - add_locale: ~
-{{ end }}
diff --git a/filebeat/module/system/syslog/manifest.yml b/filebeat/module/system/syslog/manifest.yml
@@ -8,13 +8,6 @@ var:
     os.darwin:
       - /var/log/system.log*
     os.windows: []
-  - name: convert_timezone
-    default: false
-    # if ES < 6.1.0, this flag switches to false automatically when evaluating the
-    # pipeline
-    min_elasticsearch_version:
-      version: 6.1.0
-      value: false
 
 ingest_pipeline: ingest/pipeline.json
 input: config/syslog.yml
diff --git a/filebeat/modules.d/system.yml.disabled b/filebeat/modules.d/system.yml.disabled
@@ -10,16 +10,10 @@
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
 
-    # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1.
-    #var.convert_timezone: false
-
   # Authorization logs
   auth:
     enabled: true
 
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
-
-    # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1.
-    #var.convert_timezone: false
diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml
@@ -21,9 +21,6 @@ filebeat.modules:
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
 
-    # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1.
-    #var.convert_timezone: false
-
     # Input configuration (advanced). Any input configuration option
     # can be added under this section.
     #input:
@@ -36,9 +33,6 @@ filebeat.modules:
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
 
-    # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1.
-    #var.convert_timezone: false
-
     # Input configuration (advanced). Any input configuration option
     # can be added under this section.
     #input: