Convert timezones if event.timezone is set (elastic#12311)

Follow up of elastic#12253
exekias · May 29, 2019 · 3af1f29 · 3af1f29
1 parent d1446d4
commit 3af1f29
Show file tree

Hide file tree

Showing 14 changed files with 129 additions and 26 deletions.
diff --git a/filebeat/module/elasticsearch/audit/ingest/pipeline.json b/filebeat/module/elasticsearch/audit/ingest/pipeline.json
@@ -37,10 +37,18 @@
                 "formats": [
                     "ISO8601"
                 ],
-                {< if .convert_timezone >}"timezone": "{{ event.timezone }}",{< end >}
                 "ignore_failure": true
             }
         },
+        {
+            "date": {
+                "if": "ctx.event.timezone != null",
+                "field": "@timestamp",
+                "formats": ["ISO8601"],
+                "timezone": "{{ event.timezone }}",
+                "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}]
+            }
+        },
         {
             "remove": {
                 "field": "elasticsearch.audit.@timestamp"

diff --git a/filebeat/module/elasticsearch/deprecation/ingest/pipeline.json b/filebeat/module/elasticsearch/deprecation/ingest/pipeline.json
@@ -37,10 +37,18 @@
                 "formats": [
                     "ISO8601"
                 ],
-                {< if .convert_timezone >}"timezone": "{{ event.timezone }}",{< end >}
                 "ignore_failure": true
             }
         },
+        {
+            "date": {
+                "if": "ctx.event.timezone != null",
+                "field": "@timestamp",
+                "formats": ["ISO8601"],
+                "timezone": "{{ event.timezone }}",
+                "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}]
+            }
+        },
         {
             "remove": {
                 "field": "elasticsearch.deprecation.timestamp"

diff --git a/filebeat/module/elasticsearch/server/ingest/pipeline.json b/filebeat/module/elasticsearch/server/ingest/pipeline.json
@@ -61,10 +61,18 @@
                 "formats": [
                     "ISO8601"
                 ],
-                {< if .convert_timezone >}"timezone": "{{ event.timezone }}",{< end >}
                 "ignore_failure": true
             }
         },
+        {
+            "date": {
+                "if": "ctx.event.timezone != null",
+                "field": "@timestamp",
+                "formats": ["ISO8601"],
+                "timezone": "{{ event.timezone }}",
+                "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}]
+            }
+        },
         {
             "remove": {
                 "field": "elasticsearch.server.timestamp"

diff --git a/filebeat/module/elasticsearch/slowlog/ingest/pipeline.json b/filebeat/module/elasticsearch/slowlog/ingest/pipeline.json
@@ -37,10 +37,18 @@
                 "formats": [
                     "ISO8601"
                 ],
-                {< if .convert_timezone >}"timezone": "{{ event.timezone }}",{< end >}
                 "ignore_failure": true
             }
         },
+        {
+            "date": {
+                "if": "ctx.event.timezone != null",
+                "field": "@timestamp",
+                "formats": ["ISO8601"],
+                "timezone": "{{ event.timezone }}",
+                "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}]
+            }
+        },
         {
             "remove": {
                 "field": "elasticsearch.slowlog.timestamp"

diff --git a/filebeat/module/kafka/log/ingest/pipeline.json b/filebeat/module/kafka/log/ingest/pipeline.json
@@ -62,10 +62,18 @@
         "field": "kafka.log.timestamp",
         "target_field": "@timestamp",
         "formats": ["yyyy-MM-dd HH:mm:ss,SSS"],
-        {< if .convert_timezone >}"timezone": "{{ event.timezone }}",{< end >}
         "ignore_failure": true
       }
     },
+    {
+      "date": {
+        "if": "ctx.event.timezone != null",
+        "field": "@timestamp",
+        "formats": ["ISO8601"],
+        "timezone": "{{ event.timezone }}",
+          "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}]
+        }
+    },
     {"remove": {"field": "kafka.log.timestamp" }}
   ],
   "on_failure" : [{

diff --git a/filebeat/module/logstash/log/ingest/pipeline-plain.json b/filebeat/module/logstash/log/ingest/pipeline-plain.json
@@ -35,10 +35,18 @@
                 "formats": [
                   "ISO8601"
               ],
-              {< if .convert_timezone >}"timezone": "{{ event.timezone }}",{< end >}
               "ignore_failure": true
             }
         },
+        {
+            "date": {
+                "if": "ctx.event.timezone != null",
+                "field": "@timestamp",
+                "formats": ["ISO8601"],
+                "timezone": "{{ event.timezone }}",
+                "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}]
+            }
+        },
         {
             "remove": {
                 "field": "logstash.log.timestamp"

diff --git a/filebeat/module/logstash/slowlog/ingest/pipeline-plain.json b/filebeat/module/logstash/slowlog/ingest/pipeline-plain.json
@@ -50,10 +50,18 @@
                 "formats": [
                   "ISO8601"
               ],
-              {< if .convert_timezone >}"timezone": "{{ event.timezone }}",{< end >}
               "ignore_failure": true
             }
         },
+        {
+            "date": {
+                "if": "ctx.event.timezone != null",
+                "field": "@timestamp",
+                "formats": ["ISO8601"],
+                "timezone": "{{ event.timezone }}",
+                "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}]
+            }
+        },
         {
             "remove": {
                 "field": [

diff --git a/filebeat/module/nginx/access/ingest/default.json b/filebeat/module/nginx/access/ingest/default.json
@@ -89,10 +89,18 @@
                 "formats": [
                     "dd/MMM/yyyy:H:m:s Z"
                 ],
-                {< if .convert_timezone >}"timezone": "{{ event.timezone }}",{< end >}
                 "ignore_failure": true
             }
         },
+        {
+            "date": {
+                "if": "ctx.event.timezone != null",
+                "field": "@timestamp",
+                "formats": ["ISO8601"],
+                "timezone": "{{ event.timezone }}",
+                "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}]
+            }
+        },
         {
             "remove": {
                 "field": "nginx.access.time"

diff --git a/filebeat/module/nginx/error/ingest/pipeline.json b/filebeat/module/nginx/error/ingest/pipeline.json
@@ -18,9 +18,16 @@
       "field": "nginx.error.time",
       "target_field": "@timestamp",
       "formats": ["yyyy/MM/dd H:m:s"],
-      {< if .convert_timezone >}"timezone": "{{ event.timezone }}",{< end >}
       "ignore_failure": true
     }
+  }, {
+    "date": {
+      "if": "ctx.event.timezone != null",
+      "field": "@timestamp",
+      "formats": ["ISO8601"],
+      "timezone": "{{ event.timezone }}",
+      "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}]
+    }
   }, {
     "remove": {
       "field": "nginx.error.time"

diff --git a/filebeat/module/system/auth/ingest/pipeline.json b/filebeat/module/system/auth/ingest/pipeline.json
@@ -46,10 +46,18 @@
                     "MMM  d HH:mm:ss",
                     "MMM dd HH:mm:ss"
                 ],
-                {< if .convert_timezone >}"timezone": "{{ event.timezone }}",{< end >}
                 "ignore_failure": true
             }
         },
+        {
+            "date": {
+                "if": "ctx.event.timezone != null",
+                "field": "@timestamp",
+                "formats": ["ISO8601"],
+                "timezone": "{{ event.timezone }}",
+                "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}]
+            }
+        },
         {
             "remove": {
                 "field": "system.auth.timestamp"

diff --git a/x-pack/filebeat/module/cisco/asa/ingest/pipeline.yml b/x-pack/filebeat/module/cisco/asa/ingest/pipeline.yml
@@ -22,9 +22,6 @@ processors:
  - date:
      field: "_temp_.raw_date"
      ignore_failure: true
-     {< if .convert_timezone >}
-     timezone: "{{ event.timezone }}"
-     {< end >}
      formats: 
       - "MMM  d HH:mm:ss"
       - "MMM dd HH:mm:ss"
@@ -42,6 +39,12 @@ processors:
       - "MMM dd yyyy HH:mm:ss z"
       - "EEE MMM  d yyyy HH:mm:ss z"
       - "EEE MMM dd yyyy HH:mm:ss z"
+ - date:
+     if: "ctx.event.timezone != null"
+     field: "@timestamp"
+     formats: ["ISO8601"]
+     timezone: "{{ event.timezone }}"
+     on_failure: [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}]
 
 #
 # Set log.level

diff --git a/x-pack/filebeat/module/iptables/log/ingest/pipeline.json b/x-pack/filebeat/module/iptables/log/ingest/pipeline.json
@@ -64,13 +64,21 @@
                 "date": {
                     "field": "iptables.raw_date",
                     "ignore_failure": true,
-                    {< if .convert_timezone >}"timezone": "{{ event.timezone }}",{< end >}
                     "formats": [
                         "MMM  d HH:mm:ss",
                         "MMM dd HH:mm:ss"
                     ]
                 }
             },
+            {
+                "date": {
+                    "if": "ctx.event.timezone != null",
+                    "field": "@timestamp",
+                    "formats": ["ISO8601"],
+                    "timezone": "{{ event.timezone }}",
+                    "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}]
+                }
+            },
             {
                 "remove": {
                     "field": "iptables.raw_date",

diff --git a/x-pack/filebeat/module/panw/panos/ingest/pipeline.yml b/x-pack/filebeat/module/panw/panos/ingest/pipeline.yml
@@ -10,33 +10,44 @@ processors:
  - date:
      field: "_temp_.generated_time"
      ignore_failure: true
-     #{< if .convert_timezone >}
-     timezone: "{{ event.timezone }}"
-     #{< end >}
      formats:
        - "yyyy/MM/dd HH:mm:ss"
+ - date:
+    if: "ctx.event.timezone != null"
+    field: "@timestamp"
+    formats: ["ISO8601"]
+    timezone: "{{ event.timezone }}"
+    on_failure: [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}]
 
 # event.created is the time the event was received at the management plane.
  - date:
      field: "event.created"
      target_field: "event.created"
      ignore_failure: true
-     #{< if .convert_timezone >}
-     timezone: "{{ event.timezone }}"
-     #{< end >}
      formats:
        - "yyyy/MM/dd HH:mm:ss"
+ - date:
+    if: "ctx.event.timezone != null && ctx.event.created != null "
+    field: "event.created"
+    target_field: "event.created"
+    formats: ["ISO8601"]
+    timezone: "{{ event.timezone }}"
+    on_failure: [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}]
 
 # event.start (traffic only) is the time the session started.
  - date:
      field: "event.start"
      target_field: "event.start"
      ignore_failure: true
-     #{< if .convert_timezone >}
-     timezone: "{{ event.timezone }}"
-     #{< end >}
      formats:
        - "yyyy/MM/dd HH:mm:ss"
+ - date:
+    if: "ctx.event.timezone != null && ctx.event.start != null"
+    field: "event.start"
+    target_field: "event.start"
+    formats: ["ISO8601"]
+    timezone: "{{ event.timezone }}"
+    on_failure: [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}]
 
 # convert integer fields as the output of the CSV processor is always a string.
  - convert: { type: long, ignore_missing: true, field: client.bytes }

diff --git a/x-pack/filebeat/module/rabbitmq/log/ingest/pipeline.yml b/x-pack/filebeat/module/rabbitmq/log/ingest/pipeline.yml
@@ -16,13 +16,15 @@ processors:
     formats:
     - yy-MM-dd HH:mm:ss.SSS
     ignore_failure: true
-    #{< if .convert_timezone >}
+- date:
+    if: "ctx.event.timezone != null"
+    field: "@timestamp"
+    formats: ["ISO8601"]
     timezone: "{{ event.timezone }}"
-    #{< end >}
 - remove:
     field:
     - timestamp
 on_failure:
 - set:
     field: error.message
-    value: "{{ _ingest.on_failure_message }}"
+    value: "{{ _ingest.on_failure_message }}"