Fix timezone fallback in ingest processor (elastic#38407) (elastic#38664

) If no timezone was specified in the date processor, then the conversion would lead to wrong time, as UTC was assumed by default, leading to incorrectly parsed dates. This commit does not assume a default timezone and will thus not format the dates in a wrong way.
dnhatn · Feb 9, 2019 · 56edc8e · 56edc8e
1 parent 5ab5a0a
commit 56edc8e
Show file tree

Hide file tree

Showing 5 changed files with 132 additions and 5 deletions.
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java
@@ -87,10 +87,16 @@ Function<String, ZonedDateTime> getFunction(String format, ZoneId zoneId, Locale
                 format = format.substring(1);
             }
 
+            boolean isUtc = ZoneOffset.UTC.equals(zoneId);
+
             int year = LocalDate.now(ZoneOffset.UTC).getYear();
-            DateFormatter formatter = DateFormatter.forPattern(format)
-                .withLocale(locale)
-                .withZone(zoneId);
+            DateFormatter dateFormatter = DateFormatter.forPattern(format)
+                .withLocale(locale);
+            // if UTC zone is set here, the the time zone specified in the format will be ignored, leading to wrong dates
+            if (isUtc == false) {
+                dateFormatter = dateFormatter.withZone(zoneId);
+            }
+            final DateFormatter formatter = dateFormatter;
             return text -> {
                 TemporalAccessor accessor = formatter.parse(text);
                 // if there is no year, we fall back to the current one and
@@ -106,7 +112,11 @@ Function<String, ZonedDateTime> getFunction(String format, ZoneId zoneId, Locale
                     accessor = newTime.withZoneSameLocal(zoneId);
                 }
 
-                return DateFormatters.from(accessor);
+                if (isUtc) {
+                    return DateFormatters.from(accessor).withZoneSameInstant(ZoneOffset.UTC);
+                } else {
+                    return DateFormatters.from(accessor);
+                }
             };
         }
     };

diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java
@@ -43,7 +43,7 @@ public final class DateProcessor extends AbstractProcessor {
 
     public static final String TYPE = "date";
     static final String DEFAULT_TARGET_FIELD = "@timestamp";
-    public static final DateFormatter FORMATTER = DateFormatter.forPattern("yyyy-MM-dd'T'HH:mm:ss.SSSXXX");
+    private static final DateFormatter FORMATTER = DateFormatter.forPattern("yyyy-MM-dd'T'HH:mm:ss.SSSXXX");
 
     private final TemplateScript.Factory timezone;
     private final TemplateScript.Factory locale;

diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateFormatTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateFormatTests.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.ingest.common;
 
+import org.elasticsearch.common.time.DateFormatter;
 import org.elasticsearch.common.time.DateUtils;
 import org.elasticsearch.test.ESTestCase;
 
@@ -43,6 +44,14 @@ public void testParseJava() {
                 equalTo("11 24 01:29:01"));
     }
 
+    public void testParseJavaWithTimeZone() {
+        Function<String, ZonedDateTime> javaFunction = DateFormat.Java.getFunction("yyyy-MM-dd'T'HH:mm:ss.SSSZZ",
+            ZoneOffset.UTC, Locale.ROOT);
+        ZonedDateTime datetime = javaFunction.apply("2018-02-05T13:44:56.657+0100");
+        String expectedDateTime = DateFormatter.forPattern("yyyy-MM-dd'T'HH:mm:ss.SSSXXX").withZone(ZoneOffset.UTC).format(datetime);
+        assertThat(expectedDateTime, is("2018-02-05T12:44:56.657Z"));
+    }
+
     public void testParseJavaDefaultYear() {
         String format = randomFrom("8dd/MM", "dd/MM");
         ZoneId timezone = DateUtils.of("Europe/Amsterdam");
@@ -70,6 +79,10 @@ public void testParseUnixWithMsPrecision() {
     public void testParseISO8601() {
         assertThat(DateFormat.Iso8601.getFunction(null, ZoneOffset.UTC, null).apply("2001-01-01T00:00:00-0800").toInstant().toEpochMilli(),
                 equalTo(978336000000L));
+        assertThat(DateFormat.Iso8601.getFunction(null, ZoneOffset.UTC, null).apply("2001-01-01T00:00:00-0800").toString(),
+                equalTo("2001-01-01T08:00Z"));
+        assertThat(DateFormat.Iso8601.getFunction(null, ZoneOffset.UTC, null).apply("2001-01-01T00:00:00-0800").toString(),
+                equalTo("2001-01-01T08:00Z"));
     }
 
     public void testParseISO8601Failure() {

diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorTests.java
@@ -29,6 +29,7 @@
 import java.time.ZoneOffset;
 import java.time.ZonedDateTime;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
@@ -97,6 +98,18 @@ public void testJavaPatternMultipleFormats() {
         }
     }
 
+    public void testJavaPatternNoTimezone() {
+        DateProcessor dateProcessor = new DateProcessor(randomAlphaOfLength(10),
+            null, null,
+            "date_as_string", Arrays.asList("yyyy dd MM HH:mm:ss XXX"), "date_as_date");
+
+        Map<String, Object> document = new HashMap<>();
+        document.put("date_as_string", "2010 12 06 00:00:00 -02:00");
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
+        dateProcessor.execute(ingestDocument);
+        assertThat(ingestDocument.getFieldValue("date_as_date", String.class), equalTo("2010-06-12T02:00:00.000Z"));
+    }
+
     public void testInvalidJavaPattern() {
         try {
             DateProcessor processor = new DateProcessor(randomAlphaOfLength(10),

diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/30_date_processor.yml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/30_date_processor.yml
@@ -39,3 +39,94 @@ teardown:
         id: 1
   - match: { _source.date_source_field: "12/06/2010" }
   - match: { _source.date_target_field: "2010-06-12T00:00:00.000+02:00" }
+
+---
+"Test date processor with no timezone configured":
+
+  - do:
+      ingest.put_pipeline:
+        id: "my_pipeline"
+        # sample formats from beats, featuring mongodb, icinga, apache
+        body:  >
+          {
+            "description": "_description",
+            "processors": [
+              {
+                "date" : {
+                  "field" : "date_source_1",
+                  "target_field" : "date_target_1",
+                  "formats" : ["yyyy-MM-dd'T'HH:mm:ss.SSSZZ" ]
+                }
+              },
+              {
+                "date" : {
+                  "field" : "date_source_2",
+                  "target_field" : "date_target_2",
+                  "formats" : ["yyyy-MM-dd HH:mm:ss Z" ]
+                }
+              },
+              {
+                "date" : {
+                  "field" : "date_source_3",
+                  "target_field" : "date_target_3",
+                  "formats" : [ "dd/MMM/yyyy:H:m:s Z" ]
+                }
+              },
+              {
+                "date" : {
+                  "field" : "date_source_4",
+                  "target_field" : "date_target_4",
+                  "formats" : [ "UNIX" ]
+                }
+              },
+              {
+                "date" : {
+                  "field" : "date_source_5",
+                  "target_field" : "date_target_5",
+                  "formats" : [ "UNIX_MS" ]
+                }
+              },
+              {
+                "date" : {
+                  "field" : "date_source_6",
+                  "target_field" : "date_target_6",
+                  "formats" : [ "TAI64N" ]
+                }
+              },
+              {
+                "date" : {
+                  "field" : "date_source_7",
+                  "target_field" : "date_target_7",
+                  "formats" : [ "ISO8601" ]
+                }
+              }
+            ]
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      index:
+        index: test
+        id: 1
+        pipeline: "my_pipeline"
+        body: { date_source_1: "2018-02-05T13:44:56.657+0100", date_source_2: "2017-04-04 13:43:09 +0200", date_source_3: "10/Aug/2018:09:45:56 +0200", date_source_4: "1", date_source_5: "1", date_source_6: "4000000050d506482dbdf024", date_source_7: "2018-02-05T13:44:56.657+0100" }
+
+  - do:
+      get:
+        index: test
+        id: 1
+  - match: { _source.date_source_1: "2018-02-05T13:44:56.657+0100" }
+  - match: { _source.date_target_1: "2018-02-05T12:44:56.657Z" }
+  - match: { _source.date_source_2: "2017-04-04 13:43:09 +0200" }
+  - match: { _source.date_target_2: "2017-04-04T11:43:09.000Z" }
+  - match: { _source.date_source_3: "10/Aug/2018:09:45:56 +0200" }
+  - match: { _source.date_target_3: "2018-08-10T07:45:56.000Z" }
+  - match: { _source.date_source_4: "1" }
+  - match: { _source.date_target_4: "1970-01-01T00:00:01.000Z" }
+  - match: { _source.date_source_5: "1" }
+  - match: { _source.date_target_5: "1970-01-01T00:00:00.001Z" }
+  - match: { _source.date_source_6: "4000000050d506482dbdf024" }
+  - match: { _source.date_target_6: "2012-12-22T01:00:46.767Z" }
+  - match: { _source.date_source_7: "2018-02-05T13:44:56.657+0100" }
+  - match: { _source.date_target_7: "2018-02-05T12:44:56.657Z" }
+