From 925d6bb268477edf4f5540f9ee9e41cd32d6c4fb Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Tue, 19 Jun 2018 10:42:43 -0700 Subject: [PATCH 01/24] Scripting: Conditionally use java time api in scripting This commit adds a boolean system property, `es.scripting.use_java_time`, which controls the concrete return type used by doc values within scripts. The return type of accessing doc values for a date field is changed to Object, essentially duck typing the type to allow co-existence during the transition from joda time to java time. --- .../painless/spi/org.elasticsearch.txt | 4 +- .../index/fielddata/ScriptDocValues.java | 73 ++++++++++++------- .../elasticsearch/script/ScriptModule.java | 3 + 3 files changed, 51 insertions(+), 29 deletions(-) diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.txt index 6495659d9cdc0..3dd436a97f63a 100644 --- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.txt +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.txt @@ -77,8 +77,8 @@ class org.elasticsearch.index.fielddata.ScriptDocValues$Longs { } class org.elasticsearch.index.fielddata.ScriptDocValues$Dates { - org.joda.time.ReadableDateTime get(int) - org.joda.time.ReadableDateTime getValue() + Object get(int) + Object getValue() List getValues() } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java b/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java index b99084de8de46..a7909f2a04edc 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java @@ -35,6 +35,9 @@ import org.joda.time.ReadableDateTime; import java.io.IOException; +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZonedDateTime; import java.util.AbstractList; import java.util.Arrays; import java.util.Comparator; @@ -52,6 +55,7 @@ * values form multiple documents. */ public abstract class ScriptDocValues extends AbstractList { + /** * Set the current doc ID. */ @@ -141,17 +145,30 @@ public int size() { } } - public static final class Dates extends ScriptDocValues { - protected static final DeprecationLogger deprecationLogger = new DeprecationLogger(ESLoggerFactory.getLogger(Dates.class)); + public static final class Dates extends ScriptDocValues { + + private static final boolean USE_JAVA_TIME = org.elasticsearch.common.Booleans.parseBoolean( + System.getProperty("es.scripting.use_java_time"), false); - private static final ReadableDateTime EPOCH = new DateTime(0, DateTimeZone.UTC); + private static final ZoneId JAVA_TIME_UTC = ZoneId.of("UTC"); + + private static final Object EPOCH; + static { + if (USE_JAVA_TIME) { + EPOCH = ZonedDateTime.ofInstant(Instant.EPOCH, JAVA_TIME_UTC); + } else { + EPOCH = new DateTime(0, DateTimeZone.UTC); + } + } private final SortedNumericDocValues in; + /** - * Values wrapped in {@link MutableDateTime}. Null by default an allocated on first usage so we allocate a reasonably size. We keep - * this array so we don't have allocate new {@link MutableDateTime}s on every usage. Instead we reuse them for every document. + * Values wrapped in a date time object. The concrete type depends on the system property {@code es.scripting.use_java_time}. + * When that system property is {@code false}, the date time objects are of type {@link MutableDateTime}. When the system + * property is {@code true}, the date time objects are of type {@link java.time.ZonedDateTime}. */ - private MutableDateTime[] dates; + private Object[] dates; private int count; /** @@ -161,11 +178,20 @@ public Dates(SortedNumericDocValues in) { this.in = in; } + /** Logs a deprecation warning if the joda time api is being used. */ + public static void maybeLogJodaTimeDeprecation() { + if (USE_JAVA_TIME == false) { + DeprecationLogger deprecationLogger = new DeprecationLogger(ESLoggerFactory.getLogger(Dates.class)); + deprecationLogger.deprecated("The joda time api for doc values is deprecated. Use -Des.scripting.use_java_time=true" + + "to use the java time api for date field doc values"); + } + } + /** * Fetch the first field value or 0 millis after epoch if there are no * in. */ - public ReadableDateTime getValue() { + public Object getValue() { if (count == 0) { return EPOCH; } @@ -173,7 +199,7 @@ public ReadableDateTime getValue() { } @Override - public ReadableDateTime get(int index) { + public Object get(int index) { if (index >= count) { throw new IndexOutOfBoundsException( "attempted to fetch the [" + index + "] date when there are only [" @@ -204,29 +230,22 @@ void refreshArray() throws IOException { if (count == 0) { return; } - if (dates == null) { - // Happens for the document. We delay allocating dates so we can allocate it with a reasonable size. - dates = new MutableDateTime[count]; - for (int i = 0; i < dates.length; i++) { - dates[i] = new MutableDateTime(in.nextValue(), DateTimeZone.UTC); + if (USE_JAVA_TIME) { + if (dates == null || count > dates.length) { + // Happens for the document. We delay allocating dates so we can allocate it with a reasonable size. + dates = new ZonedDateTime[count]; } - return; - } - if (count > dates.length) { - // Happens when we move to a new document and it has more dates than any documents before it. - MutableDateTime[] backup = dates; - dates = new MutableDateTime[count]; - System.arraycopy(backup, 0, dates, 0, backup.length); - for (int i = 0; i < backup.length; i++) { - dates[i].setMillis(in.nextValue()); + for (int i = 0; i < count; ++i) { + dates[i] = ZonedDateTime.ofInstant(Instant.ofEpochMilli(in.nextValue()), JAVA_TIME_UTC); } - for (int i = backup.length; i < dates.length; i++) { + } else { + if (dates == null || count > dates.length) { + // Happens for the document. We delay allocating dates so we can allocate it with a reasonable size. + dates = new MutableDateTime[count]; + } + for (int i = 0; i < count; i++) { dates[i] = new MutableDateTime(in.nextValue(), DateTimeZone.UTC); } - return; - } - for (int i = 0; i < count; i++) { - dates[i] = new MutableDateTime(in.nextValue(), DateTimeZone.UTC); } } } diff --git a/server/src/main/java/org/elasticsearch/script/ScriptModule.java b/server/src/main/java/org/elasticsearch/script/ScriptModule.java index 7074d3ad9fe44..1b68f49a3884c 100644 --- a/server/src/main/java/org/elasticsearch/script/ScriptModule.java +++ b/server/src/main/java/org/elasticsearch/script/ScriptModule.java @@ -29,6 +29,7 @@ import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.plugins.ScriptPlugin; import org.elasticsearch.search.aggregations.pipeline.movfn.MovingFunctionScript; @@ -81,6 +82,8 @@ public ScriptModule(Settings settings, List scriptPlugins) { } } scriptService = new ScriptService(settings, Collections.unmodifiableMap(engines), Collections.unmodifiableMap(contexts)); + + ScriptDocValues.Dates.maybeLogJodaTimeDeprecation(); } /** From f42474c41f9015a44a33bce8dec962a2158d9c37 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Tue, 19 Jun 2018 12:18:30 -0700 Subject: [PATCH 02/24] remove unused import --- .../java/org/elasticsearch/index/fielddata/ScriptDocValues.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java b/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java index a7909f2a04edc..bd1a7141beccd 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java @@ -19,7 +19,6 @@ package org.elasticsearch.index.fielddata; - import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.BytesRef; @@ -32,7 +31,6 @@ import org.joda.time.DateTime; import org.joda.time.DateTimeZone; import org.joda.time.MutableDateTime; -import org.joda.time.ReadableDateTime; import java.io.IOException; import java.time.Instant; From d3d2908fdd5343795910c62fad7edc0113284899 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Tue, 19 Jun 2018 12:58:20 -0700 Subject: [PATCH 03/24] fix compile --- .../java/org/elasticsearch/search/fields/SearchFieldsIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java b/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java index ab5387b6e3f48..0726ea5a5b168 100644 --- a/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java +++ b/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java @@ -110,7 +110,7 @@ protected Map, Object>> pluginScripts() { scripts.put("doc['date'].date.millis", vars -> { Map doc = (Map) vars.get("doc"); ScriptDocValues.Dates dates = (ScriptDocValues.Dates) doc.get("date"); - return dates.getValue().getMillis(); + return ((ReadableDateTime) dates.getValue()).getMillis(); }); scripts.put("_fields['num1'].value", vars -> fieldsScript(vars, "num1")); From 1a23214e70d55fe24b03620af84975a1cb424f5b Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Wed, 20 Jun 2018 16:27:02 -0700 Subject: [PATCH 04/24] change setting name and move check into ScriptModule --- .../org/elasticsearch/gradle/BuildPlugin.groovy | 3 +++ .../index/fielddata/ScriptDocValues.java | 17 +++-------------- .../org/elasticsearch/script/ScriptModule.java | 12 +++++++++++- 3 files changed, 17 insertions(+), 15 deletions(-) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy index eb3cd1dc8c6da..4fed0c18fab09 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy @@ -687,6 +687,9 @@ class BuildPlugin implements Plugin { } } + // TODO: remove this once joda time is removed from scriptin in 7.0 + systemProperty 'es.script.use_java_time', 'true' + boolean assertionsEnabled = Boolean.parseBoolean(System.getProperty('tests.asserts', 'true')) enableSystemAssertions assertionsEnabled enableAssertions assertionsEnabled diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java b/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java index bd1a7141beccd..06933a969a00a 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java @@ -28,6 +28,7 @@ import org.elasticsearch.common.geo.GeoUtils; import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.logging.ESLoggerFactory; +import org.elasticsearch.script.ScriptModule; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; import org.joda.time.MutableDateTime; @@ -145,14 +146,11 @@ public int size() { public static final class Dates extends ScriptDocValues { - private static final boolean USE_JAVA_TIME = org.elasticsearch.common.Booleans.parseBoolean( - System.getProperty("es.scripting.use_java_time"), false); - private static final ZoneId JAVA_TIME_UTC = ZoneId.of("UTC"); private static final Object EPOCH; static { - if (USE_JAVA_TIME) { + if (ScriptModule.USE_JAVA_TIME) { EPOCH = ZonedDateTime.ofInstant(Instant.EPOCH, JAVA_TIME_UTC); } else { EPOCH = new DateTime(0, DateTimeZone.UTC); @@ -176,15 +174,6 @@ public Dates(SortedNumericDocValues in) { this.in = in; } - /** Logs a deprecation warning if the joda time api is being used. */ - public static void maybeLogJodaTimeDeprecation() { - if (USE_JAVA_TIME == false) { - DeprecationLogger deprecationLogger = new DeprecationLogger(ESLoggerFactory.getLogger(Dates.class)); - deprecationLogger.deprecated("The joda time api for doc values is deprecated. Use -Des.scripting.use_java_time=true" + - "to use the java time api for date field doc values"); - } - } - /** * Fetch the first field value or 0 millis after epoch if there are no * in. @@ -228,7 +217,7 @@ void refreshArray() throws IOException { if (count == 0) { return; } - if (USE_JAVA_TIME) { + if (ScriptModule.USE_JAVA_TIME) { if (dates == null || count > dates.length) { // Happens for the document. We delay allocating dates so we can allocate it with a reasonable size. dates = new ZonedDateTime[count]; diff --git a/server/src/main/java/org/elasticsearch/script/ScriptModule.java b/server/src/main/java/org/elasticsearch/script/ScriptModule.java index 1b68f49a3884c..b80b68622813d 100644 --- a/server/src/main/java/org/elasticsearch/script/ScriptModule.java +++ b/server/src/main/java/org/elasticsearch/script/ScriptModule.java @@ -27,6 +27,9 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import org.elasticsearch.common.Booleans; +import org.elasticsearch.common.logging.DeprecationLogger; +import org.elasticsearch.common.logging.ESLoggerFactory; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.fielddata.ScriptDocValues; @@ -38,6 +41,9 @@ */ public class ScriptModule { + /** Whether scripts should expose dates as java time objects instead of joda time. */ + public static final boolean USE_JAVA_TIME = Booleans.parseBoolean(System.getProperty("es.script.use_java_time"), false); + public static final Map> CORE_CONTEXTS; static { CORE_CONTEXTS = Stream.of( @@ -83,7 +89,11 @@ public ScriptModule(Settings settings, List scriptPlugins) { } scriptService = new ScriptService(settings, Collections.unmodifiableMap(engines), Collections.unmodifiableMap(contexts)); - ScriptDocValues.Dates.maybeLogJodaTimeDeprecation(); + if (USE_JAVA_TIME == false) { + DeprecationLogger deprecationLogger = new DeprecationLogger(ESLoggerFactory.getLogger(ScriptModule.class)); + deprecationLogger.deprecated("The joda time api for doc values is deprecated. Use -Des.script.use_java_time=true" + + "to use the java time api for date field doc values"); + } } /** From 8ad8573b409d62c8e804a63459cfed293b9f3822 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Wed, 27 Jun 2018 09:41:37 -0700 Subject: [PATCH 05/24] fix checkstyle --- .../index/fielddata/ScriptDocValues.java | 2 -- .../org/elasticsearch/script/ScriptModule.java | 17 ++++++++--------- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java b/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java index 06933a969a00a..671c1365f8f3d 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java @@ -26,8 +26,6 @@ import org.elasticsearch.common.geo.GeoHashUtils; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.geo.GeoUtils; -import org.elasticsearch.common.logging.DeprecationLogger; -import org.elasticsearch.common.logging.ESLoggerFactory; import org.elasticsearch.script.ScriptModule; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; diff --git a/server/src/main/java/org/elasticsearch/script/ScriptModule.java b/server/src/main/java/org/elasticsearch/script/ScriptModule.java index 34df2ba54bf29..efb6206491531 100644 --- a/server/src/main/java/org/elasticsearch/script/ScriptModule.java +++ b/server/src/main/java/org/elasticsearch/script/ScriptModule.java @@ -19,23 +19,22 @@ package org.elasticsearch.script; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.function.Function; -import java.util.stream.Collectors; -import java.util.stream.Stream; - import org.elasticsearch.common.Booleans; import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.logging.ESLoggerFactory; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.plugins.ScriptPlugin; import org.elasticsearch.search.aggregations.pipeline.movfn.MovingFunctionScript; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; + /** * Manages building {@link ScriptService}. */ From 2bd06d97884d7b73d0844bdcbad401cf89e09072 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Tue, 3 Jul 2018 11:44:07 -0700 Subject: [PATCH 06/24] better tests --- .../elasticsearch/gradle/BuildPlugin.groovy | 2 +- server/build.gradle | 9 ++++ .../elasticsearch/script/ScriptModule.java | 2 +- .../fielddata/ScriptDocValuesDatesTests.java | 26 ++++++---- .../ScriptDocValuesJodaBwcTests.java | 52 +++++++++++++++++++ 5 files changed, 78 insertions(+), 13 deletions(-) create mode 100644 server/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesJodaBwcTests.java diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy index 58bae36a120c7..e272a29c14be0 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy @@ -706,7 +706,7 @@ class BuildPlugin implements Plugin { } // TODO: remove this once joda time is removed from scriptin in 7.0 - systemProperty 'es.script.use_java_time', 'true' + systemProperty 'es.scripting.use_java_time', 'true' boolean assertionsEnabled = Boolean.parseBoolean(System.getProperty('tests.asserts', 'true')) enableSystemAssertions assertionsEnabled diff --git a/server/build.gradle b/server/build.gradle index 3055c625ea914..47136f1ff44ee 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -156,6 +156,15 @@ if (isEclipse) { compileJava.options.compilerArgs << "-Xlint:-cast,-deprecation,-rawtypes,-try,-unchecked" compileTestJava.options.compilerArgs << "-Xlint:-cast,-deprecation,-rawtypes,-try,-unchecked" +// TODO: remove in 7.0 +additionalTest('testScriptDocValuesJodaBwc') { + include '**/ScriptDocValuesJodaBwcTests.class' + systemProperty 'es.scripting.use_java_time', 'false' +} +test { + exclude '**/ScriptDocValuesJodaBwcTests.class' +} + forbiddenPatterns { exclude '**/*.json' exclude '**/*.jmx' diff --git a/server/src/main/java/org/elasticsearch/script/ScriptModule.java b/server/src/main/java/org/elasticsearch/script/ScriptModule.java index efb6206491531..01f34c4b6757b 100644 --- a/server/src/main/java/org/elasticsearch/script/ScriptModule.java +++ b/server/src/main/java/org/elasticsearch/script/ScriptModule.java @@ -41,7 +41,7 @@ public class ScriptModule { /** Whether scripts should expose dates as java time objects instead of joda time. */ - public static final boolean USE_JAVA_TIME = Booleans.parseBoolean(System.getProperty("es.script.use_java_time"), false); + public static final boolean USE_JAVA_TIME = Booleans.parseBoolean(System.getProperty("es.scripting.use_java_time"), false); public static final Map> CORE_CONTEXTS; static { diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesDatesTests.java b/server/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesDatesTests.java index 604a11843fc2a..de116cc44098b 100644 --- a/server/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesDatesTests.java +++ b/server/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesDatesTests.java @@ -21,38 +21,42 @@ import org.elasticsearch.index.fielddata.ScriptDocValues.Dates; import org.elasticsearch.test.ESTestCase; -import org.joda.time.DateTime; -import org.joda.time.DateTimeZone; -import org.joda.time.ReadableDateTime; import java.io.IOException; +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.util.function.Function; public class ScriptDocValuesDatesTests extends ESTestCase { public void test() throws IOException { + assertDateDocValues(millis -> ZonedDateTime.ofInstant(Instant.ofEpochMilli(millis), ZoneId.of("UTC"))); + } + + public void assertDateDocValues(Function datetimeCtor) throws IOException { long[][] values = new long[between(3, 10)][]; - ReadableDateTime[][] expectedDates = new ReadableDateTime[values.length][]; + Object[][] expectedDates = new Object[values.length][]; for (int d = 0; d < values.length; d++) { values[d] = new long[randomBoolean() ? randomBoolean() ? 0 : 1 : between(2, 100)]; - expectedDates[d] = new ReadableDateTime[values[d].length]; + expectedDates[d] = new Object[values[d].length]; for (int i = 0; i < values[d].length; i++) { - expectedDates[d][i] = new DateTime(randomNonNegativeLong(), DateTimeZone.UTC); - values[d][i] = expectedDates[d][i].getMillis(); + values[d][i] = randomNonNegativeLong(); + expectedDates[d][i] = datetimeCtor.apply(values[d][i]); } } Dates dates = wrap(values); for (int round = 0; round < 10; round++) { int d = between(0, values.length - 1); dates.setNextDocId(d); - assertEquals(expectedDates[d].length > 0 ? expectedDates[d][0] : new DateTime(0, DateTimeZone.UTC), dates.getValue()); + if (dates.size() > 0) { + assertEquals(expectedDates[d][0], dates.getValue()); + } assertEquals(values[d].length, dates.size()); for (int i = 0; i < values[d].length; i++) { assertEquals(expectedDates[d][i], dates.get(i)); } - - Exception e = expectThrows(UnsupportedOperationException.class, () -> dates.add(new DateTime())); - assertEquals("doc values are unmodifiable", e.getMessage()); } } diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesJodaBwcTests.java b/server/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesJodaBwcTests.java new file mode 100644 index 0000000000000..aa9c1f2fcf23c --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesJodaBwcTests.java @@ -0,0 +1,52 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.fielddata; + +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.plugins.ScriptPlugin; +import org.elasticsearch.script.MockScriptEngine; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.script.ScriptEngine; +import org.elasticsearch.script.ScriptModule; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; + +import java.io.IOException; +import java.util.Collection; +import java.util.Collections; + +public class ScriptDocValuesJodaBwcTests extends ScriptDocValuesDatesTests { + + @Override + public void test() throws IOException { + assertDateDocValues(millis -> new DateTime(millis, DateTimeZone.UTC)); + } + + public void testDeprecationWarning() { + new ScriptModule(Settings.EMPTY, Collections.singletonList(new ScriptPlugin() { + @Override + public ScriptEngine getScriptEngine(Settings settings, Collection> contexts) { + return new MockScriptEngine(MockScriptEngine.NAME, Collections.singletonMap("1", script -> "1")); + } + })); + assertWarnings("The joda time api for doc values is deprecated. Use -Des.script.use_java_time=true" + + "to use the java time api for date field doc values"); + } +} From fc019aaf0d2a4417938bc047665aa6b424b355f7 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Tue, 3 Jul 2018 14:49:44 -0700 Subject: [PATCH 07/24] fix script conversions from date to number --- .../aggregations/support/values/ScriptDoubleValues.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptDoubleValues.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptDoubleValues.java index ac3c8f682ba62..1227efb5ea0af 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptDoubleValues.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptDoubleValues.java @@ -27,6 +27,7 @@ import java.io.IOException; import java.lang.reflect.Array; +import java.time.ZonedDateTime; import java.util.Collection; /** @@ -54,6 +55,9 @@ public boolean advanceExact(int target) throws IOException { } else if (value instanceof ReadableInstant) { resize(1); values[0] = ((ReadableInstant) value).getMillis(); + } else if (value instanceof ZonedDateTime) { + resize(1); + values[0] = ((ZonedDateTime) value).toInstant().toEpochMilli(); } else if (value.getClass().isArray()) { int length = Array.getLength(value); if (length == 0) { @@ -89,6 +93,8 @@ private static double toDoubleValue(Object o) { } else if (o instanceof ReadableInstant) { // Dates are exposed in scripts as ReadableDateTimes but aggregations want them to be numeric return ((ReadableInstant) o).getMillis(); + } else if (o instanceof ZonedDateTime) { + return ((ZonedDateTime) o).toInstant().toEpochMilli(); } else if (o instanceof Boolean) { // We do expose boolean fields as boolean in scripts, however aggregations still expect // that scripts return the same internal representation as regular fields, so boolean From 0650ac8c33e106af3927c06f54884a8b7a0ce1c6 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Tue, 3 Jul 2018 17:32:04 -0700 Subject: [PATCH 08/24] more conversions --- .../search/aggregations/support/values/ScriptLongValues.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptLongValues.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptLongValues.java index 818a9d9fd8ddf..cdc448bd04130 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptLongValues.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptLongValues.java @@ -28,6 +28,7 @@ import java.io.IOException; import java.lang.reflect.Array; +import java.time.ZonedDateTime; import java.util.Collection; import java.util.Iterator; @@ -91,6 +92,8 @@ private static long toLongValue(Object o) { } else if (o instanceof ReadableInstant) { // Dates are exposed in scripts as ReadableDateTimes but aggregations want them to be numeric return ((ReadableInstant) o).getMillis(); + } else if (o instanceof ZonedDateTime) { + return ((ZonedDateTime) o).toInstant().toEpochMilli(); } else if (o instanceof Boolean) { // We do expose boolean fields as boolean in scripts, however aggregations still expect // that scripts return the same internal representation as regular fields, so boolean From c599e465d2e11e43ed050360c8e425c48a7f3522 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Tue, 17 Jul 2018 08:56:14 -0700 Subject: [PATCH 09/24] fix test class used --- .../java/org/elasticsearch/search/fields/SearchFieldsIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java b/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java index e2ce6490f5dbe..895d954dd7a44 100644 --- a/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java +++ b/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java @@ -110,7 +110,7 @@ protected Map, Object>> pluginScripts() { scripts.put("doc['date'].date.millis", vars -> { Map doc = (Map) vars.get("doc"); ScriptDocValues.Dates dates = (ScriptDocValues.Dates) doc.get("date"); - return ((ReadableDateTime) dates.getValue()).getMillis(); + return ((ZonedDateTime) dates.getValue()).toInstant().toEpochMilli(); }); scripts.put("_fields['num1'].value", vars -> fieldsScript(vars, "num1")); From 4383c4389cd36b1218c29dcd615b05673d962e67 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Tue, 17 Jul 2018 12:40:46 -0700 Subject: [PATCH 10/24] fix test --- .../fielddata/ScriptDocValuesMissingV6BehaviourTests.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesMissingV6BehaviourTests.java b/server/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesMissingV6BehaviourTests.java index 1dc836874d847..33fa98e6a0dc2 100644 --- a/server/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesMissingV6BehaviourTests.java +++ b/server/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesMissingV6BehaviourTests.java @@ -35,6 +35,10 @@ import org.joda.time.DateTimeZone; import org.joda.time.ReadableDateTime; import java.io.IOException; +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; import java.util.Collection; import java.util.Collections; @@ -68,7 +72,7 @@ public void testZeroForMissingValueLong() throws IOException { } public void testEpochForMissingValueDate() throws IOException { - final ReadableDateTime EPOCH = new DateTime(0, DateTimeZone.UTC); + final ZonedDateTime EPOCH = ZonedDateTime.ofInstant(Instant.EPOCH, ZoneId.of("UTC")); long[][] values = new long[between(3, 10)][]; for (int d = 0; d < values.length; d++) { values[d] = new long[0]; From dee8abe7fac1e1f20d1574a1e481c68ba87009fc Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Thu, 19 Jul 2018 09:40:31 -0700 Subject: [PATCH 11/24] fix missing imports --- .../fielddata/ScriptDocValuesMissingV6BehaviourTests.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesMissingV6BehaviourTests.java b/server/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesMissingV6BehaviourTests.java index 33fa98e6a0dc2..813cd1e0cea09 100644 --- a/server/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesMissingV6BehaviourTests.java +++ b/server/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesMissingV6BehaviourTests.java @@ -31,13 +31,9 @@ import org.elasticsearch.script.ScriptModule; import org.elasticsearch.test.ESTestCase; -import org.joda.time.DateTime; -import org.joda.time.DateTimeZone; -import org.joda.time.ReadableDateTime; import java.io.IOException; import java.time.Instant; import java.time.ZoneId; -import java.time.ZoneOffset; import java.time.ZonedDateTime; import java.util.Collection; import java.util.Collections; From 7a67906b9ccb870ef61986c133a2bb0dbbce19e4 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Thu, 19 Jul 2018 13:33:15 -0700 Subject: [PATCH 12/24] fix test --- .../search/fields/SearchFieldsIT.java | 21 ++++++++----------- 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java b/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java index b2c43efcf9598..4b14fdb2538de 100644 --- a/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java +++ b/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java @@ -47,15 +47,11 @@ import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.InternalSettingsPlugin; -import org.joda.time.DateTime; -import org.joda.time.DateTimeZone; -import org.joda.time.ReadableDateTime; -import org.joda.time.base.BaseDateTime; -import org.joda.time.format.DateTimeFormat; -import org.joda.time.format.DateTimeFormatter; +import java.time.ZoneId; import java.time.ZoneOffset; import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Arrays; import java.util.Base64; @@ -64,6 +60,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.concurrent.ExecutionException; @@ -808,8 +805,8 @@ public void testDocValueFields() throws Exception { assertThat(searchResponse.getHits().getAt(0).getFields().get("long_field").getValue(), equalTo((Object) 4L)); assertThat(searchResponse.getHits().getAt(0).getFields().get("float_field").getValue(), equalTo((Object) 5.0)); assertThat(searchResponse.getHits().getAt(0).getFields().get("double_field").getValue(), equalTo((Object) 6.0d)); - BaseDateTime dateField = searchResponse.getHits().getAt(0).getFields().get("date_field").getValue(); - assertThat(dateField.getMillis(), equalTo(date.toInstant().toEpochMilli())); + ZonedDateTime dateField = searchResponse.getHits().getAt(0).getFields().get("date_field").getValue(); + assertThat(dateField.toInstant().toEpochMilli(), equalTo(date.toInstant().toEpochMilli())); assertThat(searchResponse.getHits().getAt(0).getFields().get("boolean_field").getValue(), equalTo((Object) true)); assertThat(searchResponse.getHits().getAt(0).getFields().get("text_field").getValue(), equalTo("foo")); assertThat(searchResponse.getHits().getAt(0).getFields().get("keyword_field").getValue(), equalTo("foo")); @@ -949,10 +946,10 @@ public void testDocValueFieldsWithFieldAlias() throws Exception { assertAcked(prepareCreate("test").addMapping("type", mapping)); ensureGreen("test"); - DateTime date = new DateTime(1990, 12, 29, 0, 0, DateTimeZone.UTC); - DateTimeFormatter formatter = DateTimeFormat.forPattern("yyyy-MM-dd"); + ZonedDateTime date = ZonedDateTime.of(1990, 12, 29, 0, 0, 0, 0, ZoneId.of("UTC")); + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd", Locale.ROOT); - index("test", "type", "1", "text_field", "foo", "date_field", formatter.print(date)); + index("test", "type", "1", "text_field", "foo", "date_field", formatter.format(date)); refresh("test"); SearchRequestBuilder builder = client().prepareSearch().setQuery(matchAllQuery()) @@ -980,7 +977,7 @@ public void testDocValueFieldsWithFieldAlias() throws Exception { DocumentField dateField = fields.get("date_field"); assertThat(dateField.getName(), equalTo("date_field")); - ReadableDateTime fetchedDate = dateField.getValue(); + ZonedDateTime fetchedDate = dateField.getValue(); assertThat(fetchedDate, equalTo(date)); } From f3add18a06a79ea48915872794f3d9abfea39fde Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Wed, 25 Jul 2018 11:17:39 -0700 Subject: [PATCH 13/24] rework to emit deprecation on each use of script with dates --- server/build.gradle | 6 -- .../index/fielddata/ScriptDocValues.java | 62 ++++++++++++++++--- .../elasticsearch/script/ScriptModule.java | 9 --- .../fielddata/ScriptDocValuesDatesTests.java | 61 +++++++++++++++--- .../ScriptDocValuesJodaBwcTests.java | 52 ---------------- 5 files changed, 105 insertions(+), 85 deletions(-) delete mode 100644 server/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesJodaBwcTests.java diff --git a/server/build.gradle b/server/build.gradle index eac791e33fba0..72d7641ee4500 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -156,11 +156,6 @@ if (isEclipse) { compileJava.options.compilerArgs << "-Xlint:-cast,-deprecation,-rawtypes,-try,-unchecked" compileTestJava.options.compilerArgs << "-Xlint:-cast,-deprecation,-rawtypes,-try,-unchecked" -// TODO: remove in 7.0 -additionalTest('testScriptDocValuesJodaBwc') { - include '**/ScriptDocValuesJodaBwcTests.class' - systemProperty 'es.scripting.use_java_time', 'false' -} // TODO: remove ScriptDocValuesMissingV6BehaviourTests in 7.0 additionalTest('testScriptDocValuesMissingV6Behaviour'){ include '**/ScriptDocValuesMissingV6BehaviourTests.class' @@ -168,7 +163,6 @@ additionalTest('testScriptDocValuesMissingV6Behaviour'){ } test { // these are tested explicitly in separate test tasks - exclude '**/ScriptDocValuesJodaBwcTests.class' exclude '**/*ScriptDocValuesMissingV6BehaviourTests.class' } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java b/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java index a7193822f3a64..e1e5b4adb90e5 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java @@ -23,15 +23,20 @@ import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefBuilder; +import org.elasticsearch.common.Booleans; import org.elasticsearch.common.geo.GeoHashUtils; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.geo.GeoUtils; +import org.elasticsearch.common.logging.DeprecationLogger; +import org.elasticsearch.common.logging.ESLoggerFactory; import org.elasticsearch.script.ScriptModule; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; import org.joda.time.MutableDateTime; import java.io.IOException; +import java.security.AccessController; +import java.security.PrivilegedAction; import java.time.Instant; import java.time.ZoneId; import java.time.ZonedDateTime; @@ -39,8 +44,11 @@ import java.util.Arrays; import java.util.Comparator; import java.util.List; +import java.util.function.Consumer; import java.util.function.UnaryOperator; +import static org.elasticsearch.common.Booleans.parseBoolean; + /** * Script level doc values, the assumption is that any implementation will @@ -145,19 +153,26 @@ public int size() { public static final class Dates extends ScriptDocValues { - private static final ZoneId JAVA_TIME_UTC = ZoneId.of("UTC"); + /** Whether scripts should expose dates as java time objects instead of joda time. */ + private static final boolean USE_JAVA_TIME = parseBoolean(System.getProperty("es.scripting.use_java_time"), false); - private static final Object EPOCH; - static { - if (ScriptModule.USE_JAVA_TIME) { - EPOCH = ZonedDateTime.ofInstant(Instant.EPOCH, JAVA_TIME_UTC); - } else { - EPOCH = new DateTime(0, DateTimeZone.UTC); - } - } + private static final DeprecationLogger deprecationLogger = new DeprecationLogger(ESLoggerFactory.getLogger(Dates.class)); + + private static final ZoneId JAVA_TIME_UTC = ZoneId.of("UTC"); private final SortedNumericDocValues in; + /** + * Method call to add deprecation message. Normally this is + * {@link #deprecationLogger} but tests override. + */ + private final Consumer deprecationCallback; + + /** + * Whether java time or joda time should be used. This is normally {@link #USE_JAVA_TIME} but tests override it. + */ + private final boolean useJavaTime; + /** * Values wrapped in a date time object. The concrete type depends on the system property {@code es.scripting.use_java_time}. * When that system property is {@code false}, the date time objects are of type {@link MutableDateTime}. When the system @@ -170,7 +185,16 @@ public static final class Dates extends ScriptDocValues { * Standard constructor. */ public Dates(SortedNumericDocValues in) { + this(in, message -> deprecationLogger.deprecatedAndMaybeLog("scripting_joda_time_deprecation", message), USE_JAVA_TIME); + } + + /** + * Constructor for testing with a deprecation callback. + */ + Dates(SortedNumericDocValues in, Consumer deprecationCallback, boolean useJavaTime) { this.in = in; + this.deprecationCallback = deprecationCallback; + this.useJavaTime = useJavaTime; } /** @@ -217,7 +241,7 @@ void refreshArray() throws IOException { if (count == 0) { return; } - if (ScriptModule.USE_JAVA_TIME) { + if (useJavaTime) { if (dates == null || count > dates.length) { // Happens for the document. We delay allocating dates so we can allocate it with a reasonable size. dates = new ZonedDateTime[count]; @@ -226,6 +250,8 @@ void refreshArray() throws IOException { dates[i] = ZonedDateTime.ofInstant(Instant.ofEpochMilli(in.nextValue()), JAVA_TIME_UTC); } } else { + deprecated("The joda time api for doc values is deprecated. Use -Des.scripting.use_java_time=true" + + " to use the java time api for date field doc values"); if (dates == null || count > dates.length) { // Happens for the document. We delay allocating dates so we can allocate it with a reasonable size. dates = new MutableDateTime[count]; @@ -235,6 +261,22 @@ void refreshArray() throws IOException { } } } + + /** + * Log a deprecation log, with the server's permissions, not the permissions of the + * script calling this method. We need to do this to prevent errors when rolling + * the log file. + */ + private void deprecated(String message) { + // Intentionally not calling SpecialPermission.check because this is supposed to be called by scripts + AccessController.doPrivileged(new PrivilegedAction() { + @Override + public Void run() { + deprecationCallback.accept(message); + return null; + } + }); + } } public static final class Doubles extends ScriptDocValues { diff --git a/server/src/main/java/org/elasticsearch/script/ScriptModule.java b/server/src/main/java/org/elasticsearch/script/ScriptModule.java index df69583fa3880..8351063cc96dc 100644 --- a/server/src/main/java/org/elasticsearch/script/ScriptModule.java +++ b/server/src/main/java/org/elasticsearch/script/ScriptModule.java @@ -41,9 +41,6 @@ */ public class ScriptModule { - /** Whether scripts should expose dates as java time objects instead of joda time. */ - public static final boolean USE_JAVA_TIME = Booleans.parseBoolean(System.getProperty("es.scripting.use_java_time"), false); - public static final Map> CORE_CONTEXTS; static { CORE_CONTEXTS = Stream.of( @@ -92,12 +89,6 @@ public ScriptModule(Settings settings, List scriptPlugins) { } } scriptService = new ScriptService(settings, Collections.unmodifiableMap(engines), Collections.unmodifiableMap(contexts)); - - if (USE_JAVA_TIME == false) { - DeprecationLogger deprecationLogger = new DeprecationLogger(ESLoggerFactory.getLogger(ScriptModule.class)); - deprecationLogger.deprecated("The joda time api for doc values is deprecated. Use -Des.script.use_java_time=true" + - "to use the java time api for date field doc values"); - } } /** diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesDatesTests.java b/server/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesDatesTests.java index 70bd657939a49..ba867f17ecc20 100644 --- a/server/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesDatesTests.java +++ b/server/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesDatesTests.java @@ -21,20 +21,45 @@ import org.elasticsearch.index.fielddata.ScriptDocValues.Dates; import org.elasticsearch.test.ESTestCase; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; import java.io.IOException; +import java.security.AccessControlContext; +import java.security.AccessController; +import java.security.PermissionCollection; +import java.security.Permissions; +import java.security.PrivilegedAction; +import java.security.ProtectionDomain; import java.time.Instant; import java.time.ZoneId; import java.time.ZonedDateTime; +import java.util.HashSet; +import java.util.Set; +import java.util.function.Consumer; import java.util.function.Function; +import static org.hamcrest.Matchers.containsInAnyOrder; + public class ScriptDocValuesDatesTests extends ESTestCase { - public void test() throws IOException { - assertDateDocValues(millis -> ZonedDateTime.ofInstant(Instant.ofEpochMilli(millis), ZoneId.of("UTC"))); + + public void testJavaTime() throws IOException { + assertDateDocValues(true); } - public void assertDateDocValues(Function datetimeCtor) throws IOException { + public void testJodaTimeBwc() throws IOException { + assertDateDocValues(false, "The joda time api for doc values is deprecated." + + " Use -Des.scripting.use_java_time=true to use the java time api for date field doc values"); + } + + public void assertDateDocValues(boolean useJavaTime, String... expectedWarnings) throws IOException { + final Function datetimeCtor; + if (useJavaTime) { + datetimeCtor = millis -> ZonedDateTime.ofInstant(Instant.ofEpochMilli(millis), ZoneId.of("UTC")); + } else { + datetimeCtor = millis -> new DateTime(millis, DateTimeZone.UTC); + } long[][] values = new long[between(3, 10)][]; Object[][] expectedDates = new Object[values.length][]; for (int d = 0; d < values.length; d++) { @@ -45,12 +70,28 @@ public void assertDateDocValues(Function datetimeCtor) throws IOEx expectedDates[d][i] = datetimeCtor.apply(values[d][i]); } } - Dates dates = wrap(values); + + Set warnings = new HashSet<>(); + Dates dates = wrap(values, deprecationMessage -> { + warnings.add(deprecationMessage); + /* Create a temporary directory to prove we are running with the + * server's permissions. */ + createTempDir(); + }, useJavaTime); + // each call to get or getValue will be run with limited permissions, just as they are in scripts + PermissionCollection noPermissions = new Permissions(); + AccessControlContext noPermissionsAcc = new AccessControlContext( + new ProtectionDomain[] { + new ProtectionDomain(null, noPermissions) + } + ); + for (int round = 0; round < 10; round++) { int d = between(0, values.length - 1); dates.setNextDocId(d); if (expectedDates[d].length > 0) { - assertEquals(expectedDates[d][0] , dates.getValue()); + Object dateValue = AccessController.doPrivileged((PrivilegedAction) dates::getValue, noPermissionsAcc); + assertEquals(expectedDates[d][0] , dateValue); } else { Exception e = expectThrows(IllegalStateException.class, () -> dates.getValue()); assertEquals("A document doesn't have a value for a field! " + @@ -59,12 +100,16 @@ public void assertDateDocValues(Function datetimeCtor) throws IOEx assertEquals(values[d].length, dates.size()); for (int i = 0; i < values[d].length; i++) { - assertEquals(expectedDates[d][i], dates.get(i)); + final int ndx = i; + Object dateValue = AccessController.doPrivileged((PrivilegedAction) () -> dates.get(ndx), noPermissionsAcc); + assertEquals(expectedDates[d][i], dateValue); } } + + assertThat(warnings, containsInAnyOrder(expectedWarnings)); } - private Dates wrap(long[][] values) { + private Dates wrap(long[][] values, Consumer deprecationHandler, boolean useJavaTime) { return new Dates(new AbstractSortedNumericDocValues() { long[] current; int i; @@ -83,6 +128,6 @@ public int docValueCount() { public long nextValue() { return current[i++]; } - }); + }, deprecationHandler, useJavaTime); } } diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesJodaBwcTests.java b/server/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesJodaBwcTests.java deleted file mode 100644 index aa9c1f2fcf23c..0000000000000 --- a/server/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesJodaBwcTests.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.fielddata; - -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.plugins.ScriptPlugin; -import org.elasticsearch.script.MockScriptEngine; -import org.elasticsearch.script.ScriptContext; -import org.elasticsearch.script.ScriptEngine; -import org.elasticsearch.script.ScriptModule; -import org.joda.time.DateTime; -import org.joda.time.DateTimeZone; - -import java.io.IOException; -import java.util.Collection; -import java.util.Collections; - -public class ScriptDocValuesJodaBwcTests extends ScriptDocValuesDatesTests { - - @Override - public void test() throws IOException { - assertDateDocValues(millis -> new DateTime(millis, DateTimeZone.UTC)); - } - - public void testDeprecationWarning() { - new ScriptModule(Settings.EMPTY, Collections.singletonList(new ScriptPlugin() { - @Override - public ScriptEngine getScriptEngine(Settings settings, Collection> contexts) { - return new MockScriptEngine(MockScriptEngine.NAME, Collections.singletonMap("1", script -> "1")); - } - })); - assertWarnings("The joda time api for doc values is deprecated. Use -Des.script.use_java_time=true" + - "to use the java time api for date field doc values"); - } -} From 286eb8d201be9d2b53e77fcc22e2368729a72047 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Wed, 25 Jul 2018 12:27:09 -0700 Subject: [PATCH 14/24] fix checkstyle --- .../org/elasticsearch/index/fielddata/ScriptDocValues.java | 3 --- .../src/main/java/org/elasticsearch/script/ScriptModule.java | 4 ---- 2 files changed, 7 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java b/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java index e1e5b4adb90e5..b9d1bcf12ff3b 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java @@ -23,14 +23,11 @@ import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefBuilder; -import org.elasticsearch.common.Booleans; import org.elasticsearch.common.geo.GeoHashUtils; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.geo.GeoUtils; import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.logging.ESLoggerFactory; -import org.elasticsearch.script.ScriptModule; -import org.joda.time.DateTime; import org.joda.time.DateTimeZone; import org.joda.time.MutableDateTime; diff --git a/server/src/main/java/org/elasticsearch/script/ScriptModule.java b/server/src/main/java/org/elasticsearch/script/ScriptModule.java index 8351063cc96dc..4e4c0a5a3c60f 100644 --- a/server/src/main/java/org/elasticsearch/script/ScriptModule.java +++ b/server/src/main/java/org/elasticsearch/script/ScriptModule.java @@ -19,10 +19,6 @@ package org.elasticsearch.script; -import org.elasticsearch.common.Booleans; -import org.elasticsearch.common.logging.DeprecationLogger; -import org.elasticsearch.common.logging.ESLoggerFactory; -import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.plugins.ScriptPlugin; From 32e0e8086f5ffa75d5d722c02d422c8ab4163ab7 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Thu, 26 Jul 2018 16:29:36 -0700 Subject: [PATCH 15/24] add doc note and make doc tests use joda time api --- docs/build.gradle | 3 +++ docs/painless/painless-getting-started.asciidoc | 6 +++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/build.gradle b/docs/build.gradle index a67c0217490b3..4c0502a0e06fc 100644 --- a/docs/build.gradle +++ b/docs/build.gradle @@ -37,6 +37,9 @@ integTestCluster { extraConfigFile 'hunspell/en_US/en_US.dic', '../server/src/test/resources/indices/analyze/conf_dir/hunspell/en_US/en_US.dic' // Whitelist reindexing from the local node so we can test it. setting 'reindex.remote.whitelist', '127.0.0.1:*' + + // TODO: remove this for 7.0, this exists to allow the doc examples in 6.x to continue using the defaults + systemProperty 'es.scripting.use_java_time', 'false' } // remove when https://github.com/elastic/elasticsearch/issues/31305 is fixed diff --git a/docs/painless/painless-getting-started.asciidoc b/docs/painless/painless-getting-started.asciidoc index 1dec4a33bb583..d8075fe502e25 100644 --- a/docs/painless/painless-getting-started.asciidoc +++ b/docs/painless/painless-getting-started.asciidoc @@ -198,7 +198,7 @@ POST hockey/player/1/_update ==== Dates Date fields are exposed as -`ReadableDateTime` +`ReadableDateTime` or so they support methods like `getYear`, and `getDayOfWeek`. @@ -221,6 +221,10 @@ GET hockey/_search ---------------------------------------------------------------- // CONSOLE +NOTE: Date fields are changing in 7.0 to be exposed as `ZonedDateTime` +from Java 8's time API. To switch to this functionality early, +add `-Des.scripting.use_java_time=true` to `jvm.options`. + [float] [[modules-scripting-painless-regex]] ==== Regular expressions From c750fd6b8497eb3a3bafb6bb6534b07ad22fb7c4 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Mon, 30 Jul 2018 10:18:42 -0700 Subject: [PATCH 16/24] add warning to tests --- docs/painless/painless-getting-started.asciidoc | 1 + .../aggregations/bucket/datehistogram-aggregation.asciidoc | 1 + 2 files changed, 2 insertions(+) diff --git a/docs/painless/painless-getting-started.asciidoc b/docs/painless/painless-getting-started.asciidoc index d8075fe502e25..8cff207ab04d5 100644 --- a/docs/painless/painless-getting-started.asciidoc +++ b/docs/painless/painless-getting-started.asciidoc @@ -220,6 +220,7 @@ GET hockey/_search } ---------------------------------------------------------------- // CONSOLE +// TEST[warning:The joda time api for doc values is deprecated. Use -Des.scripting.use_java_time=true to use the java time api for date field doc values] NOTE: Date fields are changing in 7.0 to be exposed as `ZonedDateTime` from Java 8's time API. To switch to this functionality early, diff --git a/docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc b/docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc index efbd8ef7389bb..e19ecac462d14 100644 --- a/docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc @@ -425,6 +425,7 @@ POST /sales/_search?size=0 -------------------------------------------------- // CONSOLE // TEST[setup:sales] +// TEST[warning:The joda time api for doc values is deprecated. Use -Des.scripting.use_java_time=true to use the java time api for date field doc values] Response: From 759210888282600b355b143831dffbb89d142fca Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Mon, 30 Jul 2018 11:32:56 -0700 Subject: [PATCH 17/24] remove accidentally added back 6.x bwc test --- server/build.gradle | 10 - ...criptDocValuesMissingV6BehaviourTests.java | 195 ------------------ 2 files changed, 205 deletions(-) delete mode 100644 server/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesMissingV6BehaviourTests.java diff --git a/server/build.gradle b/server/build.gradle index 72d7641ee4500..deb3839897918 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -156,16 +156,6 @@ if (isEclipse) { compileJava.options.compilerArgs << "-Xlint:-cast,-deprecation,-rawtypes,-try,-unchecked" compileTestJava.options.compilerArgs << "-Xlint:-cast,-deprecation,-rawtypes,-try,-unchecked" -// TODO: remove ScriptDocValuesMissingV6BehaviourTests in 7.0 -additionalTest('testScriptDocValuesMissingV6Behaviour'){ - include '**/ScriptDocValuesMissingV6BehaviourTests.class' - systemProperty 'es.scripting.exception_for_missing_value', 'false' -} -test { - // these are tested explicitly in separate test tasks - exclude '**/*ScriptDocValuesMissingV6BehaviourTests.class' -} - forbiddenPatterns { exclude '**/*.json' exclude '**/*.jmx' diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesMissingV6BehaviourTests.java b/server/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesMissingV6BehaviourTests.java deleted file mode 100644 index 813cd1e0cea09..0000000000000 --- a/server/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesMissingV6BehaviourTests.java +++ /dev/null @@ -1,195 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.fielddata; - -import org.elasticsearch.common.geo.GeoPoint; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.fielddata.ScriptDocValues.Longs; -import org.elasticsearch.index.fielddata.ScriptDocValues.Dates; -import org.elasticsearch.index.fielddata.ScriptDocValues.Booleans; -import org.elasticsearch.plugins.ScriptPlugin; -import org.elasticsearch.script.MockScriptEngine; -import org.elasticsearch.script.ScriptContext; -import org.elasticsearch.script.ScriptEngine; -import org.elasticsearch.script.ScriptModule; -import org.elasticsearch.test.ESTestCase; - -import java.io.IOException; -import java.time.Instant; -import java.time.ZoneId; -import java.time.ZonedDateTime; -import java.util.Collection; -import java.util.Collections; - -import static java.util.Collections.singletonList; - -public class ScriptDocValuesMissingV6BehaviourTests extends ESTestCase { - - public void testScriptMissingValuesWarning(){ - new ScriptModule(Settings.EMPTY, singletonList(new ScriptPlugin() { - @Override - public ScriptEngine getScriptEngine(Settings settings, Collection> contexts) { - return new MockScriptEngine(MockScriptEngine.NAME, Collections.singletonMap("1", script -> "1")); - } - })); - assertWarnings("Script: returning default values for missing document values is deprecated. " + - "Set system property '-Des.scripting.exception_for_missing_value=true' " + - "to make behaviour compatible with future major versions."); - } - - public void testZeroForMissingValueLong() throws IOException { - long[][] values = new long[between(3, 10)][]; - for (int d = 0; d < values.length; d++) { - values[d] = new long[0]; - } - Longs longs = wrap(values); - for (int round = 0; round < 10; round++) { - int d = between(0, values.length - 1); - longs.setNextDocId(d); - assertEquals(0, longs.getValue()); - } - } - - public void testEpochForMissingValueDate() throws IOException { - final ZonedDateTime EPOCH = ZonedDateTime.ofInstant(Instant.EPOCH, ZoneId.of("UTC")); - long[][] values = new long[between(3, 10)][]; - for (int d = 0; d < values.length; d++) { - values[d] = new long[0]; - } - Dates dates = wrapDates(values); - for (int round = 0; round < 10; round++) { - int d = between(0, values.length - 1); - dates.setNextDocId(d); - assertEquals(EPOCH, dates.getValue()); - } - } - - public void testFalseForMissingValueBoolean() throws IOException { - long[][] values = new long[between(3, 10)][]; - for (int d = 0; d < values.length; d++) { - values[d] = new long[0]; - } - Booleans bools = wrapBooleans(values); - for (int round = 0; round < 10; round++) { - int d = between(0, values.length - 1); - bools.setNextDocId(d); - assertEquals(false, bools.getValue()); - } - } - - public void testNullForMissingValueGeo() throws IOException{ - final MultiGeoPointValues values = wrap(new GeoPoint[0]); - final ScriptDocValues.GeoPoints script = new ScriptDocValues.GeoPoints(values); - script.setNextDocId(0); - assertEquals(null, script.getValue()); - } - - - private Longs wrap(long[][] values) { - return new Longs(new AbstractSortedNumericDocValues() { - long[] current; - int i; - @Override - public boolean advanceExact(int doc) { - i = 0; - current = values[doc]; - return current.length > 0; - } - @Override - public int docValueCount() { - return current.length; - } - @Override - public long nextValue() { - return current[i++]; - } - }); - } - - private Booleans wrapBooleans(long[][] values) { - return new Booleans(new AbstractSortedNumericDocValues() { - long[] current; - int i; - @Override - public boolean advanceExact(int doc) { - i = 0; - current = values[doc]; - return current.length > 0; - } - @Override - public int docValueCount() { - return current.length; - } - @Override - public long nextValue() { - return current[i++]; - } - }); - } - - private Dates wrapDates(long[][] values) { - return new Dates(new AbstractSortedNumericDocValues() { - long[] current; - int i; - @Override - public boolean advanceExact(int doc) { - current = values[doc]; - i = 0; - return current.length > 0; - } - @Override - public int docValueCount() { - return current.length; - } - @Override - public long nextValue() { - return current[i++]; - } - }); - } - - - private static MultiGeoPointValues wrap(final GeoPoint... points) { - return new MultiGeoPointValues() { - int docID = -1; - int i; - @Override - public GeoPoint nextValue() { - if (docID != 0) { - fail(); - } - return points[i++]; - } - @Override - public boolean advanceExact(int docId) { - docID = docId; - return points.length > 0; - } - @Override - public int docValueCount() { - if (docID != 0) { - return 0; - } - return points.length; - } - }; - } - -} From 6b95165e4913768d22c945470f7658b2dd59ed13 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Mon, 30 Jul 2018 13:24:04 -0700 Subject: [PATCH 18/24] fix painless tests --- modules/lang-painless/build.gradle | 3 +-- .../resources/rest-api-spec/test/painless/20_scriptfield.yml | 4 ++-- .../rest-api-spec/test/painless/50_script_doc_values.yml | 4 ++-- .../org/elasticsearch/index/fielddata/ScriptDocValues.java | 2 +- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/modules/lang-painless/build.gradle b/modules/lang-painless/build.gradle index fb1ea441a9dd5..e3a7ccecae2e5 100644 --- a/modules/lang-painless/build.gradle +++ b/modules/lang-painless/build.gradle @@ -17,8 +17,6 @@ * under the License. */ - - esplugin { description 'An easy, safe and fast scripting language for Elasticsearch' classname 'org.elasticsearch.painless.PainlessPlugin' @@ -26,6 +24,7 @@ esplugin { integTestCluster { module project.project(':modules:mapper-extras') + systemProperty 'es.scripting.use_java_time', 'true' } dependencies { diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/20_scriptfield.yml b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/20_scriptfield.yml index 2914e8a916ec6..3be6601521e87 100644 --- a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/20_scriptfield.yml +++ b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/20_scriptfield.yml @@ -108,7 +108,7 @@ setup: script_fields: bar: script: - source: "doc.date.value.dayOfWeek" + source: "doc.date.value.dayOfWeek.value" - match: { hits.hits.0.fields.bar.0: 7} @@ -123,7 +123,7 @@ setup: source: > StringBuilder b = new StringBuilder(); for (def date : doc.dates) { - b.append(" ").append(date.getDayOfWeek()); + b.append(" ").append(date.getDayOfWeek().value); } return b.toString().trim() diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/50_script_doc_values.yml b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/50_script_doc_values.yml index 617b8df61b6bd..4c3c204d2d9ca 100644 --- a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/50_script_doc_values.yml +++ b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/50_script_doc_values.yml @@ -95,7 +95,7 @@ setup: field: script: source: "doc.date.get(0)" - - match: { hits.hits.0.fields.field.0: '2017-01-01T12:11:12.000Z' } + - match: { hits.hits.0.fields.field.0: '2017-01-01T12:11:12Z' } - do: search: @@ -104,7 +104,7 @@ setup: field: script: source: "doc.date.value" - - match: { hits.hits.0.fields.field.0: '2017-01-01T12:11:12.000Z' } + - match: { hits.hits.0.fields.field.0: '2017-01-01T12:11:12Z' } --- "geo_point": diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java b/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java index b9d1bcf12ff3b..7db8bd695cfb7 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java @@ -155,7 +155,7 @@ public static final class Dates extends ScriptDocValues { private static final DeprecationLogger deprecationLogger = new DeprecationLogger(ESLoggerFactory.getLogger(Dates.class)); - private static final ZoneId JAVA_TIME_UTC = ZoneId.of("UTC"); + private static final ZoneId JAVA_TIME_UTC = ZoneId.of("UTC").normalized(); private final SortedNumericDocValues in; From 4042747ae7976e1382ecfbd2861b9d3ef5442210 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Mon, 30 Jul 2018 17:04:23 -0700 Subject: [PATCH 19/24] fix test --- .../index/fielddata/ScriptDocValuesDatesTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesDatesTests.java b/server/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesDatesTests.java index ba867f17ecc20..7b240d85383b6 100644 --- a/server/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesDatesTests.java +++ b/server/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesDatesTests.java @@ -56,7 +56,7 @@ public void testJodaTimeBwc() throws IOException { public void assertDateDocValues(boolean useJavaTime, String... expectedWarnings) throws IOException { final Function datetimeCtor; if (useJavaTime) { - datetimeCtor = millis -> ZonedDateTime.ofInstant(Instant.ofEpochMilli(millis), ZoneId.of("UTC")); + datetimeCtor = millis -> ZonedDateTime.ofInstant(Instant.ofEpochMilli(millis), ZoneId.of("UTC").normalized()); } else { datetimeCtor = millis -> new DateTime(millis, DateTimeZone.UTC); } From 74c9b899722bc7abfdaa69cdacb050c010ff7175 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Tue, 31 Jul 2018 08:24:53 -0700 Subject: [PATCH 20/24] fix more tests to use normalized zoneid --- .../java/org/elasticsearch/search/fields/SearchFieldsIT.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java b/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java index f1f86c8b6119f..051c26ba2758c 100644 --- a/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java +++ b/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java @@ -943,7 +943,8 @@ public void testDocValueFieldsWithFieldAlias() throws Exception { assertAcked(prepareCreate("test").addMapping("type", mapping)); ensureGreen("test"); - ZonedDateTime date = ZonedDateTime.of(1990, 12, 29, 0, 0, 0, 0, ZoneId.of("UTC")); + ZonedDateTime date = ZonedDateTime.of(1990, 12, 29, 0, 0, 0, 0, + ZoneId.of("UTC").normalized()); DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd", Locale.ROOT); index("test", "type", "1", "text_field", "foo", "date_field", formatter.format(date)); From 96e843f27e4751b54d2c30db3ecbe60b013f23cf Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Tue, 31 Jul 2018 10:08:32 -0700 Subject: [PATCH 21/24] use ZoneOffset.UTC --- .../org/elasticsearch/index/fielddata/ScriptDocValues.java | 5 ++--- .../index/fielddata/ScriptDocValuesDatesTests.java | 3 ++- .../java/org/elasticsearch/search/fields/SearchFieldsIT.java | 3 +-- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java b/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java index 7db8bd695cfb7..658955def76b5 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java @@ -36,6 +36,7 @@ import java.security.PrivilegedAction; import java.time.Instant; import java.time.ZoneId; +import java.time.ZoneOffset; import java.time.ZonedDateTime; import java.util.AbstractList; import java.util.Arrays; @@ -155,8 +156,6 @@ public static final class Dates extends ScriptDocValues { private static final DeprecationLogger deprecationLogger = new DeprecationLogger(ESLoggerFactory.getLogger(Dates.class)); - private static final ZoneId JAVA_TIME_UTC = ZoneId.of("UTC").normalized(); - private final SortedNumericDocValues in; /** @@ -244,7 +243,7 @@ void refreshArray() throws IOException { dates = new ZonedDateTime[count]; } for (int i = 0; i < count; ++i) { - dates[i] = ZonedDateTime.ofInstant(Instant.ofEpochMilli(in.nextValue()), JAVA_TIME_UTC); + dates[i] = ZonedDateTime.ofInstant(Instant.ofEpochMilli(in.nextValue()), ZoneOffset.UTC); } } else { deprecated("The joda time api for doc values is deprecated. Use -Des.scripting.use_java_time=true" + diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesDatesTests.java b/server/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesDatesTests.java index 7b240d85383b6..54b2998644bd0 100644 --- a/server/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesDatesTests.java +++ b/server/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesDatesTests.java @@ -33,6 +33,7 @@ import java.security.ProtectionDomain; import java.time.Instant; import java.time.ZoneId; +import java.time.ZoneOffset; import java.time.ZonedDateTime; import java.util.HashSet; import java.util.Set; @@ -56,7 +57,7 @@ public void testJodaTimeBwc() throws IOException { public void assertDateDocValues(boolean useJavaTime, String... expectedWarnings) throws IOException { final Function datetimeCtor; if (useJavaTime) { - datetimeCtor = millis -> ZonedDateTime.ofInstant(Instant.ofEpochMilli(millis), ZoneId.of("UTC").normalized()); + datetimeCtor = millis -> ZonedDateTime.ofInstant(Instant.ofEpochMilli(millis), ZoneOffset.UTC); } else { datetimeCtor = millis -> new DateTime(millis, DateTimeZone.UTC); } diff --git a/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java b/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java index 051c26ba2758c..04f044d858de1 100644 --- a/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java +++ b/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java @@ -943,8 +943,7 @@ public void testDocValueFieldsWithFieldAlias() throws Exception { assertAcked(prepareCreate("test").addMapping("type", mapping)); ensureGreen("test"); - ZonedDateTime date = ZonedDateTime.of(1990, 12, 29, 0, 0, 0, 0, - ZoneId.of("UTC").normalized()); + ZonedDateTime date = ZonedDateTime.of(1990, 12, 29, 0, 0, 0, 0, ZoneOffset.UTC); DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd", Locale.ROOT); index("test", "type", "1", "text_field", "foo", "date_field", formatter.format(date)); From 055d2c088a034726bffec74005a81c8d638c92bb Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Tue, 31 Jul 2018 10:21:59 -0700 Subject: [PATCH 22/24] remove unused import --- .../java/org/elasticsearch/index/fielddata/ScriptDocValues.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java b/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java index 658955def76b5..35284cb655d98 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java @@ -35,7 +35,6 @@ import java.security.AccessController; import java.security.PrivilegedAction; import java.time.Instant; -import java.time.ZoneId; import java.time.ZoneOffset; import java.time.ZonedDateTime; import java.util.AbstractList; @@ -47,7 +46,6 @@ import static org.elasticsearch.common.Booleans.parseBoolean; - /** * Script level doc values, the assumption is that any implementation will * implement a getValue and a getValues that return From b28ce90ad0a1616a97969520e4315514fb45c13e Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Tue, 31 Jul 2018 10:49:47 -0700 Subject: [PATCH 23/24] another unused import... --- .../java/org/elasticsearch/search/fields/SearchFieldsIT.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java b/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java index 04f044d858de1..aea0243a399d4 100644 --- a/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java +++ b/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java @@ -48,7 +48,6 @@ import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.InternalSettingsPlugin; -import java.time.ZoneId; import java.time.ZoneOffset; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; From 63e941bd7b4b7c982eee0d519d6dab15c8686876 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Tue, 31 Jul 2018 10:54:21 -0700 Subject: [PATCH 24/24] and one more --- .../index/fielddata/ScriptDocValuesDatesTests.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesDatesTests.java b/server/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesDatesTests.java index 54b2998644bd0..9f80a45fa8e74 100644 --- a/server/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesDatesTests.java +++ b/server/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesDatesTests.java @@ -32,7 +32,6 @@ import java.security.PrivilegedAction; import java.security.ProtectionDomain; import java.time.Instant; -import java.time.ZoneId; import java.time.ZoneOffset; import java.time.ZonedDateTime; import java.util.HashSet; @@ -42,7 +41,6 @@ import static org.hamcrest.Matchers.containsInAnyOrder; - public class ScriptDocValuesDatesTests extends ESTestCase { public void testJavaTime() throws IOException {