From ddf63a67a746f858861927b7721eb3cda31a67f5 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Mon, 11 Oct 2021 14:56:20 +0200 Subject: [PATCH 01/15] Deprecate resolution loss on date field When stroing nanoseconds on a date field the nanosecond part is lost as it cannot be stored. The date_nanos field should be used instead. This commit emits a deprecation warning to notify users about this. --- .../elasticsearch/index/mapper/DateFieldMapper.java | 5 +++++ .../index/mapper/DateFieldMapperTests.java | 11 +++++++++++ 2 files changed, 16 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 1a0a22e8824cb..b43d7047cadcb 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -78,6 +78,11 @@ public enum Resolution { MILLISECONDS(CONTENT_TYPE, NumericType.DATE) { @Override public long convert(Instant instant) { + if (instant.getNano() % 1000000 != 0) { + DEPRECATION_LOGGER.critical(DeprecationCategory.MAPPINGS, "date_field_with_nanos", + "You are attempting to store a date field with nanosecond resolution on a date field. " + + "The nanosecond part was lost. Use date_nanos field type."); + } return instant.toEpochMilli(); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java index 205f081b5ae83..e286d6a3edc11 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java @@ -142,6 +142,17 @@ public void testIgnoreMalformed() throws IOException { testIgnoreMalformedForValue("-522000000", "long overflow", "date_optional_time"); } + public void testResolutionLoss() throws Exception { + DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> b + .field("type", "date"))); + + ParsedDocument doc = mapper.parse(source(b -> b.field("field", "2018-10-03T14:42:44.123456+0000"))); + + assertWarnings("You are attempting to store a date field with nanosecond resolution on a date field." + + " The nanosecond part was lost. Use date_nanos field type."); + } + + private void testIgnoreMalformedForValue(String value, String expectedCause, String dateFormat) throws IOException { DocumentMapper mapper = createDocumentMapper(fieldMapping((builder)-> dateFieldMapping(builder, dateFormat))); From 9d129c152fc948d761017a586a318c7d4819475b Mon Sep 17 00:00:00 2001 From: pgomulka Date: Tue, 12 Oct 2021 11:00:45 +0200 Subject: [PATCH 02/15] only emit on parsing --- .../elasticsearch/index/mapper/DateFieldMapper.java | 13 +++++++------ .../index/mapper/DateFieldMapperTests.java | 13 +++++++++---- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index b43d7047cadcb..14c8836479cb0 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -78,11 +78,6 @@ public enum Resolution { MILLISECONDS(CONTENT_TYPE, NumericType.DATE) { @Override public long convert(Instant instant) { - if (instant.getNano() % 1000000 != 0) { - DEPRECATION_LOGGER.critical(DeprecationCategory.MAPPINGS, "date_field_with_nanos", - "You are attempting to store a date field with nanosecond resolution on a date field. " + - "The nanosecond part was lost. Use date_nanos field type."); - } return instant.toEpochMilli(); } @@ -383,7 +378,13 @@ protected DateMathParser dateMathParser() { // Visible for testing. public long parse(String value) { - return resolution.convert(DateFormatters.from(dateTimeFormatter().parse(value), dateTimeFormatter().locale()).toInstant()); + final Instant instant = DateFormatters.from(dateTimeFormatter().parse(value), dateTimeFormatter().locale()).toInstant(); + if (resolution == Resolution.MILLISECONDS && instant.getNano() % 1000000 != 0) { + DEPRECATION_LOGGER.critical(DeprecationCategory.MAPPINGS, "date_field_with_nanos", + "You are attempting to store a date field with nanosecond resolution on a date field. " + + "The nanosecond part was lost. Use date_nanos field type."); + } + return resolution.convert(instant); } /** diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java index e286d6a3edc11..73885d2e50186 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java @@ -417,11 +417,11 @@ public void testFetchMillisFromIso8601() throws IOException { } public void testFetchMillisFromIso8601Nanos() throws IOException { - assertFetch(dateMapperService(), "field", randomIs8601Nanos(MAX_ISO_DATE), null); + assertFetch(dateNanosMapperService(), "field", randomIs8601Nanos(MAX_NANOS), null); } public void testFetchMillisFromIso8601NanosFormatted() throws IOException { - assertFetch(dateMapperService(), "field", randomIs8601Nanos(MAX_ISO_DATE), "strict_date_optional_time_nanos"); + assertFetch(dateNanosMapperService(), "field", randomIs8601Nanos(MAX_NANOS), "strict_date_optional_time_nanos"); } /** @@ -432,7 +432,8 @@ public void testFetchMillisFromIso8601NanosFormatted() throws IOException { * way. */ public void testFetchMillisFromRoundedNanos() throws IOException { - assertFetch(dateMapperService(), "field", randomDecimalNanos(MAX_ISO_DATE), null); + assertFetch(dateMapperService(), "field", randomDecimalMillis(MAX_ISO_DATE), null); + assertFetch(dateNanosMapperService(), "field", randomDecimalNanos(MAX_NANOS), null); } /** @@ -545,7 +546,7 @@ protected Object generateRandomInputValue(MappedFieldType ft) { switch (((DateFieldType) ft).resolution()) { case MILLISECONDS: if (randomBoolean()) { - return randomIs8601Nanos(MAX_ISO_DATE); + return randomDecimalMillis(MAX_ISO_DATE); } return randomLongBetween(0, Long.MAX_VALUE); case NANOSECONDS: @@ -578,6 +579,10 @@ private String randomDecimalNanos(long maxMillis) { return Long.toString(randomLongBetween(0, maxMillis)) + "." + between(0, 999999); } + private String randomDecimalMillis(long maxMillis) { + return Long.toString(randomLongBetween(0, maxMillis)); + } + public void testScriptAndPrecludedParameters() { { Exception e = expectThrows(MapperParsingException.class, () -> createDocumentMapper(fieldMapping(b -> { From 0138a7ab1228b930249225276479eab3010ae880 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Tue, 12 Oct 2021 11:35:53 +0200 Subject: [PATCH 03/15] test fix --- .../java/org/elasticsearch/index/mapper/DateFieldMapper.java | 2 +- .../xpack/datastreams/AutoCreateDataStreamIT.java | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 14c8836479cb0..ef95f84b78cd9 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -380,7 +380,7 @@ protected DateMathParser dateMathParser() { public long parse(String value) { final Instant instant = DateFormatters.from(dateTimeFormatter().parse(value), dateTimeFormatter().locale()).toInstant(); if (resolution == Resolution.MILLISECONDS && instant.getNano() % 1000000 != 0) { - DEPRECATION_LOGGER.critical(DeprecationCategory.MAPPINGS, "date_field_with_nanos", + DEPRECATION_LOGGER.warn(DeprecationCategory.MAPPINGS, "date_field_with_nanos", "You are attempting to store a date field with nanosecond resolution on a date field. " + "The nanosecond part was lost. Use date_nanos field type."); } diff --git a/x-pack/plugin/data-streams/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/datastreams/AutoCreateDataStreamIT.java b/x-pack/plugin/data-streams/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/datastreams/AutoCreateDataStreamIT.java index 9f6cdbeedc53b..3c3d6b999ac06 100644 --- a/x-pack/plugin/data-streams/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/datastreams/AutoCreateDataStreamIT.java +++ b/x-pack/plugin/data-streams/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/datastreams/AutoCreateDataStreamIT.java @@ -14,6 +14,7 @@ import org.elasticsearch.client.ResponseException; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.Streams; +import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.json.JsonXContent; import org.elasticsearch.test.rest.ESRestTestCase; @@ -101,7 +102,9 @@ private void createTemplateWithAllowAutoCreate(Boolean allowAutoCreate) throws I private Response indexDocument() throws IOException { final Request indexDocumentRequest = new Request("POST", "recipe_kr/_doc"); - indexDocumentRequest.setJsonEntity("{ \"@timestamp\": \"" + Instant.now() + "\", \"name\": \"Kimchi\" }"); + final Instant now = Instant.now(); + final String time = DateFormatter.forPattern("strict_date_optional_time").format(now); + indexDocumentRequest.setJsonEntity("{ \"@timestamp\": \"" + time + "\", \"name\": \"Kimchi\" }"); return client().performRequest(indexDocumentRequest); } } From 6486c68ca2ea53fde22a7b2223bb9563da50c062 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Thu, 14 Oct 2021 17:57:53 +0200 Subject: [PATCH 04/15] more info in deprecation, test fixes and split of methods (no warning on fetch) --- .../elasticsearch/upgrades/IndexingIT.java | 2 +- .../60_pipeline_timestamp_date_mapping.yml | 2 +- .../index/mapper/DateFieldMapper.java | 24 +++++++++++++++---- .../ml/transforms/PainlessDomainSplitIT.java | 4 +++- 4 files changed, 24 insertions(+), 8 deletions(-) diff --git a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexingIT.java b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexingIT.java index 43346f426897a..c9b538661cb7a 100644 --- a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexingIT.java +++ b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexingIT.java @@ -200,7 +200,7 @@ public void testDateNanosFormatUpgrade() throws IOException { Request index = new Request("POST", "/" + indexName + "/_doc/"); XContentBuilder doc = XContentBuilder.builder(XContentType.JSON.xContent()) .startObject() - .field("date", "2015-01-01T12:10:30.123456789Z") + .field("date", "2015-01-01T12:10:30.123Z") .field("date_nanos", "2015-01-01T12:10:30.123456789Z") .endObject(); index.addParameter("refresh", "true"); diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/60_pipeline_timestamp_date_mapping.yml b/qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/60_pipeline_timestamp_date_mapping.yml index 0f514f2213492..a096814689a64 100644 --- a/qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/60_pipeline_timestamp_date_mapping.yml +++ b/qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/60_pipeline_timestamp_date_mapping.yml @@ -9,7 +9,7 @@ index: timetest body: mappings: - "properties": { "my_time": {"type": "date", "format": "strict_date_optional_time_nanos"}} + "properties": { "my_time": {"type": "date_nanos", "format": "strict_date_optional_time_nanos"}} - do: ingest.put_pipeline: diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index ef95f84b78cd9..54982998ccf99 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -290,7 +290,9 @@ private Long parseNullValue(DateFieldType fieldType) { return null; } try { - return fieldType.parse(nullValue.getValue()); + final String fieldName = fieldType.name(); + final String indexName = "null value... but what index?";//context.indexSettings().getIndex().getName(); + return fieldType.parseWithDeprecation(nullValue.getValue(), fieldName, nullValue.getValue()); } catch (Exception e) { if (indexCreatedVersion.onOrAfter(Version.V_8_0_0)) { throw new MapperParsingException("Error parsing [null_value] on field [" + name() + "]: " + e.getMessage(), e); @@ -307,6 +309,7 @@ private Long parseNullValue(DateFieldType fieldType) { public DateFieldMapper build(MapperBuilderContext context) { DateFieldType ft = new DateFieldType(context.buildFullName(name()), index.getValue(), store.getValue(), docValues.getValue(), buildFormatter(), resolution, nullValue.getValue(), scriptValues(), meta.getValue()); + Long nullTimestamp = parseNullValue(ft); return new DateFieldMapper(name, ft, multiFieldsBuilder.build(this, context), copyTo.build(), nullTimestamp, resolution, this); @@ -378,15 +381,24 @@ protected DateMathParser dateMathParser() { // Visible for testing. public long parse(String value) { - final Instant instant = DateFormatters.from(dateTimeFormatter().parse(value), dateTimeFormatter().locale()).toInstant(); + final Instant instant = getInstant(value); + return resolution.convert(instant); + } + + public long parseWithDeprecation(String value, String fieldName, String indexName) { + final Instant instant = getInstant(value); if (resolution == Resolution.MILLISECONDS && instant.getNano() % 1000000 != 0) { DEPRECATION_LOGGER.warn(DeprecationCategory.MAPPINGS, "date_field_with_nanos", - "You are attempting to store a date field with nanosecond resolution on a date field. " + - "The nanosecond part was lost. Use date_nanos field type."); + "You are attempting to store a nanosecond resolution on a field {} of type date on index {}. " + + "The nanosecond part was lost. Use date_nanos field type.", fieldName, indexName); } return resolution.convert(instant); } + private Instant getInstant(String value) { + return DateFormatters.from(dateTimeFormatter().parse(value), dateTimeFormatter().locale()).toInstant(); + } + /** * Format to use to resolve {@link Number}s from the source. Its valid * to send the numbers with up to six digits after the decimal place @@ -706,7 +718,9 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio timestamp = nullValue; } else { try { - timestamp = fieldType().parse(dateAsString); + final String fieldName = fieldType().name(); + final String indexName = context.indexSettings().getIndex().getName(); + timestamp = fieldType().parseWithDeprecation(dateAsString, fieldName, indexName); } catch (IllegalArgumentException | ElasticsearchParseException | DateTimeException | ArithmeticException e) { if (ignoreMalformed) { context.addIgnoredField(mappedFieldType.name()); diff --git a/x-pack/plugin/ml/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/transforms/PainlessDomainSplitIT.java b/x-pack/plugin/ml/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/transforms/PainlessDomainSplitIT.java index 32e4bd9ab4712..28ec21d6d0063 100644 --- a/x-pack/plugin/ml/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/transforms/PainlessDomainSplitIT.java +++ b/x-pack/plugin/ml/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/transforms/PainlessDomainSplitIT.java @@ -12,6 +12,8 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.time.DateFormatter; +import org.elasticsearch.common.time.DateFormatters; import org.elasticsearch.test.rest.ESRestTestCase; import java.time.ZoneOffset; @@ -278,7 +280,7 @@ public void testHRDSplit() throws Exception { for (int i = 1; i <= 100; i++) { ZonedDateTime time = baseTime.plusHours(i); - String formattedTime = time.format(DateTimeFormatter.ISO_DATE_TIME); + String formattedTime = DateFormatter.forPattern("strict_date_optional_time").format(time); if (i % 50 == 0) { // Anomaly has 100 docs, but we don't care about the value for (int j = 0; j < 100; j++) { From 15d8934d4a0d3318297cb1b8e0b43b8fe1a0f4cc Mon Sep 17 00:00:00 2001 From: pgomulka Date: Thu, 14 Oct 2021 20:25:48 +0200 Subject: [PATCH 05/15] message fix --- .../org/elasticsearch/index/mapper/DateFieldMapperTests.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java index a8c20889da8df..f2596dbf8db37 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java @@ -148,8 +148,9 @@ public void testResolutionLoss() throws Exception { ParsedDocument doc = mapper.parse(source(b -> b.field("field", "2018-10-03T14:42:44.123456+0000"))); - assertWarnings("You are attempting to store a date field with nanosecond resolution on a date field." + - " The nanosecond part was lost. Use date_nanos field type."); + assertWarnings("You are attempting to store a nanosecond resolution " + + "on a field field of type date on index index. " + + "The nanosecond part was lost. Use date_nanos field type."); } From e74e50416fc65e109fc464e421e37b2678493a95 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Thu, 14 Oct 2021 21:19:53 +0200 Subject: [PATCH 06/15] yml test fix --- .../test/search.aggregation/49_range_timezone_bug.yml | 2 +- .../rest-api-spec/test/multi_cluster/80_transform.yml | 2 +- .../rest-api-spec/test/remote_cluster/80_transform.yml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.aggregation/49_range_timezone_bug.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.aggregation/49_range_timezone_bug.yml index 481c32f688be6..ddba9d20e83d3 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.aggregation/49_range_timezone_bug.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.aggregation/49_range_timezone_bug.yml @@ -8,7 +8,7 @@ setup: mappings: properties: mydate: - type: date + type: date_nanos format: "uuuu-MM-dd'T'HH:mm:ss.SSSSSSSSSZZZZZ" - do: diff --git a/x-pack/plugin/transform/qa/multi-cluster-tests-with-security/src/test/resources/rest-api-spec/test/multi_cluster/80_transform.yml b/x-pack/plugin/transform/qa/multi-cluster-tests-with-security/src/test/resources/rest-api-spec/test/multi_cluster/80_transform.yml index cafa1fc384d06..adbb7426b1458 100644 --- a/x-pack/plugin/transform/qa/multi-cluster-tests-with-security/src/test/resources/rest-api-spec/test/multi_cluster/80_transform.yml +++ b/x-pack/plugin/transform/qa/multi-cluster-tests-with-security/src/test/resources/rest-api-spec/test/multi_cluster/80_transform.yml @@ -211,7 +211,7 @@ teardown: mappings: properties: time: - type: date + type: date_nanos user: type: keyword stars: diff --git a/x-pack/plugin/transform/qa/multi-cluster-tests-with-security/src/test/resources/rest-api-spec/test/remote_cluster/80_transform.yml b/x-pack/plugin/transform/qa/multi-cluster-tests-with-security/src/test/resources/rest-api-spec/test/remote_cluster/80_transform.yml index c550f148f956b..f9c0cfd5c616b 100644 --- a/x-pack/plugin/transform/qa/multi-cluster-tests-with-security/src/test/resources/rest-api-spec/test/remote_cluster/80_transform.yml +++ b/x-pack/plugin/transform/qa/multi-cluster-tests-with-security/src/test/resources/rest-api-spec/test/remote_cluster/80_transform.yml @@ -49,7 +49,7 @@ teardown: mappings: properties: time: - type: date + type: date_nanos user: type: keyword stars: @@ -108,7 +108,7 @@ teardown: mappings: properties: time: - type: date + type: date_nanos user: type: keyword stars: From cfa22eb6c3daa77a7d821843693f488573167892 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Thu, 14 Oct 2021 22:47:18 +0200 Subject: [PATCH 07/15] reneame field --- .../rest-api-spec/test/multi_cluster/80_transform.yml | 2 +- .../rest-api-spec/test/remote_cluster/80_transform.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/transform/qa/multi-cluster-tests-with-security/src/test/resources/rest-api-spec/test/multi_cluster/80_transform.yml b/x-pack/plugin/transform/qa/multi-cluster-tests-with-security/src/test/resources/rest-api-spec/test/multi_cluster/80_transform.yml index adbb7426b1458..4bf9247fdf9d0 100644 --- a/x-pack/plugin/transform/qa/multi-cluster-tests-with-security/src/test/resources/rest-api-spec/test/multi_cluster/80_transform.yml +++ b/x-pack/plugin/transform/qa/multi-cluster-tests-with-security/src/test/resources/rest-api-spec/test/multi_cluster/80_transform.yml @@ -210,7 +210,7 @@ teardown: test_alias: {} mappings: properties: - time: + date: type: date_nanos user: type: keyword diff --git a/x-pack/plugin/transform/qa/multi-cluster-tests-with-security/src/test/resources/rest-api-spec/test/remote_cluster/80_transform.yml b/x-pack/plugin/transform/qa/multi-cluster-tests-with-security/src/test/resources/rest-api-spec/test/remote_cluster/80_transform.yml index f9c0cfd5c616b..1cc9112508dfd 100644 --- a/x-pack/plugin/transform/qa/multi-cluster-tests-with-security/src/test/resources/rest-api-spec/test/remote_cluster/80_transform.yml +++ b/x-pack/plugin/transform/qa/multi-cluster-tests-with-security/src/test/resources/rest-api-spec/test/remote_cluster/80_transform.yml @@ -48,7 +48,7 @@ teardown: test_alias: {} mappings: properties: - time: + date: type: date_nanos user: type: keyword @@ -107,7 +107,7 @@ teardown: test_alias: {} mappings: properties: - time: + date: type: date_nanos user: type: keyword From 4dd2b173b90c671c75ce2cfe54a08d6d6561178a Mon Sep 17 00:00:00 2001 From: pgomulka Date: Fri, 15 Oct 2021 09:29:21 +0200 Subject: [PATCH 08/15] fixtransform tests --- .../continuous/TransformContinuousIT.java | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/TransformContinuousIT.java b/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/TransformContinuousIT.java index b047b39f1c70b..b6d64bba01b86 100644 --- a/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/TransformContinuousIT.java +++ b/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/TransformContinuousIT.java @@ -34,6 +34,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.XContentBuilder; @@ -53,6 +54,7 @@ import java.text.DecimalFormatSymbols; import java.time.Instant; import java.time.ZoneId; +import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Base64; import java.util.Collections; @@ -191,7 +193,7 @@ public void testContinousEvents() throws Exception { for (int i = 0; i < 100; i++) { dates.add( // create a random date between 1/1/2001 and 1/1/2006 - ContinuousTestCase.STRICT_DATE_OPTIONAL_TIME_PRINTER_NANOS.withZone(ZoneId.of("UTC")) + formatTimestmap(dateType) .format(Instant.ofEpochMilli(randomLongBetween(978307200000L, 1136073600000L))) ); } @@ -246,11 +248,11 @@ public void testContinousEvents() throws Exception { } // simulate a different timestamp that is off from the timestamp used for sync, so it can fall into the previous bucket - String metricDateString = ContinuousTestCase.STRICT_DATE_OPTIONAL_TIME_PRINTER_NANOS.withZone(ZoneId.of("UTC")) + String metricDateString = formatTimestmap(dateType) .format(runDate.minusSeconds(randomIntBetween(0, 2)).plusNanos(randomIntBetween(0, 999999))); source.append("\"metric-timestamp\":\"").append(metricDateString).append("\","); - String dateString = ContinuousTestCase.STRICT_DATE_OPTIONAL_TIME_PRINTER_NANOS.withZone(ZoneId.of("UTC")) + String dateString = formatTimestmap(dateType) .format(runDate.plusNanos(randomIntBetween(0, 999999))); source.append("\"timestamp\":\"").append(dateString).append("\","); @@ -298,6 +300,14 @@ public void testContinousEvents() throws Exception { } } + private DateFormatter formatTimestmap(String dateType) { + if(dateType == "date_nanos"){ + return DateFormatter.forPattern("strict_date_optional_time_nanos").withZone(ZoneId.of("UTC")); + } else { + return DateFormatter.forPattern("strict_date_optional_time").withZone(ZoneId.of("UTC")); + } + } + /** * Create the transform source index with randomized settings to increase test coverage, for example * index sorting, triggers query optimizations. From 6f12128f7163facd5d8c88908f92b1571469d4a0 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Fri, 15 Oct 2021 09:56:55 +0200 Subject: [PATCH 09/15] reformat message --- .../java/org/elasticsearch/index/mapper/DateFieldMapper.java | 2 +- .../org/elasticsearch/index/mapper/DateFieldMapperTests.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 54982998ccf99..bf90f3e0817af 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -389,7 +389,7 @@ public long parseWithDeprecation(String value, String fieldName, String indexNam final Instant instant = getInstant(value); if (resolution == Resolution.MILLISECONDS && instant.getNano() % 1000000 != 0) { DEPRECATION_LOGGER.warn(DeprecationCategory.MAPPINGS, "date_field_with_nanos", - "You are attempting to store a nanosecond resolution on a field {} of type date on index {}. " + + "You are attempting to store a nanosecond resolution on a field [{}] of type date on index [{}]. " + "The nanosecond part was lost. Use date_nanos field type.", fieldName, indexName); } return resolution.convert(instant); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java index f2596dbf8db37..f6ec16718a72c 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java @@ -149,7 +149,7 @@ public void testResolutionLoss() throws Exception { ParsedDocument doc = mapper.parse(source(b -> b.field("field", "2018-10-03T14:42:44.123456+0000"))); assertWarnings("You are attempting to store a nanosecond resolution " + - "on a field field of type date on index index. " + + "on a field [field] of type date on index [index]. " + "The nanosecond part was lost. Use date_nanos field type."); } From 276ac15b513281c895634ef2efc3e3cbf0eb76e7 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Fri, 15 Oct 2021 10:53:32 +0200 Subject: [PATCH 10/15] null value logging --- .../index/mapper/DateFieldMapper.java | 7 +++---- .../index/mapper/MapperBuilderContext.java | 14 ++++++++++++++ .../elasticsearch/index/mapper/MappingParser.java | 3 ++- 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index bf90f3e0817af..7bfb2057c8318 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -285,14 +285,13 @@ protected List> getParameters() { return List.of(index, docValues, store, format, locale, nullValue, ignoreMalformed, script, onScriptError, meta); } - private Long parseNullValue(DateFieldType fieldType) { + private Long parseNullValue(DateFieldType fieldType, String indexName) { if (nullValue.getValue() == null) { return null; } try { final String fieldName = fieldType.name(); - final String indexName = "null value... but what index?";//context.indexSettings().getIndex().getName(); - return fieldType.parseWithDeprecation(nullValue.getValue(), fieldName, nullValue.getValue()); + return fieldType.parseWithDeprecation(nullValue.getValue(), fieldName, indexName); } catch (Exception e) { if (indexCreatedVersion.onOrAfter(Version.V_8_0_0)) { throw new MapperParsingException("Error parsing [null_value] on field [" + name() + "]: " + e.getMessage(), e); @@ -310,7 +309,7 @@ public DateFieldMapper build(MapperBuilderContext context) { DateFieldType ft = new DateFieldType(context.buildFullName(name()), index.getValue(), store.getValue(), docValues.getValue(), buildFormatter(), resolution, nullValue.getValue(), scriptValues(), meta.getValue()); - Long nullTimestamp = parseNullValue(ft); + Long nullTimestamp = parseNullValue(ft, context.getIndexName()); return new DateFieldMapper(name, ft, multiFieldsBuilder.build(this, context), copyTo.build(), nullTimestamp, resolution, this); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperBuilderContext.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperBuilderContext.java index 45a5fa24ed538..6220c5ed0496a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperBuilderContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperBuilderContext.java @@ -9,6 +9,7 @@ package org.elasticsearch.index.mapper; import org.elasticsearch.common.Strings; +import org.elasticsearch.index.Index; /** * Holds context for building Mapper objects from their Builders @@ -30,9 +31,14 @@ public static MapperBuilderContext forPath(ContentPath path) { } private final String path; + private final String indexName; private MapperBuilderContext(String path) { + this(path,null); + } + private MapperBuilderContext(String path, String indexName) { this.path = path; + this.indexName = indexName; } /** @@ -53,4 +59,12 @@ public final String buildFullName(String name) { } return path + "." + name; } + + public MapperBuilderContext withIndexName(String indexName) { + return new MapperBuilderContext(this.path, indexName); + } + + public String getIndexName() { + return indexName; + } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappingParser.java b/server/src/main/java/org/elasticsearch/index/mapper/MappingParser.java index 727e7622c27df..1906e9cbb3051 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappingParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappingParser.java @@ -95,7 +95,8 @@ Mapping parse(@Nullable String type, CompressedXContent source) throws MapperPar private Mapping parse(String type, Map mapping) throws MapperParsingException { MappingParserContext parserContext = parserContextSupplier.get(); RootObjectMapper rootObjectMapper - = rootObjectTypeParser.parse(type, mapping, parserContext).build(MapperBuilderContext.ROOT); + = rootObjectTypeParser.parse(type, mapping, parserContext).build(MapperBuilderContext.ROOT + .withIndexName(parserContext.getIndexSettings().getIndex().getName())); Map, MetadataFieldMapper> metadataMappers = metadataMappersSupplier.get(); Map meta = null; From efb167aae49b631205bb5c70292febdc0ff72030 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Fri, 15 Oct 2021 13:12:00 +0200 Subject: [PATCH 11/15] import --- .../org/elasticsearch/index/mapper/MapperBuilderContext.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperBuilderContext.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperBuilderContext.java index 6220c5ed0496a..23b1bd50fcef8 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperBuilderContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperBuilderContext.java @@ -9,7 +9,6 @@ package org.elasticsearch.index.mapper; import org.elasticsearch.common.Strings; -import org.elasticsearch.index.Index; /** * Holds context for building Mapper objects from their Builders From 09c56c599cb49d1d19347e80497e87961abe1753 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Fri, 15 Oct 2021 15:46:39 +0200 Subject: [PATCH 12/15] store indexname in a builder --- .../index/mapper/DateFieldMapper.java | 26 ++++++++++++++----- .../index/mapper/DynamicFieldsBuilder.java | 3 ++- .../index/mapper/MapperBuilderContext.java | 12 --------- .../index/mapper/MappingParser.java | 4 +-- .../MetadataRolloverServiceTests.java | 4 +-- .../index/mapper/DateFieldMapperTests.java | 3 +-- 6 files changed, 26 insertions(+), 26 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 7bfb2057c8318..2d6cb1f530ead 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -73,6 +73,7 @@ public final class DateFieldMapper extends FieldMapper { public static final DateFormatter DEFAULT_DATE_TIME_NANOS_FORMATTER = DateFormatter.forPattern("strict_date_optional_time_nanos||epoch_millis"); private static final DateMathParser EPOCH_MILLIS_PARSER = DateFormatter.forPattern("epoch_millis").toDateMathParser(); + private final String indexName; public enum Resolution { MILLISECONDS(CONTENT_TYPE, NumericType.DATE) { @@ -231,6 +232,7 @@ public static class Builder extends FieldMapper.Builder { private final Parameter nullValue = Parameter.stringParam("null_value", false, m -> toType(m).nullValueAsString, null).acceptsNull(); private final Parameter ignoreMalformed; + private String indexName; private final Parameter