From 4616cc7cc1dfcca5f0ed93ab5e602b1b36bf1f30 Mon Sep 17 00:00:00 2001 From: Julie Tibshirani Date: Tue, 30 Jun 2020 13:39:04 -0700 Subject: [PATCH] Return null_value when the source contains a 'null' for the field. (#58623) This PR adds a version of `XContentMapValues.extractValue` that accepts a default value to return in place of 'null'. It then uses this method when looking up source values to return the configured `null_value` instead of 'null' when retrieving fields. --- .../index/mapper/ScaledFloatFieldMapper.java | 16 +++- .../mapper/ScaledFloatFieldMapperTests.java | 14 +++- .../ICUCollationKeywordFieldMapper.java | 5 ++ .../ICUCollationKeywordFieldMapperTests.java | 13 +++- .../xcontent/support/XContentMapValues.java | 74 ++++++++++++++----- .../index/mapper/BooleanFieldMapper.java | 4 + .../index/mapper/DateFieldMapper.java | 7 +- .../index/mapper/FieldMapper.java | 10 ++- .../index/mapper/IpFieldMapper.java | 12 ++- .../index/mapper/KeywordFieldMapper.java | 5 ++ .../index/mapper/NumberFieldMapper.java | 10 +++ .../search/lookup/SourceLookup.java | 17 ++++- .../support/XContentMapValuesTests.java | 29 ++++++++ .../index/mapper/BooleanFieldMapperTests.java | 15 +++- .../index/mapper/DateFieldMapperTests.java | 26 +++++++ .../index/mapper/IpFieldMapperTests.java | 12 ++- .../index/mapper/KeywordFieldMapperTests.java | 8 ++ .../index/mapper/NumberFieldMapperTests.java | 13 +++- .../mapper/FlatObjectFieldMapper.java | 5 ++ .../mapper/FlatObjectFieldMapperTests.java | 24 ++++++ .../wildcard/mapper/WildcardFieldMapper.java | 5 ++ .../mapper/WildcardFieldMapperTests.java | 10 +++ 22 files changed, 301 insertions(+), 33 deletions(-) diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java index 118220def8313..41f4cd1dfce9f 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java @@ -355,6 +355,11 @@ protected ScaledFloatFieldMapper clone() { return (ScaledFloatFieldMapper) super.clone(); } + @Override + protected Double nullValue() { + return nullValue; + } + @Override protected void parseCreateField(ParseContext context) throws IOException { @@ -479,7 +484,16 @@ protected Double parseSourceValue(Object value, String format) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } - double doubleValue = objectToDouble(value); + double doubleValue; + if (value.equals("")) { + if (nullValue == null) { + return null; + } + doubleValue = nullValue; + } else { + doubleValue = objectToDouble(value); + } + double scalingFactor = fieldType().getScalingFactor(); return Math.round(doubleValue * scalingFactor) / scalingFactor; } diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java index bcbf6fb432f79..6758750f66581 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java @@ -32,6 +32,7 @@ import org.elasticsearch.index.IndexService; import org.elasticsearch.index.mapper.MapperService.MergeReason; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.test.InternalSettingsPlugin; import org.junit.Before; @@ -405,11 +406,22 @@ public void testMeta() throws Exception { public void testParseSourceValue() { Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); + ScaledFloatFieldMapper mapper = new ScaledFloatFieldMapper.Builder("field") .scalingFactor(100) .build(context); - assertEquals(3.14, mapper.parseSourceValue(3.1415926, null), 0.00001); assertEquals(3.14, mapper.parseSourceValue("3.1415", null), 0.00001); + assertNull(mapper.parseSourceValue("", null)); + + ScaledFloatFieldMapper nullValueMapper = new ScaledFloatFieldMapper.Builder("field") + .scalingFactor(100) + .nullValue(2.71) + .build(context); + assertEquals(2.71, nullValueMapper.parseSourceValue("", null), 0.00001); + + SourceLookup sourceLookup = new SourceLookup(); + sourceLookup.setSource(Collections.singletonMap("field", null)); + assertEquals(List.of(2.71), nullValueMapper.lookupValues(sourceLookup, null)); } } diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java index 7ee03aec31287..a7a1f8686e257 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java @@ -577,6 +577,11 @@ protected String contentType() { return CONTENT_TYPE; } + @Override + protected String nullValue() { + return nullValue; + } + @Override protected void mergeOptions(FieldMapper other, List conflicts) { ICUCollationKeywordFieldMapper icuMergeWith = (ICUCollationKeywordFieldMapper) other; diff --git a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperTests.java b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperTests.java index 1784a74d11f35..b809b0fb87139 100644 --- a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperTests.java +++ b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperTests.java @@ -38,12 +38,15 @@ import org.elasticsearch.index.mapper.MapperService.MergeReason; import org.elasticsearch.plugin.analysis.icu.AnalysisICUPlugin; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.test.InternalSettingsPlugin; import org.junit.Before; import java.io.IOException; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; +import java.util.List; import java.util.Set; import static org.hamcrest.Matchers.containsString; @@ -489,9 +492,8 @@ public void testUpdateIgnoreAbove() throws IOException { public void testParseSourceValue() { Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); - ICUCollationKeywordFieldMapper mapper = new ICUCollationKeywordFieldMapper.Builder("field").build(context); - assertEquals("value", mapper.parseSourceValue("value", null)); + ICUCollationKeywordFieldMapper mapper = new ICUCollationKeywordFieldMapper.Builder("field").build(context); assertEquals("42", mapper.parseSourceValue(42L, null)); assertEquals("true", mapper.parseSourceValue(true, null)); @@ -501,5 +503,12 @@ public void testParseSourceValue() { assertNull(ignoreAboveMapper.parseSourceValue("value", null)); assertEquals("42", ignoreAboveMapper.parseSourceValue(42L, null)); assertEquals("true", ignoreAboveMapper.parseSourceValue(true, null)); + + ICUCollationKeywordFieldMapper nullValueMapper = new ICUCollationKeywordFieldMapper.Builder("field") + .nullValue("NULL") + .build(context); + SourceLookup sourceLookup = new SourceLookup(); + sourceLookup.setSource(Collections.singletonMap("field", null)); + assertEquals(List.of("NULL"), nullValueMapper.lookupValues(sourceLookup, null)); } } diff --git a/server/src/main/java/org/elasticsearch/common/xcontent/support/XContentMapValues.java b/server/src/main/java/org/elasticsearch/common/xcontent/support/XContentMapValues.java index dfbb507365fa9..df5b419e527e9 100644 --- a/server/src/main/java/org/elasticsearch/common/xcontent/support/XContentMapValues.java +++ b/server/src/main/java/org/elasticsearch/common/xcontent/support/XContentMapValues.java @@ -97,6 +97,16 @@ private static void extractRawValues(List values, List part, String[] pa } } + /** + * For the provided path, return its value in the xContent map. + * + * Note that in contrast with {@link XContentMapValues#extractRawValues}, array and object values + * can be returned. + * + * @param path the value's path in the map. + * + * @return the value associated with the path in the map or 'null' if the path does not exist. + */ public static Object extractValue(String path, Map map) { return extractValue(map, path.split("\\.")); } @@ -105,19 +115,51 @@ public static Object extractValue(Map map, String... pathElements) { if (pathElements.length == 0) { return null; } - return extractValue(pathElements, 0, map); + return XContentMapValues.extractValue(pathElements, 0, map, null); } - @SuppressWarnings({"unchecked"}) - private static Object extractValue(String[] pathElements, int index, Object currentValue) { - if (index == pathElements.length) { - return currentValue; - } - if (currentValue == null) { + /** + * For the provided path, return its value in the xContent map. + * + * Note that in contrast with {@link XContentMapValues#extractRawValues}, array and object values + * can be returned. + * + * @param path the value's path in the map. + * @param nullValue a value to return if the path exists, but the value is 'null'. This helps + * in distinguishing between a path that doesn't exist vs. a value of 'null'. + * + * @return the value associated with the path in the map or 'null' if the path does not exist. + */ + public static Object extractValue(String path, Map map, Object nullValue) { + String[] pathElements = path.split("\\."); + if (pathElements.length == 0) { return null; } + return extractValue(pathElements, 0, map, nullValue); + } + + private static Object extractValue(String[] pathElements, + int index, + Object currentValue, + Object nullValue) { + if (currentValue instanceof List) { + List valueList = (List) currentValue; + List newList = new ArrayList<>(valueList.size()); + for (Object o : valueList) { + Object listValue = extractValue(pathElements, index, o, nullValue); + if (listValue != null) { + newList.add(listValue); + } + } + return newList; + } + + if (index == pathElements.length) { + return currentValue != null ? currentValue : nullValue; + } + if (currentValue instanceof Map) { - Map map = (Map) currentValue; + Map map = (Map) currentValue; String key = pathElements[index]; Object mapValue = map.get(key); int nextIndex = index + 1; @@ -126,18 +168,12 @@ private static Object extractValue(String[] pathElements, int index, Object curr mapValue = map.get(key); nextIndex++; } - return extractValue(pathElements, nextIndex, mapValue); - } - if (currentValue instanceof List) { - List valueList = (List) currentValue; - List newList = new ArrayList(valueList.size()); - for (Object o : valueList) { - Object listValue = extractValue(pathElements, index, o); - if (listValue != null) { - newList.add(listValue); - } + + if (map.containsKey(key) == false) { + return null; } - return newList; + + return extractValue(pathElements, nextIndex, mapValue, nullValue); } return null; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java index a2686d0c3bc50..c849b00b1149b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java @@ -281,4 +281,8 @@ protected String contentType() { return CONTENT_TYPE; } + @Override + protected Object nullValue() { + return nullValue; + } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 3e964b26f1d4e..d00a878d51485 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -538,6 +538,11 @@ protected DateFieldMapper clone() { return (DateFieldMapper) super.clone(); } + @Override + protected String nullValue() { + return nullValueAsString; + } + @Override protected void parseCreateField(ParseContext context) throws IOException { String dateAsString; @@ -588,8 +593,8 @@ protected void parseCreateField(ParseContext context) throws IOException { public String parseSourceValue(Object value, String format) { String date = value.toString(); long timestamp = fieldType().parse(date); - ZonedDateTime dateTime = fieldType().resolution().toInstant(timestamp).atZone(ZoneOffset.UTC); + ZonedDateTime dateTime = fieldType().resolution().toInstant(timestamp).atZone(ZoneOffset.UTC); DateFormatter dateTimeFormatter = fieldType().dateTimeFormatter(); if (format != null) { dateTimeFormatter = DateFormatter.forPattern(format).withLocale(dateTimeFormatter.locale()); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java index c2dfa86819126..58bbf9c2b8379 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java @@ -224,6 +224,13 @@ public CopyTo copyTo() { return copyTo; } + /** + * A value to use in place of a {@code null} value in the document source. + */ + protected Object nullValue() { + return null; + } + /** * Whether this mapper can handle an array value during document parsing. If true, * when an array is encountered during parsing, the document parser will pass the @@ -286,7 +293,7 @@ public void parse(ParseContext context) throws IOException { * @return a list a standardized field values. */ public List lookupValues(SourceLookup lookup, @Nullable String format) { - Object sourceValue = lookup.extractValue(name()); + Object sourceValue = lookup.extractValue(name(), nullValue()); if (sourceValue == null) { return List.of(); } @@ -338,6 +345,7 @@ protected FieldMapper clone() { } } + @Override public FieldMapper merge(Mapper mergeWith) { FieldMapper merged = clone(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java index aa2c8f6ea1467..ca8f58e92b46c 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java @@ -348,6 +348,11 @@ protected String contentType() { return fieldType().typeName(); } + @Override + protected Object nullValue() { + return nullValue; + } + @Override protected IpFieldMapper clone() { return (IpFieldMapper) super.clone(); @@ -406,7 +411,12 @@ protected String parseSourceValue(Object value, String format) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } - InetAddress address = InetAddresses.forString(value.toString()); + InetAddress address; + if (value instanceof InetAddress) { + address = (InetAddress) value; + } else { + address = InetAddresses.forString(value.toString()); + } return InetAddresses.toAddrString(address); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 239cb88e1b92d..df4de525b7ada 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -416,6 +416,11 @@ protected String contentType() { return CONTENT_TYPE; } + @Override + protected String nullValue() { + return nullValue; + } + @Override protected void mergeOptions(FieldMapper other, List conflicts) { KeywordFieldMapper k = (KeywordFieldMapper) other; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index 4c007bfa81204..39cb363b74b41 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -1036,6 +1036,11 @@ protected NumberFieldMapper clone() { return (NumberFieldMapper) super.clone(); } + @Override + protected Number nullValue() { + return nullValue; + } + @Override protected void parseCreateField(ParseContext context) throws IOException { XContentParser parser = context.parser(); @@ -1090,6 +1095,11 @@ protected Number parseSourceValue(Object value, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } + + if (value.equals("")) { + return nullValue; + } + return fieldType().type.parse(value, coerce.value()); } diff --git a/server/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java b/server/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java index 6393d4bf50ff0..d63caed14adb0 100644 --- a/server/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java +++ b/server/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java @@ -21,6 +21,7 @@ import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.xcontent.XContentHelper; @@ -133,11 +134,19 @@ public List extractRawValues(String path) { } /** - * For the provided path, return its value in the source. Note that in contrast with - * {@link SourceLookup#extractRawValues}, array and object values can be returned. + * For the provided path, return its value in the source. + * + * Note that in contrast with {@link SourceLookup#extractRawValues}, array and object values + * can be returned. + * + * @param path the value's path in the source. + * @param nullValue a value to return if the path exists, but the value is 'null'. This helps + * in distinguishing between a path that doesn't exist vs. a value of 'null'. + * + * @return the value associated with the path in the source or 'null' if the path does not exist. */ - public Object extractValue(String path) { - return XContentMapValues.extractValue(path, loadSourceIfNeeded()); + public Object extractValue(String path, @Nullable Object nullValue) { + return XContentMapValues.extractValue(path, loadSourceIfNeeded(), nullValue); } public Object filter(FetchSourceContext context) { diff --git a/server/src/test/java/org/elasticsearch/common/xcontent/support/XContentMapValuesTests.java b/server/src/test/java/org/elasticsearch/common/xcontent/support/XContentMapValuesTests.java index d83000bd66956..957316d99dad8 100644 --- a/server/src/test/java/org/elasticsearch/common/xcontent/support/XContentMapValuesTests.java +++ b/server/src/test/java/org/elasticsearch/common/xcontent/support/XContentMapValuesTests.java @@ -164,6 +164,35 @@ public void testExtractValue() throws Exception { assertThat(XContentMapValues.extractValue("path1.xxx.path2.yyy.test", map).toString(), equalTo("value")); } + public void testExtractValueWithNullValue() throws Exception { + XContentBuilder builder = XContentFactory.jsonBuilder().startObject() + .field("field", "value") + .nullField("other_field") + .array("array", "value1", null, "value2") + .startObject("object1") + .startObject("object2").nullField("field").endObject() + .endObject() + .startArray("object_array") + .startObject().nullField("field").endObject() + .startObject().field("field", "value").endObject() + .endArray() + .endObject(); + + Map map; + try (XContentParser parser = createParser(JsonXContent.jsonXContent, Strings.toString(builder))) { + map = parser.map(); + } + assertEquals("value", XContentMapValues.extractValue("field", map, "NULL")); + assertNull(XContentMapValues.extractValue("missing", map, "NULL")); + assertNull(XContentMapValues.extractValue("field.missing", map, "NULL")); + assertNull(XContentMapValues.extractValue("object1.missing", map, "NULL")); + + assertEquals("NULL", XContentMapValues.extractValue("other_field", map, "NULL")); + assertEquals(List.of("value1", "NULL", "value2"), XContentMapValues.extractValue("array", map, "NULL")); + assertEquals(List.of("NULL", "value"), XContentMapValues.extractValue("object_array.field", map, "NULL")); + assertEquals("NULL", XContentMapValues.extractValue("object1.object2.field", map, "NULL")); + } + public void testExtractRawValue() throws Exception { XContentBuilder builder = XContentFactory.jsonBuilder().startObject() .field("test", "value") diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java index 5c34fb0017e79..253084ec4ddfa 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java @@ -47,6 +47,7 @@ import org.elasticsearch.index.mapper.MapperService.MergeReason; import org.elasticsearch.index.mapper.ParseContext.Document; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.test.InternalSettingsPlugin; import org.junit.Before; @@ -54,6 +55,9 @@ import java.io.IOException; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import static org.hamcrest.Matchers.containsString; @@ -299,10 +303,19 @@ public void testBoosts() throws Exception { public void testParseSourceValue() { Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); - BooleanFieldMapper mapper = new BooleanFieldMapper.Builder("field").build(context); + BooleanFieldMapper mapper = new BooleanFieldMapper.Builder("field").build(context); assertTrue(mapper.parseSourceValue(true, null)); assertFalse(mapper.parseSourceValue("false", null)); assertFalse(mapper.parseSourceValue("", null)); + + Map mapping = Map.of("type", "boolean", "null_value", true); + BooleanFieldMapper.Builder builder = new BooleanFieldMapper.Builder("field"); + builder.parse("field", null, new HashMap<>(mapping)); + BooleanFieldMapper nullValueMapper = builder.build(context); + + SourceLookup sourceLookup = new SourceLookup(); + sourceLookup.setSource(Collections.singletonMap("field", null)); + assertEquals(List.of(true), nullValueMapper.lookupValues(sourceLookup, null)); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java index ad6067caf9262..8323d74912df6 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java @@ -35,6 +35,7 @@ import org.elasticsearch.index.mapper.MapperService.MergeReason; import org.elasticsearch.index.termvectors.TermVectorsService; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.test.InternalSettingsPlugin; import org.junit.Before; @@ -44,6 +45,7 @@ import java.time.ZonedDateTime; import java.util.Collection; import java.util.Collections; +import java.util.List; import java.util.Locale; import java.util.Set; @@ -489,6 +491,14 @@ public void testParseSourceValue() { String dateInMillis = "662428800000"; assertEquals(dateInMillis, mapperWithMillis.parseSourceValue(dateInMillis, null)); assertEquals(dateInMillis, mapperWithMillis.parseSourceValue(662428800000L, null)); + + String nullValueDate = "2020-05-15T21:33:02.000Z"; + DateFieldMapper nullValueMapper = new DateFieldMapper.Builder("field") + .nullValue(nullValueDate) + .build(context); + SourceLookup sourceLookup = new SourceLookup(); + sourceLookup.setSource(Collections.singletonMap("field", null)); + assertEquals(List.of(nullValueDate), nullValueMapper.lookupValues(sourceLookup, null)); } public void testParseSourceValueWithFormat() { @@ -497,10 +507,15 @@ public void testParseSourceValueWithFormat() { DateFieldMapper mapper = new DateFieldMapper.Builder("field") .format("strict_date_time") + .nullValue("1970-12-29T00:00:00.000Z") .build(context); String date = "1990-12-29T00:00:00.000Z"; assertEquals("1990/12/29", mapper.parseSourceValue(date, "yyyy/MM/dd")); assertEquals("662428800000", mapper.parseSourceValue(date, "epoch_millis")); + + SourceLookup sourceLookup = new SourceLookup(); + sourceLookup.setSource(Collections.singletonMap("field", null)); + assertEquals(List.of("1970/12/29"), mapper.lookupValues(sourceLookup, "yyyy/MM/dd")); } public void testParseSourceValueNanos() { @@ -514,5 +529,16 @@ public void testParseSourceValueNanos() { String date = "2020-05-15T21:33:02.123456789Z"; assertEquals("2020-05-15T21:33:02.123456789Z", mapper.parseSourceValue(date, null)); assertEquals("2020-05-15T21:33:02.123Z", mapper.parseSourceValue(1589578382123L, null)); + + String nullValueDate = "2020-05-15T21:33:02.123456789Z"; + DateFieldMapper nullValueMapper = new DateFieldMapper.Builder("field") + .format("strict_date_time||epoch_millis") + .nullValue(nullValueDate) + .withResolution(DateFieldMapper.Resolution.NANOSECONDS) + .build(context); + + SourceLookup sourceLookup = new SourceLookup(); + sourceLookup.setSource(Collections.singletonMap("field", null)); + assertEquals(List.of(nullValueDate), nullValueMapper.lookupValues(sourceLookup, null)); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java index 055162c1b0862..405efdecb8248 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java @@ -40,12 +40,15 @@ import org.elasticsearch.index.IndexService; import org.elasticsearch.index.termvectors.TermVectorsService; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.test.InternalSettingsPlugin; import org.junit.Before; import java.io.IOException; import java.net.InetAddress; import java.util.Collection; +import java.util.Collections; +import java.util.List; import java.util.Set; import static org.hamcrest.Matchers.containsString; @@ -303,11 +306,18 @@ public void testEmptyName() throws IOException { public void testParseSourceValue() { Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); - IpFieldMapper mapper = new IpFieldMapper.Builder("field").build(context); + IpFieldMapper mapper = new IpFieldMapper.Builder("field").build(context); assertEquals("2001:db8::2:1", mapper.parseSourceValue("2001:db8::2:1", null)); assertEquals("2001:db8::2:1", mapper.parseSourceValue("2001:db8:0:0:0:0:2:1", null)); assertEquals("::1", mapper.parseSourceValue("0:0:0:0:0:0:0:1", null)); + + IpFieldMapper nullValueMapper = new IpFieldMapper.Builder("field") + .nullValue(InetAddresses.forString("2001:db8:0:0:0:0:2:7")) + .build(context); + SourceLookup sourceLookup = new SourceLookup(); + sourceLookup.setSource(Collections.singletonMap("field", null)); + assertEquals(List.of("2001:db8::2:7"), nullValueMapper.lookupValues(sourceLookup, null)); } @Override diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java index 74bf554ff2b44..0c6302779f6bf 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java @@ -46,6 +46,7 @@ import org.elasticsearch.indices.analysis.AnalysisModule; import org.elasticsearch.plugins.AnalysisPlugin; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.test.InternalSettingsPlugin; import org.junit.Before; @@ -648,5 +649,12 @@ public void testParseSourceValue() { assertNull(ignoreAboveMapper.parseSourceValue("value", null)); assertEquals("42", ignoreAboveMapper.parseSourceValue(42L, null)); assertEquals("true", ignoreAboveMapper.parseSourceValue(true, null)); + + KeywordFieldMapper nullValueMapper = new KeywordFieldMapper.Builder("field") + .nullValue("NULL") + .build(context); + SourceLookup sourceLookup = new SourceLookup(); + sourceLookup.setSource(Collections.singletonMap("field", null)); + assertEquals(List.of("NULL"), nullValueMapper.lookupValues(sourceLookup, null)); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java index 9f89122c10aa1..c2a60272225bb 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java @@ -38,11 +38,13 @@ import org.elasticsearch.index.mapper.NumberFieldTypeTests.OutOfRangeSpec; import org.elasticsearch.index.termvectors.TermVectorsService; import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.search.lookup.SourceLookup; import java.io.ByteArrayInputStream; import java.io.IOException; import java.math.BigInteger; import java.util.Arrays; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -407,10 +409,19 @@ public void testEmptyName() throws IOException { public void testParseSourceValue() { Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); - NumberFieldMapper mapper = new NumberFieldMapper.Builder("field", NumberType.INTEGER).build(context); + NumberFieldMapper mapper = new NumberFieldMapper.Builder("field", NumberType.INTEGER).build(context); assertEquals(3, mapper.parseSourceValue(3.14, null)); assertEquals(42, mapper.parseSourceValue("42.9", null)); + + NumberFieldMapper nullValueMapper = new NumberFieldMapper.Builder("field", NumberType.FLOAT) + .nullValue(2.71f) + .build(context); + assertEquals(2.71f, (float) nullValueMapper.parseSourceValue("", null), 0.00001); + + SourceLookup sourceLookup = new SourceLookup(); + sourceLookup.setSource(Collections.singletonMap("field", null)); + assertEquals(List.of(2.71f), nullValueMapper.lookupValues(sourceLookup, null)); } @Timeout(millis = 30000) diff --git a/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java b/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java index 44ab855f7de27..c02559487e32e 100644 --- a/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java +++ b/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java @@ -524,6 +524,11 @@ protected String contentType() { return CONTENT_TYPE; } + @Override + protected String nullValue() { + return nullValue; + } + @Override protected void mergeOptions(FieldMapper mergeWith, List conflicts) { FlatObjectFieldMapper other = ((FlatObjectFieldMapper) mergeWith); diff --git a/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapperTests.java b/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapperTests.java index 7a0e1a26d79bb..21f12b3a16d0e 100644 --- a/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapperTests.java +++ b/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapperTests.java @@ -9,24 +9,30 @@ import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexableField; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.mapper.ContentPath; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.DocumentMapperParser; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.FieldMapperTestCase; import org.elasticsearch.index.mapper.FieldNamesFieldMapper; +import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MapperService.MergeReason; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.mapper.SourceToParse; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; import org.elasticsearch.xpack.flattened.FlattenedMapperPlugin; import org.elasticsearch.xpack.flattened.mapper.FlatObjectFieldMapper.KeyedFlatObjectFieldType; @@ -36,6 +42,9 @@ import java.io.IOException; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; import java.util.Set; import static org.apache.lucene.analysis.BaseTokenStreamTestCase.assertTokenStreamContents; @@ -506,4 +515,19 @@ public void testSplitQueriesOnWhitespace() throws IOException { new String[] {"Hello", "World"}); } + public void testParseSourceValue() { + Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); + Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); + + Map sourceValue = Map.of("key", "value"); + FlatObjectFieldMapper mapper = new FlatObjectFieldMapper.Builder("field").build(context); + assertEquals(sourceValue, mapper.parseSourceValue(sourceValue, null)); + + FlatObjectFieldMapper nullValueMapper = new FlatObjectFieldMapper.Builder("field") + .nullValue("NULL") + .build(context); + SourceLookup sourceLookup = new SourceLookup(); + sourceLookup.setSource(Collections.singletonMap("field", null)); + assertEquals(List.of("NULL"), nullValueMapper.lookupValues(sourceLookup, null)); + } } diff --git a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java index 01ff1be1de5ed..01d5755fb6c19 100644 --- a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java +++ b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java @@ -991,6 +991,11 @@ protected String contentType() { return CONTENT_TYPE; } + @Override + protected String nullValue() { + return nullValue; + } + @Override protected void mergeOptions(FieldMapper other, List conflicts) { this.ignoreAbove = ((WildcardFieldMapper) other).ignoreAbove; diff --git a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java index a3176576c97ec..54e3b9f7757dc 100644 --- a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java +++ b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java @@ -55,6 +55,7 @@ import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.search.sort.FieldSortBuilder; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.IndexSettingsModule; @@ -64,7 +65,9 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.HashSet; +import java.util.List; import java.util.function.BiFunction; import static org.hamcrest.Matchers.equalTo; @@ -789,6 +792,13 @@ public void testParseSourceValue() { assertNull(ignoreAboveMapper.parseSourceValue("value", null)); assertEquals("42", ignoreAboveMapper.parseSourceValue(42L, null)); assertEquals("true", ignoreAboveMapper.parseSourceValue(true, null)); + + WildcardFieldMapper nullValueMapper = new WildcardFieldMapper.Builder("field") + .nullValue("NULL") + .build(context); + SourceLookup sourceLookup = new SourceLookup(); + sourceLookup.setSource(Collections.singletonMap("field", null)); + assertEquals(List.of("NULL"), nullValueMapper.lookupValues(sourceLookup, null)); } protected MappedFieldType provideMappedFieldType(String name) {