From c3a3963304a81a953bc87f25481cb188f7fcad02 Mon Sep 17 00:00:00 2001 From: Julie Tibshirani Date: Thu, 28 May 2020 08:11:19 -0700 Subject: [PATCH] Allow field mappers to retrieve fields from source. (#56928) This PR adds new method `FieldMapper#lookupValues(SourceLookup)` that extracts and parses the source values. This lets us return values like numbers and dates in a consistent format, and also handle special data types like `constant_keyword`. The `lookupValues` method calls into `parseSourceValue`, which mappers can override to specify how values should be parsed. --- .../index/mapper/RankFeatureFieldMapper.java | 19 ++++-- .../index/mapper/RankFeaturesFieldMapper.java | 5 ++ .../index/mapper/ScaledFloatFieldMapper.java | 7 ++ .../mapper/SearchAsYouTypeFieldMapper.java | 15 +++++ .../index/mapper/TokenCountFieldMapper.java | 5 ++ .../mapper/RankFeatureFieldMapperTests.java | 11 ++++ .../mapper/ScaledFloatFieldMapperTests.java | 14 ++++ .../join/mapper/MetaJoinFieldMapper.java | 5 ++ .../join/mapper/ParentIdFieldMapper.java | 5 ++ .../join/mapper/ParentJoinFieldMapper.java | 5 ++ .../percolator/PercolatorFieldMapper.java | 5 ++ .../ICUCollationKeywordFieldMapper.java | 5 ++ .../ICUCollationKeywordFieldMapperTests.java | 12 ++++ .../AnnotatedTextFieldMapper.java | 5 ++ .../AnnotatedTextFieldMapperTests.java | 17 +++++ .../mapper/murmur3/Murmur3FieldMapper.java | 6 +- .../test/search/330_fetch_fields.yml | 54 +++++++++++++-- .../common/network/InetAddresses.java | 13 ++++ .../mapper/AbstractGeometryFieldMapper.java | 5 ++ .../index/mapper/BinaryFieldMapper.java | 4 ++ .../index/mapper/BooleanFieldMapper.java | 11 ++++ .../index/mapper/CompletionFieldMapper.java | 9 +++ .../index/mapper/DateFieldMapper.java | 11 ++++ .../index/mapper/FieldMapper.java | 42 ++++++++++++ .../index/mapper/IpFieldMapper.java | 6 ++ .../index/mapper/KeywordFieldMapper.java | 5 ++ .../index/mapper/MetadataFieldMapper.java | 5 ++ .../index/mapper/NumberFieldMapper.java | 5 ++ .../index/mapper/RangeFieldMapper.java | 21 ++++++ .../elasticsearch/index/mapper/RangeType.java | 53 +++++++++++---- .../index/mapper/TextFieldMapper.java | 15 +++++ .../fetch/subphase/FetchFieldsPhase.java | 19 +++++- .../fetch/subphase/FieldValueRetriever.java | 65 +++++++------------ .../search/lookup/SourceLookup.java | 8 +++ .../index/mapper/BooleanFieldMapperTests.java | 14 +++- .../mapper/CompletionFieldMapperTests.java | 17 +++++ .../index/mapper/DateFieldMapperTests.java | 39 +++++++++++ .../mapper/DocumentFieldMapperTests.java | 5 ++ .../index/mapper/ExternalMapper.java | 5 ++ .../index/mapper/FakeStringFieldMapper.java | 5 ++ .../index/mapper/IpFieldMapperTests.java | 13 ++++ .../index/mapper/IpRangeFieldMapperTests.java | 13 ++++ .../index/mapper/KeywordFieldMapperTests.java | 15 +++++ .../index/mapper/NumberFieldMapperTests.java | 12 ++++ .../index/mapper/RangeFieldMapperTests.java | 20 ++++++ .../index/mapper/RangeFieldTypeTests.java | 10 +-- .../index/mapper/TextFieldMapperTests.java | 12 ++++ .../subphase/FieldValueRetrieverTests.java | 61 ++++++++++++++--- .../index/mapper/MockFieldMapper.java | 5 ++ .../mapper/HistogramFieldMapper.java | 5 ++ .../mapper/ConstantKeywordFieldMapper.java | 13 ++++ .../ConstantKeywordFieldMapperTests.java | 26 ++++++++ .../mapper/FlatObjectFieldMapper.java | 5 ++ .../test/constant_keyword/10_basic.yml | 15 +++++ .../mapper/DenseVectorFieldMapper.java | 5 ++ .../mapper/SparseVectorFieldMapper.java | 6 +- .../wildcard/mapper/WildcardFieldMapper.java | 5 ++ 57 files changed, 737 insertions(+), 81 deletions(-) diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java index 93f7cb4230bf3..2023d5686a13a 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java @@ -162,11 +162,7 @@ protected void parseCreateField(ParseContext context) throws IOException { float value; if (context.externalValueSet()) { Object v = context.externalValue(); - if (v instanceof Number) { - value = ((Number) v).floatValue(); - } else { - value = Float.parseFloat(v.toString()); - } + value = objectToFloat(v); } else if (context.parser().currentToken() == Token.VALUE_NULL) { // skip return; @@ -186,6 +182,19 @@ protected void parseCreateField(ParseContext context) throws IOException { context.doc().addWithKey(name(), new FeatureField("_feature", name(), value)); } + private Float objectToFloat(Object value) { + if (value instanceof Number) { + return ((Number) value).floatValue(); + } else { + return Float.parseFloat(value.toString()); + } + } + + @Override + protected Float parseSourceValue(Object value) { + return objectToFloat(value); + } + @Override protected String contentType() { return CONTENT_TYPE; diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java index b41bc86aa9a78..4faac8b29bbe0 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java @@ -168,6 +168,11 @@ protected void parseCreateField(ParseContext context) throws IOException { throw new AssertionError("parse is implemented directly"); } + @Override + protected Object parseSourceValue(Object value) { + return value; + } + @Override protected boolean indexedByDefault() { return false; diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java index 88c55c9554084..6f0da247fabfd 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java @@ -496,6 +496,13 @@ private static double objectToDouble(Object value) { return doubleValue; } + @Override + protected Double parseSourceValue(Object value) { + double doubleValue = objectToDouble(value); + double scalingFactor = fieldType().getScalingFactor(); + return Math.round(doubleValue * scalingFactor) / scalingFactor; + } + private static class ScaledFloatIndexFieldData extends IndexNumericFieldData { private final IndexNumericFieldData scaledFieldData; diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java index a38f770026596..f21bd51e1f8ad 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java @@ -491,6 +491,11 @@ protected void parseCreateField(ParseContext context) { throw new UnsupportedOperationException(); } + @Override + protected Object parseSourceValue(Object value) { + throw new UnsupportedOperationException(); + } + @Override protected void mergeOptions(FieldMapper other, List conflicts) { @@ -532,6 +537,11 @@ protected void mergeOptions(FieldMapper other, List conflicts) { } + @Override + protected Object parseSourceValue(Object value) { + throw new UnsupportedOperationException(); + } + @Override protected String contentType() { return "shingle"; @@ -684,6 +694,11 @@ protected void parseCreateField(ParseContext context) throws IOException { } } + @Override + protected String parseSourceValue(Object value) { + return value.toString(); + } + @Override protected String contentType() { return CONTENT_TYPE; diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java index fb6a41a0f7608..51da52e37e22f 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java @@ -158,6 +158,11 @@ protected void parseCreateField(ParseContext context) throws IOException { context.doc().addAll(NumberFieldMapper.NumberType.INTEGER.createFields(fieldType().name(), tokenCount, indexed, docValued, stored)); } + @Override + protected String parseSourceValue(Object value) { + return value.toString(); + } + /** * Count position increments in a token stream. Package private for testing. * @param analyzer analyzer to create token stream diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureFieldMapperTests.java index 5fa95e6e15f2f..e867370acc8ed 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureFieldMapperTests.java @@ -23,9 +23,12 @@ import org.apache.lucene.analysis.tokenattributes.TermFrequencyAttribute; import org.apache.lucene.document.FeatureField; import org.apache.lucene.index.IndexableField; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.IndexService; @@ -186,4 +189,12 @@ public void testRejectMultiValuedFields() throws MapperParsingException, IOExcep e.getCause().getMessage()); } + public void testParseSourceValue() { + Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); + Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); + RankFeatureFieldMapper mapper = new RankFeatureFieldMapper.Builder("field").build(context); + + assertEquals(3.14f, mapper.parseSourceValue(3.14), 0.0001); + assertEquals(42.9f, mapper.parseSourceValue("42.9"), 0.0001); + } } diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java index 7fa4a81ec5c13..f573c25ba176d 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java @@ -21,9 +21,12 @@ import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexableField; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.IndexService; @@ -398,4 +401,15 @@ public void testMeta() throws Exception { new CompressedXContent(mapping3), MergeReason.MAPPING_UPDATE); assertEquals(mapping3, mapper.mappingSource().toString()); } + + public void testParseSourceValue() { + Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); + Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); + ScaledFloatFieldMapper mapper = new ScaledFloatFieldMapper.Builder("field") + .scalingFactor(100) + .build(context); + + assertEquals(3.14, mapper.parseSourceValue(3.1415926), 0.00001); + assertEquals(3.14, mapper.parseSourceValue("3.1415"), 0.00001); + } } diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/MetaJoinFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/MetaJoinFieldMapper.java index 1fc0709f16e4a..51c7c9510abb5 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/MetaJoinFieldMapper.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/MetaJoinFieldMapper.java @@ -144,6 +144,11 @@ protected void parseCreateField(ParseContext context) throws IOException { throw new IllegalStateException("Should never be called"); } + @Override + protected Object parseSourceValue(Object value) { + throw new UnsupportedOperationException("The " + typeName() + " field is not stored in _source."); + } + @Override protected String contentType() { return CONTENT_TYPE; diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java index 6763e88c168ee..7022347173605 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java @@ -194,6 +194,11 @@ protected void parseCreateField(ParseContext context) throws IOException { context.doc().add(new SortedDocValuesField(fieldType().name(), binaryValue)); } + @Override + protected Object parseSourceValue(Object value) { + throw new UnsupportedOperationException("The " + typeName() + " field is not stored in _source."); + } + @Override protected void mergeOptions(FieldMapper other, List conflicts) { ParentIdFieldMapper parentMergeWith = (ParentIdFieldMapper) other; diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java index 54a849b646470..f12dbcf4866b7 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java @@ -356,6 +356,11 @@ protected void parseCreateField(ParseContext context) throws IOException { throw new UnsupportedOperationException("parsing is implemented in parse(), this method should NEVER be called"); } + @Override + protected Object parseSourceValue(Object value) { + return value; + } + @Override public void parse(ParseContext context) throws IOException { context.path().add(simpleName()); diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java index 253f86e149d12..3fa9d1e7ee64d 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java @@ -385,6 +385,11 @@ public void parse(ParseContext context) throws IOException { processQuery(query, context); } + @Override + protected Object parseSourceValue(Object value) { + return value; + } + static void createQueryBuilderField(Version indexVersion, BinaryFieldMapper qbField, QueryBuilder queryBuilder, ParseContext context) throws IOException { try (ByteArrayOutputStream stream = new ByteArrayOutputStream()) { diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java index 6c0aa7b4ddcf5..cb5cc30a06f55 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java @@ -752,4 +752,9 @@ protected void parseCreateField(ParseContext context) throws IOException { createFieldNamesField(context); } } + + @Override + protected String parseSourceValue(Object value) { + return value.toString(); + } } diff --git a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperTests.java b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperTests.java index 21a9cce88fe54..28cd339e36bcd 100644 --- a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperTests.java +++ b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperTests.java @@ -26,9 +26,12 @@ import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.IndexableFieldType; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.IndexService; @@ -483,4 +486,13 @@ public void testUpdateIgnoreAbove() throws IOException { indexService.mapperService().merge("type", new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE); } + public void testParseSourceValue() { + Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); + Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); + ICUCollationKeywordFieldMapper mapper = new ICUCollationKeywordFieldMapper.Builder("field").build(context); + + assertEquals("value", mapper.parseSourceValue("value")); + assertEquals("42", mapper.parseSourceValue(42L)); + assertEquals("true", mapper.parseSourceValue(true)); + } } diff --git a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java index e48315c5901d6..6db1952419af7 100644 --- a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java +++ b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java @@ -595,6 +595,11 @@ protected void parseCreateField(ParseContext context) throws IOException { } } + @Override + protected String parseSourceValue(Object value) { + return value.toString(); + } + @Override protected String contentType() { return CONTENT_TYPE; diff --git a/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java b/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java index 5acc8c9a82280..1f4fdf59bc817 100644 --- a/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java +++ b/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java @@ -28,10 +28,12 @@ import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.Version; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.termvectors.TermVectorsRequest; import org.elasticsearch.action.termvectors.TermVectorsResponse; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.compress.CompressedXContent; @@ -44,8 +46,10 @@ import org.elasticsearch.index.IndexService; import org.elasticsearch.index.VersionType; import org.elasticsearch.index.engine.Engine; +import org.elasticsearch.index.mapper.ContentPath; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.DocumentMapperParser; +import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperService.MergeReason; import org.elasticsearch.index.mapper.ParsedDocument; @@ -672,4 +676,17 @@ public void testEmptyName() throws IOException { assertThat(e.getMessage(), containsString("name cannot be empty string")); } + public void testParseSourceValue() { + Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); + Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); + AnnotatedTextFieldMapper mapper = new AnnotatedTextFieldMapper.Builder("field") + .indexAnalyzer(indexService.getIndexAnalyzers().getDefaultIndexAnalyzer()) + .searchAnalyzer(indexService.getIndexAnalyzers().getDefaultSearchAnalyzer()) + .searchQuoteAnalyzer(indexService.getIndexAnalyzers().getDefaultSearchQuoteAnalyzer()) + .build(context); + + assertEquals("value", mapper.parseSourceValue("value")); + assertEquals("42", mapper.parseSourceValue(42L)); + assertEquals("true", mapper.parseSourceValue(true)); + } } diff --git a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java index 0dd9b5fe76fc2..8e34cdf22f815 100644 --- a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java +++ b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java @@ -156,6 +156,11 @@ protected void parseCreateField(ParseContext context) } } + @Override + protected String parseSourceValue(Object value) { + return value.toString(); + } + @Override protected boolean indexedByDefault() { return false; @@ -165,5 +170,4 @@ protected boolean indexedByDefault() { protected void mergeOptions(FieldMapper other, List conflicts) { } - } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/330_fetch_fields.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/330_fetch_fields.yml index f716e42f9e7eb..5653ae61a6c7d 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/330_fetch_fields.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/330_fetch_fields.yml @@ -21,7 +21,7 @@ setup: index: test id: 1 body: - keyword: [ "first", "second" ] + keyword: [ "x", "y" ] integer_range: gte: 0 lte: 42 @@ -39,8 +39,8 @@ setup: - is_true: hits.hits.0._id - is_true: hits.hits.0._source - - match: { hits.hits.0.fields.keyword.0: first } - - match: { hits.hits.0.fields.keyword.1: second } + - match: { hits.hits.0.fields.keyword.0: x } + - match: { hits.hits.0.fields.keyword.1: y } - match: { hits.hits.0.fields.integer_range.0.gte: 0 } - match: { hits.hits.0.fields.integer_range.0.lte: 42 } @@ -65,7 +65,7 @@ setup: index: test id: 1 body: - keyword: [ "value" ] + keyword: [ "x" ] - do: catch: bad_request @@ -76,3 +76,49 @@ setup: - match: { error.root_cause.0.type: "illegal_argument_exception" } - match: { error.root_cause.0.reason: "Unable to retrieve the requested [fields] since _source is disabled in the mappings for index [test]" } + +--- +"Test ignore malformed": + - do: + indices.create: + index: test + body: + settings: + number_of_shards: 1 + mappings: + properties: + keyword: + type: keyword + integer: + type: integer + ignore_malformed: true + + - do: + index: + index: test + id: 1 + body: + keyword: "x" + integer: 42 + + - do: + index: + index: test + id: 2 + body: + keyword: "y" + integer: "not an integer" + + - do: + indices.refresh: + index: [ test ] + + - do: + search: + index: test + body: + sort: [ keyword ] + fields: [ integer ] + + - match: { hits.hits.0.fields.integer.0: 42 } + - is_false: hits.hits.1.fields.integer diff --git a/server/src/main/java/org/elasticsearch/common/network/InetAddresses.java b/server/src/main/java/org/elasticsearch/common/network/InetAddresses.java index 21c5b4e04b25b..444c291f36a83 100644 --- a/server/src/main/java/org/elasticsearch/common/network/InetAddresses.java +++ b/server/src/main/java/org/elasticsearch/common/network/InetAddresses.java @@ -384,4 +384,17 @@ public static Tuple parseCidr(String maskedAddress) { throw new IllegalArgumentException("Expected [ip/prefix] but was [" + maskedAddress + "]"); } } + + /** + * Given an address and prefix length, returns the string representation of the range in CIDR notation. + * + * See {@link #toAddrString} for details on how the address is represented. + */ + public static String toCidrString(InetAddress address, int prefixLength) { + return new StringBuilder() + .append(toAddrString(address)) + .append("/") + .append(prefixLength) + .toString(); + } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java index 62ed1cea72103..7fb6e60d2bb6c 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java @@ -141,6 +141,11 @@ public Builder ignoreZValue(final boolean ignoreZValue) { } } + @Override + protected Object parseSourceValue(Object value) { + throw new UnsupportedOperationException(); + } + public abstract static class TypeParser implements Mapper.TypeParser { protected abstract T newBuilder(String name, Map params); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java index aef0cd09b463f..6600e7b1e87a1 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java @@ -205,7 +205,11 @@ protected void parseCreateField(ParseContext context) throws IOException { // no doc values createFieldNamesField(context); } + } + @Override + protected Object parseSourceValue(Object value) { + return value; } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java index 9a3894bea5e74..59f30bacdedb2 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java @@ -29,6 +29,7 @@ import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TermRangeQuery; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.Booleans; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; @@ -260,6 +261,16 @@ protected void parseCreateField(ParseContext context) throws IOException { } } + @Override + public Boolean parseSourceValue(Object value) { + if (value instanceof Boolean) { + return (Boolean) value; + } else { + String textValue = value.toString(); + return Booleans.parseBoolean(textValue.toCharArray(), 0, textValue.length(), false); + } + } + @Override protected void mergeOptions(FieldMapper other, List conflicts) { // TODO ban updating null values diff --git a/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java index eddd44a58fbdb..5598a7b6a34d1 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java @@ -623,6 +623,15 @@ private void parse(ParseContext parseContext, Token token, } } + @Override + protected List parseSourceValue(Object value) { + if (value instanceof List) { + return (List) value; + } else { + return List.of(value); + } + } + static class CompletionInputMetadata { public final String input; public final Map> contexts; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 98f222e822477..524bee70fac79 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -59,6 +59,7 @@ import java.time.Instant; import java.time.ZoneId; import java.time.ZoneOffset; +import java.time.ZonedDateTime; import java.util.Collections; import java.util.Iterator; import java.util.List; @@ -356,6 +357,7 @@ protected DateMathParser dateMathParser() { return dateMathParser; } + // Visible for testing. public long parse(String value) { return resolution.convert(DateFormatters.from(dateTimeFormatter().parse(value)).toInstant()); } @@ -625,6 +627,15 @@ protected void parseCreateField(ParseContext context) throws IOException { } } + @Override + public String parseSourceValue(Object value) { + String date = value.toString(); + long timestamp = fieldType().parse(date); + + ZonedDateTime dateTime = fieldType().resolution().toInstant(timestamp).atZone(ZoneOffset.UTC); + return fieldType().dateTimeFormatter().format(dateTime); + } + @Override protected void mergeOptions(FieldMapper other, List conflicts) { final DateFieldMapper d = (DateFieldMapper) other; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java index c527e677e5bb5..fff66746a3f63 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java @@ -32,6 +32,7 @@ import org.elasticsearch.common.xcontent.support.AbstractXContentParser; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.mapper.FieldNamesFieldMapper.FieldNamesFieldType; +import org.elasticsearch.search.lookup.SourceLookup; import java.io.IOException; import java.util.ArrayList; @@ -268,6 +269,47 @@ public void parse(ParseContext context) throws IOException { */ protected abstract void parseCreateField(ParseContext context) throws IOException; + /** + * Given access to a document's _source, return this field's values. + * + * In addition to pulling out the values, mappers can parse them into a standard form. This + * method delegates parsing to {@link #parseSourceValue} for parsing. Most mappers will choose + * to override {@link #parseSourceValue} -- for example numeric field mappers make sure to + * parse the source value into a number of the right type. + * + * Some mappers may need more flexibility and can override this entire method instead. + * + * @param lookup a lookup structure over the document's source. + * @return a list a standardized field values. + */ + public List lookupValues(SourceLookup lookup) { + Object sourceValue = lookup.extractValue(name()); + if (sourceValue == null) { + return List.of(); + } + + List values = new ArrayList<>(); + if (parsesArrayValue()) { + return (List) parseSourceValue(sourceValue); + } else { + List sourceValues = sourceValue instanceof List ? (List) sourceValue : List.of(sourceValue); + for (Object value : sourceValues) { + Object parsedValue = parseSourceValue(value); + values.add(parsedValue); + } + } + return values; + } + + /** + * Given a value that has been extracted from a document's source, parse it into a standard + * format. This parsing logic should closely mirror the value parsing in + * {@link #parseCreateField} or {@link #parse}. + * + * Note that when overriding this method, {@link #lookupValues} should *not* be overridden. + */ + protected abstract Object parseSourceValue(Object value); + protected void createFieldNamesField(ParseContext context) { FieldNamesFieldType fieldNamesFieldType = context.docMapper().metadataMapper(FieldNamesFieldMapper.class).fieldType(); if (fieldNamesFieldType != null && fieldNamesFieldType.isEnabled()) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java index d836bc0284eb3..8da0b5b832d42 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java @@ -409,6 +409,12 @@ protected void parseCreateField(ParseContext context) throws IOException { } } + @Override + protected String parseSourceValue(Object value) { + InetAddress address = InetAddresses.forString(value.toString()); + return InetAddresses.toAddrString(address); + } + @Override protected void mergeOptions(FieldMapper other, List conflicts) { IpFieldMapper mergeWith = (IpFieldMapper) other; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 94fe5b3424528..856daf3f9e9a3 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -227,6 +227,11 @@ public Mapper.Builder parse(String name, Map node, ParserCont } } + @Override + protected String parseSourceValue(Object value) { + return value.toString(); + } + public static final class KeywordFieldType extends StringFieldType { boolean hasNorms; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java index 54b9aaf57027b..b06111e80823f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java @@ -78,6 +78,11 @@ public void postParse(ParseContext context) throws IOException { // do nothing } + @Override + protected Object parseSourceValue(Object value) { + throw new UnsupportedOperationException("The " + typeName() + " field is not stored in _source."); + } + @Override protected void mergeOptions(FieldMapper other, List conflicts) { } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index 94af1ce030f30..e592bc0f974c9 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -1101,6 +1101,11 @@ protected void parseCreateField(ParseContext context) throws IOException { } } + @Override + protected Number parseSourceValue(Object value) { + return fieldType().type.parse(value, coerce.value()); + } + @Override protected void mergeOptions(FieldMapper other, List conflicts) { NumberFieldMapper m = (NumberFieldMapper) other; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java index b6336887f69db..8f69d28510c85 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java @@ -52,6 +52,7 @@ import java.time.ZoneId; import java.time.ZoneOffset; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; @@ -402,6 +403,26 @@ protected void parseCreateField(ParseContext context) throws IOException { } } + @Override + @SuppressWarnings("unchecked") + protected Object parseSourceValue(Object value) { + RangeType rangeType = fieldType().rangeType(); + if (!(value instanceof Map)) { + assert rangeType == RangeType.IP; + Tuple ipRange = InetAddresses.parseCidr(value.toString()); + return InetAddresses.toCidrString(ipRange.v1(), ipRange.v2()); + } + + Map range = (Map) value; + Map parsedRange = new HashMap<>(); + for (Map.Entry entry : range.entrySet()) { + Object parsedValue = rangeType.parseValue(entry.getValue(), coerce.value(), fieldType().dateMathParser); + Object formattedValue = rangeType.formatValue(parsedValue, fieldType().dateTimeFormatter); + parsedRange.put(entry.getKey(), formattedValue); + } + return parsedRange; + } + @Override protected void mergeOptions(FieldMapper other, List conflicts) { RangeFieldMapper mergeWith = (RangeFieldMapper) other; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RangeType.java b/server/src/main/java/org/elasticsearch/index/mapper/RangeType.java index 86d79e3171573..f293d4ded6c61 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RangeType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RangeType.java @@ -36,14 +36,17 @@ import org.elasticsearch.common.Nullable; import org.elasticsearch.common.geo.ShapeRelation; import org.elasticsearch.common.network.InetAddresses; +import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.common.time.DateMathParser; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.query.QueryShardContext; import java.io.IOException; import java.net.InetAddress; +import java.time.Instant; import java.time.ZoneId; import java.time.ZoneOffset; +import java.time.ZonedDateTime; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -69,8 +72,9 @@ public InetAddress parseTo(RangeFieldMapper.RangeFieldType fieldType, XContentPa InetAddress address = InetAddresses.forString(parser.text()); return included ? address : nextDown(address); } + @Override - public InetAddress parse(Object value, boolean coerce) { + public InetAddress parseValue(Object value, boolean coerce, @Nullable DateMathParser dateMathParser) { if (value instanceof InetAddress) { return (InetAddress) value; } else { @@ -80,6 +84,12 @@ public InetAddress parse(Object value, boolean coerce) { return InetAddresses.forString(value.toString()); } } + + @Override + public Object formatValue(Object value, DateFormatter dateFormatter) { + return InetAddresses.toAddrString((InetAddress) value); + } + @Override public InetAddress minValue() { return InetAddressPoint.MIN_VALUE; @@ -170,22 +180,34 @@ private Query createQuery(String field, Object lower, Object upper, boolean incl public Field getRangeField(String name, RangeFieldMapper.Range r) { return new LongRange(name, new long[] {((Number)r.from).longValue()}, new long[] {((Number)r.to).longValue()}); } - private Number parse(DateMathParser dateMathParser, String dateStr) { - return dateMathParser.parse(dateStr, () -> {throw new IllegalArgumentException("now is not used at indexing time");}) - .toEpochMilli(); - } @Override public Number parseFrom(RangeFieldMapper.RangeFieldType fieldType, XContentParser parser, boolean coerce, boolean included) throws IOException { - Number value = parse(fieldType.dateMathParser, parser.text()); + Number value = parseValue(parser.text(), coerce, fieldType.dateMathParser); return included ? value : nextUp(value); } @Override public Number parseTo(RangeFieldMapper.RangeFieldType fieldType, XContentParser parser, boolean coerce, boolean included) throws IOException{ - Number value = parse(fieldType.dateMathParser, parser.text()); + Number value = parseValue(parser.text(), coerce, fieldType.dateMathParser); return included ? value : nextDown(value); } + + @Override + public Long parseValue(Object dateStr, boolean coerce, @Nullable DateMathParser dateMathParser) { + assert dateMathParser != null; + return dateMathParser.parse(dateStr.toString(), () -> { + throw new IllegalArgumentException("now is not used at indexing time"); + }).toEpochMilli(); + } + + @Override + public Object formatValue(Object value, DateFormatter dateFormatter) { + long timestamp = (long) value; + ZonedDateTime dateTime = Instant.ofEpochMilli(timestamp).atZone(ZoneOffset.UTC); + return dateFormatter.format(dateTime); + } + @Override public Long minValue() { return Long.MIN_VALUE; @@ -243,6 +265,7 @@ public Query rangeQuery(String field, boolean hasDocValues, Object lowerTerm, Ob return createRangeQuery(field, hasDocValues, low, high, includeLower, includeUpper, relation); } + @Override public Query withinQuery(String field, Object from, Object to, boolean includeLower, boolean includeUpper) { return LONG.withinQuery(field, from, to, includeLower, includeUpper); @@ -598,6 +621,15 @@ public List createFields(ParseContext context, String name, Rang } return fields; } + + public Object parseValue(Object value, boolean coerce, @Nullable DateMathParser dateMathParser) { + return numberType.parse(value, coerce); + } + + public Object formatValue(Object value, DateFormatter formatter) { + return value; + } + /** parses from value. rounds according to included flag */ public Object parseFrom(RangeFieldMapper.RangeFieldType fieldType, XContentParser parser, boolean coerce, boolean included) throws IOException { @@ -618,15 +650,12 @@ public Object parseTo(RangeFieldMapper.RangeFieldType fieldType, XContentParser public abstract Query withinQuery(String field, Object from, Object to, boolean includeFrom, boolean includeTo); public abstract Query containsQuery(String field, Object from, Object to, boolean includeFrom, boolean includeTo); public abstract Query intersectsQuery(String field, Object from, Object to, boolean includeFrom, boolean includeTo); - public Object parse(Object value, boolean coerce) { - return numberType.parse(value, coerce); - } public Query rangeQuery(String field, boolean hasDocValues, Object from, Object to, boolean includeFrom, boolean includeTo, ShapeRelation relation, @Nullable ZoneId timeZone, @Nullable DateMathParser dateMathParser, QueryShardContext context) { - Object lower = from == null ? minValue() : parse(from, false); - Object upper = to == null ? maxValue() : parse(to, false); + Object lower = from == null ? minValue() : parseValue(from, false, dateMathParser); + Object upper = to == null ? maxValue() : parseValue(to, false, dateMathParser); return createRangeQuery(field, hasDocValues, lower, upper, includeFrom, includeTo, relation); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java index c17f6e8b62119..9056484b2f024 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java @@ -514,6 +514,11 @@ protected void parseCreateField(ParseContext context) throws IOException { throw new UnsupportedOperationException(); } + @Override + protected Object parseSourceValue(Object value) { + throw new UnsupportedOperationException(); + } + @Override protected void mergeOptions(FieldMapper other, List conflicts) { @@ -540,6 +545,11 @@ protected void parseCreateField(ParseContext context) { throw new UnsupportedOperationException(); } + @Override + protected Object parseSourceValue(Object value) { + throw new UnsupportedOperationException(); + } + @Override protected void mergeOptions(FieldMapper other, List conflicts) { @@ -877,6 +887,11 @@ protected void parseCreateField(ParseContext context) throws IOException { } } + @Override + protected String parseSourceValue(Object value) { + return value.toString(); + } + @Override public Iterator iterator() { List subIterators = new ArrayList<>(); diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsPhase.java index e6d5558b36199..50513dee9c898 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsPhase.java @@ -23,12 +23,15 @@ import org.apache.lucene.index.ReaderUtil; import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.IgnoredFieldMapper; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.search.lookup.SourceLookup; +import java.util.HashSet; import java.util.Map; +import java.util.Set; /** * A fetch sub-phase for high-level field retrieval. Given a list of fields, it @@ -59,10 +62,24 @@ public void hitsExecute(SearchContext context, SearchHit[] hits) { LeafReaderContext readerContext = context.searcher().getIndexReader().leaves().get(readerIndex); sourceLookup.setSegmentAndDocument(readerContext, hit.docId()); - Map documentFields = fieldValueRetriever.retrieve(sourceLookup); + Set ignoredFields = getIgnoredFields(hit); + Map documentFields = fieldValueRetriever.retrieve(sourceLookup, ignoredFields); for (Map.Entry entry : documentFields.entrySet()) { hit.setDocumentField(entry.getKey(), entry.getValue()); } } } + + private Set getIgnoredFields(SearchHit hit) { + DocumentField field = hit.field(IgnoredFieldMapper.NAME); + if (field == null) { + return Set.of(); + } + + Set ignoredFields = new HashSet<>(); + for (Object value : field.getValues()) { + ignoredFields.add((String) value); + } + return ignoredFields; + } } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldValueRetriever.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldValueRetriever.java index e9d1b88a9199f..bf879c85f043a 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldValueRetriever.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldValueRetriever.java @@ -20,15 +20,14 @@ package org.elasticsearch.search.fetch.subphase; import org.elasticsearch.common.document.DocumentField; -import org.elasticsearch.common.xcontent.support.XContentMapValues; -import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.DocumentFieldMappers; +import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.search.lookup.SourceLookup; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -38,80 +37,60 @@ * Then given a specific document, it can retrieve the corresponding fields from the document's source. */ public class FieldValueRetriever { + private final DocumentFieldMappers fieldMappers; private final List fieldContexts; - private final Set sourcePaths; public static FieldValueRetriever create(MapperService mapperService, Collection fieldPatterns) { + DocumentFieldMappers fieldMappers = mapperService.documentMapper().mappers(); List fields = new ArrayList<>(); - Set sourcePaths = new HashSet<>(); for (String fieldPattern : fieldPatterns) { Collection concreteFields = mapperService.simpleMatchToFullName(fieldPattern); for (String field : concreteFields) { - MappedFieldType fieldType = mapperService.fieldType(field); - - if (fieldType != null) { + if (fieldMappers.getMapper(field) != null) { Set sourcePath = mapperService.sourcePath(field); fields.add(new FieldContext(field, sourcePath)); - sourcePaths.addAll(sourcePath); } } } - return new FieldValueRetriever(fields, sourcePaths); + return new FieldValueRetriever(fieldMappers, fields); } - private FieldValueRetriever(List fieldContexts, Set sourcePaths) { + private FieldValueRetriever(DocumentFieldMappers fieldMappers, + List fieldContexts) { + this.fieldMappers = fieldMappers; this.fieldContexts = fieldContexts; - this.sourcePaths = sourcePaths; } - @SuppressWarnings("unchecked") - public Map retrieve(SourceLookup sourceLookup) { - Map result = new HashMap<>(); - Map sourceValues = extractValues(sourceLookup, sourcePaths); - + public Map retrieve(SourceLookup sourceLookup, Set ignoredFields) { + Map documentFields = new HashMap<>(); for (FieldContext fieldContext : fieldContexts) { String field = fieldContext.fieldName; Set sourcePath = fieldContext.sourcePath; - List values = new ArrayList<>(); - for (String path : sourcePath) { - Object value = sourceValues.get(path); - if (value != null) { - if (value instanceof List) { - values.addAll((List) value); - } else { - values.add(value); - } - } + if (ignoredFields.contains(field)) { + continue; } - result.put(field, new DocumentField(field, values)); - } - return result; - } - /** - * For each of the provided paths, return its value in the source. Note that in contrast with - * {@link SourceLookup#extractRawValues}, array and object values can be returned. - */ - private static Map extractValues(SourceLookup sourceLookup, Set paths) { - Map result = new HashMap<>(paths.size()); - for (String path : paths) { - Object value = XContentMapValues.extractValue(path, sourceLookup); - if (value != null) { - result.put(path, value); + List parsedValues = new ArrayList<>(); + for (String path : sourcePath) { + FieldMapper fieldMapper = (FieldMapper) fieldMappers.getMapper(path); + List values = fieldMapper.lookupValues(sourceLookup); + parsedValues.addAll(values); } + documentFields.put(field, new DocumentField(field, parsedValues)); } - return result; + return documentFields; } private static class FieldContext { final String fieldName; final Set sourcePath; - FieldContext(String fieldName, Set sourcePath) { + FieldContext(String fieldName, + Set sourcePath) { this.fieldName = fieldName; this.sourcePath = sourcePath; } diff --git a/server/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java b/server/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java index e547f02ef0347..6393d4bf50ff0 100644 --- a/server/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java +++ b/server/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java @@ -132,6 +132,14 @@ public List extractRawValues(String path) { return XContentMapValues.extractRawValues(path, loadSourceIfNeeded()); } + /** + * For the provided path, return its value in the source. Note that in contrast with + * {@link SourceLookup#extractRawValues}, array and object values can be returned. + */ + public Object extractValue(String path) { + return XContentMapValues.extractValue(path, loadSourceIfNeeded()); + } + public Object filter(FetchSourceContext context) { return context.getFilter().apply(loadSourceIfNeeded()); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java index 564be1efdca85..56bbe608be330 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java @@ -30,9 +30,12 @@ import org.apache.lucene.store.ByteBuffersDirectory; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; @@ -285,9 +288,18 @@ public void testMeta() throws Exception { assertEquals(mapping3, mapper.mappingSource().toString()); } + public void testParseSourceValue() { + Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); + Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); + BooleanFieldMapper mapper = new BooleanFieldMapper.Builder("field").build(context); + + assertTrue(mapper.parseSourceValue(true)); + assertFalse(mapper.parseSourceValue("false")); + assertFalse(mapper.parseSourceValue("")); + } + @Override protected BooleanFieldMapper.Builder newBuilder() { return new BooleanFieldMapper.Builder("boolean"); } - } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java index 66fec4eca1fc2..147da4cefbe26 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java @@ -32,6 +32,8 @@ import org.apache.lucene.util.CharsRefBuilder; import org.apache.lucene.util.automaton.Operations; import org.apache.lucene.util.automaton.RegExp; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.compress.CompressedXContent; @@ -55,6 +57,7 @@ import java.io.IOException; import java.util.Arrays; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.function.Function; @@ -961,6 +964,20 @@ public void testLimitOfContextMappings() throws Throwable { CompletionFieldMapper.COMPLETION_CONTEXTS_LIMIT + "] has been exceeded")); } + public void testParseSourceValue() { + Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); + Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); + CompletionFieldMapper mapper = new CompletionFieldMapper.Builder("field").build(context); + + assertEquals(List.of("value"), mapper.parseSourceValue("value")); + + List list = List.of("first", "second"); + assertEquals(list, mapper.parseSourceValue(list)); + + Map object = Map.of("input", List.of("first", "second"), "weight", "2.718"); + assertEquals(List.of(object), mapper.parseSourceValue(object)); + } + private Matcher suggestField(String value) { return Matchers.allOf(hasProperty(IndexableField::stringValue, equalTo(value)), Matchers.instanceOf(SuggestField.class)); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java index 7677e55449afa..5b77e50459fa0 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java @@ -21,10 +21,13 @@ import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexableField; +import org.elasticsearch.Version; import org.elasticsearch.bootstrap.JavaVersion; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; @@ -464,4 +467,40 @@ public void testMeta() throws Exception { assertEquals(mapping3, mapper.mappingSource().toString()); } + public void testParseSourceValue() { + Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); + Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); + + DateFieldMapper mapper = new DateFieldMapper.Builder("field").build(context); + String date = "2020-05-15T21:33:02.000Z"; + assertEquals(date, mapper.parseSourceValue(date)); + assertEquals(date, mapper.parseSourceValue(1589578382000L)); + + DateFieldMapper mapperWithFormat = new DateFieldMapper.Builder("field") + .format("yyyy/MM/dd||epoch_millis") + .build(context); + String dateInFormat = "1990/12/29"; + assertEquals(dateInFormat, mapperWithFormat.parseSourceValue(dateInFormat)); + assertEquals(dateInFormat, mapperWithFormat.parseSourceValue(662428800000L)); + + DateFieldMapper mapperWithMillis = new DateFieldMapper.Builder("field") + .format("epoch_millis") + .build(context); + String dateInMillis = "662428800000"; + assertEquals(dateInMillis, mapperWithMillis.parseSourceValue(dateInMillis)); + assertEquals(dateInMillis, mapperWithMillis.parseSourceValue(662428800000L)); + } + + public void testParseSourceValueNanos() { + Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); + Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); + + DateFieldMapper mapper = new DateFieldMapper.Builder("field") + .format("strict_date_time||epoch_millis") + .withResolution(DateFieldMapper.Resolution.NANOSECONDS) + .build(context); + String date = "2020-05-15T21:33:02.123456789Z"; + assertEquals("2020-05-15T21:33:02.123456789Z", mapper.parseSourceValue(date)); + assertEquals("2020-05-15T21:33:02.123Z", mapper.parseSourceValue(1589578382123L)); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java index 695ae6a780ba0..6e516b1afa2e6 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java @@ -111,6 +111,11 @@ static class FakeFieldMapper extends FieldMapper { protected void parseCreateField(ParseContext context) throws IOException { } + @Override + protected Object parseSourceValue(Object value) { + throw new UnsupportedOperationException(); + } + @Override protected void mergeOptions(FieldMapper other, List conflicts) { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java b/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java index 9ab3ecabfd01e..fba095dd3ebfd 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java @@ -209,6 +209,11 @@ protected void parseCreateField(ParseContext context) throws IOException { throw new UnsupportedOperationException(); } + @Override + protected Object parseSourceValue(Object value) { + return value; + } + @Override public Iterator iterator() { return Iterators.concat(super.iterator(), Arrays.asList(binMapper, boolMapper, pointMapper, shapeMapper, stringMapper).iterator()); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java b/server/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java index fa93a33ed35e2..a24a2d999d07d 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java @@ -141,6 +141,11 @@ protected void parseCreateField(ParseContext context) throws IOException { } } + @Override + protected String parseSourceValue(Object value) { + return value.toString(); + } + @Override protected void mergeOptions(FieldMapper other, List conflicts) { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java index 07ed508a624a1..0706e382fe830 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java @@ -26,10 +26,13 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.network.InetAddresses; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; @@ -297,6 +300,16 @@ public void testEmptyName() throws IOException { assertThat(e.getMessage(), containsString("name cannot be empty string")); } + public void testParseSourceValue() { + Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); + Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); + IpFieldMapper mapper = new IpFieldMapper.Builder("field").build(context); + + assertEquals("2001:db8::2:1", mapper.parseSourceValue("2001:db8::2:1")); + assertEquals("2001:db8::2:1", mapper.parseSourceValue("2001:db8:0:0:0:0:2:1")); + assertEquals("::1", mapper.parseSourceValue("0:0:0:0:0:0:0:1")); + } + @Override protected IpFieldMapper.Builder newBuilder() { return new IpFieldMapper.Builder("ip"); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpRangeFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpRangeFieldMapperTests.java index 2f861ca6fc79c..8e06c938c8b7f 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IpRangeFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IpRangeFieldMapperTests.java @@ -20,10 +20,13 @@ import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexableField; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.network.InetAddresses; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; @@ -80,4 +83,14 @@ public void testStoreCidr() throws Exception { assertThat(storedField.stringValue(), containsString(strVal)); } } + + public void testParseSourceValue() { + Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); + Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); + + RangeFieldMapper mapper = new RangeFieldMapper.Builder("field", RangeType.IP).build(context); + Map range = Map.of("gte", "2001:db8:0:0:0:0:2:1"); + assertEquals(Map.of("gte", "2001:db8::2:1"), mapper.parseSourceValue(range)); + assertEquals("2001:db8::2:1/32", mapper.parseSourceValue("2001:db8:0:0:0:0:2:1/32")); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java index 82f3b1784fceb..04abbc1e1c267 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java @@ -28,6 +28,8 @@ import org.apache.lucene.search.similarities.BM25Similarity; import org.apache.lucene.search.similarities.BooleanSimilarity; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.compress.CompressedXContent; @@ -165,6 +167,9 @@ public void testDefaults() throws Exception { // used by TermVectorsService assertArrayEquals(new String[] { "1234" }, TermVectorsService.getValues(doc.rootDoc().getFields("field"))); + + FieldMapper fieldMapper = (FieldMapper) mapper.mappers().getMapper("field"); + assertEquals("1234", fieldMapper.parseSourceValue("1234")); } public void testIgnoreAbove() throws IOException { @@ -619,4 +624,14 @@ public void testMeta() throws Exception { new CompressedXContent(mapping3), MergeReason.MAPPING_UPDATE); assertEquals(mapping3, mapper.mappingSource().toString()); } + + public void testParseSourceValue() { + Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); + Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); + KeywordFieldMapper mapper = new KeywordFieldMapper.Builder("field").build(context); + + assertEquals("value", mapper.parseSourceValue("value")); + assertEquals("42", mapper.parseSourceValue(42L)); + assertEquals("true", mapper.parseSourceValue(true)); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java index 2514f93131398..7eefee2142419 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java @@ -22,11 +22,14 @@ import com.carrotsearch.randomizedtesting.annotations.Timeout; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexableField; +import org.elasticsearch.Version; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.index.IndexResponse; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; @@ -401,6 +404,15 @@ public void testEmptyName() throws IOException { } } + public void testParseSourceValue() { + Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); + Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); + NumberFieldMapper mapper = new NumberFieldMapper.Builder("field", NumberType.INTEGER).build(context); + + assertEquals(3, mapper.parseSourceValue(3.14)); + assertEquals(42, mapper.parseSourceValue("42.9")); + } + @Timeout(millis = 30000) public void testOutOfRangeValues() throws IOException { final List> inputs = Arrays.asList( diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java index 88d2db04f2b73..0659983e1ed62 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java @@ -22,10 +22,13 @@ import org.apache.lucene.document.InetAddressPoint; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexableField; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.network.InetAddresses; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; @@ -40,6 +43,7 @@ import java.util.Collection; import java.util.HashSet; import java.util.Locale; +import java.util.Map; import java.util.Set; import static org.elasticsearch.index.query.RangeQueryBuilder.GTE_FIELD; @@ -49,6 +53,7 @@ import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.containsString; + public class RangeFieldMapperTests extends AbstractNumericFieldMapperTestCase { @Override @@ -486,4 +491,19 @@ public void testIllegalFormatField() throws Exception { assertEquals("Invalid format: [[test_format]]: Unknown pattern letter: t", e.getMessage()); } + public void testParseSourceValue() { + Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); + Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); + + RangeFieldMapper longMapper = new RangeFieldMapper.Builder("field", RangeType.LONG).build(context); + Map longRange = Map.of("gte", 3.14, "lt", "42.9"); + assertEquals(Map.of("gte", 3L, "lt", 42L), longMapper.parseSourceValue(longRange)); + + RangeFieldMapper dateMapper = new RangeFieldMapper.Builder("field", RangeType.DATE) + .format("yyyy/MM/dd||epoch_millis") + .build(context); + Map dateRange = Map.of("lt", "1990/12/29", "gte", 597429487111L); + assertEquals(Map.of("lt", "1990/12/29", "gte", "1988/12/06"), + dateMapper.parseSourceValue(dateRange)); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java index 9dcdb4e32fc28..988982e002bb1 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java @@ -239,13 +239,13 @@ public void testDateRangeQueryUsingMappingFormat() { assertEquals(1466062190000L, formatter.parseMillis(to)); RangeFieldType fieldType = new RangeFieldType(FIELDNAME, true, true, formatter, Collections.emptyMap()); - final Query query = fieldType.rangeQuery(from, to, true, true, relation, null, null, context); + final Query query = fieldType.rangeQuery(from, to, true, true, relation, null, fieldType.dateMathParser(), context); assertEquals("field:", query.toString()); // compare lower and upper bounds with what we would get on a `date` field DateFieldType dateFieldType = new DateFieldType(FIELDNAME, true, true, formatter, DateFieldMapper.Resolution.MILLISECONDS, Collections.emptyMap()); - final Query queryOnDateField = dateFieldType.rangeQuery(from, to, true, true, relation, null, null, context); + final Query queryOnDateField = dateFieldType.rangeQuery(from, to, true, true, relation, null, fieldType.dateMathParser(), context); assertEquals("field:[1465975790000 TO 1466062190999]", queryOnDateField.toString()); } @@ -466,9 +466,9 @@ private Object nextTo(Object from) throws Exception { } public void testParseIp() { - assertEquals(InetAddresses.forString("::1"), RangeType.IP.parse(InetAddresses.forString("::1"), randomBoolean())); - assertEquals(InetAddresses.forString("::1"), RangeType.IP.parse("::1", randomBoolean())); - assertEquals(InetAddresses.forString("::1"), RangeType.IP.parse(new BytesRef("::1"), randomBoolean())); + assertEquals(InetAddresses.forString("::1"), RangeType.IP.parseValue(InetAddresses.forString("::1"), randomBoolean(), null)); + assertEquals(InetAddresses.forString("::1"), RangeType.IP.parseValue("::1", randomBoolean(), null)); + assertEquals(InetAddresses.forString("::1"), RangeType.IP.parseValue(new BytesRef("::1"), randomBoolean(), null)); } public void testTermQuery() throws Exception { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java index 8a4eebf3ffe76..f65002f916763 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java @@ -49,7 +49,9 @@ import org.apache.lucene.search.spans.SpanOrQuery; import org.apache.lucene.search.spans.SpanTermQuery; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.Version; import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.compress.CompressedXContent; @@ -1331,4 +1333,14 @@ public void testMeta() throws Exception { new CompressedXContent(mapping3), MergeReason.MAPPING_UPDATE); assertEquals(mapping3, mapper.mappingSource().toString()); } + + public void testParseSourceValue() { + Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); + Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); + TextFieldMapper mapper = new TextFieldMapper.Builder("field").build(context); + + assertEquals("value", mapper.parseSourceValue("value")); + assertEquals("42", mapper.parseSourceValue(42L)); + assertEquals("true", mapper.parseSourceValue(true)); + } } diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldValueRetrieverTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldValueRetrieverTests.java index 92aa7eeacde4f..2b61fa2f8620e 100644 --- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldValueRetrieverTests.java +++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldValueRetrieverTests.java @@ -32,6 +32,7 @@ import java.io.IOException; import java.util.List; import java.util.Map; +import java.util.Set; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasItems; @@ -65,8 +66,8 @@ public void testObjectValues() throws IOException { MapperService mapperService = createMapperService(); XContentBuilder source = XContentFactory.jsonBuilder().startObject() .startObject("float_range") - .field("gte", 0.0) - .field("lte", 2.718) + .field("gte", 0.0f) + .field("lte", 2.718f) .endObject() .endObject(); @@ -76,14 +77,59 @@ public void testObjectValues() throws IOException { DocumentField rangeField = fields.get("float_range"); assertNotNull(rangeField); assertThat(rangeField.getValues().size(), equalTo(1)); - assertThat(rangeField.getValue(), equalTo(Map.of("gte", 0.0, "lte", 2.718))); + assertThat(rangeField.getValue(), equalTo(Map.of("gte", 0.0f, "lte", 2.718f))); + } + + public void testNonExistentField() throws IOException { + MapperService mapperService = createMapperService(); + XContentBuilder source = XContentFactory.jsonBuilder().startObject() + .field("field", "value") + .endObject(); + + Map fields = retrieveFields(mapperService, source, "non-existent"); + assertThat(fields.size(), equalTo(0)); + } + + public void testArrayValueMappers() throws IOException { + MapperService mapperService = createMapperService(); + + XContentBuilder source = XContentFactory.jsonBuilder().startObject() + .array("completion", "first", "second") + .endObject(); + + Map fields = retrieveFields(mapperService, source, "completion"); + assertThat(fields.size(), equalTo(1)); + + DocumentField field = fields.get("completion"); + assertNotNull(field); + assertThat(field.getValues().size(), equalTo(2)); + assertThat(field.getValues(), hasItems("first", "second")); + + // Test a field with multiple geo-points. + source = XContentFactory.jsonBuilder().startObject() + .startObject("completion") + .array("input", "first", "second") + .field("weight", "2.718") + .endObject() + .endObject(); + + fields = retrieveFields(mapperService, source, "completion"); + assertThat(fields.size(), equalTo(1)); + + field = fields.get("completion"); + assertNotNull(field); + assertThat(field.getValues().size(), equalTo(1)); + + Map expected = Map.of("input", List.of("first", "second"), + "weight", "2.718"); + assertThat(field.getValues().get(0), equalTo(expected)); } public void testFieldNamesWithWildcard() throws IOException { MapperService mapperService = createMapperService();; XContentBuilder source = XContentFactory.jsonBuilder().startObject() .array("field", "first", "second") - .field("integer_field", "third") + .field("integer_field", 333) .startObject("object") .field("field", "fourth") .endObject() @@ -100,7 +146,7 @@ public void testFieldNamesWithWildcard() throws IOException { DocumentField otherField = fields.get("integer_field"); assertNotNull(otherField); assertThat(otherField.getValues().size(), equalTo(1)); - assertThat(otherField.getValues(), hasItems("third")); + assertThat(otherField.getValues(), hasItems(333)); DocumentField objectField = fields.get("object.field"); assertNotNull(objectField); @@ -108,7 +154,6 @@ public void testFieldNamesWithWildcard() throws IOException { assertThat(objectField.getValues(), hasItems("fourth")); } - public void testFieldAliases() throws IOException { XContentBuilder mapping = XContentFactory.jsonBuilder().startObject() .startObject("properties") @@ -226,15 +271,15 @@ private Map retrieveFields(MapperService mapperService, X sourceLookup.setSource(BytesReference.bytes(source)); FieldValueRetriever fetchFieldsLookup = FieldValueRetriever.create(mapperService, fieldPatterns); - return fetchFieldsLookup.retrieve(sourceLookup); + return fetchFieldsLookup.retrieve(sourceLookup, Set.of()); } - public MapperService createMapperService() throws IOException { XContentBuilder mapping = XContentFactory.jsonBuilder().startObject() .startObject("properties") .startObject("field").field("type", "keyword").endObject() .startObject("integer_field").field("type", "integer").endObject() + .startObject("completion").field("type", "completion").endObject() .startObject("float_range").field("type", "float_range").endObject() .startObject("object") .startObject("properties") diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java index 43b0f12514b4b..d65a502ca0650 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java @@ -98,6 +98,11 @@ protected String contentType() { protected void parseCreateField(ParseContext context) throws IOException { } + @Override + protected Object parseSourceValue(Object value) { + throw new UnsupportedOperationException(); + } + @Override protected void mergeOptions(FieldMapper other, List conflicts) { diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java index 6d07c70974e21..679dd93841a6d 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java @@ -161,6 +161,11 @@ protected void parseCreateField(ParseContext context) throws IOException { throw new UnsupportedOperationException("Parsing is implemented in parse(), this method should NEVER be called"); } + @Override + protected Object parseSourceValue(Object value) { + return value; + } + public static class HistogramFieldType extends MappedFieldType { public HistogramFieldType(String name, boolean hasDocValues, Map meta) { diff --git a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java index 24067a70e5c50..c75112ec5cbc0 100644 --- a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java +++ b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java @@ -38,6 +38,7 @@ import org.elasticsearch.index.mapper.TypeParsers; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.lookup.SourceLookup; import java.io.IOException; import java.time.ZoneId; @@ -283,6 +284,18 @@ protected void parseCreateField(ParseContext context) throws IOException { } } + @Override + public List lookupValues(SourceLookup lookup) { + return fieldType().value == null + ? List.of() + : List.of(fieldType().value); + } + + @Override + protected Object parseSourceValue(Object value) { + throw new UnsupportedOperationException("This should never be called, since lookupValues is implemented directly."); + } + @Override protected void mergeOptions(FieldMapper other, List conflicts) { ConstantKeywordFieldType newConstantKeywordFT = (ConstantKeywordFieldType) other.fieldType(); diff --git a/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java b/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java index c1e183d7c804c..acfd5689d6a2e 100644 --- a/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java +++ b/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java @@ -13,18 +13,21 @@ import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.FieldMapperTestCase; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperService.MergeReason; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.mapper.SourceToParse; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.xpack.constantkeyword.ConstantKeywordMapperPlugin; import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; import org.junit.Before; import java.util.Collection; import java.util.Collections; +import java.util.List; import java.util.Set; public class ConstantKeywordFieldMapperTests extends FieldMapperTestCase { @@ -133,4 +136,27 @@ public void testMeta() throws Exception { new CompressedXContent(mapping3), MergeReason.MAPPING_UPDATE); assertEquals(mapping3, mapper.mappingSource().toString()); } + + public void testLookupValues() throws Exception { + IndexService indexService = createIndex("test"); + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("_doc") + .startObject("properties").startObject("field").field("type", "constant_keyword") + .endObject().endObject().endObject().endObject()); + DocumentMapper mapper = indexService.mapperService().merge("_doc", new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE); + assertEquals(mapping, mapper.mappingSource().toString()); + + FieldMapper fieldMapper = (FieldMapper) mapper.mappers().getMapper("field"); + List values = fieldMapper.lookupValues(new SourceLookup()); + assertTrue(values.isEmpty()); + + String mapping2 = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("_doc") + .startObject("properties").startObject("field").field("type", "constant_keyword") + .field("value", "foo").endObject().endObject().endObject().endObject()); + mapper = indexService.mapperService().merge("_doc", new CompressedXContent(mapping2), MergeReason.MAPPING_UPDATE); + + fieldMapper = (FieldMapper) mapper.mappers().getMapper("field"); + values = fieldMapper.lookupValues(new SourceLookup()); + assertEquals(1, values.size()); + assertEquals("foo", values.get(0)); + } } diff --git a/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java b/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java index c45dd2b7015d6..4cc4613bd03d9 100644 --- a/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java +++ b/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java @@ -631,6 +631,11 @@ protected void parseCreateField(ParseContext context) throws IOException { } } + @Override + protected Object parseSourceValue(Object value) { + return value; + } + @Override protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException { super.doXContentBody(builder, includeDefaults, params); diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/constant_keyword/10_basic.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/constant_keyword/10_basic.yml index ad2e2677bf610..7a6c2a0d53b0d 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/constant_keyword/10_basic.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/constant_keyword/10_basic.yml @@ -184,3 +184,18 @@ setup: - match: {hits.hits.0._index: test1 } - match: {hits.hits.1._index: test1 } - match: {hits.hits.2._index: test2 } + +--- +"Field retrieval": + + - do: + search: + index: test* + body: + fields: [ foo ] + sort: [ { _index: asc } ] + + - match: { "hits.total.value": 3 } + - match: {hits.hits.0.fields.foo.0: bar } + - match: {hits.hits.1.fields.foo.0: bar } + - match: {hits.hits.2.fields.foo.0: baz } diff --git a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java index 7ee3393de3aab..6d002e4ab7517 100644 --- a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java +++ b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java @@ -214,6 +214,11 @@ public void parse(ParseContext context) throws IOException { context.doc().addWithKey(fieldType().name(), field); } + @Override + protected Object parseSourceValue(Object value) { + return value; + } + @Override protected boolean indexedByDefault() { return false; diff --git a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapper.java b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapper.java index 66904e26eafb3..cc1c3a6272af5 100644 --- a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapper.java +++ b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapper.java @@ -145,12 +145,16 @@ public void parse(ParseContext context) { throw new UnsupportedOperationException(ERROR_MESSAGE_7X); } - @Override protected void parseCreateField(ParseContext context) { throw new IllegalStateException("parse is implemented directly"); } + @Override + protected Object parseSourceValue(Object value) { + throw new UnsupportedOperationException(ERROR_MESSAGE_7X); + } + @Override protected String contentType() { return CONTENT_TYPE; diff --git a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java index c8b70af0c34c9..356e963207469 100644 --- a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java +++ b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java @@ -958,6 +958,11 @@ protected void parseCreateField(ParseContext context) throws IOException { parseDoc.addAll(fields); } + @Override + protected String parseSourceValue(Object value) { + return value.toString(); + } + void createFields(String value, Document parseDoc, Listfields) throws IOException { if (value == null || value.length() > ignoreAbove) { return;