diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java index 1f0f06a697001..39a78f4d0b19a 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java @@ -24,6 +24,8 @@ import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.function.Function; import java.util.function.Supplier; /** @@ -102,11 +104,11 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, S } @Override - public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { + public ValueFetcher valueFetcher(Function> sourcePaths, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } - return new SourceValueFetcher(name(), context) { + return new SourceValueFetcher(name(), sourcePaths.apply(name()), null) { @Override protected Float parseSourceValue(Object value) { return objectToFloat(value); diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureMetaFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureMetaFieldMapper.java index 1fbbfa2ec1fd7..031fe16a2ae08 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureMetaFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureMetaFieldMapper.java @@ -12,6 +12,8 @@ import org.elasticsearch.index.query.SearchExecutionContext; import java.util.Collections; +import java.util.Set; +import java.util.function.Function; /** * This meta field only exists because rank feature fields index everything into a @@ -40,7 +42,7 @@ public String typeName() { } @Override - public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { + public ValueFetcher valueFetcher(Function> sourcePaths, String format) { throw new UnsupportedOperationException("Cannot fetch values for internal field [" + typeName() + "]."); } diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java index a0538ebb48321..a9f74ea827795 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java @@ -20,6 +20,8 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.function.Function; import java.util.function.Supplier; /** @@ -75,8 +77,8 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, S } @Override - public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { - return SourceValueFetcher.identity(name(), context, format); + public ValueFetcher valueFetcher(Function> sourcePaths, String format) { + return SourceValueFetcher.identity(name(), sourcePaths, format); } @Override diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java index ff6f853b5eec0..a84a78ff6ad3d 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java @@ -43,6 +43,8 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.function.Function; import java.util.function.Supplier; /** A {@link FieldMapper} for scaled floats. Values are internally multiplied @@ -199,11 +201,11 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, S } @Override - public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { + public ValueFetcher valueFetcher(Function> sourcePaths, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } - return new SourceValueFetcher(name(), context) { + return new SourceValueFetcher(name(), sourcePaths.apply(name()), null) { @Override protected Double parseSourceValue(Object value) { double doubleValue; @@ -517,9 +519,9 @@ public int docValueCount() { } @Override - public DocValueFetcher.Leaf getLeafValueFetcher(DocValueFormat format) { + public Leaf getLeafValueFetcher(DocValueFormat format) { SortedNumericDoubleValues values = getDoubleValues(); - return new DocValueFetcher.Leaf() { + return new Leaf() { @Override public boolean advanceExact(int docId) throws IOException { return values.advanceExact(docId); diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java index a7bc4b9de915e..b08a1435129c3 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java @@ -51,6 +51,8 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; +import java.util.function.Function; import static org.elasticsearch.index.mapper.TextFieldMapper.TextFieldType.hasGaps; @@ -253,8 +255,8 @@ private ShingleFieldType shingleFieldForPositions(int positions) { } @Override - public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { - return SourceValueFetcher.toString(name(), context, format); + public ValueFetcher valueFetcher(Function> sourcePaths, String format) { + return SourceValueFetcher.toString(name(), sourcePaths, format); } @Override @@ -366,10 +368,10 @@ public Query prefixQuery(String value, MultiTermQuery.RewriteMethod method, bool } @Override - public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { + public ValueFetcher valueFetcher(Function> sourcePaths, String format) { // Because this internal field is modelled as a multi-field, SourceValueFetcher will look up its // parent field in _source. So we don't need to use the parent field name here. - return SourceValueFetcher.toString(name(), context, format); + return SourceValueFetcher.toString(name(), sourcePaths, format); } @Override @@ -473,10 +475,10 @@ void setPrefixFieldType(PrefixFieldType prefixFieldType) { } @Override - public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { + public ValueFetcher valueFetcher(Function> sourcePaths, String format) { // Because this internal field is modelled as a multi-field, SourceValueFetcher will look up its // parent field in _source. So we don't need to use the parent field name here. - return SourceValueFetcher.toString(name(), context, format); + return SourceValueFetcher.toString(name(), sourcePaths, format); } @Override diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java index 0ce9b9e1b6531..9a51fca626e74 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java @@ -12,12 +12,13 @@ import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; import org.elasticsearch.index.analysis.NamedAnalyzer; -import org.elasticsearch.index.query.SearchExecutionContext; import java.io.IOException; import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.function.Function; import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeIntegerValue; @@ -81,11 +82,11 @@ static class TokenCountFieldType extends NumberFieldMapper.NumberFieldType { } @Override - public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { + public ValueFetcher valueFetcher(Function> sourcePaths, String format) { if (hasDocValues() == false) { return (lookup, ignoredFields) -> List.of(); } - return new DocValueFetcher(docValueFormat(format, null), context.getForField(this)); + return new DocValueFetcher(docValueFormat(format, null), name()); } } diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/MetaJoinFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/MetaJoinFieldMapper.java index b4894e21eb72b..88872fd740a6d 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/MetaJoinFieldMapper.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/MetaJoinFieldMapper.java @@ -20,6 +20,8 @@ import java.io.IOException; import java.util.Collections; +import java.util.Set; +import java.util.function.Function; import java.util.function.Supplier; /** @@ -54,7 +56,7 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, S } @Override - public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { + public ValueFetcher valueFetcher(Function> sourcePaths, String format) { throw new UnsupportedOperationException("Cannot fetch values for metadata field [" + NAME + "]."); } diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java index e975b6a167423..7d52486ab55c2 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java @@ -21,11 +21,12 @@ import org.elasticsearch.index.mapper.StringFieldType; import org.elasticsearch.index.mapper.TextSearchInfo; import org.elasticsearch.index.mapper.ValueFetcher; -import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.lookup.SearchLookup; import java.util.Collections; +import java.util.Set; +import java.util.function.Function; import java.util.function.Supplier; /** @@ -64,7 +65,7 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, S } @Override - public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { + public ValueFetcher valueFetcher(Function> sourcePaths, String format) { throw new UnsupportedOperationException("Cannot fetch values for internal field [" + typeName() + "]."); } diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java index 88523a5fd5467..544363794d00d 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java @@ -28,7 +28,6 @@ import org.elasticsearch.index.mapper.StringFieldType; import org.elasticsearch.index.mapper.TextSearchInfo; import org.elasticsearch.index.mapper.ValueFetcher; -import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.lookup.SearchLookup; @@ -41,6 +40,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.Function; import java.util.function.Supplier; /** @@ -151,8 +151,8 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, S } @Override - public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { - return SourceValueFetcher.identity(name(), context, format); + public ValueFetcher valueFetcher(Function> sourcePaths, String format) { + return SourceValueFetcher.identity(name(), sourcePaths, format); } @Override diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java index bd8084028ef87..9e6d3a515f246 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java @@ -61,9 +61,9 @@ import org.elasticsearch.index.query.ConstantScoreQueryBuilder; import org.elasticsearch.index.query.DisMaxQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.query.QueryShardException; import org.elasticsearch.index.query.Rewriteable; +import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder; import java.io.ByteArrayOutputStream; @@ -75,6 +75,8 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.function.Function; import java.util.function.Supplier; import static org.elasticsearch.index.query.AbstractQueryBuilder.parseInnerQueryBuilder; @@ -207,8 +209,8 @@ public Query termQuery(Object value, SearchExecutionContext context) { } @Override - public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { - return SourceValueFetcher.identity(name(), context, format); + public ValueFetcher valueFetcher(Function> sourcePaths, String format) { + return SourceValueFetcher.identity(name(), sourcePaths, format); } Query percolateQuery(String name, PercolateQuery.QueryStore queryStore, List documents, diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhase.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhase.java index 21081e86a16a1..da1ade746fd90 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhase.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhase.java @@ -22,6 +22,7 @@ import org.elasticsearch.search.fetch.subphase.highlight.HighlightPhase; import org.elasticsearch.search.fetch.subphase.highlight.Highlighter; import org.elasticsearch.search.fetch.subphase.highlight.SearchHighlightContext; +import org.elasticsearch.search.lookup.ValuesLookup; import java.io.IOException; import java.util.ArrayList; @@ -82,9 +83,9 @@ public void process(HitContext hit) throws IOException { BytesReference document = percolateQuery.getDocuments().get(slot); HitContext subContext = new HitContext( new SearchHit(slot, "unknown", Collections.emptyMap(), Collections.emptyMap()), + ValuesLookup.sourceOnly(document), percolatorLeafReaderContext, slot); - subContext.sourceLookup().setSource(document); // force source because MemoryIndex does not store fields SearchHighlightContext highlight = new SearchHighlightContext(fetchContext.highlight().fields(), true); FetchSubPhaseProcessor processor = highlightPhase.getProcessor(fetchContext, highlight, query); diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorMatchedSlotSubFetchPhaseTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorMatchedSlotSubFetchPhaseTests.java index 6c16c3d049c07..89af53687fbae 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorMatchedSlotSubFetchPhaseTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorMatchedSlotSubFetchPhaseTests.java @@ -52,7 +52,7 @@ public void testHitsExecute() throws Exception { LeafReaderContext context = reader.leaves().get(0); // A match: { - HitContext hit = new HitContext(new SearchHit(0), context, 0); + HitContext hit = new HitContext(new SearchHit(0), null, context, 0); PercolateQuery.QueryStore queryStore = ctx -> docId -> new TermQuery(new Term("field", "value")); MemoryIndex memoryIndex = new MemoryIndex(); memoryIndex.addField("field", "value", new WhitespaceAnalyzer()); @@ -73,7 +73,7 @@ public void testHitsExecute() throws Exception { // No match: { - HitContext hit = new HitContext(new SearchHit(0), context, 0); + HitContext hit = new HitContext(new SearchHit(0), null, context, 0); PercolateQuery.QueryStore queryStore = ctx -> docId -> new TermQuery(new Term("field", "value")); MemoryIndex memoryIndex = new MemoryIndex(); memoryIndex.addField("field", "value1", new WhitespaceAnalyzer()); @@ -93,7 +93,7 @@ public void testHitsExecute() throws Exception { // No query: { - HitContext hit = new HitContext(new SearchHit(0), context, 0); + HitContext hit = new HitContext(new SearchHit(0), null, context, 0); PercolateQuery.QueryStore queryStore = ctx -> docId -> null; MemoryIndex memoryIndex = new MemoryIndex(); memoryIndex.addField("field", "value", new WhitespaceAnalyzer()); diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java index aef9fa2b53f04..60b5ba658b71f 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java @@ -37,6 +37,8 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.function.Function; import java.util.function.Supplier; public class ICUCollationKeywordFieldMapper extends FieldMapper { @@ -70,12 +72,12 @@ public String typeName() { } @Override - public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { + public ValueFetcher valueFetcher(Function> sourcePaths, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } - return new SourceValueFetcher(name(), context, nullValue) { + return new SourceValueFetcher(name(), sourcePaths.apply(name()), nullValue) { @Override protected String parseSourceValue(Object value) { String keywordValue = value.toString(); @@ -154,6 +156,11 @@ public String format(BytesRef value) { return new String(encoded, 0, encodedLength); } + @Override + public Object formatObject(Object in) { + return format((BytesRef)in); + } + @Override public BytesRef parseBytesRef(String value) { char[] encoded = value.toCharArray(); diff --git a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java index 75dad11e6ce50..1c1c953509b17 100644 --- a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java +++ b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java @@ -31,6 +31,8 @@ import java.io.IOException; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.function.Function; import java.util.function.Supplier; public class Murmur3FieldMapper extends FieldMapper { @@ -93,8 +95,8 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, S } @Override - public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { - return SourceValueFetcher.toString(name(), context, format); + public ValueFetcher valueFetcher(Function> sourcePaths, String format) { + return SourceValueFetcher.toString(name(), sourcePaths, format); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/LeafFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/LeafFieldData.java index 351ffd137e865..ff78e8b9d8da8 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/LeafFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/LeafFieldData.java @@ -10,7 +10,6 @@ import org.apache.lucene.util.Accountable; import org.elasticsearch.common.lease.Releasable; -import org.elasticsearch.index.mapper.DocValueFetcher; import org.elasticsearch.search.DocValueFormat; import java.io.IOException; @@ -33,9 +32,9 @@ public interface LeafFieldData extends Accountable, Releasable { /** * Return a value fetcher for this leaf implementation. */ - default DocValueFetcher.Leaf getLeafValueFetcher(DocValueFormat format) { + default Leaf getLeafValueFetcher(DocValueFormat format) { SortedBinaryDocValues values = getBytesValues(); - return new DocValueFetcher.Leaf() { + return new Leaf() { @Override public boolean advanceExact(int docId) throws IOException { return values.advanceExact(docId); @@ -52,4 +51,22 @@ public Object nextValue() throws IOException { } }; } + + interface Leaf { + /** + * Advance the doc values reader to the provided doc. + * @return false if there are no values for this document, true otherwise + */ + boolean advanceExact(int docId) throws IOException; + + /** + * A count of the number of values at this document. + */ + int docValueCount() throws IOException; + + /** + * Load and format the next value. + */ + Object nextValue() throws IOException; + } } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafDoubleFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafDoubleFieldData.java index 9e28adcc6c113..b291b419ee443 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafDoubleFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafDoubleFieldData.java @@ -15,7 +15,6 @@ import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; -import org.elasticsearch.index.mapper.DocValueFetcher; import org.elasticsearch.search.DocValueFormat; import java.io.IOException; @@ -71,9 +70,9 @@ public Collection getChildResources() { } @Override - public DocValueFetcher.Leaf getLeafValueFetcher(DocValueFormat format) { + public Leaf getLeafValueFetcher(DocValueFormat format) { SortedNumericDoubleValues values = getDoubleValues(); - return new DocValueFetcher.Leaf() { + return new Leaf() { @Override public boolean advanceExact(int docId) throws IOException { return values.advanceExact(docId); diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafLongFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafLongFieldData.java index f0f7b457b8112..b838fdd56e73e 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafLongFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafLongFieldData.java @@ -15,7 +15,6 @@ import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; -import org.elasticsearch.index.mapper.DocValueFetcher; import org.elasticsearch.search.DocValueFormat; import java.io.IOException; @@ -68,9 +67,9 @@ public final SortedNumericDoubleValues getDoubleValues() { } @Override - public DocValueFetcher.Leaf getLeafValueFetcher(DocValueFormat format) { + public Leaf getLeafValueFetcher(DocValueFormat format) { SortedNumericDocValues values = getLongValues(); - return new DocValueFetcher.Leaf() { + return new Leaf() { @Override public boolean advanceExact(int docId) throws IOException { return values.advanceExact(docId); diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericIndexFieldData.java index 31e56cfd421f6..d4e51dd48163c 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericIndexFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericIndexFieldData.java @@ -27,7 +27,6 @@ import org.elasticsearch.index.fielddata.NumericDoubleValues; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; import org.elasticsearch.index.fielddata.fieldcomparator.LongValuesComparatorSource; -import org.elasticsearch.index.mapper.DocValueFetcher; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.MultiValueMode; @@ -164,10 +163,10 @@ public SortedNumericDocValues getLongValuesAsNanos() { } @Override - public DocValueFetcher.Leaf getLeafValueFetcher(DocValueFormat format) { + public Leaf getLeafValueFetcher(DocValueFormat format) { DocValueFormat nanosFormat = DocValueFormat.withNanosecondResolution(format); SortedNumericDocValues values = getLongValuesAsNanos(); - return new DocValueFetcher.Leaf() { + return new Leaf() { @Override public boolean advanceExact(int docId) throws IOException { return values.advanceExact(docId); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java index 54e585cc1e924..6c092fc1e2976 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java @@ -26,6 +26,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.function.Function; /** @@ -112,19 +113,19 @@ public final Query termQuery(Object value, SearchExecutionContext context) { } @Override - public final ValueFetcher valueFetcher(SearchExecutionContext context, String format) { + public final ValueFetcher valueFetcher(Function> sourcePaths, String format) { String geoFormat = format != null ? format : GeoJsonGeometryFormat.NAME; Function valueParser = value -> geometryParser.parseAndFormatObject(value, geoFormat); if (parsesArrayValue) { - return new ArraySourceValueFetcher(name(), context) { + return new ArraySourceValueFetcher(name(), sourcePaths.apply(name()), null) { @Override protected Object parseSourceValue(Object value) { return valueParser.apply(value); } }; } else { - return new SourceValueFetcher(name(), context) { + return new SourceValueFetcher(name(), sourcePaths.apply(name()), null) { // TODO null value? @Override protected Object parseSourceValue(Object value) { return valueParser.apply(value); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ArraySourceValueFetcher.java b/server/src/main/java/org/elasticsearch/index/mapper/ArraySourceValueFetcher.java index bc9418711f7fb..e288b2ce8078c 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ArraySourceValueFetcher.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ArraySourceValueFetcher.java @@ -9,8 +9,7 @@ package org.elasticsearch.index.mapper; import org.elasticsearch.common.Nullable; -import org.elasticsearch.index.query.SearchExecutionContext; -import org.elasticsearch.search.lookup.SourceLookup; +import org.elasticsearch.search.lookup.ValuesLookup; import java.util.ArrayList; import java.util.List; @@ -29,29 +28,20 @@ public abstract class ArraySourceValueFetcher implements ValueFetcher { private final @Nullable Object nullValue; private final String fieldName; - public ArraySourceValueFetcher(String fieldName, SearchExecutionContext context) { - this(fieldName, context, null); - } - - /** - * @param fieldName The name of the field. - * @param context The query shard context - * @param nullValue A optional substitute value if the _source value is 'null'. - */ - public ArraySourceValueFetcher(String fieldName, SearchExecutionContext context, Object nullValue) { - this.sourcePaths = context.sourcePath(fieldName); + public ArraySourceValueFetcher(String fieldName, Set sourcePaths, Object nullValue) { + this.sourcePaths = sourcePaths; this.nullValue = nullValue; this.fieldName = fieldName; } @Override - public List fetchValues(SourceLookup lookup, Set ignoredFields) { + public List fetchValues(ValuesLookup lookup, Set ignoredFields) { List values = new ArrayList<>(); if (ignoredFields.contains(fieldName)) { return values; } for (String path : sourcePaths) { - Object sourceValue = lookup.extractValue(path, nullValue); + Object sourceValue = lookup.source().extractValue(path, nullValue); if (sourceValue == null) { return List.of(); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java index 3c0e1b00a8cd8..ec9d7bcf5de28 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java @@ -31,6 +31,8 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.function.Function; import java.util.function.Supplier; public class BinaryFieldMapper extends FieldMapper { @@ -86,8 +88,8 @@ public String typeName() { } @Override - public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { - return SourceValueFetcher.identity(name(), context, format); + public ValueFetcher valueFetcher(Function> sourcePaths, String format) { + return SourceValueFetcher.identity(name(), sourcePaths, format); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java index 113ae8d7ba144..43dd320292665 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java @@ -33,6 +33,8 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.function.Function; import java.util.function.Supplier; /** @@ -117,12 +119,12 @@ public String typeName() { } @Override - public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { + public ValueFetcher valueFetcher(Function> sourcePaths, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } - return new SourceValueFetcher(name(), context, nullValue) { + return new SourceValueFetcher(name(), sourcePaths.apply(name()), nullValue) { @Override protected Boolean parseSourceValue(Object value) { if (value instanceof Boolean) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java index abd724d6b9c83..d95c87e21396b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java @@ -30,7 +30,6 @@ import org.elasticsearch.common.xcontent.XContentParser.Token; import org.elasticsearch.index.analysis.AnalyzerScope; import org.elasticsearch.index.analysis.NamedAnalyzer; -import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.search.suggest.completion.CompletionSuggester; import org.elasticsearch.search.suggest.completion.context.ContextMapping; import org.elasticsearch.search.suggest.completion.context.ContextMappings; @@ -43,6 +42,7 @@ import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.function.Function; /** * Mapper for completion field. The field values are indexed as a weighted FST for @@ -276,12 +276,12 @@ public String typeName() { } @Override - public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { + public ValueFetcher valueFetcher(Function> sourcePaths, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } - return new ArraySourceValueFetcher(name(), context) { + return new ArraySourceValueFetcher(name(), sourcePaths.apply(name()), null) { @Override protected List parseSourceValue(Object value) { if (value instanceof List) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index ba710be928e72..4d084f428da33 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -48,6 +48,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Set; import java.util.function.BiFunction; import java.util.function.Function; import java.util.function.LongSupplier; @@ -346,13 +347,13 @@ public long parse(String value) { } @Override - public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { + public ValueFetcher valueFetcher(Function> sourcePaths, String format) { DateFormatter defaultFormatter = dateTimeFormatter(); DateFormatter formatter = format != null ? DateFormatter.forPattern(format).withLocale(defaultFormatter.locale()) : defaultFormatter; - return new SourceValueFetcher(name(), context, nullValue) { + return new SourceValueFetcher(name(), sourcePaths.apply(name()), nullValue) { @Override public String parseSourceValue(Object value) { String date = value.toString(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocCountFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DocCountFieldMapper.java index 2bc38b1fb830d..74c24c537a8e6 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocCountFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocCountFieldMapper.java @@ -10,11 +10,13 @@ import org.apache.lucene.search.Query; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParserUtils; -import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.query.QueryShardException; +import org.elasticsearch.index.query.SearchExecutionContext; import java.io.IOException; import java.util.Collections; +import java.util.Set; +import java.util.function.Function; /** Mapper for the doc_count field. */ public class DocCountFieldMapper extends MetadataFieldMapper { @@ -55,12 +57,12 @@ public Query termQuery(Object value, SearchExecutionContext context) { } @Override - public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { + public ValueFetcher valueFetcher(Function> sourcePaths, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } - return new SourceValueFetcher(name(), context, DEFAULT_VALUE) { + return new SourceValueFetcher(name(), sourcePaths.apply(name()), DEFAULT_VALUE) { @Override protected Object parseSourceValue(Object value) { if ("".equals(value)) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocValueFetcher.java b/server/src/main/java/org/elasticsearch/index/mapper/DocValueFetcher.java index 30851e01037b8..887ec6ec5c67d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocValueFetcher.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocValueFetcher.java @@ -8,63 +8,31 @@ package org.elasticsearch.index.mapper; -import org.apache.lucene.index.LeafReaderContext; -import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.search.DocValueFormat; -import org.elasticsearch.search.lookup.SourceLookup; +import org.elasticsearch.search.lookup.ValuesLookup; import java.io.IOException; -import java.util.ArrayList; import java.util.List; import java.util.Set; - -import static java.util.Collections.emptyList; +import java.util.stream.Collectors; /** * Value fetcher that loads from doc values. */ public final class DocValueFetcher implements ValueFetcher { private final DocValueFormat format; - private final IndexFieldData ifd; - private Leaf leaf; + private final String field; - public DocValueFetcher(DocValueFormat format, IndexFieldData ifd) { + public DocValueFetcher(DocValueFormat format, String field) { this.format = format; - this.ifd = ifd; - } - - @Override - public void setNextReader(LeafReaderContext context) { - leaf = ifd.load(context).getLeafValueFetcher(format); + this.field = field; } @Override - public List fetchValues(SourceLookup lookup, Set ignoredFields) throws IOException { - if (false == leaf.advanceExact(lookup.docId())) { - return emptyList(); - } - List result = new ArrayList(leaf.docValueCount()); - for (int i = 0, count = leaf.docValueCount(); i < count; ++i) { - result.add(leaf.nextValue()); - } - return result; + public List fetchValues(ValuesLookup lookup, Set ignoreFields) throws IOException { + ScriptDocValues values = lookup.doc().get(field); + return values.stream().map(format::formatObject).collect(Collectors.toList()); } - public interface Leaf { - /** - * Advance the doc values reader to the provided doc. - * @return false if there are no values for this document, true otherwise - */ - boolean advanceExact(int docId) throws IOException; - - /** - * A count of the number of values at this document. - */ - int docValueCount() throws IOException; - - /** - * Load and format the next value. - */ - Object nextValue() throws IOException; - } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index 8d4ef0f482a32..9e8ead3ae4561 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -8,14 +8,6 @@ package org.elasticsearch.index.mapper; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.Iterator; -import java.util.List; -import java.util.function.Function; - import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexableField; import org.apache.lucene.search.Query; @@ -31,6 +23,15 @@ import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.query.SearchExecutionContext; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.Iterator; +import java.util.List; +import java.util.Set; +import java.util.function.Function; + /** A parser for documents, given mappings from a DocumentMapper */ final class DocumentParser { @@ -110,6 +111,8 @@ private static void internalParseDocument(RootObjectMapper root, MetadataFieldMa parseObjectOrNested(context, root); } + root.postParse(context, new IndexTimeScriptParams(context.sourceToParse().source(), context.mappingLookup())); + for (MetadataFieldMapper metadataMapper : metadataFieldsMappers) { metadataMapper.postParse(context); } @@ -814,7 +817,7 @@ private static class NoOpFieldMapper extends FieldMapper { NoOpFieldMapper(String simpleName, RuntimeFieldType runtimeField) { super(simpleName, new MappedFieldType(runtimeField.name(), false, false, false, TextSearchInfo.NONE, Collections.emptyMap()) { @Override - public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { + public ValueFetcher valueFetcher(Function> sourcePaths, String format) { throw new UnsupportedOperationException(); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java index 8851a442f9946..d44ede821fa0c 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java @@ -175,6 +175,16 @@ public void parse(ParseContext context) throws IOException { */ protected abstract void parseCreateField(ParseContext context) throws IOException; + @Override + public final void postParse(ParseContext context, IndexTimeScriptParams params) throws IOException { + doPostParse(context, params); + for (Mapper mapper : multiFields) { + mapper.postParse(context, params); + } + } + + public void doPostParse(ParseContext context, IndexTimeScriptParams params) { } + protected final void createFieldNamesField(ParseContext context) { assert fieldType().hasDocValues() == false : "_field_names should only be used when doc_values are turned off"; FieldNamesFieldMapper fieldNamesFieldMapper = (FieldNamesFieldMapper) context.getMetadataMapper(FieldNamesFieldMapper.NAME); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldNamesFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldNamesFieldMapper.java index e228d01a37390..c059a52b33e4d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldNamesFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldNamesFieldMapper.java @@ -20,6 +20,8 @@ import java.util.Collections; import java.util.Iterator; import java.util.List; +import java.util.Set; +import java.util.function.Function; /** * A mapper that indexes the field names of a document under _field_names. This mapper is typically useful in order @@ -121,7 +123,7 @@ public boolean isEnabled() { } @Override - public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { + public ValueFetcher valueFetcher(Function> sourcePaths, String format) { throw new UnsupportedOperationException("Cannot fetch values for internal field [" + name() + "]."); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java index c4701aa4ee853..5212179f0f140 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java @@ -44,7 +44,9 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.Set; import java.util.function.BooleanSupplier; +import java.util.function.Function; import java.util.function.Supplier; /** @@ -108,7 +110,7 @@ public boolean isSearchable() { } @Override - public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { + public ValueFetcher valueFetcher(Function> sourcePaths, String format) { throw new UnsupportedOperationException("Cannot fetch values for internal field [" + name() + "]."); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredFieldMapper.java index 79497c0b6ce83..253373b8d066f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredFieldMapper.java @@ -16,6 +16,8 @@ import org.elasticsearch.index.query.SearchExecutionContext; import java.util.Collections; +import java.util.Set; +import java.util.function.Function; /** * A field mapper that records fields that have been ignored because they were malformed. @@ -65,7 +67,7 @@ public Query existsQuery(SearchExecutionContext context) { } @Override - public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { + public ValueFetcher valueFetcher(Function> sourcePaths, String format) { throw new UnsupportedOperationException("Cannot fetch values for internal field [" + name() + "]."); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java index 001313f4db545..c6d945d6ad7a0 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java @@ -18,6 +18,8 @@ import org.elasticsearch.search.lookup.SearchLookup; import java.util.Collections; +import java.util.Set; +import java.util.function.Function; import java.util.function.Supplier; public class IndexFieldMapper extends MetadataFieldMapper { @@ -62,7 +64,7 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, S } @Override - public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { + public ValueFetcher valueFetcher(Function> sourcePaths, String format) { throw new UnsupportedOperationException("Cannot fetch values for internal field [" + name() + "]."); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IndexTimeScriptParams.java b/server/src/main/java/org/elasticsearch/index/mapper/IndexTimeScriptParams.java new file mode 100644 index 0000000000000..5814e8265c38e --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/mapper/IndexTimeScriptParams.java @@ -0,0 +1,96 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.mapper; + +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.index.fielddata.ScriptDocValues; +import org.elasticsearch.search.lookup.LeafDocLookup; +import org.elasticsearch.search.lookup.SourceLookup; +import org.elasticsearch.search.lookup.ValuesLookup; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; +import java.util.function.Predicate; + +public class IndexTimeScriptParams implements ValuesLookup { + + private final BytesReference source; + private final Function fieldTypeLookup; + private final Function> sourcePaths; + + private LeafDocLookup docLookup; + private SourceLookup sourceLookup; + + public IndexTimeScriptParams(BytesReference source, MappingLookup mappingLookup) { + this(source, mappingLookup::getFieldType, mappingLookup::sourcePaths); + } + + public IndexTimeScriptParams( + BytesReference source, + Function fieldTypeLookup, + Function> sourcePaths) { + this.source = source; + this.fieldTypeLookup = fieldTypeLookup; + this.sourcePaths = sourcePaths; + } + + public SourceLookup source() { + if (sourceLookup == null) { + sourceLookup = new SourceLookup(); + sourceLookup.setSource(source); + } + return sourceLookup; + } + + public Map> doc() { + if (docLookup == null) { + Predicate fieldExists = f -> fieldTypeLookup.apply(f) != null; + Function> valueLoader = f -> { + MappedFieldType ft = fieldTypeLookup.apply(f); + if (ft == null) { + throw new IllegalArgumentException("No field found for [" + f + "] in mapping"); + } + ValueFetcher fetcher = ft.valueFetcher(sourcePaths, null); + return new SyntheticScriptDocValues(fetcher); + }; + docLookup = new LeafDocLookup(fieldExists, valueLoader); + } + return docLookup; + } + + private class SyntheticScriptDocValues extends ScriptDocValues { + + final ValueFetcher fetcher; + List values; + + private SyntheticScriptDocValues(ValueFetcher fetcher) { + this.fetcher = fetcher; + } + + @Override + public void setNextDocId(int docId) throws IOException { + values = fetcher.fetchValues(IndexTimeScriptParams.this, Collections.emptySet()); + } + + @Override + public Object get(int index) { + return values.get(index); + } + + @Override + public int size() { + return values.size(); + } + } + +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java index 4253a83b03a75..2731562e30202 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java @@ -38,7 +38,9 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.function.BiFunction; +import java.util.function.Function; import java.util.function.Supplier; /** A {@link FieldMapper} for ip addresses. */ @@ -149,11 +151,11 @@ private static InetAddress parse(Object value) { } @Override - public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { + public ValueFetcher valueFetcher(Function> sourcePaths, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } - return new SourceValueFetcher(name(), context, nullValue) { + return new SourceValueFetcher(name(), sourcePaths.apply(name()), nullValue) { @Override protected Object parseSourceValue(Object value) { InetAddress address; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 959af80fc6eb8..eb0d211b8ad54 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -21,7 +21,6 @@ import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData; -import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.similarity.SimilarityProvider; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.lookup.SearchLookup; @@ -31,6 +30,8 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.function.Function; import java.util.function.Supplier; /** @@ -228,12 +229,12 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, S } @Override - public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { + public ValueFetcher valueFetcher(Function> sourcePaths, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } - return new SourceValueFetcher(name(), context, nullValue) { + return new SourceValueFetcher(name(), sourcePaths.apply(name()), nullValue) { @Override protected String parseSourceValue(Object value) { String keywordValue = value.toString(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java index 342171490c0ab..6d1571f4f5ab8 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java @@ -36,8 +36,8 @@ import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.query.DistanceFeatureQueryBuilder; import org.elasticsearch.index.query.QueryRewriteContext; -import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.query.QueryShardException; +import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.fetch.subphase.FetchFieldsPhase; import org.elasticsearch.search.lookup.SearchLookup; @@ -47,6 +47,7 @@ import java.util.Collection; import java.util.Map; import java.util.Objects; +import java.util.Set; import java.util.function.Function; import java.util.function.Supplier; @@ -93,7 +94,7 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, S * for metadata fields, field types should not throw {@link UnsupportedOperationException} since this * could cause a search retrieving multiple fields (like "fields": ["*"]) to fail. */ - public abstract ValueFetcher valueFetcher(SearchExecutionContext context, @Nullable String format); + public abstract ValueFetcher valueFetcher(Function> sourcePaths, @Nullable String format); /** Returns the name of this type, as would be specified in mapping properties */ public abstract String typeName(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java b/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java index 6ca146bfd5afc..21fe60c80b238 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java @@ -18,6 +18,7 @@ import org.elasticsearch.index.similarity.SimilarityProvider; import org.elasticsearch.script.ScriptService; +import java.io.IOException; import java.util.Map; import java.util.Objects; import java.util.function.BooleanSupplier; @@ -212,4 +213,14 @@ public final String simpleName() { */ public abstract void validate(MappingLookup mappers); + /** + * Do any post-processing of a document after it has been parsed + * + * @param context the ParseContext + * @param params the parsed document exposed as script parameters + */ + public void postParse(ParseContext context, IndexTimeScriptParams params) throws IOException { + // default impl does nothing + } + } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NestedPathFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NestedPathFieldMapper.java index d097d3de2309f..1c37c8a7f684f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NestedPathFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NestedPathFieldMapper.java @@ -19,6 +19,8 @@ import org.elasticsearch.index.query.SearchExecutionContext; import java.util.Collections; +import java.util.Set; +import java.util.function.Function; public class NestedPathFieldMapper extends MetadataFieldMapper { @@ -72,7 +74,7 @@ public Query existsQuery(SearchExecutionContext context) { } @Override - public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { + public ValueFetcher valueFetcher(Function> sourcePaths, String format) { throw new UnsupportedOperationException("Cannot fetch values for internal field [" + name() + "]."); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NestedValueFetcher.java b/server/src/main/java/org/elasticsearch/index/mapper/NestedValueFetcher.java index 05e16d2681278..b722e68327a11 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NestedValueFetcher.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NestedValueFetcher.java @@ -11,7 +11,7 @@ import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.search.fetch.subphase.FieldFetcher; -import org.elasticsearch.search.lookup.SourceLookup; +import org.elasticsearch.search.lookup.ValuesLookup; import java.io.IOException; import java.util.ArrayList; @@ -39,20 +39,18 @@ public NestedValueFetcher(String nestedField, FieldFetcher nestedFieldFetcher) { } @Override - public List fetchValues(SourceLookup lookup, Set ignoreFields) throws IOException { + public List fetchValues(ValuesLookup lookup, Set ignoreFields) throws IOException { List nestedEntriesToReturn = new ArrayList<>(); Map filteredSource = new HashMap<>(); Map stub = createSourceMapStub(filteredSource); - List nestedValues = XContentMapValues.extractNestedValue(nestedFieldPath, lookup.source()); + List nestedValues = XContentMapValues.extractNestedValue(nestedFieldPath, lookup.source().source()); if (nestedValues == null) { return Collections.emptyList(); } for (Object entry : nestedValues) { // add this one entry only to the stub and use this as source lookup stub.put(nestedFieldName, entry); - SourceLookup nestedSourceLookup = new SourceLookup(); - nestedSourceLookup.setSource(filteredSource); - + ValuesLookup nestedSourceLookup = ValuesLookup.sourceOnly(filteredSource); Map fetchResult = nestedFieldFetcher.fetch(nestedSourceLookup, ignoreFields); Map nestedEntry = new HashMap<>(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index 3b0a77f25c000..857da5b504d2f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -30,12 +30,18 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParser.Token; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexNumericFieldData.NumericType; +import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.fielddata.plain.SortedNumericIndexFieldData; import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.lookup.SearchLookup; @@ -45,10 +51,13 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; import java.util.function.BiFunction; +import java.util.function.Consumer; import java.util.function.Function; import java.util.function.Supplier; @@ -62,6 +71,40 @@ private static NumberFieldMapper toType(FieldMapper in) { return (NumberFieldMapper) in; } + private static class ScriptParameter implements ToXContent { + + final Script script; + final NumberScript compiled; + + ScriptParameter(Script script, ScriptService service) { + this.script = script; + this.compiled = service.compile(script, SCRIPT_CONTEXT).newInstance(); + } + + @Override + public String toString() { + return script.toString(); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ScriptParameter that = (ScriptParameter) o; + return Objects.equals(script, that.script); + } + + @Override + public int hashCode() { + return Objects.hash(script); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return script.toXContent(builder, params); + } + } + public static class Builder extends FieldMapper.Builder { private final Parameter indexed = Parameter.indexParam(m -> toType(m).indexed, true); @@ -73,6 +116,14 @@ public static class Builder extends FieldMapper.Builder { private final Parameter nullValue; + final FieldMapper.Parameter script = new FieldMapper.Parameter<>( + "script", + false, + () -> null, + (n, c, o) -> o == null ? null : new ScriptParameter(Script.parse(o), c.scriptService()), + m -> toType(m).builder.script.get() + ).acceptsNull(); + private final Parameter> meta = Parameter.metaParam(); private final NumberType type; @@ -110,7 +161,7 @@ public Builder docValues(boolean hasDocValues) { @Override protected List> getParameters() { - return List.of(indexed, hasDocValues, stored, ignoreMalformed, coerce, nullValue, meta); + return List.of(indexed, hasDocValues, stored, ignoreMalformed, coerce, nullValue, script, meta); } @Override @@ -954,12 +1005,12 @@ public Object valueForDisplay(Object value) { } @Override - public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { + public ValueFetcher valueFetcher(Function> sourcePaths, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } - return new SourceValueFetcher(name(), context, nullValue) { + return new SourceValueFetcher(name(), sourcePaths.apply(name()), nullValue) { @Override protected Object parseSourceValue(Object value) { if (value.equals("")) { @@ -1001,6 +1052,9 @@ public CollapseType collapseType() { private final Explicit ignoreMalformed; private final Explicit coerce; private final Number nullValue; + private final NumberScript script; + + private final Builder builder; private final boolean ignoreMalformedByDefault; private final boolean coerceByDefault; @@ -1012,6 +1066,7 @@ private NumberFieldMapper( CopyTo copyTo, Builder builder) { super(simpleName, mappedFieldType, multiFields, copyTo); + this.builder = builder; this.type = builder.type; this.indexed = builder.indexed.getValue(); this.hasDocValues = builder.hasDocValues.getValue(); @@ -1021,6 +1076,7 @@ private NumberFieldMapper( this.nullValue = builder.nullValue.getValue(); this.ignoreMalformedByDefault = builder.ignoreMalformed.getDefaultValue().value(); this.coerceByDefault = builder.coerce.getDefaultValue().value(); + this.script = builder.script.get() == null ? null : builder.script.get().compiled; } boolean coerce() { @@ -1043,6 +1099,10 @@ protected String contentType() { @Override protected void parseCreateField(ParseContext context) throws IOException { + if (script != null) { + throw new IllegalArgumentException("Values supplied for scripted field [" + name() + "]"); + } + XContentParser parser = context.parser(); Object value; Number numericValue = null; @@ -1080,6 +1140,10 @@ protected void parseCreateField(ParseContext context) throws IOException { numericValue = fieldType().type.parse(value, coerce.value()); } + indexValue(context, numericValue); + } + + private void indexValue(ParseContext context, Number numericValue) { context.doc().addAll(fieldType().type.createFields(fieldType().name(), numericValue, indexed, hasDocValues, stored)); @@ -1092,4 +1156,61 @@ protected void parseCreateField(ParseContext context) throws IOException { public FieldMapper.Builder getMergeBuilder() { return new Builder(simpleName(), type, ignoreMalformedByDefault, coerceByDefault).init(this); } + + @Override + public void doPostParse(ParseContext context, IndexTimeScriptParams params) { + if (script != null) { + script.execute(params, v -> indexValue(context, v)); + } + } + + public static final ScriptContext SCRIPT_CONTEXT = new ScriptContext<>( + "number_script", ScriptFactory.class + ); + + public interface ScriptFactory { + NumberScript newInstance(); + } + + public abstract static class NumberScript { + + private final Map params = new HashMap<>(); + private IndexTimeScriptParams docParams; + private Consumer emitter; + + public final void execute(IndexTimeScriptParams params, Consumer emitter) { + this.docParams = params; + this.emitter = emitter; + this.params.clear(); + this.params.put("_source", params.source()); + this.execute(); + } + + public abstract void execute(); + + public final Map> getDoc() { + return docParams.doc(); + } + + public final Map getParams() { + return params; + } + + public void emit(Number number) { + this.emitter.accept(number); + } + + } + + public static final class Emit { + private final NumberScript script; + + public Emit(NumberScript script) { + this.script = script; + } + + public void emit(Number number) { + this.script.emit(number); + } + } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java index 18f3e0d53ab20..4d5943c7157a9 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java @@ -397,6 +397,13 @@ ObjectMapper copyAndReset() { return copy; } + @Override + public void postParse(ParseContext context, IndexTimeScriptParams params) throws IOException { + for (Mapper mapper : mappers.values()) { + mapper.postParse(context, params); + } + } + /** * Build a mapping update with the provided sub mapping update. */ diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java index f4a49be19a28b..b905598bfa4af 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java @@ -41,6 +41,7 @@ import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.function.Function; import java.util.function.Supplier; import static org.elasticsearch.index.query.RangeQueryBuilder.GTE_FIELD; @@ -175,13 +176,13 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, S } @Override - public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { + public ValueFetcher valueFetcher(Function> sourcePaths, String format) { DateFormatter defaultFormatter = dateTimeFormatter(); DateFormatter formatter = format != null ? DateFormatter.forPattern(format).withLocale(defaultFormatter.locale()) : defaultFormatter; - return new SourceValueFetcher(name(), context) { + return new SourceValueFetcher(name(), sourcePaths.apply(name()), null) { @Override @SuppressWarnings("unchecked") diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java index 6e5f7681ab06d..b655ae5fa507e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java @@ -12,10 +12,11 @@ import org.apache.lucene.document.FieldType; import org.apache.lucene.index.IndexOptions; import org.elasticsearch.common.lucene.Lucene; -import org.elasticsearch.index.query.SearchExecutionContext; import java.util.Collections; import java.util.List; +import java.util.Set; +import java.util.function.Function; public class RoutingFieldMapper extends MetadataFieldMapper { @@ -83,7 +84,7 @@ public String typeName() { } @Override - public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { + public ValueFetcher valueFetcher(Function> sourcePaths, String format) { throw new UnsupportedOperationException("Cannot fetch values for internal field [" + name() + "]."); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java index 10d74e46c33db..34c65d09af464 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java @@ -27,6 +27,8 @@ import java.util.Collection; import java.util.Collections; import java.util.Objects; +import java.util.Set; +import java.util.function.Function; import java.util.function.Supplier; /** @@ -111,7 +113,7 @@ private long parse(Object value) { } @Override - public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { + public ValueFetcher valueFetcher(Function> sourcePaths, String format) { throw new UnsupportedOperationException("Cannot fetch values for internal field [" + name() + "]."); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java index a7b9cab7ed992..54aae6f3bb4af 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java @@ -25,14 +25,15 @@ import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.support.XContentMapValues; -import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.query.QueryShardException; +import org.elasticsearch.index.query.SearchExecutionContext; import java.io.IOException; import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.function.Function; public class SourceFieldMapper extends MetadataFieldMapper { @@ -100,7 +101,7 @@ public String typeName() { } @Override - public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { + public ValueFetcher valueFetcher(Function> sourcePaths, String format) { throw new UnsupportedOperationException("Cannot fetch values for internal field [" + name() + "]."); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceValueFetcher.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceValueFetcher.java index 7fefb32ae648c..add029a243ac2 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceValueFetcher.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceValueFetcher.java @@ -9,14 +9,14 @@ package org.elasticsearch.index.mapper; import org.elasticsearch.common.Nullable; -import org.elasticsearch.index.query.SearchExecutionContext; -import org.elasticsearch.search.lookup.SourceLookup; +import org.elasticsearch.search.lookup.ValuesLookup; import java.util.ArrayDeque; import java.util.ArrayList; import java.util.List; import java.util.Queue; import java.util.Set; +import java.util.function.Function; /** * An implementation of {@link ValueFetcher} that knows how to extract values @@ -30,29 +30,24 @@ public abstract class SourceValueFetcher implements ValueFetcher { private final @Nullable Object nullValue; private final String fieldName; - public SourceValueFetcher(String fieldName, SearchExecutionContext context) { - this(fieldName, context, null); - } - /** - * @param fieldName The name of the field. - * @param context The query shard context - * @param nullValue A optional substitute value if the _source value is 'null'. + * @param sourcePaths The locations in _source to retrieve values from + * @param nullValue A optional substitute value if the _source value is 'null'. */ - public SourceValueFetcher(String fieldName, SearchExecutionContext context, Object nullValue) { - this.sourcePaths = context.sourcePath(fieldName); + public SourceValueFetcher(String fieldName, Set sourcePaths, Object nullValue) { + this.sourcePaths = sourcePaths; this.nullValue = nullValue; this.fieldName = fieldName; } @Override - public List fetchValues(SourceLookup lookup, Set ignoredFields) { + public List fetchValues(ValuesLookup lookup, Set ignoredFields) { List values = new ArrayList<>(); if (ignoredFields.contains(fieldName)) { return values; } for (String path : sourcePaths) { - Object sourceValue = lookup.extractValue(path, nullValue); + Object sourceValue = lookup.source().extractValue(path, nullValue); if (sourceValue == null) { continue; } @@ -86,11 +81,11 @@ public List fetchValues(SourceLookup lookup, Set ignoredFields) /** * Creates a {@link SourceValueFetcher} that passes through source values unmodified. */ - public static SourceValueFetcher identity(String fieldName, SearchExecutionContext context, String format) { + public static SourceValueFetcher identity(String fieldName, Function> sourcePaths, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + fieldName + "] doesn't support formats."); } - return new SourceValueFetcher(fieldName, context) { + return new SourceValueFetcher(fieldName, sourcePaths.apply(fieldName), null) { @Override protected Object parseSourceValue(Object value) { return value; @@ -101,11 +96,11 @@ protected Object parseSourceValue(Object value) { /** * Creates a {@link SourceValueFetcher} that converts source values to strings. */ - public static SourceValueFetcher toString(String fieldName, SearchExecutionContext context, String format) { + public static SourceValueFetcher toString(String fieldName, Function> sourcePaths, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + fieldName + "] doesn't support formats."); } - return new SourceValueFetcher(fieldName, context) { + return new SourceValueFetcher(fieldName, sourcePaths.apply(fieldName), null) { @Override protected Object parseSourceValue(Object value) { return value.toString(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java index dd3d1070b503a..e6a25470969ec 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java @@ -72,6 +72,8 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; +import java.util.function.Function; import java.util.function.IntPredicate; import java.util.function.Supplier; @@ -486,7 +488,7 @@ private static final class PrefixFieldType extends StringFieldType { } @Override - public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { + public ValueFetcher valueFetcher(Function> sourcePaths, String format) { throw new UnsupportedOperationException(); } @@ -633,8 +635,8 @@ public String typeName() { } @Override - public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { - return SourceValueFetcher.toString(name(), context, format); + public ValueFetcher valueFetcher(Function> sourcePaths, String format) { + return SourceValueFetcher.toString(name(), sourcePaths, format); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TypeFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/TypeFieldType.java new file mode 100644 index 0000000000000..b5705b6348cf4 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/mapper/TypeFieldType.java @@ -0,0 +1,83 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.mapper; + +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.Query; +import org.elasticsearch.common.logging.DeprecationCategory; +import org.elasticsearch.common.logging.DeprecationLogger; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.plain.ConstantIndexFieldData; +import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.lookup.SearchLookup; + +import java.util.Collections; +import java.util.Set; +import java.util.function.Function; +import java.util.function.Supplier; + +/** + * Mediates access to the deprecated _type field + */ +public final class TypeFieldType extends ConstantFieldType { + + private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(TypeFieldType.class); + public static final String TYPES_V7_DEPRECATION_MESSAGE = "[types removal] Using the _type field " + + "in queries and aggregations is deprecated, prefer to use a field instead."; + + public static final String NAME = "_type"; + + public static final String CONTENT_TYPE = "_type"; + + private final String type; + + TypeFieldType(String type) { + super(NAME, Collections.emptyMap()); + this.type = type; + } + + /** + * Returns the name of the current type + */ + public String getType() { + return type; + } + + @Override + public String typeName() { + return CONTENT_TYPE; + } + + @Override + public Query existsQuery(SearchExecutionContext context) { + deprecationLogger.deprecate(DeprecationCategory.QUERIES, "typefieldtype", TYPES_V7_DEPRECATION_MESSAGE); + return new MatchAllDocsQuery(); + } + + @Override + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { + deprecationLogger.deprecate(DeprecationCategory.QUERIES, "typefieldtype", TYPES_V7_DEPRECATION_MESSAGE); + return new ConstantIndexFieldData.Builder(type, name(), CoreValuesSourceType.KEYWORD); + } + + @Override + public ValueFetcher valueFetcher(Function> sourcePaths, String format) { + throw new UnsupportedOperationException("Cannot fetch values for internal field [" + name() + "]."); + } + + @Override + protected boolean matches(String pattern, boolean caseInsensitive, SearchExecutionContext context) { + deprecationLogger.deprecate(DeprecationCategory.QUERIES, "typefieldtype", TYPES_V7_DEPRECATION_MESSAGE); + if (caseInsensitive) { + return pattern.equalsIgnoreCase(type); + } + return pattern.equals(type); + } +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ValueFetcher.java b/server/src/main/java/org/elasticsearch/index/mapper/ValueFetcher.java index 83d5ce6682d55..4d7fefbb1b859 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ValueFetcher.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ValueFetcher.java @@ -8,10 +8,9 @@ package org.elasticsearch.index.mapper; -import org.apache.lucene.index.LeafReaderContext; import org.elasticsearch.common.Nullable; import org.elasticsearch.search.fetch.subphase.FetchFieldsPhase; -import org.elasticsearch.search.lookup.SourceLookup; +import org.elasticsearch.search.lookup.ValuesLookup; import java.io.IOException; import java.util.List; @@ -32,14 +31,10 @@ public interface ValueFetcher { * Note that for array values, the order in which values are returned is undefined and * should not be relied on. * - * @param lookup a lookup structure over the document's source. + * @param lookup a lookup structure over the document's values. * @param ignoredFields the fields in _ignored that have been ignored for this document because they were malformed * @return a list a standardized field values. */ - List fetchValues(SourceLookup lookup, @Nullable Set ignoredFields) throws IOException; + List fetchValues(ValuesLookup lookup, @Nullable Set ignoredFields) throws IOException; - /** - * Update the leaf reader used to fetch values. - */ - default void setNextReader(LeafReaderContext context) {} } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/VersionFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/VersionFieldMapper.java index 63d56dad929ef..2379848ed62b8 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/VersionFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/VersionFieldMapper.java @@ -12,10 +12,12 @@ import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.search.Query; import org.elasticsearch.index.mapper.ParseContext.Document; -import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.query.QueryShardException; +import org.elasticsearch.index.query.SearchExecutionContext; import java.util.Collections; +import java.util.Set; +import java.util.function.Function; /** Mapper for the _version field. */ public class VersionFieldMapper extends MetadataFieldMapper { @@ -44,7 +46,7 @@ public Query termQuery(Object value, SearchExecutionContext context) { } @Override - public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { + public ValueFetcher valueFetcher(Function> sourcePaths, String format) { throw new UnsupportedOperationException("Cannot fetch values for internal field [" + name() + "]."); } } diff --git a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java index df00b476d4f3d..63900a7615720 100644 --- a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java +++ b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java @@ -20,6 +20,7 @@ import org.elasticsearch.common.time.DateMathParser; import org.elasticsearch.geometry.utils.Geohash; import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.script.JodaCompatibleZonedDateTime; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils; import java.io.IOException; @@ -62,6 +63,9 @@ default Object format(BytesRef value) { throw new UnsupportedOperationException(); } + /** Format a general object for returning in responses */ + Object formatObject(Object in); + /** Parse a value that was formatted with {@link #format(long)} back to the * original long value. */ default long parseLong(String value, boolean roundUp, LongSupplier now) { @@ -106,6 +110,11 @@ public String format(BytesRef value) { return value.utf8ToString(); } + @Override + public Object formatObject(Object in) { + return in; + } + @Override public long parseLong(String value, boolean roundUp, LongSupplier now) { try { @@ -157,6 +166,11 @@ public String format(BytesRef value) { .encodeToString(Arrays.copyOfRange(value.bytes, value.offset, value.offset + value.length)); } + @Override + public Object formatObject(Object in) { + return format((BytesRef)in); + } + @Override public BytesRef parseBytesRef(String value) { return new BytesRef(Base64.getDecoder().decode(value)); @@ -241,6 +255,13 @@ public long parseLong(String value, boolean roundUp, LongSupplier now) { return resolution.convert(parser.parse(value, now, roundUp, timeZone)); } + @Override + public Object formatObject(Object in) { + assert in instanceof JodaCompatibleZonedDateTime; + JodaCompatibleZonedDateTime dt = (JodaCompatibleZonedDateTime) in; + return formatter.format(dt.toInstant().atZone(timeZone)); + } + @Override public double parseDouble(String value, boolean roundUp, LongSupplier now) { return parseLong(value, roundUp, now); @@ -263,6 +284,12 @@ public String getWriteableName() { public void writeTo(StreamOutput out) { } + @Override + public Object formatObject(Object in) { + assert in instanceof Number; + return format(((Number) in).longValue()); + } + @Override public String format(long value) { return Geohash.stringEncode(value); @@ -285,6 +312,12 @@ public String getWriteableName() { public void writeTo(StreamOutput out) { } + @Override + public Object formatObject(Object in) { + assert in instanceof Number; + return format(((Number) in).longValue()); + } + @Override public String format(long value) { return GeoTileUtils.stringEncode(value); @@ -307,6 +340,12 @@ public String getWriteableName() { public void writeTo(StreamOutput out) { } + @Override + public Object formatObject(Object in) { + assert in instanceof Number; + return format(((Number) in).longValue()); + } + @Override public Boolean format(long value) { return value != 0; @@ -345,6 +384,12 @@ public String getWriteableName() { public void writeTo(StreamOutput out) { } + @Override + public Object formatObject(Object in) { + assert in instanceof BytesRef; + return format((BytesRef)in); + } + @Override public String format(BytesRef value) { byte[] bytes = Arrays.copyOfRange(value.bytes, value.offset, value.offset + value.length); @@ -416,6 +461,12 @@ public String format(double value) { return format.format(value); } + @Override + public Object formatObject(Object in) { + assert in instanceof Number; + return format(((Number) in).longValue()); + } + @Override public long parseLong(String value, boolean roundUp, LongSupplier now) { Number n; @@ -468,7 +519,7 @@ public int hashCode() { @Override public String toString() { return pattern; } - }; + } /** * DocValues format for unsigned 64 bit long values, @@ -501,6 +552,12 @@ public long parseLong(String value, boolean roundUp, LongSupplier now) { return parsedValue ^ MASK_2_63; } + @Override + public Object formatObject(Object in) { + assert in instanceof Number; + return format(((Number) in).longValue()); + } + /** * Formats a raw docValue that is stored in the shifted long format to the unsigned long representation. */ diff --git a/server/src/main/java/org/elasticsearch/search/fetch/FetchContext.java b/server/src/main/java/org/elasticsearch/search/fetch/FetchContext.java index 92d6140f2e12c..077c99245fdbf 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/FetchContext.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/FetchContext.java @@ -211,7 +211,7 @@ public SourceLookup getRootSourceLookup(FetchSubPhase.HitContext hitContext) { InnerHitSubContext innerHitsContext = (InnerHitSubContext) searchContext; return innerHitsContext.getRootLookup(); } else { - return hitContext.sourceLookup(); + return hitContext.valuesLookup().source(); } } } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java index 86b8f5b0ec641..62f5988ca7d85 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java @@ -36,7 +36,9 @@ import org.elasticsearch.search.fetch.subphase.InnerHitsContext; import org.elasticsearch.search.fetch.subphase.InnerHitsPhase; import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.search.lookup.LeafSearchLookup; import org.elasticsearch.search.lookup.SourceLookup; +import org.elasticsearch.search.lookup.ValuesLookup; import org.elasticsearch.tasks.TaskCancelledException; import java.io.IOException; @@ -104,6 +106,7 @@ public void execute(SearchContext context) { int currentReaderIndex = -1; LeafReaderContext currentReaderContext = null; LeafNestedDocuments leafNestedDocuments = null; + LeafSearchLookup currentValuesLookup = null; CheckedBiConsumer fieldReader = null; boolean hasSequentialDocs = hasSequentialDocs(docs); for (int index = 0; index < context.docIdsToLoadSize(); index++) { @@ -130,9 +133,11 @@ public void execute(SearchContext context) { for (FetchSubPhaseProcessor processor : processors) { processor.setNextReader(currentReaderContext); } + currentValuesLookup = fetchContext.searchLookup().getLeafSearchLookup(currentReaderContext); leafNestedDocuments = nestedDocuments.getLeafNestedDocuments(currentReaderContext); } assert currentReaderContext != null; + currentValuesLookup.setDocument(docId); HitContext hit = prepareHitContext( context, leafNestedDocuments, @@ -141,6 +146,7 @@ public void execute(SearchContext context) { docId, storedToRequestedFields, currentReaderContext, + currentValuesLookup, fieldReader); for (FetchSubPhaseProcessor processor : processors) { processor.process(hit); @@ -248,13 +254,14 @@ private HitContext prepareHitContext(SearchContext context, int docId, Map> storedToRequestedFields, LeafReaderContext subReaderContext, + ValuesLookup valuesLookup, CheckedBiConsumer storedFieldReader) throws IOException { if (nestedDocuments.advance(docId - subReaderContext.docBase) == null) { return prepareNonNestedHitContext( - context, fieldsVisitor, docId, storedToRequestedFields, subReaderContext, storedFieldReader); + context, fieldsVisitor, docId, storedToRequestedFields, subReaderContext, valuesLookup, storedFieldReader); } else { return prepareNestedHitContext(context, docId, nestedDocuments, hasNonNestedParent, storedToRequestedFields, - subReaderContext, storedFieldReader); + valuesLookup, subReaderContext, storedFieldReader); } } @@ -262,7 +269,7 @@ private HitContext prepareHitContext(SearchContext context, * Resets the provided {@link HitContext} with information on the current * document. This includes the following: * - Adding an initial {@link SearchHit} instance. - * - Loading the document source and setting it on {@link HitContext#sourceLookup()}. This + * - Loading the document source and setting it on {@link HitContext#valuesLookup()}. This * allows fetch subphases that use the hit context to access the preloaded source. */ private HitContext prepareNonNestedHitContext(SearchContext context, @@ -270,11 +277,12 @@ private HitContext prepareNonNestedHitContext(SearchContext context, int docId, Map> storedToRequestedFields, LeafReaderContext subReaderContext, + ValuesLookup valuesLookup, CheckedBiConsumer fieldReader) throws IOException { int subDocId = docId - subReaderContext.docBase; if (fieldsVisitor == null) { SearchHit hit = new SearchHit(docId, null, null, null); - return new HitContext(hit, subReaderContext, subDocId); + return new HitContext(hit, valuesLookup, subReaderContext, subDocId); } else { SearchHit hit; loadStoredFields(context.getSearchExecutionContext()::getFieldType, fieldReader, fieldsVisitor, subDocId); @@ -287,11 +295,11 @@ private HitContext prepareNonNestedHitContext(SearchContext context, hit = new SearchHit(docId, fieldsVisitor.id(), emptyMap(), emptyMap()); } - HitContext hitContext = new HitContext(hit, subReaderContext, subDocId); + HitContext hitContext = new HitContext(hit, valuesLookup, subReaderContext, subDocId); if (fieldsVisitor.source() != null) { // Store the loaded source on the hit context so that fetch subphases can access it. // Also make it available to scripts by storing it on the shared SearchLookup instance. - hitContext.sourceLookup().setSource(fieldsVisitor.source()); + hitContext.valuesLookup().source().setSource(fieldsVisitor.source()); SourceLookup scriptSourceLookup = context.getSearchExecutionContext().lookup().source(); scriptSourceLookup.setSegmentAndDocument(subReaderContext, subDocId); @@ -306,7 +314,7 @@ private HitContext prepareNonNestedHitContext(SearchContext context, * nested document. This includes the following: * - Adding an initial {@link SearchHit} instance. * - Loading the document source, filtering it based on the nested document ID, then - * setting it on {@link HitContext#sourceLookup()}. This allows fetch subphases that + * setting it on {@link HitContext#valuesLookup()}. This allows fetch subphases that * use the hit context to access the preloaded source. */ @SuppressWarnings("unchecked") @@ -315,6 +323,7 @@ private HitContext prepareNestedHitContext(SearchContext context, LeafNestedDocuments nestedInfo, Predicate hasNonNestedParent, Map> storedToRequestedFields, + ValuesLookup valuesLookup, LeafReaderContext subReaderContext, CheckedBiConsumer storedFieldReader) throws IOException { @@ -368,7 +377,7 @@ private HitContext prepareNestedHitContext(SearchContext context, SearchHit.NestedIdentity nestedIdentity = nestedInfo.nestedIdentity(); SearchHit hit = new SearchHit(topDocId, rootId, nestedIdentity, docFields, metaFields); - HitContext hitContext = new HitContext(hit, subReaderContext, nestedInfo.doc()); + HitContext hitContext = new HitContext(hit, valuesLookup, subReaderContext, nestedInfo.doc()); if (rootSourceAsMap != null && rootSourceAsMap.isEmpty() == false) { // Isolate the nested json array object that matches with nested hit and wrap it back into the same json @@ -400,8 +409,8 @@ private HitContext prepareNestedHitContext(SearchContext context, } } - hitContext.sourceLookup().setSource(nestedSourceAsMap); - hitContext.sourceLookup().setSourceContentType(rootSourceContentType); + hitContext.valuesLookup().source().setSource(nestedSourceAsMap); + hitContext.valuesLookup().source().setSourceContentType(rootSourceContentType); } return hitContext; } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/FetchSubPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/FetchSubPhase.java index 8ab13515167eb..2acde5d186fb3 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/FetchSubPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/FetchSubPhase.java @@ -12,7 +12,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.ReaderUtil; import org.elasticsearch.search.SearchHit; -import org.elasticsearch.search.lookup.SourceLookup; +import org.elasticsearch.search.lookup.ValuesLookup; import java.io.IOException; @@ -25,14 +25,13 @@ class HitContext { private final SearchHit hit; private final LeafReaderContext readerContext; private final int docId; - private SourceLookup sourceLookup; + private ValuesLookup valuesLookup; - public HitContext(SearchHit hit, LeafReaderContext context, int docId) { + public HitContext(SearchHit hit, ValuesLookup valuesLookup, LeafReaderContext context, int docId) { this.hit = hit; this.readerContext = context; this.docId = docId; - this.sourceLookup = new SourceLookup(); - sourceLookup.setSegmentAndDocument(context, docId); + this.valuesLookup = valuesLookup; } public SearchHit hit() { @@ -61,12 +60,8 @@ public int docId() { * In most cases, the hit document's source is loaded eagerly at the start of the * {@link FetchPhase}. This lookup will contain the preloaded source. */ - public SourceLookup sourceLookup() { - return sourceLookup; - } - - public void setSourceLookup(SourceLookup sourceLookup) { - this.sourceLookup = sourceLookup; + public ValuesLookup valuesLookup() { + return valuesLookup; } public IndexReader topLevelReader() { diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesPhase.java index 72b135a9a3dd7..087f9a2160075 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesPhase.java @@ -48,7 +48,7 @@ public FetchSubPhaseProcessor getProcessor(FetchContext context) { } ValueFetcher fetcher = new DocValueFetcher( ft.docValueFormat(fieldAndFormat.format, null), - context.searchLookup().getForField(ft) + ft.name() ); fields.add(new DocValueField(fieldAndFormat.field, fetcher)); } @@ -56,9 +56,6 @@ public FetchSubPhaseProcessor getProcessor(FetchContext context) { return new FetchSubPhaseProcessor() { @Override public void setNextReader(LeafReaderContext readerContext) { - for (DocValueField f : fields) { - f.fetcher.setNextReader(readerContext); - } } @Override @@ -71,7 +68,7 @@ public void process(HitContext hit) throws IOException { // docValues fields will still be document fields, and put under "fields" section of a hit. hit.hit().setDocumentField(f.field, hitField); } - hitField.getValues().addAll(f.fetcher.fetchValues(hit.sourceLookup(), Collections.emptySet())); + hitField.getValues().addAll(f.fetcher.fetchValues(hit.valuesLookup(), Collections.emptySet())); } } }; diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsPhase.java index da7b40ce0dda3..98c9b4a64e0f7 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsPhase.java @@ -15,7 +15,6 @@ import org.elasticsearch.search.fetch.FetchContext; import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.fetch.FetchSubPhaseProcessor; -import org.elasticsearch.search.lookup.SourceLookup; import java.io.IOException; import java.util.HashSet; @@ -40,21 +39,21 @@ public FetchSubPhaseProcessor getProcessor(FetchContext fetchContext) { "in the mappings for index [" + fetchContext.getIndexName() + "]"); } - FieldFetcher fieldFetcher = FieldFetcher.create(fetchContext.getSearchExecutionContext(), fetchFieldsContext.fields()); + FieldFetcher fieldFetcher = FieldFetcher.create( + fetchContext.getSearchExecutionContext(), + fetchFieldsContext.fields()); return new FetchSubPhaseProcessor() { @Override public void setNextReader(LeafReaderContext readerContext) { - fieldFetcher.setNextReader(readerContext); } @Override public void process(HitContext hitContext) throws IOException { SearchHit hit = hitContext.hit(); - SourceLookup sourceLookup = hitContext.sourceLookup(); Set ignoredFields = getIgnoredFields(hit); - Map documentFields = fieldFetcher.fetch(sourceLookup, ignoredFields); + Map documentFields = fieldFetcher.fetch(hitContext.valuesLookup(), ignoredFields); for (Map.Entry entry : documentFields.entrySet()) { hit.setDocumentField(entry.getKey(), entry.getValue()); } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhase.java index d0afe52c9ea59..e2ceb560d80ae 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhase.java @@ -56,7 +56,7 @@ public void process(HitContext hitContext) { private void hitExecute(FetchSourceContext fetchSourceContext, HitContext hitContext) { final boolean nestedHit = hitContext.hit().getNestedIdentity() != null; - SourceLookup source = hitContext.sourceLookup(); + SourceLookup source = hitContext.valuesLookup().source(); // If this is a parent document and there are no source filters, then add the source as-is. if (nestedHit == false && containsFilters(fetchSourceContext) == false) { diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldFetcher.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldFetcher.java index e44c2a20c74cb..e6fbbe1d197d4 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldFetcher.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldFetcher.java @@ -8,7 +8,6 @@ package org.elasticsearch.search.fetch.subphase; -import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.util.automaton.CharacterRunAutomaton; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.document.DocumentField; @@ -18,7 +17,7 @@ import org.elasticsearch.index.mapper.ObjectMapper; import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.query.SearchExecutionContext; -import org.elasticsearch.search.lookup.SourceLookup; +import org.elasticsearch.search.lookup.ValuesLookup; import java.io.IOException; import java.util.ArrayList; @@ -85,7 +84,7 @@ private static FieldFetcher create(SearchExecutionContext context, } // only add concrete fields if they are not beneath a known nested field if (nestedParentPath == null) { - ValueFetcher valueFetcher = ft.valueFetcher(context, fieldAndFormat.format); + ValueFetcher valueFetcher = ft.valueFetcher(context::sourcePath, fieldAndFormat.format); fieldContexts.put(field, new FieldContext(field, valueFetcher)); } } @@ -134,20 +133,20 @@ private FieldFetcher( this.unmappedFieldsFetchAutomaton = unmappedFieldsFetchAutomaton; } - public Map fetch(SourceLookup sourceLookup, Set ignoredFields) throws IOException { + public Map fetch(ValuesLookup valuesLookup, Set ignoredFields) throws IOException { assert ignoredFields != null; Map documentFields = new HashMap<>(); for (FieldContext context : fieldContexts.values()) { String field = context.fieldName; ValueFetcher valueFetcher = context.valueFetcher; - List parsedValues = valueFetcher.fetchValues(sourceLookup, ignoredFields); + List parsedValues = valueFetcher.fetchValues(valuesLookup, ignoredFields); if (parsedValues.isEmpty() == false) { documentFields.put(field, new DocumentField(field, parsedValues)); } } if (this.unmappedFieldsFetchAutomaton != null) { - collectUnmapped(documentFields, sourceLookup.source(), "", 0); + collectUnmapped(documentFields, valuesLookup.source().source(), "", 0); } return documentFields; } @@ -240,12 +239,6 @@ private static int step(CharacterRunAutomaton automaton, String key, int state) return state; } - public void setNextReader(LeafReaderContext readerContext) { - for (FieldContext field : fieldContexts.values()) { - field.valueFetcher.setNextReader(readerContext); - } - } - private static class FieldContext { final String fieldName; final ValueFetcher valueFetcher; diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/FastVectorHighlighter.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/FastVectorHighlighter.java index e1c347c9c3daf..358fb4cd95041 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/FastVectorHighlighter.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/FastVectorHighlighter.java @@ -126,7 +126,7 @@ public HighlightField highlight(FieldHighlightContext fieldContext) throws IOExc cache.fvh.setPhraseLimit(field.fieldOptions().phraseLimit()); String[] fragments; - FragmentsBuilder fragmentsBuilder = entry.fragmentsBuilderSupplier.apply(hitContext.sourceLookup()); + FragmentsBuilder fragmentsBuilder = entry.fragmentsBuilderSupplier.apply(hitContext.valuesLookup().source()); // a HACK to make highlighter do highlighting, even though its using the single frag list builder int numberOfFragments = field.fieldOptions().numberOfFragments() == 0 ? diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightUtils.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightUtils.java index 8bf7422ec0a66..eea780b3af9ec 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightUtils.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightUtils.java @@ -46,9 +46,8 @@ public static List loadFieldValues(MappedFieldType fieldType, List textsToHighlight = fieldVisitor.fields().get(fieldType.name()); return Objects.requireNonNullElse(textsToHighlight, Collections.emptyList()); } - ValueFetcher fetcher = fieldType.valueFetcher(searchContext, null); - fetcher.setNextReader(hitContext.readerContext()); - return fetcher.fetchValues(hitContext.sourceLookup(), Collections.emptySet()); + ValueFetcher fetcher = fieldType.valueFetcher(searchContext::sourcePath, null); + return fetcher.fetchValues(hitContext.valuesLookup(), Collections.emptySet()); } public static class Encoders { diff --git a/server/src/main/java/org/elasticsearch/search/lookup/LeafDocLookup.java b/server/src/main/java/org/elasticsearch/search/lookup/LeafDocLookup.java index fed14ae4a4431..7a94fe0c8b83c 100644 --- a/server/src/main/java/org/elasticsearch/search/lookup/LeafDocLookup.java +++ b/server/src/main/java/org/elasticsearch/search/lookup/LeafDocLookup.java @@ -21,22 +21,38 @@ import java.util.Map; import java.util.Set; import java.util.function.Function; +import java.util.function.Predicate; public class LeafDocLookup implements Map> { private final Map> localCacheFieldData = new HashMap<>(4); - private final Function fieldTypeLookup; - private final Function> fieldDataLookup; - private final LeafReaderContext reader; + private final Predicate fieldExists; + private final Function> loader; private int docId = -1; + public LeafDocLookup(Predicate fieldExists, Function> loader) { + this.loader = loader; + this.fieldExists = fieldExists; + } + LeafDocLookup(Function fieldTypeLookup, Function> fieldDataLookup, LeafReaderContext reader) { - this.fieldTypeLookup = fieldTypeLookup; - this.fieldDataLookup = fieldDataLookup; - this.reader = reader; + this(s -> fieldTypeLookup.apply(s) != null, fieldName -> { + final MappedFieldType fieldType = fieldTypeLookup.apply(fieldName); + if (fieldType == null) { + throw new IllegalArgumentException("No field found for [" + fieldName + "] in mapping"); + } + // load fielddata on behalf of the script: otherwise it would need additional permissions + // to deal with pagedbytes/ramusagestimator/etc + return AccessController.doPrivileged(new PrivilegedAction<>() { + @Override + public ScriptDocValues run() { + return fieldDataLookup.apply(fieldType).load(reader).getScriptValues(); + } + }); + }); } public void setDocument(int docId) { @@ -49,18 +65,7 @@ public ScriptDocValues get(Object key) { String fieldName = key.toString(); ScriptDocValues scriptValues = localCacheFieldData.get(fieldName); if (scriptValues == null) { - final MappedFieldType fieldType = fieldTypeLookup.apply(fieldName); - if (fieldType == null) { - throw new IllegalArgumentException("No field found for [" + fieldName + "] in mapping"); - } - // load fielddata on behalf of the script: otherwise it would need additional permissions - // to deal with pagedbytes/ramusagestimator/etc - scriptValues = AccessController.doPrivileged(new PrivilegedAction>() { - @Override - public ScriptDocValues run() { - return fieldDataLookup.apply(fieldType).load(reader).getScriptValues(); - } - }); + scriptValues = loader.apply(fieldName); localCacheFieldData.put(fieldName, scriptValues); } try { @@ -76,7 +81,7 @@ public boolean containsKey(Object key) { // assume its a string... String fieldName = key.toString(); ScriptDocValues scriptValues = localCacheFieldData.get(fieldName); - return scriptValues != null || fieldTypeLookup.apply(fieldName) != null; + return scriptValues != null || fieldExists.test(fieldName); } @Override diff --git a/server/src/main/java/org/elasticsearch/search/lookup/LeafSearchLookup.java b/server/src/main/java/org/elasticsearch/search/lookup/LeafSearchLookup.java index 0564d98499695..bf5ae001a3953 100644 --- a/server/src/main/java/org/elasticsearch/search/lookup/LeafSearchLookup.java +++ b/server/src/main/java/org/elasticsearch/search/lookup/LeafSearchLookup.java @@ -15,7 +15,7 @@ /** * Per-segment version of {@link SearchLookup}. */ -public class LeafSearchLookup { +public class LeafSearchLookup implements ValuesLookup { private final LeafReaderContext ctx; private final LeafDocLookup docMap; @@ -39,6 +39,7 @@ public Map asMap() { return this.asMap; } + @Override public SourceLookup source() { return this.sourceLookup; } @@ -47,6 +48,7 @@ public LeafStoredFieldsLookup fields() { return this.fieldsLookup; } + @Override public LeafDocLookup doc() { return this.docMap; } diff --git a/server/src/main/java/org/elasticsearch/search/lookup/ValuesLookup.java b/server/src/main/java/org/elasticsearch/search/lookup/ValuesLookup.java new file mode 100644 index 0000000000000..b2d4316c56ec6 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/lookup/ValuesLookup.java @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.search.lookup; + +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.index.fielddata.ScriptDocValues; + +import java.util.Map; + +public interface ValuesLookup { + + SourceLookup source(); + + Map> doc(); + + static ValuesLookup sourceOnly(BytesReference source) { + SourceLookup sourceLookup = new SourceLookup(); + sourceLookup.setSource(source); + return new ValuesLookup() { + @Override + public SourceLookup source() { + return sourceLookup; + } + + @Override + public Map> doc() { + throw new UnsupportedOperationException("FieldData is not available from a source-only lookup"); + } + }; + } + + static ValuesLookup sourceOnly(Map source) { + SourceLookup sourceLookup = new SourceLookup(); + sourceLookup.setSource(source); + return new ValuesLookup() { + @Override + public SourceLookup source() { + return sourceLookup; + } + + @Override + public Map> doc() { + throw new UnsupportedOperationException("FieldData is not available from a source-only lookup"); + } + }; + } +} diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java index dec496c5e8ce2..c499b5e77b47a 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java @@ -60,6 +60,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Function; import java.util.stream.Collectors; import static java.util.Collections.singletonList; @@ -1599,7 +1600,7 @@ public static class MetadataTimestampFieldMapper extends MetadataFieldMapper { public MetadataTimestampFieldMapper(boolean enabled) { super(new MappedFieldType("_data_stream_timestamp", false, false, false, TextSearchInfo.NONE, Map.of()) { @Override - public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { + public ValueFetcher valueFetcher(Function> sourcePaths, String format) { throw new UnsupportedOperationException(); } diff --git a/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java b/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java index cfe9dcf2e5517..f147ccea8edf9 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java @@ -25,6 +25,8 @@ import org.elasticsearch.test.ESTestCase; import java.util.Collections; +import java.util.Set; +import java.util.function.Function; import java.util.function.Supplier; import static org.elasticsearch.common.settings.Settings.Builder.EMPTY_SETTINGS; @@ -143,7 +145,7 @@ public void testIndexSorting() { final IndexFieldDataService indexFieldDataService = new IndexFieldDataService(indexSettings, cache, circuitBreakerService, null); MappedFieldType fieldType = new RuntimeFieldType("field", Collections.emptyMap()) { @Override - public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { + public ValueFetcher valueFetcher(Function> sourcePaths, String format) { throw new UnsupportedOperationException(); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/CalculatedFieldTests.java b/server/src/test/java/org/elasticsearch/index/mapper/CalculatedFieldTests.java new file mode 100644 index 0000000000000..f3d741658f64b --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/CalculatedFieldTests.java @@ -0,0 +1,109 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.mapper; + +import org.apache.lucene.index.IndexableField; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.ScriptPlugin; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.script.ScriptEngine; +import org.elasticsearch.search.lookup.SourceLookup; + +import java.io.IOException; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +public class CalculatedFieldTests extends MapperServiceTestCase { + + @Override + protected Collection getPlugins() { + return Set.of(new TestScriptPlugin()); + } + + public void testCalculatedFieldLength() throws IOException { + DocumentMapper mapper = createDocumentMapper(mapping(b -> { + b.startObject("message").field("type", "text").endObject(); + b.startObject("message_length"); + b.field("type", "long"); + b.field("script", "length"); + b.endObject(); + })); + + ParsedDocument doc = mapper.parse(source(b -> b.field("message", "this is some text"))); + IndexableField[] lengthFields = doc.rootDoc().getFields("message_length"); + assertEquals(2, lengthFields.length); + assertEquals("LongPoint ", lengthFields[0].toString()); + assertEquals("docValuesType=SORTED_NUMERIC", lengthFields[1].toString()); + } + + public void testSerialization() throws IOException { + DocumentMapper mapper = createDocumentMapper(mapping(b -> { + b.startObject("message").field("type", "text").endObject(); + b.startObject("message_length"); + b.field("type", "long"); + b.field("script", "length"); + b.endObject(); + })); + assertEquals( + "{\"_doc\":{\"properties\":{\"message\":{\"type\":\"text\"}," + + "\"message_length\":{\"type\":\"long\",\"script\":{\"source\":\"length\",\"lang\":\"painless\"}}}}}", + Strings.toString(mapper.mapping())); + } + + public static class TestScriptPlugin extends Plugin implements ScriptPlugin { + @Override + public List> getContexts() { + return List.of(NumberFieldMapper.SCRIPT_CONTEXT); + } + + @Override + public ScriptEngine getScriptEngine(Settings settings, Collection> contexts) { + return new ScriptEngine() { + @Override + public String getType() { + return "painless"; + } + + @Override + public FactoryType compile( + String name, String code, ScriptContext context, Map params) { + return (FactoryType) factory(code); + } + + @Override + public Set> getSupportedContexts() { + return Set.of(NumberFieldMapper.SCRIPT_CONTEXT); + } + + private NumberFieldMapper.ScriptFactory factory(String code) { + switch (code) { + case "length": + return () -> new NumberFieldMapper.NumberScript() { + @Override + public void execute() { + SourceLookup source = (SourceLookup) getParams().get("_source"); + for (Object v : source.extractRawValues("message")) { + emit(Objects.toString(v).length()); + } + } + }; + default: + throw new IllegalArgumentException("Cannot compile script [" + code + "]"); + } + } + }; + } + } + +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java b/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java index b4c8ce1e94e9a..74b0bb994bf2d 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java @@ -16,7 +16,6 @@ import org.elasticsearch.index.analysis.AnalyzerScope; import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.analysis.NamedAnalyzer; -import org.elasticsearch.index.query.SearchExecutionContext; import java.io.IOException; import java.nio.charset.Charset; @@ -26,6 +25,8 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.function.Function; /** * This mapper add a new sub fields @@ -102,8 +103,8 @@ public String typeName() { } @Override - public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { - return SourceValueFetcher.identity(name(), context, format); + public ValueFetcher valueFetcher(Function> sourcePaths, String format) { + return SourceValueFetcher.identity(name(), sourcePaths, format); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java b/server/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java index 3c44b6991389a..330d4b68f24cc 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java @@ -12,11 +12,12 @@ import org.apache.lucene.document.FieldType; import org.apache.lucene.index.IndexOptions; import org.elasticsearch.common.lucene.Lucene; -import org.elasticsearch.index.query.SearchExecutionContext; import java.io.IOException; import java.util.Collections; import java.util.List; +import java.util.Set; +import java.util.function.Function; // Like a String mapper but with very few options. We just use it to test if highlighting on a custom string mapped field works as expected. public class FakeStringFieldMapper extends FieldMapper { @@ -64,8 +65,8 @@ public String typeName() { } @Override - public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { - return SourceValueFetcher.toString(name(), context, format); + public ValueFetcher valueFetcher(Function> sourcePaths, String format) { + return SourceValueFetcher.toString(name(), sourcePaths, format); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IndexTimeScriptParamsTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IndexTimeScriptParamsTests.java new file mode 100644 index 0000000000000..b4afcbaaf1a7f --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/IndexTimeScriptParamsTests.java @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.mapper; + +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.index.fielddata.ScriptDocValues; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.util.Set; + +public class IndexTimeScriptParamsTests extends ESTestCase { + + public void testSource() throws IOException { + BytesReference source = BytesReference.bytes(XContentFactory.jsonBuilder() + .startObject() + .field("foo", "bar") + .endObject()); + + IndexTimeScriptParams params = new IndexTimeScriptParams(source, f -> null, Set::of); + assertEquals("bar", params.source().get("foo")); + } + + public void testDoc() throws IOException { + MappedFieldType longField = new NumberFieldMapper.NumberFieldType("longfield", NumberFieldMapper.NumberType.LONG); + + IndexTimeScriptParams params = new IndexTimeScriptParams( + new BytesArray("{\"longfield\":[10, 20]}"), + f -> longField, + Set::of); + ScriptDocValues values = params.doc().get("longfield"); + assertEquals(10L, values.get(0)); + assertEquals(20L, values.get(1)); + assertEquals(2, values.size()); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MappingLookupTests.java b/server/src/test/java/org/elasticsearch/index/mapper/MappingLookupTests.java index 80f221539da72..c33cad3c990e4 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/MappingLookupTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/MappingLookupTests.java @@ -16,7 +16,6 @@ import org.elasticsearch.common.Explicit; import org.elasticsearch.index.analysis.AnalyzerScope; import org.elasticsearch.index.analysis.NamedAnalyzer; -import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.test.ESTestCase; import java.io.IOException; @@ -24,6 +23,8 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Set; +import java.util.function.Function; import static java.util.Collections.emptyList; import static org.hamcrest.CoreMatchers.instanceOf; @@ -148,7 +149,7 @@ private FakeFieldType(String name) { } @Override - public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { + public ValueFetcher valueFetcher(Function> sourcePaths, String format) { throw new UnsupportedOperationException(); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RootObjectMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RootObjectMapperTests.java index 6011b5dd7c143..60f319d756fc2 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/RootObjectMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/RootObjectMapperTests.java @@ -25,6 +25,8 @@ import java.util.Collection; import java.util.Collections; import java.util.Map; +import java.util.Set; +import java.util.function.Function; import static org.elasticsearch.test.VersionUtils.randomVersionBetween; import static org.hamcrest.Matchers.containsString; @@ -728,7 +730,7 @@ protected RuntimeField(String name, String prop1, String prop2) { } @Override - public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { + public ValueFetcher valueFetcher(Function> sourcePaths, String format) { return null; } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TestRuntimeField.java b/server/src/test/java/org/elasticsearch/index/mapper/TestRuntimeField.java index f1875212bd055..889c5ffffe1e5 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TestRuntimeField.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TestRuntimeField.java @@ -17,6 +17,8 @@ import java.io.IOException; import java.util.Collections; import java.util.Map; +import java.util.Set; +import java.util.function.Function; public class TestRuntimeField extends RuntimeFieldType { @@ -32,7 +34,7 @@ protected void doXContentBody(XContentBuilder builder, boolean includeDefaults) } @Override - public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { + public ValueFetcher valueFetcher(Function> sourcePaths, String format) { return null; } diff --git a/server/src/test/java/org/elasticsearch/search/collapse/CollapseBuilderTests.java b/server/src/test/java/org/elasticsearch/search/collapse/CollapseBuilderTests.java index 16aef6da8f34b..c953b75c7a86d 100644 --- a/server/src/test/java/org/elasticsearch/search/collapse/CollapseBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/collapse/CollapseBuilderTests.java @@ -36,6 +36,8 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Set; +import java.util.function.Function; import static java.util.Collections.emptyList; import static org.mockito.Mockito.mock; @@ -196,7 +198,7 @@ public String typeName() { } @Override - public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { + public ValueFetcher valueFetcher(Function> sourcePaths, String format) { throw new UnsupportedOperationException(); } diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhaseTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhaseTests.java index a6fbaaf04a7a2..97f932a1b7f7a 100644 --- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhaseTests.java +++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhaseTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.search.fetch.FetchContext; import org.elasticsearch.search.fetch.FetchSubPhase.HitContext; import org.elasticsearch.search.fetch.FetchSubPhaseProcessor; +import org.elasticsearch.search.lookup.ValuesLookup; import org.elasticsearch.test.ESTestCase; import java.io.IOException; @@ -150,8 +151,11 @@ private HitContext hitExecuteMultiple(XContentBuilder source, boolean fetchSourc // We don't need a real index, just a LeafReaderContext which cannot be mocked. MemoryIndex index = new MemoryIndex(); LeafReaderContext leafReaderContext = index.createSearcher().getIndexReader().leaves().get(0); - HitContext hitContext = new HitContext(searchHit, leafReaderContext, 1); - hitContext.sourceLookup().setSource(source == null ? null : BytesReference.bytes(source)); + HitContext hitContext = new HitContext( + searchHit, + ValuesLookup.sourceOnly(source == null ? null : BytesReference.bytes(source)), + leafReaderContext, + 1); FetchSourcePhase phase = new FetchSourcePhase(); FetchSubPhaseProcessor processor = phase.getProcessor(fetchContext); diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java index ae85b28c914b3..e1808029af990 100644 --- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java +++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java @@ -22,7 +22,7 @@ import org.elasticsearch.index.mapper.MapperServiceTestCase; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.query.SearchExecutionContext; -import org.elasticsearch.search.lookup.SourceLookup; +import org.elasticsearch.search.lookup.ValuesLookup; import java.io.IOException; import java.util.ArrayList; @@ -786,12 +786,11 @@ private static Map fetchFields( List fields, @Nullable Set ignoreFields ) throws IOException { - - SourceLookup sourceLookup = new SourceLookup(); - sourceLookup.setSource(BytesReference.bytes(source)); - FieldFetcher fieldFetcher = FieldFetcher.create(newSearchExecutionContext(mapperService), fields); - return fieldFetcher.fetch(sourceLookup, ignoreFields != null ? ignoreFields : Collections.emptySet()); + return fieldFetcher.fetch( + ValuesLookup.sourceOnly(BytesReference.bytes(source)), + ignoreFields != null ? ignoreFields : Collections.emptySet() + ); } public MapperService createMapperService() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java b/server/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java index 6c8f845ae0111..c0953b84a90cf 100644 --- a/server/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java @@ -46,7 +46,9 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Function; import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode; import static org.hamcrest.Matchers.containsString; @@ -99,7 +101,7 @@ private SearchExecutionContext createShardContext(Version indexVersionCreated, I TextSearchInfo.NONE, Collections.emptyMap()) { @Override - public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { + public ValueFetcher valueFetcher(Function> sourcePaths, String format) { throw new UnsupportedOperationException(); } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java index 0752a54ef8923..70d974e30176c 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java @@ -8,7 +8,7 @@ package org.elasticsearch.index.mapper; import org.elasticsearch.index.query.SearchExecutionContext; -import org.elasticsearch.search.lookup.SourceLookup; +import org.elasticsearch.search.lookup.ValuesLookup; import org.elasticsearch.test.ESTestCase; import java.io.IOException; @@ -41,12 +41,8 @@ public static List fetchSourceValue(MappedFieldType fieldType, Object sourceV public static List fetchSourceValue(MappedFieldType fieldType, Object sourceValue, String format) throws IOException { String field = fieldType.name(); - SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class); - when(searchExecutionContext.sourcePath(field)).thenReturn(Set.of(field)); - ValueFetcher fetcher = fieldType.valueFetcher(searchExecutionContext, format); - SourceLookup lookup = new SourceLookup(); - lookup.setSource(Collections.singletonMap(field, sourceValue)); - return fetcher.fetchValues(lookup, Collections.emptySet()); + ValueFetcher fetcher = fieldType.valueFetcher(Set::of, format); + return fetcher.fetchValues(ValuesLookup.sourceOnly(Collections.singletonMap(field, sourceValue)), Collections.emptySet()); } } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java index bf4e76a50fcd9..70b85c81eb9e1 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java @@ -32,6 +32,7 @@ import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.lookup.LeafSearchLookup; import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; @@ -304,12 +305,12 @@ protected final List fetchFromDocValues(MapperService mapperService, MappedFi iw.addDocument(mapperService.documentMapper().parse(source(b -> b.field(ft.name(), sourceValue))).rootDoc()); }, iw -> { SearchLookup lookup = new SearchLookup(mapperService::fieldType, fieldDataLookup); - ValueFetcher valueFetcher = new DocValueFetcher(format, lookup.getForField(ft)); + ValueFetcher valueFetcher = new DocValueFetcher(format, ft.name()); IndexSearcher searcher = newSearcher(iw); LeafReaderContext context = searcher.getIndexReader().leaves().get(0); - lookup.source().setSegmentAndDocument(context, 0); - valueFetcher.setNextReader(context); - result.set(valueFetcher.fetchValues(lookup.source(), Collections.emptySet())); + LeafSearchLookup leaf = lookup.getLeafSearchLookup(context); + leaf.setDocument(0); + result.set(valueFetcher.fetchValues(leaf, Collections.emptySet())); }); return result.get(); } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java index 9a46baed3edc1..4c858cf08e4ac 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java @@ -9,11 +9,12 @@ package org.elasticsearch.index.mapper; import org.elasticsearch.index.analysis.NamedAnalyzer; -import org.elasticsearch.index.query.SearchExecutionContext; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.function.Function; // this sucks how much must be overridden just do get a dummy field mapper... public class MockFieldMapper extends FieldMapper { @@ -59,7 +60,7 @@ public String typeName() { } @Override - public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { + public ValueFetcher valueFetcher(Function> sourcePaths, String format) { throw new UnsupportedOperationException(); } } diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java index 0d4a637a82e86..761d81ad2a094 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java @@ -51,6 +51,8 @@ import java.io.IOException; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.function.Function; import java.util.function.Supplier; import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken; @@ -135,8 +137,8 @@ public String typeName() { } @Override - public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { - return SourceValueFetcher.identity(name(), context, format); + public ValueFetcher valueFetcher(Function> sourcePaths, String format) { + return SourceValueFetcher.identity(name(), sourcePaths, format); } @Override diff --git a/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/mapper/DataStreamTimestampFieldMapper.java b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/mapper/DataStreamTimestampFieldMapper.java index f4dfad960fd9e..86a9e0fa2b180 100644 --- a/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/mapper/DataStreamTimestampFieldMapper.java +++ b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/mapper/DataStreamTimestampFieldMapper.java @@ -30,6 +30,8 @@ import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.function.Function; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; @@ -62,7 +64,7 @@ public Query existsQuery(SearchExecutionContext context) { } @Override - public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { + public ValueFetcher valueFetcher(Function> sourcePaths, String format) { throw new UnsupportedOperationException(); } } diff --git a/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapper.java b/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapper.java index 5c9fa514e3007..f1904e7b0aa78 100644 --- a/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapper.java +++ b/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapper.java @@ -55,6 +55,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -431,12 +432,12 @@ public BucketedSort newBucketedSort( } @Override - public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { + public ValueFetcher valueFetcher(Function> sourcePaths, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } - return new SourceValueFetcher(name(), context) { + return new SourceValueFetcher(name(), sourcePaths.apply(name()), null) { @Override @SuppressWarnings("unchecked") protected Object parseSourceValue(Object value) { diff --git a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java index d5f4539454f30..4ec868b8bb38f 100644 --- a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java +++ b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java @@ -45,6 +45,8 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; +import java.util.function.Function; import java.util.function.Supplier; /** @@ -131,7 +133,7 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, S } @Override - public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { + public ValueFetcher valueFetcher(Function> sourcePaths, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } diff --git a/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldTypeTests.java b/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldTypeTests.java index 0392fa327ed0d..9b57924eaebb4 100644 --- a/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldTypeTests.java +++ b/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldTypeTests.java @@ -10,11 +10,12 @@ import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.util.automaton.RegExp; +import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.index.mapper.FieldTypeTestCase; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.ValueFetcher; -import org.elasticsearch.search.lookup.SourceLookup; +import org.elasticsearch.search.lookup.ValuesLookup; import org.elasticsearch.xpack.constantkeyword.mapper.ConstantKeywordFieldMapper.ConstantKeywordFieldType; import java.util.Arrays; @@ -110,9 +111,8 @@ public void testFetchValue() throws Exception { MappedFieldType fieldType = new ConstantKeywordFieldMapper.ConstantKeywordFieldType("field", null); ValueFetcher fetcher = fieldType.valueFetcher(null, null); - SourceLookup missingValueLookup = new SourceLookup(); - SourceLookup nullValueLookup = new SourceLookup(); - nullValueLookup.setSource(Collections.singletonMap("field", null)); + ValuesLookup missingValueLookup = ValuesLookup.sourceOnly(new BytesArray("{}")); + ValuesLookup nullValueLookup = ValuesLookup.sourceOnly(Collections.singletonMap("field", null)); assertTrue(fetcher.fetchValues(missingValueLookup, Collections.emptySet()).isEmpty()); assertTrue(fetcher.fetchValues(nullValueLookup, Collections.emptySet()).isEmpty()); diff --git a/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlattenedFieldMapper.java b/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlattenedFieldMapper.java index 45e8cfbca607c..00a3879bda509 100644 --- a/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlattenedFieldMapper.java +++ b/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlattenedFieldMapper.java @@ -55,6 +55,8 @@ import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.function.Function; import java.util.function.Supplier; /** @@ -257,7 +259,7 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, S } @Override - public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { + public ValueFetcher valueFetcher(Function> sourcePaths, String format) { // This is an internal field but it can match a field pattern so we return an empty list. return (lookup, ignoredFields) -> List.of(); } @@ -403,8 +405,8 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, S } @Override - public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { - return SourceValueFetcher.identity(name(), context, format); + public ValueFetcher valueFetcher(Function> sourcePaths, String format) { + return SourceValueFetcher.identity(name(), sourcePaths, format); } } diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java index 2c39719eac6bc..23e034eb17d56 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java @@ -47,6 +47,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.function.Function; import java.util.function.Supplier; @@ -238,12 +239,12 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, S } @Override - public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { + public ValueFetcher valueFetcher(Function> sourcePaths, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } - return new SourceValueFetcher(name(), context, nullValueFormatted) { + return new SourceValueFetcher(name(), sourcePaths.apply(name()), nullValueFormatted) { @Override protected Object parseSourceValue(Object value) { if (value.equals("")) { diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongLeafFieldData.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongLeafFieldData.java index c56d427bca9cd..cd58a55836b18 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongLeafFieldData.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongLeafFieldData.java @@ -16,7 +16,6 @@ import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; -import org.elasticsearch.index.mapper.DocValueFetcher; import org.elasticsearch.search.DocValueFormat; import java.io.IOException; @@ -94,9 +93,9 @@ public void close() { } @Override - public DocValueFetcher.Leaf getLeafValueFetcher(DocValueFormat format) { + public Leaf getLeafValueFetcher(DocValueFormat format) { SortedNumericDocValues values = getLongValues(); - return new DocValueFetcher.Leaf() { + return new Leaf() { @Override public boolean advanceExact(int docId) throws IOException { return values.advanceExact(docId); diff --git a/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java b/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java index 88b163d33f6ba..b83c79673e0cd 100644 --- a/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java +++ b/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java @@ -58,6 +58,8 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.function.Function; import java.util.function.Supplier; import static org.elasticsearch.search.SearchService.ALLOW_EXPENSIVE_QUERIES; @@ -132,8 +134,8 @@ public String typeName() { } @Override - public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { - return SourceValueFetcher.toString(name(), context, format); + public ValueFetcher valueFetcher(Function> sourcePaths, String format) { + return SourceValueFetcher.toString(name(), sourcePaths, format); } @Override @@ -387,6 +389,11 @@ public String format(BytesRef value) { return VersionEncoder.decodeVersion(value); } + @Override + public Object formatObject(Object in) { + return format((BytesRef) in); + } + @Override public BytesRef parseBytesRef(String value) { return VersionEncoder.encodeVersion(value).bytesRef; diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/FieldValueFetcher.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/FieldValueFetcher.java index bd473075fb5d1..d0d86be43d787 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/FieldValueFetcher.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/FieldValueFetcher.java @@ -10,7 +10,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.util.BytesRef; import org.elasticsearch.index.fielddata.IndexFieldData; -import org.elasticsearch.index.mapper.DocValueFetcher; +import org.elasticsearch.index.fielddata.LeafFieldData; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.search.DocValueFormat; @@ -46,9 +46,9 @@ protected FieldValueFetcher(String name, this.valueFunc = valueFunc; } - DocValueFetcher.Leaf getLeaf(LeafReaderContext context) { - final DocValueFetcher.Leaf delegate = fieldData.load(context).getLeafValueFetcher(DocValueFormat.RAW); - return new DocValueFetcher.Leaf() { + LeafFieldData.Leaf getLeaf(LeafReaderContext context) { + final LeafFieldData.Leaf delegate = fieldData.load(context).getLeafValueFetcher(DocValueFormat.RAW); + return new LeafFieldData.Leaf() { @Override public boolean advanceExact(int docId) throws IOException { return delegate.advanceExact(docId); diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/RollupShardIndexer.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/RollupShardIndexer.java index 351efbccea1dc..8e0761b872bde 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/RollupShardIndexer.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/RollupShardIndexer.java @@ -42,9 +42,9 @@ import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.engine.Engine; +import org.elasticsearch.index.fielddata.LeafFieldData; import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.DocCountFieldMapper; -import org.elasticsearch.index.mapper.DocValueFetcher; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.shard.IndexShard; @@ -434,8 +434,8 @@ private BucketCollector(long timestamp, @Override public LeafCollector getLeafCollector(LeafReaderContext context) { - final List groupFieldLeaves = leafFetchers(context, groupFieldFetchers); - final List metricsFieldLeaves = leafFetchers(context, metricsFieldFetchers); + final List groupFieldLeaves = leafFetchers(context, groupFieldFetchers); + final List metricsFieldLeaves = leafFetchers(context, metricsFieldFetchers); return new LeafCollector() { @Override public void setScorer(Scorable scorer) { @@ -444,7 +444,7 @@ public void setScorer(Scorable scorer) { @Override public void collect(int docID) throws IOException { List> combinationKeys = new ArrayList<>(); - for (DocValueFetcher.Leaf leafField : groupFieldLeaves) { + for (LeafFieldData.Leaf leafField : groupFieldLeaves) { if (leafField.advanceExact(docID)) { List lst = new ArrayList<>(); for (int i = 0; i < leafField.docValueCount(); i++) { @@ -458,7 +458,7 @@ public void collect(int docID) throws IOException { final BytesRef valueBytes; try (BytesStreamOutput out = new BytesStreamOutput()) { - for (DocValueFetcher.Leaf leaf : metricsFieldLeaves) { + for (LeafFieldData.Leaf leaf : metricsFieldLeaves) { if (leaf.advanceExact(docID)) { out.writeVInt(leaf.docValueCount()); for (int i = 0; i < leaf.docValueCount(); i++) { @@ -487,8 +487,8 @@ public void collect(int docID) throws IOException { }; } - private List leafFetchers(LeafReaderContext context, List fetchers) { - List leaves = new ArrayList<>(); + private List leafFetchers(LeafReaderContext context, List fetchers) { + List leaves = new ArrayList<>(); for (FieldValueFetcher fetcher : fetchers) { leaves.add(fetcher.getLeaf(context)); } diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractScriptFieldType.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractScriptFieldType.java index a16db46806025..302d966815e03 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractScriptFieldType.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractScriptFieldType.java @@ -37,6 +37,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Set; import java.util.function.BiFunction; import java.util.function.Function; @@ -198,8 +199,8 @@ protected final void checkAllowExpensiveQueries(SearchExecutionContext context) } @Override - public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { - return new DocValueFetcher(docValueFormat(format, null), context.getForField(this)); + public ValueFetcher valueFetcher(Function> sourcePaths, String format) { + return new DocValueFetcher(docValueFormat(format, null), name()); } @Override diff --git a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java index 5fc3d3c235893..dd6e97f5d015c 100644 --- a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java +++ b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java @@ -34,6 +34,8 @@ import java.time.ZoneId; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.function.Function; import java.util.function.Supplier; import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken; @@ -110,11 +112,11 @@ public String typeName() { } @Override - public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { + public ValueFetcher valueFetcher(Function> sourcePaths, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } - return new ArraySourceValueFetcher(name(), context) { + return new ArraySourceValueFetcher(name(), sourcePaths.apply(name()), null) { @Override protected Object parseSourceValue(Object value) { return value; diff --git a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapper.java b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapper.java index 64abe4f46036b..b34fc37a4819c 100644 --- a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapper.java +++ b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapper.java @@ -24,6 +24,8 @@ import java.time.ZoneId; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.function.Function; /** * A {@link FieldMapper} for indexing a sparse vector of floats. @@ -88,7 +90,7 @@ public DocValueFormat docValueFormat(String format, ZoneId timeZone) { } @Override - public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { + public ValueFetcher valueFetcher(Function> sourcePaths, String format) { throw new UnsupportedOperationException(ERROR_MESSAGE_7X); } diff --git a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java index 49d044eb34acf..07235f8140838 100644 --- a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java +++ b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java @@ -78,6 +78,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.Function; import java.util.function.Supplier; /** @@ -909,12 +910,12 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, S } @Override - public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { + public ValueFetcher valueFetcher(Function> sourcePaths, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } - return new SourceValueFetcher(name(), context, nullValue) { + return new SourceValueFetcher(name(), sourcePaths.apply(name()), nullValue) { @Override protected String parseSourceValue(Object value) { String keywordValue = value.toString();