From 9a127adb4b1dd8e69ac5b6c08cc7251383562e9c Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Tue, 15 Sep 2020 15:57:26 -0400 Subject: [PATCH] Implement fields fetch for runtime fields (#61995) This implements the `fields` API in `_search` for runtime fields using doc values. Most of that implementation is stolen from the `docvalue_fields` fetch sub-phase, just moved into the same API that the `fields` API uses. At this point the `docvalue_fields` fetch phase looks like a special case of the `fields` API. While I was at it I moved the "which doc values sub-implementation should I use for fetching?" question from a bunch of `instanceof`s to a method on `LeafFieldData` so we can be much more flexible with what is returned and we're not forced to extend certain classes just to make the fetch phase happy. Relates to #59332 --- .../index/mapper/RankFeatureFieldMapper.java | 2 +- .../index/mapper/RankFeaturesFieldMapper.java | 2 +- .../index/mapper/ScaledFloatFieldMapper.java | 2 +- .../mapper/SearchAsYouTypeFieldMapper.java | 7 +- .../index/mapper/TokenCountFieldMapper.java | 3 +- .../mapper/RankFeatureFieldMapperTests.java | 2 +- .../mapper/ScaledFloatFieldMapperTests.java | 2 +- .../join/mapper/MetaJoinFieldMapper.java | 2 +- .../join/mapper/ParentIdFieldMapper.java | 2 +- .../join/mapper/ParentJoinFieldMapper.java | 2 +- .../percolator/PercolatorFieldMapper.java | 3 +- .../PercolatorHighlightSubFetchPhase.java | 6 +- .../PercolatorMatchedSlotSubFetchPhase.java | 3 +- ...PercolatorHighlightSubFetchPhaseTests.java | 4 +- ...rcolatorMatchedSlotSubFetchPhaseTests.java | 13 +- .../ICUCollationKeywordFieldMapper.java | 2 +- .../ICUCollationKeywordFieldMapperTests.java | 2 +- .../AnnotatedTextFieldMapperTests.java | 2 +- .../AnnotatedTextFieldMapper.java | 3 +- .../mapper/murmur3/Murmur3FieldMapper.java | 2 +- .../test/search/240_date_nanos.yml | 11 +- .../search/fetch/FetchSubPhasePluginIT.java | 3 +- .../fielddata/IndexNumericFieldData.java | 4 +- .../index/fielddata/LeafFieldData.java | 26 +++ .../fielddata/plain/LeafDoubleFieldData.java | 26 ++- .../fielddata/plain/LeafLongFieldData.java | 28 ++- .../plain/SortedNumericIndexFieldData.java | 24 +++ .../mapper/AbstractGeometryFieldMapper.java | 3 +- .../index/mapper/BinaryFieldMapper.java | 2 +- .../index/mapper/BooleanFieldMapper.java | 2 +- .../index/mapper/CompletionFieldMapper.java | 3 +- .../index/mapper/DateFieldMapper.java | 2 +- .../index/mapper/DocValueFetcher.java | 79 ++++++++ .../index/mapper/FieldMapper.java | 4 +- .../index/mapper/IpFieldMapper.java | 2 +- .../index/mapper/KeywordFieldMapper.java | 2 +- .../index/mapper/MetadataFieldMapper.java | 3 +- .../index/mapper/NumberFieldMapper.java | 2 +- .../index/mapper/RangeFieldMapper.java | 2 +- .../index/mapper/TextFieldMapper.java | 6 +- .../index/mapper/ValueFetcher.java | 9 +- .../index/query/InnerHitContextBuilder.java | 3 +- .../index/query/QueryShardContext.java | 20 +- .../search/DefaultSearchContext.java | 6 - .../elasticsearch/search/SearchService.java | 4 +- .../metrics/TopHitsAggregatorFactory.java | 3 +- .../search/fetch/FetchPhase.java | 45 +++-- .../search/fetch/FetchSubPhase.java | 16 +- .../search/fetch/subphase/ExplainPhase.java | 3 +- .../fetch/subphase/FetchDocValuesPhase.java | 172 +++--------------- .../fetch/subphase/FetchFieldsContext.java | 31 ++-- .../fetch/subphase/FetchFieldsPhase.java | 16 +- .../fetch/subphase/FetchScorePhase.java | 3 +- .../fetch/subphase/FetchSourcePhase.java | 3 +- .../fetch/subphase/FetchVersionPhase.java | 3 +- .../fetch/subphase/FieldValueRetriever.java | 24 ++- .../search/fetch/subphase/InnerHitsPhase.java | 3 +- .../fetch/subphase/MatchedQueriesPhase.java | 3 +- .../fetch/subphase/ScriptFieldsPhase.java | 3 +- .../fetch/subphase/SeqNoPrimaryTermPhase.java | 3 +- .../subphase/highlight/HighlightPhase.java | 3 +- .../internal/FilteredSearchContext.java | 6 - .../search/internal/SearchContext.java | 3 - .../search/lookup/SourceLookup.java | 4 + .../index/mapper/BooleanFieldMapperTests.java | 2 +- .../mapper/CompletionFieldMapperTests.java | 2 +- .../index/mapper/DateFieldMapperTests.java | 29 ++- .../mapper/DocumentFieldMapperTests.java | 3 +- .../index/mapper/ExternalMapper.java | 3 +- .../index/mapper/FakeStringFieldMapper.java | 3 +- .../mapper/GeoPointFieldMapperTests.java | 2 +- .../mapper/GeoShapeFieldMapperTests.java | 2 +- .../index/mapper/IpFieldMapperTests.java | 2 +- .../index/mapper/IpRangeFieldMapperTests.java | 3 +- .../index/mapper/KeywordFieldMapperTests.java | 2 +- .../LegacyGeoShapeFieldMapperTests.java | 2 +- .../index/mapper/NumberFieldMapperTests.java | 2 +- .../index/mapper/ParametrizedMapperTests.java | 3 +- .../index/mapper/RangeFieldMapperTests.java | 4 +- .../index/mapper/TextFieldMapperTests.java | 2 +- .../fetch/subphase/FetchSourcePhaseTests.java | 5 +- .../subphase/FieldValueRetrieverTests.java | 10 +- .../index/mapper/FieldMapperTestCase.java | 6 +- .../index/mapper/MapperTestCase.java | 40 +++- .../index/mapper/MockFieldMapper.java | 3 +- .../aggregations/AggregatorTestCase.java | 7 - .../elasticsearch/test/TestSearchContext.java | 6 - .../mapper/HistogramFieldMapper.java | 2 +- .../search/AsyncSearchSingleNodeTests.java | 2 +- .../ConstantKeywordFieldMapperTests.java | 4 +- .../mapper/ConstantKeywordFieldMapper.java | 2 +- .../mapper/FlatObjectFieldMapperTests.java | 2 +- .../mapper/FlatObjectFieldMapper.java | 2 +- .../runtime-fields/qa/rest/build.gradle | 2 - .../mapper/RuntimeFieldMapper.java | 6 +- .../index/mapper/PointFieldMapperTests.java | 2 +- .../index/mapper/ShapeFieldMapperTests.java | 2 +- .../test/runtime_fields/10_keyword.yml | 14 ++ .../test/runtime_fields/20_long.yml | 26 +++ .../test/runtime_fields/30_double.yml | 19 ++ .../test/runtime_fields/40_date.yml | 23 +++ .../test/runtime_fields/50_ip.yml | 19 ++ .../test/runtime_fields/60_boolean.yml | 13 ++ .../mapper/DenseVectorFieldMapper.java | 2 +- .../mapper/SparseVectorFieldMapper.java | 3 +- .../wildcard/mapper/WildcardFieldMapper.java | 2 +- .../mapper/WildcardFieldMapperTests.java | 2 +- 107 files changed, 640 insertions(+), 341 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/index/mapper/DocValueFetcher.java diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java index 3513406d2ae75..705d3efd46321 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java @@ -183,7 +183,7 @@ private Float objectToFloat(Object value) { } @Override - public ValueFetcher valueFetcher(MapperService mapperService, String format) { + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java index 59b8ccb7f05f9..bf265e65c9f2e 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java @@ -162,7 +162,7 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - public ValueFetcher valueFetcher(MapperService mapperService, String format) { + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java index f3975b6d57e56..664a3a9052501 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java @@ -397,7 +397,7 @@ private static double objectToDouble(Object value) { } @Override - public ValueFetcher valueFetcher(MapperService mapperService, String format) { + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java index 87ae686427f89..53e06860af776 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java @@ -54,6 +54,7 @@ import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.similarity.SimilarityProvider; import org.elasticsearch.index.similarity.SimilarityService; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.util.ArrayList; @@ -419,7 +420,7 @@ protected void parseCreateField(ParseContext context) { } @Override - public ValueFetcher valueFetcher(MapperService mapperService, String format) { + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { throw new UnsupportedOperationException(); } @@ -465,7 +466,7 @@ protected void mergeOptions(FieldMapper other, List conflicts) { } @Override - public ValueFetcher valueFetcher(MapperService mapperService, String format) { + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { throw new UnsupportedOperationException(); } @@ -588,7 +589,7 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - public ValueFetcher valueFetcher(MapperService mapperService, String format) { + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { throw new UnsupportedOperationException(); } diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java index 7e99fc1b1969e..087b828cdb232 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java @@ -25,6 +25,7 @@ import org.apache.lucene.document.FieldType; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.analysis.NamedAnalyzer; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.util.Iterator; @@ -159,7 +160,7 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - public ValueFetcher valueFetcher(MapperService mapperService, String format) { + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureFieldMapperTests.java index 26ffbe53f5e66..b397f2b3406f2 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureFieldMapperTests.java @@ -147,7 +147,7 @@ public void testRejectMultiValuedFields() throws MapperParsingException, IOExcep e.getCause().getMessage()); } - public void testFetchSourceValue() { + public void testFetchSourceValue() throws IOException { Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); RankFeatureFieldMapper mapper = new RankFeatureFieldMapper.Builder("field").build(context); diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java index 0edb1073ec6ee..f084bd8ad259e 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java @@ -259,7 +259,7 @@ public void testRejectIndexOptions() { containsString("Failed to parse mapping: unknown parameter [index_options] on mapper [field] of type [scaled_float]")); } - public void testFetchSourceValue() { + public void testFetchSourceValue() throws IOException { Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/MetaJoinFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/MetaJoinFieldMapper.java index 161c61137d8c5..e620f0d388853 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/MetaJoinFieldMapper.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/MetaJoinFieldMapper.java @@ -140,7 +140,7 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - public ValueFetcher valueFetcher(MapperService mapperService, String format) { + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { throw new UnsupportedOperationException("Cannot fetch values for metadata field [" + typeName() + "]."); } diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java index b9bf5ade734c4..9fa5a35f5bfff 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java @@ -190,7 +190,7 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - public ValueFetcher valueFetcher(MapperService mapperService, String format) { + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { throw new UnsupportedOperationException("Cannot fetch values for internal field [" + typeName() + "]."); } diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java index aef2495932ec0..2fb5d37a30957 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java @@ -352,7 +352,7 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - public ValueFetcher valueFetcher(MapperService mapperService, String format) { + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java index 83e3a7282a701..e3c514e8bf7a8 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java @@ -79,6 +79,7 @@ import org.elasticsearch.index.query.QueryShardException; import org.elasticsearch.index.query.Rewriteable; import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -366,7 +367,7 @@ public void parse(ParseContext context) throws IOException { } @Override - public ValueFetcher valueFetcher(MapperService mapperService, String format) { + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhase.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhase.java index 5bfe1bc5af626..d1b6ad5fffcdd 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhase.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhase.java @@ -34,6 +34,8 @@ import org.elasticsearch.search.fetch.subphase.highlight.Highlighter; import org.elasticsearch.search.fetch.subphase.highlight.SearchHighlightContext; import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.search.lookup.SourceLookup; import java.io.IOException; import java.util.ArrayList; @@ -54,7 +56,7 @@ final class PercolatorHighlightSubFetchPhase implements FetchSubPhase { } @Override - public FetchSubPhaseProcessor getProcessor(SearchContext searchContext) throws IOException { + public FetchSubPhaseProcessor getProcessor(SearchContext searchContext, SearchLookup lookup) throws IOException { if (searchContext.highlight() == null) { return null; } @@ -95,7 +97,7 @@ public void process(HitContext hit) throws IOException { BytesReference document = percolateQuery.getDocuments().get(slot); HitContext subContext = new HitContext( new SearchHit(slot, "unknown", Collections.emptyMap(), Collections.emptyMap()), - percolatorLeafReaderContext, slot, new HashMap<>() + percolatorLeafReaderContext, slot, new SourceLookup(), new HashMap<>() ); subContext.sourceLookup().setSource(document); // force source because MemoryIndex does not store fields diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorMatchedSlotSubFetchPhase.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorMatchedSlotSubFetchPhase.java index 56ba934486cec..aaf4c3868d89c 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorMatchedSlotSubFetchPhase.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorMatchedSlotSubFetchPhase.java @@ -36,6 +36,7 @@ import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.fetch.FetchSubPhaseProcessor; import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.util.ArrayList; @@ -56,7 +57,7 @@ final class PercolatorMatchedSlotSubFetchPhase implements FetchSubPhase { static final String FIELD_NAME_PREFIX = "_percolator_document_slot"; @Override - public FetchSubPhaseProcessor getProcessor(SearchContext searchContext) throws IOException { + public FetchSubPhaseProcessor getProcessor(SearchContext searchContext, SearchLookup lookup) throws IOException { List percolateContexts = new ArrayList<>(); List percolateQueries = locatePercolatorQuery(searchContext.query()); diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhaseTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhaseTests.java index 15d5682725dc1..16f7030f14f14 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhaseTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhaseTests.java @@ -52,9 +52,9 @@ public void testHitsExecutionNeeded() throws IOException { Mockito.when(searchContext.highlight()).thenReturn(new SearchHighlightContext(Collections.emptyList())); Mockito.when(searchContext.query()).thenReturn(new MatchAllDocsQuery()); - assertNull(subFetchPhase.getProcessor(searchContext)); + assertNull(subFetchPhase.getProcessor(searchContext, null)); Mockito.when(searchContext.query()).thenReturn(percolateQuery); - assertNotNull(subFetchPhase.getProcessor(searchContext)); + assertNotNull(subFetchPhase.getProcessor(searchContext, null)); } public void testLocatePercolatorQuery() { diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorMatchedSlotSubFetchPhaseTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorMatchedSlotSubFetchPhaseTests.java index 26351f96dc06a..bdc8db34d96ba 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorMatchedSlotSubFetchPhaseTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorMatchedSlotSubFetchPhaseTests.java @@ -40,6 +40,7 @@ import org.elasticsearch.search.fetch.FetchSubPhase.HitContext; import org.elasticsearch.search.fetch.FetchSubPhaseProcessor; import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.test.ESTestCase; import java.util.Collections; @@ -66,7 +67,7 @@ public void testHitsExecute() throws Exception { LeafReaderContext context = reader.leaves().get(0); // A match: { - HitContext hit = new HitContext(new SearchHit(0), context, 0, new HashMap<>()); + HitContext hit = new HitContext(new SearchHit(0), context, 0, new SourceLookup(), new HashMap<>()); PercolateQuery.QueryStore queryStore = ctx -> docId -> new TermQuery(new Term("field", "value")); MemoryIndex memoryIndex = new MemoryIndex(); memoryIndex.addField("field", "value", new WhitespaceAnalyzer()); @@ -77,7 +78,7 @@ public void testHitsExecute() throws Exception { SearchContext sc = mock(SearchContext.class); when(sc.query()).thenReturn(percolateQuery); - FetchSubPhaseProcessor processor = phase.getProcessor(sc); + FetchSubPhaseProcessor processor = phase.getProcessor(sc, null); assertNotNull(processor); processor.process(hit); @@ -87,7 +88,7 @@ public void testHitsExecute() throws Exception { // No match: { - HitContext hit = new HitContext(new SearchHit(0), context, 0, new HashMap<>()); + HitContext hit = new HitContext(new SearchHit(0), context, 0, new SourceLookup(), new HashMap<>()); PercolateQuery.QueryStore queryStore = ctx -> docId -> new TermQuery(new Term("field", "value")); MemoryIndex memoryIndex = new MemoryIndex(); memoryIndex.addField("field", "value1", new WhitespaceAnalyzer()); @@ -98,7 +99,7 @@ public void testHitsExecute() throws Exception { SearchContext sc = mock(SearchContext.class); when(sc.query()).thenReturn(percolateQuery); - FetchSubPhaseProcessor processor = phase.getProcessor(sc); + FetchSubPhaseProcessor processor = phase.getProcessor(sc, null); assertNotNull(processor); processor.process(hit); @@ -107,7 +108,7 @@ public void testHitsExecute() throws Exception { // No query: { - HitContext hit = new HitContext(new SearchHit(0), context, 0, new HashMap<>()); + HitContext hit = new HitContext(new SearchHit(0), context, 0, new SourceLookup(), new HashMap<>()); PercolateQuery.QueryStore queryStore = ctx -> docId -> null; MemoryIndex memoryIndex = new MemoryIndex(); memoryIndex.addField("field", "value", new WhitespaceAnalyzer()); @@ -118,7 +119,7 @@ public void testHitsExecute() throws Exception { SearchContext sc = mock(SearchContext.class); when(sc.query()).thenReturn(percolateQuery); - FetchSubPhaseProcessor processor = phase.getProcessor(sc); + FetchSubPhaseProcessor processor = phase.getProcessor(sc, null); assertNotNull(processor); processor.process(hit); diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java index 50d19c3c6684d..2e0b67643837b 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java @@ -735,7 +735,7 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - public ValueFetcher valueFetcher(MapperService mapperService, String format) { + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } diff --git a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperTests.java b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperTests.java index 3966dc5dc7279..9eea934991833 100644 --- a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperTests.java +++ b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperTests.java @@ -317,7 +317,7 @@ public void testUpdateIgnoreAbove() throws IOException { assertEquals(0, fields.length); } - public void testFetchSourceValue() { + public void testFetchSourceValue() throws IOException { Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); diff --git a/plugins/mapper-annotated-text/src/internalClusterTest/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java b/plugins/mapper-annotated-text/src/internalClusterTest/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java index 293cd781638d8..ed84512c27c6a 100644 --- a/plugins/mapper-annotated-text/src/internalClusterTest/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java +++ b/plugins/mapper-annotated-text/src/internalClusterTest/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java @@ -678,7 +678,7 @@ public void testEmptyName() throws IOException { assertThat(e.getMessage(), containsString("name cannot be empty string")); } - public void testFetchSourceValue() { + public void testFetchSourceValue() throws IOException { Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); diff --git a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java index dfc9ac93f875a..4bc255581e850 100644 --- a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java +++ b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java @@ -46,6 +46,7 @@ import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.mapper.annotatedtext.AnnotatedTextFieldMapper.AnnotatedText.AnnotationToken; import org.elasticsearch.index.similarity.SimilarityProvider; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.io.Reader; @@ -589,7 +590,7 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - public ValueFetcher valueFetcher(MapperService mapperService, String format) { + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } diff --git a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java index 117b36366e22c..614df262eba62 100644 --- a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java +++ b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java @@ -153,7 +153,7 @@ protected void parseCreateField(ParseContext context) } @Override - public ValueFetcher valueFetcher(MapperService mapperService, String format) { + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/240_date_nanos.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/240_date_nanos.yml index 2caf9c7084792..cc9bd063ff4fc 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/240_date_nanos.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/240_date_nanos.yml @@ -110,8 +110,15 @@ setup: rest_total_hits_as_int: true index: date* body: - docvalue_fields: [ { "field": "date", "format" : "strict_date_optional_time" }, { "field": "date", "format": "epoch_millis" }, { "field" : "date", "format": "uuuu-MM-dd'T'HH:mm:ss.SSSSSSSSSX" } ] - sort: [ { "date": "desc" } ] + docvalue_fields: + - field: date + format: strict_date_optional_time + - field: date + format: epoch_millis + - field: date + format: uuuu-MM-dd'T'HH:mm:ss.SSSSSSSSSX + sort: + - date: desc - match: { hits.total: 2 } - length: { hits.hits: 2 } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java index 9ed2a5397a12b..b57c6482526f4 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java @@ -39,6 +39,7 @@ import org.elasticsearch.search.SearchExtBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; @@ -114,7 +115,7 @@ private static final class TermVectorsFetchSubPhase implements FetchSubPhase { private static final String NAME = "term_vectors_fetch"; @Override - public FetchSubPhaseProcessor getProcessor(SearchContext searchContext) { + public FetchSubPhaseProcessor getProcessor(SearchContext searchContext, SearchLookup lookup) { return new FetchSubPhaseProcessor() { @Override public void setNextReader(LeafReaderContext readerContext) { diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java index 6721716bb160e..8f2776de1a826 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java @@ -32,10 +32,10 @@ import org.elasticsearch.index.fielddata.fieldcomparator.LongValuesComparatorSource; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.MultiValueMode; -import org.elasticsearch.search.sort.BucketedSort; -import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.aggregations.support.ValuesSourceType; +import org.elasticsearch.search.sort.BucketedSort; +import org.elasticsearch.search.sort.SortOrder; import java.io.IOException; import java.util.function.LongUnaryOperator; diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/LeafFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/LeafFieldData.java index 34d3cc05aa22b..a094d83e02dca 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/LeafFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/LeafFieldData.java @@ -21,6 +21,10 @@ import org.apache.lucene.util.Accountable; import org.elasticsearch.common.lease.Releasable; +import org.elasticsearch.index.mapper.DocValueFetcher; +import org.elasticsearch.search.DocValueFormat; + +import java.io.IOException; /** * The thread safe {@link org.apache.lucene.index.LeafReader} level cache of the data. @@ -37,4 +41,26 @@ public interface LeafFieldData extends Accountable, Releasable { */ SortedBinaryDocValues getBytesValues(); + /** + * Return a value fetcher for this leaf implementation. + */ + default DocValueFetcher.Leaf getLeafValueFetcher(DocValueFormat format) { + SortedBinaryDocValues values = getBytesValues(); + return new DocValueFetcher.Leaf() { + @Override + public boolean advanceExact(int docId) throws IOException { + return values.advanceExact(docId); + } + + @Override + public int docValueCount() throws IOException { + return values.docValueCount(); + } + + @Override + public Object nextValue() throws IOException { + return format.format(values.nextValue()); + } + }; + } } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafDoubleFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafDoubleFieldData.java index 3ab7847f641d7..a3277f97fb799 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafDoubleFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafDoubleFieldData.java @@ -21,12 +21,15 @@ import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.util.Accountable; -import org.elasticsearch.index.fielddata.LeafNumericFieldData; import org.elasticsearch.index.fielddata.FieldData; +import org.elasticsearch.index.fielddata.LeafNumericFieldData; import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; +import org.elasticsearch.index.mapper.DocValueFetcher; +import org.elasticsearch.search.DocValueFormat; +import java.io.IOException; import java.util.Collection; import java.util.Collections; @@ -78,6 +81,27 @@ public Collection getChildResources() { }; } + @Override + public DocValueFetcher.Leaf getLeafValueFetcher(DocValueFormat format) { + SortedNumericDoubleValues values = getDoubleValues(); + return new DocValueFetcher.Leaf() { + @Override + public boolean advanceExact(int docId) throws IOException { + return values.advanceExact(docId); + } + + @Override + public int docValueCount() throws IOException { + return values.docValueCount(); + } + + @Override + public Object nextValue() throws IOException { + return format.format(values.nextValue()); + } + }; + } + @Override public void close() { } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafLongFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafLongFieldData.java index 38dede8537b94..31895e5dbb102 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafLongFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafLongFieldData.java @@ -19,12 +19,17 @@ package org.elasticsearch.index.fielddata.plain; -import org.elasticsearch.index.fielddata.LeafNumericFieldData; +import org.apache.lucene.index.SortedNumericDocValues; import org.elasticsearch.index.fielddata.FieldData; import org.elasticsearch.index.fielddata.IndexNumericFieldData.NumericType; +import org.elasticsearch.index.fielddata.LeafNumericFieldData; import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; +import org.elasticsearch.index.mapper.DocValueFetcher; +import org.elasticsearch.search.DocValueFormat; + +import java.io.IOException; /** * Specialization of {@link LeafNumericFieldData} for integers. @@ -73,6 +78,27 @@ public final SortedNumericDoubleValues getDoubleValues() { return FieldData.castToDouble(getLongValues()); } + @Override + public DocValueFetcher.Leaf getLeafValueFetcher(DocValueFormat format) { + SortedNumericDocValues values = getLongValues(); + return new DocValueFetcher.Leaf() { + @Override + public boolean advanceExact(int docId) throws IOException { + return values.advanceExact(docId); + } + + @Override + public int docValueCount() throws IOException { + return values.docValueCount(); + } + + @Override + public Object nextValue() throws IOException { + return format.format(values.nextValue()); + } + }; + } + @Override public void close() {} } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericIndexFieldData.java index 41441ac36fced..2847a5c6dbdf5 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericIndexFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericIndexFieldData.java @@ -38,8 +38,10 @@ import org.elasticsearch.index.fielddata.NumericDoubleValues; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; import org.elasticsearch.index.fielddata.fieldcomparator.LongValuesComparatorSource; +import org.elasticsearch.index.mapper.DocValueFetcher; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.indices.breaker.CircuitBreakerService; +import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.MultiValueMode; import org.elasticsearch.search.aggregations.support.ValuesSourceType; @@ -173,6 +175,28 @@ public SortedNumericDocValues getLongValuesAsNanos() { throw new IllegalStateException("Cannot load doc values", e); } } + + @Override + public DocValueFetcher.Leaf getLeafValueFetcher(DocValueFormat format) { + DocValueFormat nanosFormat = DocValueFormat.withNanosecondResolution(format); + SortedNumericDocValues values = getLongValuesAsNanos(); + return new DocValueFetcher.Leaf() { + @Override + public boolean advanceExact(int docId) throws IOException { + return values.advanceExact(docId); + } + + @Override + public int docValueCount() throws IOException { + return values.docValueCount(); + } + + @Override + public Object nextValue() throws IOException { + return nanosFormat.format(values.nextValue()); + } + }; + } } /** diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java index f8f81109b2001..3459e7eee8492 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java @@ -40,6 +40,7 @@ import org.elasticsearch.geometry.Geometry; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.query.QueryShardException; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.io.UncheckedIOException; @@ -184,7 +185,7 @@ public Builder ignoreZValue(final boolean ignoreZValue) { } @Override - public ValueFetcher valueFetcher(MapperService mapperService, String format) { + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { String geoFormat = format != null ? format : GeoJsonGeometryFormat.NAME; AbstractGeometryFieldType mappedFieldType = fieldType(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java index 863c352876b02..1a98b2ea69883 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java @@ -192,7 +192,7 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - public ValueFetcher valueFetcher(MapperService mapperService, String format) { + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java index df452c84d9367..e87c5bd0296e7 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java @@ -253,7 +253,7 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - public ValueFetcher valueFetcher(MapperService mapperService, String format) { + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java index 7c1360c4c5f3f..7a8c7b6529d20 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java @@ -45,6 +45,7 @@ import org.elasticsearch.index.analysis.AnalyzerScope; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.search.suggest.completion.CompletionSuggester; import org.elasticsearch.search.suggest.completion.context.ContextMapping; import org.elasticsearch.search.suggest.completion.context.ContextMappings; @@ -540,7 +541,7 @@ private void parse(ParseContext parseContext, Token token, } @Override - public ValueFetcher valueFetcher(MapperService mapperService, String format) { + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 8a2247e732b33..ffa8335dae125 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -606,7 +606,7 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - public ValueFetcher valueFetcher(MapperService mapperService, String format) { + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { DateFormatter defaultFormatter = fieldType().dateTimeFormatter(); DateFormatter formatter = format != null ? DateFormatter.forPattern(format).withLocale(defaultFormatter.locale()) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocValueFetcher.java b/server/src/main/java/org/elasticsearch/index/mapper/DocValueFetcher.java new file mode 100644 index 0000000000000..6dfa4021a6859 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocValueFetcher.java @@ -0,0 +1,79 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.mapper; + +import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.lookup.SourceLookup; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static java.util.Collections.emptyList; + +/** + * Value fetcher that loads from doc values. + */ +public final class DocValueFetcher implements ValueFetcher { + private final DocValueFormat format; + private final IndexFieldData ifd; + private Leaf leaf; + + public DocValueFetcher(DocValueFormat format, IndexFieldData ifd) { + this.format = format; + this.ifd = ifd; + } + + public void setNextReader(LeafReaderContext context) { + leaf = ifd.load(context).getLeafValueFetcher(format); + } + + @Override + public List fetchValues(SourceLookup lookup) throws IOException { + if (false == leaf.advanceExact(lookup.docId())) { + return emptyList(); + } + List result = new ArrayList(leaf.docValueCount()); + for (int i = 0, count = leaf.docValueCount(); i < count; ++i) { + result.add(leaf.nextValue()); + } + return result; + } + + public interface Leaf { + /** + * Advance the doc values reader to the provided doc. + * @return false if there are no values for this document, true otherwise + */ + boolean advanceExact(int docId) throws IOException; + + /** + * A count of the number of values at this document. + */ + int docValueCount() throws IOException; + + /** + * Load and format the next value. + */ + Object nextValue() throws IOException; + } +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java index d5e1bf5386fca..3129c59fddbed 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java @@ -21,6 +21,7 @@ import com.carrotsearch.hppc.cursors.ObjectCursor; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; + import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.index.IndexOptions; @@ -35,6 +36,7 @@ import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.mapper.FieldNamesFieldMapper.FieldNamesFieldType; import org.elasticsearch.search.fetch.subphase.FetchFieldsPhase; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.util.ArrayList; @@ -278,7 +280,7 @@ public void parse(ParseContext context) throws IOException { /** * Create a helper class to fetch field values during the {@link FetchFieldsPhase}. */ - public abstract ValueFetcher valueFetcher(MapperService mapperService, @Nullable String format); + public abstract ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, @Nullable String format); protected void createFieldNamesField(ParseContext context) { FieldNamesFieldType fieldNamesFieldType = context.docMapper().metadataMapper(FieldNamesFieldMapper.class).fieldType(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java index 1b75f09c7177a..68d742961a7fa 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java @@ -404,7 +404,7 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - public ValueFetcher valueFetcher(MapperService mapperService, String format) { + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index cfe1a9151c839..b4614bf043639 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -393,7 +393,7 @@ private String normalizeValue(NamedAnalyzer normalizer, String value) throws IOE } @Override - public ValueFetcher valueFetcher(MapperService mapperService, String format) { + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java index 1737a9ec82831..0c8febc19ed62 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java @@ -22,6 +22,7 @@ import org.elasticsearch.common.Explicit; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.util.Map; @@ -165,7 +166,7 @@ public void postParse(ParseContext context) throws IOException { } @Override - public ValueFetcher valueFetcher(MapperService mapperService, String format) { + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup lookup, String format) { throw new UnsupportedOperationException("Cannot fetch values for internal field [" + name() + "]."); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index dfa43ba7ea663..9448266a3d540 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -1089,7 +1089,7 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - public ValueFetcher valueFetcher(MapperService mapperService, String format) { + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java index c30f1a3e1c37b..85c3cdb39ad18 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java @@ -360,7 +360,7 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - public ValueFetcher valueFetcher(MapperService mapperService, String format) { + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { DateFormatter defaultFormatter = fieldType().dateTimeFormatter(); DateFormatter formatter = format != null ? DateFormatter.forPattern(format).withLocale(defaultFormatter.locale()) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java index 690aca26f3bbd..1448e5c60b860 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java @@ -499,7 +499,7 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - public ValueFetcher valueFetcher(MapperService mapperService, String format) { + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { throw new UnsupportedOperationException(); } @@ -530,7 +530,7 @@ protected void parseCreateField(ParseContext context) { } @Override - public ValueFetcher valueFetcher(MapperService mapperService, String format) { + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { throw new UnsupportedOperationException(); } @@ -837,7 +837,7 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - public ValueFetcher valueFetcher(MapperService mapperService, String format) { + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ValueFetcher.java b/server/src/main/java/org/elasticsearch/index/mapper/ValueFetcher.java index 712be0ad3510d..7f8a5b6149613 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ValueFetcher.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ValueFetcher.java @@ -19,9 +19,11 @@ package org.elasticsearch.index.mapper; +import org.apache.lucene.index.LeafReaderContext; import org.elasticsearch.search.fetch.subphase.FetchFieldsPhase; import org.elasticsearch.search.lookup.SourceLookup; +import java.io.IOException; import java.util.List; /** @@ -42,5 +44,10 @@ public interface ValueFetcher { * @param lookup a lookup structure over the document's source. * @return a list a standardized field values. */ - List fetchValues(SourceLookup lookup); + List fetchValues(SourceLookup lookup) throws IOException; + + /** + * Update the leaf reader used to fetch values. + */ + default void setNextReader(LeafReaderContext context) {} } diff --git a/server/src/main/java/org/elasticsearch/index/query/InnerHitContextBuilder.java b/server/src/main/java/org/elasticsearch/index/query/InnerHitContextBuilder.java index ed2e4a2315957..add896d68f81c 100644 --- a/server/src/main/java/org/elasticsearch/index/query/InnerHitContextBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/InnerHitContextBuilder.java @@ -95,8 +95,7 @@ protected void setupInnerHitsContext(QueryShardContext queryShardContext, } if (innerHitBuilder.getFetchFields() != null) { String indexName = queryShardContext.index().getName(); - FetchFieldsContext fieldsContext = FetchFieldsContext.create( - indexName, queryShardContext.getMapperService(), innerHitBuilder.getFetchFields()); + FetchFieldsContext fieldsContext = new FetchFieldsContext(innerHitBuilder.getFetchFields()); innerHitsContext.fetchFieldsContext(fieldsContext); } if (innerHitBuilder.getScriptFields() != null) { diff --git a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java index 528831e3a11e8..6efe13820cae6 100644 --- a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java @@ -290,15 +290,33 @@ MappedFieldType failIfFieldMappingNotFound(String name, MappedFieldType fieldMap private SearchLookup lookup = null; + /** + * Get the lookup to use during the search. + */ public SearchLookup lookup() { if (this.lookup == null) { this.lookup = new SearchLookup( getMapperService(), - (fieldType, searchLookup) -> indexFieldDataService.apply(fieldType, fullyQualifiedIndex.getName(), searchLookup)); + (fieldType, searchLookup) -> indexFieldDataService.apply(fieldType, fullyQualifiedIndex.getName(), searchLookup) + ); } return this.lookup; } + /** + * Build a lookup customized for the fetch phase. Use {@link #lookup()} + * in other phases. + */ + public SearchLookup newFetchLookup() { + /* + * Real customization coming soon, I promise! + */ + return new SearchLookup( + getMapperService(), + (fieldType, searchLookup) -> indexFieldDataService.apply(fieldType, fullyQualifiedIndex.getName(), searchLookup) + ); + } + public NestedScope nestedScope() { return nestedScope; } diff --git a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java index 33102a1a480b9..9f8f0abc323ad 100644 --- a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java +++ b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java @@ -38,7 +38,6 @@ import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.cache.bitset.BitsetFilterCache; import org.elasticsearch.index.engine.Engine; -import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.ObjectMapper; @@ -487,11 +486,6 @@ public BitsetFilterCache bitsetFilterCache() { return indexService.cache().bitsetFilterCache(); } - @Override - public > IFD getForField(MappedFieldType fieldType) { - return queryShardContext.getForField(fieldType); - } - @Override public TimeValue timeout() { return timeout; diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java index f716eb61707a3..5543a340d30dd 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchService.java +++ b/server/src/main/java/org/elasticsearch/search/SearchService.java @@ -967,9 +967,7 @@ private void parseSource(DefaultSearchContext context, SearchSourceBuilder sourc context.docValuesContext(docValuesContext); } if (source.fetchFields() != null) { - String indexName = context.indexShard().shardId().getIndexName(); - FetchFieldsContext fetchFieldsContext = FetchFieldsContext.create( - indexName, context.mapperService(), source.fetchFields()); + FetchFieldsContext fetchFieldsContext = new FetchFieldsContext(source.fetchFields()); context.fetchFieldsContext(fetchFieldsContext); } if (source.highlighter() != null) { diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregatorFactory.java index 7b6c440d77a2b..42197d4277ed2 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregatorFactory.java @@ -114,8 +114,7 @@ public Aggregator createInternal(SearchContext searchContext, subSearchContext.docValuesContext(docValuesContext); } if (fetchFields != null) { - String indexName = searchContext.indexShard().shardId().getIndexName(); - FetchFieldsContext fieldsContext = FetchFieldsContext.create(indexName, searchContext.mapperService(), fetchFields); + FetchFieldsContext fieldsContext = new FetchFieldsContext(fetchFields); subSearchContext.fetchFieldsContext(fieldsContext); } for (ScriptFieldsContext.ScriptField field : scriptFields) { diff --git a/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java index 0e8a73875dc97..1f491768b9a11 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java @@ -52,6 +52,7 @@ import org.elasticsearch.search.fetch.subphase.InnerHitsContext; import org.elasticsearch.search.fetch.subphase.InnerHitsPhase; import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.tasks.TaskCancelledException; @@ -99,7 +100,8 @@ public void execute(SearchContext context) { SearchHit[] hits = new SearchHit[context.docIdsToLoadSize()]; Map sharedCache = new HashMap<>(); - List processors = getProcessors(context); + SearchLookup lookup = context.getQueryShardContext().newFetchLookup(); + List processors = getProcessors(context, lookup); int currentReaderIndex = -1; LeafReaderContext currentReaderContext = null; @@ -118,8 +120,15 @@ public void execute(SearchContext context) { } } assert currentReaderContext != null; - HitContext hit - = prepareHitContext(context, fieldsVisitor, docId, storedToRequestedFields, currentReaderContext, sharedCache); + HitContext hit = prepareHitContext( + context, + lookup, + fieldsVisitor, + docId, + storedToRequestedFields, + currentReaderContext, + sharedCache + ); for (FetchSubPhaseProcessor processor : processors) { processor.process(hit); } @@ -137,11 +146,11 @@ public void execute(SearchContext context) { } - List getProcessors(SearchContext context) { + List getProcessors(SearchContext context, SearchLookup lookup) { try { List processors = new ArrayList<>(); for (FetchSubPhase fsp : fetchSubPhases) { - FetchSubPhaseProcessor processor = fsp.getProcessor(context); + FetchSubPhaseProcessor processor = fsp.getProcessor(context, lookup); if (processor != null) { processors.add(processor); } @@ -231,12 +240,20 @@ private int findRootDocumentIfNested(SearchContext context, LeafReaderContext su return -1; } - private HitContext prepareHitContext(SearchContext context, FieldsVisitor fieldsVisitor, int docId, + private HitContext prepareHitContext(SearchContext context, SearchLookup lookup, FieldsVisitor fieldsVisitor, int docId, Map> storedToRequestedFields, LeafReaderContext subReaderContext, Map sharedCache) throws IOException { int rootDocId = findRootDocumentIfNested(context, subReaderContext, docId - subReaderContext.docBase); if (rootDocId == -1) { - return prepareNonNestedHitContext(context, fieldsVisitor, docId, storedToRequestedFields, subReaderContext, sharedCache); + return prepareNonNestedHitContext( + context, + lookup, + fieldsVisitor, + docId, + storedToRequestedFields, + subReaderContext, + sharedCache + ); } else { return prepareNestedHitContext(context, docId, rootDocId, storedToRequestedFields, subReaderContext, sharedCache); } @@ -250,6 +267,7 @@ private HitContext prepareHitContext(SearchContext context, FieldsVisitor fields * fetch subphases that use the hit context to access the preloaded source. */ private HitContext prepareNonNestedHitContext(SearchContext context, + SearchLookup lookup, FieldsVisitor fieldsVisitor, int docId, Map> storedToRequestedFields, @@ -258,7 +276,7 @@ private HitContext prepareNonNestedHitContext(SearchContext context, int subDocId = docId - subReaderContext.docBase; if (fieldsVisitor == null) { SearchHit hit = new SearchHit(docId, null, null, null); - return new HitContext(hit, subReaderContext, subDocId, sharedCache); + return new HitContext(hit, subReaderContext, subDocId, lookup.source(), sharedCache); } else { SearchHit hit; loadStoredFields(context.mapperService(), subReaderContext, fieldsVisitor, subDocId); @@ -271,7 +289,7 @@ private HitContext prepareNonNestedHitContext(SearchContext context, hit = new SearchHit(docId, fieldsVisitor.id(), emptyMap(), emptyMap()); } - HitContext hitContext = new HitContext(hit, subReaderContext, subDocId, sharedCache); + HitContext hitContext = new HitContext(hit, subReaderContext, subDocId, lookup.source(), sharedCache); if (fieldsVisitor.source() != null) { hitContext.sourceLookup().setSource(fieldsVisitor.source()); } @@ -280,7 +298,6 @@ private HitContext prepareNonNestedHitContext(SearchContext context, } /** - /** * Resets the provided {@link HitContext} with information on the current * nested document. This includes the following: * - Adding an initial {@link SearchHit} instance. @@ -348,7 +365,13 @@ private HitContext prepareNestedHitContext(SearchContext context, getInternalNestedIdentity(context, nestedDocId, subReaderContext, context.mapperService(), nestedObjectMapper); SearchHit hit = new SearchHit(nestedTopDocId, rootId, nestedIdentity, docFields, metaFields); - HitContext hitContext = new HitContext(hit, subReaderContext, nestedDocId, sharedCache); + HitContext hitContext = new HitContext( + hit, + subReaderContext, + nestedDocId, + new SourceLookup(), // Use a clean, fresh SourceLookup for the nested context + sharedCache + ); if (rootSourceAsMap != null) { // Isolate the nested json array object that matches with nested hit and wrap it back into the same json diff --git a/server/src/main/java/org/elasticsearch/search/fetch/FetchSubPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/FetchSubPhase.java index dad4bf2dfce06..4f549b20e8b35 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/FetchSubPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/FetchSubPhase.java @@ -24,6 +24,7 @@ import org.apache.lucene.index.ReaderUtil; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.search.lookup.SourceLookup; import java.io.IOException; @@ -38,14 +39,21 @@ class HitContext { private final SearchHit hit; private final LeafReaderContext readerContext; private final int docId; - private final SourceLookup sourceLookup = new SourceLookup(); + private final SourceLookup sourceLookup; private final Map cache; - public HitContext(SearchHit hit, LeafReaderContext context, int docId, Map cache) { + public HitContext( + SearchHit hit, + LeafReaderContext context, + int docId, + SourceLookup sourceLookup, + Map cache + ) { this.hit = hit; this.readerContext = context; this.docId = docId; - this.sourceLookup.setSegmentAndDocument(context, docId); + this.sourceLookup = sourceLookup; + sourceLookup.setSegmentAndDocument(context, docId); this.cache = cache; } @@ -95,5 +103,5 @@ public Map cache() { * If nothing should be executed for the provided {@link SearchContext}, then the * implementation should return {@code null} */ - FetchSubPhaseProcessor getProcessor(SearchContext searchContext) throws IOException; + FetchSubPhaseProcessor getProcessor(SearchContext searchContext, SearchLookup lookup) throws IOException; } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/ExplainPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/ExplainPhase.java index 010b9a0e12894..f1ec50a7f4af6 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/ExplainPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/ExplainPhase.java @@ -23,6 +23,7 @@ import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.fetch.FetchSubPhaseProcessor; import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.search.rescore.RescoreContext; import java.io.IOException; @@ -33,7 +34,7 @@ public final class ExplainPhase implements FetchSubPhase { @Override - public FetchSubPhaseProcessor getProcessor(SearchContext context) { + public FetchSubPhaseProcessor getProcessor(SearchContext context, SearchLookup lookup) { if (context.explain() == false || context.hasOnlySuggest()) { return null; } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesPhase.java index a3a5df6885a4e..2bac3886c7e9f 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesPhase.java @@ -19,27 +19,20 @@ package org.elasticsearch.search.fetch.subphase; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.SortedNumericDocValues; import org.elasticsearch.common.document.DocumentField; -import org.elasticsearch.index.fielddata.IndexFieldData; -import org.elasticsearch.index.fielddata.IndexNumericFieldData; -import org.elasticsearch.index.fielddata.SortedBinaryDocValues; -import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; -import org.elasticsearch.index.fielddata.plain.SortedNumericIndexFieldData; +import org.elasticsearch.index.mapper.DocValueFetcher; import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.fetch.FetchSubPhaseProcessor; import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.List; -import static org.elasticsearch.index.fielddata.IndexNumericFieldData.NumericType; -import static org.elasticsearch.search.DocValueFormat.withNanosecondResolution; - /** * Fetch sub phase which pulls data from doc values. * @@ -48,7 +41,7 @@ public final class FetchDocValuesPhase implements FetchSubPhase { @Override - public FetchSubPhaseProcessor getProcessor(SearchContext context) throws IOException { + public FetchSubPhaseProcessor getProcessor(SearchContext context, SearchLookup lookup) throws IOException { if (context.collapse() != null) { // retrieve the `doc_value` associated with the collapse field String name = context.collapse().getFieldName(); @@ -64,19 +57,26 @@ public FetchSubPhaseProcessor getProcessor(SearchContext context) throws IOExcep return null; } + /* + * Its tempting to swap this to a `Map` but that'd break backwards + * compatibility because we support fetching the same field multiple + * times with different configuration. That isn't possible with a `Map`. + */ List fields = new ArrayList<>(); for (FieldAndFormat fieldAndFormat : context.docValuesContext().fields()) { - DocValueField f = buildField(context, fieldAndFormat); - if (f != null) { - fields.add(f); + MappedFieldType ft = context.mapperService().fieldType(fieldAndFormat.field); + if (ft == null) { + continue; } + ValueFetcher fetcher = new DocValueFetcher(ft.docValueFormat(fieldAndFormat.format, null), lookup.doc().getForField(ft)); + fields.add(new DocValueField(fieldAndFormat.field, fetcher)); } return new FetchSubPhaseProcessor() { @Override public void setNextReader(LeafReaderContext readerContext) throws IOException { for (DocValueField f : fields) { - f.setNextReader(readerContext); + f.fetcher.setNextReader(readerContext); } } @@ -90,149 +90,19 @@ public void process(HitContext hit) throws IOException { // docValues fields will still be document fields, and put under "fields" section of a hit. hit.hit().setDocumentField(f.field, hitField); } - f.setValues(hit.docId(), hitField); + hitField.getValues().addAll(f.fetcher.fetchValues(hit.sourceLookup())); } } }; } - private abstract static class DocValueField { - - final String field; - final DocValueFormat format; + private class DocValueField { + private final String field; + private final ValueFetcher fetcher; - protected DocValueField(String field, DocValueFormat format) { + DocValueField(String field, ValueFetcher fetcher) { this.field = field; - this.format = format; - } - - abstract void setNextReader(LeafReaderContext context); - abstract void setValues(int doc, DocumentField hitField) throws IOException; - - } - - private static class DoubleDocValueField extends DocValueField { - - SortedNumericDoubleValues doubleValues; - IndexNumericFieldData fieldData; - - DoubleDocValueField(String field, IndexNumericFieldData fieldData, DocValueFormat format) { - super(field, format); - this.fieldData = fieldData; - } - - @Override - void setNextReader(LeafReaderContext context) { - doubleValues = fieldData.load(context).getDoubleValues(); - } - - @Override - void setValues(int doc, DocumentField hitField) throws IOException { - final List values = hitField.getValues(); - if (doubleValues.advanceExact(doc)) { - for (int i = 0, count = doubleValues.docValueCount(); i < count; ++i) { - values.add(format.format(doubleValues.nextValue())); - } - } - } - } - - private static class NanoDocValueField extends DocValueField { - - SortedNumericDocValues longValues; - IndexNumericFieldData fieldData; - - NanoDocValueField(String field, IndexNumericFieldData fieldData, DocValueFormat format) { - super(field, withNanosecondResolution(format)); - this.fieldData = fieldData; - } - - @Override - void setNextReader(LeafReaderContext context) { - longValues = ((SortedNumericIndexFieldData.NanoSecondFieldData) fieldData.load(context)).getLongValuesAsNanos(); - } - - @Override - void setValues(int doc, DocumentField hitField) throws IOException { - final List values = hitField.getValues(); - if (longValues.advanceExact(doc)) { - for (int i = 0, count = longValues.docValueCount(); i < count; ++i) { - values.add(format.format(longValues.nextValue())); - } - } - } - } - - private static class LongDocValueField extends DocValueField { - - SortedNumericDocValues longValues; - IndexNumericFieldData fieldData; - - LongDocValueField(String field, IndexNumericFieldData fieldData, DocValueFormat format) { - super(field, format); - this.fieldData = fieldData; - } - - @Override - void setNextReader(LeafReaderContext context) { - longValues = fieldData.load(context).getLongValues(); - } - - @Override - void setValues(int doc, DocumentField hitField) throws IOException { - final List values = hitField.getValues(); - if (longValues.advanceExact(doc)) { - for (int i = 0, count = longValues.docValueCount(); i < count; ++i) { - values.add(format.format(longValues.nextValue())); - } - } - } - - } - - private static class BinaryDocValueField extends DocValueField { - - SortedBinaryDocValues byteValues; - IndexFieldData fieldData; - - BinaryDocValueField(String field, IndexFieldData fieldData, DocValueFormat format) { - super(field, format); - this.fieldData = fieldData; - } - - @Override - void setNextReader(LeafReaderContext context) { - byteValues = fieldData.load(context).getBytesValues(); - } - - @Override - void setValues(int doc, DocumentField hitField) throws IOException { - final List values = hitField.getValues(); - if (byteValues.advanceExact(doc)) { - for (int i = 0, count = byteValues.docValueCount(); i < count; ++i) { - values.add(format.format(byteValues.nextValue())); - } - } - } - - } - - private static DocValueField buildField(SearchContext context, FieldAndFormat fieldAndFormat) { - MappedFieldType fieldType = context.mapperService().fieldType(fieldAndFormat.field); - if (fieldType == null) { - return null; - } - final IndexFieldData indexFieldData = context.getForField(fieldType); - DocValueFormat format = fieldType.docValueFormat(fieldAndFormat.format, null); - if (indexFieldData instanceof IndexNumericFieldData) { - if (((IndexNumericFieldData) indexFieldData).getNumericType().isFloatingPoint()) { - return new DoubleDocValueField(fieldAndFormat.field, (IndexNumericFieldData) indexFieldData, format); - } - if (((IndexNumericFieldData) indexFieldData).getNumericType() == NumericType.DATE_NANOSECONDS) { - return new NanoDocValueField(fieldAndFormat.field, (IndexNumericFieldData) indexFieldData, format); - } - return new LongDocValueField(fieldAndFormat.field, (IndexNumericFieldData) indexFieldData, format); + this.fetcher = fetcher; } - return new BinaryDocValueField(fieldAndFormat.field, indexFieldData, format); } } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsContext.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsContext.java index 79b33003da788..d28fa314b5d9b 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsContext.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsContext.java @@ -18,8 +18,8 @@ */ package org.elasticsearch.search.fetch.subphase; -import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.search.lookup.SearchLookup; import java.util.List; @@ -27,27 +27,18 @@ * The context needed to retrieve fields. */ public class FetchFieldsContext { + private final List fields; - private FieldValueRetriever fieldValueRetriever; - - public static FetchFieldsContext create(String indexName, - MapperService mapperService, - List fields) { - DocumentMapper documentMapper = mapperService.documentMapper(); - if (documentMapper.sourceMapper().enabled() == false) { - throw new IllegalArgumentException("Unable to retrieve the requested [fields] since _source is " + - "disabled in the mappings for index [" + indexName + "]"); - } - - FieldValueRetriever fieldValueRetriever = FieldValueRetriever.create(mapperService, fields); - return new FetchFieldsContext(fieldValueRetriever); + public FetchFieldsContext(List fields) { + this.fields = fields; } - private FetchFieldsContext(FieldValueRetriever fieldValueRetriever) { - this.fieldValueRetriever = fieldValueRetriever; - } - - public FieldValueRetriever fieldValueRetriever() { - return fieldValueRetriever; + public FieldValueRetriever fieldValueRetriever(String indexName, MapperService mapperService, SearchLookup searchLookup) { + if (mapperService.documentMapper().sourceMapper().enabled() == false) { + throw new IllegalArgumentException( + "Unable to retrieve the requested [fields] since _source is disabled in the mappings for index [" + indexName + "]" + ); + } + return FieldValueRetriever.create(mapperService, searchLookup, fields); } } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsPhase.java index ad3827b2879c9..511beded817ca 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsPhase.java @@ -26,8 +26,10 @@ import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.fetch.FetchSubPhaseProcessor; import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.search.lookup.SourceLookup; +import java.io.IOException; import java.util.HashSet; import java.util.Map; import java.util.Set; @@ -39,25 +41,29 @@ public final class FetchFieldsPhase implements FetchSubPhase { @Override - public FetchSubPhaseProcessor getProcessor(SearchContext searchContext) { + public FetchSubPhaseProcessor getProcessor(SearchContext searchContext, SearchLookup lookup) { FetchFieldsContext fetchFieldsContext = searchContext.fetchFieldsContext(); if (fetchFieldsContext == null) { return null; } + FieldValueRetriever retriever = fetchFieldsContext.fieldValueRetriever( + searchContext.indexShard().shardId().getIndexName(), + searchContext.mapperService(), + lookup + ); return new FetchSubPhaseProcessor() { @Override public void setNextReader(LeafReaderContext readerContext) { - + retriever.setNextReader(readerContext); } @Override - public void process(HitContext hitContext) { + public void process(HitContext hitContext) throws IOException { SearchHit hit = hitContext.hit(); SourceLookup sourceLookup = hitContext.sourceLookup(); - FieldValueRetriever fieldValueRetriever = fetchFieldsContext.fieldValueRetriever(); Set ignoredFields = getIgnoredFields(hit); - Map documentFields = fieldValueRetriever.retrieve(sourceLookup, ignoredFields); + Map documentFields = retriever.retrieve(sourceLookup, ignoredFields); for (Map.Entry entry : documentFields.entrySet()) { hit.setDocumentField(entry.getKey(), entry.getValue()); } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchScorePhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchScorePhase.java index 8f59a771da5d5..e9dd4291915cc 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchScorePhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchScorePhase.java @@ -28,13 +28,14 @@ import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.fetch.FetchSubPhaseProcessor; import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; public class FetchScorePhase implements FetchSubPhase { @Override - public FetchSubPhaseProcessor getProcessor(SearchContext context) throws IOException { + public FetchSubPhaseProcessor getProcessor(SearchContext context, SearchLookup lookup) throws IOException { if (context.trackScores() == false || context.docIdsToLoadSize() == 0 || // scores were already computed since they are needed on the coordinated node to merge top hits context.sort() == null) { diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhase.java index 9801f6d577827..56802099bb90c 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhase.java @@ -28,6 +28,7 @@ import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.fetch.FetchSubPhaseProcessor; import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.search.lookup.SourceLookup; import java.io.IOException; @@ -36,7 +37,7 @@ public final class FetchSourcePhase implements FetchSubPhase { @Override - public FetchSubPhaseProcessor getProcessor(SearchContext searchContext) { + public FetchSubPhaseProcessor getProcessor(SearchContext searchContext, SearchLookup lookup) { if (searchContext.sourceRequested() == false) { return null; } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchVersionPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchVersionPhase.java index 394d534b56e3c..0c60b91b5ece0 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchVersionPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchVersionPhase.java @@ -25,13 +25,14 @@ import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.fetch.FetchSubPhaseProcessor; import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; public final class FetchVersionPhase implements FetchSubPhase { @Override - public FetchSubPhaseProcessor getProcessor(SearchContext context) { + public FetchSubPhaseProcessor getProcessor(SearchContext context, SearchLookup lookup) { if (context.version() == false || (context.storedFieldsContext() != null && context.storedFieldsContext().fetchFields() == false)) { return null; diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldValueRetriever.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldValueRetriever.java index b6853a5a05172..cea9137ac9784 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldValueRetriever.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldValueRetriever.java @@ -19,6 +19,7 @@ package org.elasticsearch.search.fetch.subphase; +import org.apache.lucene.index.LeafReaderContext; import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.index.mapper.FieldAliasMapper; import org.elasticsearch.index.mapper.FieldMapper; @@ -26,8 +27,10 @@ import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.ValueFetcher; +import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.search.lookup.SourceLookup; +import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -40,10 +43,11 @@ * Then given a specific document, it can retrieve the corresponding fields from the document's source. */ public class FieldValueRetriever { - private final List fieldContexts; - - public static FieldValueRetriever create(MapperService mapperService, - Collection fieldAndFormats) { + public static FieldValueRetriever create( + MapperService mapperService, + SearchLookup searchLookup, + Collection fieldAndFormats + ) { MappingLookup fieldMappers = mapperService.documentMapper().mappers(); List fieldContexts = new ArrayList<>(); @@ -65,7 +69,7 @@ public static FieldValueRetriever create(MapperService mapperService, } FieldMapper fieldMapper = (FieldMapper) mapper; - ValueFetcher valueFetcher = fieldMapper.valueFetcher(mapperService, format); + ValueFetcher valueFetcher = fieldMapper.valueFetcher(mapperService, searchLookup, format); fieldContexts.add(new FieldContext(field, valueFetcher)); } } @@ -73,11 +77,13 @@ public static FieldValueRetriever create(MapperService mapperService, return new FieldValueRetriever(fieldContexts); } + private final List fieldContexts; + private FieldValueRetriever(List fieldContexts) { this.fieldContexts = fieldContexts; } - public Map retrieve(SourceLookup sourceLookup, Set ignoredFields) { + public Map retrieve(SourceLookup sourceLookup, Set ignoredFields) throws IOException { Map documentFields = new HashMap<>(); for (FieldContext context : fieldContexts) { String field = context.fieldName; @@ -95,6 +101,12 @@ public Map retrieve(SourceLookup sourceLookup, Set highlighters) { } @Override - public FetchSubPhaseProcessor getProcessor(SearchContext context) { + public FetchSubPhaseProcessor getProcessor(SearchContext context, SearchLookup lookup) { if (context.highlight() == null) { return null; } diff --git a/server/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java b/server/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java index 95d15d5dded58..3ee4878a4e5db 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java +++ b/server/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java @@ -27,7 +27,6 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.index.cache.bitset.BitsetFilterCache; -import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.ObjectMapper; @@ -244,11 +243,6 @@ public BitsetFilterCache bitsetFilterCache() { return in.bitsetFilterCache(); } - @Override - public > IFD getForField(MappedFieldType fieldType) { - return in.getForField(fieldType); - } - @Override public TimeValue timeout() { return in.timeout(); diff --git a/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java b/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java index 3bbf3ea387f29..11a477d3ebf6b 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java +++ b/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java @@ -30,7 +30,6 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.index.cache.bitset.BitsetFilterCache; -import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.ObjectMapper; @@ -230,8 +229,6 @@ public final void assignRescoreDocIds(RescoreDocIds rescoreDocIds) { public abstract BitsetFilterCache bitsetFilterCache(); - public abstract > IFD getForField(MappedFieldType fieldType); - public abstract TimeValue timeout(); public abstract void timeout(TimeValue timeout); diff --git a/server/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java b/server/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java index d63caed14adb0..9eff8f64ef6f6 100644 --- a/server/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java +++ b/server/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java @@ -55,6 +55,10 @@ public XContentType sourceContentType() { return sourceContentType; } + public int docId() { + return docId; + } + // Scripting requires this method to be public. Using source() // is not possible because certain checks use source == null as // as a determination if source is enabled/disabled, but it should diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java index f287cd635fe33..ea086a1120d19 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java @@ -172,7 +172,7 @@ public void testBoosts() throws Exception { assertEquals(new BoostQuery(new TermQuery(new Term("field", "T")), 2.0f), ft.termQuery("true", null)); } - public void testFetchSourceValue() { + public void testFetchSourceValue() throws IOException { Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java index 3284b0f89502b..6a1bd1f0bcc34 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java @@ -735,7 +735,7 @@ public void testLimitOfContextMappings() throws Throwable { CompletionFieldMapper.COMPLETION_CONTEXTS_LIMIT + "] has been exceeded")); } - public void testFetchSourceValue() { + public void testFetchSourceValue() throws IOException { Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); NamedAnalyzer defaultAnalyzer = new NamedAnalyzer("standard", AnalyzerScope.INDEX, new StandardAnalyzer()); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java index 9255b500f064a..f0f4c78804a38 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java @@ -29,6 +29,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.mapper.DateFieldMapper.Resolution; import org.elasticsearch.index.termvectors.TermVectorsService; +import org.elasticsearch.search.DocValueFormat; import java.io.IOException; import java.time.ZoneId; @@ -284,7 +285,7 @@ public void testIllegalFormatField() { assertThat(e.getMessage(), containsString("Error parsing [format] on field [field]: Invalid")); } - public void testFetchSourceValue() { + public void testFetchSourceValue() throws IOException { DateFieldMapper mapper = createMapper(Resolution.MILLISECONDS, null); String date = "2020-05-15T21:33:02.000Z"; assertEquals(List.of(date), fetchSourceValue(mapper, date)); @@ -305,7 +306,7 @@ public void testFetchSourceValue() { assertEquals(List.of(nullValueDate), fetchSourceValue(nullValueMapper, null)); } - public void testParseSourceValueWithFormat() { + public void testParseSourceValueWithFormat() throws IOException { DateFieldMapper mapper = createMapper(Resolution.NANOSECONDS, "strict_date_time", "1970-12-29T00:00:00.000Z"); String date = "1990-12-29T00:00:00.000Z"; assertEquals(List.of("1990/12/29"), fetchSourceValue(mapper, date, "yyyy/MM/dd")); @@ -313,7 +314,7 @@ public void testParseSourceValueWithFormat() { assertEquals(List.of("1970/12/29"), fetchSourceValue(mapper, null, "yyyy/MM/dd")); } - public void testParseSourceValueNanos() { + public void testParseSourceValueNanos() throws IOException { DateFieldMapper mapper = createMapper(Resolution.NANOSECONDS, "strict_date_time||epoch_millis"); String date = "2020-05-15T21:33:02.123456789Z"; assertEquals(List.of("2020-05-15T21:33:02.123456789Z"), fetchSourceValue(mapper, date)); @@ -324,6 +325,28 @@ public void testParseSourceValueNanos() { assertEquals(List.of(nullValueDate), fetchSourceValue(nullValueMapper, null)); } + public void testFetchDocValuesMillis() throws IOException { + MapperService mapperService = createMapperService( + fieldMapping(b -> b.field("type", "date").field("format", "strict_date_time||epoch_millis")) + ); + MappedFieldType ft = mapperService.fieldType("field"); + DocValueFormat format = ft.docValueFormat(null, null); + String date = "2020-05-15T21:33:02.123Z"; + assertEquals(List.of(date), fetchFromDocValues(mapperService, ft, format, date)); + assertEquals(List.of(date), fetchFromDocValues(mapperService, ft, format, 1589578382123L)); + } + + public void testFetchDocValuesNanos() throws IOException { + MapperService mapperService = createMapperService( + fieldMapping(b -> b.field("type", "date_nanos").field("format", "strict_date_time||epoch_millis")) + ); + MappedFieldType ft = mapperService.fieldType("field"); + DocValueFormat format = ft.docValueFormat(null, null); + String date = "2020-05-15T21:33:02.123456789Z"; + assertEquals(List.of(date), fetchFromDocValues(mapperService, ft, format, date)); + assertEquals(List.of("2020-05-15T21:33:02.123Z"), fetchFromDocValues(mapperService, ft, format, 1589578382123L)); + } + private DateFieldMapper createMapper(Resolution resolution, String format) { return createMapper(resolution, format, null); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java index edc40211c2853..f6565ac25ecf1 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java @@ -32,6 +32,7 @@ import org.elasticsearch.index.analysis.AnalyzerScope; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.io.StringReader; @@ -103,7 +104,7 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - public ValueFetcher valueFetcher(MapperService mapperService, String format) { + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { throw new UnsupportedOperationException(); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java b/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java index 8406ceb514631..a22164a08960c 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java @@ -30,6 +30,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.geometry.Point; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.nio.charset.Charset; @@ -201,7 +202,7 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - public ValueFetcher valueFetcher(MapperService mapperService, String format) { + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { return new SourceValueFetcher(name(), mapperService, parsesArrayValue()) { @Override protected Object parseSourceValue(Object value) { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java b/server/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java index 6c214fbc8c611..e7002668c5466 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java @@ -31,6 +31,7 @@ import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.util.Collections; @@ -134,7 +135,7 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - public ValueFetcher valueFetcher(MapperService mapperService, String format) { + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { return new SourceValueFetcher(name(), mapperService, parsesArrayValue()) { @Override protected String parseSourceValue(Object value) { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java index 19a0776b199a5..135918ad13b18 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java @@ -299,7 +299,7 @@ public void testInvalidGeopointValuesIgnored() throws Exception { ); } - public void testFetchSourceValue() { + public void testFetchSourceValue() throws IOException { Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java index 7972c5250d432..ae413bb61ac1b 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java @@ -214,7 +214,7 @@ public void testGeoShapeArrayParsing() throws Exception { assertThat(document.docs().get(0).getFields("field").length, equalTo(2)); } - public void testFetchSourceValue() { + public void testFetchSourceValue() throws IOException { Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java index 0cf6496eb8635..8ba10e4044f29 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java @@ -294,7 +294,7 @@ public void testEmptyName() throws IOException { assertThat(e.getMessage(), containsString("name cannot be empty string")); } - public void testFetchSourceValue() { + public void testFetchSourceValue() throws IOException { Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpRangeFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpRangeFieldMapperTests.java index cd427538ae3e5..f3b05abf942ce 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IpRangeFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IpRangeFieldMapperTests.java @@ -34,6 +34,7 @@ import org.elasticsearch.test.ESSingleNodeTestCase; import org.junit.Before; +import java.io.IOException; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -86,7 +87,7 @@ public void testStoreCidr() throws Exception { } } - public void testFetchSourceValue() { + public void testFetchSourceValue() throws IOException { Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java index e653fb8726ec0..7711cc2125ecd 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java @@ -401,7 +401,7 @@ public void testSplitQueriesOnWhitespace() throws IOException { ); } - public void testFetchSourceValue() { + public void testFetchSourceValue() throws IOException { Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapperTests.java index 7382d4645d25b..2afa6a6a8b064 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapperTests.java @@ -634,7 +634,7 @@ public void testGeoShapeArrayParsing() throws Exception { assertFieldWarnings("tree", "strategy"); } - public void testFetchSourceValue() { + public void testFetchSourceValue() throws IOException { Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java index 3e62dac1df51c..7aa0f9bf7d5c3 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java @@ -204,7 +204,7 @@ protected void doTestNullValue(String type) throws IOException { assertFalse(dvField.fieldType().stored()); } - public void testFetchSourceValue() { + public void testFetchSourceValue() throws IOException { Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java index 0d788878db642..c6033e27412e9 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java @@ -36,6 +36,7 @@ import org.elasticsearch.index.mapper.ParametrizedFieldMapper.Parameter; import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.test.ESSingleNodeTestCase; import java.io.IOException; @@ -192,7 +193,7 @@ protected void parseCreateField(ParseContext context) { } @Override - public ValueFetcher valueFetcher(MapperService mapperService, String format) { + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { return null; } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java index de283c1168367..8877e68192041 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java @@ -332,7 +332,7 @@ public void testIllegalFormatField() throws Exception { assertThat(e.getMessage(), containsString("Invalid format: [[test_format]]: Unknown pattern letter: t")); } - public void testFetchSourceValue() { + public void testFetchSourceValue() throws IOException { Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); @@ -348,7 +348,7 @@ public void testFetchSourceValue() { fetchSourceValue(dateMapper, dateRange)); } - public void testParseSourceValueWithFormat() { + public void testParseSourceValueWithFormat() throws IOException { Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java index 1c54b2a7d1687..3475916d354b9 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java @@ -922,7 +922,7 @@ public void testSimpleMerge() throws IOException { assertThat(mapperService.documentMapper().mappers().getMapper("other_field"), instanceOf(KeywordFieldMapper.class)); } - public void testFetchSourceValue() { + public void testFetchSourceValue() throws IOException { Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhaseTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhaseTests.java index 3da53d4879513..cdc949410bf62 100644 --- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhaseTests.java +++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhaseTests.java @@ -31,6 +31,7 @@ import org.elasticsearch.search.fetch.FetchSubPhase.HitContext; import org.elasticsearch.search.fetch.FetchSubPhaseProcessor; import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.TestSearchContext; @@ -159,11 +160,11 @@ private HitContext hitExecuteMultiple(XContentBuilder source, boolean fetchSourc // We don't need a real index, just a LeafReaderContext which cannot be mocked. MemoryIndex index = new MemoryIndex(); LeafReaderContext leafReaderContext = index.createSearcher().getIndexReader().leaves().get(0); - HitContext hitContext = new HitContext(searchHit, leafReaderContext, 1, new HashMap<>()); + HitContext hitContext = new HitContext(searchHit, leafReaderContext, 1, new SourceLookup(), new HashMap<>()); hitContext.sourceLookup().setSource(source == null ? null : BytesReference.bytes(source)); FetchSourcePhase phase = new FetchSourcePhase(); - FetchSubPhaseProcessor processor = phase.getProcessor(searchContext); + FetchSubPhaseProcessor processor = phase.getProcessor(searchContext, null); if (fetchSource == false) { assertNull(processor); } else { diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldValueRetrieverTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldValueRetrieverTests.java index 01ddf10aa26fd..e73b39bfa9a04 100644 --- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldValueRetrieverTests.java +++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldValueRetrieverTests.java @@ -360,16 +360,20 @@ public void testObjectFields() throws IOException { assertFalse(fields.containsKey("object")); } - private Map retrieveFields(MapperService mapperService, XContentBuilder source, String fieldPattern) { + private Map retrieveFields(MapperService mapperService, XContentBuilder source, String fieldPattern) + throws IOException { + List fields = List.of(new FieldAndFormat(fieldPattern, null)); return retrieveFields(mapperService, source, fields); } - private Map retrieveFields(MapperService mapperService, XContentBuilder source, List fields) { + private Map retrieveFields(MapperService mapperService, XContentBuilder source, List fields) + throws IOException { + SourceLookup sourceLookup = new SourceLookup(); sourceLookup.setSource(BytesReference.bytes(source)); - FieldValueRetriever fetchFieldsLookup = FieldValueRetriever.create(mapperService, fields); + FieldValueRetriever fetchFieldsLookup = FieldValueRetriever.create(mapperService, null, fields); return fetchFieldsLookup.retrieve(sourceLookup, Set.of()); } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/FieldMapperTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/FieldMapperTestCase.java index 4348a0d614e28..52a3ccb955b58 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/FieldMapperTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/FieldMapperTestCase.java @@ -255,17 +255,17 @@ private String mappingsToString(ToXContent builder, boolean includeDefaults) thr return Strings.toString(x); } - public static List fetchSourceValue(FieldMapper mapper, Object sourceValue) { + public static List fetchSourceValue(FieldMapper mapper, Object sourceValue) throws IOException { return fetchSourceValue(mapper, sourceValue, null); } - public static List fetchSourceValue(FieldMapper mapper, Object sourceValue, String format) { + public static List fetchSourceValue(FieldMapper mapper, Object sourceValue, String format) throws IOException { String field = mapper.name(); MapperService mapperService = mock(MapperService.class); when(mapperService.sourcePath(field)).thenReturn(Set.of(field)); - ValueFetcher fetcher = mapper.valueFetcher(mapperService, format); + ValueFetcher fetcher = mapper.valueFetcher(mapperService, null, format); SourceLookup lookup = new SourceLookup(); lookup.setSource(Collections.singletonMap(field, sourceValue)); return fetcher.fetchValues(lookup); diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java index e6fd8c6e00000..734880c2ba045 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java @@ -19,18 +19,28 @@ package org.elasticsearch.index.mapper; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.util.SetOnce; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.IndexFieldDataCache; +import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.search.lookup.SourceLookup; import java.io.IOException; import java.util.Collections; import java.util.List; import java.util.Set; +import java.util.function.BiFunction; +import java.util.function.Supplier; import static org.hamcrest.Matchers.containsString; import static org.mockito.Mockito.mock; @@ -122,19 +132,43 @@ public final void testMeta() throws IOException { ); } - public static List fetchSourceValue(FieldMapper mapper, Object sourceValue) { + public static List fetchSourceValue(FieldMapper mapper, Object sourceValue) throws IOException { return fetchSourceValue(mapper, sourceValue, null); } - public static List fetchSourceValue(FieldMapper mapper, Object sourceValue, String format) { + public static List fetchSourceValue(FieldMapper mapper, Object sourceValue, String format) throws IOException { String field = mapper.name(); MapperService mapperService = mock(MapperService.class); when(mapperService.sourcePath(field)).thenReturn(Set.of(field)); - ValueFetcher fetcher = mapper.valueFetcher(mapperService, format); + ValueFetcher fetcher = mapper.valueFetcher(mapperService, null, format); SourceLookup lookup = new SourceLookup(); lookup.setSource(Collections.singletonMap(field, sourceValue)); return fetcher.fetchValues(lookup); } + + /** + * Use a {@linkplain FieldMapper} to extract values from doc values. + */ + protected final List fetchFromDocValues(MapperService mapperService, MappedFieldType ft, DocValueFormat format, Object sourceValue) + throws IOException { + + BiFunction, IndexFieldData> fieldDataLookup = (mft, lookupSource) -> mft + .fielddataBuilder("test", () -> { throw new UnsupportedOperationException(); }) + .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService(), mapperService); + SetOnce> result = new SetOnce<>(); + withLuceneIndex(mapperService, iw -> { + iw.addDocument(mapperService.documentMapper().parse(source(b -> b.field(ft.name(), sourceValue))).rootDoc()); + }, iw -> { + SearchLookup lookup = new SearchLookup(mapperService, fieldDataLookup); + ValueFetcher valueFetcher = new DocValueFetcher(format, lookup.doc().getForField(ft)); + IndexSearcher searcher = newSearcher(iw); + LeafReaderContext context = searcher.getIndexReader().leaves().get(0); + lookup.source().setSegmentAndDocument(context, 0); + valueFetcher.setNextReader(context); + result.set(valueFetcher.fetchValues(lookup.source())); + }); + return result.get(); + } } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java index 408462dc6d81d..ae4b99ebcd998 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java @@ -28,6 +28,7 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.util.Collections; @@ -90,7 +91,7 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - public ValueFetcher valueFetcher(MapperService mapperService, String format) { + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { throw new UnsupportedOperationException(); } diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index 4ddda5fc9f084..7ff55fc27a4f5 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -101,7 +101,6 @@ import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache; import org.elasticsearch.indices.mapper.MapperRegistry; -import org.elasticsearch.mock.orig.Mockito; import org.elasticsearch.plugins.SearchPlugin; import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.SearchModule; @@ -314,12 +313,6 @@ public boolean shouldCache(Query query) { IndexFieldDataService ifds = new IndexFieldDataService(indexSettings, new IndicesFieldDataCache(Settings.EMPTY, new IndexFieldDataCache.Listener() { }), circuitBreakerService, mapperService); - when(searchContext.getForField(Mockito.any(MappedFieldType.class))) - .thenAnswer(invocationOnMock -> ifds.getForField((MappedFieldType) invocationOnMock.getArguments()[0], - indexSettings.getIndex().getName(), - () -> { - throw new UnsupportedOperationException("search lookup not available"); - })); QueryShardContext queryShardContext = queryShardContextMock(contextIndexSearcher, mapperService, indexSettings, circuitBreakerService, bigArrays); when(searchContext.getQueryShardContext()).thenReturn(queryShardContext); diff --git a/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java b/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java index 1a418d1aebf57..c04b87ccf69cb 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java +++ b/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java @@ -28,7 +28,6 @@ import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.cache.bitset.BitsetFilterCache; -import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.ObjectMapper; @@ -308,11 +307,6 @@ public BitsetFilterCache bitsetFilterCache() { return fixedBitSetFilterCache; } - @Override - public > IFD getForField(MappedFieldType fieldType) { - return queryShardContext.getForField(fieldType); - } - @Override public TimeValue timeout() { return TimeValue.ZERO; diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java index 3930a7c88189f..46c4f4a6584ef 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java @@ -168,7 +168,7 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - public ValueFetcher valueFetcher(MapperService mapperService, String format) { + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchSingleNodeTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchSingleNodeTests.java index e62c893d34750..83af1481ad009 100644 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchSingleNodeTests.java +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchSingleNodeTests.java @@ -117,7 +117,7 @@ public void testFetchFailuresOnlySomeShards() throws Exception { public static final class SubFetchPhasePlugin extends Plugin implements SearchPlugin { @Override public List getFetchSubPhases(FetchPhaseConstructionContext context) { - return Collections.singletonList(searchContext -> new FetchSubPhaseProcessor() { + return Collections.singletonList((searchContext, lookup) -> new FetchSubPhaseProcessor() { @Override public void setNextReader(LeafReaderContext readerContext) {} diff --git a/x-pack/plugin/mapper-constant-keyword/src/internalClusterTest/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java b/x-pack/plugin/mapper-constant-keyword/src/internalClusterTest/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java index a084081b0dcb2..9cbf209ecb078 100644 --- a/x-pack/plugin/mapper-constant-keyword/src/internalClusterTest/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java +++ b/x-pack/plugin/mapper-constant-keyword/src/internalClusterTest/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java @@ -113,7 +113,7 @@ protected void minimalMapping(XContentBuilder b) throws IOException { public void testFetchValue() throws Exception { MapperService mapperService = createMapperService(fieldMapping(b -> b.field("type", "constant_keyword"))); FieldMapper fieldMapper = (FieldMapper) mapperService.documentMapper().mappers().getMapper("field"); - ValueFetcher fetcher = fieldMapper.valueFetcher(mapperService, null); + ValueFetcher fetcher = fieldMapper.valueFetcher(mapperService, null, null); SourceLookup missingValueLookup = new SourceLookup(); SourceLookup nullValueLookup = new SourceLookup(); @@ -124,7 +124,7 @@ public void testFetchValue() throws Exception { merge(mapperService, fieldMapping(b -> b.field("type", "constant_keyword").field("value", "foo"))); fieldMapper = (FieldMapper) mapperService.documentMapper().mappers().getMapper("field"); - fetcher = fieldMapper.valueFetcher(mapperService, null); + fetcher = fieldMapper.valueFetcher(mapperService, null, null); assertEquals(List.of("foo"), fetcher.fetchValues(missingValueLookup)); assertEquals(List.of("foo"), fetcher.fetchValues(nullValueLookup)); diff --git a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java index 21e6c9254c216..b354d7a2dc3ba 100644 --- a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java +++ b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java @@ -267,7 +267,7 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - public ValueFetcher valueFetcher(MapperService mapperService, String format) { + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } diff --git a/x-pack/plugin/mapper-flattened/src/internalClusterTest/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapperTests.java b/x-pack/plugin/mapper-flattened/src/internalClusterTest/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapperTests.java index 2e6295a41098e..b65ecc2b5c89a 100644 --- a/x-pack/plugin/mapper-flattened/src/internalClusterTest/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapperTests.java +++ b/x-pack/plugin/mapper-flattened/src/internalClusterTest/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapperTests.java @@ -513,7 +513,7 @@ public void testSplitQueriesOnWhitespace() throws IOException { new String[] {"Hello", "World"}); } - public void testFetchSourceValue() { + public void testFetchSourceValue() throws IOException { Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); diff --git a/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java b/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java index c4283ac49a1b9..50873db863870 100644 --- a/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java +++ b/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java @@ -564,7 +564,7 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - public ValueFetcher valueFetcher(MapperService mapperService, String format) { + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } diff --git a/x-pack/plugin/runtime-fields/qa/rest/build.gradle b/x-pack/plugin/runtime-fields/qa/rest/build.gradle index 11627ae95aa77..34998468a35a2 100644 --- a/x-pack/plugin/runtime-fields/qa/rest/build.gradle +++ b/x-pack/plugin/runtime-fields/qa/rest/build.gradle @@ -32,8 +32,6 @@ yamlRestTest { systemProperty 'tests.rest.blacklist', [ /////// TO FIX /////// - 'search/330_fetch_fields/*', // The 'fields' option is not yet supported - 'search/110_field_collapsing/field collapsing, inner_hits, and fields', // Also fails because of the 'fields' option 'search.highlight/40_keyword_ignore/Plain Highligher should skip highlighting ignored keyword values', // The plain highlighter is incompatible with runtime fields. Worth fixing? 'search/115_multiple_field_collapsing/two levels fields collapsing', // Broken. Gotta fix. 'field_caps/30_filter/Field caps with index filter', // We don't support filtering field caps on runtime fields. What should we do? diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeFieldMapper.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeFieldMapper.java index 51c4154ea1c81..d52fd4f960b46 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeFieldMapper.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeFieldMapper.java @@ -10,6 +10,7 @@ import org.elasticsearch.common.util.LocaleUtils; import org.elasticsearch.index.mapper.BooleanFieldMapper; import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.index.mapper.DocValueFetcher; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.IpFieldMapper; import org.elasticsearch.index.mapper.KeywordFieldMapper; @@ -22,6 +23,7 @@ import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptType; +import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.xpack.runtimefields.BooleanScriptFieldScript; import org.elasticsearch.xpack.runtimefields.DateScriptFieldScript; import org.elasticsearch.xpack.runtimefields.DoubleScriptFieldScript; @@ -75,8 +77,8 @@ protected void parseCreateField(ParseContext context) { } @Override - public ValueFetcher valueFetcher(MapperService mapperService, String format) { - throw new UnsupportedOperationException(); + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup lookup, String format) { + return new DocValueFetcher(fieldType().docValueFormat(format, null), lookup.doc().getForField(fieldType())); } @Override diff --git a/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapperTests.java b/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapperTests.java index df6694303fe28..1dcf312c8c6d9 100644 --- a/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapperTests.java +++ b/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapperTests.java @@ -304,7 +304,7 @@ public void testIgnoreZValue() throws IOException { assertThat(ignoreZValue, equalTo(false)); } - public void testFetchSourceValue() { + public void testFetchSourceValue() throws IOException { Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); diff --git a/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapperTests.java b/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapperTests.java index 64d98ba8abbfa..47f1d62459a56 100644 --- a/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapperTests.java +++ b/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapperTests.java @@ -332,7 +332,7 @@ public String toXContentString(ShapeFieldMapper mapper) throws IOException { return toXContentString(mapper, true); } - public void testFetchSourceValue() { + public void testFetchSourceValue() throws IOException { Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_keyword.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_keyword.yml index d346e3980a328..1bd38e88bb88c 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_keyword.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_keyword.yml @@ -99,6 +99,20 @@ setup: lang: painless meta: {} +--- +"fetch fields": + - do: + search: + index: sensor + body: + sort: timestamp + fields: [day_of_week, day_of_week_from_source, day_of_week_letters, prefixed_node] + - match: {hits.total.value: 6} + - match: {hits.hits.0.fields.day_of_week: [Thursday] } + - match: {hits.hits.0.fields.day_of_week_from_source: [Thursday] } + - match: {hits.hits.0.fields.day_of_week_letters: [T, a, d, h, r, s, u, y] } + - match: {hits.hits.0.fields.prefixed_node: [node_c] } + --- "docvalue_fields": - do: diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/20_long.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/20_long.yml index 9160075a068fb..45fb63a51d24b 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/20_long.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/20_long.yml @@ -92,6 +92,32 @@ setup: - match: {sensor.mappings.properties.voltage_times_ten.script.params: {multiplier: 10} } - match: {sensor.mappings.properties.voltage_times_ten.script.lang: painless } +--- +"fetch fields": + - do: + search: + index: sensor + body: + sort: timestamp + fields: + - voltage_times_ten + - voltage_times_ten_from_source + - temperature_digits + - field: millis_ago + format: epoch_millis + - match: {hits.total.value: 6} + - match: {hits.hits.0.fields.voltage_times_ten: [40] } + - match: {hits.hits.0.fields.voltage_times_ten_from_source: [40] } + - match: {hits.hits.0.fields.temperature_digits: [0, 2, 2] } + - match: {hits.hits.0.fields.voltage_times_ten: [40] } + - match: {hits.hits.1.fields.voltage_times_ten: [42] } + - match: {hits.hits.2.fields.voltage_times_ten: [56] } + - match: {hits.hits.3.fields.voltage_times_ten: [51] } + - match: {hits.hits.4.fields.voltage_times_ten: [58] } + - match: {hits.hits.5.fields.voltage_times_ten: [52] } + # We can't check the value because it is constantly increasing. If `gt` worked on strings we could do it, but it doens't. + - is_true: hits.hits.0.fields.millis_ago.0 + --- "docvalue_fields": - do: diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/30_double.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/30_double.yml index a30ebf53f463e..12b197519c6ce 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/30_double.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/30_double.yml @@ -83,6 +83,25 @@ setup: - match: {sensor.mappings.properties.voltage_percent.script.params: {max: 5.8} } - match: {sensor.mappings.properties.voltage_percent.script.lang: painless } +--- +"fetch fields": + - do: + search: + index: sensor + body: + sort: timestamp + fields: [voltage_percent, voltage_percent_from_source, voltage_sqrts] + - match: {hits.total.value: 6} + - match: {hits.hits.0.fields.voltage_percent: [0.6896551724137931] } + - match: {hits.hits.0.fields.voltage_percent_from_source: [0.6896551724137931] } + # Scripts that scripts that emit multiple values are supported and their results are sorted + - match: {hits.hits.0.fields.voltage_sqrts: [1.4142135623730951, 2.0, 4.0] } + - match: {hits.hits.1.fields.voltage_percent: [0.7241379310344828] } + - match: {hits.hits.2.fields.voltage_percent: [0.9655172413793103] } + - match: {hits.hits.3.fields.voltage_percent: [0.8793103448275862] } + - match: {hits.hits.4.fields.voltage_percent: [1.0] } + - match: {hits.hits.5.fields.voltage_percent: [0.896551724137931] } + --- "docvalue_fields": - do: diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/40_date.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/40_date.yml index 21a3f74de06c7..6b17629629db0 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/40_date.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/40_date.yml @@ -109,6 +109,29 @@ setup: - match: {sensor.mappings.properties.formatted_tomorrow.script.lang: painless } - match: {sensor.mappings.properties.formatted_tomorrow.format: yyyy-MM-dd } +--- +"fetch fields": + - do: + search: + index: sensor + body: + sort: timestamp + fields: [tomorrow, tomorrow_from_source, the_past, all_week, formatted_tomorrow] + - match: {hits.total.value: 6} + - match: {hits.hits.0.fields.tomorrow: [2018-01-19T17:41:34.000Z] } + - match: {hits.hits.0.fields.tomorrow_from_source: [2018-01-19T17:41:34.000Z] } + - match: {hits.hits.0.fields.the_past: [2018-01-18T17:41:33.000Z] } + - match: + hits.hits.0.fields.all_week: + - 2018-01-18T17:41:34.000Z + - 2018-01-19T17:41:34.000Z + - 2018-01-20T17:41:34.000Z + - 2018-01-21T17:41:34.000Z + - 2018-01-22T17:41:34.000Z + - 2018-01-23T17:41:34.000Z + - 2018-01-24T17:41:34.000Z + - match: {hits.hits.0.fields.formatted_tomorrow: [2018-01-19] } + --- "docvalue_fields": - do: diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/50_ip.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/50_ip.yml index 32808f63a7dd6..843ea2cda257b 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/50_ip.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/50_ip.yml @@ -76,6 +76,25 @@ setup: emit(m.substring(0, end)); - match: {http_logs.mappings.properties.ip.script.lang: painless } +--- +"fetch fields": + - do: + search: + index: http_logs + body: + sort: timestamp + fields: [ip, ip_from_source, ip_many] + - match: {hits.total.value: 6} + - match: {hits.hits.0.fields.ip: ["40.135.0.0"] } + - match: {hits.hits.0.fields.ip_from_source: ["40.135.0.0"] } + - match: + hits.hits.0.fields.ip_many: + - 40.135.0.0 + - 40.135.0.1 + - 40.135.0.2 + - 40.135.0.3 + - 40.135.0.4 + --- "docvalue_fields": - do: diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/60_boolean.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/60_boolean.yml index f6e8f420694c6..aaaccaf6c1cd3 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/60_boolean.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/60_boolean.yml @@ -80,6 +80,19 @@ setup: - match: {sensor.mappings.properties.over_v.script.params: {min_v: 5.0} } - match: {sensor.mappings.properties.over_v.script.lang: painless } +--- +"fetch fields": + - do: + search: + index: sensor + body: + sort: timestamp + fields: [over_v, over_v_from_source, big_vals] + - match: {hits.total.value: 6} + - match: {hits.hits.0.fields.over_v: [false] } + - match: {hits.hits.0.fields.over_v_from_source: [false] } + - match: {hits.hits.0.fields.big_vals: [false, true] } # doc values are sorted with falses before trues + --- "docvalue_fields": - do: diff --git a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java index eeebb4e399d72..abaf41a1f6b77 100644 --- a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java +++ b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java @@ -211,7 +211,7 @@ public void parse(ParseContext context) throws IOException { } @Override - public ValueFetcher valueFetcher(MapperService mapperService, String format) { + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } diff --git a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapper.java b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapper.java index 487391f86348c..143ffac1bbd45 100644 --- a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapper.java +++ b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapper.java @@ -24,6 +24,7 @@ import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.lookup.SearchLookup; import java.time.ZoneId; import java.util.List; @@ -146,7 +147,7 @@ protected void parseCreateField(ParseContext context) { } @Override - public ValueFetcher valueFetcher(MapperService mapperService, String format) { + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } diff --git a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java index 21e9714252329..397a6f3204f9d 100644 --- a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java +++ b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java @@ -959,7 +959,7 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - public ValueFetcher valueFetcher(MapperService mapperService, String format) { + public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } diff --git a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java index 71f8afbcba649..3836547430c3f 100644 --- a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java +++ b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java @@ -820,7 +820,7 @@ protected String convertToRandomRegex(String randomValue) { return result.toString(); } - public void testFetchSourceValue() { + public void testFetchSourceValue() throws IOException { Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath());