diff --git a/docs/changelog/91465.yaml b/docs/changelog/91465.yaml new file mode 100644 index 0000000000000..fa8f78a897a42 --- /dev/null +++ b/docs/changelog/91465.yaml @@ -0,0 +1,5 @@ +pr: 91465 +summary: Support synthetic `_source` for `_doc_count` field +area: TSDB +type: enhancement +issues: [] diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureMetaFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureMetaFieldMapper.java index b47370fe0e487..65a92c0e00d64 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureMetaFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureMetaFieldMapper.java @@ -11,6 +11,7 @@ import org.apache.lucene.search.Query; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MetadataFieldMapper; +import org.elasticsearch.index.mapper.SourceLoader; import org.elasticsearch.index.mapper.TextSearchInfo; import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.query.SearchExecutionContext; @@ -68,4 +69,8 @@ protected String contentType() { return CONTENT_TYPE; } + @Override + public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { + return SourceLoader.SyntheticFieldLoader.NOTHING; + } } diff --git a/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java b/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java index cbf64affc588e..d6292a7d6451f 100644 --- a/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java +++ b/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java @@ -16,6 +16,7 @@ import org.elasticsearch.index.mapper.MetadataFieldMapper; import org.elasticsearch.index.mapper.NumberFieldMapper.NumberFieldType; import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; +import org.elasticsearch.index.mapper.SourceLoader; import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.query.SearchExecutionContext; @@ -96,4 +97,9 @@ public void postParse(DocumentParserContext context) { public FieldMapper.Builder getMergeBuilder() { return new Builder().init(this); } + + @Override + public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { + return SourceLoader.SyntheticFieldLoader.NOTHING; + } } diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/get/100_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/get/100_synthetic_source.yml index 77edf3d78874f..566cc777faa41 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/get/100_synthetic_source.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/get/100_synthetic_source.yml @@ -584,6 +584,45 @@ _source filtering: kwd: foo - is_false: fields +--- +_doc_count: + - skip: + version: " - 8.5.99" + reason: introduced in 8.6.0 + + - do: + indices.create: + index: test + body: + settings: + number_of_replicas: 0 + mappings: + _source: + mode: synthetic + + - do: + index: + index: test + id: 1 + refresh: true + body: + _doc_count: 3 + foo: bar + + - do: + get: + index: test + id: 1 + - match: {_index: "test"} + - match: {_id: "1"} + - match: {_version: 1} + - match: {found: true} + - match: + _source: + _doc_count: 3 + foo: bar + - is_false: fields + --- ip with ignore_malformed: - skip: diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java index 6bb4dcac01cd3..6656e0aa1f936 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java @@ -30,6 +30,7 @@ import org.elasticsearch.index.mapper.DocumentParserContext; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MetadataFieldMapper; +import org.elasticsearch.index.mapper.SourceLoader; import org.elasticsearch.index.mapper.TimeSeriesParams; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; @@ -718,6 +719,11 @@ protected String contentType() { return CONTENT_TYPE; } + @Override + public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { + throw new UnsupportedOperationException(); + } + private static final TypeParser PARSER = new FixedTypeParser(c -> new TestMetadataMapper()); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java index ce6da0f3db85a..ece0b082a3ccf 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java @@ -268,4 +268,9 @@ protected String contentType() { public boolean isEnabled() { return enabled; } + + @Override + public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { + return SourceLoader.SyntheticFieldLoader.NOTHING; + } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocCountFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DocCountFieldMapper.java index 907a74c0ad259..fa5b01913be83 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocCountFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocCountFieldMapper.java @@ -8,14 +8,20 @@ package org.elasticsearch.index.mapper; import org.apache.lucene.index.IndexableField; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.PostingsEnum; +import org.apache.lucene.index.Term; import org.apache.lucene.search.Query; import org.elasticsearch.common.xcontent.XContentParserUtils; import org.elasticsearch.index.query.QueryShardException; import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentParser; import java.io.IOException; import java.util.Collections; +import java.util.Map; +import java.util.stream.Stream; /** Mapper for the doc_count field. */ public class DocCountFieldMapper extends MetadataFieldMapper { @@ -25,6 +31,11 @@ public class DocCountFieldMapper extends MetadataFieldMapper { private static final DocCountFieldMapper INSTANCE = new DocCountFieldMapper(); + /** + * The term that is the key to the postings list that stores the doc counts. + */ + private static final Term TERM = new Term(NAME, NAME); + public static final TypeParser PARSER = new FixedTypeParser(c -> INSTANCE); public static final class DocCountFieldType extends MappedFieldType { @@ -115,4 +126,49 @@ protected String contentType() { public static IndexableField field(int count) { return new CustomTermFreqField(NAME, NAME, count); } + + @Override + public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { + return new SyntheticFieldLoader(); + } + + /** + * The lookup for loading values. + */ + public static PostingsEnum leafLookup(LeafReader reader) throws IOException { + return reader.postings(TERM); + } + + private class SyntheticFieldLoader implements SourceLoader.SyntheticFieldLoader { + private PostingsEnum postings; + private boolean hasValue; + + @Override + public Stream> storedFieldLoaders() { + return Stream.empty(); + } + + @Override + public DocValuesLoader docValuesLoader(LeafReader leafReader, int[] docIdsInLeaf) throws IOException { + postings = leafLookup(leafReader); + if (postings == null) { + hasValue = false; + return null; + } + return docId -> hasValue = docId == postings.advance(docId); + } + + @Override + public boolean hasValue() { + return hasValue; + } + + @Override + public void write(XContentBuilder b) throws IOException { + if (hasValue == false) { + return; + } + b.field(NAME, postings.freq()); + } + } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldNamesFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldNamesFieldMapper.java index 3d841977d3b0b..ac78cf9b041cd 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldNamesFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldNamesFieldMapper.java @@ -196,4 +196,8 @@ protected String contentType() { return CONTENT_TYPE; } + @Override + public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { + return SourceLoader.SyntheticFieldLoader.NOTHING; + } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java index 8a921366556b8..29866ca4b56b2 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java @@ -41,6 +41,11 @@ protected final String contentType() { return CONTENT_TYPE; } + @Override + public final SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { + return SourceLoader.SyntheticFieldLoader.NOTHING; + } + /** * Description of the document being parsed used in error messages. Not * called unless there is an error. diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredFieldMapper.java index ecbeff9d8cb41..94ed3ee515c92 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredFieldMapper.java @@ -88,4 +88,8 @@ protected String contentType() { return CONTENT_TYPE; } + @Override + public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { + return SourceLoader.SyntheticFieldLoader.NOTHING; + } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java index dc79a4c8195ac..fdc2b9cef07d1 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java @@ -102,4 +102,9 @@ public IndexFieldMapper() { protected String contentType() { return CONTENT_TYPE; } + + @Override + public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { + return SourceLoader.SyntheticFieldLoader.NOTHING; + } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/LegacyTypeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/LegacyTypeFieldMapper.java index a6075b9a46627..5d658e67b0904 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/LegacyTypeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/LegacyTypeFieldMapper.java @@ -106,4 +106,9 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) protected String contentType() { return CONTENT_TYPE; } + + @Override + public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { + return SourceLoader.SyntheticFieldLoader.NOTHING; + } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperBuilderContext.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperBuilderContext.java index c5a7bf2584e9f..8ed6bf3d1db7e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperBuilderContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperBuilderContext.java @@ -11,6 +11,7 @@ import org.elasticsearch.common.Strings; import java.util.Objects; +import java.util.function.BooleanSupplier; /** * Holds context for building Mapper objects from their Builders @@ -21,19 +22,28 @@ public final class MapperBuilderContext { * The root context, to be used when building a tree of mappers */ public static MapperBuilderContext root(boolean isSourceSynthetic) { - return new MapperBuilderContext(isSourceSynthetic); + return new MapperBuilderContext(null, () -> isSourceSynthetic); + } + + /** + * A context to use to build metadata fields. + */ + public static MapperBuilderContext forMetadata() { + return new MapperBuilderContext( + null, + () -> { throw new UnsupportedOperationException("metadata fields can't check if _source is synthetic"); } + ); } private final String path; - private final boolean isSourceSynthetic; + private final BooleanSupplier isSourceSynthetic; - private MapperBuilderContext(boolean isSourceSynthetic) { - this.path = null; - this.isSourceSynthetic = isSourceSynthetic; + MapperBuilderContext(String path, boolean isSourceSynthetic) { + this(Objects.requireNonNull(path), () -> isSourceSynthetic); } - MapperBuilderContext(String path, boolean isSourceSynthetic) { - this.path = Objects.requireNonNull(path); + private MapperBuilderContext(String path, BooleanSupplier isSourceSynthetic) { + this.path = path; this.isSourceSynthetic = isSourceSynthetic; } @@ -56,7 +66,10 @@ public String buildFullName(String name) { return path + "." + name; } + /** + * Is the {@code _source} field being reconstructed on the fly? + */ public boolean isSourceSynthetic() { - return isSourceSynthetic; + return isSourceSynthetic.getAsBoolean(); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/Mapping.java b/server/src/main/java/org/elasticsearch/index/mapper/Mapping.java index 04ac4063aa3a0..a67d761522912 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/Mapping.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/Mapping.java @@ -124,6 +124,10 @@ private boolean isSourceSynthetic() { return sfm != null && sfm.isSynthetic(); } + public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { + return root.syntheticFieldLoader(Arrays.stream(metadataMappers)); + } + /** * Merges a new mapping into the existing one. * diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappingParser.java b/server/src/main/java/org/elasticsearch/index/mapper/MappingParser.java index c9d9d48528c03..07a15cc6e2f5f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappingParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappingParser.java @@ -120,7 +120,7 @@ private Mapping parse(String type, Map mapping) throws MapperPar @SuppressWarnings("unchecked") Map fieldNodeMap = (Map) fieldNode; MetadataFieldMapper metadataFieldMapper = typeParser.parse(fieldName, fieldNodeMap, parserContext) - .build(MapperBuilderContext.root(false)); + .build(MapperBuilderContext.forMetadata()); metadataMappers.put(metadataFieldMapper.getClass(), metadataFieldMapper); fieldNodeMap.remove("type"); checkNoRemainingFields(fieldName, fieldNodeMap); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java index 6303cf0063b1b..b1c00f8f58a6f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java @@ -155,7 +155,7 @@ public final XContentBuilder toXContent(XContentBuilder builder, Params params) @Override protected void parseCreateField(DocumentParserContext context) throws IOException { throw new MapperParsingException( - "Field [" + name() + "] is a metadata field and cannot be added inside" + " a document. Use the index API request parameters." + "Field [" + name() + "] is a metadata field and cannot be added inside a document. Use the index API request parameters." ); } @@ -173,4 +173,6 @@ public void postParse(DocumentParserContext context) throws IOException { // do nothing } + @Override + public abstract SourceLoader.SyntheticFieldLoader syntheticFieldLoader(); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NestedPathFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NestedPathFieldMapper.java index 29ac3b86b6c72..713b25d8478e1 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NestedPathFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NestedPathFieldMapper.java @@ -97,4 +97,8 @@ protected String contentType() { return NAME; } + @Override + public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { + return SourceLoader.SyntheticFieldLoader.NOTHING; + } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java index 656eb6e722e19..bc04837ad6e2a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java @@ -578,11 +578,9 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep } - @Override - public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { + public SourceLoader.SyntheticFieldLoader syntheticFieldLoader(Stream extra) { return new SyntheticSourceFieldLoader( - mappers.values() - .stream() + Stream.concat(extra, mappers.values().stream()) .sorted(Comparator.comparing(Mapper::name)) .map(Mapper::syntheticFieldLoader) .filter(l -> l != null) @@ -590,6 +588,11 @@ public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { ); } + @Override + public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { + return syntheticFieldLoader(Stream.empty()); + } + private class SyntheticSourceFieldLoader implements SourceLoader.SyntheticFieldLoader { private final List fields; private boolean hasValue; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java index f7ef8b9113cbc..52f11ee045e63 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java @@ -131,4 +131,8 @@ protected String contentType() { return CONTENT_TYPE; } + @Override + public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { + return SourceLoader.SyntheticFieldLoader.NOTHING; + } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java index ffa98a34c551e..f4d6fdb7aa2c9 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java @@ -239,4 +239,8 @@ protected String contentType() { return CONTENT_TYPE; } + @Override + public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { + return SourceLoader.SyntheticFieldLoader.NOTHING; + } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java index fb95311437a3d..54aaaa845c81a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java @@ -270,4 +270,9 @@ public SourceLoader newSourceLoader(Mapping mapping) { public boolean isSynthetic() { return mode == Mode.SYNTHETIC; } + + @Override + public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { + return SourceLoader.SyntheticFieldLoader.NOTHING; + } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java index 394bf736a158b..a2e9ce84d425a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java @@ -89,7 +89,7 @@ class Synthetic implements SourceLoader { private final Map storedFieldLoaders; public Synthetic(Mapping mapping) { - loader = mapping.getRoot().syntheticFieldLoader(); + loader = mapping.syntheticFieldLoader(); storedFieldLoaders = Map.copyOf(loader.storedFieldLoaders().collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java index ad55d50fcf071..3032c4e696cd9 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java @@ -155,6 +155,11 @@ protected String contentType() { return CONTENT_TYPE; } + @Override + public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { + return SourceLoader.SyntheticFieldLoader.NOTHING; + } + /** * Decode the {@code _tsid} into a human readable map. */ diff --git a/server/src/main/java/org/elasticsearch/index/mapper/VersionFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/VersionFieldMapper.java index 88ed2d9224d88..0de2a27fbaac2 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/VersionFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/VersionFieldMapper.java @@ -92,4 +92,9 @@ public void postParse(DocumentParserContext context) { protected String contentType() { return CONTENT_TYPE; } + + @Override + public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { + return SourceLoader.SyntheticFieldLoader.NOTHING; + } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/DocCountProvider.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/DocCountProvider.java index 8c7ee72e7cb5a..02d5c1c84a9c9 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/DocCountProvider.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/DocCountProvider.java @@ -10,7 +10,6 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.PostingsEnum; -import org.apache.lucene.index.Term; import org.elasticsearch.index.mapper.DocCountFieldMapper; import java.io.IOException; @@ -41,7 +40,7 @@ public int getDocCount(int doc) throws IOException { } public void setLeafReaderContext(LeafReaderContext ctx) throws IOException { - docCountPostings = ctx.reader().postings(new Term(DocCountFieldMapper.NAME, DocCountFieldMapper.NAME)); + docCountPostings = DocCountFieldMapper.leafLookup(ctx.reader()); } public boolean alwaysOne() { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocCountFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocCountFieldMapperTests.java index 08bc585a400b9..21c7d3b4a8963 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DocCountFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocCountFieldMapperTests.java @@ -8,8 +8,16 @@ package org.elasticsearch.index.mapper; import org.apache.lucene.index.IndexableField; +import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.index.fieldvisitor.LeafStoredFieldLoader; +import org.elasticsearch.index.fieldvisitor.StoredFieldLoader; + +import java.io.IOException; +import java.util.List; +import java.util.stream.IntStream; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; public class DocCountFieldMapperTests extends MapperServiceTestCase { @@ -57,4 +65,35 @@ public void testInvalidDocument_ArrayDocCount() throws Exception { Exception e = expectThrows(MapperParsingException.class, () -> mapper.parse(source(b -> b.array(CONTENT_TYPE, 10, 20, 30)))); assertThat(e.getCause().getMessage(), containsString("Arrays are not allowed for field [_doc_count].")); } + + public void testSyntheticSource() throws IOException { + DocumentMapper mapper = createDocumentMapper(syntheticSourceMapping(b -> {})); + assertThat(syntheticSource(mapper, b -> b.field(CONTENT_TYPE, 10)), equalTo("{\"_doc_count\":10}")); + } + + public void testSyntheticSourceMany() throws IOException { + MapperService mapper = createMapperService(syntheticSourceMapping(b -> {})); + List counts = randomList(2, 10000, () -> between(1, Integer.MAX_VALUE)); + withLuceneIndex(mapper, iw -> { + for (int c : counts) { + iw.addDocument(mapper.documentMapper().parse(source(b -> b.field(CONTENT_TYPE, c))).rootDoc()); + } + }, reader -> { + int i = 0; + SourceLoader loader = mapper.mappingLookup().newSourceLoader(); + assertTrue(loader.requiredStoredFields().isEmpty()); + for (LeafReaderContext leaf : reader.leaves()) { + int[] docIds = IntStream.range(0, leaf.reader().maxDoc()).toArray(); + SourceLoader.Leaf sourceLoaderLeaf = loader.leaf(leaf.reader(), docIds); + LeafStoredFieldLoader storedFieldLoader = StoredFieldLoader.empty().getLoader(leaf, docIds); + for (int docId : docIds) { + assertThat( + "doc " + docId, + sourceLoaderLeaf.source(storedFieldLoader, docId).utf8ToString(), + equalTo("{\"_doc_count\":" + counts.get(i++) + "}") + ); + } + } + }); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java index 934511165289e..9f25bd6477130 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java @@ -2573,6 +2573,11 @@ protected String contentType() { return CONTENT_TYPE; } + @Override + public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { + throw new UnsupportedOperationException(); + } + private static final TypeParser PARSER = new FixedTypeParser(c -> new MockMetadataMapper()); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ExternalMetadataMapper.java b/server/src/test/java/org/elasticsearch/index/mapper/ExternalMetadataMapper.java index 02c438688d768..aca1984c502cd 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ExternalMetadataMapper.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ExternalMetadataMapper.java @@ -31,6 +31,10 @@ public void postParse(DocumentParserContext context) { context.doc().add(new StringField(FIELD_NAME, FIELD_VALUE, Store.YES)); } - public static final TypeParser PARSER = new FixedTypeParser(c -> new ExternalMetadataMapper()); + @Override + public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { + throw new UnsupportedOperationException(); + } + public static final TypeParser PARSER = new FixedTypeParser(c -> new ExternalMetadataMapper()); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/routing/allocation/mapper/DataTierFieldMapper.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/routing/allocation/mapper/DataTierFieldMapper.java index cf6229bc75458..9cfa9be606239 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/routing/allocation/mapper/DataTierFieldMapper.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/routing/allocation/mapper/DataTierFieldMapper.java @@ -17,6 +17,7 @@ import org.elasticsearch.index.mapper.ConstantFieldType; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MetadataFieldMapper; +import org.elasticsearch.index.mapper.SourceLoader; import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.query.SearchExecutionContext; @@ -109,4 +110,9 @@ public DataTierFieldMapper() { protected String contentType() { return CONTENT_TYPE; } + + @Override + public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { + return SourceLoader.SyntheticFieldLoader.NOTHING; + } }