diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/DisableGraphQueryTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/DisableGraphQueryTests.java index d1792e94f7331..c7ac35d3febce 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/DisableGraphQueryTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/DisableGraphQueryTests.java @@ -46,6 +46,7 @@ import java.util.Collection; import java.util.Collections; +import static java.util.Collections.emptyMap; import static org.hamcrest.Matchers.equalTo; /** @@ -84,7 +85,7 @@ public void setup() { indexService = createIndex("test", settings, "t", "text_shingle", "type=text,analyzer=text_shingle", "text_shingle_unigram", "type=text,analyzer=text_shingle_unigram"); - shardContext = indexService.newQueryShardContext(0, null, () -> 0L, null); + shardContext = indexService.newQueryShardContext(0, null, () -> 0L, null, emptyMap()); // parsed queries for "text_shingle_unigram:(foo bar baz)" with query parsers // that ignores position length attribute diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java index 21059f2112457..1810a1eff384a 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java @@ -85,6 +85,7 @@ import java.util.Map; import java.util.Objects; +import static java.util.Collections.emptyMap; import static org.elasticsearch.action.ValidateActions.addValidationError; import static org.elasticsearch.rest.RestRequest.Method.GET; import static org.elasticsearch.rest.RestRequest.Method.POST; @@ -555,7 +556,7 @@ private static Response prepareRamIndex(Request request, searcher.setQueryCache(null); final long absoluteStartMillis = System.currentTimeMillis(); QueryShardContext context = - indexService.newQueryShardContext(0, searcher, () -> absoluteStartMillis, null); + indexService.newQueryShardContext(0, searcher, () -> absoluteStartMillis, null, emptyMap()); return handler.apply(context, indexReader.leaves().get(0)); } } diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java index e9a6ca60509e3..ca885cdfdff6a 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java @@ -32,6 +32,8 @@ import java.util.List; import java.util.Map; +import static java.util.Collections.emptyMap; + /** * Test that needsScores() is reported correctly depending on whether _score is used */ @@ -45,7 +47,7 @@ public void testNeedsScores() { contexts.put(NumberSortScript.CONTEXT, Whitelist.BASE_WHITELISTS); PainlessScriptEngine service = new PainlessScriptEngine(Settings.EMPTY, contexts); - QueryShardContext shardContext = index.newQueryShardContext(0, null, () -> 0, null); + QueryShardContext shardContext = index.newQueryShardContext(0, null, () -> 0, null, emptyMap()); NumberSortScript.Factory factory = service.compile(null, "1.2", NumberSortScript.CONTEXT, Collections.emptyMap()); NumberSortScript.LeafFactory ss = factory.newFactory(Collections.emptyMap(), shardContext.lookup()); diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java index 19f12d6130d00..1d59ab64b8231 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java @@ -103,6 +103,7 @@ import java.util.function.Function; import java.util.stream.Collectors; +import static java.util.Collections.emptyMap; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.QueryBuilders.boolQuery; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; @@ -515,7 +516,7 @@ public void testQueryWithRewrite() throws Exception { QueryShardContext shardContext = indexService.newQueryShardContext( randomInt(20), null, () -> { throw new UnsupportedOperationException(); - }, null); + }, null, emptyMap()); PlainActionFuture future = new PlainActionFuture<>(); Rewriteable.rewriteAndFetch(queryBuilder, shardContext, future); assertQueryBuilder(qbSource, future.get()); diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchTests.java index e91a8cf1783e7..ff5d6dc7c4b7b 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchTests.java @@ -51,6 +51,7 @@ import java.util.Map; import java.util.function.Function; +import static java.util.Collections.emptyMap; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.QueryBuilders.boolQuery; import static org.elasticsearch.index.query.QueryBuilders.matchQuery; @@ -258,7 +259,7 @@ public void testRangeQueriesWithNow() throws Exception { try (Engine.Searcher searcher = indexService.getShard(0).acquireSearcher("test")) { long[] currentTime = new long[] {System.currentTimeMillis()}; QueryShardContext queryShardContext = - indexService.newQueryShardContext(0, searcher, () -> currentTime[0], null); + indexService.newQueryShardContext(0, searcher, () -> currentTime[0], null, emptyMap()); BytesReference source = BytesReference.bytes(jsonBuilder().startObject() .field("field1", "value") diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkByScrollParallelizationHelperTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkByScrollParallelizationHelperTests.java index a64415d08b1b1..b2512e6a62cfd 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkByScrollParallelizationHelperTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkByScrollParallelizationHelperTests.java @@ -38,6 +38,7 @@ public void testSliceIntoSubRequests() throws IOException { () -> null, () -> null, () -> emptyList(), + () -> null, () -> null)); if (searchRequest.source() != null) { // Clear the slice builder if there is one set. We can't call sliceIntoSubRequests if it is. diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/index/80_date_nanos.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/index/80_date_nanos.yml new file mode 100644 index 0000000000000..ada085ebb148e --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/index/80_date_nanos.yml @@ -0,0 +1,33 @@ +--- +"date_nanos requires dates after 1970 and before 2262": + + - do: + indices.create: + index: date_ns + body: + settings: + number_of_shards: 3 + number_of_replicas: 0 + mappings: + properties: + date: + type: date_nanos + field: + type: long + + - do: + bulk: + refresh: true + body: + - '{ "index" : { "_index" : "date_ns", "_id" : "date_ns_1" } }' + - '{"date" : "1969-10-28T12:12:12.123456789Z" }' + - '{ "index" : { "_index" : "date_ns", "_id" : "date_ns_2" } }' + - '{"date" : "2263-10-29T12:12:12.123456789Z" }' + + - match: { errors: true } + - match: { items.0.index.status: 400 } + - match: { items.0.index.error.type: mapper_parsing_exception } + - match: { items.0.index.error.caused_by.reason: "date[1969-10-28T12:12:12.123456789Z] is before the epoch in 1970 and cannot be stored in nanosecond resolution" } + - match: { items.1.index.status: 400 } + - match: { items.1.index.error.type: mapper_parsing_exception } + - match: { items.1.index.error.caused_by.reason: "date[2263-10-29T12:12:12.123456789Z] is after 2262-04-11T23:47:16.854775807 and cannot be stored in nanosecond resolution" } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/240_date_nanos.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/240_date_nanos.yml index cc9bd063ff4fc..007f7f7f0e88e 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/240_date_nanos.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/240_date_nanos.yml @@ -71,28 +71,6 @@ setup: - match: { hits.hits.1._id: "second" } - match: { hits.hits.1.sort: [1540815132987654321] } - ---- -"date_nanos requires dates after 1970 and before 2262": - - - do: - bulk: - refresh: true - body: - - '{ "index" : { "_index" : "date_ns", "_id" : "date_ns_1" } }' - - '{"date" : "1969-10-28T12:12:12.123456789Z" }' - - '{ "index" : { "_index" : "date_ns", "_id" : "date_ns_2" } }' - - '{"date" : "2263-10-29T12:12:12.123456789Z" }' - - - match: { errors: true } - - match: { items.0.index.status: 400 } - - match: { items.0.index.error.type: mapper_parsing_exception } - - match: { items.0.index.error.caused_by.reason: "date[1969-10-28T12:12:12.123456789Z] is before the epoch in 1970 and cannot be stored in nanosecond resolution" } - - match: { items.1.index.status: 400 } - - match: { items.1.index.error.type: mapper_parsing_exception } - - match: { items.1.index.error.caused_by.reason: "date[2263-10-29T12:12:12.123456789Z] is after 2262-04-11T23:47:16.854775807 and cannot be stored in nanosecond resolution" } - - --- "doc value fields are working as expected across date and date_nanos fields": diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java index 4cc978b166129..db129a9479934 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java @@ -56,6 +56,7 @@ import java.util.function.Function; import java.util.stream.Collectors; +import static java.util.Collections.emptyMap; import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.findConflictingV1Templates; import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.findConflictingV2Templates; import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.findV2Template; @@ -182,7 +183,7 @@ public static Template resolveTemplate(final String matchingTemplate, final Stri resolvedAliases, tempClusterState.metadata(), aliasValidator, xContentRegistry, // the context is only used for validation so it's fine to pass fake values for the // shard id and the current timestamp - tempIndexService.newQueryShardContext(0, null, () -> 0L, null))); + tempIndexService.newQueryShardContext(0, null, () -> 0L, null, emptyMap()))); Map aliasesByName = aliases.stream().collect( Collectors.toMap(AliasMetadata::getAlias, Function.identity())); diff --git a/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java b/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java index 57082ed450941..9ee9bf947a46d 100644 --- a/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java +++ b/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java @@ -89,7 +89,8 @@ public void run() { CollapseBuilder innerCollapseBuilder = innerHitBuilder.getInnerCollapseBuilder(); SearchSourceBuilder sourceBuilder = buildExpandSearchSourceBuilder(innerHitBuilder, innerCollapseBuilder) .query(groupQuery) - .postFilter(searchRequest.source().postFilter()); + .postFilter(searchRequest.source().postFilter()) + .runtimeMappings(searchRequest.source().runtimeMappings()); SearchRequest groupRequest = new SearchRequest(searchRequest); groupRequest.source(sourceBuilder); multiRequest.add(groupRequest); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java index 67fa07eb74a9e..4a75c47878d00 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java @@ -98,6 +98,7 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; +import static java.util.Collections.emptyMap; import static java.util.stream.Collectors.toList; import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING; import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING; @@ -486,7 +487,7 @@ private ClusterState applyCreateIndexRequestWithV1Templates(final ClusterState c MetadataIndexTemplateService.resolveAliases(templates), currentState.metadata(), aliasValidator, // the context is only used for validation so it's fine to pass fake values for the // shard id and the current timestamp - xContentRegistry, indexService.newQueryShardContext(0, null, () -> 0L, null)), + xContentRegistry, indexService.newQueryShardContext(0, null, () -> 0L, null, emptyMap())), templates.stream().map(IndexTemplateMetadata::getName).collect(toList()), metadataTransformer); } @@ -519,7 +520,7 @@ private ClusterState applyCreateIndexRequestWithV2Template(final ClusterState cu MetadataIndexTemplateService.resolveAliases(currentState.metadata(), templateName), currentState.metadata(), aliasValidator, // the context is only used for validation so it's fine to pass fake values for the // shard id and the current timestamp - xContentRegistry, indexService.newQueryShardContext(0, null, () -> 0L, null)), + xContentRegistry, indexService.newQueryShardContext(0, null, () -> 0L, null, emptyMap())), Collections.singletonList(templateName), metadataTransformer); } @@ -565,7 +566,7 @@ private ClusterState applyCreateIndexRequestWithExistingMetadata(final ClusterSt currentState.metadata(), aliasValidator, xContentRegistry, // the context is only used for validation so it's fine to pass fake values for the // shard id and the current timestamp - indexService.newQueryShardContext(0, null, () -> 0L, null)), + indexService.newQueryShardContext(0, null, () -> 0L, null, emptyMap())), List.of(), metadataTransformer); } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesService.java index 55c9635470a13..3ced1267655f1 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesService.java @@ -47,6 +47,7 @@ import java.util.function.Function; import static java.util.Collections.emptyList; +import static java.util.Collections.emptyMap; import static org.elasticsearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason.NO_LONGER_ASSIGNED; /** @@ -149,7 +150,7 @@ public ClusterState applyAliasActions(ClusterState currentState, Iterable System.currentTimeMillis(), null), xContentRegistry); + () -> System.currentTimeMillis(), null, emptyMap()), xContentRegistry); } }; if (action.apply(newAliasValidator, metadata, index)) { diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index 40392ba516e1c..ddab81145eff3 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -80,6 +80,7 @@ import java.util.function.Predicate; import java.util.stream.Collectors; +import static java.util.Collections.emptyMap; import static org.elasticsearch.cluster.metadata.MetadataCreateDataStreamService.validateTimestampFieldMapping; import static org.elasticsearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason.NO_LONGER_ASSIGNED; @@ -1117,7 +1118,7 @@ private static void validateCompositeTemplate(final ClusterState state, new AliasValidator(), // the context is only used for validation so it's fine to pass fake values for the // shard id and the current timestamp - xContentRegistry, tempIndexService.newQueryShardContext(0, null, () -> 0L, null)); + xContentRegistry, tempIndexService.newQueryShardContext(0, null, () -> 0L, null, emptyMap())); // triggers inclusion of _timestamp field and its validation: String indexName = DataStream.BACKING_INDEX_PREFIX + temporaryIndexName; diff --git a/server/src/main/java/org/elasticsearch/index/IndexService.java b/server/src/main/java/org/elasticsearch/index/IndexService.java index 88da415caabec..24f44c3336df7 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexService.java +++ b/server/src/main/java/org/elasticsearch/index/IndexService.java @@ -192,7 +192,7 @@ public IndexService( assert indexAnalyzers != null; this.mapperService = new MapperService(indexSettings, indexAnalyzers, xContentRegistry, similarityService, mapperRegistry, // we parse all percolator queries as they would be parsed on shard 0 - () -> newQueryShardContext(0, null, System::currentTimeMillis, null), idFieldDataEnabled, scriptService); + () -> newQueryShardContext(0, null, System::currentTimeMillis, null, emptyMap()), idFieldDataEnabled, scriptService); this.indexFieldData = new IndexFieldDataService(indexSettings, indicesFieldDataCache, circuitBreakerService, mapperService); if (indexSettings.getIndexSortConfig().hasIndexSort()) { // we delay the actual creation of the sort order for this index because the mapping has not been merged yet. @@ -586,13 +586,19 @@ public IndexSettings getIndexSettings() { * Passing a {@code null} {@link IndexSearcher} will return a valid context, however it won't be able to make * {@link IndexReader}-specific optimizations, such as rewriting containing range queries. */ - public QueryShardContext newQueryShardContext(int shardId, IndexSearcher searcher, LongSupplier nowInMillis, String clusterAlias) { + public QueryShardContext newQueryShardContext( + int shardId, + IndexSearcher searcher, + LongSupplier nowInMillis, + String clusterAlias, + Map runtimeMappings + ) { final SearchIndexNameMatcher indexNameMatcher = new SearchIndexNameMatcher(index().getName(), clusterAlias, clusterService, expressionResolver); return new QueryShardContext( shardId, indexSettings, bigArrays, indexCache.bitsetFilterCache(), indexFieldData::getForField, mapperService(), similarityService(), scriptService, xContentRegistry, namedWriteableRegistry, client, searcher, nowInMillis, clusterAlias, - indexNameMatcher, allowExpensiveQueries, valuesSourceRegistry); + indexNameMatcher, allowExpensiveQueries, valuesSourceRegistry, runtimeMappings); } /** diff --git a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java index f8131ecf19cf3..07447cf790e28 100644 --- a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java @@ -26,6 +26,7 @@ import org.apache.lucene.search.join.BitSetProducer; import org.apache.lucene.search.similarities.Similarity; import org.apache.lucene.util.SetOnce; +import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.client.Client; @@ -34,6 +35,7 @@ import org.elasticsearch.common.TriFunction; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.lucene.search.Queries; +import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentParser; @@ -68,6 +70,7 @@ import java.io.IOException; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -77,6 +80,8 @@ import java.util.function.Predicate; import java.util.function.Supplier; +import static java.util.Collections.emptyMap; + /** * Context object used to create lucene queries on the shard level. */ @@ -103,7 +108,11 @@ public class QueryShardContext extends QueryRewriteContext { private boolean mapUnmappedFieldAsString; private NestedScope nestedScope; private final ValuesSourceRegistry valuesSourceRegistry; + private final Map runtimeMappings; + /** + * Build a {@linkplain QueryShardContext} without any information from the search request. + */ public QueryShardContext(int shardId, IndexSettings indexSettings, BigArrays bigArrays, @@ -122,16 +131,63 @@ public QueryShardContext(int shardId, BooleanSupplier allowExpensiveQueries, ValuesSourceRegistry valuesSourceRegistry) { this(shardId, indexSettings, bigArrays, bitsetFilterCache, indexFieldDataLookup, mapperService, similarityService, - scriptService, xContentRegistry, namedWriteableRegistry, client, searcher, nowInMillis, indexNameMatcher, - new Index(RemoteClusterAware.buildRemoteIndexName(clusterAlias, indexSettings.getIndex().getName()), - indexSettings.getIndex().getUUID()), allowExpensiveQueries, valuesSourceRegistry); + scriptService, xContentRegistry, namedWriteableRegistry, client, searcher, nowInMillis, clusterAlias, + indexNameMatcher, allowExpensiveQueries, valuesSourceRegistry, emptyMap()); + } + + /** + * Build a {@linkplain QueryShardContext} with information from the search request. + */ + public QueryShardContext( + int shardId, + IndexSettings indexSettings, + BigArrays bigArrays, + BitsetFilterCache bitsetFilterCache, + TriFunction, IndexFieldData> indexFieldDataLookup, + MapperService mapperService, + SimilarityService similarityService, + ScriptService scriptService, + NamedXContentRegistry xContentRegistry, + NamedWriteableRegistry namedWriteableRegistry, + Client client, + IndexSearcher searcher, + LongSupplier nowInMillis, + String clusterAlias, + Predicate indexNameMatcher, + BooleanSupplier allowExpensiveQueries, + ValuesSourceRegistry valuesSourceRegistry, + Map runtimeMappings + ) { + this( + shardId, + indexSettings, + bigArrays, + bitsetFilterCache, + indexFieldDataLookup, + mapperService, + similarityService, + scriptService, + xContentRegistry, + namedWriteableRegistry, + client, + searcher, + nowInMillis, + indexNameMatcher, + new Index( + RemoteClusterAware.buildRemoteIndexName(clusterAlias, indexSettings.getIndex().getName()), + indexSettings.getIndex().getUUID() + ), + allowExpensiveQueries, + valuesSourceRegistry, + parseRuntimeMappings(runtimeMappings, mapperService, indexSettings) + ); } public QueryShardContext(QueryShardContext source) { this(source.shardId, source.indexSettings, source.bigArrays, source.bitsetFilterCache, source.indexFieldDataService, source.mapperService, source.similarityService, source.scriptService, source.getXContentRegistry(), source.getWriteableRegistry(), source.client, source.searcher, source.nowInMillis, source.indexNameMatcher, - source.fullyQualifiedIndex, source.allowExpensiveQueries, source.valuesSourceRegistry); + source.fullyQualifiedIndex, source.allowExpensiveQueries, source.valuesSourceRegistry, source.runtimeMappings); } private QueryShardContext(int shardId, @@ -150,7 +206,8 @@ private QueryShardContext(int shardId, Predicate indexNameMatcher, Index fullyQualifiedIndex, BooleanSupplier allowExpensiveQueries, - ValuesSourceRegistry valuesSourceRegistry) { + ValuesSourceRegistry valuesSourceRegistry, + Map runtimeMappings) { super(xContentRegistry, namedWriteableRegistry, client, nowInMillis); this.shardId = shardId; this.similarityService = similarityService; @@ -167,6 +224,7 @@ private QueryShardContext(int shardId, this.fullyQualifiedIndex = fullyQualifiedIndex; this.allowExpensiveQueries = allowExpensiveQueries; this.valuesSourceRegistry = valuesSourceRegistry; + this.runtimeMappings = runtimeMappings; } private void reset() { @@ -243,7 +301,20 @@ public boolean hasMappings() { * type then the fields will be returned with a type prefix. */ public Set simpleMatchToIndexNames(String pattern) { - return mapperService.simpleMatchToFullName(pattern); + if (runtimeMappings.isEmpty()) { + return mapperService.simpleMatchToFullName(pattern); + } + if (Regex.isSimpleMatchPattern(pattern) == false) { + // no wildcards + return Collections.singleton(pattern); + } + Set matches = new HashSet<>(mapperService.simpleMatchToFullName(pattern)); + for (String name : runtimeMappings.keySet()) { + if (Regex.simpleMatch(pattern, name)) { + matches.add(name); + } + } + return matches; } /** @@ -256,14 +327,19 @@ public Set simpleMatchToIndexNames(String pattern) { * @see QueryShardContext#setMapUnmappedFieldAsString(boolean) */ public MappedFieldType getFieldType(String name) { - return failIfFieldMappingNotFound(name, mapperService.fieldType(name)); + return failIfFieldMappingNotFound(name, fieldType(name)); } /** * Returns true if the field identified by the provided name is mapped, false otherwise */ public boolean isFieldMapped(String name) { - return mapperService.fieldType(name) != null; + return fieldType(name) != null; + } + + private MappedFieldType fieldType(String name) { + MappedFieldType fieldType = runtimeMappings.get(name); + return fieldType == null ? mapperService.fieldType(name) : fieldType; } public ObjectMapper getObjectMapper(String name) { @@ -287,13 +363,22 @@ public boolean isSourceEnabled() { * Generally used to handle unmapped fields in the context of sorting. */ public MappedFieldType buildAnonymousFieldType(String type) { - final Mapper.TypeParser.ParserContext parserContext = mapperService.parserContext(); + return buildFieldType(type, "__anonymous_" + type, Collections.emptyMap(), mapperService.parserContext(), indexSettings); + } + + private static MappedFieldType buildFieldType( + String type, + String field, + Map node, + Mapper.TypeParser.ParserContext parserContext, + IndexSettings indexSettings + ) { Mapper.TypeParser typeParser = parserContext.typeParser(type); if (typeParser == null) { throw new IllegalArgumentException("No mapper found for type [" + type + "]"); } - final Mapper.Builder builder = typeParser.parse("__anonymous_" + type, Collections.emptyMap(), parserContext); - Mapper mapper = builder.build(new ContentPath(1)); + Mapper.Builder builder = typeParser.parse(field, node, parserContext); + Mapper mapper = builder.build(new ContentPath(0)); if (mapper instanceof FieldMapper) { return ((FieldMapper)mapper).fieldType(); } @@ -534,4 +619,26 @@ public BigArrays bigArrays() { return bigArrays; } + private static Map parseRuntimeMappings( + Map mappings, + MapperService mapperService, + IndexSettings indexSettings + ) { + Map runtimeMappings = new HashMap<>(); + for (Map.Entry entry : mappings.entrySet()) { + String field = entry.getKey(); + if (entry.getValue() instanceof Map == false) { + throw new ElasticsearchParseException("runtime mappings must be a map type"); + } + @SuppressWarnings("unchecked") + Map node = new HashMap<>((Map) entry.getValue()); + // Replace the type until we have native support for the runtime section + Object oldRuntimeType = node.put("runtime_type", node.remove("type")); + if (oldRuntimeType != null) { + throw new ElasticsearchParseException("use [type] in [runtime_mappings] instead of [runtime_type]"); + } + runtimeMappings.put(field, buildFieldType("runtime", field, node, mapperService.parserContext(), indexSettings)); + } + return runtimeMappings; + } } diff --git a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java index 2964604b2408d..1882f98c8ab09 100644 --- a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java +++ b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java @@ -174,8 +174,13 @@ final class DefaultSearchContext extends SearchContext { this.relativeTimeSupplier = relativeTimeSupplier; this.timeout = timeout; - queryShardContext = indexService.newQueryShardContext(request.shardId().id(), this.searcher, - request::nowInMillis, shardTarget.getClusterAlias()); + queryShardContext = indexService.newQueryShardContext( + request.shardId().id(), + this.searcher, + request::nowInMillis, + shardTarget.getClusterAlias(), + request.getRuntimeMappings() + ); queryBoost = request.indexBoost(); this.lowLevelCancellation = lowLevelCancellation; } diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java index d44445ac811d5..f976a05ba510e 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchService.java +++ b/server/src/main/java/org/elasticsearch/search/SearchService.java @@ -1178,7 +1178,7 @@ private CanMatchResponse canMatch(ShardSearchRequest request, boolean checkRefre try (canMatchSearcher) { QueryShardContext context = indexService.newQueryShardContext(request.shardId().id(), canMatchSearcher, - request::nowInMillis, request.getClusterAlias()); + request::nowInMillis, request.getClusterAlias(), request.getRuntimeMappings()); Rewriteable.rewrite(request.getRewriteable(), context, false); final boolean aliasFilterCanMatch = request.getAliasFilter() .getQueryBuilder() instanceof MatchNoneQueryBuilder == false; diff --git a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java index 461cf5456c7e6..c4b0752447b49 100644 --- a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java @@ -63,8 +63,10 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Objects; +import static java.util.Collections.emptyMap; import static org.elasticsearch.index.query.AbstractQueryBuilder.parseInnerQueryBuilder; import static org.elasticsearch.search.internal.SearchContext.TRACK_TOTAL_HITS_ACCURATE; import static org.elasticsearch.search.internal.SearchContext.TRACK_TOTAL_HITS_DISABLED; @@ -110,6 +112,7 @@ public final class SearchSourceBuilder implements Writeable, ToXContentObject, R public static final ParseField COLLAPSE = new ParseField("collapse"); public static final ParseField SLICE = new ParseField("slice"); public static final ParseField POINT_IN_TIME = new ParseField("pit"); + public static final ParseField RUNTIME_MAPPINGS_FIELD = new ParseField("runtime_mappings"); public static SearchSourceBuilder fromXContent(XContentParser parser) throws IOException { return fromXContent(parser, true); @@ -191,6 +194,8 @@ public static HighlightBuilder highlight() { private PointInTimeBuilder pointInTimeBuilder = null; + private Map runtimeMappings = emptyMap(); + /** * Constructs a new search source builder. */ @@ -251,6 +256,9 @@ public SearchSourceBuilder(StreamInput in) throws IOException { } pointInTimeBuilder = in.readOptionalWriteable(PointInTimeBuilder::new); } + if (in.getVersion().onOrAfter(Version.V_8_0_0)) { + runtimeMappings = in.readMap(); + } } @Override @@ -312,6 +320,15 @@ public void writeTo(StreamOutput out) throws IOException { } out.writeOptionalWriteable(pointInTimeBuilder); } + if (out.getVersion().onOrAfter(Version.V_8_0_0)) { + out.writeMap(runtimeMappings); + } else { + if (false == runtimeMappings.isEmpty()) { + throw new IllegalArgumentException( + "Versions before 8.0.0 don't support [runtime_mappings] and search was sent to [" + out.getVersion() + "]" + ); + } + } } /** @@ -973,6 +990,21 @@ public SearchSourceBuilder pointInTimeBuilder(PointInTimeBuilder builder) { return this; } + /** + * Mappings specified on this search request that override built in mappings. + */ + public Map runtimeMappings() { + return runtimeMappings; + } + + /** + * Specify the mappings specified on this search request that override built in mappings. + */ + public SearchSourceBuilder runtimeMappings(Map runtimeMappings) { + this.runtimeMappings = runtimeMappings == null ? emptyMap() : runtimeMappings; + return this; + } + /** * Rewrites this search source builder into its primitive form. e.g. by * rewriting the QueryBuilder. If the builder did not change the identity @@ -1059,6 +1091,7 @@ private SearchSourceBuilder shallowCopy(QueryBuilder queryBuilder, QueryBuilder rewrittenBuilder.seqNoAndPrimaryTerm = seqNoAndPrimaryTerm; rewrittenBuilder.collapse = collapse; rewrittenBuilder.pointInTimeBuilder = pointInTimeBuilder; + rewrittenBuilder.runtimeMappings = runtimeMappings; return rewrittenBuilder; } @@ -1169,6 +1202,8 @@ public void parseXContent(XContentParser parser, boolean checkTrailingTokens) th collapse = CollapseBuilder.fromXContent(parser); } else if (POINT_IN_TIME.match(currentFieldName, parser.getDeprecationHandler())) { pointInTimeBuilder = PointInTimeBuilder.fromXContent(parser); + } else if (RUNTIME_MAPPINGS_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { + runtimeMappings = parser.map(); } else { throw new ParsingException(parser.getTokenLocation(), "Unknown key for a " + token + " in [" + currentFieldName + "].", parser.getTokenLocation()); @@ -1376,6 +1411,10 @@ public XContentBuilder innerToXContent(XContentBuilder builder, Params params) t if (pointInTimeBuilder != null) { pointInTimeBuilder.toXContent(builder, params); } + if (false == runtimeMappings.isEmpty()) { + builder.field(RUNTIME_MAPPINGS_FIELD.getPreferredName(), runtimeMappings); + } + return builder; } @@ -1588,7 +1627,7 @@ public int hashCode() { return Objects.hash(aggregations, explain, fetchSourceContext, fetchFields, docValueFields, storedFieldsContext, from, highlightBuilder, indexBoosts, minScore, postQueryBuilder, queryBuilder, rescoreBuilders, scriptFields, size, sorts, searchAfterBuilder, sliceBuilder, stats, suggestBuilder, terminateAfter, timeout, trackScores, version, - seqNoAndPrimaryTerm, profile, extBuilders, collapse, trackTotalHitsUpTo, pointInTimeBuilder); + seqNoAndPrimaryTerm, profile, extBuilders, collapse, trackTotalHitsUpTo, pointInTimeBuilder, runtimeMappings); } @Override @@ -1629,7 +1668,8 @@ public boolean equals(Object obj) { && Objects.equals(extBuilders, other.extBuilders) && Objects.equals(collapse, other.collapse) && Objects.equals(trackTotalHitsUpTo, other.trackTotalHitsUpTo) - && Objects.equals(pointInTimeBuilder, other.pointInTimeBuilder); + && Objects.equals(pointInTimeBuilder, other.pointInTimeBuilder) + && Objects.equals(runtimeMappings, other.runtimeMappings); } @Override diff --git a/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java b/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java index d496a10e696c2..af2e46a65dffb 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java +++ b/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java @@ -47,8 +47,8 @@ import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.AliasFilterParsingException; import org.elasticsearch.indices.InvalidAliasNameException; -import org.elasticsearch.search.SearchSortValuesAndFormats; import org.elasticsearch.search.Scroll; +import org.elasticsearch.search.SearchSortValuesAndFormats; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.query.QuerySearchResult; import org.elasticsearch.search.sort.FieldSortBuilder; @@ -61,6 +61,7 @@ import java.util.Map; import java.util.function.Function; +import static java.util.Collections.emptyMap; import static org.elasticsearch.search.internal.SearchContext.TRACK_TOTAL_HITS_DISABLED; /** @@ -521,4 +522,8 @@ public static QueryBuilder parseAliasFilter(CheckedFunction getRuntimeMappings() { + return source == null ? emptyMap() : source.runtimeMappings(); + } } diff --git a/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java b/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java index beb4ef6d66e6f..76e77ce2c4671 100644 --- a/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java @@ -26,6 +26,7 @@ import org.elasticsearch.index.query.InnerHitBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.AbstractSearchTestCase; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.builder.SearchSourceBuilder; @@ -38,10 +39,14 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; import java.util.stream.IntStream; +import static java.util.Collections.emptyMap; +import static org.hamcrest.Matchers.equalTo; + public class ExpandSearchPhaseTests extends ESTestCase { public void testCollapseSingleHit() throws IOException { @@ -58,6 +63,7 @@ public void testCollapseSingleHit() throws IOException { AtomicBoolean executedMultiSearch = new AtomicBoolean(false); QueryBuilder originalQuery = randomBoolean() ? null : QueryBuilders.termQuery("foo", "bar"); + Map runtimeMappings = randomBoolean() ? emptyMap() : AbstractSearchTestCase.randomRuntimeMappings(); final MockSearchPhaseContext mockSearchPhaseContext = new MockSearchPhaseContext(1); String collapseValue = randomBoolean() ? null : "boom"; @@ -66,7 +72,7 @@ public void testCollapseSingleHit() throws IOException { .collapse(new CollapseBuilder("someField") .setInnerHits(IntStream.range(0, numInnerHits).mapToObj(hitNum -> new InnerHitBuilder().setName("innerHit" + hitNum)) .collect(Collectors.toList())))); - mockSearchPhaseContext.getRequest().source().query(originalQuery); + mockSearchPhaseContext.getRequest().source().query(originalQuery).runtimeMappings(runtimeMappings); mockSearchPhaseContext.searchTransport = new SearchTransportService(null, null, null) { @Override void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionListener listener) { @@ -85,7 +91,7 @@ void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionL assertThat(groupBuilder.must(), Matchers.contains(QueryBuilders.termQuery("foo", "bar"))); } assertArrayEquals(mockSearchPhaseContext.getRequest().indices(), searchRequest.indices()); - + assertThat(searchRequest.source().runtimeMappings(), equalTo(runtimeMappings)); List mSearchResponses = new ArrayList<>(numInnerHits); for (int innerHitNum = 0; innerHitNum < numInnerHits; innerHitNum++) { diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java b/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java index 3725bb7b87a77..b6ac781a015d7 100644 --- a/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java +++ b/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java @@ -59,6 +59,7 @@ import java.util.Collection; import java.util.List; +import static java.util.Collections.emptyMap; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.sameInstance; @@ -139,7 +140,7 @@ public void setup() throws Exception { writer = new IndexWriter( new ByteBuffersDirectory(), new IndexWriterConfig(new StandardAnalyzer()).setMergePolicy(new LogByteSizeMergePolicy()) ); - shardContext = indexService.newQueryShardContext(0, null, () -> 0, null); + shardContext = indexService.newQueryShardContext(0, null, () -> 0, null, emptyMap()); } protected final List refreshReader() throws Exception { diff --git a/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java b/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java index c4d63e051230d..bb21086561a9a 100644 --- a/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java @@ -50,14 +50,21 @@ import org.elasticsearch.index.fielddata.LeafFieldData; import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.fielddata.plain.AbstractLeafOrdinalsFieldData; +import org.elasticsearch.index.mapper.ContentPath; +import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.IndexFieldMapper; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.index.mapper.Mapper.TypeParser; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.NumberFieldMapper; +import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.mapper.TextFieldMapper; +import org.elasticsearch.index.mapper.TextSearchInfo; +import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.indices.IndicesModule; +import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.search.lookup.LeafDocLookup; import org.elasticsearch.search.lookup.LeafSearchLookup; import org.elasticsearch.search.lookup.SearchLookup; @@ -68,6 +75,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.function.BiFunction; import java.util.function.Supplier; @@ -183,9 +191,24 @@ public void testIndexSortedOnField() { IndexSettings indexSettings = new IndexSettings(indexMetadata, settings); QueryShardContext context = new QueryShardContext( - 0, indexSettings, BigArrays.NON_RECYCLING_INSTANCE, null, null, - null, null, null, NamedXContentRegistry.EMPTY, new NamedWriteableRegistry(Collections.emptyList()), - null, null, () -> 0L, null, null, () -> true, null); + 0, + indexSettings, + BigArrays.NON_RECYCLING_INSTANCE, + null, + null, + null, + null, + null, + NamedXContentRegistry.EMPTY, + new NamedWriteableRegistry(Collections.emptyList()), + null, + null, + () -> 0L, + null, + null, + () -> true, + null + ); assertTrue(context.indexSortedOnField("sort_field")); assertFalse(context.indexSortedOnField("second_sort_field")); @@ -289,12 +312,86 @@ public void testFielddataLookupOneFieldManyReferences() throws IOException { assertEquals(List.of(expectedFirstDoc.toString(), expectedSecondDoc.toString()), collect("field", queryShardContext)); } + public void testRuntimeFields() throws IOException { + MapperService mapperService = mockMapperService("test", List.of(new MapperPlugin() { + @Override + public Map getMappers() { + return Map.of("runtime", (name, node, parserContext) -> new Mapper.Builder(name) { + @Override + public Mapper build(ContentPath path) { + return new DummyMapper(name, new DummyMappedFieldType(name)); + } + }); + } + })); + when(mapperService.fieldType("pig")).thenReturn(new DummyMappedFieldType("pig")); + when(mapperService.simpleMatchToFullName("*")).thenReturn(Set.of("pig")); + /* + * Making these immutable here test that we don't modify them. + * Modifying them would cause all kinds of problems if two + * shards are parsed on the same node. + */ + Map runtimeMappings = Map.ofEntries( + Map.entry("cat", Map.of("type", "keyword")), + Map.entry("dog", Map.of("type", "keyword")) + ); + QueryShardContext qsc = new QueryShardContext( + 0, + mapperService.getIndexSettings(), + BigArrays.NON_RECYCLING_INSTANCE, + null, + (mappedFieldType, idxName, searchLookup) -> mappedFieldType.fielddataBuilder(idxName, searchLookup).build(null, null), + mapperService, + null, + null, + NamedXContentRegistry.EMPTY, + new NamedWriteableRegistry(List.of()), + null, + null, + () -> 0, + "test", + null, + () -> true, + null, + runtimeMappings + ); + assertTrue(qsc.isFieldMapped("cat")); + assertThat(qsc.getFieldType("cat"), instanceOf(DummyMappedFieldType.class)); + assertThat(qsc.simpleMatchToIndexNames("cat"), equalTo(Set.of("cat"))); + assertTrue(qsc.isFieldMapped("dog")); + assertThat(qsc.getFieldType("dog"), instanceOf(DummyMappedFieldType.class)); + assertThat(qsc.simpleMatchToIndexNames("dog"), equalTo(Set.of("dog"))); + assertTrue(qsc.isFieldMapped("pig")); + assertThat(qsc.getFieldType("pig"), instanceOf(DummyMappedFieldType.class)); + assertThat(qsc.simpleMatchToIndexNames("pig"), equalTo(Set.of("pig"))); + assertThat(qsc.simpleMatchToIndexNames("*"), equalTo(Set.of("cat", "dog", "pig"))); + } + public static QueryShardContext createQueryShardContext(String indexUuid, String clusterAlias) { return createQueryShardContext(indexUuid, clusterAlias, null); } - private static QueryShardContext createQueryShardContext(String indexUuid, String clusterAlias, - TriFunction runtimeDocValues) { + private static QueryShardContext createQueryShardContext( + String indexUuid, + String clusterAlias, + TriFunction runtimeDocValues + ) { + MapperService mapperService = mockMapperService(indexUuid, List.of()); + if (runtimeDocValues != null) { + when(mapperService.fieldType(any())).thenAnswer(fieldTypeInv -> { + String fieldName = (String)fieldTypeInv.getArguments()[0]; + return mockFieldType(fieldName, (leafSearchLookup, docId) -> runtimeDocValues.apply(fieldName, leafSearchLookup, docId)); + }); + } + final long nowInMillis = randomNonNegativeLong(); + return new QueryShardContext( + 0, mapperService.getIndexSettings(), BigArrays.NON_RECYCLING_INSTANCE, null, + (mappedFieldType, idxName, searchLookup) -> mappedFieldType.fielddataBuilder(idxName, searchLookup).build(null, null), + mapperService, null, null, NamedXContentRegistry.EMPTY, new NamedWriteableRegistry(Collections.emptyList()), + null, null, () -> nowInMillis, clusterAlias, null, () -> true, null); + } + + private static MapperService mockMapperService(String indexUuid, List mapperPlugins) { IndexMetadata.Builder indexMetadataBuilder = new IndexMetadata.Builder("index"); indexMetadataBuilder.settings(Settings.builder().put("index.version.created", Version.CURRENT) .put("index.number_of_shards", 1) @@ -302,34 +399,24 @@ private static QueryShardContext createQueryShardContext(String indexUuid, Strin .put(IndexMetadata.SETTING_INDEX_UUID, indexUuid) ); IndexMetadata indexMetadata = indexMetadataBuilder.build(); - IndexSettings indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY); IndexAnalyzers indexAnalyzers = new IndexAnalyzers( Collections.singletonMap("default", new NamedAnalyzer("default", AnalyzerScope.INDEX, null)), Collections.emptyMap(), Collections.emptyMap() ); + IndexSettings indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY); + MapperService mapperService = mock(MapperService.class); when(mapperService.getIndexSettings()).thenReturn(indexSettings); when(mapperService.index()).thenReturn(indexMetadata.getIndex()); when(mapperService.getIndexAnalyzers()).thenReturn(indexAnalyzers); - Map typeParserMap = IndicesModule.getMappers(Collections.emptyList()); + Map typeParserMap = IndicesModule.getMappers(mapperPlugins); Mapper.TypeParser.ParserContext parserContext = new Mapper.TypeParser.ParserContext(name -> null, typeParserMap::get, Version.CURRENT, () -> null, null, null, mapperService.getIndexAnalyzers(), mapperService.getIndexSettings(), () -> { throw new UnsupportedOperationException(); }); when(mapperService.parserContext()).thenReturn(parserContext); - if (runtimeDocValues != null) { - when(mapperService.fieldType(any())).thenAnswer(fieldTypeInv -> { - String fieldName = (String)fieldTypeInv.getArguments()[0]; - return mockFieldType(fieldName, (leafSearchLookup, docId) -> runtimeDocValues.apply(fieldName, leafSearchLookup, docId)); - }); - } - final long nowInMillis = randomNonNegativeLong(); - return new QueryShardContext( - 0, indexSettings, BigArrays.NON_RECYCLING_INSTANCE, null, - (mappedFieldType, idxName, searchLookup) -> mappedFieldType.fielddataBuilder(idxName, searchLookup).build(null, null), - mapperService, null, null, NamedXContentRegistry.EMPTY, new NamedWriteableRegistry(Collections.emptyList()), - null, null, () -> nowInMillis, clusterAlias, null, () -> true, null); + return mapperService; } private static MappedFieldType mockFieldType(String fieldName, BiFunction runtimeDocValues) { @@ -427,4 +514,45 @@ public void collect(int doc) throws IOException { } } + private static class DummyMapper extends FieldMapper { + protected DummyMapper(String simpleName, MappedFieldType mappedFieldType) { + super(simpleName, mappedFieldType, Map.of(), MultiFields.empty(), CopyTo.empty()); + } + + @Override + protected void parseCreateField(ParseContext context) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public Builder getMergeBuilder() { + throw new UnsupportedOperationException(); + } + + @Override + protected String contentType() { + throw new UnsupportedOperationException(); + } + } + + private static class DummyMappedFieldType extends MappedFieldType { + DummyMappedFieldType(String name) { + super(name, true, false, true, TextSearchInfo.SIMPLE_MATCH_ONLY, null); + } + + @Override + public ValueFetcher valueFetcher(QueryShardContext context, SearchLookup searchLookup, String format) { + throw new UnsupportedOperationException(); + } + + @Override + public String typeName() { + return "runtime"; + } + + @Override + public Query termQuery(Object value, QueryShardContext context) { + throw new UnsupportedOperationException(); + } + } } diff --git a/server/src/test/java/org/elasticsearch/index/search/MultiMatchQueryTests.java b/server/src/test/java/org/elasticsearch/index/search/MultiMatchQueryTests.java index 16bb399bde087..d633b7248e2cb 100644 --- a/server/src/test/java/org/elasticsearch/index/search/MultiMatchQueryTests.java +++ b/server/src/test/java/org/elasticsearch/index/search/MultiMatchQueryTests.java @@ -58,6 +58,7 @@ import java.util.List; import java.util.Map; +import static java.util.Collections.emptyMap; import static org.elasticsearch.index.query.QueryBuilders.multiMatchQuery; import static org.hamcrest.Matchers.equalTo; @@ -99,7 +100,7 @@ public void setup() throws IOException { public void testCrossFieldMultiMatchQuery() throws IOException { QueryShardContext queryShardContext = indexService.newQueryShardContext( - randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null); + randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null, emptyMap()); queryShardContext.setAllowUnmappedFields(true); for (float tieBreaker : new float[] {0.0f, 0.5f}) { Query parsedQuery = multiMatchQuery("banon") @@ -126,8 +127,12 @@ public void testBlendTerms() { float[] boosts = new float[] {2, 3}; Query expected = BlendedTermQuery.dismaxBlendedQuery(terms, boosts, 1.0f); Query actual = MultiMatchQuery.blendTerm( - indexService.newQueryShardContext(randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null), - new BytesRef("baz"), 1f, false, Arrays.asList(new FieldAndBoost(ft1, 2), new FieldAndBoost(ft2, 3))); + indexService.newQueryShardContext(randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null, emptyMap()), + new BytesRef("baz"), + 1f, + false, + Arrays.asList(new FieldAndBoost(ft1, 2), new FieldAndBoost(ft2, 3)) + ); assertEquals(expected, actual); } @@ -146,8 +151,12 @@ public Query termQuery(Object value, QueryShardContext context) { BlendedTermQuery.dismaxBlendedQuery(terms, boosts, 1.0f) ), 1f); Query actual = MultiMatchQuery.blendTerm( - indexService.newQueryShardContext(randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null), - new BytesRef("baz"), 1f, true, Arrays.asList(new FieldAndBoost(ft1, 2), new FieldAndBoost(ft2, 3))); + indexService.newQueryShardContext(randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null, emptyMap()), + new BytesRef("baz"), + 1f, + true, + Arrays.asList(new FieldAndBoost(ft1, 2), new FieldAndBoost(ft2, 3)) + ); assertEquals(expected, actual); } @@ -159,7 +168,7 @@ public Query termQuery(Object value, QueryShardContext context) { } }; expectThrows(IllegalArgumentException.class, () -> MultiMatchQuery.blendTerm( - indexService.newQueryShardContext(randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null), + indexService.newQueryShardContext(randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null, emptyMap()), new BytesRef("baz"), 1f, false, Arrays.asList(new FieldAndBoost(ft, 1)))); } @@ -181,14 +190,18 @@ public Query termQuery(Object value, QueryShardContext context) { expectedDisjunct1 ), 1.0f); Query actual = MultiMatchQuery.blendTerm( - indexService.newQueryShardContext(randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null), - new BytesRef("baz"), 1f, false, Arrays.asList(new FieldAndBoost(ft1, 2), new FieldAndBoost(ft2, 3))); + indexService.newQueryShardContext(randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null, emptyMap()), + new BytesRef("baz"), + 1f, + false, + Arrays.asList(new FieldAndBoost(ft1, 2), new FieldAndBoost(ft2, 3)) + ); assertEquals(expected, actual); } public void testMultiMatchCrossFieldsWithSynonyms() throws IOException { QueryShardContext queryShardContext = indexService.newQueryShardContext( - randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null); + randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null, emptyMap()); MultiMatchQuery parser = new MultiMatchQuery(queryShardContext); parser.setAnalyzer(new MockSynonymAnalyzer()); @@ -220,7 +233,7 @@ public void testMultiMatchCrossFieldsWithSynonyms() throws IOException { public void testMultiMatchCrossFieldsWithSynonymsPhrase() throws IOException { QueryShardContext queryShardContext = indexService.newQueryShardContext( - randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null); + randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null, emptyMap()); MultiMatchQuery parser = new MultiMatchQuery(queryShardContext); parser.setAnalyzer(new MockSynonymAnalyzer()); Map fieldNames = new HashMap<>(); @@ -289,7 +302,7 @@ public void testKeywordSplitQueriesOnWhitespace() throws IOException { QueryShardContext queryShardContext = indexService.newQueryShardContext( randomInt(20), null, () -> { throw new UnsupportedOperationException(); - }, null); + }, null, emptyMap()); MultiMatchQuery parser = new MultiMatchQuery(queryShardContext); Map fieldNames = new HashMap<>(); fieldNames.put("field", 1.0f); diff --git a/server/src/test/java/org/elasticsearch/index/search/NestedHelperTests.java b/server/src/test/java/org/elasticsearch/index/search/NestedHelperTests.java index f33d813ebb048..d8f7405ef3387 100644 --- a/server/src/test/java/org/elasticsearch/index/search/NestedHelperTests.java +++ b/server/src/test/java/org/elasticsearch/index/search/NestedHelperTests.java @@ -43,6 +43,8 @@ import java.io.IOException; import java.util.Collections; +import static java.util.Collections.emptyMap; + public class NestedHelperTests extends ESSingleNodeTestCase { IndexService indexService; @@ -333,7 +335,7 @@ public void testConjunction() { } public void testNested() throws IOException { - QueryShardContext context = indexService.newQueryShardContext(0, new IndexSearcher(new MultiReader()), () -> 0, null); + QueryShardContext context = indexService.newQueryShardContext(0, new IndexSearcher(new MultiReader()), () -> 0, null, emptyMap()); NestedQueryBuilder queryBuilder = new NestedQueryBuilder("nested1", new MatchAllQueryBuilder(), ScoreMode.Avg); ESToParentBlockJoinQuery query = (ESToParentBlockJoinQuery) queryBuilder.toQuery(context); diff --git a/server/src/test/java/org/elasticsearch/index/search/nested/NestedSortingTests.java b/server/src/test/java/org/elasticsearch/index/search/nested/NestedSortingTests.java index d2b8c6bab9dd8..e4b0573b06cca 100644 --- a/server/src/test/java/org/elasticsearch/index/search/nested/NestedSortingTests.java +++ b/server/src/test/java/org/elasticsearch/index/search/nested/NestedSortingTests.java @@ -73,6 +73,7 @@ import java.util.Collections; import java.util.List; +import static java.util.Collections.emptyMap; import static org.elasticsearch.index.mapper.SeqNoFieldMapper.PRIMARY_TERM_NAME; import static org.hamcrest.Matchers.equalTo; @@ -607,7 +608,7 @@ public void testMultiLevelNestedSorting() throws IOException { DirectoryReader reader = DirectoryReader.open(writer); reader = ElasticsearchDirectoryReader.wrap(reader, new ShardId(indexService.index(), 0)); IndexSearcher searcher = new IndexSearcher(reader); - QueryShardContext queryShardContext = indexService.newQueryShardContext(0, searcher, () -> 0L, null); + QueryShardContext queryShardContext = indexService.newQueryShardContext(0, searcher, () -> 0L, null, emptyMap()); FieldSortBuilder sortBuilder = new FieldSortBuilder("chapters.paragraphs.word_count"); sortBuilder.setNestedSort(new NestedSortBuilder("chapters").setNestedSort(new NestedSortBuilder("chapters.paragraphs"))); diff --git a/server/src/test/java/org/elasticsearch/search/AbstractSearchTestCase.java b/server/src/test/java/org/elasticsearch/search/AbstractSearchTestCase.java index e8488d99d8a80..8b82205c5efd2 100644 --- a/server/src/test/java/org/elasticsearch/search/AbstractSearchTestCase.java +++ b/server/src/test/java/org/elasticsearch/search/AbstractSearchTestCase.java @@ -91,7 +91,22 @@ protected SearchSourceBuilder createSearchSourceBuilder() { SuggestBuilderTests::randomSuggestBuilder, QueryRescorerBuilderTests::randomRescoreBuilder, randomExtBuilders, - CollapseBuilderTests::randomCollapseBuilder); + CollapseBuilderTests::randomCollapseBuilder, + AbstractSearchTestCase::randomRuntimeMappings); + } + + public static Map randomRuntimeMappings() { + int count = between(1, 100); + Map runtimeFields = new HashMap<>(count); + while (runtimeFields.size() < count) { + int size = between(1, 10); + Map config = new HashMap<>(size); + while (config.size() < size) { + config.put(randomAlphaOfLength(5), randomAlphaOfLength(5)); + } + runtimeFields.put(randomAlphaOfLength(5), config); + } + return runtimeFields; } protected SearchRequest createSearchRequest() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java b/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java index d71958df9b1ea..01ab3c591b41c 100644 --- a/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java +++ b/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java @@ -108,7 +108,9 @@ public void testPreProcess() throws Exception { when(indexCache.query()).thenReturn(queryCache); when(indexService.cache()).thenReturn(indexCache); QueryShardContext queryShardContext = mock(QueryShardContext.class); - when(indexService.newQueryShardContext(eq(shardId.id()), anyObject(), anyObject(), anyString())).thenReturn(queryShardContext); + when(indexService.newQueryShardContext(eq(shardId.id()), anyObject(), anyObject(), anyString(), anyObject())).thenReturn( + queryShardContext + ); MapperService mapperService = mock(MapperService.class); when(mapperService.hasNested()).thenReturn(randomBoolean()); when(indexService.mapperService()).thenReturn(mapperService); diff --git a/server/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java b/server/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java index 8ce36d4295cc2..c68f8243640e9 100644 --- a/server/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java @@ -20,11 +20,10 @@ package org.elasticsearch.search.builder; import com.fasterxml.jackson.core.JsonParseException; + +import org.elasticsearch.Version; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.io.stream.BytesStreamOutput; -import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput; -import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -93,16 +92,18 @@ private static void assertParseSearchSource(SearchSourceBuilder testBuilder, XCo } public void testSerialization() throws IOException { - SearchSourceBuilder testBuilder = createSearchSourceBuilder(); - try (BytesStreamOutput output = new BytesStreamOutput()) { - testBuilder.writeTo(output); - try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) { - SearchSourceBuilder deserializedBuilder = new SearchSourceBuilder(in); - assertEquals(deserializedBuilder, testBuilder); - assertEquals(deserializedBuilder.hashCode(), testBuilder.hashCode()); - assertNotSame(deserializedBuilder, testBuilder); - } - } + SearchSourceBuilder original = createSearchSourceBuilder(); + SearchSourceBuilder copy = copyBuilder(original); + assertEquals(copy, original); + assertEquals(copy.hashCode(), original.hashCode()); + assertNotSame(copy, original); + } + + public void testSerializingWithRuntimeFieldsBeforeSupportedThrows() { + SearchSourceBuilder original = new SearchSourceBuilder().runtimeMappings(randomRuntimeMappings()); + Version v = Version.V_8_0_0.minimumCompatibilityVersion(); + Exception e = expectThrows(IllegalArgumentException.class, () -> copyBuilder(original, v)); + assertThat(e.getMessage(), equalTo("Versions before 8.0.0 don't support [runtime_mappings] and search was sent to [" + v + "]")); } public void testShallowCopy() { @@ -118,9 +119,12 @@ public void testEqualsAndHashcode() throws IOException { EqualsHashCodeTestUtils.checkEqualsAndHashCode(createSearchSourceBuilder(), this::copyBuilder); } - //we use the streaming infra to create a copy of the builder provided as argument private SearchSourceBuilder copyBuilder(SearchSourceBuilder original) throws IOException { - return ESTestCase.copyWriteable(original, namedWriteableRegistry, SearchSourceBuilder::new); + return copyBuilder(original, Version.CURRENT); + } + + private SearchSourceBuilder copyBuilder(SearchSourceBuilder original, Version version) throws IOException { + return ESTestCase.copyWriteable(original, namedWriteableRegistry, SearchSourceBuilder::new, version); } public void testParseIncludeExclude() throws IOException { diff --git a/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java b/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java index ed95561aa9100..67aaedf7d818f 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java @@ -53,6 +53,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.function.Supplier; import static java.util.Collections.emptyMap; @@ -82,7 +83,7 @@ private RandomSearchRequestGenerator() {} * Build a random search request. * * @param randomSearchSourceBuilder builds a random {@link SearchSourceBuilder}. You can use - * {@link #randomSearchSourceBuilder(Supplier, Supplier, Supplier, Supplier, Supplier)}. + * {@link #randomSearchSourceBuilder}. */ public static SearchRequest randomSearchRequest(Supplier randomSearchSourceBuilder) { SearchRequest searchRequest = new SearchRequest(); @@ -122,7 +123,8 @@ public static SearchSourceBuilder randomSearchSourceBuilder( Supplier randomSuggestBuilder, Supplier> randomRescoreBuilder, Supplier> randomExtBuilders, - Supplier randomCollapseBuilder) { + Supplier randomCollapseBuilder, + Supplier> randomRuntimeMappings) { SearchSourceBuilder builder = new SearchSourceBuilder(); if (randomBoolean()) { builder.from(randomIntBetween(0, 10000)); @@ -378,6 +380,9 @@ public static SearchSourceBuilder randomSearchSourceBuilder( } builder.pointInTimeBuilder(pit); } + if (randomBoolean()) { + builder.runtimeMappings(randomRuntimeMappings.get()); + } return builder; } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java b/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java index 9193e90bdbaf4..a99a4f0b6b077 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java +++ b/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java @@ -63,6 +63,8 @@ import java.util.List; import java.util.Map; +import static java.util.Collections.emptyMap; + public class TestSearchContext extends SearchContext { public static final SearchShardTarget SHARD_TARGET = new SearchShardTarget("test", new ShardId("test", "test", 0), null, OriginalIndices.NONE); @@ -97,7 +99,7 @@ public TestSearchContext(BigArrays bigArrays, IndexService indexService) { this.indexService = indexService; this.fixedBitSetFilterCache = indexService.cache().bitsetFilterCache(); this.indexShard = indexService.getShardOrNull(0); - queryShardContext = indexService.newQueryShardContext(0, null, () -> 0L, null); + queryShardContext = indexService.newQueryShardContext(0, null, () -> 0L, null, emptyMap()); } public TestSearchContext(QueryShardContext queryShardContext) { diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchAction.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchAction.java index dac79f04cba67..7cd1e85366989 100644 --- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchAction.java +++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchAction.java @@ -70,6 +70,8 @@ import java.util.Map; import java.util.Set; +import static java.util.Collections.emptyMap; + /** * This is an internal action, that executes msearch requests for enrich indices in a more efficient manner. * Currently each search request inside a msearch request is executed as a separate search. If many search requests @@ -229,11 +231,20 @@ protected MultiSearchResponse shardOperation(Request request, ShardId shardId) t final IndexShard indexShard = indicesService.getShardOrNull(shardId); try (Engine.Searcher searcher = indexShard.acquireSearcher("enrich_msearch")) { final FieldsVisitor visitor = new FieldsVisitor(true); + /* + * Enrich doesn't support defining runtime fields in its + * configuration. We could add support for that if we'd + * like it but, for now at least, you can't configure any + * runtime fields so it is safe to build the context without + * any. + */ + Map runtimeFields = emptyMap(); final QueryShardContext context = indexService.newQueryShardContext( shardId.id(), searcher, () -> { throw new UnsupportedOperationException(); }, - null + null, + runtimeFields ); final MultiSearchResponse.Item[] items = new MultiSearchResponse.Item[request.multiSearchRequest.requests().size()]; for (int i = 0; i < request.multiSearchRequest.requests().size(); i++) { diff --git a/x-pack/plugin/runtime-fields/qa/build.gradle b/x-pack/plugin/runtime-fields/qa/build.gradle index 7dc01b73ed9ec..61e9797913849 100644 --- a/x-pack/plugin/runtime-fields/qa/build.gradle +++ b/x-pack/plugin/runtime-fields/qa/build.gradle @@ -1 +1,74 @@ -// Empty project so we can pick up its subproject +// Shared infratructure + +apply plugin: 'elasticsearch.build' + +dependencies { + api project(":test:framework") +} + +test.enabled = false // We don't currently have any tests for this because they are test utilities. + +subprojects { + if (project.name.startsWith('core-with-')) { + apply plugin: 'elasticsearch.yaml-rest-test' + + dependencies { + yamlRestTestImplementation xpackProject("plugin:runtime-fields:qa") + } + + restResources { + restApi { + includeXpack 'async_search', 'graph', '*_point_in_time' + } + restTests { + includeCore '*' + includeXpack 'async_search', 'graph' + } + } + + testClusters.yamlRestTest { + testDistribution = 'DEFAULT' + setting 'xpack.license.self_generated.type', 'trial' + } + + yamlRestTest { + def suites = [ + 'async_search', + 'search', + 'search.aggregation', + 'search.highlight', + 'search.inner_hits', + 'search_shards', + 'suggest', + ] + if (project.name.equals('core-with-mapped')) { + suites += [ + // These two don't support runtime fields on the request. Should they? + 'field_caps', + 'graph', + // The search request tests don't know how to support msearch for now + 'msearch', + ] + } + systemProperty 'tests.rest.suite', suites.join(',') + systemProperty 'tests.rest.blacklist', + [ + /////// TO FIX /////// + 'search.highlight/40_keyword_ignore/Plain Highligher should skip highlighting ignored keyword values', // The plain highlighter is incompatible with runtime fields. Worth fixing? + 'search/115_multiple_field_collapsing/two levels fields collapsing', // Broken. Gotta fix. + 'field_caps/30_filter/Field caps with index filter', // We don't support filtering field caps on runtime fields. What should we do? + 'search.aggregation/10_histogram/*', // runtime doesn't support sub-fields. Maybe it should? + 'search/140_pre_filter_search_shards/pre_filter_shard_size with shards that have no hit', + /////// TO FIX /////// + + /////// NOT SUPPORTED /////// + 'search.aggregation/280_rare_terms/*', // Requires an index and we won't have it + // Runtime fields don't have global ords + 'search.aggregation/20_terms/string profiler via global ordinals', + 'search.aggregation/20_terms/Global ordinals are loaded with the global_ordinals execution hint', + 'search.aggregation/170_cardinality_metric/profiler string', + /////// NOT SUPPORTED /////// + ].join(',') + } + } +} \ No newline at end of file diff --git a/x-pack/plugin/runtime-fields/qa/core-with-mapped/build.gradle b/x-pack/plugin/runtime-fields/qa/core-with-mapped/build.gradle new file mode 100644 index 0000000000000..ea347a8a55e7a --- /dev/null +++ b/x-pack/plugin/runtime-fields/qa/core-with-mapped/build.gradle @@ -0,0 +1 @@ +// Configured by parent project diff --git a/x-pack/plugin/runtime-fields/qa/core-with-mapped/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/mapped/CoreWithMappedRuntimeFieldsIT.java b/x-pack/plugin/runtime-fields/qa/core-with-mapped/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/mapped/CoreWithMappedRuntimeFieldsIT.java new file mode 100644 index 0000000000000..0c19187c088d0 --- /dev/null +++ b/x-pack/plugin/runtime-fields/qa/core-with-mapped/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/mapped/CoreWithMappedRuntimeFieldsIT.java @@ -0,0 +1,76 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.test.mapped; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; +import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; +import org.elasticsearch.test.rest.yaml.section.ApiCallSection; +import org.elasticsearch.xpack.runtimefields.test.CoreTestTranslater; + +import java.util.HashMap; +import java.util.Map; + +/** + * Runs elasticsearch's core rest tests replacing all field mappings with runtime fields + * that load from {@code _source}. Tests that configure the field in a way that are not + * supported by runtime fields are skipped. + */ +public class CoreWithMappedRuntimeFieldsIT extends ESClientYamlSuiteTestCase { + public CoreWithMappedRuntimeFieldsIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { + super(testCandidate); + } + + @ParametersFactory + public static Iterable parameters() throws Exception { + return new MappingRuntimeFieldTranslater().parameters(); + } + + private static class MappingRuntimeFieldTranslater extends CoreTestTranslater { + @Override + protected Map dynamicTemplateFor(String type) { + return dynamicTemplateToAddRuntimeFields(type); + } + + @Override + protected Suite suite(ClientYamlTestCandidate candidate) { + return new Suite(candidate) { + @Override + protected boolean modifyMappingProperties(String index, Map properties) { + Map newProperties = new HashMap<>(properties.size()); + Map> runtimeProperties = new HashMap<>(properties.size()); + if (false == runtimeifyMappingProperties(properties, newProperties, runtimeProperties)) { + return false; + } + for (Map.Entry> runtimeProperty : runtimeProperties.entrySet()) { + runtimeProperty.getValue().put("runtime_type", runtimeProperty.getValue().get("type")); + runtimeProperty.getValue().put("type", "runtime"); + newProperties.put(runtimeProperty.getKey(), runtimeProperty.getValue()); + } + properties.clear(); + properties.putAll(newProperties); + return true; + } + + @Override + protected boolean modifySearch(ApiCallSection search) { + // We don't need to modify the search request if the mappings are in the index + return true; + } + + @Override + protected boolean handleIndex(IndexRequest index) { + // We don't need to scrape anything out of the index requests. + return true; + } + }; + } + } +} diff --git a/x-pack/plugin/runtime-fields/qa/core-with-search/build.gradle b/x-pack/plugin/runtime-fields/qa/core-with-search/build.gradle new file mode 100644 index 0000000000000..ea347a8a55e7a --- /dev/null +++ b/x-pack/plugin/runtime-fields/qa/core-with-search/build.gradle @@ -0,0 +1 @@ +// Configured by parent project diff --git a/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java b/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java new file mode 100644 index 0000000000000..02092699ecc02 --- /dev/null +++ b/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java @@ -0,0 +1,225 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.test.search; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.common.regex.Regex; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; +import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; +import org.elasticsearch.test.rest.yaml.section.ApiCallSection; +import org.elasticsearch.test.rest.yaml.section.ExecutableSection; +import org.elasticsearch.xpack.runtimefields.test.CoreTestTranslater; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static java.util.Collections.unmodifiableMap; + +/** + * Runs elasticsearch's core rest tests disabling all mappings and replacing them + * with runtime fields defined on the search request that load from {@code _source}. Tests + * that configure the field in a way that are not supported by runtime fields are skipped. + */ +public class CoreTestsWithSearchRuntimeFieldsIT extends ESClientYamlSuiteTestCase { + public CoreTestsWithSearchRuntimeFieldsIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { + super(testCandidate); + } + + @ParametersFactory + public static Iterable parameters() throws Exception { + return new SearchRequestRuntimeFieldTranslater().parameters(); + } + + /** + * Translating the tests is fairly difficult here because instead of ES + * tracking the mappings we have to track them. We don't have to do it as + * well as ES, just well enough that we can decorate the search requests + * with types that make most tests "just work". + */ + private static class SearchRequestRuntimeFieldTranslater extends CoreTestTranslater { + @Override + protected Map dynamicTemplateFor(String type) { + return dynamicTemplateToDisableRuntimeCompatibleFields(type); + } + + @Override + protected Suite suite(ClientYamlTestCandidate candidate) { + return new Suite(candidate) { + private Map>> runtimeMappingsAfterSetup; + private Map> mappedFieldsAfterSetup; + private Map>> runtimeMappings; + private Map> mappedFields; + + @Override + public boolean modifySections(List executables) { + if (runtimeMappingsAfterSetup == null) { + // We're modifying the setup section + runtimeMappings = new HashMap<>(); + mappedFields = new HashMap<>(); + if (false == super.modifySections(executables)) { + return false; + } + runtimeMappingsAfterSetup = unmodifiableMap(runtimeMappings); + runtimeMappings = null; + mappedFieldsAfterSetup = unmodifiableMap(mappedFields); + mappedFields = null; + return true; + } + runtimeMappings = new HashMap<>(runtimeMappingsAfterSetup); + mappedFields = new HashMap<>(mappedFieldsAfterSetup); + return super.modifySections(executables); + } + + @Override + protected boolean modifyMappingProperties(String index, Map properties) { + Map untouchedMapping = new HashMap<>(); + Map> runtimeMapping = new HashMap<>(); + if (false == runtimeifyMappingProperties(properties, untouchedMapping, runtimeMapping)) { + return false; + } + properties.clear(); + properties.putAll(untouchedMapping); + mappedFields.put(index, untouchedMapping.keySet()); + runtimeMappings.put(index, runtimeMapping); + return true; + } + + @Override + protected boolean modifySearch(ApiCallSection search) { + if (search.getBodies().isEmpty()) { + search.addBody(new HashMap<>()); + } + for (Map body : search.getBodies()) { + Map runtimeMapping = runtimeMappings(search.getParams().get("index")); + if (runtimeMapping == null) { + return false; + } + body.put("runtime_mappings", runtimeMapping); + } + return true; + } + + private Map runtimeMappings(String index) { + if (index == null) { + return mergeMappings(new String[] { "*" }); + } + String[] patterns = Arrays.stream(index.split(",")).map(m -> m.equals("_all") ? "*" : m).toArray(String[]::new); + if (patterns.length == 0 && Regex.isSimpleMatchPattern(patterns[0])) { + return runtimeMappings.get(patterns[0]); + } + return mergeMappings(patterns); + } + + private Map mergeMappings(String[] patterns) { + Map> merged = new HashMap<>(); + for (Map.Entry>> indexEntry : runtimeMappings.entrySet()) { + if (false == Regex.simpleMatch(patterns, indexEntry.getKey())) { + continue; + } + for (Map.Entry> field : indexEntry.getValue().entrySet()) { + Map mergedConfig = merged.get(field.getKey()); + if (mergedConfig == null) { + merged.put(field.getKey(), field.getValue()); + } else if (false == mergedConfig.equals(field.getValue())) { + // The two indices have different runtime mappings for a field so we have to give up on running the test. + return null; + } + } + } + for (Map.Entry> indexEntry : mappedFields.entrySet()) { + if (false == Regex.simpleMatch(patterns, indexEntry.getKey())) { + continue; + } + for (String mappedField : indexEntry.getValue()) { + if (merged.containsKey(mappedField)) { + // We have a runtime mappings for a field *and* regular mapping. We can't make this test work so skip it. + return null; + } + } + } + return merged; + } + + @Override + protected boolean handleIndex(IndexRequest index) { + /* + * Ok! Let's reverse engineer dynamic mapping. Sort of. We're + * really just looking to figure out which of the runtime fields + * is "close enough" to what dynamic mapping would do. + */ + if (index.getPipeline() != null) { + // We can't attempt local dynamic mappings with pipelines + return false; + } + Map map = XContentHelper.convertToMap(index.source(), false, index.getContentType()).v2(); + Map> indexRuntimeMappings = runtimeMappings.computeIfAbsent( + index.index(), + i -> new HashMap<>() + ); + Set indexMappedfields = mappedFields.computeIfAbsent(index.index(), i -> Set.of()); + for (Map.Entry e : map.entrySet()) { + String name = e.getKey(); + if (indexRuntimeMappings.containsKey(name) || indexMappedfields.contains(name)) { + continue; + } + Object value = e.getValue(); + if (value == null) { + continue; + } + if (value instanceof Boolean) { + indexRuntimeMappings.put(name, runtimeFieldLoadingFromSource(name, "boolean")); + continue; + } + if (value instanceof Long || value instanceof Integer) { + indexRuntimeMappings.put(name, runtimeFieldLoadingFromSource(name, "long")); + continue; + } + if (value instanceof Double) { + indexRuntimeMappings.put(name, runtimeFieldLoadingFromSource(name, "double")); + continue; + } + if (false == value instanceof String) { + continue; + } + try { + Long.parseLong(value.toString()); + indexRuntimeMappings.put(name, runtimeFieldLoadingFromSource(name, "long")); + continue; + } catch (IllegalArgumentException iae) { + // Try the next one + } + try { + Double.parseDouble(value.toString()); + indexRuntimeMappings.put(name, runtimeFieldLoadingFromSource(name, "double")); + continue; + } catch (IllegalArgumentException iae) { + // Try the next one + } + try { + DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(value.toString()); + indexRuntimeMappings.put(name, runtimeFieldLoadingFromSource(name, "date")); + continue; + } catch (IllegalArgumentException iae) { + // Try the next one + } + // Strings are funny, the regular dynamic mapping puts them in "name.keyword" so we follow along. + indexRuntimeMappings.put(name + ".keyword", runtimeFieldLoadingFromSource(name, "keyword")); + } + return true; + } + }; + } + } +} diff --git a/x-pack/plugin/runtime-fields/qa/rest/build.gradle b/x-pack/plugin/runtime-fields/qa/rest/build.gradle deleted file mode 100644 index 34998468a35a2..0000000000000 --- a/x-pack/plugin/runtime-fields/qa/rest/build.gradle +++ /dev/null @@ -1,50 +0,0 @@ -apply plugin: 'elasticsearch.yaml-rest-test' - -restResources { - restApi { - includeXpack 'async_search', 'graph', '*_point_in_time' - } - restTests { - includeCore '*' - includeXpack 'async_search', 'graph' - } -} - -testClusters.yamlRestTest { - testDistribution = 'DEFAULT' - setting 'xpack.license.self_generated.type', 'trial' -} - -yamlRestTest { - systemProperty 'tests.rest.suite', - [ - 'async_search', - 'field_caps', - 'graph', - 'msearch', - 'search', - 'search.aggregation', - 'search.highlight', - 'search.inner_hits', - 'search_shards', - 'suggest', - ].join(',') - systemProperty 'tests.rest.blacklist', - [ - /////// TO FIX /////// - 'search.highlight/40_keyword_ignore/Plain Highligher should skip highlighting ignored keyword values', // The plain highlighter is incompatible with runtime fields. Worth fixing? - 'search/115_multiple_field_collapsing/two levels fields collapsing', // Broken. Gotta fix. - 'field_caps/30_filter/Field caps with index filter', // We don't support filtering field caps on runtime fields. What should we do? - 'search.aggregation/10_histogram/*', // runtime doesn't support sub-fields. Maybe it should? - 'search/140_pre_filter_search_shards/pre_filter_shard_size with shards that have no hit', - /////// TO FIX /////// - - /////// NOT SUPPORTED /////// - 'search.aggregation/280_rare_terms/*', // Requires an index and we won't have it - // Runtime fields don't have global ords - 'search.aggregation/20_terms/string profiler via global ordinals', - 'search.aggregation/20_terms/Global ordinals are loaded with the global_ordinals execution hint', - 'search.aggregation/170_cardinality_metric/profiler string', - /////// NOT SUPPORTED /////// - ].join(',') -} diff --git a/x-pack/plugin/runtime-fields/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/rest/CoreTestsWithRuntimeFieldsIT.java b/x-pack/plugin/runtime-fields/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/rest/CoreTestsWithRuntimeFieldsIT.java deleted file mode 100644 index 0ca9981ba317f..0000000000000 --- a/x-pack/plugin/runtime-fields/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/rest/CoreTestsWithRuntimeFieldsIT.java +++ /dev/null @@ -1,259 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -package org.elasticsearch.xpack.runtimefields.rest; - -import com.carrotsearch.randomizedtesting.annotations.Name; -import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import org.elasticsearch.common.xcontent.XContentLocation; -import org.elasticsearch.index.mapper.BooleanFieldMapper; -import org.elasticsearch.index.mapper.DateFieldMapper; -import org.elasticsearch.index.mapper.IpFieldMapper; -import org.elasticsearch.index.mapper.KeywordFieldMapper; -import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; -import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; -import org.elasticsearch.test.rest.yaml.ClientYamlTestExecutionContext; -import org.elasticsearch.test.rest.yaml.ClientYamlTestResponse; -import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; -import org.elasticsearch.test.rest.yaml.section.ClientYamlTestSection; -import org.elasticsearch.test.rest.yaml.section.ClientYamlTestSuite; -import org.elasticsearch.test.rest.yaml.section.DoSection; -import org.elasticsearch.test.rest.yaml.section.ExecutableSection; -import org.elasticsearch.test.rest.yaml.section.SetupSection; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; - -import static org.hamcrest.Matchers.equalTo; - -public class CoreTestsWithRuntimeFieldsIT extends ESClientYamlSuiteTestCase { - public CoreTestsWithRuntimeFieldsIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { - super(testCandidate); - } - - /** - * Builds test parameters similarly to {@link ESClientYamlSuiteTestCase#createParameters()}, - * replacing the body of index creation commands so that fields are {@code runtime}s - * that load from {@code source} instead of their original type. Test configurations that - * do are not modified to contain runtime fields are not returned as they are tested - * elsewhere. - */ - @ParametersFactory - public static Iterable parameters() throws Exception { - Map suites = new HashMap<>(); - List result = new ArrayList<>(); - for (Object[] orig : ESClientYamlSuiteTestCase.createParameters()) { - assert orig.length == 1; - ClientYamlTestCandidate candidate = (ClientYamlTestCandidate) orig[0]; - ClientYamlTestSuite suite = suites.computeIfAbsent(candidate.getTestPath(), k -> modifiedSuite(candidate)); - if (suite == null) { - // The setup section contains an unsupported option - continue; - } - if (false == modifySection(candidate.getTestSection().getExecutableSections())) { - // The test section contains an unsupported option - continue; - } - ClientYamlTestSection modified = new ClientYamlTestSection( - candidate.getTestSection().getLocation(), - candidate.getTestSection().getName(), - candidate.getTestSection().getSkipSection(), - candidate.getTestSection().getExecutableSections() - ); - result.add(new Object[] { new ClientYamlTestCandidate(suite, modified) }); - } - return result; - } - - /** - * Modify the setup section to setup a dynamic template that replaces - * field configurations with scripts that load from source - * and replaces field configurations in {@code incides.create} - * with scripts that load from source. - */ - private static ClientYamlTestSuite modifiedSuite(ClientYamlTestCandidate candidate) { - if (false == modifySection(candidate.getSetupSection().getExecutableSections())) { - return null; - } - List setup = new ArrayList<>(candidate.getSetupSection().getExecutableSections().size() + 1); - setup.add(ADD_TEMPLATE); - setup.addAll(candidate.getSetupSection().getExecutableSections()); - return new ClientYamlTestSuite( - candidate.getApi(), - candidate.getName(), - new SetupSection(candidate.getSetupSection().getSkipSection(), setup), - candidate.getTeardownSection(), - List.of() - ); - } - - /** - * Replace field configuration in {@code indices.create} with scripts - * that load from the source. - */ - private static boolean modifySection(List executables) { - for (ExecutableSection section : executables) { - if (false == (section instanceof DoSection)) { - continue; - } - DoSection doSection = (DoSection) section; - if (false == doSection.getApiCallSection().getApi().equals("indices.create")) { - continue; - } - for (Map body : doSection.getApiCallSection().getBodies()) { - Object settings = body.get("settings"); - if (settings instanceof Map && ((Map) settings).containsKey("sort.field")) { - /* - * You can't sort the index on a runtime_keyword and it is - * hard to figure out if the sort was a runtime_keyword so - * let's just skip this test. - */ - continue; - } - Object mappings = body.get("mappings"); - if (false == (mappings instanceof Map)) { - continue; - } - Object properties = ((Map) mappings).get("properties"); - if (false == (properties instanceof Map)) { - continue; - } - for (Map.Entry property : ((Map) properties).entrySet()) { - if (false == property.getValue() instanceof Map) { - continue; - } - @SuppressWarnings("unchecked") - Map propertyMap = (Map) property.getValue(); - String name = property.getKey().toString(); - String type = Objects.toString(propertyMap.get("type")); - if ("nested".equals(type)) { - // Our loading scripts can't be made to manage nested fields so we have to skip those tests. - return false; - } - if ("false".equals(Objects.toString(propertyMap.get("doc_values")))) { - // If doc_values is false we can't emulate with scripts. `null` and `true` are fine. - continue; - } - if ("false".equals(Objects.toString(propertyMap.get("index")))) { - // If index is false we can't emulate with scripts - continue; - } - if ("true".equals(Objects.toString(propertyMap.get("store")))) { - // If store is true we can't emulate with scripts - continue; - } - if (propertyMap.containsKey("ignore_above")) { - // Scripts don't support ignore_above so we skip those fields - continue; - } - if (propertyMap.containsKey("ignore_malformed")) { - // Our source reading script doesn't emulate ignore_malformed - continue; - } - String toLoad = painlessToLoadFromSource(name, type); - if (toLoad == null) { - continue; - } - propertyMap.put("type", "runtime"); - propertyMap.put("runtime_type", type); - propertyMap.put("script", toLoad); - propertyMap.remove("store"); - propertyMap.remove("index"); - propertyMap.remove("doc_values"); - } - } - } - return true; - } - - private static String painlessToLoadFromSource(String name, String type) { - String emit = PAINLESS_TO_EMIT.get(type); - if (emit == null) { - return null; - } - StringBuilder b = new StringBuilder(); - b.append("def v = params._source['").append(name).append("'];\n"); - b.append("if (v instanceof Iterable) {\n"); - b.append(" for (def vv : ((Iterable) v)) {\n"); - b.append(" if (vv != null) {\n"); - b.append(" def value = vv;\n"); - b.append(" ").append(emit).append("\n"); - b.append(" }\n"); - b.append(" }\n"); - b.append("} else {\n"); - b.append(" if (v != null) {\n"); - b.append(" def value = v;\n"); - b.append(" ").append(emit).append("\n"); - b.append(" }\n"); - b.append("}\n"); - return b.toString(); - } - - private static final Map PAINLESS_TO_EMIT = Map.ofEntries( - Map.entry(BooleanFieldMapper.CONTENT_TYPE, "emit(Boolean.parseBoolean(value.toString()));"), - Map.entry(DateFieldMapper.CONTENT_TYPE, "emit(parse(value.toString()));"), - Map.entry( - NumberType.DOUBLE.typeName(), - "emit(value instanceof Number ? ((Number) value).doubleValue() : Double.parseDouble(value.toString()));" - ), - Map.entry(KeywordFieldMapper.CONTENT_TYPE, "emit(value.toString());"), - Map.entry(IpFieldMapper.CONTENT_TYPE, "emit(value.toString());"), - Map.entry( - NumberType.LONG.typeName(), - "emit(value instanceof Number ? ((Number) value).longValue() : Long.parseLong(value.toString()));" - ) - ); - - private static final ExecutableSection ADD_TEMPLATE = new ExecutableSection() { - @Override - public XContentLocation getLocation() { - return new XContentLocation(-1, -1); - } - - @Override - public void execute(ClientYamlTestExecutionContext executionContext) throws IOException { - Map params = Map.of("name", "convert_to_source_only", "create", "true"); - List> dynamicTemplates = new ArrayList<>(); - for (String type : PAINLESS_TO_EMIT.keySet()) { - if (type.equals("ip")) { - // There isn't a dynamic template to pick up ips. They'll just look like strings. - continue; - } - Map mapping = Map.ofEntries( - Map.entry("type", "runtime"), - Map.entry("runtime_type", type), - Map.entry("script", painlessToLoadFromSource("{name}", type)) - ); - if (type.contentEquals("keyword")) { - /* - * For "string"-type dynamic mappings emulate our default - * behavior with a top level text field and a `.keyword` - * multi-field. But instead of the default, use a runtime - * field for the multi-field. - */ - mapping = Map.of("type", "text", "fields", Map.of("keyword", mapping)); - dynamicTemplates.add(Map.of(type, Map.of("match_mapping_type", "string", "mapping", mapping))); - } else { - dynamicTemplates.add(Map.of(type, Map.of("match_mapping_type", type, "mapping", mapping))); - } - } - List> bodies = List.of( - Map.ofEntries( - Map.entry("index_patterns", "*"), - Map.entry("priority", Integer.MAX_VALUE - 1), - Map.entry("template", Map.of("settings", Map.of(), "mappings", Map.of("dynamic_templates", dynamicTemplates))) - ) - ); - ClientYamlTestResponse response = executionContext.callApi("indices.put_index_template", params, bodies, Map.of()); - assertThat(response.getStatusCode(), equalTo(200)); - // There are probably some warning about overlapping templates. Ignore them. - } - }; -} diff --git a/x-pack/plugin/runtime-fields/qa/src/main/java/org/elasticsearch/xpack/runtimefields/test/CoreTestTranslater.java b/x-pack/plugin/runtime-fields/qa/src/main/java/org/elasticsearch/xpack/runtimefields/test/CoreTestTranslater.java new file mode 100644 index 0000000000000..bc03972da2000 --- /dev/null +++ b/x-pack/plugin/runtime-fields/qa/src/main/java/org/elasticsearch/xpack/runtimefields/test/CoreTestTranslater.java @@ -0,0 +1,407 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.runtimefields.test; + +import org.elasticsearch.action.bulk.BulkRequestParser; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentLocation; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.index.mapper.BooleanFieldMapper; +import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.index.mapper.IpFieldMapper; +import org.elasticsearch.index.mapper.KeywordFieldMapper; +import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; +import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; +import org.elasticsearch.test.rest.yaml.ClientYamlTestExecutionContext; +import org.elasticsearch.test.rest.yaml.ClientYamlTestResponse; +import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; +import org.elasticsearch.test.rest.yaml.section.ApiCallSection; +import org.elasticsearch.test.rest.yaml.section.ClientYamlTestSection; +import org.elasticsearch.test.rest.yaml.section.ClientYamlTestSuite; +import org.elasticsearch.test.rest.yaml.section.DoSection; +import org.elasticsearch.test.rest.yaml.section.ExecutableSection; +import org.elasticsearch.test.rest.yaml.section.SetupSection; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import static org.hamcrest.Matchers.equalTo; +import static org.junit.Assert.assertThat; + +/** + * Builds test parameters similarly to {@link ESClientYamlSuiteTestCase#createParameters()}, + * replacing all fields with runtime fields that load from {@code _source} if possible. Tests + * that configure the field in a way that are not supported by runtime fields are skipped. + */ +public abstract class CoreTestTranslater { + public Iterable parameters() throws Exception { + Map suites = new HashMap<>(); + List result = new ArrayList<>(); + for (Object[] orig : ESClientYamlSuiteTestCase.createParameters()) { + assert orig.length == 1; + ClientYamlTestCandidate candidate = (ClientYamlTestCandidate) orig[0]; + Suite suite = suites.computeIfAbsent(candidate.getSuitePath(), k -> suite(candidate)); + if (suite.modified == null) { + // The setup section contains an unsupported option + continue; + } + if (false == suite.modifySections(candidate.getTestSection().getExecutableSections())) { + // The test section contains an unsupported option + continue; + } + ClientYamlTestSection modified = new ClientYamlTestSection( + candidate.getTestSection().getLocation(), + candidate.getTestSection().getName(), + candidate.getTestSection().getSkipSection(), + candidate.getTestSection().getExecutableSections() + ); + result.add(new Object[] { new ClientYamlTestCandidate(suite.modified, modified) }); + } + return result; + } + + protected abstract Suite suite(ClientYamlTestCandidate candidate); + + private static String painlessToLoadFromSource(String name, String type) { + String emit = PAINLESS_TO_EMIT.get(type); + if (emit == null) { + return null; + } + StringBuilder b = new StringBuilder(); + b.append("def v = params._source['").append(name).append("'];\n"); + b.append("if (v instanceof Iterable) {\n"); + b.append(" for (def vv : ((Iterable) v)) {\n"); + b.append(" if (vv != null) {\n"); + b.append(" def value = vv;\n"); + b.append(" ").append(emit).append("\n"); + b.append(" }\n"); + b.append(" }\n"); + b.append("} else {\n"); + b.append(" if (v != null) {\n"); + b.append(" def value = v;\n"); + b.append(" ").append(emit).append("\n"); + b.append(" }\n"); + b.append("}\n"); + return b.toString(); + } + + private static final Map PAINLESS_TO_EMIT = Map.ofEntries( + Map.entry(BooleanFieldMapper.CONTENT_TYPE, "emit(Boolean.parseBoolean(value.toString()));"), + Map.entry(DateFieldMapper.CONTENT_TYPE, "emit(parse(value.toString()));"), + Map.entry( + NumberType.DOUBLE.typeName(), + "emit(value instanceof Number ? ((Number) value).doubleValue() : Double.parseDouble(value.toString()));" + ), + Map.entry(KeywordFieldMapper.CONTENT_TYPE, "emit(value.toString());"), + Map.entry(IpFieldMapper.CONTENT_TYPE, "emit(value.toString());"), + Map.entry( + NumberType.LONG.typeName(), + "emit(value instanceof Number ? ((Number) value).longValue() : Long.parseLong(value.toString()));" + ) + ); + + protected abstract Map dynamicTemplateFor(String type); + + protected static Map dynamicTemplateToDisableRuntimeCompatibleFields(String type) { + return Map.of("type", type, "index", false, "doc_values", false); + } + + protected static Map dynamicTemplateToAddRuntimeFields(String type) { + return Map.ofEntries( + Map.entry("type", "runtime"), + Map.entry("runtime_type", type), + Map.entry("script", painlessToLoadFromSource("{name}", type)) + ); + } + + protected static Map runtimeFieldLoadingFromSource(String name, String type) { + return Map.of("type", type, "script", painlessToLoadFromSource(name, type)); + } + + private ExecutableSection addIndexTemplate() { + return new ExecutableSection() { + @Override + public XContentLocation getLocation() { + return new XContentLocation(-1, -1); + } + + @Override + public void execute(ClientYamlTestExecutionContext executionContext) throws IOException { + Map params = Map.of("name", "hack_dynamic_mappings", "create", "true"); + List> dynamicTemplates = new ArrayList<>(); + for (String type : PAINLESS_TO_EMIT.keySet()) { + if (type.equals("ip")) { + // There isn't a dynamic template to pick up ips. They'll just look like strings. + continue; + } + Map mapping = dynamicTemplateFor(type); + if (type.equals("keyword")) { + /* + * For "string"-type dynamic mappings emulate our default + * behavior with a top level text field and a `.keyword` + * multi-field. In our case we disable the keyword field + * and substitute it with an enabled one on the search + * request. + */ + mapping = Map.of("type", "text", "fields", Map.of("keyword", mapping)); + dynamicTemplates.add(Map.of(type, Map.of("match_mapping_type", "string", "mapping", mapping))); + } else { + dynamicTemplates.add(Map.of(type, Map.of("match_mapping_type", type, "mapping", mapping))); + } + } + Map indexTemplate = Map.of("settings", Map.of(), "mappings", Map.of("dynamic_templates", dynamicTemplates)); + List> bodies = List.of( + Map.ofEntries( + Map.entry("index_patterns", "*"), + Map.entry("priority", Integer.MAX_VALUE - 1), + Map.entry("template", indexTemplate) + ) + ); + ClientYamlTestResponse response = executionContext.callApi("indices.put_index_template", params, bodies, Map.of()); + assertThat(response.getStatusCode(), equalTo(200)); + // There are probably some warning about overlapping templates. Ignore them. + } + }; + } + + /** + * A modified suite. + */ + protected abstract class Suite { + private final ClientYamlTestSuite modified; + + public Suite(ClientYamlTestCandidate candidate) { + if (false == modifySections(candidate.getSetupSection().getExecutableSections())) { + modified = null; + return; + } + /* + * Modify the setup section to rewrite and create index commands and + * to add a dynamic template that sets up any dynamic indices how we + * expect them. + */ + List setup = new ArrayList<>(candidate.getSetupSection().getExecutableSections().size() + 1); + setup.add(addIndexTemplate()); + setup.addAll(candidate.getSetupSection().getExecutableSections()); + modified = new ClientYamlTestSuite( + candidate.getApi(), + candidate.getName(), + new SetupSection(candidate.getSetupSection().getSkipSection(), setup), + candidate.getTeardownSection(), + List.of() + ); + } + + /** + * Replace field configuration in {@code indices.create} with scripts + * that load from the source. + * + * @return true if the section is appropriate for testing with runtime fields + */ + public boolean modifySections(List executables) { + for (ExecutableSection section : executables) { + if (false == (section instanceof DoSection)) { + continue; + } + DoSection doSection = (DoSection) section; + String api = doSection.getApiCallSection().getApi(); + switch (api) { + case "indices.create": + if (false == modifyCreateIndex(doSection.getApiCallSection())) { + return false; + } + break; + case "search": + case "async_search.submit": + if (false == modifySearch(doSection.getApiCallSection())) { + return false; + } + break; + case "bulk": + if (false == handleBulk(doSection.getApiCallSection())) { + return false; + } + break; + case "index": + if (false == handleIndex(doSection.getApiCallSection())) { + return false; + } + break; + default: + continue; + } + } + return true; + } + + /** + * Modify a test search request. + */ + protected abstract boolean modifySearch(ApiCallSection search); + + private boolean modifyCreateIndex(ApiCallSection createIndex) { + String index = createIndex.getParams().get("index"); + for (Map body : createIndex.getBodies()) { + Object settings = body.get("settings"); + if (settings instanceof Map && ((Map) settings).containsKey("sort.field")) { + /* + * You can't sort the index on a runtime_keyword and it is + * hard to figure out if the sort was a runtime_keyword so + * let's just skip this test. + */ + continue; + } + Object mapping = body.get("mappings"); + if (false == (mapping instanceof Map)) { + continue; + } + Object properties = ((Map) mapping).get("properties"); + if (false == (properties instanceof Map)) { + continue; + } + @SuppressWarnings("unchecked") + Map propertiesMap = (Map) properties; + if (false == modifyMappingProperties(index, propertiesMap)) { + return false; + } + } + return true; + } + + /** + * Modify the mapping defined in the test. + */ + protected abstract boolean modifyMappingProperties(String index, Map properties); + + /** + * Modify the provided map in place, translating all fields into + * runtime fields that load from source. + * @return true if this mapping supports runtime fields, false otherwise + */ + protected final boolean runtimeifyMappingProperties( + Map properties, + Map untouchedProperties, + Map> runtimeProperties + ) { + for (Map.Entry property : properties.entrySet()) { + if (false == property.getValue() instanceof Map) { + untouchedProperties.put(property.getKey(), property.getValue()); + continue; + } + @SuppressWarnings("unchecked") + Map propertyMap = (Map) property.getValue(); + String name = property.getKey().toString(); + String type = Objects.toString(propertyMap.get("type")); + if ("nested".equals(type)) { + // Our loading scripts can't be made to manage nested fields so we have to skip those tests. + return false; + } + if ("false".equals(Objects.toString(propertyMap.get("doc_values")))) { + // If doc_values is false we can't emulate with scripts. So we keep the old definition. `null` and `true` are fine. + untouchedProperties.put(property.getKey(), property.getValue()); + continue; + } + if ("false".equals(Objects.toString(propertyMap.get("index")))) { + // If index is false we can't emulate with scripts + untouchedProperties.put(property.getKey(), property.getValue()); + continue; + } + if ("true".equals(Objects.toString(propertyMap.get("store")))) { + // If store is true we can't emulate with scripts + untouchedProperties.put(property.getKey(), property.getValue()); + continue; + } + if (propertyMap.containsKey("ignore_above")) { + // Scripts don't support ignore_above so we skip those fields + untouchedProperties.put(property.getKey(), property.getValue()); + continue; + } + if (propertyMap.containsKey("ignore_malformed")) { + // Our source reading script doesn't emulate ignore_malformed + untouchedProperties.put(property.getKey(), property.getValue()); + continue; + } + String toLoad = painlessToLoadFromSource(name, type); + if (toLoad == null) { + untouchedProperties.put(property.getKey(), property.getValue()); + continue; + } + Map runtimeConfig = new HashMap<>(propertyMap); + runtimeConfig.put("script", toLoad); + runtimeConfig.remove("store"); + runtimeConfig.remove("index"); + runtimeConfig.remove("doc_values"); + runtimeProperties.put(property.getKey(), runtimeConfig); + } + /* + * Its tempting to return false here if we didn't make any runtime + * fields, skipping the test. But that would cause us to skip any + * test uses dynamic mappings. Disaster! Instead we use a dynamic + * template to make the dynamic mappings into runtime fields too. + * The downside is that we can run tests that don't use runtime + * fields at all. That's unfortunate, but its ok. + */ + return true; + } + + private boolean handleBulk(ApiCallSection bulk) { + String defaultIndex = bulk.getParams().get("index"); + String defaultRouting = bulk.getParams().get("routing"); + String defaultPipeline = bulk.getParams().get("pipeline"); + BytesStreamOutput bos = new BytesStreamOutput(); + try { + for (Map body : bulk.getBodies()) { + try (XContentBuilder b = new XContentBuilder(JsonXContent.jsonXContent, bos)) { + b.map(body); + } + bos.write(JsonXContent.jsonXContent.streamSeparator()); + } + List indexRequests = new ArrayList<>(); + new BulkRequestParser(false).parse( + bos.bytes(), + defaultIndex, + defaultRouting, + null, + defaultPipeline, + null, + true, + XContentType.JSON, + (index, type) -> indexRequests.add(index), + u -> {}, + d -> {} + ); + for (IndexRequest index : indexRequests) { + if (false == handleIndex(index)) { + return false; + } + } + } catch (IOException e) { + throw new AssertionError(e); + } + return true; + } + + private boolean handleIndex(ApiCallSection indexRequest) { + String index = indexRequest.getParams().get("index"); + String pipeline = indexRequest.getParams().get("pipeline"); + assert indexRequest.getBodies().size() == 1; + try { + return handleIndex(new IndexRequest(index).setPipeline(pipeline).source(indexRequest.getBodies().get(0))); + } catch (IOException e) { + throw new AssertionError(e); + } + } + + protected abstract boolean handleIndex(IndexRequest index) throws IOException; + } +} diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java index 870fb9faf3ed8..8cd10494e0c11 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java @@ -285,6 +285,7 @@ import java.util.stream.Collectors; import static java.util.Collections.emptyList; +import static java.util.Collections.emptyMap; import static java.util.Collections.singletonList; import static org.elasticsearch.xpack.core.XPackSettings.API_KEY_SERVICE_ENABLED_SETTING; import static org.elasticsearch.xpack.core.XPackSettings.HTTP_SSL_ENABLED; @@ -722,7 +723,10 @@ public void onIndexModule(IndexModule module) { () -> { throw new IllegalArgumentException("permission filters are not allowed to use the current timestamp"); - }, null), + }, + null, + // Don't use runtime mappings in the security query + emptyMap()), dlsBitsetCache.get(), securityContext.get(), getLicenseState(), diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_keyword.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_keyword.yml index 840a0906dc1db..733c2f0b50190 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_keyword.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_keyword.yml @@ -360,3 +360,110 @@ setup: - match: { aggregations.to-users.users.hits.hits.2._index: test } - match: { aggregations.to-users.users.hits.hits.2._nested.field: users } - match: { aggregations.to-users.users.hits.hits.2._nested.offset: 1 } + +--- +"fetch defined on search request": + - do: + search: + index: sensor + body: + runtime_mappings: + voltage_rating: + type: keyword + script: | + double v = doc['voltage'].value; + if (v < 4.8) { + emit('low'); + } else if (v > 5.2) { + emit('high'); + } else { + emit('ok'); + } + fields: [voltage_rating] + sort: timestamp + - match: {hits.total.value: 6} + - match: {hits.hits.0._source.voltage: 4.0} + - match: {hits.hits.0.fields.voltage_rating: [low]} + +--- +"match defined on search request": + - do: + search: + index: sensor + body: + runtime_mappings: + voltage.rating: + type: keyword + script: | + double v = doc['voltage'].value; + if (v < 4.8) { + emit('low'); + } else if (v > 5.2) { + emit('high'); + } else { + emit('ok'); + } + query: + match: + voltage.rating: ok + sort: timestamp + - match: {hits.total.value: 2} + - match: {hits.hits.0._source.voltage: 5.1} + +--- +"search glob defined on search request": + - do: + search: + index: sensor + body: + runtime_mappings: + voltage.rating: + type: keyword + script: | + double v = doc['voltage'].value; + if (v < 4.8) { + emit('low'); + } else if (v > 5.2) { + emit('high'); + } else { + emit('ok'); + } + query: + simple_query_string: + fields: [voltage.*] + query: ok + sort: timestamp + - match: {hits.total.value: 2} + - match: {hits.hits.0._source.voltage: 5.1} + + +--- +"replace object field on search request": + - do: + bulk: + index: student + refresh: true + body: | + {"index":{}} + {"name": {"first": "Andrew", "last": "Wiggin"}} + {"index":{}} + {"name": {"first": "Julian", "last": "Delphiki", "suffix": "II"}} + + - do: + search: + index: student + body: + runtime_mappings: + name.first: + type: keyword + script: | + if ('Wiggin'.equals(doc['name.last.keyword'].value)) { + emit('Ender'); + } else if ('Delphiki'.equals(doc['name.last.keyword'].value)) { + emit('Bean'); + } + query: + match: + name.first: Bean + - match: {hits.total.value: 1} + - match: {hits.hits.0._source.name.first: Julian}