From 706da74631d905f928fa92d4a633356307f49871 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 29 Oct 2020 14:16:17 -0400 Subject: [PATCH 01/36] Add `runtime_mappings` to search request This adds a way to specify the `runtime_mappings` on a search request which are always "runtime" fields. It looks like: ``` curl -XDELETE -uelastic:password -HContent-Type:application/json localhost:9200/test curl -XPOST -uelastic:password -HContent-Type:application/json 'localhost:9200/test/_bulk?pretty&refresh' -d' {"index": {}} {"animal": "cat", "sound": "meow"} {"index": {}} {"animal": "dog", "sound": "woof"} {"index": {}} {"animal": "snake", "sound": "hisssssssssssssssss"} ' curl -XPOST -uelastic:password -HContent-Type:application/json localhost:9200/test/_search?pretty -d' { "runtime_mappings": { "animal.upper": { "type": "keyword", "script": "for (String s : doc[\"animal.keyword\"]) {emit(s.toUpperCase())}" } }, "query": { "match": { "animal.upper": "DOG" } } }' ``` --- .../common/DisableGraphQueryTests.java | 3 +- .../action/PainlessExecuteAction.java | 3 +- .../painless/NeedsScoreTests.java | 4 +- .../PercolatorFieldMapperTests.java | 3 +- .../PercolatorQuerySearchTests.java | 3 +- .../TransportSimulateIndexTemplateAction.java | 3 +- .../action/search/SearchRequest.java | 1 + .../metadata/MetadataCreateIndexService.java | 7 +- .../metadata/MetadataIndexAliasesService.java | 3 +- .../MetadataIndexTemplateService.java | 3 +- .../org/elasticsearch/index/IndexService.java | 12 +- .../index/query/QueryShardContext.java | 116 ++++++++++++++++-- .../search/DefaultSearchContext.java | 2 +- .../elasticsearch/search/SearchService.java | 2 +- .../search/builder/SearchSourceBuilder.java | 37 +++++- .../search/internal/ShardSearchRequest.java | 7 +- .../fielddata/AbstractFieldDataTestCase.java | 3 +- .../index/search/MultiMatchQueryTests.java | 35 ++++-- .../index/search/NestedHelperTests.java | 4 +- .../search/nested/NestedSortingTests.java | 3 +- .../search/DefaultSearchContextTests.java | 4 +- .../elasticsearch/test/TestSearchContext.java | 4 +- .../action/EnrichShardMultiSearchAction.java | 5 +- .../xpack/security/Security.java | 6 +- 24 files changed, 225 insertions(+), 48 deletions(-) diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/DisableGraphQueryTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/DisableGraphQueryTests.java index d1792e94f7331..c7ac35d3febce 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/DisableGraphQueryTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/DisableGraphQueryTests.java @@ -46,6 +46,7 @@ import java.util.Collection; import java.util.Collections; +import static java.util.Collections.emptyMap; import static org.hamcrest.Matchers.equalTo; /** @@ -84,7 +85,7 @@ public void setup() { indexService = createIndex("test", settings, "t", "text_shingle", "type=text,analyzer=text_shingle", "text_shingle_unigram", "type=text,analyzer=text_shingle_unigram"); - shardContext = indexService.newQueryShardContext(0, null, () -> 0L, null); + shardContext = indexService.newQueryShardContext(0, null, () -> 0L, null, emptyMap()); // parsed queries for "text_shingle_unigram:(foo bar baz)" with query parsers // that ignores position length attribute diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java index 21059f2112457..1810a1eff384a 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java @@ -85,6 +85,7 @@ import java.util.Map; import java.util.Objects; +import static java.util.Collections.emptyMap; import static org.elasticsearch.action.ValidateActions.addValidationError; import static org.elasticsearch.rest.RestRequest.Method.GET; import static org.elasticsearch.rest.RestRequest.Method.POST; @@ -555,7 +556,7 @@ private static Response prepareRamIndex(Request request, searcher.setQueryCache(null); final long absoluteStartMillis = System.currentTimeMillis(); QueryShardContext context = - indexService.newQueryShardContext(0, searcher, () -> absoluteStartMillis, null); + indexService.newQueryShardContext(0, searcher, () -> absoluteStartMillis, null, emptyMap()); return handler.apply(context, indexReader.leaves().get(0)); } } diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java index e9a6ca60509e3..ca885cdfdff6a 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java @@ -32,6 +32,8 @@ import java.util.List; import java.util.Map; +import static java.util.Collections.emptyMap; + /** * Test that needsScores() is reported correctly depending on whether _score is used */ @@ -45,7 +47,7 @@ public void testNeedsScores() { contexts.put(NumberSortScript.CONTEXT, Whitelist.BASE_WHITELISTS); PainlessScriptEngine service = new PainlessScriptEngine(Settings.EMPTY, contexts); - QueryShardContext shardContext = index.newQueryShardContext(0, null, () -> 0, null); + QueryShardContext shardContext = index.newQueryShardContext(0, null, () -> 0, null, emptyMap()); NumberSortScript.Factory factory = service.compile(null, "1.2", NumberSortScript.CONTEXT, Collections.emptyMap()); NumberSortScript.LeafFactory ss = factory.newFactory(Collections.emptyMap(), shardContext.lookup()); diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java index 19f12d6130d00..1d59ab64b8231 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java @@ -103,6 +103,7 @@ import java.util.function.Function; import java.util.stream.Collectors; +import static java.util.Collections.emptyMap; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.QueryBuilders.boolQuery; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; @@ -515,7 +516,7 @@ public void testQueryWithRewrite() throws Exception { QueryShardContext shardContext = indexService.newQueryShardContext( randomInt(20), null, () -> { throw new UnsupportedOperationException(); - }, null); + }, null, emptyMap()); PlainActionFuture future = new PlainActionFuture<>(); Rewriteable.rewriteAndFetch(queryBuilder, shardContext, future); assertQueryBuilder(qbSource, future.get()); diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchTests.java index e91a8cf1783e7..ff5d6dc7c4b7b 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchTests.java @@ -51,6 +51,7 @@ import java.util.Map; import java.util.function.Function; +import static java.util.Collections.emptyMap; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.QueryBuilders.boolQuery; import static org.elasticsearch.index.query.QueryBuilders.matchQuery; @@ -258,7 +259,7 @@ public void testRangeQueriesWithNow() throws Exception { try (Engine.Searcher searcher = indexService.getShard(0).acquireSearcher("test")) { long[] currentTime = new long[] {System.currentTimeMillis()}; QueryShardContext queryShardContext = - indexService.newQueryShardContext(0, searcher, () -> currentTime[0], null); + indexService.newQueryShardContext(0, searcher, () -> currentTime[0], null, emptyMap()); BytesReference source = BytesReference.bytes(jsonBuilder().startObject() .field("field1", "value") diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java index 4cc978b166129..db129a9479934 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java @@ -56,6 +56,7 @@ import java.util.function.Function; import java.util.stream.Collectors; +import static java.util.Collections.emptyMap; import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.findConflictingV1Templates; import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.findConflictingV2Templates; import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.findV2Template; @@ -182,7 +183,7 @@ public static Template resolveTemplate(final String matchingTemplate, final Stri resolvedAliases, tempClusterState.metadata(), aliasValidator, xContentRegistry, // the context is only used for validation so it's fine to pass fake values for the // shard id and the current timestamp - tempIndexService.newQueryShardContext(0, null, () -> 0L, null))); + tempIndexService.newQueryShardContext(0, null, () -> 0L, null, emptyMap()))); Map aliasesByName = aliases.stream().collect( Collectors.toMap(AliasMetadata::getAlias, Function.identity())); diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java index e6e07dbc67904..ef7aff6be399e 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java @@ -42,6 +42,7 @@ import java.util.Map; import java.util.Objects; +import static java.util.Collections.emptyMap; import static org.elasticsearch.action.ValidateActions.addValidationError; /** diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java index ecee9d8f6e40d..1884f69b259eb 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java @@ -98,6 +98,7 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; +import static java.util.Collections.emptyMap; import static java.util.stream.Collectors.toList; import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING; import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING; @@ -490,7 +491,7 @@ private ClusterState applyCreateIndexRequestWithV1Templates(final ClusterState c MetadataIndexTemplateService.resolveAliases(templates), currentState.metadata(), aliasValidator, // the context is only used for validation so it's fine to pass fake values for the // shard id and the current timestamp - xContentRegistry, indexService.newQueryShardContext(0, null, () -> 0L, null)), + xContentRegistry, indexService.newQueryShardContext(0, null, () -> 0L, null, emptyMap())), templates.stream().map(IndexTemplateMetadata::getName).collect(toList()), metadataTransformer); } @@ -523,7 +524,7 @@ private ClusterState applyCreateIndexRequestWithV2Template(final ClusterState cu MetadataIndexTemplateService.resolveAliases(currentState.metadata(), templateName), currentState.metadata(), aliasValidator, // the context is only used for validation so it's fine to pass fake values for the // shard id and the current timestamp - xContentRegistry, indexService.newQueryShardContext(0, null, () -> 0L, null)), + xContentRegistry, indexService.newQueryShardContext(0, null, () -> 0L, null, emptyMap())), Collections.singletonList(templateName), metadataTransformer); } @@ -569,7 +570,7 @@ private ClusterState applyCreateIndexRequestWithExistingMetadata(final ClusterSt currentState.metadata(), aliasValidator, xContentRegistry, // the context is only used for validation so it's fine to pass fake values for the // shard id and the current timestamp - indexService.newQueryShardContext(0, null, () -> 0L, null)), + indexService.newQueryShardContext(0, null, () -> 0L, null, emptyMap())), List.of(), metadataTransformer); } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesService.java index 2ea44d0440119..5b9ae54481030 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesService.java @@ -47,6 +47,7 @@ import java.util.function.Function; import static java.util.Collections.emptyList; +import static java.util.Collections.emptyMap; import static org.elasticsearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason.NO_LONGER_ASSIGNED; /** @@ -154,7 +155,7 @@ public ClusterState applyAliasActions(ClusterState currentState, Iterable System.currentTimeMillis(), null), xContentRegistry); + () -> System.currentTimeMillis(), null, emptyMap()), xContentRegistry); } }; if (action.apply(newAliasValidator, metadata, index)) { diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index 19b9e83fd06a4..5c9e54f097f74 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -80,6 +80,7 @@ import java.util.function.Predicate; import java.util.stream.Collectors; +import static java.util.Collections.emptyMap; import static org.elasticsearch.cluster.metadata.MetadataCreateDataStreamService.validateTimestampFieldMapping; import static org.elasticsearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason.NO_LONGER_ASSIGNED; @@ -1112,7 +1113,7 @@ private static void validateCompositeTemplate(final ClusterState state, new AliasValidator(), // the context is only used for validation so it's fine to pass fake values for the // shard id and the current timestamp - xContentRegistry, tempIndexService.newQueryShardContext(0, null, () -> 0L, null)); + xContentRegistry, tempIndexService.newQueryShardContext(0, null, () -> 0L, null, emptyMap())); // triggers inclusion of _timestamp field and its validation: String indexName = DataStream.BACKING_INDEX_PREFIX + temporaryIndexName; diff --git a/server/src/main/java/org/elasticsearch/index/IndexService.java b/server/src/main/java/org/elasticsearch/index/IndexService.java index 4d3dad75841c7..61198c312f4d3 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexService.java +++ b/server/src/main/java/org/elasticsearch/index/IndexService.java @@ -192,7 +192,7 @@ public IndexService( assert indexAnalyzers != null; this.mapperService = new MapperService(indexSettings, indexAnalyzers, xContentRegistry, similarityService, mapperRegistry, // we parse all percolator queries as they would be parsed on shard 0 - () -> newQueryShardContext(0, null, System::currentTimeMillis, null), idFieldDataEnabled, scriptService); + () -> newQueryShardContext(0, null, System::currentTimeMillis, null, emptyMap()), idFieldDataEnabled, scriptService); this.indexFieldData = new IndexFieldDataService(indexSettings, indicesFieldDataCache, circuitBreakerService, mapperService); if (indexSettings.getIndexSortConfig().hasIndexSort()) { // we delay the actual creation of the sort order for this index because the mapping has not been merged yet. @@ -586,13 +586,19 @@ public IndexSettings getIndexSettings() { * Passing a {@code null} {@link IndexSearcher} will return a valid context, however it won't be able to make * {@link IndexReader}-specific optimizations, such as rewriting containing range queries. */ - public QueryShardContext newQueryShardContext(int shardId, IndexSearcher searcher, LongSupplier nowInMillis, String clusterAlias) { + public QueryShardContext newQueryShardContext( + int shardId, + IndexSearcher searcher, + LongSupplier nowInMillis, + String clusterAlias, + Map runtimeMappings + ) { final SearchIndexNameMatcher indexNameMatcher = new SearchIndexNameMatcher(index().getName(), clusterAlias, clusterService, expressionResolver); return new QueryShardContext( shardId, indexSettings, bigArrays, indexCache.bitsetFilterCache(), indexFieldData::getForField, mapperService(), similarityService(), scriptService, xContentRegistry, namedWriteableRegistry, client, searcher, nowInMillis, clusterAlias, - indexNameMatcher, allowExpensiveQueries, valuesSourceRegistry); + indexNameMatcher, allowExpensiveQueries, valuesSourceRegistry, runtimeMappings); } /** diff --git a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java index 368daac1729b0..ff350edba5787 100644 --- a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java @@ -26,6 +26,7 @@ import org.apache.lucene.search.join.BitSetProducer; import org.apache.lucene.search.similarities.Similarity; import org.apache.lucene.util.SetOnce; +import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.client.Client; @@ -77,6 +78,8 @@ import java.util.function.Predicate; import java.util.function.Supplier; +import static java.util.Collections.emptyMap; + /** * Context object used to create lucene queries on the shard level. */ @@ -103,8 +106,12 @@ public class QueryShardContext extends QueryRewriteContext { private boolean mapUnmappedFieldAsString; private NestedScope nestedScope; private final ValuesSourceRegistry valuesSourceRegistry; + private final Map runtimeMappings; - public QueryShardContext(int shardId, + /** + * Build a {@linkplain QueryShardContext} without any information from the search request. + */ + public QueryShardContext(int shardId, // NOCOMMIT make sure we need two methods IndexSettings indexSettings, BigArrays bigArrays, BitsetFilterCache bitsetFilterCache, @@ -122,16 +129,63 @@ public QueryShardContext(int shardId, BooleanSupplier allowExpensiveQueries, ValuesSourceRegistry valuesSourceRegistry) { this(shardId, indexSettings, bigArrays, bitsetFilterCache, indexFieldDataLookup, mapperService, similarityService, - scriptService, xContentRegistry, namedWriteableRegistry, client, searcher, nowInMillis, indexNameMatcher, - new Index(RemoteClusterAware.buildRemoteIndexName(clusterAlias, indexSettings.getIndex().getName()), - indexSettings.getIndex().getUUID()), allowExpensiveQueries, valuesSourceRegistry); + scriptService, xContentRegistry, namedWriteableRegistry, client, searcher, nowInMillis, clusterAlias, + indexNameMatcher, allowExpensiveQueries, valuesSourceRegistry, emptyMap()); + } + + /** + * Build a {@linkplain QueryShardContext} with information from the search request. + */ + public QueryShardContext( + int shardId, + IndexSettings indexSettings, + BigArrays bigArrays, + BitsetFilterCache bitsetFilterCache, + TriFunction, IndexFieldData> indexFieldDataLookup, + MapperService mapperService, + SimilarityService similarityService, + ScriptService scriptService, + NamedXContentRegistry xContentRegistry, + NamedWriteableRegistry namedWriteableRegistry, + Client client, + IndexSearcher searcher, + LongSupplier nowInMillis, + String clusterAlias, + Predicate indexNameMatcher, + BooleanSupplier allowExpensiveQueries, + ValuesSourceRegistry valuesSourceRegistry, + Map runtimeMappings + ) { + this( + shardId, + indexSettings, + bigArrays, + bitsetFilterCache, + indexFieldDataLookup, + mapperService, + similarityService, + scriptService, + xContentRegistry, + namedWriteableRegistry, + client, + searcher, + nowInMillis, + indexNameMatcher, + new Index( + RemoteClusterAware.buildRemoteIndexName(clusterAlias, indexSettings.getIndex().getName()), + indexSettings.getIndex().getUUID() + ), + allowExpensiveQueries, + valuesSourceRegistry, + parseRuntimeMappings(runtimeMappings, mapperService::parserContext, indexSettings) + ); } public QueryShardContext(QueryShardContext source) { this(source.shardId, source.indexSettings, source.bigArrays, source.bitsetFilterCache, source.indexFieldDataService, source.mapperService, source.similarityService, source.scriptService, source.getXContentRegistry(), source.getWriteableRegistry(), source.client, source.searcher, source.nowInMillis, source.indexNameMatcher, - source.fullyQualifiedIndex, source.allowExpensiveQueries, source.valuesSourceRegistry); + source.fullyQualifiedIndex, source.allowExpensiveQueries, source.valuesSourceRegistry, source.runtimeMappings); } private QueryShardContext(int shardId, @@ -150,7 +204,8 @@ private QueryShardContext(int shardId, Predicate indexNameMatcher, Index fullyQualifiedIndex, BooleanSupplier allowExpensiveQueries, - ValuesSourceRegistry valuesSourceRegistry) { + ValuesSourceRegistry valuesSourceRegistry, + Map runtimeMappings) { super(xContentRegistry, namedWriteableRegistry, client, nowInMillis); this.shardId = shardId; this.similarityService = similarityService; @@ -167,6 +222,7 @@ private QueryShardContext(int shardId, this.fullyQualifiedIndex = fullyQualifiedIndex; this.allowExpensiveQueries = allowExpensiveQueries; this.valuesSourceRegistry = valuesSourceRegistry; + this.runtimeMappings = runtimeMappings; } private void reset() { @@ -256,14 +312,19 @@ public Set simpleMatchToIndexNames(String pattern) { * @see QueryShardContext#setMapUnmappedFieldAsString(boolean) */ public MappedFieldType getFieldType(String name) { - return failIfFieldMappingNotFound(name, mapperService.fieldType(name)); + return failIfFieldMappingNotFound(name, fieldType(name)); } /** * Returns true if the field identified by the provided name is mapped, false otherwise */ public boolean isFieldMapped(String name) { - return mapperService.fieldType(name) != null; + return fieldType(name) != null; + } + + private MappedFieldType fieldType(String name) { + MappedFieldType fieldType = runtimeMappings.get(name); + return fieldType == null ? mapperService.fieldType(name) : fieldType; } public ObjectMapper getObjectMapper(String name) { @@ -275,13 +336,22 @@ public ObjectMapper getObjectMapper(String name) { * Generally used to handle unmapped fields in the context of sorting. */ public MappedFieldType buildAnonymousFieldType(String type) { - final Mapper.TypeParser.ParserContext parserContext = mapperService.parserContext(); + return buildFieldType(type, "__anonymous_" + type, Collections.emptyMap(), mapperService.parserContext(), indexSettings); + } + + private static MappedFieldType buildFieldType( + String type, + String field, + Map node, + Mapper.TypeParser.ParserContext parserContext, + IndexSettings indexSettings + ) { Mapper.TypeParser typeParser = parserContext.typeParser(type); if (typeParser == null) { throw new IllegalArgumentException("No mapper found for type [" + type + "]"); } - final Mapper.Builder builder = typeParser.parse("__anonymous_" + type, Collections.emptyMap(), parserContext); - final Mapper.BuilderContext builderContext = new Mapper.BuilderContext(indexSettings.getSettings(), new ContentPath(1)); + Mapper.Builder builder = typeParser.parse(field, node, parserContext); + Mapper.BuilderContext builderContext = new Mapper.BuilderContext(indexSettings.getSettings(), new ContentPath(1)); Mapper mapper = builder.build(builderContext); if (mapper instanceof FieldMapper) { return ((FieldMapper)mapper).fieldType(); @@ -522,4 +592,28 @@ public SimilarityService getSimilarityService() { public BitsetFilterCache getBitsetFilterCache() { return bitsetFilterCache; } + + private static Map parseRuntimeMappings( + Map mappings, + Supplier parserContextSupplier, + IndexSettings indexSettings + ) { + Map runtimeMappings = new HashMap<>(); + for (Map.Entry entry : mappings.entrySet()) { + String field = entry.getKey(); + if (entry.getValue() instanceof Map == false) { + throw new ElasticsearchParseException("runtime mappings must be a map type"); + } + @SuppressWarnings("unchecked") + Map node = (Map) entry.getValue(); + // Replace the type until we have native support for the runtime section + Object oldRuntimeType = node.put("runtime_type", node.remove("type")); + if (oldRuntimeType != null) { + throw new ElasticsearchParseException("use [type] in [runtime_mappings] instead of [runtime_type]"); + } + runtimeMappings.put(field, buildFieldType("runtime", field, node, parserContextSupplier.get(), indexSettings)); + } + return runtimeMappings; + } + } diff --git a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java index 25defd6e58bcf..5c8fe1540c162 100644 --- a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java +++ b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java @@ -178,7 +178,7 @@ final class DefaultSearchContext extends SearchContext { this.relativeTimeSupplier = relativeTimeSupplier; this.timeout = timeout; queryShardContext = indexService.newQueryShardContext(request.shardId().id(), this.searcher, - request::nowInMillis, shardTarget.getClusterAlias()); + request::nowInMillis, shardTarget.getClusterAlias(), request.getRuntimeMappings()); queryBoost = request.indexBoost(); this.lowLevelCancellation = lowLevelCancellation; } diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java index e8ed6134cb883..d48260ec6e69a 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchService.java +++ b/server/src/main/java/org/elasticsearch/search/SearchService.java @@ -1178,7 +1178,7 @@ private CanMatchResponse canMatch(ShardSearchRequest request, boolean checkRefre try (canMatchSearcher) { QueryShardContext context = indexService.newQueryShardContext(request.shardId().id(), canMatchSearcher, - request::nowInMillis, request.getClusterAlias()); + request::nowInMillis, request.getClusterAlias(), request.getRuntimeMappings()); Rewriteable.rewrite(request.getRewriteable(), context, false); final boolean aliasFilterCanMatch = request.getAliasFilter() .getQueryBuilder() instanceof MatchNoneQueryBuilder == false; diff --git a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java index 461cf5456c7e6..27035d7fb77ec 100644 --- a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java @@ -63,6 +63,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Objects; import static org.elasticsearch.index.query.AbstractQueryBuilder.parseInnerQueryBuilder; @@ -110,6 +111,7 @@ public final class SearchSourceBuilder implements Writeable, ToXContentObject, R public static final ParseField COLLAPSE = new ParseField("collapse"); public static final ParseField SLICE = new ParseField("slice"); public static final ParseField POINT_IN_TIME = new ParseField("pit"); + public static final ParseField RUNTIME_MAPPINGS_FIELD = new ParseField("runtime_mappings"); public static SearchSourceBuilder fromXContent(XContentParser parser) throws IOException { return fromXContent(parser, true); @@ -191,6 +193,8 @@ public static HighlightBuilder highlight() { private PointInTimeBuilder pointInTimeBuilder = null; + private Map runtimeMappings = null; + /** * Constructs a new search source builder. */ @@ -251,6 +255,9 @@ public SearchSourceBuilder(StreamInput in) throws IOException { } pointInTimeBuilder = in.readOptionalWriteable(PointInTimeBuilder::new); } + if (in.getVersion().onOrAfter(Version.V_8_0_0)) { + runtimeMappings = in.readMap(); + } } @Override @@ -312,6 +319,9 @@ public void writeTo(StreamOutput out) throws IOException { } out.writeOptionalWriteable(pointInTimeBuilder); } + if (out.getVersion().onOrAfter(Version.V_8_0_0)) { + out.writeMap(runtimeMappings); + } } /** @@ -973,6 +983,21 @@ public SearchSourceBuilder pointInTimeBuilder(PointInTimeBuilder builder) { return this; } + /** + * Mappings specified on this search request that override built in mappings. + */ + public Map runtimeMappings() { + return runtimeMappings; + } + + /** + * Specify the mappings specified on this search request that override built in mappings. + */ + public SearchSourceBuilder runtimeMappings(Map runtimeMappings) { + this.runtimeMappings = runtimeMappings; + return this; + } + /** * Rewrites this search source builder into its primitive form. e.g. by * rewriting the QueryBuilder. If the builder did not change the identity @@ -1059,6 +1084,7 @@ private SearchSourceBuilder shallowCopy(QueryBuilder queryBuilder, QueryBuilder rewrittenBuilder.seqNoAndPrimaryTerm = seqNoAndPrimaryTerm; rewrittenBuilder.collapse = collapse; rewrittenBuilder.pointInTimeBuilder = pointInTimeBuilder; + rewrittenBuilder.runtimeMappings = runtimeMappings; return rewrittenBuilder; } @@ -1169,6 +1195,8 @@ public void parseXContent(XContentParser parser, boolean checkTrailingTokens) th collapse = CollapseBuilder.fromXContent(parser); } else if (POINT_IN_TIME.match(currentFieldName, parser.getDeprecationHandler())) { pointInTimeBuilder = PointInTimeBuilder.fromXContent(parser); + } else if (RUNTIME_MAPPINGS_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { + runtimeMappings = parser.map(); } else { throw new ParsingException(parser.getTokenLocation(), "Unknown key for a " + token + " in [" + currentFieldName + "].", parser.getTokenLocation()); @@ -1376,6 +1404,10 @@ public XContentBuilder innerToXContent(XContentBuilder builder, Params params) t if (pointInTimeBuilder != null) { pointInTimeBuilder.toXContent(builder, params); } + if (runtimeMappings != null) { + builder.field(RUNTIME_MAPPINGS_FIELD.getPreferredName(), runtimeMappings); + } + return builder; } @@ -1588,7 +1620,7 @@ public int hashCode() { return Objects.hash(aggregations, explain, fetchSourceContext, fetchFields, docValueFields, storedFieldsContext, from, highlightBuilder, indexBoosts, minScore, postQueryBuilder, queryBuilder, rescoreBuilders, scriptFields, size, sorts, searchAfterBuilder, sliceBuilder, stats, suggestBuilder, terminateAfter, timeout, trackScores, version, - seqNoAndPrimaryTerm, profile, extBuilders, collapse, trackTotalHitsUpTo, pointInTimeBuilder); + seqNoAndPrimaryTerm, profile, extBuilders, collapse, trackTotalHitsUpTo, pointInTimeBuilder, runtimeMappings); } @Override @@ -1629,7 +1661,8 @@ public boolean equals(Object obj) { && Objects.equals(extBuilders, other.extBuilders) && Objects.equals(collapse, other.collapse) && Objects.equals(trackTotalHitsUpTo, other.trackTotalHitsUpTo) - && Objects.equals(pointInTimeBuilder, other.pointInTimeBuilder); + && Objects.equals(pointInTimeBuilder, other.pointInTimeBuilder) + && Objects.equals(runtimeMappings, other.runtimeMappings); } @Override diff --git a/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java b/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java index d496a10e696c2..4b494781b8ad9 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java +++ b/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java @@ -47,8 +47,8 @@ import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.AliasFilterParsingException; import org.elasticsearch.indices.InvalidAliasNameException; -import org.elasticsearch.search.SearchSortValuesAndFormats; import org.elasticsearch.search.Scroll; +import org.elasticsearch.search.SearchSortValuesAndFormats; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.query.QuerySearchResult; import org.elasticsearch.search.sort.FieldSortBuilder; @@ -61,6 +61,7 @@ import java.util.Map; import java.util.function.Function; +import static java.util.Collections.emptyMap; import static org.elasticsearch.search.internal.SearchContext.TRACK_TOTAL_HITS_DISABLED; /** @@ -521,4 +522,8 @@ public static QueryBuilder parseAliasFilter(CheckedFunction getRuntimeMappings() { + return source != null && source.runtimeMappings() != null ? source.runtimeMappings() : emptyMap(); + } } diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java b/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java index 353dceb12494e..534ac7e929be3 100644 --- a/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java +++ b/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java @@ -60,6 +60,7 @@ import java.util.Collection; import java.util.List; +import static java.util.Collections.emptyMap; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.sameInstance; @@ -140,7 +141,7 @@ public void setup() throws Exception { writer = new IndexWriter( new ByteBuffersDirectory(), new IndexWriterConfig(new StandardAnalyzer()).setMergePolicy(new LogByteSizeMergePolicy()) ); - shardContext = indexService.newQueryShardContext(0, null, () -> 0, null); + shardContext = indexService.newQueryShardContext(0, null, () -> 0, null, emptyMap()); } protected final List refreshReader() throws Exception { diff --git a/server/src/test/java/org/elasticsearch/index/search/MultiMatchQueryTests.java b/server/src/test/java/org/elasticsearch/index/search/MultiMatchQueryTests.java index 16bb399bde087..d633b7248e2cb 100644 --- a/server/src/test/java/org/elasticsearch/index/search/MultiMatchQueryTests.java +++ b/server/src/test/java/org/elasticsearch/index/search/MultiMatchQueryTests.java @@ -58,6 +58,7 @@ import java.util.List; import java.util.Map; +import static java.util.Collections.emptyMap; import static org.elasticsearch.index.query.QueryBuilders.multiMatchQuery; import static org.hamcrest.Matchers.equalTo; @@ -99,7 +100,7 @@ public void setup() throws IOException { public void testCrossFieldMultiMatchQuery() throws IOException { QueryShardContext queryShardContext = indexService.newQueryShardContext( - randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null); + randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null, emptyMap()); queryShardContext.setAllowUnmappedFields(true); for (float tieBreaker : new float[] {0.0f, 0.5f}) { Query parsedQuery = multiMatchQuery("banon") @@ -126,8 +127,12 @@ public void testBlendTerms() { float[] boosts = new float[] {2, 3}; Query expected = BlendedTermQuery.dismaxBlendedQuery(terms, boosts, 1.0f); Query actual = MultiMatchQuery.blendTerm( - indexService.newQueryShardContext(randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null), - new BytesRef("baz"), 1f, false, Arrays.asList(new FieldAndBoost(ft1, 2), new FieldAndBoost(ft2, 3))); + indexService.newQueryShardContext(randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null, emptyMap()), + new BytesRef("baz"), + 1f, + false, + Arrays.asList(new FieldAndBoost(ft1, 2), new FieldAndBoost(ft2, 3)) + ); assertEquals(expected, actual); } @@ -146,8 +151,12 @@ public Query termQuery(Object value, QueryShardContext context) { BlendedTermQuery.dismaxBlendedQuery(terms, boosts, 1.0f) ), 1f); Query actual = MultiMatchQuery.blendTerm( - indexService.newQueryShardContext(randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null), - new BytesRef("baz"), 1f, true, Arrays.asList(new FieldAndBoost(ft1, 2), new FieldAndBoost(ft2, 3))); + indexService.newQueryShardContext(randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null, emptyMap()), + new BytesRef("baz"), + 1f, + true, + Arrays.asList(new FieldAndBoost(ft1, 2), new FieldAndBoost(ft2, 3)) + ); assertEquals(expected, actual); } @@ -159,7 +168,7 @@ public Query termQuery(Object value, QueryShardContext context) { } }; expectThrows(IllegalArgumentException.class, () -> MultiMatchQuery.blendTerm( - indexService.newQueryShardContext(randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null), + indexService.newQueryShardContext(randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null, emptyMap()), new BytesRef("baz"), 1f, false, Arrays.asList(new FieldAndBoost(ft, 1)))); } @@ -181,14 +190,18 @@ public Query termQuery(Object value, QueryShardContext context) { expectedDisjunct1 ), 1.0f); Query actual = MultiMatchQuery.blendTerm( - indexService.newQueryShardContext(randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null), - new BytesRef("baz"), 1f, false, Arrays.asList(new FieldAndBoost(ft1, 2), new FieldAndBoost(ft2, 3))); + indexService.newQueryShardContext(randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null, emptyMap()), + new BytesRef("baz"), + 1f, + false, + Arrays.asList(new FieldAndBoost(ft1, 2), new FieldAndBoost(ft2, 3)) + ); assertEquals(expected, actual); } public void testMultiMatchCrossFieldsWithSynonyms() throws IOException { QueryShardContext queryShardContext = indexService.newQueryShardContext( - randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null); + randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null, emptyMap()); MultiMatchQuery parser = new MultiMatchQuery(queryShardContext); parser.setAnalyzer(new MockSynonymAnalyzer()); @@ -220,7 +233,7 @@ public void testMultiMatchCrossFieldsWithSynonyms() throws IOException { public void testMultiMatchCrossFieldsWithSynonymsPhrase() throws IOException { QueryShardContext queryShardContext = indexService.newQueryShardContext( - randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null); + randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null, emptyMap()); MultiMatchQuery parser = new MultiMatchQuery(queryShardContext); parser.setAnalyzer(new MockSynonymAnalyzer()); Map fieldNames = new HashMap<>(); @@ -289,7 +302,7 @@ public void testKeywordSplitQueriesOnWhitespace() throws IOException { QueryShardContext queryShardContext = indexService.newQueryShardContext( randomInt(20), null, () -> { throw new UnsupportedOperationException(); - }, null); + }, null, emptyMap()); MultiMatchQuery parser = new MultiMatchQuery(queryShardContext); Map fieldNames = new HashMap<>(); fieldNames.put("field", 1.0f); diff --git a/server/src/test/java/org/elasticsearch/index/search/NestedHelperTests.java b/server/src/test/java/org/elasticsearch/index/search/NestedHelperTests.java index f33d813ebb048..d8f7405ef3387 100644 --- a/server/src/test/java/org/elasticsearch/index/search/NestedHelperTests.java +++ b/server/src/test/java/org/elasticsearch/index/search/NestedHelperTests.java @@ -43,6 +43,8 @@ import java.io.IOException; import java.util.Collections; +import static java.util.Collections.emptyMap; + public class NestedHelperTests extends ESSingleNodeTestCase { IndexService indexService; @@ -333,7 +335,7 @@ public void testConjunction() { } public void testNested() throws IOException { - QueryShardContext context = indexService.newQueryShardContext(0, new IndexSearcher(new MultiReader()), () -> 0, null); + QueryShardContext context = indexService.newQueryShardContext(0, new IndexSearcher(new MultiReader()), () -> 0, null, emptyMap()); NestedQueryBuilder queryBuilder = new NestedQueryBuilder("nested1", new MatchAllQueryBuilder(), ScoreMode.Avg); ESToParentBlockJoinQuery query = (ESToParentBlockJoinQuery) queryBuilder.toQuery(context); diff --git a/server/src/test/java/org/elasticsearch/index/search/nested/NestedSortingTests.java b/server/src/test/java/org/elasticsearch/index/search/nested/NestedSortingTests.java index d2b8c6bab9dd8..e4b0573b06cca 100644 --- a/server/src/test/java/org/elasticsearch/index/search/nested/NestedSortingTests.java +++ b/server/src/test/java/org/elasticsearch/index/search/nested/NestedSortingTests.java @@ -73,6 +73,7 @@ import java.util.Collections; import java.util.List; +import static java.util.Collections.emptyMap; import static org.elasticsearch.index.mapper.SeqNoFieldMapper.PRIMARY_TERM_NAME; import static org.hamcrest.Matchers.equalTo; @@ -607,7 +608,7 @@ public void testMultiLevelNestedSorting() throws IOException { DirectoryReader reader = DirectoryReader.open(writer); reader = ElasticsearchDirectoryReader.wrap(reader, new ShardId(indexService.index(), 0)); IndexSearcher searcher = new IndexSearcher(reader); - QueryShardContext queryShardContext = indexService.newQueryShardContext(0, searcher, () -> 0L, null); + QueryShardContext queryShardContext = indexService.newQueryShardContext(0, searcher, () -> 0L, null, emptyMap()); FieldSortBuilder sortBuilder = new FieldSortBuilder("chapters.paragraphs.word_count"); sortBuilder.setNestedSort(new NestedSortBuilder("chapters").setNestedSort(new NestedSortBuilder("chapters.paragraphs"))); diff --git a/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java b/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java index d71958df9b1ea..01ab3c591b41c 100644 --- a/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java +++ b/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java @@ -108,7 +108,9 @@ public void testPreProcess() throws Exception { when(indexCache.query()).thenReturn(queryCache); when(indexService.cache()).thenReturn(indexCache); QueryShardContext queryShardContext = mock(QueryShardContext.class); - when(indexService.newQueryShardContext(eq(shardId.id()), anyObject(), anyObject(), anyString())).thenReturn(queryShardContext); + when(indexService.newQueryShardContext(eq(shardId.id()), anyObject(), anyObject(), anyString(), anyObject())).thenReturn( + queryShardContext + ); MapperService mapperService = mock(MapperService.class); when(mapperService.hasNested()).thenReturn(randomBoolean()); when(indexService.mapperService()).thenReturn(mapperService); diff --git a/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java b/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java index 7ae94d37e6416..ec32c01d90cd0 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java +++ b/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java @@ -66,6 +66,8 @@ import java.util.List; import java.util.Map; +import static java.util.Collections.emptyMap; + public class TestSearchContext extends SearchContext { public static final SearchShardTarget SHARD_TARGET = new SearchShardTarget("test", new ShardId("test", "test", 0), null, OriginalIndices.NONE); @@ -100,7 +102,7 @@ public TestSearchContext(BigArrays bigArrays, IndexService indexService) { this.indexService = indexService; this.fixedBitSetFilterCache = indexService.cache().bitsetFilterCache(); this.indexShard = indexService.getShardOrNull(0); - queryShardContext = indexService.newQueryShardContext(0, null, () -> 0L, null); + queryShardContext = indexService.newQueryShardContext(0, null, () -> 0L, null, emptyMap()); } public TestSearchContext(QueryShardContext queryShardContext) { diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchAction.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchAction.java index dac79f04cba67..1ece72b2e4bba 100644 --- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchAction.java +++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchAction.java @@ -70,6 +70,8 @@ import java.util.Map; import java.util.Set; +import static java.util.Collections.emptyMap; + /** * This is an internal action, that executes msearch requests for enrich indices in a more efficient manner. * Currently each search request inside a msearch request is executed as a separate search. If many search requests @@ -233,7 +235,8 @@ protected MultiSearchResponse shardOperation(Request request, ShardId shardId) t shardId.id(), searcher, () -> { throw new UnsupportedOperationException(); }, - null + null, + emptyMap() // NOCOMMIT is it right not to use the runtime mappings? ); final MultiSearchResponse.Item[] items = new MultiSearchResponse.Item[request.multiSearchRequest.requests().size()]; for (int i = 0; i < request.multiSearchRequest.requests().size(); i++) { diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java index 4474e401ecbaf..62bb9052b0107 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java @@ -282,6 +282,7 @@ import java.util.stream.Collectors; import static java.util.Collections.emptyList; +import static java.util.Collections.emptyMap; import static java.util.Collections.singletonList; import static org.elasticsearch.xpack.core.XPackSettings.API_KEY_SERVICE_ENABLED_SETTING; import static org.elasticsearch.xpack.core.XPackSettings.HTTP_SSL_ENABLED; @@ -719,7 +720,10 @@ public void onIndexModule(IndexModule module) { () -> { throw new IllegalArgumentException("permission filters are not allowed to use the current timestamp"); - }, null), + }, + null, + // Don't use runtime mappings in the security query + emptyMap()), dlsBitsetCache.get(), securityContext.get(), getLicenseState(), From ff7575018d5ac8b0c64ea05f4c1ed9961930e2b5 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Mon, 2 Nov 2020 16:09:31 -0500 Subject: [PATCH 02/36] WIP --- x-pack/plugin/runtime-fields/qa/build.gradle | 8 +- .../{rest => core-with-mapped}/build.gradle | 4 + .../mapped/CoreWithMappedRuntimeFieldsIT.java | 63 ++++ .../qa/core-with-search/build.gradle | 54 ++++ .../CoreTestsWithSearchRuntimeFieldsIT.java | 84 +++++ .../rest/CoreTestsWithRuntimeFieldsIT.java | 259 --------------- .../test/CoreTestTranslater.java | 300 ++++++++++++++++++ 7 files changed, 512 insertions(+), 260 deletions(-) rename x-pack/plugin/runtime-fields/qa/{rest => core-with-mapped}/build.gradle (95%) create mode 100644 x-pack/plugin/runtime-fields/qa/core-with-mapped/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/mapped/CoreWithMappedRuntimeFieldsIT.java create mode 100644 x-pack/plugin/runtime-fields/qa/core-with-search/build.gradle create mode 100644 x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java delete mode 100644 x-pack/plugin/runtime-fields/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/rest/CoreTestsWithRuntimeFieldsIT.java create mode 100644 x-pack/plugin/runtime-fields/qa/src/main/java/org/elasticsearch/xpack/runtimefields/test/CoreTestTranslater.java diff --git a/x-pack/plugin/runtime-fields/qa/build.gradle b/x-pack/plugin/runtime-fields/qa/build.gradle index 7dc01b73ed9ec..bc22510e77458 100644 --- a/x-pack/plugin/runtime-fields/qa/build.gradle +++ b/x-pack/plugin/runtime-fields/qa/build.gradle @@ -1 +1,7 @@ -// Empty project so we can pick up its subproject +// Shared infratructure + +apply plugin: 'elasticsearch.build' + +dependencies { + api project(":test:framework") +} diff --git a/x-pack/plugin/runtime-fields/qa/rest/build.gradle b/x-pack/plugin/runtime-fields/qa/core-with-mapped/build.gradle similarity index 95% rename from x-pack/plugin/runtime-fields/qa/rest/build.gradle rename to x-pack/plugin/runtime-fields/qa/core-with-mapped/build.gradle index 34998468a35a2..7af3a9d6886da 100644 --- a/x-pack/plugin/runtime-fields/qa/rest/build.gradle +++ b/x-pack/plugin/runtime-fields/qa/core-with-mapped/build.gradle @@ -1,5 +1,9 @@ apply plugin: 'elasticsearch.yaml-rest-test' +dependencies { + yamlRestTestImplementation xpackProject("plugin:runtime-fields:qa") +} + restResources { restApi { includeXpack 'async_search', 'graph', '*_point_in_time' diff --git a/x-pack/plugin/runtime-fields/qa/core-with-mapped/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/mapped/CoreWithMappedRuntimeFieldsIT.java b/x-pack/plugin/runtime-fields/qa/core-with-mapped/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/mapped/CoreWithMappedRuntimeFieldsIT.java new file mode 100644 index 0000000000000..ea46c4ceaf023 --- /dev/null +++ b/x-pack/plugin/runtime-fields/qa/core-with-mapped/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/mapped/CoreWithMappedRuntimeFieldsIT.java @@ -0,0 +1,63 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.test.mapped; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; +import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; +import org.elasticsearch.test.rest.yaml.section.ApiCallSection; +import org.elasticsearch.xpack.runtimefields.test.CoreTestTranslater; + +import java.util.List; +import java.util.Map; + +public class CoreWithMappedRuntimeFieldsIT extends ESClientYamlSuiteTestCase { + public CoreWithMappedRuntimeFieldsIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { + super(testCandidate); + } + + @ParametersFactory + public static Iterable parameters() throws Exception { + return new MappingRuntimeFieldTranslater().parameters(); + } + + /** + * Builds test parameters similarly to {@link ESClientYamlSuiteTestCase#createParameters()}, + * replacing all fields with runtime fields that load from {@code _source} if possible. Tests + * that configure the field in a way that are not supported by runtime fields are skipped. + */ + private static class MappingRuntimeFieldTranslater extends CoreTestTranslater { + @Override + protected List> dynamicTemplates() { + return dynamicTemplatesToAddRuntimeFieldsToMappings(); + } + + @Override + protected Suite suite(ClientYamlTestCandidate candidate) { + return new Suite(candidate) { + @Override + protected boolean modifyMappingProperties(String index, Map properties) { + return runtimeifyMappingProperties(properties); + } + + @Override + protected void modifyMapping(Map mapping) { + // The top level mapping is fine for runtime fields defined in the mapping + } + + @Override + protected boolean modifySearch(ApiCallSection search) { + // We don't need to modify the search request if the mappings are in the index + return true; + } + }; + } + + } +} diff --git a/x-pack/plugin/runtime-fields/qa/core-with-search/build.gradle b/x-pack/plugin/runtime-fields/qa/core-with-search/build.gradle new file mode 100644 index 0000000000000..7af3a9d6886da --- /dev/null +++ b/x-pack/plugin/runtime-fields/qa/core-with-search/build.gradle @@ -0,0 +1,54 @@ +apply plugin: 'elasticsearch.yaml-rest-test' + +dependencies { + yamlRestTestImplementation xpackProject("plugin:runtime-fields:qa") +} + +restResources { + restApi { + includeXpack 'async_search', 'graph', '*_point_in_time' + } + restTests { + includeCore '*' + includeXpack 'async_search', 'graph' + } +} + +testClusters.yamlRestTest { + testDistribution = 'DEFAULT' + setting 'xpack.license.self_generated.type', 'trial' +} + +yamlRestTest { + systemProperty 'tests.rest.suite', + [ + 'async_search', + 'field_caps', + 'graph', + 'msearch', + 'search', + 'search.aggregation', + 'search.highlight', + 'search.inner_hits', + 'search_shards', + 'suggest', + ].join(',') + systemProperty 'tests.rest.blacklist', + [ + /////// TO FIX /////// + 'search.highlight/40_keyword_ignore/Plain Highligher should skip highlighting ignored keyword values', // The plain highlighter is incompatible with runtime fields. Worth fixing? + 'search/115_multiple_field_collapsing/two levels fields collapsing', // Broken. Gotta fix. + 'field_caps/30_filter/Field caps with index filter', // We don't support filtering field caps on runtime fields. What should we do? + 'search.aggregation/10_histogram/*', // runtime doesn't support sub-fields. Maybe it should? + 'search/140_pre_filter_search_shards/pre_filter_shard_size with shards that have no hit', + /////// TO FIX /////// + + /////// NOT SUPPORTED /////// + 'search.aggregation/280_rare_terms/*', // Requires an index and we won't have it + // Runtime fields don't have global ords + 'search.aggregation/20_terms/string profiler via global ordinals', + 'search.aggregation/20_terms/Global ordinals are loaded with the global_ordinals execution hint', + 'search.aggregation/170_cardinality_metric/profiler string', + /////// NOT SUPPORTED /////// + ].join(',') +} diff --git a/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java b/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java new file mode 100644 index 0000000000000..43c6c7630721a --- /dev/null +++ b/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java @@ -0,0 +1,84 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.test.search; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; +import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; +import org.elasticsearch.test.rest.yaml.section.ApiCallSection; +import org.elasticsearch.xpack.runtimefields.test.CoreTestTranslater; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.hamcrest.Matchers.hasSize; + +public class CoreTestsWithSearchRuntimeFieldsIT extends ESClientYamlSuiteTestCase { + public CoreTestsWithSearchRuntimeFieldsIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { + super(testCandidate); + } + + @ParametersFactory + public static Iterable parameters() throws Exception { + return new SearchRequestRuntimeFieldTranslater().parameters(); + } + + /** + * Builds test parameters similarly to {@link ESClientYamlSuiteTestCase#createParameters()}, + * replacing all fields with runtime fields that load from {@code _source} if possible. Tests + * that configure the field in a way that are not supported by runtime fields are skipped. + */ + private static class SearchRequestRuntimeFieldTranslater extends CoreTestTranslater { + @Override + protected List> dynamicTemplates() { + return dynamicTemplatesToDisableAllFields(); + } + + @Override + protected Suite suite(ClientYamlTestCandidate candidate) { + return new Suite(candidate) { + private final Map> runtimeMappings = new HashMap<>();; + + @Override + protected boolean modifyMappingProperties(String index, Map properties) { + Map indexRuntimeMappings = new HashMap<>(properties); + if (false == runtimeifyMappingProperties(properties)) { + return false; + } + runtimeMappings.put(index, indexRuntimeMappings); + return true; + } + + @Override + protected void modifyMapping(Map mapping) { + mapping.clear(); + mapping.put("dynamic", false); + } + + @Override + protected boolean modifySearch(ApiCallSection search) { + assertThat(search.getBodies(), hasSize(1)); + Map body = search.getBodies().get(0); + Object index = body.get("index"); + if (index == null) { + // NOCOMMIT hack together a mapping based on merging everything + return false; + } + Map runtimeMapping = runtimeMappings.get(index); + if (runtimeMapping == null) { + return false; + } + search.getBodies().get(0).put("runtime_mappings", runtimeMapping); + return true; + } + }; + } + } +} diff --git a/x-pack/plugin/runtime-fields/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/rest/CoreTestsWithRuntimeFieldsIT.java b/x-pack/plugin/runtime-fields/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/rest/CoreTestsWithRuntimeFieldsIT.java deleted file mode 100644 index 0ca9981ba317f..0000000000000 --- a/x-pack/plugin/runtime-fields/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/rest/CoreTestsWithRuntimeFieldsIT.java +++ /dev/null @@ -1,259 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -package org.elasticsearch.xpack.runtimefields.rest; - -import com.carrotsearch.randomizedtesting.annotations.Name; -import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import org.elasticsearch.common.xcontent.XContentLocation; -import org.elasticsearch.index.mapper.BooleanFieldMapper; -import org.elasticsearch.index.mapper.DateFieldMapper; -import org.elasticsearch.index.mapper.IpFieldMapper; -import org.elasticsearch.index.mapper.KeywordFieldMapper; -import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; -import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; -import org.elasticsearch.test.rest.yaml.ClientYamlTestExecutionContext; -import org.elasticsearch.test.rest.yaml.ClientYamlTestResponse; -import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; -import org.elasticsearch.test.rest.yaml.section.ClientYamlTestSection; -import org.elasticsearch.test.rest.yaml.section.ClientYamlTestSuite; -import org.elasticsearch.test.rest.yaml.section.DoSection; -import org.elasticsearch.test.rest.yaml.section.ExecutableSection; -import org.elasticsearch.test.rest.yaml.section.SetupSection; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; - -import static org.hamcrest.Matchers.equalTo; - -public class CoreTestsWithRuntimeFieldsIT extends ESClientYamlSuiteTestCase { - public CoreTestsWithRuntimeFieldsIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { - super(testCandidate); - } - - /** - * Builds test parameters similarly to {@link ESClientYamlSuiteTestCase#createParameters()}, - * replacing the body of index creation commands so that fields are {@code runtime}s - * that load from {@code source} instead of their original type. Test configurations that - * do are not modified to contain runtime fields are not returned as they are tested - * elsewhere. - */ - @ParametersFactory - public static Iterable parameters() throws Exception { - Map suites = new HashMap<>(); - List result = new ArrayList<>(); - for (Object[] orig : ESClientYamlSuiteTestCase.createParameters()) { - assert orig.length == 1; - ClientYamlTestCandidate candidate = (ClientYamlTestCandidate) orig[0]; - ClientYamlTestSuite suite = suites.computeIfAbsent(candidate.getTestPath(), k -> modifiedSuite(candidate)); - if (suite == null) { - // The setup section contains an unsupported option - continue; - } - if (false == modifySection(candidate.getTestSection().getExecutableSections())) { - // The test section contains an unsupported option - continue; - } - ClientYamlTestSection modified = new ClientYamlTestSection( - candidate.getTestSection().getLocation(), - candidate.getTestSection().getName(), - candidate.getTestSection().getSkipSection(), - candidate.getTestSection().getExecutableSections() - ); - result.add(new Object[] { new ClientYamlTestCandidate(suite, modified) }); - } - return result; - } - - /** - * Modify the setup section to setup a dynamic template that replaces - * field configurations with scripts that load from source - * and replaces field configurations in {@code incides.create} - * with scripts that load from source. - */ - private static ClientYamlTestSuite modifiedSuite(ClientYamlTestCandidate candidate) { - if (false == modifySection(candidate.getSetupSection().getExecutableSections())) { - return null; - } - List setup = new ArrayList<>(candidate.getSetupSection().getExecutableSections().size() + 1); - setup.add(ADD_TEMPLATE); - setup.addAll(candidate.getSetupSection().getExecutableSections()); - return new ClientYamlTestSuite( - candidate.getApi(), - candidate.getName(), - new SetupSection(candidate.getSetupSection().getSkipSection(), setup), - candidate.getTeardownSection(), - List.of() - ); - } - - /** - * Replace field configuration in {@code indices.create} with scripts - * that load from the source. - */ - private static boolean modifySection(List executables) { - for (ExecutableSection section : executables) { - if (false == (section instanceof DoSection)) { - continue; - } - DoSection doSection = (DoSection) section; - if (false == doSection.getApiCallSection().getApi().equals("indices.create")) { - continue; - } - for (Map body : doSection.getApiCallSection().getBodies()) { - Object settings = body.get("settings"); - if (settings instanceof Map && ((Map) settings).containsKey("sort.field")) { - /* - * You can't sort the index on a runtime_keyword and it is - * hard to figure out if the sort was a runtime_keyword so - * let's just skip this test. - */ - continue; - } - Object mappings = body.get("mappings"); - if (false == (mappings instanceof Map)) { - continue; - } - Object properties = ((Map) mappings).get("properties"); - if (false == (properties instanceof Map)) { - continue; - } - for (Map.Entry property : ((Map) properties).entrySet()) { - if (false == property.getValue() instanceof Map) { - continue; - } - @SuppressWarnings("unchecked") - Map propertyMap = (Map) property.getValue(); - String name = property.getKey().toString(); - String type = Objects.toString(propertyMap.get("type")); - if ("nested".equals(type)) { - // Our loading scripts can't be made to manage nested fields so we have to skip those tests. - return false; - } - if ("false".equals(Objects.toString(propertyMap.get("doc_values")))) { - // If doc_values is false we can't emulate with scripts. `null` and `true` are fine. - continue; - } - if ("false".equals(Objects.toString(propertyMap.get("index")))) { - // If index is false we can't emulate with scripts - continue; - } - if ("true".equals(Objects.toString(propertyMap.get("store")))) { - // If store is true we can't emulate with scripts - continue; - } - if (propertyMap.containsKey("ignore_above")) { - // Scripts don't support ignore_above so we skip those fields - continue; - } - if (propertyMap.containsKey("ignore_malformed")) { - // Our source reading script doesn't emulate ignore_malformed - continue; - } - String toLoad = painlessToLoadFromSource(name, type); - if (toLoad == null) { - continue; - } - propertyMap.put("type", "runtime"); - propertyMap.put("runtime_type", type); - propertyMap.put("script", toLoad); - propertyMap.remove("store"); - propertyMap.remove("index"); - propertyMap.remove("doc_values"); - } - } - } - return true; - } - - private static String painlessToLoadFromSource(String name, String type) { - String emit = PAINLESS_TO_EMIT.get(type); - if (emit == null) { - return null; - } - StringBuilder b = new StringBuilder(); - b.append("def v = params._source['").append(name).append("'];\n"); - b.append("if (v instanceof Iterable) {\n"); - b.append(" for (def vv : ((Iterable) v)) {\n"); - b.append(" if (vv != null) {\n"); - b.append(" def value = vv;\n"); - b.append(" ").append(emit).append("\n"); - b.append(" }\n"); - b.append(" }\n"); - b.append("} else {\n"); - b.append(" if (v != null) {\n"); - b.append(" def value = v;\n"); - b.append(" ").append(emit).append("\n"); - b.append(" }\n"); - b.append("}\n"); - return b.toString(); - } - - private static final Map PAINLESS_TO_EMIT = Map.ofEntries( - Map.entry(BooleanFieldMapper.CONTENT_TYPE, "emit(Boolean.parseBoolean(value.toString()));"), - Map.entry(DateFieldMapper.CONTENT_TYPE, "emit(parse(value.toString()));"), - Map.entry( - NumberType.DOUBLE.typeName(), - "emit(value instanceof Number ? ((Number) value).doubleValue() : Double.parseDouble(value.toString()));" - ), - Map.entry(KeywordFieldMapper.CONTENT_TYPE, "emit(value.toString());"), - Map.entry(IpFieldMapper.CONTENT_TYPE, "emit(value.toString());"), - Map.entry( - NumberType.LONG.typeName(), - "emit(value instanceof Number ? ((Number) value).longValue() : Long.parseLong(value.toString()));" - ) - ); - - private static final ExecutableSection ADD_TEMPLATE = new ExecutableSection() { - @Override - public XContentLocation getLocation() { - return new XContentLocation(-1, -1); - } - - @Override - public void execute(ClientYamlTestExecutionContext executionContext) throws IOException { - Map params = Map.of("name", "convert_to_source_only", "create", "true"); - List> dynamicTemplates = new ArrayList<>(); - for (String type : PAINLESS_TO_EMIT.keySet()) { - if (type.equals("ip")) { - // There isn't a dynamic template to pick up ips. They'll just look like strings. - continue; - } - Map mapping = Map.ofEntries( - Map.entry("type", "runtime"), - Map.entry("runtime_type", type), - Map.entry("script", painlessToLoadFromSource("{name}", type)) - ); - if (type.contentEquals("keyword")) { - /* - * For "string"-type dynamic mappings emulate our default - * behavior with a top level text field and a `.keyword` - * multi-field. But instead of the default, use a runtime - * field for the multi-field. - */ - mapping = Map.of("type", "text", "fields", Map.of("keyword", mapping)); - dynamicTemplates.add(Map.of(type, Map.of("match_mapping_type", "string", "mapping", mapping))); - } else { - dynamicTemplates.add(Map.of(type, Map.of("match_mapping_type", type, "mapping", mapping))); - } - } - List> bodies = List.of( - Map.ofEntries( - Map.entry("index_patterns", "*"), - Map.entry("priority", Integer.MAX_VALUE - 1), - Map.entry("template", Map.of("settings", Map.of(), "mappings", Map.of("dynamic_templates", dynamicTemplates))) - ) - ); - ClientYamlTestResponse response = executionContext.callApi("indices.put_index_template", params, bodies, Map.of()); - assertThat(response.getStatusCode(), equalTo(200)); - // There are probably some warning about overlapping templates. Ignore them. - } - }; -} diff --git a/x-pack/plugin/runtime-fields/qa/src/main/java/org/elasticsearch/xpack/runtimefields/test/CoreTestTranslater.java b/x-pack/plugin/runtime-fields/qa/src/main/java/org/elasticsearch/xpack/runtimefields/test/CoreTestTranslater.java new file mode 100644 index 0000000000000..5aa345984461f --- /dev/null +++ b/x-pack/plugin/runtime-fields/qa/src/main/java/org/elasticsearch/xpack/runtimefields/test/CoreTestTranslater.java @@ -0,0 +1,300 @@ +package org.elasticsearch.xpack.runtimefields.test; + +import org.elasticsearch.common.xcontent.XContentLocation; +import org.elasticsearch.index.mapper.BooleanFieldMapper; +import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.index.mapper.IpFieldMapper; +import org.elasticsearch.index.mapper.KeywordFieldMapper; +import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; +import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; +import org.elasticsearch.test.rest.yaml.ClientYamlTestExecutionContext; +import org.elasticsearch.test.rest.yaml.ClientYamlTestResponse; +import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; +import org.elasticsearch.test.rest.yaml.section.ApiCallSection; +import org.elasticsearch.test.rest.yaml.section.ClientYamlTestSection; +import org.elasticsearch.test.rest.yaml.section.ClientYamlTestSuite; +import org.elasticsearch.test.rest.yaml.section.DoSection; +import org.elasticsearch.test.rest.yaml.section.ExecutableSection; +import org.elasticsearch.test.rest.yaml.section.SetupSection; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import static org.hamcrest.Matchers.equalTo; +import static org.junit.Assert.assertThat; + +/** + * Builds test parameters similarly to {@link ESClientYamlSuiteTestCase#createParameters()}, + * replacing all fields with runtime fields that load from {@code _source} if possible. Tests + * that configure the field in a way that are not supported by runtime fields are skipped. + */ +public abstract class CoreTestTranslater { + public Iterable parameters() throws Exception { + Map suites = new HashMap<>(); + List result = new ArrayList<>(); + for (Object[] orig : ESClientYamlSuiteTestCase.createParameters()) { + assert orig.length == 1; + ClientYamlTestCandidate candidate = (ClientYamlTestCandidate) orig[0]; + Suite suite = suites.computeIfAbsent(candidate.getTestPath(), k -> suite(candidate)); + if (suite.modified == null) { + // The setup section contains an unsupported option + continue; + } + if (false == suite.modifySections(candidate.getTestSection().getExecutableSections())) { + // The test section contains an unsupported option + continue; + } + ClientYamlTestSection modified = new ClientYamlTestSection( + candidate.getTestSection().getLocation(), + candidate.getTestSection().getName(), + candidate.getTestSection().getSkipSection(), + candidate.getTestSection().getExecutableSections() + ); + result.add(new Object[] { new ClientYamlTestCandidate(suite.modified, modified) }); + } + return result; + } + + protected abstract Suite suite(ClientYamlTestCandidate candidate); + + private static String painlessToLoadFromSource(String name, String type) { + String emit = PAINLESS_TO_EMIT.get(type); + if (emit == null) { + return null; + } + StringBuilder b = new StringBuilder(); + b.append("def v = params._source['").append(name).append("'];\n"); + b.append("if (v instanceof Iterable) {\n"); + b.append(" for (def vv : ((Iterable) v)) {\n"); + b.append(" if (vv != null) {\n"); + b.append(" def value = vv;\n"); + b.append(" ").append(emit).append("\n"); + b.append(" }\n"); + b.append(" }\n"); + b.append("} else {\n"); + b.append(" if (v != null) {\n"); + b.append(" def value = v;\n"); + b.append(" ").append(emit).append("\n"); + b.append(" }\n"); + b.append("}\n"); + return b.toString(); + } + + private static final Map PAINLESS_TO_EMIT = Map.ofEntries( + Map.entry(BooleanFieldMapper.CONTENT_TYPE, "emit(Boolean.parseBoolean(value.toString()));"), + Map.entry(DateFieldMapper.CONTENT_TYPE, "emit(parse(value.toString()));"), + Map.entry( + NumberType.DOUBLE.typeName(), + "emit(value instanceof Number ? ((Number) value).doubleValue() : Double.parseDouble(value.toString()));" + ), + Map.entry(KeywordFieldMapper.CONTENT_TYPE, "emit(value.toString());"), + Map.entry(IpFieldMapper.CONTENT_TYPE, "emit(value.toString());"), + Map.entry( + NumberType.LONG.typeName(), + "emit(value instanceof Number ? ((Number) value).longValue() : Long.parseLong(value.toString()));" + ) + ); + + protected abstract List> dynamicTemplates(); + + protected static List> dynamicTemplatesToDisableAllFields() { + return List.of(Map.of("disable_all", Map.of("enabled", false))); + } + + protected static List> dynamicTemplatesToAddRuntimeFieldsToMappings() { + List> dynamicTemplates = new ArrayList<>(); + for (String type : PAINLESS_TO_EMIT.keySet()) { + if (type.equals("ip")) { + // There isn't a dynamic template to pick up ips. They'll just look like strings. + continue; + } + Map mapping = Map.ofEntries( + Map.entry("type", "runtime"), + Map.entry("runtime_type", type), + Map.entry("script", painlessToLoadFromSource("{name}", type)) + ); + if (type.contentEquals("keyword")) { + /* + * For "string"-type dynamic mappings emulate our default + * behavior with a top level text field and a `.keyword` + * multi-field. But instead of the default, use a runtime + * field for the multi-field. + */ + mapping = Map.of("type", "text", "fields", Map.of("keyword", mapping)); + dynamicTemplates.add(Map.of(type, Map.of("match_mapping_type", "string", "mapping", mapping))); + } else { + dynamicTemplates.add(Map.of(type, Map.of("match_mapping_type", type, "mapping", mapping))); + } + } + return dynamicTemplates; + } + + private final ExecutableSection addTemplate() { + return new ExecutableSection() { + @Override + public XContentLocation getLocation() { + return new XContentLocation(-1, -1); + } + + @Override + public void execute(ClientYamlTestExecutionContext executionContext) throws IOException { + Map params = Map.of("name", "convert_to_source_only", "create", "true"); + List> bodies = List.of( + Map.ofEntries( + Map.entry("index_patterns", "*"), + Map.entry("priority", Integer.MAX_VALUE - 1), + Map.entry("template", Map.of("settings", Map.of(), "mappings", Map.of("dynamic_templates", dynamicTemplates()))) + ) + ); + ClientYamlTestResponse response = executionContext.callApi("indices.put_index_template", params, bodies, Map.of()); + assertThat(response.getStatusCode(), equalTo(200)); + // There are probably some warning about overlapping templates. Ignore them. + } + }; + } + + /** + * NOCOMMIT write me + */ + protected abstract class Suite { + private final ClientYamlTestSuite modified; + + public Suite(ClientYamlTestCandidate candidate) { + if (false == modifySections(candidate.getSetupSection().getExecutableSections())) { + modified = null; + return; + } + /* + * Modify the setup section to rewrite and create index commands and + * to add a dynamic template that sets up any dynamic indices how we + * expect them. + */ + List setup = new ArrayList<>(candidate.getSetupSection().getExecutableSections().size() + 1); + setup.add(addTemplate()); + setup.addAll(candidate.getSetupSection().getExecutableSections()); + modified = new ClientYamlTestSuite( + candidate.getApi(), + candidate.getName(), + new SetupSection(candidate.getSetupSection().getSkipSection(), setup), + candidate.getTeardownSection(), + List.of() + ); + } + + /** + * Replace field configuration in {@code indices.create} with scripts + * that load from the source. + * + * @return true if the section is appropriate for testing with runtime fields + */ + public final boolean modifySections(List executables) { + for (ExecutableSection section : executables) { + if (false == (section instanceof DoSection)) { + continue; + } + DoSection doSection = (DoSection) section; + if (doSection.getApiCallSection().getApi().equals("indices.create")) { + if (false == modifyCreateIndex(doSection.getApiCallSection())) { + return false; + } + } + if (doSection.getApiCallSection().getApi().equals("search")) { + if (false == modifySearch(doSection.getApiCallSection())) { + return false; + } + } + } + return true; + } + + /** + * Modify a test search request. + */ + protected abstract boolean modifySearch(ApiCallSection search); + + private final boolean modifyCreateIndex(ApiCallSection createIndex) { + String index = createIndex.getParams().get("index"); + for (Map body : createIndex.getBodies()) { + Object settings = body.get("settings"); + if (settings instanceof Map && ((Map) settings).containsKey("sort.field")) { + /* + * You can't sort the index on a runtime_keyword and it is + * hard to figure out if the sort was a runtime_keyword so + * let's just skip this test. + */ + continue; + } + Object mapping = body.get("mappings"); + if (false == (mapping instanceof Map)) { + continue; + } + @SuppressWarnings("unchecked") + Map mappingMap = (Map) mapping; + Object properties = mappingMap.get("properties"); + if (false == (properties instanceof Map)) { + continue; + } + if (false == modifyMappingProperties(index, (Map) properties)) { + return false; + } + modifyMapping(mappingMap); + } + return true; + } + + protected abstract boolean modifyMappingProperties(String index, Map properties); + + protected abstract void modifyMapping(Map mapping); + + protected final boolean runtimeifyMappingProperties(Map properties) { + for (Map.Entry property : properties.entrySet()) { + if (false == property.getValue() instanceof Map) { + continue; + } + @SuppressWarnings("unchecked") + Map propertyMap = (Map) property.getValue(); + String name = property.getKey().toString(); + String type = Objects.toString(propertyMap.get("type")); + if ("nested".equals(type)) { + // Our loading scripts can't be made to manage nested fields so we have to skip those tests. + return false; + } + if ("false".equals(Objects.toString(propertyMap.get("doc_values")))) { + // If doc_values is false we can't emulate with scripts. So we keep the old definition. `null` and `true` are fine. + continue; + } + if ("false".equals(Objects.toString(propertyMap.get("index")))) { + // If index is false we can't emulate with scripts + continue; + } + if ("true".equals(Objects.toString(propertyMap.get("store")))) { + // If store is true we can't emulate with scripts + continue; + } + if (propertyMap.containsKey("ignore_above")) { + // Scripts don't support ignore_above so we skip those fields + continue; + } + if (propertyMap.containsKey("ignore_malformed")) { + // Our source reading script doesn't emulate ignore_malformed + continue; + } + String toLoad = painlessToLoadFromSource(name, type); + if (toLoad == null) { + continue; + } + propertyMap.put("type", "runtime"); + propertyMap.put("runtime_type", type); + propertyMap.put("script", toLoad); + propertyMap.remove("store"); + propertyMap.remove("index"); + propertyMap.remove("doc_values"); + } + return true; + } + } +} From 730cf922c553b19c5934adf2f5cfb8305047887c Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Mon, 2 Nov 2020 18:10:59 -0500 Subject: [PATCH 03/36] WIP --- .../elasticsearch/client/RequestLogger.java | 4 +- .../mapped/CoreWithMappedRuntimeFieldsIT.java | 42 +++++--- .../qa/core-with-search/build.gradle | 7 +- .../CoreTestsWithSearchRuntimeFieldsIT.java | 99 ++++++++++++++----- .../test/CoreTestTranslater.java | 78 +++++++++------ 5 files changed, 157 insertions(+), 73 deletions(-) diff --git a/client/rest/src/main/java/org/elasticsearch/client/RequestLogger.java b/client/rest/src/main/java/org/elasticsearch/client/RequestLogger.java index 7c56a7edf97a9..41271ab69d35c 100644 --- a/client/rest/src/main/java/org/elasticsearch/client/RequestLogger.java +++ b/client/rest/src/main/java/org/elasticsearch/client/RequestLogger.java @@ -65,7 +65,7 @@ static void logResponse(Log logger, HttpUriRequest request, HttpHost host, HttpR logger.warn(buildWarningMessage(request, host, warnings)); } } - if (tracer.isTraceEnabled()) { + if (tracer.isErrorEnabled()) { String requestLine; try { requestLine = buildTraceRequest(request, host); @@ -80,7 +80,7 @@ static void logResponse(Log logger, HttpUriRequest request, HttpHost host, HttpR responseLine = ""; tracer.trace("error while reading response for trace purposes", e); } - tracer.trace(requestLine + '\n' + responseLine); + tracer.error(requestLine + '\n' + responseLine); } } diff --git a/x-pack/plugin/runtime-fields/qa/core-with-mapped/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/mapped/CoreWithMappedRuntimeFieldsIT.java b/x-pack/plugin/runtime-fields/qa/core-with-mapped/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/mapped/CoreWithMappedRuntimeFieldsIT.java index ea46c4ceaf023..52f5df557a939 100644 --- a/x-pack/plugin/runtime-fields/qa/core-with-mapped/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/mapped/CoreWithMappedRuntimeFieldsIT.java +++ b/x-pack/plugin/runtime-fields/qa/core-with-mapped/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/mapped/CoreWithMappedRuntimeFieldsIT.java @@ -14,9 +14,14 @@ import org.elasticsearch.test.rest.yaml.section.ApiCallSection; import org.elasticsearch.xpack.runtimefields.test.CoreTestTranslater; -import java.util.List; +import java.util.HashMap; import java.util.Map; +/** + * Runs elasticsearch's core rest tests replacing all field mappings with runtime fields + * that load from {@code _source}. Tests that configure the field in a way that are not + * supported by runtime fields are skipped. + */ public class CoreWithMappedRuntimeFieldsIT extends ESClientYamlSuiteTestCase { public CoreWithMappedRuntimeFieldsIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { super(testCandidate); @@ -27,28 +32,35 @@ public static Iterable parameters() throws Exception { return new MappingRuntimeFieldTranslater().parameters(); } - /** - * Builds test parameters similarly to {@link ESClientYamlSuiteTestCase#createParameters()}, - * replacing all fields with runtime fields that load from {@code _source} if possible. Tests - * that configure the field in a way that are not supported by runtime fields are skipped. - */ private static class MappingRuntimeFieldTranslater extends CoreTestTranslater { @Override - protected List> dynamicTemplates() { - return dynamicTemplatesToAddRuntimeFieldsToMappings(); + protected Map indexTemplate() { + return indexTemplateToAddRuntimeFieldsToMappings(); } @Override protected Suite suite(ClientYamlTestCandidate candidate) { return new Suite(candidate) { @Override - protected boolean modifyMappingProperties(String index, Map properties) { - return runtimeifyMappingProperties(properties); - } - - @Override - protected void modifyMapping(Map mapping) { - // The top level mapping is fine for runtime fields defined in the mapping + protected boolean modifyMapping(String index, Map mapping) { + Object properties = mapping.get("properties"); + if (properties == null || false == (properties instanceof Map)) { + return true; + } + @SuppressWarnings("unchecked") + Map propertiesMap = (Map) properties; + Map newProperties = new HashMap<>(propertiesMap.size()); + Map> runtimeProperties = new HashMap<>(propertiesMap.size()); + if (false == runtimeifyMappingProperties(propertiesMap, newProperties, runtimeProperties)) { + return false; + } + for (Map.Entry> runtimeProperty : runtimeProperties.entrySet()) { + runtimeProperty.getValue().put("runtime_type", runtimeProperty.getValue().get("type")); + runtimeProperty.getValue().put("type", "runtime"); + newProperties.put(runtimeProperty.getKey(), runtimeProperty.getValue()); + } + mapping.put("properties", newProperties); + return true; } @Override diff --git a/x-pack/plugin/runtime-fields/qa/core-with-search/build.gradle b/x-pack/plugin/runtime-fields/qa/core-with-search/build.gradle index 7af3a9d6886da..25d1b6d47bce6 100644 --- a/x-pack/plugin/runtime-fields/qa/core-with-search/build.gradle +++ b/x-pack/plugin/runtime-fields/qa/core-with-search/build.gradle @@ -23,9 +23,9 @@ yamlRestTest { systemProperty 'tests.rest.suite', [ 'async_search', - 'field_caps', - 'graph', - 'msearch', +// 'field_caps', These two don't support runtime fields on the request. Should they? +// 'graph', +// 'msearch', This one just doesn't have infrastructure to hack the runtime fields into place 'search', 'search.aggregation', 'search.highlight', @@ -41,6 +41,7 @@ yamlRestTest { 'field_caps/30_filter/Field caps with index filter', // We don't support filtering field caps on runtime fields. What should we do? 'search.aggregation/10_histogram/*', // runtime doesn't support sub-fields. Maybe it should? 'search/140_pre_filter_search_shards/pre_filter_shard_size with shards that have no hit', + 'search/240_date_nanos/date_nanos requires dates after 1970 and before 2262', // This isn't a search test and should probably move /////// TO FIX /////// /////// NOT SUPPORTED /////// diff --git a/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java b/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java index 43c6c7630721a..1c40376c7cedd 100644 --- a/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java +++ b/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java @@ -9,75 +9,128 @@ import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.apache.logging.log4j.LogManager; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; import org.elasticsearch.test.rest.yaml.section.ApiCallSection; +import org.elasticsearch.test.rest.yaml.section.ExecutableSection; import org.elasticsearch.xpack.runtimefields.test.CoreTestTranslater; import java.util.HashMap; import java.util.List; import java.util.Map; +import static java.util.Collections.unmodifiableMap; import static org.hamcrest.Matchers.hasSize; +/** + * Runs elasticsearch's core rest tests disabling all mappings and replacing them + * with runtime fields defined on the search request that load from {@code _source}. Tests + * that configure the field in a way that are not supported by runtime fields are skipped. + */ public class CoreTestsWithSearchRuntimeFieldsIT extends ESClientYamlSuiteTestCase { public CoreTestsWithSearchRuntimeFieldsIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { super(testCandidate); } + @Override + protected boolean randomizeContentType() { // NOCOMMIT remove me + return false; + } + @ParametersFactory public static Iterable parameters() throws Exception { return new SearchRequestRuntimeFieldTranslater().parameters(); } - /** - * Builds test parameters similarly to {@link ESClientYamlSuiteTestCase#createParameters()}, - * replacing all fields with runtime fields that load from {@code _source} if possible. Tests - * that configure the field in a way that are not supported by runtime fields are skipped. - */ private static class SearchRequestRuntimeFieldTranslater extends CoreTestTranslater { @Override - protected List> dynamicTemplates() { - return dynamicTemplatesToDisableAllFields(); + protected Map indexTemplate() { + return indexTemplateToDisableAllFields(); } @Override protected Suite suite(ClientYamlTestCandidate candidate) { return new Suite(candidate) { - private final Map> runtimeMappings = new HashMap<>();; + private Map> runtimeMappingsAfterSetup; + private Map> runtimeMappings; @Override - protected boolean modifyMappingProperties(String index, Map properties) { - Map indexRuntimeMappings = new HashMap<>(properties); - if (false == runtimeifyMappingProperties(properties)) { - return false; + public boolean modifySections(List executables) { + if (runtimeMappingsAfterSetup == null) { + if (candidate.toString().contains("50_filter")) { + LogManager.getLogger().error("init"); + } + // We're modifying the setup section + runtimeMappings = new HashMap<>(); + if (false == super.modifySections(executables)) { + return false; + } + runtimeMappingsAfterSetup = unmodifiableMap(runtimeMappings); + runtimeMappings = null; + return true; } - runtimeMappings.put(index, indexRuntimeMappings); - return true; + runtimeMappings = new HashMap<>(runtimeMappingsAfterSetup); + return super.modifySections(executables); } @Override - protected void modifyMapping(Map mapping) { - mapping.clear(); + protected boolean modifyMapping(String index, Map mapping) { + Object properties = mapping.get("properties"); + if (properties == null || false == (properties instanceof Map)) { + return true; + } mapping.put("dynamic", false); + @SuppressWarnings("unchecked") + Map propertiesMap = (Map) properties; + Map untouchedMapping = new HashMap<>(); + Map> runtimeMapping = new HashMap<>(); + if (false == runtimeifyMappingProperties(propertiesMap, untouchedMapping, runtimeMapping)) { + return false; + } + mapping.put("properties", untouchedMapping); + if (candidate.toString().contains("50_filter")) { + LogManager.getLogger().error("untouched " + untouchedMapping); + LogManager.getLogger().error("runtime " + runtimeMapping); + } + runtimeMappings.put(index, runtimeMapping); + return true; } @Override protected boolean modifySearch(ApiCallSection search) { - assertThat(search.getBodies(), hasSize(1)); - Map body = search.getBodies().get(0); - Object index = body.get("index"); - if (index == null) { - // NOCOMMIT hack together a mapping based on merging everything - return false; + if (candidate.toString().contains("50_filter")) { + LogManager.getLogger().error("toString " + candidate); + LogManager.getLogger().error("runtimeMappingsAfterSetup " + runtimeMappingsAfterSetup); + LogManager.getLogger().error("runtimeMappings " + runtimeMappings); } - Map runtimeMapping = runtimeMappings.get(index); + Map body; + if (search.getBodies().isEmpty()) { + body = new HashMap<>(); + search.addBody(body); + } else { + body = search.getBodies().get(0); + assertThat(search.getBodies(), hasSize(1)); + } + Object index = body.get("index"); + Map runtimeMapping = runtimeMappings(index); if (runtimeMapping == null) { return false; } search.getBodies().get(0).put("runtime_mappings", runtimeMapping); return true; } + + private Map runtimeMappings(Object index) { + if (index != null) { + return runtimeMappings.get(index); + } + // No mapping index specified in the search, if there is just one index we can just us it + if (runtimeMappings.size() == 1) { + return runtimeMappings.values().iterator().next(); + } + return null; + } }; } } diff --git a/x-pack/plugin/runtime-fields/qa/src/main/java/org/elasticsearch/xpack/runtimefields/test/CoreTestTranslater.java b/x-pack/plugin/runtime-fields/qa/src/main/java/org/elasticsearch/xpack/runtimefields/test/CoreTestTranslater.java index 5aa345984461f..3a6130fce5636 100644 --- a/x-pack/plugin/runtime-fields/qa/src/main/java/org/elasticsearch/xpack/runtimefields/test/CoreTestTranslater.java +++ b/x-pack/plugin/runtime-fields/qa/src/main/java/org/elasticsearch/xpack/runtimefields/test/CoreTestTranslater.java @@ -39,7 +39,7 @@ public Iterable parameters() throws Exception { for (Object[] orig : ESClientYamlSuiteTestCase.createParameters()) { assert orig.length == 1; ClientYamlTestCandidate candidate = (ClientYamlTestCandidate) orig[0]; - Suite suite = suites.computeIfAbsent(candidate.getTestPath(), k -> suite(candidate)); + Suite suite = suites.computeIfAbsent(candidate.getSuitePath(), k -> suite(candidate)); if (suite.modified == null) { // The setup section contains an unsupported option continue; @@ -99,13 +99,13 @@ private static String painlessToLoadFromSource(String name, String type) { ) ); - protected abstract List> dynamicTemplates(); + protected abstract Map indexTemplate(); - protected static List> dynamicTemplatesToDisableAllFields() { - return List.of(Map.of("disable_all", Map.of("enabled", false))); + protected static Map indexTemplateToDisableAllFields() { + return Map.of("settings", Map.of(), "mappings", Map.of("dynamic", false)); } - protected static List> dynamicTemplatesToAddRuntimeFieldsToMappings() { + protected static Map indexTemplateToAddRuntimeFieldsToMappings() { List> dynamicTemplates = new ArrayList<>(); for (String type : PAINLESS_TO_EMIT.keySet()) { if (type.equals("ip")) { @@ -130,10 +130,10 @@ protected static List> dynamicTemplatesToAddRuntimeFieldsToM dynamicTemplates.add(Map.of(type, Map.of("match_mapping_type", type, "mapping", mapping))); } } - return dynamicTemplates; + return Map.of("settings", Map.of(), "mappings", Map.of("dynamic_templates", dynamicTemplates)); } - private final ExecutableSection addTemplate() { + private final ExecutableSection addIndexTemplate() { return new ExecutableSection() { @Override public XContentLocation getLocation() { @@ -142,12 +142,12 @@ public XContentLocation getLocation() { @Override public void execute(ClientYamlTestExecutionContext executionContext) throws IOException { - Map params = Map.of("name", "convert_to_source_only", "create", "true"); + Map params = Map.of("name", "hack_dynamic_mappings", "create", "true"); List> bodies = List.of( Map.ofEntries( Map.entry("index_patterns", "*"), Map.entry("priority", Integer.MAX_VALUE - 1), - Map.entry("template", Map.of("settings", Map.of(), "mappings", Map.of("dynamic_templates", dynamicTemplates()))) + Map.entry("template", indexTemplate()) ) ); ClientYamlTestResponse response = executionContext.callApi("indices.put_index_template", params, bodies, Map.of()); @@ -174,7 +174,7 @@ public Suite(ClientYamlTestCandidate candidate) { * expect them. */ List setup = new ArrayList<>(candidate.getSetupSection().getExecutableSections().size() + 1); - setup.add(addTemplate()); + setup.add(addIndexTemplate()); setup.addAll(candidate.getSetupSection().getExecutableSections()); modified = new ClientYamlTestSuite( candidate.getApi(), @@ -191,18 +191,19 @@ public Suite(ClientYamlTestCandidate candidate) { * * @return true if the section is appropriate for testing with runtime fields */ - public final boolean modifySections(List executables) { + public boolean modifySections(List executables) { for (ExecutableSection section : executables) { if (false == (section instanceof DoSection)) { continue; } DoSection doSection = (DoSection) section; - if (doSection.getApiCallSection().getApi().equals("indices.create")) { + String api = doSection.getApiCallSection().getApi(); + if (api.equals("indices.create")) { if (false == modifyCreateIndex(doSection.getApiCallSection())) { return false; } } - if (doSection.getApiCallSection().getApi().equals("search")) { + if (api.equals("search") || api.equals("async_search.submit")) { if (false == modifySearch(doSection.getApiCallSection())) { return false; } @@ -234,25 +235,28 @@ private final boolean modifyCreateIndex(ApiCallSection createIndex) { } @SuppressWarnings("unchecked") Map mappingMap = (Map) mapping; - Object properties = mappingMap.get("properties"); - if (false == (properties instanceof Map)) { - continue; - } - if (false == modifyMappingProperties(index, (Map) properties)) { + if (false == modifyMapping(index, mappingMap)) { return false; } - modifyMapping(mappingMap); } return true; } - protected abstract boolean modifyMappingProperties(String index, Map properties); + protected abstract boolean modifyMapping(String index, Map mapping); - protected abstract void modifyMapping(Map mapping); - - protected final boolean runtimeifyMappingProperties(Map properties) { - for (Map.Entry property : properties.entrySet()) { + /** + * Modify the provided map in place, translating all fields into + * runtime fields that load from source. + * @return true if this mapping supports runtime fields, false otherwise + */ + protected final boolean runtimeifyMappingProperties( + Map properties, + Map untouchedProperties, + Map> runtimeProperties + ) { + for (Map.Entry property : properties.entrySet()) { if (false == property.getValue() instanceof Map) { + untouchedProperties.put(property.getKey(), property.getValue()); continue; } @SuppressWarnings("unchecked") @@ -265,35 +269,49 @@ protected final boolean runtimeifyMappingProperties(Map properties) { } if ("false".equals(Objects.toString(propertyMap.get("doc_values")))) { // If doc_values is false we can't emulate with scripts. So we keep the old definition. `null` and `true` are fine. + untouchedProperties.put(property.getKey(), property.getValue()); continue; } if ("false".equals(Objects.toString(propertyMap.get("index")))) { // If index is false we can't emulate with scripts + untouchedProperties.put(property.getKey(), property.getValue()); continue; } if ("true".equals(Objects.toString(propertyMap.get("store")))) { // If store is true we can't emulate with scripts + untouchedProperties.put(property.getKey(), property.getValue()); continue; } if (propertyMap.containsKey("ignore_above")) { // Scripts don't support ignore_above so we skip those fields + untouchedProperties.put(property.getKey(), property.getValue()); continue; } if (propertyMap.containsKey("ignore_malformed")) { // Our source reading script doesn't emulate ignore_malformed + untouchedProperties.put(property.getKey(), property.getValue()); continue; } String toLoad = painlessToLoadFromSource(name, type); if (toLoad == null) { + untouchedProperties.put(property.getKey(), property.getValue()); continue; } - propertyMap.put("type", "runtime"); - propertyMap.put("runtime_type", type); - propertyMap.put("script", toLoad); - propertyMap.remove("store"); - propertyMap.remove("index"); - propertyMap.remove("doc_values"); + Map runtimeConfig = new HashMap<>(propertyMap); + runtimeConfig.put("script", toLoad); + runtimeConfig.remove("store"); + runtimeConfig.remove("index"); + runtimeConfig.remove("doc_values"); + runtimeProperties.put(property.getKey(), runtimeConfig); } + /* + * Its tempting to return false here if we didn't make any runtime + * fields, skipping the test. But that would cause us to skip any + * test uses dynamic mappings. Disaster! Instead we use a dynamic + * template to make the dynamic mappings into runtime fields too. + * The downside is that we can run tests that don't use runtime + * fields at all. That's unfortunate, but its ok. + */ return true; } } From 80fc471e115fffd5abd7be51e180754b4ebff994 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 4 Nov 2020 09:01:38 -0500 Subject: [PATCH 04/36] Some of my own tests --- .../elasticsearch/client/RequestLogger.java | 4 +- .../index/query/QueryShardContext.java | 18 ++++- .../test/runtime_fields/10_keyword.yml | 76 +++++++++++++++++++ 3 files changed, 95 insertions(+), 3 deletions(-) diff --git a/client/rest/src/main/java/org/elasticsearch/client/RequestLogger.java b/client/rest/src/main/java/org/elasticsearch/client/RequestLogger.java index 41271ab69d35c..7c56a7edf97a9 100644 --- a/client/rest/src/main/java/org/elasticsearch/client/RequestLogger.java +++ b/client/rest/src/main/java/org/elasticsearch/client/RequestLogger.java @@ -65,7 +65,7 @@ static void logResponse(Log logger, HttpUriRequest request, HttpHost host, HttpR logger.warn(buildWarningMessage(request, host, warnings)); } } - if (tracer.isErrorEnabled()) { + if (tracer.isTraceEnabled()) { String requestLine; try { requestLine = buildTraceRequest(request, host); @@ -80,7 +80,7 @@ static void logResponse(Log logger, HttpUriRequest request, HttpHost host, HttpR responseLine = ""; tracer.trace("error while reading response for trace purposes", e); } - tracer.error(requestLine + '\n' + responseLine); + tracer.trace(requestLine + '\n' + responseLine); } } diff --git a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java index ff350edba5787..e265299863897 100644 --- a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java @@ -35,6 +35,7 @@ import org.elasticsearch.common.TriFunction; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.lucene.search.Queries; +import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentParser; @@ -69,6 +70,7 @@ import java.io.IOException; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -299,7 +301,21 @@ public boolean hasMappings() { * type then the fields will be returned with a type prefix. */ public Set simpleMatchToIndexNames(String pattern) { - return mapperService.simpleMatchToFullName(pattern); + // TODO remove the duplication with MapperService and FieldTypeLookup + if (runtimeMappings.isEmpty()) { + return mapperService.simpleMatchToFullName(pattern); + } + if (Regex.isSimpleMatchPattern(pattern) == false) { + // no wildcards + return Collections.singleton(pattern); + } + Set matches = new HashSet<>(mapperService.simpleMatchToFullName(pattern)); + for (String name : runtimeMappings.keySet()) { + if (Regex.simpleMatch(pattern, name)) { + matches.add(name); + } + } + return matches; } /** diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_keyword.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_keyword.yml index 840a0906dc1db..d895141ffbe0c 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_keyword.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_keyword.yml @@ -360,3 +360,79 @@ setup: - match: { aggregations.to-users.users.hits.hits.2._index: test } - match: { aggregations.to-users.users.hits.hits.2._nested.field: users } - match: { aggregations.to-users.users.hits.hits.2._nested.offset: 1 } + +--- +"fetch defined on search request": + - do: + search: + index: sensor + body: + runtime_mappings: + voltage.rating: + type: keyword + script: | + double v = doc['voltage'].value; + if (v < 4.8) { + emit('low'); + } else if (v > 5.2) { + emit('high'); + } else { + emit('ok'); + } + fields: [voltage.rating] + sort: timestamp + - match: {hits.total.value: 6} + - match: {hits.hits.0._source.voltage: 4.0} + - match: {hits.hits.0.fields.voltage.rating: low} + + +--- +"match defined on search request": + - do: + search: + index: sensor + body: + runtime_mappings: + voltage.rating: + type: keyword + script: | + double v = doc['voltage'].value; + if (v < 4.8) { + emit('low'); + } else if (v > 5.2) { + emit('high'); + } else { + emit('ok'); + } + query: + match: + voltage.rating: ok + sort: timestamp + - match: {hits.total.value: 2} + - match: {hits.hits.0._source.voltage: 5.1} + +--- +"search glob defined on search request": + - do: + search: + index: sensor + body: + runtime_mappings: + voltage.rating: + type: keyword + script: | + double v = doc['voltage'].value; + if (v < 4.8) { + emit('low'); + } else if (v > 5.2) { + emit('high'); + } else { + emit('ok'); + } + query: + simple_query_string: + fields: [voltage.*] + query: ok + sort: timestamp + - match: {hits.total.value: 2} + - match: {hits.hits.0._source.voltage: 5.1} From a94ccd6d03e96d0d7e1cf8ad48403ae293cfe33d Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 4 Nov 2020 10:11:59 -0500 Subject: [PATCH 05/36] WIP --- .../index/query/QueryShardContextTests.java | 91 +++++++++++++++---- 1 file changed, 73 insertions(+), 18 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java b/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java index c4d63e051230d..7374c89e17ab4 100644 --- a/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java @@ -183,9 +183,24 @@ public void testIndexSortedOnField() { IndexSettings indexSettings = new IndexSettings(indexMetadata, settings); QueryShardContext context = new QueryShardContext( - 0, indexSettings, BigArrays.NON_RECYCLING_INSTANCE, null, null, - null, null, null, NamedXContentRegistry.EMPTY, new NamedWriteableRegistry(Collections.emptyList()), - null, null, () -> 0L, null, null, () -> true, null); + 0, + indexSettings, + BigArrays.NON_RECYCLING_INSTANCE, + null, + null, + mock(MapperService.class), + null, + null, + NamedXContentRegistry.EMPTY, + new NamedWriteableRegistry(Collections.emptyList()), + null, + null, + () -> 0L, + null, + null, + () -> true, + null + ); assertTrue(context.indexSortedOnField("sort_field")); assertFalse(context.indexSortedOnField("second_sort_field")); @@ -289,12 +304,62 @@ public void testFielddataLookupOneFieldManyReferences() throws IOException { assertEquals(List.of(expectedFirstDoc.toString(), expectedSecondDoc.toString()), collect("field", queryShardContext)); } + public void testRuntimeFields() throws IOException { + MapperService mapperService = mockMapperService("test"); + Map runtimeFields = Map.ofEntries( + Map.entry("cat", Map.of("type", "keyword", "script", "emit('cat')")), + Map.entry("dog", Map.of("type", "keyword", "script", "emit('dog')")), + Map.entry("catdog", Map.of("type", "keyword", "script", "emit(doc['cat'].value + doc['dog'].value)")) + ); + QueryShardContext qsc = new QueryShardContext( + 0, + mapperService.getIndexSettings(), + BigArrays.NON_RECYCLING_INSTANCE, + null, + (mappedFieldType, idxName, searchLookup) -> mappedFieldType.fielddataBuilder(idxName, searchLookup).build(null, null), + mapperService, + null, + null, + NamedXContentRegistry.EMPTY, + new NamedWriteableRegistry(Collections.emptyList()), + null, + null, + () -> 0, + "test", + null, + () -> true, + null, + runtimeFields + ); + assertTrue(qsc.isFieldMapped("cat")); + assertThat(qsc.getFieldType(name)) + } + public static QueryShardContext createQueryShardContext(String indexUuid, String clusterAlias) { return createQueryShardContext(indexUuid, clusterAlias, null); } - private static QueryShardContext createQueryShardContext(String indexUuid, String clusterAlias, - TriFunction runtimeDocValues) { + private static QueryShardContext createQueryShardContext( + String indexUuid, + String clusterAlias, + TriFunction runtimeDocValues + ) { + MapperService mapperService = mockMapperService(indexUuid); + if (runtimeDocValues != null) { + when(mapperService.fieldType(any())).thenAnswer(fieldTypeInv -> { + String fieldName = (String)fieldTypeInv.getArguments()[0]; + return mockFieldType(fieldName, (leafSearchLookup, docId) -> runtimeDocValues.apply(fieldName, leafSearchLookup, docId)); + }); + } + final long nowInMillis = randomNonNegativeLong(); + return new QueryShardContext( + 0, mapperService.getIndexSettings(), BigArrays.NON_RECYCLING_INSTANCE, null, + (mappedFieldType, idxName, searchLookup) -> mappedFieldType.fielddataBuilder(idxName, searchLookup).build(null, null), + mapperService, null, null, NamedXContentRegistry.EMPTY, new NamedWriteableRegistry(Collections.emptyList()), + null, null, () -> nowInMillis, clusterAlias, null, () -> true, null); + } + + private static MapperService mockMapperService(String indexUuid) { IndexMetadata.Builder indexMetadataBuilder = new IndexMetadata.Builder("index"); indexMetadataBuilder.settings(Settings.builder().put("index.version.created", Version.CURRENT) .put("index.number_of_shards", 1) @@ -302,11 +367,12 @@ private static QueryShardContext createQueryShardContext(String indexUuid, Strin .put(IndexMetadata.SETTING_INDEX_UUID, indexUuid) ); IndexMetadata indexMetadata = indexMetadataBuilder.build(); - IndexSettings indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY); IndexAnalyzers indexAnalyzers = new IndexAnalyzers( Collections.singletonMap("default", new NamedAnalyzer("default", AnalyzerScope.INDEX, null)), Collections.emptyMap(), Collections.emptyMap() ); + IndexSettings indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY); + MapperService mapperService = mock(MapperService.class); when(mapperService.getIndexSettings()).thenReturn(indexSettings); when(mapperService.index()).thenReturn(indexMetadata.getIndex()); @@ -318,18 +384,7 @@ private static QueryShardContext createQueryShardContext(String indexUuid, Strin throw new UnsupportedOperationException(); }); when(mapperService.parserContext()).thenReturn(parserContext); - if (runtimeDocValues != null) { - when(mapperService.fieldType(any())).thenAnswer(fieldTypeInv -> { - String fieldName = (String)fieldTypeInv.getArguments()[0]; - return mockFieldType(fieldName, (leafSearchLookup, docId) -> runtimeDocValues.apply(fieldName, leafSearchLookup, docId)); - }); - } - final long nowInMillis = randomNonNegativeLong(); - return new QueryShardContext( - 0, indexSettings, BigArrays.NON_RECYCLING_INSTANCE, null, - (mappedFieldType, idxName, searchLookup) -> mappedFieldType.fielddataBuilder(idxName, searchLookup).build(null, null), - mapperService, null, null, NamedXContentRegistry.EMPTY, new NamedWriteableRegistry(Collections.emptyList()), - null, null, () -> nowInMillis, clusterAlias, null, () -> true, null); + return mapperService; } private static MappedFieldType mockFieldType(String fieldName, BiFunction runtimeDocValues) { From 19f9746a73645cce3ddf626f90f0df03aa2f409a Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 4 Nov 2020 10:25:47 -0500 Subject: [PATCH 06/36] Unit tests --- .../index/query/QueryShardContextTests.java | 85 ++++++++++++++++--- .../test/runtime_fields/10_keyword.yml | 6 +- 2 files changed, 77 insertions(+), 14 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java b/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java index 7374c89e17ab4..666c71799ba3a 100644 --- a/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java @@ -50,14 +50,21 @@ import org.elasticsearch.index.fielddata.LeafFieldData; import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.fielddata.plain.AbstractLeafOrdinalsFieldData; +import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.IndexFieldMapper; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.index.mapper.Mapper.BuilderContext; +import org.elasticsearch.index.mapper.Mapper.TypeParser; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.NumberFieldMapper; +import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.mapper.TextFieldMapper; +import org.elasticsearch.index.mapper.TextSearchInfo; +import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.indices.IndicesModule; +import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.search.lookup.LeafDocLookup; import org.elasticsearch.search.lookup.LeafSearchLookup; import org.elasticsearch.search.lookup.SearchLookup; @@ -66,8 +73,10 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.function.BiFunction; import java.util.function.Supplier; @@ -305,12 +314,20 @@ public void testFielddataLookupOneFieldManyReferences() throws IOException { } public void testRuntimeFields() throws IOException { - MapperService mapperService = mockMapperService("test"); - Map runtimeFields = Map.ofEntries( - Map.entry("cat", Map.of("type", "keyword", "script", "emit('cat')")), - Map.entry("dog", Map.of("type", "keyword", "script", "emit('dog')")), - Map.entry("catdog", Map.of("type", "keyword", "script", "emit(doc['cat'].value + doc['dog'].value)")) - ); + MapperService mapperService = mockMapperService("test", List.of(new MapperPlugin() { + @Override + public Map getMappers() { + return Map.of("runtime", (name, node, parserContext) -> new Mapper.Builder(name) { + @Override + public Mapper build(BuilderContext context) { + return new DummyMapper(name, new DummyMappedFieldType(name)); + } + }); + } + })); + Map runtimeFields = new HashMap<>(); + runtimeFields.put("cat", new HashMap<>(Map.of("type", "keyword"))); + runtimeFields.put("dog", new HashMap<>(Map.of("type", "keyword"))); QueryShardContext qsc = new QueryShardContext( 0, mapperService.getIndexSettings(), @@ -321,7 +338,7 @@ public void testRuntimeFields() throws IOException { null, null, NamedXContentRegistry.EMPTY, - new NamedWriteableRegistry(Collections.emptyList()), + new NamedWriteableRegistry(List.of()), null, null, () -> 0, @@ -332,7 +349,12 @@ public void testRuntimeFields() throws IOException { runtimeFields ); assertTrue(qsc.isFieldMapped("cat")); - assertThat(qsc.getFieldType(name)) + assertThat(qsc.getFieldType("cat"), instanceOf(DummyMappedFieldType.class)); + assertThat(qsc.simpleMatchToIndexNames("cat"), equalTo(Set.of("cat"))); + assertTrue(qsc.isFieldMapped("dog")); + assertThat(qsc.getFieldType("dog"), instanceOf(DummyMappedFieldType.class)); + assertThat(qsc.simpleMatchToIndexNames("dog"), equalTo(Set.of("dog"))); + assertThat(qsc.simpleMatchToIndexNames("*"), equalTo(Set.of("cat", "dog"))); } public static QueryShardContext createQueryShardContext(String indexUuid, String clusterAlias) { @@ -344,7 +366,7 @@ private static QueryShardContext createQueryShardContext( String clusterAlias, TriFunction runtimeDocValues ) { - MapperService mapperService = mockMapperService(indexUuid); + MapperService mapperService = mockMapperService(indexUuid, List.of()); if (runtimeDocValues != null) { when(mapperService.fieldType(any())).thenAnswer(fieldTypeInv -> { String fieldName = (String)fieldTypeInv.getArguments()[0]; @@ -359,7 +381,7 @@ mapperService, null, null, NamedXContentRegistry.EMPTY, new NamedWriteableRegist null, null, () -> nowInMillis, clusterAlias, null, () -> true, null); } - private static MapperService mockMapperService(String indexUuid) { + private static MapperService mockMapperService(String indexUuid, List mapperPlugins) { IndexMetadata.Builder indexMetadataBuilder = new IndexMetadata.Builder("index"); indexMetadataBuilder.settings(Settings.builder().put("index.version.created", Version.CURRENT) .put("index.number_of_shards", 1) @@ -377,7 +399,7 @@ private static MapperService mockMapperService(String indexUuid) { when(mapperService.getIndexSettings()).thenReturn(indexSettings); when(mapperService.index()).thenReturn(indexMetadata.getIndex()); when(mapperService.getIndexAnalyzers()).thenReturn(indexAnalyzers); - Map typeParserMap = IndicesModule.getMappers(Collections.emptyList()); + Map typeParserMap = IndicesModule.getMappers(mapperPlugins); Mapper.TypeParser.ParserContext parserContext = new Mapper.TypeParser.ParserContext(name -> null, typeParserMap::get, Version.CURRENT, () -> null, null, null, mapperService.getIndexAnalyzers(), mapperService.getIndexSettings(), () -> { @@ -482,4 +504,45 @@ public void collect(int doc) throws IOException { } } + private static class DummyMapper extends FieldMapper { + protected DummyMapper(String simpleName, MappedFieldType mappedFieldType) { + super(simpleName, mappedFieldType, Map.of(), MultiFields.empty(), CopyTo.empty()); + } + + @Override + protected void parseCreateField(ParseContext context) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public Builder getMergeBuilder() { + throw new UnsupportedOperationException(); + } + + @Override + protected String contentType() { + throw new UnsupportedOperationException(); + } + } + + private static class DummyMappedFieldType extends MappedFieldType { + public DummyMappedFieldType(String name) { + super(name, true, false, true, TextSearchInfo.SIMPLE_MATCH_ONLY, null); + } + + @Override + public ValueFetcher valueFetcher(QueryShardContext context, SearchLookup searchLookup, String format) { + throw new UnsupportedOperationException(); + } + + @Override + public String typeName() { + return "runtime"; + } + + @Override + public Query termQuery(Object value, QueryShardContext context) { + throw new UnsupportedOperationException(); + } + } } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_keyword.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_keyword.yml index d895141ffbe0c..09d547029f9e8 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_keyword.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_keyword.yml @@ -368,7 +368,7 @@ setup: index: sensor body: runtime_mappings: - voltage.rating: + voltage_rating: type: keyword script: | double v = doc['voltage'].value; @@ -379,11 +379,11 @@ setup: } else { emit('ok'); } - fields: [voltage.rating] + fields: [voltage_rating] sort: timestamp - match: {hits.total.value: 6} - match: {hits.hits.0._source.voltage: 4.0} - - match: {hits.hits.0.fields.voltage.rating: low} + - match: {hits.hits.0.fields.voltage_rating: [low]} --- From 05d0c9c26a1e56134a29c3084109084290c4075f Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 4 Nov 2020 10:33:00 -0500 Subject: [PATCH 07/36] Drop nocommit Tons of callers, we need this --- .../java/org/elasticsearch/index/query/QueryShardContext.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java index 78877bb01161d..debbe09467871 100644 --- a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java @@ -113,7 +113,7 @@ public class QueryShardContext extends QueryRewriteContext { /** * Build a {@linkplain QueryShardContext} without any information from the search request. */ - public QueryShardContext(int shardId, // NOCOMMIT make sure we need two methods + public QueryShardContext(int shardId, IndexSettings indexSettings, BigArrays bigArrays, BitsetFilterCache bitsetFilterCache, From 747220e6658b3bc0d88649f5d085b34931271045 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 4 Nov 2020 10:39:29 -0500 Subject: [PATCH 08/36] Drop another nocommit --- .../xpack/enrich/action/EnrichShardMultiSearchAction.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchAction.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchAction.java index 1ece72b2e4bba..44a9248aae845 100644 --- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchAction.java +++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchAction.java @@ -236,7 +236,7 @@ protected MultiSearchResponse shardOperation(Request request, ShardId shardId) t searcher, () -> { throw new UnsupportedOperationException(); }, null, - emptyMap() // NOCOMMIT is it right not to use the runtime mappings? + emptyMap() // Enrich doesn't support defining runtime fields ); final MultiSearchResponse.Item[] items = new MultiSearchResponse.Item[request.multiSearchRequest.requests().size()]; for (int i = 0; i < request.multiSearchRequest.requests().size(); i++) { From 8000de1d49d990852be698a8b839158a275fba71 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 4 Nov 2020 10:41:02 -0500 Subject: [PATCH 09/36] checkstyle --- .../elasticsearch/client/RequestLogger.java | 4 +-- ...ulkByScrollParallelizationHelperTests.java | 1 + .../action/search/SearchRequest.java | 1 - .../index/query/QueryShardContext.java | 2 +- .../MetadataCreateIndexServiceTests.java | 3 +- .../index/mapper/DateFieldTypeTests.java | 8 +++-- .../index/mapper/IndexFieldTypeTests.java | 22 +++++++++++-- .../index/mapper/NumberFieldTypeTests.java | 4 ++- .../index/mapper/RangeFieldTypeTests.java | 3 +- .../index/query/QueryShardContextTests.java | 15 ++++++--- .../search/AbstractSearchTestCase.java | 17 +++++++++- .../highlight/HighlightBuilderTests.java | 4 ++- .../rescore/QueryRescorerBuilderTests.java | 6 ++-- .../search/sort/AbstractSortTestCase.java | 4 ++- .../search/RandomSearchRequestGenerator.java | 9 ++++-- x-pack/plugin/runtime-fields/qa/build.gradle | 2 ++ .../CoreTestsWithSearchRuntimeFieldsIT.java | 32 ++++--------------- .../test/CoreTestTranslater.java | 14 ++++++-- 18 files changed, 99 insertions(+), 52 deletions(-) diff --git a/client/rest/src/main/java/org/elasticsearch/client/RequestLogger.java b/client/rest/src/main/java/org/elasticsearch/client/RequestLogger.java index 7c56a7edf97a9..41271ab69d35c 100644 --- a/client/rest/src/main/java/org/elasticsearch/client/RequestLogger.java +++ b/client/rest/src/main/java/org/elasticsearch/client/RequestLogger.java @@ -65,7 +65,7 @@ static void logResponse(Log logger, HttpUriRequest request, HttpHost host, HttpR logger.warn(buildWarningMessage(request, host, warnings)); } } - if (tracer.isTraceEnabled()) { + if (tracer.isErrorEnabled()) { String requestLine; try { requestLine = buildTraceRequest(request, host); @@ -80,7 +80,7 @@ static void logResponse(Log logger, HttpUriRequest request, HttpHost host, HttpR responseLine = ""; tracer.trace("error while reading response for trace purposes", e); } - tracer.trace(requestLine + '\n' + responseLine); + tracer.error(requestLine + '\n' + responseLine); } } diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkByScrollParallelizationHelperTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkByScrollParallelizationHelperTests.java index a64415d08b1b1..b2512e6a62cfd 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkByScrollParallelizationHelperTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkByScrollParallelizationHelperTests.java @@ -38,6 +38,7 @@ public void testSliceIntoSubRequests() throws IOException { () -> null, () -> null, () -> emptyList(), + () -> null, () -> null)); if (searchRequest.source() != null) { // Clear the slice builder if there is one set. We can't call sliceIntoSubRequests if it is. diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java index ef7aff6be399e..e6e07dbc67904 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java @@ -42,7 +42,6 @@ import java.util.Map; import java.util.Objects; -import static java.util.Collections.emptyMap; import static org.elasticsearch.action.ValidateActions.addValidationError; /** diff --git a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java index debbe09467871..635de7d686d7a 100644 --- a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java @@ -641,7 +641,7 @@ private static Map parseRuntimeMappings( throw new ElasticsearchParseException("runtime mappings must be a map type"); } @SuppressWarnings("unchecked") - Map node = (Map) entry.getValue(); + Map node = new HashMap<>((Map) entry.getValue()); // Replace the type until we have native support for the runtime section Object oldRuntimeType = node.put("runtime_type", node.remove("type")); if (oldRuntimeType != null) { diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java index a8489ed6008b7..9006ec359f60e 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java @@ -110,6 +110,7 @@ import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.Matchers.startsWith; +import static org.mockito.Mockito.mock; public class MetadataCreateIndexServiceTests extends ESTestCase { @@ -125,7 +126,7 @@ public void setupCreateIndexRequestAndAliasValidator() { .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1).build(); queryShardContext = new QueryShardContext(0, new IndexSettings(IndexMetadata.builder("test").settings(indexSettings).build(), indexSettings), - BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, xContentRegistry(), writableRegistry(), + BigArrays.NON_RECYCLING_INSTANCE, null, null, mock(MapperService.class), null, null, xContentRegistry(), writableRegistry(), null, null, () -> randomNonNegativeLong(), null, null, () -> true, null); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java index 95e09f229256c..b4b819d6d7eb2 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java @@ -59,6 +59,8 @@ import java.util.Collections; import java.util.List; +import static org.mockito.Mockito.mock; + public class DateFieldTypeTests extends FieldTypeTestCase { private static final long nowInMillis = 0; @@ -165,7 +167,7 @@ public void testTermQuery() { .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1).build(); QueryShardContext context = new QueryShardContext(0, new IndexSettings(IndexMetadata.builder("foo").settings(indexSettings).build(), indexSettings), - BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, + BigArrays.NON_RECYCLING_INSTANCE, null, null, mock(MapperService.class), null, null, xContentRegistry(), writableRegistry(), null, null, () -> nowInMillis, null, null, () -> true, null); MappedFieldType ft = new DateFieldType("field"); String date = "2015-10-12T14:10:55"; @@ -187,7 +189,7 @@ public void testRangeQuery() throws IOException { .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1).build(); QueryShardContext context = new QueryShardContext(0, new IndexSettings(IndexMetadata.builder("foo").settings(indexSettings).build(), indexSettings), - BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, xContentRegistry(), writableRegistry(), + BigArrays.NON_RECYCLING_INSTANCE, null, null, mock(MapperService.class), null, null, xContentRegistry(), writableRegistry(), null, null, () -> nowInMillis, null, null, () -> true, null); MappedFieldType ft = new DateFieldType("field"); String date1 = "2015-10-12T14:10:55"; @@ -231,7 +233,7 @@ public void testRangeQueryWithIndexSort() { IndexSettings indexSettings = new IndexSettings(indexMetadata, settings); QueryShardContext context = new QueryShardContext(0, indexSettings, - BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, xContentRegistry(), writableRegistry(), + BigArrays.NON_RECYCLING_INSTANCE, null, null, mock(MapperService.class), null, null, xContentRegistry(), writableRegistry(), null, null, () -> 0L, null, null, () -> true, null); MappedFieldType ft = new DateFieldType("field"); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldTypeTests.java index 473eca61c71b2..dc47c003f34a3 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldTypeTests.java @@ -32,6 +32,7 @@ import java.util.function.Predicate; import static org.hamcrest.Matchers.containsString; +import static org.mockito.Mockito.mock; public class IndexFieldTypeTests extends ESTestCase { @@ -68,7 +69,24 @@ private QueryShardContext createContext() { IndexSettings indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY); Predicate indexNameMatcher = pattern -> Regex.simpleMatch(pattern, "index"); - return new QueryShardContext(0, indexSettings, null, null, null, null, null, null, xContentRegistry(), writableRegistry(), - null, null, System::currentTimeMillis, null, indexNameMatcher, () -> true, null); + return new QueryShardContext( + 0, + indexSettings, + null, + null, + null, + mock(MapperService.class), + null, + null, + xContentRegistry(), + writableRegistry(), + null, + null, + System::currentTimeMillis, + null, + indexNameMatcher, + () -> true, + null + ); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java index 6f40d9443dc77..59744532d6b9b 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java @@ -20,6 +20,7 @@ package org.elasticsearch.index.mapper; import com.carrotsearch.randomizedtesting.generators.RandomPicks; + import org.apache.lucene.document.Document; import org.apache.lucene.document.DoublePoint; import org.apache.lucene.document.FloatPoint; @@ -69,6 +70,7 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; +import static org.mockito.Mockito.mock; public class NumberFieldTypeTests extends FieldTypeTestCase { @@ -473,7 +475,7 @@ public void doTestIndexSortRangeQueries(NumberType type, Supplier valueS IndexSearcher searcher = newSearcher(reader); QueryShardContext context = new QueryShardContext(0, indexSettings, - BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, xContentRegistry(), writableRegistry(), + BigArrays.NON_RECYCLING_INSTANCE, null, null, mock(MapperService.class), null, null, xContentRegistry(), writableRegistry(), null, null, () -> 0L, null, null, () -> true, null); final int iters = 10; diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java index 1e67d7fdb0046..e9240c01d89e9 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java @@ -54,6 +54,7 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.instanceOf; +import static org.mockito.Mockito.mock; public class RangeFieldTypeTests extends FieldTypeTestCase { RangeType type; @@ -212,7 +213,7 @@ private QueryShardContext createContext() { Settings indexSettings = Settings.builder() .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).build(); IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(randomAlphaOfLengthBetween(1, 10), indexSettings); - return new QueryShardContext(0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, + return new QueryShardContext(0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, null, null, mock(MapperService.class), null, null, xContentRegistry(), writableRegistry(), null, null, () -> nowInMillis, null, null, () -> true, null); } diff --git a/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java b/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java index 666c71799ba3a..666ad6b608fd1 100644 --- a/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java @@ -73,7 +73,6 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -325,9 +324,15 @@ public Mapper build(BuilderContext context) { }); } })); - Map runtimeFields = new HashMap<>(); - runtimeFields.put("cat", new HashMap<>(Map.of("type", "keyword"))); - runtimeFields.put("dog", new HashMap<>(Map.of("type", "keyword"))); + /* + * Making these immutable here test that we don't modify them. + * Modifying them would cause all kinds of problems if two + * shards are parsed on the same node. + */ + Map runtimeFields = Map.ofEntries( + Map.entry("cat", Map.of("type", "keyword")), + Map.entry("dog", Map.of("type", "keyword")) + ); QueryShardContext qsc = new QueryShardContext( 0, mapperService.getIndexSettings(), @@ -526,7 +531,7 @@ protected String contentType() { } private static class DummyMappedFieldType extends MappedFieldType { - public DummyMappedFieldType(String name) { + DummyMappedFieldType(String name) { super(name, true, false, true, TextSearchInfo.SIMPLE_MATCH_ONLY, null); } diff --git a/server/src/test/java/org/elasticsearch/search/AbstractSearchTestCase.java b/server/src/test/java/org/elasticsearch/search/AbstractSearchTestCase.java index e8488d99d8a80..3caf5ee9ef27c 100644 --- a/server/src/test/java/org/elasticsearch/search/AbstractSearchTestCase.java +++ b/server/src/test/java/org/elasticsearch/search/AbstractSearchTestCase.java @@ -91,7 +91,22 @@ protected SearchSourceBuilder createSearchSourceBuilder() { SuggestBuilderTests::randomSuggestBuilder, QueryRescorerBuilderTests::randomRescoreBuilder, randomExtBuilders, - CollapseBuilderTests::randomCollapseBuilder); + CollapseBuilderTests::randomCollapseBuilder, + this::randomRandomMappings); + } + + protected Map randomRandomMappings() { + int count = between(1, 100); + Map runtimeFields = new HashMap<>(count); + while (runtimeFields.size() < count) { + int size = between(1, 10); + Map config = new HashMap<>(size); + while (config.size() < size) { + config.put(randomAlphaOfLength(5), randomAlphaOfLength(5)); + } + runtimeFields.put(randomAlphaOfLength(5), config); + } + return runtimeFields; } protected SearchRequest createSearchRequest() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java index 098ac2cd7a959..10cdac3e7836d 100644 --- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java @@ -42,6 +42,7 @@ import org.elasticsearch.index.mapper.ContentPath; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.index.query.IdsQueryBuilder; import org.elasticsearch.index.query.MatchAllQueryBuilder; @@ -74,6 +75,7 @@ import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.mockito.Mockito.mock; public class HighlightBuilderTests extends ESTestCase { @@ -280,7 +282,7 @@ public void testBuildSearchContextHighlight() throws IOException { IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(index, indexSettings); // shard context will only need indicesQueriesRegistry for building Query objects nested in highlighter QueryShardContext mockShardContext = new QueryShardContext(0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, - null, null, null, null, null, xContentRegistry(), namedWriteableRegistry, + null, null, mock(MapperService.class), null, null, xContentRegistry(), namedWriteableRegistry, null, null, System::currentTimeMillis, null, null, () -> true, null) { @Override public MappedFieldType getFieldType(String name) { diff --git a/server/src/test/java/org/elasticsearch/search/rescore/QueryRescorerBuilderTests.java b/server/src/test/java/org/elasticsearch/search/rescore/QueryRescorerBuilderTests.java index 88913e0aaa782..d942abf5f0e2b 100644 --- a/server/src/test/java/org/elasticsearch/search/rescore/QueryRescorerBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/rescore/QueryRescorerBuilderTests.java @@ -41,6 +41,7 @@ import org.elasticsearch.index.mapper.ContentPath; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.index.query.MatchAllQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; @@ -59,6 +60,7 @@ import static java.util.Collections.emptyList; import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode; import static org.hamcrest.Matchers.containsString; +import static org.mockito.Mockito.mock; public class QueryRescorerBuilderTests extends ESTestCase { @@ -144,7 +146,7 @@ public void testBuildRescoreSearchContext() throws ElasticsearchParseException, IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(randomAlphaOfLengthBetween(1, 10), indexSettings); // shard context will only need indicesQueriesRegistry for building Query objects nested in query rescorer QueryShardContext mockShardContext = new QueryShardContext(0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, - null, null, null, null, null, + null, null, mock(MapperService.class), null, null, xContentRegistry(), namedWriteableRegistry, null, null, () -> nowInMillis, null, null, () -> true, null) { @Override public MappedFieldType getFieldType(String name) { @@ -188,7 +190,7 @@ public void testRewritingKeepsSettings() throws IOException { IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(randomAlphaOfLengthBetween(1, 10), indexSettings); // shard context will only need indicesQueriesRegistry for building Query objects nested in query rescorer QueryShardContext mockShardContext = new QueryShardContext(0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, - null, null, null, null, null, + null, null, mock(MapperService.class), null, null, xContentRegistry(), namedWriteableRegistry, null, null, () -> nowInMillis, null, null, () -> true, null) { @Override public MappedFieldType getFieldType(String name) { diff --git a/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java b/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java index 7dd1641e6cf74..32d4c90dbc7b2 100644 --- a/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java +++ b/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java @@ -41,6 +41,7 @@ import org.elasticsearch.index.fielddata.IndexFieldDataCache; import org.elasticsearch.index.mapper.ContentPath; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.Mapper.BuilderContext; import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.index.mapper.ObjectMapper; @@ -72,6 +73,7 @@ import static java.util.Collections.emptyList; import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode; +import static org.mockito.Mockito.mock; public abstract class AbstractSortTestCase> extends ESTestCase { @@ -202,7 +204,7 @@ protected final QueryShardContext createMockShardContext(IndexSearcher searcher) return builder.build(new IndexFieldDataCache.None(), null); }; return new QueryShardContext(0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, bitsetFilterCache, indexFieldDataLookup, - null, null, scriptService, xContentRegistry(), namedWriteableRegistry, null, searcher, + mock(MapperService.class), null, scriptService, xContentRegistry(), namedWriteableRegistry, null, searcher, () -> randomNonNegativeLong(), null, null, () -> true, null) { @Override diff --git a/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java b/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java index ed95561aa9100..67aaedf7d818f 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java @@ -53,6 +53,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.function.Supplier; import static java.util.Collections.emptyMap; @@ -82,7 +83,7 @@ private RandomSearchRequestGenerator() {} * Build a random search request. * * @param randomSearchSourceBuilder builds a random {@link SearchSourceBuilder}. You can use - * {@link #randomSearchSourceBuilder(Supplier, Supplier, Supplier, Supplier, Supplier)}. + * {@link #randomSearchSourceBuilder}. */ public static SearchRequest randomSearchRequest(Supplier randomSearchSourceBuilder) { SearchRequest searchRequest = new SearchRequest(); @@ -122,7 +123,8 @@ public static SearchSourceBuilder randomSearchSourceBuilder( Supplier randomSuggestBuilder, Supplier> randomRescoreBuilder, Supplier> randomExtBuilders, - Supplier randomCollapseBuilder) { + Supplier randomCollapseBuilder, + Supplier> randomRuntimeMappings) { SearchSourceBuilder builder = new SearchSourceBuilder(); if (randomBoolean()) { builder.from(randomIntBetween(0, 10000)); @@ -378,6 +380,9 @@ public static SearchSourceBuilder randomSearchSourceBuilder( } builder.pointInTimeBuilder(pit); } + if (randomBoolean()) { + builder.runtimeMappings(randomRuntimeMappings.get()); + } return builder; } } diff --git a/x-pack/plugin/runtime-fields/qa/build.gradle b/x-pack/plugin/runtime-fields/qa/build.gradle index bc22510e77458..4f108f906c60c 100644 --- a/x-pack/plugin/runtime-fields/qa/build.gradle +++ b/x-pack/plugin/runtime-fields/qa/build.gradle @@ -5,3 +5,5 @@ apply plugin: 'elasticsearch.build' dependencies { api project(":test:framework") } + +test.enabled = false // We don't currently have any tests for this because they are test utilities. diff --git a/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java b/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java index 1c40376c7cedd..d610e6750bbc6 100644 --- a/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java +++ b/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java @@ -9,7 +9,6 @@ import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import org.apache.logging.log4j.LogManager; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; import org.elasticsearch.test.rest.yaml.section.ApiCallSection; @@ -21,7 +20,6 @@ import java.util.Map; import static java.util.Collections.unmodifiableMap; -import static org.hamcrest.Matchers.hasSize; /** * Runs elasticsearch's core rest tests disabling all mappings and replacing them @@ -58,9 +56,6 @@ protected Suite suite(ClientYamlTestCandidate candidate) { @Override public boolean modifySections(List executables) { if (runtimeMappingsAfterSetup == null) { - if (candidate.toString().contains("50_filter")) { - LogManager.getLogger().error("init"); - } // We're modifying the setup section runtimeMappings = new HashMap<>(); if (false == super.modifySections(executables)) { @@ -89,35 +84,22 @@ protected boolean modifyMapping(String index, Map mapping) { return false; } mapping.put("properties", untouchedMapping); - if (candidate.toString().contains("50_filter")) { - LogManager.getLogger().error("untouched " + untouchedMapping); - LogManager.getLogger().error("runtime " + runtimeMapping); - } runtimeMappings.put(index, runtimeMapping); return true; } @Override protected boolean modifySearch(ApiCallSection search) { - if (candidate.toString().contains("50_filter")) { - LogManager.getLogger().error("toString " + candidate); - LogManager.getLogger().error("runtimeMappingsAfterSetup " + runtimeMappingsAfterSetup); - LogManager.getLogger().error("runtimeMappings " + runtimeMappings); - } - Map body; if (search.getBodies().isEmpty()) { - body = new HashMap<>(); - search.addBody(body); - } else { - body = search.getBodies().get(0); - assertThat(search.getBodies(), hasSize(1)); + search.addBody(new HashMap<>()); } - Object index = body.get("index"); - Map runtimeMapping = runtimeMappings(index); - if (runtimeMapping == null) { - return false; + for (Map body : search.getBodies()) { + Map runtimeMapping = runtimeMappings(body.get("index")); + if (runtimeMapping == null) { + return false; + } + body.put("runtime_mappings", runtimeMapping); } - search.getBodies().get(0).put("runtime_mappings", runtimeMapping); return true; } diff --git a/x-pack/plugin/runtime-fields/qa/src/main/java/org/elasticsearch/xpack/runtimefields/test/CoreTestTranslater.java b/x-pack/plugin/runtime-fields/qa/src/main/java/org/elasticsearch/xpack/runtimefields/test/CoreTestTranslater.java index 3a6130fce5636..ffcd747b1a8e5 100644 --- a/x-pack/plugin/runtime-fields/qa/src/main/java/org/elasticsearch/xpack/runtimefields/test/CoreTestTranslater.java +++ b/x-pack/plugin/runtime-fields/qa/src/main/java/org/elasticsearch/xpack/runtimefields/test/CoreTestTranslater.java @@ -1,3 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ package org.elasticsearch.xpack.runtimefields.test; import org.elasticsearch.common.xcontent.XContentLocation; @@ -133,7 +138,7 @@ protected static Map indexTemplateToAddRuntimeFieldsToMappings() return Map.of("settings", Map.of(), "mappings", Map.of("dynamic_templates", dynamicTemplates)); } - private final ExecutableSection addIndexTemplate() { + private ExecutableSection addIndexTemplate() { return new ExecutableSection() { @Override public XContentLocation getLocation() { @@ -158,7 +163,7 @@ public void execute(ClientYamlTestExecutionContext executionContext) throws IOEx } /** - * NOCOMMIT write me + * A modified suite. */ protected abstract class Suite { private final ClientYamlTestSuite modified; @@ -217,7 +222,7 @@ public boolean modifySections(List executables) { */ protected abstract boolean modifySearch(ApiCallSection search); - private final boolean modifyCreateIndex(ApiCallSection createIndex) { + private boolean modifyCreateIndex(ApiCallSection createIndex) { String index = createIndex.getParams().get("index"); for (Map body : createIndex.getBodies()) { Object settings = body.get("settings"); @@ -242,6 +247,9 @@ private final boolean modifyCreateIndex(ApiCallSection createIndex) { return true; } + /** + * Modify the mapping defined in the test. + */ protected abstract boolean modifyMapping(String index, Map mapping); /** From a1d09159297a96680da6dc3c6feff91acc9dfab8 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 4 Nov 2020 16:39:20 -0500 Subject: [PATCH 10/36] Can we do dynamic mappings? --- .../elasticsearch/client/RequestLogger.java | 4 +- .../search.aggregation/340_geo_distance.yml | 4 +- .../mapped/CoreWithMappedRuntimeFieldsIT.java | 8 +- .../CoreTestsWithSearchRuntimeFieldsIT.java | 84 +++++++++++++++- .../test/CoreTestTranslater.java | 98 +++++++++++++++++-- 5 files changed, 182 insertions(+), 16 deletions(-) diff --git a/client/rest/src/main/java/org/elasticsearch/client/RequestLogger.java b/client/rest/src/main/java/org/elasticsearch/client/RequestLogger.java index 41271ab69d35c..7c56a7edf97a9 100644 --- a/client/rest/src/main/java/org/elasticsearch/client/RequestLogger.java +++ b/client/rest/src/main/java/org/elasticsearch/client/RequestLogger.java @@ -65,7 +65,7 @@ static void logResponse(Log logger, HttpUriRequest request, HttpHost host, HttpR logger.warn(buildWarningMessage(request, host, warnings)); } } - if (tracer.isErrorEnabled()) { + if (tracer.isTraceEnabled()) { String requestLine; try { requestLine = buildTraceRequest(request, host); @@ -80,7 +80,7 @@ static void logResponse(Log logger, HttpUriRequest request, HttpHost host, HttpR responseLine = ""; tracer.trace("error while reading response for trace purposes", e); } - tracer.error(requestLine + '\n' + responseLine); + tracer.trace(requestLine + '\n' + responseLine); } } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/340_geo_distance.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/340_geo_distance.yml index 3d37f1948278c..58fa1b04a7b7f 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/340_geo_distance.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/340_geo_distance.yml @@ -1,7 +1,7 @@ setup: - do: indices.create: - index: test + index: test_geoadsf body: mappings: properties: @@ -9,7 +9,7 @@ setup: type: geo_point - do: bulk: - index: test + index: test_geoadsf refresh: true body: - '{"index": {}}' diff --git a/x-pack/plugin/runtime-fields/qa/core-with-mapped/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/mapped/CoreWithMappedRuntimeFieldsIT.java b/x-pack/plugin/runtime-fields/qa/core-with-mapped/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/mapped/CoreWithMappedRuntimeFieldsIT.java index 52f5df557a939..7ea291c1c95ba 100644 --- a/x-pack/plugin/runtime-fields/qa/core-with-mapped/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/mapped/CoreWithMappedRuntimeFieldsIT.java +++ b/x-pack/plugin/runtime-fields/qa/core-with-mapped/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/mapped/CoreWithMappedRuntimeFieldsIT.java @@ -9,6 +9,7 @@ import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; import org.elasticsearch.test.rest.yaml.section.ApiCallSection; @@ -68,8 +69,13 @@ protected boolean modifySearch(ApiCallSection search) { // We don't need to modify the search request if the mappings are in the index return true; } + + @Override + protected boolean handleIndex(IndexRequest index) { + // We don't need to scrape anything out of the index requests. + return true; + } }; } - } } diff --git a/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java b/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java index d610e6750bbc6..fda33645df7a3 100644 --- a/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java +++ b/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java @@ -9,6 +9,10 @@ import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.apache.logging.log4j.LogManager; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; import org.elasticsearch.test.rest.yaml.section.ApiCallSection; @@ -18,6 +22,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import static java.util.Collections.unmodifiableMap; @@ -50,22 +55,28 @@ protected Map indexTemplate() { @Override protected Suite suite(ClientYamlTestCandidate candidate) { return new Suite(candidate) { - private Map> runtimeMappingsAfterSetup; - private Map> runtimeMappings; + private Map>> runtimeMappingsAfterSetup; + private Map> mappedFieldsAfterSetup; + private Map>> runtimeMappings; + private Map> mappedFields; @Override public boolean modifySections(List executables) { if (runtimeMappingsAfterSetup == null) { // We're modifying the setup section runtimeMappings = new HashMap<>(); + mappedFields = new HashMap<>(); if (false == super.modifySections(executables)) { return false; } runtimeMappingsAfterSetup = unmodifiableMap(runtimeMappings); runtimeMappings = null; + mappedFieldsAfterSetup = unmodifiableMap(mappedFields); + mappedFields = null; return true; } runtimeMappings = new HashMap<>(runtimeMappingsAfterSetup); + mappedFields = new HashMap<>(mappedFieldsAfterSetup); return super.modifySections(executables); } @@ -84,6 +95,7 @@ protected boolean modifyMapping(String index, Map mapping) { return false; } mapping.put("properties", untouchedMapping); + mappedFields.put(index, untouchedMapping.keySet()); runtimeMappings.put(index, runtimeMapping); return true; } @@ -111,8 +123,76 @@ protected boolean modifySearch(ApiCallSection search) { if (runtimeMappings.size() == 1) { return runtimeMappings.values().iterator().next(); } + // NOCOMMIT try and merge the mappings return null; } + + @Override + protected boolean handleIndex(IndexRequest index) { + /* + * Ok! Let's reverse engineer dynamic mapping. Sort of. We're + * really just looking to figure out which of the runtime fields + * is "close enough" to what dynamic mapping would do. + */ + if (index.getPipeline() != null) { + // We can't attempt local dynamic mappings with pipelines + return false; + } + Map map = XContentHelper.convertToMap(index.source(), false, index.getContentType()).v2(); + Map> indexRuntimeMappings = runtimeMappings.computeIfAbsent( + index.index(), + i -> new HashMap<>() + ); + Set indexMappedfields = mappedFields.computeIfAbsent(index.index(), i -> Set.of()); + for (Map.Entry e : map.entrySet()) { + String name = e.getKey(); + if (indexRuntimeMappings.containsKey(name) || indexMappedfields.contains(name)) { + continue; + } + Object value = e.getValue(); + if (value == null) { + continue; + } + if (value instanceof Boolean) { + indexRuntimeMappings.put(name, runtimeFieldLoadingFromSource(name, "boolean")); + continue; + } + if (value instanceof Long || value instanceof Integer) { + indexRuntimeMappings.put(name, runtimeFieldLoadingFromSource(name, "long")); + continue; + } + if (value instanceof Double) { + indexRuntimeMappings.put(name, runtimeFieldLoadingFromSource(name, "double")); + continue; + } + if (false == value instanceof String) { + continue; + } + try { + Long.parseLong(value.toString()); + indexRuntimeMappings.put(name, runtimeFieldLoadingFromSource(name, "long")); + continue; + } catch (IllegalArgumentException iae) { + // Try the next one + } + try { + Double.parseDouble(value.toString()); + indexRuntimeMappings.put(name, runtimeFieldLoadingFromSource(name, "double")); + continue; + } catch (IllegalArgumentException iae) { + // Try the next one + } + try { + DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(value.toString()); + indexRuntimeMappings.put(name, runtimeFieldLoadingFromSource(name, "date")); + continue; + } catch (IllegalArgumentException iae) { + // Try the next one + } + indexRuntimeMappings.put(name, runtimeFieldLoadingFromSource(name, "keyword")); + } + return true; + } }; } } diff --git a/x-pack/plugin/runtime-fields/qa/src/main/java/org/elasticsearch/xpack/runtimefields/test/CoreTestTranslater.java b/x-pack/plugin/runtime-fields/qa/src/main/java/org/elasticsearch/xpack/runtimefields/test/CoreTestTranslater.java index ffcd747b1a8e5..26d6a131b4cf4 100644 --- a/x-pack/plugin/runtime-fields/qa/src/main/java/org/elasticsearch/xpack/runtimefields/test/CoreTestTranslater.java +++ b/x-pack/plugin/runtime-fields/qa/src/main/java/org/elasticsearch/xpack/runtimefields/test/CoreTestTranslater.java @@ -5,7 +5,13 @@ */ package org.elasticsearch.xpack.runtimefields.test; +import org.elasticsearch.action.bulk.BulkRequestParser; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentLocation; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.index.mapper.BooleanFieldMapper; import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.IpFieldMapper; @@ -138,6 +144,13 @@ protected static Map indexTemplateToAddRuntimeFieldsToMappings() return Map.of("settings", Map.of(), "mappings", Map.of("dynamic_templates", dynamicTemplates)); } + protected static Map runtimeFieldLoadingFromSource(String name, String type) { + return Map.ofEntries( + Map.entry("type", type), + Map.entry("script", painlessToLoadFromSource(name, type)) + ); + } + private ExecutableSection addIndexTemplate() { return new ExecutableSection() { @Override @@ -203,15 +216,30 @@ public boolean modifySections(List executables) { } DoSection doSection = (DoSection) section; String api = doSection.getApiCallSection().getApi(); - if (api.equals("indices.create")) { - if (false == modifyCreateIndex(doSection.getApiCallSection())) { - return false; - } - } - if (api.equals("search") || api.equals("async_search.submit")) { - if (false == modifySearch(doSection.getApiCallSection())) { - return false; - } + switch (api) { + case "indices.create": + if (false == modifyCreateIndex(doSection.getApiCallSection())) { + return false; + } + break; + case "search": + case "async_search.submit": + if (false == modifySearch(doSection.getApiCallSection())) { + return false; + } + break; + case "bulk": + if (false == handleBulk(doSection.getApiCallSection())) { + return false; + } + break; + case "index": + if (false == handleIndex(doSection.getApiCallSection())) { + return false; + } + break; + default: + continue; } } return true; @@ -322,5 +350,57 @@ protected final boolean runtimeifyMappingProperties( */ return true; } + + private boolean handleBulk(ApiCallSection bulk) { + String defaultIndex = bulk.getParams().get("index"); + String defaultRouting = bulk.getParams().get("routing"); + String defaultPipeline = bulk.getParams().get("pipeline"); + BytesStreamOutput bos = new BytesStreamOutput(); + try { + for (Map body : bulk.getBodies()) { + try (XContentBuilder b = new XContentBuilder(JsonXContent.jsonXContent, bos)) { + b.map(body); + } + bos.write(JsonXContent.jsonXContent.streamSeparator()); + } + List indexRequests = new ArrayList<>(); + new BulkRequestParser(false).parse( + bos.bytes(), + defaultIndex, + defaultRouting, + null, + defaultPipeline, + null, + true, + XContentType.JSON, + (index, type) -> { + indexRequests.add(index); + }, + u -> {}, + d -> {} + ); + for (IndexRequest index : indexRequests) { + if (false == handleIndex(index)) { + return false; + } + } + } catch (IOException e) { + throw new AssertionError(e); + } + return true; + } + + private boolean handleIndex(ApiCallSection indexRequest) { + String index = indexRequest.getParams().get("index"); + String pipeline = indexRequest.getParams().get("pipeline"); + assert indexRequest.getBodies().size() == 1; + try { + return handleIndex(new IndexRequest(index).setPipeline(pipeline).source(indexRequest.getBodies().get(0))); + } catch (IOException e) { + throw new AssertionError(e); + } + } + + protected abstract boolean handleIndex(IndexRequest index) throws IOException; } } From 0d7cae13d68ff809106cd1bb52bf636dc708ac73 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 4 Nov 2020 20:09:29 -0500 Subject: [PATCH 11/36] Add runtime mappings to collapse --- .../org/elasticsearch/action/search/ExpandSearchPhase.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java b/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java index 57082ed450941..9ee9bf947a46d 100644 --- a/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java +++ b/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java @@ -89,7 +89,8 @@ public void run() { CollapseBuilder innerCollapseBuilder = innerHitBuilder.getInnerCollapseBuilder(); SearchSourceBuilder sourceBuilder = buildExpandSearchSourceBuilder(innerHitBuilder, innerCollapseBuilder) .query(groupQuery) - .postFilter(searchRequest.source().postFilter()); + .postFilter(searchRequest.source().postFilter()) + .runtimeMappings(searchRequest.source().runtimeMappings()); SearchRequest groupRequest = new SearchRequest(searchRequest); groupRequest.source(sourceBuilder); multiRequest.add(groupRequest); From b17eff0d7389819cb8c8efd3ea31da662e5af3fa Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 4 Nov 2020 20:16:35 -0500 Subject: [PATCH 12/36] Fixup dynamic mapping --- .../test/search/CoreTestsWithSearchRuntimeFieldsIT.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java b/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java index fda33645df7a3..073a9c3555b1c 100644 --- a/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java +++ b/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java @@ -189,7 +189,8 @@ protected boolean handleIndex(IndexRequest index) { } catch (IllegalArgumentException iae) { // Try the next one } - indexRuntimeMappings.put(name, runtimeFieldLoadingFromSource(name, "keyword")); + // Strings are funny, the regular dynamic mapping puts them in "name.keyword" so we follow along. + indexRuntimeMappings.put(name + ".keyword", runtimeFieldLoadingFromSource(name, "keyword")); } return true; } From e95ed207ae4f74079bcd3a02b91bd698629a5916 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 5 Nov 2020 08:40:57 -0500 Subject: [PATCH 13/36] Handle _all --- .../elasticsearch/client/RequestLogger.java | 4 +-- .../search/builder/SearchSourceBuilder.java | 11 +++++-- .../action/search/ExpandSearchPhaseTests.java | 4 +++ .../mapped/CoreWithMappedRuntimeFieldsIT.java | 2 +- .../CoreTestsWithSearchRuntimeFieldsIT.java | 7 ++--- .../test/CoreTestTranslater.java | 29 ++++++++++++++++--- 6 files changed, 44 insertions(+), 13 deletions(-) diff --git a/client/rest/src/main/java/org/elasticsearch/client/RequestLogger.java b/client/rest/src/main/java/org/elasticsearch/client/RequestLogger.java index 7c56a7edf97a9..41271ab69d35c 100644 --- a/client/rest/src/main/java/org/elasticsearch/client/RequestLogger.java +++ b/client/rest/src/main/java/org/elasticsearch/client/RequestLogger.java @@ -65,7 +65,7 @@ static void logResponse(Log logger, HttpUriRequest request, HttpHost host, HttpR logger.warn(buildWarningMessage(request, host, warnings)); } } - if (tracer.isTraceEnabled()) { + if (tracer.isErrorEnabled()) { String requestLine; try { requestLine = buildTraceRequest(request, host); @@ -80,7 +80,7 @@ static void logResponse(Log logger, HttpUriRequest request, HttpHost host, HttpR responseLine = ""; tracer.trace("error while reading response for trace purposes", e); } - tracer.trace(requestLine + '\n' + responseLine); + tracer.error(requestLine + '\n' + responseLine); } } diff --git a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java index 27035d7fb77ec..d342ed4223115 100644 --- a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java @@ -256,7 +256,9 @@ public SearchSourceBuilder(StreamInput in) throws IOException { pointInTimeBuilder = in.readOptionalWriteable(PointInTimeBuilder::new); } if (in.getVersion().onOrAfter(Version.V_8_0_0)) { - runtimeMappings = in.readMap(); + if (in.readBoolean()) { + runtimeMappings = in.readMap(); + } } } @@ -320,7 +322,12 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalWriteable(pointInTimeBuilder); } if (out.getVersion().onOrAfter(Version.V_8_0_0)) { - out.writeMap(runtimeMappings); + if (runtimeMappings == null) { + out.writeBoolean(false); + } else { + out.writeBoolean(true); + out.writeMap(runtimeMappings); + } } } diff --git a/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java b/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java index beb4ef6d66e6f..a55a3b0e3c000 100644 --- a/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java @@ -240,4 +240,8 @@ void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionL mockSearchPhaseContext.assertNoFailure(); assertNotNull(mockSearchPhaseContext.searchResponse.get()); } + + public void testCollapseWithRuntimeField() { + // NOCOMMIT write me. + } } diff --git a/x-pack/plugin/runtime-fields/qa/core-with-mapped/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/mapped/CoreWithMappedRuntimeFieldsIT.java b/x-pack/plugin/runtime-fields/qa/core-with-mapped/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/mapped/CoreWithMappedRuntimeFieldsIT.java index 7ea291c1c95ba..b126cf919fbd8 100644 --- a/x-pack/plugin/runtime-fields/qa/core-with-mapped/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/mapped/CoreWithMappedRuntimeFieldsIT.java +++ b/x-pack/plugin/runtime-fields/qa/core-with-mapped/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/mapped/CoreWithMappedRuntimeFieldsIT.java @@ -36,7 +36,7 @@ public static Iterable parameters() throws Exception { private static class MappingRuntimeFieldTranslater extends CoreTestTranslater { @Override protected Map indexTemplate() { - return indexTemplateToAddRuntimeFieldsToMappings(); + return indexTemplateToAddRuntimeFields(); } @Override diff --git a/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java b/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java index 073a9c3555b1c..c3d8301373452 100644 --- a/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java +++ b/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java @@ -9,7 +9,6 @@ import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import org.apache.logging.log4j.LogManager; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.index.mapper.DateFieldMapper; @@ -49,7 +48,7 @@ public static Iterable parameters() throws Exception { private static class SearchRequestRuntimeFieldTranslater extends CoreTestTranslater { @Override protected Map indexTemplate() { - return indexTemplateToDisableAllFields(); + return indexTemplateToDisableRuntimeCompatibleFields(); } @Override @@ -116,14 +115,14 @@ protected boolean modifySearch(ApiCallSection search) { } private Map runtimeMappings(Object index) { - if (index != null) { + if (index != null && false == "_all".equals(index)) { return runtimeMappings.get(index); } // No mapping index specified in the search, if there is just one index we can just us it if (runtimeMappings.size() == 1) { return runtimeMappings.values().iterator().next(); } - // NOCOMMIT try and merge the mappings + // TODO try and merge the mappings if targeting more than one index return null; } diff --git a/x-pack/plugin/runtime-fields/qa/src/main/java/org/elasticsearch/xpack/runtimefields/test/CoreTestTranslater.java b/x-pack/plugin/runtime-fields/qa/src/main/java/org/elasticsearch/xpack/runtimefields/test/CoreTestTranslater.java index 26d6a131b4cf4..62625cf575f16 100644 --- a/x-pack/plugin/runtime-fields/qa/src/main/java/org/elasticsearch/xpack/runtimefields/test/CoreTestTranslater.java +++ b/x-pack/plugin/runtime-fields/qa/src/main/java/org/elasticsearch/xpack/runtimefields/test/CoreTestTranslater.java @@ -112,11 +112,32 @@ private static String painlessToLoadFromSource(String name, String type) { protected abstract Map indexTemplate(); - protected static Map indexTemplateToDisableAllFields() { - return Map.of("settings", Map.of(), "mappings", Map.of("dynamic", false)); + protected static Map indexTemplateToDisableRuntimeCompatibleFields() { + List> dynamicTemplates = new ArrayList<>(); + for (String type : PAINLESS_TO_EMIT.keySet()) { + if (type.equals("ip")) { + // There isn't a dynamic template to pick up ips. They'll just look like strings. + continue; + } + Map mapping = Map.of("type", type, "index", false, "doc_values", false); + if (type.equals("keyword")) { + /* + * For "string"-type dynamic mappings emulate our default + * behavior with a top level text field and a `.keyword` + * multi-field. In our case we disable the keyword field + * and substitute it with an enabled one on the search + * request. + */ + mapping = Map.of("type", "text", "fields", Map.of("keyword", mapping)); + dynamicTemplates.add(Map.of(type, Map.of("match_mapping_type", "string", "mapping", mapping))); + } else { + dynamicTemplates.add(Map.of(type, Map.of("match_mapping_type", type, "mapping", mapping))); + } + } + return Map.of("settings", Map.of(), "mappings", Map.of("dynamic_templates", dynamicTemplates)); } - protected static Map indexTemplateToAddRuntimeFieldsToMappings() { + protected static Map indexTemplateToAddRuntimeFields() { List> dynamicTemplates = new ArrayList<>(); for (String type : PAINLESS_TO_EMIT.keySet()) { if (type.equals("ip")) { @@ -128,7 +149,7 @@ protected static Map indexTemplateToAddRuntimeFieldsToMappings() Map.entry("runtime_type", type), Map.entry("script", painlessToLoadFromSource("{name}", type)) ); - if (type.contentEquals("keyword")) { + if (type.equals("keyword")) { /* * For "string"-type dynamic mappings emulate our default * behavior with a top level text field and a `.keyword` From c2100371ffd6e03a55a0fd3e1c84330618ab6f4e Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 5 Nov 2020 09:02:11 -0500 Subject: [PATCH 14/36] Refactor --- .../elasticsearch/client/RequestLogger.java | 4 +- .../index/query/QueryShardContext.java | 4 + .../index/search/QueryParserHelper.java | 5 ++ .../index/query/QueryShardContextTests.java | 11 ++- .../mapped/CoreWithMappedRuntimeFieldsIT.java | 4 +- .../CoreTestsWithSearchRuntimeFieldsIT.java | 13 ++- .../test/CoreTestTranslater.java | 83 +++++++------------ 7 files changed, 62 insertions(+), 62 deletions(-) diff --git a/client/rest/src/main/java/org/elasticsearch/client/RequestLogger.java b/client/rest/src/main/java/org/elasticsearch/client/RequestLogger.java index 41271ab69d35c..7c56a7edf97a9 100644 --- a/client/rest/src/main/java/org/elasticsearch/client/RequestLogger.java +++ b/client/rest/src/main/java/org/elasticsearch/client/RequestLogger.java @@ -65,7 +65,7 @@ static void logResponse(Log logger, HttpUriRequest request, HttpHost host, HttpR logger.warn(buildWarningMessage(request, host, warnings)); } } - if (tracer.isErrorEnabled()) { + if (tracer.isTraceEnabled()) { String requestLine; try { requestLine = buildTraceRequest(request, host); @@ -80,7 +80,7 @@ static void logResponse(Log logger, HttpUriRequest request, HttpHost host, HttpR responseLine = ""; tracer.trace("error while reading response for trace purposes", e); } - tracer.error(requestLine + '\n' + responseLine); + tracer.trace(requestLine + '\n' + responseLine); } } diff --git a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java index 635de7d686d7a..563165596f3f0 100644 --- a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java @@ -19,6 +19,7 @@ package org.elasticsearch.index.query; +import org.apache.logging.log4j.LogManager; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.index.IndexReader; import org.apache.lucene.search.IndexSearcher; @@ -32,6 +33,7 @@ import org.elasticsearch.client.Client; import org.elasticsearch.common.CheckedFunction; import org.elasticsearch.common.ParsingException; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.TriFunction; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.lucene.search.Queries; @@ -339,6 +341,8 @@ public boolean isFieldMapped(String name) { } private MappedFieldType fieldType(String name) { + LogManager.getLogger().error("ADSFDSAFaaa {} {} {}", name, runtimeMappings, Strings.toString(mapperService.documentMapper())); + MappedFieldType fieldType = runtimeMappings.get(name); return fieldType == null ? mapperService.fieldType(name) : fieldType; } diff --git a/server/src/main/java/org/elasticsearch/index/search/QueryParserHelper.java b/server/src/main/java/org/elasticsearch/index/search/QueryParserHelper.java index 4ad45e2752f93..54032873a01df 100644 --- a/server/src/main/java/org/elasticsearch/index/search/QueryParserHelper.java +++ b/server/src/main/java/org/elasticsearch/index/search/QueryParserHelper.java @@ -19,6 +19,7 @@ package org.elasticsearch.index.search; +import org.apache.logging.log4j.LogManager; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.index.mapper.MappedFieldType; @@ -117,22 +118,26 @@ static Map resolveMappingFields(QueryShardContext context, static Map resolveMappingField(QueryShardContext context, String fieldOrPattern, float weight, boolean acceptAllTypes, boolean acceptMetadataField, String fieldSuffix) { Set allFields = context.simpleMatchToIndexNames(fieldOrPattern); + LogManager.getLogger().error("ADSFDSAF {} {}", fieldOrPattern, allFields); Map fields = new HashMap<>(); for (String fieldName : allFields) { if (fieldSuffix != null && context.getFieldType(fieldName + fieldSuffix) != null) { fieldName = fieldName + fieldSuffix; } + LogManager.getLogger().error("ADSFDSAF1 {}", fieldName); if (context.isFieldMapped(fieldName) == false) { continue; } + LogManager.getLogger().error("ADSFDSAF2 {}", fieldName); MappedFieldType fieldType = context.getFieldType(fieldName); if (acceptMetadataField == false && fieldType.name().startsWith("_")) { // Ignore metadata fields continue; } + LogManager.getLogger().error("ADSFDSAF3 {}", fieldName); if (acceptAllTypes == false) { if (fieldType.getTextSearchInfo() == TextSearchInfo.NONE) { diff --git a/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java b/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java index 666ad6b608fd1..9ac959a8249e3 100644 --- a/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java @@ -324,12 +324,14 @@ public Mapper build(BuilderContext context) { }); } })); + when(mapperService.fieldType("pig")).thenReturn(new DummyMappedFieldType("pig")); + when(mapperService.simpleMatchToFullName("*")).thenReturn(Set.of("pig")); /* * Making these immutable here test that we don't modify them. * Modifying them would cause all kinds of problems if two * shards are parsed on the same node. */ - Map runtimeFields = Map.ofEntries( + Map runtimeMappings = Map.ofEntries( Map.entry("cat", Map.of("type", "keyword")), Map.entry("dog", Map.of("type", "keyword")) ); @@ -351,7 +353,7 @@ public Mapper build(BuilderContext context) { null, () -> true, null, - runtimeFields + runtimeMappings ); assertTrue(qsc.isFieldMapped("cat")); assertThat(qsc.getFieldType("cat"), instanceOf(DummyMappedFieldType.class)); @@ -359,7 +361,10 @@ public Mapper build(BuilderContext context) { assertTrue(qsc.isFieldMapped("dog")); assertThat(qsc.getFieldType("dog"), instanceOf(DummyMappedFieldType.class)); assertThat(qsc.simpleMatchToIndexNames("dog"), equalTo(Set.of("dog"))); - assertThat(qsc.simpleMatchToIndexNames("*"), equalTo(Set.of("cat", "dog"))); + assertTrue(qsc.isFieldMapped("pig")); + assertThat(qsc.getFieldType("pig"), instanceOf(DummyMappedFieldType.class)); + assertThat(qsc.simpleMatchToIndexNames("pig"), equalTo(Set.of("pig"))); + assertThat(qsc.simpleMatchToIndexNames("*"), equalTo(Set.of("cat", "dog", "pig"))); } public static QueryShardContext createQueryShardContext(String indexUuid, String clusterAlias) { diff --git a/x-pack/plugin/runtime-fields/qa/core-with-mapped/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/mapped/CoreWithMappedRuntimeFieldsIT.java b/x-pack/plugin/runtime-fields/qa/core-with-mapped/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/mapped/CoreWithMappedRuntimeFieldsIT.java index b126cf919fbd8..9a2973ac8eac2 100644 --- a/x-pack/plugin/runtime-fields/qa/core-with-mapped/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/mapped/CoreWithMappedRuntimeFieldsIT.java +++ b/x-pack/plugin/runtime-fields/qa/core-with-mapped/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/mapped/CoreWithMappedRuntimeFieldsIT.java @@ -35,8 +35,8 @@ public static Iterable parameters() throws Exception { private static class MappingRuntimeFieldTranslater extends CoreTestTranslater { @Override - protected Map indexTemplate() { - return indexTemplateToAddRuntimeFields(); + protected Map dynamicTemplateFor(String type) { + return dynamicTemplateToAddRuntimeFields(type); } @Override diff --git a/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java b/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java index c3d8301373452..2427af0a31037 100644 --- a/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java +++ b/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java @@ -45,10 +45,16 @@ public static Iterable parameters() throws Exception { return new SearchRequestRuntimeFieldTranslater().parameters(); } + /** + * Translating the tests is fairly difficult here because instead of ES + * tracking the mappings we have to track them. We don't have to do it as + * well as ES, just well enough that we can decorate the search requests + * with types that make most tests "just work". + */ private static class SearchRequestRuntimeFieldTranslater extends CoreTestTranslater { @Override - protected Map indexTemplate() { - return indexTemplateToDisableRuntimeCompatibleFields(); + protected Map dynamicTemplateFor(String type) { + return dynamicTemplateToDisableRuntimeCompatibleFields(type); } @Override @@ -85,7 +91,6 @@ protected boolean modifyMapping(String index, Map mapping) { if (properties == null || false == (properties instanceof Map)) { return true; } - mapping.put("dynamic", false); @SuppressWarnings("unchecked") Map propertiesMap = (Map) properties; Map untouchedMapping = new HashMap<>(); @@ -105,7 +110,7 @@ protected boolean modifySearch(ApiCallSection search) { search.addBody(new HashMap<>()); } for (Map body : search.getBodies()) { - Map runtimeMapping = runtimeMappings(body.get("index")); + Map runtimeMapping = runtimeMappings(search.getParams().get("index")); if (runtimeMapping == null) { return false; } diff --git a/x-pack/plugin/runtime-fields/qa/src/main/java/org/elasticsearch/xpack/runtimefields/test/CoreTestTranslater.java b/x-pack/plugin/runtime-fields/qa/src/main/java/org/elasticsearch/xpack/runtimefields/test/CoreTestTranslater.java index 62625cf575f16..a513a97e929e0 100644 --- a/x-pack/plugin/runtime-fields/qa/src/main/java/org/elasticsearch/xpack/runtimefields/test/CoreTestTranslater.java +++ b/x-pack/plugin/runtime-fields/qa/src/main/java/org/elasticsearch/xpack/runtimefields/test/CoreTestTranslater.java @@ -110,59 +110,18 @@ private static String painlessToLoadFromSource(String name, String type) { ) ); - protected abstract Map indexTemplate(); + protected abstract Map dynamicTemplateFor(String type); - protected static Map indexTemplateToDisableRuntimeCompatibleFields() { - List> dynamicTemplates = new ArrayList<>(); - for (String type : PAINLESS_TO_EMIT.keySet()) { - if (type.equals("ip")) { - // There isn't a dynamic template to pick up ips. They'll just look like strings. - continue; - } - Map mapping = Map.of("type", type, "index", false, "doc_values", false); - if (type.equals("keyword")) { - /* - * For "string"-type dynamic mappings emulate our default - * behavior with a top level text field and a `.keyword` - * multi-field. In our case we disable the keyword field - * and substitute it with an enabled one on the search - * request. - */ - mapping = Map.of("type", "text", "fields", Map.of("keyword", mapping)); - dynamicTemplates.add(Map.of(type, Map.of("match_mapping_type", "string", "mapping", mapping))); - } else { - dynamicTemplates.add(Map.of(type, Map.of("match_mapping_type", type, "mapping", mapping))); - } - } - return Map.of("settings", Map.of(), "mappings", Map.of("dynamic_templates", dynamicTemplates)); + protected static Map dynamicTemplateToDisableRuntimeCompatibleFields(String type) { + return Map.of("type", type, "index", false, "doc_values", false); } - protected static Map indexTemplateToAddRuntimeFields() { - List> dynamicTemplates = new ArrayList<>(); - for (String type : PAINLESS_TO_EMIT.keySet()) { - if (type.equals("ip")) { - // There isn't a dynamic template to pick up ips. They'll just look like strings. - continue; - } - Map mapping = Map.ofEntries( - Map.entry("type", "runtime"), - Map.entry("runtime_type", type), - Map.entry("script", painlessToLoadFromSource("{name}", type)) - ); - if (type.equals("keyword")) { - /* - * For "string"-type dynamic mappings emulate our default - * behavior with a top level text field and a `.keyword` - * multi-field. But instead of the default, use a runtime - * field for the multi-field. - */ - mapping = Map.of("type", "text", "fields", Map.of("keyword", mapping)); - dynamicTemplates.add(Map.of(type, Map.of("match_mapping_type", "string", "mapping", mapping))); - } else { - dynamicTemplates.add(Map.of(type, Map.of("match_mapping_type", type, "mapping", mapping))); - } - } - return Map.of("settings", Map.of(), "mappings", Map.of("dynamic_templates", dynamicTemplates)); + protected static Map dynamicTemplateToAddRuntimeFields(String type) { + return Map.ofEntries( + Map.entry("type", "runtime"), + Map.entry("runtime_type", type), + Map.entry("script", painlessToLoadFromSource("{name}", type)) + ); } protected static Map runtimeFieldLoadingFromSource(String name, String type) { @@ -182,11 +141,33 @@ public XContentLocation getLocation() { @Override public void execute(ClientYamlTestExecutionContext executionContext) throws IOException { Map params = Map.of("name", "hack_dynamic_mappings", "create", "true"); + List> dynamicTemplates = new ArrayList<>(); + for (String type : PAINLESS_TO_EMIT.keySet()) { + if (type.equals("ip")) { + // There isn't a dynamic template to pick up ips. They'll just look like strings. + continue; + } + Map mapping = dynamicTemplateFor(type); + if (type.equals("keyword")) { + /* + * For "string"-type dynamic mappings emulate our default + * behavior with a top level text field and a `.keyword` + * multi-field. In our case we disable the keyword field + * and substitute it with an enabled one on the search + * request. + */ + mapping = Map.of("type", "text", "fields", Map.of("keyword", mapping)); + dynamicTemplates.add(Map.of(type, Map.of("match_mapping_type", "string", "mapping", mapping))); + } else { + dynamicTemplates.add(Map.of(type, Map.of("match_mapping_type", type, "mapping", mapping))); + } + } + Map indexTemplate = Map.of("settings", Map.of(), "mappings", Map.of("dynamic_templates", dynamicTemplates)); List> bodies = List.of( Map.ofEntries( Map.entry("index_patterns", "*"), Map.entry("priority", Integer.MAX_VALUE - 1), - Map.entry("template", indexTemplate()) + Map.entry("template", indexTemplate) ) ); ClientYamlTestResponse response = executionContext.callApi("indices.put_index_template", params, bodies, Map.of()); From 685085695d0ce26ba55d0ba290755b5eabdd0311 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 5 Nov 2020 11:56:39 -0500 Subject: [PATCH 15/36] Refactor --- .../mapped/CoreWithMappedRuntimeFieldsIT.java | 17 ++++++----------- .../CoreTestsWithSearchRuntimeFieldsIT.java | 13 ++++--------- .../test/CoreTestTranslater.java | 19 +++++++++---------- 3 files changed, 19 insertions(+), 30 deletions(-) diff --git a/x-pack/plugin/runtime-fields/qa/core-with-mapped/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/mapped/CoreWithMappedRuntimeFieldsIT.java b/x-pack/plugin/runtime-fields/qa/core-with-mapped/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/mapped/CoreWithMappedRuntimeFieldsIT.java index 9a2973ac8eac2..0c19187c088d0 100644 --- a/x-pack/plugin/runtime-fields/qa/core-with-mapped/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/mapped/CoreWithMappedRuntimeFieldsIT.java +++ b/x-pack/plugin/runtime-fields/qa/core-with-mapped/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/mapped/CoreWithMappedRuntimeFieldsIT.java @@ -43,16 +43,10 @@ protected Map dynamicTemplateFor(String type) { protected Suite suite(ClientYamlTestCandidate candidate) { return new Suite(candidate) { @Override - protected boolean modifyMapping(String index, Map mapping) { - Object properties = mapping.get("properties"); - if (properties == null || false == (properties instanceof Map)) { - return true; - } - @SuppressWarnings("unchecked") - Map propertiesMap = (Map) properties; - Map newProperties = new HashMap<>(propertiesMap.size()); - Map> runtimeProperties = new HashMap<>(propertiesMap.size()); - if (false == runtimeifyMappingProperties(propertiesMap, newProperties, runtimeProperties)) { + protected boolean modifyMappingProperties(String index, Map properties) { + Map newProperties = new HashMap<>(properties.size()); + Map> runtimeProperties = new HashMap<>(properties.size()); + if (false == runtimeifyMappingProperties(properties, newProperties, runtimeProperties)) { return false; } for (Map.Entry> runtimeProperty : runtimeProperties.entrySet()) { @@ -60,7 +54,8 @@ protected boolean modifyMapping(String index, Map mapping) { runtimeProperty.getValue().put("type", "runtime"); newProperties.put(runtimeProperty.getKey(), runtimeProperty.getValue()); } - mapping.put("properties", newProperties); + properties.clear(); + properties.putAll(newProperties); return true; } diff --git a/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java b/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java index 2427af0a31037..b3844d83930c6 100644 --- a/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java +++ b/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java @@ -86,19 +86,14 @@ public boolean modifySections(List executables) { } @Override - protected boolean modifyMapping(String index, Map mapping) { - Object properties = mapping.get("properties"); - if (properties == null || false == (properties instanceof Map)) { - return true; - } - @SuppressWarnings("unchecked") - Map propertiesMap = (Map) properties; + protected boolean modifyMappingProperties(String index, Map properties) { Map untouchedMapping = new HashMap<>(); Map> runtimeMapping = new HashMap<>(); - if (false == runtimeifyMappingProperties(propertiesMap, untouchedMapping, runtimeMapping)) { + if (false == runtimeifyMappingProperties(properties, untouchedMapping, runtimeMapping)) { return false; } - mapping.put("properties", untouchedMapping); + properties.clear(); + properties.putAll(untouchedMapping); mappedFields.put(index, untouchedMapping.keySet()); runtimeMappings.put(index, runtimeMapping); return true; diff --git a/x-pack/plugin/runtime-fields/qa/src/main/java/org/elasticsearch/xpack/runtimefields/test/CoreTestTranslater.java b/x-pack/plugin/runtime-fields/qa/src/main/java/org/elasticsearch/xpack/runtimefields/test/CoreTestTranslater.java index a513a97e929e0..bc03972da2000 100644 --- a/x-pack/plugin/runtime-fields/qa/src/main/java/org/elasticsearch/xpack/runtimefields/test/CoreTestTranslater.java +++ b/x-pack/plugin/runtime-fields/qa/src/main/java/org/elasticsearch/xpack/runtimefields/test/CoreTestTranslater.java @@ -125,10 +125,7 @@ protected static Map dynamicTemplateToAddRuntimeFields(String ty } protected static Map runtimeFieldLoadingFromSource(String name, String type) { - return Map.ofEntries( - Map.entry("type", type), - Map.entry("script", painlessToLoadFromSource(name, type)) - ); + return Map.of("type", type, "script", painlessToLoadFromSource(name, type)); } private ExecutableSection addIndexTemplate() { @@ -268,9 +265,13 @@ private boolean modifyCreateIndex(ApiCallSection createIndex) { if (false == (mapping instanceof Map)) { continue; } + Object properties = ((Map) mapping).get("properties"); + if (false == (properties instanceof Map)) { + continue; + } @SuppressWarnings("unchecked") - Map mappingMap = (Map) mapping; - if (false == modifyMapping(index, mappingMap)) { + Map propertiesMap = (Map) properties; + if (false == modifyMappingProperties(index, propertiesMap)) { return false; } } @@ -280,7 +281,7 @@ private boolean modifyCreateIndex(ApiCallSection createIndex) { /** * Modify the mapping defined in the test. */ - protected abstract boolean modifyMapping(String index, Map mapping); + protected abstract boolean modifyMappingProperties(String index, Map properties); /** * Modify the provided map in place, translating all fields into @@ -375,9 +376,7 @@ private boolean handleBulk(ApiCallSection bulk) { null, true, XContentType.JSON, - (index, type) -> { - indexRequests.add(index); - }, + (index, type) -> indexRequests.add(index), u -> {}, d -> {} ); From 88926d44ed9992615319b4d52c096d98be2d8d61 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 5 Nov 2020 12:12:56 -0500 Subject: [PATCH 16/36] Missing tests --- .../search/internal/ShardSearchRequest.java | 4 ++++ .../action/search/ExpandSearchPhaseTests.java | 13 +++++++------ .../search/AbstractSearchTestCase.java | 4 ++-- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java b/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java index 4b494781b8ad9..ecb8fed920ffa 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java +++ b/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java @@ -523,6 +523,10 @@ public static QueryBuilder parseAliasFilter(CheckedFunction getRuntimeMappings() { return source != null && source.runtimeMappings() != null ? source.runtimeMappings() : emptyMap(); } diff --git a/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java b/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java index a55a3b0e3c000..31846467a6e35 100644 --- a/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java @@ -26,6 +26,7 @@ import org.elasticsearch.index.query.InnerHitBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.AbstractSearchTestCase; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.builder.SearchSourceBuilder; @@ -38,10 +39,13 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; import java.util.stream.IntStream; +import static org.hamcrest.Matchers.equalTo; + public class ExpandSearchPhaseTests extends ESTestCase { public void testCollapseSingleHit() throws IOException { @@ -58,6 +62,7 @@ public void testCollapseSingleHit() throws IOException { AtomicBoolean executedMultiSearch = new AtomicBoolean(false); QueryBuilder originalQuery = randomBoolean() ? null : QueryBuilders.termQuery("foo", "bar"); + Map runtimeMappings = randomBoolean() ? null : AbstractSearchTestCase.randomRandomMappings(); final MockSearchPhaseContext mockSearchPhaseContext = new MockSearchPhaseContext(1); String collapseValue = randomBoolean() ? null : "boom"; @@ -66,7 +71,7 @@ public void testCollapseSingleHit() throws IOException { .collapse(new CollapseBuilder("someField") .setInnerHits(IntStream.range(0, numInnerHits).mapToObj(hitNum -> new InnerHitBuilder().setName("innerHit" + hitNum)) .collect(Collectors.toList())))); - mockSearchPhaseContext.getRequest().source().query(originalQuery); + mockSearchPhaseContext.getRequest().source().query(originalQuery).runtimeMappings(runtimeMappings); mockSearchPhaseContext.searchTransport = new SearchTransportService(null, null, null) { @Override void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionListener listener) { @@ -85,7 +90,7 @@ void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionL assertThat(groupBuilder.must(), Matchers.contains(QueryBuilders.termQuery("foo", "bar"))); } assertArrayEquals(mockSearchPhaseContext.getRequest().indices(), searchRequest.indices()); - + assertThat(searchRequest.source().runtimeMappings(), equalTo(runtimeMappings)); List mSearchResponses = new ArrayList<>(numInnerHits); for (int innerHitNum = 0; innerHitNum < numInnerHits; innerHitNum++) { @@ -240,8 +245,4 @@ void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionL mockSearchPhaseContext.assertNoFailure(); assertNotNull(mockSearchPhaseContext.searchResponse.get()); } - - public void testCollapseWithRuntimeField() { - // NOCOMMIT write me. - } } diff --git a/server/src/test/java/org/elasticsearch/search/AbstractSearchTestCase.java b/server/src/test/java/org/elasticsearch/search/AbstractSearchTestCase.java index 3caf5ee9ef27c..7bef2d5811fa7 100644 --- a/server/src/test/java/org/elasticsearch/search/AbstractSearchTestCase.java +++ b/server/src/test/java/org/elasticsearch/search/AbstractSearchTestCase.java @@ -92,10 +92,10 @@ protected SearchSourceBuilder createSearchSourceBuilder() { QueryRescorerBuilderTests::randomRescoreBuilder, randomExtBuilders, CollapseBuilderTests::randomCollapseBuilder, - this::randomRandomMappings); + AbstractSearchTestCase::randomRandomMappings); } - protected Map randomRandomMappings() { + public static Map randomRandomMappings() { int count = between(1, 100); Map runtimeFields = new HashMap<>(count); while (runtimeFields.size() < count) { From 99d3e7d69514af5ea489ac4a40fc33c67baa786b Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 5 Nov 2020 12:25:01 -0500 Subject: [PATCH 17/36] Handle matching many --- .../CoreTestsWithSearchRuntimeFieldsIT.java | 35 +++++++++++++++---- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java b/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java index b3844d83930c6..2022ef94cffc7 100644 --- a/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java +++ b/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java @@ -10,6 +10,7 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; @@ -115,15 +116,35 @@ protected boolean modifySearch(ApiCallSection search) { } private Map runtimeMappings(Object index) { - if (index != null && false == "_all".equals(index)) { - return runtimeMappings.get(index); + if (index == null) { + return mergeMappings("*"); } - // No mapping index specified in the search, if there is just one index we can just us it - if (runtimeMappings.size() == 1) { - return runtimeMappings.values().iterator().next(); + String indexStr = index.toString(); + if ("_all".equals(indexStr)) { + return mergeMappings("*"); } - // TODO try and merge the mappings if targeting more than one index - return null; + if (Regex.isSimpleMatchPattern(indexStr)) { + return runtimeMappings.get(indexStr); + } + return mergeMappings(indexStr); + } + + private Map mergeMappings(String pattern) { + Map> merged = new HashMap<>(); + for (Map.Entry>> indexEntry: runtimeMappings.entrySet()) { + if (false == Regex.simpleMatch(pattern, indexEntry.getKey())) { + continue; + } + for (Map.Entry> field : indexEntry.getValue().entrySet()) { + Map mergedConfig = merged.get(field.getKey()); + if (mergedConfig == null) { + merged.put(field.getKey(), field.getValue()); + } else if (false == mergedConfig.equals(field.getValue())) { + return null; + } + } + } + return merged; } @Override From 88be56b1c07bb4312aee68b7985a97a6302d0b30 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 5 Nov 2020 12:40:36 -0500 Subject: [PATCH 18/36] cleanup --- .../test/search/CoreTestsWithSearchRuntimeFieldsIT.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java b/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java index 2022ef94cffc7..59b3255e51edf 100644 --- a/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java +++ b/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java @@ -131,7 +131,7 @@ protected boolean modifySearch(ApiCallSection search) { private Map mergeMappings(String pattern) { Map> merged = new HashMap<>(); - for (Map.Entry>> indexEntry: runtimeMappings.entrySet()) { + for (Map.Entry>> indexEntry : runtimeMappings.entrySet()) { if (false == Regex.simpleMatch(pattern, indexEntry.getKey())) { continue; } @@ -140,6 +140,7 @@ protected boolean modifySearch(ApiCallSection search) { if (mergedConfig == null) { merged.put(field.getKey(), field.getValue()); } else if (false == mergedConfig.equals(field.getValue())) { + // The two indices have different runtime mappings for a field so we have to give up on running the test. return null; } } From 458df74bc3de9800f7e73069ad9a6a2cbf83e46c Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 5 Nov 2020 12:50:25 -0500 Subject: [PATCH 19/36] precommit --- .../org/elasticsearch/index/query/QueryShardContextTests.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java b/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java index 9ac959a8249e3..6ec3ffe790250 100644 --- a/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java @@ -50,12 +50,12 @@ import org.elasticsearch.index.fielddata.LeafFieldData; import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.fielddata.plain.AbstractLeafOrdinalsFieldData; +import org.elasticsearch.index.mapper.ContentPath; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.IndexFieldMapper; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; -import org.elasticsearch.index.mapper.Mapper.BuilderContext; import org.elasticsearch.index.mapper.Mapper.TypeParser; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.NumberFieldMapper; @@ -318,7 +318,7 @@ public void testRuntimeFields() throws IOException { public Map getMappers() { return Map.of("runtime", (name, node, parserContext) -> new Mapper.Builder(name) { @Override - public Mapper build(BuilderContext context) { + public Mapper build(ContentPath path) { return new DummyMapper(name, new DummyMappedFieldType(name)); } }); From f82f41ece7f5e6a94cebb1ac29df143a4d6c5245 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 5 Nov 2020 13:40:12 -0500 Subject: [PATCH 20/36] Fixup other test --- .../index/query/QueryShardContext.java | 4 -- .../index/search/QueryParserHelper.java | 5 --- .../CoreTestsWithSearchRuntimeFieldsIT.java | 38 +++++++++++++------ 3 files changed, 27 insertions(+), 20 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java index dd882978c8285..528daa70b7594 100644 --- a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java @@ -19,7 +19,6 @@ package org.elasticsearch.index.query; -import org.apache.logging.log4j.LogManager; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.index.IndexReader; import org.apache.lucene.search.IndexSearcher; @@ -33,7 +32,6 @@ import org.elasticsearch.client.Client; import org.elasticsearch.common.CheckedFunction; import org.elasticsearch.common.ParsingException; -import org.elasticsearch.common.Strings; import org.elasticsearch.common.TriFunction; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.lucene.search.Queries; @@ -341,8 +339,6 @@ public boolean isFieldMapped(String name) { } private MappedFieldType fieldType(String name) { - LogManager.getLogger().error("ADSFDSAFaaa {} {} {}", name, runtimeMappings, Strings.toString(mapperService.documentMapper())); - MappedFieldType fieldType = runtimeMappings.get(name); return fieldType == null ? mapperService.fieldType(name) : fieldType; } diff --git a/server/src/main/java/org/elasticsearch/index/search/QueryParserHelper.java b/server/src/main/java/org/elasticsearch/index/search/QueryParserHelper.java index 54032873a01df..4ad45e2752f93 100644 --- a/server/src/main/java/org/elasticsearch/index/search/QueryParserHelper.java +++ b/server/src/main/java/org/elasticsearch/index/search/QueryParserHelper.java @@ -19,7 +19,6 @@ package org.elasticsearch.index.search; -import org.apache.logging.log4j.LogManager; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.index.mapper.MappedFieldType; @@ -118,26 +117,22 @@ static Map resolveMappingFields(QueryShardContext context, static Map resolveMappingField(QueryShardContext context, String fieldOrPattern, float weight, boolean acceptAllTypes, boolean acceptMetadataField, String fieldSuffix) { Set allFields = context.simpleMatchToIndexNames(fieldOrPattern); - LogManager.getLogger().error("ADSFDSAF {} {}", fieldOrPattern, allFields); Map fields = new HashMap<>(); for (String fieldName : allFields) { if (fieldSuffix != null && context.getFieldType(fieldName + fieldSuffix) != null) { fieldName = fieldName + fieldSuffix; } - LogManager.getLogger().error("ADSFDSAF1 {}", fieldName); if (context.isFieldMapped(fieldName) == false) { continue; } - LogManager.getLogger().error("ADSFDSAF2 {}", fieldName); MappedFieldType fieldType = context.getFieldType(fieldName); if (acceptMetadataField == false && fieldType.name().startsWith("_")) { // Ignore metadata fields continue; } - LogManager.getLogger().error("ADSFDSAF3 {}", fieldName); if (acceptAllTypes == false) { if (fieldType.getTextSearchInfo() == TextSearchInfo.NONE) { diff --git a/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java b/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java index 59b3255e51edf..5c3c226a4bb87 100644 --- a/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java +++ b/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java @@ -9,6 +9,7 @@ import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.apache.logging.log4j.LogManager; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.xcontent.XContentHelper; @@ -19,6 +20,7 @@ import org.elasticsearch.test.rest.yaml.section.ExecutableSection; import org.elasticsearch.xpack.runtimefields.test.CoreTestTranslater; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -115,24 +117,24 @@ protected boolean modifySearch(ApiCallSection search) { return true; } - private Map runtimeMappings(Object index) { + private Map runtimeMappings(String index) { if (index == null) { - return mergeMappings("*"); + return mergeMappings(index, new String[] { "*" }); } - String indexStr = index.toString(); - if ("_all".equals(indexStr)) { - return mergeMappings("*"); + String[] patterns = Arrays.stream(index.split(",")).map(m -> m.equals("_all") ? "*" : m).toArray(String[]::new); + if (patterns.length == 0 && Regex.isSimpleMatchPattern(patterns[0])) { + return runtimeMappings.get(patterns[0]); } - if (Regex.isSimpleMatchPattern(indexStr)) { - return runtimeMappings.get(indexStr); - } - return mergeMappings(indexStr); + return mergeMappings(index, patterns); } - private Map mergeMappings(String pattern) { + private Map mergeMappings(String index, String[] patterns) { Map> merged = new HashMap<>(); for (Map.Entry>> indexEntry : runtimeMappings.entrySet()) { - if (false == Regex.simpleMatch(pattern, indexEntry.getKey())) { + if (index != null && index.equals("date*")) { + LogManager.getLogger().error("ADSFASFDDAF {} {} {}", patterns, indexEntry.getKey(), Regex.simpleMatch(patterns, indexEntry.getKey())); + } + if (false == Regex.simpleMatch(patterns, indexEntry.getKey())) { continue; } for (Map.Entry> field : indexEntry.getValue().entrySet()) { @@ -145,6 +147,20 @@ protected boolean modifySearch(ApiCallSection search) { } } } + for (Map.Entry> indexEntry : mappedFields.entrySet()) { + if (index != null && index.equals("date*")) { + LogManager.getLogger().error("ADSFASFDDAF {} {} {}", patterns, indexEntry.getKey(), Regex.simpleMatch(patterns, indexEntry.getKey())); + } + if (false == Regex.simpleMatch(patterns, indexEntry.getKey())) { + continue; + } + for (String mappedField :indexEntry.getValue()) { + if (merged.containsKey(mappedField)) { + // We have a runtime mappings for a field *and* regular mapping. We can't make this test work so skip it. + return null; + } + } + } return merged; } From f348d663d62b1422314cd20c84234d9b5f224e06 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 5 Nov 2020 14:47:24 -0500 Subject: [PATCH 21/36] Fixup --- .../CoreTestsWithSearchRuntimeFieldsIT.java | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java b/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java index 5c3c226a4bb87..40723d997c996 100644 --- a/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java +++ b/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java @@ -9,7 +9,6 @@ import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import org.apache.logging.log4j.LogManager; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.xcontent.XContentHelper; @@ -119,21 +118,18 @@ protected boolean modifySearch(ApiCallSection search) { private Map runtimeMappings(String index) { if (index == null) { - return mergeMappings(index, new String[] { "*" }); + return mergeMappings(new String[] { "*" }); } String[] patterns = Arrays.stream(index.split(",")).map(m -> m.equals("_all") ? "*" : m).toArray(String[]::new); if (patterns.length == 0 && Regex.isSimpleMatchPattern(patterns[0])) { return runtimeMappings.get(patterns[0]); } - return mergeMappings(index, patterns); + return mergeMappings(patterns); } - private Map mergeMappings(String index, String[] patterns) { + private Map mergeMappings(String[] patterns) { Map> merged = new HashMap<>(); for (Map.Entry>> indexEntry : runtimeMappings.entrySet()) { - if (index != null && index.equals("date*")) { - LogManager.getLogger().error("ADSFASFDDAF {} {} {}", patterns, indexEntry.getKey(), Regex.simpleMatch(patterns, indexEntry.getKey())); - } if (false == Regex.simpleMatch(patterns, indexEntry.getKey())) { continue; } @@ -148,15 +144,12 @@ protected boolean modifySearch(ApiCallSection search) { } } for (Map.Entry> indexEntry : mappedFields.entrySet()) { - if (index != null && index.equals("date*")) { - LogManager.getLogger().error("ADSFASFDDAF {} {} {}", patterns, indexEntry.getKey(), Regex.simpleMatch(patterns, indexEntry.getKey())); - } if (false == Regex.simpleMatch(patterns, indexEntry.getKey())) { continue; } - for (String mappedField :indexEntry.getValue()) { + for (String mappedField : indexEntry.getValue()) { if (merged.containsKey(mappedField)) { - // We have a runtime mappings for a field *and* regular mapping. We can't make this test work so skip it. + // We have a runtime mappings for a field *and* regular mapping. We can't make this test work so skip it. return null; } } From 67736fb6a595295c96e3fb3120d393ca40eeb0de Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Fri, 6 Nov 2020 09:01:27 -0500 Subject: [PATCH 22/36] Extra test! --- .../xpack/rollup/job/RollupIndexerIndexingTests.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java index c013a56df675c..c7d623f0ebbb7 100644 --- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java +++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java @@ -38,6 +38,7 @@ import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.query.RangeQueryBuilder; @@ -79,6 +80,7 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.core.IsInstanceOf.instanceOf; import static org.hamcrest.number.OrderingComparison.greaterThan; +import static org.mockito.Mockito.mock; public class RollupIndexerIndexingTests extends AggregatorTestCase { private QueryShardContext queryShardContext; @@ -88,7 +90,7 @@ public class RollupIndexerIndexingTests extends AggregatorTestCase { private void setup() { settings = createIndexSettings(); queryShardContext = new QueryShardContext(0, settings, - BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, + BigArrays.NON_RECYCLING_INSTANCE, null, null, mock(MapperService.class), null, null, null, null, null, null, () -> 0L, null, null, () -> true, null); } From 139ef5f0542361ff15af2dfd20d11a53c6c23c67 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Fri, 6 Nov 2020 10:58:25 -0500 Subject: [PATCH 23/36] More test --- .../xpack/wildcard/mapper/WildcardFieldMapperTests.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java index 7840b6c527d22..438d28b5730bc 100644 --- a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java +++ b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java @@ -53,6 +53,7 @@ import org.elasticsearch.index.mapper.ContentPath; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MapperTestCase; import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.query.QueryShardContext; @@ -897,7 +898,7 @@ protected final QueryShardContext createMockShardContext() { return builder.build(new IndexFieldDataCache.None(), null); }; return new QueryShardContext(0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, bitsetFilterCache, indexFieldDataLookup, - null, null, null, xContentRegistry(), null, null, null, + mock(MapperService.class), null, null, xContentRegistry(), null, null, null, () -> randomNonNegativeLong(), null, null, () -> true, null) { @Override From b336ff3f87026cb3d9481f74b16ba9841bdab266 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Mon, 9 Nov 2020 10:23:23 -0500 Subject: [PATCH 24/36] Iter --- .../index/query/QueryShardContext.java | 6 ++-- .../MetadataCreateIndexServiceTests.java | 3 +- .../index/mapper/DateFieldTypeTests.java | 8 ++--- .../index/mapper/IndexFieldTypeTests.java | 22 ++----------- .../index/mapper/NumberFieldTypeTests.java | 3 +- .../index/mapper/RangeFieldTypeTests.java | 3 +- .../index/query/QueryShardContextTests.java | 2 +- .../highlight/HighlightBuilderTests.java | 4 +-- .../rescore/QueryRescorerBuilderTests.java | 6 ++-- .../search/sort/AbstractSortTestCase.java | 4 +-- .../action/EnrichShardMultiSearchAction.java | 8 ++++- .../job/RollupIndexerIndexingTests.java | 4 +-- .../CoreTestsWithSearchRuntimeFieldsIT.java | 5 --- .../test/runtime_fields/10_keyword.yml | 33 ++++++++++++++++++- .../mapper/WildcardFieldMapperTests.java | 3 +- 15 files changed, 57 insertions(+), 57 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java index 715702a40c46a..41c7ab793b652 100644 --- a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java @@ -179,7 +179,7 @@ public QueryShardContext( ), allowExpensiveQueries, valuesSourceRegistry, - parseRuntimeMappings(runtimeMappings, mapperService::parserContext, indexSettings) + parseRuntimeMappings(runtimeMappings, mapperService, indexSettings) ); } @@ -622,7 +622,7 @@ public BigArrays bigArrays() { private static Map parseRuntimeMappings( Map mappings, - Supplier parserContextSupplier, + MapperService mapperService, IndexSettings indexSettings ) { Map runtimeMappings = new HashMap<>(); @@ -638,7 +638,7 @@ private static Map parseRuntimeMappings( if (oldRuntimeType != null) { throw new ElasticsearchParseException("use [type] in [runtime_mappings] instead of [runtime_type]"); } - runtimeMappings.put(field, buildFieldType("runtime", field, node, parserContextSupplier.get(), indexSettings)); + runtimeMappings.put(field, buildFieldType("runtime", field, node, mapperService.parserContext(), indexSettings)); } return runtimeMappings; } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java index 9006ec359f60e..a8489ed6008b7 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java @@ -110,7 +110,6 @@ import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.Matchers.startsWith; -import static org.mockito.Mockito.mock; public class MetadataCreateIndexServiceTests extends ESTestCase { @@ -126,7 +125,7 @@ public void setupCreateIndexRequestAndAliasValidator() { .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1).build(); queryShardContext = new QueryShardContext(0, new IndexSettings(IndexMetadata.builder("test").settings(indexSettings).build(), indexSettings), - BigArrays.NON_RECYCLING_INSTANCE, null, null, mock(MapperService.class), null, null, xContentRegistry(), writableRegistry(), + BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, xContentRegistry(), writableRegistry(), null, null, () -> randomNonNegativeLong(), null, null, () -> true, null); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java index b4b819d6d7eb2..95e09f229256c 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java @@ -59,8 +59,6 @@ import java.util.Collections; import java.util.List; -import static org.mockito.Mockito.mock; - public class DateFieldTypeTests extends FieldTypeTestCase { private static final long nowInMillis = 0; @@ -167,7 +165,7 @@ public void testTermQuery() { .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1).build(); QueryShardContext context = new QueryShardContext(0, new IndexSettings(IndexMetadata.builder("foo").settings(indexSettings).build(), indexSettings), - BigArrays.NON_RECYCLING_INSTANCE, null, null, mock(MapperService.class), null, null, + BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, xContentRegistry(), writableRegistry(), null, null, () -> nowInMillis, null, null, () -> true, null); MappedFieldType ft = new DateFieldType("field"); String date = "2015-10-12T14:10:55"; @@ -189,7 +187,7 @@ public void testRangeQuery() throws IOException { .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1).build(); QueryShardContext context = new QueryShardContext(0, new IndexSettings(IndexMetadata.builder("foo").settings(indexSettings).build(), indexSettings), - BigArrays.NON_RECYCLING_INSTANCE, null, null, mock(MapperService.class), null, null, xContentRegistry(), writableRegistry(), + BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, xContentRegistry(), writableRegistry(), null, null, () -> nowInMillis, null, null, () -> true, null); MappedFieldType ft = new DateFieldType("field"); String date1 = "2015-10-12T14:10:55"; @@ -233,7 +231,7 @@ public void testRangeQueryWithIndexSort() { IndexSettings indexSettings = new IndexSettings(indexMetadata, settings); QueryShardContext context = new QueryShardContext(0, indexSettings, - BigArrays.NON_RECYCLING_INSTANCE, null, null, mock(MapperService.class), null, null, xContentRegistry(), writableRegistry(), + BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, xContentRegistry(), writableRegistry(), null, null, () -> 0L, null, null, () -> true, null); MappedFieldType ft = new DateFieldType("field"); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldTypeTests.java index dc47c003f34a3..473eca61c71b2 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldTypeTests.java @@ -32,7 +32,6 @@ import java.util.function.Predicate; import static org.hamcrest.Matchers.containsString; -import static org.mockito.Mockito.mock; public class IndexFieldTypeTests extends ESTestCase { @@ -69,24 +68,7 @@ private QueryShardContext createContext() { IndexSettings indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY); Predicate indexNameMatcher = pattern -> Regex.simpleMatch(pattern, "index"); - return new QueryShardContext( - 0, - indexSettings, - null, - null, - null, - mock(MapperService.class), - null, - null, - xContentRegistry(), - writableRegistry(), - null, - null, - System::currentTimeMillis, - null, - indexNameMatcher, - () -> true, - null - ); + return new QueryShardContext(0, indexSettings, null, null, null, null, null, null, xContentRegistry(), writableRegistry(), + null, null, System::currentTimeMillis, null, indexNameMatcher, () -> true, null); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java index 7e5a9a49ae0e1..18ab76de7c93c 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java @@ -70,7 +70,6 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; -import static org.mockito.Mockito.mock; public class NumberFieldTypeTests extends FieldTypeTestCase { @@ -475,7 +474,7 @@ public void doTestIndexSortRangeQueries(NumberType type, Supplier valueS IndexSearcher searcher = newSearcher(reader); QueryShardContext context = new QueryShardContext(0, indexSettings, - BigArrays.NON_RECYCLING_INSTANCE, null, null, mock(MapperService.class), null, null, xContentRegistry(), writableRegistry(), + BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, xContentRegistry(), writableRegistry(), null, null, () -> 0L, null, null, () -> true, null); final int iters = 10; diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java index d289cab19a874..a7dbb0d2bc480 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java @@ -54,7 +54,6 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.instanceOf; -import static org.mockito.Mockito.mock; public class RangeFieldTypeTests extends FieldTypeTestCase { RangeType type; @@ -213,7 +212,7 @@ private QueryShardContext createContext() { Settings indexSettings = Settings.builder() .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).build(); IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(randomAlphaOfLengthBetween(1, 10), indexSettings); - return new QueryShardContext(0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, null, null, mock(MapperService.class), null, null, + return new QueryShardContext(0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, xContentRegistry(), writableRegistry(), null, null, () -> nowInMillis, null, null, () -> true, null); } diff --git a/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java b/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java index 6ec3ffe790250..bb21086561a9a 100644 --- a/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java @@ -196,7 +196,7 @@ public void testIndexSortedOnField() { BigArrays.NON_RECYCLING_INSTANCE, null, null, - mock(MapperService.class), + null, null, null, NamedXContentRegistry.EMPTY, diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java index b01ac76db228f..da8ba282e7d4b 100644 --- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java @@ -41,7 +41,6 @@ import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.mapper.ContentPath; import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.index.query.IdsQueryBuilder; import org.elasticsearch.index.query.MatchAllQueryBuilder; @@ -74,7 +73,6 @@ import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; -import static org.mockito.Mockito.mock; public class HighlightBuilderTests extends ESTestCase { @@ -281,7 +279,7 @@ public void testBuildSearchContextHighlight() throws IOException { IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(index, indexSettings); // shard context will only need indicesQueriesRegistry for building Query objects nested in highlighter QueryShardContext mockShardContext = new QueryShardContext(0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, - null, null, mock(MapperService.class), null, null, xContentRegistry(), namedWriteableRegistry, + null, null, null, null, null, xContentRegistry(), namedWriteableRegistry, null, null, System::currentTimeMillis, null, null, () -> true, null) { @Override public MappedFieldType getFieldType(String name) { diff --git a/server/src/test/java/org/elasticsearch/search/rescore/QueryRescorerBuilderTests.java b/server/src/test/java/org/elasticsearch/search/rescore/QueryRescorerBuilderTests.java index ce863c71873a6..0c04510442a9e 100644 --- a/server/src/test/java/org/elasticsearch/search/rescore/QueryRescorerBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/rescore/QueryRescorerBuilderTests.java @@ -40,7 +40,6 @@ import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.mapper.ContentPath; import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.index.query.MatchAllQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; @@ -59,7 +58,6 @@ import static java.util.Collections.emptyList; import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode; import static org.hamcrest.Matchers.containsString; -import static org.mockito.Mockito.mock; public class QueryRescorerBuilderTests extends ESTestCase { @@ -145,7 +143,7 @@ public void testBuildRescoreSearchContext() throws ElasticsearchParseException, IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(randomAlphaOfLengthBetween(1, 10), indexSettings); // shard context will only need indicesQueriesRegistry for building Query objects nested in query rescorer QueryShardContext mockShardContext = new QueryShardContext(0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, - null, null, mock(MapperService.class), null, null, + null, null, null, null, null, xContentRegistry(), namedWriteableRegistry, null, null, () -> nowInMillis, null, null, () -> true, null) { @Override public MappedFieldType getFieldType(String name) { @@ -189,7 +187,7 @@ public void testRewritingKeepsSettings() throws IOException { IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(randomAlphaOfLengthBetween(1, 10), indexSettings); // shard context will only need indicesQueriesRegistry for building Query objects nested in query rescorer QueryShardContext mockShardContext = new QueryShardContext(0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, - null, null, mock(MapperService.class), null, null, + null, null, null, null, null, xContentRegistry(), namedWriteableRegistry, null, null, () -> nowInMillis, null, null, () -> true, null) { @Override public MappedFieldType getFieldType(String name) { diff --git a/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java b/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java index bd94c7eb841d8..0da8c39ef9c42 100644 --- a/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java +++ b/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java @@ -41,7 +41,6 @@ import org.elasticsearch.index.fielddata.IndexFieldDataCache; import org.elasticsearch.index.mapper.ContentPath; import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.index.mapper.ObjectMapper; import org.elasticsearch.index.mapper.ObjectMapper.Nested; @@ -72,7 +71,6 @@ import static java.util.Collections.emptyList; import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode; -import static org.mockito.Mockito.mock; public abstract class AbstractSortTestCase> extends ESTestCase { @@ -203,7 +201,7 @@ protected final QueryShardContext createMockShardContext(IndexSearcher searcher) return builder.build(new IndexFieldDataCache.None(), null); }; return new QueryShardContext(0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, bitsetFilterCache, indexFieldDataLookup, - mock(MapperService.class), null, scriptService, xContentRegistry(), namedWriteableRegistry, null, searcher, + null, null, scriptService, xContentRegistry(), namedWriteableRegistry, null, searcher, () -> randomNonNegativeLong(), null, null, () -> true, null) { @Override diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchAction.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchAction.java index 44a9248aae845..52f821199a06a 100644 --- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchAction.java +++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchAction.java @@ -231,12 +231,18 @@ protected MultiSearchResponse shardOperation(Request request, ShardId shardId) t final IndexShard indexShard = indicesService.getShardOrNull(shardId); try (Engine.Searcher searcher = indexShard.acquireSearcher("enrich_msearch")) { final FieldsVisitor visitor = new FieldsVisitor(true); + /* + * Enrich doesn't support defining runtime fields in the search + * request so we use an empty map to signal that they aren't + * any. + */ + Map runtimeFields = emptyMap(); final QueryShardContext context = indexService.newQueryShardContext( shardId.id(), searcher, () -> { throw new UnsupportedOperationException(); }, null, - emptyMap() // Enrich doesn't support defining runtime fields + runtimeFields ); final MultiSearchResponse.Item[] items = new MultiSearchResponse.Item[request.multiSearchRequest.requests().size()]; for (int i = 0; i < request.multiSearchRequest.requests().size(); i++) { diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java index c7d623f0ebbb7..c013a56df675c 100644 --- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java +++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java @@ -38,7 +38,6 @@ import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.query.RangeQueryBuilder; @@ -80,7 +79,6 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.core.IsInstanceOf.instanceOf; import static org.hamcrest.number.OrderingComparison.greaterThan; -import static org.mockito.Mockito.mock; public class RollupIndexerIndexingTests extends AggregatorTestCase { private QueryShardContext queryShardContext; @@ -90,7 +88,7 @@ public class RollupIndexerIndexingTests extends AggregatorTestCase { private void setup() { settings = createIndexSettings(); queryShardContext = new QueryShardContext(0, settings, - BigArrays.NON_RECYCLING_INSTANCE, null, null, mock(MapperService.class), null, null, + BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, null, null, null, null, () -> 0L, null, null, () -> true, null); } diff --git a/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java b/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java index 40723d997c996..02092699ecc02 100644 --- a/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java +++ b/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java @@ -37,11 +37,6 @@ public CoreTestsWithSearchRuntimeFieldsIT(@Name("yaml") ClientYamlTestCandidate super(testCandidate); } - @Override - protected boolean randomizeContentType() { // NOCOMMIT remove me - return false; - } - @ParametersFactory public static Iterable parameters() throws Exception { return new SearchRequestRuntimeFieldTranslater().parameters(); diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_keyword.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_keyword.yml index 09d547029f9e8..733c2f0b50190 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_keyword.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_keyword.yml @@ -385,7 +385,6 @@ setup: - match: {hits.hits.0._source.voltage: 4.0} - match: {hits.hits.0.fields.voltage_rating: [low]} - --- "match defined on search request": - do: @@ -436,3 +435,35 @@ setup: sort: timestamp - match: {hits.total.value: 2} - match: {hits.hits.0._source.voltage: 5.1} + + +--- +"replace object field on search request": + - do: + bulk: + index: student + refresh: true + body: | + {"index":{}} + {"name": {"first": "Andrew", "last": "Wiggin"}} + {"index":{}} + {"name": {"first": "Julian", "last": "Delphiki", "suffix": "II"}} + + - do: + search: + index: student + body: + runtime_mappings: + name.first: + type: keyword + script: | + if ('Wiggin'.equals(doc['name.last.keyword'].value)) { + emit('Ender'); + } else if ('Delphiki'.equals(doc['name.last.keyword'].value)) { + emit('Bean'); + } + query: + match: + name.first: Bean + - match: {hits.total.value: 1} + - match: {hits.hits.0._source.name.first: Julian} diff --git a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java index 438d28b5730bc..7840b6c527d22 100644 --- a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java +++ b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java @@ -53,7 +53,6 @@ import org.elasticsearch.index.mapper.ContentPath; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MapperTestCase; import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.query.QueryShardContext; @@ -898,7 +897,7 @@ protected final QueryShardContext createMockShardContext() { return builder.build(new IndexFieldDataCache.None(), null); }; return new QueryShardContext(0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, bitsetFilterCache, indexFieldDataLookup, - mock(MapperService.class), null, null, xContentRegistry(), null, null, null, + null, null, null, xContentRegistry(), null, null, null, () -> randomNonNegativeLong(), null, null, () -> true, null) { @Override From f0f6d69aa3cdc93145955ee82d11ccfcd9735ff4 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Mon, 9 Nov 2020 10:24:45 -0500 Subject: [PATCH 25/36] Fixup --- .../test/search.aggregation/340_geo_distance.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/340_geo_distance.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/340_geo_distance.yml index 58fa1b04a7b7f..3d37f1948278c 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/340_geo_distance.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/340_geo_distance.yml @@ -1,7 +1,7 @@ setup: - do: indices.create: - index: test_geoadsf + index: test body: mappings: properties: @@ -9,7 +9,7 @@ setup: type: geo_point - do: bulk: - index: test_geoadsf + index: test refresh: true body: - '{"index": {}}' From 218314e03d602f35d297a25d832d6db3db99fddd Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Mon, 9 Nov 2020 10:26:17 -0500 Subject: [PATCH 26/36] remove TODO not much duplication --- .../java/org/elasticsearch/index/query/QueryShardContext.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java index 41c7ab793b652..ec5d3140e9943 100644 --- a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java @@ -301,7 +301,6 @@ public boolean hasMappings() { * type then the fields will be returned with a type prefix. */ public Set simpleMatchToIndexNames(String pattern) { - // TODO remove the duplication with MapperService and FieldTypeLookup if (runtimeMappings.isEmpty()) { return mapperService.simpleMatchToFullName(pattern); } From 93ebadc911f207bad26818d08010b98ee17a4518 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Mon, 9 Nov 2020 10:27:37 -0500 Subject: [PATCH 27/36] Drop noop change --- .../org/elasticsearch/index/mapper/NumberFieldTypeTests.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java index 18ab76de7c93c..07ad7e6a7282b 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java @@ -20,7 +20,6 @@ package org.elasticsearch.index.mapper; import com.carrotsearch.randomizedtesting.generators.RandomPicks; - import org.apache.lucene.document.Document; import org.apache.lucene.document.DoublePoint; import org.apache.lucene.document.FloatPoint; From a1843ddbdabedec91ab2fdbb9313fa03e40dc7a2 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Mon, 9 Nov 2020 10:46:46 -0500 Subject: [PATCH 28/36] Clean up build --- .../test/index/80_date_nanos.yml | 33 ++++++++++ .../test/search/240_date_nanos.yml | 22 ------- .../action/search/ExpandSearchPhaseTests.java | 2 +- .../search/AbstractSearchTestCase.java | 4 +- x-pack/plugin/runtime-fields/qa/build.gradle | 65 +++++++++++++++++++ .../qa/core-with-mapped/build.gradle | 55 +--------------- .../qa/core-with-search/build.gradle | 56 +--------------- 7 files changed, 103 insertions(+), 134 deletions(-) create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/index/80_date_nanos.yml diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/index/80_date_nanos.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/index/80_date_nanos.yml new file mode 100644 index 0000000000000..ada085ebb148e --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/index/80_date_nanos.yml @@ -0,0 +1,33 @@ +--- +"date_nanos requires dates after 1970 and before 2262": + + - do: + indices.create: + index: date_ns + body: + settings: + number_of_shards: 3 + number_of_replicas: 0 + mappings: + properties: + date: + type: date_nanos + field: + type: long + + - do: + bulk: + refresh: true + body: + - '{ "index" : { "_index" : "date_ns", "_id" : "date_ns_1" } }' + - '{"date" : "1969-10-28T12:12:12.123456789Z" }' + - '{ "index" : { "_index" : "date_ns", "_id" : "date_ns_2" } }' + - '{"date" : "2263-10-29T12:12:12.123456789Z" }' + + - match: { errors: true } + - match: { items.0.index.status: 400 } + - match: { items.0.index.error.type: mapper_parsing_exception } + - match: { items.0.index.error.caused_by.reason: "date[1969-10-28T12:12:12.123456789Z] is before the epoch in 1970 and cannot be stored in nanosecond resolution" } + - match: { items.1.index.status: 400 } + - match: { items.1.index.error.type: mapper_parsing_exception } + - match: { items.1.index.error.caused_by.reason: "date[2263-10-29T12:12:12.123456789Z] is after 2262-04-11T23:47:16.854775807 and cannot be stored in nanosecond resolution" } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/240_date_nanos.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/240_date_nanos.yml index cc9bd063ff4fc..007f7f7f0e88e 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/240_date_nanos.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/240_date_nanos.yml @@ -71,28 +71,6 @@ setup: - match: { hits.hits.1._id: "second" } - match: { hits.hits.1.sort: [1540815132987654321] } - ---- -"date_nanos requires dates after 1970 and before 2262": - - - do: - bulk: - refresh: true - body: - - '{ "index" : { "_index" : "date_ns", "_id" : "date_ns_1" } }' - - '{"date" : "1969-10-28T12:12:12.123456789Z" }' - - '{ "index" : { "_index" : "date_ns", "_id" : "date_ns_2" } }' - - '{"date" : "2263-10-29T12:12:12.123456789Z" }' - - - match: { errors: true } - - match: { items.0.index.status: 400 } - - match: { items.0.index.error.type: mapper_parsing_exception } - - match: { items.0.index.error.caused_by.reason: "date[1969-10-28T12:12:12.123456789Z] is before the epoch in 1970 and cannot be stored in nanosecond resolution" } - - match: { items.1.index.status: 400 } - - match: { items.1.index.error.type: mapper_parsing_exception } - - match: { items.1.index.error.caused_by.reason: "date[2263-10-29T12:12:12.123456789Z] is after 2262-04-11T23:47:16.854775807 and cannot be stored in nanosecond resolution" } - - --- "doc value fields are working as expected across date and date_nanos fields": diff --git a/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java b/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java index 31846467a6e35..f43e785529a98 100644 --- a/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java @@ -62,7 +62,7 @@ public void testCollapseSingleHit() throws IOException { AtomicBoolean executedMultiSearch = new AtomicBoolean(false); QueryBuilder originalQuery = randomBoolean() ? null : QueryBuilders.termQuery("foo", "bar"); - Map runtimeMappings = randomBoolean() ? null : AbstractSearchTestCase.randomRandomMappings(); + Map runtimeMappings = randomBoolean() ? null : AbstractSearchTestCase.randomRuntimeMappings(); final MockSearchPhaseContext mockSearchPhaseContext = new MockSearchPhaseContext(1); String collapseValue = randomBoolean() ? null : "boom"; diff --git a/server/src/test/java/org/elasticsearch/search/AbstractSearchTestCase.java b/server/src/test/java/org/elasticsearch/search/AbstractSearchTestCase.java index 7bef2d5811fa7..8b82205c5efd2 100644 --- a/server/src/test/java/org/elasticsearch/search/AbstractSearchTestCase.java +++ b/server/src/test/java/org/elasticsearch/search/AbstractSearchTestCase.java @@ -92,10 +92,10 @@ protected SearchSourceBuilder createSearchSourceBuilder() { QueryRescorerBuilderTests::randomRescoreBuilder, randomExtBuilders, CollapseBuilderTests::randomCollapseBuilder, - AbstractSearchTestCase::randomRandomMappings); + AbstractSearchTestCase::randomRuntimeMappings); } - public static Map randomRandomMappings() { + public static Map randomRuntimeMappings() { int count = between(1, 100); Map runtimeFields = new HashMap<>(count); while (runtimeFields.size() < count) { diff --git a/x-pack/plugin/runtime-fields/qa/build.gradle b/x-pack/plugin/runtime-fields/qa/build.gradle index 4f108f906c60c..61e9797913849 100644 --- a/x-pack/plugin/runtime-fields/qa/build.gradle +++ b/x-pack/plugin/runtime-fields/qa/build.gradle @@ -7,3 +7,68 @@ dependencies { } test.enabled = false // We don't currently have any tests for this because they are test utilities. + +subprojects { + if (project.name.startsWith('core-with-')) { + apply plugin: 'elasticsearch.yaml-rest-test' + + dependencies { + yamlRestTestImplementation xpackProject("plugin:runtime-fields:qa") + } + + restResources { + restApi { + includeXpack 'async_search', 'graph', '*_point_in_time' + } + restTests { + includeCore '*' + includeXpack 'async_search', 'graph' + } + } + + testClusters.yamlRestTest { + testDistribution = 'DEFAULT' + setting 'xpack.license.self_generated.type', 'trial' + } + + yamlRestTest { + def suites = [ + 'async_search', + 'search', + 'search.aggregation', + 'search.highlight', + 'search.inner_hits', + 'search_shards', + 'suggest', + ] + if (project.name.equals('core-with-mapped')) { + suites += [ + // These two don't support runtime fields on the request. Should they? + 'field_caps', + 'graph', + // The search request tests don't know how to support msearch for now + 'msearch', + ] + } + systemProperty 'tests.rest.suite', suites.join(',') + systemProperty 'tests.rest.blacklist', + [ + /////// TO FIX /////// + 'search.highlight/40_keyword_ignore/Plain Highligher should skip highlighting ignored keyword values', // The plain highlighter is incompatible with runtime fields. Worth fixing? + 'search/115_multiple_field_collapsing/two levels fields collapsing', // Broken. Gotta fix. + 'field_caps/30_filter/Field caps with index filter', // We don't support filtering field caps on runtime fields. What should we do? + 'search.aggregation/10_histogram/*', // runtime doesn't support sub-fields. Maybe it should? + 'search/140_pre_filter_search_shards/pre_filter_shard_size with shards that have no hit', + /////// TO FIX /////// + + /////// NOT SUPPORTED /////// + 'search.aggregation/280_rare_terms/*', // Requires an index and we won't have it + // Runtime fields don't have global ords + 'search.aggregation/20_terms/string profiler via global ordinals', + 'search.aggregation/20_terms/Global ordinals are loaded with the global_ordinals execution hint', + 'search.aggregation/170_cardinality_metric/profiler string', + /////// NOT SUPPORTED /////// + ].join(',') + } + } +} \ No newline at end of file diff --git a/x-pack/plugin/runtime-fields/qa/core-with-mapped/build.gradle b/x-pack/plugin/runtime-fields/qa/core-with-mapped/build.gradle index 7af3a9d6886da..ea347a8a55e7a 100644 --- a/x-pack/plugin/runtime-fields/qa/core-with-mapped/build.gradle +++ b/x-pack/plugin/runtime-fields/qa/core-with-mapped/build.gradle @@ -1,54 +1 @@ -apply plugin: 'elasticsearch.yaml-rest-test' - -dependencies { - yamlRestTestImplementation xpackProject("plugin:runtime-fields:qa") -} - -restResources { - restApi { - includeXpack 'async_search', 'graph', '*_point_in_time' - } - restTests { - includeCore '*' - includeXpack 'async_search', 'graph' - } -} - -testClusters.yamlRestTest { - testDistribution = 'DEFAULT' - setting 'xpack.license.self_generated.type', 'trial' -} - -yamlRestTest { - systemProperty 'tests.rest.suite', - [ - 'async_search', - 'field_caps', - 'graph', - 'msearch', - 'search', - 'search.aggregation', - 'search.highlight', - 'search.inner_hits', - 'search_shards', - 'suggest', - ].join(',') - systemProperty 'tests.rest.blacklist', - [ - /////// TO FIX /////// - 'search.highlight/40_keyword_ignore/Plain Highligher should skip highlighting ignored keyword values', // The plain highlighter is incompatible with runtime fields. Worth fixing? - 'search/115_multiple_field_collapsing/two levels fields collapsing', // Broken. Gotta fix. - 'field_caps/30_filter/Field caps with index filter', // We don't support filtering field caps on runtime fields. What should we do? - 'search.aggregation/10_histogram/*', // runtime doesn't support sub-fields. Maybe it should? - 'search/140_pre_filter_search_shards/pre_filter_shard_size with shards that have no hit', - /////// TO FIX /////// - - /////// NOT SUPPORTED /////// - 'search.aggregation/280_rare_terms/*', // Requires an index and we won't have it - // Runtime fields don't have global ords - 'search.aggregation/20_terms/string profiler via global ordinals', - 'search.aggregation/20_terms/Global ordinals are loaded with the global_ordinals execution hint', - 'search.aggregation/170_cardinality_metric/profiler string', - /////// NOT SUPPORTED /////// - ].join(',') -} +// Configured by parent project diff --git a/x-pack/plugin/runtime-fields/qa/core-with-search/build.gradle b/x-pack/plugin/runtime-fields/qa/core-with-search/build.gradle index 25d1b6d47bce6..ea347a8a55e7a 100644 --- a/x-pack/plugin/runtime-fields/qa/core-with-search/build.gradle +++ b/x-pack/plugin/runtime-fields/qa/core-with-search/build.gradle @@ -1,55 +1 @@ -apply plugin: 'elasticsearch.yaml-rest-test' - -dependencies { - yamlRestTestImplementation xpackProject("plugin:runtime-fields:qa") -} - -restResources { - restApi { - includeXpack 'async_search', 'graph', '*_point_in_time' - } - restTests { - includeCore '*' - includeXpack 'async_search', 'graph' - } -} - -testClusters.yamlRestTest { - testDistribution = 'DEFAULT' - setting 'xpack.license.self_generated.type', 'trial' -} - -yamlRestTest { - systemProperty 'tests.rest.suite', - [ - 'async_search', -// 'field_caps', These two don't support runtime fields on the request. Should they? -// 'graph', -// 'msearch', This one just doesn't have infrastructure to hack the runtime fields into place - 'search', - 'search.aggregation', - 'search.highlight', - 'search.inner_hits', - 'search_shards', - 'suggest', - ].join(',') - systemProperty 'tests.rest.blacklist', - [ - /////// TO FIX /////// - 'search.highlight/40_keyword_ignore/Plain Highligher should skip highlighting ignored keyword values', // The plain highlighter is incompatible with runtime fields. Worth fixing? - 'search/115_multiple_field_collapsing/two levels fields collapsing', // Broken. Gotta fix. - 'field_caps/30_filter/Field caps with index filter', // We don't support filtering field caps on runtime fields. What should we do? - 'search.aggregation/10_histogram/*', // runtime doesn't support sub-fields. Maybe it should? - 'search/140_pre_filter_search_shards/pre_filter_shard_size with shards that have no hit', - 'search/240_date_nanos/date_nanos requires dates after 1970 and before 2262', // This isn't a search test and should probably move - /////// TO FIX /////// - - /////// NOT SUPPORTED /////// - 'search.aggregation/280_rare_terms/*', // Requires an index and we won't have it - // Runtime fields don't have global ords - 'search.aggregation/20_terms/string profiler via global ordinals', - 'search.aggregation/20_terms/Global ordinals are loaded with the global_ordinals execution hint', - 'search.aggregation/170_cardinality_metric/profiler string', - /////// NOT SUPPORTED /////// - ].join(',') -} +// Configured by parent project From 34abd68240e9111965339454ee5361fff682cac6 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Mon, 9 Nov 2020 11:25:22 -0500 Subject: [PATCH 29/36] Fail if sending runtime fields to old --- .../search/builder/SearchSourceBuilder.java | 6 ++++ .../builder/SearchSourceBuilderTests.java | 34 +++++++++++-------- 2 files changed, 25 insertions(+), 15 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java index d342ed4223115..943f1a1aea164 100644 --- a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java @@ -328,6 +328,12 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(true); out.writeMap(runtimeMappings); } + } else { + if (runtimeMappings != null) { + throw new IllegalArgumentException( + "Versions before 8.0.0 don't support [runtime_mappings] and search was sent to [" + out.getVersion() + "]" + ); + } } } diff --git a/server/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java b/server/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java index 8ce36d4295cc2..a219e5540d9eb 100644 --- a/server/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java @@ -20,11 +20,10 @@ package org.elasticsearch.search.builder; import com.fasterxml.jackson.core.JsonParseException; + +import org.elasticsearch.Version; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.io.stream.BytesStreamOutput; -import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput; -import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -93,16 +92,18 @@ private static void assertParseSearchSource(SearchSourceBuilder testBuilder, XCo } public void testSerialization() throws IOException { - SearchSourceBuilder testBuilder = createSearchSourceBuilder(); - try (BytesStreamOutput output = new BytesStreamOutput()) { - testBuilder.writeTo(output); - try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) { - SearchSourceBuilder deserializedBuilder = new SearchSourceBuilder(in); - assertEquals(deserializedBuilder, testBuilder); - assertEquals(deserializedBuilder.hashCode(), testBuilder.hashCode()); - assertNotSame(deserializedBuilder, testBuilder); - } - } + SearchSourceBuilder original = createSearchSourceBuilder(); + SearchSourceBuilder copy = copyBuilder(original); + assertEquals(copy, original); + assertEquals(copy.hashCode(), original.hashCode()); + assertNotSame(copy, original); + } + + public void testSerializingWithRuntimeFieldsBeforeSupportedThrows() { + SearchSourceBuilder original = new SearchSourceBuilder().runtimeMappings(randomRuntimeMappings()); + Version v = Version.CURRENT.minimumCompatibilityVersion(); + Exception e = expectThrows(IllegalArgumentException.class, () -> copyBuilder(original, v)); + assertThat(e.getMessage(), equalTo("Versions before 8.0.0 don't support [runtime_mappings] and search was sent to [" + v + "]")); } public void testShallowCopy() { @@ -118,9 +119,12 @@ public void testEqualsAndHashcode() throws IOException { EqualsHashCodeTestUtils.checkEqualsAndHashCode(createSearchSourceBuilder(), this::copyBuilder); } - //we use the streaming infra to create a copy of the builder provided as argument private SearchSourceBuilder copyBuilder(SearchSourceBuilder original) throws IOException { - return ESTestCase.copyWriteable(original, namedWriteableRegistry, SearchSourceBuilder::new); + return copyBuilder(original, Version.CURRENT); + } + + private SearchSourceBuilder copyBuilder(SearchSourceBuilder original, Version version) throws IOException { + return ESTestCase.copyWriteable(original, namedWriteableRegistry, SearchSourceBuilder::new, version); } public void testParseIncludeExclude() throws IOException { From f91f5670c613baec68bbfb906e001f19e4bee97d Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Mon, 9 Nov 2020 11:52:01 -0500 Subject: [PATCH 30/36] Pin version --- .../elasticsearch/search/builder/SearchSourceBuilderTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java b/server/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java index a219e5540d9eb..c68f8243640e9 100644 --- a/server/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java @@ -101,7 +101,7 @@ public void testSerialization() throws IOException { public void testSerializingWithRuntimeFieldsBeforeSupportedThrows() { SearchSourceBuilder original = new SearchSourceBuilder().runtimeMappings(randomRuntimeMappings()); - Version v = Version.CURRENT.minimumCompatibilityVersion(); + Version v = Version.V_8_0_0.minimumCompatibilityVersion(); Exception e = expectThrows(IllegalArgumentException.class, () -> copyBuilder(original, v)); assertThat(e.getMessage(), equalTo("Versions before 8.0.0 don't support [runtime_mappings] and search was sent to [" + v + "]")); } From a2d8e2045a6eb6ba90933c9e5e152035d2492d4b Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Mon, 9 Nov 2020 12:32:48 -0500 Subject: [PATCH 31/36] iter --- .../org/elasticsearch/index/query/QueryShardContext.java | 2 +- .../xpack/enrich/action/EnrichShardMultiSearchAction.java | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java index ec5d3140e9943..07447cf790e28 100644 --- a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java @@ -378,7 +378,7 @@ private static MappedFieldType buildFieldType( throw new IllegalArgumentException("No mapper found for type [" + type + "]"); } Mapper.Builder builder = typeParser.parse(field, node, parserContext); - Mapper mapper = builder.build(new ContentPath(1)); + Mapper mapper = builder.build(new ContentPath(0)); if (mapper instanceof FieldMapper) { return ((FieldMapper)mapper).fieldType(); } diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchAction.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchAction.java index 52f821199a06a..7cd1e85366989 100644 --- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchAction.java +++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchAction.java @@ -232,8 +232,10 @@ protected MultiSearchResponse shardOperation(Request request, ShardId shardId) t try (Engine.Searcher searcher = indexShard.acquireSearcher("enrich_msearch")) { final FieldsVisitor visitor = new FieldsVisitor(true); /* - * Enrich doesn't support defining runtime fields in the search - * request so we use an empty map to signal that they aren't + * Enrich doesn't support defining runtime fields in its + * configuration. We could add support for that if we'd + * like it but, for now at least, you can't configure any + * runtime fields so it is safe to build the context without * any. */ Map runtimeFields = emptyMap(); From f19c2c1cc3b5d468a2479f3d62f5d66953b54425 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Mon, 9 Nov 2020 12:52:00 -0500 Subject: [PATCH 32/36] Default to emptyMap --- .../search/DefaultSearchContext.java | 2 +- .../elasticsearch/search/SearchService.java | 2 +- .../search/builder/SearchSourceBuilder.java | 18 ++++++------------ .../search/internal/ShardSearchRequest.java | 8 -------- 4 files changed, 8 insertions(+), 22 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java index a604b5a97a147..713e591152c2e 100644 --- a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java +++ b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java @@ -175,7 +175,7 @@ final class DefaultSearchContext extends SearchContext { this.relativeTimeSupplier = relativeTimeSupplier; this.timeout = timeout; queryShardContext = indexService.newQueryShardContext(request.shardId().id(), this.searcher, - request::nowInMillis, shardTarget.getClusterAlias(), request.getRuntimeMappings()); + request::nowInMillis, shardTarget.getClusterAlias(), request.source().runtimeMappings()); queryBoost = request.indexBoost(); this.lowLevelCancellation = lowLevelCancellation; } diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java index f976a05ba510e..467f35fa28db6 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchService.java +++ b/server/src/main/java/org/elasticsearch/search/SearchService.java @@ -1178,7 +1178,7 @@ private CanMatchResponse canMatch(ShardSearchRequest request, boolean checkRefre try (canMatchSearcher) { QueryShardContext context = indexService.newQueryShardContext(request.shardId().id(), canMatchSearcher, - request::nowInMillis, request.getClusterAlias(), request.getRuntimeMappings()); + request::nowInMillis, request.getClusterAlias(), request.source().runtimeMappings()); Rewriteable.rewrite(request.getRewriteable(), context, false); final boolean aliasFilterCanMatch = request.getAliasFilter() .getQueryBuilder() instanceof MatchNoneQueryBuilder == false; diff --git a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java index 943f1a1aea164..09b62b36248ca 100644 --- a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java @@ -66,6 +66,7 @@ import java.util.Map; import java.util.Objects; +import static java.util.Collections.emptyMap; import static org.elasticsearch.index.query.AbstractQueryBuilder.parseInnerQueryBuilder; import static org.elasticsearch.search.internal.SearchContext.TRACK_TOTAL_HITS_ACCURATE; import static org.elasticsearch.search.internal.SearchContext.TRACK_TOTAL_HITS_DISABLED; @@ -193,7 +194,7 @@ public static HighlightBuilder highlight() { private PointInTimeBuilder pointInTimeBuilder = null; - private Map runtimeMappings = null; + private Map runtimeMappings = emptyMap(); /** * Constructs a new search source builder. @@ -256,9 +257,7 @@ public SearchSourceBuilder(StreamInput in) throws IOException { pointInTimeBuilder = in.readOptionalWriteable(PointInTimeBuilder::new); } if (in.getVersion().onOrAfter(Version.V_8_0_0)) { - if (in.readBoolean()) { - runtimeMappings = in.readMap(); - } + runtimeMappings = in.readMap(); } } @@ -322,14 +321,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalWriteable(pointInTimeBuilder); } if (out.getVersion().onOrAfter(Version.V_8_0_0)) { - if (runtimeMappings == null) { - out.writeBoolean(false); - } else { - out.writeBoolean(true); - out.writeMap(runtimeMappings); - } + out.writeMap(runtimeMappings); } else { - if (runtimeMappings != null) { + if (false == runtimeMappings.isEmpty()) { throw new IllegalArgumentException( "Versions before 8.0.0 don't support [runtime_mappings] and search was sent to [" + out.getVersion() + "]" ); @@ -1007,7 +1001,7 @@ public Map runtimeMappings() { * Specify the mappings specified on this search request that override built in mappings. */ public SearchSourceBuilder runtimeMappings(Map runtimeMappings) { - this.runtimeMappings = runtimeMappings; + this.runtimeMappings = runtimeMappings == null ? emptyMap() : runtimeMappings; return this; } diff --git a/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java b/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java index ecb8fed920ffa..38ce395ba6d8c 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java +++ b/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java @@ -522,12 +522,4 @@ public static QueryBuilder parseAliasFilter(CheckedFunction getRuntimeMappings() { - return source != null && source.runtimeMappings() != null ? source.runtimeMappings() : emptyMap(); - } } From 862d02adbadba602aebcaaf6bb04fe122bde11ce Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Mon, 9 Nov 2020 13:08:31 -0500 Subject: [PATCH 33/36] fixup --- .../org/elasticsearch/search/builder/SearchSourceBuilder.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java index 09b62b36248ca..c4b0752447b49 100644 --- a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java @@ -1411,7 +1411,7 @@ public XContentBuilder innerToXContent(XContentBuilder builder, Params params) t if (pointInTimeBuilder != null) { pointInTimeBuilder.toXContent(builder, params); } - if (runtimeMappings != null) { + if (false == runtimeMappings.isEmpty()) { builder.field(RUNTIME_MAPPINGS_FIELD.getPreferredName(), runtimeMappings); } From 3a26fe4a0671555d1cf14bf04535e79f7b9e9b38 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Mon, 9 Nov 2020 13:10:44 -0500 Subject: [PATCH 34/36] iter --- .../elasticsearch/search/DefaultSearchContext.java | 12 ++++++++++-- .../search/internal/ShardSearchRequest.java | 1 - 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java index 713e591152c2e..c7b2152d0e47f 100644 --- a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java +++ b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java @@ -77,6 +77,8 @@ import java.util.Map; import java.util.function.LongSupplier; +import static java.util.Collections.emptyMap; + final class DefaultSearchContext extends SearchContext { private final ReaderContext readerContext; @@ -174,8 +176,14 @@ final class DefaultSearchContext extends SearchContext { this.relativeTimeSupplier = relativeTimeSupplier; this.timeout = timeout; - queryShardContext = indexService.newQueryShardContext(request.shardId().id(), this.searcher, - request::nowInMillis, shardTarget.getClusterAlias(), request.source().runtimeMappings()); + Map runtimeMappings = request.source() == null ? emptyMap() : request.source().runtimeMappings(); + queryShardContext = indexService.newQueryShardContext( + request.shardId().id(), + this.searcher, + request::nowInMillis, + shardTarget.getClusterAlias(), + runtimeMappings + ); queryBoost = request.indexBoost(); this.lowLevelCancellation = lowLevelCancellation; } diff --git a/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java b/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java index 38ce395ba6d8c..ef5db6e8c3dba 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java +++ b/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java @@ -61,7 +61,6 @@ import java.util.Map; import java.util.function.Function; -import static java.util.Collections.emptyMap; import static org.elasticsearch.search.internal.SearchContext.TRACK_TOTAL_HITS_DISABLED; /** From f66b3e2024f2628451141eaa7e3be9b3c61c8d41 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Mon, 9 Nov 2020 14:08:07 -0500 Subject: [PATCH 35/36] Put it back --- .../java/org/elasticsearch/search/DefaultSearchContext.java | 5 +---- .../main/java/org/elasticsearch/search/SearchService.java | 2 +- .../elasticsearch/search/internal/ShardSearchRequest.java | 5 +++++ 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java index c7b2152d0e47f..1882f98c8ab09 100644 --- a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java +++ b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java @@ -77,8 +77,6 @@ import java.util.Map; import java.util.function.LongSupplier; -import static java.util.Collections.emptyMap; - final class DefaultSearchContext extends SearchContext { private final ReaderContext readerContext; @@ -176,13 +174,12 @@ final class DefaultSearchContext extends SearchContext { this.relativeTimeSupplier = relativeTimeSupplier; this.timeout = timeout; - Map runtimeMappings = request.source() == null ? emptyMap() : request.source().runtimeMappings(); queryShardContext = indexService.newQueryShardContext( request.shardId().id(), this.searcher, request::nowInMillis, shardTarget.getClusterAlias(), - runtimeMappings + request.getRuntimeMappings() ); queryBoost = request.indexBoost(); this.lowLevelCancellation = lowLevelCancellation; diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java index 467f35fa28db6..f976a05ba510e 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchService.java +++ b/server/src/main/java/org/elasticsearch/search/SearchService.java @@ -1178,7 +1178,7 @@ private CanMatchResponse canMatch(ShardSearchRequest request, boolean checkRefre try (canMatchSearcher) { QueryShardContext context = indexService.newQueryShardContext(request.shardId().id(), canMatchSearcher, - request::nowInMillis, request.getClusterAlias(), request.source().runtimeMappings()); + request::nowInMillis, request.getClusterAlias(), request.getRuntimeMappings()); Rewriteable.rewrite(request.getRewriteable(), context, false); final boolean aliasFilterCanMatch = request.getAliasFilter() .getQueryBuilder() instanceof MatchNoneQueryBuilder == false; diff --git a/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java b/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java index ef5db6e8c3dba..af2e46a65dffb 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java +++ b/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java @@ -61,6 +61,7 @@ import java.util.Map; import java.util.function.Function; +import static java.util.Collections.emptyMap; import static org.elasticsearch.search.internal.SearchContext.TRACK_TOTAL_HITS_DISABLED; /** @@ -521,4 +522,8 @@ public static QueryBuilder parseAliasFilter(CheckedFunction getRuntimeMappings() { + return source == null ? emptyMap() : source.runtimeMappings(); + } } From 6dfb44fe9a5b2b01cb42b56da45b9e8f9a3ff8dc Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Mon, 9 Nov 2020 15:05:02 -0500 Subject: [PATCH 36/36] Fixup --- .../elasticsearch/action/search/ExpandSearchPhaseTests.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java b/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java index f43e785529a98..76e77ce2c4671 100644 --- a/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java @@ -44,6 +44,7 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; +import static java.util.Collections.emptyMap; import static org.hamcrest.Matchers.equalTo; public class ExpandSearchPhaseTests extends ESTestCase { @@ -62,7 +63,7 @@ public void testCollapseSingleHit() throws IOException { AtomicBoolean executedMultiSearch = new AtomicBoolean(false); QueryBuilder originalQuery = randomBoolean() ? null : QueryBuilders.termQuery("foo", "bar"); - Map runtimeMappings = randomBoolean() ? null : AbstractSearchTestCase.randomRuntimeMappings(); + Map runtimeMappings = randomBoolean() ? emptyMap() : AbstractSearchTestCase.randomRuntimeMappings(); final MockSearchPhaseContext mockSearchPhaseContext = new MockSearchPhaseContext(1); String collapseValue = randomBoolean() ? null : "boom";