From a08b52f3bd736fd741a38f3a4493a183c2ad8a69 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Tue, 10 Nov 2020 12:38:59 -0500 Subject: [PATCH] Add `runtime_mappings` to search request (#64374) This adds a way to specify the `runtime_mappings` on a search request which are always "runtime" fields. It looks like: ``` curl -XDELETE -uelastic:password -HContent-Type:application/json localhost:9200/test curl -XPOST -uelastic:password -HContent-Type:application/json 'localhost:9200/test/_bulk?pretty&refresh' -d' {"index": {}} {"animal": "cat", "sound": "meow"} {"index": {}} {"animal": "dog", "sound": "woof"} {"index": {}} {"animal": "snake", "sound": "hisssssssssssssssss"} ' curl -XPOST -uelastic:password -HContent-Type:application/json localhost:9200/test/_search?pretty -d' { "runtime_mappings": { "animal.upper": { "type": "keyword", "script": "for (String s : doc[\"animal.keyword\"]) {emit(s.toUpperCase())}" } }, "query": { "match": { "animal.upper": "DOG" } } }' ``` NOTE: If we have to send a search request with runtime mappings to a node that doesn't support runtime mappings at all then we'll fail the search request entirely. The alternative would be to not send those runtime mappings and let the node fail the search request with an "unknown field" error. I believe this is would be hard to surprising because you defined the field in the search request. NOTE: It isn't obvious but you can also use `runtime_mappings` to override fields inside objects by naming the runtime fields with `.` in them. Like this: ``` curl -XDELETE -uelastic:password -HContent-Type:application/json localhost:9200/test curl -uelastic:password -XPOST -HContent-Type:application/json localhost:9200/test/_bulk?refresh -d' {"index":{}} {"name": {"first": "Andrew", "last": "Wiggin"}} {"index":{}} {"name": {"first": "Julian", "last": "Delphiki", "suffix": "II"}} ' curl -uelastic:password -XPOST -HContent-Type:application/json localhost:9200/test/_search?pretty -d'{ "runtime_mappings": { "name.first": { "type": "keyword", "script": "if (\"Wiggin\".equals(doc[\"name.last.keyword\"].value)) {emit(\"Ender\");} else if (\"Delphiki\".equals(doc[\"name.last.keyword\"].value)) {emit(\"Bean\");}" } }, "query": { "match": { "name.first": "Bean" } } }' ``` Relates to #59332 --- .../common/DisableGraphQueryTests.java | 3 +- .../action/PainlessExecuteAction.java | 3 +- .../painless/NeedsScoreTests.java | 4 +- .../PercolatorFieldMapperTests.java | 3 +- .../PercolatorQuerySearchTests.java | 3 +- ...ulkByScrollParallelizationHelperTests.java | 1 + .../test/index/80_date_nanos.yml | 33 ++ .../test/search/240_date_nanos.yml | 22 - .../TransportSimulateIndexTemplateAction.java | 3 +- .../action/search/ExpandSearchPhase.java | 3 +- .../metadata/MetadataCreateIndexService.java | 7 +- .../metadata/MetadataIndexAliasesService.java | 3 +- .../MetadataIndexTemplateService.java | 3 +- .../org/elasticsearch/index/IndexService.java | 12 +- .../index/query/QueryShardContext.java | 129 +++++- .../search/DefaultSearchContext.java | 9 +- .../elasticsearch/search/SearchService.java | 2 +- .../search/builder/SearchSourceBuilder.java | 44 +- .../search/internal/ShardSearchRequest.java | 7 +- .../action/search/ExpandSearchPhaseTests.java | 10 +- .../fielddata/AbstractFieldDataTestCase.java | 3 +- .../index/query/QueryShardContextTests.java | 166 ++++++- .../index/search/MultiMatchQueryTests.java | 35 +- .../index/search/NestedHelperTests.java | 4 +- .../search/nested/NestedSortingTests.java | 3 +- .../search/AbstractSearchTestCase.java | 17 +- .../search/DefaultSearchContextTests.java | 4 +- .../builder/SearchSourceBuilderTests.java | 34 +- .../search/RandomSearchRequestGenerator.java | 9 +- .../elasticsearch/test/TestSearchContext.java | 4 +- .../action/EnrichShardMultiSearchAction.java | 13 +- x-pack/plugin/runtime-fields/qa/build.gradle | 75 +++- .../qa/core-with-mapped/build.gradle | 1 + .../mapped/CoreWithMappedRuntimeFieldsIT.java | 76 ++++ .../qa/core-with-search/build.gradle | 1 + .../CoreTestsWithSearchRuntimeFieldsIT.java | 225 ++++++++++ .../runtime-fields/qa/rest/build.gradle | 50 --- .../rest/CoreTestsWithRuntimeFieldsIT.java | 259 ----------- .../test/CoreTestTranslater.java | 407 ++++++++++++++++++ .../xpack/security/Security.java | 6 +- .../test/runtime_fields/10_keyword.yml | 107 +++++ 41 files changed, 1382 insertions(+), 421 deletions(-) create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/index/80_date_nanos.yml create mode 100644 x-pack/plugin/runtime-fields/qa/core-with-mapped/build.gradle create mode 100644 x-pack/plugin/runtime-fields/qa/core-with-mapped/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/mapped/CoreWithMappedRuntimeFieldsIT.java create mode 100644 x-pack/plugin/runtime-fields/qa/core-with-search/build.gradle create mode 100644 x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java delete mode 100644 x-pack/plugin/runtime-fields/qa/rest/build.gradle delete mode 100644 x-pack/plugin/runtime-fields/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/rest/CoreTestsWithRuntimeFieldsIT.java create mode 100644 x-pack/plugin/runtime-fields/qa/src/main/java/org/elasticsearch/xpack/runtimefields/test/CoreTestTranslater.java diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/DisableGraphQueryTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/DisableGraphQueryTests.java index d1792e94f7331..c7ac35d3febce 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/DisableGraphQueryTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/DisableGraphQueryTests.java @@ -46,6 +46,7 @@ import java.util.Collection; import java.util.Collections; +import static java.util.Collections.emptyMap; import static org.hamcrest.Matchers.equalTo; /** @@ -84,7 +85,7 @@ public void setup() { indexService = createIndex("test", settings, "t", "text_shingle", "type=text,analyzer=text_shingle", "text_shingle_unigram", "type=text,analyzer=text_shingle_unigram"); - shardContext = indexService.newQueryShardContext(0, null, () -> 0L, null); + shardContext = indexService.newQueryShardContext(0, null, () -> 0L, null, emptyMap()); // parsed queries for "text_shingle_unigram:(foo bar baz)" with query parsers // that ignores position length attribute diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java index 21059f2112457..1810a1eff384a 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java @@ -85,6 +85,7 @@ import java.util.Map; import java.util.Objects; +import static java.util.Collections.emptyMap; import static org.elasticsearch.action.ValidateActions.addValidationError; import static org.elasticsearch.rest.RestRequest.Method.GET; import static org.elasticsearch.rest.RestRequest.Method.POST; @@ -555,7 +556,7 @@ private static Response prepareRamIndex(Request request, searcher.setQueryCache(null); final long absoluteStartMillis = System.currentTimeMillis(); QueryShardContext context = - indexService.newQueryShardContext(0, searcher, () -> absoluteStartMillis, null); + indexService.newQueryShardContext(0, searcher, () -> absoluteStartMillis, null, emptyMap()); return handler.apply(context, indexReader.leaves().get(0)); } } diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java index e9a6ca60509e3..ca885cdfdff6a 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java @@ -32,6 +32,8 @@ import java.util.List; import java.util.Map; +import static java.util.Collections.emptyMap; + /** * Test that needsScores() is reported correctly depending on whether _score is used */ @@ -45,7 +47,7 @@ public void testNeedsScores() { contexts.put(NumberSortScript.CONTEXT, Whitelist.BASE_WHITELISTS); PainlessScriptEngine service = new PainlessScriptEngine(Settings.EMPTY, contexts); - QueryShardContext shardContext = index.newQueryShardContext(0, null, () -> 0, null); + QueryShardContext shardContext = index.newQueryShardContext(0, null, () -> 0, null, emptyMap()); NumberSortScript.Factory factory = service.compile(null, "1.2", NumberSortScript.CONTEXT, Collections.emptyMap()); NumberSortScript.LeafFactory ss = factory.newFactory(Collections.emptyMap(), shardContext.lookup()); diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java index 19f12d6130d00..1d59ab64b8231 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java @@ -103,6 +103,7 @@ import java.util.function.Function; import java.util.stream.Collectors; +import static java.util.Collections.emptyMap; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.QueryBuilders.boolQuery; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; @@ -515,7 +516,7 @@ public void testQueryWithRewrite() throws Exception { QueryShardContext shardContext = indexService.newQueryShardContext( randomInt(20), null, () -> { throw new UnsupportedOperationException(); - }, null); + }, null, emptyMap()); PlainActionFuture future = new PlainActionFuture<>(); Rewriteable.rewriteAndFetch(queryBuilder, shardContext, future); assertQueryBuilder(qbSource, future.get()); diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchTests.java index e91a8cf1783e7..ff5d6dc7c4b7b 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchTests.java @@ -51,6 +51,7 @@ import java.util.Map; import java.util.function.Function; +import static java.util.Collections.emptyMap; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.QueryBuilders.boolQuery; import static org.elasticsearch.index.query.QueryBuilders.matchQuery; @@ -258,7 +259,7 @@ public void testRangeQueriesWithNow() throws Exception { try (Engine.Searcher searcher = indexService.getShard(0).acquireSearcher("test")) { long[] currentTime = new long[] {System.currentTimeMillis()}; QueryShardContext queryShardContext = - indexService.newQueryShardContext(0, searcher, () -> currentTime[0], null); + indexService.newQueryShardContext(0, searcher, () -> currentTime[0], null, emptyMap()); BytesReference source = BytesReference.bytes(jsonBuilder().startObject() .field("field1", "value") diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkByScrollParallelizationHelperTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkByScrollParallelizationHelperTests.java index a64415d08b1b1..b2512e6a62cfd 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkByScrollParallelizationHelperTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkByScrollParallelizationHelperTests.java @@ -38,6 +38,7 @@ public void testSliceIntoSubRequests() throws IOException { () -> null, () -> null, () -> emptyList(), + () -> null, () -> null)); if (searchRequest.source() != null) { // Clear the slice builder if there is one set. We can't call sliceIntoSubRequests if it is. diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/index/80_date_nanos.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/index/80_date_nanos.yml new file mode 100644 index 0000000000000..ada085ebb148e --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/index/80_date_nanos.yml @@ -0,0 +1,33 @@ +--- +"date_nanos requires dates after 1970 and before 2262": + + - do: + indices.create: + index: date_ns + body: + settings: + number_of_shards: 3 + number_of_replicas: 0 + mappings: + properties: + date: + type: date_nanos + field: + type: long + + - do: + bulk: + refresh: true + body: + - '{ "index" : { "_index" : "date_ns", "_id" : "date_ns_1" } }' + - '{"date" : "1969-10-28T12:12:12.123456789Z" }' + - '{ "index" : { "_index" : "date_ns", "_id" : "date_ns_2" } }' + - '{"date" : "2263-10-29T12:12:12.123456789Z" }' + + - match: { errors: true } + - match: { items.0.index.status: 400 } + - match: { items.0.index.error.type: mapper_parsing_exception } + - match: { items.0.index.error.caused_by.reason: "date[1969-10-28T12:12:12.123456789Z] is before the epoch in 1970 and cannot be stored in nanosecond resolution" } + - match: { items.1.index.status: 400 } + - match: { items.1.index.error.type: mapper_parsing_exception } + - match: { items.1.index.error.caused_by.reason: "date[2263-10-29T12:12:12.123456789Z] is after 2262-04-11T23:47:16.854775807 and cannot be stored in nanosecond resolution" } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/240_date_nanos.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/240_date_nanos.yml index cc9bd063ff4fc..007f7f7f0e88e 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/240_date_nanos.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/240_date_nanos.yml @@ -71,28 +71,6 @@ setup: - match: { hits.hits.1._id: "second" } - match: { hits.hits.1.sort: [1540815132987654321] } - ---- -"date_nanos requires dates after 1970 and before 2262": - - - do: - bulk: - refresh: true - body: - - '{ "index" : { "_index" : "date_ns", "_id" : "date_ns_1" } }' - - '{"date" : "1969-10-28T12:12:12.123456789Z" }' - - '{ "index" : { "_index" : "date_ns", "_id" : "date_ns_2" } }' - - '{"date" : "2263-10-29T12:12:12.123456789Z" }' - - - match: { errors: true } - - match: { items.0.index.status: 400 } - - match: { items.0.index.error.type: mapper_parsing_exception } - - match: { items.0.index.error.caused_by.reason: "date[1969-10-28T12:12:12.123456789Z] is before the epoch in 1970 and cannot be stored in nanosecond resolution" } - - match: { items.1.index.status: 400 } - - match: { items.1.index.error.type: mapper_parsing_exception } - - match: { items.1.index.error.caused_by.reason: "date[2263-10-29T12:12:12.123456789Z] is after 2262-04-11T23:47:16.854775807 and cannot be stored in nanosecond resolution" } - - --- "doc value fields are working as expected across date and date_nanos fields": diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java index 4cc978b166129..db129a9479934 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java @@ -56,6 +56,7 @@ import java.util.function.Function; import java.util.stream.Collectors; +import static java.util.Collections.emptyMap; import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.findConflictingV1Templates; import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.findConflictingV2Templates; import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.findV2Template; @@ -182,7 +183,7 @@ public static Template resolveTemplate(final String matchingTemplate, final Stri resolvedAliases, tempClusterState.metadata(), aliasValidator, xContentRegistry, // the context is only used for validation so it's fine to pass fake values for the // shard id and the current timestamp - tempIndexService.newQueryShardContext(0, null, () -> 0L, null))); + tempIndexService.newQueryShardContext(0, null, () -> 0L, null, emptyMap()))); Map aliasesByName = aliases.stream().collect( Collectors.toMap(AliasMetadata::getAlias, Function.identity())); diff --git a/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java b/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java index 57082ed450941..9ee9bf947a46d 100644 --- a/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java +++ b/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java @@ -89,7 +89,8 @@ public void run() { CollapseBuilder innerCollapseBuilder = innerHitBuilder.getInnerCollapseBuilder(); SearchSourceBuilder sourceBuilder = buildExpandSearchSourceBuilder(innerHitBuilder, innerCollapseBuilder) .query(groupQuery) - .postFilter(searchRequest.source().postFilter()); + .postFilter(searchRequest.source().postFilter()) + .runtimeMappings(searchRequest.source().runtimeMappings()); SearchRequest groupRequest = new SearchRequest(searchRequest); groupRequest.source(sourceBuilder); multiRequest.add(groupRequest); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java index 67fa07eb74a9e..4a75c47878d00 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java @@ -98,6 +98,7 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; +import static java.util.Collections.emptyMap; import static java.util.stream.Collectors.toList; import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING; import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING; @@ -486,7 +487,7 @@ private ClusterState applyCreateIndexRequestWithV1Templates(final ClusterState c MetadataIndexTemplateService.resolveAliases(templates), currentState.metadata(), aliasValidator, // the context is only used for validation so it's fine to pass fake values for the // shard id and the current timestamp - xContentRegistry, indexService.newQueryShardContext(0, null, () -> 0L, null)), + xContentRegistry, indexService.newQueryShardContext(0, null, () -> 0L, null, emptyMap())), templates.stream().map(IndexTemplateMetadata::getName).collect(toList()), metadataTransformer); } @@ -519,7 +520,7 @@ private ClusterState applyCreateIndexRequestWithV2Template(final ClusterState cu MetadataIndexTemplateService.resolveAliases(currentState.metadata(), templateName), currentState.metadata(), aliasValidator, // the context is only used for validation so it's fine to pass fake values for the // shard id and the current timestamp - xContentRegistry, indexService.newQueryShardContext(0, null, () -> 0L, null)), + xContentRegistry, indexService.newQueryShardContext(0, null, () -> 0L, null, emptyMap())), Collections.singletonList(templateName), metadataTransformer); } @@ -565,7 +566,7 @@ private ClusterState applyCreateIndexRequestWithExistingMetadata(final ClusterSt currentState.metadata(), aliasValidator, xContentRegistry, // the context is only used for validation so it's fine to pass fake values for the // shard id and the current timestamp - indexService.newQueryShardContext(0, null, () -> 0L, null)), + indexService.newQueryShardContext(0, null, () -> 0L, null, emptyMap())), List.of(), metadataTransformer); } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesService.java index 55c9635470a13..3ced1267655f1 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesService.java @@ -47,6 +47,7 @@ import java.util.function.Function; import static java.util.Collections.emptyList; +import static java.util.Collections.emptyMap; import static org.elasticsearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason.NO_LONGER_ASSIGNED; /** @@ -149,7 +150,7 @@ public ClusterState applyAliasActions(ClusterState currentState, Iterable System.currentTimeMillis(), null), xContentRegistry); + () -> System.currentTimeMillis(), null, emptyMap()), xContentRegistry); } }; if (action.apply(newAliasValidator, metadata, index)) { diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index 40392ba516e1c..ddab81145eff3 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -80,6 +80,7 @@ import java.util.function.Predicate; import java.util.stream.Collectors; +import static java.util.Collections.emptyMap; import static org.elasticsearch.cluster.metadata.MetadataCreateDataStreamService.validateTimestampFieldMapping; import static org.elasticsearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason.NO_LONGER_ASSIGNED; @@ -1117,7 +1118,7 @@ private static void validateCompositeTemplate(final ClusterState state, new AliasValidator(), // the context is only used for validation so it's fine to pass fake values for the // shard id and the current timestamp - xContentRegistry, tempIndexService.newQueryShardContext(0, null, () -> 0L, null)); + xContentRegistry, tempIndexService.newQueryShardContext(0, null, () -> 0L, null, emptyMap())); // triggers inclusion of _timestamp field and its validation: String indexName = DataStream.BACKING_INDEX_PREFIX + temporaryIndexName; diff --git a/server/src/main/java/org/elasticsearch/index/IndexService.java b/server/src/main/java/org/elasticsearch/index/IndexService.java index 88da415caabec..24f44c3336df7 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexService.java +++ b/server/src/main/java/org/elasticsearch/index/IndexService.java @@ -192,7 +192,7 @@ public IndexService( assert indexAnalyzers != null; this.mapperService = new MapperService(indexSettings, indexAnalyzers, xContentRegistry, similarityService, mapperRegistry, // we parse all percolator queries as they would be parsed on shard 0 - () -> newQueryShardContext(0, null, System::currentTimeMillis, null), idFieldDataEnabled, scriptService); + () -> newQueryShardContext(0, null, System::currentTimeMillis, null, emptyMap()), idFieldDataEnabled, scriptService); this.indexFieldData = new IndexFieldDataService(indexSettings, indicesFieldDataCache, circuitBreakerService, mapperService); if (indexSettings.getIndexSortConfig().hasIndexSort()) { // we delay the actual creation of the sort order for this index because the mapping has not been merged yet. @@ -586,13 +586,19 @@ public IndexSettings getIndexSettings() { * Passing a {@code null} {@link IndexSearcher} will return a valid context, however it won't be able to make * {@link IndexReader}-specific optimizations, such as rewriting containing range queries. */ - public QueryShardContext newQueryShardContext(int shardId, IndexSearcher searcher, LongSupplier nowInMillis, String clusterAlias) { + public QueryShardContext newQueryShardContext( + int shardId, + IndexSearcher searcher, + LongSupplier nowInMillis, + String clusterAlias, + Map runtimeMappings + ) { final SearchIndexNameMatcher indexNameMatcher = new SearchIndexNameMatcher(index().getName(), clusterAlias, clusterService, expressionResolver); return new QueryShardContext( shardId, indexSettings, bigArrays, indexCache.bitsetFilterCache(), indexFieldData::getForField, mapperService(), similarityService(), scriptService, xContentRegistry, namedWriteableRegistry, client, searcher, nowInMillis, clusterAlias, - indexNameMatcher, allowExpensiveQueries, valuesSourceRegistry); + indexNameMatcher, allowExpensiveQueries, valuesSourceRegistry, runtimeMappings); } /** diff --git a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java index f8131ecf19cf3..07447cf790e28 100644 --- a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java @@ -26,6 +26,7 @@ import org.apache.lucene.search.join.BitSetProducer; import org.apache.lucene.search.similarities.Similarity; import org.apache.lucene.util.SetOnce; +import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.client.Client; @@ -34,6 +35,7 @@ import org.elasticsearch.common.TriFunction; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.lucene.search.Queries; +import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentParser; @@ -68,6 +70,7 @@ import java.io.IOException; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -77,6 +80,8 @@ import java.util.function.Predicate; import java.util.function.Supplier; +import static java.util.Collections.emptyMap; + /** * Context object used to create lucene queries on the shard level. */ @@ -103,7 +108,11 @@ public class QueryShardContext extends QueryRewriteContext { private boolean mapUnmappedFieldAsString; private NestedScope nestedScope; private final ValuesSourceRegistry valuesSourceRegistry; + private final Map runtimeMappings; + /** + * Build a {@linkplain QueryShardContext} without any information from the search request. + */ public QueryShardContext(int shardId, IndexSettings indexSettings, BigArrays bigArrays, @@ -122,16 +131,63 @@ public QueryShardContext(int shardId, BooleanSupplier allowExpensiveQueries, ValuesSourceRegistry valuesSourceRegistry) { this(shardId, indexSettings, bigArrays, bitsetFilterCache, indexFieldDataLookup, mapperService, similarityService, - scriptService, xContentRegistry, namedWriteableRegistry, client, searcher, nowInMillis, indexNameMatcher, - new Index(RemoteClusterAware.buildRemoteIndexName(clusterAlias, indexSettings.getIndex().getName()), - indexSettings.getIndex().getUUID()), allowExpensiveQueries, valuesSourceRegistry); + scriptService, xContentRegistry, namedWriteableRegistry, client, searcher, nowInMillis, clusterAlias, + indexNameMatcher, allowExpensiveQueries, valuesSourceRegistry, emptyMap()); + } + + /** + * Build a {@linkplain QueryShardContext} with information from the search request. + */ + public QueryShardContext( + int shardId, + IndexSettings indexSettings, + BigArrays bigArrays, + BitsetFilterCache bitsetFilterCache, + TriFunction, IndexFieldData> indexFieldDataLookup, + MapperService mapperService, + SimilarityService similarityService, + ScriptService scriptService, + NamedXContentRegistry xContentRegistry, + NamedWriteableRegistry namedWriteableRegistry, + Client client, + IndexSearcher searcher, + LongSupplier nowInMillis, + String clusterAlias, + Predicate indexNameMatcher, + BooleanSupplier allowExpensiveQueries, + ValuesSourceRegistry valuesSourceRegistry, + Map runtimeMappings + ) { + this( + shardId, + indexSettings, + bigArrays, + bitsetFilterCache, + indexFieldDataLookup, + mapperService, + similarityService, + scriptService, + xContentRegistry, + namedWriteableRegistry, + client, + searcher, + nowInMillis, + indexNameMatcher, + new Index( + RemoteClusterAware.buildRemoteIndexName(clusterAlias, indexSettings.getIndex().getName()), + indexSettings.getIndex().getUUID() + ), + allowExpensiveQueries, + valuesSourceRegistry, + parseRuntimeMappings(runtimeMappings, mapperService, indexSettings) + ); } public QueryShardContext(QueryShardContext source) { this(source.shardId, source.indexSettings, source.bigArrays, source.bitsetFilterCache, source.indexFieldDataService, source.mapperService, source.similarityService, source.scriptService, source.getXContentRegistry(), source.getWriteableRegistry(), source.client, source.searcher, source.nowInMillis, source.indexNameMatcher, - source.fullyQualifiedIndex, source.allowExpensiveQueries, source.valuesSourceRegistry); + source.fullyQualifiedIndex, source.allowExpensiveQueries, source.valuesSourceRegistry, source.runtimeMappings); } private QueryShardContext(int shardId, @@ -150,7 +206,8 @@ private QueryShardContext(int shardId, Predicate indexNameMatcher, Index fullyQualifiedIndex, BooleanSupplier allowExpensiveQueries, - ValuesSourceRegistry valuesSourceRegistry) { + ValuesSourceRegistry valuesSourceRegistry, + Map runtimeMappings) { super(xContentRegistry, namedWriteableRegistry, client, nowInMillis); this.shardId = shardId; this.similarityService = similarityService; @@ -167,6 +224,7 @@ private QueryShardContext(int shardId, this.fullyQualifiedIndex = fullyQualifiedIndex; this.allowExpensiveQueries = allowExpensiveQueries; this.valuesSourceRegistry = valuesSourceRegistry; + this.runtimeMappings = runtimeMappings; } private void reset() { @@ -243,7 +301,20 @@ public boolean hasMappings() { * type then the fields will be returned with a type prefix. */ public Set simpleMatchToIndexNames(String pattern) { - return mapperService.simpleMatchToFullName(pattern); + if (runtimeMappings.isEmpty()) { + return mapperService.simpleMatchToFullName(pattern); + } + if (Regex.isSimpleMatchPattern(pattern) == false) { + // no wildcards + return Collections.singleton(pattern); + } + Set matches = new HashSet<>(mapperService.simpleMatchToFullName(pattern)); + for (String name : runtimeMappings.keySet()) { + if (Regex.simpleMatch(pattern, name)) { + matches.add(name); + } + } + return matches; } /** @@ -256,14 +327,19 @@ public Set simpleMatchToIndexNames(String pattern) { * @see QueryShardContext#setMapUnmappedFieldAsString(boolean) */ public MappedFieldType getFieldType(String name) { - return failIfFieldMappingNotFound(name, mapperService.fieldType(name)); + return failIfFieldMappingNotFound(name, fieldType(name)); } /** * Returns true if the field identified by the provided name is mapped, false otherwise */ public boolean isFieldMapped(String name) { - return mapperService.fieldType(name) != null; + return fieldType(name) != null; + } + + private MappedFieldType fieldType(String name) { + MappedFieldType fieldType = runtimeMappings.get(name); + return fieldType == null ? mapperService.fieldType(name) : fieldType; } public ObjectMapper getObjectMapper(String name) { @@ -287,13 +363,22 @@ public boolean isSourceEnabled() { * Generally used to handle unmapped fields in the context of sorting. */ public MappedFieldType buildAnonymousFieldType(String type) { - final Mapper.TypeParser.ParserContext parserContext = mapperService.parserContext(); + return buildFieldType(type, "__anonymous_" + type, Collections.emptyMap(), mapperService.parserContext(), indexSettings); + } + + private static MappedFieldType buildFieldType( + String type, + String field, + Map node, + Mapper.TypeParser.ParserContext parserContext, + IndexSettings indexSettings + ) { Mapper.TypeParser typeParser = parserContext.typeParser(type); if (typeParser == null) { throw new IllegalArgumentException("No mapper found for type [" + type + "]"); } - final Mapper.Builder builder = typeParser.parse("__anonymous_" + type, Collections.emptyMap(), parserContext); - Mapper mapper = builder.build(new ContentPath(1)); + Mapper.Builder builder = typeParser.parse(field, node, parserContext); + Mapper mapper = builder.build(new ContentPath(0)); if (mapper instanceof FieldMapper) { return ((FieldMapper)mapper).fieldType(); } @@ -534,4 +619,26 @@ public BigArrays bigArrays() { return bigArrays; } + private static Map parseRuntimeMappings( + Map mappings, + MapperService mapperService, + IndexSettings indexSettings + ) { + Map runtimeMappings = new HashMap<>(); + for (Map.Entry entry : mappings.entrySet()) { + String field = entry.getKey(); + if (entry.getValue() instanceof Map == false) { + throw new ElasticsearchParseException("runtime mappings must be a map type"); + } + @SuppressWarnings("unchecked") + Map node = new HashMap<>((Map) entry.getValue()); + // Replace the type until we have native support for the runtime section + Object oldRuntimeType = node.put("runtime_type", node.remove("type")); + if (oldRuntimeType != null) { + throw new ElasticsearchParseException("use [type] in [runtime_mappings] instead of [runtime_type]"); + } + runtimeMappings.put(field, buildFieldType("runtime", field, node, mapperService.parserContext(), indexSettings)); + } + return runtimeMappings; + } } diff --git a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java index 2964604b2408d..1882f98c8ab09 100644 --- a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java +++ b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java @@ -174,8 +174,13 @@ final class DefaultSearchContext extends SearchContext { this.relativeTimeSupplier = relativeTimeSupplier; this.timeout = timeout; - queryShardContext = indexService.newQueryShardContext(request.shardId().id(), this.searcher, - request::nowInMillis, shardTarget.getClusterAlias()); + queryShardContext = indexService.newQueryShardContext( + request.shardId().id(), + this.searcher, + request::nowInMillis, + shardTarget.getClusterAlias(), + request.getRuntimeMappings() + ); queryBoost = request.indexBoost(); this.lowLevelCancellation = lowLevelCancellation; } diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java index d44445ac811d5..f976a05ba510e 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchService.java +++ b/server/src/main/java/org/elasticsearch/search/SearchService.java @@ -1178,7 +1178,7 @@ private CanMatchResponse canMatch(ShardSearchRequest request, boolean checkRefre try (canMatchSearcher) { QueryShardContext context = indexService.newQueryShardContext(request.shardId().id(), canMatchSearcher, - request::nowInMillis, request.getClusterAlias()); + request::nowInMillis, request.getClusterAlias(), request.getRuntimeMappings()); Rewriteable.rewrite(request.getRewriteable(), context, false); final boolean aliasFilterCanMatch = request.getAliasFilter() .getQueryBuilder() instanceof MatchNoneQueryBuilder == false; diff --git a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java index 461cf5456c7e6..c4b0752447b49 100644 --- a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java @@ -63,8 +63,10 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Objects; +import static java.util.Collections.emptyMap; import static org.elasticsearch.index.query.AbstractQueryBuilder.parseInnerQueryBuilder; import static org.elasticsearch.search.internal.SearchContext.TRACK_TOTAL_HITS_ACCURATE; import static org.elasticsearch.search.internal.SearchContext.TRACK_TOTAL_HITS_DISABLED; @@ -110,6 +112,7 @@ public final class SearchSourceBuilder implements Writeable, ToXContentObject, R public static final ParseField COLLAPSE = new ParseField("collapse"); public static final ParseField SLICE = new ParseField("slice"); public static final ParseField POINT_IN_TIME = new ParseField("pit"); + public static final ParseField RUNTIME_MAPPINGS_FIELD = new ParseField("runtime_mappings"); public static SearchSourceBuilder fromXContent(XContentParser parser) throws IOException { return fromXContent(parser, true); @@ -191,6 +194,8 @@ public static HighlightBuilder highlight() { private PointInTimeBuilder pointInTimeBuilder = null; + private Map runtimeMappings = emptyMap(); + /** * Constructs a new search source builder. */ @@ -251,6 +256,9 @@ public SearchSourceBuilder(StreamInput in) throws IOException { } pointInTimeBuilder = in.readOptionalWriteable(PointInTimeBuilder::new); } + if (in.getVersion().onOrAfter(Version.V_8_0_0)) { + runtimeMappings = in.readMap(); + } } @Override @@ -312,6 +320,15 @@ public void writeTo(StreamOutput out) throws IOException { } out.writeOptionalWriteable(pointInTimeBuilder); } + if (out.getVersion().onOrAfter(Version.V_8_0_0)) { + out.writeMap(runtimeMappings); + } else { + if (false == runtimeMappings.isEmpty()) { + throw new IllegalArgumentException( + "Versions before 8.0.0 don't support [runtime_mappings] and search was sent to [" + out.getVersion() + "]" + ); + } + } } /** @@ -973,6 +990,21 @@ public SearchSourceBuilder pointInTimeBuilder(PointInTimeBuilder builder) { return this; } + /** + * Mappings specified on this search request that override built in mappings. + */ + public Map runtimeMappings() { + return runtimeMappings; + } + + /** + * Specify the mappings specified on this search request that override built in mappings. + */ + public SearchSourceBuilder runtimeMappings(Map runtimeMappings) { + this.runtimeMappings = runtimeMappings == null ? emptyMap() : runtimeMappings; + return this; + } + /** * Rewrites this search source builder into its primitive form. e.g. by * rewriting the QueryBuilder. If the builder did not change the identity @@ -1059,6 +1091,7 @@ private SearchSourceBuilder shallowCopy(QueryBuilder queryBuilder, QueryBuilder rewrittenBuilder.seqNoAndPrimaryTerm = seqNoAndPrimaryTerm; rewrittenBuilder.collapse = collapse; rewrittenBuilder.pointInTimeBuilder = pointInTimeBuilder; + rewrittenBuilder.runtimeMappings = runtimeMappings; return rewrittenBuilder; } @@ -1169,6 +1202,8 @@ public void parseXContent(XContentParser parser, boolean checkTrailingTokens) th collapse = CollapseBuilder.fromXContent(parser); } else if (POINT_IN_TIME.match(currentFieldName, parser.getDeprecationHandler())) { pointInTimeBuilder = PointInTimeBuilder.fromXContent(parser); + } else if (RUNTIME_MAPPINGS_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { + runtimeMappings = parser.map(); } else { throw new ParsingException(parser.getTokenLocation(), "Unknown key for a " + token + " in [" + currentFieldName + "].", parser.getTokenLocation()); @@ -1376,6 +1411,10 @@ public XContentBuilder innerToXContent(XContentBuilder builder, Params params) t if (pointInTimeBuilder != null) { pointInTimeBuilder.toXContent(builder, params); } + if (false == runtimeMappings.isEmpty()) { + builder.field(RUNTIME_MAPPINGS_FIELD.getPreferredName(), runtimeMappings); + } + return builder; } @@ -1588,7 +1627,7 @@ public int hashCode() { return Objects.hash(aggregations, explain, fetchSourceContext, fetchFields, docValueFields, storedFieldsContext, from, highlightBuilder, indexBoosts, minScore, postQueryBuilder, queryBuilder, rescoreBuilders, scriptFields, size, sorts, searchAfterBuilder, sliceBuilder, stats, suggestBuilder, terminateAfter, timeout, trackScores, version, - seqNoAndPrimaryTerm, profile, extBuilders, collapse, trackTotalHitsUpTo, pointInTimeBuilder); + seqNoAndPrimaryTerm, profile, extBuilders, collapse, trackTotalHitsUpTo, pointInTimeBuilder, runtimeMappings); } @Override @@ -1629,7 +1668,8 @@ public boolean equals(Object obj) { && Objects.equals(extBuilders, other.extBuilders) && Objects.equals(collapse, other.collapse) && Objects.equals(trackTotalHitsUpTo, other.trackTotalHitsUpTo) - && Objects.equals(pointInTimeBuilder, other.pointInTimeBuilder); + && Objects.equals(pointInTimeBuilder, other.pointInTimeBuilder) + && Objects.equals(runtimeMappings, other.runtimeMappings); } @Override diff --git a/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java b/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java index d496a10e696c2..af2e46a65dffb 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java +++ b/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java @@ -47,8 +47,8 @@ import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.AliasFilterParsingException; import org.elasticsearch.indices.InvalidAliasNameException; -import org.elasticsearch.search.SearchSortValuesAndFormats; import org.elasticsearch.search.Scroll; +import org.elasticsearch.search.SearchSortValuesAndFormats; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.query.QuerySearchResult; import org.elasticsearch.search.sort.FieldSortBuilder; @@ -61,6 +61,7 @@ import java.util.Map; import java.util.function.Function; +import static java.util.Collections.emptyMap; import static org.elasticsearch.search.internal.SearchContext.TRACK_TOTAL_HITS_DISABLED; /** @@ -521,4 +522,8 @@ public static QueryBuilder parseAliasFilter(CheckedFunction getRuntimeMappings() { + return source == null ? emptyMap() : source.runtimeMappings(); + } } diff --git a/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java b/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java index beb4ef6d66e6f..76e77ce2c4671 100644 --- a/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java @@ -26,6 +26,7 @@ import org.elasticsearch.index.query.InnerHitBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.AbstractSearchTestCase; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.builder.SearchSourceBuilder; @@ -38,10 +39,14 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; import java.util.stream.IntStream; +import static java.util.Collections.emptyMap; +import static org.hamcrest.Matchers.equalTo; + public class ExpandSearchPhaseTests extends ESTestCase { public void testCollapseSingleHit() throws IOException { @@ -58,6 +63,7 @@ public void testCollapseSingleHit() throws IOException { AtomicBoolean executedMultiSearch = new AtomicBoolean(false); QueryBuilder originalQuery = randomBoolean() ? null : QueryBuilders.termQuery("foo", "bar"); + Map runtimeMappings = randomBoolean() ? emptyMap() : AbstractSearchTestCase.randomRuntimeMappings(); final MockSearchPhaseContext mockSearchPhaseContext = new MockSearchPhaseContext(1); String collapseValue = randomBoolean() ? null : "boom"; @@ -66,7 +72,7 @@ public void testCollapseSingleHit() throws IOException { .collapse(new CollapseBuilder("someField") .setInnerHits(IntStream.range(0, numInnerHits).mapToObj(hitNum -> new InnerHitBuilder().setName("innerHit" + hitNum)) .collect(Collectors.toList())))); - mockSearchPhaseContext.getRequest().source().query(originalQuery); + mockSearchPhaseContext.getRequest().source().query(originalQuery).runtimeMappings(runtimeMappings); mockSearchPhaseContext.searchTransport = new SearchTransportService(null, null, null) { @Override void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionListener listener) { @@ -85,7 +91,7 @@ void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionL assertThat(groupBuilder.must(), Matchers.contains(QueryBuilders.termQuery("foo", "bar"))); } assertArrayEquals(mockSearchPhaseContext.getRequest().indices(), searchRequest.indices()); - + assertThat(searchRequest.source().runtimeMappings(), equalTo(runtimeMappings)); List mSearchResponses = new ArrayList<>(numInnerHits); for (int innerHitNum = 0; innerHitNum < numInnerHits; innerHitNum++) { diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java b/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java index 3725bb7b87a77..b6ac781a015d7 100644 --- a/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java +++ b/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java @@ -59,6 +59,7 @@ import java.util.Collection; import java.util.List; +import static java.util.Collections.emptyMap; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.sameInstance; @@ -139,7 +140,7 @@ public void setup() throws Exception { writer = new IndexWriter( new ByteBuffersDirectory(), new IndexWriterConfig(new StandardAnalyzer()).setMergePolicy(new LogByteSizeMergePolicy()) ); - shardContext = indexService.newQueryShardContext(0, null, () -> 0, null); + shardContext = indexService.newQueryShardContext(0, null, () -> 0, null, emptyMap()); } protected final List refreshReader() throws Exception { diff --git a/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java b/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java index c4d63e051230d..bb21086561a9a 100644 --- a/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java @@ -50,14 +50,21 @@ import org.elasticsearch.index.fielddata.LeafFieldData; import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.fielddata.plain.AbstractLeafOrdinalsFieldData; +import org.elasticsearch.index.mapper.ContentPath; +import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.IndexFieldMapper; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.index.mapper.Mapper.TypeParser; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.NumberFieldMapper; +import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.mapper.TextFieldMapper; +import org.elasticsearch.index.mapper.TextSearchInfo; +import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.indices.IndicesModule; +import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.search.lookup.LeafDocLookup; import org.elasticsearch.search.lookup.LeafSearchLookup; import org.elasticsearch.search.lookup.SearchLookup; @@ -68,6 +75,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.function.BiFunction; import java.util.function.Supplier; @@ -183,9 +191,24 @@ public void testIndexSortedOnField() { IndexSettings indexSettings = new IndexSettings(indexMetadata, settings); QueryShardContext context = new QueryShardContext( - 0, indexSettings, BigArrays.NON_RECYCLING_INSTANCE, null, null, - null, null, null, NamedXContentRegistry.EMPTY, new NamedWriteableRegistry(Collections.emptyList()), - null, null, () -> 0L, null, null, () -> true, null); + 0, + indexSettings, + BigArrays.NON_RECYCLING_INSTANCE, + null, + null, + null, + null, + null, + NamedXContentRegistry.EMPTY, + new NamedWriteableRegistry(Collections.emptyList()), + null, + null, + () -> 0L, + null, + null, + () -> true, + null + ); assertTrue(context.indexSortedOnField("sort_field")); assertFalse(context.indexSortedOnField("second_sort_field")); @@ -289,12 +312,86 @@ public void testFielddataLookupOneFieldManyReferences() throws IOException { assertEquals(List.of(expectedFirstDoc.toString(), expectedSecondDoc.toString()), collect("field", queryShardContext)); } + public void testRuntimeFields() throws IOException { + MapperService mapperService = mockMapperService("test", List.of(new MapperPlugin() { + @Override + public Map getMappers() { + return Map.of("runtime", (name, node, parserContext) -> new Mapper.Builder(name) { + @Override + public Mapper build(ContentPath path) { + return new DummyMapper(name, new DummyMappedFieldType(name)); + } + }); + } + })); + when(mapperService.fieldType("pig")).thenReturn(new DummyMappedFieldType("pig")); + when(mapperService.simpleMatchToFullName("*")).thenReturn(Set.of("pig")); + /* + * Making these immutable here test that we don't modify them. + * Modifying them would cause all kinds of problems if two + * shards are parsed on the same node. + */ + Map runtimeMappings = Map.ofEntries( + Map.entry("cat", Map.of("type", "keyword")), + Map.entry("dog", Map.of("type", "keyword")) + ); + QueryShardContext qsc = new QueryShardContext( + 0, + mapperService.getIndexSettings(), + BigArrays.NON_RECYCLING_INSTANCE, + null, + (mappedFieldType, idxName, searchLookup) -> mappedFieldType.fielddataBuilder(idxName, searchLookup).build(null, null), + mapperService, + null, + null, + NamedXContentRegistry.EMPTY, + new NamedWriteableRegistry(List.of()), + null, + null, + () -> 0, + "test", + null, + () -> true, + null, + runtimeMappings + ); + assertTrue(qsc.isFieldMapped("cat")); + assertThat(qsc.getFieldType("cat"), instanceOf(DummyMappedFieldType.class)); + assertThat(qsc.simpleMatchToIndexNames("cat"), equalTo(Set.of("cat"))); + assertTrue(qsc.isFieldMapped("dog")); + assertThat(qsc.getFieldType("dog"), instanceOf(DummyMappedFieldType.class)); + assertThat(qsc.simpleMatchToIndexNames("dog"), equalTo(Set.of("dog"))); + assertTrue(qsc.isFieldMapped("pig")); + assertThat(qsc.getFieldType("pig"), instanceOf(DummyMappedFieldType.class)); + assertThat(qsc.simpleMatchToIndexNames("pig"), equalTo(Set.of("pig"))); + assertThat(qsc.simpleMatchToIndexNames("*"), equalTo(Set.of("cat", "dog", "pig"))); + } + public static QueryShardContext createQueryShardContext(String indexUuid, String clusterAlias) { return createQueryShardContext(indexUuid, clusterAlias, null); } - private static QueryShardContext createQueryShardContext(String indexUuid, String clusterAlias, - TriFunction runtimeDocValues) { + private static QueryShardContext createQueryShardContext( + String indexUuid, + String clusterAlias, + TriFunction runtimeDocValues + ) { + MapperService mapperService = mockMapperService(indexUuid, List.of()); + if (runtimeDocValues != null) { + when(mapperService.fieldType(any())).thenAnswer(fieldTypeInv -> { + String fieldName = (String)fieldTypeInv.getArguments()[0]; + return mockFieldType(fieldName, (leafSearchLookup, docId) -> runtimeDocValues.apply(fieldName, leafSearchLookup, docId)); + }); + } + final long nowInMillis = randomNonNegativeLong(); + return new QueryShardContext( + 0, mapperService.getIndexSettings(), BigArrays.NON_RECYCLING_INSTANCE, null, + (mappedFieldType, idxName, searchLookup) -> mappedFieldType.fielddataBuilder(idxName, searchLookup).build(null, null), + mapperService, null, null, NamedXContentRegistry.EMPTY, new NamedWriteableRegistry(Collections.emptyList()), + null, null, () -> nowInMillis, clusterAlias, null, () -> true, null); + } + + private static MapperService mockMapperService(String indexUuid, List mapperPlugins) { IndexMetadata.Builder indexMetadataBuilder = new IndexMetadata.Builder("index"); indexMetadataBuilder.settings(Settings.builder().put("index.version.created", Version.CURRENT) .put("index.number_of_shards", 1) @@ -302,34 +399,24 @@ private static QueryShardContext createQueryShardContext(String indexUuid, Strin .put(IndexMetadata.SETTING_INDEX_UUID, indexUuid) ); IndexMetadata indexMetadata = indexMetadataBuilder.build(); - IndexSettings indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY); IndexAnalyzers indexAnalyzers = new IndexAnalyzers( Collections.singletonMap("default", new NamedAnalyzer("default", AnalyzerScope.INDEX, null)), Collections.emptyMap(), Collections.emptyMap() ); + IndexSettings indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY); + MapperService mapperService = mock(MapperService.class); when(mapperService.getIndexSettings()).thenReturn(indexSettings); when(mapperService.index()).thenReturn(indexMetadata.getIndex()); when(mapperService.getIndexAnalyzers()).thenReturn(indexAnalyzers); - Map typeParserMap = IndicesModule.getMappers(Collections.emptyList()); + Map typeParserMap = IndicesModule.getMappers(mapperPlugins); Mapper.TypeParser.ParserContext parserContext = new Mapper.TypeParser.ParserContext(name -> null, typeParserMap::get, Version.CURRENT, () -> null, null, null, mapperService.getIndexAnalyzers(), mapperService.getIndexSettings(), () -> { throw new UnsupportedOperationException(); }); when(mapperService.parserContext()).thenReturn(parserContext); - if (runtimeDocValues != null) { - when(mapperService.fieldType(any())).thenAnswer(fieldTypeInv -> { - String fieldName = (String)fieldTypeInv.getArguments()[0]; - return mockFieldType(fieldName, (leafSearchLookup, docId) -> runtimeDocValues.apply(fieldName, leafSearchLookup, docId)); - }); - } - final long nowInMillis = randomNonNegativeLong(); - return new QueryShardContext( - 0, indexSettings, BigArrays.NON_RECYCLING_INSTANCE, null, - (mappedFieldType, idxName, searchLookup) -> mappedFieldType.fielddataBuilder(idxName, searchLookup).build(null, null), - mapperService, null, null, NamedXContentRegistry.EMPTY, new NamedWriteableRegistry(Collections.emptyList()), - null, null, () -> nowInMillis, clusterAlias, null, () -> true, null); + return mapperService; } private static MappedFieldType mockFieldType(String fieldName, BiFunction runtimeDocValues) { @@ -427,4 +514,45 @@ public void collect(int doc) throws IOException { } } + private static class DummyMapper extends FieldMapper { + protected DummyMapper(String simpleName, MappedFieldType mappedFieldType) { + super(simpleName, mappedFieldType, Map.of(), MultiFields.empty(), CopyTo.empty()); + } + + @Override + protected void parseCreateField(ParseContext context) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public Builder getMergeBuilder() { + throw new UnsupportedOperationException(); + } + + @Override + protected String contentType() { + throw new UnsupportedOperationException(); + } + } + + private static class DummyMappedFieldType extends MappedFieldType { + DummyMappedFieldType(String name) { + super(name, true, false, true, TextSearchInfo.SIMPLE_MATCH_ONLY, null); + } + + @Override + public ValueFetcher valueFetcher(QueryShardContext context, SearchLookup searchLookup, String format) { + throw new UnsupportedOperationException(); + } + + @Override + public String typeName() { + return "runtime"; + } + + @Override + public Query termQuery(Object value, QueryShardContext context) { + throw new UnsupportedOperationException(); + } + } } diff --git a/server/src/test/java/org/elasticsearch/index/search/MultiMatchQueryTests.java b/server/src/test/java/org/elasticsearch/index/search/MultiMatchQueryTests.java index 16bb399bde087..d633b7248e2cb 100644 --- a/server/src/test/java/org/elasticsearch/index/search/MultiMatchQueryTests.java +++ b/server/src/test/java/org/elasticsearch/index/search/MultiMatchQueryTests.java @@ -58,6 +58,7 @@ import java.util.List; import java.util.Map; +import static java.util.Collections.emptyMap; import static org.elasticsearch.index.query.QueryBuilders.multiMatchQuery; import static org.hamcrest.Matchers.equalTo; @@ -99,7 +100,7 @@ public void setup() throws IOException { public void testCrossFieldMultiMatchQuery() throws IOException { QueryShardContext queryShardContext = indexService.newQueryShardContext( - randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null); + randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null, emptyMap()); queryShardContext.setAllowUnmappedFields(true); for (float tieBreaker : new float[] {0.0f, 0.5f}) { Query parsedQuery = multiMatchQuery("banon") @@ -126,8 +127,12 @@ public void testBlendTerms() { float[] boosts = new float[] {2, 3}; Query expected = BlendedTermQuery.dismaxBlendedQuery(terms, boosts, 1.0f); Query actual = MultiMatchQuery.blendTerm( - indexService.newQueryShardContext(randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null), - new BytesRef("baz"), 1f, false, Arrays.asList(new FieldAndBoost(ft1, 2), new FieldAndBoost(ft2, 3))); + indexService.newQueryShardContext(randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null, emptyMap()), + new BytesRef("baz"), + 1f, + false, + Arrays.asList(new FieldAndBoost(ft1, 2), new FieldAndBoost(ft2, 3)) + ); assertEquals(expected, actual); } @@ -146,8 +151,12 @@ public Query termQuery(Object value, QueryShardContext context) { BlendedTermQuery.dismaxBlendedQuery(terms, boosts, 1.0f) ), 1f); Query actual = MultiMatchQuery.blendTerm( - indexService.newQueryShardContext(randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null), - new BytesRef("baz"), 1f, true, Arrays.asList(new FieldAndBoost(ft1, 2), new FieldAndBoost(ft2, 3))); + indexService.newQueryShardContext(randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null, emptyMap()), + new BytesRef("baz"), + 1f, + true, + Arrays.asList(new FieldAndBoost(ft1, 2), new FieldAndBoost(ft2, 3)) + ); assertEquals(expected, actual); } @@ -159,7 +168,7 @@ public Query termQuery(Object value, QueryShardContext context) { } }; expectThrows(IllegalArgumentException.class, () -> MultiMatchQuery.blendTerm( - indexService.newQueryShardContext(randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null), + indexService.newQueryShardContext(randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null, emptyMap()), new BytesRef("baz"), 1f, false, Arrays.asList(new FieldAndBoost(ft, 1)))); } @@ -181,14 +190,18 @@ public Query termQuery(Object value, QueryShardContext context) { expectedDisjunct1 ), 1.0f); Query actual = MultiMatchQuery.blendTerm( - indexService.newQueryShardContext(randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null), - new BytesRef("baz"), 1f, false, Arrays.asList(new FieldAndBoost(ft1, 2), new FieldAndBoost(ft2, 3))); + indexService.newQueryShardContext(randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null, emptyMap()), + new BytesRef("baz"), + 1f, + false, + Arrays.asList(new FieldAndBoost(ft1, 2), new FieldAndBoost(ft2, 3)) + ); assertEquals(expected, actual); } public void testMultiMatchCrossFieldsWithSynonyms() throws IOException { QueryShardContext queryShardContext = indexService.newQueryShardContext( - randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null); + randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null, emptyMap()); MultiMatchQuery parser = new MultiMatchQuery(queryShardContext); parser.setAnalyzer(new MockSynonymAnalyzer()); @@ -220,7 +233,7 @@ public void testMultiMatchCrossFieldsWithSynonyms() throws IOException { public void testMultiMatchCrossFieldsWithSynonymsPhrase() throws IOException { QueryShardContext queryShardContext = indexService.newQueryShardContext( - randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null); + randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null, emptyMap()); MultiMatchQuery parser = new MultiMatchQuery(queryShardContext); parser.setAnalyzer(new MockSynonymAnalyzer()); Map fieldNames = new HashMap<>(); @@ -289,7 +302,7 @@ public void testKeywordSplitQueriesOnWhitespace() throws IOException { QueryShardContext queryShardContext = indexService.newQueryShardContext( randomInt(20), null, () -> { throw new UnsupportedOperationException(); - }, null); + }, null, emptyMap()); MultiMatchQuery parser = new MultiMatchQuery(queryShardContext); Map fieldNames = new HashMap<>(); fieldNames.put("field", 1.0f); diff --git a/server/src/test/java/org/elasticsearch/index/search/NestedHelperTests.java b/server/src/test/java/org/elasticsearch/index/search/NestedHelperTests.java index f33d813ebb048..d8f7405ef3387 100644 --- a/server/src/test/java/org/elasticsearch/index/search/NestedHelperTests.java +++ b/server/src/test/java/org/elasticsearch/index/search/NestedHelperTests.java @@ -43,6 +43,8 @@ import java.io.IOException; import java.util.Collections; +import static java.util.Collections.emptyMap; + public class NestedHelperTests extends ESSingleNodeTestCase { IndexService indexService; @@ -333,7 +335,7 @@ public void testConjunction() { } public void testNested() throws IOException { - QueryShardContext context = indexService.newQueryShardContext(0, new IndexSearcher(new MultiReader()), () -> 0, null); + QueryShardContext context = indexService.newQueryShardContext(0, new IndexSearcher(new MultiReader()), () -> 0, null, emptyMap()); NestedQueryBuilder queryBuilder = new NestedQueryBuilder("nested1", new MatchAllQueryBuilder(), ScoreMode.Avg); ESToParentBlockJoinQuery query = (ESToParentBlockJoinQuery) queryBuilder.toQuery(context); diff --git a/server/src/test/java/org/elasticsearch/index/search/nested/NestedSortingTests.java b/server/src/test/java/org/elasticsearch/index/search/nested/NestedSortingTests.java index d2b8c6bab9dd8..e4b0573b06cca 100644 --- a/server/src/test/java/org/elasticsearch/index/search/nested/NestedSortingTests.java +++ b/server/src/test/java/org/elasticsearch/index/search/nested/NestedSortingTests.java @@ -73,6 +73,7 @@ import java.util.Collections; import java.util.List; +import static java.util.Collections.emptyMap; import static org.elasticsearch.index.mapper.SeqNoFieldMapper.PRIMARY_TERM_NAME; import static org.hamcrest.Matchers.equalTo; @@ -607,7 +608,7 @@ public void testMultiLevelNestedSorting() throws IOException { DirectoryReader reader = DirectoryReader.open(writer); reader = ElasticsearchDirectoryReader.wrap(reader, new ShardId(indexService.index(), 0)); IndexSearcher searcher = new IndexSearcher(reader); - QueryShardContext queryShardContext = indexService.newQueryShardContext(0, searcher, () -> 0L, null); + QueryShardContext queryShardContext = indexService.newQueryShardContext(0, searcher, () -> 0L, null, emptyMap()); FieldSortBuilder sortBuilder = new FieldSortBuilder("chapters.paragraphs.word_count"); sortBuilder.setNestedSort(new NestedSortBuilder("chapters").setNestedSort(new NestedSortBuilder("chapters.paragraphs"))); diff --git a/server/src/test/java/org/elasticsearch/search/AbstractSearchTestCase.java b/server/src/test/java/org/elasticsearch/search/AbstractSearchTestCase.java index e8488d99d8a80..8b82205c5efd2 100644 --- a/server/src/test/java/org/elasticsearch/search/AbstractSearchTestCase.java +++ b/server/src/test/java/org/elasticsearch/search/AbstractSearchTestCase.java @@ -91,7 +91,22 @@ protected SearchSourceBuilder createSearchSourceBuilder() { SuggestBuilderTests::randomSuggestBuilder, QueryRescorerBuilderTests::randomRescoreBuilder, randomExtBuilders, - CollapseBuilderTests::randomCollapseBuilder); + CollapseBuilderTests::randomCollapseBuilder, + AbstractSearchTestCase::randomRuntimeMappings); + } + + public static Map randomRuntimeMappings() { + int count = between(1, 100); + Map runtimeFields = new HashMap<>(count); + while (runtimeFields.size() < count) { + int size = between(1, 10); + Map config = new HashMap<>(size); + while (config.size() < size) { + config.put(randomAlphaOfLength(5), randomAlphaOfLength(5)); + } + runtimeFields.put(randomAlphaOfLength(5), config); + } + return runtimeFields; } protected SearchRequest createSearchRequest() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java b/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java index d71958df9b1ea..01ab3c591b41c 100644 --- a/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java +++ b/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java @@ -108,7 +108,9 @@ public void testPreProcess() throws Exception { when(indexCache.query()).thenReturn(queryCache); when(indexService.cache()).thenReturn(indexCache); QueryShardContext queryShardContext = mock(QueryShardContext.class); - when(indexService.newQueryShardContext(eq(shardId.id()), anyObject(), anyObject(), anyString())).thenReturn(queryShardContext); + when(indexService.newQueryShardContext(eq(shardId.id()), anyObject(), anyObject(), anyString(), anyObject())).thenReturn( + queryShardContext + ); MapperService mapperService = mock(MapperService.class); when(mapperService.hasNested()).thenReturn(randomBoolean()); when(indexService.mapperService()).thenReturn(mapperService); diff --git a/server/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java b/server/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java index 8ce36d4295cc2..c68f8243640e9 100644 --- a/server/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java @@ -20,11 +20,10 @@ package org.elasticsearch.search.builder; import com.fasterxml.jackson.core.JsonParseException; + +import org.elasticsearch.Version; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.io.stream.BytesStreamOutput; -import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput; -import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -93,16 +92,18 @@ private static void assertParseSearchSource(SearchSourceBuilder testBuilder, XCo } public void testSerialization() throws IOException { - SearchSourceBuilder testBuilder = createSearchSourceBuilder(); - try (BytesStreamOutput output = new BytesStreamOutput()) { - testBuilder.writeTo(output); - try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) { - SearchSourceBuilder deserializedBuilder = new SearchSourceBuilder(in); - assertEquals(deserializedBuilder, testBuilder); - assertEquals(deserializedBuilder.hashCode(), testBuilder.hashCode()); - assertNotSame(deserializedBuilder, testBuilder); - } - } + SearchSourceBuilder original = createSearchSourceBuilder(); + SearchSourceBuilder copy = copyBuilder(original); + assertEquals(copy, original); + assertEquals(copy.hashCode(), original.hashCode()); + assertNotSame(copy, original); + } + + public void testSerializingWithRuntimeFieldsBeforeSupportedThrows() { + SearchSourceBuilder original = new SearchSourceBuilder().runtimeMappings(randomRuntimeMappings()); + Version v = Version.V_8_0_0.minimumCompatibilityVersion(); + Exception e = expectThrows(IllegalArgumentException.class, () -> copyBuilder(original, v)); + assertThat(e.getMessage(), equalTo("Versions before 8.0.0 don't support [runtime_mappings] and search was sent to [" + v + "]")); } public void testShallowCopy() { @@ -118,9 +119,12 @@ public void testEqualsAndHashcode() throws IOException { EqualsHashCodeTestUtils.checkEqualsAndHashCode(createSearchSourceBuilder(), this::copyBuilder); } - //we use the streaming infra to create a copy of the builder provided as argument private SearchSourceBuilder copyBuilder(SearchSourceBuilder original) throws IOException { - return ESTestCase.copyWriteable(original, namedWriteableRegistry, SearchSourceBuilder::new); + return copyBuilder(original, Version.CURRENT); + } + + private SearchSourceBuilder copyBuilder(SearchSourceBuilder original, Version version) throws IOException { + return ESTestCase.copyWriteable(original, namedWriteableRegistry, SearchSourceBuilder::new, version); } public void testParseIncludeExclude() throws IOException { diff --git a/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java b/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java index ed95561aa9100..67aaedf7d818f 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java @@ -53,6 +53,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.function.Supplier; import static java.util.Collections.emptyMap; @@ -82,7 +83,7 @@ private RandomSearchRequestGenerator() {} * Build a random search request. * * @param randomSearchSourceBuilder builds a random {@link SearchSourceBuilder}. You can use - * {@link #randomSearchSourceBuilder(Supplier, Supplier, Supplier, Supplier, Supplier)}. + * {@link #randomSearchSourceBuilder}. */ public static SearchRequest randomSearchRequest(Supplier randomSearchSourceBuilder) { SearchRequest searchRequest = new SearchRequest(); @@ -122,7 +123,8 @@ public static SearchSourceBuilder randomSearchSourceBuilder( Supplier randomSuggestBuilder, Supplier> randomRescoreBuilder, Supplier> randomExtBuilders, - Supplier randomCollapseBuilder) { + Supplier randomCollapseBuilder, + Supplier> randomRuntimeMappings) { SearchSourceBuilder builder = new SearchSourceBuilder(); if (randomBoolean()) { builder.from(randomIntBetween(0, 10000)); @@ -378,6 +380,9 @@ public static SearchSourceBuilder randomSearchSourceBuilder( } builder.pointInTimeBuilder(pit); } + if (randomBoolean()) { + builder.runtimeMappings(randomRuntimeMappings.get()); + } return builder; } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java b/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java index 9193e90bdbaf4..a99a4f0b6b077 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java +++ b/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java @@ -63,6 +63,8 @@ import java.util.List; import java.util.Map; +import static java.util.Collections.emptyMap; + public class TestSearchContext extends SearchContext { public static final SearchShardTarget SHARD_TARGET = new SearchShardTarget("test", new ShardId("test", "test", 0), null, OriginalIndices.NONE); @@ -97,7 +99,7 @@ public TestSearchContext(BigArrays bigArrays, IndexService indexService) { this.indexService = indexService; this.fixedBitSetFilterCache = indexService.cache().bitsetFilterCache(); this.indexShard = indexService.getShardOrNull(0); - queryShardContext = indexService.newQueryShardContext(0, null, () -> 0L, null); + queryShardContext = indexService.newQueryShardContext(0, null, () -> 0L, null, emptyMap()); } public TestSearchContext(QueryShardContext queryShardContext) { diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchAction.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchAction.java index dac79f04cba67..7cd1e85366989 100644 --- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchAction.java +++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchAction.java @@ -70,6 +70,8 @@ import java.util.Map; import java.util.Set; +import static java.util.Collections.emptyMap; + /** * This is an internal action, that executes msearch requests for enrich indices in a more efficient manner. * Currently each search request inside a msearch request is executed as a separate search. If many search requests @@ -229,11 +231,20 @@ protected MultiSearchResponse shardOperation(Request request, ShardId shardId) t final IndexShard indexShard = indicesService.getShardOrNull(shardId); try (Engine.Searcher searcher = indexShard.acquireSearcher("enrich_msearch")) { final FieldsVisitor visitor = new FieldsVisitor(true); + /* + * Enrich doesn't support defining runtime fields in its + * configuration. We could add support for that if we'd + * like it but, for now at least, you can't configure any + * runtime fields so it is safe to build the context without + * any. + */ + Map runtimeFields = emptyMap(); final QueryShardContext context = indexService.newQueryShardContext( shardId.id(), searcher, () -> { throw new UnsupportedOperationException(); }, - null + null, + runtimeFields ); final MultiSearchResponse.Item[] items = new MultiSearchResponse.Item[request.multiSearchRequest.requests().size()]; for (int i = 0; i < request.multiSearchRequest.requests().size(); i++) { diff --git a/x-pack/plugin/runtime-fields/qa/build.gradle b/x-pack/plugin/runtime-fields/qa/build.gradle index 7dc01b73ed9ec..61e9797913849 100644 --- a/x-pack/plugin/runtime-fields/qa/build.gradle +++ b/x-pack/plugin/runtime-fields/qa/build.gradle @@ -1 +1,74 @@ -// Empty project so we can pick up its subproject +// Shared infratructure + +apply plugin: 'elasticsearch.build' + +dependencies { + api project(":test:framework") +} + +test.enabled = false // We don't currently have any tests for this because they are test utilities. + +subprojects { + if (project.name.startsWith('core-with-')) { + apply plugin: 'elasticsearch.yaml-rest-test' + + dependencies { + yamlRestTestImplementation xpackProject("plugin:runtime-fields:qa") + } + + restResources { + restApi { + includeXpack 'async_search', 'graph', '*_point_in_time' + } + restTests { + includeCore '*' + includeXpack 'async_search', 'graph' + } + } + + testClusters.yamlRestTest { + testDistribution = 'DEFAULT' + setting 'xpack.license.self_generated.type', 'trial' + } + + yamlRestTest { + def suites = [ + 'async_search', + 'search', + 'search.aggregation', + 'search.highlight', + 'search.inner_hits', + 'search_shards', + 'suggest', + ] + if (project.name.equals('core-with-mapped')) { + suites += [ + // These two don't support runtime fields on the request. Should they? + 'field_caps', + 'graph', + // The search request tests don't know how to support msearch for now + 'msearch', + ] + } + systemProperty 'tests.rest.suite', suites.join(',') + systemProperty 'tests.rest.blacklist', + [ + /////// TO FIX /////// + 'search.highlight/40_keyword_ignore/Plain Highligher should skip highlighting ignored keyword values', // The plain highlighter is incompatible with runtime fields. Worth fixing? + 'search/115_multiple_field_collapsing/two levels fields collapsing', // Broken. Gotta fix. + 'field_caps/30_filter/Field caps with index filter', // We don't support filtering field caps on runtime fields. What should we do? + 'search.aggregation/10_histogram/*', // runtime doesn't support sub-fields. Maybe it should? + 'search/140_pre_filter_search_shards/pre_filter_shard_size with shards that have no hit', + /////// TO FIX /////// + + /////// NOT SUPPORTED /////// + 'search.aggregation/280_rare_terms/*', // Requires an index and we won't have it + // Runtime fields don't have global ords + 'search.aggregation/20_terms/string profiler via global ordinals', + 'search.aggregation/20_terms/Global ordinals are loaded with the global_ordinals execution hint', + 'search.aggregation/170_cardinality_metric/profiler string', + /////// NOT SUPPORTED /////// + ].join(',') + } + } +} \ No newline at end of file diff --git a/x-pack/plugin/runtime-fields/qa/core-with-mapped/build.gradle b/x-pack/plugin/runtime-fields/qa/core-with-mapped/build.gradle new file mode 100644 index 0000000000000..ea347a8a55e7a --- /dev/null +++ b/x-pack/plugin/runtime-fields/qa/core-with-mapped/build.gradle @@ -0,0 +1 @@ +// Configured by parent project diff --git a/x-pack/plugin/runtime-fields/qa/core-with-mapped/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/mapped/CoreWithMappedRuntimeFieldsIT.java b/x-pack/plugin/runtime-fields/qa/core-with-mapped/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/mapped/CoreWithMappedRuntimeFieldsIT.java new file mode 100644 index 0000000000000..0c19187c088d0 --- /dev/null +++ b/x-pack/plugin/runtime-fields/qa/core-with-mapped/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/mapped/CoreWithMappedRuntimeFieldsIT.java @@ -0,0 +1,76 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.test.mapped; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; +import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; +import org.elasticsearch.test.rest.yaml.section.ApiCallSection; +import org.elasticsearch.xpack.runtimefields.test.CoreTestTranslater; + +import java.util.HashMap; +import java.util.Map; + +/** + * Runs elasticsearch's core rest tests replacing all field mappings with runtime fields + * that load from {@code _source}. Tests that configure the field in a way that are not + * supported by runtime fields are skipped. + */ +public class CoreWithMappedRuntimeFieldsIT extends ESClientYamlSuiteTestCase { + public CoreWithMappedRuntimeFieldsIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { + super(testCandidate); + } + + @ParametersFactory + public static Iterable parameters() throws Exception { + return new MappingRuntimeFieldTranslater().parameters(); + } + + private static class MappingRuntimeFieldTranslater extends CoreTestTranslater { + @Override + protected Map dynamicTemplateFor(String type) { + return dynamicTemplateToAddRuntimeFields(type); + } + + @Override + protected Suite suite(ClientYamlTestCandidate candidate) { + return new Suite(candidate) { + @Override + protected boolean modifyMappingProperties(String index, Map properties) { + Map newProperties = new HashMap<>(properties.size()); + Map> runtimeProperties = new HashMap<>(properties.size()); + if (false == runtimeifyMappingProperties(properties, newProperties, runtimeProperties)) { + return false; + } + for (Map.Entry> runtimeProperty : runtimeProperties.entrySet()) { + runtimeProperty.getValue().put("runtime_type", runtimeProperty.getValue().get("type")); + runtimeProperty.getValue().put("type", "runtime"); + newProperties.put(runtimeProperty.getKey(), runtimeProperty.getValue()); + } + properties.clear(); + properties.putAll(newProperties); + return true; + } + + @Override + protected boolean modifySearch(ApiCallSection search) { + // We don't need to modify the search request if the mappings are in the index + return true; + } + + @Override + protected boolean handleIndex(IndexRequest index) { + // We don't need to scrape anything out of the index requests. + return true; + } + }; + } + } +} diff --git a/x-pack/plugin/runtime-fields/qa/core-with-search/build.gradle b/x-pack/plugin/runtime-fields/qa/core-with-search/build.gradle new file mode 100644 index 0000000000000..ea347a8a55e7a --- /dev/null +++ b/x-pack/plugin/runtime-fields/qa/core-with-search/build.gradle @@ -0,0 +1 @@ +// Configured by parent project diff --git a/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java b/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java new file mode 100644 index 0000000000000..02092699ecc02 --- /dev/null +++ b/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java @@ -0,0 +1,225 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.test.search; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.common.regex.Regex; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; +import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; +import org.elasticsearch.test.rest.yaml.section.ApiCallSection; +import org.elasticsearch.test.rest.yaml.section.ExecutableSection; +import org.elasticsearch.xpack.runtimefields.test.CoreTestTranslater; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static java.util.Collections.unmodifiableMap; + +/** + * Runs elasticsearch's core rest tests disabling all mappings and replacing them + * with runtime fields defined on the search request that load from {@code _source}. Tests + * that configure the field in a way that are not supported by runtime fields are skipped. + */ +public class CoreTestsWithSearchRuntimeFieldsIT extends ESClientYamlSuiteTestCase { + public CoreTestsWithSearchRuntimeFieldsIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { + super(testCandidate); + } + + @ParametersFactory + public static Iterable parameters() throws Exception { + return new SearchRequestRuntimeFieldTranslater().parameters(); + } + + /** + * Translating the tests is fairly difficult here because instead of ES + * tracking the mappings we have to track them. We don't have to do it as + * well as ES, just well enough that we can decorate the search requests + * with types that make most tests "just work". + */ + private static class SearchRequestRuntimeFieldTranslater extends CoreTestTranslater { + @Override + protected Map dynamicTemplateFor(String type) { + return dynamicTemplateToDisableRuntimeCompatibleFields(type); + } + + @Override + protected Suite suite(ClientYamlTestCandidate candidate) { + return new Suite(candidate) { + private Map>> runtimeMappingsAfterSetup; + private Map> mappedFieldsAfterSetup; + private Map>> runtimeMappings; + private Map> mappedFields; + + @Override + public boolean modifySections(List executables) { + if (runtimeMappingsAfterSetup == null) { + // We're modifying the setup section + runtimeMappings = new HashMap<>(); + mappedFields = new HashMap<>(); + if (false == super.modifySections(executables)) { + return false; + } + runtimeMappingsAfterSetup = unmodifiableMap(runtimeMappings); + runtimeMappings = null; + mappedFieldsAfterSetup = unmodifiableMap(mappedFields); + mappedFields = null; + return true; + } + runtimeMappings = new HashMap<>(runtimeMappingsAfterSetup); + mappedFields = new HashMap<>(mappedFieldsAfterSetup); + return super.modifySections(executables); + } + + @Override + protected boolean modifyMappingProperties(String index, Map properties) { + Map untouchedMapping = new HashMap<>(); + Map> runtimeMapping = new HashMap<>(); + if (false == runtimeifyMappingProperties(properties, untouchedMapping, runtimeMapping)) { + return false; + } + properties.clear(); + properties.putAll(untouchedMapping); + mappedFields.put(index, untouchedMapping.keySet()); + runtimeMappings.put(index, runtimeMapping); + return true; + } + + @Override + protected boolean modifySearch(ApiCallSection search) { + if (search.getBodies().isEmpty()) { + search.addBody(new HashMap<>()); + } + for (Map body : search.getBodies()) { + Map runtimeMapping = runtimeMappings(search.getParams().get("index")); + if (runtimeMapping == null) { + return false; + } + body.put("runtime_mappings", runtimeMapping); + } + return true; + } + + private Map runtimeMappings(String index) { + if (index == null) { + return mergeMappings(new String[] { "*" }); + } + String[] patterns = Arrays.stream(index.split(",")).map(m -> m.equals("_all") ? "*" : m).toArray(String[]::new); + if (patterns.length == 0 && Regex.isSimpleMatchPattern(patterns[0])) { + return runtimeMappings.get(patterns[0]); + } + return mergeMappings(patterns); + } + + private Map mergeMappings(String[] patterns) { + Map> merged = new HashMap<>(); + for (Map.Entry>> indexEntry : runtimeMappings.entrySet()) { + if (false == Regex.simpleMatch(patterns, indexEntry.getKey())) { + continue; + } + for (Map.Entry> field : indexEntry.getValue().entrySet()) { + Map mergedConfig = merged.get(field.getKey()); + if (mergedConfig == null) { + merged.put(field.getKey(), field.getValue()); + } else if (false == mergedConfig.equals(field.getValue())) { + // The two indices have different runtime mappings for a field so we have to give up on running the test. + return null; + } + } + } + for (Map.Entry> indexEntry : mappedFields.entrySet()) { + if (false == Regex.simpleMatch(patterns, indexEntry.getKey())) { + continue; + } + for (String mappedField : indexEntry.getValue()) { + if (merged.containsKey(mappedField)) { + // We have a runtime mappings for a field *and* regular mapping. We can't make this test work so skip it. + return null; + } + } + } + return merged; + } + + @Override + protected boolean handleIndex(IndexRequest index) { + /* + * Ok! Let's reverse engineer dynamic mapping. Sort of. We're + * really just looking to figure out which of the runtime fields + * is "close enough" to what dynamic mapping would do. + */ + if (index.getPipeline() != null) { + // We can't attempt local dynamic mappings with pipelines + return false; + } + Map map = XContentHelper.convertToMap(index.source(), false, index.getContentType()).v2(); + Map> indexRuntimeMappings = runtimeMappings.computeIfAbsent( + index.index(), + i -> new HashMap<>() + ); + Set indexMappedfields = mappedFields.computeIfAbsent(index.index(), i -> Set.of()); + for (Map.Entry e : map.entrySet()) { + String name = e.getKey(); + if (indexRuntimeMappings.containsKey(name) || indexMappedfields.contains(name)) { + continue; + } + Object value = e.getValue(); + if (value == null) { + continue; + } + if (value instanceof Boolean) { + indexRuntimeMappings.put(name, runtimeFieldLoadingFromSource(name, "boolean")); + continue; + } + if (value instanceof Long || value instanceof Integer) { + indexRuntimeMappings.put(name, runtimeFieldLoadingFromSource(name, "long")); + continue; + } + if (value instanceof Double) { + indexRuntimeMappings.put(name, runtimeFieldLoadingFromSource(name, "double")); + continue; + } + if (false == value instanceof String) { + continue; + } + try { + Long.parseLong(value.toString()); + indexRuntimeMappings.put(name, runtimeFieldLoadingFromSource(name, "long")); + continue; + } catch (IllegalArgumentException iae) { + // Try the next one + } + try { + Double.parseDouble(value.toString()); + indexRuntimeMappings.put(name, runtimeFieldLoadingFromSource(name, "double")); + continue; + } catch (IllegalArgumentException iae) { + // Try the next one + } + try { + DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(value.toString()); + indexRuntimeMappings.put(name, runtimeFieldLoadingFromSource(name, "date")); + continue; + } catch (IllegalArgumentException iae) { + // Try the next one + } + // Strings are funny, the regular dynamic mapping puts them in "name.keyword" so we follow along. + indexRuntimeMappings.put(name + ".keyword", runtimeFieldLoadingFromSource(name, "keyword")); + } + return true; + } + }; + } + } +} diff --git a/x-pack/plugin/runtime-fields/qa/rest/build.gradle b/x-pack/plugin/runtime-fields/qa/rest/build.gradle deleted file mode 100644 index 34998468a35a2..0000000000000 --- a/x-pack/plugin/runtime-fields/qa/rest/build.gradle +++ /dev/null @@ -1,50 +0,0 @@ -apply plugin: 'elasticsearch.yaml-rest-test' - -restResources { - restApi { - includeXpack 'async_search', 'graph', '*_point_in_time' - } - restTests { - includeCore '*' - includeXpack 'async_search', 'graph' - } -} - -testClusters.yamlRestTest { - testDistribution = 'DEFAULT' - setting 'xpack.license.self_generated.type', 'trial' -} - -yamlRestTest { - systemProperty 'tests.rest.suite', - [ - 'async_search', - 'field_caps', - 'graph', - 'msearch', - 'search', - 'search.aggregation', - 'search.highlight', - 'search.inner_hits', - 'search_shards', - 'suggest', - ].join(',') - systemProperty 'tests.rest.blacklist', - [ - /////// TO FIX /////// - 'search.highlight/40_keyword_ignore/Plain Highligher should skip highlighting ignored keyword values', // The plain highlighter is incompatible with runtime fields. Worth fixing? - 'search/115_multiple_field_collapsing/two levels fields collapsing', // Broken. Gotta fix. - 'field_caps/30_filter/Field caps with index filter', // We don't support filtering field caps on runtime fields. What should we do? - 'search.aggregation/10_histogram/*', // runtime doesn't support sub-fields. Maybe it should? - 'search/140_pre_filter_search_shards/pre_filter_shard_size with shards that have no hit', - /////// TO FIX /////// - - /////// NOT SUPPORTED /////// - 'search.aggregation/280_rare_terms/*', // Requires an index and we won't have it - // Runtime fields don't have global ords - 'search.aggregation/20_terms/string profiler via global ordinals', - 'search.aggregation/20_terms/Global ordinals are loaded with the global_ordinals execution hint', - 'search.aggregation/170_cardinality_metric/profiler string', - /////// NOT SUPPORTED /////// - ].join(',') -} diff --git a/x-pack/plugin/runtime-fields/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/rest/CoreTestsWithRuntimeFieldsIT.java b/x-pack/plugin/runtime-fields/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/rest/CoreTestsWithRuntimeFieldsIT.java deleted file mode 100644 index 0ca9981ba317f..0000000000000 --- a/x-pack/plugin/runtime-fields/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/rest/CoreTestsWithRuntimeFieldsIT.java +++ /dev/null @@ -1,259 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -package org.elasticsearch.xpack.runtimefields.rest; - -import com.carrotsearch.randomizedtesting.annotations.Name; -import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import org.elasticsearch.common.xcontent.XContentLocation; -import org.elasticsearch.index.mapper.BooleanFieldMapper; -import org.elasticsearch.index.mapper.DateFieldMapper; -import org.elasticsearch.index.mapper.IpFieldMapper; -import org.elasticsearch.index.mapper.KeywordFieldMapper; -import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; -import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; -import org.elasticsearch.test.rest.yaml.ClientYamlTestExecutionContext; -import org.elasticsearch.test.rest.yaml.ClientYamlTestResponse; -import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; -import org.elasticsearch.test.rest.yaml.section.ClientYamlTestSection; -import org.elasticsearch.test.rest.yaml.section.ClientYamlTestSuite; -import org.elasticsearch.test.rest.yaml.section.DoSection; -import org.elasticsearch.test.rest.yaml.section.ExecutableSection; -import org.elasticsearch.test.rest.yaml.section.SetupSection; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; - -import static org.hamcrest.Matchers.equalTo; - -public class CoreTestsWithRuntimeFieldsIT extends ESClientYamlSuiteTestCase { - public CoreTestsWithRuntimeFieldsIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { - super(testCandidate); - } - - /** - * Builds test parameters similarly to {@link ESClientYamlSuiteTestCase#createParameters()}, - * replacing the body of index creation commands so that fields are {@code runtime}s - * that load from {@code source} instead of their original type. Test configurations that - * do are not modified to contain runtime fields are not returned as they are tested - * elsewhere. - */ - @ParametersFactory - public static Iterable parameters() throws Exception { - Map suites = new HashMap<>(); - List result = new ArrayList<>(); - for (Object[] orig : ESClientYamlSuiteTestCase.createParameters()) { - assert orig.length == 1; - ClientYamlTestCandidate candidate = (ClientYamlTestCandidate) orig[0]; - ClientYamlTestSuite suite = suites.computeIfAbsent(candidate.getTestPath(), k -> modifiedSuite(candidate)); - if (suite == null) { - // The setup section contains an unsupported option - continue; - } - if (false == modifySection(candidate.getTestSection().getExecutableSections())) { - // The test section contains an unsupported option - continue; - } - ClientYamlTestSection modified = new ClientYamlTestSection( - candidate.getTestSection().getLocation(), - candidate.getTestSection().getName(), - candidate.getTestSection().getSkipSection(), - candidate.getTestSection().getExecutableSections() - ); - result.add(new Object[] { new ClientYamlTestCandidate(suite, modified) }); - } - return result; - } - - /** - * Modify the setup section to setup a dynamic template that replaces - * field configurations with scripts that load from source - * and replaces field configurations in {@code incides.create} - * with scripts that load from source. - */ - private static ClientYamlTestSuite modifiedSuite(ClientYamlTestCandidate candidate) { - if (false == modifySection(candidate.getSetupSection().getExecutableSections())) { - return null; - } - List setup = new ArrayList<>(candidate.getSetupSection().getExecutableSections().size() + 1); - setup.add(ADD_TEMPLATE); - setup.addAll(candidate.getSetupSection().getExecutableSections()); - return new ClientYamlTestSuite( - candidate.getApi(), - candidate.getName(), - new SetupSection(candidate.getSetupSection().getSkipSection(), setup), - candidate.getTeardownSection(), - List.of() - ); - } - - /** - * Replace field configuration in {@code indices.create} with scripts - * that load from the source. - */ - private static boolean modifySection(List executables) { - for (ExecutableSection section : executables) { - if (false == (section instanceof DoSection)) { - continue; - } - DoSection doSection = (DoSection) section; - if (false == doSection.getApiCallSection().getApi().equals("indices.create")) { - continue; - } - for (Map body : doSection.getApiCallSection().getBodies()) { - Object settings = body.get("settings"); - if (settings instanceof Map && ((Map) settings).containsKey("sort.field")) { - /* - * You can't sort the index on a runtime_keyword and it is - * hard to figure out if the sort was a runtime_keyword so - * let's just skip this test. - */ - continue; - } - Object mappings = body.get("mappings"); - if (false == (mappings instanceof Map)) { - continue; - } - Object properties = ((Map) mappings).get("properties"); - if (false == (properties instanceof Map)) { - continue; - } - for (Map.Entry property : ((Map) properties).entrySet()) { - if (false == property.getValue() instanceof Map) { - continue; - } - @SuppressWarnings("unchecked") - Map propertyMap = (Map) property.getValue(); - String name = property.getKey().toString(); - String type = Objects.toString(propertyMap.get("type")); - if ("nested".equals(type)) { - // Our loading scripts can't be made to manage nested fields so we have to skip those tests. - return false; - } - if ("false".equals(Objects.toString(propertyMap.get("doc_values")))) { - // If doc_values is false we can't emulate with scripts. `null` and `true` are fine. - continue; - } - if ("false".equals(Objects.toString(propertyMap.get("index")))) { - // If index is false we can't emulate with scripts - continue; - } - if ("true".equals(Objects.toString(propertyMap.get("store")))) { - // If store is true we can't emulate with scripts - continue; - } - if (propertyMap.containsKey("ignore_above")) { - // Scripts don't support ignore_above so we skip those fields - continue; - } - if (propertyMap.containsKey("ignore_malformed")) { - // Our source reading script doesn't emulate ignore_malformed - continue; - } - String toLoad = painlessToLoadFromSource(name, type); - if (toLoad == null) { - continue; - } - propertyMap.put("type", "runtime"); - propertyMap.put("runtime_type", type); - propertyMap.put("script", toLoad); - propertyMap.remove("store"); - propertyMap.remove("index"); - propertyMap.remove("doc_values"); - } - } - } - return true; - } - - private static String painlessToLoadFromSource(String name, String type) { - String emit = PAINLESS_TO_EMIT.get(type); - if (emit == null) { - return null; - } - StringBuilder b = new StringBuilder(); - b.append("def v = params._source['").append(name).append("'];\n"); - b.append("if (v instanceof Iterable) {\n"); - b.append(" for (def vv : ((Iterable) v)) {\n"); - b.append(" if (vv != null) {\n"); - b.append(" def value = vv;\n"); - b.append(" ").append(emit).append("\n"); - b.append(" }\n"); - b.append(" }\n"); - b.append("} else {\n"); - b.append(" if (v != null) {\n"); - b.append(" def value = v;\n"); - b.append(" ").append(emit).append("\n"); - b.append(" }\n"); - b.append("}\n"); - return b.toString(); - } - - private static final Map PAINLESS_TO_EMIT = Map.ofEntries( - Map.entry(BooleanFieldMapper.CONTENT_TYPE, "emit(Boolean.parseBoolean(value.toString()));"), - Map.entry(DateFieldMapper.CONTENT_TYPE, "emit(parse(value.toString()));"), - Map.entry( - NumberType.DOUBLE.typeName(), - "emit(value instanceof Number ? ((Number) value).doubleValue() : Double.parseDouble(value.toString()));" - ), - Map.entry(KeywordFieldMapper.CONTENT_TYPE, "emit(value.toString());"), - Map.entry(IpFieldMapper.CONTENT_TYPE, "emit(value.toString());"), - Map.entry( - NumberType.LONG.typeName(), - "emit(value instanceof Number ? ((Number) value).longValue() : Long.parseLong(value.toString()));" - ) - ); - - private static final ExecutableSection ADD_TEMPLATE = new ExecutableSection() { - @Override - public XContentLocation getLocation() { - return new XContentLocation(-1, -1); - } - - @Override - public void execute(ClientYamlTestExecutionContext executionContext) throws IOException { - Map params = Map.of("name", "convert_to_source_only", "create", "true"); - List> dynamicTemplates = new ArrayList<>(); - for (String type : PAINLESS_TO_EMIT.keySet()) { - if (type.equals("ip")) { - // There isn't a dynamic template to pick up ips. They'll just look like strings. - continue; - } - Map mapping = Map.ofEntries( - Map.entry("type", "runtime"), - Map.entry("runtime_type", type), - Map.entry("script", painlessToLoadFromSource("{name}", type)) - ); - if (type.contentEquals("keyword")) { - /* - * For "string"-type dynamic mappings emulate our default - * behavior with a top level text field and a `.keyword` - * multi-field. But instead of the default, use a runtime - * field for the multi-field. - */ - mapping = Map.of("type", "text", "fields", Map.of("keyword", mapping)); - dynamicTemplates.add(Map.of(type, Map.of("match_mapping_type", "string", "mapping", mapping))); - } else { - dynamicTemplates.add(Map.of(type, Map.of("match_mapping_type", type, "mapping", mapping))); - } - } - List> bodies = List.of( - Map.ofEntries( - Map.entry("index_patterns", "*"), - Map.entry("priority", Integer.MAX_VALUE - 1), - Map.entry("template", Map.of("settings", Map.of(), "mappings", Map.of("dynamic_templates", dynamicTemplates))) - ) - ); - ClientYamlTestResponse response = executionContext.callApi("indices.put_index_template", params, bodies, Map.of()); - assertThat(response.getStatusCode(), equalTo(200)); - // There are probably some warning about overlapping templates. Ignore them. - } - }; -} diff --git a/x-pack/plugin/runtime-fields/qa/src/main/java/org/elasticsearch/xpack/runtimefields/test/CoreTestTranslater.java b/x-pack/plugin/runtime-fields/qa/src/main/java/org/elasticsearch/xpack/runtimefields/test/CoreTestTranslater.java new file mode 100644 index 0000000000000..bc03972da2000 --- /dev/null +++ b/x-pack/plugin/runtime-fields/qa/src/main/java/org/elasticsearch/xpack/runtimefields/test/CoreTestTranslater.java @@ -0,0 +1,407 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.runtimefields.test; + +import org.elasticsearch.action.bulk.BulkRequestParser; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentLocation; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.index.mapper.BooleanFieldMapper; +import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.index.mapper.IpFieldMapper; +import org.elasticsearch.index.mapper.KeywordFieldMapper; +import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; +import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; +import org.elasticsearch.test.rest.yaml.ClientYamlTestExecutionContext; +import org.elasticsearch.test.rest.yaml.ClientYamlTestResponse; +import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; +import org.elasticsearch.test.rest.yaml.section.ApiCallSection; +import org.elasticsearch.test.rest.yaml.section.ClientYamlTestSection; +import org.elasticsearch.test.rest.yaml.section.ClientYamlTestSuite; +import org.elasticsearch.test.rest.yaml.section.DoSection; +import org.elasticsearch.test.rest.yaml.section.ExecutableSection; +import org.elasticsearch.test.rest.yaml.section.SetupSection; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import static org.hamcrest.Matchers.equalTo; +import static org.junit.Assert.assertThat; + +/** + * Builds test parameters similarly to {@link ESClientYamlSuiteTestCase#createParameters()}, + * replacing all fields with runtime fields that load from {@code _source} if possible. Tests + * that configure the field in a way that are not supported by runtime fields are skipped. + */ +public abstract class CoreTestTranslater { + public Iterable parameters() throws Exception { + Map suites = new HashMap<>(); + List result = new ArrayList<>(); + for (Object[] orig : ESClientYamlSuiteTestCase.createParameters()) { + assert orig.length == 1; + ClientYamlTestCandidate candidate = (ClientYamlTestCandidate) orig[0]; + Suite suite = suites.computeIfAbsent(candidate.getSuitePath(), k -> suite(candidate)); + if (suite.modified == null) { + // The setup section contains an unsupported option + continue; + } + if (false == suite.modifySections(candidate.getTestSection().getExecutableSections())) { + // The test section contains an unsupported option + continue; + } + ClientYamlTestSection modified = new ClientYamlTestSection( + candidate.getTestSection().getLocation(), + candidate.getTestSection().getName(), + candidate.getTestSection().getSkipSection(), + candidate.getTestSection().getExecutableSections() + ); + result.add(new Object[] { new ClientYamlTestCandidate(suite.modified, modified) }); + } + return result; + } + + protected abstract Suite suite(ClientYamlTestCandidate candidate); + + private static String painlessToLoadFromSource(String name, String type) { + String emit = PAINLESS_TO_EMIT.get(type); + if (emit == null) { + return null; + } + StringBuilder b = new StringBuilder(); + b.append("def v = params._source['").append(name).append("'];\n"); + b.append("if (v instanceof Iterable) {\n"); + b.append(" for (def vv : ((Iterable) v)) {\n"); + b.append(" if (vv != null) {\n"); + b.append(" def value = vv;\n"); + b.append(" ").append(emit).append("\n"); + b.append(" }\n"); + b.append(" }\n"); + b.append("} else {\n"); + b.append(" if (v != null) {\n"); + b.append(" def value = v;\n"); + b.append(" ").append(emit).append("\n"); + b.append(" }\n"); + b.append("}\n"); + return b.toString(); + } + + private static final Map PAINLESS_TO_EMIT = Map.ofEntries( + Map.entry(BooleanFieldMapper.CONTENT_TYPE, "emit(Boolean.parseBoolean(value.toString()));"), + Map.entry(DateFieldMapper.CONTENT_TYPE, "emit(parse(value.toString()));"), + Map.entry( + NumberType.DOUBLE.typeName(), + "emit(value instanceof Number ? ((Number) value).doubleValue() : Double.parseDouble(value.toString()));" + ), + Map.entry(KeywordFieldMapper.CONTENT_TYPE, "emit(value.toString());"), + Map.entry(IpFieldMapper.CONTENT_TYPE, "emit(value.toString());"), + Map.entry( + NumberType.LONG.typeName(), + "emit(value instanceof Number ? ((Number) value).longValue() : Long.parseLong(value.toString()));" + ) + ); + + protected abstract Map dynamicTemplateFor(String type); + + protected static Map dynamicTemplateToDisableRuntimeCompatibleFields(String type) { + return Map.of("type", type, "index", false, "doc_values", false); + } + + protected static Map dynamicTemplateToAddRuntimeFields(String type) { + return Map.ofEntries( + Map.entry("type", "runtime"), + Map.entry("runtime_type", type), + Map.entry("script", painlessToLoadFromSource("{name}", type)) + ); + } + + protected static Map runtimeFieldLoadingFromSource(String name, String type) { + return Map.of("type", type, "script", painlessToLoadFromSource(name, type)); + } + + private ExecutableSection addIndexTemplate() { + return new ExecutableSection() { + @Override + public XContentLocation getLocation() { + return new XContentLocation(-1, -1); + } + + @Override + public void execute(ClientYamlTestExecutionContext executionContext) throws IOException { + Map params = Map.of("name", "hack_dynamic_mappings", "create", "true"); + List> dynamicTemplates = new ArrayList<>(); + for (String type : PAINLESS_TO_EMIT.keySet()) { + if (type.equals("ip")) { + // There isn't a dynamic template to pick up ips. They'll just look like strings. + continue; + } + Map mapping = dynamicTemplateFor(type); + if (type.equals("keyword")) { + /* + * For "string"-type dynamic mappings emulate our default + * behavior with a top level text field and a `.keyword` + * multi-field. In our case we disable the keyword field + * and substitute it with an enabled one on the search + * request. + */ + mapping = Map.of("type", "text", "fields", Map.of("keyword", mapping)); + dynamicTemplates.add(Map.of(type, Map.of("match_mapping_type", "string", "mapping", mapping))); + } else { + dynamicTemplates.add(Map.of(type, Map.of("match_mapping_type", type, "mapping", mapping))); + } + } + Map indexTemplate = Map.of("settings", Map.of(), "mappings", Map.of("dynamic_templates", dynamicTemplates)); + List> bodies = List.of( + Map.ofEntries( + Map.entry("index_patterns", "*"), + Map.entry("priority", Integer.MAX_VALUE - 1), + Map.entry("template", indexTemplate) + ) + ); + ClientYamlTestResponse response = executionContext.callApi("indices.put_index_template", params, bodies, Map.of()); + assertThat(response.getStatusCode(), equalTo(200)); + // There are probably some warning about overlapping templates. Ignore them. + } + }; + } + + /** + * A modified suite. + */ + protected abstract class Suite { + private final ClientYamlTestSuite modified; + + public Suite(ClientYamlTestCandidate candidate) { + if (false == modifySections(candidate.getSetupSection().getExecutableSections())) { + modified = null; + return; + } + /* + * Modify the setup section to rewrite and create index commands and + * to add a dynamic template that sets up any dynamic indices how we + * expect them. + */ + List setup = new ArrayList<>(candidate.getSetupSection().getExecutableSections().size() + 1); + setup.add(addIndexTemplate()); + setup.addAll(candidate.getSetupSection().getExecutableSections()); + modified = new ClientYamlTestSuite( + candidate.getApi(), + candidate.getName(), + new SetupSection(candidate.getSetupSection().getSkipSection(), setup), + candidate.getTeardownSection(), + List.of() + ); + } + + /** + * Replace field configuration in {@code indices.create} with scripts + * that load from the source. + * + * @return true if the section is appropriate for testing with runtime fields + */ + public boolean modifySections(List executables) { + for (ExecutableSection section : executables) { + if (false == (section instanceof DoSection)) { + continue; + } + DoSection doSection = (DoSection) section; + String api = doSection.getApiCallSection().getApi(); + switch (api) { + case "indices.create": + if (false == modifyCreateIndex(doSection.getApiCallSection())) { + return false; + } + break; + case "search": + case "async_search.submit": + if (false == modifySearch(doSection.getApiCallSection())) { + return false; + } + break; + case "bulk": + if (false == handleBulk(doSection.getApiCallSection())) { + return false; + } + break; + case "index": + if (false == handleIndex(doSection.getApiCallSection())) { + return false; + } + break; + default: + continue; + } + } + return true; + } + + /** + * Modify a test search request. + */ + protected abstract boolean modifySearch(ApiCallSection search); + + private boolean modifyCreateIndex(ApiCallSection createIndex) { + String index = createIndex.getParams().get("index"); + for (Map body : createIndex.getBodies()) { + Object settings = body.get("settings"); + if (settings instanceof Map && ((Map) settings).containsKey("sort.field")) { + /* + * You can't sort the index on a runtime_keyword and it is + * hard to figure out if the sort was a runtime_keyword so + * let's just skip this test. + */ + continue; + } + Object mapping = body.get("mappings"); + if (false == (mapping instanceof Map)) { + continue; + } + Object properties = ((Map) mapping).get("properties"); + if (false == (properties instanceof Map)) { + continue; + } + @SuppressWarnings("unchecked") + Map propertiesMap = (Map) properties; + if (false == modifyMappingProperties(index, propertiesMap)) { + return false; + } + } + return true; + } + + /** + * Modify the mapping defined in the test. + */ + protected abstract boolean modifyMappingProperties(String index, Map properties); + + /** + * Modify the provided map in place, translating all fields into + * runtime fields that load from source. + * @return true if this mapping supports runtime fields, false otherwise + */ + protected final boolean runtimeifyMappingProperties( + Map properties, + Map untouchedProperties, + Map> runtimeProperties + ) { + for (Map.Entry property : properties.entrySet()) { + if (false == property.getValue() instanceof Map) { + untouchedProperties.put(property.getKey(), property.getValue()); + continue; + } + @SuppressWarnings("unchecked") + Map propertyMap = (Map) property.getValue(); + String name = property.getKey().toString(); + String type = Objects.toString(propertyMap.get("type")); + if ("nested".equals(type)) { + // Our loading scripts can't be made to manage nested fields so we have to skip those tests. + return false; + } + if ("false".equals(Objects.toString(propertyMap.get("doc_values")))) { + // If doc_values is false we can't emulate with scripts. So we keep the old definition. `null` and `true` are fine. + untouchedProperties.put(property.getKey(), property.getValue()); + continue; + } + if ("false".equals(Objects.toString(propertyMap.get("index")))) { + // If index is false we can't emulate with scripts + untouchedProperties.put(property.getKey(), property.getValue()); + continue; + } + if ("true".equals(Objects.toString(propertyMap.get("store")))) { + // If store is true we can't emulate with scripts + untouchedProperties.put(property.getKey(), property.getValue()); + continue; + } + if (propertyMap.containsKey("ignore_above")) { + // Scripts don't support ignore_above so we skip those fields + untouchedProperties.put(property.getKey(), property.getValue()); + continue; + } + if (propertyMap.containsKey("ignore_malformed")) { + // Our source reading script doesn't emulate ignore_malformed + untouchedProperties.put(property.getKey(), property.getValue()); + continue; + } + String toLoad = painlessToLoadFromSource(name, type); + if (toLoad == null) { + untouchedProperties.put(property.getKey(), property.getValue()); + continue; + } + Map runtimeConfig = new HashMap<>(propertyMap); + runtimeConfig.put("script", toLoad); + runtimeConfig.remove("store"); + runtimeConfig.remove("index"); + runtimeConfig.remove("doc_values"); + runtimeProperties.put(property.getKey(), runtimeConfig); + } + /* + * Its tempting to return false here if we didn't make any runtime + * fields, skipping the test. But that would cause us to skip any + * test uses dynamic mappings. Disaster! Instead we use a dynamic + * template to make the dynamic mappings into runtime fields too. + * The downside is that we can run tests that don't use runtime + * fields at all. That's unfortunate, but its ok. + */ + return true; + } + + private boolean handleBulk(ApiCallSection bulk) { + String defaultIndex = bulk.getParams().get("index"); + String defaultRouting = bulk.getParams().get("routing"); + String defaultPipeline = bulk.getParams().get("pipeline"); + BytesStreamOutput bos = new BytesStreamOutput(); + try { + for (Map body : bulk.getBodies()) { + try (XContentBuilder b = new XContentBuilder(JsonXContent.jsonXContent, bos)) { + b.map(body); + } + bos.write(JsonXContent.jsonXContent.streamSeparator()); + } + List indexRequests = new ArrayList<>(); + new BulkRequestParser(false).parse( + bos.bytes(), + defaultIndex, + defaultRouting, + null, + defaultPipeline, + null, + true, + XContentType.JSON, + (index, type) -> indexRequests.add(index), + u -> {}, + d -> {} + ); + for (IndexRequest index : indexRequests) { + if (false == handleIndex(index)) { + return false; + } + } + } catch (IOException e) { + throw new AssertionError(e); + } + return true; + } + + private boolean handleIndex(ApiCallSection indexRequest) { + String index = indexRequest.getParams().get("index"); + String pipeline = indexRequest.getParams().get("pipeline"); + assert indexRequest.getBodies().size() == 1; + try { + return handleIndex(new IndexRequest(index).setPipeline(pipeline).source(indexRequest.getBodies().get(0))); + } catch (IOException e) { + throw new AssertionError(e); + } + } + + protected abstract boolean handleIndex(IndexRequest index) throws IOException; + } +} diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java index 870fb9faf3ed8..8cd10494e0c11 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java @@ -285,6 +285,7 @@ import java.util.stream.Collectors; import static java.util.Collections.emptyList; +import static java.util.Collections.emptyMap; import static java.util.Collections.singletonList; import static org.elasticsearch.xpack.core.XPackSettings.API_KEY_SERVICE_ENABLED_SETTING; import static org.elasticsearch.xpack.core.XPackSettings.HTTP_SSL_ENABLED; @@ -722,7 +723,10 @@ public void onIndexModule(IndexModule module) { () -> { throw new IllegalArgumentException("permission filters are not allowed to use the current timestamp"); - }, null), + }, + null, + // Don't use runtime mappings in the security query + emptyMap()), dlsBitsetCache.get(), securityContext.get(), getLicenseState(), diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_keyword.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_keyword.yml index 840a0906dc1db..733c2f0b50190 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_keyword.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_keyword.yml @@ -360,3 +360,110 @@ setup: - match: { aggregations.to-users.users.hits.hits.2._index: test } - match: { aggregations.to-users.users.hits.hits.2._nested.field: users } - match: { aggregations.to-users.users.hits.hits.2._nested.offset: 1 } + +--- +"fetch defined on search request": + - do: + search: + index: sensor + body: + runtime_mappings: + voltage_rating: + type: keyword + script: | + double v = doc['voltage'].value; + if (v < 4.8) { + emit('low'); + } else if (v > 5.2) { + emit('high'); + } else { + emit('ok'); + } + fields: [voltage_rating] + sort: timestamp + - match: {hits.total.value: 6} + - match: {hits.hits.0._source.voltage: 4.0} + - match: {hits.hits.0.fields.voltage_rating: [low]} + +--- +"match defined on search request": + - do: + search: + index: sensor + body: + runtime_mappings: + voltage.rating: + type: keyword + script: | + double v = doc['voltage'].value; + if (v < 4.8) { + emit('low'); + } else if (v > 5.2) { + emit('high'); + } else { + emit('ok'); + } + query: + match: + voltage.rating: ok + sort: timestamp + - match: {hits.total.value: 2} + - match: {hits.hits.0._source.voltage: 5.1} + +--- +"search glob defined on search request": + - do: + search: + index: sensor + body: + runtime_mappings: + voltage.rating: + type: keyword + script: | + double v = doc['voltage'].value; + if (v < 4.8) { + emit('low'); + } else if (v > 5.2) { + emit('high'); + } else { + emit('ok'); + } + query: + simple_query_string: + fields: [voltage.*] + query: ok + sort: timestamp + - match: {hits.total.value: 2} + - match: {hits.hits.0._source.voltage: 5.1} + + +--- +"replace object field on search request": + - do: + bulk: + index: student + refresh: true + body: | + {"index":{}} + {"name": {"first": "Andrew", "last": "Wiggin"}} + {"index":{}} + {"name": {"first": "Julian", "last": "Delphiki", "suffix": "II"}} + + - do: + search: + index: student + body: + runtime_mappings: + name.first: + type: keyword + script: | + if ('Wiggin'.equals(doc['name.last.keyword'].value)) { + emit('Ender'); + } else if ('Delphiki'.equals(doc['name.last.keyword'].value)) { + emit('Bean'); + } + query: + match: + name.first: Bean + - match: {hits.total.value: 1} + - match: {hits.hits.0._source.name.first: Julian}