From fa1a9727dba2dadeb8057a3f3cce585f7343d4fa Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Thu, 1 Nov 2018 15:10:38 +0100 Subject: [PATCH 1/6] Ignore date ranges containing 'now' when pre-processing a percolator query Today when a percolator query containing a date range then the query analyzer extracts that range, so that at search time the `percolate` query can exclude percolator queries efficiently that are never going to match. The problem is that if 'now' is used it is evaluated at index time. So the idea is to exclude date ranges with 'now', so that the query analyzer can't extract it and the `percolate` query is then able to evualate 'now' at query time. --- .../percolator/PercolatorFieldMapper.java | 95 ++++++++++++++++++- .../PercolatorQuerySearchTests.java | 50 ++++++++++ 2 files changed, 143 insertions(+), 2 deletions(-) diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java index 4b46537bb1650..b30f663e44fa3 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java @@ -76,6 +76,7 @@ import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.query.QueryShardException; +import org.elasticsearch.index.query.RangeQueryBuilder; import org.elasticsearch.index.query.Rewriteable; import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder; @@ -409,8 +410,14 @@ public void parse(ParseContext context) throws IOException { Version indexVersion = context.mapperService().getIndexSettings().getIndexVersionCreated(); createQueryBuilderField(indexVersion, queryBuilderField, queryBuilder, context); - Query query = toQuery(queryShardContext, isMapUnmappedFieldAsText(), queryBuilder); - processQuery(query, context); + QueryBuilder queryBuilderForProcessing = rewrite(queryBuilder); + if (queryBuilderForProcessing != null) { + Query query = toQuery(queryShardContext, isMapUnmappedFieldAsText(), queryBuilderForProcessing); + processQuery(query, context); + } else { + FieldType pft = (FieldType) this.fieldType(); + context.doc().add(new Field(pft.extractionResultField.name(), EXTRACTION_FAILED, extractionResultField.fieldType())); + } } static void createQueryBuilderField(Version indexVersion, BinaryFieldMapper qbField, @@ -566,6 +573,90 @@ static void verifyQuery(QueryBuilder queryBuilder) { } } + static QueryBuilder rewrite(QueryBuilder queryBuilder) { + if (queryBuilder instanceof BoolQueryBuilder) { + BoolQueryBuilder boolQueryBuilder = (BoolQueryBuilder) queryBuilder; + BoolQueryBuilder newBoolQueryBuilder = new BoolQueryBuilder(); + newBoolQueryBuilder.minimumShouldMatch(boolQueryBuilder.minimumShouldMatch()); + newBoolQueryBuilder.adjustPureNegative(boolQueryBuilder.adjustPureNegative()); + + for (QueryBuilder clause : boolQueryBuilder.filter()) { + QueryBuilder result = rewrite(clause); + if (result != null) { + newBoolQueryBuilder.filter(result); + } + } + + for (QueryBuilder clause : boolQueryBuilder.must()) { + QueryBuilder result = rewrite(clause); + if (result != null) { + newBoolQueryBuilder.must(result); + } + } + + for (QueryBuilder clause : boolQueryBuilder.should()) { + QueryBuilder result = rewrite(clause); + if (result != null) { + newBoolQueryBuilder.should(result); + } + } + + for (QueryBuilder clause : boolQueryBuilder.mustNot()) { + QueryBuilder result = rewrite(clause); + if (result != null) { + newBoolQueryBuilder.mustNot(result); + } + } + + return newBoolQueryBuilder; + } else if (queryBuilder instanceof ConstantScoreQueryBuilder) { + QueryBuilder innerQuery = rewrite(((ConstantScoreQueryBuilder) queryBuilder).innerQuery()); + if (innerQuery != null) { + return queryBuilder; + } else { + return null; + } + } else if (queryBuilder instanceof FunctionScoreQueryBuilder) { + QueryBuilder innerQuery = rewrite(((FunctionScoreQueryBuilder) queryBuilder).query()); + if (innerQuery != null) { + return queryBuilder; + } else { + return null; + } + } else if (queryBuilder instanceof BoostingQueryBuilder) { + QueryBuilder negativeQuery = rewrite(((BoostingQueryBuilder) queryBuilder).negativeQuery()); + QueryBuilder positiveQuery = rewrite(((BoostingQueryBuilder) queryBuilder).positiveQuery()); + if (negativeQuery != null && positiveQuery != null) { + return null; + } else { + return queryBuilder; + } + } else if (queryBuilder instanceof DisMaxQueryBuilder) { + DisMaxQueryBuilder disMaxQueryBuilder = (DisMaxQueryBuilder) queryBuilder; + + DisMaxQueryBuilder newDisMaxQueryBuilder = new DisMaxQueryBuilder(); + newDisMaxQueryBuilder.tieBreaker(disMaxQueryBuilder.tieBreaker()); + for (QueryBuilder innerQueryBuilder : disMaxQueryBuilder.innerQueries()) { + QueryBuilder result = rewrite(innerQueryBuilder); + if (result != null) { + newDisMaxQueryBuilder.add(result); + } + } + + return newDisMaxQueryBuilder; + } else if (queryBuilder instanceof RangeQueryBuilder) { + RangeQueryBuilder rangeQueryBuilder = (RangeQueryBuilder) queryBuilder; + if ((rangeQueryBuilder.from() != null && rangeQueryBuilder.from().toString().contains("now")) || + (rangeQueryBuilder.to() != null && rangeQueryBuilder.to().toString().contains("now"))) { + return null; + } else { + return queryBuilder; + } + } else { + return queryBuilder; + } + } + static byte[] encodeRange(String rangeFieldName, byte[] minEncoded, byte[] maxEncoded) { assert minEncoded.length == maxEncoded.length; byte[] bytes = new byte[BinaryRange.BYTES * 2]; diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchTests.java index b7693f514393b..a3880266ade76 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchTests.java @@ -47,7 +47,12 @@ import java.util.function.Function; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.index.query.QueryBuilders.boolQuery; import static org.elasticsearch.index.query.QueryBuilders.matchQuery; +import static org.elasticsearch.index.query.QueryBuilders.rangeQuery; +import static org.elasticsearch.index.query.QueryBuilders.scriptQuery; +import static org.elasticsearch.index.query.QueryBuilders.termQuery; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHits; @@ -221,4 +226,49 @@ public void testMapUnmappedFieldAsText() throws IOException { assertSearchHits(response, "1"); } + public void testRangeQueriesWithNow() throws Exception { + assertAcked(client().admin().indices().prepareCreate("test") + .addMapping("_doc", "field1", "type=keyword", "field2", "type=date", "query", "type=percolator") + ); + + client().prepareIndex("test", "_doc", "1") + .setSource(jsonBuilder().startObject().field("query", rangeQuery("field2").from("now-3s").to("now+3s")).endObject()) + .get(); + client().prepareIndex("test", "_doc", "2") + .setSource(jsonBuilder().startObject().field("query", boolQuery() + .filter(termQuery("field1", "value")) + .filter(rangeQuery("field2").from("now-3s").to("now+3s")) + ).endObject()) + .get(); + + + Script script = new Script(ScriptType.INLINE, MockScriptPlugin.NAME, "1==1", Collections.emptyMap()); + client().prepareIndex("test", "_doc", "3") + .setSource(jsonBuilder().startObject().field("query", boolQuery() + .filter(scriptQuery(script)) + .filter(rangeQuery("field2").from("now-3s").to("now+3s")) + ).endObject()) + .get(); + client().admin().indices().prepareRefresh().get(); + + BytesReference source = BytesReference.bytes(jsonBuilder().startObject() + .field("field1", "value") + .field("field2", System.currentTimeMillis()) + .endObject()); + SearchResponse response = client().prepareSearch() + .setQuery(new PercolateQueryBuilder("query", source, XContentType.JSON)) + .get(); + assertHitCount(response, 3); + + Thread.sleep(5000); + source = BytesReference.bytes(jsonBuilder().startObject() + .field("field1", "value") + .field("field2", System.currentTimeMillis()) + .endObject()); + response = client().prepareSearch() + .setQuery(new PercolateQueryBuilder("query", source, XContentType.JSON)) + .get(); + assertHitCount(response, 3); + } + } From 21b7b869d8ac14ab266ea13ed69dd6eee8683c0b Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Tue, 6 Nov 2018 07:38:52 +0100 Subject: [PATCH 2/6] Added `supportsNowInRangeQueries()` to `QueryRewriteContext` that controls whether a context supports `now` in date ranges. This defaults to `true`. In the case of the percolator, it passes down a special rewrite context that returns `false.`. --- .../percolator/PercolatorFieldMapper.java | 103 ++---------------- .../index/query/QueryRewriteContext.java | 7 ++ .../index/query/RangeQueryBuilder.java | 7 ++ 3 files changed, 24 insertions(+), 93 deletions(-) diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java index b30f663e44fa3..c2e5cf5d141d1 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java @@ -76,7 +76,6 @@ import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.query.QueryShardException; -import org.elasticsearch.index.query.RangeQueryBuilder; import org.elasticsearch.index.query.Rewriteable; import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder; @@ -410,14 +409,16 @@ public void parse(ParseContext context) throws IOException { Version indexVersion = context.mapperService().getIndexSettings().getIndexVersionCreated(); createQueryBuilderField(indexVersion, queryBuilderField, queryBuilder, context); - QueryBuilder queryBuilderForProcessing = rewrite(queryBuilder); - if (queryBuilderForProcessing != null) { - Query query = toQuery(queryShardContext, isMapUnmappedFieldAsText(), queryBuilderForProcessing); - processQuery(query, context); - } else { - FieldType pft = (FieldType) this.fieldType(); - context.doc().add(new Field(pft.extractionResultField.name(), EXTRACTION_FAILED, extractionResultField.fieldType())); - } + + QueryBuilder queryBuilderForProcessing = queryBuilder.rewrite(new QueryShardContext(queryShardContext) { + + @Override + public boolean supportsNowInRangeQueries() { + return false; + } + }); + Query query = toQuery(queryShardContext, isMapUnmappedFieldAsText(), queryBuilderForProcessing); + processQuery(query, context); } static void createQueryBuilderField(Version indexVersion, BinaryFieldMapper qbField, @@ -573,90 +574,6 @@ static void verifyQuery(QueryBuilder queryBuilder) { } } - static QueryBuilder rewrite(QueryBuilder queryBuilder) { - if (queryBuilder instanceof BoolQueryBuilder) { - BoolQueryBuilder boolQueryBuilder = (BoolQueryBuilder) queryBuilder; - BoolQueryBuilder newBoolQueryBuilder = new BoolQueryBuilder(); - newBoolQueryBuilder.minimumShouldMatch(boolQueryBuilder.minimumShouldMatch()); - newBoolQueryBuilder.adjustPureNegative(boolQueryBuilder.adjustPureNegative()); - - for (QueryBuilder clause : boolQueryBuilder.filter()) { - QueryBuilder result = rewrite(clause); - if (result != null) { - newBoolQueryBuilder.filter(result); - } - } - - for (QueryBuilder clause : boolQueryBuilder.must()) { - QueryBuilder result = rewrite(clause); - if (result != null) { - newBoolQueryBuilder.must(result); - } - } - - for (QueryBuilder clause : boolQueryBuilder.should()) { - QueryBuilder result = rewrite(clause); - if (result != null) { - newBoolQueryBuilder.should(result); - } - } - - for (QueryBuilder clause : boolQueryBuilder.mustNot()) { - QueryBuilder result = rewrite(clause); - if (result != null) { - newBoolQueryBuilder.mustNot(result); - } - } - - return newBoolQueryBuilder; - } else if (queryBuilder instanceof ConstantScoreQueryBuilder) { - QueryBuilder innerQuery = rewrite(((ConstantScoreQueryBuilder) queryBuilder).innerQuery()); - if (innerQuery != null) { - return queryBuilder; - } else { - return null; - } - } else if (queryBuilder instanceof FunctionScoreQueryBuilder) { - QueryBuilder innerQuery = rewrite(((FunctionScoreQueryBuilder) queryBuilder).query()); - if (innerQuery != null) { - return queryBuilder; - } else { - return null; - } - } else if (queryBuilder instanceof BoostingQueryBuilder) { - QueryBuilder negativeQuery = rewrite(((BoostingQueryBuilder) queryBuilder).negativeQuery()); - QueryBuilder positiveQuery = rewrite(((BoostingQueryBuilder) queryBuilder).positiveQuery()); - if (negativeQuery != null && positiveQuery != null) { - return null; - } else { - return queryBuilder; - } - } else if (queryBuilder instanceof DisMaxQueryBuilder) { - DisMaxQueryBuilder disMaxQueryBuilder = (DisMaxQueryBuilder) queryBuilder; - - DisMaxQueryBuilder newDisMaxQueryBuilder = new DisMaxQueryBuilder(); - newDisMaxQueryBuilder.tieBreaker(disMaxQueryBuilder.tieBreaker()); - for (QueryBuilder innerQueryBuilder : disMaxQueryBuilder.innerQueries()) { - QueryBuilder result = rewrite(innerQueryBuilder); - if (result != null) { - newDisMaxQueryBuilder.add(result); - } - } - - return newDisMaxQueryBuilder; - } else if (queryBuilder instanceof RangeQueryBuilder) { - RangeQueryBuilder rangeQueryBuilder = (RangeQueryBuilder) queryBuilder; - if ((rangeQueryBuilder.from() != null && rangeQueryBuilder.from().toString().contains("now")) || - (rangeQueryBuilder.to() != null && rangeQueryBuilder.to().toString().contains("now"))) { - return null; - } else { - return queryBuilder; - } - } else { - return queryBuilder; - } - } - static byte[] encodeRange(String rangeFieldName, byte[] minEncoded, byte[] maxEncoded) { assert minEncoded.length == maxEncoded.length; byte[] bytes = new byte[BinaryRange.BYTES * 2]; diff --git a/server/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java b/server/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java index baf60bbbc0912..27fba14f8bff1 100644 --- a/server/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java @@ -124,4 +124,11 @@ public void onFailure(Exception e) { } } + /** + * @return whether the query rewrite context supports 'now' (current time) in range queries with data ranges. + */ + public boolean supportsNowInRangeQueries() { + return true; + } + } diff --git a/server/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java index 756c6456a9f13..eab62fe938ea1 100644 --- a/server/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java @@ -459,6 +459,13 @@ protected MappedFieldType.Relation getRelation(QueryRewriteContext queryRewriteC @Override protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws IOException { + if (queryRewriteContext.supportsNowInRangeQueries() == false) { + if ((from() != null && from().toString().contains("now")) || + (to() != null && to().toString().contains("now"))) { + return new MatchAllQueryBuilder(); + } + } + final MappedFieldType.Relation relation = getRelation(queryRewriteContext); switch (relation) { case DISJOINT: From 183ffc3191af562bd5d1f4a10f6891b2c69fc6c3 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Wed, 7 Nov 2018 11:40:13 +0100 Subject: [PATCH 3/6] renamed method and added jdocs / comments --- .../percolator/PercolatorFieldMapper.java | 4 ++-- .../elasticsearch/index/query/QueryRewriteContext.java | 10 +++++++--- .../elasticsearch/index/query/RangeQueryBuilder.java | 5 ++++- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java index e4480f2f2ab1b..6ac073ef90a02 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java @@ -413,8 +413,8 @@ public void parse(ParseContext context) throws IOException { QueryBuilder queryBuilderForProcessing = queryBuilder.rewrite(new QueryShardContext(queryShardContext) { @Override - public boolean supportsNowInRangeQueries() { - return false; + public boolean convertNowRangeToMatchAll() { + return true; } }); Query query = toQuery(queryShardContext, isMapUnmappedFieldAsText(), queryBuilderForProcessing); diff --git a/server/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java b/server/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java index 27fba14f8bff1..b275088d89441 100644 --- a/server/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java @@ -125,10 +125,14 @@ public void onFailure(Exception e) { } /** - * @return whether the query rewrite context supports 'now' (current time) in range queries with data ranges. + * In pre-processing contexts that happen at index time 'now' date ranges should be replaced by a {@link MatchAllQueryBuilder}. + * Otherwise documents that should match at query time would never match and the document that have fallen outside the + * date range would continue to match. + * + * @return indicates whether range queries with date ranges using 'now' are rewritten to a {@link MatchAllQueryBuilder}. */ - public boolean supportsNowInRangeQueries() { - return true; + public boolean convertNowRangeToMatchAll() { + return false; } } diff --git a/server/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java index eab62fe938ea1..6b8a47e8ce23e 100644 --- a/server/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java @@ -459,7 +459,10 @@ protected MappedFieldType.Relation getRelation(QueryRewriteContext queryRewriteC @Override protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws IOException { - if (queryRewriteContext.supportsNowInRangeQueries() == false) { + // Percolator queries get rewritten and pre-processed at index time. + // If a range query has a date range using 'now' and 'now' gets resolved at index time then + // the pre-processing uses that to pre-process. This can then lead to mismatches at query time. + if (queryRewriteContext.convertNowRangeToMatchAll()) { if ((from() != null && from().toString().contains("now")) || (to() != null && to().toString().contains("now"))) { return new MatchAllQueryBuilder(); From d2a6f4d31b131b9ef58e0b6e1aceee35eb5f26c1 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Wed, 7 Nov 2018 14:32:53 +0100 Subject: [PATCH 4/6] no sleeps --- .../PercolatorQuerySearchTests.java | 59 +++++++++++-------- 1 file changed, 36 insertions(+), 23 deletions(-) diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchTests.java index a3880266ade76..17781e2b59366 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchTests.java @@ -18,22 +18,30 @@ */ package org.elasticsearch.percolator; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; import org.apache.lucene.search.join.ScoreMode; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.IndexService; import org.elasticsearch.index.cache.bitset.BitsetFilterCache; +import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.query.Operator; +import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.MockScriptPlugin; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptType; +import org.elasticsearch.search.SearchService; import org.elasticsearch.search.lookup.LeafDocLookup; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ESSingleNodeTestCase; @@ -55,6 +63,7 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHits; +import static org.hamcrest.Matchers.equalTo; public class PercolatorQuerySearchTests extends ESSingleNodeTestCase { @@ -227,17 +236,16 @@ public void testMapUnmappedFieldAsText() throws IOException { } public void testRangeQueriesWithNow() throws Exception { - assertAcked(client().admin().indices().prepareCreate("test") - .addMapping("_doc", "field1", "type=keyword", "field2", "type=date", "query", "type=percolator") - ); + IndexService indexService = createIndex("test", Settings.builder().put("index.number_of_shards", 1).build(), "_doc", + "field1", "type=keyword", "field2", "type=date", "query", "type=percolator"); client().prepareIndex("test", "_doc", "1") - .setSource(jsonBuilder().startObject().field("query", rangeQuery("field2").from("now-3s").to("now+3s")).endObject()) + .setSource(jsonBuilder().startObject().field("query", rangeQuery("field2").from("now-1h").to("now+1h")).endObject()) .get(); client().prepareIndex("test", "_doc", "2") .setSource(jsonBuilder().startObject().field("query", boolQuery() .filter(termQuery("field1", "value")) - .filter(rangeQuery("field2").from("now-3s").to("now+3s")) + .filter(rangeQuery("field2").from("now-1h").to("now+1h")) ).endObject()) .get(); @@ -246,29 +254,34 @@ public void testRangeQueriesWithNow() throws Exception { client().prepareIndex("test", "_doc", "3") .setSource(jsonBuilder().startObject().field("query", boolQuery() .filter(scriptQuery(script)) - .filter(rangeQuery("field2").from("now-3s").to("now+3s")) + .filter(rangeQuery("field2").from("now-1h").to("now+1h")) ).endObject()) .get(); client().admin().indices().prepareRefresh().get(); - BytesReference source = BytesReference.bytes(jsonBuilder().startObject() - .field("field1", "value") - .field("field2", System.currentTimeMillis()) - .endObject()); - SearchResponse response = client().prepareSearch() - .setQuery(new PercolateQueryBuilder("query", source, XContentType.JSON)) - .get(); - assertHitCount(response, 3); + try (Engine.Searcher engineSearcher = indexService.getShard(0).acquireSearcher("test")) { + IndexSearcher indexSearcher = engineSearcher.searcher(); + long[] currentTime = new long[] {System.currentTimeMillis()}; + QueryShardContext queryShardContext = + indexService.newQueryShardContext(0, engineSearcher.reader(), () -> currentTime[0], null); - Thread.sleep(5000); - source = BytesReference.bytes(jsonBuilder().startObject() - .field("field1", "value") - .field("field2", System.currentTimeMillis()) - .endObject()); - response = client().prepareSearch() - .setQuery(new PercolateQueryBuilder("query", source, XContentType.JSON)) - .get(); - assertHitCount(response, 3); + BytesReference source = BytesReference.bytes(jsonBuilder().startObject() + .field("field1", "value") + .field("field2", currentTime[0]) + .endObject()); + QueryBuilder queryBuilder = new PercolateQueryBuilder("query", source, XContentType.JSON); + Query query = queryBuilder.toQuery(queryShardContext); + assertThat(indexSearcher.count(query), equalTo(3)); + + currentTime[0] = currentTime[0] + 10800000; // + 3 hours + source = BytesReference.bytes(jsonBuilder().startObject() + .field("field1", "value") + .field("field2", currentTime[0]) + .endObject()); + queryBuilder = new PercolateQueryBuilder("query", source, XContentType.JSON); + query = queryBuilder.toQuery(queryShardContext); + assertThat(indexSearcher.count(query), equalTo(3)); + } } } From 54cc60899f1294a976ed6e701f38f5b080db6c00 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Wed, 7 Nov 2018 14:40:56 +0100 Subject: [PATCH 5/6] added a unit test for range query builder --- .../index/query/RangeQueryBuilderTests.java | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/server/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java index 1fe157255b6b7..28349994c63e3 100644 --- a/server/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java @@ -563,4 +563,33 @@ public void testParseRelation() { builder.relation("intersects"); assertEquals(ShapeRelation.INTERSECTS, builder.relation()); } + + public void testConvertNowRangeToMatchAll() throws IOException { + RangeQueryBuilder query = new RangeQueryBuilder(DATE_FIELD_NAME); + DateTime queryFromValue = new DateTime(2019, 1, 1, 0, 0, 0, ISOChronology.getInstanceUTC()); + DateTime queryToValue = new DateTime(2020, 1, 1, 0, 0, 0, ISOChronology.getInstanceUTC()); + if (randomBoolean()) { + query.from("now"); + query.to(queryToValue); + } else if (randomBoolean()) { + query.from(queryFromValue); + query.to("now"); + } else { + query.from("now"); + query.to("now+1h"); + } + QueryShardContext queryShardContext = createShardContext(); + QueryBuilder rewritten = query.rewrite(queryShardContext); + assertThat(rewritten, instanceOf(RangeQueryBuilder.class)); + + queryShardContext = new QueryShardContext(queryShardContext) { + + @Override + public boolean convertNowRangeToMatchAll() { + return true; + } + }; + rewritten = query.rewrite(queryShardContext); + assertThat(rewritten, instanceOf(MatchAllQueryBuilder.class)); + } } From a188c278221fc864116d9e890673ce26e217673c Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Wed, 7 Nov 2018 15:18:58 +0100 Subject: [PATCH 6/6] fixed checkstyle violation --- .../elasticsearch/percolator/PercolatorQuerySearchTests.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchTests.java index 17781e2b59366..90c456fee9a6d 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchTests.java @@ -23,7 +23,6 @@ import org.apache.lucene.search.join.ScoreMode; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.support.WriteRequest; -import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -41,7 +40,6 @@ import org.elasticsearch.script.MockScriptPlugin; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptType; -import org.elasticsearch.search.SearchService; import org.elasticsearch.search.lookup.LeafDocLookup; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ESSingleNodeTestCase; @@ -60,7 +58,6 @@ import static org.elasticsearch.index.query.QueryBuilders.rangeQuery; import static org.elasticsearch.index.query.QueryBuilders.scriptQuery; import static org.elasticsearch.index.query.QueryBuilders.termQuery; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHits; import static org.hamcrest.Matchers.equalTo;