diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy index 4feda1eba8767..85a8cea67e8eb 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy @@ -872,6 +872,9 @@ class BuildPlugin implements Plugin { // TODO: remove this once ctx isn't added to update script params in 7.0 test.systemProperty 'es.scripting.update.ctx_in_params', 'false' + // TODO: remove when sort optimization is merged + test.systemProperty 'es.search.long_sort_optimized', 'true' + test.testLogging { TestLoggingContainer logging -> logging.showExceptions = true logging.showCauses = true diff --git a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java index 64621277f6e6f..48e238c3e3168 100644 --- a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java +++ b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java @@ -21,26 +21,36 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.lucene.document.LongPoint; +import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.PointValues; import org.apache.lucene.queries.MinDocQuery; import org.apache.lucene.queries.SearchAfterSortedDocQuery; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.Collector; import org.apache.lucene.search.ConstantScoreQuery; +import org.apache.lucene.search.DocValuesFieldExistsQuery; import org.apache.lucene.search.FieldDoc; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.Sort; +import org.apache.lucene.search.SortField; import org.apache.lucene.search.TopDocs; +import org.apache.lucene.search.TopFieldDocs; import org.apache.lucene.search.TotalHits; import org.elasticsearch.action.search.SearchTask; +import org.elasticsearch.common.Booleans; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.lucene.search.TopDocsAndMaxScore; import org.elasticsearch.common.util.concurrent.QueueResizingEsThreadPoolExecutor; +import org.elasticsearch.index.IndexSortConfig; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.DateFieldMapper.DateFieldType; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.SearchPhase; import org.elasticsearch.search.SearchService; @@ -57,6 +67,8 @@ import org.elasticsearch.tasks.TaskCancelledException; import org.elasticsearch.threadpool.ThreadPool; +import java.io.IOException; +import java.util.Arrays; import java.util.LinkedList; import java.util.concurrent.ExecutorService; import java.util.function.Consumer; @@ -67,6 +79,7 @@ import static org.elasticsearch.search.query.QueryCollectorContext.createMinScoreCollectorContext; import static org.elasticsearch.search.query.QueryCollectorContext.createMultiCollectorContext; import static org.elasticsearch.search.query.TopDocsCollectorContext.createTopDocsCollectorContext; +import static org.elasticsearch.search.query.TopDocsCollectorContext.shortcutTotalHitCount; /** @@ -75,6 +88,8 @@ */ public class QueryPhase implements SearchPhase { private static final Logger LOGGER = LogManager.getLogger(QueryPhase.class); + public static final boolean SYS_PROP_LONG_SORT_OPTIMIZED = + Booleans.parseBoolean(System.getProperty("es.search.long_sort_optimized", "false")); private final AggregationPhase aggregationPhase; private final SuggestPhase suggestPhase; @@ -133,6 +148,7 @@ public void execute(SearchContext searchContext) throws QueryPhaseExecutionExcep static boolean execute(SearchContext searchContext, final IndexSearcher searcher, Consumer checkCancellationSetter) throws QueryPhaseExecutionException { + SortAndFormats sortAndFormatsForRewrittenNumericSort = null; final IndexReader reader = searcher.getIndexReader(); QuerySearchResult queryResult = searchContext.queryResult(); queryResult.searchTimedOut(false); @@ -204,6 +220,25 @@ static boolean execute(SearchContext searchContext, hasFilterCollector = true; } + // try to rewrite numeric or date sort to the optimized distanceFeatureQuery + if ((searchContext.sort() != null) && SYS_PROP_LONG_SORT_OPTIMIZED) { + Query rewrittenQuery = tryRewriteLongSort(searchContext, searcher.getIndexReader(), query, hasFilterCollector); + if (rewrittenQuery != null) { + query = rewrittenQuery; + // modify sorts: add sort on _score as 1st sort, and move the sort on the original field as the 2nd sort + SortField[] oldSortFields = searchContext.sort().sort.getSort(); + DocValueFormat[] oldFormats = searchContext.sort().formats; + SortField[] newSortFields = new SortField[oldSortFields.length + 1]; + DocValueFormat[] newFormats = new DocValueFormat[oldSortFields.length + 1]; + newSortFields[0] = SortField.FIELD_SCORE; + newFormats[0] = DocValueFormat.RAW; + System.arraycopy(oldSortFields, 0, newSortFields, 1, oldSortFields.length); + System.arraycopy(oldFormats, 0, newFormats, 1, oldFormats.length); + sortAndFormatsForRewrittenNumericSort = searchContext.sort(); // stash SortAndFormats to restore it later + searchContext.sort(new SortAndFormats(new Sort(newSortFields), newFormats)); + } + } + boolean timeoutSet = scrollContext == null && searchContext.timeout() != null && searchContext.timeout().equals(SearchService.NO_TIMEOUT) == false; @@ -290,6 +325,13 @@ static boolean execute(SearchContext searchContext, for (QueryCollectorContext ctx : collectors) { ctx.postProcess(result); } + + // if we rewrote numeric long or date sort, restore fieldDocs based on the original sort + if (sortAndFormatsForRewrittenNumericSort != null) { + searchContext.sort(sortAndFormatsForRewrittenNumericSort); // restore SortAndFormats + restoreTopFieldDocs(result, sortAndFormatsForRewrittenNumericSort); + } + ExecutorService executor = searchContext.indexShard().getThreadPool().executor(ThreadPool.Names.SEARCH); if (executor instanceof QueueResizingEsThreadPoolExecutor) { QueueResizingEsThreadPoolExecutor rExecutor = (QueueResizingEsThreadPoolExecutor) executor; @@ -306,6 +348,92 @@ static boolean execute(SearchContext searchContext, } } + private static Query tryRewriteLongSort(SearchContext searchContext, IndexReader reader, + Query query, boolean hasFilterCollector) throws IOException { + if (searchContext.searchAfter() != null) return null; + if (searchContext.scrollContext() != null) return null; + if (searchContext.collapse() != null) return null; + if (searchContext.trackScores()) return null; + if (searchContext.aggregations() != null) return null; + Sort sort = searchContext.sort().sort; + SortField sortField = sort.getSort()[0]; + if (SortField.Type.LONG.equals(IndexSortConfig.getSortFieldType(sortField)) == false) return null; + + // check if this is a field of type Long or Date, that is indexed and has doc values + String fieldName = sortField.getField(); + if (fieldName == null) return null; // happens when _score or _doc is the 1st sort field + if (searchContext.mapperService() == null) return null; // mapperService can be null in tests + final MappedFieldType fieldType = searchContext.mapperService().fullName(fieldName); + if (fieldType == null) return null; // for unmapped fields, default behaviour depending on "unmapped_type" flag + if ((fieldType.typeName().equals("long") == false) && (fieldType instanceof DateFieldType == false)) return null; + if (fieldType.indexOptions() == IndexOptions.NONE) return null; //TODO: change to pointDataDimensionCount() when implemented + if (fieldType.hasDocValues() == false) return null; + + // check that all sorts are actual document fields or _doc + for (int i = 1; i < sort.getSort().length; i++) { + SortField sField = sort.getSort()[i]; + String sFieldName = sField.getField(); + if (sFieldName == null) { + if (SortField.FIELD_DOC.equals(sField) == false) return null; + } else { + if (searchContext.mapperService().fullName(sFieldName) == null) return null; // could be _script field that uses _score + } + } + + // check that setting of missing values allows optimization + if (sortField.getMissingValue() == null) return null; + Long missingValue = (Long) sortField.getMissingValue(); + boolean missingValuesAccordingToSort = (sortField.getReverse() && (missingValue == Long.MIN_VALUE)) || + ((sortField.getReverse() == false) && (missingValue == Long.MAX_VALUE)); + if (missingValuesAccordingToSort == false) return null; + + // check for multiple values + if (PointValues.size(reader, fieldName) != PointValues.getDocCount(reader, fieldName)) return null; //TODO: handle multiple values + + // check if the optimization makes sense with the track_total_hits setting + if (searchContext.trackTotalHitsUpTo() == Integer.MAX_VALUE) { + // with filter, we can't pre-calculate hitsCount, we need to explicitly calculate them => optimization does't make sense + if (hasFilterCollector) return null; + // if we can't pre-calculate hitsCount based on the query type, optimization does't make sense + if (shortcutTotalHitCount(reader, query) == -1) return null; + } + + byte[] minValueBytes = PointValues.getMinPackedValue(reader, fieldName); + byte[] maxValueBytes = PointValues.getMaxPackedValue(reader, fieldName); + if ((maxValueBytes == null) || (minValueBytes == null)) return null; + long minValue = LongPoint.decodeDimension(minValueBytes, 0); + long maxValue = LongPoint.decodeDimension(maxValueBytes, 0); + + Query rewrittenQuery; + if (minValue == maxValue) { + rewrittenQuery = new DocValuesFieldExistsQuery(fieldName); + } else { + long origin = (sortField.getReverse()) ? maxValue : minValue; + long pivotDistance = (maxValue - minValue) >>> 1; // division by 2 on the unsigned representation to avoid overflow + if (pivotDistance == 0) { // 0 if maxValue = (minValue + 1) + pivotDistance = 1; + } + rewrittenQuery = LongPoint.newDistanceFeatureQuery(sortField.getField(), 1, origin, pivotDistance); + } + rewrittenQuery = new BooleanQuery.Builder() + .add(query, BooleanClause.Occur.FILTER) // filter for original query + .add(rewrittenQuery, BooleanClause.Occur.SHOULD) //should for rewrittenQuery + .build(); + return rewrittenQuery; + } + + // Restore fieldsDocs to remove the first _score sort + // updating in place without creating new FieldDoc objects + static void restoreTopFieldDocs(QuerySearchResult result, SortAndFormats originalSortAndFormats) { + TopDocs topDocs = result.topDocs().topDocs; + for (ScoreDoc scoreDoc : topDocs.scoreDocs) { + FieldDoc fieldDoc = (FieldDoc) scoreDoc; + fieldDoc.fields = Arrays.copyOfRange(fieldDoc.fields, 1, fieldDoc.fields.length); + } + TopFieldDocs newTopDocs = new TopFieldDocs(topDocs.totalHits, topDocs.scoreDocs, originalSortAndFormats.sort.getSort()); + result.topDocs(new TopDocsAndMaxScore(newTopDocs, Float.NaN), originalSortAndFormats.formats); + } + /** * Returns true if the provided query returns docs in index order (internal doc ids). * @param query The query to execute diff --git a/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java b/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java index 582e1caa7ce87..e1ecf998fb045 100644 --- a/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java +++ b/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java @@ -24,6 +24,7 @@ import org.apache.lucene.document.Field.Store; import org.apache.lucene.document.LatLonDocValuesField; import org.apache.lucene.document.LatLonPoint; +import org.apache.lucene.document.LongPoint; import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.SortedSetDocValuesField; import org.apache.lucene.document.StringField; @@ -36,6 +37,7 @@ import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; import org.apache.lucene.queries.MinDocQuery; +import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanClause.Occur; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.Collector; @@ -50,9 +52,11 @@ import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; +import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.Sort; import org.apache.lucene.search.SortField; import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.TotalHitCountCollector; import org.apache.lucene.search.TotalHits; import org.apache.lucene.search.Weight; @@ -65,6 +69,10 @@ import org.apache.lucene.util.FixedBitSet; import org.elasticsearch.action.search.SearchTask; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.index.query.ParsedQuery; import org.elasticsearch.index.search.ESToParentBlockJoinQuery; import org.elasticsearch.index.shard.IndexShard; @@ -84,6 +92,9 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.instanceOf; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.mockito.Mockito.spy; public class QueryPhaseTests extends IndexShardTestCase { @@ -579,7 +590,6 @@ public void testIndexSortScrollOptimization() throws Exception { dir.close(); } - public void testDisableTopScoreCollection() throws Exception { Directory dir = newDirectory(); IndexWriterConfig iwc = newIndexWriterConfig(new StandardAnalyzer()); @@ -630,6 +640,75 @@ public void testDisableTopScoreCollection() throws Exception { dir.close(); } + + public void testNumericLongOrDateSortOptimization() throws Exception { + final String fieldNameLong = "long-field"; + final String fieldNameDate = "date-field"; + MappedFieldType fieldTypeLong = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG); + MappedFieldType fieldTypeDate = new DateFieldMapper.Builder(fieldNameDate).fieldType(); + MapperService mapperService = mock(MapperService.class); + when(mapperService.fullName(fieldNameLong)).thenReturn(fieldTypeLong); + when(mapperService.fullName(fieldNameDate)).thenReturn(fieldTypeDate); + TestSearchContext searchContext = spy(new TestSearchContext(null, indexShard)); + when(searchContext.mapperService()).thenReturn(mapperService); + + final int numDocs = scaledRandomIntBetween(50, 100); + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + for (int i = 0; i < numDocs; ++i) { + Document doc = new Document(); + long longValue = randomLongBetween(-10000000L, 10000000L); + doc.add(new LongPoint(fieldNameLong, longValue)); + doc.add(new NumericDocValuesField(fieldNameLong, longValue)); + longValue = randomLongBetween(0, 3000000000000L); + doc.add(new LongPoint(fieldNameDate, longValue)); + doc.add(new NumericDocValuesField(fieldNameDate, longValue)); + writer.addDocument(doc); + } + writer.close(); + final IndexReader reader = DirectoryReader.open(dir); + IndexSearcher searcher = getAssertingSortOptimizedSearcher(reader, 0); + + // 1. Test a sort on long field + final SortField sortFieldLong = new SortField(fieldNameLong, SortField.Type.LONG); + sortFieldLong.setMissingValue(Long.MAX_VALUE); + final Sort longSort = new Sort(sortFieldLong); + SortAndFormats sortAndFormats = new SortAndFormats(longSort, new DocValueFormat[]{DocValueFormat.RAW}); + searchContext.sort(sortAndFormats); + searchContext.parsedQuery(new ParsedQuery(new MatchAllDocsQuery())); + searchContext.setTask(new SearchTask(123L, "", "", "", null, Collections.emptyMap())); + searchContext.setSize(10); + QueryPhase.execute(searchContext, searcher, checkCancelled -> {}); + assertSortResults(searchContext.queryResult().topDocs().topDocs, (long) numDocs, false); + + // 2. Test a sort on long field + date field + final SortField sortFieldDate = new SortField(fieldNameDate, SortField.Type.LONG); + DocValueFormat dateFormat = fieldTypeDate.docValueFormat(null, null); + final Sort longDateSort = new Sort(sortFieldLong, sortFieldDate); + sortAndFormats = new SortAndFormats(longDateSort, new DocValueFormat[]{DocValueFormat.RAW, dateFormat}); + searchContext.sort(sortAndFormats); + QueryPhase.execute(searchContext, searcher, checkCancelled -> {}); + assertSortResults(searchContext.queryResult().topDocs().topDocs, (long) numDocs, true); + + // 3. Test a sort on date field + sortFieldDate.setMissingValue(Long.MAX_VALUE); + final Sort dateSort = new Sort(sortFieldDate); + sortAndFormats = new SortAndFormats(dateSort, new DocValueFormat[]{dateFormat}); + searchContext.sort(sortAndFormats); + QueryPhase.execute(searchContext, searcher, checkCancelled -> {}); + assertSortResults(searchContext.queryResult().topDocs().topDocs, (long) numDocs, false); + + // 4. Test a sort on date field + long field + final Sort dateLongSort = new Sort(sortFieldDate, sortFieldLong); + sortAndFormats = new SortAndFormats(dateLongSort, new DocValueFormat[]{dateFormat, DocValueFormat.RAW}); + searchContext.sort(sortAndFormats); + QueryPhase.execute(searchContext, searcher, checkCancelled -> {}); + assertSortResults(searchContext.queryResult().topDocs().topDocs, (long) numDocs, true); + reader.close(); + dir.close(); + } + + public void testMaxScoreQueryVisitor() { BitSetProducer producer = context -> new FixedBitSet(1); Query query = new ESToParentBlockJoinQuery(new MatchAllDocsQuery(), producer, ScoreMode.Avg, "nested"); @@ -681,6 +760,83 @@ public void testMaxScoreQueryVisitor() { } } + public void testNumericLongSortOptimizationDocsHaveTheSameValue() throws Exception { + final String fieldNameLong = "long-field"; + MappedFieldType fieldTypeLong = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG); + MapperService mapperService = mock(MapperService.class); + when(mapperService.fullName(fieldNameLong)).thenReturn(fieldTypeLong); + TestSearchContext searchContext = spy(new TestSearchContext(null, indexShard)); + when(searchContext.mapperService()).thenReturn(mapperService); + + final int numDocs = scaledRandomIntBetween(5, 10); + long longValue = randomLongBetween(-10000000L, 10000000L); // all docs have the same value + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + for (int i = 0; i < numDocs; ++i) { + Document doc = new Document(); + doc.add(new LongPoint(fieldNameLong, longValue)); + doc.add(new NumericDocValuesField(fieldNameLong, longValue)); + writer.addDocument(doc); + } + writer.close(); + final IndexReader reader = DirectoryReader.open(dir); + IndexSearcher searcher = getAssertingSortOptimizedSearcher(reader, 1); + + final SortField sortFieldLong = new SortField(fieldNameLong, SortField.Type.LONG); + sortFieldLong.setMissingValue(Long.MAX_VALUE); + final Sort longSort = new Sort(sortFieldLong); + SortAndFormats sortAndFormats = new SortAndFormats(longSort, new DocValueFormat[]{DocValueFormat.RAW}); + searchContext.sort(sortAndFormats); + searchContext.parsedQuery(new ParsedQuery(new MatchAllDocsQuery())); + searchContext.setTask(new SearchTask(123L, "", "", "", null, Collections.emptyMap())); + searchContext.setSize(10); + QueryPhase.execute(searchContext, searcher, checkCancelled -> {}); + assertSortResults(searchContext.queryResult().topDocs().topDocs, (long) numDocs, false); + reader.close(); + dir.close(); + } + + // used to check that numeric long or date sort optimization was run + private static IndexSearcher getAssertingSortOptimizedSearcher(IndexReader reader, int queryType) { + return new IndexSearcher(reader) { + @Override + public void search(Query query, Collector results) throws IOException { + assertTrue(query instanceof BooleanQuery); + List clauses = ((BooleanQuery) query).clauses(); + assertTrue(clauses.size() == 2); + assertTrue(clauses.get(0).getOccur() == Occur.FILTER); + assertTrue(clauses.get(1).getOccur() == Occur.SHOULD); + if (queryType == 0) { + assertTrue (clauses.get(1).getQuery().getClass() == + LongPoint.newDistanceFeatureQuery("random_field", 1, 1, 1).getClass() + ); + } + if (queryType == 1) assertTrue(clauses.get(1).getQuery() instanceof DocValuesFieldExistsQuery); + super.search(query, results); + } + }; + } + + // assert score docs are in order and their number is as expected + private void assertSortResults(TopDocs topDocs, long expectedNumDocs, boolean isDoubleSort) { + assertEquals(topDocs.totalHits.value, expectedNumDocs); + long cur1, cur2; + long prev1 = Long.MIN_VALUE; + long prev2 = Long.MIN_VALUE; + for (ScoreDoc scoreDoc : topDocs.scoreDocs) { + cur1 = (long) ((FieldDoc) scoreDoc).fields[0]; + assertThat(cur1, greaterThanOrEqualTo(prev1)); // test that docs are properly sorted on the first sort + if (isDoubleSort) { + cur2 = (long) ((FieldDoc) scoreDoc).fields[1]; + if (cur1 == prev1) { + assertThat(cur2, greaterThanOrEqualTo(prev2)); // test that docs are properly sorted on the secondary sort + } + prev2 = cur2; + } + prev1 = cur1; + } + } + private static IndexSearcher getAssertingEarlyTerminationSearcher(IndexReader reader, int size) { return new IndexSearcher(reader) { @Override diff --git a/server/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java b/server/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java index d3f21867ab1d1..8b97aed63eb55 100644 --- a/server/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java +++ b/server/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java @@ -80,8 +80,10 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.lessThan; +import static org.hamcrest.Matchers.lessThanOrEqualTo; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.nullValue; @@ -1800,4 +1802,101 @@ public void testCastNumericTypeExceptions() throws Exception { } } } + + public void testLongSortOptimizationCorrectResults() { + assertAcked(prepareCreate("test1") + .setSettings(Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 5)) + .addMapping("_doc", "long_field", "type=long", "int_field", "type=integer", "keyword_field", "type=keyword").get()); + + long currentLong; + long previousLong = 0; + int currentInt; + int previousInt = 0; + // fill data with some equal values + for (int i = 1; i <= 50; i++) { + currentLong = randomBoolean() ? randomLong() : previousLong; + currentInt = randomBoolean() ? randomInt() : previousInt; + String source = "{\"long_field\":" + currentLong + ", \"int_field\":" + currentInt + + ", \"keyword_field\": \"" + randomAlphaOfLength(5) + "\"}"; + client().prepareIndex("test1", "_doc", Integer.toString(i)).setSource(source, XContentType.JSON).get(); + previousLong = currentLong; + previousInt = currentInt; + } + refresh(); + + //*** 1. sort DESC on long_field + SearchResponse searchResponse = client().prepareSearch() + .addSort(new FieldSortBuilder("long_field").order(SortOrder.DESC)) + .setSize(10).get(); + assertSearchResponse(searchResponse); + previousLong = Long.MAX_VALUE; + for (int i = 0; i < searchResponse.getHits().getHits().length; i++) { + // check the correct sort order + SearchHit hit = searchResponse.getHits().getHits()[i]; + currentLong = (long) searchResponse.getHits().getHits()[i].getSourceAsMap().get("long_field"); + assertThat(searchResponse.toString(), currentLong, lessThanOrEqualTo(previousLong)); + + // check that sort values filled correctly + long longSortValue = (long) hit.getSortValues()[0]; + assertEquals(currentLong, longSortValue); + + previousLong = currentLong; + } + + //*** 2. sort ASC on long_field + searchResponse = client().prepareSearch() + .addSort(new FieldSortBuilder("long_field").order(SortOrder.ASC)) + .setSize(10).get(); + assertSearchResponse(searchResponse); + previousLong = Long.MIN_VALUE; + for (int i = 0; i < searchResponse.getHits().getHits().length; i++) { + // check the correct sort order + SearchHit hit = searchResponse.getHits().getHits()[i]; + currentLong = (long) searchResponse.getHits().getHits()[i].getSourceAsMap().get("long_field"); + assertThat(searchResponse.toString(), currentLong, greaterThanOrEqualTo(previousLong)); + + // check that sort values filled correctly + long longSortValue = (long) hit.getSortValues()[0]; + assertEquals(currentLong, longSortValue); + + previousLong = currentLong; + } + + //*** 3. multi sort on long_field, int_field, keyword + searchResponse = client().prepareSearch() + .addSort(new FieldSortBuilder("long_field").order(SortOrder.ASC)) + .addSort(new FieldSortBuilder("int_field").order(SortOrder.ASC)) + .addSort(new FieldSortBuilder("keyword_field").order(SortOrder.ASC)) + .setSize(10).get(); + assertSearchResponse(searchResponse); + previousLong = Long.MIN_VALUE; + previousInt = Integer.MIN_VALUE; + String previousKeyword = ""; + for (int i = 0; i < searchResponse.getHits().getHits().length; i++) { + // check the correct sort order + SearchHit hit = searchResponse.getHits().getHits()[i]; + currentLong = (long) searchResponse.getHits().getHits()[i].getSourceAsMap().get("long_field"); + currentInt = (int) searchResponse.getHits().getHits()[i].getSourceAsMap().get("int_field"); + String currentKeyword = (String) searchResponse.getHits().getHits()[i].getSourceAsMap().get("keyword_field"); + assertThat(searchResponse.toString(), currentLong, greaterThanOrEqualTo(previousLong)); + if (currentLong == previousLong) { + assertThat(searchResponse.toString(), currentInt, greaterThanOrEqualTo(previousInt)); + if (currentInt == previousInt) { + assertThat(searchResponse.toString(), currentKeyword, greaterThanOrEqualTo(previousKeyword)); + } + } + + // check that sort values filled correctly + long longSortValue = (long) hit.getSortValues()[0]; + int intSortValue = ((Long) hit.getSortValues()[1]).intValue(); + String keywordSortValue = (String) hit.getSortValues()[2]; + assertEquals(currentLong, longSortValue); + assertEquals(currentInt, intSortValue); + assertEquals(currentKeyword, keywordSortValue); + + previousLong = currentLong; + previousInt = currentInt; + previousKeyword = currentKeyword; + } + } }