Support for pagination in v2 engine of SELECT * FROM <table> queries (

opensearch-project#1666) v2 SQL engine can now paginate simple queries. Pagination is initiated by setting fetch_size property in the request JSON. Pagination is implemented using the OpenSearch Scroll API. Please see pagination-v2.md for implementation details. --------- Signed-off-by: MaxKsyunz <maxk@bitquilltech.com> Signed-off-by: Yury-Fridlyand <yury.fridlyand@improving.com> Signed-off-by: Max Ksyunz <maxk@bitquilltech.com> Co-authored-by: Yury-Fridlyand <yury.fridlyand@improving.com> Co-authored-by: GabeFernandez310 <Gabriel.Fernandez@improving.com> Co-authored-by: Andrew Carbonetto <andrewc@bitquilltech.com> Signed-off-by: Mitchell Gale <Mitchell.Gale@improving.com>
Bit-Quill · Jun 12, 2023 · 630a715 · 630a715
1 parent 7f70a8f
commit 630a715
Show file tree

Hide file tree

Showing 3 changed files with 15 additions and 17 deletions.
diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilder.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilder.java
@@ -14,7 +14,6 @@
 import static org.opensearch.search.sort.SortOrder.ASC;
 
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collection;
 import java.util.List;
 import java.util.Map;
@@ -281,6 +280,10 @@ private List<NestedQueryBuilder> extractNestedQueries(QueryBuilder query) {
     return result;
   }
 
+  public int getMaxResponseSize() {
+    return pageSize == null ? requestedTotalSize : pageSize;
+  }
+
   /**
    * Initialize bool query for push down.
    */

diff --git a/...src/main/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilder.java b/...src/main/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilder.java
@@ -14,18 +14,15 @@
 import java.util.Map;
 import java.util.function.BiFunction;
 import lombok.RequiredArgsConstructor;
-import org.apache.lucene.search.join.ScoreMode;
 import org.opensearch.index.query.BoolQueryBuilder;
 import org.opensearch.index.query.QueryBuilder;
 import org.opensearch.index.query.QueryBuilders;
 import org.opensearch.index.query.ScriptQueryBuilder;
 import org.opensearch.script.Script;
-import org.opensearch.sql.ast.expression.Function;
 import org.opensearch.sql.common.antlr.SyntaxCheckException;
 import org.opensearch.sql.expression.Expression;
 import org.opensearch.sql.expression.ExpressionNodeVisitor;
 import org.opensearch.sql.expression.FunctionExpression;
-import org.opensearch.sql.expression.ReferenceExpression;
 import org.opensearch.sql.expression.function.BuiltinFunctionName;
 import org.opensearch.sql.expression.function.FunctionName;
 import org.opensearch.sql.opensearch.storage.script.filter.lucene.LikeQuery;

diff --git a/...rch/src/test/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilderTest.java b/...rch/src/test/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilderTest.java
@@ -384,22 +384,20 @@ void testPushDownNestedWithNestedFilter() {
         .innerHit(new InnerHitBuilder().setFetchSourceContext(
             new FetchSourceContext(true, new String[]{"message.info"}, null)));
 
-    assertEquals(
-        new SearchSourceBuilder()
-            .query(
-                QueryBuilders.boolQuery().filter(
-                    QueryBuilders.boolQuery()
-                        .must(filterQuery)
-                )
-            )
-            .from(DEFAULT_OFFSET)
-            .size(DEFAULT_LIMIT)
-            .timeout(DEFAULT_QUERY_TIMEOUT),
-        requestBuilder.getSourceBuilder());
+    assertSearchSourceBuilder(new SearchSourceBuilder()
+        .query(
+          QueryBuilders.boolQuery().filter(
+            QueryBuilders.boolQuery()
+              .must(filterQuery)
+        )
+      )
+        .from(DEFAULT_OFFSET)
+        .size(DEFAULT_LIMIT)
+        .timeout(DEFAULT_QUERY_TIMEOUT), requestBuilder);
   }
 
   @Test
-  void testPushTypeMapping() {
+  void test_push_type_mapping() {
     Map<String, OpenSearchDataType> typeMapping = Map.of("intA", OpenSearchDataType.of(INTEGER));
     requestBuilder.pushTypeMapping(typeMapping);