diff --git a/integ-test/src/test/resources/clickbench/mappings/clickbench_index_mapping.json b/integ-test/src/test/resources/clickbench/mappings/clickbench_index_mapping.json index c695dcc0502..4d278868186 100644 --- a/integ-test/src/test/resources/clickbench/mappings/clickbench_index_mapping.json +++ b/integ-test/src/test/resources/clickbench/mappings/clickbench_index_mapping.json @@ -4,7 +4,8 @@ "number_of_shards": 2, "number_of_replicas": 0, "sort.field": [ "CounterID", "EventDate", "UserID", "EventTime", "WatchID" ], - "sort.order": [ "desc", "desc", "desc", "desc", "desc" ] + "sort.order": [ "desc", "desc", "desc", "desc", "desc" ], + "optimized.enabled": true } }, "mappings" : { diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchClient.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchClient.java index 0261bc98120..859a6a06432 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchClient.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchClient.java @@ -54,6 +54,14 @@ public interface OpenSearchClient { */ Map getIndexMaxResultWindows(String... indexExpression); + /** + * Check if index is optimized. + * + * @param indexName index name + * @return true if index is optimized, false otherwise + */ + boolean isIndexOptimized(String indexName); + /** * Perform search query in the search request. * diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java index 152fe499d2e..11d3747cab1 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java @@ -135,6 +135,18 @@ public Map getIndexMaxResultWindows(String... indexExpression) } } + @Override + public boolean isIndexOptimized(String indexName) { + try { + GetSettingsResponse settingsResponse = + client.admin().indices().prepareGetSettings(indexName).setLocal(true).get(); + Settings settings = settingsResponse.getIndexToSettings().get(indexName); + return settings != null && settings.getAsBoolean("index.optimized.enabled", false); + } catch (Exception e) { + return false; + } + } + /** TODO: Scroll doesn't work for aggregation. Support aggregation later. */ @Override public OpenSearchResponse search(OpenSearchRequest request) { diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java index 87b171707bb..c0afabf1db0 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java @@ -261,4 +261,9 @@ public void deletePit(DeletePitRequest deletePitRequest) { "Error occurred while deleting PIT for internal plugin operation", e); } } + + @Override + public boolean isIndexOptimized(String indexName) { + return false; + } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequest.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequest.java index 79b6db64122..00a7e90c130 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequest.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequest.java @@ -77,6 +77,7 @@ import org.opensearch.sql.expression.function.BuiltinFunctionName; import org.opensearch.sql.expression.function.PPLBuiltinOperators; import org.opensearch.sql.expression.function.udf.datetime.ExtractFunction; +import org.opensearch.sql.opensearch.client.OpenSearchClient; import org.opensearch.sql.opensearch.data.value.OpenSearchExprValueFactory; import org.opensearch.sql.opensearch.response.OpenSearchResponse; import org.opensearch.sql.opensearch.storage.OpenSearchIndex; @@ -138,6 +139,10 @@ public class OpenSearchQueryRequest implements OpenSearchRequest { /** List of includes expected in the response. */ @EqualsAndHashCode.Exclude @ToString.Exclude private final List includes; + /** OpenSearch client. */ + @EqualsAndHashCode.Exclude @ToString.Exclude + private final OpenSearchClient client; + @EqualsAndHashCode.Exclude private boolean needClean = true; /** Indicate the search already done. */ @@ -158,6 +163,8 @@ public class OpenSearchQueryRequest implements OpenSearchRequest { private static final Logger LOGGER = LogManager.getLogger(OpenSearchQueryRequest.class); + private final boolean indexOptimized; + /** Constructor of OpenSearchQueryRequest. */ public OpenSearchQueryRequest( String indexName, int size, OpenSearchExprValueFactory factory, List includes) { @@ -175,6 +182,8 @@ public OpenSearchQueryRequest( this.exprValueFactory = factory; this.includes = includes; this.pushedDownRelNodeTree = null; + this.client = null; + this.indexOptimized = false; } /** Constructor of OpenSearchQueryRequest. */ @@ -183,7 +192,7 @@ public OpenSearchQueryRequest( SearchSourceBuilder sourceBuilder, OpenSearchExprValueFactory factory, List includes) { - this(indexName, sourceBuilder, factory, includes, (RelNode) null); + this(indexName, sourceBuilder, factory, includes, (RelNode) null, false); } /** Constructor of OpenSearchQueryRequest with RelNode tree support. */ @@ -193,11 +202,24 @@ public OpenSearchQueryRequest( OpenSearchExprValueFactory factory, List includes, RelNode pushedDownRelNodeTree) { + this(indexName, sourceBuilder, factory, includes, pushedDownRelNodeTree, false); + } + + /** Constructor of OpenSearchQueryRequest with RelNode tree and index optimization support. */ + public OpenSearchQueryRequest( + IndexName indexName, + SearchSourceBuilder sourceBuilder, + OpenSearchExprValueFactory factory, + List includes, + RelNode pushedDownRelNodeTree, + boolean indexOptimized) { this.indexName = indexName; this.sourceBuilder = sourceBuilder; this.exprValueFactory = factory; this.includes = includes; this.pushedDownRelNodeTree = pushedDownRelNodeTree; + this.client = null; + this.indexOptimized = indexOptimized; } /** Constructor of OpenSearchQueryRequest with PIT support. */ @@ -208,7 +230,7 @@ public OpenSearchQueryRequest( List includes, TimeValue cursorKeepAlive, String pitId) { - this(indexName, sourceBuilder, factory, includes, cursorKeepAlive, pitId, null); + this(indexName, sourceBuilder, factory, includes, cursorKeepAlive, pitId, null, false); } /** Constructor of OpenSearchQueryRequest with PIT and RelNode tree support. */ @@ -220,6 +242,19 @@ public OpenSearchQueryRequest( TimeValue cursorKeepAlive, String pitId, RelNode pushedDownRelNodeTree) { + this(indexName, sourceBuilder, factory, includes, cursorKeepAlive, pitId, pushedDownRelNodeTree, false); + } + + /** Constructor of OpenSearchQueryRequest with PIT, RelNode tree and index optimization support. */ + public OpenSearchQueryRequest( + IndexName indexName, + SearchSourceBuilder sourceBuilder, + OpenSearchExprValueFactory factory, + List includes, + TimeValue cursorKeepAlive, + String pitId, + RelNode pushedDownRelNodeTree, + boolean indexOptimized) { this.indexName = indexName; this.sourceBuilder = sourceBuilder; this.exprValueFactory = factory; @@ -227,6 +262,8 @@ public OpenSearchQueryRequest( this.cursorKeepAlive = cursorKeepAlive; this.pitId = pitId; this.pushedDownRelNodeTree = pushedDownRelNodeTree; + this.client = null; + this.indexOptimized = indexOptimized; } /** true if the request is a count aggregation request. */ @@ -272,6 +309,8 @@ public OpenSearchQueryRequest(StreamInput in, OpenSearchStorageEngine engine) th exprValueFactory = new OpenSearchExprValueFactory( index.getFieldOpenSearchTypes(), index.isFieldTypeTolerance()); + client = index.getClient(); + this.indexOptimized = client.isIndexOptimized(indexName.toString()); // RelNode tree is not serialized/deserialized for now // It is only used during the initial query execution @@ -291,8 +330,9 @@ public OpenSearchResponse search( // get the value before set searchDone = true boolean isCountAggRequest = isCountAggRequest(); searchDone = true; - // Convert pushed-down RelNode tree to Substrait bytes - sourceBuilder.queryPlanIR(convertToSubstraitAndSerialize(pushedDownRelNodeTree)); + if (indexOptimized) { + sourceBuilder.queryPlanIR(convertToSubstraitAndSerialize(pushedDownRelNodeTree)); + } return new OpenSearchResponse( searchAction.apply( new SearchRequest().indices(indexName.getIndexNames()).source(sourceBuilder)), @@ -314,8 +354,9 @@ public OpenSearchResponse searchWithPIT(Function SearchHits.empty(), exprValueFactory, includes, isCountAggRequest()); } else { this.sourceBuilder.pointInTimeBuilder(new PointInTimeBuilder(this.pitId)); - // Convert pushed-down RelNode tree to Substrait bytes - sourceBuilder.queryPlanIR(convertToSubstraitAndSerialize(pushedDownRelNodeTree)); + if (indexOptimized) { + sourceBuilder.queryPlanIR(convertToSubstraitAndSerialize(pushedDownRelNodeTree)); + } this.sourceBuilder.timeout(cursorKeepAlive); // check for search after if (searchAfter != null) { diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilder.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilder.java index 703107901ac..077bda15246 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilder.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilder.java @@ -75,6 +75,9 @@ public class OpenSearchRequestBuilder { @ToString.Exclude private final Settings settings; + /** Whether the index is optimized. */ + private boolean indexOptimized = false; + /** RelNode tree for pushed-down operations (optional). * -- SETTER -- * Set the RelNode tree for pushed-down operations. @@ -121,13 +124,14 @@ public OpenSearchRequest build( TimeValue cursorKeepAlive, OpenSearchClient client, boolean isMappingEmpty) { + indexOptimized = client.isIndexOptimized(indexName.toString()); /* Don't use PIT search: * 1. If the size of source is 0. It means this is an aggregation request and no need to use pit. * 2. If mapping is empty. It means no data in the index. PIT search relies on `_id` fields to do sort, thus it will fail if using PIT search in this case. */ if (sourceBuilder.size() == 0 || isMappingEmpty) { return new OpenSearchQueryRequest( - indexName, sourceBuilder, exprValueFactory, List.of(), pushedDownRelNodeTree); + indexName, sourceBuilder, exprValueFactory, List.of(), pushedDownRelNodeTree, indexOptimized); } return buildRequestWithPit(indexName, cursorKeepAlive, client); } @@ -144,13 +148,13 @@ private OpenSearchRequest buildRequestWithPit( // Search with PIT request String pitId = createPit(indexName, cursorKeepAlive, client); return new OpenSearchQueryRequest( - indexName, sourceBuilder, exprValueFactory, includes, cursorKeepAlive, pitId, pushedDownRelNodeTree); + indexName, sourceBuilder, exprValueFactory, includes, cursorKeepAlive, pitId, pushedDownRelNodeTree, indexOptimized); } else { sourceBuilder.from(startFrom); sourceBuilder.size(size); // Search with non-Pit request return new OpenSearchQueryRequest( - indexName, sourceBuilder, exprValueFactory, includes, pushedDownRelNodeTree); + indexName, sourceBuilder, exprValueFactory, includes, pushedDownRelNodeTree, indexOptimized); } } else { if (startFrom != 0) { @@ -160,7 +164,7 @@ private OpenSearchRequest buildRequestWithPit( // Search with PIT request String pitId = createPit(indexName, cursorKeepAlive, client); return new OpenSearchQueryRequest( - indexName, sourceBuilder, exprValueFactory, includes, cursorKeepAlive, pitId, pushedDownRelNodeTree); + indexName, sourceBuilder, exprValueFactory, includes, cursorKeepAlive, pitId, pushedDownRelNodeTree, indexOptimized); } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteEnumerableIndexScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteEnumerableIndexScan.java index 6988890919b..5d82ca2f315 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteEnumerableIndexScan.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteEnumerableIndexScan.java @@ -120,6 +120,7 @@ public Result implement(EnumerableRelImplementor implementor, Prefer pref) { RelNode pushedDownTree = null; try { if (pushDownContext != null && !pushDownContext.isEmpty()) { + // FIXME : Make this optional based on the index setting LOG.info("Full RelNode tree:\n{}", RelOptUtil.toString(CalciteToolsHelper.OpenSearchRelRunners.getCurrentRelNode())); LOG.info("=== PushDownContext contains {} operations ===", pushDownContext.size()); int index = 0;