Merge branch 'main' into deprecate-delete-statement

Signed-off-by: Chen Dai <daichen@amazon.com>

.github/workflows/integ-tests-with-security.yml

@@ -21,7 +21,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        java: [21]
+        java: [21, 23]
     runs-on: ubuntu-latest
     container:
       # using the same image which is used by opensearch-build team to build the OpenSearch Distribution
@@ -62,7 +62,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ windows-latest, macos-13 ]
-        java: [21]
+        java: [21, 23]
 
     runs-on: ${{ matrix.os }}
 

.github/workflows/sql-pitest.yml

@@ -22,6 +22,7 @@ jobs:
       matrix:
         java:
           - 21
+          - 23
     runs-on: ubuntu-latest
     container:
       # using the same image which is used by opensearch-build team to build the OpenSearch Distribution

.github/workflows/sql-test-and-build-workflow.yml

@@ -29,7 +29,7 @@ jobs:
       # Run all jobs
       fail-fast: false
       matrix:
-        java: [21]
+        java: [21, 23]
     runs-on: ubuntu-latest
     container:
       # using the same image which is used by opensearch-build team to build the OpenSearch Distribution
@@ -101,7 +101,9 @@ jobs:
       matrix:
         entry:
           - { os: windows-latest, java: 21, os_build_args: -x doctest -PbuildPlatform=windows }
+          - { os: windows-latest, java: 23, os_build_args: -x doctest -PbuildPlatform=windows }
           - { os: macos-13, java: 21 }
+          - { os: macos-13, java: 23 }
     runs-on: ${{ matrix.entry.os }}
 
     steps:
@@ -161,7 +163,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        java: [21]
+        java: [21, 23]
     container:
       image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
       options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }}

.github/workflows/sql-test-workflow.yml

@@ -22,6 +22,7 @@ jobs:
       matrix:
         java:
           - 21
+          - 23
     runs-on: ubuntu-latest
     container:
       # using the same image which is used by opensearch-build team to build the OpenSearch Distribution

async-query-core/build.gradle

@@ -9,7 +9,7 @@ plugins {
     id 'jacoco'
     id 'antlr'
     id 'com.diffplug.spotless' version '6.22.0'
-    id 'com.github.johnrengelman.shadow'
+    id 'com.gradleup.shadow'
 }
 
 repositories {
@@ -147,4 +147,4 @@ shadowJar {
 
     from sourceSets.main.output
     configurations = [project.configurations.runtimeClasspath]
-}
+}

async-query/src/main/java/org/opensearch/sql/spark/cluster/ClusterManagerEventListener.java

@@ -12,7 +12,6 @@
 import java.time.Duration;
 import java.util.Arrays;
 import java.util.List;
-import org.opensearch.client.Client;
 import org.opensearch.cluster.LocalNodeClusterManagerListener;
 import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.common.lifecycle.LifecycleListener;
@@ -25,6 +24,7 @@
 import org.opensearch.sql.spark.flint.operation.FlintIndexOpFactory;
 import org.opensearch.threadpool.Scheduler.Cancellable;
 import org.opensearch.threadpool.ThreadPool;
+import org.opensearch.transport.client.Client;
 
 public class ClusterManagerEventListener implements LocalNodeClusterManagerListener {
 

async-query/src/main/java/org/opensearch/sql/spark/cluster/IndexCleanup.java

@@ -8,13 +8,13 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.opensearch.action.support.IndicesOptions;
-import org.opensearch.client.Client;
 import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.common.util.concurrent.ThreadContext;
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.index.query.QueryBuilder;
 import org.opensearch.index.reindex.DeleteByQueryAction;
 import org.opensearch.index.reindex.DeleteByQueryRequest;
+import org.opensearch.transport.client.Client;
 
 /** Clean up the old docs for indices. */
 public class IndexCleanup {

async-query/src/main/java/org/opensearch/sql/spark/execution/statestore/StateStore.java

@@ -30,7 +30,6 @@
 import org.opensearch.action.support.WriteRequest;
 import org.opensearch.action.update.UpdateRequest;
 import org.opensearch.action.update.UpdateResponse;
-import org.opensearch.client.Client;
 import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.common.action.ActionFuture;
 import org.opensearch.common.util.concurrent.ThreadContext;
@@ -60,6 +59,7 @@
 import org.opensearch.sql.spark.execution.xcontent.XContentSerializerUtil;
 import org.opensearch.sql.spark.flint.FlintIndexState;
 import org.opensearch.sql.spark.flint.FlintIndexStateModel;
+import org.opensearch.transport.client.Client;
 
 /**
  * State Store maintain the state of Session and Statement. State State create/update/get doc on
@@ -260,7 +260,7 @@ public long count(String indexName, QueryBuilder query) {
       throw new RuntimeException(
           "Fetching job metadata information failed with status : " + searchResponse.status());
     } else {
-      return searchResponse.getHits().getTotalHits().value;
+      return searchResponse.getHits().getTotalHits().value();
     }
   }
 

async-query/src/main/java/org/opensearch/sql/spark/flint/FlintIndexMetadataServiceImpl.java

@@ -22,9 +22,9 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.opensearch.action.admin.indices.mapping.get.GetMappingsResponse;
-import org.opensearch.client.Client;
 import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext;
 import org.opensearch.sql.spark.dispatcher.model.FlintIndexOptions;
+import org.opensearch.transport.client.Client;
 
 /** Implementation of {@link FlintIndexMetadataService} */
 @AllArgsConstructor

async-query/src/main/java/org/opensearch/sql/spark/flint/OpenSearchFlintIndexClient.java

@@ -9,8 +9,8 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.opensearch.action.admin.indices.delete.DeleteIndexRequest;
-import org.opensearch.action.support.master.AcknowledgedResponse;
-import org.opensearch.client.Client;
+import org.opensearch.action.support.clustermanager.AcknowledgedResponse;
+import org.opensearch.transport.client.Client;
 
 @RequiredArgsConstructor
 public class OpenSearchFlintIndexClient implements FlintIndexClient {

async-query/src/main/java/org/opensearch/sql/spark/response/OpenSearchJobExecutionResponseReader.java

@@ -14,7 +14,6 @@
 import org.json.JSONObject;
 import org.opensearch.action.search.SearchRequest;
 import org.opensearch.action.search.SearchResponse;
-import org.opensearch.client.Client;
 import org.opensearch.common.action.ActionFuture;
 import org.opensearch.index.IndexNotFoundException;
 import org.opensearch.index.query.QueryBuilder;
@@ -23,6 +22,7 @@
 import org.opensearch.search.builder.SearchSourceBuilder;
 import org.opensearch.sql.spark.asyncquery.model.AsyncQueryJobMetadata;
 import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext;
+import org.opensearch.transport.client.Client;
 
 /** JobExecutionResponseReader implementation for reading response from OpenSearch index. */
 public class OpenSearchJobExecutionResponseReader implements JobExecutionResponseReader {

async-query/src/main/java/org/opensearch/sql/spark/rest/RestAsyncQueryManagementAction.java

@@ -20,7 +20,6 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.opensearch.OpenSearchException;
-import org.opensearch.client.node.NodeClient;
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.core.rest.RestStatus;
 import org.opensearch.rest.BaseRestHandler;
@@ -48,6 +47,7 @@
 import org.opensearch.sql.spark.transport.model.CreateAsyncQueryActionResponse;
 import org.opensearch.sql.spark.transport.model.GetAsyncQueryResultActionRequest;
 import org.opensearch.sql.spark.transport.model.GetAsyncQueryResultActionResponse;
+import org.opensearch.transport.client.node.NodeClient;
 
 @RequiredArgsConstructor
 public class RestAsyncQueryManagementAction extends BaseRestHandler {

async-query/src/main/java/org/opensearch/sql/spark/scheduler/OpenSearchAsyncQueryScheduler.java

@@ -28,7 +28,6 @@
 import org.opensearch.action.support.WriteRequest;
 import org.opensearch.action.update.UpdateRequest;
 import org.opensearch.action.update.UpdateResponse;
-import org.opensearch.client.Client;
 import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.common.action.ActionFuture;
 import org.opensearch.common.xcontent.XContentType;
@@ -40,6 +39,7 @@
 import org.opensearch.sql.spark.scheduler.job.ScheduledAsyncQueryJobRunner;
 import org.opensearch.sql.spark.scheduler.model.AsyncQuerySchedulerRequest;
 import org.opensearch.sql.spark.scheduler.model.ScheduledAsyncQueryJobRequest;
+import org.opensearch.transport.client.Client;
 
 /** Scheduler class for managing asynchronous query jobs. */
 @RequiredArgsConstructor

async-query/src/main/java/org/opensearch/sql/spark/scheduler/job/ScheduledAsyncQueryJobRunner.java

@@ -7,7 +7,6 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.opensearch.client.Client;
 import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.jobscheduler.spi.JobExecutionContext;
 import org.opensearch.jobscheduler.spi.ScheduledJobParameter;
@@ -20,14 +19,15 @@
 import org.opensearch.sql.spark.rest.model.CreateAsyncQueryResponse;
 import org.opensearch.sql.spark.scheduler.model.ScheduledAsyncQueryJobRequest;
 import org.opensearch.threadpool.ThreadPool;
+import org.opensearch.transport.client.Client;
 
 /**
  * The job runner class for scheduling async query.
  *
  * <p>The job runner should be a singleton class if it uses OpenSearch client or other objects
  * passed from OpenSearch. Because when registering the job runner to JobScheduler plugin,
  * OpenSearch has not invoked plugins' createComponents() method. That is saying the plugin is not
- * completely initialized, and the OpenSearch {@link org.opensearch.client.Client}, {@link
+ * completely initialized, and the OpenSearch {@link org.opensearch.transport.client.Client}, {@link
  * ClusterService} and other objects are not available to plugin and this job runner.
  *
  * <p>So we have to move this job runner initialization to {@link Plugin} createComponents() method,

async-query/src/main/java/org/opensearch/sql/spark/transport/config/AsyncExecutorServiceModule.java

@@ -9,7 +9,6 @@
 
 import com.google.common.collect.ImmutableMap;
 import lombok.RequiredArgsConstructor;
-import org.opensearch.client.node.NodeClient;
 import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.common.inject.AbstractModule;
 import org.opensearch.common.inject.Provides;
@@ -71,6 +70,7 @@
 import org.opensearch.sql.spark.validator.S3GlueSQLGrammarElementValidator;
 import org.opensearch.sql.spark.validator.SQLQueryValidator;
 import org.opensearch.sql.spark.validator.SecurityLakeSQLGrammarElementValidator;
+import org.opensearch.transport.client.node.NodeClient;
 
 @RequiredArgsConstructor
 public class AsyncExecutorServiceModule extends AbstractModule {

async-query/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceSpec.java

@@ -38,7 +38,6 @@
 import org.opensearch.action.admin.indices.delete.DeleteIndexRequest;
 import org.opensearch.action.search.SearchRequest;
 import org.opensearch.action.search.SearchResponse;
-import org.opensearch.client.node.NodeClient;
 import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.common.settings.ClusterSettings;
 import org.opensearch.common.settings.Setting;
@@ -109,6 +108,7 @@
 import org.opensearch.sql.spark.validator.SQLQueryValidator;
 import org.opensearch.sql.storage.DataSourceFactory;
 import org.opensearch.test.OpenSearchIntegTestCase;
+import org.opensearch.transport.client.node.NodeClient;
 
 public class AsyncQueryExecutorServiceSpec extends OpenSearchIntegTestCase {
 

async-query/src/test/java/org/opensearch/sql/spark/asyncquery/model/MockFlintIndex.java

@@ -10,9 +10,9 @@
 import lombok.SneakyThrows;
 import org.opensearch.action.admin.indices.delete.DeleteIndexRequest;
 import org.opensearch.action.admin.indices.mapping.get.GetMappingsResponse;
-import org.opensearch.client.Client;
 import org.opensearch.sql.spark.flint.FlintIndexType;
 import org.opensearch.sql.spark.utils.TestUtils;
+import org.opensearch.transport.client.Client;
 
 @Getter
 public class MockFlintIndex {

async-query/src/test/java/org/opensearch/sql/spark/flint/FlintIndexMetadataServiceImplTest.java

@@ -22,7 +22,6 @@
 import org.mockito.Mock;
 import org.mockito.junit.jupiter.MockitoExtension;
 import org.opensearch.action.admin.indices.mapping.get.GetMappingsResponse;
-import org.opensearch.client.Client;
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.cluster.metadata.MappingMetadata;
 import org.opensearch.common.xcontent.XContentType;
@@ -34,6 +33,7 @@
 import org.opensearch.sql.spark.dispatcher.model.FullyQualifiedTableName;
 import org.opensearch.sql.spark.dispatcher.model.IndexQueryActionType;
 import org.opensearch.sql.spark.dispatcher.model.IndexQueryDetails;
+import org.opensearch.transport.client.Client;
 
 @ExtendWith(MockitoExtension.class)
 public class FlintIndexMetadataServiceImplTest {

async-query/src/test/java/org/opensearch/sql/spark/flint/OpenSearchFlintIndexClientTest.java

@@ -16,8 +16,8 @@
 import org.mockito.Mock;
 import org.mockito.junit.jupiter.MockitoExtension;
 import org.opensearch.action.admin.indices.delete.DeleteIndexRequest;
-import org.opensearch.action.support.master.AcknowledgedResponse;
-import org.opensearch.client.Client;
+import org.opensearch.action.support.clustermanager.AcknowledgedResponse;
+import org.opensearch.transport.client.Client;
 
 @ExtendWith(MockitoExtension.class)
 public class OpenSearchFlintIndexClientTest {

async-query/src/test/java/org/opensearch/sql/spark/response/OpenSearchJobExecutionResponseReaderTest.java

@@ -23,13 +23,13 @@
 import org.mockito.Mockito;
 import org.mockito.junit.jupiter.MockitoExtension;
 import org.opensearch.action.search.SearchResponse;
-import org.opensearch.client.Client;
 import org.opensearch.common.action.ActionFuture;
 import org.opensearch.core.rest.RestStatus;
 import org.opensearch.index.IndexNotFoundException;
 import org.opensearch.search.SearchHit;
 import org.opensearch.search.SearchHits;
 import org.opensearch.sql.spark.asyncquery.model.AsyncQueryJobMetadata;
+import org.opensearch.transport.client.Client;
 
 @ExtendWith(MockitoExtension.class)
 public class OpenSearchJobExecutionResponseReaderTest {

async-query/src/test/java/org/opensearch/sql/spark/rest/RestAsyncQueryManagementActionTest.java

@@ -9,13 +9,13 @@
 import org.mockito.ArgumentCaptor;
 import org.mockito.ArgumentMatchers;
 import org.mockito.Mockito;
-import org.opensearch.client.node.NodeClient;
 import org.opensearch.rest.RestChannel;
 import org.opensearch.rest.RestRequest;
 import org.opensearch.rest.RestResponse;
 import org.opensearch.sql.common.setting.Settings;
 import org.opensearch.sql.opensearch.setting.OpenSearchSettings;
 import org.opensearch.threadpool.ThreadPool;
+import org.opensearch.transport.client.node.NodeClient;
 
 public class RestAsyncQueryManagementActionTest {
 

async-query/src/test/java/org/opensearch/sql/spark/scheduler/OpenSearchAsyncQuerySchedulerTest.java

@@ -36,7 +36,6 @@
 import org.opensearch.action.support.WriteRequest;
 import org.opensearch.action.update.UpdateRequest;
 import org.opensearch.action.update.UpdateResponse;
-import org.opensearch.client.Client;
 import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.common.action.ActionFuture;
 import org.opensearch.index.engine.DocumentMissingException;
@@ -45,6 +44,7 @@
 import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext;
 import org.opensearch.sql.spark.scheduler.model.AsyncQuerySchedulerRequest;
 import org.opensearch.sql.spark.scheduler.model.ScheduledAsyncQueryJobRequest;
+import org.opensearch.transport.client.Client;
 
 public class OpenSearchAsyncQuerySchedulerTest {
 

async-query/src/test/java/org/opensearch/sql/spark/scheduler/job/ScheduledAsyncQueryJobRunnerTest.java

@@ -27,7 +27,6 @@
 import org.mockito.ArgumentCaptor;
 import org.mockito.Mock;
 import org.mockito.MockitoAnnotations;
-import org.opensearch.client.Client;
 import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.jobscheduler.spi.JobExecutionContext;
 import org.opensearch.jobscheduler.spi.ScheduledJobParameter;
@@ -38,6 +37,7 @@
 import org.opensearch.sql.spark.rest.model.LangType;
 import org.opensearch.sql.spark.scheduler.model.ScheduledAsyncQueryJobRequest;
 import org.opensearch.threadpool.ThreadPool;
+import org.opensearch.transport.client.Client;
 
 public class ScheduledAsyncQueryJobRunnerTest {
 

async-query/src/test/java/org/opensearch/sql/spark/transport/config/AsyncExecutorServiceModuleTest.java

@@ -11,14 +11,14 @@
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.mockito.Mock;
 import org.mockito.junit.jupiter.MockitoExtension;
-import org.opensearch.client.node.NodeClient;
 import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.common.inject.Injector;
 import org.opensearch.common.inject.ModulesBuilder;
 import org.opensearch.sql.common.setting.Settings;
 import org.opensearch.sql.datasource.DataSourceService;
 import org.opensearch.sql.legacy.metrics.Metrics;
 import org.opensearch.sql.spark.asyncquery.AsyncQueryExecutorService;
+import org.opensearch.transport.client.node.NodeClient;
 
 @ExtendWith(MockitoExtension.class)
 public class AsyncExecutorServiceModuleTest {

async-query/src/test/java/org/opensearch/sql/spark/utils/TestUtils.java

@@ -10,8 +10,8 @@
 import java.net.URL;
 import lombok.SneakyThrows;
 import org.opensearch.action.admin.indices.create.CreateIndexRequest;
-import org.opensearch.client.Client;
 import org.opensearch.common.xcontent.XContentType;
+import org.opensearch.transport.client.Client;
 
 public class TestUtils {
   @SneakyThrows