POC implementation

async-query-core/build.gradle

@@ -50,6 +50,8 @@ dependencies {
     implementation 'com.google.code.gson:gson:2.8.9'
     api group: 'com.amazonaws', name: 'aws-java-sdk-emrserverless', version: "${aws_java_sdk_version}"
 
+    api "org.opensearch:common-utils:${common_utils_snapshot_version}@jar"
+
     testImplementation(platform("org.junit:junit-bom:5.9.3"))
 
     testCompileOnly('org.junit.jupiter:junit-jupiter')

async-query-core/src/main/java/org/opensearch/sql/spark/asyncquery/model/AsyncQueryJobMetadata.java

@@ -10,10 +10,10 @@
 import lombok.Data;
 import lombok.EqualsAndHashCode;
 import lombok.experimental.SuperBuilder;
+import org.opensearch.commons.ppl.serde.SerializeUtils;
 import org.opensearch.sql.spark.dispatcher.model.JobType;
 import org.opensearch.sql.spark.execution.statestore.StateModel;
 import org.opensearch.sql.spark.rest.model.LangType;
-import org.opensearch.sql.utils.SerializeUtils;
 
 /** This class models all the metadata required for a job. */
 @Data

async-query-core/src/main/java/org/opensearch/sql/spark/parameter/SparkParameterComposerCollection.java

@@ -11,8 +11,8 @@
 import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.Map;
+import org.opensearch.commons.ppl.serde.DataSourceType;
 import org.opensearch.sql.datasource.model.DataSourceMetadata;
-import org.opensearch.sql.datasource.model.DataSourceType;
 import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext;
 import org.opensearch.sql.spark.dispatcher.model.DispatchQueryRequest;
 

async-query-core/src/main/java/org/opensearch/sql/spark/validator/GrammarElementValidatorProvider.java

@@ -7,7 +7,7 @@
 
 import java.util.Map;
 import lombok.AllArgsConstructor;
-import org.opensearch.sql.datasource.model.DataSourceType;
+import org.opensearch.commons.ppl.serde.DataSourceType;
 
 /** Provides GrammarElementValidator based on DataSourceType. */
 @AllArgsConstructor

async-query-core/src/main/java/org/opensearch/sql/spark/validator/PPLQueryValidator.java

@@ -9,9 +9,9 @@
 import org.antlr.v4.runtime.CommonTokenStream;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.opensearch.commons.ppl.serde.DataSourceType;
 import org.opensearch.sql.common.antlr.CaseInsensitiveCharStream;
 import org.opensearch.sql.common.antlr.SyntaxAnalysisErrorListener;
-import org.opensearch.sql.datasource.model.DataSourceType;
 import org.opensearch.sql.spark.antlr.parser.OpenSearchPPLLexer;
 import org.opensearch.sql.spark.antlr.parser.OpenSearchPPLParser;
 

async-query-core/src/main/java/org/opensearch/sql/spark/validator/SQLQueryValidator.java

@@ -8,7 +8,7 @@
 import lombok.AllArgsConstructor;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.opensearch.sql.datasource.model.DataSourceType;
+import org.opensearch.commons.ppl.serde.DataSourceType;
 import org.opensearch.sql.spark.utils.SQLQueryUtils;
 
 /** Validate input SQL query based on the DataSourceType. */

async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryCoreIntegTest.java

@@ -36,9 +36,9 @@
 import org.mockito.Captor;
 import org.mockito.Mock;
 import org.mockito.junit.jupiter.MockitoExtension;
+import org.opensearch.commons.ppl.serde.DataSourceType;
 import org.opensearch.sql.datasource.DataSourceService;
 import org.opensearch.sql.datasource.model.DataSourceMetadata;
-import org.opensearch.sql.datasource.model.DataSourceType;
 import org.opensearch.sql.spark.asyncquery.model.AsyncQueryExecutionResponse;
 import org.opensearch.sql.spark.asyncquery.model.AsyncQueryJobMetadata;
 import org.opensearch.sql.spark.asyncquery.model.AsyncQueryJobMetadata.AsyncQueryJobMetadataBuilder;

async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/IndexDMLHandlerTest.java

@@ -27,7 +27,6 @@
 import org.mockito.Mockito;
 import org.mockito.junit.jupiter.MockitoExtension;
 import org.opensearch.sql.datasource.model.DataSourceMetadata;
-import org.opensearch.sql.datasource.model.DataSourceType;
 import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext;
 import org.opensearch.sql.spark.config.SparkSubmitParameterModifier;
 import org.opensearch.sql.spark.dispatcher.model.DispatchQueryContext;

async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java

@@ -60,9 +60,9 @@
 import org.mockito.Captor;
 import org.mockito.Mock;
 import org.mockito.junit.jupiter.MockitoExtension;
+import org.opensearch.commons.ppl.serde.DataSourceType;
 import org.opensearch.sql.datasource.DataSourceService;
 import org.opensearch.sql.datasource.model.DataSourceMetadata;
-import org.opensearch.sql.datasource.model.DataSourceType;
 import org.opensearch.sql.spark.asyncquery.model.AsyncQueryJobMetadata;
 import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext;
 import org.opensearch.sql.spark.client.EMRServerlessClient;

async-query-core/src/test/java/org/opensearch/sql/spark/parameter/SparkParameterComposerCollectionTest.java

@@ -15,8 +15,8 @@
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.mockito.Mock;
 import org.mockito.junit.jupiter.MockitoExtension;
+import org.opensearch.commons.ppl.serde.DataSourceType;
 import org.opensearch.sql.datasource.model.DataSourceMetadata;
-import org.opensearch.sql.datasource.model.DataSourceType;
 import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext;
 import org.opensearch.sql.spark.dispatcher.model.DispatchQueryRequest;
 

async-query-core/src/test/java/org/opensearch/sql/spark/parameter/SparkSubmitParametersBuilderTest.java

@@ -21,7 +21,6 @@
 import org.mockito.Mock;
 import org.mockito.junit.jupiter.MockitoExtension;
 import org.opensearch.sql.datasource.model.DataSourceMetadata;
-import org.opensearch.sql.datasource.model.DataSourceType;
 import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext;
 import org.opensearch.sql.spark.config.SparkSubmitParameterModifier;
 import org.opensearch.sql.spark.dispatcher.model.DispatchQueryRequest;

async-query-core/src/test/java/org/opensearch/sql/spark/validator/GrammarElementValidatorProviderTest.java

@@ -9,7 +9,6 @@
 
 import com.google.common.collect.ImmutableMap;
 import org.junit.jupiter.api.Test;
-import org.opensearch.sql.datasource.model.DataSourceType;
 
 class GrammarElementValidatorProviderTest {
   final S3GlueSQLGrammarElementValidator s3GlueSQLGrammarElementValidator =

async-query-core/src/test/java/org/opensearch/sql/spark/validator/PPLQueryValidatorTest.java

@@ -32,7 +32,6 @@
 import org.mockito.Mock;
 import org.mockito.junit.jupiter.MockitoExtension;
 import org.opensearch.sql.common.antlr.CaseInsensitiveCharStream;
-import org.opensearch.sql.datasource.model.DataSourceType;
 import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.SingleStatementContext;
 
 @ExtendWith(MockitoExtension.class)

async-query-core/src/test/java/org/opensearch/sql/spark/validator/SQLQueryValidatorTest.java

@@ -22,7 +22,6 @@
 import org.mockito.Mock;
 import org.mockito.junit.jupiter.MockitoExtension;
 import org.opensearch.sql.common.antlr.CaseInsensitiveCharStream;
-import org.opensearch.sql.datasource.model.DataSourceType;
 import org.opensearch.sql.spark.antlr.parser.SqlBaseLexer;
 import org.opensearch.sql.spark.antlr.parser.SqlBaseParser;
 import org.opensearch.sql.spark.antlr.parser.SqlBaseParser.SingleStatementContext;

async-query/src/main/java/org/opensearch/sql/spark/transport/TransportCreateAsyncQueryRequestAction.java

@@ -12,10 +12,10 @@
 import org.opensearch.action.support.ActionFilters;
 import org.opensearch.action.support.HandledTransportAction;
 import org.opensearch.common.inject.Inject;
+import org.opensearch.commons.ppl.format.JsonResponseFormatter;
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.sql.common.setting.Settings;
 import org.opensearch.sql.opensearch.setting.OpenSearchSettings;
-import org.opensearch.sql.protocol.response.format.JsonResponseFormatter;
 import org.opensearch.sql.spark.asyncquery.AsyncQueryExecutorService;
 import org.opensearch.sql.spark.asyncquery.AsyncQueryExecutorServiceImpl;
 import org.opensearch.sql.spark.asyncquery.model.NullAsyncQueryRequestContext;

async-query/src/main/java/org/opensearch/sql/spark/transport/TransportGetAsyncQueryResultAction.java

@@ -9,10 +9,10 @@
 import org.opensearch.action.support.ActionFilters;
 import org.opensearch.action.support.HandledTransportAction;
 import org.opensearch.common.inject.Inject;
+import org.opensearch.commons.ppl.format.JsonResponseFormatter;
+import org.opensearch.commons.ppl.format.ResponseFormatter;
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.sql.executor.pagination.Cursor;
-import org.opensearch.sql.protocol.response.format.JsonResponseFormatter;
-import org.opensearch.sql.protocol.response.format.ResponseFormatter;
 import org.opensearch.sql.spark.asyncquery.AsyncQueryExecutorService;
 import org.opensearch.sql.spark.asyncquery.AsyncQueryExecutorServiceImpl;
 import org.opensearch.sql.spark.asyncquery.model.AsyncQueryExecutionResponse;

async-query/src/main/java/org/opensearch/sql/spark/transport/config/AsyncExecutorServiceModule.java

@@ -13,9 +13,9 @@
 import org.opensearch.common.inject.AbstractModule;
 import org.opensearch.common.inject.Provides;
 import org.opensearch.common.inject.Singleton;
+import org.opensearch.commons.ppl.serde.DataSourceType;
 import org.opensearch.sql.common.setting.Settings;
 import org.opensearch.sql.datasource.DataSourceService;
-import org.opensearch.sql.datasource.model.DataSourceType;
 import org.opensearch.sql.legacy.metrics.GaugeMetric;
 import org.opensearch.sql.legacy.metrics.Metrics;
 import org.opensearch.sql.spark.asyncquery.AsyncQueryExecutorService;

async-query/src/main/java/org/opensearch/sql/spark/transport/format/AsyncQueryResultResponseFormatter.java

@@ -11,9 +11,9 @@
 import lombok.Builder;
 import lombok.Getter;
 import lombok.RequiredArgsConstructor;
+import org.opensearch.commons.ppl.format.JsonResponseFormatter;
 import org.opensearch.core.common.Strings;
 import org.opensearch.sql.protocol.response.QueryResult;
-import org.opensearch.sql.protocol.response.format.JsonResponseFormatter;
 import org.opensearch.sql.spark.transport.model.AsyncQueryResult;
 
 /**

async-query/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImplSpecTest.java

@@ -23,12 +23,12 @@
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Disabled;
 import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
+import org.opensearch.commons.ppl.serde.DataSourceType;
 import org.opensearch.core.common.Strings;
 import org.opensearch.index.query.QueryBuilders;
 import org.opensearch.sql.common.setting.Settings;
 import org.opensearch.sql.datasource.model.DataSourceMetadata;
 import org.opensearch.sql.datasource.model.DataSourceStatus;
-import org.opensearch.sql.datasource.model.DataSourceType;
 import org.opensearch.sql.datasources.exceptions.DatasourceDisabledException;
 import org.opensearch.sql.spark.asyncquery.model.AsyncQueryExecutionResponse;
 import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext;

async-query/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceSpec.java

@@ -43,12 +43,12 @@
 import org.opensearch.common.settings.Setting;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.xcontent.XContentType;
+import org.opensearch.commons.ppl.serde.DataSourceType;
 import org.opensearch.index.query.QueryBuilder;
 import org.opensearch.plugins.Plugin;
 import org.opensearch.search.builder.SearchSourceBuilder;
 import org.opensearch.sql.common.setting.Settings.Key;
 import org.opensearch.sql.datasource.model.DataSourceMetadata;
-import org.opensearch.sql.datasource.model.DataSourceType;
 import org.opensearch.sql.datasources.encryptor.EncryptorImpl;
 import org.opensearch.sql.datasources.glue.GlueDataSourceFactory;
 import org.opensearch.sql.datasources.service.DataSourceMetadataStorage;

async-query/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryGetResultSpecTest.java

@@ -19,10 +19,10 @@
 import org.junit.Before;
 import org.junit.Test;
 import org.opensearch.action.index.IndexRequest;
+import org.opensearch.commons.ppl.format.JsonResponseFormatter;
+import org.opensearch.commons.ppl.format.ResponseFormatter;
 import org.opensearch.sql.data.model.ExprValue;
 import org.opensearch.sql.executor.pagination.Cursor;
-import org.opensearch.sql.protocol.response.format.JsonResponseFormatter;
-import org.opensearch.sql.protocol.response.format.ResponseFormatter;
 import org.opensearch.sql.spark.asyncquery.model.AsyncQueryExecutionResponse;
 import org.opensearch.sql.spark.asyncquery.model.AsyncQueryJobMetadata;
 import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext;

async-query/src/test/java/org/opensearch/sql/spark/parameter/S3GlueDataSourceSparkParameterComposerTest.java

@@ -20,10 +20,10 @@
 import org.mockito.Mock;
 import org.mockito.Mockito;
 import org.mockito.junit.jupiter.MockitoExtension;
+import org.opensearch.commons.ppl.serde.DataSourceType;
 import org.opensearch.sql.common.setting.Settings.Key;
 import org.opensearch.sql.datasource.model.DataSourceMetadata;
 import org.opensearch.sql.datasource.model.DataSourceStatus;
-import org.opensearch.sql.datasource.model.DataSourceType;
 import org.opensearch.sql.datasources.auth.AuthenticationType;
 import org.opensearch.sql.datasources.glue.GlueDataSourceFactory;
 import org.opensearch.sql.opensearch.setting.OpenSearchSettings;

async-query/src/test/java/org/opensearch/sql/spark/transport/format/AsyncQueryResultResponseFormatterTest.java

@@ -9,7 +9,7 @@
 import static org.opensearch.sql.data.model.ExprValueUtils.tupleValue;
 import static org.opensearch.sql.data.type.ExprCoreType.INTEGER;
 import static org.opensearch.sql.data.type.ExprCoreType.STRING;
-import static org.opensearch.sql.protocol.response.format.JsonResponseFormatter.Style.COMPACT;
+import static org.opensearch.commons.ppl.format.JsonResponseFormatter.Style.COMPACT;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;

build.gradle

@@ -13,6 +13,7 @@ buildscript {
         opensearch_build = version_tokens[0] + '.0'
         prometheus_binary_version = "2.37.2"
         common_utils_version = System.getProperty("common_utils.version", opensearch_build)
+        common_utils_snapshot_version = common_utils_version + '-SNAPSHOT'
         if (buildVersionQualifier) {
             opensearch_build += "-${buildVersionQualifier}"
         }

core/build.gradle

@@ -68,6 +68,8 @@ dependencies {
     annotationProcessor('org.immutables:value:2.8.8')
     compileOnly('org.immutables:value-annotations:2.8.8')
 
+    api "org.opensearch:common-utils:${common_utils_snapshot_version}@jar"
+
     testImplementation('org.junit.jupiter:junit-jupiter:5.9.3')
     testImplementation group: 'org.hamcrest', name: 'hamcrest-library', version: "${hamcrest_version}"
     testImplementation group: 'org.mockito', name: 'mockito-core', version: "${mockito_version}"

core/src/main/java/org/opensearch/sql/datasource/model/DataSource.java

@@ -10,6 +10,7 @@
 import lombok.EqualsAndHashCode;
 import lombok.Getter;
 import lombok.RequiredArgsConstructor;
+import org.opensearch.commons.ppl.serde.DataSourceType;
 import org.opensearch.sql.storage.StorageEngine;
 
 /** Each user configured datasource mapping to one instance of DataSource per JVM. */

core/src/main/java/org/opensearch/sql/datasource/model/DataSourceMetadata.java

@@ -21,6 +21,7 @@
 import lombok.EqualsAndHashCode;
 import lombok.Getter;
 import org.apache.commons.lang3.StringUtils;
+import org.opensearch.commons.ppl.serde.DataSourceType;
 import org.opensearch.sql.datasource.DataSourceService;
 
 @Getter

core/src/main/java/org/opensearch/sql/storage/DataSourceFactory.java

@@ -7,10 +7,10 @@
 
 package org.opensearch.sql.storage;
 
+import org.opensearch.commons.ppl.serde.DataSourceType;
 import org.opensearch.sql.datasource.DataSourceService;
 import org.opensearch.sql.datasource.model.DataSource;
 import org.opensearch.sql.datasource.model.DataSourceMetadata;
-import org.opensearch.sql.datasource.model.DataSourceType;
 
 /**
  * {@link DataSourceFactory} is used to create {@link DataSource} from {@link DataSourceMetadata}.