From bc02815ffde3af801f7685123b032a59f889bb97 Mon Sep 17 00:00:00 2001 From: Vamsi Manohar Date: Tue, 11 Jul 2023 16:21:17 -0700 Subject: [PATCH 1/9] Remov:e Default master key (#1851) Signed-off-by: Vamsi Manohar --- .../datasources/encryptor/EncryptorImpl.java | 17 +++++- .../rest/RestDataSourceQueryAction.java | 3 +- .../encryptor/EncryptorImplTest.java | 58 +++++++++++++++++++ docs/user/ppl/admin/datasources.rst | 10 +++- integ-test/build.gradle | 1 + .../sql/datasource/DataSourceAPIsIT.java | 5 +- .../setting/OpenSearchSettings.java | 1 - .../org/opensearch/sql/plugin/SQLPlugin.java | 12 +++- 8 files changed, 99 insertions(+), 8 deletions(-) diff --git a/datasources/src/main/java/org/opensearch/sql/datasources/encryptor/EncryptorImpl.java b/datasources/src/main/java/org/opensearch/sql/datasources/encryptor/EncryptorImpl.java index 4838cd41a5..18e3e2f257 100644 --- a/datasources/src/main/java/org/opensearch/sql/datasources/encryptor/EncryptorImpl.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/encryptor/EncryptorImpl.java @@ -15,6 +15,7 @@ import java.util.Base64; import javax.crypto.spec.SecretKeySpec; import lombok.RequiredArgsConstructor; +import org.apache.commons.lang3.StringUtils; @RequiredArgsConstructor public class EncryptorImpl implements Encryptor { @@ -23,7 +24,7 @@ public class EncryptorImpl implements Encryptor { @Override public String encrypt(String plainText) { - + validate(masterKey); final AwsCrypto crypto = AwsCrypto.builder() .withCommitmentPolicy(CommitmentPolicy.RequireEncryptRequireDecrypt) .build(); @@ -39,6 +40,7 @@ public String encrypt(String plainText) { @Override public String decrypt(String encryptedText) { + validate(masterKey); final AwsCrypto crypto = AwsCrypto.builder() .withCommitmentPolicy(CommitmentPolicy.RequireEncryptRequireDecrypt) .build(); @@ -52,4 +54,17 @@ public String decrypt(String encryptedText) { return new String(decryptedResult.getResult()); } + private void validate(String masterKey) { + if (StringUtils.isEmpty(masterKey)) { + throw new IllegalStateException( + "Master key is a required config for using create and update datasource APIs." + + "Please set plugins.query.datasources.encryption.masterkey config " + + "in opensearch.yml in all the cluster nodes. " + + "More details can be found here: " + + "https://github.com/opensearch-project/sql/blob/main/docs/user/ppl/" + + "admin/datasources.rst#master-key-config-for-encrypting-credential-information"); + } + } + + } \ No newline at end of file diff --git a/datasources/src/main/java/org/opensearch/sql/datasources/rest/RestDataSourceQueryAction.java b/datasources/src/main/java/org/opensearch/sql/datasources/rest/RestDataSourceQueryAction.java index 95efd2e8f5..6cc5b3bd86 100644 --- a/datasources/src/main/java/org/opensearch/sql/datasources/rest/RestDataSourceQueryAction.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/rest/RestDataSourceQueryAction.java @@ -247,7 +247,8 @@ private void reportError(final RestChannel channel, final Exception e, final Res private static boolean isClientError(Exception e) { return e instanceof NullPointerException // NPE is hard to differentiate but more likely caused by bad query - || e instanceof IllegalArgumentException; + || e instanceof IllegalArgumentException + || e instanceof IllegalStateException; } } \ No newline at end of file diff --git a/datasources/src/test/java/org/opensearch/sql/datasources/encryptor/EncryptorImplTest.java b/datasources/src/test/java/org/opensearch/sql/datasources/encryptor/EncryptorImplTest.java index 22f5b09255..8c9b140f09 100644 --- a/datasources/src/test/java/org/opensearch/sql/datasources/encryptor/EncryptorImplTest.java +++ b/datasources/src/test/java/org/opensearch/sql/datasources/encryptor/EncryptorImplTest.java @@ -84,4 +84,62 @@ public void testDecryptWithDifferentKey() { encryptor2.decrypt(encrypted); }); } + + @Test + public void testEncryptionAndDecryptionWithNullMasterKey() { + String input = "This is a test input"; + Encryptor encryptor = new EncryptorImpl(null); + IllegalStateException illegalStateException + = Assertions.assertThrows(IllegalStateException.class, + () -> encryptor.encrypt(input)); + Assertions.assertEquals("Master key is a required config for using create and" + + " update datasource APIs." + + "Please set plugins.query.datasources.encryption.masterkey config " + + "in opensearch.yml in all the cluster nodes. " + + "More details can be found here: " + + "https://github.com/opensearch-project/sql/blob/main/docs/user/ppl/" + + "admin/datasources.rst#master-key-config-for-encrypting-credential-information", + illegalStateException.getMessage()); + illegalStateException + = Assertions.assertThrows(IllegalStateException.class, + () -> encryptor.decrypt(input)); + Assertions.assertEquals("Master key is a required config for using create and" + + " update datasource APIs." + + "Please set plugins.query.datasources.encryption.masterkey config " + + "in opensearch.yml in all the cluster nodes. " + + "More details can be found here: " + + "https://github.com/opensearch-project/sql/blob/main/docs/user/ppl/" + + "admin/datasources.rst#master-key-config-for-encrypting-credential-information", + illegalStateException.getMessage()); + } + + @Test + public void testEncryptionAndDecryptionWithEmptyMasterKey() { + String masterKey = ""; + String input = "This is a test input"; + Encryptor encryptor = new EncryptorImpl(masterKey); + IllegalStateException illegalStateException + = Assertions.assertThrows(IllegalStateException.class, + () -> encryptor.encrypt(input)); + Assertions.assertEquals("Master key is a required config for using create and" + + " update datasource APIs." + + "Please set plugins.query.datasources.encryption.masterkey config " + + "in opensearch.yml in all the cluster nodes. " + + "More details can be found here: " + + "https://github.com/opensearch-project/sql/blob/main/docs/user/ppl/" + + "admin/datasources.rst#master-key-config-for-encrypting-credential-information", + illegalStateException.getMessage()); + illegalStateException + = Assertions.assertThrows(IllegalStateException.class, + () -> encryptor.decrypt(input)); + Assertions.assertEquals("Master key is a required config for using create and" + + " update datasource APIs." + + "Please set plugins.query.datasources.encryption.masterkey config " + + "in opensearch.yml in all the cluster nodes. " + + "More details can be found here: " + + "https://github.com/opensearch-project/sql/blob/main/docs/user/ppl/" + + "admin/datasources.rst#master-key-config-for-encrypting-credential-information", + illegalStateException.getMessage()); + } + } \ No newline at end of file diff --git a/docs/user/ppl/admin/datasources.rst b/docs/user/ppl/admin/datasources.rst index 6a5871a9e9..0f026f72cf 100644 --- a/docs/user/ppl/admin/datasources.rst +++ b/docs/user/ppl/admin/datasources.rst @@ -121,10 +121,14 @@ Only users mapped with roles having above actions are authorized to execute data Master Key config for encrypting credential information ======================================================== * When users provide credentials for a data source, the system encrypts and securely stores them in the metadata index. System uses "AES/GCM/NoPadding" symmetric encryption algorithm. -* Users can set up a master key to use with this encryption method by configuring the plugins.query.datasources.encryption.masterkey setting in the opensearch.yml file. +* Master key is a required config and users can set this up by configuring the `plugins.query.datasources.encryption.masterkey` setting in the opensearch.yml file. * The master key must be 16, 24, or 32 characters long. -* It's highly recommended that users configure a master key for better security. -* If users don't provide a master key, the system will default to "0000000000000000". +* Sample Bash Script to generate a 24 character master key :: + + #!/bin/bash + # Generate a 24-character key + master_key=$(openssl rand -hex 12) + echo "Master Key: $master_key" * Sample python script to generate a 24 character master key :: import random diff --git a/integ-test/build.gradle b/integ-test/build.gradle index 5bb42e81dd..12cf91ed7f 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -129,6 +129,7 @@ testClusters.all { if (System.getProperty("debugJVM") != null) { jvmArgs '-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005' } + setting "plugins.query.datasources.encryption.masterkey", "1234567812345678" } testClusters.integTest { diff --git a/integ-test/src/test/java/org/opensearch/sql/datasource/DataSourceAPIsIT.java b/integ-test/src/test/java/org/opensearch/sql/datasource/DataSourceAPIsIT.java index c942962fb8..86af85727d 100644 --- a/integ-test/src/test/java/org/opensearch/sql/datasource/DataSourceAPIsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/datasource/DataSourceAPIsIT.java @@ -50,7 +50,10 @@ public void createDataSourceAPITest() { //create datasource DataSourceMetadata createDSM = new DataSourceMetadata("create_prometheus", DataSourceType.PROMETHEUS, - ImmutableList.of(), ImmutableMap.of("prometheus.uri", "https://localhost:9090")); + ImmutableList.of(), ImmutableMap.of("prometheus.uri", "https://localhost:9090", + "prometheus.auth.type","basicauth", + "prometheus.auth.username", "username", + "prometheus.auth.password", "password")); Request createRequest = getCreateDataSourceRequest(createDSM); Response response = client().performRequest(createRequest); Assert.assertEquals(201, response.getStatusLine().getStatusCode()); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java index 01c3aeb30d..0810312974 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java @@ -111,7 +111,6 @@ public class OpenSearchSettings extends Settings { public static final Setting DATASOURCE_MASTER_SECRET_KEY = Setting.simpleString( ENCYRPTION_MASTER_KEY.getKeyValue(), - "0000000000000000", Setting.Property.NodeScope, Setting.Property.Final, Setting.Property.Filtered); diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java b/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java index 7e867be967..3adaf3eac5 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java @@ -15,6 +15,7 @@ import java.util.List; import java.util.Objects; import java.util.function.Supplier; +import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.action.ActionRequest; @@ -90,7 +91,8 @@ public class SQLPlugin extends Plugin implements ActionPlugin, ScriptPlugin { - private static final Logger LOG = LogManager.getLogger(); + private static final Logger LOGGER = LogManager.getLogger(SQLPlugin.class); + private ClusterService clusterService; /** * Settings should be inited when bootstrap the plugin. @@ -212,6 +214,14 @@ public ScriptEngine getScriptEngine(Settings settings, Collection Date: Tue, 11 Jul 2023 17:48:55 -0700 Subject: [PATCH 2/9] Restrict master key config only to integTest cluster (#1856) Signed-off-by: Vamsi Manohar --- integ-test/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integ-test/build.gradle b/integ-test/build.gradle index 12cf91ed7f..95762d079b 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -129,11 +129,11 @@ testClusters.all { if (System.getProperty("debugJVM") != null) { jvmArgs '-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005' } - setting "plugins.query.datasources.encryption.masterkey", "1234567812345678" } testClusters.integTest { plugin ":opensearch-sql-plugin" + setting "plugins.query.datasources.encryption.masterkey", "1234567812345678" } testClusters { From e186cf7d263a87a04c69875e6ad04d1f75e03d6a Mon Sep 17 00:00:00 2001 From: Max Ksyunz Date: Wed, 12 Jul 2023 09:53:35 -0700 Subject: [PATCH 3/9] Add release notes for 2.9 (#1857) Signed-off-by: Max Ksyunz --- .../opensearch-sql.release-notes-2.9.0.0.md | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 release-notes/opensearch-sql.release-notes-2.9.0.0.md diff --git a/release-notes/opensearch-sql.release-notes-2.9.0.0.md b/release-notes/opensearch-sql.release-notes-2.9.0.0.md new file mode 100644 index 0000000000..a374676cf8 --- /dev/null +++ b/release-notes/opensearch-sql.release-notes-2.9.0.0.md @@ -0,0 +1,44 @@ +Compatible with OpenSearch and OpenSearch Dashboards Version 2.9.0 + +### Features + +* Enable Table Function and PromQL function ([#1719](https://github.com/opensearch-project/sql/pull/1719)) +* Add spark connector ([#1780](https://github.com/opensearch-project/sql/pull/1780)) + +### Enhancements + +* Pagination: Support WHERE clause, column list in SELECT clause and for functions and expressions in the query ([#1500](https://github.com/opensearch-project/sql/pull/1500)) +* Pagination: Support ORDER BY clauses and queries without FROM clause ([#1599](https://github.com/opensearch-project/sql/pull/1599)) +* Remove backticks on by field in stats ([#1728](https://github.com/opensearch-project/sql/pull/1728)) +* Support Array and ExprValue Parsing With Inner Hits ([#1737](https://github.com/opensearch-project/sql/pull/1737)) +* Add Support for Nested Function in Order By Clause ([#1789](https://github.com/opensearch-project/sql/pull/1789)) +* Add Support for Field Star in Nested Function ([#1773](https://github.com/opensearch-project/sql/pull/1773)) +* Guarantee datasource read api is strong consistent read (compatibility with segment replication) ([#1815](https://github.com/opensearch-project/sql/pull/1815)) +* Added new datetime functions and aliases to PPL ([#1807](https://github.com/opensearch-project/sql/pull/1807)) +* Support user-defined and incomplete date formats ([#1821](https://github.com/opensearch-project/sql/pull/1821)) +* Add _routing to SQL includes list ([#1771](https://github.com/opensearch-project/sql/pull/1771)) +* Disable read of plugins.query.datasources.encryption.masterkey from cluster settings GET API ([#1825](https://github.com/opensearch-project/sql/pull/1825)) +* Add EMR client to spark connector ([#1790](https://github.com/opensearch-project/sql/pull/1790)) +* Improved error codes in case of data sourcde API security exception ([#1753](https://github.com/opensearch-project/sql/pull/1753)) +* Remove Default master encryption key from settings ([#1851](https://github.com/opensearch-project/sql/pull/1851)) + +### Bug Fixes + +* Fixed bug of byte/short not handling 0 denominator in divide/modulus equations ([#1716](https://github.com/opensearch-project/sql/pull/1716)) +* Fix CSV/RAW output header being application/json rather than plain/text ([#1779](https://github.com/opensearch-project/sql/pull/1779)) + +### Documentation + +* Updated documentation of round function return type ([#1725](https://github.com/opensearch-project/sql/pull/1725)) +* Updated `protocol.rst` with new wording for error message ([#1662](https://github.com/opensearch-project/sql/pull/1662)) + +### Infrastructure + +* stopPrometheus task in doctest build.gradle now runs upon project failure in startOpenSearch ([#1747](https://github.com/opensearch-project/sql/pull/1747)) +* Upgrade guava to 32.0.1 +* Disable CrossClusterSearchIT test ([#1814](https://github.com/opensearch-project/sql/pull/1814)) +* fix flakytest when tests.locale=tr ([#1827](https://github.com/opensearch-project/sql/pull/1827)) + +### Refactoring + +* Simplify OpenSearchIndexScanBuilder ([#1738](https://github.com/opensearch-project/sql/pull/1738)) \ No newline at end of file From dae5a85000cf052b24e06c108a977f949237e2d6 Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Fri, 14 Jul 2023 09:29:42 -0700 Subject: [PATCH 4/9] Fix core refactor: StreamIO from common to core.common (#296) (#1869) * Fix core refactor: StreamIO from common to core.common * Fix core refactor: StreamIO from common to core.common --------- Signed-off-by: acarbonetto --- .../sql/datasources/exceptions/ErrorMessage.java | 2 +- .../model/transport/CreateDataSourceActionRequest.java | 2 +- .../transport/CreateDataSourceActionResponse.java | 4 ++-- .../model/transport/DeleteDataSourceActionRequest.java | 2 +- .../transport/DeleteDataSourceActionResponse.java | 4 ++-- .../model/transport/GetDataSourceActionRequest.java | 2 +- .../model/transport/GetDataSourceActionResponse.java | 4 ++-- .../model/transport/UpdateDataSourceActionRequest.java | 2 +- .../transport/UpdateDataSourceActionResponse.java | 4 ++-- .../datasources/rest/RestDataSourceQueryAction.java | 8 ++++---- .../sql/datasources/utils/XContentParserUtils.java | 2 +- .../OpenSearchDataSourceMetadataStorageTest.java | 4 ++-- .../sql/datasources/utils/XContentParserUtilsTest.java | 2 +- .../org/opensearch/sql/legacy/NestedFieldQueryIT.java | 2 +- .../org/opensearch/sql/legacy/QueryAnalysisIT.java | 8 ++++---- .../test/java/org/opensearch/sql/legacy/QueryIT.java | 2 +- .../org/opensearch/sql/legacy/RestIntegTestCase.java | 2 +- .../org/opensearch/sql/legacy/TermQueryExplainIT.java | 2 +- .../java/org/opensearch/sql/sql/QueryValidationIT.java | 4 ++-- .../sql/legacy/executor/AsyncRestExecutor.java | 2 +- .../legacy/executor/ElasticDefaultRestExecutor.java | 2 +- .../legacy/executor/GetIndexRequestRestListener.java | 2 +- .../sql/legacy/executor/csv/CSVResultRestExecutor.java | 2 +- .../executor/cursor/CursorAsyncRestExecutor.java | 2 +- .../legacy/executor/cursor/CursorCloseExecutor.java | 2 +- .../legacy/executor/cursor/CursorResultExecutor.java | 2 +- .../sql/legacy/executor/format/ErrorMessage.java | 2 +- .../executor/format/PrettyFormatRestExecutor.java | 2 +- .../sql/legacy/executor/join/ElasticJoinExecutor.java | 3 +-- .../sql/legacy/executor/join/ElasticUtils.java | 2 +- .../legacy/executor/join/HashJoinElasticExecutor.java | 1 - .../executor/join/NestedLoopsElasticExecutor.java | 1 - .../sql/legacy/executor/multi/MinusExecutor.java | 1 - .../sql/legacy/executor/multi/UnionExecutor.java | 1 - .../sql/legacy/plugin/RestSQLQueryAction.java | 4 ++-- .../opensearch/sql/legacy/plugin/RestSqlAction.java | 8 ++++---- .../sql/legacy/plugin/RestSqlStatsAction.java | 4 ++-- .../sql/legacy/query/join/JoinRequestBuilder.java | 2 +- .../query/join/NestedLoopsElasticRequestBuilder.java | 2 +- .../opensearch/sql/legacy/query/maker/AggMaker.java | 2 +- .../legacy/query/multi/MultiQueryRequestBuilder.java | 2 +- .../planner/physical/node/scroll/SearchHitRow.java | 1 - .../rewriter/matchtoterm/VerificationException.java | 2 +- .../sql/legacy/esdomain/mapping/FieldMappingTest.java | 2 +- .../planner/physical/node/scroll/SearchHitRowTest.java | 2 +- .../sql/legacy/unittest/ErrorMessageFactoryTest.java | 2 +- .../sql/legacy/unittest/SqlRequestFactoryTest.java | 2 +- .../sql/legacy/unittest/planner/QueryPlannerTest.java | 3 +-- .../sql/opensearch/request/OpenSearchQueryRequest.java | 2 +- .../sql/opensearch/request/OpenSearchRequest.java | 6 +++--- .../opensearch/request/OpenSearchScrollRequest.java | 4 ++-- .../sql/opensearch/response/OpenSearchResponse.java | 2 +- .../sql/opensearch/response/error/ErrorMessage.java | 2 +- .../opensearch/storage/scan/OpenSearchIndexScan.java | 2 +- .../opensearch/request/OpenSearchQueryRequestTest.java | 2 +- .../request/OpenSearchScrollRequestTest.java | 2 +- .../opensearch/response/OpenSearchResponseTest.java | 6 +++--- .../response/error/ErrorMessageFactoryTest.java | 2 +- .../opensearch/response/error/ErrorMessageTest.java | 4 ++-- .../response/error/OpenSearchErrorMessageTest.java | 2 +- .../storage/scan/OpenSearchIndexScanTest.java | 3 +-- .../aggregation/dsl/BucketAggregationBuilderTest.java | 2 +- .../main/java/org/opensearch/sql/plugin/SQLPlugin.java | 2 +- .../opensearch/sql/plugin/rest/RestPPLQueryAction.java | 10 +++++----- .../opensearch/sql/plugin/rest/RestPPLStatsAction.java | 4 ++-- .../sql/plugin/rest/RestQuerySettingsAction.java | 2 +- .../sql/plugin/transport/TransportPPLQueryRequest.java | 8 ++++---- .../plugin/transport/TransportPPLQueryResponse.java | 4 ++-- .../plugin/transport/TransportPPLQueryRequestTest.java | 2 +- .../sql/spark/response/SparkResponseTest.java | 2 +- 70 files changed, 97 insertions(+), 105 deletions(-) diff --git a/datasources/src/main/java/org/opensearch/sql/datasources/exceptions/ErrorMessage.java b/datasources/src/main/java/org/opensearch/sql/datasources/exceptions/ErrorMessage.java index c96640693c..265b3ddf31 100644 --- a/datasources/src/main/java/org/opensearch/sql/datasources/exceptions/ErrorMessage.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/exceptions/ErrorMessage.java @@ -10,7 +10,7 @@ import com.google.gson.GsonBuilder; import com.google.gson.JsonObject; import lombok.Getter; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; /** * Error Message. diff --git a/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/CreateDataSourceActionRequest.java b/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/CreateDataSourceActionRequest.java index 333564c10a..2926f95220 100644 --- a/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/CreateDataSourceActionRequest.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/CreateDataSourceActionRequest.java @@ -14,7 +14,7 @@ import lombok.Getter; import org.opensearch.action.ActionRequest; import org.opensearch.action.ActionRequestValidationException; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.sql.datasource.model.DataSourceMetadata; public class CreateDataSourceActionRequest diff --git a/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/CreateDataSourceActionResponse.java b/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/CreateDataSourceActionResponse.java index 4531c3d9fe..3b997a7d21 100644 --- a/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/CreateDataSourceActionResponse.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/CreateDataSourceActionResponse.java @@ -11,8 +11,8 @@ import lombok.Getter; import lombok.RequiredArgsConstructor; import org.opensearch.action.ActionResponse; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; @RequiredArgsConstructor public class CreateDataSourceActionResponse diff --git a/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/DeleteDataSourceActionRequest.java b/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/DeleteDataSourceActionRequest.java index 6bcbd7a561..bc9d4491d5 100644 --- a/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/DeleteDataSourceActionRequest.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/DeleteDataSourceActionRequest.java @@ -14,7 +14,7 @@ import org.apache.commons.lang3.StringUtils; import org.opensearch.action.ActionRequest; import org.opensearch.action.ActionRequestValidationException; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; public class DeleteDataSourceActionRequest extends ActionRequest { diff --git a/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/DeleteDataSourceActionResponse.java b/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/DeleteDataSourceActionResponse.java index c6847ed9ed..11d4a72274 100644 --- a/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/DeleteDataSourceActionResponse.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/DeleteDataSourceActionResponse.java @@ -11,8 +11,8 @@ import lombok.Getter; import lombok.RequiredArgsConstructor; import org.opensearch.action.ActionResponse; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; @RequiredArgsConstructor public class DeleteDataSourceActionResponse extends ActionResponse { diff --git a/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/GetDataSourceActionRequest.java b/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/GetDataSourceActionRequest.java index 6cafe1972a..1e617f1866 100644 --- a/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/GetDataSourceActionRequest.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/GetDataSourceActionRequest.java @@ -14,7 +14,7 @@ import lombok.NoArgsConstructor; import org.opensearch.action.ActionRequest; import org.opensearch.action.ActionRequestValidationException; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; @NoArgsConstructor public class GetDataSourceActionRequest extends ActionRequest { diff --git a/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/GetDataSourceActionResponse.java b/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/GetDataSourceActionResponse.java index 030493cb51..15437e6a3f 100644 --- a/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/GetDataSourceActionResponse.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/GetDataSourceActionResponse.java @@ -11,8 +11,8 @@ import lombok.Getter; import lombok.RequiredArgsConstructor; import org.opensearch.action.ActionResponse; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; @RequiredArgsConstructor public class GetDataSourceActionResponse extends ActionResponse { diff --git a/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/UpdateDataSourceActionRequest.java b/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/UpdateDataSourceActionRequest.java index fe66483edd..cd5c761d85 100644 --- a/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/UpdateDataSourceActionRequest.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/UpdateDataSourceActionRequest.java @@ -14,7 +14,7 @@ import lombok.Getter; import org.opensearch.action.ActionRequest; import org.opensearch.action.ActionRequestValidationException; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.sql.datasource.model.DataSourceMetadata; public class UpdateDataSourceActionRequest diff --git a/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/UpdateDataSourceActionResponse.java b/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/UpdateDataSourceActionResponse.java index faa3b1139b..8407b9675d 100644 --- a/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/UpdateDataSourceActionResponse.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/UpdateDataSourceActionResponse.java @@ -11,8 +11,8 @@ import lombok.Getter; import lombok.RequiredArgsConstructor; import org.opensearch.action.ActionResponse; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; @RequiredArgsConstructor public class UpdateDataSourceActionResponse diff --git a/datasources/src/main/java/org/opensearch/sql/datasources/rest/RestDataSourceQueryAction.java b/datasources/src/main/java/org/opensearch/sql/datasources/rest/RestDataSourceQueryAction.java index 6cc5b3bd86..aa34488675 100644 --- a/datasources/src/main/java/org/opensearch/sql/datasources/rest/RestDataSourceQueryAction.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/rest/RestDataSourceQueryAction.java @@ -7,13 +7,13 @@ package org.opensearch.sql.datasources.rest; +import static org.opensearch.core.rest.RestStatus.BAD_REQUEST; +import static org.opensearch.core.rest.RestStatus.NOT_FOUND; +import static org.opensearch.core.rest.RestStatus.SERVICE_UNAVAILABLE; import static org.opensearch.rest.RestRequest.Method.DELETE; import static org.opensearch.rest.RestRequest.Method.GET; import static org.opensearch.rest.RestRequest.Method.POST; import static org.opensearch.rest.RestRequest.Method.PUT; -import static org.opensearch.rest.RestStatus.BAD_REQUEST; -import static org.opensearch.rest.RestStatus.NOT_FOUND; -import static org.opensearch.rest.RestStatus.SERVICE_UNAVAILABLE; import com.google.common.collect.ImmutableList; import java.io.IOException; @@ -24,11 +24,11 @@ import org.opensearch.OpenSearchException; import org.opensearch.action.ActionListener; import org.opensearch.client.node.NodeClient; +import org.opensearch.core.rest.RestStatus; import org.opensearch.rest.BaseRestHandler; import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestChannel; import org.opensearch.rest.RestRequest; -import org.opensearch.rest.RestStatus; import org.opensearch.sql.datasource.model.DataSourceMetadata; import org.opensearch.sql.datasources.exceptions.DataSourceNotFoundException; import org.opensearch.sql.datasources.exceptions.ErrorMessage; diff --git a/datasources/src/main/java/org/opensearch/sql/datasources/utils/XContentParserUtils.java b/datasources/src/main/java/org/opensearch/sql/datasources/utils/XContentParserUtils.java index a8643a35f3..38a500afae 100644 --- a/datasources/src/main/java/org/opensearch/sql/datasources/utils/XContentParserUtils.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/utils/XContentParserUtils.java @@ -5,7 +5,7 @@ package org.opensearch.sql.datasources.utils; -import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; import java.io.IOException; import java.util.ArrayList; diff --git a/datasources/src/test/java/org/opensearch/sql/datasources/storage/OpenSearchDataSourceMetadataStorageTest.java b/datasources/src/test/java/org/opensearch/sql/datasources/storage/OpenSearchDataSourceMetadataStorageTest.java index 5a9efaba67..b1554ed585 100644 --- a/datasources/src/test/java/org/opensearch/sql/datasources/storage/OpenSearchDataSourceMetadataStorageTest.java +++ b/datasources/src/test/java/org/opensearch/sql/datasources/storage/OpenSearchDataSourceMetadataStorageTest.java @@ -34,10 +34,10 @@ import org.opensearch.action.update.UpdateResponse; import org.opensearch.client.Client; import org.opensearch.cluster.service.ClusterService; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.core.rest.RestStatus; import org.opensearch.index.engine.DocumentMissingException; import org.opensearch.index.engine.VersionConflictEngineException; -import org.opensearch.index.shard.ShardId; -import org.opensearch.rest.RestStatus; import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; import org.opensearch.sql.datasource.model.DataSourceMetadata; diff --git a/datasources/src/test/java/org/opensearch/sql/datasources/utils/XContentParserUtilsTest.java b/datasources/src/test/java/org/opensearch/sql/datasources/utils/XContentParserUtilsTest.java index 605d641bda..f47d0503e7 100644 --- a/datasources/src/test/java/org/opensearch/sql/datasources/utils/XContentParserUtilsTest.java +++ b/datasources/src/test/java/org/opensearch/sql/datasources/utils/XContentParserUtilsTest.java @@ -11,7 +11,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.junit.jupiter.MockitoExtension; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.sql.datasource.model.DataSourceMetadata; import org.opensearch.sql.datasource.model.DataSourceType; diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/NestedFieldQueryIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/NestedFieldQueryIT.java index 8fee963e96..85b5aab4fe 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/NestedFieldQueryIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/NestedFieldQueryIT.java @@ -36,7 +36,7 @@ import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.search.SearchHit; /** diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/QueryAnalysisIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/QueryAnalysisIT.java index b18ce1c7f9..3a58b7ffc0 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/QueryAnalysisIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/QueryAnalysisIT.java @@ -8,9 +8,9 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; -import static org.opensearch.rest.RestStatus.BAD_REQUEST; -import static org.opensearch.rest.RestStatus.OK; -import static org.opensearch.rest.RestStatus.SERVICE_UNAVAILABLE; +import static org.opensearch.core.rest.RestStatus.BAD_REQUEST; +import static org.opensearch.core.rest.RestStatus.OK; +import static org.opensearch.core.rest.RestStatus.SERVICE_UNAVAILABLE; import java.io.IOException; import org.junit.Assert; @@ -18,7 +18,7 @@ import org.opensearch.client.Request; import org.opensearch.client.Response; import org.opensearch.client.ResponseException; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.sql.legacy.antlr.semantic.SemanticAnalysisException; import org.opensearch.sql.legacy.antlr.syntax.SyntaxAnalysisException; import org.opensearch.sql.legacy.exception.SqlFeatureNotImplementedException; diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/QueryIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/QueryIT.java index eeff107f15..f99285a90b 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/QueryIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/QueryIT.java @@ -38,7 +38,7 @@ import org.junit.Ignore; import org.junit.Test; import org.opensearch.client.ResponseException; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.sql.legacy.utils.StringUtils; public class QueryIT extends SQLIntegTestCase { diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/RestIntegTestCase.java b/integ-test/src/test/java/org/opensearch/sql/legacy/RestIntegTestCase.java index 555d0af0d2..e6f567e2db 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/RestIntegTestCase.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/RestIntegTestCase.java @@ -47,7 +47,7 @@ import org.opensearch.common.Strings; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; /** * SQL plugin integration test base class (migrated from SQLIntegTestCase) diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/TermQueryExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/TermQueryExplainIT.java index 5dc0f0901b..fcc9b048c9 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/TermQueryExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/TermQueryExplainIT.java @@ -15,7 +15,7 @@ import org.junit.Ignore; import org.junit.Test; import org.opensearch.client.ResponseException; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; public class TermQueryExplainIT extends SQLIntegTestCase { diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/QueryValidationIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/QueryValidationIT.java index 62869be168..5a16cd3f64 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/QueryValidationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/QueryValidationIT.java @@ -7,7 +7,7 @@ package org.opensearch.sql.sql; import static org.hamcrest.Matchers.is; -import static org.opensearch.rest.RestStatus.BAD_REQUEST; +import static org.opensearch.core.rest.RestStatus.BAD_REQUEST; import static org.opensearch.sql.legacy.plugin.RestSqlAction.QUERY_API_ENDPOINT; import static org.opensearch.sql.util.MatcherUtils.featureValueOf; @@ -21,7 +21,7 @@ import org.opensearch.client.Request; import org.opensearch.client.RequestOptions; import org.opensearch.client.ResponseException; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.sql.legacy.SQLIntegTestCase; /** diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/AsyncRestExecutor.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/AsyncRestExecutor.java index 4ad6e55777..1df0036bab 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/AsyncRestExecutor.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/AsyncRestExecutor.java @@ -17,7 +17,7 @@ import org.opensearch.common.unit.TimeValue; import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestChannel; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.common.utils.QueryContext; import org.opensearch.sql.legacy.esdomain.LocalClusterState; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/ElasticDefaultRestExecutor.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/ElasticDefaultRestExecutor.java index b5132d7ffb..35ab9facb3 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/ElasticDefaultRestExecutor.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/ElasticDefaultRestExecutor.java @@ -22,7 +22,7 @@ import org.opensearch.index.reindex.DeleteByQueryRequest; import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestChannel; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.rest.action.RestStatusToXContentListener; import org.opensearch.search.SearchHits; import org.opensearch.sql.legacy.exception.SqlParseException; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/GetIndexRequestRestListener.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/GetIndexRequestRestListener.java index 1adb0e745e..591319c74c 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/GetIndexRequestRestListener.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/GetIndexRequestRestListener.java @@ -19,7 +19,7 @@ import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestChannel; import org.opensearch.rest.RestResponse; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.rest.action.RestBuilderListener; import org.opensearch.sql.legacy.antlr.semantic.SemanticAnalysisException; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/csv/CSVResultRestExecutor.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/csv/CSVResultRestExecutor.java index 2f11d2c606..ae7623e3a2 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/csv/CSVResultRestExecutor.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/csv/CSVResultRestExecutor.java @@ -12,7 +12,7 @@ import org.opensearch.client.Client; import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestChannel; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.sql.legacy.executor.QueryActionElasticExecutor; import org.opensearch.sql.legacy.executor.RestExecutor; import org.opensearch.sql.legacy.query.QueryAction; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/cursor/CursorAsyncRestExecutor.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/cursor/CursorAsyncRestExecutor.java index 7bb6421502..92703dde2a 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/cursor/CursorAsyncRestExecutor.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/cursor/CursorAsyncRestExecutor.java @@ -15,7 +15,7 @@ import org.opensearch.common.unit.TimeValue; import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestChannel; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.common.utils.QueryContext; import org.opensearch.sql.legacy.esdomain.LocalClusterState; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/cursor/CursorCloseExecutor.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/cursor/CursorCloseExecutor.java index 901962dbbe..98e89c12e4 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/cursor/CursorCloseExecutor.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/cursor/CursorCloseExecutor.java @@ -6,7 +6,7 @@ package org.opensearch.sql.legacy.executor.cursor; -import static org.opensearch.rest.RestStatus.OK; +import static org.opensearch.core.rest.RestStatus.OK; import java.util.Map; import org.apache.logging.log4j.LogManager; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/cursor/CursorResultExecutor.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/cursor/CursorResultExecutor.java index b7d6da1137..9753f8049c 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/cursor/CursorResultExecutor.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/cursor/CursorResultExecutor.java @@ -6,7 +6,7 @@ package org.opensearch.sql.legacy.executor.cursor; -import static org.opensearch.rest.RestStatus.OK; +import static org.opensearch.core.rest.RestStatus.OK; import java.util.Arrays; import java.util.Map; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/ErrorMessage.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/ErrorMessage.java index 1ba9490916..5297fa38ff 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/ErrorMessage.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/ErrorMessage.java @@ -7,7 +7,7 @@ package org.opensearch.sql.legacy.executor.format; import org.json.JSONObject; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; public class ErrorMessage { diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/PrettyFormatRestExecutor.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/PrettyFormatRestExecutor.java index cf034a3d0b..65fd6b7022 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/PrettyFormatRestExecutor.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/PrettyFormatRestExecutor.java @@ -15,7 +15,7 @@ import org.opensearch.core.common.Strings; import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestChannel; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.sql.legacy.cursor.Cursor; import org.opensearch.sql.legacy.cursor.DefaultCursor; import org.opensearch.sql.legacy.exception.SqlParseException; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/ElasticJoinExecutor.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/ElasticJoinExecutor.java index 3bbfcff1dd..f7d1fbf641 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/ElasticJoinExecutor.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/ElasticJoinExecutor.java @@ -21,13 +21,12 @@ import org.opensearch.action.search.SearchResponse; import org.opensearch.client.Client; import org.opensearch.common.document.DocumentField; -import org.opensearch.common.text.Text; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.mapper.MapperService; import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestChannel; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; import org.opensearch.search.sort.FieldSortBuilder; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/ElasticUtils.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/ElasticUtils.java index 5a91184a09..6e42687fa5 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/ElasticUtils.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/ElasticUtils.java @@ -17,7 +17,7 @@ import org.opensearch.action.search.SearchRequestBuilder; import org.opensearch.action.search.SearchResponse; import org.opensearch.client.Client; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.xcontent.XContentFactory; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/HashJoinElasticExecutor.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/HashJoinElasticExecutor.java index c5511eefc4..5703cf2ef5 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/HashJoinElasticExecutor.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/HashJoinElasticExecutor.java @@ -21,7 +21,6 @@ import org.opensearch.action.search.SearchResponse; import org.opensearch.client.Client; import org.opensearch.common.document.DocumentField; -import org.opensearch.common.text.Text; import org.opensearch.common.unit.TimeValue; import org.opensearch.index.mapper.MapperService; import org.opensearch.index.query.BoolQueryBuilder; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/NestedLoopsElasticExecutor.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/NestedLoopsElasticExecutor.java index cc6971a220..21a9a6054f 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/NestedLoopsElasticExecutor.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/join/NestedLoopsElasticExecutor.java @@ -19,7 +19,6 @@ import org.opensearch.action.search.SearchResponse; import org.opensearch.client.Client; import org.opensearch.common.document.DocumentField; -import org.opensearch.common.text.Text; import org.opensearch.common.unit.TimeValue; import org.opensearch.index.mapper.MapperService; import org.opensearch.search.SearchHit; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/multi/MinusExecutor.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/multi/MinusExecutor.java index 8d37cb0656..83901f1acb 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/multi/MinusExecutor.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/multi/MinusExecutor.java @@ -19,7 +19,6 @@ import org.opensearch.action.search.SearchResponse; import org.opensearch.client.Client; import org.opensearch.common.document.DocumentField; -import org.opensearch.common.text.Text; import org.opensearch.common.unit.TimeValue; import org.opensearch.index.mapper.MapperService; import org.opensearch.search.SearchHit; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/multi/UnionExecutor.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/multi/UnionExecutor.java index fba2b2dcaf..4b4080156d 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/multi/UnionExecutor.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/multi/UnionExecutor.java @@ -16,7 +16,6 @@ import org.opensearch.action.search.SearchResponse; import org.opensearch.client.Client; import org.opensearch.common.document.DocumentField; -import org.opensearch.common.text.Text; import org.opensearch.index.mapper.MapperService; import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/plugin/RestSQLQueryAction.java b/legacy/src/main/java/org/opensearch/sql/legacy/plugin/RestSQLQueryAction.java index bbc1c293be..9ee9a8a683 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/plugin/RestSQLQueryAction.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/plugin/RestSQLQueryAction.java @@ -6,7 +6,7 @@ package org.opensearch.sql.legacy.plugin; -import static org.opensearch.rest.RestStatus.OK; +import static org.opensearch.core.rest.RestStatus.OK; import static org.opensearch.sql.executor.ExecutionEngine.QueryResponse; import static org.opensearch.sql.protocol.response.format.JsonResponseFormatter.Style.PRETTY; @@ -20,7 +20,7 @@ import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestChannel; import org.opensearch.rest.RestRequest; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.sql.common.antlr.SyntaxCheckException; import org.opensearch.sql.common.response.ResponseListener; import org.opensearch.sql.common.utils.QueryContext; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/plugin/RestSqlAction.java b/legacy/src/main/java/org/opensearch/sql/legacy/plugin/RestSqlAction.java index 0408a61342..9a15cc9e21 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/plugin/RestSqlAction.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/plugin/RestSqlAction.java @@ -6,9 +6,9 @@ package org.opensearch.sql.legacy.plugin; -import static org.opensearch.rest.RestStatus.BAD_REQUEST; -import static org.opensearch.rest.RestStatus.OK; -import static org.opensearch.rest.RestStatus.SERVICE_UNAVAILABLE; +import static org.opensearch.core.rest.RestStatus.BAD_REQUEST; +import static org.opensearch.core.rest.RestStatus.OK; +import static org.opensearch.core.rest.RestStatus.SERVICE_UNAVAILABLE; import com.alibaba.druid.sql.parser.ParserException; import com.google.common.collect.ImmutableList; @@ -33,7 +33,7 @@ import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestChannel; import org.opensearch.rest.RestRequest; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.sql.common.antlr.SyntaxCheckException; import org.opensearch.sql.common.utils.QueryContext; import org.opensearch.sql.exception.ExpressionEvaluationException; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/plugin/RestSqlStatsAction.java b/legacy/src/main/java/org/opensearch/sql/legacy/plugin/RestSqlStatsAction.java index 5b48ef6710..d300ea7177 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/plugin/RestSqlStatsAction.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/plugin/RestSqlStatsAction.java @@ -6,7 +6,7 @@ package org.opensearch.sql.legacy.plugin; -import static org.opensearch.rest.RestStatus.SERVICE_UNAVAILABLE; +import static org.opensearch.core.rest.RestStatus.SERVICE_UNAVAILABLE; import com.google.common.collect.ImmutableList; import java.util.Arrays; @@ -21,7 +21,7 @@ import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestController; import org.opensearch.rest.RestRequest; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.sql.common.utils.QueryContext; import org.opensearch.sql.legacy.executor.format.ErrorMessageFactory; import org.opensearch.sql.legacy.metrics.Metrics; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/query/join/JoinRequestBuilder.java b/legacy/src/main/java/org/opensearch/sql/legacy/query/join/JoinRequestBuilder.java index de6cc0cddd..622bd5a75b 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/query/join/JoinRequestBuilder.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/query/join/JoinRequestBuilder.java @@ -12,7 +12,7 @@ import org.opensearch.action.ActionRequestBuilder; import org.opensearch.action.ActionResponse; import org.opensearch.action.search.MultiSearchRequest; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.ToXContent; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/query/join/NestedLoopsElasticRequestBuilder.java b/legacy/src/main/java/org/opensearch/sql/legacy/query/join/NestedLoopsElasticRequestBuilder.java index 6783441931..f89a7eb8e1 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/query/join/NestedLoopsElasticRequestBuilder.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/query/join/NestedLoopsElasticRequestBuilder.java @@ -10,7 +10,7 @@ import java.io.IOException; import org.json.JSONObject; import org.json.JSONStringer; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.ToXContent; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/query/maker/AggMaker.java b/legacy/src/main/java/org/opensearch/sql/legacy/query/maker/AggMaker.java index c3cb6c8016..0c9caab03d 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/query/maker/AggMaker.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/query/maker/AggMaker.java @@ -18,7 +18,7 @@ import java.util.Map; import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; -import org.opensearch.common.ParsingException; +import org.opensearch.core.common.ParsingException; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.common.xcontent.json.JsonXContentParser; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/query/multi/MultiQueryRequestBuilder.java b/legacy/src/main/java/org/opensearch/sql/legacy/query/multi/MultiQueryRequestBuilder.java index 1b20fbe3fb..c9c53acf1f 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/query/multi/MultiQueryRequestBuilder.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/query/multi/MultiQueryRequestBuilder.java @@ -15,7 +15,7 @@ import org.opensearch.action.ActionRequestBuilder; import org.opensearch.action.ActionResponse; import org.opensearch.action.search.SearchRequestBuilder; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.ToXContent; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/scroll/SearchHitRow.java b/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/scroll/SearchHitRow.java index b8cc2bb965..27e3072bab 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/scroll/SearchHitRow.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/scroll/SearchHitRow.java @@ -10,7 +10,6 @@ import java.util.HashMap; import java.util.Map; import org.opensearch.common.document.DocumentField; -import org.opensearch.common.text.Text; import org.opensearch.index.mapper.MapperService; import org.opensearch.search.SearchHit; import org.opensearch.sql.legacy.query.planner.physical.Row; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/rewriter/matchtoterm/VerificationException.java b/legacy/src/main/java/org/opensearch/sql/legacy/rewriter/matchtoterm/VerificationException.java index 29e2a3ff7c..51b936bdc3 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/rewriter/matchtoterm/VerificationException.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/rewriter/matchtoterm/VerificationException.java @@ -7,7 +7,7 @@ package org.opensearch.sql.legacy.rewriter.matchtoterm; import org.opensearch.OpenSearchException; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; public class VerificationException extends OpenSearchException { diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/esdomain/mapping/FieldMappingTest.java b/legacy/src/test/java/org/opensearch/sql/legacy/esdomain/mapping/FieldMappingTest.java index 6821261420..fe8b25ed1c 100644 --- a/legacy/src/test/java/org/opensearch/sql/legacy/esdomain/mapping/FieldMappingTest.java +++ b/legacy/src/test/java/org/opensearch/sql/legacy/esdomain/mapping/FieldMappingTest.java @@ -17,7 +17,7 @@ import java.util.stream.Collectors; import org.hamcrest.Matcher; import org.junit.Test; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.sql.legacy.domain.Field; import org.opensearch.sql.legacy.util.MatcherUtils; diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/query/planner/physical/node/scroll/SearchHitRowTest.java b/legacy/src/test/java/org/opensearch/sql/legacy/query/planner/physical/node/scroll/SearchHitRowTest.java index 6c2f789a47..03976bbbd8 100644 --- a/legacy/src/test/java/org/opensearch/sql/legacy/query/planner/physical/node/scroll/SearchHitRowTest.java +++ b/legacy/src/test/java/org/opensearch/sql/legacy/query/planner/physical/node/scroll/SearchHitRowTest.java @@ -10,7 +10,7 @@ import com.google.common.collect.ImmutableMap; import org.junit.Test; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.search.SearchHit; public class SearchHitRowTest { diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/ErrorMessageFactoryTest.java b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/ErrorMessageFactoryTest.java index 50b69d349b..c4c9504486 100644 --- a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/ErrorMessageFactoryTest.java +++ b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/ErrorMessageFactoryTest.java @@ -9,7 +9,7 @@ import org.junit.Assert; import org.junit.Test; import org.opensearch.OpenSearchException; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.sql.legacy.executor.format.ErrorMessage; import org.opensearch.sql.legacy.executor.format.ErrorMessageFactory; import org.opensearch.sql.legacy.executor.format.OpenSearchErrorMessage; diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/SqlRequestFactoryTest.java b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/SqlRequestFactoryTest.java index efc424789d..f93461724d 100644 --- a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/SqlRequestFactoryTest.java +++ b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/SqlRequestFactoryTest.java @@ -18,7 +18,7 @@ import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.runners.MockitoJUnitRunner; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.rest.RestRequest; import org.opensearch.sql.legacy.esdomain.LocalClusterState; import org.opensearch.sql.legacy.request.PreparedStatementRequest; diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/planner/QueryPlannerTest.java b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/planner/QueryPlannerTest.java index a894ccedc2..775417d669 100644 --- a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/planner/QueryPlannerTest.java +++ b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/planner/QueryPlannerTest.java @@ -39,9 +39,8 @@ import org.opensearch.action.search.SearchScrollRequestBuilder; import org.opensearch.client.Client; import org.opensearch.cluster.ClusterName; -import org.opensearch.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.common.settings.ClusterSettings; -import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequest.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequest.java index 2d035f8a5b..6da34dd678 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequest.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequest.java @@ -17,7 +17,7 @@ import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.SearchResponse; import org.opensearch.action.search.SearchScrollRequest; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.search.SearchHits; import org.opensearch.search.builder.SearchSourceBuilder; import org.opensearch.search.fetch.subphase.FetchSourceContext; diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchRequest.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchRequest.java index cae988ea56..f070ac11db 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchRequest.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchRequest.java @@ -13,10 +13,10 @@ import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.SearchResponse; import org.opensearch.action.search.SearchScrollRequest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; import org.opensearch.common.unit.TimeValue; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.sql.opensearch.data.value.OpenSearchExprValueFactory; import org.opensearch.sql.opensearch.response.OpenSearchResponse; diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchScrollRequest.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchScrollRequest.java index 6a5d001684..9ffcc42ff7 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchScrollRequest.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchScrollRequest.java @@ -19,9 +19,9 @@ import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.SearchResponse; import org.opensearch.action.search.SearchScrollRequest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; import org.opensearch.common.unit.TimeValue; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.search.builder.SearchSourceBuilder; import org.opensearch.sql.opensearch.data.value.OpenSearchExprValueFactory; import org.opensearch.sql.opensearch.response.OpenSearchResponse; diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/OpenSearchResponse.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/OpenSearchResponse.java index 0bbab796be..03abfbf6c1 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/OpenSearchResponse.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/OpenSearchResponse.java @@ -23,7 +23,7 @@ import lombok.EqualsAndHashCode; import lombok.ToString; import org.opensearch.action.search.SearchResponse; -import org.opensearch.common.text.Text; +import org.opensearch.core.common.text.Text; import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; import org.opensearch.search.aggregations.Aggregations; diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/error/ErrorMessage.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/error/ErrorMessage.java index c26c833f67..f828c2c485 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/error/ErrorMessage.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/error/ErrorMessage.java @@ -8,7 +8,7 @@ import lombok.Getter; import org.json.JSONObject; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; /** * Error Message. diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScan.java index e216e1e2fe..2ee863b6db 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScan.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScan.java @@ -14,8 +14,8 @@ import java.util.Iterator; import lombok.EqualsAndHashCode; import lombok.ToString; -import org.opensearch.common.io.stream.BytesStreamInput; import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.common.io.stream.BytesStreamInput; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.exception.NoCursorException; import org.opensearch.sql.executor.pagination.PlanSerializer; diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequestTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequestTest.java index e188bd7c5c..cf548d44f9 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequestTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequestTest.java @@ -28,7 +28,7 @@ import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.SearchResponse; import org.opensearch.action.search.SearchScrollRequest; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.index.query.QueryBuilders; import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/OpenSearchScrollRequestTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/OpenSearchScrollRequestTest.java index 57ba6c201b..63c6a5ca7d 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/OpenSearchScrollRequestTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/OpenSearchScrollRequestTest.java @@ -36,9 +36,9 @@ import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.SearchResponse; import org.opensearch.action.search.SearchScrollRequest; -import org.opensearch.common.io.stream.BytesStreamInput; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.unit.TimeValue; +import org.opensearch.core.common.io.stream.BytesStreamInput; import org.opensearch.index.query.QueryBuilders; import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/response/OpenSearchResponseTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/response/OpenSearchResponseTest.java index 672fca12d7..e77819a453 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/response/OpenSearchResponseTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/response/OpenSearchResponseTest.java @@ -30,9 +30,9 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.action.search.SearchResponse; -import org.opensearch.common.bytes.BytesArray; -import org.opensearch.common.text.Text; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.text.Text; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; import org.opensearch.search.SearchShardTarget; diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/response/error/ErrorMessageFactoryTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/response/error/ErrorMessageFactoryTest.java index ffd1a9d14e..c3ae5d139d 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/response/error/ErrorMessageFactoryTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/response/error/ErrorMessageFactoryTest.java @@ -11,7 +11,7 @@ import org.junit.jupiter.api.Test; import org.opensearch.OpenSearchException; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; public class ErrorMessageFactoryTest { private Throwable nonOpenSearchThrowable = new Throwable(); diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/response/error/ErrorMessageTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/response/error/ErrorMessageTest.java index 0c5e018f88..ac0d46938a 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/response/error/ErrorMessageTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/response/error/ErrorMessageTest.java @@ -7,8 +7,8 @@ package org.opensearch.sql.opensearch.response.error; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.opensearch.rest.RestStatus.BAD_REQUEST; -import static org.opensearch.rest.RestStatus.SERVICE_UNAVAILABLE; +import static org.opensearch.core.rest.RestStatus.BAD_REQUEST; +import static org.opensearch.core.rest.RestStatus.SERVICE_UNAVAILABLE; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/response/error/OpenSearchErrorMessageTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/response/error/OpenSearchErrorMessageTest.java index 91d4a15ef3..3dcb38a558 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/response/error/OpenSearchErrorMessageTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/response/error/OpenSearchErrorMessageTest.java @@ -8,7 +8,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.Mockito.when; -import static org.opensearch.rest.RestStatus.SERVICE_UNAVAILABLE; +import static org.opensearch.core.rest.RestStatus.SERVICE_UNAVAILABLE; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanTest.java index 08590f8021..9e96e3cb6a 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanTest.java @@ -36,8 +36,8 @@ import org.mockito.junit.jupiter.MockitoExtension; import org.mockito.stubbing.Answer; import org.opensearch.action.search.SearchResponse; -import org.opensearch.common.bytes.BytesArray; import org.opensearch.common.unit.TimeValue; +import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryBuilders; import org.opensearch.search.SearchHit; @@ -46,7 +46,6 @@ import org.opensearch.search.fetch.subphase.highlight.HighlightBuilder; import org.opensearch.sql.ast.expression.DataType; import org.opensearch.sql.ast.expression.Literal; -import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.data.model.ExprValueUtils; import org.opensearch.sql.exception.NoCursorException; diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/BucketAggregationBuilderTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/BucketAggregationBuilderTest.java index f93c69de28..521f93f2e7 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/BucketAggregationBuilderTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/BucketAggregationBuilderTest.java @@ -34,9 +34,9 @@ import org.junit.jupiter.params.provider.ValueSource; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import org.opensearch.common.bytes.BytesReference; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.aggregations.bucket.composite.CompositeValuesSourceBuilder; import org.opensearch.search.aggregations.bucket.missing.MissingOrder; diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java b/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java index 3adaf3eac5..12d6358c7a 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java @@ -28,13 +28,13 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Injector; import org.opensearch.common.inject.ModulesBuilder; -import org.opensearch.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.IndexScopedSettings; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.common.settings.SettingsFilter; import org.opensearch.common.util.concurrent.OpenSearchExecutors; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.env.Environment; import org.opensearch.env.NodeEnvironment; diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestPPLQueryAction.java b/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestPPLQueryAction.java index 0f4d2f7d0c..8b5074a5aa 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestPPLQueryAction.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestPPLQueryAction.java @@ -5,10 +5,10 @@ package org.opensearch.sql.plugin.rest; -import static org.opensearch.rest.RestStatus.BAD_REQUEST; -import static org.opensearch.rest.RestStatus.INTERNAL_SERVER_ERROR; -import static org.opensearch.rest.RestStatus.OK; -import static org.opensearch.rest.RestStatus.SERVICE_UNAVAILABLE; +import static org.opensearch.core.rest.RestStatus.BAD_REQUEST; +import static org.opensearch.core.rest.RestStatus.INTERNAL_SERVER_ERROR; +import static org.opensearch.core.rest.RestStatus.OK; +import static org.opensearch.core.rest.RestStatus.SERVICE_UNAVAILABLE; import com.google.common.collect.ImmutableList; import java.util.Arrays; @@ -20,12 +20,12 @@ import org.apache.logging.log4j.Logger; import org.opensearch.action.ActionListener; import org.opensearch.client.node.NodeClient; +import org.opensearch.core.rest.RestStatus; import org.opensearch.index.IndexNotFoundException; import org.opensearch.rest.BaseRestHandler; import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestChannel; import org.opensearch.rest.RestRequest; -import org.opensearch.rest.RestStatus; import org.opensearch.sql.common.antlr.SyntaxCheckException; import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.exception.ExpressionEvaluationException; diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestPPLStatsAction.java b/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestPPLStatsAction.java index 5b9c792c7d..ef9f68a2a7 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestPPLStatsAction.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestPPLStatsAction.java @@ -6,7 +6,7 @@ package org.opensearch.sql.plugin.rest; -import static org.opensearch.rest.RestStatus.SERVICE_UNAVAILABLE; +import static org.opensearch.core.rest.RestStatus.SERVICE_UNAVAILABLE; import com.google.common.collect.ImmutableList; import java.util.Arrays; @@ -17,11 +17,11 @@ import org.apache.logging.log4j.Logger; import org.opensearch.client.node.NodeClient; import org.opensearch.common.settings.Settings; +import org.opensearch.core.rest.RestStatus; import org.opensearch.rest.BaseRestHandler; import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestController; import org.opensearch.rest.RestRequest; -import org.opensearch.rest.RestStatus; import org.opensearch.sql.common.utils.QueryContext; import org.opensearch.sql.legacy.executor.format.ErrorMessageFactory; import org.opensearch.sql.legacy.metrics.Metrics; diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestQuerySettingsAction.java b/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestQuerySettingsAction.java index 6249cfba6c..9022817106 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestQuerySettingsAction.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestQuerySettingsAction.java @@ -5,7 +5,7 @@ package org.opensearch.sql.plugin.rest; -import static org.opensearch.rest.RestStatus.INTERNAL_SERVER_ERROR; +import static org.opensearch.core.rest.RestStatus.INTERNAL_SERVER_ERROR; import com.google.common.collect.ImmutableList; import java.io.IOException; diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryRequest.java b/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryRequest.java index 5ced07c02e..8cdf27ef3c 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryRequest.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryRequest.java @@ -17,10 +17,10 @@ import org.json.JSONObject; import org.opensearch.action.ActionRequest; import org.opensearch.action.ActionRequestValidationException; -import org.opensearch.common.io.stream.InputStreamStreamInput; -import org.opensearch.common.io.stream.OutputStreamStreamOutput; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.InputStreamStreamInput; +import org.opensearch.core.common.io.stream.OutputStreamStreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.sql.ppl.domain.PPLQueryRequest; import org.opensearch.sql.protocol.response.format.Format; import org.opensearch.sql.protocol.response.format.JsonResponseFormatter; diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryResponse.java b/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryResponse.java index bfd4994b07..4f26686f30 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryResponse.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryResponse.java @@ -9,8 +9,8 @@ import lombok.Getter; import lombok.RequiredArgsConstructor; import org.opensearch.action.ActionResponse; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; @RequiredArgsConstructor public class TransportPPLQueryResponse extends ActionResponse { diff --git a/plugin/src/test/java/org/opensearch/sql/plugin/transport/TransportPPLQueryRequestTest.java b/plugin/src/test/java/org/opensearch/sql/plugin/transport/TransportPPLQueryRequestTest.java index 6e086f4d4c..0e5d99ae35 100644 --- a/plugin/src/test/java/org/opensearch/sql/plugin/transport/TransportPPLQueryRequestTest.java +++ b/plugin/src/test/java/org/opensearch/sql/plugin/transport/TransportPPLQueryRequestTest.java @@ -16,7 +16,7 @@ import org.junit.rules.ExpectedException; import org.opensearch.action.ActionRequest; import org.opensearch.action.ActionRequestValidationException; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamOutput; public class TransportPPLQueryRequestTest { diff --git a/spark/src/test/java/org/opensearch/sql/spark/response/SparkResponseTest.java b/spark/src/test/java/org/opensearch/sql/spark/response/SparkResponseTest.java index 20210ea7e5..f6bc23b827 100644 --- a/spark/src/test/java/org/opensearch/sql/spark/response/SparkResponseTest.java +++ b/spark/src/test/java/org/opensearch/sql/spark/response/SparkResponseTest.java @@ -26,7 +26,7 @@ import org.opensearch.action.delete.DeleteResponse; import org.opensearch.action.search.SearchResponse; import org.opensearch.client.Client; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; From 8c8e08c248f229e5bb96d8e5882714f68c4abebe Mon Sep 17 00:00:00 2001 From: Matthew Wells Date: Fri, 14 Jul 2023 10:05:48 -0700 Subject: [PATCH 5/9] Added new datetime functions and aliases to PPL [Part 2] (#283) (#1852) * Added new datetime functions, documentation, and tests Signed-off-by: Matthew Wells --- docs/user/ppl/functions/datetime.rst | 289 +++++++++++++++++- .../sql/ppl/DateTimeFunctionIT.java | 81 +++++ ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 10 + ppl/src/main/antlr/OpenSearchPPLParser.g4 | 64 ++++ .../sql/ppl/parser/AstExpressionBuilder.java | 54 ++++ .../sql/ppl/antlr/PPLSyntaxParserTest.java | 61 +++- .../ppl/parser/AstExpressionBuilderTest.java | 60 ++++ 7 files changed, 615 insertions(+), 4 deletions(-) diff --git a/docs/user/ppl/functions/datetime.rst b/docs/user/ppl/functions/datetime.rst index fccfefca6b..8f844c75a2 100644 --- a/docs/user/ppl/functions/datetime.rst +++ b/docs/user/ppl/functions/datetime.rst @@ -813,6 +813,77 @@ Example:: +-----------------------------------+ +EXTRACT +------- + +Description +>>>>>>>>>>> + +Usage: extract(part FROM date) returns a LONG with digits in order according to the given 'part' arguments. +The specific format of the returned long is determined by the table below. + +Argument type: PART, where PART is one of the following tokens in the table below. + +The format specifiers found in this table are the same as those found in the `DATE_FORMAT`_ function. + +.. list-table:: The following table describes the mapping of a 'part' to a particular format. + :widths: 20 80 + :header-rows: 1 + + * - Part + - Format + * - MICROSECOND + - %f + * - SECOND + - %s + * - MINUTE + - %i + * - HOUR + - %H + * - DAY + - %d + * - WEEK + - %X + * - MONTH + - %m + * - YEAR + - %V + * - SECOND_MICROSECOND + - %s%f + * - MINUTE_MICROSECOND + - %i%s%f + * - MINUTE_SECOND + - %i%s + * - HOUR_MICROSECOND + - %H%i%s%f + * - HOUR_SECOND + - %H%i%s + * - HOUR_MINUTE + - %H%i + * - DAY_MICROSECOND + - %d%H%i%s%f + * - DAY_SECOND + - %d%H%i%s + * - DAY_MINUTE + - %d%H%i + * - DAY_HOUR + - %d%H% + * - YEAR_MONTH + - %V%m + +Return type: LONG + +Example:: + + os> source=people | eval `extract(YEAR_MONTH FROM "2023-02-07 10:11:12")` = extract(YEAR_MONTH FROM "2023-02-07 10:11:12") | fields `extract(YEAR_MONTH FROM "2023-02-07 10:11:12")` + fetched rows / total rows = 1/1 + +--------------------------------------------------+ + | extract(YEAR_MONTH FROM "2023-02-07 10:11:12") | + |--------------------------------------------------| + | 202302 | + +--------------------------------------------------+ + + FROM_DAYS --------- @@ -872,6 +943,28 @@ Examples:: +-----------------------------------+ +GET_FORMAT +---------- + +Description +>>>>>>>>>>> + +Usage: Returns a string value containing string format specifiers based on the input arguments. + +Argument type: TYPE, STRING, where TYPE must be one of the following tokens: [DATE, TIME, DATETIME, TIMESTAMP], and +STRING must be one of the following tokens: ["USA", "JIS", "ISO", "EUR", "INTERNAL"] (" can be replaced by '). + +Examples:: + + os> source=people | eval `GET_FORMAT(DATE, 'USA')` = GET_FORMAT(DATE, 'USA') | fields `GET_FORMAT(DATE, 'USA')` + fetched rows / total rows = 1/1 + +---------------------------+ + | GET_FORMAT(DATE, 'USA') | + |---------------------------| + | %m.%d.%Y | + +---------------------------+ + + HOUR ---- @@ -922,6 +1015,26 @@ Example:: +---------------------------------+ +LAST_DAY +-------- + +Usage: Returns the last day of the month as a DATE for a valid argument. + +Argument type: DATE/DATETIME/STRING/TIMESTAMP/TIME + +Return type: DATE + +Example:: + + os> source=people | eval `last_day('2023-02-06')` = last_day('2023-02-06') | fields `last_day('2023-02-06')` + fetched rows / total rows = 1/1 + +--------------------------+ + | last_day('2023-02-06') | + |--------------------------| + | 2023-02-28 | + +--------------------------+ + + LOCALTIMESTAMP -------------- @@ -1136,7 +1249,6 @@ Return type: INTEGER Synonyms: `MONTH_OF_YEAR`_ - Example:: os> source=people | eval `MONTH(DATE('2020-08-26'))` = MONTH(DATE('2020-08-26')) | fields `MONTH(DATE('2020-08-26'))` @@ -1289,6 +1401,32 @@ Example:: +-------------------------------+ +SEC_TO_TIME +----------- + +Description +>>>>>>>>>>> + +Usage: sec_to_time(number) returns the time in HH:mm:ssss[.nnnnnn] format. +Note that the function returns a time between 00:00:00 and 23:59:59. +If an input value is too large (greater than 86399), the function will wrap around and begin returning outputs starting from 00:00:00. +If an input value is too small (less than 0), the function will wrap around and begin returning outputs counting down from 23:59:59. + +Argument type: INTEGER, LONG, DOUBLE, FLOAT + +Return type: TIME + +Example:: + + os> source=people | eval `SEC_TO_TIME(3601)` = SEC_TO_TIME(3601) | eval `SEC_TO_TIME(1234.123)` = SEC_TO_TIME(1234.123) | fields `SEC_TO_TIME(3601)`, `SEC_TO_TIME(1234.123)` + fetched rows / total rows = 1/1 + +---------------------+-------------------------+ + | SEC_TO_TIME(3601) | SEC_TO_TIME(1234.123) | + |---------------------+-------------------------| + | 01:00:01 | 00:20:34.123 | + +---------------------+-------------------------+ + + SECOND ------ @@ -1339,6 +1477,32 @@ Example:: +--------------------------------------+ +STR_TO_DATE +----------- + +Description +>>>>>>>>>>> + +Usage: str_to_date(string, string) is used to extract a DATETIME from the first argument string using the formats specified in the second argument string. +The input argument must have enough information to be parsed as a DATE, DATETIME, or TIME. +Acceptable string format specifiers are the same as those used in the `DATE_FORMAT`_ function. +It returns NULL when a statement cannot be parsed due to an invalid pair of arguments, and when 0 is provided for any DATE field. Otherwise, it will return a DATETIME with the parsed values (as well as default values for any field that was not parsed). + +Argument type: STRING, STRING + +Return type: DATETIME + +Example:: + + OS> source=people | eval `str_to_date("01,5,2013", "%d,%m,%Y")` = str_to_date("01,5,2013", "%d,%m,%Y") | fields = `str_to_date("01,5,2013", "%d,%m,%Y")` + fetched rows / total rows = 1/1 + +----------------------------------------+ + | str_to_date("01,5,2013", "%d,%m,%Y") | + |----------------------------------------| + | 2013-05-01 00:00:00 | + +----------------------------------------+ + + SUBDATE ------- @@ -1637,6 +1801,57 @@ Example:: +------------------------------------+------------------------------------------------------+ +TIMESTAMPADD +------------ + +Description +>>>>>>>>>>> + +Usage: Returns a DATETIME value based on a passed in DATE/DATETIME/TIME/TIMESTAMP/STRING argument and an INTERVAL and INTEGER argument which determine the amount of time to be added. +If the third argument is a STRING, it must be formatted as a valid DATETIME. If only a TIME is provided, a DATETIME is still returned with the DATE portion filled in using the current date. +If the third argument is a DATE, it will be automatically converted to a DATETIME. + +Argument type: INTERVAL, INTEGER, DATE/DATETIME/TIME/TIMESTAMP/STRING + +INTERVAL must be one of the following tokens: [MICROSECOND, SECOND, MINUTE, HOUR, DAY, WEEK, MONTH, QUARTER, YEAR] + +Examples:: + + os> source=people | eval `TIMESTAMPADD(DAY, 17, '2000-01-01 00:00:00')` = TIMESTAMPADD(DAY, 17, '2000-01-01 00:00:00') | eval `TIMESTAMPADD(QUARTER, -1, '2000-01-01 00:00:00')` = TIMESTAMPADD(QUARTER, -1, '2000-01-01 00:00:00') | fields `TIMESTAMPADD(DAY, 17, '2000-01-01 00:00:00')`, `TIMESTAMPADD(QUARTER, -1, '2000-01-01 00:00:00')` + fetched rows / total rows = 1/1 + +------------------------------------------------+----------------------------------------------------+ + | TIMESTAMPADD(DAY, 17, '2000-01-01 00:00:00') | TIMESTAMPADD(QUARTER, -1, '2000-01-01 00:00:00') | + |------------------------------------------------+----------------------------------------------------| + | 2000-01-18 00:00:00 | 1999-10-01 00:00:00 | + +------------------------------------------------+----------------------------------------------------+ + + +TIMESTAMPDIFF +------------- + +Description +>>>>>>>>>>> + +Usage: TIMESTAMPDIFF(interval, start, end) returns the difference between the start and end date/times in interval units. +If a TIME is provided as an argument, it will be converted to a DATETIME with the DATE portion filled in using the current date. +Arguments will be automatically converted to a DATETIME/TIME/TIMESTAMP when appropriate. +Any argument that is a STRING must be formatted as a valid DATETIME. + +Argument type: INTERVAL, DATE/DATETIME/TIME/TIMESTAMP/STRING, DATE/DATETIME/TIME/TIMESTAMP/STRING + +INTERVAL must be one of the following tokens: [MICROSECOND, SECOND, MINUTE, HOUR, DAY, WEEK, MONTH, QUARTER, YEAR] + +Examples:: + + os> source=people | eval `TIMESTAMPDIFF(YEAR, '1997-01-01 00:00:00', '2001-03-06 00:00:00')` = TIMESTAMPDIFF(YEAR, '1997-01-01 00:00:00', '2001-03-06 00:00:00') | eval `TIMESTAMPDIFF(SECOND, time('00:00:23'), time('00:00:00'))` = TIMESTAMPDIFF(SECOND, time('00:00:23'), time('00:00:00')) | fields `TIMESTAMPDIFF(YEAR, '1997-01-01 00:00:00', '2001-03-06 00:00:00')`, `TIMESTAMPDIFF(SECOND, time('00:00:23'), time('00:00:00'))` + fetched rows / total rows = 1/1 + +---------------------------------------------------------------------+-------------------------------------------------------------+ + | TIMESTAMPDIFF(YEAR, '1997-01-01 00:00:00', '2001-03-06 00:00:00') | TIMESTAMPDIFF(SECOND, time('00:00:23'), time('00:00:00')) | + |---------------------------------------------------------------------+-------------------------------------------------------------| + | 4 | -23 | + +---------------------------------------------------------------------+-------------------------------------------------------------+ + + TO_DAYS ------- @@ -1660,6 +1875,30 @@ Example:: +-------------------------------+ +TO_SECONDS +---------- + +Description +>>>>>>>>>>> + +Usage: to_seconds(date) returns the number of seconds since the year 0 of the given value. Returns NULL if value is invalid. +An argument of a LONG type can be used. It must be formatted as YMMDD, YYMMDD, YYYMMDD or YYYYMMDD. Note that a LONG type argument cannot have leading 0s as it will be parsed using an octal numbering system. + +Argument type: STRING/LONG/DATE/DATETIME/TIME/TIMESTAMP + +Return type: LONG + +Example:: + + os> source=people | eval `TO_SECONDS(DATE('2008-10-07'))` = TO_SECONDS(DATE('2008-10-07')) | eval `TO_SECONDS(950228)` = TO_SECONDS(950228) | fields `TO_SECONDS(DATE('2008-10-07'))`, `TO_SECONDS(950228)` + fetched rows / total rows = 1/1 + +----------------------------------+----------------------+ + | TO_SECONDS(DATE('2008-10-07')) | TO_SECONDS(950228) | + |----------------------------------+----------------------| + | 63390556800 | 62961148800 | + +----------------------------------+----------------------+ + + UNIX_TIMESTAMP -------------- @@ -1821,6 +2060,31 @@ Example:: +----------------------------+-------------------------------+ +WEEKDAY +------- + +Description +>>>>>>>>>>> + +Usage: weekday(date) returns the weekday index for date (0 = Monday, 1 = Tuesday, ..., 6 = Sunday). + +It is similar to the `dayofweek`_ function, but returns different indexes for each day. + +Argument type: STRING/DATE/DATETIME/TIME/TIMESTAMP + +Return type: INTEGER + +Example:: + + os> source=people | eval `weekday(DATE('2020-08-26'))` = weekday(DATE('2020-08-26')) | eval `weekday(DATE('2020-08-27'))` = weekday(DATE('2020-08-27')) | fields `weekday(DATE('2020-08-26'))`, `weekday(DATE('2020-08-27'))` + fetched rows / total rows = 1/1 + +-------------------------------+-------------------------------+ + | weekday(DATE('2020-08-26')) | weekday(DATE('2020-08-27')) | + |-------------------------------+-------------------------------| + | 2 | 3 | + +-------------------------------+-------------------------------+ + + WEEK_OF_YEAR ------------ @@ -1910,3 +2174,26 @@ Example:: +----------------------------+ +YEARWEEK +-------- + +Description +>>>>>>>>>>> + +Usage: yearweek(date) returns the year and week for date as an integer. It accepts and optional mode arguments aligned with those available for the `WEEK`_ function. + +Argument type: STRING/DATE/DATETIME/TIME/TIMESTAMP + +Return type: INTEGER + +Example:: + + os> source=people | eval `YEARWEEK('2020-08-26')` = YEARWEEK('2020-08-26') | eval `YEARWEEK('2019-01-05', 1)` = YEARWEEK('2019-01-05', 1) | fields `YEARWEEK('2020-08-26')`, `YEARWEEK('2019-01-05', 1)` + fetched rows / total rows = 1/1 + +--------------------------+-----------------------------+ + | YEARWEEK('2020-08-26') | YEARWEEK('2019-01-05', 1) | + |--------------------------+-----------------------------| + | 202034 | 201901 | + +--------------------------+-----------------------------+ + + diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeFunctionIT.java index a10d7f3771..b75b0ecaef 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeFunctionIT.java @@ -1062,4 +1062,85 @@ public void testTimeDiff() throws IOException { verifySchema(result, schema("f", null, "time")); verifySome(result.getJSONArray("datarows"), rows("10:59:59")); } + + @Test + public void testGetFormat() throws IOException{ + var result = executeQuery(String.format("source=%s | eval f = date_format('2003-10-03', get_format(DATE,'USA')) | fields f", TEST_INDEX_DATE)); + verifySchema(result, schema("f", null, "string")); + verifySome(result.getJSONArray("datarows"), rows("10.03.2003")); + } + + @Test + public void testLastDay() throws IOException{ + var result = executeQuery(String.format("source=%s | eval f = last_day('2003-10-03') | fields f", TEST_INDEX_DATE)); + verifySchema(result, schema("f", null, "date")); + verifySome(result.getJSONArray("datarows"), rows("2003-10-31")); + } + + @Test + public void testSecToTime() throws IOException{ + var result = executeQuery(String.format("source=%s | eval f = sec_to_time(123456) | fields f", TEST_INDEX_DATE)); + verifySchema(result, schema("f", null, "time")); + verifySome(result.getJSONArray("datarows"), rows("10:17:36")); + } + + @Test + public void testYearWeek() throws IOException{ + var result = executeQuery(String.format("source=%s | eval f1 = yearweek('2003-10-03') | eval f2 = yearweek('2003-10-03', 3) | fields f1, f2", TEST_INDEX_DATE)); + verifySchema(result, + schema("f1", null, "integer"), + schema("f2", null, "integer")); + verifySome(result.getJSONArray("datarows"), rows(200339, 200340)); + } + + @Test + public void testWeekDay() throws IOException{ + var result = executeQuery(String.format("source=%s | eval f = weekday('2003-10-03') | fields f", TEST_INDEX_DATE)); + verifySchema(result, schema("f", null, "integer")); + verifySome(result.getJSONArray("datarows"), rows(4)); + } + + @Test + public void testToSeconds() throws IOException{ + var result = executeQuery(String.format("source=%s | eval f1 = to_seconds(date('2008-10-07')) | " + + "eval f2 = to_seconds('2020-09-16 07:40:00') | " + + "eval f3 = to_seconds(DATETIME('2020-09-16 07:40:00')) | fields f1, f2, f3", TEST_INDEX_DATE)); + verifySchema(result, + schema("f1", null, "long"), + schema("f2", null, "long"), + schema("f3", null, "long")); + verifySome(result.getJSONArray("datarows"), rows(63390556800L, 63767461200L, 63767461200L)); + } + + @Test + public void testStrToDate() throws IOException{ + var result = executeQuery(String.format("source=%s | eval f = str_to_date('01,5,2013', '%s') | fields f", TEST_INDEX_DATE, "%d,%m,%Y")); + verifySchema(result, schema("f", null, "datetime")); + verifySome(result.getJSONArray("datarows"), rows("2013-05-01 00:00:00")); + } + + @Test + public void testTimeStampAdd() throws IOException{ + var result = executeQuery(String.format("source=%s | eval f = timestampadd(YEAR, 15, '2001-03-06 00:00:00') | fields f", TEST_INDEX_DATE)); + verifySchema(result, schema("f", null, "datetime")); + verifySome(result.getJSONArray("datarows"), rows("2016-03-06 00:00:00")); + } + + @Test + public void testTimestampDiff() throws IOException{ + var result = executeQuery(String.format("source=%s | eval f = timestampdiff(YEAR, '1997-01-01 00:00:00', '2001-03-06 00:00:00') | fields f", TEST_INDEX_DATE)); + verifySchema(result, schema("f", null, "datetime")); + verifySome(result.getJSONArray("datarows"), rows(4)); + } + + @Test + public void testExtract() throws IOException{ + var result = executeQuery(String.format("source=%s | eval f1 = extract(YEAR FROM '1997-01-01 00:00:00') | eval f2 = extract(MINUTE FROM time('10:17:36')) | fields f1, f2", TEST_INDEX_DATE)); + verifySchema(result, + schema("f1", null, "long"), + schema("f2", null, "long")); + verifySome(result.getJSONArray("datarows"), rows(1997L, 17L)); + } + + } diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index 908dbe5262..e74aed30eb 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -262,8 +262,11 @@ DAYOFWEEK: 'DAYOFWEEK'; DAYOFYEAR: 'DAYOFYEAR'; DAY_OF_MONTH: 'DAY_OF_MONTH'; DAY_OF_WEEK: 'DAY_OF_WEEK'; +EXTRACT: 'EXTRACT'; FROM_DAYS: 'FROM_DAYS'; FROM_UNIXTIME: 'FROM_UNIXTIME'; +GET_FORMAT: 'GET_FORMAT'; +LAST_DAY: 'LAST_DAY'; LOCALTIME: 'LOCALTIME'; LOCALTIMESTAMP: 'LOCALTIMESTAMP'; MAKEDATE: 'MAKEDATE'; @@ -272,19 +275,26 @@ MONTHNAME: 'MONTHNAME'; NOW: 'NOW'; PERIOD_ADD: 'PERIOD_ADD'; PERIOD_DIFF: 'PERIOD_DIFF'; +SEC_TO_TIME: 'SEC_TO_TIME'; +STR_TO_DATE: 'STR_TO_DATE'; SUBDATE: 'SUBDATE'; SUBTIME: 'SUBTIME'; SYSDATE: 'SYSDATE'; TIME: 'TIME'; TIMEDIFF: 'TIMEDIFF'; TIMESTAMP: 'TIMESTAMP'; +TIMESTAMPADD: 'TIMESTAMPADD'; +TIMESTAMPDIFF: 'TIMESTAMPDIFF'; TIME_FORMAT: 'TIME_FORMAT'; TIME_TO_SEC: 'TIME_TO_SEC'; TO_DAYS: 'TO_DAYS'; +TO_SECONDS: 'TO_SECONDS'; UNIX_TIMESTAMP: 'UNIX_TIMESTAMP'; UTC_DATE: 'UTC_DATE'; UTC_TIME: 'UTC_TIME'; UTC_TIMESTAMP: 'UTC_TIMESTAMP'; +WEEKDAY: 'WEEKDAY'; +YEARWEEK: 'YEARWEEK'; // TEXT FUNCTIONS SUBSTR: 'SUBSTR'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index 853af31443..9cde1bfbb8 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -287,6 +287,9 @@ valueExpression right=valueExpression #binaryArithmetic | primaryExpression #valueExpressionDefault | positionFunction #positionFunctionCall + | extractFunction #extractFunctionCall + | getFormatFunction #getFormatFunctionCall + | timestampFunction #timestampFunctionCall | LT_PRTHS valueExpression RT_PRTHS #parentheticValueExpr ; @@ -538,6 +541,7 @@ dateTimeFunctionName | FROM_UNIXTIME | HOUR | HOUR_OF_DAY + | LAST_DAY | LOCALTIME | LOCALTIMESTAMP | MAKEDATE @@ -555,6 +559,8 @@ dateTimeFunctionName | QUARTER | SECOND | SECOND_OF_MINUTE + | SEC_TO_TIME + | STR_TO_DATE | SUBDATE | SUBTIME | SYSDATE @@ -564,13 +570,71 @@ dateTimeFunctionName | TIME_FORMAT | TIME_TO_SEC | TO_DAYS + | TO_SECONDS | UNIX_TIMESTAMP | UTC_DATE | UTC_TIME | UTC_TIMESTAMP | WEEK + | WEEKDAY | WEEK_OF_YEAR | YEAR + | YEARWEEK + ; + +getFormatFunction + : GET_FORMAT LT_PRTHS getFormatType COMMA functionArg RT_PRTHS + ; + +getFormatType + : DATE + | DATETIME + | TIME + | TIMESTAMP + ; + +extractFunction + : EXTRACT LT_PRTHS datetimePart FROM functionArg RT_PRTHS + ; + +simpleDateTimePart + : MICROSECOND + | SECOND + | MINUTE + | HOUR + | DAY + | WEEK + | MONTH + | QUARTER + | YEAR + ; + +complexDateTimePart + : SECOND_MICROSECOND + | MINUTE_MICROSECOND + | MINUTE_SECOND + | HOUR_MICROSECOND + | HOUR_SECOND + | HOUR_MINUTE + | DAY_MICROSECOND + | DAY_SECOND + | DAY_MINUTE + | DAY_HOUR + | YEAR_MONTH + ; + +datetimePart + : simpleDateTimePart + | complexDateTimePart + ; + +timestampFunction + : timestampFunctionName LT_PRTHS simpleDateTimePart COMMA firstArg=functionArg COMMA secondArg=functionArg RT_PRTHS + ; + +timestampFunctionName + : TIMESTAMPADD + | TIMESTAMPDIFF ; /** condition function return boolean value */ diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java index eddee3064e..c775747ec4 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java @@ -297,6 +297,60 @@ public UnresolvedExpression visitPositionFunction( visitFunctionArg(ctx.functionArg(1)))); } + @Override + public UnresolvedExpression visitExtractFunctionCall( + OpenSearchPPLParser.ExtractFunctionCallContext ctx) { + return new Function( + ctx.extractFunction().EXTRACT().toString(), + getExtractFunctionArguments(ctx)); + } + + private List getExtractFunctionArguments( + OpenSearchPPLParser.ExtractFunctionCallContext ctx) { + List args = Arrays.asList( + new Literal(ctx.extractFunction().datetimePart().getText(), DataType.STRING), + visitFunctionArg(ctx.extractFunction().functionArg()) + ); + return args; + } + + @Override + public UnresolvedExpression visitGetFormatFunctionCall( + OpenSearchPPLParser.GetFormatFunctionCallContext ctx) { + return new Function( + ctx.getFormatFunction().GET_FORMAT().toString(), + getFormatFunctionArguments(ctx)); + } + + private List getFormatFunctionArguments( + OpenSearchPPLParser.GetFormatFunctionCallContext ctx) { + List args = Arrays.asList( + new Literal(ctx.getFormatFunction().getFormatType().getText(), DataType.STRING), + visitFunctionArg(ctx.getFormatFunction().functionArg()) + ); + return args; + } + + @Override + public UnresolvedExpression visitTimestampFunctionCall( + OpenSearchPPLParser.TimestampFunctionCallContext ctx) { + return new Function( + ctx.timestampFunction().timestampFunctionName().getText(), + timestampFunctionArguments(ctx)); + } + + private List timestampFunctionArguments( + OpenSearchPPLParser.TimestampFunctionCallContext ctx) { + List args = Arrays.asList( + new Literal( + ctx.timestampFunction().simpleDateTimePart().getText(), + DataType.STRING), + visitFunctionArg(ctx.timestampFunction().firstArg), + visitFunctionArg(ctx.timestampFunction().secondArg) + ); + return args; + } + /** * Literal and value. */ diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/antlr/PPLSyntaxParserTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/antlr/PPLSyntaxParserTest.java index bbc566e2ba..8ca6c5c84e 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/antlr/PPLSyntaxParserTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/antlr/PPLSyntaxParserTest.java @@ -7,11 +7,15 @@ package org.opensearch.sql.ppl.antlr; import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThrows; +import java.util.List; import org.antlr.v4.runtime.tree.ParseTree; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; +import org.opensearch.sql.common.antlr.SyntaxCheckException; public class PPLSyntaxParserTest { @@ -135,7 +139,7 @@ public void testTopCommandWithoutNAndGroupByShouldPass() { } @Test - public void can_parse_multi_match_relevance_function() { + public void testCanParseMultiMatchRelevanceFunction() { assertNotEquals(null, new PPLSyntaxParser().parse( "SOURCE=test | WHERE multi_match(['address'], 'query')")); assertNotEquals(null, new PPLSyntaxParser().parse( @@ -168,7 +172,7 @@ public void can_parse_multi_match_relevance_function() { } @Test - public void can_parse_simple_query_string_relevance_function() { + public void testCanParseSimpleQueryStringRelevanceFunction() { assertNotEquals(null, new PPLSyntaxParser().parse( "SOURCE=test | WHERE simple_query_string(['address'], 'query')")); assertNotEquals(null, new PPLSyntaxParser().parse( @@ -201,7 +205,7 @@ public void can_parse_simple_query_string_relevance_function() { } @Test - public void can_parse_query_string_relevance_function() { + public void testCanParseQueryStringRelevanceFunction() { assertNotEquals(null, new PPLSyntaxParser().parse( "SOURCE=test | WHERE query_string(['address'], 'query')")); assertNotEquals(null, new PPLSyntaxParser().parse( @@ -268,5 +272,56 @@ public void testDescribeCommandWithSourceShouldFail() { new PPLSyntaxParser().parse("describe source=t"); } + + @Test + public void testCanParseExtractFunction() { + String[] parts = List.of("MICROSECOND", "SECOND", "MINUTE", "HOUR", "DAY", + "WEEK", "MONTH", "QUARTER", "YEAR", "SECOND_MICROSECOND", + "MINUTE_MICROSECOND", "MINUTE_SECOND", "HOUR_MICROSECOND", + "HOUR_SECOND", "HOUR_MINUTE", "DAY_MICROSECOND", + "DAY_SECOND", "DAY_MINUTE", "DAY_HOUR", "YEAR_MONTH").toArray(new String[0]); + + for (String part : parts) { + assertNotNull(new PPLSyntaxParser().parse( + String.format("SOURCE=test | eval k = extract(%s FROM \"2023-02-06\")", part))); + } + } + + @Test + public void testCanParseGetFormatFunction() { + String[] types = {"DATE", "DATETIME", "TIME", "TIMESTAMP"}; + String[] formats = {"'USA'", "'JIS'", "'ISO'", "'EUR'", "'INTERNAL'"}; + + for (String type : types) { + for (String format : formats) { + assertNotNull(new PPLSyntaxParser().parse( + String.format("SOURCE=test | eval k = get_format(%s, %s)", type, format))); + } + } + } + + @Test + public void testCannotParseGetFormatFunctionWithBadArg() { + assertThrows( + SyntaxCheckException.class, + () -> new PPLSyntaxParser().parse( + "SOURCE=test | eval k = GET_FORMAT(NONSENSE_ARG,'INTERNAL')")); + } + + @Test + public void testCanParseTimestampaddFunction() { + assertNotNull(new PPLSyntaxParser().parse( + "SOURCE=test | eval k = TIMESTAMPADD(MINUTE, 1, '2003-01-02')")); + assertNotNull(new PPLSyntaxParser().parse( + "SOURCE=test | eval k = TIMESTAMPADD(WEEK,1,'2003-01-02')")); + } + + @Test + public void testCanParseTimestampdiffFunction() { + assertNotNull(new PPLSyntaxParser().parse( + "SOURCE=test | eval k = TIMESTAMPDIFF(MINUTE, '2003-01-02', '2003-01-02')")); + assertNotNull(new PPLSyntaxParser().parse( + "SOURCE=test | eval k = TIMESTAMPDIFF(WEEK,'2003-01-02','2003-01-02')")); + } } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java index 3f3c0e50ba..a6e130eed3 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java @@ -16,6 +16,7 @@ import static org.opensearch.sql.ast.dsl.AstDSL.booleanLiteral; import static org.opensearch.sql.ast.dsl.AstDSL.cast; import static org.opensearch.sql.ast.dsl.AstDSL.compare; +import static org.opensearch.sql.ast.dsl.AstDSL.dateLiteral; import static org.opensearch.sql.ast.dsl.AstDSL.defaultFieldsArgs; import static org.opensearch.sql.ast.dsl.AstDSL.defaultSortFieldArgs; import static org.opensearch.sql.ast.dsl.AstDSL.defaultStatsArgs; @@ -40,6 +41,7 @@ import static org.opensearch.sql.ast.dsl.AstDSL.relation; import static org.opensearch.sql.ast.dsl.AstDSL.sort; import static org.opensearch.sql.ast.dsl.AstDSL.stringLiteral; +import static org.opensearch.sql.ast.dsl.AstDSL.timestampLiteral; import static org.opensearch.sql.ast.dsl.AstDSL.unresolvedArg; import static org.opensearch.sql.ast.dsl.AstDSL.xor; @@ -869,4 +871,62 @@ public void indexCanBeId() { defaultStatsArgs() )); } + + @Test + public void testExtractFunctionExpr() { + assertEqual("source=t | eval f=extract(day from '2001-05-07 10:11:12')", + eval( + relation("t"), + let( + field("f"), + function("extract", + stringLiteral("day"), stringLiteral("2001-05-07 10:11:12")) + ) + )); + } + + + @Test + public void testGet_FormatFunctionExpr() { + assertEqual("source=t | eval f=get_format(DATE,'USA')", + eval( + relation("t"), + let( + field("f"), + function("get_format", + stringLiteral("DATE"), stringLiteral("USA")) + ) + )); + } + + @Test + public void testTimeStampAddFunctionExpr() { + assertEqual("source=t | eval f=timestampadd(YEAR, 15, '2001-03-06 00:00:00')", + eval( + relation("t"), + let( + field("f"), + function("timestampadd", + stringLiteral("YEAR"), + intLiteral(15), + stringLiteral("2001-03-06 00:00:00")) + ) + )); + } + + @Test + public void testTimeStampDiffFunctionExpr() { + assertEqual("source=t | eval f=timestampdiff(" + + "YEAR, '1997-01-01 00:00:00', '2001-03-06 00:00:00')", + eval( + relation("t"), + let( + field("f"), + function("timestampdiff", + stringLiteral("YEAR"), + stringLiteral("1997-01-01 00:00:00"), + stringLiteral("2001-03-06 00:00:00")) + ) + )); + } } From 761e6a4016856d89bac46e32ea5a2df42c8cb733 Mon Sep 17 00:00:00 2001 From: Max Ksyunz Date: Tue, 18 Jul 2023 10:46:05 -0700 Subject: [PATCH 6/9] Addressing comments on 2.9 release note (#1863) Signed-off-by: Max Ksyunz --- release-notes/opensearch-sql.release-notes-2.9.0.0.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/release-notes/opensearch-sql.release-notes-2.9.0.0.md b/release-notes/opensearch-sql.release-notes-2.9.0.0.md index a374676cf8..6bbdbf453f 100644 --- a/release-notes/opensearch-sql.release-notes-2.9.0.0.md +++ b/release-notes/opensearch-sql.release-notes-2.9.0.0.md @@ -3,7 +3,7 @@ Compatible with OpenSearch and OpenSearch Dashboards Version 2.9.0 ### Features * Enable Table Function and PromQL function ([#1719](https://github.com/opensearch-project/sql/pull/1719)) -* Add spark connector ([#1780](https://github.com/opensearch-project/sql/pull/1780)) +* Add Spark connector ([#1780](https://github.com/opensearch-project/sql/pull/1780)) ### Enhancements @@ -19,7 +19,7 @@ Compatible with OpenSearch and OpenSearch Dashboards Version 2.9.0 * Add _routing to SQL includes list ([#1771](https://github.com/opensearch-project/sql/pull/1771)) * Disable read of plugins.query.datasources.encryption.masterkey from cluster settings GET API ([#1825](https://github.com/opensearch-project/sql/pull/1825)) * Add EMR client to spark connector ([#1790](https://github.com/opensearch-project/sql/pull/1790)) -* Improved error codes in case of data sourcde API security exception ([#1753](https://github.com/opensearch-project/sql/pull/1753)) +* Improved error codes in case of data source API security exception ([#1753](https://github.com/opensearch-project/sql/pull/1753)) * Remove Default master encryption key from settings ([#1851](https://github.com/opensearch-project/sql/pull/1851)) ### Bug Fixes @@ -31,14 +31,15 @@ Compatible with OpenSearch and OpenSearch Dashboards Version 2.9.0 * Updated documentation of round function return type ([#1725](https://github.com/opensearch-project/sql/pull/1725)) * Updated `protocol.rst` with new wording for error message ([#1662](https://github.com/opensearch-project/sql/pull/1662)) +* Updated documentation for temporal data types ([#1826](https://github.com/opensearch-project/sql/pull/1826)) ### Infrastructure * stopPrometheus task in doctest build.gradle now runs upon project failure in startOpenSearch ([#1747](https://github.com/opensearch-project/sql/pull/1747)) -* Upgrade guava to 32.0.1 +* Bump guava to 32.0.1 ([#1829](https://github.com/opensearch-project/sql/pull/1829)) * Disable CrossClusterSearchIT test ([#1814](https://github.com/opensearch-project/sql/pull/1814)) -* fix flakytest when tests.locale=tr ([#1827](https://github.com/opensearch-project/sql/pull/1827)) +* Fix flakytest when tests.locale=tr ([#1827](https://github.com/opensearch-project/sql/pull/1827)) ### Refactoring -* Simplify OpenSearchIndexScanBuilder ([#1738](https://github.com/opensearch-project/sql/pull/1738)) \ No newline at end of file +* Simplify OpenSearchIndexScanBuilder ([#1738](https://github.com/opensearch-project/sql/pull/1738)) From f4883c93f1aa09d9a3ed66ae9218369183abb99f Mon Sep 17 00:00:00 2001 From: Forest Vey Date: Tue, 18 Jul 2023 12:44:49 -0700 Subject: [PATCH 7/9] Removing breaking dependency. (#298) (#1883) Signed-off-by: forestmvey --- .../opensearch/sql/opensearch/client/OpenSearchNodeClient.java | 1 - 1 file changed, 1 deletion(-) diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java index b6ca5471e5..c6d44e2c23 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java @@ -6,7 +6,6 @@ package org.opensearch.sql.opensearch.client; -import com.carrotsearch.hppc.cursors.ObjectObjectCursor; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import java.util.Arrays; From a34ac9bf949faa949709542bf8b358af5d048a6b Mon Sep 17 00:00:00 2001 From: Rupal Mahajan Date: Wed, 19 Jul 2023 09:27:41 -0700 Subject: [PATCH 8/9] Enable hive support in spark application (#1845) * Initial spark application draft Signed-off-by: Rupal Mahajan * Remove temp table Signed-off-by: Rupal Mahajan * Add license header Signed-off-by: Rupal Mahajan * Add scalastyle-config and update readme Signed-off-by: Rupal Mahajan * Fix datatype for result and schema Signed-off-by: Rupal Mahajan * Add test Signed-off-by: Rupal Mahajan * Simplify code using toJSON.collect.toList Signed-off-by: Rupal Mahajan * Add example in readme Signed-off-by: Rupal Mahajan * Fix triple quotes issue Signed-off-by: Rupal Mahajan * Update method name and description Signed-off-by: Rupal Mahajan * Add applicationId Signed-off-by: Rupal Mahajan * Enable Hive Support Signed-off-by: Rupal Mahajan * Set flint config Signed-off-by: Rupal Mahajan * Fix json parsing error Signed-off-by: Rupal Mahajan --------- Signed-off-by: Rupal Mahajan --- .../src/main/scala/org/opensearch/sql/SQLJob.scala | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/spark-sql-application/src/main/scala/org/opensearch/sql/SQLJob.scala b/spark-sql-application/src/main/scala/org/opensearch/sql/SQLJob.scala index 04fa92b25b..98a3a08134 100644 --- a/spark-sql-application/src/main/scala/org/opensearch/sql/SQLJob.scala +++ b/spark-sql-application/src/main/scala/org/opensearch/sql/SQLJob.scala @@ -5,6 +5,7 @@ package org.opensearch.sql +import org.apache.spark.SparkConf import org.apache.spark.sql.{DataFrame, SparkSession, Row} import org.apache.spark.sql.types._ @@ -31,8 +32,17 @@ object SQLJob { val auth = args(5) val region = args(6) + val conf: SparkConf = new SparkConf() + .setAppName("SQLJob") + .set("spark.sql.extensions", "org.opensearch.flint.spark.FlintSparkExtensions") + .set("spark.datasource.flint.host", host) + .set("spark.datasource.flint.port", port) + .set("spark.datasource.flint.scheme", scheme) + .set("spark.datasource.flint.auth", auth) + .set("spark.datasource.flint.region", region) + // Create a SparkSession - val spark = SparkSession.builder().appName("SQLJob").getOrCreate() + val spark = SparkSession.builder().config(conf).enableHiveSupport().getOrCreate() try { // Execute SQL query @@ -89,7 +99,7 @@ object SQLJob { // Create the data rows val rows = Seq(( - result.toJSON.collect.toList.map(_.replaceAll("\"", "'")), + result.toJSON.collect.toList.map(_.replaceAll("'", "\\\\'").replaceAll("\"", "'")), resultSchema.toJSON.collect.toList.map(_.replaceAll("\"", "'")), sys.env.getOrElse("EMR_STEP_ID", "unknown"), spark.sparkContext.applicationId)) From f92e48ca2b2d5121769e9d3252048dc282e33465 Mon Sep 17 00:00:00 2001 From: Matthew Wells Date: Wed, 19 Jul 2023 13:49:18 -0700 Subject: [PATCH 9/9] fixed missing underscores in day_of_week doctest (#300) (#1885) Signed-off-by: Matthew Wells --- docs/user/ppl/functions/datetime.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/user/ppl/functions/datetime.rst b/docs/user/ppl/functions/datetime.rst index 8f844c75a2..f7c4091753 100644 --- a/docs/user/ppl/functions/datetime.rst +++ b/docs/user/ppl/functions/datetime.rst @@ -754,13 +754,13 @@ Synonyms: `DAYOFWEEK`_ Example:: - os> source=people | eval `DAYOFWEEK(DATE('2020-08-26'))` = DAYOFWEEK(DATE('2020-08-26')) | fields `DAYOFWEEK(DATE('2020-08-26'))` + os> source=people | eval `DAY_OF_WEEK(DATE('2020-08-26'))` = DAY_OF_WEEK(DATE('2020-08-26')) | fields `DAY_OF_WEEK(DATE('2020-08-26'))` fetched rows / total rows = 1/1 - +---------------------------------+ - | DAYOFWEEK(DATE('2020-08-26')) | - |---------------------------------| - | 4 | - +---------------------------------+ + +-----------------------------------+ + | DAY_OF_WEEK(DATE('2020-08-26')) | + |-----------------------------------| + | 4 | + +-----------------------------------+ DAYOFYEAR