From a9111a99fcdc72b5d231fb5ba14a4f0474532fc8 Mon Sep 17 00:00:00 2001 From: vamsi-amazon Date: Thu, 6 Apr 2023 19:26:30 -0700 Subject: [PATCH] Refactored all the datasource releated code to new datasources module Signed-off-by: vamsi-amazon --- config/checkstyle/google_checks.xml | 3 +- .../datasource/model/DataSourceMetadata.java | 3 - .../sql/storage/DataSourceFactory.java | 1 - datasources/build.gradle | 6 +- .../datasources}/auth/AuthenticationType.java | 2 +- .../DataSourceUserAuthorizationHelper.java | 8 +- ...DataSourceUserAuthorizationHelperImpl.java | 1 - .../datasources/encryptor/EncryptorImpl.java | 8 +- .../DataSourceNotFoundException.java | 2 +- .../datasources/exceptions/ErrorMessage.java | 78 ++++ .../CreateDataSourceActionRequest.java | 2 +- .../CreateDataSourceActionResponse.java | 2 +- .../DeleteDataSourceActionRequest.java | 2 +- .../DeleteDataSourceActionResponse.java | 2 +- .../GetDataSourceActionRequest.java | 2 +- .../GetDataSourceActionResponse.java | 2 +- .../UpdateDataSourceActionRequest.java | 2 +- .../UpdateDataSourceActionResponse.java | 2 +- .../rest/RestDataSourceQueryAction.java | 22 +- .../service}/DataSourceLoaderCache.java | 2 +- .../service}/DataSourceLoaderCacheImpl.java | 2 +- .../service}/DataSourceMetadataStorage.java | 3 +- .../service}/DataSourceServiceImpl.java | 14 +- .../OpenSearchDataSourceMetadataStorage.java | 49 ++- .../TransportCreateDataSourceAction.java | 28 +- .../TransportDeleteDataSourceAction.java | 22 +- .../TransportGetDataSourceAction.java | 14 +- .../TransportUpdateDataSourceAction.java | 25 +- .../utils/XContentParserUtils.java | 7 +- .../resources/datasources-index-mapping.yml | 0 .../resources/datasources-index-settings.yml | 0 .../auth/AuthenticationTypeTest.java | 2 +- ...SourceUserAuthorizationHelperImplTest.java | 36 +- .../encryptor/EncryptorImplTest.java | 87 ++++ .../DataSourceLoaderCacheImplTest.java | 2 +- .../service}/DataSourceServiceImplTest.java | 6 +- ...enSearchDataSourceMetadataStorageTest.java | 404 +++++++++++++++++- .../TransportCreateDataSourceActionTest.java | 86 ++++ .../TransportDeleteDataSourceActionTest.java | 78 ++++ .../TransportGetDataSourceActionTest.java | 137 ++++++ .../TransportUpdateDataSourceActionTest.java | 87 ++++ .../sql/datasources/utils/SchedulerTest.java | 8 +- .../utils/XContentParserUtilsTest.java | 101 +++++ .../org/opensearch/sql/ppl/StandaloneIT.java | 6 +- .../org/opensearch/sql/plugin/SQLPlugin.java | 24 +- .../transport/TransportPPLQueryAction.java | 2 +- prometheus/build.gradle | 1 + .../storage/PrometheusStorageFactory.java | 2 +- 48 files changed, 1206 insertions(+), 179 deletions(-) rename {core/src/main/java/org/opensearch/sql/datasource/model => datasources/src/main/java/org/opensearch/sql/datasources}/auth/AuthenticationType.java (94%) rename {core/src/main/java/org/opensearch/sql/datasource => datasources/src/main/java/org/opensearch/sql/datasources/auth}/DataSourceUserAuthorizationHelper.java (79%) rename {core/src/main/java/org/opensearch/sql/datasource => datasources/src/main/java/org/opensearch/sql/datasources}/exceptions/DataSourceNotFoundException.java (84%) create mode 100644 datasources/src/main/java/org/opensearch/sql/datasources/exceptions/ErrorMessage.java rename datasources/src/main/java/org/opensearch/sql/datasources/model/{ => transport}/CreateDataSourceActionRequest.java (96%) rename datasources/src/main/java/org/opensearch/sql/datasources/model/{ => transport}/CreateDataSourceActionResponse.java (92%) rename datasources/src/main/java/org/opensearch/sql/datasources/model/{ => transport}/DeleteDataSourceActionRequest.java (96%) rename datasources/src/main/java/org/opensearch/sql/datasources/model/{ => transport}/DeleteDataSourceActionResponse.java (92%) rename datasources/src/main/java/org/opensearch/sql/datasources/model/{ => transport}/GetDataSourceActionRequest.java (95%) rename datasources/src/main/java/org/opensearch/sql/datasources/model/{ => transport}/GetDataSourceActionResponse.java (92%) rename datasources/src/main/java/org/opensearch/sql/datasources/model/{ => transport}/UpdateDataSourceActionRequest.java (96%) rename datasources/src/main/java/org/opensearch/sql/datasources/model/{ => transport}/UpdateDataSourceActionResponse.java (92%) rename {core/src/main/java/org/opensearch/sql/datasource => datasources/src/main/java/org/opensearch/sql/datasources/service}/DataSourceLoaderCache.java (91%) rename {core/src/main/java/org/opensearch/sql/datasource => datasources/src/main/java/org/opensearch/sql/datasources/service}/DataSourceLoaderCacheImpl.java (97%) rename {core/src/main/java/org/opensearch/sql/datasource => datasources/src/main/java/org/opensearch/sql/datasources/service}/DataSourceMetadataStorage.java (95%) rename {core/src/main/java/org/opensearch/sql/datasource => datasources/src/main/java/org/opensearch/sql/datasources/service}/DataSourceServiceImpl.java (93%) rename {plugin => datasources}/src/main/resources/datasources-index-mapping.yml (100%) rename {plugin => datasources}/src/main/resources/datasources-index-settings.yml (100%) rename {core/src/test/java/org/opensearch/sql/datasource/model => datasources/src/test/java/org/opensearch/sql/datasources}/auth/AuthenticationTypeTest.java (93%) create mode 100644 datasources/src/test/java/org/opensearch/sql/datasources/encryptor/EncryptorImplTest.java rename {core/src/test/java/org/opensearch/sql/datasource => datasources/src/test/java/org/opensearch/sql/datasources/service}/DataSourceLoaderCacheImplTest.java (98%) rename {core/src/test/java/org/opensearch/sql/datasource => datasources/src/test/java/org/opensearch/sql/datasources/service}/DataSourceServiceImplTest.java (98%) create mode 100644 datasources/src/test/java/org/opensearch/sql/datasources/transport/TransportCreateDataSourceActionTest.java create mode 100644 datasources/src/test/java/org/opensearch/sql/datasources/transport/TransportDeleteDataSourceActionTest.java create mode 100644 datasources/src/test/java/org/opensearch/sql/datasources/transport/TransportGetDataSourceActionTest.java create mode 100644 datasources/src/test/java/org/opensearch/sql/datasources/transport/TransportUpdateDataSourceActionTest.java create mode 100644 datasources/src/test/java/org/opensearch/sql/datasources/utils/XContentParserUtilsTest.java diff --git a/config/checkstyle/google_checks.xml b/config/checkstyle/google_checks.xml index a0c7d90fd9..12c90f8495 100644 --- a/config/checkstyle/google_checks.xml +++ b/config/checkstyle/google_checks.xml @@ -39,8 +39,9 @@ - + + diff --git a/core/src/main/java/org/opensearch/sql/datasource/model/DataSourceMetadata.java b/core/src/main/java/org/opensearch/sql/datasource/model/DataSourceMetadata.java index 27d06d8151..7945f8aec3 100644 --- a/core/src/main/java/org/opensearch/sql/datasource/model/DataSourceMetadata.java +++ b/core/src/main/java/org/opensearch/sql/datasource/model/DataSourceMetadata.java @@ -12,8 +12,6 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.collect.ImmutableMap; -import com.google.gson.Gson; -import java.io.IOException; import java.util.Collections; import java.util.List; import java.util.Map; @@ -21,7 +19,6 @@ import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.NoArgsConstructor; -import lombok.RequiredArgsConstructor; import lombok.Setter; import org.opensearch.sql.datasource.DataSourceService; diff --git a/core/src/main/java/org/opensearch/sql/storage/DataSourceFactory.java b/core/src/main/java/org/opensearch/sql/storage/DataSourceFactory.java index d0f24d0e5a..8512eddbe3 100644 --- a/core/src/main/java/org/opensearch/sql/storage/DataSourceFactory.java +++ b/core/src/main/java/org/opensearch/sql/storage/DataSourceFactory.java @@ -7,7 +7,6 @@ package org.opensearch.sql.storage; -import java.util.Map; import org.opensearch.sql.datasource.DataSourceService; import org.opensearch.sql.datasource.model.DataSource; import org.opensearch.sql.datasource.model.DataSourceMetadata; diff --git a/datasources/build.gradle b/datasources/build.gradle index 2306ba99a6..cc2219f325 100644 --- a/datasources/build.gradle +++ b/datasources/build.gradle @@ -56,7 +56,11 @@ jacocoTestCoverageVerification { violationRules { rule { element = 'CLASS' - excludes = [] + excludes = [ + 'org.opensearch.sql.datasources.settings.DataSourceSettings', + 'org.opensearch.sql.datasources.exceptions.*', + 'org.opensearch.sql.datasources.rest.*' + ] limit { counter = 'LINE' minimum = 1.0 diff --git a/core/src/main/java/org/opensearch/sql/datasource/model/auth/AuthenticationType.java b/datasources/src/main/java/org/opensearch/sql/datasources/auth/AuthenticationType.java similarity index 94% rename from core/src/main/java/org/opensearch/sql/datasource/model/auth/AuthenticationType.java rename to datasources/src/main/java/org/opensearch/sql/datasources/auth/AuthenticationType.java index 9cf3e01509..715e72c0c3 100644 --- a/core/src/main/java/org/opensearch/sql/datasource/model/auth/AuthenticationType.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/auth/AuthenticationType.java @@ -5,7 +5,7 @@ * */ -package org.opensearch.sql.datasource.model.auth; +package org.opensearch.sql.datasources.auth; import java.util.Collections; import java.util.HashMap; diff --git a/core/src/main/java/org/opensearch/sql/datasource/DataSourceUserAuthorizationHelper.java b/datasources/src/main/java/org/opensearch/sql/datasources/auth/DataSourceUserAuthorizationHelper.java similarity index 79% rename from core/src/main/java/org/opensearch/sql/datasource/DataSourceUserAuthorizationHelper.java rename to datasources/src/main/java/org/opensearch/sql/datasources/auth/DataSourceUserAuthorizationHelper.java index dbbe82a527..adcfb0bdfd 100644 --- a/core/src/main/java/org/opensearch/sql/datasource/DataSourceUserAuthorizationHelper.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/auth/DataSourceUserAuthorizationHelper.java @@ -1,6 +1,10 @@ -package org.opensearch.sql.datasource; +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.datasources.auth; -import java.util.List; import org.opensearch.sql.datasource.model.DataSourceMetadata; /** diff --git a/datasources/src/main/java/org/opensearch/sql/datasources/auth/DataSourceUserAuthorizationHelperImpl.java b/datasources/src/main/java/org/opensearch/sql/datasources/auth/DataSourceUserAuthorizationHelperImpl.java index 5519f4ad8b..cd55991d00 100644 --- a/datasources/src/main/java/org/opensearch/sql/datasources/auth/DataSourceUserAuthorizationHelperImpl.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/auth/DataSourceUserAuthorizationHelperImpl.java @@ -12,7 +12,6 @@ import org.opensearch.client.Client; import org.opensearch.commons.ConfigConstants; import org.opensearch.commons.authuser.User; -import org.opensearch.sql.datasource.DataSourceUserAuthorizationHelper; import org.opensearch.sql.datasource.model.DataSourceMetadata; @AllArgsConstructor diff --git a/datasources/src/main/java/org/opensearch/sql/datasources/encryptor/EncryptorImpl.java b/datasources/src/main/java/org/opensearch/sql/datasources/encryptor/EncryptorImpl.java index 618d5be9ad..4838cd41a5 100644 --- a/datasources/src/main/java/org/opensearch/sql/datasources/encryptor/EncryptorImpl.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/encryptor/EncryptorImpl.java @@ -29,8 +29,8 @@ public String encrypt(String plainText) { .build(); JceMasterKey jceMasterKey - = JceMasterKey.getInstance(new SecretKeySpec(masterKey.getBytes(), "AES"), "Custom", "", - "AES/GCM/NoPadding"); + = JceMasterKey.getInstance(new SecretKeySpec(masterKey.getBytes(), "AES"), "Custom", + "opensearch.config.master.key", "AES/GCM/NoPadding"); final CryptoResult encryptResult = crypto.encryptData(jceMasterKey, plainText.getBytes(StandardCharsets.UTF_8)); @@ -44,8 +44,8 @@ public String decrypt(String encryptedText) { .build(); JceMasterKey jceMasterKey - = JceMasterKey.getInstance(new SecretKeySpec(masterKey.getBytes(), "AES"), "Custom", "", - "AES/GCM/NoPadding"); + = JceMasterKey.getInstance(new SecretKeySpec(masterKey.getBytes(), "AES"), "Custom", + "opensearch.config.master.key", "AES/GCM/NoPadding"); final CryptoResult decryptedResult = crypto.decryptData(jceMasterKey, Base64.getDecoder().decode(encryptedText)); diff --git a/core/src/main/java/org/opensearch/sql/datasource/exceptions/DataSourceNotFoundException.java b/datasources/src/main/java/org/opensearch/sql/datasources/exceptions/DataSourceNotFoundException.java similarity index 84% rename from core/src/main/java/org/opensearch/sql/datasource/exceptions/DataSourceNotFoundException.java rename to datasources/src/main/java/org/opensearch/sql/datasources/exceptions/DataSourceNotFoundException.java index 0a068ccdfc..484b0b92b2 100644 --- a/core/src/main/java/org/opensearch/sql/datasource/exceptions/DataSourceNotFoundException.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/exceptions/DataSourceNotFoundException.java @@ -5,7 +5,7 @@ * */ -package org.opensearch.sql.datasource.exceptions; +package org.opensearch.sql.datasources.exceptions; /** * DataSourceNotFoundException. diff --git a/datasources/src/main/java/org/opensearch/sql/datasources/exceptions/ErrorMessage.java b/datasources/src/main/java/org/opensearch/sql/datasources/exceptions/ErrorMessage.java new file mode 100644 index 0000000000..6dbd9bcfb5 --- /dev/null +++ b/datasources/src/main/java/org/opensearch/sql/datasources/exceptions/ErrorMessage.java @@ -0,0 +1,78 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + + +package org.opensearch.sql.datasources.exceptions; + +import com.google.gson.Gson; +import com.google.gson.JsonObject; +import lombok.Getter; +import org.opensearch.rest.RestStatus; + +/** + * Error Message. + */ +public class ErrorMessage { + + protected Throwable exception; + + private final int status; + + @Getter + private final String type; + + @Getter + private final String reason; + + @Getter + private final String details; + + /** + * Error Message Constructor. + */ + public ErrorMessage(Throwable exception, int status) { + this.exception = exception; + this.status = status; + + this.type = fetchType(); + this.reason = fetchReason(); + this.details = fetchDetails(); + } + + private String fetchType() { + return exception.getClass().getSimpleName(); + } + + protected String fetchReason() { + return status == RestStatus.BAD_REQUEST.getStatus() + ? "Invalid Request" + : "There was internal problem at backend"; + } + + protected String fetchDetails() { + // Some exception prints internal information (full class name) which is security concern + return emptyStringIfNull(exception.getLocalizedMessage()); + } + + private String emptyStringIfNull(String str) { + return str != null ? str : ""; + } + + @Override + public String toString() { + JsonObject jsonObject = new JsonObject(); + jsonObject.addProperty("status", status); + jsonObject.add("error", getErrorAsJson()); + return new Gson().toJson(jsonObject); + } + + private JsonObject getErrorAsJson() { + JsonObject errorJson = new JsonObject(); + errorJson.addProperty("type", type); + errorJson.addProperty("reason", reason); + errorJson.addProperty("details", details); + return errorJson; + } +} diff --git a/datasources/src/main/java/org/opensearch/sql/datasources/model/CreateDataSourceActionRequest.java b/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/CreateDataSourceActionRequest.java similarity index 96% rename from datasources/src/main/java/org/opensearch/sql/datasources/model/CreateDataSourceActionRequest.java rename to datasources/src/main/java/org/opensearch/sql/datasources/model/transport/CreateDataSourceActionRequest.java index d7ba15abc6..333564c10a 100644 --- a/datasources/src/main/java/org/opensearch/sql/datasources/model/CreateDataSourceActionRequest.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/CreateDataSourceActionRequest.java @@ -5,7 +5,7 @@ * */ -package org.opensearch.sql.datasources.model; +package org.opensearch.sql.datasources.model.transport; import static org.opensearch.sql.analysis.DataSourceSchemaIdentifierNameResolver.DEFAULT_DATASOURCE_NAME; diff --git a/datasources/src/main/java/org/opensearch/sql/datasources/model/CreateDataSourceActionResponse.java b/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/CreateDataSourceActionResponse.java similarity index 92% rename from datasources/src/main/java/org/opensearch/sql/datasources/model/CreateDataSourceActionResponse.java rename to datasources/src/main/java/org/opensearch/sql/datasources/model/transport/CreateDataSourceActionResponse.java index a1838c574b..4531c3d9fe 100644 --- a/datasources/src/main/java/org/opensearch/sql/datasources/model/CreateDataSourceActionResponse.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/CreateDataSourceActionResponse.java @@ -5,7 +5,7 @@ * */ -package org.opensearch.sql.datasources.model; +package org.opensearch.sql.datasources.model.transport; import java.io.IOException; import lombok.Getter; diff --git a/datasources/src/main/java/org/opensearch/sql/datasources/model/DeleteDataSourceActionRequest.java b/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/DeleteDataSourceActionRequest.java similarity index 96% rename from datasources/src/main/java/org/opensearch/sql/datasources/model/DeleteDataSourceActionRequest.java rename to datasources/src/main/java/org/opensearch/sql/datasources/model/transport/DeleteDataSourceActionRequest.java index 63b2a585fe..6bcbd7a561 100644 --- a/datasources/src/main/java/org/opensearch/sql/datasources/model/DeleteDataSourceActionRequest.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/DeleteDataSourceActionRequest.java @@ -5,7 +5,7 @@ * */ -package org.opensearch.sql.datasources.model; +package org.opensearch.sql.datasources.model.transport; import static org.opensearch.sql.analysis.DataSourceSchemaIdentifierNameResolver.DEFAULT_DATASOURCE_NAME; diff --git a/datasources/src/main/java/org/opensearch/sql/datasources/model/DeleteDataSourceActionResponse.java b/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/DeleteDataSourceActionResponse.java similarity index 92% rename from datasources/src/main/java/org/opensearch/sql/datasources/model/DeleteDataSourceActionResponse.java rename to datasources/src/main/java/org/opensearch/sql/datasources/model/transport/DeleteDataSourceActionResponse.java index 7b1c99a9af..c6847ed9ed 100644 --- a/datasources/src/main/java/org/opensearch/sql/datasources/model/DeleteDataSourceActionResponse.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/DeleteDataSourceActionResponse.java @@ -5,7 +5,7 @@ * */ -package org.opensearch.sql.datasources.model; +package org.opensearch.sql.datasources.model.transport; import java.io.IOException; import lombok.Getter; diff --git a/datasources/src/main/java/org/opensearch/sql/datasources/model/GetDataSourceActionRequest.java b/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/GetDataSourceActionRequest.java similarity index 95% rename from datasources/src/main/java/org/opensearch/sql/datasources/model/GetDataSourceActionRequest.java rename to datasources/src/main/java/org/opensearch/sql/datasources/model/transport/GetDataSourceActionRequest.java index 5286db14a6..6cafe1972a 100644 --- a/datasources/src/main/java/org/opensearch/sql/datasources/model/GetDataSourceActionRequest.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/GetDataSourceActionRequest.java @@ -5,7 +5,7 @@ * */ -package org.opensearch.sql.datasources.model; +package org.opensearch.sql.datasources.model.transport; import static org.opensearch.sql.analysis.DataSourceSchemaIdentifierNameResolver.DEFAULT_DATASOURCE_NAME; diff --git a/datasources/src/main/java/org/opensearch/sql/datasources/model/GetDataSourceActionResponse.java b/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/GetDataSourceActionResponse.java similarity index 92% rename from datasources/src/main/java/org/opensearch/sql/datasources/model/GetDataSourceActionResponse.java rename to datasources/src/main/java/org/opensearch/sql/datasources/model/transport/GetDataSourceActionResponse.java index fbf189ecf2..030493cb51 100644 --- a/datasources/src/main/java/org/opensearch/sql/datasources/model/GetDataSourceActionResponse.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/GetDataSourceActionResponse.java @@ -5,7 +5,7 @@ * */ -package org.opensearch.sql.datasources.model; +package org.opensearch.sql.datasources.model.transport; import java.io.IOException; import lombok.Getter; diff --git a/datasources/src/main/java/org/opensearch/sql/datasources/model/UpdateDataSourceActionRequest.java b/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/UpdateDataSourceActionRequest.java similarity index 96% rename from datasources/src/main/java/org/opensearch/sql/datasources/model/UpdateDataSourceActionRequest.java rename to datasources/src/main/java/org/opensearch/sql/datasources/model/transport/UpdateDataSourceActionRequest.java index 6f40289e1d..fe66483edd 100644 --- a/datasources/src/main/java/org/opensearch/sql/datasources/model/UpdateDataSourceActionRequest.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/UpdateDataSourceActionRequest.java @@ -5,7 +5,7 @@ * */ -package org.opensearch.sql.datasources.model; +package org.opensearch.sql.datasources.model.transport; import static org.opensearch.sql.analysis.DataSourceSchemaIdentifierNameResolver.DEFAULT_DATASOURCE_NAME; diff --git a/datasources/src/main/java/org/opensearch/sql/datasources/model/UpdateDataSourceActionResponse.java b/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/UpdateDataSourceActionResponse.java similarity index 92% rename from datasources/src/main/java/org/opensearch/sql/datasources/model/UpdateDataSourceActionResponse.java rename to datasources/src/main/java/org/opensearch/sql/datasources/model/transport/UpdateDataSourceActionResponse.java index 1f213c1074..faa3b1139b 100644 --- a/datasources/src/main/java/org/opensearch/sql/datasources/model/UpdateDataSourceActionResponse.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/UpdateDataSourceActionResponse.java @@ -5,7 +5,7 @@ * */ -package org.opensearch.sql.datasources.model; +package org.opensearch.sql.datasources.model.transport; import java.io.IOException; import lombok.Getter; diff --git a/datasources/src/main/java/org/opensearch/sql/datasources/rest/RestDataSourceQueryAction.java b/datasources/src/main/java/org/opensearch/sql/datasources/rest/RestDataSourceQueryAction.java index 4ec04b8782..c75170c355 100644 --- a/datasources/src/main/java/org/opensearch/sql/datasources/rest/RestDataSourceQueryAction.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/rest/RestDataSourceQueryAction.java @@ -28,23 +28,23 @@ import org.opensearch.rest.RestChannel; import org.opensearch.rest.RestRequest; import org.opensearch.rest.RestStatus; -import org.opensearch.sql.datasource.exceptions.DataSourceNotFoundException; import org.opensearch.sql.datasource.model.DataSourceMetadata; +import org.opensearch.sql.datasources.exceptions.DataSourceNotFoundException; +import org.opensearch.sql.datasources.exceptions.ErrorMessage; +import org.opensearch.sql.datasources.model.transport.CreateDataSourceActionRequest; +import org.opensearch.sql.datasources.model.transport.CreateDataSourceActionResponse; +import org.opensearch.sql.datasources.model.transport.DeleteDataSourceActionRequest; +import org.opensearch.sql.datasources.model.transport.DeleteDataSourceActionResponse; +import org.opensearch.sql.datasources.model.transport.GetDataSourceActionRequest; +import org.opensearch.sql.datasources.model.transport.GetDataSourceActionResponse; +import org.opensearch.sql.datasources.model.transport.UpdateDataSourceActionRequest; +import org.opensearch.sql.datasources.model.transport.UpdateDataSourceActionResponse; import org.opensearch.sql.datasources.transport.TransportCreateDataSourceAction; import org.opensearch.sql.datasources.transport.TransportDeleteDataSourceAction; import org.opensearch.sql.datasources.transport.TransportGetDataSourceAction; import org.opensearch.sql.datasources.transport.TransportUpdateDataSourceAction; import org.opensearch.sql.datasources.utils.Scheduler; import org.opensearch.sql.datasources.utils.XContentParserUtils; -import org.opensearch.sql.opensearch.response.error.ErrorMessageFactory; -import org.opensearch.sql.datasources.model.CreateDataSourceActionRequest; -import org.opensearch.sql.datasources.model.CreateDataSourceActionResponse; -import org.opensearch.sql.datasources.model.DeleteDataSourceActionRequest; -import org.opensearch.sql.datasources.model.DeleteDataSourceActionResponse; -import org.opensearch.sql.datasources.model.GetDataSourceActionRequest; -import org.opensearch.sql.datasources.model.GetDataSourceActionResponse; -import org.opensearch.sql.datasources.model.UpdateDataSourceActionRequest; -import org.opensearch.sql.datasources.model.UpdateDataSourceActionResponse; public class RestDataSourceQueryAction extends BaseRestHandler { @@ -237,7 +237,7 @@ private void handleException(Exception e, RestChannel restChannel) { private void reportError(final RestChannel channel, final Exception e, final RestStatus status) { channel.sendResponse( new BytesRestResponse( - status, ErrorMessageFactory.createErrorMessage(e, status.getStatus()).toString())); + status, new ErrorMessage(e, status.getStatus()).toString())); } private static boolean isClientError(Exception e) { diff --git a/core/src/main/java/org/opensearch/sql/datasource/DataSourceLoaderCache.java b/datasources/src/main/java/org/opensearch/sql/datasources/service/DataSourceLoaderCache.java similarity index 91% rename from core/src/main/java/org/opensearch/sql/datasource/DataSourceLoaderCache.java rename to datasources/src/main/java/org/opensearch/sql/datasources/service/DataSourceLoaderCache.java index cce70fe584..3fe2954c12 100644 --- a/core/src/main/java/org/opensearch/sql/datasource/DataSourceLoaderCache.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/service/DataSourceLoaderCache.java @@ -1,4 +1,4 @@ -package org.opensearch.sql.datasource; +package org.opensearch.sql.datasources.service; import org.opensearch.sql.datasource.model.DataSource; import org.opensearch.sql.datasource.model.DataSourceMetadata; diff --git a/core/src/main/java/org/opensearch/sql/datasource/DataSourceLoaderCacheImpl.java b/datasources/src/main/java/org/opensearch/sql/datasources/service/DataSourceLoaderCacheImpl.java similarity index 97% rename from core/src/main/java/org/opensearch/sql/datasource/DataSourceLoaderCacheImpl.java rename to datasources/src/main/java/org/opensearch/sql/datasources/service/DataSourceLoaderCacheImpl.java index 56b7bec08b..ba9520fc0c 100644 --- a/core/src/main/java/org/opensearch/sql/datasource/DataSourceLoaderCacheImpl.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/service/DataSourceLoaderCacheImpl.java @@ -1,4 +1,4 @@ -package org.opensearch.sql.datasource; +package org.opensearch.sql.datasources.service; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; diff --git a/core/src/main/java/org/opensearch/sql/datasource/DataSourceMetadataStorage.java b/datasources/src/main/java/org/opensearch/sql/datasources/service/DataSourceMetadataStorage.java similarity index 95% rename from core/src/main/java/org/opensearch/sql/datasource/DataSourceMetadataStorage.java rename to datasources/src/main/java/org/opensearch/sql/datasources/service/DataSourceMetadataStorage.java index 85ffd0a1b3..b54af3195e 100644 --- a/core/src/main/java/org/opensearch/sql/datasource/DataSourceMetadataStorage.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/service/DataSourceMetadataStorage.java @@ -5,11 +5,10 @@ * */ -package org.opensearch.sql.datasource; +package org.opensearch.sql.datasources.service; import java.util.List; import java.util.Optional; -import javax.xml.crypto.Data; import org.opensearch.sql.datasource.model.DataSource; import org.opensearch.sql.datasource.model.DataSourceMetadata; diff --git a/core/src/main/java/org/opensearch/sql/datasource/DataSourceServiceImpl.java b/datasources/src/main/java/org/opensearch/sql/datasources/service/DataSourceServiceImpl.java similarity index 93% rename from core/src/main/java/org/opensearch/sql/datasource/DataSourceServiceImpl.java rename to datasources/src/main/java/org/opensearch/sql/datasources/service/DataSourceServiceImpl.java index bc3f2d0abf..86afa90c2b 100644 --- a/core/src/main/java/org/opensearch/sql/datasource/DataSourceServiceImpl.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/service/DataSourceServiceImpl.java @@ -3,30 +3,24 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.sql.datasource; +package org.opensearch.sql.datasources.service; import static org.opensearch.sql.analysis.DataSourceSchemaIdentifierNameResolver.DEFAULT_DATASOURCE_NAME; import com.google.common.base.Preconditions; import com.google.common.base.Strings; -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; import java.util.HashMap; import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.stream.Collectors; -import javax.xml.crypto.Data; import org.opensearch.sql.common.utils.StringUtils; -import org.opensearch.sql.datasource.exceptions.DataSourceNotFoundException; +import org.opensearch.sql.datasource.DataSourceService; import org.opensearch.sql.datasource.model.DataSource; import org.opensearch.sql.datasource.model.DataSourceMetadata; -import org.opensearch.sql.datasource.model.DataSourceType; +import org.opensearch.sql.datasources.auth.DataSourceUserAuthorizationHelper; +import org.opensearch.sql.datasources.exceptions.DataSourceNotFoundException; import org.opensearch.sql.storage.DataSourceFactory; /** diff --git a/datasources/src/main/java/org/opensearch/sql/datasources/storage/OpenSearchDataSourceMetadataStorage.java b/datasources/src/main/java/org/opensearch/sql/datasources/storage/OpenSearchDataSourceMetadataStorage.java index 8f0d67b9a4..f76e1ba9dc 100644 --- a/datasources/src/main/java/org/opensearch/sql/datasources/storage/OpenSearchDataSourceMetadataStorage.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/storage/OpenSearchDataSourceMetadataStorage.java @@ -11,7 +11,6 @@ import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; @@ -43,11 +42,11 @@ import org.opensearch.index.query.QueryBuilders; import org.opensearch.search.SearchHit; import org.opensearch.search.builder.SearchSourceBuilder; -import org.opensearch.sql.datasource.DataSourceMetadataStorage; -import org.opensearch.sql.datasource.exceptions.DataSourceNotFoundException; import org.opensearch.sql.datasource.model.DataSourceMetadata; -import org.opensearch.sql.datasource.model.auth.AuthenticationType; +import org.opensearch.sql.datasources.auth.AuthenticationType; import org.opensearch.sql.datasources.encryptor.Encryptor; +import org.opensearch.sql.datasources.exceptions.DataSourceNotFoundException; +import org.opensearch.sql.datasources.service.DataSourceMetadataStorage; import org.opensearch.sql.datasources.utils.XContentParserUtils; public class OpenSearchDataSourceMetadataStorage implements DataSourceMetadataStorage { @@ -92,6 +91,7 @@ public List getDataSourceMetadata() { public Optional getDataSourceMetadata(String datasourceName) { if (!this.clusterService.state().routingTable().hasIndex(DATASOURCE_INDEX_NAME)) { createDataSourcesIndex(); + return Optional.empty(); } return searchInDataSourcesIndex(QueryBuilders.termQuery("name", datasourceName)) .stream() @@ -187,13 +187,12 @@ private void createDataSourcesIndex() { InputStream settingsFileStream = OpenSearchDataSourceMetadataStorage.class.getClassLoader() .getResourceAsStream(DATASOURCE_INDEX_SETTINGS_FILE_NAME); CreateIndexRequest createIndexRequest = new CreateIndexRequest(DATASOURCE_INDEX_NAME); - createIndexRequest - .mapping(IOUtils.toString(mappingFileStream, StandardCharsets.UTF_8), + createIndexRequest.mapping(IOUtils.toString(mappingFileStream, StandardCharsets.UTF_8), XContentType.YAML) .settings(IOUtils.toString(settingsFileStream, StandardCharsets.UTF_8), XContentType.YAML); ActionFuture createIndexResponseActionFuture; - try (ThreadContext.StoredContext storedContext = client.threadPool().getThreadContext() + try (ThreadContext.StoredContext ignored = client.threadPool().getThreadContext() .stashContext()) { createIndexResponseActionFuture = client.admin().indices().create(createIndexRequest); } @@ -202,11 +201,11 @@ private void createDataSourcesIndex() { LOG.info("Index: {} creation Acknowledged", DATASOURCE_INDEX_NAME); } else { throw new RuntimeException( - String.format("Index: %s creation failed", DATASOURCE_INDEX_NAME)); + "Index creation is not acknowledged."); } } catch (Throwable e) { throw new RuntimeException( - "Internal server error while creating" + DATASOURCE_INDEX_NAME + " index" + "Internal server error while creating" + DATASOURCE_INDEX_NAME + " index:: " + e.getMessage()); } } @@ -219,7 +218,7 @@ private List searchInDataSourcesIndex(QueryBuilder query) { searchSourceBuilder.size(DATASOURCE_QUERY_RESULT_SIZE); searchRequest.source(searchSourceBuilder); ActionFuture searchResponseActionFuture; - try (ThreadContext.StoredContext storedContext = client.threadPool().getThreadContext() + try (ThreadContext.StoredContext ignored = client.threadPool().getThreadContext() .stashContext()) { searchResponseActionFuture = client.search(searchRequest); } @@ -243,6 +242,7 @@ private List searchInDataSourcesIndex(QueryBuilder query) { } } + @SuppressWarnings("missingswitchdefault") private DataSourceMetadata encryptDecryptAuthenticationData(DataSourceMetadata dataSourceMetadata, Boolean isEncryption) { Map propertiesMap = dataSourceMetadata.getProperties(); @@ -259,8 +259,6 @@ private DataSourceMetadata encryptDecryptAuthenticationData(DataSourceMetadata d case AWSSIGV4AUTH: handleSigV4PropertiesEncryptionDecryption(propertiesMap, isEncryption); break; - default: - break; } } return dataSourceMetadata; @@ -268,14 +266,16 @@ private DataSourceMetadata encryptDecryptAuthenticationData(DataSourceMetadata d private void handleBasicAuthPropertiesEncryptionDecryption(Map propertiesMap, Boolean isEncryption) { - Optional usernameKey = propertiesMap.keySet().stream() + ArrayList list = new ArrayList<>(); + propertiesMap.keySet().stream() .filter(s -> s.endsWith("auth.username")) - .findFirst(); - Optional passwordKey = propertiesMap.keySet().stream() + .findFirst() + .ifPresent(list::add); + propertiesMap.keySet().stream() .filter(s -> s.endsWith("auth.password")) - .findFirst(); - encryptOrDecrypt(propertiesMap, isEncryption, - Arrays.asList(usernameKey.get(), passwordKey.get())); + .findFirst() + .ifPresent(list::add); + encryptOrDecrypt(propertiesMap, isEncryption, list); } private void encryptOrDecrypt(Map propertiesMap, Boolean isEncryption, @@ -293,13 +293,16 @@ private void encryptOrDecrypt(Map propertiesMap, Boolean isEncry private void handleSigV4PropertiesEncryptionDecryption(Map propertiesMap, Boolean isEncryption) { - Optional accessKey = propertiesMap.keySet().stream() + ArrayList list = new ArrayList<>(); + propertiesMap.keySet().stream() .filter(s -> s.endsWith("auth.access_key")) - .findFirst(); - Optional secretKey = propertiesMap.keySet().stream() + .findFirst() + .ifPresent(list::add); + propertiesMap.keySet().stream() .filter(s -> s.endsWith("auth.secret_key")) - .findFirst(); - encryptOrDecrypt(propertiesMap, isEncryption, Arrays.asList(accessKey.get(), secretKey.get())); + .findFirst() + .ifPresent(list::add); + encryptOrDecrypt(propertiesMap, isEncryption, list); } } \ No newline at end of file diff --git a/datasources/src/main/java/org/opensearch/sql/datasources/transport/TransportCreateDataSourceAction.java b/datasources/src/main/java/org/opensearch/sql/datasources/transport/TransportCreateDataSourceAction.java index d202f26955..4d8c51fac7 100644 --- a/datasources/src/main/java/org/opensearch/sql/datasources/transport/TransportCreateDataSourceAction.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/transport/TransportCreateDataSourceAction.java @@ -7,60 +7,54 @@ package org.opensearch.sql.datasources.transport; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.opensearch.action.ActionListener; import org.opensearch.action.ActionType; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.HandledTransportAction; -import org.opensearch.client.Client; -import org.opensearch.client.node.NodeClient; import org.opensearch.common.inject.Inject; import org.opensearch.sql.datasource.DataSourceService; -import org.opensearch.sql.datasource.DataSourceServiceImpl; import org.opensearch.sql.datasource.model.DataSourceMetadata; -import org.opensearch.sql.datasources.model.CreateDataSourceActionRequest; -import org.opensearch.sql.datasources.model.CreateDataSourceActionResponse; +import org.opensearch.sql.datasources.model.transport.CreateDataSourceActionRequest; +import org.opensearch.sql.datasources.model.transport.CreateDataSourceActionResponse; +import org.opensearch.sql.datasources.service.DataSourceServiceImpl; import org.opensearch.tasks.Task; import org.opensearch.transport.TransportService; public class TransportCreateDataSourceAction extends HandledTransportAction { - - private static final Logger LOG = LogManager.getLogger(); public static final String NAME = "cluster:admin/opensearch/ql/datasources/create"; public static final ActionType ACTION_TYPE = new ActionType<>(NAME, CreateDataSourceActionResponse::new); private DataSourceService dataSourceService; - private Client client; /** * TransportCreateDataSourceAction action for creating datasource. * * @param transportService transportService. * @param actionFilters actionFilters. - * @param client client. * @param dataSourceService dataSourceService. */ @Inject public TransportCreateDataSourceAction(TransportService transportService, ActionFilters actionFilters, - NodeClient client, DataSourceServiceImpl dataSourceService) { super(TransportCreateDataSourceAction.NAME, transportService, actionFilters, CreateDataSourceActionRequest::new); this.dataSourceService = dataSourceService; - this.client = client; } @Override protected void doExecute(Task task, CreateDataSourceActionRequest request, ActionListener actionListener) { - DataSourceMetadata dataSourceMetadata = request.getDataSourceMetadata(); - dataSourceService.createDataSource(dataSourceMetadata); - actionListener.onResponse(new CreateDataSourceActionResponse("Created DataSource with name " - + dataSourceMetadata.getName())); + try { + DataSourceMetadata dataSourceMetadata = request.getDataSourceMetadata(); + dataSourceService.createDataSource(dataSourceMetadata); + actionListener.onResponse(new CreateDataSourceActionResponse("Created DataSource with name " + + dataSourceMetadata.getName())); + } catch (Exception e) { + actionListener.onFailure(e); + } } } \ No newline at end of file diff --git a/datasources/src/main/java/org/opensearch/sql/datasources/transport/TransportDeleteDataSourceAction.java b/datasources/src/main/java/org/opensearch/sql/datasources/transport/TransportDeleteDataSourceAction.java index ea5c61310a..1d109ca7fc 100644 --- a/datasources/src/main/java/org/opensearch/sql/datasources/transport/TransportDeleteDataSourceAction.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/transport/TransportDeleteDataSourceAction.java @@ -11,13 +11,11 @@ import org.opensearch.action.ActionType; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.HandledTransportAction; -import org.opensearch.client.Client; -import org.opensearch.client.node.NodeClient; import org.opensearch.common.inject.Inject; import org.opensearch.sql.datasource.DataSourceService; -import org.opensearch.sql.datasource.DataSourceServiceImpl; -import org.opensearch.sql.datasources.model.DeleteDataSourceActionRequest; -import org.opensearch.sql.datasources.model.DeleteDataSourceActionResponse; +import org.opensearch.sql.datasources.model.transport.DeleteDataSourceActionRequest; +import org.opensearch.sql.datasources.model.transport.DeleteDataSourceActionResponse; +import org.opensearch.sql.datasources.service.DataSourceServiceImpl; import org.opensearch.tasks.Task; import org.opensearch.transport.TransportService; @@ -29,33 +27,33 @@ public class TransportDeleteDataSourceAction ACTION_TYPE = new ActionType<>(NAME, DeleteDataSourceActionResponse::new); private DataSourceService dataSourceService; - private Client client; /** * TransportDeleteDataSourceAction action for deleting datasource. * * @param transportService transportService. * @param actionFilters actionFilters. - * @param client client. * @param dataSourceService dataSourceService. */ @Inject public TransportDeleteDataSourceAction(TransportService transportService, ActionFilters actionFilters, - NodeClient client, DataSourceServiceImpl dataSourceService) { super(TransportDeleteDataSourceAction.NAME, transportService, actionFilters, DeleteDataSourceActionRequest::new); - this.client = client; this.dataSourceService = dataSourceService; } @Override protected void doExecute(Task task, DeleteDataSourceActionRequest request, ActionListener actionListener) { - dataSourceService.deleteDataSource(request.getDataSourceName()); - actionListener.onResponse(new DeleteDataSourceActionResponse("Deleted DataSource with name " - + request.getDataSourceName())); + try { + dataSourceService.deleteDataSource(request.getDataSourceName()); + actionListener.onResponse(new DeleteDataSourceActionResponse("Deleted DataSource with name " + + request.getDataSourceName())); + } catch (Exception e) { + actionListener.onFailure(e); + } } } \ No newline at end of file diff --git a/datasources/src/main/java/org/opensearch/sql/datasources/transport/TransportGetDataSourceAction.java b/datasources/src/main/java/org/opensearch/sql/datasources/transport/TransportGetDataSourceAction.java index 4bd9bf95c4..33d08f7cd2 100644 --- a/datasources/src/main/java/org/opensearch/sql/datasources/transport/TransportGetDataSourceAction.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/transport/TransportGetDataSourceAction.java @@ -7,21 +7,17 @@ package org.opensearch.sql.datasources.transport; - - import java.util.Set; import org.opensearch.action.ActionListener; import org.opensearch.action.ActionType; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.HandledTransportAction; -import org.opensearch.client.Client; -import org.opensearch.client.node.NodeClient; import org.opensearch.common.inject.Inject; import org.opensearch.sql.datasource.DataSourceService; -import org.opensearch.sql.datasource.DataSourceServiceImpl; import org.opensearch.sql.datasource.model.DataSourceMetadata; -import org.opensearch.sql.datasources.model.GetDataSourceActionRequest; -import org.opensearch.sql.datasources.model.GetDataSourceActionResponse; +import org.opensearch.sql.datasources.model.transport.GetDataSourceActionRequest; +import org.opensearch.sql.datasources.model.transport.GetDataSourceActionResponse; +import org.opensearch.sql.datasources.service.DataSourceServiceImpl; import org.opensearch.sql.protocol.response.format.JsonResponseFormatter; import org.opensearch.tasks.Task; import org.opensearch.transport.TransportService; @@ -34,24 +30,20 @@ public class TransportGetDataSourceAction ACTION_TYPE = new ActionType<>(NAME, GetDataSourceActionResponse::new); private DataSourceService dataSourceService; - private Client client; /** * TransportGetDataSourceAction action for getting datasource. * * @param transportService transportService. * @param actionFilters actionFilters. - * @param client client. * @param dataSourceService dataSourceService. */ @Inject public TransportGetDataSourceAction(TransportService transportService, ActionFilters actionFilters, - NodeClient client, DataSourceServiceImpl dataSourceService) { super(TransportGetDataSourceAction.NAME, transportService, actionFilters, GetDataSourceActionRequest::new); - this.client = client; this.dataSourceService = dataSourceService; } diff --git a/datasources/src/main/java/org/opensearch/sql/datasources/transport/TransportUpdateDataSourceAction.java b/datasources/src/main/java/org/opensearch/sql/datasources/transport/TransportUpdateDataSourceAction.java index aa3bc1bb6a..4aece69e5b 100644 --- a/datasources/src/main/java/org/opensearch/sql/datasources/transport/TransportUpdateDataSourceAction.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/transport/TransportUpdateDataSourceAction.java @@ -7,58 +7,53 @@ package org.opensearch.sql.datasources.transport; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.opensearch.action.ActionListener; import org.opensearch.action.ActionType; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.HandledTransportAction; -import org.opensearch.client.Client; -import org.opensearch.client.node.NodeClient; import org.opensearch.common.inject.Inject; import org.opensearch.sql.datasource.DataSourceService; -import org.opensearch.sql.datasource.DataSourceServiceImpl; -import org.opensearch.sql.datasources.model.UpdateDataSourceActionRequest; -import org.opensearch.sql.datasources.model.UpdateDataSourceActionResponse; +import org.opensearch.sql.datasources.model.transport.UpdateDataSourceActionRequest; +import org.opensearch.sql.datasources.model.transport.UpdateDataSourceActionResponse; +import org.opensearch.sql.datasources.service.DataSourceServiceImpl; import org.opensearch.tasks.Task; import org.opensearch.transport.TransportService; public class TransportUpdateDataSourceAction extends HandledTransportAction { - private static final Logger LOG = LogManager.getLogger(); public static final String NAME = "cluster:admin/opensearch/ql/datasources/update"; public static final ActionType ACTION_TYPE = new ActionType<>(NAME, UpdateDataSourceActionResponse::new); private DataSourceService dataSourceService; - private Client client; /** * TransportUpdateDataSourceAction action for updating datasource. * * @param transportService transportService. * @param actionFilters actionFilters. - * @param client client. * @param dataSourceService dataSourceService. */ @Inject public TransportUpdateDataSourceAction(TransportService transportService, ActionFilters actionFilters, - NodeClient client, DataSourceServiceImpl dataSourceService) { super(TransportUpdateDataSourceAction.NAME, transportService, actionFilters, UpdateDataSourceActionRequest::new); this.dataSourceService = dataSourceService; - this.client = client; } @Override protected void doExecute(Task task, UpdateDataSourceActionRequest request, ActionListener actionListener) { - dataSourceService.updateDataSource(request.getDataSourceMetadata()); - actionListener.onResponse(new UpdateDataSourceActionResponse("Updated DataSource with name " - + request.getDataSourceMetadata().getName())); + try { + dataSourceService.updateDataSource(request.getDataSourceMetadata()); + actionListener.onResponse(new UpdateDataSourceActionResponse("Updated DataSource with name " + + request.getDataSourceMetadata().getName())); + } catch (Exception e) { + actionListener.onFailure(e); + } } } \ No newline at end of file diff --git a/datasources/src/main/java/org/opensearch/sql/datasources/utils/XContentParserUtils.java b/datasources/src/main/java/org/opensearch/sql/datasources/utils/XContentParserUtils.java index 05fb634065..a8643a35f3 100644 --- a/datasources/src/main/java/org/opensearch/sql/datasources/utils/XContentParserUtils.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/utils/XContentParserUtils.java @@ -12,6 +12,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import lombok.experimental.UtilityClass; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.DeprecationHandler; @@ -21,6 +22,10 @@ import org.opensearch.sql.datasource.model.DataSourceMetadata; import org.opensearch.sql.datasource.model.DataSourceType; +/** + * Utitlity class to serialize and deserialize objects in XContent. + */ +@UtilityClass public class XContentParserUtils { public static final String NAME_FIELD = "name"; public static final String CONNECTOR_FIELD = "connector"; @@ -69,7 +74,7 @@ public static DataSourceMetadata toDataSourceMetadata(XContentParser parser) thr } } if (name == null || connector == null) { - throw new IllegalArgumentException("Missing required fields"); + throw new IllegalArgumentException("name and connector are required fields."); } return new DataSourceMetadata(name, connector, allowedRoles, properties); } diff --git a/plugin/src/main/resources/datasources-index-mapping.yml b/datasources/src/main/resources/datasources-index-mapping.yml similarity index 100% rename from plugin/src/main/resources/datasources-index-mapping.yml rename to datasources/src/main/resources/datasources-index-mapping.yml diff --git a/plugin/src/main/resources/datasources-index-settings.yml b/datasources/src/main/resources/datasources-index-settings.yml similarity index 100% rename from plugin/src/main/resources/datasources-index-settings.yml rename to datasources/src/main/resources/datasources-index-settings.yml diff --git a/core/src/test/java/org/opensearch/sql/datasource/model/auth/AuthenticationTypeTest.java b/datasources/src/test/java/org/opensearch/sql/datasources/auth/AuthenticationTypeTest.java similarity index 93% rename from core/src/test/java/org/opensearch/sql/datasource/model/auth/AuthenticationTypeTest.java rename to datasources/src/test/java/org/opensearch/sql/datasources/auth/AuthenticationTypeTest.java index f9e4f3ce59..23bb4688e1 100644 --- a/core/src/test/java/org/opensearch/sql/datasource/model/auth/AuthenticationTypeTest.java +++ b/datasources/src/test/java/org/opensearch/sql/datasources/auth/AuthenticationTypeTest.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.sql.datasource.model.auth; +package org.opensearch.sql.datasources.auth; import static org.junit.jupiter.api.Assertions.assertEquals; diff --git a/datasources/src/test/java/org/opensearch/sql/datasources/auth/DataSourceUserAuthorizationHelperImplTest.java b/datasources/src/test/java/org/opensearch/sql/datasources/auth/DataSourceUserAuthorizationHelperImplTest.java index 6617735894..552bd0edf9 100644 --- a/datasources/src/test/java/org/opensearch/sql/datasources/auth/DataSourceUserAuthorizationHelperImplTest.java +++ b/datasources/src/test/java/org/opensearch/sql/datasources/auth/DataSourceUserAuthorizationHelperImplTest.java @@ -10,18 +10,18 @@ import java.util.HashMap; import java.util.List; import org.junit.Assert; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Answers; import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.Mockito; -import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.client.Client; import org.opensearch.sql.datasource.model.DataSourceMetadata; import org.opensearch.sql.datasource.model.DataSourceType; -@RunWith(MockitoJUnitRunner.class) +@ExtendWith(MockitoExtension.class) public class DataSourceUserAuthorizationHelperImplTest { @Mock(answer = Answers.RETURNS_DEEP_STUBS) @@ -35,7 +35,7 @@ public class DataSourceUserAuthorizationHelperImplTest { public void testAuthorizeDataSourceWithAllowedRoles() { String userString = "myuser|bckrole1,bckrol2|prometheus_access|myTenant"; Mockito.when(client.threadPool().getThreadContext() - .getTransient(OPENSEARCH_SECURITY_USER_INFO_THREAD_CONTEXT)) + .getTransient(OPENSEARCH_SECURITY_USER_INFO_THREAD_CONTEXT)) .thenReturn(userString); DataSourceMetadata dataSourceMetadata = dataSourceMetadata(); this.dataSourceUserAuthorizationHelper @@ -46,18 +46,40 @@ public void testAuthorizeDataSourceWithAllowedRoles() { public void testAuthorizeDataSourceWithAdminRole() { String userString = "myuser|bckrole1,bckrol2|all_access|myTenant"; Mockito.when(client.threadPool().getThreadContext() - .getTransient(OPENSEARCH_SECURITY_USER_INFO_THREAD_CONTEXT)) + .getTransient(OPENSEARCH_SECURITY_USER_INFO_THREAD_CONTEXT)) .thenReturn(userString); DataSourceMetadata dataSourceMetadata = dataSourceMetadata(); this.dataSourceUserAuthorizationHelper .authorizeDataSource(dataSourceMetadata); } + @Test + public void testAuthorizeDataSourceWithNullUserString() { + Mockito.when(client.threadPool().getThreadContext() + .getTransient(OPENSEARCH_SECURITY_USER_INFO_THREAD_CONTEXT)) + .thenReturn(null); + DataSourceMetadata dataSourceMetadata = dataSourceMetadata(); + this.dataSourceUserAuthorizationHelper + .authorizeDataSource(dataSourceMetadata); + } + + @Test + public void testAuthorizeDataSourceWithDefaultDataSource() { + String userString = "myuser|bckrole1,bckrol2|role1|myTenant"; + Mockito.when(client.threadPool().getThreadContext() + .getTransient(OPENSEARCH_SECURITY_USER_INFO_THREAD_CONTEXT)) + .thenReturn(userString); + DataSourceMetadata dataSourceMetadata = + DataSourceMetadata.defaultOpenSearchDataSourceMetadata(); + this.dataSourceUserAuthorizationHelper + .authorizeDataSource(dataSourceMetadata); + } + @Test public void testAuthorizeDataSourceWithException() { String userString = "myuser|bckrole1,bckrol2|role1|myTenant"; Mockito.when(client.threadPool().getThreadContext() - .getTransient(OPENSEARCH_SECURITY_USER_INFO_THREAD_CONTEXT)) + .getTransient(OPENSEARCH_SECURITY_USER_INFO_THREAD_CONTEXT)) .thenReturn(userString); DataSourceMetadata dataSourceMetadata = dataSourceMetadata(); SecurityException securityException diff --git a/datasources/src/test/java/org/opensearch/sql/datasources/encryptor/EncryptorImplTest.java b/datasources/src/test/java/org/opensearch/sql/datasources/encryptor/EncryptorImplTest.java new file mode 100644 index 0000000000..22f5b09255 --- /dev/null +++ b/datasources/src/test/java/org/opensearch/sql/datasources/encryptor/EncryptorImplTest.java @@ -0,0 +1,87 @@ +/* + * + * * Copyright OpenSearch Contributors + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.opensearch.sql.datasources.encryptor; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import com.amazonaws.encryptionsdk.exception.AwsCryptoException; +import com.amazonaws.encryptionsdk.exception.BadCiphertextException; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.junit.jupiter.MockitoExtension; + + +@ExtendWith(MockitoExtension.class) +public class EncryptorImplTest { + + @Test + public void testEncryptAndDecrypt() { + String masterKey = "1234567890123456"; + String input = "This is a test input"; + Encryptor encryptor = new EncryptorImpl(masterKey); + + String encrypted = encryptor.encrypt(input); + String decrypted = encryptor.decrypt(encrypted); + + assertEquals(input, decrypted); + } + + @Test + public void testMasterKeySize() { + String input = "This is a test input"; + String masterKey8 = "12345678"; + Encryptor encryptor8 = new EncryptorImpl(masterKey8); + assertThrows(AwsCryptoException.class, () -> { + encryptor8.encrypt(input); + }); + + String masterKey16 = "1234567812345678"; + Encryptor encryptor16 = new EncryptorImpl(masterKey16); + String encrypted = encryptor16.encrypt(input); + Assertions.assertEquals(input, encryptor16.decrypt(encrypted)); + + String masterKey24 = "123456781234567812345678"; + Encryptor encryptor24 = new EncryptorImpl(masterKey24); + encrypted = encryptor24.encrypt(input); + Assertions.assertEquals(input, encryptor24.decrypt(encrypted)); + + String masterKey17 = "12345678123456781"; + Encryptor encryptor17 = new EncryptorImpl(masterKey17); + assertThrows(AwsCryptoException.class, () -> { + encryptor17.encrypt(input); + }); + } + + @Test + public void testInvalidBase64String() { + String encrypted = "invalidBase64String"; + Encryptor encryptor = new EncryptorImpl("randomMasterKey"); + + assertThrows(BadCiphertextException.class, () -> { + encryptor.decrypt(encrypted); + }); + } + + @Test + public void testDecryptWithDifferentKey() { + + String masterKeyOne = "1234567890123456"; + String masterKeyTwo = "1234567890123455"; + String input = "This is a test input"; + Encryptor encryptor1 = new EncryptorImpl(masterKeyOne); + Encryptor encryptor2 = new EncryptorImpl(masterKeyTwo); + + String encrypted = encryptor1.encrypt(input); + + assertThrows(Exception.class, () -> { + encryptor2.decrypt(encrypted); + }); + } +} \ No newline at end of file diff --git a/core/src/test/java/org/opensearch/sql/datasource/DataSourceLoaderCacheImplTest.java b/datasources/src/test/java/org/opensearch/sql/datasources/service/DataSourceLoaderCacheImplTest.java similarity index 98% rename from core/src/test/java/org/opensearch/sql/datasource/DataSourceLoaderCacheImplTest.java rename to datasources/src/test/java/org/opensearch/sql/datasources/service/DataSourceLoaderCacheImplTest.java index fae69e7feb..bf656857b0 100644 --- a/core/src/test/java/org/opensearch/sql/datasource/DataSourceLoaderCacheImplTest.java +++ b/datasources/src/test/java/org/opensearch/sql/datasources/service/DataSourceLoaderCacheImplTest.java @@ -1,4 +1,4 @@ -package org.opensearch.sql.datasource; +package org.opensearch.sql.datasources.service; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.lenient; diff --git a/core/src/test/java/org/opensearch/sql/datasource/DataSourceServiceImplTest.java b/datasources/src/test/java/org/opensearch/sql/datasources/service/DataSourceServiceImplTest.java similarity index 98% rename from core/src/test/java/org/opensearch/sql/datasource/DataSourceServiceImplTest.java rename to datasources/src/test/java/org/opensearch/sql/datasources/service/DataSourceServiceImplTest.java index 98e17e9166..e1312ec582 100644 --- a/core/src/test/java/org/opensearch/sql/datasource/DataSourceServiceImplTest.java +++ b/datasources/src/test/java/org/opensearch/sql/datasources/service/DataSourceServiceImplTest.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.sql.datasource; +package org.opensearch.sql.datasources.service; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -34,10 +34,12 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import org.opensearch.sql.datasource.exceptions.DataSourceNotFoundException; +import org.opensearch.sql.datasource.DataSourceService; import org.opensearch.sql.datasource.model.DataSource; import org.opensearch.sql.datasource.model.DataSourceMetadata; import org.opensearch.sql.datasource.model.DataSourceType; +import org.opensearch.sql.datasources.auth.DataSourceUserAuthorizationHelper; +import org.opensearch.sql.datasources.exceptions.DataSourceNotFoundException; import org.opensearch.sql.storage.DataSourceFactory; import org.opensearch.sql.storage.StorageEngine; diff --git a/datasources/src/test/java/org/opensearch/sql/datasources/storage/OpenSearchDataSourceMetadataStorageTest.java b/datasources/src/test/java/org/opensearch/sql/datasources/storage/OpenSearchDataSourceMetadataStorageTest.java index 54a2f5facd..f42a638588 100644 --- a/datasources/src/test/java/org/opensearch/sql/datasources/storage/OpenSearchDataSourceMetadataStorageTest.java +++ b/datasources/src/test/java/org/opensearch/sql/datasources/storage/OpenSearchDataSourceMetadataStorageTest.java @@ -11,19 +11,20 @@ import com.fasterxml.jackson.databind.ObjectMapper; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Optional; import lombok.SneakyThrows; import org.apache.lucene.search.TotalHits; -import org.junit.Assert; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Answers; import org.mockito.ArgumentMatchers; import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.Mockito; -import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.action.ActionFuture; import org.opensearch.action.DocWriteResponse; import org.opensearch.action.admin.indices.create.CreateIndexResponse; @@ -33,14 +34,18 @@ import org.opensearch.action.update.UpdateResponse; import org.opensearch.client.Client; import org.opensearch.cluster.service.ClusterService; +import org.opensearch.index.engine.DocumentMissingException; +import org.opensearch.index.engine.VersionConflictEngineException; +import org.opensearch.index.shard.ShardId; import org.opensearch.rest.RestStatus; import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; -import org.opensearch.sql.datasources.encryptor.Encryptor; import org.opensearch.sql.datasource.model.DataSourceMetadata; import org.opensearch.sql.datasource.model.DataSourceType; +import org.opensearch.sql.datasources.encryptor.Encryptor; +import org.opensearch.sql.datasources.exceptions.DataSourceNotFoundException; -@RunWith(MockitoJUnitRunner.class) +@ExtendWith(MockitoExtension.class) public class OpenSearchDataSourceMetadataStorageTest { private static final String TEST_DATASOURCE_INDEX_NAME = "testDS"; @@ -98,18 +103,57 @@ public void testGetDataSourceMetadata() { = openSearchDataSourceMetadataStorage.getDataSourceMetadata(TEST_DATASOURCE_INDEX_NAME); - Assert.assertFalse(dataSourceMetadataOptional.isEmpty()); + Assertions.assertFalse(dataSourceMetadataOptional.isEmpty()); DataSourceMetadata dataSourceMetadata = dataSourceMetadataOptional.get(); - Assert.assertEquals(TEST_DATASOURCE_INDEX_NAME, dataSourceMetadata.getName()); - Assert.assertEquals(DataSourceType.PROMETHEUS, dataSourceMetadata.getConnector()); - Assert.assertEquals("password", + Assertions.assertEquals(TEST_DATASOURCE_INDEX_NAME, dataSourceMetadata.getName()); + Assertions.assertEquals(DataSourceType.PROMETHEUS, dataSourceMetadata.getConnector()); + Assertions.assertEquals("password", dataSourceMetadata.getProperties().get("prometheus.auth.password")); - Assert.assertEquals("username", + Assertions.assertEquals("username", dataSourceMetadata.getProperties().get("prometheus.auth.username")); - Assert.assertEquals("basicauth", + Assertions.assertEquals("basicauth", dataSourceMetadata.getProperties().get("prometheus.auth.type")); } + @SneakyThrows + @Test + public void testGetDataSourceMetadataWith404SearchResponse() { + Mockito.when(clusterService.state().routingTable().hasIndex(DATASOURCE_INDEX_NAME)) + .thenReturn(true); + Mockito.when(client.search(ArgumentMatchers.any())).thenReturn(searchResponseActionFuture); + Mockito.when(searchResponseActionFuture.actionGet()).thenReturn(searchResponse); + Mockito.when(searchResponse.status()).thenReturn(RestStatus.NOT_FOUND); + + RuntimeException runtimeException = Assertions.assertThrows(RuntimeException.class, + () -> openSearchDataSourceMetadataStorage.getDataSourceMetadata( + TEST_DATASOURCE_INDEX_NAME)); + Assertions.assertEquals( + "Fetching dataSource metadata information failed with status : NOT_FOUND", + runtimeException.getMessage()); + } + + @SneakyThrows + @Test + public void testGetDataSourceMetadataWithParsingFailed() { + Mockito.when(clusterService.state().routingTable().hasIndex(DATASOURCE_INDEX_NAME)) + .thenReturn(true); + Mockito.when(client.search(ArgumentMatchers.any())).thenReturn(searchResponseActionFuture); + Mockito.when(searchResponseActionFuture.actionGet()).thenReturn(searchResponse); + Mockito.when(searchResponse.status()).thenReturn(RestStatus.OK); + Mockito.when(searchResponse.getHits()) + .thenReturn( + new SearchHits( + new SearchHit[] {searchHit}, + new TotalHits(21, TotalHits.Relation.EQUAL_TO), + 1.0F)); + Mockito.when(searchHit.getSourceAsString()) + .thenReturn("..testDs"); + + Assertions.assertThrows(RuntimeException.class, + () -> openSearchDataSourceMetadataStorage.getDataSourceMetadata( + TEST_DATASOURCE_INDEX_NAME)); + } + @SneakyThrows @Test public void testGetDataSourceMetadataWithAWSSigV4() { @@ -133,18 +177,81 @@ public void testGetDataSourceMetadataWithAWSSigV4() { = openSearchDataSourceMetadataStorage.getDataSourceMetadata(TEST_DATASOURCE_INDEX_NAME); - Assert.assertFalse(dataSourceMetadataOptional.isEmpty()); + Assertions.assertFalse(dataSourceMetadataOptional.isEmpty()); DataSourceMetadata dataSourceMetadata = dataSourceMetadataOptional.get(); - Assert.assertEquals(TEST_DATASOURCE_INDEX_NAME, dataSourceMetadata.getName()); - Assert.assertEquals(DataSourceType.PROMETHEUS, dataSourceMetadata.getConnector()); - Assert.assertEquals("secret_key", + Assertions.assertEquals(TEST_DATASOURCE_INDEX_NAME, dataSourceMetadata.getName()); + Assertions.assertEquals(DataSourceType.PROMETHEUS, dataSourceMetadata.getConnector()); + Assertions.assertEquals("secret_key", dataSourceMetadata.getProperties().get("prometheus.auth.secret_key")); - Assert.assertEquals("access_key", + Assertions.assertEquals("access_key", dataSourceMetadata.getProperties().get("prometheus.auth.access_key")); - Assert.assertEquals("awssigv4", + Assertions.assertEquals("awssigv4", dataSourceMetadata.getProperties().get("prometheus.auth.type")); } + + @SneakyThrows + @Test + public void testGetDataSourceMetadataList() { + Mockito.when(clusterService.state().routingTable().hasIndex(DATASOURCE_INDEX_NAME)) + .thenReturn(true); + Mockito.when(client.search(ArgumentMatchers.any())).thenReturn(searchResponseActionFuture); + Mockito.when(searchResponseActionFuture.actionGet()).thenReturn(searchResponse); + Mockito.when(searchResponse.status()).thenReturn(RestStatus.OK); + Mockito.when(searchResponse.getHits()) + .thenReturn( + new SearchHits( + new SearchHit[] {searchHit}, + new TotalHits(21, TotalHits.Relation.EQUAL_TO), + 1.0F)); + Mockito.when(searchHit.getSourceAsString()) + .thenReturn(getDataSourceMetadataStringWithNoAuthentication()); + + List dataSourceMetadataList + = openSearchDataSourceMetadataStorage.getDataSourceMetadata(); + + + Assertions.assertEquals(1, dataSourceMetadataList.size()); + DataSourceMetadata dataSourceMetadata = dataSourceMetadataList.get(0); + Assertions.assertEquals(TEST_DATASOURCE_INDEX_NAME, dataSourceMetadata.getName()); + Assertions.assertEquals(DataSourceType.PROMETHEUS, dataSourceMetadata.getConnector()); + } + + + @SneakyThrows + @Test + public void testGetDataSourceMetadataListWithNoIndex() { + Mockito.when(clusterService.state().routingTable().hasIndex(DATASOURCE_INDEX_NAME)) + .thenReturn(Boolean.FALSE); + Mockito.when(client.admin().indices().create(ArgumentMatchers.any())) + .thenReturn(createIndexResponseActionFuture); + Mockito.when(createIndexResponseActionFuture.actionGet()) + .thenReturn(new CreateIndexResponse(true, true, DATASOURCE_INDEX_NAME)); + Mockito.when(client.index(ArgumentMatchers.any())).thenReturn(indexResponseActionFuture); + + List dataSourceMetadataList + = openSearchDataSourceMetadataStorage.getDataSourceMetadata(); + + Assertions.assertEquals(0, dataSourceMetadataList.size()); + } + + @SneakyThrows + @Test + public void testGetDataSourceMetadataWithNoIndex() { + Mockito.when(clusterService.state().routingTable().hasIndex(DATASOURCE_INDEX_NAME)) + .thenReturn(Boolean.FALSE); + Mockito.when(client.admin().indices().create(ArgumentMatchers.any())) + .thenReturn(createIndexResponseActionFuture); + Mockito.when(createIndexResponseActionFuture.actionGet()) + .thenReturn(new CreateIndexResponse(true, true, DATASOURCE_INDEX_NAME)); + Mockito.when(client.index(ArgumentMatchers.any())).thenReturn(indexResponseActionFuture); + + Optional dataSourceMetadataOptional + = openSearchDataSourceMetadataStorage.getDataSourceMetadata(TEST_DATASOURCE_INDEX_NAME); + + Assertions.assertFalse(dataSourceMetadataOptional.isPresent()); + } + @Test public void testCreateDataSourceMetadata() { @@ -173,11 +280,149 @@ public void testCreateDataSourceMetadata() { } @Test - public void testUpdateDataSourceMetadata() { + public void testCreateDataSourceMetadataWithOutCreatingIndex() { Mockito.when(clusterService.state().routingTable().hasIndex(DATASOURCE_INDEX_NAME)) .thenReturn(Boolean.TRUE); Mockito.when(encryptor.encrypt("secret_key")).thenReturn("secret_key"); Mockito.when(encryptor.encrypt("access_key")).thenReturn("access_key"); + Mockito.when(client.index(ArgumentMatchers.any())).thenReturn(indexResponseActionFuture); + Mockito.when(indexResponseActionFuture.actionGet()).thenReturn(indexResponse); + Mockito.when(indexResponse.getResult()).thenReturn(DocWriteResponse.Result.CREATED); + DataSourceMetadata dataSourceMetadata = getDataSourceMetadata(); + + this.openSearchDataSourceMetadataStorage.createDataSourceMetadata(dataSourceMetadata); + + Mockito.verify(encryptor, Mockito.times(1)).encrypt("secret_key"); + Mockito.verify(encryptor, Mockito.times(1)).encrypt("access_key"); + Mockito.verify(client.admin().indices(), Mockito.times(0)).create(ArgumentMatchers.any()); + Mockito.verify(client, Mockito.times(1)).index(ArgumentMatchers.any()); + Mockito.verify(client.threadPool().getThreadContext(), Mockito.times(1)).stashContext(); + } + + + @Test + public void testCreateDataSourceMetadataFailedWithNotFoundResponse() { + + Mockito.when(clusterService.state().routingTable().hasIndex(DATASOURCE_INDEX_NAME)) + .thenReturn(Boolean.FALSE); + Mockito.when(encryptor.encrypt("secret_key")).thenReturn("secret_key"); + Mockito.when(encryptor.encrypt("access_key")).thenReturn("access_key"); + Mockito.when(client.admin().indices().create(ArgumentMatchers.any())) + .thenReturn(createIndexResponseActionFuture); + Mockito.when(createIndexResponseActionFuture.actionGet()) + .thenReturn(new CreateIndexResponse(true, true, DATASOURCE_INDEX_NAME)); + Mockito.when(client.index(ArgumentMatchers.any())).thenReturn(indexResponseActionFuture); + Mockito.when(indexResponseActionFuture.actionGet()).thenReturn(indexResponse); + Mockito.when(indexResponse.getResult()).thenReturn(DocWriteResponse.Result.NOT_FOUND); + DataSourceMetadata dataSourceMetadata = getDataSourceMetadata(); + + RuntimeException runtimeException = Assertions.assertThrows(RuntimeException.class, + () -> this.openSearchDataSourceMetadataStorage.createDataSourceMetadata( + dataSourceMetadata)); + Assertions.assertEquals("Saving dataSource metadata information failed with result : not_found", + runtimeException.getMessage()); + + Mockito.verify(encryptor, Mockito.times(1)).encrypt("secret_key"); + Mockito.verify(encryptor, Mockito.times(1)).encrypt("access_key"); + Mockito.verify(client.admin().indices(), Mockito.times(1)).create(ArgumentMatchers.any()); + Mockito.verify(client, Mockito.times(1)).index(ArgumentMatchers.any()); + Mockito.verify(client.threadPool().getThreadContext(), Mockito.times(2)).stashContext(); + + + } + + @Test + public void testCreateDataSourceMetadataWithVersionConflict() { + + Mockito.when(clusterService.state().routingTable().hasIndex(DATASOURCE_INDEX_NAME)) + .thenReturn(Boolean.FALSE); + Mockito.when(encryptor.encrypt("secret_key")).thenReturn("secret_key"); + Mockito.when(encryptor.encrypt("access_key")).thenReturn("access_key"); + Mockito.when(client.admin().indices().create(ArgumentMatchers.any())) + .thenReturn(createIndexResponseActionFuture); + Mockito.when(createIndexResponseActionFuture.actionGet()) + .thenReturn(new CreateIndexResponse(true, true, DATASOURCE_INDEX_NAME)); + Mockito.when(client.index(ArgumentMatchers.any())) + .thenThrow(VersionConflictEngineException.class); + DataSourceMetadata dataSourceMetadata = getDataSourceMetadata(); + IllegalArgumentException illegalArgumentException = + Assertions.assertThrows(IllegalArgumentException.class, + () -> this.openSearchDataSourceMetadataStorage.createDataSourceMetadata( + dataSourceMetadata)); + Assertions.assertEquals("A datasource already exists with name: testDS", + illegalArgumentException.getMessage()); + + + Mockito.verify(encryptor, Mockito.times(1)).encrypt("secret_key"); + Mockito.verify(encryptor, Mockito.times(1)).encrypt("access_key"); + Mockito.verify(client.admin().indices(), Mockito.times(1)).create(ArgumentMatchers.any()); + Mockito.verify(client, Mockito.times(1)).index(ArgumentMatchers.any()); + Mockito.verify(client.threadPool().getThreadContext(), Mockito.times(2)).stashContext(); + + + } + + @Test + public void testCreateDataSourceMetadataWithException() { + + Mockito.when(clusterService.state().routingTable().hasIndex(DATASOURCE_INDEX_NAME)) + .thenReturn(Boolean.FALSE); + Mockito.when(encryptor.encrypt("secret_key")).thenReturn("secret_key"); + Mockito.when(encryptor.encrypt("access_key")).thenReturn("access_key"); + Mockito.when(client.admin().indices().create(ArgumentMatchers.any())) + .thenReturn(createIndexResponseActionFuture); + Mockito.when(createIndexResponseActionFuture.actionGet()) + .thenReturn(new CreateIndexResponse(true, true, DATASOURCE_INDEX_NAME)); + Mockito.when(client.index(ArgumentMatchers.any())) + .thenThrow(new RuntimeException("error while indexing")); + DataSourceMetadata dataSourceMetadata = getDataSourceMetadata(); + + RuntimeException runtimeException = Assertions.assertThrows(RuntimeException.class, + () -> this.openSearchDataSourceMetadataStorage.createDataSourceMetadata( + dataSourceMetadata)); + Assertions.assertEquals("java.lang.RuntimeException: error while indexing", + runtimeException.getMessage()); + + Mockito.verify(encryptor, Mockito.times(1)).encrypt("secret_key"); + Mockito.verify(encryptor, Mockito.times(1)).encrypt("access_key"); + Mockito.verify(client.admin().indices(), Mockito.times(1)).create(ArgumentMatchers.any()); + Mockito.verify(client, Mockito.times(1)).index(ArgumentMatchers.any()); + Mockito.verify(client.threadPool().getThreadContext(), Mockito.times(2)).stashContext(); + + + } + + @Test + public void testCreateDataSourceMetadataWithIndexCreationFailed() { + + Mockito.when(clusterService.state().routingTable().hasIndex(DATASOURCE_INDEX_NAME)) + .thenReturn(Boolean.FALSE); + Mockito.when(encryptor.encrypt("secret_key")).thenReturn("secret_key"); + Mockito.when(encryptor.encrypt("access_key")).thenReturn("access_key"); + Mockito.when(client.admin().indices().create(ArgumentMatchers.any())) + .thenReturn(createIndexResponseActionFuture); + Mockito.when(createIndexResponseActionFuture.actionGet()) + .thenReturn(new CreateIndexResponse(false, false, DATASOURCE_INDEX_NAME)); + DataSourceMetadata dataSourceMetadata = getDataSourceMetadata(); + + RuntimeException runtimeException = Assertions.assertThrows(RuntimeException.class, + () -> this.openSearchDataSourceMetadataStorage.createDataSourceMetadata( + dataSourceMetadata)); + Assertions.assertEquals( + "Internal server error while creating.ql-datasources index:: " + + "Index creation is not acknowledged.", + runtimeException.getMessage()); + + Mockito.verify(encryptor, Mockito.times(1)).encrypt("secret_key"); + Mockito.verify(encryptor, Mockito.times(1)).encrypt("access_key"); + Mockito.verify(client.admin().indices(), Mockito.times(1)).create(ArgumentMatchers.any()); + Mockito.verify(client.threadPool().getThreadContext(), Mockito.times(1)).stashContext(); + } + + @Test + public void testUpdateDataSourceMetadata() { + Mockito.when(encryptor.encrypt("secret_key")).thenReturn("secret_key"); + Mockito.when(encryptor.encrypt("access_key")).thenReturn("access_key"); Mockito.when(client.update(ArgumentMatchers.any())).thenReturn(updateResponseActionFuture); Mockito.when(updateResponseActionFuture.actionGet()).thenReturn(updateResponse); Mockito.when(updateResponse.getResult()).thenReturn(DocWriteResponse.Result.UPDATED); @@ -193,12 +438,83 @@ public void testUpdateDataSourceMetadata() { } + @Test + public void testUpdateDataSourceMetadataWithNotFoundResult() { + Mockito.when(encryptor.encrypt("secret_key")).thenReturn("secret_key"); + Mockito.when(encryptor.encrypt("access_key")).thenReturn("access_key"); + Mockito.when(client.update(ArgumentMatchers.any())).thenReturn(updateResponseActionFuture); + Mockito.when(updateResponseActionFuture.actionGet()).thenReturn(updateResponse); + Mockito.when(updateResponse.getResult()).thenReturn(DocWriteResponse.Result.NOT_FOUND); + DataSourceMetadata dataSourceMetadata = getDataSourceMetadata(); + + RuntimeException runtimeException = Assertions.assertThrows(RuntimeException.class, + () -> this.openSearchDataSourceMetadataStorage.updateDataSourceMetadata( + dataSourceMetadata)); + Assertions.assertEquals("Saving dataSource metadata information failed with result : not_found", + runtimeException.getMessage()); + + Mockito.verify(encryptor, Mockito.times(1)).encrypt("secret_key"); + Mockito.verify(encryptor, Mockito.times(1)).encrypt("access_key"); + Mockito.verify(client.admin().indices(), Mockito.times(0)).create(ArgumentMatchers.any()); + Mockito.verify(client, Mockito.times(1)).update(ArgumentMatchers.any()); + Mockito.verify(client.threadPool().getThreadContext(), Mockito.times(1)).stashContext(); + + } + + @Test + public void testUpdateDataSourceMetadataWithDocumentMissingException() { + Mockito.when(encryptor.encrypt("secret_key")).thenReturn("secret_key"); + Mockito.when(encryptor.encrypt("access_key")).thenReturn("access_key"); + Mockito.when(client.update(ArgumentMatchers.any())).thenThrow(new DocumentMissingException( + ShardId.fromString("[2][2]"), "testDS")); + DataSourceMetadata dataSourceMetadata = getDataSourceMetadata(); + dataSourceMetadata.setName("testDS"); + + + DataSourceNotFoundException dataSourceNotFoundException = + Assertions.assertThrows(DataSourceNotFoundException.class, + () -> this.openSearchDataSourceMetadataStorage.updateDataSourceMetadata( + dataSourceMetadata)); + Assertions.assertEquals("Datasource with name: testDS doesn't exist", + dataSourceNotFoundException.getMessage()); + + Mockito.verify(encryptor, Mockito.times(1)).encrypt("secret_key"); + Mockito.verify(encryptor, Mockito.times(1)).encrypt("access_key"); + Mockito.verify(client.admin().indices(), Mockito.times(0)).create(ArgumentMatchers.any()); + Mockito.verify(client, Mockito.times(1)).update(ArgumentMatchers.any()); + Mockito.verify(client.threadPool().getThreadContext(), Mockito.times(1)).stashContext(); + + } + + @Test + public void testUpdateDataSourceMetadataWithRuntimeException() { + Mockito.when(encryptor.encrypt("secret_key")).thenReturn("secret_key"); + Mockito.when(encryptor.encrypt("access_key")).thenReturn("access_key"); + Mockito.when(client.update(ArgumentMatchers.any())) + .thenThrow(new RuntimeException("error message")); + DataSourceMetadata dataSourceMetadata = getDataSourceMetadata(); + dataSourceMetadata.setName("testDS"); + + + RuntimeException runtimeException = Assertions.assertThrows(RuntimeException.class, + () -> this.openSearchDataSourceMetadataStorage.updateDataSourceMetadata( + dataSourceMetadata)); + Assertions.assertEquals("java.lang.RuntimeException: error message", + runtimeException.getMessage()); + + Mockito.verify(encryptor, Mockito.times(1)).encrypt("secret_key"); + Mockito.verify(encryptor, Mockito.times(1)).encrypt("access_key"); + Mockito.verify(client.admin().indices(), Mockito.times(0)).create(ArgumentMatchers.any()); + Mockito.verify(client, Mockito.times(1)).update(ArgumentMatchers.any()); + Mockito.verify(client.threadPool().getThreadContext(), Mockito.times(1)).stashContext(); + + } + @Test public void testDeleteDataSourceMetadata() { Mockito.when(client.delete(ArgumentMatchers.any())).thenReturn(deleteResponseActionFuture); Mockito.when(deleteResponseActionFuture.actionGet()).thenReturn(deleteResponse); Mockito.when(deleteResponse.getResult()).thenReturn(DocWriteResponse.Result.DELETED); - DataSourceMetadata dataSourceMetadata = getDataSourceMetadata(); this.openSearchDataSourceMetadataStorage.deleteDataSourceMetadata("testDS"); @@ -208,6 +524,42 @@ public void testDeleteDataSourceMetadata() { Mockito.verify(client.threadPool().getThreadContext(), Mockito.times(1)).stashContext(); } + @Test + public void testDeleteDataSourceMetadataWhichisAlreadyDeleted() { + Mockito.when(client.delete(ArgumentMatchers.any())).thenReturn(deleteResponseActionFuture); + Mockito.when(deleteResponseActionFuture.actionGet()).thenReturn(deleteResponse); + Mockito.when(deleteResponse.getResult()).thenReturn(DocWriteResponse.Result.NOT_FOUND); + + DataSourceNotFoundException dataSourceNotFoundException = + Assertions.assertThrows(DataSourceNotFoundException.class, + () -> this.openSearchDataSourceMetadataStorage.deleteDataSourceMetadata("testDS")); + Assertions.assertEquals("Datasource with name: testDS doesn't exist", + dataSourceNotFoundException.getMessage()); + + + Mockito.verifyNoInteractions(encryptor); + Mockito.verify(client.admin().indices(), Mockito.times(0)).create(ArgumentMatchers.any()); + Mockito.verify(client, Mockito.times(1)).delete(ArgumentMatchers.any()); + Mockito.verify(client.threadPool().getThreadContext(), Mockito.times(1)).stashContext(); + } + + @Test + public void testDeleteDataSourceMetadataWithUnexpectedResult() { + Mockito.when(client.delete(ArgumentMatchers.any())).thenReturn(deleteResponseActionFuture); + Mockito.when(deleteResponseActionFuture.actionGet()).thenReturn(deleteResponse); + Mockito.when(deleteResponse.getResult()).thenReturn(DocWriteResponse.Result.NOOP); + + RuntimeException runtimeException = Assertions.assertThrows(RuntimeException.class, + () -> this.openSearchDataSourceMetadataStorage.deleteDataSourceMetadata("testDS")); + Assertions.assertEquals("Deleting dataSource metadata information failed with result : noop", + runtimeException.getMessage()); + + Mockito.verifyNoInteractions(encryptor); + Mockito.verify(client.admin().indices(), Mockito.times(0)).create(ArgumentMatchers.any()); + Mockito.verify(client, Mockito.times(1)).delete(ArgumentMatchers.any()); + Mockito.verify(client.threadPool().getThreadContext(), Mockito.times(1)).stashContext(); + } + private String getBasicDataSourceMetadataString() throws JsonProcessingException { DataSourceMetadata dataSourceMetadata = new DataSourceMetadata(); dataSourceMetadata.setName("testDS"); @@ -238,6 +590,18 @@ private String getAWSSigv4DataSourceMetadataString() throws JsonProcessingExcept return objectMapper.writeValueAsString(dataSourceMetadata); } + private String getDataSourceMetadataStringWithNoAuthentication() throws JsonProcessingException { + DataSourceMetadata dataSourceMetadata = new DataSourceMetadata(); + dataSourceMetadata.setName("testDS"); + dataSourceMetadata.setConnector(DataSourceType.PROMETHEUS); + dataSourceMetadata.setAllowedRoles(Collections.singletonList("prometheus_access")); + Map properties = new HashMap<>(); + properties.put("prometheus.auth.uri", "https://localhost:9090"); + dataSourceMetadata.setProperties(properties); + ObjectMapper objectMapper = new ObjectMapper(); + return objectMapper.writeValueAsString(dataSourceMetadata); + } + private DataSourceMetadata getDataSourceMetadata() { DataSourceMetadata dataSourceMetadata = new DataSourceMetadata(); dataSourceMetadata.setName("testDS"); diff --git a/datasources/src/test/java/org/opensearch/sql/datasources/transport/TransportCreateDataSourceActionTest.java b/datasources/src/test/java/org/opensearch/sql/datasources/transport/TransportCreateDataSourceActionTest.java new file mode 100644 index 0000000000..3dd5c21214 --- /dev/null +++ b/datasources/src/test/java/org/opensearch/sql/datasources/transport/TransportCreateDataSourceActionTest.java @@ -0,0 +1,86 @@ +package org.opensearch.sql.datasources.transport; + +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +import java.util.HashSet; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.action.ActionListener; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.sql.datasource.model.DataSourceMetadata; +import org.opensearch.sql.datasource.model.DataSourceType; +import org.opensearch.sql.datasources.model.transport.CreateDataSourceActionRequest; +import org.opensearch.sql.datasources.model.transport.CreateDataSourceActionResponse; +import org.opensearch.sql.datasources.service.DataSourceServiceImpl; +import org.opensearch.tasks.Task; +import org.opensearch.transport.TransportService; + +@ExtendWith(MockitoExtension.class) +public class TransportCreateDataSourceActionTest { + + @Mock + private TransportService transportService; + @Mock + private TransportCreateDataSourceAction action; + @Mock + private DataSourceServiceImpl dataSourceService; + @Mock + private Task task; + @Mock + private ActionListener actionListener; + @Captor + private ArgumentCaptor + createDataSourceActionResponseArgumentCaptor; + + @Captor + private ArgumentCaptor exceptionArgumentCaptor; + + @BeforeEach + public void setUp() { + action = new TransportCreateDataSourceAction(transportService, + new ActionFilters(new HashSet<>()), dataSourceService); + } + + @Test + public void testDoExecute() { + DataSourceMetadata dataSourceMetadata = new DataSourceMetadata(); + dataSourceMetadata.setName("test_datasource"); + dataSourceMetadata.setConnector(DataSourceType.PROMETHEUS); + CreateDataSourceActionRequest request = new CreateDataSourceActionRequest(dataSourceMetadata); + + action.doExecute(task, request, actionListener); + verify(dataSourceService, times(1)).createDataSource(dataSourceMetadata); + Mockito.verify(actionListener) + .onResponse(createDataSourceActionResponseArgumentCaptor.capture()); + CreateDataSourceActionResponse createDataSourceActionResponse + = createDataSourceActionResponseArgumentCaptor.getValue(); + Assertions.assertEquals("Created DataSource with name test_datasource", + createDataSourceActionResponse.getResult()); + } + + @Test + public void testDoExecuteWithException() { + DataSourceMetadata dataSourceMetadata = new DataSourceMetadata(); + dataSourceMetadata.setName("test_datasource"); + dataSourceMetadata.setConnector(DataSourceType.PROMETHEUS); + doThrow(new RuntimeException("Error")).when(dataSourceService) + .createDataSource(dataSourceMetadata); + CreateDataSourceActionRequest request = new CreateDataSourceActionRequest(dataSourceMetadata); + action.doExecute(task, request, actionListener); + verify(dataSourceService, times(1)).createDataSource(dataSourceMetadata); + Mockito.verify(actionListener).onFailure(exceptionArgumentCaptor.capture()); + Exception exception = exceptionArgumentCaptor.getValue(); + Assertions.assertTrue(exception instanceof RuntimeException); + Assertions.assertEquals("Error", + exception.getMessage()); + } +} \ No newline at end of file diff --git a/datasources/src/test/java/org/opensearch/sql/datasources/transport/TransportDeleteDataSourceActionTest.java b/datasources/src/test/java/org/opensearch/sql/datasources/transport/TransportDeleteDataSourceActionTest.java new file mode 100644 index 0000000000..9beeb1a9a9 --- /dev/null +++ b/datasources/src/test/java/org/opensearch/sql/datasources/transport/TransportDeleteDataSourceActionTest.java @@ -0,0 +1,78 @@ +package org.opensearch.sql.datasources.transport; + +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +import java.util.HashSet; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.action.ActionListener; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.sql.datasources.model.transport.DeleteDataSourceActionRequest; +import org.opensearch.sql.datasources.model.transport.DeleteDataSourceActionResponse; +import org.opensearch.sql.datasources.service.DataSourceServiceImpl; +import org.opensearch.tasks.Task; +import org.opensearch.transport.TransportService; + +@ExtendWith(MockitoExtension.class) +public class TransportDeleteDataSourceActionTest { + + @Mock + private TransportService transportService; + @Mock + private TransportDeleteDataSourceAction action; + @Mock + private DataSourceServiceImpl dataSourceService; + @Mock + private Task task; + @Mock + private ActionListener actionListener; + + @Captor + private ArgumentCaptor + deleteDataSourceActionResponseArgumentCaptor; + @Captor + private ArgumentCaptor exceptionArgumentCaptor; + + + @BeforeEach + public void setUp() { + action = new TransportDeleteDataSourceAction(transportService, + new ActionFilters(new HashSet<>()), dataSourceService); + } + + @Test + public void testDoExecute() { + DeleteDataSourceActionRequest request = new DeleteDataSourceActionRequest("test_datasource"); + + action.doExecute(task, request, actionListener); + verify(dataSourceService, times(1)).deleteDataSource("test_datasource"); + Mockito.verify(actionListener) + .onResponse(deleteDataSourceActionResponseArgumentCaptor.capture()); + DeleteDataSourceActionResponse deleteDataSourceActionResponse + = deleteDataSourceActionResponseArgumentCaptor.getValue(); + Assertions.assertEquals("Deleted DataSource with name test_datasource", + deleteDataSourceActionResponse.getResult()); + } + + @Test + public void testDoExecuteWithException() { + doThrow(new RuntimeException("Error")).when(dataSourceService).deleteDataSource("testDS"); + DeleteDataSourceActionRequest request = new DeleteDataSourceActionRequest("testDS"); + action.doExecute(task, request, actionListener); + verify(dataSourceService, times(1)).deleteDataSource("testDS"); + Mockito.verify(actionListener).onFailure(exceptionArgumentCaptor.capture()); + Exception exception = exceptionArgumentCaptor.getValue(); + Assertions.assertTrue(exception instanceof RuntimeException); + Assertions.assertEquals("Error", + exception.getMessage()); + } +} \ No newline at end of file diff --git a/datasources/src/test/java/org/opensearch/sql/datasources/transport/TransportGetDataSourceActionTest.java b/datasources/src/test/java/org/opensearch/sql/datasources/transport/TransportGetDataSourceActionTest.java new file mode 100644 index 0000000000..d5506c0a45 --- /dev/null +++ b/datasources/src/test/java/org/opensearch/sql/datasources/transport/TransportGetDataSourceActionTest.java @@ -0,0 +1,137 @@ +package org.opensearch.sql.datasources.transport; + +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; +import java.lang.reflect.Type; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.action.ActionListener; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.sql.datasource.model.DataSourceMetadata; +import org.opensearch.sql.datasource.model.DataSourceType; +import org.opensearch.sql.datasources.model.transport.GetDataSourceActionRequest; +import org.opensearch.sql.datasources.model.transport.GetDataSourceActionResponse; +import org.opensearch.sql.datasources.service.DataSourceServiceImpl; +import org.opensearch.sql.protocol.response.format.JsonResponseFormatter; +import org.opensearch.tasks.Task; +import org.opensearch.transport.TransportService; + +@ExtendWith(MockitoExtension.class) +public class TransportGetDataSourceActionTest { + + @Mock + private TransportService transportService; + @Mock + private TransportGetDataSourceAction action; + @Mock + private DataSourceServiceImpl dataSourceService; + @Mock + private Task task; + @Mock + private ActionListener actionListener; + + @Captor + private ArgumentCaptor getDataSourceActionResponseArgumentCaptor; + + @Captor + private ArgumentCaptor exceptionArgumentCaptor; + + @BeforeEach + public void setUp() { + action = new TransportGetDataSourceAction(transportService, + new ActionFilters(new HashSet<>()), dataSourceService); + } + + @Test + public void testDoExecute() { + DataSourceMetadata dataSourceMetadata = new DataSourceMetadata(); + dataSourceMetadata.setName("test_datasource"); + dataSourceMetadata.setConnector(DataSourceType.PROMETHEUS); + GetDataSourceActionRequest request = new GetDataSourceActionRequest("test_datasource"); + when(dataSourceService.getDataSourceMetadata("test_datasource")) + .thenReturn(dataSourceMetadata); + + action.doExecute(task, request, actionListener); + verify(dataSourceService, times(1)).getDataSourceMetadata("test_datasource"); + Mockito.verify(actionListener).onResponse(getDataSourceActionResponseArgumentCaptor.capture()); + GetDataSourceActionResponse getDataSourceActionResponse + = getDataSourceActionResponseArgumentCaptor.getValue(); + JsonResponseFormatter dataSourceMetadataJsonResponseFormatter = + new JsonResponseFormatter<>( + JsonResponseFormatter.Style.PRETTY) { + @Override + protected Object buildJsonObject(DataSourceMetadata response) { + return response; + } + }; + Assertions.assertEquals(dataSourceMetadataJsonResponseFormatter.format(dataSourceMetadata), + getDataSourceActionResponse.getResult()); + DataSourceMetadata result = + new Gson().fromJson(getDataSourceActionResponse.getResult(), DataSourceMetadata.class); + Assertions.assertEquals("test_datasource", result.getName()); + Assertions.assertEquals(DataSourceType.PROMETHEUS, result.getConnector()); + } + + @Test + public void testDoExecuteForGetAllDataSources() { + DataSourceMetadata dataSourceMetadata = new DataSourceMetadata(); + dataSourceMetadata.setName("test_datasource"); + dataSourceMetadata.setConnector(DataSourceType.PROMETHEUS); + + GetDataSourceActionRequest request = new GetDataSourceActionRequest(); + when(dataSourceService.getDataSourceMetadata(false)) + .thenReturn(Collections.singleton(dataSourceMetadata)); + + action.doExecute(task, request, actionListener); + verify(dataSourceService, times(1)).getDataSourceMetadata(false); + Mockito.verify(actionListener).onResponse(getDataSourceActionResponseArgumentCaptor.capture()); + GetDataSourceActionResponse getDataSourceActionResponse + = getDataSourceActionResponseArgumentCaptor.getValue(); + JsonResponseFormatter> dataSourceMetadataJsonResponseFormatter = + new JsonResponseFormatter<>( + JsonResponseFormatter.Style.PRETTY) { + @Override + protected Object buildJsonObject(Set response) { + return response; + } + }; + Type setType = new TypeToken>() { + }.getType(); + Assertions.assertEquals( + dataSourceMetadataJsonResponseFormatter.format(Collections.singleton(dataSourceMetadata)), + getDataSourceActionResponse.getResult()); + Set result = + new Gson().fromJson(getDataSourceActionResponse.getResult(), setType); + DataSourceMetadata resultDataSource = result.iterator().next(); + Assertions.assertEquals("test_datasource", resultDataSource.getName()); + Assertions.assertEquals(DataSourceType.PROMETHEUS, resultDataSource.getConnector()); + } + + @Test + public void testDoExecuteWithException() { + doThrow(new RuntimeException("Error")).when(dataSourceService).getDataSourceMetadata("testDS"); + GetDataSourceActionRequest request = new GetDataSourceActionRequest("testDS"); + action.doExecute(task, request, actionListener); + verify(dataSourceService, times(1)).getDataSourceMetadata("testDS"); + Mockito.verify(actionListener).onFailure(exceptionArgumentCaptor.capture()); + Exception exception = exceptionArgumentCaptor.getValue(); + Assertions.assertTrue(exception instanceof RuntimeException); + Assertions.assertEquals("Error", + exception.getMessage()); + } +} \ No newline at end of file diff --git a/datasources/src/test/java/org/opensearch/sql/datasources/transport/TransportUpdateDataSourceActionTest.java b/datasources/src/test/java/org/opensearch/sql/datasources/transport/TransportUpdateDataSourceActionTest.java new file mode 100644 index 0000000000..fecab012d2 --- /dev/null +++ b/datasources/src/test/java/org/opensearch/sql/datasources/transport/TransportUpdateDataSourceActionTest.java @@ -0,0 +1,87 @@ +package org.opensearch.sql.datasources.transport; + +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +import java.util.HashSet; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.action.ActionListener; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.sql.datasource.model.DataSourceMetadata; +import org.opensearch.sql.datasource.model.DataSourceType; +import org.opensearch.sql.datasources.model.transport.UpdateDataSourceActionRequest; +import org.opensearch.sql.datasources.model.transport.UpdateDataSourceActionResponse; +import org.opensearch.sql.datasources.service.DataSourceServiceImpl; +import org.opensearch.tasks.Task; +import org.opensearch.transport.TransportService; + +@ExtendWith(MockitoExtension.class) +public class TransportUpdateDataSourceActionTest { + + @Mock + private TransportService transportService; + @Mock + private TransportUpdateDataSourceAction action; + @Mock + private DataSourceServiceImpl dataSourceService; + @Mock + private Task task; + @Mock + private ActionListener actionListener; + + @Captor + private ArgumentCaptor + updateDataSourceActionResponseArgumentCaptor; + + @Captor + private ArgumentCaptor exceptionArgumentCaptor; + + @BeforeEach + public void setUp() { + action = new TransportUpdateDataSourceAction(transportService, + new ActionFilters(new HashSet<>()), dataSourceService); + } + + @Test + public void testDoExecute() { + DataSourceMetadata dataSourceMetadata = new DataSourceMetadata(); + dataSourceMetadata.setName("test_datasource"); + dataSourceMetadata.setConnector(DataSourceType.PROMETHEUS); + UpdateDataSourceActionRequest request = new UpdateDataSourceActionRequest(dataSourceMetadata); + + action.doExecute(task, request, actionListener); + verify(dataSourceService, times(1)).updateDataSource(dataSourceMetadata); + Mockito.verify(actionListener) + .onResponse(updateDataSourceActionResponseArgumentCaptor.capture()); + UpdateDataSourceActionResponse updateDataSourceActionResponse + = updateDataSourceActionResponseArgumentCaptor.getValue(); + Assertions.assertEquals("Updated DataSource with name test_datasource", + updateDataSourceActionResponse.getResult()); + } + + @Test + public void testDoExecuteWithException() { + DataSourceMetadata dataSourceMetadata = new DataSourceMetadata(); + dataSourceMetadata.setName("test_datasource"); + dataSourceMetadata.setConnector(DataSourceType.PROMETHEUS); + doThrow(new RuntimeException("Error")).when(dataSourceService) + .updateDataSource(dataSourceMetadata); + UpdateDataSourceActionRequest request = new UpdateDataSourceActionRequest(dataSourceMetadata); + action.doExecute(task, request, actionListener); + verify(dataSourceService, times(1)).updateDataSource(dataSourceMetadata); + Mockito.verify(actionListener).onFailure(exceptionArgumentCaptor.capture()); + Exception exception = exceptionArgumentCaptor.getValue(); + Assertions.assertTrue(exception instanceof RuntimeException); + Assertions.assertEquals("Error", + exception.getMessage()); + } +} \ No newline at end of file diff --git a/datasources/src/test/java/org/opensearch/sql/datasources/utils/SchedulerTest.java b/datasources/src/test/java/org/opensearch/sql/datasources/utils/SchedulerTest.java index cb5d18c2c2..d091e77044 100644 --- a/datasources/src/test/java/org/opensearch/sql/datasources/utils/SchedulerTest.java +++ b/datasources/src/test/java/org/opensearch/sql/datasources/utils/SchedulerTest.java @@ -7,16 +7,16 @@ import java.util.concurrent.atomic.AtomicBoolean; import org.junit.Assert; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.ArgumentMatchers; import org.mockito.Mock; import org.mockito.Mockito; -import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.client.node.NodeClient; import org.opensearch.threadpool.ThreadPool; -@RunWith(MockitoJUnitRunner.class) +@ExtendWith(MockitoExtension.class) public class SchedulerTest { @Mock diff --git a/datasources/src/test/java/org/opensearch/sql/datasources/utils/XContentParserUtilsTest.java b/datasources/src/test/java/org/opensearch/sql/datasources/utils/XContentParserUtilsTest.java new file mode 100644 index 0000000000..605d641bda --- /dev/null +++ b/datasources/src/test/java/org/opensearch/sql/datasources/utils/XContentParserUtilsTest.java @@ -0,0 +1,101 @@ +package org.opensearch.sql.datasources.utils; + +import static org.junit.jupiter.api.Assertions.assertThrows; + +import com.google.gson.Gson; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import lombok.SneakyThrows; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.sql.datasource.model.DataSourceMetadata; +import org.opensearch.sql.datasource.model.DataSourceType; + +@ExtendWith(MockitoExtension.class) +public class XContentParserUtilsTest { + + @SneakyThrows + @Test + public void testConvertToXContent() { + DataSourceMetadata dataSourceMetadata = new DataSourceMetadata(); + dataSourceMetadata.setName("testDS"); + dataSourceMetadata.setConnector(DataSourceType.PROMETHEUS); + dataSourceMetadata.setAllowedRoles(List.of("prometheus_access")); + dataSourceMetadata.setProperties(Map.of("prometheus.uri", "https://localhost:9090")); + + XContentBuilder contentBuilder = XContentParserUtils.convertToXContent(dataSourceMetadata); + String contentString = BytesReference.bytes(contentBuilder).utf8ToString(); + Assertions.assertEquals("{\"name\":\"testDS\",\"connector\":\"PROMETHEUS\",\"allowedRoles\":[\"prometheus_access\"],\"properties\":{\"prometheus.uri\":\"https://localhost:9090\"}}", + contentString); + } + + @SneakyThrows + @Test + public void testToDataSourceMetadataFromJson() { + DataSourceMetadata dataSourceMetadata = new DataSourceMetadata(); + dataSourceMetadata.setName("testDS"); + dataSourceMetadata.setConnector(DataSourceType.PROMETHEUS); + dataSourceMetadata.setAllowedRoles(List.of("prometheus_access")); + dataSourceMetadata.setProperties(Map.of("prometheus.uri", "https://localhost:9090")); + Gson gson = new Gson(); + String json = gson.toJson(dataSourceMetadata); + + DataSourceMetadata retrievedMetadata = XContentParserUtils.toDataSourceMetadata(json); + + Assertions.assertEquals(retrievedMetadata, dataSourceMetadata); + Assertions.assertEquals("prometheus_access", retrievedMetadata.getAllowedRoles().get(0)); + + } + + @SneakyThrows + @Test + public void testToDataSourceMetadataFromJsonWithoutName() { + DataSourceMetadata dataSourceMetadata = new DataSourceMetadata(); + dataSourceMetadata.setConnector(DataSourceType.PROMETHEUS); + dataSourceMetadata.setAllowedRoles(List.of("prometheus_access")); + dataSourceMetadata.setProperties(Map.of("prometheus.uri", "https://localhost:9090")); + Gson gson = new Gson(); + String json = gson.toJson(dataSourceMetadata); + + IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, () -> { + XContentParserUtils.toDataSourceMetadata(json); + }); + Assertions.assertEquals("name and connector are required fields.", exception.getMessage()); + } + + @SneakyThrows + @Test + public void testToDataSourceMetadataFromJsonWithoutConnector() { + DataSourceMetadata dataSourceMetadata = new DataSourceMetadata(); + dataSourceMetadata.setName("name"); + dataSourceMetadata.setAllowedRoles(List.of("prometheus_access")); + dataSourceMetadata.setProperties(Map.of("prometheus.uri", "https://localhost:9090")); + Gson gson = new Gson(); + String json = gson.toJson(dataSourceMetadata); + + IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, () -> { + XContentParserUtils.toDataSourceMetadata(json); + }); + Assertions.assertEquals("name and connector are required fields.", exception.getMessage()); + } + + @SneakyThrows + @Test + public void testToDataSourceMetadataFromJsonUsingUnknownObject() { + HashMap hashMap = new HashMap<>(); + hashMap.put("test", "test"); + Gson gson = new Gson(); + String json = gson.toJson(hashMap); + + IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, () -> { + XContentParserUtils.toDataSourceMetadata(json); + }); + Assertions.assertEquals("Unknown field: test", exception.getMessage()); + } + +} diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/StandaloneIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/StandaloneIT.java index 0c900ea234..cca7833d66 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/StandaloneIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/StandaloneIT.java @@ -31,10 +31,10 @@ import org.opensearch.sql.analysis.ExpressionAnalyzer; import org.opensearch.sql.common.response.ResponseListener; import org.opensearch.sql.common.setting.Settings; -import org.opensearch.sql.datasource.DataSourceMetadataStorage; +import org.opensearch.sql.datasources.service.DataSourceMetadataStorage; import org.opensearch.sql.datasource.DataSourceService; -import org.opensearch.sql.datasource.DataSourceServiceImpl; -import org.opensearch.sql.datasource.DataSourceUserAuthorizationHelper; +import org.opensearch.sql.datasources.service.DataSourceServiceImpl; +import org.opensearch.sql.datasources.auth.DataSourceUserAuthorizationHelper; import org.opensearch.sql.datasource.model.DataSourceMetadata; import org.opensearch.sql.executor.ExecutionEngine; import org.opensearch.sql.executor.ExecutionEngine.QueryResponse; diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java b/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java index 69bb701594..fcb66e2e43 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java @@ -46,11 +46,19 @@ import org.opensearch.script.ScriptContext; import org.opensearch.script.ScriptEngine; import org.opensearch.script.ScriptService; -import org.opensearch.sql.datasources.encryptor.EncryptorImpl; -import org.opensearch.sql.datasource.DataSourceMetadataStorage; import org.opensearch.sql.datasource.DataSourceService; -import org.opensearch.sql.datasource.DataSourceServiceImpl; -import org.opensearch.sql.datasource.DataSourceUserAuthorizationHelper; +import org.opensearch.sql.datasources.auth.DataSourceUserAuthorizationHelper; +import org.opensearch.sql.datasources.auth.DataSourceUserAuthorizationHelperImpl; +import org.opensearch.sql.datasources.encryptor.EncryptorImpl; +import org.opensearch.sql.datasources.model.transport.CreateDataSourceActionResponse; +import org.opensearch.sql.datasources.model.transport.DeleteDataSourceActionResponse; +import org.opensearch.sql.datasources.model.transport.GetDataSourceActionResponse; +import org.opensearch.sql.datasources.model.transport.UpdateDataSourceActionResponse; +import org.opensearch.sql.datasources.rest.RestDataSourceQueryAction; +import org.opensearch.sql.datasources.service.DataSourceMetadataStorage; +import org.opensearch.sql.datasources.service.DataSourceServiceImpl; +import org.opensearch.sql.datasources.settings.DataSourceSettings; +import org.opensearch.sql.datasources.storage.OpenSearchDataSourceMetadataStorage; import org.opensearch.sql.datasources.transport.TransportCreateDataSourceAction; import org.opensearch.sql.datasources.transport.TransportDeleteDataSourceAction; import org.opensearch.sql.datasources.transport.TransportGetDataSourceAction; @@ -67,14 +75,6 @@ import org.opensearch.sql.opensearch.storage.script.ExpressionScriptEngine; import org.opensearch.sql.opensearch.storage.serialization.DefaultExpressionSerializer; import org.opensearch.sql.plugin.config.OpenSearchPluginModule; -import org.opensearch.sql.datasources.settings.DataSourceSettings; -import org.opensearch.sql.datasources.auth.DataSourceUserAuthorizationHelperImpl; -import org.opensearch.sql.datasources.storage.OpenSearchDataSourceMetadataStorage; -import org.opensearch.sql.datasources.model.CreateDataSourceActionResponse; -import org.opensearch.sql.datasources.model.DeleteDataSourceActionResponse; -import org.opensearch.sql.datasources.model.GetDataSourceActionResponse; -import org.opensearch.sql.datasources.model.UpdateDataSourceActionResponse; -import org.opensearch.sql.datasources.rest.RestDataSourceQueryAction; import org.opensearch.sql.plugin.rest.RestPPLQueryAction; import org.opensearch.sql.plugin.rest.RestPPLStatsAction; import org.opensearch.sql.plugin.rest.RestQuerySettingsAction; diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryAction.java b/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryAction.java index 6825b2ac92..a5c094e956 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryAction.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryAction.java @@ -21,7 +21,7 @@ import org.opensearch.sql.common.response.ResponseListener; import org.opensearch.sql.common.utils.QueryContext; import org.opensearch.sql.datasource.DataSourceService; -import org.opensearch.sql.datasource.DataSourceServiceImpl; +import org.opensearch.sql.datasources.service.DataSourceServiceImpl; import org.opensearch.sql.executor.ExecutionEngine; import org.opensearch.sql.legacy.metrics.MetricName; import org.opensearch.sql.legacy.metrics.Metrics; diff --git a/prometheus/build.gradle b/prometheus/build.gradle index ca70813e58..b0c05f1bc8 100644 --- a/prometheus/build.gradle +++ b/prometheus/build.gradle @@ -16,6 +16,7 @@ repositories { dependencies { api project(':core') + implementation project(':datasources') implementation "io.github.resilience4j:resilience4j-retry:1.5.0" implementation group: 'com.fasterxml.jackson.core', name: 'jackson-core', version: "${versions.jackson}" implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: "${versions.jackson_databind}" diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusStorageFactory.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusStorageFactory.java index bb95ca93a2..4a0f52f4a5 100644 --- a/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusStorageFactory.java +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusStorageFactory.java @@ -21,7 +21,7 @@ import org.opensearch.sql.datasource.model.DataSource; import org.opensearch.sql.datasource.model.DataSourceMetadata; import org.opensearch.sql.datasource.model.DataSourceType; -import org.opensearch.sql.datasource.model.auth.AuthenticationType; +import org.opensearch.sql.datasources.auth.AuthenticationType; import org.opensearch.sql.prometheus.authinterceptors.AwsSigningInterceptor; import org.opensearch.sql.prometheus.authinterceptors.BasicAuthenticationInterceptor; import org.opensearch.sql.prometheus.client.PrometheusClient;