Refactored all the datasource releated code to new datasources module

Signed-off-by: vamsi-amazon <reddyvam@amazon.com>
opensearch-project · Apr 8, 2023 · a9111a9 · a9111a9
1 parent 68d677a
commit a9111a9
Show file tree

Hide file tree

Showing 48 changed files with 1,206 additions and 179 deletions.
diff --git a/config/checkstyle/google_checks.xml b/config/checkstyle/google_checks.xml
@@ -39,8 +39,9 @@
     <property name="max" value="100"/>
     <property name="ignorePattern" value="^package.*|^import.*|a href|href|http://|https://|ftp://"/>
   </module>
-
+  <module name="SuppressWarningsFilter" />
   <module name="TreeWalker">
+    <module name="SuppressWarningsHolder" />
     <module name="OuterTypeFilename"/>
     <module name="IllegalTokenText">
       <property name="tokens" value="STRING_LITERAL, CHAR_LITERAL"/>

diff --git a/core/src/main/java/org/opensearch/sql/datasource/model/DataSourceMetadata.java b/core/src/main/java/org/opensearch/sql/datasource/model/DataSourceMetadata.java
@@ -12,16 +12,13 @@
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.google.common.collect.ImmutableMap;
-import com.google.gson.Gson;
-import java.io.IOException;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import lombok.AllArgsConstructor;
 import lombok.EqualsAndHashCode;
 import lombok.Getter;
 import lombok.NoArgsConstructor;
-import lombok.RequiredArgsConstructor;
 import lombok.Setter;
 import org.opensearch.sql.datasource.DataSourceService;
 

diff --git a/core/src/main/java/org/opensearch/sql/storage/DataSourceFactory.java b/core/src/main/java/org/opensearch/sql/storage/DataSourceFactory.java
@@ -7,7 +7,6 @@
 
 package org.opensearch.sql.storage;
 
-import java.util.Map;
 import org.opensearch.sql.datasource.DataSourceService;
 import org.opensearch.sql.datasource.model.DataSource;
 import org.opensearch.sql.datasource.model.DataSourceMetadata;

diff --git a/datasources/build.gradle b/datasources/build.gradle
@@ -56,7 +56,11 @@ jacocoTestCoverageVerification {
     violationRules {
         rule {
             element = 'CLASS'
-            excludes = []
+            excludes = [
+                    'org.opensearch.sql.datasources.settings.DataSourceSettings',
+                    'org.opensearch.sql.datasources.exceptions.*',
+                    'org.opensearch.sql.datasources.rest.*'
+            ]
             limit {
                 counter = 'LINE'
                 minimum = 1.0

diff --git a/...source/model/auth/AuthenticationType.java → .../datasources/auth/AuthenticationType.java b/...source/model/auth/AuthenticationType.java → .../datasources/auth/AuthenticationType.java
@@ -5,7 +5,7 @@
  *
  */
 
-package org.opensearch.sql.datasource.model.auth;
+package org.opensearch.sql.datasources.auth;
 
 import java.util.Collections;
 import java.util.HashMap;

diff --git a/...ce/DataSourceUserAuthorizationHelper.java → ...th/DataSourceUserAuthorizationHelper.java b/...ce/DataSourceUserAuthorizationHelper.java → ...th/DataSourceUserAuthorizationHelper.java
@@ -1,6 +1,10 @@
-package org.opensearch.sql.datasource;
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.sql.datasources.auth;
 
-import java.util.List;
 import org.opensearch.sql.datasource.model.DataSourceMetadata;
 
 /**

diff --git a/.../main/java/org/opensearch/sql/datasources/auth/DataSourceUserAuthorizationHelperImpl.java b/.../main/java/org/opensearch/sql/datasources/auth/DataSourceUserAuthorizationHelperImpl.java
@@ -12,7 +12,6 @@
 import org.opensearch.client.Client;
 import org.opensearch.commons.ConfigConstants;
 import org.opensearch.commons.authuser.User;
-import org.opensearch.sql.datasource.DataSourceUserAuthorizationHelper;
 import org.opensearch.sql.datasource.model.DataSourceMetadata;
 
 @AllArgsConstructor

diff --git a/datasources/src/main/java/org/opensearch/sql/datasources/encryptor/EncryptorImpl.java b/datasources/src/main/java/org/opensearch/sql/datasources/encryptor/EncryptorImpl.java
@@ -29,8 +29,8 @@ public String encrypt(String plainText) {
         .build();
 
     JceMasterKey jceMasterKey
-        = JceMasterKey.getInstance(new SecretKeySpec(masterKey.getBytes(), "AES"), "Custom", "",
-        "AES/GCM/NoPadding");
+        = JceMasterKey.getInstance(new SecretKeySpec(masterKey.getBytes(), "AES"), "Custom",
+        "opensearch.config.master.key", "AES/GCM/NoPadding");
 
     final CryptoResult<byte[], JceMasterKey> encryptResult = crypto.encryptData(jceMasterKey,
         plainText.getBytes(StandardCharsets.UTF_8));
@@ -44,8 +44,8 @@ public String decrypt(String encryptedText) {
         .build();
 
     JceMasterKey jceMasterKey
-        = JceMasterKey.getInstance(new SecretKeySpec(masterKey.getBytes(), "AES"), "Custom", "",
-        "AES/GCM/NoPadding");
+        = JceMasterKey.getInstance(new SecretKeySpec(masterKey.getBytes(), "AES"), "Custom",
+        "opensearch.config.master.key", "AES/GCM/NoPadding");
 
     final CryptoResult<byte[], JceMasterKey> decryptedResult
         = crypto.decryptData(jceMasterKey, Base64.getDecoder().decode(encryptedText));

diff --git a/...ceptions/DataSourceNotFoundException.java → ...ceptions/DataSourceNotFoundException.java b/...ceptions/DataSourceNotFoundException.java → ...ceptions/DataSourceNotFoundException.java
@@ -5,7 +5,7 @@
  *
  */
 
-package org.opensearch.sql.datasource.exceptions;
+package org.opensearch.sql.datasources.exceptions;
 
 /**
  * DataSourceNotFoundException.

diff --git a/datasources/src/main/java/org/opensearch/sql/datasources/exceptions/ErrorMessage.java b/datasources/src/main/java/org/opensearch/sql/datasources/exceptions/ErrorMessage.java
@@ -0,0 +1,78 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+
+package org.opensearch.sql.datasources.exceptions;
+
+import com.google.gson.Gson;
+import com.google.gson.JsonObject;
+import lombok.Getter;
+import org.opensearch.rest.RestStatus;
+
+/**
+ * Error Message.
+ */
+public class ErrorMessage {
+
+  protected Throwable exception;
+
+  private final int status;
+
+  @Getter
+  private final String type;
+
+  @Getter
+  private final String reason;
+
+  @Getter
+  private final String details;
+
+  /**
+   * Error Message Constructor.
+   */
+  public ErrorMessage(Throwable exception, int status) {
+    this.exception = exception;
+    this.status = status;
+
+    this.type = fetchType();
+    this.reason = fetchReason();
+    this.details = fetchDetails();
+  }
+
+  private String fetchType() {
+    return exception.getClass().getSimpleName();
+  }
+
+  protected String fetchReason() {
+    return status == RestStatus.BAD_REQUEST.getStatus()
+        ? "Invalid Request"
+        : "There was internal problem at backend";
+  }
+
+  protected String fetchDetails() {
+    // Some exception prints internal information (full class name) which is security concern
+    return emptyStringIfNull(exception.getLocalizedMessage());
+  }
+
+  private String emptyStringIfNull(String str) {
+    return str != null ? str : "";
+  }
+
+  @Override
+  public String toString() {
+    JsonObject jsonObject = new JsonObject();
+    jsonObject.addProperty("status", status);
+    jsonObject.add("error", getErrorAsJson());
+    return new Gson().toJson(jsonObject);
+  }
+
+  private JsonObject getErrorAsJson() {
+    JsonObject errorJson = new JsonObject();
+    errorJson.addProperty("type", type);
+    errorJson.addProperty("reason", reason);
+    errorJson.addProperty("details", details);
+    return errorJson;
+  }
+}
diff --git a/.../model/CreateDataSourceActionRequest.java → ...nsport/CreateDataSourceActionRequest.java b/.../model/CreateDataSourceActionRequest.java → ...nsport/CreateDataSourceActionRequest.java
@@ -5,7 +5,7 @@
  *
  */
 
-package org.opensearch.sql.datasources.model;
+package org.opensearch.sql.datasources.model.transport;
 
 
 import static org.opensearch.sql.analysis.DataSourceSchemaIdentifierNameResolver.DEFAULT_DATASOURCE_NAME;

diff --git a/...model/CreateDataSourceActionResponse.java → ...sport/CreateDataSourceActionResponse.java b/...model/CreateDataSourceActionResponse.java → ...sport/CreateDataSourceActionResponse.java
@@ -5,7 +5,7 @@
  *
  */
 
-package org.opensearch.sql.datasources.model;
+package org.opensearch.sql.datasources.model.transport;
 
 import java.io.IOException;
 import lombok.Getter;

diff --git a/.../model/DeleteDataSourceActionRequest.java → ...nsport/DeleteDataSourceActionRequest.java b/.../model/DeleteDataSourceActionRequest.java → ...nsport/DeleteDataSourceActionRequest.java
@@ -5,7 +5,7 @@
  *
  */
 
-package org.opensearch.sql.datasources.model;
+package org.opensearch.sql.datasources.model.transport;
 
 import static org.opensearch.sql.analysis.DataSourceSchemaIdentifierNameResolver.DEFAULT_DATASOURCE_NAME;
 

diff --git a/...model/DeleteDataSourceActionResponse.java → ...sport/DeleteDataSourceActionResponse.java b/...model/DeleteDataSourceActionResponse.java → ...sport/DeleteDataSourceActionResponse.java
@@ -5,7 +5,7 @@
  *
  */
 
-package org.opensearch.sql.datasources.model;
+package org.opensearch.sql.datasources.model.transport;
 
 import java.io.IOException;
 import lombok.Getter;

diff --git a/...ces/model/GetDataSourceActionRequest.java → ...transport/GetDataSourceActionRequest.java b/...ces/model/GetDataSourceActionRequest.java → ...transport/GetDataSourceActionRequest.java
@@ -5,7 +5,7 @@
  *
  */
 
-package org.opensearch.sql.datasources.model;
+package org.opensearch.sql.datasources.model.transport;
 
 import static org.opensearch.sql.analysis.DataSourceSchemaIdentifierNameResolver.DEFAULT_DATASOURCE_NAME;
 

diff --git a/...es/model/GetDataSourceActionResponse.java → ...ransport/GetDataSourceActionResponse.java b/...es/model/GetDataSourceActionResponse.java → ...ransport/GetDataSourceActionResponse.java
@@ -5,7 +5,7 @@
  *
  */
 
-package org.opensearch.sql.datasources.model;
+package org.opensearch.sql.datasources.model.transport;
 
 import java.io.IOException;
 import lombok.Getter;

diff --git a/.../model/UpdateDataSourceActionRequest.java → ...nsport/UpdateDataSourceActionRequest.java b/.../model/UpdateDataSourceActionRequest.java → ...nsport/UpdateDataSourceActionRequest.java
@@ -5,7 +5,7 @@
  *
  */
 
-package org.opensearch.sql.datasources.model;
+package org.opensearch.sql.datasources.model.transport;
 
 
 import static org.opensearch.sql.analysis.DataSourceSchemaIdentifierNameResolver.DEFAULT_DATASOURCE_NAME;

diff --git a/...model/UpdateDataSourceActionResponse.java → ...sport/UpdateDataSourceActionResponse.java b/...model/UpdateDataSourceActionResponse.java → ...sport/UpdateDataSourceActionResponse.java
@@ -5,7 +5,7 @@
  *
  */
 
-package org.opensearch.sql.datasources.model;
+package org.opensearch.sql.datasources.model.transport;
 
 import java.io.IOException;
 import lombok.Getter;

diff --git a/datasources/src/main/java/org/opensearch/sql/datasources/rest/RestDataSourceQueryAction.java b/datasources/src/main/java/org/opensearch/sql/datasources/rest/RestDataSourceQueryAction.java
@@ -28,23 +28,23 @@
 import org.opensearch.rest.RestChannel;
 import org.opensearch.rest.RestRequest;
 import org.opensearch.rest.RestStatus;
-import org.opensearch.sql.datasource.exceptions.DataSourceNotFoundException;
 import org.opensearch.sql.datasource.model.DataSourceMetadata;
+import org.opensearch.sql.datasources.exceptions.DataSourceNotFoundException;
+import org.opensearch.sql.datasources.exceptions.ErrorMessage;
+import org.opensearch.sql.datasources.model.transport.CreateDataSourceActionRequest;
+import org.opensearch.sql.datasources.model.transport.CreateDataSourceActionResponse;
+import org.opensearch.sql.datasources.model.transport.DeleteDataSourceActionRequest;
+import org.opensearch.sql.datasources.model.transport.DeleteDataSourceActionResponse;
+import org.opensearch.sql.datasources.model.transport.GetDataSourceActionRequest;
+import org.opensearch.sql.datasources.model.transport.GetDataSourceActionResponse;
+import org.opensearch.sql.datasources.model.transport.UpdateDataSourceActionRequest;
+import org.opensearch.sql.datasources.model.transport.UpdateDataSourceActionResponse;
 import org.opensearch.sql.datasources.transport.TransportCreateDataSourceAction;
 import org.opensearch.sql.datasources.transport.TransportDeleteDataSourceAction;
 import org.opensearch.sql.datasources.transport.TransportGetDataSourceAction;
 import org.opensearch.sql.datasources.transport.TransportUpdateDataSourceAction;
 import org.opensearch.sql.datasources.utils.Scheduler;
 import org.opensearch.sql.datasources.utils.XContentParserUtils;
-import org.opensearch.sql.opensearch.response.error.ErrorMessageFactory;
-import org.opensearch.sql.datasources.model.CreateDataSourceActionRequest;
-import org.opensearch.sql.datasources.model.CreateDataSourceActionResponse;
-import org.opensearch.sql.datasources.model.DeleteDataSourceActionRequest;
-import org.opensearch.sql.datasources.model.DeleteDataSourceActionResponse;
-import org.opensearch.sql.datasources.model.GetDataSourceActionRequest;
-import org.opensearch.sql.datasources.model.GetDataSourceActionResponse;
-import org.opensearch.sql.datasources.model.UpdateDataSourceActionRequest;
-import org.opensearch.sql.datasources.model.UpdateDataSourceActionResponse;
 
 
 public class RestDataSourceQueryAction extends BaseRestHandler {
@@ -237,7 +237,7 @@ private void handleException(Exception e, RestChannel restChannel) {
   private void reportError(final RestChannel channel, final Exception e, final RestStatus status) {
     channel.sendResponse(
         new BytesRestResponse(
-            status, ErrorMessageFactory.createErrorMessage(e, status.getStatus()).toString()));
+            status, new ErrorMessage(e, status.getStatus()).toString()));
   }
 
   private static boolean isClientError(Exception e) {

diff --git a/...sql/datasource/DataSourceLoaderCache.java → ...ources/service/DataSourceLoaderCache.java b/...sql/datasource/DataSourceLoaderCache.java → ...ources/service/DataSourceLoaderCache.java
@@ -1,4 +1,4 @@
-package org.opensearch.sql.datasource;
+package org.opensearch.sql.datasources.service;
 
 import org.opensearch.sql.datasource.model.DataSource;
 import org.opensearch.sql.datasource.model.DataSourceMetadata;

diff --git a/...datasource/DataSourceLoaderCacheImpl.java → ...es/service/DataSourceLoaderCacheImpl.java b/...datasource/DataSourceLoaderCacheImpl.java → ...es/service/DataSourceLoaderCacheImpl.java
@@ -1,4 +1,4 @@
-package org.opensearch.sql.datasource;
+package org.opensearch.sql.datasources.service;
 
 import com.google.common.cache.Cache;
 import com.google.common.cache.CacheBuilder;

diff --git a/...datasource/DataSourceMetadataStorage.java → ...es/service/DataSourceMetadataStorage.java b/...datasource/DataSourceMetadataStorage.java → ...es/service/DataSourceMetadataStorage.java
@@ -5,11 +5,10 @@
  *
  */
 
-package org.opensearch.sql.datasource;
+package org.opensearch.sql.datasources.service;
 
 import java.util.List;
 import java.util.Optional;
-import javax.xml.crypto.Data;
 import org.opensearch.sql.datasource.model.DataSource;
 import org.opensearch.sql.datasource.model.DataSourceMetadata;
 

diff --git a/...sql/datasource/DataSourceServiceImpl.java → ...ources/service/DataSourceServiceImpl.java b/...sql/datasource/DataSourceServiceImpl.java → ...ources/service/DataSourceServiceImpl.java
@@ -3,30 +3,24 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-package org.opensearch.sql.datasource;
+package org.opensearch.sql.datasources.service;
 
 import static org.opensearch.sql.analysis.DataSourceSchemaIdentifierNameResolver.DEFAULT_DATASOURCE_NAME;
 
 import com.google.common.base.Preconditions;
 import com.google.common.base.Strings;
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
-import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.stream.Collectors;
-import javax.xml.crypto.Data;
 import org.opensearch.sql.common.utils.StringUtils;
-import org.opensearch.sql.datasource.exceptions.DataSourceNotFoundException;
+import org.opensearch.sql.datasource.DataSourceService;
 import org.opensearch.sql.datasource.model.DataSource;
 import org.opensearch.sql.datasource.model.DataSourceMetadata;
-import org.opensearch.sql.datasource.model.DataSourceType;
+import org.opensearch.sql.datasources.auth.DataSourceUserAuthorizationHelper;
+import org.opensearch.sql.datasources.exceptions.DataSourceNotFoundException;
 import org.opensearch.sql.storage.DataSourceFactory;
 
 /**
-Original file line number
+Diff line change
@@ Expand Up / @@ -5,7 +5,7 @@ @@
      *
      */
-    package org.opensearch.sql.datasource.exceptions;
+    package org.opensearch.sql.datasources.exceptions;
     /**
      * DataSourceNotFoundException.
@@ Expand Down @@