Reafactor into new datasources module

Signed-off-by: vamsi-amazon <reddyvam@amazon.com>
opensearch-project · Apr 6, 2023 · 68d677a · 68d677a
1 parent 3fbbaa4
commit 68d677a
Show file tree

Hide file tree

Showing 29 changed files with 233 additions and 180 deletions.
diff --git a/common/build.gradle b/common/build.gradle
@@ -36,7 +36,6 @@ dependencies {
     api group: 'com.google.guava', name: 'guava', version: '31.0.1-jre'
     api group: 'org.apache.logging.log4j', name: 'log4j-core', version:'2.17.1'
     api group: 'org.apache.commons', name: 'commons-lang3', version: '3.12.0'
-    api 'com.amazonaws:aws-encryption-sdk-java:2.4.0'
 
     testImplementation group: 'junit', name: 'junit', version: '4.13.2'
     testImplementation group: 'org.assertj', name: 'assertj-core', version: '3.9.1'

diff --git a/datasources/build.gradle b/datasources/build.gradle
@@ -0,0 +1,77 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+plugins {
+    id 'java-library'
+    id "io.freefair.lombok"
+    id 'jacoco'
+}
+
+repositories {
+    mavenCentral()
+}
+
+dependencies {
+    implementation project(':core')
+    implementation project(':protocol')
+    implementation group: 'org.opensearch', name: 'opensearch', version: "${opensearch_version}"
+    implementation group: 'org.opensearch', name: 'opensearch-x-content', version: "${opensearch_version}"
+    implementation group: 'org.opensearch', name: 'common-utils', version: "${opensearch_build}"
+    implementation group: 'commons-io', name: 'commons-io', version: '2.8.0'
+    implementation 'com.amazonaws:aws-encryption-sdk-java:2.4.0'
+
+    testImplementation group: 'junit', name: 'junit', version: '4.13.2'
+    testImplementation('org.junit.jupiter:junit-jupiter:5.6.2')
+    testImplementation group: 'net.bytebuddy', name: 'byte-buddy-agent', version: '1.12.13'
+    testImplementation group: 'org.hamcrest', name: 'hamcrest-library', version: '2.1'
+    testImplementation group: 'org.mockito', name: 'mockito-core', version: '5.2.0'
+    testImplementation group: 'org.mockito', name: 'mockito-junit-jupiter', version: '5.2.0'
+    testImplementation 'org.junit.jupiter:junit-jupiter:5.6.2'
+}
+
+test {
+    useJUnitPlatform()
+    testLogging {
+        events "passed", "skipped", "failed"
+        exceptionFormat "full"
+    }
+}
+
+jacocoTestReport {
+    reports {
+        html.enabled true
+        xml.enabled true
+    }
+    afterEvaluate {
+        classDirectories.setFrom(files(classDirectories.files.collect {
+            fileTree(dir: it)
+        }))
+    }
+}
+test.finalizedBy(project.tasks.jacocoTestReport)
+
+jacocoTestCoverageVerification {
+    violationRules {
+        rule {
+            element = 'CLASS'
+            excludes = []
+            limit {
+                counter = 'LINE'
+                minimum = 1.0
+            }
+            limit {
+                counter = 'BRANCH'
+                minimum = 1.0
+            }
+        }
+    }
+    afterEvaluate {
+        classDirectories.setFrom(files(classDirectories.files.collect {
+            fileTree(dir: it)
+        }))
+    }
+}
+check.dependsOn jacocoTestCoverageVerification
+jacocoTestCoverageVerification.dependsOn jacocoTestReport
diff --git a/datasources/lombok.config b/datasources/lombok.config
@@ -0,0 +1,3 @@
+# This file is generated by the 'io.freefair.lombok' Gradle plugin
+config.stopBubbling = true
+lombok.addLombokGeneratedAnnotation = true
diff --git a/...ataSourceUserAuthorizationHelperImpl.java → ...ataSourceUserAuthorizationHelperImpl.java b/...ataSourceUserAuthorizationHelperImpl.java → ...ataSourceUserAuthorizationHelperImpl.java
@@ -3,14 +3,14 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-package org.opensearch.sql.plugin.datasource;
+package org.opensearch.sql.datasources.auth;
 
-import static org.opensearch.commons.ConfigConstants.OPENSEARCH_SECURITY_USER_INFO_THREAD_CONTEXT;
 import static org.opensearch.sql.analysis.DataSourceSchemaIdentifierNameResolver.DEFAULT_DATASOURCE_NAME;
 
 import java.util.List;
 import lombok.AllArgsConstructor;
 import org.opensearch.client.Client;
+import org.opensearch.commons.ConfigConstants;
 import org.opensearch.commons.authuser.User;
 import org.opensearch.sql.datasource.DataSourceUserAuthorizationHelper;
 import org.opensearch.sql.datasource.model.DataSourceMetadata;
@@ -21,13 +21,15 @@ public class DataSourceUserAuthorizationHelperImpl implements DataSourceUserAuth
 
   private Boolean isAuthorizationRequired() {
     String userString = client.threadPool()
-        .getThreadContext().getTransient(OPENSEARCH_SECURITY_USER_INFO_THREAD_CONTEXT);
+        .getThreadContext().getTransient(
+            ConfigConstants.OPENSEARCH_SECURITY_USER_INFO_THREAD_CONTEXT);
     return userString != null;
   }
 
   private List<String> getUserRoles() {
     String userString = client.threadPool()
-        .getThreadContext().getTransient(OPENSEARCH_SECURITY_USER_INFO_THREAD_CONTEXT);
+        .getThreadContext().getTransient(
+            ConfigConstants.OPENSEARCH_SECURITY_USER_INFO_THREAD_CONTEXT);
     User user = User.parse(userString);
     return user.getRoles();
   }

diff --git a/...earch/sql/common/encryptor/Encryptor.java → .../sql/datasources/encryptor/Encryptor.java b/...earch/sql/common/encryptor/Encryptor.java → .../sql/datasources/encryptor/Encryptor.java
@@ -5,7 +5,7 @@
  *
  */
 
-package org.opensearch.sql.common.encryptor;
+package org.opensearch.sql.datasources.encryptor;
 
 public interface Encryptor {
 

diff --git a/...h/sql/common/encryptor/EncryptorImpl.java → .../datasources/encryptor/EncryptorImpl.java b/...h/sql/common/encryptor/EncryptorImpl.java → .../datasources/encryptor/EncryptorImpl.java
@@ -5,7 +5,7 @@
  *
  */
 
-package org.opensearch.sql.common.encryptor;
+package org.opensearch.sql.datasources.encryptor;
 
 import com.amazonaws.encryptionsdk.AwsCrypto;
 import com.amazonaws.encryptionsdk.CommitmentPolicy;

diff --git a/.../model/CreateDataSourceActionRequest.java → .../model/CreateDataSourceActionRequest.java b/.../model/CreateDataSourceActionRequest.java → .../model/CreateDataSourceActionRequest.java
@@ -5,7 +5,7 @@
  *
  */
 
-package org.opensearch.sql.plugin.model;
+package org.opensearch.sql.datasources.model;
 
 
 import static org.opensearch.sql.analysis.DataSourceSchemaIdentifierNameResolver.DEFAULT_DATASOURCE_NAME;

diff --git a/...model/CreateDataSourceActionResponse.java → ...model/CreateDataSourceActionResponse.java b/...model/CreateDataSourceActionResponse.java → ...model/CreateDataSourceActionResponse.java
@@ -5,7 +5,7 @@
  *
  */
 
-package org.opensearch.sql.plugin.model;
+package org.opensearch.sql.datasources.model;
 
 import java.io.IOException;
 import lombok.Getter;

diff --git a/.../model/DeleteDataSourceActionRequest.java → .../model/DeleteDataSourceActionRequest.java b/.../model/DeleteDataSourceActionRequest.java → .../model/DeleteDataSourceActionRequest.java
@@ -5,7 +5,7 @@
  *
  */
 
-package org.opensearch.sql.plugin.model;
+package org.opensearch.sql.datasources.model;
 
 import static org.opensearch.sql.analysis.DataSourceSchemaIdentifierNameResolver.DEFAULT_DATASOURCE_NAME;
 

diff --git a/...model/DeleteDataSourceActionResponse.java → ...model/DeleteDataSourceActionResponse.java b/...model/DeleteDataSourceActionResponse.java → ...model/DeleteDataSourceActionResponse.java
@@ -5,7 +5,7 @@
  *
  */
 
-package org.opensearch.sql.plugin.model;
+package org.opensearch.sql.datasources.model;
 
 import java.io.IOException;
 import lombok.Getter;

diff --git a/...gin/model/GetDataSourceActionRequest.java → ...ces/model/GetDataSourceActionRequest.java b/...gin/model/GetDataSourceActionRequest.java → ...ces/model/GetDataSourceActionRequest.java
@@ -5,7 +5,7 @@
  *
  */
 
-package org.opensearch.sql.plugin.model;
+package org.opensearch.sql.datasources.model;
 
 import static org.opensearch.sql.analysis.DataSourceSchemaIdentifierNameResolver.DEFAULT_DATASOURCE_NAME;
 

diff --git a/...in/model/GetDataSourceActionResponse.java → ...es/model/GetDataSourceActionResponse.java b/...in/model/GetDataSourceActionResponse.java → ...es/model/GetDataSourceActionResponse.java
@@ -5,7 +5,7 @@
  *
  */
 
-package org.opensearch.sql.plugin.model;
+package org.opensearch.sql.datasources.model;
 
 import java.io.IOException;
 import lombok.Getter;

diff --git a/.../model/UpdateDataSourceActionRequest.java → .../model/UpdateDataSourceActionRequest.java b/.../model/UpdateDataSourceActionRequest.java → .../model/UpdateDataSourceActionRequest.java
@@ -5,19 +5,17 @@
  *
  */
 
-package org.opensearch.sql.plugin.model;
+package org.opensearch.sql.datasources.model;
 
 
 import static org.opensearch.sql.analysis.DataSourceSchemaIdentifierNameResolver.DEFAULT_DATASOURCE_NAME;
 
 import java.io.IOException;
 import lombok.Getter;
-import org.json.JSONObject;
 import org.opensearch.action.ActionRequest;
 import org.opensearch.action.ActionRequestValidationException;
 import org.opensearch.common.io.stream.StreamInput;
 import org.opensearch.sql.datasource.model.DataSourceMetadata;
-import org.opensearch.sql.protocol.response.format.JsonResponseFormatter;
 
 public class UpdateDataSourceActionRequest
     extends ActionRequest {

diff --git a/...model/UpdateDataSourceActionResponse.java → ...model/UpdateDataSourceActionResponse.java b/...model/UpdateDataSourceActionResponse.java → ...model/UpdateDataSourceActionResponse.java
@@ -5,7 +5,7 @@
  *
  */
 
-package org.opensearch.sql.plugin.model;
+package org.opensearch.sql.datasources.model;
 
 import java.io.IOException;
 import lombok.Getter;

diff --git a/...lugin/rest/RestDataSourceQueryAction.java → ...urces/rest/RestDataSourceQueryAction.java b/...lugin/rest/RestDataSourceQueryAction.java → ...urces/rest/RestDataSourceQueryAction.java
@@ -5,7 +5,7 @@
  *
  */
 
-package org.opensearch.sql.plugin.rest;
+package org.opensearch.sql.datasources.rest;
 
 import static org.opensearch.rest.RestRequest.Method.DELETE;
 import static org.opensearch.rest.RestRequest.Method.GET;
@@ -14,7 +14,6 @@
 import static org.opensearch.rest.RestStatus.BAD_REQUEST;
 import static org.opensearch.rest.RestStatus.NOT_FOUND;
 import static org.opensearch.rest.RestStatus.SERVICE_UNAVAILABLE;
-import static org.opensearch.sql.plugin.utils.Scheduler.schedule;
 
 import com.google.common.collect.ImmutableList;
 import java.io.IOException;
@@ -31,22 +30,22 @@
 import org.opensearch.rest.RestStatus;
 import org.opensearch.sql.datasource.exceptions.DataSourceNotFoundException;
 import org.opensearch.sql.datasource.model.DataSourceMetadata;
-import org.opensearch.sql.legacy.metrics.MetricName;
-import org.opensearch.sql.legacy.metrics.Metrics;
+import org.opensearch.sql.datasources.transport.TransportCreateDataSourceAction;
+import org.opensearch.sql.datasources.transport.TransportDeleteDataSourceAction;
+import org.opensearch.sql.datasources.transport.TransportGetDataSourceAction;
+import org.opensearch.sql.datasources.transport.TransportUpdateDataSourceAction;
+import org.opensearch.sql.datasources.utils.Scheduler;
+import org.opensearch.sql.datasources.utils.XContentParserUtils;
 import org.opensearch.sql.opensearch.response.error.ErrorMessageFactory;
-import org.opensearch.sql.plugin.model.CreateDataSourceActionRequest;
-import org.opensearch.sql.plugin.model.CreateDataSourceActionResponse;
-import org.opensearch.sql.plugin.model.DeleteDataSourceActionRequest;
-import org.opensearch.sql.plugin.model.DeleteDataSourceActionResponse;
-import org.opensearch.sql.plugin.model.GetDataSourceActionRequest;
-import org.opensearch.sql.plugin.model.GetDataSourceActionResponse;
-import org.opensearch.sql.plugin.model.UpdateDataSourceActionRequest;
-import org.opensearch.sql.plugin.model.UpdateDataSourceActionResponse;
-import org.opensearch.sql.plugin.transport.datasource.TransportCreateDataSourceAction;
-import org.opensearch.sql.plugin.transport.datasource.TransportDeleteDataSourceAction;
-import org.opensearch.sql.plugin.transport.datasource.TransportGetDataSourceAction;
-import org.opensearch.sql.plugin.transport.datasource.TransportUpdateDataSourceAction;
-import org.opensearch.sql.plugin.utils.XContentParserUtils;
+import org.opensearch.sql.datasources.model.CreateDataSourceActionRequest;
+import org.opensearch.sql.datasources.model.CreateDataSourceActionResponse;
+import org.opensearch.sql.datasources.model.DeleteDataSourceActionRequest;
+import org.opensearch.sql.datasources.model.DeleteDataSourceActionResponse;
+import org.opensearch.sql.datasources.model.GetDataSourceActionRequest;
+import org.opensearch.sql.datasources.model.GetDataSourceActionResponse;
+import org.opensearch.sql.datasources.model.UpdateDataSourceActionRequest;
+import org.opensearch.sql.datasources.model.UpdateDataSourceActionResponse;
+
 
 public class RestDataSourceQueryAction extends BaseRestHandler {
 
@@ -136,7 +135,7 @@ private RestChannelConsumer executePostRequest(RestRequest restRequest,
 
     DataSourceMetadata dataSourceMetadata
         = XContentParserUtils.toDataSourceMetadata(restRequest.contentParser());
-    return restChannel -> schedule(nodeClient,
+    return restChannel -> Scheduler.schedule(nodeClient,
         () -> nodeClient.execute(TransportCreateDataSourceAction.ACTION_TYPE,
             new CreateDataSourceActionRequest(dataSourceMetadata),
             new ActionListener<>() {
@@ -158,7 +157,7 @@ public void onFailure(Exception e) {
   private RestChannelConsumer executeGetRequest(RestRequest restRequest,
                                                 NodeClient nodeClient) {
     String dataSourceName = restRequest.param("dataSourceName");
-    return restChannel -> schedule(nodeClient,
+    return restChannel -> Scheduler.schedule(nodeClient,
         () -> nodeClient.execute(TransportGetDataSourceAction.ACTION_TYPE,
             new GetDataSourceActionRequest(dataSourceName),
             new ActionListener<>() {
@@ -180,7 +179,7 @@ private RestChannelConsumer executeUpdateRequest(RestRequest restRequest,
                                                    NodeClient nodeClient) throws IOException {
     DataSourceMetadata dataSourceMetadata
         = XContentParserUtils.toDataSourceMetadata(restRequest.contentParser());
-    return restChannel -> schedule(nodeClient,
+    return restChannel -> Scheduler.schedule(nodeClient,
         () -> nodeClient.execute(TransportUpdateDataSourceAction.ACTION_TYPE,
             new UpdateDataSourceActionRequest(dataSourceMetadata),
             new ActionListener<>() {
@@ -203,7 +202,7 @@ private RestChannelConsumer executeDeleteRequest(RestRequest restRequest,
                                                    NodeClient nodeClient) {
 
     String dataSourceName = restRequest.param("dataSourceName");
-    return restChannel -> schedule(nodeClient,
+    return restChannel -> Scheduler.schedule(nodeClient,
         () -> nodeClient.execute(TransportDeleteDataSourceAction.ACTION_TYPE,
             new DeleteDataSourceActionRequest(dataSourceName),
             new ActionListener<>() {
@@ -228,14 +227,8 @@ private void handleException(Exception e, RestChannel restChannel) {
     } else {
       LOG.error("Error happened during request handling", e);
       if (isClientError(e)) {
-        Metrics.getInstance()
-            .getNumericalMetric(MetricName.DATASOURCE_FAILED_REQ_COUNT_CUS)
-            .increment();
         reportError(restChannel, e, BAD_REQUEST);
       } else {
-        Metrics.getInstance()
-            .getNumericalMetric(MetricName.DATASOURCE_FAILED_REQ_COUNT_SYS)
-            .increment();
         reportError(restChannel, e, SERVICE_UNAVAILABLE);
       }
     }

diff --git a/...plugin/datasource/DataSourceSettings.java → ...asources/settings/DataSourceSettings.java b/...plugin/datasource/DataSourceSettings.java → ...asources/settings/DataSourceSettings.java
@@ -3,7 +3,7 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-package org.opensearch.sql.plugin.datasource;
+package org.opensearch.sql.datasources.settings;
 
 import java.io.InputStream;
 import org.opensearch.common.settings.SecureSetting;

diff --git a/.../OpenSearchDataSourceMetadataStorage.java → .../OpenSearchDataSourceMetadataStorage.java b/.../OpenSearchDataSourceMetadataStorage.java → .../OpenSearchDataSourceMetadataStorage.java
@@ -5,7 +5,7 @@
  *
  */
 
-package org.opensearch.sql.plugin.datasource;
+package org.opensearch.sql.datasources.storage;
 
 import java.io.IOException;
 import java.io.InputStream;
@@ -43,12 +43,12 @@
 import org.opensearch.index.query.QueryBuilders;
 import org.opensearch.search.SearchHit;
 import org.opensearch.search.builder.SearchSourceBuilder;
-import org.opensearch.sql.common.encryptor.Encryptor;
 import org.opensearch.sql.datasource.DataSourceMetadataStorage;
 import org.opensearch.sql.datasource.exceptions.DataSourceNotFoundException;
 import org.opensearch.sql.datasource.model.DataSourceMetadata;
 import org.opensearch.sql.datasource.model.auth.AuthenticationType;
-import org.opensearch.sql.plugin.utils.XContentParserUtils;
+import org.opensearch.sql.datasources.encryptor.Encryptor;
+import org.opensearch.sql.datasources.utils.XContentParserUtils;
 
 public class OpenSearchDataSourceMetadataStorage implements DataSourceMetadataStorage {
 

diff --git a/...urce/TransportCreateDataSourceAction.java → ...port/TransportCreateDataSourceAction.java b/...urce/TransportCreateDataSourceAction.java → ...port/TransportCreateDataSourceAction.java
@@ -5,7 +5,7 @@
  *
  */
 
-package org.opensearch.sql.plugin.transport.datasource;
+package org.opensearch.sql.datasources.transport;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -19,10 +19,8 @@
 import org.opensearch.sql.datasource.DataSourceService;
 import org.opensearch.sql.datasource.DataSourceServiceImpl;
 import org.opensearch.sql.datasource.model.DataSourceMetadata;
-import org.opensearch.sql.legacy.metrics.MetricName;
-import org.opensearch.sql.legacy.metrics.Metrics;
-import org.opensearch.sql.plugin.model.CreateDataSourceActionRequest;
-import org.opensearch.sql.plugin.model.CreateDataSourceActionResponse;
+import org.opensearch.sql.datasources.model.CreateDataSourceActionRequest;
+import org.opensearch.sql.datasources.model.CreateDataSourceActionResponse;
 import org.opensearch.tasks.Task;
 import org.opensearch.transport.TransportService;
 
@@ -59,8 +57,6 @@ public TransportCreateDataSourceAction(TransportService transportService,
   @Override
   protected void doExecute(Task task, CreateDataSourceActionRequest request,
                            ActionListener<CreateDataSourceActionResponse> actionListener) {
-
-    Metrics.getInstance().getNumericalMetric(MetricName.DATASOURCE_REQ_COUNT).increment();
     DataSourceMetadata dataSourceMetadata = request.getDataSourceMetadata();
     dataSourceService.createDataSource(dataSourceMetadata);
     actionListener.onResponse(new CreateDataSourceActionResponse("Created DataSource with name "
-Original file line number
+Diff line change
@@ Expand Up / @@ -5,7 +5,7 @@ @@
      *
      */
-    package org.opensearch.sql.common.encryptor;
+    package org.opensearch.sql.datasources.encryptor;
     public interface Encryptor {
@@ Expand Down @@