Address PR feedback + some fixes.

Signed-off-by: Yury-Fridlyand <yury.fridlyand@improving.com>

integ-test/build.gradle

@@ -80,7 +80,12 @@ ext {
     configureSecurityPlugin = { OpenSearchCluster cluster ->
 
         cluster.getNodes().forEach { node ->
-            node.getCredentials().add(Map.of('useradd', 'admin', '-p', 'admin'))
+            var creds = node.getCredentials()
+            if (creds.isEmpty()) {
+                creds.add(Map.of('useradd', 'admin', '-p', 'admin'))
+            } else {
+                creds.get(0).putAll(Map.of('useradd', 'admin', '-p', 'admin'))
+            }
         }
 
         var projectAbsPath = projectDir.getAbsolutePath()
@@ -198,9 +203,6 @@ compileTestJava {
 }
 
 testClusters.all {
-    testDistribution = 'archive'
-    plugin ":opensearch-sql-plugin"
-
     // debug with command, ./gradlew opensearch-sql:run -DdebugJVM. --debug-jvm does not work with keystore.
     if (System.getProperty("debugJVM") != null) {
         jvmArgs '-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005'
@@ -209,13 +211,21 @@ testClusters.all {
 
 testClusters {
     integTest {
+        testDistribution = 'archive'
+        plugin ":opensearch-sql-plugin"
         setting "plugins.query.datasources.encryption.masterkey", "1234567812345678"
     }
     remoteCluster {
+        testDistribution = 'archive'
+        plugin ":opensearch-sql-plugin"
     }
     integTestWithSecurity {
+        testDistribution = 'archive'
+        plugin ":opensearch-sql-plugin"
     }
     remoteIntegTestWithSecurity {
+        testDistribution = 'archive'
+        plugin ":opensearch-sql-plugin"
     }
 }
 
@@ -299,9 +309,8 @@ task integTestWithSecurity(type: RestIntegTestTask) {
     useCluster testClusters.integTestWithSecurity
     useCluster testClusters.remoteIntegTestWithSecurity
 
-    // Don't use `getClusters`: cluster order is important. IT framework adds/uses a cluster
-    // named as the task as default and uses it to init default REST client
-    systemProperty "cluster.names", "integTestWithSecurity,anotherintegTestWithSecurity"
+    systemProperty "cluster.names",
+        getClusters().stream().map(cluster -> cluster.getName()).collect(Collectors.joining(","))
 
     getClusters().forEach { cluster ->
         configureSecurityPlugin(cluster)

integ-test/src/test/java/org/opensearch/sql/legacy/OpenSearchSQLRestTestCase.java

@@ -49,8 +49,22 @@
 public abstract class OpenSearchSQLRestTestCase extends OpenSearchRestTestCase {
 
   private static final Logger LOG = LogManager.getLogger();
-  public static final String REMOTE_CLUSTER = "remoteCluster";
   public static final String MATCH_ALL_REMOTE_CLUSTER = "*";
+  // Requires to insert cluster name and cluster transport address (host:port)
+  public static final String REMOTE_CLUSTER_SETTING =
+      "{"
+          + "\"persistent\": {"
+          + "  \"cluster\": {"
+          + "    \"remote\": {"
+          + "      \"%s\": {"
+          + "        \"seeds\": ["
+          + "          \"%s\""
+          + "        ]"
+          + "      }"
+          + "    }"
+          + "  }"
+          + "}"
+          + "}";
 
   private static RestClient remoteClient;
   /**
@@ -109,21 +123,20 @@ public void initRemoteClient(String clusterName) throws IOException {
     remoteClient = remoteAdminClient = initClient(clusterName);
   }
 
+  /** Configure http client for the given <b>cluster</b>. */
   public RestClient initClient(String clusterName) throws IOException {
-    String cluster = getTestRestCluster(clusterName);
-    String[] stringUrls = cluster.split(",");
+    String[] stringUrls = getTestRestCluster(clusterName).split(",");
     List<HttpHost> hosts = new ArrayList<>(stringUrls.length);
     for (String stringUrl : stringUrls) {
       int portSeparator = stringUrl.lastIndexOf(':');
       if (portSeparator < 0) {
         throw new IllegalArgumentException("Illegal cluster url [" + stringUrl + "]");
       }
       String host = stringUrl.substring(0, portSeparator);
-      int port = Integer.valueOf(stringUrl.substring(portSeparator + 1));
+      int port = Integer.parseInt(stringUrl.substring(portSeparator + 1));
       hosts.add(buildHttpHost(host, port));
     }
-    final List<HttpHost> clusterHosts = unmodifiableList(hosts);
-    return buildClient(restClientSettings(), clusterHosts.toArray(new HttpHost[0]));
+    return buildClient(restClientSettings(), hosts.toArray(new HttpHost[0]));
   }
 
   /**
@@ -199,6 +212,10 @@ protected static void wipeAllOpenSearchIndices(RestClient client) throws IOExcep
     }
   }
 
+  /**
+   * Configure authentication and pass <b>builder</b> to superclass to configure other stuff.<br>
+   * By default, auth is configure when <b>https</b> is set only.
+   */
   protected static void configureClient(RestClientBuilder builder, Settings settings)
       throws IOException {
     String userName = System.getProperty("user");
@@ -272,20 +289,9 @@ public void configureMultiClusters(String remote)
 
     Request connectionRequest = new Request("PUT", "_cluster/settings");
     String connectionSetting = String.format(
-        "{"
-            + "\"persistent\": {"
-            + "  \"cluster\": {"
-            + "    \"remote\": {"
-            + "      \"%s\": {"
-            + "        \"seeds\": ["
-            + "          \"%s\""
-            + "        ]"
-            + "      }"
-            + "    }"
-            + "  }"
-            + "}"
-            + "}",
-        remote, getTestTransportCluster(remote).split(",")[0]);
+        REMOTE_CLUSTER_SETTING,
+        remote,
+        getTestTransportCluster(remote).split(",")[0]);
     connectionRequest.setJsonEntity(connectionSetting);
     adminClient().performRequest(connectionRequest);
   }

integ-test/src/test/java/org/opensearch/sql/legacy/RestIntegTestCase.java

@@ -44,9 +44,9 @@
 import org.junit.Before;
 import org.opensearch.client.Request;
 import org.opensearch.client.Response;
-import org.opensearch.core.xcontent.XContentBuilder;
 import org.opensearch.common.xcontent.XContentFactory;
 import org.opensearch.core.rest.RestStatus;
+import org.opensearch.core.xcontent.XContentBuilder;
 
 /**
  * SQL plugin integration test base class (migrated from SQLIntegTestCase)

integ-test/src/test/java/org/opensearch/sql/ppl/CrossClusterSearchIT.java

@@ -23,10 +23,16 @@
 
 public class CrossClusterSearchIT extends PPLIntegTestCase {
 
-  // Set second cluster as a remote
   static {
-    String[] clusterNames = System.getProperty("cluster.names", ",remoteCluster").split(",");
-    REMOTE_CLUSTER = clusterNames[1];
+    // find a remote cluster
+    String[] clusterNames = System.getProperty("cluster.names").split(",");
+    var remote = "remoteCluster";
+    for (var cluster : clusterNames) {
+      if (cluster.startsWith("remote")) {
+         remote = cluster;
+      }
+    }
+    REMOTE_CLUSTER = remote;
   }
 
   public static final String REMOTE_CLUSTER;