Fix flaky SearchScrollWithFailingNodesIT (opensearch-project#10374)

The test intended to stop a data node and called a method named
`stopRandomNonClusterManagerNode()` in order to do that. However, that
method would stop a random node that was not the currently elected
cluster manager, regardless of node role. I have also renamed that
method hoping to be more clear.

Resolves opensearch-project#10137

Signed-off-by: Andrew Ross <andrross@amazon.com>
Signed-off-by: Siddhant Deshmukh <deshsid@amazon.com>

server/src/internalClusterTest/java/org/opensearch/cluster/MinimumClusterManagerNodesIT.java

@@ -317,8 +317,8 @@ public void testThreeNodesNoClusterManagerBlock() throws Exception {
         );
         Settings nonClusterManagerDataPathSettings1 = internalCluster().dataPathSettings(nonClusterManagerNodes.get(0));
         Settings nonClusterManagerDataPathSettings2 = internalCluster().dataPathSettings(nonClusterManagerNodes.get(1));
-        internalCluster().stopRandomNonClusterManagerNode();
-        internalCluster().stopRandomNonClusterManagerNode();
+        internalCluster().stopRandomNodeNotCurrentClusterManager();
+        internalCluster().stopRandomNodeNotCurrentClusterManager();
 
         logger.info("--> verify that there is no cluster-manager anymore on remaining node");
         // spin here to wait till the state is set

server/src/internalClusterTest/java/org/opensearch/discovery/DiscoveryDisruptionIT.java

@@ -136,7 +136,7 @@ public void testClusterJoinDespiteOfPublishingIssues() throws Exception {
 
         // shutting down the nodes, to avoid the leakage check tripping
         // on the states associated with the commit requests we may have dropped
-        internalCluster().stopRandomNonClusterManagerNode();
+        internalCluster().stopRandomNodeNotCurrentClusterManager();
     }
 
     public void testClusterFormingWithASlowNode() {

server/src/internalClusterTest/java/org/opensearch/search/scroll/SearchScrollWithFailingNodesIT.java

@@ -119,7 +119,7 @@ public void testScanScrollWithShardExceptions() throws Exception {
         assertThat(numHits, equalTo(100L));
         clearScroll("_all");
 
-        internalCluster().stopRandomNonClusterManagerNode();
+        internalCluster().stopRandomDataNode();
 
         searchResponse = client().prepareSearch().setQuery(matchAllQuery()).setSize(10).setScroll(TimeValue.timeValueMinutes(1)).get();
         assertThat(searchResponse.getSuccessfulShards(), lessThan(searchResponse.getTotalShards()));

test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java

@@ -1816,7 +1816,7 @@ public synchronized void stopCurrentClusterManagerNode() throws IOException {
     /**
      * Stops any of the current nodes but not the cluster-manager node.
      */
-    public synchronized void stopRandomNonClusterManagerNode() throws IOException {
+    public synchronized void stopRandomNodeNotCurrentClusterManager() throws IOException {
         NodeAndClient nodeAndClient = getRandomNodeAndClient(new NodeNamePredicate(getClusterManagerName()).negate());
         if (nodeAndClient != null) {
             logger.info(
@@ -1841,11 +1841,11 @@ public synchronized void stopCurrentMasterNode() throws IOException {
     /**
      * Stops any of the current nodes but not the cluster-manager node.
      *
-     * @deprecated As of 2.2, because supporting inclusive language, replaced by {@link #stopRandomNonClusterManagerNode()}
+     * @deprecated As of 2.2, because supporting inclusive language, replaced by {@link #stopRandomNodeNotCurrentClusterManager()}
      */
     @Deprecated
-    public synchronized void stopRandomNonMasterNode() throws IOException {
-        stopRandomNonClusterManagerNode();
+    public synchronized void stopRandomNodeNotCurrentMaster() throws IOException {
+        stopRandomNodeNotCurrentClusterManager();
     }
 
     /**
@@ -1860,7 +1860,7 @@ public void stopAllNodes() {
             }
             int totalClusterManagerNodes = numClusterManagerNodes();
             while (totalClusterManagerNodes > 1) {
-                stopRandomNonClusterManagerNode();
+                stopRandomNodeNotCurrentClusterManager();
                 totalClusterManagerNodes -= 1;
             }
             stopCurrentClusterManagerNode();