[Tests] Fix flakiness by ensuring stable cluster (#39300) (#39356)

In integration tests where `setBootstrapMasterNodeIndex()` is used in
combination with `autoMinMasterNodes = false` the cluster can start
bootstrapping once the number of nodes set with the
`setBootstrapMasterNodeIndex` have been started but it's not ensured
that all nodes have successfully joined to form the cluster.

This behaviour was introduced with 5db7ed2
and in order to ensure that the cluster is properly formed before proceeding
with the integration test, use `ensureStableCluster()` with the
appropriate number of expected nodes.

Fixes: #39220

Author: matriv

modules/transport-netty4/src/test/java/org/elasticsearch/rest/discovery/Zen2RestApiIT.java

@@ -118,6 +118,7 @@ public Settings onNodeStopped(String nodeName) throws IOException {
     public void testClearVotingTombstonesNotWaitingForRemoval() throws Exception {
         internalCluster().setBootstrapMasterNodeIndex(2);
         List<String> nodes = internalCluster().startNodes(3);
+        ensureStableCluster(3);
         RestClient restClient = getRestClient();
         Response response = restClient.performRequest(new Request("POST", "/_cluster/voting_config_exclusions/" + nodes.get(2)));
         assertThat(response.getStatusLine().getStatusCode(), is(200));
@@ -131,6 +132,7 @@ public void testClearVotingTombstonesNotWaitingForRemoval() throws Exception {
     public void testClearVotingTombstonesWaitingForRemoval() throws Exception {
         internalCluster().setBootstrapMasterNodeIndex(2);
         List<String> nodes = internalCluster().startNodes(3);
+        ensureStableCluster(3);
         RestClient restClient = getRestClient();
         String nodeToWithdraw = nodes.get(randomIntBetween(0, 2));
         Response response = restClient.performRequest(new Request("POST", "/_cluster/voting_config_exclusions/" + nodeToWithdraw));
@@ -145,6 +147,7 @@ public void testClearVotingTombstonesWaitingForRemoval() throws Exception {
     public void testFailsOnUnknownNode() throws Exception {
         internalCluster().setBootstrapMasterNodeIndex(2);
         internalCluster().startNodes(3);
+        ensureStableCluster(3);
         RestClient restClient = getRestClient();
         try {
             restClient.performRequest(new Request("POST", "/_cluster/voting_config_exclusions/invalid"));

server/src/test/java/org/elasticsearch/cluster/MinimumMasterNodesIT.java

@@ -284,7 +284,7 @@ public void testCannotCommitStateThreeNodes() throws Exception {
                 .build();
 
         internalCluster().startNodes(3, settings);
-        ensureGreen(); // ensure cluster state is recovered before we disrupt things
+        ensureStableCluster(3);
 
         final String master = internalCluster().getMasterName();
         Set<String> otherNodes = new HashSet<>(Arrays.asList(internalCluster().getNodeNames()));

server/src/test/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommandIT.java

@@ -250,7 +250,6 @@ public void testDetachAbortedByUser() throws IOException {
         expectThrows(() -> detachCluster(environment, 0, true), ElasticsearchNodeCommand.ABORTED_BY_USER_MSG);
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/39220")
     public void test3MasterNodes2Failed() throws Exception {
         internalCluster().setBootstrapMasterNodeIndex(2);
         List<String> masterNodes = new ArrayList<>();
@@ -268,8 +267,12 @@ public void test3MasterNodes2Failed() throws Exception {
         logger.info("--> start 2nd and 3rd master-eligible nodes and bootstrap");
         masterNodes.addAll(internalCluster().startMasterOnlyNodes(2)); // node ordinals 2 and 3
 
+        logger.info("--> wait for all nodes to join the cluster");
+        ensureStableCluster(4);
+
         logger.info("--> create index test");
         createIndex("test");
+        ensureGreen("test");
 
         logger.info("--> stop 2nd and 3d master eligible node");
         internalCluster().stopRandomNode(InternalTestCluster.nameFilter(masterNodes.get(1)));