Add voting config IT

Signed-off-by: Shivansh Arora <hishiv@amazon.com>
shiv0408 · Aug 30, 2024 · 262c169 · 262c169
1 parent 3480336
commit 262c169
Show file tree

Hide file tree

Showing 2 changed files with 51 additions and 10 deletions.
diff --git a/.../src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java b/.../src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java
@@ -8,12 +8,17 @@
 
 package org.opensearch.gateway.remote;
 
+import org.opensearch.action.admin.cluster.configuration.AddVotingConfigExclusionsAction;
+import org.opensearch.action.admin.cluster.configuration.AddVotingConfigExclusionsRequest;
+import org.opensearch.action.admin.cluster.configuration.AddVotingConfigExclusionsResponse;
 import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
 import org.opensearch.action.admin.cluster.state.ClusterStateResponse;
 import org.opensearch.client.Client;
+import org.opensearch.cluster.coordination.CoordinationState;
 import org.opensearch.cluster.coordination.PersistedStateRegistry;
 import org.opensearch.common.blobstore.BlobPath;
 import org.opensearch.common.settings.Settings;
+import org.opensearch.core.action.ActionResponse;
 import org.opensearch.gateway.GatewayMetaState;
 import org.opensearch.gateway.remote.model.RemoteClusterMetadataManifest;
 import org.opensearch.indices.recovery.RecoverySettings;
@@ -27,9 +32,12 @@
 
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
 import java.util.Base64;
+import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.concurrent.ExecutionException;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
@@ -161,8 +169,7 @@ public void testMasterReElectionUsesIncrementalUpload() throws IOException {
         ensureStableCluster(4);
 
         persistedStateRegistry = internalCluster().getClusterManagerNodeInstance(PersistedStateRegistry.class);
-        GatewayMetaState.RemotePersistedState persistedStateAfterElection = (GatewayMetaState.RemotePersistedState) persistedStateRegistry
-            .getPersistedState(PersistedStateRegistry.PersistedStateType.REMOTE);
+        CoordinationState.PersistedState persistedStateAfterElection = persistedStateRegistry.getPersistedState(PersistedStateRegistry.PersistedStateType.REMOTE);
         ClusterMetadataManifest manifestAfterElection = persistedStateAfterElection.getLastAcceptedManifest();
 
         // coordination metadata is updated, it will be unequal
@@ -177,6 +184,41 @@ public void testMasterReElectionUsesIncrementalUpload() throws IOException {
         assertEquals(manifest.getIndicesRouting(), manifestAfterElection.getIndicesRouting());
     }
 
+
+    public void testVotingConfigAreCommitted() throws ExecutionException, InterruptedException {
+        prepareCluster(3, 2, INDEX_NAME, 1, 2);
+        ensureStableCluster(5);
+        ensureGreen(INDEX_NAME);
+        // add two new nodes to the cluster, to update the voting config
+        List<String> newCMNodes = internalCluster().startClusterManagerOnlyNodes(2, Settings.EMPTY);
+        ensureStableCluster(7);
+
+        internalCluster().getInstances(PersistedStateRegistry.class).forEach(
+            persistedStateRegistry -> {
+                CoordinationState.PersistedState localState = persistedStateRegistry.getPersistedState(PersistedStateRegistry.PersistedStateType.LOCAL);
+                CoordinationState.PersistedState remoteState = persistedStateRegistry.getPersistedState(PersistedStateRegistry.PersistedStateType.REMOTE);
+                if (remoteState != null) {
+                    assertEquals(localState.getLastAcceptedState().getLastCommittedConfiguration(), remoteState.getLastAcceptedState().getLastCommittedConfiguration());
+                    assertEquals(5, remoteState.getLastAcceptedState().getLastCommittedConfiguration().getNodeIds().size());
+                }
+            }
+        );
+
+        // remove the newly added nodes from the voting config
+        client().admin().cluster().execute(AddVotingConfigExclusionsAction.INSTANCE, new AddVotingConfigExclusionsRequest(newCMNodes.get(0), newCMNodes.get(1))).get();
+
+        internalCluster().getInstances(PersistedStateRegistry.class).forEach(
+            persistedStateRegistry -> {
+                CoordinationState.PersistedState localState = persistedStateRegistry.getPersistedState(PersistedStateRegistry.PersistedStateType.LOCAL);
+                CoordinationState.PersistedState remoteState = persistedStateRegistry.getPersistedState(PersistedStateRegistry.PersistedStateType.REMOTE);
+                if (remoteState != null) {
+                    assertEquals(localState.getLastAcceptedState().getLastCommittedConfiguration(), remoteState.getLastAcceptedState().getLastCommittedConfiguration());
+                    assertEquals(3, remoteState.getLastAcceptedState().getLastCommittedConfiguration().getNodeIds().size());
+                }
+            }
+        );
+    }
+
     private Map<String, Integer> getMetadataFiles(BlobStoreRepository repository, String subDirectory) throws IOException {
         BlobPath metadataPath = repository.basePath()
             .add(

diff --git a/server/src/main/java/org/opensearch/gateway/GatewayMetaState.java b/server/src/main/java/org/opensearch/gateway/GatewayMetaState.java
@@ -797,18 +797,17 @@ public void markLastAcceptedStateAsCommitted() {
                 assert lastAcceptedManifest != null : "Last accepted manifest is not present";
                 ClusterState clusterState = lastAcceptedState;
                 boolean shouldCommitVotingConfig = shouldCommitVotingConfig();
-
+                Metadata.Builder metadataBuilder;
+                if (shouldCommitVotingConfig) {
+                    metadataBuilder = commitVotingConfiguration(lastAcceptedState);
+                } else {
+                    metadataBuilder = Metadata.builder(lastAcceptedState.metadata());
+                }
                 if (lastAcceptedState.metadata().clusterUUID().equals(Metadata.UNKNOWN_CLUSTER_UUID) == false
                     && lastAcceptedState.metadata().clusterUUIDCommitted() == false) {
-                    Metadata.Builder metadataBuilder;
-                    if (shouldCommitVotingConfig) {
-                        metadataBuilder = commitVotingConfiguration(lastAcceptedState);
-                    } else {
-                        metadataBuilder = Metadata.builder(lastAcceptedState.metadata());
-                    }
                     metadataBuilder.clusterUUIDCommitted(true);
-                    clusterState = ClusterState.builder(lastAcceptedState).metadata(metadataBuilder).build();
                 }
+                clusterState = ClusterState.builder(lastAcceptedState).metadata(metadataBuilder).build();
                 if (clusterState.getNodes().isLocalNodeElectedClusterManager()) {
                     final RemoteClusterStateManifestInfo committedManifestDetails = remoteClusterStateService.markLastStateAsCommitted(
                         clusterState,