Skip to content

Commit

Permalink
Fix flaky tests in restore flow using snapshot interop (opensearch-pr…
Browse files Browse the repository at this point in the history 
…oject#8422)

Signed-off-by: Sachin Kale <kalsac@amazon.com>
Signed-off-by: sahil buddharaju <sahilbud@amazon.com>
  • Loading branch information
@sachinpkale
sachinpkale authored and sahil buddharaju committed Jul 18, 2023
1 parent 7998814 commit ea7b725
Showing 1 changed file with 26 additions and 20 deletions.
46 changes: 26 additions & 20 deletions server/src/internalClusterTest/java/org/opensearch/snapshots/RestoreSnapshotIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -183,8 +183,8 @@ public void testRestoreRemoteStoreIndicesWithoutRemoteTranslog() throws IOExcept

public void testRestoreOperationsShallowCopyEnabled(boolean remoteTranslogEnabled) throws IOException, ExecutionException,
InterruptedException {
internalCluster().startClusterManagerOnlyNode();
final String primaryNode = internalCluster().startNode();
String clusterManagerNode = internalCluster().startClusterManagerOnlyNode();
String primary = internalCluster().startDataOnlyNode();
String indexName1 = "testindex1";
String indexName2 = "testindex2";
String snapshotRepoName = "test-restore-snapshot-repo";
Expand Down Expand Up @@ -216,7 +216,7 @@ public void testRestoreOperationsShallowCopyEnabled(boolean remoteTranslogEnable
indexDocuments(client, indexName2, numDocsInIndex2);
ensureGreen(indexName1, indexName2);

final String secondNode = internalCluster().startNode();
internalCluster().startDataOnlyNode();
logger.info("--> snapshot");
CreateSnapshotResponse createSnapshotResponse = client.admin()
.cluster()
Expand Down Expand Up @@ -273,10 +273,12 @@ public void testRestoreOperationsShallowCopyEnabled(boolean remoteTranslogEnable
assertDocsPresentInIndex(client, restoredIndexName2, numDocsInIndex2);

// deleting data for restoredIndexName1 and restoring from remote store.
internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNodeName(restoredIndexName1)));
internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primary));
ensureRed(restoredIndexName1);
assertAcked(client().admin().indices().prepareClose(restoredIndexName1));
client().admin()
// Re-initialize client to make sure we are not using client from stopped node.
client = client(clusterManagerNode);
assertAcked(client.admin().indices().prepareClose(restoredIndexName1));
client.admin()
.cluster()
.restoreRemoteStore(new RestoreRemoteStoreRequest().indices(restoredIndexName1), PlainActionFuture.newFuture());
ensureYellowAndNoInitializingShards(restoredIndexName1);
Expand All @@ -300,7 +302,7 @@ public void testRestoreOperationsShallowCopyEnabled(boolean remoteTranslogEnable
assertEquals(restoreSnapshotResponse3.status(), RestStatus.ACCEPTED);
ensureGreen(restoredIndexName1Seg);

GetIndexResponse getIndexResponse = client().admin()
GetIndexResponse getIndexResponse = client.admin()
.indices()
.getIndex(new GetIndexRequest().indices(restoredIndexName1Seg).includeDefaults(true))
.get();
Expand Down Expand Up @@ -331,7 +333,7 @@ public void testRestoreOperationsShallowCopyEnabled(boolean remoteTranslogEnable
assertEquals(restoreSnapshotResponse4.status(), RestStatus.ACCEPTED);
ensureGreen(restoredIndexName1Doc);

getIndexResponse = client().admin()
getIndexResponse = client.admin()
.indices()
.getIndex(new GetIndexRequest().indices(restoredIndexName1Doc).includeDefaults(true))
.get();
Expand All @@ -347,8 +349,8 @@ public void testRestoreOperationsShallowCopyEnabled(boolean remoteTranslogEnable
}

public void testRestoreInSameRemoteStoreEnabledIndex() throws IOException {
internalCluster().startClusterManagerOnlyNode();
internalCluster().startNode();
String clusterManagerNode = internalCluster().startClusterManagerOnlyNode();
String primary = internalCluster().startDataOnlyNode();
String indexName1 = "testindex1";
String indexName2 = "testindex2";
String snapshotRepoName = "test-restore-snapshot-repo";
Expand Down Expand Up @@ -378,7 +380,7 @@ public void testRestoreInSameRemoteStoreEnabledIndex() throws IOException {
indexDocuments(client, indexName2, numDocsInIndex2);
ensureGreen(indexName1, indexName2);

final String secondNode = internalCluster().startNode();
internalCluster().startDataOnlyNode();
logger.info("--> snapshot");
CreateSnapshotResponse createSnapshotResponse = client.admin()
.cluster()
Expand Down Expand Up @@ -435,10 +437,12 @@ public void testRestoreInSameRemoteStoreEnabledIndex() throws IOException {
assertDocsPresentInIndex(client, restoredIndexName2, numDocsInIndex2);

// deleting data for restoredIndexName1 and restoring from remote store.
internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNodeName(indexName1)));
internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primary));
ensureRed(indexName1);
assertAcked(client().admin().indices().prepareClose(indexName1));
client().admin().cluster().restoreRemoteStore(new RestoreRemoteStoreRequest().indices(indexName1), PlainActionFuture.newFuture());
// Re-initialize client to make sure we are not using client from stopped node.
client = client(clusterManagerNode);
assertAcked(client.admin().indices().prepareClose(indexName1));
client.admin().cluster().restoreRemoteStore(new RestoreRemoteStoreRequest().indices(indexName1), PlainActionFuture.newFuture());
ensureYellowAndNoInitializingShards(indexName1);
ensureGreen(indexName1);
assertDocsPresentInIndex(client(), indexName1, numDocsInIndex1);
Expand All @@ -449,8 +453,8 @@ public void testRestoreInSameRemoteStoreEnabledIndex() throws IOException {
}

public void testRestoreShallowCopySnapshotWithDifferentRepo() throws IOException {
internalCluster().startClusterManagerOnlyNode();
final String primaryNode = internalCluster().startNode();
String clusterManagerNode = internalCluster().startClusterManagerOnlyNode();
String primary = internalCluster().startDataOnlyNode();
String indexName1 = "testindex1";
String indexName2 = "testindex2";
String snapshotRepoName = "test-restore-snapshot-repo";
Expand Down Expand Up @@ -479,7 +483,7 @@ public void testRestoreShallowCopySnapshotWithDifferentRepo() throws IOException
indexDocuments(client, indexName2, numDocsInIndex2);
ensureGreen(indexName1, indexName2);

final String secondNode = internalCluster().startNode();
internalCluster().startDataOnlyNode();

logger.info("--> snapshot");
CreateSnapshotResponse createSnapshotResponse = client.admin()
Expand Down Expand Up @@ -513,9 +517,11 @@ public void testRestoreShallowCopySnapshotWithDifferentRepo() throws IOException
assertDocsPresentInIndex(client(), restoredIndexName1, numDocsInIndex1);

// deleting data for restoredIndexName1 and restoring from remote store.
internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNodeName(restoredIndexName1)));
assertAcked(client().admin().indices().prepareClose(restoredIndexName1));
client().admin()
internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primary));
// Re-initialize client to make sure we are not using client from stopped node.
client = client(clusterManagerNode);
assertAcked(client.admin().indices().prepareClose(restoredIndexName1));
client.admin()
.cluster()
.restoreRemoteStore(new RestoreRemoteStoreRequest().indices(restoredIndexName1), PlainActionFuture.newFuture());
ensureYellowAndNoInitializingShards(restoredIndexName1);
Expand Down

0 comments on commit ea7b725

Please sign in to comment.