Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix flaky tests in restore flow using snapshot interop #8422

Merged
merged 2 commits into from
Jul 4, 2023
Merged

Fix flaky tests in restore flow using snapshot interop #8422

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8a5f22a
Fix flaky tests in restore flow using snapshot interop
Jul 4, 2023
81ca831
Address PR comments
Jul 4, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 26 additions & 20 deletions server/src/internalClusterTest/java/org/opensearch/snapshots/RestoreSnapshotIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -183,8 +183,8 @@ public void testRestoreRemoteStoreIndicesWithoutRemoteTranslog() throws IOExcept

public void testRestoreOperationsShallowCopyEnabled(boolean remoteTranslogEnabled) throws IOException, ExecutionException,
InterruptedException {
internalCluster().startClusterManagerOnlyNode();
final String primaryNode = internalCluster().startNode();
String clusterManagerNode = internalCluster().startClusterManagerOnlyNode();
String primary = internalCluster().startDataOnlyNode();
String indexName1 = "testindex1";
String indexName2 = "testindex2";
String snapshotRepoName = "test-restore-snapshot-repo";
Expand Down Expand Up @@ -216,7 +216,7 @@ public void testRestoreOperationsShallowCopyEnabled(boolean remoteTranslogEnable
indexDocuments(client, indexName2, numDocsInIndex2);
ensureGreen(indexName1, indexName2);

final String secondNode = internalCluster().startNode();
internalCluster().startDataOnlyNode();
logger.info("--> snapshot");
CreateSnapshotResponse createSnapshotResponse = client.admin()
.cluster()
Expand Down Expand Up @@ -273,10 +273,12 @@ public void testRestoreOperationsShallowCopyEnabled(boolean remoteTranslogEnable
assertDocsPresentInIndex(client, restoredIndexName2, numDocsInIndex2);

// deleting data for restoredIndexName1 and restoring from remote store.
internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNodeName(restoredIndexName1)));
internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primary));
ensureRed(restoredIndexName1);
assertAcked(client().admin().indices().prepareClose(restoredIndexName1));
client().admin()
// Re-initialize client to make sure we are not using client from stopped node.
client = client(clusterManagerNode);
assertAcked(client.admin().indices().prepareClose(restoredIndexName1));
client.admin()
.cluster()
.restoreRemoteStore(new RestoreRemoteStoreRequest().indices(restoredIndexName1), PlainActionFuture.newFuture());
ensureYellowAndNoInitializingShards(restoredIndexName1);
Expand All @@ -300,7 +302,7 @@ public void testRestoreOperationsShallowCopyEnabled(boolean remoteTranslogEnable
assertEquals(restoreSnapshotResponse3.status(), RestStatus.ACCEPTED);
ensureGreen(restoredIndexName1Seg);

GetIndexResponse getIndexResponse = client().admin()
GetIndexResponse getIndexResponse = client.admin()
.indices()
.getIndex(new GetIndexRequest().indices(restoredIndexName1Seg).includeDefaults(true))
.get();
Expand Down Expand Up @@ -331,7 +333,7 @@ public void testRestoreOperationsShallowCopyEnabled(boolean remoteTranslogEnable
assertEquals(restoreSnapshotResponse4.status(), RestStatus.ACCEPTED);
ensureGreen(restoredIndexName1Doc);

getIndexResponse = client().admin()
getIndexResponse = client.admin()
.indices()
.getIndex(new GetIndexRequest().indices(restoredIndexName1Doc).includeDefaults(true))
.get();
Expand All @@ -347,8 +349,8 @@ public void testRestoreOperationsShallowCopyEnabled(boolean remoteTranslogEnable
}

public void testRestoreInSameRemoteStoreEnabledIndex() throws IOException {
internalCluster().startClusterManagerOnlyNode();
internalCluster().startNode();
String clusterManagerNode = internalCluster().startClusterManagerOnlyNode();
String primary = internalCluster().startDataOnlyNode();
String indexName1 = "testindex1";
String indexName2 = "testindex2";
String snapshotRepoName = "test-restore-snapshot-repo";
Expand Down Expand Up @@ -378,7 +380,7 @@ public void testRestoreInSameRemoteStoreEnabledIndex() throws IOException {
indexDocuments(client, indexName2, numDocsInIndex2);
ensureGreen(indexName1, indexName2);

final String secondNode = internalCluster().startNode();
internalCluster().startDataOnlyNode();
logger.info("--> snapshot");
CreateSnapshotResponse createSnapshotResponse = client.admin()
.cluster()
Expand Down Expand Up @@ -435,10 +437,12 @@ public void testRestoreInSameRemoteStoreEnabledIndex() throws IOException {
assertDocsPresentInIndex(client, restoredIndexName2, numDocsInIndex2);

// deleting data for restoredIndexName1 and restoring from remote store.
internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNodeName(indexName1)));
internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primary));
ensureRed(indexName1);
assertAcked(client().admin().indices().prepareClose(indexName1));
client().admin().cluster().restoreRemoteStore(new RestoreRemoteStoreRequest().indices(indexName1), PlainActionFuture.newFuture());
// Re-initialize client to make sure we are not using client from stopped node.
client = client(clusterManagerNode);
assertAcked(client.admin().indices().prepareClose(indexName1));
client.admin().cluster().restoreRemoteStore(new RestoreRemoteStoreRequest().indices(indexName1), PlainActionFuture.newFuture());
ensureYellowAndNoInitializingShards(indexName1);
ensureGreen(indexName1);
assertDocsPresentInIndex(client(), indexName1, numDocsInIndex1);
Expand All @@ -449,8 +453,8 @@ public void testRestoreInSameRemoteStoreEnabledIndex() throws IOException {
}

public void testRestoreShallowCopySnapshotWithDifferentRepo() throws IOException {
internalCluster().startClusterManagerOnlyNode();
final String primaryNode = internalCluster().startNode();
String clusterManagerNode = internalCluster().startClusterManagerOnlyNode();
String primary = internalCluster().startDataOnlyNode();
String indexName1 = "testindex1";
String indexName2 = "testindex2";
String snapshotRepoName = "test-restore-snapshot-repo";
Expand Down Expand Up @@ -479,7 +483,7 @@ public void testRestoreShallowCopySnapshotWithDifferentRepo() throws IOException
indexDocuments(client, indexName2, numDocsInIndex2);
ensureGreen(indexName1, indexName2);

final String secondNode = internalCluster().startNode();
internalCluster().startDataOnlyNode();

logger.info("--> snapshot");
CreateSnapshotResponse createSnapshotResponse = client.admin()
Expand Down Expand Up @@ -513,9 +517,11 @@ public void testRestoreShallowCopySnapshotWithDifferentRepo() throws IOException
assertDocsPresentInIndex(client(), restoredIndexName1, numDocsInIndex1);

// deleting data for restoredIndexName1 and restoring from remote store.
internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNodeName(restoredIndexName1)));
assertAcked(client().admin().indices().prepareClose(restoredIndexName1));
client().admin()
internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primary));
// Re-initialize client to make sure we are not using client from stopped node.
client = client(clusterManagerNode);
assertAcked(client.admin().indices().prepareClose(restoredIndexName1));
client.admin()
.cluster()
.restoreRemoteStore(new RestoreRemoteStoreRequest().indices(restoredIndexName1), PlainActionFuture.newFuture());
ensureYellowAndNoInitializingShards(restoredIndexName1);
Expand Down