[Remote Store] Fix flaky test - closes #7703 (#7879)

Signed-off-by: Ashish Singh <ssashish@amazon.com>
opensearch-project · Jun 2, 2023 · 5d5e8ad · 5d5e8ad
1 parent ecd0879
commit 5d5e8ad
Show file tree

Hide file tree

Showing 3 changed files with 12 additions and 5 deletions.
diff --git a/...rTest/java/org/opensearch/remotestore/AbstractRemoteStoreMockRepositoryIntegTestCase.java b/...rTest/java/org/opensearch/remotestore/AbstractRemoteStoreMockRepositoryIntegTestCase.java
@@ -70,7 +70,7 @@ protected void deleteRepo() {
         assertAcked(clusterAdmin().prepareDeleteRepository(REPOSITORY_NAME));
     }
 
-    protected void setup(Path repoLocation, double ioFailureRate, String skipExceptionBlobList) {
+    protected void setup(Path repoLocation, double ioFailureRate, String skipExceptionBlobList, long maxFailure) {
         logger.info("--> Creating repository={} at the path={}", REPOSITORY_NAME, repoLocation);
         // The random_control_io_exception_rate setting ensures that 10-25% of all operations to remote store results in
         /// IOException. skip_exception_on_verification_file & skip_exception_on_list_blobs settings ensures that the
@@ -85,7 +85,7 @@ protected void setup(Path repoLocation, double ioFailureRate, String skipExcepti
                 .put("skip_exception_on_list_blobs", true)
                 // Skipping is required for metadata as it is part of recovery
                 .put("skip_exception_on_blobs", skipExceptionBlobList)
-                .put("max_failure_number", Long.MAX_VALUE)
+                .put("max_failure_number", maxFailure)
         );
 
         internalCluster().startDataOnlyNodes(1);

diff --git a/...er/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBackpressureIT.java b/...er/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBackpressureIT.java
@@ -29,7 +29,7 @@ public class RemoteStoreBackpressureIT extends AbstractRemoteStoreMockRepository
 
     public void testWritesRejected() {
         Path location = randomRepoPath().toAbsolutePath();
-        setup(location, 1d, "metadata");
+        setup(location, 1d, "metadata", Long.MAX_VALUE);
 
         Settings request = Settings.builder().put(REMOTE_REFRESH_SEGMENT_PRESSURE_ENABLED.getKey(), true).build();
         ClusterUpdateSettingsResponse clusterUpdateResponse = client().admin()

diff --git a/...src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRefreshListenerIT.java b/...src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRefreshListenerIT.java
@@ -15,6 +15,9 @@
 import org.opensearch.test.OpenSearchIntegTestCase;
 
 import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
 import java.util.Locale;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
@@ -27,7 +30,7 @@ public class RemoteStoreRefreshListenerIT extends AbstractRemoteStoreMockReposit
     public void testRemoteRefreshRetryOnFailure() throws Exception {
 
         Path location = randomRepoPath().toAbsolutePath();
-        setup(location, randomDoubleBetween(0.1, 0.25, true), "metadata");
+        setup(location, randomDoubleBetween(0.1, 0.15, true), "metadata", 10L);
 
         // Here we are having flush/refresh after each iteration of indexing. However, the refresh will not always succeed
         // due to IOExceptions that are thrown while doing uploadBlobs.
@@ -47,8 +50,12 @@ public void testRemoteRefreshRetryOnFailure() throws Exception {
         assertBusy(() -> {
             Set<String> filesInLocal = getSegmentFiles(location.getRoot().resolve(segmentDataLocalPath));
             Set<String> filesInRepo = getSegmentFiles(segmentDataRepoPath);
+            List<String> sortedFilesInLocal = new ArrayList<>(filesInLocal), sortedFilesInRepo = new ArrayList<>(filesInRepo);
+            Collections.sort(sortedFilesInLocal);
+            Collections.sort(sortedFilesInRepo);
+            logger.info("Local files = {}, Repo files = {}", sortedFilesInLocal, sortedFilesInRepo);
             assertTrue(filesInRepo.containsAll(filesInLocal));
-        }, 60, TimeUnit.SECONDS);
+        }, 90, TimeUnit.SECONDS);
         deleteRepo();
     }