[Segment Replication] Wait for documents to replicate to replica shar…

…ds (#4236)

* [Segment Replication] Add thread sleep to account for replica lag in delete operations test

Signed-off-by: Suraj Singh <surajrider@gmail.com>

* Address review comments, assertBusy on doc count rather than sleep

Signed-off-by: Suraj Singh <surajrider@gmail.com>

Signed-off-by: Suraj Singh <surajrider@gmail.com>

server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java

@@ -38,6 +38,7 @@
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
+import java.util.concurrent.TimeUnit;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
@@ -293,9 +294,22 @@ public void testDeleteOperations() throws Exception {
 
             refresh(INDEX_NAME);
             waitForReplicaUpdate();
-
-            assertHitCount(client(nodeA).prepareSearch(INDEX_NAME).setSize(0).setPreference("_only_local").get(), expectedHitCount - 1);
-            assertHitCount(client(nodeB).prepareSearch(INDEX_NAME).setSize(0).setPreference("_only_local").get(), expectedHitCount - 1);
+            assertBusy(() -> {
+                final long nodeA_Count = client(nodeA).prepareSearch(INDEX_NAME)
+                    .setSize(0)
+                    .setPreference("_only_local")
+                    .get()
+                    .getHits()
+                    .getTotalHits().value;
+                assertEquals(expectedHitCount - 1, nodeA_Count);
+                final long nodeB_Count = client(nodeB).prepareSearch(INDEX_NAME)
+                    .setSize(0)
+                    .setPreference("_only_local")
+                    .get()
+                    .getHits()
+                    .getTotalHits().value;
+                assertEquals(expectedHitCount - 1, nodeB_Count);
+            }, 5, TimeUnit.SECONDS);
         }
     }