[Segment Replication] Update bwc test to rely on segments for verific…

…ation (opensearch-project#8267)

* [Segment Replication] Use _cat/segments vs index stats + _search to verify doc count

Signed-off-by: Suraj Singh <surajrider@gmail.com>

Self review

Signed-off-by: Suraj Singh <surajrider@gmail.com>

remove unused imports

Signed-off-by: Suraj Singh <surajrider@gmail.com>

Handle 0 doc count segments

Signed-off-by: Suraj Singh <surajrider@gmail.com>

* Use 1 minute timeout for assertBusy validations and comments

Signed-off-by: Suraj Singh <surajrider@gmail.com>

---------

Signed-off-by: Suraj Singh <surajrider@gmail.com>

Add missing import statement

Signed-off-by: Suraj Singh <surajrider@gmail.com>

qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/IndexingIT.java

@@ -59,6 +59,7 @@
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.either;
 
+
 /**
  * Basic test that indexed documents survive the rolling restart. See
  * {@link RecoveryIT} for much more in depth testing of the mechanism
@@ -99,6 +100,11 @@ private void waitForSearchableDocs(String index, int shardCount, int replicaCoun
 
         // Verify segment store
         assertBusy(() -> {
+            /**
+             * Use default tabular output and sort response based on shard,segment,primaryOrReplica columns to allow line by
+             * line parsing where records related to a segment (e.g. _0) are chunked together with first record belonging
+             * to primary while remaining *replicaCount* records belongs to replica copies
+             * */
             Request segrepStatsRequest = new Request("GET", "/_cat/segments/" + index + "?s=shard,segment,primaryOrReplica");
             segrepStatsRequest.addParameter("h", "index,shard,primaryOrReplica,segment,docs.count");
             Response segrepStatsResponse = client().performRequest(segrepStatsRequest);
@@ -127,7 +133,7 @@ private void waitForSearchableDocs(String index, int shardCount, int replicaCoun
                     segmentsIndex++;
                 }
             }
-        });
+        }, 1, TimeUnit.MINUTES);
     }
 
     private void waitForClusterHealthWithNoShardMigration(String indexName, String status) throws IOException {
@@ -150,7 +156,7 @@ private void verifySegmentStats(String indexName) throws Exception {
                 String[] elements = statLine.split(" +");
                 assertEquals("Replica shard " + elements[0] + "not upto date with primary ", 0, Integer.parseInt(elements[2]));
             }
-        });
+        }, 1, TimeUnit.MINUTES);
     }
 
     public void testIndexing() throws IOException, ParseException {