Fixing concurrent search tests with one slice (opensearch-project#11071)

* Fixing concurrent search tests with one slice (opensearch-project#11071)

Signed-off-by: Ticheng Lin <ticheng@amazon.com>

* Remove changes for non-flaky tests (opensearch-project#11071)

Signed-off-by: Ticheng Lin <ticheng@amazon.com>

---------

Signed-off-by: Ticheng Lin <ticheng@amazon.com>

server/src/internalClusterTest/java/org/opensearch/search/nested/SimpleNestedIT.java

@@ -293,6 +293,7 @@ public void testMultiNested() throws Exception {
         refresh();
         // check the numDocs
         assertDocumentCount("test", 7);
+        indexRandomForConcurrentSearch("test");
 
         // do some multi nested queries
         SearchResponse searchResponse = client().prepareSearch("test")
@@ -485,6 +486,7 @@ public void testExplain() throws Exception {
             )
             .setRefreshPolicy(IMMEDIATE)
             .get();
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse searchResponse = client().prepareSearch("test")
             .setQuery(nestedQuery("nested1", termQuery("nested1.n_field1", "n_value1"), ScoreMode.Total))
@@ -968,6 +970,10 @@ public void testNestedSortWithMultiLevelFiltering() throws Exception {
 
     // https://github.com/elastic/elasticsearch/issues/31554
     public void testLeakingSortValues() throws Exception {
+        assumeFalse(
+            "Concurrent search case muted pending fix: https://github.com/opensearch-project/OpenSearch/issues/11065",
+            internalCluster().clusterService().getClusterSettings().get(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING)
+        );
         assertAcked(
             prepareCreate("test").setSettings(Settings.builder().put("number_of_shards", 1))
                 .setMapping(
@@ -1035,6 +1041,7 @@ public void testLeakingSortValues() throws Exception {
             .get();
 
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse searchResponse = client().prepareSearch()
             .setQuery(termQuery("_id", 2))
@@ -1627,6 +1634,7 @@ public void testCheckFixedBitSetCache() throws Exception {
         client().prepareIndex("test").setId("1").setSource("field", "value").get();
         refresh();
         ensureSearchable("test");
+        indexRandomForConcurrentSearch("test");
 
         // No nested mapping yet, there shouldn't be anything in the fixed bit set cache
         ClusterStatsResponse clusterStatsResponse = client().admin().cluster().prepareClusterStats().get();

server/src/internalClusterTest/java/org/opensearch/search/pit/PitMultiNodeIT.java

@@ -100,6 +100,7 @@ public void clearIndex() {
     public void testPit() throws Exception {
         CreatePitRequest request = new CreatePitRequest(TimeValue.timeValueDays(1), true);
         request.setIndices(new String[] { "index" });
+        indexRandomForConcurrentSearch("index");
         ActionFuture<CreatePitResponse> execute = client().execute(CreatePitAction.INSTANCE, request);
         CreatePitResponse pitResponse = execute.get();
         SearchResponse searchResponse = client().prepareSearch("index")

server/src/internalClusterTest/java/org/opensearch/search/preference/SearchPreferenceIT.java

@@ -52,7 +52,6 @@
 import org.opensearch.test.OpenSearchIntegTestCase;
 import org.opensearch.test.ParameterizedOpenSearchIntegTestCase;
 
-import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -99,7 +98,7 @@ public Settings nodeSettings(int nodeOrdinal) {
     }
 
     // see #2896
-    public void testStopOneNodePreferenceWithRedState() throws IOException {
+    public void testStopOneNodePreferenceWithRedState() throws Exception {
         assertAcked(
             prepareCreate("test").setSettings(
                 Settings.builder().put("index.number_of_shards", cluster().numDataNodes() + 2).put("index.number_of_replicas", 0)
@@ -110,6 +109,7 @@ public void testStopOneNodePreferenceWithRedState() throws IOException {
             client().prepareIndex("test").setId("" + i).setSource("field1", "value1").get();
         }
         refresh();
+        indexRandomForConcurrentSearch("test");
         internalCluster().stopRandomDataNode();
         client().admin().cluster().prepareHealth().setWaitForStatus(ClusterHealthStatus.RED).get();
         String[] preferences = new String[] {
@@ -138,7 +138,7 @@ public void testStopOneNodePreferenceWithRedState() throws IOException {
         assertThat("_only_local", searchResponse.getFailedShards(), greaterThanOrEqualTo(0));
     }
 
-    public void testNoPreferenceRandom() {
+    public void testNoPreferenceRandom() throws Exception {
         assertAcked(
             prepareCreate("test").setSettings(
                 // this test needs at least a replica to make sure two consecutive searches go to two different copies of the same data
@@ -149,6 +149,7 @@ public void testNoPreferenceRandom() {
 
         client().prepareIndex("test").setSource("field1", "value1").get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         final Client client = internalCluster().smartClient();
         SearchResponse searchResponse = client.prepareSearch("test").setQuery(matchAllQuery()).get();
@@ -201,7 +202,7 @@ public void testThatSpecifyingNonExistingNodesReturnsUsefulError() {
         }
     }
 
-    public void testNodesOnlyRandom() {
+    public void testNodesOnlyRandom() throws Exception {
         assertAcked(
             prepareCreate("test").setSettings(
                 // this test needs at least a replica to make sure two consecutive searches go to two different copies of the same data
@@ -211,6 +212,7 @@ public void testNodesOnlyRandom() {
         ensureGreen();
         client().prepareIndex("test").setSource("field1", "value1").get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         final Client client = internalCluster().smartClient();
         // multiple wildchar to cover multi-param usecase

server/src/internalClusterTest/java/org/opensearch/search/query/QueryStringIT.java

@@ -186,6 +186,7 @@ public void testDocWithAllTypes() throws Exception {
         String docBody = copyToStringFromClasspath("/org/opensearch/search/query/all-example-document.json");
         reqs.add(client().prepareIndex("test").setId("1").setSource(docBody, MediaTypeRegistry.JSON));
         indexRandom(true, false, reqs);
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse resp = client().prepareSearch("test").setQuery(queryStringQuery("foo")).get();
         assertHits(resp.getHits(), "1");
@@ -225,6 +226,7 @@ public void testKeywordWithWhitespace() throws Exception {
         reqs.add(client().prepareIndex("test").setId("2").setSource("f1", "bar"));
         reqs.add(client().prepareIndex("test").setId("3").setSource("f1", "foo bar"));
         indexRandom(true, false, reqs);
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse resp = client().prepareSearch("test").setQuery(queryStringQuery("foo")).get();
         assertHits(resp.getHits(), "3");
@@ -245,6 +247,7 @@ public void testRegexCaseInsensitivity() throws Exception {
         indexRequests.add(client().prepareIndex("messages").setId("1").setSource("message", "message: this is a TLS handshake"));
         indexRequests.add(client().prepareIndex("messages").setId("2").setSource("message", "message: this is a tcp handshake"));
         indexRandom(true, false, indexRequests);
+        indexRandomForConcurrentSearch("messages");
 
         SearchResponse response = client().prepareSearch("messages").setQuery(queryStringQuery("/TLS/").defaultField("message")).get();
         assertNoFailures(response);
@@ -282,6 +285,7 @@ public void testAllFields() throws Exception {
         List<IndexRequestBuilder> reqs = new ArrayList<>();
         reqs.add(client().prepareIndex("test_1").setId("1").setSource("f1", "foo", "f2", "eggplant"));
         indexRandom(true, false, reqs);
+        indexRandomForConcurrentSearch("test_1");
 
         SearchResponse resp = client().prepareSearch("test_1")
             .setQuery(queryStringQuery("foo eggplant").defaultOperator(Operator.AND))
@@ -374,6 +378,7 @@ public void testLimitOnExpandedFields() throws Exception {
 
         client().prepareIndex("testindex").setId("1").setSource("field_A0", "foo bar baz").get();
         refresh();
+        indexRandomForConcurrentSearch("testindex");
 
         // single field shouldn't trigger the limit
         doAssertOneHitForQueryString("field_A0:foo");
@@ -465,6 +470,7 @@ public void testFieldAliasOnDisallowedFieldType() throws Exception {
         List<IndexRequestBuilder> indexRequests = new ArrayList<>();
         indexRequests.add(client().prepareIndex("test").setId("1").setSource("f3", "text", "f2", "one"));
         indexRandom(true, false, indexRequests);
+        indexRandomForConcurrentSearch("test");
 
         // The wildcard field matches aliases for both a text and geo_point field.
         // By default, the geo_point field should be ignored when building the query.

server/src/internalClusterTest/java/org/opensearch/search/query/ScriptScoreQueryIT.java

@@ -109,13 +109,14 @@ protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() {
     // 1) only matched docs retrieved
     // 2) score is calculated based on a script with params
     // 3) min score applied
-    public void testScriptScore() {
+    public void testScriptScore() throws Exception {
         assertAcked(prepareCreate("test-index").setMapping("field1", "type=text", "field2", "type=double"));
         int docCount = 10;
         for (int i = 1; i <= docCount; i++) {
             client().prepareIndex("test-index").setId("" + i).setSource("field1", "text" + (i % 2), "field2", i).get();
         }
         refresh();
+        indexRandomForConcurrentSearch("test-index");
 
         Map<String, Object> params = new HashMap<>();
         params.put("param1", 0.1);
@@ -135,13 +136,14 @@ public void testScriptScore() {
         assertOrderedSearchHits(resp, "10", "8", "6");
     }
 
-    public void testScriptScoreBoolQuery() {
+    public void testScriptScoreBoolQuery() throws Exception {
         assertAcked(prepareCreate("test-index").setMapping("field1", "type=text", "field2", "type=double"));
         int docCount = 10;
         for (int i = 1; i <= docCount; i++) {
             client().prepareIndex("test-index").setId("" + i).setSource("field1", "text" + i, "field2", i).get();
         }
         refresh();
+        indexRandomForConcurrentSearch("test-index");
 
         Map<String, Object> params = new HashMap<>();
         params.put("param1", 0.1);
@@ -155,7 +157,7 @@ public void testScriptScoreBoolQuery() {
     }
 
     // test that when the internal query is rewritten script_score works well
-    public void testRewrittenQuery() {
+    public void testRewrittenQuery() throws Exception {
         assertAcked(
             prepareCreate("test-index2").setSettings(Settings.builder().put("index.number_of_shards", 1))
                 .setMapping("field1", "type=date", "field2", "type=double")
@@ -164,6 +166,7 @@ public void testRewrittenQuery() {
         client().prepareIndex("test-index2").setId("2").setSource("field1", "2019-10-01", "field2", 2).get();
         client().prepareIndex("test-index2").setId("3").setSource("field1", "2019-11-01", "field2", 3).get();
         refresh();
+        indexRandomForConcurrentSearch("test-index2");
 
         RangeQueryBuilder rangeQB = new RangeQueryBuilder("field1").from("2019-01-01"); // the query should be rewritten to from:null
         Map<String, Object> params = new HashMap<>();
@@ -174,14 +177,15 @@ public void testRewrittenQuery() {
         assertOrderedSearchHits(resp, "3", "2", "1");
     }
 
-    public void testDisallowExpensiveQueries() {
+    public void testDisallowExpensiveQueries() throws Exception {
         try {
             assertAcked(prepareCreate("test-index").setMapping("field1", "type=text", "field2", "type=double"));
             int docCount = 10;
             for (int i = 1; i <= docCount; i++) {
                 client().prepareIndex("test-index").setId("" + i).setSource("field1", "text" + (i % 2), "field2", i).get();
             }
             refresh();
+            indexRandomForConcurrentSearch("test-index");
 
             Map<String, Object> params = new HashMap<>();
             params.put("param1", 0.1);

server/src/internalClusterTest/java/org/opensearch/search/query/SimpleQueryStringIT.java

@@ -150,6 +150,7 @@ public void testSimpleQueryString() throws ExecutionException, InterruptedExcept
             client().prepareIndex("test").setId("5").setSource("body", "quux baz spaghetti"),
             client().prepareIndex("test").setId("6").setSource("otherbody", "spaghetti")
         );
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse searchResponse = client().prepareSearch().setQuery(simpleQueryStringQuery("foo bar")).get();
         assertHitCount(searchResponse, 3L);
@@ -199,6 +200,7 @@ public void testSimpleQueryStringMinimumShouldMatch() throws Exception {
             client().prepareIndex("test").setId("3").setSource("body", "foo bar"),
             client().prepareIndex("test").setId("4").setSource("body", "foo baz bar")
         );
+        indexRandomForConcurrentSearch("test");
 
         logger.info("--> query 1");
         SearchResponse searchResponse = client().prepareSearch().setQuery(simpleQueryStringQuery("foo bar").minimumShouldMatch("2")).get();
@@ -235,6 +237,7 @@ public void testSimpleQueryStringMinimumShouldMatch() throws Exception {
             client().prepareIndex("test").setId("7").setSource("body2", "foo bar", "other", "foo"),
             client().prepareIndex("test").setId("8").setSource("body2", "foo baz bar", "other", "foo")
         );
+        indexRandomForConcurrentSearch("test");
 
         logger.info("--> query 5");
         searchResponse = client().prepareSearch()
@@ -256,7 +259,7 @@ public void testSimpleQueryStringMinimumShouldMatch() throws Exception {
         assertSearchHits(searchResponse, "6", "7", "8");
     }
 
-    public void testNestedFieldSimpleQueryString() throws IOException {
+    public void testNestedFieldSimpleQueryString() throws Exception {
         assertAcked(
             prepareCreate("test").setMapping(
                 jsonBuilder().startObject()
@@ -275,6 +278,7 @@ public void testNestedFieldSimpleQueryString() throws IOException {
         );
         client().prepareIndex("test").setId("1").setSource("body", "foo bar baz").get();
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse searchResponse = client().prepareSearch().setQuery(simpleQueryStringQuery("foo bar baz").field("body")).get();
         assertHitCount(searchResponse, 1L);
@@ -359,6 +363,8 @@ public void testSimpleQueryStringLenient() throws ExecutionException, Interrupte
             client().prepareIndex("test2").setId("10").setSource("field", 5)
         );
         refresh();
+        indexRandomForConcurrentSearch("test1");
+        indexRandomForConcurrentSearch("test2");
 
         SearchResponse searchResponse = client().prepareSearch()
             .setAllowPartialSearchResults(true)
@@ -419,6 +425,7 @@ public void testSimpleQueryStringUsesFieldAnalyzer() throws Exception {
         client().prepareIndex("test").setId("2").setSource("foo", 234, "bar", "bcd").get();
 
         refresh();
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse searchResponse = client().prepareSearch().setQuery(simpleQueryStringQuery("123").field("foo").field("bar")).get();
         assertHitCount(searchResponse, 1L);
@@ -469,6 +476,7 @@ public void testBasicAllQuery() throws Exception {
         reqs.add(client().prepareIndex("test").setId("2").setSource("f2", "Bar"));
         reqs.add(client().prepareIndex("test").setId("3").setSource("f3", "foo bar baz"));
         indexRandom(true, false, reqs);
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("foo")).get();
         assertHitCount(resp, 2L);
@@ -492,6 +500,7 @@ public void testWithDate() throws Exception {
         reqs.add(client().prepareIndex("test").setId("1").setSource("f1", "foo", "f_date", "2015/09/02"));
         reqs.add(client().prepareIndex("test").setId("2").setSource("f1", "bar", "f_date", "2015/09/01"));
         indexRandom(true, false, reqs);
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("foo bar")).get();
         assertHits(resp.getHits(), "1", "2");
@@ -523,6 +532,7 @@ public void testWithLotsOfTypes() throws Exception {
             client().prepareIndex("test").setId("2").setSource("f1", "bar", "f_date", "2015/09/01", "f_float", "1.8", "f_ip", "127.0.0.2")
         );
         indexRandom(true, false, reqs);
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("foo bar")).get();
         assertHits(resp.getHits(), "1", "2");
@@ -550,6 +560,7 @@ public void testDocWithAllTypes() throws Exception {
         String docBody = copyToStringFromClasspath("/org/opensearch/search/query/all-example-document.json");
         reqs.add(client().prepareIndex("test").setId("1").setSource(docBody, MediaTypeRegistry.JSON));
         indexRandom(true, false, reqs);
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("foo")).get();
         assertHits(resp.getHits(), "1");
@@ -596,6 +607,7 @@ public void testKeywordWithWhitespace() throws Exception {
         reqs.add(client().prepareIndex("test").setId("2").setSource("f1", "bar"));
         reqs.add(client().prepareIndex("test").setId("3").setSource("f1", "foo bar"));
         indexRandom(true, false, reqs);
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("foo")).get();
         assertHits(resp.getHits(), "3");
@@ -663,6 +675,7 @@ public void testFieldAlias() throws Exception {
         indexRequests.add(client().prepareIndex("test").setId("2").setSource("f3", "value", "f2", "two"));
         indexRequests.add(client().prepareIndex("test").setId("3").setSource("f3", "another value", "f2", "three"));
         indexRandom(true, false, indexRequests);
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse response = client().prepareSearch("test").setQuery(simpleQueryStringQuery("value").field("f3_alias")).get();
 
@@ -681,6 +694,7 @@ public void testFieldAliasWithWildcardField() throws Exception {
         indexRequests.add(client().prepareIndex("test").setId("2").setSource("f3", "value", "f2", "two"));
         indexRequests.add(client().prepareIndex("test").setId("3").setSource("f3", "another value", "f2", "three"));
         indexRandom(true, false, indexRequests);
+        indexRandomForConcurrentSearch("test");
 
         SearchResponse response = client().prepareSearch("test").setQuery(simpleQueryStringQuery("value").field("f3_*")).get();
 
@@ -697,6 +711,7 @@ public void testFieldAliasOnDisallowedFieldType() throws Exception {
         List<IndexRequestBuilder> indexRequests = new ArrayList<>();
         indexRequests.add(client().prepareIndex("test").setId("1").setSource("f3", "text", "f2", "one"));
         indexRandom(true, false, indexRequests);
+        indexRandomForConcurrentSearch("test");
 
         // The wildcard field matches aliases for both a text and boolean field.
         // By default, the boolean field should be ignored when building the query.