Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Backport 2.x] Add indexRandomForConcurrentSearch to tests #11265

Merged
merged 1 commit into from
Nov 20, 2023
Merged

[Backport 2.x] Add indexRandomForConcurrentSearch to tests #11265

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions ...Test/java/org/opensearch/search/aggregations/metrics/CardinalityWithRequestBreakerIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,7 @@ public void testRequestBreaker() throws Exception {
)
.get();

indexRandomForConcurrentSearch("test");
try {
client().prepareSearch("test")
.addAggregation(
Expand Down
16 changes: 16 additions & 0 deletions server/src/internalClusterTest/java/org/opensearch/search/fields/SearchFieldsIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -251,6 +251,7 @@ public void testStoredFields() throws Exception {
.get();

client().admin().indices().prepareRefresh().get();
indexRandomForConcurrentSearch("test");

SearchResponse searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addStoredField("field1").get();
assertThat(searchResponse.getHits().getTotalHits().value, equalTo(1L));
Expand Down Expand Up @@ -358,6 +359,7 @@ public void testScriptDocAndFields() throws Exception {
)
.get();
client().admin().indices().refresh(refreshRequest()).actionGet();
indexRandomForConcurrentSearch("test");

logger.info("running doc['num1'].value");
SearchResponse response = client().prepareSearch()
Expand Down Expand Up @@ -458,6 +460,7 @@ public void testScriptWithUnsignedLong() throws Exception {
)
.get();
client().admin().indices().refresh(refreshRequest()).actionGet();
indexRandomForConcurrentSearch("test");

SearchResponse response = client().prepareSearch()
.setQuery(matchAllQuery())
Expand Down Expand Up @@ -547,6 +550,7 @@ public void testScriptFieldWithNanos() throws Exception {
.setSource(jsonBuilder().startObject().field("date", "1970-01-01T00:00:00.000Z").endObject()),
client().prepareIndex("test").setId("2").setSource(jsonBuilder().startObject().field("date", date).endObject())
);
indexRandomForConcurrentSearch("test");

SearchResponse response = client().prepareSearch()
.setQuery(matchAllQuery())
Expand Down Expand Up @@ -632,6 +636,7 @@ public void testScriptFieldUsingSource() throws Exception {
)
.get();
client().admin().indices().refresh(refreshRequest()).actionGet();
indexRandomForConcurrentSearch("test");

SearchResponse response = client().prepareSearch()
.setQuery(matchAllQuery())
Expand Down Expand Up @@ -674,6 +679,7 @@ public void testScriptFieldUsingSource() throws Exception {
public void testScriptFieldsForNullReturn() throws Exception {
client().prepareIndex("test").setId("1").setSource("foo", "bar").setRefreshPolicy("true").get();

indexRandomForConcurrentSearch("test");
SearchResponse response = client().prepareSearch()
.setQuery(matchAllQuery())
.addScriptField("test_script_1", new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "return null", Collections.emptyMap()))
Expand Down Expand Up @@ -795,6 +801,7 @@ public void testStoredFieldsWithoutSource() throws Exception {
.get();

client().admin().indices().prepareRefresh().get();
indexRandomForConcurrentSearch("test");

SearchResponse searchResponse = client().prepareSearch()
.setQuery(matchAllQuery())
Expand Down Expand Up @@ -852,6 +859,7 @@ public void testSearchFieldsMetadata() throws Exception {
.setSource(jsonBuilder().startObject().field("field1", "value").endObject())
.setRefreshPolicy(IMMEDIATE)
.get();
indexRandomForConcurrentSearch("my-index");

SearchResponse searchResponse = client().prepareSearch("my-index").addStoredField("field1").addStoredField("_routing").get();

Expand All @@ -866,6 +874,7 @@ public void testSearchFieldsNonLeafField() throws Exception {
.setSource(jsonBuilder().startObject().startObject("field1").field("field2", "value1").endObject().endObject())
.setRefreshPolicy(IMMEDIATE)
.get();
indexRandomForConcurrentSearch("my-index");

assertFailures(
client().prepareSearch("my-index").addStoredField("field1"),
Expand Down Expand Up @@ -932,6 +941,7 @@ public void testGetFieldsComplexField() throws Exception {
);

client().prepareIndex("my-index").setId("1").setRefreshPolicy(IMMEDIATE).setSource(source, MediaTypeRegistry.JSON).get();
indexRandomForConcurrentSearch("my-index");

String field = "field1.field2.field3.field4";

Expand Down Expand Up @@ -1039,6 +1049,7 @@ public void testDocValueFields() throws Exception {
.get();

client().admin().indices().prepareRefresh().get();
indexRandomForConcurrentSearch("test");

SearchRequestBuilder builder = client().prepareSearch()
.setQuery(matchAllQuery())
Expand Down Expand Up @@ -1271,6 +1282,7 @@ public void testScriptFields() throws Exception {
);
}
indexRandom(true, reqs);
indexRandomForConcurrentSearch("index");
ensureSearchable();
SearchRequestBuilder req = client().prepareSearch("index");
for (String field : Arrays.asList("s", "ms", "l", "ml", "d", "md")) {
Expand Down Expand Up @@ -1326,6 +1338,7 @@ public void testDocValueFieldsWithFieldAlias() throws Exception {

index("test", MapperService.SINGLE_MAPPING_NAME, "1", "text_field", "foo", "date_field", formatter.print(date));
refresh("test");
indexRandomForConcurrentSearch("test");

SearchRequestBuilder builder = client().prepareSearch()
.setQuery(matchAllQuery())
Expand Down Expand Up @@ -1387,6 +1400,7 @@ public void testWildcardDocValueFieldsWithFieldAlias() throws Exception {

index("test", MapperService.SINGLE_MAPPING_NAME, "1", "text_field", "foo", "date_field", formatter.print(date));
refresh("test");
indexRandomForConcurrentSearch("test");

SearchRequestBuilder builder = client().prepareSearch()
.setQuery(matchAllQuery())
Expand Down Expand Up @@ -1440,6 +1454,7 @@ public void testStoredFieldsWithFieldAlias() throws Exception {

index("test", MapperService.SINGLE_MAPPING_NAME, "1", "field1", "value1", "field2", "value2");
refresh("test");
indexRandomForConcurrentSearch("test");

SearchResponse searchResponse = client().prepareSearch()
.setQuery(matchAllQuery())
Expand Down Expand Up @@ -1482,6 +1497,7 @@ public void testWildcardStoredFieldsWithFieldAlias() throws Exception {

index("test", MapperService.SINGLE_MAPPING_NAME, "1", "field1", "value1", "field2", "value2");
refresh("test");
indexRandomForConcurrentSearch("test");

SearchResponse searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addStoredField("field*").get();
assertHitCount(searchResponse, 1L);
Expand Down
10 changes: 10 additions & 0 deletions ...rc/internalClusterTest/java/org/opensearch/search/functionscore/DecayFunctionScoreIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -398,6 +398,7 @@ public void testBoostModeSettingWorks() throws Exception {
)
);
indexRandom(true, false, indexBuilders); // force no dummy docs
indexRandomForConcurrentSearch("test");

// Test Gauss
List<Float> lonlat = new ArrayList<>();
Expand Down Expand Up @@ -482,6 +483,7 @@ public void testParseGeoPoint() throws Exception {
constantScoreQuery(termQuery("test", "value")),
ScoreFunctionBuilders.weightFactorFunction(randomIntBetween(1, 10))
);
indexRandomForConcurrentSearch("test");
GeoPoint point = new GeoPoint(20, 11);
ActionFuture<SearchResponse> response = client().search(
searchRequest().searchType(SearchType.QUERY_THEN_FETCH)
Expand Down Expand Up @@ -535,6 +537,7 @@ public void testCombineModes() throws Exception {
.setRefreshPolicy(IMMEDIATE)
.setSource(jsonBuilder().startObject().field("test", "value value").field("num", 1.0).endObject())
.get();
indexRandomForConcurrentSearch("test");
FunctionScoreQueryBuilder baseQuery = functionScoreQuery(
constantScoreQuery(termQuery("test", "value")),
ScoreFunctionBuilders.weightFactorFunction(2)
Expand Down Expand Up @@ -654,6 +657,7 @@ public void testCombineModesExplain() throws Exception {
constantScoreQuery(termQuery("test", "value")).queryName("query1"),
ScoreFunctionBuilders.weightFactorFunction(2, "weight1")
);
indexRandomForConcurrentSearch("test");
// decay score should return 0.5 for this function and baseQuery should return 2.0f as it's score
ActionFuture<SearchResponse> response = client().search(
searchRequest().searchType(SearchType.QUERY_THEN_FETCH)
Expand Down Expand Up @@ -762,6 +766,7 @@ public void testParseDateMath() throws Exception {
).actionGet();
refresh();

indexRandomForConcurrentSearch("test");
SearchResponse sr = client().search(
searchRequest().source(
searchSource().query(functionScoreQuery(termQuery("test", "value"), gaussDecayFunction("num1", "now", "2d")))
Expand Down Expand Up @@ -817,6 +822,7 @@ public void testValueMissingLin() throws Exception {
).actionGet();

refresh();
indexRandomForConcurrentSearch("test");

ActionFuture<SearchResponse> response = client().search(
searchRequest().searchType(SearchType.QUERY_THEN_FETCH)
Expand Down Expand Up @@ -893,6 +899,7 @@ public void testDateWithoutOrigin() throws Exception {
).actionGet();

refresh();
indexRandomForConcurrentSearch("test");

ActionFuture<SearchResponse> response = client().search(
searchRequest().searchType(SearchType.QUERY_THEN_FETCH)
Expand Down Expand Up @@ -974,6 +981,7 @@ public void testManyDocsLin() throws Exception {
List<Float> lonlat = new ArrayList<>();
lonlat.add(100f);
lonlat.add(110f);
indexRandomForConcurrentSearch("test");
ActionFuture<SearchResponse> response = client().search(
searchRequest().searchType(SearchType.QUERY_THEN_FETCH)
.source(
Expand Down Expand Up @@ -1107,6 +1115,7 @@ public void testNoQueryGiven() throws Exception {
client().index(indexRequest("test").source(jsonBuilder().startObject().field("test", "value").field("num", 1.0).endObject()))
.actionGet();
refresh();
indexRandomForConcurrentSearch("test");
// so, we indexed a string field, but now we try to score a num field
ActionFuture<SearchResponse> response = client().search(
searchRequest().searchType(SearchType.QUERY_THEN_FETCH)
Expand Down Expand Up @@ -1171,6 +1180,7 @@ public void testMultiFieldOptions() throws Exception {
);

indexRandom(true, doc1, doc2);
indexRandomForConcurrentSearch("test");

ActionFuture<SearchResponse> response = client().search(searchRequest().source(searchSource().query(baseQuery)));
SearchResponse sr = response.actionGet();
Expand Down
7 changes: 4 additions & 3 deletions ...ternalClusterTest/java/org/opensearch/search/functionscore/FunctionScoreFieldValueIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ protected Settings featureFlagSettings() {
return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build();
}

public void testFieldValueFactor() throws IOException {
public void testFieldValueFactor() throws IOException, InterruptedException {
assertAcked(
prepareCreate("test").setMapping(
jsonBuilder().startObject()
Expand All @@ -99,8 +99,8 @@ public void testFieldValueFactor() throws IOException {
client().prepareIndex("test").setId("1").setSource("test", 5, "body", "foo").get();
client().prepareIndex("test").setId("2").setSource("test", 17, "body", "foo").get();
client().prepareIndex("test").setId("3").setSource("body", "bar").get();

refresh();
indexRandomForConcurrentSearch("test");

// document 2 scores higher because 17 > 5
SearchResponse response = client().prepareSearch("test")
Expand Down Expand Up @@ -189,7 +189,7 @@ public void testFieldValueFactor() throws IOException {
}
}

public void testFieldValueFactorExplain() throws IOException {
public void testFieldValueFactorExplain() throws IOException, InterruptedException {
assertAcked(
prepareCreate("test").setMapping(
jsonBuilder().startObject()
Expand All @@ -210,6 +210,7 @@ public void testFieldValueFactorExplain() throws IOException {
client().prepareIndex("test").setId("3").setSource("body", "bar").get();

refresh();
indexRandomForConcurrentSearch("test");

// document 2 scores higher because 17 > 5
final String functionName = "func1";
Expand Down
13 changes: 9 additions & 4 deletions server/src/internalClusterTest/java/org/opensearch/search/functionscore/FunctionScoreIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,10 +126,11 @@ protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() {
}
}

public void testScriptScoresNested() throws IOException {
public void testScriptScoresNested() throws IOException, InterruptedException {
createIndex(INDEX);
index(INDEX, TYPE, "1", jsonBuilder().startObject().field("dummy_field", 1).endObject());
refresh();
indexRandomForConcurrentSearch(INDEX);

Script scriptOne = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "1", Collections.emptyMap());
Script scriptTwo = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "get score value", Collections.emptyMap());
Expand All @@ -148,10 +149,11 @@ public void testScriptScoresNested() throws IOException {
assertThat(response.getHits().getAt(0).getScore(), equalTo(1.0f));
}

public void testScriptScoresWithAgg() throws IOException {
public void testScriptScoresWithAgg() throws IOException, InterruptedException {
createIndex(INDEX);
index(INDEX, TYPE, "1", jsonBuilder().startObject().field("dummy_field", 1).endObject());
refresh();
indexRandomForConcurrentSearch(INDEX);

Script script = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "get score value", Collections.emptyMap());

Expand All @@ -166,10 +168,11 @@ public void testScriptScoresWithAgg() throws IOException {
assertThat(((Terms) response.getAggregations().asMap().get("score_agg")).getBuckets().get(0).getDocCount(), is(1L));
}

public void testScriptScoresWithAggWithExplain() throws IOException {
public void testScriptScoresWithAggWithExplain() throws IOException, InterruptedException {
createIndex(INDEX);
index(INDEX, TYPE, "1", jsonBuilder().startObject().field("dummy_field", 1).endObject());
refresh();
indexRandomForConcurrentSearch(INDEX);

Script script = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "get score value", Collections.emptyMap());

Expand All @@ -195,7 +198,7 @@ public void testScriptScoresWithAggWithExplain() throws IOException {
assertThat(((Terms) response.getAggregations().asMap().get("score_agg")).getBuckets().get(0).getDocCount(), is(1L));
}

public void testMinScoreFunctionScoreBasic() throws IOException {
public void testMinScoreFunctionScoreBasic() throws IOException, InterruptedException {
float score = randomValueOtherThanMany((f) -> Float.compare(f, 0) < 0, OpenSearchTestCase::randomFloat);
float minScore = randomValueOtherThanMany((f) -> Float.compare(f, 0) < 0, OpenSearchTestCase::randomFloat);
index(
Expand All @@ -207,6 +210,7 @@ public void testMinScoreFunctionScoreBasic() throws IOException {
.endObject()
);
refresh();
indexRandomForConcurrentSearch(INDEX);
ensureYellow();

Script script = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "doc['random_score']", Collections.emptyMap());
Expand Down Expand Up @@ -291,6 +295,7 @@ public void testWithEmptyFunctions() throws IOException, ExecutionException, Int
assertAcked(prepareCreate("test"));
index("test", "testtype", "1", jsonBuilder().startObject().field("text", "test text").endObject());
refresh();
indexRandomForConcurrentSearch("test");

SearchResponse termQuery = client().search(searchRequest().source(searchSource().explain(true).query(termQuery("text", "text"))))
.get();
Expand Down
1 change: 1 addition & 0 deletions ...c/internalClusterTest/java/org/opensearch/search/functionscore/FunctionScorePluginIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,7 @@ public void testPlugin() throws Exception {
).actionGet();

client().admin().indices().prepareRefresh().get();
indexRandomForConcurrentSearch("test");
DecayFunctionBuilder<?> gfb = new CustomDistanceScoreBuilder("num1", "2013-05-28", "+1d");

ActionFuture<SearchResponse> response = client().search(
Expand Down
8 changes: 7 additions & 1 deletion server/src/internalClusterTest/java/org/opensearch/search/functionscore/QueryRescorerIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,14 +110,15 @@ protected Settings featureFlagSettings() {
return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build();
}

public void testEnforceWindowSize() {
public void testEnforceWindowSize() throws InterruptedException {
createIndex("test");
// this
int iters = scaledRandomIntBetween(10, 20);
for (int i = 0; i < iters; i++) {
client().prepareIndex("test").setId(Integer.toString(i)).setSource("f", Integer.toString(i)).get();
}
refresh();
indexRandomForConcurrentSearch("test");

int numShards = getNumShards("test").numPrimaries;
for (int j = 0; j < iters; j++) {
Expand Down Expand Up @@ -169,6 +170,7 @@ public void testRescorePhrase() throws Exception {
.setSource("field1", "quick huge brown", "field2", "the quick lazy huge brown fox jumps over the tree")
.get();
refresh();
indexRandomForConcurrentSearch("test");
SearchResponse searchResponse = client().prepareSearch()
.setQuery(QueryBuilders.matchQuery("field1", "the quick brown").operator(Operator.OR))
.setRescorer(
Expand Down Expand Up @@ -474,6 +476,7 @@ private static void assertEquivalent(String query, SearchResponse plain, SearchR
public void testEquivalence() throws Exception {
// no dummy docs since merges can change scores while we run queries.
int numDocs = indexRandomNumbers("whitespace", -1, false);
indexRandomForConcurrentSearch("test");

final int iters = scaledRandomIntBetween(50, 100);
for (int i = 0; i < iters; i++) {
Expand Down Expand Up @@ -545,6 +548,7 @@ public void testExplain() throws Exception {
.setSource("field1", "quick huge brown", "field2", "the quick lazy huge brown fox jumps over the tree")
.get();
refresh();
indexRandomForConcurrentSearch("test");

{
SearchResponse searchResponse = client().prepareSearch()
Expand Down Expand Up @@ -816,6 +820,7 @@ public void testFromSize() throws Exception {
client().prepareIndex("test").setId("" + i).setSource("text", "hello world").get();
}
refresh();
indexRandomForConcurrentSearch("test");

SearchRequestBuilder request = client().prepareSearch();
request.setQuery(QueryBuilders.termQuery("text", "hello"));
Expand All @@ -832,6 +837,7 @@ public void testRescorePhaseWithInvalidSort() throws Exception {
client().prepareIndex("test").setId("" + i).setSource("number", 0).get();
}
refresh();
indexRandomForConcurrentSearch("test");

Exception exc = expectThrows(
Exception.class,
Expand Down
Loading