Skip to content

Commit 0758ddc

Browse files
karenyrxkarenyrx
authored
Merge branch 'main' into 0.18.0_script
Signed-off-by: Karen X <karenxyr@gmail.com>
2 parents 258f392 + ae140b5 commit 0758ddc

File tree

11 files changed

+654
-68
lines changed

11 files changed

+654
-68
lines changed

‎.github/benchmark-configs.json‎

Lines changed: 16 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -35,30 +35,13 @@
3535
"baseline_cluster_config": "x64-r5.xlarge-single-node-1-shard-0-replica-baseline"
3636
},
3737
"id_3": {
38-
"description": "Search only test-procedure for NYC_TAXIS, uses snapshot to restore the data for OS-3.0.0",
39-
"supported_major_versions": ["3"],
40-
"cluster-benchmark-configs": {
41-
"SINGLE_NODE_CLUSTER": "true",
42-
"MIN_DISTRIBUTION": "true",
43-
"TEST_WORKLOAD": "nyc_taxis",
44-
"WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-3x\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots-3x\",\"snapshot_name\":\"nyc_taxis_1_shard\"}",
45-
"CAPTURE_NODE_STAT": "true",
46-
"TEST_PROCEDURE": "restore-from-snapshot"
47-
},
48-
"cluster_configuration": {
49-
"size": "Single-Node",
50-
"data_instance_config": "4vCPU, 32G Mem, 16G Heap"
51-
},
52-
"baseline_cluster_config": "x64-r5.xlarge-1-shard-0-replica-snapshot-baseline"
53-
},
54-
"id_4": {
5538
"description": "Search only test-procedure for big5, uses snapshot to restore the data for OS-3.x",
5639
"supported_major_versions": ["3"],
5740
"cluster-benchmark-configs": {
5841
"SINGLE_NODE_CLUSTER": "true",
5942
"MIN_DISTRIBUTION": "true",
6043
"TEST_WORKLOAD": "big5",
61-
"WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-3x\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"10.2.1\",\"snapshot_name\":\"big5_1_shard_single_client\"}",
44+
"WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-3x\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"10.3.0\",\"snapshot_name\":\"big5_1_shard_single_client\"}",
6245
"CAPTURE_NODE_STAT": "true",
6346
"TEST_PROCEDURE": "restore-from-snapshot"
6447
},
@@ -68,7 +51,7 @@
6851
},
6952
"baseline_cluster_config": "x64-r5.xlarge-1-shard-0-replica-snapshot-baseline"
7053
},
71-
"id_5": {
54+
"id_4": {
7255
"description": "Indexing and search configuration for pmc workload",
7356
"supported_major_versions": ["3"],
7457
"cluster-benchmark-configs": {
@@ -84,7 +67,7 @@
8467
},
8568
"baseline_cluster_config": "x64-r5.xlarge-single-node-1-shard-0-replica-baseline"
8669
},
87-
"id_6": {
70+
"id_5": {
8871
"description": "Indexing only configuration for stack-overflow workload",
8972
"supported_major_versions": ["3"],
9073
"cluster-benchmark-configs": {
@@ -100,15 +83,15 @@
10083
},
10184
"baseline_cluster_config": "x64-r5.xlarge-single-node-1-shard-0-replica-baseline"
10285
},
103-
"id_7": {
86+
"id_6": {
10487
"description": "Search only test-procedure for big5 with concurrent segment search setting enabled",
10588
"supported_major_versions": ["3"],
10689
"cluster-benchmark-configs": {
10790
"SINGLE_NODE_CLUSTER": "true",
10891
"MIN_DISTRIBUTION": "true",
10992
"TEST_WORKLOAD": "big5",
11093
"ADDITIONAL_CONFIG": "search.concurrent_segment_search.enabled:true",
111-
"WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-3x\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots-3x\",\"snapshot_name\":\"big5_1_shard_single_client\"}",
94+
"WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-3x\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"10.3.0\",\"snapshot_name\":\"big5_1_shard_single_client\"}",
11295
"CAPTURE_NODE_STAT": "true",
11396
"TEST_PROCEDURE": "restore-from-snapshot"
11497
},
@@ -118,15 +101,15 @@
118101
},
119102
"baseline_cluster_config": "x64-r5.xlarge-1-shard-0-replica-snapshot-baseline"
120103
},
121-
"id_8": {
104+
"id_7": {
122105
"description": "Search only test-procedure for big5 with concurrent segment search mode as all",
123106
"supported_major_versions": ["3"],
124107
"cluster-benchmark-configs": {
125108
"SINGLE_NODE_CLUSTER": "true",
126109
"MIN_DISTRIBUTION": "true",
127110
"TEST_WORKLOAD": "big5",
128111
"ADDITIONAL_CONFIG": "search.concurrent_segment_search.mode:all",
129-
"WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-3x\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots-3x\",\"snapshot_name\":\"big5_1_shard_single_client\"}",
112+
"WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-3x\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"10.3.0\",\"snapshot_name\":\"big5_1_shard_single_client\"}",
130113
"CAPTURE_NODE_STAT": "true",
131114
"TEST_PROCEDURE": "restore-from-snapshot"
132115
},
@@ -136,15 +119,15 @@
136119
},
137120
"baseline_cluster_config": "x64-r5.xlarge-1-shard-0-replica-snapshot-baseline"
138121
},
139-
"id_9": {
122+
"id_8": {
140123
"description": "Search only test-procedure for big5 with concurrent segment search mode as auto",
141124
"supported_major_versions": ["3"],
142125
"cluster-benchmark-configs": {
143126
"SINGLE_NODE_CLUSTER": "true",
144127
"MIN_DISTRIBUTION": "true",
145128
"TEST_WORKLOAD": "big5",
146129
"ADDITIONAL_CONFIG": "search.concurrent_segment_search.mode:auto",
147-
"WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-3x\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots-3x\",\"snapshot_name\":\"big5_1_shard_single_client\"}",
130+
"WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-3x\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"10.3.0\",\"snapshot_name\":\"big5_1_shard_single_client\"}",
148131
"CAPTURE_NODE_STAT": "true",
149132
"TEST_PROCEDURE": "restore-from-snapshot"
150133
},
@@ -154,15 +137,15 @@
154137
},
155138
"baseline_cluster_config": "x64-r5.xlarge-1-shard-0-replica-snapshot-baseline"
156139
},
157-
"id_10": {
140+
"id_9": {
158141
"description": "Search only test-procedure for big5, uses snapshot to restore the data for OS-3.0.0. Enables range query approximation.",
159142
"supported_major_versions": ["3"],
160143
"cluster-benchmark-configs": {
161144
"SINGLE_NODE_CLUSTER": "true",
162145
"MIN_DISTRIBUTION": "true",
163146
"TEST_WORKLOAD": "big5",
164147
"ADDITIONAL_CONFIG": "opensearch.experimental.feature.approximate_point_range_query.enabled:true",
165-
"WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-3x\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots-3x\",\"snapshot_name\":\"big5_1_shard_single_client\"}",
148+
"WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-3x\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"10.3.0\",\"snapshot_name\":\"big5_1_shard_single_client\"}",
166149
"CAPTURE_NODE_STAT": "true",
167150
"TEST_PROCEDURE": "restore-from-snapshot"
168151
},
@@ -172,7 +155,7 @@
172155
},
173156
"baseline_cluster_config": "x64-r5.xlarge-1-shard-0-replica-snapshot-baseline"
174157
},
175-
"id_11": {
158+
"id_10": {
176159
"description": "Benchmarking config for NESTED workload, benchmarks nested queries with inner-hits",
177160
"supported_major_versions": ["3"],
178161
"cluster-benchmark-configs": {
@@ -188,31 +171,14 @@
188171
},
189172
"baseline_cluster_config": "x64-r5.xlarge-single-node-1-shard-0-replica-baseline"
190173
},
191-
"id_12": {
192-
"description": "Search only test-procedure for HTTP_LOGS, uses snapshot to restore the data for OS-3.0.0",
193-
"supported_major_versions": ["3"],
194-
"cluster-benchmark-configs": {
195-
"SINGLE_NODE_CLUSTER": "true",
196-
"MIN_DISTRIBUTION": "true",
197-
"TEST_WORKLOAD": "http_logs",
198-
"WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-3x\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots-3x\",\"snapshot_name\":\"http_logs_1_shard\"}",
199-
"CAPTURE_NODE_STAT": "true",
200-
"TEST_PROCEDURE": "restore-from-snapshot"
201-
},
202-
"cluster_configuration": {
203-
"size": "Single-Node",
204-
"data_instance_config": "4vCPU, 32G Mem, 16G Heap"
205-
},
206-
"baseline_cluster_config": "x64-r5.xlarge-1-shard-0-replica-snapshot-baseline"
207-
},
208-
"id_13": {
174+
"id_11": {
209175
"description": "Search only test-procedure for HTTP_LOGS, uses snapshot to restore the data for OS-3.x",
210176
"supported_major_versions": ["3"],
211177
"cluster-benchmark-configs": {
212178
"SINGLE_NODE_CLUSTER": "true",
213179
"MIN_DISTRIBUTION": "true",
214180
"TEST_WORKLOAD": "http_logs",
215-
"WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-3x\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"10.2.1\",\"snapshot_name\":\"http_logs_1_shard\"}",
181+
"WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-3x\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"10.3.0\",\"snapshot_name\":\"http_logs_1_shard\"}",
216182
"CAPTURE_NODE_STAT": "true",
217183
"TEST_PROCEDURE": "restore-from-snapshot"
218184
},
@@ -222,14 +188,14 @@
222188
},
223189
"baseline_cluster_config": "x64-r5.xlarge-1-shard-0-replica-snapshot-baseline"
224190
},
225-
"id_14": {
191+
"id_12": {
226192
"description": "Search only test-procedure for NYC_TAXIS, uses snapshot to restore the data for OS-3.x",
227193
"supported_major_versions": ["3"],
228194
"cluster-benchmark-configs": {
229195
"SINGLE_NODE_CLUSTER": "true",
230196
"MIN_DISTRIBUTION": "true",
231197
"TEST_WORKLOAD": "nyc_taxis",
232-
"WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-3x\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"10.2.1\",\"snapshot_name\":\"nyc_taxis_1_shard\"}",
198+
"WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-3x\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"10.3.0\",\"snapshot_name\":\"nyc_taxis_1_shard\"}",
233199
"CAPTURE_NODE_STAT": "true",
234200
"TEST_PROCEDURE": "restore-from-snapshot"
235201
},

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
3131
- Add skip_list param for date, scaled float and token count fields ([#19142](https://github.com/opensearch-project/OpenSearch/pull/19142))
3232
- Implement GRPC MatchPhrase, MultiMatch queries ([#19449](https://github.com/opensearch-project/OpenSearch/pull/19449))
3333
- Optimize gRPC transport thread management for improved throughput ([#19278](https://github.com/opensearch-project/OpenSearch/pull/19278))
34+
- Implement GRPC Boolean query and inject registry for all internal query converters ([#19391](https://github.com/opensearch-project/OpenSearch/pull/19391))
3435

3536
### Changed
3637
- Refactor `if-else` chains to use `Java 17 pattern matching switch expressions`(([#18965](https://github.com/opensearch-project/OpenSearch/pull/18965))

‎modules/transport-grpc/src/main/java/org/opensearch/transport/grpc/GrpcPlugin.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -329,6 +329,10 @@ public Collection<Object> createComponents(
329329
queryRegistry.registerConverter(converter);
330330
}
331331
logger.info("Successfully injected registry and registered all {} external converters", queryConverters.size());
332+
333+
// Update the registry on all converters (including built-in ones) so they can access external converters
334+
queryRegistry.updateRegistryOnAllConverters();
335+
logger.info("Updated registry on all converters to include external converters");
332336
} else {
333337
logger.info("No external QueryBuilderProtoConverter(s) to register");
334338
}

‎modules/transport-grpc/src/main/java/org/opensearch/transport/grpc/proto/request/search/query/BoolQueryBuilderProtoConverter.java‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
*
4+
* The OpenSearch Contributors require contributions made to
5+
* this file be licensed under the Apache-2.0 license or a
6+
* compatible open source license.
7+
*/
8+
package org.opensearch.transport.grpc.proto.request.search.query;
9+
10+
import org.opensearch.index.query.QueryBuilder;
11+
import org.opensearch.protobufs.QueryContainer;
12+
import org.opensearch.transport.grpc.spi.QueryBuilderProtoConverter;
13+
import org.opensearch.transport.grpc.spi.QueryBuilderProtoConverterRegistry;
14+
15+
/**
16+
* Converter for Bool queries.
17+
* This class implements the QueryBuilderProtoConverter interface to provide Bool query support
18+
* for the gRPC transport module.
19+
*/
20+
public class BoolQueryBuilderProtoConverter implements QueryBuilderProtoConverter {
21+
22+
/**
23+
* Default constructor for BoolQueryBuilderProtoConverter.
24+
*/
25+
public BoolQueryBuilderProtoConverter() {}
26+
27+
private QueryBuilderProtoConverterRegistry registry;
28+
29+
@Override
30+
public void setRegistry(QueryBuilderProtoConverterRegistry registry) {
31+
this.registry = registry;
32+
}
33+
34+
@Override
35+
public QueryContainer.QueryContainerCase getHandledQueryCase() {
36+
return QueryContainer.QueryContainerCase.BOOL;
37+
}
38+
39+
@Override
40+
public QueryBuilder fromProto(QueryContainer queryContainer) {
41+
if (queryContainer == null || queryContainer.getQueryContainerCase() != QueryContainer.QueryContainerCase.BOOL) {
42+
throw new IllegalArgumentException("QueryContainer does not contain a Bool query");
43+
}
44+
45+
return BoolQueryBuilderProtoUtils.fromProto(queryContainer.getBool(), registry);
46+
}
47+
}

‎modules/transport-grpc/src/main/java/org/opensearch/transport/grpc/proto/request/search/query/BoolQueryBuilderProtoUtils.java‎

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
*
4+
* The OpenSearch Contributors require contributions made to
5+
* this file be licensed under the Apache-2.0 license or a
6+
* compatible open source license.
7+
*/
8+
package org.opensearch.transport.grpc.proto.request.search.query;
9+
10+
import org.opensearch.index.query.AbstractQueryBuilder;
11+
import org.opensearch.index.query.BoolQueryBuilder;
12+
import org.opensearch.index.query.QueryBuilder;
13+
import org.opensearch.protobufs.BoolQuery;
14+
import org.opensearch.protobufs.MinimumShouldMatch;
15+
import org.opensearch.protobufs.QueryContainer;
16+
import org.opensearch.transport.grpc.spi.QueryBuilderProtoConverterRegistry;
17+
18+
import java.util.List;
19+
20+
/**
21+
* Utility class for converting BoolQuery Protocol Buffers to OpenSearch objects.
22+
* This class provides methods to transform Protocol Buffer representations of bool queries
23+
* into their corresponding OpenSearch BoolQueryBuilder implementations for search operations.
24+
*/
25+
class BoolQueryBuilderProtoUtils {
26+
27+
private BoolQueryBuilderProtoUtils() {
28+
// Utility class, no instances
29+
}
30+
31+
/**
32+
* Converts a Protocol Buffer BoolQuery to an OpenSearch BoolQueryBuilder.
33+
* Similar to {@link BoolQueryBuilder#fromXContent(org.opensearch.core.xcontent.XContentParser)}, this method
34+
* parses the Protocol Buffer representation and creates a properly configured
35+
* BoolQueryBuilder with the appropriate must, must_not, should, filter clauses,
36+
* boost, query name, and minimum_should_match.
37+
*
38+
* @param boolQueryProto The Protocol Buffer BoolQuery object
39+
* @param registry The registry to use for converting nested queries
40+
* @return A configured BoolQueryBuilder instance
41+
*/
42+
static BoolQueryBuilder fromProto(BoolQuery boolQueryProto, QueryBuilderProtoConverterRegistry registry) {
43+
String queryName = null;
44+
float boost = AbstractQueryBuilder.DEFAULT_BOOST;
45+
String minimumShouldMatch = null;
46+
boolean adjustPureNegative = BoolQueryBuilder.ADJUST_PURE_NEGATIVE_DEFAULT;
47+
48+
// Create BoolQueryBuilder
49+
BoolQueryBuilder boolQuery = new BoolQueryBuilder();
50+
51+
// Process name
52+
if (boolQueryProto.hasXName()) {
53+
queryName = boolQueryProto.getXName();
54+
boolQuery.queryName(queryName);
55+
}
56+
57+
// Process boost
58+
if (boolQueryProto.hasBoost()) {
59+
boost = boolQueryProto.getBoost();
60+
boolQuery.boost(boost);
61+
}
62+
63+
// Process minimum_should_match
64+
if (boolQueryProto.hasMinimumShouldMatch()) {
65+
MinimumShouldMatch minimumShouldMatchProto = boolQueryProto.getMinimumShouldMatch();
66+
switch (minimumShouldMatchProto.getMinimumShouldMatchCase()) {
67+
case INT32:
68+
minimumShouldMatch = String.valueOf(minimumShouldMatchProto.getInt32());
69+
break;
70+
case STRING:
71+
minimumShouldMatch = minimumShouldMatchProto.getString();
72+
break;
73+
default:
74+
// No minimum_should_match specified
75+
break;
76+
}
77+
78+
if (minimumShouldMatch != null) {
79+
boolQuery.minimumShouldMatch(minimumShouldMatch);
80+
}
81+
}
82+
83+
// Process must clauses
84+
List<QueryContainer> mustClauses = boolQueryProto.getMustList();
85+
for (QueryContainer queryContainer : mustClauses) {
86+
QueryBuilder queryBuilder = registry.fromProto(queryContainer);
87+
if (queryBuilder != null) {
88+
boolQuery.must(queryBuilder);
89+
}
90+
}
91+
92+
// Process must_not clauses
93+
List<QueryContainer> mustNotClauses = boolQueryProto.getMustNotList();
94+
for (QueryContainer queryContainer : mustNotClauses) {
95+
QueryBuilder queryBuilder = registry.fromProto(queryContainer);
96+
if (queryBuilder != null) {
97+
boolQuery.mustNot(queryBuilder);
98+
}
99+
}
100+
101+
// Process should clauses
102+
List<QueryContainer> shouldClauses = boolQueryProto.getShouldList();
103+
for (QueryContainer queryContainer : shouldClauses) {
104+
QueryBuilder queryBuilder = registry.fromProto(queryContainer);
105+
if (queryBuilder != null) {
106+
boolQuery.should(queryBuilder);
107+
}
108+
}
109+
110+
// Process filter clauses
111+
List<QueryContainer> filterClauses = boolQueryProto.getFilterList();
112+
for (QueryContainer queryContainer : filterClauses) {
113+
QueryBuilder queryBuilder = registry.fromProto(queryContainer);
114+
if (queryBuilder != null) {
115+
boolQuery.filter(queryBuilder);
116+
}
117+
}
118+
119+
// Process adjust_pure_negative
120+
if (boolQueryProto.hasAdjustPureNegative()) {
121+
adjustPureNegative = boolQueryProto.getAdjustPureNegative();
122+
boolQuery.adjustPureNegative(adjustPureNegative);
123+
}
124+
125+
return boolQuery;
126+
}
127+
}

0 commit comments

Comments
 (0)