Skip to content

Commit 243cb99

Browse files
LantaoJinLantaoJin
authored
[Backport 2.19-dev] Support pushdown physical sort operator to speedup SortMergeJoin (opensearch-project#3864) (opensearch-project#3889)
* Support pushdown physical sort operator to speedup SortMergeJoin (opensearch-project#3864) * Support pushdown physical sort operator to speedup SortMergeJoin Signed-off-by: Lantao Jin <ltjin@amazon.com> * Enable sort pushdown when the type is TEXT without fields and fielddata=true Signed-off-by: Lantao Jin <ltjin@amazon.com> * Fix UT Signed-off-by: Lantao Jin <ltjin@amazon.com> * Add OpenSearchTextType.toKeywordSubField() Signed-off-by: Lantao Jin <ltjin@amazon.com> * Fix IT Signed-off-by: Lantao Jin <ltjin@amazon.com> * Fix IT Signed-off-by: Lantao Jin <ltjin@amazon.com> * Only push down sort when order is ASC for fielddata text Signed-off-by: Lantao Jin <ltjin@amazon.com> * revert CalciteNoPushdownIT Signed-off-by: Lantao Jin <ltjin@amazon.com> * revert the fielddata logic Signed-off-by: Lantao Jin <ltjin@amazon.com> * Fix IT Signed-off-by: Lantao Jin <ltjin@amazon.com> * 'gender' in test data should contain keyword subfield Signed-off-by: Lantao Jin <ltjin@amazon.com> * Fix IT Signed-off-by: Lantao Jin <ltjin@amazon.com> * revert the typo change in test Signed-off-by: Lantao Jin <ltjin@amazon.com> * fix no pushdown IT Signed-off-by: Lantao Jin <ltjin@amazon.com> --------- Signed-off-by: Lantao Jin <ltjin@amazon.com> (cherry picked from commit 71aa9ba) * Fix IT Signed-off-by: Lantao Jin <ltjin@amazon.com> * revert changes in build.gradle Signed-off-by: Lantao Jin <ltjin@amazon.com> --------- Signed-off-by: Lantao Jin <ltjin@amazon.com>
1 parent 0bedd3f commit 243cb99

File tree

26 files changed

+421
-323
lines changed

26 files changed

+421
-323
lines changed

‎benchmarks/src/jmh/java/org/opensearch/sql/expression/operator/predicate/MergeArrayAndObjectMapBenchmark.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
import java.util.Map;
1313
import org.openjdk.jmh.annotations.Benchmark;
1414
import org.opensearch.sql.opensearch.data.type.OpenSearchDataType;
15-
import org.opensearch.sql.opensearch.request.system.OpenSearchDescribeIndexRequest;
15+
import org.opensearch.sql.opensearch.util.MergeRules.MergeRuleHelper;
1616

1717
public class MergeArrayAndObjectMapBenchmark {
1818
private static final List<Map<String, OpenSearchDataType>> candidateMaps = prepareListOfMaps(120);
@@ -21,7 +21,7 @@ public class MergeArrayAndObjectMapBenchmark {
2121
public void testMerge() {
2222
Map<String, OpenSearchDataType> finalResult = new HashMap<>();
2323
for (Map<String, OpenSearchDataType> map : candidateMaps) {
24-
OpenSearchDescribeIndexRequest.mergeObjectAndArrayInsideMap(finalResult, map);
24+
MergeRuleHelper.merge(finalResult, map);
2525
}
2626
}
2727

‎integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java‎

Lines changed: 17 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -48,12 +48,23 @@ public void supportSearchSargPushDown_multiRange() throws IOException {
4848
// Only for Calcite
4949
@Test
5050
public void supportSearchSargPushDown_timeRange() throws IOException {
51+
String query =
52+
"source=opensearch-sql_test_index_bank"
53+
+ "| where birthdate >= '2016-12-08 00:00:00.000000000' "
54+
+ "and birthdate < '2018-11-09 00:00:00.000000000'";
55+
var result = explainQueryToString(query);
5156
String expected = loadExpectedPlan("explain_sarg_filter_push_time_range.json");
52-
assertJsonEqualsIgnoreId(
53-
expected,
54-
explainQueryToString(
55-
"source=opensearch-sql_test_index_bank"
56-
+ "| where birthdate >= '2016-12-08 00:00:00.000000000' "
57-
+ "and birthdate < '2018-11-09 00:00:00.000000000' "));
57+
assertJsonEqualsIgnoreId(expected, result);
58+
}
59+
60+
// Only for Calcite
61+
@Test
62+
public void supportPushDownSortMergeJoin() throws IOException {
63+
String query =
64+
"source=opensearch-sql_test_index_bank| join left=l right=r on"
65+
+ " l.account_number=r.account_number opensearch-sql_test_index_bank";
66+
var result = explainQueryToString(query);
67+
String expected = loadExpectedPlan("explain_merge_join_sort_push.json");
68+
assertJsonEqualsIgnoreId(expected, result);
5869
}
5970
}

‎integ-test/src/test/java/org/opensearch/sql/legacy/AggregationExpressionIT.java‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,7 @@ public void hasGroupKeyAvgOnIntegerShouldPass() {
7171
Index.BANK.getName()));
7272

7373
verifySchema(response, schema("gender", null, "text"), schema("AVG(age)", "avg", "double"));
74-
verifyDataRows(response, rows("m", 34.25), rows("f", 33.666666666666664d));
74+
verifyDataRows(response, rows("M", 34.25), rows("F", 33.666666666666664d));
7575
}
7676

7777
@Test
@@ -86,7 +86,7 @@ public void hasGroupKeyMaxAddMinShouldPass() {
8686
response,
8787
schema("gender", null, "text"),
8888
schema("MAX(age) + MIN(age)", "addValue", "long"));
89-
verifyDataRows(response, rows("m", 60), rows("f", 60));
89+
verifyDataRows(response, rows("M", 60), rows("F", 60));
9090
}
9191

9292
@Test
@@ -98,7 +98,7 @@ public void hasGroupKeyMaxAddLiteralShouldPass() {
9898
Index.ACCOUNT.getName()));
9999

100100
verifySchema(response, schema("gender", null, "text"), schema("MAX(age) + 1", "add", "long"));
101-
verifyDataRows(response, rows("m", 41), rows("f", 41));
101+
verifyDataRows(response, rows("M", 41), rows("F", 41));
102102
}
103103

104104
@Test
@@ -126,7 +126,7 @@ public void hasGroupKeyLogMaxAddMinShouldPass() {
126126
response,
127127
schema("gender", null, "text"),
128128
schema("Log(MAX(age) + MIN(age))", "logValue", "double"));
129-
verifyDataRows(response, rows("m", 4.0943445622221d), rows("f", 4.0943445622221d));
129+
verifyDataRows(response, rows("M", 4.0943445622221d), rows("F", 4.0943445622221d));
130130
}
131131

132132
@Test
@@ -136,7 +136,7 @@ public void AddLiteralOnGroupKeyShouldPass() {
136136
String.format(
137137
"SELECT gender, age+10, max(balance) as `max` "
138138
+ "FROM %s "
139-
+ "WHERE gender = 'm' and age < 22 "
139+
+ "WHERE gender = 'M' and age < 22 "
140140
+ "GROUP BY gender, age "
141141
+ "ORDER BY age",
142142
Index.ACCOUNT.getName()));
@@ -146,7 +146,7 @@ public void AddLiteralOnGroupKeyShouldPass() {
146146
schema("gender", null, "text"),
147147
schema("age+10", null, "long"),
148148
schema("max(balance)", "max", "long"));
149-
verifyDataRows(response, rows("m", 30, 49568), rows("m", 31, 49433));
149+
verifyDataRows(response, rows("M", 30, 49568), rows("M", 31, 49433));
150150
}
151151

152152
@Test
@@ -156,7 +156,7 @@ public void logWithAddLiteralOnGroupKeyShouldPass() {
156156
String.format(
157157
"SELECT gender, Log(age+10) as logAge, max(balance) as max "
158158
+ "FROM %s "
159-
+ "WHERE gender = 'm' and age < 22 "
159+
+ "WHERE gender = 'M' and age < 22 "
160160
+ "GROUP BY gender, age "
161161
+ "ORDER BY age",
162162
Index.ACCOUNT.getName()));
@@ -167,7 +167,7 @@ public void logWithAddLiteralOnGroupKeyShouldPass() {
167167
schema("Log(age+10)", "logAge", "double"),
168168
schema("max(balance)", "max", "long"));
169169
verifyDataRows(
170-
response, rows("m", 3.4011973816621555d, 49568), rows("m", 3.4339872044851463d, 49433));
170+
response, rows("M", 3.4011973816621555d, 49568), rows("M", 3.4339872044851463d, 49433));
171171
}
172172

173173
@Test
@@ -177,7 +177,7 @@ public void logWithAddLiteralOnGroupKeyAndMaxSubtractLiteralShouldPass() {
177177
String.format(
178178
"SELECT gender, Log(age+10) as logAge, max(balance) - 100 as max "
179179
+ "FROM %s "
180-
+ "WHERE gender = 'm' and age < 22 "
180+
+ "WHERE gender = 'M' and age < 22 "
181181
+ "GROUP BY gender, age "
182182
+ "ORDER BY age",
183183
Index.ACCOUNT.getName()));
@@ -188,7 +188,7 @@ public void logWithAddLiteralOnGroupKeyAndMaxSubtractLiteralShouldPass() {
188188
schema("Log(age+10)", "logAge", "double"),
189189
schema("max(balance) - 100", "max", "long"));
190190
verifyDataRows(
191-
response, rows("m", 3.4011973816621555d, 49468), rows("m", 3.4339872044851463d, 49333));
191+
response, rows("M", 3.4011973816621555d, 49468), rows("M", 3.4339872044851463d, 49333));
192192
}
193193

194194
/** The date is in JDBC format. */

0 commit comments

Comments
 (0)