Skip to content

Commit cd38983

Browse files
songkant-awssongkant-aws
authored
Convert like function call to wildcard query for Calcite filter pushdown (#3915)
* Convert like function call to wildcard query for Calcite filter pushdown Signed-off-by: Songkan Tang <songkant@amazon.com> * Fix V2 expression like function bug and match its behavior in Calcite Signed-off-by: Songkan Tang <songkant@amazon.com> * Fix like default escape in Calcite Signed-off-by: Songkan Tang <songkant@amazon.com> * Fix tests Signed-off-by: Songkan Tang <songkant@amazon.com> * Fix spotless check Signed-off-by: Songkan Tang <songkant@amazon.com> * Address comments Signed-off-by: Songkan Tang <songkant@amazon.com> * Fix SQL IT correctness Signed-off-by: Songkan Tang <songkant@amazon.com> * Remove test log Signed-off-by: Songkan Tang <songkant@amazon.com> * Minor improve one CalciteLikeQueryIT Signed-off-by: Songkan Tang <songkant@amazon.com> --------- Signed-off-by: Songkan Tang <songkant@amazon.com>
1 parent 739c4df commit cd38983

File tree

21 files changed

+267
-82
lines changed

21 files changed

+267
-82
lines changed

‎core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -726,7 +726,6 @@ void populate() {
726726
registerOperator(REGEXP, SqlLibraryOperators.REGEXP);
727727
registerOperator(CONCAT, SqlLibraryOperators.CONCAT_FUNCTION);
728728
registerOperator(CONCAT_WS, SqlLibraryOperators.CONCAT_WS);
729-
registerOperator(LIKE, SqlLibraryOperators.ILIKE);
730729
registerOperator(CONCAT_WS, SqlLibraryOperators.CONCAT_WS);
731730
registerOperator(REVERSE, SqlLibraryOperators.REVERSE);
732731
registerOperator(RIGHT, SqlLibraryOperators.RIGHT);
@@ -1005,6 +1004,18 @@ void populate() {
10051004
builder.makeLiteral(" "),
10061005
arg))),
10071006
PPLTypeChecker.family(SqlTypeFamily.ANY)));
1007+
register(
1008+
LIKE,
1009+
createFunctionImpWithTypeChecker(
1010+
(builder, arg1, arg2) ->
1011+
builder.makeCall(
1012+
SqlLibraryOperators.ILIKE,
1013+
arg1,
1014+
arg2,
1015+
// TODO: Figure out escaping solution. '\\' is used for JSON input but is not
1016+
// necessary for SQL function input
1017+
builder.makeLiteral("\\")),
1018+
PPLTypeChecker.family(SqlTypeFamily.STRING, SqlTypeFamily.STRING)));
10081019
}
10091020
}
10101021

‎docs/user/ppl/functions/string.rst‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,8 @@ Example::
105105
+-------------------------------+
106106

107107

108+
Limitation: The pushdown of the LIKE function to a DSL wildcard query is supported only for keyword fields.
109+
108110
LOCATE
109111
-------
110112

‎integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteLikeQueryIT.java‎

Lines changed: 3 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -6,66 +6,22 @@
66
package org.opensearch.sql.calcite.remote;
77

88
import java.io.IOException;
9-
import org.junit.Ignore;
9+
import org.junit.Assume;
1010
import org.junit.Test;
1111
import org.opensearch.sql.ppl.LikeQueryIT;
1212

13-
// TODO Like function behaviour in V2 is not correct. Remove when it was fixed in V2.
1413
public class CalciteLikeQueryIT extends LikeQueryIT {
1514
@Override
1615
public void init() throws Exception {
1716
super.init();
1817
enableCalcite();
19-
// TODO: "https://github.com/opensearch-project/sql/issues/3428"
20-
// disallowCalciteFallback();
18+
disallowCalciteFallback();
2119
}
2220

2321
@Override
2422
@Test
25-
@Ignore("https://github.com/opensearch-project/sql/issues/3428")
26-
public void test_like_with_escaped_percent() throws IOException, IOException {
27-
super.test_like_with_escaped_percent();
28-
}
29-
30-
@Override
31-
@Test
32-
@Ignore("https://github.com/opensearch-project/sql/issues/3428")
33-
public void test_like_in_where_with_escaped_underscore() throws IOException {
34-
super.test_like_in_where_with_escaped_underscore();
35-
}
36-
37-
@Override
38-
@Test
39-
@Ignore("https://github.com/opensearch-project/sql/issues/3428")
40-
public void test_like_on_text_field_with_one_word() throws IOException {
41-
super.test_like_on_text_field_with_one_word();
42-
}
43-
44-
@Override
45-
@Test
46-
@Ignore("https://github.com/opensearch-project/sql/issues/3428")
47-
public void test_like_on_text_keyword_field_with_one_word() throws IOException {
48-
super.test_like_on_text_keyword_field_with_one_word();
49-
}
50-
51-
@Override
52-
@Test
53-
@Ignore("https://github.com/opensearch-project/sql/issues/3428")
54-
public void test_like_on_text_keyword_field_with_greater_than_one_word() throws IOException {
55-
super.test_like_on_text_keyword_field_with_greater_than_one_word();
56-
}
57-
58-
@Override
59-
@Test
60-
@Ignore("https://github.com/opensearch-project/sql/issues/3428")
61-
public void test_like_on_text_field_with_greater_than_one_word() throws IOException {
62-
super.test_like_on_text_field_with_greater_than_one_word();
63-
}
64-
65-
@Override
66-
@Test
67-
@Ignore("https://github.com/opensearch-project/sql/issues/3428")
6823
public void test_convert_field_text_to_keyword() throws IOException {
24+
Assume.assumeTrue("Pushdown is not enabled, skipping this test.", isPushdownEnabled());
6925
super.test_convert_field_text_to_keyword();
7026
}
7127
}

‎integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteWhereCommandIT.java‎

Lines changed: 0 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@
55

66
package org.opensearch.sql.calcite.remote;
77

8-
import java.io.IOException;
98
import org.opensearch.sql.ppl.WhereCommandIT;
109

1110
public class CalciteWhereCommandIT extends WhereCommandIT {
@@ -16,19 +15,6 @@ public void init() throws Exception {
1615
disallowCalciteFallback();
1716
}
1817

19-
@Override
20-
public void testIsNotNullFunction() throws IOException {
21-
withFallbackEnabled(
22-
() -> {
23-
try {
24-
super.testIsNotNullFunction();
25-
} catch (IOException e) {
26-
throw new RuntimeException(e);
27-
}
28-
},
29-
"https://github.com/opensearch-project/sql/issues/3428");
30-
}
31-
3218
@Override
3319
protected String getIncompatibleTypeErrMsg() {
3420
return "In expression types are incompatible: fields type LONG, values type [INTEGER, INTEGER,"

‎integ-test/src/test/java/org/opensearch/sql/ppl/ExplainIT.java‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -434,6 +434,24 @@ public void testMultiFieldsRelevanceQueryFunctionExplain() throws IOException {
434434
+ " default_operator='or', analyzer=english)"));
435435
}
436436

437+
@Test
438+
public void testKeywordLikeFunctionExplain() throws IOException {
439+
String expected = loadExpectedPlan("explain_keyword_like_function.json");
440+
assertJsonEqualsIgnoreId(
441+
expected,
442+
explainQueryToString(
443+
"source=opensearch-sql_test_index_account | where like(firstname, '%mbe%')"));
444+
}
445+
446+
@Test
447+
public void testTextLikeFunctionExplain() throws IOException {
448+
String expected = loadExpectedPlan("explain_text_like_function.json");
449+
assertJsonEqualsIgnoreId(
450+
expected,
451+
explainQueryToString(
452+
"source=opensearch-sql_test_index_account | where like(address, '%Holmes%')"));
453+
}
454+
437455
@Ignore("The serialized string is unstable because of function properties")
438456
@Test
439457
public void testFilterScriptPushDownExplain() throws Exception {

‎integ-test/src/test/java/org/opensearch/sql/ppl/LikeQueryIT.java‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -62,17 +62,17 @@ public void test_like_in_where_with_escaped_underscore() throws IOException {
6262
@Test
6363
public void test_like_on_text_field_with_one_word() throws IOException {
6464
String query =
65-
"source=" + TEST_INDEX_WILDCARD + " | WHERE Like(TextBody, 'test*') | fields TextBody";
65+
"source=" + TEST_INDEX_WILDCARD + " | WHERE Like(TextBody, 'test%') | fields TextBody";
6666
JSONObject result = executeQuery(query);
67-
assertEquals(9, result.getInt("total"));
67+
assertEquals(8, result.getInt("total"));
6868
}
6969

7070
@Test
7171
public void test_like_on_text_keyword_field_with_one_word() throws IOException {
7272
String query =
7373
"source="
7474
+ TEST_INDEX_WILDCARD
75-
+ " | WHERE Like(TextKeywordBody, 'test*') | fields TextKeywordBody";
75+
+ " | WHERE Like(TextKeywordBody, 'test%') | fields TextKeywordBody";
7676
JSONObject result = executeQuery(query);
7777
assertEquals(8, result.getInt("total"));
7878
}
@@ -82,17 +82,17 @@ public void test_like_on_text_keyword_field_with_greater_than_one_word() throws
8282
String query =
8383
"source="
8484
+ TEST_INDEX_WILDCARD
85-
+ " | WHERE Like(TextKeywordBody, 'test wild*') | fields TextKeywordBody";
85+
+ " | WHERE Like(TextKeywordBody, 'test wild%') | fields TextKeywordBody";
8686
JSONObject result = executeQuery(query);
8787
assertEquals(7, result.getInt("total"));
8888
}
8989

9090
@Test
9191
public void test_like_on_text_field_with_greater_than_one_word() throws IOException {
9292
String query =
93-
"source=" + TEST_INDEX_WILDCARD + " | WHERE Like(TextBody, 'test wild*') | fields TextBody";
93+
"source=" + TEST_INDEX_WILDCARD + " | WHERE Like(TextBody, 'test wild%') | fields TextBody";
9494
JSONObject result = executeQuery(query);
95-
assertEquals(0, result.getInt("total"));
95+
assertEquals(7, result.getInt("total"));
9696
}
9797

9898
@Test

‎integ-test/src/test/java/org/opensearch/sql/ppl/WhereCommandIT.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,16 @@ public void testLikeFunction() throws IOException {
8888
verifyDataRows(result, rows("Amber"));
8989
}
9090

91+
@Test
92+
public void testLikeFunctionNoHit() throws IOException {
93+
JSONObject result =
94+
executeQuery(
95+
String.format(
96+
"source=%s | where like(firstname, 'Duk_') | fields lastname",
97+
TEST_INDEX_BANK_WITH_NULL_VALUES));
98+
assertEquals(0, result.getInt("total"));
99+
}
100+
91101
@Test
92102
public void testIsNullFunction() throws IOException {
93103
JSONObject result =

‎integ-test/src/test/java/org/opensearch/sql/sql/LikeQueryIT.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -117,14 +117,14 @@ public void test_like_in_where_with_escaped_underscore() throws IOException {
117117

118118
@Test
119119
public void test_like_on_text_field_with_one_word() throws IOException {
120-
String query = "SELECT * FROM " + TEST_INDEX_WILDCARD + " WHERE TextBody LIKE 'test*'";
120+
String query = "SELECT * FROM " + TEST_INDEX_WILDCARD + " WHERE TextBody LIKE 'test%'";
121121
JSONObject result = executeJdbcRequest(query);
122-
assertEquals(9, result.getInt("total"));
122+
assertEquals(8, result.getInt("total"));
123123
}
124124

125125
@Test
126126
public void test_like_on_text_keyword_field_with_one_word() throws IOException {
127-
String query = "SELECT * FROM " + TEST_INDEX_WILDCARD + " WHERE TextKeywordBody LIKE 'test*'";
127+
String query = "SELECT * FROM " + TEST_INDEX_WILDCARD + " WHERE TextKeywordBody LIKE 'test%'";
128128
JSONObject result = executeJdbcRequest(query);
129129
assertEquals(8, result.getInt("total"));
130130
}
@@ -134,7 +134,7 @@ public void test_like_on_text_keyword_field_with_greater_than_one_word() throws
134134
String query =
135135
"SELECT * FROM " + TEST_INDEX_WILDCARD + " WHERE TextKeywordBody LIKE 'test wild*'";
136136
JSONObject result = executeJdbcRequest(query);
137-
assertEquals(7, result.getInt("total"));
137+
assertEquals(0, result.getInt("total"));
138138
}
139139

140140
@Test

‎integ-test/src/test/resources/expectedOutput/calcite/explain_keyword_like_function.json‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
{
2+
"calcite": {
3+
"logical": "LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10])\n LogicalFilter(condition=[ILIKE($1, '%mbe%':VARCHAR, '\\')])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n",
4+
"physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], FILTER->ILIKE($1, '%mbe%':VARCHAR, '\\')], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"query\":{\"wildcard\":{\"firstname.keyword\":{\"wildcard\":\"*mbe*\",\"case_insensitive\":true,\"boost\":1.0}}},\"_source\":{\"includes\":[\"account_number\",\"firstname\",\"address\",\"balance\",\"gender\",\"city\",\"employer\",\"state\",\"age\",\"email\",\"lastname\"],\"excludes\":[]},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}]}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n"
5+
}
6+
}

‎integ-test/src/test/resources/expectedOutput/calcite/explain_text_like_function.json‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
{
2+
"calcite": {
3+
"logical": "LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10])\n LogicalFilter(condition=[ILIKE($2, '%Holmes%':VARCHAR, '\\')])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n",
4+
"physical": "EnumerableCalc(expr#0..10=[{inputs}], expr#11=['%Holmes%':VARCHAR], expr#12=['\\'], expr#13=[ILIKE($t2, $t11, $t12)], proj#0..10=[{exprs}], $condition=[$t13])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"account_number\",\"firstname\",\"address\",\"balance\",\"gender\",\"city\",\"employer\",\"state\",\"age\",\"email\",\"lastname\"],\"excludes\":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n"
5+
}
6+
}

0 commit comments

Comments
 (0)