diff --git a/benchmarks/src/jmh/java/org/opensearch/sql/expression/operator/predicate/MergeArrayAndObjectMapBenchmark.java b/benchmarks/src/jmh/java/org/opensearch/sql/expression/operator/predicate/MergeArrayAndObjectMapBenchmark.java index 98143618d57..62ebca24393 100644 --- a/benchmarks/src/jmh/java/org/opensearch/sql/expression/operator/predicate/MergeArrayAndObjectMapBenchmark.java +++ b/benchmarks/src/jmh/java/org/opensearch/sql/expression/operator/predicate/MergeArrayAndObjectMapBenchmark.java @@ -12,7 +12,7 @@ import java.util.Map; import org.openjdk.jmh.annotations.Benchmark; import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; -import org.opensearch.sql.opensearch.request.system.OpenSearchDescribeIndexRequest; +import org.opensearch.sql.opensearch.util.MergeRules.MergeRuleHelper; public class MergeArrayAndObjectMapBenchmark { private static final List> candidateMaps = prepareListOfMaps(120); @@ -21,7 +21,7 @@ public class MergeArrayAndObjectMapBenchmark { public void testMerge() { Map finalResult = new HashMap<>(); for (Map map : candidateMaps) { - OpenSearchDescribeIndexRequest.mergeObjectAndArrayInsideMap(finalResult, map); + MergeRuleHelper.merge(finalResult, map); } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java index 5fab2fade3f..29ab9ec9f87 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java @@ -48,12 +48,23 @@ public void supportSearchSargPushDown_multiRange() throws IOException { // Only for Calcite @Test public void supportSearchSargPushDown_timeRange() throws IOException { + String query = + "source=opensearch-sql_test_index_bank" + + "| where birthdate >= '2016-12-08 00:00:00.000000000' " + + "and birthdate < '2018-11-09 00:00:00.000000000'"; + var result = explainQueryToString(query); String expected = loadExpectedPlan("explain_sarg_filter_push_time_range.json"); - assertJsonEqualsIgnoreId( - expected, - explainQueryToString( - "source=opensearch-sql_test_index_bank" - + "| where birthdate >= '2016-12-08 00:00:00.000000000' " - + "and birthdate < '2018-11-09 00:00:00.000000000' ")); + assertJsonEqualsIgnoreId(expected, result); + } + + // Only for Calcite + @Test + public void supportPushDownSortMergeJoin() throws IOException { + String query = + "source=opensearch-sql_test_index_bank| join left=l right=r on" + + " l.account_number=r.account_number opensearch-sql_test_index_bank"; + var result = explainQueryToString(query); + String expected = loadExpectedPlan("explain_merge_join_sort_push.json"); + assertJsonEqualsIgnoreId(expected, result); } } diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/AggregationExpressionIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/AggregationExpressionIT.java index 37398220ff3..9f6a08c76ae 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/AggregationExpressionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/AggregationExpressionIT.java @@ -71,7 +71,7 @@ public void hasGroupKeyAvgOnIntegerShouldPass() { Index.BANK.getName())); verifySchema(response, schema("gender", null, "text"), schema("AVG(age)", "avg", "double")); - verifyDataRows(response, rows("m", 34.25), rows("f", 33.666666666666664d)); + verifyDataRows(response, rows("M", 34.25), rows("F", 33.666666666666664d)); } @Test @@ -86,7 +86,7 @@ public void hasGroupKeyMaxAddMinShouldPass() { response, schema("gender", null, "text"), schema("MAX(age) + MIN(age)", "addValue", "long")); - verifyDataRows(response, rows("m", 60), rows("f", 60)); + verifyDataRows(response, rows("M", 60), rows("F", 60)); } @Test @@ -98,7 +98,7 @@ public void hasGroupKeyMaxAddLiteralShouldPass() { Index.ACCOUNT.getName())); verifySchema(response, schema("gender", null, "text"), schema("MAX(age) + 1", "add", "long")); - verifyDataRows(response, rows("m", 41), rows("f", 41)); + verifyDataRows(response, rows("M", 41), rows("F", 41)); } @Test @@ -126,7 +126,7 @@ public void hasGroupKeyLogMaxAddMinShouldPass() { response, schema("gender", null, "text"), schema("Log(MAX(age) + MIN(age))", "logValue", "double")); - verifyDataRows(response, rows("m", 4.0943445622221d), rows("f", 4.0943445622221d)); + verifyDataRows(response, rows("M", 4.0943445622221d), rows("F", 4.0943445622221d)); } @Test @@ -136,7 +136,7 @@ public void AddLiteralOnGroupKeyShouldPass() { String.format( "SELECT gender, age+10, max(balance) as `max` " + "FROM %s " - + "WHERE gender = 'm' and age < 22 " + + "WHERE gender = 'M' and age < 22 " + "GROUP BY gender, age " + "ORDER BY age", Index.ACCOUNT.getName())); @@ -146,7 +146,7 @@ public void AddLiteralOnGroupKeyShouldPass() { schema("gender", null, "text"), schema("age+10", null, "long"), schema("max(balance)", "max", "long")); - verifyDataRows(response, rows("m", 30, 49568), rows("m", 31, 49433)); + verifyDataRows(response, rows("M", 30, 49568), rows("M", 31, 49433)); } @Test @@ -156,7 +156,7 @@ public void logWithAddLiteralOnGroupKeyShouldPass() { String.format( "SELECT gender, Log(age+10) as logAge, max(balance) as max " + "FROM %s " - + "WHERE gender = 'm' and age < 22 " + + "WHERE gender = 'M' and age < 22 " + "GROUP BY gender, age " + "ORDER BY age", Index.ACCOUNT.getName())); @@ -167,7 +167,7 @@ public void logWithAddLiteralOnGroupKeyShouldPass() { schema("Log(age+10)", "logAge", "double"), schema("max(balance)", "max", "long")); verifyDataRows( - response, rows("m", 3.4011973816621555d, 49568), rows("m", 3.4339872044851463d, 49433)); + response, rows("M", 3.4011973816621555d, 49568), rows("M", 3.4339872044851463d, 49433)); } @Test @@ -177,7 +177,7 @@ public void logWithAddLiteralOnGroupKeyAndMaxSubtractLiteralShouldPass() { String.format( "SELECT gender, Log(age+10) as logAge, max(balance) - 100 as max " + "FROM %s " - + "WHERE gender = 'm' and age < 22 " + + "WHERE gender = 'M' and age < 22 " + "GROUP BY gender, age " + "ORDER BY age", Index.ACCOUNT.getName())); @@ -188,7 +188,7 @@ public void logWithAddLiteralOnGroupKeyAndMaxSubtractLiteralShouldPass() { schema("Log(age+10)", "logAge", "double"), schema("max(balance) - 100", "max", "long")); verifyDataRows( - response, rows("m", 3.4011973816621555d, 49468), rows("m", 3.4339872044851463d, 49333)); + response, rows("M", 3.4011973816621555d, 49468), rows("M", 3.4339872044851463d, 49333)); } /** The date is in JDBC format. */ diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/AggregationIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/AggregationIT.java index 38fefe0fe14..803379a50ea 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/AggregationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/AggregationIT.java @@ -230,19 +230,19 @@ public void groupByUsingTableNamePrefixTest() throws Exception { private void assertResultForGroupByTest(JSONObject result) { Assert.assertThat(getTotalHits(result), equalTo(1000)); - JSONObject gender = getAggregation(result, "gender"); + JSONObject gender = getAggregation(result, "gender.keyword"); Assert.assertThat(gender.getJSONArray("buckets").length(), equalTo(2)); - final boolean isMaleFirst = gender.optQuery("/buckets/0/key").equals("m"); + final boolean isMaleFirst = gender.optQuery("/buckets/0/key").equals("M"); final int maleBucketId = isMaleFirst ? 0 : 1; final int femaleBucketId = isMaleFirst ? 1 : 0; final String maleBucketPrefix = String.format(Locale.ROOT, "/buckets/%d", maleBucketId); final String femaleBucketPrefix = String.format(Locale.ROOT, "/buckets/%d", femaleBucketId); - Assert.assertThat(gender.query(maleBucketPrefix + "/key"), equalTo("m")); + Assert.assertThat(gender.query(maleBucketPrefix + "/key"), equalTo("M")); Assert.assertThat(gender.query(maleBucketPrefix + "/COUNT(*)/value"), equalTo(507)); - Assert.assertThat(gender.query(femaleBucketPrefix + "/key"), equalTo("f")); + Assert.assertThat(gender.query(femaleBucketPrefix + "/key"), equalTo("F")); Assert.assertThat(gender.query(femaleBucketPrefix + "/COUNT(*)/value"), equalTo(493)); } @@ -281,19 +281,19 @@ public void groupByHavingUsingTableNamePrefixTest() throws Exception { private void assertResultForGroupByHavingTest(JSONObject result) { Assert.assertThat(getTotalHits(result), equalTo(1000)); - JSONObject gender = getAggregation(result, "gender"); + JSONObject gender = getAggregation(result, "gender.keyword"); Assert.assertThat(gender.getJSONArray("buckets").length(), equalTo(2)); - final boolean isMaleFirst = gender.optQuery("/buckets/0/key").equals("m"); + final boolean isMaleFirst = gender.optQuery("/buckets/0/key").equals("M"); final int maleBucketId = isMaleFirst ? 0 : 1; final int femaleBucketId = isMaleFirst ? 1 : 0; final String maleBucketPrefix = String.format(Locale.ROOT, "/buckets/%d", maleBucketId); final String femaleBucketPrefix = String.format(Locale.ROOT, "/buckets/%d", femaleBucketId); - Assert.assertThat(gender.query(maleBucketPrefix + "/key"), equalTo("m")); + Assert.assertThat(gender.query(maleBucketPrefix + "/key"), equalTo("M")); Assert.assertThat(gender.query(maleBucketPrefix + "/count_0/value"), equalTo(507)); - Assert.assertThat(gender.query(femaleBucketPrefix + "/key"), equalTo("f")); + Assert.assertThat(gender.query(femaleBucketPrefix + "/key"), equalTo("F")); Assert.assertThat(gender.query(femaleBucketPrefix + "/count_0/value"), equalTo(493)); } @@ -309,19 +309,19 @@ public void groupBySubqueryTest() throws Exception { + "GROUP BY gender", TEST_INDEX_ACCOUNT, TEST_INDEX_ACCOUNT)); Assert.assertThat(getTotalHits(result), equalTo(1000)); - JSONObject gender = getAggregation(result, "gender"); + JSONObject gender = getAggregation(result, "gender.keyword"); Assert.assertThat(gender.getJSONArray("buckets").length(), equalTo(2)); - final boolean isMaleFirst = gender.optQuery("/buckets/0/key").equals("m"); + final boolean isMaleFirst = gender.optQuery("/buckets/0/key").equals("M"); final int maleBucketId = isMaleFirst ? 0 : 1; final int femaleBucketId = isMaleFirst ? 1 : 0; final String maleBucketPrefix = String.format(Locale.ROOT, "/buckets/%d", maleBucketId); final String femaleBucketPrefix = String.format(Locale.ROOT, "/buckets/%d", femaleBucketId); - Assert.assertThat(gender.query(maleBucketPrefix + "/key"), equalTo("m")); + Assert.assertThat(gender.query(maleBucketPrefix + "/key"), equalTo("M")); Assert.assertThat(gender.query(maleBucketPrefix + "/COUNT(*)/value"), equalTo(507)); - Assert.assertThat(gender.query(femaleBucketPrefix + "/key"), equalTo("f")); + Assert.assertThat(gender.query(femaleBucketPrefix + "/key"), equalTo("F")); Assert.assertThat(gender.query(femaleBucketPrefix + "/COUNT(*)/value"), equalTo(493)); } @@ -335,19 +335,19 @@ public void postFilterTest() throws Exception { + "{\\\"gender\\\":\\\"m\\\"}}) */ COUNT(*) FROM %s GROUP BY gender", TEST_INDEX_ACCOUNT)); Assert.assertThat(getTotalHits(result), equalTo(507)); - JSONObject gender = getAggregation(result, "gender"); + JSONObject gender = getAggregation(result, "gender.keyword"); Assert.assertThat(gender.getJSONArray("buckets").length(), equalTo(2)); - final boolean isMaleFirst = gender.optQuery("/buckets/0/key").equals("m"); + final boolean isMaleFirst = gender.optQuery("/buckets/0/key").equals("M"); final int maleBucketId = isMaleFirst ? 0 : 1; final int femaleBucketId = isMaleFirst ? 1 : 0; final String maleBucketPrefix = String.format(Locale.ROOT, "/buckets/%d", maleBucketId); final String femaleBucketPrefix = String.format(Locale.ROOT, "/buckets/%d", femaleBucketId); - Assert.assertThat(gender.query(maleBucketPrefix + "/key"), equalTo("m")); + Assert.assertThat(gender.query(maleBucketPrefix + "/key"), equalTo("M")); Assert.assertThat(gender.query(maleBucketPrefix + "/COUNT(*)/value"), equalTo(507)); - Assert.assertThat(gender.query(femaleBucketPrefix + "/key"), equalTo("f")); + Assert.assertThat(gender.query(femaleBucketPrefix + "/key"), equalTo("F")); Assert.assertThat(gender.query(femaleBucketPrefix + "/COUNT(*)/value"), equalTo(493)); } @@ -361,10 +361,10 @@ public void multipleGroupByTest() throws Exception { + " terms('field'='age','size'=200,'alias'='age')", TEST_INDEX_ACCOUNT)); Assert.assertThat(getTotalHits(result), equalTo(1000)); - JSONObject gender = getAggregation(result, "gender"); + JSONObject gender = getAggregation(result, "gender.keyword"); Assert.assertThat(gender.getJSONArray("buckets").length(), equalTo(2)); - final boolean isMaleFirst = gender.optQuery("/buckets/0/key").equals("m"); + final boolean isMaleFirst = gender.optQuery("/buckets/0/key").equals("M"); final int maleBucketId = isMaleFirst ? 0 : 1; final int femaleBucketId = isMaleFirst ? 1 : 0; @@ -403,7 +403,7 @@ public void multipleGroupBysWithSize() throws Exception { + " terms('alias'='ageAgg','field'='age','size'=3)", TEST_INDEX_ACCOUNT)); Assert.assertThat(getTotalHits(result), equalTo(1000)); - JSONObject gender = getAggregation(result, "gender"); + JSONObject gender = getAggregation(result, "gender.keyword"); Assert.assertThat(gender.getJSONArray("buckets").length(), equalTo(2)); final JSONArray mAgeBuckets = (JSONArray) (gender.optQuery("/buckets/0/ageAgg/buckets")); @@ -543,7 +543,7 @@ public void orderByGroupFieldWithAlias() throws IOException { TEST_INDEX_ACCOUNT)); verifySchema(response, schema("gender", "g", "text"), schema("COUNT(*)", "count", "integer")); - verifyDataRowsInOrder(response, rows("f", 493), rows("m", 507)); + verifyDataRowsInOrder(response, rows("F", 493), rows("M", 507)); // ORDER BY field alias response = @@ -553,7 +553,7 @@ public void orderByGroupFieldWithAlias() throws IOException { TEST_INDEX_ACCOUNT)); verifySchema(response, schema("gender", "g", "text"), schema("COUNT(*)", "count", "integer")); - verifyDataRowsInOrder(response, rows("f", 493), rows("m", 507)); + verifyDataRowsInOrder(response, rows("F", 493), rows("M", 507)); } @Test @@ -648,17 +648,17 @@ public void topHitTest() throws IOException { String.format( "select topHits('size'=3,age='desc') from %s group by gender", TEST_INDEX_ACCOUNT); JSONObject result = executeQuery(query); - JSONObject gender = getAggregation(result, "gender"); + JSONObject gender = getAggregation(result, "gender.keyword"); Assert.assertThat(gender.getJSONArray("buckets").length(), equalTo(2)); - final boolean isMaleFirst = gender.optQuery("/buckets/0/key").equals("m"); + final boolean isMaleFirst = gender.optQuery("/buckets/0/key").equals("M"); final int maleBucketId = isMaleFirst ? 0 : 1; final int femaleBucketId = isMaleFirst ? 1 : 0; final String maleBucketPrefix = String.format(Locale.ROOT, "/buckets/%d", maleBucketId); final String femaleBucketPrefix = String.format(Locale.ROOT, "/buckets/%d", femaleBucketId); - Assert.assertThat(gender.query(maleBucketPrefix + "/key"), equalTo("m")); + Assert.assertThat(gender.query(maleBucketPrefix + "/key"), equalTo("M")); Assert.assertThat( gender.query(maleBucketPrefix + "/topHits(size=3,age=desc)/hits/total/value"), equalTo(507)); @@ -669,7 +669,7 @@ public void topHitTest() throws IOException { ((JSONArray) gender.query(maleBucketPrefix + "/topHits(size=3,age=desc)/hits/hits")) .length(), equalTo(3)); - Assert.assertThat(gender.query(femaleBucketPrefix + "/key"), equalTo("f")); + Assert.assertThat(gender.query(femaleBucketPrefix + "/key"), equalTo("F")); Assert.assertThat( gender.query(femaleBucketPrefix + "/topHits(size=3,age=desc)/hits/total/value"), equalTo(493)); @@ -690,17 +690,17 @@ public void topHitTest_WithInclude() throws IOException { "select topHits('size'=3,age='desc','include'=age) from %s group by gender", TEST_INDEX_ACCOUNT); JSONObject result = executeQuery(query); - JSONObject gender = getAggregation(result, "gender"); + JSONObject gender = getAggregation(result, "gender.keyword"); Assert.assertThat(gender.getJSONArray("buckets").length(), equalTo(2)); - final boolean isMaleFirst = gender.optQuery("/buckets/0/key").equals("m"); + final boolean isMaleFirst = gender.optQuery("/buckets/0/key").equals("M"); final int maleBucketId = isMaleFirst ? 0 : 1; final int femaleBucketId = isMaleFirst ? 1 : 0; final String maleBucketPrefix = String.format(Locale.ROOT, "/buckets/%d", maleBucketId); final String femaleBucketPrefix = String.format(Locale.ROOT, "/buckets/%d", femaleBucketId); - Assert.assertThat(gender.query(maleBucketPrefix + "/key"), equalTo("m")); + Assert.assertThat(gender.query(maleBucketPrefix + "/key"), equalTo("M")); Assert.assertThat( gender.query(maleBucketPrefix + "/topHits(size=3,age=desc,include=age)/hits/total/value"), equalTo(507)); @@ -714,7 +714,7 @@ public void topHitTest_WithInclude() throws IOException { .length(), equalTo(3)); - Assert.assertThat(gender.query(femaleBucketPrefix + "/key"), equalTo("f")); + Assert.assertThat(gender.query(femaleBucketPrefix + "/key"), equalTo("F")); Assert.assertThat( gender.query(femaleBucketPrefix + "/topHits(size=3,age=desc,include=age)/hits/total/value"), equalTo(493)); @@ -755,7 +755,7 @@ public void topHitTest_WithIncludeTwoFields() throws IOException { + "group by gender", TEST_INDEX_ACCOUNT); JSONObject result = executeQuery(query); - JSONObject gender = getAggregation(result, "gender"); + JSONObject gender = getAggregation(result, "gender.keyword"); Assert.assertThat(gender.getJSONArray("buckets").length(), equalTo(2)); for (int i = 0; i < 2; ++i) { @@ -786,17 +786,17 @@ public void topHitTest_WithExclude() throws IOException { "select topHits('size'=3,'exclude'='lastname',age='desc') from " + "%s group by gender", TEST_INDEX_ACCOUNT); JSONObject result = executeQuery(query); - JSONObject gender = getAggregation(result, "gender"); + JSONObject gender = getAggregation(result, "gender.keyword"); Assert.assertThat(gender.getJSONArray("buckets").length(), equalTo(2)); - final boolean isMaleFirst = gender.optQuery("/buckets/0/key").equals("m"); + final boolean isMaleFirst = gender.optQuery("/buckets/0/key").equals("M"); final int maleBucketId = isMaleFirst ? 0 : 1; final int femaleBucketId = isMaleFirst ? 1 : 0; final String maleBucketPrefix = String.format(Locale.ROOT, "/buckets/%d", maleBucketId); final String femaleBucketPrefix = String.format(Locale.ROOT, "/buckets/%d", femaleBucketId); - Assert.assertThat(gender.query(maleBucketPrefix + "/key"), equalTo("m")); + Assert.assertThat(gender.query(maleBucketPrefix + "/key"), equalTo("M")); Assert.assertThat( gender.query( maleBucketPrefix + "/topHits(size=3,exclude=lastname,age=desc)/hits/total/value"), @@ -812,7 +812,7 @@ public void topHitTest_WithExclude() throws IOException { .length(), equalTo(3)); - Assert.assertThat(gender.query(femaleBucketPrefix + "/key"), equalTo("f")); + Assert.assertThat(gender.query(femaleBucketPrefix + "/key"), equalTo("F")); Assert.assertThat( gender.query( femaleBucketPrefix + "/topHits(size=3,exclude=lastname,age=desc)/hits/total/value"), @@ -1026,8 +1026,8 @@ public void topHitTest_WithExclude() throws IOException { // } // // Assert.assertEquals(2, buckets.keySet().size()); - // Assert.assertEquals(expectedAges, buckets.get("m")); - // Assert.assertEquals(expectedAges, buckets.get("f")); + // Assert.assertEquals(expectedAges, buckets.get("M")); + // Assert.assertEquals(expectedAges, buckets.get("F")); // // Terms state = result.get("state.keyword"); // for(Terms.Bucket stateBucket : state.getBuckets()) { @@ -1053,8 +1053,8 @@ public void topHitTest_WithExclude() throws IOException { // Terms gender = result.get("gender"); // for(Terms.Bucket genderBucket : gender.getBuckets()) { // String genderKey = genderBucket.getKey().toString(); - // Assert.assertTrue("Gender should be m or f", genderKey.equals("m") || - // genderKey.equals("f")); + // Assert.assertTrue("Gender should be m or f", genderKey.equals("M") || + // genderKey.equals("F")); // } // // Assert.assertEquals(2, gender.getBuckets().size()); @@ -1125,14 +1125,14 @@ public void topHitTest_WithExclude() throws IOException { // @Test // public void groupByTestWithFilter() throws Exception { // Aggregations result = query(String.format("SELECT COUNT(*) FROM %s GROUP BY - // filter(gender='m'),gender", TEST_INDEX_ACCOUNT)); - // InternalFilter filter = result.get("filter(gender = 'm')@FILTER"); + // filter(gender='M'),gender", TEST_INDEX_ACCOUNT)); + // InternalFilter filter = result.get("filter(gender = 'M')@FILTER"); // Terms gender = filter.getAggregations().get("gender"); // // for(Terms.Bucket bucket : gender.getBuckets()) { // String key = bucket.getKey().toString(); // long count = ((ValueCount) bucket.getAggregations().get("COUNT(*)")).getValue(); - // if(key.equalsIgnoreCase("m")) { + // if(key.equalsIgnoreCase("M")) { // Assert.assertEquals(507, count); // } // else { diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/PrettyFormatResponseIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/PrettyFormatResponseIT.java index 3247975a673..bc347ace444 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/PrettyFormatResponseIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/PrettyFormatResponseIT.java @@ -431,7 +431,7 @@ public void aggregationFunctionInHaving() throws IOException { JSONArray dataRows = getDataRows(response); assertEquals(1, dataRows.length()); - assertEquals("m", dataRows.getJSONArray(0).getString(0)); + assertEquals("M", dataRows.getJSONArray(0).getString(0)); } /** diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/QueryIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/QueryIT.java index f94b80686e0..b65e393e8b3 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/QueryIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/QueryIT.java @@ -569,19 +569,19 @@ public void doubleNotTest() throws IOException { Locale.ROOT, "SELECT * FROM %s WHERE gender LIKE 'm' AND NOT gender LIKE 'f'", TEST_INDEX_ACCOUNT)); - // Assert there are results and they all have gender 'm' + // Assert there are results and they all have gender 'M' Assert.assertNotEquals(0, getTotalHits(response4)); JSONArray hits = getHits(response4); for (int i = 0; i < hits.length(); i++) { JSONObject hit = hits.getJSONObject(i); - Assert.assertEquals("m", getSource(hit).getString("gender").toLowerCase()); + Assert.assertEquals("M", getSource(hit).getString("gender")); } JSONObject response5 = executeQuery( String.format( Locale.ROOT, - "SELECT * FROM %s WHERE NOT (gender = 'm' OR gender = 'f')", + "SELECT * FROM %s WHERE NOT (gender = 'M' OR gender = 'F')", TEST_INDEX_ACCOUNT)); Assert.assertEquals(0, getTotalHits(response5)); } @@ -942,8 +942,8 @@ public void notMissFilterSearch() throws IOException { @Test public void complexConditionQuery() throws IOException { String errorMessage = - "Result does not exist to the condition (gender='m' AND (age> 25 OR account_number>5)) OR" - + " (gender='f' AND (age>30 OR account_number < 8)"; + "Result does not exist to the condition (gender='M' AND (age> 25 OR account_number>5)) OR" + + " (gender='F' AND (age>30 OR account_number < 8)"; JSONObject response = executeQuery( @@ -951,8 +951,8 @@ public void complexConditionQuery() throws IOException { Locale.ROOT, "SELECT * " + "FROM %s " - + "WHERE (gender='m' AND (age> 25 OR account_number>5)) " - + "OR (gender='f' AND (age>30 OR account_number < 8))", + + "WHERE (gender='M' AND (age> 25 OR account_number>5)) " + + "OR (gender='F' AND (age>30 OR account_number < 8))", TEST_INDEX_ACCOUNT)); JSONArray hits = getHits(response); @@ -960,14 +960,14 @@ public void complexConditionQuery() throws IOException { JSONObject hit = hits.getJSONObject(i); JSONObject source = getSource(hit); - String gender = source.getString("gender").toLowerCase(); + String gender = source.getString("gender"); int age = source.getInt("age"); int accountNumber = source.getInt("account_number"); Assert.assertTrue( errorMessage, - (gender.equals("m") && (age > 25 || accountNumber > 5)) - || (gender.equals("f") && (age > 30 || accountNumber < 8))); + (gender.equals("M") && (age > 25 || accountNumber > 5)) + || (gender.equals("F") && (age > 30 || accountNumber < 8))); } } @@ -975,8 +975,8 @@ public void complexConditionQuery() throws IOException { public void complexNotConditionQuery() throws IOException { String errorMessage = "Result does not exist to the condition " - + "NOT (gender='m' AND NOT (age > 25 OR account_number > 5)) " - + "OR (NOT gender='f' AND NOT (age > 30 OR account_number < 8))"; + + "NOT (gender='M' AND NOT (age > 25 OR account_number > 5)) " + + "OR (NOT gender='F' AND NOT (age > 30 OR account_number < 8))"; JSONObject response = executeQuery( @@ -984,8 +984,8 @@ public void complexNotConditionQuery() throws IOException { Locale.ROOT, "SELECT * " + "FROM %s " - + "WHERE NOT (gender='m' AND NOT (age > 25 OR account_number > 5)) " - + "OR (NOT gender='f' AND NOT (age > 30 OR account_number < 8))", + + "WHERE NOT (gender='M' AND NOT (age > 25 OR account_number > 5)) " + + "OR (NOT gender='F' AND NOT (age > 30 OR account_number < 8))", TEST_INDEX_ACCOUNT)); JSONArray hits = getHits(response); @@ -994,14 +994,14 @@ public void complexNotConditionQuery() throws IOException { JSONObject hit = hits.getJSONObject(i); JSONObject source = getSource(hit); - String gender = source.getString("gender").toLowerCase(); + String gender = source.getString("gender"); int age = source.getInt("age"); int accountNumber = source.getInt("account_number"); Assert.assertTrue( errorMessage, - !(gender.equals("m") && !(age > 25 || accountNumber > 5)) - || (!gender.equals("f") && !(age > 30 || accountNumber < 8))); + !(gender.equals("M") && !(age > 25 || accountNumber > 5)) + || (!gender.equals("F") && !(age > 30 || accountNumber < 8))); } } diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/SubqueryIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/SubqueryIT.java index 16614bfd2ae..c701b8167fe 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/SubqueryIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/SubqueryIT.java @@ -289,19 +289,19 @@ public void selectFromSubqueryWithCountAndGroupByShouldPass() throws Exception { TEST_INDEX_ACCOUNT)); assertThat(getTotalHits(result), equalTo(1000)); - JSONObject gender = (JSONObject) result.query("/aggregations/gender"); + JSONObject gender = (JSONObject) result.query("/aggregations/gender.keyword"); assertThat(gender.getJSONArray("buckets").length(), equalTo(2)); - boolean isMaleFirst = gender.optQuery("/buckets/0/key").equals("m"); + boolean isMaleFirst = gender.optQuery("/buckets/0/key").equals("M"); int maleBucketId = isMaleFirst ? 0 : 1; int femaleBucketId = isMaleFirst ? 1 : 0; String maleBucketPrefix = String.format(Locale.ROOT, "/buckets/%d", maleBucketId); String femaleBucketPrefix = String.format(Locale.ROOT, "/buckets/%d", femaleBucketId); - assertThat(gender.query(maleBucketPrefix + "/key"), equalTo("m")); + assertThat(gender.query(maleBucketPrefix + "/key"), equalTo("M")); assertThat(gender.query(maleBucketPrefix + "/count/value"), equalTo(507)); - assertThat(gender.query(femaleBucketPrefix + "/key"), equalTo("f")); + assertThat(gender.query(femaleBucketPrefix + "/key"), equalTo("F")); assertThat(gender.query(femaleBucketPrefix + "/count/value"), equalTo(493)); } @@ -333,7 +333,7 @@ public void selectFromSubqueryWithCountAndGroupByAndHavingShouldPass() throws Ex + " GROUP BY gender " + " HAVING T2 > 500) t", TEST_INDEX_ACCOUNT)); - assertThat(result.query("/aggregations/g/buckets/0/c/value"), equalTo(507)); + assertThat(result.query("/aggregations/gender.keyword/buckets/0/c/value"), equalTo(507)); } @Test diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/ExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/ExplainIT.java index 1015a12a0a9..272fe3162e4 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/ExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/ExplainIT.java @@ -134,7 +134,7 @@ public void testMultiSortPushDownExplain() throws IOException { explainQueryToString( "source=opensearch-sql_test_index_account " + "| sort account_number, firstname, address, balance " - + "| sort - balance, - gender, address " + + "| sort - balance, - gender, account_number " + "| fields account_number, firstname, address, balance, gender")); } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/RareCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/RareCommandIT.java index 8383fb994b3..8b7355993ca 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/RareCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/RareCommandIT.java @@ -59,7 +59,7 @@ public void testRareWithGroup() throws IOException { rows("F", "OK", 7), rows("F", "KS", 7), rows("F", "CO", 7), - rows("F", "NV", 8), + isPushdownEnabled() ? rows("F", "AR", 8) : rows("F", "NV", 8), rows("M", "NE", 5), rows("M", "RI", 5), rows("M", "NV", 5), diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/StatsCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/StatsCommandIT.java index e62722ed2ae..48e2f506e61 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/StatsCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/StatsCommandIT.java @@ -11,7 +11,6 @@ import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; -import static org.opensearch.sql.util.MatcherUtils.verifyDataRowsInOrder; import static org.opensearch.sql.util.MatcherUtils.verifySchema; import static org.opensearch.sql.util.MatcherUtils.verifySchemaInOrder; @@ -358,27 +357,15 @@ public void testStatsBySpanAndMultipleFields() throws IOException { schema("span(age,10)", null, "int"), schema("gender", null, "string"), schema("state", null, "string")); - if (isCalciteEnabled()) { - verifyDataRows( - response, - rows(1, 20, "F", "VA"), - rows(1, 30, "F", "IN"), - rows(1, 30, "F", "PA"), - rows(1, 30, "M", "IL"), - rows(1, 30, "M", "MD"), - rows(1, 30, "M", "TN"), - rows(1, 30, "M", "WA")); - } else { - verifyDataRowsInOrder( - response, - rows(1, 20, "f", "VA"), - rows(1, 30, "f", "IN"), - rows(1, 30, "f", "PA"), - rows(1, 30, "m", "IL"), - rows(1, 30, "m", "MD"), - rows(1, 30, "m", "TN"), - rows(1, 30, "m", "WA")); - } + verifyDataRows( + response, + rows(1, 20, "F", "VA"), + rows(1, 30, "F", "IN"), + rows(1, 30, "F", "PA"), + rows(1, 30, "M", "IL"), + rows(1, 30, "M", "MD"), + rows(1, 30, "M", "TN"), + rows(1, 30, "M", "WA")); } @Test @@ -395,27 +382,15 @@ public void testStatsByMultipleFieldsAndSpan() throws IOException { schema("span(age,10)", null, "int"), schema("gender", null, "string"), schema("state", null, "string")); - if (isCalciteEnabled()) { - verifyDataRows( - response, - rows(1, 20, "F", "VA"), - rows(1, 30, "F", "IN"), - rows(1, 30, "F", "PA"), - rows(1, 30, "M", "IL"), - rows(1, 30, "M", "MD"), - rows(1, 30, "M", "TN"), - rows(1, 30, "M", "WA")); - } else { - verifyDataRowsInOrder( - response, - rows(1, 20, "f", "VA"), - rows(1, 30, "f", "IN"), - rows(1, 30, "f", "PA"), - rows(1, 30, "m", "IL"), - rows(1, 30, "m", "MD"), - rows(1, 30, "m", "TN"), - rows(1, 30, "m", "WA")); - } + verifyDataRows( + response, + rows(1, 20, "F", "VA"), + rows(1, 30, "F", "IN"), + rows(1, 30, "F", "PA"), + rows(1, 30, "M", "IL"), + rows(1, 30, "M", "MD"), + rows(1, 30, "M", "TN"), + rows(1, 30, "M", "WA")); } @Test diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_merge_join_sort_push.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_merge_join_sort_push.json new file mode 100644 index 00000000000..4c8026fd1f8 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_merge_join_sort_push.json @@ -0,0 +1,6 @@ +{ + "calcite": { + "logical": "LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], r.account_number=[$13], r.firstname=[$14], r.address=[$15], r.birthdate=[$16], r.gender=[$17], r.city=[$18], r.lastname=[$19], r.balance=[$20], r.employer=[$21], r.state=[$22], r.age=[$23], r.email=[$24], r.male=[$25])\n LogicalJoin(condition=[=($0, $13)], joinType=[inner])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", + "physical": "EnumerableMergeJoin(condition=[=($0, $13)], joinType=[inner])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], SORT->[{\n \"account_number\" : {\n \"order\" : \"asc\",\n \"missing\" : \"_last\"\n }\n}]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"account_number\",\"firstname\",\"address\",\"birthdate\",\"gender\",\"city\",\"lastname\",\"balance\",\"employer\",\"state\",\"age\",\"email\",\"male\"],\"excludes\":[]},\"sort\":[{\"account_number\":{\"order\":\"asc\",\"missing\":\"_last\"}}]}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], SORT->[{\n \"account_number\" : {\n \"order\" : \"asc\",\n \"missing\" : \"_last\"\n }\n}]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"account_number\",\"firstname\",\"address\",\"birthdate\",\"gender\",\"city\",\"lastname\",\"balance\",\"employer\",\"state\",\"age\",\"email\",\"male\"],\"excludes\":[]},\"sort\":[{\"account_number\":{\"order\":\"asc\",\"missing\":\"_last\"}}]}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + } +} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_multi_sort_push.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_multi_sort_push.json index 98084cf146a..a4ddbe9dfb0 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_multi_sort_push.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_multi_sort_push.json @@ -1,6 +1,6 @@ { "calcite": { - "logical": "LogicalSort(sort0=[$3], sort1=[$4], sort2=[$2], dir0=[DESC-nulls-last], dir1=[DESC-nulls-last], dir2=[ASC-nulls-first])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4])\n LogicalSort(sort0=[$3], sort1=[$4], sort2=[$2], dir0=[DESC-nulls-last], dir1=[DESC-nulls-last], dir2=[ASC-nulls-first])\n LogicalSort(sort0=[$0], sort1=[$1], sort2=[$2], sort3=[$3], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first], dir2=[ASC-nulls-first], dir3=[ASC-nulls-first])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender], SORT->[{\n \"balance\" : {\n \"order\" : \"desc\",\n \"missing\" : \"_last\"\n }\n}, {\n \"gender\" : {\n \"order\" : \"desc\",\n \"missing\" : \"_last\"\n }\n}, {\n \"address\" : {\n \"order\" : \"asc\",\n \"missing\" : \"_first\"\n }\n}]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"account_number\",\"firstname\",\"address\",\"balance\",\"gender\"],\"excludes\":[]},\"sort\":[{\"balance\":{\"order\":\"desc\",\"missing\":\"_last\"}},{\"gender\":{\"order\":\"desc\",\"missing\":\"_last\"}},{\"address\":{\"order\":\"asc\",\"missing\":\"_first\"}}]}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + "logical": "LogicalSort(sort0=[$3], sort1=[$4], sort2=[$0], dir0=[DESC-nulls-last], dir1=[DESC-nulls-last], dir2=[ASC-nulls-first])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4])\n LogicalSort(sort0=[$3], sort1=[$4], sort2=[$0], dir0=[DESC-nulls-last], dir1=[DESC-nulls-last], dir2=[ASC-nulls-first])\n LogicalSort(sort0=[$0], sort1=[$1], sort2=[$2], sort3=[$3], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first], dir2=[ASC-nulls-first], dir3=[ASC-nulls-first])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", + "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender], SORT->[{\n \"balance\" : {\n \"order\" : \"desc\",\n \"missing\" : \"_last\"\n }\n}, {\n \"gender.keyword\" : {\n \"order\" : \"desc\",\n \"missing\" : \"_last\"\n }\n}, {\n \"account_number\" : {\n \"order\" : \"asc\",\n \"missing\" : \"_first\"\n }\n}]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"account_number\",\"firstname\",\"address\",\"balance\",\"gender\"],\"excludes\":[]},\"sort\":[{\"balance\":{\"order\":\"desc\",\"missing\":\"_last\"}},{\"gender.keyword\":{\"order\":\"desc\",\"missing\":\"_last\"}},{\"account_number\":{\"order\":\"asc\",\"missing\":\"_first\"}}]}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_merge_join_sort_push.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_merge_join_sort_push.json new file mode 100644 index 00000000000..3daeece7f1a --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_merge_join_sort_push.json @@ -0,0 +1,6 @@ +{ + "calcite": { + "logical": "LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], r.account_number=[$13], r.firstname=[$14], r.address=[$15], r.birthdate=[$16], r.gender=[$17], r.city=[$18], r.lastname=[$19], r.balance=[$20], r.employer=[$21], r.state=[$22], r.age=[$23], r.email=[$24], r.male=[$25])\n LogicalJoin(condition=[=($0, $13)], joinType=[inner])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", + "physical": "EnumerableMergeJoin(condition=[=($0, $13)], joinType=[inner])\n EnumerableSort(sort0=[$0], dir0=[ASC])\n EnumerableCalc(expr#0..18=[{inputs}], proj#0..12=[{exprs}])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n EnumerableSort(sort0=[$0], dir0=[ASC])\n EnumerableCalc(expr#0..18=[{inputs}], proj#0..12=[{exprs}])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n" + } +} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_multi_sort_push.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_multi_sort_push.json index 2914ec95b6a..b1bb221d982 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_multi_sort_push.json +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_multi_sort_push.json @@ -1,6 +1,6 @@ { "calcite": { - "logical": "LogicalSort(sort0=[$3], sort1=[$4], sort2=[$2], dir0=[DESC-nulls-last], dir1=[DESC-nulls-last], dir2=[ASC-nulls-first])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4])\n LogicalSort(sort0=[$3], sort1=[$4], sort2=[$2], dir0=[DESC-nulls-last], dir1=[DESC-nulls-last], dir2=[ASC-nulls-first])\n LogicalSort(sort0=[$0], sort1=[$1], sort2=[$2], sort3=[$3], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first], dir2=[ASC-nulls-first], dir3=[ASC-nulls-first])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableSort(sort0=[$3], sort1=[$4], sort2=[$2], dir0=[DESC-nulls-last], dir1=[DESC-nulls-last], dir2=[ASC-nulls-first])\n EnumerableCalc(expr#0..16=[{inputs}], proj#0..4=[{exprs}])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" + "logical": "LogicalSort(sort0=[$3], sort1=[$4], sort2=[$0], dir0=[DESC-nulls-last], dir1=[DESC-nulls-last], dir2=[ASC-nulls-first])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4])\n LogicalSort(sort0=[$3], sort1=[$4], sort2=[$0], dir0=[DESC-nulls-last], dir1=[DESC-nulls-last], dir2=[ASC-nulls-first])\n LogicalSort(sort0=[$0], sort1=[$1], sort2=[$2], sort3=[$3], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first], dir2=[ASC-nulls-first], dir3=[ASC-nulls-first])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", + "physical": "EnumerableSort(sort0=[$3], sort1=[$4], sort2=[$0], dir0=[DESC-nulls-last], dir1=[DESC-nulls-last], dir2=[ASC-nulls-first])\n EnumerableCalc(expr#0..16=[{inputs}], proj#0..4=[{exprs}])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_multi_sort_push.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_multi_sort_push.json index b327d41bb66..147184ed5e4 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_multi_sort_push.json +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_multi_sort_push.json @@ -4,14 +4,12 @@ "description": { "fields": "[account_number, firstname, address, balance, gender]" }, - "children": [ - { - "name": "OpenSearchIndexScan", - "description": { - "request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"account_number\",\"firstname\",\"address\",\"balance\",\"gender\"],\"excludes\":[]},\"sort\":[{\"account_number\":{\"order\":\"asc\",\"missing\":\"_first\"}},{\"firstname.keyword\":{\"order\":\"asc\",\"missing\":\"_first\"}},{\"address\":{\"order\":\"asc\",\"missing\":\"_first\"}},{\"balance\":{\"order\":\"asc\",\"missing\":\"_first\"}},{\"balance\":{\"order\":\"desc\",\"missing\":\"_last\"}},{\"gender\":{\"order\":\"desc\",\"missing\":\"_last\"}},{\"address\":{\"order\":\"asc\",\"missing\":\"_first\"}}]}, needClean=true, searchDone=false, pitId=s9y3QQEhb3BlbnNlYXJjaC1zcWxfdGVzdF9pbmRleF9hY2NvdW50Fndla1VpMi1kVHh5Qi1lYnhPQnlSbXcAFkU4Qm9UVURIUWI2a3pjNkhmQkxvc2cAAAAAAAAAAAEWdTZkUG5oYkZTWWl2UnpFVXlhNmhXZwEWd2VrVWkyLWRUeHlCLWVieE9CeVJtdwAA, cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" - }, - "children": [] - } - ] + "children": [{ + "name": "OpenSearchIndexScan", + "description": { + "request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"account_number\",\"firstname\",\"address\",\"balance\",\"gender\"],\"excludes\":[]},\"sort\":[{\"account_number\":{\"order\":\"asc\",\"missing\":\"_first\"}},{\"firstname.keyword\":{\"order\":\"asc\",\"missing\":\"_first\"}},{\"address\":{\"order\":\"asc\",\"missing\":\"_first\"}},{\"balance\":{\"order\":\"asc\",\"missing\":\"_first\"}},{\"balance\":{\"order\":\"desc\",\"missing\":\"_last\"}},{\"gender.keyword\":{\"order\":\"desc\",\"missing\":\"_last\"}},{\"account_number\":{\"order\":\"asc\",\"missing\":\"_first\"}}]}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" + }, + "children": [] + }] } -} +} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_patterns_simple_pattern_agg_push.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_patterns_simple_pattern_agg_push.json index c6f90418f74..2e5bf031582 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_patterns_simple_pattern_agg_push.json +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_patterns_simple_pattern_agg_push.json @@ -1,16 +1,15 @@ { "root": { - "name":"ProjectOperator", - "description":{ - "fields":"[pattern_count, sample_logs, patterns_field]" + "name": "ProjectOperator", + "description": { + "fields": "[pattern_count, sample_logs, patterns_field]" }, - "children":[ - { - "name":"OpenSearchIndexScan", - "description":{ - "request":"OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account, sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"patterns_field\":{\"terms\":{\"script\":{\"source\":\"rO0ABXNyADZvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5wYXJzZS5QYXR0ZXJuc0V4cHJlc3Npb26h4+bazqpHBgIAAloAEHVzZUN1c3RvbVBhdHRlcm5MAAdwYXR0ZXJudAAZTGphdmEvdXRpbC9yZWdleC9QYXR0ZXJuO3hyADNvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5wYXJzZS5QYXJzZUV4cHJlc3Npb27CZfCltUMmOQIABEwACmlkZW50aWZpZXJ0ACpMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vRXhwcmVzc2lvbjtMAA1pZGVudGlmaWVyU3RydAASTGphdmEvbGFuZy9TdHJpbmc7TAAHcGF0dGVybnEAfgADTAALc291cmNlRmllbGRxAH4AA3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3QAEExqYXZhL3V0aWwvTGlzdDtMAAxmdW5jdGlvbk5hbWV0ADVMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25OYW1lO3hwc3IAPXNoYWRlZC5jb20uZ29vZ2xlLmNvbW1vbi5jb2xsZWN0LkltbXV0YWJsZUxpc3QkU2VyaWFsaXplZEZvcm0AAAAAAAAAAAIAAVsACGVsZW1lbnRzdAATW0xqYXZhL2xhbmcvT2JqZWN0O3hwdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAA3NyADFvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5SZWZlcmVuY2VFeHByZXNzaW9uq0TvXBIHhdYCAARMAARhdHRycQB+AARMAAVwYXRoc3EAfgAGTAAHcmF3UGF0aHEAfgAETAAEdHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hwdAAFZW1haWxzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWFxAH4ACnhwdXIAE1tMamF2YS5sYW5nLlN0cmluZzut0lbn6R17RwIAAHhwAAAAAXEAfgARcQB+ABFzcgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hUZXh0VHlwZbmXyjSCpmeCAgABTAAGZmllbGRzdAAPTGphdmEvdXRpbC9NYXA7eHIAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGXCY7zKAvoFNQIAA0wADGV4cHJDb3JlVHlwZXQAK0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJDb3JlVHlwZTtMAAttYXBwaW5nVHlwZXQASExvcmcvb3BlbnNlYXJjaC9zcWwvb3BlbnNlYXJjaC9kYXRhL3R5cGUvT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlO0wACnByb3BlcnRpZXNxAH4AF3hwfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAHVU5LTk9XTn5yAEZvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlAAAAAAAAAAASAAB4cQB+AB10AARUZXh0c3IAPHNoYWRlZC5jb20uZ29vZ2xlLmNvbW1vbi5jb2xsZWN0LkltbXV0YWJsZU1hcCRTZXJpYWxpemVkRm9ybQAAAAAAAAAAAgACTAAEa2V5c3QAEkxqYXZhL2xhbmcvT2JqZWN0O0wABnZhbHVlc3EAfgAkeHB1cQB+AAwAAAAAdXEAfgAMAAAAAHNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAACdAAHa2V5d29yZHNxAH4AGH5xAH4AHHQABlNUUklOR35xAH4AIHQAB0tleXdvcmRxAH4AJXhzcgAvb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uTGl0ZXJhbEV4cHJlc3Npb25FQi3wjMeCJAIAAUwACWV4cHJWYWx1ZXQAKUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7eHBzcgAtb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuRXhwclN0cmluZ1ZhbHVltW44cOB3TIUCAAFMAAV2YWx1ZXEAfgAEeHIAL29yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkFic3RyYWN0RXhwclZhbHVlyWu1dgYURIoCAAB4cHQAAHNxAH4AMHNxAH4AM3QADnBhdHRlcm5zX2ZpZWxkc3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAR4cHQACHBhdHRlcm5zcQB+ADdxAH4AOXEAfgAycQB+ABAAcA==\",\"lang\":\"opensearch_query_expression\"},\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"pattern_count\":{\"value_count\":{\"field\":\"_index\"}},\"sample_logs\":{\"top_hits\":{\"from\":0,\"size\":10,\"version\":false,\"seq_no_primary_term\":false,\"explain\":false,\"_source\":{\"includes\":[\"email\"],\"excludes\":[]}}}}}}}, needClean=true, searchDone=false, pitId=null, cursorKeepAlive=null, searchAfter=null, searchResponse=null)"}, - "children":[] - } - ] + "children": [{ + "name": "OpenSearchIndexScan", + "description": { + "request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account, sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"patterns_field\":{\"terms\":{\"script\":{\"source\":\"rO0ABXNyADZvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5wYXJzZS5QYXR0ZXJuc0V4cHJlc3Npb26h4+bazqpHBgIAAloAEHVzZUN1c3RvbVBhdHRlcm5MAAdwYXR0ZXJudAAZTGphdmEvdXRpbC9yZWdleC9QYXR0ZXJuO3hyADNvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5wYXJzZS5QYXJzZUV4cHJlc3Npb27CZfCltUMmOQIABEwACmlkZW50aWZpZXJ0ACpMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vRXhwcmVzc2lvbjtMAA1pZGVudGlmaWVyU3RydAASTGphdmEvbGFuZy9TdHJpbmc7TAAHcGF0dGVybnEAfgADTAALc291cmNlRmllbGRxAH4AA3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3QAEExqYXZhL3V0aWwvTGlzdDtMAAxmdW5jdGlvbk5hbWV0ADVMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25OYW1lO3hwc3IAPXNoYWRlZC5jb20uZ29vZ2xlLmNvbW1vbi5jb2xsZWN0LkltbXV0YWJsZUxpc3QkU2VyaWFsaXplZEZvcm0AAAAAAAAAAAIAAVsACGVsZW1lbnRzdAATW0xqYXZhL2xhbmcvT2JqZWN0O3hwdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAA3NyADFvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5SZWZlcmVuY2VFeHByZXNzaW9uq0TvXBIHhdYCAARMAARhdHRycQB+AARMAAVwYXRoc3EAfgAGTAAHcmF3UGF0aHEAfgAETAAEdHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hwdAAFZW1haWxzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWFxAH4ACnhwdXIAE1tMamF2YS5sYW5nLlN0cmluZzut0lbn6R17RwIAAHhwAAAAAXEAfgARcQB+ABFzcgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hUZXh0VHlwZa2Do5ME4zFEAgABTAAGZmllbGRzdAAPTGphdmEvdXRpbC9NYXA7eHIAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGXCY7zKAvoFNQIAA0wADGV4cHJDb3JlVHlwZXQAK0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJDb3JlVHlwZTtMAAttYXBwaW5nVHlwZXQASExvcmcvb3BlbnNlYXJjaC9zcWwvb3BlbnNlYXJjaC9kYXRhL3R5cGUvT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlO0wACnByb3BlcnRpZXNxAH4AF3hwfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAHVU5LTk9XTn5yAEZvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlAAAAAAAAAAASAAB4cQB+AB10AARUZXh0c3IAPHNoYWRlZC5jb20uZ29vZ2xlLmNvbW1vbi5jb2xsZWN0LkltbXV0YWJsZU1hcCRTZXJpYWxpemVkRm9ybQAAAAAAAAAAAgACTAAEa2V5c3QAEkxqYXZhL2xhbmcvT2JqZWN0O0wABnZhbHVlc3EAfgAkeHB1cQB+AAwAAAAAdXEAfgAMAAAAAHNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAACdAAHa2V5d29yZHNxAH4AGH5xAH4AHHQABlNUUklOR35xAH4AIHQAB0tleXdvcmRxAH4AJXhzcgAvb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uTGl0ZXJhbEV4cHJlc3Npb25FQi3wjMeCJAIAAUwACWV4cHJWYWx1ZXQAKUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7eHBzcgAtb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuRXhwclN0cmluZ1ZhbHVltW44cOB3TIUCAAFMAAV2YWx1ZXEAfgAEeHIAL29yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkFic3RyYWN0RXhwclZhbHVlyWu1dgYURIoCAAB4cHQAAHNxAH4AMHNxAH4AM3QADnBhdHRlcm5zX2ZpZWxkc3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAR4cHQACHBhdHRlcm5zcQB+ADdxAH4AOXEAfgAycQB+ABAAcA==\",\"lang\":\"opensearch_query_expression\"},\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"pattern_count\":{\"value_count\":{\"field\":\"_index\"}},\"sample_logs\":{\"top_hits\":{\"from\":0,\"size\":10,\"version\":false,\"seq_no_primary_term\":false,\"explain\":false,\"_source\":{\"includes\":[\"email\"],\"excludes\":[]}}}}}}}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + }, + "children": [] + }] } } diff --git a/integ-test/src/test/resources/indexDefinitions/account_index_mapping.json b/integ-test/src/test/resources/indexDefinitions/account_index_mapping.json index a2c5ba8577d..a6bd51d89a0 100644 --- a/integ-test/src/test/resources/indexDefinitions/account_index_mapping.json +++ b/integ-test/src/test/resources/indexDefinitions/account_index_mapping.json @@ -3,7 +3,13 @@ "properties": { "gender": { "type": "text", - "fielddata": true + "fielddata": true, + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } }, "address": { "type": "text", diff --git a/integ-test/src/test/resources/indexDefinitions/bank_index_mapping.json b/integ-test/src/test/resources/indexDefinitions/bank_index_mapping.json index a0609a04d27..4285d59d455 100644 --- a/integ-test/src/test/resources/indexDefinitions/bank_index_mapping.json +++ b/integ-test/src/test/resources/indexDefinitions/bank_index_mapping.json @@ -30,7 +30,13 @@ }, "gender": { "type": "text", - "fielddata": true + "fielddata": true, + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } }, "lastname": { "type": "keyword" diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchTextType.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchTextType.java index 169eb3f49d4..bcc8d0f1491 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchTextType.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchTextType.java @@ -67,9 +67,12 @@ protected OpenSearchDataType cloneEmpty() { } /** - * Text field doesn't have doc value (exception thrown even when you call "get") Limitation: - * assume inner field name is always "keyword". + * Text field doesn't have doc value (exception thrown even when you call "get")
+ * Limitation: assume inner field name is always "keyword". + * + * @deprecated Use {@code toKeywordSubField(fieldName, fieldType)} */ + @Deprecated public static String convertTextToKeyword(String fieldName, ExprType fieldType) { if (fieldType instanceof OpenSearchTextType && ((OpenSearchTextType) fieldType).getFields().size() > 0) { @@ -77,4 +80,28 @@ public static String convertTextToKeyword(String fieldName, ExprType fieldType) } return fieldName; } + + /** + * Get the keyword subfield of the text field. Alternative of {@code + * convertTextToKeyword(fieldName, fieldType)} in v3. + * + * @return the text type keyword subfield if exists, or null. If the type of filed is not text, + * return field name. + */ + public static String toKeywordSubField(String fieldName, ExprType exprType) { + if (exprType != null && exprType.getOriginalExprType() instanceof OpenSearchTextType) { + OpenSearchTextType textType = (OpenSearchTextType) exprType.getOriginalExprType(); + // For OpenSearch Alias type which maps to the field of text type, + // we have to use its original path + String path = exprType.getOriginalPath().orElse(fieldName); + // Find the first subfield with type keyword, return null if non-exist. + return textType.getFields().entrySet().stream() + .filter(e -> e.getValue().getMappingType() == OpenSearchDataType.MappingType.Keyword) + .findFirst() + .map(e -> path + "." + e.getKey()) + .orElse(null); + } else { + return fieldName; + } + } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/OpenSearchIndexScanRule.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/OpenSearchIndexScanRule.java index f32c2fd6cc8..010f7ee47ca 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/OpenSearchIndexScanRule.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/OpenSearchIndexScanRule.java @@ -8,24 +8,25 @@ import java.util.HashSet; import java.util.Set; import org.apache.calcite.plan.RelOptTable; +import org.apache.calcite.rel.core.Sort; import org.apache.calcite.rel.logical.LogicalProject; import org.apache.calcite.rel.logical.LogicalSort; import org.apache.calcite.rex.RexNode; import org.opensearch.sql.opensearch.storage.OpenSearchIndex; -import org.opensearch.sql.opensearch.storage.scan.CalciteLogicalIndexScan; +import org.opensearch.sql.opensearch.storage.scan.AbstractCalciteIndexScan; public interface OpenSearchIndexScanRule { /** * CalciteOpenSearchIndexScan doesn't allow push-down anymore (except Sort under some strict * condition) after Aggregate push-down. */ - static boolean noAggregatePushed(CalciteLogicalIndexScan scan) { + static boolean noAggregatePushed(AbstractCalciteIndexScan scan) { if (scan.getPushDownContext().isAggregatePushed()) return false; final RelOptTable table = scan.getTable(); return table.unwrap(OpenSearchIndex.class) != null; } - static boolean isLimitPushed(CalciteLogicalIndexScan scan) { + static boolean isLimitPushed(AbstractCalciteIndexScan scan) { return scan.getPushDownContext().isLimitPushed(); } @@ -53,7 +54,7 @@ static boolean isLogicalSortLimit(LogicalSort sort) { return sort.fetch != null; } - static boolean sortByFieldsOnly(LogicalSort sort) { + static boolean sortByFieldsOnly(Sort sort) { return !sort.getCollation().getFieldCollations().isEmpty() && sort.fetch == null; } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/OpenSearchSortIndexScanRule.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/OpenSearchSortIndexScanRule.java index 52241f39e2f..b95725c3e16 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/OpenSearchSortIndexScanRule.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/OpenSearchSortIndexScanRule.java @@ -8,9 +8,9 @@ import java.util.function.Predicate; import org.apache.calcite.plan.RelOptRuleCall; import org.apache.calcite.plan.RelRule; -import org.apache.calcite.rel.logical.LogicalSort; +import org.apache.calcite.rel.core.Sort; import org.immutables.value.Value; -import org.opensearch.sql.opensearch.storage.scan.CalciteLogicalIndexScan; +import org.opensearch.sql.opensearch.storage.scan.AbstractCalciteIndexScan; @Value.Enclosing public class OpenSearchSortIndexScanRule extends RelRule { @@ -21,11 +21,11 @@ protected OpenSearchSortIndexScanRule(Config config) { @Override public void onMatch(RelOptRuleCall call) { - final LogicalSort sort = call.rel(0); - final CalciteLogicalIndexScan scan = call.rel(1); + final Sort sort = call.rel(0); + final AbstractCalciteIndexScan scan = call.rel(1); var collations = sort.collation.getFieldCollations(); - CalciteLogicalIndexScan newScan = scan.pushDownSort(collations); + AbstractCalciteIndexScan newScan = scan.pushDownSort(collations); if (newScan != null) { call.transformTo(newScan); } @@ -39,11 +39,11 @@ public interface Config extends RelRule.Config { .build() .withOperandSupplier( b0 -> - b0.operand(LogicalSort.class) + b0.operand(Sort.class) .predicate(OpenSearchIndexScanRule::sortByFieldsOnly) .oneInput( b1 -> - b1.operand(CalciteLogicalIndexScan.class) + b1.operand(AbstractCalciteIndexScan.class) // Skip the rule if a limit has already been pushed down // because pushing down a sort after a limit will be treated // as sort-then-limit by OpenSearch DSL. diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java index 0e075300261..803260263e2 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java @@ -80,7 +80,6 @@ import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; -import org.opensearch.sql.opensearch.data.type.OpenSearchDataType.MappingType; import org.opensearch.sql.opensearch.data.type.OpenSearchTextType; import org.opensearch.sql.opensearch.storage.script.filter.lucene.relevance.MatchBoolPrefixQuery; import org.opensearch.sql.opensearch.storage.script.filter.lucene.relevance.MatchPhrasePrefixQuery; @@ -1306,23 +1305,6 @@ boolean isTextType() { return type != null && type.getOriginalExprType() instanceof OpenSearchTextType; } - String toKeywordSubField() { - ExprType type = this.type.getOriginalExprType(); - if (type instanceof OpenSearchTextType) { - OpenSearchTextType textType = (OpenSearchTextType) type; - // For OpenSearch Alias type which maps to the field of text type, - // we have to use its original path - String path = this.type.getOriginalPath().orElse(this.name); - // Find the first subfield with type keyword, return null if non-exist. - return textType.getFields().entrySet().stream() - .filter(e -> e.getValue().getMappingType() == MappingType.Keyword) - .findFirst() - .map(e -> path + "." + e.getKey()) - .orElse(null); - } - return null; - } - boolean isMetaField() { return OpenSearchConstants.METADATAFIELD_TYPE_MAP.containsKey(getRootName()); } @@ -1332,10 +1314,7 @@ String getReference() { } String getReferenceForTermQuery() { - if (isTextType()) { - return toKeywordSubField(); - } - return getRootName(); + return OpenSearchTextType.toKeywordSubField(getRootName(), this.type); } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/AbstractCalciteIndexScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/AbstractCalciteIndexScan.java index 206fd0a42fe..e8c50a8f7d7 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/AbstractCalciteIndexScan.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/AbstractCalciteIndexScan.java @@ -9,26 +9,43 @@ import static org.opensearch.sql.common.setting.Settings.Key.CALCITE_PUSHDOWN_ROWCOUNT_ESTIMATION_FACTOR; import java.util.ArrayDeque; +import java.util.ArrayList; import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; import lombok.Getter; +import org.apache.calcite.adapter.enumerable.EnumerableMergeJoin; +import org.apache.calcite.adapter.enumerable.EnumerableSort; import org.apache.calcite.plan.RelOptCluster; import org.apache.calcite.plan.RelOptTable; import org.apache.calcite.plan.RelTraitSet; +import org.apache.calcite.rel.RelCollations; +import org.apache.calcite.rel.RelFieldCollation; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.RelWriter; import org.apache.calcite.rel.core.TableScan; import org.apache.calcite.rel.hint.RelHint; +import org.apache.calcite.rel.logical.LogicalAggregate; import org.apache.calcite.rel.metadata.RelMdUtil; import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexNode; import org.apache.calcite.util.NumberUtil; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.search.sort.ScoreSortBuilder; +import org.opensearch.search.sort.SortBuilder; +import org.opensearch.search.sort.SortBuilders; +import org.opensearch.search.sort.SortOrder; +import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.opensearch.data.type.OpenSearchTextType; import org.opensearch.sql.opensearch.request.OpenSearchRequestBuilder; import org.opensearch.sql.opensearch.storage.OpenSearchIndex; /** An abstract relational operator representing a scan of an OpenSearchIndex type. */ @Getter public abstract class AbstractCalciteIndexScan extends TableScan { + private static final Logger LOG = LogManager.getLogger(AbstractCalciteIndexScan.class); public final OpenSearchIndex osIndex; // The schema of this scan operator, it's initialized with the row type of the table, but may be // changed by push down operations. @@ -137,6 +154,146 @@ public boolean isAggregatePushed() { } } + protected abstract AbstractCalciteIndexScan buildScan( + RelOptCluster cluster, + RelTraitSet traitSet, + List hints, + RelOptTable table, + OpenSearchIndex osIndex, + RelDataType schema, + PushDownContext pushDownContext); + + private List getCollationNames(List collations) { + return collations.stream() + .map(collation -> getRowType().getFieldNames().get(collation.getFieldIndex())) + .collect(Collectors.toList()); + } + + /** + * Check if the sort by collations contains any aggregators that are pushed down. E.g. In `stats + * avg(age) as avg_age by state | sort avg_age`, the sort clause has `avg_age` which is an + * aggregator. The function will return true in this case. + * + * @param collations List of collation names to check against aggregators. + * @return True if any collation name matches an aggregator output, false otherwise. + */ + private boolean hasAggregatorInSortBy(List collations) { + Stream aggregates = + pushDownContext.stream() + .filter(action -> action.getType() == PushDownType.AGGREGATION) + .map(action -> ((LogicalAggregate) action.getDigest())); + return aggregates + .map(aggregate -> isAnyCollationNameInAggregateOutput(aggregate, collations)) + .reduce(false, Boolean::logicalOr); + } + + private static boolean isAnyCollationNameInAggregateOutput( + LogicalAggregate aggregate, List collations) { + List fieldNames = aggregate.getRowType().getFieldNames(); + // The output fields of the aggregate are in the format of + // [...grouping fields, ...aggregator fields], so we set an offset to skip + // the grouping fields. + int groupOffset = aggregate.getGroupSet().cardinality(); + List fieldsWithoutGrouping = fieldNames.subList(groupOffset, fieldNames.size()); + return collations.stream() + .map(fieldsWithoutGrouping::contains) + .reduce(false, Boolean::logicalOr); + } + + /** + * Create a new {@link PushDownContext} without the collation action. + * + * @param pushDownContext The original push-down context. + * @return A new push-down context without the collation action. + */ + protected PushDownContext cloneWithoutSort(PushDownContext pushDownContext) { + PushDownContext newContext = new PushDownContext(); + for (PushDownAction action : pushDownContext) { + if (action.getType() != PushDownType.SORT) { + newContext.add(action); + } + } + return newContext; + } + + /** + * The sort pushdown is not only applied in logical plan side, but also should be applied in + * physical plan side. Because we could push down the {@link EnumerableSort} of {@link + * EnumerableMergeJoin} to OpenSearch. + */ + public AbstractCalciteIndexScan pushDownSort(List collations) { + try { + List collationNames = getCollationNames(collations); + if (getPushDownContext().isAggregatePushed() && hasAggregatorInSortBy(collationNames)) { + // If aggregation is pushed down, we cannot push down sorts where its by fields contain + // aggregators. + return null; + } + + // Propagate the sort to the new scan + RelTraitSet traitsWithCollations = getTraitSet().plus(RelCollations.of(collations)); + AbstractCalciteIndexScan newScan = + buildScan( + getCluster(), + traitsWithCollations, + hints, + table, + osIndex, + getRowType(), + // Existing collations are overridden (discarded) by the new collations, + cloneWithoutSort(pushDownContext)); + + List> builders = new ArrayList<>(); + for (RelFieldCollation collation : collations) { + int index = collation.getFieldIndex(); + String fieldName = this.getRowType().getFieldNames().get(index); + RelFieldCollation.Direction direction = collation.getDirection(); + RelFieldCollation.NullDirection nullDirection = collation.nullDirection; + // Default sort order is ASCENDING + SortOrder order = + RelFieldCollation.Direction.DESCENDING.equals(direction) + ? SortOrder.DESC + : SortOrder.ASC; + // TODO: support script sort and distance sort + SortBuilder sortBuilder; + if (ScoreSortBuilder.NAME.equals(fieldName)) { + sortBuilder = SortBuilders.scoreSort(); + } else { + String missing; + switch (nullDirection) { + case FIRST: + missing = "_first"; + break; + case LAST: + missing = "_last"; + break; + default: + missing = null; + break; + } + // Keyword field is optimized for sorting in OpenSearch + ExprType fieldType = osIndex.getFieldTypes().get(fieldName); + String field = OpenSearchTextType.toKeywordSubField(fieldName, fieldType); + sortBuilder = SortBuilders.fieldSort(field).missing(missing); + } + builders.add(sortBuilder.order(order)); + } + newScan.pushDownContext.add( + new PushDownAction( + PushDownType.SORT, + builders.toString(), + requestBuilder -> requestBuilder.pushDownSort(builders))); + return newScan; + } catch (Exception e) { + if (LOG.isDebugEnabled()) { + LOG.debug("Cannot pushdown the sort {}", getCollationNames(collations), e); + } else { + LOG.info("Cannot pushdown the sort {}, ", getCollationNames(collations)); + } + } + return null; + } + protected enum PushDownType { FILTER, PROJECT, diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteEnumerableIndexScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteEnumerableIndexScan.java index 503c98f7b96..1d9161d6153 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteEnumerableIndexScan.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteEnumerableIndexScan.java @@ -58,6 +58,19 @@ public CalciteEnumerableIndexScan( super(cluster, traitSet, hints, table, osIndex, schema, pushDownContext); } + @Override + protected AbstractCalciteIndexScan buildScan( + RelOptCluster cluster, + RelTraitSet traitSet, + List hints, + RelOptTable table, + OpenSearchIndex osIndex, + RelDataType schema, + PushDownContext pushDownContext) { + return new CalciteEnumerableIndexScan( + cluster, traitSet, hints, table, osIndex, schema, pushDownContext); + } + @Override public void register(RelOptPlanner planner) { for (RelOptRule rule : OpenSearchRules.OPEN_SEARCH_OPT_RULES) { diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteLogicalIndexScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteLogicalIndexScan.java index 224fa13b783..c3aec83a365 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteLogicalIndexScan.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteLogicalIndexScan.java @@ -6,12 +6,10 @@ package org.opensearch.sql.opensearch.storage.scan; import com.google.common.collect.ImmutableList; -import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; -import java.util.stream.Stream; import lombok.Getter; import org.apache.calcite.plan.Convention; import org.apache.calcite.plan.RelOptCluster; @@ -35,15 +33,10 @@ import org.apache.logging.log4j.Logger; import org.opensearch.index.query.QueryBuilder; import org.opensearch.search.aggregations.AggregationBuilder; -import org.opensearch.search.sort.ScoreSortBuilder; -import org.opensearch.search.sort.SortBuilder; -import org.opensearch.search.sort.SortBuilders; -import org.opensearch.search.sort.SortOrder; import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory; import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; -import org.opensearch.sql.opensearch.data.type.OpenSearchTextType; import org.opensearch.sql.opensearch.planner.physical.EnumerableIndexScanRule; import org.opensearch.sql.opensearch.planner.physical.OpenSearchIndexRules; import org.opensearch.sql.opensearch.request.AggregateAnalyzer; @@ -79,6 +72,19 @@ protected CalciteLogicalIndexScan( super(cluster, traitSet, hints, table, osIndex, schema, pushDownContext); } + @Override + protected AbstractCalciteIndexScan buildScan( + RelOptCluster cluster, + RelTraitSet traitSet, + List hints, + RelOptTable table, + OpenSearchIndex osIndex, + RelDataType schema, + PushDownContext pushDownContext) { + return new CalciteLogicalIndexScan( + cluster, traitSet, hints, table, osIndex, schema, pushDownContext); + } + public CalciteLogicalIndexScan copyWithNewSchema(RelDataType schema) { // Do shallow copy for requestBuilder, thus requestBuilder among different plans produced in the // optimization process won't affect each other. @@ -245,131 +251,4 @@ public CalciteLogicalIndexScan pushDownLimit(Integer limit, Integer offset) { } return null; } - - public CalciteLogicalIndexScan pushDownSort(List collations) { - try { - List collationNames = getCollationNames(collations); - if (getPushDownContext().isAggregatePushed() && hasAggregatorInSortBy(collationNames)) { - // If aggregation is pushed down, we cannot push down sorts where its by fields contain - // aggregators. - return null; - } - - // Propagate the sort to the new scan - RelTraitSet traitsWithCollations = getTraitSet().plus(RelCollations.of(collations)); - CalciteLogicalIndexScan newScan = - new CalciteLogicalIndexScan( - getCluster(), - traitsWithCollations, - hints, - table, - osIndex, - getRowType(), - // Existing collations are overridden (discarded) by the new collations, - cloneWithoutSort(pushDownContext)); - - List> builders = new ArrayList<>(); - for (RelFieldCollation collation : collations) { - int index = collation.getFieldIndex(); - String fieldName = this.getRowType().getFieldNames().get(index); - RelFieldCollation.Direction direction = collation.getDirection(); - RelFieldCollation.NullDirection nullDirection = collation.nullDirection; - // Default sort order is ASCENDING - SortOrder order = - RelFieldCollation.Direction.DESCENDING.equals(direction) - ? SortOrder.DESC - : SortOrder.ASC; - // TODO: support script sort and distance sort - SortBuilder sortBuilder; - if (ScoreSortBuilder.NAME.equals(fieldName)) { - sortBuilder = SortBuilders.scoreSort(); - } else { - String missing; - switch (nullDirection) { - case FIRST: - missing = "_first"; - break; - case LAST: - missing = "_last"; - break; - default: - missing = null; - break; - } - // Keyword field is optimized for sorting in OpenSearch - String fieldNameKeyword = - OpenSearchTextType.convertTextToKeyword( - fieldName, osIndex.getFieldTypes().get(fieldName)); - sortBuilder = SortBuilders.fieldSort(fieldNameKeyword).missing(missing); - } - builders.add(sortBuilder.order(order)); - } - newScan.pushDownContext.add( - new PushDownAction( - PushDownType.SORT, - builders.toString(), - requestBuilder -> requestBuilder.pushDownSort(builders))); - return newScan; - } catch (Exception e) { - if (LOG.isDebugEnabled()) { - LOG.debug("Cannot pushdown the sort {}", getCollationNames(collations), e); - } else { - LOG.info("Cannot pushdown the sort {}, ", getCollationNames(collations)); - } - } - return null; - } - - private List getCollationNames(List collations) { - return collations.stream() - .map(collation -> getRowType().getFieldNames().get(collation.getFieldIndex())) - .collect(Collectors.toList()); - } - - /** - * Check if the sort by collations contains any aggregators that are pushed down. E.g. In `stats - * avg(age) as avg_age by state | sort avg_age`, the sort clause has `avg_age` which is an - * aggregator. The function will return true in this case. - * - * @param collations List of collation names to check against aggregators. - * @return True if any collation name matches an aggregator output, false otherwise. - */ - private boolean hasAggregatorInSortBy(List collations) { - Stream aggregates = - pushDownContext.stream() - .filter(action -> action.getType() == PushDownType.AGGREGATION) - .map(action -> ((LogicalAggregate) action.getDigest())); - return aggregates - .map(aggregate -> isAnyCollationNameInAggregateOutput(aggregate, collations)) - .reduce(false, Boolean::logicalOr); - } - - private static boolean isAnyCollationNameInAggregateOutput( - LogicalAggregate aggregate, List collations) { - List fieldNames = aggregate.getRowType().getFieldNames(); - // The output fields of the aggregate are in the format of - // [...grouping fields, ...aggregator fields], so we set an offset to skip - // the grouping fields. - int groupOffset = aggregate.getGroupSet().cardinality(); - List fieldsWithoutGrouping = fieldNames.subList(groupOffset, fieldNames.size()); - return collations.stream() - .map(fieldsWithoutGrouping::contains) - .reduce(false, Boolean::logicalOr); - } - - /** - * Create a new {@link PushDownContext} without the collation action. - * - * @param pushDownContext The original push-down context. - * @return A new push-down context without the collation action. - */ - private static PushDownContext cloneWithoutSort(PushDownContext pushDownContext) { - PushDownContext newContext = new PushDownContext(); - for (PushDownAction action : pushDownContext) { - if (action.getType() != PushDownType.SORT) { - newContext.add(action); - } - } - return newContext; - } } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprTextValueTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprTextValueTest.java index 9b7e032c57a..c9b02fdce59 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprTextValueTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprTextValueTest.java @@ -62,6 +62,32 @@ void non_text_types_arent_converted() { "field", OpenSearchDataType.of(OpenSearchDataType.MappingType.Integer))), () -> assertEquals("field", OpenSearchTextType.convertTextToKeyword("field", STRING)), () -> assertEquals("field", OpenSearchTextType.convertTextToKeyword("field", INTEGER))); + assertAll( + () -> + assertEquals( + "field", + OpenSearchTextType.toKeywordSubField("field", OpenSearchDataType.of(INTEGER))), + () -> + assertEquals( + "field", + OpenSearchTextType.toKeywordSubField("field", OpenSearchDataType.of(STRING))), + () -> + assertEquals( + "field", + OpenSearchTextType.toKeywordSubField( + "field", OpenSearchDataType.of(OpenSearchDataType.MappingType.GeoPoint))), + () -> + assertEquals( + "field", + OpenSearchTextType.toKeywordSubField( + "field", OpenSearchDataType.of(OpenSearchDataType.MappingType.Keyword))), + () -> + assertEquals( + "field", + OpenSearchTextType.toKeywordSubField( + "field", OpenSearchDataType.of(OpenSearchDataType.MappingType.Integer))), + () -> assertEquals("field", OpenSearchTextType.toKeywordSubField("field", STRING)), + () -> assertEquals("field", OpenSearchTextType.toKeywordSubField("field", INTEGER))); } @Test @@ -77,6 +103,9 @@ void non_text_types_with_nested_objects_arent_converted() { assertAll( () -> assertEquals("field", OpenSearchTextType.convertTextToKeyword("field", objectType)), () -> assertEquals("field", OpenSearchTextType.convertTextToKeyword("field", arrayType))); + assertAll( + () -> assertEquals("field", OpenSearchTextType.toKeywordSubField("field", objectType)), + () -> assertEquals("field", OpenSearchTextType.toKeywordSubField("field", arrayType))); } @Test @@ -85,6 +114,10 @@ void text_type_without_fields_isnt_converted() { "field", OpenSearchTextType.convertTextToKeyword( "field", OpenSearchDataType.of(OpenSearchDataType.MappingType.Text))); + assertEquals( + null, + OpenSearchTextType.toKeywordSubField( + "field", OpenSearchDataType.of(OpenSearchDataType.MappingType.Text))); } @Test @@ -94,5 +127,7 @@ void text_type_with_fields_is_converted() { Map.of("keyword", OpenSearchDataType.of(OpenSearchDataType.MappingType.Keyword))); assertEquals( "field.keyword", OpenSearchTextType.convertTextToKeyword("field", textWithKeywordType)); + assertEquals( + "field.keyword", OpenSearchTextType.toKeywordSubField("field", textWithKeywordType)); } }