diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/clickbench/PPLClickBenchIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/clickbench/PPLClickBenchIT.java index 9e730a29ddd..af7d748f5cd 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/clickbench/PPLClickBenchIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/clickbench/PPLClickBenchIT.java @@ -48,9 +48,12 @@ public static void reset() throws IOException { System.out.println(); } - /** Ignore queries that are not supported by Calcite. */ + /** + * Ignore queries that are not supported by Calcite. Ignore q30 because of too much script push + * down, which will cause ResourceMonitor restriction. + */ protected Set ignored() { - return Set.of(29); + return Set.of(29, 30); } @Test diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java index 17f2dc95d22..0cd5269a706 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java @@ -790,4 +790,20 @@ public void testSumNull() throws IOException { + "}", response.toString()); } + + @Test + public void testAggWithFunction() throws IOException { + JSONObject response = + executeQuery( + String.format( + "source=%s | eval len = length(gender) | stats sum(balance + 100) as sum by len," + + " gender ", + TEST_INDEX_BANK)); + verifySchema( + response, + schema("sum", null, "bigint"), + schema("len", null, "int"), + schema("gender", null, "string")); + verifyDataRows(response, rows(121764, 1, "F"), rows(65909, 1, "M")); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/tpch/CalcitePPLTpchIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/tpch/CalcitePPLTpchIT.java index 508b7648156..46107a60b04 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/tpch/CalcitePPLTpchIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/tpch/CalcitePPLTpchIT.java @@ -59,45 +59,45 @@ public void testQ1() throws IOException { "A", "F", 37474, - 37569624.63999998, - 35676192.096999995, - 37101416.22242404, + isPushdownEnabled() ? 37569624.64 : 37569624.63999998, + isPushdownEnabled() ? 35676192.097 : 35676192.096999995, + isPushdownEnabled() ? 37101416.222424 : 37101416.22242404, 25.354533152909337, - 25419.231826792948, - 0.050866035182679493, + isPushdownEnabled() ? 25419.231826792962 : 25419.231826792948, + isPushdownEnabled() ? 0.0508660351826793 : 0.050866035182679493, 1478), rows( "N", "F", 1041, 1041301.07, - 999060.8979999998, - 1036450.80228, + isPushdownEnabled() ? 999060.898 : 999060.8979999998, + isPushdownEnabled() ? 1036450.8022800001 : 1036450.80228, 27.394736842105264, 27402.659736842103, - 0.042894736842105284, + isPushdownEnabled() ? 0.04289473684210526 : 0.042894736842105284, 38), rows( "N", "O", 75168, - 75384955.36999969, - 71653166.30340016, - 74498798.13307281, + isPushdownEnabled() ? 75384955.37 : 75384955.36999969, + isPushdownEnabled() ? 71653166.3034 : 71653166.30340016, + isPushdownEnabled() ? 74498798.133073 : 74498798.13307281, 25.558653519211152, - 25632.422771166166, - 0.04969738184291069, + isPushdownEnabled() ? 25632.42277116627 : 25632.422771166166, + isPushdownEnabled() ? 0.049697381842910573 : 0.04969738184291069, 2941), rows( "R", "F", 36511, 36570841.24, - 34738472.87580004, - 36169060.11219294, + isPushdownEnabled() ? 34738472.8758 : 34738472.87580004, + isPushdownEnabled() ? 36169060.112193 : 36169060.11219294, 25.059025394646532, 25100.09693891558, - 0.050027453671928686, + isPushdownEnabled() ? 0.05002745367192862 : 0.050027453671928686, 1457)); } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/ExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/ExplainIT.java index 06d1120b859..764a95417e0 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/ExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/ExplainIT.java @@ -353,7 +353,6 @@ public void testPatternsSimplePatternMethodWithoutAggExplain() throws IOExceptio @Test public void testPatternsSimplePatternMethodWithAggPushDownExplain() throws IOException { - // TODO: Correct calcite expected result once pushdown is supported String expected = loadExpectedPlan("explain_patterns_simple_pattern_agg_push.json"); assertJsonEqualsIgnoreId( expected, @@ -472,6 +471,37 @@ public void testDifferentFilterScriptPushDownBehaviorExplain() throws Exception } } + @Test + public void testExplainOnTake() throws IOException { + String expected = loadExpectedPlan("explain_take.json"); + assertJsonEqualsIgnoreId( + expected, + explainQueryToString( + "source=opensearch-sql_test_index_account | stats take(firstname, 2) as take")); + } + + @Test + public void testExplainOnPercentile() throws IOException { + String expected = loadExpectedPlan("explain_percentile.json"); + assertJsonEqualsIgnoreId( + expected, + explainQueryToString( + "source=opensearch-sql_test_index_account | stats percentile(balance, 50) as p50," + + " percentile(balance, 90) as p90")); + } + + @Test + public void testExplainOnAggregationWithFunction() throws IOException { + String expected = loadExpectedPlan("explain_agg_with_script.json"); + assertJsonEqualsIgnoreId( + expected, + explainQueryToString( + String.format( + "source=%s | eval len = length(gender) | stats sum(balance + 100) as sum by len," + + " gender ", + TEST_INDEX_BANK))); + } + protected String loadExpectedPlan(String fileName) throws IOException { String prefix; if (isCalciteEnabled()) { diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/StatsCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/StatsCommandIT.java index 48e2f506e61..f7aba663190 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/StatsCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/StatsCommandIT.java @@ -441,12 +441,12 @@ public void testStatsPercentileByNullValue() throws IOException { verifySchema(response, schema("p50", null, "bigint"), schema("age", null, "int")); verifyDataRows( response, - rows(isCalciteEnabled() ? null : 0, null), + rows(isCalciteEnabled() && !isPushdownEnabled() ? null : 0, null), rows(32838, 28), rows(39225, 32), rows(4180, 33), rows(48086, 34), - rows(isCalciteEnabled() ? null : 0, 36)); + rows(isCalciteEnabled() && !isPushdownEnabled() ? null : 0, 36)); } @Test diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_script.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_script.json new file mode 100644 index 00000000000..cd7044baec6 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_script.json @@ -0,0 +1,6 @@ +{ + "calcite": { + "logical": "LogicalProject(sum=[$2], len=[$0], gender=[$1])\n LogicalAggregate(group=[{0, 1}], sum=[SUM($2)])\n LogicalProject(len=[CHAR_LENGTH($4)], gender=[$4], $f3=[+($7, 100)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", + "physical": "EnumerableCalc(expr#0..2=[{inputs}], sum=[$t2], len=[$t0], gender=[$t1])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},sum=SUM($2))], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"len\":{\"terms\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQHjnsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhY2NvdW50X251bWJlciIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImZpcnN0bmFtZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImFkZHJlc3MiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJiaXJ0aGRhdGUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJnZW5kZXIiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJjaXR5IgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAibGFzdG5hbWUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJiYWxhbmNlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZW1wbG95ZXIiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJzdGF0ZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhZ2UiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJlbWFpbCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIkJPT0xFQU4iLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJtYWxlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiX2lkIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiX2luZGV4IgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiUkVBTCIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJuYW1lIjogIl9zY29yZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlJFQUwiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfbWF4c2NvcmUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfc29ydCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9yb3V0aW5nIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogdHJ1ZQp9dAAEZXhwcnQApnsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDSEFSX0xFTkdUSCIsCiAgICAia2luZCI6ICJDSEFSX0xFTkdUSCIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiaW5wdXQiOiA0LAogICAgICAibmFtZSI6ICIkNCIKICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAF2phdmEudXRpbC5MaW5rZWRIYXNoTWFwNMBOXBBswPsCAAFaAAthY2Nlc3NPcmRlcnhyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAABh3CAAAACAAAAANdAAOYWNjb3VudF9udW1iZXJ+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AARMT05HdAAJZmlyc3RuYW1lfnEAfgALdAAGU1RSSU5HdAAHYWRkcmVzc3NyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaFRleHRUeXBlrYOjkwTjMUQCAAFMAAZmaWVsZHN0AA9MamF2YS91dGlsL01hcDt4cgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZcJjvMoC+gU1AgADTAAMZXhwckNvcmVUeXBldAArTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL3R5cGUvRXhwckNvcmVUeXBlO0wAC21hcHBpbmdUeXBldABITG9yZy9vcGVuc2VhcmNoL3NxbC9vcGVuc2VhcmNoL2RhdGEvdHlwZS9PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGU7TAAKcHJvcGVydGllc3EAfgAUeHB+cQB+AAt0AAdVTktOT1dOfnIARm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGUAAAAAAAAAABIAAHhxAH4ADHQABFRleHRzcgA8c2hhZGVkLmNvbS5nb29nbGUuY29tbW9uLmNvbGxlY3QuSW1tdXRhYmxlTWFwJFNlcmlhbGl6ZWRGb3JtAAAAAAAAAAACAAJMAARrZXlzdAASTGphdmEvbGFuZy9PYmplY3Q7TAAGdmFsdWVzcQB+AB94cHVyABNbTGphdmEubGFuZy5PYmplY3Q7kM5YnxBzKWwCAAB4cAAAAAB1cQB+ACEAAAAAc3EAfgAAAAAAA3cEAAAAAHh0AAliaXJ0aGRhdGVzcgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRlVHlwZZ4tUq4QfcqvAgABTAAHZm9ybWF0c3QAEExqYXZhL3V0aWwvTGlzdDt4cQB+ABV+cQB+AAt0AAlUSU1FU1RBTVB+cQB+ABt0AAREYXRlcQB+ACBzcQB+AAAAAAABdwQAAAAAeHQABmdlbmRlcnNxAH4AE3EAfgAZcQB+ABxxAH4AIHNxAH4AAAAAAAN3BAAAAAJ0AAdrZXl3b3Jkc3EAfgAVcQB+ABB+cQB+ABt0AAdLZXl3b3JkcQB+ACB4dAAEY2l0eXEAfgAQdAAIbGFzdG5hbWVxAH4AEHQAB2JhbGFuY2VxAH4ADXQACGVtcGxveWVyc3EAfgATcQB+ABlxAH4AHHEAfgAgcQB+ACR0AAVzdGF0ZXNxAH4AE3EAfgAZcQB+ABxxAH4AIHNxAH4AAAAAAAN3BAAAAAJxAH4AMXEAfgAyeHQAA2FnZX5xAH4AC3QAB0lOVEVHRVJ0AAVlbWFpbHNxAH4AE3EAfgAZcQB+ABxxAH4AIHEAfgAkdAAEbWFsZX5xAH4AC3QAB0JPT0xFQU54AHg=\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*}},\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}},{\"gender\":{\"terms\":{\"field\":\"gender.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"sum\":{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQHjnsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhY2NvdW50X251bWJlciIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImZpcnN0bmFtZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImFkZHJlc3MiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJiaXJ0aGRhdGUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJnZW5kZXIiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJjaXR5IgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAibGFzdG5hbWUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJiYWxhbmNlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZW1wbG95ZXIiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJzdGF0ZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhZ2UiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJlbWFpbCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIkJPT0xFQU4iLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJtYWxlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiX2lkIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiX2luZGV4IgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiUkVBTCIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJuYW1lIjogIl9zY29yZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlJFQUwiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfbWF4c2NvcmUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfc29ydCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9yb3V0aW5nIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogdHJ1ZQp9dAAEZXhwcnQBA3sKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIlBMVVMiLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJpbnB1dCI6IDcsCiAgICAgICJuYW1lIjogIiQ3IgogICAgfSwKICAgIHsKICAgICAgImxpdGVyYWwiOiAxMDAsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlCiAgICAgIH0KICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAF2phdmEudXRpbC5MaW5rZWRIYXNoTWFwNMBOXBBswPsCAAFaAAthY2Nlc3NPcmRlcnhyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAABh3CAAAACAAAAANdAAOYWNjb3VudF9udW1iZXJ+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AARMT05HdAAJZmlyc3RuYW1lfnEAfgALdAAGU1RSSU5HdAAHYWRkcmVzc3NyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaFRleHRUeXBlrYOjkwTjMUQCAAFMAAZmaWVsZHN0AA9MamF2YS91dGlsL01hcDt4cgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZcJjvMoC+gU1AgADTAAMZXhwckNvcmVUeXBldAArTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL3R5cGUvRXhwckNvcmVUeXBlO0wAC21hcHBpbmdUeXBldABITG9yZy9vcGVuc2VhcmNoL3NxbC9vcGVuc2VhcmNoL2RhdGEvdHlwZS9PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGU7TAAKcHJvcGVydGllc3EAfgAUeHB+cQB+AAt0AAdVTktOT1dOfnIARm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGUAAAAAAAAAABIAAHhxAH4ADHQABFRleHRzcgA8c2hhZGVkLmNvbS5nb29nbGUuY29tbW9uLmNvbGxlY3QuSW1tdXRhYmxlTWFwJFNlcmlhbGl6ZWRGb3JtAAAAAAAAAAACAAJMAARrZXlzdAASTGphdmEvbGFuZy9PYmplY3Q7TAAGdmFsdWVzcQB+AB94cHVyABNbTGphdmEubGFuZy5PYmplY3Q7kM5YnxBzKWwCAAB4cAAAAAB1cQB+ACEAAAAAc3EAfgAAAAAAA3cEAAAAAHh0AAliaXJ0aGRhdGVzcgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRlVHlwZZ4tUq4QfcqvAgABTAAHZm9ybWF0c3QAEExqYXZhL3V0aWwvTGlzdDt4cQB+ABV+cQB+AAt0AAlUSU1FU1RBTVB+cQB+ABt0AAREYXRlcQB+ACBzcQB+AAAAAAABdwQAAAAAeHQABmdlbmRlcnNxAH4AE3EAfgAZcQB+ABxxAH4AIHNxAH4AAAAAAAN3BAAAAAJ0AAdrZXl3b3Jkc3EAfgAVcQB+ABB+cQB+ABt0AAdLZXl3b3JkcQB+ACB4dAAEY2l0eXEAfgAQdAAIbGFzdG5hbWVxAH4AEHQAB2JhbGFuY2VxAH4ADXQACGVtcGxveWVyc3EAfgATcQB+ABlxAH4AHHEAfgAgcQB+ACR0AAVzdGF0ZXNxAH4AE3EAfgAZcQB+ABxxAH4AIHNxAH4AAAAAAAN3BAAAAAJxAH4AMXEAfgAyeHQAA2FnZX5xAH4AC3QAB0lOVEVHRVJ0AAVlbWFpbHNxAH4AE3EAfgAZcQB+ABxxAH4AIHEAfgAkdAAEbWFsZX5xAH4AC3QAB0JPT0xFQU54AHg=\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + } +} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_output.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_output.json index f3f1b89ff49..1653c487348 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_output.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_output.json @@ -1,6 +1,6 @@ { "calcite": { "logical": "LogicalProject(age2=[$2])\n LogicalFilter(condition=[<=($3, 1)])\n LogicalProject(avg_age=[$0], state=[$1], age2=[$2], _row_number_=[ROW_NUMBER() OVER (PARTITION BY $2 ORDER BY $2)])\n LogicalFilter(condition=[IS NOT NULL($2)])\n LogicalProject(avg_age=[$0], state=[$1], age2=[+($0, 2)])\n LogicalSort(sort0=[$1], dir0=[ASC-nulls-first])\n LogicalProject(avg_age=[$2], state=[$0], city=[$1])\n LogicalAggregate(group=[{0, 1}], avg_age=[AVG($2)])\n LogicalProject(state=[$7], city=[$5], age=[$8])\n LogicalFilter(condition=[>($8, 30)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1], expr#4=[<=($t2, $t3)], age2=[$t1], $condition=[$t4])\n EnumerableWindow(window#0=[window(partition {1} order by [1] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])])\n EnumerableCalc(expr#0..2=[{inputs}], expr#3=[2], expr#4=[+($t2, $t3)], expr#5=[IS NOT NULL($t2)], state=[$t0], age2=[$t4], $condition=[$t5])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[city, state, age], FILTER->>($2, 30), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_age=AVG($2)), SORT->[{\n \"state.keyword\" : {\n \"order\" : \"asc\",\n \"missing\" : \"_first\"\n }\n}]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"range\":{\"age\":{\"from\":30,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},\"_source\":{\"includes\":[\"city\",\"state\",\"age\"],\"excludes\":[]},\"sort\":[{\"state.keyword\":{\"order\":\"asc\",\"missing\":\"_first\"}}],\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}},{\"city\":{\"terms\":{\"field\":\"city.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"avg_age\":{\"avg\":{\"field\":\"age\"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + "physical": "EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1], expr#4=[<=($t2, $t3)], age2=[$t1], $condition=[$t4])\n EnumerableWindow(window#0=[window(partition {1} order by [1] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])])\n EnumerableCalc(expr#0..2=[{inputs}], expr#3=[2], expr#4=[+($t2, $t3)], expr#5=[IS NOT NULL($t2)], state=[$t0], age2=[$t4], $condition=[$t5])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[city, state, age], FILTER->>($2, 30), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_age=AVG($2)), SORT->[0 ASC FIRST]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"range\":{\"age\":{\"from\":30,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},\"_source\":{\"includes\":[\"city\",\"state\",\"age\"],\"excludes\":[]},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}],\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":10,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}},{\"city\":{\"terms\":{\"field\":\"city.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"avg_age\":{\"avg\":{\"field\":\"age\"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_patterns_simple_pattern_agg_push.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_patterns_simple_pattern_agg_push.json index 50cfc66c07a..0a096afd754 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_patterns_simple_pattern_agg_push.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_patterns_simple_pattern_agg_push.json @@ -1,6 +1,6 @@ { - "calcite":{ - "logical":"LogicalProject(pattern_count=[$1], patterns_field=[SAFE_CAST(ITEM(PATTERN_PARSER($0, $2), 'pattern'))], tokens=[SAFE_CAST(ITEM(PATTERN_PARSER($0, $2), 'tokens'))])\n LogicalAggregate(group=[{1}], pattern_count=[COUNT($1)], sample_logs=[TAKE($0, $2)])\n LogicalProject(email=[$9], patterns_field=[REGEXP_REPLACE($9, '[a-zA-Z0-9]+':VARCHAR, '<*>')], $f18=[10])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical":"EnumerableCalc(expr#0..2=[{inputs}], expr#3=[PATTERN_PARSER($t0, $t2)], expr#4=['pattern'], expr#5=[ITEM($t3, $t4)], expr#6=[SAFE_CAST($t5)], expr#7=['tokens'], expr#8=[ITEM($t3, $t7)], expr#9=[SAFE_CAST($t8)], pattern_count=[$t1], patterns_field=[$t6], tokens=[$t9])\n EnumerableAggregate(group=[{1}], pattern_count=[COUNT($1)], sample_logs=[TAKE($0, $2)])\n EnumerableCalc(expr#0=[{inputs}], expr#1=['[a-zA-Z0-9]+':VARCHAR], expr#2=['<*>'], expr#3=[REGEXP_REPLACE($t0, $t1, $t2)], expr#4=[10], email=[$t0], $f1=[$t3], $f2=[$t4])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[email]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"email\"],\"excludes\":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + "calcite": { + "logical": "LogicalProject(pattern_count=[$1], patterns_field=[SAFE_CAST(ITEM(PATTERN_PARSER($0, $2), 'pattern'))], tokens=[SAFE_CAST(ITEM(PATTERN_PARSER($0, $2), 'tokens'))])\n LogicalAggregate(group=[{1}], pattern_count=[COUNT($1)], sample_logs=[TAKE($0, $2)])\n LogicalProject(email=[$9], patterns_field=[REGEXP_REPLACE($9, '[a-zA-Z0-9]+':VARCHAR, '<*>')], $f18=[10])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", + "physical": "EnumerableCalc(expr#0..2=[{inputs}], expr#3=[PATTERN_PARSER($t0, $t2)], expr#4=['pattern'], expr#5=[ITEM($t3, $t4)], expr#6=[SAFE_CAST($t5)], expr#7=['tokens'], expr#8=[ITEM($t3, $t7)], expr#9=[SAFE_CAST($t8)], pattern_count=[$t1], patterns_field=[$t6], tokens=[$t9])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},pattern_count=COUNT($1),sample_logs=TAKE($0, $2))], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"patterns_field\":{\"terms\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQGy3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhY2NvdW50X251bWJlciIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImZpcnN0bmFtZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImFkZHJlc3MiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJiYWxhbmNlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZ2VuZGVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiY2l0eSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImVtcGxveWVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAic3RhdGUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhZ2UiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJlbWFpbCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImxhc3RuYW1lIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiX2lkIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiX2luZGV4IgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiUkVBTCIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJuYW1lIjogIl9zY29yZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlJFQUwiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfbWF4c2NvcmUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfc29ydCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9yb3V0aW5nIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogdHJ1ZQp9dAAEZXhwcnQBx3sKICAib3AiOiB7CiAgICAibmFtZSI6ICJSRUdFWFBfUkVQTEFDRSIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiaW5wdXQiOiA5LAogICAgICAibmFtZSI6ICIkOSIKICAgIH0sCiAgICB7CiAgICAgICJsaXRlcmFsIjogIlthLXpBLVowLTldKyIsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9LAogICAgewogICAgICAibGl0ZXJhbCI6ICI8Kj4iLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZSwKICAgICAgICAicHJlY2lzaW9uIjogMwogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABdqYXZhLnV0aWwuTGlua2VkSGFzaE1hcDTATlwQbMD7AgABWgALYWNjZXNzT3JkZXJ4cgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAMdwgAAAAQAAAAC3QADmFjY291bnRfbnVtYmVyfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAETE9OR3QACWZpcnN0bmFtZXNyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaFRleHRUeXBlrYOjkwTjMUQCAAFMAAZmaWVsZHN0AA9MamF2YS91dGlsL01hcDt4cgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZcJjvMoC+gU1AgADTAAMZXhwckNvcmVUeXBldAArTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL3R5cGUvRXhwckNvcmVUeXBlO0wAC21hcHBpbmdUeXBldABITG9yZy9vcGVuc2VhcmNoL3NxbC9vcGVuc2VhcmNoL2RhdGEvdHlwZS9PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGU7TAAKcHJvcGVydGllc3EAfgAReHB+cQB+AAt0AAdVTktOT1dOfnIARm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGUAAAAAAAAAABIAAHhxAH4ADHQABFRleHRzcgA8c2hhZGVkLmNvbS5nb29nbGUuY29tbW9uLmNvbGxlY3QuSW1tdXRhYmxlTWFwJFNlcmlhbGl6ZWRGb3JtAAAAAAAAAAACAAJMAARrZXlzdAASTGphdmEvbGFuZy9PYmplY3Q7TAAGdmFsdWVzcQB+ABx4cHVyABNbTGphdmEubGFuZy5PYmplY3Q7kM5YnxBzKWwCAAB4cAAAAAB1cQB+AB4AAAAAc3EAfgAAAAAAA3cEAAAAAnQAB2tleXdvcmRzcQB+ABJ+cQB+AAt0AAZTVFJJTkd+cQB+ABh0AAdLZXl3b3JkcQB+AB14dAAHYWRkcmVzc3NxAH4AEHEAfgAWcQB+ABlxAH4AHXNxAH4AAAAAAAN3BAAAAAB4dAAHYmFsYW5jZXEAfgANdAAGZ2VuZGVyc3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAnEAfgAicQB+ACN4dAAEY2l0eXNxAH4AEHEAfgAWcQB+ABlxAH4AHXNxAH4AAAAAAAN3BAAAAAJxAH4AInEAfgAjeHQACGVtcGxveWVyc3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAnEAfgAicQB+ACN4dAAFc3RhdGVzcQB+ABBxAH4AFnEAfgAZcQB+AB1zcQB+AAAAAAADdwQAAAACcQB+ACJxAH4AI3h0AANhZ2VxAH4ADXQABWVtYWlsc3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAnEAfgAicQB+ACN4dAAIbGFzdG5hbWVzcQB+ABBxAH4AFnEAfgAZcQB+AB1zcQB+AAAAAAADdwQAAAACcQB+ACJxAH4AI3h4AHg=\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*}},\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"pattern_count\":{\"value_count\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQGy3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhY2NvdW50X251bWJlciIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImZpcnN0bmFtZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImFkZHJlc3MiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJiYWxhbmNlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZ2VuZGVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiY2l0eSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImVtcGxveWVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAic3RhdGUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhZ2UiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJlbWFpbCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImxhc3RuYW1lIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiX2lkIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiX2luZGV4IgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiUkVBTCIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJuYW1lIjogIl9zY29yZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlJFQUwiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfbWF4c2NvcmUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfc29ydCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9yb3V0aW5nIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogdHJ1ZQp9dAAEZXhwcnQBx3sKICAib3AiOiB7CiAgICAibmFtZSI6ICJSRUdFWFBfUkVQTEFDRSIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiaW5wdXQiOiA5LAogICAgICAibmFtZSI6ICIkOSIKICAgIH0sCiAgICB7CiAgICAgICJsaXRlcmFsIjogIlthLXpBLVowLTldKyIsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9LAogICAgewogICAgICAibGl0ZXJhbCI6ICI8Kj4iLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZSwKICAgICAgICAicHJlY2lzaW9uIjogMwogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABdqYXZhLnV0aWwuTGlua2VkSGFzaE1hcDTATlwQbMD7AgABWgALYWNjZXNzT3JkZXJ4cgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAMdwgAAAAQAAAAC3QADmFjY291bnRfbnVtYmVyfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAETE9OR3QACWZpcnN0bmFtZXNyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaFRleHRUeXBlrYOjkwTjMUQCAAFMAAZmaWVsZHN0AA9MamF2YS91dGlsL01hcDt4cgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZcJjvMoC+gU1AgADTAAMZXhwckNvcmVUeXBldAArTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL3R5cGUvRXhwckNvcmVUeXBlO0wAC21hcHBpbmdUeXBldABITG9yZy9vcGVuc2VhcmNoL3NxbC9vcGVuc2VhcmNoL2RhdGEvdHlwZS9PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGU7TAAKcHJvcGVydGllc3EAfgAReHB+cQB+AAt0AAdVTktOT1dOfnIARm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGUAAAAAAAAAABIAAHhxAH4ADHQABFRleHRzcgA8c2hhZGVkLmNvbS5nb29nbGUuY29tbW9uLmNvbGxlY3QuSW1tdXRhYmxlTWFwJFNlcmlhbGl6ZWRGb3JtAAAAAAAAAAACAAJMAARrZXlzdAASTGphdmEvbGFuZy9PYmplY3Q7TAAGdmFsdWVzcQB+ABx4cHVyABNbTGphdmEubGFuZy5PYmplY3Q7kM5YnxBzKWwCAAB4cAAAAAB1cQB+AB4AAAAAc3EAfgAAAAAAA3cEAAAAAnQAB2tleXdvcmRzcQB+ABJ+cQB+AAt0AAZTVFJJTkd+cQB+ABh0AAdLZXl3b3JkcQB+AB14dAAHYWRkcmVzc3NxAH4AEHEAfgAWcQB+ABlxAH4AHXNxAH4AAAAAAAN3BAAAAAB4dAAHYmFsYW5jZXEAfgANdAAGZ2VuZGVyc3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAnEAfgAicQB+ACN4dAAEY2l0eXNxAH4AEHEAfgAWcQB+ABlxAH4AHXNxAH4AAAAAAAN3BAAAAAJxAH4AInEAfgAjeHQACGVtcGxveWVyc3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAnEAfgAicQB+ACN4dAAFc3RhdGVzcQB+ABBxAH4AFnEAfgAZcQB+AB1zcQB+AAAAAAADdwQAAAACcQB+ACJxAH4AI3h0AANhZ2VxAH4ADXQABWVtYWlsc3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAnEAfgAicQB+ACN4dAAIbGFzdG5hbWVzcQB+ABBxAH4AFnEAfgAZcQB+AB1zcQB+AAAAAAADdwQAAAACcQB+ACJxAH4AI3h4AHg=\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*}}}},\"sample_logs\":{\"top_hits\":{\"from\":0,\"size\":10,\"version\":false,\"seq_no_primary_term\":false,\"explain\":false,\"_source\":{\"includes\":[\"email\"],\"excludes\":[]}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" } -} \ No newline at end of file +} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_percentile.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_percentile.json new file mode 100644 index 00000000000..81efcf6e87a --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_percentile.json @@ -0,0 +1,6 @@ +{ + "calcite": { + "logical": "LogicalAggregate(group=[{}], p50=[percentile_approx($0, $1, $2)], p90=[percentile_approx($0, $3, $2)])\n LogicalProject(balance=[$3], $f2=[50], $f3=[FLAG(BIGINT)], $f4=[90])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", + "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},p50=percentile_approx($0, $1, $2),p90=percentile_approx($0, $3, $2))], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"p50\":{\"percentiles\":{\"field\":\"balance\",\"percents\":[50.0],\"keyed\":true,\"tdigest\":{\"compression\":100.0}}},\"p90\":{\"percentiles\":{\"field\":\"balance\",\"percents\":[90.0],\"keyed\":true,\"tdigest\":{\"compression\":100.0}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + } +} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_take.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_take.json new file mode 100644 index 00000000000..87f40097fc5 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_take.json @@ -0,0 +1,6 @@ +{ + "calcite": { + "logical": "LogicalAggregate(group=[{}], take=[TAKE($0, $1)])\n LogicalProject(firstname=[$1], $f1=[2])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", + "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},take=TAKE($0, $1))], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"take\":{\"top_hits\":{\"from\":0,\"size\":2,\"version\":false,\"seq_no_primary_term\":false,\"explain\":false,\"_source\":{\"includes\":[\"firstname\"],\"excludes\":[]}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + } +} diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_script.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_script.json new file mode 100644 index 00000000000..f5efb096fc3 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_script.json @@ -0,0 +1,6 @@ +{ + "calcite": { + "logical": "LogicalProject(sum=[$2], len=[$0], gender=[$1])\n LogicalAggregate(group=[{0, 1}], sum=[SUM($2)])\n LogicalProject(len=[CHAR_LENGTH($4)], gender=[$4], $f3=[+($7, 100)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", + "physical": "EnumerableCalc(expr#0..2=[{inputs}], sum=[$t2], len=[$t0], gender=[$t1])\n EnumerableAggregate(group=[{0, 1}], sum=[SUM($2)])\n EnumerableCalc(expr#0..18=[{inputs}], expr#19=[CHAR_LENGTH($t4)], expr#20=[100], expr#21=[+($t7, $t20)], len=[$t19], gender=[$t4], $f3=[$t21])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n" + } +} diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_percentile.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_percentile.json new file mode 100644 index 00000000000..bcba13e8fe2 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_percentile.json @@ -0,0 +1,6 @@ +{ + "calcite": { + "logical": "LogicalAggregate(group=[{}], p50=[percentile_approx($0, $1, $2)], p90=[percentile_approx($0, $3, $2)])\n LogicalProject(balance=[$3], $f2=[50], $f3=[FLAG(BIGINT)], $f4=[90])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", + "physical": "EnumerableAggregate(group=[{}], p50=[percentile_approx($0, $1, $2)], p90=[percentile_approx($0, $3, $2)])\n EnumerableCalc(expr#0..16=[{inputs}], expr#17=[50], expr#18=[FLAG(BIGINT)], expr#19=[90], balance=[$t3], $f2=[$t17], $f3=[$t18], $f4=[$t19])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" + } +} diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_take.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_take.json new file mode 100644 index 00000000000..cc191b55032 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_take.json @@ -0,0 +1,6 @@ +{ + "calcite": { + "logical": "LogicalAggregate(group=[{}], take=[TAKE($0, $1)])\n LogicalProject(firstname=[$1], $f1=[2])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", + "physical": "EnumerableAggregate(group=[{}], take=[TAKE($0, $1)])\n EnumerableCalc(expr#0..16=[{inputs}], expr#17=[2], firstname=[$t1], $f1=[$t17])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" + } +} diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_agg_with_script.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_agg_with_script.json new file mode 100644 index 00000000000..7af36f9596b --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_agg_with_script.json @@ -0,0 +1,36 @@ +{ + "root": { + "name": "ProjectOperator", + "description": { + "fields": "[sum, len, gender]" + }, + "children": [ + { + "name": "AggregationOperator", + "description": { + "aggregators": "[sum]", + "groupBy": "[len, gender]" + }, + "children": [ + { + "name": "OpenSearchEvalOperator", + "description": { + "expressions": { + "len": "length(gender)" + } + }, + "children": [ + { + "name": "OpenSearchIndexScan", + "description": { + "request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_bank, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\"}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" + }, + "children": [] + } + ] + } + ] + } + ] + } +} diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_percentile.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_percentile.json new file mode 100644 index 00000000000..c55e236e3c1 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_percentile.json @@ -0,0 +1,17 @@ +{ + "root": { + "name": "ProjectOperator", + "description": { + "fields": "[p50, p90]" + }, + "children": [ + { + "name": "OpenSearchIndexScan", + "description": { + "request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account, sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"p50\":{\"percentiles\":{\"field\":\"balance\",\"percents\":[50.0],\"keyed\":true,\"tdigest\":{\"compression\":100.0}}},\"p90\":{\"percentiles\":{\"field\":\"balance\",\"percents\":[90.0],\"keyed\":true,\"tdigest\":{\"compression\":100.0}}}}}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + }, + "children": [] + } + ] + } +} diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_take.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_take.json new file mode 100644 index 00000000000..5e623b3fd26 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_take.json @@ -0,0 +1,17 @@ +{ + "root": { + "name": "ProjectOperator", + "description": { + "fields": "[take]" + }, + "children": [ + { + "name": "OpenSearchIndexScan", + "description": { + "request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account, sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"take\":{\"top_hits\":{\"from\":0,\"size\":2,\"version\":false,\"seq_no_primary_term\":false,\"explain\":false,\"_source\":{\"includes\":[\"firstname\"],\"excludes\":[]}}}}}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + }, + "children": [] + } + ] + } +} diff --git a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/3570.yml b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/3570.yml index 3176817123c..a42e8c8478d 100644 --- a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/3570.yml +++ b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/3570.yml @@ -70,7 +70,7 @@ teardown: Content-Type: 'application/json' ppl: body: - query: 'source=hdfs_logs | patterns content mode=aggregation | fields patterns_field, pattern_count' + query: 'source=hdfs_logs | patterns content mode=aggregation | sort patterns_field | fields patterns_field, pattern_count' - match: {"total": 5} - match: {"schema": [{"name": "patterns_field", "type": "string"}, {"name": "pattern_count", "type": "bigint"}]} - match: {"datarows": [ @@ -78,14 +78,14 @@ teardown: " _-", 2 ], - [ - "* .: ////_/______/-. _-", - 1 - ], [ " _", 2 ], + [ + "* .: ////_/______/-. _-", + 1 + ], [ "* .: ////_/______/-. _", 1 diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/AggregateAnalyzer.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/AggregateAnalyzer.java index 28b6b2ccc4d..f0e60362ba1 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/AggregateAnalyzer.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/AggregateAnalyzer.java @@ -30,7 +30,6 @@ import static org.opensearch.sql.data.type.ExprCoreType.DATE; import static org.opensearch.sql.data.type.ExprCoreType.TIME; import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP; -import static org.opensearch.sql.opensearch.storage.OpenSearchIndex.METADATA_FIELD_INDEX; import com.google.common.base.Throwables; import com.google.common.collect.ImmutableList; @@ -38,15 +37,21 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; +import lombok.RequiredArgsConstructor; +import org.apache.calcite.plan.RelOptCluster; import org.apache.calcite.rel.core.Aggregate; import org.apache.calcite.rel.core.AggregateCall; import org.apache.calcite.rel.core.Project; +import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexCall; import org.apache.calcite.rex.RexInputRef; import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; import org.apache.calcite.sql.SqlKind; import org.apache.commons.lang3.tuple.Pair; +import org.opensearch.script.Script; import org.opensearch.search.aggregations.AggregationBuilder; import org.opensearch.search.aggregations.AggregationBuilders; import org.opensearch.search.aggregations.AggregatorFactories; @@ -55,10 +60,12 @@ import org.opensearch.search.aggregations.bucket.composite.TermsValuesSourceBuilder; import org.opensearch.search.aggregations.bucket.missing.MissingOrder; import org.opensearch.search.aggregations.metrics.ExtendedStats; +import org.opensearch.search.aggregations.metrics.PercentilesAggregationBuilder; import org.opensearch.search.aggregations.support.ValueType; import org.opensearch.search.aggregations.support.ValuesSourceAggregationBuilder; import org.opensearch.search.sort.SortOrder; import org.opensearch.sql.ast.expression.SpanUnit; +import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory; import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.expression.function.BuiltinFunctionName; import org.opensearch.sql.opensearch.request.PredicateAnalyzer.NamedFieldExpression; @@ -66,8 +73,10 @@ import org.opensearch.sql.opensearch.response.agg.MetricParser; import org.opensearch.sql.opensearch.response.agg.NoBucketAggregationParser; import org.opensearch.sql.opensearch.response.agg.OpenSearchAggregationResponseParser; +import org.opensearch.sql.opensearch.response.agg.SinglePercentileParser; import org.opensearch.sql.opensearch.response.agg.SingleValueParser; import org.opensearch.sql.opensearch.response.agg.StatsParser; +import org.opensearch.sql.opensearch.response.agg.TopHitsParser; import org.opensearch.sql.opensearch.storage.script.aggregation.dsl.BucketAggregationBuilder; /** @@ -107,31 +116,71 @@ public static class ExpressionNotAnalyzableException extends Exception { private AggregateAnalyzer() {} + @RequiredArgsConstructor + private static class AggregateBuilderHelper { + private final RelDataType rowType; + private final Map fieldTypes; + private final RelOptCluster cluster; + + > T build(RexNode node, T aggBuilder) { + return build(node, aggBuilder::field, aggBuilder::script); + } + + > T build(RexNode node, T sourceBuilder) { + return build(node, sourceBuilder::field, sourceBuilder::script); + } + + T build(RexNode node, Function fieldBuilder, Function scriptBuilder) { + if (node == null) return fieldBuilder.apply(METADATA_FIELD); + else if (node instanceof RexInputRef) { + RexInputRef ref = (RexInputRef) node; + return fieldBuilder.apply( + new NamedFieldExpression(ref.getIndex(), rowType.getFieldNames(), fieldTypes) + .getReferenceForTermQuery()); + } else if (node instanceof RexCall || node instanceof RexLiteral) { + return scriptBuilder.apply( + (new PredicateAnalyzer.ScriptQueryExpression(node, rowType, fieldTypes, cluster)) + .getScript()); + } + throw new IllegalStateException( + String.format("Metric aggregation doesn't support RexNode %s", node)); + } + + NamedFieldExpression inferNamedField(RexNode node) { + if (node instanceof RexInputRef) { + RexInputRef ref = (RexInputRef) node; + return new NamedFieldExpression(ref.getIndex(), rowType.getFieldNames(), fieldTypes); + } + throw new IllegalStateException( + String.format("Cannot infer field name from RexNode %s", node)); + } + + T inferValue(RexNode node, Class clazz) { + if (node instanceof RexLiteral) { + RexLiteral literal = (RexLiteral) node; + return literal.getValueAs(clazz); + } + throw new IllegalStateException(String.format("Cannot infer value from RexNode %s", node)); + } + } + // TODO: should we support filter aggregation? For PPL, we don't have filter in stats command - // TODO: support script pushdown for aggregation. Calcite doesn't expression in its AggregateCall - // or GroupSet - // https://github.com/opensearch-project/sql/issues/3386 - // public static Pair, OpenSearchAggregationResponseParser> analyze( Aggregate aggregate, Project project, - List schema, + RelDataType rowType, Map fieldTypes, - List outputFields) + List outputFields, + RelOptCluster cluster) throws ExpressionNotAnalyzableException { requireNonNull(aggregate, "aggregate"); try { List groupList = aggregate.getGroupSet().asList(); - FieldExpressionCreator fieldExpressionCreator = - fieldIndex -> new NamedFieldExpression(fieldIndex, schema, fieldTypes); + AggregateBuilderHelper helper = new AggregateBuilderHelper(rowType, fieldTypes, cluster); + List aggFieldNames = outputFields.subList(groupList.size(), outputFields.size()); // Process all aggregate calls Pair> builderAndParser = - processAggregateCalls( - groupList.size(), - aggregate.getAggCallList(), - project, - fieldExpressionCreator, - outputFields); + processAggregateCalls(aggFieldNames, aggregate.getAggCallList(), project, helper); Builder metricBuilder = builderAndParser.getLeft(); List metricParserList = builderAndParser.getRight(); @@ -141,7 +190,7 @@ public static Pair, OpenSearchAggregationResponseParser new NoBucketAggregationParser(metricParserList)); } else { List> buckets = - createCompositeBuckets(groupList, project, fieldExpressionCreator); + createCompositeBuckets(groupList, project, helper); return Pair.of( Collections.singletonList( AggregationBuilders.composite("composite_buckets", buckets) @@ -156,163 +205,179 @@ public static Pair, OpenSearchAggregationResponseParser } private static Pair> processAggregateCalls( - int groupOffset, + List aggFieldNames, List aggCalls, Project project, - FieldExpressionCreator fieldExpressionCreator, - List outputFields) { - assert aggCalls.size() + groupOffset == outputFields.size() - : "groups size and agg calls size should match with output fields"; + AggregateBuilderHelper helper) { Builder metricBuilder = new AggregatorFactories.Builder(); List metricParserList = new ArrayList<>(); for (int i = 0; i < aggCalls.size(); i++) { AggregateCall aggCall = aggCalls.get(i); - String argStr = - aggCall.getAggregation().kind == SqlKind.COUNT && aggCall.getArgList().isEmpty() - ? METADATA_FIELD_INDEX - : fieldExpressionCreator - .create(convertAggArgThroughProject(aggCall, project).getIndex()) - .getReferenceForTermQuery(); - String aggField = outputFields.get(groupOffset + i); - - Pair, MetricParser> builderAndParser = - createAggregationBuilderAndParser(aggCall, argStr, aggField); + List args = convertAggArgThroughProject(aggCall, project); + String aggFieldName = aggFieldNames.get(i); + + Pair builderAndParser = + createAggregationBuilderAndParser(aggCall, args, aggFieldName, helper); metricBuilder.addAggregator(builderAndParser.getLeft()); metricParserList.add(builderAndParser.getRight()); } return Pair.of(metricBuilder, metricParserList); } - private static RexInputRef convertAggArgThroughProject(AggregateCall aggCall, Project project) { - RexNode argRex = project.getProjects().get(aggCall.getArgList().get(0)); - if (argRex instanceof RexInputRef) return (RexInputRef)argRex; - else throw new IllegalArgumentException("Unsupported aggregate argument: " + argRex); + private static List convertAggArgThroughProject(AggregateCall aggCall, Project project) { + return project == null + ? List.of() + : aggCall.getArgList().stream().map(project.getProjects()::get).collect(Collectors.toList()); } - private interface FieldExpressionCreator { - NamedFieldExpression create(int fieldIndex); - } - - private static Pair, MetricParser> - createAggregationBuilderAndParser(AggregateCall aggCall, String argStr, String aggField) { + private static Pair createAggregationBuilderAndParser( + AggregateCall aggCall, + List args, + String aggFieldName, + AggregateBuilderHelper helper) { if (aggCall.isDistinct()) { - return createDistinctAggregation(aggCall, argStr, aggField); + return createDistinctAggregation(aggCall, args, aggFieldName, helper); } else { - return createRegularAggregation(aggCall, argStr, aggField); + return createRegularAggregation(aggCall, args, aggFieldName, helper); } } - private static Pair, MetricParser> createDistinctAggregation( - AggregateCall aggCall, String argStr, String aggField) { + private static Pair createDistinctAggregation( + AggregateCall aggCall, + List args, + String aggFieldName, + AggregateBuilderHelper helper) { switch (aggCall.getAggregation().kind) { case COUNT: return Pair.of( - AggregationBuilders.cardinality(aggField).field(argStr), - new SingleValueParser(aggField)); + helper.build( + !args.isEmpty() ? args.get(0) : null, + AggregationBuilders.cardinality(aggFieldName)), + new SingleValueParser(aggFieldName)); default: throw new AggregateAnalyzer.AggregateAnalyzerException( String.format("unsupported distinct aggregator %s", aggCall.getAggregation())); } } - private static Pair, MetricParser> createRegularAggregation( - AggregateCall aggCall, String argStr, String aggField) { + private static Pair createRegularAggregation( + AggregateCall aggCall, + List args, + String aggFieldName, + AggregateBuilderHelper helper) { switch (aggCall.getAggregation().kind) { case AVG: return Pair.of( - AggregationBuilders.avg(aggField).field(argStr), - new SingleValueParser(aggField)); + helper.build(args.get(0), AggregationBuilders.avg(aggFieldName)), + new SingleValueParser(aggFieldName)); case SUM: return Pair.of( - AggregationBuilders.sum(aggField).field(argStr), - new SingleValueParser(aggField)); + helper.build(args.get(0), AggregationBuilders.sum(aggFieldName)), + new SingleValueParser(aggFieldName)); case COUNT: return Pair.of( - AggregationBuilders.count(aggField).field(argStr), - new SingleValueParser(aggField)); + helper.build( + !args.isEmpty() ? args.get(0) : null, AggregationBuilders.count(aggFieldName)), + new SingleValueParser(aggFieldName)); case MIN: return Pair.of( - AggregationBuilders.min(aggField).field(argStr), - new SingleValueParser(aggField)); + helper.build(args.get(0), AggregationBuilders.min(aggFieldName)), + new SingleValueParser(aggFieldName)); case MAX: return Pair.of( - AggregationBuilders.max(aggField).field(argStr), - new SingleValueParser(aggField)); + helper.build(args.get(0), AggregationBuilders.max(aggFieldName)), + new SingleValueParser(aggFieldName)); case VAR_SAMP: return Pair.of( - AggregationBuilders.extendedStats(aggField).field(argStr), - new StatsParser(ExtendedStats::getVarianceSampling, aggField)); + helper.build(args.get(0), AggregationBuilders.extendedStats(aggFieldName)), + new StatsParser(ExtendedStats::getVarianceSampling, aggFieldName)); case VAR_POP: return Pair.of( - AggregationBuilders.extendedStats(aggField).field(argStr), - new StatsParser(ExtendedStats::getVariancePopulation, aggField)); + helper.build(args.get(0), AggregationBuilders.extendedStats(aggFieldName)), + new StatsParser(ExtendedStats::getVariancePopulation, aggFieldName)); case STDDEV_SAMP: return Pair.of( - AggregationBuilders.extendedStats(aggField).field(argStr), - new StatsParser(ExtendedStats::getStdDeviationSampling, aggField)); + helper.build(args.get(0), AggregationBuilders.extendedStats(aggFieldName)), + new StatsParser(ExtendedStats::getStdDeviationSampling, aggFieldName)); case STDDEV_POP: return Pair.of( - AggregationBuilders.extendedStats(aggField).field(argStr), - new StatsParser(ExtendedStats::getStdDeviationPopulation, aggField)); + helper.build(args.get(0), AggregationBuilders.extendedStats(aggFieldName)), + new StatsParser(ExtendedStats::getStdDeviationPopulation, aggFieldName)); + case OTHER_FUNCTION: + BuiltinFunctionName functionName = + BuiltinFunctionName.ofAggregation(aggCall.getAggregation().getName()).get(); + switch (functionName) { + case TAKE: + return Pair.of( + AggregationBuilders.topHits(aggFieldName) + .fetchSource(helper.inferNamedField(args.get(0)).getRootName(), null) + .size(helper.inferValue(args.get(1), Integer.class)) + .from(0), + new TopHitsParser(aggFieldName)); + case PERCENTILE_APPROX: + PercentilesAggregationBuilder aggBuilder = + helper + .build(args.get(0), AggregationBuilders.percentiles(aggFieldName)) + .percentiles(helper.inferValue(args.get(1), Double.class)); + /* See {@link PercentileApproxFunction}, PERCENTILE_APPROX accepts args of [FIELD, PERCENTILE, TYPE, COMPRESSION(optional)] */ + if (args.size() > 3) { + aggBuilder.compression(helper.inferValue(args.get(3), Double.class)); + } + return Pair.of(aggBuilder, new SinglePercentileParser(aggFieldName)); + default: + throw new AggregateAnalyzer.AggregateAnalyzerException( + String.format("Unsupported push-down aggregator %s", aggCall.getAggregation())); + } default: throw new AggregateAnalyzerException( - String.format("unsupported aggregator %s", aggCall.getAggregation())); + String.format("unsupported aggregator %s", aggCall.getAggregation())); } - } private static List> createCompositeBuckets( - List groupList, Project project, FieldExpressionCreator fieldExpressionCreator) { + List groupList, Project project, AggregateAnalyzer.AggregateBuilderHelper helper) { ImmutableList.Builder> resultBuilder = ImmutableList.builder(); - groupList.forEach( - groupIndex -> resultBuilder.add(createBucket(groupIndex, project, fieldExpressionCreator))); + groupList.forEach(groupIndex -> resultBuilder.add(createBucket(groupIndex, project, helper))); return resultBuilder.build(); } private static CompositeValuesSourceBuilder createBucket( - Integer groupIndex, - Project project, - AggregateAnalyzer.FieldExpressionCreator fieldExpressionCreator) { + Integer groupIndex, Project project, AggregateAnalyzer.AggregateBuilderHelper helper) { RexNode rex = project.getProjects().get(groupIndex); - if (rex instanceof RexInputRef) { - NamedFieldExpression groupExpr = fieldExpressionCreator.create(((RexInputRef)rex).getIndex()); - return createTermsSourceBuilder(groupExpr); - } else if (rex instanceof RexCall + String bucketName = project.getRowType().getFieldList().get(groupIndex).getName(); + if (rex instanceof RexCall && rex.getKind() == SqlKind.OTHER_FUNCTION && ((RexCall) rex).getOperator().getName().equalsIgnoreCase(BuiltinFunctionName.SPAN.name()) && ((RexCall) rex).getOperands().size() == 3 && ((RexCall) rex).getOperands().get(0) instanceof RexInputRef && ((RexCall) rex).getOperands().get(1) instanceof RexLiteral && ((RexCall) rex).getOperands().get(2) instanceof RexLiteral) { - NamedFieldExpression fieldName = fieldExpressionCreator.create(((RexInputRef)((RexCall) rex).getOperands().get(0)).getIndex()); return BucketAggregationBuilder.buildHistogram( - project.getRowType().getFieldList().get(groupIndex).getName(), - fieldName.getReferenceForTermQuery(), + bucketName, + helper.inferNamedField(((RexCall) rex).getOperands().get(0)).getRootName(), ((RexLiteral)((RexCall) rex).getOperands().get(1)).getValueAs(Double.class), SpanUnit.of(((RexLiteral)((RexCall) rex).getOperands().get(2)).getValueAs(String.class)), MissingOrder.FIRST); } else { - throw new AggregateAnalyzer.AggregateAnalyzerException( - String.format("Unsupported group expression %s in project %s", rex, project)); + return createTermsSourceBuilder(bucketName, rex, helper); } } private static CompositeValuesSourceBuilder createTermsSourceBuilder( - NamedFieldExpression groupExpr) { - + String bucketName, RexNode group, AggregateAnalyzer.AggregateBuilderHelper helper) { CompositeValuesSourceBuilder sourceBuilder = - new TermsValuesSourceBuilder(groupExpr.getRootName()) - .missingBucket(true) - // TODO: use Sort's option if there is Sort push-down into aggregation - // https://github.com/opensearch-project/sql/issues/3380 - .missingOrder(MissingOrder.FIRST) - .order(SortOrder.ASC) - .field(groupExpr.getReferenceForTermQuery()); + helper.build( + group, + new TermsValuesSourceBuilder(bucketName) + .missingBucket(true) + .missingOrder(MissingOrder.FIRST) + .order(SortOrder.ASC)); // Time types values are converted to LONG in ExpressionAggregationScript::execute - if (List.of(TIMESTAMP, TIME, DATE).contains(groupExpr.getExprType())) { + if (List.of(TIMESTAMP, TIME, DATE) + .contains(OpenSearchTypeFactory.convertRelDataTypeToExprType(group.getType()))) { sourceBuilder.userValuetypeHint(ValueType.LONG); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java index d01af29eede..fffc9cef298 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java @@ -1365,18 +1365,21 @@ public ScriptQueryExpression( @Override public QueryBuilder builder() { + return new ScriptQueryBuilder(getScript()); + } + + public Script getScript() { long currentTime = Hook.CURRENT_TIME.get(-1L); if (currentTime < 0) { throw new UnsupportedScriptException( "ScriptQueryExpression requires a valid current time from hook, but it is not set"); } - return new ScriptQueryBuilder( - new Script( - DEFAULT_SCRIPT_TYPE, - COMPOUNDED_LANG_NAME, - code, - Collections.emptyMap(), - Map.of(Variable.UTC_TIMESTAMP.camelName, currentTime))); + return new Script( + DEFAULT_SCRIPT_TYPE, + COMPOUNDED_LANG_NAME, + code, + Collections.emptyMap(), + Map.of(Variable.UTC_TIMESTAMP.camelName, currentTime)); } @Override @@ -1393,138 +1396,6 @@ public void updateAnalyzedNodes(RexNode rexNode) { public List getUnAnalyzableNodes() { return List.of(); } - - @Override - public QueryExpression exists() { - throw new PredicateAnalyzerException( - "SqlOperatorImpl ['exists'] " + "cannot be applied to a script expression"); - } - - @Override - public QueryExpression contains(LiteralExpression literal) { - throw new PredicateAnalyzerException( - "SqlOperatorImpl ['contains'] " + "cannot be applied to a script expression"); - } - - @Override - public QueryExpression not() { - throw new PredicateAnalyzerException( - "SqlOperatorImpl ['not'] " + "cannot be applied to a script expression"); - } - - @Override - public QueryExpression notExists() { - throw new PredicateAnalyzerException( - "SqlOperatorImpl ['notExists'] " + "cannot be applied to a script expression"); - } - - @Override - public QueryExpression like(LiteralExpression literal) { - throw new PredicateAnalyzerException( - "SqlOperatorImpl ['like'] " + "cannot be applied to a script expression"); - } - - @Override - public QueryExpression notLike(LiteralExpression literal) { - throw new PredicateAnalyzerException( - "SqlOperatorImpl ['notLike'] " + "cannot be applied to a script expression"); - } - - @Override - public QueryExpression equals(LiteralExpression literal) { - throw new PredicateAnalyzerException( - "SqlOperatorImpl ['='] " + "cannot be applied to a script expression"); - } - - @Override - public QueryExpression notEquals(LiteralExpression literal) { - throw new PredicateAnalyzerException( - "SqlOperatorImpl ['not'] " + "cannot be applied to a script expression"); - } - - @Override - public QueryExpression gt(LiteralExpression literal) { - throw new PredicateAnalyzerException( - "SqlOperatorImpl ['>'] " + "cannot be applied to a script expression"); - } - - @Override - public QueryExpression gte(LiteralExpression literal) { - throw new PredicateAnalyzerException( - "SqlOperatorImpl ['>='] " + "cannot be applied to a script expression"); - } - - @Override - public QueryExpression lt(LiteralExpression literal) { - throw new PredicateAnalyzerException( - "SqlOperatorImpl ['<'] " + "cannot be applied to a script expression"); - } - - @Override - public QueryExpression lte(LiteralExpression literal) { - throw new PredicateAnalyzerException( - "SqlOperatorImpl ['<='] " + "cannot be applied to a script expression"); - } - - @Override - public QueryExpression match(String query, Map optionalArguments) { - throw new PredicateAnalyzerException( - "Match query " + "cannot be applied to a script expression"); - } - - @Override - public QueryExpression matchPhrase(String query, Map optionalArguments) { - throw new PredicateAnalyzerException( - "MatchPhrase query " + "cannot be applied to a script expression"); - } - - @Override - public QueryExpression matchBoolPrefix(String query, Map optionalArguments) { - throw new PredicateAnalyzerException( - "MatchBoolPrefix query " + "cannot be applied to a script expression"); - } - - @Override - public QueryExpression matchPhrasePrefix(String query, Map optionalArguments) { - throw new PredicateAnalyzerException( - "MatchPhrasePrefix query " + "cannot be applied to a script expression"); - } - - @Override - public QueryExpression simpleQueryString( - RexCall fieldsRexCall, String query, Map optionalArguments) { - throw new PredicateAnalyzerException( - "SimpleQueryString query " + "cannot be applied to a script expression"); - } - - @Override - public QueryExpression queryString( - RexCall fieldsRexCall, String query, Map optionalArguments) { - throw new PredicateAnalyzerException( - "QueryString query " + "cannot be applied to a script expression"); - } - - @Override - public QueryExpression multiMatch( - RexCall fieldsRexCall, String query, Map optionalArguments) { - throw new PredicateAnalyzerException( - "MultiMatch query " + "cannot be applied to a script expression"); - } - - @Override - public QueryExpression isTrue() { - throw new PredicateAnalyzerException("isTrue cannot be applied to a script expression"); - } - - @Override - public QueryExpression in(LiteralExpression literal) { - throw new PredicateAnalyzerException("in cannot be applied to a script expression"); - } - - @Override - public QueryExpression notIn(LiteralExpression literal) { - throw new PredicateAnalyzerException("notIn cannot be applied to a script expression"); - } } /** diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/AbstractCalciteIndexScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/AbstractCalciteIndexScan.java index a6361400e7e..68f477d834c 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/AbstractCalciteIndexScan.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/AbstractCalciteIndexScan.java @@ -10,7 +10,10 @@ import java.util.ArrayDeque; import java.util.ArrayList; +import java.util.Collections; import java.util.List; +import java.util.Map; +import java.util.stream.IntStream; import java.util.stream.Collectors; import java.util.stream.Stream; import lombok.Getter; @@ -21,6 +24,8 @@ import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.RelCollations; import org.apache.calcite.rel.RelFieldCollation; +import org.apache.calcite.rel.RelFieldCollation.Direction; +import org.apache.calcite.rel.RelFieldCollation.NullDirection; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.RelWriter; import org.apache.calcite.rel.core.TableScan; @@ -31,16 +36,25 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexNode; import org.apache.calcite.util.NumberUtil; +import org.apache.commons.lang3.tuple.Pair; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.opensearch.search.aggregations.AggregationBuilder; +import org.opensearch.search.aggregations.AggregationBuilders; +import org.opensearch.search.aggregations.AggregatorFactories.Builder; +import org.opensearch.search.aggregations.bucket.composite.CompositeAggregationBuilder; +import org.opensearch.search.aggregations.bucket.composite.CompositeValuesSourceBuilder; +import org.opensearch.search.aggregations.bucket.missing.MissingOrder; import org.opensearch.search.sort.ScoreSortBuilder; import org.opensearch.search.sort.SortBuilder; import org.opensearch.search.sort.SortBuilders; import org.opensearch.search.sort.SortOrder; import org.opensearch.sql.common.setting.Settings.Key; import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; import org.opensearch.sql.opensearch.data.type.OpenSearchTextType; import org.opensearch.sql.opensearch.request.OpenSearchRequestBuilder; +import org.opensearch.sql.opensearch.response.agg.OpenSearchAggregationResponseParser; import org.opensearch.sql.opensearch.storage.OpenSearchIndex; /** An abstract relational operator representing a scan of an OpenSearchIndex type. */ @@ -251,15 +265,23 @@ public AbstractCalciteIndexScan pushDownSort(List collations) // Existing collations are overridden (discarded) by the new collations, cloneWithoutSort(pushDownContext)); - List> builders = new ArrayList<>(); - for (RelFieldCollation collation : collations) { - int index = collation.getFieldIndex(); - String fieldName = this.getRowType().getFieldNames().get(index); - RelFieldCollation.Direction direction = collation.getDirection(); - RelFieldCollation.NullDirection nullDirection = collation.nullDirection; - // Default sort order is ASCENDING - SortOrder order = - RelFieldCollation.Direction.DESCENDING.equals(direction) + AbstractAction action; + Object digest; + if (pushDownContext.isAggregatePushed) { + // Push down the sort into the aggregation bucket + ((AggPushDownAction) requireNonNull(pushDownContext.peekLast()).action) + .pushDownSortIntoAggBucket(collations); + action = requestBuilder -> {}; + digest = collations; + } else { + List> builders = new ArrayList<>(); + for (RelFieldCollation collation : collations) { + int index = collation.getFieldIndex(); + String fieldName = this.getRowType().getFieldNames().get(index); + Direction direction = collation.getDirection(); + NullDirection nullDirection = collation.nullDirection; + // Default sort order is ASCENDING + SortOrder order = Direction.DESCENDING.equals(direction) ? SortOrder.DESC : SortOrder.ASC; // TODO: support script sort and distance sort @@ -271,7 +293,7 @@ public AbstractCalciteIndexScan pushDownSort(List collations) switch (nullDirection) { case FIRST: missing = "_first"; - break; + break; case LAST: missing = "_last"; break; @@ -279,18 +301,17 @@ public AbstractCalciteIndexScan pushDownSort(List collations) missing = null; break; } - // Keyword field is optimized for sorting in OpenSearch - ExprType fieldType = osIndex.getFieldTypes().get(fieldName); - String field = OpenSearchTextType.toKeywordSubField(fieldName, fieldType); - sortBuilder = SortBuilders.fieldSort(field).missing(missing); + // Keyword field is optimized for sorting in OpenSearch + ExprType fieldType = osIndex.getFieldTypes().get(fieldName); + String field = OpenSearchTextType.toKeywordSubField(fieldName, fieldType); + sortBuilder = SortBuilders.fieldSort(field).missing(missing); + } + builders.add(sortBuilder.order(order)); } - builders.add(sortBuilder.order(order)); + action = requestBuilder -> requestBuilder.pushDownSort(builders); + digest = builders.toString(); } - newScan.pushDownContext.add( - new PushDownAction( - PushDownType.SORT, - builders.toString(), - requestBuilder -> requestBuilder.pushDownSort(builders))); + newScan.pushDownContext.add(new PushDownAction(PushDownType.SORT, digest, action)); return newScan; } catch (Exception e) { if (LOG.isDebugEnabled()) { @@ -357,4 +378,68 @@ public AbstractAction getAction() { public interface AbstractAction { void apply(OpenSearchRequestBuilder requestBuilder); } + + public static class AggPushDownAction implements AbstractAction { + + private Pair, OpenSearchAggregationResponseParser> aggregationBuilder; + private final Map extendedTypeMapping; + + public AggPushDownAction( + Pair, OpenSearchAggregationResponseParser> aggregationBuilder, + Map extendedTypeMapping) { + this.aggregationBuilder = aggregationBuilder; + this.extendedTypeMapping = extendedTypeMapping; + } + + @Override + public void apply(OpenSearchRequestBuilder requestBuilder) { + requestBuilder.pushDownAggregation(aggregationBuilder); + requestBuilder.pushTypeMapping(extendedTypeMapping); + } + + public void pushDownSortIntoAggBucket(List collations) { + // It will always use a single CompositeAggregationBuilder for the aggregation with GroupBy + // See {@link AggregateAnalyzer} + CompositeAggregationBuilder compositeAggregationBuilder = + (CompositeAggregationBuilder) aggregationBuilder.getLeft().get(0); + List> buckets = + ((CompositeAggregationBuilder) aggregationBuilder.getLeft().get(0)).sources(); + List> newBuckets = new ArrayList<>(buckets.size()); + List selected = new ArrayList<>(collations.size()); + // Have to put the collation required buckets first, then the rest of buckets. + collations.forEach( + collation -> { + CompositeValuesSourceBuilder bucket = buckets.get(collation.getFieldIndex()); + Direction direction = collation.getDirection(); + NullDirection nullDirection = collation.nullDirection; + SortOrder order = + Direction.DESCENDING.equals(direction) ? SortOrder.DESC : SortOrder.ASC; + MissingOrder missingOrder; + switch (nullDirection) { + case FIRST: + missingOrder = MissingOrder.FIRST; + break; + case LAST: + missingOrder = MissingOrder.LAST; + break; + default: + missingOrder = MissingOrder.DEFAULT; + break; + } + newBuckets.add(bucket.order(order).missingOrder(missingOrder)); + selected.add(collation.getFieldIndex()); + }); + IntStream.range(0, buckets.size()) + .filter(i -> !selected.contains(i)) + .forEach(i -> newBuckets.add(buckets.get(i))); + Builder newAggBuilder = new Builder(); + compositeAggregationBuilder.getSubAggregations().forEach(newAggBuilder::addAggregator); + aggregationBuilder = + Pair.of( + Collections.singletonList( + AggregationBuilders.composite("composite_buckets", newBuckets) + .subAggregations(newAggBuilder)), + aggregationBuilder.getRight()); + } + } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteLogicalIndexScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteLogicalIndexScan.java index a03a782a040..a344a9b61c7 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteLogicalIndexScan.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteLogicalIndexScan.java @@ -227,11 +227,11 @@ public CalciteLogicalIndexScan pushDownAggregate(Aggregate aggregate, Project pr aggregate.getRowType(), // Aggregation will eliminate all collations. cloneWithoutSort(pushDownContext)); - List schema = this.getRowType().getFieldNames(); Map fieldTypes = this.osIndex.getFieldTypes(); List outputFields = aggregate.getRowType().getFieldNames(); final Pair, OpenSearchAggregationResponseParser> aggregationBuilder = - AggregateAnalyzer.analyze(aggregate, project, schema, fieldTypes, outputFields); + AggregateAnalyzer.analyze( + aggregate, project, getRowType(), fieldTypes, outputFields, getCluster()); Map extendedTypeMapping = aggregate.getRowType().getFieldList().stream() .collect( @@ -241,14 +241,8 @@ public CalciteLogicalIndexScan pushDownAggregate(Aggregate aggregate, Project pr OpenSearchDataType.of( OpenSearchTypeFactory.convertRelDataTypeToExprType( field.getType())))); - newScan.pushDownContext.add( - new PushDownAction( - PushDownType.AGGREGATION, - aggregate, - requestBuilder -> { - requestBuilder.pushDownAggregation(aggregationBuilder); - requestBuilder.pushTypeMapping(extendedTypeMapping); - })); + AggPushDownAction action = new AggPushDownAction(aggregationBuilder, extendedTypeMapping); + newScan.pushDownContext.add(new PushDownAction(PushDownType.AGGREGATION, aggregate, action)); return newScan; } catch (Exception e) { if (LOG.isDebugEnabled()) { diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/CalciteScriptEngine.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/CalciteScriptEngine.java index a62c4ec3c58..cebaf8c0418 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/CalciteScriptEngine.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/CalciteScriptEngine.java @@ -29,6 +29,7 @@ import static org.opensearch.sql.data.type.ExprCoreType.FLOAT; import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; +import static org.opensearch.sql.data.type.ExprCoreType.SHORT; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -37,7 +38,7 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.function.Function; +import java.util.function.BiFunction; import java.util.function.Supplier; import lombok.RequiredArgsConstructor; import org.apache.calcite.DataContext; @@ -76,6 +77,7 @@ import org.apache.commons.lang3.tuple.Pair; import org.checkerframework.checker.nullness.qual.Nullable; import org.opensearch.index.fielddata.ScriptDocValues; +import org.opensearch.script.AggregationScript; import org.opensearch.script.FilterScript; import org.opensearch.script.ScriptContext; import org.opensearch.script.ScriptEngine; @@ -83,6 +85,7 @@ import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.opensearch.request.PredicateAnalyzer.NamedFieldExpression; +import org.opensearch.sql.opensearch.storage.script.aggregation.CalciteAggregationScriptFactory; import org.opensearch.sql.opensearch.storage.script.filter.CalciteFilterScriptFactory; import org.opensearch.sql.opensearch.storage.serde.RelJsonSerializer; @@ -103,11 +106,14 @@ public CalciteScriptEngine(RelOptCluster relOptCluster) { public static final String EXPRESSION_LANG_NAME = "opensearch_calcite_expression"; /** All supported script contexts and function to create factory from expression. */ - private static final Map, Function, Object>> + private static final Map< + ScriptContext, BiFunction, RelDataType, Object>> CONTEXTS = new ImmutableMap.Builder< - ScriptContext, Function, Object>>() + ScriptContext, + BiFunction, RelDataType, Object>>() .put(FilterScript.CONTEXT, CalciteFilterScriptFactory::new) + .put(AggregationScript.CONTEXT, CalciteAggregationScriptFactory::new) .build(); @Override @@ -135,7 +141,7 @@ public T compile( new RexExecutable(code, "generated Rex code").getFunction(); if (CONTEXTS.containsKey(context)) { - return context.factoryClazz.cast(CONTEXTS.get(context).apply(function)); + return context.factoryClazz.cast(CONTEXTS.get(context).apply(function, rexNode.getType())); } throw new IllegalStateException( String.format( @@ -204,6 +210,7 @@ private Expression tryConvertDocValue(Expression docValueExpr, ExprType exprType ExprCoreType type = (ExprCoreType) exprType; switch (type) { case INTEGER: + case SHORT: docValue = EnumUtils.convert(docValueExpr, Long.class); break; case FLOAT: diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/CalciteAggregationScript.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/CalciteAggregationScript.java new file mode 100644 index 00000000000..5a4ea5158b1 --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/CalciteAggregationScript.java @@ -0,0 +1,68 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.script.aggregation; + +import static java.time.temporal.ChronoUnit.MILLIS; +import static org.opensearch.sql.data.type.ExprCoreType.DATE; +import static org.opensearch.sql.data.type.ExprCoreType.TIME; +import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP; + +import java.time.LocalTime; +import java.util.Map; +import lombok.EqualsAndHashCode; +import org.apache.calcite.DataContext; +import org.apache.calcite.linq4j.function.Function1; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.lucene.index.LeafReaderContext; +import org.opensearch.script.AggregationScript; +import org.opensearch.search.lookup.SearchLookup; +import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory; +import org.opensearch.sql.data.model.ExprValueUtils; +import org.opensearch.sql.data.type.ExprCoreType; +import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.opensearch.storage.script.core.CalciteScript; + +/** Calcite script executor that executes the generated code on each document for aggregation. */ +@EqualsAndHashCode(callSuper = false) +class CalciteAggregationScript extends AggregationScript { + + /** Calcite Script. */ + private final CalciteScript calciteScript; + + private final RelDataType type; + + public CalciteAggregationScript( + Function1 function, + RelDataType type, + SearchLookup lookup, + LeafReaderContext context, + Map params) { + super(params, lookup, context); + this.calciteScript = new CalciteScript(function, params); + this.type = type; + } + + @Override + public Object execute() { + Object value = calciteScript.execute(this::getDoc)[0]; + ExprType exprType = OpenSearchTypeFactory.convertRelDataTypeToExprType(type); + // See logic in {@link ExpressionAggregationScript::execute} + ExprCoreType coreType = (ExprCoreType) exprType; + switch (coreType) { + case TIME: + // Can't get timestamp from `ExprTimeValue` + return MILLIS.between(LocalTime.MIN, ExprValueUtils.fromObjectValue(value, TIME).timeValue()); + case DATE: + return ExprValueUtils.fromObjectValue(value, DATE).timestampValue().toEpochMilli(); + case TIMESTAMP: + return ExprValueUtils.fromObjectValue(value, TIMESTAMP) + .timestampValue() + .toEpochMilli(); + default: + return value; + } + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/CalciteAggregationScriptFactory.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/CalciteAggregationScriptFactory.java new file mode 100644 index 00000000000..0784dc48149 --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/CalciteAggregationScriptFactory.java @@ -0,0 +1,41 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.script.aggregation; + +import java.util.Map; +import lombok.EqualsAndHashCode; +import org.apache.calcite.DataContext; +import org.apache.calcite.linq4j.function.Function1; +import org.apache.calcite.rel.type.RelDataType; +import org.opensearch.script.AggregationScript; +import org.opensearch.search.lookup.SearchLookup; + +/** Calcite script factory that generates leaf factory. */ +@EqualsAndHashCode +public class CalciteAggregationScriptFactory implements AggregationScript.Factory { + + /** Generated code of calcite to execute. */ + private final Function1 function; + + private final RelDataType type; + + public CalciteAggregationScriptFactory( + Function1 function, RelDataType type) { + this.function = function; + this.type = type; + } + + @Override + public boolean isResultDeterministic() { + // This implies the results are cacheable + return true; + } + + @Override + public AggregationScript.LeafFactory newFactory(Map params, SearchLookup lookup) { + return new CalciteAggregationScriptLeafFactory(function, type, params, lookup); + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/CalciteAggregationScriptLeafFactory.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/CalciteAggregationScriptLeafFactory.java new file mode 100644 index 00000000000..8555b0a82a7 --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/CalciteAggregationScriptLeafFactory.java @@ -0,0 +1,48 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.script.aggregation; + +import java.util.Map; +import org.apache.calcite.DataContext; +import org.apache.calcite.linq4j.function.Function1; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.lucene.index.LeafReaderContext; +import org.opensearch.script.AggregationScript; +import org.opensearch.search.lookup.SearchLookup; + +/** Calcite script leaf factory that produces script executor for each leaf. */ +class CalciteAggregationScriptLeafFactory implements AggregationScript.LeafFactory { + + private final Function1 function; + private final RelDataType type; + + /** Parameters for the calcite script. */ + private final Map params; + + /** Document lookup that returns doc values. */ + private final SearchLookup lookup; + + public CalciteAggregationScriptLeafFactory( + Function1 function, + RelDataType type, + Map params, + SearchLookup lookup) { + this.function = function; + this.type = type; + this.params = params; + this.lookup = lookup; + } + + @Override + public AggregationScript newInstance(LeafReaderContext ctx) { + return new CalciteAggregationScript(function, type, lookup, ctx, params); + } + + @Override + public boolean needs_score() { + return false; + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/CalciteFilterScriptFactory.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/CalciteFilterScriptFactory.java index 351d886f42c..bf252b5fb93 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/CalciteFilterScriptFactory.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/CalciteFilterScriptFactory.java @@ -9,6 +9,7 @@ import lombok.EqualsAndHashCode; import org.apache.calcite.DataContext; import org.apache.calcite.linq4j.function.Function1; +import org.apache.calcite.rel.type.RelDataType; import org.opensearch.script.FilterScript; import org.opensearch.search.lookup.SearchLookup; @@ -19,7 +20,7 @@ public class CalciteFilterScriptFactory implements FilterScript.Factory { /** Generated code of calcite to execute. */ private final Function1 function; - public CalciteFilterScriptFactory(Function1 function) { + public CalciteFilterScriptFactory(Function1 function, RelDataType type) { this.function = function; } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/AggregateAnalyzerTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/AggregateAnalyzerTest.java index b14605e841c..470784d4142 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/AggregateAnalyzerTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/AggregateAnalyzerTest.java @@ -20,6 +20,7 @@ import org.apache.calcite.rel.core.Aggregate; import org.apache.calcite.rel.core.AggregateCall; import org.apache.calcite.rel.core.Project; +import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rel.type.RelDataTypeSystem; import org.apache.calcite.rex.RexInputRef; @@ -46,6 +47,13 @@ class AggregateAnalyzerTest { private final RelDataTypeFactory typeFactory = new SqlTypeFactoryImpl(RelDataTypeSystem.DEFAULT); private final List schema = List.of("a", "b", "c"); + private final RelDataType rowType = + typeFactory.createStructType( + ImmutableList.of( + typeFactory.createSqlType(SqlTypeName.INTEGER), + typeFactory.createSqlType(SqlTypeName.VARCHAR), + typeFactory.createSqlType(SqlTypeName.VARCHAR)), + schema); final Map fieldTypes = Map.of( "a", @@ -130,7 +138,7 @@ void analyze_aggCall_simple() throws ExpressionNotAnalyzableException { List.of(countCall, avgCall, sumCall, minCall, maxCall), ImmutableBitSet.of()); Project project = createMockProject(List.of(0)); Pair, OpenSearchAggregationResponseParser> result = - AggregateAnalyzer.analyze(aggregate, project, schema, fieldTypes, outputFields); + AggregateAnalyzer.analyze(aggregate, project, rowType, fieldTypes, outputFields, null); assertEquals( "[{\"cnt\":{\"value_count\":{\"field\":\"_index\"}}}," + " {\"avg\":{\"avg\":{\"field\":\"a\"}}}," @@ -211,7 +219,7 @@ void analyze_aggCall_extended() throws ExpressionNotAnalyzableException { List.of(varSampCall, varPopCall, stddevSampCall, stddevPopCall), ImmutableBitSet.of()); Project project = createMockProject(List.of(0)); Pair, OpenSearchAggregationResponseParser> result = - AggregateAnalyzer.analyze(aggregate, project, schema, fieldTypes, outputFields); + AggregateAnalyzer.analyze(aggregate, project, rowType, fieldTypes, outputFields, null); assertEquals( "[{\"var_samp\":{\"extended_stats\":{\"field\":\"a\",\"sigma\":2.0}}}," + " {\"var_pop\":{\"extended_stats\":{\"field\":\"a\",\"sigma\":2.0}}}," @@ -250,7 +258,7 @@ void analyze_groupBy() throws ExpressionNotAnalyzableException { Aggregate aggregate = createMockAggregate(List.of(aggCall), ImmutableBitSet.of(0, 1)); Project project = createMockProject(List.of(0, 1)); Pair, OpenSearchAggregationResponseParser> result = - AggregateAnalyzer.analyze(aggregate, project, schema, fieldTypes, outputFields); + AggregateAnalyzer.analyze(aggregate, project, rowType, fieldTypes, outputFields, null); assertEquals( "[{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[" @@ -292,7 +300,8 @@ void analyze_aggCall_TextWithoutKeyword() { assertThrows( ExpressionNotAnalyzableException.class, () -> - AggregateAnalyzer.analyze(aggregate, project, schema, fieldTypes, List.of("sum"))); + AggregateAnalyzer.analyze( + aggregate, project, rowType, fieldTypes, List.of("sum"), null)); assertEquals("[field] must not be null: [sum]", exception.getCause().getMessage()); } @@ -317,7 +326,9 @@ void analyze_groupBy_TextWithoutKeyword() { ExpressionNotAnalyzableException exception = assertThrows( ExpressionNotAnalyzableException.class, - () -> AggregateAnalyzer.analyze(aggregate, project, schema, fieldTypes, outputFields)); + () -> + AggregateAnalyzer.analyze( + aggregate, project, rowType, fieldTypes, outputFields, null)); assertEquals("[field] must not be null", exception.getCause().getMessage()); } @@ -334,9 +345,11 @@ private Project createMockProject(List refIndex) { for (Integer index : refIndex) { RexInputRef ref = mock(RexInputRef.class); when(ref.getIndex()).thenReturn(index); + when(ref.getType()).thenReturn(typeFactory.createSqlType(SqlTypeName.INTEGER)); rexNodes.add(ref); } when(project.getProjects()).thenReturn(rexNodes); + when(project.getRowType()).thenReturn(rowType); return project; } }