From 02fb46e4c05fd3fa21b3aabe660ed2488ef404a4 Mon Sep 17 00:00:00 2001 From: Yahor Yuzefovich Date: Tue, 28 Feb 2023 16:42:46 -0800 Subject: [PATCH] sql: introduce array_cat_agg aggregate builtin This commit introduces a new `array_cat_agg` aggregate builtin function that takes in an array type as its input, and then unnests each array and appends all its elements into a single result array. In other words, it behaves similar to `array_agg(unnest(array_column))`. This function doesn't have an analogue in Postgres. However, some of our SQL observability tools need this functionality, and the current workaround of using a LATERAL JOIN often results in slow apply joins, so this new builtin should speed things up significantly. In particular, `crdb_internal.statement_statistics` view is now refactored to use the new builtin which removes an apply join from it. The choice of this particular name comes from the fact that we have the `array_cat` builtin which concatenates two arrays. Release note (sql change): New aggregate builtin function `array_cat_agg` is introduced. It behaves similar to how `array_agg(unnest(array_column))` would - namely, it takes arrays as its input, unnests them into the array elements which are then aggregated into a single result array (i.e. it's similar to concatenating all input arrays into a single one). --- docs/generated/sql/aggregates.md | 44 ++++ pkg/sql/crdb_internal.go | 3 +- pkg/sql/distsql/columnar_operators_test.go | 4 +- pkg/sql/execinfrapb/aggregate_funcs.go | 1 + pkg/sql/execinfrapb/processors_sql.proto | 1 + pkg/sql/function_resolver_test.go | 2 +- .../logictest/testdata/logic_test/aggregate | 82 ++++++- .../testdata/logic_test/information_schema | 198 +++++++-------- .../logictest/testdata/logic_test/pg_catalog | 4 +- .../logictest/testdata/logic_test/pgoidtype | 4 +- .../opt/exec/execbuilder/testdata/aggregate | 178 ++++++++------ pkg/sql/opt/memo/testdata/memo | 228 +++++++++++------- pkg/sql/opt/memo/testdata/typing | 124 +++++----- pkg/sql/opt/memo/typing.go | 1 + pkg/sql/opt/norm/testdata/rules/agg | 16 +- pkg/sql/opt/operator.go | 15 +- pkg/sql/opt/ops/scalar.opt | 5 + pkg/sql/opt/optbuilder/groupby.go | 7 +- pkg/sql/sem/builtins/aggregate_builtins.go | 95 ++++++++ 19 files changed, 653 insertions(+), 359 deletions(-) diff --git a/docs/generated/sql/aggregates.md b/docs/generated/sql/aggregates.md index e92621be807..3488d264159 100644 --- a/docs/generated/sql/aggregates.md +++ b/docs/generated/sql/aggregates.md @@ -45,6 +45,50 @@ Immutable array_agg(arg1: varbit) → varbit[]

Aggregates the selected values into an array.

Immutable +array_cat_agg(arg1: bool[]) → bool[]

Unnests the selected arrays into elements that are then aggregated into a single array.

+Immutable +array_cat_agg(arg1: bytes[]) → bytes[]

Unnests the selected arrays into elements that are then aggregated into a single array.

+Immutable +array_cat_agg(arg1: date[]) → date[]

Unnests the selected arrays into elements that are then aggregated into a single array.

+Immutable +array_cat_agg(arg1: decimal[]) → decimal[]

Unnests the selected arrays into elements that are then aggregated into a single array.

+Immutable +array_cat_agg(arg1: float[]) → float[]

Unnests the selected arrays into elements that are then aggregated into a single array.

+Immutable +array_cat_agg(arg1: inet[]) → inet[]

Unnests the selected arrays into elements that are then aggregated into a single array.

+Immutable +array_cat_agg(arg1: int[]) → int[]

Unnests the selected arrays into elements that are then aggregated into a single array.

+Immutable +array_cat_agg(arg1: interval[]) → interval[]

Unnests the selected arrays into elements that are then aggregated into a single array.

+Immutable +array_cat_agg(arg1: string[]) → string[]

Unnests the selected arrays into elements that are then aggregated into a single array.

+Immutable +array_cat_agg(arg1: time[]) → time[]

Unnests the selected arrays into elements that are then aggregated into a single array.

+Immutable +array_cat_agg(arg1: timestamp[]) → timestamp[]

Unnests the selected arrays into elements that are then aggregated into a single array.

+Immutable +array_cat_agg(arg1: timestamptz[]) → timestamptz[]

Unnests the selected arrays into elements that are then aggregated into a single array.

+Immutable +array_cat_agg(arg1: uuid[]) → uuid[]

Unnests the selected arrays into elements that are then aggregated into a single array.

+Immutable +array_cat_agg(arg1: anyenum[]) → anyenum[]

Unnests the selected arrays into elements that are then aggregated into a single array.

+Immutable +array_cat_agg(arg1: box2d[]) → box2d[]

Unnests the selected arrays into elements that are then aggregated into a single array.

+Immutable +array_cat_agg(arg1: geography[]) → geography[]

Unnests the selected arrays into elements that are then aggregated into a single array.

+Immutable +array_cat_agg(arg1: geometry[]) → geometry[]

Unnests the selected arrays into elements that are then aggregated into a single array.

+Immutable +array_cat_agg(arg1: jsonb[]) → jsonb[]

Unnests the selected arrays into elements that are then aggregated into a single array.

+Immutable +array_cat_agg(arg1: oid[]) → oid[]

Unnests the selected arrays into elements that are then aggregated into a single array.

+Immutable +array_cat_agg(arg1: timetz[]) → timetz[]

Unnests the selected arrays into elements that are then aggregated into a single array.

+Immutable +array_cat_agg(arg1: tuple[]) → tuple[]

Unnests the selected arrays into elements that are then aggregated into a single array.

+Immutable +array_cat_agg(arg1: varbit[]) → varbit[]

Unnests the selected arrays into elements that are then aggregated into a single array.

+Immutable avg(arg1: decimal) → decimal

Calculates the average of the selected values.

Immutable avg(arg1: float) → float

Calculates the average of the selected values.

diff --git a/pkg/sql/crdb_internal.go b/pkg/sql/crdb_internal.go index 17e10ad6f2b..251c5481f7e 100644 --- a/pkg/sql/crdb_internal.go +++ b/pkg/sql/crdb_internal.go @@ -5723,7 +5723,7 @@ SELECT crdb_internal.merge_statement_stats(array_agg(DISTINCT statistics)), max(sampled_plan), aggregation_interval, - array_remove(array_agg(index_rec), NULL) AS index_recommendations + array_remove(array_cat_agg(index_recommendations), NULL) AS index_recommendations FROM ( SELECT aggregated_ts, @@ -5753,7 +5753,6 @@ FROM ( FROM system.statement_statistics ) -LEFT JOIN LATERAL unnest(index_recommendations) AS index_rec ON true GROUP BY aggregated_ts, fingerprint_id, diff --git a/pkg/sql/distsql/columnar_operators_test.go b/pkg/sql/distsql/columnar_operators_test.go index 13cd6bb1a3b..d255aa7ed98 100644 --- a/pkg/sql/distsql/columnar_operators_test.go +++ b/pkg/sql/distsql/columnar_operators_test.go @@ -103,6 +103,7 @@ var aggregateFuncToNumArguments = map[execinfrapb.AggregatorSpec_Func]int{ execinfrapb.FinalCovarSamp: 1, execinfrapb.FinalCorr: 1, execinfrapb.FinalSqrdiff: 3, + execinfrapb.ArrayCatAgg: 1, } // TestAggregateFuncToNumArguments ensures that all aggregate functions are @@ -261,7 +262,8 @@ func TestAggregatorAgainstProcessor(t *testing.T) { execinfrapb.StExtent, execinfrapb.StUnion, execinfrapb.StCollect, - execinfrapb.ArrayAgg: + execinfrapb.ArrayAgg, + execinfrapb.ArrayCatAgg: for _, typ := range aggFnInputTypes { if typ.Family() == types.TupleFamily || (typ.Family() == types.ArrayFamily && typ.ArrayContents().Family() == types.TupleFamily) { invalid = true diff --git a/pkg/sql/execinfrapb/aggregate_funcs.go b/pkg/sql/execinfrapb/aggregate_funcs.go index 866f75e5001..7068a7a335b 100644 --- a/pkg/sql/execinfrapb/aggregate_funcs.go +++ b/pkg/sql/execinfrapb/aggregate_funcs.go @@ -73,4 +73,5 @@ const ( FinalCovarSamp = AggregatorSpec_FINAL_COVAR_SAMP FinalCorr = AggregatorSpec_FINAL_CORR FinalSqrdiff = AggregatorSpec_FINAL_SQRDIFF + ArrayCatAgg = AggregatorSpec_ARRAY_CAT_AGG ) diff --git a/pkg/sql/execinfrapb/processors_sql.proto b/pkg/sql/execinfrapb/processors_sql.proto index caa87aa4737..8dee2c8c077 100644 --- a/pkg/sql/execinfrapb/processors_sql.proto +++ b/pkg/sql/execinfrapb/processors_sql.proto @@ -852,6 +852,7 @@ message AggregatorSpec { FINAL_COVAR_SAMP = 58; FINAL_CORR = 59; FINAL_SQRDIFF = 60; + ARRAY_CAT_AGG = 61; } enum Type { diff --git a/pkg/sql/function_resolver_test.go b/pkg/sql/function_resolver_test.go index 29705d1a831..5e1bbee2e2b 100644 --- a/pkg/sql/function_resolver_test.go +++ b/pkg/sql/function_resolver_test.go @@ -367,7 +367,7 @@ CREATE FUNCTION sc1.lower(a STRING) RETURNS STRING IMMUTABLE LANGUAGE SQL AS $$ exprStr: "lower('HI')", searchPath: []string{"sc1", "sc2"}, expectedFuncBody: "", - expectedFuncOID: 831, + expectedFuncOID: 853, desiredType: types.String, }, { diff --git a/pkg/sql/logictest/testdata/logic_test/aggregate b/pkg/sql/logictest/testdata/logic_test/aggregate index 67dcef8a815..cc02ab11907 100644 --- a/pkg/sql/logictest/testdata/logic_test/aggregate +++ b/pkg/sql/logictest/testdata/logic_test/aggregate @@ -2,11 +2,12 @@ subtest other statement ok CREATE TABLE kv ( - k INT PRIMARY KEY, - v INT, - w INT, - s STRING, - i INTERVAL + k INT PRIMARY KEY, + v INT, + w INT, + s STRING, + i INTERVAL, + arr INT[] ) # Aggregate functions return NULL if there are no rows. @@ -31,6 +32,11 @@ SELECT array_agg(1) FROM kv ---- NULL +query T +SELECT array_cat_agg('{1}'::INT[]) FROM kv +---- +NULL + query T SELECT json_agg(1) FROM kv ---- @@ -60,6 +66,11 @@ SELECT array_agg(v) FROM kv ---- NULL +query T +SELECT array_cat_agg(arr) FROM kv +---- +NULL + query T SELECT json_agg(v) FROM kv ---- @@ -85,6 +96,17 @@ SELECT array_agg(1) ---- {1} +query T +SELECT array_cat_agg('{1}'::INT[]) +---- +{1} + +# Array-concatenating empty arrays results in an empty array. +query T +SELECT array_cat_agg(arr) FROM [SELECT ('{}'::INT[]) AS arr FROM generate_series(1, 3)]; +---- +{} + query T SELECT json_agg(1) ---- @@ -164,12 +186,12 @@ SELECT (SELECT COALESCE(max(1), 0) FROM generate_series(1,0)) statement OK INSERT INTO kv VALUES -(1, 2, 3, 'a', '1min'), -(3, 4, 5, 'a', '2sec'), -(5, NULL, 5, NULL, NULL), -(6, 2, 3, 'b', '1ms'), -(7, 2, 2, 'b', '4 days'), -(8, 4, 2, 'A', '3 years') +(1, 2, 3, 'a', '1min', '{1, 2, NULL}'), +(3, 4, 5, 'a', '2sec', '{3, 4, 5}'), +(5, NULL, 5, NULL, NULL, NULL), +(6, 2, 3, 'b', '1ms', '{6, 2, 3}'), +(7, 2, 2, 'b', '4 days', '{7, 2, 2}'), +(8, 4, 2, 'A', '3 years', '{NULL, 4, 2}') # Aggregate functions triggers aggregation and computation for every row even when applied to a constant. # NB: The XOR result is 00 because \x01 is XOR'd an even number of times. @@ -186,6 +208,11 @@ SELECT array_agg(1) FROM kv ---- {1,1,1,1,1,1} +query T +SELECT array_cat_agg('{1, 2}'::INT[]) FROM kv +---- +{1,2,1,2,1,2,1,2,1,2,1,2} + query T SELECT json_agg(1) FROM kv ---- @@ -594,11 +621,26 @@ SELECT array_agg(k) || 1 FROM (SELECT k FROM kv ORDER BY k) ---- {1,3,5,6,7,8,1} +query T +SELECT array_cat_agg(arr) FROM (SELECT arr FROM kv ORDER BY k) +---- +{1,2,NULL,3,4,5,6,2,3,7,2,2,NULL,4,2} + query T SELECT array_agg(s) FROM kv WHERE s IS NULL ---- {NULL} +query T +SELECT array_cat_agg(arr) FROM kv WHERE arr IS NULL +---- +NULL + +query TTT +SELECT array_cat_agg(arr ORDER BY k), array_cat_agg(NULL::INT[]), array_cat_agg('{NULL, NULL}'::INT[]) FROM kv WHERE arr IS NOT NULL +---- +{1,2,NULL,3,4,5,6,2,3,7,2,2,NULL,4,2} NULL {NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL} + query T SELECT json_agg(s) FROM kv WHERE s IS NULL ---- @@ -2866,7 +2908,8 @@ statement ok CREATE TABLE tab ( col1 INT PRIMARY KEY, col2 INT, - col3 STRING + col3 STRING, + arr INT[] ) # Ordered aggregations when there are no rows. @@ -2876,7 +2919,8 @@ SELECT array_agg(col1 ORDER BY col2) FROM TAB NULL statement ok -INSERT INTO tab VALUES (-3, 7, 'a'), (-2, 6, 'a'), (-1, 5, 'a'), (0, 7, 'b'), (1, 5, 'b'), (2, 6, 'b') +INSERT INTO tab VALUES (-3, 7, 'a', '{-3, 7}'), (-2, 6, 'a', '{-2, 6}'), (-1, 5, 'a', '{-1, 5}'), + (0, 7, 'b', '{0, 7}'), (1, 5, 'b', '{1, 5}'), (2, 6, 'b', '{2, 6}') query T colnames SELECT array_agg(col1 ORDER BY col1) FROM tab @@ -2884,12 +2928,24 @@ SELECT array_agg(col1 ORDER BY col1) FROM tab array_agg {-3,-2,-1,0,1,2} +query T colnames +SELECT array_cat_agg(arr ORDER BY col1) FROM tab +---- +array_cat_agg +{-3,7,-2,6,-1,5,0,7,1,5,2,6} + query T colnames SELECT array_agg(col1 ORDER BY col2*100+col1) FROM tab ---- array_agg {-1,1,-2,2,-3,0} +query T colnames +SELECT array_cat_agg(arr ORDER BY col2*100+col1) FROM tab +---- +array_cat_agg +{-1,5,1,5,-2,6,2,6,-3,7,0,7} + query T colnames SELECT json_agg(col1 ORDER BY col1) FROM tab ---- diff --git a/pkg/sql/logictest/testdata/logic_test/information_schema b/pkg/sql/logictest/testdata/logic_test/information_schema index 10b0105300d..2e708e7840f 100644 --- a/pkg/sql/logictest/testdata/logic_test/information_schema +++ b/pkg/sql/logictest/testdata/logic_test/information_schema @@ -5170,102 +5170,102 @@ SELECT * FROM information_schema.role_routine_grants WHERE reverse(split_part(reverse(specific_name), '_', 1))::INT < 50 ORDER BY specific_name, grantee; ---- -grantor grantee specific_catalog specific_schema specific_name routine_catalog routine_schema routine_name privilege_type is_grantable -NULL public test pg_catalog array_agg_1 test pg_catalog array_agg EXECUTE NO -NULL root test pg_catalog array_agg_1 test pg_catalog array_agg EXECUTE NO -NULL public test pg_catalog array_agg_10 test pg_catalog array_agg EXECUTE NO -NULL root test pg_catalog array_agg_10 test pg_catalog array_agg EXECUTE NO -NULL public test pg_catalog array_agg_11 test pg_catalog array_agg EXECUTE NO -NULL root test pg_catalog array_agg_11 test pg_catalog array_agg EXECUTE NO -NULL public test pg_catalog array_agg_12 test pg_catalog array_agg EXECUTE NO -NULL root test pg_catalog array_agg_12 test pg_catalog array_agg EXECUTE NO -NULL public test pg_catalog array_agg_13 test pg_catalog array_agg EXECUTE NO -NULL root test pg_catalog array_agg_13 test pg_catalog array_agg EXECUTE NO -NULL public test pg_catalog array_agg_14 test pg_catalog array_agg EXECUTE NO -NULL root test pg_catalog array_agg_14 test pg_catalog array_agg EXECUTE NO -NULL public test pg_catalog array_agg_15 test pg_catalog array_agg EXECUTE NO -NULL root test pg_catalog array_agg_15 test pg_catalog array_agg EXECUTE NO -NULL public test pg_catalog array_agg_16 test pg_catalog array_agg EXECUTE NO -NULL root test pg_catalog array_agg_16 test pg_catalog array_agg EXECUTE NO -NULL public test pg_catalog array_agg_17 test pg_catalog array_agg EXECUTE NO -NULL root test pg_catalog array_agg_17 test pg_catalog array_agg EXECUTE NO -NULL public test pg_catalog array_agg_18 test pg_catalog array_agg EXECUTE NO -NULL root test pg_catalog array_agg_18 test pg_catalog array_agg EXECUTE NO -NULL public test pg_catalog array_agg_19 test pg_catalog array_agg EXECUTE NO -NULL root test pg_catalog array_agg_19 test pg_catalog array_agg EXECUTE NO -NULL public test pg_catalog array_agg_2 test pg_catalog array_agg EXECUTE NO -NULL root test pg_catalog array_agg_2 test pg_catalog array_agg EXECUTE NO -NULL public test pg_catalog array_agg_20 test pg_catalog array_agg EXECUTE NO -NULL root test pg_catalog array_agg_20 test pg_catalog array_agg EXECUTE NO -NULL public test pg_catalog array_agg_21 test pg_catalog array_agg EXECUTE NO -NULL root test pg_catalog array_agg_21 test pg_catalog array_agg EXECUTE NO -NULL public test pg_catalog array_agg_22 test pg_catalog array_agg EXECUTE NO -NULL root test pg_catalog array_agg_22 test pg_catalog array_agg EXECUTE NO -NULL public test pg_catalog array_agg_3 test pg_catalog array_agg EXECUTE NO -NULL root test pg_catalog array_agg_3 test pg_catalog array_agg EXECUTE NO -NULL public test pg_catalog array_agg_4 test pg_catalog array_agg EXECUTE NO -NULL root test pg_catalog array_agg_4 test pg_catalog array_agg EXECUTE NO -NULL public test pg_catalog array_agg_5 test pg_catalog array_agg EXECUTE NO -NULL root test pg_catalog array_agg_5 test pg_catalog array_agg EXECUTE NO -NULL public test pg_catalog array_agg_6 test pg_catalog array_agg EXECUTE NO -NULL root test pg_catalog array_agg_6 test pg_catalog array_agg EXECUTE NO -NULL public test pg_catalog array_agg_7 test pg_catalog array_agg EXECUTE NO -NULL root test pg_catalog array_agg_7 test pg_catalog array_agg EXECUTE NO -NULL public test pg_catalog array_agg_8 test pg_catalog array_agg EXECUTE NO -NULL root test pg_catalog array_agg_8 test pg_catalog array_agg EXECUTE NO -NULL public test pg_catalog array_agg_9 test pg_catalog array_agg EXECUTE NO -NULL root test pg_catalog array_agg_9 test pg_catalog array_agg EXECUTE NO -NULL public test pg_catalog avg_23 test pg_catalog avg EXECUTE NO -NULL root test pg_catalog avg_23 test pg_catalog avg EXECUTE NO -NULL public test pg_catalog avg_24 test pg_catalog avg EXECUTE NO -NULL root test pg_catalog avg_24 test pg_catalog avg EXECUTE NO -NULL public test pg_catalog avg_25 test pg_catalog avg EXECUTE NO -NULL root test pg_catalog avg_25 test pg_catalog avg EXECUTE NO -NULL public test pg_catalog avg_26 test pg_catalog avg EXECUTE NO -NULL root test pg_catalog avg_26 test pg_catalog avg EXECUTE NO -NULL public test pg_catalog bit_and_27 test pg_catalog bit_and EXECUTE NO -NULL root test pg_catalog bit_and_27 test pg_catalog bit_and EXECUTE NO -NULL public test pg_catalog bit_and_28 test pg_catalog bit_and EXECUTE NO -NULL root test pg_catalog bit_and_28 test pg_catalog bit_and EXECUTE NO -NULL public test pg_catalog bit_or_29 test pg_catalog bit_or EXECUTE NO -NULL root test pg_catalog bit_or_29 test pg_catalog bit_or EXECUTE NO -NULL public test pg_catalog bit_or_30 test pg_catalog bit_or EXECUTE NO -NULL root test pg_catalog bit_or_30 test pg_catalog bit_or EXECUTE NO -NULL public test pg_catalog bool_and_31 test pg_catalog bool_and EXECUTE NO -NULL root test pg_catalog bool_and_31 test pg_catalog bool_and EXECUTE NO -NULL public test pg_catalog bool_or_32 test pg_catalog bool_or EXECUTE NO -NULL root test pg_catalog bool_or_32 test pg_catalog bool_or EXECUTE NO -NULL public test pg_catalog concat_agg_33 test pg_catalog concat_agg EXECUTE NO -NULL root test pg_catalog concat_agg_33 test pg_catalog concat_agg EXECUTE NO -NULL public test pg_catalog concat_agg_34 test pg_catalog concat_agg EXECUTE NO -NULL root test pg_catalog concat_agg_34 test pg_catalog concat_agg EXECUTE NO -NULL public test pg_catalog corr_35 test pg_catalog corr EXECUTE NO -NULL root test pg_catalog corr_35 test pg_catalog corr EXECUTE NO -NULL public test pg_catalog corr_36 test pg_catalog corr EXECUTE NO -NULL root test pg_catalog corr_36 test pg_catalog corr EXECUTE NO -NULL public test pg_catalog corr_37 test pg_catalog corr EXECUTE NO -NULL root test pg_catalog corr_37 test pg_catalog corr EXECUTE NO -NULL public test pg_catalog corr_38 test pg_catalog corr EXECUTE NO -NULL root test pg_catalog corr_38 test pg_catalog corr EXECUTE NO -NULL public test pg_catalog corr_39 test pg_catalog corr EXECUTE NO -NULL root test pg_catalog corr_39 test pg_catalog corr EXECUTE NO -NULL public test pg_catalog corr_40 test pg_catalog corr EXECUTE NO -NULL root test pg_catalog corr_40 test pg_catalog corr EXECUTE NO -NULL public test pg_catalog corr_41 test pg_catalog corr EXECUTE NO -NULL root test pg_catalog corr_41 test pg_catalog corr EXECUTE NO -NULL public test pg_catalog corr_42 test pg_catalog corr EXECUTE NO -NULL root test pg_catalog corr_42 test pg_catalog corr EXECUTE NO -NULL public test pg_catalog corr_43 test pg_catalog corr EXECUTE NO -NULL root test pg_catalog corr_43 test pg_catalog corr EXECUTE NO -NULL public test pg_catalog covar_pop_44 test pg_catalog covar_pop EXECUTE NO -NULL root test pg_catalog covar_pop_44 test pg_catalog covar_pop EXECUTE NO -NULL public test pg_catalog covar_pop_45 test pg_catalog covar_pop EXECUTE NO -NULL root test pg_catalog covar_pop_45 test pg_catalog covar_pop EXECUTE NO -NULL public test pg_catalog covar_pop_46 test pg_catalog covar_pop EXECUTE NO -NULL root test pg_catalog covar_pop_46 test pg_catalog covar_pop EXECUTE NO -NULL public test pg_catalog covar_pop_47 test pg_catalog covar_pop EXECUTE NO -NULL root test pg_catalog covar_pop_47 test pg_catalog covar_pop EXECUTE NO -NULL public test pg_catalog covar_pop_48 test pg_catalog covar_pop EXECUTE NO -NULL root test pg_catalog covar_pop_48 test pg_catalog covar_pop EXECUTE NO -NULL public test pg_catalog covar_pop_49 test pg_catalog covar_pop EXECUTE NO -NULL root test pg_catalog covar_pop_49 test pg_catalog covar_pop EXECUTE NO +grantor grantee specific_catalog specific_schema specific_name routine_catalog routine_schema routine_name privilege_type is_grantable +NULL public test pg_catalog array_agg_1 test pg_catalog array_agg EXECUTE NO +NULL root test pg_catalog array_agg_1 test pg_catalog array_agg EXECUTE NO +NULL public test pg_catalog array_agg_10 test pg_catalog array_agg EXECUTE NO +NULL root test pg_catalog array_agg_10 test pg_catalog array_agg EXECUTE NO +NULL public test pg_catalog array_agg_11 test pg_catalog array_agg EXECUTE NO +NULL root test pg_catalog array_agg_11 test pg_catalog array_agg EXECUTE NO +NULL public test pg_catalog array_agg_12 test pg_catalog array_agg EXECUTE NO +NULL root test pg_catalog array_agg_12 test pg_catalog array_agg EXECUTE NO +NULL public test pg_catalog array_agg_13 test pg_catalog array_agg EXECUTE NO +NULL root test pg_catalog array_agg_13 test pg_catalog array_agg EXECUTE NO +NULL public test pg_catalog array_agg_14 test pg_catalog array_agg EXECUTE NO +NULL root test pg_catalog array_agg_14 test pg_catalog array_agg EXECUTE NO +NULL public test pg_catalog array_agg_15 test pg_catalog array_agg EXECUTE NO +NULL root test pg_catalog array_agg_15 test pg_catalog array_agg EXECUTE NO +NULL public test pg_catalog array_agg_16 test pg_catalog array_agg EXECUTE NO +NULL root test pg_catalog array_agg_16 test pg_catalog array_agg EXECUTE NO +NULL public test pg_catalog array_agg_17 test pg_catalog array_agg EXECUTE NO +NULL root test pg_catalog array_agg_17 test pg_catalog array_agg EXECUTE NO +NULL public test pg_catalog array_agg_18 test pg_catalog array_agg EXECUTE NO +NULL root test pg_catalog array_agg_18 test pg_catalog array_agg EXECUTE NO +NULL public test pg_catalog array_agg_19 test pg_catalog array_agg EXECUTE NO +NULL root test pg_catalog array_agg_19 test pg_catalog array_agg EXECUTE NO +NULL public test pg_catalog array_agg_2 test pg_catalog array_agg EXECUTE NO +NULL root test pg_catalog array_agg_2 test pg_catalog array_agg EXECUTE NO +NULL public test pg_catalog array_agg_20 test pg_catalog array_agg EXECUTE NO +NULL root test pg_catalog array_agg_20 test pg_catalog array_agg EXECUTE NO +NULL public test pg_catalog array_agg_21 test pg_catalog array_agg EXECUTE NO +NULL root test pg_catalog array_agg_21 test pg_catalog array_agg EXECUTE NO +NULL public test pg_catalog array_agg_22 test pg_catalog array_agg EXECUTE NO +NULL root test pg_catalog array_agg_22 test pg_catalog array_agg EXECUTE NO +NULL public test pg_catalog array_agg_3 test pg_catalog array_agg EXECUTE NO +NULL root test pg_catalog array_agg_3 test pg_catalog array_agg EXECUTE NO +NULL public test pg_catalog array_agg_4 test pg_catalog array_agg EXECUTE NO +NULL root test pg_catalog array_agg_4 test pg_catalog array_agg EXECUTE NO +NULL public test pg_catalog array_agg_5 test pg_catalog array_agg EXECUTE NO +NULL root test pg_catalog array_agg_5 test pg_catalog array_agg EXECUTE NO +NULL public test pg_catalog array_agg_6 test pg_catalog array_agg EXECUTE NO +NULL root test pg_catalog array_agg_6 test pg_catalog array_agg EXECUTE NO +NULL public test pg_catalog array_agg_7 test pg_catalog array_agg EXECUTE NO +NULL root test pg_catalog array_agg_7 test pg_catalog array_agg EXECUTE NO +NULL public test pg_catalog array_agg_8 test pg_catalog array_agg EXECUTE NO +NULL root test pg_catalog array_agg_8 test pg_catalog array_agg EXECUTE NO +NULL public test pg_catalog array_agg_9 test pg_catalog array_agg EXECUTE NO +NULL root test pg_catalog array_agg_9 test pg_catalog array_agg EXECUTE NO +NULL public test pg_catalog array_cat_agg_23 test pg_catalog array_cat_agg EXECUTE NO +NULL root test pg_catalog array_cat_agg_23 test pg_catalog array_cat_agg EXECUTE NO +NULL public test pg_catalog array_cat_agg_24 test pg_catalog array_cat_agg EXECUTE NO +NULL root test pg_catalog array_cat_agg_24 test pg_catalog array_cat_agg EXECUTE NO +NULL public test pg_catalog array_cat_agg_25 test pg_catalog array_cat_agg EXECUTE NO +NULL root test pg_catalog array_cat_agg_25 test pg_catalog array_cat_agg EXECUTE NO +NULL public test pg_catalog array_cat_agg_26 test pg_catalog array_cat_agg EXECUTE NO +NULL root test pg_catalog array_cat_agg_26 test pg_catalog array_cat_agg EXECUTE NO +NULL public test pg_catalog array_cat_agg_27 test pg_catalog array_cat_agg EXECUTE NO +NULL root test pg_catalog array_cat_agg_27 test pg_catalog array_cat_agg EXECUTE NO +NULL public test pg_catalog array_cat_agg_28 test pg_catalog array_cat_agg EXECUTE NO +NULL root test pg_catalog array_cat_agg_28 test pg_catalog array_cat_agg EXECUTE NO +NULL public test pg_catalog array_cat_agg_29 test pg_catalog array_cat_agg EXECUTE NO +NULL root test pg_catalog array_cat_agg_29 test pg_catalog array_cat_agg EXECUTE NO +NULL public test pg_catalog array_cat_agg_30 test pg_catalog array_cat_agg EXECUTE NO +NULL root test pg_catalog array_cat_agg_30 test pg_catalog array_cat_agg EXECUTE NO +NULL public test pg_catalog array_cat_agg_31 test pg_catalog array_cat_agg EXECUTE NO +NULL root test pg_catalog array_cat_agg_31 test pg_catalog array_cat_agg EXECUTE NO +NULL public test pg_catalog array_cat_agg_32 test pg_catalog array_cat_agg EXECUTE NO +NULL root test pg_catalog array_cat_agg_32 test pg_catalog array_cat_agg EXECUTE NO +NULL public test pg_catalog array_cat_agg_33 test pg_catalog array_cat_agg EXECUTE NO +NULL root test pg_catalog array_cat_agg_33 test pg_catalog array_cat_agg EXECUTE NO +NULL public test pg_catalog array_cat_agg_34 test pg_catalog array_cat_agg EXECUTE NO +NULL root test pg_catalog array_cat_agg_34 test pg_catalog array_cat_agg EXECUTE NO +NULL public test pg_catalog array_cat_agg_35 test pg_catalog array_cat_agg EXECUTE NO +NULL root test pg_catalog array_cat_agg_35 test pg_catalog array_cat_agg EXECUTE NO +NULL public test pg_catalog array_cat_agg_36 test pg_catalog array_cat_agg EXECUTE NO +NULL root test pg_catalog array_cat_agg_36 test pg_catalog array_cat_agg EXECUTE NO +NULL public test pg_catalog array_cat_agg_37 test pg_catalog array_cat_agg EXECUTE NO +NULL root test pg_catalog array_cat_agg_37 test pg_catalog array_cat_agg EXECUTE NO +NULL public test pg_catalog array_cat_agg_38 test pg_catalog array_cat_agg EXECUTE NO +NULL root test pg_catalog array_cat_agg_38 test pg_catalog array_cat_agg EXECUTE NO +NULL public test pg_catalog array_cat_agg_39 test pg_catalog array_cat_agg EXECUTE NO +NULL root test pg_catalog array_cat_agg_39 test pg_catalog array_cat_agg EXECUTE NO +NULL public test pg_catalog array_cat_agg_40 test pg_catalog array_cat_agg EXECUTE NO +NULL root test pg_catalog array_cat_agg_40 test pg_catalog array_cat_agg EXECUTE NO +NULL public test pg_catalog array_cat_agg_41 test pg_catalog array_cat_agg EXECUTE NO +NULL root test pg_catalog array_cat_agg_41 test pg_catalog array_cat_agg EXECUTE NO +NULL public test pg_catalog array_cat_agg_42 test pg_catalog array_cat_agg EXECUTE NO +NULL root test pg_catalog array_cat_agg_42 test pg_catalog array_cat_agg EXECUTE NO +NULL public test pg_catalog array_cat_agg_43 test pg_catalog array_cat_agg EXECUTE NO +NULL root test pg_catalog array_cat_agg_43 test pg_catalog array_cat_agg EXECUTE NO +NULL public test pg_catalog array_cat_agg_44 test pg_catalog array_cat_agg EXECUTE NO +NULL root test pg_catalog array_cat_agg_44 test pg_catalog array_cat_agg EXECUTE NO +NULL public test pg_catalog avg_45 test pg_catalog avg EXECUTE NO +NULL root test pg_catalog avg_45 test pg_catalog avg EXECUTE NO +NULL public test pg_catalog avg_46 test pg_catalog avg EXECUTE NO +NULL root test pg_catalog avg_46 test pg_catalog avg EXECUTE NO +NULL public test pg_catalog avg_47 test pg_catalog avg EXECUTE NO +NULL root test pg_catalog avg_47 test pg_catalog avg EXECUTE NO +NULL public test pg_catalog avg_48 test pg_catalog avg EXECUTE NO +NULL root test pg_catalog avg_48 test pg_catalog avg EXECUTE NO +NULL public test pg_catalog bit_and_49 test pg_catalog bit_and EXECUTE NO +NULL root test pg_catalog bit_and_49 test pg_catalog bit_and EXECUTE NO diff --git a/pkg/sql/logictest/testdata/logic_test/pg_catalog b/pkg/sql/logictest/testdata/logic_test/pg_catalog index 77b1e6e8f3e..ef14d704999 100644 --- a/pkg/sql/logictest/testdata/logic_test/pg_catalog +++ b/pkg/sql/logictest/testdata/logic_test/pg_catalog @@ -3270,7 +3270,7 @@ FROM pg_proc p JOIN pg_type t ON t.typinput = p.oid WHERE t.typname = '_int4' ---- -2024 array_in array_in +2046 array_in array_in ## #16285 ## int2vectors should be 0-indexed @@ -3306,7 +3306,7 @@ SELECT cur_max_builtin_oid FROM [SELECT max(oid) as cur_max_builtin_oid FROM pg_ query TT SELECT proname, oid FROM pg_catalog.pg_proc WHERE oid = $cur_max_builtin_oid ---- -to_regtype 2044 +to_regtype 2066 ## Ensure that unnest works with oid wrapper arrays diff --git a/pkg/sql/logictest/testdata/logic_test/pgoidtype b/pkg/sql/logictest/testdata/logic_test/pgoidtype index 717436012d0..d5766f981ce 100644 --- a/pkg/sql/logictest/testdata/logic_test/pgoidtype +++ b/pkg/sql/logictest/testdata/logic_test/pgoidtype @@ -92,7 +92,7 @@ WHERE relname = 'pg_constraint' query OOOO SELECT 'upper'::REGPROC, 'upper(string)'::REGPROCEDURE, 'pg_catalog.upper(string)'::REGPROCEDURE, 'upper'::REGPROC::OID ---- -upper upper upper 833 +upper upper upper 855 query error pq: invalid function signature: invalid.more.pg_catalog.upper: at or near ".": syntax error SELECT 'invalid.more.pg_catalog.upper'::REGPROCEDURE @@ -100,7 +100,7 @@ SELECT 'invalid.more.pg_catalog.upper'::REGPROCEDURE query OOO SELECT 'upper'::REGPROC, 'upper(string)'::REGPROCEDURE, 'upper'::REGPROC::OID ---- -upper upper 833 +upper upper 855 query error pq: unknown function: blah\(ignored\)\(\): function undefined SELECT 'blah(ignored)'::REGPROC diff --git a/pkg/sql/opt/exec/execbuilder/testdata/aggregate b/pkg/sql/opt/exec/execbuilder/testdata/aggregate index 43baa0ced1a..95ce12571bb 100644 --- a/pkg/sql/opt/exec/execbuilder/testdata/aggregate +++ b/pkg/sql/opt/exec/execbuilder/testdata/aggregate @@ -2,10 +2,11 @@ statement ok CREATE TABLE kv ( - k INT PRIMARY KEY, - v INT, - w INT, - s STRING + k INT PRIMARY KEY, + v INT, + w INT, + s STRING, + arr STRING[] ) query T @@ -21,38 +22,38 @@ vectorized: true • group (scalar) │ columns: (min int, max int, count int, sum_int int, avg decimal, sum decimal, stddev decimal, variance decimal, bool_and bool, bool_or bool, xor_agg bytes, corr float, covar_pop float, covar_samp float, sqrdiff decimal, regr_intercept float, regr_r2 float, regr_slope float, regr_sxx float, regr_sxy float, regr_syy float, regr_count int, regr_avgx float, regr_avgy float) │ estimated row count: 1 (missing stats) -│ aggregate 0: min(column7) -│ aggregate 1: max(column7) -│ aggregate 2: count(column10) -│ aggregate 3: sum_int(column7) -│ aggregate 4: avg(column7) -│ aggregate 5: sum(column7) -│ aggregate 6: stddev(column7) -│ aggregate 7: variance(column7) -│ aggregate 8: bool_and(column17) -│ aggregate 9: bool_or(column19) -│ aggregate 10: xor_agg(column21) -│ aggregate 11: corr(column7, column7) -│ aggregate 12: covar_pop(column7, column7) -│ aggregate 13: covar_samp(column7, column7) -│ aggregate 14: sqrdiff(column7) -│ aggregate 15: regr_intercept(column7, column7) -│ aggregate 16: regr_r2(column7, column7) -│ aggregate 17: regr_slope(column7, column7) -│ aggregate 18: regr_sxx(column7, column7) -│ aggregate 19: regr_sxy(column7, column7) -│ aggregate 20: regr_syy(column7, column7) +│ aggregate 0: min(column8) +│ aggregate 1: max(column8) +│ aggregate 2: count(column11) +│ aggregate 3: sum_int(column8) +│ aggregate 4: avg(column8) +│ aggregate 5: sum(column8) +│ aggregate 6: stddev(column8) +│ aggregate 7: variance(column8) +│ aggregate 8: bool_and(column18) +│ aggregate 9: bool_or(column20) +│ aggregate 10: xor_agg(column22) +│ aggregate 11: corr(column8, column8) +│ aggregate 12: covar_pop(column8, column8) +│ aggregate 13: covar_samp(column8, column8) +│ aggregate 14: sqrdiff(column8) +│ aggregate 15: regr_intercept(column8, column8) +│ aggregate 16: regr_r2(column8, column8) +│ aggregate 17: regr_slope(column8, column8) +│ aggregate 18: regr_sxx(column8, column8) +│ aggregate 19: regr_sxy(column8, column8) +│ aggregate 20: regr_syy(column8, column8) │ aggregate 21: count_rows() -│ aggregate 22: regr_avgx(column7, column7) -│ aggregate 23: regr_avgy(column7, column7) +│ aggregate 22: regr_avgx(column8, column8) +│ aggregate 23: regr_avgy(column8, column8) │ └── • render - │ columns: (column7 int, column10 unknown, column17 bool, column19 bool, column21 bytes) - │ render column7: (1)[int] - │ render column10: (NULL)[unknown] - │ render column17: (true)[bool] - │ render column19: (false)[bool] - │ render column21: ('\x01')[bytes] + │ columns: (column8 int, column11 unknown, column18 bool, column20 bool, column22 bytes) + │ render column8: (1)[int] + │ render column11: (NULL)[unknown] + │ render column18: (true)[bool] + │ render column20: (false)[bool] + │ render column22: ('\x01')[bytes] │ └── • scan columns: () @@ -76,33 +77,33 @@ vectorized: true │ aggregate 0: min(v) │ aggregate 1: max(v) │ aggregate 2: count(v) -│ aggregate 3: sum_int(column10) +│ aggregate 3: sum_int(column11) │ aggregate 4: avg(v) │ aggregate 5: sum(v) │ aggregate 6: stddev(v) │ aggregate 7: variance(v) -│ aggregate 8: bool_and(column16) -│ aggregate 9: bool_or(column16) -│ aggregate 10: xor_agg(column19) +│ aggregate 8: bool_and(column17) +│ aggregate 9: bool_or(column17) +│ aggregate 10: xor_agg(column20) │ aggregate 11: corr(v, k) │ aggregate 12: covar_pop(v, k) │ aggregate 13: covar_samp(v, k) │ aggregate 14: sqrdiff(v) -│ aggregate 15: regr_intercept(column10, column10) -│ aggregate 16: regr_r2(column10, column10) -│ aggregate 17: regr_slope(column10, column10) -│ aggregate 18: regr_sxx(column10, column10) -│ aggregate 19: regr_sxy(column10, column10) -│ aggregate 20: regr_syy(column10, column10) +│ aggregate 15: regr_intercept(column11, column11) +│ aggregate 16: regr_r2(column11, column11) +│ aggregate 17: regr_slope(column11, column11) +│ aggregate 18: regr_sxx(column11, column11) +│ aggregate 19: regr_sxy(column11, column11) +│ aggregate 20: regr_syy(column11, column11) │ aggregate 21: count_rows() -│ aggregate 22: regr_avgx(column10, column10) -│ aggregate 23: regr_avgy(column10, column10) +│ aggregate 22: regr_avgx(column11, column11) +│ aggregate 23: regr_avgy(column11, column11) │ └── • render - │ columns: (column10 int, column16 bool, column19 bytes, k int, v int) - │ render column10: (1)[int] - │ render column16: ((v)[int] = (1)[int])[bool] - │ render column19: ((s)[string]::BYTES)[bytes] + │ columns: (column11 int, column17 bool, column20 bytes, k int, v int) + │ render column11: (1)[int] + │ render column17: ((v)[int] = (1)[int])[bool] + │ render column20: ((s)[string]::BYTES)[bytes] │ render k: (k)[int] │ render v: (v)[int] │ @@ -216,11 +217,11 @@ vectorized: true │ columns: (count int, r int) │ estimated row count: 1,000 (missing stats) │ aggregate 0: count_rows() -│ group by: column8 +│ group by: column9 │ └── • render - │ columns: (column8 int) - │ render column8: ((k)[int] + (v)[int])[int] + │ columns: (column9 int) + │ render column9: ((k)[int] + (v)[int])[int] │ └── • scan columns: (k int, v int) @@ -322,23 +323,23 @@ vectorized: true • group (scalar) │ columns: (count) │ estimated row count: 1 (missing stats) -│ aggregate 0: count(column13) +│ aggregate 0: count(column15) │ └── • distinct - │ columns: (column13) + │ columns: (column15) │ estimated row count: 1,000 (missing stats) - │ distinct on: column13 + │ distinct on: column15 │ └── • render - │ columns: (column13) - │ render column13: ((k, v, w, s) AS k, v, w, s) + │ columns: (column15) + │ render column15: ((k, v, w, s, arr) AS k, v, w, s, arr) │ └── • cross join (inner) - │ columns: (k, v, w, s) + │ columns: (k, v, w, s, arr) │ estimated row count: 1,000,000 (missing stats) │ ├── • scan - │ columns: (k, v, w, s) + │ columns: (k, v, w, s, arr) │ estimated row count: 1,000 (missing stats) │ table: kv@kv_pkey │ spans: FULL SCAN @@ -1224,13 +1225,13 @@ vectorized: true └── • group (hash) │ columns: (v int, count int, count_rows int) │ estimated row count: 100 (missing stats) - │ aggregate 0: count(column7) + │ aggregate 0: count(column8) │ aggregate 1: count_rows() │ group by: v │ └── • render - │ columns: (column7 unknown, v int) - │ render column7: (NULL)[unknown] + │ columns: (column8 unknown, v int) + │ render column8: (NULL)[unknown] │ render v: (v)[int] │ └── • scan @@ -1249,12 +1250,12 @@ vectorized: true • group (hash) │ columns: (v, count) │ estimated row count: 33 (missing stats) -│ aggregate 0: count(column7) +│ aggregate 0: count(column8) │ group by: v │ └── • render - │ columns: (column7, v) - │ render column7: NULL + │ columns: (column8, v) + │ render column8: NULL │ render v: v │ └── • filter @@ -1677,18 +1678,18 @@ vectorized: true │ render a: (1)[int] │ └── • distinct - │ columns: (column7 decimal, v int) + │ columns: (column8 decimal, v int) │ estimated row count: 333 (missing stats) - │ distinct on: column7, v + │ distinct on: column8, v │ └── • filter - │ columns: (column7 decimal, v int) + │ columns: (column8 decimal, v int) │ estimated row count: 333 (missing stats) - │ filter: ((column7)[decimal] > (1)[decimal])[bool] + │ filter: ((column8)[decimal] > (1)[decimal])[bool] │ └── • render - │ columns: (column7 decimal, v int) - │ render column7: ((w)[int]::DECIMAL)[decimal] + │ columns: (column8 decimal, v int) + │ render column8: ((w)[int]::DECIMAL)[decimal] │ render v: (v)[int] │ └── • scan @@ -1776,6 +1777,29 @@ vectorized: true table: kv@kv_pkey spans: FULL SCAN +query T +EXPLAIN (VERBOSE) SELECT array_cat_agg(arr) FROM (SELECT * FROM kv ORDER BY v) +---- +distribution: local +vectorized: true +· +• group (scalar) +│ columns: (array_cat_agg) +│ estimated row count: 1 (missing stats) +│ aggregate 0: array_cat_agg(arr) +│ +└── • sort + │ columns: (v, arr) + │ ordering: +v + │ estimated row count: 1,000 (missing stats) + │ order: +v + │ + └── • scan + columns: (v, arr) + estimated row count: 1,000 (missing stats) + table: kv@kv_pkey + spans: FULL SCAN + query T EXPLAIN (VERBOSE) SELECT k FROM kv ORDER BY s ---- @@ -1847,12 +1871,12 @@ vectorized: true • group (scalar) │ columns: (string_agg) │ estimated row count: 1 (missing stats) -│ aggregate 0: string_agg(s, column7) +│ aggregate 0: string_agg(s, column8) │ └── • render - │ columns: (column7, k, s) + │ columns: (column8, k, s) │ ordering: +k - │ render column7: ',' + │ render column8: ',' │ render k: k │ render s: s │ @@ -1930,11 +1954,11 @@ vectorized: true • group (scalar) │ columns: (string_agg) │ estimated row count: 1 (missing stats) -│ aggregate 0: string_agg(s, column7) +│ aggregate 0: string_agg(s, column8) │ └── • render - │ columns: (column7, s) - │ render column7: ', ' + │ columns: (column8, s) + │ render column8: ', ' │ render s: s │ └── • scan diff --git a/pkg/sql/opt/memo/testdata/memo b/pkg/sql/opt/memo/testdata/memo index cc982542701..7e5426d306a 100644 --- a/pkg/sql/opt/memo/testdata/memo +++ b/pkg/sql/opt/memo/testdata/memo @@ -1,5 +1,5 @@ exec-ddl -CREATE TABLE a (x INT PRIMARY KEY, y INT) +CREATE TABLE a (x INT PRIMARY KEY, y INT, arr INT[]) ---- exec-ddl @@ -14,51 +14,51 @@ ORDER BY y LIMIT 10 ---- limit - ├── columns: y:2(int!null) x:5(string!null) c:9(int!null) + ├── columns: y:2(int!null) x:6(string!null) c:10(int!null) ├── internal-ordering: +2 ├── cardinality: [0 - 10] ├── immutable - ├── fd: (2)-->(9) + ├── fd: (2)-->(10) ├── ordering: +2 ├── sort - │ ├── columns: y:2(int!null) b.x:5(string!null) c:9(int!null) + │ ├── columns: y:2(int!null) b.x:6(string!null) c:10(int!null) │ ├── immutable - │ ├── fd: (2)-->(9) + │ ├── fd: (2)-->(10) │ ├── ordering: +2 │ ├── limit hint: 10.00 │ └── project - │ ├── columns: c:9(int!null) y:2(int!null) b.x:5(string!null) + │ ├── columns: c:10(int!null) y:2(int!null) b.x:6(string!null) │ ├── immutable - │ ├── fd: (2)-->(9) + │ ├── fd: (2)-->(10) │ ├── select - │ │ ├── columns: a.x:1(int!null) y:2(int!null) a.crdb_internal_mvcc_timestamp:3(decimal) a.tableoid:4(oid) b.x:5(string!null) z:6(decimal!null) b.crdb_internal_mvcc_timestamp:7(decimal) b.tableoid:8(oid) + │ │ ├── columns: a.x:1(int!null) y:2(int!null) arr:3(int[]) a.crdb_internal_mvcc_timestamp:4(decimal) a.tableoid:5(oid) b.x:6(string!null) z:7(decimal!null) b.crdb_internal_mvcc_timestamp:8(decimal) b.tableoid:9(oid) │ │ ├── immutable - │ │ ├── key: (1,5) - │ │ ├── fd: (1)-->(2-4), (5)-->(6-8) + │ │ ├── key: (1,6) + │ │ ├── fd: (1)-->(2-5), (6)-->(7-9) │ │ ├── inner-join (cross) - │ │ │ ├── columns: a.x:1(int!null) y:2(int) a.crdb_internal_mvcc_timestamp:3(decimal) a.tableoid:4(oid) b.x:5(string!null) z:6(decimal!null) b.crdb_internal_mvcc_timestamp:7(decimal) b.tableoid:8(oid) - │ │ │ ├── key: (1,5) - │ │ │ ├── fd: (1)-->(2-4), (5)-->(6-8) + │ │ │ ├── columns: a.x:1(int!null) y:2(int) arr:3(int[]) a.crdb_internal_mvcc_timestamp:4(decimal) a.tableoid:5(oid) b.x:6(string!null) z:7(decimal!null) b.crdb_internal_mvcc_timestamp:8(decimal) b.tableoid:9(oid) + │ │ │ ├── key: (1,6) + │ │ │ ├── fd: (1)-->(2-5), (6)-->(7-9) │ │ │ ├── scan a - │ │ │ │ ├── columns: a.x:1(int!null) y:2(int) a.crdb_internal_mvcc_timestamp:3(decimal) a.tableoid:4(oid) + │ │ │ │ ├── columns: a.x:1(int!null) y:2(int) arr:3(int[]) a.crdb_internal_mvcc_timestamp:4(decimal) a.tableoid:5(oid) │ │ │ │ ├── key: (1) - │ │ │ │ └── fd: (1)-->(2-4) + │ │ │ │ └── fd: (1)-->(2-5) │ │ │ ├── scan b - │ │ │ │ ├── columns: b.x:5(string!null) z:6(decimal!null) b.crdb_internal_mvcc_timestamp:7(decimal) b.tableoid:8(oid) - │ │ │ │ ├── key: (5) - │ │ │ │ └── fd: (5)-->(6-8) + │ │ │ │ ├── columns: b.x:6(string!null) z:7(decimal!null) b.crdb_internal_mvcc_timestamp:8(decimal) b.tableoid:9(oid) + │ │ │ │ ├── key: (6) + │ │ │ │ └── fd: (6)-->(7-9) │ │ │ └── filters (true) │ │ └── filters - │ │ └── and [type=bool, outer=(1,2,5), immutable, constraints=(/2: [/2 - ])] + │ │ └── and [type=bool, outer=(1,2,6), immutable, constraints=(/2: [/2 - ])] │ │ ├── gt [type=bool] │ │ │ ├── variable: y:2 [type=int] │ │ │ └── const: 1 [type=int] │ │ └── eq [type=bool] │ │ ├── cast: STRING [type=string] │ │ │ └── variable: a.x:1 [type=int] - │ │ └── variable: b.x:5 [type=string] + │ │ └── variable: b.x:6 [type=string] │ └── projections - │ └── plus [as=c:9, type=int, outer=(2), immutable] + │ └── plus [as=c:10, type=int, outer=(2), immutable] │ ├── variable: y:2 [type=int] │ └── const: 1 [type=int] └── const: 10 [type=int] @@ -71,33 +71,33 @@ ORDER BY y LIMIT 10 ---- project - ├── columns: y:2(int!null) x:5(string!null) c:10(int!null) + ├── columns: y:2(int!null) x:6(string!null) c:11(int!null) ├── cardinality: [0 - 10] ├── immutable - ├── fd: (2)-->(10) + ├── fd: (2)-->(11) ├── ordering: +2 ├── limit - │ ├── columns: y:2(int!null) b.x:5(string!null) column9:9(string!null) + │ ├── columns: y:2(int!null) b.x:6(string!null) column10:10(string!null) │ ├── internal-ordering: +2 │ ├── cardinality: [0 - 10] │ ├── immutable - │ ├── fd: (5)==(9), (9)==(5) + │ ├── fd: (6)==(10), (10)==(6) │ ├── ordering: +2 │ ├── inner-join (lookup b) - │ │ ├── columns: y:2(int!null) b.x:5(string!null) column9:9(string!null) - │ │ ├── key columns: [9] = [5] + │ │ ├── columns: y:2(int!null) b.x:6(string!null) column10:10(string!null) + │ │ ├── key columns: [10] = [6] │ │ ├── lookup columns are key │ │ ├── immutable - │ │ ├── fd: (5)==(9), (9)==(5) + │ │ ├── fd: (6)==(10), (10)==(6) │ │ ├── ordering: +2 │ │ ├── limit hint: 10.00 │ │ ├── sort - │ │ │ ├── columns: y:2(int!null) column9:9(string!null) + │ │ │ ├── columns: y:2(int!null) column10:10(string!null) │ │ │ ├── immutable │ │ │ ├── ordering: +2 │ │ │ ├── limit hint: 100.00 │ │ │ └── project - │ │ │ ├── columns: column9:9(string!null) y:2(int!null) + │ │ │ ├── columns: column10:10(string!null) y:2(int!null) │ │ │ ├── immutable │ │ │ ├── select │ │ │ │ ├── columns: a.x:1(int!null) y:2(int!null) @@ -112,12 +112,12 @@ project │ │ │ │ ├── variable: y:2 [type=int] │ │ │ │ └── const: 1 [type=int] │ │ │ └── projections - │ │ │ └── cast: STRING [as=column9:9, type=string, outer=(1), immutable] + │ │ │ └── cast: STRING [as=column10:10, type=string, outer=(1), immutable] │ │ │ └── variable: a.x:1 [type=int] │ │ └── filters (true) │ └── const: 10 [type=int] └── projections - └── plus [as=c:10, type=int, outer=(2), immutable] + └── plus [as=c:11, type=int, outer=(2), immutable] ├── variable: y:2 [type=int] └── const: 1 [type=int] @@ -128,47 +128,47 @@ WHERE a.y>1 AND a.x::string=b.x ORDER BY y LIMIT 10 ---- -memo (optimized, ~24KB, required=[presentation: y:2,x:5,c:10] [ordering: +2]) +memo (optimized, ~24KB, required=[presentation: y:2,x:6,c:11] [ordering: +2]) ├── G1: (project G2 G3 y x) - │ ├── [presentation: y:2,x:5,c:10] [ordering: +2] + │ ├── [presentation: y:2,x:6,c:11] [ordering: +2] │ │ ├── best: (project G2="[ordering: +2]" G3 y x) - │ │ └── cost: 1754.72 + │ │ └── cost: 1764.82 │ └── [] │ ├── best: (project G2 G3 y x) - │ └── cost: 1754.72 + │ └── cost: 1764.82 ├── G2: (limit G4 G5 ordering=+2) (top-k G4 &{10 +2 }) │ ├── [ordering: +2] │ │ ├── best: (limit G4="[ordering: +2] [limit hint: 10.00]" G5 ordering=+2) - │ │ └── cost: 1754.51 + │ │ └── cost: 1764.61 │ └── [] │ ├── best: (limit G4="[ordering: +2] [limit hint: 10.00]" G5 ordering=+2) - │ └── cost: 1754.51 + │ └── cost: 1764.61 ├── G3: (projections G6) - ├── G4: (inner-join G7 G8 G9) (inner-join G8 G7 G9) (lookup-join G7 G10 b,keyCols=[9],outCols=(2,5,9)) (merge-join G8 G7 G10 inner-join,+5,+9) + ├── G4: (inner-join G7 G8 G9) (inner-join G8 G7 G9) (lookup-join G7 G10 b,keyCols=[10],outCols=(2,6,10)) (merge-join G8 G7 G10 inner-join,+6,+10) │ ├── [ordering: +2] [limit hint: 10.00] - │ │ ├── best: (lookup-join G7="[ordering: +2] [limit hint: 100.00]" G10 b,keyCols=[9],outCols=(2,5,9)) - │ │ └── cost: 1754.40 + │ │ ├── best: (lookup-join G7="[ordering: +2] [limit hint: 100.00]" G10 b,keyCols=[10],outCols=(2,6,10)) + │ │ └── cost: 1764.50 │ └── [] │ ├── best: (inner-join G8 G7 G9) - │ └── cost: 2157.16 + │ └── cost: 2167.26 ├── G5: (const 10) ├── G6: (plus G11 G12) ├── G7: (project G13 G14 y) - │ ├── [ordering: +2] [limit hint: 100.00] + │ ├── [ordering: +10] │ │ ├── best: (sort G7) - │ │ └── cost: 1150.38 - │ ├── [ordering: +9] + │ │ └── cost: 1160.48 + │ ├── [ordering: +2] [limit hint: 100.00] │ │ ├── best: (sort G7) - │ │ └── cost: 1150.38 + │ │ └── cost: 1160.48 │ └── [] │ ├── best: (project G13 G14 y) - │ └── cost: 1081.14 - ├── G8: (scan b,cols=(5)) - │ ├── [ordering: +5] - │ │ ├── best: (scan b,cols=(5)) + │ └── cost: 1091.24 + ├── G8: (scan b,cols=(6)) + │ ├── [ordering: +6] + │ │ ├── best: (scan b,cols=(6)) │ │ └── cost: 1054.32 │ └── [] - │ ├── best: (scan b,cols=(5)) + │ ├── best: (scan b,cols=(6)) │ └── cost: 1054.32 ├── G9: (filters G15) ├── G10: (filters) @@ -177,22 +177,22 @@ memo (optimized, ~24KB, required=[presentation: y:2,x:5,c:10] [ordering: +2]) ├── G13: (select G16 G17) │ ├── [ordering: +2] [limit hint: 100.00] │ │ ├── best: (sort G13) - │ │ └── cost: 1143.69 + │ │ └── cost: 1153.79 │ └── [] │ ├── best: (select G16 G17) - │ └── cost: 1074.45 + │ └── cost: 1084.55 ├── G14: (projections G18) ├── G15: (eq G19 G20) ├── G16: (scan a,cols=(1,2)) │ ├── [ordering: +2] [limit hint: 300.00] │ │ ├── best: (sort G16) - │ │ └── cost: 1303.90 + │ │ └── cost: 1314.00 │ └── [] │ ├── best: (scan a,cols=(1,2)) - │ └── cost: 1064.42 + │ └── cost: 1074.52 ├── G17: (filters G21) ├── G18: (cast G22 STRING) - ├── G19: (variable column9) + ├── G19: (variable column10) ├── G20: (variable b.x) ├── G21: (gt G11 G12) └── G22: (variable a.x) @@ -242,21 +242,21 @@ memo (optimized, ~8KB, required=[presentation: x:1]) ├── G1: (project G2 G3 x) │ └── [presentation: x:1] │ ├── best: (project G2 G3 x) - │ └── cost: 5.10 + │ └── cost: 5.11 ├── G2: (select G4 G5) (select G6 G7) │ └── [] │ ├── best: (select G6 G7) - │ └── cost: 5.08 + │ └── cost: 5.09 ├── G3: (projections) ├── G4: (scan a,cols=(1,2)) │ └── [] │ ├── best: (scan a,cols=(1,2)) - │ └── cost: 1064.42 + │ └── cost: 1074.52 ├── G5: (filters G8 G9) ├── G6: (scan a,cols=(1,2),constrained) │ └── [] │ ├── best: (scan a,cols=(1,2),constrained) - │ └── cost: 5.05 + │ └── cost: 5.06 ├── G7: (filters G9) ├── G8: (eq G10 G11) ├── G9: (eq G12 G13) @@ -268,29 +268,29 @@ memo (optimized, ~8KB, required=[presentation: x:1]) memo SELECT x, y FROM a UNION SELECT x+1, y+1 FROM a ---- -memo (optimized, ~8KB, required=[presentation: x:11,y:12]) - ├── G1: (union G2 G3) (union G2 G3 ordering=+11,+12) - │ └── [presentation: x:11,y:12] +memo (optimized, ~8KB, required=[presentation: x:13,y:14]) + ├── G1: (union G2 G3) (union G2 G3 ordering=+13,+14) + │ └── [presentation: x:13,y:14] │ ├── best: (union G2 G3) - │ └── cost: 2199.49 + │ └── cost: 2219.69 ├── G2: (scan a,cols=(1,2)) │ ├── [ordering: +1] │ │ ├── best: (scan a,cols=(1,2)) - │ │ └── cost: 1064.42 + │ │ └── cost: 1074.52 │ └── [] │ ├── best: (scan a,cols=(1,2)) - │ └── cost: 1064.42 + │ └── cost: 1074.52 ├── G3: (project G4 G5) - │ ├── [ordering: +9,+10] + │ ├── [ordering: +11,+12] │ │ ├── best: (sort G3) - │ │ └── cost: 1344.89 + │ │ └── cost: 1354.99 │ └── [] │ ├── best: (project G4 G5) - │ └── cost: 1094.44 - ├── G4: (scan a,cols=(5,6)) + │ └── cost: 1104.54 + ├── G4: (scan a,cols=(6,7)) │ └── [] - │ ├── best: (scan a,cols=(5,6)) - │ └── cost: 1064.42 + │ ├── best: (scan a,cols=(6,7)) + │ └── cost: 1074.52 ├── G5: (projections G6 G7) ├── G6: (plus G8 G9) ├── G7: (plus G10 G9) @@ -301,15 +301,15 @@ memo (optimized, ~8KB, required=[presentation: x:11,y:12]) memo SELECT array_agg(x) FROM (SELECT * FROM a) ---- -memo (optimized, ~5KB, required=[presentation: array_agg:5]) +memo (optimized, ~5KB, required=[presentation: array_agg:6]) ├── G1: (scalar-group-by G2 G3 cols=()) - │ └── [presentation: array_agg:5] + │ └── [presentation: array_agg:6] │ ├── best: (scalar-group-by G2 G3 cols=()) - │ └── cost: 1064.35 + │ └── cost: 1074.45 ├── G2: (scan a,cols=(1)) │ └── [] │ ├── best: (scan a,cols=(1)) - │ └── cost: 1054.32 + │ └── cost: 1064.42 ├── G3: (aggregations G4) ├── G4: (array-agg G5) └── G5: (variable x) @@ -317,20 +317,20 @@ memo (optimized, ~5KB, required=[presentation: array_agg:5]) memo SELECT array_agg(x) FROM (SELECT * FROM a) GROUP BY y ---- -memo (optimized, ~5KB, required=[presentation: array_agg:5]) +memo (optimized, ~6KB, required=[presentation: array_agg:6]) ├── G1: (project G2 G3 array_agg) - │ └── [presentation: array_agg:5] + │ └── [presentation: array_agg:6] │ ├── best: (project G2 G3 array_agg) - │ └── cost: 1096.47 + │ └── cost: 1106.57 ├── G2: (group-by G4 G5 cols=(2)) │ └── [] │ ├── best: (group-by G4 G5 cols=(2)) - │ └── cost: 1095.45 + │ └── cost: 1105.55 ├── G3: (projections) ├── G4: (scan a,cols=(1,2)) │ └── [] │ ├── best: (scan a,cols=(1,2)) - │ └── cost: 1064.42 + │ └── cost: 1074.52 ├── G5: (aggregations G6) ├── G6: (array-agg G7) └── G7: (variable x) @@ -338,22 +338,78 @@ memo (optimized, ~5KB, required=[presentation: array_agg:5]) memo SELECT array_agg(x) FROM (SELECT * FROM a ORDER BY y) ---- -memo (optimized, ~4KB, required=[presentation: array_agg:5]) +memo (optimized, ~5KB, required=[presentation: array_agg:6]) ├── G1: (scalar-group-by G2 G3 cols=(),ordering=+2) - │ └── [presentation: array_agg:5] + │ └── [presentation: array_agg:6] │ ├── best: (scalar-group-by G2="[ordering: +2]" G3 cols=(),ordering=+2) - │ └── cost: 1313.93 + │ └── cost: 1324.03 ├── G2: (scan a,cols=(1,2)) │ ├── [ordering: +2] │ │ ├── best: (sort G2) - │ │ └── cost: 1303.90 + │ │ └── cost: 1314.00 │ └── [] │ ├── best: (scan a,cols=(1,2)) - │ └── cost: 1064.42 + │ └── cost: 1074.52 ├── G3: (aggregations G4) ├── G4: (array-agg G5) └── G5: (variable x) +memo +SELECT array_cat_agg(arr) FROM (SELECT * FROM a) +---- +memo (optimized, ~5KB, required=[presentation: array_cat_agg:6]) + ├── G1: (scalar-group-by G2 G3 cols=()) + │ └── [presentation: array_cat_agg:6] + │ ├── best: (scalar-group-by G2 G3 cols=()) + │ └── cost: 1074.45 + ├── G2: (scan a,cols=(3)) + │ └── [] + │ ├── best: (scan a,cols=(3)) + │ └── cost: 1064.42 + ├── G3: (aggregations G4) + ├── G4: (array-cat-agg G5) + └── G5: (variable arr) + +memo +SELECT array_cat_agg(arr) FROM (SELECT * FROM a) GROUP BY y +---- +memo (optimized, ~6KB, required=[presentation: array_cat_agg:6]) + ├── G1: (project G2 G3 array_cat_agg) + │ └── [presentation: array_cat_agg:6] + │ ├── best: (project G2 G3 array_cat_agg) + │ └── cost: 1106.57 + ├── G2: (group-by G4 G5 cols=(2)) + │ └── [] + │ ├── best: (group-by G4 G5 cols=(2)) + │ └── cost: 1105.55 + ├── G3: (projections) + ├── G4: (scan a,cols=(2,3)) + │ └── [] + │ ├── best: (scan a,cols=(2,3)) + │ └── cost: 1074.52 + ├── G5: (aggregations G6) + ├── G6: (array-cat-agg G7) + └── G7: (variable arr) + +memo +SELECT array_cat_agg(arr) FROM (SELECT * FROM a ORDER BY y) +---- +memo (optimized, ~5KB, required=[presentation: array_cat_agg:6]) + ├── G1: (scalar-group-by G2 G3 cols=(),ordering=+2) + │ └── [presentation: array_cat_agg:6] + │ ├── best: (scalar-group-by G2="[ordering: +2]" G3 cols=(),ordering=+2) + │ └── cost: 1324.03 + ├── G2: (scan a,cols=(2,3)) + │ ├── [ordering: +2] + │ │ ├── best: (sort G2) + │ │ └── cost: 1314.00 + │ └── [] + │ ├── best: (scan a,cols=(2,3)) + │ └── cost: 1074.52 + ├── G3: (aggregations G4) + ├── G4: (array-cat-agg G5) + └── G5: (variable arr) + memo SELECT DISTINCT info FROM [EXPLAIN SELECT 123 AS k] ---- diff --git a/pkg/sql/opt/memo/testdata/typing b/pkg/sql/opt/memo/testdata/typing index 4bf01bb7191..09dba8cefce 100644 --- a/pkg/sql/opt/memo/testdata/typing +++ b/pkg/sql/opt/memo/testdata/typing @@ -3,7 +3,7 @@ CREATE TABLE a (x INT PRIMARY KEY, y INT) ---- exec-ddl -CREATE TABLE b (x STRING PRIMARY KEY, z DECIMAL NOT NULL) +CREATE TABLE b (x STRING PRIMARY KEY, z DECIMAL NOT NULL, arr STRING[]) ---- exec-ddl @@ -114,11 +114,11 @@ build SELECT * FROM b WHERE b.x LIKE '%foo%' AND b.x NOT LIKE '%bar%' ---- project - ├── columns: x:1(string!null) z:2(decimal!null) + ├── columns: x:1(string!null) z:2(decimal!null) arr:3(string[]) └── select - ├── columns: x:1(string!null) z:2(decimal!null) crdb_internal_mvcc_timestamp:3(decimal) tableoid:4(oid) + ├── columns: x:1(string!null) z:2(decimal!null) arr:3(string[]) crdb_internal_mvcc_timestamp:4(decimal) tableoid:5(oid) ├── scan b - │ └── columns: x:1(string!null) z:2(decimal!null) crdb_internal_mvcc_timestamp:3(decimal) tableoid:4(oid) + │ └── columns: x:1(string!null) z:2(decimal!null) arr:3(string[]) crdb_internal_mvcc_timestamp:4(decimal) tableoid:5(oid) └── filters └── (x:1 LIKE '%foo%') AND (x:1 NOT LIKE '%bar%') [type=bool] @@ -127,11 +127,11 @@ build SELECT * FROM b WHERE b.x ILIKE '%foo%' AND b.x NOT ILIKE '%bar%' ---- project - ├── columns: x:1(string!null) z:2(decimal!null) + ├── columns: x:1(string!null) z:2(decimal!null) arr:3(string[]) └── select - ├── columns: x:1(string!null) z:2(decimal!null) crdb_internal_mvcc_timestamp:3(decimal) tableoid:4(oid) + ├── columns: x:1(string!null) z:2(decimal!null) arr:3(string[]) crdb_internal_mvcc_timestamp:4(decimal) tableoid:5(oid) ├── scan b - │ └── columns: x:1(string!null) z:2(decimal!null) crdb_internal_mvcc_timestamp:3(decimal) tableoid:4(oid) + │ └── columns: x:1(string!null) z:2(decimal!null) arr:3(string[]) crdb_internal_mvcc_timestamp:4(decimal) tableoid:5(oid) └── filters └── (x:1 ILIKE '%foo%') AND (x:1 NOT ILIKE '%bar%') [type=bool] @@ -140,11 +140,11 @@ build SELECT * FROM b WHERE b.x ~ 'foo' AND b.x !~ 'bar' AND b.x ~* 'foo' AND b.x !~* 'bar' ---- project - ├── columns: x:1(string!null) z:2(decimal!null) + ├── columns: x:1(string!null) z:2(decimal!null) arr:3(string[]) └── select - ├── columns: x:1(string!null) z:2(decimal!null) crdb_internal_mvcc_timestamp:3(decimal) tableoid:4(oid) + ├── columns: x:1(string!null) z:2(decimal!null) arr:3(string[]) crdb_internal_mvcc_timestamp:4(decimal) tableoid:5(oid) ├── scan b - │ └── columns: x:1(string!null) z:2(decimal!null) crdb_internal_mvcc_timestamp:3(decimal) tableoid:4(oid) + │ └── columns: x:1(string!null) z:2(decimal!null) arr:3(string[]) crdb_internal_mvcc_timestamp:4(decimal) tableoid:5(oid) └── filters └── (((x:1 ~ 'foo') AND (x:1 !~ 'bar')) AND (x:1 ~* 'foo')) AND (x:1 !~* 'bar') [type=bool] @@ -235,11 +235,11 @@ build SELECT b.x || 'more' AS r FROM b ---- project - ├── columns: r:5(string!null) + ├── columns: r:6(string!null) ├── scan b - │ └── columns: x:1(string!null) z:2(decimal!null) crdb_internal_mvcc_timestamp:3(decimal) tableoid:4(oid) + │ └── columns: x:1(string!null) z:2(decimal!null) arr:3(string[]) crdb_internal_mvcc_timestamp:4(decimal) tableoid:5(oid) └── projections - └── x:1 || 'more' [as=r:5, type=string] + └── x:1 || 'more' [as=r:6, type=string] # UnaryMinus, UnaryComplement build @@ -327,55 +327,57 @@ project # Aggregate functions. build SELECT - array_agg(z), avg(z), bool_and(z=0), bool_or(z=0), concat_agg(x), count(z), - count(*), max(x), max(z), sum_int(x::int), sum(z), sqrdiff(z), variance(x::int), + array_agg(z), array_cat_agg(arr), avg(z), bool_and(z=0), bool_or(z=0), concat_agg(x), + count(z), count(*), max(x), max(z), sum_int(x::int), sum(z), sqrdiff(z), variance(x::int), stddev(z), xor_agg(x::int), json_agg(x::json), jsonb_agg(x::jsonb) FROM b ---- scalar-group-by - ├── columns: array_agg:5(decimal[]) avg:6(decimal) bool_and:8(bool) bool_or:9(bool) concat_agg:10(string) count:11(int!null) count:12(int!null) max:13(string) max:14(decimal) sum_int:16(int) sum:17(decimal) sqrdiff:18(decimal) variance:19(decimal) stddev:20(decimal) xor_agg:21(int) json_agg:23(jsonb) jsonb_agg:24(jsonb) + ├── columns: array_agg:6(decimal[]) array_cat_agg:7(string[]) avg:8(decimal) bool_and:10(bool) bool_or:11(bool) concat_agg:12(string) count:13(int!null) count:14(int!null) max:15(string) max:16(decimal) sum_int:18(int) sum:19(decimal) sqrdiff:20(decimal) variance:21(decimal) stddev:22(decimal) xor_agg:23(int) json_agg:25(jsonb) jsonb_agg:26(jsonb) ├── project - │ ├── columns: column7:7(bool!null) column15:15(int!null) column22:22(jsonb!null) x:1(string!null) z:2(decimal!null) + │ ├── columns: column9:9(bool!null) column17:17(int!null) column24:24(jsonb!null) x:1(string!null) z:2(decimal!null) arr:3(string[]) │ ├── scan b - │ │ └── columns: x:1(string!null) z:2(decimal!null) crdb_internal_mvcc_timestamp:3(decimal) tableoid:4(oid) + │ │ └── columns: x:1(string!null) z:2(decimal!null) arr:3(string[]) crdb_internal_mvcc_timestamp:4(decimal) tableoid:5(oid) │ └── projections - │ ├── z:2 = 0 [as=column7:7, type=bool] - │ ├── x:1::INT8 [as=column15:15, type=int] - │ └── x:1::JSONB [as=column22:22, type=jsonb] + │ ├── z:2 = 0 [as=column9:9, type=bool] + │ ├── x:1::INT8 [as=column17:17, type=int] + │ └── x:1::JSONB [as=column24:24, type=jsonb] └── aggregations - ├── array-agg [as=array_agg:5, type=decimal[]] + ├── array-agg [as=array_agg:6, type=decimal[]] │ └── z:2 [type=decimal] - ├── avg [as=avg:6, type=decimal] + ├── array-cat-agg [as=array_cat_agg:7, type=string[]] + │ └── arr:3 [type=string[]] + ├── avg [as=avg:8, type=decimal] │ └── z:2 [type=decimal] - ├── bool-and [as=bool_and:8, type=bool] - │ └── column7:7 [type=bool] - ├── bool-or [as=bool_or:9, type=bool] - │ └── column7:7 [type=bool] - ├── concat-agg [as=concat_agg:10, type=string] + ├── bool-and [as=bool_and:10, type=bool] + │ └── column9:9 [type=bool] + ├── bool-or [as=bool_or:11, type=bool] + │ └── column9:9 [type=bool] + ├── concat-agg [as=concat_agg:12, type=string] │ └── x:1 [type=string] - ├── count [as=count:11, type=int] + ├── count [as=count:13, type=int] │ └── z:2 [type=decimal] - ├── count-rows [as=count_rows:12, type=int] - ├── max [as=max:13, type=string] + ├── count-rows [as=count_rows:14, type=int] + ├── max [as=max:15, type=string] │ └── x:1 [type=string] - ├── max [as=max:14, type=decimal] + ├── max [as=max:16, type=decimal] │ └── z:2 [type=decimal] - ├── sum-int [as=sum_int:16, type=int] - │ └── column15:15 [type=int] - ├── sum [as=sum:17, type=decimal] + ├── sum-int [as=sum_int:18, type=int] + │ └── column17:17 [type=int] + ├── sum [as=sum:19, type=decimal] │ └── z:2 [type=decimal] - ├── sqr-diff [as=sqrdiff:18, type=decimal] + ├── sqr-diff [as=sqrdiff:20, type=decimal] │ └── z:2 [type=decimal] - ├── variance [as=variance:19, type=decimal] - │ └── column15:15 [type=int] - ├── std-dev [as=stddev:20, type=decimal] + ├── variance [as=variance:21, type=decimal] + │ └── column17:17 [type=int] + ├── std-dev [as=stddev:22, type=decimal] │ └── z:2 [type=decimal] - ├── xor-agg [as=xor_agg:21, type=int] - │ └── column15:15 [type=int] - ├── json-agg [as=json_agg:23, type=jsonb] - │ └── column22:22 [type=jsonb] - └── jsonb-agg [as=jsonb_agg:24, type=jsonb] - └── column22:22 [type=jsonb] + ├── xor-agg [as=xor_agg:23, type=int] + │ └── column17:17 [type=int] + ├── json-agg [as=json_agg:25, type=jsonb] + │ └── column24:24 [type=jsonb] + └── jsonb-agg [as=jsonb_agg:26, type=jsonb] + └── column24:24 [type=jsonb] # ConstAgg internal aggregate function. opt @@ -384,29 +386,29 @@ SELECT * FROM (SELECT x, x::string, y FROM a) WHERE (SELECT max(x) FROM b WHERE project ├── columns: x:1(int!null) x:5(string!null) y:2(int!null) ├── select - │ ├── columns: a.x:1(int!null) y:2(int!null) max:10(string!null) + │ ├── columns: a.x:1(int!null) y:2(int!null) max:11(string!null) │ ├── group-by (hash) - │ │ ├── columns: a.x:1(int!null) y:2(int!null) max:10(string!null) + │ │ ├── columns: a.x:1(int!null) y:2(int!null) max:11(string!null) │ │ ├── grouping columns: a.x:1(int!null) │ │ ├── inner-join (hash) - │ │ │ ├── columns: a.x:1(int!null) y:2(int!null) b.x:6(string!null) column11:11(int!null) + │ │ │ ├── columns: a.x:1(int!null) y:2(int!null) b.x:6(string!null) column12:12(int!null) │ │ │ ├── scan a │ │ │ │ └── columns: a.x:1(int!null) y:2(int) │ │ │ ├── project - │ │ │ │ ├── columns: column11:11(int!null) b.x:6(string!null) + │ │ │ │ ├── columns: column12:12(int!null) b.x:6(string!null) │ │ │ │ ├── scan b │ │ │ │ │ └── columns: b.x:6(string!null) z:7(decimal!null) │ │ │ │ └── projections - │ │ │ │ └── z:7::INT8 [as=column11:11, type=int] + │ │ │ │ └── z:7::INT8 [as=column12:12, type=int] │ │ │ └── filters - │ │ │ └── y:2 = column11:11 [type=bool] + │ │ │ └── y:2 = column12:12 [type=bool] │ │ └── aggregations - │ │ ├── max [as=max:10, type=string] + │ │ ├── max [as=max:11, type=string] │ │ │ └── b.x:6 [type=string] │ │ └── const-agg [as=y:2, type=int] │ │ └── y:2 [type=int] │ └── filters - │ └── max:10 > 'foo' [type=bool] + │ └── max:11 > 'foo' [type=bool] └── projections └── a.x:1::STRING [as=x:5, type=string] @@ -447,11 +449,11 @@ build SELECT x::VARCHAR(2) FROM b ---- project - ├── columns: x:5(varchar!null) + ├── columns: x:6(varchar!null) ├── scan b - │ └── columns: b.x:1(string!null) z:2(decimal!null) crdb_internal_mvcc_timestamp:3(decimal) tableoid:4(oid) + │ └── columns: b.x:1(string!null) z:2(decimal!null) arr:3(string[]) crdb_internal_mvcc_timestamp:4(decimal) tableoid:5(oid) └── projections - └── b.x:1::VARCHAR(2) [as=x:5, type=varchar] + └── b.x:1::VARCHAR(2) [as=x:6, type=varchar] # Cast same type with different precisions. # See #42571. @@ -459,13 +461,13 @@ build SELECT z::decimal(10, 3), z::decimal(10, 1), z::decimal(10, 4) FROM b ---- project - ├── columns: z:5(decimal!null) z:6(decimal!null) z:7(decimal!null) + ├── columns: z:6(decimal!null) z:7(decimal!null) z:8(decimal!null) ├── scan b - │ └── columns: x:1(string!null) b.z:2(decimal!null) crdb_internal_mvcc_timestamp:3(decimal) tableoid:4(oid) + │ └── columns: x:1(string!null) b.z:2(decimal!null) arr:3(string[]) crdb_internal_mvcc_timestamp:4(decimal) tableoid:5(oid) └── projections - ├── b.z:2::DECIMAL(10,3) [as=z:5, type=decimal] - ├── b.z:2::DECIMAL(10,1) [as=z:6, type=decimal] - └── b.z:2::DECIMAL(10,4) [as=z:7, type=decimal] + ├── b.z:2::DECIMAL(10,3) [as=z:6, type=decimal] + ├── b.z:2::DECIMAL(10,1) [as=z:7, type=decimal] + └── b.z:2::DECIMAL(10,4) [as=z:8, type=decimal] build SELECT ARRAY[1,2] IS NULL diff --git a/pkg/sql/opt/memo/typing.go b/pkg/sql/opt/memo/typing.go index 632b1312b36..04ed228b73a 100644 --- a/pkg/sql/opt/memo/typing.go +++ b/pkg/sql/opt/memo/typing.go @@ -191,6 +191,7 @@ func init() { // a large number of possible overloads or where ReturnType depends on // argument types. typingFuncMap[opt.ArrayAggOp] = typeArrayAgg + typingFuncMap[opt.ArrayCatAggOp] = typeAsFirstArg typingFuncMap[opt.MaxOp] = typeAsFirstArg typingFuncMap[opt.MinOp] = typeAsFirstArg typingFuncMap[opt.ConstAggOp] = typeAsFirstArg diff --git a/pkg/sql/opt/norm/testdata/rules/agg b/pkg/sql/opt/norm/testdata/rules/agg index 08028d524c4..18d981e3c07 100644 --- a/pkg/sql/opt/norm/testdata/rules/agg +++ b/pkg/sql/opt/norm/testdata/rules/agg @@ -100,22 +100,23 @@ SELECT variance(DISTINCT f), xor_agg(DISTINCT s::BYTES), array_agg(DISTINCT i), + array_cat_agg(DISTINCT arr), json_agg(DISTINCT j), regr_count(DISTINCT i, f) FROM a ---- scalar-group-by - ├── columns: count:9!null sum:11 sum_int:12 avg:13 stddev:14 variance:15 xor_agg:17 array_agg:18 json_agg:19 regr_count:20!null + ├── columns: count:9!null sum:11 sum_int:12 avg:13 stddev:14 variance:15 xor_agg:17 array_agg:18 array_cat_agg:19 json_agg:20 regr_count:21!null ├── cardinality: [1 - 1] ├── immutable ├── key: () - ├── fd: ()-->(9,11-15,17-20) + ├── fd: ()-->(9,11-15,17-21) ├── project - │ ├── columns: column10:10 column16:16 i:2 f:3 j:5 + │ ├── columns: column10:10 column16:16 i:2 f:3 j:5 arr:6 │ ├── immutable │ ├── fd: (2)-->(10) │ ├── scan a - │ │ └── columns: i:2 f:3 s:4 j:5 + │ │ └── columns: i:2 f:3 s:4 j:5 arr:6 │ └── projections │ ├── i:2 > 5 [as=column10:10, outer=(2)] │ └── s:4::BYTES [as=column16:16, outer=(4), immutable] @@ -146,10 +147,13 @@ scalar-group-by ├── agg-distinct [as=array_agg:18, outer=(2)] │ └── array-agg │ └── i:2 - ├── agg-distinct [as=json_agg:19, outer=(5)] + ├── agg-distinct [as=array_cat_agg:19, outer=(6)] + │ └── array-cat-agg + │ └── arr:6 + ├── agg-distinct [as=json_agg:20, outer=(5)] │ └── json-agg │ └── j:5 - └── agg-distinct [as=regr_count:20, outer=(2,3)] + └── agg-distinct [as=regr_count:21, outer=(2,3)] └── regression-count ├── i:2 └── f:3 diff --git a/pkg/sql/opt/operator.go b/pkg/sql/opt/operator.go index 7e865e521e2..96d657a11f6 100644 --- a/pkg/sql/opt/operator.go +++ b/pkg/sql/opt/operator.go @@ -183,6 +183,7 @@ var UnaryOpReverseMap = map[Operator]tree.UnaryOperatorSymbol{ // aggregation function. var AggregateOpReverseMap = map[Operator]string{ ArrayAggOp: "array_agg", + ArrayCatAggOp: "array_cat_agg", AvgOp: "avg", BitAndAggOp: "bit_and", BitOrAggOp: "bit_or", @@ -328,8 +329,8 @@ func AggregateIgnoresNulls(op Operator) bool { RegressionSXYOp, RegressionSYYOp, RegressionCountOp: return true - case ArrayAggOp, ConcatAggOp, ConstAggOp, CountRowsOp, FirstAggOp, JsonAggOp, - JsonbAggOp, JsonObjectAggOp, JsonbObjectAggOp: + case ArrayAggOp, ArrayCatAggOp, ConcatAggOp, ConstAggOp, CountRowsOp, + FirstAggOp, JsonAggOp, JsonbAggOp, JsonObjectAggOp, JsonbObjectAggOp: return false default: @@ -343,7 +344,7 @@ func AggregateIgnoresNulls(op Operator) bool { func AggregateIsNullOnEmpty(op Operator) bool { switch op { - case AnyNotNullAggOp, ArrayAggOp, AvgOp, BitAndAggOp, + case AnyNotNullAggOp, ArrayAggOp, ArrayCatAggOp, AvgOp, BitAndAggOp, BitOrAggOp, BoolAndOp, BoolOrOp, ConcatAggOp, ConstAggOp, ConstNotNullAggOp, CorrOp, FirstAggOp, JsonAggOp, JsonbAggOp, MaxOp, MinOp, SqrDiffOp, StdDevOp, STMakeLineOp, StringAggOp, SumOp, SumIntOp, @@ -372,7 +373,7 @@ func AggregateIsNullOnEmpty(op Operator) bool { func AggregateIsNeverNullOnNonNullInput(op Operator) bool { switch op { - case AnyNotNullAggOp, ArrayAggOp, AvgOp, BitAndAggOp, + case AnyNotNullAggOp, ArrayAggOp, ArrayCatAggOp, AvgOp, BitAndAggOp, BitOrAggOp, BoolAndOp, BoolOrOp, ConcatAggOp, ConstAggOp, ConstNotNullAggOp, CountOp, CountRowsOp, FirstAggOp, JsonAggOp, JsonbAggOp, MaxOp, MinOp, SqrDiffOp, @@ -429,7 +430,7 @@ func AggregatesCanMerge(inner, outer Operator) bool { // while CountOp and CountRowsOp both output int values. return outer == SumIntOp - case ArrayAggOp, AvgOp, ConcatAggOp, CorrOp, JsonAggOp, JsonbAggOp, + case ArrayAggOp, ArrayCatAggOp, AvgOp, ConcatAggOp, CorrOp, JsonAggOp, JsonbAggOp, JsonObjectAggOp, JsonbObjectAggOp, PercentileContOp, PercentileDiscOp, SqrDiffOp, STCollectOp, StdDevOp, StringAggOp, VarianceOp, StdDevPopOp, VarPopOp, CovarPopOp, CovarSampOp, RegressionAvgXOp, RegressionAvgYOp, @@ -450,8 +451,8 @@ func AggregateIgnoresDuplicates(op Operator) bool { ConstAggOp, ConstNotNullAggOp, FirstAggOp, MaxOp, MinOp, STExtentOp, STUnionOp: return true - case ArrayAggOp, AvgOp, ConcatAggOp, CountOp, CorrOp, CountRowsOp, SumIntOp, - SumOp, SqrDiffOp, VarianceOp, StdDevOp, XorAggOp, JsonAggOp, JsonbAggOp, + case ArrayAggOp, ArrayCatAggOp, AvgOp, ConcatAggOp, CountOp, CorrOp, CountRowsOp, + SumIntOp, SumOp, SqrDiffOp, VarianceOp, StdDevOp, XorAggOp, JsonAggOp, JsonbAggOp, StringAggOp, PercentileDiscOp, PercentileContOp, StdDevPopOp, STMakeLineOp, VarPopOp, JsonObjectAggOp, JsonbObjectAggOp, STCollectOp, CovarPopOp, CovarSampOp, RegressionAvgXOp, RegressionAvgYOp, RegressionInterceptOp, diff --git a/pkg/sql/opt/ops/scalar.opt b/pkg/sql/opt/ops/scalar.opt index 5e1ea80f9e1..0bb836ac60f 100644 --- a/pkg/sql/opt/ops/scalar.opt +++ b/pkg/sql/opt/ops/scalar.opt @@ -795,6 +795,11 @@ define ArrayAgg { Input ScalarExpr } +[Scalar, Aggregate] +define ArrayCatAgg { + Input ScalarExpr +} + [Scalar, Aggregate] define Avg { Input ScalarExpr diff --git a/pkg/sql/opt/optbuilder/groupby.go b/pkg/sql/opt/optbuilder/groupby.go index 81a45211bd8..f00c5cbc1a2 100644 --- a/pkg/sql/opt/optbuilder/groupby.go +++ b/pkg/sql/opt/optbuilder/groupby.go @@ -251,8 +251,9 @@ func (a aggregateInfo) isOrderingSensitive() bool { return true } switch a.def.Name { - case "array_agg", "concat_agg", "string_agg", "json_agg", "jsonb_agg", "json_object_agg", "jsonb_object_agg", - "st_makeline", "st_collect", "st_memcollect": + case "array_agg", "array_cat_agg", "concat_agg", "string_agg", "json_agg", + "jsonb_agg", "json_object_agg", "jsonb_object_agg", "st_makeline", + "st_collect", "st_memcollect": return true default: return false @@ -806,6 +807,8 @@ func (b *Builder) constructAggregate(name string, args []opt.ScalarExpr) opt.Sca switch name { case "array_agg": return b.factory.ConstructArrayAgg(args[0]) + case "array_cat_agg": + return b.factory.ConstructArrayCatAgg(args[0]) case "avg": return b.factory.ConstructAvg(args[0]) case "bit_and": diff --git a/pkg/sql/sem/builtins/aggregate_builtins.go b/pkg/sql/sem/builtins/aggregate_builtins.go index c45218bd6a9..12bdb4f69c1 100644 --- a/pkg/sql/sem/builtins/aggregate_builtins.go +++ b/pkg/sql/sem/builtins/aggregate_builtins.go @@ -117,6 +117,28 @@ var aggregates = map[string]builtinDefinition{ }), ), + "array_cat_agg": setProps(aggProps(), + arrayBuiltin(func(t *types.T) tree.Overload { + tArray := types.MakeArray(t) + return makeAggOverloadWithReturnType( + []*types.T{tArray}, + func(args []tree.TypedExpr) *types.T { + if len(args) == 0 { + return tArray + } + // Whenever possible, use the expression's type, so we can + // properly handle aliased types that don't explicitly have + // overloads. + return args[0].ResolvedType() + }, + newArrayCatAggregate, + "Unnests the selected arrays into elements that are then aggregated into a single array.", + volatility.Immutable, + true, /* calledOnNullInput */ + ) + }), + ), + "avg": makeBuiltin(aggProps(), makeImmutableAggOverload([]*types.T{types.Int}, types.Decimal, newIntAvgAggregate, "Calculates the average of the selected values."), @@ -1226,6 +1248,7 @@ var _ eval.AggregateFunc = ®ressionAvgXAggregate{} var _ eval.AggregateFunc = ®ressionAvgYAggregate{} const sizeOfArrayAggregate = int64(unsafe.Sizeof(arrayAggregate{})) +const sizeOfArrayCatAggregate = int64(unsafe.Sizeof(arrayCatAggregate{})) const sizeOfAvgAggregate = int64(unsafe.Sizeof(avgAggregate{})) const sizeOfRegressionAccumulatorDecimalBase = int64(unsafe.Sizeof(regressionAccumulatorDecimalBase{})) const sizeOfFinalRegressionAccumulatorDecimalBase = int64(unsafe.Sizeof(finalRegressionAccumulatorDecimalBase{})) @@ -1473,6 +1496,78 @@ func (a *arrayAggregate) Size() int64 { return sizeOfArrayAggregate } +type arrayCatAggregate struct { + arr *tree.DArray + // Note that we do not embed singleDatumAggregateBase struct to help with + // memory accounting because arrayCatAggregate stores multiple datums + // inside of arr. + acc mon.BoundAccount + // seenNonNull tracks whether at least one non-NULL datum was added. This is + // needed to handle a case of only empty arrays added correctly (we want to + // return an empty array too rather that NULL). + seenNonNull bool +} + +func newArrayCatAggregate( + params []*types.T, evalCtx *eval.Context, _ tree.Datums, +) eval.AggregateFunc { + return &arrayCatAggregate{ + arr: tree.NewDArray(params[0].ArrayContents()), + acc: evalCtx.Mon.MakeBoundAccount(), + } +} + +// Add unnests the passed datum into elements which are then accumulated into +// the array. +func (a *arrayCatAggregate) Add(ctx context.Context, datum tree.Datum, _ ...tree.Datum) error { + if datum == tree.DNull { + // If we're given a NULL array, then we don't have any elements to + // include. + return nil + } + array, ok := datum.(*tree.DArray) + if !ok { + return errors.AssertionFailedf("got %T when *tree.DArray is expected", datum) + } + a.seenNonNull = true + for _, d := range array.Array { + if err := a.acc.Grow(ctx, int64(d.Size())); err != nil { + return err + } + if err := a.arr.Append(d); err != nil { + return err + } + } + return nil +} + +// Result returns a copy of the array of all datums passed to Add. +func (a *arrayCatAggregate) Result() (tree.Datum, error) { + if len(a.arr.Array) > 0 || a.seenNonNull { + arrCopy := *a.arr + return &arrCopy, nil + } + return tree.DNull, nil +} + +// Reset implements eval.AggregateFunc interface. +func (a *arrayCatAggregate) Reset(ctx context.Context) { + a.arr = tree.NewDArray(a.arr.ParamTyp) + a.acc.Empty(ctx) + a.seenNonNull = false +} + +// Close allows the aggregate to release the memory it requested during +// operation. +func (a *arrayCatAggregate) Close(ctx context.Context) { + a.acc.Close(ctx) +} + +// Size is part of the eval.AggregateFunc interface. +func (a *arrayCatAggregate) Size() int64 { + return sizeOfArrayCatAggregate +} + type avgAggregate struct { agg eval.AggregateFunc count int