diff --git a/pkg/sql/buffer.go b/pkg/sql/buffer.go index 6d55e9f81bfa..93f044cae0b6 100644 --- a/pkg/sql/buffer.go +++ b/pkg/sql/buffer.go @@ -29,6 +29,9 @@ type bufferNode struct { // processors, but this node is local. bufferedRows *rowcontainer.RowContainer passThruNextRowIdx int + + // label is a string used to describe the node in an EXPLAIN plan. + label string } func (n *bufferNode) startExec(params runParams) error { @@ -64,7 +67,11 @@ func (n *bufferNode) Values() tree.Datums { func (n *bufferNode) Close(ctx context.Context) { n.plan.Close(ctx) - n.bufferedRows.Close(ctx) + // It's valid to be Closed without startExec having been called, in which + // case n.bufferedRows will be nil. + if n.bufferedRows != nil { + n.bufferedRows.Close(ctx) + } } // scanBufferNode behaves like an iterator into the bufferNode it is @@ -74,6 +81,9 @@ type scanBufferNode struct { buffer *bufferNode nextRowIdx int + + // label is a string used to describe the node in an EXPLAIN plan. + label string } func (n *scanBufferNode) startExec(runParams) error { diff --git a/pkg/sql/logictest/testdata/logic_test/with b/pkg/sql/logictest/testdata/logic_test/with index fc00ba60d4ff..23f0a6872588 100644 --- a/pkg/sql/logictest/testdata/logic_test/with +++ b/pkg/sql/logictest/testdata/logic_test/with @@ -1,8 +1,5 @@ # LogicTest: local local-opt fakedist fakedist-opt fakedist-metadata -query error unsupported multiple use of CTE clause "a" -WITH a AS (SELECT 1) SELECT * FROM a CROSS JOIN a - statement ok CREATE TABLE x(a) AS SELECT generate_series(1, 3) @@ -144,41 +141,6 @@ WITH t AS ( ) SELECT * FROM t -# Regression test for #24307 until CockroachDB learns how to execute -# side effects no matter what. -query error unimplemented: common table expression "t" with side effects was not used in query -WITH t AS ( - INSERT INTO x(a) VALUES(0) RETURNING a -) -SELECT 1 - -query error unimplemented: common table expression "t" with side effects was not used in query -WITH t AS ( - SELECT * FROM ( - WITH b AS (INSERT INTO x(a) VALUES(0) RETURNING a) - TABLE b - ) -) -SELECT 1 - -query error unimplemented: common table expression "t" with side effects was not used in query -WITH t AS ( - DELETE FROM x RETURNING a -) -SELECT 1 - -query error unimplemented: common table expression "t" with side effects was not used in query -WITH t AS ( - UPSERT INTO x(a) VALUES(0) RETURNING a -) -SELECT 1 - -query error unimplemented: common table expression "t" with side effects was not used in query -WITH t AS ( - UPDATE x SET a = 0 RETURNING a -) -SELECT 1 - # however if there are no side effects, no errors are required. query I WITH t AS (SELECT 1) SELECT 2 @@ -237,3 +199,18 @@ query I ((WITH lim(x) AS (SELECT 1) SELECT 123) LIMIT (SELECT x FROM lim)) ---- 123 + +# CTE with an ORDER BY. + +statement ok +CREATE TABLE ab (a INT PRIMARY KEY, b INT) + +statement ok +INSERT INTO ab VALUES (1, 2), (3, 4), (5, 6) + +query I +WITH a AS (SELECT a FROM ab ORDER BY b) SELECT * FROM a +---- +1 +3 +5 diff --git a/pkg/sql/logictest/testdata/logic_test/with-hp b/pkg/sql/logictest/testdata/logic_test/with-hp new file mode 100644 index 000000000000..2a5e7770a35f --- /dev/null +++ b/pkg/sql/logictest/testdata/logic_test/with-hp @@ -0,0 +1,42 @@ +# LogicTest: local fakedist + +statement ok +CREATE TABLE x(a) AS SELECT generate_series(1, 3) + +statement ok +CREATE TABLE y(a) AS SELECT generate_series(2, 4) + +# Regression test for #24307 until CockroachDB learns how to execute +# side effects no matter what. +query error unimplemented: common table expression "t" with side effects was not used in query +WITH t AS ( + INSERT INTO x(a) VALUES(0) RETURNING a +) +SELECT 1 + +query error unimplemented: common table expression "t" with side effects was not used in query +WITH t AS ( + SELECT * FROM ( + WITH b AS (INSERT INTO x(a) VALUES(0) RETURNING a) + TABLE b + ) +) +SELECT 1 + +query error unimplemented: common table expression "t" with side effects was not used in query +WITH t AS ( + DELETE FROM x RETURNING a +) +SELECT 1 + +query error unimplemented: common table expression "t" with side effects was not used in query +WITH t AS ( + UPSERT INTO x(a) VALUES(0) RETURNING a +) +SELECT 1 + +query error unimplemented: common table expression "t" with side effects was not used in query +WITH t AS ( + UPDATE x SET a = 0 RETURNING a +) +SELECT 1 diff --git a/pkg/sql/logictest/testdata/logic_test/with-opt b/pkg/sql/logictest/testdata/logic_test/with-opt new file mode 100644 index 000000000000..d3267053735e --- /dev/null +++ b/pkg/sql/logictest/testdata/logic_test/with-opt @@ -0,0 +1,70 @@ +# LogicTest: local-opt fakedist-opt + +statement ok +CREATE TABLE x(a) AS SELECT generate_series(1, 3) + +statement ok +CREATE TABLE y(b) AS SELECT generate_series(2, 4) + +# Referencing a CTE multiple times. +query II +WITH t AS (SELECT b FROM y) SELECT * FROM t JOIN t AS q ON true +---- +2 2 +2 3 +2 4 +3 2 +3 3 +3 4 +4 2 +4 3 +4 4 + +query II +WITH + one AS (SELECT a AS u FROM x), + two AS (SELECT b AS v FROM (SELECT b FROM y UNION ALL SELECT u FROM one)) +SELECT + * +FROM + one JOIN two ON u = v +---- +1 1 +2 2 +3 3 +2 2 +3 3 + +# Mutation CTEs that aren't referenced elsewhere in the query. +statement ok +CREATE TABLE z (c INT PRIMARY KEY); + +query I +WITH foo AS (INSERT INTO z VALUES (10) RETURNING 1) SELECT 2 +---- +2 + +query I +SELECT * FROM z +---- +10 + +query I +WITH foo AS (UPDATE z SET c = 20 RETURNING 1) SELECT 3 +---- +3 + +query I +SELECT * FROM z +---- +20 + +query I +WITH foo AS (DELETE FROM z RETURNING 1) SELECT 4 +---- +4 + +query I +SELECT count(*) FROM z +---- +0 diff --git a/pkg/sql/opt/bench/stub_factory.go b/pkg/sql/opt/bench/stub_factory.go index 10aad2b9fb90..6e1243b5beed 100644 --- a/pkg/sql/opt/bench/stub_factory.go +++ b/pkg/sql/opt/bench/stub_factory.go @@ -315,3 +315,11 @@ func (f *stubFactory) ConstructAlterTableRelocate( ) (exec.Node, error) { return struct{}{}, nil } + +func (f *stubFactory) ConstructBuffer(value exec.Node, label string) (exec.Node, error) { + return struct{}{}, nil +} + +func (f *stubFactory) ConstructScanBuffer(ref exec.Node, label string) (exec.Node, error) { + return struct{}{}, nil +} diff --git a/pkg/sql/opt/exec/execbuilder/builder.go b/pkg/sql/opt/exec/execbuilder/builder.go index 586d81f67b38..96c88044e6d3 100644 --- a/pkg/sql/opt/exec/execbuilder/builder.go +++ b/pkg/sql/opt/exec/execbuilder/builder.go @@ -50,6 +50,22 @@ type Builder struct { // each relational subexpression when evalCtx.SessionData.SaveTablesPrefix is // non-empty. nameGen *memo.ExprNameGenerator + + // withExprs is the set of With expressions which may be referenced elsewhere + // in the query. + // TODO(justin): set this up so that we can look them up by index lookups + // rather than scans. + withExprs []builtWithExpr +} + +// builtWithExpr is metadata regarding a With expression which has already been +// added to the set of subqueries for the query. +type builtWithExpr struct { + id opt.WithID + // outputCols maps the output ColumnIDs of the With expression to the ordinal + // positions they are output to. See execPlan.outputCols for more details. + outputCols opt.ColMap + bufferNode exec.Node } // New constructs an instance of the execution node builder using the diff --git a/pkg/sql/opt/exec/execbuilder/relational.go b/pkg/sql/opt/exec/execbuilder/relational.go index 781d2ae8f4a1..23e1471023fc 100644 --- a/pkg/sql/opt/exec/execbuilder/relational.go +++ b/pkg/sql/opt/exec/execbuilder/relational.go @@ -11,6 +11,7 @@ package execbuilder import ( + "bytes" "fmt" "github.com/cockroachdb/cockroach/pkg/server/telemetry" @@ -237,6 +238,12 @@ func (b *Builder) buildRelational(e memo.RelExpr) (execPlan, error) { case *memo.CreateTableExpr: ep, err = b.buildCreateTable(t) + case *memo.WithExpr: + ep, err = b.buildWith(t) + + case *memo.WithScanExpr: + ep, err = b.buildWithScan(t) + case *memo.ExplainExpr: ep, err = b.buildExplain(t) @@ -1321,6 +1328,97 @@ func (b *Builder) buildMax1Row(max1Row *memo.Max1RowExpr) (execPlan, error) { } +func (b *Builder) buildWith(with *memo.WithExpr) (execPlan, error) { + value, err := b.buildRelational(with.Binding) + if err != nil { + return execPlan{}, err + } + + var label bytes.Buffer + fmt.Fprintf(&label, "buffer %d", with.ID) + if with.Name != "" { + fmt.Fprintf(&label, " (%s)", with.Name) + } + + buffer, err := b.factory.ConstructBuffer(value.root, label.String()) + if err != nil { + return execPlan{}, err + } + + // TODO(justin): if the binding here has a spoolNode at its root, we can + // remove it, since subquery execution also guarantees complete execution. + + // Add the buffer as a subquery so it gets executed ahead of time, and is + // available to be referenced by other queries. + b.subqueries = append(b.subqueries, exec.Subquery{ + ExprNode: &tree.Subquery{ + Select: with.OriginalExpr.Select, + }, + // TODO(justin): this is wasteful: both the subquery and the bufferNode + // will buffer up all the results. This should be fixed by either making + // the buffer point directly to the subquery results or adding a new + // subquery mode that reads and discards all rows. This could possibly also + // be fixed by ensuring that bufferNode exhausts its input (and forcing it + // to behave like a spoolNode) and using the EXISTS mode. + Mode: exec.SubqueryAllRows, + Root: buffer, + }) + + b.withExprs = append(b.withExprs, builtWithExpr{ + id: with.ID, + outputCols: value.outputCols, + bufferNode: buffer, + }) + + return b.buildRelational(with.Input) +} + +func (b *Builder) buildWithScan(withScan *memo.WithScanExpr) (execPlan, error) { + id := withScan.ID + var e *builtWithExpr + for i := range b.withExprs { + if b.withExprs[i].id == id { + e = &b.withExprs[i] + break + } + } + if e == nil { + panic(errors.AssertionFailedf("couldn't find With expression with ID %d", id)) + } + + var label bytes.Buffer + fmt.Fprintf(&label, "buffer %d", withScan.ID) + if withScan.Name != "" { + fmt.Fprintf(&label, " (%s)", withScan.Name) + } + + node, err := b.factory.ConstructScanBuffer(e.bufferNode, label.String()) + if err != nil { + return execPlan{}, err + } + + // The ColumnIDs from the With expression need to get remapped according to + // the mapping in the withScan to get the actual colMap for this expression. + var outputCols opt.ColMap + + referencedExpr := b.mem.WithExpr(withScan.ID) + if !referencedExpr.Relational().OutputCols.Equals(withScan.InCols.ToSet()) { + panic(errors.AssertionFailedf( + "columns being output from WITH do not match expected columns", + )) + } + + for i := range withScan.InCols { + idx, _ := e.outputCols.Get(int(withScan.InCols[i])) + outputCols.Set(int(withScan.OutCols[i]), idx) + } + + return execPlan{ + root: node, + outputCols: outputCols, + }, nil +} + func (b *Builder) buildProjectSet(projectSet *memo.ProjectSetExpr) (execPlan, error) { input, err := b.buildRelational(projectSet.Input) if err != nil { diff --git a/pkg/sql/opt/exec/execbuilder/testdata/spool b/pkg/sql/opt/exec/execbuilder/testdata/spool index 8382ad3ce3a8..f1185e2e688e 100644 --- a/pkg/sql/opt/exec/execbuilder/testdata/spool +++ b/pkg/sql/opt/exec/execbuilder/testdata/spool @@ -11,67 +11,99 @@ query TTT EXPLAIN WITH a AS (INSERT INTO t SELECT * FROM t2 RETURNING x) SELECT * FROM a LIMIT 1 ---- -limit · · - │ count 1 - └── spool · · - │ limit 1 - └── run · · - └── insert · · - │ into t(x) - │ strategy inserter - └── scan · · -· table t2@primary -· spans ALL +root · · + ├── limit · · + │ │ count 1 + │ └── scan buffer node · · + │ label buffer 1 (a) + └── subquery · · + │ id @S1 + │ original sql + │ exec mode all rows + └── buffer node · · + │ label buffer 1 (a) + └── spool · · + └── run · · + └── insert · · + │ into t(x) + │ strategy inserter + └── scan · · +· table t2@primary +· spans ALL query TTT EXPLAIN WITH a AS (DELETE FROM t RETURNING x) SELECT * FROM a LIMIT 1 ---- -limit · · - │ count 1 - └── spool · · - │ limit 1 - └── run · · - └── delete · · - │ from t - │ strategy deleter - └── scan · · -· table t@primary -· spans ALL +root · · + ├── limit · · + │ │ count 1 + │ └── scan buffer node · · + │ label buffer 1 (a) + └── subquery · · + │ id @S1 + │ original sql + │ exec mode all rows + └── buffer node · · + │ label buffer 1 (a) + └── spool · · + └── run · · + └── delete · · + │ from t + │ strategy deleter + └── scan · · +· table t@primary +· spans ALL query TTT EXPLAIN WITH a AS (UPDATE t SET x = x + 1 RETURNING x) SELECT * FROM a LIMIT 1 ---- -limit · · - │ count 1 - └── spool · · - │ limit 1 - └── run · · - └── update · · - │ table t - │ set x - │ strategy updater - └── render · · - └── scan · · -· table t@primary -· spans ALL +root · · + ├── limit · · + │ │ count 1 + │ └── scan buffer node · · + │ label buffer 1 (a) + └── subquery · · + │ id @S1 + │ original sql + │ exec mode all rows + └── buffer node · · + │ label buffer 1 (a) + └── spool · · + └── run · · + └── update · · + │ table t + │ set x + │ strategy updater + └── render · · + └── scan · · +· table t@primary +· spans ALL query TTT EXPLAIN WITH a AS (UPSERT INTO t VALUES (2), (3) RETURNING x) SELECT * FROM a LIMIT 1 ---- -limit · · - │ count 1 - └── spool · · - │ limit 1 - └── run · · - └── upsert · · - │ into t(x) - │ strategy opt upserter - └── values · · -· size 1 column, 2 rows +root · · + ├── limit · · + │ │ count 1 + │ └── scan buffer node · · + │ label buffer 1 (a) + └── subquery · · + │ id @S1 + │ original sql + │ exec mode all rows + └── buffer node · · + │ label buffer 1 (a) + └── spool · · + └── run · · + └── upsert · · + │ into t(x) + │ strategy opt upserter + └── values · · +· size 1 column, 2 rows # Ditto all mutations, with the statement source syntax. query TTT @@ -177,23 +209,39 @@ EXPLAIN WITH a AS (INSERT INTO t SELECT * FROM t2 RETURNING x), b AS (INSERT INTO t SELECT x+1 FROM a RETURNING x) SELECT * FROM b LIMIT 1 ---- -limit · · - │ count 1 - └── spool · · - │ limit 1 - └── run · · - └── insert · · - │ into t(x) - │ strategy inserter - └── spool · · - └── render · · - └── run · · - └── insert · · - │ into t(x) - │ strategy inserter - └── scan · · -· table t2@primary -· spans ALL +root · · + ├── limit · · + │ │ count 1 + │ └── scan buffer node · · + │ label buffer 2 (b) + ├── subquery · · + │ │ id @S1 + │ │ original sql + │ │ exec mode all rows + │ └── buffer node · · + │ │ label buffer 1 (a) + │ └── spool · · + │ └── run · · + │ └── insert · · + │ │ into t(x) + │ │ strategy inserter + │ └── scan · · + │ table t2@primary + │ spans ALL + └── subquery · · + │ id @S2 + │ original sql + │ exec mode all rows + └── buffer node · · + │ label buffer 2 (b) + └── spool · · + └── run · · + └── insert · · + │ into t(x) + │ strategy inserter + └── render · · + └── scan buffer node · · +· label buffer 1 (a) # Check that no spool is inserted if a top-level render is elided. query TTT diff --git a/pkg/sql/opt/exec/execbuilder/testdata/with b/pkg/sql/opt/exec/execbuilder/testdata/with new file mode 100644 index 000000000000..f601cd13fef5 --- /dev/null +++ b/pkg/sql/opt/exec/execbuilder/testdata/with @@ -0,0 +1,70 @@ +# LogicTest: local-opt + +statement ok +CREATE TABLE x(a INT) + +statement ok +CREATE TABLE y(a INT) + +query TTTTT +EXPLAIN (VERBOSE) + WITH t AS (SELECT a FROM y) SELECT * FROM t JOIN t AS q ON true +---- +root · · (a, a) · + ├── hash-join · · (a, a) · + │ │ type cross · · + │ ├── scan buffer node · · (a) · + │ │ label buffer 1 (t) · · + │ └── scan buffer node · · (a) · + │ label buffer 1 (t) · · + └── subquery · · (a, a) · + │ id @S1 · · + │ original sql SELECT a FROM y · · + │ exec mode all rows · · + └── buffer node · · (a) · + │ label buffer 1 (t) · · + └── scan · · (a) · +· table y@primary · · +· spans ALL · · + +query TTTTT +EXPLAIN (VERBOSE) + WITH t AS (SELECT a FROM y) SELECT * FROM t +---- +root · · (a) · + ├── scan buffer node · · (a) · + │ label buffer 1 (t) · · + └── subquery · · (a) · + │ id @S1 · · + │ original sql SELECT a FROM y · · + │ exec mode all rows · · + └── buffer node · · (a) · + │ label buffer 1 (t) · · + └── scan · · (a) · +· table y@primary · · +· spans ALL · · + +query TTTTT +EXPLAIN (VERBOSE) + WITH t AS (INSERT INTO x VALUES (1) RETURNING a) SELECT * FROM t +---- +root · · (a) · + ├── scan buffer node · · (a) · + │ label buffer 1 (t) · · + └── subquery · · (a) · + │ id @S1 · · + │ original sql · · + │ exec mode all rows · · + └── buffer node · · (a) · + │ label buffer 1 (t) · · + └── spool · · (a) · + └── render · · (a) · + │ render 0 a · · + └── run · · (a, rowid[hidden]) · + └── insert · · (a, rowid[hidden]) · + │ into x(a, rowid) · · + │ strategy inserter · · + └── values · · (column1, column4) · +· size 2 columns, 1 row · · +· row 0, expr 0 1 · · +· row 0, expr 1 unique_rowid() · · diff --git a/pkg/sql/opt/exec/factory.go b/pkg/sql/opt/exec/factory.go index be7e6a8a596f..3d301f9e828f 100644 --- a/pkg/sql/opt/exec/factory.go +++ b/pkg/sql/opt/exec/factory.go @@ -422,6 +422,14 @@ type Factory interface { // ConstructAlterTableRelocate creates a node that implements ALTER TABLE/INDEX // UNSPLIT AT. ConstructAlterTableRelocate(index cat.Index, input Node, relocateLease bool) (Node, error) + + // ConstructBuffer constructs a node whose input can be referenced from + // elsewhere in the query. + ConstructBuffer(value Node, label string) (Node, error) + + // ConstructScanBuffer constructs a node which refers to a node constructed by + // ConstructBuffer. + ConstructScanBuffer(ref Node, label string) (Node, error) } // OutputOrdering indicates the required output ordering on a Node that is being diff --git a/pkg/sql/opt/memo/expr_format.go b/pkg/sql/opt/memo/expr_format.go index 2468dcbd184e..7291087fcb5e 100644 --- a/pkg/sql/opt/memo/expr_format.go +++ b/pkg/sql/opt/memo/expr_format.go @@ -198,6 +198,21 @@ func (f *ExprFmtCtx) formatRelational(e RelExpr, tp treeprinter.Node) { } else { fmt.Fprintf(f.Buffer, "%v (segmented)", e.Op()) } + + case *WithExpr: + w := e.(*WithExpr) + fmt.Fprintf(f.Buffer, "%v &%d", e.Op(), w.ID) + if w.Name != "" { + fmt.Fprintf(f.Buffer, " (%s)", w.Name) + } + + case *WithScanExpr: + ws := e.(*WithScanExpr) + fmt.Fprintf(f.Buffer, "%v &%d", e.Op(), ws.ID) + if ws.Name != "" { + fmt.Fprintf(f.Buffer, " (%s)", ws.Name) + } + default: fmt.Fprintf(f.Buffer, "%v", e.Op()) if opt.IsJoinNonApplyOp(t) { @@ -384,6 +399,18 @@ func (f *ExprFmtCtx) formatRelational(e RelExpr, tp treeprinter.Node) { f.formatColList(e, tp, "fetch columns:", t.FetchCols) } + case *WithScanExpr: + if !f.HasFlags(ExprFmtHideColumns) { + child := tp.Child("mapping:") + for i := range t.InCols { + f.Buffer.Reset() + formatCol(f, "" /* label */, t.InCols[i], opt.ColSet{}, false /* omitType */) + f.Buffer.WriteString(" =>") + formatCol(f, "" /* label */, t.OutCols[i], opt.ColSet{}, false /* omitType */) + child.Child(f.Buffer.String()) + } + } + case *CreateTableExpr: tp.Child(t.Syntax.String()) diff --git a/pkg/sql/opt/memo/interner.go b/pkg/sql/opt/memo/interner.go index ad8fde33a0d2..3c468bf8da59 100644 --- a/pkg/sql/opt/memo/interner.go +++ b/pkg/sql/opt/memo/interner.go @@ -460,6 +460,10 @@ func (h *hasher) HashValuesID(val opt.ValuesID) { h.HashUint64(uint64(val)) } +func (h *hasher) HashWithID(val opt.WithID) { + h.HashUint64(uint64(val)) +} + func (h *hasher) HashScanLimit(val ScanLimit) { h.HashUint64(uint64(val)) } @@ -752,6 +756,10 @@ func (h *hasher) IsValuesIDEqual(l, r opt.ValuesID) bool { return l == r } +func (h *hasher) IsWithIDEqual(l, r opt.WithID) bool { + return l == r +} + func (h *hasher) IsScanLimitEqual(l, r ScanLimit) bool { return l == r } diff --git a/pkg/sql/opt/memo/logical_props_builder.go b/pkg/sql/opt/memo/logical_props_builder.go index 4b4d99fc52e6..9ad1a1fa573e 100644 --- a/pkg/sql/opt/memo/logical_props_builder.go +++ b/pkg/sql/opt/memo/logical_props_builder.go @@ -715,6 +715,85 @@ func (b *logicalPropsBuilder) buildBasicProps(e opt.Expr, cols opt.ColList, rel } } +func (b *logicalPropsBuilder) buildWithProps(with *WithExpr, rel *props.Relational) { + BuildSharedProps(b.mem, with, &rel.Shared) + + // Copy over the props from the input. + *rel = *with.Input.Relational() + + // Side Effects + // ------------ + // This expression has side effects if either Binding or Input has side + // effects, which is what is computed by the call to BuildSharedProps. + + // Output Columns + // -------------- + // Passed through from the call above to b.buildProps. + + // Not Null Columns + // ---------------- + // Passed through from the call above to b.buildProps. + + // Outer Columns + // ------------- + // Passed through from the call above to b.buildProps. + + // Functional Dependencies + // ----------------------- + // Passed through from the call above to b.buildProps. + + // Cardinality + // ----------- + // Passed through from the call above to b.buildProps. + + // Statistics + // ---------- + if !b.disableStats { + b.sb.statsFromChild(with, 1) + } +} + +func (b *logicalPropsBuilder) buildWithScanProps(ref *WithScanExpr, rel *props.Relational) { + e := b.mem.WithExpr(ref.ID) + + // WithScan inherits most of the logical properties of the expression it + // references. + *rel = *e.Relational() + + // Side Effects + // ------------ + // TODO(justin): these shouldn't have side-effects, but mutating that here + // has complications with the way that shared props are built. + + // Output Columns + // -------------- + rel.OutputCols = ref.OutCols.ToSet() + + // Not Null Columns + // ---------------- + rel.NotNullCols = translateColSet(rel.NotNullCols, ref.InCols, ref.OutCols) + + // Outer Columns + // ------------- + // Copied from the referenced expression. + + // Functional Dependencies + // ----------------------- + rel.FuncDeps = props.FuncDepSet{} + rel.FuncDeps.CopyFrom(&e.Relational().FuncDeps) + for i := range ref.InCols { + rel.FuncDeps.AddSynthesizedCol(opt.MakeColSet(ref.InCols[i]), ref.OutCols[i]) + } + + // Cardinality + // ----------- + // Copied from the referenced expression. + + // Statistics + // ---------- + // Copied from the referenced expression. +} + func (b *logicalPropsBuilder) buildExplainProps(explain *ExplainExpr, rel *props.Relational) { b.buildBasicProps(explain, explain.ColList, rel) } diff --git a/pkg/sql/opt/memo/memo.go b/pkg/sql/opt/memo/memo.go index db85409c319f..f0e056000404 100644 --- a/pkg/sql/opt/memo/memo.go +++ b/pkg/sql/opt/memo/memo.go @@ -142,6 +142,10 @@ type Memo struct { // curID is the highest currently in-use scalar expression ID. curID opt.ScalarID + // withExprs is the set of With expressions that have been constructed thus + // far. + withExprs []RelExpr + // WARNING: if you add more members, add initialization code in Init. } @@ -158,6 +162,7 @@ func (m *Memo) Init(evalCtx *tree.EvalContext) { m.rootExpr = nil m.rootProps = nil m.memEstimate = 0 + m.withExprs = nil m.dataConversion = evalCtx.SessionData.DataConversion m.reorderJoinsLimit = evalCtx.SessionData.ReorderJoinsLimit @@ -353,3 +358,16 @@ func (m *Memo) RequestColStat( } return nil, false } + +// AddWithBinding adds a new expression to the set of referenceable With +// expressions, returning its associated ID that can be used to retrieve it. +func (m *Memo) AddWithBinding(e RelExpr) opt.WithID { + m.withExprs = append(m.withExprs, e) + return opt.WithID(len(m.withExprs)) +} + +// WithExpr returns the expression associated with the given WithID. +// This shouldn't be used for anything relating to physical props. +func (m *Memo) WithExpr(id opt.WithID) RelExpr { + return m.withExprs[id-1] +} diff --git a/pkg/sql/opt/memo/statistics_builder.go b/pkg/sql/opt/memo/statistics_builder.go index 85bc1ecd4614..4caddf0b077a 100644 --- a/pkg/sql/opt/memo/statistics_builder.go +++ b/pkg/sql/opt/memo/statistics_builder.go @@ -345,6 +345,12 @@ func (sb *statisticsBuilder) colStat(colSet opt.ColSet, e RelExpr) *props.Column case opt.ExplainOp, opt.ShowTraceForSessionOp, opt.OpaqueRelOp: return sb.colStatUnknown(colSet, e.Relational()) + case opt.WithOp: + return sb.colStat(colSet, e.Child(1).(RelExpr)) + + case opt.WithScanOp: + return sb.colStatWithScan(colSet, e.(*WithScanExpr)) + case opt.FakeRelOp: panic(errors.AssertionFailedf("FakeRelOp does not contain col stat for %v", colSet)) } @@ -2050,6 +2056,30 @@ func (sb *statisticsBuilder) colStatSequenceSelect( return colStat } +// +-----------+ +// | With Scan | +// +-----------+ + +func (sb *statisticsBuilder) colStatWithScan( + colSet opt.ColSet, ws *WithScanExpr, +) *props.ColumnStatistic { + relProps := ws.Relational() + s := &relProps.Stats + + withExpr := ws.Memo().WithExpr(ws.ID) + + // We need to pass on the colStat request to the referenced expression, but + // we need to translate the columns to the ones returned by the original + // expression, rather than the reference. + cols := translateColSet(colSet, ws.OutCols, ws.InCols) + + colstat, _ := s.ColStats.Add(colSet) + *colstat = *sb.colStat(cols, withExpr) + colstat.Cols = colSet + + return colstat +} + // +---------+ // | Unknown | // +---------+ diff --git a/pkg/sql/opt/memo/testdata/logprops/with b/pkg/sql/opt/memo/testdata/logprops/with new file mode 100644 index 000000000000..653eb5a03e0c --- /dev/null +++ b/pkg/sql/opt/memo/testdata/logprops/with @@ -0,0 +1,101 @@ +exec-ddl +CREATE TABLE xy (x INT PRIMARY KEY, y INT) +---- + +build +WITH foo AS (SELECT * FROM xy) SELECT * FROM foo +---- +with &1 (foo) + ├── columns: x:3(int!null) y:4(int) + ├── key: (1) + ├── fd: (1)-->(2,3), (2)-->(4) + ├── scan xy + │ ├── columns: xy.x:1(int!null) xy.y:2(int) + │ ├── key: (1) + │ ├── fd: (1)-->(2) + │ ├── prune: (1,2) + │ └── interesting orderings: (+1) + └── with-scan &1 (foo) + ├── columns: x:3(int!null) y:4(int) + ├── mapping: + │ ├── xy.x:1(int) => x:3(int) + │ └── xy.y:2(int) => y:4(int) + ├── key: (1) + └── fd: (1)-->(2,3), (2)-->(4) + +# Side effects should not be propagated up to the top-level from the Value +# side of a WITH. +build +WITH foo AS (SELECT 1/0) SELECT * FROM foo +---- +with &1 (foo) + ├── columns: "?column?":2(decimal) + ├── cardinality: [1 - 1] + ├── side-effects + ├── key: () + ├── fd: ()-->(1), (1)-->(2) + ├── project + │ ├── columns: "?column?":1(decimal) + │ ├── cardinality: [1 - 1] + │ ├── side-effects + │ ├── key: () + │ ├── fd: ()-->(1) + │ ├── prune: (1) + │ ├── values + │ │ ├── cardinality: [1 - 1] + │ │ ├── key: () + │ │ └── tuple [type=tuple] + │ └── projections + │ └── div [type=decimal, side-effects] + │ ├── const: 1 [type=int] + │ └── const: 0 [type=int] + └── with-scan &1 (foo) + ├── columns: "?column?":2(decimal) + ├── mapping: + │ └── "?column?":1(decimal) => "?column?":2(decimal) + ├── cardinality: [1 - 1] + ├── side-effects + ├── key: () + └── fd: ()-->(1), (1)-->(2) + +# Side effects should be propagated up to the top-level from the Input side of +# a With. +build +WITH foo AS (SELECT 1) SELECT 1/0 FROM foo +---- +with &1 (foo) + ├── columns: "?column?":3(decimal) + ├── cardinality: [1 - 1] + ├── side-effects + ├── key: () + ├── fd: ()-->(3) + ├── project + │ ├── columns: "?column?":1(int!null) + │ ├── cardinality: [1 - 1] + │ ├── key: () + │ ├── fd: ()-->(1) + │ ├── prune: (1) + │ ├── values + │ │ ├── cardinality: [1 - 1] + │ │ ├── key: () + │ │ └── tuple [type=tuple] + │ └── projections + │ └── const: 1 [type=int] + └── project + ├── columns: "?column?":3(decimal) + ├── cardinality: [1 - 1] + ├── side-effects + ├── key: () + ├── fd: ()-->(3) + ├── prune: (3) + ├── with-scan &1 (foo) + │ ├── columns: "?column?":2(int!null) + │ ├── mapping: + │ │ └── "?column?":1(int) => "?column?":2(int) + │ ├── cardinality: [1 - 1] + │ ├── key: () + │ └── fd: ()-->(1), (1)-->(2) + └── projections + └── div [type=decimal, side-effects] + ├── const: 1 [type=int] + └── const: 0 [type=int] diff --git a/pkg/sql/opt/memo/testdata/stats/groupby b/pkg/sql/opt/memo/testdata/stats/groupby index df8fbdf0f33e..a89172d1e28d 100644 --- a/pkg/sql/opt/memo/testdata/stats/groupby +++ b/pkg/sql/opt/memo/testdata/stats/groupby @@ -465,42 +465,55 @@ WITH q (a, b) AS (SELECT * FROM (VALUES (true, NULL), (false, NULL), (true, 5))) GROUP BY q.b HAVING bool_or(q.a) ---- -project - ├── columns: "?column?":4(int!null) +with &1 (q) + ├── columns: "?column?":6(int!null) ├── cardinality: [0 - 3] ├── stats: [rows=0.22654092] - ├── fd: ()-->(4) - ├── select - │ ├── columns: column2:2(int) bool_or:3(bool!null) - │ ├── cardinality: [0 - 3] - │ ├── stats: [rows=0.22654092, distinct(3)=0.22654092, null(3)=0] - │ ├── key: (2) - │ ├── fd: ()-->(3) - │ ├── group-by - │ │ ├── columns: column2:2(int) bool_or:3(bool) - │ │ ├── grouping columns: column2:2(int) - │ │ ├── cardinality: [0 - 3] - │ │ ├── stats: [rows=1.29289322, distinct(2)=1.29289322, null(2)=1, distinct(3)=1.29289322, null(3)=1] - │ │ ├── key: (2) - │ │ ├── fd: (2)-->(3) - │ │ ├── select - │ │ │ ├── columns: column1:1(bool!null) column2:2(int) - │ │ │ ├── cardinality: [0 - 3] - │ │ │ ├── stats: [rows=1.5, distinct(1)=1, null(1)=0, distinct(2)=1.29289322, null(2)=1] - │ │ │ ├── fd: ()-->(1) - │ │ │ ├── values - │ │ │ │ ├── columns: column1:1(bool!null) column2:2(int) - │ │ │ │ ├── cardinality: [3 - 3] - │ │ │ │ ├── stats: [rows=3, distinct(1)=2, null(1)=0, distinct(2)=2, null(2)=2] - │ │ │ │ ├── (true, NULL) [type=tuple{bool, int}] - │ │ │ │ ├── (false, NULL) [type=tuple{bool, int}] - │ │ │ │ └── (true, 5) [type=tuple{bool, int}] - │ │ │ └── filters - │ │ │ └── variable: column1 [type=bool, outer=(1), constraints=(/1: [/true - /true]; tight), fd=()-->(1)] - │ │ └── aggregations - │ │ └── bool-or [type=bool, outer=(1)] - │ │ └── variable: column1 [type=bool] - │ └── filters - │ └── variable: bool_or [type=bool, outer=(3), constraints=(/3: [/true - /true]; tight), fd=()-->(3)] - └── projections - └── const: 1 [type=int] + ├── fd: ()-->(6) + ├── values + │ ├── columns: column1:1(bool!null) column2:2(int) + │ ├── cardinality: [3 - 3] + │ ├── stats: [rows=3, distinct(1)=2, null(1)=0, distinct(2)=2, null(2)=2] + │ ├── (true, NULL) [type=tuple{bool, int}] + │ ├── (false, NULL) [type=tuple{bool, int}] + │ └── (true, 5) [type=tuple{bool, int}] + └── project + ├── columns: "?column?":6(int!null) + ├── cardinality: [0 - 3] + ├── stats: [rows=0.22654092] + ├── fd: ()-->(6) + ├── select + │ ├── columns: b:4(int) bool_or:5(bool!null) + │ ├── cardinality: [0 - 3] + │ ├── stats: [rows=0.22654092, distinct(5)=0.22654092, null(5)=0] + │ ├── key: (4) + │ ├── fd: ()-->(5) + │ ├── group-by + │ │ ├── columns: b:4(int) bool_or:5(bool) + │ │ ├── grouping columns: b:4(int) + │ │ ├── cardinality: [0 - 3] + │ │ ├── stats: [rows=1.29289322, distinct(4)=1.29289322, null(4)=1, distinct(5)=1.29289322, null(5)=1] + │ │ ├── key: (4) + │ │ ├── fd: (4)-->(5) + │ │ ├── select + │ │ │ ├── columns: a:3(bool!null) b:4(int) + │ │ │ ├── cardinality: [0 - 3] + │ │ │ ├── stats: [rows=1.5, distinct(3)=1, null(3)=0, distinct(4)=1.29289322, null(4)=1] + │ │ │ ├── fd: ()-->(3) + │ │ │ ├── with-scan &1 (q) + │ │ │ │ ├── columns: a:3(bool!null) b:4(int) + │ │ │ │ ├── mapping: + │ │ │ │ │ ├── column1:1(bool) => a:3(bool) + │ │ │ │ │ └── column2:2(int) => b:4(int) + │ │ │ │ ├── cardinality: [3 - 3] + │ │ │ │ ├── stats: [rows=3, distinct(3)=2, null(3)=0, distinct(4)=2, null(4)=2] + │ │ │ │ └── fd: (1)-->(3), (2)-->(4) + │ │ │ └── filters + │ │ │ └── variable: a [type=bool, outer=(3), constraints=(/3: [/true - /true]; tight), fd=()-->(3)] + │ │ └── aggregations + │ │ └── bool-or [type=bool, outer=(3)] + │ │ └── variable: a [type=bool] + │ └── filters + │ └── variable: bool_or [type=bool, outer=(5), constraints=(/5: [/true - /true]; tight), fd=()-->(5)] + └── projections + └── const: 1 [type=int] diff --git a/pkg/sql/opt/memo/testdata/stats/scan b/pkg/sql/opt/memo/testdata/stats/scan index 15738a6bb1e5..dd126d6aa422 100644 --- a/pkg/sql/opt/memo/testdata/stats/scan +++ b/pkg/sql/opt/memo/testdata/stats/scan @@ -537,34 +537,45 @@ FROM WHERE subq.col1; ---- -project - ├── columns: "?column?":24(int!null) +with &1 (subq) + ├── columns: "?column?":26(int!null) ├── stats: [rows=0.95099005] - ├── fd: ()-->(24) - ├── select - │ ├── columns: col1:23(bool!null) - │ ├── stats: [rows=0.95099005, distinct(23)=0.95099005, null(23)=0] - │ ├── fd: ()-->(23) - │ ├── project - │ │ ├── columns: col1:23(bool) - │ │ ├── stats: [rows=333333.333, distinct(23)=333333.333, null(23)=16336.65] - │ │ ├── inner-join (hash) - │ │ │ ├── columns: tab0.e:5(varchar) tab0.f:6("char") tab0.h:8(varchar) tab0.j:10(float!null) tab1.e:16(varchar) tab1.f:17("char") tab1.j:21(float!null) - │ │ │ ├── stats: [rows=333333.333, distinct(10)=100, null(10)=0, distinct(21)=100, null(21)=0, distinct(5,6,8,16,17)=333333.333, null(5,6,8,16,17)=16336.65] - │ │ │ ├── scan tab0 - │ │ │ │ ├── columns: tab0.e:5(varchar) tab0.f:6("char") tab0.h:8(varchar) tab0.j:10(float!null) - │ │ │ │ └── stats: [rows=1000, distinct(10)=100, null(10)=0, distinct(5,6,8)=1000, null(5,6,8)=29.701] - │ │ │ ├── scan tab1 - │ │ │ │ ├── columns: tab1.e:16(varchar) tab1.f:17("char") tab1.j:21(float!null) - │ │ │ │ └── stats: [rows=1000, distinct(21)=100, null(21)=0, distinct(16,17)=1000, null(16,17)=19.9] - │ │ │ └── filters - │ │ │ └── tab0.j IN (tab1.j,) [type=bool, outer=(10,21)] - │ │ └── projections - │ │ └── CASE WHEN ilike_escape(regexp_replace(tab0.h, tab1.e, tab0.f, tab0.e::STRING), tab1.f, '') THEN true ELSE false END [type=bool, outer=(5,6,8,16,17)] - │ └── filters - │ └── variable: col1 [type=bool, outer=(23), constraints=(/23: [/true - /true]; tight), fd=()-->(23)] - └── projections - └── const: 1 [type=int] + ├── fd: ()-->(26) + ├── project + │ ├── columns: col1:23(bool) tab1.g:18(int4!null) + │ ├── stats: [rows=333333.333, distinct(18)=100, null(18)=0, distinct(23)=333333.333, null(23)=16336.65] + │ ├── inner-join (hash) + │ │ ├── columns: tab0.e:5(varchar) tab0.f:6("char") tab0.h:8(varchar) tab0.j:10(float!null) tab1.e:16(varchar) tab1.f:17("char") tab1.g:18(int4!null) tab1.j:21(float!null) + │ │ ├── stats: [rows=333333.333, distinct(10)=100, null(10)=0, distinct(18)=100, null(18)=0, distinct(21)=100, null(21)=0, distinct(5,6,8,16,17)=333333.333, null(5,6,8,16,17)=16336.65] + │ │ ├── scan tab0 + │ │ │ ├── columns: tab0.e:5(varchar) tab0.f:6("char") tab0.h:8(varchar) tab0.j:10(float!null) + │ │ │ └── stats: [rows=1000, distinct(10)=100, null(10)=0, distinct(5,6,8)=1000, null(5,6,8)=29.701] + │ │ ├── scan tab1 + │ │ │ ├── columns: tab1.e:16(varchar) tab1.f:17("char") tab1.g:18(int4!null) tab1.j:21(float!null) + │ │ │ └── stats: [rows=1000, distinct(18)=100, null(18)=0, distinct(21)=100, null(21)=0, distinct(16,17)=1000, null(16,17)=19.9] + │ │ └── filters + │ │ └── tab0.j IN (tab1.j,) [type=bool, outer=(10,21)] + │ └── projections + │ └── CASE WHEN ilike_escape(regexp_replace(tab0.h, tab1.e, tab0.f, tab0.e::STRING), tab1.f, '') THEN true ELSE false END [type=bool, outer=(5,6,8,16,17)] + └── project + ├── columns: "?column?":26(int!null) + ├── stats: [rows=0.95099005] + ├── fd: ()-->(26) + ├── select + │ ├── columns: col0:24(int4!null) col1:25(bool!null) + │ ├── stats: [rows=0.95099005, distinct(24)=0.95099005, null(24)=0, distinct(25)=0.95099005, null(25)=0] + │ ├── fd: ()-->(25) + │ ├── with-scan &1 (subq) + │ │ ├── columns: col0:24(int4!null) col1:25(bool) + │ │ ├── mapping: + │ │ │ ├── tab1.g:18(int4) => col0:24(int4) + │ │ │ └── col1:23(bool) => col1:25(bool) + │ │ ├── stats: [rows=333333.333, distinct(24)=100, null(24)=0, distinct(25)=333333.333, null(25)=16336.65] + │ │ └── fd: (18)-->(24), (23)-->(25) + │ └── filters + │ └── variable: col1 [type=bool, outer=(25), constraints=(/25: [/true - /true]; tight), fd=()-->(25)] + └── projections + └── const: 1 [type=int] # --------------------- # Tests with Histograms diff --git a/pkg/sql/opt/memo/testdata/stats/select b/pkg/sql/opt/memo/testdata/stats/select index 4adc0ec609aa..4705e4e23fef 100644 --- a/pkg/sql/opt/memo/testdata/stats/select +++ b/pkg/sql/opt/memo/testdata/stats/select @@ -1367,10 +1367,10 @@ WITH t(x) AS ( ) SELECT x FROM t WHERE x ---- -select - ├── columns: x:5(bool!null) - ├── stats: [rows=4e+10, distinct(5)=1, null(5)=0] - ├── fd: ()-->(5) +with &1 (t) + ├── columns: x:6(bool!null) + ├── stats: [rows=4e+10, distinct(6)=1, null(6)=0] + ├── fd: ()-->(6) ├── project │ ├── columns: x:5(bool) │ ├── stats: [rows=4e+20, distinct(5)=1, null(5)=4e+20] @@ -1386,8 +1386,18 @@ select │ │ └── filters (true) │ └── projections │ └── (t1.x::INT8 << 5533)::BOOL OR t2.x [type=bool, outer=(1,3)] - └── filters - └── variable: x [type=bool, outer=(5), constraints=(/5: [/true - /true]; tight), fd=()-->(5)] + └── select + ├── columns: x:6(bool!null) + ├── stats: [rows=4e+10, distinct(6)=1, null(6)=0] + ├── fd: ()-->(6) + ├── with-scan &1 (t) + │ ├── columns: x:6(bool) + │ ├── mapping: + │ │ └── x:5(bool) => x:6(bool) + │ ├── stats: [rows=4e+20, distinct(6)=1, null(6)=4e+20] + │ └── fd: (5)-->(6) + └── filters + └── variable: x [type=bool, outer=(6), constraints=(/6: [/true - /true]; tight), fd=()-->(6)] # Regression test for #38375. Avoid floating point precision errors. exec-ddl diff --git a/pkg/sql/opt/memo/testdata/stats/with b/pkg/sql/opt/memo/testdata/stats/with new file mode 100644 index 000000000000..ec8f2257bf96 --- /dev/null +++ b/pkg/sql/opt/memo/testdata/stats/with @@ -0,0 +1,55 @@ +exec-ddl +CREATE TABLE a (x INT PRIMARY KEY, y INT, s STRING) +---- + +exec-ddl +ALTER TABLE a INJECT STATISTICS '[ + { + "columns": ["x"], + "created_at": "2018-01-01 1:00:00.00000+00:00", + "row_count": 5000, + "distinct_count": 5000 + }, + { + "columns": ["y"], + "created_at": "2018-01-01 1:30:00.00000+00:00", + "row_count": 5000, + "distinct_count": 400 + }, + { + "columns": ["s"], + "created_at": "2018-01-01 1:30:00.00000+00:00", + "row_count": 5000, + "distinct_count": 10 + }, + { + "columns": ["y","s"], + "created_at": "2018-01-01 1:30:00.00000+00:00", + "row_count": 5000, + "distinct_count": 1000 + } +]' +---- + +build colstat=4 colstat=5 colstat=6 +WITH foo AS (SELECT * FROM a) SELECT * FROM foo +---- +with &1 (foo) + ├── columns: x:4(int!null) y:5(int) s:6(string) + ├── stats: [rows=5000] + ├── key: (1) + ├── fd: (1)-->(2-4), (2)-->(5), (3)-->(6) + ├── scan a + │ ├── columns: a.x:1(int!null) a.y:2(int) a.s:3(string) + │ ├── stats: [rows=5000, distinct(1)=5000, null(1)=0, distinct(2)=400, null(2)=0, distinct(3)=10, null(3)=0] + │ ├── key: (1) + │ └── fd: (1)-->(2,3) + └── with-scan &1 (foo) + ├── columns: x:4(int!null) y:5(int) s:6(string) + ├── mapping: + │ ├── a.x:1(int) => x:4(int) + │ ├── a.y:2(int) => y:5(int) + │ └── a.s:3(string) => s:6(string) + ├── stats: [rows=5000, distinct(4)=5000, null(4)=0, distinct(5)=400, null(5)=0, distinct(6)=10, null(6)=0] + ├── key: (1) + └── fd: (1)-->(2-4), (2)-->(5), (3)-->(6) diff --git a/pkg/sql/opt/metadata.go b/pkg/sql/opt/metadata.go index c66a69de0eb8..63de6d9d3941 100644 --- a/pkg/sql/opt/metadata.go +++ b/pkg/sql/opt/metadata.go @@ -441,3 +441,7 @@ func (md *Metadata) AddView(v cat.View) { func (md *Metadata) AllViews() []cat.View { return md.views } + +// WithID uniquely identifies a With expression within the scope of a query. +// See the comment for Metadata for more details on identifiers. +type WithID uint64 diff --git a/pkg/sql/opt/ops/relational.opt b/pkg/sql/opt/ops/relational.opt index 2bc8def088db..6df6a3073324 100644 --- a/pkg/sql/opt/ops/relational.opt +++ b/pkg/sql/opt/ops/relational.opt @@ -823,6 +823,60 @@ define WindowPrivate { Ordering OrderingChoice } +# With executes Binding, making its results available to Input. Within Input, +# Binding may be referenced by a WithScan expression containing the ID of this +# With. +[Relational] +define With { + Binding RelExpr + Input RelExpr + + _ WithPrivate +} + +[Private] +define WithPrivate { + ID WithID + + # OriginalExpr contains the original Subquery expression if it's a SELECT + # so that we can display it in the EXPLAIN plan. + OriginalExpr Subquery + + # Name is used to identify the with for debugging purposes. + Name string +} + +# WithScan returns the results present in the With expression referenced +# by ID. +[Relational] +define WithScan { + _ WithScanPrivate +} + +[Private] +define WithScanPrivate { + ID WithID + + # Name is used to identify the with being referenced for debugging purposes. + Name string + + # InCols are the columns output by the expression referenced by this + # expression. They correspond elementwise to the columns listed in OutCols. + # Every WithScanPrivate with the same WithID should have the same set of + # InCols, being the OutputCols of the binding for the referenced With. + # TODO(justin): this should be relaxed eventually so that we can prune + # these. + InCols ColList + + # OutCols contains a list of columns which correspond elementwise to the + # columns in InCols, which are the IDs output by the referenced With + # expression. WithScan cannot reuse the column IDs used in the original With + # expression, since multiple WithScans referencing the same With can occur in + # the same tree, so we maintain a mapping from the columns returned from + # the referenced expression to the referencing expression. + OutCols ColList +} + # FakeRel is a mock relational operator used for testing; its logical properties # are pre-determined and stored in the private. It can be used as the child of # an operator for which we are calculating properties or statistics. diff --git a/pkg/sql/opt/optbuilder/builder.go b/pkg/sql/opt/optbuilder/builder.go index 57838348570c..e577226e9a4a 100644 --- a/pkg/sql/opt/optbuilder/builder.go +++ b/pkg/sql/opt/optbuilder/builder.go @@ -92,6 +92,7 @@ type Builder struct { evalCtx *tree.EvalContext catalog cat.Catalog scopeAlloc []scope + ctes []cteSource // If set, the planner will skip checking for the SELECT privilege when // resolving data sources (tables, views, etc). This is used when compiling diff --git a/pkg/sql/opt/optbuilder/delete.go b/pkg/sql/opt/optbuilder/delete.go index 826c12f931a1..525da1dc0c84 100644 --- a/pkg/sql/opt/optbuilder/delete.go +++ b/pkg/sql/opt/optbuilder/delete.go @@ -39,9 +39,9 @@ func (b *Builder) buildDelete(del *tree.Delete, inScope *scope) (outScope *scope "DELETE statement requires LIMIT when ORDER BY is used")) } + var ctes []cteSource if del.With != nil { - inScope = b.buildCTE(del.With.CTEList, inScope) - defer b.checkCTEUsage(inScope) + inScope, ctes = b.buildCTE(del.With.CTEList, inScope) } // DELETE FROM xx AS yy - we want to know about xx (tn) because @@ -77,6 +77,8 @@ func (b *Builder) buildDelete(del *tree.Delete, inScope *scope) (outScope *scope mb.buildDelete(nil /* returning */) } + mb.outScope.expr = b.wrapWithCTEs(mb.outScope.expr, ctes) + return mb.outScope } diff --git a/pkg/sql/opt/optbuilder/insert.go b/pkg/sql/opt/optbuilder/insert.go index 78288ced6849..a114138b7ae2 100644 --- a/pkg/sql/opt/optbuilder/insert.go +++ b/pkg/sql/opt/optbuilder/insert.go @@ -154,9 +154,9 @@ func init() { // ON CONFLICT clause is present, since it joins a new set of rows to the input // and thereby scrambles the input ordering. func (b *Builder) buildInsert(ins *tree.Insert, inScope *scope) (outScope *scope) { + var ctes []cteSource if ins.With != nil { - inScope = b.buildCTE(ins.With.CTEList, inScope) - defer b.checkCTEUsage(inScope) + inScope, ctes = b.buildCTE(ins.With.CTEList, inScope) } // INSERT INTO xx AS yy - we want to know about xx (tn) because @@ -311,6 +311,8 @@ func (b *Builder) buildInsert(ins *tree.Insert, inScope *scope) (outScope *scope mb.buildUpsert(returning) } + mb.outScope.expr = b.wrapWithCTEs(mb.outScope.expr, ctes) + return mb.outScope } diff --git a/pkg/sql/opt/optbuilder/scope.go b/pkg/sql/opt/optbuilder/scope.go index ad00341686c2..b0b098c4efc8 100644 --- a/pkg/sql/opt/optbuilder/scope.go +++ b/pkg/sql/opt/optbuilder/scope.go @@ -94,14 +94,11 @@ type scope struct { // cteSource represents a CTE in the given query. type cteSource struct { - name tree.AliasClause - cols []scopeColumn - expr memo.RelExpr - - // used tracks if this CTE has been referenced. We are currently limited - // to only having a single reference to a given CTE, so if this is set then - // this CTE has already been referenced and may not be referenced again. - used bool + name tree.AliasClause + cols []scopeColumn + originalExpr tree.SelectStatement + expr memo.RelExpr + id opt.WithID } // groupByStrSet is a set of stringified GROUP BY expressions that map to the diff --git a/pkg/sql/opt/optbuilder/select.go b/pkg/sql/opt/optbuilder/select.go index 70f07cb300b0..cc463e4c9b05 100644 --- a/pkg/sql/opt/optbuilder/select.go +++ b/pkg/sql/opt/optbuilder/select.go @@ -73,17 +73,28 @@ func (b *Builder) buildDataSource( // CTEs take precedence over other data sources. if cte := inScope.resolveCTE(tn); cte != nil { - if cte.used { - panic(unimplementedWithIssueDetailf(21084, "", "unsupported multiple use of CTE clause %q", tn)) - } - cte.used = true - outScope = inScope.push() - // TODO(justin): once we support mutations here, we will want to include a - // spool operation. - outScope.expr = cte.expr - outScope.cols = cte.cols + inCols := make(opt.ColList, len(cte.cols)) + outCols := make(opt.ColList, len(cte.cols)) + outScope.cols = nil + i := 0 + for _, col := range cte.cols { + id := col.id + c := b.factory.Metadata().ColumnMeta(id) + newCol := b.synthesizeColumn(outScope, string(col.name), c.Type, nil, nil) + newCol.table = *tn + inCols[i] = id + outCols[i] = newCol.id + i++ + } + + outScope.expr = b.factory.ConstructWithScan(&memo.WithScanPrivate{ + ID: cte.id, + Name: string(cte.name.Alias), + InCols: inCols, + OutCols: outCols, + }) return outScope } @@ -476,15 +487,24 @@ func (b *Builder) buildWithOrdinality(colName string, inScope *scope) (outScope return inScope } -func (b *Builder) buildCTE(ctes []*tree.CTE, inScope *scope) (outScope *scope) { +func (b *Builder) buildCTE( + ctes []*tree.CTE, inScope *scope, +) (outScope *scope, addedCTEs []cteSource) { outScope = inScope.push() + start := len(b.ctes) + outScope.ctes = make(map[string]*cteSource) for i := range ctes { cteScope := b.buildStmt(ctes[i].Stmt, nil /* desiredTypes */, outScope) cols := cteScope.cols name := ctes[i].Name.Alias + // TODO(justin): lift this restriction when possible. WITH should be hoistable. + if b.subquery != nil && !b.subquery.outerCols.Empty() { + panic(pgerror.Newf(pgcode.FeatureNotSupported, "CTEs may not be correlated")) + } + if _, ok := outScope.ctes[name.String()]; ok { panic(pgerror.Newf( pgcode.DuplicateAlias, @@ -516,27 +536,52 @@ func (b *Builder) buildCTE(ctes []*tree.CTE, inScope *scope) (outScope *scope) { "WITH clause %q does not have a RETURNING clause", tree.ErrString(&name))) } - outScope.ctes[ctes[i].Name.Alias.String()] = &cteSource{ - name: ctes[i].Name, - cols: cols, - expr: cteScope.expr, + projectionsScope := cteScope.replace() + projectionsScope.appendColumnsFromScope(cteScope) + b.constructProjectForScope(cteScope, projectionsScope) + + cteScope = projectionsScope + + id := b.factory.Memo().AddWithBinding(cteScope.expr) + + // No good way to show non-select expressions, like INSERT, here. + var stmt tree.SelectStatement + if sel, ok := ctes[i].Stmt.(*tree.Select); ok { + stmt = sel.Select } + + b.ctes = append(b.ctes, cteSource{ + name: ctes[i].Name, + cols: cols, + originalExpr: stmt, + expr: cteScope.expr, + id: id, + }) + cte := &b.ctes[len(b.ctes)-1] + outScope.ctes[ctes[i].Name.Alias.String()] = cte } telemetry.Inc(sqltelemetry.CteUseCounter) - return outScope + return outScope, b.ctes[start:] } -// checkCTEUsage ensures that a CTE that contains a mutation (like INSERT) is -// used at least once by the query. Otherwise, it might not be executed. -func (b *Builder) checkCTEUsage(inScope *scope) { - for alias, source := range inScope.ctes { - if !source.used && source.expr.Relational().CanMutate { - panic(unimplemented.NewWithIssuef(24307, - "common table expression %q with side effects was not used in query", alias)) - } - } +// wrapWithCTEs adds With expressions on top of an expression. +func (b *Builder) wrapWithCTEs(expr memo.RelExpr, ctes []cteSource) memo.RelExpr { + // Since later CTEs can refer to earlier ones, we want to add these in + // reverse order. + for i := len(ctes) - 1; i >= 0; i-- { + expr = b.factory.ConstructWith( + ctes[i].expr, + expr, + &memo.WithPrivate{ + ID: ctes[i].id, + Name: string(ctes[i].name.Alias), + OriginalExpr: &tree.Subquery{Select: ctes[i].originalExpr}, + }, + ) + } + return expr } // buildSelectStmt builds a set of memo groups that represent the given select @@ -609,9 +654,9 @@ func (b *Builder) buildSelect( } } + var ctes []cteSource if with != nil { - inScope = b.buildCTE(with.CTEList, inScope) - defer b.checkCTEUsage(inScope) + inScope, ctes = b.buildCTE(with.CTEList, inScope) } // NB: The case statements are sorted lexicographically. @@ -648,6 +693,8 @@ func (b *Builder) buildSelect( b.buildLimit(limit, inScope, outScope) } + outScope.expr = b.wrapWithCTEs(outScope.expr, ctes) + // TODO(rytaft): Support FILTER expression. return outScope } diff --git a/pkg/sql/opt/optbuilder/testdata/delete b/pkg/sql/opt/optbuilder/testdata/delete index 90035f346de0..e7899003fb92 100644 --- a/pkg/sql/opt/optbuilder/testdata/delete +++ b/pkg/sql/opt/optbuilder/testdata/delete @@ -149,19 +149,24 @@ select build WITH cte AS (SELECT x FROM xyz) DELETE FROM abcde WHERE EXISTS(SELECT * FROM cte) ---- -delete abcde - ├── columns: - ├── fetch columns: a:10(int) b:11(int) c:12(int) d:13(int) e:14(int) rowid:15(int) - └── select - ├── columns: a:10(int!null) b:11(int) c:12(int) d:13(int) e:14(int) rowid:15(int!null) - ├── scan abcde - │ └── columns: a:10(int!null) b:11(int) c:12(int) d:13(int) e:14(int) rowid:15(int!null) - └── filters - └── exists [type=bool] - └── project - ├── columns: x:1(string!null) - └── scan xyz - └── columns: x:1(string!null) y:2(int) z:3(float) +with &1 (cte) + ├── project + │ ├── columns: xyz.x:1(string!null) + │ └── scan xyz + │ └── columns: xyz.x:1(string!null) y:2(int) z:3(float) + └── delete abcde + ├── columns: + ├── fetch columns: a:10(int) b:11(int) c:12(int) d:13(int) e:14(int) rowid:15(int) + └── select + ├── columns: a:10(int!null) b:11(int) c:12(int) d:13(int) e:14(int) rowid:15(int!null) + ├── scan abcde + │ └── columns: a:10(int!null) b:11(int) c:12(int) d:13(int) e:14(int) rowid:15(int!null) + └── filters + └── exists [type=bool] + └── with-scan &1 (cte) + ├── columns: x:16(string!null) + └── mapping: + └── xyz.x:1(string) => x:16(string) # Unknown target table. build @@ -179,7 +184,33 @@ error (42703): statement source "DELETE FROM abcde WHERE a = 1" does not return build WITH cte AS (SELECT b FROM [DELETE FROM abcde WHERE a=b RETURNING *]) DELETE FROM abcde WHERE a=b ---- -error (0A000): unimplemented: common table expression "cte" with side effects was not used in query +with &1 (cte) + ├── project + │ ├── columns: b:2(int!null) + │ └── project + │ ├── columns: a:1(int!null) b:2(int!null) c:3(int) d:4(int) e:5(int) + │ └── delete abcde + │ ├── columns: a:1(int!null) b:2(int!null) c:3(int) d:4(int) e:5(int) rowid:6(int!null) + │ ├── fetch columns: a:7(int) b:8(int) c:9(int) d:10(int) e:11(int) rowid:12(int) + │ └── select + │ ├── columns: a:7(int!null) b:8(int!null) c:9(int) d:10(int) e:11(int) rowid:12(int!null) + │ ├── scan abcde + │ │ └── columns: a:7(int!null) b:8(int) c:9(int) d:10(int) e:11(int) rowid:12(int!null) + │ └── filters + │ └── eq [type=bool] + │ ├── variable: a [type=int] + │ └── variable: b [type=int] + └── delete abcde + ├── columns: + ├── fetch columns: a:19(int) b:20(int) c:21(int) d:22(int) e:23(int) rowid:24(int) + └── select + ├── columns: a:19(int!null) b:20(int!null) c:21(int) d:22(int) e:23(int) rowid:24(int!null) + ├── scan abcde + │ └── columns: a:19(int!null) b:20(int) c:21(int) d:22(int) e:23(int) rowid:24(int!null) + └── filters + └── eq [type=bool] + ├── variable: a [type=int] + └── variable: b [type=int] # With alias, original table name should be inaccessible. build diff --git a/pkg/sql/opt/optbuilder/testdata/insert b/pkg/sql/opt/optbuilder/testdata/insert index a6bb5ccda5b3..98f710421cc6 100644 --- a/pkg/sql/opt/optbuilder/testdata/insert +++ b/pkg/sql/opt/optbuilder/testdata/insert @@ -504,90 +504,168 @@ error (42703): statement source "INSERT INTO abcde VALUES (1)" does not return a build WITH a AS (SELECT y, y+1 FROM xyz) INSERT INTO abcde SELECT * FROM a ---- -insert abcde - ├── columns: - ├── insert-mapping: - │ ├── y:2 => a:5 - │ ├── "?column?":4 => b:6 - │ ├── column11:11 => c:7 - │ ├── column13:13 => d:8 - │ ├── y:2 => e:9 - │ └── column12:12 => rowid:10 - └── project - ├── columns: column13:13(int) y:2(int) "?column?":4(int) column11:11(int!null) column12:12(int) - ├── project - │ ├── columns: column11:11(int!null) column12:12(int) y:2(int) "?column?":4(int) - │ ├── project - │ │ ├── columns: "?column?":4(int) y:2(int) - │ │ ├── scan xyz - │ │ │ └── columns: x:1(string!null) y:2(int) z:3(float) - │ │ └── projections - │ │ └── plus [type=int] - │ │ ├── variable: y [type=int] - │ │ └── const: 1 [type=int] - │ └── projections - │ ├── const: 10 [type=int] - │ └── function: unique_rowid [type=int] - └── projections - └── plus [type=int] - ├── plus [type=int] - │ ├── variable: ?column? [type=int] - │ └── variable: column11 [type=int] - └── const: 1 [type=int] +with &1 (a) + ├── project + │ ├── columns: "?column?":4(int) xyz.y:2(int) + │ ├── scan xyz + │ │ └── columns: x:1(string!null) xyz.y:2(int) z:3(float) + │ └── projections + │ └── plus [type=int] + │ ├── variable: xyz.y [type=int] + │ └── const: 1 [type=int] + └── insert abcde + ├── columns: + ├── insert-mapping: + │ ├── y:11 => a:5 + │ ├── "?column?":12 => b:6 + │ ├── column13:13 => c:7 + │ ├── column15:15 => d:8 + │ ├── y:11 => e:9 + │ └── column14:14 => rowid:10 + └── project + ├── columns: column15:15(int) y:11(int) "?column?":12(int) column13:13(int!null) column14:14(int) + ├── project + │ ├── columns: column13:13(int!null) column14:14(int) y:11(int) "?column?":12(int) + │ ├── with-scan &1 (a) + │ │ ├── columns: y:11(int) "?column?":12(int) + │ │ └── mapping: + │ │ ├── xyz.y:2(int) => y:11(int) + │ │ └── "?column?":4(int) => "?column?":12(int) + │ └── projections + │ ├── const: 10 [type=int] + │ └── function: unique_rowid [type=int] + └── projections + └── plus [type=int] + ├── plus [type=int] + │ ├── variable: ?column? [type=int] + │ └── variable: column13 [type=int] + └── const: 1 [type=int] # Use CTE with multiple variables. build WITH a AS (SELECT y, y+1 FROM xyz), b AS (SELECT y+1, y FROM xyz) INSERT INTO abcde TABLE a UNION TABLE b ---- -insert abcde - ├── columns: - ├── insert-mapping: - │ ├── y:15 => a:9 - │ ├── "?column?":16 => b:10 - │ ├── column17:17 => c:11 - │ ├── column19:19 => d:12 - │ ├── y:15 => e:13 - │ └── column18:18 => rowid:14 - └── project - ├── columns: column19:19(int) y:15(int) "?column?":16(int) column17:17(int!null) column18:18(int) +with &1 (a) + ├── project + │ ├── columns: "?column?":4(int) xyz.y:2(int) + │ ├── scan xyz + │ │ └── columns: x:1(string!null) xyz.y:2(int) z:3(float) + │ └── projections + │ └── plus [type=int] + │ ├── variable: xyz.y [type=int] + │ └── const: 1 [type=int] + └── with &2 (b) ├── project - │ ├── columns: column17:17(int!null) column18:18(int) y:15(int) "?column?":16(int) - │ ├── union - │ │ ├── columns: y:15(int) "?column?":16(int) - │ │ ├── left columns: xyz.y:2(int) "?column?":4(int) - │ │ ├── right columns: "?column?":8(int) xyz.y:6(int) - │ │ ├── project - │ │ │ ├── columns: "?column?":4(int) xyz.y:2(int) - │ │ │ ├── scan xyz - │ │ │ │ └── columns: x:1(string!null) xyz.y:2(int) z:3(float) - │ │ │ └── projections - │ │ │ └── plus [type=int] - │ │ │ ├── variable: xyz.y [type=int] - │ │ │ └── const: 1 [type=int] - │ │ └── project - │ │ ├── columns: "?column?":8(int) xyz.y:6(int) - │ │ ├── scan xyz - │ │ │ └── columns: x:5(string!null) xyz.y:6(int) z:7(float) - │ │ └── projections - │ │ └── plus [type=int] - │ │ ├── variable: xyz.y [type=int] - │ │ └── const: 1 [type=int] + │ ├── columns: "?column?":8(int) xyz.y:6(int) + │ ├── scan xyz + │ │ └── columns: x:5(string!null) xyz.y:6(int) z:7(float) │ └── projections - │ ├── const: 10 [type=int] - │ └── function: unique_rowid [type=int] - └── projections - └── plus [type=int] - ├── plus [type=int] - │ ├── variable: ?column? [type=int] - │ └── variable: column17 [type=int] - └── const: 1 [type=int] + │ └── plus [type=int] + │ ├── variable: xyz.y [type=int] + │ └── const: 1 [type=int] + └── insert abcde + ├── columns: + ├── insert-mapping: + │ ├── y:19 => a:9 + │ ├── "?column?":20 => b:10 + │ ├── column21:21 => c:11 + │ ├── column23:23 => d:12 + │ ├── y:19 => e:13 + │ └── column22:22 => rowid:14 + └── project + ├── columns: column23:23(int) y:19(int) "?column?":20(int) column21:21(int!null) column22:22(int) + ├── project + │ ├── columns: column21:21(int!null) column22:22(int) y:19(int) "?column?":20(int) + │ ├── union + │ │ ├── columns: y:19(int) "?column?":20(int) + │ │ ├── left columns: y:15(int) "?column?":16(int) + │ │ ├── right columns: "?column?":17(int) y:18(int) + │ │ ├── with-scan &1 (a) + │ │ │ ├── columns: y:15(int) "?column?":16(int) + │ │ │ └── mapping: + │ │ │ ├── xyz.y:2(int) => y:15(int) + │ │ │ └── "?column?":4(int) => "?column?":16(int) + │ │ └── with-scan &2 (b) + │ │ ├── columns: "?column?":17(int) y:18(int) + │ │ └── mapping: + │ │ ├── "?column?":8(int) => "?column?":17(int) + │ │ └── xyz.y:6(int) => y:18(int) + │ └── projections + │ ├── const: 10 [type=int] + │ └── function: unique_rowid [type=int] + └── projections + └── plus [type=int] + ├── plus [type=int] + │ ├── variable: ?column? [type=int] + │ └── variable: column21 [type=int] + └── const: 1 [type=int] # Non-referenced CTE with mutation. build WITH cte AS (SELECT b FROM [INSERT INTO abcde VALUES (1) RETURNING *]) INSERT INTO abcde VALUES (1) ---- -error (0A000): unimplemented: common table expression "cte" with side effects was not used in query +with &1 (cte) + ├── project + │ ├── columns: b:2(int) + │ └── project + │ ├── columns: a:1(int!null) b:2(int) c:3(int!null) d:4(int) e:5(int!null) + │ └── insert abcde + │ ├── columns: a:1(int!null) b:2(int) c:3(int!null) d:4(int) e:5(int!null) rowid:6(int!null) + │ ├── insert-mapping: + │ │ ├── column1:7 => a:1 + │ │ ├── column8:8 => b:2 + │ │ ├── column9:9 => c:3 + │ │ ├── column11:11 => d:4 + │ │ ├── column1:7 => e:5 + │ │ └── column10:10 => rowid:6 + │ └── project + │ ├── columns: column11:11(int) column1:7(int!null) column8:8(int) column9:9(int!null) column10:10(int) + │ ├── project + │ │ ├── columns: column8:8(int) column9:9(int!null) column10:10(int) column1:7(int!null) + │ │ ├── values + │ │ │ ├── columns: column1:7(int!null) + │ │ │ └── tuple [type=tuple{int}] + │ │ │ └── const: 1 [type=int] + │ │ └── projections + │ │ ├── cast: INT8 [type=int] + │ │ │ └── null [type=unknown] + │ │ ├── const: 10 [type=int] + │ │ └── function: unique_rowid [type=int] + │ └── projections + │ └── plus [type=int] + │ ├── plus [type=int] + │ │ ├── variable: column8 [type=int] + │ │ └── variable: column9 [type=int] + │ └── const: 1 [type=int] + └── insert abcde + ├── columns: + ├── insert-mapping: + │ ├── column1:18 => a:12 + │ ├── column19:19 => b:13 + │ ├── column20:20 => c:14 + │ ├── column22:22 => d:15 + │ ├── column1:18 => e:16 + │ └── column21:21 => rowid:17 + └── project + ├── columns: column22:22(int) column1:18(int!null) column19:19(int) column20:20(int!null) column21:21(int) + ├── project + │ ├── columns: column19:19(int) column20:20(int!null) column21:21(int) column1:18(int!null) + │ ├── values + │ │ ├── columns: column1:18(int!null) + │ │ └── tuple [type=tuple{int}] + │ │ └── const: 1 [type=int] + │ └── projections + │ ├── cast: INT8 [type=int] + │ │ └── null [type=unknown] + │ ├── const: 10 [type=int] + │ └── function: unique_rowid [type=int] + └── projections + └── plus [type=int] + ├── plus [type=int] + │ ├── variable: column19 [type=int] + │ └── variable: column20 [type=int] + └── const: 1 [type=int] # Insert CTE that returns no columns. build diff --git a/pkg/sql/opt/optbuilder/testdata/select b/pkg/sql/opt/optbuilder/testdata/select index c73498710551..bdd920451c35 100644 --- a/pkg/sql/opt/optbuilder/testdata/select +++ b/pkg/sql/opt/optbuilder/testdata/select @@ -1229,4 +1229,27 @@ project build WITH cte AS (SELECT b FROM [INSERT INTO abc VALUES (1) RETURNING *] LIMIT 1) SELECT * FROM abc ---- -error (0A000): unimplemented: common table expression "cte" with side effects was not used in query +with &1 (cte) + ├── columns: a:6(int!null) b:7(int) c:8(int) + ├── limit + │ ├── columns: b:2(int) + │ ├── project + │ │ ├── columns: b:2(int) + │ │ └── insert abc + │ │ ├── columns: a:1(int!null) b:2(int) c:3(int) + │ │ ├── insert-mapping: + │ │ │ ├── column1:4 => a:1 + │ │ │ ├── column5:5 => b:2 + │ │ │ └── column5:5 => c:3 + │ │ └── project + │ │ ├── columns: column5:5(int) column1:4(int!null) + │ │ ├── values + │ │ │ ├── columns: column1:4(int!null) + │ │ │ └── tuple [type=tuple{int}] + │ │ │ └── const: 1 [type=int] + │ │ └── projections + │ │ └── cast: INT8 [type=int] + │ │ └── null [type=unknown] + │ └── const: 1 [type=int] + └── scan abc + └── columns: a:6(int!null) b:7(int) c:8(int) diff --git a/pkg/sql/opt/optbuilder/testdata/update b/pkg/sql/opt/optbuilder/testdata/update index 1da69b3e20b7..743c0386b10b 100644 --- a/pkg/sql/opt/optbuilder/testdata/update +++ b/pkg/sql/opt/optbuilder/testdata/update @@ -423,7 +423,45 @@ error (42703): statement source "UPDATE abcde SET a = 1" does not return any col build WITH cte AS (SELECT b FROM [UPDATE abcde SET a=b RETURNING *]) UPDATE abcde SET a=b ---- -error (0A000): unimplemented: common table expression "cte" with side effects was not used in query +with &1 (cte) + ├── project + │ ├── columns: b:2(int) + │ └── project + │ ├── columns: a:1(int!null) b:2(int) c:3(int) d:4(int) e:5(int) + │ └── update abcde + │ ├── columns: a:1(int!null) b:2(int) c:3(int) d:4(int) e:5(int) rowid:6(int!null) + │ ├── fetch columns: a:7(int) b:8(int) c:9(int) d:10(int) e:11(int) rowid:12(int) + │ ├── update-mapping: + │ │ ├── b:8 => a:1 + │ │ ├── column13:13 => d:4 + │ │ └── b:8 => e:5 + │ └── project + │ ├── columns: column13:13(int) a:7(int!null) b:8(int) c:9(int) d:10(int) e:11(int) rowid:12(int!null) + │ ├── scan abcde + │ │ └── columns: a:7(int!null) b:8(int) c:9(int) d:10(int) e:11(int) rowid:12(int!null) + │ └── projections + │ └── plus [type=int] + │ ├── plus [type=int] + │ │ ├── variable: b [type=int] + │ │ └── variable: c [type=int] + │ └── const: 1 [type=int] + └── update abcde + ├── columns: + ├── fetch columns: a:20(int) b:21(int) c:22(int) d:23(int) e:24(int) rowid:25(int) + ├── update-mapping: + │ ├── b:21 => a:14 + │ ├── column26:26 => d:17 + │ └── b:21 => e:18 + └── project + ├── columns: column26:26(int) a:20(int!null) b:21(int) c:22(int) d:23(int) e:24(int) rowid:25(int!null) + ├── scan abcde + │ └── columns: a:20(int!null) b:21(int) c:22(int) d:23(int) e:24(int) rowid:25(int!null) + └── projections + └── plus [type=int] + ├── plus [type=int] + │ ├── variable: b [type=int] + │ └── variable: c [type=int] + └── const: 1 [type=int] # With alias, original table name should be inaccessible. build @@ -1124,67 +1162,78 @@ error (42804): value type int doesn't match type string of column "x" build WITH cte AS (SELECT x FROM xyz) UPDATE abcde SET a=b WHERE EXISTS(SELECT * FROM cte) ---- -update abcde - ├── columns: - ├── fetch columns: a:10(int) b:11(int) c:12(int) d:13(int) e:14(int) rowid:15(int) - ├── update-mapping: - │ ├── b:11 => a:4 - │ ├── column16:16 => d:7 - │ └── b:11 => e:8 - └── project - ├── columns: column16:16(int) a:10(int!null) b:11(int) c:12(int) d:13(int) e:14(int) rowid:15(int!null) - ├── select - │ ├── columns: a:10(int!null) b:11(int) c:12(int) d:13(int) e:14(int) rowid:15(int!null) - │ ├── scan abcde - │ │ └── columns: a:10(int!null) b:11(int) c:12(int) d:13(int) e:14(int) rowid:15(int!null) - │ └── filters - │ └── exists [type=bool] - │ └── project - │ ├── columns: x:1(string!null) - │ └── scan xyz - │ └── columns: x:1(string!null) y:2(int) z:3(float) - └── projections - └── plus [type=int] - ├── plus [type=int] - │ ├── variable: b [type=int] - │ └── variable: c [type=int] - └── const: 1 [type=int] +with &1 (cte) + ├── project + │ ├── columns: xyz.x:1(string!null) + │ └── scan xyz + │ └── columns: xyz.x:1(string!null) y:2(int) z:3(float) + └── update abcde + ├── columns: + ├── fetch columns: a:10(int) b:11(int) c:12(int) d:13(int) e:14(int) rowid:15(int) + ├── update-mapping: + │ ├── b:11 => a:4 + │ ├── column17:17 => d:7 + │ └── b:11 => e:8 + └── project + ├── columns: column17:17(int) a:10(int!null) b:11(int) c:12(int) d:13(int) e:14(int) rowid:15(int!null) + ├── select + │ ├── columns: a:10(int!null) b:11(int) c:12(int) d:13(int) e:14(int) rowid:15(int!null) + │ ├── scan abcde + │ │ └── columns: a:10(int!null) b:11(int) c:12(int) d:13(int) e:14(int) rowid:15(int!null) + │ └── filters + │ └── exists [type=bool] + │ └── with-scan &1 (cte) + │ ├── columns: x:16(string!null) + │ └── mapping: + │ └── xyz.x:1(string) => x:16(string) + └── projections + └── plus [type=int] + ├── plus [type=int] + │ ├── variable: b [type=int] + │ └── variable: c [type=int] + └── const: 1 [type=int] # Use CTE within SET expression. build WITH a AS (SELECT y, y+1 AS y1 FROM xyz) UPDATE abcde SET (a, b) = (SELECT * FROM a) ---- -update abcde - ├── columns: - ├── fetch columns: a:11(int) b:12(int) c:13(int) d:14(int) e:15(int) rowid:16(int) - ├── update-mapping: - │ ├── y:2 => a:5 - │ ├── y1:4 => b:6 - │ ├── column17:17 => d:8 - │ └── y:2 => e:9 - └── project - ├── columns: column17:17(int) y:2(int) y1:4(int) a:11(int!null) b:12(int) c:13(int) d:14(int) e:15(int) rowid:16(int!null) - ├── left-join-apply - │ ├── columns: y:2(int) y1:4(int) a:11(int!null) b:12(int) c:13(int) d:14(int) e:15(int) rowid:16(int!null) - │ ├── scan abcde - │ │ └── columns: a:11(int!null) b:12(int) c:13(int) d:14(int) e:15(int) rowid:16(int!null) - │ ├── max1-row - │ │ ├── columns: y:2(int) y1:4(int) - │ │ └── project - │ │ ├── columns: y1:4(int) y:2(int) - │ │ ├── scan xyz - │ │ │ └── columns: x:1(string!null) y:2(int) z:3(float) - │ │ └── projections - │ │ └── plus [type=int] - │ │ ├── variable: y [type=int] - │ │ └── const: 1 [type=int] - │ └── filters (true) - └── projections - └── plus [type=int] - ├── plus [type=int] - │ ├── variable: y1 [type=int] - │ └── variable: c [type=int] - └── const: 1 [type=int] +with &1 (a) + ├── project + │ ├── columns: y1:4(int) xyz.y:2(int) + │ ├── scan xyz + │ │ └── columns: x:1(string!null) xyz.y:2(int) z:3(float) + │ └── projections + │ └── plus [type=int] + │ ├── variable: xyz.y [type=int] + │ └── const: 1 [type=int] + └── update abcde + ├── columns: + ├── fetch columns: a:11(int) b:12(int) c:13(int) d:14(int) e:15(int) rowid:16(int) + ├── update-mapping: + │ ├── y:17 => a:5 + │ ├── y1:18 => b:6 + │ ├── column19:19 => d:8 + │ └── y:17 => e:9 + └── project + ├── columns: column19:19(int) a:11(int!null) b:12(int) c:13(int) d:14(int) e:15(int) rowid:16(int!null) y:17(int) y1:18(int) + ├── left-join-apply + │ ├── columns: a:11(int!null) b:12(int) c:13(int) d:14(int) e:15(int) rowid:16(int!null) y:17(int) y1:18(int) + │ ├── scan abcde + │ │ └── columns: a:11(int!null) b:12(int) c:13(int) d:14(int) e:15(int) rowid:16(int!null) + │ ├── max1-row + │ │ ├── columns: y:17(int) y1:18(int) + │ │ └── with-scan &1 (a) + │ │ ├── columns: y:17(int) y1:18(int) + │ │ └── mapping: + │ │ ├── xyz.y:2(int) => y:17(int) + │ │ └── y1:4(int) => y1:18(int) + │ └── filters (true) + └── projections + └── plus [type=int] + ├── plus [type=int] + │ ├── variable: y1 [type=int] + │ └── variable: c [type=int] + └── const: 1 [type=int] # ------------------------------------------------------------------------------ # Tests with mutations. diff --git a/pkg/sql/opt/optbuilder/testdata/with b/pkg/sql/opt/optbuilder/testdata/with index 8f648311babc..d999f2e5eab2 100644 --- a/pkg/sql/opt/optbuilder/testdata/with +++ b/pkg/sql/opt/optbuilder/testdata/with @@ -1,5 +1,5 @@ exec-ddl -CREATE TABLE x(a INT) +CREATE TABLE x(a INT, b INT) ---- exec-ddl @@ -10,41 +10,53 @@ build WITH t AS (SELECT a FROM y WHERE a < 3) SELECT * FROM x NATURAL JOIN t ---- -project - ├── columns: a:3(int!null) - └── inner-join (hash) - ├── columns: y.a:1(int!null) x.a:3(int!null) x.rowid:4(int!null) - ├── scan x - │ └── columns: x.a:3(int) x.rowid:4(int!null) - ├── project - │ ├── columns: y.a:1(int!null) - │ └── select - │ ├── columns: y.a:1(int!null) y.rowid:2(int!null) - │ ├── scan y - │ │ └── columns: y.a:1(int) y.rowid:2(int!null) - │ └── filters - │ └── lt [type=bool] - │ ├── variable: y.a [type=int] - │ └── const: 3 [type=int] - └── filters - └── eq [type=bool] - ├── variable: x.a [type=int] - └── variable: y.a [type=int] +with &1 (t) + ├── columns: a:3(int!null) b:4(int) + ├── project + │ ├── columns: y.a:1(int!null) + │ └── select + │ ├── columns: y.a:1(int!null) y.rowid:2(int!null) + │ ├── scan y + │ │ └── columns: y.a:1(int) y.rowid:2(int!null) + │ └── filters + │ └── lt [type=bool] + │ ├── variable: y.a [type=int] + │ └── const: 3 [type=int] + └── project + ├── columns: x.a:3(int!null) b:4(int) + └── inner-join (hash) + ├── columns: x.a:3(int!null) b:4(int) x.rowid:5(int!null) a:6(int!null) + ├── scan x + │ └── columns: x.a:3(int) b:4(int) x.rowid:5(int!null) + ├── with-scan &1 (t) + │ ├── columns: a:6(int!null) + │ └── mapping: + │ └── y.a:1(int) => a:6(int) + └── filters + └── eq [type=bool] + ├── variable: x.a [type=int] + └── variable: a [type=int] build WITH t AS (SELECT a FROM y WHERE a < 3) SELECT * FROM t ---- -project - ├── columns: a:1(int!null) - └── select - ├── columns: a:1(int!null) rowid:2(int!null) - ├── scan y - │ └── columns: a:1(int) rowid:2(int!null) - └── filters - └── lt [type=bool] - ├── variable: a [type=int] - └── const: 3 [type=int] +with &1 (t) + ├── columns: a:3(int!null) + ├── project + │ ├── columns: y.a:1(int!null) + │ └── select + │ ├── columns: y.a:1(int!null) rowid:2(int!null) + │ ├── scan y + │ │ └── columns: y.a:1(int) rowid:2(int!null) + │ └── filters + │ └── lt [type=bool] + │ ├── variable: y.a [type=int] + │ └── const: 3 [type=int] + └── with-scan &1 (t) + ├── columns: a:3(int!null) + └── mapping: + └── y.a:1(int) => a:3(int) # Chaining multiple CTEs. build @@ -53,22 +65,34 @@ WITH t2 AS (SELECT * FROM t1 WHERE a > 1) SELECT * FROM t2 ---- -select - ├── columns: a:1(int!null) +with &1 (t1) + ├── columns: a:4(int!null) ├── project - │ ├── columns: a:1(int!null) + │ ├── columns: y.a:1(int!null) │ └── select - │ ├── columns: a:1(int!null) rowid:2(int!null) + │ ├── columns: y.a:1(int!null) rowid:2(int!null) │ ├── scan y - │ │ └── columns: a:1(int) rowid:2(int!null) + │ │ └── columns: y.a:1(int) rowid:2(int!null) │ └── filters │ └── lt [type=bool] - │ ├── variable: a [type=int] + │ ├── variable: y.a [type=int] │ └── const: 3 [type=int] - └── filters - └── gt [type=bool] - ├── variable: a [type=int] - └── const: 1 [type=int] + └── with &2 (t2) + ├── columns: a:4(int!null) + ├── select + │ ├── columns: a:3(int!null) + │ ├── with-scan &1 (t1) + │ │ ├── columns: a:3(int!null) + │ │ └── mapping: + │ │ └── y.a:1(int) => a:3(int) + │ └── filters + │ └── gt [type=bool] + │ ├── variable: a [type=int] + │ └── const: 1 [type=int] + └── with-scan &2 (t2) + ├── columns: a:4(int!null) + └── mapping: + └── a:3(int) => a:4(int) build WITH @@ -77,28 +101,46 @@ WITH t3 AS (SELECT * FROM t2 WHERE a = 2) SELECT * FROM t3 ---- -select - ├── columns: a:1(int!null) - ├── select - │ ├── columns: a:1(int!null) - │ ├── project - │ │ ├── columns: a:1(int!null) - │ │ └── select - │ │ ├── columns: a:1(int!null) rowid:2(int!null) - │ │ ├── scan y - │ │ │ └── columns: a:1(int) rowid:2(int!null) - │ │ └── filters - │ │ └── lt [type=bool] - │ │ ├── variable: a [type=int] - │ │ └── const: 3 [type=int] - │ └── filters - │ └── gt [type=bool] - │ ├── variable: a [type=int] - │ └── const: 1 [type=int] - └── filters - └── eq [type=bool] - ├── variable: a [type=int] - └── const: 2 [type=int] +with &1 (t1) + ├── columns: a:5(int!null) + ├── project + │ ├── columns: y.a:1(int!null) + │ └── select + │ ├── columns: y.a:1(int!null) rowid:2(int!null) + │ ├── scan y + │ │ └── columns: y.a:1(int) rowid:2(int!null) + │ └── filters + │ └── lt [type=bool] + │ ├── variable: y.a [type=int] + │ └── const: 3 [type=int] + └── with &2 (t2) + ├── columns: a:5(int!null) + ├── select + │ ├── columns: a:3(int!null) + │ ├── with-scan &1 (t1) + │ │ ├── columns: a:3(int!null) + │ │ └── mapping: + │ │ └── y.a:1(int) => a:3(int) + │ └── filters + │ └── gt [type=bool] + │ ├── variable: a [type=int] + │ └── const: 1 [type=int] + └── with &3 (t3) + ├── columns: a:5(int!null) + ├── select + │ ├── columns: a:4(int!null) + │ ├── with-scan &2 (t2) + │ │ ├── columns: a:4(int!null) + │ │ └── mapping: + │ │ └── a:3(int) => a:4(int) + │ └── filters + │ └── eq [type=bool] + │ ├── variable: a [type=int] + │ └── const: 2 [type=int] + └── with-scan &3 (t3) + ├── columns: a:5(int!null) + └── mapping: + └── a:4(int) => a:5(int) build WITH @@ -108,57 +150,95 @@ WITH t4 AS (SELECT * FROM t2 WHERE a > 3) SELECT * FROM t3 NATURAL JOIN t4 ---- -project - ├── columns: a:1(int!null) - └── inner-join (hash) - ├── columns: a:1(int!null) a:3(int!null) - ├── select - │ ├── columns: a:1(int!null) - │ ├── project - │ │ ├── columns: a:1(int!null) - │ │ └── select - │ │ ├── columns: a:1(int!null) rowid:2(int!null) - │ │ ├── scan y - │ │ │ └── columns: a:1(int) rowid:2(int!null) - │ │ └── filters - │ │ └── lt [type=bool] - │ │ ├── variable: a [type=int] - │ │ └── const: 3 [type=int] - │ └── filters - │ └── lt [type=bool] - │ ├── variable: a [type=int] - │ └── const: 4 [type=int] - ├── select - │ ├── columns: a:3(int!null) - │ ├── project - │ │ ├── columns: a:3(int!null) - │ │ └── select - │ │ ├── columns: a:3(int!null) rowid:4(int!null) - │ │ ├── scan y - │ │ │ └── columns: a:3(int) rowid:4(int!null) - │ │ └── filters - │ │ └── gt [type=bool] - │ │ ├── variable: a [type=int] - │ │ └── const: 1 [type=int] - │ └── filters - │ └── gt [type=bool] - │ ├── variable: a [type=int] - │ └── const: 3 [type=int] - └── filters - └── eq [type=bool] - ├── variable: a [type=int] - └── variable: a [type=int] +with &1 (t1) + ├── columns: a:7(int!null) + ├── project + │ ├── columns: y.a:1(int!null) + │ └── select + │ ├── columns: y.a:1(int!null) rowid:2(int!null) + │ ├── scan y + │ │ └── columns: y.a:1(int) rowid:2(int!null) + │ └── filters + │ └── lt [type=bool] + │ ├── variable: y.a [type=int] + │ └── const: 3 [type=int] + └── with &2 (t2) + ├── columns: a:7(int!null) + ├── project + │ ├── columns: y.a:3(int!null) + │ └── select + │ ├── columns: y.a:3(int!null) rowid:4(int!null) + │ ├── scan y + │ │ └── columns: y.a:3(int) rowid:4(int!null) + │ └── filters + │ └── gt [type=bool] + │ ├── variable: y.a [type=int] + │ └── const: 1 [type=int] + └── with &3 (t3) + ├── columns: a:7(int!null) + ├── select + │ ├── columns: a:5(int!null) + │ ├── with-scan &1 (t1) + │ │ ├── columns: a:5(int!null) + │ │ └── mapping: + │ │ └── y.a:1(int) => a:5(int) + │ └── filters + │ └── lt [type=bool] + │ ├── variable: a [type=int] + │ └── const: 4 [type=int] + └── with &4 (t4) + ├── columns: a:7(int!null) + ├── select + │ ├── columns: a:6(int!null) + │ ├── with-scan &2 (t2) + │ │ ├── columns: a:6(int!null) + │ │ └── mapping: + │ │ └── y.a:3(int) => a:6(int) + │ └── filters + │ └── gt [type=bool] + │ ├── variable: a [type=int] + │ └── const: 3 [type=int] + └── project + ├── columns: a:7(int!null) + └── inner-join (hash) + ├── columns: a:7(int!null) a:8(int!null) + ├── with-scan &3 (t3) + │ ├── columns: a:7(int!null) + │ └── mapping: + │ └── a:5(int) => a:7(int) + ├── with-scan &4 (t4) + │ ├── columns: a:8(int!null) + │ └── mapping: + │ └── a:6(int) => a:8(int) + └── filters + └── eq [type=bool] + ├── variable: a [type=int] + └── variable: a [type=int] # Make sure they scope properly. build WITH t AS (SELECT true) SELECT * FROM (WITH t AS (SELECT false) SELECT * FROM t) ---- -project - ├── columns: bool:2(bool!null) - ├── values - │ └── tuple [type=tuple] - └── projections - └── false [type=bool] +with &1 (t) + ├── columns: bool:3(bool!null) + ├── project + │ ├── columns: bool:1(bool!null) + │ ├── values + │ │ └── tuple [type=tuple] + │ └── projections + │ └── true [type=bool] + └── with &2 (t) + ├── columns: bool:3(bool!null) + ├── project + │ ├── columns: bool:2(bool!null) + │ ├── values + │ │ └── tuple [type=tuple] + │ └── projections + │ └── false [type=bool] + └── with-scan &2 (t) + ├── columns: bool:3(bool!null) + └── mapping: + └── bool:2(bool) => bool:3(bool) build WITH @@ -175,7 +255,36 @@ WITH t2 AS (SELECT * FROM t1) SELECT * FROM t1 NATURAL JOIN t2 ---- -error (0A000): unimplemented: unsupported multiple use of CTE clause "t1" +with &1 (t1) + ├── columns: bool:3(bool!null) + ├── project + │ ├── columns: bool:1(bool!null) + │ ├── values + │ │ └── tuple [type=tuple] + │ └── projections + │ └── true [type=bool] + └── with &2 (t2) + ├── columns: bool:3(bool!null) + ├── with-scan &1 (t1) + │ ├── columns: bool:2(bool!null) + │ └── mapping: + │ └── bool:1(bool) => bool:2(bool) + └── project + ├── columns: bool:3(bool!null) + └── inner-join (hash) + ├── columns: bool:3(bool!null) bool:4(bool!null) + ├── with-scan &1 (t1) + │ ├── columns: bool:3(bool!null) + │ └── mapping: + │ └── bool:1(bool) => bool:3(bool) + ├── with-scan &2 (t2) + │ ├── columns: bool:4(bool!null) + │ └── mapping: + │ └── bool:2(bool) => bool:4(bool) + └── filters + └── eq [type=bool] + ├── variable: bool [type=bool] + └── variable: bool [type=bool] build WITH @@ -183,105 +292,157 @@ WITH t2 AS (SELECT * FROM x NATURAL JOIN t1) SELECT * FROM t2 NATURAL JOIN x ---- -project - ├── columns: a:3(int!null) - └── inner-join (hash) - ├── columns: a:3(int!null) a:5(int!null) rowid:6(int!null) +with &1 (t1) + ├── columns: a:9(int!null) b:10(int!null) + ├── project + │ ├── columns: x.a:1(int) x.b:2(int) + │ └── scan x + │ └── columns: x.a:1(int) x.b:2(int) rowid:3(int!null) + └── with &2 (t2) + ├── columns: a:9(int!null) b:10(int!null) ├── project - │ ├── columns: a:3(int!null) + │ ├── columns: x.a:4(int!null) x.b:5(int!null) │ └── inner-join (hash) - │ ├── columns: a:1(int!null) a:3(int!null) rowid:4(int!null) + │ ├── columns: x.a:4(int!null) x.b:5(int!null) rowid:6(int!null) a:7(int!null) b:8(int!null) │ ├── scan x - │ │ └── columns: a:3(int) rowid:4(int!null) - │ ├── project - │ │ ├── columns: a:1(int) - │ │ └── scan x - │ │ └── columns: a:1(int) rowid:2(int!null) + │ │ └── columns: x.a:4(int) x.b:5(int) rowid:6(int!null) + │ ├── with-scan &1 (t1) + │ │ ├── columns: a:7(int) b:8(int) + │ │ └── mapping: + │ │ ├── x.a:1(int) => a:7(int) + │ │ └── x.b:2(int) => b:8(int) │ └── filters + │ ├── eq [type=bool] + │ │ ├── variable: x.a [type=int] + │ │ └── variable: a [type=int] │ └── eq [type=bool] - │ ├── variable: a [type=int] - │ └── variable: a [type=int] - ├── scan x - │ └── columns: a:5(int) rowid:6(int!null) - └── filters - └── eq [type=bool] - ├── variable: a [type=int] - └── variable: a [type=int] + │ ├── variable: x.b [type=int] + │ └── variable: b [type=int] + └── project + ├── columns: a:9(int!null) b:10(int!null) + └── inner-join (hash) + ├── columns: a:9(int!null) b:10(int!null) x.a:11(int!null) x.b:12(int!null) rowid:13(int!null) + ├── with-scan &2 (t2) + │ ├── columns: a:9(int!null) b:10(int!null) + │ └── mapping: + │ ├── x.a:4(int) => a:9(int) + │ └── x.b:5(int) => b:10(int) + ├── scan x + │ └── columns: x.a:11(int) x.b:12(int) rowid:13(int!null) + └── filters + ├── eq [type=bool] + │ ├── variable: a [type=int] + │ └── variable: x.a [type=int] + └── eq [type=bool] + ├── variable: b [type=int] + └── variable: x.b [type=int] build WITH t AS (SELECT a FROM y WHERE a < 3) SELECT * FROM t NATURAL JOIN t ---- -error (0A000): unimplemented: unsupported multiple use of CTE clause "t" +error (42712): source name "t" specified more than once (missing AS clause) build WITH t(x) AS (SELECT a FROM x) SELECT x FROM (SELECT x FROM t) ---- -project - ├── columns: x:1(int) - └── scan x - └── columns: a:1(int) rowid:2(int!null) +with &1 (t) + ├── columns: x:4(int) + ├── project + │ ├── columns: a:1(int) + │ └── scan x + │ └── columns: a:1(int) b:2(int) rowid:3(int!null) + └── with-scan &1 (t) + ├── columns: x:4(int) + └── mapping: + └── a:1(int) => x:4(int) build WITH t(a, b) AS (SELECT true a, false b) SELECT a, b FROM t ---- -project - ├── columns: a:1(bool!null) b:2(bool!null) - ├── values - │ └── tuple [type=tuple] - └── projections - ├── true [type=bool] - └── false [type=bool] +with &1 (t) + ├── columns: a:3(bool!null) b:4(bool!null) + ├── project + │ ├── columns: a:1(bool!null) b:2(bool!null) + │ ├── values + │ │ └── tuple [type=tuple] + │ └── projections + │ ├── true [type=bool] + │ └── false [type=bool] + └── with-scan &1 (t) + ├── columns: a:3(bool!null) b:4(bool!null) + └── mapping: + ├── a:1(bool) => a:3(bool) + └── b:2(bool) => b:4(bool) build WITH t(b, a) AS (SELECT true a, false b) SELECT a, b FROM t ---- -project - ├── columns: a:2(bool!null) b:1(bool!null) - ├── values - │ └── tuple [type=tuple] - └── projections - ├── true [type=bool] - └── false [type=bool] +with &1 (t) + ├── columns: a:4(bool!null) b:3(bool!null) + ├── project + │ ├── columns: a:1(bool!null) b:2(bool!null) + │ ├── values + │ │ └── tuple [type=tuple] + │ └── projections + │ ├── true [type=bool] + │ └── false [type=bool] + └── with-scan &1 (t) + ├── columns: b:3(bool!null) a:4(bool!null) + └── mapping: + ├── a:1(bool) => b:3(bool) + └── b:2(bool) => a:4(bool) build WITH t AS (SELECT a FROM x) SELECT * FROM y WHERE a IN (SELECT * FROM t) ---- -project - ├── columns: a:3(int) - └── select - ├── columns: y.a:3(int) y.rowid:4(int!null) - ├── scan y - │ └── columns: y.a:3(int) y.rowid:4(int!null) - └── filters - └── any: eq [type=bool] - ├── project - │ ├── columns: x.a:1(int) - │ └── scan x - │ └── columns: x.a:1(int) x.rowid:2(int!null) - └── variable: y.a [type=int] +with &1 (t) + ├── columns: a:4(int) + ├── project + │ ├── columns: x.a:1(int) + │ └── scan x + │ └── columns: x.a:1(int) b:2(int) x.rowid:3(int!null) + └── project + ├── columns: y.a:4(int) + └── select + ├── columns: y.a:4(int) y.rowid:5(int!null) + ├── scan y + │ └── columns: y.a:4(int) y.rowid:5(int!null) + └── filters + └── any: eq [type=bool] + ├── with-scan &1 (t) + │ ├── columns: a:6(int) + │ └── mapping: + │ └── x.a:1(int) => a:6(int) + └── variable: y.a [type=int] build WITH t(x) AS (SELECT a FROM x) SELECT * FROM y WHERE a IN (SELECT x FROM t) ---- -project - ├── columns: a:3(int) - └── select - ├── columns: y.a:3(int) y.rowid:4(int!null) - ├── scan y - │ └── columns: y.a:3(int) y.rowid:4(int!null) - └── filters - └── any: eq [type=bool] - ├── project - │ ├── columns: x.a:1(int) - │ └── scan x - │ └── columns: x.a:1(int) x.rowid:2(int!null) - └── variable: y.a [type=int] +with &1 (t) + ├── columns: a:4(int) + ├── project + │ ├── columns: x.a:1(int) + │ └── scan x + │ └── columns: x.a:1(int) b:2(int) x.rowid:3(int!null) + └── project + ├── columns: y.a:4(int) + └── select + ├── columns: y.a:4(int) y.rowid:5(int!null) + ├── scan y + │ └── columns: y.a:4(int) y.rowid:5(int!null) + └── filters + └── any: eq [type=bool] + ├── with-scan &1 (t) + │ ├── columns: x:6(int) + │ └── mapping: + │ └── x.a:1(int) => x:6(int) + └── variable: y.a [type=int] # Using a subquery inside a CTE build @@ -289,59 +450,51 @@ SELECT * FROM x WHERE a IN (WITH t AS (SELECT * FROM y WHERE a < 3) SELECT * FROM t) ---- project - ├── columns: a:1(int) + ├── columns: a:1(int) b:2(int) └── select - ├── columns: x.a:1(int) x.rowid:2(int!null) + ├── columns: x.a:1(int) b:2(int) x.rowid:3(int!null) ├── scan x - │ └── columns: x.a:1(int) x.rowid:2(int!null) + │ └── columns: x.a:1(int) b:2(int) x.rowid:3(int!null) └── filters └── any: eq [type=bool] - ├── project - │ ├── columns: y.a:3(int!null) - │ └── select - │ ├── columns: y.a:3(int!null) y.rowid:4(int!null) - │ ├── scan y - │ │ └── columns: y.a:3(int) y.rowid:4(int!null) - │ └── filters - │ └── lt [type=bool] - │ ├── variable: y.a [type=int] - │ └── const: 3 [type=int] + ├── with &1 (t) + │ ├── columns: a:6(int!null) + │ ├── project + │ │ ├── columns: y.a:4(int!null) + │ │ └── select + │ │ ├── columns: y.a:4(int!null) y.rowid:5(int!null) + │ │ ├── scan y + │ │ │ └── columns: y.a:4(int) y.rowid:5(int!null) + │ │ └── filters + │ │ └── lt [type=bool] + │ │ ├── variable: y.a [type=int] + │ │ └── const: 3 [type=int] + │ └── with-scan &1 (t) + │ ├── columns: a:6(int!null) + │ └── mapping: + │ └── y.a:4(int) => a:6(int) └── variable: x.a [type=int] # Using a correlated subquery inside a CTE build SELECT (WITH t AS (SELECT * FROM y WHERE x.a = y.a) SELECT * FROM t LIMIT 1) FROM x ---- -project - ├── columns: "?column?":5(int) - ├── scan x - │ └── columns: x.a:1(int) x.rowid:2(int!null) - └── projections - └── subquery [type=int] - └── max1-row - ├── columns: y.a:3(int!null) - └── limit - ├── columns: y.a:3(int!null) - ├── project - │ ├── columns: y.a:3(int!null) - │ └── select - │ ├── columns: y.a:3(int!null) y.rowid:4(int!null) - │ ├── scan y - │ │ └── columns: y.a:3(int) y.rowid:4(int!null) - │ └── filters - │ └── eq [type=bool] - │ ├── variable: x.a [type=int] - │ └── variable: y.a [type=int] - └── const: 1 [type=int] +error (0A000): CTEs may not be correlated # Rename columns build WITH t(b) AS (SELECT a FROM x) SELECT b, t.b FROM t ---- -project - ├── columns: b:1(int) b:1(int) - └── scan x - └── columns: a:1(int) rowid:2(int!null) +with &1 (t) + ├── columns: b:4(int) b:4(int) + ├── project + │ ├── columns: a:1(int) + │ └── scan x + │ └── columns: a:1(int) x.b:2(int) rowid:3(int!null) + └── with-scan &1 (t) + ├── columns: b:4(int) + └── mapping: + └── a:1(int) => b:4(int) build WITH t(b, c) AS (SELECT a FROM x) SELECT b, t.b FROM t @@ -358,21 +511,200 @@ error (42P01): no data source matches prefix: x build WITH t(x) AS (WITH t(x) AS (SELECT 1) SELECT x * 10 FROM t) SELECT x + 2 FROM t ---- -project - ├── columns: "?column?":3(int) +with &1 (t) + ├── columns: "?column?":5(int) ├── project - │ ├── columns: "?column?":2(int) - │ ├── project - │ │ ├── columns: "?column?":1(int!null) - │ │ ├── values - │ │ │ └── tuple [type=tuple] - │ │ └── projections - │ │ └── const: 1 [type=int] + │ ├── columns: "?column?":1(int!null) + │ ├── values + │ │ └── tuple [type=tuple] │ └── projections - │ └── mult [type=int] - │ ├── variable: ?column? [type=int] - │ └── const: 10 [type=int] - └── projections - └── plus [type=int] - ├── variable: ?column? [type=int] - └── const: 2 [type=int] + │ └── const: 1 [type=int] + └── with &2 (t) + ├── columns: "?column?":5(int) + ├── with &1 (t) + │ ├── columns: "?column?":3(int) + │ ├── project + │ │ ├── columns: "?column?":1(int!null) + │ │ ├── values + │ │ │ └── tuple [type=tuple] + │ │ └── projections + │ │ └── const: 1 [type=int] + │ └── project + │ ├── columns: "?column?":3(int) + │ ├── with-scan &1 (t) + │ │ ├── columns: x:2(int!null) + │ │ └── mapping: + │ │ └── "?column?":1(int) => x:2(int) + │ └── projections + │ └── mult [type=int] + │ ├── variable: x [type=int] + │ └── const: 10 [type=int] + └── project + ├── columns: "?column?":5(int) + ├── with-scan &2 (t) + │ ├── columns: x:4(int) + │ └── mapping: + │ └── "?column?":3(int) => x:4(int) + └── projections + └── plus [type=int] + ├── variable: x [type=int] + └── const: 2 [type=int] + +build +WITH one AS (SELECT a AS u FROM x), + two AS (SELECT a AS v FROM (SELECT a FROM y UNION ALL SELECT u FROM one)) + SELECT * FROM one JOIN two ON u = v +---- +with &1 (one) + ├── columns: u:8(int!null) v:9(int!null) + ├── project + │ ├── columns: x.a:1(int) + │ └── scan x + │ └── columns: x.a:1(int) b:2(int) x.rowid:3(int!null) + └── with &2 (two) + ├── columns: u:8(int!null) v:9(int!null) + ├── union-all + │ ├── columns: a:7(int) + │ ├── left columns: y.a:4(int) + │ ├── right columns: u:6(int) + │ ├── project + │ │ ├── columns: y.a:4(int) + │ │ └── scan y + │ │ └── columns: y.a:4(int) y.rowid:5(int!null) + │ └── with-scan &1 (one) + │ ├── columns: u:6(int) + │ └── mapping: + │ └── x.a:1(int) => u:6(int) + └── inner-join (hash) + ├── columns: u:8(int!null) v:9(int!null) + ├── with-scan &1 (one) + │ ├── columns: u:8(int) + │ └── mapping: + │ └── x.a:1(int) => u:8(int) + ├── with-scan &2 (two) + │ ├── columns: v:9(int) + │ └── mapping: + │ └── a:7(int) => v:9(int) + └── filters + └── eq [type=bool] + ├── variable: u [type=int] + └── variable: v [type=int] + +build +WITH foo AS (SELECT x.a FROM x ORDER by x.a) SELECT * FROM foo +---- +with &1 (foo) + ├── columns: a:4(int) + ├── project + │ ├── columns: x.a:1(int) + │ └── scan x + │ └── columns: x.a:1(int) b:2(int) rowid:3(int!null) + └── with-scan &1 (foo) + ├── columns: a:4(int) + └── mapping: + └── x.a:1(int) => a:4(int) + +# Mutations. +build +WITH t AS (SELECT a FROM x) INSERT INTO x SELECT a + 20 FROM t RETURNING * +---- +with &1 (t) + ├── columns: a:4(int) b:5(int) + ├── project + │ ├── columns: x.a:1(int) + │ └── scan x + │ └── columns: x.a:1(int) b:2(int) rowid:3(int!null) + └── project + ├── columns: x.a:4(int) b:5(int) + └── insert x + ├── columns: x.a:4(int) b:5(int) rowid:6(int!null) + ├── insert-mapping: + │ ├── "?column?":8 => x.a:4 + │ ├── column9:9 => b:5 + │ └── column10:10 => rowid:6 + └── project + ├── columns: column9:9(int) column10:10(int) "?column?":8(int) + ├── project + │ ├── columns: "?column?":8(int) + │ ├── with-scan &1 (t) + │ │ ├── columns: a:7(int) + │ │ └── mapping: + │ │ └── x.a:1(int) => a:7(int) + │ └── projections + │ └── plus [type=int] + │ ├── variable: a [type=int] + │ └── const: 20 [type=int] + └── projections + ├── cast: INT8 [type=int] + │ └── null [type=unknown] + └── function: unique_rowid [type=int] + +build +WITH t AS (SELECT a FROM x) UPDATE x SET a = (SELECT * FROM t) RETURNING * +---- +with &1 (t) + ├── columns: a:4(int) b:5(int) + ├── project + │ ├── columns: x.a:1(int) + │ └── scan x + │ └── columns: x.a:1(int) b:2(int) rowid:3(int!null) + └── project + ├── columns: x.a:4(int) b:5(int) + └── update x + ├── columns: x.a:4(int) b:5(int) rowid:6(int!null) + ├── fetch columns: x.a:7(int) b:8(int) rowid:9(int) + ├── update-mapping: + │ └── column11:11 => x.a:4 + └── project + ├── columns: column11:11(int) x.a:7(int) b:8(int) rowid:9(int!null) + ├── scan x + │ └── columns: x.a:7(int) b:8(int) rowid:9(int!null) + └── projections + └── subquery [type=int] + └── max1-row + ├── columns: a:10(int) + └── with-scan &1 (t) + ├── columns: a:10(int) + └── mapping: + └── x.a:1(int) => a:10(int) + +build +WITH t AS (SELECT a FROM x) DELETE FROM x WHERE a = (SELECT * FROM t) RETURNING * +---- +with &1 (t) + ├── columns: a:4(int!null) b:5(int) + ├── project + │ ├── columns: x.a:1(int) + │ └── scan x + │ └── columns: x.a:1(int) b:2(int) rowid:3(int!null) + └── project + ├── columns: x.a:4(int!null) b:5(int) + └── delete x + ├── columns: x.a:4(int!null) b:5(int) rowid:6(int!null) + ├── fetch columns: x.a:7(int) b:8(int) rowid:9(int) + └── select + ├── columns: x.a:7(int!null) b:8(int) rowid:9(int!null) + ├── scan x + │ └── columns: x.a:7(int) b:8(int) rowid:9(int!null) + └── filters + └── eq [type=bool] + ├── variable: x.a [type=int] + └── subquery [type=int] + └── max1-row + ├── columns: a:10(int) + └── with-scan &1 (t) + ├── columns: a:10(int) + └── mapping: + └── x.a:1(int) => a:10(int) + +# Correlated WITH is not allowed. + +build +SELECT (WITH foo AS (SELECT x.a FROM x WHERE x.a = y.a) SELECT a FROM foo) FROM y +---- +error (0A000): CTEs may not be correlated + +build +SELECT (WITH foo AS (SELECT (SELECT y.a) FROM x) SELECT a FROM foo) FROM y +---- +error (0A000): CTEs may not be correlated diff --git a/pkg/sql/opt/optbuilder/update.go b/pkg/sql/opt/optbuilder/update.go index 53e98637c0b3..7353e6f30465 100644 --- a/pkg/sql/opt/optbuilder/update.go +++ b/pkg/sql/opt/optbuilder/update.go @@ -74,9 +74,9 @@ func (b *Builder) buildUpdate(upd *tree.Update, inScope *scope) (outScope *scope panic(pgerror.DangerousStatementf("UPDATE without WHERE clause")) } + var ctes []cteSource if upd.With != nil { - inScope = b.buildCTE(upd.With.CTEList, inScope) - defer b.checkCTEUsage(inScope) + inScope, ctes = b.buildCTE(upd.With.CTEList, inScope) } // UPDATE xx AS yy - we want to know about xx (tn) because @@ -122,6 +122,8 @@ func (b *Builder) buildUpdate(upd *tree.Update, inScope *scope) (outScope *scope mb.buildUpdate(nil /* returning */) } + mb.outScope.expr = b.wrapWithCTEs(mb.outScope.expr, ctes) + return mb.outScope } diff --git a/pkg/sql/opt/optgen/cmd/optgen/metadata.go b/pkg/sql/opt/optgen/cmd/optgen/metadata.go index 24a0edf653cd..4c0783f939bc 100644 --- a/pkg/sql/opt/optgen/cmd/optgen/metadata.go +++ b/pkg/sql/opt/optgen/cmd/optgen/metadata.go @@ -150,6 +150,7 @@ func newMetadata(compiled *lang.CompiledExpr, pkg string) *metadata { "SchemaID": {fullName: "opt.SchemaID", passByVal: true}, "SequenceID": {fullName: "opt.SequenceID", passByVal: true}, "ValuesID": {fullName: "opt.ValuesID", passByVal: true}, + "WithID": {fullName: "opt.WithID", passByVal: true}, "Ordering": {fullName: "opt.Ordering", passByVal: true}, "OrderingChoice": {fullName: "physical.OrderingChoice", passByVal: true}, "TupleOrdinal": {fullName: "memo.TupleOrdinal", passByVal: true}, diff --git a/pkg/sql/opt_exec_factory.go b/pkg/sql/opt_exec_factory.go index 94315ead1b31..c6d1653792cd 100644 --- a/pkg/sql/opt_exec_factory.go +++ b/pkg/sql/opt_exec_factory.go @@ -783,6 +783,22 @@ func (ef *execFactory) ConstructMax1Row(input exec.Node) (exec.Node, error) { }, nil } +// ConstructBuffer is part of the exec.Factory interface. +func (ef *execFactory) ConstructBuffer(input exec.Node, label string) (exec.Node, error) { + return &bufferNode{ + plan: input.(planNode), + label: label, + }, nil +} + +// ConstructScanBuffer is part of the exec.Factory interface. +func (ef *execFactory) ConstructScanBuffer(ref exec.Node, label string) (exec.Node, error) { + return &scanBufferNode{ + buffer: ref.(*bufferNode), + label: label, + }, nil +} + // ConstructProjectSet is part of the exec.Factory interface. func (ef *execFactory) ConstructProjectSet( n exec.Node, exprs tree.TypedExprs, zipCols sqlbase.ResultColumns, numColsPerGen []int, diff --git a/pkg/sql/plan.go b/pkg/sql/plan.go index 31035658fc3c..8c4a08f4ccdc 100644 --- a/pkg/sql/plan.go +++ b/pkg/sql/plan.go @@ -144,7 +144,7 @@ type planNode interface { // Close terminates the planNode execution and releases its resources. // This method should be called if the node has been used in any way (any // methods on it have been called) after it was constructed. Note that this - // doesn't imply that startPlan() has been necessarily called. + // doesn't imply that startExec() has been necessarily called. // // This method must not be called during execution - the planNode // tree must remain "live" and readable via walk() even after diff --git a/pkg/sql/walk.go b/pkg/sql/walk.go index d559e1db76c5..868b1e9e9d12 100644 --- a/pkg/sql/walk.go +++ b/pkg/sql/walk.go @@ -649,7 +649,15 @@ func (v *planVisitor) visitInternal(plan planNode, name string) { case *errorIfRowsNode: n.plan = v.visit(n.plan) + case *scanBufferNode: + if v.observer.attr != nil { + v.observer.attr(name, "label", n.label) + } + case *bufferNode: + if v.observer.attr != nil { + v.observer.attr(name, "label", n.label) + } n.plan = v.visit(n.plan) } }