CTE: support explain CTE plan (#24986)

cmd/explaintest/r/explain_cte.result

@@ -0,0 +1,129 @@
+use test;
+drop table if exists t1, t2;
+create table t1 (c1 int primary key, c2 int, index c2 (c2));
+create table t2 (c1 int unique, c2 int);
+insert into t1 values(1, 0), (2, 1);
+insert into t2 values(1, 0), (2, 1);
+explain with cte(a) as (select 1) select * from cte;
+id	estRows	task	access object	operator info
+CTEFullScan_8	1.00	root	CTE:cte	data:CTE_0
+CTE_0	1.00	root		None Recursive CTE
+└─Projection_6(Seed Part)	1.00	root		1->Column#1
+  └─TableDual_7	1.00	root		rows:1
+explain with cte(a) as (select c1 from t1) select * from cte;
+id	estRows	task	access object	operator info
+CTEFullScan_11	1.00	root	CTE:cte	data:CTE_0
+CTE_0	1.00	root		None Recursive CTE
+└─TableReader_8(Seed Part)	10000.00	root		data:TableFullScan_7
+  └─TableFullScan_7	10000.00	cop[tikv]	table:t1	keep order:false, stats:pseudo
+explain with cte(a,b,c,d) as (select * from t1, t2) select * from cte;
+id	estRows	task	access object	operator info
+CTEFullScan_18	1.00	root	CTE:cte	data:CTE_0
+CTE_0	1.00	root		None Recursive CTE
+└─HashJoin_10(Seed Part)	100000000.00	root		CARTESIAN inner join
+  ├─TableReader_17(Build)	10000.00	root		data:TableFullScan_16
+  │ └─TableFullScan_16	10000.00	cop[tikv]	table:t2	keep order:false, stats:pseudo
+  └─TableReader_13(Probe)	10000.00	root		data:TableFullScan_12
+    └─TableFullScan_12	10000.00	cop[tikv]	table:t1	keep order:false, stats:pseudo
+explain with recursive cte(a) as (select 1 union select a+1 from cte where a < 10) select * from cte;
+id	estRows	task	access object	operator info
+CTEFullScan_17	1.00	root	CTE:cte	data:CTE_0
+CTE_0	1.00	root		Recursive CTE
+├─Projection_12(Seed Part)	1.00	root		1->Column#2
+│ └─TableDual_13	1.00	root		rows:1
+└─Projection_14(Recursive Part)	0.80	root		cast(plus(Column#3, 1), bigint(1) BINARY)->Column#5
+  └─Selection_15	0.80	root		lt(Column#3, 10)
+    └─CTETable_16	1.00	root		Scan on CTE_0
+explain with recursive cte(a) as (select c2 from t1 union select a+1 from cte where a < 10) select * from cte;
+id	estRows	task	access object	operator info
+CTEFullScan_20	1.00	root	CTE:cte	data:CTE_0
+CTE_0	1.00	root		Recursive CTE
+├─TableReader_14(Seed Part)	10000.00	root		data:TableFullScan_13
+│ └─TableFullScan_13	10000.00	cop[tikv]	table:t1	keep order:false, stats:pseudo
+└─Projection_17(Recursive Part)	0.80	root		cast(plus(test.t1.c2, 1), int(11))->test.t1.c2
+  └─Selection_18	0.80	root		lt(test.t1.c2, 10)
+    └─CTETable_19	1.00	root		Scan on CTE_0
+explain with cte(a) as (with recursive cte1(a) as (select 1 union select a + 1 from cte1 where a < 10) select * from cte1) select * from cte;
+id	estRows	task	access object	operator info
+CTEFullScan_21	1.00	root	CTE:cte	data:CTE_0
+CTE_0	1.00	root		None Recursive CTE
+└─CTEFullScan_20(Seed Part)	1.00	root	CTE:cte1	data:CTE_1
+CTE_1	1.00	root		Recursive CTE
+├─Projection_15(Seed Part)	1.00	root		1->Column#2
+│ └─TableDual_16	1.00	root		rows:1
+└─Projection_17(Recursive Part)	0.80	root		cast(plus(Column#3, 1), bigint(1) BINARY)->Column#5
+  └─Selection_18	0.80	root		lt(Column#3, 10)
+    └─CTETable_19	1.00	root		Scan on CTE_1
+explain with recursive cte(a) as (select 1 union select a+1 from cte where a < 10) select * from cte t1, cte t2;
+id	estRows	task	access object	operator info
+HashJoin_15	1.00	root		CARTESIAN inner join
+├─CTEFullScan_23(Build)	1.00	root	CTE:t2	data:CTE_0
+└─CTEFullScan_22(Probe)	1.00	root	CTE:t1	data:CTE_0
+CTE_0	1.00	root		Recursive CTE
+├─Projection_17(Seed Part)	1.00	root		1->Column#2
+│ └─TableDual_18	1.00	root		rows:1
+└─Projection_19(Recursive Part)	0.80	root		cast(plus(Column#3, 1), bigint(1) BINARY)->Column#5
+  └─Selection_20	0.80	root		lt(Column#3, 10)
+    └─CTETable_21	1.00	root		Scan on CTE_0
+explain with cte(a) as (with recursive cte1(a) as (select 1 union select a + 1 from cte1 where a < 10) select * from cte1) select * from cte t1, cte t2;
+id	estRows	task	access object	operator info
+HashJoin_17	1.00	root		CARTESIAN inner join
+├─CTEFullScan_27(Build)	1.00	root	CTE:t2	data:CTE_0
+└─CTEFullScan_26(Probe)	1.00	root	CTE:t1	data:CTE_0
+CTE_0	1.00	root		None Recursive CTE
+└─CTEFullScan_25(Seed Part)	1.00	root	CTE:cte1	data:CTE_1
+CTE_1	1.00	root		Recursive CTE
+├─Projection_20(Seed Part)	1.00	root		1->Column#2
+│ └─TableDual_21	1.00	root		rows:1
+└─Projection_22(Recursive Part)	0.80	root		cast(plus(Column#3, 1), bigint(1) BINARY)->Column#5
+  └─Selection_23	0.80	root		lt(Column#3, 10)
+    └─CTETable_24	1.00	root		Scan on CTE_1
+explain with recursive cte1(a) as (select 1 union select a+1 from cte1 where a < 10), cte2(a) as (select c2 from t1 union select a+1 from cte2 where a < 10) select * from cte1, cte2;
+id	estRows	task	access object	operator info
+HashJoin_23	1.00	root		CARTESIAN inner join
+├─CTEFullScan_39(Build)	1.00	root	CTE:cte2	data:CTE_1
+└─CTEFullScan_30(Probe)	1.00	root	CTE:cte1	data:CTE_0
+CTE_1	1.00	root		Recursive CTE
+├─TableReader_33(Seed Part)	10000.00	root		data:TableFullScan_32
+│ └─TableFullScan_32	10000.00	cop[tikv]	table:t1	keep order:false, stats:pseudo
+└─Projection_36(Recursive Part)	0.80	root		cast(plus(test.t1.c2, 1), int(11))->test.t1.c2
+  └─Selection_37	0.80	root		lt(test.t1.c2, 10)
+    └─CTETable_38	1.00	root		Scan on CTE_1
+CTE_0	1.00	root		Recursive CTE
+├─Projection_25(Seed Part)	1.00	root		1->Column#2
+│ └─TableDual_26	1.00	root		rows:1
+└─Projection_27(Recursive Part)	0.80	root		cast(plus(Column#3, 1), bigint(1) BINARY)->Column#5
+  └─Selection_28	0.80	root		lt(Column#3, 10)
+    └─CTETable_29	1.00	root		Scan on CTE_0
+explain with q(a,b) as (select * from t1) select /*+ merge(q) no_merge(q1) */ * from q, q q1 where q.a=1 and q1.a=2;
+id	estRows	task	access object	operator info
+HashJoin_12	0.64	root		CARTESIAN inner join
+├─Selection_21(Build)	0.80	root		eq(test.t1.c1, 2)
+│ └─CTEFullScan_22	1.00	root	CTE:q1	data:CTE_0
+└─Selection_14(Probe)	0.80	root		eq(test.t1.c1, 1)
+  └─CTEFullScan_20	1.00	root	CTE:q	data:CTE_0
+CTE_0	1.00	root		None Recursive CTE
+└─TableReader_17(Seed Part)	10000.00	root		data:TableFullScan_16
+  └─TableFullScan_16	10000.00	cop[tikv]	table:t1	keep order:false, stats:pseudo
+explain with recursive cte(a,b) as (select 1, concat('a', 1) union select a+1, concat(b, 1) from cte where a < 5) select * from cte;
+id	estRows	task	access object	operator info
+CTEFullScan_17	1.00	root	CTE:cte	data:CTE_0
+CTE_0	1.00	root		Recursive CTE
+├─Projection_12(Seed Part)	1.00	root		1->Column#3, a1->Column#4
+│ └─TableDual_13	1.00	root		rows:1
+└─Projection_14(Recursive Part)	0.80	root		cast(plus(Column#5, 1), bigint(1) BINARY)->Column#9, cast(concat(Column#6, 1), var_string(21))->Column#10
+  └─Selection_15	0.80	root		lt(Column#5, 5)
+    └─CTETable_16	1.00	root		Scan on CTE_0
+explain select * from t1 dt where exists(with recursive qn as (select c1*0+1 as b union all select b+1 from qn where b=0) select * from qn where b=1);
+id	estRows	task	access object	operator info
+Apply_19	10000.00	root		CARTESIAN semi join
+├─TableReader_21(Build)	10000.00	root		data:TableFullScan_20
+│ └─TableFullScan_20	10000.00	cop[tikv]	table:dt	keep order:false, stats:pseudo
+└─Selection_24(Probe)	0.80	root		eq(Column#8, 1)
+  └─CTEFullScan_30	1.00	root	CTE:qn	data:CTE_0
+CTE_0	1.00	root		Recursive CTE
+├─Projection_25(Seed Part)	1.00	root		plus(mul(test.t1.c1, 0), 1)->Column#4
+│ └─TableDual_26	1.00	root		rows:1
+└─Projection_27(Recursive Part)	0.80	root		plus(Column#5, 1)->Column#7
+  └─Selection_28	0.80	root		eq(Column#5, 0)
+    └─CTETable_29	1.00	root		Scan on CTE_0

cmd/explaintest/t/explain_cte.test

@@ -0,0 +1,31 @@
+use test;
+drop table if exists t1, t2;
+create table t1 (c1 int primary key, c2 int, index c2 (c2));
+create table t2 (c1 int unique, c2 int);
+insert into t1 values(1, 0), (2, 1);
+insert into t2 values(1, 0), (2, 1);
+
+# simple cte
+explain with cte(a) as (select 1) select * from cte;
+explain with cte(a) as (select c1 from t1) select * from cte;
+explain with cte(a,b,c,d) as (select * from t1, t2) select * from cte;
+
+# recursive cte
+explain with recursive cte(a) as (select 1 union select a+1 from cte where a < 10) select * from cte;
+explain with recursive cte(a) as (select c2 from t1 union select a+1 from cte where a < 10) select * from cte;
+
+# nested cte
+explain with cte(a) as (with recursive cte1(a) as (select 1 union select a + 1 from cte1 where a < 10) select * from cte1) select * from cte;
+
+# cte with join
+explain with recursive cte(a) as (select 1 union select a+1 from cte where a < 10) select * from cte t1, cte t2;
+explain with cte(a) as (with recursive cte1(a) as (select 1 union select a + 1 from cte1 where a < 10) select * from cte1) select * from cte t1, cte t2;
+
+# multiple cte
+explain with recursive cte1(a) as (select 1 union select a+1 from cte1 where a < 10), cte2(a) as (select c2 from t1 union select a+1 from cte2 where a < 10) select * from cte1, cte2;
+
+# other
+explain with q(a,b) as (select * from t1) select /*+ merge(q) no_merge(q1) */ * from q, q q1 where q.a=1 and q1.a=2;
+# explain with cte(a,b) as (select * from t1) select (select 1 from cte limit 1) from cte;
+explain with recursive cte(a,b) as (select 1, concat('a', 1) union select a+1, concat(b, 1) from cte where a < 5) select * from cte;
+explain select * from t1 dt where exists(with recursive qn as (select c1*0+1 as b union all select b+1 from qn where b=0) select * from qn where b=1);

planner/core/common_plans.go

@@ -955,6 +955,8 @@ type Explain struct {
 	Rows           [][]string
 	ExplainRows    [][]string
 	explainedPlans map[int]bool
+
+	ctes []*PhysicalCTE
 }
 
 // GetExplainRowsForPlan get explain rows for plan.
@@ -1016,6 +1018,10 @@ func (e *Explain) RenderResult() error {
 			if err != nil {
 				return err
 			}
+			err = e.explainPlanInRowFormatCTE()
+			if err != nil {
+				return err
+			}
 		}
 	case ast.ExplainFormatDOT:
 		if physicalPlan, ok := e.TargetPlan.(PhysicalPlan); ok {
@@ -1031,6 +1037,26 @@ func (e *Explain) RenderResult() error {
 	return nil
 }
 
+func (e *Explain) explainPlanInRowFormatCTE() (err error) {
+	explainedCTEPlan := make(map[int]struct{})
+	for i := 0; i < len(e.ctes); i++ {
+		x := (*CTEDefinition)(e.ctes[i])
+		// skip if the CTE has been explained, the same CTE has same IDForStorage
+		if _, ok := explainedCTEPlan[x.CTE.IDForStorage]; ok {
+			continue
+		}
+		e.prepareOperatorInfo(x, "root", "", "", true)
+		childIndent := texttree.Indent4Child("", true)
+		err = e.explainPlanInRowFormat(x.SeedPlan, "root", "(Seed Part)", childIndent, x.RecurPlan == nil)
+		if x.RecurPlan != nil {
+			err = e.explainPlanInRowFormat(x.RecurPlan, "root", "(Recursive Part)", childIndent, true)
+		}
+		explainedCTEPlan[x.CTE.IDForStorage] = struct{}{}
+	}
+
+	return
+}
+
 // explainPlanInRowFormat generates explain information for root-tasks.
 func (e *Explain) explainPlanInRowFormat(p Plan, taskType, driverSide, indent string, isLastChild bool) (err error) {
 	e.prepareOperatorInfo(p, taskType, driverSide, indent, isLastChild)
@@ -1134,6 +1160,8 @@ func (e *Explain) explainPlanInRowFormat(p Plan, taskType, driverSide, indent st
 		if x.Plan != nil {
 			err = e.explainPlanInRowFormat(x.Plan, "root", "", indent, true)
 		}
+	case *PhysicalCTE:
+		e.ctes = append(e.ctes, x)
 	}
 	return
 }

planner/core/encode.go

@@ -34,6 +34,8 @@ var encoderPool = sync.Pool{
 type planEncoder struct {
 	buf          bytes.Buffer
 	encodedPlans map[int]bool
+
+	ctes []*PhysicalCTE
 }
 
 // EncodePlan is used to encodePlan the plan to the plan tree with compressing.
@@ -59,9 +61,33 @@ func (pn *planEncoder) encodePlanTree(p Plan) string {
 	pn.encodedPlans = make(map[int]bool)
 	pn.buf.Reset()
 	pn.encodePlan(p, true, kv.TiKV, 0)
+	pn.encodeCTEPlan()
 	return plancodec.Compress(pn.buf.Bytes())
 }
 
+func (pn *planEncoder) encodeCTEPlan() {
+	explainedCTEPlan := make(map[int]struct{})
+	for i := 0; i < len(pn.ctes); i++ {
+		x := (*CTEDefinition)(pn.ctes[i])
+		// skip if the CTE has been explained, the same CTE has same IDForStorage
+		if _, ok := explainedCTEPlan[x.CTE.IDForStorage]; ok {
+			continue
+		}
+		taskTypeInfo := plancodec.EncodeTaskType(true, kv.TiKV)
+		actRows, analyzeInfo, memoryInfo, diskInfo := getRuntimeInfo(x.SCtx(), x, nil)
+		rowCount := 0.0
+		if statsInfo := x.statsInfo(); statsInfo != nil {
+			rowCount = x.statsInfo().RowCount
+		}
+		plancodec.EncodePlanNode(0, x.CTE.IDForStorage, plancodec.TypeCTEDefinition, rowCount, taskTypeInfo, x.ExplainInfo(), actRows, analyzeInfo, memoryInfo, diskInfo, &pn.buf)
+		pn.encodePlan(x.SeedPlan, true, kv.TiKV, 1)
+		if x.RecurPlan != nil {
+			pn.encodePlan(x.RecurPlan, true, kv.TiKV, 1)
+		}
+		explainedCTEPlan[x.CTE.IDForStorage] = struct{}{}
+	}
+}
+
 func (pn *planEncoder) encodePlan(p Plan, isRoot bool, store kv.StoreType, depth int) {
 	taskTypeInfo := plancodec.EncodeTaskType(isRoot, store)
 	actRows, analyzeInfo, memoryInfo, diskInfo := getRuntimeInfo(p.SCtx(), p, nil)
@@ -102,6 +128,8 @@ func (pn *planEncoder) encodePlan(p Plan, isRoot bool, store kv.StoreType, depth
 		if copPlan.tablePlan != nil {
 			pn.encodePlan(copPlan.tablePlan, false, store, depth)
 		}
+	case *PhysicalCTE:
+		pn.ctes = append(pn.ctes, copPlan)
 	}
 }
 

planner/core/find_best_task.go

@@ -1962,7 +1962,7 @@ func (p *LogicalCTE) findBestTask(prop *property.PhysicalProperty, planCounter *
 		}
 	}
 
-	pcte := PhysicalCTE{SeedPlan: sp, RecurPlan: rp, CTE: p.cte}.Init(p.ctx, p.stats)
+	pcte := PhysicalCTE{SeedPlan: sp, RecurPlan: rp, CTE: p.cte, cteAsName: p.cteAsName}.Init(p.ctx, p.stats)
 	pcte.SetSchema(p.schema)
 	t = &rootTask{pcte, sp.statsInfo().RowCount}
 	p.cte.cteTask = t

planner/core/logical_plan_builder.go

@@ -3668,11 +3668,12 @@ func (b *PlanBuilder) tryBuildCTE(ctx context.Context, tn *ast.TableName, asName
 
 			b.handleHelper.pushMap(nil)
 			var p LogicalPlan
-			lp := LogicalCTE{cte: &CTEClass{IsDistinct: cte.isDistinct, seedPartLogicalPlan: cte.seedLP, recursivePartLogicalPlan: cte.recurLP, IDForStorage: cte.storageID, optFlag: cte.optFlag}}.Init(b.ctx, b.getSelectOffset())
+			lp := LogicalCTE{cteAsName: tn.Name, cte: &CTEClass{IsDistinct: cte.isDistinct, seedPartLogicalPlan: cte.seedLP, recursivePartLogicalPlan: cte.recurLP, IDForStorage: cte.storageID, optFlag: cte.optFlag}}.Init(b.ctx, b.getSelectOffset())
 			lp.SetSchema(getResultCTESchema(cte.seedLP.Schema(), b.ctx.GetSessionVars()))
 			p = lp
 			p.SetOutputNames(cte.seedLP.OutputNames())
 			if len(asName.String()) > 0 {
+				lp.cteAsName = *asName
 				var on types.NameSlice
 				for _, name := range p.OutputNames() {
 					cpOn := *name

planner/core/logical_plans.go

@@ -1190,7 +1190,8 @@ type CTEClass struct {
 type LogicalCTE struct {
 	logicalSchemaProducer
 
-	cte *CTEClass
+	cte       *CTEClass
+	cteAsName model.CIStr
 }
 
 // LogicalCTETable is for CTE table

planner/core/physical_plans.go

@@ -14,6 +14,8 @@
 package core
 
 import (
+	"fmt"
+	"strconv"
 	"unsafe"
 
 	"github.com/pingcap/errors"
@@ -31,6 +33,7 @@ import (
 	"github.com/pingcap/tidb/table/tables"
 	"github.com/pingcap/tidb/types"
 	"github.com/pingcap/tidb/util/ranger"
+	"github.com/pingcap/tidb/util/stringutil"
 	"github.com/pingcap/tipb/go-tipb"
 )
 
@@ -1367,6 +1370,7 @@ type PhysicalCTE struct {
 	SeedPlan  PhysicalPlan
 	RecurPlan PhysicalPlan
 	CTE       *CTEClass
+	cteAsName model.CIStr
 }
 
 // PhysicalCTETable is for CTE table.
@@ -1384,3 +1388,51 @@ func (p *PhysicalCTE) ExtractCorrelatedCols() []*expression.CorrelatedColumn {
 	}
 	return corCols
 }
+
+// AccessObject implements physicalScan interface.
+func (p *PhysicalCTE) AccessObject(normalized bool) string {
+	return fmt.Sprintf("CTE:%s", p.cteAsName.L)
+}
+
+// OperatorInfo implements dataAccesser interface.
+func (p *PhysicalCTE) OperatorInfo(normalized bool) string {
+	return fmt.Sprintf("data:%s", (*CTEDefinition)(p).ExplainID())
+}
+
+// ExplainInfo implements Plan interface.
+func (p *PhysicalCTE) ExplainInfo() string {
+	return p.AccessObject(false) + ", " + p.OperatorInfo(false)
+}
+
+// ExplainID overrides the ExplainID.
+func (p *PhysicalCTE) ExplainID() fmt.Stringer {
+	return stringutil.MemoizeStr(func() string {
+		if p.ctx != nil && p.ctx.GetSessionVars().StmtCtx.IgnoreExplainIDSuffix {
+			return p.TP()
+		}
+		return p.TP() + "_" + strconv.Itoa(p.id)
+	})
+}
+
+// ExplainInfo overrides the ExplainInfo
+func (p *PhysicalCTETable) ExplainInfo() string {
+	return "Scan on CTE_" + strconv.Itoa(p.IDForStorage)
+}
+
+// CTEDefinition is CTE definition for explain.
+type CTEDefinition PhysicalCTE
+
+// ExplainInfo overrides the ExplainInfo
+func (p *CTEDefinition) ExplainInfo() string {
+	if p.RecurPlan != nil {
+		return "Recursive CTE"
+	}
+	return "None Recursive CTE"
+}
+
+// ExplainID overrides the ExplainID.
+func (p *CTEDefinition) ExplainID() fmt.Stringer {
+	return stringutil.MemoizeStr(func() string {
+		return "CTE_" + strconv.Itoa(p.CTE.IDForStorage)
+	})
+}