planner, CTE: Fix default inline CTE which contains agg or window function and refactor inline CTE strategy (#48436)

close #47711

Author: ti-chi-bot

cmd/explaintest/r/cte.result

@@ -790,15 +790,3 @@ with cte1 as (select 1), cte2 as (select 2) select * from cte1 union (with cte2
 1
 1
 3
-explain with cte1 as (select 1), cte2 as (select 2) select * from cte1 union (with cte2 as (select 3) select * from cte2 union all select * from cte2);
-id	estRows	task	access object	operator info
-HashAgg_24	3.00	root		group by:Column#9, funcs:firstrow(Column#9)->Column#9
-└─Union_25	3.00	root		
-  ├─Projection_26	1.00	root		1->Column#9
-  │ └─TableDual_27	1.00	root		rows:1
-  └─Union_29	2.00	root		
-    ├─CTEFullScan_31	1.00	root	CTE:cte2	data:CTE_2
-    └─CTEFullScan_33	1.00	root	CTE:cte2	data:CTE_2
-CTE_2	1.00	root		Non-Recursive CTE
-└─Projection_22(Seed Part)	1.00	root		3->Column#5
-  └─TableDual_23	1.00	root		rows:1

cmd/explaintest/r/explain_cte.result

@@ -167,33 +167,6 @@ CTE_0	8001.00	root		Recursive CTE, limit(offset:0, count:0)
 │ └─TableFullScan_17	10000.00	cop[tikv]	table:t1	keep order:false, stats:pseudo
 └─Projection_21(Recursive Part)	10000.00	root		cast(plus(test.t1.c1, 1), int(11))->test.t1.c1
   └─CTETable_22	10000.00	root		Scan on CTE_0
-explain with recursive cte1(c1) as (select c1 from t1 union select c1 from t2 limit 1) select * from cte1;
-id	estRows	task	access object	operator info
-CTEFullScan_34	1.00	root	CTE:cte1	data:CTE_0
-CTE_0	1.00	root		Non-Recursive CTE
-└─Limit_20(Seed Part)	1.00	root		offset:0, count:1
-  └─HashAgg_21	1.00	root		group by:Column#11, funcs:firstrow(Column#11)->Column#11
-    └─Union_22	20000.00	root		
-      ├─TableReader_25	10000.00	root		data:TableFullScan_24
-      │ └─TableFullScan_24	10000.00	cop[tikv]	table:t1	keep order:false, stats:pseudo
-      └─IndexReader_32	10000.00	root		index:IndexFullScan_31
-        └─IndexFullScan_31	10000.00	cop[tikv]	table:t2, index:c1(c1)	keep order:false, stats:pseudo
-explain with recursive cte1(c1) as (select c1 from t1 union select c1 from t2 limit 100 offset 100) select * from cte1;
-id	estRows	task	access object	operator info
-CTEFullScan_34	100.00	root	CTE:cte1	data:CTE_0
-CTE_0	100.00	root		Non-Recursive CTE
-└─Limit_20(Seed Part)	100.00	root		offset:100, count:100
-  └─HashAgg_21	200.00	root		group by:Column#11, funcs:firstrow(Column#11)->Column#11
-    └─Union_22	20000.00	root		
-      ├─TableReader_25	10000.00	root		data:TableFullScan_24
-      │ └─TableFullScan_24	10000.00	cop[tikv]	table:t1	keep order:false, stats:pseudo
-      └─IndexReader_32	10000.00	root		index:IndexFullScan_31
-        └─IndexFullScan_31	10000.00	cop[tikv]	table:t2, index:c1(c1)	keep order:false, stats:pseudo
-explain with recursive cte1(c1) as (select c1 from t1 union select c1 from t2 limit 0 offset 0) select * from cte1;
-id	estRows	task	access object	operator info
-CTEFullScan_18	0.00	root	CTE:cte1	data:CTE_0
-CTE_0	0.00	root		Non-Recursive CTE
-└─TableDual_16(Seed Part)	0.00	root		rows:0
 CREATE TABLE `customer` (
 `c_customer_sk` int(11) NOT NULL,
 `c_customer_id` char(16) NOT NULL,

cmd/explaintest/t/cte.test

@@ -336,4 +336,3 @@ INSERT INTO `t_dnmxh` VALUES (104,571000,NULL),(104,572000,44.37),(104,573000,59
 WITH cte_0 AS (select distinct ref_0.wkey as c0, ref_0.pkey as c1, ref_0.c_xhsndb as c2 from t_dnmxh as ref_0 where (1 <= ( select ref_1.pkey not in ( select ref_5.wkey as c0 from t_dnmxh as ref_5 where (ref_5.wkey < ( select ref_6.pkey as c0 from t_cqmg3b as ref_6 where 88 between 96 and 76)) ) as c0 from (t_cqmg3b as ref_1 left outer join t_dnmxh as ref_2 on (ref_1.wkey = ref_2.wkey )) where ref_0.c_xhsndb is NULL union select 33 <= 91 as c0 from t_cqmg3b as ref_8 ))), cte_1 AS (select ref_9.wkey as c0, ref_9.pkey as c1, ref_9.c_anpf_c as c2, ref_9.c_b_fp_c as c3, ref_9.c_ndccfb as c4, ref_9.c_8rswc as c5 from t_cqmg3b as ref_9) select count(1) from cte_0 as ref_10 where case when 56 < 50 then case when 100 in ( select distinct ref_11.c4 as c0 from cte_1 as ref_11 where (ref_11.c4 > ( select ref_13.pkey as c0 from t_dnmxh as ref_13 where (ref_13.wkey > ( select distinct ref_11.c1 as c0 from cte_0 as ref_14)) )) or (1 = 1)) then null else null end else '7mxv6' end not like 'ki4%vc';
 #case
 with cte1 as (select 1), cte2 as (select 2) select * from cte1 union (with cte2 as (select 3) select * from cte2 union all select * from cte2) order by 1;
-explain with cte1 as (select 1), cte2 as (select 2) select * from cte1 union (with cte2 as (select 3) select * from cte2 union all select * from cte2);

cmd/explaintest/t/explain_cte.test

@@ -39,11 +39,6 @@ explain with recursive cte1(c1) as (select c1 from t1 union select c1 + 1 c1 fro
 explain with recursive cte1(c1) as (select c1 from t1 union select c1 + 1 c1 from cte1 limit 1) select * from cte1 dt1 join cte1 dt2 on dt1.c1 = dt2.c1;
 explain with recursive cte1(c1) as (select c1 from t1 union select c1 + 1 c1 from cte1 limit 0 offset 0) select * from cte1 dt1 join cte1 dt2 on dt1.c1 = dt2.c1;
 
-# non-recursive limit
-explain with recursive cte1(c1) as (select c1 from t1 union select c1 from t2 limit 1) select * from cte1;
-explain with recursive cte1(c1) as (select c1 from t1 union select c1 from t2 limit 100 offset 100) select * from cte1;
-explain with recursive cte1(c1) as (select c1 from t1 union select c1 from t2 limit 0 offset 0) select * from cte1;
-
 # TPC-DS Q11
 CREATE TABLE `customer` (
   `c_customer_sk` int(11) NOT NULL,

planner/core/casetest/physical_plan_test.go

@@ -1432,6 +1432,8 @@ func TestSingleConsumerCTE(t *testing.T) {
 	tk.MustExec("use test")
 	tk.MustExec("drop table if exists t;")
 	tk.MustExec("CREATE TABLE `t` (`a` int(11));")
+	tk.MustExec("create table t1 (c1 int primary key, c2 int, index c2 (c2));")
+	tk.MustExec("create table t2 (c1 int unique, c2 int);")
 	tk.MustExec("insert into t values (1), (5), (10), (15), (20), (30), (50);")
 
 	var (

planner/core/casetest/testdata/flat_plan_suite_out.json

@@ -212,8 +212,8 @@
           {
             "Depth": 2,
             "Label": 0,
-            "IsRoot": true,
-            "StoreType": 2,
+            "IsRoot": false,
+            "StoreType": 0,
             "ReqType": 0,
             "IsPhysicalPlan": true,
             "TextTreeIndent": "│ │ ",
@@ -232,80 +232,15 @@
           {
             "Depth": 2,
             "Label": 0,
-            "IsRoot": true,
-            "StoreType": 2,
+            "IsRoot": false,
+            "StoreType": 0,
             "ReqType": 0,
             "IsPhysicalPlan": true,
             "TextTreeIndent": "  │ ",
             "IsLastChild": true
           }
         ],
-        "CTEs": [
-          [
-            {
-              "Depth": 0,
-              "Label": 0,
-              "IsRoot": true,
-              "StoreType": 2,
-              "ReqType": 0,
-              "IsPhysicalPlan": true,
-              "TextTreeIndent": "",
-              "IsLastChild": true
-            },
-            {
-              "Depth": 1,
-              "Label": 3,
-              "IsRoot": true,
-              "StoreType": 2,
-              "ReqType": 0,
-              "IsPhysicalPlan": true,
-              "TextTreeIndent": "│ ",
-              "IsLastChild": true
-            },
-            {
-              "Depth": 2,
-              "Label": 0,
-              "IsRoot": false,
-              "StoreType": 0,
-              "ReqType": 0,
-              "IsPhysicalPlan": true,
-              "TextTreeIndent": "  │ ",
-              "IsLastChild": true
-            }
-          ],
-          [
-            {
-              "Depth": 0,
-              "Label": 0,
-              "IsRoot": true,
-              "StoreType": 2,
-              "ReqType": 0,
-              "IsPhysicalPlan": true,
-              "TextTreeIndent": "",
-              "IsLastChild": true
-            },
-            {
-              "Depth": 1,
-              "Label": 3,
-              "IsRoot": true,
-              "StoreType": 2,
-              "ReqType": 0,
-              "IsPhysicalPlan": true,
-              "TextTreeIndent": "│ ",
-              "IsLastChild": true
-            },
-            {
-              "Depth": 2,
-              "Label": 0,
-              "IsRoot": false,
-              "StoreType": 0,
-              "ReqType": 0,
-              "IsPhysicalPlan": true,
-              "TextTreeIndent": "  │ ",
-              "IsLastChild": true
-            }
-          ]
-        ]
+        "CTEs": null
       },
       {
         "SQL": "WITH RECURSIVE cte (n) AS(  SELECT 1  UNION ALL  SELECT n + 1 FROM cte WHERE n < 5)SELECT * FROM cte;",

planner/core/casetest/testdata/integration_suite_out.json

@@ -6593,24 +6593,21 @@
       {
         "SQL": "explain format = 'brief' select /*+ qb_name(qb_v8, v8), merge(@qb_v8) */ * from v8;",
         "Plan": [
-          "HashAgg 16000.00 root  group by:Column#21, funcs:firstrow(Column#21)->Column#21",
+          "HashAgg 16000.00 root  group by:Column#41, funcs:firstrow(Column#41)->Column#41",
           "└─Union 1000000010000.00 root  ",
           "  ├─HashJoin 1000000000000.00 root  CARTESIAN inner join",
           "  │ ├─TableReader(Build) 10000.00 root  data:TableFullScan",
           "  │ │ └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo",
-          "  │ └─CTEFullScan(Probe) 100000000.00 root CTE:cte2 data:CTE_1",
+          "  │ └─Projection(Probe) 100000000.00 root  1->Column#55",
+          "  │   └─HashJoin 100000000.00 root  CARTESIAN inner join",
+          "  │     ├─Projection(Build) 10000.00 root  1->Column#54",
+          "  │     │ └─IndexReader 10000.00 root  index:IndexFullScan",
+          "  │     │   └─IndexFullScan 10000.00 cop[tikv] table:t3, index:idx_a(a) keep order:false, stats:pseudo",
+          "  │     └─Projection(Probe) 10000.00 root  1->Column#53",
+          "  │       └─IndexReader 10000.00 root  index:IndexFullScan",
+          "  │         └─IndexFullScan 10000.00 cop[tikv] table:t2, index:idx_a(a) keep order:false, stats:pseudo",
           "  └─TableReader 10000.00 root  data:TableFullScan",
-          "    └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo",
-          "CTE_1 100000000.00 root  Non-Recursive CTE",
-          "└─HashJoin(Seed Part) 100000000.00 root  CARTESIAN inner join",
-          "  ├─CTEFullScan(Build) 10000.00 root CTE:cte4 data:CTE_3",
-          "  └─CTEFullScan(Probe) 10000.00 root CTE:cte3 data:CTE_2",
-          "CTE_3 10000.00 root  Non-Recursive CTE",
-          "└─IndexReader(Seed Part) 10000.00 root  index:IndexFullScan",
-          "  └─IndexFullScan 10000.00 cop[tikv] table:t3, index:idx_a(a) keep order:false, stats:pseudo",
-          "CTE_2 10000.00 root  Non-Recursive CTE",
-          "└─IndexReader(Seed Part) 10000.00 root  index:IndexFullScan",
-          "  └─IndexFullScan 10000.00 cop[tikv] table:t2, index:idx_a(a) keep order:false, stats:pseudo"
+          "    └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo"
         ],
         "Warn": null
       },

planner/core/casetest/testdata/plan_suite_in.json

@@ -796,6 +796,10 @@
   {
     "name": "TestSingleConsumerCTE",
     "cases": [
+      "with recursive cte1(c1) as (select c1 from t1 union select c1 from t2 limit 1) select * from cte1; -- non-recursive limit, inline cte1",
+      "with recursive cte1(c1) as (select c1 from t1 union select c1 from t2 limit 100 offset 100) select * from cte1; -- non-recursive limit, inline cte1",
+      "with recursive cte1(c1) as (select c1 from t1 union select c1 from t2 limit 0 offset 0) select * from cte1; -- non-recursive limit, inline cte1",
+      "with cte1 as (select 1), cte2 as (select 2) select * from cte1 union (with cte2 as (select 3) select * from cte2 union all select * from cte2) -- inline cte1, not inline cte2",
       "with base1 as (WITH RECURSIVE cte(a) AS (with tmp as (select 1 as a) SELECT a from tmp UNION SELECT a+1 FROM cte) SELECT * FROM cte) select * from base1; -- issue #43318",
       "with cte as (select 1) select * from cte; -- inline cte",
       "with cte1 as (select 1), cte2 as (select 2) select * from cte1 union select * from cte2; -- inline cte1, cte2",
@@ -810,7 +814,10 @@
       "set tidb_opt_force_inline_cte=1; -- enable force inline CTE",
       "with cte as (select 1) select * from cte union select * from cte; -- force inline cte while multi-consumer",
       "set tidb_opt_force_inline_cte=0; -- disable force inline CTE",
-      "with cte as (select 1) select /*+ MERGE() */ * from cte union select * from cte; -- firstly inline cte, secondly cannot be inlined"
+      "with cte as (select 1) select /*+ MERGE() */ * from cte union select * from cte; -- firstly inline cte, secondly cannot be inlined",
+      "with a as (select 8 as id from dual),maxa as (select max(id) as max_id from a),b as (with recursive temp as (select 1 as lvl from dual union all select lvl+1 from temp, maxa where lvl < max_id)select * from temp) select * from b; -- issue #47711, maxa cannot be inlined because it contains agg and in the recursive part of cte temp",
+      "with a as (select count(*) from t1), b as (select 2 as bb from a), c as (with recursive tmp as (select 1 as res from t1 union all select res+1 from tmp,b where res+1 < bb) select * from tmp) select * from c; -- inline a, cannot be inline b because b indirectly contains agg and in the recursive part of cte tmp",
+      "with a as (select count(*) from t1), b as (select 2 as bb from a), c as (with recursive tmp as (select bb as res from b union all select res+1 from tmp where res +1 < 10) select * from tmp) select * from c; -- inline a, b, cannot be inline tmp, c"
     ]
   },
   {