From e33b37789c616d7b7c997748cdd7eac6d6ec835c Mon Sep 17 00:00:00 2001 From: wjHuang Date: Thu, 3 Dec 2020 21:42:49 +0800 Subject: [PATCH] planner: add projection if expand virtual generated column (#20478) Signed-off-by: wjhuang2016 --- .../explain_generate_column_substitute.result | 44 +++++++++---------- cmd/explaintest/r/generated_columns.result | 26 ++++++----- expression/integration_test.go | 13 ++++++ planner/core/task.go | 26 ++++++++++- 4 files changed, 74 insertions(+), 35 deletions(-) diff --git a/cmd/explaintest/r/explain_generate_column_substitute.result b/cmd/explaintest/r/explain_generate_column_substitute.result index 803e11973e5a6..0e43618421be3 100644 --- a/cmd/explaintest/r/explain_generate_column_substitute.result +++ b/cmd/explaintest/r/explain_generate_column_substitute.result @@ -109,8 +109,8 @@ a b c e 5 -1 6 4 desc select a+1 from t order by a+1; id estRows task access object operator info -IndexReader_13 10000.00 root index:IndexFullScan_12 -└─IndexFullScan_12 10000.00 cop[tikv] table:t, index:idx_c(c) keep order:true, stats:pseudo +IndexReader_14 10000.00 root index:IndexFullScan_13 +└─IndexFullScan_13 10000.00 cop[tikv] table:t, index:idx_c(c) keep order:true, stats:pseudo select a+1 from t order by a+1; a+1 0 @@ -122,8 +122,8 @@ a+1 6 desc select b+a from t order by b+a; id estRows task access object operator info -IndexReader_13 10000.00 root index:IndexFullScan_12 -└─IndexFullScan_12 10000.00 cop[tikv] table:t, index:idx_e(e) keep order:true, stats:pseudo +IndexReader_14 10000.00 root index:IndexFullScan_13 +└─IndexFullScan_13 10000.00 cop[tikv] table:t, index:idx_e(e) keep order:true, stats:pseudo select b+a from t order by b+a; b+a -3 @@ -183,9 +183,9 @@ a+1 desc select c from t where a+1=3; id estRows task access object operator info Projection_4 10.00 root test.t.c -└─IndexLookUp_10 10.00 root - ├─IndexRangeScan_8(Build) 10.00 cop[tikv] table:t, index:expr_idx_c(`a` + 1) range:[3,3], keep order:false, stats:pseudo - └─TableRowIDScan_9(Probe) 10.00 cop[tikv] table:t keep order:false, stats:pseudo +└─IndexLookUp_11 10.00 root + ├─IndexRangeScan_9(Build) 10.00 cop[tikv] table:t, index:expr_idx_c(`a` + 1) range:[3,3], keep order:false, stats:pseudo + └─TableRowIDScan_10(Probe) 10.00 cop[tikv] table:t keep order:false, stats:pseudo select c from t where a+1=3; c 3 @@ -208,9 +208,9 @@ b+a desc select e from t where b+a=3; id estRows task access object operator info Projection_4 10.00 root test.t.e -└─IndexLookUp_10 10.00 root - ├─IndexRangeScan_8(Build) 10.00 cop[tikv] table:t, index:expr_idx_e(`b` + `a`) range:[3,3], keep order:false, stats:pseudo - └─TableRowIDScan_9(Probe) 10.00 cop[tikv] table:t keep order:false, stats:pseudo +└─IndexLookUp_11 10.00 root + ├─IndexRangeScan_9(Build) 10.00 cop[tikv] table:t, index:expr_idx_e(`b` + `a`) range:[3,3], keep order:false, stats:pseudo + └─TableRowIDScan_10(Probe) 10.00 cop[tikv] table:t keep order:false, stats:pseudo select e from t where b+a=3; e 3 @@ -276,8 +276,8 @@ a b c e 5 -1 6 4 desc select a+1 from t order by a+1; id estRows task access object operator info -IndexReader_13 10000.00 root index:IndexFullScan_12 -└─IndexFullScan_12 10000.00 cop[tikv] table:t, index:expr_idx_c(`a` + 1) keep order:true, stats:pseudo +IndexReader_14 10000.00 root index:IndexFullScan_13 +└─IndexFullScan_13 10000.00 cop[tikv] table:t, index:expr_idx_c(`a` + 1) keep order:true, stats:pseudo select a+1 from t order by a+1; a+1 0 @@ -289,8 +289,8 @@ a+1 6 desc select b+a from t order by b+a; id estRows task access object operator info -IndexReader_13 10000.00 root index:IndexFullScan_12 -└─IndexFullScan_12 10000.00 cop[tikv] table:t, index:expr_idx_e(`b` + `a`) keep order:true, stats:pseudo +IndexReader_14 10000.00 root index:IndexFullScan_13 +└─IndexFullScan_13 10000.00 cop[tikv] table:t, index:expr_idx_e(`b` + `a`) keep order:true, stats:pseudo select b+a from t order by b+a; b+a -3 @@ -345,29 +345,29 @@ drop table if exists t0; create table t0(c0 float(25), c1 double as (c0) unique); explain select c0 from t0; id estRows task access object operator info -IndexReader_7 10000.00 root index:IndexFullScan_6 -└─IndexFullScan_6 10000.00 cop[tikv] table:t0, index:c1(c1) keep order:false, stats:pseudo +IndexReader_8 10000.00 root index:IndexFullScan_7 +└─IndexFullScan_7 10000.00 cop[tikv] table:t0, index:c1(c1) keep order:false, stats:pseudo -- IndexRead drop table if exists t0; create table t0(c0 double, c1 double as (c0) unique); explain select c0 from t0; id estRows task access object operator info -IndexReader_7 10000.00 root index:IndexFullScan_6 -└─IndexFullScan_6 10000.00 cop[tikv] table:t0, index:c1(c1) keep order:false, stats:pseudo +IndexReader_8 10000.00 root index:IndexFullScan_7 +└─IndexFullScan_7 10000.00 cop[tikv] table:t0, index:c1(c1) keep order:false, stats:pseudo -- IndexRead drop table if exists t0; create table t0(c0 double, c1 double as (c0) unique); explain select c0 from t0; id estRows task access object operator info -IndexReader_7 10000.00 root index:IndexFullScan_6 -└─IndexFullScan_6 10000.00 cop[tikv] table:t0, index:c1(c1) keep order:false, stats:pseudo +IndexReader_8 10000.00 root index:IndexFullScan_7 +└─IndexFullScan_7 10000.00 cop[tikv] table:t0, index:c1(c1) keep order:false, stats:pseudo -- IndexRead drop table if exists t0; create table t0(c0 float(24), c1 float as (c0) unique); explain select c0 from t0; id estRows task access object operator info -IndexReader_7 10000.00 root index:IndexFullScan_6 -└─IndexFullScan_6 10000.00 cop[tikv] table:t0, index:c1(c1) keep order:false, stats:pseudo +IndexReader_8 10000.00 root index:IndexFullScan_7 +└─IndexFullScan_7 10000.00 cop[tikv] table:t0, index:c1(c1) keep order:false, stats:pseudo -- IndexRead drop table if exists t0; create table t0(c0 float(25), c1 float as (c0) unique); diff --git a/cmd/explaintest/r/generated_columns.result b/cmd/explaintest/r/generated_columns.result index dad75f5aeca9e..77585086bb35b 100644 --- a/cmd/explaintest/r/generated_columns.result +++ b/cmd/explaintest/r/generated_columns.result @@ -123,9 +123,9 @@ IndexReader_6 10.00 root index:IndexRangeScan_5 EXPLAIN SELECT b, c, d FROM t1 WHERE b=1; id estRows task access object operator info Projection_4 10.00 root test.t1.b, test.t1.c, test.t1.d -└─IndexLookUp_10 10.00 root - ├─IndexRangeScan_8(Build) 10.00 cop[tikv] table:t1, index:b(b) range:[1,1], keep order:false, stats:pseudo - └─TableRowIDScan_9(Probe) 10.00 cop[tikv] table:t1 keep order:false, stats:pseudo +└─IndexLookUp_11 10.00 root + ├─IndexRangeScan_9(Build) 10.00 cop[tikv] table:t1, index:b(b) range:[1,1], keep order:false, stats:pseudo + └─TableRowIDScan_10(Probe) 10.00 cop[tikv] table:t1 keep order:false, stats:pseudo EXPLAIN SELECT * FROM t1 WHERE b=1; id estRows task access object operator info IndexLookUp_10 10.00 root @@ -147,9 +147,9 @@ INSERT INTO person (name, address_info) VALUES ("John", CAST('{"city_no": 1}' AS EXPLAIN SELECT name FROM person where city_no=1; id estRows task access object operator info Projection_4 10.00 root test.person.name -└─IndexLookUp_10 10.00 root - ├─IndexRangeScan_8(Build) 10.00 cop[tikv] table:person, index:city_no(city_no) range:[1,1], keep order:false, stats:pseudo - └─TableRowIDScan_9(Probe) 10.00 cop[tikv] table:person keep order:false, stats:pseudo +└─IndexLookUp_11 10.00 root + ├─IndexRangeScan_9(Build) 10.00 cop[tikv] table:person, index:city_no(city_no) range:[1,1], keep order:false, stats:pseudo + └─TableRowIDScan_10(Probe) 10.00 cop[tikv] table:person keep order:false, stats:pseudo DROP TABLE IF EXISTS t1; CREATE TABLE t1 (a INT, b INT GENERATED ALWAYS AS (-a) VIRTUAL, @@ -177,9 +177,10 @@ HashAgg_5 8000.00 root group by:Column#7, funcs:sum(Column#6)->Column#5 EXPLAIN SELECT sum(b) FROM t1 GROUP BY c; id estRows task access object operator info HashAgg_5 8000.00 root group by:Column#9, funcs:sum(Column#8)->Column#5 -└─Projection_18 10000.00 root cast(test.t1.b, decimal(32,0) BINARY)->Column#8, test.t1.c - └─TableReader_11 10000.00 root data:TableFullScan_10 - └─TableFullScan_10 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo +└─Projection_20 10000.00 root cast(test.t1.b, decimal(32,0) BINARY)->Column#8, test.t1.c + └─Projection_12 10000.00 root test.t1.b, test.t1.c + └─TableReader_11 10000.00 root data:TableFullScan_10 + └─TableFullScan_10 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo EXPLAIN SELECT sum(c) FROM t1 GROUP BY a; id estRows task access object operator info HashAgg_9 8000.00 root group by:test.t1.a, funcs:sum(Column#6)->Column#5 @@ -189,9 +190,10 @@ HashAgg_9 8000.00 root group by:test.t1.a, funcs:sum(Column#6)->Column#5 EXPLAIN SELECT sum(c) FROM t1 GROUP BY b; id estRows task access object operator info HashAgg_5 8000.00 root group by:Column#7, funcs:sum(Column#6)->Column#5 -└─Projection_12 10000.00 root cast(test.t1.c, decimal(32,0) BINARY)->Column#6, test.t1.b - └─TableReader_9 10000.00 root data:TableFullScan_8 - └─TableFullScan_8 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo +└─Projection_14 10000.00 root cast(test.t1.c, decimal(32,0) BINARY)->Column#6, test.t1.b + └─Projection_10 10000.00 root test.t1.b, test.t1.c + └─TableReader_9 10000.00 root data:TableFullScan_8 + └─TableFullScan_8 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo DROP TABLE IF EXISTS tu; CREATE TABLE tu (a INT, b INT, c INT GENERATED ALWAYS AS (a + b) VIRTUAL, primary key (a), unique key uk(c)); INSERT INTO tu(a, b) VALUES(1, 2); diff --git a/expression/integration_test.go b/expression/integration_test.go index 5e16c86a16eaf..49c43affbc742 100644 --- a/expression/integration_test.go +++ b/expression/integration_test.go @@ -6055,6 +6055,19 @@ func (s *testIntegrationSerialSuite) TestCacheConstEval(c *C) { tk.MustExec("admin reload expr_pushdown_blacklist") } +func (s *testSuite) TestIssue20071(c *C) { + tk := testkit.NewTestKitWithInit(c, s.store) + tk.MustExec("drop table if exists table_30_utf8_4") + tk.MustExec("drop table if exists t") + tk.MustExec("create table t(a int)") + tk.MustExec("insert into t values(1)") + tk.MustExec("create table table_30_utf8_4 ( `pk` int primary key, `col_int_key_unsigned` int unsigned , `col_int_key_signed` int, `col_float_key_signed` float , `col_float_key_unsigned` float unsigned) character set utf8 partition by hash(pk) partitions 4;") + tk.MustExec("insert ignore into table_30_utf8_4 values (0,91, 10, 14,19.0495)") + tk.MustExec("alter table table_30_utf8_4 add column a int as (col_int_key_signed * 2)") + tk.MustExec("SELECT count(1) AS val FROM table_30_utf8_4 WHERE table_30_utf8_4.col_int_key_unsigned!=table_30_utf8_4.a OR (SELECT count(1) AS val FROM t WHERE table_30_utf8_4.col_float_key_signed!=table_30_utf8_4.col_float_key_unsigned )!=7984764426240273913;") + tk.MustExec("select a from table_30_utf8_4 order by a") +} + func (s *testSuite) TestVirtualGeneratedColumnAndLimit(c *C) { tk := testkit.NewTestKitWithInit(c, s.store) tk.MustExec("drop table if exists t;") diff --git a/planner/core/task.go b/planner/core/task.go index 9ea179e7c6ca3..ae54c46b89e78 100644 --- a/planner/core/task.go +++ b/planner/core/task.go @@ -172,6 +172,15 @@ func (p *basePhysicalPlan) attach2Task(tasks ...task) task { } func (p *PhysicalUnionScan) attach2Task(tasks ...task) task { + if pj, ok := tasks[0].plan().(*PhysicalProjection); ok { + // Convert unionScan->projection to projection->unionScan, because unionScan can't handle projection as its children. + p.SetChildren(pj.children...) + p.stats = tasks[0].plan().statsInfo() + rt, _ := tasks[0].(*rootTask) + rt.p = pj.children[0] + pj.SetChildren(p) + return pj.attach2Task(p.basePhysicalPlan.attach2Task(tasks...)) + } p.stats = tasks[0].plan().statsInfo() return p.basePhysicalPlan.attach2Task(tasks...) } @@ -724,6 +733,8 @@ func finishCopTask(ctx sessionctx.Context, task task) task { // the number of regions involved, we simply use DistSQLScanConcurrency. copIterWorkers := float64(t.plan().SCtx().GetSessionVars().DistSQLScanConcurrency()) t.finishIndexPlan() + needExtraProj := false + var prevSchema *expression.Schema // Network cost of transferring rows of table scan to TiDB. if t.tablePlan != nil { t.cst += t.count() * sessVars.NetworkFactor * t.tblColHists.GetAvgRowSize(ctx, t.tablePlan.Schema().Columns, false, false) @@ -738,7 +749,13 @@ func finishCopTask(ctx sessionctx.Context, task task) task { } } ts := tp.(*PhysicalTableScan) + prevColumnLen := len(ts.Columns) + prevSchema = ts.schema.Clone() ts.Columns = ExpandVirtualColumn(ts.Columns, ts.schema, ts.Table.Columns) + if len(ts.Columns) > prevColumnLen { + // Add an projection to make sure not to output extract columns. + needExtraProj = true + } } t.cst /= copIterWorkers newTask := &rootTask{ @@ -779,7 +796,14 @@ func finishCopTask(ctx sessionctx.Context, task task) task { }.Init(ctx, t.tablePlan.SelectBlockOffset()) p.PartitionInfo = t.partitionInfo p.stats = t.tablePlan.statsInfo() - newTask.p = p + if needExtraProj { + proj := PhysicalProjection{Exprs: expression.Column2Exprs(prevSchema.Columns)}.Init(ts.ctx, ts.stats, ts.SelectBlockOffset(), nil) + proj.SetSchema(prevSchema) + proj.SetChildren(p) + newTask.p = proj + } else { + newTask.p = p + } } if len(t.rootTaskConds) > 0 {