Skip to content

Commit

Permalink
planner: add projection if expand virtual generated column (#20478)
Browse files Browse the repository at this point in the history 
Signed-off-by: wjhuang2016 <huangwenjun1997@gmail.com>
  • Loading branch information
@wjhuang2016
wjhuang2016 authored Dec 3, 2020
1 parent 8381461 commit e33b377
Show file tree
Hide file tree
Showing 4 changed files with 74 additions and 35 deletions.
44 changes: 22 additions & 22 deletions cmd/explaintest/r/explain_generate_column_substitute.result
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,8 +109,8 @@ a b c e
5 -1 6 4
desc select a+1 from t order by a+1;
id estRows task access object operator info
IndexReader_13 10000.00 root index:IndexFullScan_12
└─IndexFullScan_12 10000.00 cop[tikv] table:t, index:idx_c(c) keep order:true, stats:pseudo
IndexReader_14 10000.00 root index:IndexFullScan_13
└─IndexFullScan_13 10000.00 cop[tikv] table:t, index:idx_c(c) keep order:true, stats:pseudo
select a+1 from t order by a+1;
a+1
0
Expand All @@ -122,8 +122,8 @@ a+1
6
desc select b+a from t order by b+a;
id estRows task access object operator info
IndexReader_13 10000.00 root index:IndexFullScan_12
└─IndexFullScan_12 10000.00 cop[tikv] table:t, index:idx_e(e) keep order:true, stats:pseudo
IndexReader_14 10000.00 root index:IndexFullScan_13
└─IndexFullScan_13 10000.00 cop[tikv] table:t, index:idx_e(e) keep order:true, stats:pseudo
select b+a from t order by b+a;
b+a
-3
Expand Down Expand Up @@ -183,9 +183,9 @@ a+1
desc select c from t where a+1=3;
id estRows task access object operator info
Projection_4 10.00 root test.t.c
└─IndexLookUp_10 10.00 root
├─IndexRangeScan_8(Build) 10.00 cop[tikv] table:t, index:expr_idx_c(`a` + 1) range:[3,3], keep order:false, stats:pseudo
└─TableRowIDScan_9(Probe) 10.00 cop[tikv] table:t keep order:false, stats:pseudo
└─IndexLookUp_11 10.00 root
├─IndexRangeScan_9(Build) 10.00 cop[tikv] table:t, index:expr_idx_c(`a` + 1) range:[3,3], keep order:false, stats:pseudo
└─TableRowIDScan_10(Probe) 10.00 cop[tikv] table:t keep order:false, stats:pseudo
select c from t where a+1=3;
c
3
Expand All @@ -208,9 +208,9 @@ b+a
desc select e from t where b+a=3;
id estRows task access object operator info
Projection_4 10.00 root test.t.e
└─IndexLookUp_10 10.00 root
├─IndexRangeScan_8(Build) 10.00 cop[tikv] table:t, index:expr_idx_e(`b` + `a`) range:[3,3], keep order:false, stats:pseudo
└─TableRowIDScan_9(Probe) 10.00 cop[tikv] table:t keep order:false, stats:pseudo
└─IndexLookUp_11 10.00 root
├─IndexRangeScan_9(Build) 10.00 cop[tikv] table:t, index:expr_idx_e(`b` + `a`) range:[3,3], keep order:false, stats:pseudo
└─TableRowIDScan_10(Probe) 10.00 cop[tikv] table:t keep order:false, stats:pseudo
select e from t where b+a=3;
e
3
Expand Down Expand Up @@ -276,8 +276,8 @@ a b c e
5 -1 6 4
desc select a+1 from t order by a+1;
id estRows task access object operator info
IndexReader_13 10000.00 root index:IndexFullScan_12
└─IndexFullScan_12 10000.00 cop[tikv] table:t, index:expr_idx_c(`a` + 1) keep order:true, stats:pseudo
IndexReader_14 10000.00 root index:IndexFullScan_13
└─IndexFullScan_13 10000.00 cop[tikv] table:t, index:expr_idx_c(`a` + 1) keep order:true, stats:pseudo
select a+1 from t order by a+1;
a+1
0
Expand All @@ -289,8 +289,8 @@ a+1
6
desc select b+a from t order by b+a;
id estRows task access object operator info
IndexReader_13 10000.00 root index:IndexFullScan_12
└─IndexFullScan_12 10000.00 cop[tikv] table:t, index:expr_idx_e(`b` + `a`) keep order:true, stats:pseudo
IndexReader_14 10000.00 root index:IndexFullScan_13
└─IndexFullScan_13 10000.00 cop[tikv] table:t, index:expr_idx_e(`b` + `a`) keep order:true, stats:pseudo
select b+a from t order by b+a;
b+a
-3
Expand Down Expand Up @@ -345,29 +345,29 @@ drop table if exists t0;
create table t0(c0 float(25), c1 double as (c0) unique);
explain select c0 from t0;
id estRows task access object operator info
IndexReader_7 10000.00 root index:IndexFullScan_6
└─IndexFullScan_6 10000.00 cop[tikv] table:t0, index:c1(c1) keep order:false, stats:pseudo
IndexReader_8 10000.00 root index:IndexFullScan_7
└─IndexFullScan_7 10000.00 cop[tikv] table:t0, index:c1(c1) keep order:false, stats:pseudo
-- IndexRead
drop table if exists t0;
create table t0(c0 double, c1 double as (c0) unique);
explain select c0 from t0;
id estRows task access object operator info
IndexReader_7 10000.00 root index:IndexFullScan_6
└─IndexFullScan_6 10000.00 cop[tikv] table:t0, index:c1(c1) keep order:false, stats:pseudo
IndexReader_8 10000.00 root index:IndexFullScan_7
└─IndexFullScan_7 10000.00 cop[tikv] table:t0, index:c1(c1) keep order:false, stats:pseudo
-- IndexRead
drop table if exists t0;
create table t0(c0 double, c1 double as (c0) unique);
explain select c0 from t0;
id estRows task access object operator info
IndexReader_7 10000.00 root index:IndexFullScan_6
└─IndexFullScan_6 10000.00 cop[tikv] table:t0, index:c1(c1) keep order:false, stats:pseudo
IndexReader_8 10000.00 root index:IndexFullScan_7
└─IndexFullScan_7 10000.00 cop[tikv] table:t0, index:c1(c1) keep order:false, stats:pseudo
-- IndexRead
drop table if exists t0;
create table t0(c0 float(24), c1 float as (c0) unique);
explain select c0 from t0;
id estRows task access object operator info
IndexReader_7 10000.00 root index:IndexFullScan_6
└─IndexFullScan_6 10000.00 cop[tikv] table:t0, index:c1(c1) keep order:false, stats:pseudo
IndexReader_8 10000.00 root index:IndexFullScan_7
└─IndexFullScan_7 10000.00 cop[tikv] table:t0, index:c1(c1) keep order:false, stats:pseudo
-- IndexRead
drop table if exists t0;
create table t0(c0 float(25), c1 float as (c0) unique);
Expand Down
26 changes: 14 additions & 12 deletions cmd/explaintest/r/generated_columns.result
View file
Original file line number Diff line number Diff line change
Expand Up @@ -123,9 +123,9 @@ IndexReader_6 10.00 root index:IndexRangeScan_5
EXPLAIN SELECT b, c, d FROM t1 WHERE b=1;
id estRows task access object operator info
Projection_4 10.00 root test.t1.b, test.t1.c, test.t1.d
└─IndexLookUp_10 10.00 root
├─IndexRangeScan_8(Build) 10.00 cop[tikv] table:t1, index:b(b) range:[1,1], keep order:false, stats:pseudo
└─TableRowIDScan_9(Probe) 10.00 cop[tikv] table:t1 keep order:false, stats:pseudo
└─IndexLookUp_11 10.00 root
├─IndexRangeScan_9(Build) 10.00 cop[tikv] table:t1, index:b(b) range:[1,1], keep order:false, stats:pseudo
└─TableRowIDScan_10(Probe) 10.00 cop[tikv] table:t1 keep order:false, stats:pseudo
EXPLAIN SELECT * FROM t1 WHERE b=1;
id estRows task access object operator info
IndexLookUp_10 10.00 root
Expand All @@ -147,9 +147,9 @@ INSERT INTO person (name, address_info) VALUES ("John", CAST('{"city_no": 1}' AS
EXPLAIN SELECT name FROM person where city_no=1;
id estRows task access object operator info
Projection_4 10.00 root test.person.name
└─IndexLookUp_10 10.00 root
├─IndexRangeScan_8(Build) 10.00 cop[tikv] table:person, index:city_no(city_no) range:[1,1], keep order:false, stats:pseudo
└─TableRowIDScan_9(Probe) 10.00 cop[tikv] table:person keep order:false, stats:pseudo
└─IndexLookUp_11 10.00 root
├─IndexRangeScan_9(Build) 10.00 cop[tikv] table:person, index:city_no(city_no) range:[1,1], keep order:false, stats:pseudo
└─TableRowIDScan_10(Probe) 10.00 cop[tikv] table:person keep order:false, stats:pseudo
DROP TABLE IF EXISTS t1;
CREATE TABLE t1 (a INT,
b INT GENERATED ALWAYS AS (-a) VIRTUAL,
Expand Down Expand Up @@ -177,9 +177,10 @@ HashAgg_5 8000.00 root group by:Column#7, funcs:sum(Column#6)->Column#5
EXPLAIN SELECT sum(b) FROM t1 GROUP BY c;
id estRows task access object operator info
HashAgg_5 8000.00 root group by:Column#9, funcs:sum(Column#8)->Column#5
└─Projection_18 10000.00 root cast(test.t1.b, decimal(32,0) BINARY)->Column#8, test.t1.c
└─TableReader_11 10000.00 root data:TableFullScan_10
└─TableFullScan_10 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo
└─Projection_20 10000.00 root cast(test.t1.b, decimal(32,0) BINARY)->Column#8, test.t1.c
└─Projection_12 10000.00 root test.t1.b, test.t1.c
└─TableReader_11 10000.00 root data:TableFullScan_10
└─TableFullScan_10 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo
EXPLAIN SELECT sum(c) FROM t1 GROUP BY a;
id estRows task access object operator info
HashAgg_9 8000.00 root group by:test.t1.a, funcs:sum(Column#6)->Column#5
Expand All @@ -189,9 +190,10 @@ HashAgg_9 8000.00 root group by:test.t1.a, funcs:sum(Column#6)->Column#5
EXPLAIN SELECT sum(c) FROM t1 GROUP BY b;
id estRows task access object operator info
HashAgg_5 8000.00 root group by:Column#7, funcs:sum(Column#6)->Column#5
└─Projection_12 10000.00 root cast(test.t1.c, decimal(32,0) BINARY)->Column#6, test.t1.b
└─TableReader_9 10000.00 root data:TableFullScan_8
└─TableFullScan_8 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo
└─Projection_14 10000.00 root cast(test.t1.c, decimal(32,0) BINARY)->Column#6, test.t1.b
└─Projection_10 10000.00 root test.t1.b, test.t1.c
└─TableReader_9 10000.00 root data:TableFullScan_8
└─TableFullScan_8 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo
DROP TABLE IF EXISTS tu;
CREATE TABLE tu (a INT, b INT, c INT GENERATED ALWAYS AS (a + b) VIRTUAL, primary key (a), unique key uk(c));
INSERT INTO tu(a, b) VALUES(1, 2);
Expand Down
13 changes: 13 additions & 0 deletions expression/integration_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6055,6 +6055,19 @@ func (s *testIntegrationSerialSuite) TestCacheConstEval(c *C) {
tk.MustExec("admin reload expr_pushdown_blacklist")
}

func (s *testSuite) TestIssue20071(c *C) {
tk := testkit.NewTestKitWithInit(c, s.store)
tk.MustExec("drop table if exists table_30_utf8_4")
tk.MustExec("drop table if exists t")
tk.MustExec("create table t(a int)")
tk.MustExec("insert into t values(1)")
tk.MustExec("create table table_30_utf8_4 ( `pk` int primary key, `col_int_key_unsigned` int unsigned , `col_int_key_signed` int, `col_float_key_signed` float , `col_float_key_unsigned` float unsigned) character set utf8 partition by hash(pk) partitions 4;")
tk.MustExec("insert ignore into table_30_utf8_4 values (0,91, 10, 14,19.0495)")
tk.MustExec("alter table table_30_utf8_4 add column a int as (col_int_key_signed * 2)")
tk.MustExec("SELECT count(1) AS val FROM table_30_utf8_4 WHERE table_30_utf8_4.col_int_key_unsigned!=table_30_utf8_4.a OR (SELECT count(1) AS val FROM t WHERE table_30_utf8_4.col_float_key_signed!=table_30_utf8_4.col_float_key_unsigned )!=7984764426240273913;")
tk.MustExec("select a from table_30_utf8_4 order by a")
}

func (s *testSuite) TestVirtualGeneratedColumnAndLimit(c *C) {
tk := testkit.NewTestKitWithInit(c, s.store)
tk.MustExec("drop table if exists t;")
Expand Down
26 changes: 25 additions & 1 deletion planner/core/task.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,15 @@ func (p *basePhysicalPlan) attach2Task(tasks ...task) task {
}

func (p *PhysicalUnionScan) attach2Task(tasks ...task) task {
if pj, ok := tasks[0].plan().(*PhysicalProjection); ok {
// Convert unionScan->projection to projection->unionScan, because unionScan can't handle projection as its children.
p.SetChildren(pj.children...)
p.stats = tasks[0].plan().statsInfo()
rt, _ := tasks[0].(*rootTask)
rt.p = pj.children[0]
pj.SetChildren(p)
return pj.attach2Task(p.basePhysicalPlan.attach2Task(tasks...))
}
p.stats = tasks[0].plan().statsInfo()
return p.basePhysicalPlan.attach2Task(tasks...)
}
Expand Down Expand Up @@ -724,6 +733,8 @@ func finishCopTask(ctx sessionctx.Context, task task) task {
// the number of regions involved, we simply use DistSQLScanConcurrency.
copIterWorkers := float64(t.plan().SCtx().GetSessionVars().DistSQLScanConcurrency())
t.finishIndexPlan()
needExtraProj := false
var prevSchema *expression.Schema
// Network cost of transferring rows of table scan to TiDB.
if t.tablePlan != nil {
t.cst += t.count() * sessVars.NetworkFactor * t.tblColHists.GetAvgRowSize(ctx, t.tablePlan.Schema().Columns, false, false)
Expand All @@ -738,7 +749,13 @@ func finishCopTask(ctx sessionctx.Context, task task) task {
}
}
ts := tp.(*PhysicalTableScan)
prevColumnLen := len(ts.Columns)
prevSchema = ts.schema.Clone()
ts.Columns = ExpandVirtualColumn(ts.Columns, ts.schema, ts.Table.Columns)
if len(ts.Columns) > prevColumnLen {
// Add an projection to make sure not to output extract columns.
needExtraProj = true
}
}
t.cst /= copIterWorkers
newTask := &rootTask{
Expand Down Expand Up @@ -779,7 +796,14 @@ func finishCopTask(ctx sessionctx.Context, task task) task {
}.Init(ctx, t.tablePlan.SelectBlockOffset())
p.PartitionInfo = t.partitionInfo
p.stats = t.tablePlan.statsInfo()
newTask.p = p
if needExtraProj {
proj := PhysicalProjection{Exprs: expression.Column2Exprs(prevSchema.Columns)}.Init(ts.ctx, ts.stats, ts.SelectBlockOffset(), nil)
proj.SetSchema(prevSchema)
proj.SetChildren(p)
newTask.p = proj
} else {
newTask.p = p
}
}

if len(t.rootTaskConds) > 0 {
Expand Down

0 comments on commit e33b377

Please sign in to comment.