Skip to content

Commit

Permalink
expression, cmd: fix ColumnSubstitute and allow some cases to substit…
Browse files Browse the repository at this point in the history
  • Loading branch information
wjhuang2016 authored and ti-chi-bot committed Jan 31, 2023
1 parent f54825b commit d6adc09
Show file tree
Hide file tree
Showing 8 changed files with 97 additions and 21 deletions.
24 changes: 24 additions & 0 deletions cmd/explaintest/r/collation_check_use_collation_disabled.result
Original file line number Diff line number Diff line change
Expand Up @@ -154,4 +154,28 @@ insert into t1 values ('-1');
insert into t2 values (0x2d31, '');
select * from t1, t2 where t1.a in (t2.b, 3);
a b c
drop table if exists t0;
drop table if exists t1;
CREATE TABLE t0(c0 BOOL, c1 INT);
CREATE TABLE t1 LIKE t0;
CREATE VIEW v0(c0) AS SELECT IS_IPV4(t0.c1) FROM t0, t1;
INSERT INTO t0(c0, c1) VALUES (true, 0);
INSERT INTO t1(c0, c1) VALUES (true, 2);
SELECT v0.c0 FROM v0;
c0
0
SELECT (v0.c0)NOT LIKE(BINARY v0.c0) FROM v0;
(v0.c0)NOT LIKE(BINARY v0.c0)
0
SELECT v0.c0 FROM v0 WHERE (v0.c0)NOT LIKE(BINARY v0.c0);
c0
desc format='brief' SELECT v0.c0 FROM v0 WHERE (v0.c0)NOT LIKE(BINARY v0.c0);
id estRows task access object operator info
Projection 80000000.00 root is_ipv4(cast(collation_check_use_collation.t0.c1, var_string(20)))->Column#7
└─HashJoin 80000000.00 root CARTESIAN inner join
├─Selection(Build) 8000.00 root not(like(cast(is_ipv4(cast(collation_check_use_collation.t0.c1, var_string(20))), var_string(20)), cast(is_ipv4(cast(collation_check_use_collation.t0.c1, var_string(20))), binary(1)), 92))
│ └─TableReader 10000.00 root data:TableFullScan
│ └─TableFullScan 10000.00 cop[tikv] table:t0 keep order:false, stats:pseudo
└─TableReader(Probe) 10000.00 root data:TableFullScan
└─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo
use test
24 changes: 24 additions & 0 deletions cmd/explaintest/r/collation_check_use_collation_enabled.result
Original file line number Diff line number Diff line change
Expand Up @@ -173,4 +173,28 @@ insert into t1 values ('-1');
insert into t2 values (0x2d31, '');
select * from t1, t2 where t1.a in (t2.b, 3);
a b c
drop table if exists t0;
drop table if exists t1;
CREATE TABLE t0(c0 BOOL, c1 INT);
CREATE TABLE t1 LIKE t0;
CREATE VIEW v0(c0) AS SELECT IS_IPV4(t0.c1) FROM t0, t1;
INSERT INTO t0(c0, c1) VALUES (true, 0);
INSERT INTO t1(c0, c1) VALUES (true, 2);
SELECT v0.c0 FROM v0;
c0
0
SELECT (v0.c0)NOT LIKE(BINARY v0.c0) FROM v0;
(v0.c0)NOT LIKE(BINARY v0.c0)
0
SELECT v0.c0 FROM v0 WHERE (v0.c0)NOT LIKE(BINARY v0.c0);
c0
desc format='brief' SELECT v0.c0 FROM v0 WHERE (v0.c0)NOT LIKE(BINARY v0.c0);
id estRows task access object operator info
Projection 80000000.00 root is_ipv4(cast(collation_check_use_collation.t0.c1, var_string(20)))->Column#7
└─HashJoin 80000000.00 root CARTESIAN inner join
├─Selection(Build) 8000.00 root not(like(cast(is_ipv4(cast(collation_check_use_collation.t0.c1, var_string(20))), var_string(20)), cast(is_ipv4(cast(collation_check_use_collation.t0.c1, var_string(20))), binary(1)), 92))
│ └─TableReader 10000.00 root data:TableFullScan
│ └─TableFullScan 10000.00 cop[tikv] table:t0 keep order:false, stats:pseudo
└─TableReader(Probe) 10000.00 root data:TableFullScan
└─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo
use test
10 changes: 4 additions & 6 deletions cmd/explaintest/r/subquery.result
Original file line number Diff line number Diff line change
Expand Up @@ -56,13 +56,11 @@ insert into exam values(1, 'math', 100);
set names utf8 collate utf8_general_ci;
explain format = 'brief' select * from stu where stu.name not in (select 'guo' from exam where exam.stu_id = stu.id);
id estRows task access object operator info
Apply 10000.00 root CARTESIAN anti semi join, other cond:eq(test.stu.name, Column#8)
HashJoin 8000.00 root anti semi join, equal:[eq(test.stu.id, test.exam.stu_id)], other cond:eq(test.stu.name, "guo")
├─TableReader(Build) 10000.00 root data:TableFullScan
│ └─TableFullScan 10000.00 cop[tikv] table:stu keep order:false, stats:pseudo
└─Projection(Probe) 100000.00 root guo->Column#8
└─TableReader 100000.00 root data:Selection
└─Selection 100000.00 cop[tikv] eq(test.exam.stu_id, test.stu.id)
└─TableFullScan 100000000.00 cop[tikv] table:exam keep order:false, stats:pseudo
│ └─TableFullScan 10000.00 cop[tikv] table:exam keep order:false, stats:pseudo
└─TableReader(Probe) 10000.00 root data:TableFullScan
└─TableFullScan 10000.00 cop[tikv] table:stu keep order:false, stats:pseudo
select * from stu where stu.name not in (select 'guo' from exam where exam.stu_id = stu.id);
id name
set names utf8mb4;
Expand Down
14 changes: 14 additions & 0 deletions cmd/explaintest/t/collation_check_use_collation.test
Original file line number Diff line number Diff line change
Expand Up @@ -110,5 +110,19 @@ insert into t1 values ('-1');
insert into t2 values (0x2d31, '');
select * from t1, t2 where t1.a in (t2.b, 3);

# issue 38736
drop table if exists t0;
drop table if exists t1;
CREATE TABLE t0(c0 BOOL, c1 INT);
CREATE TABLE t1 LIKE t0;
CREATE VIEW v0(c0) AS SELECT IS_IPV4(t0.c1) FROM t0, t1;
INSERT INTO t0(c0, c1) VALUES (true, 0);
INSERT INTO t1(c0, c1) VALUES (true, 2);

SELECT v0.c0 FROM v0;
SELECT (v0.c0)NOT LIKE(BINARY v0.c0) FROM v0;
SELECT v0.c0 FROM v0 WHERE (v0.c0)NOT LIKE(BINARY v0.c0);
desc format='brief' SELECT v0.c0 FROM v0 WHERE (v0.c0)NOT LIKE(BINARY v0.c0);

# cleanup environment
use test
8 changes: 0 additions & 8 deletions expression/collation.go
Original file line number Diff line number Diff line change
Expand Up @@ -299,14 +299,6 @@ func deriveCollation(ctx sessionctx.Context, funcName string, args []Expression,
return ec, nil
}

// DeriveCollationFromExprs derives collation information from these expressions.
// Deprecated, use CheckAndDeriveCollationFromExprs instead.
// TODO: remove this function after the all usage is replaced by CheckAndDeriveCollationFromExprs
func DeriveCollationFromExprs(ctx sessionctx.Context, exprs ...Expression) (dstCharset, dstCollation string) {
collation := inferCollation(exprs...)
return collation.Charset, collation.Collation
}

// CheckAndDeriveCollationFromExprs derives collation information from these expressions, return error if derives collation error.
func CheckAndDeriveCollationFromExprs(ctx sessionctx.Context, funcName string, evalType types.EvalType, args ...Expression) (et *ExprCollation, err error) {
ec := inferCollation(args...)
Expand Down
9 changes: 9 additions & 0 deletions expression/integration_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -7895,3 +7895,12 @@ func TestIssue39146(t *testing.T) {
tk.MustExec("set @@tidb_enable_vectorized_expression = off;")
tk.MustQuery(`select str_to_date(substr(dest,1,6),'%H%i%s') from sun;`).Check(testkit.Rows("20:23:10"))
}

func TestIssue40536(t *testing.T) {
store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
tk.MustExec("CREATE TABLE `6bf9e76d-ab44-4031-8a07-418b10741580` (\n `e0b5f703-6cfe-49b4-bc21-16a6455e43a7` set('7','va','ung60','ow','1g','gxwz5','uhnh','k','5la1','q8d9c','1f') NOT NULL DEFAULT '7,1g,uhnh,5la1,q8d9c',\n `fbc3527f-9617-4b9d-a5dc-4be31c00d8a5` datetime DEFAULT '6449-09-28 14:39:04',\n PRIMARY KEY (`e0b5f703-6cfe-49b4-bc21-16a6455e43a7`) /*T![clustered_index] CLUSTERED */\n) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_bin;")
tk.MustExec("CREATE TABLE `8919f3f4-25be-4a1a-904a-bb5e863d8fc8` (\n `9804d5f2-cbc7-43b7-b241-ea2656dc941a` enum('s951','36d','ua65','49yru','6l2em','4ea','jf2d2','vprsc','3yl7n','hz','ov') DEFAULT '4ea',\n `323cdbcb-0c14-4362-90ab-ea42caaed6a5` year(4) NOT NULL DEFAULT '1983',\n `b9b70f39-1a02-4114-9d7d-fa6259c1b691` time DEFAULT '20:18:04',\n PRIMARY KEY (`323cdbcb-0c14-4362-90ab-ea42caaed6a5`) /*T![clustered_index] CLUSTERED */,\n KEY `a704d6bb-772b-44ea-8cb0-6f7491c1aaa6` (`323cdbcb-0c14-4362-90ab-ea42caaed6a5`,`9804d5f2-cbc7-43b7-b241-ea2656dc941a`)\n) ENGINE=InnoDB DEFAULT CHARSET=ascii COLLATE=ascii_bin;")
tk.MustExec("delete from `6bf9e76d-ab44-4031-8a07-418b10741580` where not( `6bf9e76d-ab44-4031-8a07-418b10741580`.`e0b5f703-6cfe-49b4-bc21-16a6455e43a7` in ( select `9804d5f2-cbc7-43b7-b241-ea2656dc941a` from `8919f3f4-25be-4a1a-904a-bb5e863d8fc8` where `6bf9e76d-ab44-4031-8a07-418b10741580`.`e0b5f703-6cfe-49b4-bc21-16a6455e43a7` in ( '1f' ) and `6bf9e76d-ab44-4031-8a07-418b10741580`.`e0b5f703-6cfe-49b4-bc21-16a6455e43a7` in ( '1g' ,'va' ,'uhnh' ) ) ) and not( IsNull( `6bf9e76d-ab44-4031-8a07-418b10741580`.`e0b5f703-6cfe-49b4-bc21-16a6455e43a7` ) );\n")
}
22 changes: 17 additions & 5 deletions expression/util.go
Original file line number Diff line number Diff line change
Expand Up @@ -415,7 +415,6 @@ func ColumnSubstituteImpl(expr Expression, schema *Schema, newExprs []Expression
if v.InOperand {
newExpr = SetExprColumnInOperand(newExpr)
}
newExpr.SetCoercibility(v.Coercibility())
return true, false, newExpr
case *ScalarFunction:
substituted := false
Expand All @@ -436,7 +435,11 @@ func ColumnSubstituteImpl(expr Expression, schema *Schema, newExprs []Expression
// cowExprRef is a copy-on-write util, args array allocation happens only
// when expr in args is changed
refExprArr := cowExprRef{v.GetArgs(), nil}
_, coll := DeriveCollationFromExprs(v.GetCtx(), v.GetArgs()...)
oldCollEt, err := CheckAndDeriveCollationFromExprs(v.GetCtx(), v.FuncName.L, v.RetType.EvalType(), v.GetArgs()...)
if err != nil {
logutil.BgLogger().Error("Unexpected error happened during ColumnSubstitution", zap.Stack("stack"))
return false, false, v
}
var tmpArgForCollCheck []Expression
if collate.NewCollationEnabled() {
tmpArgForCollCheck = make([]Expression, len(v.GetArgs()))
Expand All @@ -452,9 +455,18 @@ func ColumnSubstituteImpl(expr Expression, schema *Schema, newExprs []Expression
changed = false
copy(tmpArgForCollCheck, refExprArr.Result())
tmpArgForCollCheck[idx] = newFuncExpr
_, newColl := DeriveCollationFromExprs(v.GetCtx(), tmpArgForCollCheck...)
if coll == newColl {
changed = checkCollationStrictness(coll, newFuncExpr.GetType().GetCollate())
newCollEt, err := CheckAndDeriveCollationFromExprs(v.GetCtx(), v.FuncName.L, v.RetType.EvalType(), tmpArgForCollCheck...)
if err != nil {
logutil.BgLogger().Error("Unexpected error happened during ColumnSubstitution", zap.Stack("stack"))
return false, failed, v
}
if oldCollEt.Collation == newCollEt.Collation {
if newFuncExpr.GetType().GetCollate() == arg.GetType().GetCollate() && newFuncExpr.Coercibility() == arg.Coercibility() {
// It's safe to use the new expression, otherwise some cases in projection push-down will be wrong.
changed = true
} else {
changed = checkCollationStrictness(oldCollEt.Collation, newFuncExpr.GetType().GetCollate())
}
}
}
hasFail = hasFail || failed || oldChanged != changed
Expand Down
7 changes: 5 additions & 2 deletions planner/core/exhaust_physical_plans.go
Original file line number Diff line number Diff line change
Expand Up @@ -1325,8 +1325,11 @@ func (ijHelper *indexJoinBuildHelper) resetContextForIndex(innerKeys []*expressi
if ijHelper.curIdxOff2KeyOff[i] >= 0 {
// Don't use the join columns if their collations are unmatched and the new collation is enabled.
if collate.NewCollationEnabled() && types.IsString(idxCol.RetType.GetType()) && types.IsString(outerKeys[ijHelper.curIdxOff2KeyOff[i]].RetType.GetType()) {
_, coll := expression.DeriveCollationFromExprs(nil, idxCol, outerKeys[ijHelper.curIdxOff2KeyOff[i]])
if !collate.CompatibleCollate(idxCol.GetType().GetCollate(), coll) {
et, err := expression.CheckAndDeriveCollationFromExprs(ijHelper.innerPlan.ctx, "equal", types.ETInt, idxCol, outerKeys[ijHelper.curIdxOff2KeyOff[i]])
if err != nil {
logutil.BgLogger().Error("Unexpected error happened during constructing index join", zap.Stack("stack"))
}
if !collate.CompatibleCollate(idxCol.GetType().GetCollate(), et.Collation) {
ijHelper.curIdxOff2KeyOff[i] = -1
}
}
Expand Down

0 comments on commit d6adc09

Please sign in to comment.