From 3d98ddbebfc0db825be3fa4d8bb841f47d2219d8 Mon Sep 17 00:00:00 2001 From: Evan Zhou Date: Tue, 2 Jun 2020 12:02:07 +0800 Subject: [PATCH 1/3] update unistore to fix unstable test (#17576) For the failed test in https://github.com/pingcap/tidb/pull/17397 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 8f44ba3106700..c3245ed61d2dc 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/klauspost/cpuid v1.2.1 github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef - github.com/ngaut/unistore v0.0.0-20200601072616-c5558b00c578 + github.com/ngaut/unistore v0.0.0-20200602034536-a59a860991d8 github.com/opentracing/basictracer-go v1.0.0 github.com/opentracing/opentracing-go v1.1.0 github.com/pingcap/br v0.0.0-20200521085655-53201addd4ad diff --git a/go.sum b/go.sum index 257768cef892e..4b16031724d09 100644 --- a/go.sum +++ b/go.sum @@ -377,8 +377,8 @@ github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 h1:7KAv7KMGTTqSmYZtNdc github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7/go.mod h1:iWMfgwqYW+e8n5lC/jjNEhwcjbRDpl5NT7n2h+4UNcI= github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef h1:K0Fn+DoFqNqktdZtdV3bPQ/0cuYh2H4rkg0tytX/07k= github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef/go.mod h1:7WjlapSfwQyo6LNmIvEWzsW1hbBQfpUO4JWnuQRmva8= -github.com/ngaut/unistore v0.0.0-20200601072616-c5558b00c578 h1:om0QlwAhWSVjpaUks9+90Ampe1vJgOMClFi9fEnqTOg= -github.com/ngaut/unistore v0.0.0-20200601072616-c5558b00c578/go.mod h1:9mpqZeS1CkNlgZwJ0LZXb+Qd7xVO5o55ngys7T1/oH8= +github.com/ngaut/unistore v0.0.0-20200602034536-a59a860991d8 h1:9RRf+Dqt0x+0ORKQ/CTwd2r0T782B/ovD4/6cLwXwkg= +github.com/ngaut/unistore v0.0.0-20200602034536-a59a860991d8/go.mod h1:9mpqZeS1CkNlgZwJ0LZXb+Qd7xVO5o55ngys7T1/oH8= github.com/nicksnyder/go-i18n v1.10.0/go.mod h1:HrK7VCrbOvQoUAQ7Vpy7i87N7JZZZ7R2xBGjv0j365Q= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= From 23759514ec8d4bae275e0591f1f45d9fd2cdba1f Mon Sep 17 00:00:00 2001 From: wjHuang Date: Tue, 2 Jun 2020 15:18:47 +0800 Subject: [PATCH 2/3] expression: support collation for REGEXP function (#17255) --- expression/builtin_like.go | 11 ++++++--- expression/integration_test.go | 41 ++++++++++++++++++++++++++++------ 2 files changed, 42 insertions(+), 10 deletions(-) diff --git a/expression/builtin_like.go b/expression/builtin_like.go index 3a74924a3dd67..5700e69a8d807 100644 --- a/expression/builtin_like.go +++ b/expression/builtin_like.go @@ -17,6 +17,7 @@ import ( "regexp" "sync" + "github.com/pingcap/parser/charset" "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/chunk" @@ -121,7 +122,7 @@ func (c *regexpFunctionClass) getFunction(ctx sessionctx.Context, args []Express } bf.tp.Flen = 1 var sig builtinFunc - if types.IsBinaryStr(args[0].GetType()) || types.IsBinaryStr(args[1].GetType()) { + if bf.collation == charset.CollationBin { sig = newBuiltinRegexpSig(bf) sig.setPbCode(tipb.ScalarFuncSig_RegexpSig) } else { @@ -189,8 +190,12 @@ type builtinRegexpUTF8Sig struct { func newBuiltinRegexpUTF8Sig(bf baseBuiltinFunc) *builtinRegexpUTF8Sig { shared := builtinRegexpSharedSig{baseBuiltinFunc: bf} - shared.compile = func(pat string) (*regexp.Regexp, error) { - return regexp.Compile("(?i)" + pat) + if collate.IsCICollation(bf.collation) { + shared.compile = func(pat string) (*regexp.Regexp, error) { + return regexp.Compile("(?i)" + pat) + } + } else { + shared.compile = regexp.Compile } return &builtinRegexpUTF8Sig{builtinRegexpSharedSig: shared} } diff --git a/expression/integration_test.go b/expression/integration_test.go index a09150cdb60ff..f40ae9c8d59e0 100755 --- a/expression/integration_test.go +++ b/expression/integration_test.go @@ -938,7 +938,7 @@ func (s *testIntegrationSuite2) TestStringBuiltin(c *C) { tk.MustQuery(`select quote(null) REGEXP binary 'null'`).Check(testkit.Rows(`0`)) tk.MustQuery(`select quote(null) REGEXP binary 'NULL'`).Check(testkit.Rows(`1`)) tk.MustQuery(`select quote(null) REGEXP 'NULL'`).Check(testkit.Rows(`1`)) - tk.MustQuery(`select quote(null) REGEXP 'null'`).Check(testkit.Rows(`1`)) + tk.MustQuery(`select quote(null) REGEXP 'null'`).Check(testkit.Rows(`0`)) // for convert result = tk.MustQuery(`select convert("123" using "binary"), convert("中文" using "binary"), convert("中文" using "utf8"), convert("中文" using "utf8mb4"), convert(cast("中文" as binary) using "utf8");`) @@ -2783,9 +2783,9 @@ func (s *testIntegrationSuite2) TestBuiltin(c *C) { tk.MustExec(`drop table if exists t;`) tk.MustExec(`create table t (a char(10), b varchar(10), c binary(10), d varbinary(10));`) tk.MustExec(`insert into t values ('text','text','text','text');`) - result = tk.MustQuery(`select a regexp 'Xt' from t;`) + result = tk.MustQuery(`select a regexp 'xt' from t;`) result.Check(testkit.Rows("1")) - result = tk.MustQuery(`select b regexp 'Xt' from t;`) + result = tk.MustQuery(`select b regexp 'xt' from t;`) result.Check(testkit.Rows("1")) result = tk.MustQuery(`select b regexp binary 'Xt' from t;`) result.Check(testkit.Rows("0")) @@ -2793,18 +2793,18 @@ func (s *testIntegrationSuite2) TestBuiltin(c *C) { result.Check(testkit.Rows("0")) result = tk.MustQuery(`select d regexp 'Xt' from t;`) result.Check(testkit.Rows("0")) - result = tk.MustQuery(`select a rlike 'Xt' from t;`) + result = tk.MustQuery(`select a rlike 'xt' from t;`) result.Check(testkit.Rows("1")) result = tk.MustQuery(`select a rlike binary 'Xt' from t;`) result.Check(testkit.Rows("0")) - result = tk.MustQuery(`select b rlike 'Xt' from t;`) + result = tk.MustQuery(`select b rlike 'xt' from t;`) result.Check(testkit.Rows("1")) result = tk.MustQuery(`select c rlike 'Xt' from t;`) result.Check(testkit.Rows("0")) result = tk.MustQuery(`select d rlike 'Xt' from t;`) result.Check(testkit.Rows("0")) result = tk.MustQuery(`select 'a' regexp 'A', 'a' regexp binary 'A'`) - result.Check(testkit.Rows("1 0")) + result.Check(testkit.Rows("0 0")) // testCase is for like and regexp type testCase struct { @@ -6162,7 +6162,7 @@ func (s *testIntegrationSerialSuite) TestCollateStringFunction(c *C) { tk.MustQuery("select INSTR(p4,n4) from t1;").Check(testkit.Rows("1")) tk.MustExec("truncate table t1;") - tk.MustExec("insert into t1 (a,p1,p2,p3,p4,n1,n2,n3,n4) values (2,'0aA1!测试テストמבחן ','0aA1!测试テストמבחן ','0aA1!测试テストמבחן ','0aA1!测试テストמבחן ','0Aa1!测试テストמבחן','0Aa1!测试テストמבחן','0Aa1!测试テストמבחן','0Aa1!测试テストמבחן');") + tk.MustExec("insert into t1 (a,p1,p2,p3,p4,n1,n2,n3,n4) values (1,'0aA1!测试テストמבחן ','0aA1!测试テストמבחן ','0aA1!测试テストמבחן ','0aA1!测试テストמבחן ','0Aa1!测试テストמבחן','0Aa1!测试テストמבחן','0Aa1!测试テストמבחן','0Aa1!测试テストמבחן');") tk.MustExec("insert into t1 (a,p1,p2,p3,p4,n1,n2,n3,n4) values (2,'0aA1!测试テストמבחן','0aA1!测试テストמבחן','0aA1!测试テストמבחן','0aA1!测试テストמבחן','0Aa1!测试テストמבחן','0Aa1!测试テストמבחן','0Aa1!测试テストמבחן','0Aa1!测试テストמבחן');") tk.MustExec("insert into t1 (a,p1,p2,p3,p4,n1,n2,n3,n4) values (3,'0aA1!测试テストמבחן','0aA1!测试テストמבחן','0aA1!测试テストמבחן','0aA1!测试テストמבחן','0Aa1!测试テストמבחן ','0Aa1!测试テストמבחן ','0Aa1!测试テストמבחן ','0Aa1!测试テストמבחן ');") @@ -6183,6 +6183,33 @@ func (s *testIntegrationSerialSuite) TestCollateStringFunction(c *C) { tk.MustQuery("select LOCATE(p4,n3) from t1;").Check(testkit.Rows("0", "0", "0")) tk.MustQuery("select LOCATE(p4,n4) from t1;").Check(testkit.Rows("0", "1", "1")) + tk.MustExec("truncate table t1;") + tk.MustExec("insert into t1 (a) values (1);") + tk.MustExec("insert into t1 (a,p1,p2,p3,p4,n1,n2,n3,n4) values (2,'0aA1!测试テストמבחן ','0aA1!测试テストמבחן ','0aA1!测试テストמבחן ','0aA1!测试テストמבחן ','0Aa1!测试テストמבחן','0Aa1!测试テストמבחן','0Aa1!测试テストמבחן','0Aa1!测试テストמבחן');") + tk.MustExec("insert into t1 (a,p1,p2,p3,p4,n1,n2,n3,n4) values (3,'0aA1!测试テストמבחן','0aA1!测试テストמבחן','0aA1!测试テストמבחן','0aA1!测试テストמבחן','0Aa1!测试テストמבחן','0Aa1!测试テストמבחן','0Aa1!测试テストמבחן','0Aa1!测试テストמבחן');") + tk.MustExec("insert into t1 (a,p1,p2,p3,p4,n1,n2,n3,n4) values (4,'0aA1!测试テストמבחן','0aA1!测试テストמבחן','0aA1!测试テストמבחן','0aA1!测试テストמבחן','0Aa1!测试テストמבחן ','0Aa1!测试テストמבחן ','0Aa1!测试テストמבחן ','0Aa1!测试テストמבחן ');") + tk.MustExec("insert into t1 (a,p1,p2,p3,p4,n1,n2,n3,n4) values (5,'0aA1!测试テストמבחן0aA1!测试','0aA1!测试テストמבחן0aA1!测试','0aA1!测试テストמבחן0aA1!测试','0aA1!测试テストמבחן0aA1!测试','0Aa1!测试','0Aa1!测试','0Aa1!测试','0Aa1!测试');") + tk.MustExec("insert into t1 (a,p1,p2,p3,p4,n1,n2,n3,n4) values (6,'0aA1!测试テストמבחן0aA1!测试','0aA1!测试テストמבחן0aA1!测试','0aA1!测试テストמבחן0aA1!测试','0aA1!测试テストמבחן0aA1!测试','0aA1!测试','0aA1!测试','0aA1!测试','0aA1!测试');") + tk.MustExec("insert into t1 (a,p1,p2,p3,p4,n1,n2,n3,n4) values (7,'0aA1!测试テストמבחן ','0aA1!测试テストמבחן ','0aA1!测试テストמבחן ','0aA1!测试テストמבחן ','0aA1!测试テストמבחן','0aA1!测试テストמבחן','0aA1!测试テストמבחן','0aA1!测试テストמבחן');") + tk.MustExec("insert into t1 (a,p1,p2,p3,p4,n1,n2,n3,n4) values (8,'0aA1!测试テストמבחן','0aA1!测试テストמבחן','0aA1!测试テストמבחן','0aA1!测试テストמבחן','0aA1!测试テストמבחן ','0aA1!测试テストמבחן ','0aA1!测试テストמבחן ','0aA1!测试テストמבחן ');") + + tk.MustQuery("select p1 REGEXP n1 from t1;").Check(testkit.Rows("", "0", "0", "0", "0", "1", "1", "0")) + tk.MustQuery("select p1 REGEXP n2 from t1;").Check(testkit.Rows("", "0", "0", "0", "0", "1", "1", "0")) + tk.MustQuery("select p1 REGEXP n3 from t1;").Check(testkit.Rows("", "0", "0", "0", "0", "1", "1", "0")) + tk.MustQuery("select p1 REGEXP n4 from t1;").Check(testkit.Rows("", "1", "1", "0", "1", "1", "1", "0")) + tk.MustQuery("select p2 REGEXP n1 from t1;").Check(testkit.Rows("", "0", "0", "0", "0", "1", "1", "0")) + tk.MustQuery("select p2 REGEXP n2 from t1;").Check(testkit.Rows("", "1", "1", "0", "1", "1", "1", "0")) + tk.MustQuery("select p2 REGEXP n3 from t1;").Check(testkit.Rows("", "0", "0", "0", "0", "1", "1", "0")) + tk.MustQuery("select p2 REGEXP n4 from t1;").Check(testkit.Rows("", "1", "1", "0", "1", "1", "1", "0")) + tk.MustQuery("select p3 REGEXP n1 from t1;").Check(testkit.Rows("", "0", "0", "0", "0", "1", "1", "0")) + tk.MustQuery("select p3 REGEXP n2 from t1;").Check(testkit.Rows("", "0", "0", "0", "0", "1", "1", "0")) + tk.MustQuery("select p3 REGEXP n3 from t1;").Check(testkit.Rows("", "0", "0", "0", "0", "1", "1", "0")) + tk.MustQuery("select p3 REGEXP n4 from t1;").Check(testkit.Rows("", "0", "0", "0", "0", "1", "1", "0")) + tk.MustQuery("select p4 REGEXP n1 from t1;").Check(testkit.Rows("", "1", "1", "0", "1", "1", "1", "0")) + tk.MustQuery("select p4 REGEXP n2 from t1;").Check(testkit.Rows("", "1", "1", "0", "1", "1", "1", "0")) + tk.MustQuery("select p4 REGEXP n3 from t1;").Check(testkit.Rows("", "0", "0", "0", "0", "1", "1", "0")) + tk.MustQuery("select p4 REGEXP n4 from t1;").Check(testkit.Rows("", "1", "1", "0", "1", "1", "1", "0")) + tk.MustExec("drop table t1;") } From 219f99a642efd72f9c53520a306b4f357d2e7f34 Mon Sep 17 00:00:00 2001 From: Xintao Date: Tue, 2 Jun 2020 15:57:03 +0800 Subject: [PATCH 3/3] Associate each region with an independent backoffer (#17568) --- store/tikv/coprocessor.go | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/store/tikv/coprocessor.go b/store/tikv/coprocessor.go index 0f49d960c8508..370852a61de43 100644 --- a/store/tikv/coprocessor.go +++ b/store/tikv/coprocessor.go @@ -490,8 +490,7 @@ func (worker *copIteratorWorker) run(ctx context.Context) { respCh = task.respChan } - bo := NewBackoffer(ctx, copNextMaxBackoff).WithVars(worker.vars) - worker.handleTask(bo, task, respCh) + worker.handleTask(ctx, task, respCh) close(task.respChan) select { case <-worker.finishCh: @@ -654,8 +653,20 @@ func (it *copIterator) Next(ctx context.Context) (kv.ResultSubset, error) { return resp, nil } +// Associate each region with an independent backoffer. In this way, when multiple regions are +// unavailable, TiDB can execute very quickly without blocking +func chooseBackoffer(ctx context.Context, backoffermap map[uint64]*Backoffer, task *copTask, worker *copIteratorWorker) *Backoffer { + bo, ok := backoffermap[task.region.id] + if ok { + return bo + } + newbo := NewBackoffer(ctx, copNextMaxBackoff).WithVars(worker.vars) + backoffermap[task.region.id] = newbo + return newbo +} + // handleTask handles single copTask, sends the result to channel, retry automatically on error. -func (worker *copIteratorWorker) handleTask(bo *Backoffer, task *copTask, respCh chan<- *copResponse) { +func (worker *copIteratorWorker) handleTask(ctx context.Context, task *copTask, respCh chan<- *copResponse) { defer func() { r := recover() if r != nil { @@ -668,8 +679,11 @@ func (worker *copIteratorWorker) handleTask(bo *Backoffer, task *copTask, respCh } }() remainTasks := []*copTask{task} + backoffermap := make(map[uint64]*Backoffer) for len(remainTasks) > 0 { - tasks, err := worker.handleTaskOnce(bo, remainTasks[0], respCh) + curTask := remainTasks[0] + bo := chooseBackoffer(ctx, backoffermap, curTask, worker) + tasks, err := worker.handleTaskOnce(bo, curTask, respCh) if err != nil { resp := &copResponse{err: errors.Trace(err)} worker.sendToRespCh(resp, respCh, true)