merge master (pingcap#10)

* merge master

* fix bug

* fix bug

* fix ut

.github/workflows/assign_project.yml

@@ -23,7 +23,8 @@ jobs:
       uses: srggrs/assign-one-project-github-action@1.2.0
       if: |
         contains(github.event.issue.labels.*.name, 'component/planner') ||
-        contains(github.event.issue.labels.*.name, 'component/statistics')
+        contains(github.event.issue.labels.*.name, 'component/statistics') ||
+        contains(github.event.issue.labels.*.name, 'component/bindinfo')
       with:
         project: 'https://github.com/pingcap/tidb/projects/39'
         column_name: 'Issue Backlog: Need Triage'

.github/workflows/issue_assigned.yml

@@ -0,0 +1,20 @@
+name: Auto Assign Project Local
+
+on:
+  issues:
+    types: [assigned]
+env:
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+jobs:
+  assign_to_project_column:
+    runs-on: ubuntu-latest
+    name: Assign to Project Column
+    steps:
+    - name: Run issues assignment to Robust test
+      uses: srggrs/assign-one-project-github-action@1.2.0
+      if: |
+        contains(github.event.issue.labels.*.name, 'component/test')
+      with:
+        project: 'https://github.com/pingcap/tidb/projects/32'
+        column_name: 'Task Assigned'

Makefile

@@ -82,14 +82,15 @@ build:
 # Install the check tools.
 check-setup:tools/bin/revive tools/bin/goword tools/bin/gometalinter tools/bin/gosec
 
-check: fmt errcheck lint tidy testSuite check-static vet staticcheck
+check: fmt errcheck unconvert lint tidy testSuite check-static vet staticcheck
 
 # These need to be fixed before they can be ran regularly
 check-fail: goword check-slow
 
 fmt:
 	@echo "gofmt (simplify)"
 	@gofmt -s -l -w $(FILES) 2>&1 | $(FAIL_ON_STDOUT)
+	@cd cmd/importcheck && $(GO) run . ../..
 
 goword:tools/bin/goword
 	tools/bin/goword $(FILES) 2>&1 | $(FAIL_ON_STDOUT)
@@ -112,6 +113,10 @@ errcheck:tools/bin/errcheck
 	@echo "errcheck"
 	@GO111MODULE=on tools/bin/errcheck -exclude ./tools/check/errcheck_excludes.txt -ignoretests -blank $(PACKAGES)
 
+unconvert:tools/bin/unconvert
+	@echo "unconvert check"
+	@GO111MODULE=on tools/bin/unconvert ./...
+
 gogenerate:
 	@echo "go generate ./..."
 	./tools/check/check-gogenerate.sh
@@ -283,6 +288,10 @@ tools/bin/errcheck: tools/check/go.mod
 	cd tools/check; \
 	$(GO) build -o ../bin/errcheck github.com/kisielk/errcheck
 
+tools/bin/unconvert: tools/check/go.mod
+	cd tools/check; \
+	$(GO) build -o ../bin/unconvert github.com/mdempsky/unconvert
+
 tools/bin/failpoint-ctl: go.mod
 	$(GO) build -o $@ github.com/pingcap/failpoint/failpoint-ctl
 

bindinfo/bind_test.go

@@ -25,13 +25,15 @@ import (
 	"github.com/pingcap/parser"
 	"github.com/pingcap/parser/auth"
 	"github.com/pingcap/parser/model"
+	"github.com/pingcap/parser/terror"
 	"github.com/pingcap/tidb/bindinfo"
 	"github.com/pingcap/tidb/domain"
 	"github.com/pingcap/tidb/kv"
 	"github.com/pingcap/tidb/meta/autoid"
 	"github.com/pingcap/tidb/metrics"
 	"github.com/pingcap/tidb/session"
 	"github.com/pingcap/tidb/store/mockstore"
+	"github.com/pingcap/tidb/store/mockstore/cluster"
 	"github.com/pingcap/tidb/store/mockstore/mocktikv"
 	"github.com/pingcap/tidb/util/logutil"
 	"github.com/pingcap/tidb/util/stmtsummary"
@@ -51,10 +53,9 @@ func TestT(t *testing.T) {
 var _ = Suite(&testSuite{})
 
 type testSuite struct {
-	cluster   *mocktikv.Cluster
-	mvccStore mocktikv.MVCCStore
-	store     kv.Storage
-	domain    *domain.Domain
+	cluster cluster.Cluster
+	store   kv.Storage
+	domain  *domain.Domain
 	*parser.Parser
 }
 
@@ -66,12 +67,15 @@ func (s *testSuite) SetUpSuite(c *C) {
 	flag.Lookup("mockTikv")
 	useMockTikv := *mockTikv
 	if useMockTikv {
-		s.cluster = mocktikv.NewCluster()
-		mocktikv.BootstrapWithSingleStore(s.cluster)
-		s.mvccStore = mocktikv.MustNewMVCCStore()
+		cluster := mocktikv.NewCluster()
+		mocktikv.BootstrapWithSingleStore(cluster)
+		s.cluster = cluster
+
+		mvccStore := mocktikv.MustNewMVCCStore()
+		cluster.SetMvccStore(mvccStore)
 		store, err := mockstore.NewMockTikvStore(
-			mockstore.WithCluster(s.cluster),
-			mockstore.WithMVCCStore(s.mvccStore),
+			mockstore.WithCluster(cluster),
+			mockstore.WithMVCCStore(mvccStore),
 		)
 		c.Assert(err, IsNil)
 		s.store = store
@@ -947,6 +951,17 @@ func (s *testSuite) TestHintsSetID(c *C) {
 	c.Assert(len(bindData.Bindings), Equals, 1)
 	bind = bindData.Bindings[0]
 	c.Assert(bind.ID, Equals, "use_index(@`sel_1` `test`.`t` `idx_a`)")
+
+	s.cleanBindingEnv(tk)
+	err := tk.ExecToErr("create global binding for select * from t using select /*+ non_exist_hint() */ * from t")
+	c.Assert(terror.ErrorEqual(err, parser.ErrWarnOptimizerHintParseError), IsTrue)
+	tk.MustExec("create global binding for select * from t where a > 10 using select * from t where a > 10")
+	bindData = bindHandle.GetBindRecord(hash, sql, "test")
+	c.Check(bindData, NotNil)
+	c.Check(bindData.OriginalSQL, Equals, "select * from t where a > ?")
+	c.Assert(len(bindData.Bindings), Equals, 1)
+	bind = bindData.Bindings[0]
+	c.Assert(bind.ID, Equals, "")
 }
 
 func (s *testSuite) TestCapturePlanBaselineIgnoreTiFlash(c *C) {

bindinfo/cache.go

@@ -117,14 +117,20 @@ func (br *BindRecord) prepareHints(sctx sessionctx.Context) error {
 				return err
 			}
 		}
-		hintsSet, err := hint.ParseHintsSet(p, bind.BindSQL, bind.Charset, bind.Collation, br.Db)
+		hintsSet, warns, err := hint.ParseHintsSet(p, bind.BindSQL, bind.Charset, bind.Collation, br.Db)
 		if err != nil {
 			return err
 		}
 		hintsStr, err := hintsSet.Restore()
 		if err != nil {
 			return err
 		}
+		// For `create global binding for select * from t using select * from t`, we allow it though hintsStr is empty.
+		// For `create global binding for select * from t using select /*+ non_exist_hint() */ * from t`,
+		// the hint is totally invaild, we escalate warning to error.
+		if hintsStr == "" && len(warns) > 0 {
+			return warns[0]
+		}
 		br.Bindings[i].Hint = hintsSet
 		br.Bindings[i].ID = hintsStr
 	}

cmd/ddltest/ddl_test.go

@@ -745,7 +745,7 @@ func (s *TestDDLSuite) TestSimpleConflictUpdate(c *C) {
 				k := randomNum(rowCount)
 				s.mustExec(c, fmt.Sprintf("update test_conflict_update set c2 = %d where c1 = %d", defaultValue, k))
 				mu.Lock()
-				keysMap[int64(k)] = int64(defaultValue)
+				keysMap[int64(k)] = defaultValue
 				mu.Unlock()
 			}
 		}()

cmd/ddltest/index_test.go

@@ -78,8 +78,8 @@ func (s *TestDDLSuite) checkAddIndex(c *C, indexInfo *model.IndexInfo) {
 		}
 
 		c.Assert(err, IsNil)
-		c.Assert(handles, HasKey, h)
-		delete(handles, h)
+		c.Assert(handles, HasKey, h.IntValue())
+		delete(handles, h.IntValue())
 	}
 
 	c.Assert(handles, HasLen, 0)
@@ -116,7 +116,7 @@ func (s *TestDDLSuite) checkDropIndex(c *C, indexInfo *model.IndexInfo) {
 		}
 
 		c.Assert(err, IsNil)
-		handles[h] = struct{}{}
+		handles[h.IntValue()] = struct{}{}
 	}
 
 	// TODO: Uncomment this after apply pool is finished

cmd/explaintest/r/black_list.result

@@ -22,31 +22,31 @@ id	estRows	task	access object	operator info
 TableReader_7	3323.33	root		data:Selection_6
 └─Selection_6	3323.33	cop[tikv]		lt(test.t.a, 1)
   └─TableFullScan_5	10000.00	cop[tikv]	table:t	keep order:false, stats:pseudo
-insert into mysql.expr_pushdown_blacklist values('<');
+insert into mysql.expr_pushdown_blacklist values('<', 'tikv,tiflash,tidb', 'for test');
 admin reload expr_pushdown_blacklist;
 
 explain select * from t where a < 1;
 id	estRows	task	access object	operator info
 Selection_5	8000.00	root		lt(test.t.a, 1)
 └─TableReader_7	10000.00	root		data:TableFullScan_6
   └─TableFullScan_6	10000.00	cop[tikv]	table:t	keep order:false, stats:pseudo
-delete from mysql.expr_pushdown_blacklist where name='<';
+delete from mysql.expr_pushdown_blacklist where name='<' and store_type = 'tikv,tiflash,tidb' and reason = 'for test';
 admin reload expr_pushdown_blacklist;
 
 explain select * from t where a < 1;
 id	estRows	task	access object	operator info
 TableReader_7	3323.33	root		data:Selection_6
 └─Selection_6	3323.33	cop[tikv]		lt(test.t.a, 1)
   └─TableFullScan_5	10000.00	cop[tikv]	table:t	keep order:false, stats:pseudo
-insert into mysql.expr_pushdown_blacklist values('lt');
+insert into mysql.expr_pushdown_blacklist values('lt', 'tikv,tiflash,tidb', 'for test');
 admin reload expr_pushdown_blacklist;
 
 explain select * from t where a < 1;
 id	estRows	task	access object	operator info
 Selection_5	8000.00	root		lt(test.t.a, 1)
 └─TableReader_7	10000.00	root		data:TableFullScan_6
   └─TableFullScan_6	10000.00	cop[tikv]	table:t	keep order:false, stats:pseudo
-delete from mysql.expr_pushdown_blacklist where name='lt';
+delete from mysql.expr_pushdown_blacklist where name='lt' and store_type = 'tikv,tiflash,tidb' and reason = 'for test';
 admin reload expr_pushdown_blacklist;
 
 explain select * from t where a < 1;

cmd/explaintest/r/explain_easy.result

@@ -164,10 +164,10 @@ id	estRows	task	access object	operator info
 Union_17	26000.00	root		
 ├─HashAgg_21	16000.00	root		group by:Column#10, funcs:firstrow(Column#12)->Column#10
 │ └─Union_22	16000.00	root		
-│   ├─StreamAgg_27	8000.00	root		group by:test.t2.c1, funcs:firstrow(test.t2.c1)->Column#12, funcs:firstrow(test.t2.c1)->Column#10
+│   ├─StreamAgg_27	8000.00	root		group by:test.t2.c1, funcs:firstrow(test.t2.c1)->Column#12
 │   │ └─IndexReader_40	10000.00	root		index:IndexFullScan_39
 │   │   └─IndexFullScan_39	10000.00	cop[tikv]	table:t2, index:c1(c1)	keep order:true, stats:pseudo
-│   └─StreamAgg_45	8000.00	root		group by:test.t2.c1, funcs:firstrow(test.t2.c1)->Column#12, funcs:firstrow(test.t2.c1)->Column#10
+│   └─StreamAgg_45	8000.00	root		group by:test.t2.c1, funcs:firstrow(test.t2.c1)->Column#12
 │     └─IndexReader_58	10000.00	root		index:IndexFullScan_57
 │       └─IndexFullScan_57	10000.00	cop[tikv]	table:t2, index:c1(c1)	keep order:true, stats:pseudo
 └─IndexReader_63	10000.00	root		index:IndexFullScan_62
@@ -176,13 +176,13 @@ explain select c1 from t2 union all select c1 from t2 union select c1 from t2;
 id	estRows	task	access object	operator info
 HashAgg_18	24000.00	root		group by:Column#10, funcs:firstrow(Column#11)->Column#10
 └─Union_19	24000.00	root		
-  ├─StreamAgg_24	8000.00	root		group by:test.t2.c1, funcs:firstrow(test.t2.c1)->Column#11, funcs:firstrow(test.t2.c1)->Column#10
+  ├─StreamAgg_24	8000.00	root		group by:test.t2.c1, funcs:firstrow(test.t2.c1)->Column#11
   │ └─IndexReader_37	10000.00	root		index:IndexFullScan_36
   │   └─IndexFullScan_36	10000.00	cop[tikv]	table:t2, index:c1(c1)	keep order:true, stats:pseudo
-  ├─StreamAgg_42	8000.00	root		group by:test.t2.c1, funcs:firstrow(test.t2.c1)->Column#11, funcs:firstrow(test.t2.c1)->Column#10
+  ├─StreamAgg_42	8000.00	root		group by:test.t2.c1, funcs:firstrow(test.t2.c1)->Column#11
   │ └─IndexReader_55	10000.00	root		index:IndexFullScan_54
   │   └─IndexFullScan_54	10000.00	cop[tikv]	table:t2, index:c1(c1)	keep order:true, stats:pseudo
-  └─StreamAgg_60	8000.00	root		group by:test.t2.c1, funcs:firstrow(test.t2.c1)->Column#11, funcs:firstrow(test.t2.c1)->Column#10
+  └─StreamAgg_60	8000.00	root		group by:test.t2.c1, funcs:firstrow(test.t2.c1)->Column#11
     └─IndexReader_73	10000.00	root		index:IndexFullScan_72
       └─IndexFullScan_72	10000.00	cop[tikv]	table:t2, index:c1(c1)	keep order:true, stats:pseudo
 select * from information_schema.tidb_indexes where table_name='t4';
@@ -669,17 +669,17 @@ id	estRows	task	access object	operator info
 Sort_13	2.00	root		Column#3:asc
 └─HashAgg_17	2.00	root		group by:Column#3, funcs:firstrow(Column#6)->Column#3
   └─Union_18	2.00	root		
-    ├─HashAgg_19	1.00	root		group by:1, funcs:firstrow(0)->Column#6, funcs:firstrow(0)->Column#3
+    ├─HashAgg_19	1.00	root		group by:1, funcs:firstrow(0)->Column#6
     │ └─TableDual_22	1.00	root		rows:1
-    └─HashAgg_25	1.00	root		group by:1, funcs:firstrow(1)->Column#6, funcs:firstrow(1)->Column#3
+    └─HashAgg_25	1.00	root		group by:1, funcs:firstrow(1)->Column#6
       └─TableDual_28	1.00	root		rows:1
 explain SELECT 0 AS a FROM dual UNION (SELECT 1 AS a FROM dual ORDER BY a);
 id	estRows	task	access object	operator info
 HashAgg_15	2.00	root		group by:Column#3, funcs:firstrow(Column#6)->Column#3
 └─Union_16	2.00	root		
-  ├─HashAgg_17	1.00	root		group by:1, funcs:firstrow(0)->Column#6, funcs:firstrow(0)->Column#3
+  ├─HashAgg_17	1.00	root		group by:1, funcs:firstrow(0)->Column#6
   │ └─TableDual_20	1.00	root		rows:1
-  └─StreamAgg_27	1.00	root		group by:Column#1, funcs:firstrow(Column#1)->Column#6, funcs:firstrow(Column#1)->Column#3
+  └─StreamAgg_27	1.00	root		group by:Column#1, funcs:firstrow(Column#1)->Column#6
     └─Projection_32	1.00	root		1->Column#1
       └─TableDual_33	1.00	root		rows:1
 create table t (i int key, j int, unique key (i, j));

cmd/explaintest/r/explain_generate_column_substitute.result

@@ -1,11 +1,11 @@
 use test;
 drop table if exists t;
-create table t(a int, b real, c int as ((a+1)) virtual, e real as ((b+a)));
+create table t(a int, b real, c bigint as ((a+1)) virtual, e real as ((b+a)));
 insert into t values (1, 2.0, default, default), (2, 2.1, default, default), (5, 3.0, default, default),
 (5, -1.0, default, default), (0, 0.0, default, default), (-1, -2.0, default, default), (0, 0, default, default);
 alter table t add index idx_c(c);
 alter table t add index idx_e(e);
-set @@sql_mode=""
+set @@sql_mode="";
 desc select * from t where a+1=3;
 id	estRows	task	access object	operator info
 IndexLookUp_10	10.00	root		
@@ -324,3 +324,58 @@ Delete_4	N/A	root		N/A
 └─IndexLookUp_11	10.00	root		
   ├─IndexRangeScan_9(Build)	10.00	cop[tikv]	table:t, index:expr_idx_e(_V$_expr_idx_e_0)	range:[0,0], keep order:false, stats:pseudo
   └─TableRowIDScan_10(Probe)	10.00	cop[tikv]	table:t	keep order:false, stats:pseudo
+drop table if exists t0;
+create table t0(c0 float(24), c1 double as (c0) unique);
+explain select c0 from t0;
+id	estRows	task	access object	operator info
+TableReader_5	10000.00	root		data:TableFullScan_4
+└─TableFullScan_4	10000.00	cop[tikv]	table:t0	keep order:false, stats:pseudo
+ -- TableRead
+drop table if exists t0;
+create table t0(c0 float(25), c1 double as (c0) unique);
+explain select c0 from t0;
+id	estRows	task	access object	operator info
+IndexReader_7	10000.00	root		index:IndexFullScan_6
+└─IndexFullScan_6	10000.00	cop[tikv]	table:t0, index:c1(c1)	keep order:false, stats:pseudo
+ -- IndexRead
+drop table if exists t0;
+create table t0(c0 double(24), c1 double as (c0) unique);
+explain select c0 from t0;
+id	estRows	task	access object	operator info
+IndexReader_7	10000.00	root		index:IndexFullScan_6
+└─IndexFullScan_6	10000.00	cop[tikv]	table:t0, index:c1(c1)	keep order:false, stats:pseudo
+ -- IndexRead
+drop table if exists t0;
+create table t0(c0 double(25), c1 double as (c0) unique);
+explain select c0 from t0;
+id	estRows	task	access object	operator info
+IndexReader_7	10000.00	root		index:IndexFullScan_6
+└─IndexFullScan_6	10000.00	cop[tikv]	table:t0, index:c1(c1)	keep order:false, stats:pseudo
+ -- IndexRead
+drop table if exists t0;
+create table t0(c0 float(24), c1 float as (c0) unique);
+explain select c0 from t0;
+id	estRows	task	access object	operator info
+IndexReader_7	10000.00	root		index:IndexFullScan_6
+└─IndexFullScan_6	10000.00	cop[tikv]	table:t0, index:c1(c1)	keep order:false, stats:pseudo
+ -- IndexRead
+drop table if exists t0;
+create table t0(c0 float(25), c1 float as (c0) unique);
+explain select c0 from t0;
+id	estRows	task	access object	operator info
+TableReader_5	10000.00	root		data:TableFullScan_4
+└─TableFullScan_4	10000.00	cop[tikv]	table:t0	keep order:false, stats:pseudo
+ -- TableRead
+drop table if exists t0;
+create table t0(c0 double(24), c1 float as (c0) unique);
+explain select c0 from t0;
+id	estRows	task	access object	operator info
+TableReader_5	10000.00	root		data:TableFullScan_4
+└─TableFullScan_4	10000.00	cop[tikv]	table:t0	keep order:false, stats:pseudo
+ -- TableRead
+drop table if exists t0;
+create table t0(c0 double(25), c1 float as (c0) unique);
+explain select c0 from t0;
+id	estRows	task	access object	operator info
+TableReader_5	10000.00	root		data:TableFullScan_4
+└─TableFullScan_4	10000.00	cop[tikv]	table:t0	keep order:false, stats:pseudo