Merge branch 'master' into temporary-no-network-point

Dockerfile

@@ -12,7 +12,7 @@
 # limitations under the License.
 
 # Builder image
-FROM golang:1.13-alpine as builder
+FROM golang:1.16-alpine as builder
 
 RUN apk add --no-cache \
     wget \

Makefile

@@ -239,11 +239,13 @@ tools/bin/unconvert: tools/check/go.mod
 	cd tools/check; \
 	$(GO) build -o ../bin/unconvert github.com/mdempsky/unconvert
 
-tools/bin/failpoint-ctl: go.mod
-	$(GO) build -o $@ github.com/pingcap/failpoint/failpoint-ctl
+tools/bin/failpoint-ctl: tools/check/go.mod
+	cd tools/check; \
+	$(GO) build -o ../bin/failpoint-ctl github.com/pingcap/failpoint/failpoint-ctl
 
-tools/bin/errdoc-gen: go.mod
-	$(GO) build -o $@ github.com/pingcap/errors/errdoc-gen
+tools/bin/errdoc-gen: tools/check/go.mod
+	cd tools/check; \
+	$(GO) build -o ../bin/errdoc-gen github.com/pingcap/errors/errdoc-gen
 
 tools/bin/golangci-lint:
 	curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s -- -b ./tools/bin v1.29.0

README.md

@@ -55,7 +55,6 @@ In addition, you may enjoy following:
 - [@PingCAP](https://twitter.com/PingCAP) on Twitter
 - Question tagged [#tidb on StackOverflow](https://stackoverflow.com/questions/tagged/tidb)
 - The PingCAP Team [English Blog](https://en.pingcap.com/blog) and [Chinese Blog](https://pingcap.com/blog-cn/)
-- [TiDB Monthly](https://pingcap.com/weekly/)
 
 For support, please contact [PingCAP](http://bit.ly/contact_us_via_github).
 

bindinfo/bind_test.go

@@ -156,7 +156,8 @@ func normalizeWithDefaultDB(c *C, sql, db string) (string, string) {
 	testParser := parser.New()
 	stmt, err := testParser.ParseOneStmt(sql, "", "")
 	c.Assert(err, IsNil)
-	return parser.NormalizeDigest(utilparser.RestoreWithDefaultDB(stmt, "test", ""))
+	normalized, digest := parser.NormalizeDigest(utilparser.RestoreWithDefaultDB(stmt, "test", ""))
+	return normalized, digest.String()
 }
 
 func (s *testSuite) TestBindParse(c *C) {
@@ -182,7 +183,7 @@ func (s *testSuite) TestBindParse(c *C) {
 	c.Check(bindHandle.Size(), Equals, 1)
 
 	sql, hash := parser.NormalizeDigest("select * from test . t")
-	bindData := bindHandle.GetBindRecord(hash, sql, "test")
+	bindData := bindHandle.GetBindRecord(hash.String(), sql, "test")
 	c.Check(bindData, NotNil)
 	c.Check(bindData.OriginalSQL, Equals, "select * from `test` . `t`")
 	bind := bindData.Bindings[0]
@@ -656,7 +657,7 @@ func (s *testSuite) TestBindingSymbolList(c *C) {
 	// Normalize
 	sql, hash := parser.NormalizeDigest("select a, b from test . t where a = 1 limit 0, 1")
 
-	bindData := s.domain.BindHandle().GetBindRecord(hash, sql, "test")
+	bindData := s.domain.BindHandle().GetBindRecord(hash.String(), sql, "test")
 	c.Assert(bindData, NotNil)
 	c.Check(bindData.OriginalSQL, Equals, "select `a` , `b` from `test` . `t` where `a` = ? limit ...")
 	bind := bindData.Bindings[0]
@@ -776,7 +777,7 @@ func (s *testSuite) TestErrorBind(c *C) {
 	c.Assert(err, IsNil, Commentf("err %v", err))
 
 	sql, hash := parser.NormalizeDigest("select * from test . t where i > ?")
-	bindData := s.domain.BindHandle().GetBindRecord(hash, sql, "test")
+	bindData := s.domain.BindHandle().GetBindRecord(hash.String(), sql, "test")
 	c.Check(bindData, NotNil)
 	c.Check(bindData.OriginalSQL, Equals, "select * from `test` . `t` where `i` > ?")
 	bind := bindData.Bindings[0]

bindinfo/handle.go

@@ -210,7 +210,7 @@ func (h *BindHandle) CreateBindRecord(sctx sessionctx.Context, record *BindRecor
 		}
 
 		sqlDigest := parser.DigestNormalized(record.OriginalSQL)
-		h.setBindRecord(sqlDigest, record)
+		h.setBindRecord(sqlDigest.String(), record)
 	}()
 
 	// Lock mysql.bind_info to synchronize with CreateBindRecord / AddBindRecord / DropBindRecord on other tidb instances.
@@ -256,7 +256,7 @@ func (h *BindHandle) AddBindRecord(sctx sessionctx.Context, record *BindRecord)
 	}
 
 	record.Db = strings.ToLower(record.Db)
-	oldRecord := h.GetBindRecord(parser.DigestNormalized(record.OriginalSQL), record.OriginalSQL, record.Db)
+	oldRecord := h.GetBindRecord(parser.DigestNormalized(record.OriginalSQL).String(), record.OriginalSQL, record.Db)
 	var duplicateBinding *Binding
 	if oldRecord != nil {
 		binding := oldRecord.FindBinding(record.Bindings[0].ID)
@@ -294,7 +294,7 @@ func (h *BindHandle) AddBindRecord(sctx sessionctx.Context, record *BindRecord)
 			return
 		}
 
-		h.appendBindRecord(parser.DigestNormalized(record.OriginalSQL), record)
+		h.appendBindRecord(parser.DigestNormalized(record.OriginalSQL).String(), record)
 	}()
 
 	// Lock mysql.bind_info to synchronize with CreateBindRecord / AddBindRecord / DropBindRecord on other tidb instances.
@@ -367,7 +367,7 @@ func (h *BindHandle) DropBindRecord(originalSQL, db string, binding *Binding) (e
 		if binding != nil {
 			record.Bindings = append(record.Bindings, *binding)
 		}
-		h.removeBindRecord(parser.DigestNormalized(originalSQL), record)
+		h.removeBindRecord(parser.DigestNormalized(originalSQL).String(), record)
 	}()
 
 	// Lock mysql.bind_info to synchronize with CreateBindRecord / AddBindRecord / DropBindRecord on other tidb instances.
@@ -515,7 +515,7 @@ func (h *BindHandle) newBindRecord(row chunk.Row) (string, *BindRecord, error) {
 	defer h.sctx.Unlock()
 	h.sctx.GetSessionVars().CurrentDB = bindRecord.Db
 	err := bindRecord.prepareHints(h.sctx.Context)
-	return hash, bindRecord, err
+	return hash.String(), bindRecord, err
 }
 
 // setBindRecord sets the BindRecord to the cache, if there already exists a BindRecord,
@@ -624,7 +624,7 @@ func (h *BindHandle) CaptureBaselines() {
 		}
 		dbName := utilparser.GetDefaultDB(stmt, bindableStmt.Schema)
 		normalizedSQL, digest := parser.NormalizeDigest(utilparser.RestoreWithDefaultDB(stmt, dbName, bindableStmt.Query))
-		if r := h.GetBindRecord(digest, normalizedSQL, dbName); r != nil && r.HasUsingBinding() {
+		if r := h.GetBindRecord(digest.String(), normalizedSQL, dbName); r != nil && r.HasUsingBinding() {
 			continue
 		}
 		bindSQL := GenerateBindSQL(context.TODO(), stmt, bindableStmt.PlanHint, true, dbName)

bindinfo/session_handle.go

@@ -60,7 +60,7 @@ func (h *SessionHandle) CreateBindRecord(sctx sessionctx.Context, record *BindRe
 	}
 
 	// update the BindMeta to the cache.
-	h.appendBindRecord(parser.DigestNormalized(record.OriginalSQL), record)
+	h.appendBindRecord(parser.DigestNormalized(record.OriginalSQL).String(), record)
 	return nil
 }
 
@@ -78,14 +78,14 @@ func (h *SessionHandle) DropBindRecord(originalSQL, db string, binding *Binding)
 	} else {
 		newRecord = record
 	}
-	h.ch.setBindRecord(parser.DigestNormalized(record.OriginalSQL), newRecord)
+	h.ch.setBindRecord(parser.DigestNormalized(record.OriginalSQL).String(), newRecord)
 	updateMetrics(metrics.ScopeSession, oldRecord, newRecord, false)
 	return nil
 }
 
 // GetBindRecord return the BindMeta of the (normdOrigSQL,db) if BindMeta exist.
 func (h *SessionHandle) GetBindRecord(normdOrigSQL, db string) *BindRecord {
-	hash := parser.DigestNormalized(normdOrigSQL)
+	hash := parser.DigestNormalized(normdOrigSQL).String()
 	bindRecords := h.ch[hash]
 	for _, bindRecord := range bindRecords {
 		if bindRecord.OriginalSQL == normdOrigSQL {

cmd/benchfilesort/main.go

@@ -17,7 +17,6 @@ import (
 	"encoding/binary"
 	"flag"
 	"fmt"
-	"io/ioutil"
 	"math/rand"
 	"os"
 	"path/filepath"
@@ -332,7 +331,7 @@ func driveRunCmd() {
 	for i := 0; i < keySize; i++ {
 		byDesc[i] = false
 	}
-	dir, err = ioutil.TempDir(tmpDir, "benchfilesort_test")
+	dir, err = os.MkdirTemp(tmpDir, "benchfilesort_test")
 	terror.MustNil(err)
 	fs, err = fsBuilder.SetSC(sc).SetSchema(keySize, valSize).SetBuf(bufSize).SetWorkers(nWorkers).SetDesc(byDesc).SetDir(dir).Build()
 	terror.MustNil(err)

cmd/benchkv/main.go

@@ -17,7 +17,7 @@ import (
 	"context"
 	"flag"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"net/http"
 	_ "net/http/pprof"
 	"sync"
@@ -129,7 +129,7 @@ func main() {
 	terror.MustNil(err)
 
 	defer terror.Call(resp.Body.Close)
-	text, err1 := ioutil.ReadAll(resp.Body)
+	text, err1 := io.ReadAll(resp.Body)
 	terror.Log(errors.Trace(err1))
 
 	fmt.Println(string(text))

cmd/explaintest/main.go

@@ -19,7 +19,6 @@ import (
 	"flag"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"os"
 	"os/exec"
@@ -163,7 +162,7 @@ LOOP:
 }
 
 func (t *tester) loadQueries() ([]query, error) {
-	data, err := ioutil.ReadFile(t.testFileName())
+	data, err := os.ReadFile(t.testFileName())
 	if err != nil {
 		return nil, err
 	}
@@ -427,12 +426,12 @@ func (t *tester) create(tableName string, qText string) error {
 		return err
 	}
 
-	js, err := ioutil.ReadAll(resp.Body)
+	js, err := io.ReadAll(resp.Body)
 	if err != nil {
 		return err
 	}
 
-	return ioutil.WriteFile(t.statsFileName(tableName), js, 0644)
+	return os.WriteFile(t.statsFileName(tableName), js, 0644)
 }
 
 func (t *tester) commit() error {
@@ -531,7 +530,7 @@ func (t *tester) flushResult() error {
 	if !record {
 		return nil
 	}
-	return ioutil.WriteFile(t.resultFileName(), t.buf.Bytes(), 0644)
+	return os.WriteFile(t.resultFileName(), t.buf.Bytes(), 0644)
 }
 
 func (t *tester) statsFileName(tableName string) string {
@@ -550,7 +549,7 @@ func (t *tester) resultFileName() string {
 
 func loadAllTests() ([]string, error) {
 	// tests must be in t folder
-	files, err := ioutil.ReadDir("./t")
+	files, err := os.ReadDir("./t")
 	if err != nil {
 		return nil, err
 	}

cmd/explaintest/r/explain_generate_column_substitute.result

@@ -389,3 +389,28 @@ explain format = 'brief' select c0 from t0;
 id	estRows	task	access object	operator info
 TableReader	10000.00	root		data:TableFullScan
 └─TableFullScan	10000.00	cop[tikv]	table:t0	keep order:false, stats:pseudo
+ -- TableRead
+drop table if exists tbl1;
+create table tbl1 (id int unsigned not null auto_increment primary key, s int, index((md5(s))));
+insert into tbl1 (id) select null;
+ insert into tbl1 (id) select null from tbl1;
+ insert into tbl1 (id) select null from tbl1;
+ insert into tbl1 (id) select null from tbl1;
+ insert into tbl1 (id) select null from tbl1;
+ insert into tbl1 (id) select null from tbl1;
+ insert into tbl1 (id) select null from tbl1;
+ insert into tbl1 (id) select null from tbl1;
+ insert into tbl1 (id) select null from tbl1;
+ insert into tbl1 (id) select null from tbl1;
+ insert into tbl1 (id) select null from tbl1;
+ insert into tbl1 (id) select null from tbl1;
+update tbl1 set s=id%32;
+explain format = 'brief' select count(*) from tbl1 where md5(s) like '02e74f10e0327ad868d138f2b4fdd6f%';
+id	estRows	task	access object	operator info
+StreamAgg	1.00	root		funcs:count(Column#6)->Column#4
+└─IndexReader	1.00	root		index:StreamAgg
+  └─StreamAgg	1.00	cop[tikv]		funcs:count(1)->Column#6
+    └─IndexRangeScan	250.00	cop[tikv]	table:tbl1, index:expression_index(md5(`s`))	range:["02e74f10e0327ad868d138f2b4fdd6f","02e74f10e0327ad868d138f2b4fdd6g"), keep order:false, stats:pseudo
+select count(*) from tbl1 use index() where md5(s) like '02e74f10e0327ad868d138f2b4fdd6f%';
+count(*)
+64

cmd/explaintest/r/partition_pruning.result

@@ -206,50 +206,43 @@ a
 1
 explain format = 'brief' SELECT * FROM t1 WHERE a = 1;
 id	estRows	task	access object	operator info
-TableReader	1.00	root	partition:p1	data:TableRangeScan
-└─TableRangeScan	1.00	cop[tikv]	table:t1	range:[1,1], keep order:false, stats:pseudo
+Point_Get	1.00	root	table:t1, partition:p1	handle:1
 SELECT * FROM t1 WHERE a = 2 order by a;
 a
 2
 explain format = 'brief' SELECT * FROM t1 WHERE a = 2;
 id	estRows	task	access object	operator info
-TableReader	1.00	root	partition:p2	data:TableRangeScan
-└─TableRangeScan	1.00	cop[tikv]	table:t1	range:[2,2], keep order:false, stats:pseudo
+Point_Get	1.00	root	table:t1, partition:p2	handle:2
 SELECT * FROM t1 WHERE a = 3 order by a;
 a
 3
 explain format = 'brief' SELECT * FROM t1 WHERE a = 3;
 id	estRows	task	access object	operator info
-TableReader	1.00	root	partition:p3	data:TableRangeScan
-└─TableRangeScan	1.00	cop[tikv]	table:t1	range:[3,3], keep order:false, stats:pseudo
+Point_Get	1.00	root	table:t1, partition:p3	handle:3
 SELECT * FROM t1 WHERE a = 4 order by a;
 a
 4
 explain format = 'brief' SELECT * FROM t1 WHERE a = 4;
 id	estRows	task	access object	operator info
-TableReader	1.00	root	partition:p4	data:TableRangeScan
-└─TableRangeScan	1.00	cop[tikv]	table:t1	range:[4,4], keep order:false, stats:pseudo
+Point_Get	1.00	root	table:t1, partition:p4	handle:4
 SELECT * FROM t1 WHERE a = 5 order by a;
 a
 5
 explain format = 'brief' SELECT * FROM t1 WHERE a = 5;
 id	estRows	task	access object	operator info
-TableReader	1.00	root	partition:p5	data:TableRangeScan
-└─TableRangeScan	1.00	cop[tikv]	table:t1	range:[5,5], keep order:false, stats:pseudo
+Point_Get	1.00	root	table:t1, partition:p5	handle:5
 SELECT * FROM t1 WHERE a = 6 order by a;
 a
 6
 explain format = 'brief' SELECT * FROM t1 WHERE a = 6;
 id	estRows	task	access object	operator info
-TableReader	1.00	root	partition:max	data:TableRangeScan
-└─TableRangeScan	1.00	cop[tikv]	table:t1	range:[6,6], keep order:false, stats:pseudo
+Point_Get	1.00	root	table:t1, partition:max	handle:6
 SELECT * FROM t1 WHERE a = 7 order by a;
 a
 7
 explain format = 'brief' SELECT * FROM t1 WHERE a = 7;
 id	estRows	task	access object	operator info
-TableReader	1.00	root	partition:max	data:TableRangeScan
-└─TableRangeScan	1.00	cop[tikv]	table:t1	range:[7,7], keep order:false, stats:pseudo
+Point_Get	1.00	root	table:t1, partition:max	handle:7
 SELECT * FROM t1 WHERE a >= 1 order by a;
 a
 1
@@ -544,43 +537,37 @@ a
 1
 explain format = 'brief' SELECT * FROM t1 WHERE a = 1;
 id	estRows	task	access object	operator info
-TableReader	1.00	root	partition:p1	data:TableRangeScan
-└─TableRangeScan	1.00	cop[tikv]	table:t1	range:[1,1], keep order:false, stats:pseudo
+Point_Get	1.00	root	table:t1, partition:p1	handle:1
 SELECT * FROM t1 WHERE a = 2;
 a
 2
 explain format = 'brief' SELECT * FROM t1 WHERE a = 2;
 id	estRows	task	access object	operator info
-TableReader	1.00	root	partition:p2	data:TableRangeScan
-└─TableRangeScan	1.00	cop[tikv]	table:t1	range:[2,2], keep order:false, stats:pseudo
+Point_Get	1.00	root	table:t1, partition:p2	handle:2
 SELECT * FROM t1 WHERE a = 3;
 a
 3
 explain format = 'brief' SELECT * FROM t1 WHERE a = 3;
 id	estRows	task	access object	operator info
-TableReader	1.00	root	partition:p3	data:TableRangeScan
-└─TableRangeScan	1.00	cop[tikv]	table:t1	range:[3,3], keep order:false, stats:pseudo
+Point_Get	1.00	root	table:t1, partition:p3	handle:3
 SELECT * FROM t1 WHERE a = 4;
 a
 4
 explain format = 'brief' SELECT * FROM t1 WHERE a = 4;
 id	estRows	task	access object	operator info
-TableReader	1.00	root	partition:p4	data:TableRangeScan
-└─TableRangeScan	1.00	cop[tikv]	table:t1	range:[4,4], keep order:false, stats:pseudo
+Point_Get	1.00	root	table:t1, partition:p4	handle:4
 SELECT * FROM t1 WHERE a = 5;
 a
 5
 explain format = 'brief' SELECT * FROM t1 WHERE a = 5;
 id	estRows	task	access object	operator info
-TableReader	1.00	root	partition:max	data:TableRangeScan
-└─TableRangeScan	1.00	cop[tikv]	table:t1	range:[5,5], keep order:false, stats:pseudo
+Point_Get	1.00	root	table:t1, partition:max	handle:5
 SELECT * FROM t1 WHERE a = 6;
 a
 6
 explain format = 'brief' SELECT * FROM t1 WHERE a = 6;
 id	estRows	task	access object	operator info
-TableReader	1.00	root	partition:max	data:TableRangeScan
-└─TableRangeScan	1.00	cop[tikv]	table:t1	range:[6,6], keep order:false, stats:pseudo
+Point_Get	1.00	root	table:t1, partition:max	handle:6
 SELECT * FROM t1 WHERE a >= 1 order by a;
 a
 1
@@ -1795,9 +1782,7 @@ TableReader	3323.33	root	partition:all	data:Selection
   └─TableFullScan	10000.00	cop[tikv]	table:t7	keep order:false, stats:pseudo
 explain format = 'brief' select * from t7 where a = 90;
 id	estRows	task	access object	operator info
-TableReader	10.00	root	partition:dual	data:Selection
-└─Selection	10.00	cop[tikv]		eq(test.t7.a, 90)
-  └─TableFullScan	10000.00	cop[tikv]	table:t7	keep order:false, stats:pseudo
+TableDual	0.00	root		rows:0
 explain format = 'brief' select * from t7 where a > 90;
 id	estRows	task	access object	operator info
 TableReader	3333.33	root	partition:dual	data:Selection
@@ -1919,9 +1904,7 @@ TableReader	3323.33	root	partition:all	data:Selection
   └─TableFullScan	10000.00	cop[tikv]	table:t7	keep order:false, stats:pseudo
 explain format = 'brief' select * from t7 where a = 90;
 id	estRows	task	access object	operator info
-TableReader	10.00	root	partition:dual	data:Selection
-└─Selection	10.00	cop[tikv]		eq(test.t7.a, 90)
-  └─TableFullScan	10000.00	cop[tikv]	table:t7	keep order:false, stats:pseudo
+TableDual	0.00	root		rows:0
 explain format = 'brief' select * from t7 where a > 90;
 id	estRows	task	access object	operator info
 TableReader	3333.33	root	partition:dual	data:Selection