Merge remote-tracking branch 'origin/enhance_max_min_eli' into enhanc…

…e_max_min_eli

Makefile

@@ -189,9 +189,9 @@ else
 	$(GOBUILD) $(RACE_FLAG) -ldflags '$(CHECK_LDFLAGS)' -o '$(TARGET)' tidb-server/main.go
 endif
 
-coverage_server:
+server_coverage:
 ifeq ($(TARGET), "")
-	$(GOBUILDCOVERAGE) $(RACE_FLAG) -ldflags '$(LDFLAGS) $(COVERAGE_SERVER_LDFLAGS) $(CHECK_FLAG)' -o ../bin/tidb-coverage-server
+	$(GOBUILDCOVERAGE) $(RACE_FLAG) -ldflags '$(LDFLAGS) $(COVERAGE_SERVER_LDFLAGS) $(CHECK_FLAG)' -o ../bin/tidb-server-coverage
 else
 	$(GOBUILDCOVERAGE) $(RACE_FLAG) -ldflags '$(LDFLAGS) $(COVERAGE_SERVER_LDFLAGS) $(CHECK_FLAG)' -o '$(TARGET)'
 endif

README.md

@@ -11,7 +11,8 @@
 [![GoDoc](https://img.shields.io/badge/Godoc-reference-blue.svg)](https://godoc.org/github.com/pingcap/tidb)
 
 - [**Stack Overflow**](https://stackoverflow.com/questions/tagged/tidb)
-- Community [**Slack Channel**](https://pingcap.com/tidbslack/)
+- [**Slack Channel**](https://pingcap.com/tidbslack/)
+- [**User Group (Chinese)**](https://asktug.com)
 - **Twitter**: [@PingCAP](https://twitter.com/PingCAP)
 - [**Reddit**](https://www.reddit.com/r/TiDB/)
 - **Mailing list**: [Google Group](https://groups.google.com/forum/#!forum/tidb-user)

executor/hash_table.go

@@ -15,6 +15,7 @@ package executor
 
 import (
 	"hash"
+	"hash/fnv"
 
 	"github.com/pingcap/errors"
 	"github.com/pingcap/tidb/sessionctx"
@@ -47,8 +48,28 @@ const (
 type hashContext struct {
 	allTypes  []*types.FieldType
 	keyColIdx []int
-	h         hash.Hash64
 	buf       []byte
+	hashVals  []hash.Hash64
+	hasNull   []bool
+}
+
+func (hc *hashContext) initHash(rows int) {
+	if hc.buf == nil {
+		hc.buf = make([]byte, 1)
+	}
+
+	if len(hc.hashVals) < rows {
+		hc.hasNull = make([]bool, rows)
+		hc.hashVals = make([]hash.Hash64, rows)
+		for i := 0; i < rows; i++ {
+			hc.hashVals[i] = fnv.New64()
+		}
+	} else {
+		for i := 0; i < rows; i++ {
+			hc.hasNull[i] = false
+			hc.hashVals[i].Reset()
+		}
+	}
 }
 
 // hashRowContainer handles the rows and the hash map of a table.
@@ -133,22 +154,24 @@ func (c *hashRowContainer) matchJoinKey(buildRow, probeRow chunk.Row, probeHCtx
 // value of hash table: RowPtr of the corresponded row
 func (c *hashRowContainer) PutChunk(chk *chunk.Chunk) error {
 	chkIdx := uint32(c.records.NumChunks())
-	c.records.Add(chk)
-	var (
-		hasNull bool
-		err     error
-		key     uint64
-	)
 	numRows := chk.NumRows()
-	for j := 0; j < numRows; j++ {
-		hasNull, key, err = c.getJoinKeyFromChkRow(c.sc, chk.GetRow(j), c.hCtx)
+
+	c.records.Add(chk)
+	c.hCtx.initHash(numRows)
+
+	hCtx := c.hCtx
+	for _, colIdx := range c.hCtx.keyColIdx {
+		err := codec.HashChunkColumns(c.sc, hCtx.hashVals, chk, hCtx.allTypes[colIdx], colIdx, hCtx.buf, hCtx.hasNull)
 		if err != nil {
 			return errors.Trace(err)
 		}
-		if hasNull {
+	}
+	for i := 0; i < numRows; i++ {
+		if c.hCtx.hasNull[i] {
 			continue
 		}
-		rowPtr := chunk.RowPtr{ChkIdx: chkIdx, RowIdx: uint32(j)}
+		key := c.hCtx.hashVals[i].Sum64()
+		rowPtr := chunk.RowPtr{ChkIdx: chkIdx, RowIdx: uint32(i)}
 		c.hashTable.Put(key, rowPtr)
 	}
 	return nil
@@ -161,9 +184,9 @@ func (*hashRowContainer) getJoinKeyFromChkRow(sc *stmtctx.StatementContext, row
 			return true, 0, nil
 		}
 	}
-	hCtx.h.Reset()
-	err = codec.HashChunkRow(sc, hCtx.h, row, hCtx.allTypes, hCtx.keyColIdx, hCtx.buf)
-	return false, hCtx.h.Sum64(), err
+	hCtx.initHash(1)
+	err = codec.HashChunkRow(sc, hCtx.hashVals[0], row, hCtx.allTypes, hCtx.keyColIdx, hCtx.buf)
+	return false, hCtx.hashVals[0].Sum64(), err
 }
 
 func (c hashRowContainer) Len() int {

executor/join.go

@@ -16,7 +16,6 @@ package executor
 import (
 	"context"
 	"fmt"
-	"hash/fnv"
 	"sync"
 	"sync/atomic"
 
@@ -334,8 +333,6 @@ func (e *HashJoinExec) runJoinWorker(workerID uint, outerKeyColIdx []int) {
 	hCtx := &hashContext{
 		allTypes:  retTypes(e.outerExec),
 		keyColIdx: outerKeyColIdx,
-		h:         fnv.New64(),
-		buf:       make([]byte, 1),
 	}
 	for ok := true; ok; {
 		if e.finished.Load().(bool) {
@@ -506,8 +503,6 @@ func (e *HashJoinExec) buildHashTableForList(innerResultCh <-chan *chunk.Chunk)
 	hCtx := &hashContext{
 		allTypes:  allTypes,
 		keyColIdx: innerKeyColIdx,
-		h:         fnv.New64(),
-		buf:       make([]byte, 1),
 	}
 	initList := chunk.NewList(allTypes, e.initCap, e.maxChunkSize)
 	e.rowContainer = newHashRowContainer(e.ctx, int(e.innerEstCount), hCtx, initList)

infoschema/tables.go

@@ -590,8 +590,8 @@ var tableTiKVStoreStatusCols = []columnInfo{
 	{"CAPACITY", mysql.TypeVarchar, 64, 0, nil, nil},
 	{"AVAILABLE", mysql.TypeVarchar, 64, 0, nil, nil},
 	{"LEADER_COUNT", mysql.TypeLonglong, 21, 0, nil, nil},
-	{"LEADER_WEIGHT", mysql.TypeLonglong, 21, 0, nil, nil},
-	{"LEADER_SCORE", mysql.TypeLonglong, 21, 0, nil, nil},
+	{"LEADER_WEIGHT", mysql.TypeDouble, 22, 0, nil, nil},
+	{"LEADER_SCORE", mysql.TypeDouble, 22, 0, nil, nil},
 	{"LEADER_SIZE", mysql.TypeLonglong, 21, 0, nil, nil},
 	{"REGION_COUNT", mysql.TypeLonglong, 21, 0, nil, nil},
 	{"REGION_WEIGHT", mysql.TypeDouble, 22, 0, nil, nil},
@@ -818,8 +818,8 @@ func dataForTiKVStoreStatus(ctx sessionctx.Context) (records [][]types.Datum, er
 		row[6].SetString(storeStat.Status.Capacity)
 		row[7].SetString(storeStat.Status.Available)
 		row[8].SetInt64(storeStat.Status.LeaderCount)
-		row[9].SetInt64(storeStat.Status.LeaderWeight)
-		row[10].SetInt64(storeStat.Status.LeaderScore)
+		row[9].SetFloat64(storeStat.Status.LeaderWeight)
+		row[10].SetFloat64(storeStat.Status.LeaderScore)
 		row[11].SetInt64(storeStat.Status.LeaderSize)
 		row[12].SetInt64(storeStat.Status.RegionCount)
 		row[13].SetFloat64(storeStat.Status.RegionWeight)

statistics/feedback.go

@@ -313,15 +313,21 @@ func buildBucketFeedback(h *Histogram, feedback *QueryFeedback) (map[int]*Bucket
 		if skip {
 			continue
 		}
-		idx, _ := h.Bounds.LowerBound(0, fb.Lower)
+		idx := h.Bounds.UpperBound(0, fb.Lower)
 		bktIdx := 0
 		// The last bucket also stores the feedback that falls outside the upper bound.
-		if idx >= h.Bounds.NumRows()-2 {
+		if idx >= h.Bounds.NumRows()-1 {
 			bktIdx = h.Len() - 1
+		} else if h.Len() == 1 {
+			bktIdx = 0
 		} else {
-			bktIdx = idx / 2
+			if idx == 0 {
+				bktIdx = 0
+			} else {
+				bktIdx = (idx - 1) / 2
+			}
 			// Make sure that this feedback lies within the bucket.
-			if chunk.Compare(h.Bounds.GetRow(2*bktIdx+1), 0, fb.Upper) < 0 {
+			if chunk.Compare(h.Bounds.GetRow(2*(bktIdx+1)), 0, fb.Upper) < 0 {
 				continue
 			}
 		}

statistics/feedback_test.go

@@ -72,14 +72,13 @@ func (s *testFeedbackSuite) TestUpdateHistogram(c *C) {
 	defaultBucketCount = 7
 	defer func() { defaultBucketCount = originBucketCount }()
 	c.Assert(UpdateHistogram(q.Hist, q).ToString(0), Equals,
-		"column:0 ndv:10058 totColSize:0\n"+
-			"num: 10000 lower_bound: 0 upper_bound: 1 repeats: 0\n"+
-			"num: 9 lower_bound: 2 upper_bound: 7 repeats: 0\n"+
-			"num: 11 lower_bound: 8 upper_bound: 19 repeats: 0\n"+
-			"num: 0 lower_bound: 20 upper_bound: 20 repeats: 0\n"+
-			"num: 18 lower_bound: 21 upper_bound: 39 repeats: 0\n"+
-			"num: 18 lower_bound: 40 upper_bound: 58 repeats: 0\n"+
-			"num: 2 lower_bound: 59 upper_bound: 60 repeats: 0")
+		"column:0 ndv:10053 totColSize:0\n"+
+			"num: 10001 lower_bound: 0 upper_bound: 2 repeats: 0\n"+
+			"num: 7 lower_bound: 2 upper_bound: 5 repeats: 0\n"+
+			"num: 4 lower_bound: 5 upper_bound: 7 repeats: 0\n"+
+			"num: 11 lower_bound: 10 upper_bound: 20 repeats: 0\n"+
+			"num: 19 lower_bound: 30 upper_bound: 49 repeats: 0\n"+
+			"num: 11 lower_bound: 50 upper_bound: 60 repeats: 0")
 }
 
 func (s *testFeedbackSuite) TestSplitBuckets(c *C) {

statistics/handle/update.go

@@ -766,11 +766,11 @@ func formatBuckets(hg *statistics.Histogram, lowBkt, highBkt, idxCols int) strin
 		return hg.BucketToString(lowBkt, idxCols)
 	}
 	if lowBkt+1 == highBkt {
-		return fmt.Sprintf("%s, %s", hg.BucketToString(lowBkt, 0), hg.BucketToString(highBkt, 0))
+		return fmt.Sprintf("%s, %s", hg.BucketToString(lowBkt, idxCols), hg.BucketToString(highBkt, idxCols))
 	}
 	// do not care the middle buckets
-	return fmt.Sprintf("%s, (%d buckets, total count %d), %s", hg.BucketToString(lowBkt, 0),
-		highBkt-lowBkt-1, hg.Buckets[highBkt-1].Count-hg.Buckets[lowBkt].Count, hg.BucketToString(highBkt, 0))
+	return fmt.Sprintf("%s, (%d buckets, total count %d), %s", hg.BucketToString(lowBkt, idxCols),
+		highBkt-lowBkt-1, hg.Buckets[highBkt-1].Count-hg.Buckets[lowBkt].Count, hg.BucketToString(highBkt, idxCols))
 }
 
 func colRangeToStr(c *statistics.Column, ran *ranger.Range, actual int64, factor float64) string {