Skip to content

Commit

Permalink
Merge branch 'master' into move-index-calc-front
Browse files Browse the repository at this point in the history
  • Loading branch information
@winoros
winoros committed Apr 25, 2018
2 parents ba77bdf + e41b908 commit d1a7691
Show file tree
Hide file tree
Showing 57 changed files with 1,994 additions and 627 deletions.
15 changes: 9 additions & 6 deletions Gopkg.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

14 changes: 11 additions & 3 deletions Gopkg.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,12 @@ ignored = ["github.com/coreos/etcd/etcdserver", "github.com/coreos/etcd/mvcc", "
branch = "master"
name = "github.com/blacktear23/go-proxyprotocol"

# lock etcd version at 3.2.14
# lock etcd version at 3.2.18
[[constraint]]
name = "github.com/coreos/etcd"
version = "=3.2.14"
version = "=3.2.18"

# lock grpc version at 1.7.5 becuase etcd depdens on this version
# lock grpc version at 1.7.5 because etcd depdens on this version
[[constraint]]
name = "google.golang.org/grpc"
source = "https://github.com/grpc/grpc-go.git"
Expand Down Expand Up @@ -68,6 +68,14 @@ ignored = ["github.com/coreos/etcd/etcdserver", "github.com/coreos/etcd/mvcc", "
name = "github.com/uber/jaeger-client-go"
version = "2.8.0"

[[constraint]]
name = "github.com/pingcap/pd"
branch = "master"

[[constraint]]
name = "github.com/pingcap/kvproto"
branch = "master"

[[constraint]]
name = "gopkg.in/natefinch/lumberjack.v2"
version = "2.0.0"
2 changes: 1 addition & 1 deletion Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ errcheck:
@ GOPATH=$(GOPATH) errcheck -blank $(PACKAGES) | grep -v "_test\.go" | awk '{print} END{if(NR>0) {exit 1}}'

lint:
go get github.com/golang/lint/golint
go get golang.org/x/lint/golint
@echo "golint"
@ golint -set_exit_status $(PACKAGES)

Expand Down
8 changes: 4 additions & 4 deletions ddl/ddl_db_change_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -566,13 +566,13 @@ func (s *testStateChangeSuite) TestParallelChangeColumnName(c *C) {
// Make sure the both DDL statements have entered the DDL queue before running the DDL jobs.
once.Do(func() {
var qLen int64
var err error
var err2 error
for {
kv.RunInNewTxn(s.store, false, func(txn kv.Transaction) error {
m := meta.NewMeta(txn)
qLen, err = m.DDLJobQueueLen()
if err != nil {
return err
qLen, err2 = m.DDLJobQueueLen()
if err2 != nil {
return err2
}
return nil
})
Expand Down
46 changes: 20 additions & 26 deletions distsql/request_builder.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ func (builder *RequestBuilder) Build() (*kv.Request, error) {

// SetTableRanges sets "KeyRanges" for "kv.Request" by converting "tableRanges"
// to "KeyRanges" firstly.
func (builder *RequestBuilder) SetTableRanges(tid int64, tableRanges []*ranger.NewRange, fb *statistics.QueryFeedback) *RequestBuilder {
func (builder *RequestBuilder) SetTableRanges(tid int64, tableRanges []*ranger.Range, fb *statistics.QueryFeedback) *RequestBuilder {
if builder.err != nil {
return builder
}
Expand All @@ -54,7 +54,7 @@ func (builder *RequestBuilder) SetTableRanges(tid int64, tableRanges []*ranger.N

// SetIndexRanges sets "KeyRanges" for "kv.Request" by converting index range
// "ranges" to "KeyRanges" firstly.
func (builder *RequestBuilder) SetIndexRanges(sc *stmtctx.StatementContext, tid, idxID int64, ranges []*ranger.NewRange) *RequestBuilder {
func (builder *RequestBuilder) SetIndexRanges(sc *stmtctx.StatementContext, tid, idxID int64, ranges []*ranger.Range) *RequestBuilder {
if builder.err != nil {
return builder
}
Expand Down Expand Up @@ -174,28 +174,28 @@ func (builder *RequestBuilder) SetConcurrency(concurrency int) *RequestBuilder {
}

// TableRangesToKVRanges converts table ranges to "KeyRange".
func TableRangesToKVRanges(tid int64, ranges []*ranger.NewRange, fb *statistics.QueryFeedback) []kv.KeyRange {
func TableRangesToKVRanges(tid int64, ranges []*ranger.Range, fb *statistics.QueryFeedback) []kv.KeyRange {
if fb == nil || fb.Hist() == nil {
return tableRangesToKVRangesWithoutSplit(tid, ranges)
}
ranges = fb.Hist().SplitRange(ranges)
krs := make([]kv.KeyRange, 0, len(ranges))
newRanges := make([]*ranger.NewRange, 0, len(ranges))
newRanges := make([]*ranger.Range, 0, len(ranges))
for _, ran := range ranges {
low, high := encodeHandleKey(ran)
startKey := tablecodec.EncodeRowKey(tid, low)
endKey := tablecodec.EncodeRowKey(tid, high)
krs = append(krs, kv.KeyRange{StartKey: startKey, EndKey: endKey})

r := &ranger.NewRange{LowVal: []types.Datum{types.NewBytesDatum(low)},
r := &ranger.Range{LowVal: []types.Datum{types.NewBytesDatum(low)},
HighVal: []types.Datum{types.NewBytesDatum(high)}}
newRanges = append(newRanges, r)
}
fb.StoreRanges(newRanges)
return krs
}

func tableRangesToKVRangesWithoutSplit(tid int64, ranges []*ranger.NewRange) []kv.KeyRange {
func tableRangesToKVRangesWithoutSplit(tid int64, ranges []*ranger.Range) []kv.KeyRange {
krs := make([]kv.KeyRange, 0, len(ranges))
for _, ran := range ranges {
low, high := encodeHandleKey(ran)
Expand All @@ -206,7 +206,7 @@ func tableRangesToKVRangesWithoutSplit(tid int64, ranges []*ranger.NewRange) []k
return krs
}

func encodeHandleKey(ran *ranger.NewRange) ([]byte, []byte) {
func encodeHandleKey(ran *ranger.Range) ([]byte, []byte) {
low := codec.EncodeInt(nil, ran.LowVal[0].GetInt64())
high := codec.EncodeInt(nil, ran.HighVal[0].GetInt64())
if ran.LowExclude {
Expand All @@ -224,41 +224,35 @@ func TableHandlesToKVRanges(tid int64, handles []int64) []kv.KeyRange {
krs := make([]kv.KeyRange, 0, len(handles))
i := 0
for i < len(handles) {
h := handles[i]
if h == math.MaxInt64 {
// We can't convert MaxInt64 into an left closed, right open range.
i++
continue
}
j := i + 1
endHandle := h + 1
for ; j < len(handles); j++ {
if handles[j] == endHandle {
endHandle = handles[j] + 1
continue
for ; j < len(handles) && handles[j-1] != math.MaxInt64; j++ {
if handles[j] != handles[j-1]+1 {
break
}
break
}
startKey := tablecodec.EncodeRowKeyWithHandle(tid, h)
endKey := tablecodec.EncodeRowKeyWithHandle(tid, endHandle)
low := codec.EncodeInt(nil, handles[i])
high := codec.EncodeInt(nil, handles[j-1])
high = []byte(kv.Key(high).PrefixNext())
startKey := tablecodec.EncodeRowKey(tid, low)
endKey := tablecodec.EncodeRowKey(tid, high)
krs = append(krs, kv.KeyRange{StartKey: startKey, EndKey: endKey})
i = j
}
return krs
}

// IndexRangesToKVRanges converts index ranges to "KeyRange".
func IndexRangesToKVRanges(sc *stmtctx.StatementContext, tid, idxID int64, ranges []*ranger.NewRange, fb *statistics.QueryFeedback) ([]kv.KeyRange, error) {
func IndexRangesToKVRanges(sc *stmtctx.StatementContext, tid, idxID int64, ranges []*ranger.Range, fb *statistics.QueryFeedback) ([]kv.KeyRange, error) {
if fb == nil || fb.Hist() == nil {
return indexRangesToKVWithoutSplit(sc, tid, idxID, ranges)
}
newRanges := make([]*ranger.NewRange, 0, len(ranges))
newRanges := make([]*ranger.Range, 0, len(ranges))
for _, ran := range ranges {
low, high, err := encodeIndexKey(sc, ran)
if err != nil {
return nil, errors.Trace(err)
}
newRanges = append(newRanges, &ranger.NewRange{LowVal: []types.Datum{types.NewBytesDatum(low)},
newRanges = append(newRanges, &ranger.Range{LowVal: []types.Datum{types.NewBytesDatum(low)},
HighVal: []types.Datum{types.NewBytesDatum(high)}, LowExclude: false, HighExclude: true})
}
ranges = fb.Hist().SplitRange(newRanges)
Expand All @@ -278,7 +272,7 @@ func IndexRangesToKVRanges(sc *stmtctx.StatementContext, tid, idxID int64, range
return krs, nil
}

func indexRangesToKVWithoutSplit(sc *stmtctx.StatementContext, tid, idxID int64, ranges []*ranger.NewRange) ([]kv.KeyRange, error) {
func indexRangesToKVWithoutSplit(sc *stmtctx.StatementContext, tid, idxID int64, ranges []*ranger.Range) ([]kv.KeyRange, error) {
krs := make([]kv.KeyRange, 0, len(ranges))
for _, ran := range ranges {
low, high, err := encodeIndexKey(sc, ran)
Expand All @@ -292,7 +286,7 @@ func indexRangesToKVWithoutSplit(sc *stmtctx.StatementContext, tid, idxID int64,
return krs, nil
}

func encodeIndexKey(sc *stmtctx.StatementContext, ran *ranger.NewRange) ([]byte, []byte, error) {
func encodeIndexKey(sc *stmtctx.StatementContext, ran *ranger.Range) ([]byte, []byte, error) {
low, err := codec.EncodeKey(sc, nil, ran.LowVal...)
if err != nil {
return nil, nil, errors.Trace(err)
Expand Down
27 changes: 16 additions & 11 deletions distsql/request_builder_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import (
"github.com/pingcap/tidb/sessionctx/variable"
"github.com/pingcap/tidb/tablecodec"
"github.com/pingcap/tidb/types"
"github.com/pingcap/tidb/util/codec"
"github.com/pingcap/tidb/util/logutil"
"github.com/pingcap/tidb/util/mock"
"github.com/pingcap/tidb/util/ranger"
Expand Down Expand Up @@ -83,22 +84,26 @@ type handleRange struct {
func (s *testSuite) getExpectedRanges(tid int64, hrs []*handleRange) []kv.KeyRange {
krs := make([]kv.KeyRange, 0, len(hrs))
for _, hr := range hrs {
startKey := tablecodec.EncodeRowKeyWithHandle(tid, hr.start)
endKey := tablecodec.EncodeRowKeyWithHandle(tid, hr.end)
low := codec.EncodeInt(nil, hr.start)
high := codec.EncodeInt(nil, hr.end)
high = []byte(kv.Key(high).PrefixNext())
startKey := tablecodec.EncodeRowKey(tid, low)
endKey := tablecodec.EncodeRowKey(tid, high)
krs = append(krs, kv.KeyRange{StartKey: startKey, EndKey: endKey})
}
return krs
}

func (s *testSuite) TestTableHandlesToKVRanges(c *C) {
handles := []int64{0, 2, 3, 4, 5, 10, 11, 100}
handles := []int64{0, 2, 3, 4, 5, 10, 11, 100, 9223372036854775806, 9223372036854775807}

// Build expected key ranges.
hrs := make([]*handleRange, 0, len(handles))
hrs = append(hrs, &handleRange{start: 0, end: 1})
hrs = append(hrs, &handleRange{start: 2, end: 6})
hrs = append(hrs, &handleRange{start: 10, end: 12})
hrs = append(hrs, &handleRange{start: 100, end: 101})
hrs = append(hrs, &handleRange{start: 0, end: 0})
hrs = append(hrs, &handleRange{start: 2, end: 5})
hrs = append(hrs, &handleRange{start: 10, end: 11})
hrs = append(hrs, &handleRange{start: 100, end: 100})
hrs = append(hrs, &handleRange{start: 9223372036854775806, end: 9223372036854775807})

// Build key ranges.
expect := s.getExpectedRanges(1, hrs)
Expand All @@ -113,7 +118,7 @@ func (s *testSuite) TestTableHandlesToKVRanges(c *C) {
}

func (s *testSuite) TestTableRangesToKVRanges(c *C) {
ranges := []*ranger.NewRange{
ranges := []*ranger.Range{
{
LowVal: []types.Datum{types.NewIntDatum(1)},
HighVal: []types.Datum{types.NewIntDatum(2)},
Expand Down Expand Up @@ -170,7 +175,7 @@ func (s *testSuite) TestTableRangesToKVRanges(c *C) {
}

func (s *testSuite) TestIndexRangesToKVRanges(c *C) {
ranges := []*ranger.NewRange{
ranges := []*ranger.Range{
{
LowVal: []types.Datum{types.NewIntDatum(1)},
HighVal: []types.Datum{types.NewIntDatum(2)},
Expand Down Expand Up @@ -229,7 +234,7 @@ func (s *testSuite) TestIndexRangesToKVRanges(c *C) {
}

func (s *testSuite) TestRequestBuilder1(c *C) {
ranges := []*ranger.NewRange{
ranges := []*ranger.Range{
{
LowVal: []types.Datum{types.NewIntDatum(1)},
HighVal: []types.Datum{types.NewIntDatum(2)},
Expand Down Expand Up @@ -303,7 +308,7 @@ func (s *testSuite) TestRequestBuilder1(c *C) {
}

func (s *testSuite) TestRequestBuilder2(c *C) {
ranges := []*ranger.NewRange{
ranges := []*ranger.Range{
{
LowVal: []types.Datum{types.NewIntDatum(1)},
HighVal: []types.Datum{types.NewIntDatum(2)},
Expand Down
6 changes: 3 additions & 3 deletions executor/admin.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ func (e *CheckIndexRangeExec) Open(ctx context.Context) error {
}
sc := e.ctx.GetSessionVars().StmtCtx
var builder distsql.RequestBuilder
kvReq, err := builder.SetIndexRanges(sc, e.table.ID, e.index.ID, ranger.FullNewRange()).
kvReq, err := builder.SetIndexRanges(sc, e.table.ID, e.index.ID, ranger.FullRange()).
SetDAGRequest(dagPB).
SetKeepOrder(true).
SetFromSessionVars(e.ctx.GetSessionVars()).
Expand Down Expand Up @@ -247,7 +247,7 @@ func (e *RecoverIndexExec) buildTableScan(ctx context.Context, txn kv.Transactio
return nil, errors.Trace(err)
}
tblInfo := e.table.Meta()
ranges := []*ranger.NewRange{{LowVal: []types.Datum{types.NewIntDatum(startHandle)}, HighVal: []types.Datum{types.NewIntDatum(math.MaxInt64)}}}
ranges := []*ranger.Range{{LowVal: []types.Datum{types.NewIntDatum(startHandle)}, HighVal: []types.Datum{types.NewIntDatum(math.MaxInt64)}}}
var builder distsql.RequestBuilder
kvReq, err := builder.SetTableRanges(tblInfo.ID, ranges, nil).
SetDAGRequest(dagPB).
Expand Down Expand Up @@ -609,7 +609,7 @@ func (e *CleanupIndexExec) buildIndexScan(ctx context.Context, txn kv.Transactio
}
sc := e.ctx.GetSessionVars().StmtCtx
var builder distsql.RequestBuilder
ranges := ranger.FullNewRange()
ranges := ranger.FullRange()
kvReq, err := builder.SetIndexRanges(sc, e.table.Meta().ID, e.index.Meta().ID, ranges).
SetDAGRequest(dagPB).
SetKeepOrder(true).
Expand Down
10 changes: 5 additions & 5 deletions executor/analyze.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -176,7 +176,7 @@ type AnalyzeIndexExec struct {

func (e *AnalyzeIndexExec) open() error {
var builder distsql.RequestBuilder
kvReq, err := builder.SetIndexRanges(e.ctx.GetSessionVars().StmtCtx, e.tblInfo.ID, e.idxInfo.ID, ranger.FullNewRange()).
kvReq, err := builder.SetIndexRanges(e.ctx.GetSessionVars().StmtCtx, e.tblInfo.ID, e.idxInfo.ID, ranger.FullRange()).
SetAnalyzeRequest(e.analyzePB).
SetKeepOrder(true).
Build()
Expand Down Expand Up @@ -264,11 +264,11 @@ type AnalyzeColumnsExec struct {
}

func (e *AnalyzeColumnsExec) open() error {
var ranges []*ranger.NewRange
var ranges []*ranger.Range
if e.pkInfo != nil {
ranges = ranger.FullIntNewRange(mysql.HasUnsignedFlag(e.pkInfo.Flag))
ranges = ranger.FullIntRange(mysql.HasUnsignedFlag(e.pkInfo.Flag))
} else {
ranges = ranger.FullIntNewRange(false)
ranges = ranger.FullIntRange(false)
}
e.resultHandler = &tableResultHandler{}
firstPartRanges, secondPartRanges := splitRanges(ranges, e.keepOrder)
Expand All @@ -290,7 +290,7 @@ func (e *AnalyzeColumnsExec) open() error {
return nil
}

func (e *AnalyzeColumnsExec) buildResp(ranges []*ranger.NewRange) (distsql.SelectResult, error) {
func (e *AnalyzeColumnsExec) buildResp(ranges []*ranger.Range) (distsql.SelectResult, error) {
var builder distsql.RequestBuilder
kvReq, err := builder.SetTableRanges(e.tblInfo.ID, ranges, nil).
SetAnalyzeRequest(e.analyzePB).
Expand Down
Loading

0 comments on commit d1a7691

Please sign in to comment.