backup/checksum: add cluster index support for ranges (pingcap#1120)

pkg/backup/client.go

@@ -266,8 +266,18 @@ func BuildTableRanges(tbl *model.TableInfo) ([]kv.KeyRange, error) {
 }
 
 func appendRanges(tbl *model.TableInfo, tblID int64) ([]kv.KeyRange, error) {
-	ranges := ranger.FullIntRange(false)
-	kvRanges := distsql.TableRangesToKVRanges(tblID, ranges, nil)
+	var ranges []*ranger.Range
+	if tbl.IsCommonHandle {
+		ranges = ranger.FullNotNullRange()
+	} else {
+		ranges = ranger.FullIntRange(false)
+	}
+
+	kvRanges, err := distsql.TableHandleRangesToKVRanges(nil, []int64{tblID}, tbl.IsCommonHandle, ranges, nil)
+	if err != nil {
+		return nil, errors.Trace(err)
+	}
+
 	for _, index := range tbl.Indices {
 		if index.State != model.StatePublic {
 			continue

pkg/backup/client_test.go

@@ -17,6 +17,7 @@ import (
 	"github.com/pingcap/tidb/store/tikv/mockstore/mocktikv"
 	"github.com/pingcap/tidb/store/tikv/oracle"
 	"github.com/pingcap/tidb/tablecodec"
+	"github.com/pingcap/tidb/types"
 	"github.com/pingcap/tidb/util/codec"
 	pd "github.com/tikv/pd/client"
 
@@ -104,7 +105,7 @@ func (r *testBackup) TestGetTS(c *C) {
 	c.Assert(ts, Equals, backupts)
 }
 
-func (r *testBackup) TestBuildTableRange(c *C) {
+func (r *testBackup) TestBuildTableRangeIntHandle(c *C) {
 	type Case struct {
 		ids []int64
 		trs []kv.KeyRange
@@ -145,6 +146,50 @@ func (r *testBackup) TestBuildTableRange(c *C) {
 	})
 }
 
+func (r *testBackup) TestBuildTableRangeCommonHandle(c *C) {
+	type Case struct {
+		ids []int64
+		trs []kv.KeyRange
+	}
+	low, err_l := codec.EncodeKey(nil, nil, []types.Datum{types.MinNotNullDatum()}...)
+	c.Assert(err_l, IsNil)
+	high, err_h := codec.EncodeKey(nil, nil, []types.Datum{types.MaxValueDatum()}...)
+	c.Assert(err_h, IsNil)
+	high = kv.Key(high).PrefixNext()
+	cases := []Case{
+		{ids: []int64{1}, trs: []kv.KeyRange{
+			{StartKey: tablecodec.EncodeRowKey(1, low), EndKey: tablecodec.EncodeRowKey(1, high)},
+		}},
+		{ids: []int64{1, 2, 3}, trs: []kv.KeyRange{
+			{StartKey: tablecodec.EncodeRowKey(1, low), EndKey: tablecodec.EncodeRowKey(1, high)},
+			{StartKey: tablecodec.EncodeRowKey(2, low), EndKey: tablecodec.EncodeRowKey(2, high)},
+			{StartKey: tablecodec.EncodeRowKey(3, low), EndKey: tablecodec.EncodeRowKey(3, high)},
+		}},
+		{ids: []int64{1, 3}, trs: []kv.KeyRange{
+			{StartKey: tablecodec.EncodeRowKey(1, low), EndKey: tablecodec.EncodeRowKey(1, high)},
+			{StartKey: tablecodec.EncodeRowKey(3, low), EndKey: tablecodec.EncodeRowKey(3, high)},
+		}},
+	}
+	for _, cs := range cases {
+		c.Log(cs)
+		tbl := &model.TableInfo{Partition: &model.PartitionInfo{Enable: true}, IsCommonHandle: true}
+		for _, id := range cs.ids {
+			tbl.Partition.Definitions = append(tbl.Partition.Definitions,
+				model.PartitionDefinition{ID: id})
+		}
+		ranges, err := backup.BuildTableRanges(tbl)
+		c.Assert(err, IsNil)
+		c.Assert(ranges, DeepEquals, cs.trs)
+	}
+
+	tbl := &model.TableInfo{ID: 7, IsCommonHandle: true}
+	ranges, err_r := backup.BuildTableRanges(tbl)
+	c.Assert(err_r, IsNil)
+	c.Assert(ranges, DeepEquals, []kv.KeyRange{
+		{StartKey: tablecodec.EncodeRowKey(7, low), EndKey: tablecodec.EncodeRowKey(7, high)},
+	})
+}
+
 func (r *testBackup) TestOnBackupRegionErrorResponse(c *C) {
 	type Case struct {
 		storeID           uint64

pkg/checksum/executor.go

@@ -111,7 +111,7 @@ func buildRequest(
 	concurrency uint,
 ) ([]*kv.Request, error) {
 	reqs := make([]*kv.Request, 0)
-	req, err := buildTableRequest(tableID, oldTable, oldTableID, startTS, concurrency)
+	req, err := buildTableRequest(tableInfo, tableID, oldTable, oldTableID, startTS, concurrency)
 	if err != nil {
 		return nil, errors.Trace(err)
 	}
@@ -151,6 +151,7 @@ func buildRequest(
 }
 
 func buildTableRequest(
+	tableInfo *model.TableInfo,
 	tableID int64,
 	oldTable *utils.Table,
 	oldTableID int64,
@@ -171,12 +172,17 @@ func buildTableRequest(
 		Rule:      rule,
 	}
 
-	ranges := ranger.FullIntRange(false)
+	var ranges []*ranger.Range
+	if tableInfo.IsCommonHandle {
+		ranges = ranger.FullNotNullRange()
+	} else {
+		ranges = ranger.FullIntRange(false)
+	}
 
 	var builder distsql.RequestBuilder
 	// Use low priority to reducing impact to other requests.
 	builder.Request.Priority = kv.PriorityLow
-	return builder.SetTableRanges(tableID, ranges, nil).
+	return builder.SetHandleRanges(nil, tableID, tableInfo.IsCommonHandle, ranges, nil).
 		SetStartTS(startTS).
 		SetChecksumRequest(checksum).
 		SetConcurrency(int(concurrency)).

pkg/checksum/executor_test.go

@@ -14,6 +14,7 @@ import (
 	"github.com/pingcap/tidb/util/testkit"
 	"github.com/pingcap/tidb/util/testleak"
 
+	"github.com/pingcap/br/pkg/backup"
 	"github.com/pingcap/br/pkg/checksum"
 	"github.com/pingcap/br/pkg/mock"
 	"github.com/pingcap/br/pkg/utils"
@@ -108,4 +109,23 @@ func (s *testChecksumSuite) TestChecksum(c *C) {
 	resp2, err = exe2.Execute(context.TODO(), s.mock.Storage.GetClient(), func() {})
 	c.Assert(err, IsNil)
 	c.Assert(resp2, NotNil)
+
+	// Test commonHandle ranges
+
+	tk.MustExec("drop table if exists t3;")
+	tk.MustExec("create table t3 (a char(255), b int, primary key(a) CLUSTERED);")
+	tk.MustExec("insert into t3 values ('fffffffff', 1), ('010101010', 2), ('394393fj39efefe', 3);")
+	tableInfo3 := s.getTableInfo(c, "test", "t3")
+	exe3, err := checksum.NewExecutorBuilder(tableInfo3, math.MaxUint64).Build()
+	c.Assert(err, IsNil)
+	first := true
+	exe3.Each(func(req *kv.Request) error {
+		if first {
+			first = false
+			ranges, err := backup.BuildTableRanges(tableInfo3)
+			c.Assert(err, IsNil)
+			c.Assert(req.KeyRanges, DeepEquals, ranges[:1], Commentf("%v", req.KeyRanges))
+		}
+		return nil
+	})
 }

tests/run.sh

@@ -41,8 +41,6 @@ fi
 
 echo "selected test cases: $SELECTED_TEST_NAME"
 
-# disable cluster index by default
-run_sql 'set @@global.tidb_enable_clustered_index = 0' || echo "tidb does not support cluster index yet, skipped!"
 # wait for global variable cache invalid
 sleep 2