From c5f6d6bc02d1bdc70dd142aa0ed77c1bd010bd80 Mon Sep 17 00:00:00 2001 From: qw4990 Date: Wed, 6 Mar 2019 13:38:37 +0800 Subject: [PATCH 01/11] add unit test for chunk size control on Limit+TableScan --- executor/chunk_size_control_test.go | 134 ++++++++++++++++++++++++++++ 1 file changed, 134 insertions(+) create mode 100644 executor/chunk_size_control_test.go diff --git a/executor/chunk_size_control_test.go b/executor/chunk_size_control_test.go new file mode 100644 index 0000000000000..38e46cc53391d --- /dev/null +++ b/executor/chunk_size_control_test.go @@ -0,0 +1,134 @@ +package executor_test + +import ( + "context" + "fmt" + "strings" + "time" + + . "github.com/pingcap/check" + "github.com/pingcap/parser/model" + "github.com/pingcap/tidb/domain" + "github.com/pingcap/tidb/kv" + "github.com/pingcap/tidb/session" + "github.com/pingcap/tidb/store/mockstore" + "github.com/pingcap/tidb/store/mockstore/mocktikv" + "github.com/pingcap/tidb/store/tikv" + "github.com/pingcap/tidb/store/tikv/tikvrpc" + "github.com/pingcap/tidb/tablecodec" + "github.com/pingcap/tidb/util/codec" + "github.com/pingcap/tidb/util/testkit" +) + +var ( + _ = Suite(&testChunkSizeControlSuite{}) +) + +type testChunkSizeControlClient struct { + tikv.Client + + regionDelay map[uint64]time.Duration +} + +func (c *testChunkSizeControlClient) SendRequest(ctx context.Context, addr string, req *tikvrpc.Request, timeout time.Duration) (*tikvrpc.Response, error) { + regionID := req.RegionId + if req.Type == tikvrpc.CmdCop && c.regionDelay[regionID] > 0 { + time.Sleep(c.regionDelay[regionID]) + } + return c.Client.SendRequest(ctx, addr, req, timeout) +} + +func (c *testChunkSizeControlClient) SetDelay(regionID uint64, dur time.Duration) { + c.regionDelay[regionID] = dur +} + +func manipulateCluster(cluster *mocktikv.Cluster, region1ID uint64, splitKeys [][]byte) []uint64 { + allRegionIDs := make([]uint64, 0, len(splitKeys)+1) + allRegionIDs = append(allRegionIDs, region1ID) + for i, key := range splitKeys { + newRegionID := cluster.AllocID() + newPeerID := cluster.AllocID() + cluster.Split(allRegionIDs[i], newRegionID, key, []uint64{newPeerID}, newPeerID) + allRegionIDs = append(allRegionIDs, newRegionID) + } + return allRegionIDs +} + +func generateSplitKeyForInt(tid int64, splitNum []int) [][]byte { + results := make([][]byte, 0, len(splitNum)) + for _, num := range splitNum { + results = append(results, tablecodec.EncodeRowKey(tid, codec.EncodeInt(nil, int64(num)))) + } + return results +} + +type testChunkSizeControlSuite struct { + store kv.Storage + dom *domain.Domain +} + +func (s *testChunkSizeControlSuite) SetUpSuite(c *C) { +} + +func (s *testChunkSizeControlSuite) initClusterAndStore( + c *C, cluster *mocktikv.Cluster, client *testChunkSizeControlClient) { + hijackClient := func(c tikv.Client) tikv.Client { + client.Client = c + return client + } + + var err error + s.store, err = mockstore.NewMockTikvStore( + mockstore.WithCluster(cluster), + mockstore.WithHijackClient(hijackClient), + ) + c.Assert(err, IsNil) + + s.dom, err = session.BootstrapSession(s.store) + c.Assert(err, IsNil) +} + +func (s *testChunkSizeControlSuite) TestChunkSizeControl(c *C) { + // init store + client := &testChunkSizeControlClient{regionDelay: make(map[uint64]time.Duration)} + cluster := mocktikv.NewCluster() + _, _, regionID := mocktikv.BootstrapWithSingleStore(cluster) + s.initClusterAndStore(c, cluster, client) + tk := testkit.NewTestKitWithInit(c, s.store) + + // create a test table + tk.MustExec("create table t (a int, primary key (a))") + tbl, err := s.dom.InfoSchema().TableByName(model.NewCIStr("test"), model.NewCIStr("t")) + c.Assert(err, IsNil) + tid := tbl.Meta().ID + + // construct two regions split by 100 + splitKeys := generateSplitKeyForInt(tid, []int{100}) + regionIDs := manipulateCluster(cluster, regionID, splitKeys) + + // insert one record into each regions + tk.MustExec("insert into t values (1), (101)") + + noBlockThreshold := time.Millisecond * 100 + client.SetDelay(regionIDs[0], time.Second) + results := tk.MustQuery("explain analyze select * from t where t.a > 0 and t.a < 200 limit 1") + cost := s.parseTimeCost(c, results.Rows()[0]) + c.Assert(cost, Less, noBlockThreshold) + + results = tk.MustQuery("explain analyze select * from t where t.a > 0 and t.a < 200 limit 2") + cost = s.parseTimeCost(c, results.Rows()[0]) + c.Assert(cost, Not(Less), time.Second) +} + +func (s *testChunkSizeControlSuite) parseTimeCost(c *C, line []interface{}) time.Duration { + lineStr := fmt.Sprintf("%v", line) + idx := strings.Index(lineStr, "time:") + c.Assert(idx, Not(Equals), -1) + lineStr = lineStr[idx+len("time:"):] + idx = strings.Index(lineStr, ",") + c.Assert(idx, Not(Equals), -1) + timeStr := lineStr[:idx] + d, err := time.ParseDuration(timeStr) + c.Assert(err, IsNil) + return d +} From e261dc1fea61b1d1bd6e53349c789ac8cdce3b9c Mon Sep 17 00:00:00 2001 From: qw4990 Date: Wed, 6 Mar 2019 13:51:24 +0800 Subject: [PATCH 02/11] WIP --- executor/chunk_size_control_test.go | 30 +++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/executor/chunk_size_control_test.go b/executor/chunk_size_control_test.go index 38e46cc53391d..1304ed9b3eaf0 100644 --- a/executor/chunk_size_control_test.go +++ b/executor/chunk_size_control_test.go @@ -42,7 +42,13 @@ func (c *testChunkSizeControlClient) SetDelay(regionID uint64, dur time.Duration c.regionDelay[regionID] = dur } -func manipulateCluster(cluster *mocktikv.Cluster, region1ID uint64, splitKeys [][]byte) []uint64 { +func manipulateCluster(cluster *mocktikv.Cluster, splitKeys [][]byte) []uint64 { + regions := cluster.GetAllRegions() + if len(regions) != 1 { + panic("invalid length of regions") + } + region1ID := regions[0].Meta.Id + allRegionIDs := make([]uint64, 0, len(splitKeys)+1) allRegionIDs = append(allRegionIDs, region1ID) for i, key := range splitKeys { @@ -88,23 +94,27 @@ func (s *testChunkSizeControlSuite) initClusterAndStore( c.Assert(err, IsNil) } -func (s *testChunkSizeControlSuite) TestChunkSizeControl(c *C) { +func (s *testChunkSizeControlSuite) initTable(c *C, tableName, tableSQL string) (*testkit.TestKit, *testChunkSizeControlClient, *mocktikv.Cluster, int64) { // init store client := &testChunkSizeControlClient{regionDelay: make(map[uint64]time.Duration)} cluster := mocktikv.NewCluster() - _, _, regionID := mocktikv.BootstrapWithSingleStore(cluster) + mocktikv.BootstrapWithSingleStore(cluster) s.initClusterAndStore(c, cluster, client) tk := testkit.NewTestKitWithInit(c, s.store) - // create a test table - tk.MustExec("create table t (a int, primary key (a))") - tbl, err := s.dom.InfoSchema().TableByName(model.NewCIStr("test"), model.NewCIStr("t")) + // create the test table + tk.MustExec(tableSQL) + tbl, err := s.dom.InfoSchema().TableByName(model.NewCIStr("test"), model.NewCIStr(tableName)) c.Assert(err, IsNil) - tid := tbl.Meta().ID + return tk, client, cluster, tbl.Meta().ID +} + +func (s *testChunkSizeControlSuite) TestLimitAndTableScan(c *C) { + tk, client, cluster, tid := s.initTable(c, "t", "create table t (a int, primary key (a))") // construct two regions split by 100 splitKeys := generateSplitKeyForInt(tid, []int{100}) - regionIDs := manipulateCluster(cluster, regionID, splitKeys) + regionIDs := manipulateCluster(cluster, splitKeys) // insert one record into each regions tk.MustExec("insert into t values (1), (101)") @@ -120,6 +130,10 @@ func (s *testChunkSizeControlSuite) TestChunkSizeControl(c *C) { c.Assert(cost, Not(Less), time.Second) } +func (s *testChunkSizeControlSuite) TestLimitAndIndexScan(c *C) { + +} + func (s *testChunkSizeControlSuite) parseTimeCost(c *C, line []interface{}) time.Duration { lineStr := fmt.Sprintf("%v", line) idx := strings.Index(lineStr, "time:") From a0aaece2b39fabdf03b9d8fb83fed25cc5289f01 Mon Sep 17 00:00:00 2001 From: qw4990 Date: Wed, 6 Mar 2019 15:12:50 +0800 Subject: [PATCH 03/11] add TestLimitAndIndexScan --- executor/chunk_size_control_test.go | 73 ++++++++++++++++++++--------- 1 file changed, 52 insertions(+), 21 deletions(-) diff --git a/executor/chunk_size_control_test.go b/executor/chunk_size_control_test.go index 1304ed9b3eaf0..3e488ba7c2ae2 100644 --- a/executor/chunk_size_control_test.go +++ b/executor/chunk_size_control_test.go @@ -16,6 +16,7 @@ import ( "github.com/pingcap/tidb/store/tikv" "github.com/pingcap/tidb/store/tikv/tikvrpc" "github.com/pingcap/tidb/tablecodec" + "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/codec" "github.com/pingcap/tidb/util/testkit" ) @@ -68,6 +69,20 @@ func generateSplitKeyForInt(tid int64, splitNum []int) [][]byte { return results } +func generateIndexSplitKeyForInt(tid, idx int64, splitNum []int) [][]byte { + results := make([][]byte, 0, len(splitNum)) + for _, num := range splitNum { + d := types.Datum{} + d.SetInt64(int64(num)) + b, err := codec.EncodeKey(nil, nil, d) + if err != nil { + panic(err) + } + results = append(results, tablecodec.EncodeIndexSeekKey(tid, idx, b)) + } + return results +} + type testChunkSizeControlSuite struct { store kv.Storage dom *domain.Domain @@ -76,41 +91,36 @@ type testChunkSizeControlSuite struct { func (s *testChunkSizeControlSuite) SetUpSuite(c *C) { } -func (s *testChunkSizeControlSuite) initClusterAndStore( - c *C, cluster *mocktikv.Cluster, client *testChunkSizeControlClient) { - hijackClient := func(c tikv.Client) tikv.Client { - client.Client = c - return client - } - +func (s *testChunkSizeControlSuite) initTable(c *C, tableSQL string) (*testkit.TestKit, *testChunkSizeControlClient, *mocktikv.Cluster) { + // init store + client := &testChunkSizeControlClient{regionDelay: make(map[uint64]time.Duration)} + cluster := mocktikv.NewCluster() + mocktikv.BootstrapWithSingleStore(cluster) var err error s.store, err = mockstore.NewMockTikvStore( mockstore.WithCluster(cluster), - mockstore.WithHijackClient(hijackClient), + mockstore.WithHijackClient(func(c tikv.Client) tikv.Client { + client.Client = c + return client + }), ) c.Assert(err, IsNil) + // init domain s.dom, err = session.BootstrapSession(s.store) c.Assert(err, IsNil) -} - -func (s *testChunkSizeControlSuite) initTable(c *C, tableName, tableSQL string) (*testkit.TestKit, *testChunkSizeControlClient, *mocktikv.Cluster, int64) { - // init store - client := &testChunkSizeControlClient{regionDelay: make(map[uint64]time.Duration)} - cluster := mocktikv.NewCluster() - mocktikv.BootstrapWithSingleStore(cluster) - s.initClusterAndStore(c, cluster, client) - tk := testkit.NewTestKitWithInit(c, s.store) // create the test table + tk := testkit.NewTestKitWithInit(c, s.store) tk.MustExec(tableSQL) - tbl, err := s.dom.InfoSchema().TableByName(model.NewCIStr("test"), model.NewCIStr(tableName)) - c.Assert(err, IsNil) - return tk, client, cluster, tbl.Meta().ID + return tk, client, cluster } func (s *testChunkSizeControlSuite) TestLimitAndTableScan(c *C) { - tk, client, cluster, tid := s.initTable(c, "t", "create table t (a int, primary key (a))") + tk, client, cluster := s.initTable(c, "create table t (a int, primary key (a))") + tbl, err := s.dom.InfoSchema().TableByName(model.NewCIStr("test"), model.NewCIStr("t")) + c.Assert(err, IsNil) + tid := tbl.Meta().ID // construct two regions split by 100 splitKeys := generateSplitKeyForInt(tid, []int{100}) @@ -131,7 +141,28 @@ func (s *testChunkSizeControlSuite) TestLimitAndTableScan(c *C) { } func (s *testChunkSizeControlSuite) TestLimitAndIndexScan(c *C) { + tk, client, cluster := s.initTable(c, "create table t (a int, index idx_a(a))") + tbl, err := s.dom.InfoSchema().TableByName(model.NewCIStr("test"), model.NewCIStr("t")) + c.Assert(err, IsNil) + tid := tbl.Meta().ID + idx := tbl.Meta().Indices[0].ID + + // construct two regions split by 100 + splitKeys := generateIndexSplitKeyForInt(tid, idx, []int{100}) + regionIDs := manipulateCluster(cluster, splitKeys) + + // insert one record into each regions + tk.MustExec("insert into t values (1), (101)") + noBlockThreshold := time.Millisecond * 100 + client.SetDelay(regionIDs[0], time.Second) + results := tk.MustQuery("explain analyze select * from t where t.a > 0 and t.a < 200 limit 1") + cost := s.parseTimeCost(c, results.Rows()[0]) + c.Assert(cost, Less, noBlockThreshold) + + results = tk.MustQuery("explain analyze select * from t where t.a > 0 and t.a < 200 limit 2") + cost = s.parseTimeCost(c, results.Rows()[0]) + c.Assert(cost, Not(Less), time.Second) } func (s *testChunkSizeControlSuite) parseTimeCost(c *C, line []interface{}) time.Duration { From eb33a46a405b966ee2205433dcdad0cf1b626de0 Mon Sep 17 00:00:00 2001 From: qw4990 Date: Wed, 6 Mar 2019 15:21:49 +0800 Subject: [PATCH 04/11] fixup --- executor/chunk_size_control_test.go | 32 +++++++++++++---------------- 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/executor/chunk_size_control_test.go b/executor/chunk_size_control_test.go index 3e488ba7c2ae2..a7ea7b975bc6b 100644 --- a/executor/chunk_size_control_test.go +++ b/executor/chunk_size_control_test.go @@ -25,13 +25,12 @@ var ( _ = Suite(&testChunkSizeControlSuite{}) ) -type testChunkSizeControlClient struct { +type testSlowClient struct { tikv.Client - regionDelay map[uint64]time.Duration } -func (c *testChunkSizeControlClient) SendRequest(ctx context.Context, addr string, req *tikvrpc.Request, timeout time.Duration) (*tikvrpc.Response, error) { +func (c *testSlowClient) SendRequest(ctx context.Context, addr string, req *tikvrpc.Request, timeout time.Duration) (*tikvrpc.Response, error) { regionID := req.RegionId if req.Type == tikvrpc.CmdCop && c.regionDelay[regionID] > 0 { time.Sleep(c.regionDelay[regionID]) @@ -39,29 +38,27 @@ func (c *testChunkSizeControlClient) SendRequest(ctx context.Context, addr strin return c.Client.SendRequest(ctx, addr, req, timeout) } -func (c *testChunkSizeControlClient) SetDelay(regionID uint64, dur time.Duration) { +func (c *testSlowClient) SetDelay(regionID uint64, dur time.Duration) { c.regionDelay[regionID] = dur } +// manipulateCluster splits this cluster's region by splitKeys and returns regionIDs after split func manipulateCluster(cluster *mocktikv.Cluster, splitKeys [][]byte) []uint64 { regions := cluster.GetAllRegions() if len(regions) != 1 { - panic("invalid length of regions") + panic("this cluster has already split") } - region1ID := regions[0].Meta.Id - allRegionIDs := make([]uint64, 0, len(splitKeys)+1) - allRegionIDs = append(allRegionIDs, region1ID) + allRegionIDs := []uint64{regions[0].Meta.Id} for i, key := range splitKeys { - newRegionID := cluster.AllocID() - newPeerID := cluster.AllocID() + newRegionID, newPeerID := cluster.AllocID(), cluster.AllocID() cluster.Split(allRegionIDs[i], newRegionID, key, []uint64{newPeerID}, newPeerID) allRegionIDs = append(allRegionIDs, newRegionID) } return allRegionIDs } -func generateSplitKeyForInt(tid int64, splitNum []int) [][]byte { +func generateTableSplitKeyForInt(tid int64, splitNum []int) [][]byte { results := make([][]byte, 0, len(splitNum)) for _, num := range splitNum { results = append(results, tablecodec.EncodeRowKey(tid, codec.EncodeInt(nil, int64(num)))) @@ -72,9 +69,9 @@ func generateSplitKeyForInt(tid int64, splitNum []int) [][]byte { func generateIndexSplitKeyForInt(tid, idx int64, splitNum []int) [][]byte { results := make([][]byte, 0, len(splitNum)) for _, num := range splitNum { - d := types.Datum{} + d := new(types.Datum) d.SetInt64(int64(num)) - b, err := codec.EncodeKey(nil, nil, d) + b, err := codec.EncodeKey(nil, nil, *d) if err != nil { panic(err) } @@ -88,12 +85,11 @@ type testChunkSizeControlSuite struct { dom *domain.Domain } -func (s *testChunkSizeControlSuite) SetUpSuite(c *C) { -} +func (s *testChunkSizeControlSuite) SetUpSuite(c *C) {} -func (s *testChunkSizeControlSuite) initTable(c *C, tableSQL string) (*testkit.TestKit, *testChunkSizeControlClient, *mocktikv.Cluster) { +func (s *testChunkSizeControlSuite) initTable(c *C, tableSQL string) (*testkit.TestKit, *testSlowClient, *mocktikv.Cluster) { // init store - client := &testChunkSizeControlClient{regionDelay: make(map[uint64]time.Duration)} + client := &testSlowClient{regionDelay: make(map[uint64]time.Duration)} cluster := mocktikv.NewCluster() mocktikv.BootstrapWithSingleStore(cluster) var err error @@ -123,7 +119,7 @@ func (s *testChunkSizeControlSuite) TestLimitAndTableScan(c *C) { tid := tbl.Meta().ID // construct two regions split by 100 - splitKeys := generateSplitKeyForInt(tid, []int{100}) + splitKeys := generateTableSplitKeyForInt(tid, []int{100}) regionIDs := manipulateCluster(cluster, splitKeys) // insert one record into each regions From c2b15ad694ed110215a8cc30b582d6e3b7cdcc17 Mon Sep 17 00:00:00 2001 From: qw4990 Date: Wed, 6 Mar 2019 15:34:03 +0800 Subject: [PATCH 05/11] add some comments --- executor/chunk_size_control_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/executor/chunk_size_control_test.go b/executor/chunk_size_control_test.go index a7ea7b975bc6b..034726c405518 100644 --- a/executor/chunk_size_control_test.go +++ b/executor/chunk_size_control_test.go @@ -129,11 +129,11 @@ func (s *testChunkSizeControlSuite) TestLimitAndTableScan(c *C) { client.SetDelay(regionIDs[0], time.Second) results := tk.MustQuery("explain analyze select * from t where t.a > 0 and t.a < 200 limit 1") cost := s.parseTimeCost(c, results.Rows()[0]) - c.Assert(cost, Less, noBlockThreshold) + c.Assert(cost, Less, noBlockThreshold) // finish quickly since there is one record in region1 results = tk.MustQuery("explain analyze select * from t where t.a > 0 and t.a < 200 limit 2") cost = s.parseTimeCost(c, results.Rows()[0]) - c.Assert(cost, Not(Less), time.Second) + c.Assert(cost, Not(Less), time.Second) // have to wait for the record returned by region2 } func (s *testChunkSizeControlSuite) TestLimitAndIndexScan(c *C) { @@ -154,11 +154,11 @@ func (s *testChunkSizeControlSuite) TestLimitAndIndexScan(c *C) { client.SetDelay(regionIDs[0], time.Second) results := tk.MustQuery("explain analyze select * from t where t.a > 0 and t.a < 200 limit 1") cost := s.parseTimeCost(c, results.Rows()[0]) - c.Assert(cost, Less, noBlockThreshold) + c.Assert(cost, Less, noBlockThreshold) // finish quickly since there is one record in region1 results = tk.MustQuery("explain analyze select * from t where t.a > 0 and t.a < 200 limit 2") cost = s.parseTimeCost(c, results.Rows()[0]) - c.Assert(cost, Not(Less), time.Second) + c.Assert(cost, Not(Less), time.Second) // have to wait for the record returned by region2 } func (s *testChunkSizeControlSuite) parseTimeCost(c *C, line []interface{}) time.Duration { From ac57b3dcf741121af09e8e1c8a597818b2ce8a92 Mon Sep 17 00:00:00 2001 From: qw4990 Date: Wed, 6 Mar 2019 15:43:43 +0800 Subject: [PATCH 06/11] fixup --- executor/chunk_size_control_test.go | 34 +++++++++++++++++++---------- 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/executor/chunk_size_control_test.go b/executor/chunk_size_control_test.go index 034726c405518..ad374c5e13303 100644 --- a/executor/chunk_size_control_test.go +++ b/executor/chunk_size_control_test.go @@ -122,18 +122,23 @@ func (s *testChunkSizeControlSuite) TestLimitAndTableScan(c *C) { splitKeys := generateTableSplitKeyForInt(tid, []int{100}) regionIDs := manipulateCluster(cluster, splitKeys) - // insert one record into each regions - tk.MustExec("insert into t values (1), (101)") + noDelayThreshold := time.Millisecond * 100 + delayThreshold := time.Second + tk.MustExec("insert into t values (1)") // insert one record into region1, and set a delay duration + client.SetDelay(regionIDs[0], delayThreshold) - noBlockThreshold := time.Millisecond * 100 - client.SetDelay(regionIDs[0], time.Second) results := tk.MustQuery("explain analyze select * from t where t.a > 0 and t.a < 200 limit 1") cost := s.parseTimeCost(c, results.Rows()[0]) - c.Assert(cost, Less, noBlockThreshold) // finish quickly since there is one record in region1 + c.Assert(cost, Not(Less), delayThreshold) // have to wait for region1 + + tk.MustExec("insert into t values (101)") // insert one record into region2 + results = tk.MustQuery("explain analyze select * from t where t.a > 0 and t.a < 200 limit 1") + cost = s.parseTimeCost(c, results.Rows()[0]) + c.Assert(cost, Less, noDelayThreshold) // region2 return quickly results = tk.MustQuery("explain analyze select * from t where t.a > 0 and t.a < 200 limit 2") cost = s.parseTimeCost(c, results.Rows()[0]) - c.Assert(cost, Not(Less), time.Second) // have to wait for the record returned by region2 + c.Assert(cost, Not(Less), delayThreshold) // have to wait } func (s *testChunkSizeControlSuite) TestLimitAndIndexScan(c *C) { @@ -147,18 +152,23 @@ func (s *testChunkSizeControlSuite) TestLimitAndIndexScan(c *C) { splitKeys := generateIndexSplitKeyForInt(tid, idx, []int{100}) regionIDs := manipulateCluster(cluster, splitKeys) - // insert one record into each regions - tk.MustExec("insert into t values (1), (101)") + noDelayThreshold := time.Millisecond * 100 + delayThreshold := time.Second + tk.MustExec("insert into t values (1)") // insert one record into region1, and set a delay duration + client.SetDelay(regionIDs[0], delayThreshold) - noBlockThreshold := time.Millisecond * 100 - client.SetDelay(regionIDs[0], time.Second) results := tk.MustQuery("explain analyze select * from t where t.a > 0 and t.a < 200 limit 1") cost := s.parseTimeCost(c, results.Rows()[0]) - c.Assert(cost, Less, noBlockThreshold) // finish quickly since there is one record in region1 + c.Assert(cost, Not(Less), delayThreshold) // have to wait for region1 + + tk.MustExec("insert into t values (101)") // insert one record into region2 + results = tk.MustQuery("explain analyze select * from t where t.a > 0 and t.a < 200 limit 1") + cost = s.parseTimeCost(c, results.Rows()[0]) + c.Assert(cost, Less, noDelayThreshold) // region2 return quickly results = tk.MustQuery("explain analyze select * from t where t.a > 0 and t.a < 200 limit 2") cost = s.parseTimeCost(c, results.Rows()[0]) - c.Assert(cost, Not(Less), time.Second) // have to wait for the record returned by region2 + c.Assert(cost, Not(Less), delayThreshold) // have to wait } func (s *testChunkSizeControlSuite) parseTimeCost(c *C, line []interface{}) time.Duration { From a9c5366846c9b04a700d3080126d696556c2c4d0 Mon Sep 17 00:00:00 2001 From: qw4990 Date: Thu, 7 Mar 2019 11:48:58 +0800 Subject: [PATCH 07/11] add license --- executor/chunk_size_control_test.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/executor/chunk_size_control_test.go b/executor/chunk_size_control_test.go index ad374c5e13303..6f4fe5e320b9a 100644 --- a/executor/chunk_size_control_test.go +++ b/executor/chunk_size_control_test.go @@ -1,3 +1,16 @@ +// Copyright 2015 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + package executor_test import ( From 3ff577c3e4aea73ee577b63c54c5e2ae19ee882e Mon Sep 17 00:00:00 2001 From: qw4990 Date: Thu, 7 Mar 2019 11:52:05 +0800 Subject: [PATCH 08/11] update license --- executor/chunk_size_control_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/executor/chunk_size_control_test.go b/executor/chunk_size_control_test.go index 6f4fe5e320b9a..a0c293e2a8c48 100644 --- a/executor/chunk_size_control_test.go +++ b/executor/chunk_size_control_test.go @@ -1,4 +1,4 @@ -// Copyright 2015 PingCAP, Inc. +// Copyright 2019 PingCAP, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. From eb8865704393f3569f60a67fb8b5dcd1190d59e4 Mon Sep 17 00:00:00 2001 From: qw4990 Date: Thu, 7 Mar 2019 13:00:33 +0800 Subject: [PATCH 09/11] fix CI problem --- executor/chunk_size_control_test.go | 31 ++++++++++++++++------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/executor/chunk_size_control_test.go b/executor/chunk_size_control_test.go index a0c293e2a8c48..5e257e87c3551 100644 --- a/executor/chunk_size_control_test.go +++ b/executor/chunk_size_control_test.go @@ -93,20 +93,17 @@ func generateIndexSplitKeyForInt(tid, idx int64, splitNum []int) [][]byte { return results } -type testChunkSizeControlSuite struct { - store kv.Storage - dom *domain.Domain -} +type testChunkSizeControlSuite struct{} func (s *testChunkSizeControlSuite) SetUpSuite(c *C) {} -func (s *testChunkSizeControlSuite) initTable(c *C, tableSQL string) (*testkit.TestKit, *testSlowClient, *mocktikv.Cluster) { +func (s *testChunkSizeControlSuite) initTable(c *C, tableSQL string) ( + kv.Storage, *domain.Domain, *testkit.TestKit, *testSlowClient, *mocktikv.Cluster) { // init store client := &testSlowClient{regionDelay: make(map[uint64]time.Duration)} cluster := mocktikv.NewCluster() mocktikv.BootstrapWithSingleStore(cluster) - var err error - s.store, err = mockstore.NewMockTikvStore( + store, err := mockstore.NewMockTikvStore( mockstore.WithCluster(cluster), mockstore.WithHijackClient(func(c tikv.Client) tikv.Client { client.Client = c @@ -116,18 +113,18 @@ func (s *testChunkSizeControlSuite) initTable(c *C, tableSQL string) (*testkit.T c.Assert(err, IsNil) // init domain - s.dom, err = session.BootstrapSession(s.store) + dom, err := session.BootstrapSession(store) c.Assert(err, IsNil) // create the test table - tk := testkit.NewTestKitWithInit(c, s.store) + tk := testkit.NewTestKitWithInit(c, store) tk.MustExec(tableSQL) - return tk, client, cluster + return store, dom, tk, client, cluster } func (s *testChunkSizeControlSuite) TestLimitAndTableScan(c *C) { - tk, client, cluster := s.initTable(c, "create table t (a int, primary key (a))") - tbl, err := s.dom.InfoSchema().TableByName(model.NewCIStr("test"), model.NewCIStr("t")) + store, dom, tk, client, cluster := s.initTable(c, "create table t (a int, primary key (a))") + tbl, err := dom.InfoSchema().TableByName(model.NewCIStr("test"), model.NewCIStr("t")) c.Assert(err, IsNil) tid := tbl.Meta().ID @@ -152,11 +149,14 @@ func (s *testChunkSizeControlSuite) TestLimitAndTableScan(c *C) { results = tk.MustQuery("explain analyze select * from t where t.a > 0 and t.a < 200 limit 2") cost = s.parseTimeCost(c, results.Rows()[0]) c.Assert(cost, Not(Less), delayThreshold) // have to wait + + dom.Close() + store.Close() } func (s *testChunkSizeControlSuite) TestLimitAndIndexScan(c *C) { - tk, client, cluster := s.initTable(c, "create table t (a int, index idx_a(a))") - tbl, err := s.dom.InfoSchema().TableByName(model.NewCIStr("test"), model.NewCIStr("t")) + store, dom, tk, client, cluster := s.initTable(c, "create table t (a int, index idx_a(a))") + tbl, err := dom.InfoSchema().TableByName(model.NewCIStr("test"), model.NewCIStr("t")) c.Assert(err, IsNil) tid := tbl.Meta().ID idx := tbl.Meta().Indices[0].ID @@ -182,6 +182,9 @@ func (s *testChunkSizeControlSuite) TestLimitAndIndexScan(c *C) { results = tk.MustQuery("explain analyze select * from t where t.a > 0 and t.a < 200 limit 2") cost = s.parseTimeCost(c, results.Rows()[0]) c.Assert(cost, Not(Less), delayThreshold) // have to wait + + dom.Close() + store.Close() } func (s *testChunkSizeControlSuite) parseTimeCost(c *C, line []interface{}) time.Duration { From a3d80e026548e3e0a0b1a6c39bb450cc7a85f896 Mon Sep 17 00:00:00 2001 From: qw4990 Date: Thu, 7 Mar 2019 13:35:03 +0800 Subject: [PATCH 10/11] fix CI problems --- executor/chunk_size_control_test.go | 80 +++++++++++++++++------------ 1 file changed, 48 insertions(+), 32 deletions(-) diff --git a/executor/chunk_size_control_test.go b/executor/chunk_size_control_test.go index 5e257e87c3551..d503b0d8baeef 100644 --- a/executor/chunk_size_control_test.go +++ b/executor/chunk_size_control_test.go @@ -93,37 +93,59 @@ func generateIndexSplitKeyForInt(tid, idx int64, splitNum []int) [][]byte { return results } -type testChunkSizeControlSuite struct{} - -func (s *testChunkSizeControlSuite) SetUpSuite(c *C) {} +type testChunkSizeControlKit struct { + store kv.Storage + dom *domain.Domain + tk *testkit.TestKit + client *testSlowClient + cluster *mocktikv.Cluster +} -func (s *testChunkSizeControlSuite) initTable(c *C, tableSQL string) ( - kv.Storage, *domain.Domain, *testkit.TestKit, *testSlowClient, *mocktikv.Cluster) { - // init store - client := &testSlowClient{regionDelay: make(map[uint64]time.Duration)} - cluster := mocktikv.NewCluster() - mocktikv.BootstrapWithSingleStore(cluster) - store, err := mockstore.NewMockTikvStore( - mockstore.WithCluster(cluster), - mockstore.WithHijackClient(func(c tikv.Client) tikv.Client { - client.Client = c - return client - }), - ) - c.Assert(err, IsNil) +type testChunkSizeControlSuite struct { + m map[string]*testChunkSizeControlKit +} - // init domain - dom, err := session.BootstrapSession(store) - c.Assert(err, IsNil) +func (s *testChunkSizeControlSuite) SetUpSuite(c *C) { + tableSQLs := map[string]string{} + tableSQLs["Limit&TableScan"] = "create table t (a int, primary key (a))" + tableSQLs["Limit&IndexScan"] = "create table t (a int, index idx_a(a))" + + s.m = make(map[string]*testChunkSizeControlKit) + for name, sql := range tableSQLs { + kit := new(testChunkSizeControlKit) + s.m[name] = kit + kit.client = &testSlowClient{regionDelay: make(map[uint64]time.Duration)} + kit.cluster = mocktikv.NewCluster() + mocktikv.BootstrapWithSingleStore(kit.cluster) + + var err error + kit.store, err = mockstore.NewMockTikvStore( + mockstore.WithCluster(kit.cluster), + mockstore.WithHijackClient(func(c tikv.Client) tikv.Client { + kit.client.Client = c + return kit.client + }), + ) + c.Assert(err, IsNil) + + // init domain + kit.dom, err = session.BootstrapSession(kit.store) + c.Assert(err, IsNil) + + // create the test table + kit.tk = testkit.NewTestKitWithInit(c, kit.store) + kit.tk.MustExec(sql) + } +} - // create the test table - tk := testkit.NewTestKitWithInit(c, store) - tk.MustExec(tableSQL) - return store, dom, tk, client, cluster +func (s *testChunkSizeControlSuite) getKit(name string) ( + kv.Storage, *domain.Domain, *testkit.TestKit, *testSlowClient, *mocktikv.Cluster) { + x := s.m[name] + return x.store, x.dom, x.tk, x.client, x.cluster } func (s *testChunkSizeControlSuite) TestLimitAndTableScan(c *C) { - store, dom, tk, client, cluster := s.initTable(c, "create table t (a int, primary key (a))") + _, dom, tk, client, cluster := s.getKit("Limit&TableScan") tbl, err := dom.InfoSchema().TableByName(model.NewCIStr("test"), model.NewCIStr("t")) c.Assert(err, IsNil) tid := tbl.Meta().ID @@ -149,13 +171,10 @@ func (s *testChunkSizeControlSuite) TestLimitAndTableScan(c *C) { results = tk.MustQuery("explain analyze select * from t where t.a > 0 and t.a < 200 limit 2") cost = s.parseTimeCost(c, results.Rows()[0]) c.Assert(cost, Not(Less), delayThreshold) // have to wait - - dom.Close() - store.Close() } func (s *testChunkSizeControlSuite) TestLimitAndIndexScan(c *C) { - store, dom, tk, client, cluster := s.initTable(c, "create table t (a int, index idx_a(a))") + _, dom, tk, client, cluster := s.getKit("Limit&IndexScan") tbl, err := dom.InfoSchema().TableByName(model.NewCIStr("test"), model.NewCIStr("t")) c.Assert(err, IsNil) tid := tbl.Meta().ID @@ -182,9 +201,6 @@ func (s *testChunkSizeControlSuite) TestLimitAndIndexScan(c *C) { results = tk.MustQuery("explain analyze select * from t where t.a > 0 and t.a < 200 limit 2") cost = s.parseTimeCost(c, results.Rows()[0]) c.Assert(cost, Not(Less), delayThreshold) // have to wait - - dom.Close() - store.Close() } func (s *testChunkSizeControlSuite) parseTimeCost(c *C, line []interface{}) time.Duration { From 96846c0b5db59d8523589e7261dbcdf90d975e65 Mon Sep 17 00:00:00 2001 From: qw4990 Date: Thu, 7 Mar 2019 14:33:36 +0800 Subject: [PATCH 11/11] update delayThreshold --- executor/chunk_size_control_test.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/executor/chunk_size_control_test.go b/executor/chunk_size_control_test.go index d503b0d8baeef..6b9861e85059f 100644 --- a/executor/chunk_size_control_test.go +++ b/executor/chunk_size_control_test.go @@ -112,6 +112,7 @@ func (s *testChunkSizeControlSuite) SetUpSuite(c *C) { s.m = make(map[string]*testChunkSizeControlKit) for name, sql := range tableSQLs { + // BootstrapSession is not thread-safe, so we have to prepare all resources in SetUp. kit := new(testChunkSizeControlKit) s.m[name] = kit kit.client = &testSlowClient{regionDelay: make(map[uint64]time.Duration)} @@ -155,9 +156,10 @@ func (s *testChunkSizeControlSuite) TestLimitAndTableScan(c *C) { regionIDs := manipulateCluster(cluster, splitKeys) noDelayThreshold := time.Millisecond * 100 - delayThreshold := time.Second + delayDuration := time.Second + delayThreshold := delayDuration * 9 / 10 tk.MustExec("insert into t values (1)") // insert one record into region1, and set a delay duration - client.SetDelay(regionIDs[0], delayThreshold) + client.SetDelay(regionIDs[0], delayDuration) results := tk.MustQuery("explain analyze select * from t where t.a > 0 and t.a < 200 limit 1") cost := s.parseTimeCost(c, results.Rows()[0]) @@ -185,9 +187,10 @@ func (s *testChunkSizeControlSuite) TestLimitAndIndexScan(c *C) { regionIDs := manipulateCluster(cluster, splitKeys) noDelayThreshold := time.Millisecond * 100 - delayThreshold := time.Second + delayDuration := time.Second + delayThreshold := delayDuration * 9 / 10 tk.MustExec("insert into t values (1)") // insert one record into region1, and set a delay duration - client.SetDelay(regionIDs[0], delayThreshold) + client.SetDelay(regionIDs[0], delayDuration) results := tk.MustQuery("explain analyze select * from t where t.a > 0 and t.a < 200 limit 1") cost := s.parseTimeCost(c, results.Rows()[0])