From a1ce222b5e696b8d98dc7ced94f3c7b62666b6c2 Mon Sep 17 00:00:00 2001 From: tison Date: Sun, 23 Jan 2022 13:19:46 +0800 Subject: [PATCH] br/pkg/backup: migrate test-infra to testify (#31288) close pingcap/tidb#28164 --- pkg/backup/client_test.go | 251 +++++++++++++++++++------------------- pkg/backup/main_test.go | 37 ++++++ pkg/backup/schema_test.go | 236 +++++++++++++++++------------------ pkg/pdutil/pd.go | 4 +- 4 files changed, 287 insertions(+), 241 deletions(-) create mode 100644 pkg/backup/main_test.go diff --git a/pkg/backup/client_test.go b/pkg/backup/client_test.go index e341f1541..e4799b319 100644 --- a/pkg/backup/client_test.go +++ b/pkg/backup/client_test.go @@ -10,7 +10,6 @@ import ( "time" "github.com/golang/protobuf/proto" - . "github.com/pingcap/check" backuppb "github.com/pingcap/kvproto/pkg/brpb" "github.com/pingcap/kvproto/pkg/encryptionpb" "github.com/pingcap/kvproto/pkg/errorpb" @@ -23,9 +22,10 @@ import ( "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/parser/model" "github.com/pingcap/tidb/tablecodec" + "github.com/pingcap/tidb/testkit" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/codec" - "github.com/pingcap/tidb/util/testkit" + "github.com/stretchr/testify/require" "github.com/tikv/client-go/v2/oracle" "github.com/tikv/client-go/v2/testutils" "github.com/tikv/client-go/v2/tikv" @@ -44,92 +44,88 @@ type testBackup struct { storage storage.ExternalStorage } -var _ = Suite(&testBackup{}) - -func TestT(t *testing.T) { - TestingT(t) -} - -func (r *testBackup) SetUpSuite(c *C) { +func createBackupSuite(t *testing.T) (s *testBackup, clean func()) { _, _, pdClient, err := testutils.NewMockTiKV("", nil) - c.Assert(err, IsNil) - r.mockPDClient = pdClient - r.ctx, r.cancel = context.WithCancel(context.Background()) + require.NoError(t, err) + s = new(testBackup) + s.mockPDClient = pdClient + s.ctx, s.cancel = context.WithCancel(context.Background()) mockMgr := &conn.Mgr{PdController: &pdutil.PdController{}} - mockMgr.SetPDClient(r.mockPDClient) + mockMgr.SetPDClient(s.mockPDClient) mockMgr.SetHTTP([]string{"test"}, nil) - r.backupClient, err = backup.NewBackupClient(r.ctx, mockMgr) - c.Assert(err, IsNil) - - r.cluster, err = mock.NewCluster() - c.Assert(err, IsNil) - base := c.MkDir() - r.storage, err = storage.NewLocalStorage(base) - c.Assert(err, IsNil) - //c.Assert(r.cluster.Start(), IsNil) - + s.backupClient, err = backup.NewBackupClient(s.ctx, mockMgr) + require.NoError(t, err) + + s.cluster, err = mock.NewCluster() + require.NoError(t, err) + base := t.TempDir() + s.storage, err = storage.NewLocalStorage(base) + require.NoError(t, err) + require.NoError(t, s.cluster.Start()) + + clean = func() { + mockMgr.Close() + s.cluster.Stop() + } + return } -func (r *testBackup) resetStorage(c *C) { - var err error - base := c.MkDir() - r.storage, err = storage.NewLocalStorage(base) - c.Assert(err, IsNil) -} +func TestGetTS(t *testing.T) { + s, clean := createBackupSuite(t) + defer clean() -func (r *testBackup) TestGetTS(c *C) { - var ( - err error - // mockPDClient' physical ts and current ts will have deviation - // so make this deviation tolerance 100ms - deviation = 100 - ) + // mockPDClient' physical ts and current ts will have deviation + // so make this deviation tolerance 100ms + deviation := 100 // timeago not work expectedDuration := 0 currentTS := time.Now().UnixNano() / int64(time.Millisecond) - ts, err := r.backupClient.GetTS(r.ctx, 0, 0) - c.Assert(err, IsNil) + ts, err := s.backupClient.GetTS(s.ctx, 0, 0) + require.NoError(t, err) pdTS := oracle.ExtractPhysical(ts) duration := int(currentTS - pdTS) - c.Assert(duration, Greater, expectedDuration-deviation) - c.Assert(duration, Less, expectedDuration+deviation) + require.Greater(t, duration, expectedDuration-deviation) + require.Less(t, duration, expectedDuration+deviation) // timeago = "1.5m" expectedDuration = 90000 currentTS = time.Now().UnixNano() / int64(time.Millisecond) - ts, err = r.backupClient.GetTS(r.ctx, 90*time.Second, 0) - c.Assert(err, IsNil) + ts, err = s.backupClient.GetTS(s.ctx, 90*time.Second, 0) + require.NoError(t, err) pdTS = oracle.ExtractPhysical(ts) duration = int(currentTS - pdTS) - c.Assert(duration, Greater, expectedDuration-deviation) - c.Assert(duration, Less, expectedDuration+deviation) + require.Greater(t, duration, expectedDuration-deviation) + require.Less(t, duration, expectedDuration+deviation) // timeago = "-1m" - _, err = r.backupClient.GetTS(r.ctx, -time.Minute, 0) - c.Assert(err, ErrorMatches, "negative timeago is not allowed.*") + _, err = s.backupClient.GetTS(s.ctx, -time.Minute, 0) + require.Error(t, err) + require.Regexp(t, "negative timeago is not allowed.*", err.Error()) // timeago = "1000000h" overflows - _, err = r.backupClient.GetTS(r.ctx, 1000000*time.Hour, 0) - c.Assert(err, ErrorMatches, ".*backup ts overflow.*") + _, err = s.backupClient.GetTS(s.ctx, 1000000*time.Hour, 0) + require.Error(t, err) + require.Regexp(t, ".*backup ts overflow.*", err.Error()) // timeago = "10h" exceed GCSafePoint - p, l, err := r.mockPDClient.GetTS(r.ctx) - c.Assert(err, IsNil) + p, l, err := s.mockPDClient.GetTS(s.ctx) + require.NoError(t, err) now := oracle.ComposeTS(p, l) - _, err = r.mockPDClient.UpdateGCSafePoint(r.ctx, now) - c.Assert(err, IsNil) - _, err = r.backupClient.GetTS(r.ctx, 10*time.Hour, 0) - c.Assert(err, ErrorMatches, ".*GC safepoint [0-9]+ exceed TS [0-9]+.*") + _, err = s.mockPDClient.UpdateGCSafePoint(s.ctx, now) + require.NoError(t, err) + _, err = s.backupClient.GetTS(s.ctx, 10*time.Hour, 0) + require.Error(t, err) + require.Regexp(t, ".*GC safepoint [0-9]+ exceed TS [0-9]+.*", err.Error()) // timeago and backupts both exists, use backupts backupts := oracle.ComposeTS(p+10, l) - ts, err = r.backupClient.GetTS(r.ctx, time.Minute, backupts) - c.Assert(err, IsNil) - c.Assert(ts, Equals, backupts) + ts, err = s.backupClient.GetTS(s.ctx, time.Minute, backupts) + require.NoError(t, err) + require.Equal(t, backupts, ts) } -func (r *testBackup) TestBuildTableRangeIntHandle(c *C) { +func TestBuildTableRangeIntHandle(t *testing.T) { type Case struct { ids []int64 trs []kv.KeyRange @@ -151,34 +147,34 @@ func (r *testBackup) TestBuildTableRangeIntHandle(c *C) { }}, } for _, cs := range cases { - c.Log(cs) + t.Log(cs) tbl := &model.TableInfo{Partition: &model.PartitionInfo{Enable: true}} for _, id := range cs.ids { tbl.Partition.Definitions = append(tbl.Partition.Definitions, model.PartitionDefinition{ID: id}) } ranges, err := backup.BuildTableRanges(tbl) - c.Assert(err, IsNil) - c.Assert(ranges, DeepEquals, cs.trs) + require.NoError(t, err) + require.Equal(t, cs.trs, ranges) } tbl := &model.TableInfo{ID: 7} ranges, err := backup.BuildTableRanges(tbl) - c.Assert(err, IsNil) - c.Assert(ranges, DeepEquals, []kv.KeyRange{ + require.NoError(t, err) + require.Equal(t, []kv.KeyRange{ {StartKey: tablecodec.EncodeRowKey(7, low), EndKey: tablecodec.EncodeRowKey(7, high)}, - }) + }, ranges) } -func (r *testBackup) TestBuildTableRangeCommonHandle(c *C) { +func TestBuildTableRangeCommonHandle(t *testing.T) { type Case struct { ids []int64 trs []kv.KeyRange } low, err_l := codec.EncodeKey(nil, nil, []types.Datum{types.MinNotNullDatum()}...) - c.Assert(err_l, IsNil) + require.NoError(t, err_l) high, err_h := codec.EncodeKey(nil, nil, []types.Datum{types.MaxValueDatum()}...) - c.Assert(err_h, IsNil) + require.NoError(t, err_h) high = kv.Key(high).PrefixNext() cases := []Case{ {ids: []int64{1}, trs: []kv.KeyRange{ @@ -195,26 +191,26 @@ func (r *testBackup) TestBuildTableRangeCommonHandle(c *C) { }}, } for _, cs := range cases { - c.Log(cs) + t.Log(cs) tbl := &model.TableInfo{Partition: &model.PartitionInfo{Enable: true}, IsCommonHandle: true} for _, id := range cs.ids { tbl.Partition.Definitions = append(tbl.Partition.Definitions, model.PartitionDefinition{ID: id}) } ranges, err := backup.BuildTableRanges(tbl) - c.Assert(err, IsNil) - c.Assert(ranges, DeepEquals, cs.trs) + require.NoError(t, err) + require.Equal(t, cs.trs, ranges) } tbl := &model.TableInfo{ID: 7, IsCommonHandle: true} ranges, err_r := backup.BuildTableRanges(tbl) - c.Assert(err_r, IsNil) - c.Assert(ranges, DeepEquals, []kv.KeyRange{ + require.NoError(t, err_r) + require.Equal(t, []kv.KeyRange{ {StartKey: tablecodec.EncodeRowKey(7, low), EndKey: tablecodec.EncodeRowKey(7, high)}, - }) + }, ranges) } -func (r *testBackup) TestOnBackupRegionErrorResponse(c *C) { +func TestOnBackupRegionErrorResponse(t *testing.T) { type Case struct { storeID uint64 bo *tikv.Backoffer @@ -241,18 +237,21 @@ func (r *testBackup) TestOnBackupRegionErrorResponse(c *C) { {storeID: 1, backupTS: 421123291611137, resp: newBackupRegionErrorResp(&errorpb.Error{ProposalInMergingMode: &errorpb.ProposalInMergingMode{}}), exceptedBackoffMs: 1000, exceptedErr: false}, } for _, cs := range cases { - c.Log(cs) + t.Log(cs) _, backoffMs, err := backup.OnBackupResponse(cs.storeID, cs.bo, cs.backupTS, cs.lockResolver, cs.resp) - c.Assert(backoffMs, Equals, cs.exceptedBackoffMs) + require.Equal(t, cs.exceptedBackoffMs, backoffMs) if cs.exceptedErr { - c.Assert(err, NotNil) + require.Error(t, err) } else { - c.Assert(err, IsNil) + require.NoError(t, err) } } } -func (r *testBackup) TestSendCreds(c *C) { +func TestSendCreds(t *testing.T) { + s, clean := createBackupSuite(t) + defer clean() + accessKey := "ab" secretAccessKey := "cd" backendOpt := storage.BackendOptions{ @@ -262,16 +261,16 @@ func (r *testBackup) TestSendCreds(c *C) { }, } backend, err := storage.ParseBackend("s3://bucket/prefix/", &backendOpt) - c.Assert(err, IsNil) + require.NoError(t, err) opts := &storage.ExternalStorageOptions{ SendCredentials: true, } - _, err = storage.New(r.ctx, backend, opts) - c.Assert(err, IsNil) + _, err = storage.New(s.ctx, backend, opts) + require.NoError(t, err) access_key := backend.GetS3().AccessKey - c.Assert(access_key, Equals, "ab") + require.Equal(t, "ab", access_key) secret_access_key := backend.GetS3().SecretAccessKey - c.Assert(secret_access_key, Equals, "cd") + require.Equal(t, "cd", secret_access_key) backendOpt = storage.BackendOptions{ S3: storage.S3BackendOptions{ @@ -280,24 +279,27 @@ func (r *testBackup) TestSendCreds(c *C) { }, } backend, err = storage.ParseBackend("s3://bucket/prefix/", &backendOpt) - c.Assert(err, IsNil) + require.NoError(t, err) opts = &storage.ExternalStorageOptions{ SendCredentials: false, } - _, err = storage.New(r.ctx, backend, opts) - c.Assert(err, IsNil) + _, err = storage.New(s.ctx, backend, opts) + require.NoError(t, err) access_key = backend.GetS3().AccessKey - c.Assert(access_key, Equals, "") + require.Equal(t, "", access_key) secret_access_key = backend.GetS3().SecretAccessKey - c.Assert(secret_access_key, Equals, "") + require.Equal(t, "", secret_access_key) } -func (r *testBackup) TestskipUnsupportedDDLJob(c *C) { - tk := testkit.NewTestKit(c, r.cluster.Storage) +func TestSkipUnsupportedDDLJob(t *testing.T) { + s, clean := createBackupSuite(t) + defer clean() + + tk := testkit.NewTestKit(t, s.cluster.Storage) tk.MustExec("CREATE DATABASE IF NOT EXISTS test_db;") tk.MustExec("CREATE TABLE IF NOT EXISTS test_db.test_table (c1 INT);") - lastTS, err := r.cluster.GetOracle().GetTimestamp(context.Background(), &oracle.Option{TxnScope: oracle.GlobalTxnScope}) - c.Assert(err, IsNil, Commentf("Error get last ts: %s", err)) + lastTS, err := s.cluster.GetOracle().GetTimestamp(context.Background(), &oracle.Option{TxnScope: oracle.GlobalTxnScope}) + require.NoErrorf(t, err, "Error get last ts: %s", err) tk.MustExec("RENAME TABLE test_db.test_table to test_db.test_table1;") tk.MustExec("DROP TABLE test_db.test_table1;") tk.MustExec("DROP DATABASE test_db;") @@ -312,58 +314,61 @@ func (r *testBackup) TestskipUnsupportedDDLJob(c *C) { tk.MustExec("ALTER TABLE tb attributes \"merge_option=allow\"") tk.MustExec("ALTER TABLE tb PARTITION p0 attributes \"merge_option=deny\"") - ts, err := r.cluster.GetOracle().GetTimestamp(context.Background(), &oracle.Option{TxnScope: oracle.GlobalTxnScope}) - c.Assert(err, IsNil, Commentf("Error get ts: %s", err)) + ts, err := s.cluster.GetOracle().GetTimestamp(context.Background(), &oracle.Option{TxnScope: oracle.GlobalTxnScope}) + require.NoErrorf(t, err, "Error get ts: %s", err) cipher := backuppb.CipherInfo{CipherType: encryptionpb.EncryptionMethod_PLAINTEXT} - metaWriter := metautil.NewMetaWriter(r.storage, metautil.MetaFileSize, false, &cipher) + metaWriter := metautil.NewMetaWriter(s.storage, metautil.MetaFileSize, false, &cipher) ctx := context.Background() metaWriter.StartWriteMetasAsync(ctx, metautil.AppendDDL) - err = backup.WriteBackupDDLJobs(metaWriter, r.cluster.Storage, lastTS, ts) - c.Assert(err, IsNil, Commentf("Error get ddl jobs: %s", err)) + err = backup.WriteBackupDDLJobs(metaWriter, s.cluster.Storage, lastTS, ts) + require.NoErrorf(t, err, "Error get ddl jobs: %s", err) err = metaWriter.FinishWriteMetas(ctx, metautil.AppendDDL) - c.Assert(err, IsNil, Commentf("Flush failed", err)) + require.NoError(t, err, "Flush failed", err) err = metaWriter.FlushBackupMeta(ctx) - c.Assert(err, IsNil, Commentf("Finially flush backupmeta failed", err)) + require.NoError(t, err, "Finally flush backup meta failed", err) - metaBytes, err := r.storage.ReadFile(ctx, metautil.MetaFile) - c.Assert(err, IsNil) + metaBytes, err := s.storage.ReadFile(ctx, metautil.MetaFile) + require.NoError(t, err) mockMeta := &backuppb.BackupMeta{} err = proto.Unmarshal(metaBytes, mockMeta) - c.Assert(err, IsNil) + require.NoError(t, err) // check the schema version - metaReader := metautil.NewMetaReader(mockMeta, r.storage, &cipher) + metaReader := metautil.NewMetaReader(mockMeta, s.storage, &cipher) allDDLJobsBytes, err := metaReader.ReadDDLs(ctx) - c.Assert(err, IsNil) + require.NoError(t, err) var allDDLJobs []*model.Job err = json.Unmarshal(allDDLJobsBytes, &allDDLJobs) - c.Assert(err, IsNil) - c.Assert(len(allDDLJobs), Equals, 8) + require.NoError(t, err) + require.Len(t, allDDLJobs, 8) } -func (r *testBackup) TestCheckBackupIsLocked(c *C) { +func TestCheckBackupIsLocked(t *testing.T) { + s, clean := createBackupSuite(t) + defer clean() + ctx := context.Background() - r.resetStorage(c) // check passed with an empty storage - err := backup.CheckBackupStorageIsLocked(ctx, r.storage) - c.Assert(err, IsNil) + err := backup.CheckBackupStorageIsLocked(ctx, s.storage) + require.NoError(t, err) // check passed with only a lock file - err = r.storage.WriteFile(ctx, metautil.LockFile, nil) - c.Assert(err, IsNil) - err = backup.CheckBackupStorageIsLocked(ctx, r.storage) - c.Assert(err, IsNil) + err = s.storage.WriteFile(ctx, metautil.LockFile, nil) + require.NoError(t, err) + err = backup.CheckBackupStorageIsLocked(ctx, s.storage) + require.NoError(t, err) // check passed with a lock file and other non-sst files. - err = r.storage.WriteFile(ctx, "1.txt", nil) - c.Assert(err, IsNil) - err = backup.CheckBackupStorageIsLocked(ctx, r.storage) - c.Assert(err, IsNil) + err = s.storage.WriteFile(ctx, "1.txt", nil) + require.NoError(t, err) + err = backup.CheckBackupStorageIsLocked(ctx, s.storage) + require.NoError(t, err) // check failed - err = r.storage.WriteFile(ctx, "1.sst", nil) - c.Assert(err, IsNil) - err = backup.CheckBackupStorageIsLocked(ctx, r.storage) - c.Assert(err, ErrorMatches, "backup lock file and sst file exist in(.+)") + err = s.storage.WriteFile(ctx, "1.sst", nil) + require.NoError(t, err) + err = backup.CheckBackupStorageIsLocked(ctx, s.storage) + require.Error(t, err) + require.Regexp(t, "backup lock file and sst file exist in(.+)", err.Error()) } diff --git a/pkg/backup/main_test.go b/pkg/backup/main_test.go new file mode 100644 index 000000000..ee9d4be00 --- /dev/null +++ b/pkg/backup/main_test.go @@ -0,0 +1,37 @@ +// Copyright 2021 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package backup + +import ( + "testing" + + "github.com/pingcap/tidb/util/testbridge" + "go.uber.org/goleak" +) + +func TestMain(m *testing.M) { + testbridge.SetupForCommonTest() + opts := []goleak.Option{ + goleak.IgnoreTopFunction("github.com/klauspost/compress/zstd.(*blockDec).startDecoder"), + goleak.IgnoreTopFunction("github.com/pingcap/goleveldb/leveldb.(*DB).compactionError"), + goleak.IgnoreTopFunction("github.com/pingcap/goleveldb/leveldb.(*DB).mCompaction"), + goleak.IgnoreTopFunction("github.com/pingcap/goleveldb/leveldb.(*DB).mpoolDrain"), + goleak.IgnoreTopFunction("github.com/pingcap/goleveldb/leveldb.(*DB).tCompaction"), + goleak.IgnoreTopFunction("github.com/pingcap/goleveldb/leveldb/util.(*BufferPool).drain"), + goleak.IgnoreTopFunction("go.etcd.io/etcd/pkg/logutil.(*MergeLogger).outputLoop"), + goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"), + } + goleak.VerifyTestMain(m, opts...) +} diff --git a/pkg/backup/schema_test.go b/pkg/backup/schema_test.go index c858d556f..dc4038bf9 100644 --- a/pkg/backup/schema_test.go +++ b/pkg/backup/schema_test.go @@ -8,9 +8,9 @@ import ( "math" "strings" "sync/atomic" + "testing" "github.com/golang/protobuf/proto" - . "github.com/pingcap/check" backuppb "github.com/pingcap/kvproto/pkg/brpb" "github.com/pingcap/kvproto/pkg/encryptionpb" filter "github.com/pingcap/tidb-tools/pkg/table-filter" @@ -20,42 +20,35 @@ import ( "github.com/pingcap/tidb/br/pkg/storage" "github.com/pingcap/tidb/br/pkg/utils" "github.com/pingcap/tidb/sessionctx/variable" - "github.com/pingcap/tidb/util/testkit" - "github.com/pingcap/tidb/util/testleak" + "github.com/pingcap/tidb/testkit" + "github.com/stretchr/testify/require" ) -var _ = Suite(&testBackupSchemaSuite{}) - -type testBackupSchemaSuite struct { - mock *mock.Cluster -} - -func (s *testBackupSchemaSuite) SetUpSuite(c *C) { +func createMockCluster(t *testing.T) (m *mock.Cluster, clean func()) { var err error - s.mock, err = mock.NewCluster() - c.Assert(err, IsNil) - c.Assert(s.mock.Start(), IsNil) -} - -func (s *testBackupSchemaSuite) TearDownSuite(c *C) { - s.mock.Stop() - testleak.AfterTest(c)() + m, err = mock.NewCluster() + require.NoError(t, err) + require.NoError(t, m.Start()) + clean = func() { + m.Stop() + } + return } -func (s *testBackupSchemaSuite) GetRandomStorage(c *C) storage.ExternalStorage { - base := c.MkDir() +func GetRandomStorage(t *testing.T) storage.ExternalStorage { + base := t.TempDir() es, err := storage.NewLocalStorage(base) - c.Assert(err, IsNil) + require.NoError(t, err) return es } -func (s *testBackupSchemaSuite) GetSchemasFromMeta(c *C, es storage.ExternalStorage) []*metautil.Table { +func GetSchemasFromMeta(t *testing.T, es storage.ExternalStorage) []*metautil.Table { ctx := context.Background() metaBytes, err := es.ReadFile(ctx, metautil.MetaFile) - c.Assert(err, IsNil) + require.NoError(t, err) mockMeta := &backuppb.BackupMeta{} err = proto.Unmarshal(metaBytes, mockMeta) - c.Assert(err, IsNil) + require.NoError(t, err) metaReader := metautil.NewMetaReader(mockMeta, es, &backuppb.CipherInfo{ @@ -66,7 +59,7 @@ func (s *testBackupSchemaSuite) GetSchemasFromMeta(c *C, es storage.ExternalStor output := make(chan *metautil.Table, 4) go func() { err = metaReader.ReadSchemasFiles(ctx, output) - c.Assert(err, IsNil) + require.NoError(t, err) close(output) }() @@ -95,33 +88,36 @@ func (sp *simpleProgress) get() int64 { return atomic.LoadInt64(&sp.counter) } -func (s *testBackupSchemaSuite) TestBuildBackupRangeAndSchema(c *C) { - tk := testkit.NewTestKit(c, s.mock.Storage) +func TestBuildBackupRangeAndSchema(t *testing.T) { + m, clean := createMockCluster(t) + defer clean() + + tk := testkit.NewTestKit(t, m.Storage) // Table t1 is not exist. testFilter, err := filter.Parse([]string{"test.t1"}) - c.Assert(err, IsNil) + require.NoError(t, err) _, backupSchemas, err := backup.BuildBackupRangeAndSchema( - s.mock.Storage, testFilter, math.MaxUint64) - c.Assert(err, IsNil) - c.Assert(backupSchemas, IsNil) + m.Storage, testFilter, math.MaxUint64) + require.NoError(t, err) + require.Nil(t, backupSchemas) // Database is not exist. fooFilter, err := filter.Parse([]string{"foo.t1"}) - c.Assert(err, IsNil) + require.NoError(t, err) _, backupSchemas, err = backup.BuildBackupRangeAndSchema( - s.mock.Storage, fooFilter, math.MaxUint64) - c.Assert(err, IsNil) - c.Assert(backupSchemas, IsNil) + m.Storage, fooFilter, math.MaxUint64) + require.NoError(t, err) + require.Nil(t, backupSchemas) // Empty database. // Filter out system tables manually. noFilter, err := filter.Parse([]string{"*.*", "!mysql.*"}) - c.Assert(err, IsNil) + require.NoError(t, err) _, backupSchemas, err = backup.BuildBackupRangeAndSchema( - s.mock.Storage, noFilter, math.MaxUint64) - c.Assert(err, IsNil) - c.Assert(backupSchemas, IsNil) + m.Storage, noFilter, math.MaxUint64) + require.NoError(t, err) + require.Nil(t, backupSchemas) tk.MustExec("use test") tk.MustExec("drop table if exists t1;") @@ -129,30 +125,30 @@ func (s *testBackupSchemaSuite) TestBuildBackupRangeAndSchema(c *C) { tk.MustExec("insert into t1 values (10);") _, backupSchemas, err = backup.BuildBackupRangeAndSchema( - s.mock.Storage, testFilter, math.MaxUint64) - c.Assert(err, IsNil) - c.Assert(backupSchemas.Len(), Equals, 1) + m.Storage, testFilter, math.MaxUint64) + require.NoError(t, err) + require.Equal(t, 1, backupSchemas.Len()) updateCh := new(simpleProgress) skipChecksum := false - es := s.GetRandomStorage(c) + es := GetRandomStorage(t) cipher := backuppb.CipherInfo{ CipherType: encryptionpb.EncryptionMethod_PLAINTEXT, } metaWriter := metautil.NewMetaWriter(es, metautil.MetaFileSize, false, &cipher) ctx := context.Background() err = backupSchemas.BackupSchemas( - ctx, metaWriter, s.mock.Storage, nil, math.MaxUint64, 1, variable.DefChecksumTableConcurrency, skipChecksum, updateCh) - c.Assert(updateCh.get(), Equals, int64(1)) - c.Assert(err, IsNil) + ctx, metaWriter, m.Storage, nil, math.MaxUint64, 1, variable.DefChecksumTableConcurrency, skipChecksum, updateCh) + require.Equal(t, int64(1), updateCh.get()) + require.NoError(t, err) err = metaWriter.FlushBackupMeta(ctx) - c.Assert(err, IsNil) + require.NoError(t, err) - schemas := s.GetSchemasFromMeta(c, es) - c.Assert(len(schemas), Equals, 1) + schemas := GetSchemasFromMeta(t, es) + require.Len(t, schemas, 1) // Cluster returns a dummy checksum (all fields are 1). - c.Assert(schemas[0].Crc64Xor, Not(Equals), 0, Commentf("%v", schemas[0])) - c.Assert(schemas[0].TotalKvs, Not(Equals), 0, Commentf("%v", schemas[0])) - c.Assert(schemas[0].TotalBytes, Not(Equals), 0, Commentf("%v", schemas[0])) + require.NotZerof(t, schemas[0].Crc64Xor, "%v", schemas[0]) + require.NotZerof(t, schemas[0].TotalKvs, "%v", schemas[0]) + require.NotZerof(t, schemas[0].TotalBytes, "%v", schemas[0]) tk.MustExec("drop table if exists t2;") tk.MustExec("create table t2 (a int);") @@ -160,34 +156,37 @@ func (s *testBackupSchemaSuite) TestBuildBackupRangeAndSchema(c *C) { tk.MustExec("insert into t2 values (11);") _, backupSchemas, err = backup.BuildBackupRangeAndSchema( - s.mock.Storage, noFilter, math.MaxUint64) - c.Assert(err, IsNil) - c.Assert(backupSchemas.Len(), Equals, 2) + m.Storage, noFilter, math.MaxUint64) + require.NoError(t, err) + require.Equal(t, 2, backupSchemas.Len()) updateCh.reset() - es2 := s.GetRandomStorage(c) + es2 := GetRandomStorage(t) metaWriter2 := metautil.NewMetaWriter(es2, metautil.MetaFileSize, false, &cipher) err = backupSchemas.BackupSchemas( - ctx, metaWriter2, s.mock.Storage, nil, math.MaxUint64, 2, variable.DefChecksumTableConcurrency, skipChecksum, updateCh) - c.Assert(updateCh.get(), Equals, int64(2)) - c.Assert(err, IsNil) + ctx, metaWriter2, m.Storage, nil, math.MaxUint64, 2, variable.DefChecksumTableConcurrency, skipChecksum, updateCh) + require.Equal(t, int64(2), updateCh.get()) + require.NoError(t, err) err = metaWriter2.FlushBackupMeta(ctx) - c.Assert(err, IsNil) + require.NoError(t, err) - schemas = s.GetSchemasFromMeta(c, es2) + schemas = GetSchemasFromMeta(t, es2) - c.Assert(len(schemas), Equals, 2) + require.Len(t, schemas, 2) // Cluster returns a dummy checksum (all fields are 1). - c.Assert(schemas[0].Crc64Xor, Not(Equals), 0, Commentf("%v", schemas[0])) - c.Assert(schemas[0].TotalKvs, Not(Equals), 0, Commentf("%v", schemas[0])) - c.Assert(schemas[0].TotalBytes, Not(Equals), 0, Commentf("%v", schemas[0])) - c.Assert(schemas[1].Crc64Xor, Not(Equals), 0, Commentf("%v", schemas[1])) - c.Assert(schemas[1].TotalKvs, Not(Equals), 0, Commentf("%v", schemas[1])) - c.Assert(schemas[1].TotalBytes, Not(Equals), 0, Commentf("%v", schemas[1])) + require.NotZerof(t, schemas[0].Crc64Xor, "%v", schemas[0]) + require.NotZerof(t, schemas[0].TotalKvs, "%v", schemas[0]) + require.NotZerof(t, schemas[0].TotalBytes, "%v", schemas[0]) + require.NotZerof(t, schemas[1].Crc64Xor, "%v", schemas[1]) + require.NotZerof(t, schemas[1].TotalKvs, "%v", schemas[1]) + require.NotZerof(t, schemas[1].TotalBytes, "%v", schemas[1]) } -func (s *testBackupSchemaSuite) TestBuildBackupRangeAndSchemaWithBrokenStats(c *C) { - tk := testkit.NewTestKit(c, s.mock.Storage) +func TestBuildBackupRangeAndSchemaWithBrokenStats(t *testing.T) { + m, clean := createMockCluster(t) + defer clean() + + tk := testkit.NewTestKit(t, m.Storage) tk.MustExec("use test") tk.MustExec("drop table if exists t3;") tk.MustExec("create table t3 (a char(1));") @@ -203,11 +202,11 @@ func (s *testBackupSchemaSuite) TestBuildBackupRangeAndSchemaWithBrokenStats(c * `) f, err := filter.Parse([]string{"test.t3"}) - c.Assert(err, IsNil) + require.NoError(t, err) - _, backupSchemas, err := backup.BuildBackupRangeAndSchema(s.mock.Storage, f, math.MaxUint64) - c.Assert(err, IsNil) - c.Assert(backupSchemas.Len(), Equals, 1) + _, backupSchemas, err := backup.BuildBackupRangeAndSchema(m.Storage, f, math.MaxUint64) + require.NoError(t, err) + require.Equal(t, 1, backupSchemas.Len()) skipChecksum := false updateCh := new(simpleProgress) @@ -216,57 +215,60 @@ func (s *testBackupSchemaSuite) TestBuildBackupRangeAndSchemaWithBrokenStats(c * CipherType: encryptionpb.EncryptionMethod_PLAINTEXT, } - es := s.GetRandomStorage(c) + es := GetRandomStorage(t) metaWriter := metautil.NewMetaWriter(es, metautil.MetaFileSize, false, &cipher) ctx := context.Background() err = backupSchemas.BackupSchemas( - ctx, metaWriter, s.mock.Storage, nil, math.MaxUint64, 1, variable.DefChecksumTableConcurrency, skipChecksum, updateCh) - c.Assert(err, IsNil) + ctx, metaWriter, m.Storage, nil, math.MaxUint64, 1, variable.DefChecksumTableConcurrency, skipChecksum, updateCh) + require.NoError(t, err) err = metaWriter.FlushBackupMeta(ctx) - c.Assert(err, IsNil) + require.NoError(t, err) - schemas := s.GetSchemasFromMeta(c, es) - c.Assert(err, IsNil) - c.Assert(schemas, HasLen, 1) + schemas := GetSchemasFromMeta(t, es) + require.NoError(t, err) + require.Len(t, schemas, 1) // the stats should be empty, but other than that everything should be backed up. - c.Assert(schemas[0].Stats, IsNil) - c.Assert(schemas[0].Crc64Xor, Not(Equals), 0) - c.Assert(schemas[0].TotalKvs, Not(Equals), 0) - c.Assert(schemas[0].TotalBytes, Not(Equals), 0) - c.Assert(schemas[0].Info, NotNil) - c.Assert(schemas[0].DB, NotNil) + require.Nil(t, schemas[0].Stats) + require.NotZerof(t, schemas[0].Crc64Xor, "%v", schemas[0]) + require.NotZerof(t, schemas[0].TotalKvs, "%v", schemas[0]) + require.NotZerof(t, schemas[0].TotalBytes, "%v", schemas[0]) + require.NotNil(t, schemas[0].Info) + require.NotNil(t, schemas[0].DB) // recover the statistics. tk.MustExec("analyze table t3;") - _, backupSchemas, err = backup.BuildBackupRangeAndSchema(s.mock.Storage, f, math.MaxUint64) - c.Assert(err, IsNil) - c.Assert(backupSchemas.Len(), Equals, 1) + _, backupSchemas, err = backup.BuildBackupRangeAndSchema(m.Storage, f, math.MaxUint64) + require.NoError(t, err) + require.Equal(t, 1, backupSchemas.Len()) updateCh.reset() - statsHandle := s.mock.Domain.StatsHandle() - es2 := s.GetRandomStorage(c) + statsHandle := m.Domain.StatsHandle() + es2 := GetRandomStorage(t) metaWriter2 := metautil.NewMetaWriter(es2, metautil.MetaFileSize, false, &cipher) err = backupSchemas.BackupSchemas( - ctx, metaWriter2, s.mock.Storage, statsHandle, math.MaxUint64, 1, variable.DefChecksumTableConcurrency, skipChecksum, updateCh) - c.Assert(err, IsNil) + ctx, metaWriter2, m.Storage, statsHandle, math.MaxUint64, 1, variable.DefChecksumTableConcurrency, skipChecksum, updateCh) + require.NoError(t, err) err = metaWriter2.FlushBackupMeta(ctx) - c.Assert(err, IsNil) + require.NoError(t, err) - schemas2 := s.GetSchemasFromMeta(c, es2) - c.Assert(schemas2, HasLen, 1) + schemas2 := GetSchemasFromMeta(t, es2) + require.Len(t, schemas2, 1) // the stats should now be filled, and other than that the result should be equivalent to the first backup. - c.Assert(schemas2[0].Stats, NotNil) - c.Assert(schemas2[0].Crc64Xor, Equals, schemas[0].Crc64Xor) - c.Assert(schemas2[0].TotalKvs, Equals, schemas[0].TotalKvs) - c.Assert(schemas2[0].TotalBytes, Equals, schemas[0].TotalBytes) - c.Assert(schemas2[0].Info, DeepEquals, schemas[0].Info) - c.Assert(schemas2[0].DB, DeepEquals, schemas[0].DB) + require.NotNil(t, schemas2[0].Stats) + require.Equal(t, schemas[0].Crc64Xor, schemas2[0].Crc64Xor) + require.Equal(t, schemas[0].TotalKvs, schemas2[0].TotalKvs) + require.Equal(t, schemas[0].TotalBytes, schemas2[0].TotalBytes) + require.Equal(t, schemas[0].Info, schemas2[0].Info) + require.Equal(t, schemas[0].DB, schemas2[0].DB) } -func (s *testBackupSchemaSuite) TestBackupSchemasForSystemTable(c *C) { - tk := testkit.NewTestKit(c, s.mock.Storage) - es2 := s.GetRandomStorage(c) +func TestBackupSchemasForSystemTable(t *testing.T) { + m, clean := createMockCluster(t) + defer clean() + + tk := testkit.NewTestKit(t, m.Storage) + es2 := GetRandomStorage(t) systemTablesCount := 32 tablePrefix := "systable" @@ -277,10 +279,10 @@ func (s *testBackupSchemaSuite) TestBackupSchemasForSystemTable(c *C) { } f, err := filter.Parse([]string{"mysql.systable*"}) - c.Assert(err, IsNil) - _, backupSchemas, err := backup.BuildBackupRangeAndSchema(s.mock.Storage, f, math.MaxUint64) - c.Assert(err, IsNil) - c.Assert(backupSchemas.Len(), Equals, systemTablesCount) + require.NoError(t, err) + _, backupSchemas, err := backup.BuildBackupRangeAndSchema(m.Storage, f, math.MaxUint64) + require.NoError(t, err) + require.Equal(t, systemTablesCount, backupSchemas.Len()) ctx := context.Background() cipher := backuppb.CipherInfo{ @@ -289,16 +291,16 @@ func (s *testBackupSchemaSuite) TestBackupSchemasForSystemTable(c *C) { updateCh := new(simpleProgress) metaWriter2 := metautil.NewMetaWriter(es2, metautil.MetaFileSize, false, &cipher) - err = backupSchemas.BackupSchemas(ctx, metaWriter2, s.mock.Storage, nil, + err = backupSchemas.BackupSchemas(ctx, metaWriter2, m.Storage, nil, math.MaxUint64, 1, variable.DefChecksumTableConcurrency, true, updateCh) - c.Assert(err, IsNil) + require.NoError(t, err) err = metaWriter2.FlushBackupMeta(ctx) - c.Assert(err, IsNil) + require.NoError(t, err) - schemas2 := s.GetSchemasFromMeta(c, es2) - c.Assert(schemas2, HasLen, systemTablesCount) + schemas2 := GetSchemasFromMeta(t, es2) + require.Len(t, schemas2, systemTablesCount) for _, schema := range schemas2 { - c.Assert(schema.DB.Name, Equals, utils.TemporaryDBName("mysql")) - c.Assert(strings.HasPrefix(schema.Info.Name.O, tablePrefix), Equals, true) + require.Equal(t, utils.TemporaryDBName("mysql"), schema.DB.Name) + require.Equal(t, true, strings.HasPrefix(schema.Info.Name.O, tablePrefix)) } } diff --git a/pkg/pdutil/pd.go b/pkg/pdutil/pd.go index 3f4c45d1d..a3487187f 100644 --- a/pkg/pdutil/pd.go +++ b/pkg/pdutil/pd.go @@ -703,7 +703,9 @@ func (p *PdController) doRemoveSchedulersWith( // Close close the connection to pd. func (p *PdController) Close() { p.pdClient.Close() - close(p.schedulerPauseCh) + if p.schedulerPauseCh != nil { + close(p.schedulerPauseCh) + } } // FetchPDVersion get pd version