Merge branch 'master' into bb7133/fix_30642

pingcap · Dec 14, 2021 · cdc7ee6 · cdc7ee6
2 parents a4f1fdd + 6c0fcea
commit cdc7ee6
Show file tree

Hide file tree

Showing 384 changed files with 564 additions and 3,568 deletions.
diff --git a/.golangci.yml b/.golangci.yml
@@ -22,6 +22,7 @@ linters:
     - rowserrcheck
     - unconvert
     - makezero
+    - durationcheck
 
 linters-settings:
   staticcheck:

diff --git a/Makefile b/Makefile
@@ -122,7 +122,7 @@ devgotest: failpoint-enable
 gotest: failpoint-enable
 	@echo "Running in native mode."
 	@export log_level=info; export TZ='Asia/Shanghai'; \
-	$(GOTEST) -ldflags '$(TEST_LDFLAGS)' $(EXTRA_TEST_ARGS) -cover $(PACKAGES_TIDB_TESTS) -coverprofile=coverage.txt -check.p true > gotest.log || { $(FAILPOINT_DISABLE); cat 'gotest.log'; exit 1; }
+	$(GOTEST) -ldflags '$(TEST_LDFLAGS)' $(EXTRA_TEST_ARGS) -timeout 20m -cover $(PACKAGES_TIDB_TESTS) -coverprofile=coverage.txt -check.p true > gotest.log || { $(FAILPOINT_DISABLE); cat 'gotest.log'; exit 1; }
 	@$(FAILPOINT_DISABLE)
 
 gotest_in_verify_ci_part_1: failpoint-enable tools/bin/gotestsum tools/bin/gocov tools/bin/gocov-xml

diff --git a/br/pkg/backup/client_test.go b/br/pkg/backup/client_test.go
@@ -265,7 +265,6 @@ func (r *testBackup) TestSendCreds(c *C) {
 	c.Assert(err, IsNil)
 	opts := &storage.ExternalStorageOptions{
 		SendCredentials: true,
-		SkipCheckPath:   true,
 	}
 	_, err = storage.New(r.ctx, backend, opts)
 	c.Assert(err, IsNil)
@@ -284,7 +283,6 @@ func (r *testBackup) TestSendCreds(c *C) {
 	c.Assert(err, IsNil)
 	opts = &storage.ExternalStorageOptions{
 		SendCredentials: false,
-		SkipCheckPath:   true,
 	}
 	_, err = storage.New(r.ctx, backend, opts)
 	c.Assert(err, IsNil)

diff --git a/br/pkg/conn/conn_test.go b/br/pkg/conn/conn_test.go
@@ -165,8 +165,6 @@ func TestCheckStoresAlive(t *testing.T) {
 }
 
 func TestGetAllTiKVStores(t *testing.T) {
-	t.Parallel()
-
 	testCases := []struct {
 		stores         []*metapb.Store
 		storeBehavior  StoreBehavior
@@ -258,8 +256,6 @@ func TestGetAllTiKVStores(t *testing.T) {
 }
 
 func TestGetConnOnCanceledContext(t *testing.T) {
-	t.Parallel()
-
 	ctx, cancel := context.WithCancel(context.Background())
 	cancel()
 

diff --git a/br/pkg/kv/checksum_test.go b/br/pkg/kv/checksum_test.go
@@ -25,8 +25,6 @@ import (
 func uint64NotEqual(a uint64, b uint64) bool { return a != b }
 
 func TestChecksum(t *testing.T) {
-	t.Parallel()
-
 	checksum := kv.NewKVChecksum(0)
 	require.Equal(t, uint64(0), checksum.Sum())
 
@@ -69,8 +67,6 @@ func TestChecksum(t *testing.T) {
 }
 
 func TestChecksumJSON(t *testing.T) {
-	t.Parallel()
-
 	testStruct := &struct {
 		Checksum kv.Checksum
 	}{

diff --git a/br/pkg/kv/kv_test.go b/br/pkg/kv/kv_test.go
@@ -26,8 +26,6 @@ import (
 )
 
 func TestMarshal(t *testing.T) {
-	t.Parallel()
-
 	dats := make([]types.Datum, 4)
 	dats[0].SetInt64(1)
 	dats[1].SetNull()
@@ -43,8 +41,6 @@ func TestMarshal(t *testing.T) {
 }
 
 func TestSimplePairIter(t *testing.T) {
-	t.Parallel()
-
 	pairs := []Pair{
 		{Key: []byte("1"), Val: []byte("a")},
 		{Key: []byte("2"), Val: []byte("b")},

diff --git a/br/pkg/kv/session_test.go b/br/pkg/kv/session_test.go
@@ -22,8 +22,6 @@ import (
 )
 
 func TestSession(t *testing.T) {
-	t.Parallel()
-
 	session := newSession(&SessionOptions{SQLMode: mysql.ModeNone, Timestamp: 1234567890, RowFormatVersion: "1"})
 	_, err := session.Txn(true)
 	require.NoError(t, err)

diff --git a/br/pkg/lightning/backend/backend_test.go b/br/pkg/lightning/backend/backend_test.go
@@ -39,7 +39,6 @@ func (s *backendSuite) tearDownTest() {
 }
 
 func TestOpenCloseImportCleanUpEngine(t *testing.T) {
-	t.Parallel()
 	s := createBackendSuite(t)
 	defer s.tearDownTest()
 	ctx := context.Background()
@@ -72,7 +71,6 @@ func TestOpenCloseImportCleanUpEngine(t *testing.T) {
 }
 
 func TestUnsafeCloseEngine(t *testing.T) {
-	t.Parallel()
 	s := createBackendSuite(t)
 	defer s.tearDownTest()
 
@@ -94,7 +92,6 @@ func TestUnsafeCloseEngine(t *testing.T) {
 }
 
 func TestUnsafeCloseEngineWithUUID(t *testing.T) {
-	t.Parallel()
 	s := createBackendSuite(t)
 	defer s.tearDownTest()
 
@@ -116,7 +113,6 @@ func TestUnsafeCloseEngineWithUUID(t *testing.T) {
 }
 
 func TestWriteEngine(t *testing.T) {
-	t.Parallel()
 	s := createBackendSuite(t)
 	defer s.tearDownTest()
 
@@ -155,7 +151,6 @@ func TestWriteEngine(t *testing.T) {
 }
 
 func TestWriteToEngineWithNothing(t *testing.T) {
-	t.Parallel()
 	s := createBackendSuite(t)
 	defer s.tearDownTest()
 
@@ -179,7 +174,6 @@ func TestWriteToEngineWithNothing(t *testing.T) {
 }
 
 func TestOpenEngineFailed(t *testing.T) {
-	t.Parallel()
 	s := createBackendSuite(t)
 	defer s.tearDownTest()
 
@@ -193,7 +187,6 @@ func TestOpenEngineFailed(t *testing.T) {
 }
 
 func TestWriteEngineFailed(t *testing.T) {
-	t.Parallel()
 	s := createBackendSuite(t)
 	defer s.tearDownTest()
 
@@ -221,7 +214,6 @@ func TestWriteEngineFailed(t *testing.T) {
 }
 
 func TestWriteBatchSendFailedWithRetry(t *testing.T) {
-	t.Parallel()
 	s := createBackendSuite(t)
 	defer s.tearDownTest()
 
@@ -249,7 +241,6 @@ func TestWriteBatchSendFailedWithRetry(t *testing.T) {
 }
 
 func TestImportFailedNoRetry(t *testing.T) {
-	t.Parallel()
 	s := createBackendSuite(t)
 	defer s.tearDownTest()
 
@@ -268,7 +259,6 @@ func TestImportFailedNoRetry(t *testing.T) {
 }
 
 func TestImportFailedWithRetry(t *testing.T) {
-	t.Parallel()
 	s := createBackendSuite(t)
 	defer s.tearDownTest()
 
@@ -289,7 +279,6 @@ func TestImportFailedWithRetry(t *testing.T) {
 }
 
 func TestImportFailedRecovered(t *testing.T) {
-	t.Parallel()
 	s := createBackendSuite(t)
 	defer s.tearDownTest()
 
@@ -312,7 +301,6 @@ func TestImportFailedRecovered(t *testing.T) {
 
 //nolint:interfacer // change test case signature causes check panicking.
 func TestClose(t *testing.T) {
-	t.Parallel()
 	s := createBackendSuite(t)
 	defer s.tearDownTest()
 
@@ -322,7 +310,6 @@ func TestClose(t *testing.T) {
 }
 
 func TestMakeEmptyRows(t *testing.T) {
-	t.Parallel()
 	s := createBackendSuite(t)
 	defer s.tearDownTest()
 
@@ -332,7 +319,6 @@ func TestMakeEmptyRows(t *testing.T) {
 }
 
 func TestNewEncoder(t *testing.T) {
-	t.Parallel()
 	s := createBackendSuite(t)
 	defer s.tearDownTest()
 
@@ -346,7 +332,6 @@ func TestNewEncoder(t *testing.T) {
 }
 
 func TestCheckDiskQuota(t *testing.T) {
-	t.Parallel()
 	s := createBackendSuite(t)
 	defer s.tearDownTest()
 

diff --git a/br/pkg/lightning/backend/importer/importer_test.go b/br/pkg/lightning/backend/importer/importer_test.go
@@ -79,7 +79,6 @@ func (s *importerSuite) tearDownTest() {
 }
 
 func TestWriteRows(t *testing.T) {
-	t.Parallel()
 	s := createImportSuite(t)
 	defer s.tearDownTest()
 
@@ -117,7 +116,6 @@ func TestWriteRows(t *testing.T) {
 }
 
 func TestWriteHeadSendFailed(t *testing.T) {
-	t.Parallel()
 	s := createImportSuite(t)
 	defer s.tearDownTest()
 
@@ -142,7 +140,6 @@ func TestWriteHeadSendFailed(t *testing.T) {
 }
 
 func TestWriteBatchSendFailed(t *testing.T) {
-	t.Parallel()
 	s := createImportSuite(t)
 	defer s.tearDownTest()
 
@@ -174,7 +171,6 @@ func TestWriteBatchSendFailed(t *testing.T) {
 }
 
 func TestWriteCloseFailed(t *testing.T) {
-	t.Parallel()
 	s := createImportSuite(t)
 	defer s.tearDownTest()
 
@@ -206,7 +202,6 @@ func TestWriteCloseFailed(t *testing.T) {
 }
 
 func TestCloseImportCleanupEngine(t *testing.T) {
-	t.Parallel()
 	s := createImportSuite(t)
 	defer s.tearDownTest()
 

diff --git a/br/pkg/lightning/backend/tidb/tidb_test.go b/br/pkg/lightning/backend/tidb/tidb_test.go
@@ -73,7 +73,6 @@ func (s *mysqlSuite) TearDownTest(t *testing.T) {
 }
 
 func TestWriteRowsReplaceOnDup(t *testing.T) {
-	t.Parallel()
 	s := createMysqlSuite(t)
 	defer s.TearDownTest(t)
 	s.mockDB.
@@ -132,7 +131,6 @@ func TestWriteRowsReplaceOnDup(t *testing.T) {
 }
 
 func TestWriteRowsIgnoreOnDup(t *testing.T) {
-	t.Parallel()
 	s := createMysqlSuite(t)
 	defer s.TearDownTest(t)
 	s.mockDB.
@@ -179,7 +177,6 @@ func TestWriteRowsIgnoreOnDup(t *testing.T) {
 }
 
 func TestWriteRowsErrorOnDup(t *testing.T) {
-	t.Parallel()
 	s := createMysqlSuite(t)
 	defer s.TearDownTest(t)
 	s.mockDB.
@@ -259,7 +256,6 @@ func testStrictMode(t *testing.T) {
 }
 
 func TestFetchRemoteTableModels_3_x(t *testing.T) {
-	t.Parallel()
 	s := createMysqlSuite(t)
 	defer s.TearDownTest(t)
 	s.mockDB.ExpectBegin()
@@ -294,7 +290,6 @@ func TestFetchRemoteTableModels_3_x(t *testing.T) {
 }
 
 func TestFetchRemoteTableModels_4_0(t *testing.T) {
-	t.Parallel()
 	s := createMysqlSuite(t)
 	defer s.TearDownTest(t)
 	s.mockDB.ExpectBegin()
@@ -332,7 +327,6 @@ func TestFetchRemoteTableModels_4_0(t *testing.T) {
 }
 
 func TestFetchRemoteTableModels_4_x_auto_increment(t *testing.T) {
-	t.Parallel()
 	s := createMysqlSuite(t)
 	defer s.TearDownTest(t)
 	s.mockDB.ExpectBegin()
@@ -370,7 +364,6 @@ func TestFetchRemoteTableModels_4_x_auto_increment(t *testing.T) {
 }
 
 func TestFetchRemoteTableModels_4_x_auto_random(t *testing.T) {
-	t.Parallel()
 	s := createMysqlSuite(t)
 	defer s.TearDownTest(t)
 	s.mockDB.ExpectBegin()
@@ -410,7 +403,6 @@ func TestFetchRemoteTableModels_4_x_auto_random(t *testing.T) {
 }
 
 func TestWriteRowsErrorNoRetry(t *testing.T) {
-	t.Parallel()
 	nonRetryableError := sql.ErrNoRows
 	s := createMysqlSuite(t)
 	defer s.TearDownTest(t)
@@ -436,7 +428,6 @@ func TestWriteRowsErrorNoRetry(t *testing.T) {
 }
 
 func TestWriteRowsErrorDowngradingAll(t *testing.T) {
-	t.Parallel()
 	nonRetryableError := sql.ErrNoRows
 	s := createMysqlSuite(t)
 	defer s.TearDownTest(t)
@@ -503,7 +494,6 @@ func TestWriteRowsErrorDowngradingAll(t *testing.T) {
 }
 
 func TestWriteRowsErrorDowngradingExceedThreshold(t *testing.T) {
-	t.Parallel()
 	nonRetryableError := sql.ErrNoRows
 	s := createMysqlSuite(t)
 	defer s.TearDownTest(t)
@@ -608,7 +598,6 @@ func encodeRowsTiDB(t *testing.T, b backend.Backend, tbl table.Table) kv.Rows {
 }
 
 func TestEncodeRowForRecord(t *testing.T) {
-	t.Parallel()
 	s := createMysqlSuite(t)
 
 	// for a correct row, the will encode a correct result

diff --git a/br/pkg/lightning/lightning.go b/br/pkg/lightning/lightning.go
@@ -296,6 +296,19 @@ func (l *Lightning) run(taskCtx context.Context, taskCfg *config.Config, g glue.
 		return errors.Annotate(err, "create storage failed")
 	}
 
+	// return expectedErr means at least meet one file
+	expectedErr := errors.New("Stop Iter")
+	walkErr := s.WalkDir(ctx, &storage.WalkOption{ListCount: 1}, func(string, int64) error {
+		// return an error when meet the first regular file to break the walk loop
+		return expectedErr
+	})
+	if !errors.ErrorEqual(walkErr, expectedErr) {
+		if walkErr == nil {
+			return errors.Errorf("data-source-dir '%s' doesn't exist or contains no files", taskCfg.Mydumper.SourceDir)
+		}
+		return errors.Annotatef(walkErr, "visit data-source-dir '%s' failed", taskCfg.Mydumper.SourceDir)
+	}
+
 	loadTask := log.L().Begin(zap.InfoLevel, "load data source")
 	var mdl *mydump.MDLoader
 	mdl, err = mydump.NewMyDumpLoaderWithStore(ctx, taskCfg, s)