*: reduce dumpling accessing database and information_schema usage to…

… improve its stability (#305) (#313)

Makefile

@@ -96,6 +96,7 @@ static: tools
 		--disable gosec \
 		--disable errorlint \
 		--disable sqlclosecheck \
+		--disable scopelint \
 		$$($(PACKAGE_DIRECTORIES))
 	# pingcap/errors APIs are mixed with multiple patterns 'pkg/errors',
 	# 'juju/errors' and 'pingcap/parser'. To avoid confusion and mistake,

go.mod

@@ -29,7 +29,6 @@ require (
 	github.com/pingcap/tidb-tools v4.0.9-0.20201127090955-2707c97b3853+incompatible
 	github.com/prometheus/client_golang v1.5.1
 	github.com/prometheus/client_model v0.2.0
-	github.com/rivo/uniseg v0.2.0 // indirect
 	github.com/soheilhy/cmux v0.1.4
 	github.com/spf13/pflag v1.0.5
 	github.com/tikv/pd v0.0.0-20210105112549-e5be7fd38659

go.sum

@@ -477,9 +477,8 @@ github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40T
 github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk=
 github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
+github.com/rivo/uniseg v0.1.0 h1:+2KBaVoUmb9XzDsrx/Ct0W/EYOSFf/nWTauy++DprtY=
 github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
-github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
-github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=

tests/run.sh

@@ -4,8 +4,6 @@
 
 # To avoid permission denied error, please run `chmod +x tests/_utils/*`.
 
-set -e
-
 DUMPLING_TEST_DIR=${DUMPLING_TEST_DIR:-"/tmp/dumpling_test_result"}
 DUMPLING_TEST_USER=${DUMPLING_TEST_USER:-"root"}
 

tests/views/run.sh

@@ -17,8 +17,18 @@ run_dumpling --no-views
 file_not_exist "$DUMPLING_OUTPUT_DIR/views.v-schema.sql"
 file_not_exist "$DUMPLING_OUTPUT_DIR/views.v-schema-view.sql"
 
+rm -rf $DUMPLING_OUTPUT_DIR
 run_dumpling --no-views=false
 #diff "$DUMPLING_BASE_NAME/data/views-schema-create.sql" "$DUMPLING_OUTPUT_DIR/views-schema-create.sql"
 diff "$DUMPLING_BASE_NAME/data/views.v-schema.sql" "$DUMPLING_OUTPUT_DIR/views.v-schema.sql"
 diff "$DUMPLING_BASE_NAME/data/views.v-schema-view.sql" "$DUMPLING_OUTPUT_DIR/views.v-schema-view.sql"
-file_not_exist "$DUMPLING_BASE_NAME/data/views.v.000000000.sql"
+file_not_exist "$DUMPLING_OUTPUT_DIR/views.v.000000000.sql"
+
+# test --no-schemas
+rm -rf $DUMPLING_OUTPUT_DIR
+run_dumpling --no-schemas
+file_not_exist "$DUMPLING_OUTPUT_DIR/views-schema-create.sql"
+file_not_exist "$DUMPLING_OUTPUT_DIR/views.t-schema.sql"
+file_not_exist "$DUMPLING_OUTPUT_DIR/views.v-schema.sql"
+file_not_exist "$DUMPLING_OUTPUT_DIR/views.v-schema-view.sql"
+file_not_exist "$DUMPLING_OUTPUT_DIR/views.v.000000000.sql"

v4/export/block_allow_list.go

@@ -9,13 +9,17 @@ import (
 )
 
 func filterTables(tctx *tcontext.Context, conf *Config) {
+	filterTablesFunc(tctx, conf, conf.TableFilter.MatchTable)
+}
+
+func filterTablesFunc(tctx *tcontext.Context, conf *Config, matchTable func(string, string) bool) {
 	tctx.L().Debug("start to filter tables")
 	dbTables := DatabaseTables{}
 	ignoredDBTable := DatabaseTables{}
 
 	for dbName, tables := range conf.Tables {
 		for _, table := range tables {
-			if conf.TableFilter.MatchTable(dbName, table.Name) {
+			if matchTable(dbName, table.Name) {
 				dbTables.AppendTable(dbName, table)
 			} else {
 				ignoredDBTable.AppendTable(dbName, table)

v4/export/block_allow_list_test.go

@@ -20,11 +20,11 @@ func (s *testBWListSuite) TestFilterTables(c *C) {
 	dbTables := DatabaseTables{}
 	expectedDBTables := DatabaseTables{}
 
-	dbTables.AppendTables(filter.InformationSchemaName, []string{"xxx"}...)
-	dbTables.AppendTables(strings.ToUpper(filter.PerformanceSchemaName), []string{"xxx"}...)
-	dbTables.AppendTables("xxx", []string{"yyy"}...)
-	expectedDBTables.AppendTables("xxx", []string{"yyy"}...)
-	dbTables.AppendTables("yyy", []string{"xxx"}...)
+	dbTables.AppendTables(filter.InformationSchemaName, []string{"xxx"}, []uint64{0})
+	dbTables.AppendTables(strings.ToUpper(filter.PerformanceSchemaName), []string{"xxx"}, []uint64{0})
+	dbTables.AppendTables("xxx", []string{"yyy"}, []uint64{0})
+	expectedDBTables.AppendTables("xxx", []string{"yyy"}, []uint64{0})
+	dbTables.AppendTables("yyy", []string{"xxx"}, []uint64{0})
 
 	tableFilter, err := tf.Parse([]string{"*.*"})
 	c.Assert(err, IsNil)

v4/export/consistency.go

@@ -34,8 +34,8 @@ func NewConsistencyController(ctx context.Context, conf *Config, session *sql.DB
 		}, nil
 	case consistencyTypeLock:
 		return &ConsistencyLockDumpingTables{
-			conn:      conn,
-			allTables: conf.Tables,
+			conn: conn,
+			conf: conf,
 		}, nil
 	case consistencyTypeSnapshot:
 		if conf.ServerInfo.ServerType != ServerTypeTiDB {
@@ -110,16 +110,28 @@ func (c *ConsistencyFlushTableWithReadLock) PingContext(ctx context.Context) err
 
 // ConsistencyLockDumpingTables execute lock tables read on all tables before dump
 type ConsistencyLockDumpingTables struct {
-	conn      *sql.Conn
-	allTables DatabaseTables
+	conn *sql.Conn
+	conf *Config
 }
 
 // Setup implements ConsistencyController.Setup
 func (c *ConsistencyLockDumpingTables) Setup(tctx *tcontext.Context) error {
 	blockList := make(map[string]map[string]interface{})
 	return utils.WithRetry(tctx, func() error {
-		lockTablesSQL := buildLockTablesSQL(c.allTables, blockList)
+		lockTablesSQL := buildLockTablesSQL(c.conf.Tables, blockList)
 		_, err := c.conn.ExecContext(tctx, lockTablesSQL)
+		if err == nil {
+			if len(blockList) > 0 {
+				filterTablesFunc(tctx, c.conf, func(db string, tbl string) bool {
+					if blockTable, ok := blockList[db]; ok {
+						if _, ok := blockTable[tbl]; ok {
+							return false
+						}
+					}
+					return true
+				})
+			}
+		}
 		return errors.Trace(err)
 	}, newLockTablesBackoffer(tctx, blockList))
 }

v4/export/consistency_test.go

@@ -65,17 +65,15 @@ func (s *testConsistencySuite) TestConsistencyController(c *C) {
 
 	conf.Consistency = consistencyTypeLock
 	conf.Tables = NewDatabaseTables().
-		AppendTables("db1", "t1", "t2", "t3").
+		AppendTables("db1", []string{"t1", "t2", "t3"}, []uint64{1, 2, 3}).
 		AppendViews("db2", "t4")
 	mock.ExpectExec("LOCK TABLES `db1`.`t1` READ,`db1`.`t2` READ,`db1`.`t3` READ").WillReturnResult(resultOk)
 	mock.ExpectExec("UNLOCK TABLES").WillReturnResult(resultOk)
 	ctrl, _ = NewConsistencyController(ctx, conf, db)
 	_, ok = ctrl.(*ConsistencyLockDumpingTables)
 	c.Assert(ok, IsTrue)
 	s.assertLifetimeErrNil(tctx, ctrl, c)
-	if err = mock.ExpectationsWereMet(); err != nil {
-		c.Fatal(err.Error())
-	}
+	c.Assert(mock.ExpectationsWereMet(), IsNil)
 }
 
 func (s *testConsistencySuite) TestConsistencyLockControllerRetry(c *C) {
@@ -90,7 +88,7 @@ func (s *testConsistencySuite) TestConsistencyLockControllerRetry(c *C) {
 
 	conf.Consistency = consistencyTypeLock
 	conf.Tables = NewDatabaseTables().
-		AppendTables("db1", "t1", "t2", "t3").
+		AppendTables("db1", []string{"t1", "t2", "t3"}, []uint64{1, 2, 3}).
 		AppendViews("db2", "t4")
 	mock.ExpectExec("LOCK TABLES `db1`.`t1` READ,`db1`.`t2` READ,`db1`.`t3` READ").
 		WillReturnError(&mysql.MySQLError{Number: ErrNoSuchTable, Message: "Table 'db1.t3' doesn't exist"})
@@ -100,9 +98,12 @@ func (s *testConsistencySuite) TestConsistencyLockControllerRetry(c *C) {
 	_, ok := ctrl.(*ConsistencyLockDumpingTables)
 	c.Assert(ok, IsTrue)
 	s.assertLifetimeErrNil(tctx, ctrl, c)
-	if err = mock.ExpectationsWereMet(); err != nil {
-		c.Fatal(err.Error())
-	}
+	// should remove table db1.t3 in tables to dump
+	expectedDumpTables := NewDatabaseTables().
+		AppendTables("db1", []string{"t1", "t2"}, []uint64{1, 2}).
+		AppendViews("db2", "t4")
+	c.Assert(conf.Tables, DeepEquals, expectedDumpTables)
+	c.Assert(mock.ExpectationsWereMet(), IsNil)
 }
 
 func (s *testConsistencySuite) TestResolveAutoConsistency(c *C) {
@@ -156,7 +157,7 @@ func (s *testConsistencySuite) TestConsistencyControllerError(c *C) {
 
 	// lock table fail
 	conf.Consistency = consistencyTypeLock
-	conf.Tables = NewDatabaseTables().AppendTables("db", "t")
+	conf.Tables = NewDatabaseTables().AppendTables("db", []string{"t"}, []uint64{1})
 	mock.ExpectExec("LOCK TABLE").WillReturnError(errors.New(""))
 	ctrl, _ = NewConsistencyController(ctx, conf, db)
 	err = ctrl.Setup(tctx)