Merge branch 'master' into e2e_test

.gitignore

@@ -2,4 +2,4 @@
 bin/
 coverage.txt
 .idea
-var
+var

README.md

@@ -0,0 +1,36 @@
+🥟 Dumpling
+============
+
+[![Build Status](https://travis-ci.org/pingcap/dumpling.svg?branch=master)](https://travis-ci.org/pingcap/dumpling)
+[![codecov](https://codecov.io/gh/pingcap/dumpling/branch/master/graph/badge.svg)](https://codecov.io/gh/pingcap/dumpling)
+[![API Docs](https://img.shields.io/badge/go.dev-reference-007d9c?logo=go&logoColor=white)](https://pkg.go.dev/github.com/pingcap/dumpling)
+[![Go Report Card](https://goreportcard.com/badge/github.com/pingcap/dumpling)](https://goreportcard.com/report/github.com/pingcap/dumpling)
+
+**Dumpling** is a tool and a Go library for creating SQL dump from a MySQL-compatible database.
+It is intended to replace `mysqldump` and `mydumper` when targeting TiDB.
+
+You may read the [design document](https://github.com/pingcap/community/blob/master/rfc/2019-12-06-dumpling.md) for details.
+
+Features
+--------
+
+> Dumpling is currently in early development stage, and most features are incomplete. Contributions are welcomed!
+
+- [x] SQL dump is split into multiple files (like `mydumper`) for easy management.
+- [x] Export multiple tables in parallel to speed up execution.
+- [ ] Multiple output formats: SQL, CSV, ...
+- [ ] Write to cloud storage (S3, GCS) natively
+- [ ] Advanced table filtering via black-/white-lists
+
+Building
+--------
+
+1. Install Go 1.13 or above
+2. Run `make build` to compile. The output is in `bin/dumpling`.
+3. Run `make test` to run the unit tests.
+4. Run `make integration_test` to run integration tests.
+
+License
+-------
+
+Dumpling is under the Apache 2.0 license. See the [LICENSE](./LICENSE) file for details.

cmd/dumpling/main.go

@@ -38,6 +38,7 @@ var (
 	logLevel    string
 	consistency string
 	snapshot    string
+	noViews     bool
 
 	rootCmd = &cobra.Command{
 		Use:   "dumpling",
@@ -67,6 +68,7 @@ func init() {
 	rootCmd.PersistentFlags().StringVar(&logLevel, "loglevel", "info", "Log level: {debug|info|warn|error|dpanic|panic|fatal}")
 	rootCmd.PersistentFlags().StringVar(&consistency, "consistency", "auto", "Consistency level during dumping: {auto|none|flush|lock|snapshot}")
 	rootCmd.PersistentFlags().StringVar(&snapshot, "snapshot", "", "Snapshot position. Valid only when consistency=snapshot")
+	rootCmd.PersistentFlags().BoolVarP(&noViews, "no-views", "W", true, "Do not dump views")
 }
 
 func run() {
@@ -87,6 +89,7 @@ func run() {
 	conf.Threads = threads
 	conf.FileSize = fileSize
 	conf.OutputDirPath = outputDir
+	conf.NoViews = noViews
 
 	err = export.Dump(conf)
 	if err != nil {

tests/_utils/file_not_exist

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+set -eu
+
+if [ -f "$1" ]; then
+  echo "[$(date)] File $1 already exists." && exit 1
+fi

tests/run.sh

@@ -1,5 +1,7 @@
 #!/bin/sh
 
+# To avoid permission denied error, please run `chmod +x tests/_utils/*`.
+
 set -e
 
 DUMPLING_TEST_DIR=${DUMPLING_TEST_DIR:-"/tmp/dumpling_test_result"}
@@ -37,6 +39,9 @@ for script in tests/*/run.sh; do
     DUMPLING_OUTPUT_DIR="$DUMPLING_TEST_DIR"/sql_res."$TEST_NAME"
     export DUMPLING_OUTPUT_DIR
 
+    echo "Cleaning up test output dir: $DUMPLING_OUTPUT_DIR"
+    rm "$DUMPLING_OUTPUT_DIR" -rf
+
     PATH="tests/_utils:$PATH" \
     sh "$script"
 done

tests/views/data/views-schema-create.sql

@@ -0,0 +1 @@
+CREATE DATABASE `views` /*!40100 DEFAULT CHARACTER SET latin1 */;

tests/views/data/views.v-schema.sql

@@ -0,0 +1 @@
+CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`localhost` SQL SECURITY DEFINER VIEW `views`.`v` AS select `views`.`t`.`a` AS `a`,`views`.`t`.`b` AS `b` from `views`.`t`;

tests/views/run.sh

@@ -0,0 +1,22 @@
+#!/bin/sh
+
+set -eu
+
+run_sql "drop database if exists views"
+run_sql "create database views"
+export DUMPLING_TEST_DATABASE="views"
+
+run_sql "create table t (a bigint, b varchar(255))"
+run_sql "create definer = 'root'@'localhost' view v as select * from t;"
+# insert 20 records to `t`.
+i=0; while [ $i -lt 20 ]; do
+  run_sql "insert into t values ($i, \"$i\")"
+  i=$(( i + 1 ))
+done
+
+run_dumpling --no-views
+file_not_exist "$DUMPLING_OUTPUT_DIR/views.v-schema.sql"
+
+run_dumpling --no-views=false
+diff "$DUMPLING_BASE_NAME/data/views-schema-create.sql" "$DUMPLING_OUTPUT_DIR/views-schema-create.sql"
+diff "$DUMPLING_BASE_NAME/data/views.v-schema.sql" "$DUMPLING_OUTPUT_DIR/views.v-schema.sql"

v4/export/config.go

@@ -25,6 +25,7 @@ type Config struct {
 	Tables        DatabaseTables
 	Snapshot      string
 	Consistency   string
+	NoViews       bool
 }
 
 func DefaultConfig() *Config {
@@ -42,6 +43,7 @@ func DefaultConfig() *Config {
 		Tables:        nil,
 		Snapshot:      "",
 		Consistency:   "auto",
+		NoViews:       true,
 	}
 }
 
@@ -128,9 +130,3 @@ func init() {
 	serverTypeString[ServerTypeMariaDB] = "MariaDB"
 	serverTypeString[ServerTypeTiDB] = "TiDB"
 }
-
-type databaseName = string
-
-type tableName = string
-
-type DatabaseTables map[databaseName][]tableName

v4/export/consistency.go

@@ -75,7 +75,7 @@ type ConsistencyLockDumpingTables struct {
 func (c *ConsistencyLockDumpingTables) Setup() error {
 	for dbName, tables := range c.allTables {
 		for _, table := range tables {
-			err := LockTables(c.db, dbName, table)
+			err := LockTables(c.db, dbName, table.Name)
 			if err != nil {
 				return err
 			}

v4/export/consistency_test.go

@@ -64,10 +64,9 @@ func (s *testConsistencySuite) TestConsistencyController(c *C) {
 	}
 
 	conf.Consistency = "lock"
-	conf.Tables = map[databaseName][]tableName{
-		"db1": {"t1", "t2", "t3"},
-		"db2": {"t4"},
-	}
+	conf.Tables = NewDatabaseTables().
+		AppendTables("db1", "t1", "t2", "t3").
+		AppendViews("db2", "t4")
 	for i := 0; i < 4; i++ {
 		mock.ExpectExec("LOCK TABLES").WillReturnResult(resultOk)
 	}
@@ -129,7 +128,7 @@ func (s *testConsistencySuite) TestConsistencyControllerError(c *C) {
 
 	// lock table fail
 	conf.Consistency = "lock"
-	conf.Tables = map[databaseName][]tableName{"db": {"t"}}
+	conf.Tables = NewDatabaseTables().AppendTables("db", "t")
 	mock.ExpectExec("LOCK TABLE").WillReturnError(errors.New(""))
 	ctrl, _ = NewConsistencyController(conf, db)
 	err = ctrl.Setup()

v4/export/dump.go

@@ -3,9 +3,9 @@ package export
 import (
 	"context"
 	"database/sql"
-	"golang.org/x/sync/errgroup"
 
 	_ "github.com/go-sql-driver/mysql"
+	"golang.org/x/sync/errgroup"
 )
 
 func Dump(conf *Config) (err error) {
@@ -30,6 +30,16 @@ func Dump(conf *Config) (err error) {
 		return err
 	}
 
+	if !conf.NoViews {
+		views, err := listAllViews(pool, databases)
+		if err != nil {
+			return err
+		}
+		conf.Tables.Merge(views)
+	}
+
+	filterDirtySchemaTables(conf)
+
 	conCtrl, err := NewConsistencyController(conf, pool)
 	if err != nil {
 		return err
@@ -78,16 +88,26 @@ func dumpDatabases(ctx context.Context, conf *Config, db *sql.DB, writer Writer)
 	return nil
 }
 
-func dumpTable(ctx context.Context, conf *Config, db *sql.DB, dbName, table string, writer Writer) error {
-	createTableSQL, err := ShowCreateTable(db, dbName, table)
+func dumpTable(ctx context.Context, conf *Config, db *sql.DB, dbName string, table *TableInfo, writer Writer) error {
+	if table.Type == TableTypeView {
+		viewName := table.Name
+		createViewSQL, err := ShowCreateView(db, dbName, viewName)
+		if err != nil {
+			return err
+		}
+		return writer.WriteTableMeta(ctx, dbName, viewName, createViewSQL)
+	}
+
+	tableName := table.Name
+	createTableSQL, err := ShowCreateTable(db, dbName, tableName)
 	if err != nil {
 		return err
 	}
-	if err := writer.WriteTableMeta(ctx, dbName, table, createTableSQL); err != nil {
+	if err := writer.WriteTableMeta(ctx, dbName, tableName, createTableSQL); err != nil {
 		return err
 	}
 
-	tableIR, err := SelectAllFromTable(conf, db, dbName, table)
+	tableIR, err := SelectAllFromTable(conf, db, dbName, tableName)
 	if err != nil {
 		return err
 	}

v4/export/dump_test.go

@@ -45,7 +45,7 @@ func (s *testDumpSuite) TestDumpDatabase(c *C) {
 	mockConfig := DefaultConfig()
 	mockConfig.SortByPk = false
 	mockConfig.Database = "test"
-	mockConfig.Tables = map[string][]tableName{"test": {"t"}}
+	mockConfig.Tables = NewDatabaseTables().AppendTables("test", "t")
 	db, mock, err := sqlmock.New()
 	c.Assert(err, IsNil)
 
@@ -85,7 +85,7 @@ func (s *testDumpSuite) TestDumpTable(c *C) {
 	mock.ExpectQuery("SELECT (.) FROM test.t").WillReturnRows(rows)
 
 	mockWriter := newMockWriter()
-	err = dumpTable(context.Background(), mockConfig, db, "test", "t", mockWriter)
+	err = dumpTable(context.Background(), mockConfig, db, "test", &TableInfo{Name: "t"}, mockWriter)
 	c.Assert(err, IsNil)
 
 	c.Assert(mockWriter.tableMeta["test.t"], Equals, showCreateTableResult)

v4/export/prepare.go

@@ -5,6 +5,7 @@ import (
 	"strings"
 
 	"github.com/pingcap/dumpling/v4/log"
+	"go.uber.org/zap"
 )
 
 func detectServerInfo(db *sql.DB) (ServerInfo, error) {
@@ -27,15 +28,81 @@ func listAllTables(db *sql.DB, databaseNames []string) (DatabaseTables, error) {
 	log.Zap().Debug("list all the tables")
 	dbTables := DatabaseTables{}
 	for _, dbName := range databaseNames {
-		err := UseDatabase(db, dbName)
+		tables, err := ListAllTables(db, dbName)
 		if err != nil {
 			return nil, err
 		}
-		tables, err := ShowTables(db)
+		dbTables = dbTables.AppendTables(dbName, tables...)
+	}
+	return dbTables, nil
+}
+
+func listAllViews(db *sql.DB, databaseNames []string) (DatabaseTables, error) {
+	log.Zap().Debug("list all the views")
+	dbTables := DatabaseTables{}
+	for _, dbName := range databaseNames {
+		views, err := ListAllViews(db, dbName)
 		if err != nil {
 			return nil, err
 		}
-		dbTables[dbName] = tables
+		dbTables = dbTables.AppendViews(dbName, views...)
 	}
 	return dbTables, nil
 }
+
+type databaseName = string
+
+type TableType int8
+
+const (
+	TableTypeBase TableType = iota
+	TableTypeView
+)
+
+type TableInfo struct {
+	Name string
+	Type TableType
+}
+
+func (t *TableInfo) Equals(other *TableInfo) bool {
+	return t.Name == other.Name && t.Type == other.Type
+}
+
+type DatabaseTables map[databaseName][]*TableInfo
+
+func NewDatabaseTables() DatabaseTables {
+	return DatabaseTables{}
+}
+
+func (d DatabaseTables) AppendTables(dbName string, tableNames ...string) DatabaseTables {
+	for _, t := range tableNames {
+		d[dbName] = append(d[dbName], &TableInfo{t, TableTypeBase})
+	}
+	return d
+}
+
+func (d DatabaseTables) AppendViews(dbName string, viewNames ...string) DatabaseTables {
+	for _, v := range viewNames {
+		d[dbName] = append(d[dbName], &TableInfo{v, TableTypeView})
+	}
+	return d
+}
+
+func (d DatabaseTables) Merge(other DatabaseTables) {
+	for name, infos := range other {
+		d[name] = append(d[name], infos...)
+	}
+}
+
+func filterDirtySchemaTables(conf *Config) {
+	switch conf.ServerInfo.ServerType {
+	case ServerTypeTiDB:
+		for dbName := range conf.Tables {
+			switch strings.ToUpper(dbName) {
+			case "INSPECTION_SCHEMA", "METRICS_SCHEMA", "PERFORMANCE_SCHEMA", "INFORMATION_SCHEMA":
+				log.Zap().Warn("unsupported dump schema in TiDB now", zap.String("schema", dbName))
+				delete(conf.Tables, dbName)
+			}
+		}
+	}
+}