Merge pull request #1 from pingcap/master

merge from upstream

.github/workflows/go.yml

@@ -3,11 +3,11 @@ on:
   push:
     branches:
       - master
-      - 'release-.*'
+      - release-*
   pull_request:
     branches:
       - master
-      - 'release-.*'
+      - release-*
 
 jobs:
   unit-test:

Makefile

@@ -27,7 +27,7 @@ test: failpoint-enable
 	$(GOTEST) $(RACEFLAG) -coverprofile=coverage.txt -covermode=atomic -tags leak ./... || ( make failpoint-disable && exit 1 )
 	@make failpoint-disable
 
-integration_test: failpoint-enable bin/dumpling
+integration_test: bins failpoint-enable bin/dumpling
 	@make failpoint-disable
 	./tests/run.sh
 
@@ -115,4 +115,10 @@ tidy:
 	cd tools && GO111MODULE=on go mod tidy
 	git diff --exit-code go.mod go.sum tools/go.mod tools/go.sum
 
-.PHONY: build test integration_test tools failpoint-enable failpoint-disable check static lint tidy
+bins:
+	@which bin/tidb-server
+	@which bin/minio
+	@which bin/tidb-lightning
+	@which bin/sync_diff_inspector
+
+.PHONY: build test integration_test tools failpoint-enable failpoint-disable check static lint tidy bins

tests/basic/run.sh

@@ -15,12 +15,17 @@ run_sql "create database \`$DB_NAME\`;"
 run_sql "create table \`$DB_NAME\`.\`$TABLE_NAME\` (a int);"
 run_sql "insert into \`$DB_NAME\`.\`$TABLE_NAME\` values (1), (2);"
 
-run_dumpling -f "$DB_NAME.$TABLE_NAME"
+run_dumpling -f "$DB_NAME.$TABLE_NAME" -L ${DUMPLING_OUTPUT_DIR}/dumpling.log
 
-cnt=`grep -w "(.*)" ${DUMPLING_OUTPUT_DIR}/${DB_NAME}.${TABLE_NAME}.000000000.sql|wc -l`
+cnt=$(grep -w "(.*)" ${DUMPLING_OUTPUT_DIR}/${DB_NAME}.${TABLE_NAME}.000000000.sql|wc -l)
 echo "records count is ${cnt}"
 [ "$cnt" = 2 ]
 
+# make sure that dumpling log contains version infomation
+cnt=$(grep -w "Welcome to dumpling.*Release Version.*Git Commit Hash.*Go Version" ${DUMPLING_OUTPUT_DIR}/dumpling.log|wc -l)
+echo "version info count is ${cnt}"
+[ "$cnt" = 1 ]
+
 # Test for simple WHERE case.
 run_sql "drop database if exists \`$DB_NAME\`;"
 run_sql "create database \`$DB_NAME\`;"
@@ -35,6 +40,15 @@ expected=$(seq 3 9)
 echo "expected ${expected}, actual ${actual}"
 [ "$actual" = "$expected" ]
 
+# Test for specifying --filetype sql with --sql, should report an error
+set +e
+run_dumpling --sql "select * from \`$DB_NAME\`.\`$TABLE_NAME\`" --filetype sql > ${DUMPLING_OUTPUT_DIR}/dumpling.log
+set -e
+
+actual=$(grep -w "unsupported config.FileType 'sql' when we specify --sql, please unset --filetype or set it to 'csv'" ${DUMPLING_OUTPUT_DIR}/dumpling.log|wc -l)
+echo "expected 1 return error when specifying --filetype sql and --sql, actual ${actual}"
+[ "$actual" = 1 ]
+
 export DUMPLING_TEST_PORT=4000
 # Test for --sql option.
 run_sql "drop database if exists \`$DB_NAME\`;"
@@ -43,36 +57,36 @@ run_sql "create sequence \`$DB_NAME\`.\`$SEQUENCE_NAME\` increment by 1;"
 
 run_dumpling --sql "select nextval(\`$DB_NAME\`.\`$SEQUENCE_NAME\`)"
 
-actual=$(grep -w "(.*)[,|;]" ${DUMPLING_OUTPUT_DIR}/result.000000000.sql | cut -c2-2)
+actual=$(sed -n '2p' ${DUMPLING_OUTPUT_DIR}/result.000000000.csv)
 echo "expected 1, actual ${actual}"
 [ "$actual" = 1 ]
 
 run_dumpling --sql "select nextval(\`$DB_NAME\`.\`$SEQUENCE_NAME\`)"
 
-actual=$(grep -w "(.*)[,|;]" ${DUMPLING_OUTPUT_DIR}/result.000000000.sql | cut -c2-2)
+actual=$(sed -n '2p' ${DUMPLING_OUTPUT_DIR}/result.000000000.csv)
 echo "expected 2, actual ${actual}"
 [ "$actual" = 2 ]
 
 # Test for tidb_mem_quota_query configuration
 export GO_FAILPOINTS="github.com/pingcap/dumpling/v4/export/PrintTiDBMemQuotaQuery=1*return"
 run_dumpling > ${DUMPLING_OUTPUT_DIR}/dumpling.log
-actual=`grep -w "tidb_mem_quota_query == 1073741824" ${DUMPLING_OUTPUT_DIR}/dumpling.log|wc -l`
+actual=$(grep -w "tidb_mem_quota_query == 1073741824" ${DUMPLING_OUTPUT_DIR}/dumpling.log|wc -l)
 echo "expected 1, actual ${actual}"
 [ "$actual" = 1 ]
 
 export GO_FAILPOINTS=""
 
-set +e
 # Test for wrong sql causing panic problem: https://github.com/pingcap/dumpling/pull/234#issuecomment-759996695
-run_dumpling --sql "test" >> ${DUMPLING_OUTPUT_DIR}/dumpling.log 2>> ${DUMPLING_OUTPUT_DIR}/dumpling.err
+set +e
+run_dumpling --sql "test" > ${DUMPLING_OUTPUT_DIR}/dumpling.log 2> ${DUMPLING_OUTPUT_DIR}/dumpling.err
 set -e
 
 # check stderr, should not contain panic info
-actual=`grep -w "panic" ${DUMPLING_OUTPUT_DIR}/dumpling.err|wc -l`
+actual=$(grep -w "panic" ${DUMPLING_OUTPUT_DIR}/dumpling.err|wc -l)
 echo "expected panic 0, actual ${actual}"
 [ "$actual" = 0 ]
 
 # check stdout, should contain mysql error log
-actual=`grep -w "Error 1064: You have an error in your SQL syntax" ${DUMPLING_OUTPUT_DIR}/dumpling.log|wc -l`
+actual=$(grep -w "Error 1064: You have an error in your SQL syntax" ${DUMPLING_OUTPUT_DIR}/dumpling.log|wc -l)
 echo "expect contain Error 1064, actual ${actual}"
 [ "$actual" -ge 1 ]

tests/no_table_and_db_name/run.sh

@@ -30,12 +30,13 @@ assert [ $( ls -lh $DUMPLING_OUTPUT_DIR | grep -e ".csv$" | wc -l ) -eq 10 ]
 # 10 files with header.
 assert [ $( cat $DUMPLING_OUTPUT_DIR/*.csv | wc -l ) -eq $(( 100 + 10 )) ]
 
+# TODO: disable --filetype sql --sql "xxx" until dumpling can product a correct sql file
 # dumping with file size = 311 bytes, actually 10 rows
-run_dumpling -F 311B --filetype sql --sql "select * from $TEST_NAME.t"
+# run_dumpling -F 311B --filetype sql --sql "select * from $TEST_NAME.t"
 
-assert [ $( ls -lh $DUMPLING_OUTPUT_DIR | grep -e ".sql$" | wc -l ) -eq 10 ]
+# assert [ $( ls -lh $DUMPLING_OUTPUT_DIR | grep -e ".sql$" | wc -l ) -eq 10 ]
 
 # 10 files with header.
-assert [ $( cat $DUMPLING_OUTPUT_DIR/*.sql | wc -l ) -eq $(( 100 + 10 * 2 )) ]
+# assert [ $( cat $DUMPLING_OUTPUT_DIR/*.sql | wc -l ) -eq $(( 100 + 10 * 2 )) ]
 
 echo "TEST: [$TEST_NAME] passed."

v4/cli/versions.go

@@ -15,6 +15,10 @@ package cli
 
 import (
 	"fmt"
+
+	"go.uber.org/zap"
+
+	"github.com/pingcap/dumpling/v4/log"
 )
 
 var (
@@ -45,3 +49,13 @@ func LongVersion() string {
 		GoVersion,
 	)
 }
+
+// LogLongVersion logs the version information of this program to the logger.
+func LogLongVersion(logger log.Logger) {
+	logger.Info("Welcome to dumpling",
+		zap.String("Release Version", ReleaseVersion),
+		zap.String("Git Commit Hash", GitHash),
+		zap.String("Git Branch", GitBranch),
+		zap.String("Build timestamp", BuildTimestamp),
+		zap.String("Go Version", GoVersion))
+}

v4/export/config.go

@@ -155,7 +155,7 @@ func DefaultConfig() *Config {
 		NoViews:            true,
 		Rows:               UnspecifiedSize,
 		Where:              "",
-		FileType:           FileFormatSQLTextString,
+		FileType:           "",
 		NoHeader:           false,
 		NoSchemas:          false,
 		NoData:             false,
@@ -221,12 +221,13 @@ func (conf *Config) DefineFlags(flags *pflag.FlagSet) {
 	flags.Uint64P(flagRows, "r", UnspecifiedSize, "Split table into chunks of this many rows, default unlimited")
 	flags.String(flagWhere, "", "Dump only selected records")
 	flags.Bool(flagEscapeBackslash, true, "use backslash to escape special characters")
-	flags.String(flagFiletype, FileFormatSQLTextString, "The type of export file (sql/csv)")
+	flags.String(flagFiletype, "", "The type of export file (sql/csv)")
 	flags.Bool(flagNoHeader, false, "whether not to dump CSV table header")
 	flags.BoolP(flagNoSchemas, "m", false, "Do not dump table schemas with the data")
 	flags.BoolP(flagNoData, "d", false, "Do not dump table data")
 	flags.String(flagCsvNullValue, "\\N", "The null value used when export to csv")
 	flags.StringP(flagSQL, "S", "", "Dump data with given sql. This argument doesn't support concurrent dump")
+	_ = flags.MarkHidden(flagSQL)
 	flags.StringSliceP(flagFilter, "f", []string{"*.*", DefaultTableFilter}, "filter to select which tables to dump")
 	flags.Bool(flagCaseSensitive, false, "whether the filter should be case-sensitive")
 	flags.Bool(flagDumpEmptyDatabase, true, "whether to dump empty database")
@@ -243,6 +244,7 @@ func (conf *Config) DefineFlags(flags *pflag.FlagSet) {
 	flags.Duration(flagReadTimeout, 15*time.Minute, "I/O read timeout for db connection.")
 	_ = flags.MarkHidden(flagReadTimeout)
 	flags.Bool(flagTransactionalConsistency, true, "Only support transactional consistency")
+	_ = flags.MarkHidden(flagTransactionalConsistency)
 	flags.StringP(flagCompress, "c", "", "Compress output file type, support 'gzip', 'no-compression' now")
 }
 
@@ -558,7 +560,7 @@ const (
 
 var (
 	gcSafePointVersion = semver.New("4.0.0")
-	tableSampleVersion = semver.New("5.0.0")
+	tableSampleVersion = semver.New("5.0.0-nightly")
 )
 
 // ServerInfo is the combination of ServerType and ServerInfo
@@ -681,11 +683,22 @@ func validateSpecifiedSQL(conf *Config) error {
 	return nil
 }
 
-func validateFileFormat(conf *Config) error {
+func adjustFileFormat(conf *Config) error {
 	conf.FileType = strings.ToLower(conf.FileType)
 	switch conf.FileType {
-	case FileFormatSQLTextString, FileFormatCSVString:
-		return nil
+	case "":
+		if conf.SQL != "" {
+			conf.FileType = FileFormatCSVString
+		} else {
+			conf.FileType = FileFormatSQLTextString
+		}
+	case FileFormatSQLTextString:
+		if conf.SQL != "" {
+			return errors.Errorf("unsupported config.FileType '%s' when we specify --sql, please unset --filetype or set it to 'csv'", conf.FileType)
+		}
+	case FileFormatCSVString:
+	default:
+		return errors.Errorf("unknown config.FileType '%s'", conf.FileType)
 	}
-	return errors.Errorf("unknown config.FileType '%s'", conf.FileType)
+	return nil
 }

v4/export/config_test.go

@@ -13,7 +13,7 @@ var _ = Suite(&testConfigSuite{})
 type testConfigSuite struct{}
 
 func (s *testConfigSuite) TestCreateExternalStorage(c *C) {
-	mockConfig := DefaultConfig()
+	mockConfig := defaultConfigForTest(c)
 	loc, err := mockConfig.createExternalStorage(context.Background())
 	c.Assert(err, IsNil)
 	c.Assert(loc.URI(), Matches, "file:.*")

v4/export/consistency_test.go

@@ -36,7 +36,7 @@ func (s *testConsistencySuite) TestConsistencyController(c *C) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 	tctx := tcontext.Background().WithContext(ctx)
-	conf := DefaultConfig()
+	conf := defaultConfigForTest(c)
 	resultOk := sqlmock.NewResult(0, 1)
 
 	conf.Consistency = consistencyTypeNone
@@ -85,7 +85,7 @@ func (s *testConsistencySuite) TestConsistencyLockControllerRetry(c *C) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 	tctx := tcontext.Background().WithContext(ctx)
-	conf := DefaultConfig()
+	conf := defaultConfigForTest(c)
 	resultOk := sqlmock.NewResult(0, 1)
 
 	conf.Consistency = consistencyTypeLock
@@ -106,7 +106,7 @@ func (s *testConsistencySuite) TestConsistencyLockControllerRetry(c *C) {
 }
 
 func (s *testConsistencySuite) TestResolveAutoConsistency(c *C) {
-	conf := DefaultConfig()
+	conf := defaultConfigForTest(c)
 	cases := []struct {
 		serverTp            ServerType
 		resolvedConsistency string
@@ -134,7 +134,7 @@ func (s *testConsistencySuite) TestConsistencyControllerError(c *C) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 	tctx := tcontext.Background().WithContext(ctx)
-	conf := DefaultConfig()
+	conf := defaultConfigForTest(c)
 
 	conf.Consistency = "invalid_str"
 	_, err = NewConsistencyController(ctx, conf, db)