diff --git a/dumpling/export/BUILD.bazel b/dumpling/export/BUILD.bazel index 08cc7fe2e664e..cf4d938de6042 100644 --- a/dumpling/export/BUILD.bazel +++ b/dumpling/export/BUILD.bazel @@ -103,6 +103,7 @@ go_test( "//util/filter", "//util/promutil", "//util/table-filter", + "@com_github_coreos_go_semver//semver", "@com_github_data_dog_go_sqlmock//:go-sqlmock", "@com_github_go_sql_driver_mysql//:mysql", "@com_github_pingcap_errors//:errors", diff --git a/dumpling/export/dump.go b/dumpling/export/dump.go index 818f91f937231..d9c363eda39dc 100644 --- a/dumpling/export/dump.go +++ b/dumpling/export/dump.go @@ -16,6 +16,7 @@ import ( "sync/atomic" "time" + "github.com/coreos/go-semver/semver" // import mysql driver "github.com/go-sql-driver/mysql" "github.com/google/uuid" @@ -46,6 +47,10 @@ var openDBFunc = openDB var errEmptyHandleVals = errors.New("empty handleVals for TiDB table") +// After TiDB v6.2.0 we always enable tidb_enable_paging by default. +// see https://docs.pingcap.com/zh/tidb/dev/system-variables#tidb_enable_paging-%E4%BB%8E-v540-%E7%89%88%E6%9C%AC%E5%BC%80%E5%A7%8B%E5%BC%95%E5%85%A5 +var enablePagingVersion = semver.New("6.2.0") + // Dumper is the dump progress structure type Dumper struct { tctx *tcontext.Context @@ -1539,6 +1544,19 @@ func updateServiceSafePoint(tctx *tcontext.Context, pdClient pd.Client, ttl int6 } } +// setDefaultSessionParams is a step to set default params for session params. +func setDefaultSessionParams(si version.ServerInfo, sessionParams map[string]interface{}) { + defaultSessionParams := map[string]interface{}{} + if si.ServerType == version.ServerTypeTiDB && si.HasTiKV && si.ServerVersion.Compare(*enablePagingVersion) >= 0 { + defaultSessionParams["tidb_enable_paging"] = "ON" + } + for k, v := range defaultSessionParams { + if _, ok := sessionParams[k]; !ok { + sessionParams[k] = v + } + } +} + // setSessionParam is an initialization step of Dumper. func setSessionParam(d *Dumper) error { conf, pool := d.conf, d.dbHandle diff --git a/dumpling/export/dump_test.go b/dumpling/export/dump_test.go index c9a40bba28d6f..7d621857f3a85 100644 --- a/dumpling/export/dump_test.go +++ b/dumpling/export/dump_test.go @@ -9,6 +9,7 @@ import ( "time" "github.com/DATA-DOG/go-sqlmock" + "github.com/coreos/go-semver/semver" "github.com/pingcap/errors" "github.com/pingcap/tidb/br/pkg/version" tcontext "github.com/pingcap/tidb/dumpling/context" @@ -224,3 +225,67 @@ func TestUnregisterMetrics(t *testing.T) { // should not panic require.Error(t, err) } + +func TestSetDefaultSessionParams(t *testing.T) { + testCases := []struct { + si version.ServerInfo + sessionParams map[string]interface{} + expectedParams map[string]interface{} + }{ + { + si: version.ServerInfo{ + ServerType: version.ServerTypeTiDB, + HasTiKV: true, + ServerVersion: semver.New("6.1.0"), + }, + sessionParams: map[string]interface{}{ + "tidb_snapshot": "2020-01-01 00:00:00", + }, + expectedParams: map[string]interface{}{ + "tidb_snapshot": "2020-01-01 00:00:00", + }, + }, + { + si: version.ServerInfo{ + ServerType: version.ServerTypeTiDB, + HasTiKV: true, + ServerVersion: semver.New("6.2.0"), + }, + sessionParams: map[string]interface{}{ + "tidb_snapshot": "2020-01-01 00:00:00", + }, + expectedParams: map[string]interface{}{ + "tidb_enable_paging": "ON", + "tidb_snapshot": "2020-01-01 00:00:00", + }, + }, + { + si: version.ServerInfo{ + ServerType: version.ServerTypeTiDB, + HasTiKV: true, + ServerVersion: semver.New("6.2.0"), + }, + sessionParams: map[string]interface{}{ + "tidb_enable_paging": "OFF", + "tidb_snapshot": "2020-01-01 00:00:00", + }, + expectedParams: map[string]interface{}{ + "tidb_enable_paging": "OFF", + "tidb_snapshot": "2020-01-01 00:00:00", + }, + }, + { + si: version.ServerInfo{ + ServerType: version.ServerTypeMySQL, + ServerVersion: semver.New("8.0.32"), + }, + sessionParams: map[string]interface{}{}, + expectedParams: map[string]interface{}{}, + }, + } + + for _, testCase := range testCases { + setDefaultSessionParams(testCase.si, testCase.sessionParams) + require.Equal(t, testCase.expectedParams, testCase.sessionParams) + } +} diff --git a/dumpling/export/sql_test.go b/dumpling/export/sql_test.go index 04615637be8f1..5ae4c7278efa4 100644 --- a/dumpling/export/sql_test.go +++ b/dumpling/export/sql_test.go @@ -1308,6 +1308,8 @@ func buildMockNewRows(mock sqlmock.Sqlmock, columns []string, driverValues [][]d } func readRegionCsvDriverValues(t *testing.T) [][]driver.Value { + t.Helper() + csvFilename := "region_results.csv" file, err := os.Open(csvFilename) require.NoError(t, err) diff --git a/dumpling/export/writer_test.go b/dumpling/export/writer_test.go index b5f54f3debcb9..09e43c469e5bf 100644 --- a/dumpling/export/writer_test.go +++ b/dumpling/export/writer_test.go @@ -5,7 +5,6 @@ package export import ( "context" "database/sql/driver" - "io/ioutil" "os" "path" "sync" @@ -31,7 +30,7 @@ func TestWriteDatabaseMeta(t *testing.T) { _, err = os.Stat(p) require.NoError(t, err) - bytes, err := ioutil.ReadFile(p) + bytes, err := os.ReadFile(p) require.NoError(t, err) require.Equal(t, "/*!40101 SET NAMES binary*/;\nCREATE DATABASE `test`;\n", string(bytes)) } @@ -50,7 +49,7 @@ func TestWritePolicyMeta(t *testing.T) { _, err = os.Stat(p) require.NoError(t, err) - bytes, err := ioutil.ReadFile(p) + bytes, err := os.ReadFile(p) require.NoError(t, err) require.Equal(t, "/*!40101 SET NAMES binary*/;\ncreate placement policy `y` followers=2;\n", string(bytes)) } @@ -68,7 +67,7 @@ func TestWriteTableMeta(t *testing.T) { p := path.Join(dir, "test.t-schema.sql") _, err = os.Stat(p) require.NoError(t, err) - bytes, err := ioutil.ReadFile(p) + bytes, err := os.ReadFile(p) require.NoError(t, err) require.Equal(t, "/*!40101 SET NAMES binary*/;\nCREATE TABLE t (a INT);\n", string(bytes)) } @@ -89,14 +88,14 @@ func TestWriteViewMeta(t *testing.T) { p := path.Join(dir, "test.v-schema.sql") _, err = os.Stat(p) require.NoError(t, err) - bytes, err := ioutil.ReadFile(p) + bytes, err := os.ReadFile(p) require.NoError(t, err) require.Equal(t, specCmt+createTableSQL, string(bytes)) p = path.Join(dir, "test.v-schema-view.sql") _, err = os.Stat(p) require.NoError(t, err) - bytes, err = ioutil.ReadFile(p) + bytes, err = os.ReadFile(p) require.NoError(t, err) require.Equal(t, specCmt+createViewSQL, string(bytes)) } @@ -126,7 +125,7 @@ func TestWriteTableData(t *testing.T) { p := path.Join(dir, "test.employee.000000000.sql") _, err = os.Stat(p) require.NoError(t, err) - bytes, err := ioutil.ReadFile(p) + bytes, err := os.ReadFile(p) require.NoError(t, err) expected := "/*!40101 SET NAMES binary*/;\n" + @@ -182,7 +181,7 @@ func TestWriteTableDataWithFileSize(t *testing.T) { p = path.Join(dir, p) _, err := os.Stat(p) require.NoError(t, err) - bytes, err := ioutil.ReadFile(p) + bytes, err := os.ReadFile(p) require.NoError(t, err) require.Equal(t, expected, string(bytes)) } @@ -232,7 +231,7 @@ func TestWriteTableDataWithFileSizeAndRows(t *testing.T) { p = path.Join(dir, p) _, err = os.Stat(p) require.NoError(t, err) - bytes, err := ioutil.ReadFile(p) + bytes, err := os.ReadFile(p) require.NoError(t, err) require.Equal(t, expected, string(bytes)) } @@ -281,7 +280,7 @@ func TestWriteTableDataWithStatementSize(t *testing.T) { p = path.Join(config.OutputDirPath, p) _, err = os.Stat(p) require.NoError(t, err) - bytes, err1 := ioutil.ReadFile(p) + bytes, err1 := os.ReadFile(p) require.NoError(t, err1) require.Equal(t, expected, string(bytes)) } @@ -297,7 +296,7 @@ func TestWriteTableDataWithStatementSize(t *testing.T) { require.NoError(t, err) err = os.RemoveAll(config.OutputDirPath) require.NoError(t, err) - config.OutputDirPath, err = ioutil.TempDir("", "dumpling") + config.OutputDirPath, err = os.MkdirTemp("", "dumpling") writer = createTestWriter(config, t) @@ -322,7 +321,7 @@ func TestWriteTableDataWithStatementSize(t *testing.T) { p = path.Join(config.OutputDirPath, p) _, err = os.Stat(p) require.NoError(t, err) - bytes, err := ioutil.ReadFile(p) + bytes, err := os.ReadFile(p) require.NoError(t, err) require.Equal(t, expected, string(bytes)) } @@ -331,6 +330,8 @@ func TestWriteTableDataWithStatementSize(t *testing.T) { var mu sync.Mutex func createTestWriter(conf *Config, t *testing.T) *Writer { + t.Helper() + mu.Lock() extStore, err := conf.createExternalStorage(context.Background()) mu.Unlock()