dumpling: add enable paging for tidb server (#39791)

close #39790

dumpling/export/BUILD.bazel

@@ -103,6 +103,7 @@ go_test(
  "//util/filter",
  "//util/promutil",
  "//util/table-filter",
+ "@com_github_coreos_go_semver//semver",
  "@com_github_data_dog_go_sqlmock//:go-sqlmock",
  "@com_github_go_sql_driver_mysql//:mysql",
  "@com_github_pingcap_errors//:errors",

dumpling/export/dump.go

@@ -16,6 +16,7 @@ import (
  "sync/atomic"
  "time"
 
+ "github.com/coreos/go-semver/semver"
  // import mysql driver
  "github.com/go-sql-driver/mysql"
  "github.com/google/uuid"
@@ -46,6 +47,10 @@ var openDBFunc = openDB
 
 var errEmptyHandleVals = errors.New("empty handleVals for TiDB table")
 
+// After TiDB v6.2.0 we always enable tidb_enable_paging by default.
+// see https://docs.pingcap.com/zh/tidb/dev/system-variables#tidb_enable_paging-%E4%BB%8E-v540-%E7%89%88%E6%9C%AC%E5%BC%80%E5%A7%8B%E5%BC%95%E5%85%A5
+var enablePagingVersion = semver.New("6.2.0")
+
 // Dumper is the dump progress structure
 type Dumper struct {
  tctx *tcontext.Context
@@ -1539,6 +1544,19 @@ func updateServiceSafePoint(tctx *tcontext.Context, pdClient pd.Client, ttl int6
  }
 }
 
+// setDefaultSessionParams is a step to set default params for session params.
+func setDefaultSessionParams(si version.ServerInfo, sessionParams map[string]interface{}) {
+ defaultSessionParams := map[string]interface{}{}
+ if si.ServerType == version.ServerTypeTiDB && si.HasTiKV && si.ServerVersion.Compare(*enablePagingVersion) >= 0 {
+ defaultSessionParams["tidb_enable_paging"] = "ON"
+ }
+ for k, v := range defaultSessionParams {
+ if _, ok := sessionParams[k]; !ok {
+ sessionParams[k] = v
+ }
+ }
+}
+
 // setSessionParam is an initialization step of Dumper.
 func setSessionParam(d *Dumper) error {
  conf, pool := d.conf, d.dbHandle

dumpling/export/dump_test.go

@@ -9,6 +9,7 @@ import (
  "time"
 
  "github.com/DATA-DOG/go-sqlmock"
+ "github.com/coreos/go-semver/semver"
  "github.com/pingcap/errors"
  "github.com/pingcap/tidb/br/pkg/version"
  tcontext "github.com/pingcap/tidb/dumpling/context"
@@ -224,3 +225,67 @@ func TestUnregisterMetrics(t *testing.T) {
  // should not panic
  require.Error(t, err)
 }
+
+func TestSetDefaultSessionParams(t *testing.T) {
+ testCases := []struct {
+ si version.ServerInfo
+ sessionParams map[string]interface{}
+ expectedParams map[string]interface{}
+ }{
+ {
+ si: version.ServerInfo{
+ ServerType: version.ServerTypeTiDB,
+ HasTiKV: true,
+ ServerVersion: semver.New("6.1.0"),
+ },
+ sessionParams: map[string]interface{}{
+ "tidb_snapshot": "2020-01-01 00:00:00",
+ },
+ expectedParams: map[string]interface{}{
+ "tidb_snapshot": "2020-01-01 00:00:00",
+ },
+ },
+ {
+ si: version.ServerInfo{
+ ServerType: version.ServerTypeTiDB,
+ HasTiKV: true,
+ ServerVersion: semver.New("6.2.0"),
+ },
+ sessionParams: map[string]interface{}{
+ "tidb_snapshot": "2020-01-01 00:00:00",
+ },
+ expectedParams: map[string]interface{}{
+ "tidb_enable_paging": "ON",
+ "tidb_snapshot": "2020-01-01 00:00:00",
+ },
+ },
+ {
+ si: version.ServerInfo{
+ ServerType: version.ServerTypeTiDB,
+ HasTiKV: true,
+ ServerVersion: semver.New("6.2.0"),
+ },
+ sessionParams: map[string]interface{}{
+ "tidb_enable_paging": "OFF",
+ "tidb_snapshot": "2020-01-01 00:00:00",
+ },
+ expectedParams: map[string]interface{}{
+ "tidb_enable_paging": "OFF",
+ "tidb_snapshot": "2020-01-01 00:00:00",
+ },
+ },
+ {
+ si: version.ServerInfo{
+ ServerType: version.ServerTypeMySQL,
+ ServerVersion: semver.New("8.0.32"),
+ },
+ sessionParams: map[string]interface{}{},
+ expectedParams: map[string]interface{}{},
+ },
+ }
+
+ for _, testCase := range testCases {
+ setDefaultSessionParams(testCase.si, testCase.sessionParams)
+ require.Equal(t, testCase.expectedParams, testCase.sessionParams)
+ }
+}

dumpling/export/sql_test.go

@@ -1308,6 +1308,8 @@ func buildMockNewRows(mock sqlmock.Sqlmock, columns []string, driverValues [][]d
 }
 
 func readRegionCsvDriverValues(t *testing.T) [][]driver.Value {
+ t.Helper()
+
  csvFilename := "region_results.csv"
  file, err := os.Open(csvFilename)
  require.NoError(t, err)

dumpling/export/writer_test.go

@@ -5,7 +5,6 @@ package export
 import (
  "context"
  "database/sql/driver"
- "io/ioutil"
  "os"
  "path"
  "sync"
@@ -31,7 +30,7 @@ func TestWriteDatabaseMeta(t *testing.T) {
  _, err = os.Stat(p)
  require.NoError(t, err)
 
- bytes, err := ioutil.ReadFile(p)
+ bytes, err := os.ReadFile(p)
  require.NoError(t, err)
  require.Equal(t, "/*!40101 SET NAMES binary*/;\nCREATE DATABASE `test`;\n", string(bytes))
 }
@@ -50,7 +49,7 @@ func TestWritePolicyMeta(t *testing.T) {
  _, err = os.Stat(p)
  require.NoError(t, err)
 
- bytes, err := ioutil.ReadFile(p)
+ bytes, err := os.ReadFile(p)
  require.NoError(t, err)
  require.Equal(t, "/*!40101 SET NAMES binary*/;\ncreate placement policy `y` followers=2;\n", string(bytes))
 }
@@ -68,7 +67,7 @@ func TestWriteTableMeta(t *testing.T) {
  p := path.Join(dir, "test.t-schema.sql")
  _, err = os.Stat(p)
  require.NoError(t, err)
- bytes, err := ioutil.ReadFile(p)
+ bytes, err := os.ReadFile(p)
  require.NoError(t, err)
  require.Equal(t, "/*!40101 SET NAMES binary*/;\nCREATE TABLE t (a INT);\n", string(bytes))
 }
@@ -89,14 +88,14 @@ func TestWriteViewMeta(t *testing.T) {
  p := path.Join(dir, "test.v-schema.sql")
  _, err = os.Stat(p)
  require.NoError(t, err)
- bytes, err := ioutil.ReadFile(p)
+ bytes, err := os.ReadFile(p)
  require.NoError(t, err)
  require.Equal(t, specCmt+createTableSQL, string(bytes))
 
  p = path.Join(dir, "test.v-schema-view.sql")
  _, err = os.Stat(p)
  require.NoError(t, err)
- bytes, err = ioutil.ReadFile(p)
+ bytes, err = os.ReadFile(p)
  require.NoError(t, err)
  require.Equal(t, specCmt+createViewSQL, string(bytes))
 }
@@ -126,7 +125,7 @@ func TestWriteTableData(t *testing.T) {
  p := path.Join(dir, "test.employee.000000000.sql")
  _, err = os.Stat(p)
  require.NoError(t, err)
- bytes, err := ioutil.ReadFile(p)
+ bytes, err := os.ReadFile(p)
  require.NoError(t, err)
 
  expected := "/*!40101 SET NAMES binary*/;\n" +
@@ -182,7 +181,7 @@ func TestWriteTableDataWithFileSize(t *testing.T) {
  p = path.Join(dir, p)
  _, err := os.Stat(p)
  require.NoError(t, err)
- bytes, err := ioutil.ReadFile(p)
+ bytes, err := os.ReadFile(p)
  require.NoError(t, err)
  require.Equal(t, expected, string(bytes))
  }
@@ -232,7 +231,7 @@ func TestWriteTableDataWithFileSizeAndRows(t *testing.T) {
  p = path.Join(dir, p)
  _, err = os.Stat(p)
  require.NoError(t, err)
- bytes, err := ioutil.ReadFile(p)
+ bytes, err := os.ReadFile(p)
  require.NoError(t, err)
  require.Equal(t, expected, string(bytes))
  }
@@ -281,7 +280,7 @@ func TestWriteTableDataWithStatementSize(t *testing.T) {
  p = path.Join(config.OutputDirPath, p)
  _, err = os.Stat(p)
  require.NoError(t, err)
- bytes, err1 := ioutil.ReadFile(p)
+ bytes, err1 := os.ReadFile(p)
  require.NoError(t, err1)
  require.Equal(t, expected, string(bytes))
  }
@@ -297,7 +296,7 @@ func TestWriteTableDataWithStatementSize(t *testing.T) {
  require.NoError(t, err)
  err = os.RemoveAll(config.OutputDirPath)
  require.NoError(t, err)
- config.OutputDirPath, err = ioutil.TempDir("", "dumpling")
+ config.OutputDirPath, err = os.MkdirTemp("", "dumpling")
 
  writer = createTestWriter(config, t)
 
@@ -322,7 +321,7 @@ func TestWriteTableDataWithStatementSize(t *testing.T) {
  p = path.Join(config.OutputDirPath, p)
  _, err = os.Stat(p)
  require.NoError(t, err)
- bytes, err := ioutil.ReadFile(p)
+ bytes, err := os.ReadFile(p)
  require.NoError(t, err)
  require.Equal(t, expected, string(bytes))
  }
@@ -331,6 +330,8 @@ func TestWriteTableDataWithStatementSize(t *testing.T) {
 var mu sync.Mutex
 
 func createTestWriter(conf *Config, t *testing.T) *Writer {
+ t.Helper()
+
  mu.Lock()
  extStore, err := conf.createExternalStorage(context.Background())
  mu.Unlock()