diff --git a/v4/export/dump.go b/v4/export/dump.go index 77f847b3..16263784 100755 --- a/v4/export/dump.go +++ b/v4/export/dump.go @@ -343,7 +343,7 @@ func (d *Dumper) buildConcatTask(tctx *tcontext.Context, conn *sql.Conn, meta Ta errCh := make(chan error, 1) go func() { // adjust rows to suitable rows for this table - d.conf.Rows = GetSuitableRows(tctx, conn, meta.DatabaseName(), meta.TableName()) + d.conf.Rows = GetSuitableRows(meta.AvgRowLength()) err := d.concurrentDumpTable(tctx, conn, meta, tableChan) d.conf.Rows = UnspecifiedSize if err != nil { @@ -873,6 +873,7 @@ func dumpTableMeta(conf *Config, conn *sql.Conn, db string, table *TableInfo) (T } meta := &tableMeta{ + avgRowLength: table.AvgRowLength, database: db, table: tbl, colTypes: colTypes, diff --git a/v4/export/ir.go b/v4/export/ir.go index bbe22809..016bc8af 100644 --- a/v4/export/ir.go +++ b/v4/export/ir.go @@ -33,6 +33,7 @@ type TableMeta interface { SpecialComments() StringIter ShowCreateTable() string ShowCreateView() string + AvgRowLength() uint64 } // SQLRowIter is the iterator on a collection of sql.Row. diff --git a/v4/export/ir_impl.go b/v4/export/ir_impl.go index 0a398acc..16aa3bd7 100644 --- a/v4/export/ir_impl.go +++ b/v4/export/ir_impl.go @@ -257,6 +257,7 @@ type tableMeta struct { specCmts []string showCreateTable string showCreateView string + avgRowLength uint64 } func (tm *tableMeta) ColumnTypes() []string { @@ -307,6 +308,10 @@ func (tm *tableMeta) ShowCreateView() string { return tm.showCreateView } +func (tm *tableMeta) AvgRowLength() uint64 { + return tm.avgRowLength +} + type metaData struct { target string metaSQL string diff --git a/v4/export/sql.go b/v4/export/sql.go index 949b594e..02172558 100644 --- a/v4/export/sql.go +++ b/v4/export/sql.go @@ -320,15 +320,13 @@ func SelectTiDBRowID(db *sql.Conn, database, table string) (bool, error) { } // GetSuitableRows gets suitable rows for each table -func GetSuitableRows(tctx *tcontext.Context, db *sql.Conn, database, table string) uint64 { +func GetSuitableRows(avgRowLength uint64) uint64 { const ( defaultRows = 200000 maxRows = 1000000 bytesPerFile = 128 * 1024 * 1024 // 128MB per file by default ) - avgRowLength, err := GetAVGRowLength(tctx, db, database, table) - if err != nil || avgRowLength == 0 { - tctx.L().Debug("fail to get average row length", zap.Uint64("averageRowLength", avgRowLength), zap.Error(err)) + if avgRowLength == 0 { return defaultRows } estimateRows := bytesPerFile / avgRowLength @@ -338,18 +336,6 @@ func GetSuitableRows(tctx *tcontext.Context, db *sql.Conn, database, table strin return estimateRows } -// GetAVGRowLength gets whether this table's average row length -func GetAVGRowLength(tctx *tcontext.Context, db *sql.Conn, database, table string) (uint64, error) { - const query = "select AVG_ROW_LENGTH from INFORMATION_SCHEMA.TABLES where table_schema=? and table_name=?;" - var avgRowLength uint64 - row := db.QueryRowContext(tctx, query, database, table) - err := row.Scan(&avgRowLength) - if err != nil { - return 0, errors.Annotatef(err, "sql: %s", query) - } - return avgRowLength, nil -} - // GetColumnTypes gets *sql.ColumnTypes from a specified table func GetColumnTypes(db *sql.Conn, fields, database, table string) ([]*sql.ColumnType, error) { query := fmt.Sprintf("SELECT %s FROM `%s`.`%s` LIMIT 1", fields, escapeString(database), escapeString(table)) diff --git a/v4/export/sql_test.go b/v4/export/sql_test.go index 45b36398..2339fc56 100644 --- a/v4/export/sql_test.go +++ b/v4/export/sql_test.go @@ -4,7 +4,6 @@ package export import ( "context" - "database/sql" "database/sql/driver" "errors" "fmt" @@ -353,60 +352,29 @@ func (s *testSQLSuite) TestShowCreateView(c *C) { } func (s *testSQLSuite) TestGetSuitableRows(c *C) { - db, mock, err := sqlmock.New() - c.Assert(err, IsNil) - defer db.Close() - conn, err := db.Conn(context.Background()) - c.Assert(err, IsNil) - tctx, cancel := tcontext.Background().WithCancel() - defer cancel() - const ( - query = "select AVG_ROW_LENGTH from INFORMATION_SCHEMA.TABLES where table_schema=\\? and table_name=\\?;" - database = "foo" - table = "bar" - ) - testCases := []struct { avgRowLength uint64 expectedRows uint64 - returnErr error }{ - { - 32, - 200000, - sql.ErrNoRows, - }, { 0, 200000, - nil, }, { 32, 1000000, - nil, }, { 1024, 131072, - nil, }, { 4096, 32768, - nil, }, } for _, testCase := range testCases { - if testCase.returnErr == nil { - mock.ExpectQuery(query).WithArgs(database, table). - WillReturnRows(sqlmock.NewRows([]string{"AVG_ROW_LENGTH"}). - AddRow(testCase.avgRowLength)) - } else { - mock.ExpectQuery(query).WithArgs(database, table). - WillReturnError(testCase.returnErr) - } - rows := GetSuitableRows(tctx, conn, database, table) + rows := GetSuitableRows(testCase.avgRowLength) c.Assert(rows, Equals, testCase.expectedRows) } } diff --git a/v4/export/test_util.go b/v4/export/test_util.go index 776fd767..970841b0 100644 --- a/v4/export/test_util.go +++ b/v4/export/test_util.go @@ -85,6 +85,10 @@ func (m *mockTableIR) ShowCreateView() string { return "" } +func (m *mockTableIR) AvgRowLength() uint64 { + return 0 +} + func (m *mockTableIR) Start(_ *tcontext.Context, conn *sql.Conn) error { return nil }