Skip to content
This repository has been archived by the owner on Aug 21, 2023. It is now read-only.

*: cherry-pick recent PRs for dumpling v5.0.1 #271

Merged
merged 11 commits into from
Apr 16, 2021
Merged

*: cherry-pick recent PRs for dumpling v5.0.1 #271

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
1677aea
Add conn session vars (#265)
recall704 Apr 5, 2021
7f739b1
add estimate total rows count to metrics for dump table (#254)
recall704 Mar 26, 2021
f47b9f6
sql: fix table sample sql generator bug (#262)
lichunzhu Apr 7, 2021
7d1fccd
export: fix a metric not using labelNames (#266)
lance6716 Apr 7, 2021
236f366
export: fix estimateTotalRowsCounter nil pointer (#267)
lichunzhu Apr 7, 2021
fb89ccc
log: replace pingcap global logger when start dumpling binary (#270)
lichunzhu Apr 14, 2021
1b33d20
fix bug
lichunzhu Apr 15, 2021
278a128
add integration test
lichunzhu Apr 16, 2021
0f32fad
fix integration test
lichunzhu Apr 16, 2021
ba962eb
Revert "fix bug"
lichunzhu Apr 16, 2021
a7c8c08
tmp
lichunzhu Apr 16, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ require (
github.com/pingcap/tidb-tools v4.0.9-0.20201127090955-2707c97b3853+incompatible
github.com/prometheus/client_golang v1.5.1
github.com/prometheus/client_model v0.2.0
github.com/siddontang/go-mysql v0.0.0-20200222075837-12e89848f047
github.com/soheilhy/cmux v0.1.4
github.com/spf13/pflag v1.0.5
github.com/syndtr/goleveldb v1.0.1-0.20190625010220-02440ea7a285 // indirect
Expand Down
4 changes: 4 additions & 0 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -766,6 +766,7 @@ github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/sasha-s/go-deadlock v0.2.0/go.mod h1:StQn567HiB1fF2yJ44N9au7wOhrPS3iZqiDbRupzT10=
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b h1:gQZ0qzfKHQIybLANtM3mBXNUtOfsCFXeTsnBqCsx1KM=
github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
github.com/segmentio/kafka-go v0.1.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo=
github.com/segmentio/kafka-go v0.2.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo=
Expand All @@ -789,9 +790,12 @@ github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeV
github.com/shurcooL/vfsgen v0.0.0-20181020040650-a97a25d856ca/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw=
github.com/shurcooL/vfsgen v0.0.0-20181202132449-6a9ea43bcacd h1:ug7PpSOB5RBPK1Kg6qskGBoP3Vnj/aNYFTznWvlkGo0=
github.com/shurcooL/vfsgen v0.0.0-20181202132449-6a9ea43bcacd/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw=
github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 h1:xT+JlYxNGqyT+XcU8iUrN18JYed2TvG9yN5ULG2jATM=
github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726/go.mod h1:3yhqj7WBBfRhbBlzyOC3gUxftwsU0u8gqevxwIHQpMw=
github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07/go.mod h1:yFdBgwXP24JziuRl2NMUahT7nGLNOKi1SIiFxMttVD4=
github.com/siddontang/go-log v0.0.0-20190221022429-1e957dd83bed h1:KMgQoLJGCq1IoZpLZE3AIffh9veYWoVlsvA4ib55TMM=
github.com/siddontang/go-log v0.0.0-20190221022429-1e957dd83bed/go.mod h1:yFdBgwXP24JziuRl2NMUahT7nGLNOKi1SIiFxMttVD4=
github.com/siddontang/go-mysql v0.0.0-20200222075837-12e89848f047 h1:boyJ8EgQN/aC3grvx8QUoJrptt7RvneezSJSCbW25a4=
github.com/siddontang/go-mysql v0.0.0-20200222075837-12e89848f047/go.mod h1:+W4RCzesQDI11HvIkaDjS8yM36SpAnGNQ7jmTLn5BnU=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
Expand Down
21 changes: 18 additions & 3 deletions v4/export/dump.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (
"github.com/pingcap/br/pkg/summary"
"github.com/pingcap/errors"
"github.com/pingcap/failpoint"
pclog "github.com/pingcap/log"
pd "github.com/tikv/pd/client"
"go.uber.org/zap"
"golang.org/x/sync/errgroup"
Expand Down Expand Up @@ -212,6 +213,12 @@ func (d *Dumper) Dump() (dumpErr error) {
}
})

// get estimate total count
err = d.getEstimateTotalRowsCount(tctx, metaConn)
if err != nil {
tctx.L().Error("fail to get estimate total count", zap.Error(err))
}

if conf.SQL == "" {
if err = d.dumpDatabases(writerCtx, metaConn, taskChan); err != nil && !errors.ErrorEqual(err, context.Canceled) {
return err
Expand Down Expand Up @@ -610,6 +617,9 @@ func selectTiDBTableSample(conn *sql.Conn, dbName, tableName string) (pkFields [
if hasImplicitRowID {
pkFields, pkColTypes = []string{"_tidb_rowid"}, []string{"BIGINT"}
}
if len(pkFields) == 0 {
return pkFields, pkVals, nil
}
query := buildTiDBTableSampleQuery(pkFields, dbName, tableName)
rows, err := conn.QueryContext(context.Background(), query)
if err != nil {
Expand Down Expand Up @@ -757,19 +767,24 @@ func runSteps(d *Dumper, steps ...func(*Dumper) error) error {

func initLogger(d *Dumper) error {
conf := d.conf
var logger log.Logger
var (
logger log.Logger
err error
props *pclog.ZapProperties
)
// conf.Logger != nil means dumpling is used as a library
if conf.Logger != nil {
logger = log.NewAppLogger(conf.Logger)
} else {
var err error
logger, err = log.InitAppLogger(&log.Config{
logger, props, err = log.InitAppLogger(&log.Config{
Level: conf.LogLevel,
File: conf.LogFile,
Format: conf.LogFormat,
})
if err != nil {
return errors.Trace(err)
}
pclog.ReplaceGlobals(logger.Logger, props)
cli.LogLongVersion(logger)
}
d.tctx = d.tctx.WithLogger(logger)
Expand Down
2 changes: 1 addition & 1 deletion v4/export/dump_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ func (s *testSQLSuite) TestDumpBlock(c *C) {
WillReturnRows(sqlmock.NewRows([]string{"Database", "Create Database"}).
AddRow("test", "CREATE DATABASE `test` /*!40100 DEFAULT CHARACTER SET utf8mb4 */"))

tctx, cancel := tcontext.Background().WithCancel()
tctx, cancel := tcontext.Background().WithLogger(appLogger).WithCancel()
defer cancel()
conn, err := db.Conn(tctx)
c.Assert(err, IsNil)
Expand Down
10 changes: 10 additions & 0 deletions v4/export/metrics.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ var (
finishedSizeCounter *prometheus.CounterVec
finishedRowsCounter *prometheus.CounterVec
finishedTablesCounter *prometheus.CounterVec
estimateTotalRowsCounter *prometheus.CounterVec
writeTimeHistogram *prometheus.HistogramVec
receiveWriteChunkTimeHistogram *prometheus.HistogramVec
errorCount *prometheus.CounterVec
Expand All @@ -33,6 +34,13 @@ func InitMetricsVector(labels prometheus.Labels) {
Name: "finished_size",
Help: "counter for dumpling finished file size",
}, labelNames)
estimateTotalRowsCounter = prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: "dumpling",
Subsystem: "dump",
Name: "estimate_total_rows",
Help: "estimate total rows for dumpling tables",
}, labelNames)
finishedRowsCounter = prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: "dumpling",
Expand Down Expand Up @@ -86,6 +94,7 @@ func RegisterMetrics(registry *prometheus.Registry) {
}
registry.MustRegister(finishedSizeCounter)
registry.MustRegister(finishedRowsCounter)
registry.MustRegister(estimateTotalRowsCounter)
registry.MustRegister(finishedTablesCounter)
registry.MustRegister(writeTimeHistogram)
registry.MustRegister(receiveWriteChunkTimeHistogram)
Expand All @@ -100,6 +109,7 @@ func RemoveLabelValuesWithTaskInMetrics(labels prometheus.Labels) {
}
finishedSizeCounter.Delete(labels)
finishedRowsCounter.Delete(labels)
estimateTotalRowsCounter.Delete(labels)
finishedTablesCounter.Delete(labels)
writeTimeHistogram.Delete(labels)
receiveWriteChunkTimeHistogram.Delete(labels)
Expand Down
23 changes: 20 additions & 3 deletions v4/export/sql.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -514,8 +514,20 @@ func isUnknownSystemVariableErr(err error) bool {
func resetDBWithSessionParams(tctx *tcontext.Context, db *sql.DB, dsn string, params map[string]interface{}) (*sql.DB, error) {
support := make(map[string]interface{})
for k, v := range params {
var pv interface{}
if str, ok := v.(string); ok {
if pvi, err := strconv.ParseInt(str, 10, 64); err == nil {
pv = pvi
} else if pvf, err := strconv.ParseFloat(str, 64); err == nil {
pv = pvf
} else {
pv = str
}
} else {
pv = v
}
s := fmt.Sprintf("SET SESSION %s = ?", k)
Copy link
Collaborator

@lance6716 lance6716 Apr 14, 2021 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Here using ? will let sql driver to check the type of pv, to decide if pv need quotes.

Instead, I prefer always use the input value as-is, that is to say we always use %s and printf to construct a SQL. So user should quote the value if it's a string type variable in SQL.

https://github.com/go-sql-driver/mysql/blob/bcc459a906419e2890a50fc2c99ea6dd927a88f2/connection.go#L68-L78

(I deleted previous misleading comment)

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we might append --param to dsn 😂 otherwise we can't control https://github.com/go-sql-driver/mysql#parameters and have to write a doc about dealing with , in parameter KVs.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Here using ? will let sql driver to check the type of pv, to decide if pv need quotes.

Instead, I prefer always use the input value as-is, that is to say we always use %s and printf to construct a SQL. So user should quote the value if it's a string type variable in SQL.

https://github.com/go-sql-driver/mysql/blob/bcc459a906419e2890a50fc2c99ea6dd927a88f2/connection.go#L68-L78

(I deleted previous misleading comment)

conf.SessionParams is a map[string]interface{}. I think using an original value to construct SQL is in-proper for being used as a library.

Copy link
Collaborator

@lance6716 lance6716 Apr 15, 2021 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

what's the detailed in-proper case?

if user input key=val, successful parsing val as a integer does not mean key is an interger MySQL variable, at that time it will raise an error.

Copy link
Collaborator

@kennytm kennytm Apr 15, 2021 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think that SET SESSION X = 123 and SET SESSION X = '123' are equivalent for system variables.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@lance6716 @kennytm
Fixed in 1b33d20, and add an integration test in 278a128. PTAL again

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

sorry I didn't reply to above comment in time 😂 I didn't found a case for system variables that SET SESSION X = 123 will cause error if X is a string type variable. Will check it now.

_, err := db.ExecContext(tctx, s, v)
_, err := db.ExecContext(tctx, s, pv)
if err != nil {
if isUnknownSystemVariableErr(err) {
tctx.L().Info("session variable is not supported by db", zap.String("variable", k), zap.Reflect("value", v))
Expand All @@ -524,7 +536,7 @@ func resetDBWithSessionParams(tctx *tcontext.Context, db *sql.DB, dsn string, pa
return nil, errors.Trace(err)
}

support[k] = v
support[k] = pv
}

for k, v := range support {
Expand Down Expand Up @@ -837,7 +849,12 @@ func pickupPossibleField(dbName, tableName string, db *sql.Conn, conf *Config) (
}

func estimateCount(tctx *tcontext.Context, dbName, tableName string, db *sql.Conn, field string, conf *Config) uint64 {
query := fmt.Sprintf("EXPLAIN SELECT `%s` FROM `%s`.`%s`", escapeString(field), escapeString(dbName), escapeString(tableName))
var query string
if strings.TrimSpace(field) == "*" || strings.TrimSpace(field) == "" {
query = fmt.Sprintf("EXPLAIN SELECT * FROM `%s`.`%s`", escapeString(dbName), escapeString(tableName))
} else {
query = fmt.Sprintf("EXPLAIN SELECT `%s` FROM `%s`.`%s`", escapeString(field), escapeString(dbName), escapeString(tableName))
}

if conf.Where != "" {
query += " WHERE "
Expand Down
Loading