Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

dumpling: Add support for Create Placement #29724

Merged
merged 29 commits into from
Nov 29, 2021
Merged
Show file tree
Hide file tree
Changes from 27 commits
Commits
Show all changes
29 commits
Select commit Hold shift + click to select a range
e58df44
add mock
sylzd Nov 12, 2021
9086c28
Merge branch 'master' of https://github.com/pingcap/tidb into dumplin…
sylzd Nov 16, 2021
06a1f86
Merge branch 'master' into dumplint_show_create_policy
sylzd Nov 23, 2021
1d2638a
complete function
sylzd Nov 23, 2021
20bd536
fix lint check
sylzd Nov 23, 2021
37b13ec
fix UT
sylzd Nov 23, 2021
eda00c5
add test case
sylzd Nov 23, 2021
788fbc8
fix mistake
sylzd Nov 23, 2021
28785e6
fix old test case
sylzd Nov 23, 2021
0ef7fa1
fix compatibility
sylzd Nov 24, 2021
c7ca8cd
restore dumpblock test
sylzd Nov 24, 2021
c43507c
add anno
sylzd Nov 24, 2021
0aea48a
skip error when encounter sequence
sylzd Nov 24, 2021
47d733e
debug in remote ci
sylzd Nov 24, 2021
e2adebc
change the tidb version
sylzd Nov 25, 2021
13aafdf
change tidb version to master
sylzd Nov 25, 2021
b975983
fix basic test for easy debug
sylzd Nov 25, 2021
0f51610
Update dumpling/tests/placement_policy/run.sh
sylzd Nov 25, 2021
6f7d245
skip the unsupport
sylzd Nov 25, 2021
fd75073
Merge branch 'dumplint_show_create_policy' of github.com:sylzd/tidb i…
sylzd Nov 25, 2021
778be14
fix
sylzd Nov 25, 2021
1ab82da
fix lint
sylzd Nov 25, 2021
4fe1e5f
fix error
sylzd Nov 26, 2021
a774f6b
Update dumpling/export/dump.go
sylzd Nov 29, 2021
3d8bd89
Update dumpling/export/sql.go
sylzd Nov 29, 2021
f52b863
Update dumpling/export/dump.go
sylzd Nov 29, 2021
c028f54
fix error info
sylzd Nov 29, 2021
f46b4a7
Merge branch 'master' into dumplint_show_create_policy
lichunzhu Nov 29, 2021
a472230
Merge branch 'master' into dumplint_show_create_policy
ti-chi-bot Nov 29, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 29 additions & 0 deletions dumpling/export/dump.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"database/sql"
"encoding/hex"
"fmt"
"github.com/go-sql-driver/mysql"
"math/big"
"sort"
"strconv"
Expand Down Expand Up @@ -298,6 +299,34 @@ func (d *Dumper) startWriters(tctx *tcontext.Context, wg *errgroup.Group, taskCh
func (d *Dumper) dumpDatabases(tctx *tcontext.Context, metaConn *sql.Conn, taskChan chan<- Task) error {
conf := d.conf
allTables := conf.Tables

// policy should be created before database
// placement policy in other server type can be different, so we only handle the tidb server
if conf.ServerInfo.ServerType == version.ServerTypeTiDB {
policyNames, err := ListAllPlacementPolicyNames(metaConn)
sylzd marked this conversation as resolved.
Show resolved Hide resolved
if err != nil {
errCause := errors.Cause(err)
if mysqlErr, ok := errCause.(*mysql.MySQLError); ok && mysqlErr.Number == ErrNoSuchTable {
// some old tidb version and other server type doesn't support placement rules, we can skip it.
tctx.L().Debug("cannot dump placement policy, maybe the server doesn't support it", log.ShortError(err))
} else {
tctx.L().Warn("fail to dump placement policy: ", log.ShortError(err))
}
}
for _, policy := range policyNames {
createPolicySQL, err := ShowCreatePlacementPolicy(metaConn, policy)
if err != nil {
return err
}
wrappedCreatePolicySQL := fmt.Sprintf("/*T![placement] %s */", createPolicySQL)
task := NewTaskPolicyMeta(policy, wrappedCreatePolicySQL)
ctxDone := d.sendTaskToChan(tctx, task, taskChan)
if ctxDone {
return tctx.Err()
}
}
}

for dbName, tables := range allTables {
if !conf.NoSchemas {
createDatabaseSQL, err := ShowCreateDatabase(metaConn, dbName)
Expand Down
1 change: 1 addition & 0 deletions dumpling/export/dump_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ func TestDumpBlock(t *testing.T) {
taskChan := make(chan Task, 1)
taskChan <- &TaskDatabaseMeta{}
d.conf.Tables = DatabaseTables{}.AppendTable(database, nil)
d.conf.ServerInfo.ServerType = version.ServerTypeMySQL
require.ErrorIs(t, d.dumpDatabases(writerCtx, conn, taskChan), context.Canceled)
require.ErrorIs(t, wg.Wait(), writerErr)
}
Expand Down
34 changes: 34 additions & 0 deletions dumpling/export/sql.go
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,21 @@ func ShowCreateTable(db *sql.Conn, database, table string) (string, error) {
return oneRow[1], nil
}

// ShowCreatePlacementPolicy constructs the create policy SQL for a specified table
// returns (createPoilicySQL, error)
func ShowCreatePlacementPolicy(db *sql.Conn, policy string) (string, error) {
var oneRow [2]string
handleOneRow := func(rows *sql.Rows) error {
return rows.Scan(&oneRow[0], &oneRow[1])
}
query := fmt.Sprintf("SHOW CREATE PLACEMENT POLICY `%s`", escapeString(policy))
err := simpleQuery(db, query, handleOneRow)
if err != nil {
return "", errors.Annotatef(err, "sql: %s", query)
}
return oneRow[1], nil
}

// ShowCreateView constructs the create view SQL for a specified view
// returns (createFakeTableSQL, createViewSQL, error)
func ShowCreateView(db *sql.Conn, database, view string) (createFakeTableSQL string, createRealViewSQL string, err error) {
Expand Down Expand Up @@ -279,6 +294,25 @@ func ListAllDatabasesTables(tctx *tcontext.Context, db *sql.Conn, databaseNames
return dbTables, nil
}

func ListAllPlacementPolicyNames(db *sql.Conn) ([]string, error) {
var policyList []string
var policy string
const query = "select distinct policy_name from information_schema.placement_rules where policy_name is not null;"
rows, err := db.QueryContext(context.Background(), query)
if err != nil {
return policyList, errors.Annotatef(err, "sql: %s", query)
}
defer rows.Close()
for rows.Next() {
err := rows.Scan(&policy)
if err != nil {
return policyList, errors.Trace(err)
}
policyList = append(policyList, policy)
}
return policyList, errors.Annotatef(rows.Err(), "sql: %s", query)
}

// SelectVersion gets the version information from the database server
func SelectVersion(db *sql.DB) (string, error) {
var versionInfo string
Expand Down
54 changes: 54 additions & 0 deletions dumpling/export/sql_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"encoding/csv"
"encoding/json"
"fmt"
"github.com/go-sql-driver/mysql"
"io"
"os"
"path"
Expand Down Expand Up @@ -340,6 +341,59 @@ func TestShowCreateView(t *testing.T) {
require.NoError(t, mock.ExpectationsWereMet())
}

func TestShowCreatePolicy(t *testing.T) {
t.Parallel()

db, mock, err := sqlmock.New()
require.NoError(t, err)
defer func() {
require.NoError(t, db.Close())
}()

conn, err := db.Conn(context.Background())
require.NoError(t, err)

mock.ExpectQuery("SHOW CREATE PLACEMENT POLICY `policy_x`").
WillReturnRows(sqlmock.NewRows([]string{"Policy", "Create Policy"}).
AddRow("policy_x", "CREATE PLACEMENT POLICY `policy_x` LEARNERS=1"))

createPolicySQL, err := ShowCreatePlacementPolicy(conn, "policy_x")
require.NoError(t, err)
require.Equal(t, "CREATE PLACEMENT POLICY `policy_x` LEARNERS=1", createPolicySQL)
require.NoError(t, mock.ExpectationsWereMet())

}

func TestListPolicyNames(t *testing.T) {
t.Parallel()

db, mock, err := sqlmock.New()
require.NoError(t, err)
defer func() {
require.NoError(t, db.Close())
}()

conn, err := db.Conn(context.Background())
require.NoError(t, err)

mock.ExpectQuery("select distinct policy_name from information_schema.placement_rules where policy_name is not null;").
WillReturnRows(sqlmock.NewRows([]string{"policy_name"}).
AddRow("policy_x"))
policies, err := ListAllPlacementPolicyNames(conn)
require.NoError(t, err)
require.Equal(t, []string{"policy_x"}, policies)
require.NoError(t, mock.ExpectationsWereMet())

// some old tidb version doesn't support placement rules returns error
expectedErr := &mysql.MySQLError{Number: ErrNoSuchTable, Message: "Table 'information_schema.placement_rules' doesn't exist"}
mock.ExpectExec("select distinct policy_name from information_schema.placement_rules where policy_name is not null;").
WillReturnError(expectedErr)
policies, err = ListAllPlacementPolicyNames(conn)
if mysqlErr, ok := err.(*mysql.MySQLError); ok {
require.Equal(t, mysqlErr.Number, ErrNoSuchTable)
}
}

func TestGetSuitableRows(t *testing.T) {
t.Parallel()

Expand Down
22 changes: 21 additions & 1 deletion dumpling/export/task.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ package export

import "fmt"

// Task is a file dump task for dumpling, it could either be dumping database/table/view metadata, table data
// Task is a file dump task for dumpling, it could either be dumping database/table/view/policy metadata, table data
type Task interface {
// Brief is the brief for a dumping task
Brief() string
Expand Down Expand Up @@ -34,6 +34,13 @@ type TaskViewMeta struct {
CreateViewSQL string
}

// TaskPolicyMeta is a dumping view metadata task
type TaskPolicyMeta struct {
Task
PolicyName string
CreatePolicySQL string
}

// TaskTableData is a dumping table data task
type TaskTableData struct {
Task
Expand Down Expand Up @@ -70,6 +77,14 @@ func NewTaskViewMeta(dbName, tblName, createTableSQL, createViewSQL string) *Tas
}
}

// NewTaskPolicyMeta returns a new dumping placement policy metadata task
func NewTaskPolicyMeta(policyName, createPolicySQL string) *TaskPolicyMeta {
return &TaskPolicyMeta{
PolicyName: policyName,
CreatePolicySQL: createPolicySQL,
}
}

// NewTaskTableData returns a new dumping table data task
func NewTaskTableData(meta TableMeta, data TableDataIR, currentChunk, totalChunks int) *TaskTableData {
return &TaskTableData{
Expand All @@ -95,6 +110,11 @@ func (t *TaskViewMeta) Brief() string {
return fmt.Sprintf("meta of view '%s'.'%s'", t.DatabaseName, t.ViewName)
}

// Brief implements task.Brief
func (t *TaskPolicyMeta) Brief() string {
return fmt.Sprintf("meta of placement policy '%s'", t.PolicyName)
}

// Brief implements task.Brief
func (t *TaskTableData) Brief() string {
db, tbl := t.Meta.DatabaseName(), t.Meta.TableName()
Expand Down
2 changes: 2 additions & 0 deletions dumpling/export/writer.go
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,8 @@ func (w *Writer) handleTask(task Task) error {
return w.WriteTableMeta(t.DatabaseName, t.TableName, t.CreateTableSQL)
case *TaskViewMeta:
return w.WriteViewMeta(t.DatabaseName, t.ViewName, t.CreateTableSQL, t.CreateViewSQL)
case *TaskPolicyMeta:
return w.WritePolicyMeta(t.PolicyName, t.CreatePolicySQL)
case *TaskTableData:
err := w.WriteTableData(t.Meta, t.Data, t.ChunkIndex)
if err != nil {
Expand Down
2 changes: 1 addition & 1 deletion dumpling/install.sh
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ cd $pwd/tidb-lightning && make
cd $pwd
mv tidb-lightning/bin/tidb-lightning bin/

TIDB_TAG="v4.0.4"
TIDB_TAG="master"
# download tidb-server
git clone -b $TIDB_TAG https://github.com/pingcap/tidb
cd $pwd/tidb && make
Expand Down
2 changes: 1 addition & 1 deletion dumpling/tests/basic/run.sh
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ echo "expected 2, actual ${actual}"

# Test for tidb_mem_quota_query configuration
export GO_FAILPOINTS="github.com/pingcap/tidb/dumpling/export/PrintTiDBMemQuotaQuery=1*return"
run_dumpling > ${DUMPLING_OUTPUT_DIR}/dumpling.log
run_dumpling | tee ${DUMPLING_OUTPUT_DIR}/dumpling.log
actual=$(grep -w "tidb_mem_quota_query == 1073741824" ${DUMPLING_OUTPUT_DIR}/dumpling.log|wc -l)
echo "expected 1, actual ${actual}"
[ "$actual" = 1 ]
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
/*!40101 SET NAMES binary*/;
/*T![placement] SET PLACEMENT_CHECKS = 0*/;
/*T![placement] CREATE PLACEMENT POLICY `x` PRIMARY_REGION="cn-east-1" REGIONS="cn-east-1,cn-east" */;
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
/*!40101 SET NAMES binary*/;
/*T![placement] SET PLACEMENT_CHECKS = 0*/;
/*T![placement] CREATE PLACEMENT POLICY `x1` FOLLOWERS=4 */;
30 changes: 30 additions & 0 deletions dumpling/tests/placement_policy/run.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
#!/bin/sh
#
# Copyright 2021 PingCAP, Inc. Licensed under Apache-2.0.

set -eu

export DUMPLING_TEST_PORT=4000

run_sql "drop database if exists policy"
run_sql "drop placement policy if exists x"
run_sql "drop placement policy if exists x1"
run_sql "create database policy"

export DUMPLING_TEST_DATABASE="policy"

run_sql 'CREATE PLACEMENT POLICY x PRIMARY_REGION="cn-east-1" REGIONS="cn-east-1,cn-east";'
run_sql 'CREATE PLACEMENT POLICY x1 FOLLOWERS=4;'

run_dumpling

file_should_exist "$DUMPLING_OUTPUT_DIR/policy-schema-create.sql"
file_should_exist "$DUMPLING_OUTPUT_DIR/x-placement-policy-create.sql"
file_should_exist "$DUMPLING_OUTPUT_DIR/x1-placement-policy-create.sql"

diff "$DUMPLING_BASE_NAME/result/x-placement-policy-create.sql" "$DUMPLING_OUTPUT_DIR/x-placement-policy-create.sql"
diff "$DUMPLING_BASE_NAME/result/x1-placement-policy-create.sql" "$DUMPLING_OUTPUT_DIR/x1-placement-policy-create.sql"
sylzd marked this conversation as resolved.
Show resolved Hide resolved

run_sql "drop database if exists policy"
run_sql "drop placement policy if exists x"
run_sql "drop placement policy if exists x1"