syncer: fix the case-sensitive issue (#1738)

chaos/cases/task.go

@@ -24,11 +24,10 @@ import (
 	"github.com/chaos-mesh/go-sqlsmith"
 	"github.com/pingcap/errors"
 	"github.com/pingcap/parser/mysql"
+	"github.com/pingcap/tidb-tools/pkg/dbutil"
 	"go.uber.org/zap"
 	"golang.org/x/sync/errgroup"
 
-	"github.com/pingcap/tidb-tools/pkg/dbutil"
-
 	config2 "github.com/pingcap/dm/dm/config"
 	"github.com/pingcap/dm/dm/pb"
 	"github.com/pingcap/dm/pkg/conn"
@@ -90,6 +89,17 @@ func newTask(ctx context.Context, cli pb.MasterClient, taskFile string, schema s
 		if err2 != nil {
 			return nil, err2
 		}
+		if taskCfg.CaseSensitive {
+			lcSetting, err2 := utils.FetchLowerCaseTableNamesSetting(ctx, conn.baseConn.DBConn)
+			if err2 != nil {
+				return nil, err2
+			}
+			if lcSetting == utils.LCTableNamesMixed {
+				msg := "can not set `case-sensitive = true` when upstream `lower_case_table_names = 2`"
+				log.L().Error(msg, zap.Any("instance", cfg))
+				return nil, errors.New(msg)
+			}
+		}
 		sourceDBs = append(sourceDBs, db)
 		sourceConns = append(sourceConns, conn)
 		res = append(res, singleResult{})

go.mod

@@ -24,7 +24,7 @@ require (
 	github.com/pingcap/log v0.0.0-20210317133921-96f4fcab92a4
 	github.com/pingcap/parser v0.0.0-20210415081931-48e7f467fd74
 	github.com/pingcap/tidb v1.1.0-beta.0.20210330094614-60111e1c4b6f
-	github.com/pingcap/tidb-tools v5.0.1-0.20210420102153-beed8ddc59e9+incompatible
+	github.com/pingcap/tidb-tools v5.1.0-alpha.0.20210603090026-288ab02f1c79+incompatible
 	github.com/prometheus/client_golang v1.5.1
 	github.com/rakyll/statik v0.1.6
 	github.com/shopspring/decimal v0.0.0-20200105231215-408a2507e114

go.sum

@@ -847,8 +847,8 @@ github.com/pingcap/tidb-tools v4.0.1+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnw
 github.com/pingcap/tidb-tools v4.0.5-0.20200820082341-afeaaaaaa153+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM=
 github.com/pingcap/tidb-tools v4.0.5-0.20200820092506-34ea90c93237+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM=
 github.com/pingcap/tidb-tools v4.0.9-0.20201127090955-2707c97b3853+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM=
-github.com/pingcap/tidb-tools v5.0.1-0.20210420102153-beed8ddc59e9+incompatible h1:jvsCYfIx30AnyQQxfzoCdzm8xRX0ZRJT3BxpFSKqrTo=
-github.com/pingcap/tidb-tools v5.0.1-0.20210420102153-beed8ddc59e9+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM=
+github.com/pingcap/tidb-tools v5.1.0-alpha.0.20210603090026-288ab02f1c79+incompatible h1:yCmJpDxoxm7kU+K5ORfLmzwLwJ16CdIYoJtP7INyN4w=
+github.com/pingcap/tidb-tools v5.1.0-alpha.0.20210603090026-288ab02f1c79+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM=
 github.com/pingcap/tipb v0.0.0-20190428032612-535e1abaa330/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI=
 github.com/pingcap/tipb v0.0.0-20200417094153-7316d94df1ee/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI=
 github.com/pingcap/tipb v0.0.0-20200604070248-508f03b0b342/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI=

loader/loader.go

@@ -663,7 +663,7 @@ func (l *Loader) skipSchemaAndTable(table *filter.Table) bool {
 	table.Name = unescapePercent(table.Name, l.logger)
 
 	tbs := []*filter.Table{table}
-	tbs = l.baList.ApplyOn(tbs)
+	tbs = l.baList.Apply(tbs)
 	return len(tbs) == 0
 }
 

pkg/parser/common.go

@@ -18,6 +18,7 @@ import (
 
 	"github.com/pingcap/dm/pkg/log"
 	"github.com/pingcap/dm/pkg/terror"
+	"github.com/pingcap/dm/pkg/utils"
 
 	"github.com/pingcap/parser"
 	"github.com/pingcap/parser/ast"
@@ -46,13 +47,20 @@ func Parse(p *parser.Parser, sql, charset, collation string) (stmt []ast.StmtNod
 
 // ref: https://github.com/pingcap/tidb/blob/09feccb529be2830944e11f5fed474020f50370f/server/sql_info_fetcher.go#L46
 type tableNameExtractor struct {
-	curDB string
-	names []*filter.Table
+	curDB  string
+	flavor utils.LowerCaseTableNamesFlavor
+	names  []*filter.Table
 }
 
 func (tne *tableNameExtractor) Enter(in ast.Node) (ast.Node, bool) {
 	if t, ok := in.(*ast.TableName); ok {
-		tb := &filter.Table{Schema: t.Schema.L, Name: t.Name.L}
+		var tb *filter.Table
+		if tne.flavor == utils.LCTableNamesSensitive {
+			tb = &filter.Table{Schema: t.Schema.O, Name: t.Name.O}
+		} else {
+			tb = &filter.Table{Schema: t.Schema.L, Name: t.Name.L}
+		}
+
 		if tb.Schema == "" {
 			tb.Schema = tne.curDB
 		}
@@ -71,7 +79,7 @@ func (tne *tableNameExtractor) Leave(in ast.Node) (ast.Node, bool) {
 // specifically, for `create table like` DDL, result contains [sourceTableName, sourceRefTableName]
 // for rename table ddl, result contains [old1, new1, old1, new1, old2, new2, old3, new3, ...] because of TiDB parser
 // for other DDL, order of tableName is the node visit order.
-func FetchDDLTableNames(schema string, stmt ast.StmtNode) ([]*filter.Table, error) {
+func FetchDDLTableNames(schema string, stmt ast.StmtNode, flavor utils.LowerCaseTableNamesFlavor) ([]*filter.Table, error) {
 	switch stmt.(type) {
 	case ast.DDLNode:
 	default:
@@ -89,8 +97,9 @@ func FetchDDLTableNames(schema string, stmt ast.StmtNode) ([]*filter.Table, erro
 	}
 
 	e := &tableNameExtractor{
-		curDB: schema,
-		names: make([]*filter.Table, 0),
+		curDB:  schema,
+		flavor: flavor,
+		names:  make([]*filter.Table, 0),
 	}
 	stmt.Accept(e)
 

pkg/parser/common_test.go

@@ -21,6 +21,7 @@ import (
 	"github.com/pingcap/tidb-tools/pkg/filter"
 
 	"github.com/pingcap/dm/pkg/terror"
+	"github.com/pingcap/dm/pkg/utils"
 )
 
 var _ = Suite(&testParserSuite{})
@@ -63,11 +64,11 @@ var testCases = []testCase{
 		[]string{"DROP DATABASE IF EXISTS `xs1`"},
 	},
 	{
-		"drop table `s1`.`t1`",
-		[]string{"DROP TABLE IF EXISTS `s1`.`t1`"},
-		[][]*filter.Table{{genTableName("s1", "t1")}},
-		[][]*filter.Table{{genTableName("xs1", "xt1")}},
-		[]string{"DROP TABLE IF EXISTS `xs1`.`xt1`"},
+		"drop table `Ss1`.`tT1`",
+		[]string{"DROP TABLE IF EXISTS `Ss1`.`tT1`"},
+		[][]*filter.Table{{genTableName("Ss1", "tT1")}},
+		[][]*filter.Table{{genTableName("xSs1", "xtT1")}},
+		[]string{"DROP TABLE IF EXISTS `xSs1`.`xtT1`"},
 	},
 	{
 		"drop table `s1`.`t1`, `s2`.`t2`",
@@ -374,7 +375,7 @@ func (t *testParserSuite) TestError(c *C) {
 
 	stmts, err := Parse(p, dml, "", "")
 	c.Assert(err, IsNil)
-	_, err = FetchDDLTableNames("test", stmts[0])
+	_, err = FetchDDLTableNames("test", stmts[0], utils.LCTableNamesInsensitive)
 	c.Assert(terror.ErrUnknownTypeDDL.Equal(err), IsTrue)
 
 	_, err = RenameDDLTable(stmts[0], nil)
@@ -406,7 +407,7 @@ func (t *testParserSuite) TestResolveDDL(c *C) {
 			c.Assert(err, IsNil)
 			c.Assert(s, HasLen, 1)
 
-			tableNames, err := FetchDDLTableNames("test", s[0])
+			tableNames, err := FetchDDLTableNames("test", s[0], utils.LCTableNamesSensitive)
 			c.Assert(err, IsNil)
 			c.Assert(tableNames, DeepEquals, tbs[j])
 

pkg/shardddl/optimism/column.go

@@ -109,3 +109,55 @@ func deleteDroppedColumnByColumnOp(lockID, column string) clientv3.Op {
 func deleteDroppedColumnsByLockOp(lockID string) clientv3.Op {
 	return clientv3.OpDelete(common.ShardDDLOptimismDroppedColumnsKeyAdapter.Encode(lockID), clientv3.WithPrefix())
 }
+
+// deleteSourceDroppedColumnsOp return a DELETE etcd operation for the specified lock relate to a upstream table.
+func deleteSourceDroppedColumnsOp(lockID, column, source, upSchema, upTable string) clientv3.Op {
+	return clientv3.OpDelete(common.ShardDDLOptimismDroppedColumnsKeyAdapter.Encode(lockID, column, source, upSchema, upTable))
+}
+
+// CheckColumns try to check and fix all the schema and table names for delete columns infos.
+func CheckColumns(cli *clientv3.Client, source string, schemaMap map[string]string, tablesMap map[string]map[string]string) error {
+	allColInfos, _, err := GetAllDroppedColumns(cli)
+	if err != nil {
+		return err
+	}
+
+	for lockID, colDropInfo := range allColInfos {
+		for columnName, sourceDropInfo := range colDropInfo {
+			tableInfos, ok := sourceDropInfo[source]
+			if !ok {
+				continue
+			}
+			for schema, tableDropInfo := range tableInfos {
+				realSchema, hasChange := schemaMap[schema]
+				if !hasChange {
+					realSchema = schema
+				}
+				tableMap := tablesMap[schema]
+				for table, stage := range tableDropInfo {
+					realTable, tblChange := tableMap[table]
+					if !tblChange {
+						realTable = table
+						tblChange = hasChange
+					}
+					if tblChange {
+						key := common.ShardDDLOptimismDroppedColumnsKeyAdapter.Encode(lockID, columnName, source, realSchema, realTable)
+						val, err := json.Marshal(stage)
+						if err != nil {
+							return err
+						}
+						opPut := clientv3.OpPut(key, string(val))
+						opDel := deleteSourceDroppedColumnsOp(lockID, columnName, source, schema, table)
+
+						_, _, err = etcdutil.DoOpsInOneTxnWithRetry(cli, opPut, opDel)
+						if err != nil {
+							return err
+						}
+					}
+				}
+			}
+		}
+	}
+
+	return nil
+}

pkg/shardddl/optimism/info.go

@@ -344,3 +344,47 @@ func (oldInfo *OldInfo) toInfo() Info {
 		TableInfosAfter: []*model.TableInfo{oldInfo.TableInfoAfter},
 	}
 }
+
+// CheckDDLInfos try to check and fix all the schema and table names for DDL info.
+func CheckDDLInfos(cli *clientv3.Client, source string, schemaMap map[string]string, tablesMap map[string]map[string]string) error {
+	allInfos, _, err := GetAllInfo(cli)
+	if err != nil {
+		return err
+	}
+
+	for _, taskTableInfos := range allInfos {
+		sourceInfos, ok := taskTableInfos[source]
+		if !ok {
+			continue
+		}
+		for schema, tblInfos := range sourceInfos {
+			realSchema, hasChange := schemaMap[schema]
+			if !hasChange {
+				realSchema = schema
+			}
+
+			tblMap := tablesMap[schema]
+			for tbl, info := range tblInfos {
+				realTable, tableChange := tblMap[tbl]
+				if !tableChange {
+					realTable = tbl
+					tableChange = hasChange
+				}
+				if tableChange {
+					delOp := deleteInfoOp(info)
+					info.UpSchema = realSchema
+					info.UpTable = realTable
+					putOp, err := putInfoOp(info)
+					if err != nil {
+						return err
+					}
+					_, _, err = etcdutil.DoOpsInOneTxnWithRetry(cli, delOp, putOp)
+					if err != nil {
+						return err
+					}
+				}
+			}
+		}
+	}
+	return nil
+}

pkg/shardddl/optimism/operation.go

@@ -282,3 +282,48 @@ func WatchOperationPut(ctx context.Context, cli *clientv3.Client,
 func deleteOperationOp(op Operation) clientv3.Op {
 	return clientv3.OpDelete(common.ShardDDLOptimismOperationKeyAdapter.Encode(op.Task, op.Source, op.UpSchema, op.UpTable))
 }
+
+// CheckOperations try to check and fix all the schema and table names for operation infos.
+func CheckOperations(cli *clientv3.Client, source string, schemaMap map[string]string, tablesMap map[string]map[string]string) error {
+	allOperations, rev, err := GetAllOperations(cli)
+	if err != nil {
+		return err
+	}
+
+	for _, taskTableOps := range allOperations {
+		sourceOps, ok := taskTableOps[source]
+		if !ok {
+			continue
+		}
+		for schema, tblOps := range sourceOps {
+			realSchema, hasChange := schemaMap[schema]
+			if !hasChange {
+				realSchema = schema
+			}
+
+			tblMap := tablesMap[schema]
+			for tbl, info := range tblOps {
+				realTable, tblChange := tblMap[tbl]
+				if !tblChange {
+					realTable = tbl
+					tblChange = hasChange
+				}
+				if tblChange {
+					newOperation := info
+					newOperation.UpSchema = realSchema
+					newOperation.UpTable = realTable
+					_, _, err = PutOperation(cli, false, newOperation, rev)
+					if err != nil {
+						return err
+					}
+					deleteOp := deleteOperationOp(info)
+					_, _, err = etcdutil.DoOpsInOneTxnWithRetry(cli, deleteOp)
+					if err != nil {
+						return err
+					}
+				}
+			}
+		}
+	}
+	return err
+}

pkg/shardddl/optimism/table.go

@@ -307,3 +307,58 @@ func putSourceTablesOp(st SourceTables) (clientv3.Op, error) {
 	key := common.ShardDDLOptimismSourceTablesKeyAdapter.Encode(st.Task, st.Source)
 	return clientv3.OpPut(key, value), nil
 }
+
+// CheckSourceTables try to check and fix all the source schemas and table names.
+func CheckSourceTables(cli *clientv3.Client, source string, schemaMap map[string]string, talesMap map[string]map[string]string) error {
+	allSourceTables, _, err := GetAllSourceTables(cli)
+	if err != nil {
+		return err
+	}
+
+	for _, taskSourceTables := range allSourceTables {
+		sourceTables, ok := taskSourceTables[source]
+		if !ok {
+			continue
+		}
+		schemaKeys := make([]string, 0)
+		tblKeys := make([]string, 0)
+		hasChange := false
+		for _, tableSources := range sourceTables.Tables {
+			for _, sources := range tableSources {
+				for schema, tbls := range sources {
+					if _, ok := schemaMap[schema]; ok {
+						schemaKeys = append(schemaKeys, schema)
+						hasChange = true
+					}
+
+					tblMap, ok := talesMap[schema]
+					if !ok {
+						continue
+					}
+					for tbl := range tbls {
+						if t, ok := tblMap[tbl]; ok {
+							tblKeys = append(tblKeys, t)
+							hasChange = true
+						}
+					}
+					for _, t := range tblKeys {
+						tbls[tblMap[t]] = tbls[t]
+						delete(tbls, t)
+					}
+					tblKeys = tblKeys[:0]
+				}
+				for _, s := range schemaKeys {
+					sources[schemaMap[s]] = sources[s]
+					delete(sources, s)
+				}
+				schemaKeys = schemaKeys[:0]
+			}
+		}
+		if hasChange {
+			if _, err = PutSourceTables(cli, sourceTables); err != nil {
+				return err
+			}
+		}
+	}
+	return err
+}