loader, syncer: Update sql_mode to more compatible (#1869) (#1894)

loader/loader.go

@@ -528,10 +528,17 @@ func (l *Loader) Init(ctx context.Context) (err error) {
  break
  }
  }
+
  if !hasSQLMode {
- lcfg.To.Session["sql_mode"] = l.cfg.LoaderConfig.SQLMode
+ sqlModes, err3 := utils.AdjustSQLModeCompatible(l.cfg.LoaderConfig.SQLMode)
+ if err3 != nil {
+ l.logger.Warn("cannot adjust sql_mode compatible, the sql_mode will stay the same", log.ShortError(err3))
+ }
+ lcfg.To.Session["sql_mode"] = sqlModes
  }
 
+ l.logger.Info("loader's sql_mode is", zap.String("sqlmode", lcfg.To.Session["sql_mode"]))
+
  l.toDB, l.toDBConns, err = createConns(tctx, lcfg, l.cfg.PoolSize)
  if err != nil {
  return err

pkg/utils/db.go

@@ -524,3 +524,56 @@ func AddGSetWithPurged(ctx context.Context, gset gtid.Set, conn *sql.Conn) (gtid
  _ = ret.Set(newGset)
  return ret, nil
 }
+
+// AdjustSQLModeCompatible adjust downstream sql mode to compatible.
+// TODO: When upstream's datatime is 2020-00-00, 2020-00-01, 2020-06-00
+// and so on, downstream will be 2019-11-30, 2019-12-01, 2020-05-31,
+// as if set the 'NO_ZERO_IN_DATE', 'NO_ZERO_DATE'.
+// This is because the implementation of go-mysql, that you can see
+// https://github.com/go-mysql-org/go-mysql/blob/master/replication/row_event.go#L1063-L1087
+func AdjustSQLModeCompatible(sqlModes string) (string, error) {
+ needDisable := []string{
+ "NO_ZERO_IN_DATE",
+ "NO_ZERO_DATE",
+ "ERROR_FOR_DIVISION_BY_ZERO",
+ "NO_AUTO_CREATE_USER",
+ "STRICT_TRANS_TABLES",
+ "STRICT_ALL_TABLES",
+ }
+ needEnable := []string{
+ "IGNORE_SPACE",
+ "NO_AUTO_VALUE_ON_ZERO",
+ "ALLOW_INVALID_DATES",
+ }
+ disable := strings.Join(needDisable, ",")
+ enable := strings.Join(needEnable, ",")
+
+ mode, err := tmysql.GetSQLMode(sqlModes)
+ if err != nil {
+ return sqlModes, err
+ }
+ disableMode, err2 := tmysql.GetSQLMode(disable)
+ if err2 != nil {
+ return sqlModes, err2
+ }
+ enableMode, err3 := tmysql.GetSQLMode(enable)
+ if err3 != nil {
+ return sqlModes, err3
+ }
+ // About this bit manipulation, details can be seen
+ // https://github.com/pingcap/dm/pull/1869#discussion_r669771966
+ mode = (mode &^ disableMode) | enableMode
+
+ return GetSQLModeStrBySQLMode(mode), nil
+}
+
+// GetSQLModeStrBySQLMode get string represent of sql_mode by sql_mode.
+func GetSQLModeStrBySQLMode(sqlMode tmysql.SQLMode) string {
+ var sqlModeStr []string
+ for str, SQLMode := range tmysql.Str2SQLMode {
+ if sqlMode&SQLMode != 0 {
+ sqlModeStr = append(sqlModeStr, str)
+ }
+ }
+ return strings.Join(sqlModeStr, ",")
+}

syncer/syncer.go

@@ -2777,10 +2777,13 @@ func (s *Syncer) createDBs(ctx context.Context) error {
  if !hasSQLMode {
  sqlMode, err2 := utils.GetGlobalVariable(ctx, s.fromDB.BaseDB.DB, "sql_mode")
  if err2 != nil {
- s.tctx.L().Warn("cannot get sql_mode from upstream database", log.ShortError(err2))
- } else {
- s.cfg.To.Session["sql_mode"] = sqlMode
+ s.tctx.L().Warn("cannot get sql_mode from upstream database, the sql_mode will be assigned \"IGNORE_SPACE, NO_AUTO_VALUE_ON_ZERO, ALLOW_INVALID_DATES\"", log.ShortError(err2))
+ }
+ sqlModes, err3 := utils.AdjustSQLModeCompatible(sqlMode)
+ if err3 != nil {
+ s.tctx.L().Warn("cannot adjust sql_mode compatible, the sql_mode will be assigned stay the same", log.ShortError(err3))
  }
+ s.cfg.To.Session["sql_mode"] = sqlModes
  }
 
  dbCfg = s.cfg.To

tests/handle_error/conf/double-source-no-sharding.yaml

@@ -10,6 +10,9 @@ target-database:
  user: "test"
  password: "/Q7B9DizNLLTTfiZHv9WoEAKamfpIUs="
 
+ session:
+ sql_mode: "STRICT_TRANS_TABLES"
+
 mysql-instances:
  - source-id: "mysql-replica-01"
  block-allow-list: "instance"

tests/others_integration.txt

@@ -11,3 +11,5 @@ downstream_more_column
 expression_filter
 fake_rotate_event
 metrics
+sql_mode
+

tests/sql_mode/conf/diff_config.toml

@@ -0,0 +1,56 @@
+# diff Configuration.
+
+log-level = "info"
+
+chunk-size = 20
+
+check-thread-count = 4
+
+sample-percent = 100
+
+use-checksum = true
+
+fix-sql-file = "fix.sql"
+
+# tables need to check.
+[[check-tables]]
+schema = "sql_mode"
+tables = ["~.*"]
+
+[[source-db]]
+host = "127.0.0.1"
+port = 3306
+user = "root"
+password = "123456"
+instance-id = "source-1"
+
+[[table-config]]
+schema = "sql_mode"
+table = "t_1"
+
+[[table-config.source-tables]]
+instance-id = "source-1"
+schema = "sql_mode"
+table = "t_1"
+
+[[source-db]]
+host = "127.0.0.1"
+port = 3307
+user = "root"
+password = "123456"
+instance-id = "source-2"
+
+[[table-config]]
+schema = "sql_mode"
+table = "t_2"
+
+[[table-config.source-tables]]
+instance-id = "source-2"
+schema = "sql_mode"
+table = "t_2"
+
+[target-db]
+host = "127.0.0.1"
+port = 4000
+user = "test"
+password = "123456"

tests/sql_mode/conf/dm-master.toml

@@ -0,0 +1,4 @@
+# Master Configuration.
+master-addr = ":8261"
+advertise-addr = "127.0.0.1:8261"
+auto-compaction-retention = "3s"

tests/sql_mode/conf/dm-task.yaml

@@ -0,0 +1,46 @@
+---
+name: sql_mode
+task-mode: all
+is-sharding: false
+meta-schema: "dm_meta"
+enable-heartbeat: false
+
+target-database:
+ host: "127.0.0.1"
+ port: 4000
+ user: "root"
+ password: ""
+
+mysql-instances:
+ - source-id: "mysql-replica-01"
+ block-allow-list: "instance"
+ mydumper-config-name: "global"
+ loader-config-name: "global"
+ syncer-config-name: "global"
+
+ - source-id: "mysql-replica-02"
+ block-allow-list: "instance"
+ mydumper-config-name: "global"
+ loader-config-name: "global"
+ syncer-config-name: "global"
+
+block-allow-list:
+ instance:
+ do-dbs: ["sql_mode"]
+
+mydumpers:
+ global:
+ threads: 4
+ chunk-filesize: 0
+ skip-tz-utc: true
+ extra-args: "--statement-size=4000"
+
+loaders:
+ global:
+ pool-size: 16
+ dir: "./dumped_data"
+
+syncers:
+ global:
+ worker-count: 16
+ batch: 100

tests/sql_mode/conf/dm-worker1.toml

@@ -0,0 +1,2 @@
+name = "worker1"
+join = "127.0.0.1:8261"

tests/sql_mode/conf/dm-worker2.toml

@@ -0,0 +1,2 @@
+name = "worker2"
+join = "127.0.0.1:8261"

tests/sql_mode/conf/source1.yaml

@@ -0,0 +1,11 @@
+source-id: mysql-replica-01
+flavor: ''
+enable-gtid: false
+enable-relay: true
+relay-binlog-name: ''
+relay-binlog-gtid: ''
+from:
+ host: 127.0.0.1
+ user: root
+ password: /Q7B9DizNLLTTfiZHv9WoEAKamfpIUs=
+ port: 3306

tests/sql_mode/conf/source2.yaml

@@ -0,0 +1,11 @@
+source-id: mysql-replica-02
+flavor: ''
+enable-gtid: false
+enable-relay: true
+relay-binlog-name: ''
+relay-binlog-gtid: ''
+from:
+ host: 127.0.0.1
+ user: root
+ password: /Q7B9DizNLLTTfiZHv9WoEAKamfpIUs=
+ port: 3307

tests/sql_mode/data/db1.increment.sql

@@ -0,0 +1,32 @@
+set @@session.SQL_MODE='';
+use `sql_mode`;
+
+-- test sql_mode PIPES_AS_CONCAT
+insert into t_1(num) values('pipes'||'as'||'concat');
+
+-- test sql_mode ANSI_QUOTES
+insert into t_1(name) values("a");
+
+-- test sql_mode IGNORE_SPACE
+create table sum (id int not null, primary key(id));
+
+-- test sql_mode NO_AUTO_VALUE_ON_ZERO
+insert into t_1(id, name) values (30, 'a');
+insert into t_1(id, name) values (0, 'b');
+insert into t_1(id, name) values (0, 'c');
+
+-- test sql_mode NO_BACKSLASH_ESCAPES
+insert into t_1(name) values ('\\b');
+
+-- test sql_mode STRICT_TRANS_TABLES && STRICT_ALL_TABLES && NO_ZERO_IN_DATE && NO_ZERO_DATE && ALLOW_INVALID_DATES
+insert into t_1(dt) values('0000-06-00');
+insert into t_1(dt) values('0000-00-01');
+insert into t_1(dt) values('0000-06-01');
+insert into t_1(dt) values('0000-00-00');
+
+-- test sql_mode ERROR_FOR_DIVISION_BY_ZERO
+insert into t_1(num) values(4/0);
+
+-- test sql_mode NO_AUTO_CREATE_USER
+drop user if exists 'no_auto_create_user';
+grant select on *.* to 'no_auto_create_user';

tests/sql_mode/data/db1.prepare.sql

@@ -0,0 +1,42 @@
+set @@session.SQL_MODE='';
+
+drop database if exists `sql_mode`;
+create database `sql_mode`;
+use `sql_mode`;
+CREATE TABLE `t_1` (
+ `id` int NOT NULL AUTO_INCREMENT,
+ `name` varchar(60),
+ `num` int,
+ `dt` datetime,
+ PRIMARY KEY (id)
+);
+
+-- test sql_mode PIPES_AS_CONCAT
+insert into t_1(num) values('pipes'||'as'||'concat');
+
+-- test sql_mode ANSI_QUOTES
+insert into t_1(name) values("a");
+
+-- test sql_mode IGNORE_SPACE
+create table count (id int not null, primary key(id));
+
+-- test sql_mode NO_AUTO_VALUE_ON_ZERO
+insert into t_1(id, name) values (10, 'a');
+insert into t_1(id, name) values (0, 'b');
+insert into t_1(id, name) values (0, 'c');
+
+-- test sql_mode NO_BACKSLASH_ESCAPES
+insert into t_1(name) values ('\\a');
+
+-- test sql_mode STRICT_TRANS_TABLES && STRICT_ALL_TABLES && NO_ZERO_IN_DATE && NO_ZERO_DATE && ALLOW_INVALID_DATES
+insert into t_1(dt) values('0000-06-00');
+insert into t_1(dt) values('0000-00-01');
+insert into t_1(dt) values('0000-06-01');
+insert into t_1(dt) values('0000-00-00');
+
+-- test sql_mode ERROR_FOR_DIVISION_BY_ZERO
+insert into t_1(num) values(4/0);
+
+-- test sql_mode NO_AUTO_CREATE_USER
+drop user if exists 'no_auto_create_user';
+grant select on *.* to 'no_auto_create_user';

tests/sql_mode/data/db2.increment.sql

@@ -0,0 +1,24 @@
+set @@session.sql_mode='ONLY_FULL_GROUP_BY,NO_UNSIGNED_SUBTRACTION,NO_DIR_IN_CREATE,STRICT_TRANS_TABLES,STRICT_ALL_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ALLOW_INVALID_DATES,ERROR_FOR_DIVISION_BY_ZERO,HIGH_NOT_PRECEDENCE,NO_ENGINE_SUBSTITUTION,REAL_AS_FLOAT';
+-- NO_AUTO_CREATE_USER set failed in mysql8.0
+
+use sql_mode;
+
+-- test sql_mode PIPES_AS_CONCAT
+set @@session.sql_mode=concat(@@session.sql_mode, ',PIPES_AS_CONCAT');
+insert into t_2(name) values('pipes'||'as'||'concat');
+
+-- test sql_mode ANSI_QUOTES
+insert into t_2(name) values("a");
+
+-- test sql_mode IGNORE_SPACE
+set @@session.sql_mode=concat(@@session.sql_mode, ',IGNORE_SPACE');
+insert into t_2(name) values(concat ('ignore', 'space'));
+
+-- test sql_mode NO_AUTO_VALUE_ON_ZERO
+set @@session.sql_mode=concat(@@session.sql_mode, ',NO_AUTO_VALUE_ON_ZERO');
+insert into t_2(id, name) values (20, 'a');
+replace into t_2(id, name) values (0, 'c');
+
+-- test sql_mode NO_BACKSLASH_ESCAPES
+set @@session.sql_mode=concat(@@session.sql_mode, ',NO_BACKSLASH_ESCAPES');
+insert into t_2(name) values ('\\a');

tests/sql_mode/data/db2.prepare.sql

@@ -0,0 +1,33 @@
+set @@session.sql_mode='ONLY_FULL_GROUP_BY,NO_UNSIGNED_SUBTRACTION,NO_DIR_IN_CREATE,STRICT_TRANS_TABLES,STRICT_ALL_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ALLOW_INVALID_DATES,ERROR_FOR_DIVISION_BY_ZERO,HIGH_NOT_PRECEDENCE,NO_ENGINE_SUBSTITUTION,REAL_AS_FLOAT';
+-- NO_AUTO_CREATE_USER set failed in mysql8.0
+
+drop database if exists `sql_mode`;
+create database `sql_mode`;
+use `sql_mode`;
+CREATE TABLE `t_2` (
+ `id` int NOT NULL AUTO_INCREMENT,
+ `name` varchar(60),
+ `num` int,
+ `dt` datetime,
+ PRIMARY KEY (id)
+);
+
+-- test sql_mode PIPES_AS_CONCAT
+set @@session.sql_mode=concat(@@session.sql_mode, ',PIPES_AS_CONCAT');
+insert into t_2(name) values('pipes'||'as'||'concat');
+
+-- test sql_mode ANSI_QUOTES
+insert into t_2(name) values("a");
+
+-- test sql_mode IGNORE_SPACE
+set @@session.sql_mode=concat(@@session.sql_mode, ',IGNORE_SPACE');
+insert into t_2(name) values(concat ('ignore', 'space'));
+
+-- test sql_mode NO_AUTO_VALUE_ON_ZERO
+set @@session.sql_mode=concat(@@session.sql_mode, ',NO_AUTO_VALUE_ON_ZERO');
+insert into t_2(id, name) values (10, 'a');
+insert into t_2(id, name) values (0, 'b');
+
+-- test sql_mode NO_BACKSLASH_ESCAPES
+set @@session.sql_mode=concat(@@session.sql_mode, ',NO_BACKSLASH_ESCAPES');
+insert into t_2(name) values ('\\a');