diff --git a/loader/loader.go b/loader/loader.go index d9c835ed63..23c8a4da4b 100644 --- a/loader/loader.go +++ b/loader/loader.go @@ -528,10 +528,17 @@ func (l *Loader) Init(ctx context.Context) (err error) { break } } + if !hasSQLMode { - lcfg.To.Session["sql_mode"] = l.cfg.LoaderConfig.SQLMode + sqlModes, err3 := utils.AdjustSQLModeCompatible(l.cfg.LoaderConfig.SQLMode) + if err3 != nil { + l.logger.Warn("cannot adjust sql_mode compatible, the sql_mode will stay the same", log.ShortError(err3)) + } + lcfg.To.Session["sql_mode"] = sqlModes } + l.logger.Info("loader's sql_mode is", zap.String("sqlmode", lcfg.To.Session["sql_mode"])) + l.toDB, l.toDBConns, err = createConns(tctx, lcfg, l.cfg.PoolSize) if err != nil { return err diff --git a/pkg/utils/db.go b/pkg/utils/db.go index f935c43bc5..948126c972 100644 --- a/pkg/utils/db.go +++ b/pkg/utils/db.go @@ -524,3 +524,56 @@ func AddGSetWithPurged(ctx context.Context, gset gtid.Set, conn *sql.Conn) (gtid _ = ret.Set(newGset) return ret, nil } + +// AdjustSQLModeCompatible adjust downstream sql mode to compatible. +// TODO: When upstream's datatime is 2020-00-00, 2020-00-01, 2020-06-00 +// and so on, downstream will be 2019-11-30, 2019-12-01, 2020-05-31, +// as if set the 'NO_ZERO_IN_DATE', 'NO_ZERO_DATE'. +// This is because the implementation of go-mysql, that you can see +// https://github.com/go-mysql-org/go-mysql/blob/master/replication/row_event.go#L1063-L1087 +func AdjustSQLModeCompatible(sqlModes string) (string, error) { + needDisable := []string{ + "NO_ZERO_IN_DATE", + "NO_ZERO_DATE", + "ERROR_FOR_DIVISION_BY_ZERO", + "NO_AUTO_CREATE_USER", + "STRICT_TRANS_TABLES", + "STRICT_ALL_TABLES", + } + needEnable := []string{ + "IGNORE_SPACE", + "NO_AUTO_VALUE_ON_ZERO", + "ALLOW_INVALID_DATES", + } + disable := strings.Join(needDisable, ",") + enable := strings.Join(needEnable, ",") + + mode, err := tmysql.GetSQLMode(sqlModes) + if err != nil { + return sqlModes, err + } + disableMode, err2 := tmysql.GetSQLMode(disable) + if err2 != nil { + return sqlModes, err2 + } + enableMode, err3 := tmysql.GetSQLMode(enable) + if err3 != nil { + return sqlModes, err3 + } + // About this bit manipulation, details can be seen + // https://github.com/pingcap/dm/pull/1869#discussion_r669771966 + mode = (mode &^ disableMode) | enableMode + + return GetSQLModeStrBySQLMode(mode), nil +} + +// GetSQLModeStrBySQLMode get string represent of sql_mode by sql_mode. +func GetSQLModeStrBySQLMode(sqlMode tmysql.SQLMode) string { + var sqlModeStr []string + for str, SQLMode := range tmysql.Str2SQLMode { + if sqlMode&SQLMode != 0 { + sqlModeStr = append(sqlModeStr, str) + } + } + return strings.Join(sqlModeStr, ",") +} diff --git a/syncer/syncer.go b/syncer/syncer.go index 40aba960ca..9a8f09db3a 100644 --- a/syncer/syncer.go +++ b/syncer/syncer.go @@ -2777,10 +2777,13 @@ func (s *Syncer) createDBs(ctx context.Context) error { if !hasSQLMode { sqlMode, err2 := utils.GetGlobalVariable(ctx, s.fromDB.BaseDB.DB, "sql_mode") if err2 != nil { - s.tctx.L().Warn("cannot get sql_mode from upstream database", log.ShortError(err2)) - } else { - s.cfg.To.Session["sql_mode"] = sqlMode + s.tctx.L().Warn("cannot get sql_mode from upstream database, the sql_mode will be assigned \"IGNORE_SPACE, NO_AUTO_VALUE_ON_ZERO, ALLOW_INVALID_DATES\"", log.ShortError(err2)) + } + sqlModes, err3 := utils.AdjustSQLModeCompatible(sqlMode) + if err3 != nil { + s.tctx.L().Warn("cannot adjust sql_mode compatible, the sql_mode will be assigned stay the same", log.ShortError(err3)) } + s.cfg.To.Session["sql_mode"] = sqlModes } dbCfg = s.cfg.To diff --git a/tests/handle_error/conf/double-source-no-sharding.yaml b/tests/handle_error/conf/double-source-no-sharding.yaml index c483994ec3..2da17837c7 100644 --- a/tests/handle_error/conf/double-source-no-sharding.yaml +++ b/tests/handle_error/conf/double-source-no-sharding.yaml @@ -10,6 +10,9 @@ target-database: user: "test" password: "/Q7B9DizNLLTTfiZHv9WoEAKamfpIUs=" + session: + sql_mode: "STRICT_TRANS_TABLES" + mysql-instances: - source-id: "mysql-replica-01" block-allow-list: "instance" diff --git a/tests/others_integration.txt b/tests/others_integration.txt index 9ece51e2b2..37e85735df 100644 --- a/tests/others_integration.txt +++ b/tests/others_integration.txt @@ -11,3 +11,5 @@ downstream_more_column expression_filter fake_rotate_event metrics +sql_mode + diff --git a/tests/sql_mode/conf/diff_config.toml b/tests/sql_mode/conf/diff_config.toml new file mode 100644 index 0000000000..29ffdd5b0d --- /dev/null +++ b/tests/sql_mode/conf/diff_config.toml @@ -0,0 +1,56 @@ +# diff Configuration. + +log-level = "info" + +chunk-size = 20 + +check-thread-count = 4 + +sample-percent = 100 + +use-checksum = true + +fix-sql-file = "fix.sql" + +# tables need to check. +[[check-tables]] +schema = "sql_mode" +tables = ["~.*"] + +[[source-db]] +host = "127.0.0.1" +port = 3306 +user = "root" +password = "123456" +instance-id = "source-1" + +[[table-config]] +schema = "sql_mode" +table = "t_1" + +[[table-config.source-tables]] +instance-id = "source-1" +schema = "sql_mode" +table = "t_1" + +[[source-db]] +host = "127.0.0.1" +port = 3307 +user = "root" +password = "123456" +instance-id = "source-2" + +[[table-config]] +schema = "sql_mode" +table = "t_2" + +[[table-config.source-tables]] +instance-id = "source-2" +schema = "sql_mode" +table = "t_2" + +[target-db] +host = "127.0.0.1" +port = 4000 +user = "test" +password = "123456" diff --git a/tests/sql_mode/conf/dm-master.toml b/tests/sql_mode/conf/dm-master.toml new file mode 100644 index 0000000000..7cecf59ad8 --- /dev/null +++ b/tests/sql_mode/conf/dm-master.toml @@ -0,0 +1,4 @@ +# Master Configuration. +master-addr = ":8261" +advertise-addr = "127.0.0.1:8261" +auto-compaction-retention = "3s" diff --git a/tests/sql_mode/conf/dm-task.yaml b/tests/sql_mode/conf/dm-task.yaml new file mode 100644 index 0000000000..dd1acd6924 --- /dev/null +++ b/tests/sql_mode/conf/dm-task.yaml @@ -0,0 +1,46 @@ +--- +name: sql_mode +task-mode: all +is-sharding: false +meta-schema: "dm_meta" +enable-heartbeat: false + +target-database: + host: "127.0.0.1" + port: 4000 + user: "root" + password: "" + +mysql-instances: + - source-id: "mysql-replica-01" + block-allow-list: "instance" + mydumper-config-name: "global" + loader-config-name: "global" + syncer-config-name: "global" + + - source-id: "mysql-replica-02" + block-allow-list: "instance" + mydumper-config-name: "global" + loader-config-name: "global" + syncer-config-name: "global" + +block-allow-list: + instance: + do-dbs: ["sql_mode"] + +mydumpers: + global: + threads: 4 + chunk-filesize: 0 + skip-tz-utc: true + extra-args: "--statement-size=4000" + +loaders: + global: + pool-size: 16 + dir: "./dumped_data" + +syncers: + global: + worker-count: 16 + batch: 100 diff --git a/tests/sql_mode/conf/dm-worker1.toml b/tests/sql_mode/conf/dm-worker1.toml new file mode 100644 index 0000000000..7a72ea72bf --- /dev/null +++ b/tests/sql_mode/conf/dm-worker1.toml @@ -0,0 +1,2 @@ +name = "worker1" +join = "127.0.0.1:8261" diff --git a/tests/sql_mode/conf/dm-worker2.toml b/tests/sql_mode/conf/dm-worker2.toml new file mode 100644 index 0000000000..010e21c73e --- /dev/null +++ b/tests/sql_mode/conf/dm-worker2.toml @@ -0,0 +1,2 @@ +name = "worker2" +join = "127.0.0.1:8261" diff --git a/tests/sql_mode/conf/source1.yaml b/tests/sql_mode/conf/source1.yaml new file mode 100644 index 0000000000..7d67feb8cf --- /dev/null +++ b/tests/sql_mode/conf/source1.yaml @@ -0,0 +1,11 @@ +source-id: mysql-replica-01 +flavor: '' +enable-gtid: false +enable-relay: true +relay-binlog-name: '' +relay-binlog-gtid: '' +from: + host: 127.0.0.1 + user: root + password: /Q7B9DizNLLTTfiZHv9WoEAKamfpIUs= + port: 3306 diff --git a/tests/sql_mode/conf/source2.yaml b/tests/sql_mode/conf/source2.yaml new file mode 100644 index 0000000000..3df6bdf955 --- /dev/null +++ b/tests/sql_mode/conf/source2.yaml @@ -0,0 +1,11 @@ +source-id: mysql-replica-02 +flavor: '' +enable-gtid: false +enable-relay: true +relay-binlog-name: '' +relay-binlog-gtid: '' +from: + host: 127.0.0.1 + user: root + password: /Q7B9DizNLLTTfiZHv9WoEAKamfpIUs= + port: 3307 diff --git a/tests/sql_mode/data/db1.increment.sql b/tests/sql_mode/data/db1.increment.sql new file mode 100644 index 0000000000..3ca2eaa30b --- /dev/null +++ b/tests/sql_mode/data/db1.increment.sql @@ -0,0 +1,32 @@ +set @@session.SQL_MODE=''; +use `sql_mode`; + +-- test sql_mode PIPES_AS_CONCAT +insert into t_1(num) values('pipes'||'as'||'concat'); + +-- test sql_mode ANSI_QUOTES +insert into t_1(name) values("a"); + +-- test sql_mode IGNORE_SPACE +create table sum (id int not null, primary key(id)); + +-- test sql_mode NO_AUTO_VALUE_ON_ZERO +insert into t_1(id, name) values (30, 'a'); +insert into t_1(id, name) values (0, 'b'); +insert into t_1(id, name) values (0, 'c'); + +-- test sql_mode NO_BACKSLASH_ESCAPES +insert into t_1(name) values ('\\b'); + +-- test sql_mode STRICT_TRANS_TABLES && STRICT_ALL_TABLES && NO_ZERO_IN_DATE && NO_ZERO_DATE && ALLOW_INVALID_DATES +insert into t_1(dt) values('0000-06-00'); +insert into t_1(dt) values('0000-00-01'); +insert into t_1(dt) values('0000-06-01'); +insert into t_1(dt) values('0000-00-00'); + +-- test sql_mode ERROR_FOR_DIVISION_BY_ZERO +insert into t_1(num) values(4/0); + +-- test sql_mode NO_AUTO_CREATE_USER +drop user if exists 'no_auto_create_user'; +grant select on *.* to 'no_auto_create_user'; \ No newline at end of file diff --git a/tests/sql_mode/data/db1.prepare.sql b/tests/sql_mode/data/db1.prepare.sql new file mode 100644 index 0000000000..f41249dcb8 --- /dev/null +++ b/tests/sql_mode/data/db1.prepare.sql @@ -0,0 +1,42 @@ +set @@session.SQL_MODE=''; + +drop database if exists `sql_mode`; +create database `sql_mode`; +use `sql_mode`; +CREATE TABLE `t_1` ( + `id` int NOT NULL AUTO_INCREMENT, + `name` varchar(60), + `num` int, + `dt` datetime, + PRIMARY KEY (id) +); + +-- test sql_mode PIPES_AS_CONCAT +insert into t_1(num) values('pipes'||'as'||'concat'); + +-- test sql_mode ANSI_QUOTES +insert into t_1(name) values("a"); + +-- test sql_mode IGNORE_SPACE +create table count (id int not null, primary key(id)); + +-- test sql_mode NO_AUTO_VALUE_ON_ZERO +insert into t_1(id, name) values (10, 'a'); +insert into t_1(id, name) values (0, 'b'); +insert into t_1(id, name) values (0, 'c'); + +-- test sql_mode NO_BACKSLASH_ESCAPES +insert into t_1(name) values ('\\a'); + +-- test sql_mode STRICT_TRANS_TABLES && STRICT_ALL_TABLES && NO_ZERO_IN_DATE && NO_ZERO_DATE && ALLOW_INVALID_DATES +insert into t_1(dt) values('0000-06-00'); +insert into t_1(dt) values('0000-00-01'); +insert into t_1(dt) values('0000-06-01'); +insert into t_1(dt) values('0000-00-00'); + +-- test sql_mode ERROR_FOR_DIVISION_BY_ZERO +insert into t_1(num) values(4/0); + +-- test sql_mode NO_AUTO_CREATE_USER +drop user if exists 'no_auto_create_user'; +grant select on *.* to 'no_auto_create_user'; \ No newline at end of file diff --git a/tests/sql_mode/data/db2.increment.sql b/tests/sql_mode/data/db2.increment.sql new file mode 100644 index 0000000000..9301ae86d3 --- /dev/null +++ b/tests/sql_mode/data/db2.increment.sql @@ -0,0 +1,24 @@ +set @@session.sql_mode='ONLY_FULL_GROUP_BY,NO_UNSIGNED_SUBTRACTION,NO_DIR_IN_CREATE,STRICT_TRANS_TABLES,STRICT_ALL_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ALLOW_INVALID_DATES,ERROR_FOR_DIVISION_BY_ZERO,HIGH_NOT_PRECEDENCE,NO_ENGINE_SUBSTITUTION,REAL_AS_FLOAT'; +-- NO_AUTO_CREATE_USER set failed in mysql8.0 + +use sql_mode; + +-- test sql_mode PIPES_AS_CONCAT +set @@session.sql_mode=concat(@@session.sql_mode, ',PIPES_AS_CONCAT'); +insert into t_2(name) values('pipes'||'as'||'concat'); + +-- test sql_mode ANSI_QUOTES +insert into t_2(name) values("a"); + +-- test sql_mode IGNORE_SPACE +set @@session.sql_mode=concat(@@session.sql_mode, ',IGNORE_SPACE'); +insert into t_2(name) values(concat ('ignore', 'space')); + +-- test sql_mode NO_AUTO_VALUE_ON_ZERO +set @@session.sql_mode=concat(@@session.sql_mode, ',NO_AUTO_VALUE_ON_ZERO'); +insert into t_2(id, name) values (20, 'a'); +replace into t_2(id, name) values (0, 'c'); + +-- test sql_mode NO_BACKSLASH_ESCAPES +set @@session.sql_mode=concat(@@session.sql_mode, ',NO_BACKSLASH_ESCAPES'); +insert into t_2(name) values ('\\a'); \ No newline at end of file diff --git a/tests/sql_mode/data/db2.prepare.sql b/tests/sql_mode/data/db2.prepare.sql new file mode 100644 index 0000000000..f956c7aaae --- /dev/null +++ b/tests/sql_mode/data/db2.prepare.sql @@ -0,0 +1,33 @@ +set @@session.sql_mode='ONLY_FULL_GROUP_BY,NO_UNSIGNED_SUBTRACTION,NO_DIR_IN_CREATE,STRICT_TRANS_TABLES,STRICT_ALL_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ALLOW_INVALID_DATES,ERROR_FOR_DIVISION_BY_ZERO,HIGH_NOT_PRECEDENCE,NO_ENGINE_SUBSTITUTION,REAL_AS_FLOAT'; +-- NO_AUTO_CREATE_USER set failed in mysql8.0 + +drop database if exists `sql_mode`; +create database `sql_mode`; +use `sql_mode`; +CREATE TABLE `t_2` ( + `id` int NOT NULL AUTO_INCREMENT, + `name` varchar(60), + `num` int, + `dt` datetime, + PRIMARY KEY (id) +); + +-- test sql_mode PIPES_AS_CONCAT +set @@session.sql_mode=concat(@@session.sql_mode, ',PIPES_AS_CONCAT'); +insert into t_2(name) values('pipes'||'as'||'concat'); + +-- test sql_mode ANSI_QUOTES +insert into t_2(name) values("a"); + +-- test sql_mode IGNORE_SPACE +set @@session.sql_mode=concat(@@session.sql_mode, ',IGNORE_SPACE'); +insert into t_2(name) values(concat ('ignore', 'space')); + +-- test sql_mode NO_AUTO_VALUE_ON_ZERO +set @@session.sql_mode=concat(@@session.sql_mode, ',NO_AUTO_VALUE_ON_ZERO'); +insert into t_2(id, name) values (10, 'a'); +insert into t_2(id, name) values (0, 'b'); + +-- test sql_mode NO_BACKSLASH_ESCAPES +set @@session.sql_mode=concat(@@session.sql_mode, ',NO_BACKSLASH_ESCAPES'); +insert into t_2(name) values ('\\a'); \ No newline at end of file diff --git a/tests/sql_mode/run.sh b/tests/sql_mode/run.sh new file mode 100644 index 0000000000..92022b1cac --- /dev/null +++ b/tests/sql_mode/run.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +set -eu + +cur=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +source $cur/../_utils/test_prepare +WORK_DIR=$TEST_DIR/$TEST_NAME + +function run() { + run_sql_source1 "SET @@GLOBAL.SQL_MODE='PIPES_AS_CONCAT,IGNORE_SPACE,ONLY_FULL_GROUP_BY,NO_UNSIGNED_SUBTRACTION,NO_DIR_IN_CREATE,NO_AUTO_VALUE_ON_ZERO,NO_BACKSLASH_ESCAPES,STRICT_TRANS_TABLES,STRICT_ALL_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ALLOW_INVALID_DATES,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,HIGH_NOT_PRECEDENCE,NO_ENGINE_SUBSTITUTION,REAL_AS_FLOAT'" + run_sql_source2 "SET @@GLOBAL.SQL_MODE=''" + + run_sql_file $cur/data/db1.prepare.sql $MYSQL_HOST1 $MYSQL_PORT1 $MYSQL_PASSWORD1 + run_sql_file $cur/data/db2.prepare.sql $MYSQL_HOST2 $MYSQL_PORT2 $MYSQL_PASSWORD2 + + run_dm_master $WORK_DIR/master $MASTER_PORT $cur/conf/dm-master.toml + check_rpc_alive $cur/../bin/check_master_online 127.0.0.1:$MASTER_PORT + run_dm_worker $WORK_DIR/worker1 $WORKER1_PORT $cur/conf/dm-worker1.toml + check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER1_PORT + run_dm_worker $WORK_DIR/worker2 $WORKER2_PORT $cur/conf/dm-worker2.toml + check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER2_PORT + + # operate mysql config to worker + cp $cur/conf/source1.yaml $WORK_DIR/source1.yaml + sed -i "/relay-binlog-name/i\relay-dir: $WORK_DIR/worker1/relay_log" $WORK_DIR/source1.yaml + dmctl_operate_source create $WORK_DIR/source1.yaml $SOURCE_ID1 + cp $cur/conf/source2.yaml $WORK_DIR/source2.yaml + sed -i "/relay-binlog-name/i\relay-dir: $WORK_DIR/worker2/relay_log" $WORK_DIR/source2.yaml + dmctl_operate_source create $WORK_DIR/source2.yaml $SOURCE_ID2 + + # start DM task only + dmctl_start_task $cur/conf/dm-task.yaml + + # use sync_diff_inspector to check full dump loader + check_sync_diff $WORK_DIR $cur/conf/diff_config.toml + + run_sql_file $cur/data/db1.increment.sql $MYSQL_HOST1 $MYSQL_PORT1 $MYSQL_PASSWORD1 + check_sync_diff $WORK_DIR $cur/conf/diff_config.toml + run_sql_file $cur/data/db2.increment.sql $MYSQL_HOST2 $MYSQL_PORT2 $MYSQL_PASSWORD2 + check_sync_diff $WORK_DIR $cur/conf/diff_config.toml +} + +cleanup_data $TEST_NAME +# also cleanup dm processes in case of last run failed +cleanup_process $* +run $* +cleanup_process $* + +echo "[$(date)] <<<<<< test case $TEST_NAME success! >>>>>>"