From 4d48162442d0d113f5842beff4ba95f5a954e9b2 Mon Sep 17 00:00:00 2001 From: glorv Date: Tue, 11 May 2021 10:45:39 +0800 Subject: [PATCH 01/15] be compatible with source db with different time zone setting --- cmd/dm-syncer/config.go | 13 +++++++++++ dm/config/subtask.go | 5 +++- dm/config/task.go | 32 ++++++++++++++++++-------- dm/master/server.go | 1 + dumpling/dumpling.go | 7 ++++++ loader/loader.go | 3 +++ tests/all_mode/data/db1.increment.sql | 9 +++++--- tests/all_mode/data/db1.increment0.sql | 3 ++- tests/all_mode/data/db1.prepare.sql | 9 ++++++-- tests/all_mode/data/db2.increment.sql | 1 - tests/all_mode/data/db2.prepare.sql | 9 ++++++-- tests/all_mode/run.sh | 3 +++ tests/full_mode/data/db1.prepare.sql | 8 +++++-- tests/full_mode/data/db2.prepare.sql | 7 ++++-- tests/full_mode/run.sh | 3 +++ 15 files changed, 89 insertions(+), 24 deletions(-) diff --git a/cmd/dm-syncer/config.go b/cmd/dm-syncer/config.go index a5780680ee..300dbf0a58 100644 --- a/cmd/dm-syncer/config.go +++ b/cmd/dm-syncer/config.go @@ -27,6 +27,7 @@ import ( router "github.com/pingcap/tidb-tools/pkg/table-router" "github.com/pingcap/dm/dm/config" + "github.com/pingcap/dm/pkg/log" "github.com/pingcap/dm/pkg/utils" ) @@ -177,6 +178,11 @@ func (c *commonConfig) newSubTaskConfig(args []string) (*config.SubTaskConfig, e cfg.ServerID = uint32(serverID) } + if cfg.Timezone != "" { + log.L().Warn("timezone is deprecated, will auto use UTC instead.") + cfg.Timezone = "+00:00" + } + return cfg, nil } @@ -203,8 +209,15 @@ func newCommonConfig() *commonConfig { fs.BoolVar(&cfg.SafeMode, "safe-mode", false, "enable safe mode to make syncer reentrant") fs.IntVar(&cfg.MaxRetry, "max-retry", 100, "maxinum retry when network interruption") fs.StringVar(&cfg.TimezoneStr, "timezone", "", "target database timezone location string") + + if cfg.TimezoneStr != "" { + log.L().Warn("timezone is deprecated, will auto use UTC instead.") + cfg.TimezoneStr = "+00:00" + } + fs.BoolVar(&cfg.SyncerConfigFormat, "syncer-config-format", false, "read syncer config format") + return cfg } diff --git a/dm/config/subtask.go b/dm/config/subtask.go index d534b83c85..9b36bed6aa 100644 --- a/dm/config/subtask.go +++ b/dm/config/subtask.go @@ -118,6 +118,8 @@ func (db *DBConfig) Decode(data string) error { // Adjust adjusts the config. func (db *DBConfig) Adjust() { + // force set session time zone to UTC here. + AdjustTargetDBTimeZone(db) } // SubTaskConfig is the configuration for SubTask. @@ -150,7 +152,8 @@ type SubTaskConfig struct { HeartbeatReportInterval int `toml:"heartbeat-report-interval" json:"heartbeat-report-interval"` EnableHeartbeat bool `toml:"enable-heartbeat" json:"enable-heartbeat"` Meta *Meta `toml:"meta" json:"meta"` - Timezone string `toml:"timezone" josn:"timezone"` + // deprecated, will auto adjust source and target time_zone + Timezone string `toml:"timezone" json:"timezone"` // RelayDir get value from dm-worker config RelayDir string `toml:"relay-dir" json:"relay-dir"` diff --git a/dm/config/task.go b/dm/config/task.go index 99af958b70..bec1b0c55f 100644 --- a/dm/config/task.go +++ b/dm/config/task.go @@ -20,20 +20,18 @@ import ( "io/ioutil" "sort" "strings" - "time" + "github.com/coreos/go-semver/semver" + "github.com/dustin/go-humanize" bf "github.com/pingcap/tidb-tools/pkg/binlog-filter" "github.com/pingcap/tidb-tools/pkg/column-mapping" "github.com/pingcap/tidb-tools/pkg/filter" router "github.com/pingcap/tidb-tools/pkg/table-router" + "go.uber.org/zap" + "gopkg.in/yaml.v2" "github.com/pingcap/dm/pkg/log" "github.com/pingcap/dm/pkg/terror" - - "github.com/coreos/go-semver/semver" - "github.com/dustin/go-humanize" - "go.uber.org/zap" - yaml "gopkg.in/yaml.v2" ) // Online DDL Scheme. @@ -71,6 +69,8 @@ var ( defaultBatch = 100 defaultQueueSize = 1024 // do not give too large default value to avoid OOM defaultCheckpointFlushInterval = 30 // in seconds + // force use UTC time_zone + defaultTimeZone = "+00:00" // TargetDBConfig. defaultSessionCfg = []struct { @@ -285,6 +285,7 @@ type TaskConfig struct { EnableHeartbeat bool `yaml:"enable-heartbeat" toml:"enable-heartbeat" json:"enable-heartbeat"` HeartbeatUpdateInterval int `yaml:"heartbeat-update-interval" toml:"heartbeat-update-interval" json:"heartbeat-update-interval"` HeartbeatReportInterval int `yaml:"heartbeat-report-interval" toml:"heartbeat-report-interval" json:"heartbeat-report-interval"` + // deprecated, needn't set anymore, will always use UTC instead Timezone string `yaml:"timezone" toml:"timezone" json:"timezone"` // handle schema/table name mode, and only for schema/table name @@ -595,11 +596,9 @@ func (c *TaskConfig) adjust() error { } if c.Timezone != "" { - _, err := time.LoadLocation(c.Timezone) - if err != nil { - return terror.ErrConfigInvalidTimezone.Delegate(err, c.Timezone) - } + log.L().Warn("timezone config is deprecated, will automatically use UTC instead") } + c.Timezone = defaultTimeZone if c.RemoveMeta { log.L().Warn("`remove-meta` in task config is deprecated, please use `start-task ... --remove-meta` instead") @@ -815,5 +814,18 @@ func AdjustTargetDBSessionCfg(dbConfig *DBConfig, version *semver.Version) { lowerMap[cfg.key] = cfg.val } } + // force set time zone to UTC + lowerMap["time_zone"] = defaultTimeZone dbConfig.Session = lowerMap } + +// force adjust session `time_zone` to UTC +func AdjustTargetDBTimeZone(config *DBConfig) { + for k := range config.Session { + if strings.ToLower(k) == "time_zone" { + config.Session[k] = defaultTimeZone + return + } + } + config.Session["time_zone"] = defaultTimeZone +} diff --git a/dm/master/server.go b/dm/master/server.go index f0d1f6630a..d4c5e7f743 100644 --- a/dm/master/server.go +++ b/dm/master/server.go @@ -1144,6 +1144,7 @@ func adjustTargetDB(ctx context.Context, dbConfig *config.DBConfig) error { config.AdjustTargetDBSessionCfg(dbConfig, version) } else { log.L().Warn("get tidb version", log.ShortError(err)) + config.AdjustTargetDBTimeZone(dbConfig) } return nil } diff --git a/dumpling/dumpling.go b/dumpling/dumpling.go index b19a0f9b8d..85e9edb4ef 100644 --- a/dumpling/dumpling.go +++ b/dumpling/dumpling.go @@ -60,6 +60,9 @@ func (m *Dumpling) Init(ctx context.Context) error { var err error m.dumpConfig, err = m.constructArgs() m.detectSQLMode(ctx) + if m.cfg.Timezone != "" { + m.dumpConfig.SessionParams["time_zone"] = m.cfg.Timezone + } m.logger.Info("create dumpling", zap.Stringer("config", m.dumpConfig)) return err } @@ -219,6 +222,10 @@ func (m *Dumpling) constructArgs() (*export.Config, error) { dumpConfig.TableFilter = tableFilter dumpConfig.CompleteInsert = true // always keep column name in `INSERT INTO` statements. dumpConfig.Logger = m.logger.Logger + // force using UTC timezone + dumpConfig.SessionParams = map[string]interface{}{ + "time_zone": "+00:00", + } if cfg.Threads > 0 { dumpConfig.Threads = cfg.Threads diff --git a/loader/loader.go b/loader/loader.go index b9809a0fba..a19ce47547 100644 --- a/loader/loader.go +++ b/loader/loader.go @@ -495,6 +495,9 @@ func (l *Loader) Init(ctx context.Context) (err error) { if lcfg.To.Session == nil { lcfg.To.Session = make(map[string]string) } + if lcfg.Timezone != "" { + lcfg.To.Session["time_zone"] = lcfg.Timezone + } hasSQLMode := false for k := range l.cfg.To.Session { diff --git a/tests/all_mode/data/db1.increment.sql b/tests/all_mode/data/db1.increment.sql index a172eb5aa9..f602f85878 100644 --- a/tests/all_mode/data/db1.increment.sql +++ b/tests/all_mode/data/db1.increment.sql @@ -6,10 +6,10 @@ update t1 set name = 'Catelyn Stark' where name = 'catelyn'; -- test multi column index with generated column alter table t1 add column info json; alter table t1 add column gen_id int as (info->'$.id'); -alter table t1 add index multi_col(`id`, `gen_id`); +alter table t1 add index multi_col(`id`, `gen_id`, ts); insert into t1 (id, name, info) values (4, 'gentest', '{"id": 123}'); insert into t1 (id, name, info) values (5, 'gentest', '{"id": 124}'); -update t1 set info = '{"id": 120}' where id = 1; +update t1 set info = '{"id": 120}', ts = '2021-05-11 12:02:05' where id = 1; update t1 set info = '{"id": 121}' where id = 2; update t1 set info = '{"id": 122}' where id = 3; @@ -20,7 +20,7 @@ alter table t1 add unique key gen_idx(`gen_id`); update t1 set name = 'gentestxx' where gen_id = 123; insert into t1 (id, name, info) values (7, 'gentest', '{"id": 126}'); -update t1 set name = 'gentestxxxxxx' where gen_id = 124; +update t1 set name = 'gentestxxxxxx', dt = '2021-05-11 12:03:05', ts = '2021-05-11 12:03:05' where gen_id = 124; -- delete with unique key delete from t1 where gen_id > 124; @@ -43,3 +43,6 @@ alter table t1 add column big2 bigint unsigned; insert into t1 (id, name, info, lat, big1, big2) values (11, 'gentest', '{"id":130}', '123.123', -9223372036854775808, 0); insert into t1 (id, name, info, lat, big1, big2) values (12, 'gentest', '{"id":131}', '123.123', 9223372036854775807, 18446744073709551615); +-- test with different session time_zone +SET @@session.time_zone = '+07:00'; +insert into t1 (id, name, info) values (13, 'tztest', '{"id": 122}'); diff --git a/tests/all_mode/data/db1.increment0.sql b/tests/all_mode/data/db1.increment0.sql index 99e094c474..fa406496ae 100644 --- a/tests/all_mode/data/db1.increment0.sql +++ b/tests/all_mode/data/db1.increment0.sql @@ -1,2 +1,3 @@ +SET @@SESSION.TIME_ZONE = "+03:00"; use all_mode; -insert into t1 (id, name) values (100, 'Eddard Stark'); +insert into t1 (id, name, ts) values (100, 'Eddard Stark', '2021-05-11 12:01:05'); diff --git a/tests/all_mode/data/db1.prepare.sql b/tests/all_mode/data/db1.prepare.sql index 631062d40e..eef8893e84 100644 --- a/tests/all_mode/data/db1.prepare.sql +++ b/tests/all_mode/data/db1.prepare.sql @@ -1,9 +1,14 @@ drop database if exists `all_mode`; create database `all_mode`; use `all_mode`; -create table t1 (id int NOT NULL AUTO_INCREMENT, name varchar(20), PRIMARY KEY (id)); +create table t1 ( + id int NOT NULL AUTO_INCREMENT, + name varchar(20), + dt datetime, + ts timestamp DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (id)); -- test ANSI_QUOTES works with quote in string -insert into t1 (id, name) values (1, 'ar"ya'), (2, 'catelyn'); +insert into t1 (id, name, dt, ts) values (1, 'ar"ya', now(), now()), (2, 'catelyn', '2021-05-11 10:01:05', '2021-05-11 10:01:05'); -- test sql_mode=NO_AUTO_VALUE_ON_ZERO insert into t1 (id, name) values (0, 'lalala'); diff --git a/tests/all_mode/data/db2.increment.sql b/tests/all_mode/data/db2.increment.sql index 524f4cacd3..bce1755e76 100644 --- a/tests/all_mode/data/db2.increment.sql +++ b/tests/all_mode/data/db2.increment.sql @@ -1,6 +1,5 @@ use all_mode; delete from t2 where name = 'Sansa'; - -- test sql_mode=NO_AUTO_VALUE_ON_ZERO insert into t2 (id, name) values (0,'haha') \ No newline at end of file diff --git a/tests/all_mode/data/db2.prepare.sql b/tests/all_mode/data/db2.prepare.sql index c7688c5a7c..3d30ab4778 100644 --- a/tests/all_mode/data/db2.prepare.sql +++ b/tests/all_mode/data/db2.prepare.sql @@ -1,8 +1,13 @@ drop database if exists `all_mode`; create database `all_mode`; use `all_mode`; -create table t2 (id int auto_increment, name varchar(20), primary key (`id`)); -insert into t2 (name) values ('Arya'), ('Bran'), ('Sansa'); +create table t2 ( + id int NOT NULL AUTO_INCREMENT, + name varchar(20), + ts timestamp, + PRIMARY KEY (id));; +insert into t2 (name, ts) values ('Arya', now()), ('Bran', '2021-05-11 10:01:05'); +insert into ts (name) values ('Sansa'); -- test block-allow-list drop database if exists `ignore_db`; diff --git a/tests/all_mode/run.sh b/tests/all_mode/run.sh index 2f23b9dad1..7f5618fdc7 100755 --- a/tests/all_mode/run.sh +++ b/tests/all_mode/run.sh @@ -171,6 +171,8 @@ function test_stop_task_before_checkpoint(){ function run() { run_sql_both_source "SET @@GLOBAL.SQL_MODE='ANSI_QUOTES,NO_AUTO_VALUE_ON_ZERO'" + run_sql_source1 "SET @@global.time_zone = '+01:00';" + run_sql_source2 "SET @@global.time_zone = '+02:00';" test_session_config @@ -339,6 +341,7 @@ function run() { export GO_FAILPOINTS='' run_sql_both_source "SET @@GLOBAL.SQL_MODE='ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION'" + run_sql_both_source "SET @@global.time_zone = 'SYSTEM';" } cleanup_data all_mode diff --git a/tests/full_mode/data/db1.prepare.sql b/tests/full_mode/data/db1.prepare.sql index 77592b929f..98ea937311 100644 --- a/tests/full_mode/data/db1.prepare.sql +++ b/tests/full_mode/data/db1.prepare.sql @@ -1,8 +1,12 @@ drop database if exists `full_mode`; create database `full_mode`; use `full_mode`; -create table t1 (id int, name varchar(20), primary key(`id`)); -insert into t1 (id, name) values (1, 'arya'), (2, 'catelyn'); +create table t1 ( + id int, + name varchar(20), + ts timestamp DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + primary key(`id`)); +insert into t1 (id, name, ts) values (1, 'arya', now()), (2, 'catelyn', '2021-05-11 10:01:03'); insert into t1 (id, name) values (3, 'Eddard Stark'); update t1 set name = 'Arya S\\\\tark' where id = 1; diff --git a/tests/full_mode/data/db2.prepare.sql b/tests/full_mode/data/db2.prepare.sql index 0eb75594ac..1f66dfc0fd 100644 --- a/tests/full_mode/data/db2.prepare.sql +++ b/tests/full_mode/data/db2.prepare.sql @@ -1,5 +1,8 @@ drop database if exists `full_mode`; create database `full_mode`; use `full_mode`; -create table t2 (id int auto_increment, name varchar(20), primary key (`id`)); -insert into t2 (name) values ('Arya'), ('Bran'), ('Sansa'); +create table t2 (id int auto_increment, name varchar(20), dt datetime, ts timestamp, primary key (`id`)); +insert into t2 (name, dt, ts) values + ('Arya', now(), now()), + ('Bran', '2021-05-11 10:09:03', '2021-05-11 10:09:03'), + ('Sansa', NULL, NULL); diff --git a/tests/full_mode/run.sh b/tests/full_mode/run.sh index ffda4a75ab..273f42af93 100755 --- a/tests/full_mode/run.sh +++ b/tests/full_mode/run.sh @@ -147,6 +147,8 @@ function run() { empty_data run_sql_both_source "SET @@GLOBAL.SQL_MODE='NO_BACKSLASH_ESCAPES'" + run_sql_source1 "SET @@global.time_zone = '+01:00';" + run_sql_source2 "SET @@global.time_zone = '+02:00';" run_sql_file $cur/data/db1.prepare.sql $MYSQL_HOST1 $MYSQL_PORT1 $MYSQL_PASSWORD1 check_contains 'Query OK, 2 rows affected' @@ -193,6 +195,7 @@ function run() { check_metric_not_contains $WORKER1_PORT 'dm_worker_task_state{source_id="mysql-replica-01",task="test"}' 3 check_metric_not_contains $WORKER2_PORT 'dm_worker_task_state{source_id="mysql-replica-02",task="test"}' 3 run_sql_both_source "SET @@GLOBAL.SQL_MODE='ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION'" + run_sql_both_source "SET @@GLOBAL.TIME_ZONE='SYSTEM';" } cleanup_data full_mode From 15cc9f214c478cc8221a88bada3e7c0c3a63de24 Mon Sep 17 00:00:00 2001 From: glorv Date: Tue, 11 May 2021 11:02:49 +0800 Subject: [PATCH 02/15] format code --- dm/config/subtask.go | 2 +- dm/config/task.go | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/dm/config/subtask.go b/dm/config/subtask.go index 9b36bed6aa..b64d971815 100644 --- a/dm/config/subtask.go +++ b/dm/config/subtask.go @@ -153,7 +153,7 @@ type SubTaskConfig struct { EnableHeartbeat bool `toml:"enable-heartbeat" json:"enable-heartbeat"` Meta *Meta `toml:"meta" json:"meta"` // deprecated, will auto adjust source and target time_zone - Timezone string `toml:"timezone" json:"timezone"` + Timezone string `toml:"timezone" json:"timezone"` // RelayDir get value from dm-worker config RelayDir string `toml:"relay-dir" json:"relay-dir"` diff --git a/dm/config/task.go b/dm/config/task.go index bec1b0c55f..6bb649ded9 100644 --- a/dm/config/task.go +++ b/dm/config/task.go @@ -69,7 +69,7 @@ var ( defaultBatch = 100 defaultQueueSize = 1024 // do not give too large default value to avoid OOM defaultCheckpointFlushInterval = 30 // in seconds - // force use UTC time_zone + // force use UTC time_zone. defaultTimeZone = "+00:00" // TargetDBConfig. @@ -282,11 +282,11 @@ type TaskConfig struct { // don't save configuration into it MetaSchema string `yaml:"meta-schema" toml:"meta-schema" json:"meta-schema"` - EnableHeartbeat bool `yaml:"enable-heartbeat" toml:"enable-heartbeat" json:"enable-heartbeat"` - HeartbeatUpdateInterval int `yaml:"heartbeat-update-interval" toml:"heartbeat-update-interval" json:"heartbeat-update-interval"` - HeartbeatReportInterval int `yaml:"heartbeat-report-interval" toml:"heartbeat-report-interval" json:"heartbeat-report-interval"` + EnableHeartbeat bool `yaml:"enable-heartbeat" toml:"enable-heartbeat" json:"enable-heartbeat"` + HeartbeatUpdateInterval int `yaml:"heartbeat-update-interval" toml:"heartbeat-update-interval" json:"heartbeat-update-interval"` + HeartbeatReportInterval int `yaml:"heartbeat-report-interval" toml:"heartbeat-report-interval" json:"heartbeat-report-interval"` // deprecated, needn't set anymore, will always use UTC instead - Timezone string `yaml:"timezone" toml:"timezone" json:"timezone"` + Timezone string `yaml:"timezone" toml:"timezone" json:"timezone"` // handle schema/table name mode, and only for schema/table name // if case insensitive, we would convert schema/table name to lower case @@ -819,7 +819,7 @@ func AdjustTargetDBSessionCfg(dbConfig *DBConfig, version *semver.Version) { dbConfig.Session = lowerMap } -// force adjust session `time_zone` to UTC +// AdjustTargetDBTimeZone force adjust session `time_zone` to UTC. func AdjustTargetDBTimeZone(config *DBConfig) { for k := range config.Session { if strings.ToLower(k) == "time_zone" { From dac941aebc9050cc8f70f925598a148090ead542 Mon Sep 17 00:00:00 2001 From: glorv Date: Tue, 11 May 2021 11:30:02 +0800 Subject: [PATCH 03/15] avoid lint --- relay/meta.go | 1 + syncer/syncer_test.go | 2 ++ 2 files changed, 3 insertions(+) diff --git a/relay/meta.go b/relay/meta.go index c9a1a3844d..0187eb5333 100644 --- a/relay/meta.go +++ b/relay/meta.go @@ -303,6 +303,7 @@ func (lm *LocalMeta) AddDir(serverUUID string, newPos *mysql.Position, newGTID g } // update UUID index file + // nolint:gocritic uuids := append(lm.uuids, newUUID) err = lm.updateIndexFile(uuids) if err != nil { diff --git a/syncer/syncer_test.go b/syncer/syncer_test.go index d2101ffe9a..a6276b5232 100644 --- a/syncer/syncer_test.go +++ b/syncer/syncer_test.go @@ -694,6 +694,7 @@ func (s *testSyncerSuite) TestColumnMapping(c *C) { mapping, err := cm.NewMapping(false, rules) c.Assert(err, IsNil) + // nolint:gocritic allEvents := append(createEvents, dmlEvents...) allEvents = append(allEvents, dropEvents...) dmlIndex := 0 @@ -1314,6 +1315,7 @@ func (s *testSyncerSuite) TestExitSafeModeByConfig(c *C) { } generatedEvents2 := s.generateEvents(events2, c) + // nolint:gocritic generatedEvents := append(generatedEvents1, generatedEvents2...) mockStreamerProducer := &MockStreamProducer{generatedEvents} From b1c99668a2292b8612135ac318b8e10fabefe890 Mon Sep 17 00:00:00 2001 From: glorv Date: Tue, 11 May 2021 12:09:20 +0800 Subject: [PATCH 04/15] fix test --- tests/all_mode/data/db2.prepare.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/all_mode/data/db2.prepare.sql b/tests/all_mode/data/db2.prepare.sql index 3d30ab4778..10199a4381 100644 --- a/tests/all_mode/data/db2.prepare.sql +++ b/tests/all_mode/data/db2.prepare.sql @@ -7,7 +7,7 @@ create table t2 ( ts timestamp, PRIMARY KEY (id));; insert into t2 (name, ts) values ('Arya', now()), ('Bran', '2021-05-11 10:01:05'); -insert into ts (name) values ('Sansa'); +insert into t2 (name) values ('Sansa'); -- test block-allow-list drop database if exists `ignore_db`; From 32db4b2fc9899a974538ba076923cb0a17167626 Mon Sep 17 00:00:00 2001 From: glorv Date: Tue, 11 May 2021 14:19:09 +0800 Subject: [PATCH 05/15] fix test --- dm/config/task.go | 3 +++ tests/all_mode/data/db2.prepare.sql | 3 +-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/dm/config/task.go b/dm/config/task.go index 6bb649ded9..ff90b842dd 100644 --- a/dm/config/task.go +++ b/dm/config/task.go @@ -827,5 +827,8 @@ func AdjustTargetDBTimeZone(config *DBConfig) { return } } + if config.Session == nil { + config.Session = make(map[string]string, 1) + } config.Session["time_zone"] = defaultTimeZone } diff --git a/tests/all_mode/data/db2.prepare.sql b/tests/all_mode/data/db2.prepare.sql index 10199a4381..fc8900ae73 100644 --- a/tests/all_mode/data/db2.prepare.sql +++ b/tests/all_mode/data/db2.prepare.sql @@ -6,8 +6,7 @@ create table t2 ( name varchar(20), ts timestamp, PRIMARY KEY (id));; -insert into t2 (name, ts) values ('Arya', now()), ('Bran', '2021-05-11 10:01:05'); -insert into t2 (name) values ('Sansa'); +insert into t2 (name, ts) values ('Arya', now()), ('Bran', '2021-05-11 10:01:05'), ('Sansa', NULL); -- test block-allow-list drop database if exists `ignore_db`; From ef59ee960486e122c97af80f7ab450fb7b46cc9a Mon Sep 17 00:00:00 2001 From: glorv Date: Tue, 11 May 2021 16:55:28 +0800 Subject: [PATCH 06/15] fix test --- dm/config/source_config.go | 2 +- dm/config/source_config_test.go | 24 ++++++++++++------- dm/config/subtask.go | 11 ++++----- dm/config/task_test.go | 16 ++++++++----- pkg/utils/common.go | 42 +++++++++++++++++++++++++++++++++ pkg/utils/common_test.go | 30 +++++++++++++++++++++++ syncer/db_test.go | 4 ++-- syncer/syncer.go | 2 +- 8 files changed, 107 insertions(+), 24 deletions(-) diff --git a/dm/config/source_config.go b/dm/config/source_config.go index d2ed9a2825..f603fba8bb 100644 --- a/dm/config/source_config.go +++ b/dm/config/source_config.go @@ -141,7 +141,7 @@ func (c *SourceConfig) Parse(content string) error { // ParseYaml parses flag definitions from the argument list, content should be yaml format. func (c *SourceConfig) ParseYaml(content string) error { - if err := yaml.UnmarshalStrict([]byte(content), c); err != nil { + if err := yaml.Unmarshal([]byte(content), c); err != nil { return terror.ErrConfigYamlTransform.Delegate(err, "decode source config") } c.adjust() diff --git a/dm/config/source_config_test.go b/dm/config/source_config_test.go index 793c0b18c0..d33580133d 100644 --- a/dm/config/source_config_test.go +++ b/dm/config/source_config_test.go @@ -63,18 +63,24 @@ func (t *testConfig) TestConfig(c *C) { // test update config file and reload c.Assert(cfg.Parse(tomlStr), IsNil) c.Assert(cfg.ServerID, Equals, uint32(100)) - c.Assert(cfg.ParseYaml(yamlStr), IsNil) - c.Assert(cfg.ServerID, Equals, uint32(100)) - c.Assert(cfg.Parse(originCfgStr), IsNil) - c.Assert(cfg.ServerID, Equals, uint32(101)) - c.Assert(cfg.ParseYaml(originCfgYamlStr), IsNil) - c.Assert(cfg.ServerID, Equals, uint32(101)) + var cfg1 SourceConfig + c.Assert(cfg1.ParseYaml(yamlStr), IsNil) + c.Assert(cfg1.ServerID, Equals, uint32(100)) + cfg.Filters = []*bf.BinlogEventRule{} + cfg.Tracer = map[string]interface{}{} + + var cfg2 SourceConfig + c.Assert(cfg2.Parse(originCfgStr), IsNil) + c.Assert(cfg2.ServerID, Equals, uint32(101)) + + var cfg3 SourceConfig + c.Assert(cfg3.ParseYaml(originCfgYamlStr), IsNil) + c.Assert(cfg3.ServerID, Equals, uint32(101)) // test decrypt password clone1.From.Password = "1234" - clone1.ServerID = 101 // fix empty map after marshal/unmarshal becomes nil - clone1.From.Session = map[string]string{} + clone1.From.Session = cfg.From.Session clone1.Tracer = map[string]interface{}{} clone1.Filters = []*bf.BinlogEventRule{} clone2 := cfg.DecryptPassword() @@ -96,6 +102,7 @@ func (t *testConfig) TestConfig(c *C) { c.Assert(err, IsNil) c.Assert(clone4toml, Matches, "(.|\n)*backoff-rollback = \"5m(.|\n)*") c.Assert(clone4toml, Matches, "(.|\n)*backoff-max = \"5m(.|\n)*") + var clone5 SourceConfig c.Assert(clone5.Parse(clone4toml), IsNil) c.Assert(clone5, DeepEquals, *clone4) @@ -103,6 +110,7 @@ func (t *testConfig) TestConfig(c *C) { c.Assert(err, IsNil) c.Assert(clone4yaml, Matches, "(.|\n)*backoff-rollback: 5m(.|\n)*") c.Assert(clone4yaml, Matches, "(.|\n)*backoff-max: 5m(.|\n)*") + var clone6 SourceConfig c.Assert(clone6.ParseYaml(clone4yaml), IsNil) c.Assert(clone6, DeepEquals, *clone4) diff --git a/dm/config/subtask.go b/dm/config/subtask.go index b64d971815..254722bb44 100644 --- a/dm/config/subtask.go +++ b/dm/config/subtask.go @@ -19,11 +19,6 @@ import ( "flag" "fmt" "strings" - "time" - - "github.com/pingcap/dm/pkg/log" - "github.com/pingcap/dm/pkg/terror" - "github.com/pingcap/dm/pkg/utils" "github.com/BurntSushi/toml" bf "github.com/pingcap/tidb-tools/pkg/binlog-filter" @@ -31,6 +26,10 @@ import ( "github.com/pingcap/tidb-tools/pkg/filter" router "github.com/pingcap/tidb-tools/pkg/table-router" "go.uber.org/zap" + + "github.com/pingcap/dm/pkg/log" + "github.com/pingcap/dm/pkg/terror" + "github.com/pingcap/dm/pkg/utils" ) // task modes. @@ -277,7 +276,7 @@ func (c *SubTaskConfig) Adjust(verifyDecryptPassword bool) error { } if c.Timezone != "" { - _, err := time.LoadLocation(c.Timezone) + _, err := utils.ParseTimeZone(c.Timezone) if err != nil { return terror.ErrConfigInvalidTimezone.Delegate(err, c.Timezone) } diff --git a/dm/config/task_test.go b/dm/config/task_test.go index 9fe4a4b6bb..a1f4cd36f0 100644 --- a/dm/config/task_test.go +++ b/dm/config/task_test.go @@ -546,10 +546,12 @@ func (t *testConfig) TestGenAndFromSubTaskConfigs(c *C) { timezone = "Asia/Shanghai" maxAllowedPacket = 10244201 fromSession = map[string]string{ - "sql_mode": " NO_AUTO_VALUE_ON_ZERO,ANSI_QUOTES", + "sql_mode": " NO_AUTO_VALUE_ON_ZERO,ANSI_QUOTES", + "time_zone": "+00:00", } toSession = map[string]string{ - "sql_mode": " NO_AUTO_VALUE_ON_ZERO,ANSI_QUOTES", + "sql_mode": " NO_AUTO_VALUE_ON_ZERO,ANSI_QUOTES", + "time_zone": "+00:00", } security = Security{ SSLCA: "/path/to/ca", @@ -810,6 +812,8 @@ func (t *testConfig) TestGenAndFromSubTaskConfigs(c *C) { c.Assert(stCfg2.EnableHeartbeat, IsTrue) stCfg1.EnableHeartbeat = false stCfg2.EnableHeartbeat = false + stCfg1.Timezone = defaultTimeZone + stCfg2.Timezone = defaultTimeZone c.Assert(stCfgs[0].String(), Equals, stCfg1.String()) c.Assert(stCfgs[1].String(), Equals, stCfg2.String()) } @@ -889,22 +893,22 @@ func (t *testConfig) TestAdjustTargetDBConfig(c *C) { }{ { DBConfig{}, - DBConfig{Session: map[string]string{}}, + DBConfig{Session: map[string]string{"time_zone": "+00:00"}}, semver.New("0.0.0"), }, { DBConfig{Session: map[string]string{"SQL_MODE": "ANSI_QUOTES"}}, - DBConfig{Session: map[string]string{"sql_mode": "ANSI_QUOTES"}}, + DBConfig{Session: map[string]string{"sql_mode": "ANSI_QUOTES", "time_zone": "+00:00"}}, semver.New("2.0.7"), }, { DBConfig{}, - DBConfig{Session: map[string]string{tidbTxnMode: tidbTxnOptimistic}}, + DBConfig{Session: map[string]string{tidbTxnMode: tidbTxnOptimistic, "time_zone": "+00:00"}}, semver.New("3.0.1"), }, { DBConfig{Session: map[string]string{"SQL_MODE": "", tidbTxnMode: "pessimistic"}}, - DBConfig{Session: map[string]string{"sql_mode": "", tidbTxnMode: "pessimistic"}}, + DBConfig{Session: map[string]string{"sql_mode": "", tidbTxnMode: "pessimistic", "time_zone": "+00:00"}}, semver.New("4.0.0-beta.2"), }, } diff --git a/pkg/utils/common.go b/pkg/utils/common.go index 922fc40005..ef59108843 100644 --- a/pkg/utils/common.go +++ b/pkg/utils/common.go @@ -19,6 +19,9 @@ import ( "fmt" "regexp" "strings" + "time" + + "github.com/pingcap/errors" "github.com/pingcap/dm/pkg/log" "github.com/pingcap/dm/pkg/terror" @@ -28,6 +31,8 @@ import ( "github.com/pingcap/tidb-tools/pkg/dbutil" "github.com/pingcap/tidb-tools/pkg/filter" router "github.com/pingcap/tidb-tools/pkg/table-router" + "github.com/pingcap/tidb/types" + "github.com/pingcap/tidb/util/timeutil" "go.uber.org/zap" ) @@ -209,3 +214,40 @@ func NonRepeatStringsEqual(a, b []string) bool { } return true } + +// ParseTimeZone parse location info from time offset("+08:00") or location string ("Asia/Shanghai") +// +// Copy from https://github.com/pingcap/tidb/blob/263a47e85ce04f74ec80d1d35b426618bc89b5a3/sessionctx/variable/varsutil.go#L411 +func ParseTimeZone(s string) (*time.Location, error) { + if strings.EqualFold(s, "SYSTEM") { + return timeutil.SystemLocation(), nil + } + loc, err := time.LoadLocation(s) + if err == nil { + return loc, nil + } + // The value can be given as a string indicating an offset from UTC, such as '+10:00' or '-6:00'. + // The time zone's value should in [-12:59,+14:00]. + if strings.HasPrefix(s, "+") || strings.HasPrefix(s, "-") { + d, err := types.ParseDuration(nil, s[1:], 0) + if err == nil { + if s[0] == '-' { + if d.Duration > 12*time.Hour+59*time.Minute { + return nil, errors.Errorf("invalid time zone '%s'", s) + } + } else { + if d.Duration > 14*time.Hour { + return nil, errors.Errorf("invalid time zone '%s'", s) + } + } + + ofst := int(d.Duration / time.Second) + if s[0] == '-' { + ofst = -ofst + } + return time.FixedZone("", ofst), nil + } + } + + return nil, errors.Errorf("invalid time zone '%s'", s) +} diff --git a/pkg/utils/common_test.go b/pkg/utils/common_test.go index 5b2b8de634..9e48766760 100644 --- a/pkg/utils/common_test.go +++ b/pkg/utils/common_test.go @@ -17,6 +17,7 @@ import ( "bytes" "context" "fmt" + "time" "github.com/DATA-DOG/go-sqlmock" . "github.com/pingcap/check" @@ -256,3 +257,32 @@ func (s *testCommonSuite) TestNonRepeatStringsEqual(c *C) { c.Assert(NonRepeatStringsEqual([]string{}, []string{"1"}), IsFalse) c.Assert(NonRepeatStringsEqual([]string{"1", "2"}, []string{"2", "3"}), IsFalse) } + +func (s *testCommonSuite) TestParseTimeZone(c *C) { + cases := map[string]time.Duration{ + "+00:00": time.Duration(0), + "+01:00": time.Hour, + "-08:03": -1 * (8*time.Hour + 3*time.Minute), + "-12:59": -1 * (12*time.Hour + 59*time.Minute), + "+12:59": 12*time.Hour + 59*time.Minute, + "Asia/Shanghai": 8 * time.Hour, + "UTC": time.Duration(0), + "CST": -1 * 6 * time.Hour, + } + for k, v := range cases { + dur, err := ParseTimeZone(k) + c.Assert(err, IsNil) + c.Assert(dur, Equals, v) + } + + badCases := []string{ + "test", + "-13:00", + "+14:05", + } + + for _, s := range badCases { + _, err := ParseTimeZone(s) + c.Assert(err, ErrorMatches, "invalid time zone.*") + } +} diff --git a/syncer/db_test.go b/syncer/db_test.go index 104ede3b45..601dcedffd 100644 --- a/syncer/db_test.go +++ b/syncer/db_test.go @@ -83,7 +83,7 @@ func (s *testDBSuite) resetBinlogSyncer(c *C) { VerifyChecksum: true, } if s.cfg.Timezone != "" { - timezone, err2 := time.LoadLocation(s.cfg.Timezone) + timezone, err2 := utils.ParseTimeZone(s.cfg.Timezone) c.Assert(err2, IsNil) cfg.TimestampStringLocation = timezone } @@ -260,7 +260,7 @@ func (s *testDBSuite) TestTimezone(c *C) { err = txn.Commit() c.Assert(err, IsNil) - location, err := time.LoadLocation(testCase.timezone) + location, err := utils.ParseTimeZone(testCase.timezone) c.Assert(err, IsNil) idx := 0 diff --git a/syncer/syncer.go b/syncer/syncer.go index 1e2168baa0..ec44ecadaa 100644 --- a/syncer/syncer.go +++ b/syncer/syncer.go @@ -2770,7 +2770,7 @@ func (s *Syncer) setTimezone() { var loc *time.Location if s.cfg.Timezone != "" { - loc, _ = time.LoadLocation(s.cfg.Timezone) + loc, _ = utils.ParseTimeZone(s.cfg.Timezone) } if loc == nil { loc = time.Now().Location() From 7b061e0ecab6117a68f47bf71bc9cd7f49fd8911 Mon Sep 17 00:00:00 2001 From: glorv Date: Tue, 11 May 2021 18:14:27 +0800 Subject: [PATCH 07/15] remove useless timezone setting --- chaos/cases/conf/task-optimistic.yaml | 1 - chaos/cases/conf/task-pessimistic.yaml | 1 - chaos/cases/conf/task-single.yaml | 1 - cmd/dm-syncer/config.go | 27 +------------------ dm/config/subtask.go | 10 ------- dm/config/subtask_test.go | 10 ------- dm/config/task.go | 9 ------- dm/config/task_test.go | 14 ---------- dm/dm-ansible/conf/task_advanced.yaml.example | 1 - dm/dm-ansible/scripts/dm_instances.json | 1 - dm/master/server_test.go | 1 - dm/master/task_advanced.yaml | 1 - dm/worker/subtask.toml | 4 --- dumpling/dumpling.go | 6 ++--- loader/loader.go | 4 +-- pkg/terror/error_list.go | 2 -- syncer/db_test.go | 7 +---- syncer/syncer.go | 14 ++-------- tests/all_mode/conf/dm-task.yaml | 1 - tests/compatibility/conf/dm-task.yaml | 1 - tests/dm_syncer/conf/dm-syncer-1.toml | 4 --- tests/dm_syncer/conf/dm-syncer-2.toml | 4 --- tests/dm_syncer/conf/dm-task.yaml | 1 - tests/dmctl_basic/conf/dm-task.yaml | 1 - tests/dmctl_basic/conf/dm-task3.yaml | 1 - tests/dmctl_basic/conf/get_task.yaml | 1 - tests/dmctl_command/conf/dm-task.yaml | 1 - .../drop_column_with_index/conf/dm-task.yaml | 1 - tests/full_mode/conf/dm-task.yaml | 1 - tests/ha/conf/dm-task.yaml | 1 - tests/ha_cases/conf/dm-task.yaml | 1 - tests/ha_cases/conf/dm-task2.yaml | 1 - tests/ha_cases/conf/standalone-task.yaml | 1 - tests/ha_cases/conf/standalone-task2.yaml | 1 - tests/ha_cases2/conf/dm-task.yaml | 1 - tests/ha_cases2/conf/dm-task2.yaml | 1 - tests/ha_cases2/conf/standalone-task.yaml | 1 - tests/ha_cases2/conf/standalone-task2.yaml | 1 - tests/ha_master/conf/dm-task.yaml | 1 - .../conf/double-source-no-sharding.yaml | 1 - .../conf/double-source-optimistic.yaml | 1 - .../conf/double-source-pessimistic.yaml | 1 - .../conf/single-source-no-sharding.yaml | 1 - tests/http_apis/conf/dm-task.yaml | 1 - tests/import_goroutine_leak/conf/dm-task.yaml | 1 - tests/import_v10x/conf/task.yaml | 1 - tests/incremental_mode/conf/dm-task.yaml | 1 - tests/initial_unit/conf/dm-task.yaml | 1 - tests/load_interrupt/conf/dm-task.yaml | 1 - tests/many_tables/conf/dm-task.yaml | 1 - tests/new_relay/conf/dm-task.yaml | 1 - tests/online_ddl/conf/dm-task.yaml | 1 - tests/only_dml/conf/dm-task.yaml | 1 - tests/print_status/conf/dm-task.yaml | 1 - tests/relay_interrupt/conf/dm-task.yaml | 1 - tests/retry_cancel/conf/dm-task.yaml | 1 - tests/safe_mode/conf/dm-task.yaml | 1 - tests/sequence_safe_mode/conf/dm-task.yaml | 1 - tests/sequence_sharding/conf/dm-task.yaml | 1 - .../conf/dm-task.yaml | 1 - .../conf/dm-task.yaml | 1 - .../conf/double-source-optimistic.yaml | 1 - .../conf/double-source-pessimistic.yaml | 1 - .../conf/single-source-no-routes.yaml | 1 - .../conf/single-source-no-sharding.yaml | 1 - .../conf/single-source-optimistic.yaml | 1 - .../conf/single-source-pessimistic.yaml | 1 - .../conf/double-source-optimistic.yaml | 1 - .../conf/double-source-pessimistic.yaml | 1 - .../conf/double-source-optimistic.yaml | 1 - .../conf/double-source-pessimistic.yaml | 1 - .../conf/single-source-optimistic-2.yaml | 1 - .../conf/single-source-optimistic.yaml | 1 - .../conf/single-source-pessimistic-2.yaml | 1 - .../conf/single-source-pessimistic.yaml | 1 - .../conf/double-source-optimistic.yaml | 1 - .../conf/double-source-pessimistic.yaml | 1 - .../conf/single-source-optimistic-2.yaml | 1 - .../conf/single-source-optimistic.yaml | 1 - .../conf/single-source-pessimistic-2.yaml | 1 - .../conf/single-source-pessimistic.yaml | 1 - tests/sharding/conf/dm-task.yaml | 1 - tests/sharding2/conf/dm-task.yaml | 1 - tests/start_task/conf/dm-task.yaml | 1 - tests/tls/conf/dm-task.yaml | 1 - 85 files changed, 7 insertions(+), 180 deletions(-) diff --git a/chaos/cases/conf/task-optimistic.yaml b/chaos/cases/conf/task-optimistic.yaml index 9c77787274..84113731be 100644 --- a/chaos/cases/conf/task-optimistic.yaml +++ b/chaos/cases/conf/task-optimistic.yaml @@ -7,7 +7,6 @@ shard-mode: optimistic # so ignore sharding table check, seems has no effect on sync-diff-inspector # see https://dev.mysql.com/doc/relnotes/mysql/8.0/en/news-8-0-19.html ignore-checking-items: ["schema_of_shard_tables","auto_increment_ID"] # tables generated by go-sqlsmith may have auto increment ID -timezone: "UTC" target-database: host: "tidb-0.tidb" diff --git a/chaos/cases/conf/task-pessimistic.yaml b/chaos/cases/conf/task-pessimistic.yaml index 4fb63cfbdb..0af0c2b7a2 100644 --- a/chaos/cases/conf/task-pessimistic.yaml +++ b/chaos/cases/conf/task-pessimistic.yaml @@ -7,7 +7,6 @@ shard-mode: pessimistic # so ignore sharding table check, seems has no effect on sync-diff-inspector # see https://dev.mysql.com/doc/relnotes/mysql/8.0/en/news-8-0-19.html ignore-checking-items: ["schema_of_shard_tables","auto_increment_ID"] # tables generated by go-sqlsmith may have auto increment ID -timezone: "UTC" target-database: host: "tidb-0.tidb" diff --git a/chaos/cases/conf/task-single.yaml b/chaos/cases/conf/task-single.yaml index 31d91b60c7..e8b725b7d5 100644 --- a/chaos/cases/conf/task-single.yaml +++ b/chaos/cases/conf/task-single.yaml @@ -1,7 +1,6 @@ --- name: "task_single" task-mode: all -timezone: "UTC" target-database: host: "tidb-0.tidb" diff --git a/cmd/dm-syncer/config.go b/cmd/dm-syncer/config.go index 300dbf0a58..b50143ae1b 100644 --- a/cmd/dm-syncer/config.go +++ b/cmd/dm-syncer/config.go @@ -16,18 +16,15 @@ package main import ( "flag" "fmt" - "os" - "time" - "github.com/BurntSushi/toml" "github.com/go-mysql-org/go-mysql/mysql" "github.com/pingcap/errors" bf "github.com/pingcap/tidb-tools/pkg/binlog-filter" "github.com/pingcap/tidb-tools/pkg/filter" router "github.com/pingcap/tidb-tools/pkg/table-router" + "os" "github.com/pingcap/dm/dm/config" - "github.com/pingcap/dm/pkg/log" "github.com/pingcap/dm/pkg/utils" ) @@ -55,8 +52,6 @@ type commonConfig struct { SafeMode bool MaxRetry int - TimezoneStr string - SyncerConfigFormat bool } @@ -77,7 +72,6 @@ func (c *commonConfig) newConfigFromSyncerConfig(args []string) (*config.SubTask EnableGTID: c.EnableGTID, SafeMode: c.SafeMode, MaxRetry: c.MaxRetry, - TimezoneStr: c.TimezoneStr, } cfg.FlagSet = flag.NewFlagSet("dm-syncer", flag.ContinueOnError) @@ -102,7 +96,6 @@ func (c *commonConfig) newConfigFromSyncerConfig(args []string) (*config.SubTask fs.BoolVar(&cfg.EnableGTID, "enable-gtid", false, "enable gtid mode") fs.BoolVar(&cfg.SafeMode, "safe-mode", false, "enable safe mode to make syncer reentrant") fs.IntVar(&cfg.MaxRetry, "max-retry", 100, "maxinum retry when network interruption") - fs.StringVar(&cfg.TimezoneStr, "timezone", "", "target database timezone location string") fs.BoolVar(&SyncerConfigFormat, "syncer-config-format", false, "read syncer config format") if err := fs.Parse(args); err != nil { @@ -164,7 +157,6 @@ func (c *commonConfig) newSubTaskConfig(args []string) (*config.SubTaskConfig, e fs.BoolVar(&cfg.EnableGTID, "enable-gtid", false, "enable gtid mode") fs.BoolVar(&cfg.SafeMode, "safe-mode", false, "enable safe mode to make syncer reentrant") fs.IntVar(&cfg.MaxRetry, "max-retry", 100, "maxinum retry when network interruption") - fs.StringVar(&cfg.Timezone, "timezone", "", "target database timezone location string") fs.StringVar(&cfg.Name, "cp-table-prefix", "dm-syncer", "the prefix of the checkpoint table name") fs.BoolVar(&syncerConfigFormat, "syncer-config-format", false, "read syncer config format") @@ -178,11 +170,6 @@ func (c *commonConfig) newSubTaskConfig(args []string) (*config.SubTaskConfig, e cfg.ServerID = uint32(serverID) } - if cfg.Timezone != "" { - log.L().Warn("timezone is deprecated, will auto use UTC instead.") - cfg.Timezone = "+00:00" - } - return cfg, nil } @@ -208,16 +195,8 @@ func newCommonConfig() *commonConfig { fs.BoolVar(&cfg.EnableGTID, "enable-gtid", false, "enable gtid mode") fs.BoolVar(&cfg.SafeMode, "safe-mode", false, "enable safe mode to make syncer reentrant") fs.IntVar(&cfg.MaxRetry, "max-retry", 100, "maxinum retry when network interruption") - fs.StringVar(&cfg.TimezoneStr, "timezone", "", "target database timezone location string") - - if cfg.TimezoneStr != "" { - log.L().Warn("timezone is deprecated, will auto use UTC instead.") - cfg.TimezoneStr = "+00:00" - } - fs.BoolVar(&cfg.SyncerConfigFormat, "syncer-config-format", false, "read syncer config format") - return cfg } @@ -278,9 +257,6 @@ type syncerConfig struct { // MaxDMLConnectionTimeout string `toml:"execute-dml-timeout" json:"execute-dml-timeout"` // ExecutionQueueLength int `toml:"execute-queue-length" json:"execute-queue-length"` - TimezoneStr string `toml:"timezone" json:"timezone"` - Timezone *time.Location `json:"-"` - printVersion bool } @@ -356,7 +332,6 @@ func (oc *syncerConfig) convertToNewFormat() (*config.SubTaskConfig, error) { }, ConfigFile: oc.ConfigFile, - Timezone: oc.TimezoneStr, From: oc.From, To: oc.To, } diff --git a/dm/config/subtask.go b/dm/config/subtask.go index 254722bb44..02b0b5a0d6 100644 --- a/dm/config/subtask.go +++ b/dm/config/subtask.go @@ -151,9 +151,6 @@ type SubTaskConfig struct { HeartbeatReportInterval int `toml:"heartbeat-report-interval" json:"heartbeat-report-interval"` EnableHeartbeat bool `toml:"enable-heartbeat" json:"enable-heartbeat"` Meta *Meta `toml:"meta" json:"meta"` - // deprecated, will auto adjust source and target time_zone - Timezone string `toml:"timezone" json:"timezone"` - // RelayDir get value from dm-worker config RelayDir string `toml:"relay-dir" json:"relay-dir"` @@ -275,13 +272,6 @@ func (c *SubTaskConfig) Adjust(verifyDecryptPassword bool) error { c.MetaSchema = defaultMetaSchema } - if c.Timezone != "" { - _, err := utils.ParseTimeZone(c.Timezone) - if err != nil { - return terror.ErrConfigInvalidTimezone.Delegate(err, c.Timezone) - } - } - dirSuffix := "." + c.Name if !strings.HasSuffix(c.LoaderConfig.Dir, dirSuffix) { // check to support multiple times calling // if not ends with the task name, we append the task name to the tail diff --git a/dm/config/subtask_test.go b/dm/config/subtask_test.go index 101d9b7512..b0241206fb 100644 --- a/dm/config/subtask_test.go +++ b/dm/config/subtask_test.go @@ -25,7 +25,6 @@ func (t *testConfig) TestSubTask(c *C) { ShardMode: "optimistic", SourceID: "mysql-instance-01", OnlineDDLScheme: "pt", - Timezone: "Asia/Shanghai", From: DBConfig{ Host: "127.0.0.1", Port: 3306, @@ -74,7 +73,6 @@ func (t *testConfig) TestSubTaskAdjustFail(c *C) { Name: "test-task", SourceID: "mysql-instance-01", OnlineDDLScheme: "pt", - Timezone: "Asia/Shanghai", From: DBConfig{ Host: "127.0.0.1", Port: 3306, @@ -133,14 +131,6 @@ func (t *testConfig) TestSubTaskAdjustFail(c *C) { }, "\\[.*\\], Message: online scheme rtc not supported.*", }, - { - func() *SubTaskConfig { - cfg := newSubTaskConfig() - cfg.Timezone = "my-house" - return cfg - }, - "\\[.*\\], Message: invalid timezone string: my-house.*", - }, } for _, tc := range testCases { diff --git a/dm/config/task.go b/dm/config/task.go index ff90b842dd..576d050ca4 100644 --- a/dm/config/task.go +++ b/dm/config/task.go @@ -285,8 +285,6 @@ type TaskConfig struct { EnableHeartbeat bool `yaml:"enable-heartbeat" toml:"enable-heartbeat" json:"enable-heartbeat"` HeartbeatUpdateInterval int `yaml:"heartbeat-update-interval" toml:"heartbeat-update-interval" json:"heartbeat-update-interval"` HeartbeatReportInterval int `yaml:"heartbeat-report-interval" toml:"heartbeat-report-interval" json:"heartbeat-report-interval"` - // deprecated, needn't set anymore, will always use UTC instead - Timezone string `yaml:"timezone" toml:"timezone" json:"timezone"` // handle schema/table name mode, and only for schema/table name // if case insensitive, we would convert schema/table name to lower case @@ -595,11 +593,6 @@ func (c *TaskConfig) adjust() error { return terror.ErrConfigGlobalConfigsUnused.Generate(unusedConfigs) } - if c.Timezone != "" { - log.L().Warn("timezone config is deprecated, will automatically use UTC instead") - } - c.Timezone = defaultTimeZone - if c.RemoveMeta { log.L().Warn("`remove-meta` in task config is deprecated, please use `start-task ... --remove-meta` instead") } @@ -631,7 +624,6 @@ func (c *TaskConfig) SubTaskConfigs(sources map[string]DBConfig) ([]*SubTaskConf } cfg.HeartbeatUpdateInterval = c.HeartbeatUpdateInterval cfg.HeartbeatReportInterval = c.HeartbeatReportInterval - cfg.Timezone = c.Timezone cfg.Meta = inst.Meta cfg.From = dbCfg @@ -709,7 +701,6 @@ func FromSubTaskConfigs(stCfgs ...*SubTaskConfig) *TaskConfig { c.EnableHeartbeat = stCfg0.EnableHeartbeat c.HeartbeatUpdateInterval = stCfg0.HeartbeatUpdateInterval c.HeartbeatReportInterval = stCfg0.HeartbeatReportInterval - c.Timezone = stCfg0.Timezone c.CaseSensitive = stCfg0.CaseSensitive c.TargetDB = &stCfg0.To // just ref c.OnlineDDLScheme = stCfg0.OnlineDDLScheme diff --git a/dm/config/task_test.go b/dm/config/task_test.go index a1f4cd36f0..462c9ceb8e 100644 --- a/dm/config/task_test.go +++ b/dm/config/task_test.go @@ -35,7 +35,6 @@ name: test task-mode: all shard-mode: "pessimistic" meta-schema: "dm_meta" -timezone: "Asia/Shanghai" case-sensitive: false online-ddl-scheme: "gh-ost" clean-dump-file: true @@ -137,7 +136,6 @@ name: test task-mode: all shard-mode: "pessimistic" meta-schema: "dm_meta" -timezone: "Asia/Shanghai" case-sensitive: false online-ddl-scheme: "gh-ost" clean-dump-file: true @@ -245,7 +243,6 @@ task-mode: all is-sharding: true meta-schema: "dm_meta" enable-heartbeat: true -timezone: "Asia/Shanghai" ignore-checking-items: ["all"] target-database: @@ -271,7 +268,6 @@ task-mode: all is-sharding: true meta-schema: "dm_meta" enable-heartbeat: true -timezone: "Asia/Shanghai" ignore-checking-items: ["all"] target-database: @@ -308,7 +304,6 @@ task-mode: all is-sharding: true meta-schema: "dm_meta" enable-heartbeat: true -timezone: "Asia/Shanghai" ignore-checking-items: ["all"] `) err = ioutil.WriteFile(filepath, configContent, 0o644) @@ -325,7 +320,6 @@ task-mode: all is-sharding: true meta-schema: "dm_meta" enable-heartbeat: true -timezone: "Asia/Shanghai" ignore-checking-items: ["all"] `) err = ioutil.WriteFile(filepath, configContent, 0o644) @@ -340,7 +334,6 @@ name: test is-sharding: true meta-schema: "dm_meta" enable-heartbeat: true -timezone: "Asia/Shanghai" ignore-checking-items: ["all"] `) err = ioutil.WriteFile(filepath, configContent, 0o644) @@ -358,7 +351,6 @@ meta-schema: "dm_meta" enable-heartbeat: true heartbeat-update-interval: 1 heartbeat-report-interval: 1 -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" @@ -436,7 +428,6 @@ meta-schema: "dm_meta" enable-heartbeat: true heartbeat-update-interval: 1 heartbeat-report-interval: 1 -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" @@ -543,7 +534,6 @@ func (t *testConfig) TestGenAndFromSubTaskConfigs(c *C) { metaSchema = "meta-sub-tasks" heartbeatUI = 12 heartbeatRI = 21 - timezone = "Asia/Shanghai" maxAllowedPacket = 10244201 fromSession = map[string]string{ "sql_mode": " NO_AUTO_VALUE_ON_ZERO,ANSI_QUOTES", @@ -650,7 +640,6 @@ func (t *testConfig) TestGenAndFromSubTaskConfigs(c *C) { BinLogPos: 456, BinLogGTID: "1-1-12,4-4-4", }, - Timezone: timezone, From: source1DBCfg, To: DBConfig{ Host: "127.0.0.1", @@ -719,7 +708,6 @@ func (t *testConfig) TestGenAndFromSubTaskConfigs(c *C) { EnableHeartbeat: stCfg1.EnableHeartbeat, HeartbeatUpdateInterval: heartbeatUI, HeartbeatReportInterval: heartbeatRI, - Timezone: timezone, CaseSensitive: stCfg1.CaseSensitive, TargetDB: &stCfg1.To, MySQLInstances: []*MySQLInstance{ @@ -812,8 +800,6 @@ func (t *testConfig) TestGenAndFromSubTaskConfigs(c *C) { c.Assert(stCfg2.EnableHeartbeat, IsTrue) stCfg1.EnableHeartbeat = false stCfg2.EnableHeartbeat = false - stCfg1.Timezone = defaultTimeZone - stCfg2.Timezone = defaultTimeZone c.Assert(stCfgs[0].String(), Equals, stCfg1.String()) c.Assert(stCfgs[1].String(), Equals, stCfg2.String()) } diff --git a/dm/dm-ansible/conf/task_advanced.yaml.example b/dm/dm-ansible/conf/task_advanced.yaml.example index 6b1fbe2c66..9b56179388 100644 --- a/dm/dm-ansible/conf/task_advanced.yaml.example +++ b/dm/dm-ansible/conf/task_advanced.yaml.example @@ -6,7 +6,6 @@ meta-schema: "dm_meta" # meta schema in downstreaming database to store meta in enable-heartbeat: false # whether to enable heartbeat for calculating lag between master and syncer # heartbeat-update-interval: 1 # interval to do heartbeat and save timestamp, default 1s # heartbeat-report-interval: 10 # interval to report time lap to prometheus, default 10s -# timezone: "Asia/Shanghai" # target database timezone, all timestamp event in binlog will translate to format time based on this timezone, default use local timezone target-database: host: "192.168.0.1" diff --git a/dm/dm-ansible/scripts/dm_instances.json b/dm/dm-ansible/scripts/dm_instances.json index 8862183488..95c6fef83d 100644 --- a/dm/dm-ansible/scripts/dm_instances.json +++ b/dm/dm-ansible/scripts/dm_instances.json @@ -1504,7 +1504,6 @@ "30d" ] }, - "timezone": "", "title": "Test-Cluster-DM-worker-instances", "uid": "le1FDN_Wz", "version": 1 diff --git a/dm/master/server_test.go b/dm/master/server_test.go index 46b6226838..c81c0c3e0f 100644 --- a/dm/master/server_test.go +++ b/dm/master/server_test.go @@ -68,7 +68,6 @@ is-sharding: true shard-mode: "" meta-schema: "dm_meta" enable-heartbeat: true -timezone: "Asia/Shanghai" ignore-checking-items: ["all"] target-database: diff --git a/dm/master/task_advanced.yaml b/dm/master/task_advanced.yaml index 6b1fbe2c66..9b56179388 100644 --- a/dm/master/task_advanced.yaml +++ b/dm/master/task_advanced.yaml @@ -6,7 +6,6 @@ meta-schema: "dm_meta" # meta schema in downstreaming database to store meta in enable-heartbeat: false # whether to enable heartbeat for calculating lag between master and syncer # heartbeat-update-interval: 1 # interval to do heartbeat and save timestamp, default 1s # heartbeat-report-interval: 10 # interval to report time lap to prometheus, default 10s -# timezone: "Asia/Shanghai" # target database timezone, all timestamp event in binlog will translate to format time based on this timezone, default use local timezone target-database: host: "192.168.0.1" diff --git a/dm/worker/subtask.toml b/dm/worker/subtask.toml index f6b7e6c802..b46f8f0aa4 100644 --- a/dm/worker/subtask.toml +++ b/dm/worker/subtask.toml @@ -53,10 +53,6 @@ meta-file = "./syncer.subTaskA.meta" worker-count = 16 batch = 1000 - -# target database timezone, all timestamp event in binlog will translate to format time based on this timezone, default use local timezone -# timezone = "Asia/Shanghai" - # filter # block allow list provides a library to filter replicate on schema/table by given rules diff --git a/dumpling/dumpling.go b/dumpling/dumpling.go index 85e9edb4ef..9c9de1ae3d 100644 --- a/dumpling/dumpling.go +++ b/dumpling/dumpling.go @@ -60,10 +60,8 @@ func (m *Dumpling) Init(ctx context.Context) error { var err error m.dumpConfig, err = m.constructArgs() m.detectSQLMode(ctx) - if m.cfg.Timezone != "" { - m.dumpConfig.SessionParams["time_zone"] = m.cfg.Timezone - } - m.logger.Info("create dumpling", zap.Stringer("config", m.dumpConfig)) + m.dumpConfig.SessionParams["time_zone"] = "+00:00" + m.logger.Info("create dumpling", zap.Stringer("config", m.dumpConfig)) return err } diff --git a/loader/loader.go b/loader/loader.go index a19ce47547..1d951a204c 100644 --- a/loader/loader.go +++ b/loader/loader.go @@ -495,9 +495,7 @@ func (l *Loader) Init(ctx context.Context) (err error) { if lcfg.To.Session == nil { lcfg.To.Session = make(map[string]string) } - if lcfg.Timezone != "" { - lcfg.To.Session["time_zone"] = lcfg.Timezone - } + lcfg.To.Session["time_zone"] = "+00:00" hasSQLMode := false for k := range l.cfg.To.Session { diff --git a/pkg/terror/error_list.go b/pkg/terror/error_list.go index fb83de36a9..34fe022348 100644 --- a/pkg/terror/error_list.go +++ b/pkg/terror/error_list.go @@ -196,7 +196,6 @@ const ( codeConfigEmptySourceID codeConfigTooLongSourceID codeConfigOnlineSchemeNotSupport - codeConfigInvalidTimezone codeConfigParseFlagSet codeConfigDecryptDBPassword codeConfigMetaInvalid @@ -804,7 +803,6 @@ var ( ErrConfigEmptySourceID = New(codeConfigEmptySourceID, ClassConfig, ScopeInternal, LevelMedium, "empty source-id not valid", "Please check the `source-id` config in configuration file.") ErrConfigTooLongSourceID = New(codeConfigTooLongSourceID, ClassConfig, ScopeInternal, LevelMedium, "too long source-id not valid", "Please check the `source-id` config in configuration file. The max source id length is 32.") ErrConfigOnlineSchemeNotSupport = New(codeConfigOnlineSchemeNotSupport, ClassConfig, ScopeInternal, LevelMedium, "online scheme %s not supported", "Please check the `online-ddl-scheme` config in task configuration file. Only `ghost` and `pt` are currently supported.") - ErrConfigInvalidTimezone = New(codeConfigInvalidTimezone, ClassConfig, ScopeInternal, LevelMedium, "invalid timezone string: %s", "Please check the `timezone` config in task configuration file.") ErrConfigParseFlagSet = New(codeConfigParseFlagSet, ClassConfig, ScopeInternal, LevelMedium, "parse subtask config flag set", "") ErrConfigDecryptDBPassword = New(codeConfigDecryptDBPassword, ClassConfig, ScopeInternal, LevelMedium, "decrypt DB password %s failed", "") ErrConfigMetaInvalid = New(codeConfigMetaInvalid, ClassConfig, ScopeInternal, LevelMedium, "must specify `binlog-name` without GTID enabled for the source or specify `binlog-gtid` with GTID enabled for the source", "Please check the `meta` config in task configuration file.") diff --git a/syncer/db_test.go b/syncer/db_test.go index 601dcedffd..9aad570b05 100644 --- a/syncer/db_test.go +++ b/syncer/db_test.go @@ -82,11 +82,7 @@ func (s *testDBSuite) resetBinlogSyncer(c *C) { UseDecimal: true, VerifyChecksum: true, } - if s.cfg.Timezone != "" { - timezone, err2 := utils.ParseTimeZone(s.cfg.Timezone) - c.Assert(err2, IsNil) - cfg.TimestampStringLocation = timezone - } + cfg.TimestampStringLocation = time.UTC if s.syncer != nil { s.syncer.Close() @@ -236,7 +232,6 @@ func (s *testDBSuite) TestTimezone(c *C) { } for _, testCase := range testCases { - s.cfg.Timezone = testCase.timezone syncer := NewSyncer(s.cfg, nil) c.Assert(syncer.genRouter(), IsNil) s.resetBinlogSyncer(c) diff --git a/syncer/syncer.go b/syncer/syncer.go index ec44ecadaa..5d226ece23 100644 --- a/syncer/syncer.go +++ b/syncer/syncer.go @@ -2714,7 +2714,6 @@ func (s *Syncer) Update(cfg *config.SubTaskConfig) error { s.cfg.RouteRules = cfg.RouteRules s.cfg.FilterRules = cfg.FilterRules s.cfg.ColumnMappingRules = cfg.ColumnMappingRules - s.cfg.Timezone = cfg.Timezone // update timezone s.setTimezone() @@ -2767,17 +2766,8 @@ func (s *Syncer) UpdateFromConfig(cfg *config.SubTaskConfig) error { } func (s *Syncer) setTimezone() { - var loc *time.Location - - if s.cfg.Timezone != "" { - loc, _ = utils.ParseTimeZone(s.cfg.Timezone) - } - if loc == nil { - loc = time.Now().Location() - s.tctx.L().Warn("use system default time location") - } - s.tctx.L().Info("use timezone", log.WrapStringerField("location", loc)) - s.timezone = loc + s.tctx.L().Info("use timezone", log.WrapStringerField("location", time.UTC)) + s.timezone = time.UTC } func (s *Syncer) setSyncCfg() error { diff --git a/tests/all_mode/conf/dm-task.yaml b/tests/all_mode/conf/dm-task.yaml index c1de2ff48b..3d303f76a5 100644 --- a/tests/all_mode/conf/dm-task.yaml +++ b/tests/all_mode/conf/dm-task.yaml @@ -6,7 +6,6 @@ meta-schema: "dm_meta" # enable-heartbeat: true heartbeat-update-interval: 1 heartbeat-report-interval: 1 -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/compatibility/conf/dm-task.yaml b/tests/compatibility/conf/dm-task.yaml index 17b0a9809c..5abe754c52 100644 --- a/tests/compatibility/conf/dm-task.yaml +++ b/tests/compatibility/conf/dm-task.yaml @@ -6,7 +6,6 @@ meta-schema: "dm_meta" # enable-heartbeat: true heartbeat-update-interval: 1 heartbeat-report-interval: 1 -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/dm_syncer/conf/dm-syncer-1.toml b/tests/dm_syncer/conf/dm-syncer-1.toml index 4826a33c03..2707f65d0d 100644 --- a/tests/dm_syncer/conf/dm-syncer-1.toml +++ b/tests/dm_syncer/conf/dm-syncer-1.toml @@ -52,10 +52,6 @@ binlog-pos = binlog-pos-placeholder-1 worker-count = 16 batch = 1000 - -# target database timezone, all timestamp event in binlog will translate to format time based on this timezone, default use local timezone -# timezone = "Asia/Shanghai" - # filter # block allow list provides a library to filter replicate on schema/table by given rules diff --git a/tests/dm_syncer/conf/dm-syncer-2.toml b/tests/dm_syncer/conf/dm-syncer-2.toml index b3ab30e2d7..825746bab6 100644 --- a/tests/dm_syncer/conf/dm-syncer-2.toml +++ b/tests/dm_syncer/conf/dm-syncer-2.toml @@ -12,10 +12,6 @@ flavor = "mysql" worker-count = 16 batch = 1000 - -# target database timezone, all timestamp event in binlog will translate to format time based on this timezone, default use local timezone -# timezone = "Asia/Shanghai" - # filter # block allow list provides a library to filter replicate on schema/table by given rules diff --git a/tests/dm_syncer/conf/dm-task.yaml b/tests/dm_syncer/conf/dm-task.yaml index 53bb403668..1a8d9e122a 100644 --- a/tests/dm_syncer/conf/dm-task.yaml +++ b/tests/dm_syncer/conf/dm-task.yaml @@ -6,7 +6,6 @@ meta-schema: "dm_meta" # enable-heartbeat: true heartbeat-update-interval: 1 heartbeat-report-interval: 1 -timezone: "Asia/Shanghai" clean-dump-file: false target-database: diff --git a/tests/dmctl_basic/conf/dm-task.yaml b/tests/dmctl_basic/conf/dm-task.yaml index fe29e72e69..a048ca36b3 100644 --- a/tests/dmctl_basic/conf/dm-task.yaml +++ b/tests/dmctl_basic/conf/dm-task.yaml @@ -4,7 +4,6 @@ task-mode: all is-sharding: true meta-schema: "dm_meta" enable-heartbeat: false -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/dmctl_basic/conf/dm-task3.yaml b/tests/dmctl_basic/conf/dm-task3.yaml index 13a759e3f4..a36d82e80c 100644 --- a/tests/dmctl_basic/conf/dm-task3.yaml +++ b/tests/dmctl_basic/conf/dm-task3.yaml @@ -4,7 +4,6 @@ task-mode: all is-sharding: true meta-schema: "dm_meta" enable-heartbeat: false -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/dmctl_basic/conf/get_task.yaml b/tests/dmctl_basic/conf/get_task.yaml index 2c2c812fc0..d03f85fb6d 100644 --- a/tests/dmctl_basic/conf/get_task.yaml +++ b/tests/dmctl_basic/conf/get_task.yaml @@ -7,7 +7,6 @@ meta-schema: dm_meta enable-heartbeat: false heartbeat-update-interval: 1 heartbeat-report-interval: 10 -timezone: Asia/Shanghai case-sensitive: false target-database: host: 127.0.0.1 diff --git a/tests/dmctl_command/conf/dm-task.yaml b/tests/dmctl_command/conf/dm-task.yaml index 4c5473c5f5..21b6540b32 100644 --- a/tests/dmctl_command/conf/dm-task.yaml +++ b/tests/dmctl_command/conf/dm-task.yaml @@ -6,7 +6,6 @@ meta-schema: "dm_meta" # enable-heartbeat: true heartbeat-update-interval: 1 heartbeat-report-interval: 1 -timezone: "Asia/Shanghai" remove-meta: true target-database: diff --git a/tests/drop_column_with_index/conf/dm-task.yaml b/tests/drop_column_with_index/conf/dm-task.yaml index e29507a35e..54ae455a51 100644 --- a/tests/drop_column_with_index/conf/dm-task.yaml +++ b/tests/drop_column_with_index/conf/dm-task.yaml @@ -6,7 +6,6 @@ meta-schema: "dm_meta" # enable-heartbeat: true heartbeat-update-interval: 1 heartbeat-report-interval: 1 -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/full_mode/conf/dm-task.yaml b/tests/full_mode/conf/dm-task.yaml index 84c3c528f5..75c21d4f8b 100644 --- a/tests/full_mode/conf/dm-task.yaml +++ b/tests/full_mode/conf/dm-task.yaml @@ -6,7 +6,6 @@ meta-schema: "dm_meta" # enable-heartbeat: true heartbeat-update-interval: 1 heartbeat-report-interval: 1 -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/ha/conf/dm-task.yaml b/tests/ha/conf/dm-task.yaml index e818fbf085..2610bdc6fa 100644 --- a/tests/ha/conf/dm-task.yaml +++ b/tests/ha/conf/dm-task.yaml @@ -6,7 +6,6 @@ meta-schema: "dm_meta" # enable-heartbeat: true heartbeat-update-interval: 1 heartbeat-report-interval: 1 -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/ha_cases/conf/dm-task.yaml b/tests/ha_cases/conf/dm-task.yaml index e818fbf085..2610bdc6fa 100644 --- a/tests/ha_cases/conf/dm-task.yaml +++ b/tests/ha_cases/conf/dm-task.yaml @@ -6,7 +6,6 @@ meta-schema: "dm_meta" # enable-heartbeat: true heartbeat-update-interval: 1 heartbeat-report-interval: 1 -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/ha_cases/conf/dm-task2.yaml b/tests/ha_cases/conf/dm-task2.yaml index dc233a8af1..ea23a367ff 100644 --- a/tests/ha_cases/conf/dm-task2.yaml +++ b/tests/ha_cases/conf/dm-task2.yaml @@ -6,7 +6,6 @@ meta-schema: "dm_meta" # enable-heartbeat: true heartbeat-update-interval: 1 heartbeat-report-interval: 1 -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/ha_cases/conf/standalone-task.yaml b/tests/ha_cases/conf/standalone-task.yaml index a201efcb43..be9c235e84 100644 --- a/tests/ha_cases/conf/standalone-task.yaml +++ b/tests/ha_cases/conf/standalone-task.yaml @@ -6,7 +6,6 @@ meta-schema: "dm_meta" # enable-heartbeat: true heartbeat-update-interval: 1 heartbeat-report-interval: 1 -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/ha_cases/conf/standalone-task2.yaml b/tests/ha_cases/conf/standalone-task2.yaml index 4de5d77670..d51f51df00 100644 --- a/tests/ha_cases/conf/standalone-task2.yaml +++ b/tests/ha_cases/conf/standalone-task2.yaml @@ -6,7 +6,6 @@ meta-schema: "dm_meta" # enable-heartbeat: true heartbeat-update-interval: 1 heartbeat-report-interval: 1 -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/ha_cases2/conf/dm-task.yaml b/tests/ha_cases2/conf/dm-task.yaml index e818fbf085..2610bdc6fa 100644 --- a/tests/ha_cases2/conf/dm-task.yaml +++ b/tests/ha_cases2/conf/dm-task.yaml @@ -6,7 +6,6 @@ meta-schema: "dm_meta" # enable-heartbeat: true heartbeat-update-interval: 1 heartbeat-report-interval: 1 -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/ha_cases2/conf/dm-task2.yaml b/tests/ha_cases2/conf/dm-task2.yaml index dc233a8af1..ea23a367ff 100644 --- a/tests/ha_cases2/conf/dm-task2.yaml +++ b/tests/ha_cases2/conf/dm-task2.yaml @@ -6,7 +6,6 @@ meta-schema: "dm_meta" # enable-heartbeat: true heartbeat-update-interval: 1 heartbeat-report-interval: 1 -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/ha_cases2/conf/standalone-task.yaml b/tests/ha_cases2/conf/standalone-task.yaml index a201efcb43..be9c235e84 100644 --- a/tests/ha_cases2/conf/standalone-task.yaml +++ b/tests/ha_cases2/conf/standalone-task.yaml @@ -6,7 +6,6 @@ meta-schema: "dm_meta" # enable-heartbeat: true heartbeat-update-interval: 1 heartbeat-report-interval: 1 -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/ha_cases2/conf/standalone-task2.yaml b/tests/ha_cases2/conf/standalone-task2.yaml index 4de5d77670..d51f51df00 100644 --- a/tests/ha_cases2/conf/standalone-task2.yaml +++ b/tests/ha_cases2/conf/standalone-task2.yaml @@ -6,7 +6,6 @@ meta-schema: "dm_meta" # enable-heartbeat: true heartbeat-update-interval: 1 heartbeat-report-interval: 1 -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/ha_master/conf/dm-task.yaml b/tests/ha_master/conf/dm-task.yaml index 16093491f9..e4f597edfa 100644 --- a/tests/ha_master/conf/dm-task.yaml +++ b/tests/ha_master/conf/dm-task.yaml @@ -6,7 +6,6 @@ meta-schema: "dm_meta" # enable-heartbeat: true heartbeat-update-interval: 1 heartbeat-report-interval: 1 -timezone: "Asia/Shanghai" target-database: host: "localhost" diff --git a/tests/handle_error/conf/double-source-no-sharding.yaml b/tests/handle_error/conf/double-source-no-sharding.yaml index d2116075a0..c483994ec3 100644 --- a/tests/handle_error/conf/double-source-no-sharding.yaml +++ b/tests/handle_error/conf/double-source-no-sharding.yaml @@ -3,7 +3,6 @@ name: test task-mode: all is-sharding: false meta-schema: "dm_meta" -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/handle_error/conf/double-source-optimistic.yaml b/tests/handle_error/conf/double-source-optimistic.yaml index d1b6dda25e..f92e748d31 100644 --- a/tests/handle_error/conf/double-source-optimistic.yaml +++ b/tests/handle_error/conf/double-source-optimistic.yaml @@ -4,7 +4,6 @@ task-mode: all is-sharding: true shard-mode: "optimistic" meta-schema: "dm_meta" -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/handle_error/conf/double-source-pessimistic.yaml b/tests/handle_error/conf/double-source-pessimistic.yaml index 739ac8581c..85875049de 100644 --- a/tests/handle_error/conf/double-source-pessimistic.yaml +++ b/tests/handle_error/conf/double-source-pessimistic.yaml @@ -4,7 +4,6 @@ task-mode: all is-sharding: true shard-mode: "pessimistic" meta-schema: "dm_meta" -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/handle_error/conf/single-source-no-sharding.yaml b/tests/handle_error/conf/single-source-no-sharding.yaml index cf75a92fe5..e3c6c18da9 100644 --- a/tests/handle_error/conf/single-source-no-sharding.yaml +++ b/tests/handle_error/conf/single-source-no-sharding.yaml @@ -3,7 +3,6 @@ name: test task-mode: all is-sharding: false meta-schema: "dm_meta" -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/http_apis/conf/dm-task.yaml b/tests/http_apis/conf/dm-task.yaml index dfc6f8e108..4ffcab07cb 100644 --- a/tests/http_apis/conf/dm-task.yaml +++ b/tests/http_apis/conf/dm-task.yaml @@ -4,7 +4,6 @@ task-mode: all is-sharding: false meta-schema: "dm_meta" # enable-heartbeat: true -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/import_goroutine_leak/conf/dm-task.yaml b/tests/import_goroutine_leak/conf/dm-task.yaml index 232e2b27ab..210bca31e2 100644 --- a/tests/import_goroutine_leak/conf/dm-task.yaml +++ b/tests/import_goroutine_leak/conf/dm-task.yaml @@ -4,7 +4,6 @@ task-mode: full is-sharding: false meta-schema: "dm_meta" # enable-heartbeat: true -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/import_v10x/conf/task.yaml b/tests/import_v10x/conf/task.yaml index fddd0483cb..0f8f6ac42b 100644 --- a/tests/import_v10x/conf/task.yaml +++ b/tests/import_v10x/conf/task.yaml @@ -7,7 +7,6 @@ meta-schema: dm_meta enable-heartbeat: false heartbeat-update-interval: 1 heartbeat-report-interval: 1 -timezone: Asia/Shanghai case-sensitive: false target-database: host: 127.0.0.1 diff --git a/tests/incremental_mode/conf/dm-task.yaml b/tests/incremental_mode/conf/dm-task.yaml index d21aeb8000..a9464ea524 100644 --- a/tests/incremental_mode/conf/dm-task.yaml +++ b/tests/incremental_mode/conf/dm-task.yaml @@ -6,7 +6,6 @@ meta-schema: "dm_meta" # enable-heartbeat: true heartbeat-update-interval: 1 heartbeat-report-interval: 1 -timezone: "Asia/Shanghai" clean-dump-file: false target-database: diff --git a/tests/initial_unit/conf/dm-task.yaml b/tests/initial_unit/conf/dm-task.yaml index f54d4e918a..5166d1f672 100644 --- a/tests/initial_unit/conf/dm-task.yaml +++ b/tests/initial_unit/conf/dm-task.yaml @@ -4,7 +4,6 @@ task-mode: all is-sharding: false meta-schema: "dm_meta" # enable-heartbeat: true -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/load_interrupt/conf/dm-task.yaml b/tests/load_interrupt/conf/dm-task.yaml index 0c91d590ea..d09b9d0434 100644 --- a/tests/load_interrupt/conf/dm-task.yaml +++ b/tests/load_interrupt/conf/dm-task.yaml @@ -4,7 +4,6 @@ task-mode: full is-sharding: false meta-schema: "dm_meta" # enable-heartbeat: true -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/many_tables/conf/dm-task.yaml b/tests/many_tables/conf/dm-task.yaml index 7e6fb62aa3..6637f03585 100644 --- a/tests/many_tables/conf/dm-task.yaml +++ b/tests/many_tables/conf/dm-task.yaml @@ -4,7 +4,6 @@ task-mode: all is-sharding: false meta-schema: "dm_meta" # enable-heartbeat: true -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/new_relay/conf/dm-task.yaml b/tests/new_relay/conf/dm-task.yaml index 3abc7a87f8..b083e8f2d2 100644 --- a/tests/new_relay/conf/dm-task.yaml +++ b/tests/new_relay/conf/dm-task.yaml @@ -6,7 +6,6 @@ meta-schema: "dm_meta" # enable-heartbeat: true heartbeat-update-interval: 1 heartbeat-report-interval: 1 -timezone: "Asia/Shanghai" clean-dump-file: false target-database: diff --git a/tests/online_ddl/conf/dm-task.yaml b/tests/online_ddl/conf/dm-task.yaml index 57decfc1cb..05be92be76 100644 --- a/tests/online_ddl/conf/dm-task.yaml +++ b/tests/online_ddl/conf/dm-task.yaml @@ -4,7 +4,6 @@ task-mode: all is-sharding: true meta-schema: "dm_meta" enable-heartbeat: false -timezone: "Asia/Shanghai" online-ddl-scheme: online-ddl-scheme-placeholder target-database: diff --git a/tests/only_dml/conf/dm-task.yaml b/tests/only_dml/conf/dm-task.yaml index 0c82cb7cdc..56eb08585f 100644 --- a/tests/only_dml/conf/dm-task.yaml +++ b/tests/only_dml/conf/dm-task.yaml @@ -6,7 +6,6 @@ meta-schema: "dm_meta" # enable-heartbeat: true heartbeat-update-interval: 1 heartbeat-report-interval: 1 -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/print_status/conf/dm-task.yaml b/tests/print_status/conf/dm-task.yaml index d7c9e4077f..d5a483479f 100644 --- a/tests/print_status/conf/dm-task.yaml +++ b/tests/print_status/conf/dm-task.yaml @@ -4,7 +4,6 @@ task-mode: all is-sharding: false meta-schema: "dm_meta" enable-heartbeat: false -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/relay_interrupt/conf/dm-task.yaml b/tests/relay_interrupt/conf/dm-task.yaml index 25286d2653..b71ab8dace 100644 --- a/tests/relay_interrupt/conf/dm-task.yaml +++ b/tests/relay_interrupt/conf/dm-task.yaml @@ -4,7 +4,6 @@ task-mode: all is-sharding: false meta-schema: "dm_meta" # enable-heartbeat: true -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/retry_cancel/conf/dm-task.yaml b/tests/retry_cancel/conf/dm-task.yaml index 1f8f3cc5c9..92d93803cf 100644 --- a/tests/retry_cancel/conf/dm-task.yaml +++ b/tests/retry_cancel/conf/dm-task.yaml @@ -2,7 +2,6 @@ name: test task-mode: all # enable-heartbeat: true -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/safe_mode/conf/dm-task.yaml b/tests/safe_mode/conf/dm-task.yaml index 84d3de1161..a77684ff97 100644 --- a/tests/safe_mode/conf/dm-task.yaml +++ b/tests/safe_mode/conf/dm-task.yaml @@ -4,7 +4,6 @@ task-mode: all is-sharding: true meta-schema: "dm_meta" enable-heartbeat: false -timezone: "Asia/Shanghai" ignore-checking-items: ["auto_increment_ID"] target-database: diff --git a/tests/sequence_safe_mode/conf/dm-task.yaml b/tests/sequence_safe_mode/conf/dm-task.yaml index 1ee51c3d72..7074d98535 100644 --- a/tests/sequence_safe_mode/conf/dm-task.yaml +++ b/tests/sequence_safe_mode/conf/dm-task.yaml @@ -4,7 +4,6 @@ task-mode: all is-sharding: true meta-schema: "dm_meta" enable-heartbeat: false -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/sequence_sharding/conf/dm-task.yaml b/tests/sequence_sharding/conf/dm-task.yaml index 3203afcf6e..4dd936aac5 100644 --- a/tests/sequence_sharding/conf/dm-task.yaml +++ b/tests/sequence_sharding/conf/dm-task.yaml @@ -4,7 +4,6 @@ task-mode: all is-sharding: true meta-schema: "dm_meta" # enable-heartbeat: true -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/sequence_sharding_optimistic/conf/dm-task.yaml b/tests/sequence_sharding_optimistic/conf/dm-task.yaml index 44f4a9b815..3a8afdfe5d 100644 --- a/tests/sequence_sharding_optimistic/conf/dm-task.yaml +++ b/tests/sequence_sharding_optimistic/conf/dm-task.yaml @@ -5,7 +5,6 @@ is-sharding: true shard-mode: "optimistic" meta-schema: "dm_meta" # enable-heartbeat: true -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/sequence_sharding_removemeta/conf/dm-task.yaml b/tests/sequence_sharding_removemeta/conf/dm-task.yaml index b9d0f13617..1f5395c2c2 100644 --- a/tests/sequence_sharding_removemeta/conf/dm-task.yaml +++ b/tests/sequence_sharding_removemeta/conf/dm-task.yaml @@ -4,7 +4,6 @@ task-mode: all is-sharding: true meta-schema: "dm_meta" # enable-heartbeat: true -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/shardddl1/conf/double-source-optimistic.yaml b/tests/shardddl1/conf/double-source-optimistic.yaml index ffee908e20..4ea5f0aa25 100644 --- a/tests/shardddl1/conf/double-source-optimistic.yaml +++ b/tests/shardddl1/conf/double-source-optimistic.yaml @@ -4,7 +4,6 @@ task-mode: all is-sharding: true shard-mode: "optimistic" meta-schema: "dm_meta" -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/shardddl1/conf/double-source-pessimistic.yaml b/tests/shardddl1/conf/double-source-pessimistic.yaml index fa62c11612..07f8c0c121 100644 --- a/tests/shardddl1/conf/double-source-pessimistic.yaml +++ b/tests/shardddl1/conf/double-source-pessimistic.yaml @@ -4,7 +4,6 @@ task-mode: all is-sharding: true shard-mode: "pessimistic" meta-schema: "dm_meta" -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/shardddl1/conf/single-source-no-routes.yaml b/tests/shardddl1/conf/single-source-no-routes.yaml index 9fbb7c596f..ea36ff2058 100644 --- a/tests/shardddl1/conf/single-source-no-routes.yaml +++ b/tests/shardddl1/conf/single-source-no-routes.yaml @@ -3,7 +3,6 @@ name: test task-mode: all is-sharding: false meta-schema: "dm_meta" -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/shardddl1/conf/single-source-no-sharding.yaml b/tests/shardddl1/conf/single-source-no-sharding.yaml index b65cdb73fb..90987e1399 100644 --- a/tests/shardddl1/conf/single-source-no-sharding.yaml +++ b/tests/shardddl1/conf/single-source-no-sharding.yaml @@ -3,7 +3,6 @@ name: test task-mode: all is-sharding: false meta-schema: "dm_meta" -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/shardddl1/conf/single-source-optimistic.yaml b/tests/shardddl1/conf/single-source-optimistic.yaml index 2ddd12e9ca..bf8501bbf0 100644 --- a/tests/shardddl1/conf/single-source-optimistic.yaml +++ b/tests/shardddl1/conf/single-source-optimistic.yaml @@ -4,7 +4,6 @@ task-mode: all is-sharding: true shard-mode: "optimistic" meta-schema: "dm_meta" -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/shardddl1/conf/single-source-pessimistic.yaml b/tests/shardddl1/conf/single-source-pessimistic.yaml index 0e966f76d5..1884130992 100644 --- a/tests/shardddl1/conf/single-source-pessimistic.yaml +++ b/tests/shardddl1/conf/single-source-pessimistic.yaml @@ -4,7 +4,6 @@ task-mode: all is-sharding: true shard-mode: "pessimistic" meta-schema: "dm_meta" -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/shardddl2/conf/double-source-optimistic.yaml b/tests/shardddl2/conf/double-source-optimistic.yaml index ffee908e20..4ea5f0aa25 100644 --- a/tests/shardddl2/conf/double-source-optimistic.yaml +++ b/tests/shardddl2/conf/double-source-optimistic.yaml @@ -4,7 +4,6 @@ task-mode: all is-sharding: true shard-mode: "optimistic" meta-schema: "dm_meta" -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/shardddl2/conf/double-source-pessimistic.yaml b/tests/shardddl2/conf/double-source-pessimistic.yaml index fa62c11612..07f8c0c121 100644 --- a/tests/shardddl2/conf/double-source-pessimistic.yaml +++ b/tests/shardddl2/conf/double-source-pessimistic.yaml @@ -4,7 +4,6 @@ task-mode: all is-sharding: true shard-mode: "pessimistic" meta-schema: "dm_meta" -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/shardddl3/conf/double-source-optimistic.yaml b/tests/shardddl3/conf/double-source-optimistic.yaml index 871b1fc378..c87c6448c3 100644 --- a/tests/shardddl3/conf/double-source-optimistic.yaml +++ b/tests/shardddl3/conf/double-source-optimistic.yaml @@ -4,7 +4,6 @@ task-mode: all is-sharding: true shard-mode: "optimistic" meta-schema: "dm_meta" -timezone: "Asia/Shanghai" ignore-checking-items: ["auto_increment_ID"] target-database: diff --git a/tests/shardddl3/conf/double-source-pessimistic.yaml b/tests/shardddl3/conf/double-source-pessimistic.yaml index 9821f1d649..db256a06ed 100644 --- a/tests/shardddl3/conf/double-source-pessimistic.yaml +++ b/tests/shardddl3/conf/double-source-pessimistic.yaml @@ -4,7 +4,6 @@ task-mode: all is-sharding: true shard-mode: "pessimistic" meta-schema: "dm_meta" -timezone: "Asia/Shanghai" ignore-checking-items: ["auto_increment_ID"] target-database: diff --git a/tests/shardddl3/conf/single-source-optimistic-2.yaml b/tests/shardddl3/conf/single-source-optimistic-2.yaml index 831ed9bdad..2439cce00a 100644 --- a/tests/shardddl3/conf/single-source-optimistic-2.yaml +++ b/tests/shardddl3/conf/single-source-optimistic-2.yaml @@ -4,7 +4,6 @@ task-mode: all is-sharding: true shard-mode: "optimistic" meta-schema: "dm_meta" -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/shardddl3/conf/single-source-optimistic.yaml b/tests/shardddl3/conf/single-source-optimistic.yaml index 2ddd12e9ca..bf8501bbf0 100644 --- a/tests/shardddl3/conf/single-source-optimistic.yaml +++ b/tests/shardddl3/conf/single-source-optimistic.yaml @@ -4,7 +4,6 @@ task-mode: all is-sharding: true shard-mode: "optimistic" meta-schema: "dm_meta" -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/shardddl3/conf/single-source-pessimistic-2.yaml b/tests/shardddl3/conf/single-source-pessimistic-2.yaml index 7801725515..c468c59ca1 100644 --- a/tests/shardddl3/conf/single-source-pessimistic-2.yaml +++ b/tests/shardddl3/conf/single-source-pessimistic-2.yaml @@ -4,7 +4,6 @@ task-mode: all is-sharding: true shard-mode: "pessimistic" meta-schema: "dm_meta" -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/shardddl3/conf/single-source-pessimistic.yaml b/tests/shardddl3/conf/single-source-pessimistic.yaml index 0e966f76d5..1884130992 100644 --- a/tests/shardddl3/conf/single-source-pessimistic.yaml +++ b/tests/shardddl3/conf/single-source-pessimistic.yaml @@ -4,7 +4,6 @@ task-mode: all is-sharding: true shard-mode: "pessimistic" meta-schema: "dm_meta" -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/shardddl4/conf/double-source-optimistic.yaml b/tests/shardddl4/conf/double-source-optimistic.yaml index 871b1fc378..c87c6448c3 100644 --- a/tests/shardddl4/conf/double-source-optimistic.yaml +++ b/tests/shardddl4/conf/double-source-optimistic.yaml @@ -4,7 +4,6 @@ task-mode: all is-sharding: true shard-mode: "optimistic" meta-schema: "dm_meta" -timezone: "Asia/Shanghai" ignore-checking-items: ["auto_increment_ID"] target-database: diff --git a/tests/shardddl4/conf/double-source-pessimistic.yaml b/tests/shardddl4/conf/double-source-pessimistic.yaml index 9821f1d649..db256a06ed 100644 --- a/tests/shardddl4/conf/double-source-pessimistic.yaml +++ b/tests/shardddl4/conf/double-source-pessimistic.yaml @@ -4,7 +4,6 @@ task-mode: all is-sharding: true shard-mode: "pessimistic" meta-schema: "dm_meta" -timezone: "Asia/Shanghai" ignore-checking-items: ["auto_increment_ID"] target-database: diff --git a/tests/shardddl4/conf/single-source-optimistic-2.yaml b/tests/shardddl4/conf/single-source-optimistic-2.yaml index 831ed9bdad..2439cce00a 100644 --- a/tests/shardddl4/conf/single-source-optimistic-2.yaml +++ b/tests/shardddl4/conf/single-source-optimistic-2.yaml @@ -4,7 +4,6 @@ task-mode: all is-sharding: true shard-mode: "optimistic" meta-schema: "dm_meta" -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/shardddl4/conf/single-source-optimistic.yaml b/tests/shardddl4/conf/single-source-optimistic.yaml index 2ddd12e9ca..bf8501bbf0 100644 --- a/tests/shardddl4/conf/single-source-optimistic.yaml +++ b/tests/shardddl4/conf/single-source-optimistic.yaml @@ -4,7 +4,6 @@ task-mode: all is-sharding: true shard-mode: "optimistic" meta-schema: "dm_meta" -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/shardddl4/conf/single-source-pessimistic-2.yaml b/tests/shardddl4/conf/single-source-pessimistic-2.yaml index 7801725515..c468c59ca1 100644 --- a/tests/shardddl4/conf/single-source-pessimistic-2.yaml +++ b/tests/shardddl4/conf/single-source-pessimistic-2.yaml @@ -4,7 +4,6 @@ task-mode: all is-sharding: true shard-mode: "pessimistic" meta-schema: "dm_meta" -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/shardddl4/conf/single-source-pessimistic.yaml b/tests/shardddl4/conf/single-source-pessimistic.yaml index 0e966f76d5..1884130992 100644 --- a/tests/shardddl4/conf/single-source-pessimistic.yaml +++ b/tests/shardddl4/conf/single-source-pessimistic.yaml @@ -4,7 +4,6 @@ task-mode: all is-sharding: true shard-mode: "pessimistic" meta-schema: "dm_meta" -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/sharding/conf/dm-task.yaml b/tests/sharding/conf/dm-task.yaml index 7e72a97cf3..fa225c0785 100644 --- a/tests/sharding/conf/dm-task.yaml +++ b/tests/sharding/conf/dm-task.yaml @@ -4,7 +4,6 @@ task-mode: all is-sharding: true meta-schema: "dm_meta" # enable-heartbeat: true -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/sharding2/conf/dm-task.yaml b/tests/sharding2/conf/dm-task.yaml index 60b8b87b68..11a406fb9c 100644 --- a/tests/sharding2/conf/dm-task.yaml +++ b/tests/sharding2/conf/dm-task.yaml @@ -4,7 +4,6 @@ task-mode: all is-sharding: true meta-schema: "dm_meta" # enable-heartbeat: true -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/start_task/conf/dm-task.yaml b/tests/start_task/conf/dm-task.yaml index be785ce8e9..1114b35704 100644 --- a/tests/start_task/conf/dm-task.yaml +++ b/tests/start_task/conf/dm-task.yaml @@ -4,7 +4,6 @@ task-mode: all is-sharding: false meta-schema: "dm_meta" # enable-heartbeat: true -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" diff --git a/tests/tls/conf/dm-task.yaml b/tests/tls/conf/dm-task.yaml index 89ffbf6b1c..990666658d 100644 --- a/tests/tls/conf/dm-task.yaml +++ b/tests/tls/conf/dm-task.yaml @@ -4,7 +4,6 @@ task-mode: all is-sharding: false meta-schema: "dm_meta" # enable-heartbeat: true -timezone: "Asia/Shanghai" target-database: host: "127.0.0.1" From f2218cbfa4375a465d8184ac2e92dd94ae4f1d26 Mon Sep 17 00:00:00 2001 From: glorv Date: Tue, 11 May 2021 19:27:35 +0800 Subject: [PATCH 08/15] fix test --- dm/config/task_test.go | 4 ++-- dumpling/dumpling.go | 2 +- pkg/utils/common_test.go | 1 - syncer/db_test.go | 5 +---- tests/all_mode/data/db1.increment.sql | 2 +- tests/dmctl_basic/conf/get_task.yaml | 1 + tests/full_mode/run.sh | 2 +- tests/import_v10x/conf/task.yaml | 1 + tests/safe_mode/run.sh | 2 +- 9 files changed, 9 insertions(+), 11 deletions(-) diff --git a/dm/config/task_test.go b/dm/config/task_test.go index 462c9ceb8e..305b84484a 100644 --- a/dm/config/task_test.go +++ b/dm/config/task_test.go @@ -289,7 +289,7 @@ mysql-instances: err := taskConfig.Decode(errorTaskConfig1) // field server-id is not a member of TaskConfig c.Check(err, NotNil) - c.Assert(err, ErrorMatches, "*line 18: field server-id not found in type config.MySQLInstance.*") + c.Assert(err, ErrorMatches, "*line 17: field server-id not found in type config.MySQLInstance.*") err = taskConfig.Decode(errorTaskConfig2) // field name duplicate @@ -640,7 +640,7 @@ func (t *testConfig) TestGenAndFromSubTaskConfigs(c *C) { BinLogPos: 456, BinLogGTID: "1-1-12,4-4-4", }, - From: source1DBCfg, + From: source1DBCfg, To: DBConfig{ Host: "127.0.0.1", Port: 4000, diff --git a/dumpling/dumpling.go b/dumpling/dumpling.go index 9c9de1ae3d..975945f1eb 100644 --- a/dumpling/dumpling.go +++ b/dumpling/dumpling.go @@ -61,7 +61,7 @@ func (m *Dumpling) Init(ctx context.Context) error { m.dumpConfig, err = m.constructArgs() m.detectSQLMode(ctx) m.dumpConfig.SessionParams["time_zone"] = "+00:00" - m.logger.Info("create dumpling", zap.Stringer("config", m.dumpConfig)) + m.logger.Info("create dumpling", zap.Stringer("config", m.dumpConfig)) return err } diff --git a/pkg/utils/common_test.go b/pkg/utils/common_test.go index 9e48766760..c199c74df0 100644 --- a/pkg/utils/common_test.go +++ b/pkg/utils/common_test.go @@ -267,7 +267,6 @@ func (s *testCommonSuite) TestParseTimeZone(c *C) { "+12:59": 12*time.Hour + 59*time.Minute, "Asia/Shanghai": 8 * time.Hour, "UTC": time.Duration(0), - "CST": -1 * 6 * time.Hour, } for k, v := range cases { dur, err := ParseTimeZone(k) diff --git a/syncer/db_test.go b/syncer/db_test.go index 9aad570b05..93af69963a 100644 --- a/syncer/db_test.go +++ b/syncer/db_test.go @@ -255,9 +255,6 @@ func (s *testDBSuite) TestTimezone(c *C) { err = txn.Commit() c.Assert(err, IsNil) - location, err := utils.ParseTimeZone(testCase.timezone) - c.Assert(err, IsNil) - idx := 0 for { if idx >= len(testCase.sqls) { @@ -280,7 +277,7 @@ func (s *testDBSuite) TestTimezone(c *C) { c.Assert(ts.Valid, IsTrue) raw := ev.Rows[0][1].(string) - data, err := time.ParseInLocation("2006-01-02 15:04:05", raw, location) + data, err := time.ParseInLocation("2006-01-02 15:04:05", raw, time.UTC) c.Assert(err, IsNil) c.Assert(data.Unix(), DeepEquals, ts.Int64) idx++ diff --git a/tests/all_mode/data/db1.increment.sql b/tests/all_mode/data/db1.increment.sql index f602f85878..4509ea3a3a 100644 --- a/tests/all_mode/data/db1.increment.sql +++ b/tests/all_mode/data/db1.increment.sql @@ -45,4 +45,4 @@ insert into t1 (id, name, info, lat, big1, big2) values (12, 'gentest', '{"id":1 -- test with different session time_zone SET @@session.time_zone = '+07:00'; -insert into t1 (id, name, info) values (13, 'tztest', '{"id": 122}'); +insert into t1 (id, name, info) values (13, 'tztest', '{"id": 132}'); diff --git a/tests/dmctl_basic/conf/get_task.yaml b/tests/dmctl_basic/conf/get_task.yaml index d03f85fb6d..eff8d4d84d 100644 --- a/tests/dmctl_basic/conf/get_task.yaml +++ b/tests/dmctl_basic/conf/get_task.yaml @@ -16,6 +16,7 @@ target-database: max-allowed-packet: null session: tidb_txn_mode: optimistic + time_zone: "+00:00" security: null mysql-instances: - source-id: mysql-replica-01 diff --git a/tests/full_mode/run.sh b/tests/full_mode/run.sh index 273f42af93..e0c5b4fa7d 100755 --- a/tests/full_mode/run.sh +++ b/tests/full_mode/run.sh @@ -42,7 +42,7 @@ function fail_acquire_global_lock() { "\"result\": true" 1 cp $cur/conf/dm-task.yaml $WORK_DIR/dm-task.yaml - sed -i '/timezone/i\ignore-checking-items: ["dump_privilege"]' $WORK_DIR/dm-task.yaml + sed -i '/heartbeat-report-interval/i\ignore-checking-items: ["dump_privilege"]' $WORK_DIR/dm-task.yaml run_dm_ctl $WORK_DIR "127.0.0.1:$MASTER_PORT" \ "start-task $WORK_DIR/dm-task.yaml --remove-meta" diff --git a/tests/import_v10x/conf/task.yaml b/tests/import_v10x/conf/task.yaml index 0f8f6ac42b..41ece6babd 100644 --- a/tests/import_v10x/conf/task.yaml +++ b/tests/import_v10x/conf/task.yaml @@ -18,6 +18,7 @@ target-database: tidb_disable_txn_auto_retry: "off" tidb_retry_limit: "10" tidb_skip_utf8_check: "1" + time_zone: "+00:00" security: null mysql-instances: - source-id: mysql-replica-01 diff --git a/tests/safe_mode/run.sh b/tests/safe_mode/run.sh index 780b061682..48750e5b05 100755 --- a/tests/safe_mode/run.sh +++ b/tests/safe_mode/run.sh @@ -27,7 +27,7 @@ function consistency_none() { dmctl_operate_source create $WORK_DIR/source2.yaml $SOURCE_ID2 cp $cur/conf/dm-task.yaml $WORK_DIR/dm-task.yaml - sed -i "/timezone/i\clean-dump-file: false" $WORK_DIR/dm-task.yaml + sed -i "/enable-heartbeat/i\clean-dump-file: false" $WORK_DIR/dm-task.yaml sed -i "s/extra-args: \"\"/extra-args: \"--consistency none\"/g" $WORK_DIR/dm-task.yaml dmctl_start_task "$WORK_DIR/dm-task.yaml" "--remove-meta" check_sync_diff $WORK_DIR $cur/conf/diff_config.toml From 46f79c7a1ac1a8da2dc7c808f0b059882d77af35 Mon Sep 17 00:00:00 2001 From: glorv Date: Tue, 11 May 2021 20:08:01 +0800 Subject: [PATCH 09/15] fix unit test --- chaos/cases/source.go | 12 +++++----- dm/config/source_config.go | 29 +++++++++++++++-------- dm/config/source_config_test.go | 33 +++++++++++++-------------- dm/master/bootstrap_test.go | 11 ++++----- dm/master/scheduler/scheduler.go | 10 ++++---- dm/master/scheduler/scheduler_test.go | 32 +++++++++++++------------- dm/master/server.go | 9 ++++---- dm/master/server_test.go | 4 ++-- dm/worker/relay_test.go | 2 +- dm/worker/server_test.go | 7 +++--- pkg/ha/bound_test.go | 4 ++-- pkg/ha/ops_test.go | 4 ++-- pkg/ha/relay_test.go | 4 ++-- pkg/ha/source.go | 12 +++++----- pkg/ha/source_test.go | 5 ++-- pkg/terror/error_list.go | 2 ++ 16 files changed, 95 insertions(+), 85 deletions(-) diff --git a/chaos/cases/source.go b/chaos/cases/source.go index 545be2bb01..46e2f11780 100644 --- a/chaos/cases/source.go +++ b/chaos/cases/source.go @@ -45,16 +45,16 @@ func createSources(ctx context.Context, cli pb.MasterClient, cfg *config) error return err } - cfg1 := config2.NewSourceConfig() - cfg2 := config2.NewSourceConfig() - cfg3 := config2.NewSourceConfig() - if err = cfg1.ParseYaml(string(s1Content)); err != nil { + cfg1, err := config2.ParseYaml(string(s1Content)) + if err != nil { return err } - if err = cfg2.ParseYaml(string(s2Content)); err != nil { + cfg2, err := config2.ParseYaml(string(s2Content)) + if err != nil { return err } - if err = cfg3.ParseYaml(string(s3Content)); err != nil { + cfg3, err := config2.ParseYaml(string(s3Content)) + if err != nil { return err } diff --git a/dm/config/source_config.go b/dm/config/source_config.go index f603fba8bb..f6e7027b25 100644 --- a/dm/config/source_config.go +++ b/dm/config/source_config.go @@ -82,6 +82,13 @@ type SourceConfig struct { // NewSourceConfig creates a new base config for upstream MySQL/MariaDB source. func NewSourceConfig() *SourceConfig { + c := newSourceConfig() + c.adjust() + return c +} + +// NewSourceConfig creates a new base config without adjust +func newSourceConfig() *SourceConfig { c := &SourceConfig{ Purge: PurgeConfig{ Interval: 60 * 60, @@ -94,7 +101,6 @@ func NewSourceConfig() *SourceConfig { BackoffMax: Duration{DefaultBackoffMax}, }, } - c.adjust() return c } @@ -140,12 +146,13 @@ func (c *SourceConfig) Parse(content string) error { } // ParseYaml parses flag definitions from the argument list, content should be yaml format. -func (c *SourceConfig) ParseYaml(content string) error { - if err := yaml.Unmarshal([]byte(content), c); err != nil { - return terror.ErrConfigYamlTransform.Delegate(err, "decode source config") +func ParseYaml(content string) (*SourceConfig, error) { + c := newSourceConfig() + if err := yaml.UnmarshalStrict([]byte(content), c); err != nil { + return nil, terror.ErrConfigYamlTransform.Delegate(err, "decode source config") } c.adjust() - return nil + return c, nil } // EncodeToml encodes config. @@ -311,16 +318,20 @@ func (c *SourceConfig) AdjustServerID(ctx context.Context, db *sql.DB) error { } // LoadFromFile loads config from file. -func (c *SourceConfig) LoadFromFile(path string) error { +func LoadFromFile(path string) (*SourceConfig, error) { + c := newSourceConfig() content, err := ioutil.ReadFile(path) if err != nil { - return terror.ErrConfigReadCfgFromFile.Delegate(err, path) + return nil, terror.ErrConfigReadCfgFromFile.Delegate(err, path) } if err := yaml.UnmarshalStrict(content, c); err != nil { - return terror.ErrConfigYamlTransform.Delegate(err, "decode source config") + return nil, terror.ErrConfigYamlTransform.Delegate(err, "decode source config") } c.adjust() - return c.Verify() + if err = c.Verify(); err != nil { + return nil, err + } + return c, nil } func (c *SourceConfig) check(metaData *toml.MetaData, err error) error { diff --git a/dm/config/source_config_test.go b/dm/config/source_config_test.go index d33580133d..a59523a931 100644 --- a/dm/config/source_config_test.go +++ b/dm/config/source_config_test.go @@ -32,9 +32,8 @@ import ( const sourceSampleFile = "../worker/source.yaml" func (t *testConfig) TestConfig(c *C) { - cfg := NewSourceConfig() - - c.Assert(cfg.LoadFromFile(sourceSampleFile), IsNil) + cfg, err := LoadFromFile(sourceSampleFile) + c.Assert(err, IsNil) cfg.RelayDir = "./xx" c.Assert(cfg.RelayDir, Equals, "./xx") c.Assert(cfg.ServerID, Equals, uint32(101)) @@ -63,8 +62,8 @@ func (t *testConfig) TestConfig(c *C) { // test update config file and reload c.Assert(cfg.Parse(tomlStr), IsNil) c.Assert(cfg.ServerID, Equals, uint32(100)) - var cfg1 SourceConfig - c.Assert(cfg1.ParseYaml(yamlStr), IsNil) + cfg1, err := ParseYaml(yamlStr) + c.Assert(err, IsNil) c.Assert(cfg1.ServerID, Equals, uint32(100)) cfg.Filters = []*bf.BinlogEventRule{} cfg.Tracer = map[string]interface{}{} @@ -73,8 +72,8 @@ func (t *testConfig) TestConfig(c *C) { c.Assert(cfg2.Parse(originCfgStr), IsNil) c.Assert(cfg2.ServerID, Equals, uint32(101)) - var cfg3 SourceConfig - c.Assert(cfg3.ParseYaml(originCfgYamlStr), IsNil) + cfg3, err := ParseYaml(originCfgYamlStr) + c.Assert(err, IsNil) c.Assert(cfg3.ServerID, Equals, uint32(101)) // test decrypt password @@ -111,8 +110,8 @@ func (t *testConfig) TestConfig(c *C) { c.Assert(clone4yaml, Matches, "(.|\n)*backoff-rollback: 5m(.|\n)*") c.Assert(clone4yaml, Matches, "(.|\n)*backoff-max: 5m(.|\n)*") - var clone6 SourceConfig - c.Assert(clone6.ParseYaml(clone4yaml), IsNil) + clone6, err := ParseYaml(clone4yaml) + c.Assert(err, IsNil) c.Assert(clone6, DeepEquals, *clone4) // test invalid config @@ -124,15 +123,15 @@ aaa: xxx `) err = ioutil.WriteFile(configFile, configContent, 0o644) c.Assert(err, IsNil) - err = cfg.LoadFromFile(configFile) + _, err = LoadFromFile(configFile) c.Assert(err, NotNil) c.Assert(err, ErrorMatches, "(.|\n)*field aaa not found in type config.SourceConfig(.|\n)*") } func (t *testConfig) TestConfigVerify(c *C) { newConfig := func() *SourceConfig { - cfg := NewSourceConfig() - c.Assert(cfg.LoadFromFile(sourceSampleFile), IsNil) + cfg, err := LoadFromFile(sourceSampleFile) + c.Assert(err, IsNil) cfg.RelayDir = "./xx" return cfg } @@ -255,12 +254,12 @@ func subtestFlavor(c *C, cfg *SourceConfig, sqlInfo, expectedFlavor, expectedErr } func (t *testConfig) TestAdjustFlavor(c *C) { - cfg := NewSourceConfig() - c.Assert(cfg.LoadFromFile(sourceSampleFile), IsNil) + cfg, err := LoadFromFile(sourceSampleFile) + c.Assert(err, IsNil) cfg.RelayDir = "./xx" cfg.Flavor = "mariadb" - err := cfg.AdjustFlavor(context.Background(), nil) + err = cfg.AdjustFlavor(context.Background(), nil) c.Assert(err, IsNil) c.Assert(cfg.Flavor, Equals, mysql.MariaDBFlavor) cfg.Flavor = "MongoDB" @@ -278,8 +277,8 @@ func (t *testConfig) TestAdjustServerID(c *C) { }() getAllServerIDFunc = getMockServerIDs - cfg := NewSourceConfig() - c.Assert(cfg.LoadFromFile(sourceSampleFile), IsNil) + cfg, err := LoadFromFile(sourceSampleFile) + c.Assert(err, IsNil) cfg.RelayDir = "./xx" c.Assert(cfg.AdjustServerID(context.Background(), nil), IsNil) diff --git a/dm/master/bootstrap_test.go b/dm/master/bootstrap_test.go index 66bf4b0b03..258d9fe3f7 100644 --- a/dm/master/bootstrap_test.go +++ b/dm/master/bootstrap_test.go @@ -68,13 +68,12 @@ func (t *testMaster) TestCollectSourceConfigFilesV1Import(c *C) { } password := os.Getenv("MYSQL_PSWD") - // load a valid source file. - cfg1 := config.NewSourceConfig() + cfg1, err := config.LoadFromFile("./source.yaml") + c.Assert(err, IsNil) // fix empty map after marshal/unmarshal becomes nil cfg1.From.Session = map[string]string{} cfg1.Tracer = map[string]interface{}{} cfg1.Filters = []*filter.BinlogEventRule{} - c.Assert(cfg1.LoadFromFile("./source.yaml"), IsNil) cfg1.From.Host = host cfg1.From.Port = port cfg1.From.User = user @@ -122,8 +121,8 @@ func (t *testMaster) TestWaitWorkersReadyV1Import(c *C) { s.cfg.V1SourcesPath = c.MkDir() c.Assert(s.scheduler.Start(ctx, etcdTestCli), IsNil) - cfg1 := config.NewSourceConfig() - c.Assert(cfg1.LoadFromFile("./source.yaml"), IsNil) + cfg1, err := config.LoadFromFile("./source.yaml") + c.Assert(err, IsNil) cfg2 := cfg1.Clone() cfg2.SourceID = "mysql-replica-02" cfgs := map[string]config.SourceConfig{ @@ -132,7 +131,7 @@ func (t *testMaster) TestWaitWorkersReadyV1Import(c *C) { } // no worker registered, timeout. - err := s.waitWorkersReadyV1Import(tctx, cfgs) + err = s.waitWorkersReadyV1Import(tctx, cfgs) c.Assert(err, ErrorMatches, ".*wait for DM-worker instances timeout.*") // register one worker. diff --git a/dm/master/scheduler/scheduler.go b/dm/master/scheduler/scheduler.go index eadedf69c5..0caf2acaad 100644 --- a/dm/master/scheduler/scheduler.go +++ b/dm/master/scheduler/scheduler.go @@ -76,7 +76,7 @@ type Scheduler struct { // - recover from etcd (calling `recoverSources`). // delete: // - remove source by user request (calling `RemoveSourceCfg`). - sourceCfgs map[string]config.SourceConfig + sourceCfgs map[string]*config.SourceConfig // all subtask configs, task name -> source ID -> subtask config. // add: @@ -151,7 +151,7 @@ type Scheduler struct { func NewScheduler(pLogger *log.Logger, securityCfg config.Security) *Scheduler { return &Scheduler{ logger: pLogger.WithFields(zap.String("component", "scheduler")), - sourceCfgs: make(map[string]config.SourceConfig), + sourceCfgs: make(map[string]*config.SourceConfig), subTaskCfgs: make(map[string]map[string]config.SubTaskConfig), workers: make(map[string]*Worker), bounds: make(map[string]*Worker), @@ -255,7 +255,7 @@ func (s *Scheduler) CloseAllWorkers() { // AddSourceCfg adds the upstream source config to the cluster. // NOTE: please verify the config before call this. -func (s *Scheduler) AddSourceCfg(cfg config.SourceConfig) error { +func (s *Scheduler) AddSourceCfg(cfg *config.SourceConfig) error { s.mu.Lock() defer s.mu.Unlock() @@ -380,7 +380,7 @@ func (s *Scheduler) GetSourceCfgByID(source string) *config.SourceConfig { if !ok { return nil } - clone := cfg + clone := *cfg return &clone } @@ -1745,7 +1745,7 @@ func (s *Scheduler) updateStatusForUnbound(source string) { // reset resets the internal status. func (s *Scheduler) reset() { - s.sourceCfgs = make(map[string]config.SourceConfig) + s.sourceCfgs = make(map[string]*config.SourceConfig) s.subTaskCfgs = make(map[string]map[string]config.SubTaskConfig) s.workers = make(map[string]*Worker) s.bounds = make(map[string]*Worker) diff --git a/dm/master/scheduler/scheduler_test.go b/dm/master/scheduler/scheduler_test.go index 75581797d4..0db2e17ddd 100644 --- a/dm/master/scheduler/scheduler_test.go +++ b/dm/master/scheduler/scheduler_test.go @@ -100,7 +100,6 @@ func (t *testScheduler) testSchedulerProgress(c *C, restart int) { taskName2 = "task-2" workerInfo1 = ha.NewWorkerInfo(workerName1, workerAddr1) workerInfo2 = ha.NewWorkerInfo(workerName2, workerAddr2) - sourceCfg1 config.SourceConfig subtaskCfg1 config.SubTaskConfig keepAliveTTL = int64(5) // NOTE: this should be >= minLeaseTTL, in second. @@ -116,7 +115,8 @@ func (t *testScheduler) testSchedulerProgress(c *C, restart int) { } } ) - c.Assert(sourceCfg1.LoadFromFile(sourceSampleFile), IsNil) + sourceCfg1, err := config.LoadFromFile(sourceSampleFile) + c.Assert(err, IsNil) sourceCfg1.SourceID = sourceID1 sourceCfg1.EnableRelay = true sourceCfg2 := sourceCfg1 @@ -550,7 +550,7 @@ func (t *testScheduler) sourceCfgNotExist(c *C, s *Scheduler, source string) { c.Assert(scm, HasLen, 0) } -func (t *testScheduler) sourceCfgExist(c *C, s *Scheduler, expectCfg config.SourceConfig) { +func (t *testScheduler) sourceCfgExist(c *C, s *Scheduler, expectCfg *config.SourceConfig) { cfgP := s.GetSourceCfgByID(expectCfg.SourceID) c.Assert(cfgP, DeepEquals, &expectCfg) scm, _, err := ha.GetSourceCfg(etcdTestCli, expectCfg.SourceID, 0) @@ -708,11 +708,11 @@ func (t *testScheduler) TestRestartScheduler(c *C) { workerAddr1 = "127.0.0.1:8262" workerInfo1 = ha.NewWorkerInfo(workerName1, workerAddr1) sourceBound1 = ha.NewSourceBound(sourceID1, workerName1) - sourceCfg1 config.SourceConfig wg sync.WaitGroup keepAliveTTL = int64(2) // NOTE: this should be >= minLeaseTTL, in second. ) - c.Assert(sourceCfg1.LoadFromFile(sourceSampleFile), IsNil) + sourceCfg1, err := config.LoadFromFile(sourceSampleFile) + c.Assert(err, IsNil) sourceCfg1.SourceID = sourceID1 s := NewScheduler(&logger, config.Security{}) @@ -822,10 +822,10 @@ func (t *testScheduler) TestWatchWorkerEventEtcdCompact(c *C) { workerAddr2 = "127.0.0.1:18262" workerAddr3 = "127.0.0.1:18362" workerAddr4 = "127.0.0.1:18462" - sourceCfg1 config.SourceConfig keepAliveTTL = int64(2) // NOTE: this should be >= minLeaseTTL, in second. ) - c.Assert(sourceCfg1.LoadFromFile(sourceSampleFile), IsNil) + sourceCfg1, err := config.LoadFromFile(sourceSampleFile) + c.Assert(err, IsNil) sourceCfg1.SourceID = sourceID1 sourceCfg2 := sourceCfg1 sourceCfg2.SourceID = sourceID2 @@ -884,7 +884,7 @@ func (t *testScheduler) TestWatchWorkerEventEtcdCompact(c *C) { // step 4: trigger etcd compaction and check whether we can receive it through watcher var startRev int64 = 1 - _, err := etcdTestCli.Compact(ctx, rev) + _, err = etcdTestCli.Compact(ctx, rev) c.Assert(err, IsNil) workerEvCh := make(chan ha.WorkerEvent, 10) workerErrCh := make(chan error, 10) @@ -962,10 +962,10 @@ func (t *testScheduler) TestLastBound(c *C) { workerName2 = "dm-worker-2" workerName3 = "dm-worker-3" workerName4 = "dm-worker-4" - sourceCfg1 config.SourceConfig ) - c.Assert(sourceCfg1.LoadFromFile(sourceSampleFile), IsNil) + sourceCfg1, err := config.LoadFromFile(sourceSampleFile) + c.Assert(err, IsNil) sourceCfg1.SourceID = sourceID1 sourceCfg2 := sourceCfg1 sourceCfg2.SourceID = sourceID2 @@ -1052,8 +1052,8 @@ func (t *testScheduler) TestTransferSource(c *C) { s.workers[workerName2] = worker2 s.workers[workerName3] = worker3 s.workers[workerName4] = worker4 - s.sourceCfgs[sourceID1] = config.SourceConfig{} - s.sourceCfgs[sourceID2] = config.SourceConfig{} + s.sourceCfgs[sourceID1] = &config.SourceConfig{} + s.sourceCfgs[sourceID2] = &config.SourceConfig{} worker1.ToFree() c.Assert(s.boundSourceToWorker(sourceID1, worker1), IsNil) @@ -1075,7 +1075,7 @@ func (t *testScheduler) TestTransferSource(c *C) { c.Assert(worker1.Stage(), Equals, WorkerFree) // test valid transfer: source -> worker = unbound -> free - s.sourceCfgs[sourceID3] = config.SourceConfig{} + s.sourceCfgs[sourceID3] = &config.SourceConfig{} s.unbounds[sourceID3] = struct{}{} c.Assert(s.TransferSource(sourceID3, workerName3), IsNil) c.Assert(s.bounds[sourceID3], DeepEquals, worker3) @@ -1095,7 +1095,7 @@ func (t *testScheduler) TestTransferSource(c *C) { c.Assert(s.bounds[sourceID1], DeepEquals, worker4) // test invalid transfer: source -> worker = unbound -> bound - s.sourceCfgs[sourceID4] = config.SourceConfig{} + s.sourceCfgs[sourceID4] = &config.SourceConfig{} s.unbounds[sourceID4] = struct{}{} c.Assert(s.TransferSource(sourceID4, workerName3), NotNil) c.Assert(s.bounds[sourceID3], DeepEquals, worker3) @@ -1147,8 +1147,8 @@ func (t *testScheduler) TestStartStopSource(c *C) { s.workers[workerName2] = worker2 s.workers[workerName3] = worker3 s.workers[workerName4] = worker4 - s.sourceCfgs[sourceID1] = config.SourceConfig{} - s.sourceCfgs[sourceID2] = config.SourceConfig{} + s.sourceCfgs[sourceID1] = &config.SourceConfig{} + s.sourceCfgs[sourceID2] = &config.SourceConfig{} worker1.ToFree() c.Assert(s.boundSourceToWorker(sourceID1, worker1), IsNil) diff --git a/dm/master/server.go b/dm/master/server.go index d4c5e7f743..08c4d673a3 100644 --- a/dm/master/server.go +++ b/dm/master/server.go @@ -1079,10 +1079,11 @@ func (s *Server) CheckTask(ctx context.Context, req *pb.CheckTaskRequest) (*pb.C func parseAndAdjustSourceConfig(ctx context.Context, contents []string) ([]*config.SourceConfig, error) { cfgs := make([]*config.SourceConfig, len(contents)) for i, content := range contents { - cfg := config.NewSourceConfig() - if err := cfg.ParseYaml(content); err != nil { + cfg, err := config.ParseYaml(content) + if err != nil { return cfgs, err } + cfg.From.Adjust() dbConfig := cfg.GenerateDBConfig() @@ -1112,8 +1113,8 @@ func parseAndAdjustSourceConfig(ctx context.Context, contents []string) ([]*conf func parseSourceConfig(contents []string) ([]*config.SourceConfig, error) { cfgs := make([]*config.SourceConfig, len(contents)) for i, content := range contents { - cfg := config.NewSourceConfig() - if err := cfg.ParseYaml(content); err != nil { + cfg, err := config.ParseYaml(content) + if err != nil { return cfgs, err } cfgs[i] = cfg diff --git a/dm/master/server_test.go b/dm/master/server_test.go index c81c0c3e0f..85021c28ee 100644 --- a/dm/master/server_test.go +++ b/dm/master/server_test.go @@ -1468,8 +1468,8 @@ func (t *testMaster) TestOperateSource(c *check.C) { s1.leader.Store(oneselfLeader) c.Assert(s1.Start(ctx), check.IsNil) defer s1.Close() - mysqlCfg := config.NewSourceConfig() - c.Assert(mysqlCfg.LoadFromFile("./source.yaml"), check.IsNil) + mysqlCfg, err := config.LoadFromFile("./source.yaml") + c.Assert(err, check.IsNil) mysqlCfg.From.Password = os.Getenv("MYSQL_PSWD") task, err := mysqlCfg.Yaml() c.Assert(err, check.IsNil) diff --git a/dm/worker/relay_test.go b/dm/worker/relay_test.go index 4b31463567..4bbec4a6b9 100644 --- a/dm/worker/relay_test.go +++ b/dm/worker/relay_test.go @@ -145,7 +145,7 @@ func (t *testRelay) TestRelay(c *C) { cfg.RelayDir = dir cfg.MetaDir = dir - relayHolder := NewRealRelayHolder(&cfg) + relayHolder := NewRealRelayHolder(cfg) c.Assert(relayHolder, NotNil) holder, ok := relayHolder.(*realRelayHolder) diff --git a/dm/worker/server_test.go b/dm/worker/server_test.go index 1d078a3b88..2b44f379f2 100644 --- a/dm/worker/server_test.go +++ b/dm/worker/server_test.go @@ -386,7 +386,7 @@ func (t *testServer) TestWatchSourceBoundEtcdCompact(c *C) { rev, err := ha.DeleteSourceBound(etcdCli, cfg.Name) c.Assert(err, IsNil) // step 2: start source at this worker - w, err := s.getOrStartWorker(&sourceCfg, true) + w, err := s.getOrStartWorker(sourceCfg, true) c.Assert(err, IsNil) c.Assert(w.EnableHandleSubtasks(), IsNil) // step 3: trigger etcd compaction and check whether we can receive it through watcher @@ -633,9 +633,8 @@ func checkRelayStatus(cli pb.WorkerClient, expect pb.Stage) bool { return status.SourceStatus.RelayStatus.Stage == expect } -func loadSourceConfigWithoutPassword(c *C) config.SourceConfig { - var sourceCfg config.SourceConfig - err := sourceCfg.LoadFromFile(sourceSampleFile) +func loadSourceConfigWithoutPassword(c *C) *config.SourceConfig { + sourceCfg, err := config.LoadFromFile(sourceSampleFile) c.Assert(err, IsNil) sourceCfg.From.Password = "" // no password set return sourceCfg diff --git a/pkg/ha/bound_test.go b/pkg/ha/bound_test.go index fa95f6c0b7..9dd8f714d2 100644 --- a/pkg/ha/bound_test.go +++ b/pkg/ha/bound_test.go @@ -128,10 +128,10 @@ func (t *testForEtcd) TestGetSourceBoundConfigEtcd(c *C) { worker = "dm-worker-1" source = "mysql-replica-1" bound = NewSourceBound(source, worker) - cfg config.SourceConfig emptyCfg config.SourceConfig ) - c.Assert(cfg.LoadFromFile(sourceSampleFile), IsNil) + cfg, err := config.LoadFromFile(sourceSampleFile) + c.Assert(err, IsNil) cfg.SourceID = source // no source bound and config bound1, cfg1, rev1, err := GetSourceBoundConfig(etcdTestCli, worker) diff --git a/pkg/ha/ops_test.go b/pkg/ha/ops_test.go index 3ad51caaff..80f129ae54 100644 --- a/pkg/ha/ops_test.go +++ b/pkg/ha/ops_test.go @@ -34,11 +34,11 @@ func (t *testForEtcd) TestOpsEtcd(c *C) { bound = NewSourceBound(source, worker) emptyStage Stage - sourceCfg config.SourceConfig subtaskCfg1 config.SubTaskConfig ) - c.Assert(sourceCfg.LoadFromFile(sourceSampleFile), IsNil) + sourceCfg, err := config.LoadFromFile(sourceSampleFile) + c.Assert(err, IsNil) sourceCfg.SourceID = source c.Assert(subtaskCfg1.DecodeFile(subTaskSampleFile, true), IsNil) subtaskCfg1.SourceID = source diff --git a/pkg/ha/relay_test.go b/pkg/ha/relay_test.go index 08d62aca46..05d7f9bb45 100644 --- a/pkg/ha/relay_test.go +++ b/pkg/ha/relay_test.go @@ -25,9 +25,9 @@ func (t *testForEtcd) TestGetRelayConfigEtcd(c *C) { var ( worker = "dm-worker-1" source = "mysql-replica-1" - cfg config.SourceConfig ) - c.Assert(cfg.LoadFromFile(sourceSampleFile), IsNil) + cfg, err := config.LoadFromFile(sourceSampleFile) + c.Assert(err, IsNil) cfg.SourceID = source // no relay source and config cfg1, rev1, err := GetRelayConfig(etcdTestCli, worker) diff --git a/pkg/ha/source.go b/pkg/ha/source.go index de0d7c661f..bb9cc49684 100644 --- a/pkg/ha/source.go +++ b/pkg/ha/source.go @@ -27,7 +27,7 @@ import ( // PutSourceCfg puts the config of the upstream source into etcd. // k/v: sourceID -> source config. -func PutSourceCfg(cli *clientv3.Client, cfg config.SourceConfig) (int64, error) { +func PutSourceCfg(cli *clientv3.Client, cfg *config.SourceConfig) (int64, error) { value, err := cfg.Toml() if err != nil { return 0, err @@ -42,12 +42,12 @@ func PutSourceCfg(cli *clientv3.Client, cfg config.SourceConfig) (int64, error) // if the source config for the sourceID not exist, return with `err == nil`. // if the source name is "", it will return all source configs as a map{sourceID: config}. // if the source name is given, it will return a map{sourceID: config} whose length is 1. -func GetSourceCfg(cli *clientv3.Client, source string, rev int64) (map[string]config.SourceConfig, int64, error) { +func GetSourceCfg(cli *clientv3.Client, source string, rev int64) (map[string]*config.SourceConfig, int64, error) { ctx, cancel := context.WithTimeout(cli.Ctx(), etcdutil.DefaultRequestTimeout) defer cancel() var ( - scm = make(map[string]config.SourceConfig) + scm = make(map[string]*config.SourceConfig) resp *clientv3.GetResponse err error ) @@ -77,8 +77,8 @@ func deleteSourceCfgOp(source string) clientv3.Op { return clientv3.OpDelete(common.UpstreamConfigKeyAdapter.Encode(source)) } -func sourceCfgFromResp(source string, resp *clientv3.GetResponse) (map[string]config.SourceConfig, error) { - scm := make(map[string]config.SourceConfig) +func sourceCfgFromResp(source string, resp *clientv3.GetResponse) (map[string]*config.SourceConfig, error) { + scm := make(map[string]*config.SourceConfig) if resp.Count == 0 { return scm, nil } else if source != "" && resp.Count > 1 { @@ -92,7 +92,7 @@ func sourceCfgFromResp(source string, resp *clientv3.GetResponse) (map[string]co if err != nil { return scm, terror.ErrConfigEtcdParse.Delegate(err, kv.Key) } - scm[cfg.SourceID] = cfg + scm[cfg.SourceID] = &cfg } return scm, nil } diff --git a/pkg/ha/source_test.go b/pkg/ha/source_test.go index af7568c1e3..44d4da4704 100644 --- a/pkg/ha/source_test.go +++ b/pkg/ha/source_test.go @@ -52,9 +52,8 @@ var _ = Suite(&testForEtcd{}) func (t *testForEtcd) TestSourceEtcd(c *C) { defer clearTestInfoOperation(c) - var cfg config.SourceConfig - - c.Assert(cfg.LoadFromFile(sourceSampleFile), IsNil) + cfg, err := config.LoadFromFile(sourceSampleFile) + c.Assert(err, IsNil) source := cfg.SourceID cfgExtra := cfg cfgExtra.SourceID = "mysql-replica-2" diff --git a/pkg/terror/error_list.go b/pkg/terror/error_list.go index 34fe022348..f214422a29 100644 --- a/pkg/terror/error_list.go +++ b/pkg/terror/error_list.go @@ -196,6 +196,7 @@ const ( codeConfigEmptySourceID codeConfigTooLongSourceID codeConfigOnlineSchemeNotSupport + codeConfigInvalidTimezone codeConfigParseFlagSet codeConfigDecryptDBPassword codeConfigMetaInvalid @@ -803,6 +804,7 @@ var ( ErrConfigEmptySourceID = New(codeConfigEmptySourceID, ClassConfig, ScopeInternal, LevelMedium, "empty source-id not valid", "Please check the `source-id` config in configuration file.") ErrConfigTooLongSourceID = New(codeConfigTooLongSourceID, ClassConfig, ScopeInternal, LevelMedium, "too long source-id not valid", "Please check the `source-id` config in configuration file. The max source id length is 32.") ErrConfigOnlineSchemeNotSupport = New(codeConfigOnlineSchemeNotSupport, ClassConfig, ScopeInternal, LevelMedium, "online scheme %s not supported", "Please check the `online-ddl-scheme` config in task configuration file. Only `ghost` and `pt` are currently supported.") + ErrConfigInvalidTimezone = New(codeConfigInvalidTimezone, ClassConfig, ScopeInternal, LevelMedium, "invalid timezone string: %s", "Please check the `timezone` config in task configuration file.") ErrConfigParseFlagSet = New(codeConfigParseFlagSet, ClassConfig, ScopeInternal, LevelMedium, "parse subtask config flag set", "") ErrConfigDecryptDBPassword = New(codeConfigDecryptDBPassword, ClassConfig, ScopeInternal, LevelMedium, "decrypt DB password %s failed", "") ErrConfigMetaInvalid = New(codeConfigMetaInvalid, ClassConfig, ScopeInternal, LevelMedium, "must specify `binlog-name` without GTID enabled for the source or specify `binlog-gtid` with GTID enabled for the source", "Please check the `meta` config in task configuration file.") From b65304bd08d0bf5f48d24e6a963d43fba4d42a9c Mon Sep 17 00:00:00 2001 From: glorv Date: Tue, 11 May 2021 20:29:45 +0800 Subject: [PATCH 10/15] fix --- cmd/dm-syncer/config.go | 3 +- dm/config/source_config.go | 2 +- dm/master/bootstrap.go | 10 +++--- dm/master/bootstrap_test.go | 10 +++--- dm/master/scheduler/scheduler_test.go | 8 ++--- dm/master/server.go | 2 +- dm/master/server_test.go | 4 +-- dm/worker/server.go | 8 ++--- dm/worker/task_checker_test.go | 4 +-- dm/worker/worker_test.go | 16 ++++----- pkg/ha/bound.go | 4 +-- pkg/ha/bound_test.go | 9 +++-- pkg/ha/relay.go | 2 +- pkg/terror/error_list.go | 2 +- pkg/utils/common.go | 48 ++------------------------- pkg/utils/common_test.go | 29 ---------------- 16 files changed, 45 insertions(+), 116 deletions(-) diff --git a/cmd/dm-syncer/config.go b/cmd/dm-syncer/config.go index b50143ae1b..6ecd71012d 100644 --- a/cmd/dm-syncer/config.go +++ b/cmd/dm-syncer/config.go @@ -16,13 +16,14 @@ package main import ( "flag" "fmt" + "os" + "github.com/BurntSushi/toml" "github.com/go-mysql-org/go-mysql/mysql" "github.com/pingcap/errors" bf "github.com/pingcap/tidb-tools/pkg/binlog-filter" "github.com/pingcap/tidb-tools/pkg/filter" router "github.com/pingcap/tidb-tools/pkg/table-router" - "os" "github.com/pingcap/dm/dm/config" "github.com/pingcap/dm/pkg/utils" diff --git a/dm/config/source_config.go b/dm/config/source_config.go index f6e7027b25..397dd025d0 100644 --- a/dm/config/source_config.go +++ b/dm/config/source_config.go @@ -324,7 +324,7 @@ func LoadFromFile(path string) (*SourceConfig, error) { if err != nil { return nil, terror.ErrConfigReadCfgFromFile.Delegate(err, path) } - if err := yaml.UnmarshalStrict(content, c); err != nil { + if err = yaml.UnmarshalStrict(content, c); err != nil { return nil, terror.ErrConfigYamlTransform.Delegate(err, "decode source config") } c.adjust() diff --git a/dm/master/bootstrap.go b/dm/master/bootstrap.go index 1697c5cf48..13d5db0b13 100644 --- a/dm/master/bootstrap.go +++ b/dm/master/bootstrap.go @@ -162,13 +162,13 @@ func (s *Server) importFromV10x(ctx context.Context) error { } // collectSourceConfigFilesV1Import tries to collect source config files for v1.0.x importing. -func (s *Server) collectSourceConfigFilesV1Import(tctx *tcontext.Context) (map[string]config.SourceConfig, error) { +func (s *Server) collectSourceConfigFilesV1Import(tctx *tcontext.Context) (map[string]*config.SourceConfig, error) { files, err := ioutil.ReadDir(s.cfg.V1SourcesPath) if err != nil { return nil, err } - cfgs := make(map[string]config.SourceConfig) + cfgs := make(map[string]*config.SourceConfig) for _, f := range files { if f.IsDir() { continue // ignore sub directories. @@ -186,7 +186,7 @@ func (s *Server) collectSourceConfigFilesV1Import(tctx *tcontext.Context) (map[s return nil, err } - cfgs[cfgs2[0].SourceID] = *cfgs2[0] + cfgs[cfgs2[0].SourceID] = cfgs2[0] tctx.Logger.Info("collected source config", zap.Stringer("config", cfgs2[0])) } @@ -195,7 +195,7 @@ func (s *Server) collectSourceConfigFilesV1Import(tctx *tcontext.Context) (map[s // waitWorkersReadyV1Import waits for DM-worker instances ready for v1.0.x importing. // NOTE: in v1.0.x, `count of DM-worker instances` equals `count of source config files`. -func (s *Server) waitWorkersReadyV1Import(tctx *tcontext.Context, sourceCfgs map[string]config.SourceConfig) error { +func (s *Server) waitWorkersReadyV1Import(tctx *tcontext.Context, sourceCfgs map[string]*config.SourceConfig) error { // now, we simply check count repeatedly. count := len(sourceCfgs) ctx2, cancel2 := context.WithTimeout(context.Background(), waitWorkerV1Timeout) @@ -291,7 +291,7 @@ func (s *Server) getSubtaskCfgsStagesV1Import(tctx *tcontext.Context) ( } // addSourcesV1Import tries to add source config into the cluster for v1.0.x importing. -func (s *Server) addSourcesV1Import(tctx *tcontext.Context, cfgs map[string]config.SourceConfig) error { +func (s *Server) addSourcesV1Import(tctx *tcontext.Context, cfgs map[string]*config.SourceConfig) error { var ( added []string err error diff --git a/dm/master/bootstrap_test.go b/dm/master/bootstrap_test.go index 258d9fe3f7..bec21ff9a3 100644 --- a/dm/master/bootstrap_test.go +++ b/dm/master/bootstrap_test.go @@ -94,8 +94,8 @@ func (t *testMaster) TestCollectSourceConfigFilesV1Import(c *C) { cfgs, err = s.collectSourceConfigFilesV1Import(tctx) c.Assert(err, IsNil) c.Assert(cfgs, HasLen, 2) - c.Assert(cfgs[cfg1.SourceID], DeepEquals, *cfg1) - c.Assert(cfgs[cfg2.SourceID], DeepEquals, *cfg2) + c.Assert(cfgs[cfg1.SourceID], DeepEquals, cfg1) + c.Assert(cfgs[cfg2.SourceID], DeepEquals, cfg2) // put a invalid source file. c.Assert(ioutil.WriteFile(filepath.Join(s.cfg.V1SourcesPath, "invalid.yaml"), []byte("invalid-source-data"), 0o644), IsNil) @@ -125,9 +125,9 @@ func (t *testMaster) TestWaitWorkersReadyV1Import(c *C) { c.Assert(err, IsNil) cfg2 := cfg1.Clone() cfg2.SourceID = "mysql-replica-02" - cfgs := map[string]config.SourceConfig{ - cfg1.SourceID: *cfg1, - cfg2.SourceID: *cfg2, + cfgs := map[string]*config.SourceConfig{ + cfg1.SourceID: cfg1, + cfg2.SourceID: cfg2, } // no worker registered, timeout. diff --git a/dm/master/scheduler/scheduler_test.go b/dm/master/scheduler/scheduler_test.go index 0db2e17ddd..56c4d13a2a 100644 --- a/dm/master/scheduler/scheduler_test.go +++ b/dm/master/scheduler/scheduler_test.go @@ -868,8 +868,8 @@ func (t *testScheduler) TestWatchWorkerEventEtcdCompact(c *C) { c.Assert(ha.KeepAlive(ctx1, etcdTestCli, workerName2, keepAliveTTL), IsNil) }() c.Assert(utils.WaitSomething(30, 100*time.Millisecond, func() bool { - kam, _, err := ha.GetKeepAliveWorkers(etcdTestCli) - return err == nil && len(kam) == 2 + kam, _, e := ha.GetKeepAliveWorkers(etcdTestCli) + return e == nil && len(kam) == 2 }), IsTrue) cancel1() wg.Wait() @@ -877,9 +877,9 @@ func (t *testScheduler) TestWatchWorkerEventEtcdCompact(c *C) { time.Sleep(time.Duration(keepAliveTTL) * time.Second) var rev int64 c.Assert(utils.WaitSomething(30, 100*time.Millisecond, func() bool { - kam, rev1, err := ha.GetKeepAliveWorkers(etcdTestCli) + kam, rev1, e := ha.GetKeepAliveWorkers(etcdTestCli) rev = rev1 - return err == nil && len(kam) == 0 + return e == nil && len(kam) == 0 }), IsTrue) // step 4: trigger etcd compaction and check whether we can receive it through watcher diff --git a/dm/master/server.go b/dm/master/server.go index 08c4d673a3..548ced8eeb 100644 --- a/dm/master/server.go +++ b/dm/master/server.go @@ -1192,7 +1192,7 @@ func (s *Server) OperateSource(ctx context.Context, req *pb.OperateSourceRequest err error ) for _, cfg := range cfgs { - err = s.scheduler.AddSourceCfg(*cfg) + err = s.scheduler.AddSourceCfg(cfg) // return first error and try to revert, so user could copy-paste same start command after error if err != nil { resp.Msg = err.Error() diff --git a/dm/master/server_test.go b/dm/master/server_test.go index 85021c28ee..c3d47c8008 100644 --- a/dm/master/server_test.go +++ b/dm/master/server_test.go @@ -271,7 +271,7 @@ func testMockScheduler(ctx context.Context, wg *sync.WaitGroup, c *check.C, sour cfg := config.NewSourceConfig() cfg.SourceID = sources[i] cfg.From.Password = password - c.Assert(scheduler2.AddSourceCfg(*cfg), check.IsNil, check.Commentf("all sources: %v", sources)) + c.Assert(scheduler2.AddSourceCfg(cfg), check.IsNil, check.Commentf("all sources: %v", sources)) wg.Add(1) ctx1, cancel1 := context.WithCancel(ctx) cancels = append(cancels, cancel1) @@ -303,7 +303,7 @@ func testMockSchedulerForRelay(ctx context.Context, wg *sync.WaitGroup, c *check cfg := config.NewSourceConfig() cfg.SourceID = sources[i] cfg.From.Password = password - c.Assert(scheduler2.AddSourceCfg(*cfg), check.IsNil, check.Commentf("all sources: %v", sources)) + c.Assert(scheduler2.AddSourceCfg(cfg), check.IsNil, check.Commentf("all sources: %v", sources)) wg.Add(1) ctx1, cancel1 := context.WithCancel(ctx) cancels = append(cancels, cancel1) diff --git a/dm/worker/server.go b/dm/worker/server.go index 3ceb65cbbc..3efd1fe58b 100644 --- a/dm/worker/server.go +++ b/dm/worker/server.go @@ -156,7 +156,7 @@ func (s *Server) Start() error { } if !bound.IsEmpty() { log.L().Warn("worker has been assigned source before keepalive", zap.Stringer("bound", bound), zap.Bool("is deleted", bound.IsDeleted)) - if err2 := s.enableHandleSubtasks(&sourceCfg, true); err2 != nil { + if err2 := s.enableHandleSubtasks(sourceCfg, true); err2 != nil { return err2 } log.L().Info("started to handle mysql source", zap.String("sourceCfg", sourceCfg.String())) @@ -452,7 +452,7 @@ func (s *Server) observeSourceBound(ctx context.Context, rev int64) error { } log.L().Info("will recover observeSourceBound", zap.String("relay source", cfg.SourceID)) - return s.enableHandleSubtasks(&cfg, false) + return s.enableHandleSubtasks(cfg, false) }() if err2 != nil { return err2 @@ -653,7 +653,7 @@ func (s *Server) operateSourceBound(bound ha.SourceBound) error { if !ok { return terror.ErrWorkerFailToGetSourceConfigFromEtcd.Generate(bound.Source) } - return s.enableHandleSubtasks(&sourceCfg, true) + return s.enableHandleSubtasks(sourceCfg, true) } func (s *Server) enableHandleSubtasks(sourceCfg *config.SourceConfig, needLock bool) error { @@ -704,7 +704,7 @@ func (s *Server) operateRelaySource(relaySource ha.RelaySource) error { if !ok { return terror.ErrWorkerFailToGetSourceConfigFromEtcd.Generate(relaySource.Source) } - return s.enableRelay(&sourceCfg, true) + return s.enableRelay(sourceCfg, true) } func (s *Server) enableRelay(sourceCfg *config.SourceConfig, needLock bool) error { diff --git a/dm/worker/task_checker_test.go b/dm/worker/task_checker_test.go index 466536e4e7..ac3db40e38 100644 --- a/dm/worker/task_checker_test.go +++ b/dm/worker/task_checker_test.go @@ -89,7 +89,7 @@ func (s *testTaskCheckerSuite) TestCheck(c *check.C) { cfg := loadSourceConfigWithoutPassword(c) cfg.RelayDir = dir cfg.MetaDir = dir - w, err := NewWorker(&cfg, nil, "") + w, err := NewWorker(cfg, nil, "") c.Assert(err, check.IsNil) w.closed.Store(false) @@ -204,7 +204,7 @@ func (s *testTaskCheckerSuite) TestCheckTaskIndependent(c *check.C) { cfg := loadSourceConfigWithoutPassword(c) cfg.RelayDir = dir cfg.MetaDir = dir - w, err := NewWorker(&cfg, nil, "") + w, err := NewWorker(cfg, nil, "") c.Assert(err, check.IsNil) w.closed.Store(false) diff --git a/dm/worker/worker_test.go b/dm/worker/worker_test.go index 878bba480c..0dc7aa9fdb 100644 --- a/dm/worker/worker_test.go +++ b/dm/worker/worker_test.go @@ -71,12 +71,12 @@ func (t *testServer) testWorker(c *C) { defer func() { NewRelayHolder = NewRealRelayHolder }() - w, err := NewWorker(&cfg, etcdCli, "") + w, err := NewWorker(cfg, etcdCli, "") c.Assert(err, IsNil) c.Assert(w.EnableRelay(), ErrorMatches, "init error") NewRelayHolder = NewDummyRelayHolder - w, err = NewWorker(&cfg, etcdCli, "") + w, err = NewWorker(cfg, etcdCli, "") c.Assert(err, IsNil) c.Assert(w.StatusJSON(""), HasLen, emptyWorkerStatusInfoJSONLength) @@ -175,7 +175,7 @@ func (t *testServer2) TestTaskAutoResume(c *C) { if s.closed.Load() { return false } - w, err2 := s.getOrStartWorker(&sourceConfig, true) + w, err2 := s.getOrStartWorker(sourceConfig, true) c.Assert(err2, IsNil) // we set sourceConfig.EnableRelay = true above c.Assert(w.EnableRelay(), IsNil) @@ -279,7 +279,7 @@ func (t *testWorkerFunctionalities) TestWorkerFunctionalities(c *C) { c.Assert(err, IsNil) // start worker - w, err := NewWorker(&sourceCfg, etcdCli, "") + w, err := NewWorker(sourceCfg, etcdCli, "") c.Assert(err, IsNil) defer w.Close() go func() { @@ -349,7 +349,7 @@ func (t *testWorkerFunctionalities) TestWorkerFunctionalities(c *C) { } func (t *testWorkerFunctionalities) testEnableRelay(c *C, w *Worker, etcdCli *clientv3.Client, - sourceCfg config.SourceConfig, cfg *Config) { + sourceCfg *config.SourceConfig, cfg *Config) { c.Assert(w.EnableRelay(), IsNil) c.Assert(w.relayEnabled.Load(), IsTrue) @@ -379,7 +379,7 @@ func (t *testWorkerFunctionalities) testDisableRelay(c *C, w *Worker) { } func (t *testWorkerFunctionalities) testEnableHandleSubtasks(c *C, w *Worker, etcdCli *clientv3.Client, - subtaskCfg config.SubTaskConfig, sourceCfg config.SourceConfig) { + subtaskCfg config.SubTaskConfig, sourceCfg *config.SourceConfig) { c.Assert(w.EnableHandleSubtasks(), IsNil) c.Assert(w.subTaskEnabled.Load(), IsTrue) @@ -469,7 +469,7 @@ func (t *testWorkerEtcdCompact) TestWatchSubtaskStageEtcdCompact(c *C) { sourceCfg.EnableRelay = false // step 1: start worker - w, err := NewWorker(&sourceCfg, etcdCli, "") + w, err := NewWorker(sourceCfg, etcdCli, "") c.Assert(err, IsNil) ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -585,7 +585,7 @@ func (t *testWorkerEtcdCompact) TestWatchRelayStageEtcdCompact(c *C) { sourceCfg.MetaDir = c.MkDir() // step 1: start worker - w, err := NewWorker(&sourceCfg, etcdCli, "") + w, err := NewWorker(sourceCfg, etcdCli, "") c.Assert(err, IsNil) ctx, cancel := context.WithCancel(context.Background()) defer cancel() diff --git a/pkg/ha/bound.go b/pkg/ha/bound.go index 1d7eb67516..1aff76700b 100644 --- a/pkg/ha/bound.go +++ b/pkg/ha/bound.go @@ -185,11 +185,11 @@ func GetLastSourceBounds(cli *clientv3.Client) (map[string]SourceBound, int64, e // if source bound is empty, will return an empty sourceBound and an empty source config // if source bound is not empty but sourceConfig is empty, will return an error // if the source bound is different for over retryNum times, will return an error. -func GetSourceBoundConfig(cli *clientv3.Client, worker string) (SourceBound, config.SourceConfig, int64, error) { +func GetSourceBoundConfig(cli *clientv3.Client, worker string) (SourceBound, *config.SourceConfig, int64, error) { var ( bound SourceBound newBound SourceBound - cfg config.SourceConfig + cfg *config.SourceConfig ok bool retryNum = defaultGetSourceBoundConfigRetry ) diff --git a/pkg/ha/bound_test.go b/pkg/ha/bound_test.go index 9dd8f714d2..5a40b81e4d 100644 --- a/pkg/ha/bound_test.go +++ b/pkg/ha/bound_test.go @@ -125,10 +125,9 @@ func (t *testForEtcd) TestGetSourceBoundConfigEtcd(c *C) { defer clearTestInfoOperation(c) var ( - worker = "dm-worker-1" - source = "mysql-replica-1" - bound = NewSourceBound(source, worker) - emptyCfg config.SourceConfig + worker = "dm-worker-1" + source = "mysql-replica-1" + bound = NewSourceBound(source, worker) ) cfg, err := config.LoadFromFile(sourceSampleFile) c.Assert(err, IsNil) @@ -138,7 +137,7 @@ func (t *testForEtcd) TestGetSourceBoundConfigEtcd(c *C) { c.Assert(err, IsNil) c.Assert(rev1, Greater, int64(0)) c.Assert(bound1.IsEmpty(), IsTrue) - c.Assert(cfg1, DeepEquals, emptyCfg) + c.Assert(cfg1, IsNil) rev2, err := PutSourceBound(etcdTestCli, bound) c.Assert(err, IsNil) diff --git a/pkg/ha/relay.go b/pkg/ha/relay.go index b75268aa3a..9b041573a6 100644 --- a/pkg/ha/relay.go +++ b/pkg/ha/relay.go @@ -173,7 +173,7 @@ func GetRelayConfig(cli *clientv3.Client, worker string) (*config.SourceConfig, return nil, 0, terror.ErrConfigMissingForBound.Generate(source) } - return &cfg, rev2, nil + return cfg, rev2, nil } return nil, 0, terror.ErrWorkerRelayConfigChanging.Generate(worker, source, newSource) } diff --git a/pkg/terror/error_list.go b/pkg/terror/error_list.go index f214422a29..fb83de36a9 100644 --- a/pkg/terror/error_list.go +++ b/pkg/terror/error_list.go @@ -804,7 +804,7 @@ var ( ErrConfigEmptySourceID = New(codeConfigEmptySourceID, ClassConfig, ScopeInternal, LevelMedium, "empty source-id not valid", "Please check the `source-id` config in configuration file.") ErrConfigTooLongSourceID = New(codeConfigTooLongSourceID, ClassConfig, ScopeInternal, LevelMedium, "too long source-id not valid", "Please check the `source-id` config in configuration file. The max source id length is 32.") ErrConfigOnlineSchemeNotSupport = New(codeConfigOnlineSchemeNotSupport, ClassConfig, ScopeInternal, LevelMedium, "online scheme %s not supported", "Please check the `online-ddl-scheme` config in task configuration file. Only `ghost` and `pt` are currently supported.") - ErrConfigInvalidTimezone = New(codeConfigInvalidTimezone, ClassConfig, ScopeInternal, LevelMedium, "invalid timezone string: %s", "Please check the `timezone` config in task configuration file.") + ErrConfigInvalidTimezone = New(codeConfigInvalidTimezone, ClassConfig, ScopeInternal, LevelMedium, "invalid timezone string: %s", "Please check the `timezone` config in task configuration file.") ErrConfigParseFlagSet = New(codeConfigParseFlagSet, ClassConfig, ScopeInternal, LevelMedium, "parse subtask config flag set", "") ErrConfigDecryptDBPassword = New(codeConfigDecryptDBPassword, ClassConfig, ScopeInternal, LevelMedium, "decrypt DB password %s failed", "") ErrConfigMetaInvalid = New(codeConfigMetaInvalid, ClassConfig, ScopeInternal, LevelMedium, "must specify `binlog-name` without GTID enabled for the source or specify `binlog-gtid` with GTID enabled for the source", "Please check the `meta` config in task configuration file.") diff --git a/pkg/utils/common.go b/pkg/utils/common.go index ef59108843..65ef2711f7 100644 --- a/pkg/utils/common.go +++ b/pkg/utils/common.go @@ -19,21 +19,16 @@ import ( "fmt" "regexp" "strings" - "time" - - "github.com/pingcap/errors" - - "github.com/pingcap/dm/pkg/log" - "github.com/pingcap/dm/pkg/terror" "github.com/pingcap/failpoint" tmysql "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb-tools/pkg/dbutil" "github.com/pingcap/tidb-tools/pkg/filter" router "github.com/pingcap/tidb-tools/pkg/table-router" - "github.com/pingcap/tidb/types" - "github.com/pingcap/tidb/util/timeutil" "go.uber.org/zap" + + "github.com/pingcap/dm/pkg/log" + "github.com/pingcap/dm/pkg/terror" ) // TrimCtrlChars returns a slice of the string s with all leading @@ -214,40 +209,3 @@ func NonRepeatStringsEqual(a, b []string) bool { } return true } - -// ParseTimeZone parse location info from time offset("+08:00") or location string ("Asia/Shanghai") -// -// Copy from https://github.com/pingcap/tidb/blob/263a47e85ce04f74ec80d1d35b426618bc89b5a3/sessionctx/variable/varsutil.go#L411 -func ParseTimeZone(s string) (*time.Location, error) { - if strings.EqualFold(s, "SYSTEM") { - return timeutil.SystemLocation(), nil - } - loc, err := time.LoadLocation(s) - if err == nil { - return loc, nil - } - // The value can be given as a string indicating an offset from UTC, such as '+10:00' or '-6:00'. - // The time zone's value should in [-12:59,+14:00]. - if strings.HasPrefix(s, "+") || strings.HasPrefix(s, "-") { - d, err := types.ParseDuration(nil, s[1:], 0) - if err == nil { - if s[0] == '-' { - if d.Duration > 12*time.Hour+59*time.Minute { - return nil, errors.Errorf("invalid time zone '%s'", s) - } - } else { - if d.Duration > 14*time.Hour { - return nil, errors.Errorf("invalid time zone '%s'", s) - } - } - - ofst := int(d.Duration / time.Second) - if s[0] == '-' { - ofst = -ofst - } - return time.FixedZone("", ofst), nil - } - } - - return nil, errors.Errorf("invalid time zone '%s'", s) -} diff --git a/pkg/utils/common_test.go b/pkg/utils/common_test.go index c199c74df0..5b2b8de634 100644 --- a/pkg/utils/common_test.go +++ b/pkg/utils/common_test.go @@ -17,7 +17,6 @@ import ( "bytes" "context" "fmt" - "time" "github.com/DATA-DOG/go-sqlmock" . "github.com/pingcap/check" @@ -257,31 +256,3 @@ func (s *testCommonSuite) TestNonRepeatStringsEqual(c *C) { c.Assert(NonRepeatStringsEqual([]string{}, []string{"1"}), IsFalse) c.Assert(NonRepeatStringsEqual([]string{"1", "2"}, []string{"2", "3"}), IsFalse) } - -func (s *testCommonSuite) TestParseTimeZone(c *C) { - cases := map[string]time.Duration{ - "+00:00": time.Duration(0), - "+01:00": time.Hour, - "-08:03": -1 * (8*time.Hour + 3*time.Minute), - "-12:59": -1 * (12*time.Hour + 59*time.Minute), - "+12:59": 12*time.Hour + 59*time.Minute, - "Asia/Shanghai": 8 * time.Hour, - "UTC": time.Duration(0), - } - for k, v := range cases { - dur, err := ParseTimeZone(k) - c.Assert(err, IsNil) - c.Assert(dur, Equals, v) - } - - badCases := []string{ - "test", - "-13:00", - "+14:05", - } - - for _, s := range badCases { - _, err := ParseTimeZone(s) - c.Assert(err, ErrorMatches, "invalid time zone.*") - } -} From 81871427111f78b30577bbd7a64029baeddccac4 Mon Sep 17 00:00:00 2001 From: glorv Date: Wed, 12 May 2021 10:23:51 +0800 Subject: [PATCH 11/15] fix test --- dm/config/source_config_test.go | 2 +- dm/master/bootstrap_test.go | 2 +- dm/master/scheduler/scheduler_test.go | 6 +++--- dm/master/server.go | 1 - dm/worker/server_test.go | 4 ++-- pkg/ha/relay_test.go | 2 +- pkg/ha/source_test.go | 6 +++--- tests/dmctl_basic/conf/get_source1.yaml | 3 ++- tests/dmctl_basic/conf/get_source2.yaml | 3 ++- 9 files changed, 15 insertions(+), 14 deletions(-) diff --git a/dm/config/source_config_test.go b/dm/config/source_config_test.go index a59523a931..3aaddb3653 100644 --- a/dm/config/source_config_test.go +++ b/dm/config/source_config_test.go @@ -112,7 +112,7 @@ func (t *testConfig) TestConfig(c *C) { clone6, err := ParseYaml(clone4yaml) c.Assert(err, IsNil) - c.Assert(clone6, DeepEquals, *clone4) + c.Assert(clone6, DeepEquals, clone4) // test invalid config dir2 := c.MkDir() diff --git a/dm/master/bootstrap_test.go b/dm/master/bootstrap_test.go index bec21ff9a3..bf1797c4b8 100644 --- a/dm/master/bootstrap_test.go +++ b/dm/master/bootstrap_test.go @@ -71,7 +71,7 @@ func (t *testMaster) TestCollectSourceConfigFilesV1Import(c *C) { cfg1, err := config.LoadFromFile("./source.yaml") c.Assert(err, IsNil) // fix empty map after marshal/unmarshal becomes nil - cfg1.From.Session = map[string]string{} + cfg1.From.Adjust() cfg1.Tracer = map[string]interface{}{} cfg1.Filters = []*filter.BinlogEventRule{} cfg1.From.Host = host diff --git a/dm/master/scheduler/scheduler_test.go b/dm/master/scheduler/scheduler_test.go index 56c4d13a2a..d1193796f5 100644 --- a/dm/master/scheduler/scheduler_test.go +++ b/dm/master/scheduler/scheduler_test.go @@ -552,7 +552,7 @@ func (t *testScheduler) sourceCfgNotExist(c *C, s *Scheduler, source string) { func (t *testScheduler) sourceCfgExist(c *C, s *Scheduler, expectCfg *config.SourceConfig) { cfgP := s.GetSourceCfgByID(expectCfg.SourceID) - c.Assert(cfgP, DeepEquals, &expectCfg) + c.Assert(cfgP, DeepEquals, expectCfg) scm, _, err := ha.GetSourceCfg(etcdTestCli, expectCfg.SourceID, 0) c.Assert(err, IsNil) cfgV := scm[expectCfg.SourceID] @@ -827,7 +827,7 @@ func (t *testScheduler) TestWatchWorkerEventEtcdCompact(c *C) { sourceCfg1, err := config.LoadFromFile(sourceSampleFile) c.Assert(err, IsNil) sourceCfg1.SourceID = sourceID1 - sourceCfg2 := sourceCfg1 + sourceCfg2 := *sourceCfg1 sourceCfg2.SourceID = sourceID2 ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -839,7 +839,7 @@ func (t *testScheduler) TestWatchWorkerEventEtcdCompact(c *C) { // step 2: add two sources and register four workers c.Assert(s.AddSourceCfg(sourceCfg1), IsNil) - c.Assert(s.AddSourceCfg(sourceCfg2), IsNil) + c.Assert(s.AddSourceCfg(&sourceCfg2), IsNil) c.Assert(s.unbounds, HasLen, 2) c.Assert(s.unbounds, HasKey, sourceID1) c.Assert(s.unbounds, HasKey, sourceID2) diff --git a/dm/master/server.go b/dm/master/server.go index 548ced8eeb..d0998bb6c2 100644 --- a/dm/master/server.go +++ b/dm/master/server.go @@ -1083,7 +1083,6 @@ func parseAndAdjustSourceConfig(ctx context.Context, contents []string) ([]*conf if err != nil { return cfgs, err } - cfg.From.Adjust() dbConfig := cfg.GenerateDBConfig() diff --git a/dm/worker/server_test.go b/dm/worker/server_test.go index 2b44f379f2..ed9525bd5a 100644 --- a/dm/worker/server_test.go +++ b/dm/worker/server_test.go @@ -420,7 +420,7 @@ func (t *testServer) TestWatchSourceBoundEtcdCompact(c *C) { return s.getWorker(true) != nil }), IsTrue) cfg2 := s.getWorker(true).cfg - c.Assert(*cfg2, DeepEquals, sourceCfg) + c.Assert(cfg2, DeepEquals, sourceCfg) cancel1() wg.Wait() c.Assert(s.stopWorker(sourceCfg.SourceID, true), IsNil) @@ -435,7 +435,7 @@ func (t *testServer) TestWatchSourceBoundEtcdCompact(c *C) { return s.getWorker(true) != nil }), IsTrue) cfg2 = s.getWorker(true).cfg - c.Assert(*cfg2, DeepEquals, sourceCfg) + c.Assert(cfg2, DeepEquals, sourceCfg) cancel2() wg.Wait() } diff --git a/pkg/ha/relay_test.go b/pkg/ha/relay_test.go index 05d7f9bb45..bd357cde2b 100644 --- a/pkg/ha/relay_test.go +++ b/pkg/ha/relay_test.go @@ -50,7 +50,7 @@ func (t *testForEtcd) TestGetRelayConfigEtcd(c *C) { cfg2, rev4, err := GetRelayConfig(etcdTestCli, worker) c.Assert(err, IsNil) c.Assert(rev4, Equals, rev3) - c.Assert(*cfg2, DeepEquals, cfg) + c.Assert(cfg2, DeepEquals, cfg) rev5, err := DeleteRelayConfig(etcdTestCli, worker) c.Assert(err, IsNil) diff --git a/pkg/ha/source_test.go b/pkg/ha/source_test.go index 44d4da4704..42cc9944bf 100644 --- a/pkg/ha/source_test.go +++ b/pkg/ha/source_test.go @@ -55,7 +55,7 @@ func (t *testForEtcd) TestSourceEtcd(c *C) { cfg, err := config.LoadFromFile(sourceSampleFile) c.Assert(err, IsNil) source := cfg.SourceID - cfgExtra := cfg + cfgExtra := *cfg cfgExtra.SourceID = "mysql-replica-2" // no source config exist. @@ -80,7 +80,7 @@ func (t *testForEtcd) TestSourceEtcd(c *C) { c.Assert(cfg2, DeepEquals, cfg) // put another source config. - rev2, err = PutSourceCfg(etcdTestCli, cfgExtra) + rev2, err = PutSourceCfg(etcdTestCli, &cfgExtra) c.Assert(err, IsNil) // get all two config. @@ -89,7 +89,7 @@ func (t *testForEtcd) TestSourceEtcd(c *C) { c.Assert(rev3, Equals, rev2) c.Assert(cfgM, HasLen, 2) c.Assert(cfgM[source], DeepEquals, cfg) - c.Assert(cfgM[cfgExtra.SourceID], DeepEquals, cfgExtra) + c.Assert(cfgM[cfgExtra.SourceID], DeepEquals, &cfgExtra) // delete the config. deleteOp := deleteSourceCfgOp(source) diff --git a/tests/dmctl_basic/conf/get_source1.yaml b/tests/dmctl_basic/conf/get_source1.yaml index 6e921fcf73..c73cc1ebb2 100644 --- a/tests/dmctl_basic/conf/get_source1.yaml +++ b/tests/dmctl_basic/conf/get_source1.yaml @@ -14,7 +14,8 @@ from: user: root password: '******' max-allowed-packet: null - session: {} + session: + time_zone: "+00:00" security: null purge: interval: 3600 diff --git a/tests/dmctl_basic/conf/get_source2.yaml b/tests/dmctl_basic/conf/get_source2.yaml index 2948f2f878..114d04acc6 100644 --- a/tests/dmctl_basic/conf/get_source2.yaml +++ b/tests/dmctl_basic/conf/get_source2.yaml @@ -14,7 +14,8 @@ from: user: root password: '******' max-allowed-packet: null - session: {} + session: + time_zone: "+00:00" security: null purge: interval: 3600 From f174d7ea00fe9c8c3c6734b3c751eb05794516b8 Mon Sep 17 00:00:00 2001 From: glorv Date: Wed, 12 May 2021 11:55:46 +0800 Subject: [PATCH 12/15] add back deprecated timezone config field --- cmd/dm-syncer/config.go | 17 +++++++++++++++++ dm/config/subtask.go | 7 +++++++ dm/config/task.go | 7 ++++++- dm/config/task_test.go | 3 ++- 4 files changed, 32 insertions(+), 2 deletions(-) diff --git a/cmd/dm-syncer/config.go b/cmd/dm-syncer/config.go index 6ecd71012d..f443380902 100644 --- a/cmd/dm-syncer/config.go +++ b/cmd/dm-syncer/config.go @@ -26,6 +26,7 @@ import ( router "github.com/pingcap/tidb-tools/pkg/table-router" "github.com/pingcap/dm/dm/config" + "github.com/pingcap/dm/pkg/log" "github.com/pingcap/dm/pkg/utils" ) @@ -53,6 +54,9 @@ type commonConfig struct { SafeMode bool MaxRetry int + // deprecated + TimezoneStr string + SyncerConfigFormat bool } @@ -79,6 +83,7 @@ func (c *commonConfig) newConfigFromSyncerConfig(args []string) (*config.SubTask fs := cfg.FlagSet var SyncerConfigFormat bool + var timezoneStr string fs.BoolVar(&cfg.printVersion, "V", false, "prints version and exit") fs.StringVar(&cfg.Name, "name", "", "the task name") @@ -97,6 +102,7 @@ func (c *commonConfig) newConfigFromSyncerConfig(args []string) (*config.SubTask fs.BoolVar(&cfg.EnableGTID, "enable-gtid", false, "enable gtid mode") fs.BoolVar(&cfg.SafeMode, "safe-mode", false, "enable safe mode to make syncer reentrant") fs.IntVar(&cfg.MaxRetry, "max-retry", 100, "maxinum retry when network interruption") + fs.StringVar(&timezoneStr, "timezone", "", "target database timezone location string") fs.BoolVar(&SyncerConfigFormat, "syncer-config-format", false, "read syncer config format") if err := fs.Parse(args); err != nil { @@ -114,6 +120,10 @@ func (c *commonConfig) newConfigFromSyncerConfig(args []string) (*config.SubTask return nil, errors.Trace(err) } + if timezoneStr != "" { + log.L().Warn("'--timezone' is deprecated, needn't set it anymore.") + } + return cfg.convertToNewFormat() } @@ -141,6 +151,7 @@ func (c *commonConfig) newSubTaskConfig(args []string) (*config.SubTaskConfig, e var syncerConfigFormat bool var printVersion bool var serverID uint + var timezoneStr string fs.BoolVar(&printVersion, "V", false, "prints version and exit") fs.StringVar(&cfg.Name, "name", "", "the task name") @@ -158,6 +169,7 @@ func (c *commonConfig) newSubTaskConfig(args []string) (*config.SubTaskConfig, e fs.BoolVar(&cfg.EnableGTID, "enable-gtid", false, "enable gtid mode") fs.BoolVar(&cfg.SafeMode, "safe-mode", false, "enable safe mode to make syncer reentrant") fs.IntVar(&cfg.MaxRetry, "max-retry", 100, "maxinum retry when network interruption") + fs.StringVar(&timezoneStr, "timezone", "", "target database timezone location string") fs.StringVar(&cfg.Name, "cp-table-prefix", "dm-syncer", "the prefix of the checkpoint table name") fs.BoolVar(&syncerConfigFormat, "syncer-config-format", false, "read syncer config format") @@ -166,6 +178,9 @@ func (c *commonConfig) newSubTaskConfig(args []string) (*config.SubTaskConfig, e if err := cfg.Parse(args, false); err != nil { return nil, errors.Trace(err) } + if timezoneStr != "" { + log.L().Warn("'--timezone' is deprecated, needn't set it anymore.") + } if serverID != 101 { cfg.ServerID = uint32(serverID) @@ -258,6 +273,8 @@ type syncerConfig struct { // MaxDMLConnectionTimeout string `toml:"execute-dml-timeout" json:"execute-dml-timeout"` // ExecutionQueueLength int `toml:"execute-queue-length" json:"execute-queue-length"` + TimezoneStr string `toml:"timezone" json:"timezone"` + printVersion bool } diff --git a/dm/config/subtask.go b/dm/config/subtask.go index 02b0b5a0d6..f52219fb7d 100644 --- a/dm/config/subtask.go +++ b/dm/config/subtask.go @@ -151,6 +151,8 @@ type SubTaskConfig struct { HeartbeatReportInterval int `toml:"heartbeat-report-interval" json:"heartbeat-report-interval"` EnableHeartbeat bool `toml:"enable-heartbeat" json:"enable-heartbeat"` Meta *Meta `toml:"meta" json:"meta"` + // deprecated + Timezone string `toml:"timezone" json:"timezone"` // RelayDir get value from dm-worker config RelayDir string `toml:"relay-dir" json:"relay-dir"` @@ -272,6 +274,11 @@ func (c *SubTaskConfig) Adjust(verifyDecryptPassword bool) error { c.MetaSchema = defaultMetaSchema } + if c.Timezone != "" { + log.L().Warn("'timezone' is deprecated, please remove this field.") + c.Timezone = "" + } + dirSuffix := "." + c.Name if !strings.HasSuffix(c.LoaderConfig.Dir, dirSuffix) { // check to support multiple times calling // if not ends with the task name, we append the task name to the tail diff --git a/dm/config/task.go b/dm/config/task.go index 576d050ca4..8db83f4e45 100644 --- a/dm/config/task.go +++ b/dm/config/task.go @@ -285,6 +285,8 @@ type TaskConfig struct { EnableHeartbeat bool `yaml:"enable-heartbeat" toml:"enable-heartbeat" json:"enable-heartbeat"` HeartbeatUpdateInterval int `yaml:"heartbeat-update-interval" toml:"heartbeat-update-interval" json:"heartbeat-update-interval"` HeartbeatReportInterval int `yaml:"heartbeat-report-interval" toml:"heartbeat-report-interval" json:"heartbeat-report-interval"` + // deprecated + Timezone string `yaml:"timezone" toml:"timezone" json:"timezone"` // handle schema/table name mode, and only for schema/table name // if case insensitive, we would convert schema/table name to lower case @@ -592,7 +594,10 @@ func (c *TaskConfig) adjust() error { sort.Strings(unusedConfigs) return terror.ErrConfigGlobalConfigsUnused.Generate(unusedConfigs) } - + if c.Timezone != "" { + log.L().Warn("`timezone` is deprecated and useless anymore, please remove it.") + c.Timezone = "" + } if c.RemoveMeta { log.L().Warn("`remove-meta` in task config is deprecated, please use `start-task ... --remove-meta` instead") } diff --git a/dm/config/task_test.go b/dm/config/task_test.go index 305b84484a..8ec6bf7b75 100644 --- a/dm/config/task_test.go +++ b/dm/config/task_test.go @@ -268,6 +268,7 @@ task-mode: all is-sharding: true meta-schema: "dm_meta" enable-heartbeat: true +timezone: "Asia/Shanghai" ignore-checking-items: ["all"] target-database: @@ -289,7 +290,7 @@ mysql-instances: err := taskConfig.Decode(errorTaskConfig1) // field server-id is not a member of TaskConfig c.Check(err, NotNil) - c.Assert(err, ErrorMatches, "*line 17: field server-id not found in type config.MySQLInstance.*") + c.Assert(err, ErrorMatches, "*line 18: field server-id not found in type config.MySQLInstance.*") err = taskConfig.Decode(errorTaskConfig2) // field name duplicate From cc6d2dfc2f949204a536e4d59d692525e4b4b6ab Mon Sep 17 00:00:00 2001 From: glorv Date: Wed, 12 May 2021 13:45:47 +0800 Subject: [PATCH 13/15] fix --- dm/config/task_test.go | 1 + tests/dmctl_basic/conf/get_task.yaml | 1 + tests/import_v10x/conf/task.yaml | 1 + 3 files changed, 3 insertions(+) diff --git a/dm/config/task_test.go b/dm/config/task_test.go index 8ec6bf7b75..ba09f6df46 100644 --- a/dm/config/task_test.go +++ b/dm/config/task_test.go @@ -242,6 +242,7 @@ name: test task-mode: all is-sharding: true meta-schema: "dm_meta" +timezone: "Asia/Shanghai" enable-heartbeat: true ignore-checking-items: ["all"] diff --git a/tests/dmctl_basic/conf/get_task.yaml b/tests/dmctl_basic/conf/get_task.yaml index eff8d4d84d..8ce3b7dae3 100644 --- a/tests/dmctl_basic/conf/get_task.yaml +++ b/tests/dmctl_basic/conf/get_task.yaml @@ -7,6 +7,7 @@ meta-schema: dm_meta enable-heartbeat: false heartbeat-update-interval: 1 heartbeat-report-interval: 10 +timezone: "" case-sensitive: false target-database: host: 127.0.0.1 diff --git a/tests/import_v10x/conf/task.yaml b/tests/import_v10x/conf/task.yaml index 41ece6babd..0524883740 100644 --- a/tests/import_v10x/conf/task.yaml +++ b/tests/import_v10x/conf/task.yaml @@ -7,6 +7,7 @@ meta-schema: dm_meta enable-heartbeat: false heartbeat-update-interval: 1 heartbeat-report-interval: 1 +timezone: "" case-sensitive: false target-database: host: 127.0.0.1 From 2753b415d3836a05c1dbfe30bd19fe472a6408dd Mon Sep 17 00:00:00 2001 From: glorv Date: Thu, 13 May 2021 14:59:09 +0800 Subject: [PATCH 14/15] fix lint --- dm/config/source_config.go | 2 +- relay/meta.go | 1 - syncer/syncer_test.go | 2 -- tests/all_mode/run.sh | 6 +++--- tests/full_mode/run.sh | 4 ++-- 5 files changed, 6 insertions(+), 9 deletions(-) diff --git a/dm/config/source_config.go b/dm/config/source_config.go index 397dd025d0..a4303d5c83 100644 --- a/dm/config/source_config.go +++ b/dm/config/source_config.go @@ -87,7 +87,7 @@ func NewSourceConfig() *SourceConfig { return c } -// NewSourceConfig creates a new base config without adjust +// NewSourceConfig creates a new base config without adjust. func newSourceConfig() *SourceConfig { c := &SourceConfig{ Purge: PurgeConfig{ diff --git a/relay/meta.go b/relay/meta.go index 0187eb5333..c9a1a3844d 100644 --- a/relay/meta.go +++ b/relay/meta.go @@ -303,7 +303,6 @@ func (lm *LocalMeta) AddDir(serverUUID string, newPos *mysql.Position, newGTID g } // update UUID index file - // nolint:gocritic uuids := append(lm.uuids, newUUID) err = lm.updateIndexFile(uuids) if err != nil { diff --git a/syncer/syncer_test.go b/syncer/syncer_test.go index a6276b5232..d2101ffe9a 100644 --- a/syncer/syncer_test.go +++ b/syncer/syncer_test.go @@ -694,7 +694,6 @@ func (s *testSyncerSuite) TestColumnMapping(c *C) { mapping, err := cm.NewMapping(false, rules) c.Assert(err, IsNil) - // nolint:gocritic allEvents := append(createEvents, dmlEvents...) allEvents = append(allEvents, dropEvents...) dmlIndex := 0 @@ -1315,7 +1314,6 @@ func (s *testSyncerSuite) TestExitSafeModeByConfig(c *C) { } generatedEvents2 := s.generateEvents(events2, c) - // nolint:gocritic generatedEvents := append(generatedEvents1, generatedEvents2...) mockStreamerProducer := &MockStreamProducer{generatedEvents} diff --git a/tests/all_mode/run.sh b/tests/all_mode/run.sh index d170d8729e..990b0b7857 100755 --- a/tests/all_mode/run.sh +++ b/tests/all_mode/run.sh @@ -171,8 +171,8 @@ function test_stop_task_before_checkpoint() { function run() { run_sql_both_source "SET @@GLOBAL.SQL_MODE='ANSI_QUOTES,NO_AUTO_VALUE_ON_ZERO'" - run_sql_source1 "SET @@global.time_zone = '+01:00';" - run_sql_source2 "SET @@global.time_zone = '+02:00';" + run_sql_source1 "SET @@global.time_zone = '+01:00';" + run_sql_source2 "SET @@global.time_zone = '+02:00';" test_session_config test_query_timeout @@ -339,7 +339,7 @@ function run() { export GO_FAILPOINTS='' run_sql_both_source "SET @@GLOBAL.SQL_MODE='ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION'" - run_sql_both_source "SET @@global.time_zone = 'SYSTEM';" + run_sql_both_source "SET @@global.time_zone = 'SYSTEM';" } cleanup_data all_mode diff --git a/tests/full_mode/run.sh b/tests/full_mode/run.sh index 53a35ce305..16346f3128 100755 --- a/tests/full_mode/run.sh +++ b/tests/full_mode/run.sh @@ -148,7 +148,7 @@ function run() { run_sql_both_source "SET @@GLOBAL.SQL_MODE='NO_BACKSLASH_ESCAPES'" run_sql_source1 "SET @@global.time_zone = '+01:00';" - run_sql_source2 "SET @@global.time_zone = '+02:00';" + run_sql_source2 "SET @@global.time_zone = '+02:00';" run_sql_file $cur/data/db1.prepare.sql $MYSQL_HOST1 $MYSQL_PORT1 $MYSQL_PASSWORD1 check_contains 'Query OK, 2 rows affected' @@ -195,7 +195,7 @@ function run() { check_metric_not_contains $WORKER1_PORT 'dm_worker_task_state{source_id="mysql-replica-01",task="test"}' 3 check_metric_not_contains $WORKER2_PORT 'dm_worker_task_state{source_id="mysql-replica-02",task="test"}' 3 run_sql_both_source "SET @@GLOBAL.SQL_MODE='ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION'" - run_sql_both_source "SET @@GLOBAL.TIME_ZONE='SYSTEM';" + run_sql_both_source "SET @@GLOBAL.TIME_ZONE='SYSTEM';" } cleanup_data full_mode From 90c683be22fd6e65ea4aeef35d1b46301141af65 Mon Sep 17 00:00:00 2001 From: glorv Date: Thu, 13 May 2021 18:03:24 +0800 Subject: [PATCH 15/15] resolve comment and format --- cmd/dm-syncer/config.go | 2 +- dm/config/task.go | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/cmd/dm-syncer/config.go b/cmd/dm-syncer/config.go index f443380902..85a879bfe3 100644 --- a/cmd/dm-syncer/config.go +++ b/cmd/dm-syncer/config.go @@ -273,7 +273,7 @@ type syncerConfig struct { // MaxDMLConnectionTimeout string `toml:"execute-dml-timeout" json:"execute-dml-timeout"` // ExecutionQueueLength int `toml:"execute-queue-length" json:"execute-queue-length"` - TimezoneStr string `toml:"timezone" json:"timezone"` + TimezoneStr string `toml:"timezone" json:"timezone"` printVersion bool } diff --git a/dm/config/task.go b/dm/config/task.go index 8db83f4e45..e133cc04c4 100644 --- a/dm/config/task.go +++ b/dm/config/task.go @@ -811,6 +811,10 @@ func AdjustTargetDBSessionCfg(dbConfig *DBConfig, version *semver.Version) { } } // force set time zone to UTC + if tz, ok := lowerMap["time_zone"]; ok { + log.L().Warn("session variable 'time_zone' is overwritten with UTC timezone.", + zap.String("time_zone", tz)) + } lowerMap["time_zone"] = defaultTimeZone dbConfig.Session = lowerMap } @@ -819,6 +823,8 @@ func AdjustTargetDBSessionCfg(dbConfig *DBConfig, version *semver.Version) { func AdjustTargetDBTimeZone(config *DBConfig) { for k := range config.Session { if strings.ToLower(k) == "time_zone" { + log.L().Warn("session variable 'time_zone' is overwritten by default UTC timezone.", + zap.String("time_zone", config.Session[k])) config.Session[k] = defaultTimeZone return }