From 77cb1ef293cdba8010b3491e5fdf3968d0b1eaf7 Mon Sep 17 00:00:00 2001 From: lance6716 Date: Thu, 17 Mar 2022 18:12:53 +0800 Subject: [PATCH 1/3] config(dm): fix can't downgrade config Signed-off-by: lance6716 --- dm/dm/config/task.go | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/dm/dm/config/task.go b/dm/dm/config/task.go index b559194478c..cb9905b8057 100644 --- a/dm/dm/config/task.go +++ b/dm/dm/config/task.go @@ -989,6 +989,25 @@ func NewMySQLInstancesForDowngrade(mysqlInstances []*MySQLInstance) []*MySQLInst return mysqlInstancesForDowngrade } +// LoaderConfigForDowngrade is the base configuration for loader in v2.0. +// This config is used for downgrade(config export) from a higher dmctl version. +// When we add any new config item into LoaderConfig, we should update it also. +type LoaderConfigForDowngrade struct { + PoolSize int `yaml:"pool-size" toml:"pool-size" json:"pool-size"` + Dir string `yaml:"dir" toml:"dir" json:"dir"` +} + +func NewLoaderConfigForDowngrade(loaderConfigs map[string]*LoaderConfig) map[string]*LoaderConfigForDowngrade { + loaderConfigsForDowngrade := make(map[string]*LoaderConfigForDowngrade, len(loaderConfigs)) + for k, v := range loaderConfigs { + loaderConfigsForDowngrade[k] = &LoaderConfigForDowngrade{ + PoolSize: v.PoolSize, + Dir: v.Dir, + } + } + return loaderConfigsForDowngrade +} + // SyncerConfigForDowngrade is the base configuration for syncer in v2.0. // This config is used for downgrade(config export) from a higher dmctl version. // When we add any new config item into SyncerConfig, we should update it also. @@ -1056,7 +1075,7 @@ type TaskConfigForDowngrade struct { BWList map[string]*filter.Rules `yaml:"black-white-list"` BAList map[string]*filter.Rules `yaml:"block-allow-list"` Mydumpers map[string]*MydumperConfig `yaml:"mydumpers"` - Loaders map[string]*LoaderConfig `yaml:"loaders"` + Loaders map[string]*LoaderConfigForDowngrade `yaml:"loaders"` Syncers map[string]*SyncerConfigForDowngrade `yaml:"syncers"` CleanDumpFile bool `yaml:"clean-dump-file"` EnableANSIQuotes bool `yaml:"ansi-quotes"` @@ -1091,7 +1110,7 @@ func NewTaskConfigForDowngrade(taskConfig *TaskConfig) *TaskConfigForDowngrade { BWList: taskConfig.BWList, BAList: taskConfig.BAList, Mydumpers: taskConfig.Mydumpers, - Loaders: taskConfig.Loaders, + Loaders: NewLoaderConfigForDowngrade(taskConfig.Loaders), Syncers: NewSyncerConfigsForDowngrade(taskConfig.Syncers), CleanDumpFile: taskConfig.CleanDumpFile, EnableANSIQuotes: taskConfig.EnableANSIQuotes, From 3bd6be1e9a2ad95030fec0cb4c8aa610c2a7f3f8 Mon Sep 17 00:00:00 2001 From: lance6716 Date: Thu, 17 Mar 2022 19:16:03 +0800 Subject: [PATCH 2/3] fix CI Signed-off-by: lance6716 --- dm/tests/dmctl_basic/conf/get_task.yaml | 2 -- dm/tests/import_v10x/conf/task.yaml | 2 -- dm/tests/new_relay/configs/tasks/test.yaml | 2 -- 3 files changed, 6 deletions(-) diff --git a/dm/tests/dmctl_basic/conf/get_task.yaml b/dm/tests/dmctl_basic/conf/get_task.yaml index 0ae500fb0d8..36ab127ce36 100644 --- a/dm/tests/dmctl_basic/conf/get_task.yaml +++ b/dm/tests/dmctl_basic/conf/get_task.yaml @@ -144,8 +144,6 @@ loaders: load-01: pool-size: 16 dir: ./dumped_data - import-mode: sql - on-duplicate: replace syncers: sync-01: meta-file: "" diff --git a/dm/tests/import_v10x/conf/task.yaml b/dm/tests/import_v10x/conf/task.yaml index ac256843270..f182be7bd58 100644 --- a/dm/tests/import_v10x/conf/task.yaml +++ b/dm/tests/import_v10x/conf/task.yaml @@ -89,8 +89,6 @@ loaders: load-01: pool-size: 16 dir: ./dumped_data - import-mode: sql - on-duplicate: replace syncers: sync-01: meta-file: "" diff --git a/dm/tests/new_relay/configs/tasks/test.yaml b/dm/tests/new_relay/configs/tasks/test.yaml index 3040c12592c..8c174e41f7b 100644 --- a/dm/tests/new_relay/configs/tasks/test.yaml +++ b/dm/tests/new_relay/configs/tasks/test.yaml @@ -43,8 +43,6 @@ loaders: load-01: pool-size: 16 dir: ./dumped_data - import-mode: sql - on-duplicate: replace syncers: sync-01: meta-file: "" From f553eb2ed9e7b20fc60db0bf7315fecd3240233b Mon Sep 17 00:00:00 2001 From: lance6716 Date: Thu, 17 Mar 2022 19:27:16 +0800 Subject: [PATCH 3/3] revert some config change :P Signed-off-by: lance6716 --- dm/tests/dmctl_basic/conf/get_task.yaml | 2 ++ dm/tests/import_v10x/conf/task.yaml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/dm/tests/dmctl_basic/conf/get_task.yaml b/dm/tests/dmctl_basic/conf/get_task.yaml index 36ab127ce36..0ae500fb0d8 100644 --- a/dm/tests/dmctl_basic/conf/get_task.yaml +++ b/dm/tests/dmctl_basic/conf/get_task.yaml @@ -144,6 +144,8 @@ loaders: load-01: pool-size: 16 dir: ./dumped_data + import-mode: sql + on-duplicate: replace syncers: sync-01: meta-file: "" diff --git a/dm/tests/import_v10x/conf/task.yaml b/dm/tests/import_v10x/conf/task.yaml index f182be7bd58..ac256843270 100644 --- a/dm/tests/import_v10x/conf/task.yaml +++ b/dm/tests/import_v10x/conf/task.yaml @@ -89,6 +89,8 @@ loaders: load-01: pool-size: 16 dir: ./dumped_data + import-mode: sql + on-duplicate: replace syncers: sync-01: meta-file: ""