Skip to content
This repository has been archived by the owner on Nov 24, 2023. It is now read-only.

config: add warning, when users set configuration to mydumper/loader/syncer b… #1410

Merged
merged 12 commits into from
Feb 2, 2021
Merged

config: add warning, when users set configuration to mydumper/loader/syncer b… #1410

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
7572e73
Add warning, when users set configuration to mydumper/loader/syncer b…
zeminzhou Jan 29, 2021
5c3dcce
TODO: generate new errors.toml; add unit test
zeminzhou Jan 31, 2021
c185dd3
generate errors.toml
zeminzhou Feb 1, 2021
0f0fe15
add unit test
zeminzhou Feb 1, 2021
b84f9e0
fine adjustment
zeminzhou Feb 1, 2021
fe703dc
adjustments
zeminzhou Feb 1, 2021
800b610
generate errors.toml
zeminzhou Feb 1, 2021
6a954ae
ajustment
zeminzhou Feb 1, 2021
c2e16fd
adjustment
zeminzhou Feb 2, 2021
1ed3dd7
adjustment
zeminzhou Feb 2, 2021
81ce23c
adjustment
zeminzhou Feb 2, 2021
6c73b20
Merge branch 'master' into issue_1409
lance6716 Feb 2, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions _utils/terror_gen/errors_release.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,8 @@ ErrConfigMoreThanOne,[code=20034:class=config:scope=internal:level=high], "Messa
ErrConfigEtcdParse,[code=20035:class=config:scope=internal:level=high], "Message: incapable config of %s from etcd"
ErrConfigMissingForBound,[code=20036:class=config:scope=internal:level=high], "Message: source bound %s doesn't have related source config in etcd"
ErrConfigBinlogEventFilter,[code=20037:class=config:scope=internal:level=high], "Message: generate binlog event filter, Workaround: Please check the `filters` config in source and task configuration files."
ErrConfigUnreferingCofig,[code=20038:class=config:scope=internal:level=high], "Message: instances don't refer configurations with xxx-config-name, Workaround: Please check the configuration files."
ErrConfigGobalConfigUnused,[code=20039:class=config:scope=internal:level=high], "Message: partial function configurations are set in gobal configurations but instances don't use them, Workaround: Please check the configuration files."
ErrBinlogExtractPosition,[code=22001:class=binlog-op:scope=internal:level=high]
ErrBinlogInvalidFilename,[code=22002:class=binlog-op:scope=internal:level=high], "Message: invalid binlog filename"
ErrBinlogParsePosFromStr,[code=22003:class=binlog-op:scope=internal:level=high]
Expand Down
56 changes: 56 additions & 0 deletions dm/config/task.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -418,6 +418,8 @@ func (c *TaskConfig) adjust() error {
}

iids := make(map[string]int) // source-id -> instance-index
gobalConfigReferCount := map[string]int{}
lance6716 marked this conversation as resolved.
Show resolved Hide resolved
prefixs := []string{"RouteRules", "FilterRules", "ColumnMappingRules", "Mydumper", "Loader", "Syncer"}
duplicateErrorStrings := make([]string, 0)
for i, inst := range c.MySQLInstances {
if err := inst.VerifyAndAdjust(); err != nil {
Expand Down Expand Up @@ -447,16 +449,19 @@ func (c *TaskConfig) adjust() error {
if _, ok := c.Routes[name]; !ok {
return terror.ErrConfigRouteRuleNotFound.Generate(i, name)
}
gobalConfigReferCount[prefixs[0]+name]++
}
for _, name := range inst.FilterRules {
if _, ok := c.Filters[name]; !ok {
return terror.ErrConfigFilterRuleNotFound.Generate(i, name)
}
gobalConfigReferCount[prefixs[1]+name]++
}
for _, name := range inst.ColumnMappingRules {
if _, ok := c.ColumnMappings[name]; !ok {
return terror.ErrConfigColumnMappingNotFound.Generate(i, name)
}
gobalConfigReferCount[prefixs[2]+name]++
}

// only when BAList is empty use BWList
Expand All @@ -472,10 +477,15 @@ func (c *TaskConfig) adjust() error {
if !ok {
return terror.ErrConfigMydumperCfgNotFound.Generate(i, inst.MydumperConfigName)
}
gobalConfigReferCount[prefixs[3]+inst.MydumperConfigName]++
inst.Mydumper = new(MydumperConfig)
*inst.Mydumper = *rule // ref mydumper config
}
if inst.Mydumper == nil {
if len(c.Mydumpers) != 0 && inst.MydumperThread == 0 {
log.L().Warn("mysql instance don't refer mydumper's configuration with mydumper-config-name, the default configuration will be used", zap.Int("mysql instance", i))
return terror.ErrConfigUnreferingCofig.Generate()
lance6716 marked this conversation as resolved.
Show resolved Hide resolved
}
defaultCfg := defaultMydumperConfig()
inst.Mydumper = &defaultCfg
} else if inst.Mydumper.ChunkFilesize == "" {
Expand All @@ -496,10 +506,15 @@ func (c *TaskConfig) adjust() error {
if !ok {
return terror.ErrConfigLoaderCfgNotFound.Generate(i, inst.LoaderConfigName)
}
gobalConfigReferCount[prefixs[4]+inst.LoaderConfigName]++
inst.Loader = new(LoaderConfig)
*inst.Loader = *rule // ref loader config
}
if inst.Loader == nil {
if len(c.Loaders) != 0 && inst.LoaderThread == 0 {
log.L().Warn("mysql instance don't refer loader's configuration with loader-config-name, the default configuration will be used", zap.Int("mysql instance", i))
return terror.ErrConfigUnreferingCofig.Generate()
}
defaultCfg := defaultLoaderConfig()
inst.Loader = &defaultCfg
}
Expand All @@ -512,10 +527,15 @@ func (c *TaskConfig) adjust() error {
if !ok {
return terror.ErrConfigSyncerCfgNotFound.Generate(i, inst.SyncerConfigName)
}
gobalConfigReferCount[prefixs[5]+inst.SyncerConfigName]++
inst.Syncer = new(SyncerConfig)
*inst.Syncer = *rule // ref syncer config
}
if inst.Syncer == nil {
if len(c.Syncers) != 0 && inst.SyncerThread == 0 {
log.L().Warn("mysql instance don't refer syncer's configuration with syncer-config-name, the default configuration will be used", zap.Int("mysql instance", i))
lance6716 marked this conversation as resolved.
Show resolved Hide resolved
return terror.ErrConfigUnreferingCofig.Generate()
}
defaultCfg := defaultSyncerConfig()
inst.Syncer = &defaultCfg
}
Expand All @@ -539,6 +559,42 @@ func (c *TaskConfig) adjust() error {
return terror.ErrConfigDuplicateCfgItem.Generate(strings.Join(duplicateErrorStrings, "\n"))
}

for route := range c.Routes {
if gobalConfigReferCount[prefixs[0]+route] == 0 {
log.L().Error("route in gobal configuration isn't used", zap.String("route", route))
return terror.ErrConfigGobalConfigUnused.Generate()
lance6716 marked this conversation as resolved.
Show resolved Hide resolved
}
}
for filter := range c.Filters {
if gobalConfigReferCount[prefixs[1]+filter] == 0 {
log.L().Error("filter in gobal configuration isn't used", zap.String("filter", filter))
return terror.ErrConfigGobalConfigUnused.Generate()
}
}
for columnMapping := range c.ColumnMappings {
if gobalConfigReferCount[prefixs[2]+columnMapping] == 0 {
log.L().Error("column-mapping in gobal configuration isn't used", zap.String("columnMapping", columnMapping))
return terror.ErrConfigGobalConfigUnused.Generate()
}
}
for mydumper := range c.Mydumpers {
if gobalConfigReferCount[prefixs[3]+mydumper] == 0 {
log.L().Error("mydumper in gobal configuration isn't used", zap.String("mydumper", mydumper))
return terror.ErrConfigGobalConfigUnused.Generate()
}
}
for loader := range c.Loaders {
if gobalConfigReferCount[prefixs[4]+loader] == 0 {
log.L().Error("loader in gobal configuration isn't used", zap.String("loader", loader))
return terror.ErrConfigGobalConfigUnused.Generate()
}
}
for syncer := range c.Syncers {
if gobalConfigReferCount[prefixs[5]+syncer] == 0 {
log.L().Error("syncer in gobal configuration isn't used", zap.String("syncer", syncer))
return terror.ErrConfigGobalConfigUnused.Generate()
}
}
if c.Timezone != "" {
_, err := time.LoadLocation(c.Timezone)
if err != nil {
Expand Down
Loading