Skip to content
This repository has been archived by the owner on Nov 24, 2023. It is now read-only.

config: fix empty flavor for source #1385

Merged
merged 7 commits into from
Jan 21, 2021
Merged

config: fix empty flavor for source #1385

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
6e8cae7
support empty flavor
GMHDBJD Jan 18, 2021
9a0ce09
use empty flavor for relay load meta
GMHDBJD Jan 18, 2021
2e2353b
verify source config after adjust
GMHDBJD Jan 18, 2021
fdb5eb0
add ut
GMHDBJD Jan 18, 2021
af7d73a
Merge remote-tracking branch 'upstream/master' into ParseGTID
GMHDBJD Jan 18, 2021
25eada5
fix it
GMHDBJD Jan 18, 2021
c587fb1
Merge branch 'master' into ParseGTID
GMHDBJD Jan 19, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion dm/config/source_config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,7 @@ func (c *SourceConfig) ParseYaml(content string) error {
return terror.ErrConfigYamlTransform.Delegate(err, "decode source config")
}
c.adjust()
return c.Verify()
return nil
}

// EncodeToml encodes config.
Expand Down
4 changes: 4 additions & 0 deletions dm/master/server.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -996,6 +996,10 @@ func parseAndAdjustSourceConfig(ctx context.Context, contents []string) ([]*conf
return cfgs, err
}

if err = cfg.Verify(); err != nil {
return cfgs, err
}

fromDB.Close()
cfgs[i] = cfg
}
Expand Down
5 changes: 3 additions & 2 deletions dm/worker/source.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,14 @@ server-id: 101
source-id: mysql-replica-01

#flavor: mysql/mariadb
flavor: mysql

#directory that used to store relay log
relay-dir: ./relay_log

#enable gtid in relay log unit
enable-gtid: false
enable-gtid: true

relay-binlog-gtid: "e68f6068-53ec-11eb-9c5f-0242ac110003:1-50"
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

there's a case that user didn't specify flavor or gtid, expecting starting relay at position of SHOW MASTER STATUS. Will SHOW MASTER STATUS returned empty Executed_Gtid_Set? If so, error message of "empty flavor with empty gtid is invalid" is misleading because he could only left them empty to use position of SHOW MASTER STATUS

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The flavor should not be empty since we have adjusted it or specify it, otherwise it is a bug. So empty flavor with empty gtid is invalid should not happen.


#charset of DSN of source mysql/mariadb instance
# charset: ''
Expand Down
30 changes: 25 additions & 5 deletions pkg/gtid/gtid.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
package gtid

import (
"github.com/pingcap/errors"
"github.com/siddontang/go-mysql/mysql"

"github.com/pingcap/dm/pkg/terror"
Expand Down Expand Up @@ -47,16 +48,35 @@ type Set interface {
// ParserGTID parses GTID from string
func ParserGTID(flavor, gtidStr string) (Set, error) {
var (
m Set
err error
m Set
err error
gtid mysql.GTIDSet
)

gtid, err := mysql.ParseGTIDSet(flavor, gtidStr)
if len(flavor) == 0 && len(gtidStr) == 0 {
return nil, errors.Errorf("empty flavor with empty gtid is invalid")
}

fla := flavor
3pointer marked this conversation as resolved.
Show resolved Hide resolved
switch fla {
case mysql.MySQLFlavor, mysql.MariaDBFlavor:
gtid, err = mysql.ParseGTIDSet(fla, gtidStr)
case "":
fla = mysql.MySQLFlavor
gtid, err = mysql.ParseGTIDSet(fla, gtidStr)
if err != nil {
fla = mysql.MariaDBFlavor
gtid, err = mysql.ParseGTIDSet(fla, gtidStr)
}
default:
err = terror.ErrNotSupportedFlavor.Generate(flavor)
}

if err != nil {
return nil, terror.ErrParseGTID.Delegate(err, gtidStr)
return nil, err
}

switch flavor {
switch fla {
case mysql.MariaDBFlavor:
m = &MariadbGTIDSet{}
case mysql.MySQLFlavor:
Expand Down
8 changes: 8 additions & 0 deletions pkg/gtid/gtid_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,14 @@ func (s *testGTIDSuite) TestGTID(c *C) {
{"mysql", []interface{}{matserUUIDs[0]}, fmt.Sprintf("%s:1-2", matserUUIDs[1]), "", ""},
{"mysql", []interface{}{matserUUIDs[0], matserUUIDs[1]}, fmt.Sprintf("%s:1-2,%s:1-2", matserUUIDs[0], matserUUIDs[1]), fmt.Sprintf("%s:1-12,%s:1-4,%s:1-4", matserUUIDs[0], matserUUIDs[1], matserUUIDs[2]), fmt.Sprintf("%s:1-2,%s:1-2,%s:1-4", matserUUIDs[0], matserUUIDs[1], matserUUIDs[2])},
{"mysql", []interface{}{matserUUIDs[0], matserUUIDs[2]}, fmt.Sprintf("%s:1-2", matserUUIDs[1]), fmt.Sprintf("%s:1-12,%s:1-3,%s:1-4", matserUUIDs[0], matserUUIDs[1], matserUUIDs[2]), fmt.Sprintf("%s:1-2", matserUUIDs[1])},
{"", []interface{}{uint32(1)}, "1-1-1,2-2-2", "1-1-12,4-4-4", "1-1-1,4-4-4"},
{"", []interface{}{uint32(1)}, "2-2-2", "1-2-12,2-2-3,4-4-4", "2-2-2,4-4-4"},
{"", []interface{}{uint32(1), uint32(3)}, "1-1-1,3-3-4,2-2-2", "1-1-12,3-3-8,4-4-4", "1-1-1,3-3-4,4-4-4"},
{"", []interface{}{uint32(1), uint32(3)}, "2-2-2", "1-2-12,2-2-3,3-3-8,4-4-4", "2-2-2,4-4-4"},
{"", []interface{}{matserUUIDs[0]}, fmt.Sprintf("%s:1-2,%s:1-2", matserUUIDs[0], matserUUIDs[1]), fmt.Sprintf("%s:1-12,%s:1-4", matserUUIDs[0], matserUUIDs[2]), fmt.Sprintf("%s:1-2,%s:1-4", matserUUIDs[0], matserUUIDs[2])},
{"", []interface{}{matserUUIDs[0]}, fmt.Sprintf("%s:1-2", matserUUIDs[1]), fmt.Sprintf("%s:1-12,%s:1-3,%s:1-4", matserUUIDs[0], matserUUIDs[1], matserUUIDs[2]), fmt.Sprintf("%s:1-2,%s:1-4", matserUUIDs[1], matserUUIDs[2])},
{"", []interface{}{matserUUIDs[0], matserUUIDs[1]}, fmt.Sprintf("%s:1-2,%s:1-2", matserUUIDs[0], matserUUIDs[1]), fmt.Sprintf("%s:1-12,%s:1-4,%s:1-4", matserUUIDs[0], matserUUIDs[1], matserUUIDs[2]), fmt.Sprintf("%s:1-2,%s:1-2,%s:1-4", matserUUIDs[0], matserUUIDs[1], matserUUIDs[2])},
{"", []interface{}{matserUUIDs[0], matserUUIDs[2]}, fmt.Sprintf("%s:1-2", matserUUIDs[1]), fmt.Sprintf("%s:1-12,%s:1-3,%s:1-4", matserUUIDs[0], matserUUIDs[1], matserUUIDs[2]), fmt.Sprintf("%s:1-2", matserUUIDs[1])},
}

for _, cs := range cases {
Expand Down
10 changes: 6 additions & 4 deletions relay/meta.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -473,11 +473,13 @@ func (lm *LocalMeta) loadMetaData() error {
return terror.ErrRelayLoadMetaData.Delegate(err)
}

gset, err := gtid.ParserGTID(lm.flavor, lm.BinlogGTID)
if err != nil {
return terror.ErrRelayLoadMetaData.Delegate(err)
if len(lm.BinlogGTID) != 0 {
gset, err := gtid.ParserGTID("", lm.BinlogGTID)
if err != nil {
return terror.ErrRelayLoadMetaData.Delegate(err)
}
lm.gset = gset
}
lm.gset = gset

return nil
}