Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ddl: implement the placement rules inheritance logic #28365

Merged
merged 29 commits into from
Oct 8, 2021
Merged

ddl: implement the placement rules inheritance logic #28365

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
29 commits
Select commit Hold shift + click to select a range
76baf02
use the persistent policy in parent objects and avoid use pd default …
AilinKid Sep 26, 2021
f830bd5
add bundle index and override field
AilinKid Sep 26, 2021
9e0bc16
resolve the nil pointer
AilinKid Sep 26, 2021
38d95f9
add create-table placement inheritance and build independent partion …
AilinKid Sep 28, 2021
8d410fb
add cancel
AilinKid Sep 28, 2021
198d6ec
add comment
AilinKid Sep 28, 2021
2e5be09
address comment
AilinKid Sep 28, 2021
808b5e3
fix test
AilinKid Sep 28, 2021
1603f0d
refactor reset bundle
AilinKid Sep 28, 2021
9b3df33
fix panic
AilinKid Sep 28, 2021
9aadfae
remove useless code
AilinKid Sep 28, 2021
9467df2
fix the alter policy
AilinKid Sep 28, 2021
41d2df0
.
AilinKid Sep 28, 2021
edf50df
show the database and partition rules
AilinKid Sep 29, 2021
f08f0ff
fix comment blank
AilinKid Sep 29, 2021
d94d838
fix the placement sql test
AilinKid Sep 30, 2021
905107b
fix the rules id
AilinKid Sep 30, 2021
6e73fa4
address comment from chao and he
AilinKid Oct 2, 2021
f6cd6a9
.
AilinKid Oct 2, 2021
23c81fc
fix test
AilinKid Oct 8, 2021
4545f19
fix bundle
AilinKid Oct 8, 2021
7050f69
fix test
AilinKid Oct 8, 2021
247a8df
change the rule de-duplication to index indentification
AilinKid Oct 8, 2021
ad907f3
move tidy to constructors: NewBundleFromOptions
AilinKid Oct 8, 2021
e606439
fmt
AilinKid Oct 8, 2021
45a2081
Merge branch 'master' into avoid-sync-default-bundle-to-pd
AilinKid Oct 8, 2021
1ae67cb
Merge branch 'master' into avoid-sync-default-bundle-to-pd
AilinKid Oct 8, 2021
dcb801e
fix test
AilinKid Oct 8, 2021
6806823
Merge branch 'master' into avoid-sync-default-bundle-to-pd
AilinKid Oct 8, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 2 additions & 11 deletions ddl/ddl_api.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6191,10 +6191,9 @@ func (d *ddl) AlterTableAlterPartition(ctx sessionctx.Context, ident ast.Ident,
return errors.Trace(err)
}

// TODO: the old placement rules should be migrated to new format. use the bundle from meta directly.
bundle := infoschema.GetBundle(d.infoCache.GetLatest(), []int64{partitionID, meta.ID, schema.ID})

bundle.ID = placement.GroupID(partitionID)

err = bundle.ApplyPlacementSpec(spec.PlacementSpecs)
if err != nil {
var sb strings.Builder
Expand All @@ -6212,15 +6211,7 @@ func (d *ddl) AlterTableAlterPartition(ctx sessionctx.Context, ident ast.Ident,
if err != nil {
return errors.Trace(err)
}
bundle.Reset(partitionID)

if len(bundle.Rules) == 0 {
bundle.Index = 0
bundle.Override = false
} else {
bundle.Index = placement.RuleIndexPartition
bundle.Override = true
}
bundle.Reset(placement.RuleIndexPartition, []int64{partitionID})

job := &model.Job{
SchemaID: schema.ID,
Expand Down
45 changes: 38 additions & 7 deletions ddl/partition.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -487,13 +487,44 @@ func buildRangePartitionDefinitions(ctx sessionctx.Context, defs []*ast.Partitio
}
}
comment, _ := def.Comment()
var directPlacementOpts *model.PlacementSettings
var placementPolicyRef *model.PolicyRefInfo
// the partition inheritance of placement rules don't have to copy the placement elements to themselves.
// For example:
// t placement policy x (p1 placement policy y, p2)
// p2 will share the same rule as table t does, but it won't copy the meta to itself. we will
// append p2 range to the coverage of table t's rules. This mechanism is good for cascading change
// when policy x is altered.
for _, opt := range def.Options {
switch opt.Tp {
case ast.TableOptionPlacementPrimaryRegion, ast.TableOptionPlacementRegions,
ast.TableOptionPlacementFollowerCount, ast.TableOptionPlacementVoterCount,
ast.TableOptionPlacementLearnerCount, ast.TableOptionPlacementSchedule,
ast.TableOptionPlacementConstraints, ast.TableOptionPlacementLeaderConstraints,
ast.TableOptionPlacementLearnerConstraints, ast.TableOptionPlacementFollowerConstraints,
ast.TableOptionPlacementVoterConstraints:
if directPlacementOpts == nil {
directPlacementOpts = &model.PlacementSettings{}
}
err := SetDirectPlacementOpt(directPlacementOpts, ast.PlacementOptionType(opt.Tp), opt.StrValue, opt.UintValue)
if err != nil {
return nil, err
}
case ast.TableOptionPlacementPolicy:
placementPolicyRef = &model.PolicyRefInfo{
Name: model.NewCIStr(opt.StrValue),
}
}
}
err := checkTooLongTable(def.Name)
if err != nil {
return nil, err
}
piDef := model.PartitionDefinition{
Name: def.Name,
Comment: comment,
Name: def.Name,
Comment: comment,
DirectPlacementOpts: directPlacementOpts,
PlacementPolicyRef: placementPolicyRef,
}

buf := new(bytes.Buffer)
Expand Down Expand Up @@ -1124,7 +1155,7 @@ func onTruncateTablePartition(d *ddlCtx, t *meta.Meta, job *model.Job) (int64, e
oldBundle, ok := d.infoCache.GetLatest().BundleByName(placement.GroupID(oldID))
if ok && !oldBundle.IsEmpty() {
bundles = append(bundles, placement.NewBundle(oldID))
bundles = append(bundles, oldBundle.Clone().Reset(newPartitions[i].ID))
bundles = append(bundles, oldBundle.Clone().Reset(placement.RuleIndexPartition, []int64{newPartitions[i].ID}))
}
}

Expand Down Expand Up @@ -1331,14 +1362,14 @@ func (w *worker) onExchangeTablePartition(d *ddlCtx, t *meta.Meta, job *model.Jo
ntBundle, ntOK := d.infoCache.GetLatest().BundleByName(placement.GroupID(nt.ID))
ntOK = ntOK && !ntBundle.IsEmpty()
if ptOK && ntOK {
bundles = append(bundles, ptBundle.Clone().Reset(nt.ID))
bundles = append(bundles, ntBundle.Clone().Reset(partDef.ID))
bundles = append(bundles, ptBundle.Clone().Reset(placement.RuleIndexPartition, []int64{nt.ID}))
bundles = append(bundles, ntBundle.Clone().Reset(placement.RuleIndexPartition, []int64{partDef.ID}))
} else if ptOK {
bundles = append(bundles, placement.NewBundle(partDef.ID))
bundles = append(bundles, ptBundle.Clone().Reset(nt.ID))
bundles = append(bundles, ptBundle.Clone().Reset(placement.RuleIndexPartition, []int64{nt.ID}))
} else if ntOK {
bundles = append(bundles, placement.NewBundle(nt.ID))
bundles = append(bundles, ntBundle.Clone().Reset(partDef.ID))
bundles = append(bundles, ntBundle.Clone().Reset(placement.RuleIndexPartition, []int64{partDef.ID}))
}
err = infosync.PutRuleBundles(context.TODO(), bundles)
if err != nil {
Expand Down
86 changes: 73 additions & 13 deletions ddl/placement/bundle.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -187,8 +187,9 @@ func NewBundleFromSugarOptions(options *model.PlacementSettings) (*Bundle, error
return &Bundle{Rules: Rules}, nil
}

// NewBundleFromOptions will transform options into the bundle.
func NewBundleFromOptions(options *model.PlacementSettings) (*Bundle, error) {
// Non-Exported functionality function, do not use it directly but NewBundleFromOptions
// here is for only directly used in the test.
func newBundleFromOptions(options *model.PlacementSettings) (bundle *Bundle, err error) {
var isSyntaxSugar bool

if options == nil {
Expand All @@ -200,9 +201,27 @@ func NewBundleFromOptions(options *model.PlacementSettings) (*Bundle, error) {
}

if isSyntaxSugar {
return NewBundleFromSugarOptions(options)
bundle, err = NewBundleFromSugarOptions(options)
} else {
bundle, err = NewBundleFromConstraintsOptions(options)
}
return bundle, err
}

// NewBundleFromOptions will transform options into the bundle.
func NewBundleFromOptions(options *model.PlacementSettings) (bundle *Bundle, err error) {
bundle, err = newBundleFromOptions(options)
if err != nil {
return nil, err
}
if bundle == nil {
return nil, nil
}
return NewBundleFromConstraintsOptions(options)
err = bundle.Tidy()
if err != nil {
return nil, err
}
return bundle, err
}

// ApplyPlacementSpec will apply actions defined in PlacementSpec to the bundle.
Expand Down Expand Up @@ -326,16 +345,57 @@ func (b *Bundle) Tidy() error {
}

// Reset resets the bundle ID and keyrange of all rules.
func (b *Bundle) Reset(newID int64) *Bundle {
b.ID = GroupID(newID)
// Involve all the table level objects.
startKey := hex.EncodeToString(codec.EncodeBytes(nil, tablecodec.GenTablePrefix(newID)))
endKey := hex.EncodeToString(codec.EncodeBytes(nil, tablecodec.GenTablePrefix(newID+1)))
for _, rule := range b.Rules {
rule.GroupID = b.ID
rule.StartKeyHex = startKey
rule.EndKeyHex = endKey
func (b *Bundle) Reset(ruleIndex int, newIDs []int64) *Bundle {
Copy link
Collaborator

@lcwangchao lcwangchao Oct 1, 2021 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The current code about bundles seems some what weird and a little complex now. How about refactoring them to below methods:

// Build bundle from table info, and the partition rules and included in the return
NewBundleFromTable(tbl *model.TableInfo) (*Bundle, error)

// Build bundle from partition
NewBundleFromPartition(partition *model.PartitionDefinition) (*Bundle, error)

I think the above methods is enough for ddl to use. It's not necessary to use BundleByName in infoschema because we'll finally delete it and the logic that directly building bundle from meta is more brief and easy to maintain.

We do not need method Tidy too. We can tidy the bundle when we constructing it, I think no where requires a not tidied bundle.

Another advices is the reuse of the code, for example, we can introduce a private method like

newBundleFromPolicyOrDirectOptions(infoschema.InfoSchema, *model.PolicyRefInfo, *model.PlacementSettings) (*Bundle, error)

It will return a semi-finished Bundle and just for code reuse and it should only be visible in placement package

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

1: refactored to func NewBundleFromTblInfo(t *meta.Meta, job *model.Job, tbInfo *model.TableInfo) and func NewBundleFromPartition(t *meta.Meta, job *model.Job, partition *model.PartitionInfo)
2: tidy moved to constructors--- NewBundleFromOptions
3: the newBundleFromPolicyOrDirectOptions function is built

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@AilinKid, where is the change of 2? I don't see that.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

added, missed in my commits

// eliminate the redundant rules.
var basicRules []*Rule
if len(b.Rules) != 0 {
// Make priority for rules with RuleIndexTable cause of duplication rules existence with RuleIndexPartition.
// If RuleIndexTable doesn't exist, bundle itself is a independent series of rules for a partition.
for _, rule := range b.Rules {
if rule.Index == RuleIndexTable {
basicRules = append(basicRules, rule)
}
}
if len(basicRules) == 0 {
basicRules = b.Rules
}
}

// extend and reset basic rules for all new ids, the first id should be the group id.
b.ID = GroupID(newIDs[0])
b.Index = ruleIndex
b.Override = true
newRules := make([]*Rule, 0, len(basicRules)*len(newIDs))
for i, newID := range newIDs {
// rule.id should be distinguished with each other, otherwise it will be de-duplicated in pd http api.
var ruleID string
if ruleIndex == RuleIndexPartition {
ruleID = "partition_rule_" + strconv.FormatInt(newID, 10)
} else {
if i == 0 {
ruleID = "table_rule_" + strconv.FormatInt(newID, 10)
} else {
ruleID = "partition_rule_" + strconv.FormatInt(newID, 10)
}
}
// Involve all the table level objects.
startKey := hex.EncodeToString(codec.EncodeBytes(nil, tablecodec.GenTablePrefix(newID)))
endKey := hex.EncodeToString(codec.EncodeBytes(nil, tablecodec.GenTablePrefix(newID+1)))
for _, rule := range basicRules {
clone := rule.Clone()
clone.ID = ruleID
clone.GroupID = b.ID
clone.StartKeyHex = startKey
clone.EndKeyHex = endKey
if i == 0 {
clone.Index = RuleIndexTable
} else {
clone.Index = RuleIndexPartition
}
newRules = append(newRules, clone)
}
}
b.Rules = newRules
return b
}

Expand Down
102 changes: 99 additions & 3 deletions ddl/placement/bundle_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -789,7 +789,7 @@ func (s *testBundleSuite) TestNewBundleFromOptions(c *C) {
})

for _, t := range tests {
bundle, err := NewBundleFromOptions(t.input)
bundle, err := newBundleFromOptions(t.input)
comment := Commentf("[%s]\nerr1 %s\nerr2 %s", t.name, err, t.err)
if t.err != nil {
c.Assert(errors.Is(err, t.err), IsTrue, comment)
Expand All @@ -800,7 +800,7 @@ func (s *testBundleSuite) TestNewBundleFromOptions(c *C) {
}
}

func (s *testBundleSuite) TestReset(c *C) {
func (s *testBundleSuite) TestResetBundleWithSingleRule(c *C) {
bundle := &Bundle{
ID: GroupID(1),
}
Expand All @@ -809,8 +809,10 @@ func (s *testBundleSuite) TestReset(c *C) {
c.Assert(err, IsNil)
bundle.Rules = rules

bundle.Reset(3)
bundle.Reset(RuleIndexTable, []int64{3})
c.Assert(bundle.ID, Equals, GroupID(3))
c.Assert(bundle.Override, Equals, true)
c.Assert(bundle.Index, Equals, RuleIndexTable)
c.Assert(bundle.Rules, HasLen, 1)
c.Assert(bundle.Rules[0].GroupID, Equals, bundle.ID)

Expand All @@ -821,6 +823,100 @@ func (s *testBundleSuite) TestReset(c *C) {
c.Assert(bundle.Rules[0].EndKeyHex, Equals, endKey)
}

func (s *testBundleSuite) TestResetBundleWithMultiRules(c *C) {
// build a bundle with three rules.
bundle, err := NewBundleFromOptions(&model.PlacementSettings{
LeaderConstraints: `["+zone=bj"]`,
Followers: 2,
FollowerConstraints: `["+zone=hz"]`,
Learners: 1,
LearnerConstraints: `["+zone=cd"]`,
Constraints: `["+disk=ssd"]`,
})
c.Assert(err, IsNil)
c.Assert(len(bundle.Rules), Equals, 3)

// test if all the three rules are basic rules even the start key are not set.
bundle.Reset(RuleIndexTable, []int64{1, 2, 3})
c.Assert(bundle.ID, Equals, GroupID(1))
c.Assert(bundle.Index, Equals, RuleIndexTable)
c.Assert(bundle.Override, Equals, true)
c.Assert(len(bundle.Rules), Equals, 3*3)
// for id 1.
startKey := hex.EncodeToString(codec.EncodeBytes(nil, tablecodec.GenTablePrefix(1)))
endKey := hex.EncodeToString(codec.EncodeBytes(nil, tablecodec.GenTablePrefix(2)))
c.Assert(bundle.Rules[0].StartKeyHex, Equals, startKey)
c.Assert(bundle.Rules[0].EndKeyHex, Equals, endKey)
c.Assert(bundle.Rules[1].StartKeyHex, Equals, startKey)
c.Assert(bundle.Rules[1].EndKeyHex, Equals, endKey)
c.Assert(bundle.Rules[2].StartKeyHex, Equals, startKey)
c.Assert(bundle.Rules[2].EndKeyHex, Equals, endKey)
// for id 2.
startKey = hex.EncodeToString(codec.EncodeBytes(nil, tablecodec.GenTablePrefix(2)))
endKey = hex.EncodeToString(codec.EncodeBytes(nil, tablecodec.GenTablePrefix(3)))
c.Assert(bundle.Rules[3].StartKeyHex, Equals, startKey)
c.Assert(bundle.Rules[3].EndKeyHex, Equals, endKey)
c.Assert(bundle.Rules[4].StartKeyHex, Equals, startKey)
c.Assert(bundle.Rules[4].EndKeyHex, Equals, endKey)
c.Assert(bundle.Rules[5].StartKeyHex, Equals, startKey)
c.Assert(bundle.Rules[5].EndKeyHex, Equals, endKey)
// for id 3.
startKey = hex.EncodeToString(codec.EncodeBytes(nil, tablecodec.GenTablePrefix(3)))
endKey = hex.EncodeToString(codec.EncodeBytes(nil, tablecodec.GenTablePrefix(4)))
c.Assert(bundle.Rules[6].StartKeyHex, Equals, startKey)
c.Assert(bundle.Rules[6].EndKeyHex, Equals, endKey)
c.Assert(bundle.Rules[7].StartKeyHex, Equals, startKey)
c.Assert(bundle.Rules[7].EndKeyHex, Equals, endKey)
c.Assert(bundle.Rules[8].StartKeyHex, Equals, startKey)
c.Assert(bundle.Rules[8].EndKeyHex, Equals, endKey)

// test if bundle has redundant rules.
// for now, the bundle has 9 rules, each table id or partition id has the three with them.
// once we reset this bundle for another ids, for example, adding partitions. we should
// extend the basic rules(3 of them) to the new partition id.
bundle.Reset(RuleIndexTable, []int64{1, 3, 4, 5})
c.Assert(bundle.ID, Equals, GroupID(1))
c.Assert(bundle.Index, Equals, RuleIndexTable)
c.Assert(bundle.Override, Equals, true)
c.Assert(len(bundle.Rules), Equals, 3*4)
// for id 1.
startKey = hex.EncodeToString(codec.EncodeBytes(nil, tablecodec.GenTablePrefix(1)))
endKey = hex.EncodeToString(codec.EncodeBytes(nil, tablecodec.GenTablePrefix(2)))
c.Assert(bundle.Rules[0].StartKeyHex, Equals, startKey)
c.Assert(bundle.Rules[0].EndKeyHex, Equals, endKey)
c.Assert(bundle.Rules[1].StartKeyHex, Equals, startKey)
c.Assert(bundle.Rules[1].EndKeyHex, Equals, endKey)
c.Assert(bundle.Rules[2].StartKeyHex, Equals, startKey)
c.Assert(bundle.Rules[2].EndKeyHex, Equals, endKey)
// for id 3.
startKey = hex.EncodeToString(codec.EncodeBytes(nil, tablecodec.GenTablePrefix(3)))
endKey = hex.EncodeToString(codec.EncodeBytes(nil, tablecodec.GenTablePrefix(4)))
c.Assert(bundle.Rules[3].StartKeyHex, Equals, startKey)
c.Assert(bundle.Rules[3].EndKeyHex, Equals, endKey)
c.Assert(bundle.Rules[4].StartKeyHex, Equals, startKey)
c.Assert(bundle.Rules[4].EndKeyHex, Equals, endKey)
c.Assert(bundle.Rules[5].StartKeyHex, Equals, startKey)
c.Assert(bundle.Rules[5].EndKeyHex, Equals, endKey)
// for id 4.
startKey = hex.EncodeToString(codec.EncodeBytes(nil, tablecodec.GenTablePrefix(4)))
endKey = hex.EncodeToString(codec.EncodeBytes(nil, tablecodec.GenTablePrefix(5)))
c.Assert(bundle.Rules[6].StartKeyHex, Equals, startKey)
c.Assert(bundle.Rules[6].EndKeyHex, Equals, endKey)
c.Assert(bundle.Rules[7].StartKeyHex, Equals, startKey)
c.Assert(bundle.Rules[7].EndKeyHex, Equals, endKey)
c.Assert(bundle.Rules[8].StartKeyHex, Equals, startKey)
c.Assert(bundle.Rules[8].EndKeyHex, Equals, endKey)
// for id 5.
startKey = hex.EncodeToString(codec.EncodeBytes(nil, tablecodec.GenTablePrefix(5)))
endKey = hex.EncodeToString(codec.EncodeBytes(nil, tablecodec.GenTablePrefix(6)))
c.Assert(bundle.Rules[9].StartKeyHex, Equals, startKey)
c.Assert(bundle.Rules[9].EndKeyHex, Equals, endKey)
c.Assert(bundle.Rules[10].StartKeyHex, Equals, startKey)
c.Assert(bundle.Rules[10].EndKeyHex, Equals, endKey)
c.Assert(bundle.Rules[11].StartKeyHex, Equals, startKey)
c.Assert(bundle.Rules[11].EndKeyHex, Equals, endKey)
}

func (s *testBundleSuite) TestTidy(c *C) {
bundle := &Bundle{
ID: GroupID(1),
Expand Down
Loading