Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

mcs: make scheduling server test stable #7367

Merged
merged 6 commits into from
Nov 16, 2023
Merged

mcs: make scheduling server test stable #7367

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
dbaeceb
mcs: make scheduling server test stable
lhy1024 Nov 14, 2023
3298197
refactor test
lhy1024 Nov 14, 2023
ce4bb95
refacator test
lhy1024 Nov 15, 2023
e274d1e
make TestAPI, TestConfigWatch and TestConfigReplication more stable
lhy1024 Nov 15, 2023
52ad5fa
Merge branch 'master' into fix-test14
ti-chi-bot[bot] Nov 16, 2023
1c408ba
Merge branch 'master' into fix-test14
ti-chi-bot[bot] Nov 16, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions pkg/utils/testutil/api_check.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,21 @@
return checkResp(resp, checkOpts...)
}

// CheckGetUntilStatusCode is used to do get request and do check options.
func CheckGetUntilStatusCode(re *require.Assertions, client *http.Client, url string, code int) error {
var err error
Eventually(re, func() bool {
resp, err2 := apiutil.GetJSON(client, url, nil)
if err2 != nil {
err = err2
return true

Check warning on line 124 in pkg/utils/testutil/api_check.go

View check run for this annotation

Codecov / codecov/patch

pkg/utils/testutil/api_check.go#L123-L124 

Added lines #L123 - L124 were not covered by tests
}
defer resp.Body.Close()
return resp.StatusCode == code
})
return err
}

// CheckPatchJSON is used to do patch request and do check options.
func CheckPatchJSON(client *http.Client, url string, data []byte, checkOpts ...func([]byte, int, http.Header)) error {
resp, err := apiutil.PatchJSON(client, url, data)
Expand Down
3 changes: 3 additions & 0 deletions server/api/diagnostic_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ package api
import (
"encoding/json"
"fmt"
"net/http"
"testing"
"time"

Expand Down Expand Up @@ -63,6 +64,8 @@ func (suite *diagnosticTestSuite) TearDownSuite() {

func (suite *diagnosticTestSuite) checkStatus(status string, url string) {
re := suite.Require()
err := tu.CheckGetUntilStatusCode(re, testDialClient, url, http.StatusOK)
suite.NoError(err)
suite.Eventually(func() bool {
result := &schedulers.DiagnosticResult{}
err := tu.ReadGetJSON(re, testDialClient, url, result)
Expand Down
3 changes: 3 additions & 0 deletions tests/integrations/mcs/scheduling/config_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,9 @@ func (suite *configTestSuite) TestConfigWatch() {
re.Equal(sc.DefaultSplitMergeInterval, watcher.GetScheduleConfig().SplitMergeInterval.Duration)
re.Equal("0.0.0", watcher.GetClusterVersion().String())
// Update the config and check if the scheduling config watcher can get the latest value.
testutil.Eventually(re, func() bool {
return watcher.GetReplicationConfig().MaxReplicas == 3
})
persistOpts := suite.pdLeaderServer.GetPersistOptions()
persistOpts.SetMaxReplicas(5)
persistConfig(re, suite.pdLeaderServer)
Expand Down
183 changes: 98 additions & 85 deletions tests/pdctl/config/config_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -315,25 +315,15 @@ func (suite *configTestSuite) checkPlacementRules(cluster *tests.TestCluster) {
re.Contains(string(output), "Success!")

// test show
var rules []placement.Rule
output, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "show")
re.NoError(err)
re.NoError(json.Unmarshal(output, &rules))
re.Len(rules, 1)
re.Equal([2]string{"pd", "default"}, rules[0].Key())
suite.checkShowRuleKey(pdAddr, [][2]string{{"pd", "default"}})

f, _ := os.CreateTemp("/tmp", "pd_tests")
fname := f.Name()
f.Close()
defer os.RemoveAll(fname)

// test load
_, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "load", "--out="+fname)
re.NoError(err)
b, _ := os.ReadFile(fname)
re.NoError(json.Unmarshal(b, &rules))
re.Len(rules, 1)
re.Equal([2]string{"pd", "default"}, rules[0].Key())
rules := suite.checkLoadRule(pdAddr, fname, [][2]string{{"pd", "default"}})

// test save
rules = append(rules, placement.Rule{
Expand All @@ -347,42 +337,26 @@ func (suite *configTestSuite) checkPlacementRules(cluster *tests.TestCluster) {
Role: "voter",
Count: 2,
})
b, _ = json.Marshal(rules)
b, _ := json.Marshal(rules)
os.WriteFile(fname, b, 0600)
_, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "save", "--in="+fname)
re.NoError(err)

// test show group
var rules2 []placement.Rule
output, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "show", "--group=pd")
re.NoError(err)
re.NoError(json.Unmarshal(output, &rules2))
re.Len(rules2, 2)
re.Equal([2]string{"pd", "default"}, rules2[0].Key())
re.Equal([2]string{"pd", "test1"}, rules2[1].Key())
suite.checkShowRuleKey(pdAddr, [][2]string{{"pd", "default"}, {"pd", "test1"}}, "--group=pd")

// test rule region detail
tests.MustPutRegion(re, cluster, 1, 1, []byte("a"), []byte("b"))
fit := &placement.RegionFit{}
// need clear up args, so create new a cobra.Command. Otherwise gourp still exists.
cmd2 := pdctlCmd.GetRootCmd()
output, err = pdctl.ExecuteCommand(cmd2, "-u", pdAddr, "config", "placement-rules", "show", "--region=1", "--detail")
re.NoError(err)
re.NoError(json.Unmarshal(output, fit))
re.Len(fit.RuleFits, 3)
re.Equal([2]string{"pd", "default"}, fit.RuleFits[0].Rule.Key())
suite.checkShowRuleKey(pdAddr, [][2]string{{"pd", "default"}}, "--region=1", "--detail")

// test delete
// need clear up args, so create new a cobra.Command. Otherwise gourp still exists.
rules[0].Count = 0
b, _ = json.Marshal(rules)
os.WriteFile(fname, b, 0600)
_, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "save", "--in="+fname)
re.NoError(err)
output, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "show", "--group=pd")
re.NoError(err)
re.NoError(json.Unmarshal(output, &rules))
re.Len(rules, 1)
re.Equal([2]string{"pd", "test1"}, rules[0].Key())
suite.checkShowRuleKey(pdAddr, [][2]string{{"pd", "test1"}}, "--group=pd")
}

func (suite *configTestSuite) TestPlacementRuleGroups() {
Expand Down Expand Up @@ -431,32 +405,38 @@ func (suite *configTestSuite) checkPlacementRuleGroups(cluster *tests.TestCluste

// show all
var groups []placement.RuleGroup
output, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-group", "show")
re.NoError(err)
re.NoError(json.Unmarshal(output, &groups))
re.Equal([]placement.RuleGroup{
{ID: "pd", Index: 42, Override: true},
{ID: "group2", Index: 100, Override: false},
{ID: "group3", Index: 200, Override: false},
}, groups)
testutil.Eventually(re, func() bool { // wait for the config to be synced to the scheduling server
output, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-group", "show")
re.NoError(err)
re.NoError(json.Unmarshal(output, &groups))
return reflect.DeepEqual([]placement.RuleGroup{
{ID: "pd", Index: 42, Override: true},
{ID: "group2", Index: 100, Override: false},
{ID: "group3", Index: 200, Override: false},
}, groups)
})

// delete
output, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-group", "delete", "group2")
re.NoError(err)
re.Contains(string(output), "Delete group and rules successfully.")

// show again
output, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-group", "show", "group2")
re.NoError(err)
re.Contains(string(output), "404")
testutil.Eventually(re, func() bool { // wait for the config to be synced to the scheduling server
output, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-group", "show", "group2")
re.NoError(err)
return strings.Contains(string(output), "404")
})

// delete using regex
_, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-group", "delete", "--regexp", ".*3")
re.NoError(err)

_, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-group", "show", "group3")
re.NoError(err)
re.Contains(string(output), "404")
testutil.Eventually(re, func() bool { // wait for the config to be synced to the scheduling server
output, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-group", "show", "group3")
re.NoError(err)
return strings.Contains(string(output), "404")
})
}

func (suite *configTestSuite) TestPlacementRuleBundle() {
Expand Down Expand Up @@ -496,28 +476,19 @@ func (suite *configTestSuite) checkPlacementRuleBundle(cluster *tests.TestCluste
defer os.RemoveAll(fname)

// test load
var bundles []placement.GroupBundle
_, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-bundle", "load", "--out="+fname)
re.NoError(err)
b, _ := os.ReadFile(fname)
re.NoError(json.Unmarshal(b, &bundles))
re.Len(bundles, 1)
re.Equal(placement.GroupBundle{ID: "pd", Index: 0, Override: false, Rules: []*placement.Rule{{GroupID: "pd", ID: "default", Role: "voter", Count: 3}}}, bundles[0])
suite.checkLoadRuleBundle(pdAddr, fname, []placement.GroupBundle{
{ID: "pd", Index: 0, Override: false, Rules: []*placement.Rule{{GroupID: "pd", ID: "default", Role: "voter", Count: 3}}},
})

// test set
bundle.ID = "pe"
bundle.Rules[0].GroupID = "pe"
b, err = json.Marshal(bundle)
b, err := json.Marshal(bundle)
re.NoError(err)
re.NoError(os.WriteFile(fname, b, 0600))
_, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-bundle", "set", "--in="+fname)
re.NoError(err)

_, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-bundle", "load", "--out="+fname)
re.NoError(err)
b, _ = os.ReadFile(fname)
re.NoError(json.Unmarshal(b, &bundles))
assertBundles(re, bundles, []placement.GroupBundle{
suite.checkLoadRuleBundle(pdAddr, fname, []placement.GroupBundle{
{ID: "pd", Index: 0, Override: false, Rules: []*placement.Rule{{GroupID: "pd", ID: "default", Role: "voter", Count: 3}}},
{ID: "pe", Index: 0, Override: false, Rules: []*placement.Rule{{GroupID: "pe", ID: "default", Role: "voter", Count: 3}}},
})
Expand All @@ -526,11 +497,7 @@ func (suite *configTestSuite) checkPlacementRuleBundle(cluster *tests.TestCluste
_, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-bundle", "delete", "pd")
re.NoError(err)

_, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-bundle", "load", "--out="+fname)
re.NoError(err)
b, _ = os.ReadFile(fname)
re.NoError(json.Unmarshal(b, &bundles))
assertBundles(re, bundles, []placement.GroupBundle{
suite.checkLoadRuleBundle(pdAddr, fname, []placement.GroupBundle{
{ID: "pe", Index: 0, Override: false, Rules: []*placement.Rule{{GroupID: "pe", ID: "default", Role: "voter", Count: 3}}},
})

Expand All @@ -542,17 +509,18 @@ func (suite *configTestSuite) checkPlacementRuleBundle(cluster *tests.TestCluste
re.NoError(os.WriteFile(fname, b, 0600))
_, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-bundle", "set", "--in="+fname)
re.NoError(err)
suite.checkLoadRuleBundle(pdAddr, fname, []placement.GroupBundle{
{ID: "pe", Index: 0, Override: false, Rules: []*placement.Rule{{GroupID: "pe", ID: "default", Role: "voter", Count: 3}}},
{ID: "pf", Index: 0, Override: false, Rules: []*placement.Rule{{GroupID: "pf", ID: "default", Role: "voter", Count: 3}}},
})

_, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-bundle", "delete", "--regexp", ".*f")
re.NoError(err)

_, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-bundle", "load", "--out="+fname)
re.NoError(err)
b, _ = os.ReadFile(fname)
re.NoError(json.Unmarshal(b, &bundles))
assertBundles(re, bundles, []placement.GroupBundle{
bundles := []placement.GroupBundle{
{ID: "pe", Index: 0, Override: false, Rules: []*placement.Rule{{GroupID: "pe", ID: "default", Role: "voter", Count: 3}}},
})
}
suite.checkLoadRuleBundle(pdAddr, fname, bundles)

// test save
bundle.Rules = []*placement.Rule{{GroupID: "pf", ID: "default", Role: "voter", Count: 3}}
Expand All @@ -562,13 +530,7 @@ func (suite *configTestSuite) checkPlacementRuleBundle(cluster *tests.TestCluste
re.NoError(os.WriteFile(fname, b, 0600))
_, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-bundle", "save", "--in="+fname)
re.NoError(err)

_, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-bundle", "load", "--out="+fname)
re.NoError(err)
b, err = os.ReadFile(fname)
re.NoError(err)
re.NoError(json.Unmarshal(b, &bundles))
assertBundles(re, bundles, []placement.GroupBundle{
suite.checkLoadRuleBundle(pdAddr, fname, []placement.GroupBundle{
{ID: "pe", Index: 0, Override: false, Rules: []*placement.Rule{{GroupID: "pe", ID: "default", Role: "voter", Count: 3}}},
{ID: "pf", Index: 0, Override: false, Rules: []*placement.Rule{{GroupID: "pf", ID: "default", Role: "voter", Count: 3}}},
})
Expand All @@ -581,16 +543,67 @@ func (suite *configTestSuite) checkPlacementRuleBundle(cluster *tests.TestCluste
_, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-bundle", "save", "--in="+fname, "--partial")
re.NoError(err)

_, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-bundle", "load", "--out="+fname)
re.NoError(err)
b, err = os.ReadFile(fname)
re.NoError(err)
re.NoError(json.Unmarshal(b, &bundles))
assertBundles(re, bundles, []placement.GroupBundle{
suite.checkLoadRuleBundle(pdAddr, fname, []placement.GroupBundle{
{ID: "pf", Index: 0, Override: false, Rules: []*placement.Rule{{GroupID: "pf", ID: "default", Role: "voter", Count: 3}}},
})
}

func (suite *configTestSuite) checkLoadRuleBundle(pdAddr string, fname string, expectValues []placement.GroupBundle) {
var bundles []placement.GroupBundle
cmd := pdctlCmd.GetRootCmd()
testutil.Eventually(suite.Require(), func() bool { // wait for the config to be synced to the scheduling server
_, err := pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-bundle", "load", "--out="+fname)
suite.NoError(err)
b, _ := os.ReadFile(fname)
suite.NoError(json.Unmarshal(b, &bundles))
return len(bundles) == len(expectValues)
})
assertBundles(suite.Require(), bundles, expectValues)
}

func (suite *configTestSuite) checkLoadRule(pdAddr string, fname string, expectValues [][2]string) []placement.Rule {
var rules []placement.Rule
cmd := pdctlCmd.GetRootCmd()
testutil.Eventually(suite.Require(), func() bool { // wait for the config to be synced to the scheduling server
_, err := pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "load", "--out="+fname)
suite.NoError(err)
b, _ := os.ReadFile(fname)
suite.NoError(json.Unmarshal(b, &rules))
return len(rules) == len(expectValues)
})
for i, v := range expectValues {
suite.Equal(v, rules[i].Key())
}
return rules
}

func (suite *configTestSuite) checkShowRuleKey(pdAddr string, expectValues [][2]string, opts ...string) {
var rules []placement.Rule
var fit placement.RegionFit
cmd := pdctlCmd.GetRootCmd()
testutil.Eventually(suite.Require(), func() bool { // wait for the config to be synced to the scheduling server
args := []string{"-u", pdAddr, "config", "placement-rules", "show"}
output, err := pdctl.ExecuteCommand(cmd, append(args, opts...)...)
suite.NoError(err)
err = json.Unmarshal(output, &rules)
if err == nil {
return len(rules) == len(expectValues)
}
suite.NoError(json.Unmarshal(output, &fit))
return len(fit.RuleFits) != 0
})
if len(rules) != 0 {
for i, v := range expectValues {
suite.Equal(v, rules[i].Key())
}
}
if len(fit.RuleFits) != 0 {
for i, v := range expectValues {
suite.Equal(v, fit.RuleFits[i].Rule.Key())
}
}
}

func TestReplicationMode(t *testing.T) {
re := require.New(t)
ctx, cancel := context.WithCancel(context.Background())
Expand Down
Loading
Loading