From 073cc5e43271172880d7ed667f04fcd3bdb62430 Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Thu, 19 Dec 2024 17:55:19 +0800 Subject: [PATCH] test: make test stable after enable ms mode dynamic switch Signed-off-by: lhy1024 --- tests/integrations/client/client_test.go | 2 +- .../mcs/discovery/register_test.go | 6 ++--- .../mcs/keyspace/tso_keyspace_group_test.go | 16 ++++++------- tests/integrations/mcs/members/member_test.go | 14 +++++------ .../integrations/mcs/scheduling/rule_test.go | 7 +++++- .../mcs/scheduling/server_test.go | 24 +++++++++---------- tests/integrations/mcs/tso/api_test.go | 6 ++--- .../mcs/tso/keyspace_group_manager_test.go | 7 +++--- tests/integrations/mcs/tso/proxy_test.go | 2 +- tests/integrations/mcs/tso/server_test.go | 14 +++++------ tests/integrations/tso/client_test.go | 6 ++--- tests/scheduling_cluster.go | 11 ++++++--- tests/testutil.go | 8 ++++--- tests/tso_cluster.go | 9 ++++--- tools/pd-ctl/tests/config/config_test.go | 4 +++- .../tests/keyspace/keyspace_group_test.go | 10 ++++---- tools/pd-ctl/tests/keyspace/keyspace_test.go | 2 +- 17 files changed, 82 insertions(+), 66 deletions(-) diff --git a/tests/integrations/client/client_test.go b/tests/integrations/client/client_test.go index fadfb952e4c..cfd1a2fbdc0 100644 --- a/tests/integrations/client/client_test.go +++ b/tests/integrations/client/client_test.go @@ -344,7 +344,7 @@ func TestTSOFollowerProxyWithTSOService(t *testing.T) { pdLeaderServer := cluster.GetServer(leaderName) re.NoError(pdLeaderServer.BootstrapCluster()) backendEndpoints := pdLeaderServer.GetAddr() - tsoCluster, err := tests.NewTestTSOCluster(ctx, 2, backendEndpoints) + tsoCluster, err := tests.NewTestTSOCluster(ctx, 2, cluster) re.NoError(err) defer tsoCluster.Destroy() time.Sleep(100 * time.Millisecond) diff --git a/tests/integrations/mcs/discovery/register_test.go b/tests/integrations/mcs/discovery/register_test.go index da6fa158307..6687d17cf43 100644 --- a/tests/integrations/mcs/discovery/register_test.go +++ b/tests/integrations/mcs/discovery/register_test.go @@ -93,7 +93,7 @@ func (suite *serverRegisterTestSuite) checkServerRegister(serviceName string) { re.Equal(addr, returnedEntry.ServiceAddr) // test primary when only one server - expectedPrimary := tests.WaitForPrimaryServing(re, map[string]bs.Server{addr: s}) + expectedPrimary := tests.WaitForPrimaryServing(re, map[string]bs.Server{addr: s}, suite.cluster) primary, exist := suite.pdLeader.GetServer().GetServicePrimaryAddr(suite.ctx, serviceName) re.True(exist) re.Equal(expectedPrimary, primary) @@ -131,7 +131,7 @@ func (suite *serverRegisterTestSuite) checkServerPrimaryChange(serviceName strin serverMap[s.GetAddr()] = s } - expectedPrimary := tests.WaitForPrimaryServing(re, serverMap) + expectedPrimary := tests.WaitForPrimaryServing(re, serverMap, suite.cluster) primary, exist = suite.pdLeader.GetServer().GetServicePrimaryAddr(suite.ctx, serviceName) re.True(exist) re.Equal(expectedPrimary, primary) @@ -139,7 +139,7 @@ func (suite *serverRegisterTestSuite) checkServerPrimaryChange(serviceName strin serverMap[primary].Close() delete(serverMap, primary) - expectedPrimary = tests.WaitForPrimaryServing(re, serverMap) + expectedPrimary = tests.WaitForPrimaryServing(re, serverMap, suite.cluster) // test API server discovery client := suite.pdLeader.GetEtcdClient() endpoints, err := discovery.Discover(client, serviceName) diff --git a/tests/integrations/mcs/keyspace/tso_keyspace_group_test.go b/tests/integrations/mcs/keyspace/tso_keyspace_group_test.go index 44347b4757d..23f100dcd90 100644 --- a/tests/integrations/mcs/keyspace/tso_keyspace_group_test.go +++ b/tests/integrations/mcs/keyspace/tso_keyspace_group_test.go @@ -95,7 +95,7 @@ func (suite *keyspaceGroupTestSuite) TestAllocNodesUpdate() { cleanups = append(cleanups, cleanup) nodes[s.GetAddr()] = s } - tests.WaitForPrimaryServing(re, nodes) + tests.WaitForPrimaryServing(re, nodes, suite.cluster) // create a keyspace group. kgs := &handlers.CreateKeyspaceGroupParams{KeyspaceGroups: []*endpoint.KeyspaceGroup{ @@ -151,7 +151,7 @@ func (suite *keyspaceGroupTestSuite) TestAllocReplica() { cleanups = append(cleanups, cleanup) nodes[s.GetAddr()] = s } - tests.WaitForPrimaryServing(re, nodes) + tests.WaitForPrimaryServing(re, nodes, suite.cluster) // miss replica. id := 1 @@ -203,7 +203,7 @@ func (suite *keyspaceGroupTestSuite) TestAllocReplica() { s2, cleanup2 := tests.StartSingleTSOTestServer(suite.ctx, re, suite.backendEndpoints, tempurl.Alloc()) defer cleanup2() nodes[s2.GetAddr()] = s2 - tests.WaitForPrimaryServing(re, nodes) + tests.WaitForPrimaryServing(re, nodes, suite.cluster) params = &handlers.AllocNodesForKeyspaceGroupParams{ Replica: constant.DefaultKeyspaceGroupReplicaCount + 1, } @@ -252,7 +252,7 @@ func (suite *keyspaceGroupTestSuite) TestSetNodes() { nodes[s.GetAddr()] = s nodesList = append(nodesList, s.GetAddr()) } - tests.WaitForPrimaryServing(re, nodes) + tests.WaitForPrimaryServing(re, nodes, suite.cluster) // the keyspace group is not exist. id := 1 @@ -318,7 +318,7 @@ func (suite *keyspaceGroupTestSuite) TestDefaultKeyspaceGroup() { cleanups = append(cleanups, cleanup) nodes[s.GetAddr()] = s } - tests.WaitForPrimaryServing(re, nodes) + tests.WaitForPrimaryServing(re, nodes, suite.cluster) // the default keyspace group is exist. var kg *endpoint.KeyspaceGroup @@ -352,7 +352,7 @@ func (suite *keyspaceGroupTestSuite) TestAllocNodes() { cleanups = append(cleanups, cleanup) nodes[s.GetAddr()] = s } - tests.WaitForPrimaryServing(re, nodes) + tests.WaitForPrimaryServing(re, nodes, suite.cluster) // create a keyspace group. kgs := &handlers.CreateKeyspaceGroupParams{KeyspaceGroups: []*endpoint.KeyspaceGroup{ @@ -394,7 +394,7 @@ func (suite *keyspaceGroupTestSuite) TestAllocOneNode() { defer cleanupOldTSOserver() nodes[oldTSOServer.GetAddr()] = oldTSOServer - tests.WaitForPrimaryServing(re, nodes) + tests.WaitForPrimaryServing(re, nodes, suite.cluster) // create a keyspace group. kgs := &handlers.CreateKeyspaceGroupParams{KeyspaceGroups: []*endpoint.KeyspaceGroup{ @@ -421,7 +421,7 @@ func (suite *keyspaceGroupTestSuite) TestAllocOneNode() { defer cleanupNewTSOServer() nodes[newTSOServer.GetAddr()] = newTSOServer - tests.WaitForPrimaryServing(re, nodes) + tests.WaitForPrimaryServing(re, nodes, suite.cluster) // the member list will be updated testutil.Eventually(re, func() bool { diff --git a/tests/integrations/mcs/members/member_test.go b/tests/integrations/mcs/members/member_test.go index 28275849073..48545ac907b 100644 --- a/tests/integrations/mcs/members/member_test.go +++ b/tests/integrations/mcs/members/member_test.go @@ -84,7 +84,7 @@ func (suite *memberTestSuite) SetupTest() { cleanup() }) } - primary := tests.WaitForPrimaryServing(re, nodes) + primary := tests.WaitForPrimaryServing(re, nodes, suite.cluster) members := mustGetKeyspaceGroupMembers(re, nodes[primary].(*tso.Server)) // Get the tso nodes suite.tsoNodes = nodes @@ -104,7 +104,7 @@ func (suite *memberTestSuite) SetupTest() { cleanup() }) } - tests.WaitForPrimaryServing(re, nodes) + tests.WaitForPrimaryServing(re, nodes, suite.cluster) suite.schedulingNodes = nodes suite.cleanupFunc = append(suite.cleanupFunc, func() { @@ -171,7 +171,7 @@ func (suite *memberTestSuite) TestPrimaryWorkWhileOtherServerClose() { nodes[member.Name()].Close() } } - tests.WaitForPrimaryServing(re, nodes) + tests.WaitForPrimaryServing(re, nodes, suite.cluster) // primary should be same with before. curPrimary, err := suite.pdClient.GetMicroServicePrimary(suite.ctx, service) @@ -291,14 +291,14 @@ func (suite *memberTestSuite) TestCampaignPrimaryAfterTransfer() { re.Equal(http.StatusOK, resp.StatusCode) resp.Body.Close() - tests.WaitForPrimaryServing(re, nodes) + tests.WaitForPrimaryServing(re, nodes, suite.cluster) newPrimary, err = suite.pdClient.GetMicroServicePrimary(suite.ctx, service) re.NoError(err) re.NotEqual(primary, newPrimary) // Close primary to push other nodes campaign primary nodes[newPrimary].Close() - tests.WaitForPrimaryServing(re, nodes) + tests.WaitForPrimaryServing(re, nodes, suite.cluster) // Primary should be different with before anotherPrimary, err := suite.pdClient.GetMicroServicePrimary(suite.ctx, service) re.NoError(err) @@ -356,7 +356,7 @@ func (suite *memberTestSuite) TestTransferPrimaryWhileLeaseExpired() { // TODO: Add campaign times check in mcs to avoid frequent campaign re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/election/skipGrantLeader")) // Can still work after lease expired - tests.WaitForPrimaryServing(re, nodes) + tests.WaitForPrimaryServing(re, nodes, suite.cluster) } } @@ -413,7 +413,7 @@ func (suite *memberTestSuite) TestTransferPrimaryWhileLeaseExpiredAndServerDown( nodes[newPrimary].Close() re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/election/skipGrantLeader")) - tests.WaitForPrimaryServing(re, nodes) + tests.WaitForPrimaryServing(re, nodes, suite.cluster) // Primary should be different with before onlyPrimary, err := suite.pdClient.GetMicroServicePrimary(suite.ctx, service) re.NoError(err) diff --git a/tests/integrations/mcs/scheduling/rule_test.go b/tests/integrations/mcs/scheduling/rule_test.go index a137619afbf..0584f852da9 100644 --- a/tests/integrations/mcs/scheduling/rule_test.go +++ b/tests/integrations/mcs/scheduling/rule_test.go @@ -21,6 +21,8 @@ import ( "github.com/stretchr/testify/suite" + "github.com/pingcap/failpoint" + "github.com/tikv/pd/pkg/schedule/labeler" "github.com/tikv/pd/pkg/schedule/placement" "github.com/tikv/pd/pkg/utils/testutil" @@ -46,6 +48,7 @@ func TestRule(t *testing.T) { func (suite *ruleTestSuite) SetupSuite() { re := suite.Require() + re.NoError(failpoint.Enable("github.com/tikv/pd/server/cluster/highFrequencyClusterJobs", `return(true)`)) var err error suite.ctx, suite.cancel = context.WithCancel(context.Background()) @@ -63,12 +66,14 @@ func (suite *ruleTestSuite) SetupSuite() { func (suite *ruleTestSuite) TearDownSuite() { suite.cancel() suite.cluster.Destroy() + re := suite.Require() + re.NoError(failpoint.Disable("github.com/tikv/pd/server/cluster/highFrequencyClusterJobs")) } func (suite *ruleTestSuite) TestRuleWatch() { re := suite.Require() - tc, err := tests.NewTestSchedulingCluster(suite.ctx, 1, suite.backendEndpoint) + tc, err := tests.NewTestSchedulingCluster(suite.ctx, 1, suite.cluster) re.NoError(err) defer tc.Destroy() diff --git a/tests/integrations/mcs/scheduling/server_test.go b/tests/integrations/mcs/scheduling/server_test.go index d3850e4667c..ea1e9df0b50 100644 --- a/tests/integrations/mcs/scheduling/server_test.go +++ b/tests/integrations/mcs/scheduling/server_test.go @@ -91,7 +91,7 @@ func (suite *serverTestSuite) TearDownSuite() { func (suite *serverTestSuite) TestAllocID() { re := suite.Require() re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/mcs/scheduling/server/fastUpdateMember", `return(true)`)) - tc, err := tests.NewTestSchedulingCluster(suite.ctx, 1, suite.backendEndpoints) + tc, err := tests.NewTestSchedulingCluster(suite.ctx, 1, suite.cluster) re.NoError(err) defer tc.Destroy() tc.WaitForPrimaryServing(re) @@ -110,7 +110,7 @@ func (suite *serverTestSuite) TestAllocIDAfterLeaderChange() { err = pd2.Run() re.NotEmpty(suite.cluster.WaitLeader()) re.NoError(err) - tc, err := tests.NewTestSchedulingCluster(suite.ctx, 1, suite.backendEndpoints) + tc, err := tests.NewTestSchedulingCluster(suite.ctx, 1, suite.cluster) re.NoError(err) defer tc.Destroy() tc.WaitForPrimaryServing(re) @@ -138,7 +138,7 @@ func (suite *serverTestSuite) TestAllocIDAfterLeaderChange() { func (suite *serverTestSuite) TestPrimaryChange() { re := suite.Require() - tc, err := tests.NewTestSchedulingCluster(suite.ctx, 2, suite.backendEndpoints) + tc, err := tests.NewTestSchedulingCluster(suite.ctx, 2, suite.cluster) re.NoError(err) defer tc.Destroy() tc.WaitForPrimaryServing(re) @@ -164,7 +164,7 @@ func (suite *serverTestSuite) TestPrimaryChange() { func (suite *serverTestSuite) TestForwardStoreHeartbeat() { re := suite.Require() - tc, err := tests.NewTestSchedulingCluster(suite.ctx, 1, suite.backendEndpoints) + tc, err := tests.NewTestSchedulingCluster(suite.ctx, 1, suite.cluster) re.NoError(err) defer tc.Destroy() tc.WaitForPrimaryServing(re) @@ -225,7 +225,7 @@ func (suite *serverTestSuite) TestSchedulingServiceFallback() { return suite.pdLeader.GetServer().GetRaftCluster().IsSchedulingControllerRunning() }) - tc, err := tests.NewTestSchedulingCluster(suite.ctx, 1, suite.backendEndpoints) + tc, err := tests.NewTestSchedulingCluster(suite.ctx, 1, suite.cluster) re.NoError(err) defer tc.Destroy() tc.WaitForPrimaryServing(re) @@ -242,7 +242,7 @@ func (suite *serverTestSuite) TestSchedulingServiceFallback() { testutil.Eventually(re, func() bool { return suite.pdLeader.GetServer().GetRaftCluster().IsSchedulingControllerRunning() }) - tc1, err := tests.NewTestSchedulingCluster(suite.ctx, 1, suite.backendEndpoints) + tc1, err := tests.NewTestSchedulingCluster(suite.ctx, 1, suite.cluster) re.NoError(err) defer tc1.Destroy() tc1.WaitForPrimaryServing(re) @@ -278,7 +278,7 @@ func (suite *serverTestSuite) TestDisableSchedulingServiceFallback() { return suite.pdLeader.GetServer().GetRaftCluster().IsSchedulingControllerRunning() }) - tc, err := tests.NewTestSchedulingCluster(suite.ctx, 1, suite.backendEndpoints) + tc, err := tests.NewTestSchedulingCluster(suite.ctx, 1, suite.cluster) re.NoError(err) defer tc.Destroy() tc.WaitForPrimaryServing(re) @@ -302,7 +302,7 @@ func (suite *serverTestSuite) TestDisableSchedulingServiceFallback() { func (suite *serverTestSuite) TestSchedulerSync() { re := suite.Require() - tc, err := tests.NewTestSchedulingCluster(suite.ctx, 1, suite.backendEndpoints) + tc, err := tests.NewTestSchedulingCluster(suite.ctx, 1, suite.cluster) re.NoError(err) defer tc.Destroy() tc.WaitForPrimaryServing(re) @@ -422,7 +422,7 @@ func checkEvictLeaderStoreIDs(re *require.Assertions, sc *schedulers.Controller, func (suite *serverTestSuite) TestForwardRegionHeartbeat() { re := suite.Require() - tc, err := tests.NewTestSchedulingCluster(suite.ctx, 1, suite.backendEndpoints) + tc, err := tests.NewTestSchedulingCluster(suite.ctx, 1, suite.cluster) re.NoError(err) defer tc.Destroy() tc.WaitForPrimaryServing(re) @@ -499,7 +499,7 @@ func (suite *serverTestSuite) TestForwardRegionHeartbeat() { func (suite *serverTestSuite) TestStoreLimit() { re := suite.Require() - tc, err := tests.NewTestSchedulingCluster(suite.ctx, 1, suite.backendEndpoints) + tc, err := tests.NewTestSchedulingCluster(suite.ctx, 1, suite.cluster) re.NoError(err) defer tc.Destroy() tc.WaitForPrimaryServing(re) @@ -660,7 +660,7 @@ func (suite *multipleServerTestSuite) TestReElectLeader() { defer func() { re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/member/skipCampaignLeaderCheck")) }() - tc, err := tests.NewTestSchedulingCluster(suite.ctx, 1, suite.backendEndpoints) + tc, err := tests.NewTestSchedulingCluster(suite.ctx, 1, suite.cluster) re.NoError(err) defer tc.Destroy() tc.WaitForPrimaryServing(re) @@ -692,7 +692,7 @@ func (suite *multipleServerTestSuite) TestReElectLeader() { func (suite *serverTestSuite) TestOnlineProgress() { re := suite.Require() - tc, err := tests.NewTestSchedulingCluster(suite.ctx, 1, suite.backendEndpoints) + tc, err := tests.NewTestSchedulingCluster(suite.ctx, 1, suite.cluster) re.NoError(err) defer tc.Destroy() tc.WaitForPrimaryServing(re) diff --git a/tests/integrations/mcs/tso/api_test.go b/tests/integrations/mcs/tso/api_test.go index 91614530ef1..fdf525d3069 100644 --- a/tests/integrations/mcs/tso/api_test.go +++ b/tests/integrations/mcs/tso/api_test.go @@ -71,7 +71,7 @@ func (suite *tsoAPITestSuite) SetupTest() { pdLeaderServer := suite.pdCluster.GetServer(leaderName) re.NoError(pdLeaderServer.BootstrapCluster()) suite.backendEndpoints = pdLeaderServer.GetAddr() - suite.tsoCluster, err = tests.NewTestTSOCluster(suite.ctx, 1, suite.backendEndpoints) + suite.tsoCluster, err = tests.NewTestTSOCluster(suite.ctx, 1, suite.pdCluster) re.NoError(err) } @@ -148,7 +148,7 @@ func TestTSOServerStartFirst(t *testing.T) { clusterCh := make(chan *tests.TestTSOCluster) defer close(clusterCh) go func() { - tsoCluster, err := tests.NewTestTSOCluster(ctx, 2, addr) + tsoCluster, err := tests.NewTestTSOCluster(ctx, 2, apiCluster) re.NoError(err) primary := tsoCluster.WaitForDefaultPrimaryServing(re) re.NotNil(primary) @@ -206,7 +206,7 @@ func TestForwardOnlyTSONoScheduling(t *testing.T) { err = tc.RunInitialServers() re.NoError(err) pdAddr := tc.GetConfig().GetClientURL() - ttc, err := tests.NewTestTSOCluster(ctx, 2, pdAddr) + ttc, err := tests.NewTestTSOCluster(ctx, 2, tc) re.NoError(err) tc.WaitLeader() leaderServer := tc.GetLeaderServer() diff --git a/tests/integrations/mcs/tso/keyspace_group_manager_test.go b/tests/integrations/mcs/tso/keyspace_group_manager_test.go index 2c19f6588e5..4f7e84cd66b 100644 --- a/tests/integrations/mcs/tso/keyspace_group_manager_test.go +++ b/tests/integrations/mcs/tso/keyspace_group_manager_test.go @@ -90,7 +90,7 @@ func (suite *tsoKeyspaceGroupManagerTestSuite) SetupSuite() { re.NotEmpty(leaderName) suite.pdLeaderServer = suite.cluster.GetServer(leaderName) re.NoError(suite.pdLeaderServer.BootstrapCluster()) - suite.tsoCluster, err = tests.NewTestTSOCluster(suite.ctx, 2, suite.pdLeaderServer.GetAddr()) + suite.tsoCluster, err = tests.NewTestTSOCluster(suite.ctx, 2, suite.cluster) re.NoError(err) suite.allocator = mockid.NewIDAllocator() suite.allocator.SetBase(uint64(time.Now().Second())) @@ -544,7 +544,6 @@ func TestTwiceSplitKeyspaceGroup(t *testing.T) { } }) re.NoError(err) - pdAddr := tc.GetConfig().GetClientURL() // Start api server and tso server. err = tc.RunInitialServers() @@ -554,7 +553,7 @@ func TestTwiceSplitKeyspaceGroup(t *testing.T) { leaderServer := tc.GetLeaderServer() re.NoError(leaderServer.BootstrapCluster()) - tsoCluster, err := tests.NewTestTSOCluster(ctx, 2, pdAddr) + tsoCluster, err := tests.NewTestTSOCluster(ctx, 2, tc) re.NoError(err) defer tsoCluster.Destroy() tsoCluster.WaitForDefaultPrimaryServing(re) @@ -751,7 +750,7 @@ func TestGetTSOImmediately(t *testing.T) { leaderServer := tc.GetLeaderServer() re.NoError(leaderServer.BootstrapCluster()) - tsoCluster, err := tests.NewTestTSOCluster(ctx, 2, pdAddr) + tsoCluster, err := tests.NewTestTSOCluster(ctx, 2, tc) re.NoError(err) defer tsoCluster.Destroy() tsoCluster.WaitForDefaultPrimaryServing(re) diff --git a/tests/integrations/mcs/tso/proxy_test.go b/tests/integrations/mcs/tso/proxy_test.go index b564076c1f0..467780ea241 100644 --- a/tests/integrations/mcs/tso/proxy_test.go +++ b/tests/integrations/mcs/tso/proxy_test.go @@ -73,7 +73,7 @@ func (s *tsoProxyTestSuite) SetupSuite() { re.NoError(s.apiLeader.BootstrapCluster()) // Create a TSO cluster with 2 servers - s.tsoCluster, err = tests.NewTestTSOCluster(s.ctx, 2, s.backendEndpoints) + s.tsoCluster, err = tests.NewTestTSOCluster(s.ctx, 2, s.apiCluster) re.NoError(err) s.tsoCluster.WaitForDefaultPrimaryServing(re) diff --git a/tests/integrations/mcs/tso/server_test.go b/tests/integrations/mcs/tso/server_test.go index 09a199c2d52..deedba0f30c 100644 --- a/tests/integrations/mcs/tso/server_test.go +++ b/tests/integrations/mcs/tso/server_test.go @@ -281,7 +281,7 @@ func TestForwardTSORelated(t *testing.T) { leaderServer.SetMicroServiceConfig(*cfg) // Unable to use the tso-related interface without tso server suite.checkUnavailableTSO(re) - tc, err := tests.NewTestTSOCluster(suite.ctx, 1, suite.backendEndpoints) + tc, err := tests.NewTestTSOCluster(suite.ctx, 1, suite.cluster) re.NoError(err) defer tc.Destroy() tc.WaitForDefaultPrimaryServing(re) @@ -293,7 +293,7 @@ func TestForwardTSOWhenPrimaryChanged(t *testing.T) { suite := NewAPIServerForward(re) defer suite.ShutDown() - tc, err := tests.NewTestTSOCluster(suite.ctx, 2, suite.backendEndpoints) + tc, err := tests.NewTestTSOCluster(suite.ctx, 2, suite.cluster) re.NoError(err) defer tc.Destroy() tc.WaitForDefaultPrimaryServing(re) @@ -333,7 +333,7 @@ func TestResignTSOPrimaryForward(t *testing.T) { suite := NewAPIServerForward(re) defer suite.ShutDown() // TODO: test random kill primary with 3 nodes - tc, err := tests.NewTestTSOCluster(suite.ctx, 2, suite.backendEndpoints) + tc, err := tests.NewTestTSOCluster(suite.ctx, 2, suite.cluster) re.NoError(err) defer tc.Destroy() tc.WaitForDefaultPrimaryServing(re) @@ -359,7 +359,7 @@ func TestResignAPIPrimaryForward(t *testing.T) { suite := NewAPIServerForward(re) defer suite.ShutDown() - tc, err := tests.NewTestTSOCluster(suite.ctx, 2, suite.backendEndpoints) + tc, err := tests.NewTestTSOCluster(suite.ctx, 2, suite.cluster) re.NoError(err) defer tc.Destroy() tc.WaitForDefaultPrimaryServing(re) @@ -417,7 +417,7 @@ func TestForwardTSOUnexpectedToFollower3(t *testing.T) { func (suite *APIServerForward) checkForwardTSOUnexpectedToFollower(checkTSO func()) { re := suite.re - tc, err := tests.NewTestTSOCluster(suite.ctx, 2, suite.backendEndpoints) + tc, err := tests.NewTestTSOCluster(suite.ctx, 2, suite.cluster) re.NoError(err) tc.WaitForDefaultPrimaryServing(re) @@ -524,7 +524,7 @@ func (suite *CommonTestSuite) SetupSuite() { suite.backendEndpoints = suite.pdLeader.GetAddr() re.NoError(suite.pdLeader.BootstrapCluster()) - suite.tsoCluster, err = tests.NewTestTSOCluster(suite.ctx, 1, suite.backendEndpoints) + suite.tsoCluster, err = tests.NewTestTSOCluster(suite.ctx, 1, suite.cluster) re.NoError(err) suite.tsoCluster.WaitForDefaultPrimaryServing(re) suite.tsoDefaultPrimaryServer = suite.tsoCluster.GetPrimaryServer(constant.DefaultKeyspaceID, constant.DefaultKeyspaceGroupID) @@ -624,7 +624,7 @@ func TestTSOServiceSwitch(t *testing.T) { re.NoError(checkTSOMonotonic(ctx, pdClient, &globalLastTS, 10)) // Start TSO server - tsoCluster, err := tests.NewTestTSOCluster(ctx, 1, pdLeader.GetAddr()) + tsoCluster, err := tests.NewTestTSOCluster(ctx, 1, tc) re.NoError(err) tsoCluster.WaitForDefaultPrimaryServing(re) diff --git a/tests/integrations/tso/client_test.go b/tests/integrations/tso/client_test.go index 2cda9f8734f..f3709a35167 100644 --- a/tests/integrations/tso/client_test.go +++ b/tests/integrations/tso/client_test.go @@ -113,7 +113,7 @@ func (suite *tsoClientTestSuite) SetupSuite() { suite.keyspaceIDs = make([]uint32, 0) if !suite.legacy { - suite.tsoCluster, err = tests.NewTestTSOCluster(suite.ctx, 3, suite.backendEndpoints) + suite.tsoCluster, err = tests.NewTestTSOCluster(suite.ctx, 3, suite.cluster) re.NoError(err) suite.keyspaceGroups = []struct { @@ -517,7 +517,7 @@ func TestMixedTSODeployment(t *testing.T) { s, cleanup := tests.StartSingleTSOTestServer(ctx, re, backendEndpoints, tempurl.Alloc()) defer cleanup() - tests.WaitForPrimaryServing(re, map[string]bs.Server{s.GetAddr(): s}) + tests.WaitForPrimaryServing(re, map[string]bs.Server{s.GetAddr(): s}, cluster) ctx1, cancel1 := context.WithCancel(context.Background()) var wg sync.WaitGroup @@ -562,7 +562,7 @@ func TestUpgradingAPIandTSOClusters(t *testing.T) { defer pdClient.Close() // Create a TSO cluster which has 2 servers - tsoCluster, err := tests.NewTestTSOCluster(ctx, 2, backendEndpoints) + tsoCluster, err := tests.NewTestTSOCluster(ctx, 2, apiCluster) re.NoError(err) tsoCluster.WaitForDefaultPrimaryServing(re) // The TSO service should be eventually healthy diff --git a/tests/scheduling_cluster.go b/tests/scheduling_cluster.go index b5fc2429043..5aa1e220be9 100644 --- a/tests/scheduling_cluster.go +++ b/tests/scheduling_cluster.go @@ -22,6 +22,7 @@ import ( scheduling "github.com/tikv/pd/pkg/mcs/scheduling/server" sc "github.com/tikv/pd/pkg/mcs/scheduling/server/config" + "github.com/tikv/pd/pkg/mcs/utils/constant" "github.com/tikv/pd/pkg/schedule/schedulers" "github.com/tikv/pd/pkg/utils/tempurl" "github.com/tikv/pd/pkg/utils/testutil" @@ -31,17 +32,19 @@ import ( type TestSchedulingCluster struct { ctx context.Context + pd *TestCluster backendEndpoints string servers map[string]*scheduling.Server cleanupFuncs map[string]testutil.CleanupFunc } // NewTestSchedulingCluster creates a new scheduling test cluster. -func NewTestSchedulingCluster(ctx context.Context, initialServerCount int, backendEndpoints string) (tc *TestSchedulingCluster, err error) { +func NewTestSchedulingCluster(ctx context.Context, initialServerCount int, pd *TestCluster) (tc *TestSchedulingCluster, err error) { schedulers.Register() tc = &TestSchedulingCluster{ ctx: ctx, - backendEndpoints: backendEndpoints, + pd: pd, + backendEndpoints: pd.GetLeaderServer().GetAddr(), servers: make(map[string]*scheduling.Server, initialServerCount), cleanupFuncs: make(map[string]testutil.CleanupFunc, initialServerCount), } @@ -115,7 +118,9 @@ func (tc *TestSchedulingCluster) WaitForPrimaryServing(re *require.Assertions) * } return false }, testutil.WithWaitFor(10*time.Second), testutil.WithTickInterval(50*time.Millisecond)) - + testutil.Eventually(re, func() bool { + return tc.pd.GetLeaderServer().GetRaftCluster().IsServiceIndependent(constant.SchedulingServiceName) + }) return primary } diff --git a/tests/testutil.go b/tests/testutil.go index 4f2a6beb261..e50b6af9c43 100644 --- a/tests/testutil.go +++ b/tests/testutil.go @@ -178,7 +178,7 @@ func NewSchedulingTestServer(ctx context.Context, cfg *sc.Config) (*scheduling.S } // WaitForPrimaryServing waits for one of servers being elected to be the primary/leader -func WaitForPrimaryServing(re *require.Assertions, serverMap map[string]bs.Server) string { +func WaitForPrimaryServing(re *require.Assertions, serverMap map[string]bs.Server, pd *TestCluster) string { var primary string testutil.Eventually(re, func() bool { for name, s := range serverMap { @@ -189,7 +189,9 @@ func WaitForPrimaryServing(re *require.Assertions, serverMap map[string]bs.Serve } return false }, testutil.WithWaitFor(10*time.Second), testutil.WithTickInterval(50*time.Millisecond)) - + testutil.Eventually(re, func() bool { + return pd.GetLeaderServer().GetRaftCluster().IsServiceIndependent(constant.TSOServiceName) + }) return primary } @@ -389,7 +391,7 @@ func (s *SchedulingTestEnvironment) startCluster(m SchedulerMode) { re.NoError(leaderServer.BootstrapCluster()) leaderServer.GetRaftCluster().SetPrepared() // start scheduling cluster - tc, err := NewTestSchedulingCluster(ctx, 1, leaderServer.GetAddr()) + tc, err := NewTestSchedulingCluster(ctx, 1, cluster) re.NoError(err) tc.WaitForPrimaryServing(re) tc.GetPrimaryServer().GetCluster().SetPrepared() diff --git a/tests/tso_cluster.go b/tests/tso_cluster.go index a6e664abb5b..d49143cab94 100644 --- a/tests/tso_cluster.go +++ b/tests/tso_cluster.go @@ -35,16 +35,17 @@ import ( type TestTSOCluster struct { ctx context.Context + pd *TestCluster backendEndpoints string servers map[string]*tso.Server cleanupFuncs map[string]testutil.CleanupFunc } // NewTestTSOCluster creates a new TSO test cluster. -func NewTestTSOCluster(ctx context.Context, initialServerCount int, backendEndpoints string) (tc *TestTSOCluster, err error) { +func NewTestTSOCluster(ctx context.Context, initialServerCount int, pd *TestCluster) (tc *TestTSOCluster, err error) { tc = &TestTSOCluster{ ctx: ctx, - backendEndpoints: backendEndpoints, + backendEndpoints: pd.GetLeaderServer().GetAddr(), servers: make(map[string]*tso.Server, initialServerCount), cleanupFuncs: make(map[string]testutil.CleanupFunc, initialServerCount), } @@ -177,7 +178,9 @@ func (tc *TestTSOCluster) WaitForPrimaryServing(re *require.Assertions, keyspace } return false }, testutil.WithWaitFor(30*time.Second), testutil.WithTickInterval(100*time.Millisecond)) - + testutil.Eventually(re, func() bool { + return tc.pd.GetLeaderServer().GetRaftCluster().IsServiceIndependent(constant.TSOServiceName) + }) return primary } diff --git a/tools/pd-ctl/tests/config/config_test.go b/tools/pd-ctl/tests/config/config_test.go index cf9e4163457..b6c58fe2bc6 100644 --- a/tools/pd-ctl/tests/config/config_test.go +++ b/tools/pd-ctl/tests/config/config_test.go @@ -740,7 +740,9 @@ func (suite *configTestSuite) checkPlacementRuleBundle(cluster *pdTests.TestClus output, err = tests.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-bundle", "get", placement.DefaultGroupID) re.NoError(err) re.NoError(json.Unmarshal(output, &bundle)) - re.Equal(placement.GroupBundle{ID: placement.DefaultGroupID, Index: 0, Override: false, Rules: []*placement.Rule{{GroupID: placement.DefaultGroupID, ID: placement.DefaultRuleID, Role: placement.Voter, Count: 3}}}, bundle) + expect := placement.GroupBundle{ID: placement.DefaultGroupID, Index: 0, Override: false, Rules: []*placement.Rule{{GroupID: placement.DefaultGroupID, ID: placement.DefaultRuleID, Role: placement.Voter, Count: 3}}} + expect.Rules[0].CreateTimestamp = bundle.Rules[0].CreateTimestamp // skip create timestamp in mcs + re.Equal(expect, bundle) f, err := os.CreateTemp("", "pd_tests") re.NoError(err) diff --git a/tools/pd-ctl/tests/keyspace/keyspace_group_test.go b/tools/pd-ctl/tests/keyspace/keyspace_group_test.go index fca00f2fd3c..15e7dafac65 100644 --- a/tools/pd-ctl/tests/keyspace/keyspace_group_test.go +++ b/tools/pd-ctl/tests/keyspace/keyspace_group_test.go @@ -111,7 +111,7 @@ func TestSplitKeyspaceGroup(t *testing.T) { re.NoError(err) pdAddr := tc.GetConfig().GetClientURL() - ttc, err := pdTests.NewTestTSOCluster(ctx, 2, pdAddr) + ttc, err := pdTests.NewTestTSOCluster(ctx, 2, tc) re.NoError(err) defer ttc.Destroy() cmd := ctl.GetRootCmd() @@ -206,7 +206,7 @@ func TestSetNodeAndPriorityKeyspaceGroup(t *testing.T) { re.NoError(err) pdAddr := tc.GetConfig().GetClientURL() - ttc, err := pdTests.NewTestTSOCluster(ctx, 2, pdAddr) + ttc, err := pdTests.NewTestTSOCluster(ctx, 2, tc) re.NoError(err) defer ttc.Destroy() tsoAddrs := ttc.GetAddrs() @@ -310,7 +310,7 @@ func TestMergeKeyspaceGroup(t *testing.T) { re.NoError(err) pdAddr := tc.GetConfig().GetClientURL() - ttc, err := pdTests.NewTestTSOCluster(ctx, 2, pdAddr) + ttc, err := pdTests.NewTestTSOCluster(ctx, 2, tc) re.NoError(err) defer ttc.Destroy() cmd := ctl.GetRootCmd() @@ -429,7 +429,7 @@ func TestKeyspaceGroupState(t *testing.T) { re.NoError(err) pdAddr := tc.GetConfig().GetClientURL() - ttc, err := pdTests.NewTestTSOCluster(ctx, 2, pdAddr) + ttc, err := pdTests.NewTestTSOCluster(ctx, 2, tc) re.NoError(err) defer ttc.Destroy() cmd := ctl.GetRootCmd() @@ -520,7 +520,7 @@ func TestShowKeyspaceGroupPrimary(t *testing.T) { re.NoError(err) pdAddr := tc.GetConfig().GetClientURL() - ttc, err := pdTests.NewTestTSOCluster(ctx, 2, pdAddr) + ttc, err := pdTests.NewTestTSOCluster(ctx, 2, tc) re.NoError(err) defer ttc.Destroy() tsoAddrs := ttc.GetAddrs() diff --git a/tools/pd-ctl/tests/keyspace/keyspace_test.go b/tools/pd-ctl/tests/keyspace/keyspace_test.go index 23a1148cd66..7eb09cfe8ac 100644 --- a/tools/pd-ctl/tests/keyspace/keyspace_test.go +++ b/tools/pd-ctl/tests/keyspace/keyspace_test.go @@ -58,7 +58,7 @@ func TestKeyspace(t *testing.T) { re.NoError(err) pdAddr := tc.GetConfig().GetClientURL() - ttc, err := pdTests.NewTestTSOCluster(ctx, 2, pdAddr) + ttc, err := pdTests.NewTestTSOCluster(ctx, 2, tc) re.NoError(err) defer ttc.Destroy() cmd := ctl.GetRootCmd()