From 2c07c241114fe9afabd9927ecbee61c4252f2d8e Mon Sep 17 00:00:00 2001 From: Sparkle <1284531+baurine@users.noreply.github.com> Date: Wed, 8 Nov 2023 18:18:42 +0800 Subject: [PATCH 001/137] chore(dashboard): update tidb dashboard verstion to v2023.11.08.1 (#7339) close tikv/pd#7340 Signed-off-by: baurine <2008.hbl@gmail.com> --- go.mod | 2 +- go.sum | 4 ++-- tests/integrations/client/go.mod | 2 +- tests/integrations/client/go.sum | 4 ++-- tests/integrations/mcs/go.mod | 2 +- tests/integrations/mcs/go.sum | 4 ++-- tests/integrations/tso/go.mod | 2 +- tests/integrations/tso/go.sum | 4 ++-- 8 files changed, 12 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index e8da2542be2..0306d70f7a3 100644 --- a/go.mod +++ b/go.mod @@ -36,7 +36,7 @@ require ( github.com/pingcap/kvproto v0.0.0-20231018065736-c0689aded40c github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 - github.com/pingcap/tidb-dashboard v0.0.0-20231102083420-865955cd15d9 + github.com/pingcap/tidb-dashboard v0.0.0-20231108071238-7cb8b7ff0537 github.com/prometheus/client_golang v1.11.1 github.com/prometheus/common v0.26.0 github.com/sasha-s/go-deadlock v0.2.0 diff --git a/go.sum b/go.sum index 28e210ef1cd..fb178321864 100644 --- a/go.sum +++ b/go.sum @@ -446,8 +446,8 @@ github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8 github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 h1:QV6jqlfOkh8hqvEAgwBZa+4bSgO0EeKC7s5c6Luam2I= github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21/go.mod h1:QYnjfA95ZaMefyl1NO8oPtKeb8pYUdnDVhQgf+qdpjM= -github.com/pingcap/tidb-dashboard v0.0.0-20231102083420-865955cd15d9 h1:xIeaDUq2ItkYMIgpWXAYKC/N3hs8aurfFvvz79lhHYE= -github.com/pingcap/tidb-dashboard v0.0.0-20231102083420-865955cd15d9/go.mod h1:EZ90+V5S4TttbYag6oKZ3jcNKRwZe1Mc9vXwOt9JBYw= +github.com/pingcap/tidb-dashboard v0.0.0-20231108071238-7cb8b7ff0537 h1:wnHt7ETIB0vm+gbLx8QhcIEmRtrT4QlWlfpcI9vjxOk= +github.com/pingcap/tidb-dashboard v0.0.0-20231108071238-7cb8b7ff0537/go.mod h1:EZ90+V5S4TttbYag6oKZ3jcNKRwZe1Mc9vXwOt9JBYw= github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e h1:FBaTXU8C3xgt/drM58VHxojHo/QoG1oPsgWTGvaSpO4= github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= diff --git a/tests/integrations/client/go.mod b/tests/integrations/client/go.mod index b9b868cf8e3..a4aca195f3f 100644 --- a/tests/integrations/client/go.mod +++ b/tests/integrations/client/go.mod @@ -119,7 +119,7 @@ require ( github.com/pingcap/errcode v0.3.0 // indirect github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c // indirect github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 // indirect - github.com/pingcap/tidb-dashboard v0.0.0-20231102083420-865955cd15d9 // indirect + github.com/pingcap/tidb-dashboard v0.0.0-20231108071238-7cb8b7ff0537 // indirect github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect diff --git a/tests/integrations/client/go.sum b/tests/integrations/client/go.sum index 81fa6fd7b39..ef9c4d2a5f3 100644 --- a/tests/integrations/client/go.sum +++ b/tests/integrations/client/go.sum @@ -410,8 +410,8 @@ github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8 github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 h1:QV6jqlfOkh8hqvEAgwBZa+4bSgO0EeKC7s5c6Luam2I= github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21/go.mod h1:QYnjfA95ZaMefyl1NO8oPtKeb8pYUdnDVhQgf+qdpjM= -github.com/pingcap/tidb-dashboard v0.0.0-20231102083420-865955cd15d9 h1:xIeaDUq2ItkYMIgpWXAYKC/N3hs8aurfFvvz79lhHYE= -github.com/pingcap/tidb-dashboard v0.0.0-20231102083420-865955cd15d9/go.mod h1:EZ90+V5S4TttbYag6oKZ3jcNKRwZe1Mc9vXwOt9JBYw= +github.com/pingcap/tidb-dashboard v0.0.0-20231108071238-7cb8b7ff0537 h1:wnHt7ETIB0vm+gbLx8QhcIEmRtrT4QlWlfpcI9vjxOk= +github.com/pingcap/tidb-dashboard v0.0.0-20231108071238-7cb8b7ff0537/go.mod h1:EZ90+V5S4TttbYag6oKZ3jcNKRwZe1Mc9vXwOt9JBYw= github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e h1:FBaTXU8C3xgt/drM58VHxojHo/QoG1oPsgWTGvaSpO4= github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= diff --git a/tests/integrations/mcs/go.mod b/tests/integrations/mcs/go.mod index c2dfdbe96ef..f6df0eb4de0 100644 --- a/tests/integrations/mcs/go.mod +++ b/tests/integrations/mcs/go.mod @@ -119,7 +119,7 @@ require ( github.com/pingcap/errcode v0.3.0 // indirect github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c // indirect github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 // indirect - github.com/pingcap/tidb-dashboard v0.0.0-20231102083420-865955cd15d9 // indirect + github.com/pingcap/tidb-dashboard v0.0.0-20231108071238-7cb8b7ff0537 // indirect github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect diff --git a/tests/integrations/mcs/go.sum b/tests/integrations/mcs/go.sum index d1b0962ab55..fc1dc1bbea5 100644 --- a/tests/integrations/mcs/go.sum +++ b/tests/integrations/mcs/go.sum @@ -414,8 +414,8 @@ github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8 github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 h1:QV6jqlfOkh8hqvEAgwBZa+4bSgO0EeKC7s5c6Luam2I= github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21/go.mod h1:QYnjfA95ZaMefyl1NO8oPtKeb8pYUdnDVhQgf+qdpjM= -github.com/pingcap/tidb-dashboard v0.0.0-20231102083420-865955cd15d9 h1:xIeaDUq2ItkYMIgpWXAYKC/N3hs8aurfFvvz79lhHYE= -github.com/pingcap/tidb-dashboard v0.0.0-20231102083420-865955cd15d9/go.mod h1:EZ90+V5S4TttbYag6oKZ3jcNKRwZe1Mc9vXwOt9JBYw= +github.com/pingcap/tidb-dashboard v0.0.0-20231108071238-7cb8b7ff0537 h1:wnHt7ETIB0vm+gbLx8QhcIEmRtrT4QlWlfpcI9vjxOk= +github.com/pingcap/tidb-dashboard v0.0.0-20231108071238-7cb8b7ff0537/go.mod h1:EZ90+V5S4TttbYag6oKZ3jcNKRwZe1Mc9vXwOt9JBYw= github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e h1:FBaTXU8C3xgt/drM58VHxojHo/QoG1oPsgWTGvaSpO4= github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= diff --git a/tests/integrations/tso/go.mod b/tests/integrations/tso/go.mod index e5131f15d91..7e833943e6e 100644 --- a/tests/integrations/tso/go.mod +++ b/tests/integrations/tso/go.mod @@ -117,7 +117,7 @@ require ( github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c // indirect github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 // indirect github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 // indirect - github.com/pingcap/tidb-dashboard v0.0.0-20231102083420-865955cd15d9 // indirect + github.com/pingcap/tidb-dashboard v0.0.0-20231108071238-7cb8b7ff0537 // indirect github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect diff --git a/tests/integrations/tso/go.sum b/tests/integrations/tso/go.sum index 576c3e75765..65a7f3e3558 100644 --- a/tests/integrations/tso/go.sum +++ b/tests/integrations/tso/go.sum @@ -408,8 +408,8 @@ github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8 github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 h1:QV6jqlfOkh8hqvEAgwBZa+4bSgO0EeKC7s5c6Luam2I= github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21/go.mod h1:QYnjfA95ZaMefyl1NO8oPtKeb8pYUdnDVhQgf+qdpjM= -github.com/pingcap/tidb-dashboard v0.0.0-20231102083420-865955cd15d9 h1:xIeaDUq2ItkYMIgpWXAYKC/N3hs8aurfFvvz79lhHYE= -github.com/pingcap/tidb-dashboard v0.0.0-20231102083420-865955cd15d9/go.mod h1:EZ90+V5S4TttbYag6oKZ3jcNKRwZe1Mc9vXwOt9JBYw= +github.com/pingcap/tidb-dashboard v0.0.0-20231108071238-7cb8b7ff0537 h1:wnHt7ETIB0vm+gbLx8QhcIEmRtrT4QlWlfpcI9vjxOk= +github.com/pingcap/tidb-dashboard v0.0.0-20231108071238-7cb8b7ff0537/go.mod h1:EZ90+V5S4TttbYag6oKZ3jcNKRwZe1Mc9vXwOt9JBYw= github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e h1:FBaTXU8C3xgt/drM58VHxojHo/QoG1oPsgWTGvaSpO4= github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= From 0c352271d7413bdf6ac948e11f1a3fb905fe2ccb Mon Sep 17 00:00:00 2001 From: disksing Date: Fri, 10 Nov 2023 12:03:42 +0800 Subject: [PATCH 002/137] dr-autosync: add recover timeout (#6295) ref tikv/pd#4399 Signed-off-by: husharp --- pkg/replication/replication_mode.go | 15 +++++- pkg/replication/replication_mode_test.go | 60 ++++++++++++++++-------- server/config/config.go | 15 +++--- 3 files changed, 62 insertions(+), 28 deletions(-) diff --git a/pkg/replication/replication_mode.go b/pkg/replication/replication_mode.go index 30b34e4596a..9093f911901 100644 --- a/pkg/replication/replication_mode.go +++ b/pkg/replication/replication_mode.go @@ -212,6 +212,7 @@ const ( type drAutoSyncStatus struct { State string `json:"state,omitempty"` StateID uint64 `json:"state_id,omitempty"` + AsyncStartTime *time.Time `json:"async_start,omitempty"` RecoverStartTime *time.Time `json:"recover_start,omitempty"` TotalRegions int `json:"total_regions,omitempty"` SyncedRegions int `json:"synced_regions,omitempty"` @@ -262,7 +263,8 @@ func (m *ModeManager) drSwitchToAsyncWithLock(availableStores []uint64) error { log.Warn("failed to switch to async state", zap.String("replicate-mode", modeDRAutoSync), errs.ZapError(err)) return err } - dr := drAutoSyncStatus{State: drStateAsync, StateID: id, AvailableStores: availableStores} + now := time.Now() + dr := drAutoSyncStatus{State: drStateAsync, StateID: id, AvailableStores: availableStores, AsyncStartTime: &now} if err := m.storage.SaveReplicationStatus(modeDRAutoSync, dr); err != nil { log.Warn("failed to switch to async state", zap.String("replicate-mode", modeDRAutoSync), errs.ZapError(err)) return err @@ -272,6 +274,15 @@ func (m *ModeManager) drSwitchToAsyncWithLock(availableStores []uint64) error { return nil } +func (m *ModeManager) drDurationSinceAsyncStart() time.Duration { + m.RLock() + defer m.RUnlock() + if m.drAutoSync.AsyncStartTime == nil { + return 0 + } + return time.Since(*m.drAutoSync.AsyncStartTime) +} + func (m *ModeManager) drSwitchToSyncRecover() error { m.Lock() defer m.Unlock() @@ -477,7 +488,7 @@ func (m *ModeManager) tickUpdateState() { m.drSwitchToAsync(storeIDs[primaryUp]) } case drStateAsync: - if canSync { + if canSync && m.drDurationSinceAsyncStart() > m.config.DRAutoSync.WaitRecoverTimeout.Duration { m.drSwitchToSyncRecover() break } diff --git a/pkg/replication/replication_mode_test.go b/pkg/replication/replication_mode_test.go index e01fb7a0b9a..5cf9f1a1450 100644 --- a/pkg/replication/replication_mode_test.go +++ b/pkg/replication/replication_mode_test.go @@ -16,6 +16,7 @@ package replication import ( "context" + "encoding/json" "errors" "fmt" "testing" @@ -159,6 +160,20 @@ func newMockReplicator(ids []uint64) *mockFileReplicator { } } +func assertLastData(t *testing.T, data string, state string, stateID uint64, availableStores []uint64) { + type status struct { + State string `json:"state"` + StateID uint64 `json:"state_id"` + AvailableStores []uint64 `json:"available_stores"` + } + var s status + err := json.Unmarshal([]byte(data), &s) + require.NoError(t, err) + require.Equal(t, state, s.State) + require.Equal(t, stateID, s.StateID) + require.Equal(t, availableStores, s.AvailableStores) +} + func TestStateSwitch(t *testing.T) { re := require.New(t) ctx, cancel := context.WithCancel(context.Background()) @@ -190,7 +205,7 @@ func TestStateSwitch(t *testing.T) { stateID := rep.drAutoSync.StateID re.NotEqual(uint64(0), stateID) rep.tickReplicateStatus() - re.Equal(fmt.Sprintf(`{"state":"sync","state_id":%d}`, stateID), replicator.lastData[1]) + assertLastData(t, replicator.lastData[1], "sync", stateID, nil) assertStateIDUpdate := func() { re.NotEqual(stateID, rep.drAutoSync.StateID) stateID = rep.drAutoSync.StateID @@ -207,7 +222,7 @@ func TestStateSwitch(t *testing.T) { re.Equal(drStateAsyncWait, rep.drGetState()) assertStateIDUpdate() rep.tickReplicateStatus() - re.Equal(fmt.Sprintf(`{"state":"async_wait","state_id":%d,"available_stores":[1,2,3,4]}`, stateID), replicator.lastData[1]) + assertLastData(t, replicator.lastData[1], "async_wait", stateID, []uint64{1, 2, 3, 4}) re.False(rep.GetReplicationStatus().GetDrAutoSync().GetPauseRegionSplit()) conf.DRAutoSync.PauseRegionSplit = true @@ -218,7 +233,7 @@ func TestStateSwitch(t *testing.T) { rep.tickUpdateState() assertStateIDUpdate() rep.tickReplicateStatus() - re.Equal(fmt.Sprintf(`{"state":"async","state_id":%d,"available_stores":[1,2,3,4]}`, stateID), replicator.lastData[1]) + assertLastData(t, replicator.lastData[1], "async", stateID, []uint64{1, 2, 3, 4}) // add new store in dr zone. cluster.AddLabelsStore(5, 1, map[string]string{"zone": "zone2"}) @@ -268,18 +283,19 @@ func TestStateSwitch(t *testing.T) { rep.tickUpdateState() re.Equal(drStateAsyncWait, rep.drGetState()) assertStateIDUpdate() + rep.tickReplicateStatus() - re.Equal(fmt.Sprintf(`{"state":"async_wait","state_id":%d,"available_stores":[1,2,3,4]}`, stateID), replicator.lastData[1]) + assertLastData(t, replicator.lastData[1], "async_wait", stateID, []uint64{1, 2, 3, 4}) setStoreState(cluster, "down", "up", "up", "up", "down", "down") rep.tickUpdateState() assertStateIDUpdate() rep.tickReplicateStatus() - re.Equal(fmt.Sprintf(`{"state":"async_wait","state_id":%d,"available_stores":[2,3,4]}`, stateID), replicator.lastData[1]) + assertLastData(t, replicator.lastData[1], "async_wait", stateID, []uint64{2, 3, 4}) setStoreState(cluster, "up", "down", "up", "up", "down", "down") rep.tickUpdateState() assertStateIDUpdate() rep.tickReplicateStatus() - re.Equal(fmt.Sprintf(`{"state":"async_wait","state_id":%d,"available_stores":[1,3,4]}`, stateID), replicator.lastData[1]) + assertLastData(t, replicator.lastData[1], "async_wait", stateID, []uint64{1, 3, 4}) // async_wait -> async rep.tickUpdateState() @@ -291,26 +307,32 @@ func TestStateSwitch(t *testing.T) { rep.tickUpdateState() assertStateIDUpdate() rep.tickReplicateStatus() - re.Equal(fmt.Sprintf(`{"state":"async","state_id":%d,"available_stores":[1,3,4]}`, stateID), replicator.lastData[1]) + assertLastData(t, replicator.lastData[1], "async", stateID, []uint64{1, 3, 4}) // async -> async setStoreState(cluster, "up", "up", "up", "up", "down", "down") rep.tickUpdateState() // store 2 won't be available before it syncs status. rep.tickReplicateStatus() - re.Equal(fmt.Sprintf(`{"state":"async","state_id":%d,"available_stores":[1,3,4]}`, stateID), replicator.lastData[1]) + assertLastData(t, replicator.lastData[1], "async", stateID, []uint64{1, 3, 4}) syncStoreStatus(1, 2, 3, 4) rep.tickUpdateState() assertStateIDUpdate() rep.tickReplicateStatus() - re.Equal(fmt.Sprintf(`{"state":"async","state_id":%d,"available_stores":[1,2,3,4]}`, stateID), replicator.lastData[1]) + assertLastData(t, replicator.lastData[1], "async", stateID, []uint64{1, 2, 3, 4}) // async -> sync_recover setStoreState(cluster, "up", "up", "up", "up", "up", "up") rep.tickUpdateState() re.Equal(drStateSyncRecover, rep.drGetState()) assertStateIDUpdate() + rep.drSwitchToAsync([]uint64{1, 2, 3, 4, 5}) + rep.config.DRAutoSync.WaitRecoverTimeout = typeutil.NewDuration(time.Hour) + rep.tickUpdateState() + re.Equal(drStateAsync, rep.drGetState()) // wait recover timeout + + rep.config.DRAutoSync.WaitRecoverTimeout = typeutil.NewDuration(0) setStoreState(cluster, "down", "up", "up", "up", "up", "up") rep.tickUpdateState() re.Equal(drStateSyncRecover, rep.drGetState()) @@ -387,27 +409,27 @@ func TestReplicateState(t *testing.T) { stateID := rep.drAutoSync.StateID // replicate after initialized rep.tickReplicateStatus() - re.Equal(fmt.Sprintf(`{"state":"sync","state_id":%d}`, stateID), replicator.lastData[1]) + assertLastData(t, replicator.lastData[1], "sync", stateID, nil) // repliate state to new member replicator.memberIDs = append(replicator.memberIDs, 2, 3) rep.tickReplicateStatus() - re.Equal(fmt.Sprintf(`{"state":"sync","state_id":%d}`, stateID), replicator.lastData[2]) - re.Equal(fmt.Sprintf(`{"state":"sync","state_id":%d}`, stateID), replicator.lastData[3]) + assertLastData(t, replicator.lastData[2], "sync", stateID, nil) + assertLastData(t, replicator.lastData[3], "sync", stateID, nil) // inject error replicator.errors[2] = errors.New("failed to persist") rep.tickUpdateState() // switch async_wait since there is only one zone newStateID := rep.drAutoSync.StateID rep.tickReplicateStatus() - re.Equal(fmt.Sprintf(`{"state":"async_wait","state_id":%d,"available_stores":[1,2]}`, newStateID), replicator.lastData[1]) - re.Equal(fmt.Sprintf(`{"state":"sync","state_id":%d}`, stateID), replicator.lastData[2]) - re.Equal(fmt.Sprintf(`{"state":"async_wait","state_id":%d,"available_stores":[1,2]}`, newStateID), replicator.lastData[3]) + assertLastData(t, replicator.lastData[1], "async_wait", newStateID, []uint64{1, 2}) + assertLastData(t, replicator.lastData[2], "sync", stateID, nil) + assertLastData(t, replicator.lastData[3], "async_wait", newStateID, []uint64{1, 2}) // clear error, replicate to node 2 next time delete(replicator.errors, 2) rep.tickReplicateStatus() - re.Equal(fmt.Sprintf(`{"state":"async_wait","state_id":%d,"available_stores":[1,2]}`, newStateID), replicator.lastData[2]) + assertLastData(t, replicator.lastData[2], "async_wait", newStateID, []uint64{1, 2}) } func TestAsynctimeout(t *testing.T) { @@ -637,7 +659,7 @@ func TestComplexPlacementRules(t *testing.T) { rep.tickUpdateState() re.Equal(drStateAsyncWait, rep.drGetState()) rep.tickReplicateStatus() - re.Equal(fmt.Sprintf(`{"state":"async_wait","state_id":%d,"available_stores":[1,2,3,4,5,6]}`, rep.drAutoSync.StateID), replicator.lastData[1]) + assertLastData(t, replicator.lastData[1], "async_wait", rep.drAutoSync.StateID, []uint64{1, 2, 3, 4, 5, 6}) // reset to sync setStoreState(cluster, "up", "up", "up", "up", "up", "up", "up", "up", "up", "up") @@ -698,7 +720,7 @@ func TestComplexPlacementRules2(t *testing.T) { rep.tickUpdateState() re.Equal(drStateAsyncWait, rep.drGetState()) rep.tickReplicateStatus() - re.Equal(fmt.Sprintf(`{"state":"async_wait","state_id":%d,"available_stores":[1,2,3,4]}`, rep.drAutoSync.StateID), replicator.lastData[1]) + assertLastData(t, replicator.lastData[1], "async_wait", rep.drAutoSync.StateID, []uint64{1, 2, 3, 4}) } func TestComplexPlacementRules3(t *testing.T) { @@ -737,7 +759,7 @@ func TestComplexPlacementRules3(t *testing.T) { rep.tickUpdateState() re.Equal(drStateAsyncWait, rep.drGetState()) rep.tickReplicateStatus() - re.Equal(fmt.Sprintf(`{"state":"async_wait","state_id":%d,"available_stores":[1,2,3,4]}`, rep.drAutoSync.StateID), replicator.lastData[1]) + assertLastData(t, replicator.lastData[1], "async_wait", rep.drAutoSync.StateID, []uint64{1, 2, 3, 4}) } func genRegions(cluster *mockcluster.Cluster, stateID uint64, n int) []*core.RegionInfo { diff --git a/server/config/config.go b/server/config/config.go index 0485e077c67..da6b0e29e07 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -831,13 +831,14 @@ func NormalizeReplicationMode(m string) string { // DRAutoSyncReplicationConfig is the configuration for auto sync mode between 2 data centers. type DRAutoSyncReplicationConfig struct { - LabelKey string `toml:"label-key" json:"label-key"` - Primary string `toml:"primary" json:"primary"` - DR string `toml:"dr" json:"dr"` - PrimaryReplicas int `toml:"primary-replicas" json:"primary-replicas"` - DRReplicas int `toml:"dr-replicas" json:"dr-replicas"` - WaitStoreTimeout typeutil.Duration `toml:"wait-store-timeout" json:"wait-store-timeout"` - PauseRegionSplit bool `toml:"pause-region-split" json:"pause-region-split,string"` + LabelKey string `toml:"label-key" json:"label-key"` + Primary string `toml:"primary" json:"primary"` + DR string `toml:"dr" json:"dr"` + PrimaryReplicas int `toml:"primary-replicas" json:"primary-replicas"` + DRReplicas int `toml:"dr-replicas" json:"dr-replicas"` + WaitStoreTimeout typeutil.Duration `toml:"wait-store-timeout" json:"wait-store-timeout"` + WaitRecoverTimeout typeutil.Duration `toml:"wait-recover-timeout" json:"wait-recover-timeout"` + PauseRegionSplit bool `toml:"pause-region-split" json:"pause-region-split,string"` } func (c *DRAutoSyncReplicationConfig) adjust(meta *configutil.ConfigMetaData) { From f1cee6c3971e18c6ab201e50555261a8c51c3041 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Fri, 10 Nov 2023 15:48:43 +0800 Subject: [PATCH 003/137] mcs/resourcemanager: delete expire tokenSlot (#7344) close tikv/pd#7346 Signed-off-by: guo-shaoge --- .../resourcemanager/server/token_buckets.go | 23 +++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/pkg/mcs/resourcemanager/server/token_buckets.go b/pkg/mcs/resourcemanager/server/token_buckets.go index a0acba3b54d..05a93c32673 100644 --- a/pkg/mcs/resourcemanager/server/token_buckets.go +++ b/pkg/mcs/resourcemanager/server/token_buckets.go @@ -20,6 +20,8 @@ import ( "github.com/gogo/protobuf/proto" rmpb "github.com/pingcap/kvproto/pkg/resource_manager" + "github.com/pingcap/log" + "go.uber.org/zap" ) const ( @@ -31,6 +33,7 @@ const ( defaultReserveRatio = 0.5 defaultLoanCoefficient = 2 maxAssignTokens = math.MaxFloat64 / 1024 // assume max client connect is 1024 + slotExpireTimeout = 10 * time.Minute ) // GroupTokenBucket is a token bucket for a resource group. @@ -62,6 +65,7 @@ type TokenSlot struct { // tokenCapacity is the number of tokens in the slot. tokenCapacity float64 lastTokenCapacity float64 + lastReqTime time.Time } // GroupTokenBucketState is the running state of TokenBucket. @@ -75,7 +79,8 @@ type GroupTokenBucketState struct { LastUpdate *time.Time `json:"last_update,omitempty"` Initialized bool `json:"initialized"` // settingChanged is used to avoid that the number of tokens returned is jitter because of changing fill rate. - settingChanged bool + settingChanged bool + lastCheckExpireSlot time.Time } // Clone returns the copy of GroupTokenBucketState @@ -95,6 +100,7 @@ func (gts *GroupTokenBucketState) Clone() *GroupTokenBucketState { Initialized: gts.Initialized, tokenSlots: tokenSlots, clientConsumptionTokensSum: gts.clientConsumptionTokensSum, + lastCheckExpireSlot: gts.lastCheckExpireSlot, } } @@ -119,16 +125,18 @@ func (gts *GroupTokenBucketState) balanceSlotTokens( clientUniqueID uint64, settings *rmpb.TokenLimitSettings, requiredToken, elapseTokens float64) { + now := time.Now() slot, exist := gts.tokenSlots[clientUniqueID] if !exist { // Only slots that require a positive number will be considered alive, // but still need to allocate the elapsed tokens as well. if requiredToken != 0 { - slot = &TokenSlot{} + slot = &TokenSlot{lastReqTime: now} gts.tokenSlots[clientUniqueID] = slot gts.clientConsumptionTokensSum = 0 } } else { + slot.lastReqTime = now if gts.clientConsumptionTokensSum >= maxAssignTokens { gts.clientConsumptionTokensSum = 0 } @@ -139,6 +147,16 @@ func (gts *GroupTokenBucketState) balanceSlotTokens( } } + if time.Since(gts.lastCheckExpireSlot) >= slotExpireTimeout { + gts.lastCheckExpireSlot = now + for clientUniqueID, slot := range gts.tokenSlots { + if time.Since(slot.lastReqTime) >= slotExpireTimeout { + delete(gts.tokenSlots, clientUniqueID) + log.Info("delete resource group slot because expire", zap.Time("last-req-time", slot.lastReqTime), + zap.Any("expire timeout", slotExpireTimeout), zap.Any("del client id", clientUniqueID), zap.Any("len", len(gts.tokenSlots))) + } + } + } if len(gts.tokenSlots) == 0 { return } @@ -264,6 +282,7 @@ func (gtb *GroupTokenBucket) init(now time.Time, clientID uint64) { lastTokenCapacity: gtb.Tokens, } gtb.LastUpdate = &now + gtb.lastCheckExpireSlot = now gtb.Initialized = true } From b5119ea4bf2c3bc1d94256810c7e3e3670e96f45 Mon Sep 17 00:00:00 2001 From: lucasliang Date: Fri, 10 Nov 2023 16:32:12 +0800 Subject: [PATCH 004/137] scheduler: refine the interval of scheduling tick in evict-slow-trend-scheduler. (#7326) ref tikv/pd#7156 Implement the `GetNextInterval` for `evict-slow-trend-scheduler`, to refine the ticking interval. Default `GetNextInterval` is not appropriate for `evict-slow-trend-scheduler`, as it might delay the checking of other nodes' slowness status. This pr adjusts the ticking interval of the evict-slow-trend-scheduler to optimize its behavior. If a slow node is already identified as a candidate, the next interval is now set to be shorter, ensuring quicker subsequent scheduling. This refinement aims to decrease response time. Signed-off-by: lucasliang Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- pkg/schedule/schedulers/evict_slow_trend.go | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/pkg/schedule/schedulers/evict_slow_trend.go b/pkg/schedule/schedulers/evict_slow_trend.go index 3983e9c345d..f31ba420c97 100644 --- a/pkg/schedule/schedulers/evict_slow_trend.go +++ b/pkg/schedule/schedulers/evict_slow_trend.go @@ -108,8 +108,12 @@ func (conf *evictSlowTrendSchedulerConfig) getKeyRangesByID(id uint64) []core.Ke return []core.KeyRange{core.NewKeyRange("", "")} } +func (conf *evictSlowTrendSchedulerConfig) hasEvictedStores() bool { + return len(conf.EvictedStores) > 0 +} + func (conf *evictSlowTrendSchedulerConfig) evictedStore() uint64 { - if len(conf.EvictedStores) == 0 { + if !conf.hasEvictedStores() { return 0 } // If a candidate passes all checks and proved to be slow, it will be @@ -237,6 +241,19 @@ type evictSlowTrendScheduler struct { handler http.Handler } +func (s *evictSlowTrendScheduler) GetNextInterval(interval time.Duration) time.Duration { + var growthType intervalGrowthType + // If it already found a slow node as candidate, the next interval should be shorter + // to make the next scheduling as soon as possible. This adjustment will decrease the + // response time, as heartbeats from other nodes will be received and updated more quickly. + if s.conf.hasEvictedStores() { + growthType = zeroGrowth + } else { + growthType = exponentialGrowth + } + return intervalGrow(s.GetMinInterval(), MaxScheduleInterval, growthType) +} + func (s *evictSlowTrendScheduler) ServeHTTP(w http.ResponseWriter, r *http.Request) { s.handler.ServeHTTP(w, r) } From fe8a393e5cc898ab65c8d683b2f7aaa33252dfc1 Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Fri, 10 Nov 2023 16:45:42 +0800 Subject: [PATCH 005/137] mcs: tso service should not forward again (#7348) ref tikv/pd#5836 Signed-off-by: Ryan Leung Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- pkg/mcs/tso/server/grpc_service.go | 34 ------------------------------ pkg/mcs/tso/server/server.go | 3 --- 2 files changed, 37 deletions(-) diff --git a/pkg/mcs/tso/server/grpc_service.go b/pkg/mcs/tso/server/grpc_service.go index 40a308c72f8..9006faf49da 100644 --- a/pkg/mcs/tso/server/grpc_service.go +++ b/pkg/mcs/tso/server/grpc_service.go @@ -28,8 +28,6 @@ import ( bs "github.com/tikv/pd/pkg/basicserver" "github.com/tikv/pd/pkg/mcs/registry" "github.com/tikv/pd/pkg/utils/apiutil" - "github.com/tikv/pd/pkg/utils/grpcutil" - "github.com/tikv/pd/pkg/utils/tsoutil" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -88,21 +86,9 @@ func (s *Service) RegisterRESTHandler(userDefineHandlers map[string]http.Handler // Tso returns a stream of timestamps func (s *Service) Tso(stream tsopb.TSO_TsoServer) error { - var ( - doneCh chan struct{} - errCh chan error - ) ctx, cancel := context.WithCancel(stream.Context()) defer cancel() for { - // Prevent unnecessary performance overhead of the channel. - if errCh != nil { - select { - case err := <-errCh: - return errors.WithStack(err) - default: - } - } request, err := stream.Recv() if err == io.EOF { return nil @@ -111,26 +97,6 @@ func (s *Service) Tso(stream tsopb.TSO_TsoServer) error { return errors.WithStack(err) } - streamCtx := stream.Context() - forwardedHost := grpcutil.GetForwardedHost(streamCtx) - if !s.IsLocalRequest(forwardedHost) { - clientConn, err := s.GetDelegateClient(s.Context(), s.GetTLSConfig(), forwardedHost) - if err != nil { - return errors.WithStack(err) - } - - if errCh == nil { - doneCh = make(chan struct{}) - defer close(doneCh) - errCh = make(chan error) - } - - tsoProtoFactory := s.tsoProtoFactory - tsoRequest := tsoutil.NewTSOProtoRequest(forwardedHost, clientConn, request, stream) - s.tsoDispatcher.DispatchRequest(ctx, tsoRequest, tsoProtoFactory, doneCh, errCh) - continue - } - start := time.Now() // TSO uses leader lease to determine validity. No need to check leader here. if s.IsClosed() { diff --git a/pkg/mcs/tso/server/server.go b/pkg/mcs/tso/server/server.go index 16ef3216c62..1a2430477d8 100644 --- a/pkg/mcs/tso/server/server.go +++ b/pkg/mcs/tso/server/server.go @@ -78,9 +78,6 @@ type Server struct { service *Service keyspaceGroupManager *tso.KeyspaceGroupManager - // tsoDispatcher is used to dispatch the TSO requests to - // the corresponding forwarding TSO channels. - tsoDispatcher *tsoutil.TSODispatcher // tsoProtoFactory is the abstract factory for creating tso // related data structures defined in the tso grpc protocol tsoProtoFactory *tsoutil.TSOProtoFactory From afe6afccf9ddbf35c4210d40e00c6d69a030d3b3 Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Fri, 10 Nov 2023 17:09:42 +0800 Subject: [PATCH 006/137] mcs: support rules http interface in scheduling server (#7199) ref tikv/pd#5839 Signed-off-by: lhy1024 Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- errors.toml | 20 + pkg/errs/errno.go | 13 +- pkg/mcs/scheduling/server/apis/v1/api.go | 296 ++++++++++++- pkg/schedule/handler/handler.go | 43 ++ pkg/utils/apiutil/serverapi/middleware.go | 5 +- server/api/region_test.go | 12 +- server/api/rule.go | 296 ++++++++----- server/api/server.go | 25 ++ tests/integrations/mcs/scheduling/api_test.go | 102 ++++- tests/pdctl/config/config_test.go | 10 +- {server => tests/server}/api/rule_test.go | 390 ++++++++++++------ 11 files changed, 932 insertions(+), 280 deletions(-) rename {server => tests/server}/api/rule_test.go (67%) diff --git a/errors.toml b/errors.toml index 1d10d40d294..b6123058310 100644 --- a/errors.toml +++ b/errors.toml @@ -551,6 +551,11 @@ error = ''' build rule list failed, %s ''' +["PD:placement:ErrKeyFormat"] +error = ''' +key should be in hex format, %s +''' + ["PD:placement:ErrLoadRule"] error = ''' load rule failed @@ -561,11 +566,21 @@ error = ''' load rule group failed ''' +["PD:placement:ErrPlacementDisabled"] +error = ''' +placement rules feature is disabled +''' + ["PD:placement:ErrRuleContent"] error = ''' invalid rule content, %s ''' +["PD:placement:ErrRuleNotFound"] +error = ''' +rule not found +''' + ["PD:plugin:ErrLoadPlugin"] error = ''' failed to load plugin @@ -616,6 +631,11 @@ error = ''' region %v has abnormal peer ''' +["PD:region:ErrRegionInvalidID"] +error = ''' +invalid region id +''' + ["PD:region:ErrRegionNotAdjacent"] error = ''' two regions are not adjacent diff --git a/pkg/errs/errno.go b/pkg/errs/errno.go index e5bac8519be..b8a882cd187 100644 --- a/pkg/errs/errno.go +++ b/pkg/errs/errno.go @@ -102,6 +102,8 @@ var ( // region errors var ( + // ErrRegionInvalidID is error info for region id invalid. + ErrRegionInvalidID = errors.Normalize("invalid region id", errors.RFCCodeText("PD:region:ErrRegionInvalidID")) // ErrRegionNotAdjacent is error info for region not adjacent. ErrRegionNotAdjacent = errors.Normalize("two regions are not adjacent", errors.RFCCodeText("PD:region:ErrRegionNotAdjacent")) // ErrRegionNotFound is error info for region not found. @@ -153,10 +155,13 @@ var ( // placement errors var ( - ErrRuleContent = errors.Normalize("invalid rule content, %s", errors.RFCCodeText("PD:placement:ErrRuleContent")) - ErrLoadRule = errors.Normalize("load rule failed", errors.RFCCodeText("PD:placement:ErrLoadRule")) - ErrLoadRuleGroup = errors.Normalize("load rule group failed", errors.RFCCodeText("PD:placement:ErrLoadRuleGroup")) - ErrBuildRuleList = errors.Normalize("build rule list failed, %s", errors.RFCCodeText("PD:placement:ErrBuildRuleList")) + ErrRuleContent = errors.Normalize("invalid rule content, %s", errors.RFCCodeText("PD:placement:ErrRuleContent")) + ErrLoadRule = errors.Normalize("load rule failed", errors.RFCCodeText("PD:placement:ErrLoadRule")) + ErrLoadRuleGroup = errors.Normalize("load rule group failed", errors.RFCCodeText("PD:placement:ErrLoadRuleGroup")) + ErrBuildRuleList = errors.Normalize("build rule list failed, %s", errors.RFCCodeText("PD:placement:ErrBuildRuleList")) + ErrPlacementDisabled = errors.Normalize("placement rules feature is disabled", errors.RFCCodeText("PD:placement:ErrPlacementDisabled")) + ErrKeyFormat = errors.Normalize("key should be in hex format, %s", errors.RFCCodeText("PD:placement:ErrKeyFormat")) + ErrRuleNotFound = errors.Normalize("rule not found", errors.RFCCodeText("PD:placement:ErrRuleNotFound")) ) // region label errors diff --git a/pkg/mcs/scheduling/server/apis/v1/api.go b/pkg/mcs/scheduling/server/apis/v1/api.go index 47fdb95543f..172515d8620 100644 --- a/pkg/mcs/scheduling/server/apis/v1/api.go +++ b/pkg/mcs/scheduling/server/apis/v1/api.go @@ -15,6 +15,7 @@ package apis import ( + "encoding/hex" "net/http" "strconv" "sync" @@ -127,12 +128,6 @@ func (s *Service) RegisterAdminRouter() { router.DELETE("cache/regions/:id", deleteRegionCacheByID) } -// RegisterConfigRouter registers the router of the config handler. -func (s *Service) RegisterConfigRouter() { - router := s.root.Group("config") - router.GET("", getConfig) -} - // RegisterSchedulersRouter registers the router of the schedulers handler. func (s *Service) RegisterSchedulersRouter() { router := s.root.Group("schedulers") @@ -172,6 +167,32 @@ func (s *Service) RegisterOperatorsRouter() { router.GET("/records", getOperatorRecords) } +// RegisterConfigRouter registers the router of the config handler. +func (s *Service) RegisterConfigRouter() { + router := s.root.Group("config") + router.GET("", getConfig) + + rules := router.Group("rules") + rules.GET("", getAllRules) + rules.GET("/group/:group", getRuleByGroup) + rules.GET("/region/:region", getRulesByRegion) + rules.GET("/region/:region/detail", checkRegionPlacementRule) + rules.GET("/key/:key", getRulesByKey) + + // We cannot merge `/rule` and `/rules`, because we allow `group_id` to be "group", + // which is the same as the prefix of `/rules/group/:group`. + rule := router.Group("rule") + rule.GET("/:group/:id", getRuleByGroupAndID) + + groups := router.Group("rule_groups") + groups.GET("", getAllGroupConfigs) + groups.GET("/:id", getRuleGroupConfig) + + placementRule := router.Group("placement-rule") + placementRule.GET("", getPlacementRules) + placementRule.GET("/:group", getPlacementRuleByGroup) +} + // @Tags admin // @Summary Change the log level. // @Produce json @@ -671,3 +692,266 @@ func getHistoryHotRegions(c *gin.Context) { var res storage.HistoryHotRegions c.IndentedJSON(http.StatusOK, res) } + +// @Tags rule +// @Summary List all rules of cluster. +// @Produce json +// @Success 200 {array} placement.Rule +// @Failure 412 {string} string "Placement rules feature is disabled." +// @Failure 500 {string} string "PD server failed to proceed the request." +// @Router /config/rules [get] +func getAllRules(c *gin.Context) { + handler := c.MustGet(handlerKey).(*handler.Handler) + manager, err := handler.GetRuleManager() + if err == errs.ErrPlacementDisabled { + c.String(http.StatusPreconditionFailed, err.Error()) + return + } + if err != nil { + c.String(http.StatusInternalServerError, err.Error()) + return + } + rules := manager.GetAllRules() + c.IndentedJSON(http.StatusOK, rules) +} + +// @Tags rule +// @Summary List all rules of cluster by group. +// @Param group path string true "The name of group" +// @Produce json +// @Success 200 {array} placement.Rule +// @Failure 412 {string} string "Placement rules feature is disabled." +// @Failure 500 {string} string "PD server failed to proceed the request." +// @Router /config/rules/group/{group} [get] +func getRuleByGroup(c *gin.Context) { + handler := c.MustGet(handlerKey).(*handler.Handler) + manager, err := handler.GetRuleManager() + if err == errs.ErrPlacementDisabled { + c.String(http.StatusPreconditionFailed, err.Error()) + return + } + if err != nil { + c.String(http.StatusInternalServerError, err.Error()) + return + } + group := c.Param("group") + rules := manager.GetRulesByGroup(group) + c.IndentedJSON(http.StatusOK, rules) +} + +// @Tags rule +// @Summary List all rules of cluster by region. +// @Param id path integer true "Region Id" +// @Produce json +// @Success 200 {array} placement.Rule +// @Failure 400 {string} string "The input is invalid." +// @Failure 404 {string} string "The region does not exist." +// @Failure 412 {string} string "Placement rules feature is disabled." +// @Failure 500 {string} string "PD server failed to proceed the request." +// @Router /config/rules/region/{region} [get] +func getRulesByRegion(c *gin.Context) { + handler := c.MustGet(handlerKey).(*handler.Handler) + manager, err := handler.GetRuleManager() + if err == errs.ErrPlacementDisabled { + c.String(http.StatusPreconditionFailed, err.Error()) + return + } + if err != nil { + c.String(http.StatusInternalServerError, err.Error()) + return + } + regionStr := c.Param("region") + region, code, err := handler.PreCheckForRegion(regionStr) + if err != nil { + c.String(code, err.Error()) + return + } + rules := manager.GetRulesForApplyRegion(region) + c.IndentedJSON(http.StatusOK, rules) +} + +// @Tags rule +// @Summary List rules and matched peers related to the given region. +// @Param id path integer true "Region Id" +// @Produce json +// @Success 200 {object} placement.RegionFit +// @Failure 400 {string} string "The input is invalid." +// @Failure 404 {string} string "The region does not exist." +// @Failure 412 {string} string "Placement rules feature is disabled." +// @Failure 500 {string} string "PD server failed to proceed the request." +// @Router /config/rules/region/{region}/detail [get] +func checkRegionPlacementRule(c *gin.Context) { + handler := c.MustGet(handlerKey).(*handler.Handler) + regionStr := c.Param("region") + region, code, err := handler.PreCheckForRegion(regionStr) + if err != nil { + c.String(code, err.Error()) + return + } + regionFit, err := handler.CheckRegionPlacementRule(region) + if err == errs.ErrPlacementDisabled { + c.String(http.StatusPreconditionFailed, err.Error()) + return + } + if err != nil { + c.String(http.StatusInternalServerError, err.Error()) + return + } + c.IndentedJSON(http.StatusOK, regionFit) +} + +// @Tags rule +// @Summary List all rules of cluster by key. +// @Param key path string true "The name of key" +// @Produce json +// @Success 200 {array} placement.Rule +// @Failure 400 {string} string "The input is invalid." +// @Failure 412 {string} string "Placement rules feature is disabled." +// @Failure 500 {string} string "PD server failed to proceed the request." +// @Router /config/rules/key/{key} [get] +func getRulesByKey(c *gin.Context) { + handler := c.MustGet(handlerKey).(*handler.Handler) + manager, err := handler.GetRuleManager() + if err == errs.ErrPlacementDisabled { + c.String(http.StatusPreconditionFailed, err.Error()) + return + } + if err != nil { + c.String(http.StatusInternalServerError, err.Error()) + return + } + keyHex := c.Param("key") + key, err := hex.DecodeString(keyHex) + if err != nil { + c.String(http.StatusBadRequest, errs.ErrKeyFormat.Error()) + return + } + rules := manager.GetRulesByKey(key) + c.IndentedJSON(http.StatusOK, rules) +} + +// @Tags rule +// @Summary Get rule of cluster by group and id. +// @Param group path string true "The name of group" +// @Param id path string true "Rule Id" +// @Produce json +// @Success 200 {object} placement.Rule +// @Failure 404 {string} string "The rule does not exist." +// @Failure 412 {string} string "Placement rules feature is disabled." +// @Router /config/rule/{group}/{id} [get] +func getRuleByGroupAndID(c *gin.Context) { + handler := c.MustGet(handlerKey).(*handler.Handler) + manager, err := handler.GetRuleManager() + if err == errs.ErrPlacementDisabled { + c.String(http.StatusPreconditionFailed, err.Error()) + return + } + if err != nil { + c.String(http.StatusInternalServerError, err.Error()) + return + } + group, id := c.Param("group"), c.Param("id") + rule := manager.GetRule(group, id) + if rule == nil { + c.String(http.StatusNotFound, errs.ErrRuleNotFound.Error()) + return + } + c.IndentedJSON(http.StatusOK, rule) +} + +// @Tags rule +// @Summary List all rule group configs. +// @Produce json +// @Success 200 {array} placement.RuleGroup +// @Failure 412 {string} string "Placement rules feature is disabled." +// @Failure 500 {string} string "PD server failed to proceed the request." +// @Router /config/rule_groups [get] +func getAllGroupConfigs(c *gin.Context) { + handler := c.MustGet(handlerKey).(*handler.Handler) + manager, err := handler.GetRuleManager() + if err == errs.ErrPlacementDisabled { + c.String(http.StatusPreconditionFailed, err.Error()) + return + } + if err != nil { + c.String(http.StatusInternalServerError, err.Error()) + return + } + ruleGroups := manager.GetRuleGroups() + c.IndentedJSON(http.StatusOK, ruleGroups) +} + +// @Tags rule +// @Summary Get rule group config by group id. +// @Param id path string true "Group Id" +// @Produce json +// @Success 200 {object} placement.RuleGroup +// @Failure 404 {string} string "The RuleGroup does not exist." +// @Failure 412 {string} string "Placement rules feature is disabled." +// @Failure 500 {string} string "PD server failed to proceed the request." +// @Router /config/rule_groups/{id} [get] +func getRuleGroupConfig(c *gin.Context) { + handler := c.MustGet(handlerKey).(*handler.Handler) + manager, err := handler.GetRuleManager() + if err == errs.ErrPlacementDisabled { + c.String(http.StatusPreconditionFailed, err.Error()) + return + } + if err != nil { + c.String(http.StatusInternalServerError, err.Error()) + return + } + id := c.Param("id") + group := manager.GetRuleGroup(id) + if group == nil { + c.String(http.StatusNotFound, errs.ErrRuleNotFound.Error()) + return + } + c.IndentedJSON(http.StatusOK, group) +} + +// @Tags rule +// @Summary List all rules and groups configuration. +// @Produce json +// @Success 200 {array} placement.GroupBundle +// @Failure 412 {string} string "Placement rules feature is disabled." +// @Failure 500 {string} string "PD server failed to proceed the request." +// @Router /config/placement-rules [get] +func getPlacementRules(c *gin.Context) { + handler := c.MustGet(handlerKey).(*handler.Handler) + manager, err := handler.GetRuleManager() + if err == errs.ErrPlacementDisabled { + c.String(http.StatusPreconditionFailed, err.Error()) + return + } + if err != nil { + c.String(http.StatusInternalServerError, err.Error()) + return + } + bundles := manager.GetAllGroupBundles() + c.IndentedJSON(http.StatusOK, bundles) +} + +// @Tags rule +// @Summary Get group config and all rules belong to the group. +// @Param group path string true "The name of group" +// @Produce json +// @Success 200 {object} placement.GroupBundle +// @Failure 412 {string} string "Placement rules feature is disabled." +// @Failure 500 {string} string "PD server failed to proceed the request." +// @Router /config/placement-rules/{group} [get] +func getPlacementRuleByGroup(c *gin.Context) { + handler := c.MustGet(handlerKey).(*handler.Handler) + manager, err := handler.GetRuleManager() + if err == errs.ErrPlacementDisabled { + c.String(http.StatusPreconditionFailed, err.Error()) + return + } + if err != nil { + c.String(http.StatusInternalServerError, err.Error()) + return + } + g := c.Param("group") + group := manager.GetGroupBundle(g) + c.IndentedJSON(http.StatusOK, group) +} diff --git a/pkg/schedule/handler/handler.go b/pkg/schedule/handler/handler.go index 45b0eaf502f..3f9f4f96622 100644 --- a/pkg/schedule/handler/handler.go +++ b/pkg/schedule/handler/handler.go @@ -18,6 +18,7 @@ import ( "bytes" "encoding/hex" "net/http" + "strconv" "strings" "time" @@ -1061,3 +1062,45 @@ func (h *Handler) GetHotBuckets(regionIDs ...uint64) (HotBucketsResponse, error) } return ret, nil } + +// GetRuleManager returns the rule manager. +func (h *Handler) GetRuleManager() (*placement.RuleManager, error) { + c := h.GetCluster() + if c == nil { + return nil, errs.ErrNotBootstrapped + } + if !c.GetSharedConfig().IsPlacementRulesEnabled() { + return nil, errs.ErrPlacementDisabled + } + return c.GetRuleManager(), nil +} + +// PreCheckForRegion checks if the region is valid. +func (h *Handler) PreCheckForRegion(regionStr string) (*core.RegionInfo, int, error) { + c := h.GetCluster() + if c == nil { + return nil, http.StatusInternalServerError, errs.ErrNotBootstrapped.GenWithStackByArgs() + } + regionID, err := strconv.ParseUint(regionStr, 10, 64) + if err != nil { + return nil, http.StatusBadRequest, errs.ErrRegionInvalidID.FastGenByArgs() + } + region := c.GetRegion(regionID) + if region == nil { + return nil, http.StatusNotFound, errs.ErrRegionNotFound.FastGenByArgs(regionID) + } + return region, http.StatusOK, nil +} + +// CheckRegionPlacementRule checks if the region matches the placement rules. +func (h *Handler) CheckRegionPlacementRule(region *core.RegionInfo) (*placement.RegionFit, error) { + c := h.GetCluster() + if c == nil { + return nil, errs.ErrNotBootstrapped.GenWithStackByArgs() + } + manager, err := h.GetRuleManager() + if err != nil { + return nil, err + } + return manager.FitRegion(c, region), nil +} diff --git a/pkg/utils/apiutil/serverapi/middleware.go b/pkg/utils/apiutil/serverapi/middleware.go index 19438ad0f91..2bb742ccbba 100644 --- a/pkg/utils/apiutil/serverapi/middleware.go +++ b/pkg/utils/apiutil/serverapi/middleware.go @@ -117,6 +117,7 @@ func (h *redirector) matchMicroServiceRedirectRules(r *http.Request) (bool, stri r.URL.Path = strings.TrimRight(r.URL.Path, "/") for _, rule := range h.microserviceRedirectRules { if strings.HasPrefix(r.URL.Path, rule.matchPath) && slice.Contains(rule.matchMethods, r.Method) { + origin := r.URL.Path addr, ok := h.s.GetServicePrimaryAddr(r.Context(), rule.targetServiceName) if !ok || addr == "" { log.Warn("failed to get the service primary addr when trying to match redirect rules", @@ -134,8 +135,8 @@ func (h *redirector) matchMicroServiceRedirectRules(r *http.Request) (bool, stri } else { r.URL.Path = rule.targetPath } - log.Debug("redirect to micro service", zap.String("path", r.URL.Path), zap.String("target", addr), - zap.String("method", r.Method)) + log.Debug("redirect to micro service", zap.String("path", r.URL.Path), zap.String("origin-path", origin), + zap.String("target", addr), zap.String("method", r.Method)) return true, addr } } diff --git a/server/api/region_test.go b/server/api/region_test.go index a39a1e5c5fd..379fcf7d463 100644 --- a/server/api/region_test.go +++ b/server/api/region_test.go @@ -241,14 +241,14 @@ func (suite *regionTestSuite) TestRegions() { mustRegionHeartbeat(re, suite.svr, r) } url := fmt.Sprintf("%s/regions", suite.urlPrefix) - RegionsInfo := &RegionsInfo{} - err := tu.ReadGetJSON(re, testDialClient, url, RegionsInfo) + regionsInfo := &RegionsInfo{} + err := tu.ReadGetJSON(re, testDialClient, url, regionsInfo) suite.NoError(err) - suite.Len(regions, RegionsInfo.Count) - sort.Slice(RegionsInfo.Regions, func(i, j int) bool { - return RegionsInfo.Regions[i].ID < RegionsInfo.Regions[j].ID + suite.Len(regions, regionsInfo.Count) + sort.Slice(regionsInfo.Regions, func(i, j int) bool { + return regionsInfo.Regions[i].ID < regionsInfo.Regions[j].ID }) - for i, r := range RegionsInfo.Regions { + for i, r := range regionsInfo.Regions { suite.Equal(regions[i].ID, r.ID) suite.Equal(regions[i].ApproximateSize, r.ApproximateSize) suite.Equal(regions[i].ApproximateKeys, r.ApproximateKeys) diff --git a/server/api/rule.go b/server/api/rule.go index b3a720ece41..77aad42eb42 100644 --- a/server/api/rule.go +++ b/server/api/rule.go @@ -19,30 +19,26 @@ import ( "fmt" "net/http" "net/url" - "strconv" "github.com/gorilla/mux" - "github.com/pingcap/errors" - "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/errs" "github.com/tikv/pd/pkg/schedule/placement" "github.com/tikv/pd/pkg/utils/apiutil" "github.com/tikv/pd/server" - "github.com/tikv/pd/server/cluster" "github.com/unrolled/render" ) -var errPlacementDisabled = errors.New("placement rules feature is disabled") - type ruleHandler struct { + *server.Handler svr *server.Server rd *render.Render } func newRulesHandler(svr *server.Server, rd *render.Render) *ruleHandler { return &ruleHandler{ - svr: svr, - rd: rd, + Handler: svr.GetHandler(), + svr: svr, + rd: rd, } } @@ -51,14 +47,19 @@ func newRulesHandler(svr *server.Server, rd *render.Render) *ruleHandler { // @Produce json // @Success 200 {array} placement.Rule // @Failure 412 {string} string "Placement rules feature is disabled." +// @Failure 500 {string} string "PD server failed to proceed the request." // @Router /config/rules [get] func (h *ruleHandler) GetAllRules(w http.ResponseWriter, r *http.Request) { - cluster := getCluster(r) - if !cluster.GetOpts().IsPlacementRulesEnabled() { - h.rd.JSON(w, http.StatusPreconditionFailed, errPlacementDisabled.Error()) + manager, err := h.Handler.GetRuleManager() + if err == errs.ErrPlacementDisabled { + h.rd.JSON(w, http.StatusPreconditionFailed, err.Error()) + return + } + if err != nil { + h.rd.JSON(w, http.StatusInternalServerError, err.Error()) return } - rules := cluster.GetRuleManager().GetAllRules() + rules := manager.GetAllRules() h.rd.JSON(w, http.StatusOK, rules) } @@ -72,9 +73,13 @@ func (h *ruleHandler) GetAllRules(w http.ResponseWriter, r *http.Request) { // @Failure 500 {string} string "PD server failed to proceed the request." // @Router /config/rules [post] func (h *ruleHandler) SetAllRules(w http.ResponseWriter, r *http.Request) { - cluster := getCluster(r) - if !cluster.GetOpts().IsPlacementRulesEnabled() { - h.rd.JSON(w, http.StatusPreconditionFailed, errPlacementDisabled.Error()) + manager, err := h.Handler.GetRuleManager() + if err == errs.ErrPlacementDisabled { + h.rd.JSON(w, http.StatusPreconditionFailed, err.Error()) + return + } + if err != nil { + h.rd.JSON(w, http.StatusInternalServerError, err.Error()) return } var rules []*placement.Rule @@ -87,7 +92,7 @@ func (h *ruleHandler) SetAllRules(w http.ResponseWriter, r *http.Request) { return } } - if err := cluster.GetRuleManager().SetKeyType(h.svr.GetConfig().PDServerCfg.KeyType). + if err := manager.SetKeyType(h.svr.GetConfig().PDServerCfg.KeyType). SetRules(rules); err != nil { if errs.ErrRuleContent.Equal(err) || errs.ErrHexDecodingString.Equal(err) { h.rd.JSON(w, http.StatusBadRequest, err.Error()) @@ -105,15 +110,20 @@ func (h *ruleHandler) SetAllRules(w http.ResponseWriter, r *http.Request) { // @Produce json // @Success 200 {array} placement.Rule // @Failure 412 {string} string "Placement rules feature is disabled." +// @Failure 500 {string} string "PD server failed to proceed the request." // @Router /config/rules/group/{group} [get] func (h *ruleHandler) GetRuleByGroup(w http.ResponseWriter, r *http.Request) { - cluster := getCluster(r) - if !cluster.GetOpts().IsPlacementRulesEnabled() { - h.rd.JSON(w, http.StatusPreconditionFailed, errPlacementDisabled.Error()) + manager, err := h.Handler.GetRuleManager() + if err == errs.ErrPlacementDisabled { + h.rd.JSON(w, http.StatusPreconditionFailed, err.Error()) + return + } + if err != nil { + h.rd.JSON(w, http.StatusInternalServerError, err.Error()) return } group := mux.Vars(r)["group"] - rules := cluster.GetRuleManager().GetRulesByGroup(group) + rules := manager.GetRulesByGroup(group) h.rd.JSON(w, http.StatusOK, rules) } @@ -125,13 +135,25 @@ func (h *ruleHandler) GetRuleByGroup(w http.ResponseWriter, r *http.Request) { // @Failure 400 {string} string "The input is invalid." // @Failure 404 {string} string "The region does not exist." // @Failure 412 {string} string "Placement rules feature is disabled." +// @Failure 500 {string} string "PD server failed to proceed the request." // @Router /config/rules/region/{region} [get] func (h *ruleHandler) GetRulesByRegion(w http.ResponseWriter, r *http.Request) { - cluster, region := h.preCheckForRegionAndRule(w, r) - if cluster == nil || region == nil { + manager, err := h.Handler.GetRuleManager() + if err == errs.ErrPlacementDisabled { + h.rd.JSON(w, http.StatusPreconditionFailed, err.Error()) + return + } + if err != nil { + h.rd.JSON(w, http.StatusInternalServerError, err.Error()) return } - rules := cluster.GetRuleManager().GetRulesForApplyRegion(region) + regionStr := mux.Vars(r)["region"] + region, code, err := h.PreCheckForRegion(regionStr) + if err != nil { + h.rd.JSON(w, code, err.Error()) + return + } + rules := manager.GetRulesForApplyRegion(region) h.rd.JSON(w, http.StatusOK, rules) } @@ -143,34 +165,25 @@ func (h *ruleHandler) GetRulesByRegion(w http.ResponseWriter, r *http.Request) { // @Failure 400 {string} string "The input is invalid." // @Failure 404 {string} string "The region does not exist." // @Failure 412 {string} string "Placement rules feature is disabled." +// @Failure 500 {string} string "PD server failed to proceed the request." // @Router /config/rules/region/{region}/detail [get] func (h *ruleHandler) CheckRegionPlacementRule(w http.ResponseWriter, r *http.Request) { - cluster, region := h.preCheckForRegionAndRule(w, r) - if cluster == nil || region == nil { + regionStr := mux.Vars(r)["region"] + region, code, err := h.PreCheckForRegion(regionStr) + if err != nil { + h.rd.JSON(w, code, err.Error()) return } - regionFit := cluster.GetRuleManager().FitRegion(cluster, region) - h.rd.JSON(w, http.StatusOK, regionFit) -} - -func (h *ruleHandler) preCheckForRegionAndRule(w http.ResponseWriter, r *http.Request) (*cluster.RaftCluster, *core.RegionInfo) { - cluster := getCluster(r) - if !cluster.GetOpts().IsPlacementRulesEnabled() { - h.rd.JSON(w, http.StatusPreconditionFailed, errPlacementDisabled.Error()) - return cluster, nil + regionFit, err := h.Handler.CheckRegionPlacementRule(region) + if err == errs.ErrPlacementDisabled { + h.rd.JSON(w, http.StatusPreconditionFailed, err.Error()) + return } - regionStr := mux.Vars(r)["region"] - regionID, err := strconv.ParseUint(regionStr, 10, 64) if err != nil { - h.rd.JSON(w, http.StatusBadRequest, "invalid region id") - return cluster, nil - } - region := cluster.GetRegion(regionID) - if region == nil { - h.rd.JSON(w, http.StatusNotFound, errs.ErrRegionNotFound.FastGenByArgs(regionID).Error()) - return cluster, nil + h.rd.JSON(w, http.StatusInternalServerError, err.Error()) + return } - return cluster, region + h.rd.JSON(w, http.StatusOK, regionFit) } // @Tags rule @@ -180,20 +193,25 @@ func (h *ruleHandler) preCheckForRegionAndRule(w http.ResponseWriter, r *http.Re // @Success 200 {array} placement.Rule // @Failure 400 {string} string "The input is invalid." // @Failure 412 {string} string "Placement rules feature is disabled." +// @Failure 500 {string} string "PD server failed to proceed the request." // @Router /config/rules/key/{key} [get] func (h *ruleHandler) GetRulesByKey(w http.ResponseWriter, r *http.Request) { - cluster := getCluster(r) - if !cluster.GetOpts().IsPlacementRulesEnabled() { - h.rd.JSON(w, http.StatusPreconditionFailed, errPlacementDisabled.Error()) + manager, err := h.Handler.GetRuleManager() + if err == errs.ErrPlacementDisabled { + h.rd.JSON(w, http.StatusPreconditionFailed, err.Error()) + return + } + if err != nil { + h.rd.JSON(w, http.StatusInternalServerError, err.Error()) return } keyHex := mux.Vars(r)["key"] key, err := hex.DecodeString(keyHex) if err != nil { - h.rd.JSON(w, http.StatusBadRequest, "key should be in hex format") + h.rd.JSON(w, http.StatusBadRequest, errs.ErrKeyFormat.FastGenByArgs(err).Error()) return } - rules := cluster.GetRuleManager().GetRulesByKey(key) + rules := manager.GetRulesByKey(key) h.rd.JSON(w, http.StatusOK, rules) } @@ -207,15 +225,19 @@ func (h *ruleHandler) GetRulesByKey(w http.ResponseWriter, r *http.Request) { // @Failure 412 {string} string "Placement rules feature is disabled." // @Router /config/rule/{group}/{id} [get] func (h *ruleHandler) GetRuleByGroupAndID(w http.ResponseWriter, r *http.Request) { - cluster := getCluster(r) - if !cluster.GetOpts().IsPlacementRulesEnabled() { - h.rd.JSON(w, http.StatusPreconditionFailed, errPlacementDisabled.Error()) + manager, err := h.Handler.GetRuleManager() + if err == errs.ErrPlacementDisabled { + h.rd.JSON(w, http.StatusPreconditionFailed, err.Error()) + return + } + if err != nil { + h.rd.JSON(w, http.StatusInternalServerError, err.Error()) return } group, id := mux.Vars(r)["group"], mux.Vars(r)["id"] - rule := cluster.GetRuleManager().GetRule(group, id) + rule := manager.GetRule(group, id) if rule == nil { - h.rd.JSON(w, http.StatusNotFound, nil) + h.rd.JSON(w, http.StatusNotFound, errs.ErrRuleNotFound.Error()) return } h.rd.JSON(w, http.StatusOK, rule) @@ -232,21 +254,25 @@ func (h *ruleHandler) GetRuleByGroupAndID(w http.ResponseWriter, r *http.Request // @Failure 500 {string} string "PD server failed to proceed the request." // @Router /config/rule [post] func (h *ruleHandler) SetRule(w http.ResponseWriter, r *http.Request) { - cluster := getCluster(r) - if !cluster.GetOpts().IsPlacementRulesEnabled() { - h.rd.JSON(w, http.StatusPreconditionFailed, errPlacementDisabled.Error()) + manager, err := h.Handler.GetRuleManager() + if err == errs.ErrPlacementDisabled { + h.rd.JSON(w, http.StatusPreconditionFailed, err.Error()) + return + } + if err != nil { + h.rd.JSON(w, http.StatusInternalServerError, err.Error()) return } var rule placement.Rule if err := apiutil.ReadJSONRespondError(h.rd, w, r.Body, &rule); err != nil { return } - oldRule := cluster.GetRuleManager().GetRule(rule.GroupID, rule.ID) + oldRule := manager.GetRule(rule.GroupID, rule.ID) if err := h.syncReplicateConfigWithDefaultRule(&rule); err != nil { h.rd.JSON(w, http.StatusBadRequest, err.Error()) return } - if err := cluster.GetRuleManager().SetKeyType(h.svr.GetConfig().PDServerCfg.KeyType). + if err := manager.SetKeyType(h.svr.GetConfig().PDServerCfg.KeyType). SetRule(&rule); err != nil { if errs.ErrRuleContent.Equal(err) || errs.ErrHexDecodingString.Equal(err) { h.rd.JSON(w, http.StatusBadRequest, err.Error()) @@ -255,6 +281,7 @@ func (h *ruleHandler) SetRule(w http.ResponseWriter, r *http.Request) { } return } + cluster := getCluster(r) cluster.AddSuspectKeyRange(rule.StartKey, rule.EndKey) if oldRule != nil { cluster.AddSuspectKeyRange(oldRule.StartKey, oldRule.EndKey) @@ -285,18 +312,23 @@ func (h *ruleHandler) syncReplicateConfigWithDefaultRule(rule *placement.Rule) e // @Failure 500 {string} string "PD server failed to proceed the request." // @Router /config/rule/{group}/{id} [delete] func (h *ruleHandler) DeleteRuleByGroup(w http.ResponseWriter, r *http.Request) { - cluster := getCluster(r) - if !cluster.GetOpts().IsPlacementRulesEnabled() { - h.rd.JSON(w, http.StatusPreconditionFailed, errPlacementDisabled.Error()) + manager, err := h.Handler.GetRuleManager() + if err == errs.ErrPlacementDisabled { + h.rd.JSON(w, http.StatusPreconditionFailed, err.Error()) + return + } + if err != nil { + h.rd.JSON(w, http.StatusInternalServerError, err.Error()) return } group, id := mux.Vars(r)["group"], mux.Vars(r)["id"] - rule := cluster.GetRuleManager().GetRule(group, id) - if err := cluster.GetRuleManager().DeleteRule(group, id); err != nil { + rule := manager.GetRule(group, id) + if err := manager.DeleteRule(group, id); err != nil { h.rd.JSON(w, http.StatusInternalServerError, err.Error()) return } if rule != nil { + cluster := getCluster(r) cluster.AddSuspectKeyRange(rule.StartKey, rule.EndKey) } @@ -313,16 +345,20 @@ func (h *ruleHandler) DeleteRuleByGroup(w http.ResponseWriter, r *http.Request) // @Failure 500 {string} string "PD server failed to proceed the request." // @Router /config/rules/batch [post] func (h *ruleHandler) BatchRules(w http.ResponseWriter, r *http.Request) { - cluster := getCluster(r) - if !cluster.GetOpts().IsPlacementRulesEnabled() { - h.rd.JSON(w, http.StatusPreconditionFailed, errPlacementDisabled.Error()) + manager, err := h.Handler.GetRuleManager() + if err == errs.ErrPlacementDisabled { + h.rd.JSON(w, http.StatusPreconditionFailed, err.Error()) + return + } + if err != nil { + h.rd.JSON(w, http.StatusInternalServerError, err.Error()) return } var opts []placement.RuleOp if err := apiutil.ReadJSONRespondError(h.rd, w, r.Body, &opts); err != nil { return } - if err := cluster.GetRuleManager().SetKeyType(h.svr.GetConfig().PDServerCfg.KeyType). + if err := manager.SetKeyType(h.svr.GetConfig().PDServerCfg.KeyType). Batch(opts); err != nil { if errs.ErrRuleContent.Equal(err) || errs.ErrHexDecodingString.Equal(err) { h.rd.JSON(w, http.StatusBadRequest, err.Error()) @@ -341,15 +377,20 @@ func (h *ruleHandler) BatchRules(w http.ResponseWriter, r *http.Request) { // @Success 200 {object} placement.RuleGroup // @Failure 404 {string} string "The RuleGroup does not exist." // @Failure 412 {string} string "Placement rules feature is disabled." +// @Failure 500 {string} string "PD server failed to proceed the request." // @Router /config/rule_group/{id} [get] func (h *ruleHandler) GetGroupConfig(w http.ResponseWriter, r *http.Request) { - cluster := getCluster(r) - if !cluster.GetOpts().IsPlacementRulesEnabled() { - h.rd.JSON(w, http.StatusPreconditionFailed, errPlacementDisabled.Error()) + manager, err := h.Handler.GetRuleManager() + if err == errs.ErrPlacementDisabled { + h.rd.JSON(w, http.StatusPreconditionFailed, err.Error()) + return + } + if err != nil { + h.rd.JSON(w, http.StatusInternalServerError, err.Error()) return } id := mux.Vars(r)["id"] - group := cluster.GetRuleManager().GetRuleGroup(id) + group := manager.GetRuleGroup(id) if group == nil { h.rd.JSON(w, http.StatusNotFound, nil) return @@ -368,21 +409,26 @@ func (h *ruleHandler) GetGroupConfig(w http.ResponseWriter, r *http.Request) { // @Failure 500 {string} string "PD server failed to proceed the request." // @Router /config/rule_group [post] func (h *ruleHandler) SetGroupConfig(w http.ResponseWriter, r *http.Request) { - cluster := getCluster(r) - if !cluster.GetOpts().IsPlacementRulesEnabled() { - h.rd.JSON(w, http.StatusPreconditionFailed, errPlacementDisabled.Error()) + manager, err := h.Handler.GetRuleManager() + if err == errs.ErrPlacementDisabled { + h.rd.JSON(w, http.StatusPreconditionFailed, err.Error()) + return + } + if err != nil { + h.rd.JSON(w, http.StatusInternalServerError, err.Error()) return } var ruleGroup placement.RuleGroup if err := apiutil.ReadJSONRespondError(h.rd, w, r.Body, &ruleGroup); err != nil { return } - if err := cluster.GetRuleManager().SetRuleGroup(&ruleGroup); err != nil { + if err := manager.SetRuleGroup(&ruleGroup); err != nil { h.rd.JSON(w, http.StatusInternalServerError, err.Error()) return } - for _, r := range cluster.GetRuleManager().GetRulesByGroup(ruleGroup.ID) { - cluster.AddSuspectKeyRange(r.StartKey, r.EndKey) + cluster := getCluster(r) + for _, rule := range manager.GetRulesByGroup(ruleGroup.ID) { + cluster.AddSuspectKeyRange(rule.StartKey, rule.EndKey) } h.rd.JSON(w, http.StatusOK, "Update rule group successfully.") } @@ -396,18 +442,23 @@ func (h *ruleHandler) SetGroupConfig(w http.ResponseWriter, r *http.Request) { // @Failure 500 {string} string "PD server failed to proceed the request." // @Router /config/rule_group/{id} [delete] func (h *ruleHandler) DeleteGroupConfig(w http.ResponseWriter, r *http.Request) { - cluster := getCluster(r) - if !cluster.GetOpts().IsPlacementRulesEnabled() { - h.rd.JSON(w, http.StatusPreconditionFailed, errPlacementDisabled.Error()) + manager, err := h.Handler.GetRuleManager() + if err == errs.ErrPlacementDisabled { + h.rd.JSON(w, http.StatusPreconditionFailed, err.Error()) + return + } + if err != nil { + h.rd.JSON(w, http.StatusInternalServerError, err.Error()) return } id := mux.Vars(r)["id"] - err := cluster.GetRuleManager().DeleteRuleGroup(id) + err = manager.DeleteRuleGroup(id) if err != nil { h.rd.JSON(w, http.StatusInternalServerError, err.Error()) return } - for _, r := range cluster.GetRuleManager().GetRulesByGroup(id) { + cluster := getCluster(r) + for _, r := range manager.GetRulesByGroup(id) { cluster.AddSuspectKeyRange(r.StartKey, r.EndKey) } h.rd.JSON(w, http.StatusOK, "Delete rule group successfully.") @@ -418,14 +469,19 @@ func (h *ruleHandler) DeleteGroupConfig(w http.ResponseWriter, r *http.Request) // @Produce json // @Success 200 {array} placement.RuleGroup // @Failure 412 {string} string "Placement rules feature is disabled." +// @Failure 500 {string} string "PD server failed to proceed the request." // @Router /config/rule_groups [get] func (h *ruleHandler) GetAllGroupConfigs(w http.ResponseWriter, r *http.Request) { - cluster := getCluster(r) - if !cluster.GetOpts().IsPlacementRulesEnabled() { - h.rd.JSON(w, http.StatusPreconditionFailed, errPlacementDisabled.Error()) + manager, err := h.Handler.GetRuleManager() + if err == errs.ErrPlacementDisabled { + h.rd.JSON(w, http.StatusPreconditionFailed, err.Error()) return } - ruleGroups := cluster.GetRuleManager().GetRuleGroups() + if err != nil { + h.rd.JSON(w, http.StatusInternalServerError, err.Error()) + return + } + ruleGroups := manager.GetRuleGroups() h.rd.JSON(w, http.StatusOK, ruleGroups) } @@ -434,14 +490,19 @@ func (h *ruleHandler) GetAllGroupConfigs(w http.ResponseWriter, r *http.Request) // @Produce json // @Success 200 {array} placement.GroupBundle // @Failure 412 {string} string "Placement rules feature is disabled." +// @Failure 500 {string} string "PD server failed to proceed the request." // @Router /config/placement-rule [get] func (h *ruleHandler) GetPlacementRules(w http.ResponseWriter, r *http.Request) { - cluster := getCluster(r) - if !cluster.GetOpts().IsPlacementRulesEnabled() { - h.rd.JSON(w, http.StatusPreconditionFailed, errPlacementDisabled.Error()) + manager, err := h.Handler.GetRuleManager() + if err == errs.ErrPlacementDisabled { + h.rd.JSON(w, http.StatusPreconditionFailed, err.Error()) + return + } + if err != nil { + h.rd.JSON(w, http.StatusInternalServerError, err.Error()) return } - bundles := cluster.GetRuleManager().GetAllGroupBundles() + bundles := manager.GetAllGroupBundles() h.rd.JSON(w, http.StatusOK, bundles) } @@ -455,9 +516,13 @@ func (h *ruleHandler) GetPlacementRules(w http.ResponseWriter, r *http.Request) // @Failure 500 {string} string "PD server failed to proceed the request." // @Router /config/placement-rule [post] func (h *ruleHandler) SetPlacementRules(w http.ResponseWriter, r *http.Request) { - cluster := getCluster(r) - if !cluster.GetOpts().IsPlacementRulesEnabled() { - h.rd.JSON(w, http.StatusPreconditionFailed, errPlacementDisabled.Error()) + manager, err := h.Handler.GetRuleManager() + if err == errs.ErrPlacementDisabled { + h.rd.JSON(w, http.StatusPreconditionFailed, err.Error()) + return + } + if err != nil { + h.rd.JSON(w, http.StatusInternalServerError, err.Error()) return } var groups []placement.GroupBundle @@ -465,7 +530,7 @@ func (h *ruleHandler) SetPlacementRules(w http.ResponseWriter, r *http.Request) return } _, partial := r.URL.Query()["partial"] - if err := cluster.GetRuleManager().SetKeyType(h.svr.GetConfig().PDServerCfg.KeyType). + if err := manager.SetKeyType(h.svr.GetConfig().PDServerCfg.KeyType). SetAllGroupBundles(groups, !partial); err != nil { if errs.ErrRuleContent.Equal(err) || errs.ErrHexDecodingString.Equal(err) { h.rd.JSON(w, http.StatusBadRequest, err.Error()) @@ -483,14 +548,20 @@ func (h *ruleHandler) SetPlacementRules(w http.ResponseWriter, r *http.Request) // @Produce json // @Success 200 {object} placement.GroupBundle // @Failure 412 {string} string "Placement rules feature is disabled." +// @Failure 500 {string} string "PD server failed to proceed the request." // @Router /config/placement-rule/{group} [get] func (h *ruleHandler) GetPlacementRuleByGroup(w http.ResponseWriter, r *http.Request) { - cluster := getCluster(r) - if !cluster.GetOpts().IsPlacementRulesEnabled() { - h.rd.JSON(w, http.StatusPreconditionFailed, errPlacementDisabled.Error()) + manager, err := h.Handler.GetRuleManager() + if err == errs.ErrPlacementDisabled { + h.rd.JSON(w, http.StatusPreconditionFailed, err.Error()) return } - group := cluster.GetRuleManager().GetGroupBundle(mux.Vars(r)["group"]) + if err != nil { + h.rd.JSON(w, http.StatusInternalServerError, err.Error()) + return + } + g := mux.Vars(r)["group"] + group := manager.GetGroupBundle(g) h.rd.JSON(w, http.StatusOK, group) } @@ -502,21 +573,26 @@ func (h *ruleHandler) GetPlacementRuleByGroup(w http.ResponseWriter, r *http.Req // @Success 200 {string} string "Delete group and rules successfully." // @Failure 400 {string} string "Bad request." // @Failure 412 {string} string "Placement rules feature is disabled." +// @Failure 500 {string} string "PD server failed to proceed the request." // @Router /config/placement-rule [delete] func (h *ruleHandler) DeletePlacementRuleByGroup(w http.ResponseWriter, r *http.Request) { - cluster := getCluster(r) - if !cluster.GetOpts().IsPlacementRulesEnabled() { - h.rd.JSON(w, http.StatusPreconditionFailed, errPlacementDisabled.Error()) + manager, err := h.Handler.GetRuleManager() + if err == errs.ErrPlacementDisabled { + h.rd.JSON(w, http.StatusPreconditionFailed, err.Error()) + return + } + if err != nil { + h.rd.JSON(w, http.StatusInternalServerError, err.Error()) return } group := mux.Vars(r)["group"] - group, err := url.PathUnescape(group) + group, err = url.PathUnescape(group) if err != nil { h.rd.JSON(w, http.StatusBadRequest, err.Error()) return } _, regex := r.URL.Query()["regexp"] - if err := cluster.GetRuleManager().DeleteGroupBundle(group, regex); err != nil { + if err := manager.DeleteGroupBundle(group, regex); err != nil { h.rd.JSON(w, http.StatusInternalServerError, err.Error()) return } @@ -532,9 +608,13 @@ func (h *ruleHandler) DeletePlacementRuleByGroup(w http.ResponseWriter, r *http. // @Failure 500 {string} string "PD server failed to proceed the request." // @Router /config/placement-rule/{group} [post] func (h *ruleHandler) SetPlacementRuleByGroup(w http.ResponseWriter, r *http.Request) { - cluster := getCluster(r) - if !cluster.GetOpts().IsPlacementRulesEnabled() { - h.rd.JSON(w, http.StatusPreconditionFailed, errPlacementDisabled.Error()) + manager, err := h.Handler.GetRuleManager() + if err == errs.ErrPlacementDisabled { + h.rd.JSON(w, http.StatusPreconditionFailed, err.Error()) + return + } + if err != nil { + h.rd.JSON(w, http.StatusInternalServerError, err.Error()) return } groupID := mux.Vars(r)["group"] @@ -549,7 +629,7 @@ func (h *ruleHandler) SetPlacementRuleByGroup(w http.ResponseWriter, r *http.Req h.rd.JSON(w, http.StatusBadRequest, fmt.Sprintf("group id %s does not match request URI %s", group.ID, groupID)) return } - if err := cluster.GetRuleManager().SetKeyType(h.svr.GetConfig().PDServerCfg.KeyType). + if err := manager.SetKeyType(h.svr.GetConfig().PDServerCfg.KeyType). SetGroupBundle(group); err != nil { if errs.ErrRuleContent.Equal(err) || errs.ErrHexDecodingString.Equal(err) { h.rd.JSON(w, http.StatusBadRequest, err.Error()) diff --git a/server/api/server.go b/server/api/server.go index ae877b8407c..77a51eb04e5 100644 --- a/server/api/server.go +++ b/server/api/server.go @@ -84,6 +84,31 @@ func NewHandler(_ context.Context, svr *server.Server) (http.Handler, apiutil.AP scheapi.APIPathPrefix+"/hotspot", mcs.SchedulingServiceName, []string{http.MethodGet}), + serverapi.MicroserviceRedirectRule( + prefix+"/config/rules", + scheapi.APIPathPrefix+"/config/rules", + mcs.SchedulingServiceName, + []string{http.MethodGet}), + serverapi.MicroserviceRedirectRule( + prefix+"/config/rule/", + scheapi.APIPathPrefix+"/config/rule", + mcs.SchedulingServiceName, + []string{http.MethodGet}), + serverapi.MicroserviceRedirectRule( + prefix+"/config/rule_group/", + scheapi.APIPathPrefix+"/config/rule_groups", // Note: this is a typo in the original code + mcs.SchedulingServiceName, + []string{http.MethodGet}), + serverapi.MicroserviceRedirectRule( + prefix+"/config/rule_groups", + scheapi.APIPathPrefix+"/config/rule_groups", + mcs.SchedulingServiceName, + []string{http.MethodGet}), + serverapi.MicroserviceRedirectRule( + prefix+"/config/placement-rule", + scheapi.APIPathPrefix+"/config/placement-rule", + mcs.SchedulingServiceName, + []string{http.MethodGet}), // because the writing of all the meta information of the scheduling service is in the API server, // we should not post and delete the scheduler directly in the scheduling service. serverapi.MicroserviceRedirectRule( diff --git a/tests/integrations/mcs/scheduling/api_test.go b/tests/integrations/mcs/scheduling/api_test.go index 15c66ce5829..cfeaa4db033 100644 --- a/tests/integrations/mcs/scheduling/api_test.go +++ b/tests/integrations/mcs/scheduling/api_test.go @@ -15,10 +15,10 @@ import ( _ "github.com/tikv/pd/pkg/mcs/scheduling/server/apis/v1" "github.com/tikv/pd/pkg/mcs/scheduling/server/config" "github.com/tikv/pd/pkg/schedule/handler" + "github.com/tikv/pd/pkg/schedule/placement" "github.com/tikv/pd/pkg/statistics" "github.com/tikv/pd/pkg/storage" "github.com/tikv/pd/pkg/utils/apiutil" - "github.com/tikv/pd/pkg/utils/tempurl" "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/tests" ) @@ -43,7 +43,7 @@ func TestAPI(t *testing.T) { suite.Run(t, &apiTestSuite{}) } -func (suite *apiTestSuite) SetupSuite() { +func (suite *apiTestSuite) SetupTest() { ctx, cancel := context.WithCancel(context.Background()) suite.ctx = ctx cluster, err := tests.NewTestAPICluster(suite.ctx, 1) @@ -62,14 +62,19 @@ func (suite *apiTestSuite) SetupSuite() { suite.cleanupFunc = func() { cancel() } + tc, err := tests.NewTestSchedulingCluster(suite.ctx, 2, suite.backendEndpoints) + suite.NoError(err) + suite.cluster.SetSchedulingCluster(tc) + tc.WaitForPrimaryServing(suite.Require()) } -func (suite *apiTestSuite) TearDownSuite() { +func (suite *apiTestSuite) TearDownTest() { suite.cluster.Destroy() suite.cleanupFunc() } func (suite *apiTestSuite) TestGetCheckerByName() { + re := suite.Require() testCases := []struct { name string }{ @@ -81,14 +86,8 @@ func (suite *apiTestSuite) TestGetCheckerByName() { {name: "joint-state"}, } - re := suite.Require() - s, cleanup := tests.StartSingleSchedulingTestServer(suite.ctx, re, suite.backendEndpoints, tempurl.Alloc()) - defer cleanup() - testutil.Eventually(re, func() bool { - return s.IsServing() - }, testutil.WithWaitFor(5*time.Second), testutil.WithTickInterval(50*time.Millisecond)) - addr := s.GetAddr() - urlPrefix := fmt.Sprintf("%s/scheduling/api/v1/checkers", addr) + s := suite.cluster.GetSchedulingPrimaryServer() + urlPrefix := fmt.Sprintf("%s/scheduling/api/v1/checkers", s.GetAddr()) co := s.GetCoordinator() for _, testCase := range testCases { @@ -123,17 +122,12 @@ func (suite *apiTestSuite) TestAPIForward() { re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/utils/apiutil/serverapi/checkHeader")) }() - tc, err := tests.NewTestSchedulingCluster(suite.ctx, 2, suite.backendEndpoints) - re.NoError(err) - defer tc.Destroy() - tc.WaitForPrimaryServing(re) - urlPrefix := fmt.Sprintf("%s/pd/api/v1", suite.backendEndpoints) var slice []string var resp map[string]interface{} // Test opeartor - err = testutil.ReadGetJSON(re, testDialClient, fmt.Sprintf("%s/%s", urlPrefix, "operators"), &slice, + err := testutil.ReadGetJSON(re, testDialClient, fmt.Sprintf("%s/%s", urlPrefix, "operators"), &slice, testutil.WithHeader(re, apiutil.ForwardToMicroServiceHeader, "true")) re.NoError(err) re.Len(slice, 0) @@ -241,6 +235,80 @@ func (suite *apiTestSuite) TestAPIForward() { err = testutil.ReadGetJSON(re, testDialClient, fmt.Sprintf("%s/%s", urlPrefix, "hotspot/regions/history"), &history, testutil.WithHeader(re, apiutil.ForwardToMicroServiceHeader, "true")) re.NoError(err) + + // Test rules: only forward `GET` request + var rules []*placement.Rule + tests.MustPutRegion(re, suite.cluster, 2, 1, []byte("a"), []byte("b"), core.SetApproximateSize(60)) + rules = []*placement.Rule{ + { + GroupID: "pd", + ID: "default", + Role: "voter", + Count: 3, + LocationLabels: []string{}, + }, + } + rulesArgs, err := json.Marshal(rules) + suite.NoError(err) + + err = testutil.ReadGetJSON(re, testDialClient, fmt.Sprintf("%s/%s", urlPrefix, "/config/rules"), &rules, + testutil.WithHeader(re, apiutil.ForwardToMicroServiceHeader, "true")) + re.NoError(err) + err = testutil.CheckPostJSON(testDialClient, fmt.Sprintf("%s/%s", urlPrefix, "config/rules"), rulesArgs, + testutil.WithoutHeader(re, apiutil.ForwardToMicroServiceHeader)) + re.NoError(err) + err = testutil.CheckPostJSON(testDialClient, fmt.Sprintf("%s/%s", urlPrefix, "config/rules/batch"), rulesArgs, + testutil.WithoutHeader(re, apiutil.ForwardToMicroServiceHeader)) + re.NoError(err) + err = testutil.ReadGetJSON(re, testDialClient, fmt.Sprintf("%s/%s", urlPrefix, "config/rules/group/pd"), &rules, + testutil.WithHeader(re, apiutil.ForwardToMicroServiceHeader, "true")) + re.NoError(err) + err = testutil.ReadGetJSON(re, testDialClient, fmt.Sprintf("%s/%s", urlPrefix, "config/rules/region/2"), &rules, + testutil.WithHeader(re, apiutil.ForwardToMicroServiceHeader, "true")) + re.NoError(err) + var fit placement.RegionFit + err = testutil.ReadGetJSON(re, testDialClient, fmt.Sprintf("%s/%s", urlPrefix, "config/rules/region/2/detail"), &fit, + testutil.WithHeader(re, apiutil.ForwardToMicroServiceHeader, "true")) + re.NoError(err) + err = testutil.ReadGetJSON(re, testDialClient, fmt.Sprintf("%s/%s", urlPrefix, "config/rules/key/0000000000000001"), &rules, + testutil.WithHeader(re, apiutil.ForwardToMicroServiceHeader, "true")) + re.NoError(err) + err = testutil.CheckGetJSON(testDialClient, fmt.Sprintf("%s/%s", urlPrefix, "config/rule/pd/2"), nil, + testutil.WithHeader(re, apiutil.ForwardToMicroServiceHeader, "true")) + re.NoError(err) + err = testutil.CheckDelete(testDialClient, fmt.Sprintf("%s/%s", urlPrefix, "config/rule/pd/2"), + testutil.WithoutHeader(re, apiutil.ForwardToMicroServiceHeader)) + re.NoError(err) + err = testutil.CheckPostJSON(testDialClient, fmt.Sprintf("%s/%s", urlPrefix, "config/rule"), rulesArgs, + testutil.WithoutHeader(re, apiutil.ForwardToMicroServiceHeader)) + re.NoError(err) + err = testutil.CheckGetJSON(testDialClient, fmt.Sprintf("%s/%s", urlPrefix, "config/rule_group/pd"), nil, + testutil.WithHeader(re, apiutil.ForwardToMicroServiceHeader, "true")) + re.NoError(err) + err = testutil.CheckDelete(testDialClient, fmt.Sprintf("%s/%s", urlPrefix, "config/rule_group/pd"), + testutil.WithoutHeader(re, apiutil.ForwardToMicroServiceHeader)) + re.NoError(err) + err = testutil.CheckPostJSON(testDialClient, fmt.Sprintf("%s/%s", urlPrefix, "config/rule_group"), rulesArgs, + testutil.WithoutHeader(re, apiutil.ForwardToMicroServiceHeader)) + re.NoError(err) + err = testutil.CheckGetJSON(testDialClient, fmt.Sprintf("%s/%s", urlPrefix, "config/rule_groups"), nil, + testutil.WithHeader(re, apiutil.ForwardToMicroServiceHeader, "true")) + re.NoError(err) + err = testutil.CheckGetJSON(testDialClient, fmt.Sprintf("%s/%s", urlPrefix, "config/placement-rule"), nil, + testutil.WithHeader(re, apiutil.ForwardToMicroServiceHeader, "true")) + re.NoError(err) + err = testutil.CheckPostJSON(testDialClient, fmt.Sprintf("%s/%s", urlPrefix, "config/placement-rule"), rulesArgs, + testutil.WithoutHeader(re, apiutil.ForwardToMicroServiceHeader)) + re.NoError(err) + err = testutil.CheckGetJSON(testDialClient, fmt.Sprintf("%s/%s", urlPrefix, "config/placement-rule/pd"), nil, + testutil.WithHeader(re, apiutil.ForwardToMicroServiceHeader, "true")) + re.NoError(err) + err = testutil.CheckDelete(testDialClient, fmt.Sprintf("%s/%s", urlPrefix, "config/placement-rule/pd"), + testutil.WithoutHeader(re, apiutil.ForwardToMicroServiceHeader)) + re.NoError(err) + err = testutil.CheckPostJSON(testDialClient, fmt.Sprintf("%s/%s", urlPrefix, "config/placement-rule/pd"), rulesArgs, + testutil.WithoutHeader(re, apiutil.ForwardToMicroServiceHeader)) + re.NoError(err) } func (suite *apiTestSuite) TestConfig() { diff --git a/tests/pdctl/config/config_test.go b/tests/pdctl/config/config_test.go index 26d70bb955f..2cc8427911a 100644 --- a/tests/pdctl/config/config_test.go +++ b/tests/pdctl/config/config_test.go @@ -19,6 +19,7 @@ import ( "encoding/json" "os" "reflect" + "strings" "testing" "time" @@ -409,9 +410,12 @@ func (suite *configTestSuite) checkPlacementRuleGroups(cluster *tests.TestCluste // test show var group placement.RuleGroup - output, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-group", "show", "pd") - re.NoError(err) - re.NoError(json.Unmarshal(output, &group)) + testutil.Eventually(re, func() bool { // wait for the config to be synced to the scheduling server + output, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-group", "show", "pd") + re.NoError(err) + return !strings.Contains(string(output), "404") + }) + re.NoError(json.Unmarshal(output, &group), string(output)) re.Equal(placement.RuleGroup{ID: "pd"}, group) // test set diff --git a/server/api/rule_test.go b/tests/server/api/rule_test.go similarity index 67% rename from server/api/rule_test.go rename to tests/server/api/rule_test.go index d2dc50f1119..3ee3357e031 100644 --- a/server/api/rule_test.go +++ b/tests/server/api/rule_test.go @@ -25,57 +25,37 @@ import ( "github.com/pingcap/kvproto/pkg/metapb" "github.com/stretchr/testify/suite" "github.com/tikv/pd/pkg/core" + "github.com/tikv/pd/pkg/errs" "github.com/tikv/pd/pkg/schedule/placement" tu "github.com/tikv/pd/pkg/utils/testutil" - "github.com/tikv/pd/server" "github.com/tikv/pd/server/config" + "github.com/tikv/pd/tests" ) type ruleTestSuite struct { suite.Suite - svr *server.Server - cleanup tu.CleanupFunc - urlPrefix string } func TestRuleTestSuite(t *testing.T) { suite.Run(t, new(ruleTestSuite)) } -func (suite *ruleTestSuite) SetupSuite() { - re := suite.Require() - suite.svr, suite.cleanup = mustNewServer(re) - server.MustWaitLeader(re, []*server.Server{suite.svr}) - - addr := suite.svr.GetAddr() - suite.urlPrefix = fmt.Sprintf("%s%s/api/v1/config", addr, apiPrefix) - - mustBootstrapCluster(re, suite.svr) - PDServerCfg := suite.svr.GetConfig().PDServerCfg - PDServerCfg.KeyType = "raw" - err := suite.svr.SetPDServerConfig(PDServerCfg) - suite.NoError(err) - suite.NoError(tu.CheckPostJSON(testDialClient, suite.urlPrefix, []byte(`{"enable-placement-rules":"true"}`), tu.StatusOK(re))) -} - -func (suite *ruleTestSuite) TearDownSuite() { - suite.cleanup() -} - -func (suite *ruleTestSuite) TearDownTest() { - def := placement.GroupBundle{ - ID: "pd", - Rules: []*placement.Rule{ - {GroupID: "pd", ID: "default", Role: "voter", Count: 3}, +func (suite *ruleTestSuite) TestSet() { + opts := []tests.ConfigOption{ + func(conf *config.Config, serverName string) { + conf.PDServerCfg.KeyType = "raw" + conf.Replication.EnablePlacementRules = true }, } - data, err := json.Marshal([]placement.GroupBundle{def}) - suite.NoError(err) - err = tu.CheckPostJSON(testDialClient, suite.urlPrefix+"/placement-rule", data, tu.StatusOK(suite.Require())) - suite.NoError(err) + env := tests.NewSchedulingTestEnvironment(suite.T(), opts...) + env.RunTestInTwoModes(suite.checkSet) } -func (suite *ruleTestSuite) TestSet() { +func (suite *ruleTestSuite) checkSet(cluster *tests.TestCluster) { + leaderServer := cluster.GetLeaderServer() + pdAddr := leaderServer.GetAddr() + urlPrefix := fmt.Sprintf("%s%s/api/v1/config", pdAddr, apiPrefix) + rule := placement.Rule{GroupID: "a", ID: "10", StartKeyHex: "1111", EndKeyHex: "3333", Role: "voter", Count: 1} successData, err := json.Marshal(rule) suite.NoError(err) @@ -159,12 +139,12 @@ func (suite *ruleTestSuite) TestSet() { for _, testCase := range testCases { suite.T().Log(testCase.name) // clear suspect keyRanges to prevent test case from others - suite.svr.GetRaftCluster().ClearSuspectKeyRanges() + leaderServer.GetRaftCluster().ClearSuspectKeyRanges() if testCase.success { - err = tu.CheckPostJSON(testDialClient, suite.urlPrefix+"/rule", testCase.rawData, tu.StatusOK(re)) + err = tu.CheckPostJSON(testDialClient, urlPrefix+"/rule", testCase.rawData, tu.StatusOK(re)) popKeyRangeMap := map[string]struct{}{} for i := 0; i < len(testCase.popKeyRange)/2; i++ { - v, got := suite.svr.GetRaftCluster().PopOneSuspectKeyRange() + v, got := leaderServer.GetRaftCluster().PopOneSuspectKeyRange() suite.True(got) popKeyRangeMap[hex.EncodeToString(v[0])] = struct{}{} popKeyRangeMap[hex.EncodeToString(v[1])] = struct{}{} @@ -175,7 +155,7 @@ func (suite *ruleTestSuite) TestSet() { suite.True(ok) } } else { - err = tu.CheckPostJSON(testDialClient, suite.urlPrefix+"/rule", testCase.rawData, + err = tu.CheckPostJSON(testDialClient, urlPrefix+"/rule", testCase.rawData, tu.StatusNotOK(re), tu.StringEqual(re, testCase.response)) } @@ -184,11 +164,26 @@ func (suite *ruleTestSuite) TestSet() { } func (suite *ruleTestSuite) TestGet() { + opts := []tests.ConfigOption{ + func(conf *config.Config, serverName string) { + conf.PDServerCfg.KeyType = "raw" + conf.Replication.EnablePlacementRules = true + }, + } + env := tests.NewSchedulingTestEnvironment(suite.T(), opts...) + env.RunTestInTwoModes(suite.checkGet) +} + +func (suite *ruleTestSuite) checkGet(cluster *tests.TestCluster) { + leaderServer := cluster.GetLeaderServer() + pdAddr := leaderServer.GetAddr() + urlPrefix := fmt.Sprintf("%s%s/api/v1/config", pdAddr, apiPrefix) + rule := placement.Rule{GroupID: "a", ID: "20", StartKeyHex: "1111", EndKeyHex: "3333", Role: "voter", Count: 1} data, err := json.Marshal(rule) suite.NoError(err) re := suite.Require() - err = tu.CheckPostJSON(testDialClient, suite.urlPrefix+"/rule", data, tu.StatusOK(re)) + err = tu.CheckPostJSON(testDialClient, urlPrefix+"/rule", data, tu.StatusOK(re)) suite.NoError(err) testCases := []struct { @@ -213,7 +208,7 @@ func (suite *ruleTestSuite) TestGet() { for _, testCase := range testCases { suite.T().Log(testCase.name) var resp placement.Rule - url := fmt.Sprintf("%s/rule/%s/%s", suite.urlPrefix, testCase.rule.GroupID, testCase.rule.ID) + url := fmt.Sprintf("%s/rule/%s/%s", urlPrefix, testCase.rule.GroupID, testCase.rule.ID) if testCase.found { err = tu.ReadGetJSON(re, testDialClient, url, &resp) suite.compareRule(&resp, &testCase.rule) @@ -225,20 +220,50 @@ func (suite *ruleTestSuite) TestGet() { } func (suite *ruleTestSuite) TestGetAll() { + opts := []tests.ConfigOption{ + func(conf *config.Config, serverName string) { + conf.PDServerCfg.KeyType = "raw" + conf.Replication.EnablePlacementRules = true + }, + } + env := tests.NewSchedulingTestEnvironment(suite.T(), opts...) + env.RunTestInTwoModes(suite.checkGetAll) +} + +func (suite *ruleTestSuite) checkGetAll(cluster *tests.TestCluster) { + leaderServer := cluster.GetLeaderServer() + pdAddr := leaderServer.GetAddr() + urlPrefix := fmt.Sprintf("%s%s/api/v1/config", pdAddr, apiPrefix) + rule := placement.Rule{GroupID: "b", ID: "20", StartKeyHex: "1111", EndKeyHex: "3333", Role: "voter", Count: 1} data, err := json.Marshal(rule) suite.NoError(err) re := suite.Require() - err = tu.CheckPostJSON(testDialClient, suite.urlPrefix+"/rule", data, tu.StatusOK(re)) + err = tu.CheckPostJSON(testDialClient, urlPrefix+"/rule", data, tu.StatusOK(re)) suite.NoError(err) var resp2 []*placement.Rule - err = tu.ReadGetJSON(re, testDialClient, suite.urlPrefix+"/rules", &resp2) + err = tu.ReadGetJSON(re, testDialClient, urlPrefix+"/rules", &resp2) suite.NoError(err) suite.GreaterOrEqual(len(resp2), 1) } func (suite *ruleTestSuite) TestSetAll() { + opts := []tests.ConfigOption{ + func(conf *config.Config, serverName string) { + conf.PDServerCfg.KeyType = "raw" + conf.Replication.EnablePlacementRules = true + }, + } + env := tests.NewSchedulingTestEnvironment(suite.T(), opts...) + env.RunTestInTwoModes(suite.checkSetAll) +} + +func (suite *ruleTestSuite) checkSetAll(cluster *tests.TestCluster) { + leaderServer := cluster.GetLeaderServer() + pdAddr := leaderServer.GetAddr() + urlPrefix := fmt.Sprintf("%s%s/api/v1/config", pdAddr, apiPrefix) + rule1 := placement.Rule{GroupID: "a", ID: "12", StartKeyHex: "1111", EndKeyHex: "3333", Role: "voter", Count: 1} rule2 := placement.Rule{GroupID: "b", ID: "12", StartKeyHex: "1111", EndKeyHex: "3333", Role: "voter", Count: 1} rule3 := placement.Rule{GroupID: "a", ID: "12", StartKeyHex: "XXXX", EndKeyHex: "3333", Role: "voter", Count: 1} @@ -247,10 +272,10 @@ func (suite *ruleTestSuite) TestSetAll() { LocationLabels: []string{"host"}} rule6 := placement.Rule{GroupID: "pd", ID: "default", StartKeyHex: "", EndKeyHex: "", Role: "voter", Count: 3} - suite.svr.GetPersistOptions().GetReplicationConfig().LocationLabels = []string{"host"} - defaultRule := suite.svr.GetRaftCluster().GetRuleManager().GetRule("pd", "default") + leaderServer.GetPersistOptions().GetReplicationConfig().LocationLabels = []string{"host"} + defaultRule := leaderServer.GetRaftCluster().GetRuleManager().GetRule("pd", "default") defaultRule.LocationLabels = []string{"host"} - suite.svr.GetRaftCluster().GetRuleManager().SetRule(defaultRule) + leaderServer.GetRaftCluster().GetRuleManager().SetRule(defaultRule) successData, err := json.Marshal([]*placement.Rule{&rule1, &rule2}) suite.NoError(err) @@ -333,13 +358,13 @@ func (suite *ruleTestSuite) TestSetAll() { for _, testCase := range testCases { suite.T().Log(testCase.name) if testCase.success { - err := tu.CheckPostJSON(testDialClient, suite.urlPrefix+"/rules", testCase.rawData, tu.StatusOK(re)) + err := tu.CheckPostJSON(testDialClient, urlPrefix+"/rules", testCase.rawData, tu.StatusOK(re)) suite.NoError(err) if testCase.isDefaultRule { - suite.Equal(int(suite.svr.GetPersistOptions().GetReplicationConfig().MaxReplicas), testCase.count) + suite.Equal(int(leaderServer.GetPersistOptions().GetReplicationConfig().MaxReplicas), testCase.count) } } else { - err := tu.CheckPostJSON(testDialClient, suite.urlPrefix+"/rules", testCase.rawData, + err := tu.CheckPostJSON(testDialClient, urlPrefix+"/rules", testCase.rawData, tu.StringEqual(re, testCase.response)) suite.NoError(err) } @@ -347,17 +372,32 @@ func (suite *ruleTestSuite) TestSetAll() { } func (suite *ruleTestSuite) TestGetAllByGroup() { + opts := []tests.ConfigOption{ + func(conf *config.Config, serverName string) { + conf.PDServerCfg.KeyType = "raw" + conf.Replication.EnablePlacementRules = true + }, + } + env := tests.NewSchedulingTestEnvironment(suite.T(), opts...) + env.RunTestInTwoModes(suite.checkGetAllByGroup) +} + +func (suite *ruleTestSuite) checkGetAllByGroup(cluster *tests.TestCluster) { + leaderServer := cluster.GetLeaderServer() + pdAddr := leaderServer.GetAddr() + urlPrefix := fmt.Sprintf("%s%s/api/v1/config", pdAddr, apiPrefix) + re := suite.Require() rule := placement.Rule{GroupID: "c", ID: "20", StartKeyHex: "1111", EndKeyHex: "3333", Role: "voter", Count: 1} data, err := json.Marshal(rule) suite.NoError(err) - err = tu.CheckPostJSON(testDialClient, suite.urlPrefix+"/rule", data, tu.StatusOK(re)) + err = tu.CheckPostJSON(testDialClient, urlPrefix+"/rule", data, tu.StatusOK(re)) suite.NoError(err) rule1 := placement.Rule{GroupID: "c", ID: "30", StartKeyHex: "1111", EndKeyHex: "3333", Role: "voter", Count: 1} data, err = json.Marshal(rule1) suite.NoError(err) - err = tu.CheckPostJSON(testDialClient, suite.urlPrefix+"/rule", data, tu.StatusOK(re)) + err = tu.CheckPostJSON(testDialClient, urlPrefix+"/rule", data, tu.StatusOK(re)) suite.NoError(err) testCases := []struct { @@ -380,7 +420,7 @@ func (suite *ruleTestSuite) TestGetAllByGroup() { for _, testCase := range testCases { suite.T().Log(testCase.name) var resp []*placement.Rule - url := fmt.Sprintf("%s/rules/group/%s", suite.urlPrefix, testCase.groupID) + url := fmt.Sprintf("%s/rules/group/%s", urlPrefix, testCase.groupID) err = tu.ReadGetJSON(re, testDialClient, url, &resp) suite.NoError(err) suite.Len(resp, testCase.count) @@ -392,15 +432,30 @@ func (suite *ruleTestSuite) TestGetAllByGroup() { } func (suite *ruleTestSuite) TestGetAllByRegion() { + opts := []tests.ConfigOption{ + func(conf *config.Config, serverName string) { + conf.PDServerCfg.KeyType = "raw" + conf.Replication.EnablePlacementRules = true + }, + } + env := tests.NewSchedulingTestEnvironment(suite.T(), opts...) + env.RunTestInTwoModes(suite.checkGetAllByRegion) +} + +func (suite *ruleTestSuite) checkGetAllByRegion(cluster *tests.TestCluster) { + leaderServer := cluster.GetLeaderServer() + pdAddr := leaderServer.GetAddr() + urlPrefix := fmt.Sprintf("%s%s/api/v1/config", pdAddr, apiPrefix) + rule := placement.Rule{GroupID: "e", ID: "20", StartKeyHex: "1111", EndKeyHex: "3333", Role: "voter", Count: 1} data, err := json.Marshal(rule) suite.NoError(err) re := suite.Require() - err = tu.CheckPostJSON(testDialClient, suite.urlPrefix+"/rule", data, tu.StatusOK(re)) + err = tu.CheckPostJSON(testDialClient, urlPrefix+"/rule", data, tu.StatusOK(re)) suite.NoError(err) r := core.NewTestRegionInfo(4, 1, []byte{0x22, 0x22}, []byte{0x33, 0x33}) - mustRegionHeartbeat(re, suite.svr, r) + tests.MustPutRegionInfo(re, cluster, r) testCases := []struct { name string @@ -429,7 +484,7 @@ func (suite *ruleTestSuite) TestGetAllByRegion() { for _, testCase := range testCases { suite.T().Log(testCase.name) var resp []*placement.Rule - url := fmt.Sprintf("%s/rules/region/%s", suite.urlPrefix, testCase.regionID) + url := fmt.Sprintf("%s/rules/region/%s", urlPrefix, testCase.regionID) if testCase.success { err = tu.ReadGetJSON(re, testDialClient, url, &resp) @@ -446,11 +501,26 @@ func (suite *ruleTestSuite) TestGetAllByRegion() { } func (suite *ruleTestSuite) TestGetAllByKey() { + opts := []tests.ConfigOption{ + func(conf *config.Config, serverName string) { + conf.PDServerCfg.KeyType = "raw" + conf.Replication.EnablePlacementRules = true + }, + } + env := tests.NewSchedulingTestEnvironment(suite.T(), opts...) + env.RunTestInTwoModes(suite.checkGetAllByKey) +} + +func (suite *ruleTestSuite) checkGetAllByKey(cluster *tests.TestCluster) { + leaderServer := cluster.GetLeaderServer() + pdAddr := leaderServer.GetAddr() + urlPrefix := fmt.Sprintf("%s%s/api/v1/config", pdAddr, apiPrefix) + rule := placement.Rule{GroupID: "f", ID: "40", StartKeyHex: "8888", EndKeyHex: "9111", Role: "voter", Count: 1} data, err := json.Marshal(rule) suite.NoError(err) re := suite.Require() - err = tu.CheckPostJSON(testDialClient, suite.urlPrefix+"/rule", data, tu.StatusOK(re)) + err = tu.CheckPostJSON(testDialClient, urlPrefix+"/rule", data, tu.StatusOK(re)) suite.NoError(err) testCases := []struct { @@ -483,7 +553,7 @@ func (suite *ruleTestSuite) TestGetAllByKey() { for _, testCase := range testCases { suite.T().Log(testCase.name) var resp []*placement.Rule - url := fmt.Sprintf("%s/rules/key/%s", suite.urlPrefix, testCase.key) + url := fmt.Sprintf("%s/rules/key/%s", urlPrefix, testCase.key) if testCase.success { err = tu.ReadGetJSON(re, testDialClient, url, &resp) suite.Len(resp, testCase.respSize) @@ -495,10 +565,25 @@ func (suite *ruleTestSuite) TestGetAllByKey() { } func (suite *ruleTestSuite) TestDelete() { + opts := []tests.ConfigOption{ + func(conf *config.Config, serverName string) { + conf.PDServerCfg.KeyType = "raw" + conf.Replication.EnablePlacementRules = true + }, + } + env := tests.NewSchedulingTestEnvironment(suite.T(), opts...) + env.RunTestInTwoModes(suite.checkDelete) +} + +func (suite *ruleTestSuite) checkDelete(cluster *tests.TestCluster) { + leaderServer := cluster.GetLeaderServer() + pdAddr := leaderServer.GetAddr() + urlPrefix := fmt.Sprintf("%s%s/api/v1/config", pdAddr, apiPrefix) + rule := placement.Rule{GroupID: "g", ID: "10", StartKeyHex: "8888", EndKeyHex: "9111", Role: "voter", Count: 1} data, err := json.Marshal(rule) suite.NoError(err) - err = tu.CheckPostJSON(testDialClient, suite.urlPrefix+"/rule", data, tu.StatusOK(suite.Require())) + err = tu.CheckPostJSON(testDialClient, urlPrefix+"/rule", data, tu.StatusOK(suite.Require())) suite.NoError(err) oldStartKey, err := hex.DecodeString(rule.StartKeyHex) suite.NoError(err) @@ -529,15 +614,15 @@ func (suite *ruleTestSuite) TestDelete() { } for _, testCase := range testCases { suite.T().Log(testCase.name) - url := fmt.Sprintf("%s/rule/%s/%s", suite.urlPrefix, testCase.groupID, testCase.id) + url := fmt.Sprintf("%s/rule/%s/%s", urlPrefix, testCase.groupID, testCase.id) // clear suspect keyRanges to prevent test case from others - suite.svr.GetRaftCluster().ClearSuspectKeyRanges() + leaderServer.GetRaftCluster().ClearSuspectKeyRanges() err = tu.CheckDelete(testDialClient, url, tu.StatusOK(suite.Require())) suite.NoError(err) if len(testCase.popKeyRange) > 0 { popKeyRangeMap := map[string]struct{}{} for i := 0; i < len(testCase.popKeyRange)/2; i++ { - v, got := suite.svr.GetRaftCluster().PopOneSuspectKeyRange() + v, got := leaderServer.GetRaftCluster().PopOneSuspectKeyRange() suite.True(got) popKeyRangeMap[hex.EncodeToString(v[0])] = struct{}{} popKeyRangeMap[hex.EncodeToString(v[1])] = struct{}{} @@ -551,16 +636,22 @@ func (suite *ruleTestSuite) TestDelete() { } } -func (suite *ruleTestSuite) compareRule(r1 *placement.Rule, r2 *placement.Rule) { - suite.Equal(r2.GroupID, r1.GroupID) - suite.Equal(r2.ID, r1.ID) - suite.Equal(r2.StartKeyHex, r1.StartKeyHex) - suite.Equal(r2.EndKeyHex, r1.EndKeyHex) - suite.Equal(r2.Role, r1.Role) - suite.Equal(r2.Count, r1.Count) +func (suite *ruleTestSuite) TestBatch() { + opts := []tests.ConfigOption{ + func(conf *config.Config, serverName string) { + conf.PDServerCfg.KeyType = "raw" + conf.Replication.EnablePlacementRules = true + }, + } + env := tests.NewSchedulingTestEnvironment(suite.T(), opts...) + env.RunTestInTwoModes(suite.checkBatch) } -func (suite *ruleTestSuite) TestBatch() { +func (suite *ruleTestSuite) checkBatch(cluster *tests.TestCluster) { + leaderServer := cluster.GetLeaderServer() + pdAddr := leaderServer.GetAddr() + urlPrefix := fmt.Sprintf("%s%s/api/v1/config", pdAddr, apiPrefix) + opt1 := placement.RuleOp{ Action: placement.RuleOpAdd, Rule: &placement.Rule{GroupID: "a", ID: "13", StartKeyHex: "1111", EndKeyHex: "3333", Role: "voter", Count: 1}, @@ -670,10 +761,10 @@ func (suite *ruleTestSuite) TestBatch() { for _, testCase := range testCases { suite.T().Log(testCase.name) if testCase.success { - err := tu.CheckPostJSON(testDialClient, suite.urlPrefix+"/rules/batch", testCase.rawData, tu.StatusOK(re)) + err := tu.CheckPostJSON(testDialClient, urlPrefix+"/rules/batch", testCase.rawData, tu.StatusOK(re)) suite.NoError(err) } else { - err := tu.CheckPostJSON(testDialClient, suite.urlPrefix+"/rules/batch", testCase.rawData, + err := tu.CheckPostJSON(testDialClient, urlPrefix+"/rules/batch", testCase.rawData, tu.StatusNotOK(re), tu.StringEqual(re, testCase.response)) suite.NoError(err) @@ -682,6 +773,21 @@ func (suite *ruleTestSuite) TestBatch() { } func (suite *ruleTestSuite) TestBundle() { + opts := []tests.ConfigOption{ + func(conf *config.Config, serverName string) { + conf.PDServerCfg.KeyType = "raw" + conf.Replication.EnablePlacementRules = true + }, + } + env := tests.NewSchedulingTestEnvironment(suite.T(), opts...) + env.RunTestInTwoModes(suite.checkBundle) +} + +func (suite *ruleTestSuite) checkBundle(cluster *tests.TestCluster) { + leaderServer := cluster.GetLeaderServer() + pdAddr := leaderServer.GetAddr() + urlPrefix := fmt.Sprintf("%s%s/api/v1/config", pdAddr, apiPrefix) + re := suite.Require() // GetAll b1 := placement.GroupBundle{ @@ -691,7 +797,7 @@ func (suite *ruleTestSuite) TestBundle() { }, } var bundles []placement.GroupBundle - err := tu.ReadGetJSON(re, testDialClient, suite.urlPrefix+"/placement-rule", &bundles) + err := tu.ReadGetJSON(re, testDialClient, urlPrefix+"/placement-rule", &bundles) suite.NoError(err) suite.Len(bundles, 1) suite.compareBundle(bundles[0], b1) @@ -707,28 +813,28 @@ func (suite *ruleTestSuite) TestBundle() { } data, err := json.Marshal(b2) suite.NoError(err) - err = tu.CheckPostJSON(testDialClient, suite.urlPrefix+"/placement-rule/foo", data, tu.StatusOK(re)) + err = tu.CheckPostJSON(testDialClient, urlPrefix+"/placement-rule/foo", data, tu.StatusOK(re)) suite.NoError(err) // Get var bundle placement.GroupBundle - err = tu.ReadGetJSON(re, testDialClient, suite.urlPrefix+"/placement-rule/foo", &bundle) + err = tu.ReadGetJSON(re, testDialClient, urlPrefix+"/placement-rule/foo", &bundle) suite.NoError(err) suite.compareBundle(bundle, b2) // GetAll again - err = tu.ReadGetJSON(re, testDialClient, suite.urlPrefix+"/placement-rule", &bundles) + err = tu.ReadGetJSON(re, testDialClient, urlPrefix+"/placement-rule", &bundles) suite.NoError(err) suite.Len(bundles, 2) suite.compareBundle(bundles[0], b1) suite.compareBundle(bundles[1], b2) // Delete - err = tu.CheckDelete(testDialClient, suite.urlPrefix+"/placement-rule/pd", tu.StatusOK(suite.Require())) + err = tu.CheckDelete(testDialClient, urlPrefix+"/placement-rule/pd", tu.StatusOK(suite.Require())) suite.NoError(err) // GetAll again - err = tu.ReadGetJSON(re, testDialClient, suite.urlPrefix+"/placement-rule", &bundles) + err = tu.ReadGetJSON(re, testDialClient, urlPrefix+"/placement-rule", &bundles) suite.NoError(err) suite.Len(bundles, 1) suite.compareBundle(bundles[0], b2) @@ -739,11 +845,11 @@ func (suite *ruleTestSuite) TestBundle() { b3 := placement.GroupBundle{ID: "foobar", Index: 100} data, err = json.Marshal([]placement.GroupBundle{b1, b2, b3}) suite.NoError(err) - err = tu.CheckPostJSON(testDialClient, suite.urlPrefix+"/placement-rule", data, tu.StatusOK(re)) + err = tu.CheckPostJSON(testDialClient, urlPrefix+"/placement-rule", data, tu.StatusOK(re)) suite.NoError(err) // GetAll again - err = tu.ReadGetJSON(re, testDialClient, suite.urlPrefix+"/placement-rule", &bundles) + err = tu.ReadGetJSON(re, testDialClient, urlPrefix+"/placement-rule", &bundles) suite.NoError(err) suite.Len(bundles, 3) suite.compareBundle(bundles[0], b2) @@ -751,11 +857,11 @@ func (suite *ruleTestSuite) TestBundle() { suite.compareBundle(bundles[2], b3) // Delete using regexp - err = tu.CheckDelete(testDialClient, suite.urlPrefix+"/placement-rule/"+url.PathEscape("foo.*")+"?regexp", tu.StatusOK(suite.Require())) + err = tu.CheckDelete(testDialClient, urlPrefix+"/placement-rule/"+url.PathEscape("foo.*")+"?regexp", tu.StatusOK(suite.Require())) suite.NoError(err) // GetAll again - err = tu.ReadGetJSON(re, testDialClient, suite.urlPrefix+"/placement-rule", &bundles) + err = tu.ReadGetJSON(re, testDialClient, urlPrefix+"/placement-rule", &bundles) suite.NoError(err) suite.Len(bundles, 1) suite.compareBundle(bundles[0], b1) @@ -770,19 +876,19 @@ func (suite *ruleTestSuite) TestBundle() { } data, err = json.Marshal(b4) suite.NoError(err) - err = tu.CheckPostJSON(testDialClient, suite.urlPrefix+"/placement-rule/"+id, data, tu.StatusOK(re)) + err = tu.CheckPostJSON(testDialClient, urlPrefix+"/placement-rule/"+id, data, tu.StatusOK(re)) suite.NoError(err) b4.ID = id b4.Rules[0].GroupID = b4.ID // Get - err = tu.ReadGetJSON(re, testDialClient, suite.urlPrefix+"/placement-rule/"+id, &bundle) + err = tu.ReadGetJSON(re, testDialClient, urlPrefix+"/placement-rule/"+id, &bundle) suite.NoError(err) suite.compareBundle(bundle, b4) // GetAll again - err = tu.ReadGetJSON(re, testDialClient, suite.urlPrefix+"/placement-rule", &bundles) + err = tu.ReadGetJSON(re, testDialClient, urlPrefix+"/placement-rule", &bundles) suite.NoError(err) suite.Len(bundles, 2) suite.compareBundle(bundles[0], b1) @@ -798,13 +904,13 @@ func (suite *ruleTestSuite) TestBundle() { } data, err = json.Marshal([]placement.GroupBundle{b1, b4, b5}) suite.NoError(err) - err = tu.CheckPostJSON(testDialClient, suite.urlPrefix+"/placement-rule", data, tu.StatusOK(re)) + err = tu.CheckPostJSON(testDialClient, urlPrefix+"/placement-rule", data, tu.StatusOK(re)) suite.NoError(err) b5.Rules[0].GroupID = b5.ID // GetAll again - err = tu.ReadGetJSON(re, testDialClient, suite.urlPrefix+"/placement-rule", &bundles) + err = tu.ReadGetJSON(re, testDialClient, urlPrefix+"/placement-rule", &bundles) suite.NoError(err) suite.Len(bundles, 3) suite.compareBundle(bundles[0], b1) @@ -813,6 +919,21 @@ func (suite *ruleTestSuite) TestBundle() { } func (suite *ruleTestSuite) TestBundleBadRequest() { + opts := []tests.ConfigOption{ + func(conf *config.Config, serverName string) { + conf.PDServerCfg.KeyType = "raw" + conf.Replication.EnablePlacementRules = true + }, + } + env := tests.NewSchedulingTestEnvironment(suite.T(), opts...) + env.RunTestInTwoModes(suite.checkBundleBadRequest) +} + +func (suite *ruleTestSuite) checkBundleBadRequest(cluster *tests.TestCluster) { + leaderServer := cluster.GetLeaderServer() + pdAddr := leaderServer.GetAddr() + urlPrefix := fmt.Sprintf("%s%s/api/v1/config", pdAddr, apiPrefix) + testCases := []struct { uri string data string @@ -826,7 +947,7 @@ func (suite *ruleTestSuite) TestBundleBadRequest() { {"/placement-rule", `[{"group_id":"foo", "rules": [{"group_id":"bar", "id":"baz", "role":"voter", "count":1}]}]`, false}, } for _, testCase := range testCases { - err := tu.CheckPostJSON(testDialClient, suite.urlPrefix+testCase.uri, []byte(testCase.data), + err := tu.CheckPostJSON(testDialClient, urlPrefix+testCase.uri, []byte(testCase.data), func(_ []byte, code int, _ http.Header) { suite.Equal(testCase.ok, code == http.StatusOK) }) @@ -844,22 +965,42 @@ func (suite *ruleTestSuite) compareBundle(b1, b2 placement.GroupBundle) { } } +func (suite *ruleTestSuite) compareRule(r1 *placement.Rule, r2 *placement.Rule) { + suite.Equal(r2.GroupID, r1.GroupID) + suite.Equal(r2.ID, r1.ID) + suite.Equal(r2.StartKeyHex, r1.StartKeyHex) + suite.Equal(r2.EndKeyHex, r1.EndKeyHex) + suite.Equal(r2.Role, r1.Role) + suite.Equal(r2.Count, r1.Count) +} + type regionRuleTestSuite struct { suite.Suite - svr *server.Server - grpcSvr *server.GrpcServer - cleanup tu.CleanupFunc - urlPrefix string - stores []*metapb.Store - regions []*core.RegionInfo } func TestRegionRuleTestSuite(t *testing.T) { suite.Run(t, new(regionRuleTestSuite)) } -func (suite *regionRuleTestSuite) SetupSuite() { - suite.stores = []*metapb.Store{ +func (suite *regionRuleTestSuite) TestRegionPlacementRule() { + opts := []tests.ConfigOption{ + func(conf *config.Config, serverName string) { + conf.Replication.EnablePlacementRules = true + conf.Replication.MaxReplicas = 1 + }, + } + env := tests.NewSchedulingTestEnvironment(suite.T(), opts...) + // FIXME: enable this test in two modes after we support region label forward. + env.RunTestInPDMode(suite.checkRegionPlacementRule) +} + +func (suite *regionRuleTestSuite) checkRegionPlacementRule(cluster *tests.TestCluster) { + re := suite.Require() + leaderServer := cluster.GetLeaderServer() + pdAddr := leaderServer.GetAddr() + urlPrefix := fmt.Sprintf("%s%s/api/v1", pdAddr, apiPrefix) + + stores := []*metapb.Store{ { Id: 1, Address: "tikv1", @@ -875,49 +1016,30 @@ func (suite *regionRuleTestSuite) SetupSuite() { Version: "2.0.0", }, } - re := suite.Require() - suite.svr, suite.cleanup = mustNewServer(re, func(cfg *config.Config) { - cfg.Replication.EnablePlacementRules = true - cfg.Replication.MaxReplicas = 1 - }) - server.MustWaitLeader(re, []*server.Server{suite.svr}) - - addr := suite.svr.GetAddr() - suite.grpcSvr = &server.GrpcServer{Server: suite.svr} - suite.urlPrefix = fmt.Sprintf("%s%s/api/v1", addr, apiPrefix) - - mustBootstrapCluster(re, suite.svr) - - for _, store := range suite.stores { - mustPutStore(re, suite.svr, store.Id, store.State, store.NodeState, nil) + for _, store := range stores { + tests.MustPutStore(re, cluster, store) } - suite.regions = make([]*core.RegionInfo, 0) + regions := make([]*core.RegionInfo, 0) peers1 := []*metapb.Peer{ {Id: 102, StoreId: 1, Role: metapb.PeerRole_Voter}, {Id: 103, StoreId: 2, Role: metapb.PeerRole_Voter}} - suite.regions = append(suite.regions, core.NewRegionInfo(&metapb.Region{Id: 1, Peers: peers1, RegionEpoch: &metapb.RegionEpoch{ConfVer: 1, Version: 1}}, peers1[0], + regions = append(regions, core.NewRegionInfo(&metapb.Region{Id: 1, Peers: peers1, RegionEpoch: &metapb.RegionEpoch{ConfVer: 1, Version: 1}}, peers1[0], core.WithStartKey([]byte("abc")), core.WithEndKey([]byte("def")))) peers2 := []*metapb.Peer{ {Id: 104, StoreId: 1, Role: metapb.PeerRole_Voter}, {Id: 105, StoreId: 2, Role: metapb.PeerRole_Learner}} - suite.regions = append(suite.regions, core.NewRegionInfo(&metapb.Region{Id: 2, Peers: peers2, RegionEpoch: &metapb.RegionEpoch{ConfVer: 2, Version: 2}}, peers2[0], + regions = append(regions, core.NewRegionInfo(&metapb.Region{Id: 2, Peers: peers2, RegionEpoch: &metapb.RegionEpoch{ConfVer: 2, Version: 2}}, peers2[0], core.WithStartKey([]byte("ghi")), core.WithEndKey([]byte("jkl")))) peers3 := []*metapb.Peer{ {Id: 106, StoreId: 1, Role: metapb.PeerRole_Voter}, {Id: 107, StoreId: 2, Role: metapb.PeerRole_Learner}} - suite.regions = append(suite.regions, core.NewRegionInfo(&metapb.Region{Id: 3, Peers: peers3, RegionEpoch: &metapb.RegionEpoch{ConfVer: 3, Version: 3}}, peers3[0], + regions = append(regions, core.NewRegionInfo(&metapb.Region{Id: 3, Peers: peers3, RegionEpoch: &metapb.RegionEpoch{ConfVer: 3, Version: 3}}, peers3[0], core.WithStartKey([]byte("mno")), core.WithEndKey([]byte("pqr")))) - for _, rg := range suite.regions { - suite.svr.GetBasicCluster().PutRegion(rg) + for _, rg := range regions { + tests.MustPutRegionInfo(re, cluster, rg) } -} - -func (suite *regionRuleTestSuite) TearDownSuite() { - suite.cleanup() -} -func (suite *regionRuleTestSuite) TestRegionPlacementRule() { - ruleManager := suite.svr.GetRaftCluster().GetRuleManager() + ruleManager := leaderServer.GetRaftCluster().GetRuleManager() ruleManager.SetRule(&placement.Rule{ GroupID: "test", ID: "test2", @@ -934,38 +1056,38 @@ func (suite *regionRuleTestSuite) TestRegionPlacementRule() { Role: placement.Learner, Count: 1, }) - re := suite.Require() - url := fmt.Sprintf("%s/config/rules/region/%d/detail", suite.urlPrefix, 1) fit := &placement.RegionFit{} + + url := fmt.Sprintf("%s/config/rules/region/%d/detail", urlPrefix, 1) err := tu.ReadGetJSON(re, testDialClient, url, fit) + suite.NoError(err) suite.Equal(len(fit.RuleFits), 1) suite.Equal(len(fit.OrphanPeers), 1) - suite.NoError(err) - url = fmt.Sprintf("%s/config/rules/region/%d/detail", suite.urlPrefix, 2) + url = fmt.Sprintf("%s/config/rules/region/%d/detail", urlPrefix, 2) fit = &placement.RegionFit{} err = tu.ReadGetJSON(re, testDialClient, url, fit) + suite.NoError(err) suite.Equal(len(fit.RuleFits), 2) suite.Equal(len(fit.OrphanPeers), 0) - suite.NoError(err) - url = fmt.Sprintf("%s/config/rules/region/%d/detail", suite.urlPrefix, 3) + url = fmt.Sprintf("%s/config/rules/region/%d/detail", urlPrefix, 3) fit = &placement.RegionFit{} err = tu.ReadGetJSON(re, testDialClient, url, fit) + suite.NoError(err) suite.Equal(len(fit.RuleFits), 0) suite.Equal(len(fit.OrphanPeers), 2) - suite.NoError(err) - url = fmt.Sprintf("%s/config/rules/region/%d/detail", suite.urlPrefix, 4) + url = fmt.Sprintf("%s/config/rules/region/%d/detail", urlPrefix, 4) err = tu.CheckGetJSON(testDialClient, url, nil, tu.Status(re, http.StatusNotFound), tu.StringContain( re, "region 4 not found")) suite.NoError(err) - url = fmt.Sprintf("%s/config/rules/region/%s/detail", suite.urlPrefix, "id") + url = fmt.Sprintf("%s/config/rules/region/%s/detail", urlPrefix, "id") err = tu.CheckGetJSON(testDialClient, url, nil, tu.Status(re, http.StatusBadRequest), tu.StringContain( - re, "invalid region id")) + re, errs.ErrRegionInvalidID.Error())) suite.NoError(err) - suite.svr.GetRaftCluster().GetReplicationConfig().EnablePlacementRules = false - url = fmt.Sprintf("%s/config/rules/region/%d/detail", suite.urlPrefix, 1) + leaderServer.GetRaftCluster().GetReplicationConfig().EnablePlacementRules = false + url = fmt.Sprintf("%s/config/rules/region/%d/detail", urlPrefix, 1) err = tu.CheckGetJSON(testDialClient, url, nil, tu.Status(re, http.StatusPreconditionFailed), tu.StringContain( re, "placement rules feature is disabled")) suite.NoError(err) From da30175bdbb44a6dd7180e89fbc1076c781aba3a Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Fri, 10 Nov 2023 17:26:13 +0800 Subject: [PATCH 007/137] etcdutil, leadership: avoid redundant created watch channel (#7352) close tikv/pd#7351 Signed-off-by: lhy1024 Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- pkg/election/leadership.go | 1 + pkg/utils/etcdutil/etcdutil.go | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/election/leadership.go b/pkg/election/leadership.go index d5d73e90b58..8cfdcf423ac 100644 --- a/pkg/election/leadership.go +++ b/pkg/election/leadership.go @@ -260,6 +260,7 @@ func (ls *Leadership) Watch(serverCtx context.Context, revision int64) { continue } } + lastReceivedResponseTime = time.Now() log.Info("watch channel is created", zap.Int64("revision", revision), zap.String("leader-key", ls.leaderKey), zap.String("purpose", ls.purpose)) watchChanLoop: diff --git a/pkg/utils/etcdutil/etcdutil.go b/pkg/utils/etcdutil/etcdutil.go index 1432b6e37c3..e004247c6d0 100644 --- a/pkg/utils/etcdutil/etcdutil.go +++ b/pkg/utils/etcdutil/etcdutil.go @@ -707,7 +707,6 @@ func (lw *LoopWatcher) watch(ctx context.Context, revision int64) (nextRevision }() ticker := time.NewTicker(RequestProgressInterval) defer ticker.Stop() - lastReceivedResponseTime := time.Now() for { if watcherCancel != nil { @@ -736,8 +735,10 @@ func (lw *LoopWatcher) watch(ctx context.Context, revision int64) (nextRevision continue } } + lastReceivedResponseTime := time.Now() log.Info("watch channel is created in watch loop", zap.Int64("revision", revision), zap.String("name", lw.name), zap.String("key", lw.key)) + watchChanLoop: select { case <-ctx.Done(): From 71621d3cc296190f11cba4532bc4afcb73e314c6 Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Fri, 10 Nov 2023 17:42:43 +0800 Subject: [PATCH 008/137] checker: refactor fixOrphanPeers (#7342) ref tikv/pd#4399 Signed-off-by: lhy1024 Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- pkg/schedule/checker/rule_checker.go | 42 +++++++++++++---------- pkg/schedule/checker/rule_checker_test.go | 1 + server/cluster/cluster_test.go | 2 +- 3 files changed, 25 insertions(+), 20 deletions(-) diff --git a/pkg/schedule/checker/rule_checker.go b/pkg/schedule/checker/rule_checker.go index c4e7c242dea..08ef5f7b45c 100644 --- a/pkg/schedule/checker/rule_checker.go +++ b/pkg/schedule/checker/rule_checker.go @@ -449,23 +449,31 @@ func (c *RuleChecker) fixOrphanPeers(region *core.RegionInfo, fit *placement.Reg return nil, nil } - isUnhealthyPeer := func(id uint64) bool { - for _, downPeer := range region.GetDownPeers() { - if downPeer.Peer.GetId() == id { + isPendingPeer := func(id uint64) bool { + for _, pendingPeer := range region.GetPendingPeers() { + if pendingPeer.GetId() == id { return true } } - for _, pendingPeer := range region.GetPendingPeers() { - if pendingPeer.GetId() == id { + return false + } + + isDownPeer := func(id uint64) bool { + for _, downPeer := range region.GetDownPeers() { + if downPeer.Peer.GetId() == id { return true } } return false } - isDisconnectedPeer := func(p *metapb.Peer) bool { + isUnhealthyPeer := func(id uint64) bool { + return isPendingPeer(id) || isDownPeer(id) + } + + isInDisconnectedStore := func(p *metapb.Peer) bool { // avoid to meet down store when fix orphan peers, - // Isdisconnected is more strictly than IsUnhealthy. + // isInDisconnectedStore is usually more strictly than IsUnhealthy. store := c.cluster.GetStore(p.GetStoreId()) if store == nil { return true @@ -475,16 +483,12 @@ func (c *RuleChecker) fixOrphanPeers(region *core.RegionInfo, fit *placement.Reg checkDownPeer := func(peers []*metapb.Peer) (*metapb.Peer, bool) { for _, p := range peers { - if isUnhealthyPeer(p.GetId()) { - // make sure is down peer. - if region.GetDownPeer(p.GetId()) != nil { - return p, true - } - return nil, true - } - if isDisconnectedPeer(p) { + if isInDisconnectedStore(p) || isDownPeer(p.GetId()) { return p, true } + if isPendingPeer(p.GetId()) { + return nil, true + } } return nil, false } @@ -517,7 +521,7 @@ func (c *RuleChecker) fixOrphanPeers(region *core.RegionInfo, fit *placement.Reg continue } // make sure the orphan peer is healthy. - if isUnhealthyPeer(orphanPeer.GetId()) || isDisconnectedPeer(orphanPeer) { + if isUnhealthyPeer(orphanPeer.GetId()) || isInDisconnectedStore(orphanPeer) { continue } // no consider witness in this path. @@ -525,7 +529,7 @@ func (c *RuleChecker) fixOrphanPeers(region *core.RegionInfo, fit *placement.Reg continue } // pinDownPeer's store should be disconnected, because we use more strict judge before. - if !isDisconnectedPeer(pinDownPeer) { + if !isInDisconnectedStore(pinDownPeer) { continue } // check if down peer can replace with orphan peer. @@ -539,7 +543,7 @@ func (c *RuleChecker) fixOrphanPeers(region *core.RegionInfo, fit *placement.Reg return operator.CreatePromoteLearnerOperatorAndRemovePeer("replace-down-peer-with-orphan-peer", c.cluster, region, orphanPeer, pinDownPeer) case orphanPeerRole == metapb.PeerRole_Voter && destRole == metapb.PeerRole_Learner: return operator.CreateDemoteLearnerOperatorAndRemovePeer("replace-down-peer-with-orphan-peer", c.cluster, region, orphanPeer, pinDownPeer) - case orphanPeerRole == destRole && isDisconnectedPeer(pinDownPeer) && !dstStore.IsDisconnected(): + case orphanPeerRole == destRole && isInDisconnectedStore(pinDownPeer) && !dstStore.IsDisconnected(): return operator.CreateRemovePeerOperator("remove-replaced-orphan-peer", c.cluster, 0, region, pinDownPeer.GetStoreId()) default: // destRole should not same with orphanPeerRole. if role is same, it fit with orphanPeer should be better than now. @@ -557,7 +561,7 @@ func (c *RuleChecker) fixOrphanPeers(region *core.RegionInfo, fit *placement.Reg hasHealthPeer := false var disconnectedPeer *metapb.Peer for _, orphanPeer := range fit.OrphanPeers { - if isDisconnectedPeer(orphanPeer) { + if isInDisconnectedStore(orphanPeer) { disconnectedPeer = orphanPeer break } diff --git a/pkg/schedule/checker/rule_checker_test.go b/pkg/schedule/checker/rule_checker_test.go index eb357f302b7..4185ce6c167 100644 --- a/pkg/schedule/checker/rule_checker_test.go +++ b/pkg/schedule/checker/rule_checker_test.go @@ -1052,6 +1052,7 @@ func (suite *ruleCheckerTestSuite) TestPriorityFitHealthWithDifferentRole1() { suite.Equal("replace-down-peer-with-orphan-peer", op.Desc()) // set peer3 only pending + suite.cluster.GetStore(3).GetMeta().LastHeartbeat = time.Now().UnixNano() r1 = r1.Clone(core.WithDownPeers(nil)) suite.cluster.PutRegion(r1) op = suite.rc.Check(suite.cluster.GetRegion(1)) diff --git a/server/cluster/cluster_test.go b/server/cluster/cluster_test.go index 89c9ea32f19..70782e27cd3 100644 --- a/server/cluster/cluster_test.go +++ b/server/cluster/cluster_test.go @@ -2792,7 +2792,7 @@ func TestReplica(t *testing.T) { re.NoError(dispatchHeartbeat(co, region, stream)) waitNoResponse(re, stream) - // Remove peer from store 4. + // Remove peer from store 3. re.NoError(tc.addLeaderRegion(2, 1, 2, 3, 4)) region = tc.GetRegion(2) re.NoError(dispatchHeartbeat(co, region, stream)) From ad96bf1f3522e6e73c14990eba24971dedfa315a Mon Sep 17 00:00:00 2001 From: Hu# Date: Fri, 10 Nov 2023 17:59:14 +0800 Subject: [PATCH 009/137] mcs: fix error typo (#7354) ref tikv/pd#4399 Signed-off-by: husharp Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- pkg/mcs/tso/server/install/install.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/mcs/tso/server/install/install.go b/pkg/mcs/tso/server/install/install.go index 27db0c51d75..a821505474f 100644 --- a/pkg/mcs/tso/server/install/install.go +++ b/pkg/mcs/tso/server/install/install.go @@ -28,5 +28,5 @@ func init() { // Install registers the API group and grpc service. func Install(register *registry.ServiceRegistry) { - register.RegisterService("Scheduling", server.NewService[*server.Server]) + register.RegisterService("TSO", server.NewService[*server.Server]) } From 1a0233b8d598b83904024cbd160d77cac5eaa446 Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Fri, 10 Nov 2023 19:33:43 +0800 Subject: [PATCH 010/137] mcs: use a controller to manage scheduling jobs (#7270) ref tikv/pd#5839 Signed-off-by: Ryan Leung Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- server/cluster/cluster.go | 402 ++++------------- server/cluster/cluster_test.go | 13 +- server/cluster/cluster_worker.go | 3 +- server/cluster/scheduling_controller.go | 424 ++++++++++++++++++ server/grpc_service.go | 12 +- server/handler.go | 7 +- server/server.go | 11 +- .../mcs/scheduling/config_test.go | 1 - tests/server/cluster/cluster_test.go | 6 +- 9 files changed, 531 insertions(+), 348 deletions(-) create mode 100644 server/cluster/scheduling_controller.go diff --git a/server/cluster/cluster.go b/server/cluster/cluster.go index 8362ee9f331..c0fb1b15f8f 100644 --- a/server/cluster/cluster.go +++ b/server/cluster/cluster.go @@ -41,22 +41,18 @@ import ( "github.com/tikv/pd/pkg/gctuner" "github.com/tikv/pd/pkg/id" "github.com/tikv/pd/pkg/keyspace" + mcsutils "github.com/tikv/pd/pkg/mcs/utils" "github.com/tikv/pd/pkg/memory" "github.com/tikv/pd/pkg/progress" "github.com/tikv/pd/pkg/replication" "github.com/tikv/pd/pkg/schedule" - "github.com/tikv/pd/pkg/schedule/checker" sc "github.com/tikv/pd/pkg/schedule/config" "github.com/tikv/pd/pkg/schedule/hbstream" "github.com/tikv/pd/pkg/schedule/labeler" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/placement" - "github.com/tikv/pd/pkg/schedule/scatter" - "github.com/tikv/pd/pkg/schedule/schedulers" - "github.com/tikv/pd/pkg/schedule/splitter" "github.com/tikv/pd/pkg/slice" "github.com/tikv/pd/pkg/statistics" - "github.com/tikv/pd/pkg/statistics/buckets" "github.com/tikv/pd/pkg/statistics/utils" "github.com/tikv/pd/pkg/storage" "github.com/tikv/pd/pkg/storage/endpoint" @@ -97,6 +93,7 @@ const ( clientTimeout = 3 * time.Second defaultChangedRegionsLimit = 10000 gcTombstoneInterval = 30 * 24 * time.Hour + serviceCheckInterval = 10 * time.Second // persistLimitRetryTimes is used to reduce the probability of the persistent error // since the once the store is added or removed, we shouldn't return an error even if the store limit is failed to persist. persistLimitRetryTimes = 5 @@ -155,16 +152,12 @@ type RaftCluster struct { prevStoreLimit map[uint64]map[storelimit.Type]float64 // This below fields are all read-only, we cannot update itself after the raft cluster starts. - clusterID uint64 - id id.Allocator - core *core.BasicCluster // cached cluster info - opt *config.PersistOptions - limiter *StoreLimiter - coordinator *schedule.Coordinator - labelLevelStats *statistics.LabelStatistics - regionStats *statistics.RegionStatistics - hotStat *statistics.HotStat - slowStat *statistics.SlowStat + clusterID uint64 + id id.Allocator + core *core.BasicCluster // cached cluster info + opt *config.PersistOptions + limiter *StoreLimiter + *schedulingController ruleManager *placement.RuleManager regionLabeler *labeler.RegionLabeler replicationMode *replication.ModeManager @@ -173,6 +166,8 @@ type RaftCluster struct { regionSyncer *syncer.RegionSyncer changedRegions chan *core.RegionInfo keyspaceGroupManager *keyspace.GroupManager + independentServices sync.Map + hbstreams *hbstream.HeartbeatStreams } // Status saves some state information. @@ -266,17 +261,17 @@ func (c *RaftCluster) InitCluster( opt sc.ConfProvider, storage storage.Storage, basicCluster *core.BasicCluster, + hbstreams *hbstream.HeartbeatStreams, keyspaceGroupManager *keyspace.GroupManager) { c.core, c.opt, c.storage, c.id = basicCluster, opt.(*config.PersistOptions), storage, id c.ctx, c.cancel = context.WithCancel(c.serverCtx) - c.labelLevelStats = statistics.NewLabelStatistics() - c.hotStat = statistics.NewHotStat(c.ctx) - c.slowStat = statistics.NewSlowStat(c.ctx) c.progressManager = progress.NewManager() c.changedRegions = make(chan *core.RegionInfo, defaultChangedRegionsLimit) c.prevStoreLimit = make(map[uint64]map[storelimit.Type]float64) c.unsafeRecoveryController = unsaferecovery.NewController(c) c.keyspaceGroupManager = keyspaceGroupManager + c.hbstreams = hbstreams + c.schedulingController = newSchedulingController(c.ctx) } // Start starts a cluster. @@ -290,7 +285,7 @@ func (c *RaftCluster) Start(s Server) error { } c.isAPIServiceMode = s.IsAPIServiceMode() - c.InitCluster(s.GetAllocator(), s.GetPersistOptions(), s.GetStorage(), s.GetBasicCluster(), s.GetKeyspaceGroupManager()) + c.InitCluster(s.GetAllocator(), s.GetPersistOptions(), s.GetStorage(), s.GetBasicCluster(), s.GetHBStreams(), s.GetKeyspaceGroupManager()) cluster, err := c.LoadClusterInfo() if err != nil { return err @@ -316,8 +311,7 @@ func (c *RaftCluster) Start(s Server) error { return err } - c.coordinator = schedule.NewCoordinator(c.ctx, cluster, s.GetHBStreams()) - c.regionStats = statistics.NewRegionStatistics(c.core, c.opt, c.ruleManager) + c.schedulingController.init(c.core, c.opt, schedule.NewCoordinator(c.ctx, c, c.GetHeartbeatStreams()), c.ruleManager) c.limiter = NewStoreLimiter(s.GetPersistOptions()) c.externalTS, err = c.storage.LoadExternalTS() if err != nil { @@ -331,14 +325,9 @@ func (c *RaftCluster) Start(s Server) error { if err != nil { return err } - c.initSchedulers() - } else { - c.wg.Add(2) - go c.runCoordinator() - go c.runStatsBackgroundJobs() } - - c.wg.Add(8) + c.wg.Add(9) + go c.runServiceCheckJob() go c.runMetricsCollectionJob() go c.runNodeStateCheckJob() go c.syncRegions() @@ -352,6 +341,38 @@ func (c *RaftCluster) Start(s Server) error { return nil } +func (c *RaftCluster) runServiceCheckJob() { + defer logutil.LogPanic() + defer c.wg.Done() + + var once sync.Once + + checkFn := func() { + if c.isAPIServiceMode { + once.Do(c.initSchedulers) + c.independentServices.Store(mcsutils.SchedulingServiceName, true) + return + } + if c.startSchedulingJobs() { + c.independentServices.Delete(mcsutils.SchedulingServiceName) + } + } + checkFn() + + ticker := time.NewTicker(serviceCheckInterval) + defer ticker.Stop() + + for { + select { + case <-c.ctx.Done(): + log.Info("service check job is stopped") + return + case <-ticker.C: + checkFn() + } + } +} + // startGCTuner func (c *RaftCluster) startGCTuner() { defer logutil.LogPanic() @@ -600,10 +621,9 @@ func (c *RaftCluster) LoadClusterInfo() (*RaftCluster, error) { zap.Int("count", c.core.GetTotalRegionCount()), zap.Duration("cost", time.Since(start)), ) - if !c.isAPIServiceMode { + if !c.IsServiceIndependent(mcsutils.SchedulingServiceName) { for _, store := range c.GetStores() { storeID := store.GetID() - c.hotStat.GetOrCreateRollingStoreStats(storeID) c.slowStat.ObserveSlowStoreStatus(storeID, store.IsSlow()) } } @@ -619,7 +639,6 @@ func (c *RaftCluster) runMetricsCollectionJob() { ticker.Stop() ticker = time.NewTicker(time.Microsecond) }) - defer ticker.Stop() for { @@ -657,24 +676,6 @@ func (c *RaftCluster) runNodeStateCheckJob() { } } -func (c *RaftCluster) runStatsBackgroundJobs() { - defer logutil.LogPanic() - defer c.wg.Done() - - ticker := time.NewTicker(statistics.RegionsStatsObserveInterval) - defer ticker.Stop() - - for { - select { - case <-c.ctx.Done(): - log.Info("statistics background jobs has been stopped") - return - case <-ticker.C: - c.hotStat.ObserveRegionsStats(c.core.GetStoresWriteRate()) - } - } -} - func (c *RaftCluster) runUpdateStoreStats() { defer logutil.LogPanic() defer c.wg.Done() @@ -696,13 +697,6 @@ func (c *RaftCluster) runUpdateStoreStats() { } } -// runCoordinator runs the main scheduling loop. -func (c *RaftCluster) runCoordinator() { - defer logutil.LogPanic() - defer c.wg.Done() - c.coordinator.RunUntilStop() -} - func (c *RaftCluster) syncRegions() { defer logutil.LogPanic() defer c.wg.Done() @@ -723,8 +717,8 @@ func (c *RaftCluster) Stop() { return } c.running = false - if !c.isAPIServiceMode { - c.coordinator.Stop() + if !c.IsServiceIndependent(mcsutils.SchedulingServiceName) { + c.stopSchedulingJobs() } c.cancel() c.Unlock() @@ -750,6 +744,11 @@ func (c *RaftCluster) Context() context.Context { return nil } +// GetHeartbeatStreams returns the heartbeat streams. +func (c *RaftCluster) GetHeartbeatStreams() *hbstream.HeartbeatStreams { + return c.hbstreams +} + // GetCoordinator returns the coordinator. func (c *RaftCluster) GetCoordinator() *schedule.Coordinator { return c.coordinator @@ -760,71 +759,6 @@ func (c *RaftCluster) GetOperatorController() *operator.Controller { return c.coordinator.GetOperatorController() } -// SetPrepared set the prepare check to prepared. Only for test purpose. -func (c *RaftCluster) SetPrepared() { - c.coordinator.GetPrepareChecker().SetPrepared() -} - -// GetRegionScatterer returns the region scatter. -func (c *RaftCluster) GetRegionScatterer() *scatter.RegionScatterer { - return c.coordinator.GetRegionScatterer() -} - -// GetRegionSplitter returns the region splitter -func (c *RaftCluster) GetRegionSplitter() *splitter.RegionSplitter { - return c.coordinator.GetRegionSplitter() -} - -// GetMergeChecker returns merge checker. -func (c *RaftCluster) GetMergeChecker() *checker.MergeChecker { - return c.coordinator.GetMergeChecker() -} - -// GetRuleChecker returns rule checker. -func (c *RaftCluster) GetRuleChecker() *checker.RuleChecker { - return c.coordinator.GetRuleChecker() -} - -// GetSchedulers gets all schedulers. -func (c *RaftCluster) GetSchedulers() []string { - return c.coordinator.GetSchedulersController().GetSchedulerNames() -} - -// GetSchedulerHandlers gets all scheduler handlers. -func (c *RaftCluster) GetSchedulerHandlers() map[string]http.Handler { - return c.coordinator.GetSchedulersController().GetSchedulerHandlers() -} - -// AddSchedulerHandler adds a scheduler handler. -func (c *RaftCluster) AddSchedulerHandler(scheduler schedulers.Scheduler, args ...string) error { - return c.coordinator.GetSchedulersController().AddSchedulerHandler(scheduler, args...) -} - -// RemoveSchedulerHandler removes a scheduler handler. -func (c *RaftCluster) RemoveSchedulerHandler(name string) error { - return c.coordinator.GetSchedulersController().RemoveSchedulerHandler(name) -} - -// AddScheduler adds a scheduler. -func (c *RaftCluster) AddScheduler(scheduler schedulers.Scheduler, args ...string) error { - return c.coordinator.GetSchedulersController().AddScheduler(scheduler, args...) -} - -// RemoveScheduler removes a scheduler. -func (c *RaftCluster) RemoveScheduler(name string) error { - return c.coordinator.GetSchedulersController().RemoveScheduler(name) -} - -// PauseOrResumeScheduler pauses or resumes a scheduler. -func (c *RaftCluster) PauseOrResumeScheduler(name string, t int64) error { - return c.coordinator.GetSchedulersController().PauseOrResumeScheduler(name, t) -} - -// PauseOrResumeChecker pauses or resumes checker. -func (c *RaftCluster) PauseOrResumeChecker(name string, t int64) error { - return c.coordinator.PauseOrResumeChecker(name, t) -} - // AllocID returns a global unique ID. func (c *RaftCluster) AllocID() (uint64, error) { return c.id.Alloc() @@ -861,10 +795,6 @@ func (c *RaftCluster) GetOpts() sc.ConfProvider { return c.opt } -func (c *RaftCluster) initSchedulers() { - c.coordinator.InitSchedulers(false) -} - // GetScheduleConfig returns scheduling configurations. func (c *RaftCluster) GetScheduleConfig() *sc.ScheduleConfig { return c.opt.GetScheduleConfig() @@ -890,60 +820,11 @@ func (c *RaftCluster) SetPDServerConfig(cfg *config.PDServerConfig) { c.opt.SetPDServerConfig(cfg) } -// AddSuspectRegions adds regions to suspect list. -func (c *RaftCluster) AddSuspectRegions(regionIDs ...uint64) { - c.coordinator.GetCheckerController().AddSuspectRegions(regionIDs...) -} - -// GetSuspectRegions gets all suspect regions. -func (c *RaftCluster) GetSuspectRegions() []uint64 { - return c.coordinator.GetCheckerController().GetSuspectRegions() -} - -// GetHotStat gets hot stat. -func (c *RaftCluster) GetHotStat() *statistics.HotStat { - return c.hotStat -} - -// GetRegionStats gets region statistics. -func (c *RaftCluster) GetRegionStats() *statistics.RegionStatistics { - return c.regionStats -} - -// GetLabelStats gets label statistics. -func (c *RaftCluster) GetLabelStats() *statistics.LabelStatistics { - return c.labelLevelStats -} - -// RemoveSuspectRegion removes region from suspect list. -func (c *RaftCluster) RemoveSuspectRegion(id uint64) { - c.coordinator.GetCheckerController().RemoveSuspectRegion(id) -} - // GetUnsafeRecoveryController returns the unsafe recovery controller. func (c *RaftCluster) GetUnsafeRecoveryController() *unsaferecovery.Controller { return c.unsafeRecoveryController } -// AddSuspectKeyRange adds the key range with the its ruleID as the key -// The instance of each keyRange is like following format: -// [2][]byte: start key/end key -func (c *RaftCluster) AddSuspectKeyRange(start, end []byte) { - c.coordinator.GetCheckerController().AddSuspectKeyRange(start, end) -} - -// PopOneSuspectKeyRange gets one suspect keyRange group. -// it would return value and true if pop success, or return empty [][2][]byte and false -// if suspectKeyRanges couldn't pop keyRange group. -func (c *RaftCluster) PopOneSuspectKeyRange() ([2][]byte, bool) { - return c.coordinator.GetCheckerController().PopOneSuspectKeyRange() -} - -// ClearSuspectKeyRanges clears the suspect keyRanges, only for unit test -func (c *RaftCluster) ClearSuspectKeyRanges() { - c.coordinator.GetCheckerController().ClearSuspectKeyRanges() -} - // HandleStoreHeartbeat updates the store status. func (c *RaftCluster) HandleStoreHeartbeat(heartbeat *pdpb.StoreHeartbeatRequest, resp *pdpb.StoreHeartbeatResponse) error { stats := heartbeat.GetStats() @@ -970,7 +851,7 @@ func (c *RaftCluster) HandleStoreHeartbeat(heartbeat *pdpb.StoreHeartbeatRequest nowTime := time.Now() var newStore *core.StoreInfo // If this cluster has slow stores, we should awaken hibernated regions in other stores. - if !c.isAPIServiceMode { + if !c.IsServiceIndependent(mcsutils.SchedulingServiceName) { if needAwaken, slowStoreIDs := c.NeedAwakenAllRegionsInStore(storeID); needAwaken { log.Info("forcely awaken hibernated regions", zap.Uint64("store-id", storeID), zap.Uint64s("slow-stores", slowStoreIDs)) newStore = store.Clone(core.SetStoreStats(stats), core.SetLastHeartbeatTS(nowTime), core.SetLastAwakenTime(nowTime), opt) @@ -1005,7 +886,7 @@ func (c *RaftCluster) HandleStoreHeartbeat(heartbeat *pdpb.StoreHeartbeatRequest regions map[uint64]*core.RegionInfo interval uint64 ) - if !c.isAPIServiceMode { + if !c.IsServiceIndependent(mcsutils.SchedulingServiceName) { c.hotStat.Observe(storeID, newStore.GetStoreStats()) c.hotStat.FilterUnhealthyStore(c) c.slowStat.ObserveSlowStoreStatus(storeID, newStore.IsSlow()) @@ -1061,7 +942,7 @@ func (c *RaftCluster) HandleStoreHeartbeat(heartbeat *pdpb.StoreHeartbeatRequest e := int64(dur)*2 - int64(stat.GetTotalDurationSec()) store.Feedback(float64(e)) } - if !c.isAPIServiceMode { + if !c.IsServiceIndependent(mcsutils.SchedulingServiceName) { // Here we will compare the reported regions with the previous hot peers to decide if it is still hot. c.hotStat.CheckReadAsync(statistics.NewCollectUnReportedPeerTask(storeID, regions, interval)) } @@ -1097,11 +978,6 @@ func (c *RaftCluster) processReportBuckets(buckets *metapb.Buckets) error { return nil } -// IsPrepared return true if the prepare checker is ready. -func (c *RaftCluster) IsPrepared() bool { - return c.coordinator.GetPrepareChecker().IsPrepared() -} - var regionGuide = core.GenerateRegionGuideFunc(true) // processRegionHeartbeat updates the region information. @@ -1112,7 +988,7 @@ func (c *RaftCluster) processRegionHeartbeat(region *core.RegionInfo) error { } region.Inherit(origin, c.GetStoreConfig().IsEnableRegionBucket()) - if !c.isAPIServiceMode { + if !c.IsServiceIndependent(mcsutils.SchedulingServiceName) { cluster.HandleStatsAsync(c, region) } @@ -1121,7 +997,7 @@ func (c *RaftCluster) processRegionHeartbeat(region *core.RegionInfo) error { // Save to cache if meta or leader is updated, or contains any down/pending peer. // Mark isNew if the region in cache does not have leader. isNew, saveKV, saveCache, needSync := regionGuide(region, origin) - if !c.isAPIServiceMode && !saveKV && !saveCache && !isNew { + if !c.IsServiceIndependent(mcsutils.SchedulingServiceName) && !saveKV && !saveCache && !isNew { // Due to some config changes need to update the region stats as well, // so we do some extra checks here. if hasRegionStats && c.regionStats.RegionStatsNeedUpdate(region) { @@ -1146,13 +1022,13 @@ func (c *RaftCluster) processRegionHeartbeat(region *core.RegionInfo) error { if overlaps, err = c.core.AtomicCheckAndPutRegion(region); err != nil { return err } - if !c.isAPIServiceMode { + if !c.IsServiceIndependent(mcsutils.SchedulingServiceName) { cluster.HandleOverlaps(c, overlaps) } regionUpdateCacheEventCounter.Inc() } - if !c.isAPIServiceMode { + if !c.IsServiceIndependent(mcsutils.SchedulingServiceName) { cluster.Collect(c, region, c.GetRegionStores(region), hasRegionStats, isNew, c.IsPrepared()) } @@ -1566,24 +1442,6 @@ func (c *RaftCluster) checkReplicaBeforeOfflineStore(storeID uint64) error { return nil } -func (c *RaftCluster) getEvictLeaderStores() (evictStores []uint64) { - if c.coordinator == nil { - return nil - } - handler, ok := c.coordinator.GetSchedulersController().GetSchedulerHandlers()[schedulers.EvictLeaderName] - if !ok { - return - } - type evictLeaderHandler interface { - EvictStoreIDs() []uint64 - } - h, ok := handler.(evictLeaderHandler) - if !ok { - return - } - return h.EvictStoreIDs() -} - func (c *RaftCluster) getUpStores() []uint64 { upStores := make([]uint64, 0) for _, store := range c.GetStores() { @@ -1634,9 +1492,8 @@ func (c *RaftCluster) BuryStore(storeID uint64, forceBury bool) error { c.resetProgress(storeID, addr) storeIDStr := strconv.FormatUint(storeID, 10) statistics.ResetStoreStatistics(addr, storeIDStr) - if !c.isAPIServiceMode { - c.hotStat.RemoveRollingStoreStats(storeID) - c.slowStat.RemoveSlowStoreStatus(storeID) + if !c.IsServiceIndependent(mcsutils.SchedulingServiceName) { + c.removeStoreStatistics(storeID) } } return err @@ -1811,9 +1668,8 @@ func (c *RaftCluster) putStoreLocked(store *core.StoreInfo) error { } } c.core.PutStore(store) - if !c.isAPIServiceMode { - c.hotStat.GetOrCreateRollingStoreStats(store.GetID()) - c.slowStat.ObserveSlowStoreStatus(store.GetID(), store.IsSlow()) + if !c.IsServiceIndependent(mcsutils.SchedulingServiceName) { + c.updateStoreStatistics(store.GetID(), store.IsSlow()) } return nil } @@ -2162,53 +2018,14 @@ func (c *RaftCluster) deleteStore(store *core.StoreInfo) error { } func (c *RaftCluster) collectMetrics() { - if !c.isAPIServiceMode { - statsMap := statistics.NewStoreStatisticsMap(c.opt) - stores := c.GetStores() - for _, s := range stores { - statsMap.Observe(s) - statsMap.ObserveHotStat(s, c.hotStat.StoresStats) - } - statsMap.Collect() - c.coordinator.GetSchedulersController().CollectSchedulerMetrics() - c.coordinator.CollectHotSpotMetrics() - c.collectClusterMetrics() - } c.collectHealthStatus() } func (c *RaftCluster) resetMetrics() { - statistics.Reset() - - if !c.isAPIServiceMode { - c.coordinator.GetSchedulersController().ResetSchedulerMetrics() - c.coordinator.ResetHotSpotMetrics() - c.resetClusterMetrics() - } c.resetHealthStatus() c.resetProgressIndicator() } -func (c *RaftCluster) collectClusterMetrics() { - if c.regionStats == nil { - return - } - c.regionStats.Collect() - c.labelLevelStats.Collect() - // collect hot cache metrics - c.hotStat.CollectMetrics() -} - -func (c *RaftCluster) resetClusterMetrics() { - if c.regionStats == nil { - return - } - c.regionStats.Reset() - c.labelLevelStats.Reset() - // reset hot cache metrics - c.hotStat.ResetMetrics() -} - func (c *RaftCluster) collectHealthStatus() { members, err := GetMembers(c.etcdClient) if err != nil { @@ -2235,21 +2052,6 @@ func (c *RaftCluster) resetProgressIndicator() { storesETAGauge.Reset() } -// GetRegionStatsByType gets the status of the region by types. -func (c *RaftCluster) GetRegionStatsByType(typ statistics.RegionStatisticType) []*core.RegionInfo { - if c.regionStats == nil { - return nil - } - return c.regionStats.GetRegionStatsByType(typ) -} - -// UpdateRegionsLabelLevelStats updates the status of the region label level by types. -func (c *RaftCluster) UpdateRegionsLabelLevelStats(regions []*core.RegionInfo) { - for _, region := range regions { - c.labelLevelStats.Observe(region, c.getStoresWithoutLabelLocked(region, core.EngineKey, core.EngineTiFlash), c.opt.GetLocationLabels()) - } -} - func (c *RaftCluster) getRegionStoresLocked(region *core.RegionInfo) []*core.StoreInfo { stores := make([]*core.StoreInfo, 0, len(region.GetPeers())) for _, p := range region.GetPeers() { @@ -2260,16 +2062,6 @@ func (c *RaftCluster) getRegionStoresLocked(region *core.RegionInfo) []*core.Sto return stores } -func (c *RaftCluster) getStoresWithoutLabelLocked(region *core.RegionInfo, key, value string) []*core.StoreInfo { - stores := make([]*core.StoreInfo, 0, len(region.GetPeers())) - for _, p := range region.GetPeers() { - if store := c.core.GetStore(p.StoreId); store != nil && !core.IsStoreContainLabel(store.GetMeta(), key, value) { - stores = append(stores, store) - } - } - return stores -} - // OnStoreVersionChange changes the version of the cluster when needed. func (c *RaftCluster) OnStoreVersionChange() { c.RLock() @@ -2345,49 +2137,6 @@ func (c *RaftCluster) GetRegionCount(startKey, endKey []byte) *statistics.Region return stats } -// GetStoresStats returns stores' statistics from cluster. -// And it will be unnecessary to filter unhealthy store, because it has been solved in process heartbeat -func (c *RaftCluster) GetStoresStats() *statistics.StoresStats { - return c.hotStat.StoresStats -} - -// GetStoresLoads returns load stats of all stores. -func (c *RaftCluster) GetStoresLoads() map[uint64][]float64 { - return c.hotStat.GetStoresLoads() -} - -// IsRegionHot checks if a region is in hot state. -func (c *RaftCluster) IsRegionHot(region *core.RegionInfo) bool { - return c.hotStat.IsRegionHot(region, c.opt.GetHotRegionCacheHitsThreshold()) -} - -// GetHotPeerStat returns hot peer stat with specified regionID and storeID. -func (c *RaftCluster) GetHotPeerStat(rw utils.RWType, regionID, storeID uint64) *statistics.HotPeerStat { - return c.hotStat.GetHotPeerStat(rw, regionID, storeID) -} - -// RegionReadStats returns hot region's read stats. -// The result only includes peers that are hot enough. -// RegionStats is a thread-safe method -func (c *RaftCluster) RegionReadStats() map[uint64][]*statistics.HotPeerStat { - // As read stats are reported by store heartbeat, the threshold needs to be adjusted. - threshold := c.GetOpts().GetHotRegionCacheHitsThreshold() * - (utils.RegionHeartBeatReportInterval / utils.StoreHeartBeatReportInterval) - return c.hotStat.RegionStats(utils.Read, threshold) -} - -// RegionWriteStats returns hot region's write stats. -// The result only includes peers that are hot enough. -func (c *RaftCluster) RegionWriteStats() map[uint64][]*statistics.HotPeerStat { - // RegionStats is a thread-safe method - return c.hotStat.RegionStats(utils.Write, c.GetOpts().GetHotRegionCacheHitsThreshold()) -} - -// BucketsStats returns hot region's buckets stats. -func (c *RaftCluster) BucketsStats(degree int, regionIDs ...uint64) map[uint64][]*buckets.BucketStat { - return c.hotStat.BucketsStats(degree, regionIDs...) -} - // TODO: remove me. // only used in test. func (c *RaftCluster) putRegion(region *core.RegionInfo) error { @@ -2775,12 +2524,11 @@ func IsClientURL(addr string, etcdClient *clientv3.Client) bool { return false } -// GetPausedSchedulerDelayAt returns DelayAt of a paused scheduler -func (c *RaftCluster) GetPausedSchedulerDelayAt(name string) (int64, error) { - return c.coordinator.GetSchedulersController().GetPausedSchedulerDelayAt(name) -} - -// GetPausedSchedulerDelayUntil returns DelayUntil of a paused scheduler -func (c *RaftCluster) GetPausedSchedulerDelayUntil(name string) (int64, error) { - return c.coordinator.GetSchedulersController().GetPausedSchedulerDelayUntil(name) +// IsServiceIndependent returns whether the service is independent. +func (c *RaftCluster) IsServiceIndependent(name string) bool { + independent, exist := c.independentServices.Load(name) + if !exist { + return false + } + return independent.(bool) } diff --git a/server/cluster/cluster_test.go b/server/cluster/cluster_test.go index 70782e27cd3..7ebd012a6a2 100644 --- a/server/cluster/cluster_test.go +++ b/server/cluster/cluster_test.go @@ -1144,7 +1144,7 @@ func TestRegionLabelIsolationLevel(t *testing.T) { re.NoError(cluster.putRegion(r)) cluster.UpdateRegionsLabelLevelStats([]*core.RegionInfo{r}) - counter := cluster.labelLevelStats.GetLabelCounter() + counter := cluster.labelStats.GetLabelCounter() re.Equal(0, counter["none"]) re.Equal(1, counter["zone"]) } @@ -2130,7 +2130,7 @@ func newTestRaftCluster( basicCluster *core.BasicCluster, ) *RaftCluster { rc := &RaftCluster{serverCtx: ctx} - rc.InitCluster(id, opt, s, basicCluster, nil) + rc.InitCluster(id, opt, s, basicCluster, nil, nil) rc.ruleManager = placement.NewRuleManager(storage.NewStorageWithMemoryBackend(), rc, opt) if opt.IsPlacementRulesEnabled() { err := rc.ruleManager.Initialize(opt.GetMaxReplicas(), opt.GetLocationLabels(), opt.GetIsolationLevel()) @@ -2138,6 +2138,7 @@ func newTestRaftCluster( panic(err) } } + rc.schedulingController.init(basicCluster, opt, nil, rc.ruleManager) return rc } @@ -2502,11 +2503,11 @@ func TestCollectMetricsConcurrent(t *testing.T) { for i := 0; i < 1000; i++ { co.CollectHotSpotMetrics() controller.CollectSchedulerMetrics() - co.GetCluster().(*RaftCluster).collectClusterMetrics() + co.GetCluster().(*RaftCluster).collectStatisticsMetrics() } co.ResetHotSpotMetrics() controller.ResetSchedulerMetrics() - co.GetCluster().(*RaftCluster).resetClusterMetrics() + co.GetCluster().(*RaftCluster).resetStatisticsMetrics() wg.Wait() } @@ -2537,7 +2538,7 @@ func TestCollectMetrics(t *testing.T) { for i := 0; i < 1000; i++ { co.CollectHotSpotMetrics() controller.CollectSchedulerMetrics() - co.GetCluster().(*RaftCluster).collectClusterMetrics() + co.GetCluster().(*RaftCluster).collectStatisticsMetrics() } stores := co.GetCluster().GetStores() regionStats := co.GetCluster().RegionWriteStats() @@ -2552,7 +2553,7 @@ func TestCollectMetrics(t *testing.T) { re.Equal(status1, status2) co.ResetHotSpotMetrics() controller.ResetSchedulerMetrics() - co.GetCluster().(*RaftCluster).resetClusterMetrics() + co.GetCluster().(*RaftCluster).resetStatisticsMetrics() } func prepare(setCfg func(*sc.ScheduleConfig), setTc func(*testCluster), run func(*schedule.Coordinator), re *require.Assertions) (*testCluster, *schedule.Coordinator, func()) { diff --git a/server/cluster/cluster_worker.go b/server/cluster/cluster_worker.go index a38ae86123f..3a319c48196 100644 --- a/server/cluster/cluster_worker.go +++ b/server/cluster/cluster_worker.go @@ -23,6 +23,7 @@ import ( "github.com/pingcap/log" "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/errs" + mcsutils "github.com/tikv/pd/pkg/mcs/utils" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/statistics/buckets" "github.com/tikv/pd/pkg/utils/logutil" @@ -233,7 +234,7 @@ func (c *RaftCluster) HandleReportBuckets(b *metapb.Buckets) error { if err := c.processReportBuckets(b); err != nil { return err } - if !c.isAPIServiceMode { + if !c.IsServiceIndependent(mcsutils.SchedulingServiceName) { c.hotStat.CheckAsync(buckets.NewCheckPeerTask(b)) } return nil diff --git a/server/cluster/scheduling_controller.go b/server/cluster/scheduling_controller.go new file mode 100644 index 00000000000..1c41c830cf6 --- /dev/null +++ b/server/cluster/scheduling_controller.go @@ -0,0 +1,424 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cluster + +import ( + "context" + "net/http" + "sync" + "time" + + "github.com/pingcap/failpoint" + "github.com/pingcap/log" + "github.com/tikv/pd/pkg/core" + "github.com/tikv/pd/pkg/schedule" + "github.com/tikv/pd/pkg/schedule/checker" + "github.com/tikv/pd/pkg/schedule/placement" + "github.com/tikv/pd/pkg/schedule/scatter" + "github.com/tikv/pd/pkg/schedule/schedulers" + "github.com/tikv/pd/pkg/schedule/splitter" + "github.com/tikv/pd/pkg/statistics" + "github.com/tikv/pd/pkg/statistics/buckets" + "github.com/tikv/pd/pkg/statistics/utils" + "github.com/tikv/pd/pkg/utils/logutil" + "github.com/tikv/pd/server/config" +) + +type schedulingController struct { + parentCtx context.Context + ctx context.Context + cancel context.CancelFunc + mu sync.RWMutex + wg sync.WaitGroup + *core.BasicCluster + opt *config.PersistOptions + coordinator *schedule.Coordinator + labelStats *statistics.LabelStatistics + regionStats *statistics.RegionStatistics + hotStat *statistics.HotStat + slowStat *statistics.SlowStat + running bool +} + +func newSchedulingController(parentCtx context.Context) *schedulingController { + ctx, cancel := context.WithCancel(parentCtx) + return &schedulingController{ + parentCtx: parentCtx, + ctx: ctx, + cancel: cancel, + labelStats: statistics.NewLabelStatistics(), + hotStat: statistics.NewHotStat(parentCtx), + slowStat: statistics.NewSlowStat(parentCtx), + } +} + +func (sc *schedulingController) init(basicCluster *core.BasicCluster, opt *config.PersistOptions, coordinator *schedule.Coordinator, ruleManager *placement.RuleManager) { + sc.BasicCluster = basicCluster + sc.opt = opt + sc.coordinator = coordinator + sc.regionStats = statistics.NewRegionStatistics(basicCluster, opt, ruleManager) +} + +func (sc *schedulingController) stopSchedulingJobs() bool { + sc.mu.Lock() + defer sc.mu.Unlock() + if !sc.running { + return false + } + sc.coordinator.Stop() + sc.cancel() + sc.wg.Wait() + sc.running = false + log.Info("scheduling service is stopped") + return true +} + +func (sc *schedulingController) startSchedulingJobs() bool { + sc.mu.Lock() + defer sc.mu.Unlock() + if sc.running { + return false + } + sc.ctx, sc.cancel = context.WithCancel(sc.parentCtx) + sc.wg.Add(3) + go sc.runCoordinator() + go sc.runStatsBackgroundJobs() + go sc.runSchedulingMetricsCollectionJob() + sc.running = true + log.Info("scheduling service is started") + return true +} + +// runCoordinator runs the main scheduling loop. +func (sc *schedulingController) runCoordinator() { + defer logutil.LogPanic() + defer sc.wg.Done() + sc.coordinator.RunUntilStop() +} + +func (sc *schedulingController) runStatsBackgroundJobs() { + defer logutil.LogPanic() + defer sc.wg.Done() + + ticker := time.NewTicker(statistics.RegionsStatsObserveInterval) + defer ticker.Stop() + + for _, store := range sc.GetStores() { + storeID := store.GetID() + sc.hotStat.GetOrCreateRollingStoreStats(storeID) + } + for { + select { + case <-sc.ctx.Done(): + log.Info("statistics background jobs has been stopped") + return + case <-ticker.C: + sc.hotStat.ObserveRegionsStats(sc.GetStoresWriteRate()) + } + } +} + +func (sc *schedulingController) runSchedulingMetricsCollectionJob() { + defer logutil.LogPanic() + defer sc.wg.Done() + + ticker := time.NewTicker(metricsCollectionJobInterval) + failpoint.Inject("highFrequencyClusterJobs", func() { + ticker.Stop() + ticker = time.NewTicker(time.Microsecond) + }) + defer ticker.Stop() + + for { + select { + case <-sc.ctx.Done(): + log.Info("scheduling metrics are reset") + sc.resetSchedulingMetrics() + log.Info("scheduling metrics collection job has been stopped") + return + case <-ticker.C: + sc.collectSchedulingMetrics() + } + } +} + +func (sc *schedulingController) resetSchedulingMetrics() { + statistics.Reset() + sc.coordinator.GetSchedulersController().ResetSchedulerMetrics() + sc.coordinator.ResetHotSpotMetrics() + sc.resetStatisticsMetrics() +} + +func (sc *schedulingController) collectSchedulingMetrics() { + statsMap := statistics.NewStoreStatisticsMap(sc.opt) + stores := sc.GetStores() + for _, s := range stores { + statsMap.Observe(s) + statsMap.ObserveHotStat(s, sc.hotStat.StoresStats) + } + statsMap.Collect() + sc.coordinator.GetSchedulersController().CollectSchedulerMetrics() + sc.coordinator.CollectHotSpotMetrics() + sc.collectStatisticsMetrics() +} + +func (sc *schedulingController) resetStatisticsMetrics() { + if sc.regionStats == nil { + return + } + sc.regionStats.Reset() + sc.labelStats.Reset() + // reset hot cache metrics + sc.hotStat.ResetMetrics() +} + +func (sc *schedulingController) collectStatisticsMetrics() { + if sc.regionStats == nil { + return + } + sc.regionStats.Collect() + sc.labelStats.Collect() + // collect hot cache metrics + sc.hotStat.CollectMetrics() +} + +func (sc *schedulingController) removeStoreStatistics(storeID uint64) { + sc.hotStat.RemoveRollingStoreStats(storeID) + sc.slowStat.RemoveSlowStoreStatus(storeID) +} + +func (sc *schedulingController) updateStoreStatistics(storeID uint64, isSlow bool) { + sc.hotStat.GetOrCreateRollingStoreStats(storeID) + sc.slowStat.ObserveSlowStoreStatus(storeID, isSlow) +} + +// GetHotStat gets hot stat. +func (sc *schedulingController) GetHotStat() *statistics.HotStat { + return sc.hotStat +} + +// GetRegionStats gets region statistics. +func (sc *schedulingController) GetRegionStats() *statistics.RegionStatistics { + return sc.regionStats +} + +// GetLabelStats gets label statistics. +func (sc *schedulingController) GetLabelStats() *statistics.LabelStatistics { + return sc.labelStats +} + +// GetRegionStatsByType gets the status of the region by types. +func (sc *schedulingController) GetRegionStatsByType(typ statistics.RegionStatisticType) []*core.RegionInfo { + if sc.regionStats == nil { + return nil + } + return sc.regionStats.GetRegionStatsByType(typ) +} + +// UpdateRegionsLabelLevelStats updates the status of the region label level by types. +func (sc *schedulingController) UpdateRegionsLabelLevelStats(regions []*core.RegionInfo) { + for _, region := range regions { + sc.labelStats.Observe(region, sc.getStoresWithoutLabelLocked(region, core.EngineKey, core.EngineTiFlash), sc.opt.GetLocationLabels()) + } +} + +func (sc *schedulingController) getStoresWithoutLabelLocked(region *core.RegionInfo, key, value string) []*core.StoreInfo { + stores := make([]*core.StoreInfo, 0, len(region.GetPeers())) + for _, p := range region.GetPeers() { + if store := sc.GetStore(p.StoreId); store != nil && !core.IsStoreContainLabel(store.GetMeta(), key, value) { + stores = append(stores, store) + } + } + return stores +} + +// GetStoresStats returns stores' statistics from cluster. +// And it will be unnecessary to filter unhealthy store, because it has been solved in process heartbeat +func (sc *schedulingController) GetStoresStats() *statistics.StoresStats { + return sc.hotStat.StoresStats +} + +// GetStoresLoads returns load stats of all stores. +func (sc *schedulingController) GetStoresLoads() map[uint64][]float64 { + return sc.hotStat.GetStoresLoads() +} + +// IsRegionHot checks if a region is in hot state. +func (sc *schedulingController) IsRegionHot(region *core.RegionInfo) bool { + return sc.hotStat.IsRegionHot(region, sc.opt.GetHotRegionCacheHitsThreshold()) +} + +// GetHotPeerStat returns hot peer stat with specified regionID and storeID. +func (sc *schedulingController) GetHotPeerStat(rw utils.RWType, regionID, storeID uint64) *statistics.HotPeerStat { + return sc.hotStat.GetHotPeerStat(rw, regionID, storeID) +} + +// RegionReadStats returns hot region's read stats. +// The result only includes peers that are hot enough. +// RegionStats is a thread-safe method +func (sc *schedulingController) RegionReadStats() map[uint64][]*statistics.HotPeerStat { + // As read stats are reported by store heartbeat, the threshold needs to be adjusted. + threshold := sc.opt.GetHotRegionCacheHitsThreshold() * + (utils.RegionHeartBeatReportInterval / utils.StoreHeartBeatReportInterval) + return sc.hotStat.RegionStats(utils.Read, threshold) +} + +// RegionWriteStats returns hot region's write stats. +// The result only includes peers that are hot enough. +func (sc *schedulingController) RegionWriteStats() map[uint64][]*statistics.HotPeerStat { + // RegionStats is a thread-safe method + return sc.hotStat.RegionStats(utils.Write, sc.opt.GetHotRegionCacheHitsThreshold()) +} + +// BucketsStats returns hot region's buckets stats. +func (sc *schedulingController) BucketsStats(degree int, regionIDs ...uint64) map[uint64][]*buckets.BucketStat { + return sc.hotStat.BucketsStats(degree, regionIDs...) +} + +// GetPausedSchedulerDelayAt returns DelayAt of a paused scheduler +func (sc *schedulingController) GetPausedSchedulerDelayAt(name string) (int64, error) { + return sc.coordinator.GetSchedulersController().GetPausedSchedulerDelayAt(name) +} + +// GetPausedSchedulerDelayUntil returns DelayUntil of a paused scheduler +func (sc *schedulingController) GetPausedSchedulerDelayUntil(name string) (int64, error) { + return sc.coordinator.GetSchedulersController().GetPausedSchedulerDelayUntil(name) +} + +// GetRegionScatterer returns the region scatter. +func (sc *schedulingController) GetRegionScatterer() *scatter.RegionScatterer { + return sc.coordinator.GetRegionScatterer() +} + +// GetRegionSplitter returns the region splitter +func (sc *schedulingController) GetRegionSplitter() *splitter.RegionSplitter { + return sc.coordinator.GetRegionSplitter() +} + +// GetMergeChecker returns merge checker. +func (sc *schedulingController) GetMergeChecker() *checker.MergeChecker { + return sc.coordinator.GetMergeChecker() +} + +// GetRuleChecker returns rule checker. +func (sc *schedulingController) GetRuleChecker() *checker.RuleChecker { + return sc.coordinator.GetRuleChecker() +} + +// GetSchedulers gets all schedulers. +func (sc *schedulingController) GetSchedulers() []string { + return sc.coordinator.GetSchedulersController().GetSchedulerNames() +} + +// GetSchedulerHandlers gets all scheduler handlers. +func (sc *schedulingController) GetSchedulerHandlers() map[string]http.Handler { + return sc.coordinator.GetSchedulersController().GetSchedulerHandlers() +} + +// AddSchedulerHandler adds a scheduler handler. +func (sc *schedulingController) AddSchedulerHandler(scheduler schedulers.Scheduler, args ...string) error { + return sc.coordinator.GetSchedulersController().AddSchedulerHandler(scheduler, args...) +} + +// RemoveSchedulerHandler removes a scheduler handler. +func (sc *schedulingController) RemoveSchedulerHandler(name string) error { + return sc.coordinator.GetSchedulersController().RemoveSchedulerHandler(name) +} + +// AddScheduler adds a scheduler. +func (sc *schedulingController) AddScheduler(scheduler schedulers.Scheduler, args ...string) error { + return sc.coordinator.GetSchedulersController().AddScheduler(scheduler, args...) +} + +// RemoveScheduler removes a scheduler. +func (sc *schedulingController) RemoveScheduler(name string) error { + return sc.coordinator.GetSchedulersController().RemoveScheduler(name) +} + +// PauseOrResumeScheduler pauses or resumes a scheduler. +func (sc *schedulingController) PauseOrResumeScheduler(name string, t int64) error { + return sc.coordinator.GetSchedulersController().PauseOrResumeScheduler(name, t) +} + +// PauseOrResumeChecker pauses or resumes checker. +func (sc *schedulingController) PauseOrResumeChecker(name string, t int64) error { + return sc.coordinator.PauseOrResumeChecker(name, t) +} + +// AddSuspectRegions adds regions to suspect list. +func (sc *schedulingController) AddSuspectRegions(regionIDs ...uint64) { + sc.coordinator.GetCheckerController().AddSuspectRegions(regionIDs...) +} + +// GetSuspectRegions gets all suspect regions. +func (sc *schedulingController) GetSuspectRegions() []uint64 { + return sc.coordinator.GetCheckerController().GetSuspectRegions() +} + +// RemoveSuspectRegion removes region from suspect list. +func (sc *schedulingController) RemoveSuspectRegion(id uint64) { + sc.coordinator.GetCheckerController().RemoveSuspectRegion(id) +} + +// PopOneSuspectKeyRange gets one suspect keyRange group. +// it would return value and true if pop success, or return empty [][2][]byte and false +// if suspectKeyRanges couldn't pop keyRange group. +func (sc *schedulingController) PopOneSuspectKeyRange() ([2][]byte, bool) { + return sc.coordinator.GetCheckerController().PopOneSuspectKeyRange() +} + +// ClearSuspectKeyRanges clears the suspect keyRanges, only for unit test +func (sc *schedulingController) ClearSuspectKeyRanges() { + sc.coordinator.GetCheckerController().ClearSuspectKeyRanges() +} + +// AddSuspectKeyRange adds the key range with the its ruleID as the key +// The instance of each keyRange is like following format: +// [2][]byte: start key/end key +func (sc *schedulingController) AddSuspectKeyRange(start, end []byte) { + sc.coordinator.GetCheckerController().AddSuspectKeyRange(start, end) +} + +func (sc *schedulingController) initSchedulers() { + sc.coordinator.InitSchedulers(false) +} + +func (sc *schedulingController) getEvictLeaderStores() (evictStores []uint64) { + if sc.coordinator == nil { + return nil + } + handler, ok := sc.coordinator.GetSchedulersController().GetSchedulerHandlers()[schedulers.EvictLeaderName] + if !ok { + return + } + type evictLeaderHandler interface { + EvictStoreIDs() []uint64 + } + h, ok := handler.(evictLeaderHandler) + if !ok { + return + } + return h.EvictStoreIDs() +} + +// IsPrepared return true if the prepare checker is ready. +func (sc *schedulingController) IsPrepared() bool { + return sc.coordinator.GetPrepareChecker().IsPrepared() +} + +// SetPrepared set the prepare check to prepared. Only for test purpose. +func (sc *schedulingController) SetPrepared() { + sc.coordinator.GetPrepareChecker().SetPrepared() +} diff --git a/server/grpc_service.go b/server/grpc_service.go index 05ec38919cb..34741d4da5b 100644 --- a/server/grpc_service.go +++ b/server/grpc_service.go @@ -1002,7 +1002,7 @@ func (s *GrpcServer) StoreHeartbeat(ctx context.Context, request *pdpb.StoreHear s.handleDamagedStore(request.GetStats()) storeHeartbeatHandleDuration.WithLabelValues(storeAddress, storeLabel).Observe(time.Since(start).Seconds()) - if s.IsAPIServiceMode() { + if s.IsServiceIndependent(utils.SchedulingServiceName) { forwardCli, _ := s.updateSchedulingClient(ctx) if forwardCli != nil { req := &schedulingpb.StoreHeartbeatRequest{ @@ -1360,7 +1360,7 @@ func (s *GrpcServer) RegionHeartbeat(stream pdpb.PD_RegionHeartbeatServer) error continue } - if s.IsAPIServiceMode() { + if s.IsServiceIndependent(utils.SchedulingServiceName) { ctx := stream.Context() primaryAddr, _ := s.GetServicePrimaryAddr(ctx, utils.SchedulingServiceName) if schedulingStream == nil || lastPrimaryAddr != primaryAddr { @@ -1632,7 +1632,7 @@ func (s *GrpcServer) AskSplit(ctx context.Context, request *pdpb.AskSplitRequest // AskBatchSplit implements gRPC PDServer. func (s *GrpcServer) AskBatchSplit(ctx context.Context, request *pdpb.AskBatchSplitRequest) (*pdpb.AskBatchSplitResponse, error) { - if s.IsAPIServiceMode() { + if s.IsServiceIndependent(utils.SchedulingServiceName) { forwardCli, err := s.updateSchedulingClient(ctx) if err != nil { return &pdpb.AskBatchSplitResponse{ @@ -1805,7 +1805,7 @@ func (s *GrpcServer) PutClusterConfig(ctx context.Context, request *pdpb.PutClus // ScatterRegion implements gRPC PDServer. func (s *GrpcServer) ScatterRegion(ctx context.Context, request *pdpb.ScatterRegionRequest) (*pdpb.ScatterRegionResponse, error) { - if s.IsAPIServiceMode() { + if s.IsServiceIndependent(utils.SchedulingServiceName) { forwardCli, err := s.updateSchedulingClient(ctx) if err != nil { return &pdpb.ScatterRegionResponse{ @@ -2028,7 +2028,7 @@ func (s *GrpcServer) UpdateServiceGCSafePoint(ctx context.Context, request *pdpb // GetOperator gets information about the operator belonging to the specify region. func (s *GrpcServer) GetOperator(ctx context.Context, request *pdpb.GetOperatorRequest) (*pdpb.GetOperatorResponse, error) { - if s.IsAPIServiceMode() { + if s.IsServiceIndependent(utils.SchedulingServiceName) { forwardCli, err := s.updateSchedulingClient(ctx) if err != nil { return &pdpb.GetOperatorResponse{ @@ -2300,7 +2300,7 @@ func (s *GrpcServer) SyncMaxTS(_ context.Context, request *pdpb.SyncMaxTSRequest // SplitRegions split regions by the given split keys func (s *GrpcServer) SplitRegions(ctx context.Context, request *pdpb.SplitRegionsRequest) (*pdpb.SplitRegionsResponse, error) { - if s.IsAPIServiceMode() { + if s.IsServiceIndependent(utils.SchedulingServiceName) { forwardCli, err := s.updateSchedulingClient(ctx) if err != nil { return &pdpb.SplitRegionsResponse{ diff --git a/server/handler.go b/server/handler.go index dc4b43238d0..6c0679bd9f9 100644 --- a/server/handler.go +++ b/server/handler.go @@ -30,6 +30,7 @@ import ( "github.com/tikv/pd/pkg/core/storelimit" "github.com/tikv/pd/pkg/encryption" "github.com/tikv/pd/pkg/errs" + mcsutils "github.com/tikv/pd/pkg/mcs/utils" "github.com/tikv/pd/pkg/schedule" sc "github.com/tikv/pd/pkg/schedule/config" sche "github.com/tikv/pd/pkg/schedule/core" @@ -192,7 +193,7 @@ func (h *Handler) AddScheduler(name string, args ...string) error { } var removeSchedulerCb func(string) error - if h.s.IsAPIServiceMode() { + if c.IsServiceIndependent(mcsutils.SchedulingServiceName) { removeSchedulerCb = c.GetCoordinator().GetSchedulersController().RemoveSchedulerHandler } else { removeSchedulerCb = c.GetCoordinator().GetSchedulersController().RemoveScheduler @@ -202,7 +203,7 @@ func (h *Handler) AddScheduler(name string, args ...string) error { return err } log.Info("create scheduler", zap.String("scheduler-name", s.GetName()), zap.Strings("scheduler-args", args)) - if h.s.IsAPIServiceMode() { + if c.IsServiceIndependent(mcsutils.SchedulingServiceName) { if err = c.AddSchedulerHandler(s, args...); err != nil { log.Error("can not add scheduler handler", zap.String("scheduler-name", s.GetName()), zap.Strings("scheduler-args", args), errs.ZapError(err)) return err @@ -229,7 +230,7 @@ func (h *Handler) RemoveScheduler(name string) error { if err != nil { return err } - if h.s.IsAPIServiceMode() { + if c.IsServiceIndependent(mcsutils.SchedulingServiceName) { if err = c.RemoveSchedulerHandler(name); err != nil { log.Error("can not remove scheduler handler", zap.String("scheduler-name", name), errs.ZapError(err)) } else { diff --git a/server/server.go b/server/server.go index 9cd7f18578e..a2c99d0cbec 100644 --- a/server/server.go +++ b/server/server.go @@ -489,7 +489,7 @@ func (s *Server) startServer(ctx context.Context) error { s.safePointV2Manager = gc.NewSafePointManagerV2(s.ctx, s.storage, s.storage, s.storage) s.hbStreams = hbstream.NewHeartbeatStreams(ctx, s.clusterID, "", s.cluster) // initial hot_region_storage in here. - if !s.IsAPIServiceMode() { + if !s.IsServiceIndependent(mcs.SchedulingServiceName) { s.hotRegionStorage, err = storage.NewHotRegionsStorage( ctx, filepath.Join(s.cfg.DataDir, "hot-region"), s.encryptionKeyManager, s.handler) if err != nil { @@ -1394,6 +1394,15 @@ func (s *Server) GetRegions() []*core.RegionInfo { return nil } +// IsServiceIndependent returns if the service is enabled +func (s *Server) IsServiceIndependent(name string) bool { + rc := s.GetRaftCluster() + if rc != nil { + return rc.IsServiceIndependent(name) + } + return false +} + // GetServiceLabels returns ApiAccessPaths by given service label // TODO: this function will be used for updating api rate limit config func (s *Server) GetServiceLabels(serviceLabel string) []apiutil.AccessPath { diff --git a/tests/integrations/mcs/scheduling/config_test.go b/tests/integrations/mcs/scheduling/config_test.go index 8b8e284f765..42ba051eb84 100644 --- a/tests/integrations/mcs/scheduling/config_test.go +++ b/tests/integrations/mcs/scheduling/config_test.go @@ -133,7 +133,6 @@ func persistConfig(re *require.Assertions, pdLeaderServer *tests.TestServer) { func (suite *configTestSuite) TestSchedulerConfigWatch() { re := suite.Require() - // Make sure the config is persisted before the watcher is created. persistConfig(re, suite.pdLeaderServer) // Create a config watcher. diff --git a/tests/server/cluster/cluster_test.go b/tests/server/cluster/cluster_test.go index 701eb9b5d69..ebf0a4e574d 100644 --- a/tests/server/cluster/cluster_test.go +++ b/tests/server/cluster/cluster_test.go @@ -815,7 +815,7 @@ func TestLoadClusterInfo(t *testing.T) { rc := cluster.NewRaftCluster(ctx, svr.ClusterID(), syncer.NewRegionSyncer(svr), svr.GetClient(), svr.GetHTTPClient()) // Cluster is not bootstrapped. - rc.InitCluster(svr.GetAllocator(), svr.GetPersistOptions(), svr.GetStorage(), svr.GetBasicCluster(), svr.GetKeyspaceGroupManager()) + rc.InitCluster(svr.GetAllocator(), svr.GetPersistOptions(), svr.GetStorage(), svr.GetBasicCluster(), svr.GetHBStreams(), svr.GetKeyspaceGroupManager()) raftCluster, err := rc.LoadClusterInfo() re.NoError(err) re.Nil(raftCluster) @@ -853,7 +853,7 @@ func TestLoadClusterInfo(t *testing.T) { re.NoError(testStorage.Flush()) raftCluster = cluster.NewRaftCluster(ctx, svr.ClusterID(), syncer.NewRegionSyncer(svr), svr.GetClient(), svr.GetHTTPClient()) - raftCluster.InitCluster(mockid.NewIDAllocator(), svr.GetPersistOptions(), testStorage, basicCluster, svr.GetKeyspaceGroupManager()) + raftCluster.InitCluster(mockid.NewIDAllocator(), svr.GetPersistOptions(), testStorage, basicCluster, svr.GetHBStreams(), svr.GetKeyspaceGroupManager()) raftCluster, err = raftCluster.LoadClusterInfo() re.NoError(err) re.NotNil(raftCluster) @@ -1561,7 +1561,7 @@ func TestTransferLeaderBack(t *testing.T) { leaderServer := tc.GetLeaderServer() svr := leaderServer.GetServer() rc := cluster.NewRaftCluster(ctx, svr.ClusterID(), syncer.NewRegionSyncer(svr), svr.GetClient(), svr.GetHTTPClient()) - rc.InitCluster(svr.GetAllocator(), svr.GetPersistOptions(), svr.GetStorage(), svr.GetBasicCluster(), svr.GetKeyspaceGroupManager()) + rc.InitCluster(svr.GetAllocator(), svr.GetPersistOptions(), svr.GetStorage(), svr.GetBasicCluster(), svr.GetHBStreams(), svr.GetKeyspaceGroupManager()) storage := rc.GetStorage() meta := &metapb.Cluster{Id: 123} re.NoError(storage.SaveMeta(meta)) From e6e35fdd4eb5b77a5d8b9a5bb96144be3c1461e9 Mon Sep 17 00:00:00 2001 From: Yongbo Jiang Date: Mon, 13 Nov 2023 12:15:43 +0800 Subject: [PATCH 011/137] api: add rule middleware (#7357) ref tikv/pd#5839 Signed-off-by: Cabinfever_B Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- server/api/router.go | 44 ++++----- server/api/rule.go | 220 +++++++++++-------------------------------- 2 files changed, 80 insertions(+), 184 deletions(-) diff --git a/server/api/router.go b/server/api/router.go index 0473e3e1bf7..d3c8f10cbf2 100644 --- a/server/api/router.go +++ b/server/api/router.go @@ -174,29 +174,31 @@ func createRouter(prefix string, svr *server.Server) *mux.Router { registerFunc(apiRouter, "/config/replication-mode", confHandler.SetReplicationModeConfig, setMethods(http.MethodPost), setAuditBackend(localLog, prometheus)) rulesHandler := newRulesHandler(svr, rd) - registerFunc(clusterRouter, "/config/rules", rulesHandler.GetAllRules, setMethods(http.MethodGet), setAuditBackend(prometheus)) - registerFunc(clusterRouter, "/config/rules", rulesHandler.SetAllRules, setMethods(http.MethodPost), setAuditBackend(localLog, prometheus)) - registerFunc(clusterRouter, "/config/rules/batch", rulesHandler.BatchRules, setMethods(http.MethodPost), setAuditBackend(localLog, prometheus)) - registerFunc(clusterRouter, "/config/rules/group/{group}", rulesHandler.GetRuleByGroup, setMethods(http.MethodGet), setAuditBackend(prometheus)) - registerFunc(clusterRouter, "/config/rules/region/{region}", rulesHandler.GetRulesByRegion, setMethods(http.MethodGet), setAuditBackend(prometheus)) - registerFunc(clusterRouter, "/config/rules/region/{region}/detail", rulesHandler.CheckRegionPlacementRule, setMethods(http.MethodGet), setAuditBackend(prometheus)) - registerFunc(clusterRouter, "/config/rules/key/{key}", rulesHandler.GetRulesByKey, setMethods(http.MethodGet), setAuditBackend(prometheus)) - registerFunc(clusterRouter, "/config/rule/{group}/{id}", rulesHandler.GetRuleByGroupAndID, setMethods(http.MethodGet), setAuditBackend(prometheus)) - registerFunc(clusterRouter, "/config/rule", rulesHandler.SetRule, setMethods(http.MethodPost), setAuditBackend(localLog, prometheus)) - registerFunc(clusterRouter, "/config/rule/{group}/{id}", rulesHandler.DeleteRuleByGroup, setMethods(http.MethodDelete), setAuditBackend(localLog, prometheus)) - - registerFunc(clusterRouter, "/config/rule_group/{id}", rulesHandler.GetGroupConfig, setMethods(http.MethodGet), setAuditBackend(prometheus)) - registerFunc(clusterRouter, "/config/rule_group", rulesHandler.SetGroupConfig, setMethods(http.MethodPost), setAuditBackend(localLog, prometheus)) - registerFunc(clusterRouter, "/config/rule_group/{id}", rulesHandler.DeleteGroupConfig, setMethods(http.MethodDelete), setAuditBackend(localLog, prometheus)) - registerFunc(clusterRouter, "/config/rule_groups", rulesHandler.GetAllGroupConfigs, setMethods(http.MethodGet), setAuditBackend(prometheus)) - - registerFunc(clusterRouter, "/config/placement-rule", rulesHandler.GetPlacementRules, setMethods(http.MethodGet), setAuditBackend(prometheus)) - registerFunc(clusterRouter, "/config/placement-rule", rulesHandler.SetPlacementRules, setMethods(http.MethodPost), setAuditBackend(localLog, prometheus)) + ruleRouter := clusterRouter.NewRoute().Subrouter() + ruleRouter.Use(newRuleMiddleware(svr, rd).Middleware) + registerFunc(ruleRouter, "/config/rules", rulesHandler.GetAllRules, setMethods(http.MethodGet), setAuditBackend(prometheus)) + registerFunc(ruleRouter, "/config/rules", rulesHandler.SetAllRules, setMethods(http.MethodPost), setAuditBackend(localLog, prometheus)) + registerFunc(ruleRouter, "/config/rules/batch", rulesHandler.BatchRules, setMethods(http.MethodPost), setAuditBackend(localLog, prometheus)) + registerFunc(ruleRouter, "/config/rules/group/{group}", rulesHandler.GetRuleByGroup, setMethods(http.MethodGet), setAuditBackend(prometheus)) + registerFunc(ruleRouter, "/config/rules/region/{region}", rulesHandler.GetRulesByRegion, setMethods(http.MethodGet), setAuditBackend(prometheus)) + registerFunc(ruleRouter, "/config/rules/region/{region}/detail", rulesHandler.CheckRegionPlacementRule, setMethods(http.MethodGet), setAuditBackend(prometheus)) + registerFunc(ruleRouter, "/config/rules/key/{key}", rulesHandler.GetRulesByKey, setMethods(http.MethodGet), setAuditBackend(prometheus)) + registerFunc(ruleRouter, "/config/rule/{group}/{id}", rulesHandler.GetRuleByGroupAndID, setMethods(http.MethodGet), setAuditBackend(prometheus)) + registerFunc(ruleRouter, "/config/rule", rulesHandler.SetRule, setMethods(http.MethodPost), setAuditBackend(localLog, prometheus)) + registerFunc(ruleRouter, "/config/rule/{group}/{id}", rulesHandler.DeleteRuleByGroup, setMethods(http.MethodDelete), setAuditBackend(localLog, prometheus)) + + registerFunc(ruleRouter, "/config/rule_group/{id}", rulesHandler.GetGroupConfig, setMethods(http.MethodGet), setAuditBackend(prometheus)) + registerFunc(ruleRouter, "/config/rule_group", rulesHandler.SetGroupConfig, setMethods(http.MethodPost), setAuditBackend(localLog, prometheus)) + registerFunc(ruleRouter, "/config/rule_group/{id}", rulesHandler.DeleteGroupConfig, setMethods(http.MethodDelete), setAuditBackend(localLog, prometheus)) + registerFunc(ruleRouter, "/config/rule_groups", rulesHandler.GetAllGroupConfigs, setMethods(http.MethodGet), setAuditBackend(prometheus)) + + registerFunc(ruleRouter, "/config/placement-rule", rulesHandler.GetPlacementRules, setMethods(http.MethodGet), setAuditBackend(prometheus)) + registerFunc(ruleRouter, "/config/placement-rule", rulesHandler.SetPlacementRules, setMethods(http.MethodPost), setAuditBackend(localLog, prometheus)) // {group} can be a regular expression, we should enable path encode to // support special characters. - registerFunc(clusterRouter, "/config/placement-rule/{group}", rulesHandler.GetPlacementRuleByGroup, setMethods(http.MethodGet), setAuditBackend(prometheus)) - registerFunc(clusterRouter, "/config/placement-rule/{group}", rulesHandler.SetPlacementRuleByGroup, setMethods(http.MethodPost), setAuditBackend(localLog, prometheus)) - registerFunc(escapeRouter, "/config/placement-rule/{group}", rulesHandler.DeletePlacementRuleByGroup, setMethods(http.MethodDelete), setAuditBackend(localLog, prometheus)) + registerFunc(ruleRouter, "/config/placement-rule/{group}", rulesHandler.GetPlacementRuleByGroup, setMethods(http.MethodGet), setAuditBackend(prometheus)) + registerFunc(ruleRouter, "/config/placement-rule/{group}", rulesHandler.SetPlacementRuleByGroup, setMethods(http.MethodPost), setAuditBackend(localLog, prometheus)) + registerFunc(ruleRouter, "/config/placement-rule/{group}", rulesHandler.DeletePlacementRuleByGroup, setMethods(http.MethodDelete), setAuditBackend(localLog, prometheus)) regionLabelHandler := newRegionLabelHandler(svr, rd) registerFunc(clusterRouter, "/config/region-label/rules", regionLabelHandler.GetAllRegionLabelRules, setMethods(http.MethodGet), setAuditBackend(prometheus)) diff --git a/server/api/rule.go b/server/api/rule.go index 77aad42eb42..47964d594be 100644 --- a/server/api/rule.go +++ b/server/api/rule.go @@ -15,6 +15,7 @@ package api import ( + "context" "encoding/hex" "fmt" "net/http" @@ -42,6 +43,42 @@ func newRulesHandler(svr *server.Server, rd *render.Render) *ruleHandler { } } +type ruleMiddleware struct { + s *server.Server + rd *render.Render + *server.Handler +} + +func newRuleMiddleware(s *server.Server, rd *render.Render) ruleMiddleware { + return ruleMiddleware{ + s: s, + rd: rd, + Handler: s.GetHandler(), + } +} + +func (m ruleMiddleware) Middleware(h http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + manager, err := m.GetRuleManager() + if err == errs.ErrPlacementDisabled { + m.rd.JSON(w, http.StatusPreconditionFailed, err.Error()) + return + } + if err != nil { + m.rd.JSON(w, http.StatusInternalServerError, err.Error()) + return + } + ctx := context.WithValue(r.Context(), ruleCtxKey{}, manager) + h.ServeHTTP(w, r.WithContext(ctx)) + }) +} + +type ruleCtxKey struct{} + +func getRuleManager(r *http.Request) *placement.RuleManager { + return r.Context().Value(ruleCtxKey{}).(*placement.RuleManager) +} + // @Tags rule // @Summary List all rules of cluster. // @Produce json @@ -50,15 +87,7 @@ func newRulesHandler(svr *server.Server, rd *render.Render) *ruleHandler { // @Failure 500 {string} string "PD server failed to proceed the request." // @Router /config/rules [get] func (h *ruleHandler) GetAllRules(w http.ResponseWriter, r *http.Request) { - manager, err := h.Handler.GetRuleManager() - if err == errs.ErrPlacementDisabled { - h.rd.JSON(w, http.StatusPreconditionFailed, err.Error()) - return - } - if err != nil { - h.rd.JSON(w, http.StatusInternalServerError, err.Error()) - return - } + manager := getRuleManager(r) rules := manager.GetAllRules() h.rd.JSON(w, http.StatusOK, rules) } @@ -73,15 +102,7 @@ func (h *ruleHandler) GetAllRules(w http.ResponseWriter, r *http.Request) { // @Failure 500 {string} string "PD server failed to proceed the request." // @Router /config/rules [post] func (h *ruleHandler) SetAllRules(w http.ResponseWriter, r *http.Request) { - manager, err := h.Handler.GetRuleManager() - if err == errs.ErrPlacementDisabled { - h.rd.JSON(w, http.StatusPreconditionFailed, err.Error()) - return - } - if err != nil { - h.rd.JSON(w, http.StatusInternalServerError, err.Error()) - return - } + manager := getRuleManager(r) var rules []*placement.Rule if err := apiutil.ReadJSONRespondError(h.rd, w, r.Body, &rules); err != nil { return @@ -113,15 +134,7 @@ func (h *ruleHandler) SetAllRules(w http.ResponseWriter, r *http.Request) { // @Failure 500 {string} string "PD server failed to proceed the request." // @Router /config/rules/group/{group} [get] func (h *ruleHandler) GetRuleByGroup(w http.ResponseWriter, r *http.Request) { - manager, err := h.Handler.GetRuleManager() - if err == errs.ErrPlacementDisabled { - h.rd.JSON(w, http.StatusPreconditionFailed, err.Error()) - return - } - if err != nil { - h.rd.JSON(w, http.StatusInternalServerError, err.Error()) - return - } + manager := getRuleManager(r) group := mux.Vars(r)["group"] rules := manager.GetRulesByGroup(group) h.rd.JSON(w, http.StatusOK, rules) @@ -138,15 +151,7 @@ func (h *ruleHandler) GetRuleByGroup(w http.ResponseWriter, r *http.Request) { // @Failure 500 {string} string "PD server failed to proceed the request." // @Router /config/rules/region/{region} [get] func (h *ruleHandler) GetRulesByRegion(w http.ResponseWriter, r *http.Request) { - manager, err := h.Handler.GetRuleManager() - if err == errs.ErrPlacementDisabled { - h.rd.JSON(w, http.StatusPreconditionFailed, err.Error()) - return - } - if err != nil { - h.rd.JSON(w, http.StatusInternalServerError, err.Error()) - return - } + manager := getRuleManager(r) regionStr := mux.Vars(r)["region"] region, code, err := h.PreCheckForRegion(regionStr) if err != nil { @@ -196,15 +201,7 @@ func (h *ruleHandler) CheckRegionPlacementRule(w http.ResponseWriter, r *http.Re // @Failure 500 {string} string "PD server failed to proceed the request." // @Router /config/rules/key/{key} [get] func (h *ruleHandler) GetRulesByKey(w http.ResponseWriter, r *http.Request) { - manager, err := h.Handler.GetRuleManager() - if err == errs.ErrPlacementDisabled { - h.rd.JSON(w, http.StatusPreconditionFailed, err.Error()) - return - } - if err != nil { - h.rd.JSON(w, http.StatusInternalServerError, err.Error()) - return - } + manager := getRuleManager(r) keyHex := mux.Vars(r)["key"] key, err := hex.DecodeString(keyHex) if err != nil { @@ -225,15 +222,7 @@ func (h *ruleHandler) GetRulesByKey(w http.ResponseWriter, r *http.Request) { // @Failure 412 {string} string "Placement rules feature is disabled." // @Router /config/rule/{group}/{id} [get] func (h *ruleHandler) GetRuleByGroupAndID(w http.ResponseWriter, r *http.Request) { - manager, err := h.Handler.GetRuleManager() - if err == errs.ErrPlacementDisabled { - h.rd.JSON(w, http.StatusPreconditionFailed, err.Error()) - return - } - if err != nil { - h.rd.JSON(w, http.StatusInternalServerError, err.Error()) - return - } + manager := getRuleManager(r) group, id := mux.Vars(r)["group"], mux.Vars(r)["id"] rule := manager.GetRule(group, id) if rule == nil { @@ -254,15 +243,7 @@ func (h *ruleHandler) GetRuleByGroupAndID(w http.ResponseWriter, r *http.Request // @Failure 500 {string} string "PD server failed to proceed the request." // @Router /config/rule [post] func (h *ruleHandler) SetRule(w http.ResponseWriter, r *http.Request) { - manager, err := h.Handler.GetRuleManager() - if err == errs.ErrPlacementDisabled { - h.rd.JSON(w, http.StatusPreconditionFailed, err.Error()) - return - } - if err != nil { - h.rd.JSON(w, http.StatusInternalServerError, err.Error()) - return - } + manager := getRuleManager(r) var rule placement.Rule if err := apiutil.ReadJSONRespondError(h.rd, w, r.Body, &rule); err != nil { return @@ -312,15 +293,7 @@ func (h *ruleHandler) syncReplicateConfigWithDefaultRule(rule *placement.Rule) e // @Failure 500 {string} string "PD server failed to proceed the request." // @Router /config/rule/{group}/{id} [delete] func (h *ruleHandler) DeleteRuleByGroup(w http.ResponseWriter, r *http.Request) { - manager, err := h.Handler.GetRuleManager() - if err == errs.ErrPlacementDisabled { - h.rd.JSON(w, http.StatusPreconditionFailed, err.Error()) - return - } - if err != nil { - h.rd.JSON(w, http.StatusInternalServerError, err.Error()) - return - } + manager := getRuleManager(r) group, id := mux.Vars(r)["group"], mux.Vars(r)["id"] rule := manager.GetRule(group, id) if err := manager.DeleteRule(group, id); err != nil { @@ -345,15 +318,7 @@ func (h *ruleHandler) DeleteRuleByGroup(w http.ResponseWriter, r *http.Request) // @Failure 500 {string} string "PD server failed to proceed the request." // @Router /config/rules/batch [post] func (h *ruleHandler) BatchRules(w http.ResponseWriter, r *http.Request) { - manager, err := h.Handler.GetRuleManager() - if err == errs.ErrPlacementDisabled { - h.rd.JSON(w, http.StatusPreconditionFailed, err.Error()) - return - } - if err != nil { - h.rd.JSON(w, http.StatusInternalServerError, err.Error()) - return - } + manager := getRuleManager(r) var opts []placement.RuleOp if err := apiutil.ReadJSONRespondError(h.rd, w, r.Body, &opts); err != nil { return @@ -380,15 +345,7 @@ func (h *ruleHandler) BatchRules(w http.ResponseWriter, r *http.Request) { // @Failure 500 {string} string "PD server failed to proceed the request." // @Router /config/rule_group/{id} [get] func (h *ruleHandler) GetGroupConfig(w http.ResponseWriter, r *http.Request) { - manager, err := h.Handler.GetRuleManager() - if err == errs.ErrPlacementDisabled { - h.rd.JSON(w, http.StatusPreconditionFailed, err.Error()) - return - } - if err != nil { - h.rd.JSON(w, http.StatusInternalServerError, err.Error()) - return - } + manager := getRuleManager(r) id := mux.Vars(r)["id"] group := manager.GetRuleGroup(id) if group == nil { @@ -409,15 +366,7 @@ func (h *ruleHandler) GetGroupConfig(w http.ResponseWriter, r *http.Request) { // @Failure 500 {string} string "PD server failed to proceed the request." // @Router /config/rule_group [post] func (h *ruleHandler) SetGroupConfig(w http.ResponseWriter, r *http.Request) { - manager, err := h.Handler.GetRuleManager() - if err == errs.ErrPlacementDisabled { - h.rd.JSON(w, http.StatusPreconditionFailed, err.Error()) - return - } - if err != nil { - h.rd.JSON(w, http.StatusInternalServerError, err.Error()) - return - } + manager := getRuleManager(r) var ruleGroup placement.RuleGroup if err := apiutil.ReadJSONRespondError(h.rd, w, r.Body, &ruleGroup); err != nil { return @@ -442,17 +391,9 @@ func (h *ruleHandler) SetGroupConfig(w http.ResponseWriter, r *http.Request) { // @Failure 500 {string} string "PD server failed to proceed the request." // @Router /config/rule_group/{id} [delete] func (h *ruleHandler) DeleteGroupConfig(w http.ResponseWriter, r *http.Request) { - manager, err := h.Handler.GetRuleManager() - if err == errs.ErrPlacementDisabled { - h.rd.JSON(w, http.StatusPreconditionFailed, err.Error()) - return - } - if err != nil { - h.rd.JSON(w, http.StatusInternalServerError, err.Error()) - return - } + manager := getRuleManager(r) id := mux.Vars(r)["id"] - err = manager.DeleteRuleGroup(id) + err := manager.DeleteRuleGroup(id) if err != nil { h.rd.JSON(w, http.StatusInternalServerError, err.Error()) return @@ -472,15 +413,7 @@ func (h *ruleHandler) DeleteGroupConfig(w http.ResponseWriter, r *http.Request) // @Failure 500 {string} string "PD server failed to proceed the request." // @Router /config/rule_groups [get] func (h *ruleHandler) GetAllGroupConfigs(w http.ResponseWriter, r *http.Request) { - manager, err := h.Handler.GetRuleManager() - if err == errs.ErrPlacementDisabled { - h.rd.JSON(w, http.StatusPreconditionFailed, err.Error()) - return - } - if err != nil { - h.rd.JSON(w, http.StatusInternalServerError, err.Error()) - return - } + manager := getRuleManager(r) ruleGroups := manager.GetRuleGroups() h.rd.JSON(w, http.StatusOK, ruleGroups) } @@ -493,15 +426,7 @@ func (h *ruleHandler) GetAllGroupConfigs(w http.ResponseWriter, r *http.Request) // @Failure 500 {string} string "PD server failed to proceed the request." // @Router /config/placement-rule [get] func (h *ruleHandler) GetPlacementRules(w http.ResponseWriter, r *http.Request) { - manager, err := h.Handler.GetRuleManager() - if err == errs.ErrPlacementDisabled { - h.rd.JSON(w, http.StatusPreconditionFailed, err.Error()) - return - } - if err != nil { - h.rd.JSON(w, http.StatusInternalServerError, err.Error()) - return - } + manager := getRuleManager(r) bundles := manager.GetAllGroupBundles() h.rd.JSON(w, http.StatusOK, bundles) } @@ -516,15 +441,7 @@ func (h *ruleHandler) GetPlacementRules(w http.ResponseWriter, r *http.Request) // @Failure 500 {string} string "PD server failed to proceed the request." // @Router /config/placement-rule [post] func (h *ruleHandler) SetPlacementRules(w http.ResponseWriter, r *http.Request) { - manager, err := h.Handler.GetRuleManager() - if err == errs.ErrPlacementDisabled { - h.rd.JSON(w, http.StatusPreconditionFailed, err.Error()) - return - } - if err != nil { - h.rd.JSON(w, http.StatusInternalServerError, err.Error()) - return - } + manager := getRuleManager(r) var groups []placement.GroupBundle if err := apiutil.ReadJSONRespondError(h.rd, w, r.Body, &groups); err != nil { return @@ -551,15 +468,7 @@ func (h *ruleHandler) SetPlacementRules(w http.ResponseWriter, r *http.Request) // @Failure 500 {string} string "PD server failed to proceed the request." // @Router /config/placement-rule/{group} [get] func (h *ruleHandler) GetPlacementRuleByGroup(w http.ResponseWriter, r *http.Request) { - manager, err := h.Handler.GetRuleManager() - if err == errs.ErrPlacementDisabled { - h.rd.JSON(w, http.StatusPreconditionFailed, err.Error()) - return - } - if err != nil { - h.rd.JSON(w, http.StatusInternalServerError, err.Error()) - return - } + manager := getRuleManager(r) g := mux.Vars(r)["group"] group := manager.GetGroupBundle(g) h.rd.JSON(w, http.StatusOK, group) @@ -576,16 +485,9 @@ func (h *ruleHandler) GetPlacementRuleByGroup(w http.ResponseWriter, r *http.Req // @Failure 500 {string} string "PD server failed to proceed the request." // @Router /config/placement-rule [delete] func (h *ruleHandler) DeletePlacementRuleByGroup(w http.ResponseWriter, r *http.Request) { - manager, err := h.Handler.GetRuleManager() - if err == errs.ErrPlacementDisabled { - h.rd.JSON(w, http.StatusPreconditionFailed, err.Error()) - return - } - if err != nil { - h.rd.JSON(w, http.StatusInternalServerError, err.Error()) - return - } + manager := getRuleManager(r) group := mux.Vars(r)["group"] + var err error group, err = url.PathUnescape(group) if err != nil { h.rd.JSON(w, http.StatusBadRequest, err.Error()) @@ -608,15 +510,7 @@ func (h *ruleHandler) DeletePlacementRuleByGroup(w http.ResponseWriter, r *http. // @Failure 500 {string} string "PD server failed to proceed the request." // @Router /config/placement-rule/{group} [post] func (h *ruleHandler) SetPlacementRuleByGroup(w http.ResponseWriter, r *http.Request) { - manager, err := h.Handler.GetRuleManager() - if err == errs.ErrPlacementDisabled { - h.rd.JSON(w, http.StatusPreconditionFailed, err.Error()) - return - } - if err != nil { - h.rd.JSON(w, http.StatusInternalServerError, err.Error()) - return - } + manager := getRuleManager(r) groupID := mux.Vars(r)["group"] var group placement.GroupBundle if err := apiutil.ReadJSONRespondError(h.rd, w, r.Body, &group); err != nil { From 7dbe6079ca2f303a2f5886ee7e8aac6c54c2532d Mon Sep 17 00:00:00 2001 From: JmPotato Date: Mon, 13 Nov 2023 13:41:13 +0800 Subject: [PATCH 012/137] client: introduce the HTTP client (#7304) ref tikv/pd#7300 Introduce the HTTP client. Signed-off-by: JmPotato Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- client/client.go | 4 +- client/http/api.go | 54 +++ client/http/client.go | 337 ++++++++++++++++++ client/http/types.go | 178 +++++++++ client/http/types_test.go | 49 +++ tests/integrations/client/http_client_test.go | 87 +++++ 6 files changed, 707 insertions(+), 2 deletions(-) create mode 100644 client/http/api.go create mode 100644 client/http/client.go create mode 100644 client/http/types.go create mode 100644 client/http/types_test.go create mode 100644 tests/integrations/client/http_client_test.go diff --git a/client/client.go b/client/client.go index 067872d2d39..56923b697e2 100644 --- a/client/client.go +++ b/client/client.go @@ -74,7 +74,7 @@ type GlobalConfigItem struct { PayLoad []byte } -// Client is a PD (Placement Driver) client. +// Client is a PD (Placement Driver) RPC client. // It should not be used after calling Close(). type Client interface { // GetClusterID gets the cluster ID from PD. @@ -1062,7 +1062,7 @@ func (c *client) ScanRegions(ctx context.Context, key, endKey []byte, limit int) defer span.Finish() } start := time.Now() - defer cmdDurationScanRegions.Observe(time.Since(start).Seconds()) + defer func() { cmdDurationScanRegions.Observe(time.Since(start).Seconds()) }() var cancel context.CancelFunc scanCtx := ctx diff --git a/client/http/api.go b/client/http/api.go new file mode 100644 index 00000000000..5326919561d --- /dev/null +++ b/client/http/api.go @@ -0,0 +1,54 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package http + +import ( + "fmt" + "net/url" +) + +// The following constants are the paths of PD HTTP APIs. +const ( + HotRead = "/pd/api/v1/hotspot/regions/read" + HotWrite = "/pd/api/v1/hotspot/regions/write" + Regions = "/pd/api/v1/regions" + regionByID = "/pd/api/v1/region/id" + regionByKey = "/pd/api/v1/region/key" + regionsByKey = "/pd/api/v1/regions/key" + regionsByStoreID = "/pd/api/v1/regions/store" + Stores = "/pd/api/v1/stores" + MinResolvedTSPrefix = "/pd/api/v1/min-resolved-ts" +) + +// RegionByID returns the path of PD HTTP API to get region by ID. +func RegionByID(regionID uint64) string { + return fmt.Sprintf("%s/%d", regionByID, regionID) +} + +// RegionByKey returns the path of PD HTTP API to get region by key. +func RegionByKey(key []byte) string { + return fmt.Sprintf("%s/%s", regionByKey, url.QueryEscape(string(key))) +} + +// RegionsByKey returns the path of PD HTTP API to scan regions with given start key, end key and limit parameters. +func RegionsByKey(startKey, endKey []byte, limit int) string { + return fmt.Sprintf("%s?start_key=%s&end_key=%s&limit=%d", + regionsByKey, url.QueryEscape(string(startKey)), url.QueryEscape(string(endKey)), limit) +} + +// RegionsByStoreID returns the path of PD HTTP API to get regions by store ID. +func RegionsByStoreID(storeID uint64) string { + return fmt.Sprintf("%s/%d", regionsByStoreID, storeID) +} diff --git a/client/http/client.go b/client/http/client.go new file mode 100644 index 00000000000..6cb1277dfcb --- /dev/null +++ b/client/http/client.go @@ -0,0 +1,337 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package http + +import ( + "context" + "crypto/tls" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" + + "github.com/pingcap/errors" + "github.com/pingcap/log" + "github.com/prometheus/client_golang/prometheus" + "go.uber.org/zap" +) + +const ( + httpScheme = "http" + httpsScheme = "https" + networkErrorStatus = "network error" + + defaultTimeout = 30 * time.Second +) + +// Client is a PD (Placement Driver) HTTP client. +type Client interface { + GetRegionByID(context.Context, uint64) (*RegionInfo, error) + GetRegionByKey(context.Context, []byte) (*RegionInfo, error) + GetRegions(context.Context) (*RegionsInfo, error) + GetRegionsByKey(context.Context, []byte, []byte, int) (*RegionsInfo, error) + GetRegionsByStoreID(context.Context, uint64) (*RegionsInfo, error) + GetHotReadRegions(context.Context) (*StoreHotPeersInfos, error) + GetHotWriteRegions(context.Context) (*StoreHotPeersInfos, error) + GetStores(context.Context) (*StoresInfo, error) + GetMinResolvedTSByStoresIDs(context.Context, []uint64) (uint64, map[uint64]uint64, error) + Close() +} + +var _ Client = (*client)(nil) + +type client struct { + pdAddrs []string + tlsConf *tls.Config + cli *http.Client + + requestCounter *prometheus.CounterVec + executionDuration *prometheus.HistogramVec +} + +// ClientOption configures the HTTP client. +type ClientOption func(c *client) + +// WithHTTPClient configures the client with the given initialized HTTP client. +func WithHTTPClient(cli *http.Client) ClientOption { + return func(c *client) { + c.cli = cli + } +} + +// WithTLSConfig configures the client with the given TLS config. +// This option won't work if the client is configured with WithHTTPClient. +func WithTLSConfig(tlsConf *tls.Config) ClientOption { + return func(c *client) { + c.tlsConf = tlsConf + } +} + +// WithMetrics configures the client with metrics. +func WithMetrics( + requestCounter *prometheus.CounterVec, + executionDuration *prometheus.HistogramVec, +) ClientOption { + return func(c *client) { + c.requestCounter = requestCounter + c.executionDuration = executionDuration + } +} + +// NewClient creates a PD HTTP client with the given PD addresses and TLS config. +func NewClient( + pdAddrs []string, + opts ...ClientOption, +) Client { + c := &client{} + // Apply the options first. + for _, opt := range opts { + opt(c) + } + // Normalize the addresses with correct scheme prefix. + for i, addr := range pdAddrs { + if !strings.HasPrefix(addr, httpScheme) { + var scheme string + if c.tlsConf != nil { + scheme = httpsScheme + } else { + scheme = httpScheme + } + pdAddrs[i] = fmt.Sprintf("%s://%s", scheme, addr) + } + } + c.pdAddrs = pdAddrs + // Init the HTTP client if it's not configured. + if c.cli == nil { + c.cli = &http.Client{Timeout: defaultTimeout} + if c.tlsConf != nil { + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.TLSClientConfig = c.tlsConf + c.cli.Transport = transport + } + } + + return c +} + +// Close closes the HTTP client. +func (c *client) Close() { + if c.cli != nil { + c.cli.CloseIdleConnections() + } + log.Info("[pd] http client closed") +} + +func (c *client) reqCounter(name, status string) { + if c.requestCounter == nil { + return + } + c.requestCounter.WithLabelValues(name, status).Inc() +} + +func (c *client) execDuration(name string, duration time.Duration) { + if c.executionDuration == nil { + return + } + c.executionDuration.WithLabelValues(name).Observe(duration.Seconds()) +} + +// At present, we will use the retry strategy of polling by default to keep +// it consistent with the current implementation of some clients (e.g. TiDB). +func (c *client) requestWithRetry( + ctx context.Context, + name, uri string, + res interface{}, +) error { + var ( + err error + addr string + ) + for idx := 0; idx < len(c.pdAddrs); idx++ { + addr = c.pdAddrs[idx] + err = c.request(ctx, name, addr, uri, res) + if err == nil { + break + } + log.Debug("[pd] request one addr failed", + zap.Int("idx", idx), zap.String("addr", addr), zap.Error(err)) + } + return err +} + +func (c *client) request( + ctx context.Context, + name, addr, uri string, + res interface{}, +) error { + reqURL := fmt.Sprintf("%s%s", addr, uri) + logFields := []zap.Field{ + zap.String("name", name), + zap.String("url", reqURL), + } + log.Debug("[pd] request the http url", logFields...) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, reqURL, nil) + if err != nil { + log.Error("[pd] create http request failed", append(logFields, zap.Error(err))...) + return errors.Trace(err) + } + start := time.Now() + resp, err := c.cli.Do(req) + if err != nil { + c.reqCounter(name, networkErrorStatus) + log.Error("[pd] do http request failed", append(logFields, zap.Error(err))...) + return errors.Trace(err) + } + c.execDuration(name, time.Since(start)) + c.reqCounter(name, resp.Status) + defer func() { + err = resp.Body.Close() + if err != nil { + log.Warn("[pd] close http response body failed", append(logFields, zap.Error(err))...) + } + }() + + if resp.StatusCode != http.StatusOK { + logFields = append(logFields, zap.String("status", resp.Status)) + + bs, readErr := io.ReadAll(resp.Body) + if readErr != nil { + logFields = append(logFields, zap.NamedError("read-body-error", err)) + } else { + logFields = append(logFields, zap.ByteString("body", bs)) + } + + log.Error("[pd] request failed with a non-200 status", logFields...) + return errors.Errorf("request pd http api failed with status: '%s'", resp.Status) + } + + err = json.NewDecoder(resp.Body).Decode(res) + if err != nil { + return errors.Trace(err) + } + return nil +} + +// GetRegionByID gets the region info by ID. +func (c *client) GetRegionByID(ctx context.Context, regionID uint64) (*RegionInfo, error) { + var region RegionInfo + err := c.requestWithRetry(ctx, "GetRegionByID", RegionByID(regionID), ®ion) + if err != nil { + return nil, err + } + return ®ion, nil +} + +// GetRegionByKey gets the region info by key. +func (c *client) GetRegionByKey(ctx context.Context, key []byte) (*RegionInfo, error) { + var region RegionInfo + err := c.requestWithRetry(ctx, "GetRegionByKey", RegionByKey(key), ®ion) + if err != nil { + return nil, err + } + return ®ion, nil +} + +// GetRegions gets the regions info. +func (c *client) GetRegions(ctx context.Context) (*RegionsInfo, error) { + var regions RegionsInfo + err := c.requestWithRetry(ctx, "GetRegions", Regions, ®ions) + if err != nil { + return nil, err + } + return ®ions, nil +} + +// GetRegionsByKey gets the regions info by key range. If the limit is -1, it will return all regions within the range. +func (c *client) GetRegionsByKey(ctx context.Context, startKey, endKey []byte, limit int) (*RegionsInfo, error) { + var regions RegionsInfo + err := c.requestWithRetry(ctx, "GetRegionsByKey", RegionsByKey(startKey, endKey, limit), ®ions) + if err != nil { + return nil, err + } + return ®ions, nil +} + +// GetRegionsByStoreID gets the regions info by store ID. +func (c *client) GetRegionsByStoreID(ctx context.Context, storeID uint64) (*RegionsInfo, error) { + var regions RegionsInfo + err := c.requestWithRetry(ctx, "GetRegionsByStoreID", RegionsByStoreID(storeID), ®ions) + if err != nil { + return nil, err + } + return ®ions, nil +} + +// GetHotReadRegions gets the hot read region statistics info. +func (c *client) GetHotReadRegions(ctx context.Context) (*StoreHotPeersInfos, error) { + var hotReadRegions StoreHotPeersInfos + err := c.requestWithRetry(ctx, "GetHotReadRegions", HotRead, &hotReadRegions) + if err != nil { + return nil, err + } + return &hotReadRegions, nil +} + +// GetHotWriteRegions gets the hot write region statistics info. +func (c *client) GetHotWriteRegions(ctx context.Context) (*StoreHotPeersInfos, error) { + var hotWriteRegions StoreHotPeersInfos + err := c.requestWithRetry(ctx, "GetHotWriteRegions", HotWrite, &hotWriteRegions) + if err != nil { + return nil, err + } + return &hotWriteRegions, nil +} + +// GetStores gets the stores info. +func (c *client) GetStores(ctx context.Context) (*StoresInfo, error) { + var stores StoresInfo + err := c.requestWithRetry(ctx, "GetStores", Stores, &stores) + if err != nil { + return nil, err + } + return &stores, nil +} + +// GetMinResolvedTSByStoresIDs get min-resolved-ts by stores IDs. +func (c *client) GetMinResolvedTSByStoresIDs(ctx context.Context, storeIDs []uint64) (uint64, map[uint64]uint64, error) { + uri := MinResolvedTSPrefix + // scope is an optional parameter, it can be `cluster` or specified store IDs. + // - When no scope is given, cluster-level's min_resolved_ts will be returned and storesMinResolvedTS will be nil. + // - When scope is `cluster`, cluster-level's min_resolved_ts will be returned and storesMinResolvedTS will be filled. + // - When scope given a list of stores, min_resolved_ts will be provided for each store + // and the scope-specific min_resolved_ts will be returned. + if len(storeIDs) != 0 { + storeIDStrs := make([]string, len(storeIDs)) + for idx, id := range storeIDs { + storeIDStrs[idx] = fmt.Sprintf("%d", id) + } + uri = fmt.Sprintf("%s?scope=%s", uri, strings.Join(storeIDStrs, ",")) + } + resp := struct { + MinResolvedTS uint64 `json:"min_resolved_ts"` + IsRealTime bool `json:"is_real_time,omitempty"` + StoresMinResolvedTS map[uint64]uint64 `json:"stores_min_resolved_ts"` + }{} + err := c.requestWithRetry(ctx, "GetMinResolvedTSByStoresIDs", uri, &resp) + if err != nil { + return 0, nil, err + } + if !resp.IsRealTime { + return 0, nil, errors.Trace(errors.New("min resolved ts is not enabled")) + } + return resp.MinResolvedTS, resp.StoresMinResolvedTS, nil +} diff --git a/client/http/types.go b/client/http/types.go new file mode 100644 index 00000000000..66eb31ec3a1 --- /dev/null +++ b/client/http/types.go @@ -0,0 +1,178 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package http + +import "time" + +// NOTICE: the structures below are copied from the PD API definitions. +// Please make sure the consistency if any change happens to the PD API. + +// RegionInfo stores the information of one region. +type RegionInfo struct { + ID int64 `json:"id"` + StartKey string `json:"start_key"` + EndKey string `json:"end_key"` + Epoch RegionEpoch `json:"epoch"` + Peers []RegionPeer `json:"peers"` + Leader RegionPeer `json:"leader"` + DownPeers []RegionPeerStat `json:"down_peers"` + PendingPeers []RegionPeer `json:"pending_peers"` + WrittenBytes uint64 `json:"written_bytes"` + ReadBytes uint64 `json:"read_bytes"` + ApproximateSize int64 `json:"approximate_size"` + ApproximateKeys int64 `json:"approximate_keys"` + + ReplicationStatus *ReplicationStatus `json:"replication_status,omitempty"` +} + +// GetStartKey gets the start key of the region. +func (r *RegionInfo) GetStartKey() string { return r.StartKey } + +// GetEndKey gets the end key of the region. +func (r *RegionInfo) GetEndKey() string { return r.EndKey } + +// RegionEpoch stores the information about its epoch. +type RegionEpoch struct { + ConfVer int64 `json:"conf_ver"` + Version int64 `json:"version"` +} + +// RegionPeer stores information of one peer. +type RegionPeer struct { + ID int64 `json:"id"` + StoreID int64 `json:"store_id"` + IsLearner bool `json:"is_learner"` +} + +// RegionPeerStat stores one field `DownSec` which indicates how long it's down than `RegionPeer`. +type RegionPeerStat struct { + Peer RegionPeer `json:"peer"` + DownSec int64 `json:"down_seconds"` +} + +// ReplicationStatus represents the replication mode status of the region. +type ReplicationStatus struct { + State string `json:"state"` + StateID int64 `json:"state_id"` +} + +// RegionsInfo stores the information of regions. +type RegionsInfo struct { + Count int64 `json:"count"` + Regions []RegionInfo `json:"regions"` +} + +// Merge merges two RegionsInfo together and returns a new one. +func (ri *RegionsInfo) Merge(other *RegionsInfo) *RegionsInfo { + newRegionsInfo := &RegionsInfo{ + Regions: make([]RegionInfo, 0, ri.Count+other.Count), + } + m := make(map[int64]RegionInfo, ri.Count+other.Count) + for _, region := range ri.Regions { + m[region.ID] = region + } + for _, region := range other.Regions { + m[region.ID] = region + } + for _, region := range m { + newRegionsInfo.Regions = append(newRegionsInfo.Regions, region) + } + newRegionsInfo.Count = int64(len(newRegionsInfo.Regions)) + return newRegionsInfo +} + +// StoreHotPeersInfos is used to get human-readable description for hot regions. +type StoreHotPeersInfos struct { + AsPeer StoreHotPeersStat `json:"as_peer"` + AsLeader StoreHotPeersStat `json:"as_leader"` +} + +// StoreHotPeersStat is used to record the hot region statistics group by store. +type StoreHotPeersStat map[uint64]*HotPeersStat + +// HotPeersStat records all hot regions statistics +type HotPeersStat struct { + StoreByteRate float64 `json:"store_bytes"` + StoreKeyRate float64 `json:"store_keys"` + StoreQueryRate float64 `json:"store_query"` + TotalBytesRate float64 `json:"total_flow_bytes"` + TotalKeysRate float64 `json:"total_flow_keys"` + TotalQueryRate float64 `json:"total_flow_query"` + Count int `json:"regions_count"` + Stats []HotPeerStatShow `json:"statistics"` +} + +// HotPeerStatShow records the hot region statistics for output +type HotPeerStatShow struct { + StoreID uint64 `json:"store_id"` + Stores []uint64 `json:"stores"` + IsLeader bool `json:"is_leader"` + IsLearner bool `json:"is_learner"` + RegionID uint64 `json:"region_id"` + HotDegree int `json:"hot_degree"` + ByteRate float64 `json:"flow_bytes"` + KeyRate float64 `json:"flow_keys"` + QueryRate float64 `json:"flow_query"` + AntiCount int `json:"anti_count"` + LastUpdateTime time.Time `json:"last_update_time,omitempty"` +} + +// StoresInfo represents the information of all TiKV/TiFlash stores. +type StoresInfo struct { + Count int `json:"count"` + Stores []StoreInfo `json:"stores"` +} + +// StoreInfo represents the information of one TiKV/TiFlash store. +type StoreInfo struct { + Store MetaStore `json:"store"` + Status StoreStatus `json:"status"` +} + +// MetaStore represents the meta information of one store. +type MetaStore struct { + ID int64 `json:"id"` + Address string `json:"address"` + State int64 `json:"state"` + StateName string `json:"state_name"` + Version string `json:"version"` + Labels []StoreLabel `json:"labels"` + StatusAddress string `json:"status_address"` + GitHash string `json:"git_hash"` + StartTimestamp int64 `json:"start_timestamp"` +} + +// StoreLabel stores the information of one store label. +type StoreLabel struct { + Key string `json:"key"` + Value string `json:"value"` +} + +// StoreStatus stores the detail information of one store. +type StoreStatus struct { + Capacity string `json:"capacity"` + Available string `json:"available"` + LeaderCount int64 `json:"leader_count"` + LeaderWeight float64 `json:"leader_weight"` + LeaderScore float64 `json:"leader_score"` + LeaderSize int64 `json:"leader_size"` + RegionCount int64 `json:"region_count"` + RegionWeight float64 `json:"region_weight"` + RegionScore float64 `json:"region_score"` + RegionSize int64 `json:"region_size"` + StartTS time.Time `json:"start_ts"` + LastHeartbeatTS time.Time `json:"last_heartbeat_ts"` + Uptime string `json:"uptime"` +} diff --git a/client/http/types_test.go b/client/http/types_test.go new file mode 100644 index 00000000000..0dfebacbdcf --- /dev/null +++ b/client/http/types_test.go @@ -0,0 +1,49 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package http + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestMergeRegionsInfo(t *testing.T) { + re := require.New(t) + regionsInfo1 := &RegionsInfo{ + Count: 1, + Regions: []RegionInfo{ + { + ID: 1, + StartKey: "", + EndKey: "a", + }, + }, + } + regionsInfo2 := &RegionsInfo{ + Count: 1, + Regions: []RegionInfo{ + { + ID: 2, + StartKey: "a", + EndKey: "", + }, + }, + } + regionsInfo := regionsInfo1.Merge(regionsInfo2) + re.Equal(int64(2), regionsInfo.Count) + re.Equal(2, len(regionsInfo.Regions)) + re.Equal(append(regionsInfo1.Regions, regionsInfo2.Regions...), regionsInfo.Regions) +} diff --git a/tests/integrations/client/http_client_test.go b/tests/integrations/client/http_client_test.go new file mode 100644 index 00000000000..03d90c6cd32 --- /dev/null +++ b/tests/integrations/client/http_client_test.go @@ -0,0 +1,87 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package client_test + +import ( + "context" + "math" + "testing" + + "github.com/stretchr/testify/suite" + pd "github.com/tikv/pd/client/http" + "github.com/tikv/pd/tests" +) + +type httpClientTestSuite struct { + suite.Suite + ctx context.Context + cancelFunc context.CancelFunc + cluster *tests.TestCluster + client pd.Client +} + +func TestHTTPClientTestSuite(t *testing.T) { + suite.Run(t, new(httpClientTestSuite)) +} + +func (suite *httpClientTestSuite) SetupSuite() { + re := suite.Require() + var err error + suite.ctx, suite.cancelFunc = context.WithCancel(context.Background()) + suite.cluster, err = tests.NewTestCluster(suite.ctx, 1) + re.NoError(err) + err = suite.cluster.RunInitialServers() + re.NoError(err) + leader := suite.cluster.WaitLeader() + re.NotEmpty(leader) + err = suite.cluster.GetLeaderServer().BootstrapCluster() + re.NoError(err) + var ( + testServers = suite.cluster.GetServers() + endpoints = make([]string, 0, len(testServers)) + ) + for _, s := range testServers { + endpoints = append(endpoints, s.GetConfig().AdvertiseClientUrls) + } + suite.client = pd.NewClient(endpoints) +} + +func (suite *httpClientTestSuite) TearDownSuite() { + suite.cancelFunc() + suite.client.Close() + suite.cluster.Destroy() +} + +func (suite *httpClientTestSuite) TestGetMinResolvedTSByStoresIDs() { + re := suite.Require() + // Get the cluster-level min resolved TS. + minResolvedTS, storeMinResolvedTSMap, err := suite.client.GetMinResolvedTSByStoresIDs(suite.ctx, nil) + re.NoError(err) + re.Greater(minResolvedTS, uint64(0)) + re.Empty(storeMinResolvedTSMap) + // Get the store-level min resolved TS. + minResolvedTS, storeMinResolvedTSMap, err = suite.client.GetMinResolvedTSByStoresIDs(suite.ctx, []uint64{1}) + re.NoError(err) + re.Greater(minResolvedTS, uint64(0)) + re.Len(storeMinResolvedTSMap, 1) + re.Equal(minResolvedTS, storeMinResolvedTSMap[1]) + // Get the store-level min resolved TS with an invalid store ID. + minResolvedTS, storeMinResolvedTSMap, err = suite.client.GetMinResolvedTSByStoresIDs(suite.ctx, []uint64{1, 2}) + re.NoError(err) + re.Greater(minResolvedTS, uint64(0)) + re.Len(storeMinResolvedTSMap, 2) + re.Equal(minResolvedTS, storeMinResolvedTSMap[1]) + re.Equal(uint64(math.MaxUint64), storeMinResolvedTSMap[2]) +} From be31c08186fa2b6c154532d1130e5727a4631473 Mon Sep 17 00:00:00 2001 From: JmPotato Date: Mon, 13 Nov 2023 17:24:44 +0800 Subject: [PATCH 013/137] resource_controller: prevent loadServerConfig from panic (#7361) close tikv/pd#7360 Prevent `loadServerConfig` from panic. Signed-off-by: JmPotato --- client/resource_group/controller/controller.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/client/resource_group/controller/controller.go b/client/resource_group/controller/controller.go index e3495a21ff1..b528351bedf 100755 --- a/client/resource_group/controller/controller.go +++ b/client/resource_group/controller/controller.go @@ -171,12 +171,13 @@ func loadServerConfig(ctx context.Context, provider ResourceGroupProvider) (*Con if err != nil { return nil, err } - if len(resp.Kvs) == 0 { + kvs := resp.GetKvs() + if len(kvs) == 0 { log.Warn("[resource group controller] server does not save config, load config failed") return DefaultConfig(), nil } config := &Config{} - err = json.Unmarshal(resp.Kvs[0].GetValue(), config) + err = json.Unmarshal(kvs[0].GetValue(), config) if err != nil { return nil, err } From 8dcd49720cd9999119d27212220bc0b03f82a75e Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Tue, 14 Nov 2023 11:29:14 +0800 Subject: [PATCH 014/137] *: Improve region forward (#7305) ref tikv/pd#5839 Signed-off-by: Ryan Leung Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- server/forward.go | 504 +++++++++++++++++++++++ server/grpc_service.go | 891 ++++++++++++----------------------------- 2 files changed, 750 insertions(+), 645 deletions(-) create mode 100644 server/forward.go diff --git a/server/forward.go b/server/forward.go new file mode 100644 index 00000000000..e765d442539 --- /dev/null +++ b/server/forward.go @@ -0,0 +1,504 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "context" + "io" + "strings" + "time" + + "github.com/pingcap/errors" + "github.com/pingcap/failpoint" + "github.com/pingcap/kvproto/pkg/pdpb" + "github.com/pingcap/kvproto/pkg/schedulingpb" + "github.com/pingcap/kvproto/pkg/tsopb" + "github.com/pingcap/log" + "github.com/tikv/pd/pkg/errs" + "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/tso" + "github.com/tikv/pd/pkg/utils/grpcutil" + "github.com/tikv/pd/pkg/utils/logutil" + "github.com/tikv/pd/pkg/utils/tsoutil" + "go.uber.org/zap" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func (s *GrpcServer) forwardTSORequest( + ctx context.Context, + request *pdpb.TsoRequest, + forwardStream tsopb.TSO_TsoClient) (*tsopb.TsoResponse, error) { + tsopbReq := &tsopb.TsoRequest{ + Header: &tsopb.RequestHeader{ + ClusterId: request.GetHeader().GetClusterId(), + SenderId: request.GetHeader().GetSenderId(), + KeyspaceId: utils.DefaultKeyspaceID, + KeyspaceGroupId: utils.DefaultKeyspaceGroupID, + }, + Count: request.GetCount(), + DcLocation: request.GetDcLocation(), + } + + failpoint.Inject("tsoProxySendToTSOTimeout", func() { + // block until watchDeadline routine cancels the context. + <-ctx.Done() + }) + + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + } + + if err := forwardStream.Send(tsopbReq); err != nil { + return nil, err + } + + failpoint.Inject("tsoProxyRecvFromTSOTimeout", func() { + // block until watchDeadline routine cancels the context. + <-ctx.Done() + }) + + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + } + + return forwardStream.Recv() +} + +// forwardTSO forward the TSO requests to the TSO service. +func (s *GrpcServer) forwardTSO(stream pdpb.PD_TsoServer) error { + var ( + server = &tsoServer{stream: stream} + forwardStream tsopb.TSO_TsoClient + forwardCtx context.Context + cancelForward context.CancelFunc + lastForwardedHost string + ) + defer func() { + s.concurrentTSOProxyStreamings.Add(-1) + if cancelForward != nil { + cancelForward() + } + }() + + maxConcurrentTSOProxyStreamings := int32(s.GetMaxConcurrentTSOProxyStreamings()) + if maxConcurrentTSOProxyStreamings >= 0 { + if newCount := s.concurrentTSOProxyStreamings.Add(1); newCount > maxConcurrentTSOProxyStreamings { + return errors.WithStack(ErrMaxCountTSOProxyRoutinesExceeded) + } + } + + tsDeadlineCh := make(chan *tsoutil.TSDeadline, 1) + go tsoutil.WatchTSDeadline(stream.Context(), tsDeadlineCh) + + for { + select { + case <-s.ctx.Done(): + return errors.WithStack(s.ctx.Err()) + case <-stream.Context().Done(): + return stream.Context().Err() + default: + } + + request, err := server.Recv(s.GetTSOProxyRecvFromClientTimeout()) + if err == io.EOF { + return nil + } + if err != nil { + return errors.WithStack(err) + } + if request.GetCount() == 0 { + err = errs.ErrGenerateTimestamp.FastGenByArgs("tso count should be positive") + return status.Errorf(codes.Unknown, err.Error()) + } + + forwardedHost, ok := s.GetServicePrimaryAddr(stream.Context(), utils.TSOServiceName) + if !ok || len(forwardedHost) == 0 { + return errors.WithStack(ErrNotFoundTSOAddr) + } + if forwardStream == nil || lastForwardedHost != forwardedHost { + if cancelForward != nil { + cancelForward() + } + + clientConn, err := s.getDelegateClient(s.ctx, forwardedHost) + if err != nil { + return errors.WithStack(err) + } + forwardStream, forwardCtx, cancelForward, err = s.createTSOForwardStream(stream.Context(), clientConn) + if err != nil { + return errors.WithStack(err) + } + lastForwardedHost = forwardedHost + } + + tsopbResp, err := s.forwardTSORequestWithDeadLine(forwardCtx, cancelForward, forwardStream, request, tsDeadlineCh) + if err != nil { + return errors.WithStack(err) + } + + // The error types defined for tsopb and pdpb are different, so we need to convert them. + var pdpbErr *pdpb.Error + tsopbErr := tsopbResp.GetHeader().GetError() + if tsopbErr != nil { + if tsopbErr.Type == tsopb.ErrorType_OK { + pdpbErr = &pdpb.Error{ + Type: pdpb.ErrorType_OK, + Message: tsopbErr.GetMessage(), + } + } else { + // TODO: specify FORWARD FAILURE error type instead of UNKNOWN. + pdpbErr = &pdpb.Error{ + Type: pdpb.ErrorType_UNKNOWN, + Message: tsopbErr.GetMessage(), + } + } + } + + response := &pdpb.TsoResponse{ + Header: &pdpb.ResponseHeader{ + ClusterId: tsopbResp.GetHeader().GetClusterId(), + Error: pdpbErr, + }, + Count: tsopbResp.GetCount(), + Timestamp: tsopbResp.GetTimestamp(), + } + if err := server.Send(response); err != nil { + return errors.WithStack(err) + } + } +} + +func (s *GrpcServer) forwardTSORequestWithDeadLine( + forwardCtx context.Context, + cancelForward context.CancelFunc, + forwardStream tsopb.TSO_TsoClient, + request *pdpb.TsoRequest, + tsDeadlineCh chan<- *tsoutil.TSDeadline) (*tsopb.TsoResponse, error) { + done := make(chan struct{}) + dl := tsoutil.NewTSDeadline(tsoutil.DefaultTSOProxyTimeout, done, cancelForward) + select { + case tsDeadlineCh <- dl: + case <-forwardCtx.Done(): + return nil, forwardCtx.Err() + } + + start := time.Now() + resp, err := s.forwardTSORequest(forwardCtx, request, forwardStream) + close(done) + if err != nil { + if strings.Contains(err.Error(), errs.NotLeaderErr) { + s.tsoPrimaryWatcher.ForceLoad() + } + return nil, err + } + tsoProxyBatchSize.Observe(float64(request.GetCount())) + tsoProxyHandleDuration.Observe(time.Since(start).Seconds()) + return resp, nil +} + +func (s *GrpcServer) createTSOForwardStream(ctx context.Context, client *grpc.ClientConn) (tsopb.TSO_TsoClient, context.Context, context.CancelFunc, error) { + done := make(chan struct{}) + forwardCtx, cancelForward := context.WithCancel(ctx) + go grpcutil.CheckStream(forwardCtx, cancelForward, done) + forwardStream, err := tsopb.NewTSOClient(client).Tso(forwardCtx) + done <- struct{}{} + return forwardStream, forwardCtx, cancelForward, err +} + +func (s *GrpcServer) createRegionHeartbeatForwardStream(client *grpc.ClientConn) (pdpb.PD_RegionHeartbeatClient, context.CancelFunc, error) { + done := make(chan struct{}) + ctx, cancel := context.WithCancel(s.ctx) + go grpcutil.CheckStream(ctx, cancel, done) + forwardStream, err := pdpb.NewPDClient(client).RegionHeartbeat(ctx) + done <- struct{}{} + return forwardStream, cancel, err +} + +func (s *GrpcServer) createRegionHeartbeatSchedulingStream(ctx context.Context, client *grpc.ClientConn) (schedulingpb.Scheduling_RegionHeartbeatClient, context.Context, context.CancelFunc, error) { + done := make(chan struct{}) + forwardCtx, cancelForward := context.WithCancel(ctx) + go grpcutil.CheckStream(forwardCtx, cancelForward, done) + forwardStream, err := schedulingpb.NewSchedulingClient(client).RegionHeartbeat(forwardCtx) + done <- struct{}{} + return forwardStream, forwardCtx, cancelForward, err +} + +func forwardRegionHeartbeatToScheduling(forwardStream schedulingpb.Scheduling_RegionHeartbeatClient, server *heartbeatServer, errCh chan error) { + defer logutil.LogPanic() + defer close(errCh) + for { + resp, err := forwardStream.Recv() + if err == io.EOF { + errCh <- errors.WithStack(err) + return + } + if err != nil { + errCh <- errors.WithStack(err) + return + } + // The error types defined for schedulingpb and pdpb are different, so we need to convert them. + var pdpbErr *pdpb.Error + schedulingpbErr := resp.GetHeader().GetError() + if schedulingpbErr != nil { + if schedulingpbErr.Type == schedulingpb.ErrorType_OK { + pdpbErr = &pdpb.Error{ + Type: pdpb.ErrorType_OK, + Message: schedulingpbErr.GetMessage(), + } + } else { + // TODO: specify FORWARD FAILURE error type instead of UNKNOWN. + pdpbErr = &pdpb.Error{ + Type: pdpb.ErrorType_UNKNOWN, + Message: schedulingpbErr.GetMessage(), + } + } + } + response := &pdpb.RegionHeartbeatResponse{ + Header: &pdpb.ResponseHeader{ + ClusterId: resp.GetHeader().GetClusterId(), + Error: pdpbErr, + }, + ChangePeer: resp.GetChangePeer(), + TransferLeader: resp.GetTransferLeader(), + RegionId: resp.GetRegionId(), + RegionEpoch: resp.GetRegionEpoch(), + TargetPeer: resp.GetTargetPeer(), + Merge: resp.GetMerge(), + SplitRegion: resp.GetSplitRegion(), + ChangePeerV2: resp.GetChangePeerV2(), + SwitchWitnesses: resp.GetSwitchWitnesses(), + } + + if err := server.Send(response); err != nil { + errCh <- errors.WithStack(err) + return + } + } +} + +func forwardRegionHeartbeatClientToServer(forwardStream pdpb.PD_RegionHeartbeatClient, server *heartbeatServer, errCh chan error) { + defer logutil.LogPanic() + defer close(errCh) + for { + resp, err := forwardStream.Recv() + if err != nil { + errCh <- errors.WithStack(err) + return + } + if err := server.Send(resp); err != nil { + errCh <- errors.WithStack(err) + return + } + } +} + +func forwardReportBucketClientToServer(forwardStream pdpb.PD_ReportBucketsClient, server *bucketHeartbeatServer, errCh chan error) { + defer logutil.LogPanic() + defer close(errCh) + for { + resp, err := forwardStream.CloseAndRecv() + if err != nil { + errCh <- errors.WithStack(err) + return + } + if err := server.Send(resp); err != nil { + errCh <- errors.WithStack(err) + return + } + } +} + +func (s *GrpcServer) createReportBucketsForwardStream(client *grpc.ClientConn) (pdpb.PD_ReportBucketsClient, context.CancelFunc, error) { + done := make(chan struct{}) + ctx, cancel := context.WithCancel(s.ctx) + go grpcutil.CheckStream(ctx, cancel, done) + forwardStream, err := pdpb.NewPDClient(client).ReportBuckets(ctx) + done <- struct{}{} + return forwardStream, cancel, err +} + +func (s *GrpcServer) getDelegateClient(ctx context.Context, forwardedHost string) (*grpc.ClientConn, error) { + client, ok := s.clientConns.Load(forwardedHost) + if ok { + // Mostly, the connection is already established, and return it directly. + return client.(*grpc.ClientConn), nil + } + + tlsConfig, err := s.GetTLSConfig().ToTLSConfig() + if err != nil { + return nil, err + } + ctxTimeout, cancel := context.WithTimeout(ctx, defaultGRPCDialTimeout) + defer cancel() + newConn, err := grpcutil.GetClientConn(ctxTimeout, forwardedHost, tlsConfig) + if err != nil { + return nil, err + } + conn, loaded := s.clientConns.LoadOrStore(forwardedHost, newConn) + if !loaded { + // Successfully stored the connection we created. + return newConn, nil + } + // Loaded a connection created/stored by another goroutine, so close the one we created + // and return the one we loaded. + newConn.Close() + return conn.(*grpc.ClientConn), nil +} + +func (s *GrpcServer) getForwardedHost(ctx, streamCtx context.Context, serviceName ...string) (forwardedHost string, err error) { + if s.IsAPIServiceMode() { + var ok bool + if len(serviceName) == 0 { + return "", ErrNotFoundService + } + forwardedHost, ok = s.GetServicePrimaryAddr(ctx, serviceName[0]) + if !ok || len(forwardedHost) == 0 { + switch serviceName[0] { + case utils.TSOServiceName: + return "", ErrNotFoundTSOAddr + case utils.SchedulingServiceName: + return "", ErrNotFoundSchedulingAddr + } + } + } else if fh := grpcutil.GetForwardedHost(streamCtx); !s.isLocalRequest(fh) { + forwardedHost = fh + } + return forwardedHost, nil +} + +func (s *GrpcServer) isLocalRequest(forwardedHost string) bool { + failpoint.Inject("useForwardRequest", func() { + failpoint.Return(false) + }) + if forwardedHost == "" { + return true + } + memberAddrs := s.GetMember().Member().GetClientUrls() + for _, addr := range memberAddrs { + if addr == forwardedHost { + return true + } + } + return false +} + +func (s *GrpcServer) getGlobalTSO(ctx context.Context) (pdpb.Timestamp, error) { + if !s.IsAPIServiceMode() { + return s.tsoAllocatorManager.HandleRequest(ctx, tso.GlobalDCLocation, 1) + } + request := &tsopb.TsoRequest{ + Header: &tsopb.RequestHeader{ + ClusterId: s.clusterID, + KeyspaceId: utils.DefaultKeyspaceID, + KeyspaceGroupId: utils.DefaultKeyspaceGroupID, + }, + Count: 1, + } + var ( + forwardedHost string + forwardStream tsopb.TSO_TsoClient + ts *tsopb.TsoResponse + err error + ok bool + ) + handleStreamError := func(err error) (needRetry bool) { + if strings.Contains(err.Error(), errs.NotLeaderErr) { + s.tsoPrimaryWatcher.ForceLoad() + log.Warn("force to load tso primary address due to error", zap.Error(err), zap.String("tso-addr", forwardedHost)) + return true + } + if grpcutil.NeedRebuildConnection(err) { + s.tsoClientPool.Lock() + delete(s.tsoClientPool.clients, forwardedHost) + s.tsoClientPool.Unlock() + log.Warn("client connection removed due to error", zap.Error(err), zap.String("tso-addr", forwardedHost)) + return true + } + return false + } + for i := 0; i < maxRetryTimesRequestTSOServer; i++ { + if i > 0 { + time.Sleep(retryIntervalRequestTSOServer) + } + forwardedHost, ok = s.GetServicePrimaryAddr(ctx, utils.TSOServiceName) + if !ok || forwardedHost == "" { + return pdpb.Timestamp{}, ErrNotFoundTSOAddr + } + forwardStream, err = s.getTSOForwardStream(forwardedHost) + if err != nil { + return pdpb.Timestamp{}, err + } + err = forwardStream.Send(request) + if err != nil { + if needRetry := handleStreamError(err); needRetry { + continue + } + log.Error("send request to tso primary server failed", zap.Error(err), zap.String("tso-addr", forwardedHost)) + return pdpb.Timestamp{}, err + } + ts, err = forwardStream.Recv() + if err != nil { + if needRetry := handleStreamError(err); needRetry { + continue + } + log.Error("receive response from tso primary server failed", zap.Error(err), zap.String("tso-addr", forwardedHost)) + return pdpb.Timestamp{}, err + } + return *ts.GetTimestamp(), nil + } + log.Error("get global tso from tso primary server failed after retry", zap.Error(err), zap.String("tso-addr", forwardedHost)) + return pdpb.Timestamp{}, err +} + +func (s *GrpcServer) getTSOForwardStream(forwardedHost string) (tsopb.TSO_TsoClient, error) { + s.tsoClientPool.RLock() + forwardStream, ok := s.tsoClientPool.clients[forwardedHost] + s.tsoClientPool.RUnlock() + if ok { + // This is the common case to return here + return forwardStream, nil + } + + s.tsoClientPool.Lock() + defer s.tsoClientPool.Unlock() + + // Double check after entering the critical section + forwardStream, ok = s.tsoClientPool.clients[forwardedHost] + if ok { + return forwardStream, nil + } + + // Now let's create the client connection and the forward stream + client, err := s.getDelegateClient(s.ctx, forwardedHost) + if err != nil { + return nil, err + } + done := make(chan struct{}) + ctx, cancel := context.WithCancel(s.ctx) + go grpcutil.CheckStream(ctx, cancel, done) + forwardStream, err = tsopb.NewTSOClient(client).Tso(ctx) + done <- struct{}{} + if err != nil { + return nil, err + } + s.tsoClientPool.clients[forwardedHost] = forwardStream + return forwardStream, nil +} diff --git a/server/grpc_service.go b/server/grpc_service.go index 34741d4da5b..b0384a7d629 100644 --- a/server/grpc_service.go +++ b/server/grpc_service.go @@ -70,6 +70,7 @@ var ( ErrSendHeartbeatTimeout = status.Errorf(codes.DeadlineExceeded, "send heartbeat timeout") ErrNotFoundTSOAddr = status.Errorf(codes.NotFound, "not found tso address") ErrNotFoundSchedulingAddr = status.Errorf(codes.NotFound, "not found scheduling address") + ErrNotFoundService = status.Errorf(codes.NotFound, "not found service") ErrForwardTSOTimeout = status.Errorf(codes.DeadlineExceeded, "forward tso request timeout") ErrMaxCountTSOProxyRoutinesExceeded = status.Errorf(codes.ResourceExhausted, "max count of concurrent tso proxy routines exceeded") ErrTSOProxyRecvFromClientTimeout = status.Errorf(codes.DeadlineExceeded, "tso proxy timeout when receiving from client; stream closed by server") @@ -83,9 +84,120 @@ type GrpcServer struct { concurrentTSOProxyStreamings atomic.Int32 } +// tsoServer wraps PD_TsoServer to ensure when any error +// occurs on Send() or Recv(), both endpoints will be closed. +type tsoServer struct { + stream pdpb.PD_TsoServer + closed int32 +} + +type pdpbTSORequest struct { + request *pdpb.TsoRequest + err error +} + +func (s *tsoServer) Send(m *pdpb.TsoResponse) error { + if atomic.LoadInt32(&s.closed) == 1 { + return io.EOF + } + done := make(chan error, 1) + go func() { + defer logutil.LogPanic() + failpoint.Inject("tsoProxyFailToSendToClient", func() { + done <- errors.New("injected error") + failpoint.Return() + }) + done <- s.stream.Send(m) + }() + timer := time.NewTimer(tsoutil.DefaultTSOProxyTimeout) + defer timer.Stop() + select { + case err := <-done: + if err != nil { + atomic.StoreInt32(&s.closed, 1) + } + return errors.WithStack(err) + case <-timer.C: + atomic.StoreInt32(&s.closed, 1) + return ErrForwardTSOTimeout + } +} + +func (s *tsoServer) Recv(timeout time.Duration) (*pdpb.TsoRequest, error) { + if atomic.LoadInt32(&s.closed) == 1 { + return nil, io.EOF + } + failpoint.Inject("tsoProxyRecvFromClientTimeout", func(val failpoint.Value) { + if customTimeoutInSeconds, ok := val.(int); ok { + timeout = time.Duration(customTimeoutInSeconds) * time.Second + } + }) + requestCh := make(chan *pdpbTSORequest, 1) + go func() { + defer logutil.LogPanic() + request, err := s.stream.Recv() + requestCh <- &pdpbTSORequest{request: request, err: err} + }() + timer := time.NewTimer(timeout) + defer timer.Stop() + select { + case req := <-requestCh: + if req.err != nil { + atomic.StoreInt32(&s.closed, 1) + return nil, errors.WithStack(req.err) + } + return req.request, nil + case <-timer.C: + atomic.StoreInt32(&s.closed, 1) + return nil, ErrTSOProxyRecvFromClientTimeout + } +} + +// heartbeatServer wraps PD_RegionHeartbeatServer to ensure when any error +// occurs on Send() or Recv(), both endpoints will be closed. +type heartbeatServer struct { + stream pdpb.PD_RegionHeartbeatServer + closed int32 +} + +func (s *heartbeatServer) Send(m core.RegionHeartbeatResponse) error { + if atomic.LoadInt32(&s.closed) == 1 { + return io.EOF + } + done := make(chan error, 1) + go func() { + defer logutil.LogPanic() + done <- s.stream.Send(m.(*pdpb.RegionHeartbeatResponse)) + }() + timer := time.NewTimer(heartbeatSendTimeout) + defer timer.Stop() + select { + case err := <-done: + if err != nil { + atomic.StoreInt32(&s.closed, 1) + } + return errors.WithStack(err) + case <-timer.C: + atomic.StoreInt32(&s.closed, 1) + return ErrSendHeartbeatTimeout + } +} + +func (s *heartbeatServer) Recv() (*pdpb.RegionHeartbeatRequest, error) { + if atomic.LoadInt32(&s.closed) == 1 { + return nil, io.EOF + } + req, err := s.stream.Recv() + if err != nil { + atomic.StoreInt32(&s.closed, 1) + return nil, errors.WithStack(err) + } + return req, nil +} + type schedulingClient struct { - client schedulingpb.SchedulingClient - lastPrimary string + client schedulingpb.SchedulingClient + primary string } func (s *schedulingClient) getClient() schedulingpb.SchedulingClient { @@ -99,7 +211,7 @@ func (s *schedulingClient) getPrimaryAddr() string { if s == nil { return "" } - return s.lastPrimary + return s.primary } type request interface { @@ -393,7 +505,7 @@ func (s *GrpcServer) Tso(stream pdpb.PD_TsoServer) error { return errors.WithStack(err) } - if forwardedHost, err := s.getForwardedHost(ctx, stream.Context()); err != nil { + if forwardedHost, err := s.getForwardedHost(ctx, stream.Context(), utils.TSOServiceName); err != nil { return err } else if len(forwardedHost) > 0 { clientConn, err := s.getDelegateClient(s.ctx, forwardedHost) @@ -440,268 +552,6 @@ func (s *GrpcServer) Tso(stream pdpb.PD_TsoServer) error { } } -// forwardTSO forward the TSO requests to the TSO service. -func (s *GrpcServer) forwardTSO(stream pdpb.PD_TsoServer) error { - var ( - server = &tsoServer{stream: stream} - forwardStream tsopb.TSO_TsoClient - forwardCtx context.Context - cancelForward context.CancelFunc - lastForwardedHost string - ) - defer func() { - s.concurrentTSOProxyStreamings.Add(-1) - if cancelForward != nil { - cancelForward() - } - }() - - maxConcurrentTSOProxyStreamings := int32(s.GetMaxConcurrentTSOProxyStreamings()) - if maxConcurrentTSOProxyStreamings >= 0 { - if newCount := s.concurrentTSOProxyStreamings.Add(1); newCount > maxConcurrentTSOProxyStreamings { - return errors.WithStack(ErrMaxCountTSOProxyRoutinesExceeded) - } - } - - tsDeadlineCh := make(chan *tsoutil.TSDeadline, 1) - go tsoutil.WatchTSDeadline(stream.Context(), tsDeadlineCh) - - for { - select { - case <-s.ctx.Done(): - return errors.WithStack(s.ctx.Err()) - case <-stream.Context().Done(): - return stream.Context().Err() - default: - } - - request, err := server.Recv(s.GetTSOProxyRecvFromClientTimeout()) - if err == io.EOF { - return nil - } - if err != nil { - return errors.WithStack(err) - } - if request.GetCount() == 0 { - err = errs.ErrGenerateTimestamp.FastGenByArgs("tso count should be positive") - return status.Errorf(codes.Unknown, err.Error()) - } - - forwardedHost, ok := s.GetServicePrimaryAddr(stream.Context(), utils.TSOServiceName) - if !ok || len(forwardedHost) == 0 { - return errors.WithStack(ErrNotFoundTSOAddr) - } - if forwardStream == nil || lastForwardedHost != forwardedHost { - if cancelForward != nil { - cancelForward() - } - - clientConn, err := s.getDelegateClient(s.ctx, forwardedHost) - if err != nil { - return errors.WithStack(err) - } - forwardStream, forwardCtx, cancelForward, err = - s.createTSOForwardStream(stream.Context(), clientConn) - if err != nil { - return errors.WithStack(err) - } - lastForwardedHost = forwardedHost - } - - tsopbResp, err := s.forwardTSORequestWithDeadLine( - forwardCtx, cancelForward, forwardStream, request, tsDeadlineCh) - if err != nil { - return errors.WithStack(err) - } - - // The error types defined for tsopb and pdpb are different, so we need to convert them. - var pdpbErr *pdpb.Error - tsopbErr := tsopbResp.GetHeader().GetError() - if tsopbErr != nil { - if tsopbErr.Type == tsopb.ErrorType_OK { - pdpbErr = &pdpb.Error{ - Type: pdpb.ErrorType_OK, - Message: tsopbErr.GetMessage(), - } - } else { - // TODO: specify FORWARD FAILURE error type instead of UNKNOWN. - pdpbErr = &pdpb.Error{ - Type: pdpb.ErrorType_UNKNOWN, - Message: tsopbErr.GetMessage(), - } - } - } - - response := &pdpb.TsoResponse{ - Header: &pdpb.ResponseHeader{ - ClusterId: tsopbResp.GetHeader().GetClusterId(), - Error: pdpbErr, - }, - Count: tsopbResp.GetCount(), - Timestamp: tsopbResp.GetTimestamp(), - } - if err := server.Send(response); err != nil { - return errors.WithStack(err) - } - } -} - -func (s *GrpcServer) forwardTSORequestWithDeadLine( - forwardCtx context.Context, - cancelForward context.CancelFunc, - forwardStream tsopb.TSO_TsoClient, - request *pdpb.TsoRequest, - tsDeadlineCh chan<- *tsoutil.TSDeadline, -) (*tsopb.TsoResponse, error) { - done := make(chan struct{}) - dl := tsoutil.NewTSDeadline(tsoutil.DefaultTSOProxyTimeout, done, cancelForward) - select { - case tsDeadlineCh <- dl: - case <-forwardCtx.Done(): - return nil, forwardCtx.Err() - } - - start := time.Now() - resp, err := s.forwardTSORequest(forwardCtx, request, forwardStream) - close(done) - if err != nil { - if strings.Contains(err.Error(), errs.NotLeaderErr) { - s.tsoPrimaryWatcher.ForceLoad() - } - return nil, err - } - tsoProxyBatchSize.Observe(float64(request.GetCount())) - tsoProxyHandleDuration.Observe(time.Since(start).Seconds()) - return resp, nil -} - -func (s *GrpcServer) forwardTSORequest( - ctx context.Context, - request *pdpb.TsoRequest, - forwardStream tsopb.TSO_TsoClient, -) (*tsopb.TsoResponse, error) { - tsopbReq := &tsopb.TsoRequest{ - Header: &tsopb.RequestHeader{ - ClusterId: request.GetHeader().GetClusterId(), - SenderId: request.GetHeader().GetSenderId(), - KeyspaceId: utils.DefaultKeyspaceID, - KeyspaceGroupId: utils.DefaultKeyspaceGroupID, - }, - Count: request.GetCount(), - DcLocation: request.GetDcLocation(), - } - - failpoint.Inject("tsoProxySendToTSOTimeout", func() { - // block until watchDeadline routine cancels the context. - <-ctx.Done() - }) - - select { - case <-ctx.Done(): - return nil, ctx.Err() - default: - } - - if err := forwardStream.Send(tsopbReq); err != nil { - return nil, err - } - - failpoint.Inject("tsoProxyRecvFromTSOTimeout", func() { - // block until watchDeadline routine cancels the context. - <-ctx.Done() - }) - - select { - case <-ctx.Done(): - return nil, ctx.Err() - default: - } - - return forwardStream.Recv() -} - -// tsoServer wraps PD_TsoServer to ensure when any error -// occurs on Send() or Recv(), both endpoints will be closed. -type tsoServer struct { - stream pdpb.PD_TsoServer - closed int32 -} - -type pdpbTSORequest struct { - request *pdpb.TsoRequest - err error -} - -func (s *tsoServer) Send(m *pdpb.TsoResponse) error { - if atomic.LoadInt32(&s.closed) == 1 { - return io.EOF - } - done := make(chan error, 1) - go func() { - defer logutil.LogPanic() - failpoint.Inject("tsoProxyFailToSendToClient", func() { - done <- errors.New("injected error") - failpoint.Return() - }) - done <- s.stream.Send(m) - }() - timer := time.NewTimer(tsoutil.DefaultTSOProxyTimeout) - defer timer.Stop() - select { - case err := <-done: - if err != nil { - atomic.StoreInt32(&s.closed, 1) - } - return errors.WithStack(err) - case <-timer.C: - atomic.StoreInt32(&s.closed, 1) - return ErrForwardTSOTimeout - } -} - -func (s *tsoServer) Recv(timeout time.Duration) (*pdpb.TsoRequest, error) { - if atomic.LoadInt32(&s.closed) == 1 { - return nil, io.EOF - } - failpoint.Inject("tsoProxyRecvFromClientTimeout", func(val failpoint.Value) { - if customTimeoutInSeconds, ok := val.(int); ok { - timeout = time.Duration(customTimeoutInSeconds) * time.Second - } - }) - requestCh := make(chan *pdpbTSORequest, 1) - go func() { - defer logutil.LogPanic() - request, err := s.stream.Recv() - requestCh <- &pdpbTSORequest{request: request, err: err} - }() - timer := time.NewTimer(timeout) - defer timer.Stop() - select { - case req := <-requestCh: - if req.err != nil { - atomic.StoreInt32(&s.closed, 1) - return nil, errors.WithStack(req.err) - } - return req.request, nil - case <-timer.C: - atomic.StoreInt32(&s.closed, 1) - return nil, ErrTSOProxyRecvFromClientTimeout - } -} - -func (s *GrpcServer) getForwardedHost(ctx, streamCtx context.Context) (forwardedHost string, err error) { - if s.IsAPIServiceMode() { - var ok bool - forwardedHost, ok = s.GetServicePrimaryAddr(ctx, utils.TSOServiceName) - if !ok || len(forwardedHost) == 0 { - return "", ErrNotFoundTSOAddr - } - } else if fh := grpcutil.GetForwardedHost(streamCtx); !s.isLocalRequest(fh) { - forwardedHost = fh - } - return forwardedHost, nil -} - // Bootstrap implements gRPC PDServer. func (s *GrpcServer) Bootstrap(ctx context.Context, request *pdpb.BootstrapRequest) (*pdpb.BootstrapResponse, error) { fn := func(ctx context.Context, client *grpc.ClientConn) (interface{}, error) { @@ -1004,7 +854,8 @@ func (s *GrpcServer) StoreHeartbeat(ctx context.Context, request *pdpb.StoreHear storeHeartbeatHandleDuration.WithLabelValues(storeAddress, storeLabel).Observe(time.Since(start).Seconds()) if s.IsServiceIndependent(utils.SchedulingServiceName) { forwardCli, _ := s.updateSchedulingClient(ctx) - if forwardCli != nil { + cli := forwardCli.getClient() + if cli != nil { req := &schedulingpb.StoreHeartbeatRequest{ Header: &schedulingpb.RequestHeader{ ClusterId: request.GetHeader().GetClusterId(), @@ -1012,9 +863,10 @@ func (s *GrpcServer) StoreHeartbeat(ctx context.Context, request *pdpb.StoreHear }, Stats: request.GetStats(), } - if _, err := forwardCli.StoreHeartbeat(ctx, req); err != nil { + if _, err := cli.StoreHeartbeat(ctx, req); err != nil { + log.Debug("forward store heartbeat failed", zap.Error(err)) // reset to let it be updated in the next request - s.schedulingClient.Store(&schedulingClient{}) + s.schedulingClient.CompareAndSwap(forwardCli, &schedulingClient{}) } } } @@ -1031,28 +883,38 @@ func (s *GrpcServer) StoreHeartbeat(ctx context.Context, request *pdpb.StoreHear return resp, nil } -func (s *GrpcServer) updateSchedulingClient(ctx context.Context) (schedulingpb.SchedulingClient, error) { +// 1. forwardedHost is empty, return nil +// 2. forwardedHost is not empty and forwardedHost is equal to pre, return pre +// 3. the rest of cases, update forwardedHost and return new client +func (s *GrpcServer) updateSchedulingClient(ctx context.Context) (*schedulingClient, error) { forwardedHost, _ := s.GetServicePrimaryAddr(ctx, utils.SchedulingServiceName) + if forwardedHost == "" { + return nil, ErrNotFoundSchedulingAddr + } + pre := s.schedulingClient.Load() - // 1. forwardedHost is not empty and pre is empty, update the schedulingClient - // 2. forwardedHost is not empty and forwardedHost is not equal to pre, update the schedulingClient - // 3. forwardedHost is not empty and forwardedHost is equal to pre, return pre - // 4. forwardedHost is empty, return nil - if forwardedHost != "" && ((pre == nil) || (pre != nil && forwardedHost != pre.(*schedulingClient).getPrimaryAddr())) { - client, err := s.getDelegateClient(ctx, forwardedHost) - if err != nil { - log.Error("get delegate client failed", zap.Error(err)) - } - forwardCli := &schedulingClient{ - client: schedulingpb.NewSchedulingClient(client), - lastPrimary: forwardedHost, + if pre != nil && forwardedHost == pre.(*schedulingClient).getPrimaryAddr() { + return pre.(*schedulingClient), nil + } + + client, err := s.getDelegateClient(ctx, forwardedHost) + if err != nil { + log.Error("get delegate client failed", zap.Error(err)) + return nil, err + } + forwardCli := &schedulingClient{ + client: schedulingpb.NewSchedulingClient(client), + primary: forwardedHost, + } + swapped := s.schedulingClient.CompareAndSwap(pre, forwardCli) + if swapped { + oldForwardedHost := "" + if pre != nil { + oldForwardedHost = pre.(*schedulingClient).getPrimaryAddr() } - s.schedulingClient.Store(forwardCli) - return forwardCli.getClient(), nil - } else if forwardedHost != "" && (pre != nil && forwardedHost == pre.(*schedulingClient).getPrimaryAddr()) { - return pre.(*schedulingClient).getClient(), nil + log.Info("update scheduling client", zap.String("old-forwarded-host", oldForwardedHost), zap.String("new-forwarded-host", forwardedHost)) } - return nil, ErrNotFoundSchedulingAddr + return forwardCli, nil } // bucketHeartbeatServer wraps PD_ReportBucketsServer to ensure when any error @@ -1097,48 +959,6 @@ func (b *bucketHeartbeatServer) Recv() (*pdpb.ReportBucketsRequest, error) { return req, nil } -// heartbeatServer wraps PD_RegionHeartbeatServer to ensure when any error -// occurs on Send() or Recv(), both endpoints will be closed. -type heartbeatServer struct { - stream pdpb.PD_RegionHeartbeatServer - closed int32 -} - -func (s *heartbeatServer) Send(m core.RegionHeartbeatResponse) error { - if atomic.LoadInt32(&s.closed) == 1 { - return io.EOF - } - done := make(chan error, 1) - go func() { - defer logutil.LogPanic() - done <- s.stream.Send(m.(*pdpb.RegionHeartbeatResponse)) - }() - timer := time.NewTimer(heartbeatSendTimeout) - defer timer.Stop() - select { - case err := <-done: - if err != nil { - atomic.StoreInt32(&s.closed, 1) - } - return errors.WithStack(err) - case <-timer.C: - atomic.StoreInt32(&s.closed, 1) - return ErrSendHeartbeatTimeout - } -} - -func (s *heartbeatServer) Recv() (*pdpb.RegionHeartbeatRequest, error) { - if atomic.LoadInt32(&s.closed) == 1 { - return nil, io.EOF - } - req, err := s.stream.Recv() - if err != nil { - atomic.StoreInt32(&s.closed, 1) - return nil, errors.WithStack(err) - } - return req, nil -} - // ReportBuckets implements gRPC PDServer func (s *GrpcServer) ReportBuckets(stream pdpb.PD_ReportBucketsServer) error { var ( @@ -1236,16 +1056,16 @@ func (s *GrpcServer) ReportBuckets(stream pdpb.PD_ReportBucketsServer) error { // RegionHeartbeat implements gRPC PDServer. func (s *GrpcServer) RegionHeartbeat(stream pdpb.PD_RegionHeartbeatServer) error { var ( - server = &heartbeatServer{stream: stream} - flowRoundOption = core.WithFlowRoundByDigit(s.persistOptions.GetPDServerConfig().FlowRoundByDigit) - forwardStream pdpb.PD_RegionHeartbeatClient - cancel context.CancelFunc - lastForwardedHost string - lastBind time.Time - errCh chan error - schedulingStream schedulingpb.Scheduling_RegionHeartbeatClient - cancel1 context.CancelFunc - lastPrimaryAddr string + server = &heartbeatServer{stream: stream} + flowRoundOption = core.WithFlowRoundByDigit(s.persistOptions.GetPDServerConfig().FlowRoundByDigit) + cancel context.CancelFunc + lastBind time.Time + errCh chan error + forwardStream pdpb.PD_RegionHeartbeatClient + lastForwardedHost string + forwardErrCh chan error + forwardSchedulingStream schedulingpb.Scheduling_RegionHeartbeatClient + lastForwardedSchedulingHost string ) defer func() { // cancel the forward stream @@ -1262,8 +1082,10 @@ func (s *GrpcServer) RegionHeartbeat(stream pdpb.PD_RegionHeartbeatServer) error if err != nil { return errors.WithStack(err) } - forwardedHost := grpcutil.GetForwardedHost(stream.Context()) + failpoint.Inject("grpcClientClosed", func() { + forwardedHost = s.GetMember().Member().GetClientUrls()[0] + }) if !s.isLocalRequest(forwardedHost) { if forwardStream == nil || lastForwardedHost != forwardedHost { if cancel != nil { @@ -1274,7 +1096,7 @@ func (s *GrpcServer) RegionHeartbeat(stream pdpb.PD_RegionHeartbeatServer) error return err } log.Info("create region heartbeat forward stream", zap.String("forwarded-host", forwardedHost)) - forwardStream, cancel, err = s.createHeartbeatForwardStream(client) + forwardStream, cancel, err = s.createRegionHeartbeatForwardStream(client) if err != nil { return err } @@ -1360,56 +1182,83 @@ func (s *GrpcServer) RegionHeartbeat(stream pdpb.PD_RegionHeartbeatServer) error continue } + regionHeartbeatHandleDuration.WithLabelValues(storeAddress, storeLabel).Observe(time.Since(start).Seconds()) + regionHeartbeatCounter.WithLabelValues(storeAddress, storeLabel, "report", "ok").Inc() + if s.IsServiceIndependent(utils.SchedulingServiceName) { - ctx := stream.Context() - primaryAddr, _ := s.GetServicePrimaryAddr(ctx, utils.SchedulingServiceName) - if schedulingStream == nil || lastPrimaryAddr != primaryAddr { - if cancel1 != nil { - cancel1() + if forwardErrCh != nil { + select { + case err, ok := <-forwardErrCh: + if ok { + if cancel != nil { + cancel() + } + forwardSchedulingStream = nil + log.Error("meet error and need to re-establish the stream", zap.Error(err)) + } + default: } - client, err := s.getDelegateClient(ctx, primaryAddr) + } + forwardedSchedulingHost, ok := s.GetServicePrimaryAddr(stream.Context(), utils.SchedulingServiceName) + if !ok || len(forwardedSchedulingHost) == 0 { + log.Debug("failed to find scheduling service primary address") + if cancel != nil { + cancel() + } + continue + } + if forwardSchedulingStream == nil || lastForwardedSchedulingHost != forwardedSchedulingHost { + if cancel != nil { + cancel() + } + client, err := s.getDelegateClient(s.ctx, forwardedSchedulingHost) if err != nil { - log.Error("get delegate client failed", zap.Error(err)) + log.Error("failed to get client", zap.Error(err)) + continue } - - log.Info("create region heartbeat forward stream", zap.String("forwarded-host", primaryAddr)) - schedulingStream, cancel1, err = s.createSchedulingStream(client) + log.Info("create scheduling forwarding stream", zap.String("forwarded-host", forwardedSchedulingHost)) + forwardSchedulingStream, _, cancel, err = s.createRegionHeartbeatSchedulingStream(stream.Context(), client) if err != nil { - log.Error("create region heartbeat forward stream failed", zap.Error(err)) - } else { - lastPrimaryAddr = primaryAddr - errCh = make(chan error, 1) - go forwardSchedulingToServer(schedulingStream, server, errCh) + log.Error("failed to create stream", zap.Error(err)) + continue } + lastForwardedSchedulingHost = forwardedSchedulingHost + forwardErrCh = make(chan error, 1) + go forwardRegionHeartbeatToScheduling(forwardSchedulingStream, server, forwardErrCh) } - if schedulingStream != nil { - req := &schedulingpb.RegionHeartbeatRequest{ - Header: &schedulingpb.RequestHeader{ - ClusterId: request.GetHeader().GetClusterId(), - SenderId: request.GetHeader().GetSenderId(), - }, - Region: request.GetRegion(), - Leader: request.GetLeader(), - DownPeers: request.GetDownPeers(), - PendingPeers: request.GetPendingPeers(), - BytesWritten: request.GetBytesWritten(), - BytesRead: request.GetBytesRead(), - KeysWritten: request.GetKeysWritten(), - KeysRead: request.GetKeysRead(), - ApproximateSize: request.GetApproximateSize(), - ApproximateKeys: request.GetApproximateKeys(), - Interval: request.GetInterval(), - Term: request.GetTerm(), - QueryStats: request.GetQueryStats(), - } - if err := schedulingStream.Send(req); err != nil { - log.Error("forward region heartbeat failed", zap.Error(err)) + schedulingpbReq := &schedulingpb.RegionHeartbeatRequest{ + Header: &schedulingpb.RequestHeader{ + ClusterId: request.GetHeader().GetClusterId(), + SenderId: request.GetHeader().GetSenderId(), + }, + Region: request.GetRegion(), + Leader: request.GetLeader(), + DownPeers: request.GetDownPeers(), + PendingPeers: request.GetPendingPeers(), + BytesWritten: request.GetBytesWritten(), + BytesRead: request.GetBytesRead(), + KeysWritten: request.GetKeysWritten(), + KeysRead: request.GetKeysRead(), + ApproximateSize: request.GetApproximateSize(), + ApproximateKeys: request.GetApproximateKeys(), + Interval: request.GetInterval(), + Term: request.GetTerm(), + QueryStats: request.GetQueryStats(), + } + if err := forwardSchedulingStream.Send(schedulingpbReq); err != nil { + forwardSchedulingStream = nil + log.Error("failed to send request to scheduling service", zap.Error(err)) + } + + select { + case err, ok := <-forwardErrCh: + if ok { + forwardSchedulingStream = nil + log.Error("failed to send response", zap.Error(err)) } + default: } } - - regionHeartbeatHandleDuration.WithLabelValues(storeAddress, storeLabel).Observe(time.Since(start).Seconds()) - regionHeartbeatCounter.WithLabelValues(storeAddress, storeLabel, "report", "ok").Inc() } } @@ -1639,7 +1488,8 @@ func (s *GrpcServer) AskBatchSplit(ctx context.Context, request *pdpb.AskBatchSp Header: s.wrapErrorToHeader(pdpb.ErrorType_UNKNOWN, err.Error()), }, nil } - if forwardCli != nil { + cli := forwardCli.getClient() + if cli != nil { req := &schedulingpb.AskBatchSplitRequest{ Header: &schedulingpb.RequestHeader{ ClusterId: request.GetHeader().GetClusterId(), @@ -1648,10 +1498,10 @@ func (s *GrpcServer) AskBatchSplit(ctx context.Context, request *pdpb.AskBatchSp Region: request.GetRegion(), SplitCount: request.GetSplitCount(), } - resp, err := s.schedulingClient.Load().(*schedulingClient).getClient().AskBatchSplit(ctx, req) + resp, err := cli.AskBatchSplit(ctx, req) if err != nil { // reset to let it be updated in the next request - s.schedulingClient.Store(&schedulingClient{}) + s.schedulingClient.CompareAndSwap(forwardCli, &schedulingClient{}) return s.convertAskSplitResponse(resp), err } return s.convertAskSplitResponse(resp), nil @@ -1812,7 +1662,8 @@ func (s *GrpcServer) ScatterRegion(ctx context.Context, request *pdpb.ScatterReg Header: s.wrapErrorToHeader(pdpb.ErrorType_UNKNOWN, err.Error()), }, nil } - if forwardCli != nil { + cli := forwardCli.getClient() + if cli != nil { var regionsID []uint64 // nolint if request.GetRegionId() != 0 { @@ -1836,10 +1687,10 @@ func (s *GrpcServer) ScatterRegion(ctx context.Context, request *pdpb.ScatterReg RetryLimit: request.GetRetryLimit(), SkipStoreLimit: request.GetSkipStoreLimit(), } - resp, err := forwardCli.ScatterRegions(ctx, req) + resp, err := cli.ScatterRegions(ctx, req) if err != nil { // reset to let it be updated in the next request - s.schedulingClient.Store(&schedulingClient{}) + s.schedulingClient.CompareAndSwap(forwardCli, &schedulingClient{}) return s.convertScatterResponse(resp), err } return s.convertScatterResponse(resp), nil @@ -2035,7 +1886,8 @@ func (s *GrpcServer) GetOperator(ctx context.Context, request *pdpb.GetOperatorR Header: s.wrapErrorToHeader(pdpb.ErrorType_UNKNOWN, err.Error()), }, nil } - if forwardCli != nil { + cli := forwardCli.getClient() + if cli != nil { req := &schedulingpb.GetOperatorRequest{ Header: &schedulingpb.RequestHeader{ ClusterId: request.GetHeader().GetClusterId(), @@ -2043,10 +1895,10 @@ func (s *GrpcServer) GetOperator(ctx context.Context, request *pdpb.GetOperatorR }, RegionId: request.GetRegionId(), } - resp, err := forwardCli.GetOperator(ctx, req) + resp, err := cli.GetOperator(ctx, req) if err != nil { // reset to let it be updated in the next request - s.schedulingClient.Store(&schedulingClient{}) + s.schedulingClient.CompareAndSwap(forwardCli, &schedulingClient{}) return s.convertOperatorResponse(resp), err } return s.convertOperatorResponse(resp), nil @@ -2307,7 +2159,8 @@ func (s *GrpcServer) SplitRegions(ctx context.Context, request *pdpb.SplitRegion Header: s.wrapErrorToHeader(pdpb.ErrorType_UNKNOWN, err.Error()), }, nil } - if forwardCli != nil { + cli := forwardCli.getClient() + if cli != nil { req := &schedulingpb.SplitRegionsRequest{ Header: &schedulingpb.RequestHeader{ ClusterId: request.GetHeader().GetClusterId(), @@ -2316,10 +2169,10 @@ func (s *GrpcServer) SplitRegions(ctx context.Context, request *pdpb.SplitRegion SplitKeys: request.GetSplitKeys(), RetryLimit: request.GetRetryLimit(), } - resp, err := forwardCli.SplitRegions(ctx, req) + resp, err := cli.SplitRegions(ctx, req) if err != nil { // reset to let it be updated in the next request - s.schedulingClient.Store(&schedulingClient{}) + s.schedulingClient.CompareAndSwap(forwardCli, &schedulingClient{}) return s.convertSplitResponse(resp), err } return s.convertSplitResponse(resp), nil @@ -2451,258 +2304,6 @@ func (s *GrpcServer) validateInternalRequest(header *pdpb.RequestHeader, onlyAll return nil } -func (s *GrpcServer) getDelegateClient(ctx context.Context, forwardedHost string) (*grpc.ClientConn, error) { - client, ok := s.clientConns.Load(forwardedHost) - if ok { - // Mostly, the connection is already established, and return it directly. - return client.(*grpc.ClientConn), nil - } - - tlsConfig, err := s.GetTLSConfig().ToTLSConfig() - if err != nil { - return nil, err - } - ctxTimeout, cancel := context.WithTimeout(ctx, defaultGRPCDialTimeout) - defer cancel() - newConn, err := grpcutil.GetClientConn(ctxTimeout, forwardedHost, tlsConfig) - if err != nil { - return nil, err - } - conn, loaded := s.clientConns.LoadOrStore(forwardedHost, newConn) - if !loaded { - // Successfully stored the connection we created. - return newConn, nil - } - // Loaded a connection created/stored by another goroutine, so close the one we created - // and return the one we loaded. - newConn.Close() - return conn.(*grpc.ClientConn), nil -} - -func (s *GrpcServer) isLocalRequest(forwardedHost string) bool { - failpoint.Inject("useForwardRequest", func() { - failpoint.Return(false) - }) - if forwardedHost == "" { - return true - } - memberAddrs := s.GetMember().Member().GetClientUrls() - for _, addr := range memberAddrs { - if addr == forwardedHost { - return true - } - } - return false -} - -func (s *GrpcServer) createHeartbeatForwardStream(client *grpc.ClientConn) (pdpb.PD_RegionHeartbeatClient, context.CancelFunc, error) { - done := make(chan struct{}) - ctx, cancel := context.WithCancel(s.ctx) - go grpcutil.CheckStream(ctx, cancel, done) - forwardStream, err := pdpb.NewPDClient(client).RegionHeartbeat(ctx) - done <- struct{}{} - return forwardStream, cancel, err -} - -func forwardRegionHeartbeatClientToServer(forwardStream pdpb.PD_RegionHeartbeatClient, server *heartbeatServer, errCh chan error) { - defer logutil.LogPanic() - defer close(errCh) - for { - resp, err := forwardStream.Recv() - if err != nil { - errCh <- errors.WithStack(err) - return - } - if err := server.Send(resp); err != nil { - errCh <- errors.WithStack(err) - return - } - } -} - -func (s *GrpcServer) createSchedulingStream(client *grpc.ClientConn) (schedulingpb.Scheduling_RegionHeartbeatClient, context.CancelFunc, error) { - if client == nil { - return nil, nil, errors.New("connection is not set") - } - done := make(chan struct{}) - ctx, cancel := context.WithCancel(s.ctx) - go grpcutil.CheckStream(ctx, cancel, done) - forwardStream, err := schedulingpb.NewSchedulingClient(client).RegionHeartbeat(ctx) - done <- struct{}{} - return forwardStream, cancel, err -} - -func forwardSchedulingToServer(forwardStream schedulingpb.Scheduling_RegionHeartbeatClient, server *heartbeatServer, errCh chan error) { - defer logutil.LogPanic() - defer close(errCh) - for { - resp, err := forwardStream.Recv() - if err != nil { - errCh <- errors.WithStack(err) - return - } - response := &pdpb.RegionHeartbeatResponse{ - Header: &pdpb.ResponseHeader{ - ClusterId: resp.GetHeader().GetClusterId(), - // ignore error here - }, - ChangePeer: resp.GetChangePeer(), - TransferLeader: resp.GetTransferLeader(), - RegionId: resp.GetRegionId(), - RegionEpoch: resp.GetRegionEpoch(), - TargetPeer: resp.GetTargetPeer(), - Merge: resp.GetMerge(), - SplitRegion: resp.GetSplitRegion(), - ChangePeerV2: resp.GetChangePeerV2(), - SwitchWitnesses: resp.GetSwitchWitnesses(), - } - - if err := server.Send(response); err != nil { - errCh <- errors.WithStack(err) - return - } - } -} - -func (s *GrpcServer) createTSOForwardStream( - ctx context.Context, client *grpc.ClientConn, -) (tsopb.TSO_TsoClient, context.Context, context.CancelFunc, error) { - done := make(chan struct{}) - forwardCtx, cancelForward := context.WithCancel(ctx) - go grpcutil.CheckStream(forwardCtx, cancelForward, done) - forwardStream, err := tsopb.NewTSOClient(client).Tso(forwardCtx) - done <- struct{}{} - return forwardStream, forwardCtx, cancelForward, err -} - -func (s *GrpcServer) createReportBucketsForwardStream(client *grpc.ClientConn) (pdpb.PD_ReportBucketsClient, context.CancelFunc, error) { - done := make(chan struct{}) - ctx, cancel := context.WithCancel(s.ctx) - go grpcutil.CheckStream(ctx, cancel, done) - forwardStream, err := pdpb.NewPDClient(client).ReportBuckets(ctx) - done <- struct{}{} - return forwardStream, cancel, err -} - -func forwardReportBucketClientToServer(forwardStream pdpb.PD_ReportBucketsClient, server *bucketHeartbeatServer, errCh chan error) { - defer logutil.LogPanic() - defer close(errCh) - for { - resp, err := forwardStream.CloseAndRecv() - if err != nil { - errCh <- errors.WithStack(err) - return - } - if err := server.Send(resp); err != nil { - errCh <- errors.WithStack(err) - return - } - } -} - -func (s *GrpcServer) getGlobalTSO(ctx context.Context) (pdpb.Timestamp, error) { - if !s.IsAPIServiceMode() { - return s.tsoAllocatorManager.HandleRequest(ctx, tso.GlobalDCLocation, 1) - } - request := &tsopb.TsoRequest{ - Header: &tsopb.RequestHeader{ - ClusterId: s.clusterID, - KeyspaceId: utils.DefaultKeyspaceID, - KeyspaceGroupId: utils.DefaultKeyspaceGroupID, - }, - Count: 1, - } - var ( - forwardedHost string - forwardStream tsopb.TSO_TsoClient - ts *tsopb.TsoResponse - err error - ok bool - ) - handleStreamError := func(err error) (needRetry bool) { - if strings.Contains(err.Error(), errs.NotLeaderErr) { - s.tsoPrimaryWatcher.ForceLoad() - log.Warn("force to load tso primary address due to error", zap.Error(err), zap.String("tso-addr", forwardedHost)) - return true - } - if grpcutil.NeedRebuildConnection(err) { - s.tsoClientPool.Lock() - delete(s.tsoClientPool.clients, forwardedHost) - s.tsoClientPool.Unlock() - log.Warn("client connection removed due to error", zap.Error(err), zap.String("tso-addr", forwardedHost)) - return true - } - return false - } - for i := 0; i < maxRetryTimesRequestTSOServer; i++ { - if i > 0 { - time.Sleep(retryIntervalRequestTSOServer) - } - forwardedHost, ok = s.GetServicePrimaryAddr(ctx, utils.TSOServiceName) - if !ok || forwardedHost == "" { - return pdpb.Timestamp{}, ErrNotFoundTSOAddr - } - forwardStream, err = s.getTSOForwardStream(forwardedHost) - if err != nil { - return pdpb.Timestamp{}, err - } - err = forwardStream.Send(request) - if err != nil { - if needRetry := handleStreamError(err); needRetry { - continue - } - log.Error("send request to tso primary server failed", zap.Error(err), zap.String("tso-addr", forwardedHost)) - return pdpb.Timestamp{}, err - } - ts, err = forwardStream.Recv() - if err != nil { - if needRetry := handleStreamError(err); needRetry { - continue - } - log.Error("receive response from tso primary server failed", zap.Error(err), zap.String("tso-addr", forwardedHost)) - return pdpb.Timestamp{}, err - } - return *ts.GetTimestamp(), nil - } - log.Error("get global tso from tso primary server failed after retry", zap.Error(err), zap.String("tso-addr", forwardedHost)) - return pdpb.Timestamp{}, err -} - -func (s *GrpcServer) getTSOForwardStream(forwardedHost string) (tsopb.TSO_TsoClient, error) { - s.tsoClientPool.RLock() - forwardStream, ok := s.tsoClientPool.clients[forwardedHost] - s.tsoClientPool.RUnlock() - if ok { - // This is the common case to return here - return forwardStream, nil - } - - s.tsoClientPool.Lock() - defer s.tsoClientPool.Unlock() - - // Double check after entering the critical section - forwardStream, ok = s.tsoClientPool.clients[forwardedHost] - if ok { - return forwardStream, nil - } - - // Now let's create the client connection and the forward stream - client, err := s.getDelegateClient(s.ctx, forwardedHost) - if err != nil { - return nil, err - } - done := make(chan struct{}) - ctx, cancel := context.WithCancel(s.ctx) - go grpcutil.CheckStream(ctx, cancel, done) - forwardStream, err = tsopb.NewTSOClient(client).Tso(ctx) - done <- struct{}{} - if err != nil { - return nil, err - } - s.tsoClientPool.clients[forwardedHost] = forwardStream - return forwardStream, nil -} - // for CDC compatibility, we need to initialize config path to `globalConfigPath` const globalConfigPath = "/global/config/" From 86831ce7186525bdbcd33f92ba9008080a2c7a05 Mon Sep 17 00:00:00 2001 From: Hu# Date: Tue, 14 Nov 2023 12:11:14 +0800 Subject: [PATCH 015/137] prepare_check: remove redundant check (#7217) ref tikv/pd#7016 remove redundant check in prepare_check Signed-off-by: husharp Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- pkg/cluster/cluster.go | 3 --- pkg/core/region.go | 18 ++++++++++--- pkg/core/region_tree.go | 7 +++++ pkg/schedule/prepare_checker.go | 35 +++++-------------------- server/cluster/cluster.go | 2 +- server/cluster/cluster_test.go | 18 ++++++------- tests/pdctl/scheduler/scheduler_test.go | 2 +- tests/server/cluster/cluster_test.go | 2 +- tests/testutil.go | 1 + 9 files changed, 40 insertions(+), 48 deletions(-) diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 0b3e0351b16..8809a706936 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -59,7 +59,4 @@ func Collect(c Cluster, region *core.RegionInfo, stores []*core.StoreInfo, hasRe if hasRegionStats { c.GetRegionStats().Observe(region, stores) } - if !isPrepared && isNew { - c.GetCoordinator().GetPrepareChecker().Collect(region) - } } diff --git a/pkg/core/region.go b/pkg/core/region.go index c9daa69c477..b141e8478da 100644 --- a/pkg/core/region.go +++ b/pkg/core/region.go @@ -1340,11 +1340,23 @@ func (r *RegionsInfo) GetStoreWriteRate(storeID uint64) (bytesRate, keysRate flo return } -// GetClusterNotFromStorageRegionsCnt gets the total count of regions that not loaded from storage anymore +// GetClusterNotFromStorageRegionsCnt gets the `NotFromStorageRegionsCnt` count of regions that not loaded from storage anymore. func (r *RegionsInfo) GetClusterNotFromStorageRegionsCnt() int { r.t.RLock() defer r.t.RUnlock() - return r.tree.notFromStorageRegionsCnt + return r.tree.notFromStorageRegionsCount() +} + +// GetNotFromStorageRegionsCntByStore gets the `NotFromStorageRegionsCnt` count of a store's leader, follower and learner by storeID. +func (r *RegionsInfo) GetNotFromStorageRegionsCntByStore(storeID uint64) int { + r.st.RLock() + defer r.st.RUnlock() + return r.getNotFromStorageRegionsCntByStoreLocked(storeID) +} + +// getNotFromStorageRegionsCntByStoreLocked gets the `NotFromStorageRegionsCnt` count of a store's leader, follower and learner by storeID. +func (r *RegionsInfo) getNotFromStorageRegionsCntByStoreLocked(storeID uint64) int { + return r.leaders[storeID].notFromStorageRegionsCount() + r.followers[storeID].notFromStorageRegionsCount() + r.learners[storeID].notFromStorageRegionsCount() } // GetMetaRegions gets a set of metapb.Region from regionMap @@ -1380,7 +1392,7 @@ func (r *RegionsInfo) GetStoreRegionCount(storeID uint64) int { return r.getStoreRegionCountLocked(storeID) } -// GetStoreRegionCount gets the total count of a store's leader, follower and learner RegionInfo by storeID +// getStoreRegionCountLocked gets the total count of a store's leader, follower and learner RegionInfo by storeID func (r *RegionsInfo) getStoreRegionCountLocked(storeID uint64) int { return r.leaders[storeID].length() + r.followers[storeID].length() + r.learners[storeID].length() } diff --git a/pkg/core/region_tree.go b/pkg/core/region_tree.go index ed3445de6b6..ecc988d97d8 100644 --- a/pkg/core/region_tree.go +++ b/pkg/core/region_tree.go @@ -82,6 +82,13 @@ func (t *regionTree) length() int { return t.tree.Len() } +func (t *regionTree) notFromStorageRegionsCount() int { + if t == nil { + return 0 + } + return t.notFromStorageRegionsCnt +} + // GetOverlaps returns the range items that has some intersections with the given items. func (t *regionTree) overlaps(item *regionItem) []*regionItem { // note that Find() gets the last item that is less or equal than the item. diff --git a/pkg/schedule/prepare_checker.go b/pkg/schedule/prepare_checker.go index c7faa57af81..34618427930 100644 --- a/pkg/schedule/prepare_checker.go +++ b/pkg/schedule/prepare_checker.go @@ -25,16 +25,13 @@ import ( type prepareChecker struct { syncutil.RWMutex - reactiveRegions map[uint64]int - start time.Time - sum int - prepared bool + start time.Time + prepared bool } func newPrepareChecker() *prepareChecker { return &prepareChecker{ - start: time.Now(), - reactiveRegions: make(map[uint64]int), + start: time.Now(), } } @@ -51,13 +48,8 @@ func (checker *prepareChecker) check(c *core.BasicCluster) bool { } notLoadedFromRegionsCnt := c.GetClusterNotFromStorageRegionsCnt() totalRegionsCnt := c.GetTotalRegionCount() - if float64(notLoadedFromRegionsCnt) > float64(totalRegionsCnt)*collectFactor { - log.Info("meta not loaded from region number is satisfied, finish prepare checker", zap.Int("not-from-storage-region", notLoadedFromRegionsCnt), zap.Int("total-region", totalRegionsCnt)) - checker.prepared = true - return true - } // The number of active regions should be more than total region of all stores * collectFactor - if float64(totalRegionsCnt)*collectFactor > float64(checker.sum) { + if float64(totalRegionsCnt)*collectFactor > float64(notLoadedFromRegionsCnt) { return false } for _, store := range c.GetStores() { @@ -66,23 +58,15 @@ func (checker *prepareChecker) check(c *core.BasicCluster) bool { } storeID := store.GetID() // For each store, the number of active regions should be more than total region of the store * collectFactor - if float64(c.GetStoreRegionCount(storeID))*collectFactor > float64(checker.reactiveRegions[storeID]) { + if float64(c.GetStoreRegionCount(storeID))*collectFactor > float64(c.GetNotFromStorageRegionsCntByStore(storeID)) { return false } } + log.Info("not loaded from storage region number is satisfied, finish prepare checker", zap.Int("not-from-storage-region", notLoadedFromRegionsCnt), zap.Int("total-region", totalRegionsCnt)) checker.prepared = true return true } -func (checker *prepareChecker) Collect(region *core.RegionInfo) { - checker.Lock() - defer checker.Unlock() - for _, p := range region.GetPeers() { - checker.reactiveRegions[p.GetStoreId()]++ - } - checker.sum++ -} - func (checker *prepareChecker) IsPrepared() bool { checker.RLock() defer checker.RUnlock() @@ -95,10 +79,3 @@ func (checker *prepareChecker) SetPrepared() { defer checker.Unlock() checker.prepared = true } - -// for test purpose -func (checker *prepareChecker) GetSum() int { - checker.RLock() - defer checker.RUnlock() - return checker.sum -} diff --git a/server/cluster/cluster.go b/server/cluster/cluster.go index c0fb1b15f8f..3b50ae16d9b 100644 --- a/server/cluster/cluster.go +++ b/server/cluster/cluster.go @@ -1018,7 +1018,7 @@ func (c *RaftCluster) processRegionHeartbeat(region *core.RegionInfo) error { // To prevent a concurrent heartbeat of another region from overriding the up-to-date region info by a stale one, // check its validation again here. // - // However it can't solve the race condition of concurrent heartbeats from the same region. + // However, it can't solve the race condition of concurrent heartbeats from the same region. if overlaps, err = c.core.AtomicCheckAndPutRegion(region); err != nil { return err } diff --git a/server/cluster/cluster_test.go b/server/cluster/cluster_test.go index 7ebd012a6a2..4b9b401e0c9 100644 --- a/server/cluster/cluster_test.go +++ b/server/cluster/cluster_test.go @@ -2384,7 +2384,7 @@ func (c *testCluster) LoadRegion(regionID uint64, followerStoreIDs ...uint64) er peer, _ := c.AllocPeer(id) region.Peers = append(region.Peers, peer) } - return c.putRegion(core.NewRegionInfo(region, nil)) + return c.putRegion(core.NewRegionInfo(region, nil, core.SetSource(core.Storage))) } func TestBasic(t *testing.T) { @@ -2469,7 +2469,7 @@ func TestDispatch(t *testing.T) { func dispatchHeartbeat(co *schedule.Coordinator, region *core.RegionInfo, stream hbstream.HeartbeatStream) error { co.GetHeartbeatStreams().BindStream(region.GetLeader().GetStoreId(), stream) - if err := co.GetCluster().(*RaftCluster).putRegion(region.Clone()); err != nil { + if err := co.GetCluster().(*RaftCluster).putRegion(region.Clone(core.SetSource(core.Heartbeat))); err != nil { return err } co.GetOperatorController().Dispatch(region, operator.DispatchFromHeartBeat, nil) @@ -2943,14 +2943,14 @@ func TestShouldRun(t *testing.T) { for _, testCase := range testCases { r := tc.GetRegion(testCase.regionID) - nr := r.Clone(core.WithLeader(r.GetPeers()[0])) + nr := r.Clone(core.WithLeader(r.GetPeers()[0]), core.SetSource(core.Heartbeat)) re.NoError(tc.processRegionHeartbeat(nr)) re.Equal(testCase.ShouldRun, co.ShouldRun()) } nr := &metapb.Region{Id: 6, Peers: []*metapb.Peer{}} - newRegion := core.NewRegionInfo(nr, nil) + newRegion := core.NewRegionInfo(nr, nil, core.SetSource(core.Heartbeat)) re.Error(tc.processRegionHeartbeat(newRegion)) - re.Equal(7, co.GetPrepareChecker().GetSum()) + re.Equal(7, tc.core.GetClusterNotFromStorageRegionsCnt()) } func TestShouldRunWithNonLeaderRegions(t *testing.T) { @@ -2986,14 +2986,14 @@ func TestShouldRunWithNonLeaderRegions(t *testing.T) { for _, testCase := range testCases { r := tc.GetRegion(testCase.regionID) - nr := r.Clone(core.WithLeader(r.GetPeers()[0])) + nr := r.Clone(core.WithLeader(r.GetPeers()[0]), core.SetSource(core.Heartbeat)) re.NoError(tc.processRegionHeartbeat(nr)) re.Equal(testCase.ShouldRun, co.ShouldRun()) } nr := &metapb.Region{Id: 9, Peers: []*metapb.Peer{}} - newRegion := core.NewRegionInfo(nr, nil) + newRegion := core.NewRegionInfo(nr, nil, core.SetSource(core.Heartbeat)) re.Error(tc.processRegionHeartbeat(newRegion)) - re.Equal(9, co.GetPrepareChecker().GetSum()) + re.Equal(9, tc.core.GetClusterNotFromStorageRegionsCnt()) // Now, after server is prepared, there exist some regions with no leader. re.Equal(uint64(0), tc.GetRegion(10).GetLeader().GetStoreId()) @@ -3263,7 +3263,6 @@ func TestRestart(t *testing.T) { re.NoError(tc.addRegionStore(3, 3)) re.NoError(tc.addLeaderRegion(1, 1)) region := tc.GetRegion(1) - co.GetPrepareChecker().Collect(region) // Add 1 replica on store 2. stream := mockhbstream.NewHeartbeatStream() @@ -3277,7 +3276,6 @@ func TestRestart(t *testing.T) { // Recreate coordinator then add another replica on store 3. co = schedule.NewCoordinator(ctx, tc.RaftCluster, hbStreams) - co.GetPrepareChecker().Collect(region) co.Run() re.NoError(dispatchHeartbeat(co, region, stream)) region = waitAddLearner(re, stream, region, 3) diff --git a/tests/pdctl/scheduler/scheduler_test.go b/tests/pdctl/scheduler/scheduler_test.go index cd599405124..d0fac2c1137 100644 --- a/tests/pdctl/scheduler/scheduler_test.go +++ b/tests/pdctl/scheduler/scheduler_test.go @@ -539,7 +539,7 @@ func (suite *schedulerTestSuite) checkSchedulerDiagnostic(cluster *tests.TestClu tests.MustPutStore(re, cluster, store) } - // note: because pdqsort is a unstable sort algorithm, set ApproximateSize for this region. + // note: because pdqsort is an unstable sort algorithm, set ApproximateSize for this region. tests.MustPutRegion(re, cluster, 1, 1, []byte("a"), []byte("b"), core.SetApproximateSize(10)) echo := mustExec(re, cmd, []string{"-u", pdAddr, "config", "set", "enable-diagnostic", "true"}, nil) diff --git a/tests/server/cluster/cluster_test.go b/tests/server/cluster/cluster_test.go index ebf0a4e574d..b7a428e3683 100644 --- a/tests/server/cluster/cluster_test.go +++ b/tests/server/cluster/cluster_test.go @@ -1402,7 +1402,7 @@ func putRegionWithLeader(re *require.Assertions, rc *cluster.RaftCluster, id id. StartKey: []byte{byte(i)}, EndKey: []byte{byte(i + 1)}, } - rc.HandleRegionHeartbeat(core.NewRegionInfo(region, region.Peers[0])) + rc.HandleRegionHeartbeat(core.NewRegionInfo(region, region.Peers[0], core.SetSource(core.Heartbeat))) } time.Sleep(50 * time.Millisecond) diff --git a/tests/testutil.go b/tests/testutil.go index 613705d3eb6..059a152f06f 100644 --- a/tests/testutil.go +++ b/tests/testutil.go @@ -197,6 +197,7 @@ func MustPutRegion(re *require.Assertions, cluster *TestCluster, regionID, store Peers: []*metapb.Peer{leader}, RegionEpoch: &metapb.RegionEpoch{ConfVer: 1, Version: 1}, } + opts = append(opts, core.SetSource(core.Heartbeat)) r := core.NewRegionInfo(metaRegion, leader, opts...) MustPutRegionInfo(re, cluster, r) return r From 181fdc95be65fd8c83155c76f0a69ddb2cf143bf Mon Sep 17 00:00:00 2001 From: Yongbo Jiang Date: Wed, 15 Nov 2023 14:45:46 +0800 Subject: [PATCH 016/137] makefile: support build with `boringcrypto` to support Fips (#7275) close tikv/pd#7274 Signed-off-by: Cabinfever_B Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- Makefile | 28 +++++++++++++++++++++++----- pkg/versioninfo/fips.go | 26 ++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 5 deletions(-) create mode 100644 pkg/versioninfo/fips.go diff --git a/Makefile b/Makefile index 54ad331aea4..906dd9414f9 100644 --- a/Makefile +++ b/Makefile @@ -15,6 +15,8 @@ dev-basic: build check basic-test BUILD_FLAGS ?= BUILD_TAGS ?= BUILD_CGO_ENABLED := 0 +BUILD_TOOL_CGO_ENABLED := 0 +BUILD_GOEXPERIMENT ?= PD_EDITION ?= Community # Ensure PD_EDITION is set to Community or Enterprise before running build process. ifneq "$(PD_EDITION)" "Community" @@ -46,6 +48,13 @@ ifeq ($(PLUGIN), 1) BUILD_TAGS += with_plugin endif +ifeq ($(ENABLE_FIPS), 1) + BUILD_TAGS+=boringcrypto + BUILD_GOEXPERIMENT=boringcrypto + BUILD_CGO_ENABLED := 1 + BUILD_TOOL_CGO_ENABLED := 1 +endif + LDFLAGS += -X "$(PD_PKG)/pkg/versioninfo.PDReleaseVersion=$(shell git describe --tags --dirty --always)" LDFLAGS += -X "$(PD_PKG)/pkg/versioninfo.PDBuildTS=$(shell date -u '+%Y-%m-%d %I:%M:%S')" LDFLAGS += -X "$(PD_PKG)/pkg/versioninfo.PDGitHash=$(shell git rev-parse HEAD)" @@ -66,6 +75,8 @@ BUILD_BIN_PATH := $(ROOT_PATH)/bin build: pd-server pd-ctl pd-recover +build-fips: pd-server-fips pd-ctl-fips pd-recover-fips + tools: pd-tso-bench pd-heartbeat-bench regions-dump stores-dump pd-api-bench PD_SERVER_DEP := @@ -79,7 +90,7 @@ endif PD_SERVER_DEP += dashboard-ui pd-server: ${PD_SERVER_DEP} - CGO_ENABLED=$(BUILD_CGO_ENABLED) go build $(BUILD_FLAGS) -gcflags '$(GCFLAGS)' -ldflags '$(LDFLAGS)' -tags "$(BUILD_TAGS)" -o $(BUILD_BIN_PATH)/pd-server cmd/pd-server/main.go + GOEXPERIMENT=$(BUILD_GOEXPERIMENT) CGO_ENABLED=$(BUILD_CGO_ENABLED) go build $(BUILD_FLAGS) -gcflags '$(GCFLAGS)' -ldflags '$(LDFLAGS)' -tags "$(BUILD_TAGS)" -o $(BUILD_BIN_PATH)/pd-server cmd/pd-server/main.go pd-server-failpoint: @$(FAILPOINT_ENABLE) @@ -89,18 +100,25 @@ pd-server-failpoint: pd-server-basic: SWAGGER=0 DASHBOARD=0 $(MAKE) pd-server -.PHONY: build tools pd-server pd-server-basic +pd-server-fips: + ENABLE_FIPS=1 $(MAKE) pd-server + +.PHONY: build tools pd-server pd-server-basic pd-server-fips # Tools pd-ctl: - CGO_ENABLED=0 go build -gcflags '$(GCFLAGS)' -ldflags '$(LDFLAGS)' -o $(BUILD_BIN_PATH)/pd-ctl tools/pd-ctl/main.go + GOEXPERIMENT=$(BUILD_GOEXPERIMENT) CGO_ENABLED=$(BUILD_TOOL_CGO_ENABLED) go build -gcflags '$(GCFLAGS)' -ldflags '$(LDFLAGS)' -o $(BUILD_BIN_PATH)/pd-ctl tools/pd-ctl/main.go +pd-ctl-fips: + ENABLE_FIPS=1 $(MAKE) pd-ctl pd-tso-bench: cd tools/pd-tso-bench && CGO_ENABLED=0 go build -o $(BUILD_BIN_PATH)/pd-tso-bench main.go pd-api-bench: cd tools/pd-api-bench && CGO_ENABLED=0 go build -o $(BUILD_BIN_PATH)/pd-api-bench main.go pd-recover: - CGO_ENABLED=0 go build -gcflags '$(GCFLAGS)' -ldflags '$(LDFLAGS)' -o $(BUILD_BIN_PATH)/pd-recover tools/pd-recover/main.go + GOEXPERIMENT=$(BUILD_GOEXPERIMENT) CGO_ENABLED=$(BUILD_TOOL_CGO_ENABLED) go build -gcflags '$(GCFLAGS)' -ldflags '$(LDFLAGS)' -o $(BUILD_BIN_PATH)/pd-recover tools/pd-recover/main.go +pd-recover-fips: + ENABLE_FIPS=1 $(MAKE) pd-recover pd-analysis: CGO_ENABLED=0 go build -gcflags '$(GCFLAGS)' -ldflags '$(LDFLAGS)' -o $(BUILD_BIN_PATH)/pd-analysis tools/pd-analysis/main.go pd-heartbeat-bench: @@ -112,7 +130,7 @@ regions-dump: stores-dump: CGO_ENABLED=0 go build -gcflags '$(GCFLAGS)' -ldflags '$(LDFLAGS)' -o $(BUILD_BIN_PATH)/stores-dump tools/stores-dump/main.go -.PHONY: pd-ctl pd-tso-bench pd-recover pd-analysis pd-heartbeat-bench simulator regions-dump stores-dump pd-api-bench +.PHONY: pd-ctl pd-ctl-fips pd-tso-bench pd-recover pd-recover-fips pd-analysis pd-heartbeat-bench simulator regions-dump stores-dump pd-api-bench #### Docker image #### diff --git a/pkg/versioninfo/fips.go b/pkg/versioninfo/fips.go new file mode 100644 index 00000000000..02478b103fa --- /dev/null +++ b/pkg/versioninfo/fips.go @@ -0,0 +1,26 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build boringcrypto +// +build boringcrypto + +package versioninfo + +import ( + _ "crypto/tls/fipsonly" +) + +func init() { + PDReleaseVersion += "-fips" +} From 0ebf4b26421f8347e02ca7a8f73f168f34b1ad0d Mon Sep 17 00:00:00 2001 From: Hu# Date: Thu, 16 Nov 2023 10:09:46 +0800 Subject: [PATCH 017/137] member: avoid frequent campaign times (#7301) close tikv/pd#7251, ref tikv/pd#7377 when pd leader frequently campaign leader, but etcd leader did not change. We need to prevent this pd leader campaign and resign to another member. Signed-off-by: husharp Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- client/client.go | 2 +- pkg/election/leadership.go | 32 ++++++++++++++++++++---- pkg/mcs/resourcemanager/server/server.go | 2 +- pkg/mcs/scheduling/server/server.go | 2 +- pkg/member/member.go | 12 ++++++++- pkg/member/participant.go | 2 +- pkg/tso/allocator_manager.go | 8 +++--- pkg/tso/global_allocator.go | 2 +- server/server.go | 2 +- tests/cluster.go | 7 ++++++ tests/server/member/member_test.go | 24 ++++++++++++++++++ 11 files changed, 79 insertions(+), 16 deletions(-) diff --git a/client/client.go b/client/client.go index 56923b697e2..2d30d9fb6c4 100644 --- a/client/client.go +++ b/client/client.go @@ -136,7 +136,7 @@ type Client interface { LoadGlobalConfig(ctx context.Context, names []string, configPath string) ([]GlobalConfigItem, int64, error) // StoreGlobalConfig set the config from etcd StoreGlobalConfig(ctx context.Context, configPath string, items []GlobalConfigItem) error - // WatchGlobalConfig returns an stream with all global config and updates + // WatchGlobalConfig returns a stream with all global config and updates WatchGlobalConfig(ctx context.Context, configPath string, revision int64) (chan []GlobalConfigItem, error) // UpdateOption updates the client option. UpdateOption(option DynamicOption, value interface{}) error diff --git a/pkg/election/leadership.go b/pkg/election/leadership.go index 8cfdcf423ac..572dae132b6 100644 --- a/pkg/election/leadership.go +++ b/pkg/election/leadership.go @@ -32,7 +32,10 @@ import ( "go.uber.org/zap" ) -const watchLoopUnhealthyTimeout = 60 * time.Second +const ( + watchLoopUnhealthyTimeout = 60 * time.Second + campaignTimesRecordTimeout = 5 * time.Minute +) // GetLeader gets the corresponding leader from etcd by given leaderPath (as the key). func GetLeader(c *clientv3.Client, leaderPath string) (*pdpb.Member, int64, error) { @@ -62,20 +65,24 @@ type Leadership struct { keepAliveCtx context.Context keepAliveCancelFunc context.CancelFunc keepAliveCancelFuncLock syncutil.Mutex + // CampaignTimes is used to record the campaign times of the leader within `campaignTimesRecordTimeout`. + // It is ordered by time to prevent the leader from campaigning too frequently. + CampaignTimes []time.Time } // NewLeadership creates a new Leadership. func NewLeadership(client *clientv3.Client, leaderKey, purpose string) *Leadership { leadership := &Leadership{ - purpose: purpose, - client: client, - leaderKey: leaderKey, + purpose: purpose, + client: client, + leaderKey: leaderKey, + CampaignTimes: make([]time.Time, 0, 10), } return leadership } // getLease gets the lease of leadership, only if leadership is valid, -// i.e the owner is a true leader, the lease is not nil. +// i.e. the owner is a true leader, the lease is not nil. func (ls *Leadership) getLease() *lease { l := ls.lease.Load() if l == nil { @@ -104,8 +111,23 @@ func (ls *Leadership) GetLeaderKey() string { return ls.leaderKey } +// addCampaignTimes is used to add the campaign times of the leader. +func (ls *Leadership) addCampaignTimes() { + for i := len(ls.CampaignTimes) - 1; i >= 0; i-- { + if time.Since(ls.CampaignTimes[i]) > campaignTimesRecordTimeout { + // remove the time which is more than `campaignTimesRecordTimeout` + // array is sorted by time + ls.CampaignTimes = ls.CampaignTimes[i:] + break + } + } + + ls.CampaignTimes = append(ls.CampaignTimes, time.Now()) +} + // Campaign is used to campaign the leader with given lease and returns a leadership func (ls *Leadership) Campaign(leaseTimeout int64, leaderData string, cmps ...clientv3.Cmp) error { + ls.addCampaignTimes() ls.leaderValue = leaderData // Create a new lease to campaign newLease := &lease{ diff --git a/pkg/mcs/resourcemanager/server/server.go b/pkg/mcs/resourcemanager/server/server.go index 7b660c07605..2a1be3e0ca5 100644 --- a/pkg/mcs/resourcemanager/server/server.go +++ b/pkg/mcs/resourcemanager/server/server.go @@ -152,7 +152,7 @@ func (s *Server) primaryElectionLoop() { func (s *Server) campaignLeader() { log.Info("start to campaign the primary/leader", zap.String("campaign-resource-manager-primary-name", s.participant.Name())) - if err := s.participant.CampaignLeader(s.cfg.LeaderLease); err != nil { + if err := s.participant.CampaignLeader(s.Context(), s.cfg.LeaderLease); err != nil { if err.Error() == errs.ErrEtcdTxnConflict.Error() { log.Info("campaign resource manager primary meets error due to txn conflict, another server may campaign successfully", zap.String("campaign-resource-manager-primary-name", s.participant.Name())) diff --git a/pkg/mcs/scheduling/server/server.go b/pkg/mcs/scheduling/server/server.go index 4304ffb218a..32b241fee91 100644 --- a/pkg/mcs/scheduling/server/server.go +++ b/pkg/mcs/scheduling/server/server.go @@ -241,7 +241,7 @@ func (s *Server) primaryElectionLoop() { func (s *Server) campaignLeader() { log.Info("start to campaign the primary/leader", zap.String("campaign-scheduling-primary-name", s.participant.Name())) - if err := s.participant.CampaignLeader(s.cfg.LeaderLease); err != nil { + if err := s.participant.CampaignLeader(s.Context(), s.cfg.LeaderLease); err != nil { if err.Error() == errs.ErrEtcdTxnConflict.Error() { log.Info("campaign scheduling primary meets error due to txn conflict, another server may campaign successfully", zap.String("campaign-scheduling-primary-name", s.participant.Name())) diff --git a/pkg/member/member.go b/pkg/member/member.go index 80332a65f94..6eddf9a7c77 100644 --- a/pkg/member/member.go +++ b/pkg/member/member.go @@ -42,6 +42,8 @@ const ( // The timeout to wait transfer etcd leader to complete. moveLeaderTimeout = 5 * time.Second dcLocationConfigEtcdPrefix = "dc-location" + // If the campaign times is more than this value in `campaignTimesRecordTimeout`, the PD will resign and campaign again. + campaignLeaderFrequencyTimes = 3 ) // EmbeddedEtcdMember is used for the election related logic. It implements Member interface. @@ -177,7 +179,15 @@ func (m *EmbeddedEtcdMember) GetLastLeaderUpdatedTime() time.Time { // CampaignLeader is used to campaign a PD member's leadership // and make it become a PD leader. -func (m *EmbeddedEtcdMember) CampaignLeader(leaseTimeout int64) error { +// leader should be changed when campaign leader frequently. +func (m *EmbeddedEtcdMember) CampaignLeader(ctx context.Context, leaseTimeout int64) error { + if len(m.leadership.CampaignTimes) >= campaignLeaderFrequencyTimes { + log.Warn("campaign times is too frequent, resign and campaign again", + zap.String("leader-name", m.Name()), zap.String("leader-key", m.GetLeaderPath())) + // remove all campaign times + m.leadership.CampaignTimes = nil + return m.ResignEtcdLeader(ctx, m.Name(), "") + } return m.leadership.Campaign(leaseTimeout, m.MemberValue()) } diff --git a/pkg/member/participant.go b/pkg/member/participant.go index b3034a86807..82cd7e05f5e 100644 --- a/pkg/member/participant.go +++ b/pkg/member/participant.go @@ -196,7 +196,7 @@ func (m *Participant) GetLeadership() *election.Leadership { } // CampaignLeader is used to campaign the leadership and make it become a leader. -func (m *Participant) CampaignLeader(leaseTimeout int64) error { +func (m *Participant) CampaignLeader(_ context.Context, leaseTimeout int64) error { if !m.campaignCheck() { return errs.ErrCheckCampaign } diff --git a/pkg/tso/allocator_manager.go b/pkg/tso/allocator_manager.go index df0ca0affc9..251a3aaf2e6 100644 --- a/pkg/tso/allocator_manager.go +++ b/pkg/tso/allocator_manager.go @@ -101,13 +101,13 @@ func (info *DCLocationInfo) clone() DCLocationInfo { type ElectionMember interface { // ID returns the unique ID in the election group. For example, it can be unique // server id of a cluster or the unique keyspace group replica id of the election - // group comprised of the replicas of a keyspace group. + // group composed of the replicas of a keyspace group. ID() uint64 - // ID returns the unique name in the election group. + // Name returns the unique name in the election group. Name() string // MemberValue returns the member value. MemberValue() string - // GetMember() returns the current member + // GetMember returns the current member GetMember() interface{} // Client returns the etcd client. Client() *clientv3.Client @@ -124,7 +124,7 @@ type ElectionMember interface { // KeepLeader is used to keep the leader's leadership. KeepLeader(ctx context.Context) // CampaignLeader is used to campaign the leadership and make it become a leader in an election group. - CampaignLeader(leaseTimeout int64) error + CampaignLeader(ctx context.Context, leaseTimeout int64) error // ResetLeader is used to reset the member's current leadership. // Basically it will reset the leader lease and unset leader info. ResetLeader() diff --git a/pkg/tso/global_allocator.go b/pkg/tso/global_allocator.go index 613ceb3eafc..a37bcc73881 100644 --- a/pkg/tso/global_allocator.go +++ b/pkg/tso/global_allocator.go @@ -568,7 +568,7 @@ func (gta *GlobalTSOAllocator) campaignLeader() { log.Info("start to campaign the primary", logutil.CondUint32("keyspace-group-id", gta.getGroupID(), gta.getGroupID() > 0), zap.String("campaign-tso-primary-name", gta.member.Name())) - if err := gta.am.member.CampaignLeader(gta.am.leaderLease); err != nil { + if err := gta.am.member.CampaignLeader(gta.ctx, gta.am.leaderLease); err != nil { if errors.Is(err, errs.ErrEtcdTxnConflict) { log.Info("campaign tso primary meets error due to txn conflict, another tso server may campaign successfully", logutil.CondUint32("keyspace-group-id", gta.getGroupID(), gta.getGroupID() > 0), diff --git a/server/server.go b/server/server.go index a2c99d0cbec..38064a3b92f 100644 --- a/server/server.go +++ b/server/server.go @@ -1636,7 +1636,7 @@ func (s *Server) leaderLoop() { func (s *Server) campaignLeader() { log.Info(fmt.Sprintf("start to campaign %s leader", s.mode), zap.String("campaign-leader-name", s.Name())) - if err := s.member.CampaignLeader(s.cfg.LeaderLease); err != nil { + if err := s.member.CampaignLeader(s.ctx, s.cfg.LeaderLease); err != nil { if err.Error() == errs.ErrEtcdTxnConflict.Error() { log.Info(fmt.Sprintf("campaign %s leader meets error due to txn conflict, another PD/API server may campaign successfully", s.mode), zap.String("campaign-leader-name", s.Name())) diff --git a/tests/cluster.go b/tests/cluster.go index ae1ae331856..41efc2b045d 100644 --- a/tests/cluster.go +++ b/tests/cluster.go @@ -155,6 +155,13 @@ func (s *TestServer) Destroy() error { return nil } +// ResetPDLeader resigns the leader of the server. +func (s *TestServer) ResetPDLeader() { + s.Lock() + defer s.Unlock() + s.server.GetMember().ResetLeader() +} + // ResignLeader resigns the leader of the server. func (s *TestServer) ResignLeader() error { s.Lock() diff --git a/tests/server/member/member_test.go b/tests/server/member/member_test.go index 26d4fa2a904..5965f9e22a6 100644 --- a/tests/server/member/member_test.go +++ b/tests/server/member/member_test.go @@ -323,6 +323,30 @@ func TestMoveLeader(t *testing.T) { } } +func TestCampaignLeaderFrequently(t *testing.T) { + re := require.New(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + cluster, err := tests.NewTestCluster(ctx, 5) + defer cluster.Destroy() + re.NoError(err) + + err = cluster.RunInitialServers() + re.NoError(err) + cluster.WaitLeader() + leader := cluster.GetLeader() + re.NotEmpty(cluster.GetLeader()) + + for i := 0; i < 3; i++ { + cluster.GetServers()[cluster.GetLeader()].ResetPDLeader() + cluster.WaitLeader() + } + // leader should be changed when campaign leader frequently + cluster.WaitLeader() + re.NotEmpty(cluster.GetLeader()) + re.NotEqual(leader, cluster.GetLeader()) +} + func TestGetLeader(t *testing.T) { re := require.New(t) ctx, cancel := context.WithCancel(context.Background()) From 112d6dc1cb7b57eba1639e6b465297de4490d8dd Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Thu, 16 Nov 2023 11:09:46 +0800 Subject: [PATCH 018/137] mcs: make scheduling server test stable (#7367) close tikv/pd#7362 Signed-off-by: lhy1024 Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- pkg/utils/testutil/api_check.go | 15 ++ server/api/diagnostic_test.go | 3 + .../mcs/scheduling/config_test.go | 3 + tests/pdctl/config/config_test.go | 183 ++++++++++-------- tests/server/api/rule_test.go | 69 ++++--- tests/server/api/scheduler_test.go | 17 +- tests/server/config/config_test.go | 10 +- 7 files changed, 177 insertions(+), 123 deletions(-) diff --git a/pkg/utils/testutil/api_check.go b/pkg/utils/testutil/api_check.go index 4ce5e859f3f..ea91654b149 100644 --- a/pkg/utils/testutil/api_check.go +++ b/pkg/utils/testutil/api_check.go @@ -114,6 +114,21 @@ func CheckGetJSON(client *http.Client, url string, data []byte, checkOpts ...fun return checkResp(resp, checkOpts...) } +// CheckGetUntilStatusCode is used to do get request and do check options. +func CheckGetUntilStatusCode(re *require.Assertions, client *http.Client, url string, code int) error { + var err error + Eventually(re, func() bool { + resp, err2 := apiutil.GetJSON(client, url, nil) + if err2 != nil { + err = err2 + return true + } + defer resp.Body.Close() + return resp.StatusCode == code + }) + return err +} + // CheckPatchJSON is used to do patch request and do check options. func CheckPatchJSON(client *http.Client, url string, data []byte, checkOpts ...func([]byte, int, http.Header)) error { resp, err := apiutil.PatchJSON(client, url, data) diff --git a/server/api/diagnostic_test.go b/server/api/diagnostic_test.go index 1774c221539..4e08426ea43 100644 --- a/server/api/diagnostic_test.go +++ b/server/api/diagnostic_test.go @@ -17,6 +17,7 @@ package api import ( "encoding/json" "fmt" + "net/http" "testing" "time" @@ -63,6 +64,8 @@ func (suite *diagnosticTestSuite) TearDownSuite() { func (suite *diagnosticTestSuite) checkStatus(status string, url string) { re := suite.Require() + err := tu.CheckGetUntilStatusCode(re, testDialClient, url, http.StatusOK) + suite.NoError(err) suite.Eventually(func() bool { result := &schedulers.DiagnosticResult{} err := tu.ReadGetJSON(re, testDialClient, url, result) diff --git a/tests/integrations/mcs/scheduling/config_test.go b/tests/integrations/mcs/scheduling/config_test.go index 42ba051eb84..06d73caf130 100644 --- a/tests/integrations/mcs/scheduling/config_test.go +++ b/tests/integrations/mcs/scheduling/config_test.go @@ -93,6 +93,9 @@ func (suite *configTestSuite) TestConfigWatch() { re.Equal(sc.DefaultSplitMergeInterval, watcher.GetScheduleConfig().SplitMergeInterval.Duration) re.Equal("0.0.0", watcher.GetClusterVersion().String()) // Update the config and check if the scheduling config watcher can get the latest value. + testutil.Eventually(re, func() bool { + return watcher.GetReplicationConfig().MaxReplicas == 3 + }) persistOpts := suite.pdLeaderServer.GetPersistOptions() persistOpts.SetMaxReplicas(5) persistConfig(re, suite.pdLeaderServer) diff --git a/tests/pdctl/config/config_test.go b/tests/pdctl/config/config_test.go index 2cc8427911a..315ec3cf7c7 100644 --- a/tests/pdctl/config/config_test.go +++ b/tests/pdctl/config/config_test.go @@ -315,12 +315,7 @@ func (suite *configTestSuite) checkPlacementRules(cluster *tests.TestCluster) { re.Contains(string(output), "Success!") // test show - var rules []placement.Rule - output, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "show") - re.NoError(err) - re.NoError(json.Unmarshal(output, &rules)) - re.Len(rules, 1) - re.Equal([2]string{"pd", "default"}, rules[0].Key()) + suite.checkShowRuleKey(pdAddr, [][2]string{{"pd", "default"}}) f, _ := os.CreateTemp("/tmp", "pd_tests") fname := f.Name() @@ -328,12 +323,7 @@ func (suite *configTestSuite) checkPlacementRules(cluster *tests.TestCluster) { defer os.RemoveAll(fname) // test load - _, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "load", "--out="+fname) - re.NoError(err) - b, _ := os.ReadFile(fname) - re.NoError(json.Unmarshal(b, &rules)) - re.Len(rules, 1) - re.Equal([2]string{"pd", "default"}, rules[0].Key()) + rules := suite.checkLoadRule(pdAddr, fname, [][2]string{{"pd", "default"}}) // test save rules = append(rules, placement.Rule{ @@ -347,42 +337,26 @@ func (suite *configTestSuite) checkPlacementRules(cluster *tests.TestCluster) { Role: "voter", Count: 2, }) - b, _ = json.Marshal(rules) + b, _ := json.Marshal(rules) os.WriteFile(fname, b, 0600) _, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "save", "--in="+fname) re.NoError(err) // test show group - var rules2 []placement.Rule - output, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "show", "--group=pd") - re.NoError(err) - re.NoError(json.Unmarshal(output, &rules2)) - re.Len(rules2, 2) - re.Equal([2]string{"pd", "default"}, rules2[0].Key()) - re.Equal([2]string{"pd", "test1"}, rules2[1].Key()) + suite.checkShowRuleKey(pdAddr, [][2]string{{"pd", "default"}, {"pd", "test1"}}, "--group=pd") // test rule region detail tests.MustPutRegion(re, cluster, 1, 1, []byte("a"), []byte("b")) - fit := &placement.RegionFit{} - // need clear up args, so create new a cobra.Command. Otherwise gourp still exists. - cmd2 := pdctlCmd.GetRootCmd() - output, err = pdctl.ExecuteCommand(cmd2, "-u", pdAddr, "config", "placement-rules", "show", "--region=1", "--detail") - re.NoError(err) - re.NoError(json.Unmarshal(output, fit)) - re.Len(fit.RuleFits, 3) - re.Equal([2]string{"pd", "default"}, fit.RuleFits[0].Rule.Key()) + suite.checkShowRuleKey(pdAddr, [][2]string{{"pd", "default"}}, "--region=1", "--detail") // test delete + // need clear up args, so create new a cobra.Command. Otherwise gourp still exists. rules[0].Count = 0 b, _ = json.Marshal(rules) os.WriteFile(fname, b, 0600) _, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "save", "--in="+fname) re.NoError(err) - output, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "show", "--group=pd") - re.NoError(err) - re.NoError(json.Unmarshal(output, &rules)) - re.Len(rules, 1) - re.Equal([2]string{"pd", "test1"}, rules[0].Key()) + suite.checkShowRuleKey(pdAddr, [][2]string{{"pd", "test1"}}, "--group=pd") } func (suite *configTestSuite) TestPlacementRuleGroups() { @@ -431,14 +405,16 @@ func (suite *configTestSuite) checkPlacementRuleGroups(cluster *tests.TestCluste // show all var groups []placement.RuleGroup - output, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-group", "show") - re.NoError(err) - re.NoError(json.Unmarshal(output, &groups)) - re.Equal([]placement.RuleGroup{ - {ID: "pd", Index: 42, Override: true}, - {ID: "group2", Index: 100, Override: false}, - {ID: "group3", Index: 200, Override: false}, - }, groups) + testutil.Eventually(re, func() bool { // wait for the config to be synced to the scheduling server + output, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-group", "show") + re.NoError(err) + re.NoError(json.Unmarshal(output, &groups)) + return reflect.DeepEqual([]placement.RuleGroup{ + {ID: "pd", Index: 42, Override: true}, + {ID: "group2", Index: 100, Override: false}, + {ID: "group3", Index: 200, Override: false}, + }, groups) + }) // delete output, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-group", "delete", "group2") @@ -446,17 +422,21 @@ func (suite *configTestSuite) checkPlacementRuleGroups(cluster *tests.TestCluste re.Contains(string(output), "Delete group and rules successfully.") // show again - output, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-group", "show", "group2") - re.NoError(err) - re.Contains(string(output), "404") + testutil.Eventually(re, func() bool { // wait for the config to be synced to the scheduling server + output, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-group", "show", "group2") + re.NoError(err) + return strings.Contains(string(output), "404") + }) // delete using regex _, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-group", "delete", "--regexp", ".*3") re.NoError(err) - _, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-group", "show", "group3") - re.NoError(err) - re.Contains(string(output), "404") + testutil.Eventually(re, func() bool { // wait for the config to be synced to the scheduling server + output, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-group", "show", "group3") + re.NoError(err) + return strings.Contains(string(output), "404") + }) } func (suite *configTestSuite) TestPlacementRuleBundle() { @@ -496,28 +476,19 @@ func (suite *configTestSuite) checkPlacementRuleBundle(cluster *tests.TestCluste defer os.RemoveAll(fname) // test load - var bundles []placement.GroupBundle - _, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-bundle", "load", "--out="+fname) - re.NoError(err) - b, _ := os.ReadFile(fname) - re.NoError(json.Unmarshal(b, &bundles)) - re.Len(bundles, 1) - re.Equal(placement.GroupBundle{ID: "pd", Index: 0, Override: false, Rules: []*placement.Rule{{GroupID: "pd", ID: "default", Role: "voter", Count: 3}}}, bundles[0]) + suite.checkLoadRuleBundle(pdAddr, fname, []placement.GroupBundle{ + {ID: "pd", Index: 0, Override: false, Rules: []*placement.Rule{{GroupID: "pd", ID: "default", Role: "voter", Count: 3}}}, + }) // test set bundle.ID = "pe" bundle.Rules[0].GroupID = "pe" - b, err = json.Marshal(bundle) + b, err := json.Marshal(bundle) re.NoError(err) re.NoError(os.WriteFile(fname, b, 0600)) _, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-bundle", "set", "--in="+fname) re.NoError(err) - - _, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-bundle", "load", "--out="+fname) - re.NoError(err) - b, _ = os.ReadFile(fname) - re.NoError(json.Unmarshal(b, &bundles)) - assertBundles(re, bundles, []placement.GroupBundle{ + suite.checkLoadRuleBundle(pdAddr, fname, []placement.GroupBundle{ {ID: "pd", Index: 0, Override: false, Rules: []*placement.Rule{{GroupID: "pd", ID: "default", Role: "voter", Count: 3}}}, {ID: "pe", Index: 0, Override: false, Rules: []*placement.Rule{{GroupID: "pe", ID: "default", Role: "voter", Count: 3}}}, }) @@ -526,11 +497,7 @@ func (suite *configTestSuite) checkPlacementRuleBundle(cluster *tests.TestCluste _, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-bundle", "delete", "pd") re.NoError(err) - _, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-bundle", "load", "--out="+fname) - re.NoError(err) - b, _ = os.ReadFile(fname) - re.NoError(json.Unmarshal(b, &bundles)) - assertBundles(re, bundles, []placement.GroupBundle{ + suite.checkLoadRuleBundle(pdAddr, fname, []placement.GroupBundle{ {ID: "pe", Index: 0, Override: false, Rules: []*placement.Rule{{GroupID: "pe", ID: "default", Role: "voter", Count: 3}}}, }) @@ -542,17 +509,18 @@ func (suite *configTestSuite) checkPlacementRuleBundle(cluster *tests.TestCluste re.NoError(os.WriteFile(fname, b, 0600)) _, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-bundle", "set", "--in="+fname) re.NoError(err) + suite.checkLoadRuleBundle(pdAddr, fname, []placement.GroupBundle{ + {ID: "pe", Index: 0, Override: false, Rules: []*placement.Rule{{GroupID: "pe", ID: "default", Role: "voter", Count: 3}}}, + {ID: "pf", Index: 0, Override: false, Rules: []*placement.Rule{{GroupID: "pf", ID: "default", Role: "voter", Count: 3}}}, + }) _, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-bundle", "delete", "--regexp", ".*f") re.NoError(err) - _, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-bundle", "load", "--out="+fname) - re.NoError(err) - b, _ = os.ReadFile(fname) - re.NoError(json.Unmarshal(b, &bundles)) - assertBundles(re, bundles, []placement.GroupBundle{ + bundles := []placement.GroupBundle{ {ID: "pe", Index: 0, Override: false, Rules: []*placement.Rule{{GroupID: "pe", ID: "default", Role: "voter", Count: 3}}}, - }) + } + suite.checkLoadRuleBundle(pdAddr, fname, bundles) // test save bundle.Rules = []*placement.Rule{{GroupID: "pf", ID: "default", Role: "voter", Count: 3}} @@ -562,13 +530,7 @@ func (suite *configTestSuite) checkPlacementRuleBundle(cluster *tests.TestCluste re.NoError(os.WriteFile(fname, b, 0600)) _, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-bundle", "save", "--in="+fname) re.NoError(err) - - _, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-bundle", "load", "--out="+fname) - re.NoError(err) - b, err = os.ReadFile(fname) - re.NoError(err) - re.NoError(json.Unmarshal(b, &bundles)) - assertBundles(re, bundles, []placement.GroupBundle{ + suite.checkLoadRuleBundle(pdAddr, fname, []placement.GroupBundle{ {ID: "pe", Index: 0, Override: false, Rules: []*placement.Rule{{GroupID: "pe", ID: "default", Role: "voter", Count: 3}}}, {ID: "pf", Index: 0, Override: false, Rules: []*placement.Rule{{GroupID: "pf", ID: "default", Role: "voter", Count: 3}}}, }) @@ -581,16 +543,67 @@ func (suite *configTestSuite) checkPlacementRuleBundle(cluster *tests.TestCluste _, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-bundle", "save", "--in="+fname, "--partial") re.NoError(err) - _, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-bundle", "load", "--out="+fname) - re.NoError(err) - b, err = os.ReadFile(fname) - re.NoError(err) - re.NoError(json.Unmarshal(b, &bundles)) - assertBundles(re, bundles, []placement.GroupBundle{ + suite.checkLoadRuleBundle(pdAddr, fname, []placement.GroupBundle{ {ID: "pf", Index: 0, Override: false, Rules: []*placement.Rule{{GroupID: "pf", ID: "default", Role: "voter", Count: 3}}}, }) } +func (suite *configTestSuite) checkLoadRuleBundle(pdAddr string, fname string, expectValues []placement.GroupBundle) { + var bundles []placement.GroupBundle + cmd := pdctlCmd.GetRootCmd() + testutil.Eventually(suite.Require(), func() bool { // wait for the config to be synced to the scheduling server + _, err := pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-bundle", "load", "--out="+fname) + suite.NoError(err) + b, _ := os.ReadFile(fname) + suite.NoError(json.Unmarshal(b, &bundles)) + return len(bundles) == len(expectValues) + }) + assertBundles(suite.Require(), bundles, expectValues) +} + +func (suite *configTestSuite) checkLoadRule(pdAddr string, fname string, expectValues [][2]string) []placement.Rule { + var rules []placement.Rule + cmd := pdctlCmd.GetRootCmd() + testutil.Eventually(suite.Require(), func() bool { // wait for the config to be synced to the scheduling server + _, err := pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "load", "--out="+fname) + suite.NoError(err) + b, _ := os.ReadFile(fname) + suite.NoError(json.Unmarshal(b, &rules)) + return len(rules) == len(expectValues) + }) + for i, v := range expectValues { + suite.Equal(v, rules[i].Key()) + } + return rules +} + +func (suite *configTestSuite) checkShowRuleKey(pdAddr string, expectValues [][2]string, opts ...string) { + var rules []placement.Rule + var fit placement.RegionFit + cmd := pdctlCmd.GetRootCmd() + testutil.Eventually(suite.Require(), func() bool { // wait for the config to be synced to the scheduling server + args := []string{"-u", pdAddr, "config", "placement-rules", "show"} + output, err := pdctl.ExecuteCommand(cmd, append(args, opts...)...) + suite.NoError(err) + err = json.Unmarshal(output, &rules) + if err == nil { + return len(rules) == len(expectValues) + } + suite.NoError(json.Unmarshal(output, &fit)) + return len(fit.RuleFits) != 0 + }) + if len(rules) != 0 { + for i, v := range expectValues { + suite.Equal(v, rules[i].Key()) + } + } + if len(fit.RuleFits) != 0 { + for i, v := range expectValues { + suite.Equal(v, fit.RuleFits[i].Rule.Key()) + } + } +} + func TestReplicationMode(t *testing.T) { re := require.New(t) ctx, cancel := context.WithCancel(context.Background()) diff --git a/tests/server/api/rule_test.go b/tests/server/api/rule_test.go index 3ee3357e031..6d292021767 100644 --- a/tests/server/api/rule_test.go +++ b/tests/server/api/rule_test.go @@ -210,8 +210,10 @@ func (suite *ruleTestSuite) checkGet(cluster *tests.TestCluster) { var resp placement.Rule url := fmt.Sprintf("%s/rule/%s/%s", urlPrefix, testCase.rule.GroupID, testCase.rule.ID) if testCase.found { - err = tu.ReadGetJSON(re, testDialClient, url, &resp) - suite.compareRule(&resp, &testCase.rule) + tu.Eventually(suite.Require(), func() bool { + err = tu.ReadGetJSON(re, testDialClient, url, &resp) + return suite.compareRule(&resp, &testCase.rule) + }) } else { err = tu.CheckGetJSON(testDialClient, url, nil, tu.Status(re, testCase.code)) } @@ -421,13 +423,17 @@ func (suite *ruleTestSuite) checkGetAllByGroup(cluster *tests.TestCluster) { suite.T().Log(testCase.name) var resp []*placement.Rule url := fmt.Sprintf("%s/rules/group/%s", urlPrefix, testCase.groupID) - err = tu.ReadGetJSON(re, testDialClient, url, &resp) - suite.NoError(err) - suite.Len(resp, testCase.count) - if testCase.count == 2 { - suite.compareRule(resp[0], &rule) - suite.compareRule(resp[1], &rule1) - } + tu.Eventually(re, func() bool { + err = tu.ReadGetJSON(re, testDialClient, url, &resp) + suite.NoError(err) + if len(resp) != testCase.count { + return false + } + if testCase.count == 2 { + return suite.compareRule(resp[0], &rule) && suite.compareRule(resp[1], &rule1) + } + return true + }) } } @@ -487,12 +493,15 @@ func (suite *ruleTestSuite) checkGetAllByRegion(cluster *tests.TestCluster) { url := fmt.Sprintf("%s/rules/region/%s", urlPrefix, testCase.regionID) if testCase.success { - err = tu.ReadGetJSON(re, testDialClient, url, &resp) - for _, r := range resp { - if r.GroupID == "e" { - suite.compareRule(r, &rule) + tu.Eventually(suite.Require(), func() bool { + err = tu.ReadGetJSON(re, testDialClient, url, &resp) + for _, r := range resp { + if r.GroupID == "e" { + return suite.compareRule(r, &rule) + } } - } + return true + }) } else { err = tu.CheckGetJSON(testDialClient, url, nil, tu.Status(re, testCase.code)) } @@ -956,22 +965,26 @@ func (suite *ruleTestSuite) checkBundleBadRequest(cluster *tests.TestCluster) { } func (suite *ruleTestSuite) compareBundle(b1, b2 placement.GroupBundle) { - suite.Equal(b2.ID, b1.ID) - suite.Equal(b2.Index, b1.Index) - suite.Equal(b2.Override, b1.Override) - suite.Len(b2.Rules, len(b1.Rules)) - for i := range b1.Rules { - suite.compareRule(b1.Rules[i], b2.Rules[i]) - } + tu.Eventually(suite.Require(), func() bool { + if b2.ID != b1.ID || b2.Index != b1.Index || b2.Override != b1.Override || len(b2.Rules) != len(b1.Rules) { + return false + } + for i := range b1.Rules { + if !suite.compareRule(b1.Rules[i], b2.Rules[i]) { + return false + } + } + return true + }) } -func (suite *ruleTestSuite) compareRule(r1 *placement.Rule, r2 *placement.Rule) { - suite.Equal(r2.GroupID, r1.GroupID) - suite.Equal(r2.ID, r1.ID) - suite.Equal(r2.StartKeyHex, r1.StartKeyHex) - suite.Equal(r2.EndKeyHex, r1.EndKeyHex) - suite.Equal(r2.Role, r1.Role) - suite.Equal(r2.Count, r1.Count) +func (suite *ruleTestSuite) compareRule(r1 *placement.Rule, r2 *placement.Rule) bool { + return r2.GroupID == r1.GroupID && + r2.ID == r1.ID && + r2.StartKeyHex == r1.StartKeyHex && + r2.EndKeyHex == r1.EndKeyHex && + r2.Role == r1.Role && + r2.Count == r1.Count } type regionRuleTestSuite struct { diff --git a/tests/server/api/scheduler_test.go b/tests/server/api/scheduler_test.go index 38f691a4eda..4d6dde6f2b9 100644 --- a/tests/server/api/scheduler_test.go +++ b/tests/server/api/scheduler_test.go @@ -18,6 +18,7 @@ import ( "encoding/json" "fmt" "net/http" + "reflect" "testing" "time" @@ -447,18 +448,22 @@ func (suite *scheduleTestSuite) checkAPI(cluster *tests.TestCluster) { suite.NoError(err) suite.NoError(tu.CheckPostJSON(testDialClient, updateURL, body, tu.StatusOK(re))) resp = make(map[string]interface{}) - suite.NoError(tu.ReadGetJSON(re, testDialClient, listURL, &resp)) - exceptMap["4"] = []interface{}{map[string]interface{}{"end-key": "", "start-key": ""}} - suite.Equal(exceptMap, resp["store-id-ranges"]) + tu.Eventually(re, func() bool { + suite.NoError(tu.ReadGetJSON(re, testDialClient, listURL, &resp)) + exceptMap["4"] = []interface{}{map[string]interface{}{"end-key": "", "start-key": ""}} + return reflect.DeepEqual(exceptMap, resp["store-id-ranges"]) + }) // using /pd/v1/schedule-config/evict-leader-scheduler/config to delete exist store from evict-leader-scheduler deleteURL := fmt.Sprintf("%s%s%s/%s/delete/%s", leaderAddr, apiPrefix, server.SchedulerConfigHandlerPath, name, "4") err = tu.CheckDelete(testDialClient, deleteURL, tu.StatusOK(re)) suite.NoError(err) resp = make(map[string]interface{}) - suite.NoError(tu.ReadGetJSON(re, testDialClient, listURL, &resp)) - delete(exceptMap, "4") - suite.Equal(exceptMap, resp["store-id-ranges"]) + tu.Eventually(re, func() bool { + suite.NoError(tu.ReadGetJSON(re, testDialClient, listURL, &resp)) + delete(exceptMap, "4") + return reflect.DeepEqual(exceptMap, resp["store-id-ranges"]) + }) err = tu.CheckDelete(testDialClient, deleteURL, tu.Status(re, http.StatusNotFound)) suite.NoError(err) }, diff --git a/tests/server/config/config_test.go b/tests/server/config/config_test.go index 8d8cf40e692..4a4a91f2661 100644 --- a/tests/server/config/config_test.go +++ b/tests/server/config/config_test.go @@ -20,6 +20,7 @@ import ( "encoding/json" "fmt" "net/http" + "reflect" "testing" "time" @@ -272,10 +273,11 @@ func (suite *configTestSuite) checkConfigReplication(cluster *tests.TestCluster) suite.NoError(err) rc4 := &sc.ReplicationConfig{} - err = tu.ReadGetJSON(re, testDialClient, addr, rc4) - suite.NoError(err) - - suite.Equal(*rc4, *rc) + tu.Eventually(re, func() bool { + err = tu.ReadGetJSON(re, testDialClient, addr, rc4) + suite.NoError(err) + return reflect.DeepEqual(*rc4, *rc) + }) } func (suite *configTestSuite) TestConfigLabelProperty() { From 4e600c227e83b0b9f92693ae21ff04e2028e1e7d Mon Sep 17 00:00:00 2001 From: glorv Date: Thu, 16 Nov 2023 11:44:17 +0800 Subject: [PATCH 019/137] resourcemanager: return resource-group priority in OnRequestWait (#7378) close tikv/pd#7379, ref tikv/tikv#15994 Signed-off-by: glorv Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- .../resource_group/controller/controller.go | 12 +++---- .../controller/controller_test.go | 8 +++-- .../resourcemanager/resource_manager_test.go | 34 +++++++++---------- 3 files changed, 28 insertions(+), 26 deletions(-) diff --git a/client/resource_group/controller/controller.go b/client/resource_group/controller/controller.go index b528351bedf..01011c2c30a 100755 --- a/client/resource_group/controller/controller.go +++ b/client/resource_group/controller/controller.go @@ -57,7 +57,7 @@ const ( // ResourceGroupKVInterceptor is used as quota limit controller for resource group using kv store. type ResourceGroupKVInterceptor interface { // OnRequestWait is used to check whether resource group has enough tokens. It maybe needs to wait some time. - OnRequestWait(ctx context.Context, resourceGroupName string, info RequestInfo) (*rmpb.Consumption, *rmpb.Consumption, error) + OnRequestWait(ctx context.Context, resourceGroupName string, info RequestInfo) (*rmpb.Consumption, *rmpb.Consumption, uint32, error) // OnResponse is used to consume tokens after receiving response. OnResponse(resourceGroupName string, req RequestInfo, resp ResponseInfo) (*rmpb.Consumption, error) // IsBackgroundRequest If the resource group has background jobs, we should not record consumption and wait for it. @@ -526,10 +526,10 @@ func (c *ResourceGroupsController) sendTokenBucketRequests(ctx context.Context, // OnRequestWait is used to check whether resource group has enough tokens. It maybe needs to wait some time. func (c *ResourceGroupsController) OnRequestWait( ctx context.Context, resourceGroupName string, info RequestInfo, -) (*rmpb.Consumption, *rmpb.Consumption, error) { +) (*rmpb.Consumption, *rmpb.Consumption, uint32, error) { gc, err := c.tryGetResourceGroup(ctx, resourceGroupName) if err != nil { - return nil, nil, err + return nil, nil, 0, err } return gc.onRequestWait(ctx, info) } @@ -1176,7 +1176,7 @@ func (gc *groupCostController) calcRequest(counter *tokenCounter) float64 { func (gc *groupCostController) onRequestWait( ctx context.Context, info RequestInfo, -) (*rmpb.Consumption, *rmpb.Consumption, error) { +) (*rmpb.Consumption, *rmpb.Consumption, uint32, error) { delta := &rmpb.Consumption{} for _, calc := range gc.calculators { calc.BeforeKVRequest(delta, info) @@ -1226,7 +1226,7 @@ func (gc *groupCostController) onRequestWait( failpoint.Inject("triggerUpdate", func() { gc.lowRUNotifyChan <- struct{}{} }) - return nil, nil, err + return nil, nil, 0, err } gc.successfulRequestDuration.Observe(d.Seconds()) } @@ -1245,7 +1245,7 @@ func (gc *groupCostController) onRequestWait( *gc.mu.storeCounter[info.StoreID()] = *gc.mu.globalCounter gc.mu.Unlock() - return delta, penalty, nil + return delta, penalty, gc.meta.Priority, nil } func (gc *groupCostController) onResponse( diff --git a/client/resource_group/controller/controller_test.go b/client/resource_group/controller/controller_test.go index 6877f8206f3..1db19787a81 100644 --- a/client/resource_group/controller/controller_test.go +++ b/client/resource_group/controller/controller_test.go @@ -30,8 +30,9 @@ import ( func createTestGroupCostController(re *require.Assertions) *groupCostController { group := &rmpb.ResourceGroup{ - Name: "test", - Mode: rmpb.GroupMode_RUMode, + Name: "test", + Mode: rmpb.GroupMode_RUMode, + Priority: 1, RUSettings: &rmpb.GroupRequestUnitSettings{ RU: &rmpb.TokenBucket{ Settings: &rmpb.TokenLimitSettings{ @@ -100,8 +101,9 @@ func TestRequestAndResponseConsumption(t *testing.T) { kvCalculator := gc.getKVCalculator() for idx, testCase := range testCases { caseNum := fmt.Sprintf("case %d", idx) - consumption, _, err := gc.onRequestWait(context.TODO(), testCase.req) + consumption, _, priority, err := gc.onRequestWait(context.TODO(), testCase.req) re.NoError(err, caseNum) + re.Equal(priority, gc.meta.Priority) expectedConsumption := &rmpb.Consumption{} if testCase.req.IsWrite() { kvCalculator.calculateWriteCost(expectedConsumption, testCase.req) diff --git a/tests/integrations/mcs/resourcemanager/resource_manager_test.go b/tests/integrations/mcs/resourcemanager/resource_manager_test.go index ed6a3ee501c..91a21caf91b 100644 --- a/tests/integrations/mcs/resourcemanager/resource_manager_test.go +++ b/tests/integrations/mcs/resourcemanager/resource_manager_test.go @@ -438,9 +438,9 @@ func (suite *resourceManagerClientTestSuite) TestResourceGroupController() { rres := cas.tcs[i].makeReadResponse() wres := cas.tcs[i].makeWriteResponse() startTime := time.Now() - _, _, err := controller.OnRequestWait(suite.ctx, cas.resourceGroupName, rreq) + _, _, _, err := controller.OnRequestWait(suite.ctx, cas.resourceGroupName, rreq) re.NoError(err) - _, _, err = controller.OnRequestWait(suite.ctx, cas.resourceGroupName, wreq) + _, _, _, err = controller.OnRequestWait(suite.ctx, cas.resourceGroupName, wreq) re.NoError(err) sum += time.Since(startTime) controller.OnResponse(cas.resourceGroupName, rreq, rres) @@ -457,7 +457,7 @@ func (suite *resourceManagerClientTestSuite) TestResourceGroupController() { re.NoError(failpoint.Enable("github.com/tikv/pd/client/resource_group/controller/triggerUpdate", "return(true)")) tcs := tokenConsumptionPerSecond{rruTokensAtATime: 1, wruTokensAtATime: 900000000, times: 1, waitDuration: 0} wreq := tcs.makeWriteRequest() - _, _, err = controller.OnRequestWait(suite.ctx, rg.Name, wreq) + _, _, _, err = controller.OnRequestWait(suite.ctx, rg.Name, wreq) re.Error(err) time.Sleep(time.Millisecond * 200) re.NoError(failpoint.Disable("github.com/tikv/pd/client/resource_group/controller/triggerUpdate")) @@ -512,9 +512,9 @@ func (suite *resourceManagerClientTestSuite) TestSwitchBurst() { wreq := tcs.makeWriteRequest() rres := tcs.makeReadResponse() wres := tcs.makeWriteResponse() - _, _, err := controller.OnRequestWait(suite.ctx, resourceGroupName, rreq) + _, _, _, err := controller.OnRequestWait(suite.ctx, resourceGroupName, rreq) re.NoError(err) - _, _, err = controller.OnRequestWait(suite.ctx, resourceGroupName, wreq) + _, _, _, err = controller.OnRequestWait(suite.ctx, resourceGroupName, wreq) re.NoError(err) controller.OnResponse(resourceGroupName, rreq, rres) controller.OnResponse(resourceGroupName, wreq, wres) @@ -551,9 +551,9 @@ func (suite *resourceManagerClientTestSuite) TestSwitchBurst() { rres := cas.tcs[i].makeReadResponse() wres := cas.tcs[i].makeWriteResponse() startTime := time.Now() - _, _, err := controller.OnRequestWait(suite.ctx, resourceGroupName, rreq) + _, _, _, err := controller.OnRequestWait(suite.ctx, resourceGroupName, rreq) re.NoError(err) - _, _, err = controller.OnRequestWait(suite.ctx, resourceGroupName, wreq) + _, _, _, err = controller.OnRequestWait(suite.ctx, resourceGroupName, wreq) re.NoError(err) sum += time.Since(startTime) controller.OnResponse(resourceGroupName, rreq, rres) @@ -571,14 +571,14 @@ func (suite *resourceManagerClientTestSuite) TestSwitchBurst() { resourceGroupName2 := suite.initGroups[2].Name tcs = tokenConsumptionPerSecond{rruTokensAtATime: 1, wruTokensAtATime: 100000, times: 1, waitDuration: 0} wreq := tcs.makeWriteRequest() - _, _, err := controller.OnRequestWait(suite.ctx, resourceGroupName2, wreq) + _, _, _, err := controller.OnRequestWait(suite.ctx, resourceGroupName2, wreq) re.NoError(err) re.NoError(failpoint.Enable("github.com/tikv/pd/client/resource_group/controller/acceleratedSpeedTrend", "return(true)")) resourceGroupName3 := suite.initGroups[3].Name tcs = tokenConsumptionPerSecond{rruTokensAtATime: 1, wruTokensAtATime: 1000, times: 1, waitDuration: 0} wreq = tcs.makeWriteRequest() - _, _, err = controller.OnRequestWait(suite.ctx, resourceGroupName3, wreq) + _, _, _, err = controller.OnRequestWait(suite.ctx, resourceGroupName3, wreq) re.NoError(err) time.Sleep(110 * time.Millisecond) tcs = tokenConsumptionPerSecond{rruTokensAtATime: 1, wruTokensAtATime: 10, times: 1010, waitDuration: 0} @@ -586,7 +586,7 @@ func (suite *resourceManagerClientTestSuite) TestSwitchBurst() { for i := 0; i < tcs.times; i++ { wreq = tcs.makeWriteRequest() startTime := time.Now() - _, _, err = controller.OnRequestWait(suite.ctx, resourceGroupName3, wreq) + _, _, _, err = controller.OnRequestWait(suite.ctx, resourceGroupName3, wreq) duration += time.Since(startTime) re.NoError(err) } @@ -635,7 +635,7 @@ func (suite *resourceManagerClientTestSuite) TestResourcePenalty() { // init req := controller.NewTestRequestInfo(false, 0, 2 /* store2 */) resp := controller.NewTestResponseInfo(0, time.Duration(30), true) - _, penalty, err := c.OnRequestWait(suite.ctx, resourceGroupName, req) + _, penalty, _, err := c.OnRequestWait(suite.ctx, resourceGroupName, req) re.NoError(err) re.Equal(penalty.WriteBytes, 0.0) re.Equal(penalty.TotalCpuTimeMs, 0.0) @@ -644,7 +644,7 @@ func (suite *resourceManagerClientTestSuite) TestResourcePenalty() { req = controller.NewTestRequestInfo(true, 60, 1 /* store1 */) resp = controller.NewTestResponseInfo(0, time.Duration(10), true) - _, penalty, err = c.OnRequestWait(suite.ctx, resourceGroupName, req) + _, penalty, _, err = c.OnRequestWait(suite.ctx, resourceGroupName, req) re.NoError(err) re.Equal(penalty.WriteBytes, 0.0) re.Equal(penalty.TotalCpuTimeMs, 0.0) @@ -654,7 +654,7 @@ func (suite *resourceManagerClientTestSuite) TestResourcePenalty() { // failed request, shouldn't be counted in penalty req = controller.NewTestRequestInfo(true, 20, 1 /* store1 */) resp = controller.NewTestResponseInfo(0, time.Duration(0), false) - _, penalty, err = c.OnRequestWait(suite.ctx, resourceGroupName, req) + _, penalty, _, err = c.OnRequestWait(suite.ctx, resourceGroupName, req) re.NoError(err) re.Equal(penalty.WriteBytes, 0.0) re.Equal(penalty.TotalCpuTimeMs, 0.0) @@ -664,7 +664,7 @@ func (suite *resourceManagerClientTestSuite) TestResourcePenalty() { // from same store, should be zero req1 := controller.NewTestRequestInfo(false, 0, 1 /* store1 */) resp1 := controller.NewTestResponseInfo(0, time.Duration(10), true) - _, penalty, err = c.OnRequestWait(suite.ctx, resourceGroupName, req1) + _, penalty, _, err = c.OnRequestWait(suite.ctx, resourceGroupName, req1) re.NoError(err) re.Equal(penalty.WriteBytes, 0.0) _, err = c.OnResponse(resourceGroupName, req1, resp1) @@ -673,7 +673,7 @@ func (suite *resourceManagerClientTestSuite) TestResourcePenalty() { // from different store, should be non-zero req2 := controller.NewTestRequestInfo(true, 50, 2 /* store2 */) resp2 := controller.NewTestResponseInfo(0, time.Duration(10), true) - _, penalty, err = c.OnRequestWait(suite.ctx, resourceGroupName, req2) + _, penalty, _, err = c.OnRequestWait(suite.ctx, resourceGroupName, req2) re.NoError(err) re.Equal(penalty.WriteBytes, 60.0) re.InEpsilon(penalty.TotalCpuTimeMs, 10.0/1000.0/1000.0, 1e-6) @@ -683,7 +683,7 @@ func (suite *resourceManagerClientTestSuite) TestResourcePenalty() { // from new store, should be zero req3 := controller.NewTestRequestInfo(true, 0, 3 /* store3 */) resp3 := controller.NewTestResponseInfo(0, time.Duration(10), true) - _, penalty, err = c.OnRequestWait(suite.ctx, resourceGroupName, req3) + _, penalty, _, err = c.OnRequestWait(suite.ctx, resourceGroupName, req3) re.NoError(err) re.Equal(penalty.WriteBytes, 0.0) _, err = c.OnResponse(resourceGroupName, req3, resp3) @@ -693,7 +693,7 @@ func (suite *resourceManagerClientTestSuite) TestResourcePenalty() { resourceGroupName = groupNames[1] req4 := controller.NewTestRequestInfo(true, 50, 1 /* store2 */) resp4 := controller.NewTestResponseInfo(0, time.Duration(10), true) - _, penalty, err = c.OnRequestWait(suite.ctx, resourceGroupName, req4) + _, penalty, _, err = c.OnRequestWait(suite.ctx, resourceGroupName, req4) re.NoError(err) re.Equal(penalty.WriteBytes, 0.0) _, err = c.OnResponse(resourceGroupName, req4, resp4) From 95557847375274910fe624ddae9bbae5f7dca003 Mon Sep 17 00:00:00 2001 From: Yongbo Jiang Date: Thu, 16 Nov 2023 17:11:16 +0800 Subject: [PATCH 020/137] makefile: delete redundant build commands (#7376) ref tikv/pd#7274 Signed-off-by: Cabinfever_B Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- Makefile | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index 906dd9414f9..bf76fd57f2f 100644 --- a/Makefile +++ b/Makefile @@ -75,8 +75,6 @@ BUILD_BIN_PATH := $(ROOT_PATH)/bin build: pd-server pd-ctl pd-recover -build-fips: pd-server-fips pd-ctl-fips pd-recover-fips - tools: pd-tso-bench pd-heartbeat-bench regions-dump stores-dump pd-api-bench PD_SERVER_DEP := @@ -100,25 +98,18 @@ pd-server-failpoint: pd-server-basic: SWAGGER=0 DASHBOARD=0 $(MAKE) pd-server -pd-server-fips: - ENABLE_FIPS=1 $(MAKE) pd-server - -.PHONY: build tools pd-server pd-server-basic pd-server-fips +.PHONY: build tools pd-server pd-server-basic # Tools pd-ctl: GOEXPERIMENT=$(BUILD_GOEXPERIMENT) CGO_ENABLED=$(BUILD_TOOL_CGO_ENABLED) go build -gcflags '$(GCFLAGS)' -ldflags '$(LDFLAGS)' -o $(BUILD_BIN_PATH)/pd-ctl tools/pd-ctl/main.go -pd-ctl-fips: - ENABLE_FIPS=1 $(MAKE) pd-ctl pd-tso-bench: cd tools/pd-tso-bench && CGO_ENABLED=0 go build -o $(BUILD_BIN_PATH)/pd-tso-bench main.go pd-api-bench: cd tools/pd-api-bench && CGO_ENABLED=0 go build -o $(BUILD_BIN_PATH)/pd-api-bench main.go pd-recover: GOEXPERIMENT=$(BUILD_GOEXPERIMENT) CGO_ENABLED=$(BUILD_TOOL_CGO_ENABLED) go build -gcflags '$(GCFLAGS)' -ldflags '$(LDFLAGS)' -o $(BUILD_BIN_PATH)/pd-recover tools/pd-recover/main.go -pd-recover-fips: - ENABLE_FIPS=1 $(MAKE) pd-recover pd-analysis: CGO_ENABLED=0 go build -gcflags '$(GCFLAGS)' -ldflags '$(LDFLAGS)' -o $(BUILD_BIN_PATH)/pd-analysis tools/pd-analysis/main.go pd-heartbeat-bench: @@ -130,7 +121,7 @@ regions-dump: stores-dump: CGO_ENABLED=0 go build -gcflags '$(GCFLAGS)' -ldflags '$(LDFLAGS)' -o $(BUILD_BIN_PATH)/stores-dump tools/stores-dump/main.go -.PHONY: pd-ctl pd-ctl-fips pd-tso-bench pd-recover pd-recover-fips pd-analysis pd-heartbeat-bench simulator regions-dump stores-dump pd-api-bench +.PHONY: pd-ctl pd-tso-bench pd-recover pd-analysis pd-heartbeat-bench simulator regions-dump stores-dump pd-api-bench #### Docker image #### From a6800a9dfad52f228f242fec2d335ea9bbaa170b Mon Sep 17 00:00:00 2001 From: iosmanthus Date: Thu, 16 Nov 2023 17:42:46 +0800 Subject: [PATCH 021/137] etcdutil: remove stale client endpoints for `healthyChecker` (#7227) close tikv/pd#7226 remove stale client endpoints for `healthyChecker` Signed-off-by: iosmanthus Co-authored-by: lhy1024 --- pkg/utils/etcdutil/etcdutil.go | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/pkg/utils/etcdutil/etcdutil.go b/pkg/utils/etcdutil/etcdutil.go index e004247c6d0..03c2374efc6 100644 --- a/pkg/utils/etcdutil/etcdutil.go +++ b/pkg/utils/etcdutil/etcdutil.go @@ -382,13 +382,18 @@ func (checker *healthyChecker) patrol(ctx context.Context) []string { } func (checker *healthyChecker) update(eps []string) { + epMap := make(map[string]struct{}) for _, ep := range eps { + epMap[ep] = struct{}{} + } + + for ep := range epMap { // check if client exists, if not, create one, if exists, check if it's offline or disconnected. if client, ok := checker.Load(ep); ok { lastHealthy := client.(*healthyClient).lastHealth if time.Since(lastHealthy) > etcdServerOfflineTimeout { log.Info("some etcd server maybe offline", zap.String("endpoint", ep)) - checker.Delete(ep) + checker.removeClient(ep) } if time.Since(lastHealthy) > etcdServerDisconnectedTimeout { // try to reset client endpoint to trigger reconnect @@ -399,6 +404,16 @@ func (checker *healthyChecker) update(eps []string) { } checker.addClient(ep, time.Now()) } + + // check if there are some stale clients, if exists, remove them. + checker.Range(func(key, value interface{}) bool { + ep := key.(string) + if _, ok := epMap[ep]; !ok { + log.Info("remove stale etcd client", zap.String("endpoint", ep)) + checker.removeClient(ep) + } + return true + }) } func (checker *healthyChecker) addClient(ep string, lastHealth time.Time) { @@ -413,6 +428,15 @@ func (checker *healthyChecker) addClient(ep string, lastHealth time.Time) { }) } +func (checker *healthyChecker) removeClient(ep string) { + if client, ok := checker.LoadAndDelete(ep); ok { + err := client.(*healthyClient).Close() + if err != nil { + log.Error("failed to close etcd healthy client", zap.Error(err)) + } + } +} + func syncUrls(client *clientv3.Client) []string { // See https://github.com/etcd-io/etcd/blob/85b640cee793e25f3837c47200089d14a8392dc7/clientv3/client.go#L170-L183 ctx, cancel := context.WithTimeout(clientv3.WithRequireLeader(client.Ctx()), DefaultRequestTimeout) From f2eaf23e94d2ea267fb423a1058eab17ed5ab754 Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Thu, 16 Nov 2023 18:45:46 +0800 Subject: [PATCH 022/137] tests: make TestUpdateAfterResetTSO stable (#7385) close tikv/pd#7381 Signed-off-by: lhy1024 Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- pkg/member/member.go | 4 ++++ tests/integrations/tso/client_test.go | 5 ++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/pkg/member/member.go b/pkg/member/member.go index 6eddf9a7c77..dd36214a595 100644 --- a/pkg/member/member.go +++ b/pkg/member/member.go @@ -27,6 +27,7 @@ import ( "time" "github.com/pingcap/errors" + "github.com/pingcap/failpoint" "github.com/pingcap/kvproto/pkg/pdpb" "github.com/pingcap/log" "github.com/tikv/pd/pkg/election" @@ -181,6 +182,9 @@ func (m *EmbeddedEtcdMember) GetLastLeaderUpdatedTime() time.Time { // and make it become a PD leader. // leader should be changed when campaign leader frequently. func (m *EmbeddedEtcdMember) CampaignLeader(ctx context.Context, leaseTimeout int64) error { + failpoint.Inject("skipCampaignLeaderCheck", func() { + failpoint.Return(m.leadership.Campaign(leaseTimeout, m.MemberValue())) + }) if len(m.leadership.CampaignTimes) >= campaignLeaderFrequencyTimes { log.Warn("campaign times is too frequent, resign and campaign again", zap.String("leader-name", m.Name()), zap.String("leader-key", m.GetLeaderPath())) diff --git a/tests/integrations/tso/client_test.go b/tests/integrations/tso/client_test.go index 63243214e81..73198690966 100644 --- a/tests/integrations/tso/client_test.go +++ b/tests/integrations/tso/client_test.go @@ -300,7 +300,10 @@ func (suite *tsoClientTestSuite) TestUpdateAfterResetTSO() { re := suite.Require() ctx, cancel := context.WithCancel(suite.ctx) defer cancel() - + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/member/skipCampaignLeaderCheck", "return(true)")) + defer func() { + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/member/skipCampaignLeaderCheck")) + }() for i := 0; i < len(suite.clients); i++ { client := suite.clients[i] testutil.Eventually(re, func() bool { From dda748abe55dffbb9b0b67fa582eb5e7231918f2 Mon Sep 17 00:00:00 2001 From: JmPotato Date: Fri, 17 Nov 2023 12:17:18 +0800 Subject: [PATCH 023/137] client/http: implement more HTTP APIs (#7371) ref tikv/pd#7300 - Implement more HTTP APIs. - Use consts more in `Rule` structure. Signed-off-by: JmPotato --- client/http/api.go | 112 +++++++++++++-- client/http/client.go | 126 ++++++++++++++--- client/http/types.go | 70 ++++++++++ pkg/mock/mockcluster/config.go | 4 +- pkg/schedule/checker/merge_checker_test.go | 4 +- pkg/schedule/checker/rule_checker_test.go | 130 +++++++++--------- pkg/schedule/filter/filters_test.go | 2 +- pkg/schedule/operator/create_operator_test.go | 4 +- pkg/schedule/placement/fit_region_test.go | 46 +++---- .../placement/region_rule_cache_test.go | 12 +- pkg/schedule/placement/rule_manager.go | 21 ++- pkg/schedule/placement/rule_manager_test.go | 118 ++++++++-------- pkg/schedule/placement/rule_test.go | 16 +-- pkg/schedule/scatter/region_scatterer_test.go | 8 +- .../schedulers/balance_witness_test.go | 4 +- pkg/schedule/schedulers/hot_region_test.go | 8 +- pkg/schedule/schedulers/scheduler_test.go | 18 +-- server/api/cluster_test.go | 5 +- server/api/region_test.go | 6 +- server/api/rule.go | 2 +- server/cluster/cluster_test.go | 16 +-- server/server.go | 2 +- tests/integrations/client/http_client_test.go | 43 ++++++ tests/integrations/mcs/scheduling/api_test.go | 6 +- .../integrations/mcs/scheduling/rule_test.go | 10 +- tests/pdctl/config/config_test.go | 62 ++++----- tests/server/api/operator_test.go | 2 +- tests/server/api/rule_test.go | 67 ++++----- .../simulator/cases/diagnose_rule.go | 4 +- 29 files changed, 616 insertions(+), 312 deletions(-) diff --git a/client/http/api.go b/client/http/api.go index 5326919561d..2fae562dd20 100644 --- a/client/http/api.go +++ b/client/http/api.go @@ -17,24 +17,58 @@ package http import ( "fmt" "net/url" + "time" ) // The following constants are the paths of PD HTTP APIs. const ( - HotRead = "/pd/api/v1/hotspot/regions/read" - HotWrite = "/pd/api/v1/hotspot/regions/write" - Regions = "/pd/api/v1/regions" - regionByID = "/pd/api/v1/region/id" - regionByKey = "/pd/api/v1/region/key" - regionsByKey = "/pd/api/v1/regions/key" - regionsByStoreID = "/pd/api/v1/regions/store" - Stores = "/pd/api/v1/stores" + // Metadata + HotRead = "/pd/api/v1/hotspot/regions/read" + HotWrite = "/pd/api/v1/hotspot/regions/write" + HotHistory = "/pd/api/v1/hotspot/regions/history" + RegionByIDPrefix = "/pd/api/v1/region/id" + regionByKey = "/pd/api/v1/region/key" + Regions = "/pd/api/v1/regions" + regionsByKey = "/pd/api/v1/regions/key" + RegionsByStoreIDPrefix = "/pd/api/v1/regions/store" + EmptyRegions = "/pd/api/v1/regions/check/empty-region" + accelerateSchedule = "/pd/api/v1/regions/accelerate-schedule" + store = "/pd/api/v1/store" + Stores = "/pd/api/v1/stores" + StatsRegion = "/pd/api/v1/stats/region" + // Config + Config = "/pd/api/v1/config" + ClusterVersion = "/pd/api/v1/config/cluster-version" + ScheduleConfig = "/pd/api/v1/config/schedule" + ReplicateConfig = "/pd/api/v1/config/replicate" + // Rule + PlacementRule = "/pd/api/v1/config/rule" + PlacementRules = "/pd/api/v1/config/rules" + placementRulesByGroup = "/pd/api/v1/config/rules/group" + RegionLabelRule = "/pd/api/v1/config/region-label/rule" + // Scheduler + Schedulers = "/pd/api/v1/schedulers" + scatterRangeScheduler = "/pd/api/v1/schedulers/scatter-range-" + // Admin + ResetTS = "/pd/api/v1/admin/reset-ts" + BaseAllocID = "/pd/api/v1/admin/base-alloc-id" + SnapshotRecoveringMark = "/pd/api/v1/admin/cluster/markers/snapshot-recovering" + // Debug + PProfProfile = "/pd/api/v1/debug/pprof/profile" + PProfHeap = "/pd/api/v1/debug/pprof/heap" + PProfMutex = "/pd/api/v1/debug/pprof/mutex" + PProfAllocs = "/pd/api/v1/debug/pprof/allocs" + PProfBlock = "/pd/api/v1/debug/pprof/block" + PProfGoroutine = "/pd/api/v1/debug/pprof/goroutine" + // Others MinResolvedTSPrefix = "/pd/api/v1/min-resolved-ts" + Status = "/pd/api/v1/status" + Version = "/pd/api/v1/version" ) // RegionByID returns the path of PD HTTP API to get region by ID. func RegionByID(regionID uint64) string { - return fmt.Sprintf("%s/%d", regionByID, regionID) + return fmt.Sprintf("%s/%d", RegionByIDPrefix, regionID) } // RegionByKey returns the path of PD HTTP API to get region by key. @@ -45,10 +79,66 @@ func RegionByKey(key []byte) string { // RegionsByKey returns the path of PD HTTP API to scan regions with given start key, end key and limit parameters. func RegionsByKey(startKey, endKey []byte, limit int) string { return fmt.Sprintf("%s?start_key=%s&end_key=%s&limit=%d", - regionsByKey, url.QueryEscape(string(startKey)), url.QueryEscape(string(endKey)), limit) + regionsByKey, + url.QueryEscape(string(startKey)), + url.QueryEscape(string(endKey)), + limit) } // RegionsByStoreID returns the path of PD HTTP API to get regions by store ID. func RegionsByStoreID(storeID uint64) string { - return fmt.Sprintf("%s/%d", regionsByStoreID, storeID) + return fmt.Sprintf("%s/%d", RegionsByStoreIDPrefix, storeID) +} + +// RegionStatsByKeyRange returns the path of PD HTTP API to get region stats by start key and end key. +func RegionStatsByKeyRange(startKey, endKey []byte) string { + return fmt.Sprintf("%s?start_key=%s&end_key=%s", + StatsRegion, + url.QueryEscape(string(startKey)), + url.QueryEscape(string(endKey))) +} + +// StoreByID returns the store API with store ID parameter. +func StoreByID(id uint64) string { + return fmt.Sprintf("%s/%d", store, id) +} + +// StoreLabelByID returns the store label API with store ID parameter. +func StoreLabelByID(id uint64) string { + return fmt.Sprintf("%s/%d/label", store, id) +} + +// ConfigWithTTLSeconds returns the config API with the TTL seconds parameter. +func ConfigWithTTLSeconds(ttlSeconds float64) string { + return fmt.Sprintf("%s?ttlSecond=%.0f", Config, ttlSeconds) +} + +// PlacementRulesByGroup returns the path of PD HTTP API to get placement rules by group. +func PlacementRulesByGroup(group string) string { + return fmt.Sprintf("%s/%s", placementRulesByGroup, group) +} + +// PlacementRuleByGroupAndID returns the path of PD HTTP API to get placement rule by group and ID. +func PlacementRuleByGroupAndID(group, id string) string { + return fmt.Sprintf("%s/%s/%s", PlacementRule, group, id) +} + +// SchedulerByName returns the scheduler API with the given scheduler name. +func SchedulerByName(name string) string { + return fmt.Sprintf("%s/%s", Schedulers, name) +} + +// ScatterRangeSchedulerWithName returns the scatter range scheduler API with name parameter. +func ScatterRangeSchedulerWithName(name string) string { + return fmt.Sprintf("%s%s", scatterRangeScheduler, name) +} + +// PProfProfileAPIWithInterval returns the pprof profile API with interval parameter. +func PProfProfileAPIWithInterval(interval time.Duration) string { + return fmt.Sprintf("%s?seconds=%d", PProfProfile, interval/time.Second) +} + +// PProfGoroutineWithDebugLevel returns the pprof goroutine API with debug level parameter. +func PProfGoroutineWithDebugLevel(level int) string { + return fmt.Sprintf("%s?debug=%d", PProfGoroutine, level) } diff --git a/client/http/client.go b/client/http/client.go index 6cb1277dfcb..6fa2dd8cdfd 100644 --- a/client/http/client.go +++ b/client/http/client.go @@ -15,12 +15,14 @@ package http import ( + "bytes" "context" "crypto/tls" "encoding/json" "fmt" "io" "net/http" + "net/url" "strings" "time" @@ -43,12 +45,17 @@ type Client interface { GetRegionByID(context.Context, uint64) (*RegionInfo, error) GetRegionByKey(context.Context, []byte) (*RegionInfo, error) GetRegions(context.Context) (*RegionsInfo, error) - GetRegionsByKey(context.Context, []byte, []byte, int) (*RegionsInfo, error) + GetRegionsByKeyRange(context.Context, []byte, []byte, int) (*RegionsInfo, error) GetRegionsByStoreID(context.Context, uint64) (*RegionsInfo, error) GetHotReadRegions(context.Context) (*StoreHotPeersInfos, error) GetHotWriteRegions(context.Context) (*StoreHotPeersInfos, error) + GetRegionStatusByKeyRange(context.Context, []byte, []byte) (*RegionStats, error) GetStores(context.Context) (*StoresInfo, error) + GetPlacementRulesByGroup(context.Context, string) ([]*Rule, error) + SetPlacementRule(context.Context, *Rule) error + DeletePlacementRule(context.Context, string, string) error GetMinResolvedTSByStoresIDs(context.Context, []uint64) (uint64, map[uint64]uint64, error) + AccelerateSchedule(context.Context, []byte, []byte) error Close() } @@ -154,8 +161,8 @@ func (c *client) execDuration(name string, duration time.Duration) { // it consistent with the current implementation of some clients (e.g. TiDB). func (c *client) requestWithRetry( ctx context.Context, - name, uri string, - res interface{}, + name, uri, method string, + body io.Reader, res interface{}, ) error { var ( err error @@ -163,7 +170,7 @@ func (c *client) requestWithRetry( ) for idx := 0; idx < len(c.pdAddrs); idx++ { addr = c.pdAddrs[idx] - err = c.request(ctx, name, addr, uri, res) + err = c.request(ctx, name, fmt.Sprintf("%s%s", addr, uri), method, body, res) if err == nil { break } @@ -175,16 +182,15 @@ func (c *client) requestWithRetry( func (c *client) request( ctx context.Context, - name, addr, uri string, - res interface{}, + name, url, method string, + body io.Reader, res interface{}, ) error { - reqURL := fmt.Sprintf("%s%s", addr, uri) logFields := []zap.Field{ zap.String("name", name), - zap.String("url", reqURL), + zap.String("url", url), } log.Debug("[pd] request the http url", logFields...) - req, err := http.NewRequestWithContext(ctx, http.MethodGet, reqURL, nil) + req, err := http.NewRequestWithContext(ctx, method, url, body) if err != nil { log.Error("[pd] create http request failed", append(logFields, zap.Error(err))...) return errors.Trace(err) @@ -219,6 +225,10 @@ func (c *client) request( return errors.Errorf("request pd http api failed with status: '%s'", resp.Status) } + if res == nil { + return nil + } + err = json.NewDecoder(resp.Body).Decode(res) if err != nil { return errors.Trace(err) @@ -229,7 +239,9 @@ func (c *client) request( // GetRegionByID gets the region info by ID. func (c *client) GetRegionByID(ctx context.Context, regionID uint64) (*RegionInfo, error) { var region RegionInfo - err := c.requestWithRetry(ctx, "GetRegionByID", RegionByID(regionID), ®ion) + err := c.requestWithRetry(ctx, + "GetRegionByID", RegionByID(regionID), + http.MethodGet, nil, ®ion) if err != nil { return nil, err } @@ -239,7 +251,9 @@ func (c *client) GetRegionByID(ctx context.Context, regionID uint64) (*RegionInf // GetRegionByKey gets the region info by key. func (c *client) GetRegionByKey(ctx context.Context, key []byte) (*RegionInfo, error) { var region RegionInfo - err := c.requestWithRetry(ctx, "GetRegionByKey", RegionByKey(key), ®ion) + err := c.requestWithRetry(ctx, + "GetRegionByKey", RegionByKey(key), + http.MethodGet, nil, ®ion) if err != nil { return nil, err } @@ -249,17 +263,21 @@ func (c *client) GetRegionByKey(ctx context.Context, key []byte) (*RegionInfo, e // GetRegions gets the regions info. func (c *client) GetRegions(ctx context.Context) (*RegionsInfo, error) { var regions RegionsInfo - err := c.requestWithRetry(ctx, "GetRegions", Regions, ®ions) + err := c.requestWithRetry(ctx, + "GetRegions", Regions, + http.MethodGet, nil, ®ions) if err != nil { return nil, err } return ®ions, nil } -// GetRegionsByKey gets the regions info by key range. If the limit is -1, it will return all regions within the range. -func (c *client) GetRegionsByKey(ctx context.Context, startKey, endKey []byte, limit int) (*RegionsInfo, error) { +// GetRegionsByKeyRange gets the regions info by key range. If the limit is -1, it will return all regions within the range. +func (c *client) GetRegionsByKeyRange(ctx context.Context, startKey, endKey []byte, limit int) (*RegionsInfo, error) { var regions RegionsInfo - err := c.requestWithRetry(ctx, "GetRegionsByKey", RegionsByKey(startKey, endKey, limit), ®ions) + err := c.requestWithRetry(ctx, + "GetRegionsByKeyRange", RegionsByKey(startKey, endKey, limit), + http.MethodGet, nil, ®ions) if err != nil { return nil, err } @@ -269,7 +287,9 @@ func (c *client) GetRegionsByKey(ctx context.Context, startKey, endKey []byte, l // GetRegionsByStoreID gets the regions info by store ID. func (c *client) GetRegionsByStoreID(ctx context.Context, storeID uint64) (*RegionsInfo, error) { var regions RegionsInfo - err := c.requestWithRetry(ctx, "GetRegionsByStoreID", RegionsByStoreID(storeID), ®ions) + err := c.requestWithRetry(ctx, + "GetRegionsByStoreID", RegionsByStoreID(storeID), + http.MethodGet, nil, ®ions) if err != nil { return nil, err } @@ -279,7 +299,9 @@ func (c *client) GetRegionsByStoreID(ctx context.Context, storeID uint64) (*Regi // GetHotReadRegions gets the hot read region statistics info. func (c *client) GetHotReadRegions(ctx context.Context) (*StoreHotPeersInfos, error) { var hotReadRegions StoreHotPeersInfos - err := c.requestWithRetry(ctx, "GetHotReadRegions", HotRead, &hotReadRegions) + err := c.requestWithRetry(ctx, + "GetHotReadRegions", HotRead, + http.MethodGet, nil, &hotReadRegions) if err != nil { return nil, err } @@ -289,23 +311,70 @@ func (c *client) GetHotReadRegions(ctx context.Context) (*StoreHotPeersInfos, er // GetHotWriteRegions gets the hot write region statistics info. func (c *client) GetHotWriteRegions(ctx context.Context) (*StoreHotPeersInfos, error) { var hotWriteRegions StoreHotPeersInfos - err := c.requestWithRetry(ctx, "GetHotWriteRegions", HotWrite, &hotWriteRegions) + err := c.requestWithRetry(ctx, + "GetHotWriteRegions", HotWrite, + http.MethodGet, nil, &hotWriteRegions) if err != nil { return nil, err } return &hotWriteRegions, nil } +// GetRegionStatusByKeyRange gets the region status by key range. +func (c *client) GetRegionStatusByKeyRange(ctx context.Context, startKey, endKey []byte) (*RegionStats, error) { + var regionStats RegionStats + err := c.requestWithRetry(ctx, + "GetRegionStatusByKeyRange", RegionStatsByKeyRange(startKey, endKey), + http.MethodGet, nil, ®ionStats, + ) + if err != nil { + return nil, err + } + return ®ionStats, nil +} + // GetStores gets the stores info. func (c *client) GetStores(ctx context.Context) (*StoresInfo, error) { var stores StoresInfo - err := c.requestWithRetry(ctx, "GetStores", Stores, &stores) + err := c.requestWithRetry(ctx, + "GetStores", Stores, + http.MethodGet, nil, &stores) if err != nil { return nil, err } return &stores, nil } +// GetPlacementRulesByGroup gets the placement rules by group. +func (c *client) GetPlacementRulesByGroup(ctx context.Context, group string) ([]*Rule, error) { + var rules []*Rule + err := c.requestWithRetry(ctx, + "GetPlacementRulesByGroup", PlacementRulesByGroup(group), + http.MethodGet, nil, &rules) + if err != nil { + return nil, err + } + return rules, nil +} + +// SetPlacementRule sets the placement rule. +func (c *client) SetPlacementRule(ctx context.Context, rule *Rule) error { + ruleJSON, err := json.Marshal(rule) + if err != nil { + return errors.Trace(err) + } + return c.requestWithRetry(ctx, + "SetPlacementRule", PlacementRule, + http.MethodPost, bytes.NewBuffer(ruleJSON), nil) +} + +// DeletePlacementRule deletes the placement rule. +func (c *client) DeletePlacementRule(ctx context.Context, group, id string) error { + return c.requestWithRetry(ctx, + "DeletePlacementRule", PlacementRuleByGroupAndID(group, id), + http.MethodDelete, nil, nil) +} + // GetMinResolvedTSByStoresIDs get min-resolved-ts by stores IDs. func (c *client) GetMinResolvedTSByStoresIDs(ctx context.Context, storeIDs []uint64) (uint64, map[uint64]uint64, error) { uri := MinResolvedTSPrefix @@ -326,7 +395,9 @@ func (c *client) GetMinResolvedTSByStoresIDs(ctx context.Context, storeIDs []uin IsRealTime bool `json:"is_real_time,omitempty"` StoresMinResolvedTS map[uint64]uint64 `json:"stores_min_resolved_ts"` }{} - err := c.requestWithRetry(ctx, "GetMinResolvedTSByStoresIDs", uri, &resp) + err := c.requestWithRetry(ctx, + "GetMinResolvedTSByStoresIDs", uri, + http.MethodGet, nil, &resp) if err != nil { return 0, nil, err } @@ -335,3 +406,18 @@ func (c *client) GetMinResolvedTSByStoresIDs(ctx context.Context, storeIDs []uin } return resp.MinResolvedTS, resp.StoresMinResolvedTS, nil } + +// AccelerateSchedule accelerates the scheduling of the regions within the given key range. +func (c *client) AccelerateSchedule(ctx context.Context, startKey, endKey []byte) error { + input := map[string]string{ + "start_key": url.QueryEscape(string(startKey)), + "end_key": url.QueryEscape(string(endKey)), + } + inputJSON, err := json.Marshal(input) + if err != nil { + return errors.Trace(err) + } + return c.requestWithRetry(ctx, + "AccelerateSchedule", accelerateSchedule, + http.MethodPost, bytes.NewBuffer(inputJSON), nil) +} diff --git a/client/http/types.go b/client/http/types.go index 66eb31ec3a1..c6bb0256c14 100644 --- a/client/http/types.go +++ b/client/http/types.go @@ -176,3 +176,73 @@ type StoreStatus struct { LastHeartbeatTS time.Time `json:"last_heartbeat_ts"` Uptime string `json:"uptime"` } + +// RegionStats stores the statistics of regions. +type RegionStats struct { + Count int `json:"count"` + EmptyCount int `json:"empty_count"` + StorageSize int64 `json:"storage_size"` + StorageKeys int64 `json:"storage_keys"` + StoreLeaderCount map[uint64]int `json:"store_leader_count"` + StorePeerCount map[uint64]int `json:"store_peer_count"` +} + +// PeerRoleType is the expected peer type of the placement rule. +type PeerRoleType string + +const ( + // Voter can either match a leader peer or follower peer + Voter PeerRoleType = "voter" + // Leader matches a leader. + Leader PeerRoleType = "leader" + // Follower matches a follower. + Follower PeerRoleType = "follower" + // Learner matches a learner. + Learner PeerRoleType = "learner" +) + +// LabelConstraint is used to filter store when trying to place peer of a region. +type LabelConstraint struct { + Key string `json:"key,omitempty"` + Op LabelConstraintOp `json:"op,omitempty"` + Values []string `json:"values,omitempty"` +} + +// LabelConstraintOp defines how a LabelConstraint matches a store. It can be one of +// 'in', 'notIn', 'exists', or 'notExists'. +type LabelConstraintOp string + +const ( + // In restricts the store label value should in the value list. + // If label does not exist, `in` is always false. + In LabelConstraintOp = "in" + // NotIn restricts the store label value should not in the value list. + // If label does not exist, `notIn` is always true. + NotIn LabelConstraintOp = "notIn" + // Exists restricts the store should have the label. + Exists LabelConstraintOp = "exists" + // NotExists restricts the store should not have the label. + NotExists LabelConstraintOp = "notExists" +) + +// Rule is the placement rule that can be checked against a region. When +// applying rules (apply means schedule regions to match selected rules), the +// apply order is defined by the tuple [GroupIndex, GroupID, Index, ID]. +type Rule struct { + GroupID string `json:"group_id"` // mark the source that add the rule + ID string `json:"id"` // unique ID within a group + Index int `json:"index,omitempty"` // rule apply order in a group, rule with less ID is applied first when indexes are equal + Override bool `json:"override,omitempty"` // when it is true, all rules with less indexes are disabled + StartKey []byte `json:"-"` // range start key + StartKeyHex string `json:"start_key"` // hex format start key, for marshal/unmarshal + EndKey []byte `json:"-"` // range end key + EndKeyHex string `json:"end_key"` // hex format end key, for marshal/unmarshal + Role PeerRoleType `json:"role"` // expected role of the peers + IsWitness bool `json:"is_witness"` // when it is true, it means the role is also a witness + Count int `json:"count"` // expected count of the peers + LabelConstraints []LabelConstraint `json:"label_constraints,omitempty"` // used to select stores to place peers + LocationLabels []string `json:"location_labels,omitempty"` // used to make peers isolated physically + IsolationLevel string `json:"isolation_level,omitempty"` // used to isolate replicas explicitly and forcibly + Version uint64 `json:"version,omitempty"` // only set at runtime, add 1 each time rules updated, begin from 0. + CreateTimestamp uint64 `json:"create_timestamp,omitempty"` // only set at runtime, recorded rule create timestamp +} diff --git a/pkg/mock/mockcluster/config.go b/pkg/mock/mockcluster/config.go index 6febba026e8..a2e11b43deb 100644 --- a/pkg/mock/mockcluster/config.go +++ b/pkg/mock/mockcluster/config.go @@ -154,8 +154,8 @@ func (mc *Cluster) SetMaxReplicasWithLabel(enablePlacementRules bool, num int, l } if enablePlacementRules { rule := &placement.Rule{ - GroupID: "pd", - ID: "default", + GroupID: placement.DefaultGroupID, + ID: placement.DefaultRuleID, Index: 1, StartKey: []byte(""), EndKey: []byte(""), diff --git a/pkg/schedule/checker/merge_checker_test.go b/pkg/schedule/checker/merge_checker_test.go index 6478eb0b2c4..5e9311c76cd 100644 --- a/pkg/schedule/checker/merge_checker_test.go +++ b/pkg/schedule/checker/merge_checker_test.go @@ -188,7 +188,7 @@ func (suite *mergeCheckerTestSuite) TestBasic() { // merge cannot across rule key. suite.cluster.SetEnablePlacementRules(true) suite.cluster.RuleManager.SetRule(&placement.Rule{ - GroupID: "pd", + GroupID: placement.DefaultGroupID, ID: "test", Index: 1, Override: true, @@ -202,7 +202,7 @@ func (suite *mergeCheckerTestSuite) TestBasic() { suite.NotNil(ops) suite.Equal(suite.regions[2].GetID(), ops[0].RegionID()) suite.Equal(suite.regions[1].GetID(), ops[1].RegionID()) - suite.cluster.RuleManager.DeleteRule("pd", "test") + suite.cluster.RuleManager.DeleteRule(placement.DefaultGroupID, "test") // check 'merge_option' label suite.cluster.GetRegionLabeler().SetLabelRule(&labeler.LabelRule{ diff --git a/pkg/schedule/checker/rule_checker_test.go b/pkg/schedule/checker/rule_checker_test.go index 4185ce6c167..e77830fac49 100644 --- a/pkg/schedule/checker/rule_checker_test.go +++ b/pkg/schedule/checker/rule_checker_test.go @@ -88,7 +88,7 @@ func (suite *ruleCheckerTestSuite) TestAddRulePeerWithIsolationLevel() { suite.cluster.AddLabelsStore(4, 1, map[string]string{"zone": "z1", "rack": "r3", "host": "h1"}) suite.cluster.AddLeaderRegionWithRange(1, "", "", 1, 2) suite.ruleManager.SetRule(&placement.Rule{ - GroupID: "pd", + GroupID: placement.DefaultGroupID, ID: "test", Index: 100, Override: true, @@ -101,7 +101,7 @@ func (suite *ruleCheckerTestSuite) TestAddRulePeerWithIsolationLevel() { suite.Nil(op) suite.cluster.AddLeaderRegionWithRange(1, "", "", 1, 3) suite.ruleManager.SetRule(&placement.Rule{ - GroupID: "pd", + GroupID: placement.DefaultGroupID, ID: "test", Index: 100, Override: true, @@ -125,9 +125,9 @@ func (suite *ruleCheckerTestSuite) TestReplaceDownPeerWithIsolationLevel() { suite.cluster.AddLabelsStore(5, 1, map[string]string{"zone": "z3", "host": "h5"}) suite.cluster.AddLabelsStore(6, 1, map[string]string{"zone": "z3", "host": "h6"}) suite.cluster.AddLeaderRegionWithRange(1, "", "", 1, 3, 5) - suite.ruleManager.DeleteRule("pd", "default") + suite.ruleManager.DeleteRule(placement.DefaultGroupID, placement.DefaultRuleID) suite.ruleManager.SetRule(&placement.Rule{ - GroupID: "pd", + GroupID: placement.DefaultGroupID, ID: "test", Index: 100, Override: true, @@ -331,7 +331,7 @@ func (suite *ruleCheckerTestSuite) TestFixOrphanPeers2() { suite.cluster.AddLabelsStore(3, 1, map[string]string{"foo": "baz"}) suite.cluster.AddLeaderRegionWithRange(1, "", "", 1, 3) suite.ruleManager.SetRule(&placement.Rule{ - GroupID: "pd", + GroupID: placement.DefaultGroupID, ID: "r1", Index: 100, Override: true, @@ -367,7 +367,7 @@ func (suite *ruleCheckerTestSuite) TestFixRoleLeader() { suite.cluster.AddLabelsStore(3, 1, map[string]string{"role": "voter"}) suite.cluster.AddLeaderRegionWithRange(1, "", "", 1, 2, 3) suite.ruleManager.SetRule(&placement.Rule{ - GroupID: "pd", + GroupID: placement.DefaultGroupID, ID: "r1", Index: 100, Override: true, @@ -378,7 +378,7 @@ func (suite *ruleCheckerTestSuite) TestFixRoleLeader() { }, }) suite.ruleManager.SetRule(&placement.Rule{ - GroupID: "pd", + GroupID: placement.DefaultGroupID, ID: "r2", Index: 101, Role: placement.Follower, @@ -398,7 +398,7 @@ func (suite *ruleCheckerTestSuite) TestFixRoleLeaderIssue3130() { suite.cluster.AddLabelsStore(2, 1, map[string]string{"role": "leader"}) suite.cluster.AddLeaderRegion(1, 1, 2) suite.ruleManager.SetRule(&placement.Rule{ - GroupID: "pd", + GroupID: placement.DefaultGroupID, ID: "r1", Index: 100, Override: true, @@ -471,7 +471,7 @@ func (suite *ruleCheckerTestSuite) TestFixRuleWitness() { suite.cluster.AddLeaderRegion(1, 1) suite.ruleManager.SetRule(&placement.Rule{ - GroupID: "pd", + GroupID: placement.DefaultGroupID, ID: "r1", Index: 100, Override: true, @@ -497,7 +497,7 @@ func (suite *ruleCheckerTestSuite) TestFixRuleWitness2() { suite.cluster.AddLeaderRegion(1, 1, 2, 3, 4) suite.ruleManager.SetRule(&placement.Rule{ - GroupID: "pd", + GroupID: placement.DefaultGroupID, ID: "r1", Index: 100, Override: false, @@ -544,8 +544,8 @@ func (suite *ruleCheckerTestSuite) TestFixRuleWitness4() { err := suite.ruleManager.SetRules([]*placement.Rule{ { - GroupID: "pd", - ID: "default", + GroupID: placement.DefaultGroupID, + ID: placement.DefaultRuleID, Index: 100, Override: true, Role: placement.Voter, @@ -553,7 +553,7 @@ func (suite *ruleCheckerTestSuite) TestFixRuleWitness4() { IsWitness: false, }, { - GroupID: "pd", + GroupID: placement.DefaultGroupID, ID: "r1", Index: 100, Override: false, @@ -580,7 +580,7 @@ func (suite *ruleCheckerTestSuite) TestFixRuleWitness5() { suite.cluster.AddLeaderRegion(1, 1, 2, 3) err := suite.ruleManager.SetRule(&placement.Rule{ - GroupID: "pd", + GroupID: placement.DefaultGroupID, ID: "r1", Index: 100, Override: true, @@ -603,15 +603,15 @@ func (suite *ruleCheckerTestSuite) TestFixRuleWitness6() { err := suite.ruleManager.SetRules([]*placement.Rule{ { - GroupID: "pd", - ID: "default", + GroupID: placement.DefaultGroupID, + ID: placement.DefaultRuleID, Index: 100, Role: placement.Voter, IsWitness: false, Count: 2, }, { - GroupID: "pd", + GroupID: placement.DefaultGroupID, ID: "r1", Index: 100, Role: placement.Voter, @@ -641,15 +641,15 @@ func (suite *ruleCheckerTestSuite) TestDisableWitness() { err := suite.ruleManager.SetRules([]*placement.Rule{ { - GroupID: "pd", - ID: "default", + GroupID: placement.DefaultGroupID, + ID: placement.DefaultRuleID, Index: 100, Role: placement.Voter, IsWitness: false, Count: 2, }, { - GroupID: "pd", + GroupID: placement.DefaultGroupID, ID: "r1", Index: 100, Role: placement.Voter, @@ -680,7 +680,7 @@ func (suite *ruleCheckerTestSuite) TestBetterReplacement() { suite.cluster.AddLabelsStore(4, 1, map[string]string{"host": "host3"}) suite.cluster.AddLeaderRegionWithRange(1, "", "", 1, 2, 3) suite.ruleManager.SetRule(&placement.Rule{ - GroupID: "pd", + GroupID: placement.DefaultGroupID, ID: "test", Index: 100, Override: true, @@ -704,7 +704,7 @@ func (suite *ruleCheckerTestSuite) TestBetterReplacement2() { suite.cluster.AddLabelsStore(4, 1, map[string]string{"zone": "z2", "host": "host1"}) suite.cluster.AddLeaderRegionWithRange(1, "", "", 1, 2, 3) suite.ruleManager.SetRule(&placement.Rule{ - GroupID: "pd", + GroupID: placement.DefaultGroupID, ID: "test", Index: 100, Override: true, @@ -727,7 +727,7 @@ func (suite *ruleCheckerTestSuite) TestNoBetterReplacement() { suite.cluster.AddLabelsStore(3, 1, map[string]string{"host": "host2"}) suite.cluster.AddLeaderRegionWithRange(1, "", "", 1, 2, 3) suite.ruleManager.SetRule(&placement.Rule{ - GroupID: "pd", + GroupID: placement.DefaultGroupID, ID: "test", Index: 100, Override: true, @@ -835,8 +835,8 @@ func (suite *ruleCheckerTestSuite) TestFixOrphanPeerWithDisconnectedStoreAndRule suite.cluster.AddLabelsStore(5, 1, map[string]string{"host": "host5"}) suite.cluster.AddLeaderRegionWithRange(1, "", "", leader, followers...) rule := &placement.Rule{ - GroupID: "pd", - ID: "default", + GroupID: placement.DefaultGroupID, + ID: placement.DefaultRuleID, Role: placement.Voter, Count: 5, StartKey: []byte{}, @@ -853,8 +853,8 @@ func (suite *ruleCheckerTestSuite) TestFixOrphanPeerWithDisconnectedStoreAndRule // change rule to 3 replicas rule = &placement.Rule{ - GroupID: "pd", - ID: "default", + GroupID: placement.DefaultGroupID, + ID: placement.DefaultRuleID, Role: placement.Voter, Count: 3, StartKey: []byte{}, @@ -941,8 +941,8 @@ func (suite *ruleCheckerTestSuite) TestFixOrphanPeerWithDisconnectedStoreAndRule suite.cluster.AddLeaderRegionWithRange(1, "", "", leader, voterFollowers...) err := suite.ruleManager.SetRules([]*placement.Rule{ { - GroupID: "pd", - ID: "default", + GroupID: placement.DefaultGroupID, + ID: placement.DefaultRuleID, Index: 100, Override: true, Role: placement.Voter, @@ -950,7 +950,7 @@ func (suite *ruleCheckerTestSuite) TestFixOrphanPeerWithDisconnectedStoreAndRule IsWitness: false, }, { - GroupID: "pd", + GroupID: placement.DefaultGroupID, ID: "r1", Index: 100, Override: false, @@ -975,10 +975,10 @@ func (suite *ruleCheckerTestSuite) TestFixOrphanPeerWithDisconnectedStoreAndRule suite.cluster.SetStoreDisconnect(testCase[2]) // change rule to 3 replicas - suite.ruleManager.DeleteRule("pd", "r1") + suite.ruleManager.DeleteRule(placement.DefaultGroupID, "r1") suite.ruleManager.SetRule(&placement.Rule{ - GroupID: "pd", - ID: "default", + GroupID: placement.DefaultGroupID, + ID: placement.DefaultRuleID, Role: placement.Voter, Count: 3, StartKey: []byte{}, @@ -1106,13 +1106,13 @@ func (suite *ruleCheckerTestSuite) TestPriorityFitHealthPeersAndTiFlash() { suite.cluster.AddLabelsStore(4, 1, map[string]string{"host": "host4", "engine": "tiflash"}) suite.cluster.AddRegionWithLearner(1, 1, []uint64{2, 3}, []uint64{4}) rule := &placement.Rule{ - GroupID: "pd", + GroupID: placement.DefaultGroupID, ID: "test", Role: placement.Voter, Count: 3, } rule2 := &placement.Rule{ - GroupID: "pd", + GroupID: placement.DefaultGroupID, ID: "test2", Role: placement.Learner, Count: 1, @@ -1126,7 +1126,7 @@ func (suite *ruleCheckerTestSuite) TestPriorityFitHealthPeersAndTiFlash() { } suite.ruleManager.SetRule(rule) suite.ruleManager.SetRule(rule2) - suite.ruleManager.DeleteRule("pd", "default") + suite.ruleManager.DeleteRule(placement.DefaultGroupID, placement.DefaultRuleID) r1 := suite.cluster.GetRegion(1) // set peer3 to pending and down @@ -1177,12 +1177,12 @@ func (suite *ruleCheckerTestSuite) TestIssue3293() { suite.cluster.DeleteStore(suite.cluster.GetStore(5)) err = suite.ruleManager.SetRule(&placement.Rule{ GroupID: "TiDB_DDL_51", - ID: "default", + ID: placement.DefaultRuleID, Role: placement.Voter, Count: 3, }) suite.NoError(err) - err = suite.ruleManager.DeleteRule("pd", "default") + err = suite.ruleManager.DeleteRule(placement.DefaultGroupID, placement.DefaultRuleID) suite.NoError(err) op := suite.rc.Check(suite.cluster.GetRegion(1)) suite.NotNil(op) @@ -1290,7 +1290,7 @@ func (suite *ruleCheckerTestSuite) TestFixDownPeer() { suite.cluster.AddLabelsStore(5, 1, map[string]string{"zone": "z3"}) suite.cluster.AddLeaderRegion(1, 1, 3, 4) rule := &placement.Rule{ - GroupID: "pd", + GroupID: placement.DefaultGroupID, ID: "test", Index: 100, Override: true, @@ -1346,13 +1346,13 @@ func (suite *ruleCheckerTestSuite) TestFixDownWitnessPeer() { r = r.Clone(core.WithWitnesses([]*metapb.Peer{r.GetPeer(2)})) suite.ruleManager.SetRule(&placement.Rule{ - GroupID: "pd", - ID: "default", + GroupID: placement.DefaultGroupID, + ID: placement.DefaultRuleID, Role: placement.Voter, Count: 2, }) suite.ruleManager.SetRule(&placement.Rule{ - GroupID: "pd", + GroupID: placement.DefaultGroupID, ID: "r1", Role: placement.Voter, Count: 1, @@ -1379,13 +1379,13 @@ func (suite *ruleCheckerTestSuite) TestFixDownPeerWithAvailableWitness() { r = r.Clone(core.WithWitnesses([]*metapb.Peer{r.GetPeer(3)})) suite.ruleManager.SetRule(&placement.Rule{ - GroupID: "pd", - ID: "default", + GroupID: placement.DefaultGroupID, + ID: placement.DefaultRuleID, Role: placement.Voter, Count: 2, }) suite.ruleManager.SetRule(&placement.Rule{ - GroupID: "pd", + GroupID: placement.DefaultGroupID, ID: "r1", Role: placement.Voter, Count: 1, @@ -1417,13 +1417,13 @@ func (suite *ruleCheckerTestSuite) TestFixDownPeerWithAvailableWitness2() { r = r.Clone(core.WithWitnesses([]*metapb.Peer{r.GetPeer(3)})) suite.ruleManager.SetRule(&placement.Rule{ - GroupID: "pd", - ID: "default", + GroupID: placement.DefaultGroupID, + ID: placement.DefaultRuleID, Role: placement.Voter, Count: 2, }) suite.ruleManager.SetRule(&placement.Rule{ - GroupID: "pd", + GroupID: placement.DefaultGroupID, ID: "r1", Role: placement.Voter, Count: 1, @@ -1451,13 +1451,13 @@ func (suite *ruleCheckerTestSuite) TestFixDownPeerWithAvailableWitness3() { r = r.Clone(core.WithWitnesses([]*metapb.Peer{r.GetPeer(3)})) suite.ruleManager.SetRule(&placement.Rule{ - GroupID: "pd", - ID: "default", + GroupID: placement.DefaultGroupID, + ID: placement.DefaultRuleID, Role: placement.Voter, Count: 2, }) suite.ruleManager.SetRule(&placement.Rule{ - GroupID: "pd", + GroupID: placement.DefaultGroupID, ID: "r1", Role: placement.Voter, Count: 1, @@ -1508,7 +1508,7 @@ func (suite *ruleCheckerTestSuite) TestFixOfflinePeer() { suite.cluster.AddLabelsStore(5, 1, map[string]string{"zone": "z3"}) suite.cluster.AddLeaderRegion(1, 1, 3, 4) rule := &placement.Rule{ - GroupID: "pd", + GroupID: placement.DefaultGroupID, ID: "test", Index: 100, Override: true, @@ -1543,13 +1543,13 @@ func (suite *ruleCheckerTestSuite) TestFixOfflinePeerWithAvaliableWitness() { r := suite.cluster.GetRegion(1) r = r.Clone(core.WithWitnesses([]*metapb.Peer{r.GetPeer(2)})) suite.ruleManager.SetRule(&placement.Rule{ - GroupID: "pd", - ID: "default", + GroupID: placement.DefaultGroupID, + ID: placement.DefaultRuleID, Role: placement.Voter, Count: 2, }) suite.ruleManager.SetRule(&placement.Rule{ - GroupID: "pd", + GroupID: placement.DefaultGroupID, ID: "r1", Role: placement.Voter, Count: 1, @@ -1573,7 +1573,7 @@ func (suite *ruleCheckerTestSuite) TestRuleCache() { suite.cluster.AddRegionStore(999, 1) suite.cluster.AddLeaderRegion(1, 1, 3, 4) rule := &placement.Rule{ - GroupID: "pd", + GroupID: placement.DefaultGroupID, ID: "test", Index: 100, Override: true, @@ -1592,7 +1592,7 @@ func (suite *ruleCheckerTestSuite) TestRuleCache() { stillCached bool }{ { - name: "default", + name: placement.DefaultRuleID, region: region, stillCached: true, }, @@ -1718,7 +1718,7 @@ func (suite *ruleCheckerTestSuite) TestDemoteVoter() { suite.cluster.AddLabelsStore(4, 1, map[string]string{"zone": "z4"}) region := suite.cluster.AddLeaderRegion(1, 1, 4) rule := &placement.Rule{ - GroupID: "pd", + GroupID: placement.DefaultGroupID, ID: "test", Role: placement.Voter, Count: 1, @@ -1731,7 +1731,7 @@ func (suite *ruleCheckerTestSuite) TestDemoteVoter() { }, } rule2 := &placement.Rule{ - GroupID: "pd", + GroupID: placement.DefaultGroupID, ID: "test2", Role: placement.Learner, Count: 1, @@ -1745,7 +1745,7 @@ func (suite *ruleCheckerTestSuite) TestDemoteVoter() { } suite.ruleManager.SetRule(rule) suite.ruleManager.SetRule(rule2) - suite.ruleManager.DeleteRule("pd", "default") + suite.ruleManager.DeleteRule(placement.DefaultGroupID, placement.DefaultRuleID) op := suite.rc.Check(region) suite.NotNil(op) suite.Equal("fix-demote-voter", op.Desc()) @@ -1807,7 +1807,7 @@ func (suite *ruleCheckerTestSuite) TestLocationLabels() { suite.cluster.AddLabelsStore(6, 1, map[string]string{"zone": "z2", "rack": "r3", "host": "h2"}) suite.cluster.AddLeaderRegionWithRange(1, "", "", 1, 2, 5) rule1 := &placement.Rule{ - GroupID: "pd", + GroupID: placement.DefaultGroupID, ID: "test1", Role: placement.Leader, Count: 1, @@ -1821,7 +1821,7 @@ func (suite *ruleCheckerTestSuite) TestLocationLabels() { LocationLabels: []string{"rack"}, } rule2 := &placement.Rule{ - GroupID: "pd", + GroupID: placement.DefaultGroupID, ID: "test2", Role: placement.Voter, Count: 1, @@ -1835,7 +1835,7 @@ func (suite *ruleCheckerTestSuite) TestLocationLabels() { LocationLabels: []string{"rack"}, } rule3 := &placement.Rule{ - GroupID: "pd", + GroupID: placement.DefaultGroupID, ID: "test3", Role: placement.Voter, Count: 1, @@ -1851,7 +1851,7 @@ func (suite *ruleCheckerTestSuite) TestLocationLabels() { suite.ruleManager.SetRule(rule1) suite.ruleManager.SetRule(rule2) suite.ruleManager.SetRule(rule3) - suite.ruleManager.DeleteRule("pd", "default") + suite.ruleManager.DeleteRule(placement.DefaultGroupID, placement.DefaultRuleID) op := suite.rc.Check(suite.cluster.GetRegion(1)) suite.NotNil(op) suite.Equal("move-to-better-location", op.Desc()) @@ -1882,7 +1882,7 @@ func (suite *ruleCheckerTestSuite) TestTiFlashLocationLabels() { }, } suite.ruleManager.SetRule(rule1) - rule := suite.ruleManager.GetRule("pd", "default") + rule := suite.ruleManager.GetRule(placement.DefaultGroupID, placement.DefaultRuleID) rule.LocationLabels = []string{"zone", "rack", "host"} suite.ruleManager.SetRule(rule) op := suite.rc.Check(suite.cluster.GetRegion(1)) diff --git a/pkg/schedule/filter/filters_test.go b/pkg/schedule/filter/filters_test.go index fa085890694..f030dff81a4 100644 --- a/pkg/schedule/filter/filters_test.go +++ b/pkg/schedule/filter/filters_test.go @@ -159,7 +159,7 @@ func TestRuleFitFilterWithPlacementRule(t *testing.T) { testCluster := mockcluster.NewCluster(ctx, opt) testCluster.SetEnablePlacementRules(true) ruleManager := testCluster.RuleManager - ruleManager.DeleteRule("pd", "default") + ruleManager.DeleteRule(placement.DefaultGroupID, placement.DefaultRuleID) err := ruleManager.SetRules([]*placement.Rule{ { GroupID: "test", diff --git a/pkg/schedule/operator/create_operator_test.go b/pkg/schedule/operator/create_operator_test.go index 08a30680303..2fcd45d11f2 100644 --- a/pkg/schedule/operator/create_operator_test.go +++ b/pkg/schedule/operator/create_operator_test.go @@ -1145,8 +1145,8 @@ func TestCreateLeaveJointStateOperatorWithoutFitRules(t *testing.T) { cluster := mockcluster.NewCluster(ctx, opts) re.NoError(cluster.SetRules([]*placement.Rule{ { - GroupID: "pd", - ID: "default", + GroupID: placement.DefaultGroupID, + ID: placement.DefaultRuleID, StartKeyHex: hex.EncodeToString([]byte("")), EndKeyHex: hex.EncodeToString([]byte("")), Role: placement.Voter, diff --git a/pkg/schedule/placement/fit_region_test.go b/pkg/schedule/placement/fit_region_test.go index 0ec67b2a2aa..5bc62d9cc12 100644 --- a/pkg/schedule/placement/fit_region_test.go +++ b/pkg/schedule/placement/fit_region_test.go @@ -55,8 +55,8 @@ func (ms mockStoresSet) GetStore(id uint64) *core.StoreInfo { func addExtraRules(extraRules int) []*Rule { rules := make([]*Rule, 0) rules = append(rules, &Rule{ - GroupID: "pd", - ID: "default", + GroupID: DefaultGroupID, + ID: DefaultRuleID, Role: Voter, Count: 3, LocationLabels: []string{}, @@ -110,8 +110,8 @@ func BenchmarkFitRegion(b *testing.B) { region := mockRegion(3, 0) rules := []*Rule{ { - GroupID: "pd", - ID: "default", + GroupID: DefaultGroupID, + ID: DefaultRuleID, Role: Voter, Count: 3, LocationLabels: []string{}, @@ -129,8 +129,8 @@ func BenchmarkFitRegionMoreStores(b *testing.B) { region := mockRegion(3, 0) rules := []*Rule{ { - GroupID: "pd", - ID: "default", + GroupID: DefaultGroupID, + ID: DefaultRuleID, Role: Voter, Count: 3, LocationLabels: []string{}, @@ -148,8 +148,8 @@ func BenchmarkFitRegionMorePeers(b *testing.B) { region := mockRegion(5, 0) rules := []*Rule{ { - GroupID: "pd", - ID: "default", + GroupID: DefaultGroupID, + ID: DefaultRuleID, Role: Voter, Count: 5, LocationLabels: []string{}, @@ -167,14 +167,14 @@ func BenchmarkFitRegionMorePeersEquals(b *testing.B) { region := mockRegion(3, 0) rules := []*Rule{ { - GroupID: "pd", - ID: "default", + GroupID: DefaultGroupID, + ID: DefaultRuleID, Role: Leader, Count: 1, LocationLabels: []string{}, }, { - GroupID: "pd", + GroupID: DefaultGroupID, ID: "default-2", Role: Follower, Count: 4, @@ -193,8 +193,8 @@ func BenchmarkFitRegionMorePeersSplitRules(b *testing.B) { region := mockRegion(3, 0) rules := []*Rule{ { - GroupID: "pd", - ID: "default", + GroupID: DefaultGroupID, + ID: DefaultRuleID, Role: Leader, Count: 1, LocationLabels: []string{}, @@ -202,7 +202,7 @@ func BenchmarkFitRegionMorePeersSplitRules(b *testing.B) { } for i := 0; i < 4; i++ { rules = append(rules, &Rule{ - GroupID: "pd", + GroupID: DefaultGroupID, ID: fmt.Sprintf("%v", i), Role: Follower, Count: 1, @@ -221,8 +221,8 @@ func BenchmarkFitRegionMoreVotersSplitRules(b *testing.B) { region := mockRegion(5, 0) rules := []*Rule{ { - GroupID: "pd", - ID: "default", + GroupID: DefaultGroupID, + ID: DefaultRuleID, Role: Voter, Count: 1, LocationLabels: []string{}, @@ -230,7 +230,7 @@ func BenchmarkFitRegionMoreVotersSplitRules(b *testing.B) { } for i := 0; i < 4; i++ { rules = append(rules, &Rule{ - GroupID: "pd", + GroupID: DefaultGroupID, ID: fmt.Sprintf("%v", i), Role: Voter, Count: 1, @@ -260,7 +260,7 @@ func BenchmarkFitRegionCrossRegion(b *testing.B) { region := mockRegion(5, 0) rules := make([]*Rule, 0) rules = append(rules, &Rule{ - GroupID: "pd", + GroupID: DefaultGroupID, ID: "1", Role: Leader, Count: 1, @@ -268,7 +268,7 @@ func BenchmarkFitRegionCrossRegion(b *testing.B) { }) for i := 0; i < 2; i++ { rules = append(rules, &Rule{ - GroupID: "pd", + GroupID: DefaultGroupID, ID: fmt.Sprintf("%v", i), Role: Follower, Count: 1, @@ -289,7 +289,7 @@ func BenchmarkFitRegionWithMoreRulesAndStoreLabels(b *testing.B) { // create 100 rules, with each rule has 101 LabelConstraints. for i := 0; i < 100; i++ { rule := &Rule{ - GroupID: "pd", + GroupID: DefaultGroupID, ID: fmt.Sprintf("%v", i), Role: Follower, Count: 3, @@ -351,7 +351,7 @@ func BenchmarkFitRegionWithLocationLabels(b *testing.B) { region := mockRegion(5, 5) rules := []*Rule{} rule := &Rule{ - GroupID: "pd", + GroupID: DefaultGroupID, ID: "followers", Role: Follower, Count: 3, @@ -360,7 +360,7 @@ func BenchmarkFitRegionWithLocationLabels(b *testing.B) { } rules = append(rules, rule) rule = &Rule{ - GroupID: "pd", + GroupID: DefaultGroupID, ID: "learner", Role: Learner, Count: 3, @@ -369,7 +369,7 @@ func BenchmarkFitRegionWithLocationLabels(b *testing.B) { } rules = append(rules, rule) rule = &Rule{ - GroupID: "pd", + GroupID: DefaultGroupID, ID: "voters", Role: Voter, Count: 4, diff --git a/pkg/schedule/placement/region_rule_cache_test.go b/pkg/schedule/placement/region_rule_cache_test.go index b4164e85530..835203bed26 100644 --- a/pkg/schedule/placement/region_rule_cache_test.go +++ b/pkg/schedule/placement/region_rule_cache_test.go @@ -99,8 +99,8 @@ func TestRegionRuleFitCache(t *testing.T) { region: mockRegion(3, 0), rules: []*Rule{ { - GroupID: "pd", - ID: "default", + GroupID: DefaultGroupID, + ID: DefaultRuleID, Role: Voter, Count: 4, Version: 1, @@ -114,8 +114,8 @@ func TestRegionRuleFitCache(t *testing.T) { region: mockRegion(3, 0), rules: []*Rule{ { - GroupID: "pd", - ID: "default", + GroupID: DefaultGroupID, + ID: DefaultRuleID, Role: Voter, Count: 3, CreateTimestamp: 1, @@ -141,7 +141,7 @@ func TestRegionRuleFitCache(t *testing.T) { region: mockRegion(3, 0), rules: []*Rule{ { - GroupID: "pd", + GroupID: DefaultGroupID, ID: "default-2", Role: Voter, Count: 3, @@ -155,7 +155,7 @@ func TestRegionRuleFitCache(t *testing.T) { region: nil, rules: []*Rule{ { - GroupID: "pd", + GroupID: DefaultGroupID, ID: "default-2", Role: Voter, Count: 3, diff --git a/pkg/schedule/placement/rule_manager.go b/pkg/schedule/placement/rule_manager.go index a7e169b74aa..e25b8802b45 100644 --- a/pkg/schedule/placement/rule_manager.go +++ b/pkg/schedule/placement/rule_manager.go @@ -37,6 +37,15 @@ import ( "golang.org/x/exp/slices" ) +const ( + // DefaultGroupID is the default rule group ID. + DefaultGroupID = "pd" + // DefaultRuleID is the default rule ID. + DefaultRuleID = "default" + // defaultWitnessRuleID is the default witness rule ID. + defaultWitnessRuleID = "witness" +) + // RuleManager is responsible for the lifecycle of all placement Rules. // It is thread safe. type RuleManager struct { @@ -88,16 +97,16 @@ func (m *RuleManager) Initialize(maxReplica int, locationLabels []string, isolat defaultRules = append(defaultRules, []*Rule{ { - GroupID: "pd", - ID: "default", + GroupID: DefaultGroupID, + ID: DefaultRuleID, Role: Voter, Count: maxReplica - witnessCount, LocationLabels: locationLabels, IsolationLevel: isolationLevel, }, { - GroupID: "pd", - ID: "witness", + GroupID: DefaultGroupID, + ID: defaultWitnessRuleID, Role: Voter, Count: witnessCount, IsWitness: true, @@ -108,8 +117,8 @@ func (m *RuleManager) Initialize(maxReplica int, locationLabels []string, isolat ) } else { defaultRules = append(defaultRules, &Rule{ - GroupID: "pd", - ID: "default", + GroupID: DefaultGroupID, + ID: DefaultRuleID, Role: Voter, Count: maxReplica, LocationLabels: locationLabels, diff --git a/pkg/schedule/placement/rule_manager_test.go b/pkg/schedule/placement/rule_manager_test.go index dad50a2d881..68a18b538d4 100644 --- a/pkg/schedule/placement/rule_manager_test.go +++ b/pkg/schedule/placement/rule_manager_test.go @@ -44,8 +44,8 @@ func TestDefault(t *testing.T) { _, manager := newTestManager(t, false) rules := manager.GetAllRules() re.Len(rules, 1) - re.Equal("pd", rules[0].GroupID) - re.Equal("default", rules[0].ID) + re.Equal(DefaultGroupID, rules[0].GroupID) + re.Equal(DefaultRuleID, rules[0].ID) re.Equal(0, rules[0].Index) re.Empty(rules[0].StartKey) re.Empty(rules[0].EndKey) @@ -58,15 +58,15 @@ func TestDefault2(t *testing.T) { _, manager := newTestManager(t, true) rules := manager.GetAllRules() re.Len(rules, 2) - re.Equal("pd", rules[0].GroupID) - re.Equal("default", rules[0].ID) + re.Equal(DefaultGroupID, rules[0].GroupID) + re.Equal(DefaultRuleID, rules[0].ID) re.Equal(0, rules[0].Index) re.Empty(rules[0].StartKey) re.Empty(rules[0].EndKey) re.Equal(Voter, rules[0].Role) re.Equal([]string{"zone", "rack", "host"}, rules[0].LocationLabels) - re.Equal("pd", rules[1].GroupID) - re.Equal("witness", rules[1].ID) + re.Equal(DefaultGroupID, rules[1].GroupID) + re.Equal(defaultWitnessRuleID, rules[1].ID) re.Equal(0, rules[1].Index) re.Empty(rules[1].StartKey) re.Empty(rules[1].EndKey) @@ -79,16 +79,16 @@ func TestAdjustRule(t *testing.T) { re := require.New(t) _, manager := newTestManager(t, false) rules := []Rule{ - {GroupID: "group", ID: "id", StartKeyHex: "123abc", EndKeyHex: "123abf", Role: "voter", Count: 3}, - {GroupID: "", ID: "id", StartKeyHex: "123abc", EndKeyHex: "123abf", Role: "voter", Count: 3}, - {GroupID: "group", ID: "", StartKeyHex: "123abc", EndKeyHex: "123abf", Role: "voter", Count: 3}, - {GroupID: "group", ID: "id", StartKeyHex: "123ab", EndKeyHex: "123abf", Role: "voter", Count: 3}, - {GroupID: "group", ID: "id", StartKeyHex: "123abc", EndKeyHex: "1123abf", Role: "voter", Count: 3}, - {GroupID: "group", ID: "id", StartKeyHex: "123abc", EndKeyHex: "123aaa", Role: "voter", Count: 3}, + {GroupID: "group", ID: "id", StartKeyHex: "123abc", EndKeyHex: "123abf", Role: Voter, Count: 3}, + {GroupID: "", ID: "id", StartKeyHex: "123abc", EndKeyHex: "123abf", Role: Voter, Count: 3}, + {GroupID: "group", ID: "", StartKeyHex: "123abc", EndKeyHex: "123abf", Role: Voter, Count: 3}, + {GroupID: "group", ID: "id", StartKeyHex: "123ab", EndKeyHex: "123abf", Role: Voter, Count: 3}, + {GroupID: "group", ID: "id", StartKeyHex: "123abc", EndKeyHex: "1123abf", Role: Voter, Count: 3}, + {GroupID: "group", ID: "id", StartKeyHex: "123abc", EndKeyHex: "123aaa", Role: Voter, Count: 3}, {GroupID: "group", ID: "id", StartKeyHex: "123abc", EndKeyHex: "123abf", Role: "master", Count: 3}, - {GroupID: "group", ID: "id", StartKeyHex: "123abc", EndKeyHex: "123abf", Role: "voter", Count: 0}, - {GroupID: "group", ID: "id", StartKeyHex: "123abc", EndKeyHex: "123abf", Role: "voter", Count: -1}, - {GroupID: "group", ID: "id", StartKeyHex: "123abc", EndKeyHex: "123abf", Role: "voter", Count: 3, LabelConstraints: []LabelConstraint{{Op: "foo"}}}, + {GroupID: "group", ID: "id", StartKeyHex: "123abc", EndKeyHex: "123abf", Role: Voter, Count: 0}, + {GroupID: "group", ID: "id", StartKeyHex: "123abc", EndKeyHex: "123abf", Role: Voter, Count: -1}, + {GroupID: "group", ID: "id", StartKeyHex: "123abc", EndKeyHex: "123abf", Role: Voter, Count: 3, LabelConstraints: []LabelConstraint{{Op: "foo"}}}, } re.NoError(manager.adjustRule(&rules[0], "group")) @@ -101,17 +101,17 @@ func TestAdjustRule(t *testing.T) { } manager.SetKeyType(constant.Table.String()) - re.Error(manager.adjustRule(&Rule{GroupID: "group", ID: "id", StartKeyHex: "123abc", EndKeyHex: "123abf", Role: "voter", Count: 3}, "group")) + re.Error(manager.adjustRule(&Rule{GroupID: "group", ID: "id", StartKeyHex: "123abc", EndKeyHex: "123abf", Role: Voter, Count: 3}, "group")) manager.SetKeyType(constant.Txn.String()) - re.Error(manager.adjustRule(&Rule{GroupID: "group", ID: "id", StartKeyHex: "123abc", EndKeyHex: "123abf", Role: "voter", Count: 3}, "group")) + re.Error(manager.adjustRule(&Rule{GroupID: "group", ID: "id", StartKeyHex: "123abc", EndKeyHex: "123abf", Role: Voter, Count: 3}, "group")) re.Error(manager.adjustRule(&Rule{ GroupID: "group", ID: "id", StartKeyHex: hex.EncodeToString(codec.EncodeBytes([]byte{0})), EndKeyHex: "123abf", - Role: "voter", + Role: Voter, Count: 3, }, "group")) @@ -120,7 +120,7 @@ func TestAdjustRule(t *testing.T) { ID: "id", StartKeyHex: hex.EncodeToString(codec.EncodeBytes([]byte{0})), EndKeyHex: hex.EncodeToString(codec.EncodeBytes([]byte{1})), - Role: "learner", + Role: Learner, Count: 1, IsWitness: true, LabelConstraints: []LabelConstraint{{Key: "engine", Op: "in", Values: []string{"tiflash"}}}, @@ -130,15 +130,15 @@ func TestAdjustRule(t *testing.T) { func TestLeaderCheck(t *testing.T) { re := require.New(t) _, manager := newTestManager(t, false) - re.Regexp(".*needs at least one leader or voter.*", manager.SetRule(&Rule{GroupID: "pd", ID: "default", Role: "learner", Count: 3}).Error()) - re.Regexp(".*define multiple leaders by count 2.*", manager.SetRule(&Rule{GroupID: "g2", ID: "33", Role: "leader", Count: 2}).Error()) + re.Regexp(".*needs at least one leader or voter.*", manager.SetRule(&Rule{GroupID: DefaultGroupID, ID: DefaultRuleID, Role: Learner, Count: 3}).Error()) + re.Regexp(".*define multiple leaders by count 2.*", manager.SetRule(&Rule{GroupID: "g2", ID: "33", Role: Leader, Count: 2}).Error()) re.Regexp(".*multiple leader replicas.*", manager.Batch([]RuleOp{ { - Rule: &Rule{GroupID: "g2", ID: "foo1", Role: "leader", Count: 1}, + Rule: &Rule{GroupID: "g2", ID: "foo1", Role: Leader, Count: 1}, Action: RuleOpAdd, }, { - Rule: &Rule{GroupID: "g2", ID: "foo2", Role: "leader", Count: 1}, + Rule: &Rule{GroupID: "g2", ID: "foo2", Role: Leader, Count: 1}, Action: RuleOpAdd, }, }).Error()) @@ -148,9 +148,9 @@ func TestSaveLoad(t *testing.T) { re := require.New(t) store, manager := newTestManager(t, false) rules := []*Rule{ - {GroupID: "pd", ID: "default", Role: "voter", Count: 5}, - {GroupID: "foo", ID: "baz", StartKeyHex: "", EndKeyHex: "abcd", Role: "voter", Count: 1}, - {GroupID: "foo", ID: "bar", Role: "learner", Count: 1}, + {GroupID: DefaultGroupID, ID: DefaultRuleID, Role: Voter, Count: 5}, + {GroupID: "foo", ID: "baz", StartKeyHex: "", EndKeyHex: "abcd", Role: Voter, Count: 1}, + {GroupID: "foo", ID: "bar", Role: Learner, Count: 1}, } for _, r := range rules { re.NoError(manager.SetRule(r.Clone())) @@ -160,7 +160,7 @@ func TestSaveLoad(t *testing.T) { err := m2.Initialize(3, []string{"no", "labels"}, "") re.NoError(err) re.Len(m2.GetAllRules(), 3) - re.Equal(rules[0].String(), m2.GetRule("pd", "default").String()) + re.Equal(rules[0].String(), m2.GetRule(DefaultGroupID, DefaultRuleID).String()) re.Equal(rules[1].String(), m2.GetRule("foo", "baz").String()) re.Equal(rules[2].String(), m2.GetRule("foo", "bar").String()) re.Equal(manager.GetRulesCount(), 3) @@ -170,14 +170,14 @@ func TestSaveLoad(t *testing.T) { func TestSetAfterGet(t *testing.T) { re := require.New(t) store, manager := newTestManager(t, false) - rule := manager.GetRule("pd", "default") + rule := manager.GetRule(DefaultGroupID, DefaultRuleID) rule.Count = 1 manager.SetRule(rule) m2 := NewRuleManager(store, nil, nil) err := m2.Initialize(100, []string{}, "") re.NoError(err) - rule = m2.GetRule("pd", "default") + rule = m2.GetRule(DefaultGroupID, DefaultRuleID) re.Equal(1, rule.Count) } @@ -193,9 +193,9 @@ func TestKeys(t *testing.T) { re := require.New(t) _, manager := newTestManager(t, false) rules := []*Rule{ - {GroupID: "1", ID: "1", Role: "voter", Count: 1, StartKeyHex: "", EndKeyHex: ""}, - {GroupID: "2", ID: "2", Role: "voter", Count: 1, StartKeyHex: "11", EndKeyHex: "ff"}, - {GroupID: "2", ID: "3", Role: "voter", Count: 1, StartKeyHex: "22", EndKeyHex: "dd"}, + {GroupID: "1", ID: "1", Role: Voter, Count: 1, StartKeyHex: "", EndKeyHex: ""}, + {GroupID: "2", ID: "2", Role: Voter, Count: 1, StartKeyHex: "11", EndKeyHex: "ff"}, + {GroupID: "2", ID: "3", Role: Voter, Count: 1, StartKeyHex: "22", EndKeyHex: "dd"}, } toDelete := []RuleOp{} @@ -207,16 +207,16 @@ func TestKeys(t *testing.T) { DeleteByIDPrefix: false, }) } - checkRules(t, manager.GetAllRules(), [][2]string{{"1", "1"}, {"2", "2"}, {"2", "3"}, {"pd", "default"}}) + checkRules(t, manager.GetAllRules(), [][2]string{{"1", "1"}, {"2", "2"}, {"2", "3"}, {DefaultGroupID, DefaultRuleID}}) manager.Batch(toDelete) - checkRules(t, manager.GetAllRules(), [][2]string{{"pd", "default"}}) + checkRules(t, manager.GetAllRules(), [][2]string{{DefaultGroupID, DefaultRuleID}}) - rules = append(rules, &Rule{GroupID: "3", ID: "4", Role: "voter", Count: 1, StartKeyHex: "44", EndKeyHex: "ee"}, - &Rule{GroupID: "3", ID: "5", Role: "voter", Count: 1, StartKeyHex: "44", EndKeyHex: "dd"}) + rules = append(rules, &Rule{GroupID: "3", ID: "4", Role: Voter, Count: 1, StartKeyHex: "44", EndKeyHex: "ee"}, + &Rule{GroupID: "3", ID: "5", Role: Voter, Count: 1, StartKeyHex: "44", EndKeyHex: "dd"}) manager.SetRules(rules) - checkRules(t, manager.GetAllRules(), [][2]string{{"1", "1"}, {"2", "2"}, {"2", "3"}, {"3", "4"}, {"3", "5"}, {"pd", "default"}}) + checkRules(t, manager.GetAllRules(), [][2]string{{"1", "1"}, {"2", "2"}, {"2", "3"}, {"3", "4"}, {"3", "5"}, {DefaultGroupID, DefaultRuleID}}) - manager.DeleteRule("pd", "default") + manager.DeleteRule(DefaultGroupID, DefaultRuleID) checkRules(t, manager.GetAllRules(), [][2]string{{"1", "1"}, {"2", "2"}, {"2", "3"}, {"3", "4"}, {"3", "5"}}) splitKeys := [][]string{ @@ -282,12 +282,12 @@ func TestKeys(t *testing.T) { func TestDeleteByIDPrefix(t *testing.T) { _, manager := newTestManager(t, false) manager.SetRules([]*Rule{ - {GroupID: "g1", ID: "foo1", Role: "voter", Count: 1}, - {GroupID: "g2", ID: "foo1", Role: "voter", Count: 1}, - {GroupID: "g2", ID: "foobar", Role: "voter", Count: 1}, - {GroupID: "g2", ID: "baz2", Role: "voter", Count: 1}, + {GroupID: "g1", ID: "foo1", Role: Voter, Count: 1}, + {GroupID: "g2", ID: "foo1", Role: Voter, Count: 1}, + {GroupID: "g2", ID: "foobar", Role: Voter, Count: 1}, + {GroupID: "g2", ID: "baz2", Role: Voter, Count: 1}, }) - manager.DeleteRule("pd", "default") + manager.DeleteRule(DefaultGroupID, DefaultRuleID) checkRules(t, manager.GetAllRules(), [][2]string{{"g1", "foo1"}, {"g2", "baz2"}, {"g2", "foo1"}, {"g2", "foobar"}}) manager.Batch([]RuleOp{{ @@ -301,40 +301,40 @@ func TestDeleteByIDPrefix(t *testing.T) { func TestRangeGap(t *testing.T) { re := require.New(t) _, manager := newTestManager(t, false) - err := manager.DeleteRule("pd", "default") + err := manager.DeleteRule(DefaultGroupID, DefaultRuleID) re.Error(err) - err = manager.SetRule(&Rule{GroupID: "pd", ID: "foo", StartKeyHex: "", EndKeyHex: "abcd", Role: "voter", Count: 1}) + err = manager.SetRule(&Rule{GroupID: DefaultGroupID, ID: "foo", StartKeyHex: "", EndKeyHex: "abcd", Role: Voter, Count: 1}) re.NoError(err) // |-- default --| // |-- foo --| // still cannot delete default since it will cause ("abcd", "") has no rules inside. - err = manager.DeleteRule("pd", "default") + err = manager.DeleteRule(DefaultGroupID, DefaultRuleID) re.Error(err) - err = manager.SetRule(&Rule{GroupID: "pd", ID: "bar", StartKeyHex: "abcd", EndKeyHex: "", Role: "voter", Count: 1}) + err = manager.SetRule(&Rule{GroupID: DefaultGroupID, ID: "bar", StartKeyHex: "abcd", EndKeyHex: "", Role: Voter, Count: 1}) re.NoError(err) // now default can be deleted. - err = manager.DeleteRule("pd", "default") + err = manager.DeleteRule(DefaultGroupID, DefaultRuleID) re.NoError(err) // cannot change range since it will cause ("abaa", "abcd") has no rules inside. - err = manager.SetRule(&Rule{GroupID: "pd", ID: "foo", StartKeyHex: "", EndKeyHex: "abaa", Role: "voter", Count: 1}) + err = manager.SetRule(&Rule{GroupID: DefaultGroupID, ID: "foo", StartKeyHex: "", EndKeyHex: "abaa", Role: Voter, Count: 1}) re.Error(err) } func TestGroupConfig(t *testing.T) { re := require.New(t) _, manager := newTestManager(t, false) - pd1 := &RuleGroup{ID: "pd"} - re.Equal(pd1, manager.GetRuleGroup("pd")) + pd1 := &RuleGroup{ID: DefaultGroupID} + re.Equal(pd1, manager.GetRuleGroup(DefaultGroupID)) // update group pd - pd2 := &RuleGroup{ID: "pd", Index: 100, Override: true} + pd2 := &RuleGroup{ID: DefaultGroupID, Index: 100, Override: true} err := manager.SetRuleGroup(pd2) re.NoError(err) - re.Equal(pd2, manager.GetRuleGroup("pd")) + re.Equal(pd2, manager.GetRuleGroup(DefaultGroupID)) // new group g without config - err = manager.SetRule(&Rule{GroupID: "g", ID: "1", Role: "voter", Count: 1}) + err = manager.SetRule(&Rule{GroupID: "g", ID: "1", Role: Voter, Count: 1}) re.NoError(err) g1 := &RuleGroup{ID: "g"} re.Equal(g1, manager.GetRuleGroup("g")) @@ -347,12 +347,12 @@ func TestGroupConfig(t *testing.T) { re.Equal([]*RuleGroup{g2, pd2}, manager.GetRuleGroups()) // delete pd group, restore to default config - err = manager.DeleteRuleGroup("pd") + err = manager.DeleteRuleGroup(DefaultGroupID) re.NoError(err) re.Equal([]*RuleGroup{pd1, g2}, manager.GetRuleGroups()) // delete rule, the group is removed too - err = manager.DeleteRule("pd", "default") + err = manager.DeleteRule(DefaultGroupID, DefaultRuleID) re.NoError(err) re.Equal([]*RuleGroup{g2}, manager.GetRuleGroups()) } @@ -360,16 +360,16 @@ func TestGroupConfig(t *testing.T) { func TestRuleVersion(t *testing.T) { re := require.New(t) _, manager := newTestManager(t, false) - rule1 := manager.GetRule("pd", "default") + rule1 := manager.GetRule(DefaultGroupID, DefaultRuleID) re.Equal(uint64(0), rule1.Version) // create new rule - newRule := &Rule{GroupID: "g1", ID: "id", StartKeyHex: "123abc", EndKeyHex: "123abf", Role: "voter", Count: 3} + newRule := &Rule{GroupID: "g1", ID: "id", StartKeyHex: "123abc", EndKeyHex: "123abf", Role: Voter, Count: 3} err := manager.SetRule(newRule) re.NoError(err) newRule = manager.GetRule("g1", "id") re.Equal(uint64(0), newRule.Version) // update rule - newRule = &Rule{GroupID: "g1", ID: "id", StartKeyHex: "123abc", EndKeyHex: "123abf", Role: "voter", Count: 2} + newRule = &Rule{GroupID: "g1", ID: "id", StartKeyHex: "123abc", EndKeyHex: "123abf", Role: Voter, Count: 2} err = manager.SetRule(newRule) re.NoError(err) newRule = manager.GetRule("g1", "id") diff --git a/pkg/schedule/placement/rule_test.go b/pkg/schedule/placement/rule_test.go index b91a1f22d65..75d7bab23c9 100644 --- a/pkg/schedule/placement/rule_test.go +++ b/pkg/schedule/placement/rule_test.go @@ -110,9 +110,9 @@ func TestGroupProperties(t *testing.T) { func TestBuildRuleList(t *testing.T) { re := require.New(t) defaultRule := &Rule{ - GroupID: "pd", - ID: "default", - Role: "voter", + GroupID: DefaultGroupID, + ID: DefaultRuleID, + Role: Voter, StartKey: []byte{}, EndKey: []byte{}, Count: 3, @@ -122,13 +122,13 @@ func TestBuildRuleList(t *testing.T) { byteEnd, err := hex.DecodeString("a2") re.NoError(err) ruleMeta := &Rule{ - GroupID: "pd", + GroupID: DefaultGroupID, ID: "meta", Index: 1, Override: true, StartKey: byteStart, EndKey: byteEnd, - Role: "voter", + Role: Voter, Count: 5, } @@ -140,7 +140,7 @@ func TestBuildRuleList(t *testing.T) { { name: "default rule", rules: map[[2]string]*Rule{ - {"pd", "default"}: defaultRule, + {DefaultGroupID, DefaultRuleID}: defaultRule, }, expect: ruleList{ ranges: []rangeRules{ @@ -155,8 +155,8 @@ func TestBuildRuleList(t *testing.T) { { name: "metadata case", rules: map[[2]string]*Rule{ - {"pd", "default"}: defaultRule, - {"pd", "meta"}: ruleMeta, + {DefaultGroupID, DefaultRuleID}: defaultRule, + {DefaultGroupID, "meta"}: ruleMeta, }, expect: ruleList{ranges: []rangeRules{ { diff --git a/pkg/schedule/scatter/region_scatterer_test.go b/pkg/schedule/scatter/region_scatterer_test.go index 681b863aea6..70517d23fee 100644 --- a/pkg/schedule/scatter/region_scatterer_test.go +++ b/pkg/schedule/scatter/region_scatterer_test.go @@ -185,7 +185,7 @@ func scatterSpecial(re *require.Assertions, numOrdinaryStores, numSpecialStores, } tc.SetEnablePlacementRules(true) re.NoError(tc.RuleManager.SetRule(&placement.Rule{ - GroupID: "pd", ID: "learner", Role: placement.Learner, Count: 3, + GroupID: placement.DefaultGroupID, ID: "learner", Role: placement.Learner, Count: 3, LabelConstraints: []placement.LabelConstraint{{Key: "engine", Op: placement.In, Values: []string{"tiflash"}}}})) // Region 1 has the same distribution with the Region 2, which is used to test selectPeerToReplace. @@ -575,8 +575,8 @@ func TestRegionHasLearner(t *testing.T) { tc.AddLabelsStore(i, 0, map[string]string{"zone": "z2"}) } tc.RuleManager.SetRule(&placement.Rule{ - GroupID: "pd", - ID: "default", + GroupID: placement.DefaultGroupID, + ID: placement.DefaultRuleID, Role: placement.Voter, Count: 3, LabelConstraints: []placement.LabelConstraint{ @@ -588,7 +588,7 @@ func TestRegionHasLearner(t *testing.T) { }, }) tc.RuleManager.SetRule(&placement.Rule{ - GroupID: "pd", + GroupID: placement.DefaultGroupID, ID: "learner", Role: placement.Learner, Count: 1, diff --git a/pkg/schedule/schedulers/balance_witness_test.go b/pkg/schedule/schedulers/balance_witness_test.go index abd4a3b3bba..59bf04c2303 100644 --- a/pkg/schedule/schedulers/balance_witness_test.go +++ b/pkg/schedule/schedulers/balance_witness_test.go @@ -43,8 +43,8 @@ func (suite *balanceWitnessSchedulerTestSuite) SetupTest() { suite.cancel, suite.conf, suite.tc, suite.oc = prepareSchedulersTest() suite.tc.RuleManager.SetRules([]*placement.Rule{ { - GroupID: "pd", - ID: "default", + GroupID: placement.DefaultGroupID, + ID: placement.DefaultRuleID, Role: placement.Voter, Count: 4, }, diff --git a/pkg/schedule/schedulers/hot_region_test.go b/pkg/schedule/schedulers/hot_region_test.go index d8f9bbc532c..15c037ddd22 100644 --- a/pkg/schedule/schedulers/hot_region_test.go +++ b/pkg/schedule/schedulers/hot_region_test.go @@ -582,8 +582,8 @@ func TestHotWriteRegionScheduleByteRateOnlyWithTiFlash(t *testing.T) { tc.SetHotRegionCacheHitsThreshold(0) re.NoError(tc.RuleManager.SetRules([]*placement.Rule{ { - GroupID: "pd", - ID: "default", + GroupID: placement.DefaultGroupID, + ID: placement.DefaultRuleID, Role: placement.Voter, Count: 3, LocationLabels: []string{"zone", "host"}, @@ -1143,7 +1143,7 @@ func TestHotWriteRegionScheduleWithRuleEnabled(t *testing.T) { tc.AddRegionStore(3, 20) err = tc.SetRule(&placement.Rule{ - GroupID: "pd", + GroupID: placement.DefaultGroupID, ID: "leader", Index: 1, Override: true, @@ -1161,7 +1161,7 @@ func TestHotWriteRegionScheduleWithRuleEnabled(t *testing.T) { }) re.NoError(err) err = tc.SetRule(&placement.Rule{ - GroupID: "pd", + GroupID: placement.DefaultGroupID, ID: "voter", Index: 2, Override: false, diff --git a/pkg/schedule/schedulers/scheduler_test.go b/pkg/schedule/schedulers/scheduler_test.go index 12ab9f8aa2f..57f1fcf1e3f 100644 --- a/pkg/schedule/schedulers/scheduler_test.go +++ b/pkg/schedule/schedulers/scheduler_test.go @@ -261,13 +261,13 @@ func TestShuffleRegionRole(t *testing.T) { // update rule to 1leader+1follower+1learner tc.SetEnablePlacementRules(true) tc.RuleManager.SetRule(&placement.Rule{ - GroupID: "pd", - ID: "default", + GroupID: placement.DefaultGroupID, + ID: placement.DefaultRuleID, Role: placement.Voter, Count: 2, }) tc.RuleManager.SetRule(&placement.Rule{ - GroupID: "pd", + GroupID: placement.DefaultGroupID, ID: "learner", Role: placement.Learner, Count: 1, @@ -428,8 +428,8 @@ func TestBalanceLeaderWithConflictRule(t *testing.T) { { name: "default Rule", rule: &placement.Rule{ - GroupID: "pd", - ID: "default", + GroupID: placement.DefaultGroupID, + ID: placement.DefaultRuleID, Index: 1, StartKey: []byte(""), EndKey: []byte(""), @@ -442,8 +442,8 @@ func TestBalanceLeaderWithConflictRule(t *testing.T) { { name: "single store allowed to be placed leader", rule: &placement.Rule{ - GroupID: "pd", - ID: "default", + GroupID: placement.DefaultGroupID, + ID: placement.DefaultRuleID, Index: 1, StartKey: []byte(""), EndKey: []byte(""), @@ -463,8 +463,8 @@ func TestBalanceLeaderWithConflictRule(t *testing.T) { { name: "2 store allowed to be placed leader", rule: &placement.Rule{ - GroupID: "pd", - ID: "default", + GroupID: placement.DefaultGroupID, + ID: placement.DefaultRuleID, Index: 1, StartKey: []byte(""), EndKey: []byte(""), diff --git a/server/api/cluster_test.go b/server/api/cluster_test.go index 01aa6ba5f24..d6d8effa365 100644 --- a/server/api/cluster_test.go +++ b/server/api/cluster_test.go @@ -22,6 +22,7 @@ import ( "github.com/pingcap/kvproto/pkg/metapb" "github.com/stretchr/testify/suite" sc "github.com/tikv/pd/pkg/schedule/config" + "github.com/tikv/pd/pkg/schedule/placement" tu "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/server" "github.com/tikv/pd/server/cluster" @@ -57,7 +58,7 @@ func (suite *clusterTestSuite) TestCluster() { suite.svr.GetPersistOptions().SetPlacementRuleEnabled(true) suite.svr.GetPersistOptions().GetReplicationConfig().LocationLabels = []string{"host"} rm := suite.svr.GetRaftCluster().GetRuleManager() - rule := rm.GetRule("pd", "default") + rule := rm.GetRule(placement.DefaultGroupID, placement.DefaultRuleID) rule.LocationLabels = []string{"host"} rule.Count = 1 rm.SetRule(rule) @@ -81,7 +82,7 @@ func (suite *clusterTestSuite) TestCluster() { c1.MaxPeerCount = 6 suite.Equal(c2, c1) - suite.Equal(int(r.MaxReplicas), suite.svr.GetRaftCluster().GetRuleManager().GetRule("pd", "default").Count) + suite.Equal(int(r.MaxReplicas), suite.svr.GetRaftCluster().GetRuleManager().GetRule(placement.DefaultGroupID, placement.DefaultRuleID).Count) } func (suite *clusterTestSuite) testGetClusterStatus() { diff --git a/server/api/region_test.go b/server/api/region_test.go index 379fcf7d463..0f8f84bfc37 100644 --- a/server/api/region_test.go +++ b/server/api/region_test.go @@ -697,7 +697,7 @@ func (suite *regionsReplicatedTestSuite) TestCheckRegionsReplicated() { Index: 5, Rules: []*placement.Rule{ { - ID: "foo", Index: 1, Role: "voter", Count: 1, + ID: "foo", Index: 1, Role: placement.Voter, Count: 1, }, }, }, @@ -738,7 +738,7 @@ func (suite *regionsReplicatedTestSuite) TestCheckRegionsReplicated() { mustRegionHeartbeat(re, suite.svr, r1) bundle[0].Rules = append(bundle[0].Rules, &placement.Rule{ - ID: "bar", Index: 1, Role: "voter", Count: 1, + ID: "bar", Index: 1, Role: placement.Voter, Count: 1, }) data, err = json.Marshal(bundle) suite.NoError(err) @@ -755,7 +755,7 @@ func (suite *regionsReplicatedTestSuite) TestCheckRegionsReplicated() { Index: 6, Rules: []*placement.Rule{ { - ID: "foo", Index: 1, Role: "voter", Count: 2, + ID: "foo", Index: 1, Role: placement.Voter, Count: 2, }, }, }) diff --git a/server/api/rule.go b/server/api/rule.go index 47964d594be..bdb3db2016d 100644 --- a/server/api/rule.go +++ b/server/api/rule.go @@ -273,7 +273,7 @@ func (h *ruleHandler) SetRule(w http.ResponseWriter, r *http.Request) { // sync replicate config with default-rule func (h *ruleHandler) syncReplicateConfigWithDefaultRule(rule *placement.Rule) error { // sync default rule with replicate config - if rule.GroupID == "pd" && rule.ID == "default" { + if rule.GroupID == placement.DefaultGroupID && rule.ID == placement.DefaultRuleID { cfg := h.svr.GetReplicationConfig().Clone() cfg.MaxReplicas = uint64(rule.Count) if err := h.svr.SetReplicationConfig(*cfg); err != nil { diff --git a/server/cluster/cluster_test.go b/server/cluster/cluster_test.go index 4b9b401e0c9..d424ea98e7b 100644 --- a/server/cluster/cluster_test.go +++ b/server/cluster/cluster_test.go @@ -1669,7 +1669,7 @@ func TestCalculateStoreSize1(t *testing.T) { } cluster.ruleManager.SetRule( - &placement.Rule{GroupID: "pd", ID: "zone1", StartKey: []byte(""), EndKey: []byte(""), Role: "voter", Count: 2, + &placement.Rule{GroupID: placement.DefaultGroupID, ID: "zone1", StartKey: []byte(""), EndKey: []byte(""), Role: placement.Voter, Count: 2, LabelConstraints: []placement.LabelConstraint{ {Key: "zone", Op: "in", Values: []string{"zone1"}}, }, @@ -1677,7 +1677,7 @@ func TestCalculateStoreSize1(t *testing.T) { ) cluster.ruleManager.SetRule( - &placement.Rule{GroupID: "pd", ID: "zone2", StartKey: []byte(""), EndKey: []byte(""), Role: "voter", Count: 2, + &placement.Rule{GroupID: placement.DefaultGroupID, ID: "zone2", StartKey: []byte(""), EndKey: []byte(""), Role: placement.Voter, Count: 2, LabelConstraints: []placement.LabelConstraint{ {Key: "zone", Op: "in", Values: []string{"zone2"}}, }, @@ -1685,13 +1685,13 @@ func TestCalculateStoreSize1(t *testing.T) { ) cluster.ruleManager.SetRule( - &placement.Rule{GroupID: "pd", ID: "zone3", StartKey: []byte(""), EndKey: []byte(""), Role: "follower", Count: 1, + &placement.Rule{GroupID: placement.DefaultGroupID, ID: "zone3", StartKey: []byte(""), EndKey: []byte(""), Role: placement.Follower, Count: 1, LabelConstraints: []placement.LabelConstraint{ {Key: "zone", Op: "in", Values: []string{"zone3"}}, }, LocationLabels: []string{"rack", "host"}}, ) - cluster.ruleManager.DeleteRule("pd", "default") + cluster.ruleManager.DeleteRule(placement.DefaultGroupID, placement.DefaultRuleID) regions := newTestRegions(100, 10, 5) for _, region := range regions { @@ -1753,7 +1753,7 @@ func TestCalculateStoreSize2(t *testing.T) { } cluster.ruleManager.SetRule( - &placement.Rule{GroupID: "pd", ID: "dc1", StartKey: []byte(""), EndKey: []byte(""), Role: "voter", Count: 2, + &placement.Rule{GroupID: placement.DefaultGroupID, ID: "dc1", StartKey: []byte(""), EndKey: []byte(""), Role: placement.Voter, Count: 2, LabelConstraints: []placement.LabelConstraint{ {Key: "dc", Op: "in", Values: []string{"dc1"}}, }, @@ -1761,7 +1761,7 @@ func TestCalculateStoreSize2(t *testing.T) { ) cluster.ruleManager.SetRule( - &placement.Rule{GroupID: "pd", ID: "logic3", StartKey: []byte(""), EndKey: []byte(""), Role: "voter", Count: 1, + &placement.Rule{GroupID: placement.DefaultGroupID, ID: "logic3", StartKey: []byte(""), EndKey: []byte(""), Role: placement.Voter, Count: 1, LabelConstraints: []placement.LabelConstraint{ {Key: "logic", Op: "in", Values: []string{"logic3"}}, }, @@ -1769,13 +1769,13 @@ func TestCalculateStoreSize2(t *testing.T) { ) cluster.ruleManager.SetRule( - &placement.Rule{GroupID: "pd", ID: "logic4", StartKey: []byte(""), EndKey: []byte(""), Role: "learner", Count: 1, + &placement.Rule{GroupID: placement.DefaultGroupID, ID: "logic4", StartKey: []byte(""), EndKey: []byte(""), Role: placement.Learner, Count: 1, LabelConstraints: []placement.LabelConstraint{ {Key: "logic", Op: "in", Values: []string{"logic4"}}, }, LocationLabels: []string{"dc", "logic", "rack", "host"}}, ) - cluster.ruleManager.DeleteRule("pd", "default") + cluster.ruleManager.DeleteRule(placement.DefaultGroupID, placement.DefaultRuleID) regions := newTestRegions(100, 10, 5) for _, region := range regions { diff --git a/server/server.go b/server/server.go index 38064a3b92f..d4b40af9c18 100644 --- a/server/server.go +++ b/server/server.go @@ -1043,7 +1043,7 @@ func (s *Server) SetReplicationConfig(cfg sc.ReplicationConfig) error { return errs.ErrNotBootstrapped.GenWithStackByArgs() } // replication.MaxReplicas won't work when placement rule is enabled and not only have one default rule. - defaultRule := rc.GetRuleManager().GetRule("pd", "default") + defaultRule := rc.GetRuleManager().GetRule(placement.DefaultGroupID, placement.DefaultRuleID) CheckInDefaultRule := func() error { // replication config won't work when placement rule is enabled and exceeds one default rule diff --git a/tests/integrations/client/http_client_test.go b/tests/integrations/client/http_client_test.go index 03d90c6cd32..d2c88d01f09 100644 --- a/tests/integrations/client/http_client_test.go +++ b/tests/integrations/client/http_client_test.go @@ -21,6 +21,7 @@ import ( "github.com/stretchr/testify/suite" pd "github.com/tikv/pd/client/http" + "github.com/tikv/pd/pkg/schedule/placement" "github.com/tikv/pd/tests" ) @@ -85,3 +86,45 @@ func (suite *httpClientTestSuite) TestGetMinResolvedTSByStoresIDs() { re.Equal(minResolvedTS, storeMinResolvedTSMap[1]) re.Equal(uint64(math.MaxUint64), storeMinResolvedTSMap[2]) } + +func (suite *httpClientTestSuite) TestRule() { + re := suite.Require() + rules, err := suite.client.GetPlacementRulesByGroup(suite.ctx, placement.DefaultGroupID) + re.NoError(err) + re.Len(rules, 1) + re.Equal(placement.DefaultGroupID, rules[0].GroupID) + re.Equal(placement.DefaultRuleID, rules[0].ID) + re.Equal(pd.Voter, rules[0].Role) + re.Equal(3, rules[0].Count) + err = suite.client.SetPlacementRule(suite.ctx, &pd.Rule{ + GroupID: placement.DefaultGroupID, + ID: "test", + Role: pd.Learner, + Count: 3, + }) + re.NoError(err) + rules, err = suite.client.GetPlacementRulesByGroup(suite.ctx, placement.DefaultGroupID) + re.NoError(err) + re.Len(rules, 2) + re.Equal(placement.DefaultGroupID, rules[1].GroupID) + re.Equal("test", rules[1].ID) + re.Equal(pd.Learner, rules[1].Role) + re.Equal(3, rules[1].Count) + err = suite.client.DeletePlacementRule(suite.ctx, placement.DefaultGroupID, "test") + re.NoError(err) + rules, err = suite.client.GetPlacementRulesByGroup(suite.ctx, placement.DefaultGroupID) + re.NoError(err) + re.Len(rules, 1) + re.Equal(placement.DefaultGroupID, rules[0].GroupID) + re.Equal(placement.DefaultRuleID, rules[0].ID) +} + +func (suite *httpClientTestSuite) TestAccelerateSchedule() { + re := suite.Require() + suspectRegions := suite.cluster.GetLeaderServer().GetRaftCluster().GetSuspectRegions() + re.Len(suspectRegions, 0) + err := suite.client.AccelerateSchedule(suite.ctx, []byte("a1"), []byte("a2")) + re.NoError(err) + suspectRegions = suite.cluster.GetLeaderServer().GetRaftCluster().GetSuspectRegions() + re.Len(suspectRegions, 1) +} diff --git a/tests/integrations/mcs/scheduling/api_test.go b/tests/integrations/mcs/scheduling/api_test.go index cfeaa4db033..f6a7f66a66f 100644 --- a/tests/integrations/mcs/scheduling/api_test.go +++ b/tests/integrations/mcs/scheduling/api_test.go @@ -241,9 +241,9 @@ func (suite *apiTestSuite) TestAPIForward() { tests.MustPutRegion(re, suite.cluster, 2, 1, []byte("a"), []byte("b"), core.SetApproximateSize(60)) rules = []*placement.Rule{ { - GroupID: "pd", - ID: "default", - Role: "voter", + GroupID: placement.DefaultGroupID, + ID: placement.DefaultRuleID, + Role: placement.Voter, Count: 3, LocationLabels: []string{}, }, diff --git a/tests/integrations/mcs/scheduling/rule_test.go b/tests/integrations/mcs/scheduling/rule_test.go index bffa58d0fe6..761e9b1ecbc 100644 --- a/tests/integrations/mcs/scheduling/rule_test.go +++ b/tests/integrations/mcs/scheduling/rule_test.go @@ -76,8 +76,8 @@ func (suite *ruleTestSuite) TestRuleWatch() { // Check the default rule and rule group. rules := ruleManager.GetAllRules() re.Len(rules, 1) - re.Equal("pd", rules[0].GroupID) - re.Equal("default", rules[0].ID) + re.Equal(placement.DefaultGroupID, rules[0].GroupID) + re.Equal(placement.DefaultRuleID, rules[0].ID) re.Equal(0, rules[0].Index) re.Empty(rules[0].StartKey) re.Empty(rules[0].EndKey) @@ -85,7 +85,7 @@ func (suite *ruleTestSuite) TestRuleWatch() { re.Empty(rules[0].LocationLabels) ruleGroups := ruleManager.GetRuleGroups() re.Len(ruleGroups, 1) - re.Equal("pd", ruleGroups[0].ID) + re.Equal(placement.DefaultGroupID, ruleGroups[0].ID) re.Equal(0, ruleGroups[0].Index) re.False(ruleGroups[0].Override) // Set a new rule via the PD API server. @@ -93,7 +93,7 @@ func (suite *ruleTestSuite) TestRuleWatch() { rule := &placement.Rule{ GroupID: "2", ID: "3", - Role: "voter", + Role: placement.Voter, Count: 1, StartKeyHex: "22", EndKeyHex: "dd", @@ -122,7 +122,7 @@ func (suite *ruleTestSuite) TestRuleWatch() { return len(rules) == 1 }) re.Len(rules, 1) - re.Equal("pd", rules[0].GroupID) + re.Equal(placement.DefaultGroupID, rules[0].GroupID) // Create a new rule group. ruleGroup := &placement.RuleGroup{ ID: "2", diff --git a/tests/pdctl/config/config_test.go b/tests/pdctl/config/config_test.go index 315ec3cf7c7..91d6723c2ac 100644 --- a/tests/pdctl/config/config_test.go +++ b/tests/pdctl/config/config_test.go @@ -315,7 +315,7 @@ func (suite *configTestSuite) checkPlacementRules(cluster *tests.TestCluster) { re.Contains(string(output), "Success!") // test show - suite.checkShowRuleKey(pdAddr, [][2]string{{"pd", "default"}}) + suite.checkShowRuleKey(pdAddr, [][2]string{{placement.DefaultGroupID, placement.DefaultRuleID}}) f, _ := os.CreateTemp("/tmp", "pd_tests") fname := f.Name() @@ -323,18 +323,18 @@ func (suite *configTestSuite) checkPlacementRules(cluster *tests.TestCluster) { defer os.RemoveAll(fname) // test load - rules := suite.checkLoadRule(pdAddr, fname, [][2]string{{"pd", "default"}}) + rules := suite.checkLoadRule(pdAddr, fname, [][2]string{{placement.DefaultGroupID, placement.DefaultRuleID}}) // test save rules = append(rules, placement.Rule{ - GroupID: "pd", + GroupID: placement.DefaultGroupID, ID: "test1", - Role: "voter", + Role: placement.Voter, Count: 1, }, placement.Rule{ GroupID: "test-group", ID: "test2", - Role: "voter", + Role: placement.Voter, Count: 2, }) b, _ := json.Marshal(rules) @@ -343,11 +343,11 @@ func (suite *configTestSuite) checkPlacementRules(cluster *tests.TestCluster) { re.NoError(err) // test show group - suite.checkShowRuleKey(pdAddr, [][2]string{{"pd", "default"}, {"pd", "test1"}}, "--group=pd") + suite.checkShowRuleKey(pdAddr, [][2]string{{placement.DefaultGroupID, placement.DefaultRuleID}, {placement.DefaultGroupID, "test1"}}, "--group=pd") // test rule region detail tests.MustPutRegion(re, cluster, 1, 1, []byte("a"), []byte("b")) - suite.checkShowRuleKey(pdAddr, [][2]string{{"pd", "default"}}, "--region=1", "--detail") + suite.checkShowRuleKey(pdAddr, [][2]string{{placement.DefaultGroupID, placement.DefaultRuleID}}, "--region=1", "--detail") // test delete // need clear up args, so create new a cobra.Command. Otherwise gourp still exists. @@ -356,7 +356,7 @@ func (suite *configTestSuite) checkPlacementRules(cluster *tests.TestCluster) { os.WriteFile(fname, b, 0600) _, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "save", "--in="+fname) re.NoError(err) - suite.checkShowRuleKey(pdAddr, [][2]string{{"pd", "test1"}}, "--group=pd") + suite.checkShowRuleKey(pdAddr, [][2]string{{placement.DefaultGroupID, "test1"}}, "--group=pd") } func (suite *configTestSuite) TestPlacementRuleGroups() { @@ -385,15 +385,15 @@ func (suite *configTestSuite) checkPlacementRuleGroups(cluster *tests.TestCluste // test show var group placement.RuleGroup testutil.Eventually(re, func() bool { // wait for the config to be synced to the scheduling server - output, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-group", "show", "pd") + output, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-group", "show", placement.DefaultGroupID) re.NoError(err) return !strings.Contains(string(output), "404") }) re.NoError(json.Unmarshal(output, &group), string(output)) - re.Equal(placement.RuleGroup{ID: "pd"}, group) + re.Equal(placement.RuleGroup{ID: placement.DefaultGroupID}, group) // test set - output, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-group", "set", "pd", "42", "true") + output, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-group", "set", placement.DefaultGroupID, "42", "true") re.NoError(err) re.Contains(string(output), "Success!") output, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-group", "set", "group2", "100", "false") @@ -410,7 +410,7 @@ func (suite *configTestSuite) checkPlacementRuleGroups(cluster *tests.TestCluste re.NoError(err) re.NoError(json.Unmarshal(output, &groups)) return reflect.DeepEqual([]placement.RuleGroup{ - {ID: "pd", Index: 42, Override: true}, + {ID: placement.DefaultGroupID, Index: 42, Override: true}, {ID: "group2", Index: 100, Override: false}, {ID: "group3", Index: 200, Override: false}, }, groups) @@ -464,10 +464,10 @@ func (suite *configTestSuite) checkPlacementRuleBundle(cluster *tests.TestCluste // test get var bundle placement.GroupBundle - output, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-bundle", "get", "pd") + output, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-bundle", "get", placement.DefaultGroupID) re.NoError(err) re.NoError(json.Unmarshal(output, &bundle)) - re.Equal(placement.GroupBundle{ID: "pd", Index: 0, Override: false, Rules: []*placement.Rule{{GroupID: "pd", ID: "default", Role: "voter", Count: 3}}}, bundle) + re.Equal(placement.GroupBundle{ID: placement.DefaultGroupID, Index: 0, Override: false, Rules: []*placement.Rule{{GroupID: placement.DefaultGroupID, ID: placement.DefaultRuleID, Role: placement.Voter, Count: 3}}}, bundle) f, err := os.CreateTemp("/tmp", "pd_tests") re.NoError(err) @@ -477,7 +477,7 @@ func (suite *configTestSuite) checkPlacementRuleBundle(cluster *tests.TestCluste // test load suite.checkLoadRuleBundle(pdAddr, fname, []placement.GroupBundle{ - {ID: "pd", Index: 0, Override: false, Rules: []*placement.Rule{{GroupID: "pd", ID: "default", Role: "voter", Count: 3}}}, + {ID: placement.DefaultGroupID, Index: 0, Override: false, Rules: []*placement.Rule{{GroupID: placement.DefaultGroupID, ID: placement.DefaultRuleID, Role: placement.Voter, Count: 3}}}, }) // test set @@ -489,41 +489,41 @@ func (suite *configTestSuite) checkPlacementRuleBundle(cluster *tests.TestCluste _, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-bundle", "set", "--in="+fname) re.NoError(err) suite.checkLoadRuleBundle(pdAddr, fname, []placement.GroupBundle{ - {ID: "pd", Index: 0, Override: false, Rules: []*placement.Rule{{GroupID: "pd", ID: "default", Role: "voter", Count: 3}}}, - {ID: "pe", Index: 0, Override: false, Rules: []*placement.Rule{{GroupID: "pe", ID: "default", Role: "voter", Count: 3}}}, + {ID: placement.DefaultGroupID, Index: 0, Override: false, Rules: []*placement.Rule{{GroupID: placement.DefaultGroupID, ID: placement.DefaultRuleID, Role: placement.Voter, Count: 3}}}, + {ID: "pe", Index: 0, Override: false, Rules: []*placement.Rule{{GroupID: "pe", ID: placement.DefaultRuleID, Role: placement.Voter, Count: 3}}}, }) // test delete - _, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-bundle", "delete", "pd") + _, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-bundle", "delete", placement.DefaultGroupID) re.NoError(err) suite.checkLoadRuleBundle(pdAddr, fname, []placement.GroupBundle{ - {ID: "pe", Index: 0, Override: false, Rules: []*placement.Rule{{GroupID: "pe", ID: "default", Role: "voter", Count: 3}}}, + {ID: "pe", Index: 0, Override: false, Rules: []*placement.Rule{{GroupID: "pe", ID: placement.DefaultRuleID, Role: placement.Voter, Count: 3}}}, }) // test delete regexp bundle.ID = "pf" - bundle.Rules = []*placement.Rule{{GroupID: "pf", ID: "default", Role: "voter", Count: 3}} + bundle.Rules = []*placement.Rule{{GroupID: "pf", ID: placement.DefaultRuleID, Role: placement.Voter, Count: 3}} b, err = json.Marshal(bundle) re.NoError(err) re.NoError(os.WriteFile(fname, b, 0600)) _, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-bundle", "set", "--in="+fname) re.NoError(err) suite.checkLoadRuleBundle(pdAddr, fname, []placement.GroupBundle{ - {ID: "pe", Index: 0, Override: false, Rules: []*placement.Rule{{GroupID: "pe", ID: "default", Role: "voter", Count: 3}}}, - {ID: "pf", Index: 0, Override: false, Rules: []*placement.Rule{{GroupID: "pf", ID: "default", Role: "voter", Count: 3}}}, + {ID: "pe", Index: 0, Override: false, Rules: []*placement.Rule{{GroupID: "pe", ID: placement.DefaultRuleID, Role: placement.Voter, Count: 3}}}, + {ID: "pf", Index: 0, Override: false, Rules: []*placement.Rule{{GroupID: "pf", ID: placement.DefaultRuleID, Role: placement.Voter, Count: 3}}}, }) _, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-bundle", "delete", "--regexp", ".*f") re.NoError(err) bundles := []placement.GroupBundle{ - {ID: "pe", Index: 0, Override: false, Rules: []*placement.Rule{{GroupID: "pe", ID: "default", Role: "voter", Count: 3}}}, + {ID: "pe", Index: 0, Override: false, Rules: []*placement.Rule{{GroupID: "pe", ID: placement.DefaultRuleID, Role: placement.Voter, Count: 3}}}, } suite.checkLoadRuleBundle(pdAddr, fname, bundles) // test save - bundle.Rules = []*placement.Rule{{GroupID: "pf", ID: "default", Role: "voter", Count: 3}} + bundle.Rules = []*placement.Rule{{GroupID: "pf", ID: placement.DefaultRuleID, Role: placement.Voter, Count: 3}} bundles = append(bundles, bundle) b, err = json.Marshal(bundles) re.NoError(err) @@ -531,8 +531,8 @@ func (suite *configTestSuite) checkPlacementRuleBundle(cluster *tests.TestCluste _, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-bundle", "save", "--in="+fname) re.NoError(err) suite.checkLoadRuleBundle(pdAddr, fname, []placement.GroupBundle{ - {ID: "pe", Index: 0, Override: false, Rules: []*placement.Rule{{GroupID: "pe", ID: "default", Role: "voter", Count: 3}}}, - {ID: "pf", Index: 0, Override: false, Rules: []*placement.Rule{{GroupID: "pf", ID: "default", Role: "voter", Count: 3}}}, + {ID: "pe", Index: 0, Override: false, Rules: []*placement.Rule{{GroupID: "pe", ID: placement.DefaultRuleID, Role: placement.Voter, Count: 3}}}, + {ID: "pf", Index: 0, Override: false, Rules: []*placement.Rule{{GroupID: "pf", ID: placement.DefaultRuleID, Role: placement.Voter, Count: 3}}}, }) // partial update, so still one group is left, no error @@ -544,7 +544,7 @@ func (suite *configTestSuite) checkPlacementRuleBundle(cluster *tests.TestCluste re.NoError(err) suite.checkLoadRuleBundle(pdAddr, fname, []placement.GroupBundle{ - {ID: "pf", Index: 0, Override: false, Rules: []*placement.Rule{{GroupID: "pf", ID: "default", Role: "voter", Count: 3}}}, + {ID: "pf", Index: 0, Override: false, Rules: []*placement.Rule{{GroupID: "pf", ID: placement.DefaultRuleID, Role: placement.Voter, Count: 3}}}, }) } @@ -715,7 +715,7 @@ func (suite *configTestSuite) checkUpdateDefaultReplicaConfig(cluster *tests.Tes } checkRuleCount := func(expect int) { - args := []string{"-u", pdAddr, "config", "placement-rules", "show", "--group", "pd", "--id", "default"} + args := []string{"-u", pdAddr, "config", "placement-rules", "show", "--group", placement.DefaultGroupID, "--id", placement.DefaultRuleID} output, err := pdctl.ExecuteCommand(cmd, args...) re.NoError(err) rule := placement.Rule{} @@ -726,7 +726,7 @@ func (suite *configTestSuite) checkUpdateDefaultReplicaConfig(cluster *tests.Tes } checkRuleLocationLabels := func(expect int) { - args := []string{"-u", pdAddr, "config", "placement-rules", "show", "--group", "pd", "--id", "default"} + args := []string{"-u", pdAddr, "config", "placement-rules", "show", "--group", placement.DefaultGroupID, "--id", placement.DefaultRuleID} output, err := pdctl.ExecuteCommand(cmd, args...) re.NoError(err) rule := placement.Rule{} @@ -737,7 +737,7 @@ func (suite *configTestSuite) checkUpdateDefaultReplicaConfig(cluster *tests.Tes } checkRuleIsolationLevel := func(expect string) { - args := []string{"-u", pdAddr, "config", "placement-rules", "show", "--group", "pd", "--id", "default"} + args := []string{"-u", pdAddr, "config", "placement-rules", "show", "--group", placement.DefaultGroupID, "--id", placement.DefaultRuleID} output, err := pdctl.ExecuteCommand(cmd, args...) re.NoError(err) rule := placement.Rule{} @@ -791,7 +791,7 @@ func (suite *configTestSuite) checkUpdateDefaultReplicaConfig(cluster *tests.Tes fname := suite.T().TempDir() rules := []placement.Rule{ { - GroupID: "pd", + GroupID: placement.DefaultGroupID, ID: "test1", Role: "voter", Count: 1, diff --git a/tests/server/api/operator_test.go b/tests/server/api/operator_test.go index 908daf21aac..e36ead7e44d 100644 --- a/tests/server/api/operator_test.go +++ b/tests/server/api/operator_test.go @@ -461,7 +461,7 @@ func (suite *operatorTestSuite) checkTransferRegionWithPlacementRule(cluster *te // add customized rule first and then remove default rule err := manager.SetRules(testCase.rules) suite.NoError(err) - err = manager.DeleteRule("pd", "default") + err = manager.DeleteRule(placement.DefaultGroupID, placement.DefaultRuleID) suite.NoError(err) } if testCase.expectedError == nil { diff --git a/tests/server/api/rule_test.go b/tests/server/api/rule_test.go index 6d292021767..861fbe5cf32 100644 --- a/tests/server/api/rule_test.go +++ b/tests/server/api/rule_test.go @@ -56,7 +56,7 @@ func (suite *ruleTestSuite) checkSet(cluster *tests.TestCluster) { pdAddr := leaderServer.GetAddr() urlPrefix := fmt.Sprintf("%s%s/api/v1/config", pdAddr, apiPrefix) - rule := placement.Rule{GroupID: "a", ID: "10", StartKeyHex: "1111", EndKeyHex: "3333", Role: "voter", Count: 1} + rule := placement.Rule{GroupID: "a", ID: "10", StartKeyHex: "1111", EndKeyHex: "3333", Role: placement.Voter, Count: 1} successData, err := json.Marshal(rule) suite.NoError(err) oldStartKey, err := hex.DecodeString(rule.StartKeyHex) @@ -64,13 +64,13 @@ func (suite *ruleTestSuite) checkSet(cluster *tests.TestCluster) { oldEndKey, err := hex.DecodeString(rule.EndKeyHex) suite.NoError(err) parseErrData := []byte("foo") - rule1 := placement.Rule{GroupID: "a", ID: "10", StartKeyHex: "XXXX", EndKeyHex: "3333", Role: "voter", Count: 1} + rule1 := placement.Rule{GroupID: "a", ID: "10", StartKeyHex: "XXXX", EndKeyHex: "3333", Role: placement.Voter, Count: 1} checkErrData, err := json.Marshal(rule1) suite.NoError(err) - rule2 := placement.Rule{GroupID: "a", ID: "10", StartKeyHex: "1111", EndKeyHex: "3333", Role: "voter", Count: -1} + rule2 := placement.Rule{GroupID: "a", ID: "10", StartKeyHex: "1111", EndKeyHex: "3333", Role: placement.Voter, Count: -1} setErrData, err := json.Marshal(rule2) suite.NoError(err) - rule3 := placement.Rule{GroupID: "a", ID: "10", StartKeyHex: "1111", EndKeyHex: "3333", Role: "follower", Count: 3} + rule3 := placement.Rule{GroupID: "a", ID: "10", StartKeyHex: "1111", EndKeyHex: "3333", Role: placement.Follower, Count: 3} updateData, err := json.Marshal(rule3) suite.NoError(err) newStartKey, err := hex.DecodeString(rule.StartKeyHex) @@ -179,7 +179,7 @@ func (suite *ruleTestSuite) checkGet(cluster *tests.TestCluster) { pdAddr := leaderServer.GetAddr() urlPrefix := fmt.Sprintf("%s%s/api/v1/config", pdAddr, apiPrefix) - rule := placement.Rule{GroupID: "a", ID: "20", StartKeyHex: "1111", EndKeyHex: "3333", Role: "voter", Count: 1} + rule := placement.Rule{GroupID: "a", ID: "20", StartKeyHex: "1111", EndKeyHex: "3333", Role: placement.Voter, Count: 1} data, err := json.Marshal(rule) suite.NoError(err) re := suite.Require() @@ -200,7 +200,7 @@ func (suite *ruleTestSuite) checkGet(cluster *tests.TestCluster) { }, { name: "not found", - rule: placement.Rule{GroupID: "a", ID: "30", StartKeyHex: "1111", EndKeyHex: "3333", Role: "voter", Count: 1}, + rule: placement.Rule{GroupID: "a", ID: "30", StartKeyHex: "1111", EndKeyHex: "3333", Role: placement.Voter, Count: 1}, found: false, code: http.StatusNotFound, }, @@ -237,7 +237,7 @@ func (suite *ruleTestSuite) checkGetAll(cluster *tests.TestCluster) { pdAddr := leaderServer.GetAddr() urlPrefix := fmt.Sprintf("%s%s/api/v1/config", pdAddr, apiPrefix) - rule := placement.Rule{GroupID: "b", ID: "20", StartKeyHex: "1111", EndKeyHex: "3333", Role: "voter", Count: 1} + rule := placement.Rule{GroupID: "b", ID: "20", StartKeyHex: "1111", EndKeyHex: "3333", Role: placement.Voter, Count: 1} data, err := json.Marshal(rule) suite.NoError(err) re := suite.Require() @@ -266,16 +266,16 @@ func (suite *ruleTestSuite) checkSetAll(cluster *tests.TestCluster) { pdAddr := leaderServer.GetAddr() urlPrefix := fmt.Sprintf("%s%s/api/v1/config", pdAddr, apiPrefix) - rule1 := placement.Rule{GroupID: "a", ID: "12", StartKeyHex: "1111", EndKeyHex: "3333", Role: "voter", Count: 1} - rule2 := placement.Rule{GroupID: "b", ID: "12", StartKeyHex: "1111", EndKeyHex: "3333", Role: "voter", Count: 1} - rule3 := placement.Rule{GroupID: "a", ID: "12", StartKeyHex: "XXXX", EndKeyHex: "3333", Role: "voter", Count: 1} - rule4 := placement.Rule{GroupID: "a", ID: "12", StartKeyHex: "1111", EndKeyHex: "3333", Role: "voter", Count: -1} - rule5 := placement.Rule{GroupID: "pd", ID: "default", StartKeyHex: "", EndKeyHex: "", Role: "voter", Count: 1, + rule1 := placement.Rule{GroupID: "a", ID: "12", StartKeyHex: "1111", EndKeyHex: "3333", Role: placement.Voter, Count: 1} + rule2 := placement.Rule{GroupID: "b", ID: "12", StartKeyHex: "1111", EndKeyHex: "3333", Role: placement.Voter, Count: 1} + rule3 := placement.Rule{GroupID: "a", ID: "12", StartKeyHex: "XXXX", EndKeyHex: "3333", Role: placement.Voter, Count: 1} + rule4 := placement.Rule{GroupID: "a", ID: "12", StartKeyHex: "1111", EndKeyHex: "3333", Role: placement.Voter, Count: -1} + rule5 := placement.Rule{GroupID: placement.DefaultGroupID, ID: placement.DefaultRuleID, StartKeyHex: "", EndKeyHex: "", Role: placement.Voter, Count: 1, LocationLabels: []string{"host"}} - rule6 := placement.Rule{GroupID: "pd", ID: "default", StartKeyHex: "", EndKeyHex: "", Role: "voter", Count: 3} + rule6 := placement.Rule{GroupID: placement.DefaultGroupID, ID: placement.DefaultRuleID, StartKeyHex: "", EndKeyHex: "", Role: placement.Voter, Count: 3} leaderServer.GetPersistOptions().GetReplicationConfig().LocationLabels = []string{"host"} - defaultRule := leaderServer.GetRaftCluster().GetRuleManager().GetRule("pd", "default") + defaultRule := leaderServer.GetRaftCluster().GetRuleManager().GetRule(placement.DefaultGroupID, placement.DefaultRuleID) defaultRule.LocationLabels = []string{"host"} leaderServer.GetRaftCluster().GetRuleManager().SetRule(defaultRule) @@ -390,13 +390,13 @@ func (suite *ruleTestSuite) checkGetAllByGroup(cluster *tests.TestCluster) { urlPrefix := fmt.Sprintf("%s%s/api/v1/config", pdAddr, apiPrefix) re := suite.Require() - rule := placement.Rule{GroupID: "c", ID: "20", StartKeyHex: "1111", EndKeyHex: "3333", Role: "voter", Count: 1} + rule := placement.Rule{GroupID: "c", ID: "20", StartKeyHex: "1111", EndKeyHex: "3333", Role: placement.Voter, Count: 1} data, err := json.Marshal(rule) suite.NoError(err) err = tu.CheckPostJSON(testDialClient, urlPrefix+"/rule", data, tu.StatusOK(re)) suite.NoError(err) - rule1 := placement.Rule{GroupID: "c", ID: "30", StartKeyHex: "1111", EndKeyHex: "3333", Role: "voter", Count: 1} + rule1 := placement.Rule{GroupID: "c", ID: "30", StartKeyHex: "1111", EndKeyHex: "3333", Role: placement.Voter, Count: 1} data, err = json.Marshal(rule1) suite.NoError(err) err = tu.CheckPostJSON(testDialClient, urlPrefix+"/rule", data, tu.StatusOK(re)) @@ -453,7 +453,7 @@ func (suite *ruleTestSuite) checkGetAllByRegion(cluster *tests.TestCluster) { pdAddr := leaderServer.GetAddr() urlPrefix := fmt.Sprintf("%s%s/api/v1/config", pdAddr, apiPrefix) - rule := placement.Rule{GroupID: "e", ID: "20", StartKeyHex: "1111", EndKeyHex: "3333", Role: "voter", Count: 1} + rule := placement.Rule{GroupID: "e", ID: "20", StartKeyHex: "1111", EndKeyHex: "3333", Role: placement.Voter, Count: 1} data, err := json.Marshal(rule) suite.NoError(err) re := suite.Require() @@ -525,7 +525,7 @@ func (suite *ruleTestSuite) checkGetAllByKey(cluster *tests.TestCluster) { pdAddr := leaderServer.GetAddr() urlPrefix := fmt.Sprintf("%s%s/api/v1/config", pdAddr, apiPrefix) - rule := placement.Rule{GroupID: "f", ID: "40", StartKeyHex: "8888", EndKeyHex: "9111", Role: "voter", Count: 1} + rule := placement.Rule{GroupID: "f", ID: "40", StartKeyHex: "8888", EndKeyHex: "9111", Role: placement.Voter, Count: 1} data, err := json.Marshal(rule) suite.NoError(err) re := suite.Require() @@ -589,7 +589,7 @@ func (suite *ruleTestSuite) checkDelete(cluster *tests.TestCluster) { pdAddr := leaderServer.GetAddr() urlPrefix := fmt.Sprintf("%s%s/api/v1/config", pdAddr, apiPrefix) - rule := placement.Rule{GroupID: "g", ID: "10", StartKeyHex: "8888", EndKeyHex: "9111", Role: "voter", Count: 1} + rule := placement.Rule{GroupID: "g", ID: "10", StartKeyHex: "8888", EndKeyHex: "9111", Role: placement.Voter, Count: 1} data, err := json.Marshal(rule) suite.NoError(err) err = tu.CheckPostJSON(testDialClient, urlPrefix+"/rule", data, tu.StatusOK(suite.Require())) @@ -663,19 +663,19 @@ func (suite *ruleTestSuite) checkBatch(cluster *tests.TestCluster) { opt1 := placement.RuleOp{ Action: placement.RuleOpAdd, - Rule: &placement.Rule{GroupID: "a", ID: "13", StartKeyHex: "1111", EndKeyHex: "3333", Role: "voter", Count: 1}, + Rule: &placement.Rule{GroupID: "a", ID: "13", StartKeyHex: "1111", EndKeyHex: "3333", Role: placement.Voter, Count: 1}, } opt2 := placement.RuleOp{ Action: placement.RuleOpAdd, - Rule: &placement.Rule{GroupID: "b", ID: "13", StartKeyHex: "1111", EndKeyHex: "3333", Role: "voter", Count: 1}, + Rule: &placement.Rule{GroupID: "b", ID: "13", StartKeyHex: "1111", EndKeyHex: "3333", Role: placement.Voter, Count: 1}, } opt3 := placement.RuleOp{ Action: placement.RuleOpAdd, - Rule: &placement.Rule{GroupID: "a", ID: "14", StartKeyHex: "1111", EndKeyHex: "3333", Role: "voter", Count: 1}, + Rule: &placement.Rule{GroupID: "a", ID: "14", StartKeyHex: "1111", EndKeyHex: "3333", Role: placement.Voter, Count: 1}, } opt4 := placement.RuleOp{ Action: placement.RuleOpAdd, - Rule: &placement.Rule{GroupID: "a", ID: "15", StartKeyHex: "1111", EndKeyHex: "3333", Role: "voter", Count: 1}, + Rule: &placement.Rule{GroupID: "a", ID: "15", StartKeyHex: "1111", EndKeyHex: "3333", Role: placement.Voter, Count: 1}, } opt5 := placement.RuleOp{ Action: placement.RuleOpDel, @@ -692,11 +692,11 @@ func (suite *ruleTestSuite) checkBatch(cluster *tests.TestCluster) { } opt8 := placement.RuleOp{ Action: placement.RuleOpAdd, - Rule: &placement.Rule{GroupID: "a", ID: "16", StartKeyHex: "XXXX", EndKeyHex: "3333", Role: "voter", Count: 1}, + Rule: &placement.Rule{GroupID: "a", ID: "16", StartKeyHex: "XXXX", EndKeyHex: "3333", Role: placement.Voter, Count: 1}, } opt9 := placement.RuleOp{ Action: placement.RuleOpAdd, - Rule: &placement.Rule{GroupID: "a", ID: "17", StartKeyHex: "1111", EndKeyHex: "3333", Role: "voter", Count: -1}, + Rule: &placement.Rule{GroupID: "a", ID: "17", StartKeyHex: "1111", EndKeyHex: "3333", Role: placement.Voter, Count: -1}, } successData1, err := json.Marshal([]placement.RuleOp{opt1, opt2, opt3}) @@ -800,9 +800,14 @@ func (suite *ruleTestSuite) checkBundle(cluster *tests.TestCluster) { re := suite.Require() // GetAll b1 := placement.GroupBundle{ - ID: "pd", + ID: placement.DefaultGroupID, Rules: []*placement.Rule{ - {GroupID: "pd", ID: "default", Role: "voter", Count: 3}, + { + GroupID: placement.DefaultGroupID, + ID: placement.DefaultRuleID, + Role: placement.Voter, + Count: 3, + }, }, } var bundles []placement.GroupBundle @@ -817,7 +822,7 @@ func (suite *ruleTestSuite) checkBundle(cluster *tests.TestCluster) { Index: 42, Override: true, Rules: []*placement.Rule{ - {GroupID: "foo", ID: "bar", Index: 1, Override: true, Role: "voter", Count: 1}, + {GroupID: "foo", ID: "bar", Index: 1, Override: true, Role: placement.Voter, Count: 1}, }, } data, err := json.Marshal(b2) @@ -849,7 +854,7 @@ func (suite *ruleTestSuite) checkBundle(cluster *tests.TestCluster) { suite.compareBundle(bundles[0], b2) // SetAll - b2.Rules = append(b2.Rules, &placement.Rule{GroupID: "foo", ID: "baz", Index: 2, Role: "follower", Count: 1}) + b2.Rules = append(b2.Rules, &placement.Rule{GroupID: "foo", ID: "baz", Index: 2, Role: placement.Follower, Count: 1}) b2.Index, b2.Override = 0, false b3 := placement.GroupBundle{ID: "foobar", Index: 100} data, err = json.Marshal([]placement.GroupBundle{b1, b2, b3}) @@ -880,7 +885,7 @@ func (suite *ruleTestSuite) checkBundle(cluster *tests.TestCluster) { b4 := placement.GroupBundle{ Index: 4, Rules: []*placement.Rule{ - {ID: "bar", Index: 1, Override: true, Role: "voter", Count: 1}, + {ID: "bar", Index: 1, Override: true, Role: placement.Voter, Count: 1}, }, } data, err = json.Marshal(b4) @@ -908,7 +913,7 @@ func (suite *ruleTestSuite) checkBundle(cluster *tests.TestCluster) { ID: "rule-without-group-id-2", Index: 5, Rules: []*placement.Rule{ - {ID: "bar", Index: 1, Override: true, Role: "voter", Count: 1}, + {ID: "bar", Index: 1, Override: true, Role: placement.Voter, Count: 1}, }, } data, err = json.Marshal([]placement.GroupBundle{b1, b4, b5}) diff --git a/tools/pd-simulator/simulator/cases/diagnose_rule.go b/tools/pd-simulator/simulator/cases/diagnose_rule.go index b4b30fdc772..6cd76c854b7 100644 --- a/tools/pd-simulator/simulator/cases/diagnose_rule.go +++ b/tools/pd-simulator/simulator/cases/diagnose_rule.go @@ -46,8 +46,8 @@ func newRule1() *Case { }, LocationLabels: []string{"host"}, }, &placement.Rule{ - GroupID: "pd", - ID: "default", + GroupID: placement.DefaultGroupID, + ID: placement.DefaultRuleID, StartKeyHex: "", EndKeyHex: "", Role: placement.Voter, From 9845c12d2a40ef3e28e0ddf9c803f33994102f81 Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Mon, 20 Nov 2023 11:10:11 +0800 Subject: [PATCH 024/137] mcs: dynamic enable scheduling jobs (#7325) ref tikv/pd#5839, close tikv/pd#7375 Signed-off-by: Ryan Leung --- pkg/mcs/scheduling/server/cluster.go | 9 +- pkg/mcs/scheduling/server/server.go | 1 + pkg/schedule/coordinator.go | 6 +- .../schedulers/scheduler_controller.go | 11 +- server/api/middleware.go | 4 +- server/cluster/cluster.go | 103 +++++++------- server/cluster/cluster_test.go | 10 +- server/cluster/cluster_worker.go | 3 + server/cluster/scheduling_controller.go | 127 ++++++++++++++---- .../mcs/scheduling/config_test.go | 3 +- .../mcs/scheduling/server_test.go | 38 ++++++ tests/pdctl/hot/hot_test.go | 1 + tests/pdctl/keyspace/keyspace_group_test.go | 3 +- tests/server/api/operator_test.go | 46 +++---- tests/testutil.go | 3 + 15 files changed, 249 insertions(+), 119 deletions(-) diff --git a/pkg/mcs/scheduling/server/cluster.go b/pkg/mcs/scheduling/server/cluster.go index 028c2a12b37..ac15212553b 100644 --- a/pkg/mcs/scheduling/server/cluster.go +++ b/pkg/mcs/scheduling/server/cluster.go @@ -502,8 +502,8 @@ func (c *Cluster) collectClusterMetrics() { func (c *Cluster) resetMetrics() { statistics.Reset() - c.coordinator.GetSchedulersController().ResetSchedulerMetrics() - c.coordinator.ResetHotSpotMetrics() + schedulers.ResetSchedulerMetrics() + schedule.ResetHotSpotMetrics() c.resetClusterMetrics() } @@ -538,6 +538,11 @@ func (c *Cluster) StopBackgroundJobs() { c.wg.Wait() } +// IsBackgroundJobsRunning returns whether the background jobs are running. Only for test purpose. +func (c *Cluster) IsBackgroundJobsRunning() bool { + return c.running.Load() +} + // HandleRegionHeartbeat processes RegionInfo reports from client. func (c *Cluster) HandleRegionHeartbeat(region *core.RegionInfo) error { if err := c.processRegionHeartbeat(region); err != nil { diff --git a/pkg/mcs/scheduling/server/server.go b/pkg/mcs/scheduling/server/server.go index 32b241fee91..c5b73dea5fc 100644 --- a/pkg/mcs/scheduling/server/server.go +++ b/pkg/mcs/scheduling/server/server.go @@ -405,6 +405,7 @@ func (s *Server) startServer() (err error) { // different service modes provided by the same pd-server binary serverInfo.WithLabelValues(versioninfo.PDReleaseVersion, versioninfo.PDGitHash).Set(float64(time.Now().Unix())) + s.serviceID = &discovery.ServiceRegistryEntry{ServiceAddr: s.cfg.AdvertiseListenAddr} uniqueName := s.cfg.GetAdvertiseListenAddr() uniqueID := memberutil.GenerateUniqueID(uniqueName) log.Info("joining primary election", zap.String("participant-name", uniqueName), zap.Uint64("participant-id", uniqueID)) diff --git a/pkg/schedule/coordinator.go b/pkg/schedule/coordinator.go index 8fb9ec8b286..6a02e68811d 100644 --- a/pkg/schedule/coordinator.go +++ b/pkg/schedule/coordinator.go @@ -88,8 +88,8 @@ type Coordinator struct { } // NewCoordinator creates a new Coordinator. -func NewCoordinator(ctx context.Context, cluster sche.ClusterInformer, hbStreams *hbstream.HeartbeatStreams) *Coordinator { - ctx, cancel := context.WithCancel(ctx) +func NewCoordinator(parentCtx context.Context, cluster sche.ClusterInformer, hbStreams *hbstream.HeartbeatStreams) *Coordinator { + ctx, cancel := context.WithCancel(parentCtx) opController := operator.NewController(ctx, cluster.GetBasicCluster(), cluster.GetSharedConfig(), hbStreams) schedulers := schedulers.NewController(ctx, cluster, cluster.GetStorage(), opController) checkers := checker.NewController(ctx, cluster, cluster.GetCheckerConfig(), cluster.GetRuleManager(), cluster.GetRegionLabeler(), opController) @@ -714,7 +714,7 @@ func collectHotMetrics(cluster sche.ClusterInformer, stores []*core.StoreInfo, t } // ResetHotSpotMetrics resets hot spot metrics. -func (c *Coordinator) ResetHotSpotMetrics() { +func ResetHotSpotMetrics() { hotSpotStatusGauge.Reset() schedulers.HotPendingSum.Reset() } diff --git a/pkg/schedule/schedulers/scheduler_controller.go b/pkg/schedule/schedulers/scheduler_controller.go index 79c8cbfbc92..5097a5f3f1c 100644 --- a/pkg/schedule/schedulers/scheduler_controller.go +++ b/pkg/schedule/schedulers/scheduler_controller.go @@ -38,8 +38,6 @@ const maxScheduleRetries = 10 var ( denySchedulersByLabelerCounter = labeler.LabelerEventCounter.WithLabelValues("schedulers", "deny") - rulesCntStatusGauge = ruleStatusGauge.WithLabelValues("rule_count") - groupsCntStatusGauge = ruleStatusGauge.WithLabelValues("group_count") ) // Controller is used to manage all schedulers. @@ -128,8 +126,8 @@ func (c *Controller) CollectSchedulerMetrics() { } ruleCnt := ruleMgr.GetRulesCount() groupCnt := ruleMgr.GetGroupsCount() - rulesCntStatusGauge.Set(float64(ruleCnt)) - groupsCntStatusGauge.Set(float64(groupCnt)) + ruleStatusGauge.WithLabelValues("rule_count").Set(float64(ruleCnt)) + ruleStatusGauge.WithLabelValues("group_count").Set(float64(groupCnt)) } func (c *Controller) isSchedulingHalted() bool { @@ -137,12 +135,9 @@ func (c *Controller) isSchedulingHalted() bool { } // ResetSchedulerMetrics resets metrics of all schedulers. -func (c *Controller) ResetSchedulerMetrics() { +func ResetSchedulerMetrics() { schedulerStatusGauge.Reset() ruleStatusGauge.Reset() - // create in map again - rulesCntStatusGauge = ruleStatusGauge.WithLabelValues("rule_count") - groupsCntStatusGauge = ruleStatusGauge.WithLabelValues("group_count") } // AddSchedulerHandler adds the HTTP handler for a scheduler. diff --git a/server/api/middleware.go b/server/api/middleware.go index cfeb0844dcf..627d7fecc92 100644 --- a/server/api/middleware.go +++ b/server/api/middleware.go @@ -114,7 +114,7 @@ func newAuditMiddleware(s *server.Server) negroni.Handler { return &auditMiddleware{svr: s} } -// ServeHTTP is used to implememt negroni.Handler for auditMiddleware +// ServeHTTP is used to implement negroni.Handler for auditMiddleware func (s *auditMiddleware) ServeHTTP(w http.ResponseWriter, r *http.Request, next http.HandlerFunc) { if !s.svr.GetServiceMiddlewarePersistOptions().IsAuditEnabled() { next(w, r) @@ -164,7 +164,7 @@ func newRateLimitMiddleware(s *server.Server) negroni.Handler { return &rateLimitMiddleware{svr: s} } -// ServeHTTP is used to implememt negroni.Handler for rateLimitMiddleware +// ServeHTTP is used to implement negroni.Handler for rateLimitMiddleware func (s *rateLimitMiddleware) ServeHTTP(w http.ResponseWriter, r *http.Request, next http.HandlerFunc) { if !s.svr.GetServiceMiddlewarePersistOptions().IsRateLimitEnabled() { next(w, r) diff --git a/server/cluster/cluster.go b/server/cluster/cluster.go index 3b50ae16d9b..0df543c96c2 100644 --- a/server/cluster/cluster.go +++ b/server/cluster/cluster.go @@ -41,15 +41,14 @@ import ( "github.com/tikv/pd/pkg/gctuner" "github.com/tikv/pd/pkg/id" "github.com/tikv/pd/pkg/keyspace" + "github.com/tikv/pd/pkg/mcs/discovery" mcsutils "github.com/tikv/pd/pkg/mcs/utils" "github.com/tikv/pd/pkg/memory" "github.com/tikv/pd/pkg/progress" "github.com/tikv/pd/pkg/replication" - "github.com/tikv/pd/pkg/schedule" sc "github.com/tikv/pd/pkg/schedule/config" "github.com/tikv/pd/pkg/schedule/hbstream" "github.com/tikv/pd/pkg/schedule/labeler" - "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/placement" "github.com/tikv/pd/pkg/slice" "github.com/tikv/pd/pkg/statistics" @@ -262,7 +261,7 @@ func (c *RaftCluster) InitCluster( storage storage.Storage, basicCluster *core.BasicCluster, hbstreams *hbstream.HeartbeatStreams, - keyspaceGroupManager *keyspace.GroupManager) { + keyspaceGroupManager *keyspace.GroupManager) error { c.core, c.opt, c.storage, c.id = basicCluster, opt.(*config.PersistOptions), storage, id c.ctx, c.cancel = context.WithCancel(c.serverCtx) c.progressManager = progress.NewManager() @@ -271,7 +270,15 @@ func (c *RaftCluster) InitCluster( c.unsafeRecoveryController = unsaferecovery.NewController(c) c.keyspaceGroupManager = keyspaceGroupManager c.hbstreams = hbstreams - c.schedulingController = newSchedulingController(c.ctx) + c.ruleManager = placement.NewRuleManager(c.storage, c, c.GetOpts()) + if c.opt.IsPlacementRulesEnabled() { + err := c.ruleManager.Initialize(c.opt.GetMaxReplicas(), c.opt.GetLocationLabels(), c.opt.GetIsolationLevel()) + if err != nil { + return err + } + } + c.schedulingController = newSchedulingController(c.ctx, c.core, c.opt, c.ruleManager) + return nil } // Start starts a cluster. @@ -285,7 +292,10 @@ func (c *RaftCluster) Start(s Server) error { } c.isAPIServiceMode = s.IsAPIServiceMode() - c.InitCluster(s.GetAllocator(), s.GetPersistOptions(), s.GetStorage(), s.GetBasicCluster(), s.GetHBStreams(), s.GetKeyspaceGroupManager()) + err := c.InitCluster(s.GetAllocator(), s.GetPersistOptions(), s.GetStorage(), s.GetBasicCluster(), s.GetHBStreams(), s.GetKeyspaceGroupManager()) + if err != nil { + return err + } cluster, err := c.LoadClusterInfo() if err != nil { return err @@ -294,24 +304,21 @@ func (c *RaftCluster) Start(s Server) error { return nil } - c.ruleManager = placement.NewRuleManager(c.storage, c, c.GetOpts()) - if c.opt.IsPlacementRulesEnabled() { - err = c.ruleManager.Initialize(c.opt.GetMaxReplicas(), c.opt.GetLocationLabels(), c.opt.GetIsolationLevel()) - if err != nil { - return err - } - } c.regionLabeler, err = labeler.NewRegionLabeler(c.ctx, c.storage, regionLabelGCInterval) if err != nil { return err } + if !c.IsServiceIndependent(mcsutils.SchedulingServiceName) { + for _, store := range c.GetStores() { + storeID := store.GetID() + c.slowStat.ObserveSlowStoreStatus(storeID, store.IsSlow()) + } + } c.replicationMode, err = replication.NewReplicationModeManager(s.GetConfig().ReplicationMode, c.storage, cluster, s) if err != nil { return err } - - c.schedulingController.init(c.core, c.opt, schedule.NewCoordinator(c.ctx, c, c.GetHeartbeatStreams()), c.ruleManager) c.limiter = NewStoreLimiter(s.GetPersistOptions()) c.externalTS, err = c.storage.LoadExternalTS() if err != nil { @@ -326,6 +333,7 @@ func (c *RaftCluster) Start(s Server) error { return err } } + c.checkServices() c.wg.Add(9) go c.runServiceCheckJob() go c.runMetricsCollectionJob() @@ -341,25 +349,39 @@ func (c *RaftCluster) Start(s Server) error { return nil } -func (c *RaftCluster) runServiceCheckJob() { - defer logutil.LogPanic() - defer c.wg.Done() - - var once sync.Once +var once sync.Once - checkFn := func() { - if c.isAPIServiceMode { - once.Do(c.initSchedulers) - c.independentServices.Store(mcsutils.SchedulingServiceName, true) - return - } - if c.startSchedulingJobs() { +func (c *RaftCluster) checkServices() { + if c.isAPIServiceMode { + servers, err := discovery.Discover(c.etcdClient, strconv.FormatUint(c.clusterID, 10), mcsutils.SchedulingServiceName) + if err != nil || len(servers) == 0 { + c.startSchedulingJobs(c, c.hbstreams) c.independentServices.Delete(mcsutils.SchedulingServiceName) + } else { + if c.stopSchedulingJobs() { + c.initCoordinator(c.ctx, c, c.hbstreams) + } else { + once.Do(func() { + c.initCoordinator(c.ctx, c, c.hbstreams) + }) + } + c.independentServices.Store(mcsutils.SchedulingServiceName, true) } + } else { + c.startSchedulingJobs(c, c.hbstreams) + c.independentServices.Delete(mcsutils.SchedulingServiceName) } - checkFn() +} + +func (c *RaftCluster) runServiceCheckJob() { + defer logutil.LogPanic() + defer c.wg.Done() ticker := time.NewTicker(serviceCheckInterval) + failpoint.Inject("highFrequencyClusterJobs", func() { + ticker.Stop() + ticker = time.NewTicker(time.Millisecond * 10) + }) defer ticker.Stop() for { @@ -368,7 +390,7 @@ func (c *RaftCluster) runServiceCheckJob() { log.Info("service check job is stopped") return case <-ticker.C: - checkFn() + c.checkServices() } } } @@ -621,12 +643,7 @@ func (c *RaftCluster) LoadClusterInfo() (*RaftCluster, error) { zap.Int("count", c.core.GetTotalRegionCount()), zap.Duration("cost", time.Since(start)), ) - if !c.IsServiceIndependent(mcsutils.SchedulingServiceName) { - for _, store := range c.GetStores() { - storeID := store.GetID() - c.slowStat.ObserveSlowStoreStatus(storeID, store.IsSlow()) - } - } + return c, nil } @@ -724,7 +741,7 @@ func (c *RaftCluster) Stop() { c.Unlock() c.wg.Wait() - log.Info("raftcluster is stopped") + log.Info("raft cluster is stopped") } // IsRunning return if the cluster is running. @@ -749,16 +766,6 @@ func (c *RaftCluster) GetHeartbeatStreams() *hbstream.HeartbeatStreams { return c.hbstreams } -// GetCoordinator returns the coordinator. -func (c *RaftCluster) GetCoordinator() *schedule.Coordinator { - return c.coordinator -} - -// GetOperatorController returns the operator controller. -func (c *RaftCluster) GetOperatorController() *operator.Controller { - return c.coordinator.GetOperatorController() -} - // AllocID returns a global unique ID. func (c *RaftCluster) AllocID() (uint64, error) { return c.id.Alloc() @@ -997,7 +1004,7 @@ func (c *RaftCluster) processRegionHeartbeat(region *core.RegionInfo) error { // Save to cache if meta or leader is updated, or contains any down/pending peer. // Mark isNew if the region in cache does not have leader. isNew, saveKV, saveCache, needSync := regionGuide(region, origin) - if !c.IsServiceIndependent(mcsutils.SchedulingServiceName) && !saveKV && !saveCache && !isNew { + if !saveKV && !saveCache && !isNew { // Due to some config changes need to update the region stats as well, // so we do some extra checks here. if hasRegionStats && c.regionStats.RegionStatsNeedUpdate(region) { @@ -1028,9 +1035,11 @@ func (c *RaftCluster) processRegionHeartbeat(region *core.RegionInfo) error { regionUpdateCacheEventCounter.Inc() } + isPrepared := true if !c.IsServiceIndependent(mcsutils.SchedulingServiceName) { - cluster.Collect(c, region, c.GetRegionStores(region), hasRegionStats, isNew, c.IsPrepared()) + isPrepared = c.IsPrepared() } + cluster.Collect(c, region, c.GetRegionStores(region), hasRegionStats, isNew, isPrepared) if c.storage != nil { // If there are concurrent heartbeats from the same region, the last write will win even if diff --git a/server/cluster/cluster_test.go b/server/cluster/cluster_test.go index d424ea98e7b..0e34ba4c743 100644 --- a/server/cluster/cluster_test.go +++ b/server/cluster/cluster_test.go @@ -2138,7 +2138,7 @@ func newTestRaftCluster( panic(err) } } - rc.schedulingController.init(basicCluster, opt, nil, rc.ruleManager) + rc.schedulingController = newSchedulingController(rc.ctx, rc.core, rc.opt, rc.ruleManager) return rc } @@ -2505,8 +2505,8 @@ func TestCollectMetricsConcurrent(t *testing.T) { controller.CollectSchedulerMetrics() co.GetCluster().(*RaftCluster).collectStatisticsMetrics() } - co.ResetHotSpotMetrics() - controller.ResetSchedulerMetrics() + schedule.ResetHotSpotMetrics() + schedulers.ResetSchedulerMetrics() co.GetCluster().(*RaftCluster).resetStatisticsMetrics() wg.Wait() } @@ -2551,8 +2551,8 @@ func TestCollectMetrics(t *testing.T) { s.Stats = nil } re.Equal(status1, status2) - co.ResetHotSpotMetrics() - controller.ResetSchedulerMetrics() + schedule.ResetHotSpotMetrics() + schedulers.ResetSchedulerMetrics() co.GetCluster().(*RaftCluster).resetStatisticsMetrics() } diff --git a/server/cluster/cluster_worker.go b/server/cluster/cluster_worker.go index 3a319c48196..74a445ad78e 100644 --- a/server/cluster/cluster_worker.go +++ b/server/cluster/cluster_worker.go @@ -38,6 +38,9 @@ func (c *RaftCluster) HandleRegionHeartbeat(region *core.RegionInfo) error { return err } + if c.IsServiceIndependent(mcsutils.SchedulingServiceName) { + return nil + } c.coordinator.GetOperatorController().Dispatch(region, operator.DispatchFromHeartBeat, c.coordinator.RecordOpStepWithTTL) return nil } diff --git a/server/cluster/scheduling_controller.go b/server/cluster/scheduling_controller.go index 1c41c830cf6..bb6470252b0 100644 --- a/server/cluster/scheduling_controller.go +++ b/server/cluster/scheduling_controller.go @@ -25,6 +25,10 @@ import ( "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/schedule" "github.com/tikv/pd/pkg/schedule/checker" + sc "github.com/tikv/pd/pkg/schedule/config" + sche "github.com/tikv/pd/pkg/schedule/core" + "github.com/tikv/pd/pkg/schedule/hbstream" + "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/placement" "github.com/tikv/pd/pkg/schedule/scatter" "github.com/tikv/pd/pkg/schedule/schedulers" @@ -33,9 +37,9 @@ import ( "github.com/tikv/pd/pkg/statistics/buckets" "github.com/tikv/pd/pkg/statistics/utils" "github.com/tikv/pd/pkg/utils/logutil" - "github.com/tikv/pd/server/config" ) +// schedulingController is used to manage all schedulers and checkers. type schedulingController struct { parentCtx context.Context ctx context.Context @@ -43,7 +47,7 @@ type schedulingController struct { mu sync.RWMutex wg sync.WaitGroup *core.BasicCluster - opt *config.PersistOptions + opt sc.ConfProvider coordinator *schedule.Coordinator labelStats *statistics.LabelStatistics regionStats *statistics.RegionStatistics @@ -52,25 +56,22 @@ type schedulingController struct { running bool } -func newSchedulingController(parentCtx context.Context) *schedulingController { +// newSchedulingController creates a new scheduling controller. +func newSchedulingController(parentCtx context.Context, basicCluster *core.BasicCluster, opt sc.ConfProvider, ruleManager *placement.RuleManager) *schedulingController { ctx, cancel := context.WithCancel(parentCtx) return &schedulingController{ - parentCtx: parentCtx, - ctx: ctx, - cancel: cancel, - labelStats: statistics.NewLabelStatistics(), - hotStat: statistics.NewHotStat(parentCtx), - slowStat: statistics.NewSlowStat(parentCtx), + parentCtx: parentCtx, + ctx: ctx, + cancel: cancel, + BasicCluster: basicCluster, + opt: opt, + labelStats: statistics.NewLabelStatistics(), + hotStat: statistics.NewHotStat(parentCtx), + slowStat: statistics.NewSlowStat(parentCtx), + regionStats: statistics.NewRegionStatistics(basicCluster, opt, ruleManager), } } -func (sc *schedulingController) init(basicCluster *core.BasicCluster, opt *config.PersistOptions, coordinator *schedule.Coordinator, ruleManager *placement.RuleManager) { - sc.BasicCluster = basicCluster - sc.opt = opt - sc.coordinator = coordinator - sc.regionStats = statistics.NewRegionStatistics(basicCluster, opt, ruleManager) -} - func (sc *schedulingController) stopSchedulingJobs() bool { sc.mu.Lock() defer sc.mu.Unlock() @@ -85,20 +86,31 @@ func (sc *schedulingController) stopSchedulingJobs() bool { return true } -func (sc *schedulingController) startSchedulingJobs() bool { +func (sc *schedulingController) startSchedulingJobs(cluster sche.ClusterInformer, hbstreams *hbstream.HeartbeatStreams) { sc.mu.Lock() defer sc.mu.Unlock() if sc.running { - return false + return } - sc.ctx, sc.cancel = context.WithCancel(sc.parentCtx) + sc.initCoordinatorLocked(sc.parentCtx, cluster, hbstreams) sc.wg.Add(3) go sc.runCoordinator() go sc.runStatsBackgroundJobs() go sc.runSchedulingMetricsCollectionJob() sc.running = true log.Info("scheduling service is started") - return true +} + +func (sc *schedulingController) initCoordinator(ctx context.Context, cluster sche.ClusterInformer, hbstreams *hbstream.HeartbeatStreams) { + sc.mu.Lock() + defer sc.mu.Unlock() + sc.initCoordinatorLocked(ctx, cluster, hbstreams) + sc.coordinator.InitSchedulers(false) +} + +func (sc *schedulingController) initCoordinatorLocked(ctx context.Context, cluster sche.ClusterInformer, hbstreams *hbstream.HeartbeatStreams) { + sc.ctx, sc.cancel = context.WithCancel(ctx) + sc.coordinator = schedule.NewCoordinator(sc.ctx, cluster, hbstreams) } // runCoordinator runs the main scheduling loop. @@ -156,8 +168,8 @@ func (sc *schedulingController) runSchedulingMetricsCollectionJob() { func (sc *schedulingController) resetSchedulingMetrics() { statistics.Reset() - sc.coordinator.GetSchedulersController().ResetSchedulerMetrics() - sc.coordinator.ResetHotSpotMetrics() + schedulers.ResetSchedulerMetrics() + schedule.ResetHotSpotMetrics() sc.resetStatisticsMetrics() } @@ -287,88 +299,136 @@ func (sc *schedulingController) BucketsStats(degree int, regionIDs ...uint64) ma return sc.hotStat.BucketsStats(degree, regionIDs...) } +// GetCoordinator returns the coordinator. +func (sc *schedulingController) GetCoordinator() *schedule.Coordinator { + sc.mu.RLock() + defer sc.mu.RUnlock() + return sc.coordinator +} + // GetPausedSchedulerDelayAt returns DelayAt of a paused scheduler func (sc *schedulingController) GetPausedSchedulerDelayAt(name string) (int64, error) { + sc.mu.RLock() + defer sc.mu.RUnlock() return sc.coordinator.GetSchedulersController().GetPausedSchedulerDelayAt(name) } // GetPausedSchedulerDelayUntil returns DelayUntil of a paused scheduler func (sc *schedulingController) GetPausedSchedulerDelayUntil(name string) (int64, error) { + sc.mu.RLock() + defer sc.mu.RUnlock() return sc.coordinator.GetSchedulersController().GetPausedSchedulerDelayUntil(name) } +// GetOperatorController returns the operator controller. +func (sc *schedulingController) GetOperatorController() *operator.Controller { + sc.mu.RLock() + defer sc.mu.RUnlock() + return sc.coordinator.GetOperatorController() +} + // GetRegionScatterer returns the region scatter. func (sc *schedulingController) GetRegionScatterer() *scatter.RegionScatterer { + sc.mu.RLock() + defer sc.mu.RUnlock() return sc.coordinator.GetRegionScatterer() } // GetRegionSplitter returns the region splitter func (sc *schedulingController) GetRegionSplitter() *splitter.RegionSplitter { + sc.mu.RLock() + defer sc.mu.RUnlock() return sc.coordinator.GetRegionSplitter() } // GetMergeChecker returns merge checker. func (sc *schedulingController) GetMergeChecker() *checker.MergeChecker { + sc.mu.RLock() + defer sc.mu.RUnlock() return sc.coordinator.GetMergeChecker() } // GetRuleChecker returns rule checker. func (sc *schedulingController) GetRuleChecker() *checker.RuleChecker { + sc.mu.RLock() + defer sc.mu.RUnlock() return sc.coordinator.GetRuleChecker() } // GetSchedulers gets all schedulers. func (sc *schedulingController) GetSchedulers() []string { + sc.mu.RLock() + defer sc.mu.RUnlock() return sc.coordinator.GetSchedulersController().GetSchedulerNames() } // GetSchedulerHandlers gets all scheduler handlers. func (sc *schedulingController) GetSchedulerHandlers() map[string]http.Handler { + sc.mu.RLock() + defer sc.mu.RUnlock() return sc.coordinator.GetSchedulersController().GetSchedulerHandlers() } // AddSchedulerHandler adds a scheduler handler. func (sc *schedulingController) AddSchedulerHandler(scheduler schedulers.Scheduler, args ...string) error { + sc.mu.RLock() + defer sc.mu.RUnlock() return sc.coordinator.GetSchedulersController().AddSchedulerHandler(scheduler, args...) } // RemoveSchedulerHandler removes a scheduler handler. func (sc *schedulingController) RemoveSchedulerHandler(name string) error { + sc.mu.RLock() + defer sc.mu.RUnlock() return sc.coordinator.GetSchedulersController().RemoveSchedulerHandler(name) } // AddScheduler adds a scheduler. func (sc *schedulingController) AddScheduler(scheduler schedulers.Scheduler, args ...string) error { + sc.mu.RLock() + defer sc.mu.RUnlock() return sc.coordinator.GetSchedulersController().AddScheduler(scheduler, args...) } // RemoveScheduler removes a scheduler. func (sc *schedulingController) RemoveScheduler(name string) error { + sc.mu.RLock() + defer sc.mu.RUnlock() return sc.coordinator.GetSchedulersController().RemoveScheduler(name) } // PauseOrResumeScheduler pauses or resumes a scheduler. func (sc *schedulingController) PauseOrResumeScheduler(name string, t int64) error { + sc.mu.RLock() + defer sc.mu.RUnlock() return sc.coordinator.GetSchedulersController().PauseOrResumeScheduler(name, t) } // PauseOrResumeChecker pauses or resumes checker. func (sc *schedulingController) PauseOrResumeChecker(name string, t int64) error { + sc.mu.RLock() + defer sc.mu.RUnlock() return sc.coordinator.PauseOrResumeChecker(name, t) } // AddSuspectRegions adds regions to suspect list. func (sc *schedulingController) AddSuspectRegions(regionIDs ...uint64) { + sc.mu.RLock() + defer sc.mu.RUnlock() sc.coordinator.GetCheckerController().AddSuspectRegions(regionIDs...) } // GetSuspectRegions gets all suspect regions. func (sc *schedulingController) GetSuspectRegions() []uint64 { + sc.mu.RLock() + defer sc.mu.RUnlock() return sc.coordinator.GetCheckerController().GetSuspectRegions() } // RemoveSuspectRegion removes region from suspect list. func (sc *schedulingController) RemoveSuspectRegion(id uint64) { + sc.mu.RLock() + defer sc.mu.RUnlock() sc.coordinator.GetCheckerController().RemoveSuspectRegion(id) } @@ -376,11 +436,15 @@ func (sc *schedulingController) RemoveSuspectRegion(id uint64) { // it would return value and true if pop success, or return empty [][2][]byte and false // if suspectKeyRanges couldn't pop keyRange group. func (sc *schedulingController) PopOneSuspectKeyRange() ([2][]byte, bool) { + sc.mu.RLock() + defer sc.mu.RUnlock() return sc.coordinator.GetCheckerController().PopOneSuspectKeyRange() } // ClearSuspectKeyRanges clears the suspect keyRanges, only for unit test func (sc *schedulingController) ClearSuspectKeyRanges() { + sc.mu.RLock() + defer sc.mu.RUnlock() sc.coordinator.GetCheckerController().ClearSuspectKeyRanges() } @@ -388,14 +452,14 @@ func (sc *schedulingController) ClearSuspectKeyRanges() { // The instance of each keyRange is like following format: // [2][]byte: start key/end key func (sc *schedulingController) AddSuspectKeyRange(start, end []byte) { + sc.mu.RLock() + defer sc.mu.RUnlock() sc.coordinator.GetCheckerController().AddSuspectKeyRange(start, end) } -func (sc *schedulingController) initSchedulers() { - sc.coordinator.InitSchedulers(false) -} - func (sc *schedulingController) getEvictLeaderStores() (evictStores []uint64) { + sc.mu.RLock() + defer sc.mu.RUnlock() if sc.coordinator == nil { return nil } @@ -415,10 +479,21 @@ func (sc *schedulingController) getEvictLeaderStores() (evictStores []uint64) { // IsPrepared return true if the prepare checker is ready. func (sc *schedulingController) IsPrepared() bool { + sc.mu.RLock() + defer sc.mu.RUnlock() return sc.coordinator.GetPrepareChecker().IsPrepared() } // SetPrepared set the prepare check to prepared. Only for test purpose. func (sc *schedulingController) SetPrepared() { + sc.mu.RLock() + defer sc.mu.RUnlock() sc.coordinator.GetPrepareChecker().SetPrepared() } + +// IsSchedulingControllerRunning returns whether the scheduling controller is running. Only for test purpose. +func (sc *schedulingController) IsSchedulingControllerRunning() bool { + sc.mu.RLock() + defer sc.mu.RUnlock() + return sc.running +} diff --git a/tests/integrations/mcs/scheduling/config_test.go b/tests/integrations/mcs/scheduling/config_test.go index 06d73caf130..ccf7cdaf48c 100644 --- a/tests/integrations/mcs/scheduling/config_test.go +++ b/tests/integrations/mcs/scheduling/config_test.go @@ -149,8 +149,9 @@ func (suite *configTestSuite) TestSchedulerConfigWatch() { ) re.NoError(err) // Get all default scheduler names. - var namesFromAPIServer, _, _ = suite.pdLeaderServer.GetRaftCluster().GetStorage().LoadAllSchedulerConfigs() + var namesFromAPIServer []string testutil.Eventually(re, func() bool { + namesFromAPIServer, _, _ = suite.pdLeaderServer.GetRaftCluster().GetStorage().LoadAllSchedulerConfigs() return len(namesFromAPIServer) == len(sc.DefaultSchedulers) }) // Check all default schedulers' configs. diff --git a/tests/integrations/mcs/scheduling/server_test.go b/tests/integrations/mcs/scheduling/server_test.go index a359e1d023a..41c00b8e9b4 100644 --- a/tests/integrations/mcs/scheduling/server_test.go +++ b/tests/integrations/mcs/scheduling/server_test.go @@ -195,6 +195,44 @@ func (suite *serverTestSuite) TestForwardStoreHeartbeat() { }) } +func (suite *serverTestSuite) TestDynamicSwitch() { + re := suite.Require() + // API server will execute scheduling jobs since there is no scheduler server. + testutil.Eventually(re, func() bool { + return suite.pdLeader.GetServer().GetRaftCluster().IsSchedulingControllerRunning() + }) + + tc, err := tests.NewTestSchedulingCluster(suite.ctx, 1, suite.backendEndpoints) + re.NoError(err) + defer tc.Destroy() + tc.WaitForPrimaryServing(re) + // After scheduling server is started, API server will not execute scheduling jobs. + testutil.Eventually(re, func() bool { + return !suite.pdLeader.GetServer().GetRaftCluster().IsSchedulingControllerRunning() + }) + // Scheduling server is responsible for executing scheduling jobs. + testutil.Eventually(re, func() bool { + return tc.GetPrimaryServer().GetCluster().IsBackgroundJobsRunning() + }) + tc.GetPrimaryServer().Close() + // Stop scheduling server. API server will execute scheduling jobs again. + testutil.Eventually(re, func() bool { + return suite.pdLeader.GetServer().GetRaftCluster().IsSchedulingControllerRunning() + }) + tc1, err := tests.NewTestSchedulingCluster(suite.ctx, 1, suite.backendEndpoints) + re.NoError(err) + defer tc1.Destroy() + tc1.WaitForPrimaryServing(re) + // After scheduling server is started, API server will not execute scheduling jobs. + testutil.Eventually(re, func() bool { + return !suite.pdLeader.GetServer().GetRaftCluster().IsSchedulingControllerRunning() + }) + // Scheduling server is responsible for executing scheduling jobs again. + testutil.Eventually(re, func() bool { + return tc1.GetPrimaryServer().GetCluster().IsBackgroundJobsRunning() + }) +} + func (suite *serverTestSuite) TestSchedulerSync() { re := suite.Require() tc, err := tests.NewTestSchedulingCluster(suite.ctx, 1, suite.backendEndpoints) diff --git a/tests/pdctl/hot/hot_test.go b/tests/pdctl/hot/hot_test.go index ac9bb3d83bf..8cab8ea9ab2 100644 --- a/tests/pdctl/hot/hot_test.go +++ b/tests/pdctl/hot/hot_test.go @@ -349,6 +349,7 @@ func (suite *hotTestSuite) checkHotWithoutHotPeer(cluster *tests.TestCluster) { hotRegion := statistics.StoreHotPeersInfos{} re.NoError(err) re.NoError(json.Unmarshal(output, &hotRegion)) + re.NotNil(hotRegion.AsPeer[1]) re.Equal(hotRegion.AsPeer[1].Count, 0) re.Equal(0.0, hotRegion.AsPeer[1].TotalBytesRate) re.Equal(load, hotRegion.AsPeer[1].StoreByteRate) diff --git a/tests/pdctl/keyspace/keyspace_group_test.go b/tests/pdctl/keyspace/keyspace_group_test.go index 0b09550d967..cbfdf1d099a 100644 --- a/tests/pdctl/keyspace/keyspace_group_test.go +++ b/tests/pdctl/keyspace/keyspace_group_test.go @@ -503,7 +503,7 @@ func TestShowKeyspaceGroupPrimary(t *testing.T) { for i := 0; i < 10; i++ { keyspaces = append(keyspaces, fmt.Sprintf("keyspace_%d", i)) } - tc, err := tests.NewTestAPICluster(ctx, 3, func(conf *config.Config, serverName string) { + tc, err := tests.NewTestAPICluster(ctx, 1, func(conf *config.Config, serverName string) { conf.Keyspace.PreAlloc = keyspaces }) re.NoError(err) @@ -528,7 +528,6 @@ func TestShowKeyspaceGroupPrimary(t *testing.T) { args := []string{"-u", pdAddr, "keyspace-group"} output, err := pdctl.ExecuteCommand(cmd, append(args, defaultKeyspaceGroupID)...) re.NoError(err) - err = json.Unmarshal(output, &keyspaceGroup) re.NoError(err) re.Equal(utils.DefaultKeyspaceGroupID, keyspaceGroup.ID) diff --git a/tests/server/api/operator_test.go b/tests/server/api/operator_test.go index e36ead7e44d..14b8618f6a6 100644 --- a/tests/server/api/operator_test.go +++ b/tests/server/api/operator_test.go @@ -27,7 +27,7 @@ import ( "github.com/pingcap/kvproto/pkg/metapb" "github.com/stretchr/testify/suite" "github.com/tikv/pd/pkg/core" - pdoperator "github.com/tikv/pd/pkg/schedule/operator" + "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/placement" tu "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/server/config" @@ -285,10 +285,10 @@ func (suite *operatorTestSuite) checkTransferRegionWithPlacementRule(cluster *te input: []byte(`{"name":"transfer-region", "region_id": 1, "to_store_ids": [2, 3]}`), expectedError: nil, expectSteps: convertStepsToStr([]string{ - pdoperator.AddLearner{ToStore: 3, PeerID: 1}.String(), - pdoperator.PromoteLearner{ToStore: 3, PeerID: 1}.String(), - pdoperator.TransferLeader{FromStore: 1, ToStore: 2}.String(), - pdoperator.RemovePeer{FromStore: 1, PeerID: 1}.String(), + operator.AddLearner{ToStore: 3, PeerID: 1}.String(), + operator.PromoteLearner{ToStore: 3, PeerID: 1}.String(), + operator.TransferLeader{FromStore: 1, ToStore: 2}.String(), + operator.RemovePeer{FromStore: 1, PeerID: 1}.String(), }), }, { @@ -297,11 +297,11 @@ func (suite *operatorTestSuite) checkTransferRegionWithPlacementRule(cluster *te input: []byte(`{"name":"transfer-region", "region_id": 1, "to_store_ids": [2, 3], "peer_roles":["follower", "leader"]}`), expectedError: nil, expectSteps: convertStepsToStr([]string{ - pdoperator.AddLearner{ToStore: 3, PeerID: 2}.String(), - pdoperator.PromoteLearner{ToStore: 3, PeerID: 2}.String(), - pdoperator.TransferLeader{FromStore: 1, ToStore: 2}.String(), - pdoperator.RemovePeer{FromStore: 1, PeerID: 2}.String(), - pdoperator.TransferLeader{FromStore: 2, ToStore: 3}.String(), + operator.AddLearner{ToStore: 3, PeerID: 2}.String(), + operator.PromoteLearner{ToStore: 3, PeerID: 2}.String(), + operator.TransferLeader{FromStore: 1, ToStore: 2}.String(), + operator.RemovePeer{FromStore: 1, PeerID: 2}.String(), + operator.TransferLeader{FromStore: 2, ToStore: 3}.String(), }), }, { @@ -316,11 +316,11 @@ func (suite *operatorTestSuite) checkTransferRegionWithPlacementRule(cluster *te placementRuleEnable: true, input: []byte(`{"name":"transfer-region", "region_id": 1, "to_store_ids": [2, 3], "peer_roles":["follower", "leader"]}`), expectSteps: convertStepsToStr([]string{ - pdoperator.AddLearner{ToStore: 3, PeerID: 3}.String(), - pdoperator.PromoteLearner{ToStore: 3, PeerID: 3}.String(), - pdoperator.TransferLeader{FromStore: 1, ToStore: 2}.String(), - pdoperator.RemovePeer{FromStore: 1, PeerID: 1}.String(), - pdoperator.TransferLeader{FromStore: 2, ToStore: 3}.String(), + operator.AddLearner{ToStore: 3, PeerID: 3}.String(), + operator.PromoteLearner{ToStore: 3, PeerID: 3}.String(), + operator.TransferLeader{FromStore: 1, ToStore: 2}.String(), + operator.RemovePeer{FromStore: 1, PeerID: 1}.String(), + operator.TransferLeader{FromStore: 2, ToStore: 3}.String(), }), }, { @@ -377,10 +377,10 @@ func (suite *operatorTestSuite) checkTransferRegionWithPlacementRule(cluster *te input: []byte(`{"name":"transfer-region", "region_id": 1, "to_store_ids": [2, 3], "peer_roles":["follower", "leader"]}`), expectedError: nil, expectSteps: convertStepsToStr([]string{ - pdoperator.AddLearner{ToStore: 3, PeerID: 5}.String(), - pdoperator.PromoteLearner{ToStore: 3, PeerID: 5}.String(), - pdoperator.TransferLeader{FromStore: 1, ToStore: 3}.String(), - pdoperator.RemovePeer{FromStore: 1, PeerID: 1}.String(), + operator.AddLearner{ToStore: 3, PeerID: 5}.String(), + operator.PromoteLearner{ToStore: 3, PeerID: 5}.String(), + operator.TransferLeader{FromStore: 1, ToStore: 3}.String(), + operator.RemovePeer{FromStore: 1, PeerID: 1}.String(), }), }, { @@ -417,10 +417,10 @@ func (suite *operatorTestSuite) checkTransferRegionWithPlacementRule(cluster *te input: []byte(`{"name":"transfer-region", "region_id": 1, "to_store_ids": [2, 3], "peer_roles":["leader", "follower"]}`), expectedError: nil, expectSteps: convertStepsToStr([]string{ - pdoperator.AddLearner{ToStore: 3, PeerID: 6}.String(), - pdoperator.PromoteLearner{ToStore: 3, PeerID: 6}.String(), - pdoperator.TransferLeader{FromStore: 1, ToStore: 2}.String(), - pdoperator.RemovePeer{FromStore: 1, PeerID: 1}.String(), + operator.AddLearner{ToStore: 3, PeerID: 6}.String(), + operator.PromoteLearner{ToStore: 3, PeerID: 6}.String(), + operator.TransferLeader{FromStore: 1, ToStore: 2}.String(), + operator.RemovePeer{FromStore: 1, PeerID: 1}.String(), }), }, } diff --git a/tests/testutil.go b/tests/testutil.go index 059a152f06f..2ccf6fb76be 100644 --- a/tests/testutil.go +++ b/tests/testutil.go @@ -299,6 +299,7 @@ func (s *SchedulingTestEnvironment) startCluster(m mode) { leaderServer := s.cluster.GetServer(s.cluster.GetLeader()) re.NoError(leaderServer.BootstrapCluster()) case apiMode: + re.NoError(failpoint.Enable("github.com/tikv/pd/server/cluster/highFrequencyClusterJobs", `return(true)`)) s.cluster, err = NewTestAPICluster(s.ctx, 1, s.opts...) re.NoError(err) err = s.cluster.RunInitialServers() @@ -306,11 +307,13 @@ func (s *SchedulingTestEnvironment) startCluster(m mode) { re.NotEmpty(s.cluster.WaitLeader()) leaderServer := s.cluster.GetServer(s.cluster.GetLeader()) re.NoError(leaderServer.BootstrapCluster()) + leaderServer.GetRaftCluster().SetPrepared() // start scheduling cluster tc, err := NewTestSchedulingCluster(s.ctx, 1, leaderServer.GetAddr()) re.NoError(err) tc.WaitForPrimaryServing(re) s.cluster.SetSchedulingCluster(tc) time.Sleep(200 * time.Millisecond) // wait for scheduling cluster to update member + re.NoError(failpoint.Disable("github.com/tikv/pd/server/cluster/highFrequencyClusterJobs")) } } From 49b32511c19127c3e0db772dfaace833caa1fb6e Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Mon, 20 Nov 2023 14:05:41 +0800 Subject: [PATCH 025/137] *: fix data race of `TestRaftClusterMultipleRestart` (#7392) close tikv/pd#7391 Signed-off-by: Ryan Leung --- server/cluster/cluster.go | 12 +++--- server/cluster/cluster_test.go | 61 +++++++++++++-------------- server/cluster/cluster_worker_test.go | 5 +-- server/server.go | 2 +- tests/server/cluster/cluster_test.go | 12 +++--- 5 files changed, 45 insertions(+), 47 deletions(-) diff --git a/server/cluster/cluster.go b/server/cluster/cluster.go index 0df543c96c2..3b826d8d33e 100644 --- a/server/cluster/cluster.go +++ b/server/cluster/cluster.go @@ -178,7 +178,7 @@ type Status struct { } // NewRaftCluster create a new cluster. -func NewRaftCluster(ctx context.Context, clusterID uint64, regionSyncer *syncer.RegionSyncer, etcdClient *clientv3.Client, +func NewRaftCluster(ctx context.Context, clusterID uint64, basicCluster *core.BasicCluster, storage storage.Storage, regionSyncer *syncer.RegionSyncer, etcdClient *clientv3.Client, httpClient *http.Client) *RaftCluster { return &RaftCluster{ serverCtx: ctx, @@ -186,6 +186,8 @@ func NewRaftCluster(ctx context.Context, clusterID uint64, regionSyncer *syncer. regionSyncer: regionSyncer, httpClient: httpClient, etcdClient: etcdClient, + core: basicCluster, + storage: storage, } } @@ -258,11 +260,9 @@ func (c *RaftCluster) loadBootstrapTime() (time.Time, error) { func (c *RaftCluster) InitCluster( id id.Allocator, opt sc.ConfProvider, - storage storage.Storage, - basicCluster *core.BasicCluster, hbstreams *hbstream.HeartbeatStreams, keyspaceGroupManager *keyspace.GroupManager) error { - c.core, c.opt, c.storage, c.id = basicCluster, opt.(*config.PersistOptions), storage, id + c.opt, c.id = opt.(*config.PersistOptions), id c.ctx, c.cancel = context.WithCancel(c.serverCtx) c.progressManager = progress.NewManager() c.changedRegions = make(chan *core.RegionInfo, defaultChangedRegionsLimit) @@ -292,7 +292,7 @@ func (c *RaftCluster) Start(s Server) error { } c.isAPIServiceMode = s.IsAPIServiceMode() - err := c.InitCluster(s.GetAllocator(), s.GetPersistOptions(), s.GetStorage(), s.GetBasicCluster(), s.GetHBStreams(), s.GetKeyspaceGroupManager()) + err := c.InitCluster(s.GetAllocator(), s.GetPersistOptions(), s.GetHBStreams(), s.GetKeyspaceGroupManager()) if err != nil { return err } @@ -380,7 +380,7 @@ func (c *RaftCluster) runServiceCheckJob() { ticker := time.NewTicker(serviceCheckInterval) failpoint.Inject("highFrequencyClusterJobs", func() { ticker.Stop() - ticker = time.NewTicker(time.Millisecond * 10) + ticker = time.NewTicker(time.Millisecond) }) defer ticker.Stop() diff --git a/server/cluster/cluster_test.go b/server/cluster/cluster_test.go index 0e34ba4c743..e9ce35dfb54 100644 --- a/server/cluster/cluster_test.go +++ b/server/cluster/cluster_test.go @@ -68,7 +68,7 @@ func TestStoreHeartbeat(t *testing.T) { _, opt, err := newTestScheduleConfig() opt.GetScheduleConfig().StoreLimitVersion = "v2" re.NoError(err) - cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) + cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend()) n, np := uint64(3), uint64(3) stores := newTestStores(n, "2.0.0") @@ -201,7 +201,7 @@ func TestFilterUnhealthyStore(t *testing.T) { _, opt, err := newTestScheduleConfig() re.NoError(err) - cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) + cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend()) stores := newTestStores(3, "2.0.0") req := &pdpb.StoreHeartbeatRequest{} @@ -239,7 +239,7 @@ func TestSetOfflineStore(t *testing.T) { _, opt, err := newTestScheduleConfig() re.NoError(err) - cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) + cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend()) cluster.coordinator = schedule.NewCoordinator(ctx, cluster, nil) cluster.ruleManager = placement.NewRuleManager(storage.NewStorageWithMemoryBackend(), cluster, cluster.GetOpts()) if opt.IsPlacementRulesEnabled() { @@ -305,7 +305,7 @@ func TestSetOfflineWithReplica(t *testing.T) { _, opt, err := newTestScheduleConfig() re.NoError(err) - cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) + cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend()) cluster.coordinator = schedule.NewCoordinator(ctx, cluster, nil) // Put 4 stores. @@ -344,7 +344,7 @@ func TestSetOfflineStoreWithEvictLeader(t *testing.T) { _, opt, err := newTestScheduleConfig() re.NoError(err) opt.SetMaxReplicas(1) - cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) + cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend()) cluster.coordinator = schedule.NewCoordinator(ctx, cluster, nil) // Put 3 stores. @@ -371,7 +371,7 @@ func TestForceBuryStore(t *testing.T) { _, opt, err := newTestScheduleConfig() re.NoError(err) - cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) + cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend()) // Put 2 stores. stores := newTestStores(2, "5.3.0") stores[1] = stores[1].Clone(core.SetLastHeartbeatTS(time.Now())) @@ -390,7 +390,7 @@ func TestReuseAddress(t *testing.T) { _, opt, err := newTestScheduleConfig() re.NoError(err) - cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) + cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend()) cluster.coordinator = schedule.NewCoordinator(ctx, cluster, nil) // Put 4 stores. for _, store := range newTestStores(4, "2.0.0") { @@ -436,7 +436,7 @@ func TestUpStore(t *testing.T) { _, opt, err := newTestScheduleConfig() re.NoError(err) - cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) + cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend()) cluster.coordinator = schedule.NewCoordinator(ctx, cluster, nil) cluster.ruleManager = placement.NewRuleManager(storage.NewStorageWithMemoryBackend(), cluster, cluster.GetOpts()) if opt.IsPlacementRulesEnabled() { @@ -481,7 +481,7 @@ func TestRemovingProcess(t *testing.T) { _, opt, err := newTestScheduleConfig() re.NoError(err) - cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) + cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend()) cluster.coordinator = schedule.NewCoordinator(ctx, cluster, nil) cluster.SetPrepared() @@ -539,7 +539,7 @@ func TestDeleteStoreUpdatesClusterVersion(t *testing.T) { _, opt, err := newTestScheduleConfig() re.NoError(err) - cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) + cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend()) cluster.coordinator = schedule.NewCoordinator(ctx, cluster, nil) cluster.ruleManager = placement.NewRuleManager(storage.NewStorageWithMemoryBackend(), cluster, cluster.GetOpts()) if opt.IsPlacementRulesEnabled() { @@ -574,7 +574,7 @@ func TestStoreClusterVersion(t *testing.T) { _, opt, err := newTestScheduleConfig() re.NoError(err) - cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) + cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend()) stores := newTestStores(3, "5.0.0") s1, s2, s3 := stores[0].GetMeta(), stores[1].GetMeta(), stores[2].GetMeta() s1.Version = "5.0.1" @@ -599,7 +599,7 @@ func TestRegionHeartbeatHotStat(t *testing.T) { _, opt, err := newTestScheduleConfig() re.NoError(err) - cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) + cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend()) cluster.coordinator = schedule.NewCoordinator(ctx, cluster, nil) newTestStores(4, "2.0.0") peers := []*metapb.Peer{ @@ -661,7 +661,7 @@ func TestBucketHeartbeat(t *testing.T) { _, opt, err := newTestScheduleConfig() re.NoError(err) - cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) + cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend()) cluster.coordinator = schedule.NewCoordinator(ctx, cluster, nil) // case1: region is not exist @@ -718,7 +718,7 @@ func TestRegionHeartbeat(t *testing.T) { _, opt, err := newTestScheduleConfig() re.NoError(err) - cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) + cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend()) cluster.coordinator = schedule.NewCoordinator(ctx, cluster, nil) n, np := uint64(3), uint64(3) cluster.wg.Add(1) @@ -963,7 +963,7 @@ func TestRegionFlowChanged(t *testing.T) { _, opt, err := newTestScheduleConfig() re.NoError(err) - cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) + cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend()) cluster.coordinator = schedule.NewCoordinator(ctx, cluster, nil) regions := []*core.RegionInfo{core.NewTestRegionInfo(1, 1, []byte{}, []byte{})} processRegions := func(regions []*core.RegionInfo) { @@ -988,7 +988,7 @@ func TestRegionSizeChanged(t *testing.T) { _, opt, err := newTestScheduleConfig() re.NoError(err) - cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) + cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend()) cluster.coordinator = schedule.NewCoordinator(ctx, cluster, nil) cluster.regionStats = statistics.NewRegionStatistics( cluster.GetBasicCluster(), @@ -1034,7 +1034,7 @@ func TestConcurrentReportBucket(t *testing.T) { _, opt, err := newTestScheduleConfig() re.NoError(err) - cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) + cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend()) cluster.coordinator = schedule.NewCoordinator(ctx, cluster, nil) regions := []*core.RegionInfo{core.NewTestRegionInfo(1, 1, []byte{}, []byte{})} @@ -1064,7 +1064,7 @@ func TestConcurrentRegionHeartbeat(t *testing.T) { _, opt, err := newTestScheduleConfig() re.NoError(err) - cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) + cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend()) cluster.coordinator = schedule.NewCoordinator(ctx, cluster, nil) regions := []*core.RegionInfo{core.NewTestRegionInfo(1, 1, []byte{}, []byte{})} @@ -1105,7 +1105,7 @@ func TestRegionLabelIsolationLevel(t *testing.T) { cfg.LocationLabels = []string{"zone"} opt.SetReplicationConfig(cfg) re.NoError(err) - cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) + cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend()) for i := uint64(1); i <= 4; i++ { var labels []*metapb.StoreLabel @@ -1184,7 +1184,7 @@ func TestHeartbeatSplit(t *testing.T) { _, opt, err := newTestScheduleConfig() re.NoError(err) - cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) + cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend()) cluster.coordinator = schedule.NewCoordinator(ctx, cluster, nil) // 1: [nil, nil) @@ -1228,7 +1228,7 @@ func TestRegionSplitAndMerge(t *testing.T) { _, opt, err := newTestScheduleConfig() re.NoError(err) - cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) + cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend()) cluster.coordinator = schedule.NewCoordinator(ctx, cluster, nil) regions := []*core.RegionInfo{core.NewTestRegionInfo(1, 1, []byte{}, []byte{})} @@ -1266,7 +1266,7 @@ func TestOfflineAndMerge(t *testing.T) { _, opt, err := newTestScheduleConfig() re.NoError(err) - cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) + cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend()) cluster.coordinator = schedule.NewCoordinator(ctx, cluster, nil) cluster.ruleManager = placement.NewRuleManager(storage.NewStorageWithMemoryBackend(), cluster, cluster.GetOpts()) if opt.IsPlacementRulesEnabled() { @@ -1634,7 +1634,7 @@ func TestCalculateStoreSize1(t *testing.T) { cfg := opt.GetReplicationConfig() cfg.EnablePlacementRules = true opt.SetReplicationConfig(cfg) - cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) + cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend()) cluster.coordinator = schedule.NewCoordinator(ctx, cluster, nil) cluster.regionStats = statistics.NewRegionStatistics( cluster.GetBasicCluster(), @@ -1720,7 +1720,7 @@ func TestCalculateStoreSize2(t *testing.T) { cfg.EnablePlacementRules = true opt.SetReplicationConfig(cfg) opt.SetMaxReplicas(3) - cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) + cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend()) cluster.coordinator = schedule.NewCoordinator(ctx, cluster, nil) cluster.regionStats = statistics.NewRegionStatistics( cluster.GetBasicCluster(), @@ -1829,7 +1829,7 @@ func Test(t *testing.T) { regions := newTestRegions(n, n, np) _, opts, err := newTestScheduleConfig() re.NoError(err) - tc := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opts, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) + tc := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opts, storage.NewStorageWithMemoryBackend()) cache := tc.core for i := uint64(0); i < n; i++ { @@ -1943,7 +1943,7 @@ func TestAwakenStore(t *testing.T) { _, opt, err := newTestScheduleConfig() re.NoError(err) - cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) + cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend()) n := uint64(3) stores := newTestStores(n, "6.5.0") re.True(stores[0].NeedAwakenStore()) @@ -1997,7 +1997,7 @@ func TestUpdateAndDeleteLabel(t *testing.T) { _, opt, err := newTestScheduleConfig() re.NoError(err) - cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) + cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend()) stores := newTestStores(1, "6.5.1") for _, store := range stores { re.NoError(cluster.PutStore(store.GetMeta())) @@ -2115,7 +2115,7 @@ func newTestScheduleConfig() (*sc.ScheduleConfig, *config.PersistOptions, error) } func newTestCluster(ctx context.Context, opt *config.PersistOptions) *testCluster { - rc := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) + rc := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend()) storage := storage.NewStorageWithMemoryBackend() rc.regionLabeler, _ = labeler.NewRegionLabeler(ctx, storage, time.Second*5) @@ -2127,10 +2127,9 @@ func newTestRaftCluster( id id.Allocator, opt *config.PersistOptions, s storage.Storage, - basicCluster *core.BasicCluster, ) *RaftCluster { - rc := &RaftCluster{serverCtx: ctx} - rc.InitCluster(id, opt, s, basicCluster, nil, nil) + rc := &RaftCluster{serverCtx: ctx, core: core.NewBasicCluster(), storage: s} + rc.InitCluster(id, opt, nil, nil) rc.ruleManager = placement.NewRuleManager(storage.NewStorageWithMemoryBackend(), rc, opt) if opt.IsPlacementRulesEnabled() { err := rc.ruleManager.Initialize(opt.GetMaxReplicas(), opt.GetLocationLabels(), opt.GetIsolationLevel()) diff --git a/server/cluster/cluster_worker_test.go b/server/cluster/cluster_worker_test.go index b376b38edc3..afc979e2b97 100644 --- a/server/cluster/cluster_worker_test.go +++ b/server/cluster/cluster_worker_test.go @@ -21,7 +21,6 @@ import ( "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/kvproto/pkg/pdpb" "github.com/stretchr/testify/require" - "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/mock/mockid" "github.com/tikv/pd/pkg/storage" ) @@ -33,7 +32,7 @@ func TestReportSplit(t *testing.T) { _, opt, err := newTestScheduleConfig() re.NoError(err) - cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) + cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend()) left := &metapb.Region{Id: 1, StartKey: []byte("a"), EndKey: []byte("b")} right := &metapb.Region{Id: 2, StartKey: []byte("b"), EndKey: []byte("c")} _, err = cluster.HandleReportSplit(&pdpb.ReportSplitRequest{Left: left, Right: right}) @@ -49,7 +48,7 @@ func TestReportBatchSplit(t *testing.T) { _, opt, err := newTestScheduleConfig() re.NoError(err) - cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) + cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend()) regions := []*metapb.Region{ {Id: 1, StartKey: []byte(""), EndKey: []byte("a")}, {Id: 2, StartKey: []byte("a"), EndKey: []byte("b")}, diff --git a/server/server.go b/server/server.go index d4b40af9c18..76893c24388 100644 --- a/server/server.go +++ b/server/server.go @@ -473,7 +473,7 @@ func (s *Server) startServer(ctx context.Context) error { s.gcSafePointManager = gc.NewSafePointManager(s.storage, s.cfg.PDServerCfg) s.basicCluster = core.NewBasicCluster() - s.cluster = cluster.NewRaftCluster(ctx, s.clusterID, syncer.NewRegionSyncer(s), s.client, s.httpClient) + s.cluster = cluster.NewRaftCluster(ctx, s.clusterID, s.GetBasicCluster(), s.GetStorage(), syncer.NewRegionSyncer(s), s.client, s.httpClient) keyspaceIDAllocator := id.NewAllocator(&id.AllocatorParams{ Client: s.client, RootPath: s.rootPath, diff --git a/tests/server/cluster/cluster_test.go b/tests/server/cluster/cluster_test.go index b7a428e3683..ccb469c04cb 100644 --- a/tests/server/cluster/cluster_test.go +++ b/tests/server/cluster/cluster_test.go @@ -812,10 +812,10 @@ func TestLoadClusterInfo(t *testing.T) { tc.WaitLeader() leaderServer := tc.GetLeaderServer() svr := leaderServer.GetServer() - rc := cluster.NewRaftCluster(ctx, svr.ClusterID(), syncer.NewRegionSyncer(svr), svr.GetClient(), svr.GetHTTPClient()) + rc := cluster.NewRaftCluster(ctx, svr.ClusterID(), svr.GetBasicCluster(), svr.GetStorage(), syncer.NewRegionSyncer(svr), svr.GetClient(), svr.GetHTTPClient()) // Cluster is not bootstrapped. - rc.InitCluster(svr.GetAllocator(), svr.GetPersistOptions(), svr.GetStorage(), svr.GetBasicCluster(), svr.GetHBStreams(), svr.GetKeyspaceGroupManager()) + rc.InitCluster(svr.GetAllocator(), svr.GetPersistOptions(), svr.GetHBStreams(), svr.GetKeyspaceGroupManager()) raftCluster, err := rc.LoadClusterInfo() re.NoError(err) re.Nil(raftCluster) @@ -852,8 +852,8 @@ func TestLoadClusterInfo(t *testing.T) { } re.NoError(testStorage.Flush()) - raftCluster = cluster.NewRaftCluster(ctx, svr.ClusterID(), syncer.NewRegionSyncer(svr), svr.GetClient(), svr.GetHTTPClient()) - raftCluster.InitCluster(mockid.NewIDAllocator(), svr.GetPersistOptions(), testStorage, basicCluster, svr.GetHBStreams(), svr.GetKeyspaceGroupManager()) + raftCluster = cluster.NewRaftCluster(ctx, svr.ClusterID(), basicCluster, testStorage, syncer.NewRegionSyncer(svr), svr.GetClient(), svr.GetHTTPClient()) + raftCluster.InitCluster(mockid.NewIDAllocator(), svr.GetPersistOptions(), svr.GetHBStreams(), svr.GetKeyspaceGroupManager()) raftCluster, err = raftCluster.LoadClusterInfo() re.NoError(err) re.NotNil(raftCluster) @@ -1560,8 +1560,8 @@ func TestTransferLeaderBack(t *testing.T) { tc.WaitLeader() leaderServer := tc.GetLeaderServer() svr := leaderServer.GetServer() - rc := cluster.NewRaftCluster(ctx, svr.ClusterID(), syncer.NewRegionSyncer(svr), svr.GetClient(), svr.GetHTTPClient()) - rc.InitCluster(svr.GetAllocator(), svr.GetPersistOptions(), svr.GetStorage(), svr.GetBasicCluster(), svr.GetHBStreams(), svr.GetKeyspaceGroupManager()) + rc := cluster.NewRaftCluster(ctx, svr.ClusterID(), svr.GetBasicCluster(), svr.GetStorage(), syncer.NewRegionSyncer(svr), svr.GetClient(), svr.GetHTTPClient()) + rc.InitCluster(svr.GetAllocator(), svr.GetPersistOptions(), svr.GetHBStreams(), svr.GetKeyspaceGroupManager()) storage := rc.GetStorage() meta := &metapb.Cluster{Id: 123} re.NoError(storage.SaveMeta(meta)) From 1e1817d0bcec05fe3349dbeb7a964663cb7fd41a Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Mon, 20 Nov 2023 15:50:41 +0800 Subject: [PATCH 026/137] mcs: support region label http interface in scheduling server (#7283) ref tikv/pd#5839 Signed-off-by: lhy1024 Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- pkg/mcs/scheduling/server/apis/v1/api.go | 167 +++++++++++++++++- pkg/schedule/handler/handler.go | 19 ++ pkg/utils/apiutil/serverapi/middleware.go | 30 +++- pkg/utils/testutil/api_check.go | 2 +- server/api/region_label.go | 2 +- server/api/server.go | 21 +++ tests/integrations/mcs/scheduling/api_test.go | 23 +++ tests/server/api/rule_test.go | 73 ++++++-- 8 files changed, 310 insertions(+), 27 deletions(-) diff --git a/pkg/mcs/scheduling/server/apis/v1/api.go b/pkg/mcs/scheduling/server/apis/v1/api.go index 172515d8620..822b4164da1 100644 --- a/pkg/mcs/scheduling/server/apis/v1/api.go +++ b/pkg/mcs/scheduling/server/apis/v1/api.go @@ -17,6 +17,7 @@ package apis import ( "encoding/hex" "net/http" + "net/url" "strconv" "sync" @@ -191,12 +192,22 @@ func (s *Service) RegisterConfigRouter() { placementRule := router.Group("placement-rule") placementRule.GET("", getPlacementRules) placementRule.GET("/:group", getPlacementRuleByGroup) + + regionLabel := router.Group("region-label") + regionLabel.GET("/rules", getAllRegionLabelRules) + regionLabel.GET("/rules/ids", getRegionLabelRulesByIDs) + regionLabel.GET("/rules/:id", getRegionLabelRuleByID) + + regions := router.Group("regions") + regions.GET("/:id/label/:key", getRegionLabelByKey) + regions.GET("/:id/labels", getRegionLabels) } // @Tags admin // @Summary Change the log level. // @Produce json // @Success 200 {string} string "The log level is updated." +// @Failure 400 {string} string "The input is invalid." // @Router /admin/log [put] func changeLogLevel(c *gin.Context) { svr := c.MustGet(multiservicesapi.ServiceContextKey).(*scheserver.Server) @@ -230,6 +241,7 @@ func getConfig(c *gin.Context) { // @Summary Drop all regions from cache. // @Produce json // @Success 200 {string} string "All regions are removed from server cache." +// @Failure 500 {string} string "PD server failed to proceed the request." // @Router /admin/cache/regions [delete] func deleteAllRegionCache(c *gin.Context) { svr := c.MustGet(multiservicesapi.ServiceContextKey).(*scheserver.Server) @@ -248,6 +260,7 @@ func deleteAllRegionCache(c *gin.Context) { // @Produce json // @Success 200 {string} string "The region is removed from server cache." // @Failure 400 {string} string "The input is invalid." +// @Failure 500 {string} string "PD server failed to proceed the request." // @Router /admin/cache/regions/{id} [delete] func deleteRegionCacheByID(c *gin.Context) { svr := c.MustGet(multiservicesapi.ServiceContextKey).(*scheserver.Server) @@ -683,8 +696,6 @@ func getHotBuckets(c *gin.Context) { // @Accept json // @Produce json // @Success 200 {object} storage.HistoryHotRegions -// @Failure 400 {string} string "The input is invalid." -// @Failure 500 {string} string "PD server failed to proceed the request." // @Router /hotspot/regions/history [get] func getHistoryHotRegions(c *gin.Context) { // TODO: support history hotspot in scheduling server with stateless in the future. @@ -955,3 +966,155 @@ func getPlacementRuleByGroup(c *gin.Context) { group := manager.GetGroupBundle(g) c.IndentedJSON(http.StatusOK, group) } + +// @Tags region_label +// @Summary Get label of a region. +// @Param id path integer true "Region Id" +// @Param key path string true "Label key" +// @Produce json +// @Success 200 {string} string +// @Failure 400 {string} string "The input is invalid." +// @Failure 404 {string} string "The region does not exist." +// @Failure 500 {string} string "PD server failed to proceed the request." +// @Router /config/regions/{id}/label/{key} [get] +func getRegionLabelByKey(c *gin.Context) { + handler := c.MustGet(handlerKey).(*handler.Handler) + + idStr := c.Param("id") + labelKey := c.Param("key") // TODO: test https://github.com/tikv/pd/pull/4004 + + id, err := strconv.ParseUint(idStr, 10, 64) + if err != nil { + c.String(http.StatusBadRequest, err.Error()) + return + } + + region, err := handler.GetRegion(id) + if err != nil { + c.String(http.StatusInternalServerError, err.Error()) + return + } + if region == nil { + c.String(http.StatusNotFound, errs.ErrRegionNotFound.FastGenByArgs().Error()) + return + } + + l, err := handler.GetRegionLabeler() + if err != nil { + c.String(http.StatusInternalServerError, err.Error()) + return + } + labelValue := l.GetRegionLabel(region, labelKey) + c.IndentedJSON(http.StatusOK, labelValue) +} + +// @Tags region_label +// @Summary Get labels of a region. +// @Param id path integer true "Region Id" +// @Produce json +// @Success 200 {string} string +// @Failure 400 {string} string "The input is invalid." +// @Failure 404 {string} string "The region does not exist." +// @Router /config/regions/{id}/labels [get] +func getRegionLabels(c *gin.Context) { + handler := c.MustGet(handlerKey).(*handler.Handler) + + idStr := c.Param("id") + id, err := strconv.ParseUint(idStr, 10, 64) + if err != nil { + c.String(http.StatusBadRequest, err.Error()) + return + } + + region, err := handler.GetRegion(id) + if err != nil { + c.String(http.StatusInternalServerError, err.Error()) + return + } + if region == nil { + c.String(http.StatusNotFound, errs.ErrRegionNotFound.FastGenByArgs().Error()) + return + } + l, err := handler.GetRegionLabeler() + if err != nil { + c.String(http.StatusInternalServerError, err.Error()) + return + } + labels := l.GetRegionLabels(region) + c.IndentedJSON(http.StatusOK, labels) +} + +// @Tags region_label +// @Summary List all label rules of cluster. +// @Produce json +// @Success 200 {array} labeler.LabelRule +// @Failure 500 {string} string "PD server failed to proceed the request." +// @Router /config/region-label/rules [get] +func getAllRegionLabelRules(c *gin.Context) { + handler := c.MustGet(handlerKey).(*handler.Handler) + l, err := handler.GetRegionLabeler() + if err != nil { + c.String(http.StatusInternalServerError, err.Error()) + return + } + rules := l.GetAllLabelRules() + c.IndentedJSON(http.StatusOK, rules) +} + +// @Tags region_label +// @Summary Get label rules of cluster by ids. +// @Param body body []string true "IDs of query rules" +// @Produce json +// @Success 200 {array} labeler.LabelRule +// @Failure 400 {string} string "The input is invalid." +// @Failure 500 {string} string "PD server failed to proceed the request." +// @Router /config/region-label/rules/ids [get] +func getRegionLabelRulesByIDs(c *gin.Context) { + handler := c.MustGet(handlerKey).(*handler.Handler) + l, err := handler.GetRegionLabeler() + if err != nil { + c.String(http.StatusInternalServerError, err.Error()) + return + } + var ids []string + if err := c.BindJSON(&ids); err != nil { + c.String(http.StatusBadRequest, err.Error()) + return + } + rules, err := l.GetLabelRules(ids) + if err != nil { + c.String(http.StatusInternalServerError, err.Error()) + return + } + c.IndentedJSON(http.StatusOK, rules) +} + +// @Tags region_label +// @Summary Get label rule of cluster by id. +// @Param id path string true "Rule Id" +// @Produce json +// @Success 200 {object} labeler.LabelRule +// @Failure 404 {string} string "The rule does not exist." +// @Failure 500 {string} string "PD server failed to proceed the request." +// @Router /config/region-label/rules/{id} [get] +func getRegionLabelRuleByID(c *gin.Context) { + handler := c.MustGet(handlerKey).(*handler.Handler) + + id, err := url.PathUnescape(c.Param("id")) + if err != nil { + c.String(http.StatusBadRequest, err.Error()) + return + } + + l, err := handler.GetRegionLabeler() + if err != nil { + c.String(http.StatusInternalServerError, err.Error()) + return + } + rule := l.GetLabelRule(id) + if rule == nil { + c.String(http.StatusNotFound, errs.ErrRegionRuleNotFound.FastGenByArgs().Error()) + return + } + c.IndentedJSON(http.StatusOK, rule) +} diff --git a/pkg/schedule/handler/handler.go b/pkg/schedule/handler/handler.go index 3f9f4f96622..8da84647f77 100644 --- a/pkg/schedule/handler/handler.go +++ b/pkg/schedule/handler/handler.go @@ -31,6 +31,7 @@ import ( "github.com/tikv/pd/pkg/schedule" sche "github.com/tikv/pd/pkg/schedule/core" "github.com/tikv/pd/pkg/schedule/filter" + "github.com/tikv/pd/pkg/schedule/labeler" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/placement" "github.com/tikv/pd/pkg/schedule/scatter" @@ -1063,6 +1064,24 @@ func (h *Handler) GetHotBuckets(regionIDs ...uint64) (HotBucketsResponse, error) return ret, nil } +// GetRegion returns the region labeler. +func (h *Handler) GetRegion(id uint64) (*core.RegionInfo, error) { + c := h.GetCluster() + if c == nil { + return nil, errs.ErrNotBootstrapped.GenWithStackByArgs() + } + return c.GetRegion(id), nil +} + +// GetRegionLabeler returns the region labeler. +func (h *Handler) GetRegionLabeler() (*labeler.RegionLabeler, error) { + c := h.GetCluster() + if c == nil || c.GetRegionLabeler() == nil { + return nil, errs.ErrNotBootstrapped + } + return c.GetRegionLabeler(), nil +} + // GetRuleManager returns the rule manager. func (h *Handler) GetRuleManager() (*placement.RuleManager, error) { c := h.GetCluster() diff --git a/pkg/utils/apiutil/serverapi/middleware.go b/pkg/utils/apiutil/serverapi/middleware.go index 2bb742ccbba..e26327cb3ff 100644 --- a/pkg/utils/apiutil/serverapi/middleware.go +++ b/pkg/utils/apiutil/serverapi/middleware.go @@ -79,6 +79,7 @@ type microserviceRedirectRule struct { targetPath string targetServiceName string matchMethods []string + filter func(*http.Request) bool } // NewRedirector redirects request to the leader if needs to be handled in the leader. @@ -94,14 +95,19 @@ func NewRedirector(s *server.Server, opts ...RedirectorOption) negroni.Handler { type RedirectorOption func(*redirector) // MicroserviceRedirectRule new a microservice redirect rule option -func MicroserviceRedirectRule(matchPath, targetPath, targetServiceName string, methods []string) RedirectorOption { +func MicroserviceRedirectRule(matchPath, targetPath, targetServiceName string, + methods []string, filters ...func(*http.Request) bool) RedirectorOption { return func(s *redirector) { - s.microserviceRedirectRules = append(s.microserviceRedirectRules, µserviceRedirectRule{ - matchPath, - targetPath, - targetServiceName, - methods, - }) + rule := µserviceRedirectRule{ + matchPath: matchPath, + targetPath: targetPath, + targetServiceName: targetServiceName, + matchMethods: methods, + } + if len(filters) > 0 { + rule.filter = filters[0] + } + s.microserviceRedirectRules = append(s.microserviceRedirectRules, rule) } } @@ -117,18 +123,26 @@ func (h *redirector) matchMicroServiceRedirectRules(r *http.Request) (bool, stri r.URL.Path = strings.TrimRight(r.URL.Path, "/") for _, rule := range h.microserviceRedirectRules { if strings.HasPrefix(r.URL.Path, rule.matchPath) && slice.Contains(rule.matchMethods, r.Method) { + if rule.filter != nil && !rule.filter(r) { + continue + } origin := r.URL.Path addr, ok := h.s.GetServicePrimaryAddr(r.Context(), rule.targetServiceName) if !ok || addr == "" { log.Warn("failed to get the service primary addr when trying to match redirect rules", zap.String("path", r.URL.Path)) } + // If the URL contains escaped characters, use RawPath instead of Path + path := r.URL.Path + if r.URL.RawPath != "" { + path = r.URL.RawPath + } // Extract parameters from the URL path // e.g. r.URL.Path = /pd/api/v1/operators/1 (before redirect) // matchPath = /pd/api/v1/operators // targetPath = /scheduling/api/v1/operators // r.URL.Path = /scheduling/api/v1/operator/1 (after redirect) - pathParams := strings.TrimPrefix(r.URL.Path, rule.matchPath) + pathParams := strings.TrimPrefix(path, rule.matchPath) pathParams = strings.Trim(pathParams, "/") // Remove leading and trailing '/' if len(pathParams) > 0 { r.URL.Path = rule.targetPath + "/" + pathParams diff --git a/pkg/utils/testutil/api_check.go b/pkg/utils/testutil/api_check.go index ea91654b149..58934bf08f6 100644 --- a/pkg/utils/testutil/api_check.go +++ b/pkg/utils/testutil/api_check.go @@ -88,7 +88,7 @@ func ReadGetJSON(re *require.Assertions, client *http.Client, url string, data i } // ReadGetJSONWithBody is used to do get request with input and check whether given data can be extracted successfully. -func ReadGetJSONWithBody(re *require.Assertions, client *http.Client, url string, input []byte, data interface{}) error { +func ReadGetJSONWithBody(re *require.Assertions, client *http.Client, url string, input []byte, data interface{}, checkOpts ...func([]byte, int, http.Header)) error { resp, err := apiutil.GetJSON(client, url, input) if err != nil { return err diff --git a/server/api/region_label.go b/server/api/region_label.go index 003dfb1132f..7958bacd371 100644 --- a/server/api/region_label.go +++ b/server/api/region_label.go @@ -83,7 +83,7 @@ func (h *regionLabelHandler) PatchRegionLabelRules(w http.ResponseWriter, r *htt // @Success 200 {array} labeler.LabelRule // @Failure 400 {string} string "The input is invalid." // @Failure 500 {string} string "PD server failed to proceed the request." -// @Router /config/region-label/rule/ids [get] +// @Router /config/region-label/rules/ids [get] func (h *regionLabelHandler) GetRegionLabelRulesByIDs(w http.ResponseWriter, r *http.Request) { cluster := getCluster(r) var ids []string diff --git a/server/api/server.go b/server/api/server.go index 77a51eb04e5..2c015bec7ac 100644 --- a/server/api/server.go +++ b/server/api/server.go @@ -17,6 +17,7 @@ package api import ( "context" "net/http" + "strings" "github.com/gorilla/mux" scheapi "github.com/tikv/pd/pkg/mcs/scheduling/server/apis/v1" @@ -79,6 +80,26 @@ func NewHandler(_ context.Context, svr *server.Server) (http.Handler, apiutil.AP scheapi.APIPathPrefix+"/checkers", mcs.SchedulingServiceName, []string{http.MethodPost, http.MethodGet}), + serverapi.MicroserviceRedirectRule( + prefix+"/region/id", + scheapi.APIPathPrefix+"/config/regions", + mcs.SchedulingServiceName, + []string{http.MethodGet}, + func(r *http.Request) bool { + // The original code uses the path "/region/id" to get the region id. + // However, the path "/region/id" is used to get the region by id, which is not what we want. + return strings.Contains(r.URL.Path, "label") + }), + serverapi.MicroserviceRedirectRule( + prefix+"/config/region-label/rules", + scheapi.APIPathPrefix+"/config/region-label/rules", + mcs.SchedulingServiceName, + []string{http.MethodGet}), + serverapi.MicroserviceRedirectRule( + prefix+"/config/region-label/rule/", // Note: this is a typo in the original code + scheapi.APIPathPrefix+"/config/region-label/rules", + mcs.SchedulingServiceName, + []string{http.MethodGet}), serverapi.MicroserviceRedirectRule( prefix+"/hotspot", scheapi.APIPathPrefix+"/hotspot", diff --git a/tests/integrations/mcs/scheduling/api_test.go b/tests/integrations/mcs/scheduling/api_test.go index f6a7f66a66f..b07568580d7 100644 --- a/tests/integrations/mcs/scheduling/api_test.go +++ b/tests/integrations/mcs/scheduling/api_test.go @@ -15,6 +15,7 @@ import ( _ "github.com/tikv/pd/pkg/mcs/scheduling/server/apis/v1" "github.com/tikv/pd/pkg/mcs/scheduling/server/config" "github.com/tikv/pd/pkg/schedule/handler" + "github.com/tikv/pd/pkg/schedule/labeler" "github.com/tikv/pd/pkg/schedule/placement" "github.com/tikv/pd/pkg/statistics" "github.com/tikv/pd/pkg/storage" @@ -236,6 +237,28 @@ func (suite *apiTestSuite) TestAPIForward() { testutil.WithHeader(re, apiutil.ForwardToMicroServiceHeader, "true")) re.NoError(err) + // Test region label + var labelRules []*labeler.LabelRule + err = testutil.ReadGetJSON(re, testDialClient, fmt.Sprintf("%s/%s", urlPrefix, "config/region-label/rules"), &labelRules, + testutil.WithHeader(re, apiutil.ForwardToMicroServiceHeader, "true")) + re.NoError(err) + err = testutil.ReadGetJSONWithBody(re, testDialClient, fmt.Sprintf("%s/%s", urlPrefix, "config/region-label/rules/ids"), []byte(`["rule1", "rule3"]`), + &labelRules, testutil.WithHeader(re, apiutil.ForwardToMicroServiceHeader, "true")) + re.NoError(err) + err = testutil.CheckGetJSON(testDialClient, fmt.Sprintf("%s/%s", urlPrefix, "config/region-label/rule/rule1"), nil, + testutil.StatusNotOK(re), testutil.WithHeader(re, apiutil.ForwardToMicroServiceHeader, "true")) + re.NoError(err) + + err = testutil.CheckGetJSON(testDialClient, fmt.Sprintf("%s/%s", urlPrefix, "region/id/1"), nil, + testutil.WithoutHeader(re, apiutil.ForwardToMicroServiceHeader)) + re.NoError(err) + err = testutil.CheckGetJSON(testDialClient, fmt.Sprintf("%s/%s", urlPrefix, "region/id/1/label/key"), nil, + testutil.WithHeader(re, apiutil.ForwardToMicroServiceHeader, "true")) + re.NoError(err) + err = testutil.CheckGetJSON(testDialClient, fmt.Sprintf("%s/%s", urlPrefix, "region/id/1/labels"), nil, + testutil.WithHeader(re, apiutil.ForwardToMicroServiceHeader, "true")) + re.NoError(err) + // Test rules: only forward `GET` request var rules []*placement.Rule tests.MustPutRegion(re, suite.cluster, 2, 1, []byte("a"), []byte("b"), core.SetApproximateSize(60)) diff --git a/tests/server/api/rule_test.go b/tests/server/api/rule_test.go index 861fbe5cf32..9176a00e66d 100644 --- a/tests/server/api/rule_test.go +++ b/tests/server/api/rule_test.go @@ -26,6 +26,7 @@ import ( "github.com/stretchr/testify/suite" "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/errs" + "github.com/tikv/pd/pkg/schedule/labeler" "github.com/tikv/pd/pkg/schedule/placement" tu "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/server/config" @@ -1008,8 +1009,7 @@ func (suite *regionRuleTestSuite) TestRegionPlacementRule() { }, } env := tests.NewSchedulingTestEnvironment(suite.T(), opts...) - // FIXME: enable this test in two modes after we support region label forward. - env.RunTestInPDMode(suite.checkRegionPlacementRule) + env.RunTestInTwoModes(suite.checkRegionPlacementRule) } func (suite *regionRuleTestSuite) checkRegionPlacementRule(cluster *tests.TestCluster) { @@ -1076,37 +1076,80 @@ func (suite *regionRuleTestSuite) checkRegionPlacementRule(cluster *tests.TestCl }) fit := &placement.RegionFit{} - url := fmt.Sprintf("%s/config/rules/region/%d/detail", urlPrefix, 1) - err := tu.ReadGetJSON(re, testDialClient, url, fit) + u := fmt.Sprintf("%s/config/rules/region/%d/detail", urlPrefix, 1) + err := tu.ReadGetJSON(re, testDialClient, u, fit) suite.NoError(err) suite.Equal(len(fit.RuleFits), 1) suite.Equal(len(fit.OrphanPeers), 1) - url = fmt.Sprintf("%s/config/rules/region/%d/detail", urlPrefix, 2) + u = fmt.Sprintf("%s/config/rules/region/%d/detail", urlPrefix, 2) fit = &placement.RegionFit{} - err = tu.ReadGetJSON(re, testDialClient, url, fit) + err = tu.ReadGetJSON(re, testDialClient, u, fit) suite.NoError(err) suite.Equal(len(fit.RuleFits), 2) suite.Equal(len(fit.OrphanPeers), 0) - url = fmt.Sprintf("%s/config/rules/region/%d/detail", urlPrefix, 3) + u = fmt.Sprintf("%s/config/rules/region/%d/detail", urlPrefix, 3) fit = &placement.RegionFit{} - err = tu.ReadGetJSON(re, testDialClient, url, fit) + err = tu.ReadGetJSON(re, testDialClient, u, fit) suite.NoError(err) suite.Equal(len(fit.RuleFits), 0) suite.Equal(len(fit.OrphanPeers), 2) - url = fmt.Sprintf("%s/config/rules/region/%d/detail", urlPrefix, 4) - err = tu.CheckGetJSON(testDialClient, url, nil, tu.Status(re, http.StatusNotFound), tu.StringContain( + var label labeler.LabelRule + escapedID := url.PathEscape("keyspaces/0") + u = fmt.Sprintf("%s/config/region-label/rule/%s", urlPrefix, escapedID) + fmt.Println("u====", u) + err = tu.ReadGetJSON(re, testDialClient, u, &label) + suite.NoError(err) + suite.Equal(label.ID, "keyspaces/0") + + var labels []labeler.LabelRule + u = fmt.Sprintf("%s/config/region-label/rules", urlPrefix) + err = tu.ReadGetJSON(re, testDialClient, u, &labels) + suite.NoError(err) + suite.Len(labels, 1) + suite.Equal(labels[0].ID, "keyspaces/0") + + u = fmt.Sprintf("%s/config/region-label/rules/ids", urlPrefix) + err = tu.CheckGetJSON(testDialClient, u, []byte(`["rule1", "rule3"]`), func(resp []byte, statusCode int, _ http.Header) { + err := json.Unmarshal(resp, &labels) + suite.NoError(err) + suite.Len(labels, 0) + }) + suite.NoError(err) + + err = tu.CheckGetJSON(testDialClient, u, []byte(`["keyspaces/0"]`), func(resp []byte, statusCode int, _ http.Header) { + err := json.Unmarshal(resp, &labels) + suite.NoError(err) + suite.Len(labels, 1) + suite.Equal(labels[0].ID, "keyspaces/0") + }) + suite.NoError(err) + + u = fmt.Sprintf("%s/config/rules/region/%d/detail", urlPrefix, 4) + err = tu.CheckGetJSON(testDialClient, u, nil, tu.Status(re, http.StatusNotFound), tu.StringContain( re, "region 4 not found")) suite.NoError(err) - url = fmt.Sprintf("%s/config/rules/region/%s/detail", urlPrefix, "id") - err = tu.CheckGetJSON(testDialClient, url, nil, tu.Status(re, http.StatusBadRequest), tu.StringContain( + u = fmt.Sprintf("%s/config/rules/region/%s/detail", urlPrefix, "id") + err = tu.CheckGetJSON(testDialClient, u, nil, tu.Status(re, http.StatusBadRequest), tu.StringContain( re, errs.ErrRegionInvalidID.Error())) suite.NoError(err) - leaderServer.GetRaftCluster().GetReplicationConfig().EnablePlacementRules = false - url = fmt.Sprintf("%s/config/rules/region/%d/detail", urlPrefix, 1) - err = tu.CheckGetJSON(testDialClient, url, nil, tu.Status(re, http.StatusPreconditionFailed), tu.StringContain( + data := make(map[string]interface{}) + data["enable-placement-rules"] = "false" + reqData, e := json.Marshal(data) + re.NoError(e) + u = fmt.Sprintf("%s/config", urlPrefix) + err = tu.CheckPostJSON(testDialClient, u, reqData, tu.StatusOK(re)) + re.NoError(err) + if sche := cluster.GetSchedulingPrimaryServer(); sche != nil { + // wait for the scheduler server to update the config + tu.Eventually(re, func() bool { + return !sche.GetCluster().GetCheckerConfig().IsPlacementRulesEnabled() + }) + } + u = fmt.Sprintf("%s/config/rules/region/%d/detail", urlPrefix, 1) + err = tu.CheckGetJSON(testDialClient, u, nil, tu.Status(re, http.StatusPreconditionFailed), tu.StringContain( re, "placement rules feature is disabled")) suite.NoError(err) } From 5a4e9efd846e0ee0d15ae4f9f91aab81f4382da3 Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Mon, 20 Nov 2023 17:46:10 +0800 Subject: [PATCH 027/137] mcs: fix scheduler memory sync in api server (#7389) close tikv/pd#7388 Signed-off-by: lhy1024 Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- pkg/mcs/scheduling/server/config/watcher.go | 3 +- pkg/schedule/schedulers/base_scheduler.go | 8 +- pkg/schedule/schedulers/evict_leader.go | 4 +- pkg/schedule/schedulers/evict_slow_store.go | 4 +- .../schedulers/evict_slow_store_test.go | 4 +- pkg/schedule/schedulers/evict_slow_trend.go | 4 +- .../schedulers/evict_slow_trend_test.go | 4 +- pkg/schedule/schedulers/grant_leader.go | 4 +- pkg/schedule/schedulers/scheduler.go | 4 +- .../schedulers/scheduler_controller.go | 8 +- plugin/scheduler_example/evict_leader.go | 4 +- tests/pdctl/scheduler/scheduler_test.go | 126 +++++++++++++----- 12 files changed, 121 insertions(+), 56 deletions(-) diff --git a/pkg/mcs/scheduling/server/config/watcher.go b/pkg/mcs/scheduling/server/config/watcher.go index 6ad37045000..433933674ea 100644 --- a/pkg/mcs/scheduling/server/config/watcher.go +++ b/pkg/mcs/scheduling/server/config/watcher.go @@ -147,7 +147,8 @@ func (cw *Watcher) initializeSchedulerConfigWatcher() error { prefixToTrim := cw.schedulerConfigPathPrefix + "/" putFn := func(kv *mvccpb.KeyValue) error { name := strings.TrimPrefix(string(kv.Key), prefixToTrim) - log.Info("update scheduler config", zap.String("name", string(kv.Value))) + log.Info("update scheduler config", zap.String("name", name), + zap.String("value", string(kv.Value))) err := cw.storage.SaveSchedulerConfig(name, kv.Value) if err != nil { log.Warn("failed to save scheduler config", diff --git a/pkg/schedule/schedulers/base_scheduler.go b/pkg/schedule/schedulers/base_scheduler.go index 6e712c18fe3..f4c8c577767 100644 --- a/pkg/schedule/schedulers/base_scheduler.go +++ b/pkg/schedule/schedulers/base_scheduler.go @@ -92,8 +92,8 @@ func (s *BaseScheduler) GetNextInterval(interval time.Duration) time.Duration { return intervalGrow(interval, MaxScheduleInterval, exponentialGrowth) } -// Prepare does some prepare work -func (s *BaseScheduler) Prepare(cluster sche.SchedulerCluster) error { return nil } +// PrepareConfig does some prepare work about config. +func (s *BaseScheduler) PrepareConfig(cluster sche.SchedulerCluster) error { return nil } -// Cleanup does some cleanup work -func (s *BaseScheduler) Cleanup(cluster sche.SchedulerCluster) {} +// CleanConfig does some cleanup work about config. +func (s *BaseScheduler) CleanConfig(cluster sche.SchedulerCluster) {} diff --git a/pkg/schedule/schedulers/evict_leader.go b/pkg/schedule/schedulers/evict_leader.go index a5c67856df8..332002043a3 100644 --- a/pkg/schedule/schedulers/evict_leader.go +++ b/pkg/schedule/schedulers/evict_leader.go @@ -239,7 +239,7 @@ func pauseAndResumeLeaderTransfer(cluster *core.BasicCluster, old, new map[uint6 } } -func (s *evictLeaderScheduler) Prepare(cluster sche.SchedulerCluster) error { +func (s *evictLeaderScheduler) PrepareConfig(cluster sche.SchedulerCluster) error { s.conf.mu.RLock() defer s.conf.mu.RUnlock() var res error @@ -251,7 +251,7 @@ func (s *evictLeaderScheduler) Prepare(cluster sche.SchedulerCluster) error { return res } -func (s *evictLeaderScheduler) Cleanup(cluster sche.SchedulerCluster) { +func (s *evictLeaderScheduler) CleanConfig(cluster sche.SchedulerCluster) { s.conf.mu.RLock() defer s.conf.mu.RUnlock() for id := range s.conf.StoreIDWithRanges { diff --git a/pkg/schedule/schedulers/evict_slow_store.go b/pkg/schedule/schedulers/evict_slow_store.go index cc1b16300c5..563f9f68c45 100644 --- a/pkg/schedule/schedulers/evict_slow_store.go +++ b/pkg/schedule/schedulers/evict_slow_store.go @@ -189,7 +189,7 @@ func (s *evictSlowStoreScheduler) EncodeConfig() ([]byte, error) { return EncodeConfig(s.conf) } -func (s *evictSlowStoreScheduler) Prepare(cluster sche.SchedulerCluster) error { +func (s *evictSlowStoreScheduler) PrepareConfig(cluster sche.SchedulerCluster) error { evictStore := s.conf.evictStore() if evictStore != 0 { return cluster.SlowStoreEvicted(evictStore) @@ -197,7 +197,7 @@ func (s *evictSlowStoreScheduler) Prepare(cluster sche.SchedulerCluster) error { return nil } -func (s *evictSlowStoreScheduler) Cleanup(cluster sche.SchedulerCluster) { +func (s *evictSlowStoreScheduler) CleanConfig(cluster sche.SchedulerCluster) { s.cleanupEvictLeader(cluster) } diff --git a/pkg/schedule/schedulers/evict_slow_store_test.go b/pkg/schedule/schedulers/evict_slow_store_test.go index 813d17ae541..11cd69e60f7 100644 --- a/pkg/schedule/schedulers/evict_slow_store_test.go +++ b/pkg/schedule/schedulers/evict_slow_store_test.go @@ -123,13 +123,13 @@ func (suite *evictSlowStoreTestSuite) TestEvictSlowStorePrepare() { suite.True(ok) suite.Zero(es2.conf.evictStore()) // prepare with no evict store. - suite.es.Prepare(suite.tc) + suite.es.PrepareConfig(suite.tc) es2.conf.setStoreAndPersist(1) suite.Equal(uint64(1), es2.conf.evictStore()) suite.False(es2.conf.readyForRecovery()) // prepare with evict store. - suite.es.Prepare(suite.tc) + suite.es.PrepareConfig(suite.tc) } func (suite *evictSlowStoreTestSuite) TestEvictSlowStorePersistFail() { diff --git a/pkg/schedule/schedulers/evict_slow_trend.go b/pkg/schedule/schedulers/evict_slow_trend.go index f31ba420c97..0d2c10e2bfe 100644 --- a/pkg/schedule/schedulers/evict_slow_trend.go +++ b/pkg/schedule/schedulers/evict_slow_trend.go @@ -270,7 +270,7 @@ func (s *evictSlowTrendScheduler) EncodeConfig() ([]byte, error) { return EncodeConfig(s.conf) } -func (s *evictSlowTrendScheduler) Prepare(cluster sche.SchedulerCluster) error { +func (s *evictSlowTrendScheduler) PrepareConfig(cluster sche.SchedulerCluster) error { evictedStoreID := s.conf.evictedStore() if evictedStoreID == 0 { return nil @@ -278,7 +278,7 @@ func (s *evictSlowTrendScheduler) Prepare(cluster sche.SchedulerCluster) error { return cluster.SlowTrendEvicted(evictedStoreID) } -func (s *evictSlowTrendScheduler) Cleanup(cluster sche.SchedulerCluster) { +func (s *evictSlowTrendScheduler) CleanConfig(cluster sche.SchedulerCluster) { s.cleanupEvictLeader(cluster) } diff --git a/pkg/schedule/schedulers/evict_slow_trend_test.go b/pkg/schedule/schedulers/evict_slow_trend_test.go index c6ad058455f..75ea50d73b4 100644 --- a/pkg/schedule/schedulers/evict_slow_trend_test.go +++ b/pkg/schedule/schedulers/evict_slow_trend_test.go @@ -255,10 +255,10 @@ func (suite *evictSlowTrendTestSuite) TestEvictSlowTrendPrepare() { suite.True(ok) suite.Zero(es2.conf.evictedStore()) // prepare with no evict store. - suite.es.Prepare(suite.tc) + suite.es.PrepareConfig(suite.tc) es2.conf.setStoreAndPersist(1) suite.Equal(uint64(1), es2.conf.evictedStore()) // prepare with evict store. - suite.es.Prepare(suite.tc) + suite.es.PrepareConfig(suite.tc) } diff --git a/pkg/schedule/schedulers/grant_leader.go b/pkg/schedule/schedulers/grant_leader.go index f244228a10f..84f830f368b 100644 --- a/pkg/schedule/schedulers/grant_leader.go +++ b/pkg/schedule/schedulers/grant_leader.go @@ -197,7 +197,7 @@ func (s *grantLeaderScheduler) ReloadConfig() error { return nil } -func (s *grantLeaderScheduler) Prepare(cluster sche.SchedulerCluster) error { +func (s *grantLeaderScheduler) PrepareConfig(cluster sche.SchedulerCluster) error { s.conf.mu.RLock() defer s.conf.mu.RUnlock() var res error @@ -209,7 +209,7 @@ func (s *grantLeaderScheduler) Prepare(cluster sche.SchedulerCluster) error { return res } -func (s *grantLeaderScheduler) Cleanup(cluster sche.SchedulerCluster) { +func (s *grantLeaderScheduler) CleanConfig(cluster sche.SchedulerCluster) { s.conf.mu.RLock() defer s.conf.mu.RUnlock() for id := range s.conf.StoreIDWithRanges { diff --git a/pkg/schedule/schedulers/scheduler.go b/pkg/schedule/schedulers/scheduler.go index 9262f7d0a65..1c788989454 100644 --- a/pkg/schedule/schedulers/scheduler.go +++ b/pkg/schedule/schedulers/scheduler.go @@ -42,8 +42,8 @@ type Scheduler interface { ReloadConfig() error GetMinInterval() time.Duration GetNextInterval(interval time.Duration) time.Duration - Prepare(cluster sche.SchedulerCluster) error - Cleanup(cluster sche.SchedulerCluster) + PrepareConfig(cluster sche.SchedulerCluster) error + CleanConfig(cluster sche.SchedulerCluster) Schedule(cluster sche.SchedulerCluster, dryRun bool) ([]*operator.Operator, []plan.Plan) IsScheduleAllowed(cluster sche.SchedulerCluster) bool } diff --git a/pkg/schedule/schedulers/scheduler_controller.go b/pkg/schedule/schedulers/scheduler_controller.go index 5097a5f3f1c..b65173c1f5b 100644 --- a/pkg/schedule/schedulers/scheduler_controller.go +++ b/pkg/schedule/schedulers/scheduler_controller.go @@ -156,7 +156,8 @@ func (c *Controller) AddSchedulerHandler(scheduler Scheduler, args ...string) er return err } c.cluster.GetSchedulerConfig().AddSchedulerCfg(scheduler.GetType(), args) - return nil + err := scheduler.PrepareConfig(c.cluster) + return err } // RemoveSchedulerHandler removes the HTTP handler for a scheduler. @@ -183,6 +184,7 @@ func (c *Controller) RemoveSchedulerHandler(name string) error { return err } + s.(Scheduler).CleanConfig(c.cluster) delete(c.schedulerHandlers, name) return nil @@ -198,7 +200,7 @@ func (c *Controller) AddScheduler(scheduler Scheduler, args ...string) error { } s := NewScheduleController(c.ctx, c.cluster, c.opController, scheduler) - if err := s.Scheduler.Prepare(c.cluster); err != nil { + if err := s.Scheduler.PrepareConfig(c.cluster); err != nil { return err } @@ -343,7 +345,7 @@ func (c *Controller) IsSchedulerExisted(name string) (bool, error) { func (c *Controller) runScheduler(s *ScheduleController) { defer logutil.LogPanic() defer c.wg.Done() - defer s.Scheduler.Cleanup(c.cluster) + defer s.Scheduler.CleanConfig(c.cluster) ticker := time.NewTicker(s.GetInterval()) defer ticker.Stop() diff --git a/plugin/scheduler_example/evict_leader.go b/plugin/scheduler_example/evict_leader.go index 8919d1bdb4b..063ae9eb150 100644 --- a/plugin/scheduler_example/evict_leader.go +++ b/plugin/scheduler_example/evict_leader.go @@ -186,7 +186,7 @@ func (s *evictLeaderScheduler) EncodeConfig() ([]byte, error) { return schedulers.EncodeConfig(s.conf) } -func (s *evictLeaderScheduler) Prepare(cluster sche.SchedulerCluster) error { +func (s *evictLeaderScheduler) PrepareConfig(cluster sche.SchedulerCluster) error { s.conf.mu.RLock() defer s.conf.mu.RUnlock() var res error @@ -198,7 +198,7 @@ func (s *evictLeaderScheduler) Prepare(cluster sche.SchedulerCluster) error { return res } -func (s *evictLeaderScheduler) Cleanup(cluster sche.SchedulerCluster) { +func (s *evictLeaderScheduler) CleanConfig(cluster sche.SchedulerCluster) { s.conf.mu.RLock() defer s.conf.mu.RUnlock() for id := range s.conf.StoreIDWitRanges { diff --git a/tests/pdctl/scheduler/scheduler_test.go b/tests/pdctl/scheduler/scheduler_test.go index d0fac2c1137..7098637c84a 100644 --- a/tests/pdctl/scheduler/scheduler_test.go +++ b/tests/pdctl/scheduler/scheduler_test.go @@ -17,6 +17,7 @@ package scheduler_test import ( "context" "encoding/json" + "fmt" "reflect" "strings" "testing" @@ -28,6 +29,7 @@ import ( "github.com/stretchr/testify/suite" "github.com/tikv/pd/pkg/core" sc "github.com/tikv/pd/pkg/schedule/config" + "github.com/tikv/pd/pkg/slice" "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/pkg/versioninfo" "github.com/tikv/pd/tests" @@ -84,7 +86,8 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) { checkSchedulerCommand := func(args []string, expected map[string]bool) { if args != nil { - mustExec(re, cmd, args, nil) + echo := mustExec(re, cmd, args, nil) + re.Contains(echo, "Success!") } testutil.Eventually(re, func() bool { var schedulers []string @@ -137,9 +140,40 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) { } checkSchedulerCommand(args, expected) - schedulers := []string{"evict-leader-scheduler", "grant-leader-scheduler"} + // avoid the influence of the scheduler order + schedulers := []string{"evict-leader-scheduler", "grant-leader-scheduler", "evict-leader-scheduler", "grant-leader-scheduler"} + + checkStorePause := func(changedStores []uint64, schedulerName string) { + status := func() string { + switch schedulerName { + case "evict-leader-scheduler": + return "paused" + case "grant-leader-scheduler": + return "resumed" + default: + re.Fail(fmt.Sprintf("unknown scheduler %s", schedulerName)) + return "" + } + }() + for _, store := range stores { + isStorePaused := !cluster.GetLeaderServer().GetRaftCluster().GetStore(store.GetId()).AllowLeaderTransfer() + if slice.AnyOf(changedStores, func(i int) bool { + return store.GetId() == changedStores[i] + }) { + re.True(isStorePaused, + fmt.Sprintf("store %d should be %s with %s", store.GetId(), status, schedulerName)) + } else { + re.False(isStorePaused, + fmt.Sprintf("store %d should not be %s with %s", store.GetId(), status, schedulerName)) + } + if sche := cluster.GetSchedulingPrimaryServer(); sche != nil { + re.Equal(isStorePaused, !sche.GetCluster().GetStore(store.GetId()).AllowLeaderTransfer()) + } + } + } for idx := range schedulers { + checkStorePause([]uint64{}, schedulers[idx]) // scheduler add command args = []string{"-u", pdAddr, "scheduler", "add", schedulers[idx], "2"} expected = map[string]bool{ @@ -155,6 +189,7 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) { expectedConfig := make(map[string]interface{}) expectedConfig["store-id-ranges"] = map[string]interface{}{"2": []interface{}{map[string]interface{}{"end-key": "", "start-key": ""}}} checkSchedulerConfigCommand(expectedConfig, schedulers[idx]) + checkStorePause([]uint64{2}, schedulers[idx]) // scheduler config update command args = []string{"-u", pdAddr, "scheduler", "config", schedulers[idx], "add-store", "3"} @@ -165,14 +200,12 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) { "transfer-witness-leader-scheduler": true, "balance-witness-scheduler": true, } - checkSchedulerCommand(args, expected) // check update success - // FIXME: remove this check after scheduler config is updated - if cluster.GetSchedulingPrimaryServer() == nil && schedulers[idx] == "grant-leader-scheduler" { - expectedConfig["store-id-ranges"] = map[string]interface{}{"2": []interface{}{map[string]interface{}{"end-key": "", "start-key": ""}}, "3": []interface{}{map[string]interface{}{"end-key": "", "start-key": ""}}} - checkSchedulerConfigCommand(expectedConfig, schedulers[idx]) - } + checkSchedulerCommand(args, expected) + expectedConfig["store-id-ranges"] = map[string]interface{}{"2": []interface{}{map[string]interface{}{"end-key": "", "start-key": ""}}, "3": []interface{}{map[string]interface{}{"end-key": "", "start-key": ""}}} + checkSchedulerConfigCommand(expectedConfig, schedulers[idx]) + checkStorePause([]uint64{2, 3}, schedulers[idx]) // scheduler delete command args = []string{"-u", pdAddr, "scheduler", "remove", schedulers[idx]} @@ -183,6 +216,7 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) { "balance-witness-scheduler": true, } checkSchedulerCommand(args, expected) + checkStorePause([]uint64{}, schedulers[idx]) // scheduler add command args = []string{"-u", pdAddr, "scheduler", "add", schedulers[idx], "2"} @@ -194,6 +228,7 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) { "balance-witness-scheduler": true, } checkSchedulerCommand(args, expected) + checkStorePause([]uint64{2}, schedulers[idx]) // scheduler add command twice args = []string{"-u", pdAddr, "scheduler", "add", schedulers[idx], "4"} @@ -209,6 +244,7 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) { // check add success expectedConfig["store-id-ranges"] = map[string]interface{}{"2": []interface{}{map[string]interface{}{"end-key": "", "start-key": ""}}, "4": []interface{}{map[string]interface{}{"end-key": "", "start-key": ""}}} checkSchedulerConfigCommand(expectedConfig, schedulers[idx]) + checkStorePause([]uint64{2, 4}, schedulers[idx]) // scheduler remove command [old] args = []string{"-u", pdAddr, "scheduler", "remove", schedulers[idx] + "-4"} @@ -224,6 +260,7 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) { // check remove success expectedConfig["store-id-ranges"] = map[string]interface{}{"2": []interface{}{map[string]interface{}{"end-key": "", "start-key": ""}}} checkSchedulerConfigCommand(expectedConfig, schedulers[idx]) + checkStorePause([]uint64{2}, schedulers[idx]) // scheduler remove command, when remove the last store, it should remove whole scheduler args = []string{"-u", pdAddr, "scheduler", "remove", schedulers[idx] + "-2"} @@ -234,6 +271,7 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) { "balance-witness-scheduler": true, } checkSchedulerCommand(args, expected) + checkStorePause([]uint64{}, schedulers[idx]) } // test shuffle region config @@ -247,7 +285,8 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) { var roles []string mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "shuffle-region-scheduler", "show-roles"}, &roles) re.Equal([]string{"leader", "follower", "learner"}, roles) - mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "shuffle-region-scheduler", "set-roles", "learner"}, nil) + echo := mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "shuffle-region-scheduler", "set-roles", "learner"}, nil) // todo:add check output + re.Contains(echo, "Success!") mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "shuffle-region-scheduler", "show-roles"}, &roles) re.Equal([]string{"learner"}, roles) mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "shuffle-region-scheduler"}, &roles) @@ -270,7 +309,8 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) { mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "grant-hot-region-scheduler"}, &conf3) re.Equal(expected3, conf3) - mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "grant-hot-region-scheduler", "set", "2", "1,2,3"}, nil) + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "grant-hot-region-scheduler", "set", "2", "1,2,3"}, nil) + re.Contains(echo, "Success!") expected3["store-leader-id"] = float64(2) // FIXME: remove this check after scheduler config is updated if cluster.GetSchedulingPrimaryServer() == nil { // "grant-hot-region-scheduler" @@ -279,7 +319,7 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) { } // test remove and add scheduler - echo := mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "add", "balance-region-scheduler"}, nil) + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "add", "balance-region-scheduler"}, nil) re.Contains(echo, "Success!") echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "remove", "balance-region-scheduler"}, nil) re.Contains(echo, "Success!") @@ -326,7 +366,8 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) { re.Equal(expected1, conf) mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "show"}, &conf) re.Equal(expected1, conf) - mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "src-tolerance-ratio", "1.02"}, nil) + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "src-tolerance-ratio", "1.02"}, nil) + re.Contains(echo, "Success!") expected1["src-tolerance-ratio"] = 1.02 var conf1 map[string]interface{} // FIXME: remove this check after scheduler config is updated @@ -334,52 +375,66 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) { mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1) re.Equal(expected1, conf1) - mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "read-priorities", "byte,key"}, nil) + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "read-priorities", "byte,key"}, nil) + re.Contains(echo, "Success!") expected1["read-priorities"] = []interface{}{"byte", "key"} mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1) re.Equal(expected1, conf1) - mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "read-priorities", "key"}, nil) + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "read-priorities", "key"}, nil) + re.Contains(echo, "Failed!") mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1) re.Equal(expected1, conf1) - mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "read-priorities", "key,byte"}, nil) + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "read-priorities", "key,byte"}, nil) + re.Contains(echo, "Success!") expected1["read-priorities"] = []interface{}{"key", "byte"} mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1) re.Equal(expected1, conf1) - mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "read-priorities", "foo,bar"}, nil) + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "read-priorities", "foo,bar"}, nil) + re.Contains(echo, "Failed!") mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1) re.Equal(expected1, conf1) - mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "read-priorities", ""}, nil) + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "read-priorities", ""}, nil) + re.Contains(echo, "Failed!") mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1) re.Equal(expected1, conf1) - mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "read-priorities", "key,key"}, nil) + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "read-priorities", "key,key"}, nil) + re.Contains(echo, "Failed!") mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1) re.Equal(expected1, conf1) - mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "read-priorities", "byte,byte"}, nil) + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "read-priorities", "byte,byte"}, nil) + re.Contains(echo, "Failed!") mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1) re.Equal(expected1, conf1) - mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "read-priorities", "key,key,byte"}, nil) + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "read-priorities", "key,key,byte"}, nil) + re.Contains(echo, "Failed!") mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1) re.Equal(expected1, conf1) // write-priorities is divided into write-leader-priorities and write-peer-priorities - mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "write-priorities", "key,byte"}, nil) + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "write-priorities", "key,byte"}, nil) + re.Contains(echo, "Failed!") + re.Contains(echo, "Config item is not found.") mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1) re.Equal(expected1, conf1) - mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "rank-formula-version", "v0"}, nil) + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "rank-formula-version", "v0"}, nil) + re.Contains(echo, "Failed!") mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1) expected1["rank-formula-version"] = "v2" - mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "rank-formula-version", "v2"}, nil) + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "rank-formula-version", "v2"}, nil) + re.Contains(echo, "Success!") mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1) re.Equal(expected1, conf1) expected1["rank-formula-version"] = "v1" - mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "rank-formula-version", "v1"}, nil) + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "rank-formula-version", "v1"}, nil) + re.Contains(echo, "Success!") mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1) re.Equal(expected1, conf1) expected1["forbid-rw-type"] = "read" - mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "forbid-rw-type", "read"}, nil) + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "forbid-rw-type", "read"}, nil) + re.Contains(echo, "Success!") mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1) re.Equal(expected1, conf1) @@ -412,7 +467,8 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) { conf1 = make(map[string]interface{}) mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-leader-scheduler", "show"}, &conf) re.Equal(4., conf["batch"]) - mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-leader-scheduler", "set", "batch", "3"}, nil) + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-leader-scheduler", "set", "batch", "3"}, nil) + re.Contains(echo, "Success!") testutil.Eventually(re, func() bool { mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-leader-scheduler"}, &conf1) return conf1["batch"] == 3. @@ -465,7 +521,8 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) { } mustUsage([]string{"-u", pdAddr, "scheduler", "pause", "balance-leader-scheduler"}) - mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "pause", "balance-leader-scheduler", "60"}, nil) + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "pause", "balance-leader-scheduler", "60"}, nil) + re.Contains(echo, "Success!") checkSchedulerWithStatusCommand("paused", []string{ "balance-leader-scheduler", }) @@ -476,7 +533,8 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) { }, testutil.WithWaitFor(30*time.Second)) mustUsage([]string{"-u", pdAddr, "scheduler", "resume", "balance-leader-scheduler", "60"}) - mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "resume", "balance-leader-scheduler"}, nil) + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "resume", "balance-leader-scheduler"}, nil) + re.Contains(echo, "Success!") checkSchedulerWithStatusCommand("paused", nil) // set label scheduler to disabled manually. @@ -547,11 +605,14 @@ func (suite *schedulerTestSuite) checkSchedulerDiagnostic(cluster *tests.TestClu checkSchedulerDescribeCommand("balance-region-scheduler", "pending", "1 store(s) RegionNotMatchRule; ") // scheduler delete command - mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "remove", "balance-region-scheduler"}, nil) + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "remove", "balance-region-scheduler"}, nil) + re.Contains(echo, "Success!") checkSchedulerDescribeCommand("balance-region-scheduler", "disabled", "") - mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "pause", "balance-leader-scheduler", "60"}, nil) - mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "resume", "balance-leader-scheduler"}, nil) + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "pause", "balance-leader-scheduler", "60"}, nil) + re.Contains(echo, "Success!") + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "resume", "balance-leader-scheduler"}, nil) + re.Contains(echo, "Success!") checkSchedulerDescribeCommand("balance-leader-scheduler", "normal", "") } @@ -604,7 +665,8 @@ func TestForwardSchedulerRequest(t *testing.T) { re.Contains(string(output), "Usage") } mustUsage([]string{"-u", backendEndpoints, "scheduler", "pause", "balance-leader-scheduler"}) - mustExec(re, cmd, []string{"-u", backendEndpoints, "scheduler", "pause", "balance-leader-scheduler", "60"}, nil) + echo := mustExec(re, cmd, []string{"-u", backendEndpoints, "scheduler", "pause", "balance-leader-scheduler", "60"}, nil) + re.Contains(echo, "Success!") checkSchedulerWithStatusCommand := func(status string, expected []string) { var schedulers []string mustExec(re, cmd, []string{"-u", backendEndpoints, "scheduler", "show", "--status", status}, &schedulers) From 89c83748a253df8effd6d8b77c5a8a5bec4e0f74 Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Mon, 20 Nov 2023 18:01:12 +0800 Subject: [PATCH 028/137] mcs: fix sync store label (#7396) close tikv/pd#7391, close tikv/pd#7393, close tikv/pd#7394 Signed-off-by: Ryan Leung --- pkg/core/store_option.go | 20 +++++++++++++++++++ pkg/mcs/scheduling/server/cluster.go | 3 +-- pkg/mcs/scheduling/server/meta/watcher.go | 2 +- .../integrations/mcs/scheduling/meta_test.go | 11 ++++++++++ .../mcs/scheduling/server_test.go | 3 ++- tests/server/cluster/cluster_test.go | 1 + 6 files changed, 36 insertions(+), 4 deletions(-) diff --git a/pkg/core/store_option.go b/pkg/core/store_option.go index 8a2aa1ef089..0bdfaffa44f 100644 --- a/pkg/core/store_option.go +++ b/pkg/core/store_option.go @@ -274,3 +274,23 @@ func SetLastAwakenTime(lastAwaken time.Time) StoreCreateOption { store.lastAwakenTime = lastAwaken } } + +// SetStoreMeta sets the meta for the store. +func SetStoreMeta(newMeta *metapb.Store) StoreCreateOption { + return func(store *StoreInfo) { + meta := typeutil.DeepClone(store.meta, StoreFactory) + meta.Version = newMeta.GetVersion() + meta.GitHash = newMeta.GetGitHash() + meta.Address = newMeta.GetAddress() + meta.StatusAddress = newMeta.GetStatusAddress() + meta.PeerAddress = newMeta.GetPeerAddress() + meta.StartTimestamp = newMeta.GetStartTimestamp() + meta.DeployPath = newMeta.GetDeployPath() + meta.LastHeartbeat = newMeta.GetLastHeartbeat() + meta.State = newMeta.GetState() + meta.Labels = newMeta.GetLabels() + meta.NodeState = newMeta.GetNodeState() + meta.PhysicallyDestroyed = newMeta.GetPhysicallyDestroyed() + store.meta = meta + } +} diff --git a/pkg/mcs/scheduling/server/cluster.go b/pkg/mcs/scheduling/server/cluster.go index ac15212553b..96ee88259da 100644 --- a/pkg/mcs/scheduling/server/cluster.go +++ b/pkg/mcs/scheduling/server/cluster.go @@ -381,8 +381,7 @@ func (c *Cluster) HandleStoreHeartbeat(heartbeat *schedulingpb.StoreHeartbeatReq return errors.Errorf("store %v not found", storeID) } - nowTime := time.Now() - newStore := store.Clone(core.SetStoreStats(stats), core.SetLastHeartbeatTS(nowTime)) + newStore := store.Clone(core.SetStoreStats(stats)) if store := c.GetStore(storeID); store != nil { statistics.UpdateStoreHeartbeatMetrics(store) diff --git a/pkg/mcs/scheduling/server/meta/watcher.go b/pkg/mcs/scheduling/server/meta/watcher.go index 3dbd0fc8c92..3a04c261163 100644 --- a/pkg/mcs/scheduling/server/meta/watcher.go +++ b/pkg/mcs/scheduling/server/meta/watcher.go @@ -81,7 +81,7 @@ func (w *Watcher) initializeStoreWatcher() error { w.basicCluster.PutStore(core.NewStoreInfo(store)) return nil } - w.basicCluster.PutStore(origin.Clone(core.SetStoreState(store.GetState(), store.GetPhysicallyDestroyed()))) + w.basicCluster.PutStore(origin.Clone(core.SetStoreMeta(store))) return nil } deleteFn := func(kv *mvccpb.KeyValue) error { diff --git a/tests/integrations/mcs/scheduling/meta_test.go b/tests/integrations/mcs/scheduling/meta_test.go index 74497e0b552..ce0dc620aef 100644 --- a/tests/integrations/mcs/scheduling/meta_test.go +++ b/tests/integrations/mcs/scheduling/meta_test.go @@ -99,4 +99,15 @@ func (suite *metaTestSuite) TestStoreWatch() { testutil.Eventually(re, func() bool { return cluster.GetStore(2) == nil }) + + // test synchronized store labels + suite.pdLeaderServer.GetServer().GetRaftCluster().PutStore( + &metapb.Store{Id: 5, Address: "mock-5", State: metapb.StoreState_Up, NodeState: metapb.NodeState_Serving, LastHeartbeat: time.Now().UnixNano(), Labels: []*metapb.StoreLabel{{Key: "zone", Value: "z1"}}}, + ) + testutil.Eventually(re, func() bool { + if len(cluster.GetStore(5).GetLabels()) == 0 { + return false + } + return cluster.GetStore(5).GetLabels()[0].GetValue() == "z1" + }) } diff --git a/tests/integrations/mcs/scheduling/server_test.go b/tests/integrations/mcs/scheduling/server_test.go index 41c00b8e9b4..eb99411d27e 100644 --- a/tests/integrations/mcs/scheduling/server_test.go +++ b/tests/integrations/mcs/scheduling/server_test.go @@ -59,7 +59,7 @@ func TestServerTestSuite(t *testing.T) { func (suite *serverTestSuite) SetupSuite() { var err error re := suite.Require() - + re.NoError(failpoint.Enable("github.com/tikv/pd/server/cluster/highFrequencyClusterJobs", `return(true)`)) suite.ctx, suite.cancel = context.WithCancel(context.Background()) suite.cluster, err = tests.NewTestAPICluster(suite.ctx, 3) re.NoError(err) @@ -76,6 +76,7 @@ func (suite *serverTestSuite) SetupSuite() { func (suite *serverTestSuite) TearDownSuite() { suite.cluster.Destroy() suite.cancel() + suite.NoError(failpoint.Disable("github.com/tikv/pd/server/cluster/highFrequencyClusterJobs")) } func (suite *serverTestSuite) TestAllocID() { diff --git a/tests/server/cluster/cluster_test.go b/tests/server/cluster/cluster_test.go index ccb469c04cb..6d233a8c8ab 100644 --- a/tests/server/cluster/cluster_test.go +++ b/tests/server/cluster/cluster_test.go @@ -510,6 +510,7 @@ func TestRaftClusterMultipleRestart(t *testing.T) { err = rc.PutStore(store) re.NoError(err) re.NotNil(tc) + rc.Stop() // let the job run at small interval re.NoError(failpoint.Enable("github.com/tikv/pd/server/cluster/highFrequencyClusterJobs", `return(true)`)) From f9f9be60531d0561860f1f34a741252f843554b0 Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Tue, 21 Nov 2023 12:04:10 +0800 Subject: [PATCH 029/137] *: clean up handling metrics process (#7370) ref tikv/pd#5839, close tikv/pd#7391 Signed-off-by: Ryan Leung --- pkg/mcs/scheduling/server/cluster.go | 16 ---------------- pkg/statistics/hot_cache.go | 4 ++-- pkg/statistics/region_collection.go | 8 ++++---- pkg/statistics/store_collection.go | 3 +++ server/cluster/cluster.go | 4 ++-- server/cluster/cluster_test.go | 21 ++++++++++++++------- server/cluster/scheduling_controller.go | 21 +++++---------------- tests/server/cluster/cluster_test.go | 2 -- 8 files changed, 30 insertions(+), 49 deletions(-) diff --git a/pkg/mcs/scheduling/server/cluster.go b/pkg/mcs/scheduling/server/cluster.go index 96ee88259da..59b2e691658 100644 --- a/pkg/mcs/scheduling/server/cluster.go +++ b/pkg/mcs/scheduling/server/cluster.go @@ -485,10 +485,6 @@ func (c *Cluster) collectMetrics() { c.coordinator.GetSchedulersController().CollectSchedulerMetrics() c.coordinator.CollectHotSpotMetrics() - c.collectClusterMetrics() -} - -func (c *Cluster) collectClusterMetrics() { if c.regionStats == nil { return } @@ -500,20 +496,8 @@ func (c *Cluster) collectClusterMetrics() { func (c *Cluster) resetMetrics() { statistics.Reset() - schedulers.ResetSchedulerMetrics() schedule.ResetHotSpotMetrics() - c.resetClusterMetrics() -} - -func (c *Cluster) resetClusterMetrics() { - if c.regionStats == nil { - return - } - c.regionStats.Reset() - c.labelStats.Reset() - // reset hot cache metrics - c.hotStat.ResetMetrics() } // StartBackgroundJobs starts background jobs. diff --git a/pkg/statistics/hot_cache.go b/pkg/statistics/hot_cache.go index de7189a1332..1868e323b0f 100644 --- a/pkg/statistics/hot_cache.go +++ b/pkg/statistics/hot_cache.go @@ -125,8 +125,8 @@ func (w *HotCache) CollectMetrics() { w.CheckReadAsync(newCollectMetricsTask()) } -// ResetMetrics resets the hot cache metrics. -func (w *HotCache) ResetMetrics() { +// ResetHotCacheStatusMetrics resets the hot cache metrics. +func ResetHotCacheStatusMetrics() { hotCacheStatusGauge.Reset() } diff --git a/pkg/statistics/region_collection.go b/pkg/statistics/region_collection.go index 26cbea9ef92..21af8e152fd 100644 --- a/pkg/statistics/region_collection.go +++ b/pkg/statistics/region_collection.go @@ -272,8 +272,8 @@ func (r *RegionStatistics) Collect() { regionWitnessLeaderRegionCounter.Set(float64(len(r.stats[WitnessLeader]))) } -// Reset resets the metrics of the regions' status. -func (r *RegionStatistics) Reset() { +// ResetRegionStatsMetrics resets the metrics of the regions' status. +func ResetRegionStatsMetrics() { regionMissPeerRegionCounter.Set(0) regionExtraPeerRegionCounter.Set(0) regionDownPeerRegionCounter.Set(0) @@ -326,8 +326,8 @@ func (l *LabelStatistics) Collect() { } } -// Reset resets the metrics of the label status. -func (l *LabelStatistics) Reset() { +// ResetLabelStatsMetrics resets the metrics of the label status. +func ResetLabelStatsMetrics() { regionLabelLevelGauge.Reset() } diff --git a/pkg/statistics/store_collection.go b/pkg/statistics/store_collection.go index dcdd77d9112..aacd45338d1 100644 --- a/pkg/statistics/store_collection.go +++ b/pkg/statistics/store_collection.go @@ -322,4 +322,7 @@ func Reset() { storeStatusGauge.Reset() clusterStatusGauge.Reset() placementStatusGauge.Reset() + ResetRegionStatsMetrics() + ResetLabelStatsMetrics() + ResetHotCacheStatusMetrics() } diff --git a/server/cluster/cluster.go b/server/cluster/cluster.go index 3b826d8d33e..e9cece0faa7 100644 --- a/server/cluster/cluster.go +++ b/server/cluster/cluster.go @@ -654,7 +654,7 @@ func (c *RaftCluster) runMetricsCollectionJob() { ticker := time.NewTicker(metricsCollectionJobInterval) failpoint.Inject("highFrequencyClusterJobs", func() { ticker.Stop() - ticker = time.NewTicker(time.Microsecond) + ticker = time.NewTicker(time.Millisecond) }) defer ticker.Stop() @@ -734,10 +734,10 @@ func (c *RaftCluster) Stop() { return } c.running = false + c.cancel() if !c.IsServiceIndependent(mcsutils.SchedulingServiceName) { c.stopSchedulingJobs() } - c.cancel() c.Unlock() c.wg.Wait() diff --git a/server/cluster/cluster_test.go b/server/cluster/cluster_test.go index e9ce35dfb54..b1a3535e90f 100644 --- a/server/cluster/cluster_test.go +++ b/server/cluster/cluster_test.go @@ -2485,7 +2485,10 @@ func TestCollectMetricsConcurrent(t *testing.T) { nil) }, func(co *schedule.Coordinator) { co.Run() }, re) defer cleanup() - + rc := co.GetCluster().(*RaftCluster) + rc.schedulingController = newSchedulingController(rc.serverCtx, rc.GetBasicCluster(), rc.GetOpts(), rc.GetRuleManager()) + rc.schedulingController.coordinator = co + controller := co.GetSchedulersController() // Make sure there are no problem when concurrent write and read var wg sync.WaitGroup count := 10 @@ -2498,15 +2501,14 @@ func TestCollectMetricsConcurrent(t *testing.T) { } }(i) } - controller := co.GetSchedulersController() for i := 0; i < 1000; i++ { co.CollectHotSpotMetrics() controller.CollectSchedulerMetrics() - co.GetCluster().(*RaftCluster).collectStatisticsMetrics() + rc.collectSchedulingMetrics() } schedule.ResetHotSpotMetrics() schedulers.ResetSchedulerMetrics() - co.GetCluster().(*RaftCluster).resetStatisticsMetrics() + rc.resetSchedulingMetrics() wg.Wait() } @@ -2520,6 +2522,11 @@ func TestCollectMetrics(t *testing.T) { nil) }, func(co *schedule.Coordinator) { co.Run() }, re) defer cleanup() + + rc := co.GetCluster().(*RaftCluster) + rc.schedulingController = newSchedulingController(rc.serverCtx, rc.GetBasicCluster(), rc.GetOpts(), rc.GetRuleManager()) + rc.schedulingController.coordinator = co + controller := co.GetSchedulersController() count := 10 for i := 0; i <= count; i++ { for k := 0; k < 200; k++ { @@ -2533,11 +2540,11 @@ func TestCollectMetrics(t *testing.T) { tc.hotStat.HotCache.Update(item, utils.Write) } } - controller := co.GetSchedulersController() + for i := 0; i < 1000; i++ { co.CollectHotSpotMetrics() controller.CollectSchedulerMetrics() - co.GetCluster().(*RaftCluster).collectStatisticsMetrics() + rc.collectSchedulingMetrics() } stores := co.GetCluster().GetStores() regionStats := co.GetCluster().RegionWriteStats() @@ -2552,7 +2559,7 @@ func TestCollectMetrics(t *testing.T) { re.Equal(status1, status2) schedule.ResetHotSpotMetrics() schedulers.ResetSchedulerMetrics() - co.GetCluster().(*RaftCluster).resetStatisticsMetrics() + rc.resetSchedulingMetrics() } func prepare(setCfg func(*sc.ScheduleConfig), setTc func(*testCluster), run func(*schedule.Coordinator), re *require.Assertions) (*testCluster, *schedule.Coordinator, func()) { diff --git a/server/cluster/scheduling_controller.go b/server/cluster/scheduling_controller.go index bb6470252b0..04c77498948 100644 --- a/server/cluster/scheduling_controller.go +++ b/server/cluster/scheduling_controller.go @@ -149,7 +149,7 @@ func (sc *schedulingController) runSchedulingMetricsCollectionJob() { ticker := time.NewTicker(metricsCollectionJobInterval) failpoint.Inject("highFrequencyClusterJobs", func() { ticker.Stop() - ticker = time.NewTicker(time.Microsecond) + ticker = time.NewTicker(time.Millisecond) }) defer ticker.Stop() @@ -170,7 +170,10 @@ func (sc *schedulingController) resetSchedulingMetrics() { statistics.Reset() schedulers.ResetSchedulerMetrics() schedule.ResetHotSpotMetrics() - sc.resetStatisticsMetrics() + statistics.ResetRegionStatsMetrics() + statistics.ResetLabelStatsMetrics() + // reset hot cache metrics + statistics.ResetHotCacheStatusMetrics() } func (sc *schedulingController) collectSchedulingMetrics() { @@ -183,20 +186,6 @@ func (sc *schedulingController) collectSchedulingMetrics() { statsMap.Collect() sc.coordinator.GetSchedulersController().CollectSchedulerMetrics() sc.coordinator.CollectHotSpotMetrics() - sc.collectStatisticsMetrics() -} - -func (sc *schedulingController) resetStatisticsMetrics() { - if sc.regionStats == nil { - return - } - sc.regionStats.Reset() - sc.labelStats.Reset() - // reset hot cache metrics - sc.hotStat.ResetMetrics() -} - -func (sc *schedulingController) collectStatisticsMetrics() { if sc.regionStats == nil { return } diff --git a/tests/server/cluster/cluster_test.go b/tests/server/cluster/cluster_test.go index 6d233a8c8ab..a5861f1ba43 100644 --- a/tests/server/cluster/cluster_test.go +++ b/tests/server/cluster/cluster_test.go @@ -518,8 +518,6 @@ func TestRaftClusterMultipleRestart(t *testing.T) { err = rc.Start(leaderServer.GetServer()) re.NoError(err) time.Sleep(time.Millisecond) - rc = leaderServer.GetRaftCluster() - re.NotNil(rc) rc.Stop() } re.NoError(failpoint.Disable("github.com/tikv/pd/server/cluster/highFrequencyClusterJobs")) From 5a2a8d6965b8921230124ce478aed5b4203d2363 Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Tue, 21 Nov 2023 14:14:41 +0800 Subject: [PATCH 030/137] *: make TestConfigTestSuite stable (#7398) close tikv/pd#7395 Signed-off-by: lhy1024 --- client/retry/backoff.go | 4 +++- pkg/schedule/operator/operator_test.go | 3 +-- tests/integrations/mcs/tso/server_test.go | 2 +- tests/pdctl/config/config_test.go | 5 +++-- tests/server/api/rule_test.go | 1 - tests/server/cluster/cluster_test.go | 6 +++--- tests/server/member/member_test.go | 4 ++-- tests/server/region_syncer/region_syncer_test.go | 2 +- 8 files changed, 14 insertions(+), 13 deletions(-) diff --git a/client/retry/backoff.go b/client/retry/backoff.go index e2ca9ab3972..b47a39d8eaa 100644 --- a/client/retry/backoff.go +++ b/client/retry/backoff.go @@ -34,9 +34,11 @@ func (bo *BackOffer) Exec( fn func() error, ) error { if err := fn(); err != nil { + after := time.NewTimer(bo.nextInterval()) + defer after.Stop() select { case <-ctx.Done(): - case <-time.After(bo.nextInterval()): + case <-after.C: failpoint.Inject("backOffExecute", func() { testBackOffExecuteFlag = true }) diff --git a/pkg/schedule/operator/operator_test.go b/pkg/schedule/operator/operator_test.go index 9d924738543..c16d929f379 100644 --- a/pkg/schedule/operator/operator_test.go +++ b/pkg/schedule/operator/operator_test.go @@ -17,7 +17,6 @@ package operator import ( "context" "encoding/json" - "fmt" "sync/atomic" "testing" "time" @@ -514,7 +513,7 @@ func (suite *operatorTestSuite) TestOpStepTimeout() { }, } for i, v := range testData { - fmt.Printf("case:%d\n", i) + suite.T().Logf("case: %d", i) for _, step := range v.step { suite.Equal(v.expect, step.Timeout(v.regionSize)) } diff --git a/tests/integrations/mcs/tso/server_test.go b/tests/integrations/mcs/tso/server_test.go index 58006b87eeb..643fb3c7911 100644 --- a/tests/integrations/mcs/tso/server_test.go +++ b/tests/integrations/mcs/tso/server_test.go @@ -89,7 +89,7 @@ func (suite *tsoServerTestSuite) TearDownSuite() { func (suite *tsoServerTestSuite) TestTSOServerStartAndStopNormally() { defer func() { if r := recover(); r != nil { - fmt.Println("Recovered from an unexpected panic", r) + suite.T().Log("Recovered from an unexpected panic", r) suite.T().Errorf("Expected no panic, but something bad occurred with") } }() diff --git a/tests/pdctl/config/config_test.go b/tests/pdctl/config/config_test.go index 91d6723c2ac..badccd9becc 100644 --- a/tests/pdctl/config/config_test.go +++ b/tests/pdctl/config/config_test.go @@ -832,7 +832,6 @@ func (suite *configTestSuite) checkPDServerConfig(cluster *tests.TestCluster) { LastHeartbeat: time.Now().UnixNano(), } tests.MustPutStore(re, cluster, store) - defer cluster.Destroy() output, err := pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "show", "server") re.NoError(err) @@ -844,7 +843,9 @@ func (suite *configTestSuite) checkPDServerConfig(cluster *tests.TestCluster) { re.Equal("table", conf.KeyType) re.Equal(typeutil.StringSlice([]string{}), conf.RuntimeServices) re.Equal("", conf.MetricStorage) - re.Equal("auto", conf.DashboardAddress) + if conf.DashboardAddress != "auto" { // dashboard has been assigned + re.Equal(leaderServer.GetAddr(), conf.DashboardAddress) + } re.Equal(int(3), conf.FlowRoundByDigit) } diff --git a/tests/server/api/rule_test.go b/tests/server/api/rule_test.go index 9176a00e66d..ffdf56b6567 100644 --- a/tests/server/api/rule_test.go +++ b/tests/server/api/rule_test.go @@ -1097,7 +1097,6 @@ func (suite *regionRuleTestSuite) checkRegionPlacementRule(cluster *tests.TestCl var label labeler.LabelRule escapedID := url.PathEscape("keyspaces/0") u = fmt.Sprintf("%s/config/region-label/rule/%s", urlPrefix, escapedID) - fmt.Println("u====", u) err = tu.ReadGetJSON(re, testDialClient, u, &label) suite.NoError(err) suite.Equal(label.ID, "keyspaces/0") diff --git a/tests/server/cluster/cluster_test.go b/tests/server/cluster/cluster_test.go index a5861f1ba43..18a82bcf0fe 100644 --- a/tests/server/cluster/cluster_test.go +++ b/tests/server/cluster/cluster_test.go @@ -1288,7 +1288,7 @@ func TestTransferLeaderForScheduler(t *testing.T) { re.NoError(err) tc.WaitLeader() // start - leaderServer := tc.GetServer(tc.GetLeader()) + leaderServer := tc.GetLeaderServer() re.NoError(leaderServer.BootstrapCluster()) rc := leaderServer.GetServer().GetRaftCluster() re.NotNil(rc) @@ -1327,7 +1327,7 @@ func TestTransferLeaderForScheduler(t *testing.T) { tc.ResignLeader() rc.Stop() tc.WaitLeader() - leaderServer = tc.GetServer(tc.GetLeader()) + leaderServer = tc.GetLeaderServer() rc1 := leaderServer.GetServer().GetRaftCluster() rc1.Start(leaderServer.GetServer()) re.NoError(err) @@ -1347,7 +1347,7 @@ func TestTransferLeaderForScheduler(t *testing.T) { tc.ResignLeader() rc1.Stop() tc.WaitLeader() - leaderServer = tc.GetServer(tc.GetLeader()) + leaderServer = tc.GetLeaderServer() rc = leaderServer.GetServer().GetRaftCluster() rc.Start(leaderServer.GetServer()) re.NotNil(rc) diff --git a/tests/server/member/member_test.go b/tests/server/member/member_test.go index 5965f9e22a6..e6657ffd223 100644 --- a/tests/server/member/member_test.go +++ b/tests/server/member/member_test.go @@ -260,7 +260,7 @@ func TestPDLeaderLostWhileEtcdLeaderIntact(t *testing.T) { re.NoError(err) leader1 := cluster.WaitLeader() - memberID := cluster.GetServer(leader1).GetLeader().GetMemberId() + memberID := cluster.GetLeaderServer().GetLeader().GetMemberId() re.NoError(failpoint.Enable("github.com/tikv/pd/server/leaderLoopCheckAgain", fmt.Sprintf("return(\"%d\")", memberID))) re.NoError(failpoint.Enable("github.com/tikv/pd/server/exitCampaignLeader", fmt.Sprintf("return(\"%d\")", memberID))) @@ -338,7 +338,7 @@ func TestCampaignLeaderFrequently(t *testing.T) { re.NotEmpty(cluster.GetLeader()) for i := 0; i < 3; i++ { - cluster.GetServers()[cluster.GetLeader()].ResetPDLeader() + cluster.GetLeaderServer().ResetPDLeader() cluster.WaitLeader() } // leader should be changed when campaign leader frequently diff --git a/tests/server/region_syncer/region_syncer_test.go b/tests/server/region_syncer/region_syncer_test.go index b73d4abb9b5..6521432c0dc 100644 --- a/tests/server/region_syncer/region_syncer_test.go +++ b/tests/server/region_syncer/region_syncer_test.go @@ -259,7 +259,7 @@ func TestPrepareCheckerWithTransferLeader(t *testing.T) { err = cluster.RunInitialServers() re.NoError(err) cluster.WaitLeader() - leaderServer := cluster.GetServer(cluster.GetLeader()) + leaderServer := cluster.GetLeaderServer() re.NoError(leaderServer.BootstrapCluster()) rc := leaderServer.GetServer().GetRaftCluster() re.NotNil(rc) From 2349f011e4ef42359de7624ac6f5b594abd6e040 Mon Sep 17 00:00:00 2001 From: JmPotato Date: Tue, 21 Nov 2023 14:41:40 +0800 Subject: [PATCH 031/137] client/http: encapsulate rule-related PD HTTP interfaces (#7397) ref tikv/pd#7300 Expand the PD HTTP interfaces to include more encapsulation, particularly focusing on the rule-related interfaces. Signed-off-by: JmPotato Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- client/http/api.go | 15 +- client/http/client.go | 148 ++++++++++++++++-- client/http/types.go | 31 ++++ tests/integrations/client/http_client_test.go | 89 ++++++++++- 4 files changed, 262 insertions(+), 21 deletions(-) diff --git a/client/http/api.go b/client/http/api.go index 2fae562dd20..1826e2231ee 100644 --- a/client/http/api.go +++ b/client/http/api.go @@ -32,7 +32,7 @@ const ( regionsByKey = "/pd/api/v1/regions/key" RegionsByStoreIDPrefix = "/pd/api/v1/regions/store" EmptyRegions = "/pd/api/v1/regions/check/empty-region" - accelerateSchedule = "/pd/api/v1/regions/accelerate-schedule" + AccelerateSchedule = "/pd/api/v1/regions/accelerate-schedule" store = "/pd/api/v1/store" Stores = "/pd/api/v1/stores" StatsRegion = "/pd/api/v1/stats/region" @@ -45,7 +45,10 @@ const ( PlacementRule = "/pd/api/v1/config/rule" PlacementRules = "/pd/api/v1/config/rules" placementRulesByGroup = "/pd/api/v1/config/rules/group" + PlacementRuleBundle = "/pd/api/v1/config/placement-rule" RegionLabelRule = "/pd/api/v1/config/region-label/rule" + RegionLabelRules = "/pd/api/v1/config/region-label/rules" + RegionLabelRulesByIDs = "/pd/api/v1/config/region-label/rules/ids" // Scheduler Schedulers = "/pd/api/v1/schedulers" scatterRangeScheduler = "/pd/api/v1/schedulers/scatter-range-" @@ -123,6 +126,16 @@ func PlacementRuleByGroupAndID(group, id string) string { return fmt.Sprintf("%s/%s/%s", PlacementRule, group, id) } +// PlacementRuleBundleByGroup returns the path of PD HTTP API to get placement rule bundle by group. +func PlacementRuleBundleByGroup(group string) string { + return fmt.Sprintf("%s/%s", PlacementRuleBundle, group) +} + +// PlacementRuleBundleWithPartialParameter returns the path of PD HTTP API to get placement rule bundle with partial parameter. +func PlacementRuleBundleWithPartialParameter(partial bool) string { + return fmt.Sprintf("%s?partial=%t", PlacementRuleBundle, partial) +} + // SchedulerByName returns the scheduler API with the given scheduler name. func SchedulerByName(name string) string { return fmt.Sprintf("%s/%s", Schedulers, name) diff --git a/client/http/client.go b/client/http/client.go index 6fa2dd8cdfd..5e438a77076 100644 --- a/client/http/client.go +++ b/client/http/client.go @@ -42,6 +42,7 @@ const ( // Client is a PD (Placement Driver) HTTP client. type Client interface { + /* Meta-related interfaces */ GetRegionByID(context.Context, uint64) (*RegionInfo, error) GetRegionByKey(context.Context, []byte) (*RegionInfo, error) GetRegions(context.Context) (*RegionsInfo, error) @@ -51,11 +52,24 @@ type Client interface { GetHotWriteRegions(context.Context) (*StoreHotPeersInfos, error) GetRegionStatusByKeyRange(context.Context, []byte, []byte) (*RegionStats, error) GetStores(context.Context) (*StoresInfo, error) + /* Rule-related interfaces */ + GetAllPlacementRuleBundles(context.Context) ([]*GroupBundle, error) + GetPlacementRuleBundleByGroup(context.Context, string) (*GroupBundle, error) GetPlacementRulesByGroup(context.Context, string) ([]*Rule, error) SetPlacementRule(context.Context, *Rule) error + SetPlacementRuleBundles(context.Context, []*GroupBundle, bool) error DeletePlacementRule(context.Context, string, string) error - GetMinResolvedTSByStoresIDs(context.Context, []uint64) (uint64, map[uint64]uint64, error) + GetAllRegionLabelRules(context.Context) ([]*LabelRule, error) + GetRegionLabelRulesByIDs(context.Context, []string) ([]*LabelRule, error) + SetRegionLabelRule(context.Context, *LabelRule) error + PatchRegionLabelRules(context.Context, *LabelRulePatch) error + /* Scheduling-related interfaces */ AccelerateSchedule(context.Context, []byte, []byte) error + /* Other interfaces */ + GetMinResolvedTSByStoresIDs(context.Context, []uint64) (uint64, map[uint64]uint64, error) + + /* Client-related methods */ + WithRespHandler(func(resp *http.Response) error) Client Close() } @@ -66,6 +80,8 @@ type client struct { tlsConf *tls.Config cli *http.Client + respHandler func(resp *http.Response) error + requestCounter *prometheus.CounterVec executionDuration *prometheus.HistogramVec } @@ -143,6 +159,14 @@ func (c *client) Close() { log.Info("[pd] http client closed") } +// WithRespHandler sets and returns a new client with the given HTTP response handler. +// This allows the caller to customize how the response is handled, including error handling logic. +func (c *client) WithRespHandler(handler func(resp *http.Response) error) Client { + newClient := *c + newClient.respHandler = handler + return &newClient +} + func (c *client) reqCounter(name, status string) { if c.requestCounter == nil { return @@ -204,6 +228,12 @@ func (c *client) request( } c.execDuration(name, time.Since(start)) c.reqCounter(name, resp.Status) + + // Give away the response handling to the caller if the handler is set. + if c.respHandler != nil { + return c.respHandler(resp) + } + defer func() { err = resp.Body.Close() if err != nil { @@ -345,6 +375,30 @@ func (c *client) GetStores(ctx context.Context) (*StoresInfo, error) { return &stores, nil } +// GetAllPlacementRuleBundles gets all placement rules bundles. +func (c *client) GetAllPlacementRuleBundles(ctx context.Context) ([]*GroupBundle, error) { + var bundles []*GroupBundle + err := c.requestWithRetry(ctx, + "GetPlacementRuleBundle", PlacementRuleBundle, + http.MethodGet, nil, &bundles) + if err != nil { + return nil, err + } + return bundles, nil +} + +// GetPlacementRuleBundleByGroup gets the placement rules bundle by group. +func (c *client) GetPlacementRuleBundleByGroup(ctx context.Context, group string) (*GroupBundle, error) { + var bundle GroupBundle + err := c.requestWithRetry(ctx, + "GetPlacementRuleBundleByGroup", PlacementRuleBundleByGroup(group), + http.MethodGet, nil, &bundle) + if err != nil { + return nil, err + } + return &bundle, nil +} + // GetPlacementRulesByGroup gets the placement rules by group. func (c *client) GetPlacementRulesByGroup(ctx context.Context, group string) ([]*Rule, error) { var rules []*Rule @@ -368,6 +422,18 @@ func (c *client) SetPlacementRule(ctx context.Context, rule *Rule) error { http.MethodPost, bytes.NewBuffer(ruleJSON), nil) } +// SetPlacementRuleBundles sets the placement rule bundles. +// If `partial` is false, all old configurations will be over-written and dropped. +func (c *client) SetPlacementRuleBundles(ctx context.Context, bundles []*GroupBundle, partial bool) error { + bundlesJSON, err := json.Marshal(bundles) + if err != nil { + return errors.Trace(err) + } + return c.requestWithRetry(ctx, + "SetPlacementRuleBundles", PlacementRuleBundleWithPartialParameter(partial), + http.MethodPost, bytes.NewBuffer(bundlesJSON), nil) +} + // DeletePlacementRule deletes the placement rule. func (c *client) DeletePlacementRule(ctx context.Context, group, id string) error { return c.requestWithRetry(ctx, @@ -375,6 +441,71 @@ func (c *client) DeletePlacementRule(ctx context.Context, group, id string) erro http.MethodDelete, nil, nil) } +// GetAllRegionLabelRules gets all region label rules. +func (c *client) GetAllRegionLabelRules(ctx context.Context) ([]*LabelRule, error) { + var labelRules []*LabelRule + err := c.requestWithRetry(ctx, + "GetAllRegionLabelRules", RegionLabelRules, + http.MethodGet, nil, &labelRules) + if err != nil { + return nil, err + } + return labelRules, nil +} + +// GetRegionLabelRulesByIDs gets the region label rules by IDs. +func (c *client) GetRegionLabelRulesByIDs(ctx context.Context, ruleIDs []string) ([]*LabelRule, error) { + idsJSON, err := json.Marshal(ruleIDs) + if err != nil { + return nil, errors.Trace(err) + } + var labelRules []*LabelRule + err = c.requestWithRetry(ctx, + "GetRegionLabelRulesByIDs", RegionLabelRulesByIDs, + http.MethodGet, bytes.NewBuffer(idsJSON), &labelRules) + if err != nil { + return nil, err + } + return labelRules, nil +} + +// SetRegionLabelRule sets the region label rule. +func (c *client) SetRegionLabelRule(ctx context.Context, labelRule *LabelRule) error { + labelRuleJSON, err := json.Marshal(labelRule) + if err != nil { + return errors.Trace(err) + } + return c.requestWithRetry(ctx, + "SetRegionLabelRule", RegionLabelRule, + http.MethodPost, bytes.NewBuffer(labelRuleJSON), nil) +} + +// PatchRegionLabelRules patches the region label rules. +func (c *client) PatchRegionLabelRules(ctx context.Context, labelRulePatch *LabelRulePatch) error { + labelRulePatchJSON, err := json.Marshal(labelRulePatch) + if err != nil { + return errors.Trace(err) + } + return c.requestWithRetry(ctx, + "PatchRegionLabelRules", RegionLabelRules, + http.MethodPatch, bytes.NewBuffer(labelRulePatchJSON), nil) +} + +// AccelerateSchedule accelerates the scheduling of the regions within the given key range. +func (c *client) AccelerateSchedule(ctx context.Context, startKey, endKey []byte) error { + input := map[string]string{ + "start_key": url.QueryEscape(string(startKey)), + "end_key": url.QueryEscape(string(endKey)), + } + inputJSON, err := json.Marshal(input) + if err != nil { + return errors.Trace(err) + } + return c.requestWithRetry(ctx, + "AccelerateSchedule", AccelerateSchedule, + http.MethodPost, bytes.NewBuffer(inputJSON), nil) +} + // GetMinResolvedTSByStoresIDs get min-resolved-ts by stores IDs. func (c *client) GetMinResolvedTSByStoresIDs(ctx context.Context, storeIDs []uint64) (uint64, map[uint64]uint64, error) { uri := MinResolvedTSPrefix @@ -406,18 +537,3 @@ func (c *client) GetMinResolvedTSByStoresIDs(ctx context.Context, storeIDs []uin } return resp.MinResolvedTS, resp.StoresMinResolvedTS, nil } - -// AccelerateSchedule accelerates the scheduling of the regions within the given key range. -func (c *client) AccelerateSchedule(ctx context.Context, startKey, endKey []byte) error { - input := map[string]string{ - "start_key": url.QueryEscape(string(startKey)), - "end_key": url.QueryEscape(string(endKey)), - } - inputJSON, err := json.Marshal(input) - if err != nil { - return errors.Trace(err) - } - return c.requestWithRetry(ctx, - "AccelerateSchedule", accelerateSchedule, - http.MethodPost, bytes.NewBuffer(inputJSON), nil) -} diff --git a/client/http/types.go b/client/http/types.go index c6bb0256c14..f948286c2b5 100644 --- a/client/http/types.go +++ b/client/http/types.go @@ -246,3 +246,34 @@ type Rule struct { Version uint64 `json:"version,omitempty"` // only set at runtime, add 1 each time rules updated, begin from 0. CreateTimestamp uint64 `json:"create_timestamp,omitempty"` // only set at runtime, recorded rule create timestamp } + +// GroupBundle represents a rule group and all rules belong to the group. +type GroupBundle struct { + ID string `json:"group_id"` + Index int `json:"group_index"` + Override bool `json:"group_override"` + Rules []*Rule `json:"rules"` +} + +// RegionLabel is the label of a region. +type RegionLabel struct { + Key string `json:"key"` + Value string `json:"value"` + TTL string `json:"ttl,omitempty"` + StartAt string `json:"start_at,omitempty"` +} + +// LabelRule is the rule to assign labels to a region. +type LabelRule struct { + ID string `json:"id"` + Index int `json:"index"` + Labels []RegionLabel `json:"labels"` + RuleType string `json:"rule_type"` + Data interface{} `json:"data"` +} + +// LabelRulePatch is the patch to update the label rules. +type LabelRulePatch struct { + SetRules []*LabelRule `json:"sets"` + DeleteRules []string `json:"deletes"` +} diff --git a/tests/integrations/client/http_client_test.go b/tests/integrations/client/http_client_test.go index d2c88d01f09..213aa57de46 100644 --- a/tests/integrations/client/http_client_test.go +++ b/tests/integrations/client/http_client_test.go @@ -17,10 +17,12 @@ package client_test import ( "context" "math" + "sort" "testing" "github.com/stretchr/testify/suite" pd "github.com/tikv/pd/client/http" + "github.com/tikv/pd/pkg/schedule/labeler" "github.com/tikv/pd/pkg/schedule/placement" "github.com/tikv/pd/tests" ) @@ -89,6 +91,13 @@ func (suite *httpClientTestSuite) TestGetMinResolvedTSByStoresIDs() { func (suite *httpClientTestSuite) TestRule() { re := suite.Require() + bundles, err := suite.client.GetAllPlacementRuleBundles(suite.ctx) + re.NoError(err) + re.Len(bundles, 1) + re.Equal(bundles[0].ID, placement.DefaultGroupID) + bundle, err := suite.client.GetPlacementRuleBundleByGroup(suite.ctx, placement.DefaultGroupID) + re.NoError(err) + re.Equal(bundles[0], bundle) rules, err := suite.client.GetPlacementRulesByGroup(suite.ctx, placement.DefaultGroupID) re.NoError(err) re.Len(rules, 1) @@ -96,19 +105,22 @@ func (suite *httpClientTestSuite) TestRule() { re.Equal(placement.DefaultRuleID, rules[0].ID) re.Equal(pd.Voter, rules[0].Role) re.Equal(3, rules[0].Count) - err = suite.client.SetPlacementRule(suite.ctx, &pd.Rule{ + // Should be the same as the rules in the bundle. + re.Equal(bundle.Rules, rules) + testRule := &pd.Rule{ GroupID: placement.DefaultGroupID, ID: "test", - Role: pd.Learner, + Role: pd.Voter, Count: 3, - }) + } + err = suite.client.SetPlacementRule(suite.ctx, testRule) re.NoError(err) rules, err = suite.client.GetPlacementRulesByGroup(suite.ctx, placement.DefaultGroupID) re.NoError(err) re.Len(rules, 2) re.Equal(placement.DefaultGroupID, rules[1].GroupID) re.Equal("test", rules[1].ID) - re.Equal(pd.Learner, rules[1].Role) + re.Equal(pd.Voter, rules[1].Role) re.Equal(3, rules[1].Count) err = suite.client.DeletePlacementRule(suite.ctx, placement.DefaultGroupID, "test") re.NoError(err) @@ -117,6 +129,75 @@ func (suite *httpClientTestSuite) TestRule() { re.Len(rules, 1) re.Equal(placement.DefaultGroupID, rules[0].GroupID) re.Equal(placement.DefaultRuleID, rules[0].ID) + err = suite.client.SetPlacementRuleBundles(suite.ctx, []*pd.GroupBundle{ + { + ID: placement.DefaultGroupID, + Rules: []*pd.Rule{testRule}, + }, + }, true) + re.NoError(err) + bundles, err = suite.client.GetAllPlacementRuleBundles(suite.ctx) + re.NoError(err) + re.Len(bundles, 1) + re.Equal(placement.DefaultGroupID, bundles[0].ID) + re.Len(bundles[0].Rules, 1) + // Make sure the create timestamp is not zero to pass the later assertion. + testRule.CreateTimestamp = bundles[0].Rules[0].CreateTimestamp + re.Equal(testRule, bundles[0].Rules[0]) +} + +func (suite *httpClientTestSuite) TestRegionLabel() { + re := suite.Require() + labelRules, err := suite.client.GetAllRegionLabelRules(suite.ctx) + re.NoError(err) + re.Len(labelRules, 1) + re.Equal("keyspaces/0", labelRules[0].ID) + // Set a new region label rule. + labelRule := &pd.LabelRule{ + ID: "rule1", + Labels: []pd.RegionLabel{{Key: "k1", Value: "v1"}}, + RuleType: "key-range", + Data: labeler.MakeKeyRanges("1234", "5678"), + } + err = suite.client.SetRegionLabelRule(suite.ctx, labelRule) + re.NoError(err) + labelRules, err = suite.client.GetAllRegionLabelRules(suite.ctx) + re.NoError(err) + re.Len(labelRules, 2) + sort.Slice(labelRules, func(i, j int) bool { + return labelRules[i].ID < labelRules[j].ID + }) + re.Equal(labelRule.ID, labelRules[1].ID) + re.Equal(labelRule.Labels, labelRules[1].Labels) + re.Equal(labelRule.RuleType, labelRules[1].RuleType) + // Patch the region label rule. + labelRule = &pd.LabelRule{ + ID: "rule2", + Labels: []pd.RegionLabel{{Key: "k2", Value: "v2"}}, + RuleType: "key-range", + Data: labeler.MakeKeyRanges("ab12", "cd12"), + } + patch := &pd.LabelRulePatch{ + SetRules: []*pd.LabelRule{labelRule}, + DeleteRules: []string{"rule1"}, + } + err = suite.client.PatchRegionLabelRules(suite.ctx, patch) + re.NoError(err) + allLabelRules, err := suite.client.GetAllRegionLabelRules(suite.ctx) + re.NoError(err) + re.Len(labelRules, 2) + sort.Slice(allLabelRules, func(i, j int) bool { + return allLabelRules[i].ID < allLabelRules[j].ID + }) + re.Equal(labelRule.ID, allLabelRules[1].ID) + re.Equal(labelRule.Labels, allLabelRules[1].Labels) + re.Equal(labelRule.RuleType, allLabelRules[1].RuleType) + labelRules, err = suite.client.GetRegionLabelRulesByIDs(suite.ctx, []string{"keyspaces/0", "rule2"}) + re.NoError(err) + sort.Slice(labelRules, func(i, j int) bool { + return labelRules[i].ID < labelRules[j].ID + }) + re.Equal(allLabelRules, labelRules) } func (suite *httpClientTestSuite) TestAccelerateSchedule() { From 8919bc11f7708736f513b9ae39763a717dc8d6df Mon Sep 17 00:00:00 2001 From: JmPotato Date: Tue, 21 Nov 2023 16:05:41 +0800 Subject: [PATCH 032/137] client/http: pass res interface into respHandler (#7404) ref tikv/pd#7300 Pass `res` interface into `respHandler`. Signed-off-by: JmPotato --- client/http/client.go | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/client/http/client.go b/client/http/client.go index 5e438a77076..d7eb1b1b801 100644 --- a/client/http/client.go +++ b/client/http/client.go @@ -69,7 +69,11 @@ type Client interface { GetMinResolvedTSByStoresIDs(context.Context, []uint64) (uint64, map[uint64]uint64, error) /* Client-related methods */ - WithRespHandler(func(resp *http.Response) error) Client + // WithRespHandler sets and returns a new client with the given HTTP response handler. + // This allows the caller to customize how the response is handled, including error handling logic. + // Additionally, it is important for the caller to handle the content of the response body properly + // in order to ensure that it can be read and marshaled correctly into `res`. + WithRespHandler(func(resp *http.Response, res interface{}) error) Client Close() } @@ -80,7 +84,7 @@ type client struct { tlsConf *tls.Config cli *http.Client - respHandler func(resp *http.Response) error + respHandler func(resp *http.Response, res interface{}) error requestCounter *prometheus.CounterVec executionDuration *prometheus.HistogramVec @@ -160,8 +164,9 @@ func (c *client) Close() { } // WithRespHandler sets and returns a new client with the given HTTP response handler. -// This allows the caller to customize how the response is handled, including error handling logic. -func (c *client) WithRespHandler(handler func(resp *http.Response) error) Client { +func (c *client) WithRespHandler( + handler func(resp *http.Response, res interface{}) error, +) Client { newClient := *c newClient.respHandler = handler return &newClient @@ -231,7 +236,7 @@ func (c *client) request( // Give away the response handling to the caller if the handler is set. if c.respHandler != nil { - return c.respHandler(resp) + return c.respHandler(resp, res) } defer func() { From 57fb02008865a77251c7bf30ce49953859993d41 Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Tue, 21 Nov 2023 19:14:41 +0800 Subject: [PATCH 033/137] dr: add more log when state change (#7390) ref tikv/pd#4399 Signed-off-by: lhy1024 Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- pkg/replication/replication_mode.go | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/pkg/replication/replication_mode.go b/pkg/replication/replication_mode.go index 9093f911901..9776a36a8f3 100644 --- a/pkg/replication/replication_mode.go +++ b/pkg/replication/replication_mode.go @@ -442,15 +442,6 @@ func (m *ModeManager) tickUpdateState() { canSync := primaryHasVoter && drHasVoter hasMajority := totalUpVoter*2 > totalVoter - log.Debug("replication store status", - zap.Uint64s("up-primary", storeIDs[primaryUp]), - zap.Uint64s("up-dr", storeIDs[drUp]), - zap.Uint64s("down-primary", storeIDs[primaryDown]), - zap.Uint64s("down-dr", storeIDs[drDown]), - zap.Bool("can-sync", canSync), - zap.Bool("has-majority", hasMajority), - ) - /* +----+ all region sync +------------+ @@ -469,7 +460,8 @@ func (m *ModeManager) tickUpdateState() { */ - switch m.drGetState() { + state := m.drGetState() + switch state { case drStateSync: // If hasMajority is false, the cluster is always unavailable. Switch to async won't help. if !canSync && hasMajority { @@ -511,6 +503,19 @@ func (m *ModeManager) tickUpdateState() { } } } + + logFunc := log.Debug + if state != m.drGetState() { + logFunc = log.Info + } + logFunc("replication store status", + zap.Uint64s("up-primary", storeIDs[primaryUp]), + zap.Uint64s("up-dr", storeIDs[drUp]), + zap.Uint64s("down-primary", storeIDs[primaryDown]), + zap.Uint64s("down-dr", storeIDs[drDown]), + zap.Bool("can-sync", canSync), + zap.Bool("has-majority", hasMajority), + ) } func (m *ModeManager) tickReplicateStatus() { From 0e2dad1c4b69c039553a4e46a9a8ec352175be73 Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Wed, 22 Nov 2023 12:38:41 +0800 Subject: [PATCH 034/137] tests: make TestFallbackTSOConsistency stable (#7408) close tikv/pd#7407 Signed-off-by: lhy1024 --- tests/integrations/tso/consistency_test.go | 19 +++++++++++-------- tests/integrations/tso/server_test.go | 4 ++-- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/tests/integrations/tso/consistency_test.go b/tests/integrations/tso/consistency_test.go index 1d35e8bf5e2..74f5090bf3b 100644 --- a/tests/integrations/tso/consistency_test.go +++ b/tests/integrations/tso/consistency_test.go @@ -27,7 +27,7 @@ import ( tso "github.com/tikv/pd/pkg/mcs/tso/server" tsopkg "github.com/tikv/pd/pkg/tso" "github.com/tikv/pd/pkg/utils/tempurl" - pd "github.com/tikv/pd/pkg/utils/testutil" + tu "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/pkg/utils/tsoutil" "github.com/tikv/pd/tests" "google.golang.org/grpc" @@ -82,7 +82,7 @@ func (suite *tsoConsistencyTestSuite) SetupSuite() { suite.pdLeaderServer = suite.cluster.GetServer(leaderName) backendEndpoints := suite.pdLeaderServer.GetAddr() if suite.legacy { - suite.pdClient = pd.MustNewGrpcClient(re, backendEndpoints) + suite.pdClient = tu.MustNewGrpcClient(re, backendEndpoints) } else { suite.tsoServer, suite.tsoServerCleanup = tests.StartSingleTSOTestServer(suite.ctx, re, backendEndpoints, tempurl.Alloc()) suite.tsoClientConn, suite.tsoClient = tso.MustNewGrpcClient(re, suite.tsoServer.GetAddr()) @@ -127,12 +127,15 @@ func (suite *tsoConsistencyTestSuite) request(ctx context.Context, count uint32) DcLocation: tsopkg.GlobalDCLocation, Count: count, } - tsoClient, err := suite.tsoClient.Tso(ctx) - re.NoError(err) - defer tsoClient.CloseSend() - re.NoError(tsoClient.Send(req)) - resp, err := tsoClient.Recv() - re.NoError(err) + var resp *tsopb.TsoResponse + tu.Eventually(re, func() bool { + tsoClient, err := suite.tsoClient.Tso(ctx) + re.NoError(err) + defer tsoClient.CloseSend() + re.NoError(tsoClient.Send(req)) + resp, err = tsoClient.Recv() + return err == nil && resp != nil + }) return checkAndReturnTimestampResponse(re, resp) } diff --git a/tests/integrations/tso/server_test.go b/tests/integrations/tso/server_test.go index c44f4967a84..ac3d914aa80 100644 --- a/tests/integrations/tso/server_test.go +++ b/tests/integrations/tso/server_test.go @@ -26,7 +26,7 @@ import ( tso "github.com/tikv/pd/pkg/mcs/tso/server" tsopkg "github.com/tikv/pd/pkg/tso" "github.com/tikv/pd/pkg/utils/tempurl" - pd "github.com/tikv/pd/pkg/utils/testutil" + tu "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/tests" "google.golang.org/grpc" ) @@ -80,7 +80,7 @@ func (suite *tsoServerTestSuite) SetupSuite() { suite.pdLeaderServer = suite.cluster.GetServer(leaderName) backendEndpoints := suite.pdLeaderServer.GetAddr() if suite.legacy { - suite.pdClient = pd.MustNewGrpcClient(re, backendEndpoints) + suite.pdClient = tu.MustNewGrpcClient(re, backendEndpoints) } else { suite.tsoServer, suite.tsoServerCleanup = tests.StartSingleTSOTestServer(suite.ctx, re, backendEndpoints, tempurl.Alloc()) suite.tsoClientConn, suite.tsoClient = tso.MustNewGrpcClient(re, suite.tsoServer.GetAddr()) From 20da1c7a40f0cfa9f24ce6d897c69fe4f4c0ce23 Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Wed, 22 Nov 2023 12:53:40 +0800 Subject: [PATCH 035/137] *: make resign leader test more stable (#7402) close tikv/pd#7400 Signed-off-by: lhy1024 Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- tests/integrations/client/client_test.go | 4 ++++ .../mcs/resourcemanager/resource_manager_test.go | 2 ++ tests/integrations/mcs/tso/server_test.go | 5 +++++ tests/integrations/tso/client_test.go | 6 +++++- 4 files changed, 16 insertions(+), 1 deletion(-) diff --git a/tests/integrations/client/client_test.go b/tests/integrations/client/client_test.go index 9cabbb03090..3834d9b53bf 100644 --- a/tests/integrations/client/client_test.go +++ b/tests/integrations/client/client_test.go @@ -158,6 +158,10 @@ func TestLeaderTransfer(t *testing.T) { cluster, err := tests.NewTestCluster(ctx, 2) re.NoError(err) defer cluster.Destroy() + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/member/skipCampaignLeaderCheck", "return(true)")) + defer func() { + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/member/skipCampaignLeaderCheck")) + }() endpoints := runServer(re, cluster) cli := setupCli(re, ctx, endpoints) diff --git a/tests/integrations/mcs/resourcemanager/resource_manager_test.go b/tests/integrations/mcs/resourcemanager/resource_manager_test.go index 91a21caf91b..b98686222fe 100644 --- a/tests/integrations/mcs/resourcemanager/resource_manager_test.go +++ b/tests/integrations/mcs/resourcemanager/resource_manager_test.go @@ -63,6 +63,7 @@ func (suite *resourceManagerClientTestSuite) SetupSuite() { re := suite.Require() re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/mcs/resourcemanager/server/enableDegradedMode", `return(true)`)) + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/member/skipCampaignLeaderCheck", "return(true)")) suite.ctx, suite.clean = context.WithCancel(context.Background()) @@ -148,6 +149,7 @@ func (suite *resourceManagerClientTestSuite) TearDownSuite() { suite.cluster.Destroy() suite.clean() re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/mcs/resourcemanager/server/enableDegradedMode")) + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/member/skipCampaignLeaderCheck")) } func (suite *resourceManagerClientTestSuite) TearDownTest() { diff --git a/tests/integrations/mcs/tso/server_test.go b/tests/integrations/mcs/tso/server_test.go index 643fb3c7911..c81c39af094 100644 --- a/tests/integrations/mcs/tso/server_test.go +++ b/tests/integrations/mcs/tso/server_test.go @@ -392,6 +392,11 @@ func (suite *APIServerForwardTestSuite) TestResignAPIPrimaryForward() { defer tc.Destroy() tc.WaitForDefaultPrimaryServing(re) + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/member/skipCampaignLeaderCheck", "return(true)")) + defer func() { + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/member/skipCampaignLeaderCheck")) + }() + for j := 0; j < 10; j++ { suite.pdLeader.ResignLeader() suite.pdLeader = suite.cluster.GetServer(suite.cluster.WaitLeader()) diff --git a/tests/integrations/tso/client_test.go b/tests/integrations/tso/client_test.go index 73198690966..8ff20d68f52 100644 --- a/tests/integrations/tso/client_test.go +++ b/tests/integrations/tso/client_test.go @@ -339,6 +339,11 @@ func (suite *tsoClientTestSuite) TestUpdateAfterResetTSO() { func (suite *tsoClientTestSuite) TestRandomResignLeader() { re := suite.Require() re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/tso/fastUpdatePhysicalInterval", "return(true)")) + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/member/skipCampaignLeaderCheck", "return(true)")) + defer func() { + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/tso/fastUpdatePhysicalInterval")) + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/member/skipCampaignLeaderCheck")) + }() parallelAct := func() { // After https://github.com/tikv/pd/issues/6376 is fixed, we can use a smaller number here. @@ -376,7 +381,6 @@ func (suite *tsoClientTestSuite) TestRandomResignLeader() { } mcs.CheckMultiKeyspacesTSO(suite.ctx, re, suite.clients, parallelAct) - re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/tso/fastUpdatePhysicalInterval")) } func (suite *tsoClientTestSuite) TestRandomShutdown() { From 7783c7407365b081ea9f75479b3de1cc36c0253e Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Wed, 22 Nov 2023 15:33:02 +0800 Subject: [PATCH 036/137] make TestPrimaryChange stable (#7412) Signed-off-by: Ryan Leung --- tests/integrations/mcs/scheduling/server_test.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/integrations/mcs/scheduling/server_test.go b/tests/integrations/mcs/scheduling/server_test.go index eb99411d27e..afd213a624b 100644 --- a/tests/integrations/mcs/scheduling/server_test.go +++ b/tests/integrations/mcs/scheduling/server_test.go @@ -127,21 +127,21 @@ func (suite *serverTestSuite) TestPrimaryChange() { tc.WaitForPrimaryServing(re) primary := tc.GetPrimaryServer() oldPrimaryAddr := primary.GetAddr() - re.Len(primary.GetCluster().GetCoordinator().GetSchedulersController().GetSchedulerNames(), 5) testutil.Eventually(re, func() bool { watchedAddr, ok := suite.pdLeader.GetServicePrimaryAddr(suite.ctx, mcs.SchedulingServiceName) - return ok && oldPrimaryAddr == watchedAddr + return ok && oldPrimaryAddr == watchedAddr && + len(primary.GetCluster().GetCoordinator().GetSchedulersController().GetSchedulerNames()) == 5 }) - // transfer leader + // change primary primary.Close() tc.WaitForPrimaryServing(re) primary = tc.GetPrimaryServer() newPrimaryAddr := primary.GetAddr() re.NotEqual(oldPrimaryAddr, newPrimaryAddr) - re.Len(primary.GetCluster().GetCoordinator().GetSchedulersController().GetSchedulerNames(), 5) testutil.Eventually(re, func() bool { watchedAddr, ok := suite.pdLeader.GetServicePrimaryAddr(suite.ctx, mcs.SchedulingServiceName) - return ok && newPrimaryAddr == watchedAddr + return ok && newPrimaryAddr == watchedAddr && + len(primary.GetCluster().GetCoordinator().GetSchedulersController().GetSchedulerNames()) == 5 }) } From 8e6dcfbad0392bbacfc6fa1611457f75a8f0f2b4 Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Wed, 22 Nov 2023 15:47:41 +0800 Subject: [PATCH 037/137] api: use suitable status code and make TestResetTS stable (#7368) close tikv/pd#7411 Signed-off-by: lhy1024 Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- pkg/mcs/tso/server/apis/v1/api.go | 8 ++++++++ pkg/tso/admin.go | 8 ++++++++ server/api/admin_test.go | 23 ++++++++++++++++++++--- 3 files changed, 36 insertions(+), 3 deletions(-) diff --git a/pkg/mcs/tso/server/apis/v1/api.go b/pkg/mcs/tso/server/apis/v1/api.go index 1b8f68778af..33e1e0801aa 100644 --- a/pkg/mcs/tso/server/apis/v1/api.go +++ b/pkg/mcs/tso/server/apis/v1/api.go @@ -15,6 +15,7 @@ package apis import ( + "fmt" "net/http" "strconv" "sync" @@ -150,6 +151,7 @@ type ResetTSParams struct { // @Failure 400 {string} string "The input is invalid." // @Failure 403 {string} string "Reset ts is forbidden." // @Failure 500 {string} string "TSO server failed to proceed the request." +// @Failure 503 {string} string "It's a temporary failure, please retry." // @Router /admin/reset-ts [post] // if force-use-larger=true: // @@ -185,6 +187,12 @@ func ResetTS(c *gin.Context) { if err = svr.ResetTS(ts, ignoreSmaller, skipUpperBoundCheck, 0); err != nil { if err == errs.ErrServerNotStarted { c.String(http.StatusInternalServerError, err.Error()) + } else if err == errs.ErrEtcdTxnConflict { + // If the error is ErrEtcdTxnConflict, it means there is a temporary failure. + // Return 503 to let the client retry. + // Ref: https://datatracker.ietf.org/doc/html/rfc7231#section-6.6.4 + c.String(http.StatusServiceUnavailable, + fmt.Sprintf("It's a temporary failure with error %s, please retry.", err.Error())) } else { c.String(http.StatusForbidden, err.Error()) } diff --git a/pkg/tso/admin.go b/pkg/tso/admin.go index 7d510cdef65..f19d8e71d05 100644 --- a/pkg/tso/admin.go +++ b/pkg/tso/admin.go @@ -15,6 +15,7 @@ package tso import ( + "fmt" "net/http" "strconv" @@ -53,6 +54,7 @@ func NewAdminHandler(handler Handler, rd *render.Render) *AdminHandler { // @Failure 400 {string} string "The input is invalid." // @Failure 403 {string} string "Reset ts is forbidden." // @Failure 500 {string} string "TSO server failed to proceed the request." +// @Failure 503 {string} string "It's a temporary failure, please retry." // @Router /admin/reset-ts [post] // if force-use-larger=true: // @@ -96,6 +98,12 @@ func (h *AdminHandler) ResetTS(w http.ResponseWriter, r *http.Request) { if err = handler.ResetTS(ts, ignoreSmaller, skipUpperBoundCheck, 0); err != nil { if err == errs.ErrServerNotStarted { h.rd.JSON(w, http.StatusInternalServerError, err.Error()) + } else if err == errs.ErrEtcdTxnConflict { + // If the error is ErrEtcdTxnConflict, it means there is a temporary failure. + // Return 503 to let the client retry. + // Ref: https://datatracker.ietf.org/doc/html/rfc7231#section-6.6.4 + h.rd.JSON(w, http.StatusServiceUnavailable, + fmt.Sprintf("It's a temporary failure with error %s, please retry.", err.Error())) } else { h.rd.JSON(w, http.StatusForbidden, err.Error()) } diff --git a/server/api/admin_test.go b/server/api/admin_test.go index 09130fd8385..ad761d532ea 100644 --- a/server/api/admin_test.go +++ b/server/api/admin_test.go @@ -18,6 +18,7 @@ import ( "context" "encoding/json" "fmt" + "io" "net/http" "testing" "time" @@ -27,6 +28,7 @@ import ( "github.com/stretchr/testify/suite" "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/replication" + "github.com/tikv/pd/pkg/utils/apiutil" tu "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/server" ) @@ -188,9 +190,24 @@ func (suite *adminTestSuite) TestResetTS() { values, err := json.Marshal(args) suite.NoError(err) re := suite.Require() - err = tu.CheckPostJSON(testDialClient, url, values, - tu.StatusOK(re), - tu.StringEqual(re, "\"Reset ts successfully.\"\n")) + tu.Eventually(re, func() bool { + resp, err := apiutil.PostJSON(testDialClient, url, values) + re.NoError(err) + defer resp.Body.Close() + b, err := io.ReadAll(resp.Body) + re.NoError(err) + switch resp.StatusCode { + case http.StatusOK: + re.Contains(string(b), "Reset ts successfully.") + return true + case http.StatusServiceUnavailable: + re.Contains(string(b), "[PD:etcd:ErrEtcdTxnConflict]etcd transaction failed, conflicted and rolled back") + return false + default: + re.FailNow("unexpected status code %d", resp.StatusCode) + return false + } + }) suite.NoError(err) t2 := makeTS(32 * time.Hour) args["tso"] = fmt.Sprintf("%d", t2) From fe848299f28a11d47e11bd93b506d6d0f1c0477f Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Wed, 22 Nov 2023 17:17:42 +0800 Subject: [PATCH 038/137] schedulers: add reload config function (#7406) close tikv/pd#7257 Signed-off-by: lhy1024 --- pkg/schedule/schedulers/balance_leader.go | 22 +- pkg/schedule/schedulers/balance_region.go | 1 + pkg/schedule/schedulers/balance_witness.go | 22 +- pkg/schedule/schedulers/evict_leader.go | 72 +++---- pkg/schedule/schedulers/evict_slow_store.go | 75 +++++-- pkg/schedule/schedulers/evict_slow_trend.go | 89 ++++++-- pkg/schedule/schedulers/grant_hot_region.go | 45 ++-- pkg/schedule/schedulers/grant_leader.go | 52 ++--- pkg/schedule/schedulers/hot_region.go | 38 ++++ pkg/schedule/schedulers/init.go | 2 + pkg/schedule/schedulers/label.go | 1 + pkg/schedule/schedulers/random_merge.go | 1 + pkg/schedule/schedulers/scatter_range.go | 47 ++-- .../schedulers/scheduler_controller.go | 3 +- pkg/schedule/schedulers/shuffle_hot_region.go | 106 ++++++++- pkg/schedule/schedulers/shuffle_leader.go | 1 + pkg/schedule/schedulers/split_bucket.go | 22 +- .../schedulers/transfer_witness_leader.go | 1 + pkg/schedule/schedulers/utils.go | 16 ++ pkg/window/policy.go | 5 +- server/cluster/scheduling_controller.go | 3 +- tests/pdctl/scheduler/scheduler_test.go | 202 ++++++++++-------- tests/server/api/scheduler_test.go | 38 ++-- tools/pd-ctl/pdctl/command/scheduler.go | 20 ++ tools/pd-simulator/simulator/node.go | 5 +- 25 files changed, 605 insertions(+), 284 deletions(-) diff --git a/pkg/schedule/schedulers/balance_leader.go b/pkg/schedule/schedulers/balance_leader.go index e5516317f46..6a64edf7e70 100644 --- a/pkg/schedule/schedulers/balance_leader.go +++ b/pkg/schedule/schedulers/balance_leader.go @@ -67,7 +67,7 @@ var ( ) type balanceLeaderSchedulerConfig struct { - mu syncutil.RWMutex + syncutil.RWMutex storage endpoint.ConfigStorage Ranges []core.KeyRange `json:"ranges"` // Batch is used to generate multiple operators by one scheduling @@ -75,8 +75,8 @@ type balanceLeaderSchedulerConfig struct { } func (conf *balanceLeaderSchedulerConfig) Update(data []byte) (int, interface{}) { - conf.mu.Lock() - defer conf.mu.Unlock() + conf.Lock() + defer conf.Unlock() oldc, _ := json.Marshal(conf) @@ -109,8 +109,8 @@ func (conf *balanceLeaderSchedulerConfig) validate() bool { } func (conf *balanceLeaderSchedulerConfig) Clone() *balanceLeaderSchedulerConfig { - conf.mu.RLock() - defer conf.mu.RUnlock() + conf.RLock() + defer conf.RUnlock() ranges := make([]core.KeyRange, len(conf.Ranges)) copy(ranges, conf.Ranges) return &balanceLeaderSchedulerConfig{ @@ -210,14 +210,14 @@ func (l *balanceLeaderScheduler) GetType() string { } func (l *balanceLeaderScheduler) EncodeConfig() ([]byte, error) { - l.conf.mu.RLock() - defer l.conf.mu.RUnlock() + l.conf.RLock() + defer l.conf.RUnlock() return EncodeConfig(l.conf) } func (l *balanceLeaderScheduler) ReloadConfig() error { - l.conf.mu.Lock() - defer l.conf.mu.Unlock() + l.conf.Lock() + defer l.conf.Unlock() cfgData, err := l.conf.storage.LoadSchedulerConfig(l.GetName()) if err != nil { return err @@ -335,8 +335,8 @@ func (cs *candidateStores) resortStoreWithPos(pos int) { } func (l *balanceLeaderScheduler) Schedule(cluster sche.SchedulerCluster, dryRun bool) ([]*operator.Operator, []plan.Plan) { - l.conf.mu.RLock() - defer l.conf.mu.RUnlock() + l.conf.RLock() + defer l.conf.RUnlock() basePlan := plan.NewBalanceSchedulerPlan() var collector *plan.Collector if dryRun { diff --git a/pkg/schedule/schedulers/balance_region.go b/pkg/schedule/schedulers/balance_region.go index 1343600af06..1cef3a4615b 100644 --- a/pkg/schedule/schedulers/balance_region.go +++ b/pkg/schedule/schedulers/balance_region.go @@ -51,6 +51,7 @@ var ( type balanceRegionSchedulerConfig struct { Name string `json:"name"` Ranges []core.KeyRange `json:"ranges"` + // TODO: When we prepare to use Ranges, we will need to implement the ReloadConfig function for this scheduler. } type balanceRegionScheduler struct { diff --git a/pkg/schedule/schedulers/balance_witness.go b/pkg/schedule/schedulers/balance_witness.go index e9bab6c1bc7..bf3fbbb83da 100644 --- a/pkg/schedule/schedulers/balance_witness.go +++ b/pkg/schedule/schedulers/balance_witness.go @@ -53,7 +53,7 @@ const ( ) type balanceWitnessSchedulerConfig struct { - mu syncutil.RWMutex + syncutil.RWMutex storage endpoint.ConfigStorage Ranges []core.KeyRange `json:"ranges"` // Batch is used to generate multiple operators by one scheduling @@ -61,8 +61,8 @@ type balanceWitnessSchedulerConfig struct { } func (conf *balanceWitnessSchedulerConfig) Update(data []byte) (int, interface{}) { - conf.mu.Lock() - defer conf.mu.Unlock() + conf.Lock() + defer conf.Unlock() oldc, _ := json.Marshal(conf) @@ -95,8 +95,8 @@ func (conf *balanceWitnessSchedulerConfig) validate() bool { } func (conf *balanceWitnessSchedulerConfig) Clone() *balanceWitnessSchedulerConfig { - conf.mu.RLock() - defer conf.mu.RUnlock() + conf.RLock() + defer conf.RUnlock() ranges := make([]core.KeyRange, len(conf.Ranges)) copy(ranges, conf.Ranges) return &balanceWitnessSchedulerConfig{ @@ -205,14 +205,14 @@ func (b *balanceWitnessScheduler) GetType() string { } func (b *balanceWitnessScheduler) EncodeConfig() ([]byte, error) { - b.conf.mu.RLock() - defer b.conf.mu.RUnlock() + b.conf.RLock() + defer b.conf.RUnlock() return EncodeConfig(b.conf) } func (b *balanceWitnessScheduler) ReloadConfig() error { - b.conf.mu.Lock() - defer b.conf.mu.Unlock() + b.conf.Lock() + defer b.conf.Unlock() cfgData, err := b.conf.storage.LoadSchedulerConfig(b.GetName()) if err != nil { return err @@ -238,8 +238,8 @@ func (b *balanceWitnessScheduler) IsScheduleAllowed(cluster sche.SchedulerCluste } func (b *balanceWitnessScheduler) Schedule(cluster sche.SchedulerCluster, dryRun bool) ([]*operator.Operator, []plan.Plan) { - b.conf.mu.RLock() - defer b.conf.mu.RUnlock() + b.conf.RLock() + defer b.conf.RUnlock() basePlan := plan.NewBalanceSchedulerPlan() var collector *plan.Collector if dryRun { diff --git a/pkg/schedule/schedulers/evict_leader.go b/pkg/schedule/schedulers/evict_leader.go index 332002043a3..879aa9869b3 100644 --- a/pkg/schedule/schedulers/evict_leader.go +++ b/pkg/schedule/schedulers/evict_leader.go @@ -56,7 +56,7 @@ var ( ) type evictLeaderSchedulerConfig struct { - mu syncutil.RWMutex + syncutil.RWMutex storage endpoint.ConfigStorage StoreIDWithRanges map[uint64][]core.KeyRange `json:"store-id-ranges"` cluster *core.BasicCluster @@ -64,8 +64,8 @@ type evictLeaderSchedulerConfig struct { } func (conf *evictLeaderSchedulerConfig) getStores() []uint64 { - conf.mu.RLock() - defer conf.mu.RUnlock() + conf.RLock() + defer conf.RUnlock() stores := make([]uint64, 0, len(conf.StoreIDWithRanges)) for storeID := range conf.StoreIDWithRanges { stores = append(stores, storeID) @@ -86,15 +86,15 @@ func (conf *evictLeaderSchedulerConfig) BuildWithArgs(args []string) error { if err != nil { return err } - conf.mu.Lock() - defer conf.mu.Unlock() + conf.Lock() + defer conf.Unlock() conf.StoreIDWithRanges[id] = ranges return nil } func (conf *evictLeaderSchedulerConfig) Clone() *evictLeaderSchedulerConfig { - conf.mu.RLock() - defer conf.mu.RUnlock() + conf.RLock() + defer conf.RUnlock() storeIDWithRanges := make(map[uint64][]core.KeyRange) for id, ranges := range conf.StoreIDWithRanges { storeIDWithRanges[id] = append(storeIDWithRanges[id], ranges...) @@ -106,8 +106,8 @@ func (conf *evictLeaderSchedulerConfig) Clone() *evictLeaderSchedulerConfig { func (conf *evictLeaderSchedulerConfig) Persist() error { name := conf.getSchedulerName() - conf.mu.RLock() - defer conf.mu.RUnlock() + conf.RLock() + defer conf.RUnlock() data, err := EncodeConfig(conf) failpoint.Inject("persistFail", func() { err = errors.New("fail to persist") @@ -123,8 +123,8 @@ func (conf *evictLeaderSchedulerConfig) getSchedulerName() string { } func (conf *evictLeaderSchedulerConfig) getRanges(id uint64) []string { - conf.mu.RLock() - defer conf.mu.RUnlock() + conf.RLock() + defer conf.RUnlock() ranges := conf.StoreIDWithRanges[id] res := make([]string, 0, len(ranges)*2) for index := range ranges { @@ -134,8 +134,8 @@ func (conf *evictLeaderSchedulerConfig) getRanges(id uint64) []string { } func (conf *evictLeaderSchedulerConfig) removeStore(id uint64) (succ bool, last bool) { - conf.mu.Lock() - defer conf.mu.Unlock() + conf.Lock() + defer conf.Unlock() _, exists := conf.StoreIDWithRanges[id] succ, last = false, false if exists { @@ -148,15 +148,15 @@ func (conf *evictLeaderSchedulerConfig) removeStore(id uint64) (succ bool, last } func (conf *evictLeaderSchedulerConfig) resetStore(id uint64, keyRange []core.KeyRange) { - conf.mu.Lock() - defer conf.mu.Unlock() + conf.Lock() + defer conf.Unlock() conf.cluster.PauseLeaderTransfer(id) conf.StoreIDWithRanges[id] = keyRange } func (conf *evictLeaderSchedulerConfig) getKeyRangesByID(id uint64) []core.KeyRange { - conf.mu.RLock() - defer conf.mu.RUnlock() + conf.RLock() + defer conf.RUnlock() if ranges, exist := conf.StoreIDWithRanges[id]; exist { return ranges } @@ -199,14 +199,14 @@ func (s *evictLeaderScheduler) GetType() string { } func (s *evictLeaderScheduler) EncodeConfig() ([]byte, error) { - s.conf.mu.RLock() - defer s.conf.mu.RUnlock() + s.conf.RLock() + defer s.conf.RUnlock() return EncodeConfig(s.conf) } func (s *evictLeaderScheduler) ReloadConfig() error { - s.conf.mu.Lock() - defer s.conf.mu.Unlock() + s.conf.Lock() + defer s.conf.Unlock() cfgData, err := s.conf.storage.LoadSchedulerConfig(s.GetName()) if err != nil { return err @@ -223,25 +223,9 @@ func (s *evictLeaderScheduler) ReloadConfig() error { return nil } -// pauseAndResumeLeaderTransfer checks the old and new store IDs, and pause or resume the leader transfer. -func pauseAndResumeLeaderTransfer(cluster *core.BasicCluster, old, new map[uint64][]core.KeyRange) { - for id := range old { - if _, ok := new[id]; ok { - continue - } - cluster.ResumeLeaderTransfer(id) - } - for id := range new { - if _, ok := old[id]; ok { - continue - } - cluster.PauseLeaderTransfer(id) - } -} - func (s *evictLeaderScheduler) PrepareConfig(cluster sche.SchedulerCluster) error { - s.conf.mu.RLock() - defer s.conf.mu.RUnlock() + s.conf.RLock() + defer s.conf.RUnlock() var res error for id := range s.conf.StoreIDWithRanges { if err := cluster.PauseLeaderTransfer(id); err != nil { @@ -252,8 +236,8 @@ func (s *evictLeaderScheduler) PrepareConfig(cluster sche.SchedulerCluster) erro } func (s *evictLeaderScheduler) CleanConfig(cluster sche.SchedulerCluster) { - s.conf.mu.RLock() - defer s.conf.mu.RUnlock() + s.conf.RLock() + defer s.conf.RUnlock() for id := range s.conf.StoreIDWithRanges { cluster.ResumeLeaderTransfer(id) } @@ -382,15 +366,15 @@ func (handler *evictLeaderHandler) UpdateConfig(w http.ResponseWriter, r *http.R idFloat, ok := input["store_id"].(float64) if ok { id = (uint64)(idFloat) - handler.config.mu.RLock() + handler.config.RLock() if _, exists = handler.config.StoreIDWithRanges[id]; !exists { if err := handler.config.cluster.PauseLeaderTransfer(id); err != nil { - handler.config.mu.RUnlock() + handler.config.RUnlock() handler.rd.JSON(w, http.StatusInternalServerError, err.Error()) return } } - handler.config.mu.RUnlock() + handler.config.RUnlock() args = append(args, strconv.FormatUint(id, 10)) } diff --git a/pkg/schedule/schedulers/evict_slow_store.go b/pkg/schedule/schedulers/evict_slow_store.go index 563f9f68c45..713920828cc 100644 --- a/pkg/schedule/schedulers/evict_slow_store.go +++ b/pkg/schedule/schedulers/evict_slow_store.go @@ -16,7 +16,6 @@ package schedulers import ( "net/http" - "sync/atomic" "time" "github.com/gorilla/mux" @@ -29,6 +28,7 @@ import ( "github.com/tikv/pd/pkg/schedule/plan" "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/utils/apiutil" + "github.com/tikv/pd/pkg/utils/syncutil" "github.com/unrolled/render" "go.uber.org/zap" ) @@ -47,6 +47,8 @@ const ( var evictSlowStoreCounter = schedulerCounter.WithLabelValues(EvictSlowStoreName, "schedule") type evictSlowStoreSchedulerConfig struct { + syncutil.RWMutex + cluster *core.BasicCluster storage endpoint.ConfigStorage // Last timestamp of the chosen slow store for eviction. lastSlowStoreCaptureTS time.Time @@ -65,13 +67,15 @@ func initEvictSlowStoreSchedulerConfig(storage endpoint.ConfigStorage) *evictSlo } func (conf *evictSlowStoreSchedulerConfig) Clone() *evictSlowStoreSchedulerConfig { + conf.RLock() + defer conf.RUnlock() return &evictSlowStoreSchedulerConfig{ - RecoveryDurationGap: atomic.LoadUint64(&conf.RecoveryDurationGap), + RecoveryDurationGap: conf.RecoveryDurationGap, } } -func (conf *evictSlowStoreSchedulerConfig) Persist() error { - name := conf.getSchedulerName() +func (conf *evictSlowStoreSchedulerConfig) persistLocked() error { + name := EvictSlowStoreName data, err := EncodeConfig(conf) failpoint.Inject("persistFail", func() { err = errors.New("fail to persist") @@ -82,11 +86,9 @@ func (conf *evictSlowStoreSchedulerConfig) Persist() error { return conf.storage.SaveSchedulerConfig(name, data) } -func (conf *evictSlowStoreSchedulerConfig) getSchedulerName() string { - return EvictSlowStoreName -} - func (conf *evictSlowStoreSchedulerConfig) getStores() []uint64 { + conf.RLock() + defer conf.RUnlock() return conf.EvictedStores } @@ -98,15 +100,17 @@ func (conf *evictSlowStoreSchedulerConfig) getKeyRangesByID(id uint64) []core.Ke } func (conf *evictSlowStoreSchedulerConfig) evictStore() uint64 { - if len(conf.EvictedStores) == 0 { + if len(conf.getStores()) == 0 { return 0 } - return conf.EvictedStores[0] + return conf.getStores()[0] } // readyForRecovery checks whether the last cpatured candidate is ready for recovery. func (conf *evictSlowStoreSchedulerConfig) readyForRecovery() bool { - recoveryDurationGap := atomic.LoadUint64(&conf.RecoveryDurationGap) + conf.RLock() + defer conf.RUnlock() + recoveryDurationGap := conf.RecoveryDurationGap failpoint.Inject("transientRecoveryGap", func() { recoveryDurationGap = 0 }) @@ -114,17 +118,21 @@ func (conf *evictSlowStoreSchedulerConfig) readyForRecovery() bool { } func (conf *evictSlowStoreSchedulerConfig) setStoreAndPersist(id uint64) error { + conf.Lock() + defer conf.Unlock() conf.EvictedStores = []uint64{id} conf.lastSlowStoreCaptureTS = time.Now() - return conf.Persist() + return conf.persistLocked() } func (conf *evictSlowStoreSchedulerConfig) clearAndPersist() (oldID uint64, err error) { oldID = conf.evictStore() + conf.Lock() + defer conf.Unlock() if oldID > 0 { conf.EvictedStores = []uint64{} conf.lastSlowStoreCaptureTS = time.Time{} - err = conf.Persist() + err = conf.persistLocked() } return } @@ -155,9 +163,16 @@ func (handler *evictSlowStoreHandler) UpdateConfig(w http.ResponseWriter, r *htt handler.rd.JSON(w, http.StatusInternalServerError, errors.New("invalid argument for 'recovery-duration'").Error()) return } - recoveryDurationGap := (uint64)(recoveryDurationGapFloat) - prevRecoveryDurationGap := atomic.LoadUint64(&handler.config.RecoveryDurationGap) - atomic.StoreUint64(&handler.config.RecoveryDurationGap, recoveryDurationGap) + handler.config.Lock() + defer handler.config.Unlock() + prevRecoveryDurationGap := handler.config.RecoveryDurationGap + recoveryDurationGap := uint64(recoveryDurationGapFloat) + handler.config.RecoveryDurationGap = recoveryDurationGap + if err := handler.config.persistLocked(); err != nil { + handler.rd.JSON(w, http.StatusInternalServerError, err.Error()) + handler.config.RecoveryDurationGap = prevRecoveryDurationGap + return + } log.Info("evict-slow-store-scheduler update 'recovery-duration' - unit: s", zap.Uint64("prev", prevRecoveryDurationGap), zap.Uint64("cur", recoveryDurationGap)) handler.rd.JSON(w, http.StatusOK, nil) } @@ -189,6 +204,34 @@ func (s *evictSlowStoreScheduler) EncodeConfig() ([]byte, error) { return EncodeConfig(s.conf) } +func (s *evictSlowStoreScheduler) ReloadConfig() error { + s.conf.Lock() + defer s.conf.Unlock() + cfgData, err := s.conf.storage.LoadSchedulerConfig(s.GetName()) + if err != nil { + return err + } + if len(cfgData) == 0 { + return nil + } + newCfg := &evictSlowStoreSchedulerConfig{} + if err = DecodeConfig([]byte(cfgData), newCfg); err != nil { + return err + } + old := make(map[uint64]struct{}) + for _, id := range s.conf.EvictedStores { + old[id] = struct{}{} + } + new := make(map[uint64]struct{}) + for _, id := range newCfg.EvictedStores { + new[id] = struct{}{} + } + pauseAndResumeLeaderTransfer(s.conf.cluster, old, new) + s.conf.RecoveryDurationGap = newCfg.RecoveryDurationGap + s.conf.EvictedStores = newCfg.EvictedStores + return nil +} + func (s *evictSlowStoreScheduler) PrepareConfig(cluster sche.SchedulerCluster) error { evictStore := s.conf.evictStore() if evictStore != 0 { diff --git a/pkg/schedule/schedulers/evict_slow_trend.go b/pkg/schedule/schedulers/evict_slow_trend.go index 0d2c10e2bfe..53e096baec7 100644 --- a/pkg/schedule/schedulers/evict_slow_trend.go +++ b/pkg/schedule/schedulers/evict_slow_trend.go @@ -17,7 +17,6 @@ package schedulers import ( "net/http" "strconv" - "sync/atomic" "time" "github.com/gorilla/mux" @@ -30,6 +29,7 @@ import ( "github.com/tikv/pd/pkg/schedule/plan" "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/utils/apiutil" + "github.com/tikv/pd/pkg/utils/syncutil" "github.com/unrolled/render" "go.uber.org/zap" ) @@ -54,6 +54,8 @@ type slowCandidate struct { } type evictSlowTrendSchedulerConfig struct { + syncutil.RWMutex + cluster *core.BasicCluster storage endpoint.ConfigStorage // Candidate for eviction in current tick. evictCandidate slowCandidate @@ -76,13 +78,15 @@ func initEvictSlowTrendSchedulerConfig(storage endpoint.ConfigStorage) *evictSlo } func (conf *evictSlowTrendSchedulerConfig) Clone() *evictSlowTrendSchedulerConfig { + conf.RLock() + defer conf.RUnlock() return &evictSlowTrendSchedulerConfig{ - RecoveryDurationGap: atomic.LoadUint64(&conf.RecoveryDurationGap), + RecoveryDurationGap: conf.RecoveryDurationGap, } } -func (conf *evictSlowTrendSchedulerConfig) Persist() error { - name := conf.getSchedulerName() +func (conf *evictSlowTrendSchedulerConfig) persistLocked() error { + name := EvictSlowTrendName data, err := EncodeConfig(conf) failpoint.Inject("persistFail", func() { err = errors.New("fail to persist") @@ -93,11 +97,9 @@ func (conf *evictSlowTrendSchedulerConfig) Persist() error { return conf.storage.SaveSchedulerConfig(name, data) } -func (conf *evictSlowTrendSchedulerConfig) getSchedulerName() string { - return EvictSlowTrendName -} - func (conf *evictSlowTrendSchedulerConfig) getStores() []uint64 { + conf.RLock() + defer conf.RUnlock() return conf.EvictedStores } @@ -109,6 +111,8 @@ func (conf *evictSlowTrendSchedulerConfig) getKeyRangesByID(id uint64) []core.Ke } func (conf *evictSlowTrendSchedulerConfig) hasEvictedStores() bool { + conf.RLock() + defer conf.RUnlock() return len(conf.EvictedStores) > 0 } @@ -116,6 +120,8 @@ func (conf *evictSlowTrendSchedulerConfig) evictedStore() uint64 { if !conf.hasEvictedStores() { return 0 } + conf.RLock() + defer conf.RUnlock() // If a candidate passes all checks and proved to be slow, it will be // recorded in `conf.EvictStores`, and `conf.lastEvictCandidate` will record // the captured timestamp of this store. @@ -123,18 +129,26 @@ func (conf *evictSlowTrendSchedulerConfig) evictedStore() uint64 { } func (conf *evictSlowTrendSchedulerConfig) candidate() uint64 { + conf.RLock() + defer conf.RUnlock() return conf.evictCandidate.storeID } func (conf *evictSlowTrendSchedulerConfig) captureTS() time.Time { + conf.RLock() + defer conf.RUnlock() return conf.evictCandidate.captureTS } func (conf *evictSlowTrendSchedulerConfig) candidateCapturedSecs() uint64 { + conf.RLock() + defer conf.RUnlock() return DurationSinceAsSecs(conf.evictCandidate.captureTS) } func (conf *evictSlowTrendSchedulerConfig) lastCapturedCandidate() *slowCandidate { + conf.RLock() + defer conf.RUnlock() return &conf.lastEvictCandidate } @@ -144,7 +158,9 @@ func (conf *evictSlowTrendSchedulerConfig) lastCandidateCapturedSecs() uint64 { // readyForRecovery checks whether the last cpatured candidate is ready for recovery. func (conf *evictSlowTrendSchedulerConfig) readyForRecovery() bool { - recoveryDurationGap := atomic.LoadUint64(&conf.RecoveryDurationGap) + conf.RLock() + defer conf.RUnlock() + recoveryDurationGap := conf.RecoveryDurationGap failpoint.Inject("transientRecoveryGap", func() { recoveryDurationGap = 0 }) @@ -152,6 +168,8 @@ func (conf *evictSlowTrendSchedulerConfig) readyForRecovery() bool { } func (conf *evictSlowTrendSchedulerConfig) captureCandidate(id uint64) { + conf.Lock() + defer conf.Unlock() conf.evictCandidate = slowCandidate{ storeID: id, captureTS: time.Now(), @@ -163,6 +181,8 @@ func (conf *evictSlowTrendSchedulerConfig) captureCandidate(id uint64) { } func (conf *evictSlowTrendSchedulerConfig) popCandidate(updLast bool) uint64 { + conf.Lock() + defer conf.Unlock() id := conf.evictCandidate.storeID if updLast { conf.lastEvictCandidate = conf.evictCandidate @@ -172,14 +192,18 @@ func (conf *evictSlowTrendSchedulerConfig) popCandidate(updLast bool) uint64 { } func (conf *evictSlowTrendSchedulerConfig) markCandidateRecovered() { + conf.Lock() + defer conf.Unlock() if conf.lastEvictCandidate != (slowCandidate{}) { conf.lastEvictCandidate.recoverTS = time.Now() } } func (conf *evictSlowTrendSchedulerConfig) setStoreAndPersist(id uint64) error { + conf.Lock() + defer conf.Unlock() conf.EvictedStores = []uint64{id} - return conf.Persist() + return conf.persistLocked() } func (conf *evictSlowTrendSchedulerConfig) clearAndPersist(cluster sche.SchedulerCluster) (oldID uint64, err error) { @@ -193,8 +217,10 @@ func (conf *evictSlowTrendSchedulerConfig) clearAndPersist(cluster sche.Schedule address = store.GetAddress() } storeSlowTrendEvictedStatusGauge.WithLabelValues(address, strconv.FormatUint(oldID, 10)).Set(0) + conf.Lock() + defer conf.Unlock() conf.EvictedStores = []uint64{} - return oldID, conf.Persist() + return oldID, conf.persistLocked() } type evictSlowTrendHandler struct { @@ -223,9 +249,16 @@ func (handler *evictSlowTrendHandler) UpdateConfig(w http.ResponseWriter, r *htt handler.rd.JSON(w, http.StatusInternalServerError, errors.New("invalid argument for 'recovery-duration'").Error()) return } - recoveryDurationGap := (uint64)(recoveryDurationGapFloat) - prevRecoveryDurationGap := atomic.LoadUint64(&handler.config.RecoveryDurationGap) - atomic.StoreUint64(&handler.config.RecoveryDurationGap, recoveryDurationGap) + handler.config.Lock() + defer handler.config.Unlock() + prevRecoveryDurationGap := handler.config.RecoveryDurationGap + recoveryDurationGap := uint64(recoveryDurationGapFloat) + handler.config.RecoveryDurationGap = recoveryDurationGap + if err := handler.config.persistLocked(); err != nil { + handler.rd.JSON(w, http.StatusInternalServerError, err.Error()) + handler.config.RecoveryDurationGap = prevRecoveryDurationGap + return + } log.Info("evict-slow-trend-scheduler update 'recovery-duration' - unit: s", zap.Uint64("prev", prevRecoveryDurationGap), zap.Uint64("cur", recoveryDurationGap)) handler.rd.JSON(w, http.StatusOK, nil) } @@ -270,6 +303,34 @@ func (s *evictSlowTrendScheduler) EncodeConfig() ([]byte, error) { return EncodeConfig(s.conf) } +func (s *evictSlowTrendScheduler) ReloadConfig() error { + s.conf.Lock() + defer s.conf.Unlock() + cfgData, err := s.conf.storage.LoadSchedulerConfig(s.GetName()) + if err != nil { + return err + } + if len(cfgData) == 0 { + return nil + } + newCfg := &evictSlowTrendSchedulerConfig{} + if err = DecodeConfig([]byte(cfgData), newCfg); err != nil { + return err + } + old := make(map[uint64]struct{}) + for _, id := range s.conf.EvictedStores { + old[id] = struct{}{} + } + new := make(map[uint64]struct{}) + for _, id := range newCfg.EvictedStores { + new[id] = struct{}{} + } + pauseAndResumeLeaderTransfer(s.conf.cluster, old, new) + s.conf.RecoveryDurationGap = newCfg.RecoveryDurationGap + s.conf.EvictedStores = newCfg.EvictedStores + return nil +} + func (s *evictSlowTrendScheduler) PrepareConfig(cluster sche.SchedulerCluster) error { evictedStoreID := s.conf.evictedStore() if evictedStoreID == 0 { diff --git a/pkg/schedule/schedulers/grant_hot_region.go b/pkg/schedule/schedulers/grant_hot_region.go index 5a68da069b8..6ab689ea5d4 100644 --- a/pkg/schedule/schedulers/grant_hot_region.go +++ b/pkg/schedule/schedulers/grant_hot_region.go @@ -54,7 +54,7 @@ var ( ) type grantHotRegionSchedulerConfig struct { - mu syncutil.RWMutex + syncutil.RWMutex storage endpoint.ConfigStorage cluster *core.BasicCluster StoreIDs []uint64 `json:"store-id"` @@ -62,8 +62,8 @@ type grantHotRegionSchedulerConfig struct { } func (conf *grantHotRegionSchedulerConfig) setStore(leaderID uint64, peers []uint64) bool { - conf.mu.Lock() - defer conf.mu.Unlock() + conf.Lock() + defer conf.Unlock() ret := slice.AnyOf(peers, func(i int) bool { return leaderID == peers[i] }) @@ -75,20 +75,20 @@ func (conf *grantHotRegionSchedulerConfig) setStore(leaderID uint64, peers []uin } func (conf *grantHotRegionSchedulerConfig) GetStoreLeaderID() uint64 { - conf.mu.RLock() - defer conf.mu.RUnlock() + conf.RLock() + defer conf.RUnlock() return conf.StoreLeaderID } func (conf *grantHotRegionSchedulerConfig) SetStoreLeaderID(id uint64) { - conf.mu.Lock() - defer conf.mu.Unlock() + conf.Lock() + defer conf.Unlock() conf.StoreLeaderID = id } func (conf *grantHotRegionSchedulerConfig) Clone() *grantHotRegionSchedulerConfig { - conf.mu.RLock() - defer conf.mu.RUnlock() + conf.RLock() + defer conf.RUnlock() newStoreIDs := make([]uint64, len(conf.StoreIDs)) copy(newStoreIDs, conf.StoreIDs) return &grantHotRegionSchedulerConfig{ @@ -99,8 +99,8 @@ func (conf *grantHotRegionSchedulerConfig) Clone() *grantHotRegionSchedulerConfi func (conf *grantHotRegionSchedulerConfig) Persist() error { name := conf.getSchedulerName() - conf.mu.RLock() - defer conf.mu.RUnlock() + conf.RLock() + defer conf.RUnlock() data, err := EncodeConfig(conf) if err != nil { return err @@ -113,8 +113,8 @@ func (conf *grantHotRegionSchedulerConfig) getSchedulerName() string { } func (conf *grantHotRegionSchedulerConfig) has(storeID uint64) bool { - conf.mu.RLock() - defer conf.mu.RUnlock() + conf.RLock() + defer conf.RUnlock() return slice.AnyOf(conf.StoreIDs, func(i int) bool { return storeID == conf.StoreIDs[i] }) @@ -151,6 +151,25 @@ func (s *grantHotRegionScheduler) EncodeConfig() ([]byte, error) { return EncodeConfig(s.conf) } +func (s *grantHotRegionScheduler) ReloadConfig() error { + s.conf.Lock() + defer s.conf.Unlock() + cfgData, err := s.conf.storage.LoadSchedulerConfig(s.GetName()) + if err != nil { + return err + } + if len(cfgData) == 0 { + return nil + } + newCfg := &grantHotRegionSchedulerConfig{} + if err := DecodeConfig([]byte(cfgData), newCfg); err != nil { + return err + } + s.conf.StoreIDs = newCfg.StoreIDs + s.conf.StoreLeaderID = newCfg.StoreLeaderID + return nil +} + // IsScheduleAllowed returns whether the scheduler is allowed to schedule. // TODO it should check if there is any scheduler such as evict or hot region scheduler func (s *grantHotRegionScheduler) IsScheduleAllowed(cluster sche.SchedulerCluster) bool { diff --git a/pkg/schedule/schedulers/grant_leader.go b/pkg/schedule/schedulers/grant_leader.go index 84f830f368b..47e14af4902 100644 --- a/pkg/schedule/schedulers/grant_leader.go +++ b/pkg/schedule/schedulers/grant_leader.go @@ -49,7 +49,7 @@ var ( ) type grantLeaderSchedulerConfig struct { - mu syncutil.RWMutex + syncutil.RWMutex storage endpoint.ConfigStorage StoreIDWithRanges map[uint64][]core.KeyRange `json:"store-id-ranges"` cluster *core.BasicCluster @@ -69,15 +69,15 @@ func (conf *grantLeaderSchedulerConfig) BuildWithArgs(args []string) error { if err != nil { return err } - conf.mu.Lock() - defer conf.mu.Unlock() + conf.Lock() + defer conf.Unlock() conf.StoreIDWithRanges[id] = ranges return nil } func (conf *grantLeaderSchedulerConfig) Clone() *grantLeaderSchedulerConfig { - conf.mu.RLock() - defer conf.mu.RUnlock() + conf.RLock() + defer conf.RUnlock() newStoreIDWithRanges := make(map[uint64][]core.KeyRange) for k, v := range conf.StoreIDWithRanges { newStoreIDWithRanges[k] = v @@ -89,8 +89,8 @@ func (conf *grantLeaderSchedulerConfig) Clone() *grantLeaderSchedulerConfig { func (conf *grantLeaderSchedulerConfig) Persist() error { name := conf.getSchedulerName() - conf.mu.RLock() - defer conf.mu.RUnlock() + conf.RLock() + defer conf.RUnlock() data, err := EncodeConfig(conf) if err != nil { return err @@ -103,8 +103,8 @@ func (conf *grantLeaderSchedulerConfig) getSchedulerName() string { } func (conf *grantLeaderSchedulerConfig) getRanges(id uint64) []string { - conf.mu.RLock() - defer conf.mu.RUnlock() + conf.RLock() + defer conf.RUnlock() ranges := conf.StoreIDWithRanges[id] res := make([]string, 0, len(ranges)*2) for index := range ranges { @@ -114,8 +114,8 @@ func (conf *grantLeaderSchedulerConfig) getRanges(id uint64) []string { } func (conf *grantLeaderSchedulerConfig) removeStore(id uint64) (succ bool, last bool) { - conf.mu.Lock() - defer conf.mu.Unlock() + conf.Lock() + defer conf.Unlock() _, exists := conf.StoreIDWithRanges[id] succ, last = false, false if exists { @@ -128,15 +128,15 @@ func (conf *grantLeaderSchedulerConfig) removeStore(id uint64) (succ bool, last } func (conf *grantLeaderSchedulerConfig) resetStore(id uint64, keyRange []core.KeyRange) { - conf.mu.Lock() - defer conf.mu.Unlock() + conf.Lock() + defer conf.Unlock() conf.cluster.PauseLeaderTransfer(id) conf.StoreIDWithRanges[id] = keyRange } func (conf *grantLeaderSchedulerConfig) getKeyRangesByID(id uint64) []core.KeyRange { - conf.mu.RLock() - defer conf.mu.RUnlock() + conf.RLock() + defer conf.RUnlock() if ranges, exist := conf.StoreIDWithRanges[id]; exist { return ranges } @@ -179,8 +179,8 @@ func (s *grantLeaderScheduler) EncodeConfig() ([]byte, error) { } func (s *grantLeaderScheduler) ReloadConfig() error { - s.conf.mu.Lock() - defer s.conf.mu.Unlock() + s.conf.Lock() + defer s.conf.Unlock() cfgData, err := s.conf.storage.LoadSchedulerConfig(s.GetName()) if err != nil { return err @@ -198,8 +198,8 @@ func (s *grantLeaderScheduler) ReloadConfig() error { } func (s *grantLeaderScheduler) PrepareConfig(cluster sche.SchedulerCluster) error { - s.conf.mu.RLock() - defer s.conf.mu.RUnlock() + s.conf.RLock() + defer s.conf.RUnlock() var res error for id := range s.conf.StoreIDWithRanges { if err := cluster.PauseLeaderTransfer(id); err != nil { @@ -210,8 +210,8 @@ func (s *grantLeaderScheduler) PrepareConfig(cluster sche.SchedulerCluster) erro } func (s *grantLeaderScheduler) CleanConfig(cluster sche.SchedulerCluster) { - s.conf.mu.RLock() - defer s.conf.mu.RUnlock() + s.conf.RLock() + defer s.conf.RUnlock() for id := range s.conf.StoreIDWithRanges { cluster.ResumeLeaderTransfer(id) } @@ -227,8 +227,8 @@ func (s *grantLeaderScheduler) IsScheduleAllowed(cluster sche.SchedulerCluster) func (s *grantLeaderScheduler) Schedule(cluster sche.SchedulerCluster, dryRun bool) ([]*operator.Operator, []plan.Plan) { grantLeaderCounter.Inc() - s.conf.mu.RLock() - defer s.conf.mu.RUnlock() + s.conf.RLock() + defer s.conf.RUnlock() ops := make([]*operator.Operator, 0, len(s.conf.StoreIDWithRanges)) pendingFilter := filter.NewRegionPendingFilter() downFilter := filter.NewRegionDownFilter() @@ -268,15 +268,15 @@ func (handler *grantLeaderHandler) UpdateConfig(w http.ResponseWriter, r *http.R idFloat, ok := input["store_id"].(float64) if ok { id = (uint64)(idFloat) - handler.config.mu.RLock() + handler.config.RLock() if _, exists = handler.config.StoreIDWithRanges[id]; !exists { if err := handler.config.cluster.PauseLeaderTransfer(id); err != nil { - handler.config.mu.RUnlock() + handler.config.RUnlock() handler.rd.JSON(w, http.StatusInternalServerError, err.Error()) return } } - handler.config.mu.RUnlock() + handler.config.RUnlock() args = append(args, strconv.FormatUint(id, 10)) } diff --git a/pkg/schedule/schedulers/hot_region.go b/pkg/schedule/schedulers/hot_region.go index 4806180e450..fdd07e85145 100644 --- a/pkg/schedule/schedulers/hot_region.go +++ b/pkg/schedule/schedulers/hot_region.go @@ -257,6 +257,44 @@ func (h *hotScheduler) EncodeConfig() ([]byte, error) { return h.conf.EncodeConfig() } +func (h *hotScheduler) ReloadConfig() error { + h.conf.Lock() + defer h.conf.Unlock() + cfgData, err := h.conf.storage.LoadSchedulerConfig(h.GetName()) + if err != nil { + return err + } + if len(cfgData) == 0 { + return nil + } + newCfg := &hotRegionSchedulerConfig{} + if err := DecodeConfig([]byte(cfgData), newCfg); err != nil { + return err + } + h.conf.MinHotByteRate = newCfg.MinHotByteRate + h.conf.MinHotKeyRate = newCfg.MinHotKeyRate + h.conf.MinHotQueryRate = newCfg.MinHotQueryRate + h.conf.MaxZombieRounds = newCfg.MaxZombieRounds + h.conf.MaxPeerNum = newCfg.MaxPeerNum + h.conf.ByteRateRankStepRatio = newCfg.ByteRateRankStepRatio + h.conf.KeyRateRankStepRatio = newCfg.KeyRateRankStepRatio + h.conf.QueryRateRankStepRatio = newCfg.QueryRateRankStepRatio + h.conf.CountRankStepRatio = newCfg.CountRankStepRatio + h.conf.GreatDecRatio = newCfg.GreatDecRatio + h.conf.MinorDecRatio = newCfg.MinorDecRatio + h.conf.SrcToleranceRatio = newCfg.SrcToleranceRatio + h.conf.DstToleranceRatio = newCfg.DstToleranceRatio + h.conf.WriteLeaderPriorities = newCfg.WriteLeaderPriorities + h.conf.WritePeerPriorities = newCfg.WritePeerPriorities + h.conf.ReadPriorities = newCfg.ReadPriorities + h.conf.StrictPickingStore = newCfg.StrictPickingStore + h.conf.EnableForTiFlash = newCfg.EnableForTiFlash + h.conf.RankFormulaVersion = newCfg.RankFormulaVersion + h.conf.ForbidRWType = newCfg.ForbidRWType + h.conf.SplitThresholds = newCfg.SplitThresholds + return nil +} + func (h *hotScheduler) ServeHTTP(w http.ResponseWriter, r *http.Request) { h.conf.ServeHTTP(w, r) } diff --git a/pkg/schedule/schedulers/init.go b/pkg/schedule/schedulers/init.go index d45602b90e1..f60be1e5b06 100644 --- a/pkg/schedule/schedulers/init.go +++ b/pkg/schedule/schedulers/init.go @@ -163,6 +163,7 @@ func schedulersRegister() { if err := decoder(conf); err != nil { return nil, err } + conf.cluster = opController.GetCluster() return newEvictSlowStoreScheduler(opController, conf), nil }) @@ -378,6 +379,7 @@ func schedulersRegister() { if err := decoder(conf); err != nil { return nil, err } + conf.storage = storage return newShuffleHotRegionScheduler(opController, conf), nil }) diff --git a/pkg/schedule/schedulers/label.go b/pkg/schedule/schedulers/label.go index 62a1100d16b..90310bcf10e 100644 --- a/pkg/schedule/schedulers/label.go +++ b/pkg/schedule/schedulers/label.go @@ -46,6 +46,7 @@ var ( type labelSchedulerConfig struct { Name string `json:"name"` Ranges []core.KeyRange `json:"ranges"` + // TODO: When we prepare to use Ranges, we will need to implement the ReloadConfig function for this scheduler. } type labelScheduler struct { diff --git a/pkg/schedule/schedulers/random_merge.go b/pkg/schedule/schedulers/random_merge.go index a621b595198..44bb5081ef9 100644 --- a/pkg/schedule/schedulers/random_merge.go +++ b/pkg/schedule/schedulers/random_merge.go @@ -48,6 +48,7 @@ var ( type randomMergeSchedulerConfig struct { Name string `json:"name"` Ranges []core.KeyRange `json:"ranges"` + // TODO: When we prepare to use Ranges, we will need to implement the ReloadConfig function for this scheduler. } type randomMergeScheduler struct { diff --git a/pkg/schedule/schedulers/scatter_range.go b/pkg/schedule/schedulers/scatter_range.go index e301b4c6e76..1bc6eafb58e 100644 --- a/pkg/schedule/schedulers/scatter_range.go +++ b/pkg/schedule/schedulers/scatter_range.go @@ -49,7 +49,7 @@ var ( ) type scatterRangeSchedulerConfig struct { - mu syncutil.RWMutex + syncutil.RWMutex storage endpoint.ConfigStorage RangeName string `json:"range-name"` StartKey string `json:"start-key"` @@ -60,8 +60,8 @@ func (conf *scatterRangeSchedulerConfig) BuildWithArgs(args []string) error { if len(args) != 3 { return errs.ErrSchedulerConfig.FastGenByArgs("ranges and name") } - conf.mu.Lock() - defer conf.mu.Unlock() + conf.Lock() + defer conf.Unlock() conf.RangeName = args[0] conf.StartKey = args[1] @@ -70,8 +70,8 @@ func (conf *scatterRangeSchedulerConfig) BuildWithArgs(args []string) error { } func (conf *scatterRangeSchedulerConfig) Clone() *scatterRangeSchedulerConfig { - conf.mu.RLock() - defer conf.mu.RUnlock() + conf.RLock() + defer conf.RUnlock() return &scatterRangeSchedulerConfig{ StartKey: conf.StartKey, EndKey: conf.EndKey, @@ -81,8 +81,8 @@ func (conf *scatterRangeSchedulerConfig) Clone() *scatterRangeSchedulerConfig { func (conf *scatterRangeSchedulerConfig) Persist() error { name := conf.getSchedulerName() - conf.mu.RLock() - defer conf.mu.RUnlock() + conf.RLock() + defer conf.RUnlock() data, err := EncodeConfig(conf) if err != nil { return err @@ -91,26 +91,26 @@ func (conf *scatterRangeSchedulerConfig) Persist() error { } func (conf *scatterRangeSchedulerConfig) GetRangeName() string { - conf.mu.RLock() - defer conf.mu.RUnlock() + conf.RLock() + defer conf.RUnlock() return conf.RangeName } func (conf *scatterRangeSchedulerConfig) GetStartKey() []byte { - conf.mu.RLock() - defer conf.mu.RUnlock() + conf.RLock() + defer conf.RUnlock() return []byte(conf.StartKey) } func (conf *scatterRangeSchedulerConfig) GetEndKey() []byte { - conf.mu.RLock() - defer conf.mu.RUnlock() + conf.RLock() + defer conf.RUnlock() return []byte(conf.EndKey) } func (conf *scatterRangeSchedulerConfig) getSchedulerName() string { - conf.mu.RLock() - defer conf.mu.RUnlock() + conf.RLock() + defer conf.RUnlock() return fmt.Sprintf("scatter-range-%s", conf.RangeName) } @@ -161,14 +161,14 @@ func (l *scatterRangeScheduler) GetType() string { } func (l *scatterRangeScheduler) EncodeConfig() ([]byte, error) { - l.config.mu.RLock() - defer l.config.mu.RUnlock() + l.config.RLock() + defer l.config.RUnlock() return EncodeConfig(l.config) } func (l *scatterRangeScheduler) ReloadConfig() error { - l.config.mu.Lock() - defer l.config.mu.Unlock() + l.config.Lock() + defer l.config.Unlock() cfgData, err := l.config.storage.LoadSchedulerConfig(l.GetName()) if err != nil { return err @@ -176,7 +176,14 @@ func (l *scatterRangeScheduler) ReloadConfig() error { if len(cfgData) == 0 { return nil } - return DecodeConfig([]byte(cfgData), l.config) + newCfg := &scatterRangeSchedulerConfig{} + if err := DecodeConfig([]byte(cfgData), newCfg); err != nil { + return err + } + l.config.RangeName = newCfg.RangeName + l.config.StartKey = newCfg.StartKey + l.config.EndKey = newCfg.EndKey + return nil } func (l *scatterRangeScheduler) IsScheduleAllowed(cluster sche.SchedulerCluster) bool { diff --git a/pkg/schedule/schedulers/scheduler_controller.go b/pkg/schedule/schedulers/scheduler_controller.go index b65173c1f5b..818f02685ea 100644 --- a/pkg/schedule/schedulers/scheduler_controller.go +++ b/pkg/schedule/schedulers/scheduler_controller.go @@ -16,6 +16,7 @@ package schedulers import ( "context" + "fmt" "net/http" "sync" "sync/atomic" @@ -280,7 +281,7 @@ func (c *Controller) PauseOrResumeScheduler(name string, t int64) error { // ReloadSchedulerConfig reloads a scheduler's config if it exists. func (c *Controller) ReloadSchedulerConfig(name string) error { if exist, _ := c.IsSchedulerExisted(name); !exist { - return nil + return fmt.Errorf("scheduler %s is not existed", name) } return c.GetScheduler(name).ReloadConfig() } diff --git a/pkg/schedule/schedulers/shuffle_hot_region.go b/pkg/schedule/schedulers/shuffle_hot_region.go index d5264b90428..6ad6656fd18 100644 --- a/pkg/schedule/schedulers/shuffle_hot_region.go +++ b/pkg/schedule/schedulers/shuffle_hot_region.go @@ -15,6 +15,9 @@ package schedulers import ( + "net/http" + + "github.com/gorilla/mux" "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/log" "github.com/tikv/pd/pkg/core/constant" @@ -24,6 +27,10 @@ import ( "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" "github.com/tikv/pd/pkg/statistics" + "github.com/tikv/pd/pkg/storage/endpoint" + "github.com/tikv/pd/pkg/utils/apiutil" + "github.com/tikv/pd/pkg/utils/syncutil" + "github.com/unrolled/render" "go.uber.org/zap" ) @@ -42,8 +49,32 @@ var ( ) type shuffleHotRegionSchedulerConfig struct { - Name string `json:"name"` - Limit uint64 `json:"limit"` + syncutil.RWMutex + storage endpoint.ConfigStorage + Name string `json:"name"` + Limit uint64 `json:"limit"` +} + +func (conf *shuffleHotRegionSchedulerConfig) getSchedulerName() string { + return conf.Name +} + +func (conf *shuffleHotRegionSchedulerConfig) Clone() *shuffleHotRegionSchedulerConfig { + conf.RLock() + defer conf.RUnlock() + return &shuffleHotRegionSchedulerConfig{ + Name: conf.Name, + Limit: conf.Limit, + } +} + +func (conf *shuffleHotRegionSchedulerConfig) persistLocked() error { + name := conf.getSchedulerName() + data, err := EncodeConfig(conf) + if err != nil { + return err + } + return conf.storage.SaveSchedulerConfig(name, data) } // ShuffleHotRegionScheduler mainly used to test. @@ -52,19 +83,26 @@ type shuffleHotRegionSchedulerConfig struct { // the hot peer. type shuffleHotRegionScheduler struct { *baseHotScheduler - conf *shuffleHotRegionSchedulerConfig + conf *shuffleHotRegionSchedulerConfig + handler http.Handler } // newShuffleHotRegionScheduler creates an admin scheduler that random balance hot regions func newShuffleHotRegionScheduler(opController *operator.Controller, conf *shuffleHotRegionSchedulerConfig) Scheduler { base := newBaseHotScheduler(opController) + handler := newShuffleHotRegionHandler(conf) ret := &shuffleHotRegionScheduler{ baseHotScheduler: base, conf: conf, + handler: handler, } return ret } +func (s *shuffleHotRegionScheduler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + s.handler.ServeHTTP(w, r) +} + func (s *shuffleHotRegionScheduler) GetName() string { return s.conf.Name } @@ -77,6 +115,24 @@ func (s *shuffleHotRegionScheduler) EncodeConfig() ([]byte, error) { return EncodeConfig(s.conf) } +func (s *shuffleHotRegionScheduler) ReloadConfig() error { + s.conf.Lock() + defer s.conf.Unlock() + cfgData, err := s.conf.storage.LoadSchedulerConfig(s.GetName()) + if err != nil { + return err + } + if len(cfgData) == 0 { + return nil + } + newCfg := &shuffleHotRegionSchedulerConfig{} + if err = DecodeConfig([]byte(cfgData), newCfg); err != nil { + return err + } + s.conf.Limit = newCfg.Limit + return nil +} + func (s *shuffleHotRegionScheduler) IsScheduleAllowed(cluster sche.SchedulerCluster) bool { hotRegionAllowed := s.OpController.OperatorCount(operator.OpHotRegion) < s.conf.Limit conf := cluster.GetSchedulerConfig() @@ -158,3 +214,47 @@ func (s *shuffleHotRegionScheduler) randomSchedule(cluster sche.SchedulerCluster shuffleHotRegionSkipCounter.Inc() return nil } + +type shuffleHotRegionHandler struct { + rd *render.Render + config *shuffleHotRegionSchedulerConfig +} + +func (handler *shuffleHotRegionHandler) UpdateConfig(w http.ResponseWriter, r *http.Request) { + var input map[string]interface{} + if err := apiutil.ReadJSONRespondError(handler.rd, w, r.Body, &input); err != nil { + return + } + limit, ok := input["limit"].(float64) + if !ok { + handler.rd.JSON(w, http.StatusBadRequest, "invalid limit") + return + } + handler.config.Lock() + defer handler.config.Unlock() + previous := handler.config.Limit + handler.config.Limit = uint64(limit) + err := handler.config.persistLocked() + if err != nil { + handler.rd.JSON(w, http.StatusInternalServerError, err.Error()) + handler.config.Limit = previous + return + } + handler.rd.JSON(w, http.StatusOK, nil) +} + +func (handler *shuffleHotRegionHandler) ListConfig(w http.ResponseWriter, r *http.Request) { + conf := handler.config.Clone() + handler.rd.JSON(w, http.StatusOK, conf) +} + +func newShuffleHotRegionHandler(config *shuffleHotRegionSchedulerConfig) http.Handler { + h := &shuffleHotRegionHandler{ + config: config, + rd: render.New(render.Options{IndentJSON: true}), + } + router := mux.NewRouter() + router.HandleFunc("/config", h.UpdateConfig).Methods(http.MethodPost) + router.HandleFunc("/list", h.ListConfig).Methods(http.MethodGet) + return router +} diff --git a/pkg/schedule/schedulers/shuffle_leader.go b/pkg/schedule/schedulers/shuffle_leader.go index 0e33fa802db..a6ff4baf65b 100644 --- a/pkg/schedule/schedulers/shuffle_leader.go +++ b/pkg/schedule/schedulers/shuffle_leader.go @@ -43,6 +43,7 @@ var ( type shuffleLeaderSchedulerConfig struct { Name string `json:"name"` Ranges []core.KeyRange `json:"ranges"` + // TODO: When we prepare to use Ranges, we will need to implement the ReloadConfig function for this scheduler. } type shuffleLeaderScheduler struct { diff --git a/pkg/schedule/schedulers/split_bucket.go b/pkg/schedule/schedulers/split_bucket.go index a08c84372b5..d536f3f8dc8 100644 --- a/pkg/schedule/schedulers/split_bucket.go +++ b/pkg/schedule/schedulers/split_bucket.go @@ -65,15 +65,15 @@ func initSplitBucketConfig() *splitBucketSchedulerConfig { } type splitBucketSchedulerConfig struct { - mu syncutil.RWMutex + syncutil.RWMutex storage endpoint.ConfigStorage Degree int `json:"degree"` SplitLimit uint64 `json:"split-limit"` } func (conf *splitBucketSchedulerConfig) Clone() *splitBucketSchedulerConfig { - conf.mu.RLock() - defer conf.mu.RUnlock() + conf.RLock() + defer conf.RUnlock() return &splitBucketSchedulerConfig{ Degree: conf.Degree, } @@ -104,8 +104,8 @@ func (h *splitBucketHandler) ListConfig(w http.ResponseWriter, _ *http.Request) } func (h *splitBucketHandler) UpdateConfig(w http.ResponseWriter, r *http.Request) { - h.conf.mu.Lock() - defer h.conf.mu.Unlock() + h.conf.Lock() + defer h.conf.Unlock() rd := render.New(render.Options{IndentJSON: true}) oldc, _ := json.Marshal(h.conf) data, err := io.ReadAll(r.Body) @@ -173,8 +173,8 @@ func (s *splitBucketScheduler) GetType() string { } func (s *splitBucketScheduler) ReloadConfig() error { - s.conf.mu.Lock() - defer s.conf.mu.Unlock() + s.conf.Lock() + defer s.conf.Unlock() cfgData, err := s.conf.storage.LoadSchedulerConfig(s.GetName()) if err != nil { return err @@ -182,7 +182,13 @@ func (s *splitBucketScheduler) ReloadConfig() error { if len(cfgData) == 0 { return nil } - return DecodeConfig([]byte(cfgData), s.conf) + newCfg := &splitBucketSchedulerConfig{} + if err := DecodeConfig([]byte(cfgData), newCfg); err != nil { + return err + } + s.conf.SplitLimit = newCfg.SplitLimit + s.conf.Degree = newCfg.Degree + return nil } // ServerHTTP implement Http server. diff --git a/pkg/schedule/schedulers/transfer_witness_leader.go b/pkg/schedule/schedulers/transfer_witness_leader.go index 2586065ea80..c651a8ef872 100644 --- a/pkg/schedule/schedulers/transfer_witness_leader.go +++ b/pkg/schedule/schedulers/transfer_witness_leader.go @@ -34,6 +34,7 @@ const ( // TransferWitnessLeaderBatchSize is the number of operators to to transfer // leaders by one scheduling transferWitnessLeaderBatchSize = 3 + // TODO: When we prepare to use Ranges, we will need to implement the ReloadConfig function for this scheduler. // TransferWitnessLeaderRecvMaxRegionSize is the max number of region can receive // TODO: make it a reasonable value transferWitnessLeaderRecvMaxRegionSize = 10000 diff --git a/pkg/schedule/schedulers/utils.go b/pkg/schedule/schedulers/utils.go index c7cdf9191ca..fea51798d1c 100644 --- a/pkg/schedule/schedulers/utils.go +++ b/pkg/schedule/schedulers/utils.go @@ -390,3 +390,19 @@ func (q *retryQuota) GC(keepStores []*core.StoreInfo) { } } } + +// pauseAndResumeLeaderTransfer checks the old and new store IDs, and pause or resume the leader transfer. +func pauseAndResumeLeaderTransfer[T any](cluster *core.BasicCluster, old, new map[uint64]T) { + for id := range old { + if _, ok := new[id]; ok { + continue + } + cluster.ResumeLeaderTransfer(id) + } + for id := range new { + if _, ok := old[id]; ok { + continue + } + cluster.PauseLeaderTransfer(id) + } +} diff --git a/pkg/window/policy.go b/pkg/window/policy.go index d67a8aa6e59..fed4fedc32a 100644 --- a/pkg/window/policy.go +++ b/pkg/window/policy.go @@ -18,8 +18,9 @@ package window import ( - "sync" "time" + + "github.com/tikv/pd/pkg/utils/syncutil" ) // RollingPolicy is a policy for ring window based on time duration. @@ -27,7 +28,7 @@ import ( // e.g. If the last point is appended one bucket duration ago, // RollingPolicy will increment current offset. type RollingPolicy struct { - mu sync.RWMutex + mu syncutil.RWMutex size int window *Window offset int diff --git a/server/cluster/scheduling_controller.go b/server/cluster/scheduling_controller.go index 04c77498948..5e8cb8462df 100644 --- a/server/cluster/scheduling_controller.go +++ b/server/cluster/scheduling_controller.go @@ -37,6 +37,7 @@ import ( "github.com/tikv/pd/pkg/statistics/buckets" "github.com/tikv/pd/pkg/statistics/utils" "github.com/tikv/pd/pkg/utils/logutil" + "github.com/tikv/pd/pkg/utils/syncutil" ) // schedulingController is used to manage all schedulers and checkers. @@ -44,7 +45,7 @@ type schedulingController struct { parentCtx context.Context ctx context.Context cancel context.CancelFunc - mu sync.RWMutex + mu syncutil.RWMutex wg sync.WaitGroup *core.BasicCluster opt sc.ConfProvider diff --git a/tests/pdctl/scheduler/scheduler_test.go b/tests/pdctl/scheduler/scheduler_test.go index 7098637c84a..7c5d53e387a 100644 --- a/tests/pdctl/scheduler/scheduler_test.go +++ b/tests/pdctl/scheduler/scheduler_test.go @@ -285,10 +285,12 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) { var roles []string mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "shuffle-region-scheduler", "show-roles"}, &roles) re.Equal([]string{"leader", "follower", "learner"}, roles) - echo := mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "shuffle-region-scheduler", "set-roles", "learner"}, nil) // todo:add check output + echo := mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "shuffle-region-scheduler", "set-roles", "learner"}, nil) re.Contains(echo, "Success!") - mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "shuffle-region-scheduler", "show-roles"}, &roles) - re.Equal([]string{"learner"}, roles) + testutil.Eventually(re, func() bool { + mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "shuffle-region-scheduler", "show-roles"}, &roles) + return reflect.DeepEqual([]string{"learner"}, roles) + }) mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "shuffle-region-scheduler"}, &roles) re.Equal([]string{"learner"}, roles) @@ -312,11 +314,10 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) { echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "grant-hot-region-scheduler", "set", "2", "1,2,3"}, nil) re.Contains(echo, "Success!") expected3["store-leader-id"] = float64(2) - // FIXME: remove this check after scheduler config is updated - if cluster.GetSchedulingPrimaryServer() == nil { // "grant-hot-region-scheduler" + testutil.Eventually(re, func() bool { mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "grant-hot-region-scheduler"}, &conf3) - re.Equal(expected3, conf3) - } + return reflect.DeepEqual(expected3, conf3) + }) // test remove and add scheduler echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "add", "balance-region-scheduler"}, nil) @@ -370,91 +371,88 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) { re.Contains(echo, "Success!") expected1["src-tolerance-ratio"] = 1.02 var conf1 map[string]interface{} - // FIXME: remove this check after scheduler config is updated - if cluster.GetSchedulingPrimaryServer() == nil { // "balance-hot-region-scheduler" - mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1) - re.Equal(expected1, conf1) + mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1) + re.Equal(expected1, conf1) - echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "read-priorities", "byte,key"}, nil) - re.Contains(echo, "Success!") - expected1["read-priorities"] = []interface{}{"byte", "key"} - mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1) - re.Equal(expected1, conf1) - - echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "read-priorities", "key"}, nil) - re.Contains(echo, "Failed!") - mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1) - re.Equal(expected1, conf1) - echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "read-priorities", "key,byte"}, nil) - re.Contains(echo, "Success!") - expected1["read-priorities"] = []interface{}{"key", "byte"} - mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1) - re.Equal(expected1, conf1) - echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "read-priorities", "foo,bar"}, nil) - re.Contains(echo, "Failed!") - mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1) - re.Equal(expected1, conf1) - echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "read-priorities", ""}, nil) - re.Contains(echo, "Failed!") - mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1) - re.Equal(expected1, conf1) - echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "read-priorities", "key,key"}, nil) - re.Contains(echo, "Failed!") - mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1) - re.Equal(expected1, conf1) - echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "read-priorities", "byte,byte"}, nil) - re.Contains(echo, "Failed!") - mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1) - re.Equal(expected1, conf1) - echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "read-priorities", "key,key,byte"}, nil) - re.Contains(echo, "Failed!") - mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1) - re.Equal(expected1, conf1) - - // write-priorities is divided into write-leader-priorities and write-peer-priorities - echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "write-priorities", "key,byte"}, nil) - re.Contains(echo, "Failed!") - re.Contains(echo, "Config item is not found.") - mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1) - re.Equal(expected1, conf1) - - echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "rank-formula-version", "v0"}, nil) - re.Contains(echo, "Failed!") - mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1) - expected1["rank-formula-version"] = "v2" - echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "rank-formula-version", "v2"}, nil) - re.Contains(echo, "Success!") - mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1) - re.Equal(expected1, conf1) - expected1["rank-formula-version"] = "v1" - echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "rank-formula-version", "v1"}, nil) - re.Contains(echo, "Success!") - mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1) - re.Equal(expected1, conf1) + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "read-priorities", "byte,key"}, nil) + re.Contains(echo, "Success!") + expected1["read-priorities"] = []interface{}{"byte", "key"} + mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1) + re.Equal(expected1, conf1) + + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "read-priorities", "key"}, nil) + re.Contains(echo, "Failed!") + mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1) + re.Equal(expected1, conf1) + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "read-priorities", "key,byte"}, nil) + re.Contains(echo, "Success!") + expected1["read-priorities"] = []interface{}{"key", "byte"} + mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1) + re.Equal(expected1, conf1) + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "read-priorities", "foo,bar"}, nil) + re.Contains(echo, "Failed!") + mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1) + re.Equal(expected1, conf1) + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "read-priorities", ""}, nil) + re.Contains(echo, "Failed!") + mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1) + re.Equal(expected1, conf1) + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "read-priorities", "key,key"}, nil) + re.Contains(echo, "Failed!") + mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1) + re.Equal(expected1, conf1) + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "read-priorities", "byte,byte"}, nil) + re.Contains(echo, "Failed!") + mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1) + re.Equal(expected1, conf1) + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "read-priorities", "key,key,byte"}, nil) + re.Contains(echo, "Failed!") + mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1) + re.Equal(expected1, conf1) + + // write-priorities is divided into write-leader-priorities and write-peer-priorities + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "write-priorities", "key,byte"}, nil) + re.Contains(echo, "Failed!") + re.Contains(echo, "Config item is not found.") + mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1) + re.Equal(expected1, conf1) + + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "rank-formula-version", "v0"}, nil) + re.Contains(echo, "Failed!") + mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1) + expected1["rank-formula-version"] = "v2" + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "rank-formula-version", "v2"}, nil) + re.Contains(echo, "Success!") + mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1) + re.Equal(expected1, conf1) + expected1["rank-formula-version"] = "v1" + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "rank-formula-version", "v1"}, nil) + re.Contains(echo, "Success!") + mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1) + re.Equal(expected1, conf1) - expected1["forbid-rw-type"] = "read" - echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "forbid-rw-type", "read"}, nil) - re.Contains(echo, "Success!") - mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1) - re.Equal(expected1, conf1) + expected1["forbid-rw-type"] = "read" + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "forbid-rw-type", "read"}, nil) + re.Contains(echo, "Success!") + mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1) + re.Equal(expected1, conf1) - // test compatibility - re.Equal("2.0.0", leaderServer.GetClusterVersion().String()) - for _, store := range stores { - version := versioninfo.HotScheduleWithQuery - store.Version = versioninfo.MinSupportedVersion(version).String() - tests.MustPutStore(re, cluster, store) - } - re.Equal("5.2.0", leaderServer.GetClusterVersion().String()) - // After upgrading, we should not use query. - mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1) - re.Equal(conf1["read-priorities"], []interface{}{"key", "byte"}) - // cannot set qps as write-peer-priorities - echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "write-peer-priorities", "query,byte"}, nil) - re.Contains(echo, "query is not allowed to be set in priorities for write-peer-priorities") - mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1) - re.Equal(conf1["write-peer-priorities"], []interface{}{"byte", "key"}) + // test compatibility + re.Equal("2.0.0", leaderServer.GetClusterVersion().String()) + for _, store := range stores { + version := versioninfo.HotScheduleWithQuery + store.Version = versioninfo.MinSupportedVersion(version).String() + tests.MustPutStore(re, cluster, store) } + re.Equal("5.2.0", leaderServer.GetClusterVersion().String()) + // After upgrading, we should not use query. + mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1) + re.Equal(conf1["read-priorities"], []interface{}{"key", "byte"}) + // cannot set qps as write-peer-priorities + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "write-peer-priorities", "query,byte"}, nil) + re.Contains(echo, "query is not allowed to be set in priorities for write-peer-priorities") + mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1) + re.Equal(conf1["write-peer-priorities"], []interface{}{"byte", "key"}) // test remove and add echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "remove", "balance-hot-region-scheduler"}, nil) @@ -498,11 +496,10 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) { echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", schedulerName, "set", "recovery-duration", "100"}, nil) re.Contains(echo, "Success!") conf = make(map[string]interface{}) - // FIXME: remove this check after scheduler config is updated - if cluster.GetSchedulingPrimaryServer() == nil && schedulerName == "evict-slow-store-scheduler" { + testutil.Eventually(re, func() bool { mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", schedulerName, "show"}, &conf) - re.Equal(100., conf["recovery-duration"]) - } + return conf["recovery-duration"] == 100. + }) echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "remove", schedulerName}, nil) re.Contains(echo, "Success!") testutil.Eventually(re, func() bool { @@ -511,6 +508,27 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) { }) } + // test shuffle hot region scheduler + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "add", "shuffle-hot-region-scheduler"}, nil) + re.Contains(echo, "Success!") + testutil.Eventually(re, func() bool { + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "show"}, nil) + return strings.Contains(echo, "shuffle-hot-region-scheduler") + }) + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "shuffle-hot-region-scheduler", "set", "limit", "127"}, nil) + re.Contains(echo, "Success!") + conf = make(map[string]interface{}) + testutil.Eventually(re, func() bool { + mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "shuffle-hot-region-scheduler", "show"}, &conf) + return conf["limit"] == 127. + }) + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "remove", "shuffle-hot-region-scheduler"}, nil) + re.Contains(echo, "Success!") + testutil.Eventually(re, func() bool { + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "show"}, nil) + return !strings.Contains(echo, "shuffle-hot-region-scheduler") + }) + // test show scheduler with paused and disabled status. checkSchedulerWithStatusCommand := func(status string, expected []string) { testutil.Eventually(re, func() bool { diff --git a/tests/server/api/scheduler_test.go b/tests/server/api/scheduler_test.go index 4d6dde6f2b9..86b932b0a7a 100644 --- a/tests/server/api/scheduler_test.go +++ b/tests/server/api/scheduler_test.go @@ -236,27 +236,25 @@ func (suite *scheduleTestSuite) checkAPI(cluster *tests.TestCluster) { suite.NoError(tu.CheckPostJSON(testDialClient, updateURL, body, tu.StatusOK(re))) resp = make(map[string]interface{}) suite.NoError(tu.ReadGetJSON(re, testDialClient, listURL, &resp)) - // FIXME: remove this check after scheduler config is updated - if cluster.GetSchedulingPrimaryServer() == nil { // "balance-hot-region-scheduler" - for key := range expectMap { - suite.Equal(expectMap[key], resp[key], "key %s", key) - } - - // update again - err = tu.CheckPostJSON(testDialClient, updateURL, body, - tu.StatusOK(re), - tu.StringEqual(re, "Config is the same with origin, so do nothing.")) - suite.NoError(err) - // config item not found - dataMap = map[string]interface{}{} - dataMap["error"] = 3 - body, err = json.Marshal(dataMap) - suite.NoError(err) - err = tu.CheckPostJSON(testDialClient, updateURL, body, - tu.Status(re, http.StatusBadRequest), - tu.StringEqual(re, "Config item is not found.")) - suite.NoError(err) + + for key := range expectMap { + suite.Equal(expectMap[key], resp[key], "key %s", key) } + + // update again + err = tu.CheckPostJSON(testDialClient, updateURL, body, + tu.StatusOK(re), + tu.StringEqual(re, "Config is the same with origin, so do nothing.")) + suite.NoError(err) + // config item not found + dataMap = map[string]interface{}{} + dataMap["error"] = 3 + body, err = json.Marshal(dataMap) + suite.NoError(err) + err = tu.CheckPostJSON(testDialClient, updateURL, body, + tu.Status(re, http.StatusBadRequest), + tu.StringEqual(re, "Config item is not found.")) + suite.NoError(err) }, }, { diff --git a/tools/pd-ctl/pdctl/command/scheduler.go b/tools/pd-ctl/pdctl/command/scheduler.go index 526ff2646dc..695576edf84 100644 --- a/tools/pd-ctl/pdctl/command/scheduler.go +++ b/tools/pd-ctl/pdctl/command/scheduler.go @@ -500,6 +500,7 @@ func NewConfigSchedulerCommand() *cobra.Command { newConfigBalanceLeaderCommand(), newSplitBucketCommand(), newConfigEvictSlowStoreCommand(), + newConfigShuffleHotRegionSchedulerCommand(), newConfigEvictSlowTrendCommand(), ) return c @@ -802,6 +803,25 @@ func newConfigEvictSlowStoreCommand() *cobra.Command { return c } +func newConfigShuffleHotRegionSchedulerCommand() *cobra.Command { + c := &cobra.Command{ + Use: "shuffle-hot-region-scheduler", + Short: "shuffle-hot-region-scheduler config", + Run: listSchedulerConfigCommandFunc, + } + + c.AddCommand(&cobra.Command{ + Use: "show", + Short: "list the config item", + Run: listSchedulerConfigCommandFunc, + }, &cobra.Command{ + Use: "set ", + Short: "set the config item", + Run: func(cmd *cobra.Command, args []string) { postSchedulerConfigCommandFunc(cmd, c.Name(), args) }, + }) + return c +} + func newConfigEvictSlowTrendCommand() *cobra.Command { c := &cobra.Command{ Use: "evict-slow-trend-scheduler", diff --git a/tools/pd-simulator/simulator/node.go b/tools/pd-simulator/simulator/node.go index b8fb422d6dd..68a10a8638e 100644 --- a/tools/pd-simulator/simulator/node.go +++ b/tools/pd-simulator/simulator/node.go @@ -25,6 +25,7 @@ import ( "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/kvproto/pkg/pdpb" "github.com/tikv/pd/pkg/ratelimit" + "github.com/tikv/pd/pkg/utils/syncutil" "github.com/tikv/pd/tools/pd-simulator/simulator/cases" "github.com/tikv/pd/tools/pd-simulator/simulator/info" "github.com/tikv/pd/tools/pd-simulator/simulator/simutil" @@ -39,7 +40,7 @@ const ( // Node simulates a TiKV. type Node struct { *metapb.Store - sync.RWMutex + syncutil.RWMutex stats *info.StoreStats tick uint64 wg sync.WaitGroup @@ -50,7 +51,7 @@ type Node struct { cancel context.CancelFunc raftEngine *RaftEngine limiter *ratelimit.RateLimiter - sizeMutex sync.Mutex + sizeMutex syncutil.Mutex hasExtraUsedSpace bool snapStats []*pdpb.SnapshotStat } From c0dad3516057075ec685e25b0edc9f7b2389c987 Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Wed, 22 Nov 2023 17:50:42 +0800 Subject: [PATCH 039/137] mcs: fix api wrongly send to scheduling server (#7403) ref tikv/pd#5839 Co-authored-by: lhy1024 Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- server/api/admin.go | 8 ++++---- server/api/config.go | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/server/api/admin.go b/server/api/admin.go index 246c9239f59..cd050102037 100644 --- a/server/api/admin.go +++ b/server/api/admin.go @@ -61,7 +61,7 @@ func (h *adminHandler) DeleteRegionCache(w http.ResponseWriter, r *http.Request) return } rc.DropCacheRegion(regionID) - if h.svr.IsAPIServiceMode() { + if h.svr.IsServiceIndependent(utils.SchedulingServiceName) { err = h.DeleteRegionCacheInSchedulingServer(regionID) } msg := "The region is removed from server cache." @@ -101,7 +101,7 @@ func (h *adminHandler) DeleteRegionStorage(w http.ResponseWriter, r *http.Reques } // Remove region from cache. rc.DropCacheRegion(regionID) - if h.svr.IsAPIServiceMode() { + if h.svr.IsServiceIndependent(utils.SchedulingServiceName) { err = h.DeleteRegionCacheInSchedulingServer(regionID) } msg := "The region is removed from server cache and region meta storage." @@ -117,7 +117,7 @@ func (h *adminHandler) DeleteAllRegionCache(w http.ResponseWriter, r *http.Reque var err error rc := getCluster(r) rc.DropCacheAllRegion() - if h.svr.IsAPIServiceMode() { + if h.svr.IsServiceIndependent(utils.SchedulingServiceName) { err = h.DeleteRegionCacheInSchedulingServer() } msg := "All regions are removed from server cache." @@ -241,7 +241,7 @@ func (h *adminHandler) DeleteRegionCacheInSchedulingServer(id ...uint64) error { } func (h *adminHandler) buildMsg(msg string, err error) string { - if h.svr.IsAPIServiceMode() && err != nil { + if h.svr.IsServiceIndependent(utils.SchedulingServiceName) && err != nil { return fmt.Sprintf("This operation was executed in API server but needs to be re-executed on scheduling server due to the following error: %s", err.Error()) } return msg diff --git a/server/api/config.go b/server/api/config.go index 746b1119a73..428cebdb1d8 100644 --- a/server/api/config.go +++ b/server/api/config.go @@ -62,7 +62,7 @@ func newConfHandler(svr *server.Server, rd *render.Render) *confHandler { // @Router /config [get] func (h *confHandler) GetConfig(w http.ResponseWriter, r *http.Request) { cfg := h.svr.GetConfig() - if h.svr.IsAPIServiceMode() { + if h.svr.IsServiceIndependent(utils.SchedulingServiceName) { schedulingServerConfig, err := h.GetSchedulingServerConfig() if err != nil { h.rd.JSON(w, http.StatusInternalServerError, err.Error()) @@ -313,7 +313,7 @@ func getConfigMap(cfg map[string]interface{}, key []string, value interface{}) m // @Success 200 {object} sc.ScheduleConfig // @Router /config/schedule [get] func (h *confHandler) GetScheduleConfig(w http.ResponseWriter, r *http.Request) { - if h.svr.IsAPIServiceMode() { + if h.svr.IsServiceIndependent(utils.SchedulingServiceName) { cfg, err := h.GetSchedulingServerConfig() if err != nil { h.rd.JSON(w, http.StatusInternalServerError, err.Error()) @@ -386,7 +386,7 @@ func (h *confHandler) SetScheduleConfig(w http.ResponseWriter, r *http.Request) // @Success 200 {object} sc.ReplicationConfig // @Router /config/replicate [get] func (h *confHandler) GetReplicationConfig(w http.ResponseWriter, r *http.Request) { - if h.svr.IsAPIServiceMode() { + if h.svr.IsServiceIndependent(utils.SchedulingServiceName) { cfg, err := h.GetSchedulingServerConfig() if err != nil { h.rd.JSON(w, http.StatusInternalServerError, err.Error()) From 8198735a06c4bd6ff105cd6e8ffead66edefdca2 Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Thu, 23 Nov 2023 15:44:41 +0800 Subject: [PATCH 040/137] *: fix data race when starting server (#7420) close tikv/pd#7419 Signed-off-by: Ryan Leung Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- pkg/utils/apiutil/serverapi/middleware.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/utils/apiutil/serverapi/middleware.go b/pkg/utils/apiutil/serverapi/middleware.go index e26327cb3ff..9ea38e4cce0 100644 --- a/pkg/utils/apiutil/serverapi/middleware.go +++ b/pkg/utils/apiutil/serverapi/middleware.go @@ -160,8 +160,7 @@ func (h *redirector) matchMicroServiceRedirectRules(r *http.Request) (bool, stri func (h *redirector) ServeHTTP(w http.ResponseWriter, r *http.Request, next http.HandlerFunc) { redirectToMicroService, targetAddr := h.matchMicroServiceRedirectRules(r) allowFollowerHandle := len(r.Header.Get(apiutil.PDAllowFollowerHandleHeader)) > 0 - isLeader := h.s.GetMember().IsLeader() - if !h.s.IsClosed() && (allowFollowerHandle || isLeader) && !redirectToMicroService { + if !h.s.IsClosed() && (allowFollowerHandle || h.s.GetMember().IsLeader()) && !redirectToMicroService { next(w, r) return } From fc578aa3c9310dfca9334bba55452469f9db0ba6 Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Thu, 23 Nov 2023 17:17:42 +0800 Subject: [PATCH 041/137] mcs: speed up and stablize tests (#7417) close tikv/pd#7399 Signed-off-by: Ryan Leung --- .../mcs/scheduling/server_test.go | 42 +++++++++++-------- 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/tests/integrations/mcs/scheduling/server_test.go b/tests/integrations/mcs/scheduling/server_test.go index afd213a624b..c65352114df 100644 --- a/tests/integrations/mcs/scheduling/server_test.go +++ b/tests/integrations/mcs/scheduling/server_test.go @@ -61,7 +61,7 @@ func (suite *serverTestSuite) SetupSuite() { re := suite.Require() re.NoError(failpoint.Enable("github.com/tikv/pd/server/cluster/highFrequencyClusterJobs", `return(true)`)) suite.ctx, suite.cancel = context.WithCancel(context.Background()) - suite.cluster, err = tests.NewTestAPICluster(suite.ctx, 3) + suite.cluster, err = tests.NewTestAPICluster(suite.ctx, 1) re.NoError(err) err = suite.cluster.RunInitialServers() @@ -96,6 +96,10 @@ func (suite *serverTestSuite) TestAllocID() { func (suite *serverTestSuite) TestAllocIDAfterLeaderChange() { re := suite.Require() re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/mcs/scheduling/server/fastUpdateMember", `return(true)`)) + pd2, err := suite.cluster.Join(suite.ctx) + re.NoError(err) + err = pd2.Run() + re.NoError(err) tc, err := tests.NewTestSchedulingCluster(suite.ctx, 1, suite.backendEndpoints) re.NoError(err) defer tc.Destroy() @@ -117,6 +121,8 @@ func (suite *serverTestSuite) TestAllocIDAfterLeaderChange() { // Update the pdLeader in test suite. suite.pdLeader = suite.cluster.GetServer(suite.cluster.WaitLeader()) suite.backendEndpoints = suite.pdLeader.GetAddr() + suite.TearDownSuite() + suite.SetupSuite() } func (suite *serverTestSuite) TestPrimaryChange() { @@ -167,24 +173,24 @@ func (suite *serverTestSuite) TestForwardStoreHeartbeat() { re.NoError(err) re.Empty(resp.GetHeader().GetError()) - resp1, err := s.StoreHeartbeat( - context.Background(), &pdpb.StoreHeartbeatRequest{ - Header: &pdpb.RequestHeader{ClusterId: suite.pdLeader.GetClusterID()}, - Stats: &pdpb.StoreStats{ - StoreId: 1, - Capacity: 1798985089024, - Available: 1709868695552, - UsedSize: 85150956358, - KeysWritten: 20000, - BytesWritten: 199, - KeysRead: 10000, - BytesRead: 99, - }, - }, - ) - re.NoError(err) - re.Empty(resp1.GetHeader().GetError()) testutil.Eventually(re, func() bool { + resp1, err := s.StoreHeartbeat( + context.Background(), &pdpb.StoreHeartbeatRequest{ + Header: &pdpb.RequestHeader{ClusterId: suite.pdLeader.GetClusterID()}, + Stats: &pdpb.StoreStats{ + StoreId: 1, + Capacity: 1798985089024, + Available: 1709868695552, + UsedSize: 85150956358, + KeysWritten: 20000, + BytesWritten: 199, + KeysRead: 10000, + BytesRead: 99, + }, + }, + ) + re.NoError(err) + re.Empty(resp1.GetHeader().GetError()) store := tc.GetPrimaryServer().GetCluster().GetStore(1) return store.GetStoreStats().GetCapacity() == uint64(1798985089024) && store.GetStoreStats().GetAvailable() == uint64(1709868695552) && From 0222af023dc5c5b0bdb3b1c6b3721c282c5eaf25 Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Thu, 23 Nov 2023 17:38:42 +0800 Subject: [PATCH 042/137] schedule: fix panic when switching placement rules (#7415) close tikv/pd#7414 Signed-off-by: Ryan Leung Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- pkg/schedule/checker/rule_checker.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/schedule/checker/rule_checker.go b/pkg/schedule/checker/rule_checker.go index 08ef5f7b45c..553ece09e65 100644 --- a/pkg/schedule/checker/rule_checker.go +++ b/pkg/schedule/checker/rule_checker.go @@ -132,6 +132,11 @@ func (c *RuleChecker) CheckWithFit(region *core.RegionInfo, fit *placement.Regio return } + // the placement rule is disabled + if fit == nil { + return + } + // If the fit is calculated by FitRegion, which means we get a new fit result, thus we should // invalid the cache if it exists c.ruleManager.InvalidCache(region.GetID()) From c760afd28eb2b252c1e44a1c0ad52f0dcc531942 Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Thu, 23 Nov 2023 19:47:42 +0800 Subject: [PATCH 043/137] *: fix data race of TestPersistScheduler (#7421) close tikv/pd#7401 Signed-off-by: Ryan Leung Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- server/cluster/cluster_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server/cluster/cluster_test.go b/server/cluster/cluster_test.go index b1a3535e90f..85edf911779 100644 --- a/server/cluster/cluster_test.go +++ b/server/cluster/cluster_test.go @@ -3140,7 +3140,7 @@ func TestPersistScheduler(t *testing.T) { // option have 6 items because the default scheduler do not remove. re.Len(newOpt.GetSchedulers(), defaultCount+3) re.NoError(newOpt.Persist(storage)) - tc.RaftCluster.opt = newOpt + tc.RaftCluster.SetScheduleConfig(newOpt.GetScheduleConfig()) co = schedule.NewCoordinator(ctx, tc.RaftCluster, hbStreams) co.Run() @@ -3153,7 +3153,7 @@ func TestPersistScheduler(t *testing.T) { _, newOpt, err = newTestScheduleConfig() re.NoError(err) re.NoError(newOpt.Reload(storage)) - tc.RaftCluster.opt = newOpt + tc.RaftCluster.SetScheduleConfig(newOpt.GetScheduleConfig()) co = schedule.NewCoordinator(ctx, tc.RaftCluster, hbStreams) co.Run() controller = co.GetSchedulersController() @@ -3180,7 +3180,7 @@ func TestPersistScheduler(t *testing.T) { _, newOpt, err = newTestScheduleConfig() re.NoError(err) re.NoError(newOpt.Reload(co.GetCluster().GetStorage())) - tc.RaftCluster.opt = newOpt + tc.RaftCluster.SetScheduleConfig(newOpt.GetScheduleConfig()) co = schedule.NewCoordinator(ctx, tc.RaftCluster, hbStreams) co.Run() @@ -3239,7 +3239,7 @@ func TestRemoveScheduler(t *testing.T) { _, newOpt, err := newTestScheduleConfig() re.NoError(err) re.NoError(newOpt.Reload(tc.storage)) - tc.RaftCluster.opt = newOpt + tc.RaftCluster.SetScheduleConfig(newOpt.GetScheduleConfig()) co = schedule.NewCoordinator(ctx, tc.RaftCluster, hbStreams) co.Run() re.Empty(controller.GetSchedulerNames()) From 9902be8d3feeb4822014566929380fed71f8b6fb Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Thu, 23 Nov 2023 20:46:13 +0800 Subject: [PATCH 044/137] *: Use http.NoBody instead of nil in http.NewRequest calls (#7428) ref tikv/pd#4399 Signed-off-by: lhy1024 Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- client/http/client.go | 30 +++++++++---------- pkg/audit/audit_test.go | 4 +-- pkg/dashboard/adapter/redirector_test.go | 10 +++---- pkg/utils/apiutil/apiutil.go | 2 +- server/api/admin.go | 2 +- server/api/admin_test.go | 4 +-- server/api/config.go | 2 +- server/api/region_test.go | 3 +- server/api/server_test.go | 2 +- server/api/store_test.go | 2 +- server/cluster/cluster.go | 5 ++-- server/server_test.go | 6 ++-- .../mcs/keyspace/tso_keyspace_group_test.go | 2 +- .../resourcemanager/resource_manager_test.go | 2 +- tests/integrations/mcs/tso/api_test.go | 4 +-- tests/server/api/api_test.go | 22 +++++++------- tests/server/api/testutil.go | 2 +- tests/server/apiv2/handlers/testutil.go | 10 +++---- tests/server/member/member_test.go | 2 +- tools/pd-api-bench/cases/cases.go | 6 ++-- 20 files changed, 60 insertions(+), 62 deletions(-) diff --git a/client/http/client.go b/client/http/client.go index d7eb1b1b801..880489aa85c 100644 --- a/client/http/client.go +++ b/client/http/client.go @@ -276,7 +276,7 @@ func (c *client) GetRegionByID(ctx context.Context, regionID uint64) (*RegionInf var region RegionInfo err := c.requestWithRetry(ctx, "GetRegionByID", RegionByID(regionID), - http.MethodGet, nil, ®ion) + http.MethodGet, http.NoBody, ®ion) if err != nil { return nil, err } @@ -288,7 +288,7 @@ func (c *client) GetRegionByKey(ctx context.Context, key []byte) (*RegionInfo, e var region RegionInfo err := c.requestWithRetry(ctx, "GetRegionByKey", RegionByKey(key), - http.MethodGet, nil, ®ion) + http.MethodGet, http.NoBody, ®ion) if err != nil { return nil, err } @@ -300,7 +300,7 @@ func (c *client) GetRegions(ctx context.Context) (*RegionsInfo, error) { var regions RegionsInfo err := c.requestWithRetry(ctx, "GetRegions", Regions, - http.MethodGet, nil, ®ions) + http.MethodGet, http.NoBody, ®ions) if err != nil { return nil, err } @@ -312,7 +312,7 @@ func (c *client) GetRegionsByKeyRange(ctx context.Context, startKey, endKey []by var regions RegionsInfo err := c.requestWithRetry(ctx, "GetRegionsByKeyRange", RegionsByKey(startKey, endKey, limit), - http.MethodGet, nil, ®ions) + http.MethodGet, http.NoBody, ®ions) if err != nil { return nil, err } @@ -324,7 +324,7 @@ func (c *client) GetRegionsByStoreID(ctx context.Context, storeID uint64) (*Regi var regions RegionsInfo err := c.requestWithRetry(ctx, "GetRegionsByStoreID", RegionsByStoreID(storeID), - http.MethodGet, nil, ®ions) + http.MethodGet, http.NoBody, ®ions) if err != nil { return nil, err } @@ -336,7 +336,7 @@ func (c *client) GetHotReadRegions(ctx context.Context) (*StoreHotPeersInfos, er var hotReadRegions StoreHotPeersInfos err := c.requestWithRetry(ctx, "GetHotReadRegions", HotRead, - http.MethodGet, nil, &hotReadRegions) + http.MethodGet, http.NoBody, &hotReadRegions) if err != nil { return nil, err } @@ -348,7 +348,7 @@ func (c *client) GetHotWriteRegions(ctx context.Context) (*StoreHotPeersInfos, e var hotWriteRegions StoreHotPeersInfos err := c.requestWithRetry(ctx, "GetHotWriteRegions", HotWrite, - http.MethodGet, nil, &hotWriteRegions) + http.MethodGet, http.NoBody, &hotWriteRegions) if err != nil { return nil, err } @@ -360,7 +360,7 @@ func (c *client) GetRegionStatusByKeyRange(ctx context.Context, startKey, endKey var regionStats RegionStats err := c.requestWithRetry(ctx, "GetRegionStatusByKeyRange", RegionStatsByKeyRange(startKey, endKey), - http.MethodGet, nil, ®ionStats, + http.MethodGet, http.NoBody, ®ionStats, ) if err != nil { return nil, err @@ -373,7 +373,7 @@ func (c *client) GetStores(ctx context.Context) (*StoresInfo, error) { var stores StoresInfo err := c.requestWithRetry(ctx, "GetStores", Stores, - http.MethodGet, nil, &stores) + http.MethodGet, http.NoBody, &stores) if err != nil { return nil, err } @@ -385,7 +385,7 @@ func (c *client) GetAllPlacementRuleBundles(ctx context.Context) ([]*GroupBundle var bundles []*GroupBundle err := c.requestWithRetry(ctx, "GetPlacementRuleBundle", PlacementRuleBundle, - http.MethodGet, nil, &bundles) + http.MethodGet, http.NoBody, &bundles) if err != nil { return nil, err } @@ -397,7 +397,7 @@ func (c *client) GetPlacementRuleBundleByGroup(ctx context.Context, group string var bundle GroupBundle err := c.requestWithRetry(ctx, "GetPlacementRuleBundleByGroup", PlacementRuleBundleByGroup(group), - http.MethodGet, nil, &bundle) + http.MethodGet, http.NoBody, &bundle) if err != nil { return nil, err } @@ -409,7 +409,7 @@ func (c *client) GetPlacementRulesByGroup(ctx context.Context, group string) ([] var rules []*Rule err := c.requestWithRetry(ctx, "GetPlacementRulesByGroup", PlacementRulesByGroup(group), - http.MethodGet, nil, &rules) + http.MethodGet, http.NoBody, &rules) if err != nil { return nil, err } @@ -443,7 +443,7 @@ func (c *client) SetPlacementRuleBundles(ctx context.Context, bundles []*GroupBu func (c *client) DeletePlacementRule(ctx context.Context, group, id string) error { return c.requestWithRetry(ctx, "DeletePlacementRule", PlacementRuleByGroupAndID(group, id), - http.MethodDelete, nil, nil) + http.MethodDelete, http.NoBody, nil) } // GetAllRegionLabelRules gets all region label rules. @@ -451,7 +451,7 @@ func (c *client) GetAllRegionLabelRules(ctx context.Context) ([]*LabelRule, erro var labelRules []*LabelRule err := c.requestWithRetry(ctx, "GetAllRegionLabelRules", RegionLabelRules, - http.MethodGet, nil, &labelRules) + http.MethodGet, http.NoBody, &labelRules) if err != nil { return nil, err } @@ -533,7 +533,7 @@ func (c *client) GetMinResolvedTSByStoresIDs(ctx context.Context, storeIDs []uin }{} err := c.requestWithRetry(ctx, "GetMinResolvedTSByStoresIDs", uri, - http.MethodGet, nil, &resp) + http.MethodGet, http.NoBody, &resp) if err != nil { return 0, nil, err } diff --git a/pkg/audit/audit_test.go b/pkg/audit/audit_test.go index 20f8c9344f7..d59c9627115 100644 --- a/pkg/audit/audit_test.go +++ b/pkg/audit/audit_test.go @@ -59,7 +59,7 @@ func TestPrometheusHistogramBackend(t *testing.T) { defer ts.Close() backend := NewPrometheusHistogramBackend(serviceAuditHistogramTest, true) - req, _ := http.NewRequest(http.MethodGet, "http://127.0.0.1:2379/test?test=test", nil) + req, _ := http.NewRequest(http.MethodGet, "http://127.0.0.1:2379/test?test=test", http.NoBody) info := requestutil.GetRequestInfo(req) info.ServiceLabel = "test" info.Component = "user1" @@ -79,7 +79,7 @@ func TestPrometheusHistogramBackend(t *testing.T) { // For test, sleep time needs longer than the push interval time.Sleep(time.Second) - req, _ = http.NewRequest(http.MethodGet, ts.URL, nil) + req, _ = http.NewRequest(http.MethodGet, ts.URL, http.NoBody) resp, err := http.DefaultClient.Do(req) re.NoError(err) defer resp.Body.Close() diff --git a/pkg/dashboard/adapter/redirector_test.go b/pkg/dashboard/adapter/redirector_test.go index f192fbeb1f2..c5d837507fc 100644 --- a/pkg/dashboard/adapter/redirector_test.go +++ b/pkg/dashboard/adapter/redirector_test.go @@ -70,17 +70,17 @@ func (suite *redirectorTestSuite) TestReverseProxy() { suite.redirector.SetAddress(suite.tempServer.URL) // Test normal forwarding - req, err := http.NewRequest(http.MethodGet, redirectorServer.URL, nil) + req, err := http.NewRequest(http.MethodGet, redirectorServer.URL, http.NoBody) suite.NoError(err) checkHTTPRequest(suite.Require(), suite.noRedirectHTTPClient, req, http.StatusOK, suite.tempText) // Test the requests that are forwarded by others - req, err = http.NewRequest(http.MethodGet, redirectorServer.URL, nil) + req, err = http.NewRequest(http.MethodGet, redirectorServer.URL, http.NoBody) suite.NoError(err) req.Header.Set(proxyHeader, "other") checkHTTPRequest(suite.Require(), suite.noRedirectHTTPClient, req, http.StatusOK, suite.tempText) // Test LoopDetected suite.redirector.SetAddress(redirectorServer.URL) - req, err = http.NewRequest(http.MethodGet, redirectorServer.URL, nil) + req, err = http.NewRequest(http.MethodGet, redirectorServer.URL, http.NoBody) suite.NoError(err) checkHTTPRequest(suite.Require(), suite.noRedirectHTTPClient, req, http.StatusLoopDetected, "") } @@ -90,11 +90,11 @@ func (suite *redirectorTestSuite) TestTemporaryRedirect() { defer redirectorServer.Close() suite.redirector.SetAddress(suite.tempServer.URL) // Test TemporaryRedirect - req, err := http.NewRequest(http.MethodGet, redirectorServer.URL, nil) + req, err := http.NewRequest(http.MethodGet, redirectorServer.URL, http.NoBody) suite.NoError(err) checkHTTPRequest(suite.Require(), suite.noRedirectHTTPClient, req, http.StatusTemporaryRedirect, "") // Test Response - req, err = http.NewRequest(http.MethodGet, redirectorServer.URL, nil) + req, err = http.NewRequest(http.MethodGet, redirectorServer.URL, http.NoBody) suite.NoError(err) checkHTTPRequest(suite.Require(), http.DefaultClient, req, http.StatusOK, suite.tempText) } diff --git a/pkg/utils/apiutil/apiutil.go b/pkg/utils/apiutil/apiutil.go index 633dc8fa557..8928688fed9 100644 --- a/pkg/utils/apiutil/apiutil.go +++ b/pkg/utils/apiutil/apiutil.go @@ -228,7 +228,7 @@ func PostJSONIgnoreResp(client *http.Client, url string, data []byte) error { // DoDelete is used to send delete request and return http response code. func DoDelete(client *http.Client, url string) (*http.Response, error) { - req, err := http.NewRequest(http.MethodDelete, url, nil) + req, err := http.NewRequest(http.MethodDelete, url, http.NoBody) if err != nil { return nil, err } diff --git a/server/api/admin.go b/server/api/admin.go index cd050102037..49fe7cdc567 100644 --- a/server/api/admin.go +++ b/server/api/admin.go @@ -225,7 +225,7 @@ func (h *adminHandler) DeleteRegionCacheInSchedulingServer(id ...uint64) error { idStr = strconv.FormatUint(id[0], 10) } url := fmt.Sprintf("%s/scheduling/api/v1/admin/cache/regions/%s", addr, idStr) - req, err := http.NewRequest(http.MethodDelete, url, nil) + req, err := http.NewRequest(http.MethodDelete, url, http.NoBody) if err != nil { return err } diff --git a/server/api/admin_test.go b/server/api/admin_test.go index ad761d532ea..76c5e729eb0 100644 --- a/server/api/admin_test.go +++ b/server/api/admin_test.go @@ -85,7 +85,7 @@ func (suite *adminTestSuite) TestDropRegion() { // After drop region from cache, lower version is accepted. url := fmt.Sprintf("%s/admin/cache/region/%d", suite.urlPrefix, region.GetID()) - req, err := http.NewRequest(http.MethodDelete, url, nil) + req, err := http.NewRequest(http.MethodDelete, url, http.NoBody) suite.NoError(err) res, err := testDialClient.Do(req) suite.NoError(err) @@ -147,7 +147,7 @@ func (suite *adminTestSuite) TestDropRegions() { // After drop all regions from cache, lower version is accepted. url := fmt.Sprintf("%s/admin/cache/regions", suite.urlPrefix) - req, err := http.NewRequest(http.MethodDelete, url, nil) + req, err := http.NewRequest(http.MethodDelete, url, http.NoBody) suite.NoError(err) res, err := testDialClient.Do(req) suite.NoError(err) diff --git a/server/api/config.go b/server/api/config.go index 428cebdb1d8..e075095d7a9 100644 --- a/server/api/config.go +++ b/server/api/config.go @@ -543,7 +543,7 @@ func (h *confHandler) GetSchedulingServerConfig() (*config.Config, error) { return nil, errs.ErrNotFoundSchedulingAddr.FastGenByArgs() } url := fmt.Sprintf("%s/scheduling/api/v1/config", addr) - req, err := http.NewRequest(http.MethodGet, url, nil) + req, err := http.NewRequest(http.MethodGet, url, http.NoBody) if err != nil { return nil, err } diff --git a/server/api/region_test.go b/server/api/region_test.go index 0f8f84bfc37..645a91e8dbc 100644 --- a/server/api/region_test.go +++ b/server/api/region_test.go @@ -15,7 +15,6 @@ package api import ( - "bytes" "context" "encoding/hex" "encoding/json" @@ -480,7 +479,7 @@ func TestRegionsWithKillRequest(t *testing.T) { } ctx, cancel := context.WithCancel(context.Background()) - req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, bytes.NewBuffer(nil)) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, http.NoBody) re.NoError(err) respCh := make(chan *http.Response) go func() { diff --git a/server/api/server_test.go b/server/api/server_test.go index 22989b92a03..d834941193b 100644 --- a/server/api/server_test.go +++ b/server/api/server_test.go @@ -273,7 +273,7 @@ func (suite *adminTestSuite) TestCleanPath() { // handled by router response := httptest.NewRecorder() r, _, _ := NewHandler(context.Background(), suite.svr) - request, err := http.NewRequest(http.MethodGet, url, nil) + request, err := http.NewRequest(http.MethodGet, url, http.NoBody) re.NoError(err) r.ServeHTTP(response, request) // handled by `cleanPath` which is in `mux.ServeHTTP` diff --git a/server/api/store_test.go b/server/api/store_test.go index 2b3a8dee9bb..d0961d572f8 100644 --- a/server/api/store_test.go +++ b/server/api/store_test.go @@ -51,7 +51,7 @@ func TestStoreTestSuite(t *testing.T) { } func (suite *storeTestSuite) requestStatusBody(client *http.Client, method string, url string) int { - req, err := http.NewRequest(method, url, nil) + req, err := http.NewRequest(method, url, http.NoBody) suite.NoError(err) resp, err := client.Do(req) suite.NoError(err) diff --git a/server/cluster/cluster.go b/server/cluster/cluster.go index e9cece0faa7..1c3d8a03a98 100644 --- a/server/cluster/cluster.go +++ b/server/cluster/cluster.go @@ -15,7 +15,6 @@ package cluster import ( - "bytes" "context" "encoding/json" "fmt" @@ -590,7 +589,7 @@ func (c *RaftCluster) fetchStoreConfigFromTiKV(ctx context.Context, statusAddres url = fmt.Sprintf("%s://%s/config", "http", statusAddress) } ctx, cancel := context.WithTimeout(ctx, clientTimeout) - req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, bytes.NewBuffer(nil)) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, http.NoBody) if err != nil { cancel() return nil, fmt.Errorf("failed to create store config http request: %w", err) @@ -2475,7 +2474,7 @@ func CheckHealth(client *http.Client, members []*pdpb.Member) map[uint64]*pdpb.M for _, member := range members { for _, cURL := range member.ClientUrls { ctx, cancel := context.WithTimeout(context.Background(), clientTimeout) - req, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("%s%s", cURL, healthURL), nil) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("%s%s", cURL, healthURL), http.NoBody) if err != nil { log.Error("failed to new request", errs.ZapError(errs.ErrNewHTTPRequest, err)) cancel() diff --git a/server/server_test.go b/server/server_test.go index 62cf5b168fc..a0562879057 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -218,7 +218,7 @@ func (suite *leaderServerTestSuite) TestSourceIpForHeaderForwarded() { err = svr.Run() suite.NoError(err) - req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("%s/pd/apis/mock/v1/hello", svr.GetAddr()), nil) + req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("%s/pd/apis/mock/v1/hello", svr.GetAddr()), http.NoBody) suite.NoError(err) req.Header.Add(apiutil.XForwardedForHeader, "127.0.0.2") resp, err := http.DefaultClient.Do(req) @@ -248,7 +248,7 @@ func (suite *leaderServerTestSuite) TestSourceIpForHeaderXReal() { err = svr.Run() suite.NoError(err) - req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("%s/pd/apis/mock/v1/hello", svr.GetAddr()), nil) + req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("%s/pd/apis/mock/v1/hello", svr.GetAddr()), http.NoBody) suite.NoError(err) req.Header.Add(apiutil.XRealIPHeader, "127.0.0.2") resp, err := http.DefaultClient.Do(req) @@ -278,7 +278,7 @@ func (suite *leaderServerTestSuite) TestSourceIpForHeaderBoth() { err = svr.Run() suite.NoError(err) - req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("%s/pd/apis/mock/v1/hello", svr.GetAddr()), nil) + req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("%s/pd/apis/mock/v1/hello", svr.GetAddr()), http.NoBody) suite.NoError(err) req.Header.Add(apiutil.XForwardedForHeader, "127.0.0.2") req.Header.Add(apiutil.XRealIPHeader, "127.0.0.3") diff --git a/tests/integrations/mcs/keyspace/tso_keyspace_group_test.go b/tests/integrations/mcs/keyspace/tso_keyspace_group_test.go index af7b31553b3..1415acc46d1 100644 --- a/tests/integrations/mcs/keyspace/tso_keyspace_group_test.go +++ b/tests/integrations/mcs/keyspace/tso_keyspace_group_test.go @@ -336,7 +336,7 @@ func (suite *keyspaceGroupTestSuite) tryCreateKeyspaceGroup(request *handlers.Cr } func (suite *keyspaceGroupTestSuite) tryGetKeyspaceGroup(id uint32) (*endpoint.KeyspaceGroup, int) { - httpReq, err := http.NewRequest(http.MethodGet, suite.server.GetAddr()+keyspaceGroupsPrefix+fmt.Sprintf("/%d", id), nil) + httpReq, err := http.NewRequest(http.MethodGet, suite.server.GetAddr()+keyspaceGroupsPrefix+fmt.Sprintf("/%d", id), http.NoBody) suite.NoError(err) resp, err := suite.dialClient.Do(httpReq) suite.NoError(err) diff --git a/tests/integrations/mcs/resourcemanager/resource_manager_test.go b/tests/integrations/mcs/resourcemanager/resource_manager_test.go index b98686222fe..cd6edda4628 100644 --- a/tests/integrations/mcs/resourcemanager/resource_manager_test.go +++ b/tests/integrations/mcs/resourcemanager/resource_manager_test.go @@ -1006,7 +1006,7 @@ func (suite *resourceManagerClientTestSuite) TestBasicResourceGroupCURD() { // Delete all resource groups for _, g := range groups { - req, err := http.NewRequest(http.MethodDelete, getAddr(i+1)+"/resource-manager/api/v1/config/group/"+g.Name, nil) + req, err := http.NewRequest(http.MethodDelete, getAddr(i+1)+"/resource-manager/api/v1/config/group/"+g.Name, http.NoBody) re.NoError(err) resp, err := http.DefaultClient.Do(req) re.NoError(err) diff --git a/tests/integrations/mcs/tso/api_test.go b/tests/integrations/mcs/tso/api_test.go index 81cc798851f..d08b8df32b9 100644 --- a/tests/integrations/mcs/tso/api_test.go +++ b/tests/integrations/mcs/tso/api_test.go @@ -124,7 +124,7 @@ func (suite *tsoAPITestSuite) TestForwardResetTS() { } func mustGetKeyspaceGroupMembers(re *require.Assertions, server *tso.Server) map[uint32]*apis.KeyspaceGroupMember { - httpReq, err := http.NewRequest(http.MethodGet, server.GetAddr()+tsoKeyspaceGroupsPrefix+"/members", nil) + httpReq, err := http.NewRequest(http.MethodGet, server.GetAddr()+tsoKeyspaceGroupsPrefix+"/members", http.NoBody) re.NoError(err) httpResp, err := dialClient.Do(httpReq) re.NoError(err) @@ -184,7 +184,7 @@ func TestTSOServerStartFirst(t *testing.T) { defer httpResp.Body.Close() re.Equal(http.StatusOK, httpResp.StatusCode) - httpReq, err = http.NewRequest(http.MethodGet, addr+"/pd/api/v2/tso/keyspace-groups/0", nil) + httpReq, err = http.NewRequest(http.MethodGet, addr+"/pd/api/v2/tso/keyspace-groups/0", http.NoBody) re.NoError(err) httpResp, err = dialClient.Do(httpReq) re.NoError(err) diff --git a/tests/server/api/api_test.go b/tests/server/api/api_test.go index 04bcdc0d461..905fb8ec096 100644 --- a/tests/server/api/api_test.go +++ b/tests/server/api/api_test.go @@ -374,7 +374,7 @@ func (suite *middlewareTestSuite) TestRateLimitMiddleware() { func (suite *middlewareTestSuite) TestSwaggerUrl() { leader := suite.cluster.GetLeaderServer() suite.NotNil(leader) - req, _ := http.NewRequest(http.MethodGet, leader.GetAddr()+"/swagger/ui/index", nil) + req, _ := http.NewRequest(http.MethodGet, leader.GetAddr()+"/swagger/ui/index", http.NoBody) resp, err := dialClient.Do(req) suite.NoError(err) suite.True(resp.StatusCode == http.StatusNotFound) @@ -395,14 +395,14 @@ func (suite *middlewareTestSuite) TestAuditPrometheusBackend() { resp.Body.Close() suite.True(leader.GetServer().GetServiceMiddlewarePersistOptions().IsAuditEnabled()) timeUnix := time.Now().Unix() - 20 - req, _ = http.NewRequest(http.MethodGet, fmt.Sprintf("%s/pd/api/v1/trend?from=%d", leader.GetAddr(), timeUnix), nil) + req, _ = http.NewRequest(http.MethodGet, fmt.Sprintf("%s/pd/api/v1/trend?from=%d", leader.GetAddr(), timeUnix), http.NoBody) resp, err = dialClient.Do(req) suite.NoError(err) _, err = io.ReadAll(resp.Body) resp.Body.Close() suite.NoError(err) - req, _ = http.NewRequest(http.MethodGet, leader.GetAddr()+"/metrics", nil) + req, _ = http.NewRequest(http.MethodGet, leader.GetAddr()+"/metrics", http.NoBody) resp, err = dialClient.Do(req) suite.NoError(err) defer resp.Body.Close() @@ -421,14 +421,14 @@ func (suite *middlewareTestSuite) TestAuditPrometheusBackend() { leader = suite.cluster.GetLeaderServer() timeUnix = time.Now().Unix() - 20 - req, _ = http.NewRequest(http.MethodGet, fmt.Sprintf("%s/pd/api/v1/trend?from=%d", leader.GetAddr(), timeUnix), nil) + req, _ = http.NewRequest(http.MethodGet, fmt.Sprintf("%s/pd/api/v1/trend?from=%d", leader.GetAddr(), timeUnix), http.NoBody) resp, err = dialClient.Do(req) suite.NoError(err) _, err = io.ReadAll(resp.Body) resp.Body.Close() suite.NoError(err) - req, _ = http.NewRequest(http.MethodGet, leader.GetAddr()+"/metrics", nil) + req, _ = http.NewRequest(http.MethodGet, leader.GetAddr()+"/metrics", http.NoBody) resp, err = dialClient.Do(req) suite.NoError(err) defer resp.Body.Close() @@ -542,7 +542,7 @@ func BenchmarkDoRequestWithoutServiceMiddleware(b *testing.B) { } func doTestRequestWithLogAudit(srv *tests.TestServer) { - req, _ := http.NewRequest(http.MethodDelete, fmt.Sprintf("%s/pd/api/v1/admin/cache/regions", srv.GetAddr()), nil) + req, _ := http.NewRequest(http.MethodDelete, fmt.Sprintf("%s/pd/api/v1/admin/cache/regions", srv.GetAddr()), http.NoBody) req.Header.Set("component", "test") resp, _ := dialClient.Do(req) resp.Body.Close() @@ -550,7 +550,7 @@ func doTestRequestWithLogAudit(srv *tests.TestServer) { func doTestRequestWithPrometheus(srv *tests.TestServer) { timeUnix := time.Now().Unix() - 20 - req, _ := http.NewRequest(http.MethodGet, fmt.Sprintf("%s/pd/api/v1/trend?from=%d", srv.GetAddr(), timeUnix), nil) + req, _ := http.NewRequest(http.MethodGet, fmt.Sprintf("%s/pd/api/v1/trend?from=%d", srv.GetAddr(), timeUnix), http.NoBody) req.Header.Set("component", "test") resp, _ := dialClient.Do(req) resp.Body.Close() @@ -611,7 +611,7 @@ func (suite *redirectorTestSuite) TestAllowFollowerHandle() { } addr := follower.GetAddr() + "/pd/api/v1/version" - request, err := http.NewRequest(http.MethodGet, addr, nil) + request, err := http.NewRequest(http.MethodGet, addr, http.NoBody) suite.NoError(err) request.Header.Add(apiutil.PDAllowFollowerHandleHeader, "true") resp, err := dialClient.Do(request) @@ -636,7 +636,7 @@ func (suite *redirectorTestSuite) TestNotLeader() { addr := follower.GetAddr() + "/pd/api/v1/version" // Request to follower without redirectorHeader is OK. - request, err := http.NewRequest(http.MethodGet, addr, nil) + request, err := http.NewRequest(http.MethodGet, addr, http.NoBody) suite.NoError(err) resp, err := dialClient.Do(request) suite.NoError(err) @@ -664,7 +664,7 @@ func (suite *redirectorTestSuite) TestXForwardedFor() { follower := suite.cluster.GetServer(suite.cluster.GetFollower()) addr := follower.GetAddr() + "/pd/api/v1/regions" - request, err := http.NewRequest(http.MethodGet, addr, nil) + request, err := http.NewRequest(http.MethodGet, addr, http.NoBody) suite.NoError(err) resp, err := dialClient.Do(request) suite.NoError(err) @@ -970,7 +970,7 @@ func TestPreparingProgress(t *testing.T) { } func sendRequest(re *require.Assertions, url string, method string, statusCode int) []byte { - req, _ := http.NewRequest(method, url, nil) + req, _ := http.NewRequest(method, url, http.NoBody) resp, err := dialClient.Do(req) re.NoError(err) re.Equal(statusCode, resp.StatusCode) diff --git a/tests/server/api/testutil.go b/tests/server/api/testutil.go index c6c2cc79611..6fab82ea2e3 100644 --- a/tests/server/api/testutil.go +++ b/tests/server/api/testutil.go @@ -63,7 +63,7 @@ func MustAddScheduler( // MustDeleteScheduler deletes a scheduler with HTTP API. func MustDeleteScheduler(re *require.Assertions, serverAddr, schedulerName string) { - httpReq, err := http.NewRequest(http.MethodDelete, fmt.Sprintf("%s%s/%s", serverAddr, schedulersPrefix, schedulerName), nil) + httpReq, err := http.NewRequest(http.MethodDelete, fmt.Sprintf("%s%s/%s", serverAddr, schedulersPrefix, schedulerName), http.NoBody) re.NoError(err) resp, err := dialClient.Do(httpReq) re.NoError(err) diff --git a/tests/server/apiv2/handlers/testutil.go b/tests/server/apiv2/handlers/testutil.go index aca29ebeb52..d26ce732714 100644 --- a/tests/server/apiv2/handlers/testutil.go +++ b/tests/server/apiv2/handlers/testutil.go @@ -42,7 +42,7 @@ var dialClient = &http.Client{ func sendLoadRangeRequest(re *require.Assertions, server *tests.TestServer, token, limit string) *handlers.LoadAllKeyspacesResponse { // Construct load range request. - httpReq, err := http.NewRequest(http.MethodGet, server.GetAddr()+keyspacesPrefix, nil) + httpReq, err := http.NewRequest(http.MethodGet, server.GetAddr()+keyspacesPrefix, http.NoBody) re.NoError(err) query := httpReq.URL.Query() query.Add("page_token", token) @@ -135,7 +135,7 @@ func mustLoadKeyspaces(re *require.Assertions, server *tests.TestServer, name st // MustLoadKeyspaceGroups loads all keyspace groups from the server. func MustLoadKeyspaceGroups(re *require.Assertions, server *tests.TestServer, token, limit string) []*endpoint.KeyspaceGroup { // Construct load range request. - httpReq, err := http.NewRequest(http.MethodGet, server.GetAddr()+keyspaceGroupsPrefix, nil) + httpReq, err := http.NewRequest(http.MethodGet, server.GetAddr()+keyspaceGroupsPrefix, http.NoBody) re.NoError(err) query := httpReq.URL.Query() query.Add("page_token", token) @@ -175,7 +175,7 @@ func MustLoadKeyspaceGroupByID(re *require.Assertions, server *tests.TestServer, // TryLoadKeyspaceGroupByID loads the keyspace group by ID with HTTP API. func TryLoadKeyspaceGroupByID(re *require.Assertions, server *tests.TestServer, id uint32) (*endpoint.KeyspaceGroup, int) { - httpReq, err := http.NewRequest(http.MethodGet, server.GetAddr()+keyspaceGroupsPrefix+fmt.Sprintf("/%d", id), nil) + httpReq, err := http.NewRequest(http.MethodGet, server.GetAddr()+keyspaceGroupsPrefix+fmt.Sprintf("/%d", id), http.NoBody) re.NoError(err) resp, err := dialClient.Do(httpReq) re.NoError(err) @@ -205,7 +205,7 @@ func FailCreateKeyspaceGroupWithCode(re *require.Assertions, server *tests.TestS // MustDeleteKeyspaceGroup deletes a keyspace group with HTTP API. func MustDeleteKeyspaceGroup(re *require.Assertions, server *tests.TestServer, id uint32) { - httpReq, err := http.NewRequest(http.MethodDelete, server.GetAddr()+keyspaceGroupsPrefix+fmt.Sprintf("/%d", id), nil) + httpReq, err := http.NewRequest(http.MethodDelete, server.GetAddr()+keyspaceGroupsPrefix+fmt.Sprintf("/%d", id), http.NoBody) re.NoError(err) resp, err := dialClient.Do(httpReq) re.NoError(err) @@ -232,7 +232,7 @@ func MustSplitKeyspaceGroup(re *require.Assertions, server *tests.TestServer, id // MustFinishSplitKeyspaceGroup finishes a keyspace group split with HTTP API. func MustFinishSplitKeyspaceGroup(re *require.Assertions, server *tests.TestServer, id uint32) { - httpReq, err := http.NewRequest(http.MethodDelete, server.GetAddr()+keyspaceGroupsPrefix+fmt.Sprintf("/%d/split", id), nil) + httpReq, err := http.NewRequest(http.MethodDelete, server.GetAddr()+keyspaceGroupsPrefix+fmt.Sprintf("/%d/split", id), http.NoBody) re.NoError(err) // Send request. resp, err := dialClient.Do(httpReq) diff --git a/tests/server/member/member_test.go b/tests/server/member/member_test.go index e6657ffd223..9ac0947c73f 100644 --- a/tests/server/member/member_test.go +++ b/tests/server/member/member_test.go @@ -88,7 +88,7 @@ func TestMemberDelete(t *testing.T) { t.Log(time.Now(), "try to delete:", table.path) testutil.Eventually(re, func() bool { addr := leader.GetConfig().ClientUrls + "/pd/api/v1/members/" + table.path - req, err := http.NewRequest(http.MethodDelete, addr, nil) + req, err := http.NewRequest(http.MethodDelete, addr, http.NoBody) re.NoError(err) res, err := httpClient.Do(req) re.NoError(err) diff --git a/tools/pd-api-bench/cases/cases.go b/tools/pd-api-bench/cases/cases.go index d431b6f325c..2f93d2e9454 100644 --- a/tools/pd-api-bench/cases/cases.go +++ b/tools/pd-api-bench/cases/cases.go @@ -44,7 +44,7 @@ var ( // InitCluster initializes the cluster. func InitCluster(ctx context.Context, cli pd.Client, httpClit *http.Client) error { req, _ := http.NewRequestWithContext(ctx, http.MethodGet, - PDAddress+"/pd/api/v1/stats/region?start_key=&end_key=&count", nil) + PDAddress+"/pd/api/v1/stats/region?start_key=&end_key=&count", http.NoBody) resp, err := httpClit.Do(req) if err != nil { return err @@ -158,7 +158,7 @@ type minResolvedTSStruct struct { func (c *minResolvedTS) Do(ctx context.Context, cli *http.Client) error { url := fmt.Sprintf("%s%s", PDAddress, c.path) - req, _ := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + req, _ := http.NewRequestWithContext(ctx, http.MethodGet, url, http.NoBody) res, err := cli.Do(req) if err != nil { return err @@ -212,7 +212,7 @@ func (c *regionsStats) Do(ctx context.Context, cli *http.Client) error { url.QueryEscape(string(generateKeyForSimulator(startID, 56))), url.QueryEscape(string(generateKeyForSimulator(endID, 56))), "") - req, _ := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + req, _ := http.NewRequestWithContext(ctx, http.MethodGet, url, http.NoBody) res, err := cli.Do(req) if err != nil { return err From c1e4a2a057a635d8e50fd925f5da05df9d96b1f5 Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Fri, 24 Nov 2023 02:26:43 +0800 Subject: [PATCH 045/137] mcs: support region http interface in scheduling server (#7297) ref tikv/pd#5839 Signed-off-by: lhy1024 Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- pkg/mcs/scheduling/server/apis/v1/api.go | 214 +++++++++++ pkg/schedule/handler/handler.go | 206 +++++++++++ pkg/schedule/schedulers/split_bucket.go | 4 +- pkg/utils/apiutil/serverapi/middleware.go | 6 +- scripts/ci-subtask.sh | 1 + server/api/region.go | 252 ++++--------- server/api/region_test.go | 220 ------------ server/api/server.go | 22 +- tests/integrations/mcs/scheduling/api_test.go | 31 +- tests/pdctl/scheduler/scheduler_test.go | 72 ++-- tests/server/api/operator_test.go | 4 +- tests/server/api/region_test.go | 340 ++++++++++++++++++ tests/server/api/rule_test.go | 6 +- tests/testutil.go | 8 +- 14 files changed, 923 insertions(+), 463 deletions(-) create mode 100644 tests/server/api/region_test.go diff --git a/pkg/mcs/scheduling/server/apis/v1/api.go b/pkg/mcs/scheduling/server/apis/v1/api.go index 822b4164da1..b59780b7a61 100644 --- a/pkg/mcs/scheduling/server/apis/v1/api.go +++ b/pkg/mcs/scheduling/server/apis/v1/api.go @@ -16,9 +16,12 @@ package apis import ( "encoding/hex" + "errors" + "fmt" "net/http" "net/url" "strconv" + "strings" "sync" "github.com/gin-contrib/cors" @@ -39,6 +42,7 @@ import ( "github.com/tikv/pd/pkg/utils/apiutil" "github.com/tikv/pd/pkg/utils/apiutil/multiservicesapi" "github.com/tikv/pd/pkg/utils/logutil" + "github.com/tikv/pd/pkg/utils/typeutil" "github.com/unrolled/render" ) @@ -118,6 +122,7 @@ func NewService(srv *scheserver.Service) *Service { s.RegisterSchedulersRouter() s.RegisterCheckersRouter() s.RegisterHotspotRouter() + s.RegisterRegionsRouter() return s } @@ -168,6 +173,16 @@ func (s *Service) RegisterOperatorsRouter() { router.GET("/records", getOperatorRecords) } +// RegisterRegionsRouter registers the router of the regions handler. +func (s *Service) RegisterRegionsRouter() { + router := s.root.Group("regions") + router.POST("/accelerate-schedule", accelerateRegionsScheduleInRange) + router.POST("/accelerate-schedule/batch", accelerateRegionsScheduleInRanges) + router.POST("/scatter", scatterRegions) + router.POST("/split", splitRegions) + router.GET("/replicated", checkRegionsReplicated) +} + // RegisterConfigRouter registers the router of the config handler. func (s *Service) RegisterConfigRouter() { router := s.root.Group("config") @@ -1118,3 +1133,202 @@ func getRegionLabelRuleByID(c *gin.Context) { } c.IndentedJSON(http.StatusOK, rule) } + +// @Tags region +// @Summary Accelerate regions scheduling a in given range, only receive hex format for keys +// @Accept json +// @Param body body object true "json params" +// @Param limit query integer false "Limit count" default(256) +// @Produce json +// @Success 200 {string} string "Accelerate regions scheduling in a given range [startKey, endKey)" +// @Failure 400 {string} string "The input is invalid." +// @Failure 500 {string} string "PD server failed to proceed the request." +// @Router /regions/accelerate-schedule [post] +func accelerateRegionsScheduleInRange(c *gin.Context) { + handler := c.MustGet(handlerKey).(*handler.Handler) + + var input map[string]interface{} + if err := c.BindJSON(&input); err != nil { + c.String(http.StatusBadRequest, err.Error()) + return + } + rawStartKey, ok1 := input["start_key"].(string) + rawEndKey, ok2 := input["end_key"].(string) + if !ok1 || !ok2 { + c.String(http.StatusBadRequest, "start_key or end_key is not string") + return + } + + limitStr, _ := c.GetQuery("limit") + limit, err := handler.AdjustLimit(limitStr, 256 /*default limit*/) + if err != nil { + c.String(http.StatusBadRequest, err.Error()) + return + } + + err = handler.AccelerateRegionsScheduleInRange(rawStartKey, rawEndKey, limit) + if err != nil { + c.String(http.StatusInternalServerError, err.Error()) + return + } + c.String(http.StatusOK, fmt.Sprintf("Accelerate regions scheduling in a given range [%s,%s)", rawStartKey, rawEndKey)) +} + +// @Tags region +// @Summary Accelerate regions scheduling in given ranges, only receive hex format for keys +// @Accept json +// @Param body body object true "json params" +// @Param limit query integer false "Limit count" default(256) +// @Produce json +// @Success 200 {string} string "Accelerate regions scheduling in given ranges [startKey1, endKey1), [startKey2, endKey2), ..." +// @Failure 400 {string} string "The input is invalid." +// @Failure 500 {string} string "PD server failed to proceed the request." +// @Router /regions/accelerate-schedule/batch [post] +func accelerateRegionsScheduleInRanges(c *gin.Context) { + handler := c.MustGet(handlerKey).(*handler.Handler) + + var input []map[string]interface{} + if err := c.BindJSON(&input); err != nil { + c.String(http.StatusBadRequest, err.Error()) + return + } + limitStr, _ := c.GetQuery("limit") + limit, err := handler.AdjustLimit(limitStr, 256 /*default limit*/) + if err != nil { + c.String(http.StatusBadRequest, err.Error()) + return + } + + var msgBuilder strings.Builder + msgBuilder.Grow(128) + msgBuilder.WriteString("Accelerate regions scheduling in given ranges: ") + var startKeys, endKeys [][]byte + for _, rg := range input { + startKey, rawStartKey, err := apiutil.ParseKey("start_key", rg) + if err != nil { + c.String(http.StatusBadRequest, err.Error()) + return + } + endKey, rawEndKey, err := apiutil.ParseKey("end_key", rg) + if err != nil { + c.String(http.StatusBadRequest, err.Error()) + return + } + startKeys = append(startKeys, startKey) + endKeys = append(endKeys, endKey) + msgBuilder.WriteString(fmt.Sprintf("[%s,%s), ", rawStartKey, rawEndKey)) + } + err = handler.AccelerateRegionsScheduleInRanges(startKeys, endKeys, limit) + if err != nil { + c.String(http.StatusInternalServerError, err.Error()) + return + } + c.String(http.StatusOK, msgBuilder.String()) +} + +// @Tags region +// @Summary Scatter regions by given key ranges or regions id distributed by given group with given retry limit +// @Accept json +// @Param body body object true "json params" +// @Produce json +// @Success 200 {string} string "Scatter regions by given key ranges or regions id distributed by given group with given retry limit" +// @Failure 400 {string} string "The input is invalid." +// @Failure 500 {string} string "PD server failed to proceed the request." +// @Router /regions/scatter [post] +func scatterRegions(c *gin.Context) { + handler := c.MustGet(handlerKey).(*handler.Handler) + + var input map[string]interface{} + if err := c.BindJSON(&input); err != nil { + c.String(http.StatusBadRequest, err.Error()) + return + } + rawStartKey, ok1 := input["start_key"].(string) + rawEndKey, ok2 := input["end_key"].(string) + group, _ := input["group"].(string) + retryLimit := 5 + if rl, ok := input["retry_limit"].(float64); ok { + retryLimit = int(rl) + } + + opsCount, failures, err := func() (int, map[uint64]error, error) { + if ok1 && ok2 { + return handler.ScatterRegionsByRange(rawStartKey, rawEndKey, group, retryLimit) + } + ids, ok := typeutil.JSONToUint64Slice(input["regions_id"]) + if !ok { + return 0, nil, errors.New("regions_id is invalid") + } + return handler.ScatterRegionsByID(ids, group, retryLimit, false) + }() + if err != nil { + c.String(http.StatusInternalServerError, err.Error()) + return + } + s := handler.BuildScatterRegionsResp(opsCount, failures) + c.IndentedJSON(http.StatusOK, &s) +} + +// @Tags region +// @Summary Split regions with given split keys +// @Accept json +// @Param body body object true "json params" +// @Produce json +// @Success 200 {string} string "Split regions with given split keys" +// @Failure 400 {string} string "The input is invalid." +// @Failure 500 {string} string "PD server failed to proceed the request." +// @Router /regions/split [post] +func splitRegions(c *gin.Context) { + handler := c.MustGet(handlerKey).(*handler.Handler) + + var input map[string]interface{} + if err := c.BindJSON(&input); err != nil { + c.String(http.StatusBadRequest, err.Error()) + return + } + s, ok := input["split_keys"] + if !ok { + c.String(http.StatusBadRequest, "split_keys should be provided.") + return + } + rawSplitKeys := s.([]interface{}) + if len(rawSplitKeys) < 1 { + c.String(http.StatusBadRequest, "empty split keys.") + return + } + retryLimit := 5 + if rl, ok := input["retry_limit"].(float64); ok { + retryLimit = int(rl) + } + s, err := handler.SplitRegions(c.Request.Context(), rawSplitKeys, retryLimit) + if err != nil { + c.String(http.StatusInternalServerError, err.Error()) + return + } + c.IndentedJSON(http.StatusOK, &s) +} + +// @Tags region +// @Summary Check if regions in the given key ranges are replicated. Returns 'REPLICATED', 'INPROGRESS', or 'PENDING'. 'PENDING' means that there is at least one region pending for scheduling. Similarly, 'INPROGRESS' means there is at least one region in scheduling. +// @Param startKey query string true "Regions start key, hex encoded" +// @Param endKey query string true "Regions end key, hex encoded" +// @Produce plain +// @Success 200 {string} string "INPROGRESS" +// @Failure 400 {string} string "The input is invalid." +// @Router /regions/replicated [get] +func checkRegionsReplicated(c *gin.Context) { + handler := c.MustGet(handlerKey).(*handler.Handler) + rawStartKey, ok1 := c.GetQuery("startKey") + rawEndKey, ok2 := c.GetQuery("endKey") + if !ok1 || !ok2 { + c.String(http.StatusBadRequest, "there is no start_key or end_key") + return + } + + state, err := handler.CheckRegionsReplicated(rawStartKey, rawEndKey) + if err != nil { + c.String(http.StatusBadRequest, err.Error()) + return + } + c.String(http.StatusOK, state) +} diff --git a/pkg/schedule/handler/handler.go b/pkg/schedule/handler/handler.go index 8da84647f77..353e2bb60e2 100644 --- a/pkg/schedule/handler/handler.go +++ b/pkg/schedule/handler/handler.go @@ -16,6 +16,7 @@ package handler import ( "bytes" + "context" "encoding/hex" "net/http" "strconv" @@ -23,6 +24,7 @@ import ( "time" "github.com/pingcap/errors" + "github.com/pingcap/failpoint" "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/kvproto/pkg/pdpb" "github.com/pingcap/log" @@ -43,6 +45,11 @@ import ( "go.uber.org/zap" ) +const ( + defaultRegionLimit = 16 + maxRegionLimit = 10240 +) + // Server is the interface for handler about schedule. // TODO: remove it after GetCluster is unified between PD server and Scheduling server. type Server interface { @@ -1082,6 +1089,205 @@ func (h *Handler) GetRegionLabeler() (*labeler.RegionLabeler, error) { return c.GetRegionLabeler(), nil } +// AccelerateRegionsScheduleInRange accelerates regions scheduling in a given range. +func (h *Handler) AccelerateRegionsScheduleInRange(rawStartKey, rawEndKey string, limit int) error { + startKey, err := hex.DecodeString(rawStartKey) + if err != nil { + return err + } + endKey, err := hex.DecodeString(rawEndKey) + if err != nil { + return err + } + c := h.GetCluster() + if c == nil { + return errs.ErrNotBootstrapped.GenWithStackByArgs() + } + co := h.GetCoordinator() + if co == nil { + return errs.ErrNotBootstrapped.GenWithStackByArgs() + } + regions := c.ScanRegions(startKey, endKey, limit) + if len(regions) > 0 { + regionsIDList := make([]uint64, 0, len(regions)) + for _, region := range regions { + regionsIDList = append(regionsIDList, region.GetID()) + } + co.GetCheckerController().AddSuspectRegions(regionsIDList...) + } + return nil +} + +// AccelerateRegionsScheduleInRanges accelerates regions scheduling in given ranges. +func (h *Handler) AccelerateRegionsScheduleInRanges(startKeys [][]byte, endKeys [][]byte, limit int) error { + c := h.GetCluster() + if c == nil { + return errs.ErrNotBootstrapped.GenWithStackByArgs() + } + co := h.GetCoordinator() + if co == nil { + return errs.ErrNotBootstrapped.GenWithStackByArgs() + } + if len(startKeys) != len(endKeys) { + return errors.New("startKeys and endKeys should have the same length") + } + var regions []*core.RegionInfo + for i := range startKeys { + regions = append(regions, c.ScanRegions(startKeys[i], endKeys[i], limit)...) + } + if len(regions) > 0 { + regionsIDList := make([]uint64, 0, len(regions)) + for _, region := range regions { + regionsIDList = append(regionsIDList, region.GetID()) + } + co.GetCheckerController().AddSuspectRegions(regionsIDList...) + } + return nil +} + +// AdjustLimit adjusts the limit of regions to schedule. +func (h *Handler) AdjustLimit(limitStr string, defaultLimits ...int) (int, error) { + limit := defaultRegionLimit + if len(defaultLimits) > 0 { + limit = defaultLimits[0] + } + if limitStr != "" { + var err error + limit, err = strconv.Atoi(limitStr) + if err != nil { + return 0, err + } + } + if limit > maxRegionLimit { + limit = maxRegionLimit + } + return limit, nil +} + +// ScatterRegionsResponse is the response for scatter regions. +type ScatterRegionsResponse struct { + ProcessedPercentage int `json:"processed-percentage"` +} + +// BuildScatterRegionsResp builds ScatterRegionsResponse. +func (h *Handler) BuildScatterRegionsResp(opsCount int, failures map[uint64]error) *ScatterRegionsResponse { + // If there existed any operator failed to be added into Operator Controller, add its regions into unProcessedRegions + percentage := 100 + if len(failures) > 0 { + percentage = 100 - 100*len(failures)/(opsCount+len(failures)) + log.Debug("scatter regions", zap.Errors("failures", func() []error { + r := make([]error, 0, len(failures)) + for _, err := range failures { + r = append(r, err) + } + return r + }())) + } + return &ScatterRegionsResponse{ + ProcessedPercentage: percentage, + } +} + +// ScatterRegionsByRange scatters regions by range. +func (h *Handler) ScatterRegionsByRange(rawStartKey, rawEndKey string, group string, retryLimit int) (int, map[uint64]error, error) { + startKey, err := hex.DecodeString(rawStartKey) + if err != nil { + return 0, nil, err + } + endKey, err := hex.DecodeString(rawEndKey) + if err != nil { + return 0, nil, err + } + co := h.GetCoordinator() + if co == nil { + return 0, nil, errs.ErrNotBootstrapped.GenWithStackByArgs() + } + return co.GetRegionScatterer().ScatterRegionsByRange(startKey, endKey, group, retryLimit) +} + +// ScatterRegionsByID scatters regions by id. +func (h *Handler) ScatterRegionsByID(ids []uint64, group string, retryLimit int, skipStoreLimit bool) (int, map[uint64]error, error) { + co := h.GetCoordinator() + if co == nil { + return 0, nil, errs.ErrNotBootstrapped.GenWithStackByArgs() + } + return co.GetRegionScatterer().ScatterRegionsByID(ids, group, retryLimit, false) +} + +// SplitRegionsResponse is the response for split regions. +type SplitRegionsResponse struct { + ProcessedPercentage int `json:"processed-percentage"` + NewRegionsID []uint64 `json:"regions-id"` +} + +// SplitRegions splits regions by split keys. +func (h *Handler) SplitRegions(ctx context.Context, rawSplitKeys []interface{}, retryLimit int) (*SplitRegionsResponse, error) { + co := h.GetCoordinator() + if co == nil { + return nil, errs.ErrNotBootstrapped.GenWithStackByArgs() + } + splitKeys := make([][]byte, 0, len(rawSplitKeys)) + for _, rawKey := range rawSplitKeys { + key, err := hex.DecodeString(rawKey.(string)) + if err != nil { + return nil, err + } + splitKeys = append(splitKeys, key) + } + + percentage, newRegionsID := co.GetRegionSplitter().SplitRegions(ctx, splitKeys, retryLimit) + s := &SplitRegionsResponse{ + ProcessedPercentage: percentage, + NewRegionsID: newRegionsID, + } + failpoint.Inject("splitResponses", func(val failpoint.Value) { + rawID, ok := val.(int) + if ok { + s.ProcessedPercentage = 100 + s.NewRegionsID = []uint64{uint64(rawID)} + } + }) + return s, nil +} + +// CheckRegionsReplicated checks if regions are replicated. +func (h *Handler) CheckRegionsReplicated(rawStartKey, rawEndKey string) (string, error) { + startKey, err := hex.DecodeString(rawStartKey) + if err != nil { + return "", err + } + endKey, err := hex.DecodeString(rawEndKey) + if err != nil { + return "", err + } + c := h.GetCluster() + if c == nil { + return "", errs.ErrNotBootstrapped.GenWithStackByArgs() + } + co := h.GetCoordinator() + if co == nil { + return "", errs.ErrNotBootstrapped.GenWithStackByArgs() + } + regions := c.ScanRegions(startKey, endKey, -1) + state := "REPLICATED" + for _, region := range regions { + if !filter.IsRegionReplicated(c, region) { + state = "INPROGRESS" + if co.IsPendingRegion(region.GetID()) { + state = "PENDING" + break + } + } + } + failpoint.Inject("mockPending", func(val failpoint.Value) { + aok, ok := val.(bool) + if ok && aok { + state = "PENDING" + } + }) + return state, nil +} + // GetRuleManager returns the rule manager. func (h *Handler) GetRuleManager() (*placement.RuleManager, error) { c := h.GetCluster() diff --git a/pkg/schedule/schedulers/split_bucket.go b/pkg/schedule/schedulers/split_bucket.go index d536f3f8dc8..6faf03e1fef 100644 --- a/pkg/schedule/schedulers/split_bucket.go +++ b/pkg/schedule/schedulers/split_bucket.go @@ -53,7 +53,7 @@ var ( splitBucketOperatorExistCounter = schedulerCounter.WithLabelValues(SplitBucketName, "operator-exist") splitBucketKeyRangeNotMatchCounter = schedulerCounter.WithLabelValues(SplitBucketName, "key-range-not-match") splitBucketNoSplitKeysCounter = schedulerCounter.WithLabelValues(SplitBucketName, "no-split-keys") - splitBucketCreateOpeartorFailCounter = schedulerCounter.WithLabelValues(SplitBucketName, "create-operator-fail") + splitBucketCreateOperatorFailCounter = schedulerCounter.WithLabelValues(SplitBucketName, "create-operator-fail") splitBucketNewOperatorCounter = schedulerCounter.WithLabelValues(SplitBucketName, "new-operator") ) @@ -281,7 +281,7 @@ func (s *splitBucketScheduler) splitBucket(plan *splitBucketPlan) []*operator.Op op, err := operator.CreateSplitRegionOperator(SplitBucketType, region, operator.OpSplit, pdpb.CheckPolicy_USEKEY, splitKey) if err != nil { - splitBucketCreateOpeartorFailCounter.Inc() + splitBucketCreateOperatorFailCounter.Inc() return nil } splitBucketNewOperatorCounter.Inc() diff --git a/pkg/utils/apiutil/serverapi/middleware.go b/pkg/utils/apiutil/serverapi/middleware.go index 9ea38e4cce0..1494dde27b8 100644 --- a/pkg/utils/apiutil/serverapi/middleware.go +++ b/pkg/utils/apiutil/serverapi/middleware.go @@ -122,17 +122,19 @@ func (h *redirector) matchMicroServiceRedirectRules(r *http.Request) (bool, stri // It will be helpful when matching the redirect rules "schedulers" or "schedulers/{name}" r.URL.Path = strings.TrimRight(r.URL.Path, "/") for _, rule := range h.microserviceRedirectRules { - if strings.HasPrefix(r.URL.Path, rule.matchPath) && slice.Contains(rule.matchMethods, r.Method) { + if strings.HasPrefix(r.URL.Path, rule.matchPath) && + slice.Contains(rule.matchMethods, r.Method) { if rule.filter != nil && !rule.filter(r) { continue } - origin := r.URL.Path + // we check the service primary addr here, so no need to check independently again. addr, ok := h.s.GetServicePrimaryAddr(r.Context(), rule.targetServiceName) if !ok || addr == "" { log.Warn("failed to get the service primary addr when trying to match redirect rules", zap.String("path", r.URL.Path)) } // If the URL contains escaped characters, use RawPath instead of Path + origin := r.URL.Path path := r.URL.Path if r.URL.RawPath != "" { path = r.URL.RawPath diff --git a/scripts/ci-subtask.sh b/scripts/ci-subtask.sh index 389d7f43341..a2e396088d6 100755 --- a/scripts/ci-subtask.sh +++ b/scripts/ci-subtask.sh @@ -29,6 +29,7 @@ else weight() { [[ $1 == "github.com/tikv/pd/server/api" ]] && return 30 [[ $1 == "github.com/tikv/pd/pkg/schedule" ]] && return 30 + [[ $1 == "pd/tests/server/api" ]] && return 30 [[ $1 =~ "pd/tests" ]] && return 5 return 1 } diff --git a/server/api/region.go b/server/api/region.go index 68e280f610c..62713cb6dcd 100644 --- a/server/api/region.go +++ b/server/api/region.go @@ -27,21 +27,18 @@ import ( "github.com/gorilla/mux" jwriter "github.com/mailru/easyjson/jwriter" - "github.com/pingcap/failpoint" + "github.com/pingcap/errors" "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/kvproto/pkg/pdpb" "github.com/pingcap/kvproto/pkg/replication_modepb" - "github.com/pingcap/log" "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/errs" "github.com/tikv/pd/pkg/keyspace" - "github.com/tikv/pd/pkg/schedule/filter" "github.com/tikv/pd/pkg/statistics" "github.com/tikv/pd/pkg/utils/apiutil" "github.com/tikv/pd/pkg/utils/typeutil" "github.com/tikv/pd/server" "github.com/unrolled/render" - "go.uber.org/zap" ) // MetaPeer is api compatible with *metapb.Peer. @@ -301,51 +298,28 @@ func (h *regionHandler) GetRegion(w http.ResponseWriter, r *http.Request) { // @Failure 400 {string} string "The input is invalid." // @Router /regions/replicated [get] func (h *regionsHandler) CheckRegionsReplicated(w http.ResponseWriter, r *http.Request) { - rc := getCluster(r) - vars := mux.Vars(r) - startKeyHex := vars["startKey"] - startKey, err := hex.DecodeString(startKeyHex) + rawStartKey := vars["startKey"] + rawEndKey := vars["endKey"] + state, err := h.Handler.CheckRegionsReplicated(rawStartKey, rawEndKey) if err != nil { h.rd.JSON(w, http.StatusBadRequest, err.Error()) return } - endKeyHex := vars["endKey"] - endKey, err := hex.DecodeString(endKeyHex) - if err != nil { - h.rd.JSON(w, http.StatusBadRequest, err.Error()) - return - } - - regions := rc.ScanRegions(startKey, endKey, -1) - state := "REPLICATED" - for _, region := range regions { - if !filter.IsRegionReplicated(rc, region) { - state = "INPROGRESS" - if rc.GetCoordinator().IsPendingRegion(region.GetID()) { - state = "PENDING" - break - } - } - } - failpoint.Inject("mockPending", func(val failpoint.Value) { - aok, ok := val.(bool) - if ok && aok { - state = "PENDING" - } - }) h.rd.JSON(w, http.StatusOK, state) } type regionsHandler struct { + *server.Handler svr *server.Server rd *render.Render } func newRegionsHandler(svr *server.Server, rd *render.Render) *regionsHandler { return ®ionsHandler{ - svr: svr, - rd: rd, + Handler: svr.GetHandler(), + svr: svr, + rd: rd, } } @@ -422,19 +396,12 @@ func (h *regionsHandler) ScanRegions(w http.ResponseWriter, r *http.Request) { rc := getCluster(r) startKey := r.URL.Query().Get("key") endKey := r.URL.Query().Get("end_key") - - limit := defaultRegionLimit - if limitStr := r.URL.Query().Get("limit"); limitStr != "" { - var err error - limit, err = strconv.Atoi(limitStr) - if err != nil { - h.rd.JSON(w, http.StatusBadRequest, err.Error()) - return - } - } - if limit > maxRegionLimit { - limit = maxRegionLimit + limit, err := h.AdjustLimit(r.URL.Query().Get("limit")) + if err != nil { + h.rd.JSON(w, http.StatusBadRequest, err.Error()) + return } + regions := rc.ScanRegions([]byte(startKey), []byte(endKey), limit) b, err := marshalRegionsInfoJSON(r.Context(), regions) if err != nil { @@ -509,16 +476,10 @@ func (h *regionsHandler) GetKeyspaceRegions(w http.ResponseWriter, r *http.Reque return } - limit := defaultRegionLimit - if limitStr := r.URL.Query().Get("limit"); limitStr != "" { - limit, err = strconv.Atoi(limitStr) - if err != nil { - h.rd.JSON(w, http.StatusBadRequest, err.Error()) - return - } - } - if limit > maxRegionLimit { - limit = maxRegionLimit + limit, err := h.AdjustLimit(r.URL.Query().Get("limit")) + if err != nil { + h.rd.JSON(w, http.StatusBadRequest, err.Error()) + return } regionBound := keyspace.MakeRegionBound(keyspaceID) regions := rc.ScanRegions(regionBound.RawLeftBound, regionBound.RawRightBound, limit) @@ -789,8 +750,6 @@ func (h *regionsHandler) GetRegionSiblings(w http.ResponseWriter, r *http.Reques } const ( - defaultRegionLimit = 16 - maxRegionLimit = 10240 minRegionHistogramSize = 1 minRegionHistogramKeys = 1000 ) @@ -892,43 +851,27 @@ func (h *regionsHandler) GetTopCPURegions(w http.ResponseWriter, r *http.Request // @Failure 400 {string} string "The input is invalid." // @Router /regions/accelerate-schedule [post] func (h *regionsHandler) AccelerateRegionsScheduleInRange(w http.ResponseWriter, r *http.Request) { - rc := getCluster(r) var input map[string]interface{} if err := apiutil.ReadJSONRespondError(h.rd, w, r.Body, &input); err != nil { return } - startKey, rawStartKey, err := apiutil.ParseKey("start_key", input) - if err != nil { - h.rd.JSON(w, http.StatusBadRequest, err.Error()) + rawStartKey, ok1 := input["start_key"].(string) + rawEndKey, ok2 := input["end_key"].(string) + if !ok1 || !ok2 { + h.rd.JSON(w, http.StatusBadRequest, "start_key or end_key is not string") return } - endKey, rawEndKey, err := apiutil.ParseKey("end_key", input) + limit, err := h.AdjustLimit(r.URL.Query().Get("limit"), 256 /*default limit*/) if err != nil { h.rd.JSON(w, http.StatusBadRequest, err.Error()) return } - limit := 256 - if limitStr := r.URL.Query().Get("limit"); limitStr != "" { - var err error - limit, err = strconv.Atoi(limitStr) - if err != nil { - h.rd.JSON(w, http.StatusBadRequest, err.Error()) - return - } - } - if limit > maxRegionLimit { - limit = maxRegionLimit - } - - regions := rc.ScanRegions(startKey, endKey, limit) - if len(regions) > 0 { - regionsIDList := make([]uint64, 0, len(regions)) - for _, region := range regions { - regionsIDList = append(regionsIDList, region.GetID()) - } - rc.AddSuspectRegions(regionsIDList...) + err = h.Handler.AccelerateRegionsScheduleInRange(rawStartKey, rawEndKey, limit) + if err != nil { + h.rd.JSON(w, http.StatusInternalServerError, err.Error()) + return } h.rd.Text(w, http.StatusOK, fmt.Sprintf("Accelerate regions scheduling in a given range [%s,%s)", rawStartKey, rawEndKey)) } @@ -943,27 +886,20 @@ func (h *regionsHandler) AccelerateRegionsScheduleInRange(w http.ResponseWriter, // @Failure 400 {string} string "The input is invalid." // @Router /regions/accelerate-schedule/batch [post] func (h *regionsHandler) AccelerateRegionsScheduleInRanges(w http.ResponseWriter, r *http.Request) { - rc := getCluster(r) var input []map[string]interface{} if err := apiutil.ReadJSONRespondError(h.rd, w, r.Body, &input); err != nil { return } - limit := 256 - if limitStr := r.URL.Query().Get("limit"); limitStr != "" { - var err error - limit, err = strconv.Atoi(limitStr) - if err != nil { - h.rd.JSON(w, http.StatusBadRequest, err.Error()) - return - } - } - if limit > maxRegionLimit { - limit = maxRegionLimit + limit, err := h.AdjustLimit(r.URL.Query().Get("limit"), 256 /*default limit*/) + if err != nil { + h.rd.JSON(w, http.StatusBadRequest, err.Error()) + return } + var msgBuilder strings.Builder msgBuilder.Grow(128) msgBuilder.WriteString("Accelerate regions scheduling in given ranges: ") - var regions []*core.RegionInfo + var startKeys, endKeys [][]byte for _, rg := range input { startKey, rawStartKey, err := apiutil.ParseKey("start_key", rg) if err != nil { @@ -975,32 +911,24 @@ func (h *regionsHandler) AccelerateRegionsScheduleInRanges(w http.ResponseWriter h.rd.JSON(w, http.StatusBadRequest, err.Error()) return } - regions = append(regions, rc.ScanRegions(startKey, endKey, limit)...) + startKeys = append(startKeys, startKey) + endKeys = append(endKeys, endKey) msgBuilder.WriteString(fmt.Sprintf("[%s,%s), ", rawStartKey, rawEndKey)) } - if len(regions) > 0 { - regionsIDList := make([]uint64, 0, len(regions)) - for _, region := range regions { - regionsIDList = append(regionsIDList, region.GetID()) - } - rc.AddSuspectRegions(regionsIDList...) + err = h.Handler.AccelerateRegionsScheduleInRanges(startKeys, endKeys, limit) + if err != nil { + h.rd.JSON(w, http.StatusInternalServerError, err.Error()) + return } h.rd.Text(w, http.StatusOK, msgBuilder.String()) } func (h *regionsHandler) GetTopNRegions(w http.ResponseWriter, r *http.Request, less func(a, b *core.RegionInfo) bool) { rc := getCluster(r) - limit := defaultRegionLimit - if limitStr := r.URL.Query().Get("limit"); limitStr != "" { - var err error - limit, err = strconv.Atoi(limitStr) - if err != nil { - h.rd.JSON(w, http.StatusBadRequest, err.Error()) - return - } - } - if limit > maxRegionLimit { - limit = maxRegionLimit + limit, err := h.AdjustLimit(r.URL.Query().Get("limit")) + if err != nil { + h.rd.JSON(w, http.StatusBadRequest, err.Error()) + return } regions := TopNRegions(rc.GetRegions(), less, limit) b, err := marshalRegionsInfoJSON(r.Context(), regions) @@ -1020,69 +948,33 @@ func (h *regionsHandler) GetTopNRegions(w http.ResponseWriter, r *http.Request, // @Failure 400 {string} string "The input is invalid." // @Router /regions/scatter [post] func (h *regionsHandler) ScatterRegions(w http.ResponseWriter, r *http.Request) { - rc := getCluster(r) var input map[string]interface{} if err := apiutil.ReadJSONRespondError(h.rd, w, r.Body, &input); err != nil { return } - _, ok1 := input["start_key"].(string) - _, ok2 := input["end_key"].(string) - group, ok := input["group"].(string) - if !ok { - group = "" - } + rawStartKey, ok1 := input["start_key"].(string) + rawEndKey, ok2 := input["end_key"].(string) + group, _ := input["group"].(string) retryLimit := 5 if rl, ok := input["retry_limit"].(float64); ok { retryLimit = int(rl) } - opsCount := 0 - var failures map[uint64]error - var err error - if ok1 && ok2 { - startKey, _, err := apiutil.ParseKey("start_key", input) - if err != nil { - h.rd.JSON(w, http.StatusBadRequest, err.Error()) - return - } - endKey, _, err := apiutil.ParseKey("end_key", input) - if err != nil { - h.rd.JSON(w, http.StatusBadRequest, err.Error()) - return - } - opsCount, failures, err = rc.GetRegionScatterer().ScatterRegionsByRange(startKey, endKey, group, retryLimit) - if err != nil { - h.rd.JSON(w, http.StatusInternalServerError, err.Error()) - return + + opsCount, failures, err := func() (int, map[uint64]error, error) { + if ok1 && ok2 { + return h.ScatterRegionsByRange(rawStartKey, rawEndKey, group, retryLimit) } - } else { ids, ok := typeutil.JSONToUint64Slice(input["regions_id"]) if !ok { - h.rd.JSON(w, http.StatusBadRequest, "regions_id is invalid") - return + return 0, nil, errors.New("regions_id is invalid") } - opsCount, failures, err = rc.GetRegionScatterer().ScatterRegionsByID(ids, group, retryLimit, false) - if err != nil { - h.rd.JSON(w, http.StatusInternalServerError, err.Error()) - return - } - } - // If there existed any operator failed to be added into Operator Controller, add its regions into unProcessedRegions - percentage := 100 - if len(failures) > 0 { - percentage = 100 - 100*len(failures)/(opsCount+len(failures)) - log.Debug("scatter regions", zap.Errors("failures", func() []error { - r := make([]error, 0, len(failures)) - for _, err := range failures { - r = append(r, err) - } - return r - }())) - } - s := struct { - ProcessedPercentage int `json:"processed-percentage"` - }{ - ProcessedPercentage: percentage, + return h.ScatterRegionsByID(ids, group, retryLimit, false) + }() + if err != nil { + h.rd.JSON(w, http.StatusInternalServerError, err.Error()) + return } + s := h.BuildScatterRegionsResp(opsCount, failures) h.rd.JSON(w, http.StatusOK, &s) } @@ -1095,16 +987,16 @@ func (h *regionsHandler) ScatterRegions(w http.ResponseWriter, r *http.Request) // @Failure 400 {string} string "The input is invalid." // @Router /regions/split [post] func (h *regionsHandler) SplitRegions(w http.ResponseWriter, r *http.Request) { - rc := getCluster(r) var input map[string]interface{} if err := apiutil.ReadJSONRespondError(h.rd, w, r.Body, &input); err != nil { return } - rawSplitKeys, ok := input["split_keys"].([]interface{}) + s, ok := input["split_keys"] if !ok { h.rd.JSON(w, http.StatusBadRequest, "split_keys should be provided.") return } + rawSplitKeys := s.([]interface{}) if len(rawSplitKeys) < 1 { h.rd.JSON(w, http.StatusBadRequest, "empty split keys.") return @@ -1113,29 +1005,11 @@ func (h *regionsHandler) SplitRegions(w http.ResponseWriter, r *http.Request) { if rl, ok := input["retry_limit"].(float64); ok { retryLimit = int(rl) } - splitKeys := make([][]byte, 0, len(rawSplitKeys)) - for _, rawKey := range rawSplitKeys { - key, err := hex.DecodeString(rawKey.(string)) - if err != nil { - h.rd.JSON(w, http.StatusBadRequest, err.Error()) - return - } - splitKeys = append(splitKeys, key) - } - s := struct { - ProcessedPercentage int `json:"processed-percentage"` - NewRegionsID []uint64 `json:"regions-id"` - }{} - percentage, newRegionsID := rc.GetRegionSplitter().SplitRegions(r.Context(), splitKeys, retryLimit) - s.ProcessedPercentage = percentage - s.NewRegionsID = newRegionsID - failpoint.Inject("splitResponses", func(val failpoint.Value) { - rawID, ok := val.(int) - if ok { - s.ProcessedPercentage = 100 - s.NewRegionsID = []uint64{uint64(rawID)} - } - }) + s, err := h.Handler.SplitRegions(r.Context(), rawSplitKeys, retryLimit) + if err != nil { + h.rd.JSON(w, http.StatusInternalServerError, err.Error()) + return + } h.rd.JSON(w, http.StatusOK, &s) } diff --git a/server/api/region_test.go b/server/api/region_test.go index 645a91e8dbc..ea2f2871a95 100644 --- a/server/api/region_test.go +++ b/server/api/region_test.go @@ -27,13 +27,11 @@ import ( "time" "github.com/docker/go-units" - "github.com/pingcap/failpoint" "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/kvproto/pkg/pdpb" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "github.com/tikv/pd/pkg/core" - "github.com/tikv/pd/pkg/schedule/placement" "github.com/tikv/pd/pkg/utils/apiutil" tu "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/server" @@ -336,99 +334,6 @@ func (suite *regionTestSuite) TestTop() { suite.checkTopRegions(fmt.Sprintf("%s/regions/cpu", suite.urlPrefix), []uint64{3, 2, 1}) } -func (suite *regionTestSuite) TestAccelerateRegionsScheduleInRange() { - re := suite.Require() - r1 := core.NewTestRegionInfo(557, 13, []byte("a1"), []byte("a2")) - r2 := core.NewTestRegionInfo(558, 14, []byte("a2"), []byte("a3")) - r3 := core.NewTestRegionInfo(559, 15, []byte("a3"), []byte("a4")) - mustRegionHeartbeat(re, suite.svr, r1) - mustRegionHeartbeat(re, suite.svr, r2) - mustRegionHeartbeat(re, suite.svr, r3) - body := fmt.Sprintf(`{"start_key":"%s", "end_key": "%s"}`, hex.EncodeToString([]byte("a1")), hex.EncodeToString([]byte("a3"))) - - err := tu.CheckPostJSON(testDialClient, fmt.Sprintf("%s/regions/accelerate-schedule", suite.urlPrefix), []byte(body), tu.StatusOK(re)) - suite.NoError(err) - idList := suite.svr.GetRaftCluster().GetSuspectRegions() - suite.Len(idList, 2) -} - -func (suite *regionTestSuite) TestAccelerateRegionsScheduleInRanges() { - re := suite.Require() - r1 := core.NewTestRegionInfo(557, 13, []byte("a1"), []byte("a2")) - r2 := core.NewTestRegionInfo(558, 14, []byte("a2"), []byte("a3")) - r3 := core.NewTestRegionInfo(559, 15, []byte("a3"), []byte("a4")) - r4 := core.NewTestRegionInfo(560, 16, []byte("a4"), []byte("a5")) - r5 := core.NewTestRegionInfo(561, 17, []byte("a5"), []byte("a6")) - mustRegionHeartbeat(re, suite.svr, r1) - mustRegionHeartbeat(re, suite.svr, r2) - mustRegionHeartbeat(re, suite.svr, r3) - mustRegionHeartbeat(re, suite.svr, r4) - mustRegionHeartbeat(re, suite.svr, r5) - body := fmt.Sprintf(`[{"start_key":"%s", "end_key": "%s"}, {"start_key":"%s", "end_key": "%s"}]`, hex.EncodeToString([]byte("a1")), hex.EncodeToString([]byte("a3")), hex.EncodeToString([]byte("a4")), hex.EncodeToString([]byte("a6"))) - - err := tu.CheckPostJSON(testDialClient, fmt.Sprintf("%s/regions/accelerate-schedule/batch", suite.urlPrefix), []byte(body), tu.StatusOK(re)) - suite.NoError(err) - idList := suite.svr.GetRaftCluster().GetSuspectRegions() - suite.Len(idList, 4) -} - -func (suite *regionTestSuite) TestScatterRegions() { - re := suite.Require() - r1 := core.NewTestRegionInfo(601, 13, []byte("b1"), []byte("b2")) - r1.GetMeta().Peers = append(r1.GetMeta().Peers, &metapb.Peer{Id: 5, StoreId: 14}, &metapb.Peer{Id: 6, StoreId: 15}) - r2 := core.NewTestRegionInfo(602, 13, []byte("b2"), []byte("b3")) - r2.GetMeta().Peers = append(r2.GetMeta().Peers, &metapb.Peer{Id: 7, StoreId: 14}, &metapb.Peer{Id: 8, StoreId: 15}) - r3 := core.NewTestRegionInfo(603, 13, []byte("b4"), []byte("b4")) - r3.GetMeta().Peers = append(r3.GetMeta().Peers, &metapb.Peer{Id: 9, StoreId: 14}, &metapb.Peer{Id: 10, StoreId: 15}) - mustRegionHeartbeat(re, suite.svr, r1) - mustRegionHeartbeat(re, suite.svr, r2) - mustRegionHeartbeat(re, suite.svr, r3) - mustPutStore(re, suite.svr, 13, metapb.StoreState_Up, metapb.NodeState_Serving, []*metapb.StoreLabel{}) - mustPutStore(re, suite.svr, 14, metapb.StoreState_Up, metapb.NodeState_Serving, []*metapb.StoreLabel{}) - mustPutStore(re, suite.svr, 15, metapb.StoreState_Up, metapb.NodeState_Serving, []*metapb.StoreLabel{}) - mustPutStore(re, suite.svr, 16, metapb.StoreState_Up, metapb.NodeState_Serving, []*metapb.StoreLabel{}) - body := fmt.Sprintf(`{"start_key":"%s", "end_key": "%s"}`, hex.EncodeToString([]byte("b1")), hex.EncodeToString([]byte("b3"))) - - err := tu.CheckPostJSON(testDialClient, fmt.Sprintf("%s/regions/scatter", suite.urlPrefix), []byte(body), tu.StatusOK(re)) - suite.NoError(err) - op1 := suite.svr.GetRaftCluster().GetOperatorController().GetOperator(601) - op2 := suite.svr.GetRaftCluster().GetOperatorController().GetOperator(602) - op3 := suite.svr.GetRaftCluster().GetOperatorController().GetOperator(603) - // At least one operator used to scatter region - suite.True(op1 != nil || op2 != nil || op3 != nil) - - body = `{"regions_id": [601, 602, 603]}` - err = tu.CheckPostJSON(testDialClient, fmt.Sprintf("%s/regions/scatter", suite.urlPrefix), []byte(body), tu.StatusOK(re)) - suite.NoError(err) -} - -func (suite *regionTestSuite) TestSplitRegions() { - re := suite.Require() - r1 := core.NewTestRegionInfo(601, 13, []byte("aaa"), []byte("ggg")) - r1.GetMeta().Peers = append(r1.GetMeta().Peers, &metapb.Peer{Id: 5, StoreId: 13}, &metapb.Peer{Id: 6, StoreId: 13}) - mustRegionHeartbeat(re, suite.svr, r1) - mustPutStore(re, suite.svr, 13, metapb.StoreState_Up, metapb.NodeState_Serving, []*metapb.StoreLabel{}) - newRegionID := uint64(11) - body := fmt.Sprintf(`{"retry_limit":%v, "split_keys": ["%s","%s","%s"]}`, 3, - hex.EncodeToString([]byte("bbb")), - hex.EncodeToString([]byte("ccc")), - hex.EncodeToString([]byte("ddd"))) - checkOpt := func(res []byte, code int, _ http.Header) { - s := &struct { - ProcessedPercentage int `json:"processed-percentage"` - NewRegionsID []uint64 `json:"regions-id"` - }{} - err := json.Unmarshal(res, s) - suite.NoError(err) - suite.Equal(100, s.ProcessedPercentage) - suite.Equal([]uint64{newRegionID}, s.NewRegionsID) - } - suite.NoError(failpoint.Enable("github.com/tikv/pd/server/api/splitResponses", fmt.Sprintf("return(%v)", newRegionID))) - err := tu.CheckPostJSON(testDialClient, fmt.Sprintf("%s/regions/split", suite.urlPrefix), []byte(body), checkOpt) - suite.NoError(failpoint.Disable("github.com/tikv/pd/server/api/splitResponses")) - suite.NoError(err) -} - func (suite *regionTestSuite) checkTopRegions(url string, regionIDs []uint64) { regions := &RegionsInfo{} err := tu.ReadGetJSON(suite.Require(), testDialClient, url, regions) @@ -651,131 +556,6 @@ func (suite *getRegionRangeHolesTestSuite) TestRegionRangeHoles() { }, *rangeHoles) } -type regionsReplicatedTestSuite struct { - suite.Suite - svr *server.Server - cleanup tu.CleanupFunc - urlPrefix string -} - -func TestRegionsReplicatedTestSuite(t *testing.T) { - suite.Run(t, new(regionsReplicatedTestSuite)) -} - -func (suite *regionsReplicatedTestSuite) SetupSuite() { - re := suite.Require() - suite.svr, suite.cleanup = mustNewServer(re) - server.MustWaitLeader(re, []*server.Server{suite.svr}) - - addr := suite.svr.GetAddr() - suite.urlPrefix = fmt.Sprintf("%s%s/api/v1", addr, apiPrefix) - - mustBootstrapCluster(re, suite.svr) -} - -func (suite *regionsReplicatedTestSuite) TearDownSuite() { - suite.cleanup() -} - -func (suite *regionsReplicatedTestSuite) TestCheckRegionsReplicated() { - re := suite.Require() - // enable placement rule - suite.NoError(tu.CheckPostJSON(testDialClient, suite.urlPrefix+"/config", []byte(`{"enable-placement-rules":"true"}`), tu.StatusOK(re))) - defer func() { - suite.NoError(tu.CheckPostJSON(testDialClient, suite.urlPrefix+"/config", []byte(`{"enable-placement-rules":"false"}`), tu.StatusOK(re))) - }() - - // add test region - r1 := core.NewTestRegionInfo(2, 1, []byte("a"), []byte("b")) - mustRegionHeartbeat(re, suite.svr, r1) - - // set the bundle - bundle := []placement.GroupBundle{ - { - ID: "5", - Index: 5, - Rules: []*placement.Rule{ - { - ID: "foo", Index: 1, Role: placement.Voter, Count: 1, - }, - }, - }, - } - - status := "" - - // invalid url - url := fmt.Sprintf(`%s/regions/replicated?startKey=%s&endKey=%s`, suite.urlPrefix, "_", "t") - err := tu.CheckGetJSON(testDialClient, url, nil, tu.Status(re, http.StatusBadRequest)) - suite.NoError(err) - - url = fmt.Sprintf(`%s/regions/replicated?startKey=%s&endKey=%s`, suite.urlPrefix, hex.EncodeToString(r1.GetStartKey()), "_") - err = tu.CheckGetJSON(testDialClient, url, nil, tu.Status(re, http.StatusBadRequest)) - suite.NoError(err) - - // correct test - url = fmt.Sprintf(`%s/regions/replicated?startKey=%s&endKey=%s`, suite.urlPrefix, hex.EncodeToString(r1.GetStartKey()), hex.EncodeToString(r1.GetEndKey())) - - // test one rule - data, err := json.Marshal(bundle) - suite.NoError(err) - err = tu.CheckPostJSON(testDialClient, suite.urlPrefix+"/config/placement-rule", data, tu.StatusOK(re)) - suite.NoError(err) - - err = tu.ReadGetJSON(re, testDialClient, url, &status) - suite.NoError(err) - suite.Equal("REPLICATED", status) - - suite.NoError(failpoint.Enable("github.com/tikv/pd/server/api/mockPending", "return(true)")) - err = tu.ReadGetJSON(re, testDialClient, url, &status) - suite.NoError(err) - suite.Equal("PENDING", status) - suite.NoError(failpoint.Disable("github.com/tikv/pd/server/api/mockPending")) - // test multiple rules - r1 = core.NewTestRegionInfo(2, 1, []byte("a"), []byte("b")) - r1.GetMeta().Peers = append(r1.GetMeta().Peers, &metapb.Peer{Id: 5, StoreId: 1}) - mustRegionHeartbeat(re, suite.svr, r1) - - bundle[0].Rules = append(bundle[0].Rules, &placement.Rule{ - ID: "bar", Index: 1, Role: placement.Voter, Count: 1, - }) - data, err = json.Marshal(bundle) - suite.NoError(err) - err = tu.CheckPostJSON(testDialClient, suite.urlPrefix+"/config/placement-rule", data, tu.StatusOK(re)) - suite.NoError(err) - - err = tu.ReadGetJSON(re, testDialClient, url, &status) - suite.NoError(err) - suite.Equal("REPLICATED", status) - - // test multiple bundles - bundle = append(bundle, placement.GroupBundle{ - ID: "6", - Index: 6, - Rules: []*placement.Rule{ - { - ID: "foo", Index: 1, Role: placement.Voter, Count: 2, - }, - }, - }) - data, err = json.Marshal(bundle) - suite.NoError(err) - err = tu.CheckPostJSON(testDialClient, suite.urlPrefix+"/config/placement-rule", data, tu.StatusOK(re)) - suite.NoError(err) - - err = tu.ReadGetJSON(re, testDialClient, url, &status) - suite.NoError(err) - suite.Equal("INPROGRESS", status) - - r1 = core.NewTestRegionInfo(2, 1, []byte("a"), []byte("b")) - r1.GetMeta().Peers = append(r1.GetMeta().Peers, &metapb.Peer{Id: 5, StoreId: 1}, &metapb.Peer{Id: 6, StoreId: 1}, &metapb.Peer{Id: 7, StoreId: 1}) - mustRegionHeartbeat(re, suite.svr, r1) - - err = tu.ReadGetJSON(re, testDialClient, url, &status) - suite.NoError(err) - suite.Equal("REPLICATED", status) -} - func TestRegionsInfoMarshal(t *testing.T) { re := require.New(t) regionWithNilPeer := core.NewRegionInfo(&metapb.Region{Id: 1}, &metapb.Peer{Id: 1}) diff --git a/server/api/server.go b/server/api/server.go index 2c015bec7ac..ad614593b2f 100644 --- a/server/api/server.go +++ b/server/api/server.go @@ -90,6 +90,26 @@ func NewHandler(_ context.Context, svr *server.Server) (http.Handler, apiutil.AP // However, the path "/region/id" is used to get the region by id, which is not what we want. return strings.Contains(r.URL.Path, "label") }), + serverapi.MicroserviceRedirectRule( + prefix+"/regions/accelerate-schedule", + scheapi.APIPathPrefix+"/regions/accelerate-schedule", + mcs.SchedulingServiceName, + []string{http.MethodPost}), + serverapi.MicroserviceRedirectRule( + prefix+"/regions/scatter", + scheapi.APIPathPrefix+"/regions/scatter", + mcs.SchedulingServiceName, + []string{http.MethodPost}), + serverapi.MicroserviceRedirectRule( + prefix+"/regions/split", + scheapi.APIPathPrefix+"/regions/split", + mcs.SchedulingServiceName, + []string{http.MethodPost}), + serverapi.MicroserviceRedirectRule( + prefix+"/regions/replicated", + scheapi.APIPathPrefix+"/regions/replicated", + mcs.SchedulingServiceName, + []string{http.MethodGet}), serverapi.MicroserviceRedirectRule( prefix+"/config/region-label/rules", scheapi.APIPathPrefix+"/config/region-label/rules", @@ -147,8 +167,6 @@ func NewHandler(_ context.Context, svr *server.Server) (http.Handler, apiutil.AP scheapi.APIPathPrefix+"/schedulers", mcs.SchedulingServiceName, []string{http.MethodPost}), - // TODO: we need to consider the case that v1 api not support restful api. - // we might change the previous path parameters to query parameters. ), negroni.Wrap(r)), ) diff --git a/tests/integrations/mcs/scheduling/api_test.go b/tests/integrations/mcs/scheduling/api_test.go index b07568580d7..0607b1dee9a 100644 --- a/tests/integrations/mcs/scheduling/api_test.go +++ b/tests/integrations/mcs/scheduling/api_test.go @@ -2,6 +2,7 @@ package scheduling_test import ( "context" + "encoding/hex" "encoding/json" "fmt" "net/http" @@ -14,6 +15,7 @@ import ( "github.com/tikv/pd/pkg/core" _ "github.com/tikv/pd/pkg/mcs/scheduling/server/apis/v1" "github.com/tikv/pd/pkg/mcs/scheduling/server/config" + "github.com/tikv/pd/pkg/mcs/utils" "github.com/tikv/pd/pkg/schedule/handler" "github.com/tikv/pd/pkg/schedule/labeler" "github.com/tikv/pd/pkg/schedule/placement" @@ -126,8 +128,10 @@ func (suite *apiTestSuite) TestAPIForward() { urlPrefix := fmt.Sprintf("%s/pd/api/v1", suite.backendEndpoints) var slice []string var resp map[string]interface{} - - // Test opeartor + testutil.Eventually(re, func() bool { + return suite.cluster.GetLeaderServer().GetServer().GetRaftCluster().IsServiceIndependent(utils.SchedulingServiceName) + }) + // Test operators err := testutil.ReadGetJSON(re, testDialClient, fmt.Sprintf("%s/%s", urlPrefix, "operators"), &slice, testutil.WithHeader(re, apiutil.ForwardToMicroServiceHeader, "true")) re.NoError(err) @@ -259,6 +263,29 @@ func (suite *apiTestSuite) TestAPIForward() { testutil.WithHeader(re, apiutil.ForwardToMicroServiceHeader, "true")) re.NoError(err) + // Test Region + body := fmt.Sprintf(`{"start_key":"%s", "end_key": "%s"}`, hex.EncodeToString([]byte("a1")), hex.EncodeToString([]byte("a3"))) + err = testutil.CheckPostJSON(testDialClient, fmt.Sprintf("%s/%s", urlPrefix, "regions/accelerate-schedule"), []byte(body), + testutil.StatusOK(re), testutil.WithHeader(re, apiutil.ForwardToMicroServiceHeader, "true")) + re.NoError(err) + body = fmt.Sprintf(`[{"start_key":"%s", "end_key": "%s"}, {"start_key":"%s", "end_key": "%s"}]`, hex.EncodeToString([]byte("a1")), hex.EncodeToString([]byte("a3")), hex.EncodeToString([]byte("a4")), hex.EncodeToString([]byte("a6"))) + err = testutil.CheckPostJSON(testDialClient, fmt.Sprintf("%s/%s", urlPrefix, "regions/accelerate-schedule/batch"), []byte(body), + testutil.StatusOK(re), testutil.WithHeader(re, apiutil.ForwardToMicroServiceHeader, "true")) + re.NoError(err) + body = fmt.Sprintf(`{"start_key":"%s", "end_key": "%s"}`, hex.EncodeToString([]byte("b1")), hex.EncodeToString([]byte("b3"))) + err = testutil.CheckPostJSON(testDialClient, fmt.Sprintf("%s/%s", urlPrefix, "regions/scatter"), []byte(body), + testutil.WithHeader(re, apiutil.ForwardToMicroServiceHeader, "true")) + suite.NoError(err) + body = fmt.Sprintf(`{"retry_limit":%v, "split_keys": ["%s","%s","%s"]}`, 3, + hex.EncodeToString([]byte("bbb")), + hex.EncodeToString([]byte("ccc")), + hex.EncodeToString([]byte("ddd"))) + err = testutil.CheckPostJSON(testDialClient, fmt.Sprintf("%s/%s", urlPrefix, "regions/split"), []byte(body), + testutil.StatusOK(re), testutil.WithHeader(re, apiutil.ForwardToMicroServiceHeader, "true")) + suite.NoError(err) + err = testutil.CheckGetJSON(testDialClient, fmt.Sprintf(`%s/regions/replicated?startKey=%s&endKey=%s`, urlPrefix, hex.EncodeToString([]byte("a1")), hex.EncodeToString([]byte("a2"))), nil, + testutil.StatusOK(re), testutil.WithHeader(re, apiutil.ForwardToMicroServiceHeader, "true")) + suite.NoError(err) // Test rules: only forward `GET` request var rules []*placement.Rule tests.MustPutRegion(re, suite.cluster, 2, 1, []byte("a"), []byte("b"), core.SetApproximateSize(60)) diff --git a/tests/pdctl/scheduler/scheduler_test.go b/tests/pdctl/scheduler/scheduler_test.go index 7c5d53e387a..7e9aeef16ee 100644 --- a/tests/pdctl/scheduler/scheduler_test.go +++ b/tests/pdctl/scheduler/scheduler_test.go @@ -362,6 +362,14 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) { "rank-formula-version": "v2", "split-thresholds": 0.2, } + checkHotSchedulerConfig := func(expect map[string]interface{}) { + testutil.Eventually(re, func() bool { + var conf1 map[string]interface{} + mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1) + return reflect.DeepEqual(expect, conf1) + }) + } + var conf map[string]interface{} mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "list"}, &conf) re.Equal(expected1, conf) @@ -370,72 +378,58 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) { echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "src-tolerance-ratio", "1.02"}, nil) re.Contains(echo, "Success!") expected1["src-tolerance-ratio"] = 1.02 - var conf1 map[string]interface{} - mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1) - re.Equal(expected1, conf1) + checkHotSchedulerConfig(expected1) echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "read-priorities", "byte,key"}, nil) re.Contains(echo, "Success!") expected1["read-priorities"] = []interface{}{"byte", "key"} - mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1) - re.Equal(expected1, conf1) + checkHotSchedulerConfig(expected1) echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "read-priorities", "key"}, nil) re.Contains(echo, "Failed!") - mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1) - re.Equal(expected1, conf1) + checkHotSchedulerConfig(expected1) echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "read-priorities", "key,byte"}, nil) re.Contains(echo, "Success!") expected1["read-priorities"] = []interface{}{"key", "byte"} - mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1) - re.Equal(expected1, conf1) + checkHotSchedulerConfig(expected1) echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "read-priorities", "foo,bar"}, nil) re.Contains(echo, "Failed!") - mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1) - re.Equal(expected1, conf1) + checkHotSchedulerConfig(expected1) echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "read-priorities", ""}, nil) re.Contains(echo, "Failed!") - mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1) - re.Equal(expected1, conf1) + checkHotSchedulerConfig(expected1) echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "read-priorities", "key,key"}, nil) re.Contains(echo, "Failed!") - mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1) - re.Equal(expected1, conf1) + checkHotSchedulerConfig(expected1) echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "read-priorities", "byte,byte"}, nil) re.Contains(echo, "Failed!") - mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1) - re.Equal(expected1, conf1) + checkHotSchedulerConfig(expected1) echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "read-priorities", "key,key,byte"}, nil) re.Contains(echo, "Failed!") - mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1) - re.Equal(expected1, conf1) + checkHotSchedulerConfig(expected1) // write-priorities is divided into write-leader-priorities and write-peer-priorities echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "write-priorities", "key,byte"}, nil) re.Contains(echo, "Failed!") re.Contains(echo, "Config item is not found.") - mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1) - re.Equal(expected1, conf1) + checkHotSchedulerConfig(expected1) echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "rank-formula-version", "v0"}, nil) re.Contains(echo, "Failed!") - mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1) + checkHotSchedulerConfig(expected1) expected1["rank-formula-version"] = "v2" echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "rank-formula-version", "v2"}, nil) re.Contains(echo, "Success!") - mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1) - re.Equal(expected1, conf1) + checkHotSchedulerConfig(expected1) expected1["rank-formula-version"] = "v1" echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "rank-formula-version", "v1"}, nil) re.Contains(echo, "Success!") - mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1) - re.Equal(expected1, conf1) + checkHotSchedulerConfig(expected1) expected1["forbid-rw-type"] = "read" echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "forbid-rw-type", "read"}, nil) re.Contains(echo, "Success!") - mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1) - re.Equal(expected1, conf1) + checkHotSchedulerConfig(expected1) // test compatibility re.Equal("2.0.0", leaderServer.GetClusterVersion().String()) @@ -446,13 +440,11 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) { } re.Equal("5.2.0", leaderServer.GetClusterVersion().String()) // After upgrading, we should not use query. - mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1) - re.Equal(conf1["read-priorities"], []interface{}{"key", "byte"}) + checkHotSchedulerConfig(expected1) // cannot set qps as write-peer-priorities echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "write-peer-priorities", "query,byte"}, nil) re.Contains(echo, "query is not allowed to be set in priorities for write-peer-priorities") - mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1) - re.Equal(conf1["write-peer-priorities"], []interface{}{"byte", "key"}) + checkHotSchedulerConfig(expected1) // test remove and add echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "remove", "balance-hot-region-scheduler"}, nil) @@ -462,7 +454,7 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) { // test balance leader config conf = make(map[string]interface{}) - conf1 = make(map[string]interface{}) + conf1 := make(map[string]interface{}) mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-leader-scheduler", "show"}, &conf) re.Equal(4., conf["batch"]) echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-leader-scheduler", "set", "batch", "3"}, nil) @@ -664,18 +656,20 @@ func TestForwardSchedulerRequest(t *testing.T) { server := cluster.GetLeaderServer() re.NoError(server.BootstrapCluster()) backendEndpoints := server.GetAddr() - tc, err := tests.NewTestSchedulingCluster(ctx, 2, backendEndpoints) + tc, err := tests.NewTestSchedulingCluster(ctx, 1, backendEndpoints) re.NoError(err) defer tc.Destroy() tc.WaitForPrimaryServing(re) cmd := pdctlCmd.GetRootCmd() args := []string{"-u", backendEndpoints, "scheduler", "show"} - var slice []string - output, err := pdctl.ExecuteCommand(cmd, args...) - re.NoError(err) - re.NoError(json.Unmarshal(output, &slice)) - re.Contains(slice, "balance-leader-scheduler") + var sches []string + testutil.Eventually(re, func() bool { + output, err := pdctl.ExecuteCommand(cmd, args...) + re.NoError(err) + re.NoError(json.Unmarshal(output, &sches)) + return slice.Contains(sches, "balance-leader-scheduler") + }) mustUsage := func(args []string) { output, err := pdctl.ExecuteCommand(cmd, args...) diff --git a/tests/server/api/operator_test.go b/tests/server/api/operator_test.go index 14b8618f6a6..c27ebbe7ee8 100644 --- a/tests/server/api/operator_test.go +++ b/tests/server/api/operator_test.go @@ -477,9 +477,7 @@ func (suite *operatorTestSuite) checkTransferRegionWithPlacementRule(cluster *te suite.NoError(err) err = tu.CheckDelete(testDialClient, regionURL, tu.StatusOK(re)) } else { - // FIXME: we should check the delete result, which should be failed, - // but the delete operator may be success because the cluster create a new operator to remove ophan peer. - err = tu.CheckDelete(testDialClient, regionURL) + err = tu.CheckDelete(testDialClient, regionURL, tu.StatusNotOK(re)) } suite.NoError(err) } diff --git a/tests/server/api/region_test.go b/tests/server/api/region_test.go new file mode 100644 index 00000000000..dcd31d6462d --- /dev/null +++ b/tests/server/api/region_test.go @@ -0,0 +1,340 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package api + +import ( + "encoding/hex" + "encoding/json" + "fmt" + "net/http" + "testing" + + "github.com/pingcap/failpoint" + "github.com/pingcap/kvproto/pkg/metapb" + "github.com/stretchr/testify/suite" + "github.com/tikv/pd/pkg/core" + "github.com/tikv/pd/pkg/schedule/placement" + tu "github.com/tikv/pd/pkg/utils/testutil" + "github.com/tikv/pd/server/config" + "github.com/tikv/pd/tests" +) + +type regionTestSuite struct { + suite.Suite +} + +func TestRegionTestSuite(t *testing.T) { + suite.Run(t, new(regionTestSuite)) +} + +func (suite *regionTestSuite) TestSplitRegions() { + env := tests.NewSchedulingTestEnvironment(suite.T()) + env.RunTestInTwoModes(suite.checkSplitRegions) +} + +func (suite *regionTestSuite) checkSplitRegions(cluster *tests.TestCluster) { + leader := cluster.GetLeaderServer() + urlPrefix := leader.GetAddr() + "/pd/api/v1" + re := suite.Require() + s1 := &metapb.Store{ + Id: 13, + State: metapb.StoreState_Up, + NodeState: metapb.NodeState_Serving, + } + tests.MustPutStore(re, cluster, s1) + r1 := core.NewTestRegionInfo(601, 13, []byte("aaa"), []byte("ggg")) + r1.GetMeta().Peers = append(r1.GetMeta().Peers, &metapb.Peer{Id: 5, StoreId: 14}, &metapb.Peer{Id: 6, StoreId: 15}) + tests.MustPutRegionInfo(re, cluster, r1) + suite.checkRegionCount(cluster, 1) + + newRegionID := uint64(11) + body := fmt.Sprintf(`{"retry_limit":%v, "split_keys": ["%s","%s","%s"]}`, 3, + hex.EncodeToString([]byte("bbb")), + hex.EncodeToString([]byte("ccc")), + hex.EncodeToString([]byte("ddd"))) + checkOpt := func(res []byte, code int, _ http.Header) { + s := &struct { + ProcessedPercentage int `json:"processed-percentage"` + NewRegionsID []uint64 `json:"regions-id"` + }{} + err := json.Unmarshal(res, s) + suite.NoError(err) + suite.Equal(100, s.ProcessedPercentage) + suite.Equal([]uint64{newRegionID}, s.NewRegionsID) + } + suite.NoError(failpoint.Enable("github.com/tikv/pd/pkg/schedule/handler/splitResponses", fmt.Sprintf("return(%v)", newRegionID))) + err := tu.CheckPostJSON(testDialClient, fmt.Sprintf("%s/regions/split", urlPrefix), []byte(body), checkOpt) + suite.NoError(failpoint.Disable("github.com/tikv/pd/pkg/schedule/handler/splitResponses")) + suite.NoError(err) +} + +func (suite *regionTestSuite) TestAccelerateRegionsScheduleInRange() { + env := tests.NewSchedulingTestEnvironment(suite.T(), func(conf *config.Config, serverName string) { + // FIXME: enable placement rules + conf.Replication.EnablePlacementRules = false + conf.Replication.MaxReplicas = 1 + }) + env.RunTestInTwoModes(suite.checkAccelerateRegionsScheduleInRange) +} + +func (suite *regionTestSuite) checkAccelerateRegionsScheduleInRange(cluster *tests.TestCluster) { + leader := cluster.GetLeaderServer() + urlPrefix := leader.GetAddr() + "/pd/api/v1" + re := suite.Require() + for i := 1; i <= 3; i++ { + s1 := &metapb.Store{ + Id: uint64(i), + State: metapb.StoreState_Up, + NodeState: metapb.NodeState_Serving, + } + tests.MustPutStore(re, cluster, s1) + } + r1 := core.NewTestRegionInfo(557, 1, []byte("a1"), []byte("a2")) + r2 := core.NewTestRegionInfo(558, 2, []byte("a2"), []byte("a3")) + r3 := core.NewTestRegionInfo(559, 3, []byte("a3"), []byte("a4")) + tests.MustPutRegionInfo(re, cluster, r1) + tests.MustPutRegionInfo(re, cluster, r2) + tests.MustPutRegionInfo(re, cluster, r3) + suite.checkRegionCount(cluster, 3) + + body := fmt.Sprintf(`{"start_key":"%s", "end_key": "%s"}`, hex.EncodeToString([]byte("a1")), hex.EncodeToString([]byte("a3"))) + err := tu.CheckPostJSON(testDialClient, fmt.Sprintf("%s/regions/accelerate-schedule", urlPrefix), []byte(body), + tu.StatusOK(re)) + suite.NoError(err) + idList := leader.GetRaftCluster().GetSuspectRegions() + if sche := cluster.GetSchedulingPrimaryServer(); sche != nil { + idList = sche.GetCluster().GetCoordinator().GetCheckerController().GetSuspectRegions() + } + re.Len(idList, 2, len(idList)) +} + +func (suite *regionTestSuite) TestAccelerateRegionsScheduleInRanges() { + env := tests.NewSchedulingTestEnvironment(suite.T(), func(conf *config.Config, serverName string) { + // FIXME: enable placement rules + conf.Replication.EnablePlacementRules = false + conf.Replication.MaxReplicas = 1 + }) + env.RunTestInTwoModes(suite.checkAccelerateRegionsScheduleInRanges) +} + +func (suite *regionTestSuite) checkAccelerateRegionsScheduleInRanges(cluster *tests.TestCluster) { + leader := cluster.GetLeaderServer() + urlPrefix := leader.GetAddr() + "/pd/api/v1" + re := suite.Require() + for i := 1; i <= 5; i++ { + s1 := &metapb.Store{ + Id: uint64(i), + State: metapb.StoreState_Up, + NodeState: metapb.NodeState_Serving, + } + tests.MustPutStore(re, cluster, s1) + } + r1 := core.NewTestRegionInfo(557, 1, []byte("a1"), []byte("a2")) + r2 := core.NewTestRegionInfo(558, 2, []byte("a2"), []byte("a3")) + r3 := core.NewTestRegionInfo(559, 3, []byte("a3"), []byte("a4")) + r4 := core.NewTestRegionInfo(560, 4, []byte("a4"), []byte("a5")) + r5 := core.NewTestRegionInfo(561, 5, []byte("a5"), []byte("a6")) + tests.MustPutRegionInfo(re, cluster, r1) + tests.MustPutRegionInfo(re, cluster, r2) + tests.MustPutRegionInfo(re, cluster, r3) + tests.MustPutRegionInfo(re, cluster, r4) + tests.MustPutRegionInfo(re, cluster, r5) + suite.checkRegionCount(cluster, 5) + + body := fmt.Sprintf(`[{"start_key":"%s", "end_key": "%s"}, {"start_key":"%s", "end_key": "%s"}]`, + hex.EncodeToString([]byte("a1")), hex.EncodeToString([]byte("a3")), hex.EncodeToString([]byte("a4")), hex.EncodeToString([]byte("a6"))) + err := tu.CheckPostJSON(testDialClient, fmt.Sprintf("%s/regions/accelerate-schedule/batch", urlPrefix), []byte(body), + tu.StatusOK(re)) + suite.NoError(err) + idList := leader.GetRaftCluster().GetSuspectRegions() + if sche := cluster.GetSchedulingPrimaryServer(); sche != nil { + idList = sche.GetCluster().GetCoordinator().GetCheckerController().GetSuspectRegions() + } + re.Len(idList, 4) +} + +func (suite *regionTestSuite) TestScatterRegions() { + env := tests.NewSchedulingTestEnvironment(suite.T()) + env.RunTestInTwoModes(suite.checkScatterRegions) +} + +func (suite *regionTestSuite) checkScatterRegions(cluster *tests.TestCluster) { + leader := cluster.GetLeaderServer() + urlPrefix := leader.GetAddr() + "/pd/api/v1" + re := suite.Require() + for i := 13; i <= 16; i++ { + s1 := &metapb.Store{ + Id: uint64(i), + State: metapb.StoreState_Up, + NodeState: metapb.NodeState_Serving, + } + tests.MustPutStore(re, cluster, s1) + } + r1 := core.NewTestRegionInfo(601, 13, []byte("b1"), []byte("b2")) + r1.GetMeta().Peers = append(r1.GetMeta().Peers, &metapb.Peer{Id: 5, StoreId: 14}, &metapb.Peer{Id: 6, StoreId: 15}) + r2 := core.NewTestRegionInfo(602, 13, []byte("b2"), []byte("b3")) + r2.GetMeta().Peers = append(r2.GetMeta().Peers, &metapb.Peer{Id: 7, StoreId: 14}, &metapb.Peer{Id: 8, StoreId: 15}) + r3 := core.NewTestRegionInfo(603, 13, []byte("b4"), []byte("b4")) + r3.GetMeta().Peers = append(r3.GetMeta().Peers, &metapb.Peer{Id: 9, StoreId: 14}, &metapb.Peer{Id: 10, StoreId: 15}) + tests.MustPutRegionInfo(re, cluster, r1) + tests.MustPutRegionInfo(re, cluster, r2) + tests.MustPutRegionInfo(re, cluster, r3) + suite.checkRegionCount(cluster, 3) + + body := fmt.Sprintf(`{"start_key":"%s", "end_key": "%s"}`, hex.EncodeToString([]byte("b1")), hex.EncodeToString([]byte("b3"))) + err := tu.CheckPostJSON(testDialClient, fmt.Sprintf("%s/regions/scatter", urlPrefix), []byte(body), tu.StatusOK(re)) + suite.NoError(err) + oc := leader.GetRaftCluster().GetOperatorController() + if sche := cluster.GetSchedulingPrimaryServer(); sche != nil { + oc = sche.GetCoordinator().GetOperatorController() + } + + op1 := oc.GetOperator(601) + op2 := oc.GetOperator(602) + op3 := oc.GetOperator(603) + // At least one operator used to scatter region + suite.True(op1 != nil || op2 != nil || op3 != nil) + + body = `{"regions_id": [601, 602, 603]}` + err = tu.CheckPostJSON(testDialClient, fmt.Sprintf("%s/regions/scatter", urlPrefix), []byte(body), tu.StatusOK(re)) + suite.NoError(err) +} + +func (suite *regionTestSuite) TestCheckRegionsReplicated() { + env := tests.NewSchedulingTestEnvironment(suite.T(), + func(conf *config.Config, serverName string) { + conf.Replication.EnablePlacementRules = true + }) + env.RunTestInPDMode(suite.checkRegionsReplicated) +} + +func (suite *regionTestSuite) checkRegionsReplicated(cluster *tests.TestCluster) { + leader := cluster.GetLeaderServer() + urlPrefix := leader.GetAddr() + "/pd/api/v1" + re := suite.Require() + + // add test region + s1 := &metapb.Store{ + Id: 1, + State: metapb.StoreState_Up, + NodeState: metapb.NodeState_Serving, + } + tests.MustPutStore(re, cluster, s1) + r1 := core.NewTestRegionInfo(2, 1, []byte("a"), []byte("b")) + tests.MustPutRegionInfo(re, cluster, r1) + suite.checkRegionCount(cluster, 1) + + // set the bundle + bundle := []placement.GroupBundle{ + { + ID: "5", + Index: 5, + Rules: []*placement.Rule{ + { + ID: "foo", Index: 1, Role: placement.Voter, Count: 1, + }, + }, + }, + } + + status := "" + + // invalid url + url := fmt.Sprintf(`%s/regions/replicated?startKey=%s&endKey=%s`, urlPrefix, "_", "t") + err := tu.CheckGetJSON(testDialClient, url, nil, tu.Status(re, http.StatusBadRequest)) + suite.NoError(err) + + url = fmt.Sprintf(`%s/regions/replicated?startKey=%s&endKey=%s`, urlPrefix, hex.EncodeToString(r1.GetStartKey()), "_") + err = tu.CheckGetJSON(testDialClient, url, nil, tu.Status(re, http.StatusBadRequest)) + suite.NoError(err) + + // correct test + url = fmt.Sprintf(`%s/regions/replicated?startKey=%s&endKey=%s`, urlPrefix, hex.EncodeToString(r1.GetStartKey()), hex.EncodeToString(r1.GetEndKey())) + err = tu.CheckGetJSON(testDialClient, url, nil, tu.StatusOK(re)) + suite.NoError(err) + + // test one rule + data, err := json.Marshal(bundle) + suite.NoError(err) + err = tu.CheckPostJSON(testDialClient, urlPrefix+"/config/placement-rule", data, tu.StatusOK(re)) + suite.NoError(err) + + tu.Eventually(re, func() bool { + err = tu.ReadGetJSON(re, testDialClient, url, &status) + suite.NoError(err) + return status == "REPLICATED" + }) + + suite.NoError(failpoint.Enable("github.com/tikv/pd/pkg/schedule/handler/mockPending", "return(true)")) + err = tu.ReadGetJSON(re, testDialClient, url, &status) + suite.NoError(err) + suite.Equal("PENDING", status) + suite.NoError(failpoint.Disable("github.com/tikv/pd/pkg/schedule/handler/mockPending")) + // test multiple rules + r1 = core.NewTestRegionInfo(2, 1, []byte("a"), []byte("b")) + r1.GetMeta().Peers = append(r1.GetMeta().Peers, &metapb.Peer{Id: 5, StoreId: 1}) + tests.MustPutRegionInfo(re, cluster, r1) + + bundle[0].Rules = append(bundle[0].Rules, &placement.Rule{ + ID: "bar", Index: 1, Role: placement.Voter, Count: 1, + }) + data, err = json.Marshal(bundle) + suite.NoError(err) + err = tu.CheckPostJSON(testDialClient, urlPrefix+"/config/placement-rule", data, tu.StatusOK(re)) + suite.NoError(err) + + err = tu.ReadGetJSON(re, testDialClient, url, &status) + suite.NoError(err) + suite.Equal("REPLICATED", status) + + // test multiple bundles + bundle = append(bundle, placement.GroupBundle{ + ID: "6", + Index: 6, + Rules: []*placement.Rule{ + { + ID: "foo", Index: 1, Role: placement.Voter, Count: 2, + }, + }, + }) + data, err = json.Marshal(bundle) + suite.NoError(err) + err = tu.CheckPostJSON(testDialClient, urlPrefix+"/config/placement-rule", data, tu.StatusOK(re)) + suite.NoError(err) + + err = tu.ReadGetJSON(re, testDialClient, url, &status) + suite.NoError(err) + suite.Equal("INPROGRESS", status) + + r1 = core.NewTestRegionInfo(2, 1, []byte("a"), []byte("b")) + r1.GetMeta().Peers = append(r1.GetMeta().Peers, &metapb.Peer{Id: 5, StoreId: 1}, &metapb.Peer{Id: 6, StoreId: 1}, &metapb.Peer{Id: 7, StoreId: 1}) + tests.MustPutRegionInfo(re, cluster, r1) + + err = tu.ReadGetJSON(re, testDialClient, url, &status) + suite.NoError(err) + suite.Equal("REPLICATED", status) +} + +func (suite *regionTestSuite) checkRegionCount(cluster *tests.TestCluster, count int) { + leader := cluster.GetLeaderServer() + tu.Eventually(suite.Require(), func() bool { + return leader.GetRaftCluster().GetRegionCount([]byte{}, []byte{}).Count == count + }) + if sche := cluster.GetSchedulingPrimaryServer(); sche != nil { + tu.Eventually(suite.Require(), func() bool { + return sche.GetCluster().GetRegionCount([]byte{}, []byte{}) == count + }) + } +} diff --git a/tests/server/api/rule_test.go b/tests/server/api/rule_test.go index ffdf56b6567..ac52362df4e 100644 --- a/tests/server/api/rule_test.go +++ b/tests/server/api/rule_test.go @@ -565,8 +565,10 @@ func (suite *ruleTestSuite) checkGetAllByKey(cluster *tests.TestCluster) { var resp []*placement.Rule url := fmt.Sprintf("%s/rules/key/%s", urlPrefix, testCase.key) if testCase.success { - err = tu.ReadGetJSON(re, testDialClient, url, &resp) - suite.Len(resp, testCase.respSize) + tu.Eventually(re, func() bool { + err = tu.ReadGetJSON(re, testDialClient, url, &resp) + return len(resp) == testCase.respSize + }) } else { err = tu.CheckGetJSON(testDialClient, url, nil, tu.Status(re, testCase.code)) } diff --git a/tests/testutil.go b/tests/testutil.go index 2ccf6fb76be..1bdd7ae10dd 100644 --- a/tests/testutil.go +++ b/tests/testutil.go @@ -34,6 +34,7 @@ import ( scheduling "github.com/tikv/pd/pkg/mcs/scheduling/server" sc "github.com/tikv/pd/pkg/mcs/scheduling/server/config" tso "github.com/tikv/pd/pkg/mcs/tso/server" + "github.com/tikv/pd/pkg/mcs/utils" "github.com/tikv/pd/pkg/utils/logutil" "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/pkg/versioninfo" @@ -272,11 +273,13 @@ func (s *SchedulingTestEnvironment) RunTestInPDMode(test func(*TestCluster)) { func (s *SchedulingTestEnvironment) RunTestInAPIMode(test func(*TestCluster)) { s.t.Log("start to run test in api mode") re := require.New(s.t) + re.NoError(failpoint.Enable("github.com/tikv/pd/server/cluster/highFrequencyClusterJobs", `return(true)`)) re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/mcs/scheduling/server/fastUpdateMember", `return(true)`)) s.startCluster(apiMode) test(s.cluster) s.cleanup() re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/mcs/scheduling/server/fastUpdateMember")) + re.NoError(failpoint.Disable("github.com/tikv/pd/server/cluster/highFrequencyClusterJobs")) s.t.Log("finish to run test in api mode") } @@ -299,7 +302,6 @@ func (s *SchedulingTestEnvironment) startCluster(m mode) { leaderServer := s.cluster.GetServer(s.cluster.GetLeader()) re.NoError(leaderServer.BootstrapCluster()) case apiMode: - re.NoError(failpoint.Enable("github.com/tikv/pd/server/cluster/highFrequencyClusterJobs", `return(true)`)) s.cluster, err = NewTestAPICluster(s.ctx, 1, s.opts...) re.NoError(err) err = s.cluster.RunInitialServers() @@ -314,6 +316,8 @@ func (s *SchedulingTestEnvironment) startCluster(m mode) { tc.WaitForPrimaryServing(re) s.cluster.SetSchedulingCluster(tc) time.Sleep(200 * time.Millisecond) // wait for scheduling cluster to update member - re.NoError(failpoint.Disable("github.com/tikv/pd/server/cluster/highFrequencyClusterJobs")) + testutil.Eventually(re, func() bool { + return s.cluster.GetLeaderServer().GetServer().GetRaftCluster().IsServiceIndependent(utils.SchedulingServiceName) + }) } } From eafc1ed899bc0e14cff163c1c198fc68d8319969 Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Fri, 24 Nov 2023 15:34:13 +0800 Subject: [PATCH 046/137] mcs: fix TestAllocIDAfterLeaderChange (#7426) close tikv/pd#7427 Signed-off-by: Ryan Leung --- errors.toml | 5 +++++ pkg/errs/errno.go | 2 ++ pkg/utils/apiutil/apiutil.go | 7 +------ pkg/utils/apiutil/serverapi/middleware.go | 12 +++++++++--- server/apiv2/middlewares/redirector.go | 13 ++++++++----- 5 files changed, 25 insertions(+), 14 deletions(-) diff --git a/errors.toml b/errors.toml index b6123058310..a318fc32492 100644 --- a/errors.toml +++ b/errors.toml @@ -16,6 +16,11 @@ error = ''' redirect failed ''' +["PD:apiutil:ErrRedirectToNotLeader"] +error = ''' +redirect to not leader +''' + ["PD:autoscaling:ErrEmptyMetricsResponse"] error = ''' metrics response from Prometheus is empty diff --git a/pkg/errs/errno.go b/pkg/errs/errno.go index b8a882cd187..a4320238374 100644 --- a/pkg/errs/errno.go +++ b/pkg/errs/errno.go @@ -197,6 +197,8 @@ var ( var ( ErrRedirect = errors.Normalize("redirect failed", errors.RFCCodeText("PD:apiutil:ErrRedirect")) ErrOptionNotExist = errors.Normalize("the option %s does not exist", errors.RFCCodeText("PD:apiutil:ErrOptionNotExist")) + // ErrRedirectToNotLeader is the error message for redirect to not leader. + ErrRedirectToNotLeader = errors.Normalize("redirect to not leader", errors.RFCCodeText("PD:apiutil:ErrRedirectToNotLeader")) ) // grpcutil errors diff --git a/pkg/utils/apiutil/apiutil.go b/pkg/utils/apiutil/apiutil.go index 8928688fed9..164b3c0783d 100644 --- a/pkg/utils/apiutil/apiutil.go +++ b/pkg/utils/apiutil/apiutil.go @@ -61,11 +61,6 @@ const ( // ForwardToMicroServiceHeader is used to mark the request is forwarded to micro service. ForwardToMicroServiceHeader = "Forward-To-Micro-Service" - // ErrRedirectFailed is the error message for redirect failed. - ErrRedirectFailed = "redirect failed" - // ErrRedirectToNotLeader is the error message for redirect to not leader. - ErrRedirectToNotLeader = "redirect to not leader" - chunkSize = 4096 ) @@ -459,7 +454,7 @@ func (p *customReverseProxies) ServeHTTP(w http.ResponseWriter, r *http.Request) } return } - http.Error(w, ErrRedirectFailed, http.StatusInternalServerError) + http.Error(w, errs.ErrRedirect.FastGenByArgs().Error(), http.StatusInternalServerError) } // copyHeader duplicates the HTTP headers from the source `src` to the destination `dst`. diff --git a/pkg/utils/apiutil/serverapi/middleware.go b/pkg/utils/apiutil/serverapi/middleware.go index 1494dde27b8..26b36a88ca8 100644 --- a/pkg/utils/apiutil/serverapi/middleware.go +++ b/pkg/utils/apiutil/serverapi/middleware.go @@ -162,7 +162,13 @@ func (h *redirector) matchMicroServiceRedirectRules(r *http.Request) (bool, stri func (h *redirector) ServeHTTP(w http.ResponseWriter, r *http.Request, next http.HandlerFunc) { redirectToMicroService, targetAddr := h.matchMicroServiceRedirectRules(r) allowFollowerHandle := len(r.Header.Get(apiutil.PDAllowFollowerHandleHeader)) > 0 - if !h.s.IsClosed() && (allowFollowerHandle || h.s.GetMember().IsLeader()) && !redirectToMicroService { + + if h.s.IsClosed() { + http.Error(w, errs.ErrServerNotStarted.FastGenByArgs().Error(), http.StatusInternalServerError) + return + } + + if (allowFollowerHandle || h.s.GetMember().IsLeader()) && !redirectToMicroService { next(w, r) return } @@ -170,7 +176,7 @@ func (h *redirector) ServeHTTP(w http.ResponseWriter, r *http.Request, next http // Prevent more than one redirection. if name := r.Header.Get(apiutil.PDRedirectorHeader); len(name) != 0 { log.Error("redirect but server is not leader", zap.String("from", name), zap.String("server", h.s.Name()), errs.ZapError(errs.ErrRedirect)) - http.Error(w, apiutil.ErrRedirectToNotLeader, http.StatusInternalServerError) + http.Error(w, errs.ErrRedirectToNotLeader.FastGenByArgs().Error(), http.StatusInternalServerError) return } @@ -189,7 +195,7 @@ func (h *redirector) ServeHTTP(w http.ResponseWriter, r *http.Request, next http var clientUrls []string if redirectToMicroService { if len(targetAddr) == 0 { - http.Error(w, apiutil.ErrRedirectFailed, http.StatusInternalServerError) + http.Error(w, errs.ErrRedirect.FastGenByArgs().Error(), http.StatusInternalServerError) return } clientUrls = append(clientUrls, targetAddr) diff --git a/server/apiv2/middlewares/redirector.go b/server/apiv2/middlewares/redirector.go index 285f096e823..37c06de1585 100644 --- a/server/apiv2/middlewares/redirector.go +++ b/server/apiv2/middlewares/redirector.go @@ -30,9 +30,13 @@ import ( func Redirector() gin.HandlerFunc { return func(c *gin.Context) { svr := c.MustGet(ServerContextKey).(*server.Server) + + if svr.IsClosed() { + c.AbortWithStatusJSON(http.StatusInternalServerError, errs.ErrServerNotStarted.FastGenByArgs().Error()) + return + } allowFollowerHandle := len(c.Request.Header.Get(apiutil.PDAllowFollowerHandleHeader)) > 0 - isLeader := svr.GetMember().IsLeader() - if !svr.IsClosed() && (allowFollowerHandle || isLeader) { + if allowFollowerHandle || svr.GetMember().IsLeader() { c.Next() return } @@ -46,12 +50,11 @@ func Redirector() gin.HandlerFunc { c.Request.Header.Set(apiutil.PDRedirectorHeader, svr.Name()) - leader := svr.GetMember().GetLeader() - if leader == nil { + if svr.GetMember().GetLeader() == nil { c.AbortWithStatusJSON(http.StatusServiceUnavailable, errs.ErrLeaderNil.FastGenByArgs().Error()) return } - clientUrls := leader.GetClientUrls() + clientUrls := svr.GetMember().GetLeader().GetClientUrls() urls := make([]url.URL, 0, len(clientUrls)) for _, item := range clientUrls { u, err := url.Parse(item) From 2109a0f547b77e8a56dc8030991c43ce5944413f Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Fri, 24 Nov 2023 17:08:44 +0800 Subject: [PATCH 047/137] ci: adjust the task weight algorithm to make ci tasks more uniform (#7434) ref tikv/pd#4399 Signed-off-by: lhy1024 --- scripts/ci-subtask.sh | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/scripts/ci-subtask.sh b/scripts/ci-subtask.sh index a2e396088d6..dd29895327b 100755 --- a/scripts/ci-subtask.sh +++ b/scripts/ci-subtask.sh @@ -29,11 +29,22 @@ else weight() { [[ $1 == "github.com/tikv/pd/server/api" ]] && return 30 [[ $1 == "github.com/tikv/pd/pkg/schedule" ]] && return 30 - [[ $1 == "pd/tests/server/api" ]] && return 30 + [[ $1 == "github.com/tikv/pd/pkg/core" ]] && return 30 + [[ $1 == "github.com/tikv/pd/tests/server/api" ]] && return 30 [[ $1 =~ "pd/tests" ]] && return 5 return 1 } + # Create an associative array to store the weight of each task. + declare -A task_weights + for t in ${tasks[@]}; do + weight $t + task_weights[$t]=$? + done + + # Sort tasks by weight in descending order. + tasks=($(printf "%s\n" "${tasks[@]}" | sort -rn)) + scores=($(seq "$1" | xargs -I{} echo 0)) res=() @@ -42,8 +53,7 @@ else for i in ${!scores[@]}; do [[ ${scores[i]} -lt ${scores[$min_i]} ]] && min_i=$i done - weight $t - scores[$min_i]=$((${scores[$min_i]} + $?)) + scores[$min_i]=$((${scores[$min_i]} + ${task_weights[$t]})) [[ $(($min_i + 1)) -eq $2 ]] && res+=($t) done printf "%s " "${res[@]}" From 4e9240a626855a66c849b753f34a4af3656e767b Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Fri, 24 Nov 2023 17:23:44 +0800 Subject: [PATCH 048/137] mcs: make TestDefaultKeyspaceGroup stable (#7437) close tikv/pd#7374 Signed-off-by: Ryan Leung Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- tests/integrations/mcs/keyspace/tso_keyspace_group_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/integrations/mcs/keyspace/tso_keyspace_group_test.go b/tests/integrations/mcs/keyspace/tso_keyspace_group_test.go index 1415acc46d1..7dcce498d56 100644 --- a/tests/integrations/mcs/keyspace/tso_keyspace_group_test.go +++ b/tests/integrations/mcs/keyspace/tso_keyspace_group_test.go @@ -301,7 +301,10 @@ func (suite *keyspaceGroupTestSuite) TestDefaultKeyspaceGroup() { return code == http.StatusOK && kg != nil }, testutil.WithWaitFor(time.Second*1)) suite.Equal(utils.DefaultKeyspaceGroupID, kg.ID) - suite.Len(kg.Members, utils.DefaultKeyspaceGroupReplicaCount) + // the allocNodesToAllKeyspaceGroups loop will run every 100ms. + testutil.Eventually(suite.Require(), func() bool { + return len(kg.Members) == utils.DefaultKeyspaceGroupReplicaCount + }) for _, member := range kg.Members { suite.Contains(nodes, member.Address) } From 58e9b2088b069a2c6a1e58be9dac752e46593e44 Mon Sep 17 00:00:00 2001 From: JmPotato Date: Mon, 27 Nov 2023 12:07:14 +0800 Subject: [PATCH 049/137] client/http: implement more rule and batch related interfaces (#7430) ref tikv/pd#7300 - Implement more rule and batch related interfaces. - Add more types and methods. - Refine the tests. Signed-off-by: JmPotato --- client/http/api.go | 35 ++-- client/http/client.go | 101 +++++++++-- client/http/types.go | 61 ++++++- pkg/schedule/checker/checker_controller.go | 5 + pkg/utils/tsoutil/tsoutil.go | 5 + server/cluster/cluster.go | 6 +- server/cluster/scheduling_controller.go | 7 + tests/integrations/client/http_client_test.go | 160 ++++++++++++++---- 8 files changed, 319 insertions(+), 61 deletions(-) diff --git a/client/http/api.go b/client/http/api.go index 1826e2231ee..c6d4f2dfb74 100644 --- a/client/http/api.go +++ b/client/http/api.go @@ -23,19 +23,20 @@ import ( // The following constants are the paths of PD HTTP APIs. const ( // Metadata - HotRead = "/pd/api/v1/hotspot/regions/read" - HotWrite = "/pd/api/v1/hotspot/regions/write" - HotHistory = "/pd/api/v1/hotspot/regions/history" - RegionByIDPrefix = "/pd/api/v1/region/id" - regionByKey = "/pd/api/v1/region/key" - Regions = "/pd/api/v1/regions" - regionsByKey = "/pd/api/v1/regions/key" - RegionsByStoreIDPrefix = "/pd/api/v1/regions/store" - EmptyRegions = "/pd/api/v1/regions/check/empty-region" - AccelerateSchedule = "/pd/api/v1/regions/accelerate-schedule" - store = "/pd/api/v1/store" - Stores = "/pd/api/v1/stores" - StatsRegion = "/pd/api/v1/stats/region" + HotRead = "/pd/api/v1/hotspot/regions/read" + HotWrite = "/pd/api/v1/hotspot/regions/write" + HotHistory = "/pd/api/v1/hotspot/regions/history" + RegionByIDPrefix = "/pd/api/v1/region/id" + regionByKey = "/pd/api/v1/region/key" + Regions = "/pd/api/v1/regions" + regionsByKey = "/pd/api/v1/regions/key" + RegionsByStoreIDPrefix = "/pd/api/v1/regions/store" + EmptyRegions = "/pd/api/v1/regions/check/empty-region" + AccelerateSchedule = "/pd/api/v1/regions/accelerate-schedule" + AccelerateScheduleInBatch = "/pd/api/v1/regions/accelerate-schedule/batch" + store = "/pd/api/v1/store" + Stores = "/pd/api/v1/stores" + StatsRegion = "/pd/api/v1/stats/region" // Config Config = "/pd/api/v1/config" ClusterVersion = "/pd/api/v1/config/cluster-version" @@ -44,8 +45,11 @@ const ( // Rule PlacementRule = "/pd/api/v1/config/rule" PlacementRules = "/pd/api/v1/config/rules" + PlacementRulesInBatch = "/pd/api/v1/config/rules/batch" placementRulesByGroup = "/pd/api/v1/config/rules/group" PlacementRuleBundle = "/pd/api/v1/config/placement-rule" + placementRuleGroup = "/pd/api/v1/config/rule_group" + placementRuleGroups = "/pd/api/v1/config/rule_groups" RegionLabelRule = "/pd/api/v1/config/region-label/rule" RegionLabelRules = "/pd/api/v1/config/region-label/rules" RegionLabelRulesByIDs = "/pd/api/v1/config/region-label/rules/ids" @@ -136,6 +140,11 @@ func PlacementRuleBundleWithPartialParameter(partial bool) string { return fmt.Sprintf("%s?partial=%t", PlacementRuleBundle, partial) } +// PlacementRuleGroupByID returns the path of PD HTTP API to get placement rule group by ID. +func PlacementRuleGroupByID(id string) string { + return fmt.Sprintf("%s/%s", placementRuleGroup, id) +} + // SchedulerByName returns the scheduler API with the given scheduler name. func SchedulerByName(name string) string { return fmt.Sprintf("%s/%s", Schedulers, name) diff --git a/client/http/client.go b/client/http/client.go index 880489aa85c..1d8c2d5c427 100644 --- a/client/http/client.go +++ b/client/http/client.go @@ -18,6 +18,7 @@ import ( "bytes" "context" "crypto/tls" + "encoding/hex" "encoding/json" "fmt" "io" @@ -46,25 +47,31 @@ type Client interface { GetRegionByID(context.Context, uint64) (*RegionInfo, error) GetRegionByKey(context.Context, []byte) (*RegionInfo, error) GetRegions(context.Context) (*RegionsInfo, error) - GetRegionsByKeyRange(context.Context, []byte, []byte, int) (*RegionsInfo, error) + GetRegionsByKeyRange(context.Context, *KeyRange, int) (*RegionsInfo, error) GetRegionsByStoreID(context.Context, uint64) (*RegionsInfo, error) GetHotReadRegions(context.Context) (*StoreHotPeersInfos, error) GetHotWriteRegions(context.Context) (*StoreHotPeersInfos, error) - GetRegionStatusByKeyRange(context.Context, []byte, []byte) (*RegionStats, error) + GetRegionStatusByKeyRange(context.Context, *KeyRange) (*RegionStats, error) GetStores(context.Context) (*StoresInfo, error) /* Rule-related interfaces */ GetAllPlacementRuleBundles(context.Context) ([]*GroupBundle, error) GetPlacementRuleBundleByGroup(context.Context, string) (*GroupBundle, error) GetPlacementRulesByGroup(context.Context, string) ([]*Rule, error) SetPlacementRule(context.Context, *Rule) error + SetPlacementRuleInBatch(context.Context, []*RuleOp) error SetPlacementRuleBundles(context.Context, []*GroupBundle, bool) error DeletePlacementRule(context.Context, string, string) error + GetAllPlacementRuleGroups(context.Context) ([]*RuleGroup, error) + GetPlacementRuleGroupByID(context.Context, string) (*RuleGroup, error) + SetPlacementRuleGroup(context.Context, *RuleGroup) error + DeletePlacementRuleGroupByID(context.Context, string) error GetAllRegionLabelRules(context.Context) ([]*LabelRule, error) GetRegionLabelRulesByIDs(context.Context, []string) ([]*LabelRule, error) SetRegionLabelRule(context.Context, *LabelRule) error PatchRegionLabelRules(context.Context, *LabelRulePatch) error /* Scheduling-related interfaces */ - AccelerateSchedule(context.Context, []byte, []byte) error + AccelerateSchedule(context.Context, *KeyRange) error + AccelerateScheduleInBatch(context.Context, []*KeyRange) error /* Other interfaces */ GetMinResolvedTSByStoresIDs(context.Context, []uint64) (uint64, map[uint64]uint64, error) @@ -308,10 +315,10 @@ func (c *client) GetRegions(ctx context.Context) (*RegionsInfo, error) { } // GetRegionsByKeyRange gets the regions info by key range. If the limit is -1, it will return all regions within the range. -func (c *client) GetRegionsByKeyRange(ctx context.Context, startKey, endKey []byte, limit int) (*RegionsInfo, error) { +func (c *client) GetRegionsByKeyRange(ctx context.Context, keyRange *KeyRange, limit int) (*RegionsInfo, error) { var regions RegionsInfo err := c.requestWithRetry(ctx, - "GetRegionsByKeyRange", RegionsByKey(startKey, endKey, limit), + "GetRegionsByKeyRange", RegionsByKey(keyRange.StartKey, keyRange.EndKey, limit), http.MethodGet, http.NoBody, ®ions) if err != nil { return nil, err @@ -356,10 +363,10 @@ func (c *client) GetHotWriteRegions(ctx context.Context) (*StoreHotPeersInfos, e } // GetRegionStatusByKeyRange gets the region status by key range. -func (c *client) GetRegionStatusByKeyRange(ctx context.Context, startKey, endKey []byte) (*RegionStats, error) { +func (c *client) GetRegionStatusByKeyRange(ctx context.Context, keyRange *KeyRange) (*RegionStats, error) { var regionStats RegionStats err := c.requestWithRetry(ctx, - "GetRegionStatusByKeyRange", RegionStatsByKeyRange(startKey, endKey), + "GetRegionStatusByKeyRange", RegionStatsByKeyRange(keyRange.StartKey, keyRange.StartKey), http.MethodGet, http.NoBody, ®ionStats, ) if err != nil { @@ -427,6 +434,17 @@ func (c *client) SetPlacementRule(ctx context.Context, rule *Rule) error { http.MethodPost, bytes.NewBuffer(ruleJSON), nil) } +// SetPlacementRuleInBatch sets the placement rules in batch. +func (c *client) SetPlacementRuleInBatch(ctx context.Context, ruleOps []*RuleOp) error { + ruleOpsJSON, err := json.Marshal(ruleOps) + if err != nil { + return errors.Trace(err) + } + return c.requestWithRetry(ctx, + "SetPlacementRuleInBatch", PlacementRulesInBatch, + http.MethodPost, bytes.NewBuffer(ruleOpsJSON), nil) +} + // SetPlacementRuleBundles sets the placement rule bundles. // If `partial` is false, all old configurations will be over-written and dropped. func (c *client) SetPlacementRuleBundles(ctx context.Context, bundles []*GroupBundle, partial bool) error { @@ -446,6 +464,48 @@ func (c *client) DeletePlacementRule(ctx context.Context, group, id string) erro http.MethodDelete, http.NoBody, nil) } +// GetAllPlacementRuleGroups gets all placement rule groups. +func (c *client) GetAllPlacementRuleGroups(ctx context.Context) ([]*RuleGroup, error) { + var ruleGroups []*RuleGroup + err := c.requestWithRetry(ctx, + "GetAllPlacementRuleGroups", placementRuleGroups, + http.MethodGet, http.NoBody, &ruleGroups) + if err != nil { + return nil, err + } + return ruleGroups, nil +} + +// GetPlacementRuleGroupByID gets the placement rule group by ID. +func (c *client) GetPlacementRuleGroupByID(ctx context.Context, id string) (*RuleGroup, error) { + var ruleGroup RuleGroup + err := c.requestWithRetry(ctx, + "GetPlacementRuleGroupByID", PlacementRuleGroupByID(id), + http.MethodGet, http.NoBody, &ruleGroup) + if err != nil { + return nil, err + } + return &ruleGroup, nil +} + +// SetPlacementRuleGroup sets the placement rule group. +func (c *client) SetPlacementRuleGroup(ctx context.Context, ruleGroup *RuleGroup) error { + ruleGroupJSON, err := json.Marshal(ruleGroup) + if err != nil { + return errors.Trace(err) + } + return c.requestWithRetry(ctx, + "SetPlacementRuleGroup", placementRuleGroup, + http.MethodPost, bytes.NewBuffer(ruleGroupJSON), nil) +} + +// DeletePlacementRuleGroupByID deletes the placement rule group by ID. +func (c *client) DeletePlacementRuleGroupByID(ctx context.Context, id string) error { + return c.requestWithRetry(ctx, + "DeletePlacementRuleGroupByID", PlacementRuleGroupByID(id), + http.MethodDelete, http.NoBody, nil) +} + // GetAllRegionLabelRules gets all region label rules. func (c *client) GetAllRegionLabelRules(ctx context.Context) ([]*LabelRule, error) { var labelRules []*LabelRule @@ -497,17 +557,34 @@ func (c *client) PatchRegionLabelRules(ctx context.Context, labelRulePatch *Labe } // AccelerateSchedule accelerates the scheduling of the regions within the given key range. -func (c *client) AccelerateSchedule(ctx context.Context, startKey, endKey []byte) error { - input := map[string]string{ - "start_key": url.QueryEscape(string(startKey)), - "end_key": url.QueryEscape(string(endKey)), +func (c *client) AccelerateSchedule(ctx context.Context, keyRange *KeyRange) error { + inputJSON, err := json.Marshal(map[string]string{ + "start_key": url.QueryEscape(hex.EncodeToString(keyRange.StartKey)), + "end_key": url.QueryEscape(hex.EncodeToString(keyRange.EndKey)), + }) + if err != nil { + return errors.Trace(err) + } + return c.requestWithRetry(ctx, + "AccelerateSchedule", AccelerateSchedule, + http.MethodPost, bytes.NewBuffer(inputJSON), nil) +} + +// AccelerateScheduleInBatch accelerates the scheduling of the regions within the given key ranges in batch. +func (c *client) AccelerateScheduleInBatch(ctx context.Context, keyRanges []*KeyRange) error { + input := make([]map[string]string, 0, len(keyRanges)) + for _, keyRange := range keyRanges { + input = append(input, map[string]string{ + "start_key": url.QueryEscape(hex.EncodeToString(keyRange.StartKey)), + "end_key": url.QueryEscape(hex.EncodeToString(keyRange.EndKey)), + }) } inputJSON, err := json.Marshal(input) if err != nil { return errors.Trace(err) } return c.requestWithRetry(ctx, - "AccelerateSchedule", AccelerateSchedule, + "AccelerateScheduleInBatch", AccelerateScheduleInBatch, http.MethodPost, bytes.NewBuffer(inputJSON), nil) } diff --git a/client/http/types.go b/client/http/types.go index f948286c2b5..56d59bafa58 100644 --- a/client/http/types.go +++ b/client/http/types.go @@ -14,7 +14,16 @@ package http -import "time" +import ( + "encoding/json" + "time" +) + +// KeyRange defines a range of keys. +type KeyRange struct { + StartKey []byte `json:"start_key"` + EndKey []byte `json:"end_key"` +} // NOTICE: the structures below are copied from the PD API definitions. // Please make sure the consistency if any change happens to the PD API. @@ -247,6 +256,56 @@ type Rule struct { CreateTimestamp uint64 `json:"create_timestamp,omitempty"` // only set at runtime, recorded rule create timestamp } +// String returns the string representation of this rule. +func (r *Rule) String() string { + b, _ := json.Marshal(r) + return string(b) +} + +// Clone returns a copy of Rule. +func (r *Rule) Clone() *Rule { + var clone Rule + json.Unmarshal([]byte(r.String()), &clone) + clone.StartKey = append(r.StartKey[:0:0], r.StartKey...) + clone.EndKey = append(r.EndKey[:0:0], r.EndKey...) + return &clone +} + +// RuleOpType indicates the operation type +type RuleOpType string + +const ( + // RuleOpAdd a placement rule, only need to specify the field *Rule + RuleOpAdd RuleOpType = "add" + // RuleOpDel a placement rule, only need to specify the field `GroupID`, `ID`, `MatchID` + RuleOpDel RuleOpType = "del" +) + +// RuleOp is for batching placement rule actions. +// The action type is distinguished by the field `Action`. +type RuleOp struct { + *Rule // information of the placement rule to add/delete the operation type + Action RuleOpType `json:"action"` + DeleteByIDPrefix bool `json:"delete_by_id_prefix"` // if action == delete, delete by the prefix of id +} + +func (r RuleOp) String() string { + b, _ := json.Marshal(r) + return string(b) +} + +// RuleGroup defines properties of a rule group. +type RuleGroup struct { + ID string `json:"id,omitempty"` + Index int `json:"index,omitempty"` + Override bool `json:"override,omitempty"` +} + +func (g *RuleGroup) String() string { + b, _ := json.Marshal(g) + return string(b) +} + // GroupBundle represents a rule group and all rules belong to the group. type GroupBundle struct { ID string `json:"group_id"` diff --git a/pkg/schedule/checker/checker_controller.go b/pkg/schedule/checker/checker_controller.go index 68b794f417a..355226cd2d8 100644 --- a/pkg/schedule/checker/checker_controller.go +++ b/pkg/schedule/checker/checker_controller.go @@ -221,6 +221,11 @@ func (c *Controller) ClearSuspectKeyRanges() { c.suspectKeyRanges.Clear() } +// ClearSuspectRegions clears the suspect regions, only for unit test +func (c *Controller) ClearSuspectRegions() { + c.suspectRegions.Clear() +} + // IsPendingRegion returns true if the given region is in the pending list. func (c *Controller) IsPendingRegion(regionID uint64) bool { _, exist := c.ruleChecker.pendingList.Get(regionID) diff --git a/pkg/utils/tsoutil/tsoutil.go b/pkg/utils/tsoutil/tsoutil.go index 796012ae031..43d8b09aa49 100644 --- a/pkg/utils/tsoutil/tsoutil.go +++ b/pkg/utils/tsoutil/tsoutil.go @@ -25,6 +25,11 @@ const ( logicalBits = (1 << physicalShiftBits) - 1 ) +// TimeToTS converts a `time.Time` to an `uint64` TS. +func TimeToTS(t time.Time) uint64 { + return ComposeTS(t.UnixNano()/int64(time.Millisecond), 0) +} + // ParseTS parses the ts to (physical,logical). func ParseTS(ts uint64) (time.Time, uint64) { physical, logical := ParseTSUint64(ts) diff --git a/server/cluster/cluster.go b/server/cluster/cluster.go index 1c3d8a03a98..ec8ca3a0d65 100644 --- a/server/cluster/cluster.go +++ b/server/cluster/cluster.go @@ -2241,7 +2241,9 @@ func (c *RaftCluster) SetMinResolvedTS(storeID, minResolvedTS uint64) error { return nil } -func (c *RaftCluster) checkAndUpdateMinResolvedTS() (uint64, bool) { +// CheckAndUpdateMinResolvedTS checks and updates the min resolved ts of the cluster. +// This is exported for testing purpose. +func (c *RaftCluster) CheckAndUpdateMinResolvedTS() (uint64, bool) { c.Lock() defer c.Unlock() @@ -2284,7 +2286,7 @@ func (c *RaftCluster) runMinResolvedTSJob() { case <-ticker.C: interval = c.opt.GetMinResolvedTSPersistenceInterval() if interval != 0 { - if current, needPersist := c.checkAndUpdateMinResolvedTS(); needPersist { + if current, needPersist := c.CheckAndUpdateMinResolvedTS(); needPersist { c.storage.SaveMinResolvedTS(current) } } else { diff --git a/server/cluster/scheduling_controller.go b/server/cluster/scheduling_controller.go index 5e8cb8462df..a36e7159cfd 100644 --- a/server/cluster/scheduling_controller.go +++ b/server/cluster/scheduling_controller.go @@ -438,6 +438,13 @@ func (sc *schedulingController) ClearSuspectKeyRanges() { sc.coordinator.GetCheckerController().ClearSuspectKeyRanges() } +// ClearSuspectRegions clears the suspect regions, only for unit test +func (sc *schedulingController) ClearSuspectRegions() { + sc.mu.RLock() + defer sc.mu.RUnlock() + sc.coordinator.GetCheckerController().ClearSuspectRegions() +} + // AddSuspectKeyRange adds the key range with the its ruleID as the key // The instance of each keyRange is like following format: // [2][]byte: start key/end key diff --git a/tests/integrations/client/http_client_test.go b/tests/integrations/client/http_client_test.go index 213aa57de46..dc901b1d290 100644 --- a/tests/integrations/client/http_client_test.go +++ b/tests/integrations/client/http_client_test.go @@ -17,13 +17,19 @@ package client_test import ( "context" "math" + "net/http" "sort" "testing" + "time" + "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" pd "github.com/tikv/pd/client/http" + "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/schedule/labeler" "github.com/tikv/pd/pkg/schedule/placement" + "github.com/tikv/pd/pkg/utils/testutil" + "github.com/tikv/pd/pkg/utils/tsoutil" "github.com/tikv/pd/tests" ) @@ -69,21 +75,30 @@ func (suite *httpClientTestSuite) TearDownSuite() { func (suite *httpClientTestSuite) TestGetMinResolvedTSByStoresIDs() { re := suite.Require() - // Get the cluster-level min resolved TS. + testMinResolvedTS := tsoutil.TimeToTS(time.Now()) + raftCluster := suite.cluster.GetLeaderServer().GetRaftCluster() + err := raftCluster.SetMinResolvedTS(1, testMinResolvedTS) + re.NoError(err) + // Make sure the min resolved TS is updated. + testutil.Eventually(re, func() bool { + minResolvedTS, _ := raftCluster.CheckAndUpdateMinResolvedTS() + return minResolvedTS == testMinResolvedTS + }) + // Wait for the cluster-level min resolved TS to be initialized. minResolvedTS, storeMinResolvedTSMap, err := suite.client.GetMinResolvedTSByStoresIDs(suite.ctx, nil) re.NoError(err) - re.Greater(minResolvedTS, uint64(0)) + re.Equal(testMinResolvedTS, minResolvedTS) re.Empty(storeMinResolvedTSMap) // Get the store-level min resolved TS. minResolvedTS, storeMinResolvedTSMap, err = suite.client.GetMinResolvedTSByStoresIDs(suite.ctx, []uint64{1}) re.NoError(err) - re.Greater(minResolvedTS, uint64(0)) + re.Equal(testMinResolvedTS, minResolvedTS) re.Len(storeMinResolvedTSMap, 1) re.Equal(minResolvedTS, storeMinResolvedTSMap[1]) // Get the store-level min resolved TS with an invalid store ID. minResolvedTS, storeMinResolvedTSMap, err = suite.client.GetMinResolvedTSByStoresIDs(suite.ctx, []uint64{1, 2}) re.NoError(err) - re.Greater(minResolvedTS, uint64(0)) + re.Equal(testMinResolvedTS, minResolvedTS) re.Len(storeMinResolvedTSMap, 2) re.Equal(minResolvedTS, storeMinResolvedTSMap[1]) re.Equal(uint64(math.MaxUint64), storeMinResolvedTSMap[2]) @@ -98,15 +113,15 @@ func (suite *httpClientTestSuite) TestRule() { bundle, err := suite.client.GetPlacementRuleBundleByGroup(suite.ctx, placement.DefaultGroupID) re.NoError(err) re.Equal(bundles[0], bundle) - rules, err := suite.client.GetPlacementRulesByGroup(suite.ctx, placement.DefaultGroupID) - re.NoError(err) - re.Len(rules, 1) - re.Equal(placement.DefaultGroupID, rules[0].GroupID) - re.Equal(placement.DefaultRuleID, rules[0].ID) - re.Equal(pd.Voter, rules[0].Role) - re.Equal(3, rules[0].Count) + // Check if we have the default rule. + suite.checkRule(re, &pd.Rule{ + GroupID: placement.DefaultGroupID, + ID: placement.DefaultRuleID, + Role: pd.Voter, + Count: 3, + }, 1, true) // Should be the same as the rules in the bundle. - re.Equal(bundle.Rules, rules) + suite.checkRule(re, bundle.Rules[0], 1, true) testRule := &pd.Rule{ GroupID: placement.DefaultGroupID, ID: "test", @@ -115,20 +130,24 @@ func (suite *httpClientTestSuite) TestRule() { } err = suite.client.SetPlacementRule(suite.ctx, testRule) re.NoError(err) - rules, err = suite.client.GetPlacementRulesByGroup(suite.ctx, placement.DefaultGroupID) - re.NoError(err) - re.Len(rules, 2) - re.Equal(placement.DefaultGroupID, rules[1].GroupID) - re.Equal("test", rules[1].ID) - re.Equal(pd.Voter, rules[1].Role) - re.Equal(3, rules[1].Count) + suite.checkRule(re, testRule, 2, true) err = suite.client.DeletePlacementRule(suite.ctx, placement.DefaultGroupID, "test") re.NoError(err) - rules, err = suite.client.GetPlacementRulesByGroup(suite.ctx, placement.DefaultGroupID) + suite.checkRule(re, testRule, 1, false) + testRuleOp := &pd.RuleOp{ + Rule: testRule, + Action: pd.RuleOpAdd, + } + err = suite.client.SetPlacementRuleInBatch(suite.ctx, []*pd.RuleOp{testRuleOp}) + re.NoError(err) + suite.checkRule(re, testRule, 2, true) + testRuleOp = &pd.RuleOp{ + Rule: testRule, + Action: pd.RuleOpDel, + } + err = suite.client.SetPlacementRuleInBatch(suite.ctx, []*pd.RuleOp{testRuleOp}) re.NoError(err) - re.Len(rules, 1) - re.Equal(placement.DefaultGroupID, rules[0].GroupID) - re.Equal(placement.DefaultRuleID, rules[0].ID) + suite.checkRule(re, testRule, 1, false) err = suite.client.SetPlacementRuleBundles(suite.ctx, []*pd.GroupBundle{ { ID: placement.DefaultGroupID, @@ -136,14 +155,63 @@ func (suite *httpClientTestSuite) TestRule() { }, }, true) re.NoError(err) - bundles, err = suite.client.GetAllPlacementRuleBundles(suite.ctx) + suite.checkRule(re, testRule, 1, true) + ruleGroups, err := suite.client.GetAllPlacementRuleGroups(suite.ctx) re.NoError(err) - re.Len(bundles, 1) - re.Equal(placement.DefaultGroupID, bundles[0].ID) - re.Len(bundles[0].Rules, 1) - // Make sure the create timestamp is not zero to pass the later assertion. - testRule.CreateTimestamp = bundles[0].Rules[0].CreateTimestamp - re.Equal(testRule, bundles[0].Rules[0]) + re.Len(ruleGroups, 1) + re.Equal(placement.DefaultGroupID, ruleGroups[0].ID) + ruleGroup, err := suite.client.GetPlacementRuleGroupByID(suite.ctx, placement.DefaultGroupID) + re.NoError(err) + re.Equal(ruleGroups[0], ruleGroup) + testRuleGroup := &pd.RuleGroup{ + ID: "test-group", + Index: 1, + Override: true, + } + err = suite.client.SetPlacementRuleGroup(suite.ctx, testRuleGroup) + re.NoError(err) + ruleGroup, err = suite.client.GetPlacementRuleGroupByID(suite.ctx, testRuleGroup.ID) + re.NoError(err) + re.Equal(testRuleGroup, ruleGroup) + err = suite.client.DeletePlacementRuleGroupByID(suite.ctx, testRuleGroup.ID) + re.NoError(err) + ruleGroup, err = suite.client.GetPlacementRuleGroupByID(suite.ctx, testRuleGroup.ID) + re.ErrorContains(err, http.StatusText(http.StatusNotFound)) + re.Empty(ruleGroup) +} + +func (suite *httpClientTestSuite) checkRule( + re *require.Assertions, + rule *pd.Rule, totalRuleCount int, exist bool, +) { + // Check through the `GetPlacementRulesByGroup` API. + rules, err := suite.client.GetPlacementRulesByGroup(suite.ctx, rule.GroupID) + re.NoError(err) + checkRuleFunc(re, rules, rule, totalRuleCount, exist) + // Check through the `GetPlacementRuleBundleByGroup` API. + bundle, err := suite.client.GetPlacementRuleBundleByGroup(suite.ctx, rule.GroupID) + re.NoError(err) + checkRuleFunc(re, bundle.Rules, rule, totalRuleCount, exist) +} + +func checkRuleFunc( + re *require.Assertions, + rules []*pd.Rule, rule *pd.Rule, totalRuleCount int, exist bool, +) { + re.Len(rules, totalRuleCount) + for _, r := range rules { + if r.ID != rule.ID { + continue + } + re.Equal(rule.GroupID, r.GroupID) + re.Equal(rule.ID, r.ID) + re.Equal(rule.Role, r.Role) + re.Equal(rule.Count, r.Count) + return + } + if exist { + re.Failf("Failed to check the rule", "rule %+v not found", rule) + } } func (suite *httpClientTestSuite) TestRegionLabel() { @@ -202,10 +270,36 @@ func (suite *httpClientTestSuite) TestRegionLabel() { func (suite *httpClientTestSuite) TestAccelerateSchedule() { re := suite.Require() - suspectRegions := suite.cluster.GetLeaderServer().GetRaftCluster().GetSuspectRegions() + raftCluster := suite.cluster.GetLeaderServer().GetRaftCluster() + for _, region := range []*core.RegionInfo{ + core.NewTestRegionInfo(10, 1, []byte("a1"), []byte("a2")), + core.NewTestRegionInfo(11, 1, []byte("a2"), []byte("a3")), + } { + err := raftCluster.HandleRegionHeartbeat(region) + re.NoError(err) + } + suspectRegions := raftCluster.GetSuspectRegions() re.Len(suspectRegions, 0) - err := suite.client.AccelerateSchedule(suite.ctx, []byte("a1"), []byte("a2")) + err := suite.client.AccelerateSchedule(suite.ctx, &pd.KeyRange{ + StartKey: []byte("a1"), + EndKey: []byte("a2")}) re.NoError(err) - suspectRegions = suite.cluster.GetLeaderServer().GetRaftCluster().GetSuspectRegions() + suspectRegions = raftCluster.GetSuspectRegions() re.Len(suspectRegions, 1) + raftCluster.ClearSuspectRegions() + suspectRegions = raftCluster.GetSuspectRegions() + re.Len(suspectRegions, 0) + err = suite.client.AccelerateScheduleInBatch(suite.ctx, []*pd.KeyRange{ + { + StartKey: []byte("a1"), + EndKey: []byte("a2"), + }, + { + StartKey: []byte("a2"), + EndKey: []byte("a3"), + }, + }) + re.NoError(err) + suspectRegions = raftCluster.GetSuspectRegions() + re.Len(suspectRegions, 2) } From 9f4803d8bd058b27240ff778dd29c446bf71d521 Mon Sep 17 00:00:00 2001 From: JmPotato Date: Mon, 27 Nov 2023 15:50:44 +0800 Subject: [PATCH 050/137] client/http: use KeyRange to differentiate the encoding format of the key (#7449) ref tikv/pd#7300 Use `KeyRange` to differentiate the encoding format of the key. Signed-off-by: JmPotato --- client/http/api.go | 17 ++++------ client/http/client.go | 20 ++++++----- client/http/types.go | 34 +++++++++++++++++-- tests/integrations/client/http_client_test.go | 14 ++------ 4 files changed, 53 insertions(+), 32 deletions(-) diff --git a/client/http/api.go b/client/http/api.go index c6d4f2dfb74..6b317330b61 100644 --- a/client/http/api.go +++ b/client/http/api.go @@ -83,13 +83,11 @@ func RegionByKey(key []byte) string { return fmt.Sprintf("%s/%s", regionByKey, url.QueryEscape(string(key))) } -// RegionsByKey returns the path of PD HTTP API to scan regions with given start key, end key and limit parameters. -func RegionsByKey(startKey, endKey []byte, limit int) string { +// RegionsByKeyRange returns the path of PD HTTP API to scan regions with given start key, end key and limit parameters. +func RegionsByKeyRange(keyRange *KeyRange, limit int) string { + startKeyStr, endKeyStr := keyRange.EscapeAsUTF8Str() return fmt.Sprintf("%s?start_key=%s&end_key=%s&limit=%d", - regionsByKey, - url.QueryEscape(string(startKey)), - url.QueryEscape(string(endKey)), - limit) + regionsByKey, startKeyStr, endKeyStr, limit) } // RegionsByStoreID returns the path of PD HTTP API to get regions by store ID. @@ -98,11 +96,10 @@ func RegionsByStoreID(storeID uint64) string { } // RegionStatsByKeyRange returns the path of PD HTTP API to get region stats by start key and end key. -func RegionStatsByKeyRange(startKey, endKey []byte) string { +func RegionStatsByKeyRange(keyRange *KeyRange) string { + startKeyStr, endKeyStr := keyRange.EscapeAsUTF8Str() return fmt.Sprintf("%s?start_key=%s&end_key=%s", - StatsRegion, - url.QueryEscape(string(startKey)), - url.QueryEscape(string(endKey))) + StatsRegion, startKeyStr, endKeyStr) } // StoreByID returns the store API with store ID parameter. diff --git a/client/http/client.go b/client/http/client.go index 1d8c2d5c427..cd7be205c12 100644 --- a/client/http/client.go +++ b/client/http/client.go @@ -18,12 +18,10 @@ import ( "bytes" "context" "crypto/tls" - "encoding/hex" "encoding/json" "fmt" "io" "net/http" - "net/url" "strings" "time" @@ -315,10 +313,11 @@ func (c *client) GetRegions(ctx context.Context) (*RegionsInfo, error) { } // GetRegionsByKeyRange gets the regions info by key range. If the limit is -1, it will return all regions within the range. +// The keys in the key range should be encoded in the UTF-8 bytes format. func (c *client) GetRegionsByKeyRange(ctx context.Context, keyRange *KeyRange, limit int) (*RegionsInfo, error) { var regions RegionsInfo err := c.requestWithRetry(ctx, - "GetRegionsByKeyRange", RegionsByKey(keyRange.StartKey, keyRange.EndKey, limit), + "GetRegionsByKeyRange", RegionsByKeyRange(keyRange, limit), http.MethodGet, http.NoBody, ®ions) if err != nil { return nil, err @@ -363,10 +362,11 @@ func (c *client) GetHotWriteRegions(ctx context.Context) (*StoreHotPeersInfos, e } // GetRegionStatusByKeyRange gets the region status by key range. +// The keys in the key range should be encoded in the UTF-8 bytes format. func (c *client) GetRegionStatusByKeyRange(ctx context.Context, keyRange *KeyRange) (*RegionStats, error) { var regionStats RegionStats err := c.requestWithRetry(ctx, - "GetRegionStatusByKeyRange", RegionStatsByKeyRange(keyRange.StartKey, keyRange.StartKey), + "GetRegionStatusByKeyRange", RegionStatsByKeyRange(keyRange), http.MethodGet, http.NoBody, ®ionStats, ) if err != nil { @@ -557,10 +557,12 @@ func (c *client) PatchRegionLabelRules(ctx context.Context, labelRulePatch *Labe } // AccelerateSchedule accelerates the scheduling of the regions within the given key range. +// The keys in the key range should be encoded in the hex bytes format (without encoding to the UTF-8 bytes). func (c *client) AccelerateSchedule(ctx context.Context, keyRange *KeyRange) error { + startKey, endKey := keyRange.EscapeAsHexStr() inputJSON, err := json.Marshal(map[string]string{ - "start_key": url.QueryEscape(hex.EncodeToString(keyRange.StartKey)), - "end_key": url.QueryEscape(hex.EncodeToString(keyRange.EndKey)), + "start_key": startKey, + "end_key": endKey, }) if err != nil { return errors.Trace(err) @@ -571,12 +573,14 @@ func (c *client) AccelerateSchedule(ctx context.Context, keyRange *KeyRange) err } // AccelerateScheduleInBatch accelerates the scheduling of the regions within the given key ranges in batch. +// The keys in the key ranges should be encoded in the hex bytes format (without encoding to the UTF-8 bytes). func (c *client) AccelerateScheduleInBatch(ctx context.Context, keyRanges []*KeyRange) error { input := make([]map[string]string, 0, len(keyRanges)) for _, keyRange := range keyRanges { + startKey, endKey := keyRange.EscapeAsHexStr() input = append(input, map[string]string{ - "start_key": url.QueryEscape(hex.EncodeToString(keyRange.StartKey)), - "end_key": url.QueryEscape(hex.EncodeToString(keyRange.EndKey)), + "start_key": startKey, + "end_key": endKey, }) } inputJSON, err := json.Marshal(input) diff --git a/client/http/types.go b/client/http/types.go index 56d59bafa58..df448e7e20d 100644 --- a/client/http/types.go +++ b/client/http/types.go @@ -15,14 +15,42 @@ package http import ( + "encoding/hex" "encoding/json" + "net/url" "time" ) -// KeyRange defines a range of keys. +// KeyRange defines a range of keys in bytes. type KeyRange struct { - StartKey []byte `json:"start_key"` - EndKey []byte `json:"end_key"` + startKey []byte + endKey []byte +} + +// NewKeyRange creates a new key range structure with the given start key and end key bytes. +// Notice: the actual encoding of the key range is not specified here. It should be either UTF-8 or hex. +// - UTF-8 means the key has already been encoded into a string with UTF-8 encoding, like: +// []byte{52 56 54 53 54 99 54 99 54 102 50 48 53 55 54 102 55 50 54 99 54 52}, which will later be converted to "48656c6c6f20576f726c64" +// by using `string()` method. +// - Hex means the key is just a raw hex bytes without encoding to a UTF-8 string, like: +// []byte{72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100}, which will later be converted to "48656c6c6f20576f726c64" +// by using `hex.EncodeToString()` method. +func NewKeyRange(startKey, endKey []byte) *KeyRange { + return &KeyRange{startKey, endKey} +} + +// EscapeAsUTF8Str returns the URL escaped key strings as they are UTF-8 encoded. +func (r *KeyRange) EscapeAsUTF8Str() (startKeyStr, endKeyStr string) { + startKeyStr = url.QueryEscape(string(r.startKey)) + endKeyStr = url.QueryEscape(string(r.endKey)) + return +} + +// EscapeAsHexStr returns the URL escaped key strings as they are hex encoded. +func (r *KeyRange) EscapeAsHexStr() (startKeyStr, endKeyStr string) { + startKeyStr = url.QueryEscape(hex.EncodeToString(r.startKey)) + endKeyStr = url.QueryEscape(hex.EncodeToString(r.endKey)) + return } // NOTICE: the structures below are copied from the PD API definitions. diff --git a/tests/integrations/client/http_client_test.go b/tests/integrations/client/http_client_test.go index dc901b1d290..3a52e91e1f8 100644 --- a/tests/integrations/client/http_client_test.go +++ b/tests/integrations/client/http_client_test.go @@ -280,9 +280,7 @@ func (suite *httpClientTestSuite) TestAccelerateSchedule() { } suspectRegions := raftCluster.GetSuspectRegions() re.Len(suspectRegions, 0) - err := suite.client.AccelerateSchedule(suite.ctx, &pd.KeyRange{ - StartKey: []byte("a1"), - EndKey: []byte("a2")}) + err := suite.client.AccelerateSchedule(suite.ctx, pd.NewKeyRange([]byte("a1"), []byte("a2"))) re.NoError(err) suspectRegions = raftCluster.GetSuspectRegions() re.Len(suspectRegions, 1) @@ -290,14 +288,8 @@ func (suite *httpClientTestSuite) TestAccelerateSchedule() { suspectRegions = raftCluster.GetSuspectRegions() re.Len(suspectRegions, 0) err = suite.client.AccelerateScheduleInBatch(suite.ctx, []*pd.KeyRange{ - { - StartKey: []byte("a1"), - EndKey: []byte("a2"), - }, - { - StartKey: []byte("a2"), - EndKey: []byte("a3"), - }, + pd.NewKeyRange([]byte("a1"), []byte("a2")), + pd.NewKeyRange([]byte("a2"), []byte("a3")), }) re.NoError(err) suspectRegions = raftCluster.GetSuspectRegions() From a09717bcd9da5fb0db095429a7f93c7cfa1ee660 Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Mon, 27 Nov 2023 16:43:14 +0800 Subject: [PATCH 051/137] mcs: support ttl config (#7409) close tikv/pd#7296 Signed-off-by: Ryan Leung Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- pkg/mcs/scheduling/server/config/config.go | 269 +++++++++++++----- pkg/mcs/scheduling/server/config/watcher.go | 40 +++ pkg/mcs/scheduling/server/server.go | 4 +- pkg/schedule/config/config.go | 28 ++ server/config/persist_options.go | 97 +++---- .../mcs/scheduling/config_test.go | 5 +- tests/server/config/config_test.go | 32 +-- 7 files changed, 324 insertions(+), 151 deletions(-) diff --git a/pkg/mcs/scheduling/server/config/config.go b/pkg/mcs/scheduling/server/config/config.go index a211c989c64..6c38f66dc68 100644 --- a/pkg/mcs/scheduling/server/config/config.go +++ b/pkg/mcs/scheduling/server/config/config.go @@ -19,6 +19,7 @@ import ( "os" "path/filepath" "reflect" + "strconv" "strings" "sync/atomic" "time" @@ -30,6 +31,7 @@ import ( "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/log" "github.com/spf13/pflag" + "github.com/tikv/pd/pkg/cache" "github.com/tikv/pd/pkg/core/constant" "github.com/tikv/pd/pkg/core/storelimit" "github.com/tikv/pd/pkg/mcs/utils" @@ -220,6 +222,7 @@ func (c *Config) Clone() *Config { // PersistConfig wraps all configurations that need to persist to storage and // allows to access them safely. type PersistConfig struct { + ttl *cache.TTLString // Store the global configuration that is related to the scheduling. clusterVersion unsafe.Pointer schedule atomic.Value @@ -231,7 +234,7 @@ type PersistConfig struct { } // NewPersistConfig creates a new PersistConfig instance. -func NewPersistConfig(cfg *Config) *PersistConfig { +func NewPersistConfig(cfg *Config, ttl *cache.TTLString) *PersistConfig { o := &PersistConfig{} o.SetClusterVersion(&cfg.ClusterVersion) o.schedule.Store(&cfg.Schedule) @@ -239,6 +242,7 @@ func NewPersistConfig(cfg *Config) *PersistConfig { // storeConfig will be fetched from TiKV by PD API server, // so we just set an empty value here first. o.storeConfig.Store(&sc.StoreConfig{}) + o.ttl = ttl return o } @@ -329,16 +333,6 @@ func (o *PersistConfig) GetMaxReplicas() int { return int(o.GetReplicationConfig().MaxReplicas) } -// GetMaxSnapshotCount returns the max snapshot count. -func (o *PersistConfig) GetMaxSnapshotCount() uint64 { - return o.GetScheduleConfig().MaxSnapshotCount -} - -// GetMaxPendingPeerCount returns the max pending peer count. -func (o *PersistConfig) GetMaxPendingPeerCount() uint64 { - return o.GetScheduleConfig().MaxPendingPeerCount -} - // IsPlacementRulesEnabled returns if the placement rules is enabled. func (o *PersistConfig) IsPlacementRulesEnabled() bool { return o.GetReplicationConfig().EnablePlacementRules @@ -354,31 +348,6 @@ func (o *PersistConfig) GetHighSpaceRatio() float64 { return o.GetScheduleConfig().HighSpaceRatio } -// GetHotRegionScheduleLimit returns the limit for hot region schedule. -func (o *PersistConfig) GetHotRegionScheduleLimit() uint64 { - return o.GetScheduleConfig().HotRegionScheduleLimit -} - -// GetRegionScheduleLimit returns the limit for region schedule. -func (o *PersistConfig) GetRegionScheduleLimit() uint64 { - return o.GetScheduleConfig().RegionScheduleLimit -} - -// GetLeaderScheduleLimit returns the limit for leader schedule. -func (o *PersistConfig) GetLeaderScheduleLimit() uint64 { - return o.GetScheduleConfig().LeaderScheduleLimit -} - -// GetReplicaScheduleLimit returns the limit for replica schedule. -func (o *PersistConfig) GetReplicaScheduleLimit() uint64 { - return o.GetScheduleConfig().ReplicaScheduleLimit -} - -// GetMergeScheduleLimit returns the limit for merge schedule. -func (o *PersistConfig) GetMergeScheduleLimit() uint64 { - return o.GetScheduleConfig().MergeScheduleLimit -} - // GetLeaderSchedulePolicy is to get leader schedule policy. func (o *PersistConfig) GetLeaderSchedulePolicy() constant.SchedulePolicy { return constant.StringToSchedulePolicy(o.GetScheduleConfig().LeaderSchedulePolicy) @@ -419,26 +388,11 @@ func (o *PersistConfig) IsOneWayMergeEnabled() bool { return o.GetScheduleConfig().EnableOneWayMerge } -// GetMaxMergeRegionSize returns the max region size. -func (o *PersistConfig) GetMaxMergeRegionSize() uint64 { - return o.GetScheduleConfig().MaxMergeRegionSize -} - -// GetMaxMergeRegionKeys returns the max region keys. -func (o *PersistConfig) GetMaxMergeRegionKeys() uint64 { - return o.GetScheduleConfig().MaxMergeRegionKeys -} - // GetRegionScoreFormulaVersion returns the region score formula version. func (o *PersistConfig) GetRegionScoreFormulaVersion() string { return o.GetScheduleConfig().RegionScoreFormulaVersion } -// GetSchedulerMaxWaitingOperator returns the scheduler max waiting operator. -func (o *PersistConfig) GetSchedulerMaxWaitingOperator() uint64 { - return o.GetScheduleConfig().SchedulerMaxWaitingOperator -} - // GetHotRegionCacheHitsThreshold returns the hot region cache hits threshold. func (o *PersistConfig) GetHotRegionCacheHitsThreshold() int { return int(o.GetScheduleConfig().HotRegionCacheHitsThreshold) @@ -474,11 +428,6 @@ func (o *PersistConfig) GetTolerantSizeRatio() float64 { return o.GetScheduleConfig().TolerantSizeRatio } -// GetWitnessScheduleLimit returns the limit for region schedule. -func (o *PersistConfig) GetWitnessScheduleLimit() uint64 { - return o.GetScheduleConfig().WitnessScheduleLimit -} - // IsDebugMetricsEnabled returns if debug metrics is enabled. func (o *PersistConfig) IsDebugMetricsEnabled() bool { return o.GetScheduleConfig().EnableDebugMetrics @@ -509,11 +458,6 @@ func (o *PersistConfig) IsRemoveExtraReplicaEnabled() bool { return o.GetScheduleConfig().EnableRemoveExtraReplica } -// IsLocationReplacementEnabled returns if location replace is enabled. -func (o *PersistConfig) IsLocationReplacementEnabled() bool { - return o.GetScheduleConfig().EnableLocationReplacement -} - // IsWitnessAllowed returns if the witness is allowed. func (o *PersistConfig) IsWitnessAllowed() bool { return o.GetScheduleConfig().EnableWitness @@ -534,8 +478,87 @@ func (o *PersistConfig) GetStoresLimit() map[uint64]sc.StoreLimitConfig { return o.GetScheduleConfig().StoreLimit } +// TTL related methods. + +// GetLeaderScheduleLimit returns the limit for leader schedule. +func (o *PersistConfig) GetLeaderScheduleLimit() uint64 { + return o.getTTLUintOr(sc.LeaderScheduleLimitKey, o.GetScheduleConfig().LeaderScheduleLimit) +} + +// GetRegionScheduleLimit returns the limit for region schedule. +func (o *PersistConfig) GetRegionScheduleLimit() uint64 { + return o.getTTLUintOr(sc.RegionScheduleLimitKey, o.GetScheduleConfig().RegionScheduleLimit) +} + +// GetWitnessScheduleLimit returns the limit for region schedule. +func (o *PersistConfig) GetWitnessScheduleLimit() uint64 { + return o.getTTLUintOr(sc.WitnessScheduleLimitKey, o.GetScheduleConfig().WitnessScheduleLimit) +} + +// GetReplicaScheduleLimit returns the limit for replica schedule. +func (o *PersistConfig) GetReplicaScheduleLimit() uint64 { + return o.getTTLUintOr(sc.ReplicaRescheduleLimitKey, o.GetScheduleConfig().ReplicaScheduleLimit) +} + +// GetMergeScheduleLimit returns the limit for merge schedule. +func (o *PersistConfig) GetMergeScheduleLimit() uint64 { + return o.getTTLUintOr(sc.MergeScheduleLimitKey, o.GetScheduleConfig().MergeScheduleLimit) +} + +// GetHotRegionScheduleLimit returns the limit for hot region schedule. +func (o *PersistConfig) GetHotRegionScheduleLimit() uint64 { + return o.getTTLUintOr(sc.HotRegionScheduleLimitKey, o.GetScheduleConfig().HotRegionScheduleLimit) +} + +// GetStoreLimit returns the limit of a store. +func (o *PersistConfig) GetStoreLimit(storeID uint64) (returnSC sc.StoreLimitConfig) { + defer func() { + returnSC.RemovePeer = o.getTTLFloatOr(fmt.Sprintf("remove-peer-%v", storeID), returnSC.RemovePeer) + returnSC.AddPeer = o.getTTLFloatOr(fmt.Sprintf("add-peer-%v", storeID), returnSC.AddPeer) + }() + if limit, ok := o.GetScheduleConfig().StoreLimit[storeID]; ok { + return limit + } + cfg := o.GetScheduleConfig().Clone() + sc := sc.StoreLimitConfig{ + AddPeer: sc.DefaultStoreLimit.GetDefaultStoreLimit(storelimit.AddPeer), + RemovePeer: sc.DefaultStoreLimit.GetDefaultStoreLimit(storelimit.RemovePeer), + } + v, ok1, err := o.getTTLFloat("default-add-peer") + if err != nil { + log.Warn("failed to parse default-add-peer from PersistOptions's ttl storage", zap.Error(err)) + } + canSetAddPeer := ok1 && err == nil + if canSetAddPeer { + returnSC.AddPeer = v + } + + v, ok2, err := o.getTTLFloat("default-remove-peer") + if err != nil { + log.Warn("failed to parse default-remove-peer from PersistOptions's ttl storage", zap.Error(err)) + } + canSetRemovePeer := ok2 && err == nil + if canSetRemovePeer { + returnSC.RemovePeer = v + } + + if canSetAddPeer || canSetRemovePeer { + return returnSC + } + cfg.StoreLimit[storeID] = sc + o.SetScheduleConfig(cfg) + return o.GetScheduleConfig().StoreLimit[storeID] +} + // GetStoreLimitByType returns the limit of a store with a given type. func (o *PersistConfig) GetStoreLimitByType(storeID uint64, typ storelimit.Type) (returned float64) { + defer func() { + if typ == storelimit.RemovePeer { + returned = o.getTTLFloatOr(fmt.Sprintf("remove-peer-%v", storeID), returned) + } else if typ == storelimit.AddPeer { + returned = o.getTTLFloatOr(fmt.Sprintf("add-peer-%v", storeID), returned) + } + }() limit := o.GetStoreLimit(storeID) switch typ { case storelimit.AddPeer: @@ -550,20 +573,48 @@ func (o *PersistConfig) GetStoreLimitByType(storeID uint64, typ storelimit.Type) } } -// GetStoreLimit returns the limit of a store. -func (o *PersistConfig) GetStoreLimit(storeID uint64) (returnSC sc.StoreLimitConfig) { - if limit, ok := o.GetScheduleConfig().StoreLimit[storeID]; ok { - return limit +// GetMaxSnapshotCount returns the number of the max snapshot which is allowed to send. +func (o *PersistConfig) GetMaxSnapshotCount() uint64 { + return o.getTTLUintOr(sc.MaxSnapshotCountKey, o.GetScheduleConfig().MaxSnapshotCount) +} + +// GetMaxPendingPeerCount returns the number of the max pending peers. +func (o *PersistConfig) GetMaxPendingPeerCount() uint64 { + return o.getTTLUintOr(sc.MaxPendingPeerCountKey, o.GetScheduleConfig().MaxPendingPeerCount) +} + +// GetMaxMergeRegionSize returns the max region size. +func (o *PersistConfig) GetMaxMergeRegionSize() uint64 { + return o.getTTLUintOr(sc.MaxMergeRegionSizeKey, o.GetScheduleConfig().MaxMergeRegionSize) +} + +// GetMaxMergeRegionKeys returns the max number of keys. +// It returns size * 10000 if the key of max-merge-region-Keys doesn't exist. +func (o *PersistConfig) GetMaxMergeRegionKeys() uint64 { + keys, exist, err := o.getTTLUint(sc.MaxMergeRegionKeysKey) + if exist && err == nil { + return keys } - cfg := o.GetScheduleConfig().Clone() - sc := sc.StoreLimitConfig{ - AddPeer: sc.DefaultStoreLimit.GetDefaultStoreLimit(storelimit.AddPeer), - RemovePeer: sc.DefaultStoreLimit.GetDefaultStoreLimit(storelimit.RemovePeer), + size, exist, err := o.getTTLUint(sc.MaxMergeRegionSizeKey) + if exist && err == nil { + return size * 10000 } + return o.GetScheduleConfig().GetMaxMergeRegionKeys() +} - cfg.StoreLimit[storeID] = sc - o.SetScheduleConfig(cfg) - return o.GetScheduleConfig().StoreLimit[storeID] +// GetSchedulerMaxWaitingOperator returns the number of the max waiting operators. +func (o *PersistConfig) GetSchedulerMaxWaitingOperator() uint64 { + return o.getTTLUintOr(sc.SchedulerMaxWaitingOperatorKey, o.GetScheduleConfig().SchedulerMaxWaitingOperator) +} + +// IsLocationReplacementEnabled returns if location replace is enabled. +func (o *PersistConfig) IsLocationReplacementEnabled() bool { + return o.getTTLBoolOr(sc.EnableLocationReplacement, o.GetScheduleConfig().EnableLocationReplacement) +} + +// IsTikvRegionSplitEnabled returns whether tikv split region is disabled. +func (o *PersistConfig) IsTikvRegionSplitEnabled() bool { + return o.getTTLBoolOr(sc.EnableTiKVSplitRegion, o.GetScheduleConfig().EnableTiKVSplitRegion) } // SetAllStoresLimit sets all store limit for a given type and rate. @@ -680,11 +731,6 @@ func (o *PersistConfig) IsRaftKV2() bool { return o.GetStoreConfig().IsRaftKV2() } -// IsTikvRegionSplitEnabled returns whether tikv split region is disabled. -func (o *PersistConfig) IsTikvRegionSplitEnabled() bool { - return o.GetScheduleConfig().EnableTiKVSplitRegion -} - // TODO: implement the following methods // AddSchedulerCfg adds the scheduler configurations. @@ -710,3 +756,72 @@ func (o *PersistConfig) IsTraceRegionFlow() bool { func (o *PersistConfig) Persist(storage endpoint.ConfigStorage) error { return nil } + +func (o *PersistConfig) getTTLUint(key string) (uint64, bool, error) { + stringForm, ok := o.GetTTLData(key) + if !ok { + return 0, false, nil + } + r, err := strconv.ParseUint(stringForm, 10, 64) + return r, true, err +} + +func (o *PersistConfig) getTTLUintOr(key string, defaultValue uint64) uint64 { + if v, ok, err := o.getTTLUint(key); ok { + if err == nil { + return v + } + log.Warn("failed to parse "+key+" from PersistOptions's ttl storage", zap.Error(err)) + } + return defaultValue +} + +func (o *PersistConfig) getTTLBool(key string) (result bool, contains bool, err error) { + stringForm, ok := o.GetTTLData(key) + if !ok { + return + } + result, err = strconv.ParseBool(stringForm) + contains = true + return +} + +func (o *PersistConfig) getTTLBoolOr(key string, defaultValue bool) bool { + if v, ok, err := o.getTTLBool(key); ok { + if err == nil { + return v + } + log.Warn("failed to parse "+key+" from PersistOptions's ttl storage", zap.Error(err)) + } + return defaultValue +} + +func (o *PersistConfig) getTTLFloat(key string) (float64, bool, error) { + stringForm, ok := o.GetTTLData(key) + if !ok { + return 0, false, nil + } + r, err := strconv.ParseFloat(stringForm, 64) + return r, true, err +} + +func (o *PersistConfig) getTTLFloatOr(key string, defaultValue float64) float64 { + if v, ok, err := o.getTTLFloat(key); ok { + if err == nil { + return v + } + log.Warn("failed to parse "+key+" from PersistOptions's ttl storage", zap.Error(err)) + } + return defaultValue +} + +// GetTTLData returns if there is a TTL data for a given key. +func (o *PersistConfig) GetTTLData(key string) (string, bool) { + if o.ttl == nil { + return "", false + } + if result, ok := o.ttl.Get(key); ok { + return result.(string), ok + } + return "", false +} diff --git a/pkg/mcs/scheduling/server/config/watcher.go b/pkg/mcs/scheduling/server/config/watcher.go index 433933674ea..4ded93ceb1b 100644 --- a/pkg/mcs/scheduling/server/config/watcher.go +++ b/pkg/mcs/scheduling/server/config/watcher.go @@ -20,6 +20,7 @@ import ( "strings" "sync" "sync/atomic" + "time" "github.com/coreos/go-semver/semver" "github.com/pingcap/log" @@ -48,8 +49,11 @@ type Watcher struct { // - Value: configuration JSON. schedulerConfigPathPrefix string + ttlConfigPrefix string + etcdClient *clientv3.Client configWatcher *etcdutil.LoopWatcher + ttlConfigWatcher *etcdutil.LoopWatcher schedulerConfigWatcher *etcdutil.LoopWatcher // Some data, like the global schedule config, should be loaded into `PersistConfig`. @@ -82,6 +86,7 @@ func NewWatcher( ctx: ctx, cancel: cancel, configPath: endpoint.ConfigPath(clusterID), + ttlConfigPrefix: sc.TTLConfigPrefix, schedulerConfigPathPrefix: endpoint.SchedulerConfigPathPrefix(clusterID), etcdClient: etcdClient, PersistConfig: persistConfig, @@ -91,6 +96,10 @@ func NewWatcher( if err != nil { return nil, err } + err = cw.initializeTTLConfigWatcher() + if err != nil { + return nil, err + } err = cw.initializeSchedulerConfigWatcher() if err != nil { return nil, err @@ -143,6 +152,37 @@ func (cw *Watcher) initializeConfigWatcher() error { return cw.configWatcher.WaitLoad() } +func (cw *Watcher) initializeTTLConfigWatcher() error { + putFn := func(kv *mvccpb.KeyValue) error { + key := string(kv.Key)[len(sc.TTLConfigPrefix)+1:] + value := string(kv.Value) + leaseID := kv.Lease + resp, err := cw.etcdClient.TimeToLive(cw.ctx, clientv3.LeaseID(leaseID)) + if err != nil { + return err + } + log.Info("update scheduling ttl config", zap.String("key", key), zap.String("value", value)) + cw.ttl.PutWithTTL(key, value, time.Duration(resp.TTL)*time.Second) + return nil + } + deleteFn := func(kv *mvccpb.KeyValue) error { + key := string(kv.Key)[len(sc.TTLConfigPrefix)+1:] + cw.ttl.PutWithTTL(key, nil, 0) + return nil + } + postEventFn := func() error { + return nil + } + cw.ttlConfigWatcher = etcdutil.NewLoopWatcher( + cw.ctx, &cw.wg, + cw.etcdClient, + "scheduling-ttl-config-watcher", cw.ttlConfigPrefix, + putFn, deleteFn, postEventFn, clientv3.WithPrefix(), + ) + cw.ttlConfigWatcher.StartWatchLoop() + return cw.ttlConfigWatcher.WaitLoad() +} + func (cw *Watcher) initializeSchedulerConfigWatcher() error { prefixToTrim := cw.schedulerConfigPathPrefix + "/" putFn := func(kv *mvccpb.KeyValue) error { diff --git a/pkg/mcs/scheduling/server/server.go b/pkg/mcs/scheduling/server/server.go index c5b73dea5fc..a43cbbebd86 100644 --- a/pkg/mcs/scheduling/server/server.go +++ b/pkg/mcs/scheduling/server/server.go @@ -36,6 +36,7 @@ import ( "github.com/pingcap/sysutil" "github.com/spf13/cobra" bs "github.com/tikv/pd/pkg/basicserver" + "github.com/tikv/pd/pkg/cache" "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/errs" "github.com/tikv/pd/pkg/mcs/discovery" @@ -46,6 +47,7 @@ import ( "github.com/tikv/pd/pkg/mcs/utils" "github.com/tikv/pd/pkg/member" "github.com/tikv/pd/pkg/schedule" + sc "github.com/tikv/pd/pkg/schedule/config" "github.com/tikv/pd/pkg/schedule/hbstream" "github.com/tikv/pd/pkg/schedule/schedulers" "github.com/tikv/pd/pkg/storage/endpoint" @@ -534,7 +536,7 @@ func CreateServer(ctx context.Context, cfg *config.Config) *Server { BaseServer: server.NewBaseServer(ctx), DiagnosticsServer: sysutil.NewDiagnosticsServer(cfg.Log.File.Filename), cfg: cfg, - persistConfig: config.NewPersistConfig(cfg), + persistConfig: config.NewPersistConfig(cfg, cache.NewStringTTL(ctx, sc.DefaultGCInterval, sc.DefaultTTL)), checkMembershipCh: make(chan struct{}, 1), } return svr diff --git a/pkg/schedule/config/config.go b/pkg/schedule/config/config.go index c8fa62b8aff..27b8917d1bf 100644 --- a/pkg/schedule/config/config.go +++ b/pkg/schedule/config/config.go @@ -79,6 +79,34 @@ var ( DefaultTiFlashStoreLimit = StoreLimit{AddPeer: 30, RemovePeer: 30} ) +// The following consts are used to identify the config item that needs to set TTL. +const ( + // TTLConfigPrefix is the prefix of the config item that needs to set TTL. + TTLConfigPrefix = "/config/ttl" + + MaxSnapshotCountKey = "schedule.max-snapshot-count" + MaxMergeRegionSizeKey = "schedule.max-merge-region-size" + MaxPendingPeerCountKey = "schedule.max-pending-peer-count" + MaxMergeRegionKeysKey = "schedule.max-merge-region-keys" + LeaderScheduleLimitKey = "schedule.leader-schedule-limit" + RegionScheduleLimitKey = "schedule.region-schedule-limit" + WitnessScheduleLimitKey = "schedule.witness-schedule-limit" + ReplicaRescheduleLimitKey = "schedule.replica-schedule-limit" + MergeScheduleLimitKey = "schedule.merge-schedule-limit" + HotRegionScheduleLimitKey = "schedule.hot-region-schedule-limit" + SchedulerMaxWaitingOperatorKey = "schedule.scheduler-max-waiting-operator" + EnableLocationReplacement = "schedule.enable-location-replacement" + DefaultAddPeer = "default-add-peer" + DefaultRemovePeer = "default-remove-peer" + + // EnableTiKVSplitRegion is the option to enable tikv split region. + // it's related to schedule, but it's not an explicit config + EnableTiKVSplitRegion = "schedule.enable-tikv-split-region" + + DefaultGCInterval = 5 * time.Second + DefaultTTL = 5 * time.Minute +) + // StoreLimit is the default limit of adding peer and removing peer when putting stores. type StoreLimit struct { mu syncutil.RWMutex diff --git a/server/config/persist_options.go b/server/config/persist_options.go index 49a44449a22..42ab91fce17 100644 --- a/server/config/persist_options.go +++ b/server/config/persist_options.go @@ -216,38 +216,21 @@ func (o *PersistOptions) SetMaxReplicas(replicas int) { o.SetReplicationConfig(v) } -const ( - maxSnapshotCountKey = "schedule.max-snapshot-count" - maxMergeRegionSizeKey = "schedule.max-merge-region-size" - maxPendingPeerCountKey = "schedule.max-pending-peer-count" - maxMergeRegionKeysKey = "schedule.max-merge-region-keys" - leaderScheduleLimitKey = "schedule.leader-schedule-limit" - regionScheduleLimitKey = "schedule.region-schedule-limit" - witnessScheduleLimitKey = "schedule.witness-schedule-limit" - replicaRescheduleLimitKey = "schedule.replica-schedule-limit" - mergeScheduleLimitKey = "schedule.merge-schedule-limit" - hotRegionScheduleLimitKey = "schedule.hot-region-schedule-limit" - schedulerMaxWaitingOperatorKey = "schedule.scheduler-max-waiting-operator" - enableLocationReplacement = "schedule.enable-location-replacement" - // it's related to schedule, but it's not an explicit config - enableTiKVSplitRegion = "schedule.enable-tikv-split-region" -) - var supportedTTLConfigs = []string{ - maxSnapshotCountKey, - maxMergeRegionSizeKey, - maxPendingPeerCountKey, - maxMergeRegionKeysKey, - leaderScheduleLimitKey, - regionScheduleLimitKey, - replicaRescheduleLimitKey, - mergeScheduleLimitKey, - hotRegionScheduleLimitKey, - schedulerMaxWaitingOperatorKey, - enableLocationReplacement, - enableTiKVSplitRegion, - "default-add-peer", - "default-remove-peer", + sc.MaxSnapshotCountKey, + sc.MaxMergeRegionSizeKey, + sc.MaxPendingPeerCountKey, + sc.MaxMergeRegionKeysKey, + sc.LeaderScheduleLimitKey, + sc.RegionScheduleLimitKey, + sc.ReplicaRescheduleLimitKey, + sc.MergeScheduleLimitKey, + sc.HotRegionScheduleLimitKey, + sc.SchedulerMaxWaitingOperatorKey, + sc.EnableLocationReplacement, + sc.EnableTiKVSplitRegion, + sc.DefaultAddPeer, + sc.DefaultRemovePeer, } // IsSupportedTTLConfig checks whether a key is a supported config item with ttl @@ -262,27 +245,27 @@ func IsSupportedTTLConfig(key string) bool { // GetMaxSnapshotCount returns the number of the max snapshot which is allowed to send. func (o *PersistOptions) GetMaxSnapshotCount() uint64 { - return o.getTTLUintOr(maxSnapshotCountKey, o.GetScheduleConfig().MaxSnapshotCount) + return o.getTTLUintOr(sc.MaxSnapshotCountKey, o.GetScheduleConfig().MaxSnapshotCount) } // GetMaxPendingPeerCount returns the number of the max pending peers. func (o *PersistOptions) GetMaxPendingPeerCount() uint64 { - return o.getTTLUintOr(maxPendingPeerCountKey, o.GetScheduleConfig().MaxPendingPeerCount) + return o.getTTLUintOr(sc.MaxPendingPeerCountKey, o.GetScheduleConfig().MaxPendingPeerCount) } // GetMaxMergeRegionSize returns the max region size. func (o *PersistOptions) GetMaxMergeRegionSize() uint64 { - return o.getTTLUintOr(maxMergeRegionSizeKey, o.GetScheduleConfig().MaxMergeRegionSize) + return o.getTTLUintOr(sc.MaxMergeRegionSizeKey, o.GetScheduleConfig().MaxMergeRegionSize) } // GetMaxMergeRegionKeys returns the max number of keys. // It returns size * 10000 if the key of max-merge-region-Keys doesn't exist. func (o *PersistOptions) GetMaxMergeRegionKeys() uint64 { - keys, exist, err := o.getTTLUint(maxMergeRegionKeysKey) + keys, exist, err := o.getTTLUint(sc.MaxMergeRegionKeysKey) if exist && err == nil { return keys } - size, exist, err := o.getTTLUint(maxMergeRegionSizeKey) + size, exist, err := o.getTTLUint(sc.MaxMergeRegionSizeKey) if exist && err == nil { return size * 10000 } @@ -424,32 +407,32 @@ func (o *PersistOptions) GetMaxStorePreparingTime() time.Duration { // GetLeaderScheduleLimit returns the limit for leader schedule. func (o *PersistOptions) GetLeaderScheduleLimit() uint64 { - return o.getTTLUintOr(leaderScheduleLimitKey, o.GetScheduleConfig().LeaderScheduleLimit) + return o.getTTLUintOr(sc.LeaderScheduleLimitKey, o.GetScheduleConfig().LeaderScheduleLimit) } // GetRegionScheduleLimit returns the limit for region schedule. func (o *PersistOptions) GetRegionScheduleLimit() uint64 { - return o.getTTLUintOr(regionScheduleLimitKey, o.GetScheduleConfig().RegionScheduleLimit) + return o.getTTLUintOr(sc.RegionScheduleLimitKey, o.GetScheduleConfig().RegionScheduleLimit) } // GetWitnessScheduleLimit returns the limit for region schedule. func (o *PersistOptions) GetWitnessScheduleLimit() uint64 { - return o.getTTLUintOr(witnessScheduleLimitKey, o.GetScheduleConfig().WitnessScheduleLimit) + return o.getTTLUintOr(sc.WitnessScheduleLimitKey, o.GetScheduleConfig().WitnessScheduleLimit) } // GetReplicaScheduleLimit returns the limit for replica schedule. func (o *PersistOptions) GetReplicaScheduleLimit() uint64 { - return o.getTTLUintOr(replicaRescheduleLimitKey, o.GetScheduleConfig().ReplicaScheduleLimit) + return o.getTTLUintOr(sc.ReplicaRescheduleLimitKey, o.GetScheduleConfig().ReplicaScheduleLimit) } // GetMergeScheduleLimit returns the limit for merge schedule. func (o *PersistOptions) GetMergeScheduleLimit() uint64 { - return o.getTTLUintOr(mergeScheduleLimitKey, o.GetScheduleConfig().MergeScheduleLimit) + return o.getTTLUintOr(sc.MergeScheduleLimitKey, o.GetScheduleConfig().MergeScheduleLimit) } // GetHotRegionScheduleLimit returns the limit for hot region schedule. func (o *PersistOptions) GetHotRegionScheduleLimit() uint64 { - return o.getTTLUintOr(hotRegionScheduleLimitKey, o.GetScheduleConfig().HotRegionScheduleLimit) + return o.getTTLUintOr(sc.HotRegionScheduleLimitKey, o.GetScheduleConfig().HotRegionScheduleLimit) } // GetStoreLimit returns the limit of a store. @@ -552,7 +535,7 @@ func (o *PersistOptions) GetRegionScoreFormulaVersion() string { // GetSchedulerMaxWaitingOperator returns the number of the max waiting operators. func (o *PersistOptions) GetSchedulerMaxWaitingOperator() uint64 { - return o.getTTLUintOr(schedulerMaxWaitingOperatorKey, o.GetScheduleConfig().SchedulerMaxWaitingOperator) + return o.getTTLUintOr(sc.SchedulerMaxWaitingOperatorKey, o.GetScheduleConfig().SchedulerMaxWaitingOperator) } // GetLeaderSchedulePolicy is to get leader schedule policy. @@ -622,12 +605,12 @@ func (o *PersistOptions) IsRemoveExtraReplicaEnabled() bool { // IsLocationReplacementEnabled returns if location replace is enabled. func (o *PersistOptions) IsLocationReplacementEnabled() bool { - return o.getTTLBoolOr(enableLocationReplacement, o.GetScheduleConfig().EnableLocationReplacement) + return o.getTTLBoolOr(sc.EnableLocationReplacement, o.GetScheduleConfig().EnableLocationReplacement) } // IsTikvRegionSplitEnabled returns whether tikv split region is disabled. func (o *PersistOptions) IsTikvRegionSplitEnabled() bool { - return o.getTTLBoolOr(enableTiKVSplitRegion, o.GetScheduleConfig().EnableTiKVSplitRegion) + return o.getTTLBoolOr(sc.EnableTiKVSplitRegion, o.GetScheduleConfig().EnableTiKVSplitRegion) } // GetMaxMovableHotPeerSize returns the max movable hot peer size. @@ -852,16 +835,22 @@ func (o *PersistOptions) GetMinResolvedTSPersistenceInterval() time.Duration { return o.GetPDServerConfig().MinResolvedTSPersistenceInterval.Duration } -const ttlConfigPrefix = "/config/ttl" - // SetTTLData set temporary configuration func (o *PersistOptions) SetTTLData(parCtx context.Context, client *clientv3.Client, key string, value string, ttl time.Duration) error { if o.ttl == nil { - o.ttl = cache.NewStringTTL(parCtx, time.Second*5, time.Minute*5) + o.ttl = cache.NewStringTTL(parCtx, sc.DefaultGCInterval, sc.DefaultTTL) } - _, err := etcdutil.EtcdKVPutWithTTL(parCtx, client, ttlConfigPrefix+"/"+key, value, int64(ttl.Seconds())) - if err != nil { - return err + if ttl != 0 { + // the minimum ttl is 5 seconds, if the given ttl is less than 5 seconds, we will use 5 seconds instead. + _, err := etcdutil.EtcdKVPutWithTTL(parCtx, client, sc.TTLConfigPrefix+"/"+key, value, int64(ttl.Seconds())) + if err != nil { + return err + } + } else { + _, err := client.Delete(parCtx, sc.TTLConfigPrefix+"/"+key) + if err != nil { + return err + } } o.ttl.PutWithTTL(key, value, ttl) return nil @@ -938,15 +927,15 @@ func (o *PersistOptions) GetTTLData(key string) (string, bool) { // LoadTTLFromEtcd loads temporary configuration which was persisted into etcd func (o *PersistOptions) LoadTTLFromEtcd(ctx context.Context, client *clientv3.Client) error { - resps, err := etcdutil.EtcdKVGet(client, ttlConfigPrefix, clientv3.WithPrefix()) + resps, err := etcdutil.EtcdKVGet(client, sc.TTLConfigPrefix, clientv3.WithPrefix()) if err != nil { return err } if o.ttl == nil { - o.ttl = cache.NewStringTTL(ctx, time.Second*5, time.Minute*5) + o.ttl = cache.NewStringTTL(ctx, sc.DefaultGCInterval, sc.DefaultTTL) } for _, resp := range resps.Kvs { - key := string(resp.Key)[len(ttlConfigPrefix)+1:] + key := string(resp.Key)[len(sc.TTLConfigPrefix)+1:] value := string(resp.Value) leaseID := resp.Lease resp, err := client.TimeToLive(ctx, clientv3.LeaseID(leaseID)) diff --git a/tests/integrations/mcs/scheduling/config_test.go b/tests/integrations/mcs/scheduling/config_test.go index ccf7cdaf48c..69d77bb24ac 100644 --- a/tests/integrations/mcs/scheduling/config_test.go +++ b/tests/integrations/mcs/scheduling/config_test.go @@ -23,6 +23,7 @@ import ( "github.com/pingcap/kvproto/pkg/metapb" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + "github.com/tikv/pd/pkg/cache" "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/mcs/scheduling/server/config" sc "github.com/tikv/pd/pkg/schedule/config" @@ -84,7 +85,7 @@ func (suite *configTestSuite) TestConfigWatch() { suite.ctx, suite.pdLeaderServer.GetEtcdClient(), suite.cluster.GetCluster().GetId(), - config.NewPersistConfig(config.NewConfig()), + config.NewPersistConfig(config.NewConfig(), cache.NewStringTTL(suite.ctx, sc.DefaultGCInterval, sc.DefaultTTL)), endpoint.NewStorageEndpoint(kv.NewMemoryKV(), nil), ) re.NoError(err) @@ -144,7 +145,7 @@ func (suite *configTestSuite) TestSchedulerConfigWatch() { suite.ctx, suite.pdLeaderServer.GetEtcdClient(), suite.cluster.GetCluster().GetId(), - config.NewPersistConfig(config.NewConfig()), + config.NewPersistConfig(config.NewConfig(), cache.NewStringTTL(suite.ctx, sc.DefaultGCInterval, sc.DefaultTTL)), storage, ) re.NoError(err) diff --git a/tests/server/config/config_test.go b/tests/server/config/config_test.go index 4a4a91f2661..c0e9a0e1148 100644 --- a/tests/server/config/config_test.go +++ b/tests/server/config/config_test.go @@ -448,7 +448,7 @@ func (suite *configTestSuite) assertTTLConfig( if !expectedEqual { equality = suite.NotEqual } - checkfunc := func(options ttlConfigInterface) { + checkFunc := func(options ttlConfigInterface) { equality(uint64(999), options.GetMaxSnapshotCount()) equality(false, options.IsLocationReplacementEnabled()) equality(uint64(999), options.GetMaxMergeRegionSize()) @@ -461,7 +461,7 @@ func (suite *configTestSuite) assertTTLConfig( equality(uint64(999), options.GetMergeScheduleLimit()) equality(false, options.IsTikvRegionSplitEnabled()) } - checkfunc(cluster.GetLeaderServer().GetServer().GetPersistOptions()) + checkFunc(cluster.GetLeaderServer().GetServer().GetPersistOptions()) if cluster.GetSchedulingPrimaryServer() != nil { // wait for the scheduling primary server to be synced options := cluster.GetSchedulingPrimaryServer().GetPersistConfig() @@ -471,16 +471,16 @@ func (suite *configTestSuite) assertTTLConfig( } return uint64(999) != options.GetMaxSnapshotCount() }) - checkfunc(options) + checkFunc(options) } } -func (suite *configTestSuite) assertTTLConfigItemEqaul( +func (suite *configTestSuite) assertTTLConfigItemEqual( cluster *tests.TestCluster, item string, expectedValue interface{}, ) { - checkfunc := func(options ttlConfigInterface) bool { + checkFunc := func(options ttlConfigInterface) bool { switch item { case "max-merge-region-size": return expectedValue.(uint64) == options.GetMaxMergeRegionSize() @@ -491,11 +491,11 @@ func (suite *configTestSuite) assertTTLConfigItemEqaul( } return false } - suite.True(checkfunc(cluster.GetLeaderServer().GetServer().GetPersistOptions())) + suite.True(checkFunc(cluster.GetLeaderServer().GetServer().GetPersistOptions())) if cluster.GetSchedulingPrimaryServer() != nil { // wait for the scheduling primary server to be synced tu.Eventually(suite.Require(), func() bool { - return checkfunc(cluster.GetSchedulingPrimaryServer().GetPersistConfig()) + return checkFunc(cluster.GetSchedulingPrimaryServer().GetPersistConfig()) }) } } @@ -506,8 +506,7 @@ func createTTLUrl(url string, ttl int) string { func (suite *configTestSuite) TestConfigTTL() { env := tests.NewSchedulingTestEnvironment(suite.T()) - // FIXME: enable this test in two modes after ttl config is supported. - env.RunTestInPDMode(suite.checkConfigTTL) + env.RunTestInTwoModes(suite.checkConfigTTL) } func (suite *configTestSuite) checkConfigTTL(cluster *tests.TestCluster) { @@ -523,14 +522,14 @@ func (suite *configTestSuite) checkConfigTTL(cluster *tests.TestCluster) { suite.assertTTLConfig(cluster, false) // test time goes by - err = tu.CheckPostJSON(testDialClient, createTTLUrl(urlPrefix, 1), postData, tu.StatusOK(re)) + err = tu.CheckPostJSON(testDialClient, createTTLUrl(urlPrefix, 5), postData, tu.StatusOK(re)) suite.NoError(err) suite.assertTTLConfig(cluster, true) - time.Sleep(2 * time.Second) + time.Sleep(5 * time.Second) suite.assertTTLConfig(cluster, false) // test cleaning up - err = tu.CheckPostJSON(testDialClient, createTTLUrl(urlPrefix, 1), postData, tu.StatusOK(re)) + err = tu.CheckPostJSON(testDialClient, createTTLUrl(urlPrefix, 5), postData, tu.StatusOK(re)) suite.NoError(err) suite.assertTTLConfig(cluster, true) err = tu.CheckPostJSON(testDialClient, createTTLUrl(urlPrefix, 0), postData, tu.StatusOK(re)) @@ -552,9 +551,9 @@ func (suite *configTestSuite) checkConfigTTL(cluster *tests.TestCluster) { err = tu.CheckPostJSON(testDialClient, createTTLUrl(urlPrefix, 1), postData, tu.StatusOK(re)) suite.NoError(err) - suite.assertTTLConfigItemEqaul(cluster, "max-merge-region-size", uint64(999)) + suite.assertTTLConfigItemEqual(cluster, "max-merge-region-size", uint64(999)) // max-merge-region-keys should keep consistence with max-merge-region-size. - suite.assertTTLConfigItemEqaul(cluster, "max-merge-region-keys", uint64(999*10000)) + suite.assertTTLConfigItemEqual(cluster, "max-merge-region-keys", uint64(999*10000)) // on invalid value, we use default config mergeConfig = map[string]interface{}{ @@ -564,13 +563,12 @@ func (suite *configTestSuite) checkConfigTTL(cluster *tests.TestCluster) { suite.NoError(err) err = tu.CheckPostJSON(testDialClient, createTTLUrl(urlPrefix, 10), postData, tu.StatusOK(re)) suite.NoError(err) - suite.assertTTLConfigItemEqaul(cluster, "enable-tikv-split-region", true) + suite.assertTTLConfigItemEqual(cluster, "enable-tikv-split-region", true) } func (suite *configTestSuite) TestTTLConflict() { env := tests.NewSchedulingTestEnvironment(suite.T()) - // FIXME: enable this test in two modes after ttl config is supported. - env.RunTestInPDMode(suite.checkTTLConflict) + env.RunTestInTwoModes(suite.checkTTLConflict) } func (suite *configTestSuite) checkTTLConflict(cluster *tests.TestCluster) { From 4bcac208ac81506d9e6a953417b448ab183b0270 Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Mon, 27 Nov 2023 16:59:46 +0800 Subject: [PATCH 052/137] mcs: fix the last heartbeat is not updated (#7450) close tikv/pd#7445 Signed-off-by: Ryan Leung Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- pkg/core/store_option.go | 2 +- pkg/mcs/scheduling/server/cluster.go | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/core/store_option.go b/pkg/core/store_option.go index 0bdfaffa44f..93b25562731 100644 --- a/pkg/core/store_option.go +++ b/pkg/core/store_option.go @@ -276,6 +276,7 @@ func SetLastAwakenTime(lastAwaken time.Time) StoreCreateOption { } // SetStoreMeta sets the meta for the store. +// NOTICE: LastHeartbeat is not persisted each time, so it is not set by this function. Please use SetLastHeartbeatTS instead. func SetStoreMeta(newMeta *metapb.Store) StoreCreateOption { return func(store *StoreInfo) { meta := typeutil.DeepClone(store.meta, StoreFactory) @@ -286,7 +287,6 @@ func SetStoreMeta(newMeta *metapb.Store) StoreCreateOption { meta.PeerAddress = newMeta.GetPeerAddress() meta.StartTimestamp = newMeta.GetStartTimestamp() meta.DeployPath = newMeta.GetDeployPath() - meta.LastHeartbeat = newMeta.GetLastHeartbeat() meta.State = newMeta.GetState() meta.Labels = newMeta.GetLabels() meta.NodeState = newMeta.GetNodeState() diff --git a/pkg/mcs/scheduling/server/cluster.go b/pkg/mcs/scheduling/server/cluster.go index 59b2e691658..b24db7ac805 100644 --- a/pkg/mcs/scheduling/server/cluster.go +++ b/pkg/mcs/scheduling/server/cluster.go @@ -381,7 +381,8 @@ func (c *Cluster) HandleStoreHeartbeat(heartbeat *schedulingpb.StoreHeartbeatReq return errors.Errorf("store %v not found", storeID) } - newStore := store.Clone(core.SetStoreStats(stats)) + nowTime := time.Now() + newStore := store.Clone(core.SetStoreStats(stats), core.SetLastHeartbeatTS(nowTime)) if store := c.GetStore(storeID); store != nil { statistics.UpdateStoreHeartbeatMetrics(store) From 82ace6ff98e9ae4efed8e68089642a4a35335f6f Mon Sep 17 00:00:00 2001 From: niubell Date: Tue, 28 Nov 2023 00:08:43 +0800 Subject: [PATCH 053/137] go.mod: upgrade gin version from v1.8.1 to v1.9.1 (#7451) close tikv/pd#7438 Upgrade Gin version to v1.9.1 to solve some security issues Signed-off-by: niubell --- go.mod | 26 ++++++---- go.sum | 52 ++++++++++++++----- tests/integrations/client/go.mod | 26 ++++++---- tests/integrations/client/go.sum | 52 ++++++++++++++----- tests/integrations/mcs/go.mod | 26 ++++++---- tests/integrations/mcs/go.sum | 52 ++++++++++++++----- tests/integrations/tso/go.mod | 24 +++++---- tests/integrations/tso/go.sum | 50 +++++++++++++++---- tools/pd-api-bench/go.mod | 26 ++++++---- tools/pd-api-bench/go.sum | 85 +++++++++++++++++--------------- 10 files changed, 287 insertions(+), 132 deletions(-) diff --git a/go.mod b/go.mod index 0306d70f7a3..e4171caea8c 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/gin-contrib/cors v1.4.0 github.com/gin-contrib/gzip v0.0.1 github.com/gin-contrib/pprof v1.4.0 - github.com/gin-gonic/gin v1.8.1 + github.com/gin-gonic/gin v1.9.1 github.com/go-echarts/go-echarts v1.0.0 github.com/gogo/protobuf v1.3.2 github.com/google/btree v1.1.2 @@ -45,7 +45,7 @@ require ( github.com/soheilhy/cmux v0.1.4 github.com/spf13/cobra v1.0.0 github.com/spf13/pflag v1.0.5 - github.com/stretchr/testify v1.8.2 + github.com/stretchr/testify v1.8.3 github.com/swaggo/http-swagger v1.2.6 github.com/swaggo/swag v1.8.3 github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965 @@ -84,8 +84,10 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/bitly/go-simplejson v0.5.0 // indirect github.com/breeswish/gin-jwt/v2 v2.6.4-jwt-patch // indirect + github.com/bytedance/sonic v1.9.1 // indirect github.com/cenkalti/backoff/v4 v4.0.2 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f // indirect github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect @@ -96,6 +98,7 @@ require ( github.com/fatih/structtag v1.2.0 // indirect github.com/fogleman/gg v1.3.0 // indirect github.com/fsnotify/fsnotify v1.4.9 // indirect + github.com/gabriel-vasile/mimetype v1.4.2 // indirect github.com/ghodss/yaml v1.0.0 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/go-ole/go-ole v1.2.6 // indirect @@ -103,13 +106,13 @@ require ( github.com/go-openapi/jsonreference v0.19.6 // indirect github.com/go-openapi/spec v0.20.4 // indirect github.com/go-openapi/swag v0.19.15 // indirect - github.com/go-playground/locales v0.14.0 // indirect - github.com/go-playground/universal-translator v0.18.0 // indirect - github.com/go-playground/validator/v10 v10.10.0 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/go-playground/validator/v10 v10.14.0 // indirect github.com/go-resty/resty/v2 v2.6.0 // indirect github.com/go-sql-driver/mysql v1.7.0 // indirect github.com/goccy/go-graphviz v0.0.9 // indirect - github.com/goccy/go-json v0.9.7 // indirect + github.com/goccy/go-json v0.10.2 // indirect github.com/golang-jwt/jwt v3.2.1+incompatible // indirect github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect github.com/golang/protobuf v1.5.3 // indirect @@ -128,11 +131,12 @@ require ( github.com/joomcode/errorx v1.0.1 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/cpuid/v2 v2.2.4 // indirect github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect - github.com/leodido/go-urn v1.2.1 // indirect + github.com/leodido/go-urn v1.2.4 // indirect github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a // indirect github.com/mattn/go-colorable v0.1.8 // indirect - github.com/mattn/go-isatty v0.0.14 // indirect + github.com/mattn/go-isatty v0.0.19 // indirect github.com/mattn/go-runewidth v0.0.8 // indirect github.com/mattn/go-sqlite3 v1.14.15 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect @@ -144,7 +148,7 @@ require ( github.com/oleiade/reflections v1.0.1 // indirect github.com/olekukonko/tablewriter v0.0.4 // indirect github.com/onsi/gomega v1.20.1 // indirect - github.com/pelletier/go-toml/v2 v2.0.1 // indirect + github.com/pelletier/go-toml/v2 v2.0.8 // indirect github.com/petermattis/goid v0.0.0-20211229010228-4d14c490ee36 // indirect github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e // indirect github.com/pkg/errors v0.9.1 // indirect @@ -166,7 +170,8 @@ require ( github.com/tklauser/go-sysconf v0.3.11 // indirect github.com/tklauser/numcpus v0.6.0 // indirect github.com/tmc/grpc-websocket-proxy v0.0.0-20200427203606-3cfed13b9966 // indirect - github.com/ugorji/go/codec v1.2.7 // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + github.com/ugorji/go/codec v1.2.11 // indirect github.com/urfave/cli/v2 v2.3.0 // indirect github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect @@ -177,6 +182,7 @@ require ( go.uber.org/dig v1.9.0 // indirect go.uber.org/fx v1.12.0 // indirect go.uber.org/multierr v1.11.0 // indirect + golang.org/x/arch v0.3.0 // indirect golang.org/x/crypto v0.14.0 // indirect golang.org/x/image v0.5.0 // indirect golang.org/x/mod v0.8.0 // indirect diff --git a/go.sum b/go.sum index fb178321864..76f98d677ee 100644 --- a/go.sum +++ b/go.sum @@ -72,6 +72,9 @@ github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4Yn github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/breeswish/gin-jwt/v2 v2.6.4-jwt-patch h1:KLE/YeX+9FNaGVW5MtImRVPhjDpfpgJhvkuYWBmOYbo= github.com/breeswish/gin-jwt/v2 v2.6.4-jwt-patch/go.mod h1:KjBLriHXe7L6fGceqWzTod8HUB/TP1WWDtfuSYtYXaI= +github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= +github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s= +github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 h1:BjkPE3785EwPhhyuFkbINB+2a1xATwk8SNDWnJiD41g= github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5/go.mod h1:jtAfVaU/2cu1+wdSRPWE2c1N2qeAA3K4RH9pYgqwets= github.com/cenkalti/backoff/v4 v4.0.2 h1:JIufpQLbh4DkbQoii76ItQIUFzevQSqOLZca4eamEDs= @@ -81,6 +84,9 @@ github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghf github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= +github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams= +github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk= github.com/chzyer/logex v1.1.10 h1:Swpa1K6QvQznwJRcfTfQJmTE72DqScAa40E+fbHEXEE= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8= @@ -140,6 +146,8 @@ github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzP github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= +github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/cors v1.4.0 h1:oJ6gwtUl3lqV0WEIwM/LxPF1QZ5qe2lGWdY2+bz7y0g= @@ -153,8 +161,9 @@ github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.3.0/go.mod h1:7cKuhb5qV2ggCFctp2fJQ+ErvciLZrIeoOSOm6mUr7Y= github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M= -github.com/gin-gonic/gin v1.8.1 h1:4+fr/el88TOO3ewCmQr8cx/CtZ/umlIRIs5M4NTNjf8= github.com/gin-gonic/gin v1.8.1/go.mod h1:ji8BvRH1azfM+SYow9zQ6SZMvR8qOMZHmsCuWR9tTTk= +github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg= +github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU= github.com/go-echarts/go-echarts v1.0.0 h1:n181E4iXwj4zrU9VYmdM2m8dyhERt2w9k9YhHqdp6A8= github.com/go-echarts/go-echarts v1.0.0/go.mod h1:qbmyAb/Rl1f2w7wKba1D4LoNq4U164yO4/wedFbcWyo= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= @@ -176,17 +185,21 @@ github.com/go-openapi/spec v0.20.4/go.mod h1:faYFR1CvsJZ0mNsmsphTMSoRrNV3TEDoAM7 github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.15 h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyrCM= github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= -github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= -github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU= github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= -github.com/go-playground/universal-translator v0.18.0 h1:82dyy6p4OuJq4/CByFNOn/jYrnRPArHwAcmLoJZxyho= github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI= -github.com/go-playground/validator/v10 v10.10.0 h1:I7mrTYv78z8k8VXa/qJlOlEXn/nBh+BF8dHX5nt/dr0= github.com/go-playground/validator/v10 v10.10.0/go.mod h1:74x4gJWsvQexRdW8Pn3dXSGrTK4nAUsbPlLADvpJkos= +github.com/go-playground/validator/v10 v10.14.0 h1:vgvQWe3XCz3gIeFDm/HnTIbj6UGmg/+t63MyGU2n5js= +github.com/go-playground/validator/v10 v10.14.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= github.com/go-resty/resty/v2 v2.6.0 h1:joIR5PNLM2EFqqESUjCMGXrWmXNHEU9CEiK813oKYS4= github.com/go-resty/resty/v2 v2.6.0/go.mod h1:PwvJS6hvaPkjtjNg9ph+VrSD92bi5Zq73w/BIH7cC3Q= github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc= @@ -194,8 +207,9 @@ github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/goccy/go-graphviz v0.0.9 h1:s/FMMJ1Joj6La3S5ApO3Jk2cwM4LpXECC2muFx3IPQQ= github.com/goccy/go-graphviz v0.0.9/go.mod h1:wXVsXxmyMQU6TN3zGRttjNn3h+iCAS7xQFC6TlNvLhk= -github.com/goccy/go-json v0.9.7 h1:IcB+Aqpx/iMHu5Yooh7jEzJk1JZ7Pjtmys2ukPr7EeM= github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= +github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/gogo/protobuf v0.0.0-20171007142547-342cbe0a0415/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v0.0.0-20180717141946-636bf0302bc9/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -335,6 +349,9 @@ github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= +github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -348,8 +365,9 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= -github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= +github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= +github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a h1:N9zuLhTvBSRt0gWSiJswwQ2HqDmtX/ZCDJURnKUt1Ik= github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a/go.mod h1:JKx41uQRwqlTZabZc+kILPrO/3jlKnQ2Z8b7YiVw5cE= @@ -366,8 +384,9 @@ github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNx github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.8 h1:3tS41NlGYSmhhe/8fhGRzc+z3AYCw1Fe1WAyLuujKs0= @@ -421,8 +440,9 @@ github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT9 github.com/otiai10/mint v1.3.3/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc= github.com/pascaldekloe/name v0.0.0-20180628100202-0fd16699aae1/go.mod h1:eD5JxqMiuNYyFNmyY9rkJ/slN8y59oEu4Ei7F8OoKWQ= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pelletier/go-toml/v2 v2.0.1 h1:8e3L2cCQzLFi2CR4g7vGFuFxX7Jl1kKX8gW+iV0GUKU= github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo= +github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= +github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4= github.com/petermattis/goid v0.0.0-20211229010228-4d14c490ee36 h1:64bxqeTEN0/xoEqhKGowgihNuzISS9rEG6YUMU4bzJo= github.com/petermattis/goid v0.0.0-20211229010228-4d14c490ee36/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4= github.com/phf/go-queue v0.0.0-20170504031614-9abe38d0371d h1:U+PMnTlV2tu7RuMK5etusZG3Cf+rpow5hqQByeCzJ2g= @@ -548,8 +568,10 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= +github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/swaggo/files v0.0.0-20210815190702-a29dd2bc99b2 h1:+iNTcqQJy0OZ5jk6a5NLib47eqXK8uYcPX+O4+cBpEM= github.com/swaggo/files v0.0.0-20210815190702-a29dd2bc99b2/go.mod h1:lKJPbtWzJ9JhsTN1k1gZgleJWY/cqq0psdoMmaThG3w= github.com/swaggo/http-swagger v1.2.6 h1:ihTjChUoSRMpFMjWw+0AkL1Ti4r6v8pCgVYLmQVRlRw= @@ -577,13 +599,16 @@ github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20200427203606-3cfed13b9966 h1:j6JEOq5QWFker+d7mFQYOhjTZonQ7YkLTHm56dbn+yM= github.com/tmc/grpc-websocket-proxy v0.0.0-20200427203606-3cfed13b9966/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= +github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go v1.2.7/go.mod h1:nF9osbDWLy6bDVv/Rtoh6QgnvNDpmCalQV5urGCCS6M= github.com/ugorji/go/codec v0.0.0-20181022190402-e5e69e061d4f/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= -github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0= github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY= +github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= +github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/unrolled/render v1.0.1 h1:VDDnQQVfBMsOsp3VaCJszSO0nkBIVEYoPWeRThk9spY= github.com/unrolled/render v1.0.1/go.mod h1:gN9T0NhL4Bfbwu8ann7Ry/TGHYfosul+J0obPf6NBdM= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= @@ -642,6 +667,9 @@ go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= +golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= +golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k= +golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -756,6 +784,7 @@ golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211013075003-97ac67df715c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -900,6 +929,7 @@ honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= moul.io/zapgorm2 v1.1.0 h1:qwAlMBYf+qJkJ7PAzJl4oCe6eS6QGiKAXUPeis0+RBE= moul.io/zapgorm2 v1.1.0/go.mod h1:emRfKjNqSzVj5lcgasBdovIXY1jSOwFz2GQZn1Rddks= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/tests/integrations/client/go.mod b/tests/integrations/client/go.mod index a4aca195f3f..a3c85eeb2fe 100644 --- a/tests/integrations/client/go.mod +++ b/tests/integrations/client/go.mod @@ -15,7 +15,7 @@ require ( github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 github.com/pingcap/kvproto v0.0.0-20231018065736-c0689aded40c github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 - github.com/stretchr/testify v1.8.2 + github.com/stretchr/testify v1.8.3 github.com/tikv/pd v0.0.0-00010101000000-000000000000 github.com/tikv/pd/client v0.0.0-00010101000000-000000000000 go.etcd.io/etcd v0.5.0-alpha.5.0.20220915004622-85b640cee793 @@ -50,9 +50,11 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/bitly/go-simplejson v0.5.0 // indirect github.com/breeswish/gin-jwt/v2 v2.6.4-jwt-patch // indirect + github.com/bytedance/sonic v1.9.1 // indirect github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 // indirect github.com/cenkalti/backoff/v4 v4.0.2 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect github.com/coreos/go-semver v0.3.0 // indirect github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f // indirect github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect @@ -60,23 +62,24 @@ require ( github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4 // indirect github.com/elliotchance/pie/v2 v2.1.0 // indirect github.com/fogleman/gg v1.3.0 // indirect + github.com/gabriel-vasile/mimetype v1.4.2 // indirect github.com/gin-contrib/cors v1.4.0 // indirect github.com/gin-contrib/gzip v0.0.1 // indirect github.com/gin-contrib/pprof v1.4.0 // indirect github.com/gin-contrib/sse v0.1.0 // indirect - github.com/gin-gonic/gin v1.8.1 // indirect + github.com/gin-gonic/gin v1.9.1 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect github.com/go-openapi/jsonreference v0.19.6 // indirect github.com/go-openapi/spec v0.20.4 // indirect github.com/go-openapi/swag v0.19.15 // indirect - github.com/go-playground/locales v0.14.0 // indirect - github.com/go-playground/universal-translator v0.18.0 // indirect - github.com/go-playground/validator/v10 v10.10.0 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/go-playground/validator/v10 v10.14.0 // indirect github.com/go-resty/resty/v2 v2.6.0 // indirect github.com/go-sql-driver/mysql v1.7.0 // indirect github.com/goccy/go-graphviz v0.0.9 // indirect - github.com/goccy/go-json v0.9.7 // indirect + github.com/goccy/go-json v0.10.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt v3.2.1+incompatible // indirect github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect @@ -100,11 +103,12 @@ require ( github.com/joomcode/errorx v1.0.1 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/cpuid/v2 v2.2.4 // indirect github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect - github.com/leodido/go-urn v1.2.1 // indirect + github.com/leodido/go-urn v1.2.4 // indirect github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a // indirect github.com/mailru/easyjson v0.7.6 // indirect - github.com/mattn/go-isatty v0.0.14 // indirect + github.com/mattn/go-isatty v0.0.19 // indirect github.com/mattn/go-sqlite3 v1.14.15 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/minio/sio v0.3.0 // indirect @@ -112,7 +116,7 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/oleiade/reflections v1.0.1 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect - github.com/pelletier/go-toml/v2 v2.0.1 // indirect + github.com/pelletier/go-toml/v2 v2.0.8 // indirect github.com/petermattis/goid v0.0.0-20211229010228-4d14c490ee36 // indirect github.com/phf/go-queue v0.0.0-20170504031614-9abe38d0371d // indirect github.com/pingcap/check v0.0.0-20211026125417-57bd13f7b5f0 // indirect @@ -147,7 +151,8 @@ require ( github.com/tklauser/go-sysconf v0.3.11 // indirect github.com/tklauser/numcpus v0.6.0 // indirect github.com/tmc/grpc-websocket-proxy v0.0.0-20200427203606-3cfed13b9966 // indirect - github.com/ugorji/go/codec v1.2.7 // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + github.com/ugorji/go/codec v1.2.11 // indirect github.com/unrolled/render v1.0.1 // indirect github.com/urfave/negroni v0.3.0 // indirect github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect @@ -159,6 +164,7 @@ require ( go.uber.org/dig v1.9.0 // indirect go.uber.org/fx v1.12.0 // indirect go.uber.org/multierr v1.11.0 // indirect + golang.org/x/arch v0.3.0 // indirect golang.org/x/crypto v0.14.0 // indirect golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4 // indirect golang.org/x/image v0.5.0 // indirect diff --git a/tests/integrations/client/go.sum b/tests/integrations/client/go.sum index ef9c4d2a5f3..c16faee2e7a 100644 --- a/tests/integrations/client/go.sum +++ b/tests/integrations/client/go.sum @@ -68,6 +68,9 @@ github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4Yn github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/breeswish/gin-jwt/v2 v2.6.4-jwt-patch h1:KLE/YeX+9FNaGVW5MtImRVPhjDpfpgJhvkuYWBmOYbo= github.com/breeswish/gin-jwt/v2 v2.6.4-jwt-patch/go.mod h1:KjBLriHXe7L6fGceqWzTod8HUB/TP1WWDtfuSYtYXaI= +github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= +github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s= +github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 h1:BjkPE3785EwPhhyuFkbINB+2a1xATwk8SNDWnJiD41g= github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5/go.mod h1:jtAfVaU/2cu1+wdSRPWE2c1N2qeAA3K4RH9pYgqwets= github.com/cenkalti/backoff/v4 v4.0.2 h1:JIufpQLbh4DkbQoii76ItQIUFzevQSqOLZca4eamEDs= @@ -77,6 +80,9 @@ github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghf github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= +github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams= +github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -123,6 +129,8 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv github.com/fogleman/gg v1.3.0 h1:/7zJX8F6AaYQc57WQCyN9cAIz+4bCJGO9B+dyW29am8= github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= +github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/cors v1.4.0 h1:oJ6gwtUl3lqV0WEIwM/LxPF1QZ5qe2lGWdY2+bz7y0g= github.com/gin-contrib/cors v1.4.0/go.mod h1:bs9pNM0x/UsmHPBWT2xZz9ROh8xYjYkiURUfmBoMlcs= @@ -135,8 +143,9 @@ github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.3.0/go.mod h1:7cKuhb5qV2ggCFctp2fJQ+ErvciLZrIeoOSOm6mUr7Y= github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M= -github.com/gin-gonic/gin v1.8.1 h1:4+fr/el88TOO3ewCmQr8cx/CtZ/umlIRIs5M4NTNjf8= github.com/gin-gonic/gin v1.8.1/go.mod h1:ji8BvRH1azfM+SYow9zQ6SZMvR8qOMZHmsCuWR9tTTk= +github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg= +github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= @@ -155,17 +164,21 @@ github.com/go-openapi/spec v0.20.4/go.mod h1:faYFR1CvsJZ0mNsmsphTMSoRrNV3TEDoAM7 github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.15 h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyrCM= github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= -github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= -github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU= github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= -github.com/go-playground/universal-translator v0.18.0 h1:82dyy6p4OuJq4/CByFNOn/jYrnRPArHwAcmLoJZxyho= github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI= -github.com/go-playground/validator/v10 v10.10.0 h1:I7mrTYv78z8k8VXa/qJlOlEXn/nBh+BF8dHX5nt/dr0= github.com/go-playground/validator/v10 v10.10.0/go.mod h1:74x4gJWsvQexRdW8Pn3dXSGrTK4nAUsbPlLADvpJkos= +github.com/go-playground/validator/v10 v10.14.0 h1:vgvQWe3XCz3gIeFDm/HnTIbj6UGmg/+t63MyGU2n5js= +github.com/go-playground/validator/v10 v10.14.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= github.com/go-resty/resty/v2 v2.6.0 h1:joIR5PNLM2EFqqESUjCMGXrWmXNHEU9CEiK813oKYS4= github.com/go-resty/resty/v2 v2.6.0/go.mod h1:PwvJS6hvaPkjtjNg9ph+VrSD92bi5Zq73w/BIH7cC3Q= github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc= @@ -173,8 +186,9 @@ github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/goccy/go-graphviz v0.0.9 h1:s/FMMJ1Joj6La3S5ApO3Jk2cwM4LpXECC2muFx3IPQQ= github.com/goccy/go-graphviz v0.0.9/go.mod h1:wXVsXxmyMQU6TN3zGRttjNn3h+iCAS7xQFC6TlNvLhk= -github.com/goccy/go-json v0.9.7 h1:IcB+Aqpx/iMHu5Yooh7jEzJk1JZ7Pjtmys2ukPr7EeM= github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= +github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/gogo/protobuf v0.0.0-20171007142547-342cbe0a0415/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v0.0.0-20180717141946-636bf0302bc9/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -312,6 +326,9 @@ github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= +github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -325,8 +342,9 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= -github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= +github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= +github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a h1:N9zuLhTvBSRt0gWSiJswwQ2HqDmtX/ZCDJURnKUt1Ik= github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a/go.mod h1:JKx41uQRwqlTZabZc+kILPrO/3jlKnQ2Z8b7YiVw5cE= @@ -338,8 +356,9 @@ github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJ github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-sqlite3 v1.14.15 h1:vfoHhTN1af61xCRSWzFIWzx2YskyMTwHLrExkBOjvxI= github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= @@ -383,8 +402,9 @@ github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT9 github.com/otiai10/mint v1.3.3/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc= github.com/pascaldekloe/name v0.0.0-20180628100202-0fd16699aae1/go.mod h1:eD5JxqMiuNYyFNmyY9rkJ/slN8y59oEu4Ei7F8OoKWQ= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pelletier/go-toml/v2 v2.0.1 h1:8e3L2cCQzLFi2CR4g7vGFuFxX7Jl1kKX8gW+iV0GUKU= github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo= +github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= +github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4= github.com/petermattis/goid v0.0.0-20211229010228-4d14c490ee36 h1:64bxqeTEN0/xoEqhKGowgihNuzISS9rEG6YUMU4bzJo= github.com/petermattis/goid v0.0.0-20211229010228-4d14c490ee36/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4= github.com/phf/go-queue v0.0.0-20170504031614-9abe38d0371d h1:U+PMnTlV2tu7RuMK5etusZG3Cf+rpow5hqQByeCzJ2g= @@ -508,8 +528,10 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= +github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/swaggo/files v0.0.0-20210815190702-a29dd2bc99b2 h1:+iNTcqQJy0OZ5jk6a5NLib47eqXK8uYcPX+O4+cBpEM= github.com/swaggo/files v0.0.0-20210815190702-a29dd2bc99b2/go.mod h1:lKJPbtWzJ9JhsTN1k1gZgleJWY/cqq0psdoMmaThG3w= github.com/swaggo/http-swagger v1.2.6 h1:ihTjChUoSRMpFMjWw+0AkL1Ti4r6v8pCgVYLmQVRlRw= @@ -535,13 +557,16 @@ github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20200427203606-3cfed13b9966 h1:j6JEOq5QWFker+d7mFQYOhjTZonQ7YkLTHm56dbn+yM= github.com/tmc/grpc-websocket-proxy v0.0.0-20200427203606-3cfed13b9966/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= +github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go v1.2.7/go.mod h1:nF9osbDWLy6bDVv/Rtoh6QgnvNDpmCalQV5urGCCS6M= github.com/ugorji/go/codec v0.0.0-20181022190402-e5e69e061d4f/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= -github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0= github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY= +github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= +github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/unrolled/render v1.0.1 h1:VDDnQQVfBMsOsp3VaCJszSO0nkBIVEYoPWeRThk9spY= github.com/unrolled/render v1.0.1/go.mod h1:gN9T0NhL4Bfbwu8ann7Ry/TGHYfosul+J0obPf6NBdM= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= @@ -599,6 +624,9 @@ go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= +golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= +golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k= +golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -708,6 +736,7 @@ golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211013075003-97ac67df715c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -842,6 +871,7 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= moul.io/zapgorm2 v1.1.0 h1:qwAlMBYf+qJkJ7PAzJl4oCe6eS6QGiKAXUPeis0+RBE= moul.io/zapgorm2 v1.1.0/go.mod h1:emRfKjNqSzVj5lcgasBdovIXY1jSOwFz2GQZn1Rddks= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/tests/integrations/mcs/go.mod b/tests/integrations/mcs/go.mod index f6df0eb4de0..216cdad2512 100644 --- a/tests/integrations/mcs/go.mod +++ b/tests/integrations/mcs/go.mod @@ -15,7 +15,7 @@ require ( github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 github.com/pingcap/kvproto v0.0.0-20231018065736-c0689aded40c github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 - github.com/stretchr/testify v1.8.2 + github.com/stretchr/testify v1.8.3 github.com/tikv/pd v0.0.0-00010101000000-000000000000 github.com/tikv/pd/client v0.0.0-00010101000000-000000000000 go.etcd.io/etcd v0.5.0-alpha.5.0.20220915004622-85b640cee793 @@ -50,9 +50,11 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/bitly/go-simplejson v0.5.0 // indirect github.com/breeswish/gin-jwt/v2 v2.6.4-jwt-patch // indirect + github.com/bytedance/sonic v1.9.1 // indirect github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 // indirect github.com/cenkalti/backoff/v4 v4.0.2 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect github.com/cloudfoundry/gosigar v1.3.6 // indirect github.com/coreos/go-semver v0.3.0 // indirect github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f // indirect @@ -61,23 +63,24 @@ require ( github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4 // indirect github.com/elliotchance/pie/v2 v2.1.0 // indirect github.com/fogleman/gg v1.3.0 // indirect + github.com/gabriel-vasile/mimetype v1.4.2 // indirect github.com/gin-contrib/cors v1.4.0 // indirect github.com/gin-contrib/gzip v0.0.1 // indirect github.com/gin-contrib/pprof v1.4.0 // indirect github.com/gin-contrib/sse v0.1.0 // indirect - github.com/gin-gonic/gin v1.8.1 // indirect + github.com/gin-gonic/gin v1.9.1 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect github.com/go-openapi/jsonreference v0.19.6 // indirect github.com/go-openapi/spec v0.20.4 // indirect github.com/go-openapi/swag v0.19.15 // indirect - github.com/go-playground/locales v0.14.0 // indirect - github.com/go-playground/universal-translator v0.18.0 // indirect - github.com/go-playground/validator/v10 v10.10.0 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/go-playground/validator/v10 v10.14.0 // indirect github.com/go-resty/resty/v2 v2.6.0 // indirect github.com/go-sql-driver/mysql v1.7.0 // indirect github.com/goccy/go-graphviz v0.0.9 // indirect - github.com/goccy/go-json v0.9.7 // indirect + github.com/goccy/go-json v0.10.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt v3.2.1+incompatible // indirect github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect @@ -101,11 +104,12 @@ require ( github.com/joomcode/errorx v1.0.1 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/cpuid/v2 v2.2.4 // indirect github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect - github.com/leodido/go-urn v1.2.1 // indirect + github.com/leodido/go-urn v1.2.4 // indirect github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a // indirect github.com/mailru/easyjson v0.7.6 // indirect - github.com/mattn/go-isatty v0.0.14 // indirect + github.com/mattn/go-isatty v0.0.19 // indirect github.com/mattn/go-sqlite3 v1.14.15 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/minio/sio v0.3.0 // indirect @@ -113,7 +117,7 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/oleiade/reflections v1.0.1 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect - github.com/pelletier/go-toml/v2 v2.0.1 // indirect + github.com/pelletier/go-toml/v2 v2.0.8 // indirect github.com/petermattis/goid v0.0.0-20211229010228-4d14c490ee36 // indirect github.com/phf/go-queue v0.0.0-20170504031614-9abe38d0371d // indirect github.com/pingcap/errcode v0.3.0 // indirect @@ -147,7 +151,8 @@ require ( github.com/tklauser/go-sysconf v0.3.11 // indirect github.com/tklauser/numcpus v0.6.0 // indirect github.com/tmc/grpc-websocket-proxy v0.0.0-20200427203606-3cfed13b9966 // indirect - github.com/ugorji/go/codec v1.2.7 // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + github.com/ugorji/go/codec v1.2.11 // indirect github.com/unrolled/render v1.0.1 // indirect github.com/urfave/negroni v0.3.0 // indirect github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect @@ -159,6 +164,7 @@ require ( go.uber.org/dig v1.9.0 // indirect go.uber.org/fx v1.12.0 // indirect go.uber.org/multierr v1.11.0 // indirect + golang.org/x/arch v0.3.0 // indirect golang.org/x/crypto v0.14.0 // indirect golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4 // indirect golang.org/x/image v0.5.0 // indirect diff --git a/tests/integrations/mcs/go.sum b/tests/integrations/mcs/go.sum index fc1dc1bbea5..c36a3dd2d4c 100644 --- a/tests/integrations/mcs/go.sum +++ b/tests/integrations/mcs/go.sum @@ -68,6 +68,9 @@ github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4Yn github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/breeswish/gin-jwt/v2 v2.6.4-jwt-patch h1:KLE/YeX+9FNaGVW5MtImRVPhjDpfpgJhvkuYWBmOYbo= github.com/breeswish/gin-jwt/v2 v2.6.4-jwt-patch/go.mod h1:KjBLriHXe7L6fGceqWzTod8HUB/TP1WWDtfuSYtYXaI= +github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= +github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s= +github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 h1:BjkPE3785EwPhhyuFkbINB+2a1xATwk8SNDWnJiD41g= github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5/go.mod h1:jtAfVaU/2cu1+wdSRPWE2c1N2qeAA3K4RH9pYgqwets= github.com/cenkalti/backoff/v4 v4.0.2 h1:JIufpQLbh4DkbQoii76ItQIUFzevQSqOLZca4eamEDs= @@ -77,6 +80,9 @@ github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghf github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= +github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams= +github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -127,6 +133,8 @@ github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzP github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= +github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/cors v1.4.0 h1:oJ6gwtUl3lqV0WEIwM/LxPF1QZ5qe2lGWdY2+bz7y0g= github.com/gin-contrib/cors v1.4.0/go.mod h1:bs9pNM0x/UsmHPBWT2xZz9ROh8xYjYkiURUfmBoMlcs= @@ -139,8 +147,9 @@ github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.3.0/go.mod h1:7cKuhb5qV2ggCFctp2fJQ+ErvciLZrIeoOSOm6mUr7Y= github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M= -github.com/gin-gonic/gin v1.8.1 h1:4+fr/el88TOO3ewCmQr8cx/CtZ/umlIRIs5M4NTNjf8= github.com/gin-gonic/gin v1.8.1/go.mod h1:ji8BvRH1azfM+SYow9zQ6SZMvR8qOMZHmsCuWR9tTTk= +github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg= +github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= @@ -159,17 +168,21 @@ github.com/go-openapi/spec v0.20.4/go.mod h1:faYFR1CvsJZ0mNsmsphTMSoRrNV3TEDoAM7 github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.15 h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyrCM= github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= -github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= -github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU= github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= -github.com/go-playground/universal-translator v0.18.0 h1:82dyy6p4OuJq4/CByFNOn/jYrnRPArHwAcmLoJZxyho= github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI= -github.com/go-playground/validator/v10 v10.10.0 h1:I7mrTYv78z8k8VXa/qJlOlEXn/nBh+BF8dHX5nt/dr0= github.com/go-playground/validator/v10 v10.10.0/go.mod h1:74x4gJWsvQexRdW8Pn3dXSGrTK4nAUsbPlLADvpJkos= +github.com/go-playground/validator/v10 v10.14.0 h1:vgvQWe3XCz3gIeFDm/HnTIbj6UGmg/+t63MyGU2n5js= +github.com/go-playground/validator/v10 v10.14.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= github.com/go-resty/resty/v2 v2.6.0 h1:joIR5PNLM2EFqqESUjCMGXrWmXNHEU9CEiK813oKYS4= github.com/go-resty/resty/v2 v2.6.0/go.mod h1:PwvJS6hvaPkjtjNg9ph+VrSD92bi5Zq73w/BIH7cC3Q= github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc= @@ -177,8 +190,9 @@ github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/goccy/go-graphviz v0.0.9 h1:s/FMMJ1Joj6La3S5ApO3Jk2cwM4LpXECC2muFx3IPQQ= github.com/goccy/go-graphviz v0.0.9/go.mod h1:wXVsXxmyMQU6TN3zGRttjNn3h+iCAS7xQFC6TlNvLhk= -github.com/goccy/go-json v0.9.7 h1:IcB+Aqpx/iMHu5Yooh7jEzJk1JZ7Pjtmys2ukPr7EeM= github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= +github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/gogo/protobuf v0.0.0-20171007142547-342cbe0a0415/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v0.0.0-20180717141946-636bf0302bc9/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -315,6 +329,9 @@ github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= +github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -328,8 +345,9 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= -github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= +github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= +github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a h1:N9zuLhTvBSRt0gWSiJswwQ2HqDmtX/ZCDJURnKUt1Ik= github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a/go.mod h1:JKx41uQRwqlTZabZc+kILPrO/3jlKnQ2Z8b7YiVw5cE= @@ -341,8 +359,9 @@ github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJ github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-sqlite3 v1.14.15 h1:vfoHhTN1af61xCRSWzFIWzx2YskyMTwHLrExkBOjvxI= github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= @@ -389,8 +408,9 @@ github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT9 github.com/otiai10/mint v1.3.3/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc= github.com/pascaldekloe/name v0.0.0-20180628100202-0fd16699aae1/go.mod h1:eD5JxqMiuNYyFNmyY9rkJ/slN8y59oEu4Ei7F8OoKWQ= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pelletier/go-toml/v2 v2.0.1 h1:8e3L2cCQzLFi2CR4g7vGFuFxX7Jl1kKX8gW+iV0GUKU= github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo= +github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= +github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4= github.com/petermattis/goid v0.0.0-20211229010228-4d14c490ee36 h1:64bxqeTEN0/xoEqhKGowgihNuzISS9rEG6YUMU4bzJo= github.com/petermattis/goid v0.0.0-20211229010228-4d14c490ee36/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4= github.com/phf/go-queue v0.0.0-20170504031614-9abe38d0371d h1:U+PMnTlV2tu7RuMK5etusZG3Cf+rpow5hqQByeCzJ2g= @@ -512,8 +532,10 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= +github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/swaggo/files v0.0.0-20210815190702-a29dd2bc99b2 h1:+iNTcqQJy0OZ5jk6a5NLib47eqXK8uYcPX+O4+cBpEM= github.com/swaggo/files v0.0.0-20210815190702-a29dd2bc99b2/go.mod h1:lKJPbtWzJ9JhsTN1k1gZgleJWY/cqq0psdoMmaThG3w= github.com/swaggo/http-swagger v1.2.6 h1:ihTjChUoSRMpFMjWw+0AkL1Ti4r6v8pCgVYLmQVRlRw= @@ -539,13 +561,16 @@ github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20200427203606-3cfed13b9966 h1:j6JEOq5QWFker+d7mFQYOhjTZonQ7YkLTHm56dbn+yM= github.com/tmc/grpc-websocket-proxy v0.0.0-20200427203606-3cfed13b9966/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= +github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go v1.2.7/go.mod h1:nF9osbDWLy6bDVv/Rtoh6QgnvNDpmCalQV5urGCCS6M= github.com/ugorji/go/codec v0.0.0-20181022190402-e5e69e061d4f/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= -github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0= github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY= +github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= +github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/unrolled/render v1.0.1 h1:VDDnQQVfBMsOsp3VaCJszSO0nkBIVEYoPWeRThk9spY= github.com/unrolled/render v1.0.1/go.mod h1:gN9T0NhL4Bfbwu8ann7Ry/TGHYfosul+J0obPf6NBdM= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= @@ -603,6 +628,9 @@ go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= +golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= +golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k= +golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -712,6 +740,7 @@ golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211013075003-97ac67df715c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -844,6 +873,7 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= moul.io/zapgorm2 v1.1.0 h1:qwAlMBYf+qJkJ7PAzJl4oCe6eS6QGiKAXUPeis0+RBE= moul.io/zapgorm2 v1.1.0/go.mod h1:emRfKjNqSzVj5lcgasBdovIXY1jSOwFz2GQZn1Rddks= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/tests/integrations/tso/go.mod b/tests/integrations/tso/go.mod index 7e833943e6e..17c432ba810 100644 --- a/tests/integrations/tso/go.mod +++ b/tests/integrations/tso/go.mod @@ -47,9 +47,11 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/bitly/go-simplejson v0.5.0 // indirect github.com/breeswish/gin-jwt/v2 v2.6.4-jwt-patch // indirect + github.com/bytedance/sonic v1.9.1 // indirect github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 // indirect github.com/cenkalti/backoff/v4 v4.0.2 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect github.com/coreos/go-semver v0.3.0 // indirect github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f // indirect github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect @@ -58,23 +60,24 @@ require ( github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4 // indirect github.com/elliotchance/pie/v2 v2.1.0 // indirect github.com/fogleman/gg v1.3.0 // indirect + github.com/gabriel-vasile/mimetype v1.4.2 // indirect github.com/gin-contrib/cors v1.4.0 // indirect github.com/gin-contrib/gzip v0.0.1 // indirect github.com/gin-contrib/pprof v1.4.0 // indirect github.com/gin-contrib/sse v0.1.0 // indirect - github.com/gin-gonic/gin v1.8.1 // indirect + github.com/gin-gonic/gin v1.9.1 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect github.com/go-openapi/jsonreference v0.19.6 // indirect github.com/go-openapi/spec v0.20.4 // indirect github.com/go-openapi/swag v0.19.15 // indirect - github.com/go-playground/locales v0.14.0 // indirect - github.com/go-playground/universal-translator v0.18.0 // indirect - github.com/go-playground/validator/v10 v10.10.0 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/go-playground/validator/v10 v10.14.0 // indirect github.com/go-resty/resty/v2 v2.6.0 // indirect github.com/go-sql-driver/mysql v1.7.0 // indirect github.com/goccy/go-graphviz v0.0.9 // indirect - github.com/goccy/go-json v0.9.7 // indirect + github.com/goccy/go-json v0.10.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt v3.2.1+incompatible // indirect github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect @@ -98,11 +101,12 @@ require ( github.com/joomcode/errorx v1.0.1 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/cpuid/v2 v2.2.4 // indirect github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect - github.com/leodido/go-urn v1.2.1 // indirect + github.com/leodido/go-urn v1.2.4 // indirect github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a // indirect github.com/mailru/easyjson v0.7.6 // indirect - github.com/mattn/go-isatty v0.0.14 // indirect + github.com/mattn/go-isatty v0.0.19 // indirect github.com/mattn/go-sqlite3 v1.14.15 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/minio/sio v0.3.0 // indirect @@ -110,7 +114,7 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/oleiade/reflections v1.0.1 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect - github.com/pelletier/go-toml/v2 v2.0.1 // indirect + github.com/pelletier/go-toml/v2 v2.0.8 // indirect github.com/petermattis/goid v0.0.0-20211229010228-4d14c490ee36 // indirect github.com/phf/go-queue v0.0.0-20170504031614-9abe38d0371d // indirect github.com/pingcap/errcode v0.3.0 // indirect @@ -145,7 +149,8 @@ require ( github.com/tklauser/go-sysconf v0.3.11 // indirect github.com/tklauser/numcpus v0.6.0 // indirect github.com/tmc/grpc-websocket-proxy v0.0.0-20200427203606-3cfed13b9966 // indirect - github.com/ugorji/go/codec v1.2.7 // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + github.com/ugorji/go/codec v1.2.11 // indirect github.com/unrolled/render v1.0.1 // indirect github.com/urfave/negroni v0.3.0 // indirect github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect @@ -160,6 +165,7 @@ require ( go.uber.org/goleak v1.1.12 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.24.0 // indirect + golang.org/x/arch v0.3.0 // indirect golang.org/x/crypto v0.14.0 // indirect golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4 // indirect golang.org/x/image v0.5.0 // indirect diff --git a/tests/integrations/tso/go.sum b/tests/integrations/tso/go.sum index 65a7f3e3558..fb87df47bcd 100644 --- a/tests/integrations/tso/go.sum +++ b/tests/integrations/tso/go.sum @@ -68,6 +68,9 @@ github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4Yn github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/breeswish/gin-jwt/v2 v2.6.4-jwt-patch h1:KLE/YeX+9FNaGVW5MtImRVPhjDpfpgJhvkuYWBmOYbo= github.com/breeswish/gin-jwt/v2 v2.6.4-jwt-patch/go.mod h1:KjBLriHXe7L6fGceqWzTod8HUB/TP1WWDtfuSYtYXaI= +github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= +github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s= +github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 h1:BjkPE3785EwPhhyuFkbINB+2a1xATwk8SNDWnJiD41g= github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5/go.mod h1:jtAfVaU/2cu1+wdSRPWE2c1N2qeAA3K4RH9pYgqwets= github.com/cenkalti/backoff/v4 v4.0.2 h1:JIufpQLbh4DkbQoii76ItQIUFzevQSqOLZca4eamEDs= @@ -77,6 +80,9 @@ github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghf github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= +github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams= +github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -123,6 +129,8 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv github.com/fogleman/gg v1.3.0 h1:/7zJX8F6AaYQc57WQCyN9cAIz+4bCJGO9B+dyW29am8= github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= +github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/cors v1.4.0 h1:oJ6gwtUl3lqV0WEIwM/LxPF1QZ5qe2lGWdY2+bz7y0g= github.com/gin-contrib/cors v1.4.0/go.mod h1:bs9pNM0x/UsmHPBWT2xZz9ROh8xYjYkiURUfmBoMlcs= @@ -135,8 +143,9 @@ github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.3.0/go.mod h1:7cKuhb5qV2ggCFctp2fJQ+ErvciLZrIeoOSOm6mUr7Y= github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M= -github.com/gin-gonic/gin v1.8.1 h1:4+fr/el88TOO3ewCmQr8cx/CtZ/umlIRIs5M4NTNjf8= github.com/gin-gonic/gin v1.8.1/go.mod h1:ji8BvRH1azfM+SYow9zQ6SZMvR8qOMZHmsCuWR9tTTk= +github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg= +github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= @@ -155,17 +164,21 @@ github.com/go-openapi/spec v0.20.4/go.mod h1:faYFR1CvsJZ0mNsmsphTMSoRrNV3TEDoAM7 github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.15 h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyrCM= github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= -github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= -github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU= github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= -github.com/go-playground/universal-translator v0.18.0 h1:82dyy6p4OuJq4/CByFNOn/jYrnRPArHwAcmLoJZxyho= github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI= -github.com/go-playground/validator/v10 v10.10.0 h1:I7mrTYv78z8k8VXa/qJlOlEXn/nBh+BF8dHX5nt/dr0= github.com/go-playground/validator/v10 v10.10.0/go.mod h1:74x4gJWsvQexRdW8Pn3dXSGrTK4nAUsbPlLADvpJkos= +github.com/go-playground/validator/v10 v10.14.0 h1:vgvQWe3XCz3gIeFDm/HnTIbj6UGmg/+t63MyGU2n5js= +github.com/go-playground/validator/v10 v10.14.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= github.com/go-resty/resty/v2 v2.6.0 h1:joIR5PNLM2EFqqESUjCMGXrWmXNHEU9CEiK813oKYS4= github.com/go-resty/resty/v2 v2.6.0/go.mod h1:PwvJS6hvaPkjtjNg9ph+VrSD92bi5Zq73w/BIH7cC3Q= github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc= @@ -173,8 +186,9 @@ github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/goccy/go-graphviz v0.0.9 h1:s/FMMJ1Joj6La3S5ApO3Jk2cwM4LpXECC2muFx3IPQQ= github.com/goccy/go-graphviz v0.0.9/go.mod h1:wXVsXxmyMQU6TN3zGRttjNn3h+iCAS7xQFC6TlNvLhk= -github.com/goccy/go-json v0.9.7 h1:IcB+Aqpx/iMHu5Yooh7jEzJk1JZ7Pjtmys2ukPr7EeM= github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= +github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/gogo/protobuf v0.0.0-20171007142547-342cbe0a0415/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v0.0.0-20180717141946-636bf0302bc9/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -312,6 +326,9 @@ github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= +github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -325,8 +342,9 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= -github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= +github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= +github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a h1:N9zuLhTvBSRt0gWSiJswwQ2HqDmtX/ZCDJURnKUt1Ik= github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a/go.mod h1:JKx41uQRwqlTZabZc+kILPrO/3jlKnQ2Z8b7YiVw5cE= @@ -338,8 +356,9 @@ github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJ github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-sqlite3 v1.14.15 h1:vfoHhTN1af61xCRSWzFIWzx2YskyMTwHLrExkBOjvxI= github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= @@ -383,8 +402,9 @@ github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT9 github.com/otiai10/mint v1.3.3/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc= github.com/pascaldekloe/name v0.0.0-20180628100202-0fd16699aae1/go.mod h1:eD5JxqMiuNYyFNmyY9rkJ/slN8y59oEu4Ei7F8OoKWQ= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pelletier/go-toml/v2 v2.0.1 h1:8e3L2cCQzLFi2CR4g7vGFuFxX7Jl1kKX8gW+iV0GUKU= github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo= +github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= +github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4= github.com/petermattis/goid v0.0.0-20211229010228-4d14c490ee36 h1:64bxqeTEN0/xoEqhKGowgihNuzISS9rEG6YUMU4bzJo= github.com/petermattis/goid v0.0.0-20211229010228-4d14c490ee36/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4= github.com/phf/go-queue v0.0.0-20170504031614-9abe38d0371d h1:U+PMnTlV2tu7RuMK5etusZG3Cf+rpow5hqQByeCzJ2g= @@ -506,7 +526,9 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/swaggo/files v0.0.0-20210815190702-a29dd2bc99b2 h1:+iNTcqQJy0OZ5jk6a5NLib47eqXK8uYcPX+O4+cBpEM= @@ -534,13 +556,16 @@ github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20200427203606-3cfed13b9966 h1:j6JEOq5QWFker+d7mFQYOhjTZonQ7YkLTHm56dbn+yM= github.com/tmc/grpc-websocket-proxy v0.0.0-20200427203606-3cfed13b9966/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= +github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go v1.2.7/go.mod h1:nF9osbDWLy6bDVv/Rtoh6QgnvNDpmCalQV5urGCCS6M= github.com/ugorji/go/codec v0.0.0-20181022190402-e5e69e061d4f/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= -github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0= github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY= +github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= +github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/unrolled/render v1.0.1 h1:VDDnQQVfBMsOsp3VaCJszSO0nkBIVEYoPWeRThk9spY= github.com/unrolled/render v1.0.1/go.mod h1:gN9T0NhL4Bfbwu8ann7Ry/TGHYfosul+J0obPf6NBdM= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= @@ -598,6 +623,9 @@ go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= +golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= +golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k= +golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -707,6 +735,7 @@ golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211013075003-97ac67df715c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -840,6 +869,7 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= moul.io/zapgorm2 v1.1.0 h1:qwAlMBYf+qJkJ7PAzJl4oCe6eS6QGiKAXUPeis0+RBE= moul.io/zapgorm2 v1.1.0/go.mod h1:emRfKjNqSzVj5lcgasBdovIXY1jSOwFz2GQZn1Rddks= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/tools/pd-api-bench/go.mod b/tools/pd-api-bench/go.mod index e6e896a0797..8050f433e8b 100644 --- a/tools/pd-api-bench/go.mod +++ b/tools/pd-api-bench/go.mod @@ -26,8 +26,10 @@ require ( github.com/aws/smithy-go v1.13.5 // indirect github.com/benbjohnson/clock v1.3.0 // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/bytedance/sonic v1.9.1 // indirect github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect github.com/coreos/go-semver v0.3.0 // indirect github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f // indirect github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect @@ -35,12 +37,13 @@ require ( github.com/docker/go-units v0.4.0 // indirect github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4 // indirect github.com/elliotchance/pie/v2 v2.1.0 // indirect + github.com/gabriel-vasile/mimetype v1.4.2 // indirect github.com/gin-contrib/sse v0.1.0 // indirect - github.com/gin-gonic/gin v1.8.1 // indirect - github.com/go-playground/locales v0.14.0 // indirect - github.com/go-playground/universal-translator v0.18.0 // indirect - github.com/go-playground/validator/v10 v10.10.0 // indirect - github.com/goccy/go-json v0.9.7 // indirect + github.com/gin-gonic/gin v1.9.1 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/go-playground/validator/v10 v10.14.0 // indirect + github.com/goccy/go-json v0.10.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt v3.2.1+incompatible // indirect github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef // indirect @@ -56,14 +59,15 @@ require ( github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect github.com/jonboulle/clockwork v0.2.2 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/cpuid/v2 v2.2.4 // indirect github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect - github.com/leodido/go-urn v1.2.1 // indirect - github.com/mattn/go-isatty v0.0.14 // indirect + github.com/leodido/go-urn v1.2.4 // indirect + github.com/mattn/go-isatty v0.0.19 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect - github.com/pelletier/go-toml/v2 v2.0.1 // indirect + github.com/pelletier/go-toml/v2 v2.0.8 // indirect github.com/petermattis/goid v0.0.0-20211229010228-4d14c490ee36 // indirect github.com/phf/go-queue v0.0.0-20170504031614-9abe38d0371d // indirect github.com/pingcap/errcode v0.3.0 // indirect @@ -81,10 +85,11 @@ require ( github.com/smallnest/chanx v0.0.0-20221229104322-eb4c998d2072 // indirect github.com/soheilhy/cmux v0.1.4 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/stretchr/testify v1.8.2 // indirect + github.com/stretchr/testify v1.8.3 // indirect github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965 // indirect github.com/tmc/grpc-websocket-proxy v0.0.0-20200427203606-3cfed13b9966 // indirect - github.com/ugorji/go/codec v1.2.7 // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + github.com/ugorji/go/codec v1.2.11 // indirect github.com/unrolled/render v1.0.1 // indirect github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect go.etcd.io/bbolt v1.3.6 // indirect @@ -92,6 +97,7 @@ require ( go.uber.org/atomic v1.10.0 // indirect go.uber.org/goleak v1.1.12 // indirect go.uber.org/multierr v1.11.0 // indirect + golang.org/x/arch v0.3.0 // indirect golang.org/x/crypto v0.14.0 // indirect golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4 // indirect golang.org/x/net v0.17.0 // indirect diff --git a/tools/pd-api-bench/go.sum b/tools/pd-api-bench/go.sum index f40a4fe2f5a..1e40c511586 100644 --- a/tools/pd-api-bench/go.sum +++ b/tools/pd-api-bench/go.sum @@ -42,12 +42,18 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= +github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s= +github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 h1:BjkPE3785EwPhhyuFkbINB+2a1xATwk8SNDWnJiD41g= github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5/go.mod h1:jtAfVaU/2cu1+wdSRPWE2c1N2qeAA3K4RH9pYgqwets= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= +github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams= +github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa h1:OaNxuTZr7kxeODyLWsRMC+OD03aFUH+mW6r2d+MWa5Y= @@ -61,7 +67,6 @@ github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7 github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -80,11 +85,13 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= +github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= -github.com/gin-gonic/gin v1.8.1 h1:4+fr/el88TOO3ewCmQr8cx/CtZ/umlIRIs5M4NTNjf8= -github.com/gin-gonic/gin v1.8.1/go.mod h1:ji8BvRH1azfM+SYow9zQ6SZMvR8qOMZHmsCuWR9tTTk= +github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg= +github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= @@ -93,17 +100,17 @@ github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= -github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A= -github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= -github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU= -github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs= -github.com/go-playground/universal-translator v0.18.0 h1:82dyy6p4OuJq4/CByFNOn/jYrnRPArHwAcmLoJZxyho= -github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= -github.com/go-playground/validator/v10 v10.10.0 h1:I7mrTYv78z8k8VXa/qJlOlEXn/nBh+BF8dHX5nt/dr0= -github.com/go-playground/validator/v10 v10.10.0/go.mod h1:74x4gJWsvQexRdW8Pn3dXSGrTK4nAUsbPlLADvpJkos= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.14.0 h1:vgvQWe3XCz3gIeFDm/HnTIbj6UGmg/+t63MyGU2n5js= +github.com/go-playground/validator/v10 v10.14.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/goccy/go-json v0.9.7 h1:IcB+Aqpx/iMHu5Yooh7jEzJk1JZ7Pjtmys2ukPr7EeM= -github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= +github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= @@ -183,26 +190,26 @@ github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= +github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= -github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= -github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= +github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= +github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a h1:N9zuLhTvBSRt0gWSiJswwQ2HqDmtX/ZCDJURnKUt1Ik= github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a/go.mod h1:JKx41uQRwqlTZabZc+kILPrO/3jlKnQ2Z8b7YiVw5cE= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= -github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= @@ -224,8 +231,8 @@ github.com/onsi/gomega v1.20.1 h1:PA/3qinGoukvymdIDV8pii6tiZgC8kbmJO6Z5+b002Q= github.com/onsi/gomega v1.20.1/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeREyVo= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= -github.com/pelletier/go-toml/v2 v2.0.1 h1:8e3L2cCQzLFi2CR4g7vGFuFxX7Jl1kKX8gW+iV0GUKU= -github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo= +github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= +github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4= github.com/petermattis/goid v0.0.0-20211229010228-4d14c490ee36 h1:64bxqeTEN0/xoEqhKGowgihNuzISS9rEG6YUMU4bzJo= github.com/petermattis/goid v0.0.0-20211229010228-4d14c490ee36/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4= github.com/phf/go-queue v0.0.0-20170504031614-9abe38d0371d h1:U+PMnTlV2tu7RuMK5etusZG3Cf+rpow5hqQByeCzJ2g= @@ -244,7 +251,6 @@ github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8 github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 h1:QV6jqlfOkh8hqvEAgwBZa+4bSgO0EeKC7s5c6Luam2I= github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21/go.mod h1:QYnjfA95ZaMefyl1NO8oPtKeb8pYUdnDVhQgf+qdpjM= -github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -274,9 +280,6 @@ github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3x github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= -github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= -github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/sasha-s/go-deadlock v0.2.0 h1:lMqc+fUb7RrFS3gQLtoQsJ7/6TV/pAIFvBsqX73DK8Y= github.com/sasha-s/go-deadlock v0.2.0/go.mod h1:StQn567HiB1fF2yJ44N9au7wOhrPS3iZqiDbRupzT10= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= @@ -305,12 +308,13 @@ github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpE github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= +github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965 h1:1oFLiOyVl+W7bnBzGhf7BbIv9loSFQcieWWYIjLqcAw= github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965/go.mod h1:9OrXJhf154huy1nPWmuSrkgjPUtUNhA+Zmy+6AESzuA= github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM= @@ -319,9 +323,10 @@ github.com/tklauser/numcpus v0.6.0 h1:kebhY2Qt+3U6RNK7UqpYNA+tJ23IBEGKkB7JQBfDYm github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ5UVIcaL4= github.com/tmc/grpc-websocket-proxy v0.0.0-20200427203606-3cfed13b9966 h1:j6JEOq5QWFker+d7mFQYOhjTZonQ7YkLTHm56dbn+yM= github.com/tmc/grpc-websocket-proxy v0.0.0-20200427203606-3cfed13b9966/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/ugorji/go v1.2.7/go.mod h1:nF9osbDWLy6bDVv/Rtoh6QgnvNDpmCalQV5urGCCS6M= -github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0= -github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY= +github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= +github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= +github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= +github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/unrolled/render v1.0.1 h1:VDDnQQVfBMsOsp3VaCJszSO0nkBIVEYoPWeRThk9spY= github.com/unrolled/render v1.0.1/go.mod h1:gN9T0NhL4Bfbwu8ann7Ry/TGHYfosul+J0obPf6NBdM= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= @@ -354,11 +359,13 @@ go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= +golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= +golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k= +golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= @@ -388,7 +395,6 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= @@ -425,9 +431,9 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -486,11 +492,9 @@ google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqw gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= @@ -515,6 +519,7 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= From f578a36a651e9fbe2ecf8c678c441b8d2e02af8c Mon Sep 17 00:00:00 2001 From: Yexiang Zhang Date: Tue, 28 Nov 2023 10:31:16 +0800 Subject: [PATCH 054/137] dashboard: update version to v20231127.1 (#7458) close tikv/pd#7457 Signed-off-by: mornyx --- go.mod | 2 +- go.sum | 4 ++-- tests/integrations/client/go.mod | 2 +- tests/integrations/client/go.sum | 4 ++-- tests/integrations/mcs/go.mod | 2 +- tests/integrations/mcs/go.sum | 4 ++-- tests/integrations/tso/go.mod | 2 +- tests/integrations/tso/go.sum | 4 ++-- 8 files changed, 12 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index e4171caea8c..676d350d22d 100644 --- a/go.mod +++ b/go.mod @@ -36,7 +36,7 @@ require ( github.com/pingcap/kvproto v0.0.0-20231018065736-c0689aded40c github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 - github.com/pingcap/tidb-dashboard v0.0.0-20231108071238-7cb8b7ff0537 + github.com/pingcap/tidb-dashboard v0.0.0-20231127105651-ce4097837c5e github.com/prometheus/client_golang v1.11.1 github.com/prometheus/common v0.26.0 github.com/sasha-s/go-deadlock v0.2.0 diff --git a/go.sum b/go.sum index 76f98d677ee..c7ceeee028c 100644 --- a/go.sum +++ b/go.sum @@ -466,8 +466,8 @@ github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8 github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 h1:QV6jqlfOkh8hqvEAgwBZa+4bSgO0EeKC7s5c6Luam2I= github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21/go.mod h1:QYnjfA95ZaMefyl1NO8oPtKeb8pYUdnDVhQgf+qdpjM= -github.com/pingcap/tidb-dashboard v0.0.0-20231108071238-7cb8b7ff0537 h1:wnHt7ETIB0vm+gbLx8QhcIEmRtrT4QlWlfpcI9vjxOk= -github.com/pingcap/tidb-dashboard v0.0.0-20231108071238-7cb8b7ff0537/go.mod h1:EZ90+V5S4TttbYag6oKZ3jcNKRwZe1Mc9vXwOt9JBYw= +github.com/pingcap/tidb-dashboard v0.0.0-20231127105651-ce4097837c5e h1:SJUSDejvKtj9vSh5ptRHh4iMrvPV3oKO8yp6/SYE8vc= +github.com/pingcap/tidb-dashboard v0.0.0-20231127105651-ce4097837c5e/go.mod h1:ucZBRz52icb23T/5Z4CsuUHmarYiin7p2MeiVBe+o8c= github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e h1:FBaTXU8C3xgt/drM58VHxojHo/QoG1oPsgWTGvaSpO4= github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= diff --git a/tests/integrations/client/go.mod b/tests/integrations/client/go.mod index a3c85eeb2fe..799901ff2e3 100644 --- a/tests/integrations/client/go.mod +++ b/tests/integrations/client/go.mod @@ -123,7 +123,7 @@ require ( github.com/pingcap/errcode v0.3.0 // indirect github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c // indirect github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 // indirect - github.com/pingcap/tidb-dashboard v0.0.0-20231108071238-7cb8b7ff0537 // indirect + github.com/pingcap/tidb-dashboard v0.0.0-20231127105651-ce4097837c5e // indirect github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect diff --git a/tests/integrations/client/go.sum b/tests/integrations/client/go.sum index c16faee2e7a..e13da5d8375 100644 --- a/tests/integrations/client/go.sum +++ b/tests/integrations/client/go.sum @@ -430,8 +430,8 @@ github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8 github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 h1:QV6jqlfOkh8hqvEAgwBZa+4bSgO0EeKC7s5c6Luam2I= github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21/go.mod h1:QYnjfA95ZaMefyl1NO8oPtKeb8pYUdnDVhQgf+qdpjM= -github.com/pingcap/tidb-dashboard v0.0.0-20231108071238-7cb8b7ff0537 h1:wnHt7ETIB0vm+gbLx8QhcIEmRtrT4QlWlfpcI9vjxOk= -github.com/pingcap/tidb-dashboard v0.0.0-20231108071238-7cb8b7ff0537/go.mod h1:EZ90+V5S4TttbYag6oKZ3jcNKRwZe1Mc9vXwOt9JBYw= +github.com/pingcap/tidb-dashboard v0.0.0-20231127105651-ce4097837c5e h1:SJUSDejvKtj9vSh5ptRHh4iMrvPV3oKO8yp6/SYE8vc= +github.com/pingcap/tidb-dashboard v0.0.0-20231127105651-ce4097837c5e/go.mod h1:ucZBRz52icb23T/5Z4CsuUHmarYiin7p2MeiVBe+o8c= github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e h1:FBaTXU8C3xgt/drM58VHxojHo/QoG1oPsgWTGvaSpO4= github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= diff --git a/tests/integrations/mcs/go.mod b/tests/integrations/mcs/go.mod index 216cdad2512..75d70e3cf06 100644 --- a/tests/integrations/mcs/go.mod +++ b/tests/integrations/mcs/go.mod @@ -123,7 +123,7 @@ require ( github.com/pingcap/errcode v0.3.0 // indirect github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c // indirect github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 // indirect - github.com/pingcap/tidb-dashboard v0.0.0-20231108071238-7cb8b7ff0537 // indirect + github.com/pingcap/tidb-dashboard v0.0.0-20231127105651-ce4097837c5e // indirect github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect diff --git a/tests/integrations/mcs/go.sum b/tests/integrations/mcs/go.sum index c36a3dd2d4c..dfead54afe1 100644 --- a/tests/integrations/mcs/go.sum +++ b/tests/integrations/mcs/go.sum @@ -434,8 +434,8 @@ github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8 github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 h1:QV6jqlfOkh8hqvEAgwBZa+4bSgO0EeKC7s5c6Luam2I= github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21/go.mod h1:QYnjfA95ZaMefyl1NO8oPtKeb8pYUdnDVhQgf+qdpjM= -github.com/pingcap/tidb-dashboard v0.0.0-20231108071238-7cb8b7ff0537 h1:wnHt7ETIB0vm+gbLx8QhcIEmRtrT4QlWlfpcI9vjxOk= -github.com/pingcap/tidb-dashboard v0.0.0-20231108071238-7cb8b7ff0537/go.mod h1:EZ90+V5S4TttbYag6oKZ3jcNKRwZe1Mc9vXwOt9JBYw= +github.com/pingcap/tidb-dashboard v0.0.0-20231127105651-ce4097837c5e h1:SJUSDejvKtj9vSh5ptRHh4iMrvPV3oKO8yp6/SYE8vc= +github.com/pingcap/tidb-dashboard v0.0.0-20231127105651-ce4097837c5e/go.mod h1:ucZBRz52icb23T/5Z4CsuUHmarYiin7p2MeiVBe+o8c= github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e h1:FBaTXU8C3xgt/drM58VHxojHo/QoG1oPsgWTGvaSpO4= github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= diff --git a/tests/integrations/tso/go.mod b/tests/integrations/tso/go.mod index 17c432ba810..309ea9dbc4d 100644 --- a/tests/integrations/tso/go.mod +++ b/tests/integrations/tso/go.mod @@ -121,7 +121,7 @@ require ( github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c // indirect github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 // indirect github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 // indirect - github.com/pingcap/tidb-dashboard v0.0.0-20231108071238-7cb8b7ff0537 // indirect + github.com/pingcap/tidb-dashboard v0.0.0-20231127105651-ce4097837c5e // indirect github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect diff --git a/tests/integrations/tso/go.sum b/tests/integrations/tso/go.sum index fb87df47bcd..94fbde2ad57 100644 --- a/tests/integrations/tso/go.sum +++ b/tests/integrations/tso/go.sum @@ -428,8 +428,8 @@ github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8 github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 h1:QV6jqlfOkh8hqvEAgwBZa+4bSgO0EeKC7s5c6Luam2I= github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21/go.mod h1:QYnjfA95ZaMefyl1NO8oPtKeb8pYUdnDVhQgf+qdpjM= -github.com/pingcap/tidb-dashboard v0.0.0-20231108071238-7cb8b7ff0537 h1:wnHt7ETIB0vm+gbLx8QhcIEmRtrT4QlWlfpcI9vjxOk= -github.com/pingcap/tidb-dashboard v0.0.0-20231108071238-7cb8b7ff0537/go.mod h1:EZ90+V5S4TttbYag6oKZ3jcNKRwZe1Mc9vXwOt9JBYw= +github.com/pingcap/tidb-dashboard v0.0.0-20231127105651-ce4097837c5e h1:SJUSDejvKtj9vSh5ptRHh4iMrvPV3oKO8yp6/SYE8vc= +github.com/pingcap/tidb-dashboard v0.0.0-20231127105651-ce4097837c5e/go.mod h1:ucZBRz52icb23T/5Z4CsuUHmarYiin7p2MeiVBe+o8c= github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e h1:FBaTXU8C3xgt/drM58VHxojHo/QoG1oPsgWTGvaSpO4= github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= From c8257ed6c5ec47f000a2428721baf94621cc6c81 Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Tue, 28 Nov 2023 11:37:16 +0800 Subject: [PATCH 055/137] *: make `TestUpdateDefaultReplicaConfig` stable (#7455) close tikv/pd#7410 Signed-off-by: Ryan Leung Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- tests/pdctl/config/config_test.go | 48 +++++++++++++++---------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/tests/pdctl/config/config_test.go b/tests/pdctl/config/config_test.go index badccd9becc..3f82be44399 100644 --- a/tests/pdctl/config/config_test.go +++ b/tests/pdctl/config/config_test.go @@ -683,66 +683,66 @@ func (suite *configTestSuite) checkUpdateDefaultReplicaConfig(cluster *tests.Tes checkMaxReplicas := func(expect uint64) { args := []string{"-u", pdAddr, "config", "show", "replication"} - output, err := pdctl.ExecuteCommand(cmd, args...) - re.NoError(err) - replicationCfg := sc.ReplicationConfig{} - re.NoError(json.Unmarshal(output, &replicationCfg)) testutil.Eventually(re, func() bool { // wait for the config to be synced to the scheduling server + output, err := pdctl.ExecuteCommand(cmd, args...) + re.NoError(err) + replicationCfg := sc.ReplicationConfig{} + re.NoError(json.Unmarshal(output, &replicationCfg)) return replicationCfg.MaxReplicas == expect }) } checkLocationLabels := func(expect int) { args := []string{"-u", pdAddr, "config", "show", "replication"} - output, err := pdctl.ExecuteCommand(cmd, args...) - re.NoError(err) - replicationCfg := sc.ReplicationConfig{} - re.NoError(json.Unmarshal(output, &replicationCfg)) testutil.Eventually(re, func() bool { // wait for the config to be synced to the scheduling server + output, err := pdctl.ExecuteCommand(cmd, args...) + re.NoError(err) + replicationCfg := sc.ReplicationConfig{} + re.NoError(json.Unmarshal(output, &replicationCfg)) return len(replicationCfg.LocationLabels) == expect }) } checkIsolationLevel := func(expect string) { args := []string{"-u", pdAddr, "config", "show", "replication"} - output, err := pdctl.ExecuteCommand(cmd, args...) - re.NoError(err) - replicationCfg := sc.ReplicationConfig{} - re.NoError(json.Unmarshal(output, &replicationCfg)) testutil.Eventually(re, func() bool { // wait for the config to be synced to the scheduling server + output, err := pdctl.ExecuteCommand(cmd, args...) + re.NoError(err) + replicationCfg := sc.ReplicationConfig{} + re.NoError(json.Unmarshal(output, &replicationCfg)) return replicationCfg.IsolationLevel == expect }) } checkRuleCount := func(expect int) { args := []string{"-u", pdAddr, "config", "placement-rules", "show", "--group", placement.DefaultGroupID, "--id", placement.DefaultRuleID} - output, err := pdctl.ExecuteCommand(cmd, args...) - re.NoError(err) - rule := placement.Rule{} - re.NoError(json.Unmarshal(output, &rule)) testutil.Eventually(re, func() bool { // wait for the config to be synced to the scheduling server + output, err := pdctl.ExecuteCommand(cmd, args...) + re.NoError(err) + rule := placement.Rule{} + re.NoError(json.Unmarshal(output, &rule)) return rule.Count == expect }) } checkRuleLocationLabels := func(expect int) { args := []string{"-u", pdAddr, "config", "placement-rules", "show", "--group", placement.DefaultGroupID, "--id", placement.DefaultRuleID} - output, err := pdctl.ExecuteCommand(cmd, args...) - re.NoError(err) - rule := placement.Rule{} - re.NoError(json.Unmarshal(output, &rule)) testutil.Eventually(re, func() bool { // wait for the config to be synced to the scheduling server + output, err := pdctl.ExecuteCommand(cmd, args...) + re.NoError(err) + rule := placement.Rule{} + re.NoError(json.Unmarshal(output, &rule)) return len(rule.LocationLabels) == expect }) } checkRuleIsolationLevel := func(expect string) { args := []string{"-u", pdAddr, "config", "placement-rules", "show", "--group", placement.DefaultGroupID, "--id", placement.DefaultRuleID} - output, err := pdctl.ExecuteCommand(cmd, args...) - re.NoError(err) - rule := placement.Rule{} - re.NoError(json.Unmarshal(output, &rule)) testutil.Eventually(re, func() bool { // wait for the config to be synced to the scheduling server + output, err := pdctl.ExecuteCommand(cmd, args...) + re.NoError(err) + rule := placement.Rule{} + re.NoError(json.Unmarshal(output, &rule)) return rule.IsolationLevel == expect }) } From a6e855eef6744adfac232833769219be4f806756 Mon Sep 17 00:00:00 2001 From: Yongbo Jiang Date: Tue, 28 Nov 2023 14:40:16 +0800 Subject: [PATCH 056/137] *: remove improper method in ServiceDiscovery (#7456) ref tikv/pd#4399 Signed-off-by: Cabinfever_B Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- client/pd_service_discovery.go | 4 +--- client/tso_service_discovery.go | 17 +---------------- pkg/utils/grpcutil/grpcutil.go | 1 + server/config/persist_options.go | 1 + 4 files changed, 4 insertions(+), 19 deletions(-) diff --git a/client/pd_service_discovery.go b/client/pd_service_discovery.go index e96093f598d..98ddd611326 100644 --- a/client/pd_service_discovery.go +++ b/client/pd_service_discovery.go @@ -63,8 +63,6 @@ type ServiceDiscovery interface { GetKeyspaceID() uint32 // GetKeyspaceGroupID returns the ID of the keyspace group GetKeyspaceGroupID() uint32 - // DiscoverServiceURLs discovers the microservice with the specified type and returns the server urls. - DiscoverMicroservice(svcType serviceType) ([]string, error) // GetServiceURLs returns the URLs of the servers providing the service GetServiceURLs() []string // GetServingEndpointClientConn returns the grpc client connection of the serving endpoint @@ -324,7 +322,7 @@ func (c *pdServiceDiscovery) GetKeyspaceGroupID() uint32 { } // DiscoverMicroservice discovers the microservice with the specified type and returns the server urls. -func (c *pdServiceDiscovery) DiscoverMicroservice(svcType serviceType) (urls []string, err error) { +func (c *pdServiceDiscovery) discoverMicroservice(svcType serviceType) (urls []string, err error) { switch svcType { case apiService: urls = c.GetServiceURLs() diff --git a/client/tso_service_discovery.go b/client/tso_service_discovery.go index 92f95129951..5f14c406797 100644 --- a/client/tso_service_discovery.go +++ b/client/tso_service_discovery.go @@ -288,21 +288,6 @@ func (c *tsoServiceDiscovery) GetKeyspaceGroupID() uint32 { return c.keyspaceGroupSD.group.Id } -// DiscoverServiceURLs discovers the microservice with the specified type and returns the server urls. -func (c *tsoServiceDiscovery) DiscoverMicroservice(svcType serviceType) ([]string, error) { - var urls []string - - switch svcType { - case apiService: - case tsoService: - return c.apiSvcDiscovery.DiscoverMicroservice(tsoService) - default: - panic("invalid service type") - } - - return urls, nil -} - // GetServiceURLs returns the URLs of the tso primary/secondary addresses of this keyspace group. // For testing use. It should only be called when the client is closed. func (c *tsoServiceDiscovery) GetServiceURLs() []string { @@ -582,7 +567,7 @@ func (c *tsoServiceDiscovery) getTSOServer(sd ServiceDiscovery) (string, error) ) t := c.tsoServerDiscovery if len(t.addrs) == 0 || t.failureCount == len(t.addrs) { - addrs, err = sd.DiscoverMicroservice(tsoService) + addrs, err = sd.(*pdServiceDiscovery).discoverMicroservice(tsoService) if err != nil { return "", err } diff --git a/pkg/utils/grpcutil/grpcutil.go b/pkg/utils/grpcutil/grpcutil.go index 44d45ff4c70..a001ec4bd03 100644 --- a/pkg/utils/grpcutil/grpcutil.go +++ b/pkg/utils/grpcutil/grpcutil.go @@ -163,6 +163,7 @@ func GetForwardedHost(ctx context.Context) string { md, ok := metadata.FromIncomingContext(ctx) if !ok { log.Debug("failed to get forwarding metadata") + return "" } if t, ok := md[ForwardMetadataKey]; ok { return t[0] diff --git a/server/config/persist_options.go b/server/config/persist_options.go index 42ab91fce17..ae9047c626b 100644 --- a/server/config/persist_options.go +++ b/server/config/persist_options.go @@ -1024,6 +1024,7 @@ func (o *PersistOptions) IsRaftKV2() bool { } // SetRegionBucketEnabled sets if the region bucket is enabled. +// only used for test. func (o *PersistOptions) SetRegionBucketEnabled(enabled bool) { cfg := o.GetStoreConfig().Clone() cfg.SetRegionBucketEnabled(enabled) From 8c8b4d4c78701bfda0dc247caa945684fc3b08f9 Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Wed, 29 Nov 2023 09:17:46 +0800 Subject: [PATCH 057/137] *: make TestConfig stable (#7463) close tikv/pd#7440 Signed-off-by: Ryan Leung Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- pkg/dashboard/adapter/manager.go | 5 +++++ tests/pdctl/config/config_test.go | 3 +++ 2 files changed, 8 insertions(+) diff --git a/pkg/dashboard/adapter/manager.go b/pkg/dashboard/adapter/manager.go index a3691242c8f..293d8ad6549 100644 --- a/pkg/dashboard/adapter/manager.go +++ b/pkg/dashboard/adapter/manager.go @@ -19,6 +19,7 @@ import ( "sync" "time" + "github.com/pingcap/failpoint" "github.com/pingcap/kvproto/pkg/pdpb" "github.com/pingcap/log" "github.com/pingcap/tidb-dashboard/pkg/apiserver" @@ -75,6 +76,10 @@ func (m *Manager) Stop() { func (m *Manager) serviceLoop() { defer logutil.LogPanic() defer m.wg.Done() + // TODO: After we fix the atomic problem of config, we can remove this failpoint. + failpoint.Inject("skipDashboardLoop", func() { + failpoint.Return() + }) ticker := time.NewTicker(CheckInterval) defer ticker.Stop() diff --git a/tests/pdctl/config/config_test.go b/tests/pdctl/config/config_test.go index 3f82be44399..3b3310185b1 100644 --- a/tests/pdctl/config/config_test.go +++ b/tests/pdctl/config/config_test.go @@ -24,6 +24,7 @@ import ( "time" "github.com/coreos/go-semver/semver" + "github.com/pingcap/failpoint" "github.com/pingcap/kvproto/pkg/metapb" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" @@ -60,8 +61,10 @@ func TestConfigTestSuite(t *testing.T) { } func (suite *configTestSuite) TestConfig() { + suite.NoError(failpoint.Enable("github.com/tikv/pd/pkg/dashboard/adapter/skipDashboardLoop", `return(true)`)) env := tests.NewSchedulingTestEnvironment(suite.T()) env.RunTestInTwoModes(suite.checkConfig) + suite.NoError(failpoint.Disable("github.com/tikv/pd/pkg/dashboard/adapter/skipDashboardLoop")) } func (suite *configTestSuite) checkConfig(cluster *tests.TestCluster) { From 4356aebb55f453cf91116a22461f4ab375c9ce11 Mon Sep 17 00:00:00 2001 From: David <8039876+AmoebaProtozoa@users.noreply.github.com> Date: Wed, 29 Nov 2023 11:43:17 +0800 Subject: [PATCH 058/137] pd-ctl: retry when keyspace group manager not initialized (#7442) close tikv/pd#7441 --- server/apiv2/handlers/keyspace.go | 2 +- server/apiv2/handlers/tso_keyspace_group.go | 25 ++++++++-------- tests/pdctl/keyspace/keyspace_test.go | 29 +++++++++++++++++++ .../pd-ctl/pdctl/command/keyspace_command.go | 28 ++++++++++++++---- 4 files changed, 65 insertions(+), 19 deletions(-) diff --git a/server/apiv2/handlers/keyspace.go b/server/apiv2/handlers/keyspace.go index 9602cc863ef..c2802bb939d 100644 --- a/server/apiv2/handlers/keyspace.go +++ b/server/apiv2/handlers/keyspace.go @@ -113,7 +113,7 @@ func LoadKeyspace(c *gin.Context) { if value, ok := c.GetQuery("force_refresh_group_id"); ok && value == "true" { groupManager := svr.GetKeyspaceGroupManager() if groupManager == nil { - c.AbortWithStatusJSON(http.StatusInternalServerError, managerUninitializedErr) + c.AbortWithStatusJSON(http.StatusInternalServerError, GroupManagerUninitializedErr) return } // keyspace has been checked in LoadKeyspace, so no need to check again. diff --git a/server/apiv2/handlers/tso_keyspace_group.go b/server/apiv2/handlers/tso_keyspace_group.go index a580b21f705..a9f042687f6 100644 --- a/server/apiv2/handlers/tso_keyspace_group.go +++ b/server/apiv2/handlers/tso_keyspace_group.go @@ -30,7 +30,8 @@ import ( "github.com/tikv/pd/server/apiv2/middlewares" ) -const groupManagerUninitializedErr = "keyspace group manager is not initialized" +// GroupManagerUninitializedErr is the error message for uninitialized keyspace group manager. +const GroupManagerUninitializedErr = "keyspace group manager is not initialized" // RegisterTSOKeyspaceGroup registers keyspace group handlers to the server. func RegisterTSOKeyspaceGroup(r *gin.RouterGroup) { @@ -78,7 +79,7 @@ func CreateKeyspaceGroups(c *gin.Context) { svr := c.MustGet(middlewares.ServerContextKey).(*server.Server) manager := svr.GetKeyspaceGroupManager() if manager == nil { - c.AbortWithStatusJSON(http.StatusInternalServerError, groupManagerUninitializedErr) + c.AbortWithStatusJSON(http.StatusInternalServerError, GroupManagerUninitializedErr) return } err = manager.CreateKeyspaceGroups(createParams.KeyspaceGroups) @@ -101,7 +102,7 @@ func GetKeyspaceGroups(c *gin.Context) { svr := c.MustGet(middlewares.ServerContextKey).(*server.Server) manager := svr.GetKeyspaceGroupManager() if manager == nil { - c.AbortWithStatusJSON(http.StatusInternalServerError, groupManagerUninitializedErr) + c.AbortWithStatusJSON(http.StatusInternalServerError, GroupManagerUninitializedErr) return } keyspaceGroups, err := manager.GetKeyspaceGroups(scanStart, scanLimit) @@ -152,7 +153,7 @@ func GetKeyspaceGroupByID(c *gin.Context) { svr := c.MustGet(middlewares.ServerContextKey).(*server.Server) manager := svr.GetKeyspaceGroupManager() if manager == nil { - c.AbortWithStatusJSON(http.StatusInternalServerError, groupManagerUninitializedErr) + c.AbortWithStatusJSON(http.StatusInternalServerError, GroupManagerUninitializedErr) return } @@ -189,7 +190,7 @@ func DeleteKeyspaceGroupByID(c *gin.Context) { svr := c.MustGet(middlewares.ServerContextKey).(*server.Server) manager := svr.GetKeyspaceGroupManager() if manager == nil { - c.AbortWithStatusJSON(http.StatusInternalServerError, groupManagerUninitializedErr) + c.AbortWithStatusJSON(http.StatusInternalServerError, GroupManagerUninitializedErr) return } kg, err := manager.DeleteKeyspaceGroupByID(id) @@ -250,7 +251,7 @@ func SplitKeyspaceGroupByID(c *gin.Context) { } groupManager := svr.GetKeyspaceGroupManager() if groupManager == nil { - c.AbortWithStatusJSON(http.StatusInternalServerError, groupManagerUninitializedErr) + c.AbortWithStatusJSON(http.StatusInternalServerError, GroupManagerUninitializedErr) return } @@ -289,7 +290,7 @@ func FinishSplitKeyspaceByID(c *gin.Context) { svr := c.MustGet(middlewares.ServerContextKey).(*server.Server) manager := svr.GetKeyspaceGroupManager() if manager == nil { - c.AbortWithStatusJSON(http.StatusInternalServerError, groupManagerUninitializedErr) + c.AbortWithStatusJSON(http.StatusInternalServerError, GroupManagerUninitializedErr) return } err = manager.FinishSplitKeyspaceByID(id) @@ -337,7 +338,7 @@ func MergeKeyspaceGroups(c *gin.Context) { svr := c.MustGet(middlewares.ServerContextKey).(*server.Server) groupManager := svr.GetKeyspaceGroupManager() if groupManager == nil { - c.AbortWithStatusJSON(http.StatusInternalServerError, groupManagerUninitializedErr) + c.AbortWithStatusJSON(http.StatusInternalServerError, GroupManagerUninitializedErr) return } // Merge keyspace group. @@ -364,7 +365,7 @@ func FinishMergeKeyspaceByID(c *gin.Context) { svr := c.MustGet(middlewares.ServerContextKey).(*server.Server) manager := svr.GetKeyspaceGroupManager() if manager == nil { - c.AbortWithStatusJSON(http.StatusInternalServerError, groupManagerUninitializedErr) + c.AbortWithStatusJSON(http.StatusInternalServerError, GroupManagerUninitializedErr) return } err = manager.FinishMergeKeyspaceByID(id) @@ -390,7 +391,7 @@ func AllocNodesForKeyspaceGroup(c *gin.Context) { svr := c.MustGet(middlewares.ServerContextKey).(*server.Server) manager := svr.GetKeyspaceGroupManager() if manager == nil { - c.AbortWithStatusJSON(http.StatusInternalServerError, groupManagerUninitializedErr) + c.AbortWithStatusJSON(http.StatusInternalServerError, GroupManagerUninitializedErr) return } allocParams := &AllocNodesForKeyspaceGroupParams{} @@ -437,7 +438,7 @@ func SetNodesForKeyspaceGroup(c *gin.Context) { svr := c.MustGet(middlewares.ServerContextKey).(*server.Server) manager := svr.GetKeyspaceGroupManager() if manager == nil { - c.AbortWithStatusJSON(http.StatusInternalServerError, groupManagerUninitializedErr) + c.AbortWithStatusJSON(http.StatusInternalServerError, GroupManagerUninitializedErr) return } setParams := &SetNodesForKeyspaceGroupParams{} @@ -493,7 +494,7 @@ func SetPriorityForKeyspaceGroup(c *gin.Context) { svr := c.MustGet(middlewares.ServerContextKey).(*server.Server) manager := svr.GetKeyspaceGroupManager() if manager == nil { - c.AbortWithStatusJSON(http.StatusInternalServerError, groupManagerUninitializedErr) + c.AbortWithStatusJSON(http.StatusInternalServerError, GroupManagerUninitializedErr) return } setParams := &SetPriorityForKeyspaceGroupParams{} diff --git a/tests/pdctl/keyspace/keyspace_test.go b/tests/pdctl/keyspace/keyspace_test.go index 805a30e6f18..3ff755fe601 100644 --- a/tests/pdctl/keyspace/keyspace_test.go +++ b/tests/pdctl/keyspace/keyspace_test.go @@ -105,6 +105,35 @@ func TestKeyspace(t *testing.T) { re.NoError(failpoint.Disable("github.com/tikv/pd/server/delayStartServerLoop")) } +// Show command should auto retry without refresh_group_id if keyspace group manager not initialized. +// See issue: #7441 +func TestKeyspaceGroupUninitialized(t *testing.T) { + re := require.New(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + re.NoError(failpoint.Enable("github.com/tikv/pd/server/delayStartServerLoop", `return(true)`)) + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/keyspace/skipSplitRegion", "return(true)")) + tc, err := tests.NewTestCluster(ctx, 1) + re.NoError(err) + re.NoError(tc.RunInitialServers()) + tc.WaitLeader() + re.NoError(tc.GetLeaderServer().BootstrapCluster()) + pdAddr := tc.GetConfig().GetClientURL() + + keyspaceName := "DEFAULT" + keyspaceID := uint32(0) + args := []string{"-u", pdAddr, "keyspace", "show", "name", keyspaceName} + output, err := pdctl.ExecuteCommand(pdctlCmd.GetRootCmd(), args...) + re.NoError(err) + var meta api.KeyspaceMeta + re.NoError(json.Unmarshal(output, &meta)) + re.Equal(keyspaceName, meta.GetName()) + re.Equal(keyspaceID, meta.GetId()) + + re.NoError(failpoint.Disable("github.com/tikv/pd/server/delayStartServerLoop")) + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/keyspace/skipSplitRegion")) +} + type keyspaceTestSuite struct { suite.Suite ctx context.Context diff --git a/tools/pd-ctl/pdctl/command/keyspace_command.go b/tools/pd-ctl/pdctl/command/keyspace_command.go index 7c0d3d78bf6..93a99abc39f 100644 --- a/tools/pd-ctl/pdctl/command/keyspace_command.go +++ b/tools/pd-ctl/pdctl/command/keyspace_command.go @@ -28,11 +28,12 @@ import ( const ( keyspacePrefix = "pd/api/v2/keyspaces" // flags - nmConfig = "config" - nmLimit = "limit" - nmPageToken = "page_token" - nmRemove = "remove" - nmUpdate = "update" + nmConfig = "config" + nmLimit = "limit" + nmPageToken = "page_token" + nmRemove = "remove" + nmUpdate = "update" + nmForceRefreshGroupID = "force_refresh_group_id" ) // NewKeyspaceCommand returns a keyspace subcommand of rootCmd. @@ -64,6 +65,7 @@ func newShowKeyspaceCommand() *cobra.Command { Short: "show keyspace metadata specified by keyspace name", Run: showKeyspaceNameCommandFunc, } + showByName.Flags().Bool(nmForceRefreshGroupID, true, "force refresh keyspace group id") r.AddCommand(showByID) r.AddCommand(showByName) return r @@ -87,7 +89,21 @@ func showKeyspaceNameCommandFunc(cmd *cobra.Command, args []string) { cmd.Usage() return } - resp, err := doRequest(cmd, fmt.Sprintf("%s/%s?force_refresh_group_id=true", keyspacePrefix, args[0]), http.MethodGet, http.Header{}) + refreshGroupID, err := cmd.Flags().GetBool(nmForceRefreshGroupID) + if err != nil { + cmd.PrintErrln("Failed to parse flag: ", err) + return + } + url := fmt.Sprintf("%s/%s", keyspacePrefix, args[0]) + if refreshGroupID { + url += "?force_refresh_group_id=true" + } + resp, err := doRequest(cmd, url, http.MethodGet, http.Header{}) + // Retry without the force_refresh_group_id if the keyspace group manager is not initialized. + // This can happen when PD is not running in API mode. + if err != nil && refreshGroupID && strings.Contains(err.Error(), handlers.GroupManagerUninitializedErr) { + resp, err = doRequest(cmd, fmt.Sprintf("%s/%s", keyspacePrefix, args[0]), http.MethodGet, http.Header{}) + } if err != nil { cmd.PrintErrln("Failed to get the keyspace information: ", err) return From 7fbe69d1ec7ffd3f4d4f4b1b8cd2f78f9f0d689a Mon Sep 17 00:00:00 2001 From: JmPotato Date: Wed, 29 Nov 2023 13:17:47 +0800 Subject: [PATCH 059/137] client/http: add GetHistoryHotRegions interface and more tests (#7453) ref tikv/pd#7300 - Add `GetHistoryHotRegions` interface. - Add more tests. Signed-off-by: JmPotato --- client/http/client.go | 61 +++++++++++++- client/http/types.go | 42 ++++++++++ tests/integrations/client/http_client_test.go | 79 +++++++++++++++++-- 3 files changed, 173 insertions(+), 9 deletions(-) diff --git a/client/http/client.go b/client/http/client.go index cd7be205c12..d15693e11d4 100644 --- a/client/http/client.go +++ b/client/http/client.go @@ -49,8 +49,12 @@ type Client interface { GetRegionsByStoreID(context.Context, uint64) (*RegionsInfo, error) GetHotReadRegions(context.Context) (*StoreHotPeersInfos, error) GetHotWriteRegions(context.Context) (*StoreHotPeersInfos, error) + GetHistoryHotRegions(context.Context, *HistoryHotRegionsRequest) (*HistoryHotRegions, error) GetRegionStatusByKeyRange(context.Context, *KeyRange) (*RegionStats, error) GetStores(context.Context) (*StoresInfo, error) + /* Config-related interfaces */ + GetScheduleConfig(context.Context) (map[string]interface{}, error) + SetScheduleConfig(context.Context, map[string]interface{}) error /* Rule-related interfaces */ GetAllPlacementRuleBundles(context.Context) ([]*GroupBundle, error) GetPlacementRuleBundleByGroup(context.Context, string) (*GroupBundle, error) @@ -191,12 +195,23 @@ func (c *client) execDuration(name string, duration time.Duration) { c.executionDuration.WithLabelValues(name).Observe(duration.Seconds()) } +// HeaderOption configures the HTTP header. +type HeaderOption func(header http.Header) + +// WithAllowFollowerHandle sets the header field to allow a PD follower to handle this request. +func WithAllowFollowerHandle() HeaderOption { + return func(header http.Header) { + header.Set("PD-Allow-Follower-Handle", "true") + } +} + // At present, we will use the retry strategy of polling by default to keep // it consistent with the current implementation of some clients (e.g. TiDB). func (c *client) requestWithRetry( ctx context.Context, name, uri, method string, body io.Reader, res interface{}, + headerOpts ...HeaderOption, ) error { var ( err error @@ -204,7 +219,7 @@ func (c *client) requestWithRetry( ) for idx := 0; idx < len(c.pdAddrs); idx++ { addr = c.pdAddrs[idx] - err = c.request(ctx, name, fmt.Sprintf("%s%s", addr, uri), method, body, res) + err = c.request(ctx, name, fmt.Sprintf("%s%s", addr, uri), method, body, res, headerOpts...) if err == nil { break } @@ -218,6 +233,7 @@ func (c *client) request( ctx context.Context, name, url, method string, body io.Reader, res interface{}, + headerOpts ...HeaderOption, ) error { logFields := []zap.Field{ zap.String("name", name), @@ -229,6 +245,9 @@ func (c *client) request( log.Error("[pd] create http request failed", append(logFields, zap.Error(err))...) return errors.Trace(err) } + for _, opt := range headerOpts { + opt(req.Header) + } start := time.Now() resp, err := c.cli.Do(req) if err != nil { @@ -361,6 +380,23 @@ func (c *client) GetHotWriteRegions(ctx context.Context) (*StoreHotPeersInfos, e return &hotWriteRegions, nil } +// GetHistoryHotRegions gets the history hot region statistics info. +func (c *client) GetHistoryHotRegions(ctx context.Context, req *HistoryHotRegionsRequest) (*HistoryHotRegions, error) { + reqJSON, err := json.Marshal(req) + if err != nil { + return nil, errors.Trace(err) + } + var historyHotRegions HistoryHotRegions + err = c.requestWithRetry(ctx, + "GetHistoryHotRegions", HotHistory, + http.MethodGet, bytes.NewBuffer(reqJSON), &historyHotRegions, + WithAllowFollowerHandle()) + if err != nil { + return nil, err + } + return &historyHotRegions, nil +} + // GetRegionStatusByKeyRange gets the region status by key range. // The keys in the key range should be encoded in the UTF-8 bytes format. func (c *client) GetRegionStatusByKeyRange(ctx context.Context, keyRange *KeyRange) (*RegionStats, error) { @@ -375,6 +411,29 @@ func (c *client) GetRegionStatusByKeyRange(ctx context.Context, keyRange *KeyRan return ®ionStats, nil } +// GetScheduleConfig gets the schedule configurations. +func (c *client) GetScheduleConfig(ctx context.Context) (map[string]interface{}, error) { + var config map[string]interface{} + err := c.requestWithRetry(ctx, + "GetScheduleConfig", ScheduleConfig, + http.MethodGet, http.NoBody, &config) + if err != nil { + return nil, err + } + return config, nil +} + +// SetScheduleConfig sets the schedule configurations. +func (c *client) SetScheduleConfig(ctx context.Context, config map[string]interface{}) error { + configJSON, err := json.Marshal(config) + if err != nil { + return errors.Trace(err) + } + return c.requestWithRetry(ctx, + "SetScheduleConfig", ScheduleConfig, + http.MethodPost, bytes.NewBuffer(configJSON), nil) +} + // GetStores gets the stores info. func (c *client) GetStores(ctx context.Context) (*StoresInfo, error) { var stores StoresInfo diff --git a/client/http/types.go b/client/http/types.go index df448e7e20d..4e99d911e0b 100644 --- a/client/http/types.go +++ b/client/http/types.go @@ -19,6 +19,8 @@ import ( "encoding/json" "net/url" "time" + + "github.com/pingcap/kvproto/pkg/encryptionpb" ) // KeyRange defines a range of keys in bytes. @@ -166,6 +168,46 @@ type HotPeerStatShow struct { LastUpdateTime time.Time `json:"last_update_time,omitempty"` } +// HistoryHotRegionsRequest wrap the request conditions. +type HistoryHotRegionsRequest struct { + StartTime int64 `json:"start_time,omitempty"` + EndTime int64 `json:"end_time,omitempty"` + RegionIDs []uint64 `json:"region_ids,omitempty"` + StoreIDs []uint64 `json:"store_ids,omitempty"` + PeerIDs []uint64 `json:"peer_ids,omitempty"` + IsLearners []bool `json:"is_learners,omitempty"` + IsLeaders []bool `json:"is_leaders,omitempty"` + HotRegionTypes []string `json:"hot_region_type,omitempty"` +} + +// HistoryHotRegions wraps historyHotRegion +type HistoryHotRegions struct { + HistoryHotRegion []*HistoryHotRegion `json:"history_hot_region"` +} + +// HistoryHotRegion wraps hot region info +// it is storage format of hot_region_storage +type HistoryHotRegion struct { + UpdateTime int64 `json:"update_time"` + RegionID uint64 `json:"region_id"` + PeerID uint64 `json:"peer_id"` + StoreID uint64 `json:"store_id"` + IsLeader bool `json:"is_leader"` + IsLearner bool `json:"is_learner"` + HotRegionType string `json:"hot_region_type"` + HotDegree int64 `json:"hot_degree"` + FlowBytes float64 `json:"flow_bytes"` + KeyRate float64 `json:"key_rate"` + QueryRate float64 `json:"query_rate"` + StartKey string `json:"start_key"` + EndKey string `json:"end_key"` + // Encryption metadata for start_key and end_key. encryption_meta.iv is IV for start_key. + // IV for end_key is calculated from (encryption_meta.iv + len(start_key)). + // The field is only used by PD and should be ignored otherwise. + // If encryption_meta is empty (i.e. nil), it means start_key and end_key are unencrypted. + EncryptionMeta *encryptionpb.EncryptionMeta `json:"encryption_meta,omitempty"` +} + // StoresInfo represents the information of all TiKV/TiFlash stores. type StoresInfo struct { Count int `json:"count"` diff --git a/tests/integrations/client/http_client_test.go b/tests/integrations/client/http_client_test.go index 3a52e91e1f8..a007b893187 100644 --- a/tests/integrations/client/http_client_test.go +++ b/tests/integrations/client/http_client_test.go @@ -55,8 +55,16 @@ func (suite *httpClientTestSuite) SetupSuite() { re.NoError(err) leader := suite.cluster.WaitLeader() re.NotEmpty(leader) - err = suite.cluster.GetLeaderServer().BootstrapCluster() + leaderServer := suite.cluster.GetLeaderServer() + err = leaderServer.BootstrapCluster() re.NoError(err) + for _, region := range []*core.RegionInfo{ + core.NewTestRegionInfo(10, 1, []byte("a1"), []byte("a2")), + core.NewTestRegionInfo(11, 1, []byte("a2"), []byte("a3")), + } { + err := leaderServer.GetRaftCluster().HandleRegionHeartbeat(region) + re.NoError(err) + } var ( testServers = suite.cluster.GetServers() endpoints = make([]string, 0, len(testServers)) @@ -73,6 +81,53 @@ func (suite *httpClientTestSuite) TearDownSuite() { suite.cluster.Destroy() } +func (suite *httpClientTestSuite) TestMeta() { + re := suite.Require() + region, err := suite.client.GetRegionByID(suite.ctx, 10) + re.NoError(err) + re.Equal(int64(10), region.ID) + re.Equal(core.HexRegionKeyStr([]byte("a1")), region.StartKey) + re.Equal(core.HexRegionKeyStr([]byte("a2")), region.EndKey) + region, err = suite.client.GetRegionByKey(suite.ctx, []byte("a2")) + re.NoError(err) + re.Equal(int64(11), region.ID) + re.Equal(core.HexRegionKeyStr([]byte("a2")), region.StartKey) + re.Equal(core.HexRegionKeyStr([]byte("a3")), region.EndKey) + regions, err := suite.client.GetRegions(suite.ctx) + re.NoError(err) + re.Equal(int64(2), regions.Count) + re.Len(regions.Regions, 2) + regions, err = suite.client.GetRegionsByKeyRange(suite.ctx, pd.NewKeyRange([]byte("a1"), []byte("a3")), -1) + re.NoError(err) + re.Equal(int64(2), regions.Count) + re.Len(regions.Regions, 2) + regions, err = suite.client.GetRegionsByStoreID(suite.ctx, 1) + re.NoError(err) + re.Equal(int64(2), regions.Count) + re.Len(regions.Regions, 2) + regionStats, err := suite.client.GetRegionStatusByKeyRange(suite.ctx, pd.NewKeyRange([]byte("a1"), []byte("a3"))) + re.NoError(err) + re.Equal(2, regionStats.Count) + hotReadRegions, err := suite.client.GetHotReadRegions(suite.ctx) + re.NoError(err) + re.Len(hotReadRegions.AsPeer, 1) + re.Len(hotReadRegions.AsLeader, 1) + hotWriteRegions, err := suite.client.GetHotWriteRegions(suite.ctx) + re.NoError(err) + re.Len(hotWriteRegions.AsPeer, 1) + re.Len(hotWriteRegions.AsLeader, 1) + historyHorRegions, err := suite.client.GetHistoryHotRegions(suite.ctx, &pd.HistoryHotRegionsRequest{ + StartTime: 0, + EndTime: time.Now().AddDate(0, 0, 1).UnixNano() / int64(time.Millisecond), + }) + re.NoError(err) + re.Len(historyHorRegions.HistoryHotRegion, 0) + store, err := suite.client.GetStores(suite.ctx) + re.NoError(err) + re.Equal(1, store.Count) + re.Len(store.Stores, 1) +} + func (suite *httpClientTestSuite) TestGetMinResolvedTSByStoresIDs() { re := suite.Require() testMinResolvedTS := tsoutil.TimeToTS(time.Now()) @@ -271,13 +326,6 @@ func (suite *httpClientTestSuite) TestRegionLabel() { func (suite *httpClientTestSuite) TestAccelerateSchedule() { re := suite.Require() raftCluster := suite.cluster.GetLeaderServer().GetRaftCluster() - for _, region := range []*core.RegionInfo{ - core.NewTestRegionInfo(10, 1, []byte("a1"), []byte("a2")), - core.NewTestRegionInfo(11, 1, []byte("a2"), []byte("a3")), - } { - err := raftCluster.HandleRegionHeartbeat(region) - re.NoError(err) - } suspectRegions := raftCluster.GetSuspectRegions() re.Len(suspectRegions, 0) err := suite.client.AccelerateSchedule(suite.ctx, pd.NewKeyRange([]byte("a1"), []byte("a2"))) @@ -295,3 +343,18 @@ func (suite *httpClientTestSuite) TestAccelerateSchedule() { suspectRegions = raftCluster.GetSuspectRegions() re.Len(suspectRegions, 2) } + +func (suite *httpClientTestSuite) TestScheduleConfig() { + re := suite.Require() + config, err := suite.client.GetScheduleConfig(suite.ctx) + re.NoError(err) + re.Equal(float64(4), config["leader-schedule-limit"]) + re.Equal(float64(2048), config["region-schedule-limit"]) + config["leader-schedule-limit"] = float64(8) + err = suite.client.SetScheduleConfig(suite.ctx, config) + re.NoError(err) + config, err = suite.client.GetScheduleConfig(suite.ctx) + re.NoError(err) + re.Equal(float64(8), config["leader-schedule-limit"]) + re.Equal(float64(2048), config["region-schedule-limit"]) +} From 90245e376b156a6aa2c41c6063f093bd63561dc4 Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Wed, 29 Nov 2023 14:55:20 +0800 Subject: [PATCH 060/137] mcs: fix unstable panic in tso test (#7433) close tikv/pd#7429 Signed-off-by: lhy1024 --- pkg/tso/keyspace_group_manager_test.go | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/pkg/tso/keyspace_group_manager_test.go b/pkg/tso/keyspace_group_manager_test.go index c20abfc5f79..0c1b017d7aa 100644 --- a/pkg/tso/keyspace_group_manager_test.go +++ b/pkg/tso/keyspace_group_manager_test.go @@ -1224,14 +1224,12 @@ func waitForPrimariesServing( re *require.Assertions, mgrs []*KeyspaceGroupManager, ids []uint32, ) { testutil.Eventually(re, func() bool { - for i := 0; i < 100; i++ { - for j, id := range ids { - if member, err := mgrs[j].GetElectionMember(id, id); err != nil || !member.IsLeader() { - return false - } - if _, _, err := mgrs[j].HandleTSORequest(mgrs[j].ctx, id, id, GlobalDCLocation, 1); err != nil { - return false - } + for j, id := range ids { + if member, err := mgrs[j].GetElectionMember(id, id); err != nil || member == nil || !member.IsLeader() { + return false + } + if _, _, err := mgrs[j].HandleTSORequest(mgrs[j].ctx, id, id, GlobalDCLocation, 1); err != nil { + return false } } return true From 1934450e9aa04c7aaf291110bf23a5995f9c94fd Mon Sep 17 00:00:00 2001 From: Yongbo Jiang Date: Wed, 29 Nov 2023 15:38:48 +0800 Subject: [PATCH 061/137] client: add follower option (#7465) ref tikv/pd#7431 Signed-off-by: Cabinfever_B Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- client/client.go | 22 +++++++++++++++++----- client/option.go | 17 +++++++++++++++++ client/option_test.go | 8 ++++++++ 3 files changed, 42 insertions(+), 5 deletions(-) diff --git a/client/client.go b/client/client.go index 2d30d9fb6c4..70de1322488 100644 --- a/client/client.go +++ b/client/client.go @@ -91,7 +91,7 @@ type Client interface { // client should retry later. GetRegion(ctx context.Context, key []byte, opts ...GetRegionOption) (*Region, error) // GetRegionFromMember gets a region from certain members. - GetRegionFromMember(ctx context.Context, key []byte, memberURLs []string) (*Region, error) + GetRegionFromMember(ctx context.Context, key []byte, memberURLs []string, opts ...GetRegionOption) (*Region, error) // GetPrevRegion gets the previous region and its leader Peer of the region where the key is located. GetPrevRegion(ctx context.Context, key []byte, opts ...GetRegionOption) (*Region, error) // GetRegionByID gets a region and its leader Peer from PD by id. @@ -100,7 +100,7 @@ type Client interface { // Limit limits the maximum number of regions returned. // If a region has no leader, corresponding leader will be placed by a peer // with empty value (PeerID is 0). - ScanRegions(ctx context.Context, key, endKey []byte, limit int) ([]*Region, error) + ScanRegions(ctx context.Context, key, endKey []byte, limit int, opts ...GetRegionOption) ([]*Region, error) // GetStore gets a store from PD by store id. // The store may expire later. Caller is responsible for caching and taking care // of store change. @@ -200,7 +200,8 @@ func WithSkipStoreLimit() RegionsOption { // GetRegionOp represents available options when getting regions. type GetRegionOp struct { - needBuckets bool + needBuckets bool + allowFollowerHandle bool } // GetRegionOption configures GetRegionOp. @@ -211,6 +212,11 @@ func WithBuckets() GetRegionOption { return func(op *GetRegionOp) { op.needBuckets = true } } +// WithAllowFollowerHandle means that client can send request to follower and let it handle this request. +func WithAllowFollowerHandle() GetRegionOption { + return func(op *GetRegionOp) { op.allowFollowerHandle = true } +} + // LeaderHealthCheckInterval might be changed in the unit to shorten the testing time. var LeaderHealthCheckInterval = time.Second @@ -701,6 +707,12 @@ func (c *client) UpdateOption(option DynamicOption, value interface{}) error { return errors.New("[pd] invalid value type for EnableTSOFollowerProxy option, it should be bool") } c.option.setEnableTSOFollowerProxy(enable) + case EnableFollowerHandle: + enable, ok := value.(bool) + if !ok { + return errors.New("[pd] invalid value type for EnableFollowerHandle option, it should be bool") + } + c.option.setEnableFollowerHandle(enable) default: return errors.New("[pd] unsupported client option") } @@ -952,7 +964,7 @@ func isNetworkError(code codes.Code) bool { return code == codes.Unavailable || code == codes.DeadlineExceeded } -func (c *client) GetRegionFromMember(ctx context.Context, key []byte, memberURLs []string) (*Region, error) { +func (c *client) GetRegionFromMember(ctx context.Context, key []byte, memberURLs []string, opts ...GetRegionOption) (*Region, error) { if span := opentracing.SpanFromContext(ctx); span != nil { span = opentracing.StartSpan("pdclient.GetRegionFromMember", opentracing.ChildOf(span.Context())) defer span.Finish() @@ -1056,7 +1068,7 @@ func (c *client) GetRegionByID(ctx context.Context, regionID uint64, opts ...Get return handleRegionResponse(resp), nil } -func (c *client) ScanRegions(ctx context.Context, key, endKey []byte, limit int) ([]*Region, error) { +func (c *client) ScanRegions(ctx context.Context, key, endKey []byte, limit int, opts ...GetRegionOption) ([]*Region, error) { if span := opentracing.SpanFromContext(ctx); span != nil { span = opentracing.StartSpan("pdclient.ScanRegions", opentracing.ChildOf(span.Context())) defer span.Finish() diff --git a/client/option.go b/client/option.go index d6a6d61d2f9..2a6c285cfb7 100644 --- a/client/option.go +++ b/client/option.go @@ -28,6 +28,7 @@ const ( maxInitClusterRetries = 100 defaultMaxTSOBatchWaitInterval time.Duration = 0 defaultEnableTSOFollowerProxy = false + defaultEnableFollowerHandle = false ) // DynamicOption is used to distinguish the dynamic option type. @@ -40,6 +41,8 @@ const ( // EnableTSOFollowerProxy is the TSO Follower Proxy option. // It is stored as bool. EnableTSOFollowerProxy + // EnableFollowerHandle is the follower handle option. + EnableFollowerHandle dynamicOptionCount ) @@ -72,6 +75,7 @@ func newOption() *option { co.dynamicOptions[MaxTSOBatchWaitInterval].Store(defaultMaxTSOBatchWaitInterval) co.dynamicOptions[EnableTSOFollowerProxy].Store(defaultEnableTSOFollowerProxy) + co.dynamicOptions[EnableFollowerHandle].Store(defaultEnableFollowerHandle) return co } @@ -88,6 +92,19 @@ func (o *option) setMaxTSOBatchWaitInterval(interval time.Duration) error { return nil } +// setEnableFollowerHandle set the Follower Handle option. +func (o *option) setEnableFollowerHandle(enable bool) { + old := o.getEnableFollowerHandle() + if enable != old { + o.dynamicOptions[EnableFollowerHandle].Store(enable) + } +} + +// getMaxTSOBatchWaitInterval gets the Follower Handle enable option. +func (o *option) getEnableFollowerHandle() bool { + return o.dynamicOptions[EnableFollowerHandle].Load().(bool) +} + // getMaxTSOBatchWaitInterval gets the max TSO batch wait interval option. func (o *option) getMaxTSOBatchWaitInterval() time.Duration { return o.dynamicOptions[MaxTSOBatchWaitInterval].Load().(time.Duration) diff --git a/client/option_test.go b/client/option_test.go index 1b5604f4d19..1a8faf8fcd9 100644 --- a/client/option_test.go +++ b/client/option_test.go @@ -28,6 +28,7 @@ func TestDynamicOptionChange(t *testing.T) { // Check the default value setting. re.Equal(defaultMaxTSOBatchWaitInterval, o.getMaxTSOBatchWaitInterval()) re.Equal(defaultEnableTSOFollowerProxy, o.getEnableTSOFollowerProxy()) + re.Equal(defaultEnableFollowerHandle, o.getEnableFollowerHandle()) // Check the invalid value setting. re.NotNil(o.setMaxTSOBatchWaitInterval(time.Second)) @@ -55,4 +56,11 @@ func TestDynamicOptionChange(t *testing.T) { close(o.enableTSOFollowerProxyCh) // Setting the same value should not notify the channel. o.setEnableTSOFollowerProxy(expectBool) + + expectBool = true + o.setEnableFollowerHandle(expectBool) + re.Equal(expectBool, o.getEnableFollowerHandle()) + expectBool = false + o.setEnableFollowerHandle(expectBool) + re.Equal(expectBool, o.getEnableFollowerHandle()) } From 54bf70e45e67fe4fb576409740270fe49af98ad6 Mon Sep 17 00:00:00 2001 From: Yongbo Jiang Date: Wed, 29 Nov 2023 17:52:19 +0800 Subject: [PATCH 062/137] client: update the leader even if the connection creation fails (#7443) close tikv/pd#7416 Signed-off-by: Cabinfever_B Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- client/client.go | 4 +- client/grpcutil/grpcutil.go | 8 ++ client/pd_service_discovery.go | 1 - client/tso_client.go | 3 +- client/tso_dispatcher.go | 82 +++++++++--------- tests/integrations/client/client_test.go | 101 ++++++++++++++++++++++- 6 files changed, 156 insertions(+), 43 deletions(-) diff --git a/client/client.go b/client/client.go index 70de1322488..f34f5897013 100644 --- a/client/client.go +++ b/client/client.go @@ -744,16 +744,18 @@ func (c *client) checkLeaderHealth(ctx context.Context) { if client := c.pdSvcDiscovery.GetServingEndpointClientConn(); client != nil { healthCli := healthpb.NewHealthClient(client) resp, err := healthCli.Check(ctx, &healthpb.HealthCheckRequest{Service: ""}) - rpcErr, ok := status.FromError(err) failpoint.Inject("unreachableNetwork1", func() { resp = nil err = status.New(codes.Unavailable, "unavailable").Err() }) + rpcErr, ok := status.FromError(err) if (ok && isNetworkError(rpcErr.Code())) || resp.GetStatus() != healthpb.HealthCheckResponse_SERVING { atomic.StoreInt32(&(c.leaderNetworkFailure), int32(1)) } else { atomic.StoreInt32(&(c.leaderNetworkFailure), int32(0)) } + } else { + atomic.StoreInt32(&(c.leaderNetworkFailure), int32(1)) } } diff --git a/client/grpcutil/grpcutil.go b/client/grpcutil/grpcutil.go index 125f1125721..fe149e76ecc 100644 --- a/client/grpcutil/grpcutil.go +++ b/client/grpcutil/grpcutil.go @@ -21,6 +21,8 @@ import ( "sync" "time" + "github.com/pingcap/errors" + "github.com/pingcap/failpoint" "github.com/pingcap/log" "github.com/tikv/pd/client/errs" "github.com/tikv/pd/client/tlsutil" @@ -88,6 +90,12 @@ func GetOrCreateGRPCConn(ctx context.Context, clientConns *sync.Map, addr string dCtx, cancel := context.WithTimeout(ctx, dialTimeout) defer cancel() cc, err := GetClientConn(dCtx, addr, tlsConfig, opt...) + failpoint.Inject("unreachableNetwork2", func(val failpoint.Value) { + if val, ok := val.(string); ok && val == addr { + cc = nil + err = errors.Errorf("unreachable network") + } + }) if err != nil { return nil, err } diff --git a/client/pd_service_discovery.go b/client/pd_service_discovery.go index 98ddd611326..b75276adbe9 100644 --- a/client/pd_service_discovery.go +++ b/client/pd_service_discovery.go @@ -614,7 +614,6 @@ func (c *pdServiceDiscovery) switchLeader(addrs []string) error { if _, err := c.GetOrCreateGRPCConn(addr); err != nil { log.Warn("[pd] failed to connect leader", zap.String("leader", addr), errs.ZapError(err)) - return err } // Set PD leader and Global TSO Allocator (which is also the PD leader) c.leader.Store(addr) diff --git a/client/tso_client.go b/client/tso_client.go index 35d9388c72b..fc38ee8e5ba 100644 --- a/client/tso_client.go +++ b/client/tso_client.go @@ -171,9 +171,10 @@ func (c *tsoClient) GetTSOAllocatorClientConnByDCLocation(dcLocation string) (*g if !ok { panic(fmt.Sprintf("the allocator leader in %s should exist", dcLocation)) } + // todo: if we support local tso forward, we should get or create client conns. cc, ok := c.svcDiscovery.GetClientConns().Load(url) if !ok { - panic(fmt.Sprintf("the client connection of %s in %s should exist", url, dcLocation)) + return nil, url.(string) } return cc.(*grpc.ClientConn), url.(string) } diff --git a/client/tso_dispatcher.go b/client/tso_dispatcher.go index e4c5bf3c77a..0de4dc3a49e 100644 --- a/client/tso_dispatcher.go +++ b/client/tso_dispatcher.go @@ -254,7 +254,7 @@ func (c *tsoClient) checkAllocator( requestForwarded.WithLabelValues(forwardedHostTrim, addrTrim).Set(0) }() cc, u := c.GetTSOAllocatorClientConnByDCLocation(dc) - healthCli := healthpb.NewHealthClient(cc) + var healthCli healthpb.HealthClient ticker := time.NewTicker(time.Second) defer ticker.Stop() for { @@ -263,20 +263,25 @@ func (c *tsoClient) checkAllocator( log.Info("[tso] the leader of the allocator leader is changed", zap.String("dc", dc), zap.String("origin", url), zap.String("new", u)) return } - healthCtx, healthCancel := context.WithTimeout(dispatcherCtx, c.option.timeout) - resp, err := healthCli.Check(healthCtx, &healthpb.HealthCheckRequest{Service: ""}) - failpoint.Inject("unreachableNetwork", func() { - resp.Status = healthpb.HealthCheckResponse_UNKNOWN - }) - healthCancel() - if err == nil && resp.GetStatus() == healthpb.HealthCheckResponse_SERVING { - // create a stream of the original allocator - cctx, cancel := context.WithCancel(dispatcherCtx) - stream, err := c.tsoStreamBuilderFactory.makeBuilder(cc).build(cctx, cancel, c.option.timeout) - if err == nil && stream != nil { - log.Info("[tso] recover the original tso stream since the network has become normal", zap.String("dc", dc), zap.String("url", url)) - updateAndClear(url, &tsoConnectionContext{url, stream, cctx, cancel}) - return + if healthCli == nil && cc != nil { + healthCli = healthpb.NewHealthClient(cc) + } + if healthCli != nil { + healthCtx, healthCancel := context.WithTimeout(dispatcherCtx, c.option.timeout) + resp, err := healthCli.Check(healthCtx, &healthpb.HealthCheckRequest{Service: ""}) + failpoint.Inject("unreachableNetwork", func() { + resp.Status = healthpb.HealthCheckResponse_UNKNOWN + }) + healthCancel() + if err == nil && resp.GetStatus() == healthpb.HealthCheckResponse_SERVING { + // create a stream of the original allocator + cctx, cancel := context.WithCancel(dispatcherCtx) + stream, err := c.tsoStreamBuilderFactory.makeBuilder(cc).build(cctx, cancel, c.option.timeout) + if err == nil && stream != nil { + log.Info("[tso] recover the original tso stream since the network has become normal", zap.String("dc", dc), zap.String("url", url)) + updateAndClear(url, &tsoConnectionContext{url, stream, cctx, cancel}) + return + } } } select { @@ -285,7 +290,7 @@ func (c *tsoClient) checkAllocator( case <-ticker.C: // To ensure we can get the latest allocator leader // and once the leader is changed, we can exit this function. - _, u = c.GetTSOAllocatorClientConnByDCLocation(dc) + cc, u = c.GetTSOAllocatorClientConnByDCLocation(dc) } } } @@ -597,29 +602,32 @@ func (c *tsoClient) tryConnectToTSO( for i := 0; i < maxRetryTimes; i++ { c.svcDiscovery.ScheduleCheckMemberChanged() cc, url = c.GetTSOAllocatorClientConnByDCLocation(dc) - cctx, cancel := context.WithCancel(dispatcherCtx) - stream, err = c.tsoStreamBuilderFactory.makeBuilder(cc).build(cctx, cancel, c.option.timeout) - failpoint.Inject("unreachableNetwork", func() { - stream = nil - err = status.New(codes.Unavailable, "unavailable").Err() - }) - if stream != nil && err == nil { - updateAndClear(url, &tsoConnectionContext{url, stream, cctx, cancel}) - return nil - } - - if err != nil && c.option.enableForwarding { - // The reason we need to judge if the error code is equal to "Canceled" here is that - // when we create a stream we use a goroutine to manually control the timeout of the connection. - // There is no need to wait for the transport layer timeout which can reduce the time of unavailability. - // But it conflicts with the retry mechanism since we use the error code to decide if it is caused by network error. - // And actually the `Canceled` error can be regarded as a kind of network error in some way. - if rpcErr, ok := status.FromError(err); ok && (isNetworkError(rpcErr.Code()) || rpcErr.Code() == codes.Canceled) { - networkErrNum++ + if cc != nil { + cctx, cancel := context.WithCancel(dispatcherCtx) + stream, err = c.tsoStreamBuilderFactory.makeBuilder(cc).build(cctx, cancel, c.option.timeout) + failpoint.Inject("unreachableNetwork", func() { + stream = nil + err = status.New(codes.Unavailable, "unavailable").Err() + }) + if stream != nil && err == nil { + updateAndClear(url, &tsoConnectionContext{url, stream, cctx, cancel}) + return nil } - } - cancel() + if err != nil && c.option.enableForwarding { + // The reason we need to judge if the error code is equal to "Canceled" here is that + // when we create a stream we use a goroutine to manually control the timeout of the connection. + // There is no need to wait for the transport layer timeout which can reduce the time of unavailability. + // But it conflicts with the retry mechanism since we use the error code to decide if it is caused by network error. + // And actually the `Canceled` error can be regarded as a kind of network error in some way. + if rpcErr, ok := status.FromError(err); ok && (isNetworkError(rpcErr.Code()) || rpcErr.Code() == codes.Canceled) { + networkErrNum++ + } + } + cancel() + } else { + networkErrNum++ + } select { case <-dispatcherCtx.Done(): return err diff --git a/tests/integrations/client/client_test.go b/tests/integrations/client/client_test.go index 3834d9b53bf..bb4d6851fd0 100644 --- a/tests/integrations/client/client_test.go +++ b/tests/integrations/client/client_test.go @@ -518,7 +518,7 @@ func TestCustomTimeout(t *testing.T) { re.Less(time.Since(start), 2*time.Second) } -func TestGetRegionFromFollowerClient(t *testing.T) { +func TestGetRegionByFollowerForwarding(t *testing.T) { re := require.New(t) ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -544,7 +544,7 @@ func TestGetRegionFromFollowerClient(t *testing.T) { } // case 1: unreachable -> normal -func TestGetTsoFromFollowerClient1(t *testing.T) { +func TestGetTsoByFollowerForwarding1(t *testing.T) { re := require.New(t) ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -575,7 +575,7 @@ func TestGetTsoFromFollowerClient1(t *testing.T) { } // case 2: unreachable -> leader transfer -> normal -func TestGetTsoFromFollowerClient2(t *testing.T) { +func TestGetTsoByFollowerForwarding2(t *testing.T) { re := require.New(t) ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -609,6 +609,101 @@ func TestGetTsoFromFollowerClient2(t *testing.T) { checkTS(re, cli, lastTS) } +// case 3: network partition between client and follower A -> transfer leader to follower A -> normal +func TestGetTsoAndRegionByFollowerForwarding(t *testing.T) { + re := require.New(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + pd.LeaderHealthCheckInterval = 100 * time.Millisecond + cluster, err := tests.NewTestCluster(ctx, 3) + re.NoError(err) + defer cluster.Destroy() + + endpoints := runServer(re, cluster) + re.NotEmpty(cluster.WaitLeader()) + leader := cluster.GetLeaderServer() + grpcPDClient := testutil.MustNewGrpcClient(re, leader.GetAddr()) + testutil.Eventually(re, func() bool { + regionHeartbeat, err := grpcPDClient.RegionHeartbeat(ctx) + re.NoError(err) + regionID := regionIDAllocator.alloc() + region := &metapb.Region{ + Id: regionID, + RegionEpoch: &metapb.RegionEpoch{ + ConfVer: 1, + Version: 1, + }, + Peers: peers, + } + req := &pdpb.RegionHeartbeatRequest{ + Header: newHeader(leader.GetServer()), + Region: region, + Leader: peers[0], + } + err = regionHeartbeat.Send(req) + re.NoError(err) + _, err = regionHeartbeat.Recv() + return err == nil + }) + follower := cluster.GetServer(cluster.GetFollower()) + re.NoError(failpoint.Enable("github.com/tikv/pd/client/grpcutil/unreachableNetwork2", fmt.Sprintf("return(\"%s\")", follower.GetAddr()))) + + cli := setupCli(re, ctx, endpoints, pd.WithForwardingOption(true)) + var lastTS uint64 + testutil.Eventually(re, func() bool { + physical, logical, err := cli.GetTS(context.TODO()) + if err == nil { + lastTS = tsoutil.ComposeTS(physical, logical) + return true + } + t.Log(err) + return false + }) + lastTS = checkTS(re, cli, lastTS) + r, err := cli.GetRegion(context.Background(), []byte("a")) + re.NoError(err) + re.NotNil(r) + leader.GetServer().GetMember().ResignEtcdLeader(leader.GetServer().Context(), + leader.GetServer().Name(), follower.GetServer().Name()) + re.NotEmpty(cluster.WaitLeader()) + testutil.Eventually(re, func() bool { + physical, logical, err := cli.GetTS(context.TODO()) + if err == nil { + lastTS = tsoutil.ComposeTS(physical, logical) + return true + } + t.Log(err) + return false + }) + lastTS = checkTS(re, cli, lastTS) + testutil.Eventually(re, func() bool { + r, err = cli.GetRegion(context.Background(), []byte("a")) + if err == nil && r != nil { + return true + } + return false + }) + + re.NoError(failpoint.Disable("github.com/tikv/pd/client/grpcutil/unreachableNetwork2")) + testutil.Eventually(re, func() bool { + physical, logical, err := cli.GetTS(context.TODO()) + if err == nil { + lastTS = tsoutil.ComposeTS(physical, logical) + return true + } + t.Log(err) + return false + }) + lastTS = checkTS(re, cli, lastTS) + testutil.Eventually(re, func() bool { + r, err = cli.GetRegion(context.Background(), []byte("a")) + if err == nil && r != nil { + return true + } + return false + }) +} + func checkTS(re *require.Assertions, cli pd.Client, lastTS uint64) uint64 { for i := 0; i < tsoRequestRound; i++ { physical, logical, err := cli.GetTS(context.TODO()) From 180ff57afe62c1391e3edace8a15336f4c139417 Mon Sep 17 00:00:00 2001 From: Yongbo Jiang Date: Wed, 29 Nov 2023 18:05:48 +0800 Subject: [PATCH 063/137] client: avoid to add redundant grpc metadata (#7471) close tikv/pd#7469 Signed-off-by: Cabinfever_B Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- client/client.go | 130 ++++++++++++++++++-------------------- client/gc_client.go | 10 +-- client/keyspace_client.go | 4 -- 3 files changed, 64 insertions(+), 80 deletions(-) diff --git a/client/client.go b/client/client.go index f34f5897013..b320be6d3d5 100644 --- a/client/client.go +++ b/client/client.go @@ -765,8 +765,7 @@ func (c *client) GetAllMembers(ctx context.Context) ([]*pdpb.Member, error) { ctx, cancel := context.WithTimeout(ctx, c.option.timeout) req := &pdpb.GetMembersRequest{Header: c.requestHeader()} - ctx = grpcutil.BuildForwardContext(ctx, c.GetLeaderAddr()) - protoClient := c.getClient() + protoClient, ctx := c.getClientAndContext(ctx) if protoClient == nil { cancel() return nil, errs.ErrClientGetProtoClient @@ -825,6 +824,17 @@ func (c *client) getClient() pdpb.PDClient { return c.leaderClient() } +func (c *client) getClientAndContext(ctx context.Context) (pdpb.PDClient, context.Context) { + if c.option.enableForwarding && atomic.LoadInt32(&c.leaderNetworkFailure) == 1 { + backupClientConn, addr := c.backupClientConn() + if backupClientConn != nil { + log.Debug("[pd] use follower client", zap.String("addr", addr)) + return pdpb.NewPDClient(backupClientConn), grpcutil.BuildForwardContext(ctx, c.GetLeaderAddr()) + } + } + return c.leaderClient(), ctx +} + func (c *client) GetTSAsync(ctx context.Context) TSFuture { return c.GetLocalTSAsync(ctx, globalDCLocation) } @@ -929,39 +939,6 @@ func handleRegionResponse(res *pdpb.GetRegionResponse) *Region { return r } -func (c *client) GetRegion(ctx context.Context, key []byte, opts ...GetRegionOption) (*Region, error) { - if span := opentracing.SpanFromContext(ctx); span != nil { - span = opentracing.StartSpan("pdclient.GetRegion", opentracing.ChildOf(span.Context())) - defer span.Finish() - } - start := time.Now() - defer func() { cmdDurationGetRegion.Observe(time.Since(start).Seconds()) }() - ctx, cancel := context.WithTimeout(ctx, c.option.timeout) - - options := &GetRegionOp{} - for _, opt := range opts { - opt(options) - } - req := &pdpb.GetRegionRequest{ - Header: c.requestHeader(), - RegionKey: key, - NeedBuckets: options.needBuckets, - } - ctx = grpcutil.BuildForwardContext(ctx, c.GetLeaderAddr()) - protoClient := c.getClient() - if protoClient == nil { - cancel() - return nil, errs.ErrClientGetProtoClient - } - resp, err := protoClient.GetRegion(ctx, req) - cancel() - - if err = c.respForErr(cmdFailDurationGetRegion, start, err, resp.GetHeader()); err != nil { - return nil, err - } - return handleRegionResponse(resp), nil -} - func isNetworkError(code codes.Code) bool { return code == codes.Unavailable || code == codes.DeadlineExceeded } @@ -1004,6 +981,38 @@ func (c *client) GetRegionFromMember(ctx context.Context, key []byte, memberURLs return handleRegionResponse(resp), nil } +func (c *client) GetRegion(ctx context.Context, key []byte, opts ...GetRegionOption) (*Region, error) { + if span := opentracing.SpanFromContext(ctx); span != nil { + span = opentracing.StartSpan("pdclient.GetRegion", opentracing.ChildOf(span.Context())) + defer span.Finish() + } + start := time.Now() + defer func() { cmdDurationGetRegion.Observe(time.Since(start).Seconds()) }() + ctx, cancel := context.WithTimeout(ctx, c.option.timeout) + + options := &GetRegionOp{} + for _, opt := range opts { + opt(options) + } + req := &pdpb.GetRegionRequest{ + Header: c.requestHeader(), + RegionKey: key, + NeedBuckets: options.needBuckets, + } + protoClient, ctx := c.getClientAndContext(ctx) + if protoClient == nil { + cancel() + return nil, errs.ErrClientGetProtoClient + } + resp, err := protoClient.GetRegion(ctx, req) + cancel() + + if err = c.respForErr(cmdFailDurationGetRegion, start, err, resp.GetHeader()); err != nil { + return nil, err + } + return handleRegionResponse(resp), nil +} + func (c *client) GetPrevRegion(ctx context.Context, key []byte, opts ...GetRegionOption) (*Region, error) { if span := opentracing.SpanFromContext(ctx); span != nil { span = opentracing.StartSpan("pdclient.GetPrevRegion", opentracing.ChildOf(span.Context())) @@ -1022,8 +1031,7 @@ func (c *client) GetPrevRegion(ctx context.Context, key []byte, opts ...GetRegio RegionKey: key, NeedBuckets: options.needBuckets, } - ctx = grpcutil.BuildForwardContext(ctx, c.GetLeaderAddr()) - protoClient := c.getClient() + protoClient, ctx := c.getClientAndContext(ctx) if protoClient == nil { cancel() return nil, errs.ErrClientGetProtoClient @@ -1055,8 +1063,7 @@ func (c *client) GetRegionByID(ctx context.Context, regionID uint64, opts ...Get RegionId: regionID, NeedBuckets: options.needBuckets, } - ctx = grpcutil.BuildForwardContext(ctx, c.GetLeaderAddr()) - protoClient := c.getClient() + protoClient, ctx := c.getClientAndContext(ctx) if protoClient == nil { cancel() return nil, errs.ErrClientGetProtoClient @@ -1090,8 +1097,7 @@ func (c *client) ScanRegions(ctx context.Context, key, endKey []byte, limit int, EndKey: endKey, Limit: int32(limit), } - scanCtx = grpcutil.BuildForwardContext(scanCtx, c.GetLeaderAddr()) - protoClient := c.getClient() + protoClient, scanCtx := c.getClientAndContext(scanCtx) if protoClient == nil { cancel() return nil, errs.ErrClientGetProtoClient @@ -1146,8 +1152,7 @@ func (c *client) GetStore(ctx context.Context, storeID uint64) (*metapb.Store, e Header: c.requestHeader(), StoreId: storeID, } - ctx = grpcutil.BuildForwardContext(ctx, c.GetLeaderAddr()) - protoClient := c.getClient() + protoClient, ctx := c.getClientAndContext(ctx) if protoClient == nil { cancel() return nil, errs.ErrClientGetProtoClient @@ -1191,8 +1196,7 @@ func (c *client) GetAllStores(ctx context.Context, opts ...GetStoreOption) ([]*m Header: c.requestHeader(), ExcludeTombstoneStores: options.excludeTombstone, } - ctx = grpcutil.BuildForwardContext(ctx, c.GetLeaderAddr()) - protoClient := c.getClient() + protoClient, ctx := c.getClientAndContext(ctx) if protoClient == nil { cancel() return nil, errs.ErrClientGetProtoClient @@ -1219,8 +1223,7 @@ func (c *client) UpdateGCSafePoint(ctx context.Context, safePoint uint64) (uint6 Header: c.requestHeader(), SafePoint: safePoint, } - ctx = grpcutil.BuildForwardContext(ctx, c.GetLeaderAddr()) - protoClient := c.getClient() + protoClient, ctx := c.getClientAndContext(ctx) if protoClient == nil { cancel() return 0, errs.ErrClientGetProtoClient @@ -1254,8 +1257,7 @@ func (c *client) UpdateServiceGCSafePoint(ctx context.Context, serviceID string, TTL: ttl, SafePoint: safePoint, } - ctx = grpcutil.BuildForwardContext(ctx, c.GetLeaderAddr()) - protoClient := c.getClient() + protoClient, ctx := c.getClientAndContext(ctx) if protoClient == nil { cancel() return 0, errs.ErrClientGetProtoClient @@ -1287,8 +1289,7 @@ func (c *client) scatterRegionsWithGroup(ctx context.Context, regionID uint64, g RegionId: regionID, Group: group, } - ctx = grpcutil.BuildForwardContext(ctx, c.GetLeaderAddr()) - protoClient := c.getClient() + protoClient, ctx := c.getClientAndContext(ctx) if protoClient == nil { cancel() return errs.ErrClientGetProtoClient @@ -1332,8 +1333,7 @@ func (c *client) SplitAndScatterRegions(ctx context.Context, splitKeys [][]byte, RetryLimit: options.retryLimit, } - ctx = grpcutil.BuildForwardContext(ctx, c.GetLeaderAddr()) - protoClient := c.getClient() + protoClient, ctx := c.getClientAndContext(ctx) if protoClient == nil { cancel() return nil, errs.ErrClientGetProtoClient @@ -1355,8 +1355,7 @@ func (c *client) GetOperator(ctx context.Context, regionID uint64) (*pdpb.GetOpe Header: c.requestHeader(), RegionId: regionID, } - ctx = grpcutil.BuildForwardContext(ctx, c.GetLeaderAddr()) - protoClient := c.getClient() + protoClient, ctx := c.getClientAndContext(ctx) if protoClient == nil { cancel() return nil, errs.ErrClientGetProtoClient @@ -1383,8 +1382,7 @@ func (c *client) SplitRegions(ctx context.Context, splitKeys [][]byte, opts ...R SplitKeys: splitKeys, RetryLimit: options.retryLimit, } - ctx = grpcutil.BuildForwardContext(ctx, c.GetLeaderAddr()) - protoClient := c.getClient() + protoClient, ctx := c.getClientAndContext(ctx) if protoClient == nil { cancel() return nil, errs.ErrClientGetProtoClient @@ -1414,8 +1412,7 @@ func (c *client) scatterRegionsWithOptions(ctx context.Context, regionsID []uint SkipStoreLimit: options.skipStoreLimit, } - ctx = grpcutil.BuildForwardContext(ctx, c.GetLeaderAddr()) - protoClient := c.getClient() + protoClient, ctx := c.getClientAndContext(ctx) if protoClient == nil { cancel() return nil, errs.ErrClientGetProtoClient @@ -1465,8 +1462,7 @@ func trimHTTPPrefix(str string) string { func (c *client) LoadGlobalConfig(ctx context.Context, names []string, configPath string) ([]GlobalConfigItem, int64, error) { ctx, cancel := context.WithTimeout(ctx, c.option.timeout) defer cancel() - ctx = grpcutil.BuildForwardContext(ctx, c.GetLeaderAddr()) - protoClient := c.getClient() + protoClient, ctx := c.getClientAndContext(ctx) if protoClient == nil { return nil, 0, errs.ErrClientGetProtoClient } @@ -1497,8 +1493,7 @@ func (c *client) StoreGlobalConfig(ctx context.Context, configPath string, items } ctx, cancel := context.WithTimeout(ctx, c.option.timeout) defer cancel() - ctx = grpcutil.BuildForwardContext(ctx, c.GetLeaderAddr()) - protoClient := c.getClient() + protoClient, ctx := c.getClientAndContext(ctx) if protoClient == nil { return errs.ErrClientGetProtoClient } @@ -1515,8 +1510,7 @@ func (c *client) WatchGlobalConfig(ctx context.Context, configPath string, revis globalConfigWatcherCh := make(chan []GlobalConfigItem, 16) ctx, cancel := context.WithTimeout(ctx, c.option.timeout) defer cancel() - ctx = grpcutil.BuildForwardContext(ctx, c.GetLeaderAddr()) - protoClient := c.getClient() + protoClient, ctx := c.getClientAndContext(ctx) if protoClient == nil { return nil, errs.ErrClientGetProtoClient } @@ -1564,8 +1558,7 @@ func (c *client) WatchGlobalConfig(ctx context.Context, configPath string, revis func (c *client) GetExternalTimestamp(ctx context.Context) (uint64, error) { ctx, cancel := context.WithTimeout(ctx, c.option.timeout) defer cancel() - ctx = grpcutil.BuildForwardContext(ctx, c.GetLeaderAddr()) - protoClient := c.getClient() + protoClient, ctx := c.getClientAndContext(ctx) if protoClient == nil { return 0, errs.ErrClientGetProtoClient } @@ -1585,8 +1578,7 @@ func (c *client) GetExternalTimestamp(ctx context.Context) (uint64, error) { func (c *client) SetExternalTimestamp(ctx context.Context, timestamp uint64) error { ctx, cancel := context.WithTimeout(ctx, c.option.timeout) defer cancel() - ctx = grpcutil.BuildForwardContext(ctx, c.GetLeaderAddr()) - protoClient := c.getClient() + protoClient, ctx := c.getClientAndContext(ctx) if protoClient == nil { return errs.ErrClientGetProtoClient } diff --git a/client/gc_client.go b/client/gc_client.go index b5d64e25129..fff292405c2 100644 --- a/client/gc_client.go +++ b/client/gc_client.go @@ -22,7 +22,6 @@ import ( "github.com/pingcap/kvproto/pkg/pdpb" "github.com/pingcap/log" "github.com/tikv/pd/client/errs" - "github.com/tikv/pd/client/grpcutil" "go.uber.org/zap" ) @@ -48,8 +47,7 @@ func (c *client) UpdateGCSafePointV2(ctx context.Context, keyspaceID uint32, saf KeyspaceId: keyspaceID, SafePoint: safePoint, } - ctx = grpcutil.BuildForwardContext(ctx, c.GetLeaderAddr()) - protoClient := c.getClient() + protoClient, ctx := c.getClientAndContext(ctx) if protoClient == nil { cancel() return 0, errs.ErrClientGetProtoClient @@ -80,8 +78,7 @@ func (c *client) UpdateServiceSafePointV2(ctx context.Context, keyspaceID uint32 SafePoint: safePoint, Ttl: ttl, } - ctx = grpcutil.BuildForwardContext(ctx, c.GetLeaderAddr()) - protoClient := c.getClient() + protoClient, ctx := c.getClientAndContext(ctx) if protoClient == nil { cancel() return 0, errs.ErrClientGetProtoClient @@ -104,8 +101,7 @@ func (c *client) WatchGCSafePointV2(ctx context.Context, revision int64) (chan [ ctx, cancel := context.WithTimeout(ctx, c.option.timeout) defer cancel() - ctx = grpcutil.BuildForwardContext(ctx, c.GetLeaderAddr()) - protoClient := c.getClient() + protoClient, ctx := c.getClientAndContext(ctx) if protoClient == nil { return nil, errs.ErrClientGetProtoClient } diff --git a/client/keyspace_client.go b/client/keyspace_client.go index d9b9172dd69..fedb7452412 100644 --- a/client/keyspace_client.go +++ b/client/keyspace_client.go @@ -21,7 +21,6 @@ import ( "github.com/opentracing/opentracing-go" "github.com/pingcap/errors" "github.com/pingcap/kvproto/pkg/keyspacepb" - "github.com/tikv/pd/client/grpcutil" ) // KeyspaceClient manages keyspace metadata. @@ -57,7 +56,6 @@ func (c *client) LoadKeyspace(ctx context.Context, name string) (*keyspacepb.Key Header: c.requestHeader(), Name: name, } - ctx = grpcutil.BuildForwardContext(ctx, c.GetLeaderAddr()) resp, err := c.keyspaceClient().LoadKeyspace(ctx, req) cancel() @@ -98,7 +96,6 @@ func (c *client) UpdateKeyspaceState(ctx context.Context, id uint32, state keysp Id: id, State: state, } - ctx = grpcutil.BuildForwardContext(ctx, c.GetLeaderAddr()) resp, err := c.keyspaceClient().UpdateKeyspaceState(ctx, req) cancel() @@ -138,7 +135,6 @@ func (c *client) GetAllKeyspaces(ctx context.Context, startID uint32, limit uint StartId: startID, Limit: limit, } - ctx = grpcutil.BuildForwardContext(ctx, c.GetLeaderAddr()) resp, err := c.keyspaceClient().GetAllKeyspaces(ctx, req) cancel() From 871be59ab0dfb71d8ad1ee0ff918978f01d7728d Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Wed, 29 Nov 2023 20:02:47 +0800 Subject: [PATCH 064/137] api: return `[]` rather than `null` for empty scheduler results (#7454) close tikv/pd#7452 Signed-off-by: lhy1024 --- pkg/schedule/handler/handler.go | 6 ++-- tests/pdctl/scheduler/scheduler_test.go | 4 +-- tests/server/api/scheduler_test.go | 47 +++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 5 deletions(-) diff --git a/pkg/schedule/handler/handler.go b/pkg/schedule/handler/handler.go index 353e2bb60e2..346a7254284 100644 --- a/pkg/schedule/handler/handler.go +++ b/pkg/schedule/handler/handler.go @@ -807,7 +807,7 @@ func (h *Handler) GetSchedulerByStatus(status string, needTS bool) (interface{}, schedulers := sc.GetSchedulerNames() switch status { case "paused": - var pausedSchedulers []string + pausedSchedulers := make([]string, 0, len(schedulers)) pausedPeriods := []schedulerPausedPeriod{} for _, scheduler := range schedulers { paused, err := sc.IsSchedulerPaused(scheduler) @@ -842,7 +842,7 @@ func (h *Handler) GetSchedulerByStatus(status string, needTS bool) (interface{}, } return pausedSchedulers, nil case "disabled": - var disabledSchedulers []string + disabledSchedulers := make([]string, 0, len(schedulers)) for _, scheduler := range schedulers { disabled, err := sc.IsSchedulerDisabled(scheduler) if err != nil { @@ -857,7 +857,7 @@ func (h *Handler) GetSchedulerByStatus(status string, needTS bool) (interface{}, // The default scheduler could not be deleted in scheduling server, // so schedulers could only be disabled. // We should not return the disabled schedulers here. - var enabledSchedulers []string + enabledSchedulers := make([]string, 0, len(schedulers)) for _, scheduler := range schedulers { disabled, err := sc.IsSchedulerDisabled(scheduler) if err != nil { diff --git a/tests/pdctl/scheduler/scheduler_test.go b/tests/pdctl/scheduler/scheduler_test.go index 7e9aeef16ee..d5bea895683 100644 --- a/tests/pdctl/scheduler/scheduler_test.go +++ b/tests/pdctl/scheduler/scheduler_test.go @@ -545,7 +545,7 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) { mustUsage([]string{"-u", pdAddr, "scheduler", "resume", "balance-leader-scheduler", "60"}) echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "resume", "balance-leader-scheduler"}, nil) re.Contains(echo, "Success!") - checkSchedulerWithStatusCommand("paused", nil) + checkSchedulerWithStatusCommand("paused", []string{}) // set label scheduler to disabled manually. echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "add", "label-scheduler"}, nil) @@ -560,7 +560,7 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) { cfg.Schedulers = origin err = leaderServer.GetServer().SetScheduleConfig(*cfg) re.NoError(err) - checkSchedulerWithStatusCommand("disabled", nil) + checkSchedulerWithStatusCommand("disabled", []string{}) } func (suite *schedulerTestSuite) TestSchedulerDiagnostic() { diff --git a/tests/server/api/scheduler_test.go b/tests/server/api/scheduler_test.go index 86b932b0a7a..2388f95e9df 100644 --- a/tests/server/api/scheduler_test.go +++ b/tests/server/api/scheduler_test.go @@ -17,6 +17,7 @@ package api import ( "encoding/json" "fmt" + "io" "net/http" "reflect" "testing" @@ -28,6 +29,7 @@ import ( "github.com/stretchr/testify/suite" sc "github.com/tikv/pd/pkg/schedule/config" "github.com/tikv/pd/pkg/slice" + "github.com/tikv/pd/pkg/utils/apiutil" tu "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/server" "github.com/tikv/pd/tests" @@ -673,6 +675,40 @@ func (suite *scheduleTestSuite) testPauseOrResume(urlPrefix string, name, create suite.False(isPaused) } +func (suite *scheduleTestSuite) TestEmptySchedulers() { + env := tests.NewSchedulingTestEnvironment(suite.T()) + env.RunTestInTwoModes(suite.checkEmptySchedulers) +} + +func (suite *scheduleTestSuite) checkEmptySchedulers(cluster *tests.TestCluster) { + re := suite.Require() + leaderAddr := cluster.GetLeaderServer().GetAddr() + urlPrefix := fmt.Sprintf("%s/pd/api/v1/schedulers", leaderAddr) + for i := 1; i <= 4; i++ { + store := &metapb.Store{ + Id: uint64(i), + State: metapb.StoreState_Up, + NodeState: metapb.NodeState_Serving, + LastHeartbeat: time.Now().UnixNano(), + } + tests.MustPutStore(suite.Require(), cluster, store) + } + + // test disabled and paused schedulers + suite.checkEmptySchedulersResp(urlPrefix + "?status=disabled") + suite.checkEmptySchedulersResp(urlPrefix + "?status=paused") + + // test enabled schedulers + schedulers := make([]string, 0) + suite.NoError(tu.ReadGetJSON(re, testDialClient, urlPrefix, &schedulers)) + for _, scheduler := range schedulers { + suite.deleteScheduler(urlPrefix, scheduler) + } + suite.NoError(tu.ReadGetJSON(re, testDialClient, urlPrefix, &schedulers)) + suite.Len(schedulers, 0) + suite.checkEmptySchedulersResp(urlPrefix) +} + func (suite *scheduleTestSuite) assertSchedulerExists(re *require.Assertions, urlPrefix string, scheduler string) { var schedulers []string tu.Eventually(re, func() bool { @@ -700,3 +736,14 @@ func (suite *scheduleTestSuite) isSchedulerPaused(urlPrefix, name string) bool { } return false } + +func (suite *scheduleTestSuite) checkEmptySchedulersResp(url string) { + resp, err := apiutil.GetJSON(testDialClient, url, nil) + suite.NoError(err) + defer resp.Body.Close() + suite.Equal(http.StatusOK, resp.StatusCode) + b, err := io.ReadAll(resp.Body) + suite.NoError(err) + suite.Contains(string(b), "[]") + suite.NotContains(string(b), "null") +} From 150139c48895b019938db9d1a33d5138d9254047 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 30 Nov 2023 06:29:49 +0000 Subject: [PATCH 065/137] build(deps): bump google.golang.org/grpc from 1.54.0 to 1.56.3 in /tools/pd-tso-bench (#7259) ref tikv/pd#4399 Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: disksing Co-authored-by: JmPotato --- tools/pd-tso-bench/go.mod | 2 +- tools/pd-tso-bench/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/pd-tso-bench/go.mod b/tools/pd-tso-bench/go.mod index f89f11ee082..8d4b3d18a31 100644 --- a/tools/pd-tso-bench/go.mod +++ b/tools/pd-tso-bench/go.mod @@ -9,7 +9,7 @@ require ( github.com/prometheus/client_golang v1.11.1 github.com/tikv/pd/client v0.0.0-00010101000000-000000000000 go.uber.org/zap v1.24.0 - google.golang.org/grpc v1.54.0 + google.golang.org/grpc v1.56.3 ) require ( diff --git a/tools/pd-tso-bench/go.sum b/tools/pd-tso-bench/go.sum index 1c266823dee..15ba2923695 100644 --- a/tools/pd-tso-bench/go.sum +++ b/tools/pd-tso-bench/go.sum @@ -205,8 +205,8 @@ gonum.org/v1/netlib v0.0.0-20181029234149-ec6d1f5cefe6/go.mod h1:wa6Ws7BG/ESfp6d google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= -google.golang.org/grpc v1.54.0 h1:EhTqbhiYeixwWQtAEZAxmV9MGqcjEU2mFx52xCzNyag= -google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g= +google.golang.org/grpc v1.56.3 h1:8I4C0Yq1EjstUzUJzpcRVbuYA2mODtEmpWiQoN/b2nc= +google.golang.org/grpc v1.56.3/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= From beedacb8b291b60db4108ba94cbf75fc72ab20a2 Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Thu, 30 Nov 2023 15:11:49 +0800 Subject: [PATCH 066/137] schedulers: fix config data race (#7486) close tikv/pd#7485 Signed-off-by: lhy1024 --- pkg/schedule/schedulers/balance_leader.go | 36 ++++++++++++------- pkg/schedule/schedulers/balance_witness.go | 24 +++++++++---- pkg/schedule/schedulers/grant_hot_region.go | 13 +++++-- pkg/schedule/schedulers/grant_leader.go | 17 ++++++--- pkg/schedule/schedulers/hot_region_config.go | 4 +-- pkg/schedule/schedulers/hot_region_test.go | 28 +++++++-------- pkg/schedule/schedulers/scatter_range.go | 4 +-- pkg/schedule/schedulers/shuffle_hot_region.go | 8 ++++- pkg/schedule/schedulers/split_bucket.go | 16 +++++++-- 9 files changed, 105 insertions(+), 45 deletions(-) diff --git a/pkg/schedule/schedulers/balance_leader.go b/pkg/schedule/schedulers/balance_leader.go index 6a64edf7e70..eb94752944b 100644 --- a/pkg/schedule/schedulers/balance_leader.go +++ b/pkg/schedule/schedulers/balance_leader.go @@ -78,19 +78,19 @@ func (conf *balanceLeaderSchedulerConfig) Update(data []byte) (int, interface{}) conf.Lock() defer conf.Unlock() - oldc, _ := json.Marshal(conf) + oldConfig, _ := json.Marshal(conf) if err := json.Unmarshal(data, conf); err != nil { return http.StatusInternalServerError, err.Error() } - newc, _ := json.Marshal(conf) - if !bytes.Equal(oldc, newc) { - if !conf.validate() { - json.Unmarshal(oldc, conf) + newConfig, _ := json.Marshal(conf) + if !bytes.Equal(oldConfig, newConfig) { + if !conf.validateLocked() { + json.Unmarshal(oldConfig, conf) return http.StatusBadRequest, "invalid batch size which should be an integer between 1 and 10" } conf.persistLocked() - log.Info("balance-leader-scheduler config is updated", zap.ByteString("old", oldc), zap.ByteString("new", newc)) + log.Info("balance-leader-scheduler config is updated", zap.ByteString("old", oldConfig), zap.ByteString("new", newConfig)) return http.StatusOK, "Config is updated." } m := make(map[string]interface{}) @@ -104,7 +104,7 @@ func (conf *balanceLeaderSchedulerConfig) Update(data []byte) (int, interface{}) return http.StatusBadRequest, "Config item is not found." } -func (conf *balanceLeaderSchedulerConfig) validate() bool { +func (conf *balanceLeaderSchedulerConfig) validateLocked() bool { return conf.Batch >= 1 && conf.Batch <= 10 } @@ -127,6 +127,20 @@ func (conf *balanceLeaderSchedulerConfig) persistLocked() error { return conf.storage.SaveSchedulerConfig(BalanceLeaderName, data) } +func (conf *balanceLeaderSchedulerConfig) getBatch() int { + conf.RLock() + defer conf.RUnlock() + return conf.Batch +} + +func (conf *balanceLeaderSchedulerConfig) getRanges() []core.KeyRange { + conf.RLock() + defer conf.RUnlock() + ranges := make([]core.KeyRange, len(conf.Ranges)) + copy(ranges, conf.Ranges) + return ranges +} + type balanceLeaderHandler struct { rd *render.Render config *balanceLeaderSchedulerConfig @@ -335,14 +349,12 @@ func (cs *candidateStores) resortStoreWithPos(pos int) { } func (l *balanceLeaderScheduler) Schedule(cluster sche.SchedulerCluster, dryRun bool) ([]*operator.Operator, []plan.Plan) { - l.conf.RLock() - defer l.conf.RUnlock() basePlan := plan.NewBalanceSchedulerPlan() var collector *plan.Collector if dryRun { collector = plan.NewCollector(basePlan) } - batch := l.conf.Batch + batch := l.conf.getBatch() balanceLeaderScheduleCounter.Inc() leaderSchedulePolicy := cluster.GetSchedulerConfig().GetLeaderSchedulePolicy() @@ -441,7 +453,7 @@ func makeInfluence(op *operator.Operator, plan *solver, usedRegions map[uint64]s // It randomly selects a health region from the source store, then picks // the best follower peer and transfers the leader. func (l *balanceLeaderScheduler) transferLeaderOut(solver *solver, collector *plan.Collector) *operator.Operator { - solver.Region = filter.SelectOneRegion(solver.RandLeaderRegions(solver.SourceStoreID(), l.conf.Ranges), + solver.Region = filter.SelectOneRegion(solver.RandLeaderRegions(solver.SourceStoreID(), l.conf.getRanges()), collector, filter.NewRegionPendingFilter(), filter.NewRegionDownFilter()) if solver.Region == nil { log.Debug("store has no leader", zap.String("scheduler", l.GetName()), zap.Uint64("store-id", solver.SourceStoreID())) @@ -485,7 +497,7 @@ func (l *balanceLeaderScheduler) transferLeaderOut(solver *solver, collector *pl // It randomly selects a health region from the target store, then picks // the worst follower peer and transfers the leader. func (l *balanceLeaderScheduler) transferLeaderIn(solver *solver, collector *plan.Collector) *operator.Operator { - solver.Region = filter.SelectOneRegion(solver.RandFollowerRegions(solver.TargetStoreID(), l.conf.Ranges), + solver.Region = filter.SelectOneRegion(solver.RandFollowerRegions(solver.TargetStoreID(), l.conf.getRanges()), nil, filter.NewRegionPendingFilter(), filter.NewRegionDownFilter()) if solver.Region == nil { log.Debug("store has no follower", zap.String("scheduler", l.GetName()), zap.Uint64("store-id", solver.TargetStoreID())) diff --git a/pkg/schedule/schedulers/balance_witness.go b/pkg/schedule/schedulers/balance_witness.go index bf3fbbb83da..9994866ac50 100644 --- a/pkg/schedule/schedulers/balance_witness.go +++ b/pkg/schedule/schedulers/balance_witness.go @@ -71,7 +71,7 @@ func (conf *balanceWitnessSchedulerConfig) Update(data []byte) (int, interface{} } newc, _ := json.Marshal(conf) if !bytes.Equal(oldc, newc) { - if !conf.validate() { + if !conf.validateLocked() { json.Unmarshal(oldc, conf) return http.StatusBadRequest, "invalid batch size which should be an integer between 1 and 10" } @@ -90,7 +90,7 @@ func (conf *balanceWitnessSchedulerConfig) Update(data []byte) (int, interface{} return http.StatusBadRequest, "Config item is not found." } -func (conf *balanceWitnessSchedulerConfig) validate() bool { +func (conf *balanceWitnessSchedulerConfig) validateLocked() bool { return conf.Batch >= 1 && conf.Batch <= 10 } @@ -113,6 +113,20 @@ func (conf *balanceWitnessSchedulerConfig) persistLocked() error { return conf.storage.SaveSchedulerConfig(BalanceWitnessName, data) } +func (conf *balanceWitnessSchedulerConfig) getBatch() int { + conf.RLock() + defer conf.RUnlock() + return conf.Batch +} + +func (conf *balanceWitnessSchedulerConfig) getRanges() []core.KeyRange { + conf.RLock() + defer conf.RUnlock() + ranges := make([]core.KeyRange, len(conf.Ranges)) + copy(ranges, conf.Ranges) + return ranges +} + type balanceWitnessHandler struct { rd *render.Render config *balanceWitnessSchedulerConfig @@ -238,14 +252,12 @@ func (b *balanceWitnessScheduler) IsScheduleAllowed(cluster sche.SchedulerCluste } func (b *balanceWitnessScheduler) Schedule(cluster sche.SchedulerCluster, dryRun bool) ([]*operator.Operator, []plan.Plan) { - b.conf.RLock() - defer b.conf.RUnlock() basePlan := plan.NewBalanceSchedulerPlan() var collector *plan.Collector if dryRun { collector = plan.NewCollector(basePlan) } - batch := b.conf.Batch + batch := b.conf.getBatch() schedulerCounter.WithLabelValues(b.GetName(), "schedule").Inc() opInfluence := b.OpController.GetOpInfluence(cluster.GetBasicCluster()) @@ -305,7 +317,7 @@ func createTransferWitnessOperator(cs *candidateStores, b *balanceWitnessSchedul // It randomly selects a health region from the source store, then picks // the best follower peer and transfers the witness. func (b *balanceWitnessScheduler) transferWitnessOut(solver *solver, collector *plan.Collector) *operator.Operator { - solver.Region = filter.SelectOneRegion(solver.RandWitnessRegions(solver.SourceStoreID(), b.conf.Ranges), + solver.Region = filter.SelectOneRegion(solver.RandWitnessRegions(solver.SourceStoreID(), b.conf.getRanges()), collector, filter.NewRegionPendingFilter(), filter.NewRegionDownFilter()) if solver.Region == nil { log.Debug("store has no witness", zap.String("scheduler", b.GetName()), zap.Uint64("store-id", solver.SourceStoreID())) diff --git a/pkg/schedule/schedulers/grant_hot_region.go b/pkg/schedule/schedulers/grant_hot_region.go index 6ab689ea5d4..81399b58c58 100644 --- a/pkg/schedule/schedulers/grant_hot_region.go +++ b/pkg/schedule/schedulers/grant_hot_region.go @@ -120,6 +120,14 @@ func (conf *grantHotRegionSchedulerConfig) has(storeID uint64) bool { }) } +func (conf *grantHotRegionSchedulerConfig) getStoreIDs() []uint64 { + conf.RLock() + defer conf.RUnlock() + storeIDs := make([]uint64, len(conf.StoreIDs)) + copy(storeIDs, conf.StoreIDs) + return storeIDs +} + // grantLeaderScheduler transfers all hot peers to peers and transfer leader to the fixed store type grantHotRegionScheduler struct { *baseHotScheduler @@ -313,7 +321,8 @@ func (s *grantHotRegionScheduler) transfer(cluster sche.SchedulerCluster, region filter.NewPlacementSafeguard(s.GetName(), cluster.GetSchedulerConfig(), cluster.GetBasicCluster(), cluster.GetRuleManager(), srcRegion, srcStore, nil), } - destStoreIDs := make([]uint64, 0, len(s.conf.StoreIDs)) + storeIDs := s.conf.getStoreIDs() + destStoreIDs := make([]uint64, 0, len(storeIDs)) var candidate []uint64 if isLeader { filters = append(filters, &filter.StoreStateFilter{ActionScope: s.GetName(), TransferLeader: true, OperatorLevel: constant.High}) @@ -321,7 +330,7 @@ func (s *grantHotRegionScheduler) transfer(cluster sche.SchedulerCluster, region } else { filters = append(filters, &filter.StoreStateFilter{ActionScope: s.GetName(), MoveRegion: true, OperatorLevel: constant.High}, filter.NewExcludedFilter(s.GetName(), srcRegion.GetStoreIDs(), srcRegion.GetStoreIDs())) - candidate = s.conf.StoreIDs + candidate = storeIDs } for _, storeID := range candidate { store := cluster.GetStore(storeID) diff --git a/pkg/schedule/schedulers/grant_leader.go b/pkg/schedule/schedulers/grant_leader.go index 47e14af4902..885f81e2442 100644 --- a/pkg/schedule/schedulers/grant_leader.go +++ b/pkg/schedule/schedulers/grant_leader.go @@ -143,6 +143,16 @@ func (conf *grantLeaderSchedulerConfig) getKeyRangesByID(id uint64) []core.KeyRa return nil } +func (conf *grantLeaderSchedulerConfig) getStoreIDWithRanges() map[uint64][]core.KeyRange { + conf.RLock() + defer conf.RUnlock() + storeIDWithRanges := make(map[uint64][]core.KeyRange) + for id, ranges := range conf.StoreIDWithRanges { + storeIDWithRanges[id] = ranges + } + return storeIDWithRanges +} + // grantLeaderScheduler transfers all leaders to peers in the store. type grantLeaderScheduler struct { *BaseScheduler @@ -227,12 +237,11 @@ func (s *grantLeaderScheduler) IsScheduleAllowed(cluster sche.SchedulerCluster) func (s *grantLeaderScheduler) Schedule(cluster sche.SchedulerCluster, dryRun bool) ([]*operator.Operator, []plan.Plan) { grantLeaderCounter.Inc() - s.conf.RLock() - defer s.conf.RUnlock() - ops := make([]*operator.Operator, 0, len(s.conf.StoreIDWithRanges)) + storeIDWithRanges := s.conf.getStoreIDWithRanges() + ops := make([]*operator.Operator, 0, len(storeIDWithRanges)) pendingFilter := filter.NewRegionPendingFilter() downFilter := filter.NewRegionDownFilter() - for id, ranges := range s.conf.StoreIDWithRanges { + for id, ranges := range storeIDWithRanges { region := filter.SelectOneRegion(cluster.RandFollowerRegions(id, ranges), nil, pendingFilter, downFilter) if region == nil { grantLeaderNoFollowerCounter.Inc() diff --git a/pkg/schedule/schedulers/hot_region_config.go b/pkg/schedule/schedulers/hot_region_config.go index 2ff78748f02..3f9f8b8c669 100644 --- a/pkg/schedule/schedulers/hot_region_config.go +++ b/pkg/schedule/schedulers/hot_region_config.go @@ -366,7 +366,7 @@ func isPriorityValid(priorities []string) (map[string]bool, error) { return priorityMap, nil } -func (conf *hotRegionSchedulerConfig) valid() error { +func (conf *hotRegionSchedulerConfig) validateLocked() error { if _, err := isPriorityValid(conf.ReadPriorities); err != nil { return err } @@ -409,7 +409,7 @@ func (conf *hotRegionSchedulerConfig) handleSetConfig(w http.ResponseWriter, r * rd.JSON(w, http.StatusInternalServerError, err.Error()) return } - if err := conf.valid(); err != nil { + if err := conf.validateLocked(); err != nil { // revert to old version if err2 := json.Unmarshal(oldc, conf); err2 != nil { rd.JSON(w, http.StatusInternalServerError, err2.Error()) diff --git a/pkg/schedule/schedulers/hot_region_test.go b/pkg/schedule/schedulers/hot_region_test.go index 15c037ddd22..6e7208e4251 100644 --- a/pkg/schedule/schedulers/hot_region_test.go +++ b/pkg/schedule/schedulers/hot_region_test.go @@ -2499,32 +2499,32 @@ func TestConfigValidation(t *testing.T) { re := require.New(t) hc := initHotRegionScheduleConfig() - err := hc.valid() + err := hc.validateLocked() re.NoError(err) // priorities is illegal hc.ReadPriorities = []string{"byte", "error"} - err = hc.valid() + err = hc.validateLocked() re.Error(err) // priorities should have at least 2 dimensions hc = initHotRegionScheduleConfig() hc.WriteLeaderPriorities = []string{"byte"} - err = hc.valid() + err = hc.validateLocked() re.Error(err) // query is not allowed to be set in priorities for write-peer-priorities hc = initHotRegionScheduleConfig() hc.WritePeerPriorities = []string{"query", "byte"} - err = hc.valid() + err = hc.validateLocked() re.Error(err) // priorities shouldn't be repeated hc.WritePeerPriorities = []string{"byte", "byte"} - err = hc.valid() + err = hc.validateLocked() re.Error(err) // no error hc.WritePeerPriorities = []string{"byte", "key"} - err = hc.valid() + err = hc.validateLocked() re.NoError(err) // rank-formula-version @@ -2533,17 +2533,17 @@ func TestConfigValidation(t *testing.T) { re.Equal("v2", hc.GetRankFormulaVersion()) // v1 hc.RankFormulaVersion = "v1" - err = hc.valid() + err = hc.validateLocked() re.NoError(err) re.Equal("v1", hc.GetRankFormulaVersion()) // v2 hc.RankFormulaVersion = "v2" - err = hc.valid() + err = hc.validateLocked() re.NoError(err) re.Equal("v2", hc.GetRankFormulaVersion()) // illegal hc.RankFormulaVersion = "v0" - err = hc.valid() + err = hc.validateLocked() re.Error(err) // forbid-rw-type @@ -2553,27 +2553,27 @@ func TestConfigValidation(t *testing.T) { re.False(hc.IsForbidRWType(utils.Write)) // read hc.ForbidRWType = "read" - err = hc.valid() + err = hc.validateLocked() re.NoError(err) re.True(hc.IsForbidRWType(utils.Read)) re.False(hc.IsForbidRWType(utils.Write)) // write hc.ForbidRWType = "write" - err = hc.valid() + err = hc.validateLocked() re.NoError(err) re.False(hc.IsForbidRWType(utils.Read)) re.True(hc.IsForbidRWType(utils.Write)) // illegal hc.ForbidRWType = "test" - err = hc.valid() + err = hc.validateLocked() re.Error(err) hc.SplitThresholds = 0 - err = hc.valid() + err = hc.validateLocked() re.Error(err) hc.SplitThresholds = 1.1 - err = hc.valid() + err = hc.validateLocked() re.Error(err) } diff --git a/pkg/schedule/schedulers/scatter_range.go b/pkg/schedule/schedulers/scatter_range.go index 1bc6eafb58e..977d8cff05c 100644 --- a/pkg/schedule/schedulers/scatter_range.go +++ b/pkg/schedule/schedulers/scatter_range.go @@ -214,7 +214,7 @@ func (l *scatterRangeScheduler) Schedule(cluster sche.SchedulerCluster, dryRun b if l.allowBalanceLeader(cluster) { ops, _ := l.balanceLeader.Schedule(c, false) if len(ops) > 0 { - ops[0].SetDesc(fmt.Sprintf("scatter-range-leader-%s", l.config.RangeName)) + ops[0].SetDesc(fmt.Sprintf("scatter-range-leader-%s", l.config.GetRangeName())) ops[0].AttachKind(operator.OpRange) ops[0].Counters = append(ops[0].Counters, scatterRangeNewOperatorCounter, @@ -226,7 +226,7 @@ func (l *scatterRangeScheduler) Schedule(cluster sche.SchedulerCluster, dryRun b if l.allowBalanceRegion(cluster) { ops, _ := l.balanceRegion.Schedule(c, false) if len(ops) > 0 { - ops[0].SetDesc(fmt.Sprintf("scatter-range-region-%s", l.config.RangeName)) + ops[0].SetDesc(fmt.Sprintf("scatter-range-region-%s", l.config.GetRangeName())) ops[0].AttachKind(operator.OpRange) ops[0].Counters = append(ops[0].Counters, scatterRangeNewOperatorCounter, diff --git a/pkg/schedule/schedulers/shuffle_hot_region.go b/pkg/schedule/schedulers/shuffle_hot_region.go index 6ad6656fd18..cd5c40d4e07 100644 --- a/pkg/schedule/schedulers/shuffle_hot_region.go +++ b/pkg/schedule/schedulers/shuffle_hot_region.go @@ -77,6 +77,12 @@ func (conf *shuffleHotRegionSchedulerConfig) persistLocked() error { return conf.storage.SaveSchedulerConfig(name, data) } +func (conf *shuffleHotRegionSchedulerConfig) getLimit() uint64 { + conf.RLock() + defer conf.RUnlock() + return conf.Limit +} + // ShuffleHotRegionScheduler mainly used to test. // It will randomly pick a hot peer, and move the peer // to a random store, and then transfer the leader to @@ -134,7 +140,7 @@ func (s *shuffleHotRegionScheduler) ReloadConfig() error { } func (s *shuffleHotRegionScheduler) IsScheduleAllowed(cluster sche.SchedulerCluster) bool { - hotRegionAllowed := s.OpController.OperatorCount(operator.OpHotRegion) < s.conf.Limit + hotRegionAllowed := s.OpController.OperatorCount(operator.OpHotRegion) < s.conf.getLimit() conf := cluster.GetSchedulerConfig() regionAllowed := s.OpController.OperatorCount(operator.OpRegion) < conf.GetRegionScheduleLimit() leaderAllowed := s.OpController.OperatorCount(operator.OpLeader) < conf.GetLeaderScheduleLimit() diff --git a/pkg/schedule/schedulers/split_bucket.go b/pkg/schedule/schedulers/split_bucket.go index 6faf03e1fef..5e31f58129c 100644 --- a/pkg/schedule/schedulers/split_bucket.go +++ b/pkg/schedule/schedulers/split_bucket.go @@ -87,6 +87,18 @@ func (conf *splitBucketSchedulerConfig) persistLocked() error { return conf.storage.SaveSchedulerConfig(SplitBucketName, data) } +func (conf *splitBucketSchedulerConfig) getDegree() int { + conf.RLock() + defer conf.RUnlock() + return conf.Degree +} + +func (conf *splitBucketSchedulerConfig) getSplitLimit() uint64 { + conf.RLock() + defer conf.RUnlock() + return conf.SplitLimit +} + type splitBucketScheduler struct { *BaseScheduler conf *splitBucketSchedulerConfig @@ -202,7 +214,7 @@ func (s *splitBucketScheduler) IsScheduleAllowed(cluster sche.SchedulerCluster) splitBucketDisableCounter.Inc() return false } - allowed := s.BaseScheduler.OpController.OperatorCount(operator.OpSplit) < s.conf.SplitLimit + allowed := s.BaseScheduler.OpController.OperatorCount(operator.OpSplit) < s.conf.getSplitLimit() if !allowed { splitBuckerSplitLimitCounter.Inc() operator.OperatorLimitCounter.WithLabelValues(s.GetType(), operator.OpSplit.String()).Inc() @@ -224,7 +236,7 @@ func (s *splitBucketScheduler) Schedule(cluster sche.SchedulerCluster, dryRun bo plan := &splitBucketPlan{ conf: conf, cluster: cluster, - hotBuckets: cluster.BucketsStats(conf.Degree), + hotBuckets: cluster.BucketsStats(conf.getDegree()), hotRegionSplitSize: cluster.GetSchedulerConfig().GetMaxMovableHotPeerSize(), } return s.splitBucket(plan), nil From 862eee18738eabb651601a9362547e5283ee830a Mon Sep 17 00:00:00 2001 From: JmPotato Date: Thu, 30 Nov 2023 16:16:18 +0800 Subject: [PATCH 067/137] client/http: implement the marshaler interfaces for Rule/RuleOp (#7462) ref tikv/pd#7300 Implement the marshaler interfaces for `Rule` and `RuleOP` to make sure we could set/get the correct start/end key. Ref https://github.com/pingcap/tidb/blob/46d4231c8b0ade353b98572e7c2a015bddf940f4/pkg/ddl/placement/rule.go#L76-L91. Signed-off-by: JmPotato --- client/http/api.go | 14 +- client/http/client.go | 21 ++- client/http/codec.go | 121 +++++++++++++ client/http/codec_test.go | 64 +++++++ client/http/types.go | 168 ++++++++++++++++++ client/http/types_test.go | 151 ++++++++++++++++ server/api/stats.go | 5 +- server/api/store.go | 16 +- tests/integrations/client/http_client_test.go | 46 +++-- 9 files changed, 582 insertions(+), 24 deletions(-) create mode 100644 client/http/codec.go create mode 100644 client/http/codec_test.go diff --git a/client/http/api.go b/client/http/api.go index 6b317330b61..f744fd0c395 100644 --- a/client/http/api.go +++ b/client/http/api.go @@ -31,6 +31,7 @@ const ( Regions = "/pd/api/v1/regions" regionsByKey = "/pd/api/v1/regions/key" RegionsByStoreIDPrefix = "/pd/api/v1/regions/store" + regionsReplicated = "/pd/api/v1/regions/replicated" EmptyRegions = "/pd/api/v1/regions/check/empty-region" AccelerateSchedule = "/pd/api/v1/regions/accelerate-schedule" AccelerateScheduleInBatch = "/pd/api/v1/regions/accelerate-schedule/batch" @@ -95,9 +96,20 @@ func RegionsByStoreID(storeID uint64) string { return fmt.Sprintf("%s/%d", RegionsByStoreIDPrefix, storeID) } +// RegionsReplicatedByKeyRange returns the path of PD HTTP API to get replicated regions with given start key and end key. +func RegionsReplicatedByKeyRange(keyRange *KeyRange) string { + startKeyStr, endKeyStr := keyRange.EscapeAsHexStr() + return fmt.Sprintf("%s?startKey=%s&endKey=%s", + regionsReplicated, startKeyStr, endKeyStr) +} + // RegionStatsByKeyRange returns the path of PD HTTP API to get region stats by start key and end key. -func RegionStatsByKeyRange(keyRange *KeyRange) string { +func RegionStatsByKeyRange(keyRange *KeyRange, onlyCount bool) string { startKeyStr, endKeyStr := keyRange.EscapeAsUTF8Str() + if onlyCount { + return fmt.Sprintf("%s?start_key=%s&end_key=%s&count", + StatsRegion, startKeyStr, endKeyStr) + } return fmt.Sprintf("%s?start_key=%s&end_key=%s", StatsRegion, startKeyStr, endKeyStr) } diff --git a/client/http/client.go b/client/http/client.go index d15693e11d4..36355a90d19 100644 --- a/client/http/client.go +++ b/client/http/client.go @@ -47,10 +47,11 @@ type Client interface { GetRegions(context.Context) (*RegionsInfo, error) GetRegionsByKeyRange(context.Context, *KeyRange, int) (*RegionsInfo, error) GetRegionsByStoreID(context.Context, uint64) (*RegionsInfo, error) + GetRegionsReplicatedStateByKeyRange(context.Context, *KeyRange) (string, error) GetHotReadRegions(context.Context) (*StoreHotPeersInfos, error) GetHotWriteRegions(context.Context) (*StoreHotPeersInfos, error) GetHistoryHotRegions(context.Context, *HistoryHotRegionsRequest) (*HistoryHotRegions, error) - GetRegionStatusByKeyRange(context.Context, *KeyRange) (*RegionStats, error) + GetRegionStatusByKeyRange(context.Context, *KeyRange, bool) (*RegionStats, error) GetStores(context.Context) (*StoresInfo, error) /* Config-related interfaces */ GetScheduleConfig(context.Context) (map[string]interface{}, error) @@ -356,6 +357,19 @@ func (c *client) GetRegionsByStoreID(ctx context.Context, storeID uint64) (*Regi return ®ions, nil } +// GetRegionsReplicatedStateByKeyRange gets the regions replicated state info by key range. +// The keys in the key range should be encoded in the hex bytes format (without encoding to the UTF-8 bytes). +func (c *client) GetRegionsReplicatedStateByKeyRange(ctx context.Context, keyRange *KeyRange) (string, error) { + var state string + err := c.requestWithRetry(ctx, + "GetRegionsReplicatedStateByKeyRange", RegionsReplicatedByKeyRange(keyRange), + http.MethodGet, http.NoBody, &state) + if err != nil { + return "", err + } + return state, nil +} + // GetHotReadRegions gets the hot read region statistics info. func (c *client) GetHotReadRegions(ctx context.Context) (*StoreHotPeersInfos, error) { var hotReadRegions StoreHotPeersInfos @@ -398,11 +412,12 @@ func (c *client) GetHistoryHotRegions(ctx context.Context, req *HistoryHotRegion } // GetRegionStatusByKeyRange gets the region status by key range. +// If the `onlyCount` flag is true, the result will only include the count of regions. // The keys in the key range should be encoded in the UTF-8 bytes format. -func (c *client) GetRegionStatusByKeyRange(ctx context.Context, keyRange *KeyRange) (*RegionStats, error) { +func (c *client) GetRegionStatusByKeyRange(ctx context.Context, keyRange *KeyRange, onlyCount bool) (*RegionStats, error) { var regionStats RegionStats err := c.requestWithRetry(ctx, - "GetRegionStatusByKeyRange", RegionStatsByKeyRange(keyRange), + "GetRegionStatusByKeyRange", RegionStatsByKeyRange(keyRange, onlyCount), http.MethodGet, http.NoBody, ®ionStats, ) if err != nil { diff --git a/client/http/codec.go b/client/http/codec.go new file mode 100644 index 00000000000..26be64b4f28 --- /dev/null +++ b/client/http/codec.go @@ -0,0 +1,121 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package http + +import ( + "encoding/hex" + + "github.com/pingcap/errors" +) + +const ( + encGroupSize = 8 + encMarker = byte(0xFF) + encPad = byte(0x0) +) + +var pads = make([]byte, encGroupSize) + +// encodeBytes guarantees the encoded value is in ascending order for comparison, +// encoding with the following rule: +// +// [group1][marker1]...[groupN][markerN] +// group is 8 bytes slice which is padding with 0. +// marker is `0xFF - padding 0 count` +// +// For example: +// +// [] -> [0, 0, 0, 0, 0, 0, 0, 0, 247] +// [1, 2, 3] -> [1, 2, 3, 0, 0, 0, 0, 0, 250] +// [1, 2, 3, 0] -> [1, 2, 3, 0, 0, 0, 0, 0, 251] +// [1, 2, 3, 4, 5, 6, 7, 8] -> [1, 2, 3, 4, 5, 6, 7, 8, 255, 0, 0, 0, 0, 0, 0, 0, 0, 247] +// +// Refer: https://github.com/facebook/mysql-5.6/wiki/MyRocks-record-format#memcomparable-format +func encodeBytes(data []byte) []byte { + // Allocate more space to avoid unnecessary slice growing. + // Assume that the byte slice size is about `(len(data) / encGroupSize + 1) * (encGroupSize + 1)` bytes, + // that is `(len(data) / 8 + 1) * 9` in our implement. + dLen := len(data) + result := make([]byte, 0, (dLen/encGroupSize+1)*(encGroupSize+1)) + for idx := 0; idx <= dLen; idx += encGroupSize { + remain := dLen - idx + padCount := 0 + if remain >= encGroupSize { + result = append(result, data[idx:idx+encGroupSize]...) + } else { + padCount = encGroupSize - remain + result = append(result, data[idx:]...) + result = append(result, pads[:padCount]...) + } + + marker := encMarker - byte(padCount) + result = append(result, marker) + } + return result +} + +func decodeBytes(b []byte) ([]byte, error) { + buf := make([]byte, 0, len(b)) + for { + if len(b) < encGroupSize+1 { + return nil, errors.New("insufficient bytes to decode value") + } + + groupBytes := b[:encGroupSize+1] + + group := groupBytes[:encGroupSize] + marker := groupBytes[encGroupSize] + + padCount := encMarker - marker + if padCount > encGroupSize { + return nil, errors.Errorf("invalid marker byte, group bytes %q", groupBytes) + } + + realGroupSize := encGroupSize - padCount + buf = append(buf, group[:realGroupSize]...) + b = b[encGroupSize+1:] + + if padCount != 0 { + // Check validity of padding bytes. + for _, v := range group[realGroupSize:] { + if v != encPad { + return nil, errors.Errorf("invalid padding byte, group bytes %q", groupBytes) + } + } + break + } + } + return buf, nil +} + +// rawKeyToKeyHexStr converts a raw key to a hex string after encoding. +func rawKeyToKeyHexStr(key []byte) string { + if len(key) == 0 { + return "" + } + return hex.EncodeToString(encodeBytes(key)) +} + +// keyHexStrToRawKey converts a hex string to a raw key after decoding. +func keyHexStrToRawKey(hexKey string) ([]byte, error) { + if len(hexKey) == 0 { + return make([]byte, 0), nil + } + key, err := hex.DecodeString(hexKey) + if err != nil { + return nil, err + } + return decodeBytes(key) +} diff --git a/client/http/codec_test.go b/client/http/codec_test.go new file mode 100644 index 00000000000..fa8d413a0d1 --- /dev/null +++ b/client/http/codec_test.go @@ -0,0 +1,64 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package http + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestBytesCodec(t *testing.T) { + inputs := []struct { + enc []byte + dec []byte + }{ + {[]byte{}, []byte{0, 0, 0, 0, 0, 0, 0, 0, 247}}, + {[]byte{0}, []byte{0, 0, 0, 0, 0, 0, 0, 0, 248}}, + {[]byte{1, 2, 3}, []byte{1, 2, 3, 0, 0, 0, 0, 0, 250}}, + {[]byte{1, 2, 3, 0}, []byte{1, 2, 3, 0, 0, 0, 0, 0, 251}}, + {[]byte{1, 2, 3, 4, 5, 6, 7}, []byte{1, 2, 3, 4, 5, 6, 7, 0, 254}}, + {[]byte{0, 0, 0, 0, 0, 0, 0, 0}, []byte{0, 0, 0, 0, 0, 0, 0, 0, 255, 0, 0, 0, 0, 0, 0, 0, 0, 247}}, + {[]byte{1, 2, 3, 4, 5, 6, 7, 8}, []byte{1, 2, 3, 4, 5, 6, 7, 8, 255, 0, 0, 0, 0, 0, 0, 0, 0, 247}}, + {[]byte{1, 2, 3, 4, 5, 6, 7, 8, 9}, []byte{1, 2, 3, 4, 5, 6, 7, 8, 255, 9, 0, 0, 0, 0, 0, 0, 0, 248}}, + } + + for _, input := range inputs { + b := encodeBytes(input.enc) + require.Equal(t, input.dec, b) + + d, err := decodeBytes(b) + require.NoError(t, err) + require.Equal(t, input.enc, d) + } + + // Test error decode. + errInputs := [][]byte{ + {1, 2, 3, 4}, + {0, 0, 0, 0, 0, 0, 0, 247}, + {0, 0, 0, 0, 0, 0, 0, 0, 246}, + {0, 0, 0, 0, 0, 0, 0, 1, 247}, + {1, 2, 3, 4, 5, 6, 7, 8, 0}, + {1, 2, 3, 4, 5, 6, 7, 8, 255, 1}, + {1, 2, 3, 4, 5, 6, 7, 8, 255, 1, 2, 3, 4, 5, 6, 7, 8}, + {1, 2, 3, 4, 5, 6, 7, 8, 255, 1, 2, 3, 4, 5, 6, 7, 8, 255}, + {1, 2, 3, 4, 5, 6, 7, 8, 255, 1, 2, 3, 4, 5, 6, 7, 8, 0}, + } + + for _, input := range errInputs { + _, err := decodeBytes(input) + require.Error(t, err) + } +} diff --git a/client/http/types.go b/client/http/types.go index 4e99d911e0b..1d8db36d100 100644 --- a/client/http/types.go +++ b/client/http/types.go @@ -341,6 +341,86 @@ func (r *Rule) Clone() *Rule { return &clone } +var ( + _ json.Marshaler = (*Rule)(nil) + _ json.Unmarshaler = (*Rule)(nil) +) + +// This is a helper struct used to customizing the JSON marshal/unmarshal methods of `Rule`. +type rule struct { + GroupID string `json:"group_id"` + ID string `json:"id"` + Index int `json:"index,omitempty"` + Override bool `json:"override,omitempty"` + StartKeyHex string `json:"start_key"` + EndKeyHex string `json:"end_key"` + Role PeerRoleType `json:"role"` + IsWitness bool `json:"is_witness"` + Count int `json:"count"` + LabelConstraints []LabelConstraint `json:"label_constraints,omitempty"` + LocationLabels []string `json:"location_labels,omitempty"` + IsolationLevel string `json:"isolation_level,omitempty"` +} + +// MarshalJSON implements `json.Marshaler` interface to make sure we could set the correct start/end key. +func (r *Rule) MarshalJSON() ([]byte, error) { + tempRule := &rule{ + GroupID: r.GroupID, + ID: r.ID, + Index: r.Index, + Override: r.Override, + StartKeyHex: r.StartKeyHex, + EndKeyHex: r.EndKeyHex, + Role: r.Role, + IsWitness: r.IsWitness, + Count: r.Count, + LabelConstraints: r.LabelConstraints, + LocationLabels: r.LocationLabels, + IsolationLevel: r.IsolationLevel, + } + // Converts the start/end key to hex format if the corresponding hex field is empty. + if len(r.StartKey) > 0 && len(r.StartKeyHex) == 0 { + tempRule.StartKeyHex = rawKeyToKeyHexStr(r.StartKey) + } + if len(r.EndKey) > 0 && len(r.EndKeyHex) == 0 { + tempRule.EndKeyHex = rawKeyToKeyHexStr(r.EndKey) + } + return json.Marshal(tempRule) +} + +// UnmarshalJSON implements `json.Unmarshaler` interface to make sure we could get the correct start/end key. +func (r *Rule) UnmarshalJSON(bytes []byte) error { + var tempRule rule + err := json.Unmarshal(bytes, &tempRule) + if err != nil { + return err + } + newRule := Rule{ + GroupID: tempRule.GroupID, + ID: tempRule.ID, + Index: tempRule.Index, + Override: tempRule.Override, + StartKeyHex: tempRule.StartKeyHex, + EndKeyHex: tempRule.EndKeyHex, + Role: tempRule.Role, + IsWitness: tempRule.IsWitness, + Count: tempRule.Count, + LabelConstraints: tempRule.LabelConstraints, + LocationLabels: tempRule.LocationLabels, + IsolationLevel: tempRule.IsolationLevel, + } + newRule.StartKey, err = keyHexStrToRawKey(newRule.StartKeyHex) + if err != nil { + return err + } + newRule.EndKey, err = keyHexStrToRawKey(newRule.EndKeyHex) + if err != nil { + return err + } + *r = newRule + return nil +} + // RuleOpType indicates the operation type type RuleOpType string @@ -364,6 +444,94 @@ func (r RuleOp) String() string { return string(b) } +var ( + _ json.Marshaler = (*RuleOp)(nil) + _ json.Unmarshaler = (*RuleOp)(nil) +) + +// This is a helper struct used to customizing the JSON marshal/unmarshal methods of `RuleOp`. +type ruleOp struct { + GroupID string `json:"group_id"` + ID string `json:"id"` + Index int `json:"index,omitempty"` + Override bool `json:"override,omitempty"` + StartKeyHex string `json:"start_key"` + EndKeyHex string `json:"end_key"` + Role PeerRoleType `json:"role"` + IsWitness bool `json:"is_witness"` + Count int `json:"count"` + LabelConstraints []LabelConstraint `json:"label_constraints,omitempty"` + LocationLabels []string `json:"location_labels,omitempty"` + IsolationLevel string `json:"isolation_level,omitempty"` + Action RuleOpType `json:"action"` + DeleteByIDPrefix bool `json:"delete_by_id_prefix"` +} + +// MarshalJSON implements `json.Marshaler` interface to make sure we could set the correct start/end key. +func (r *RuleOp) MarshalJSON() ([]byte, error) { + tempRuleOp := &ruleOp{ + GroupID: r.GroupID, + ID: r.ID, + Index: r.Index, + Override: r.Override, + StartKeyHex: r.StartKeyHex, + EndKeyHex: r.EndKeyHex, + Role: r.Role, + IsWitness: r.IsWitness, + Count: r.Count, + LabelConstraints: r.LabelConstraints, + LocationLabels: r.LocationLabels, + IsolationLevel: r.IsolationLevel, + Action: r.Action, + DeleteByIDPrefix: r.DeleteByIDPrefix, + } + // Converts the start/end key to hex format if the corresponding hex field is empty. + if len(r.StartKey) > 0 && len(r.StartKeyHex) == 0 { + tempRuleOp.StartKeyHex = rawKeyToKeyHexStr(r.StartKey) + } + if len(r.EndKey) > 0 && len(r.EndKeyHex) == 0 { + tempRuleOp.EndKeyHex = rawKeyToKeyHexStr(r.EndKey) + } + return json.Marshal(tempRuleOp) +} + +// UnmarshalJSON implements `json.Unmarshaler` interface to make sure we could get the correct start/end key. +func (r *RuleOp) UnmarshalJSON(bytes []byte) error { + var tempRuleOp ruleOp + err := json.Unmarshal(bytes, &tempRuleOp) + if err != nil { + return err + } + newRuleOp := RuleOp{ + Rule: &Rule{ + GroupID: tempRuleOp.GroupID, + ID: tempRuleOp.ID, + Index: tempRuleOp.Index, + Override: tempRuleOp.Override, + StartKeyHex: tempRuleOp.StartKeyHex, + EndKeyHex: tempRuleOp.EndKeyHex, + Role: tempRuleOp.Role, + IsWitness: tempRuleOp.IsWitness, + Count: tempRuleOp.Count, + LabelConstraints: tempRuleOp.LabelConstraints, + LocationLabels: tempRuleOp.LocationLabels, + IsolationLevel: tempRuleOp.IsolationLevel, + }, + Action: tempRuleOp.Action, + DeleteByIDPrefix: tempRuleOp.DeleteByIDPrefix, + } + newRuleOp.StartKey, err = keyHexStrToRawKey(newRuleOp.StartKeyHex) + if err != nil { + return err + } + newRuleOp.EndKey, err = keyHexStrToRawKey(newRuleOp.EndKeyHex) + if err != nil { + return err + } + *r = newRuleOp + return nil +} + // RuleGroup defines properties of a rule group. type RuleGroup struct { ID string `json:"id,omitempty"` diff --git a/client/http/types_test.go b/client/http/types_test.go index 0dfebacbdcf..74482e29c3c 100644 --- a/client/http/types_test.go +++ b/client/http/types_test.go @@ -15,6 +15,7 @@ package http import ( + "encoding/json" "testing" "github.com/stretchr/testify/require" @@ -47,3 +48,153 @@ func TestMergeRegionsInfo(t *testing.T) { re.Equal(2, len(regionsInfo.Regions)) re.Equal(append(regionsInfo1.Regions, regionsInfo2.Regions...), regionsInfo.Regions) } + +func TestRuleStartEndKey(t *testing.T) { + re := require.New(t) + // Empty start/end key and key hex. + ruleToMarshal := &Rule{} + rule := mustMarshalAndUnmarshal(re, ruleToMarshal) + re.Equal("", rule.StartKeyHex) + re.Equal("", rule.EndKeyHex) + re.Equal([]byte(""), rule.StartKey) + re.Equal([]byte(""), rule.EndKey) + // Empty start/end key and non-empty key hex. + ruleToMarshal = &Rule{ + StartKeyHex: rawKeyToKeyHexStr([]byte("a")), + EndKeyHex: rawKeyToKeyHexStr([]byte("b")), + } + rule = mustMarshalAndUnmarshal(re, ruleToMarshal) + re.Equal([]byte("a"), rule.StartKey) + re.Equal([]byte("b"), rule.EndKey) + re.Equal(ruleToMarshal.StartKeyHex, rule.StartKeyHex) + re.Equal(ruleToMarshal.EndKeyHex, rule.EndKeyHex) + // Non-empty start/end key and empty key hex. + ruleToMarshal = &Rule{ + StartKey: []byte("a"), + EndKey: []byte("b"), + } + rule = mustMarshalAndUnmarshal(re, ruleToMarshal) + re.Equal(ruleToMarshal.StartKey, rule.StartKey) + re.Equal(ruleToMarshal.EndKey, rule.EndKey) + re.Equal(rawKeyToKeyHexStr(ruleToMarshal.StartKey), rule.StartKeyHex) + re.Equal(rawKeyToKeyHexStr(ruleToMarshal.EndKey), rule.EndKeyHex) + // Non-empty start/end key and non-empty key hex. + ruleToMarshal = &Rule{ + StartKey: []byte("a"), + EndKey: []byte("b"), + StartKeyHex: rawKeyToKeyHexStr([]byte("c")), + EndKeyHex: rawKeyToKeyHexStr([]byte("d")), + } + rule = mustMarshalAndUnmarshal(re, ruleToMarshal) + re.Equal([]byte("c"), rule.StartKey) + re.Equal([]byte("d"), rule.EndKey) + re.Equal(ruleToMarshal.StartKeyHex, rule.StartKeyHex) + re.Equal(ruleToMarshal.EndKeyHex, rule.EndKeyHex) + // Half of each pair of keys is empty. + ruleToMarshal = &Rule{ + StartKey: []byte("a"), + EndKeyHex: rawKeyToKeyHexStr([]byte("d")), + } + rule = mustMarshalAndUnmarshal(re, ruleToMarshal) + re.Equal(ruleToMarshal.StartKey, rule.StartKey) + re.Equal([]byte("d"), rule.EndKey) + re.Equal(rawKeyToKeyHexStr(ruleToMarshal.StartKey), rule.StartKeyHex) + re.Equal(ruleToMarshal.EndKeyHex, rule.EndKeyHex) +} + +func mustMarshalAndUnmarshal(re *require.Assertions, rule *Rule) *Rule { + ruleJSON, err := json.Marshal(rule) + re.NoError(err) + var newRule *Rule + err = json.Unmarshal(ruleJSON, &newRule) + re.NoError(err) + return newRule +} + +func TestRuleOpStartEndKey(t *testing.T) { + re := require.New(t) + // Empty start/end key and key hex. + ruleOpToMarshal := &RuleOp{ + Rule: &Rule{}, + } + ruleOp := mustMarshalAndUnmarshalRuleOp(re, ruleOpToMarshal) + re.Equal("", ruleOp.StartKeyHex) + re.Equal("", ruleOp.EndKeyHex) + re.Equal([]byte(""), ruleOp.StartKey) + re.Equal([]byte(""), ruleOp.EndKey) + // Empty start/end key and non-empty key hex. + ruleOpToMarshal = &RuleOp{ + Rule: &Rule{ + StartKeyHex: rawKeyToKeyHexStr([]byte("a")), + EndKeyHex: rawKeyToKeyHexStr([]byte("b")), + }, + Action: RuleOpAdd, + DeleteByIDPrefix: true, + } + ruleOp = mustMarshalAndUnmarshalRuleOp(re, ruleOpToMarshal) + re.Equal([]byte("a"), ruleOp.StartKey) + re.Equal([]byte("b"), ruleOp.EndKey) + re.Equal(ruleOpToMarshal.StartKeyHex, ruleOp.StartKeyHex) + re.Equal(ruleOpToMarshal.EndKeyHex, ruleOp.EndKeyHex) + re.Equal(ruleOpToMarshal.Action, ruleOp.Action) + re.Equal(ruleOpToMarshal.DeleteByIDPrefix, ruleOp.DeleteByIDPrefix) + // Non-empty start/end key and empty key hex. + ruleOpToMarshal = &RuleOp{ + Rule: &Rule{ + StartKey: []byte("a"), + EndKey: []byte("b"), + }, + Action: RuleOpAdd, + DeleteByIDPrefix: true, + } + ruleOp = mustMarshalAndUnmarshalRuleOp(re, ruleOpToMarshal) + re.Equal(ruleOpToMarshal.StartKey, ruleOp.StartKey) + re.Equal(ruleOpToMarshal.EndKey, ruleOp.EndKey) + re.Equal(rawKeyToKeyHexStr(ruleOpToMarshal.StartKey), ruleOp.StartKeyHex) + re.Equal(rawKeyToKeyHexStr(ruleOpToMarshal.EndKey), ruleOp.EndKeyHex) + re.Equal(ruleOpToMarshal.Action, ruleOp.Action) + re.Equal(ruleOpToMarshal.DeleteByIDPrefix, ruleOp.DeleteByIDPrefix) + // Non-empty start/end key and non-empty key hex. + ruleOpToMarshal = &RuleOp{ + Rule: &Rule{ + StartKey: []byte("a"), + EndKey: []byte("b"), + StartKeyHex: rawKeyToKeyHexStr([]byte("c")), + EndKeyHex: rawKeyToKeyHexStr([]byte("d")), + }, + Action: RuleOpAdd, + DeleteByIDPrefix: true, + } + ruleOp = mustMarshalAndUnmarshalRuleOp(re, ruleOpToMarshal) + re.Equal([]byte("c"), ruleOp.StartKey) + re.Equal([]byte("d"), ruleOp.EndKey) + re.Equal(ruleOpToMarshal.StartKeyHex, ruleOp.StartKeyHex) + re.Equal(ruleOpToMarshal.EndKeyHex, ruleOp.EndKeyHex) + re.Equal(ruleOpToMarshal.Action, ruleOp.Action) + re.Equal(ruleOpToMarshal.DeleteByIDPrefix, ruleOp.DeleteByIDPrefix) + // Half of each pair of keys is empty. + ruleOpToMarshal = &RuleOp{ + Rule: &Rule{ + StartKey: []byte("a"), + EndKeyHex: rawKeyToKeyHexStr([]byte("d")), + }, + Action: RuleOpDel, + DeleteByIDPrefix: false, + } + ruleOp = mustMarshalAndUnmarshalRuleOp(re, ruleOpToMarshal) + re.Equal(ruleOpToMarshal.StartKey, ruleOp.StartKey) + re.Equal([]byte("d"), ruleOp.EndKey) + re.Equal(rawKeyToKeyHexStr(ruleOpToMarshal.StartKey), ruleOp.StartKeyHex) + re.Equal(ruleOpToMarshal.EndKeyHex, ruleOp.EndKeyHex) + re.Equal(ruleOpToMarshal.Action, ruleOp.Action) + re.Equal(ruleOpToMarshal.DeleteByIDPrefix, ruleOp.DeleteByIDPrefix) +} + +func mustMarshalAndUnmarshalRuleOp(re *require.Assertions, ruleOp *RuleOp) *RuleOp { + ruleOpJSON, err := json.Marshal(ruleOp) + re.NoError(err) + var newRuleOp *RuleOp + err = json.Unmarshal(ruleOpJSON, &newRuleOp) + re.NoError(err) + return newRuleOp +} diff --git a/server/api/stats.go b/server/api/stats.go index 1798597b6cc..915d33ddfdf 100644 --- a/server/api/stats.go +++ b/server/api/stats.go @@ -36,8 +36,9 @@ func newStatsHandler(svr *server.Server, rd *render.Render) *statsHandler { // @Tags stats // @Summary Get region statistics of a specified range. -// @Param start_key query string true "Start key" -// @Param end_key query string true "End key" +// @Param start_key query string true "Start key" +// @Param end_key query string true "End key" +// @Param count query bool false "Whether only count the number of regions" // @Produce json // @Success 200 {object} statistics.RegionStats // @Router /stats/region [get] diff --git a/server/api/store.go b/server/api/store.go index a44850d35cc..8537cd45c5b 100644 --- a/server/api/store.go +++ b/server/api/store.go @@ -172,14 +172,14 @@ func newStoreHandler(handler *server.Handler, rd *render.Render) *storeHandler { } } -// @Tags store +// @Tags store // @Summary Get a store's information. // @Param id path integer true "Store Id" -// @Produce json +// @Produce json // @Success 200 {object} StoreInfo // @Failure 400 {string} string "The input is invalid." // @Failure 404 {string} string "The store does not exist." -// @Failure 500 {string} string "PD server failed to proceed the request." +// @Failure 500 {string} string "PD server failed to proceed the request." // @Router /store/{id} [get] func (h *storeHandler) GetStore(w http.ResponseWriter, r *http.Request) { rc := getCluster(r) @@ -735,13 +735,13 @@ func (h *storesHandler) GetStoresProgress(w http.ResponseWriter, r *http.Request } // @Tags store -// @Summary Get all stores in the cluster. -// @Param state query array true "Specify accepted store states." +// @Summary Get all stores in the cluster. +// @Param state query array true "Specify accepted store states." // @Produce json -// @Success 200 {object} StoresInfo +// @Success 200 {object} StoresInfo // @Failure 500 {string} string "PD server failed to proceed the request." -// @Router /stores [get] -// @Deprecated Better to use /stores/check instead. +// @Router /stores [get] +// @Deprecated Better to use /stores/check instead. func (h *storesHandler) GetAllStores(w http.ResponseWriter, r *http.Request) { rc := getCluster(r) stores := rc.GetMetaStores() diff --git a/tests/integrations/client/http_client_test.go b/tests/integrations/client/http_client_test.go index a007b893187..6c636d2a2a1 100644 --- a/tests/integrations/client/http_client_test.go +++ b/tests/integrations/client/http_client_test.go @@ -105,9 +105,17 @@ func (suite *httpClientTestSuite) TestMeta() { re.NoError(err) re.Equal(int64(2), regions.Count) re.Len(regions.Regions, 2) - regionStats, err := suite.client.GetRegionStatusByKeyRange(suite.ctx, pd.NewKeyRange([]byte("a1"), []byte("a3"))) + state, err := suite.client.GetRegionsReplicatedStateByKeyRange(suite.ctx, pd.NewKeyRange([]byte("a1"), []byte("a3"))) re.NoError(err) - re.Equal(2, regionStats.Count) + re.Equal("INPROGRESS", state) + regionStats, err := suite.client.GetRegionStatusByKeyRange(suite.ctx, pd.NewKeyRange([]byte("a1"), []byte("a3")), false) + re.NoError(err) + re.Greater(regionStats.Count, 0) + re.NotEmpty(regionStats.StoreLeaderCount) + regionStats, err = suite.client.GetRegionStatusByKeyRange(suite.ctx, pd.NewKeyRange([]byte("a1"), []byte("a3")), true) + re.NoError(err) + re.Greater(regionStats.Count, 0) + re.Empty(regionStats.StoreLeaderCount) hotReadRegions, err := suite.client.GetHotReadRegions(suite.ctx) re.NoError(err) re.Len(hotReadRegions.AsPeer, 1) @@ -170,18 +178,22 @@ func (suite *httpClientTestSuite) TestRule() { re.Equal(bundles[0], bundle) // Check if we have the default rule. suite.checkRule(re, &pd.Rule{ - GroupID: placement.DefaultGroupID, - ID: placement.DefaultRuleID, - Role: pd.Voter, - Count: 3, + GroupID: placement.DefaultGroupID, + ID: placement.DefaultRuleID, + Role: pd.Voter, + Count: 3, + StartKey: []byte{}, + EndKey: []byte{}, }, 1, true) // Should be the same as the rules in the bundle. suite.checkRule(re, bundle.Rules[0], 1, true) testRule := &pd.Rule{ - GroupID: placement.DefaultGroupID, - ID: "test", - Role: pd.Voter, - Count: 3, + GroupID: placement.DefaultGroupID, + ID: "test", + Role: pd.Voter, + Count: 3, + StartKey: []byte{}, + EndKey: []byte{}, } err = suite.client.SetPlacementRule(suite.ctx, testRule) re.NoError(err) @@ -233,6 +245,18 @@ func (suite *httpClientTestSuite) TestRule() { ruleGroup, err = suite.client.GetPlacementRuleGroupByID(suite.ctx, testRuleGroup.ID) re.ErrorContains(err, http.StatusText(http.StatusNotFound)) re.Empty(ruleGroup) + // Test the start key and end key. + testRule = &pd.Rule{ + GroupID: placement.DefaultGroupID, + ID: "test", + Role: pd.Voter, + Count: 5, + StartKey: []byte("a1"), + EndKey: []byte(""), + } + err = suite.client.SetPlacementRule(suite.ctx, testRule) + re.NoError(err) + suite.checkRule(re, testRule, 1, true) } func (suite *httpClientTestSuite) checkRule( @@ -262,6 +286,8 @@ func checkRuleFunc( re.Equal(rule.ID, r.ID) re.Equal(rule.Role, r.Role) re.Equal(rule.Count, r.Count) + re.Equal(rule.StartKey, r.StartKey) + re.Equal(rule.EndKey, r.EndKey) return } if exist { From 5aacd664ce83beb36f35f0e6ddd9735369e985c9 Mon Sep 17 00:00:00 2001 From: Yongbo Jiang Date: Thu, 30 Nov 2023 16:34:18 +0800 Subject: [PATCH 068/137] syncer: add region syncer client status (#7461) ref tikv/pd#7431 Signed-off-by: Cabinfever_B Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- pkg/syncer/client.go | 15 ++++++++++++- pkg/syncer/server.go | 14 ++++++++++++- server/server.go | 6 ++++++ .../region_syncer/region_syncer_test.go | 21 ++++++++++++++++--- 4 files changed, 51 insertions(+), 5 deletions(-) diff --git a/pkg/syncer/client.go b/pkg/syncer/client.go index f61ce320a74..00dd8c5107d 100644 --- a/pkg/syncer/client.go +++ b/pkg/syncer/client.go @@ -19,6 +19,8 @@ import ( "time" "github.com/docker/go-units" + "github.com/pingcap/errors" + "github.com/pingcap/failpoint" "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/kvproto/pkg/pdpb" "github.com/pingcap/log" @@ -77,6 +79,11 @@ func (s *RegionSyncer) syncRegion(ctx context.Context, conn *grpc.ClientConn) (C var regionGuide = core.GenerateRegionGuideFunc(false) +// IsRunning returns whether the region syncer client is running. +func (s *RegionSyncer) IsRunning() bool { + return s.streamingRunning.Load() +} + // StartSyncWithLeader starts to sync with leader. func (s *RegionSyncer) StartSyncWithLeader(addr string) { s.wg.Add(1) @@ -89,6 +96,7 @@ func (s *RegionSyncer) StartSyncWithLeader(addr string) { go func() { defer logutil.LogPanic() defer s.wg.Done() + defer s.streamingRunning.Store(false) // used to load region from kv storage to cache storage. bc := s.server.GetBasicCluster() regionStorage := s.server.GetStorage() @@ -132,6 +140,9 @@ func (s *RegionSyncer) StartSyncWithLeader(addr string) { } stream, err := s.syncRegion(ctx, conn) + failpoint.Inject("disableClientStreaming", func() { + err = errors.Errorf("no stream") + }) if err != nil { if ev, ok := status.FromError(err); ok { if ev.Code() == codes.Canceled { @@ -142,11 +153,11 @@ func (s *RegionSyncer) StartSyncWithLeader(addr string) { time.Sleep(time.Second) continue } - log.Info("server starts to synchronize with leader", zap.String("server", s.server.Name()), zap.String("leader", s.server.GetLeader().GetName()), zap.Uint64("request-index", s.history.GetNextIndex())) for { resp, err := stream.Recv() if err != nil { + s.streamingRunning.Store(false) log.Error("region sync with leader meet error", errs.ZapError(errs.ErrGRPCRecv, err)) if err = stream.CloseSend(); err != nil { log.Error("failed to terminate client stream", errs.ZapError(errs.ErrGRPCCloseSend, err)) @@ -212,6 +223,8 @@ func (s *RegionSyncer) StartSyncWithLeader(addr string) { _ = regionStorage.DeleteRegion(old.GetMeta()) } } + // mark the client as running status when it finished the first history region sync. + s.streamingRunning.Store(true) } } }() diff --git a/pkg/syncer/server.go b/pkg/syncer/server.go index 7d339e75dbe..4fb38614de0 100644 --- a/pkg/syncer/server.go +++ b/pkg/syncer/server.go @@ -18,6 +18,7 @@ import ( "context" "io" "sync" + "sync/atomic" "time" "github.com/docker/go-units" @@ -83,6 +84,8 @@ type RegionSyncer struct { history *historyBuffer limit *ratelimit.RateLimiter tlsConfig *grpcutil.TLSConfig + // status when as client + streamingRunning atomic.Bool } // NewRegionSyncer returns a region syncer. @@ -228,7 +231,16 @@ func (s *RegionSyncer) syncHistoryRegion(ctx context.Context, request *pdpb.Sync if s.history.GetNextIndex() == startIndex { log.Info("requested server has already in sync with server", zap.String("requested-server", name), zap.String("server", s.server.Name()), zap.Uint64("last-index", startIndex)) - return nil + // still send a response to follower to show the history region sync. + resp := &pdpb.SyncRegionResponse{ + Header: &pdpb.ResponseHeader{ClusterId: s.server.ClusterID()}, + Regions: nil, + StartIndex: startIndex, + RegionStats: nil, + RegionLeaders: nil, + Buckets: nil, + } + return stream.Send(resp) } // do full synchronization if startIndex == 0 { diff --git a/server/server.go b/server/server.go index 76893c24388..43daa65d844 100644 --- a/server/server.go +++ b/server/server.go @@ -1363,6 +1363,12 @@ func (s *Server) GetRaftCluster() *cluster.RaftCluster { return s.cluster } +// DirectlyGetRaftCluster returns raft cluster directly. +// Only used for test. +func (s *Server) DirectlyGetRaftCluster() *cluster.RaftCluster { + return s.cluster +} + // GetCluster gets cluster. func (s *Server) GetCluster() *metapb.Cluster { return &metapb.Cluster{ diff --git a/tests/server/region_syncer/region_syncer_test.go b/tests/server/region_syncer/region_syncer_test.go index 6521432c0dc..87b5c0683c7 100644 --- a/tests/server/region_syncer/region_syncer_test.go +++ b/tests/server/region_syncer/region_syncer_test.go @@ -47,21 +47,34 @@ func (i *idAllocator) alloc() uint64 { func TestRegionSyncer(t *testing.T) { re := require.New(t) ctx, cancel := context.WithCancel(context.Background()) - defer cancel() re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/storage/regionStorageFastFlush", `return(true)`)) re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/syncer/noFastExitSync", `return(true)`)) + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/syncer/disableClientStreaming", `return(true)`)) cluster, err := tests.NewTestCluster(ctx, 3, func(conf *config.Config, serverName string) { conf.PDServerCfg.UseRegionStorage = true }) - defer cluster.Destroy() + defer func() { + cluster.Destroy() + cancel() + }() re.NoError(err) re.NoError(cluster.RunInitialServers()) cluster.WaitLeader() leaderServer := cluster.GetLeaderServer() + re.NoError(leaderServer.BootstrapCluster()) rc := leaderServer.GetServer().GetRaftCluster() re.NotNil(rc) + followerServer := cluster.GetServer(cluster.GetFollower()) + + testutil.Eventually(re, func() bool { + return !followerServer.GetServer().DirectlyGetRaftCluster().GetRegionSyncer().IsRunning() + }) + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/syncer/disableClientStreaming")) re.True(cluster.WaitRegionSyncerClientsReady(2)) + testutil.Eventually(re, func() bool { + return followerServer.GetServer().DirectlyGetRaftCluster().GetRegionSyncer().IsRunning() + }) regionLen := 110 regions := initRegions(regionLen) @@ -119,7 +132,6 @@ func TestRegionSyncer(t *testing.T) { time.Sleep(4 * time.Second) // test All regions have been synchronized to the cache of followerServer - followerServer := cluster.GetServer(cluster.GetFollower()) re.NotNil(followerServer) cacheRegions := leaderServer.GetServer().GetBasicCluster().GetRegions() re.Len(cacheRegions, regionLen) @@ -141,6 +153,9 @@ func TestRegionSyncer(t *testing.T) { re.NoError(err) cluster.WaitLeader() leaderServer = cluster.GetLeaderServer() + testutil.Eventually(re, func() bool { + return !leaderServer.GetServer().GetRaftCluster().GetRegionSyncer().IsRunning() + }) re.NotNil(leaderServer) loadRegions := leaderServer.GetServer().GetRaftCluster().GetRegions() re.Len(loadRegions, regionLen) From d91886f29a272737541bec41e100ea879bc63b44 Mon Sep 17 00:00:00 2001 From: Yongbo Jiang Date: Thu, 30 Nov 2023 16:50:48 +0800 Subject: [PATCH 069/137] *: support forward GetMinTS (#7482) close tikv/pd#7467 Signed-off-by: Cabinfever_B Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- client/client.go | 17 ++++------------- server/grpc_service.go | 11 +++++++---- tests/integrations/mcs/testutil.go | 2 +- tests/integrations/tso/client_test.go | 15 +++++++++++++-- 4 files changed, 25 insertions(+), 20 deletions(-) diff --git a/client/client.go b/client/client.go index b320be6d3d5..7053ed2be96 100644 --- a/client/client.go +++ b/client/client.go @@ -813,17 +813,6 @@ func (c *client) backupClientConn() (*grpc.ClientConn, string) { return nil, "" } -func (c *client) getClient() pdpb.PDClient { - if c.option.enableForwarding && atomic.LoadInt32(&c.leaderNetworkFailure) == 1 { - backupClientConn, addr := c.backupClientConn() - if backupClientConn != nil { - log.Debug("[pd] use follower client", zap.String("addr", addr)) - return pdpb.NewPDClient(backupClientConn) - } - } - return c.leaderClient() -} - func (c *client) getClientAndContext(ctx context.Context) (pdpb.PDClient, context.Context) { if c.option.enableForwarding && atomic.LoadInt32(&c.leaderNetworkFailure) == 1 { backupClientConn, addr := c.backupClientConn() @@ -892,16 +881,18 @@ func (c *client) GetMinTS(ctx context.Context) (physical int64, logical int64, e default: return 0, 0, errs.ErrClientGetMinTSO.FastGenByArgs("undefined service mode") } - + ctx, cancel := context.WithTimeout(ctx, c.option.timeout) // Call GetMinTS API to get the minimal TS from the API leader. - protoClient := c.getClient() + protoClient, ctx := c.getClientAndContext(ctx) if protoClient == nil { + cancel() return 0, 0, errs.ErrClientGetProtoClient } resp, err := protoClient.GetMinTS(ctx, &pdpb.GetMinTSRequest{ Header: c.requestHeader(), }) + cancel() if err != nil { if strings.Contains(err.Error(), "Unimplemented") { // If the method is not supported, we fallback to GetTS. diff --git a/server/grpc_service.go b/server/grpc_service.go index b0384a7d629..bb20fcbe484 100644 --- a/server/grpc_service.go +++ b/server/grpc_service.go @@ -271,10 +271,13 @@ func (s *GrpcServer) GetClusterInfo(ctx context.Context, _ *pdpb.GetClusterInfoR func (s *GrpcServer) GetMinTS( ctx context.Context, request *pdpb.GetMinTSRequest, ) (*pdpb.GetMinTSResponse, error) { - if err := s.validateRequest(request.GetHeader()); err != nil { - return &pdpb.GetMinTSResponse{ - Header: s.wrapErrorToHeader(pdpb.ErrorType_UNKNOWN, err.Error()), - }, nil + fn := func(ctx context.Context, client *grpc.ClientConn) (interface{}, error) { + return pdpb.NewPDClient(client).GetMinTS(ctx, request) + } + if rsp, err := s.unaryMiddleware(ctx, request, fn); err != nil { + return nil, err + } else if rsp != nil { + return rsp.(*pdpb.GetMinTSResponse), nil } var ( diff --git a/tests/integrations/mcs/testutil.go b/tests/integrations/mcs/testutil.go index bbedd65209d..d23da905f78 100644 --- a/tests/integrations/mcs/testutil.go +++ b/tests/integrations/mcs/testutil.go @@ -106,7 +106,7 @@ func WaitForMultiKeyspacesTSOAvailable( clients := make([]pd.Client, 0, len(keyspaceIDs)) for _, keyspaceID := range keyspaceIDs { - cli := SetupClientWithKeyspaceID(ctx, re, keyspaceID, backendEndpoints) + cli := SetupClientWithKeyspaceID(ctx, re, keyspaceID, backendEndpoints, pd.WithForwardingOption(true)) re.NotNil(cli) clients = append(clients, cli) diff --git a/tests/integrations/tso/client_test.go b/tests/integrations/tso/client_test.go index 8ff20d68f52..b021e73a2f9 100644 --- a/tests/integrations/tso/client_test.go +++ b/tests/integrations/tso/client_test.go @@ -98,7 +98,7 @@ func (suite *tsoClientTestSuite) SetupSuite() { suite.keyspaceIDs = make([]uint32, 0) if suite.legacy { - client, err := pd.NewClientWithContext(suite.ctx, strings.Split(suite.backendEndpoints, ","), pd.SecurityOption{}) + client, err := pd.NewClientWithContext(suite.ctx, strings.Split(suite.backendEndpoints, ","), pd.SecurityOption{}, pd.WithForwardingOption(true)) re.NoError(err) innerClient, ok := client.(interface{ GetServiceDiscovery() pd.ServiceDiscovery }) re.True(ok) @@ -263,7 +263,9 @@ func (suite *tsoClientTestSuite) TestDiscoverTSOServiceWithLegacyPath() { // TestGetMinTS tests the correctness of GetMinTS. func (suite *tsoClientTestSuite) TestGetMinTS() { re := suite.Require() - suite.waitForAllKeyspaceGroupsInServing(re) + if !suite.legacy { + suite.waitForAllKeyspaceGroupsInServing(re) + } var wg sync.WaitGroup wg.Add(tsoRequestConcurrencyNumber * len(suite.clients)) @@ -293,6 +295,15 @@ func (suite *tsoClientTestSuite) TestGetMinTS() { } } wg.Wait() + + re.NoError(failpoint.Enable("github.com/tikv/pd/client/unreachableNetwork1", "return(true)")) + time.Sleep(time.Second) + testutil.Eventually(re, func() bool { + var err error + _, _, err = suite.clients[0].GetMinTS(suite.ctx) + return err == nil + }) + re.NoError(failpoint.Disable("github.com/tikv/pd/client/unreachableNetwork1")) } // More details can be found in this issue: https://github.com/tikv/pd/issues/4884 From 3191594c91775dbf4679f48dda63a74d1de43176 Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Thu, 30 Nov 2023 17:05:48 +0800 Subject: [PATCH 070/137] *: make TestDisable stable (#7487) close tikv/pd#7468 Signed-off-by: Ryan Leung Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- pkg/schedule/coordinator.go | 1 + server/cluster/cluster.go | 5 +++++ tests/server/api/scheduler_test.go | 2 ++ 3 files changed, 8 insertions(+) diff --git a/pkg/schedule/coordinator.go b/pkg/schedule/coordinator.go index 6a02e68811d..e276589a4e9 100644 --- a/pkg/schedule/coordinator.go +++ b/pkg/schedule/coordinator.go @@ -513,6 +513,7 @@ func (c *Coordinator) InitSchedulers(needRun bool) { if err := c.cluster.GetSchedulerConfig().Persist(c.cluster.GetStorage()); err != nil { log.Error("cannot persist schedule config", errs.ZapError(err)) } + log.Info("scheduler config is updated", zap.Reflect("scheduler-config", scheduleCfg.Schedulers)) c.markSchedulersInitialized() } diff --git a/server/cluster/cluster.go b/server/cluster/cluster.go index ec8ca3a0d65..78f6ddd4364 100644 --- a/server/cluster/cluster.go +++ b/server/cluster/cluster.go @@ -467,6 +467,10 @@ func (c *RaftCluster) startGCTuner() { func (c *RaftCluster) runStoreConfigSync() { defer logutil.LogPanic() defer c.wg.Done() + // TODO: After we fix the atomic problem of config, we can remove this failpoint. + failpoint.Inject("skipStoreConfigSync", func() { + failpoint.Return() + }) var ( synced, switchRaftV2Config, needPersist bool @@ -491,6 +495,7 @@ func (c *RaftCluster) runStoreConfigSync() { if err := c.opt.Persist(c.storage); err != nil { log.Warn("store config persisted failed", zap.Error(err)) } + log.Info("store config is updated") } select { case <-c.ctx.Done(): diff --git a/tests/server/api/scheduler_test.go b/tests/server/api/scheduler_test.go index 2388f95e9df..69ee37d49e8 100644 --- a/tests/server/api/scheduler_test.go +++ b/tests/server/api/scheduler_test.go @@ -566,8 +566,10 @@ func (suite *scheduleTestSuite) checkAPI(cluster *tests.TestCluster) { } func (suite *scheduleTestSuite) TestDisable() { + suite.NoError(failpoint.Enable("github.com/tikv/pd/server/cluster/skipStoreConfigSync", `return(true)`)) env := tests.NewSchedulingTestEnvironment(suite.T()) env.RunTestInTwoModes(suite.checkDisable) + suite.NoError(failpoint.Disable("github.com/tikv/pd/server/cluster/skipStoreConfigSync")) } func (suite *scheduleTestSuite) checkDisable(cluster *tests.TestCluster) { From 259435d93ae2e081dd706545350f214bb8788256 Mon Sep 17 00:00:00 2001 From: glorv Date: Mon, 4 Dec 2023 11:46:22 +0800 Subject: [PATCH 071/137] client: return total wait duration in resource interceptor OnRequestWait call (#7488) ref tikv/pd#5851 Signed-off-by: glorv --- .../resource_group/controller/controller.go | 15 ++++---- .../controller/controller_test.go | 2 +- .../resourcemanager/resource_manager_test.go | 34 +++++++++---------- 3 files changed, 27 insertions(+), 24 deletions(-) diff --git a/client/resource_group/controller/controller.go b/client/resource_group/controller/controller.go index 01011c2c30a..56d9ef9bd1b 100755 --- a/client/resource_group/controller/controller.go +++ b/client/resource_group/controller/controller.go @@ -57,7 +57,7 @@ const ( // ResourceGroupKVInterceptor is used as quota limit controller for resource group using kv store. type ResourceGroupKVInterceptor interface { // OnRequestWait is used to check whether resource group has enough tokens. It maybe needs to wait some time. - OnRequestWait(ctx context.Context, resourceGroupName string, info RequestInfo) (*rmpb.Consumption, *rmpb.Consumption, uint32, error) + OnRequestWait(ctx context.Context, resourceGroupName string, info RequestInfo) (*rmpb.Consumption, *rmpb.Consumption, time.Duration, uint32, error) // OnResponse is used to consume tokens after receiving response. OnResponse(resourceGroupName string, req RequestInfo, resp ResponseInfo) (*rmpb.Consumption, error) // IsBackgroundRequest If the resource group has background jobs, we should not record consumption and wait for it. @@ -526,10 +526,10 @@ func (c *ResourceGroupsController) sendTokenBucketRequests(ctx context.Context, // OnRequestWait is used to check whether resource group has enough tokens. It maybe needs to wait some time. func (c *ResourceGroupsController) OnRequestWait( ctx context.Context, resourceGroupName string, info RequestInfo, -) (*rmpb.Consumption, *rmpb.Consumption, uint32, error) { +) (*rmpb.Consumption, *rmpb.Consumption, time.Duration, uint32, error) { gc, err := c.tryGetResourceGroup(ctx, resourceGroupName) if err != nil { - return nil, nil, 0, err + return nil, nil, time.Duration(0), 0, err } return gc.onRequestWait(ctx, info) } @@ -1176,7 +1176,7 @@ func (gc *groupCostController) calcRequest(counter *tokenCounter) float64 { func (gc *groupCostController) onRequestWait( ctx context.Context, info RequestInfo, -) (*rmpb.Consumption, *rmpb.Consumption, uint32, error) { +) (*rmpb.Consumption, *rmpb.Consumption, time.Duration, uint32, error) { delta := &rmpb.Consumption{} for _, calc := range gc.calculators { calc.BeforeKVRequest(delta, info) @@ -1185,6 +1185,7 @@ func (gc *groupCostController) onRequestWait( gc.mu.Lock() add(gc.mu.consumption, delta) gc.mu.Unlock() + var waitDuration time.Duration if !gc.burstable.Load() { var err error @@ -1217,6 +1218,7 @@ func (gc *groupCostController) onRequestWait( } gc.requestRetryCounter.Inc() time.Sleep(retryInterval) + waitDuration += retryInterval } if err != nil { gc.failedRequestCounter.Inc() @@ -1226,9 +1228,10 @@ func (gc *groupCostController) onRequestWait( failpoint.Inject("triggerUpdate", func() { gc.lowRUNotifyChan <- struct{}{} }) - return nil, nil, 0, err + return nil, nil, waitDuration, 0, err } gc.successfulRequestDuration.Observe(d.Seconds()) + waitDuration += d } gc.mu.Lock() @@ -1245,7 +1248,7 @@ func (gc *groupCostController) onRequestWait( *gc.mu.storeCounter[info.StoreID()] = *gc.mu.globalCounter gc.mu.Unlock() - return delta, penalty, gc.meta.Priority, nil + return delta, penalty, waitDuration, gc.meta.Priority, nil } func (gc *groupCostController) onResponse( diff --git a/client/resource_group/controller/controller_test.go b/client/resource_group/controller/controller_test.go index 1db19787a81..4d09e338222 100644 --- a/client/resource_group/controller/controller_test.go +++ b/client/resource_group/controller/controller_test.go @@ -101,7 +101,7 @@ func TestRequestAndResponseConsumption(t *testing.T) { kvCalculator := gc.getKVCalculator() for idx, testCase := range testCases { caseNum := fmt.Sprintf("case %d", idx) - consumption, _, priority, err := gc.onRequestWait(context.TODO(), testCase.req) + consumption, _, _, priority, err := gc.onRequestWait(context.TODO(), testCase.req) re.NoError(err, caseNum) re.Equal(priority, gc.meta.Priority) expectedConsumption := &rmpb.Consumption{} diff --git a/tests/integrations/mcs/resourcemanager/resource_manager_test.go b/tests/integrations/mcs/resourcemanager/resource_manager_test.go index cd6edda4628..dbd2f838456 100644 --- a/tests/integrations/mcs/resourcemanager/resource_manager_test.go +++ b/tests/integrations/mcs/resourcemanager/resource_manager_test.go @@ -440,9 +440,9 @@ func (suite *resourceManagerClientTestSuite) TestResourceGroupController() { rres := cas.tcs[i].makeReadResponse() wres := cas.tcs[i].makeWriteResponse() startTime := time.Now() - _, _, _, err := controller.OnRequestWait(suite.ctx, cas.resourceGroupName, rreq) + _, _, _, _, err := controller.OnRequestWait(suite.ctx, cas.resourceGroupName, rreq) re.NoError(err) - _, _, _, err = controller.OnRequestWait(suite.ctx, cas.resourceGroupName, wreq) + _, _, _, _, err = controller.OnRequestWait(suite.ctx, cas.resourceGroupName, wreq) re.NoError(err) sum += time.Since(startTime) controller.OnResponse(cas.resourceGroupName, rreq, rres) @@ -459,7 +459,7 @@ func (suite *resourceManagerClientTestSuite) TestResourceGroupController() { re.NoError(failpoint.Enable("github.com/tikv/pd/client/resource_group/controller/triggerUpdate", "return(true)")) tcs := tokenConsumptionPerSecond{rruTokensAtATime: 1, wruTokensAtATime: 900000000, times: 1, waitDuration: 0} wreq := tcs.makeWriteRequest() - _, _, _, err = controller.OnRequestWait(suite.ctx, rg.Name, wreq) + _, _, _, _, err = controller.OnRequestWait(suite.ctx, rg.Name, wreq) re.Error(err) time.Sleep(time.Millisecond * 200) re.NoError(failpoint.Disable("github.com/tikv/pd/client/resource_group/controller/triggerUpdate")) @@ -514,9 +514,9 @@ func (suite *resourceManagerClientTestSuite) TestSwitchBurst() { wreq := tcs.makeWriteRequest() rres := tcs.makeReadResponse() wres := tcs.makeWriteResponse() - _, _, _, err := controller.OnRequestWait(suite.ctx, resourceGroupName, rreq) + _, _, _, _, err := controller.OnRequestWait(suite.ctx, resourceGroupName, rreq) re.NoError(err) - _, _, _, err = controller.OnRequestWait(suite.ctx, resourceGroupName, wreq) + _, _, _, _, err = controller.OnRequestWait(suite.ctx, resourceGroupName, wreq) re.NoError(err) controller.OnResponse(resourceGroupName, rreq, rres) controller.OnResponse(resourceGroupName, wreq, wres) @@ -553,9 +553,9 @@ func (suite *resourceManagerClientTestSuite) TestSwitchBurst() { rres := cas.tcs[i].makeReadResponse() wres := cas.tcs[i].makeWriteResponse() startTime := time.Now() - _, _, _, err := controller.OnRequestWait(suite.ctx, resourceGroupName, rreq) + _, _, _, _, err := controller.OnRequestWait(suite.ctx, resourceGroupName, rreq) re.NoError(err) - _, _, _, err = controller.OnRequestWait(suite.ctx, resourceGroupName, wreq) + _, _, _, _, err = controller.OnRequestWait(suite.ctx, resourceGroupName, wreq) re.NoError(err) sum += time.Since(startTime) controller.OnResponse(resourceGroupName, rreq, rres) @@ -573,14 +573,14 @@ func (suite *resourceManagerClientTestSuite) TestSwitchBurst() { resourceGroupName2 := suite.initGroups[2].Name tcs = tokenConsumptionPerSecond{rruTokensAtATime: 1, wruTokensAtATime: 100000, times: 1, waitDuration: 0} wreq := tcs.makeWriteRequest() - _, _, _, err := controller.OnRequestWait(suite.ctx, resourceGroupName2, wreq) + _, _, _, _, err := controller.OnRequestWait(suite.ctx, resourceGroupName2, wreq) re.NoError(err) re.NoError(failpoint.Enable("github.com/tikv/pd/client/resource_group/controller/acceleratedSpeedTrend", "return(true)")) resourceGroupName3 := suite.initGroups[3].Name tcs = tokenConsumptionPerSecond{rruTokensAtATime: 1, wruTokensAtATime: 1000, times: 1, waitDuration: 0} wreq = tcs.makeWriteRequest() - _, _, _, err = controller.OnRequestWait(suite.ctx, resourceGroupName3, wreq) + _, _, _, _, err = controller.OnRequestWait(suite.ctx, resourceGroupName3, wreq) re.NoError(err) time.Sleep(110 * time.Millisecond) tcs = tokenConsumptionPerSecond{rruTokensAtATime: 1, wruTokensAtATime: 10, times: 1010, waitDuration: 0} @@ -588,7 +588,7 @@ func (suite *resourceManagerClientTestSuite) TestSwitchBurst() { for i := 0; i < tcs.times; i++ { wreq = tcs.makeWriteRequest() startTime := time.Now() - _, _, _, err = controller.OnRequestWait(suite.ctx, resourceGroupName3, wreq) + _, _, _, _, err = controller.OnRequestWait(suite.ctx, resourceGroupName3, wreq) duration += time.Since(startTime) re.NoError(err) } @@ -637,7 +637,7 @@ func (suite *resourceManagerClientTestSuite) TestResourcePenalty() { // init req := controller.NewTestRequestInfo(false, 0, 2 /* store2 */) resp := controller.NewTestResponseInfo(0, time.Duration(30), true) - _, penalty, _, err := c.OnRequestWait(suite.ctx, resourceGroupName, req) + _, penalty, _, _, err := c.OnRequestWait(suite.ctx, resourceGroupName, req) re.NoError(err) re.Equal(penalty.WriteBytes, 0.0) re.Equal(penalty.TotalCpuTimeMs, 0.0) @@ -646,7 +646,7 @@ func (suite *resourceManagerClientTestSuite) TestResourcePenalty() { req = controller.NewTestRequestInfo(true, 60, 1 /* store1 */) resp = controller.NewTestResponseInfo(0, time.Duration(10), true) - _, penalty, _, err = c.OnRequestWait(suite.ctx, resourceGroupName, req) + _, penalty, _, _, err = c.OnRequestWait(suite.ctx, resourceGroupName, req) re.NoError(err) re.Equal(penalty.WriteBytes, 0.0) re.Equal(penalty.TotalCpuTimeMs, 0.0) @@ -656,7 +656,7 @@ func (suite *resourceManagerClientTestSuite) TestResourcePenalty() { // failed request, shouldn't be counted in penalty req = controller.NewTestRequestInfo(true, 20, 1 /* store1 */) resp = controller.NewTestResponseInfo(0, time.Duration(0), false) - _, penalty, _, err = c.OnRequestWait(suite.ctx, resourceGroupName, req) + _, penalty, _, _, err = c.OnRequestWait(suite.ctx, resourceGroupName, req) re.NoError(err) re.Equal(penalty.WriteBytes, 0.0) re.Equal(penalty.TotalCpuTimeMs, 0.0) @@ -666,7 +666,7 @@ func (suite *resourceManagerClientTestSuite) TestResourcePenalty() { // from same store, should be zero req1 := controller.NewTestRequestInfo(false, 0, 1 /* store1 */) resp1 := controller.NewTestResponseInfo(0, time.Duration(10), true) - _, penalty, _, err = c.OnRequestWait(suite.ctx, resourceGroupName, req1) + _, penalty, _, _, err = c.OnRequestWait(suite.ctx, resourceGroupName, req1) re.NoError(err) re.Equal(penalty.WriteBytes, 0.0) _, err = c.OnResponse(resourceGroupName, req1, resp1) @@ -675,7 +675,7 @@ func (suite *resourceManagerClientTestSuite) TestResourcePenalty() { // from different store, should be non-zero req2 := controller.NewTestRequestInfo(true, 50, 2 /* store2 */) resp2 := controller.NewTestResponseInfo(0, time.Duration(10), true) - _, penalty, _, err = c.OnRequestWait(suite.ctx, resourceGroupName, req2) + _, penalty, _, _, err = c.OnRequestWait(suite.ctx, resourceGroupName, req2) re.NoError(err) re.Equal(penalty.WriteBytes, 60.0) re.InEpsilon(penalty.TotalCpuTimeMs, 10.0/1000.0/1000.0, 1e-6) @@ -685,7 +685,7 @@ func (suite *resourceManagerClientTestSuite) TestResourcePenalty() { // from new store, should be zero req3 := controller.NewTestRequestInfo(true, 0, 3 /* store3 */) resp3 := controller.NewTestResponseInfo(0, time.Duration(10), true) - _, penalty, _, err = c.OnRequestWait(suite.ctx, resourceGroupName, req3) + _, penalty, _, _, err = c.OnRequestWait(suite.ctx, resourceGroupName, req3) re.NoError(err) re.Equal(penalty.WriteBytes, 0.0) _, err = c.OnResponse(resourceGroupName, req3, resp3) @@ -695,7 +695,7 @@ func (suite *resourceManagerClientTestSuite) TestResourcePenalty() { resourceGroupName = groupNames[1] req4 := controller.NewTestRequestInfo(true, 50, 1 /* store2 */) resp4 := controller.NewTestResponseInfo(0, time.Duration(10), true) - _, penalty, _, err = c.OnRequestWait(suite.ctx, resourceGroupName, req4) + _, penalty, _, _, err = c.OnRequestWait(suite.ctx, resourceGroupName, req4) re.NoError(err) re.Equal(penalty.WriteBytes, 0.0) _, err = c.OnResponse(resourceGroupName, req4, resp4) From 080af97eeac082ad1d658e81fca2be989fc438e3 Mon Sep 17 00:00:00 2001 From: JmPotato Date: Mon, 4 Dec 2023 16:01:23 +0800 Subject: [PATCH 072/137] client/http: introduce caller ID into the HTTP client (#7490) ref tikv/pd#7300 Introduce caller ID into the HTTP client. Signed-off-by: JmPotato --- client/http/client.go | 66 +++++++++++++++++++++++++--------- client/http/client_test.go | 73 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 122 insertions(+), 17 deletions(-) create mode 100644 client/http/client_test.go diff --git a/client/http/client.go b/client/http/client.go index 36355a90d19..b79aa9ca002 100644 --- a/client/http/client.go +++ b/client/http/client.go @@ -32,6 +32,7 @@ import ( ) const ( + defaultCallerID = "pd-http-client" httpScheme = "http" httpsScheme = "https" networkErrorStatus = "network error" @@ -79,6 +80,8 @@ type Client interface { GetMinResolvedTSByStoresIDs(context.Context, []uint64) (uint64, map[uint64]uint64, error) /* Client-related methods */ + // WithCallerID sets and returns a new client with the given caller ID. + WithCallerID(string) Client // WithRespHandler sets and returns a new client with the given HTTP response handler. // This allows the caller to customize how the response is handled, including error handling logic. // Additionally, it is important for the caller to handle the content of the response body properly @@ -89,11 +92,20 @@ type Client interface { var _ Client = (*client)(nil) -type client struct { +// clientInner is the inner implementation of the PD HTTP client, which will +// implement some internal logics, such as HTTP client, service discovery, etc. +type clientInner struct { pdAddrs []string tlsConf *tls.Config cli *http.Client +} + +type client struct { + // Wrap this struct is to make sure the inner implementation + // won't be exposed and cloud be consistent during the copy. + inner *clientInner + callerID string respHandler func(resp *http.Response, res interface{}) error requestCounter *prometheus.CounterVec @@ -106,7 +118,7 @@ type ClientOption func(c *client) // WithHTTPClient configures the client with the given initialized HTTP client. func WithHTTPClient(cli *http.Client) ClientOption { return func(c *client) { - c.cli = cli + c.inner.cli = cli } } @@ -114,7 +126,7 @@ func WithHTTPClient(cli *http.Client) ClientOption { // This option won't work if the client is configured with WithHTTPClient. func WithTLSConfig(tlsConf *tls.Config) ClientOption { return func(c *client) { - c.tlsConf = tlsConf + c.inner.tlsConf = tlsConf } } @@ -134,7 +146,7 @@ func NewClient( pdAddrs []string, opts ...ClientOption, ) Client { - c := &client{} + c := &client{inner: &clientInner{}, callerID: defaultCallerID} // Apply the options first. for _, opt := range opts { opt(c) @@ -143,7 +155,7 @@ func NewClient( for i, addr := range pdAddrs { if !strings.HasPrefix(addr, httpScheme) { var scheme string - if c.tlsConf != nil { + if c.inner.tlsConf != nil { scheme = httpsScheme } else { scheme = httpScheme @@ -151,14 +163,14 @@ func NewClient( pdAddrs[i] = fmt.Sprintf("%s://%s", scheme, addr) } } - c.pdAddrs = pdAddrs + c.inner.pdAddrs = pdAddrs // Init the HTTP client if it's not configured. - if c.cli == nil { - c.cli = &http.Client{Timeout: defaultTimeout} - if c.tlsConf != nil { + if c.inner.cli == nil { + c.inner.cli = &http.Client{Timeout: defaultTimeout} + if c.inner.tlsConf != nil { transport := http.DefaultTransport.(*http.Transport).Clone() - transport.TLSClientConfig = c.tlsConf - c.cli.Transport = transport + transport.TLSClientConfig = c.inner.tlsConf + c.inner.cli.Transport = transport } } @@ -167,12 +179,22 @@ func NewClient( // Close closes the HTTP client. func (c *client) Close() { - if c.cli != nil { - c.cli.CloseIdleConnections() + if c.inner == nil { + return + } + if c.inner.cli != nil { + c.inner.cli.CloseIdleConnections() } log.Info("[pd] http client closed") } +// WithCallerID sets and returns a new client with the given caller ID. +func (c *client) WithCallerID(callerID string) Client { + newClient := *c + newClient.callerID = callerID + return &newClient +} + // WithRespHandler sets and returns a new client with the given HTTP response handler. func (c *client) WithRespHandler( handler func(resp *http.Response, res interface{}) error, @@ -196,13 +218,19 @@ func (c *client) execDuration(name string, duration time.Duration) { c.executionDuration.WithLabelValues(name).Observe(duration.Seconds()) } +// Header key definition constants. +const ( + pdAllowFollowerHandleKey = "PD-Allow-Follower-Handle" + componentSignatureKey = "component" +) + // HeaderOption configures the HTTP header. type HeaderOption func(header http.Header) // WithAllowFollowerHandle sets the header field to allow a PD follower to handle this request. func WithAllowFollowerHandle() HeaderOption { return func(header http.Header) { - header.Set("PD-Allow-Follower-Handle", "true") + header.Set(pdAllowFollowerHandleKey, "true") } } @@ -218,8 +246,8 @@ func (c *client) requestWithRetry( err error addr string ) - for idx := 0; idx < len(c.pdAddrs); idx++ { - addr = c.pdAddrs[idx] + for idx := 0; idx < len(c.inner.pdAddrs); idx++ { + addr = c.inner.pdAddrs[idx] err = c.request(ctx, name, fmt.Sprintf("%s%s", addr, uri), method, body, res, headerOpts...) if err == nil { break @@ -239,6 +267,8 @@ func (c *client) request( logFields := []zap.Field{ zap.String("name", name), zap.String("url", url), + zap.String("method", method), + zap.String("caller-id", c.callerID), } log.Debug("[pd] request the http url", logFields...) req, err := http.NewRequestWithContext(ctx, method, url, body) @@ -249,8 +279,10 @@ func (c *client) request( for _, opt := range headerOpts { opt(req.Header) } + req.Header.Set(componentSignatureKey, c.callerID) + start := time.Now() - resp, err := c.cli.Do(req) + resp, err := c.inner.cli.Do(req) if err != nil { c.reqCounter(name, networkErrorStatus) log.Error("[pd] do http request failed", append(logFields, zap.Error(err))...) diff --git a/client/http/client_test.go b/client/http/client_test.go new file mode 100644 index 00000000000..621910e29ea --- /dev/null +++ b/client/http/client_test.go @@ -0,0 +1,73 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package http + +import ( + "context" + "net/http" + "testing" + + "github.com/stretchr/testify/require" +) + +// requestChecker is used to check the HTTP request sent by the client. +type requestChecker struct { + checker func(req *http.Request) error +} + +// RoundTrip implements the `http.RoundTripper` interface. +func (rc *requestChecker) RoundTrip(req *http.Request) (resp *http.Response, err error) { + return &http.Response{StatusCode: http.StatusOK}, rc.checker(req) +} + +func newHTTPClientWithRequestChecker(checker func(req *http.Request) error) *http.Client { + return &http.Client{ + Transport: &requestChecker{checker: checker}, + } +} + +func TestPDAllowFollowerHandleHeader(t *testing.T) { + re := require.New(t) + var expectedVal string + httpClient := newHTTPClientWithRequestChecker(func(req *http.Request) error { + val := req.Header.Get(pdAllowFollowerHandleKey) + if val != expectedVal { + re.Failf("PD allow follower handler header check failed", + "should be %s, but got %s", expectedVal, val) + } + return nil + }) + c := NewClient([]string{"http://127.0.0.1"}, WithHTTPClient(httpClient)) + c.GetRegions(context.Background()) + expectedVal = "true" + c.GetHistoryHotRegions(context.Background(), &HistoryHotRegionsRequest{}) +} + +func TestCallerID(t *testing.T) { + re := require.New(t) + expectedVal := defaultCallerID + httpClient := newHTTPClientWithRequestChecker(func(req *http.Request) error { + val := req.Header.Get(componentSignatureKey) + if val != expectedVal { + re.Failf("Caller ID header check failed", + "should be %s, but got %s", expectedVal, val) + } + return nil + }) + c := NewClient([]string{"http://127.0.0.1"}, WithHTTPClient(httpClient)) + c.GetRegions(context.Background()) + expectedVal = "test" + c.WithCallerID(expectedVal).GetRegions(context.Background()) +} From dfff6903e60eb1e534916341f1aeb843ee0ffc63 Mon Sep 17 00:00:00 2001 From: Yongbo Jiang Date: Mon, 4 Dec 2023 16:13:22 +0800 Subject: [PATCH 073/137] ci/scripts: add client CI test (#7491) ref tikv/pd#4399 Signed-off-by: Cabinfever_B Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- .github/workflows/pd-tests.yaml | 2 +- Makefile | 1 + scripts/ci-subtask.sh | 4 +++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pd-tests.yaml b/.github/workflows/pd-tests.yaml index 73e31fd4ad1..517a2c480e7 100644 --- a/.github/workflows/pd-tests.yaml +++ b/.github/workflows/pd-tests.yaml @@ -45,7 +45,7 @@ jobs: env: WORKER_ID: ${{ matrix.worker_id }} WORKER_COUNT: 13 - JOB_COUNT: 10 # 11, 12 13 are for other integrations jobs + JOB_COUNT: 10 # 11, 12, 13 are for other integrations jobs run: | make ci-test-job JOB_COUNT=$(($JOB_COUNT)) JOB_INDEX=$WORKER_ID mv covprofile covprofile_$WORKER_ID diff --git a/Makefile b/Makefile index bf76fd57f2f..946493cd7ce 100644 --- a/Makefile +++ b/Makefile @@ -244,6 +244,7 @@ ci-test-job: install-tools dashboard-ui else \ for mod in $(shell ./scripts/ci-subtask.sh $(JOB_COUNT) $(JOB_INDEX)); do cd $$mod && $(MAKE) ci-test-job && cd $(ROOT_PATH) > /dev/null && cat $$mod/covprofile >> covprofile; done; \ fi + @$(FAILPOINT_DISABLE) TSO_INTEGRATION_TEST_PKGS := $(PD_PKG)/tests/server/tso diff --git a/scripts/ci-subtask.sh b/scripts/ci-subtask.sh index dd29895327b..5d7392efe11 100755 --- a/scripts/ci-subtask.sh +++ b/scripts/ci-subtask.sh @@ -10,7 +10,9 @@ if [[ $2 -gt 10 ]]; then # Currently, we only have 3 integration tests, so we can hardcode the task index. for t in ${integration_tasks[@]}; do if [[ "$t" = "./tests/integrations/client" && "$2" = 11 ]]; then - printf "%s " "$t" + res=("./client") + res+=($t) + printf "%s " "${res[@]}" break elif [[ "$t" = "./tests/integrations/tso" && "$2" = 12 ]]; then printf "%s " "$t" From 6235fead565b0d3b0a99936ddd36b06a5ee2c317 Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Tue, 5 Dec 2023 10:44:48 +0800 Subject: [PATCH 074/137] *: make TestConfigPDServer stable (#7492) ref tikv/pd#4399 Signed-off-by: Ryan Leung --- pkg/core/region_tree.go | 1 + pkg/schedule/schedulers/evict_slow_trend_test.go | 1 + .../integrations/mcs/resourcemanager/resource_manager_test.go | 2 +- tests/integrations/mcs/tso/proxy_test.go | 2 +- tests/registry/registry_test.go | 2 +- tests/server/config/config_test.go | 4 +++- 6 files changed, 8 insertions(+), 4 deletions(-) diff --git a/pkg/core/region_tree.go b/pkg/core/region_tree.go index ecc988d97d8..333e1730ec8 100644 --- a/pkg/core/region_tree.go +++ b/pkg/core/region_tree.go @@ -1,4 +1,5 @@ // Copyright 2016 TiKV Project Authors. +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at diff --git a/pkg/schedule/schedulers/evict_slow_trend_test.go b/pkg/schedule/schedulers/evict_slow_trend_test.go index 75ea50d73b4..65a70962a20 100644 --- a/pkg/schedule/schedulers/evict_slow_trend_test.go +++ b/pkg/schedule/schedulers/evict_slow_trend_test.go @@ -1,3 +1,4 @@ +// Copyright 2023 TiKV Project Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/tests/integrations/mcs/resourcemanager/resource_manager_test.go b/tests/integrations/mcs/resourcemanager/resource_manager_test.go index dbd2f838456..36eb87a83db 100644 --- a/tests/integrations/mcs/resourcemanager/resource_manager_test.go +++ b/tests/integrations/mcs/resourcemanager/resource_manager_test.go @@ -4,7 +4,7 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, diff --git a/tests/integrations/mcs/tso/proxy_test.go b/tests/integrations/mcs/tso/proxy_test.go index fc33a6a41be..60280fa892d 100644 --- a/tests/integrations/mcs/tso/proxy_test.go +++ b/tests/integrations/mcs/tso/proxy_test.go @@ -4,7 +4,7 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, diff --git a/tests/registry/registry_test.go b/tests/registry/registry_test.go index a3aff76a1cf..d2661cda616 100644 --- a/tests/registry/registry_test.go +++ b/tests/registry/registry_test.go @@ -4,7 +4,7 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, diff --git a/tests/server/config/config_test.go b/tests/server/config/config_test.go index c0e9a0e1148..eb7acb80b96 100644 --- a/tests/server/config/config_test.go +++ b/tests/server/config/config_test.go @@ -402,7 +402,9 @@ func (suite *configTestSuite) checkConfigPDServer(cluster *tests.TestCluster) { suite.Equal("table", sc.KeyType) suite.Equal(typeutil.StringSlice([]string{}), sc.RuntimeServices) suite.Equal("", sc.MetricStorage) - suite.Equal("auto", sc.DashboardAddress) + if sc.DashboardAddress != "auto" { // dashboard has been assigned + re.Equal(leaderServer.GetAddr(), sc.DashboardAddress) + } suite.Equal(int(3), sc.FlowRoundByDigit) suite.Equal(typeutil.NewDuration(time.Second), sc.MinResolvedTSPersistenceInterval) suite.Equal(24*time.Hour, sc.MaxResetTSGap.Duration) From 4eac0ef621eddc7079e48b40f5277c9151335ea5 Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Tue, 5 Dec 2023 17:41:49 +0800 Subject: [PATCH 075/137] mcs: add forward metrics (#7494) ref tikv/pd#5839 Signed-off-by: Ryan Leung --- server/grpc_service.go | 19 +++++++++++++++++++ server/metrics.go | 9 +++++++++ server/server.go | 12 ++++++------ 3 files changed, 34 insertions(+), 6 deletions(-) diff --git a/server/grpc_service.go b/server/grpc_service.go index bb20fcbe484..6acc13b8187 100644 --- a/server/grpc_service.go +++ b/server/grpc_service.go @@ -77,6 +77,17 @@ var ( ErrEtcdNotStarted = status.Errorf(codes.Unavailable, "server is started, but etcd not started") ) +var ( + errRegionHeartbeatSend = forwardFailCounter.WithLabelValues("region_heartbeat", "send") + errRegionHeartbeatClient = forwardFailCounter.WithLabelValues("region_heartbeat", "client") + errRegionHeartbeatStream = forwardFailCounter.WithLabelValues("region_heartbeat", "stream") + errRegionHeartbeatRecv = forwardFailCounter.WithLabelValues("region_heartbeat", "recv") + errScatterRegionSend = forwardFailCounter.WithLabelValues("scatter_region", "send") + errSplitRegionsSend = forwardFailCounter.WithLabelValues("split_regions", "send") + errStoreHeartbeatSend = forwardFailCounter.WithLabelValues("store_heartbeat", "send") + errGetOperatorSend = forwardFailCounter.WithLabelValues("get_operator", "send") +) + // GrpcServer wraps Server to provide grpc service. type GrpcServer struct { *Server @@ -867,6 +878,7 @@ func (s *GrpcServer) StoreHeartbeat(ctx context.Context, request *pdpb.StoreHear Stats: request.GetStats(), } if _, err := cli.StoreHeartbeat(ctx, req); err != nil { + errStoreHeartbeatSend.Inc() log.Debug("forward store heartbeat failed", zap.Error(err)) // reset to let it be updated in the next request s.schedulingClient.CompareAndSwap(forwardCli, &schedulingClient{}) @@ -1216,12 +1228,14 @@ func (s *GrpcServer) RegionHeartbeat(stream pdpb.PD_RegionHeartbeatServer) error } client, err := s.getDelegateClient(s.ctx, forwardedSchedulingHost) if err != nil { + errRegionHeartbeatClient.Inc() log.Error("failed to get client", zap.Error(err)) continue } log.Info("create scheduling forwarding stream", zap.String("forwarded-host", forwardedSchedulingHost)) forwardSchedulingStream, _, cancel, err = s.createRegionHeartbeatSchedulingStream(stream.Context(), client) if err != nil { + errRegionHeartbeatStream.Inc() log.Error("failed to create stream", zap.Error(err)) continue } @@ -1250,6 +1264,7 @@ func (s *GrpcServer) RegionHeartbeat(stream pdpb.PD_RegionHeartbeatServer) error } if err := forwardSchedulingStream.Send(schedulingpbReq); err != nil { forwardSchedulingStream = nil + errRegionHeartbeatSend.Inc() log.Error("failed to send request to scheduling service", zap.Error(err)) } @@ -1257,6 +1272,7 @@ func (s *GrpcServer) RegionHeartbeat(stream pdpb.PD_RegionHeartbeatServer) error case err, ok := <-forwardErrCh: if ok { forwardSchedulingStream = nil + errRegionHeartbeatRecv.Inc() log.Error("failed to send response", zap.Error(err)) } default: @@ -1692,6 +1708,7 @@ func (s *GrpcServer) ScatterRegion(ctx context.Context, request *pdpb.ScatterReg } resp, err := cli.ScatterRegions(ctx, req) if err != nil { + errScatterRegionSend.Inc() // reset to let it be updated in the next request s.schedulingClient.CompareAndSwap(forwardCli, &schedulingClient{}) return s.convertScatterResponse(resp), err @@ -1900,6 +1917,7 @@ func (s *GrpcServer) GetOperator(ctx context.Context, request *pdpb.GetOperatorR } resp, err := cli.GetOperator(ctx, req) if err != nil { + errGetOperatorSend.Inc() // reset to let it be updated in the next request s.schedulingClient.CompareAndSwap(forwardCli, &schedulingClient{}) return s.convertOperatorResponse(resp), err @@ -2174,6 +2192,7 @@ func (s *GrpcServer) SplitRegions(ctx context.Context, request *pdpb.SplitRegion } resp, err := cli.SplitRegions(ctx, req) if err != nil { + errSplitRegionsSend.Inc() // reset to let it be updated in the next request s.schedulingClient.CompareAndSwap(forwardCli, &schedulingClient{}) return s.convertSplitResponse(resp), err diff --git a/server/metrics.go b/server/metrics.go index 94eb9bf19a2..2d13d02d564 100644 --- a/server/metrics.go +++ b/server/metrics.go @@ -159,6 +159,14 @@ var ( Name: "maxprocs", Help: "The value of GOMAXPROCS.", }) + + forwardFailCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pd", + Subsystem: "server", + Name: "forward_fail_total", + Help: "Counter of forward fail.", + }, []string{"request", "type"}) ) func init() { @@ -179,4 +187,5 @@ func init() { prometheus.MustRegister(serviceAuditHistogram) prometheus.MustRegister(bucketReportInterval) prometheus.MustRegister(serverMaxProcs) + prometheus.MustRegister(forwardFailCounter) } diff --git a/server/server.go b/server/server.go index 43daa65d844..c815e7d50c6 100644 --- a/server/server.go +++ b/server/server.go @@ -489,13 +489,13 @@ func (s *Server) startServer(ctx context.Context) error { s.safePointV2Manager = gc.NewSafePointManagerV2(s.ctx, s.storage, s.storage, s.storage) s.hbStreams = hbstream.NewHeartbeatStreams(ctx, s.clusterID, "", s.cluster) // initial hot_region_storage in here. - if !s.IsServiceIndependent(mcs.SchedulingServiceName) { - s.hotRegionStorage, err = storage.NewHotRegionsStorage( - ctx, filepath.Join(s.cfg.DataDir, "hot-region"), s.encryptionKeyManager, s.handler) - if err != nil { - return err - } + + s.hotRegionStorage, err = storage.NewHotRegionsStorage( + ctx, filepath.Join(s.cfg.DataDir, "hot-region"), s.encryptionKeyManager, s.handler) + if err != nil { + return err } + // Run callbacks log.Info("triggering the start callback functions") for _, cb := range s.startCallbacks { From 6080557a9e9e7327a732db2c05378d1c29f04fd3 Mon Sep 17 00:00:00 2001 From: JmPotato Date: Thu, 7 Dec 2023 09:14:50 +0800 Subject: [PATCH 076/137] election: provide methods to read and write campaignTimes instead of export (#7501) ref tikv/pd#7499 Provide methods to read and write `campaignTimes` instead of export. Signed-off-by: JmPotato --- pkg/election/leadership.go | 34 ++++++++++++++++++++++++------ pkg/member/member.go | 5 ++--- tests/server/member/member_test.go | 3 +-- 3 files changed, 30 insertions(+), 12 deletions(-) diff --git a/pkg/election/leadership.go b/pkg/election/leadership.go index 572dae132b6..132b3a8aad9 100644 --- a/pkg/election/leadership.go +++ b/pkg/election/leadership.go @@ -33,6 +33,7 @@ import ( ) const ( + defaultCampaignTimesSlot = 10 watchLoopUnhealthyTimeout = 60 * time.Second campaignTimesRecordTimeout = 5 * time.Minute ) @@ -65,9 +66,9 @@ type Leadership struct { keepAliveCtx context.Context keepAliveCancelFunc context.CancelFunc keepAliveCancelFuncLock syncutil.Mutex - // CampaignTimes is used to record the campaign times of the leader within `campaignTimesRecordTimeout`. + // campaignTimes is used to record the campaign times of the leader within `campaignTimesRecordTimeout`. // It is ordered by time to prevent the leader from campaigning too frequently. - CampaignTimes []time.Time + campaignTimes []time.Time } // NewLeadership creates a new Leadership. @@ -76,7 +77,7 @@ func NewLeadership(client *clientv3.Client, leaderKey, purpose string) *Leadersh purpose: purpose, client: client, leaderKey: leaderKey, - CampaignTimes: make([]time.Time, 0, 10), + campaignTimes: make([]time.Time, 0, defaultCampaignTimesSlot), } return leadership } @@ -111,18 +112,37 @@ func (ls *Leadership) GetLeaderKey() string { return ls.leaderKey } +// GetCampaignTimesNum is used to get the campaign times of the leader within `campaignTimesRecordTimeout`. +func (ls *Leadership) GetCampaignTimesNum() int { + if ls == nil { + return 0 + } + return len(ls.campaignTimes) +} + +// ResetCampaignTimes is used to reset the campaign times of the leader. +func (ls *Leadership) ResetCampaignTimes() { + if ls == nil { + return + } + ls.campaignTimes = make([]time.Time, 0, defaultCampaignTimesSlot) +} + // addCampaignTimes is used to add the campaign times of the leader. func (ls *Leadership) addCampaignTimes() { - for i := len(ls.CampaignTimes) - 1; i >= 0; i-- { - if time.Since(ls.CampaignTimes[i]) > campaignTimesRecordTimeout { + if ls == nil { + return + } + for i := len(ls.campaignTimes) - 1; i >= 0; i-- { + if time.Since(ls.campaignTimes[i]) > campaignTimesRecordTimeout { // remove the time which is more than `campaignTimesRecordTimeout` // array is sorted by time - ls.CampaignTimes = ls.CampaignTimes[i:] + ls.campaignTimes = ls.campaignTimes[i:] break } } - ls.CampaignTimes = append(ls.CampaignTimes, time.Now()) + ls.campaignTimes = append(ls.campaignTimes, time.Now()) } // Campaign is used to campaign the leader with given lease and returns a leadership diff --git a/pkg/member/member.go b/pkg/member/member.go index dd36214a595..b411d0c957b 100644 --- a/pkg/member/member.go +++ b/pkg/member/member.go @@ -185,11 +185,10 @@ func (m *EmbeddedEtcdMember) CampaignLeader(ctx context.Context, leaseTimeout in failpoint.Inject("skipCampaignLeaderCheck", func() { failpoint.Return(m.leadership.Campaign(leaseTimeout, m.MemberValue())) }) - if len(m.leadership.CampaignTimes) >= campaignLeaderFrequencyTimes { + if m.leadership.GetCampaignTimesNum() >= campaignLeaderFrequencyTimes { log.Warn("campaign times is too frequent, resign and campaign again", zap.String("leader-name", m.Name()), zap.String("leader-key", m.GetLeaderPath())) - // remove all campaign times - m.leadership.CampaignTimes = nil + m.leadership.ResetCampaignTimes() return m.ResignEtcdLeader(ctx, m.Name(), "") } return m.leadership.Campaign(leaseTimeout, m.MemberValue()) diff --git a/tests/server/member/member_test.go b/tests/server/member/member_test.go index 9ac0947c73f..68fbdb33bd9 100644 --- a/tests/server/member/member_test.go +++ b/tests/server/member/member_test.go @@ -341,8 +341,7 @@ func TestCampaignLeaderFrequently(t *testing.T) { cluster.GetLeaderServer().ResetPDLeader() cluster.WaitLeader() } - // leader should be changed when campaign leader frequently - cluster.WaitLeader() + // PD leader should be different from before because etcd leader changed. re.NotEmpty(cluster.GetLeader()) re.NotEqual(leader, cluster.GetLeader()) } From 995fcef820f43758b94a2eeef8cdd91fa08deaea Mon Sep 17 00:00:00 2001 From: JmPotato Date: Thu, 7 Dec 2023 09:37:19 +0800 Subject: [PATCH 077/137] client/http: fix TestMergeRegionsInfo to make it stable (#7502) ref tikv/pd#7300 Fix `TestMergeRegionsInfo` to make it stable. Signed-off-by: JmPotato Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- client/http/types_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/http/types_test.go b/client/http/types_test.go index 74482e29c3c..1dedbdc7d3b 100644 --- a/client/http/types_test.go +++ b/client/http/types_test.go @@ -46,7 +46,7 @@ func TestMergeRegionsInfo(t *testing.T) { regionsInfo := regionsInfo1.Merge(regionsInfo2) re.Equal(int64(2), regionsInfo.Count) re.Equal(2, len(regionsInfo.Regions)) - re.Equal(append(regionsInfo1.Regions, regionsInfo2.Regions...), regionsInfo.Regions) + re.Subset(regionsInfo.Regions, append(regionsInfo1.Regions, regionsInfo2.Regions...)) } func TestRuleStartEndKey(t *testing.T) { From ad232d1bd88db83e93731686f39c33f2739d9e52 Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Thu, 7 Dec 2023 14:37:21 +0800 Subject: [PATCH 078/137] *: remove the tombstone store metrics (#7504) ref tikv/pd#5839 Signed-off-by: Ryan Leung --- pkg/mcs/scheduling/server/meta/watcher.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/pkg/mcs/scheduling/server/meta/watcher.go b/pkg/mcs/scheduling/server/meta/watcher.go index 3a04c261163..6fae537eab9 100644 --- a/pkg/mcs/scheduling/server/meta/watcher.go +++ b/pkg/mcs/scheduling/server/meta/watcher.go @@ -16,12 +16,14 @@ package meta import ( "context" + "strconv" "sync" "github.com/gogo/protobuf/proto" "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/log" "github.com/tikv/pd/pkg/core" + "github.com/tikv/pd/pkg/statistics" "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/utils/etcdutil" "go.etcd.io/etcd/clientv3" @@ -79,9 +81,15 @@ func (w *Watcher) initializeStoreWatcher() error { origin := w.basicCluster.GetStore(store.GetId()) if origin == nil { w.basicCluster.PutStore(core.NewStoreInfo(store)) - return nil + } else { + w.basicCluster.PutStore(origin.Clone(core.SetStoreMeta(store))) } - w.basicCluster.PutStore(origin.Clone(core.SetStoreMeta(store))) + + if store.GetNodeState() == metapb.NodeState_Removed { + statistics.ResetStoreStatistics(store.GetAddress(), strconv.FormatUint(store.GetId(), 10)) + // TODO: remove hot stats + } + return nil } deleteFn := func(kv *mvccpb.KeyValue) error { From d8550f0ac54b4680a7d6d73acf00d0fd3e70788e Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Thu, 7 Dec 2023 21:11:50 +0800 Subject: [PATCH 079/137] mcs: fix panic of get leader (#7472) close tikv/pd#7280 Signed-off-by: Ryan Leung Co-authored-by: Ti Chi Robot --- pkg/member/participant.go | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/pkg/member/participant.go b/pkg/member/participant.go index 82cd7e05f5e..189da7b96c9 100644 --- a/pkg/member/participant.go +++ b/pkg/member/participant.go @@ -121,22 +121,16 @@ func (m *Participant) Client() *clientv3.Client { // IsLeader returns whether the participant is the leader or not by checking its leadership's // lease and leader info. func (m *Participant) IsLeader() bool { - if m.GetLeader() == nil { - return false - } return m.leadership.Check() && m.GetLeader().GetId() == m.member.GetId() && m.campaignCheck() } // IsLeaderElected returns true if the leader exists; otherwise false func (m *Participant) IsLeaderElected() bool { - return m.GetLeader() != nil + return m.GetLeader().GetId() != 0 } // GetLeaderListenUrls returns current leader's listen urls func (m *Participant) GetLeaderListenUrls() []string { - if m.GetLeader() == nil { - return nil - } return m.GetLeader().GetListenUrls() } @@ -149,13 +143,9 @@ func (m *Participant) GetLeaderID() uint64 { func (m *Participant) GetLeader() participant { leader := m.leader.Load() if leader == nil { - return nil - } - member := leader.(participant) - if member.GetId() == 0 { - return nil + return NewParticipantByService(m.serviceName) } - return member + return leader.(participant) } // setLeader sets the member's leader. From ab3025c4d56dc4e89812db7d227466b906fe2a49 Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Fri, 8 Dec 2023 16:07:21 +0800 Subject: [PATCH 080/137] tests: reduce mcs test duration (#7435) ref tikv/pd#4399 Signed-off-by: lhy1024 Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- pkg/schedule/operator/operator_controller.go | 6 + pkg/statistics/hot_cache.go | 7 + pkg/statistics/hot_peer_cache.go | 12 + tests/integrations/mcs/scheduling/api_test.go | 363 +++++++++--------- tests/pdctl/config/config_test.go | 94 ++++- tests/pdctl/hot/hot_test.go | 50 ++- tests/pdctl/operator/operator_test.go | 25 +- tests/pdctl/scheduler/scheduler_test.go | 55 ++- tests/server/api/checker_test.go | 13 +- tests/server/api/operator_test.go | 59 ++- tests/server/api/region_test.go | 141 ++++--- tests/server/api/rule_test.go | 150 +++----- tests/server/api/scheduler_test.go | 73 ++-- tests/server/config/config_test.go | 40 +- tests/testutil.go | 123 ++++-- 15 files changed, 723 insertions(+), 488 deletions(-) diff --git a/pkg/schedule/operator/operator_controller.go b/pkg/schedule/operator/operator_controller.go index 3e9d2f3abcb..e3bead3ffca 100644 --- a/pkg/schedule/operator/operator_controller.go +++ b/pkg/schedule/operator/operator_controller.go @@ -799,6 +799,12 @@ func (oc *Controller) GetFastOpInfluence(cluster *core.BasicCluster, influence O } } +// CleanAllOpRecords removes all operators' records. +// It is used in tests only. +func (oc *Controller) CleanAllOpRecords() { + oc.records.ttl.Clear() +} + // AddOpInfluence add operator influence for cluster func AddOpInfluence(op *Operator, influence OpInfluence, cluster *core.BasicCluster) { region := cluster.GetRegion(op.RegionID()) diff --git a/pkg/statistics/hot_cache.go b/pkg/statistics/hot_cache.go index 1868e323b0f..799fb240d10 100644 --- a/pkg/statistics/hot_cache.go +++ b/pkg/statistics/hot_cache.go @@ -205,3 +205,10 @@ func (w *HotCache) GetThresholds(kind utils.RWType, storeID uint64) []float64 { } return nil } + +// CleanCache cleans the cache. +// This is used for test purpose. +func (w *HotCache) CleanCache() { + w.writeCache.removeAllItem() + w.readCache.removeAllItem() +} diff --git a/pkg/statistics/hot_peer_cache.go b/pkg/statistics/hot_peer_cache.go index 1ac07289a3c..0e35e0e23be 100644 --- a/pkg/statistics/hot_peer_cache.go +++ b/pkg/statistics/hot_peer_cache.go @@ -544,6 +544,18 @@ func (f *hotPeerCache) removeItem(item *HotPeerStat) { } } +// removeAllItem removes all items of the cache. +// It is used for test. +func (f *hotPeerCache) removeAllItem() { + for _, peers := range f.peersOfStore { + for _, peer := range peers.GetAll() { + item := peer.(*HotPeerStat) + item.actionType = utils.Remove + f.updateStat(item) + } + } +} + func (f *hotPeerCache) coldItem(newItem, oldItem *HotPeerStat) { newItem.HotDegree = oldItem.HotDegree - 1 newItem.AntiCount = oldItem.AntiCount - 1 diff --git a/tests/integrations/mcs/scheduling/api_test.go b/tests/integrations/mcs/scheduling/api_test.go index 0607b1dee9a..8f5d37ee1bb 100644 --- a/tests/integrations/mcs/scheduling/api_test.go +++ b/tests/integrations/mcs/scheduling/api_test.go @@ -1,7 +1,6 @@ package scheduling_test import ( - "context" "encoding/hex" "encoding/json" "fmt" @@ -10,7 +9,6 @@ import ( "time" "github.com/pingcap/failpoint" - "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "github.com/tikv/pd/pkg/core" _ "github.com/tikv/pd/pkg/mcs/scheduling/server/apis/v1" @@ -34,49 +32,26 @@ var testDialClient = &http.Client{ type apiTestSuite struct { suite.Suite - ctx context.Context - cleanupFunc testutil.CleanupFunc - cluster *tests.TestCluster - server *tests.TestServer - backendEndpoints string - dialClient *http.Client + env *tests.SchedulingTestEnvironment } func TestAPI(t *testing.T) { - suite.Run(t, &apiTestSuite{}) + suite.Run(t, new(apiTestSuite)) } -func (suite *apiTestSuite) SetupTest() { - ctx, cancel := context.WithCancel(context.Background()) - suite.ctx = ctx - cluster, err := tests.NewTestAPICluster(suite.ctx, 1) - suite.cluster = cluster - suite.NoError(err) - suite.NoError(cluster.RunInitialServers()) - suite.NotEmpty(cluster.WaitLeader()) - suite.server = cluster.GetLeaderServer() - suite.NoError(suite.server.BootstrapCluster()) - suite.backendEndpoints = suite.server.GetAddr() - suite.dialClient = &http.Client{ - Transport: &http.Transport{ - DisableKeepAlives: true, - }, - } - suite.cleanupFunc = func() { - cancel() - } - tc, err := tests.NewTestSchedulingCluster(suite.ctx, 2, suite.backendEndpoints) - suite.NoError(err) - suite.cluster.SetSchedulingCluster(tc) - tc.WaitForPrimaryServing(suite.Require()) +func (suite *apiTestSuite) SetupSuite() { + suite.env = tests.NewSchedulingTestEnvironment(suite.T()) } -func (suite *apiTestSuite) TearDownTest() { - suite.cluster.Destroy() - suite.cleanupFunc() +func (suite *apiTestSuite) TearDownSuite() { + suite.env.Cleanup() } func (suite *apiTestSuite) TestGetCheckerByName() { + suite.env.RunTestInAPIMode(suite.checkGetCheckerByName) +} + +func (suite *apiTestSuite) checkGetCheckerByName(cluster *tests.TestCluster) { re := suite.Require() testCases := []struct { name string @@ -89,7 +64,7 @@ func (suite *apiTestSuite) TestGetCheckerByName() { {name: "joint-state"}, } - s := suite.cluster.GetSchedulingPrimaryServer() + s := cluster.GetSchedulingPrimaryServer() urlPrefix := fmt.Sprintf("%s/scheduling/api/v1/checkers", s.GetAddr()) co := s.GetCoordinator() @@ -119,18 +94,24 @@ func (suite *apiTestSuite) TestGetCheckerByName() { } func (suite *apiTestSuite) TestAPIForward() { + suite.env.RunTestInAPIMode(suite.checkAPIForward) +} + +func (suite *apiTestSuite) checkAPIForward(cluster *tests.TestCluster) { re := suite.Require() re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/utils/apiutil/serverapi/checkHeader", "return(true)")) defer func() { re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/utils/apiutil/serverapi/checkHeader")) }() - urlPrefix := fmt.Sprintf("%s/pd/api/v1", suite.backendEndpoints) + leader := cluster.GetLeaderServer().GetServer() + urlPrefix := fmt.Sprintf("%s/pd/api/v1", leader.GetAddr()) var slice []string var resp map[string]interface{} testutil.Eventually(re, func() bool { - return suite.cluster.GetLeaderServer().GetServer().GetRaftCluster().IsServiceIndependent(utils.SchedulingServiceName) + return leader.GetRaftCluster().IsServiceIndependent(utils.SchedulingServiceName) }) + // Test operators err := testutil.ReadGetJSON(re, testDialClient, fmt.Sprintf("%s/%s", urlPrefix, "operators"), &slice, testutil.WithHeader(re, apiutil.ForwardToMicroServiceHeader, "true")) @@ -159,13 +140,18 @@ func (suite *apiTestSuite) TestAPIForward() { re.NoError(err) suite.False(resp["paused"].(bool)) - input := make(map[string]interface{}) - input["delay"] = 10 - pauseArgs, err := json.Marshal(input) - suite.NoError(err) - err = testutil.CheckPostJSON(testDialClient, fmt.Sprintf("%s/%s", urlPrefix, "checker/merge"), pauseArgs, - testutil.StatusOK(re), testutil.WithHeader(re, apiutil.ForwardToMicroServiceHeader, "true")) - suite.NoError(err) + // Test pause + postChecker := func(delay int) { + input := make(map[string]interface{}) + input["delay"] = delay + pauseArgs, err := json.Marshal(input) + suite.NoError(err) + err = testutil.CheckPostJSON(testDialClient, fmt.Sprintf("%s/%s", urlPrefix, "checker/merge"), pauseArgs, + testutil.StatusOK(re), testutil.WithHeader(re, apiutil.ForwardToMicroServiceHeader, "true")) + suite.NoError(err) + } + postChecker(30) + postChecker(0) // Test scheduler: // Need to redirect: @@ -183,12 +169,17 @@ func (suite *apiTestSuite) TestAPIForward() { re.NoError(err) re.Contains(slice, "balance-leader-scheduler") - input["delay"] = 30 - pauseArgs, err = json.Marshal(input) - suite.NoError(err) - err = testutil.CheckPostJSON(testDialClient, fmt.Sprintf("%s/%s", urlPrefix, "schedulers/balance-leader-scheduler"), pauseArgs, - testutil.WithHeader(re, apiutil.ForwardToMicroServiceHeader, "true")) - suite.NoError(err) + postScheduler := func(delay int) { + input := make(map[string]interface{}) + input["delay"] = delay + pauseArgs, err := json.Marshal(input) + suite.NoError(err) + err = testutil.CheckPostJSON(testDialClient, fmt.Sprintf("%s/%s", urlPrefix, "schedulers/balance-leader-scheduler"), pauseArgs, + testutil.WithHeader(re, apiutil.ForwardToMicroServiceHeader, "true")) + suite.NoError(err) + } + postScheduler(30) + postScheduler(0) err = testutil.ReadGetJSON(re, testDialClient, fmt.Sprintf("%s/%s", urlPrefix, "schedulers/diagnostic/balance-leader-scheduler"), &resp, testutil.WithHeader(re, apiutil.ForwardToMicroServiceHeader, "true")) @@ -212,7 +203,7 @@ func (suite *apiTestSuite) TestAPIForward() { suite.NoError(err) } - err = testutil.CheckPostJSON(testDialClient, fmt.Sprintf("%s/%s", urlPrefix, "schedulers"), pauseArgs, + err = testutil.CheckPostJSON(testDialClient, fmt.Sprintf("%s/%s", urlPrefix, "schedulers"), nil, testutil.WithoutHeader(re, apiutil.ForwardToMicroServiceHeader)) re.NoError(err) @@ -220,6 +211,14 @@ func (suite *apiTestSuite) TestAPIForward() { testutil.WithoutHeader(re, apiutil.ForwardToMicroServiceHeader)) re.NoError(err) + input := make(map[string]interface{}) + input["name"] = "balance-leader-scheduler" + b, err := json.Marshal(input) + re.NoError(err) + err = testutil.CheckPostJSON(testDialClient, fmt.Sprintf("%s/%s", urlPrefix, "schedulers"), b, + testutil.WithoutHeader(re, apiutil.ForwardToMicroServiceHeader)) + re.NoError(err) + // Test hotspot var hotRegions statistics.StoreHotPeersInfos err = testutil.ReadGetJSON(re, testDialClient, fmt.Sprintf("%s/%s", urlPrefix, "hotspot/regions/write"), &hotRegions, @@ -288,7 +287,7 @@ func (suite *apiTestSuite) TestAPIForward() { suite.NoError(err) // Test rules: only forward `GET` request var rules []*placement.Rule - tests.MustPutRegion(re, suite.cluster, 2, 1, []byte("a"), []byte("b"), core.SetApproximateSize(60)) + tests.MustPutRegion(re, cluster, 2, 1, []byte("a"), []byte("b"), core.SetApproximateSize(60)) rules = []*placement.Rule{ { GroupID: placement.DefaultGroupID, @@ -362,141 +361,141 @@ func (suite *apiTestSuite) TestAPIForward() { } func (suite *apiTestSuite) TestConfig() { - checkConfig := func(cluster *tests.TestCluster) { - re := suite.Require() - s := cluster.GetSchedulingPrimaryServer() - testutil.Eventually(re, func() bool { - return s.IsServing() - }, testutil.WithWaitFor(5*time.Second), testutil.WithTickInterval(50*time.Millisecond)) - addr := s.GetAddr() - urlPrefix := fmt.Sprintf("%s/scheduling/api/v1/config", addr) - - var cfg config.Config - testutil.ReadGetJSON(re, testDialClient, urlPrefix, &cfg) - suite.Equal(cfg.GetListenAddr(), s.GetConfig().GetListenAddr()) - suite.Equal(cfg.Schedule.LeaderScheduleLimit, s.GetConfig().Schedule.LeaderScheduleLimit) - suite.Equal(cfg.Schedule.EnableCrossTableMerge, s.GetConfig().Schedule.EnableCrossTableMerge) - suite.Equal(cfg.Replication.MaxReplicas, s.GetConfig().Replication.MaxReplicas) - suite.Equal(cfg.Replication.LocationLabels, s.GetConfig().Replication.LocationLabels) - suite.Equal(cfg.DataDir, s.GetConfig().DataDir) - testutil.Eventually(re, func() bool { - // wait for all schedulers to be loaded in scheduling server. - return len(cfg.Schedule.SchedulersPayload) == 5 - }) - suite.Contains(cfg.Schedule.SchedulersPayload, "balance-leader-scheduler") - suite.Contains(cfg.Schedule.SchedulersPayload, "balance-region-scheduler") - suite.Contains(cfg.Schedule.SchedulersPayload, "balance-hot-region-scheduler") - suite.Contains(cfg.Schedule.SchedulersPayload, "balance-witness-scheduler") - suite.Contains(cfg.Schedule.SchedulersPayload, "transfer-witness-leader-scheduler") - } - env := tests.NewSchedulingTestEnvironment(suite.T()) - env.RunTestInAPIMode(checkConfig) + suite.env.RunTestInAPIMode(suite.checkConfig) +} + +func (suite *apiTestSuite) checkConfig(cluster *tests.TestCluster) { + re := suite.Require() + s := cluster.GetSchedulingPrimaryServer() + testutil.Eventually(re, func() bool { + return s.IsServing() + }, testutil.WithWaitFor(5*time.Second), testutil.WithTickInterval(50*time.Millisecond)) + addr := s.GetAddr() + urlPrefix := fmt.Sprintf("%s/scheduling/api/v1/config", addr) + + var cfg config.Config + testutil.ReadGetJSON(re, testDialClient, urlPrefix, &cfg) + suite.Equal(cfg.GetListenAddr(), s.GetConfig().GetListenAddr()) + suite.Equal(cfg.Schedule.LeaderScheduleLimit, s.GetConfig().Schedule.LeaderScheduleLimit) + suite.Equal(cfg.Schedule.EnableCrossTableMerge, s.GetConfig().Schedule.EnableCrossTableMerge) + suite.Equal(cfg.Replication.MaxReplicas, s.GetConfig().Replication.MaxReplicas) + suite.Equal(cfg.Replication.LocationLabels, s.GetConfig().Replication.LocationLabels) + suite.Equal(cfg.DataDir, s.GetConfig().DataDir) + testutil.Eventually(re, func() bool { + // wait for all schedulers to be loaded in scheduling server. + return len(cfg.Schedule.SchedulersPayload) == 5 + }) + suite.Contains(cfg.Schedule.SchedulersPayload, "balance-leader-scheduler") + suite.Contains(cfg.Schedule.SchedulersPayload, "balance-region-scheduler") + suite.Contains(cfg.Schedule.SchedulersPayload, "balance-hot-region-scheduler") + suite.Contains(cfg.Schedule.SchedulersPayload, "balance-witness-scheduler") + suite.Contains(cfg.Schedule.SchedulersPayload, "transfer-witness-leader-scheduler") +} + +func (suite *apiTestSuite) TestConfigForward() { + suite.env.RunTestInAPIMode(suite.checkConfigForward) } -func TestConfigForward(t *testing.T) { - re := require.New(t) - checkConfigForward := func(cluster *tests.TestCluster) { - sche := cluster.GetSchedulingPrimaryServer() - opts := sche.GetPersistConfig() - var cfg map[string]interface{} - addr := cluster.GetLeaderServer().GetAddr() - urlPrefix := fmt.Sprintf("%s/pd/api/v1/config", addr) - - // Test config forward - // Expect to get same config in scheduling server and api server - testutil.Eventually(re, func() bool { - testutil.ReadGetJSON(re, testDialClient, urlPrefix, &cfg) - re.Equal(cfg["schedule"].(map[string]interface{})["leader-schedule-limit"], - float64(opts.GetLeaderScheduleLimit())) - re.Equal(cfg["replication"].(map[string]interface{})["max-replicas"], - float64(opts.GetReplicationConfig().MaxReplicas)) - schedulers := cfg["schedule"].(map[string]interface{})["schedulers-payload"].(map[string]interface{}) - return len(schedulers) == 5 - }) - - // Test to change config in api server - // Expect to get new config in scheduling server and api server - reqData, err := json.Marshal(map[string]interface{}{ - "max-replicas": 4, - }) - re.NoError(err) - err = testutil.CheckPostJSON(testDialClient, urlPrefix, reqData, testutil.StatusOK(re)) - re.NoError(err) - testutil.Eventually(re, func() bool { - testutil.ReadGetJSON(re, testDialClient, urlPrefix, &cfg) - return cfg["replication"].(map[string]interface{})["max-replicas"] == 4. && - opts.GetReplicationConfig().MaxReplicas == 4. - }) - - // Test to change config only in scheduling server - // Expect to get new config in scheduling server but not old config in api server - opts.GetScheduleConfig().LeaderScheduleLimit = 100 - re.Equal(100, int(opts.GetLeaderScheduleLimit())) +func (suite *apiTestSuite) checkConfigForward(cluster *tests.TestCluster) { + re := suite.Require() + sche := cluster.GetSchedulingPrimaryServer() + opts := sche.GetPersistConfig() + var cfg map[string]interface{} + addr := cluster.GetLeaderServer().GetAddr() + urlPrefix := fmt.Sprintf("%s/pd/api/v1/config", addr) + + // Test config forward + // Expect to get same config in scheduling server and api server + testutil.Eventually(re, func() bool { testutil.ReadGetJSON(re, testDialClient, urlPrefix, &cfg) - re.Equal(100., cfg["schedule"].(map[string]interface{})["leader-schedule-limit"]) - opts.GetReplicationConfig().MaxReplicas = 5 - re.Equal(5, int(opts.GetReplicationConfig().MaxReplicas)) + re.Equal(cfg["schedule"].(map[string]interface{})["leader-schedule-limit"], + float64(opts.GetLeaderScheduleLimit())) + re.Equal(cfg["replication"].(map[string]interface{})["max-replicas"], + float64(opts.GetReplicationConfig().MaxReplicas)) + schedulers := cfg["schedule"].(map[string]interface{})["schedulers-payload"].(map[string]interface{}) + return len(schedulers) == 5 + }) + + // Test to change config in api server + // Expect to get new config in scheduling server and api server + reqData, err := json.Marshal(map[string]interface{}{ + "max-replicas": 4, + }) + re.NoError(err) + err = testutil.CheckPostJSON(testDialClient, urlPrefix, reqData, testutil.StatusOK(re)) + re.NoError(err) + testutil.Eventually(re, func() bool { testutil.ReadGetJSON(re, testDialClient, urlPrefix, &cfg) - re.Equal(5., cfg["replication"].(map[string]interface{})["max-replicas"]) - } - env := tests.NewSchedulingTestEnvironment(t) - env.RunTestInAPIMode(checkConfigForward) + return cfg["replication"].(map[string]interface{})["max-replicas"] == 4. && + opts.GetReplicationConfig().MaxReplicas == 4. + }) + + // Test to change config only in scheduling server + // Expect to get new config in scheduling server but not old config in api server + opts.GetScheduleConfig().LeaderScheduleLimit = 100 + re.Equal(100, int(opts.GetLeaderScheduleLimit())) + testutil.ReadGetJSON(re, testDialClient, urlPrefix, &cfg) + re.Equal(100., cfg["schedule"].(map[string]interface{})["leader-schedule-limit"]) + opts.GetReplicationConfig().MaxReplicas = 5 + re.Equal(5, int(opts.GetReplicationConfig().MaxReplicas)) + testutil.ReadGetJSON(re, testDialClient, urlPrefix, &cfg) + re.Equal(5., cfg["replication"].(map[string]interface{})["max-replicas"]) } -func TestAdminRegionCache(t *testing.T) { - re := require.New(t) - checkAdminRegionCache := func(cluster *tests.TestCluster) { - r1 := core.NewTestRegionInfo(10, 1, []byte(""), []byte("b"), core.SetRegionConfVer(100), core.SetRegionVersion(100)) - tests.MustPutRegionInfo(re, cluster, r1) - r2 := core.NewTestRegionInfo(20, 1, []byte("b"), []byte("c"), core.SetRegionConfVer(100), core.SetRegionVersion(100)) - tests.MustPutRegionInfo(re, cluster, r2) - r3 := core.NewTestRegionInfo(30, 1, []byte("c"), []byte(""), core.SetRegionConfVer(100), core.SetRegionVersion(100)) - tests.MustPutRegionInfo(re, cluster, r3) - - schedulingServer := cluster.GetSchedulingPrimaryServer() - re.Equal(3, schedulingServer.GetCluster().GetRegionCount([]byte{}, []byte{})) - - addr := schedulingServer.GetAddr() - urlPrefix := fmt.Sprintf("%s/scheduling/api/v1/admin/cache/regions", addr) - err := testutil.CheckDelete(testDialClient, fmt.Sprintf("%s/%s", urlPrefix, "30"), testutil.StatusOK(re)) - re.NoError(err) - re.Equal(2, schedulingServer.GetCluster().GetRegionCount([]byte{}, []byte{})) - - err = testutil.CheckDelete(testDialClient, urlPrefix, testutil.StatusOK(re)) - re.NoError(err) - re.Equal(0, schedulingServer.GetCluster().GetRegionCount([]byte{}, []byte{})) - } - env := tests.NewSchedulingTestEnvironment(t) - env.RunTestInAPIMode(checkAdminRegionCache) +func (suite *apiTestSuite) TestAdminRegionCache() { + suite.env.RunTestInAPIMode(suite.checkAdminRegionCache) } -func TestAdminRegionCacheForward(t *testing.T) { - re := require.New(t) - checkAdminRegionCache := func(cluster *tests.TestCluster) { - r1 := core.NewTestRegionInfo(10, 1, []byte(""), []byte("b"), core.SetRegionConfVer(100), core.SetRegionVersion(100)) - tests.MustPutRegionInfo(re, cluster, r1) - r2 := core.NewTestRegionInfo(20, 1, []byte("b"), []byte("c"), core.SetRegionConfVer(100), core.SetRegionVersion(100)) - tests.MustPutRegionInfo(re, cluster, r2) - r3 := core.NewTestRegionInfo(30, 1, []byte("c"), []byte(""), core.SetRegionConfVer(100), core.SetRegionVersion(100)) - tests.MustPutRegionInfo(re, cluster, r3) - - apiServer := cluster.GetLeaderServer().GetServer() - schedulingServer := cluster.GetSchedulingPrimaryServer() - re.Equal(3, schedulingServer.GetCluster().GetRegionCount([]byte{}, []byte{})) - re.Equal(3, apiServer.GetRaftCluster().GetRegionCount([]byte{}, []byte{}).Count) - - addr := cluster.GetLeaderServer().GetAddr() - urlPrefix := fmt.Sprintf("%s/pd/api/v1/admin/cache/region", addr) - err := testutil.CheckDelete(testDialClient, fmt.Sprintf("%s/%s", urlPrefix, "30"), testutil.StatusOK(re)) - re.NoError(err) - re.Equal(2, schedulingServer.GetCluster().GetRegionCount([]byte{}, []byte{})) - re.Equal(2, apiServer.GetRaftCluster().GetRegionCount([]byte{}, []byte{}).Count) - - err = testutil.CheckDelete(testDialClient, urlPrefix+"s", testutil.StatusOK(re)) - re.NoError(err) - re.Equal(0, schedulingServer.GetCluster().GetRegionCount([]byte{}, []byte{})) - re.Equal(0, apiServer.GetRaftCluster().GetRegionCount([]byte{}, []byte{}).Count) - } - env := tests.NewSchedulingTestEnvironment(t) - env.RunTestInAPIMode(checkAdminRegionCache) +func (suite *apiTestSuite) checkAdminRegionCache(cluster *tests.TestCluster) { + re := suite.Require() + r1 := core.NewTestRegionInfo(10, 1, []byte(""), []byte("b"), core.SetRegionConfVer(100), core.SetRegionVersion(100)) + tests.MustPutRegionInfo(re, cluster, r1) + r2 := core.NewTestRegionInfo(20, 1, []byte("b"), []byte("c"), core.SetRegionConfVer(100), core.SetRegionVersion(100)) + tests.MustPutRegionInfo(re, cluster, r2) + r3 := core.NewTestRegionInfo(30, 1, []byte("c"), []byte(""), core.SetRegionConfVer(100), core.SetRegionVersion(100)) + tests.MustPutRegionInfo(re, cluster, r3) + + schedulingServer := cluster.GetSchedulingPrimaryServer() + re.Equal(3, schedulingServer.GetCluster().GetRegionCount([]byte{}, []byte{})) + + addr := schedulingServer.GetAddr() + urlPrefix := fmt.Sprintf("%s/scheduling/api/v1/admin/cache/regions", addr) + err := testutil.CheckDelete(testDialClient, fmt.Sprintf("%s/%s", urlPrefix, "30"), testutil.StatusOK(re)) + re.NoError(err) + re.Equal(2, schedulingServer.GetCluster().GetRegionCount([]byte{}, []byte{})) + + err = testutil.CheckDelete(testDialClient, urlPrefix, testutil.StatusOK(re)) + re.NoError(err) + re.Equal(0, schedulingServer.GetCluster().GetRegionCount([]byte{}, []byte{})) +} + +func (suite *apiTestSuite) TestAdminRegionCacheForward() { + suite.env.RunTestInAPIMode(suite.checkAdminRegionCacheForward) +} + +func (suite *apiTestSuite) checkAdminRegionCacheForward(cluster *tests.TestCluster) { + re := suite.Require() + r1 := core.NewTestRegionInfo(10, 1, []byte(""), []byte("b"), core.SetRegionConfVer(100), core.SetRegionVersion(100)) + tests.MustPutRegionInfo(re, cluster, r1) + r2 := core.NewTestRegionInfo(20, 1, []byte("b"), []byte("c"), core.SetRegionConfVer(100), core.SetRegionVersion(100)) + tests.MustPutRegionInfo(re, cluster, r2) + r3 := core.NewTestRegionInfo(30, 1, []byte("c"), []byte(""), core.SetRegionConfVer(100), core.SetRegionVersion(100)) + tests.MustPutRegionInfo(re, cluster, r3) + + apiServer := cluster.GetLeaderServer().GetServer() + schedulingServer := cluster.GetSchedulingPrimaryServer() + re.Equal(3, schedulingServer.GetCluster().GetRegionCount([]byte{}, []byte{})) + re.Equal(3, apiServer.GetRaftCluster().GetRegionCount([]byte{}, []byte{}).Count) + + addr := cluster.GetLeaderServer().GetAddr() + urlPrefix := fmt.Sprintf("%s/pd/api/v1/admin/cache/region", addr) + err := testutil.CheckDelete(testDialClient, fmt.Sprintf("%s/%s", urlPrefix, "30"), testutil.StatusOK(re)) + re.NoError(err) + re.Equal(2, schedulingServer.GetCluster().GetRegionCount([]byte{}, []byte{})) + re.Equal(2, apiServer.GetRaftCluster().GetRegionCount([]byte{}, []byte{}).Count) + + err = testutil.CheckDelete(testDialClient, urlPrefix+"s", testutil.StatusOK(re)) + re.NoError(err) + re.Equal(0, schedulingServer.GetCluster().GetRegionCount([]byte{}, []byte{})) + re.Equal(0, apiServer.GetRaftCluster().GetRegionCount([]byte{}, []byte{}).Count) } diff --git a/tests/pdctl/config/config_test.go b/tests/pdctl/config/config_test.go index 3b3310185b1..c63160a32e5 100644 --- a/tests/pdctl/config/config_test.go +++ b/tests/pdctl/config/config_test.go @@ -17,8 +17,10 @@ package config_test import ( "context" "encoding/json" + "net/http" "os" "reflect" + "strconv" "strings" "testing" "time" @@ -38,6 +40,13 @@ import ( pdctlCmd "github.com/tikv/pd/tools/pd-ctl/pdctl" ) +// testDialClient used to dial http request. only used for test. +var testDialClient = &http.Client{ + Transport: &http.Transport{ + DisableKeepAlives: true, + }, +} + type testCase struct { name string value interface{} @@ -54,16 +63,43 @@ func (t *testCase) judge(re *require.Assertions, scheduleConfigs ...*sc.Schedule type configTestSuite struct { suite.Suite + env *tests.SchedulingTestEnvironment } func TestConfigTestSuite(t *testing.T) { suite.Run(t, new(configTestSuite)) } +func (suite *configTestSuite) SetupSuite() { + suite.env = tests.NewSchedulingTestEnvironment(suite.T()) +} + +func (suite *configTestSuite) TearDownSuite() { + suite.env.Cleanup() +} + +func (suite *configTestSuite) TearDownTest() { + cleanFunc := func(cluster *tests.TestCluster) { + def := placement.GroupBundle{ + ID: "pd", + Rules: []*placement.Rule{ + {GroupID: "pd", ID: "default", Role: "voter", Count: 3}, + }, + } + data, err := json.Marshal([]placement.GroupBundle{def}) + suite.NoError(err) + leader := cluster.GetLeaderServer() + suite.NotNil(leader) + urlPrefix := leader.GetAddr() + err = testutil.CheckPostJSON(testDialClient, urlPrefix+"/pd/api/v1/config/placement-rule", data, testutil.StatusOK(suite.Require())) + suite.NoError(err) + } + suite.env.RunFuncInTwoModes(cleanFunc) +} + func (suite *configTestSuite) TestConfig() { suite.NoError(failpoint.Enable("github.com/tikv/pd/pkg/dashboard/adapter/skipDashboardLoop", `return(true)`)) - env := tests.NewSchedulingTestEnvironment(suite.T()) - env.RunTestInTwoModes(suite.checkConfig) + suite.env.RunTestInTwoModes(suite.checkConfig) suite.NoError(failpoint.Disable("github.com/tikv/pd/pkg/dashboard/adapter/skipDashboardLoop")) } @@ -79,7 +115,6 @@ func (suite *configTestSuite) checkConfig(cluster *tests.TestCluster) { } svr := leaderServer.GetServer() tests.MustPutStore(re, cluster, store) - defer cluster.Destroy() // config show args := []string{"-u", pdAddr, "config", "show"} @@ -111,6 +146,7 @@ func (suite *configTestSuite) checkConfig(cluster *tests.TestCluster) { re.NoError(err) re.False(svr.GetPDServerConfig().TraceRegionFlow) + origin := svr.GetPDServerConfig().FlowRoundByDigit args = []string{"-u", pdAddr, "config", "set", "flow-round-by-digit", "10"} _, err = pdctl.ExecuteCommand(cmd, args...) re.NoError(err) @@ -120,6 +156,17 @@ func (suite *configTestSuite) checkConfig(cluster *tests.TestCluster) { _, err = pdctl.ExecuteCommand(cmd, args...) re.Error(err) + args = []string{"-u", pdAddr, "config", "set", "flow-round-by-digit", strconv.Itoa(origin)} + _, err = pdctl.ExecuteCommand(cmd, args...) + re.NoError(err) + testutil.Eventually(re, func() bool { // wait for the config to be synced to the scheduling server + output, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "show", "server") + re.NoError(err) + var conf config.PDServerConfig + re.NoError(json.Unmarshal(output, &conf)) + return conf.FlowRoundByDigit == origin + }) + // config show schedule args = []string{"-u", pdAddr, "config", "show", "schedule"} output, err = pdctl.ExecuteCommand(cmd, args...) @@ -295,8 +342,7 @@ func (suite *configTestSuite) checkConfig(cluster *tests.TestCluster) { } func (suite *configTestSuite) TestPlacementRules() { - env := tests.NewSchedulingTestEnvironment(suite.T()) - env.RunTestInTwoModes(suite.checkPlacementRules) + suite.env.RunTestInTwoModes(suite.checkPlacementRules) } func (suite *configTestSuite) checkPlacementRules(cluster *tests.TestCluster) { @@ -311,7 +357,6 @@ func (suite *configTestSuite) checkPlacementRules(cluster *tests.TestCluster) { LastHeartbeat: time.Now().UnixNano(), } tests.MustPutStore(re, cluster, store) - defer cluster.Destroy() output, err := pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "enable") re.NoError(err) @@ -363,8 +408,7 @@ func (suite *configTestSuite) checkPlacementRules(cluster *tests.TestCluster) { } func (suite *configTestSuite) TestPlacementRuleGroups() { - env := tests.NewSchedulingTestEnvironment(suite.T()) - env.RunTestInTwoModes(suite.checkPlacementRuleGroups) + suite.env.RunTestInTwoModes(suite.checkPlacementRuleGroups) } func (suite *configTestSuite) checkPlacementRuleGroups(cluster *tests.TestCluster) { @@ -379,8 +423,6 @@ func (suite *configTestSuite) checkPlacementRuleGroups(cluster *tests.TestCluste LastHeartbeat: time.Now().UnixNano(), } tests.MustPutStore(re, cluster, store) - defer cluster.Destroy() - output, err := pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "enable") re.NoError(err) re.Contains(string(output), "Success!") @@ -443,8 +485,7 @@ func (suite *configTestSuite) checkPlacementRuleGroups(cluster *tests.TestCluste } func (suite *configTestSuite) TestPlacementRuleBundle() { - env := tests.NewSchedulingTestEnvironment(suite.T()) - env.RunTestInTwoModes(suite.checkPlacementRuleBundle) + suite.env.RunTestInTwoModes(suite.checkPlacementRuleBundle) } func (suite *configTestSuite) checkPlacementRuleBundle(cluster *tests.TestCluster) { @@ -459,7 +500,6 @@ func (suite *configTestSuite) checkPlacementRuleBundle(cluster *tests.TestCluste LastHeartbeat: time.Now().UnixNano(), } tests.MustPutStore(re, cluster, store) - defer cluster.Destroy() output, err := pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "enable") re.NoError(err) @@ -549,6 +589,25 @@ func (suite *configTestSuite) checkPlacementRuleBundle(cluster *tests.TestCluste suite.checkLoadRuleBundle(pdAddr, fname, []placement.GroupBundle{ {ID: "pf", Index: 0, Override: false, Rules: []*placement.Rule{{GroupID: "pf", ID: placement.DefaultRuleID, Role: placement.Voter, Count: 3}}}, }) + + // set default rule only + bundles = []placement.GroupBundle{{ + ID: "pd", + Rules: []*placement.Rule{ + {GroupID: "pd", ID: "default", Role: "voter", Count: 3}, + }, + }} + b, err = json.Marshal(bundles) + re.NoError(err) + re.NoError(os.WriteFile(fname, b, 0600)) + _, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-bundle", "save", "--in="+fname) + re.NoError(err) + _, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-bundle", "delete", "--regexp", ".*f") + re.NoError(err) + + suite.checkLoadRuleBundle(pdAddr, fname, []placement.GroupBundle{ + {ID: "pd", Index: 0, Override: false, Rules: []*placement.Rule{{GroupID: "pd", ID: placement.DefaultRuleID, Role: placement.Voter, Count: 3}}}, + }) } func (suite *configTestSuite) checkLoadRuleBundle(pdAddr string, fname string, expectValues []placement.GroupBundle) { @@ -627,7 +686,6 @@ func TestReplicationMode(t *testing.T) { leaderServer := cluster.GetLeaderServer() re.NoError(leaderServer.BootstrapCluster()) tests.MustPutStore(re, cluster, store) - defer cluster.Destroy() conf := config.ReplicationModeConfig{ ReplicationMode: "majority", @@ -667,8 +725,7 @@ func TestReplicationMode(t *testing.T) { } func (suite *configTestSuite) TestUpdateDefaultReplicaConfig() { - env := tests.NewSchedulingTestEnvironment(suite.T()) - env.RunTestInTwoModes(suite.checkUpdateDefaultReplicaConfig) + suite.env.RunTestInTwoModes(suite.checkUpdateDefaultReplicaConfig) } func (suite *configTestSuite) checkUpdateDefaultReplicaConfig(cluster *tests.TestCluster) { @@ -682,8 +739,6 @@ func (suite *configTestSuite) checkUpdateDefaultReplicaConfig(cluster *tests.Tes State: metapb.StoreState_Up, } tests.MustPutStore(re, cluster, store) - defer cluster.Destroy() - checkMaxReplicas := func(expect uint64) { args := []string{"-u", pdAddr, "config", "show", "replication"} testutil.Eventually(re, func() bool { // wait for the config to be synced to the scheduling server @@ -819,8 +874,7 @@ func (suite *configTestSuite) checkUpdateDefaultReplicaConfig(cluster *tests.Tes } func (suite *configTestSuite) TestPDServerConfig() { - env := tests.NewSchedulingTestEnvironment(suite.T()) - env.RunTestInTwoModes(suite.checkPDServerConfig) + suite.env.RunTestInTwoModes(suite.checkPDServerConfig) } func (suite *configTestSuite) checkPDServerConfig(cluster *tests.TestCluster) { diff --git a/tests/pdctl/hot/hot_test.go b/tests/pdctl/hot/hot_test.go index 8cab8ea9ab2..03c26f40441 100644 --- a/tests/pdctl/hot/hot_test.go +++ b/tests/pdctl/hot/hot_test.go @@ -42,30 +42,40 @@ import ( type hotTestSuite struct { suite.Suite + env *tests.SchedulingTestEnvironment } func TestHotTestSuite(t *testing.T) { suite.Run(t, new(hotTestSuite)) } -func (suite *hotTestSuite) TestHot() { - var start time.Time - start = start.Add(time.Hour) - opts := []tests.ConfigOption{ +func (suite *hotTestSuite) SetupSuite() { + suite.env = tests.NewSchedulingTestEnvironment(suite.T(), func(conf *config.Config, serverName string) { - conf.Schedule.MaxStoreDownTime.Duration = time.Since(start) + conf.Schedule.MaxStoreDownTime.Duration = time.Hour + conf.Schedule.HotRegionCacheHitsThreshold = 0 }, + ) +} + +func (suite *hotTestSuite) TearDownSuite() { + suite.env.Cleanup() +} + +func (suite *hotTestSuite) TearDownTest() { + cleanFunc := func(cluster *tests.TestCluster) { + leader := cluster.GetLeaderServer() + hotStat := leader.GetRaftCluster().GetHotStat() + if sche := cluster.GetSchedulingPrimaryServer(); sche != nil { + hotStat = sche.GetCluster().GetHotStat() + } + hotStat.HotCache.CleanCache() } - env := tests.NewSchedulingTestEnvironment(suite.T(), opts...) - env.RunTestInTwoModes(suite.checkHot) + suite.env.RunFuncInTwoModes(cleanFunc) +} - opts = append(opts, func(conf *config.Config, serverName string) { - conf.Schedule.HotRegionCacheHitsThreshold = 0 - }) - env = tests.NewSchedulingTestEnvironment(suite.T(), opts...) - env.RunTestInTwoModes(suite.checkHotWithoutHotPeer) - env = tests.NewSchedulingTestEnvironment(suite.T(), opts...) - env.RunTestInTwoModes(suite.checkHotWithStoreID) +func (suite *hotTestSuite) TestHot() { + suite.env.RunTestInTwoModes(suite.checkHot) } func (suite *hotTestSuite) checkHot(cluster *tests.TestCluster) { @@ -229,6 +239,10 @@ func (suite *hotTestSuite) checkHot(cluster *tests.TestCluster) { testCommand(reportIntervals, "read") } +func (suite *hotTestSuite) TestHotWithStoreID() { + suite.env.RunTestInTwoModes(suite.checkHotWithStoreID) +} + func (suite *hotTestSuite) checkHotWithStoreID(cluster *tests.TestCluster) { re := suite.Require() statistics.Denoising = false @@ -292,6 +306,10 @@ func (suite *hotTestSuite) checkHotWithStoreID(cluster *tests.TestCluster) { re.Equal(float64(200000000), hotRegion.AsLeader[1].TotalBytesRate) } +func (suite *hotTestSuite) TestHotWithoutHotPeer() { + suite.env.RunTestInTwoModes(suite.checkHotWithoutHotPeer) +} + func (suite *hotTestSuite) checkHotWithoutHotPeer(cluster *tests.TestCluster) { re := suite.Require() statistics.Denoising = false @@ -363,10 +381,10 @@ func (suite *hotTestSuite) checkHotWithoutHotPeer(cluster *tests.TestCluster) { hotRegion := statistics.StoreHotPeersInfos{} re.NoError(err) re.NoError(json.Unmarshal(output, &hotRegion)) - re.Equal(hotRegion.AsPeer[1].Count, 0) + re.Equal(0, hotRegion.AsPeer[1].Count) re.Equal(0.0, hotRegion.AsPeer[1].TotalBytesRate) re.Equal(load, hotRegion.AsPeer[1].StoreByteRate) - re.Equal(hotRegion.AsLeader[1].Count, 0) + re.Equal(0, hotRegion.AsLeader[1].Count) re.Equal(0.0, hotRegion.AsLeader[1].TotalBytesRate) re.Equal(0.0, hotRegion.AsLeader[1].StoreByteRate) // write leader sum } diff --git a/tests/pdctl/operator/operator_test.go b/tests/pdctl/operator/operator_test.go index 8bb034993fa..aa2fe5d1304 100644 --- a/tests/pdctl/operator/operator_test.go +++ b/tests/pdctl/operator/operator_test.go @@ -34,27 +34,30 @@ import ( type operatorTestSuite struct { suite.Suite + env *tests.SchedulingTestEnvironment } func TestOperatorTestSuite(t *testing.T) { suite.Run(t, new(operatorTestSuite)) } -func (suite *operatorTestSuite) TestOperator() { - var start time.Time - start = start.Add(time.Hour) - opts := []tests.ConfigOption{ - // TODO: enable placementrules +func (suite *operatorTestSuite) SetupSuite() { + suite.env = tests.NewSchedulingTestEnvironment(suite.T(), func(conf *config.Config, serverName string) { + // TODO: enable placement rules conf.Replication.MaxReplicas = 2 conf.Replication.EnablePlacementRules = false + conf.Schedule.MaxStoreDownTime.Duration = time.Hour }, - func(conf *config.Config, serverName string) { - conf.Schedule.MaxStoreDownTime.Duration = time.Since(start) - }, - } - env := tests.NewSchedulingTestEnvironment(suite.T(), opts...) - env.RunTestInTwoModes(suite.checkOperator) + ) +} + +func (suite *operatorTestSuite) TearDownSuite() { + suite.env.Cleanup() +} + +func (suite *operatorTestSuite) TestOperator() { + suite.env.RunTestInTwoModes(suite.checkOperator) } func (suite *operatorTestSuite) checkOperator(cluster *tests.TestCluster) { diff --git a/tests/pdctl/scheduler/scheduler_test.go b/tests/pdctl/scheduler/scheduler_test.go index d5bea895683..d8d54a79d13 100644 --- a/tests/pdctl/scheduler/scheduler_test.go +++ b/tests/pdctl/scheduler/scheduler_test.go @@ -23,6 +23,7 @@ import ( "testing" "time" + "github.com/pingcap/failpoint" "github.com/pingcap/kvproto/pkg/metapb" "github.com/spf13/cobra" "github.com/stretchr/testify/require" @@ -39,15 +40,61 @@ import ( type schedulerTestSuite struct { suite.Suite + env *tests.SchedulingTestEnvironment + defaultSchedulers []string } func TestSchedulerTestSuite(t *testing.T) { suite.Run(t, new(schedulerTestSuite)) } +func (suite *schedulerTestSuite) SetupSuite() { + suite.NoError(failpoint.Enable("github.com/tikv/pd/server/cluster/skipStoreConfigSync", `return(true)`)) + suite.env = tests.NewSchedulingTestEnvironment(suite.T()) + suite.defaultSchedulers = []string{ + "balance-leader-scheduler", + "balance-region-scheduler", + "balance-hot-region-scheduler", + "balance-witness-scheduler", + "transfer-witness-leader-scheduler", + } +} + +func (suite *schedulerTestSuite) TearDownSuite() { + suite.env.Cleanup() + suite.NoError(failpoint.Disable("github.com/tikv/pd/server/cluster/skipStoreConfigSync")) +} + +func (suite *schedulerTestSuite) TearDownTest() { + cleanFunc := func(cluster *tests.TestCluster) { + re := suite.Require() + pdAddr := cluster.GetConfig().GetClientURL() + cmd := pdctlCmd.GetRootCmd() + + var currentSchedulers []string + mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "show"}, ¤tSchedulers) + for _, scheduler := range suite.defaultSchedulers { + if slice.NoneOf(currentSchedulers, func(i int) bool { + return currentSchedulers[i] == scheduler + }) { + echo := mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "add", scheduler}, nil) + re.Contains(echo, "Success!") + } + } + for _, scheduler := range currentSchedulers { + if slice.NoneOf(suite.defaultSchedulers, func(i int) bool { + return suite.defaultSchedulers[i] == scheduler + }) { + echo := mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "remove", scheduler}, nil) + re.Contains(echo, "Success!") + } + } + } + suite.env.RunFuncInTwoModes(cleanFunc) +} + func (suite *schedulerTestSuite) TestScheduler() { - env := tests.NewSchedulingTestEnvironment(suite.T()) - env.RunTestInTwoModes(suite.checkScheduler) + suite.env.RunTestInTwoModes(suite.checkScheduler) } func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) { @@ -436,6 +483,7 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) { for _, store := range stores { version := versioninfo.HotScheduleWithQuery store.Version = versioninfo.MinSupportedVersion(version).String() + store.LastHeartbeat = time.Now().UnixNano() tests.MustPutStore(re, cluster, store) } re.Equal("5.2.0", leaderServer.GetClusterVersion().String()) @@ -564,8 +612,7 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) { } func (suite *schedulerTestSuite) TestSchedulerDiagnostic() { - env := tests.NewSchedulingTestEnvironment(suite.T()) - env.RunTestInTwoModes(suite.checkSchedulerDiagnostic) + suite.env.RunTestInTwoModes(suite.checkSchedulerDiagnostic) } func (suite *schedulerTestSuite) checkSchedulerDiagnostic(cluster *tests.TestCluster) { diff --git a/tests/server/api/checker_test.go b/tests/server/api/checker_test.go index 0f359553b73..8037fcc3989 100644 --- a/tests/server/api/checker_test.go +++ b/tests/server/api/checker_test.go @@ -27,14 +27,23 @@ import ( type checkerTestSuite struct { suite.Suite + env *tests.SchedulingTestEnvironment } func TestCheckerTestSuite(t *testing.T) { suite.Run(t, new(checkerTestSuite)) } + +func (suite *checkerTestSuite) SetupSuite() { + suite.env = tests.NewSchedulingTestEnvironment(suite.T()) +} + +func (suite *checkerTestSuite) TearDownSuite() { + suite.env.Cleanup() +} + func (suite *checkerTestSuite) TestAPI() { - env := tests.NewSchedulingTestEnvironment(suite.T()) - env.RunTestInTwoModes(suite.checkAPI) + suite.env.RunTestInTwoModes(suite.checkAPI) } func (suite *checkerTestSuite) checkAPI(cluster *tests.TestCluster) { diff --git a/tests/server/api/operator_test.go b/tests/server/api/operator_test.go index c27ebbe7ee8..41a687b1181 100644 --- a/tests/server/api/operator_test.go +++ b/tests/server/api/operator_test.go @@ -45,20 +45,26 @@ var ( type operatorTestSuite struct { suite.Suite + env *tests.SchedulingTestEnvironment } func TestOperatorTestSuite(t *testing.T) { suite.Run(t, new(operatorTestSuite)) } -func (suite *operatorTestSuite) TestAddRemovePeer() { - opts := []tests.ConfigOption{ +func (suite *operatorTestSuite) SetupSuite() { + suite.env = tests.NewSchedulingTestEnvironment(suite.T(), func(conf *config.Config, serverName string) { conf.Replication.MaxReplicas = 1 - }, - } - env := tests.NewSchedulingTestEnvironment(suite.T(), opts...) - env.RunTestInTwoModes(suite.checkAddRemovePeer) + }) +} + +func (suite *operatorTestSuite) TearDownSuite() { + suite.env.Cleanup() +} + +func (suite *operatorTestSuite) TestAddRemovePeer() { + suite.env.RunTestInTwoModes(suite.checkAddRemovePeer) } func (suite *operatorTestSuite) checkAddRemovePeer(cluster *tests.TestCluster) { @@ -168,17 +174,36 @@ func (suite *operatorTestSuite) checkAddRemovePeer(cluster *tests.TestCluster) { } func (suite *operatorTestSuite) TestMergeRegionOperator() { - opts := []tests.ConfigOption{ - func(conf *config.Config, serverName string) { - conf.Replication.MaxReplicas = 1 - }, - } - env := tests.NewSchedulingTestEnvironment(suite.T(), opts...) - env.RunTestInTwoModes(suite.checkMergeRegionOperator) + suite.env.RunTestInTwoModes(suite.checkMergeRegionOperator) } func (suite *operatorTestSuite) checkMergeRegionOperator(cluster *tests.TestCluster) { re := suite.Require() + stores := []*metapb.Store{ + { + Id: 1, + State: metapb.StoreState_Up, + NodeState: metapb.NodeState_Serving, + LastHeartbeat: time.Now().UnixNano(), + }, + { + Id: 2, + State: metapb.StoreState_Up, + NodeState: metapb.NodeState_Serving, + LastHeartbeat: time.Now().UnixNano(), + }, + { + Id: 3, + State: metapb.StoreState_Up, + NodeState: metapb.NodeState_Serving, + LastHeartbeat: time.Now().UnixNano(), + }, + } + + for _, store := range stores { + tests.MustPutStore(re, cluster, store) + } + suite.pauseRuleChecker(cluster) r1 := core.NewTestRegionInfo(10, 1, []byte(""), []byte("b"), core.SetWrittenBytes(1000), core.SetReadBytes(1000), core.SetRegionConfVer(1), core.SetRegionVersion(1)) tests.MustPutRegionInfo(re, cluster, r1) @@ -204,13 +229,13 @@ func (suite *operatorTestSuite) checkMergeRegionOperator(cluster *tests.TestClus } func (suite *operatorTestSuite) TestTransferRegionWithPlacementRule() { - opts := []tests.ConfigOption{ + // use a new environment to avoid affecting other tests + env := tests.NewSchedulingTestEnvironment(suite.T(), func(conf *config.Config, serverName string) { conf.Replication.MaxReplicas = 3 - }, - } - env := tests.NewSchedulingTestEnvironment(suite.T(), opts...) + }) env.RunTestInTwoModes(suite.checkTransferRegionWithPlacementRule) + env.Cleanup() } func (suite *operatorTestSuite) checkTransferRegionWithPlacementRule(cluster *tests.TestCluster) { diff --git a/tests/server/api/region_test.go b/tests/server/api/region_test.go index dcd31d6462d..450995a6e5e 100644 --- a/tests/server/api/region_test.go +++ b/tests/server/api/region_test.go @@ -19,6 +19,7 @@ import ( "encoding/json" "fmt" "net/http" + "strconv" "testing" "github.com/pingcap/failpoint" @@ -27,21 +28,67 @@ import ( "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/schedule/placement" tu "github.com/tikv/pd/pkg/utils/testutil" - "github.com/tikv/pd/server/config" "github.com/tikv/pd/tests" ) type regionTestSuite struct { suite.Suite + env *tests.SchedulingTestEnvironment } func TestRegionTestSuite(t *testing.T) { suite.Run(t, new(regionTestSuite)) } +func (suite *regionTestSuite) SetupSuite() { + suite.env = tests.NewSchedulingTestEnvironment(suite.T()) +} + +func (suite *regionTestSuite) TearDownSuite() { + suite.env.Cleanup() +} + +func (suite *regionTestSuite) TearDownTest() { + cleanFunc := func(cluster *tests.TestCluster) { + // clean region cache + leader := cluster.GetLeaderServer() + re := suite.Require() + pdAddr := cluster.GetConfig().GetClientURL() + for _, region := range leader.GetRegions() { + url := fmt.Sprintf("%s/pd/api/v1/admin/cache/region/%d", pdAddr, region.GetID()) + err := tu.CheckDelete(testDialClient, url, tu.StatusOK(re)) + suite.NoError(err) + } + suite.Empty(leader.GetRegions()) + // clean rules + def := placement.GroupBundle{ + ID: "pd", + Rules: []*placement.Rule{ + {GroupID: "pd", ID: "default", Role: "voter", Count: 3}, + }, + } + data, err := json.Marshal([]placement.GroupBundle{def}) + suite.NoError(err) + urlPrefix := cluster.GetLeaderServer().GetAddr() + err = tu.CheckPostJSON(testDialClient, urlPrefix+"/pd/api/v1/config/placement-rule", data, tu.StatusOK(suite.Require())) + suite.NoError(err) + // clean stores + // TODO: cannot sync to scheduling server? + for _, store := range leader.GetStores() { + suite.NoError(cluster.GetLeaderServer().GetRaftCluster().RemoveStore(store.GetId(), true)) + suite.NoError(cluster.GetLeaderServer().GetRaftCluster().BuryStore(store.GetId(), true)) + } + suite.NoError(cluster.GetLeaderServer().GetRaftCluster().RemoveTombStoneRecords()) + suite.Empty(leader.GetStores()) + } + suite.env.RunFuncInTwoModes(cleanFunc) +} + func (suite *regionTestSuite) TestSplitRegions() { + // use a new environment to avoid affecting other tests env := tests.NewSchedulingTestEnvironment(suite.T()) env.RunTestInTwoModes(suite.checkSplitRegions) + env.Cleanup() } func (suite *regionTestSuite) checkSplitRegions(cluster *tests.TestCluster) { @@ -81,12 +128,7 @@ func (suite *regionTestSuite) checkSplitRegions(cluster *tests.TestCluster) { } func (suite *regionTestSuite) TestAccelerateRegionsScheduleInRange() { - env := tests.NewSchedulingTestEnvironment(suite.T(), func(conf *config.Config, serverName string) { - // FIXME: enable placement rules - conf.Replication.EnablePlacementRules = false - conf.Replication.MaxReplicas = 1 - }) - env.RunTestInTwoModes(suite.checkAccelerateRegionsScheduleInRange) + suite.env.RunTestInTwoModes(suite.checkAccelerateRegionsScheduleInRange) } func (suite *regionTestSuite) checkAccelerateRegionsScheduleInRange(cluster *tests.TestCluster) { @@ -101,13 +143,13 @@ func (suite *regionTestSuite) checkAccelerateRegionsScheduleInRange(cluster *tes } tests.MustPutStore(re, cluster, s1) } - r1 := core.NewTestRegionInfo(557, 1, []byte("a1"), []byte("a2")) - r2 := core.NewTestRegionInfo(558, 2, []byte("a2"), []byte("a3")) - r3 := core.NewTestRegionInfo(559, 3, []byte("a3"), []byte("a4")) - tests.MustPutRegionInfo(re, cluster, r1) - tests.MustPutRegionInfo(re, cluster, r2) - tests.MustPutRegionInfo(re, cluster, r3) - suite.checkRegionCount(cluster, 3) + regionCount := uint64(3) + for i := uint64(1); i <= regionCount; i++ { + r1 := core.NewTestRegionInfo(550+i, 1, []byte("a"+strconv.FormatUint(i, 10)), []byte("a"+strconv.FormatUint(i+1, 10))) + r1.GetMeta().Peers = append(r1.GetMeta().Peers, &metapb.Peer{Id: 100 + i, StoreId: (i + 1) % regionCount}, &metapb.Peer{Id: 200 + i, StoreId: (i + 2) % regionCount}) + tests.MustPutRegionInfo(re, cluster, r1) + } + suite.checkRegionCount(cluster, regionCount) body := fmt.Sprintf(`{"start_key":"%s", "end_key": "%s"}`, hex.EncodeToString([]byte("a1")), hex.EncodeToString([]byte("a3"))) err := tu.CheckPostJSON(testDialClient, fmt.Sprintf("%s/regions/accelerate-schedule", urlPrefix), []byte(body), @@ -121,19 +163,14 @@ func (suite *regionTestSuite) checkAccelerateRegionsScheduleInRange(cluster *tes } func (suite *regionTestSuite) TestAccelerateRegionsScheduleInRanges() { - env := tests.NewSchedulingTestEnvironment(suite.T(), func(conf *config.Config, serverName string) { - // FIXME: enable placement rules - conf.Replication.EnablePlacementRules = false - conf.Replication.MaxReplicas = 1 - }) - env.RunTestInTwoModes(suite.checkAccelerateRegionsScheduleInRanges) + suite.env.RunTestInTwoModes(suite.checkAccelerateRegionsScheduleInRanges) } func (suite *regionTestSuite) checkAccelerateRegionsScheduleInRanges(cluster *tests.TestCluster) { leader := cluster.GetLeaderServer() urlPrefix := leader.GetAddr() + "/pd/api/v1" re := suite.Require() - for i := 1; i <= 5; i++ { + for i := 1; i <= 6; i++ { s1 := &metapb.Store{ Id: uint64(i), State: metapb.StoreState_Up, @@ -141,17 +178,13 @@ func (suite *regionTestSuite) checkAccelerateRegionsScheduleInRanges(cluster *te } tests.MustPutStore(re, cluster, s1) } - r1 := core.NewTestRegionInfo(557, 1, []byte("a1"), []byte("a2")) - r2 := core.NewTestRegionInfo(558, 2, []byte("a2"), []byte("a3")) - r3 := core.NewTestRegionInfo(559, 3, []byte("a3"), []byte("a4")) - r4 := core.NewTestRegionInfo(560, 4, []byte("a4"), []byte("a5")) - r5 := core.NewTestRegionInfo(561, 5, []byte("a5"), []byte("a6")) - tests.MustPutRegionInfo(re, cluster, r1) - tests.MustPutRegionInfo(re, cluster, r2) - tests.MustPutRegionInfo(re, cluster, r3) - tests.MustPutRegionInfo(re, cluster, r4) - tests.MustPutRegionInfo(re, cluster, r5) - suite.checkRegionCount(cluster, 5) + regionCount := uint64(6) + for i := uint64(1); i <= regionCount; i++ { + r1 := core.NewTestRegionInfo(550+i, 1, []byte("a"+strconv.FormatUint(i, 10)), []byte("a"+strconv.FormatUint(i+1, 10))) + r1.GetMeta().Peers = append(r1.GetMeta().Peers, &metapb.Peer{Id: 100 + i, StoreId: (i + 1) % regionCount}, &metapb.Peer{Id: 200 + i, StoreId: (i + 2) % regionCount}) + tests.MustPutRegionInfo(re, cluster, r1) + } + suite.checkRegionCount(cluster, regionCount) body := fmt.Sprintf(`[{"start_key":"%s", "end_key": "%s"}, {"start_key":"%s", "end_key": "%s"}]`, hex.EncodeToString([]byte("a1")), hex.EncodeToString([]byte("a3")), hex.EncodeToString([]byte("a4")), hex.EncodeToString([]byte("a6"))) @@ -166,8 +199,10 @@ func (suite *regionTestSuite) checkAccelerateRegionsScheduleInRanges(cluster *te } func (suite *regionTestSuite) TestScatterRegions() { + // use a new environment to avoid affecting other tests env := tests.NewSchedulingTestEnvironment(suite.T()) env.RunTestInTwoModes(suite.checkScatterRegions) + env.Cleanup() } func (suite *regionTestSuite) checkScatterRegions(cluster *tests.TestCluster) { @@ -182,11 +217,11 @@ func (suite *regionTestSuite) checkScatterRegions(cluster *tests.TestCluster) { } tests.MustPutStore(re, cluster, s1) } - r1 := core.NewTestRegionInfo(601, 13, []byte("b1"), []byte("b2")) + r1 := core.NewTestRegionInfo(701, 13, []byte("b1"), []byte("b2")) r1.GetMeta().Peers = append(r1.GetMeta().Peers, &metapb.Peer{Id: 5, StoreId: 14}, &metapb.Peer{Id: 6, StoreId: 15}) - r2 := core.NewTestRegionInfo(602, 13, []byte("b2"), []byte("b3")) + r2 := core.NewTestRegionInfo(702, 13, []byte("b2"), []byte("b3")) r2.GetMeta().Peers = append(r2.GetMeta().Peers, &metapb.Peer{Id: 7, StoreId: 14}, &metapb.Peer{Id: 8, StoreId: 15}) - r3 := core.NewTestRegionInfo(603, 13, []byte("b4"), []byte("b4")) + r3 := core.NewTestRegionInfo(703, 13, []byte("b4"), []byte("b4")) r3.GetMeta().Peers = append(r3.GetMeta().Peers, &metapb.Peer{Id: 9, StoreId: 14}, &metapb.Peer{Id: 10, StoreId: 15}) tests.MustPutRegionInfo(re, cluster, r1) tests.MustPutRegionInfo(re, cluster, r2) @@ -201,26 +236,24 @@ func (suite *regionTestSuite) checkScatterRegions(cluster *tests.TestCluster) { oc = sche.GetCoordinator().GetOperatorController() } - op1 := oc.GetOperator(601) - op2 := oc.GetOperator(602) - op3 := oc.GetOperator(603) + op1 := oc.GetOperator(701) + op2 := oc.GetOperator(702) + op3 := oc.GetOperator(703) // At least one operator used to scatter region suite.True(op1 != nil || op2 != nil || op3 != nil) - body = `{"regions_id": [601, 602, 603]}` + body = `{"regions_id": [701, 702, 703]}` err = tu.CheckPostJSON(testDialClient, fmt.Sprintf("%s/regions/scatter", urlPrefix), []byte(body), tu.StatusOK(re)) suite.NoError(err) } func (suite *regionTestSuite) TestCheckRegionsReplicated() { - env := tests.NewSchedulingTestEnvironment(suite.T(), - func(conf *config.Config, serverName string) { - conf.Replication.EnablePlacementRules = true - }) - env.RunTestInPDMode(suite.checkRegionsReplicated) + // Fixme: after delete+set rule, the key range will be empty, so the test will fail in api mode. + suite.env.RunTestInPDMode(suite.checkRegionsReplicated) } func (suite *regionTestSuite) checkRegionsReplicated(cluster *tests.TestCluster) { + suite.pauseRuleChecker(cluster) leader := cluster.GetLeaderServer() urlPrefix := leader.GetAddr() + "/pd/api/v1" re := suite.Require() @@ -327,14 +360,28 @@ func (suite *regionTestSuite) checkRegionsReplicated(cluster *tests.TestCluster) suite.Equal("REPLICATED", status) } -func (suite *regionTestSuite) checkRegionCount(cluster *tests.TestCluster, count int) { +func (suite *regionTestSuite) checkRegionCount(cluster *tests.TestCluster, count uint64) { leader := cluster.GetLeaderServer() tu.Eventually(suite.Require(), func() bool { - return leader.GetRaftCluster().GetRegionCount([]byte{}, []byte{}).Count == count + return leader.GetRaftCluster().GetRegionCount([]byte{}, []byte{}).Count == int(count) }) if sche := cluster.GetSchedulingPrimaryServer(); sche != nil { tu.Eventually(suite.Require(), func() bool { - return sche.GetCluster().GetRegionCount([]byte{}, []byte{}) == count + return sche.GetCluster().GetRegionCount([]byte{}, []byte{}) == int(count) }) } } + +// pauseRuleChecker will pause rule checker to avoid unexpected operator. +func (suite *regionTestSuite) pauseRuleChecker(cluster *tests.TestCluster) { + re := suite.Require() + checkerName := "rule" + addr := cluster.GetLeaderServer().GetAddr() + resp := make(map[string]interface{}) + url := fmt.Sprintf("%s/pd/api/v1/checker/%s", addr, checkerName) + err := tu.CheckPostJSON(testDialClient, url, []byte(`{"delay":1000}`), tu.StatusOK(re)) + re.NoError(err) + err = tu.ReadGetJSON(re, testDialClient, url, &resp) + re.NoError(err) + re.True(resp["paused"].(bool)) +} diff --git a/tests/server/api/rule_test.go b/tests/server/api/rule_test.go index ac52362df4e..0a0c3f2fb2e 100644 --- a/tests/server/api/rule_test.go +++ b/tests/server/api/rule_test.go @@ -35,21 +35,43 @@ import ( type ruleTestSuite struct { suite.Suite + env *tests.SchedulingTestEnvironment } func TestRuleTestSuite(t *testing.T) { suite.Run(t, new(ruleTestSuite)) } -func (suite *ruleTestSuite) TestSet() { - opts := []tests.ConfigOption{ - func(conf *config.Config, serverName string) { - conf.PDServerCfg.KeyType = "raw" - conf.Replication.EnablePlacementRules = true - }, +func (suite *ruleTestSuite) SetupSuite() { + suite.env = tests.NewSchedulingTestEnvironment(suite.T(), func(conf *config.Config, serverName string) { + conf.PDServerCfg.KeyType = "raw" + conf.Replication.EnablePlacementRules = true + }) +} + +func (suite *ruleTestSuite) TearDownSuite() { + suite.env.Cleanup() +} + +func (suite *ruleTestSuite) TearDownTest() { + cleanFunc := func(cluster *tests.TestCluster) { + def := placement.GroupBundle{ + ID: "pd", + Rules: []*placement.Rule{ + {GroupID: "pd", ID: "default", Role: "voter", Count: 3}, + }, + } + data, err := json.Marshal([]placement.GroupBundle{def}) + suite.NoError(err) + urlPrefix := cluster.GetLeaderServer().GetAddr() + err = tu.CheckPostJSON(testDialClient, urlPrefix+"/pd/api/v1/config/placement-rule", data, tu.StatusOK(suite.Require())) + suite.NoError(err) } - env := tests.NewSchedulingTestEnvironment(suite.T(), opts...) - env.RunTestInTwoModes(suite.checkSet) + suite.env.RunFuncInTwoModes(cleanFunc) +} + +func (suite *ruleTestSuite) TestSet() { + suite.env.RunTestInTwoModes(suite.checkSet) } func (suite *ruleTestSuite) checkSet(cluster *tests.TestCluster) { @@ -165,14 +187,7 @@ func (suite *ruleTestSuite) checkSet(cluster *tests.TestCluster) { } func (suite *ruleTestSuite) TestGet() { - opts := []tests.ConfigOption{ - func(conf *config.Config, serverName string) { - conf.PDServerCfg.KeyType = "raw" - conf.Replication.EnablePlacementRules = true - }, - } - env := tests.NewSchedulingTestEnvironment(suite.T(), opts...) - env.RunTestInTwoModes(suite.checkGet) + suite.env.RunTestInTwoModes(suite.checkGet) } func (suite *ruleTestSuite) checkGet(cluster *tests.TestCluster) { @@ -223,14 +238,7 @@ func (suite *ruleTestSuite) checkGet(cluster *tests.TestCluster) { } func (suite *ruleTestSuite) TestGetAll() { - opts := []tests.ConfigOption{ - func(conf *config.Config, serverName string) { - conf.PDServerCfg.KeyType = "raw" - conf.Replication.EnablePlacementRules = true - }, - } - env := tests.NewSchedulingTestEnvironment(suite.T(), opts...) - env.RunTestInTwoModes(suite.checkGetAll) + suite.env.RunTestInTwoModes(suite.checkGetAll) } func (suite *ruleTestSuite) checkGetAll(cluster *tests.TestCluster) { @@ -252,14 +260,7 @@ func (suite *ruleTestSuite) checkGetAll(cluster *tests.TestCluster) { } func (suite *ruleTestSuite) TestSetAll() { - opts := []tests.ConfigOption{ - func(conf *config.Config, serverName string) { - conf.PDServerCfg.KeyType = "raw" - conf.Replication.EnablePlacementRules = true - }, - } - env := tests.NewSchedulingTestEnvironment(suite.T(), opts...) - env.RunTestInTwoModes(suite.checkSetAll) + suite.env.RunTestInTwoModes(suite.checkSetAll) } func (suite *ruleTestSuite) checkSetAll(cluster *tests.TestCluster) { @@ -375,14 +376,7 @@ func (suite *ruleTestSuite) checkSetAll(cluster *tests.TestCluster) { } func (suite *ruleTestSuite) TestGetAllByGroup() { - opts := []tests.ConfigOption{ - func(conf *config.Config, serverName string) { - conf.PDServerCfg.KeyType = "raw" - conf.Replication.EnablePlacementRules = true - }, - } - env := tests.NewSchedulingTestEnvironment(suite.T(), opts...) - env.RunTestInTwoModes(suite.checkGetAllByGroup) + suite.env.RunTestInTwoModes(suite.checkGetAllByGroup) } func (suite *ruleTestSuite) checkGetAllByGroup(cluster *tests.TestCluster) { @@ -439,14 +433,7 @@ func (suite *ruleTestSuite) checkGetAllByGroup(cluster *tests.TestCluster) { } func (suite *ruleTestSuite) TestGetAllByRegion() { - opts := []tests.ConfigOption{ - func(conf *config.Config, serverName string) { - conf.PDServerCfg.KeyType = "raw" - conf.Replication.EnablePlacementRules = true - }, - } - env := tests.NewSchedulingTestEnvironment(suite.T(), opts...) - env.RunTestInTwoModes(suite.checkGetAllByRegion) + suite.env.RunTestInTwoModes(suite.checkGetAllByRegion) } func (suite *ruleTestSuite) checkGetAllByRegion(cluster *tests.TestCluster) { @@ -511,14 +498,8 @@ func (suite *ruleTestSuite) checkGetAllByRegion(cluster *tests.TestCluster) { } func (suite *ruleTestSuite) TestGetAllByKey() { - opts := []tests.ConfigOption{ - func(conf *config.Config, serverName string) { - conf.PDServerCfg.KeyType = "raw" - conf.Replication.EnablePlacementRules = true - }, - } - env := tests.NewSchedulingTestEnvironment(suite.T(), opts...) - env.RunTestInTwoModes(suite.checkGetAllByKey) + // Fixme: after delete+set rule, the key range will be empty, so the test will fail in api mode. + suite.env.RunTestInPDMode(suite.checkGetAllByKey) } func (suite *ruleTestSuite) checkGetAllByKey(cluster *tests.TestCluster) { @@ -577,14 +558,7 @@ func (suite *ruleTestSuite) checkGetAllByKey(cluster *tests.TestCluster) { } func (suite *ruleTestSuite) TestDelete() { - opts := []tests.ConfigOption{ - func(conf *config.Config, serverName string) { - conf.PDServerCfg.KeyType = "raw" - conf.Replication.EnablePlacementRules = true - }, - } - env := tests.NewSchedulingTestEnvironment(suite.T(), opts...) - env.RunTestInTwoModes(suite.checkDelete) + suite.env.RunTestInTwoModes(suite.checkDelete) } func (suite *ruleTestSuite) checkDelete(cluster *tests.TestCluster) { @@ -649,14 +623,7 @@ func (suite *ruleTestSuite) checkDelete(cluster *tests.TestCluster) { } func (suite *ruleTestSuite) TestBatch() { - opts := []tests.ConfigOption{ - func(conf *config.Config, serverName string) { - conf.PDServerCfg.KeyType = "raw" - conf.Replication.EnablePlacementRules = true - }, - } - env := tests.NewSchedulingTestEnvironment(suite.T(), opts...) - env.RunTestInTwoModes(suite.checkBatch) + suite.env.RunTestInTwoModes(suite.checkBatch) } func (suite *ruleTestSuite) checkBatch(cluster *tests.TestCluster) { @@ -785,14 +752,7 @@ func (suite *ruleTestSuite) checkBatch(cluster *tests.TestCluster) { } func (suite *ruleTestSuite) TestBundle() { - opts := []tests.ConfigOption{ - func(conf *config.Config, serverName string) { - conf.PDServerCfg.KeyType = "raw" - conf.Replication.EnablePlacementRules = true - }, - } - env := tests.NewSchedulingTestEnvironment(suite.T(), opts...) - env.RunTestInTwoModes(suite.checkBundle) + suite.env.RunTestInTwoModes(suite.checkBundle) } func (suite *ruleTestSuite) checkBundle(cluster *tests.TestCluster) { @@ -936,14 +896,7 @@ func (suite *ruleTestSuite) checkBundle(cluster *tests.TestCluster) { } func (suite *ruleTestSuite) TestBundleBadRequest() { - opts := []tests.ConfigOption{ - func(conf *config.Config, serverName string) { - conf.PDServerCfg.KeyType = "raw" - conf.Replication.EnablePlacementRules = true - }, - } - env := tests.NewSchedulingTestEnvironment(suite.T(), opts...) - env.RunTestInTwoModes(suite.checkBundleBadRequest) + suite.env.RunTestInTwoModes(suite.checkBundleBadRequest) } func (suite *ruleTestSuite) checkBundleBadRequest(cluster *tests.TestCluster) { @@ -997,21 +950,26 @@ func (suite *ruleTestSuite) compareRule(r1 *placement.Rule, r2 *placement.Rule) type regionRuleTestSuite struct { suite.Suite + env *tests.SchedulingTestEnvironment } func TestRegionRuleTestSuite(t *testing.T) { suite.Run(t, new(regionRuleTestSuite)) } +func (suite *regionRuleTestSuite) SetupSuite() { + suite.env = tests.NewSchedulingTestEnvironment(suite.T(), func(conf *config.Config, serverName string) { + conf.Replication.EnablePlacementRules = true + conf.Replication.MaxReplicas = 1 + }) +} + +func (suite *regionRuleTestSuite) TearDownSuite() { + suite.env.Cleanup() +} + func (suite *regionRuleTestSuite) TestRegionPlacementRule() { - opts := []tests.ConfigOption{ - func(conf *config.Config, serverName string) { - conf.Replication.EnablePlacementRules = true - conf.Replication.MaxReplicas = 1 - }, - } - env := tests.NewSchedulingTestEnvironment(suite.T(), opts...) - env.RunTestInTwoModes(suite.checkRegionPlacementRule) + suite.env.RunTestInTwoModes(suite.checkRegionPlacementRule) } func (suite *regionRuleTestSuite) checkRegionPlacementRule(cluster *tests.TestCluster) { diff --git a/tests/server/api/scheduler_test.go b/tests/server/api/scheduler_test.go index 69ee37d49e8..77dd22a006e 100644 --- a/tests/server/api/scheduler_test.go +++ b/tests/server/api/scheduler_test.go @@ -20,6 +20,7 @@ import ( "io" "net/http" "reflect" + "strings" "testing" "time" @@ -39,15 +40,25 @@ const apiPrefix = "/pd" type scheduleTestSuite struct { suite.Suite + env *tests.SchedulingTestEnvironment } func TestScheduleTestSuite(t *testing.T) { suite.Run(t, new(scheduleTestSuite)) } +func (suite *scheduleTestSuite) SetupSuite() { + suite.NoError(failpoint.Enable("github.com/tikv/pd/server/cluster/skipStoreConfigSync", `return(true)`)) + suite.env = tests.NewSchedulingTestEnvironment(suite.T()) +} + +func (suite *scheduleTestSuite) TearDownSuite() { + suite.env.Cleanup() + suite.NoError(failpoint.Disable("github.com/tikv/pd/server/cluster/skipStoreConfigSync")) +} + func (suite *scheduleTestSuite) TestOriginAPI() { - env := tests.NewSchedulingTestEnvironment(suite.T()) - env.RunTestInTwoModes(suite.checkOriginAPI) + suite.env.RunTestInTwoModes(suite.checkOriginAPI) } func (suite *scheduleTestSuite) checkOriginAPI(cluster *tests.TestCluster) { @@ -115,8 +126,7 @@ func (suite *scheduleTestSuite) checkOriginAPI(cluster *tests.TestCluster) { } func (suite *scheduleTestSuite) TestAPI() { - env := tests.NewSchedulingTestEnvironment(suite.T()) - env.RunTestInTwoModes(suite.checkAPI) + suite.env.RunTestInTwoModes(suite.checkAPI) } func (suite *scheduleTestSuite) checkAPI(cluster *tests.TestCluster) { @@ -566,10 +576,7 @@ func (suite *scheduleTestSuite) checkAPI(cluster *tests.TestCluster) { } func (suite *scheduleTestSuite) TestDisable() { - suite.NoError(failpoint.Enable("github.com/tikv/pd/server/cluster/skipStoreConfigSync", `return(true)`)) - env := tests.NewSchedulingTestEnvironment(suite.T()) - env.RunTestInTwoModes(suite.checkDisable) - suite.NoError(failpoint.Disable("github.com/tikv/pd/server/cluster/skipStoreConfigSync")) + suite.env.RunTestInTwoModes(suite.checkDisable) } func (suite *scheduleTestSuite) checkDisable(cluster *tests.TestCluster) { @@ -678,8 +685,7 @@ func (suite *scheduleTestSuite) testPauseOrResume(urlPrefix string, name, create } func (suite *scheduleTestSuite) TestEmptySchedulers() { - env := tests.NewSchedulingTestEnvironment(suite.T()) - env.RunTestInTwoModes(suite.checkEmptySchedulers) + suite.env.RunTestInTwoModes(suite.checkEmptySchedulers) } func (suite *scheduleTestSuite) checkEmptySchedulers(cluster *tests.TestCluster) { @@ -695,20 +701,30 @@ func (suite *scheduleTestSuite) checkEmptySchedulers(cluster *tests.TestCluster) } tests.MustPutStore(suite.Require(), cluster, store) } - - // test disabled and paused schedulers - suite.checkEmptySchedulersResp(urlPrefix + "?status=disabled") - suite.checkEmptySchedulersResp(urlPrefix + "?status=paused") - - // test enabled schedulers - schedulers := make([]string, 0) - suite.NoError(tu.ReadGetJSON(re, testDialClient, urlPrefix, &schedulers)) - for _, scheduler := range schedulers { - suite.deleteScheduler(urlPrefix, scheduler) + for _, query := range []string{"", "?status=paused", "?status=disabled"} { + schedulers := make([]string, 0) + suite.NoError(tu.ReadGetJSON(re, testDialClient, urlPrefix+query, &schedulers)) + for _, scheduler := range schedulers { + if strings.Contains(query, "disable") { + input := make(map[string]interface{}) + input["name"] = scheduler + body, err := json.Marshal(input) + suite.NoError(err) + suite.addScheduler(urlPrefix, body) + } else { + suite.deleteScheduler(urlPrefix, scheduler) + } + } + tu.Eventually(re, func() bool { + resp, err := apiutil.GetJSON(testDialClient, urlPrefix+query, nil) + suite.NoError(err) + defer resp.Body.Close() + suite.Equal(http.StatusOK, resp.StatusCode) + b, err := io.ReadAll(resp.Body) + suite.NoError(err) + return strings.Contains(string(b), "[]") && !strings.Contains(string(b), "null") + }) } - suite.NoError(tu.ReadGetJSON(re, testDialClient, urlPrefix, &schedulers)) - suite.Len(schedulers, 0) - suite.checkEmptySchedulersResp(urlPrefix) } func (suite *scheduleTestSuite) assertSchedulerExists(re *require.Assertions, urlPrefix string, scheduler string) { @@ -738,14 +754,3 @@ func (suite *scheduleTestSuite) isSchedulerPaused(urlPrefix, name string) bool { } return false } - -func (suite *scheduleTestSuite) checkEmptySchedulersResp(url string) { - resp, err := apiutil.GetJSON(testDialClient, url, nil) - suite.NoError(err) - defer resp.Body.Close() - suite.Equal(http.StatusOK, resp.StatusCode) - b, err := io.ReadAll(resp.Body) - suite.NoError(err) - suite.Contains(string(b), "[]") - suite.NotContains(string(b), "null") -} diff --git a/tests/server/config/config_test.go b/tests/server/config/config_test.go index eb7acb80b96..faa03c15329 100644 --- a/tests/server/config/config_test.go +++ b/tests/server/config/config_test.go @@ -86,15 +86,22 @@ func TestRateLimitConfigReload(t *testing.T) { type configTestSuite struct { suite.Suite + env *tests.SchedulingTestEnvironment } func TestConfigTestSuite(t *testing.T) { suite.Run(t, new(configTestSuite)) } +func (suite *configTestSuite) SetupSuite() { + suite.env = tests.NewSchedulingTestEnvironment(suite.T()) +} + +func (suite *configTestSuite) TearDownSuite() { + suite.env.Cleanup() +} func (suite *configTestSuite) TestConfigAll() { - env := tests.NewSchedulingTestEnvironment(suite.T()) - env.RunTestInTwoModes(suite.checkConfigAll) + suite.env.RunTestInTwoModes(suite.checkConfigAll) } func (suite *configTestSuite) checkConfigAll(cluster *tests.TestCluster) { @@ -212,8 +219,7 @@ func (suite *configTestSuite) checkConfigAll(cluster *tests.TestCluster) { } func (suite *configTestSuite) TestConfigSchedule() { - env := tests.NewSchedulingTestEnvironment(suite.T()) - env.RunTestInTwoModes(suite.checkConfigSchedule) + suite.env.RunTestInTwoModes(suite.checkConfigSchedule) } func (suite *configTestSuite) checkConfigSchedule(cluster *tests.TestCluster) { @@ -231,14 +237,15 @@ func (suite *configTestSuite) checkConfigSchedule(cluster *tests.TestCluster) { err = tu.CheckPostJSON(testDialClient, addr, postData, tu.StatusOK(re)) suite.NoError(err) - scheduleConfig1 := &sc.ScheduleConfig{} - suite.NoError(tu.ReadGetJSON(re, testDialClient, addr, scheduleConfig1)) - suite.Equal(*scheduleConfig1, *scheduleConfig) + tu.Eventually(re, func() bool { + scheduleConfig1 := &sc.ScheduleConfig{} + suite.NoError(tu.ReadGetJSON(re, testDialClient, addr, scheduleConfig1)) + return reflect.DeepEqual(*scheduleConfig1, *scheduleConfig) + }) } func (suite *configTestSuite) TestConfigReplication() { - env := tests.NewSchedulingTestEnvironment(suite.T()) - env.RunTestInTwoModes(suite.checkConfigReplication) + suite.env.RunTestInTwoModes(suite.checkConfigReplication) } func (suite *configTestSuite) checkConfigReplication(cluster *tests.TestCluster) { @@ -281,8 +288,7 @@ func (suite *configTestSuite) checkConfigReplication(cluster *tests.TestCluster) } func (suite *configTestSuite) TestConfigLabelProperty() { - env := tests.NewSchedulingTestEnvironment(suite.T()) - env.RunTestInTwoModes(suite.checkConfigLabelProperty) + suite.env.RunTestInTwoModes(suite.checkConfigLabelProperty) } func (suite *configTestSuite) checkConfigLabelProperty(cluster *tests.TestCluster) { @@ -334,8 +340,7 @@ func (suite *configTestSuite) checkConfigLabelProperty(cluster *tests.TestCluste } func (suite *configTestSuite) TestConfigDefault() { - env := tests.NewSchedulingTestEnvironment(suite.T()) - env.RunTestInTwoModes(suite.checkConfigDefault) + suite.env.RunTestInTwoModes(suite.checkConfigDefault) } func (suite *configTestSuite) checkConfigDefault(cluster *tests.TestCluster) { @@ -379,8 +384,7 @@ func (suite *configTestSuite) checkConfigDefault(cluster *tests.TestCluster) { } func (suite *configTestSuite) TestConfigPDServer() { - env := tests.NewSchedulingTestEnvironment(suite.T()) - env.RunTestInTwoModes(suite.checkConfigPDServer) + suite.env.RunTestInTwoModes(suite.checkConfigPDServer) } func (suite *configTestSuite) checkConfigPDServer(cluster *tests.TestCluster) { @@ -507,8 +511,7 @@ func createTTLUrl(url string, ttl int) string { } func (suite *configTestSuite) TestConfigTTL() { - env := tests.NewSchedulingTestEnvironment(suite.T()) - env.RunTestInTwoModes(suite.checkConfigTTL) + suite.env.RunTestInTwoModes(suite.checkConfigTTL) } func (suite *configTestSuite) checkConfigTTL(cluster *tests.TestCluster) { @@ -569,8 +572,7 @@ func (suite *configTestSuite) checkConfigTTL(cluster *tests.TestCluster) { } func (suite *configTestSuite) TestTTLConflict() { - env := tests.NewSchedulingTestEnvironment(suite.T()) - env.RunTestInTwoModes(suite.checkTTLConflict) + suite.env.RunTestInTwoModes(suite.checkTTLConflict) } func (suite *configTestSuite) checkTTLConflict(cluster *tests.TestCluster) { diff --git a/tests/testutil.go b/tests/testutil.go index 1bdd7ae10dd..0956cf1a4bd 100644 --- a/tests/testutil.go +++ b/tests/testutil.go @@ -18,6 +18,8 @@ import ( "context" "fmt" "os" + "runtime" + "strings" "sync" "testing" "time" @@ -173,12 +175,19 @@ func MustPutStore(re *require.Assertions, cluster *TestCluster, store *metapb.St }) re.NoError(err) + ts := store.GetLastHeartbeat() + if ts == 0 { + ts = time.Now().UnixNano() + } storeInfo := grpcServer.GetRaftCluster().GetStore(store.GetId()) - newStore := storeInfo.Clone(core.SetStoreStats(&pdpb.StoreStats{ - Capacity: uint64(10 * units.GiB), - UsedSize: uint64(9 * units.GiB), - Available: uint64(1 * units.GiB), - })) + newStore := storeInfo.Clone( + core.SetStoreStats(&pdpb.StoreStats{ + Capacity: uint64(10 * units.GiB), + UsedSize: uint64(9 * units.GiB), + Available: uint64(1 * units.GiB), + }), + core.SetLastHeartbeatTS(time.Unix(ts/1e9, ts%1e9)), + ) grpcServer.GetRaftCluster().GetBasicCluster().PutStore(newStore) if cluster.GetSchedulingPrimaryServer() != nil { cluster.GetSchedulingPrimaryServer().GetCluster().PutStore(newStore) @@ -239,18 +248,19 @@ const ( // SchedulingTestEnvironment is used for test purpose. type SchedulingTestEnvironment struct { - t *testing.T - ctx context.Context - cancel context.CancelFunc - cluster *TestCluster - opts []ConfigOption + t *testing.T + opts []ConfigOption + clusters map[mode]*TestCluster + cancels []context.CancelFunc } // NewSchedulingTestEnvironment is to create a new SchedulingTestEnvironment. func NewSchedulingTestEnvironment(t *testing.T, opts ...ConfigOption) *SchedulingTestEnvironment { return &SchedulingTestEnvironment{ - t: t, - opts: opts, + t: t, + opts: opts, + clusters: make(map[mode]*TestCluster), + cancels: make([]context.CancelFunc, 0), } } @@ -262,62 +272,95 @@ func (s *SchedulingTestEnvironment) RunTestInTwoModes(test func(*TestCluster)) { // RunTestInPDMode is to run test in pd mode. func (s *SchedulingTestEnvironment) RunTestInPDMode(test func(*TestCluster)) { - s.t.Log("start to run test in pd mode") - s.startCluster(pdMode) - test(s.cluster) - s.cleanup() - s.t.Log("finish to run test in pd mode") + s.t.Logf("start test %s in pd mode", s.getTestName()) + if _, ok := s.clusters[pdMode]; !ok { + s.startCluster(pdMode) + } + test(s.clusters[pdMode]) +} + +func (s *SchedulingTestEnvironment) getTestName() string { + pc, _, _, _ := runtime.Caller(2) + caller := runtime.FuncForPC(pc) + if caller == nil || strings.Contains(caller.Name(), "RunTestInTwoModes") { + pc, _, _, _ = runtime.Caller(3) + caller = runtime.FuncForPC(pc) + } + if caller != nil { + elements := strings.Split(caller.Name(), ".") + return elements[len(elements)-1] + } + return "" } // RunTestInAPIMode is to run test in api mode. func (s *SchedulingTestEnvironment) RunTestInAPIMode(test func(*TestCluster)) { - s.t.Log("start to run test in api mode") re := require.New(s.t) re.NoError(failpoint.Enable("github.com/tikv/pd/server/cluster/highFrequencyClusterJobs", `return(true)`)) re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/mcs/scheduling/server/fastUpdateMember", `return(true)`)) - s.startCluster(apiMode) - test(s.cluster) - s.cleanup() - re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/mcs/scheduling/server/fastUpdateMember")) - re.NoError(failpoint.Disable("github.com/tikv/pd/server/cluster/highFrequencyClusterJobs")) - s.t.Log("finish to run test in api mode") + defer func() { + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/mcs/scheduling/server/fastUpdateMember")) + re.NoError(failpoint.Disable("github.com/tikv/pd/server/cluster/highFrequencyClusterJobs")) + }() + s.t.Logf("start test %s in api mode", s.getTestName()) + if _, ok := s.clusters[apiMode]; !ok { + s.startCluster(apiMode) + } + test(s.clusters[apiMode]) } -func (s *SchedulingTestEnvironment) cleanup() { - s.cluster.Destroy() - s.cancel() +// RunFuncInTwoModes is to run func in two modes. +func (s *SchedulingTestEnvironment) RunFuncInTwoModes(f func(*TestCluster)) { + if c, ok := s.clusters[pdMode]; ok { + f(c) + } + if c, ok := s.clusters[apiMode]; ok { + f(c) + } +} + +// Cleanup is to cleanup the environment. +func (s *SchedulingTestEnvironment) Cleanup() { + for _, cluster := range s.clusters { + cluster.Destroy() + } + for _, cancel := range s.cancels { + cancel() + } } func (s *SchedulingTestEnvironment) startCluster(m mode) { - var err error re := require.New(s.t) - s.ctx, s.cancel = context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(context.Background()) + s.cancels = append(s.cancels, cancel) switch m { case pdMode: - s.cluster, err = NewTestCluster(s.ctx, 1, s.opts...) + cluster, err := NewTestCluster(ctx, 1, s.opts...) re.NoError(err) - err = s.cluster.RunInitialServers() + err = cluster.RunInitialServers() re.NoError(err) - re.NotEmpty(s.cluster.WaitLeader()) - leaderServer := s.cluster.GetServer(s.cluster.GetLeader()) + re.NotEmpty(cluster.WaitLeader()) + leaderServer := cluster.GetServer(cluster.GetLeader()) re.NoError(leaderServer.BootstrapCluster()) + s.clusters[pdMode] = cluster case apiMode: - s.cluster, err = NewTestAPICluster(s.ctx, 1, s.opts...) + cluster, err := NewTestAPICluster(ctx, 1, s.opts...) re.NoError(err) - err = s.cluster.RunInitialServers() + err = cluster.RunInitialServers() re.NoError(err) - re.NotEmpty(s.cluster.WaitLeader()) - leaderServer := s.cluster.GetServer(s.cluster.GetLeader()) + re.NotEmpty(cluster.WaitLeader()) + leaderServer := cluster.GetServer(cluster.GetLeader()) re.NoError(leaderServer.BootstrapCluster()) leaderServer.GetRaftCluster().SetPrepared() // start scheduling cluster - tc, err := NewTestSchedulingCluster(s.ctx, 1, leaderServer.GetAddr()) + tc, err := NewTestSchedulingCluster(ctx, 1, leaderServer.GetAddr()) re.NoError(err) tc.WaitForPrimaryServing(re) - s.cluster.SetSchedulingCluster(tc) + cluster.SetSchedulingCluster(tc) time.Sleep(200 * time.Millisecond) // wait for scheduling cluster to update member testutil.Eventually(re, func() bool { - return s.cluster.GetLeaderServer().GetServer().GetRaftCluster().IsServiceIndependent(utils.SchedulingServiceName) + return cluster.GetLeaderServer().GetServer().GetRaftCluster().IsServiceIndependent(utils.SchedulingServiceName) }) + s.clusters[apiMode] = cluster } } From ba3776c177ce9de2dc029b3f49ba22933b4f1f21 Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Fri, 8 Dec 2023 18:19:46 +0800 Subject: [PATCH 081/137] mcs: add IsServiceIndependent check to fix redirect (#7513) close tikv/pd#7512 Signed-off-by: lhy1024 --- pkg/mcs/scheduling/server/config/config.go | 2 +- pkg/utils/apiutil/serverapi/middleware.go | 11 +++++- server/config/persist_options.go | 2 +- tests/integrations/mcs/tso/api_test.go | 44 +++++++++++++++++++++ tests/server/api/scheduler_test.go | 45 +++++++++++++--------- 5 files changed, 82 insertions(+), 22 deletions(-) diff --git a/pkg/mcs/scheduling/server/config/config.go b/pkg/mcs/scheduling/server/config/config.go index 6c38f66dc68..3e347afc12e 100644 --- a/pkg/mcs/scheduling/server/config/config.go +++ b/pkg/mcs/scheduling/server/config/config.go @@ -612,7 +612,7 @@ func (o *PersistConfig) IsLocationReplacementEnabled() bool { return o.getTTLBoolOr(sc.EnableLocationReplacement, o.GetScheduleConfig().EnableLocationReplacement) } -// IsTikvRegionSplitEnabled returns whether tikv split region is disabled. +// IsTikvRegionSplitEnabled returns whether tikv split region is enabled. func (o *PersistConfig) IsTikvRegionSplitEnabled() bool { return o.getTTLBoolOr(sc.EnableTiKVSplitRegion, o.GetScheduleConfig().EnableTiKVSplitRegion) } diff --git a/pkg/utils/apiutil/serverapi/middleware.go b/pkg/utils/apiutil/serverapi/middleware.go index 26b36a88ca8..eb0f8a5f8eb 100644 --- a/pkg/utils/apiutil/serverapi/middleware.go +++ b/pkg/utils/apiutil/serverapi/middleware.go @@ -22,6 +22,7 @@ import ( "github.com/pingcap/failpoint" "github.com/pingcap/log" "github.com/tikv/pd/pkg/errs" + mcsutils "github.com/tikv/pd/pkg/mcs/utils" "github.com/tikv/pd/pkg/slice" "github.com/tikv/pd/pkg/utils/apiutil" "github.com/tikv/pd/server" @@ -122,16 +123,24 @@ func (h *redirector) matchMicroServiceRedirectRules(r *http.Request) (bool, stri // It will be helpful when matching the redirect rules "schedulers" or "schedulers/{name}" r.URL.Path = strings.TrimRight(r.URL.Path, "/") for _, rule := range h.microserviceRedirectRules { + // Now we only support checking the scheduling service whether it is independent + if rule.targetServiceName == mcsutils.SchedulingServiceName { + if !h.s.IsServiceIndependent(mcsutils.SchedulingServiceName) { + continue + } + } if strings.HasPrefix(r.URL.Path, rule.matchPath) && slice.Contains(rule.matchMethods, r.Method) { if rule.filter != nil && !rule.filter(r) { continue } - // we check the service primary addr here, so no need to check independently again. + // we check the service primary addr here, + // if the service is not available, we will return ErrRedirect by returning an empty addr. addr, ok := h.s.GetServicePrimaryAddr(r.Context(), rule.targetServiceName) if !ok || addr == "" { log.Warn("failed to get the service primary addr when trying to match redirect rules", zap.String("path", r.URL.Path)) + return true, "" } // If the URL contains escaped characters, use RawPath instead of Path origin := r.URL.Path diff --git a/server/config/persist_options.go b/server/config/persist_options.go index ae9047c626b..0fa1804b879 100644 --- a/server/config/persist_options.go +++ b/server/config/persist_options.go @@ -608,7 +608,7 @@ func (o *PersistOptions) IsLocationReplacementEnabled() bool { return o.getTTLBoolOr(sc.EnableLocationReplacement, o.GetScheduleConfig().EnableLocationReplacement) } -// IsTikvRegionSplitEnabled returns whether tikv split region is disabled. +// IsTikvRegionSplitEnabled returns whether tikv split region is enabled. func (o *PersistOptions) IsTikvRegionSplitEnabled() bool { return o.getTTLBoolOr(sc.EnableTiKVSplitRegion, o.GetScheduleConfig().EnableTiKVSplitRegion) } diff --git a/tests/integrations/mcs/tso/api_test.go b/tests/integrations/mcs/tso/api_test.go index d08b8df32b9..94b38dd93cb 100644 --- a/tests/integrations/mcs/tso/api_test.go +++ b/tests/integrations/mcs/tso/api_test.go @@ -18,6 +18,7 @@ import ( "bytes" "context" "encoding/json" + "fmt" "io" "net/http" "testing" @@ -200,3 +201,46 @@ func TestTSOServerStartFirst(t *testing.T) { re.NoError(failpoint.Disable("github.com/tikv/pd/server/delayStartServerLoop")) } + +func TestForwardOnlyTSONoScheduling(t *testing.T) { + re := require.New(t) + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/utils/apiutil/serverapi/checkHeader", "return(true)")) + defer func() { + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/utils/apiutil/serverapi/checkHeader")) + }() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + tc, err := tests.NewTestAPICluster(ctx, 1) + defer tc.Destroy() + re.NoError(err) + err = tc.RunInitialServers() + re.NoError(err) + pdAddr := tc.GetConfig().GetClientURL() + ttc, err := tests.NewTestTSOCluster(ctx, 2, pdAddr) + re.NoError(err) + tc.WaitLeader() + leaderServer := tc.GetLeaderServer() + re.NoError(leaderServer.BootstrapCluster()) + + urlPrefix := fmt.Sprintf("%s/pd/api/v1", pdAddr) + + // Test /operators, it should not forward when there is no scheduling server. + var slice []string + err = testutil.ReadGetJSON(re, dialClient, fmt.Sprintf("%s/%s", urlPrefix, "operators"), &slice, + testutil.WithoutHeader(re, apiutil.ForwardToMicroServiceHeader)) + re.NoError(err) + re.Len(slice, 0) + + // Test admin/reset-ts, it should forward to tso server. + input := []byte(`{"tso":"121312", "force-use-larger":true}`) + err = testutil.CheckPostJSON(dialClient, fmt.Sprintf("%s/%s", urlPrefix, "admin/reset-ts"), input, + testutil.StatusOK(re), testutil.StringContain(re, "Reset ts successfully"), testutil.WithHeader(re, apiutil.ForwardToMicroServiceHeader, "true")) + re.NoError(err) + + // If close tso server, it should try forward to tso server, but return error in api mode. + ttc.Destroy() + err = testutil.CheckPostJSON(dialClient, fmt.Sprintf("%s/%s", urlPrefix, "admin/reset-ts"), input, + testutil.Status(re, http.StatusInternalServerError), testutil.StringContain(re, "[PD:apiutil:ErrRedirect]redirect failed")) + re.NoError(err) +} diff --git a/tests/server/api/scheduler_test.go b/tests/server/api/scheduler_test.go index 77dd22a006e..b3810da154a 100644 --- a/tests/server/api/scheduler_test.go +++ b/tests/server/api/scheduler_test.go @@ -26,7 +26,6 @@ import ( "github.com/pingcap/failpoint" "github.com/pingcap/kvproto/pkg/metapb" - "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" sc "github.com/tikv/pd/pkg/schedule/config" "github.com/tikv/pd/pkg/slice" @@ -82,7 +81,7 @@ func (suite *scheduleTestSuite) checkOriginAPI(cluster *tests.TestCluster) { re := suite.Require() suite.NoError(tu.CheckPostJSON(testDialClient, urlPrefix, body, tu.StatusOK(re))) - suite.assertSchedulerExists(re, urlPrefix, "evict-leader-scheduler") + suite.assertSchedulerExists(urlPrefix, "evict-leader-scheduler") resp := make(map[string]interface{}) listURL := fmt.Sprintf("%s%s%s/%s/list", leaderAddr, apiPrefix, server.SchedulerConfigHandlerPath, "evict-leader-scheduler") suite.NoError(tu.ReadGetJSON(re, testDialClient, listURL, &resp)) @@ -94,20 +93,20 @@ func (suite *scheduleTestSuite) checkOriginAPI(cluster *tests.TestCluster) { suite.NoError(err) suite.NoError(failpoint.Enable("github.com/tikv/pd/pkg/schedule/schedulers/persistFail", "return(true)")) suite.NoError(tu.CheckPostJSON(testDialClient, urlPrefix, body, tu.StatusNotOK(re))) - suite.assertSchedulerExists(re, urlPrefix, "evict-leader-scheduler") + suite.assertSchedulerExists(urlPrefix, "evict-leader-scheduler") resp = make(map[string]interface{}) suite.NoError(tu.ReadGetJSON(re, testDialClient, listURL, &resp)) suite.Len(resp["store-id-ranges"], 1) suite.NoError(failpoint.Disable("github.com/tikv/pd/pkg/schedule/schedulers/persistFail")) suite.NoError(tu.CheckPostJSON(testDialClient, urlPrefix, body, tu.StatusOK(re))) - suite.assertSchedulerExists(re, urlPrefix, "evict-leader-scheduler") + suite.assertSchedulerExists(urlPrefix, "evict-leader-scheduler") resp = make(map[string]interface{}) suite.NoError(tu.ReadGetJSON(re, testDialClient, listURL, &resp)) suite.Len(resp["store-id-ranges"], 2) deleteURL := fmt.Sprintf("%s/%s", urlPrefix, "evict-leader-scheduler-1") err = tu.CheckDelete(testDialClient, deleteURL, tu.StatusOK(re)) suite.NoError(err) - suite.assertSchedulerExists(re, urlPrefix, "evict-leader-scheduler") + suite.assertSchedulerExists(urlPrefix, "evict-leader-scheduler") resp1 := make(map[string]interface{}) suite.NoError(tu.ReadGetJSON(re, testDialClient, listURL, &resp1)) suite.Len(resp1["store-id-ranges"], 1) @@ -115,11 +114,11 @@ func (suite *scheduleTestSuite) checkOriginAPI(cluster *tests.TestCluster) { suite.NoError(failpoint.Enable("github.com/tikv/pd/server/config/persistFail", "return(true)")) err = tu.CheckDelete(testDialClient, deleteURL, tu.Status(re, http.StatusInternalServerError)) suite.NoError(err) - suite.assertSchedulerExists(re, urlPrefix, "evict-leader-scheduler") + suite.assertSchedulerExists(urlPrefix, "evict-leader-scheduler") suite.NoError(failpoint.Disable("github.com/tikv/pd/server/config/persistFail")) err = tu.CheckDelete(testDialClient, deleteURL, tu.StatusOK(re)) suite.NoError(err) - suite.assertNoScheduler(re, urlPrefix, "evict-leader-scheduler") + suite.assertNoScheduler(urlPrefix, "evict-leader-scheduler") suite.NoError(tu.CheckGetJSON(testDialClient, listURL, nil, tu.Status(re, http.StatusNotFound))) err = tu.CheckDelete(testDialClient, deleteURL, tu.Status(re, http.StatusNotFound)) suite.NoError(err) @@ -492,7 +491,7 @@ func (suite *scheduleTestSuite) checkAPI(cluster *tests.TestCluster) { testCase.extraTestFunc(testCase.createdName) } suite.deleteScheduler(urlPrefix, testCase.createdName) - suite.assertNoScheduler(re, urlPrefix, testCase.createdName) + suite.assertNoScheduler(urlPrefix, testCase.createdName) } // test pause and resume all schedulers. @@ -507,7 +506,7 @@ func (suite *scheduleTestSuite) checkAPI(cluster *tests.TestCluster) { body, err := json.Marshal(input) suite.NoError(err) suite.addScheduler(urlPrefix, body) - suite.assertSchedulerExists(re, urlPrefix, testCase.createdName) // wait for scheduler to be synced. + suite.assertSchedulerExists(urlPrefix, testCase.createdName) // wait for scheduler to be synced. if testCase.extraTestFunc != nil { testCase.extraTestFunc(testCase.createdName) } @@ -571,7 +570,7 @@ func (suite *scheduleTestSuite) checkAPI(cluster *tests.TestCluster) { createdName = testCase.name } suite.deleteScheduler(urlPrefix, createdName) - suite.assertNoScheduler(re, urlPrefix, createdName) + suite.assertNoScheduler(urlPrefix, createdName) } } @@ -612,8 +611,8 @@ func (suite *scheduleTestSuite) checkDisable(cluster *tests.TestCluster) { err = tu.CheckPostJSON(testDialClient, u, body, tu.StatusOK(re)) suite.NoError(err) - suite.assertNoScheduler(re, urlPrefix, name) - suite.assertSchedulerExists(re, fmt.Sprintf("%s?status=disabled", urlPrefix), name) + suite.assertNoScheduler(urlPrefix, name) + suite.assertSchedulerExists(fmt.Sprintf("%s?status=disabled", urlPrefix), name) // reset schedule config scheduleConfig.Schedulers = originSchedulers @@ -623,7 +622,7 @@ func (suite *scheduleTestSuite) checkDisable(cluster *tests.TestCluster) { suite.NoError(err) suite.deleteScheduler(urlPrefix, name) - suite.assertNoScheduler(re, urlPrefix, name) + suite.assertNoScheduler(urlPrefix, name) } func (suite *scheduleTestSuite) addScheduler(urlPrefix string, body []byte) { @@ -648,7 +647,7 @@ func (suite *scheduleTestSuite) testPauseOrResume(urlPrefix string, name, create err := tu.CheckPostJSON(testDialClient, urlPrefix, body, tu.StatusOK(re)) re.NoError(err) } - suite.assertSchedulerExists(re, urlPrefix, createdName) // wait for scheduler to be synced. + suite.assertSchedulerExists(urlPrefix, createdName) // wait for scheduler to be synced. // test pause. input := make(map[string]interface{}) @@ -727,25 +726,33 @@ func (suite *scheduleTestSuite) checkEmptySchedulers(cluster *tests.TestCluster) } } -func (suite *scheduleTestSuite) assertSchedulerExists(re *require.Assertions, urlPrefix string, scheduler string) { +func (suite *scheduleTestSuite) assertSchedulerExists(urlPrefix string, scheduler string) { var schedulers []string + re := suite.Require() tu.Eventually(re, func() bool { - tu.ReadGetJSON(suite.Require(), testDialClient, urlPrefix, &schedulers) + err := tu.ReadGetJSON(re, testDialClient, urlPrefix, &schedulers, + tu.StatusOK(re)) + suite.NoError(err) return slice.Contains(schedulers, scheduler) }) } -func (suite *scheduleTestSuite) assertNoScheduler(re *require.Assertions, urlPrefix string, scheduler string) { +func (suite *scheduleTestSuite) assertNoScheduler(urlPrefix string, scheduler string) { var schedulers []string + re := suite.Require() tu.Eventually(re, func() bool { - tu.ReadGetJSON(suite.Require(), testDialClient, urlPrefix, &schedulers) + err := tu.ReadGetJSON(re, testDialClient, urlPrefix, &schedulers, + tu.StatusOK(re)) + suite.NoError(err) return !slice.Contains(schedulers, scheduler) }) } func (suite *scheduleTestSuite) isSchedulerPaused(urlPrefix, name string) bool { var schedulers []string - err := tu.ReadGetJSON(suite.Require(), testDialClient, fmt.Sprintf("%s?status=paused", urlPrefix), &schedulers) + re := suite.Require() + err := tu.ReadGetJSON(re, testDialClient, fmt.Sprintf("%s?status=paused", urlPrefix), &schedulers, + tu.StatusOK(re)) suite.NoError(err) for _, scheduler := range schedulers { if scheduler == name { From 00674d02ad93280bc1f8b12e545f59f0a7bd6570 Mon Sep 17 00:00:00 2001 From: Yongbo Jiang Date: Fri, 8 Dec 2023 18:31:46 +0800 Subject: [PATCH 082/137] pkg/window: fix panic in test (#7511) close tikv/pd#7386 Signed-off-by: Cabinfever_B Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- pkg/window/policy_test.go | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/pkg/window/policy_test.go b/pkg/window/policy_test.go index 14b3b326192..489c8428c9a 100644 --- a/pkg/window/policy_test.go +++ b/pkg/window/policy_test.go @@ -26,9 +26,12 @@ import ( "github.com/stretchr/testify/require" ) -func GetRollingPolicy() *RollingPolicy { - w := NewWindow(Options{Size: 3}) - return NewRollingPolicy(w, RollingPolicyOpts{BucketDuration: 100 * time.Millisecond}) +const defaultBucketDuration = 100 * time.Millisecond +const defaultSize = 3 + +func getRollingPolicy() *RollingPolicy { + w := NewWindow(Options{Size: defaultSize}) + return NewRollingPolicy(w, RollingPolicyOpts{BucketDuration: defaultBucketDuration}) } func TestRollingPolicy_Add(t *testing.T) { @@ -45,6 +48,7 @@ func TestRollingPolicy_Add(t *testing.T) { points: []float64{1, 1}, }, { + // In CI, the actual sleep time may be more than 100 (timeSleep = 94). timeSleep: []int{94, 250}, offset: []int{0, 0}, points: []float64{1, 1}, @@ -60,14 +64,25 @@ func TestRollingPolicy_Add(t *testing.T) { t.Run("test policy add", func(t *testing.T) { var totalTS, lastOffset int timeSleep := test.timeSleep - policy := GetRollingPolicy() + beginTime := time.Now() + policy := getRollingPolicy() + points := make([]float64, defaultSize) + asExpected := true for i, n := range timeSleep { totalTS += n time.Sleep(time.Duration(n) * time.Millisecond) - offset, point := test.offset[i], test.points[i] + point := test.points[i] + offset := int(time.Since(beginTime)/defaultBucketDuration) % defaultSize + points[i] += point policy.Add(point) - - re.Less(math.Abs(point-policy.window.buckets[offset].Points[0]), 1e-6, + if offset != test.offset[i] { + asExpected = false + } + if asExpected { + re.Less(math.Abs(point-policy.window.buckets[offset].Points[0]), 1e-6, + fmt.Sprintf("error, time since last append: %vms, last offset: %v", totalTS, lastOffset)) + } + re.Less(math.Abs(points[i]-policy.window.buckets[offset].Points[0]), 1e-6, fmt.Sprintf("error, time since last append: %vms, last offset: %v", totalTS, lastOffset)) lastOffset = offset } @@ -78,7 +93,7 @@ func TestRollingPolicy_Add(t *testing.T) { func TestRollingPolicy_AddWithTimespan(t *testing.T) { re := require.New(t) t.Run("timespan < bucket number", func(t *testing.T) { - policy := GetRollingPolicy() + policy := getRollingPolicy() // bucket 0 policy.Add(0) // bucket 1 @@ -102,7 +117,7 @@ func TestRollingPolicy_AddWithTimespan(t *testing.T) { }) t.Run("timespan > bucket number", func(t *testing.T) { - policy := GetRollingPolicy() + policy := getRollingPolicy() // bucket 0 policy.Add(0) From bd155f6d968bc1c704b709b20355c7bfccaf8c62 Mon Sep 17 00:00:00 2001 From: ShuNing Date: Mon, 11 Dec 2023 15:43:50 +0800 Subject: [PATCH 083/137] ci: fix docker build sqlite3 and alpine 3.19 incompatibility (#7522) close tikv/pd#7521 ci: fix docker build with musl compatibility issue Signed-off-by: Shuning Chen Co-authored-by: Shuning Chen --- Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 550b1c1bb72..d3f0025023b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,7 +25,9 @@ RUN GO111MODULE=on go mod download COPY . . -RUN make +# Workaround sqlite3 and alpine 3.19 incompatibility +# https://github.com/mattn/go-sqlite3/issues/1164 +RUN CGO_CFLAGS="-D_LARGEFILE64_SOURCE" make FROM alpine:3.17 From ab68192a52fe9e9aeebf981af250e41f28b96ef5 Mon Sep 17 00:00:00 2001 From: ShuNing Date: Mon, 11 Dec 2023 15:57:17 +0800 Subject: [PATCH 084/137] resource_control: improve trace logs, ctl and metrics (#7510) close tikv/pd#7509 resource_control: improve trace logs, ctl and metrics - add the ctl command to config resource cotnrol - add metrics and a switch to enable trace log Signed-off-by: nolouch Signed-off-by: Shuning Chen Co-authored-by: Shuning Chen Co-authored-by: Yongbo Jiang --- client/go.mod | 2 +- client/resource_group/controller/config.go | 4 + .../resource_group/controller/controller.go | 51 +++++--- client/resource_group/controller/limiter.go | 31 +++-- .../resource_group/controller/limiter_test.go | 2 +- client/resource_group/controller/metrics.go | 9 ++ pkg/mcs/resourcemanager/server/config.go | 3 + .../resource_manager_command_test.go | 97 +++++++++++++++ .../pdctl/command/resource_manager_command.go | 112 ++++++++++++++++++ tools/pd-ctl/pdctl/ctl.go | 1 + 10 files changed, 282 insertions(+), 30 deletions(-) create mode 100644 tests/pdctl/resourcemanager/resource_manager_command_test.go create mode 100644 tools/pd-ctl/pdctl/command/resource_manager_command.go diff --git a/client/go.mod b/client/go.mod index 948a5c22b14..54be0c96765 100644 --- a/client/go.mod +++ b/client/go.mod @@ -13,6 +13,7 @@ require ( github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 github.com/prometheus/client_golang v1.11.1 github.com/stretchr/testify v1.8.2 + go.uber.org/atomic v1.10.0 go.uber.org/goleak v1.1.11 go.uber.org/zap v1.24.0 golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4 @@ -31,7 +32,6 @@ require ( github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/common v0.26.0 // indirect github.com/prometheus/procfs v0.6.0 // indirect - go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/net v0.17.0 // indirect golang.org/x/sys v0.13.0 // indirect diff --git a/client/resource_group/controller/config.go b/client/resource_group/controller/config.go index 16a2525cd0d..ffc360c385c 100644 --- a/client/resource_group/controller/config.go +++ b/client/resource_group/controller/config.go @@ -88,6 +88,9 @@ type Config struct { // RequestUnit is the configuration determines the coefficients of the RRU and WRU cost. // This configuration should be modified carefully. RequestUnit RequestUnitConfig `toml:"request-unit" json:"request-unit"` + + // EnableControllerTraceLog is to control whether resource control client enable trace. + EnableControllerTraceLog bool `toml:"enable-controller-trace-log" json:"enable-controller-trace-log,string"` } // DefaultConfig returns the default resource manager controller configuration. @@ -96,6 +99,7 @@ func DefaultConfig() *Config { DegradedModeWaitDuration: NewDuration(defaultDegradedModeWaitDuration), LTBMaxWaitDuration: NewDuration(defaultMaxWaitDuration), RequestUnit: DefaultRequestUnitConfig(), + EnableControllerTraceLog: false, } } diff --git a/client/resource_group/controller/controller.go b/client/resource_group/controller/controller.go index 56d9ef9bd1b..f9a8caaabed 100755 --- a/client/resource_group/controller/controller.go +++ b/client/resource_group/controller/controller.go @@ -32,6 +32,7 @@ import ( "github.com/prometheus/client_golang/prometheus" pd "github.com/tikv/pd/client" "github.com/tikv/pd/client/errs" + atomicutil "go.uber.org/atomic" "go.uber.org/zap" "golang.org/x/exp/slices" ) @@ -54,6 +55,14 @@ const ( lowToken selectType = 1 ) +var enableControllerTraceLog = atomicutil.NewBool(false) + +func logControllerTrace(msg string, fields ...zap.Field) { + if enableControllerTraceLog.Load() { + log.Info(msg, fields...) + } +} + // ResourceGroupKVInterceptor is used as quota limit controller for resource group using kv store. type ResourceGroupKVInterceptor interface { // OnRequestWait is used to check whether resource group has enough tokens. It maybe needs to wait some time. @@ -369,6 +378,9 @@ func (c *ResourceGroupsController) Start(ctx context.Context) { } copyCfg := *c.ruConfig c.safeRuConfig.Store(©Cfg) + if enableControllerTraceLog.Load() != config.EnableControllerTraceLog { + enableControllerTraceLog.Store(config.EnableControllerTraceLog) + } log.Info("load resource controller config after config changed", zap.Reflect("config", config), zap.Reflect("ruConfig", c.ruConfig)) } @@ -505,7 +517,7 @@ func (c *ResourceGroupsController) sendTokenBucketRequests(ctx context.Context, c.responseDeadlineCh = c.run.responseDeadline.C } go func() { - log.Debug("[resource group controller] send token bucket request", zap.Time("now", now), zap.Any("req", req.Requests), zap.String("source", source)) + logControllerTrace("[resource group controller] send token bucket request", zap.Time("now", now), zap.Any("req", req.Requests), zap.String("source", source)) resp, err := c.provider.AcquireTokenBuckets(ctx, req) latency := time.Since(now) if err != nil { @@ -518,7 +530,7 @@ func (c *ResourceGroupsController) sendTokenBucketRequests(ctx context.Context, } else { successfulTokenRequestDuration.Observe(latency.Seconds()) } - log.Debug("[resource group controller] token bucket response", zap.Time("now", time.Now()), zap.Any("resp", resp), zap.String("source", source), zap.Duration("latency", latency)) + logControllerTrace("[resource group controller] token bucket response", zap.Time("now", time.Now()), zap.Any("resp", resp), zap.String("source", source), zap.Duration("latency", latency)) c.tokenResponseChan <- resp }() } @@ -603,10 +615,11 @@ type groupCostController struct { calculators []ResourceCalculator handleRespFunc func(*rmpb.TokenBucketResponse) - successfulRequestDuration prometheus.Observer - requestRetryCounter prometheus.Counter - failedRequestCounter prometheus.Counter - tokenRequestCounter prometheus.Counter + successfulRequestDuration prometheus.Observer + failedLimitReserveDuration prometheus.Observer + requestRetryCounter prometheus.Counter + failedRequestCounter prometheus.Counter + tokenRequestCounter prometheus.Counter mu struct { sync.Mutex @@ -696,14 +709,15 @@ func newGroupCostController( return nil, errs.ErrClientResourceGroupConfigUnavailable.FastGenByArgs("not supports the resource type") } gc := &groupCostController{ - meta: group, - name: group.Name, - mainCfg: mainCfg, - mode: group.GetMode(), - successfulRequestDuration: successfulRequestDuration.WithLabelValues(group.Name), - failedRequestCounter: failedRequestCounter.WithLabelValues(group.Name), - requestRetryCounter: requestRetryCounter.WithLabelValues(group.Name), - tokenRequestCounter: resourceGroupTokenRequestCounter.WithLabelValues(group.Name), + meta: group, + name: group.Name, + mainCfg: mainCfg, + mode: group.GetMode(), + successfulRequestDuration: successfulRequestDuration.WithLabelValues(group.Name), + failedLimitReserveDuration: failedLimitReserveDuration.WithLabelValues(group.Name), + failedRequestCounter: failedRequestCounter.WithLabelValues(group.Name), + requestRetryCounter: requestRetryCounter.WithLabelValues(group.Name), + tokenRequestCounter: resourceGroupTokenRequestCounter.WithLabelValues(group.Name), calculators: []ResourceCalculator{ newKVCalculator(mainCfg), newSQLCalculator(mainCfg), @@ -805,7 +819,7 @@ func (gc *groupCostController) updateRunState() { } *gc.run.consumption = *gc.mu.consumption gc.mu.Unlock() - log.Debug("[resource group controller] update run state", zap.Any("request-unit-consumption", gc.run.consumption)) + logControllerTrace("[resource group controller] update run state", zap.Any("request-unit-consumption", gc.run.consumption)) gc.run.now = newTime } @@ -886,7 +900,7 @@ func (gc *groupCostController) updateAvgRaWResourcePerSec() { if !gc.calcAvg(counter, getRawResourceValueFromConsumption(gc.run.consumption, typ)) { continue } - log.Debug("[resource group controller] update avg raw resource per sec", zap.String("name", gc.name), zap.String("type", rmpb.RawResourceType_name[int32(typ)]), zap.Float64("avg-ru-per-sec", counter.avgRUPerSec)) + logControllerTrace("[resource group controller] update avg raw resource per sec", zap.String("name", gc.name), zap.String("type", rmpb.RawResourceType_name[int32(typ)]), zap.Float64("avg-ru-per-sec", counter.avgRUPerSec)) } gc.burstable.Store(isBurstable) } @@ -900,7 +914,7 @@ func (gc *groupCostController) updateAvgRUPerSec() { if !gc.calcAvg(counter, getRUValueFromConsumption(gc.run.consumption, typ)) { continue } - log.Debug("[resource group controller] update avg ru per sec", zap.String("name", gc.name), zap.String("type", rmpb.RequestUnitType_name[int32(typ)]), zap.Float64("avg-ru-per-sec", counter.avgRUPerSec)) + logControllerTrace("[resource group controller] update avg ru per sec", zap.String("name", gc.name), zap.String("type", rmpb.RequestUnitType_name[int32(typ)]), zap.Float64("avg-ru-per-sec", counter.avgRUPerSec)) } gc.burstable.Store(isBurstable) } @@ -1222,6 +1236,9 @@ func (gc *groupCostController) onRequestWait( } if err != nil { gc.failedRequestCounter.Inc() + if d.Seconds() > 0 { + gc.failedLimitReserveDuration.Observe(d.Seconds()) + } gc.mu.Lock() sub(gc.mu.consumption, delta) gc.mu.Unlock() diff --git a/client/resource_group/controller/limiter.go b/client/resource_group/controller/limiter.go index f89ab17514c..63c94a9782b 100644 --- a/client/resource_group/controller/limiter.go +++ b/client/resource_group/controller/limiter.go @@ -122,10 +122,11 @@ func NewLimiterWithCfg(now time.Time, cfg tokenBucketReconfigureArgs, lowTokensN // A Reservation holds information about events that are permitted by a Limiter to happen after a delay. // A Reservation may be canceled, which may enable the Limiter to permit additional events. type Reservation struct { - ok bool - lim *Limiter - tokens float64 - timeToAct time.Time + ok bool + lim *Limiter + tokens float64 + timeToAct time.Time + needWaitDurtion time.Duration // This is the Limit at reservation time, it can change later. limit Limit } @@ -301,7 +302,7 @@ func (lim *Limiter) Reconfigure(now time.Time, ) { lim.mu.Lock() defer lim.mu.Unlock() - log.Debug("[resource group controller] before reconfigure", zap.Float64("old-tokens", lim.tokens), zap.Float64("old-rate", float64(lim.limit)), zap.Float64("old-notify-threshold", args.NotifyThreshold), zap.Int64("old-burst", lim.burst)) + logControllerTrace("[resource group controller] before reconfigure", zap.Float64("old-tokens", lim.tokens), zap.Float64("old-rate", float64(lim.limit)), zap.Float64("old-notify-threshold", args.NotifyThreshold), zap.Int64("old-burst", lim.burst)) if args.NewBurst < 0 { lim.last = now lim.tokens = args.NewTokens @@ -317,7 +318,7 @@ func (lim *Limiter) Reconfigure(now time.Time, opt(lim) } lim.maybeNotify() - log.Debug("[resource group controller] after reconfigure", zap.Float64("tokens", lim.tokens), zap.Float64("rate", float64(lim.limit)), zap.Float64("notify-threshold", args.NotifyThreshold), zap.Int64("burst", lim.burst)) + logControllerTrace("[resource group controller] after reconfigure", zap.Float64("tokens", lim.tokens), zap.Float64("rate", float64(lim.limit)), zap.Float64("notify-threshold", args.NotifyThreshold), zap.Int64("burst", lim.burst)) } // AvailableTokens decreases the amount of tokens currently available. @@ -358,9 +359,10 @@ func (lim *Limiter) reserveN(now time.Time, n float64, maxFutureReserve time.Dur // Prepare reservation r := Reservation{ - ok: ok, - lim: lim, - limit: lim.limit, + ok: ok, + lim: lim, + limit: lim.limit, + needWaitDurtion: waitDuration, } if ok { r.tokens = n @@ -372,7 +374,14 @@ func (lim *Limiter) reserveN(now time.Time, n float64, maxFutureReserve time.Dur lim.tokens = tokens lim.maybeNotify() } else { - log.Debug("[resource group controller]", zap.Float64("current-tokens", lim.tokens), zap.Float64("current-rate", float64(lim.limit)), zap.Float64("request-tokens", n), zap.Int64("burst", lim.burst), zap.Int("remaining-notify-times", lim.remainingNotifyTimes)) + log.Warn("[resource group controller] cannot reserve enough tokens", + zap.Duration("need-wait-duration", waitDuration), + zap.Duration("max-wait-duration", maxFutureReserve), + zap.Float64("current-ltb-tokens", lim.tokens), + zap.Float64("current-ltb-rate", float64(lim.limit)), + zap.Float64("request-tokens", n), + zap.Int64("burst", lim.burst), + zap.Int("remaining-notify-times", lim.remainingNotifyTimes)) lim.last = last if lim.limit == 0 { lim.notify() @@ -452,7 +461,7 @@ func WaitReservations(ctx context.Context, now time.Time, reservations []*Reserv for _, res := range reservations { if !res.ok { cancel() - return 0, errs.ErrClientResourceGroupThrottled + return res.needWaitDurtion, errs.ErrClientResourceGroupThrottled } delay := res.DelayFrom(now) if delay > longestDelayDuration { diff --git a/client/resource_group/controller/limiter_test.go b/client/resource_group/controller/limiter_test.go index b8b96ae13d6..d963f830551 100644 --- a/client/resource_group/controller/limiter_test.go +++ b/client/resource_group/controller/limiter_test.go @@ -161,7 +161,7 @@ func TestCancel(t *testing.T) { checkTokens(re, lim1, t2, 7) checkTokens(re, lim2, t2, 2) d, err := WaitReservations(ctx, t2, []*Reservation{r1, r2}) - re.Equal(d, time.Duration(0)) + re.Equal(d, 4*time.Second) re.Error(err) checkTokens(re, lim1, t3, 13) checkTokens(re, lim2, t3, 3) diff --git a/client/resource_group/controller/metrics.go b/client/resource_group/controller/metrics.go index 68eb26d0312..47e285ad775 100644 --- a/client/resource_group/controller/metrics.go +++ b/client/resource_group/controller/metrics.go @@ -42,6 +42,15 @@ var ( Help: "Bucketed histogram of wait duration of successful request.", }, []string{resourceGroupNameLabel}) + failedLimitReserveDuration = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: requestSubsystem, + Name: "limit_reserve_time_failed", + Buckets: []float64{.005, .01, .05, .1, .5, 1, 5, 10, 20, 25, 30}, // 0.005 ~ 30 + Help: "Bucketed histogram of wait duration of failed request.", + }, []string{resourceGroupNameLabel}) + failedRequestCounter = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: namespace, diff --git a/pkg/mcs/resourcemanager/server/config.go b/pkg/mcs/resourcemanager/server/config.go index 10e91612842..bcd5a853dfc 100644 --- a/pkg/mcs/resourcemanager/server/config.go +++ b/pkg/mcs/resourcemanager/server/config.go @@ -102,6 +102,9 @@ type ControllerConfig struct { // RequestUnit is the configuration determines the coefficients of the RRU and WRU cost. // This configuration should be modified carefully. RequestUnit RequestUnitConfig `toml:"request-unit" json:"request-unit"` + + // EnableControllerTraceLog is to control whether resource control client enable trace. + EnableControllerTraceLog bool `toml:"enable-controller-trace-log" json:"enable-controller-trace-log,string"` } // Adjust adjusts the configuration and initializes it with the default value if necessary. diff --git a/tests/pdctl/resourcemanager/resource_manager_command_test.go b/tests/pdctl/resourcemanager/resource_manager_command_test.go new file mode 100644 index 00000000000..ad43e0abca9 --- /dev/null +++ b/tests/pdctl/resourcemanager/resource_manager_command_test.go @@ -0,0 +1,97 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package resourcemanager_test + +import ( + "context" + "encoding/json" + "testing" + "time" + + "github.com/stretchr/testify/suite" + "github.com/tikv/pd/pkg/mcs/resourcemanager/server" + "github.com/tikv/pd/pkg/utils/typeutil" + "github.com/tikv/pd/tests" + "github.com/tikv/pd/tests/pdctl" + pdctlCmd "github.com/tikv/pd/tools/pd-ctl/pdctl" +) + +func TestResourceManagerSuite(t *testing.T) { + suite.Run(t, new(testResourceManagerSuite)) +} + +type testResourceManagerSuite struct { + suite.Suite + ctx context.Context + cancel context.CancelFunc + cluster *tests.TestCluster + pdAddr string +} + +func (s *testResourceManagerSuite) SetupSuite() { + s.ctx, s.cancel = context.WithCancel(context.Background()) + cluster, err := tests.NewTestCluster(s.ctx, 1) + s.Nil(err) + s.cluster = cluster + s.cluster.RunInitialServers() + cluster.WaitLeader() + s.pdAddr = cluster.GetConfig().GetClientURL() +} + +func (s *testResourceManagerSuite) TearDownSuite() { + s.cancel() + s.cluster.Destroy() +} + +func (s *testResourceManagerSuite) TestConfigController() { + expectCfg := server.ControllerConfig{} + expectCfg.Adjust(nil) + // Show controller config + checkShow := func() { + args := []string{"-u", s.pdAddr, "resource-manager", "config", "controller", "show"} + output, err := pdctl.ExecuteCommand(pdctlCmd.GetRootCmd(), args...) + s.Nil(err) + + actualCfg := server.ControllerConfig{} + err = json.Unmarshal(output, &actualCfg) + s.Nil(err) + s.Equal(expectCfg, actualCfg) + } + + // Check default config + checkShow() + + // Set controller config + args := []string{"-u", s.pdAddr, "resource-manager", "config", "controller", "set", "ltb-max-wait-duration", "1h"} + output, err := pdctl.ExecuteCommand(pdctlCmd.GetRootCmd(), args...) + s.Nil(err) + s.Contains(string(output), "Success!") + expectCfg.LTBMaxWaitDuration = typeutil.Duration{Duration: 1 * time.Hour} + checkShow() + + args = []string{"-u", s.pdAddr, "resource-manager", "config", "controller", "set", "enable-controller-trace-log", "true"} + output, err = pdctl.ExecuteCommand(pdctlCmd.GetRootCmd(), args...) + s.Nil(err) + s.Contains(string(output), "Success!") + expectCfg.EnableControllerTraceLog = true + checkShow() + + args = []string{"-u", s.pdAddr, "resource-manager", "config", "controller", "set", "write-base-cost", "2"} + output, err = pdctl.ExecuteCommand(pdctlCmd.GetRootCmd(), args...) + s.Nil(err) + s.Contains(string(output), "Success!") + expectCfg.RequestUnit.WriteBaseCost = 2 + checkShow() +} diff --git a/tools/pd-ctl/pdctl/command/resource_manager_command.go b/tools/pd-ctl/pdctl/command/resource_manager_command.go new file mode 100644 index 00000000000..8bc5ea85977 --- /dev/null +++ b/tools/pd-ctl/pdctl/command/resource_manager_command.go @@ -0,0 +1,112 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package command + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + "strconv" + + "github.com/spf13/cobra" +) + +const ( + resourceManagerPrefix = "resource-manager/api/v1" + // flags + rmConfigController = "config/controller" +) + +// NewResourceManagerCommand return a resource manager subcommand of rootCmd +func NewResourceManagerCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "resource-manager [flags]", + Short: "resource-manager commands", + } + cmd.AddCommand(newResourceManagerConfigCommand()) + return cmd +} + +func newResourceManagerConfigCommand() *cobra.Command { + r := &cobra.Command{ + Use: "config", + Short: "config resource manager", + } + r.AddCommand(newConfigControllerCommand()) + return r +} + +func newConfigControllerCommand() *cobra.Command { + r := &cobra.Command{ + Use: "controller", + Short: "config controller", + } + r.AddCommand(newConfigControllerSetCommand()) + r.AddCommand(newConfigControllerShowCommand()) + return r +} + +func newConfigControllerSetCommand() *cobra.Command { + r := &cobra.Command{ + Use: "set ", + Short: "set controller config", + Run: func(cmd *cobra.Command, args []string) { + if len(args) != 2 { + cmd.Println(cmd.UsageString()) + return + } + + var val interface{} + val, err := strconv.ParseFloat(args[1], 64) + if err != nil { + val = args[1] + } + data := map[string]interface{}{args[0]: val} + jsonData, err := json.Marshal(data) + if err != nil { + cmd.Println(err) + return + } + resp, err := doRequest(cmd, fmt.Sprintf("%s/%s", resourceManagerPrefix, rmConfigController), http.MethodPost, http.Header{}, WithBody(bytes.NewBuffer(jsonData))) + if err != nil { + cmd.PrintErrln("Failed to set the config: ", err) + return + } + cmd.Println(resp) + }, + } + return r +} + +func newConfigControllerShowCommand() *cobra.Command { + r := &cobra.Command{ + Use: "show", + Short: "show controller config", + Run: func(cmd *cobra.Command, args []string) { + if len(args) != 0 { + cmd.Println(cmd.UsageString()) + return + } + resp, err := doRequest(cmd, fmt.Sprintf("%s/%s", resourceManagerPrefix, rmConfigController), http.MethodGet, http.Header{}) + if err != nil { + cmd.Println(err) + return + } + cmd.Println(resp) + }, + } + return r +} diff --git a/tools/pd-ctl/pdctl/ctl.go b/tools/pd-ctl/pdctl/ctl.go index 86494c046eb..7a3c540b266 100644 --- a/tools/pd-ctl/pdctl/ctl.go +++ b/tools/pd-ctl/pdctl/ctl.go @@ -67,6 +67,7 @@ func GetRootCmd() *cobra.Command { command.NewUnsafeCommand(), command.NewKeyspaceGroupCommand(), command.NewKeyspaceCommand(), + command.NewResourceManagerCommand(), ) rootCmd.Flags().ParseErrorsWhitelist.UnknownFlags = true From fcdeec42aa9a9033c576083c1571dcd264d068c8 Mon Sep 17 00:00:00 2001 From: ShuNing Date: Mon, 11 Dec 2023 16:40:47 +0800 Subject: [PATCH 085/137] resource_control: fix data race in controller (#7520) close tikv/pd#4399, ref pingcap/tidb#49311 the meta information is not safe to visit concurrency - address the data race condition Signed-off-by: Shuning Chen Co-authored-by: Shuning Chen --- client/resource_group/controller/controller.go | 2 +- client/resource_group/controller/metrics.go | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/client/resource_group/controller/controller.go b/client/resource_group/controller/controller.go index f9a8caaabed..a07622e5f3b 100755 --- a/client/resource_group/controller/controller.go +++ b/client/resource_group/controller/controller.go @@ -1265,7 +1265,7 @@ func (gc *groupCostController) onRequestWait( *gc.mu.storeCounter[info.StoreID()] = *gc.mu.globalCounter gc.mu.Unlock() - return delta, penalty, waitDuration, gc.meta.Priority, nil + return delta, penalty, waitDuration, gc.getMeta().GetPriority(), nil } func (gc *groupCostController) onResponse( diff --git a/client/resource_group/controller/metrics.go b/client/resource_group/controller/metrics.go index 47e285ad775..7e6a559265b 100644 --- a/client/resource_group/controller/metrics.go +++ b/client/resource_group/controller/metrics.go @@ -94,6 +94,7 @@ func init() { prometheus.MustRegister(resourceGroupStatusGauge) prometheus.MustRegister(successfulRequestDuration) prometheus.MustRegister(failedRequestCounter) + prometheus.MustRegister(failedLimitReserveDuration) prometheus.MustRegister(requestRetryCounter) prometheus.MustRegister(tokenRequestDuration) prometheus.MustRegister(resourceGroupTokenRequestCounter) From ab97b9a267f3a733b587fc96a1af90e39767331d Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Tue, 12 Dec 2023 14:16:47 +0800 Subject: [PATCH 086/137] mcs: fix panic of GetMinTS (#7529) close tikv/pd#7528 Signed-off-by: Ryan Leung Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- server/grpc_service.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/server/grpc_service.go b/server/grpc_service.go index 6acc13b8187..fa74f1ea8b6 100644 --- a/server/grpc_service.go +++ b/server/grpc_service.go @@ -328,11 +328,11 @@ func (s *GrpcServer) GetMinTSFromTSOService(dcLocation string) (*pdpb.Timestamp, // Get the minimal timestamp from the TSO servers/pods var mutex syncutil.Mutex - resps := make([]*tsopb.GetMinTSResponse, len(addrs)) + resps := make([]*tsopb.GetMinTSResponse, 0) wg := sync.WaitGroup{} wg.Add(len(addrs)) - for idx, addr := range addrs { - go func(idx int, addr string) { + for _, addr := range addrs { + go func(addr string) { defer wg.Done() resp, err := s.getMinTSFromSingleServer(s.ctx, dcLocation, addr) if err != nil || resp == nil { @@ -342,8 +342,8 @@ func (s *GrpcServer) GetMinTSFromTSOService(dcLocation string) (*pdpb.Timestamp, } mutex.Lock() defer mutex.Unlock() - resps[idx] = resp - }(idx, addr) + resps = append(resps, resp) + }(addr) } wg.Wait() From f71de230a6cf9f15c54a1efd237d062c039a0bde Mon Sep 17 00:00:00 2001 From: JmPotato Date: Wed, 13 Dec 2023 13:15:18 +0800 Subject: [PATCH 087/137] audit, client/http: use X-Caller-ID to replace the component signature key (#7536) ref tikv/pd#7300 Use `X-Caller-ID` to replace the component signature key. Signed-off-by: JmPotato --- client/http/client.go | 4 +- client/http/client_test.go | 2 +- pkg/audit/audit.go | 2 +- pkg/audit/audit_test.go | 12 +++--- pkg/utils/apiutil/apiutil.go | 55 ++++++++++++++++----------- pkg/utils/apiutil/apiutil_test.go | 8 ++-- pkg/utils/requestutil/context_test.go | 4 +- pkg/utils/requestutil/request_info.go | 14 ++++--- server/api/middleware.go | 2 +- server/metrics.go | 2 +- tests/pdctl/global_test.go | 12 +++--- tests/server/api/api_test.go | 10 ++--- tools/pd-ctl/pdctl/command/global.go | 17 +++++---- 13 files changed, 80 insertions(+), 64 deletions(-) diff --git a/client/http/client.go b/client/http/client.go index b79aa9ca002..613ebf33294 100644 --- a/client/http/client.go +++ b/client/http/client.go @@ -221,7 +221,7 @@ func (c *client) execDuration(name string, duration time.Duration) { // Header key definition constants. const ( pdAllowFollowerHandleKey = "PD-Allow-Follower-Handle" - componentSignatureKey = "component" + xCallerIDKey = "X-Caller-ID" ) // HeaderOption configures the HTTP header. @@ -279,7 +279,7 @@ func (c *client) request( for _, opt := range headerOpts { opt(req.Header) } - req.Header.Set(componentSignatureKey, c.callerID) + req.Header.Set(xCallerIDKey, c.callerID) start := time.Now() resp, err := c.inner.cli.Do(req) diff --git a/client/http/client_test.go b/client/http/client_test.go index 621910e29ea..70c2ddee08b 100644 --- a/client/http/client_test.go +++ b/client/http/client_test.go @@ -59,7 +59,7 @@ func TestCallerID(t *testing.T) { re := require.New(t) expectedVal := defaultCallerID httpClient := newHTTPClientWithRequestChecker(func(req *http.Request) error { - val := req.Header.Get(componentSignatureKey) + val := req.Header.Get(xCallerIDKey) if val != expectedVal { re.Failf("Caller ID header check failed", "should be %s, but got %s", expectedVal, val) diff --git a/pkg/audit/audit.go b/pkg/audit/audit.go index 3553e5f8377..b971b09ed7e 100644 --- a/pkg/audit/audit.go +++ b/pkg/audit/audit.go @@ -98,7 +98,7 @@ func (b *PrometheusHistogramBackend) ProcessHTTPRequest(req *http.Request) bool if !ok { return false } - b.histogramVec.WithLabelValues(requestInfo.ServiceLabel, "HTTP", requestInfo.Component, requestInfo.IP).Observe(float64(endTime - requestInfo.StartTimeStamp)) + b.histogramVec.WithLabelValues(requestInfo.ServiceLabel, "HTTP", requestInfo.CallerID, requestInfo.IP).Observe(float64(endTime - requestInfo.StartTimeStamp)) return true } diff --git a/pkg/audit/audit_test.go b/pkg/audit/audit_test.go index d59c9627115..8098b36975e 100644 --- a/pkg/audit/audit_test.go +++ b/pkg/audit/audit_test.go @@ -51,7 +51,7 @@ func TestPrometheusHistogramBackend(t *testing.T) { Name: "audit_handling_seconds_test", Help: "PD server service handling audit", Buckets: prometheus.DefBuckets, - }, []string{"service", "method", "component", "ip"}) + }, []string{"service", "method", "caller_id", "ip"}) prometheus.MustRegister(serviceAuditHistogramTest) @@ -62,7 +62,7 @@ func TestPrometheusHistogramBackend(t *testing.T) { req, _ := http.NewRequest(http.MethodGet, "http://127.0.0.1:2379/test?test=test", http.NoBody) info := requestutil.GetRequestInfo(req) info.ServiceLabel = "test" - info.Component = "user1" + info.CallerID = "user1" info.IP = "localhost" req = req.WithContext(requestutil.WithRequestInfo(req.Context(), info)) re.False(backend.ProcessHTTPRequest(req)) @@ -73,7 +73,7 @@ func TestPrometheusHistogramBackend(t *testing.T) { re.True(backend.ProcessHTTPRequest(req)) re.True(backend.ProcessHTTPRequest(req)) - info.Component = "user2" + info.CallerID = "user2" req = req.WithContext(requestutil.WithRequestInfo(req.Context(), info)) re.True(backend.ProcessHTTPRequest(req)) @@ -85,8 +85,8 @@ func TestPrometheusHistogramBackend(t *testing.T) { defer resp.Body.Close() content, _ := io.ReadAll(resp.Body) output := string(content) - re.Contains(output, "pd_service_audit_handling_seconds_test_count{component=\"user1\",ip=\"localhost\",method=\"HTTP\",service=\"test\"} 2") - re.Contains(output, "pd_service_audit_handling_seconds_test_count{component=\"user2\",ip=\"localhost\",method=\"HTTP\",service=\"test\"} 1") + re.Contains(output, "pd_service_audit_handling_seconds_test_count{caller_id=\"user1\",ip=\"localhost\",method=\"HTTP\",service=\"test\"} 2") + re.Contains(output, "pd_service_audit_handling_seconds_test_count{caller_id=\"user2\",ip=\"localhost\",method=\"HTTP\",service=\"test\"} 1") } func TestLocalLogBackendUsingFile(t *testing.T) { @@ -103,7 +103,7 @@ func TestLocalLogBackendUsingFile(t *testing.T) { b, _ := os.ReadFile(fname) output := strings.SplitN(string(b), "]", 4) re.Equal( - fmt.Sprintf(" [\"audit log\"] [service-info=\"{ServiceLabel:, Method:HTTP/1.1/GET:/test, Component:anonymous, IP:, Port:, "+ + fmt.Sprintf(" [\"audit log\"] [service-info=\"{ServiceLabel:, Method:HTTP/1.1/GET:/test, CallerID:anonymous, IP:, Port:, "+ "StartTime:%s, URLParam:{\\\"test\\\":[\\\"test\\\"]}, BodyParam:testBody}\"]\n", time.Unix(info.StartTimeStamp, 0).String()), output[3], diff --git a/pkg/utils/apiutil/apiutil.go b/pkg/utils/apiutil/apiutil.go index 164b3c0783d..53fab682fcb 100644 --- a/pkg/utils/apiutil/apiutil.go +++ b/pkg/utils/apiutil/apiutil.go @@ -39,15 +39,14 @@ import ( "go.uber.org/zap" ) -var ( - // componentSignatureKey is used for http request header key - // to identify component signature +const ( + // componentSignatureKey is used for http request header key to identify component signature. + // Deprecated: please use `XCallerIDHeader` below to obtain a more granular source identification. + // This is kept for backward compatibility. componentSignatureKey = "component" - // componentAnonymousValue identifies anonymous request source - componentAnonymousValue = "anonymous" -) + // anonymousValue identifies anonymous request source + anonymousValue = "anonymous" -const ( // PDRedirectorHeader is used to mark which PD redirected this request. PDRedirectorHeader = "PD-Redirector" // PDAllowFollowerHandleHeader is used to mark whether this request is allowed to be handled by the follower PD. @@ -58,6 +57,8 @@ const ( XForwardedPortHeader = "X-Forwarded-Port" // XRealIPHeader is used to mark the real client IP. XRealIPHeader = "X-Real-Ip" + // XCallerIDHeader is used to mark the caller ID. + XCallerIDHeader = "X-Caller-ID" // ForwardToMicroServiceHeader is used to mark the request is forwarded to micro service. ForwardToMicroServiceHeader = "Forward-To-Micro-Service" @@ -112,7 +113,7 @@ func ErrorResp(rd *render.Render, w http.ResponseWriter, err error) { // GetIPPortFromHTTPRequest returns http client host IP and port from context. // Because `X-Forwarded-For ` header has been written into RFC 7239(Forwarded HTTP Extension), -// so `X-Forwarded-For` has the higher priority than `X-Real-IP`. +// so `X-Forwarded-For` has the higher priority than `X-Real-Ip`. // And both of them have the higher priority than `RemoteAddr` func GetIPPortFromHTTPRequest(r *http.Request) (ip, port string) { forwardedIPs := strings.Split(r.Header.Get(XForwardedForHeader), ",") @@ -136,32 +137,42 @@ func GetIPPortFromHTTPRequest(r *http.Request) (ip, port string) { return splitIP, splitPort } -// GetComponentNameOnHTTP returns component name from Request Header -func GetComponentNameOnHTTP(r *http.Request) string { +// getComponentNameOnHTTP returns component name from the request header. +func getComponentNameOnHTTP(r *http.Request) string { componentName := r.Header.Get(componentSignatureKey) if len(componentName) == 0 { - componentName = componentAnonymousValue + componentName = anonymousValue } return componentName } -// ComponentSignatureRoundTripper is used to add component signature in HTTP header -type ComponentSignatureRoundTripper struct { - proxied http.RoundTripper - component string +// GetCallerIDOnHTTP returns caller ID from the request header. +func GetCallerIDOnHTTP(r *http.Request) string { + callerID := r.Header.Get(XCallerIDHeader) + if len(callerID) == 0 { + // Fall back to get the component name to keep backward compatibility. + callerID = getComponentNameOnHTTP(r) + } + return callerID +} + +// CallerIDRoundTripper is used to add caller ID in the HTTP header. +type CallerIDRoundTripper struct { + proxied http.RoundTripper + callerID string } -// NewComponentSignatureRoundTripper returns a new ComponentSignatureRoundTripper. -func NewComponentSignatureRoundTripper(roundTripper http.RoundTripper, componentName string) *ComponentSignatureRoundTripper { - return &ComponentSignatureRoundTripper{ - proxied: roundTripper, - component: componentName, +// NewCallerIDRoundTripper returns a new `CallerIDRoundTripper`. +func NewCallerIDRoundTripper(roundTripper http.RoundTripper, callerID string) *CallerIDRoundTripper { + return &CallerIDRoundTripper{ + proxied: roundTripper, + callerID: callerID, } } // RoundTrip is used to implement RoundTripper -func (rt *ComponentSignatureRoundTripper) RoundTrip(req *http.Request) (resp *http.Response, err error) { - req.Header.Add(componentSignatureKey, rt.component) +func (rt *CallerIDRoundTripper) RoundTrip(req *http.Request) (resp *http.Response, err error) { + req.Header.Add(XCallerIDHeader, rt.callerID) // Send the request, get the response and the error resp, err = rt.proxied.RoundTrip(req) return diff --git a/pkg/utils/apiutil/apiutil_test.go b/pkg/utils/apiutil/apiutil_test.go index a4e7b97aa4d..106d3fb21cb 100644 --- a/pkg/utils/apiutil/apiutil_test.go +++ b/pkg/utils/apiutil/apiutil_test.go @@ -101,7 +101,7 @@ func TestGetIPPortFromHTTPRequest(t *testing.T) { ip: "127.0.0.1", port: "5299", }, - // IPv4 "X-Real-IP" with port + // IPv4 "X-Real-Ip" with port { r: &http.Request{ Header: map[string][]string{ @@ -111,7 +111,7 @@ func TestGetIPPortFromHTTPRequest(t *testing.T) { ip: "127.0.0.1", port: "5299", }, - // IPv4 "X-Real-IP" without port + // IPv4 "X-Real-Ip" without port { r: &http.Request{ Header: map[string][]string{ @@ -158,7 +158,7 @@ func TestGetIPPortFromHTTPRequest(t *testing.T) { ip: "::1", port: "", }, - // IPv6 "X-Real-IP" with port + // IPv6 "X-Real-Ip" with port { r: &http.Request{ Header: map[string][]string{ @@ -168,7 +168,7 @@ func TestGetIPPortFromHTTPRequest(t *testing.T) { ip: "::1", port: "5299", }, - // IPv6 "X-Real-IP" without port + // IPv6 "X-Real-Ip" without port { r: &http.Request{ Header: map[string][]string{ diff --git a/pkg/utils/requestutil/context_test.go b/pkg/utils/requestutil/context_test.go index 475b109e410..298fc1ff8a3 100644 --- a/pkg/utils/requestutil/context_test.go +++ b/pkg/utils/requestutil/context_test.go @@ -34,7 +34,7 @@ func TestRequestInfo(t *testing.T) { RequestInfo{ ServiceLabel: "test label", Method: http.MethodPost, - Component: "pdctl", + CallerID: "pdctl", IP: "localhost", URLParam: "{\"id\"=1}", BodyParam: "{\"state\"=\"Up\"}", @@ -45,7 +45,7 @@ func TestRequestInfo(t *testing.T) { re.True(ok) re.Equal("test label", result.ServiceLabel) re.Equal(http.MethodPost, result.Method) - re.Equal("pdctl", result.Component) + re.Equal("pdctl", result.CallerID) re.Equal("localhost", result.IP) re.Equal("{\"id\"=1}", result.URLParam) re.Equal("{\"state\"=\"Up\"}", result.BodyParam) diff --git a/pkg/utils/requestutil/request_info.go b/pkg/utils/requestutil/request_info.go index 40724bb790f..cc5403f7232 100644 --- a/pkg/utils/requestutil/request_info.go +++ b/pkg/utils/requestutil/request_info.go @@ -27,9 +27,11 @@ import ( // RequestInfo holds service information from http.Request type RequestInfo struct { - ServiceLabel string - Method string - Component string + ServiceLabel string + Method string + // CallerID is used to identify the specific source of a HTTP request, it will be marked in + // the PD HTTP client, with granularity that can be refined to a specific functionality within a component. + CallerID string IP string Port string URLParam string @@ -38,8 +40,8 @@ type RequestInfo struct { } func (info *RequestInfo) String() string { - s := fmt.Sprintf("{ServiceLabel:%s, Method:%s, Component:%s, IP:%s, Port:%s, StartTime:%s, URLParam:%s, BodyParam:%s}", - info.ServiceLabel, info.Method, info.Component, info.IP, info.Port, time.Unix(info.StartTimeStamp, 0), info.URLParam, info.BodyParam) + s := fmt.Sprintf("{ServiceLabel:%s, Method:%s, CallerID:%s, IP:%s, Port:%s, StartTime:%s, URLParam:%s, BodyParam:%s}", + info.ServiceLabel, info.Method, info.CallerID, info.IP, info.Port, time.Unix(info.StartTimeStamp, 0), info.URLParam, info.BodyParam) return s } @@ -49,7 +51,7 @@ func GetRequestInfo(r *http.Request) RequestInfo { return RequestInfo{ ServiceLabel: apiutil.GetRouteName(r), Method: fmt.Sprintf("%s/%s:%s", r.Proto, r.Method, r.URL.Path), - Component: apiutil.GetComponentNameOnHTTP(r), + CallerID: apiutil.GetCallerIDOnHTTP(r), IP: ip, Port: port, URLParam: getURLParam(r), diff --git a/server/api/middleware.go b/server/api/middleware.go index 627d7fecc92..4173c37b396 100644 --- a/server/api/middleware.go +++ b/server/api/middleware.go @@ -69,7 +69,7 @@ func (rm *requestInfoMiddleware) ServeHTTP(w http.ResponseWriter, r *http.Reques w.Header().Add("body-param", requestInfo.BodyParam) w.Header().Add("url-param", requestInfo.URLParam) w.Header().Add("method", requestInfo.Method) - w.Header().Add("component", requestInfo.Component) + w.Header().Add("caller-id", requestInfo.CallerID) w.Header().Add("ip", requestInfo.IP) }) diff --git a/server/metrics.go b/server/metrics.go index 2d13d02d564..54c5830dc52 100644 --- a/server/metrics.go +++ b/server/metrics.go @@ -151,7 +151,7 @@ var ( Name: "audit_handling_seconds", Help: "PD server service handling audit", Buckets: prometheus.DefBuckets, - }, []string{"service", "method", "component", "ip"}) + }, []string{"service", "method", "caller_id", "ip"}) serverMaxProcs = prometheus.NewGauge( prometheus.GaugeOpts{ Namespace: "pd", diff --git a/tests/pdctl/global_test.go b/tests/pdctl/global_test.go index 7e57f589249..00d31a384d5 100644 --- a/tests/pdctl/global_test.go +++ b/tests/pdctl/global_test.go @@ -30,18 +30,20 @@ import ( "go.uber.org/zap" ) +const pdControlCallerID = "pd-ctl" + func TestSendAndGetComponent(t *testing.T) { re := require.New(t) handler := func(ctx context.Context, s *server.Server) (http.Handler, apiutil.APIServiceGroup, error) { mux := http.NewServeMux() mux.HandleFunc("/pd/api/v1/health", func(w http.ResponseWriter, r *http.Request) { - component := apiutil.GetComponentNameOnHTTP(r) + callerID := apiutil.GetCallerIDOnHTTP(r) for k := range r.Header { log.Info("header", zap.String("key", k)) } - log.Info("component", zap.String("component", component)) - re.Equal("pdctl", component) - fmt.Fprint(w, component) + log.Info("caller id", zap.String("caller-id", callerID)) + re.Equal(pdControlCallerID, callerID) + fmt.Fprint(w, callerID) }) info := apiutil.APIServiceGroup{ IsCore: true, @@ -65,5 +67,5 @@ func TestSendAndGetComponent(t *testing.T) { args := []string{"-u", pdAddr, "health"} output, err := ExecuteCommand(cmd, args...) re.NoError(err) - re.Equal("pdctl\n", string(output)) + re.Equal(fmt.Sprintf("%s\n", pdControlCallerID), string(output)) } diff --git a/tests/server/api/api_test.go b/tests/server/api/api_test.go index 905fb8ec096..f5db6bb2513 100644 --- a/tests/server/api/api_test.go +++ b/tests/server/api/api_test.go @@ -164,7 +164,7 @@ func (suite *middlewareTestSuite) TestRequestInfoMiddleware() { suite.Equal("{\"force\":[\"true\"]}", resp.Header.Get("url-param")) suite.Equal("{\"testkey\":\"testvalue\"}", resp.Header.Get("body-param")) suite.Equal("HTTP/1.1/POST:/pd/api/v1/debug/pprof/profile", resp.Header.Get("method")) - suite.Equal("anonymous", resp.Header.Get("component")) + suite.Equal("anonymous", resp.Header.Get("caller-id")) suite.Equal("127.0.0.1", resp.Header.Get("ip")) input = map[string]interface{}{ @@ -408,7 +408,7 @@ func (suite *middlewareTestSuite) TestAuditPrometheusBackend() { defer resp.Body.Close() content, _ := io.ReadAll(resp.Body) output := string(content) - suite.Contains(output, "pd_service_audit_handling_seconds_count{component=\"anonymous\",ip=\"127.0.0.1\",method=\"HTTP\",service=\"GetTrend\"} 1") + suite.Contains(output, "pd_service_audit_handling_seconds_count{caller_id=\"anonymous\",ip=\"127.0.0.1\",method=\"HTTP\",service=\"GetTrend\"} 1") // resign to test persist config oldLeaderName := leader.GetServer().Name() @@ -434,7 +434,7 @@ func (suite *middlewareTestSuite) TestAuditPrometheusBackend() { defer resp.Body.Close() content, _ = io.ReadAll(resp.Body) output = string(content) - suite.Contains(output, "pd_service_audit_handling_seconds_count{component=\"anonymous\",ip=\"127.0.0.1\",method=\"HTTP\",service=\"GetTrend\"} 2") + suite.Contains(output, "pd_service_audit_handling_seconds_count{caller_id=\"anonymous\",ip=\"127.0.0.1\",method=\"HTTP\",service=\"GetTrend\"} 2") input = map[string]interface{}{ "enable-audit": "false", @@ -543,7 +543,7 @@ func BenchmarkDoRequestWithoutServiceMiddleware(b *testing.B) { func doTestRequestWithLogAudit(srv *tests.TestServer) { req, _ := http.NewRequest(http.MethodDelete, fmt.Sprintf("%s/pd/api/v1/admin/cache/regions", srv.GetAddr()), http.NoBody) - req.Header.Set("component", "test") + req.Header.Set(apiutil.XCallerIDHeader, "test") resp, _ := dialClient.Do(req) resp.Body.Close() } @@ -551,7 +551,7 @@ func doTestRequestWithLogAudit(srv *tests.TestServer) { func doTestRequestWithPrometheus(srv *tests.TestServer) { timeUnix := time.Now().Unix() - 20 req, _ := http.NewRequest(http.MethodGet, fmt.Sprintf("%s/pd/api/v1/trend?from=%d", srv.GetAddr(), timeUnix), http.NoBody) - req.Header.Set("component", "test") + req.Header.Set(apiutil.XCallerIDHeader, "test") resp, _ := dialClient.Do(req) resp.Body.Close() } diff --git a/tools/pd-ctl/pdctl/command/global.go b/tools/pd-ctl/pdctl/command/global.go index 5d8552da51a..0b1f4b4409a 100644 --- a/tools/pd-ctl/pdctl/command/global.go +++ b/tools/pd-ctl/pdctl/command/global.go @@ -29,14 +29,15 @@ import ( "go.etcd.io/etcd/pkg/transport" ) -var ( - pdControllerComponentName = "pdctl" - dialClient = &http.Client{ - Transport: apiutil.NewComponentSignatureRoundTripper(http.DefaultTransport, pdControllerComponentName), - } - pingPrefix = "pd/api/v1/ping" +const ( + pdControlCallerID = "pd-ctl" + pingPrefix = "pd/api/v1/ping" ) +var dialClient = &http.Client{ + Transport: apiutil.NewCallerIDRoundTripper(http.DefaultTransport, pdControlCallerID), +} + // InitHTTPSClient creates https client with ca file func InitHTTPSClient(caPath, certPath, keyPath string) error { tlsInfo := transport.TLSInfo{ @@ -50,8 +51,8 @@ func InitHTTPSClient(caPath, certPath, keyPath string) error { } dialClient = &http.Client{ - Transport: apiutil.NewComponentSignatureRoundTripper( - &http.Transport{TLSClientConfig: tlsConfig}, pdControllerComponentName), + Transport: apiutil.NewCallerIDRoundTripper( + &http.Transport{TLSClientConfig: tlsConfig}, pdControlCallerID), } return nil From 1eed4948168ac555d07f15574096e54b927fa802 Mon Sep 17 00:00:00 2001 From: Hu# Date: Wed, 13 Dec 2023 14:40:49 +0800 Subject: [PATCH 088/137] client/http: add func for jenkins test (#7516) ref tikv/pd#7300 Signed-off-by: husharp Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- client/http/api.go | 13 ++++ client/http/client.go | 71 +++++++++++++++++++ client/http/types.go | 10 +++ tests/integrations/client/http_client_test.go | 62 +++++++++++++++- 4 files changed, 155 insertions(+), 1 deletion(-) diff --git a/client/http/api.go b/client/http/api.go index f744fd0c395..2153cd286e8 100644 --- a/client/http/api.go +++ b/client/http/api.go @@ -38,6 +38,9 @@ const ( store = "/pd/api/v1/store" Stores = "/pd/api/v1/stores" StatsRegion = "/pd/api/v1/stats/region" + membersPrefix = "/pd/api/v1/members" + leaderPrefix = "/pd/api/v1/leader" + transferLeader = "/pd/api/v1/leader/transfer" // Config Config = "/pd/api/v1/config" ClusterVersion = "/pd/api/v1/config/cluster-version" @@ -124,6 +127,16 @@ func StoreLabelByID(id uint64) string { return fmt.Sprintf("%s/%d/label", store, id) } +// LabelByStoreID returns the path of PD HTTP API to set store label. +func LabelByStoreID(storeID int64) string { + return fmt.Sprintf("%s/%d/label", store, storeID) +} + +// TransferLeaderByID returns the path of PD HTTP API to transfer leader by ID. +func TransferLeaderByID(leaderID string) string { + return fmt.Sprintf("%s/%s", transferLeader, leaderID) +} + // ConfigWithTTLSeconds returns the config API with the TTL seconds parameter. func ConfigWithTTLSeconds(ttlSeconds float64) string { return fmt.Sprintf("%s?ttlSecond=%.0f", Config, ttlSeconds) diff --git a/client/http/client.go b/client/http/client.go index 613ebf33294..958c52489fb 100644 --- a/client/http/client.go +++ b/client/http/client.go @@ -26,6 +26,7 @@ import ( "time" "github.com/pingcap/errors" + "github.com/pingcap/kvproto/pkg/pdpb" "github.com/pingcap/log" "github.com/prometheus/client_golang/prometheus" "go.uber.org/zap" @@ -54,9 +55,16 @@ type Client interface { GetHistoryHotRegions(context.Context, *HistoryHotRegionsRequest) (*HistoryHotRegions, error) GetRegionStatusByKeyRange(context.Context, *KeyRange, bool) (*RegionStats, error) GetStores(context.Context) (*StoresInfo, error) + SetStoreLabels(context.Context, int64, map[string]string) error + GetMembers(context.Context) (*MembersInfo, error) + GetLeader(context.Context) (*pdpb.Member, error) + TransferLeader(context.Context, string) error /* Config-related interfaces */ GetScheduleConfig(context.Context) (map[string]interface{}, error) SetScheduleConfig(context.Context, map[string]interface{}) error + /* Scheduler-related interfaces */ + GetSchedulers(context.Context) ([]string, error) + CreateScheduler(ctx context.Context, name string, storeID uint64) error /* Rule-related interfaces */ GetAllPlacementRuleBundles(context.Context) ([]*GroupBundle, error) GetPlacementRuleBundleByGroup(context.Context, string) (*GroupBundle, error) @@ -458,6 +466,44 @@ func (c *client) GetRegionStatusByKeyRange(ctx context.Context, keyRange *KeyRan return ®ionStats, nil } +// SetStoreLabels sets the labels of a store. +func (c *client) SetStoreLabels(ctx context.Context, storeID int64, storeLabels map[string]string) error { + jsonInput, err := json.Marshal(storeLabels) + if err != nil { + return errors.Trace(err) + } + return c.requestWithRetry(ctx, "SetStoreLabel", LabelByStoreID(storeID), + http.MethodPost, bytes.NewBuffer(jsonInput), nil) +} + +func (c *client) GetMembers(ctx context.Context) (*MembersInfo, error) { + var members MembersInfo + err := c.requestWithRetry(ctx, + "GetMembers", membersPrefix, + http.MethodGet, http.NoBody, &members) + if err != nil { + return nil, err + } + return &members, nil +} + +// GetLeader gets the leader of PD cluster. +func (c *client) GetLeader(ctx context.Context) (*pdpb.Member, error) { + var leader pdpb.Member + err := c.requestWithRetry(ctx, "GetLeader", leaderPrefix, + http.MethodGet, http.NoBody, &leader) + if err != nil { + return nil, err + } + return &leader, nil +} + +// TransferLeader transfers the PD leader. +func (c *client) TransferLeader(ctx context.Context, newLeader string) error { + return c.requestWithRetry(ctx, "TransferLeader", TransferLeaderByID(newLeader), + http.MethodPost, http.NoBody, nil) +} + // GetScheduleConfig gets the schedule configurations. func (c *client) GetScheduleConfig(ctx context.Context) (map[string]interface{}, error) { var config map[string]interface{} @@ -662,6 +708,31 @@ func (c *client) PatchRegionLabelRules(ctx context.Context, labelRulePatch *Labe http.MethodPatch, bytes.NewBuffer(labelRulePatchJSON), nil) } +// GetSchedulers gets the schedulers from PD cluster. +func (c *client) GetSchedulers(ctx context.Context) ([]string, error) { + var schedulers []string + err := c.requestWithRetry(ctx, "GetSchedulers", Schedulers, + http.MethodGet, http.NoBody, &schedulers) + if err != nil { + return nil, err + } + return schedulers, nil +} + +// CreateScheduler creates a scheduler to PD cluster. +func (c *client) CreateScheduler(ctx context.Context, name string, storeID uint64) error { + inputJSON, err := json.Marshal(map[string]interface{}{ + "name": name, + "store_id": storeID, + }) + if err != nil { + return errors.Trace(err) + } + return c.requestWithRetry(ctx, + "CreateScheduler", Schedulers, + http.MethodPost, bytes.NewBuffer(inputJSON), nil) +} + // AccelerateSchedule accelerates the scheduling of the regions within the given key range. // The keys in the key range should be encoded in the hex bytes format (without encoding to the UTF-8 bytes). func (c *client) AccelerateSchedule(ctx context.Context, keyRange *KeyRange) error { diff --git a/client/http/types.go b/client/http/types.go index 1d8db36d100..b05e8e0efba 100644 --- a/client/http/types.go +++ b/client/http/types.go @@ -21,6 +21,7 @@ import ( "time" "github.com/pingcap/kvproto/pkg/encryptionpb" + "github.com/pingcap/kvproto/pkg/pdpb" ) // KeyRange defines a range of keys in bytes. @@ -574,3 +575,12 @@ type LabelRulePatch struct { SetRules []*LabelRule `json:"sets"` DeleteRules []string `json:"deletes"` } + +// MembersInfo is PD members info returned from PD RESTful interface +// type Members map[string][]*pdpb.Member +type MembersInfo struct { + Header *pdpb.ResponseHeader `json:"header,omitempty"` + Members []*pdpb.Member `json:"members,omitempty"` + Leader *pdpb.Member `json:"leader,omitempty"` + EtcdLeader *pdpb.Member `json:"etcd_leader,omitempty"` +} diff --git a/tests/integrations/client/http_client_test.go b/tests/integrations/client/http_client_test.go index 6c636d2a2a1..476b4d2f541 100644 --- a/tests/integrations/client/http_client_test.go +++ b/tests/integrations/client/http_client_test.go @@ -49,7 +49,7 @@ func (suite *httpClientTestSuite) SetupSuite() { re := suite.Require() var err error suite.ctx, suite.cancelFunc = context.WithCancel(context.Background()) - suite.cluster, err = tests.NewTestCluster(suite.ctx, 1) + suite.cluster, err = tests.NewTestCluster(suite.ctx, 2) re.NoError(err) err = suite.cluster.RunInitialServers() re.NoError(err) @@ -384,3 +384,63 @@ func (suite *httpClientTestSuite) TestScheduleConfig() { re.Equal(float64(8), config["leader-schedule-limit"]) re.Equal(float64(2048), config["region-schedule-limit"]) } + +func (suite *httpClientTestSuite) TestSchedulers() { + re := suite.Require() + schedulers, err := suite.client.GetSchedulers(suite.ctx) + re.NoError(err) + re.Len(schedulers, 0) + + err = suite.client.CreateScheduler(suite.ctx, "evict-leader-scheduler", 1) + re.NoError(err) + schedulers, err = suite.client.GetSchedulers(suite.ctx) + re.NoError(err) + re.Len(schedulers, 1) +} + +func (suite *httpClientTestSuite) TestSetStoreLabels() { + re := suite.Require() + resp, err := suite.client.GetStores(suite.ctx) + re.NoError(err) + setStore := resp.Stores[0] + re.Empty(setStore.Store.Labels, nil) + storeLabels := map[string]string{ + "zone": "zone1", + } + err = suite.client.SetStoreLabels(suite.ctx, 1, storeLabels) + re.NoError(err) + + resp, err = suite.client.GetStores(suite.ctx) + re.NoError(err) + for _, store := range resp.Stores { + if store.Store.ID == setStore.Store.ID { + for _, label := range store.Store.Labels { + re.Equal(label.Value, storeLabels[label.Key]) + } + } + } +} + +func (suite *httpClientTestSuite) TestTransferLeader() { + re := suite.Require() + members, err := suite.client.GetMembers(suite.ctx) + re.NoError(err) + re.Len(members.Members, 2) + + oldLeader, err := suite.client.GetLeader(suite.ctx) + re.NoError(err) + + // Transfer leader to another pd + for _, member := range members.Members { + if member.Name != oldLeader.Name { + err = suite.client.TransferLeader(suite.ctx, member.Name) + re.NoError(err) + break + } + } + + newLeader := suite.cluster.WaitLeader() + re.NotEmpty(newLeader) + re.NoError(err) + re.NotEqual(oldLeader.Name, newLeader) +} From 859502b957be228b1be2e2cd6bebeb516c8f38bb Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Wed, 13 Dec 2023 16:16:49 +0800 Subject: [PATCH 089/137] mcs: fix panic when getting the member which is not started (#7540) close tikv/pd#7539 Signed-off-by: Ryan Leung Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- pkg/mcs/scheduling/server/server.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/mcs/scheduling/server/server.go b/pkg/mcs/scheduling/server/server.go index a43cbbebd86..8ee8b81ae47 100644 --- a/pkg/mcs/scheduling/server/server.go +++ b/pkg/mcs/scheduling/server/server.go @@ -192,6 +192,10 @@ func (s *Server) updateAPIServerMemberLoop() { continue } for _, ep := range members.Members { + if len(ep.GetClientURLs()) == 0 { // This member is not started yet. + log.Info("member is not started yet", zap.String("member-id", fmt.Sprintf("%x", ep.GetID())), errs.ZapError(err)) + continue + } status, err := s.GetClient().Status(ctx, ep.ClientURLs[0]) if err != nil { log.Info("failed to get status of member", zap.String("member-id", fmt.Sprintf("%x", ep.ID)), zap.String("endpoint", ep.ClientURLs[0]), errs.ZapError(err)) From cf718a799fcddda5413e6e8c8df0a14270b3d20e Mon Sep 17 00:00:00 2001 From: Yongbo Jiang Date: Wed, 13 Dec 2023 17:30:19 +0800 Subject: [PATCH 090/137] pkg/ratelimit: refactor for BBR (#7239) ref tikv/pd#7167 Signed-off-by: Cabinfever_B Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- pkg/ratelimit/controller.go | 79 ++++++ pkg/ratelimit/controller_test.go | 426 +++++++++++++++++++++++++++++++ pkg/ratelimit/limiter.go | 170 +++++++----- pkg/ratelimit/limiter_test.go | 164 ++++++------ pkg/ratelimit/option.go | 55 +--- server/api/middleware.go | 4 +- server/grpc_service.go | 38 +-- server/server.go | 12 +- 8 files changed, 723 insertions(+), 225 deletions(-) create mode 100644 pkg/ratelimit/controller.go create mode 100644 pkg/ratelimit/controller_test.go diff --git a/pkg/ratelimit/controller.go b/pkg/ratelimit/controller.go new file mode 100644 index 00000000000..0c95be9b11b --- /dev/null +++ b/pkg/ratelimit/controller.go @@ -0,0 +1,79 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ratelimit + +import ( + "sync" + + "golang.org/x/time/rate" +) + +var emptyFunc = func() {} + +// Controller is a controller which holds multiple limiters to manage the request rate of different objects. +type Controller struct { + limiters sync.Map + // the label which is in labelAllowList won't be limited, and only inited by hard code. + labelAllowList map[string]struct{} +} + +// NewController returns a global limiter which can be updated in the later. +func NewController() *Controller { + return &Controller{ + labelAllowList: make(map[string]struct{}), + } +} + +// Allow is used to check whether it has enough token. +func (l *Controller) Allow(label string) (DoneFunc, error) { + var ok bool + lim, ok := l.limiters.Load(label) + if ok { + return lim.(*limiter).allow() + } + return emptyFunc, nil +} + +// Update is used to update Ratelimiter with Options +func (l *Controller) Update(label string, opts ...Option) UpdateStatus { + var status UpdateStatus + for _, opt := range opts { + status |= opt(label, l) + } + return status +} + +// GetQPSLimiterStatus returns the status of a given label's QPS limiter. +func (l *Controller) GetQPSLimiterStatus(label string) (limit rate.Limit, burst int) { + if limit, exist := l.limiters.Load(label); exist { + return limit.(*limiter).getQPSLimiterStatus() + } + return 0, 0 +} + +// GetConcurrencyLimiterStatus returns the status of a given label's concurrency limiter. +func (l *Controller) GetConcurrencyLimiterStatus(label string) (limit uint64, current uint64) { + if limit, exist := l.limiters.Load(label); exist { + return limit.(*limiter).getConcurrencyLimiterStatus() + } + return 0, 0 +} + +// IsInAllowList returns whether this label is in allow list. +// If returns true, the given label won't be limited +func (l *Controller) IsInAllowList(label string) bool { + _, allow := l.labelAllowList[label] + return allow +} diff --git a/pkg/ratelimit/controller_test.go b/pkg/ratelimit/controller_test.go new file mode 100644 index 00000000000..a830217cb9f --- /dev/null +++ b/pkg/ratelimit/controller_test.go @@ -0,0 +1,426 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ratelimit + +import ( + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" + "github.com/tikv/pd/pkg/utils/syncutil" + "golang.org/x/time/rate" +) + +type changeAndResult struct { + opt Option + checkOptionStatus func(string, Option) + totalRequest int + success int + fail int + release int + waitDuration time.Duration + checkStatusFunc func(string) +} + +type labelCase struct { + label string + round []changeAndResult +} + +func runMulitLabelLimiter(t *testing.T, limiter *Controller, testCase []labelCase) { + re := require.New(t) + var caseWG sync.WaitGroup + for _, tempCas := range testCase { + caseWG.Add(1) + cas := tempCas + go func() { + var lock syncutil.Mutex + successCount, failedCount := 0, 0 + var wg sync.WaitGroup + r := &releaseUtil{} + for _, rd := range cas.round { + rd.checkOptionStatus(cas.label, rd.opt) + time.Sleep(rd.waitDuration) + for i := 0; i < rd.totalRequest; i++ { + wg.Add(1) + go func() { + countRateLimiterHandleResult(limiter, cas.label, &successCount, &failedCount, &lock, &wg, r) + }() + } + wg.Wait() + re.Equal(rd.fail, failedCount) + re.Equal(rd.success, successCount) + for i := 0; i < rd.release; i++ { + r.release() + } + rd.checkStatusFunc(cas.label) + failedCount -= rd.fail + successCount -= rd.success + } + caseWG.Done() + }() + } + caseWG.Wait() +} + +func TestControllerWithConcurrencyLimiter(t *testing.T) { + t.Parallel() + re := require.New(t) + limiter := NewController() + testCase := []labelCase{ + { + label: "test1", + round: []changeAndResult{ + { + opt: UpdateConcurrencyLimiter(10), + checkOptionStatus: func(label string, o Option) { + status := limiter.Update(label, o) + re.True(status&ConcurrencyChanged != 0) + }, + totalRequest: 15, + fail: 5, + success: 10, + release: 10, + waitDuration: 0, + checkStatusFunc: func(label string) { + limit, current := limiter.GetConcurrencyLimiterStatus(label) + re.Equal(uint64(10), limit) + re.Equal(uint64(0), current) + }, + }, + { + opt: UpdateConcurrencyLimiter(10), + checkOptionStatus: func(label string, o Option) { + status := limiter.Update(label, o) + re.True(status&ConcurrencyNoChange != 0) + }, + checkStatusFunc: func(label string) {}, + }, + { + opt: UpdateConcurrencyLimiter(5), + checkOptionStatus: func(label string, o Option) { + status := limiter.Update(label, o) + re.True(status&ConcurrencyChanged != 0) + }, + totalRequest: 15, + fail: 10, + success: 5, + release: 5, + waitDuration: 0, + checkStatusFunc: func(label string) { + limit, current := limiter.GetConcurrencyLimiterStatus(label) + re.Equal(uint64(5), limit) + re.Equal(uint64(0), current) + }, + }, + { + opt: UpdateConcurrencyLimiter(0), + checkOptionStatus: func(label string, o Option) { + status := limiter.Update(label, o) + re.True(status&ConcurrencyDeleted != 0) + }, + totalRequest: 15, + fail: 0, + success: 15, + release: 5, + waitDuration: 0, + checkStatusFunc: func(label string) { + limit, current := limiter.GetConcurrencyLimiterStatus(label) + re.Equal(uint64(0), limit) + re.Equal(uint64(0), current) + }, + }, + }, + }, + { + label: "test2", + round: []changeAndResult{ + { + opt: UpdateConcurrencyLimiter(15), + checkOptionStatus: func(label string, o Option) { + status := limiter.Update(label, o) + re.True(status&ConcurrencyChanged != 0) + }, + totalRequest: 10, + fail: 0, + success: 10, + release: 0, + waitDuration: 0, + checkStatusFunc: func(label string) { + limit, current := limiter.GetConcurrencyLimiterStatus(label) + re.Equal(uint64(15), limit) + re.Equal(uint64(10), current) + }, + }, + { + opt: UpdateConcurrencyLimiter(10), + checkOptionStatus: func(label string, o Option) { + status := limiter.Update(label, o) + re.True(status&ConcurrencyChanged != 0) + }, + totalRequest: 10, + fail: 10, + success: 0, + release: 10, + waitDuration: 0, + checkStatusFunc: func(label string) { + limit, current := limiter.GetConcurrencyLimiterStatus(label) + re.Equal(uint64(10), limit) + re.Equal(uint64(0), current) + }, + }, + }, + }, + } + runMulitLabelLimiter(t, limiter, testCase) +} + +func TestBlockList(t *testing.T) { + t.Parallel() + re := require.New(t) + opts := []Option{AddLabelAllowList()} + limiter := NewController() + label := "test" + + re.False(limiter.IsInAllowList(label)) + for _, opt := range opts { + opt(label, limiter) + } + re.True(limiter.IsInAllowList(label)) + + status := UpdateQPSLimiter(float64(rate.Every(time.Second)), 1)(label, limiter) + re.True(status&InAllowList != 0) + for i := 0; i < 10; i++ { + _, err := limiter.Allow(label) + re.NoError(err) + } +} + +func TestControllerWithQPSLimiter(t *testing.T) { + t.Parallel() + re := require.New(t) + limiter := NewController() + testCase := []labelCase{ + { + label: "test1", + round: []changeAndResult{ + { + opt: UpdateQPSLimiter(float64(rate.Every(time.Second)), 1), + checkOptionStatus: func(label string, o Option) { + status := limiter.Update(label, o) + re.True(status&QPSChanged != 0) + }, + totalRequest: 3, + fail: 2, + success: 1, + waitDuration: 0, + checkStatusFunc: func(label string) { + limit, burst := limiter.GetQPSLimiterStatus(label) + re.Equal(rate.Limit(1), limit) + re.Equal(1, burst) + }, + }, + { + opt: UpdateQPSLimiter(float64(rate.Every(time.Second)), 1), + checkOptionStatus: func(label string, o Option) { + status := limiter.Update(label, o) + re.True(status&QPSNoChange != 0) + }, + checkStatusFunc: func(label string) {}, + }, + { + opt: UpdateQPSLimiter(5, 5), + checkOptionStatus: func(label string, o Option) { + status := limiter.Update(label, o) + re.True(status&QPSChanged != 0) + }, + totalRequest: 10, + fail: 5, + success: 5, + waitDuration: time.Second, + checkStatusFunc: func(label string) { + limit, burst := limiter.GetQPSLimiterStatus(label) + re.Equal(rate.Limit(5), limit) + re.Equal(5, burst) + }, + }, + { + opt: UpdateQPSLimiter(0, 0), + checkOptionStatus: func(label string, o Option) { + status := limiter.Update(label, o) + re.True(status&QPSDeleted != 0) + }, + totalRequest: 10, + fail: 0, + success: 10, + release: 0, + waitDuration: 0, + checkStatusFunc: func(label string) { + limit, burst := limiter.GetQPSLimiterStatus(label) + re.Equal(rate.Limit(0), limit) + re.Equal(0, burst) + }, + }, + }, + }, + { + label: "test2", + round: []changeAndResult{ + { + opt: UpdateQPSLimiter(50, 5), + checkOptionStatus: func(label string, o Option) { + status := limiter.Update(label, o) + re.True(status&QPSChanged != 0) + }, + totalRequest: 10, + fail: 5, + success: 5, + waitDuration: time.Second, + checkStatusFunc: func(label string) { + limit, burst := limiter.GetQPSLimiterStatus(label) + re.Equal(rate.Limit(50), limit) + re.Equal(5, burst) + }, + }, + { + opt: UpdateQPSLimiter(0, 0), + checkOptionStatus: func(label string, o Option) { + status := limiter.Update(label, o) + re.True(status&QPSDeleted != 0) + }, + totalRequest: 10, + fail: 0, + success: 10, + release: 0, + waitDuration: 0, + checkStatusFunc: func(label string) { + limit, burst := limiter.GetQPSLimiterStatus(label) + re.Equal(rate.Limit(0), limit) + re.Equal(0, burst) + }, + }, + }, + }, + } + runMulitLabelLimiter(t, limiter, testCase) +} + +func TestControllerWithTwoLimiters(t *testing.T) { + t.Parallel() + re := require.New(t) + limiter := NewController() + testCase := []labelCase{ + { + label: "test1", + round: []changeAndResult{ + { + opt: UpdateDimensionConfig(&DimensionConfig{ + QPS: 100, + QPSBurst: 100, + ConcurrencyLimit: 100, + }), + checkOptionStatus: func(label string, o Option) { + status := limiter.Update(label, o) + re.True(status&QPSChanged != 0) + }, + totalRequest: 200, + fail: 100, + success: 100, + release: 100, + waitDuration: time.Second, + checkStatusFunc: func(label string) { + limit, burst := limiter.GetQPSLimiterStatus(label) + re.Equal(rate.Limit(100), limit) + re.Equal(100, burst) + climit, current := limiter.GetConcurrencyLimiterStatus(label) + re.Equal(uint64(100), climit) + re.Equal(uint64(0), current) + }, + }, + { + opt: UpdateQPSLimiter(float64(rate.Every(time.Second)), 1), + checkOptionStatus: func(label string, o Option) { + status := limiter.Update(label, o) + re.True(status&QPSChanged != 0) + }, + totalRequest: 200, + fail: 199, + success: 1, + release: 0, + waitDuration: time.Second, + checkStatusFunc: func(label string) { + limit, current := limiter.GetConcurrencyLimiterStatus(label) + re.Equal(uint64(100), limit) + re.Equal(uint64(1), current) + }, + }, + }, + }, + { + label: "test2", + round: []changeAndResult{ + { + opt: UpdateQPSLimiter(50, 5), + checkOptionStatus: func(label string, o Option) { + status := limiter.Update(label, o) + re.True(status&QPSChanged != 0) + }, + totalRequest: 10, + fail: 5, + success: 5, + waitDuration: time.Second, + checkStatusFunc: func(label string) { + limit, burst := limiter.GetQPSLimiterStatus(label) + re.Equal(rate.Limit(50), limit) + re.Equal(5, burst) + }, + }, + { + opt: UpdateQPSLimiter(0, 0), + checkOptionStatus: func(label string, o Option) { + status := limiter.Update(label, o) + re.True(status&QPSDeleted != 0) + }, + totalRequest: 10, + fail: 0, + success: 10, + release: 0, + waitDuration: 0, + checkStatusFunc: func(label string) { + limit, burst := limiter.GetQPSLimiterStatus(label) + re.Equal(rate.Limit(0), limit) + re.Equal(0, burst) + }, + }, + }, + }, + } + runMulitLabelLimiter(t, limiter, testCase) +} + +func countRateLimiterHandleResult(limiter *Controller, label string, successCount *int, + failedCount *int, lock *syncutil.Mutex, wg *sync.WaitGroup, r *releaseUtil) { + doneFucn, err := limiter.Allow(label) + lock.Lock() + defer lock.Unlock() + if err == nil { + *successCount++ + r.append(doneFucn) + } else { + *failedCount++ + } + wg.Done() +} diff --git a/pkg/ratelimit/limiter.go b/pkg/ratelimit/limiter.go index 4bf930ed6c5..444b5aa2481 100644 --- a/pkg/ratelimit/limiter.go +++ b/pkg/ratelimit/limiter.go @@ -1,4 +1,4 @@ -// Copyright 2022 TiKV Project Authors. +// Copyright 2023 TiKV Project Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,11 +15,16 @@ package ratelimit import ( - "sync" + "math" + "github.com/tikv/pd/pkg/errs" + "github.com/tikv/pd/pkg/utils/syncutil" "golang.org/x/time/rate" ) +// DoneFunc is done function. +type DoneFunc func() + // DimensionConfig is the limit dimension config of one label type DimensionConfig struct { // qps conifg @@ -29,92 +34,125 @@ type DimensionConfig struct { ConcurrencyLimit uint64 } -// Limiter is a controller for the request rate. -type Limiter struct { - qpsLimiter sync.Map - concurrencyLimiter sync.Map - // the label which is in labelAllowList won't be limited - labelAllowList map[string]struct{} +type limiter struct { + mu syncutil.RWMutex + concurrency *concurrencyLimiter + rate *RateLimiter } -// NewLimiter returns a global limiter which can be updated in the later. -func NewLimiter() *Limiter { - return &Limiter{ - labelAllowList: make(map[string]struct{}), - } +func newLimiter() *limiter { + lim := &limiter{} + return lim } -// Allow is used to check whether it has enough token. -func (l *Limiter) Allow(label string) bool { - var cl *concurrencyLimiter - var ok bool - if limiter, exist := l.concurrencyLimiter.Load(label); exist { - if cl, ok = limiter.(*concurrencyLimiter); ok && !cl.allow() { - return false - } - } +func (l *limiter) getConcurrencyLimiter() *concurrencyLimiter { + l.mu.RLock() + defer l.mu.RUnlock() + return l.concurrency +} - if limiter, exist := l.qpsLimiter.Load(label); exist { - if ql, ok := limiter.(*RateLimiter); ok && !ql.Allow() { - if cl != nil { - cl.release() - } - return false - } - } +func (l *limiter) getRateLimiter() *RateLimiter { + l.mu.RLock() + defer l.mu.RUnlock() + return l.rate +} - return true +func (l *limiter) deleteRateLimiter() bool { + l.mu.Lock() + defer l.mu.Unlock() + l.rate = nil + return l.isEmpty() } -// Release is used to refill token. It may be not uesful for some limiters because they will refill automatically -func (l *Limiter) Release(label string) { - if limiter, exist := l.concurrencyLimiter.Load(label); exist { - if cl, ok := limiter.(*concurrencyLimiter); ok { - cl.release() - } - } +func (l *limiter) deleteConcurrency() bool { + l.mu.Lock() + defer l.mu.Unlock() + l.concurrency = nil + return l.isEmpty() } -// Update is used to update Ratelimiter with Options -func (l *Limiter) Update(label string, opts ...Option) UpdateStatus { - var status UpdateStatus - for _, opt := range opts { - status |= opt(label, l) - } - return status +func (l *limiter) isEmpty() bool { + return l.concurrency == nil && l.rate == nil } -// GetQPSLimiterStatus returns the status of a given label's QPS limiter. -func (l *Limiter) GetQPSLimiterStatus(label string) (limit rate.Limit, burst int) { - if limiter, exist := l.qpsLimiter.Load(label); exist { - return limiter.(*RateLimiter).Limit(), limiter.(*RateLimiter).Burst() +func (l *limiter) getQPSLimiterStatus() (limit rate.Limit, burst int) { + baseLimiter := l.getRateLimiter() + if baseLimiter != nil { + return baseLimiter.Limit(), baseLimiter.Burst() } - return 0, 0 } -// QPSUnlimit deletes QPS limiter of the given label -func (l *Limiter) QPSUnlimit(label string) { - l.qpsLimiter.Delete(label) +func (l *limiter) getConcurrencyLimiterStatus() (limit uint64, current uint64) { + baseLimiter := l.getConcurrencyLimiter() + if baseLimiter != nil { + return baseLimiter.getLimit(), baseLimiter.getCurrent() + } + return 0, 0 } -// GetConcurrencyLimiterStatus returns the status of a given label's concurrency limiter. -func (l *Limiter) GetConcurrencyLimiterStatus(label string) (limit uint64, current uint64) { - if limiter, exist := l.concurrencyLimiter.Load(label); exist { - return limiter.(*concurrencyLimiter).getLimit(), limiter.(*concurrencyLimiter).getCurrent() +func (l *limiter) updateConcurrencyConfig(limit uint64) UpdateStatus { + oldConcurrencyLimit, _ := l.getConcurrencyLimiterStatus() + if oldConcurrencyLimit == limit { + return ConcurrencyNoChange + } + if limit < 1 { + l.deleteConcurrency() + return ConcurrencyDeleted } - return 0, 0 + l.mu.Lock() + defer l.mu.Unlock() + if l.concurrency != nil { + l.concurrency.setLimit(limit) + } else { + l.concurrency = newConcurrencyLimiter(limit) + } + return ConcurrencyChanged +} + +func (l *limiter) updateQPSConfig(limit float64, burst int) UpdateStatus { + oldQPSLimit, oldBurst := l.getQPSLimiterStatus() + if math.Abs(float64(oldQPSLimit)-limit) < eps && oldBurst == burst { + return QPSNoChange + } + if limit <= eps || burst < 1 { + l.deleteRateLimiter() + return QPSDeleted + } + l.mu.Lock() + defer l.mu.Unlock() + if l.rate != nil { + l.rate.SetLimit(rate.Limit(limit)) + l.rate.SetBurst(burst) + } else { + l.rate = NewRateLimiter(limit, burst) + } + return QPSChanged } -// ConcurrencyUnlimit deletes concurrency limiter of the given label -func (l *Limiter) ConcurrencyUnlimit(label string) { - l.concurrencyLimiter.Delete(label) +func (l *limiter) updateDimensionConfig(cfg *DimensionConfig) UpdateStatus { + status := l.updateQPSConfig(cfg.QPS, cfg.QPSBurst) + status |= l.updateConcurrencyConfig(cfg.ConcurrencyLimit) + return status } -// IsInAllowList returns whether this label is in allow list. -// If returns true, the given label won't be limited -func (l *Limiter) IsInAllowList(label string) bool { - _, allow := l.labelAllowList[label] - return allow +func (l *limiter) allow() (DoneFunc, error) { + concurrency := l.getConcurrencyLimiter() + if concurrency != nil && !concurrency.allow() { + return nil, errs.ErrRateLimitExceeded + } + + rate := l.getRateLimiter() + if rate != nil && !rate.Allow() { + if concurrency != nil { + concurrency.release() + } + return nil, errs.ErrRateLimitExceeded + } + return func() { + if concurrency != nil { + concurrency.release() + } + }, nil } diff --git a/pkg/ratelimit/limiter_test.go b/pkg/ratelimit/limiter_test.go index d5d9829816a..8834495f3e9 100644 --- a/pkg/ratelimit/limiter_test.go +++ b/pkg/ratelimit/limiter_test.go @@ -1,4 +1,4 @@ -// Copyright 2022 TiKV Project Authors. +// Copyright 2023 TiKV Project Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -24,162 +24,145 @@ import ( "golang.org/x/time/rate" ) -func TestUpdateConcurrencyLimiter(t *testing.T) { +type releaseUtil struct { + dones []DoneFunc +} + +func (r *releaseUtil) release() { + if len(r.dones) > 0 { + r.dones[0]() + r.dones = r.dones[1:] + } +} + +func (r *releaseUtil) append(d DoneFunc) { + r.dones = append(r.dones, d) +} + +func TestWithConcurrencyLimiter(t *testing.T) { t.Parallel() re := require.New(t) - opts := []Option{UpdateConcurrencyLimiter(10)} - limiter := NewLimiter() - - label := "test" - status := limiter.Update(label, opts...) + limiter := newLimiter() + status := limiter.updateConcurrencyConfig(10) re.True(status&ConcurrencyChanged != 0) var lock syncutil.Mutex successCount, failedCount := 0, 0 var wg sync.WaitGroup + r := &releaseUtil{} for i := 0; i < 15; i++ { wg.Add(1) go func() { - countRateLimiterHandleResult(limiter, label, &successCount, &failedCount, &lock, &wg) + countSingleLimiterHandleResult(limiter, &successCount, &failedCount, &lock, &wg, r) }() } wg.Wait() re.Equal(5, failedCount) re.Equal(10, successCount) for i := 0; i < 10; i++ { - limiter.Release(label) + r.release() } - limit, current := limiter.GetConcurrencyLimiterStatus(label) + limit, current := limiter.getConcurrencyLimiterStatus() re.Equal(uint64(10), limit) re.Equal(uint64(0), current) - status = limiter.Update(label, UpdateConcurrencyLimiter(10)) + status = limiter.updateConcurrencyConfig(10) re.True(status&ConcurrencyNoChange != 0) - status = limiter.Update(label, UpdateConcurrencyLimiter(5)) + status = limiter.updateConcurrencyConfig(5) re.True(status&ConcurrencyChanged != 0) failedCount = 0 successCount = 0 for i := 0; i < 15; i++ { wg.Add(1) - go countRateLimiterHandleResult(limiter, label, &successCount, &failedCount, &lock, &wg) + go countSingleLimiterHandleResult(limiter, &successCount, &failedCount, &lock, &wg, r) } wg.Wait() re.Equal(10, failedCount) re.Equal(5, successCount) for i := 0; i < 5; i++ { - limiter.Release(label) + r.release() } - status = limiter.Update(label, UpdateConcurrencyLimiter(0)) + status = limiter.updateConcurrencyConfig(0) re.True(status&ConcurrencyDeleted != 0) failedCount = 0 successCount = 0 for i := 0; i < 15; i++ { wg.Add(1) - go countRateLimiterHandleResult(limiter, label, &successCount, &failedCount, &lock, &wg) + go countSingleLimiterHandleResult(limiter, &successCount, &failedCount, &lock, &wg, r) } wg.Wait() re.Equal(0, failedCount) re.Equal(15, successCount) - limit, current = limiter.GetConcurrencyLimiterStatus(label) + limit, current = limiter.getConcurrencyLimiterStatus() re.Equal(uint64(0), limit) re.Equal(uint64(0), current) } -func TestBlockList(t *testing.T) { +func TestWithQPSLimiter(t *testing.T) { t.Parallel() re := require.New(t) - opts := []Option{AddLabelAllowList()} - limiter := NewLimiter() - label := "test" - - re.False(limiter.IsInAllowList(label)) - for _, opt := range opts { - opt(label, limiter) - } - re.True(limiter.IsInAllowList(label)) - - status := UpdateQPSLimiter(float64(rate.Every(time.Second)), 1)(label, limiter) - re.True(status&InAllowList != 0) - for i := 0; i < 10; i++ { - re.True(limiter.Allow(label)) - } -} - -func TestUpdateQPSLimiter(t *testing.T) { - t.Parallel() - re := require.New(t) - opts := []Option{UpdateQPSLimiter(float64(rate.Every(time.Second)), 1)} - limiter := NewLimiter() - - label := "test" - status := limiter.Update(label, opts...) + limiter := newLimiter() + status := limiter.updateQPSConfig(float64(rate.Every(time.Second)), 1) re.True(status&QPSChanged != 0) var lock syncutil.Mutex successCount, failedCount := 0, 0 var wg sync.WaitGroup + r := &releaseUtil{} wg.Add(3) for i := 0; i < 3; i++ { - go countRateLimiterHandleResult(limiter, label, &successCount, &failedCount, &lock, &wg) + go countSingleLimiterHandleResult(limiter, &successCount, &failedCount, &lock, &wg, r) } wg.Wait() re.Equal(2, failedCount) re.Equal(1, successCount) - limit, burst := limiter.GetQPSLimiterStatus(label) + limit, burst := limiter.getQPSLimiterStatus() re.Equal(rate.Limit(1), limit) re.Equal(1, burst) - status = limiter.Update(label, UpdateQPSLimiter(float64(rate.Every(time.Second)), 1)) + status = limiter.updateQPSConfig(float64(rate.Every(time.Second)), 1) re.True(status&QPSNoChange != 0) - status = limiter.Update(label, UpdateQPSLimiter(5, 5)) + status = limiter.updateQPSConfig(5, 5) re.True(status&QPSChanged != 0) - limit, burst = limiter.GetQPSLimiterStatus(label) + limit, burst = limiter.getQPSLimiterStatus() re.Equal(rate.Limit(5), limit) re.Equal(5, burst) time.Sleep(time.Second) for i := 0; i < 10; i++ { if i < 5 { - re.True(limiter.Allow(label)) + _, err := limiter.allow() + re.NoError(err) } else { - re.False(limiter.Allow(label)) + _, err := limiter.allow() + re.Error(err) } } time.Sleep(time.Second) - status = limiter.Update(label, UpdateQPSLimiter(0, 0)) + status = limiter.updateQPSConfig(0, 0) re.True(status&QPSDeleted != 0) for i := 0; i < 10; i++ { - re.True(limiter.Allow(label)) + _, err := limiter.allow() + re.NoError(err) } - qLimit, qCurrent := limiter.GetQPSLimiterStatus(label) + qLimit, qCurrent := limiter.getQPSLimiterStatus() re.Equal(rate.Limit(0), qLimit) re.Equal(0, qCurrent) -} -func TestQPSLimiter(t *testing.T) { - t.Parallel() - re := require.New(t) - opts := []Option{UpdateQPSLimiter(float64(rate.Every(3*time.Second)), 100)} - limiter := NewLimiter() - - label := "test" - for _, opt := range opts { - opt(label, limiter) - } - - var lock syncutil.Mutex - successCount, failedCount := 0, 0 - var wg sync.WaitGroup + successCount = 0 + failedCount = 0 + status = limiter.updateQPSConfig(float64(rate.Every(3*time.Second)), 100) + re.True(status&QPSChanged != 0) wg.Add(200) for i := 0; i < 200; i++ { - go countRateLimiterHandleResult(limiter, label, &successCount, &failedCount, &lock, &wg) + go countSingleLimiterHandleResult(limiter, &successCount, &failedCount, &lock, &wg, r) } wg.Wait() re.Equal(200, failedCount+successCount) @@ -188,12 +171,12 @@ func TestQPSLimiter(t *testing.T) { time.Sleep(4 * time.Second) // 3+1 wg.Add(1) - countRateLimiterHandleResult(limiter, label, &successCount, &failedCount, &lock, &wg) + countSingleLimiterHandleResult(limiter, &successCount, &failedCount, &lock, &wg, r) wg.Wait() re.Equal(101, successCount) } -func TestTwoLimiters(t *testing.T) { +func TestWithTwoLimiters(t *testing.T) { t.Parallel() re := require.New(t) cfg := &DimensionConfig{ @@ -201,20 +184,18 @@ func TestTwoLimiters(t *testing.T) { QPSBurst: 100, ConcurrencyLimit: 100, } - opts := []Option{UpdateDimensionConfig(cfg)} - limiter := NewLimiter() - - label := "test" - for _, opt := range opts { - opt(label, limiter) - } + limiter := newLimiter() + status := limiter.updateDimensionConfig(cfg) + re.True(status&QPSChanged != 0) + re.True(status&ConcurrencyChanged != 0) var lock syncutil.Mutex successCount, failedCount := 0, 0 var wg sync.WaitGroup + r := &releaseUtil{} wg.Add(200) for i := 0; i < 200; i++ { - go countRateLimiterHandleResult(limiter, label, &successCount, &failedCount, &lock, &wg) + go countSingleLimiterHandleResult(limiter, &successCount, &failedCount, &lock, &wg, r) } wg.Wait() re.Equal(100, failedCount) @@ -223,35 +204,42 @@ func TestTwoLimiters(t *testing.T) { wg.Add(100) for i := 0; i < 100; i++ { - go countRateLimiterHandleResult(limiter, label, &successCount, &failedCount, &lock, &wg) + go countSingleLimiterHandleResult(limiter, &successCount, &failedCount, &lock, &wg, r) } wg.Wait() re.Equal(200, failedCount) re.Equal(100, successCount) for i := 0; i < 100; i++ { - limiter.Release(label) + r.release() } - limiter.Update(label, UpdateQPSLimiter(float64(rate.Every(10*time.Second)), 1)) + status = limiter.updateQPSConfig(float64(rate.Every(10*time.Second)), 1) + re.True(status&QPSChanged != 0) wg.Add(100) for i := 0; i < 100; i++ { - go countRateLimiterHandleResult(limiter, label, &successCount, &failedCount, &lock, &wg) + go countSingleLimiterHandleResult(limiter, &successCount, &failedCount, &lock, &wg, r) } wg.Wait() re.Equal(101, successCount) re.Equal(299, failedCount) - limit, current := limiter.GetConcurrencyLimiterStatus(label) + limit, current := limiter.getConcurrencyLimiterStatus() re.Equal(uint64(100), limit) re.Equal(uint64(1), current) + + cfg = &DimensionConfig{} + status = limiter.updateDimensionConfig(cfg) + re.True(status&ConcurrencyDeleted != 0) + re.True(status&QPSDeleted != 0) } -func countRateLimiterHandleResult(limiter *Limiter, label string, successCount *int, - failedCount *int, lock *syncutil.Mutex, wg *sync.WaitGroup) { - result := limiter.Allow(label) +func countSingleLimiterHandleResult(limiter *limiter, successCount *int, + failedCount *int, lock *syncutil.Mutex, wg *sync.WaitGroup, r *releaseUtil) { + doneFucn, err := limiter.allow() lock.Lock() defer lock.Unlock() - if result { + if err == nil { *successCount++ + r.append(doneFucn) } else { *failedCount++ } diff --git a/pkg/ratelimit/option.go b/pkg/ratelimit/option.go index 53afb9926d4..b1cc459d786 100644 --- a/pkg/ratelimit/option.go +++ b/pkg/ratelimit/option.go @@ -14,8 +14,6 @@ package ratelimit -import "golang.org/x/time/rate" - // UpdateStatus is flags for updating limiter config. type UpdateStatus uint32 @@ -40,77 +38,46 @@ const ( // Option is used to create a limiter with the optional settings. // these setting is used to add a kind of limiter for a service -type Option func(string, *Limiter) UpdateStatus +type Option func(string, *Controller) UpdateStatus // AddLabelAllowList adds a label into allow list. // It means the given label will not be limited func AddLabelAllowList() Option { - return func(label string, l *Limiter) UpdateStatus { + return func(label string, l *Controller) UpdateStatus { l.labelAllowList[label] = struct{}{} return 0 } } -func updateConcurrencyConfig(l *Limiter, label string, limit uint64) UpdateStatus { - oldConcurrencyLimit, _ := l.GetConcurrencyLimiterStatus(label) - if oldConcurrencyLimit == limit { - return ConcurrencyNoChange - } - if limit < 1 { - l.ConcurrencyUnlimit(label) - return ConcurrencyDeleted - } - if limiter, exist := l.concurrencyLimiter.LoadOrStore(label, newConcurrencyLimiter(limit)); exist { - limiter.(*concurrencyLimiter).setLimit(limit) - } - return ConcurrencyChanged -} - -func updateQPSConfig(l *Limiter, label string, limit float64, burst int) UpdateStatus { - oldQPSLimit, oldBurst := l.GetQPSLimiterStatus(label) - - if (float64(oldQPSLimit)-limit < eps && float64(oldQPSLimit)-limit > -eps) && oldBurst == burst { - return QPSNoChange - } - if limit <= eps || burst < 1 { - l.QPSUnlimit(label) - return QPSDeleted - } - if limiter, exist := l.qpsLimiter.LoadOrStore(label, NewRateLimiter(limit, burst)); exist { - limiter.(*RateLimiter).SetLimit(rate.Limit(limit)) - limiter.(*RateLimiter).SetBurst(burst) - } - return QPSChanged -} - // UpdateConcurrencyLimiter creates a concurrency limiter for a given label if it doesn't exist. func UpdateConcurrencyLimiter(limit uint64) Option { - return func(label string, l *Limiter) UpdateStatus { + return func(label string, l *Controller) UpdateStatus { if _, allow := l.labelAllowList[label]; allow { return InAllowList } - return updateConcurrencyConfig(l, label, limit) + lim, _ := l.limiters.LoadOrStore(label, newLimiter()) + return lim.(*limiter).updateConcurrencyConfig(limit) } } // UpdateQPSLimiter creates a QPS limiter for a given label if it doesn't exist. func UpdateQPSLimiter(limit float64, burst int) Option { - return func(label string, l *Limiter) UpdateStatus { + return func(label string, l *Controller) UpdateStatus { if _, allow := l.labelAllowList[label]; allow { return InAllowList } - return updateQPSConfig(l, label, limit, burst) + lim, _ := l.limiters.LoadOrStore(label, newLimiter()) + return lim.(*limiter).updateQPSConfig(limit, burst) } } // UpdateDimensionConfig creates QPS limiter and concurrency limiter for a given label by config if it doesn't exist. func UpdateDimensionConfig(cfg *DimensionConfig) Option { - return func(label string, l *Limiter) UpdateStatus { + return func(label string, l *Controller) UpdateStatus { if _, allow := l.labelAllowList[label]; allow { return InAllowList } - status := updateQPSConfig(l, label, cfg.QPS, cfg.QPSBurst) - status |= updateConcurrencyConfig(l, label, cfg.ConcurrencyLimit) - return status + lim, _ := l.limiters.LoadOrStore(label, newLimiter()) + return lim.(*limiter).updateDimensionConfig(cfg) } } diff --git a/server/api/middleware.go b/server/api/middleware.go index 4173c37b396..6536935592f 100644 --- a/server/api/middleware.go +++ b/server/api/middleware.go @@ -177,8 +177,8 @@ func (s *rateLimitMiddleware) ServeHTTP(w http.ResponseWriter, r *http.Request, // There is no need to check whether rateLimiter is nil. CreateServer ensures that it is created rateLimiter := s.svr.GetServiceRateLimiter() - if rateLimiter.Allow(requestInfo.ServiceLabel) { - defer rateLimiter.Release(requestInfo.ServiceLabel) + if done, err := rateLimiter.Allow(requestInfo.ServiceLabel); err == nil { + defer done() next(w, r) } else { http.Error(w, http.StatusText(http.StatusTooManyRequests), http.StatusTooManyRequests) diff --git a/server/grpc_service.go b/server/grpc_service.go index fa74f1ea8b6..24280f46437 100644 --- a/server/grpc_service.go +++ b/server/grpc_service.go @@ -431,11 +431,11 @@ func (s *GrpcServer) GetMembers(context.Context, *pdpb.GetMembersRequest) (*pdpb if s.GetServiceMiddlewarePersistOptions().IsGRPCRateLimitEnabled() { fName := currentFunction() limiter := s.GetGRPCRateLimiter() - if limiter.Allow(fName) { - defer limiter.Release(fName) + if done, err := limiter.Allow(fName); err == nil { + defer done() } else { return &pdpb.GetMembersResponse{ - Header: s.wrapErrorToHeader(pdpb.ErrorType_UNKNOWN, errs.ErrRateLimitExceeded.FastGenByArgs().Error()), + Header: s.wrapErrorToHeader(pdpb.ErrorType_UNKNOWN, err.Error()), }, nil } } @@ -662,11 +662,11 @@ func (s *GrpcServer) GetStore(ctx context.Context, request *pdpb.GetStoreRequest if s.GetServiceMiddlewarePersistOptions().IsGRPCRateLimitEnabled() { fName := currentFunction() limiter := s.GetGRPCRateLimiter() - if limiter.Allow(fName) { - defer limiter.Release(fName) + if done, err := limiter.Allow(fName); err == nil { + defer done() } else { return &pdpb.GetStoreResponse{ - Header: s.wrapErrorToHeader(pdpb.ErrorType_UNKNOWN, errs.ErrRateLimitExceeded.FastGenByArgs().Error()), + Header: s.wrapErrorToHeader(pdpb.ErrorType_UNKNOWN, err.Error()), }, nil } } @@ -765,11 +765,11 @@ func (s *GrpcServer) GetAllStores(ctx context.Context, request *pdpb.GetAllStore if s.GetServiceMiddlewarePersistOptions().IsGRPCRateLimitEnabled() { fName := currentFunction() limiter := s.GetGRPCRateLimiter() - if limiter.Allow(fName) { - defer limiter.Release(fName) + if done, err := limiter.Allow(fName); err == nil { + defer done() } else { return &pdpb.GetAllStoresResponse{ - Header: s.wrapErrorToHeader(pdpb.ErrorType_UNKNOWN, errs.ErrRateLimitExceeded.FastGenByArgs().Error()), + Header: s.wrapErrorToHeader(pdpb.ErrorType_UNKNOWN, err.Error()), }, nil } } @@ -810,8 +810,8 @@ func (s *GrpcServer) StoreHeartbeat(ctx context.Context, request *pdpb.StoreHear if s.GetServiceMiddlewarePersistOptions().IsGRPCRateLimitEnabled() { fName := currentFunction() limiter := s.GetGRPCRateLimiter() - if limiter.Allow(fName) { - defer limiter.Release(fName) + if done, err := limiter.Allow(fName); err == nil { + defer done() } else { return &pdpb.StoreHeartbeatResponse{ Header: s.wrapErrorToHeader(pdpb.ErrorType_UNKNOWN, errs.ErrRateLimitExceeded.FastGenByArgs().Error()), @@ -1286,8 +1286,8 @@ func (s *GrpcServer) GetRegion(ctx context.Context, request *pdpb.GetRegionReque if s.GetServiceMiddlewarePersistOptions().IsGRPCRateLimitEnabled() { fName := currentFunction() limiter := s.GetGRPCRateLimiter() - if limiter.Allow(fName) { - defer limiter.Release(fName) + if done, err := limiter.Allow(fName); err == nil { + defer done() } else { return &pdpb.GetRegionResponse{ Header: s.wrapErrorToHeader(pdpb.ErrorType_UNKNOWN, errs.ErrRateLimitExceeded.FastGenByArgs().Error()), @@ -1330,8 +1330,8 @@ func (s *GrpcServer) GetPrevRegion(ctx context.Context, request *pdpb.GetRegionR if s.GetServiceMiddlewarePersistOptions().IsGRPCRateLimitEnabled() { fName := currentFunction() limiter := s.GetGRPCRateLimiter() - if limiter.Allow(fName) { - defer limiter.Release(fName) + if done, err := limiter.Allow(fName); err == nil { + defer done() } else { return &pdpb.GetRegionResponse{ Header: s.wrapErrorToHeader(pdpb.ErrorType_UNKNOWN, errs.ErrRateLimitExceeded.FastGenByArgs().Error()), @@ -1375,8 +1375,8 @@ func (s *GrpcServer) GetRegionByID(ctx context.Context, request *pdpb.GetRegionB if s.GetServiceMiddlewarePersistOptions().IsGRPCRateLimitEnabled() { fName := currentFunction() limiter := s.GetGRPCRateLimiter() - if limiter.Allow(fName) { - defer limiter.Release(fName) + if done, err := limiter.Allow(fName); err == nil { + defer done() } else { return &pdpb.GetRegionResponse{ Header: s.wrapErrorToHeader(pdpb.ErrorType_UNKNOWN, errs.ErrRateLimitExceeded.FastGenByArgs().Error()), @@ -1419,8 +1419,8 @@ func (s *GrpcServer) ScanRegions(ctx context.Context, request *pdpb.ScanRegionsR if s.GetServiceMiddlewarePersistOptions().IsGRPCRateLimitEnabled() { fName := currentFunction() limiter := s.GetGRPCRateLimiter() - if limiter.Allow(fName) { - defer limiter.Release(fName) + if done, err := limiter.Allow(fName); err == nil { + defer done() } else { return &pdpb.ScanRegionsResponse{ Header: s.wrapErrorToHeader(pdpb.ErrorType_UNKNOWN, errs.ErrRateLimitExceeded.FastGenByArgs().Error()), diff --git a/server/server.go b/server/server.go index c815e7d50c6..187c30dbf7a 100644 --- a/server/server.go +++ b/server/server.go @@ -216,11 +216,11 @@ type Server struct { // related data structures defined in the PD grpc service pdProtoFactory *tsoutil.PDProtoFactory - serviceRateLimiter *ratelimit.Limiter + serviceRateLimiter *ratelimit.Controller serviceLabels map[string][]apiutil.AccessPath apiServiceLabelMap map[apiutil.AccessPath]string - grpcServiceRateLimiter *ratelimit.Limiter + grpcServiceRateLimiter *ratelimit.Controller grpcServiceLabels map[string]struct{} grpcServer *grpc.Server @@ -273,8 +273,8 @@ func CreateServer(ctx context.Context, cfg *config.Config, services []string, le audit.NewLocalLogBackend(true), audit.NewPrometheusHistogramBackend(serviceAuditHistogram, false), } - s.serviceRateLimiter = ratelimit.NewLimiter() - s.grpcServiceRateLimiter = ratelimit.NewLimiter() + s.serviceRateLimiter = ratelimit.NewController() + s.grpcServiceRateLimiter = ratelimit.NewController() s.serviceAuditBackendLabels = make(map[string]*audit.BackendLabels) s.serviceLabels = make(map[string][]apiutil.AccessPath) s.grpcServiceLabels = make(map[string]struct{}) @@ -1467,7 +1467,7 @@ func (s *Server) SetServiceAuditBackendLabels(serviceLabel string, labels []stri } // GetServiceRateLimiter is used to get rate limiter -func (s *Server) GetServiceRateLimiter() *ratelimit.Limiter { +func (s *Server) GetServiceRateLimiter() *ratelimit.Controller { return s.serviceRateLimiter } @@ -1482,7 +1482,7 @@ func (s *Server) UpdateServiceRateLimiter(serviceLabel string, opts ...ratelimit } // GetGRPCRateLimiter is used to get rate limiter -func (s *Server) GetGRPCRateLimiter() *ratelimit.Limiter { +func (s *Server) GetGRPCRateLimiter() *ratelimit.Controller { return s.grpcServiceRateLimiter } From e26a4f7292280345d5813d7a1990a77a8785c0d6 Mon Sep 17 00:00:00 2001 From: lance6716 Date: Wed, 13 Dec 2023 17:46:48 +0800 Subject: [PATCH 091/137] client/http: add more API for lightning's usage, and don't use body io.Reader (#7534) ref tikv/pd#7300 Signed-off-by: lance6716 Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- client/http/client.go | 124 ++++++++++++------ tests/integrations/client/http_client_test.go | 11 ++ 2 files changed, 95 insertions(+), 40 deletions(-) diff --git a/client/http/client.go b/client/http/client.go index 958c52489fb..d74c77571d6 100644 --- a/client/http/client.go +++ b/client/http/client.go @@ -55,6 +55,7 @@ type Client interface { GetHistoryHotRegions(context.Context, *HistoryHotRegionsRequest) (*HistoryHotRegions, error) GetRegionStatusByKeyRange(context.Context, *KeyRange, bool) (*RegionStats, error) GetStores(context.Context) (*StoresInfo, error) + GetStore(context.Context, uint64) (*StoreInfo, error) SetStoreLabels(context.Context, int64, map[string]string) error GetMembers(context.Context) (*MembersInfo, error) GetLeader(context.Context) (*pdpb.Member, error) @@ -62,9 +63,11 @@ type Client interface { /* Config-related interfaces */ GetScheduleConfig(context.Context) (map[string]interface{}, error) SetScheduleConfig(context.Context, map[string]interface{}) error + GetClusterVersion(context.Context) (string, error) /* Scheduler-related interfaces */ GetSchedulers(context.Context) ([]string, error) CreateScheduler(ctx context.Context, name string, storeID uint64) error + SetSchedulerDelay(context.Context, string, int64) error /* Rule-related interfaces */ GetAllPlacementRuleBundles(context.Context) ([]*GroupBundle, error) GetPlacementRuleBundleByGroup(context.Context, string) (*GroupBundle, error) @@ -247,7 +250,7 @@ func WithAllowFollowerHandle() HeaderOption { func (c *client) requestWithRetry( ctx context.Context, name, uri, method string, - body io.Reader, res interface{}, + body []byte, res interface{}, headerOpts ...HeaderOption, ) error { var ( @@ -269,7 +272,7 @@ func (c *client) requestWithRetry( func (c *client) request( ctx context.Context, name, url, method string, - body io.Reader, res interface{}, + body []byte, res interface{}, headerOpts ...HeaderOption, ) error { logFields := []zap.Field{ @@ -279,7 +282,7 @@ func (c *client) request( zap.String("caller-id", c.callerID), } log.Debug("[pd] request the http url", logFields...) - req, err := http.NewRequestWithContext(ctx, method, url, body) + req, err := http.NewRequestWithContext(ctx, method, url, bytes.NewBuffer(body)) if err != nil { log.Error("[pd] create http request failed", append(logFields, zap.Error(err))...) return errors.Trace(err) @@ -341,7 +344,7 @@ func (c *client) GetRegionByID(ctx context.Context, regionID uint64) (*RegionInf var region RegionInfo err := c.requestWithRetry(ctx, "GetRegionByID", RegionByID(regionID), - http.MethodGet, http.NoBody, ®ion) + http.MethodGet, nil, ®ion) if err != nil { return nil, err } @@ -353,7 +356,7 @@ func (c *client) GetRegionByKey(ctx context.Context, key []byte) (*RegionInfo, e var region RegionInfo err := c.requestWithRetry(ctx, "GetRegionByKey", RegionByKey(key), - http.MethodGet, http.NoBody, ®ion) + http.MethodGet, nil, ®ion) if err != nil { return nil, err } @@ -365,7 +368,7 @@ func (c *client) GetRegions(ctx context.Context) (*RegionsInfo, error) { var regions RegionsInfo err := c.requestWithRetry(ctx, "GetRegions", Regions, - http.MethodGet, http.NoBody, ®ions) + http.MethodGet, nil, ®ions) if err != nil { return nil, err } @@ -378,7 +381,7 @@ func (c *client) GetRegionsByKeyRange(ctx context.Context, keyRange *KeyRange, l var regions RegionsInfo err := c.requestWithRetry(ctx, "GetRegionsByKeyRange", RegionsByKeyRange(keyRange, limit), - http.MethodGet, http.NoBody, ®ions) + http.MethodGet, nil, ®ions) if err != nil { return nil, err } @@ -390,7 +393,7 @@ func (c *client) GetRegionsByStoreID(ctx context.Context, storeID uint64) (*Regi var regions RegionsInfo err := c.requestWithRetry(ctx, "GetRegionsByStoreID", RegionsByStoreID(storeID), - http.MethodGet, http.NoBody, ®ions) + http.MethodGet, nil, ®ions) if err != nil { return nil, err } @@ -403,7 +406,7 @@ func (c *client) GetRegionsReplicatedStateByKeyRange(ctx context.Context, keyRan var state string err := c.requestWithRetry(ctx, "GetRegionsReplicatedStateByKeyRange", RegionsReplicatedByKeyRange(keyRange), - http.MethodGet, http.NoBody, &state) + http.MethodGet, nil, &state) if err != nil { return "", err } @@ -415,7 +418,7 @@ func (c *client) GetHotReadRegions(ctx context.Context) (*StoreHotPeersInfos, er var hotReadRegions StoreHotPeersInfos err := c.requestWithRetry(ctx, "GetHotReadRegions", HotRead, - http.MethodGet, http.NoBody, &hotReadRegions) + http.MethodGet, nil, &hotReadRegions) if err != nil { return nil, err } @@ -427,7 +430,7 @@ func (c *client) GetHotWriteRegions(ctx context.Context) (*StoreHotPeersInfos, e var hotWriteRegions StoreHotPeersInfos err := c.requestWithRetry(ctx, "GetHotWriteRegions", HotWrite, - http.MethodGet, http.NoBody, &hotWriteRegions) + http.MethodGet, nil, &hotWriteRegions) if err != nil { return nil, err } @@ -443,7 +446,7 @@ func (c *client) GetHistoryHotRegions(ctx context.Context, req *HistoryHotRegion var historyHotRegions HistoryHotRegions err = c.requestWithRetry(ctx, "GetHistoryHotRegions", HotHistory, - http.MethodGet, bytes.NewBuffer(reqJSON), &historyHotRegions, + http.MethodGet, reqJSON, &historyHotRegions, WithAllowFollowerHandle()) if err != nil { return nil, err @@ -458,7 +461,7 @@ func (c *client) GetRegionStatusByKeyRange(ctx context.Context, keyRange *KeyRan var regionStats RegionStats err := c.requestWithRetry(ctx, "GetRegionStatusByKeyRange", RegionStatsByKeyRange(keyRange, onlyCount), - http.MethodGet, http.NoBody, ®ionStats, + http.MethodGet, nil, ®ionStats, ) if err != nil { return nil, err @@ -473,14 +476,14 @@ func (c *client) SetStoreLabels(ctx context.Context, storeID int64, storeLabels return errors.Trace(err) } return c.requestWithRetry(ctx, "SetStoreLabel", LabelByStoreID(storeID), - http.MethodPost, bytes.NewBuffer(jsonInput), nil) + http.MethodPost, jsonInput, nil) } func (c *client) GetMembers(ctx context.Context) (*MembersInfo, error) { var members MembersInfo err := c.requestWithRetry(ctx, "GetMembers", membersPrefix, - http.MethodGet, http.NoBody, &members) + http.MethodGet, nil, &members) if err != nil { return nil, err } @@ -491,7 +494,7 @@ func (c *client) GetMembers(ctx context.Context) (*MembersInfo, error) { func (c *client) GetLeader(ctx context.Context) (*pdpb.Member, error) { var leader pdpb.Member err := c.requestWithRetry(ctx, "GetLeader", leaderPrefix, - http.MethodGet, http.NoBody, &leader) + http.MethodGet, nil, &leader) if err != nil { return nil, err } @@ -501,7 +504,7 @@ func (c *client) GetLeader(ctx context.Context) (*pdpb.Member, error) { // TransferLeader transfers the PD leader. func (c *client) TransferLeader(ctx context.Context, newLeader string) error { return c.requestWithRetry(ctx, "TransferLeader", TransferLeaderByID(newLeader), - http.MethodPost, http.NoBody, nil) + http.MethodPost, nil, nil) } // GetScheduleConfig gets the schedule configurations. @@ -509,7 +512,7 @@ func (c *client) GetScheduleConfig(ctx context.Context) (map[string]interface{}, var config map[string]interface{} err := c.requestWithRetry(ctx, "GetScheduleConfig", ScheduleConfig, - http.MethodGet, http.NoBody, &config) + http.MethodGet, nil, &config) if err != nil { return nil, err } @@ -524,7 +527,7 @@ func (c *client) SetScheduleConfig(ctx context.Context, config map[string]interf } return c.requestWithRetry(ctx, "SetScheduleConfig", ScheduleConfig, - http.MethodPost, bytes.NewBuffer(configJSON), nil) + http.MethodPost, configJSON, nil) } // GetStores gets the stores info. @@ -532,19 +535,43 @@ func (c *client) GetStores(ctx context.Context) (*StoresInfo, error) { var stores StoresInfo err := c.requestWithRetry(ctx, "GetStores", Stores, - http.MethodGet, http.NoBody, &stores) + http.MethodGet, nil, &stores) if err != nil { return nil, err } return &stores, nil } +// GetStore gets the store info by ID. +func (c *client) GetStore(ctx context.Context, storeID uint64) (*StoreInfo, error) { + var store StoreInfo + err := c.requestWithRetry(ctx, + "GetStore", StoreByID(storeID), + http.MethodGet, nil, &store) + if err != nil { + return nil, err + } + return &store, nil +} + +// GetClusterVersion gets the cluster version. +func (c *client) GetClusterVersion(ctx context.Context) (string, error) { + var version string + err := c.requestWithRetry(ctx, + "GetClusterVersion", ClusterVersion, + http.MethodGet, nil, &version) + if err != nil { + return "", err + } + return version, nil +} + // GetAllPlacementRuleBundles gets all placement rules bundles. func (c *client) GetAllPlacementRuleBundles(ctx context.Context) ([]*GroupBundle, error) { var bundles []*GroupBundle err := c.requestWithRetry(ctx, "GetPlacementRuleBundle", PlacementRuleBundle, - http.MethodGet, http.NoBody, &bundles) + http.MethodGet, nil, &bundles) if err != nil { return nil, err } @@ -556,7 +583,7 @@ func (c *client) GetPlacementRuleBundleByGroup(ctx context.Context, group string var bundle GroupBundle err := c.requestWithRetry(ctx, "GetPlacementRuleBundleByGroup", PlacementRuleBundleByGroup(group), - http.MethodGet, http.NoBody, &bundle) + http.MethodGet, nil, &bundle) if err != nil { return nil, err } @@ -568,7 +595,7 @@ func (c *client) GetPlacementRulesByGroup(ctx context.Context, group string) ([] var rules []*Rule err := c.requestWithRetry(ctx, "GetPlacementRulesByGroup", PlacementRulesByGroup(group), - http.MethodGet, http.NoBody, &rules) + http.MethodGet, nil, &rules) if err != nil { return nil, err } @@ -583,7 +610,7 @@ func (c *client) SetPlacementRule(ctx context.Context, rule *Rule) error { } return c.requestWithRetry(ctx, "SetPlacementRule", PlacementRule, - http.MethodPost, bytes.NewBuffer(ruleJSON), nil) + http.MethodPost, ruleJSON, nil) } // SetPlacementRuleInBatch sets the placement rules in batch. @@ -594,7 +621,7 @@ func (c *client) SetPlacementRuleInBatch(ctx context.Context, ruleOps []*RuleOp) } return c.requestWithRetry(ctx, "SetPlacementRuleInBatch", PlacementRulesInBatch, - http.MethodPost, bytes.NewBuffer(ruleOpsJSON), nil) + http.MethodPost, ruleOpsJSON, nil) } // SetPlacementRuleBundles sets the placement rule bundles. @@ -606,14 +633,14 @@ func (c *client) SetPlacementRuleBundles(ctx context.Context, bundles []*GroupBu } return c.requestWithRetry(ctx, "SetPlacementRuleBundles", PlacementRuleBundleWithPartialParameter(partial), - http.MethodPost, bytes.NewBuffer(bundlesJSON), nil) + http.MethodPost, bundlesJSON, nil) } // DeletePlacementRule deletes the placement rule. func (c *client) DeletePlacementRule(ctx context.Context, group, id string) error { return c.requestWithRetry(ctx, "DeletePlacementRule", PlacementRuleByGroupAndID(group, id), - http.MethodDelete, http.NoBody, nil) + http.MethodDelete, nil, nil) } // GetAllPlacementRuleGroups gets all placement rule groups. @@ -621,7 +648,7 @@ func (c *client) GetAllPlacementRuleGroups(ctx context.Context) ([]*RuleGroup, e var ruleGroups []*RuleGroup err := c.requestWithRetry(ctx, "GetAllPlacementRuleGroups", placementRuleGroups, - http.MethodGet, http.NoBody, &ruleGroups) + http.MethodGet, nil, &ruleGroups) if err != nil { return nil, err } @@ -633,7 +660,7 @@ func (c *client) GetPlacementRuleGroupByID(ctx context.Context, id string) (*Rul var ruleGroup RuleGroup err := c.requestWithRetry(ctx, "GetPlacementRuleGroupByID", PlacementRuleGroupByID(id), - http.MethodGet, http.NoBody, &ruleGroup) + http.MethodGet, nil, &ruleGroup) if err != nil { return nil, err } @@ -648,14 +675,14 @@ func (c *client) SetPlacementRuleGroup(ctx context.Context, ruleGroup *RuleGroup } return c.requestWithRetry(ctx, "SetPlacementRuleGroup", placementRuleGroup, - http.MethodPost, bytes.NewBuffer(ruleGroupJSON), nil) + http.MethodPost, ruleGroupJSON, nil) } // DeletePlacementRuleGroupByID deletes the placement rule group by ID. func (c *client) DeletePlacementRuleGroupByID(ctx context.Context, id string) error { return c.requestWithRetry(ctx, "DeletePlacementRuleGroupByID", PlacementRuleGroupByID(id), - http.MethodDelete, http.NoBody, nil) + http.MethodDelete, nil, nil) } // GetAllRegionLabelRules gets all region label rules. @@ -663,7 +690,7 @@ func (c *client) GetAllRegionLabelRules(ctx context.Context) ([]*LabelRule, erro var labelRules []*LabelRule err := c.requestWithRetry(ctx, "GetAllRegionLabelRules", RegionLabelRules, - http.MethodGet, http.NoBody, &labelRules) + http.MethodGet, nil, &labelRules) if err != nil { return nil, err } @@ -679,7 +706,7 @@ func (c *client) GetRegionLabelRulesByIDs(ctx context.Context, ruleIDs []string) var labelRules []*LabelRule err = c.requestWithRetry(ctx, "GetRegionLabelRulesByIDs", RegionLabelRulesByIDs, - http.MethodGet, bytes.NewBuffer(idsJSON), &labelRules) + http.MethodGet, idsJSON, &labelRules) if err != nil { return nil, err } @@ -694,7 +721,7 @@ func (c *client) SetRegionLabelRule(ctx context.Context, labelRule *LabelRule) e } return c.requestWithRetry(ctx, "SetRegionLabelRule", RegionLabelRule, - http.MethodPost, bytes.NewBuffer(labelRuleJSON), nil) + http.MethodPost, labelRuleJSON, nil) } // PatchRegionLabelRules patches the region label rules. @@ -705,14 +732,14 @@ func (c *client) PatchRegionLabelRules(ctx context.Context, labelRulePatch *Labe } return c.requestWithRetry(ctx, "PatchRegionLabelRules", RegionLabelRules, - http.MethodPatch, bytes.NewBuffer(labelRulePatchJSON), nil) + http.MethodPatch, labelRulePatchJSON, nil) } // GetSchedulers gets the schedulers from PD cluster. func (c *client) GetSchedulers(ctx context.Context) ([]string, error) { var schedulers []string err := c.requestWithRetry(ctx, "GetSchedulers", Schedulers, - http.MethodGet, http.NoBody, &schedulers) + http.MethodGet, nil, &schedulers) if err != nil { return nil, err } @@ -730,7 +757,7 @@ func (c *client) CreateScheduler(ctx context.Context, name string, storeID uint6 } return c.requestWithRetry(ctx, "CreateScheduler", Schedulers, - http.MethodPost, bytes.NewBuffer(inputJSON), nil) + http.MethodPost, inputJSON, nil) } // AccelerateSchedule accelerates the scheduling of the regions within the given key range. @@ -746,7 +773,7 @@ func (c *client) AccelerateSchedule(ctx context.Context, keyRange *KeyRange) err } return c.requestWithRetry(ctx, "AccelerateSchedule", AccelerateSchedule, - http.MethodPost, bytes.NewBuffer(inputJSON), nil) + http.MethodPost, inputJSON, nil) } // AccelerateScheduleInBatch accelerates the scheduling of the regions within the given key ranges in batch. @@ -766,10 +793,27 @@ func (c *client) AccelerateScheduleInBatch(ctx context.Context, keyRanges []*Key } return c.requestWithRetry(ctx, "AccelerateScheduleInBatch", AccelerateScheduleInBatch, - http.MethodPost, bytes.NewBuffer(inputJSON), nil) + http.MethodPost, inputJSON, nil) +} + +// SetSchedulerDelay sets the delay of given scheduler. +func (c *client) SetSchedulerDelay(ctx context.Context, scheduler string, delaySec int64) error { + m := map[string]int64{ + "delay": delaySec, + } + inputJSON, err := json.Marshal(m) + if err != nil { + return errors.Trace(err) + } + return c.requestWithRetry(ctx, + "SetSchedulerDelay", SchedulerByName(scheduler), + http.MethodPost, inputJSON, nil) } // GetMinResolvedTSByStoresIDs get min-resolved-ts by stores IDs. +// - When storeIDs has zero length, it will return (cluster-level's min_resolved_ts, nil, nil) when no error. +// - When storeIDs is {"cluster"}, it will return (cluster-level's min_resolved_ts, stores_min_resolved_ts, nil) when no error. +// - When storeID is specified to ID lists, it will return (min_resolved_ts of given stores, stores_min_resolved_ts, nil) when no error. func (c *client) GetMinResolvedTSByStoresIDs(ctx context.Context, storeIDs []uint64) (uint64, map[uint64]uint64, error) { uri := MinResolvedTSPrefix // scope is an optional parameter, it can be `cluster` or specified store IDs. @@ -791,7 +835,7 @@ func (c *client) GetMinResolvedTSByStoresIDs(ctx context.Context, storeIDs []uin }{} err := c.requestWithRetry(ctx, "GetMinResolvedTSByStoresIDs", uri, - http.MethodGet, http.NoBody, &resp) + http.MethodGet, nil, &resp) if err != nil { return 0, nil, err } diff --git a/tests/integrations/client/http_client_test.go b/tests/integrations/client/http_client_test.go index 476b4d2f541..7c8f66f4826 100644 --- a/tests/integrations/client/http_client_test.go +++ b/tests/integrations/client/http_client_test.go @@ -134,6 +134,13 @@ func (suite *httpClientTestSuite) TestMeta() { re.NoError(err) re.Equal(1, store.Count) re.Len(store.Stores, 1) + storeID := uint64(store.Stores[0].Store.ID) // TODO: why type is different? + store2, err := suite.client.GetStore(suite.ctx, storeID) + re.NoError(err) + re.EqualValues(storeID, store2.Store.ID) + version, err := suite.client.GetClusterVersion(suite.ctx) + re.NoError(err) + re.Equal("0.0.0", version) } func (suite *httpClientTestSuite) TestGetMinResolvedTSByStoresIDs() { @@ -396,6 +403,10 @@ func (suite *httpClientTestSuite) TestSchedulers() { schedulers, err = suite.client.GetSchedulers(suite.ctx) re.NoError(err) re.Len(schedulers, 1) + err = suite.client.SetSchedulerDelay(suite.ctx, "evict-leader-scheduler", 100) + re.NoError(err) + err = suite.client.SetSchedulerDelay(suite.ctx, "not-exist", 100) + re.ErrorContains(err, "500 Internal Server Error") // TODO: should return friendly error message } func (suite *httpClientTestSuite) TestSetStoreLabels() { From 48fabb79e8b0197bd6752fb84b0b704b782d3b48 Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Wed, 13 Dec 2023 18:37:49 +0800 Subject: [PATCH 092/137] mcs: fix unnecessary PDRedirectorHeader (#7538) close tikv/pd#7533 Signed-off-by: lhy1024 Co-authored-by: Ryan Leung --- pkg/utils/apiutil/serverapi/middleware.go | 20 +++---- tests/integrations/mcs/scheduling/api_test.go | 55 +++++++++++++++++-- tests/pdctl/scheduler/scheduler_test.go | 45 --------------- 3 files changed, 60 insertions(+), 60 deletions(-) mode change 100644 => 100755 pkg/utils/apiutil/serverapi/middleware.go diff --git a/pkg/utils/apiutil/serverapi/middleware.go b/pkg/utils/apiutil/serverapi/middleware.go old mode 100644 new mode 100755 index eb0f8a5f8eb..c7979dcc038 --- a/pkg/utils/apiutil/serverapi/middleware.go +++ b/pkg/utils/apiutil/serverapi/middleware.go @@ -182,14 +182,6 @@ func (h *redirector) ServeHTTP(w http.ResponseWriter, r *http.Request, next http return } - // Prevent more than one redirection. - if name := r.Header.Get(apiutil.PDRedirectorHeader); len(name) != 0 { - log.Error("redirect but server is not leader", zap.String("from", name), zap.String("server", h.s.Name()), errs.ZapError(errs.ErrRedirect)) - http.Error(w, errs.ErrRedirectToNotLeader.FastGenByArgs().Error(), http.StatusInternalServerError) - return - } - - r.Header.Set(apiutil.PDRedirectorHeader, h.s.Name()) forwardedIP, forwardedPort := apiutil.GetIPPortFromHTTPRequest(r) if len(forwardedIP) > 0 { r.Header.Add(apiutil.XForwardedForHeader, forwardedIP) @@ -208,9 +200,9 @@ func (h *redirector) ServeHTTP(w http.ResponseWriter, r *http.Request, next http return } clientUrls = append(clientUrls, targetAddr) + // Add a header to the response, this is not a failure injection + // it is used for testing, to check whether the request is forwarded to the micro service failpoint.Inject("checkHeader", func() { - // add a header to the response, this is not a failure injection - // it is used for testing, to check whether the request is forwarded to the micro service w.Header().Set(apiutil.ForwardToMicroServiceHeader, "true") }) } else { @@ -220,7 +212,15 @@ func (h *redirector) ServeHTTP(w http.ResponseWriter, r *http.Request, next http return } clientUrls = leader.GetClientUrls() + // Prevent more than one redirection among PD/API servers. + if name := r.Header.Get(apiutil.PDRedirectorHeader); len(name) != 0 { + log.Error("redirect but server is not leader", zap.String("from", name), zap.String("server", h.s.Name()), errs.ZapError(errs.ErrRedirect)) + http.Error(w, errs.ErrRedirectToNotLeader.FastGenByArgs().Error(), http.StatusInternalServerError) + return + } + r.Header.Set(apiutil.PDRedirectorHeader, h.s.Name()) } + urls := make([]url.URL, 0, len(clientUrls)) for _, item := range clientUrls { u, err := url.Parse(item) diff --git a/tests/integrations/mcs/scheduling/api_test.go b/tests/integrations/mcs/scheduling/api_test.go index 8f5d37ee1bb..4c71f8f14a3 100644 --- a/tests/integrations/mcs/scheduling/api_test.go +++ b/tests/integrations/mcs/scheduling/api_test.go @@ -1,6 +1,7 @@ package scheduling_test import ( + "context" "encoding/hex" "encoding/json" "fmt" @@ -40,10 +41,12 @@ func TestAPI(t *testing.T) { } func (suite *apiTestSuite) SetupSuite() { + suite.NoError(failpoint.Enable("github.com/tikv/pd/pkg/utils/apiutil/serverapi/checkHeader", "return(true)")) suite.env = tests.NewSchedulingTestEnvironment(suite.T()) } func (suite *apiTestSuite) TearDownSuite() { + suite.NoError(failpoint.Disable("github.com/tikv/pd/pkg/utils/apiutil/serverapi/checkHeader")) suite.env.Cleanup() } @@ -99,10 +102,6 @@ func (suite *apiTestSuite) TestAPIForward() { func (suite *apiTestSuite) checkAPIForward(cluster *tests.TestCluster) { re := suite.Require() - re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/utils/apiutil/serverapi/checkHeader", "return(true)")) - defer func() { - re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/utils/apiutil/serverapi/checkHeader")) - }() leader := cluster.GetLeaderServer().GetServer() urlPrefix := fmt.Sprintf("%s/pd/api/v1", leader.GetAddr()) @@ -300,7 +299,7 @@ func (suite *apiTestSuite) checkAPIForward(cluster *tests.TestCluster) { rulesArgs, err := json.Marshal(rules) suite.NoError(err) - err = testutil.ReadGetJSON(re, testDialClient, fmt.Sprintf("%s/%s", urlPrefix, "/config/rules"), &rules, + err = testutil.ReadGetJSON(re, testDialClient, fmt.Sprintf("%s/%s", urlPrefix, "config/rules"), &rules, testutil.WithHeader(re, apiutil.ForwardToMicroServiceHeader, "true")) re.NoError(err) err = testutil.CheckPostJSON(testDialClient, fmt.Sprintf("%s/%s", urlPrefix, "config/rules"), rulesArgs, @@ -499,3 +498,49 @@ func (suite *apiTestSuite) checkAdminRegionCacheForward(cluster *tests.TestClust re.Equal(0, schedulingServer.GetCluster().GetRegionCount([]byte{}, []byte{})) re.Equal(0, apiServer.GetRaftCluster().GetRegionCount([]byte{}, []byte{}).Count) } + +func (suite *apiTestSuite) TestFollowerForward() { + suite.env.RunTestInTwoModes(suite.checkFollowerForward) +} + +func (suite *apiTestSuite) checkFollowerForward(cluster *tests.TestCluster) { + re := suite.Require() + leaderAddr := cluster.GetLeaderServer().GetAddr() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + follower, err := cluster.JoinAPIServer(ctx) + re.NoError(err) + re.NoError(follower.Run()) + re.NotEmpty(cluster.WaitLeader()) + + followerAddr := follower.GetAddr() + if cluster.GetLeaderServer().GetAddr() != leaderAddr { + followerAddr = leaderAddr + } + + urlPrefix := fmt.Sprintf("%s/pd/api/v1", followerAddr) + rules := []*placement.Rule{} + if sche := cluster.GetSchedulingPrimaryServer(); sche != nil { + // follower will forward to scheduling server directly + re.NotEqual(cluster.GetLeaderServer().GetAddr(), followerAddr) + err = testutil.ReadGetJSON(re, testDialClient, fmt.Sprintf("%s/%s", urlPrefix, "config/rules"), &rules, + testutil.WithHeader(re, apiutil.ForwardToMicroServiceHeader, "true"), + ) + re.NoError(err) + } else { + // follower will forward to leader server + re.NotEqual(cluster.GetLeaderServer().GetAddr(), followerAddr) + err = testutil.ReadGetJSON(re, testDialClient, fmt.Sprintf("%s/%s", urlPrefix, "config/rules"), &rules, + testutil.WithoutHeader(re, apiutil.ForwardToMicroServiceHeader), + ) + re.NoError(err) + } + + // follower will forward to leader server + re.NotEqual(cluster.GetLeaderServer().GetAddr(), followerAddr) + results := make(map[string]interface{}) + err = testutil.ReadGetJSON(re, testDialClient, fmt.Sprintf("%s/%s", urlPrefix, "config"), &results, + testutil.WithoutHeader(re, apiutil.ForwardToMicroServiceHeader), + ) + re.NoError(err) +} diff --git a/tests/pdctl/scheduler/scheduler_test.go b/tests/pdctl/scheduler/scheduler_test.go index d8d54a79d13..fb7c239b431 100644 --- a/tests/pdctl/scheduler/scheduler_test.go +++ b/tests/pdctl/scheduler/scheduler_test.go @@ -15,7 +15,6 @@ package scheduler_test import ( - "context" "encoding/json" "fmt" "reflect" @@ -691,47 +690,3 @@ func mightExec(re *require.Assertions, cmd *cobra.Command, args []string, v inte } json.Unmarshal(output, v) } - -func TestForwardSchedulerRequest(t *testing.T) { - re := require.New(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - cluster, err := tests.NewTestAPICluster(ctx, 1) - re.NoError(err) - re.NoError(cluster.RunInitialServers()) - re.NotEmpty(cluster.WaitLeader()) - server := cluster.GetLeaderServer() - re.NoError(server.BootstrapCluster()) - backendEndpoints := server.GetAddr() - tc, err := tests.NewTestSchedulingCluster(ctx, 1, backendEndpoints) - re.NoError(err) - defer tc.Destroy() - tc.WaitForPrimaryServing(re) - - cmd := pdctlCmd.GetRootCmd() - args := []string{"-u", backendEndpoints, "scheduler", "show"} - var sches []string - testutil.Eventually(re, func() bool { - output, err := pdctl.ExecuteCommand(cmd, args...) - re.NoError(err) - re.NoError(json.Unmarshal(output, &sches)) - return slice.Contains(sches, "balance-leader-scheduler") - }) - - mustUsage := func(args []string) { - output, err := pdctl.ExecuteCommand(cmd, args...) - re.NoError(err) - re.Contains(string(output), "Usage") - } - mustUsage([]string{"-u", backendEndpoints, "scheduler", "pause", "balance-leader-scheduler"}) - echo := mustExec(re, cmd, []string{"-u", backendEndpoints, "scheduler", "pause", "balance-leader-scheduler", "60"}, nil) - re.Contains(echo, "Success!") - checkSchedulerWithStatusCommand := func(status string, expected []string) { - var schedulers []string - mustExec(re, cmd, []string{"-u", backendEndpoints, "scheduler", "show", "--status", status}, &schedulers) - re.Equal(expected, schedulers) - } - checkSchedulerWithStatusCommand("paused", []string{ - "balance-leader-scheduler", - }) -} From 0e220b0b39a765762c48a7fa620b15bc78bd0a38 Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Wed, 13 Dec 2023 18:55:49 +0800 Subject: [PATCH 093/137] api: fix the output of some APIs (#7542) ref tikv/pd#4399 Signed-off-by: Ryan Leung Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- server/api/store.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/server/api/store.go b/server/api/store.go index 8537cd45c5b..44e178c23fd 100644 --- a/server/api/store.go +++ b/server/api/store.go @@ -334,7 +334,7 @@ func (h *storeHandler) SetStoreLabel(w http.ResponseWriter, r *http.Request) { // @Param id path integer true "Store Id" // @Param body body object true "Labels in json format" // @Produce json -// @Success 200 {string} string "The store's label is updated." +// @Success 200 {string} string "The label is deleted for store." // @Failure 400 {string} string "The input is invalid." // @Failure 500 {string} string "PD server failed to proceed the request." // @Router /store/{id}/label [delete] @@ -369,7 +369,7 @@ func (h *storeHandler) DeleteStoreLabel(w http.ResponseWriter, r *http.Request) // @Param id path integer true "Store Id" // @Param body body object true "json params" // @Produce json -// @Success 200 {string} string "The store's label is updated." +// @Success 200 {string} string "The store's weight is updated." // @Failure 400 {string} string "The input is invalid." // @Failure 500 {string} string "PD server failed to proceed the request." // @Router /store/{id}/weight [post] @@ -413,7 +413,7 @@ func (h *storeHandler) SetStoreWeight(w http.ResponseWriter, r *http.Request) { return } - h.rd.JSON(w, http.StatusOK, "The store's label is updated.") + h.rd.JSON(w, http.StatusOK, "The store's weight is updated.") } // FIXME: details of input json body params @@ -423,7 +423,7 @@ func (h *storeHandler) SetStoreWeight(w http.ResponseWriter, r *http.Request) { // @Param id path integer true "Store Id" // @Param body body object true "json params" // @Produce json -// @Success 200 {string} string "The store's label is updated." +// @Success 200 {string} string "The store's limit is updated." // @Failure 400 {string} string "The input is invalid." // @Failure 500 {string} string "PD server failed to proceed the request." // @Router /store/{id}/limit [post] @@ -486,7 +486,7 @@ func (h *storeHandler) SetStoreLimit(w http.ResponseWriter, r *http.Request) { return } } - h.rd.JSON(w, http.StatusOK, "The store's label is updated.") + h.rd.JSON(w, http.StatusOK, "The store's limit is updated.") } type storesHandler struct { From f51f9134558e793e7ea08fc32cef672c7afa37b7 Mon Sep 17 00:00:00 2001 From: Yongbo Jiang Date: Wed, 13 Dec 2023 19:27:19 +0800 Subject: [PATCH 094/137] errs: remove redundant `FastGenWithCause` in `ZapError` (#7497) close tikv/pd#7496 Signed-off-by: Cabinfever_B Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- client/errs/errno.go | 2 +- client/errs/errs.go | 2 +- client/tso_dispatcher.go | 6 ++-- errors.toml | 2 +- pkg/autoscaling/prometheus.go | 2 +- pkg/errs/errno.go | 2 +- pkg/errs/errs.go | 2 +- pkg/errs/errs_test.go | 35 +++++++++++++++------- pkg/schedule/hbstream/heartbeat_streams.go | 2 +- pkg/schedule/plugin_interface.go | 6 ++-- pkg/schedule/schedulers/evict_leader.go | 2 +- pkg/schedule/schedulers/grant_leader.go | 2 +- pkg/schedule/schedulers/init.go | 10 +++---- pkg/schedule/schedulers/scheduler.go | 4 +-- pkg/schedule/schedulers/utils.go | 4 +-- pkg/tso/keyspace_group_manager.go | 2 +- pkg/utils/logutil/log.go | 2 +- pkg/utils/tempurl/check_env_linux.go | 2 +- server/handler.go | 2 +- 19 files changed, 53 insertions(+), 38 deletions(-) diff --git a/client/errs/errno.go b/client/errs/errno.go index 646af81929d..0f93ebf1472 100644 --- a/client/errs/errno.go +++ b/client/errs/errno.go @@ -54,7 +54,7 @@ var ( ErrClientGetMultiResponse = errors.Normalize("get invalid value response %v, must only one", errors.RFCCodeText("PD:client:ErrClientGetMultiResponse")) ErrClientGetServingEndpoint = errors.Normalize("get serving endpoint failed", errors.RFCCodeText("PD:client:ErrClientGetServingEndpoint")) ErrClientFindGroupByKeyspaceID = errors.Normalize("can't find keyspace group by keyspace id", errors.RFCCodeText("PD:client:ErrClientFindGroupByKeyspaceID")) - ErrClientWatchGCSafePointV2Stream = errors.Normalize("watch gc safe point v2 stream failed, %s", errors.RFCCodeText("PD:client:ErrClientWatchGCSafePointV2Stream")) + ErrClientWatchGCSafePointV2Stream = errors.Normalize("watch gc safe point v2 stream failed", errors.RFCCodeText("PD:client:ErrClientWatchGCSafePointV2Stream")) ) // grpcutil errors diff --git a/client/errs/errs.go b/client/errs/errs.go index e715056b055..47f7c29a467 100644 --- a/client/errs/errs.go +++ b/client/errs/errs.go @@ -27,7 +27,7 @@ func ZapError(err error, causeError ...error) zap.Field { } if e, ok := err.(*errors.Error); ok { if len(causeError) >= 1 { - err = e.Wrap(causeError[0]).FastGenWithCause() + err = e.Wrap(causeError[0]) } else { err = e.FastGenByArgs() } diff --git a/client/tso_dispatcher.go b/client/tso_dispatcher.go index 0de4dc3a49e..6b2c33ca58d 100644 --- a/client/tso_dispatcher.go +++ b/client/tso_dispatcher.go @@ -412,7 +412,7 @@ tsoBatchLoop: } else { log.Error("[tso] fetch pending tso requests error", zap.String("dc-location", dc), - errs.ZapError(errs.ErrClientGetTSO.FastGenByArgs("when fetch pending tso requests"), err)) + errs.ZapError(errs.ErrClientGetTSO, err)) } return } @@ -495,10 +495,10 @@ tsoBatchLoop: default: } c.svcDiscovery.ScheduleCheckMemberChanged() - log.Error("[tso] getTS error", + log.Error("[tso] getTS error after processing requests", zap.String("dc-location", dc), zap.String("stream-addr", streamAddr), - errs.ZapError(errs.ErrClientGetTSO.FastGenByArgs("after processing requests"), err)) + errs.ZapError(errs.ErrClientGetTSO, err)) // Set `stream` to nil and remove this stream from the `connectionCtxs` due to error. connectionCtxs.Delete(streamAddr) cancel() diff --git a/errors.toml b/errors.toml index a318fc32492..69aa29d2dda 100644 --- a/errors.toml +++ b/errors.toml @@ -83,7 +83,7 @@ get min TSO failed, %v ["PD:client:ErrClientGetTSO"] error = ''' -get TSO failed, %v +get TSO failed ''' ["PD:client:ErrClientGetTSOTimeout"] diff --git a/pkg/autoscaling/prometheus.go b/pkg/autoscaling/prometheus.go index 43ba768a585..91b813b6ef2 100644 --- a/pkg/autoscaling/prometheus.go +++ b/pkg/autoscaling/prometheus.go @@ -94,7 +94,7 @@ func (prom *PrometheusQuerier) queryMetricsFromPrometheus(query string, timestam resp, warnings, err := prom.api.Query(ctx, query, timestamp) if err != nil { - return nil, errs.ErrPrometheusQuery.Wrap(err).FastGenWithCause() + return nil, errs.ErrPrometheusQuery.Wrap(err) } if len(warnings) > 0 { diff --git a/pkg/errs/errno.go b/pkg/errs/errno.go index a4320238374..03fa0f61158 100644 --- a/pkg/errs/errno.go +++ b/pkg/errs/errno.go @@ -86,7 +86,7 @@ var ( var ( ErrClientCreateTSOStream = errors.Normalize("create TSO stream failed, %s", errors.RFCCodeText("PD:client:ErrClientCreateTSOStream")) ErrClientGetTSOTimeout = errors.Normalize("get TSO timeout", errors.RFCCodeText("PD:client:ErrClientGetTSOTimeout")) - ErrClientGetTSO = errors.Normalize("get TSO failed, %v", errors.RFCCodeText("PD:client:ErrClientGetTSO")) + ErrClientGetTSO = errors.Normalize("get TSO failed", errors.RFCCodeText("PD:client:ErrClientGetTSO")) ErrClientGetLeader = errors.Normalize("get leader failed, %v", errors.RFCCodeText("PD:client:ErrClientGetLeader")) ErrClientGetMember = errors.Normalize("get member failed", errors.RFCCodeText("PD:client:ErrClientGetMember")) ErrClientGetMinTSO = errors.Normalize("get min TSO failed, %v", errors.RFCCodeText("PD:client:ErrClientGetMinTSO")) diff --git a/pkg/errs/errs.go b/pkg/errs/errs.go index acc42637733..5746b282f10 100644 --- a/pkg/errs/errs.go +++ b/pkg/errs/errs.go @@ -27,7 +27,7 @@ func ZapError(err error, causeError ...error) zap.Field { } if e, ok := err.(*errors.Error); ok { if len(causeError) >= 1 { - err = e.Wrap(causeError[0]).FastGenWithCause() + err = e.Wrap(causeError[0]) } else { err = e.FastGenByArgs() } diff --git a/pkg/errs/errs_test.go b/pkg/errs/errs_test.go index c242dd994f5..d76c02dc110 100644 --- a/pkg/errs/errs_test.go +++ b/pkg/errs/errs_test.go @@ -81,9 +81,19 @@ func TestError(t *testing.T) { log.Error("test", zap.Error(ErrEtcdLeaderNotFound.FastGenByArgs())) re.Contains(lg.Message(), rfc) err := errors.New("test error") - log.Error("test", ZapError(ErrEtcdLeaderNotFound, err)) - rfc = `[error="[PD:member:ErrEtcdLeaderNotFound]test error` - re.Contains(lg.Message(), rfc) + // use Info() because of no stack for comparing. + log.Info("test", ZapError(ErrEtcdLeaderNotFound, err)) + rfc = `[error="[PD:member:ErrEtcdLeaderNotFound]etcd leader not found: test error` + m1 := lg.Message() + re.Contains(m1, rfc) + log.Info("test", zap.Error(ErrEtcdLeaderNotFound.Wrap(err))) + m2 := lg.Message() + idx1 := strings.Index(m1, "[error") + idx2 := strings.Index(m2, "[error") + re.Equal(m1[idx1:], m2[idx2:]) + log.Info("test", zap.Error(ErrEtcdLeaderNotFound.Wrap(err).FastGenWithCause())) + m3 := lg.Message() + re.NotContains(m3, rfc) } func TestErrorEqual(t *testing.T) { @@ -94,24 +104,24 @@ func TestErrorEqual(t *testing.T) { re.True(errors.ErrorEqual(err1, err2)) err := errors.New("test") - err1 = ErrSchedulerNotFound.Wrap(err).FastGenWithCause() - err2 = ErrSchedulerNotFound.Wrap(err).FastGenWithCause() + err1 = ErrSchedulerNotFound.Wrap(err) + err2 = ErrSchedulerNotFound.Wrap(err) re.True(errors.ErrorEqual(err1, err2)) err1 = ErrSchedulerNotFound.FastGenByArgs() - err2 = ErrSchedulerNotFound.Wrap(err).FastGenWithCause() + err2 = ErrSchedulerNotFound.Wrap(err) re.False(errors.ErrorEqual(err1, err2)) err3 := errors.New("test") err4 := errors.New("test") - err1 = ErrSchedulerNotFound.Wrap(err3).FastGenWithCause() - err2 = ErrSchedulerNotFound.Wrap(err4).FastGenWithCause() + err1 = ErrSchedulerNotFound.Wrap(err3) + err2 = ErrSchedulerNotFound.Wrap(err4) re.True(errors.ErrorEqual(err1, err2)) err3 = errors.New("test1") err4 = errors.New("test") - err1 = ErrSchedulerNotFound.Wrap(err3).FastGenWithCause() - err2 = ErrSchedulerNotFound.Wrap(err4).FastGenWithCause() + err1 = ErrSchedulerNotFound.Wrap(err3) + err2 = ErrSchedulerNotFound.Wrap(err4) re.False(errors.ErrorEqual(err1, err2)) } @@ -135,11 +145,16 @@ func TestErrorWithStack(t *testing.T) { m1 := lg.Message() log.Error("test", zap.Error(errors.WithStack(err))) m2 := lg.Message() + log.Error("test", ZapError(ErrStrconvParseInt.GenWithStackByCause(), err)) + m3 := lg.Message() // This test is based on line number and the first log is in line 141, the second is in line 142. // So they have the same length stack. Move this test to another place need to change the corresponding length. idx1 := strings.Index(m1, "[stack=") re.GreaterOrEqual(idx1, -1) idx2 := strings.Index(m2, "[stack=") re.GreaterOrEqual(idx2, -1) + idx3 := strings.Index(m3, "[stack=") + re.GreaterOrEqual(idx3, -1) re.Len(m2[idx2:], len(m1[idx1:])) + re.Len(m3[idx3:], len(m1[idx1:])) } diff --git a/pkg/schedule/hbstream/heartbeat_streams.go b/pkg/schedule/hbstream/heartbeat_streams.go index e7d7f688035..57a7521c0a7 100644 --- a/pkg/schedule/hbstream/heartbeat_streams.go +++ b/pkg/schedule/hbstream/heartbeat_streams.go @@ -139,7 +139,7 @@ func (s *HeartbeatStreams) run() { if stream, ok := s.streams[storeID]; ok { if err := stream.Send(msg); err != nil { log.Error("send heartbeat message fail", - zap.Uint64("region-id", msg.GetRegionId()), errs.ZapError(errs.ErrGRPCSend.Wrap(err).GenWithStackByArgs())) + zap.Uint64("region-id", msg.GetRegionId()), errs.ZapError(errs.ErrGRPCSend, err)) delete(s.streams, storeID) heartbeatStreamCounter.WithLabelValues(storeAddress, storeLabel, "push", "err").Inc() } else { diff --git a/pkg/schedule/plugin_interface.go b/pkg/schedule/plugin_interface.go index dd1ce3471e6..62ffe2eb900 100644 --- a/pkg/schedule/plugin_interface.go +++ b/pkg/schedule/plugin_interface.go @@ -46,19 +46,19 @@ func (p *PluginInterface) GetFunction(path string, funcName string) (plugin.Symb // open plugin filePath, err := filepath.Abs(path) if err != nil { - return nil, errs.ErrFilePathAbs.Wrap(err).FastGenWithCause() + return nil, errs.ErrFilePathAbs.Wrap(err) } log.Info("open plugin file", zap.String("file-path", filePath)) plugin, err := plugin.Open(filePath) if err != nil { - return nil, errs.ErrLoadPlugin.Wrap(err).FastGenWithCause() + return nil, errs.ErrLoadPlugin.Wrap(err) } p.pluginMap[path] = plugin } // get func from plugin f, err := p.pluginMap[path].Lookup(funcName) if err != nil { - return nil, errs.ErrLookupPluginFunc.Wrap(err).FastGenWithCause() + return nil, errs.ErrLookupPluginFunc.Wrap(err) } return f, nil } diff --git a/pkg/schedule/schedulers/evict_leader.go b/pkg/schedule/schedulers/evict_leader.go index 879aa9869b3..ae8b1ecf1ea 100644 --- a/pkg/schedule/schedulers/evict_leader.go +++ b/pkg/schedule/schedulers/evict_leader.go @@ -80,7 +80,7 @@ func (conf *evictLeaderSchedulerConfig) BuildWithArgs(args []string) error { id, err := strconv.ParseUint(args[0], 10, 64) if err != nil { - return errs.ErrStrconvParseUint.Wrap(err).FastGenWithCause() + return errs.ErrStrconvParseUint.Wrap(err) } ranges, err := getKeyRanges(args[1:]) if err != nil { diff --git a/pkg/schedule/schedulers/grant_leader.go b/pkg/schedule/schedulers/grant_leader.go index 885f81e2442..027350536aa 100644 --- a/pkg/schedule/schedulers/grant_leader.go +++ b/pkg/schedule/schedulers/grant_leader.go @@ -63,7 +63,7 @@ func (conf *grantLeaderSchedulerConfig) BuildWithArgs(args []string) error { id, err := strconv.ParseUint(args[0], 10, 64) if err != nil { - return errs.ErrStrconvParseUint.Wrap(err).FastGenWithCause() + return errs.ErrStrconvParseUint.Wrap(err) } ranges, err := getKeyRanges(args[1:]) if err != nil { diff --git a/pkg/schedule/schedulers/init.go b/pkg/schedule/schedulers/init.go index f60be1e5b06..57eb4b90985 100644 --- a/pkg/schedule/schedulers/init.go +++ b/pkg/schedule/schedulers/init.go @@ -129,7 +129,7 @@ func schedulersRegister() { id, err := strconv.ParseUint(args[0], 10, 64) if err != nil { - return errs.ErrStrconvParseUint.Wrap(err).FastGenWithCause() + return errs.ErrStrconvParseUint.Wrap(err) } ranges, err := getKeyRanges(args[1:]) @@ -180,14 +180,14 @@ func schedulersRegister() { } leaderID, err := strconv.ParseUint(args[0], 10, 64) if err != nil { - return errs.ErrStrconvParseUint.Wrap(err).FastGenWithCause() + return errs.ErrStrconvParseUint.Wrap(err) } storeIDs := make([]uint64, 0) for _, id := range strings.Split(args[1], ",") { storeID, err := strconv.ParseUint(id, 10, 64) if err != nil { - return errs.ErrStrconvParseUint.Wrap(err).FastGenWithCause() + return errs.ErrStrconvParseUint.Wrap(err) } storeIDs = append(storeIDs, storeID) } @@ -248,7 +248,7 @@ func schedulersRegister() { id, err := strconv.ParseUint(args[0], 10, 64) if err != nil { - return errs.ErrStrconvParseUint.Wrap(err).FastGenWithCause() + return errs.ErrStrconvParseUint.Wrap(err) } ranges, err := getKeyRanges(args[1:]) if err != nil { @@ -365,7 +365,7 @@ func schedulersRegister() { if len(args) == 1 { limit, err := strconv.ParseUint(args[0], 10, 64) if err != nil { - return errs.ErrStrconvParseUint.Wrap(err).FastGenWithCause() + return errs.ErrStrconvParseUint.Wrap(err) } conf.Limit = limit } diff --git a/pkg/schedule/schedulers/scheduler.go b/pkg/schedule/schedulers/scheduler.go index 1c788989454..38fc8f5607d 100644 --- a/pkg/schedule/schedulers/scheduler.go +++ b/pkg/schedule/schedulers/scheduler.go @@ -52,7 +52,7 @@ type Scheduler interface { func EncodeConfig(v interface{}) ([]byte, error) { marshaled, err := json.Marshal(v) if err != nil { - return nil, errs.ErrJSONMarshal.Wrap(err).FastGenWithCause() + return nil, errs.ErrJSONMarshal.Wrap(err) } return marshaled, nil } @@ -61,7 +61,7 @@ func EncodeConfig(v interface{}) ([]byte, error) { func DecodeConfig(data []byte, v interface{}) error { err := json.Unmarshal(data, v) if err != nil { - return errs.ErrJSONUnmarshal.Wrap(err).FastGenWithCause() + return errs.ErrJSONUnmarshal.Wrap(err) } return nil } diff --git a/pkg/schedule/schedulers/utils.go b/pkg/schedule/schedulers/utils.go index fea51798d1c..a22f992bda1 100644 --- a/pkg/schedule/schedulers/utils.go +++ b/pkg/schedule/schedulers/utils.go @@ -218,11 +218,11 @@ func getKeyRanges(args []string) ([]core.KeyRange, error) { for len(args) > 1 { startKey, err := url.QueryUnescape(args[0]) if err != nil { - return nil, errs.ErrQueryUnescape.Wrap(err).FastGenWithCause() + return nil, errs.ErrQueryUnescape.Wrap(err) } endKey, err := url.QueryUnescape(args[1]) if err != nil { - return nil, errs.ErrQueryUnescape.Wrap(err).FastGenWithCause() + return nil, errs.ErrQueryUnescape.Wrap(err) } args = args[2:] ranges = append(ranges, core.NewKeyRange(startKey, endKey)) diff --git a/pkg/tso/keyspace_group_manager.go b/pkg/tso/keyspace_group_manager.go index badcb18d5d8..58534de1642 100644 --- a/pkg/tso/keyspace_group_manager.go +++ b/pkg/tso/keyspace_group_manager.go @@ -542,7 +542,7 @@ func (kgm *KeyspaceGroupManager) InitializeGroupWatchLoop() error { putFn := func(kv *mvccpb.KeyValue) error { group := &endpoint.KeyspaceGroup{} if err := json.Unmarshal(kv.Value, group); err != nil { - return errs.ErrJSONUnmarshal.Wrap(err).FastGenWithCause() + return errs.ErrJSONUnmarshal.Wrap(err) } kgm.updateKeyspaceGroup(group) if group.ID == mcsutils.DefaultKeyspaceGroupID { diff --git a/pkg/utils/logutil/log.go b/pkg/utils/logutil/log.go index 3dc4430b066..8c0977818fa 100644 --- a/pkg/utils/logutil/log.go +++ b/pkg/utils/logutil/log.go @@ -70,7 +70,7 @@ func StringToZapLogLevel(level string) zapcore.Level { func SetupLogger(logConfig log.Config, logger **zap.Logger, logProps **log.ZapProperties, enabled ...bool) error { lg, p, err := log.InitLogger(&logConfig, zap.AddStacktrace(zapcore.FatalLevel)) if err != nil { - return errs.ErrInitLogger.Wrap(err).FastGenWithCause() + return errs.ErrInitLogger.Wrap(err) } *logger = lg *logProps = p diff --git a/pkg/utils/tempurl/check_env_linux.go b/pkg/utils/tempurl/check_env_linux.go index cf0e686cada..58f902f4bb7 100644 --- a/pkg/utils/tempurl/check_env_linux.go +++ b/pkg/utils/tempurl/check_env_linux.go @@ -36,7 +36,7 @@ func checkAddr(addr string) (bool, error) { return s.RemoteAddr.String() == addr || s.LocalAddr.String() == addr }) if err != nil { - return false, errs.ErrNetstatTCPSocks.Wrap(err).FastGenWithCause() + return false, errs.ErrNetstatTCPSocks.Wrap(err) } return len(tabs) < 1, nil } diff --git a/server/handler.go b/server/handler.go index 6c0679bd9f9..b91c8e368f9 100644 --- a/server/handler.go +++ b/server/handler.go @@ -470,7 +470,7 @@ func (h *Handler) PluginLoad(pluginPath string) error { // make sure path is in data dir filePath, err := filepath.Abs(pluginPath) if err != nil || !isPathInDirectory(filePath, h.s.GetConfig().DataDir) { - return errs.ErrFilePathAbs.Wrap(err).FastGenWithCause() + return errs.ErrFilePathAbs.Wrap(err) } c.LoadPlugin(pluginPath, ch) From 7b60e0928d35bbdb1914850040825684853d63f0 Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Thu, 14 Dec 2023 18:21:20 +0800 Subject: [PATCH 095/137] mcs: fix sequence of callback functions (#7548) close tikv/pd#7543 Signed-off-by: Ryan Leung --- pkg/mcs/resourcemanager/server/server.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/mcs/resourcemanager/server/server.go b/pkg/mcs/resourcemanager/server/server.go index 2a1be3e0ca5..43d426bfc40 100644 --- a/pkg/mcs/resourcemanager/server/server.go +++ b/pkg/mcs/resourcemanager/server/server.go @@ -321,13 +321,15 @@ func (s *Server) startServer() (err error) { s.serverLoopWg.Add(1) go utils.StartGRPCAndHTTPServers(s, serverReadyChan, s.GetListener()) <-serverReadyChan - s.startServerLoop() // Run callbacks log.Info("triggering the start callback functions") for _, cb := range s.GetStartCallbacks() { cb() } + // The start callback function will initialize storage, which will be used in service ready callback. + // We should make sure the calling sequence is right. + s.startServerLoop() // Server has started. entry := &discovery.ServiceRegistryEntry{ServiceAddr: s.cfg.AdvertiseListenAddr} From 8196d84c04fb4fcd0d99a57304aa242655381056 Mon Sep 17 00:00:00 2001 From: Hu# Date: Fri, 15 Dec 2023 17:27:20 +0800 Subject: [PATCH 096/137] makefile: update golangci (#7556) close tikv/pd#7551 Signed-off-by: husharp --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 946493cd7ce..67cdac99b02 100644 --- a/Makefile +++ b/Makefile @@ -160,7 +160,7 @@ SHELL := env PATH='$(PATH)' GOBIN='$(GO_TOOLS_BIN_PATH)' $(shell which bash) install-tools: @mkdir -p $(GO_TOOLS_BIN_PATH) - @which golangci-lint >/dev/null 2>&1 || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GO_TOOLS_BIN_PATH) v1.51.2 + @which golangci-lint >/dev/null 2>&1 || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GO_TOOLS_BIN_PATH) v1.55.2 @grep '_' tools.go | sed 's/"//g' | awk '{print $$2}' | xargs go install .PHONY: install-tools From a4ab7d31da67acb2dcc5945b973e18d2a0f10d1b Mon Sep 17 00:00:00 2001 From: Hu# Date: Mon, 18 Dec 2023 14:06:22 +0800 Subject: [PATCH 097/137] ci: support real cluster test in jenkins (#7493) ref tikv/pd#7298 Signed-off-by: husharp --- Makefile | 7 +- go.mod | 4 +- go.sum | 8 +- pkg/member/member.go | 4 +- tests/integrations/client/go.mod | 2 +- tests/integrations/mcs/go.mod | 2 +- tests/integrations/realtiup/Makefile | 58 ++++ tests/integrations/realtiup/deploy.sh | 23 ++ tests/integrations/realtiup/go.mod | 47 ++++ tests/integrations/realtiup/go.sum | 252 ++++++++++++++++++ tests/integrations/realtiup/mock_db.go | 91 +++++++ tests/integrations/realtiup/reboot_pd_test.go | 71 +++++ .../realtiup/transfer_leader_test.go | 73 +++++ tests/integrations/realtiup/ts_test.go | 45 ++++ tests/integrations/realtiup/util.go | 39 +++ tests/integrations/realtiup/wait_tiup.sh | 22 ++ tests/integrations/tso/go.mod | 2 +- tools/pd-api-bench/go.mod | 2 +- tools/pd-simulator/main.go | 2 +- 19 files changed, 740 insertions(+), 14 deletions(-) create mode 100644 tests/integrations/realtiup/Makefile create mode 100755 tests/integrations/realtiup/deploy.sh create mode 100644 tests/integrations/realtiup/go.mod create mode 100644 tests/integrations/realtiup/go.sum create mode 100644 tests/integrations/realtiup/mock_db.go create mode 100644 tests/integrations/realtiup/reboot_pd_test.go create mode 100644 tests/integrations/realtiup/transfer_leader_test.go create mode 100644 tests/integrations/realtiup/ts_test.go create mode 100644 tests/integrations/realtiup/util.go create mode 100755 tests/integrations/realtiup/wait_tiup.sh diff --git a/Makefile b/Makefile index 67cdac99b02..2a506eb576f 100644 --- a/Makefile +++ b/Makefile @@ -55,7 +55,12 @@ ifeq ($(ENABLE_FIPS), 1) BUILD_TOOL_CGO_ENABLED := 1 endif -LDFLAGS += -X "$(PD_PKG)/pkg/versioninfo.PDReleaseVersion=$(shell git describe --tags --dirty --always)" +RELEASE_VERSION ?= $(shell git describe --tags --dirty --always) +ifeq ($(RUN_CI), 1) + RELEASE_VERSION := None +endif + +LDFLAGS += -X "$(PD_PKG)/pkg/versioninfo.PDReleaseVersion=$(RELEASE_VERSION)" LDFLAGS += -X "$(PD_PKG)/pkg/versioninfo.PDBuildTS=$(shell date -u '+%Y-%m-%d %I:%M:%S')" LDFLAGS += -X "$(PD_PKG)/pkg/versioninfo.PDGitHash=$(shell git rev-parse HEAD)" LDFLAGS += -X "$(PD_PKG)/pkg/versioninfo.PDGitBranch=$(shell git rev-parse --abbrev-ref HEAD)" diff --git a/go.mod b/go.mod index 676d350d22d..d5cbc41f654 100644 --- a/go.mod +++ b/go.mod @@ -55,7 +55,7 @@ require ( go.uber.org/atomic v1.10.0 go.uber.org/goleak v1.1.12 go.uber.org/zap v1.24.0 - golang.org/x/exp v0.0.0-20230108222341-4b8118a2686a + golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4 golang.org/x/text v0.13.0 golang.org/x/time v0.1.0 golang.org/x/tools v0.6.0 @@ -185,7 +185,7 @@ require ( golang.org/x/arch v0.3.0 // indirect golang.org/x/crypto v0.14.0 // indirect golang.org/x/image v0.5.0 // indirect - golang.org/x/mod v0.8.0 // indirect + golang.org/x/mod v0.11.0 // indirect golang.org/x/net v0.17.0 // indirect golang.org/x/oauth2 v0.4.0 // indirect golang.org/x/sync v0.1.0 // indirect diff --git a/go.sum b/go.sum index c7ceeee028c..bf35be7eb8c 100644 --- a/go.sum +++ b/go.sum @@ -683,8 +683,8 @@ golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20230108222341-4b8118a2686a h1:tlXy25amD5A7gOfbXdqCGN5k8ESEed/Ee1E5RcrYnqU= -golang.org/x/exp v0.0.0-20230108222341-4b8118a2686a/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= +golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4 h1:QLureRX3moex6NVu/Lr4MGakp9FdA7sBHGBmvRW7NaM= +golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/image v0.0.0-20200119044424-58c23975cae1/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.5.0 h1:5JMiNunQeQw++mMOz48/ISeNu3Iweh/JaZU8ZLqHRrI= golang.org/x/image v0.5.0/go.mod h1:FVC7BI/5Ym8R25iw5OLsgshdUBbT1h5jZTpA+mvAdZ4= @@ -700,8 +700,8 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= -golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU= +golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= diff --git a/pkg/member/member.go b/pkg/member/member.go index b411d0c957b..1b901a1d04a 100644 --- a/pkg/member/member.go +++ b/pkg/member/member.go @@ -57,7 +57,7 @@ type EmbeddedEtcdMember struct { id uint64 // etcd server id. member *pdpb.Member // current PD's info. rootPath string - // memberValue is the serialized string of `member`. It will be save in + // memberValue is the serialized string of `member`. It will be saved in // etcd leader key when the PD node is successfully elected as the PD leader // of the cluster. Every write will use it to check PD leadership. memberValue string @@ -199,7 +199,7 @@ func (m *EmbeddedEtcdMember) KeepLeader(ctx context.Context) { m.leadership.Keep(ctx) } -// PreCheckLeader does some pre-check before checking whether or not it's the leader. +// PreCheckLeader does some pre-check before checking whether it's the leader. func (m *EmbeddedEtcdMember) PreCheckLeader() error { if m.GetEtcdLeader() == 0 { return errs.ErrEtcdLeaderNotFound diff --git a/tests/integrations/client/go.mod b/tests/integrations/client/go.mod index 799901ff2e3..da130278ae0 100644 --- a/tests/integrations/client/go.mod +++ b/tests/integrations/client/go.mod @@ -17,7 +17,7 @@ require ( github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 github.com/stretchr/testify v1.8.3 github.com/tikv/pd v0.0.0-00010101000000-000000000000 - github.com/tikv/pd/client v0.0.0-00010101000000-000000000000 + github.com/tikv/pd/client v0.0.0-20231101084237-a1a1eea8dafd go.etcd.io/etcd v0.5.0-alpha.5.0.20220915004622-85b640cee793 go.uber.org/goleak v1.1.12 go.uber.org/zap v1.24.0 diff --git a/tests/integrations/mcs/go.mod b/tests/integrations/mcs/go.mod index 75d70e3cf06..1823a224fa1 100644 --- a/tests/integrations/mcs/go.mod +++ b/tests/integrations/mcs/go.mod @@ -17,7 +17,7 @@ require ( github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 github.com/stretchr/testify v1.8.3 github.com/tikv/pd v0.0.0-00010101000000-000000000000 - github.com/tikv/pd/client v0.0.0-00010101000000-000000000000 + github.com/tikv/pd/client v0.0.0-20231101084237-a1a1eea8dafd go.etcd.io/etcd v0.5.0-alpha.5.0.20220915004622-85b640cee793 go.uber.org/goleak v1.1.12 go.uber.org/zap v1.24.0 diff --git a/tests/integrations/realtiup/Makefile b/tests/integrations/realtiup/Makefile new file mode 100644 index 00000000000..c9ffd3d6599 --- /dev/null +++ b/tests/integrations/realtiup/Makefile @@ -0,0 +1,58 @@ +# Copyright 2023 TiKV Project Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ROOT_PATH := ../../.. +GO_TOOLS_BIN_PATH := $(ROOT_PATH)/.tools/bin +PATH := $(GO_TOOLS_BIN_PATH):$(PATH) +SHELL := env PATH='$(PATH)' GOBIN='$(GO_TOOLS_BIN_PATH)' $(shell which bash) + +static: install-tools + @ echo "gofmt ..." + @ gofmt -s -l -d . 2>&1 | awk '{ print } END { if (NR > 0) { exit 1 } }' + @ echo "golangci-lint ..." + @ golangci-lint run -c $(ROOT_PATH)/.golangci.yml --verbose ./... --allow-parallel-runners + @ echo "revive ..." + @ revive -formatter friendly -config $(ROOT_PATH)/revive.toml ./... + +tidy: + @ go mod tidy + git diff go.mod go.sum | cat + git diff --quiet go.mod go.sum + +check: deploy test kill_tiup + +deploy: kill_tiup + @ echo "deploying..." + ./deploy.sh + @ echo "wait tiup cluster ready..." + ./wait_tiup.sh 15 20 + @ echo "check cluster status..." + @ pid=$$(ps -ef | grep 'tiup' | grep -v grep | awk '{print $$2}' | head -n 1); \ + echo $$pid; + +kill_tiup: + @ echo "kill tiup..." + @ pid=$$(ps -ef | grep 'tiup' | grep -v grep | awk '{print $$2}' | head -n 1); \ + if [ ! -z "$$pid" ]; then \ + echo $$pid; \ + kill $$pid; \ + echo "waiting for tiup to exit..."; \ + sleep 10; \ + fi + +test: + CGO_ENABLED=1 go test ./... -v -tags deadlock -race -cover || { exit 1; } + +install-tools: + cd $(ROOT_PATH) && $(MAKE) install-tools diff --git a/tests/integrations/realtiup/deploy.sh b/tests/integrations/realtiup/deploy.sh new file mode 100755 index 00000000000..18e6de7f0b9 --- /dev/null +++ b/tests/integrations/realtiup/deploy.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# deploy `tiup playground` + +TIUP_BIN_DIR=$HOME/.tiup/bin/tiup +CUR_PATH=$(pwd) + +# See https://misc.flogisoft.com/bash/tip_colors_and_formatting. +color-green() { # Green + echo -e "\x1B[1;32m${*}\x1B[0m" +} + +# Install TiUP +color-green "install TiUP..." +curl --proto '=https' --tlsv1.2 -sSf https://tiup-mirrors.pingcap.com/install.sh | sh +$TIUP_BIN_DIR update playground + +cd ../../.. +# Run TiUP +$TIUP_BIN_DIR playground nightly --kv 3 --tiflash 1 --db 1 --pd 3 --without-monitor \ + --pd.binpath ./bin/pd-server --kv.binpath ./bin/tikv-server --db.binpath ./bin/tidb-server --tiflash.binpath ./bin/tiflash --tag pd_test \ + > $CUR_PATH/playground.log 2>&1 & + +cd $CUR_PATH diff --git a/tests/integrations/realtiup/go.mod b/tests/integrations/realtiup/go.mod new file mode 100644 index 00000000000..ccb23548f3e --- /dev/null +++ b/tests/integrations/realtiup/go.mod @@ -0,0 +1,47 @@ +module github.com/tikv/pd/tests/integrations/realtiup + +go 1.21 + +replace github.com/tikv/pd/client => ../../../client + +require ( + github.com/DATA-DOG/go-sqlmock v1.5.0 + github.com/go-sql-driver/mysql v1.7.1 + github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 + github.com/stretchr/testify v1.8.4 + github.com/tikv/pd/client v0.0.0-00010101000000-000000000000 + gorm.io/driver/mysql v1.5.2 + gorm.io/gorm v1.25.5 + moul.io/zapgorm2 v1.3.0 +) + +require ( + github.com/benbjohnson/clock v1.3.0 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/jinzhu/inflection v1.0.0 // indirect + github.com/jinzhu/now v1.1.5 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect + github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c // indirect + github.com/pingcap/kvproto v0.0.0-20230727073445-53e1f8730c30 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/client_golang v1.11.1 // indirect + github.com/prometheus/client_model v0.2.0 // indirect + github.com/prometheus/common v0.26.0 // indirect + github.com/prometheus/procfs v0.6.0 // indirect + go.uber.org/atomic v1.10.0 // indirect + go.uber.org/goleak v1.1.12 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.24.0 // indirect + golang.org/x/net v0.17.0 // indirect + golang.org/x/sys v0.13.0 // indirect + golang.org/x/text v0.13.0 // indirect + google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect + google.golang.org/grpc v1.54.0 // indirect + google.golang.org/protobuf v1.30.0 // indirect + gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/tests/integrations/realtiup/go.sum b/tests/integrations/realtiup/go.sum new file mode 100644 index 00000000000..fde38211174 --- /dev/null +++ b/tests/integrations/realtiup/go.sum @@ -0,0 +1,252 @@ +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= +github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= +github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= +github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI= +github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= +github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v1.1.4/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= +github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= +github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c h1:xpW9bvK+HuuTmyFqUwr+jcCvpVkK7sumiz+ko5H9eq4= +github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg= +github.com/pingcap/kvproto v0.0.0-20230727073445-53e1f8730c30 h1:EvqKcDT7ceGLW0mXqM8Cp5Z8DfgQRnwj2YTnlCLj2QI= +github.com/pingcap/kvproto v0.0.0-20230727073445-53e1f8730c30/go.mod h1:r0q/CFcwvyeRhKtoqzmWMBebrtpIziQQ9vR+JKh1knc= +github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8IDP+SZrdhV1Kibl9KrHxJ9eciw= +github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= +github.com/prometheus/client_golang v1.11.1 h1:+4eQaD7vAZ6DsfsxB15hbE0odUjGI5ARs9yskGu1v4s= +github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ= +github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= +github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= +go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= +go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= +go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= +go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= +go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= +go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= +google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= +google.golang.org/grpc v1.54.0 h1:EhTqbhiYeixwWQtAEZAxmV9MGqcjEU2mFx52xCzNyag= +google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= +gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= +gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gorm.io/driver/mysql v1.5.2 h1:QC2HRskSE75wBuOxe0+iCkyJZ+RqpudsQtqkp+IMuXs= +gorm.io/driver/mysql v1.5.2/go.mod h1:pQLhh1Ut/WUAySdTHwBpBv6+JKcj+ua4ZFx1QQTBzb8= +gorm.io/gorm v1.23.6/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk= +gorm.io/gorm v1.25.2-0.20230530020048-26663ab9bf55/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k= +gorm.io/gorm v1.25.5 h1:zR9lOiiYf09VNh5Q1gphfyia1JpiClIWG9hQaxB/mls= +gorm.io/gorm v1.25.5/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= +moul.io/zapgorm2 v1.3.0 h1:+CzUTMIcnafd0d/BvBce8T4uPn6DQnpIrz64cyixlkk= +moul.io/zapgorm2 v1.3.0/go.mod h1:nPVy6U9goFKHR4s+zfSo1xVFaoU7Qgd5DoCdOfzoCqs= diff --git a/tests/integrations/realtiup/mock_db.go b/tests/integrations/realtiup/mock_db.go new file mode 100644 index 00000000000..95f3af8a06c --- /dev/null +++ b/tests/integrations/realtiup/mock_db.go @@ -0,0 +1,91 @@ +// Copyright 2023 TiKV Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package realtiup + +import ( + "testing" + "time" + + "github.com/DATA-DOG/go-sqlmock" + mysqldriver "github.com/go-sql-driver/mysql" + "github.com/pingcap/log" + "github.com/stretchr/testify/require" + "gorm.io/driver/mysql" + "gorm.io/gorm" + "moul.io/zapgorm2" +) + +// TestDB is a test database +type TestDB struct { + inner *gorm.DB + require *require.Assertions + + isUnderlyingMocked bool + mock sqlmock.Sqlmock +} + +// OpenTestDB opens a test database +func OpenTestDB(t *testing.T, configModifier ...func(*mysqldriver.Config, *gorm.Config)) *TestDB { + r := require.New(t) + + dsn := mysqldriver.NewConfig() + dsn.Net = "tcp" + dsn.Addr = "127.0.0.1:4000" + dsn.Params = map[string]string{"time_zone": "'+00:00'"} + dsn.ParseTime = true + dsn.Loc = time.UTC + dsn.User = "root" + dsn.DBName = "test" + + config := &gorm.Config{ + Logger: zapgorm2.New(log.L()), + } + + for _, m := range configModifier { + m(dsn, config) + } + + db, err := gorm.Open(mysql.Open(dsn.FormatDSN()), config) + r.NoError(err) + + return &TestDB{ + inner: db.Debug(), + require: r, + } +} + +// MustClose closes the test database +func (db *TestDB) MustClose() { + if db.isUnderlyingMocked { + db.mock.ExpectClose() + } + + d, err := db.inner.DB() + db.require.NoError(err) + + err = d.Close() + db.require.NoError(err) +} + +// Gorm returns the underlying gorm.DB +func (db *TestDB) Gorm() *gorm.DB { + return db.inner +} + +// MustExec executes a query +func (db *TestDB) MustExec(sql string, values ...interface{}) { + err := db.inner.Exec(sql, values...).Error + db.require.NoError(err) +} diff --git a/tests/integrations/realtiup/reboot_pd_test.go b/tests/integrations/realtiup/reboot_pd_test.go new file mode 100644 index 00000000000..bccf465bde0 --- /dev/null +++ b/tests/integrations/realtiup/reboot_pd_test.go @@ -0,0 +1,71 @@ +// Copyright 2023 TiKV Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package realtiup + +import ( + "context" + "os/exec" + "testing" + + "github.com/pingcap/log" + "github.com/stretchr/testify/require" +) + +func restartTiUP() { + log.Info("start to restart TiUP") + cmd := exec.Command("make", "deploy") + err := cmd.Run() + if err != nil { + panic(err) + } + log.Info("TiUP restart success") +} + +// https://github.com/tikv/pd/issues/6467 +func TestReloadLabel(t *testing.T) { + re := require.New(t) + ctx := context.Background() + + resp, _ := pdHTTPCli.GetStores(ctx) + setStore := resp.Stores[0] + re.Empty(setStore.Store.Labels, nil) + storeLabel := map[string]string{ + "zone": "zone1", + } + err := pdHTTPCli.SetStoreLabels(ctx, setStore.Store.ID, storeLabel) + re.NoError(err) + + resp, err = pdHTTPCli.GetStores(ctx) + re.NoError(err) + for _, store := range resp.Stores { + if store.Store.ID == setStore.Store.ID { + for _, label := range store.Store.Labels { + re.Equal(label.Value, storeLabel[label.Key]) + } + } + } + + restartTiUP() + + resp, err = pdHTTPCli.GetStores(ctx) + re.NoError(err) + for _, store := range resp.Stores { + if store.Store.ID == setStore.Store.ID { + for _, label := range store.Store.Labels { + re.Equal(label.Value, storeLabel[label.Key]) + } + } + } +} diff --git a/tests/integrations/realtiup/transfer_leader_test.go b/tests/integrations/realtiup/transfer_leader_test.go new file mode 100644 index 00000000000..51142be03f9 --- /dev/null +++ b/tests/integrations/realtiup/transfer_leader_test.go @@ -0,0 +1,73 @@ +// Copyright 2023 TiKV Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package realtiup + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// https://github.com/tikv/pd/issues/6988#issuecomment-1694924611 +// https://github.com/tikv/pd/issues/6897 +func TestTransferLeader(t *testing.T) { + re := require.New(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + resp, err := pdHTTPCli.GetLeader(ctx) + re.NoError(err) + oldLeader := resp.Name + + var newLeader string + for i := 0; i < 2; i++ { + if resp.Name != fmt.Sprintf("pd-%d", i) { + newLeader = fmt.Sprintf("pd-%d", i) + } + } + + // record scheduler + err = pdHTTPCli.CreateScheduler(ctx, "evict-leader-scheduler", 1) + re.NoError(err) + res, err := pdHTTPCli.GetSchedulers(ctx) + re.NoError(err) + oldSchedulersLen := len(res) + + re.NoError(pdHTTPCli.TransferLeader(ctx, newLeader)) + // wait for transfer leader to new leader + time.Sleep(1 * time.Second) + resp, err = pdHTTPCli.GetLeader(ctx) + re.NoError(err) + re.Equal(newLeader, resp.Name) + + res, err = pdHTTPCli.GetSchedulers(ctx) + re.NoError(err) + re.Equal(oldSchedulersLen, len(res)) + + // transfer leader to old leader + re.NoError(pdHTTPCli.TransferLeader(ctx, oldLeader)) + // wait for transfer leader + time.Sleep(1 * time.Second) + resp, err = pdHTTPCli.GetLeader(ctx) + re.NoError(err) + re.Equal(oldLeader, resp.Name) + + res, err = pdHTTPCli.GetSchedulers(ctx) + re.NoError(err) + re.Equal(oldSchedulersLen, len(res)) +} diff --git a/tests/integrations/realtiup/ts_test.go b/tests/integrations/realtiup/ts_test.go new file mode 100644 index 00000000000..9bf8aee2d49 --- /dev/null +++ b/tests/integrations/realtiup/ts_test.go @@ -0,0 +1,45 @@ +// Copyright 2023 TiKV Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package realtiup + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestTS(t *testing.T) { + re := require.New(t) + + db := OpenTestDB(t) + db.MustExec("use test") + db.MustExec("drop table if exists t") + db.MustExec("create table t(a int, index i(a))") + db.MustExec("insert t values (1), (2), (3)") + var rows int + err := db.inner.Raw("select count(*) from t").Row().Scan(&rows) + re.NoError(err) + re.Equal(3, rows) + + re.NoError(err) + re.Equal(3, rows) + + var ts uint64 + err = db.inner.Begin().Raw("select @@tidb_current_ts").Scan(&ts).Rollback().Error + re.NoError(err) + re.NotEqual(0, GetTimeFromTS(ts)) + + db.MustClose() +} diff --git a/tests/integrations/realtiup/util.go b/tests/integrations/realtiup/util.go new file mode 100644 index 00000000000..66d6127b5c4 --- /dev/null +++ b/tests/integrations/realtiup/util.go @@ -0,0 +1,39 @@ +// Copyright 2023 TiKV Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package realtiup + +import ( + "time" + + "github.com/tikv/pd/client/http" +) + +const physicalShiftBits = 18 + +var ( + pdAddrs = []string{"127.0.0.1:2379"} + pdHTTPCli = http.NewClient(pdAddrs) +) + +// GetTimeFromTS extracts time.Time from a timestamp. +func GetTimeFromTS(ts uint64) time.Time { + ms := ExtractPhysical(ts) + return time.Unix(ms/1e3, (ms%1e3)*1e6) +} + +// ExtractPhysical returns a ts's physical part. +func ExtractPhysical(ts uint64) int64 { + return int64(ts >> physicalShiftBits) +} diff --git a/tests/integrations/realtiup/wait_tiup.sh b/tests/integrations/realtiup/wait_tiup.sh new file mode 100755 index 00000000000..497774f9e96 --- /dev/null +++ b/tests/integrations/realtiup/wait_tiup.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# Wait until `tiup playground` command runs success + +TIUP_BIN_DIR=$HOME/.tiup/bin/tiup +INTERVAL=$1 +MAX_TIMES=$2 + +if ([ -z "${INTERVAL}" ] || [ -z "${MAX_TIMES}" ]); then + echo "Usage: command " + exit 1 +fi + +for ((i=0; i<${MAX_TIMES}; i++)); do + sleep ${INTERVAL} + $TIUP_BIN_DIR playground display --tag pd_test + if [ $? -eq 0 ]; then + exit 0 + fi + cat ./playground.log +done + +exit 1 \ No newline at end of file diff --git a/tests/integrations/tso/go.mod b/tests/integrations/tso/go.mod index 309ea9dbc4d..0d734716ed5 100644 --- a/tests/integrations/tso/go.mod +++ b/tests/integrations/tso/go.mod @@ -16,7 +16,7 @@ require ( github.com/pingcap/kvproto v0.0.0-20231018065736-c0689aded40c github.com/stretchr/testify v1.8.4 github.com/tikv/pd v0.0.0-00010101000000-000000000000 - github.com/tikv/pd/client v0.0.0-00010101000000-000000000000 + github.com/tikv/pd/client v0.0.0-20231101084237-a1a1eea8dafd github.com/tikv/pd/tests/integrations/mcs v0.0.0-00010101000000-000000000000 google.golang.org/grpc v1.54.0 ) diff --git a/tools/pd-api-bench/go.mod b/tools/pd-api-bench/go.mod index 8050f433e8b..77b4891d3d8 100644 --- a/tools/pd-api-bench/go.mod +++ b/tools/pd-api-bench/go.mod @@ -4,7 +4,7 @@ go 1.21 require ( github.com/tikv/pd v0.0.0-00010101000000-000000000000 - github.com/tikv/pd/client v0.0.0-00010101000000-000000000000 + github.com/tikv/pd/client v0.0.0-20231101084237-a1a1eea8dafd go.uber.org/zap v1.24.0 google.golang.org/grpc v1.54.0 ) diff --git a/tools/pd-simulator/main.go b/tools/pd-simulator/main.go index 60d8874d083..5d781757b39 100644 --- a/tools/pd-simulator/main.go +++ b/tools/pd-simulator/main.go @@ -56,7 +56,7 @@ var ( ) func main() { - // wait PD start. Otherwise it will happen error when getting cluster ID. + // wait PD start. Otherwise, it will happen error when getting cluster ID. time.Sleep(3 * time.Second) // ignore some undefined flag flag.CommandLine.ParseErrorsWhitelist.UnknownFlags = true From a16f99ee5cdaaef29acdd8b6d1a7f66be0269d45 Mon Sep 17 00:00:00 2001 From: Hu# Date: Mon, 18 Dec 2023 14:37:52 +0800 Subject: [PATCH 098/137] api: support mcs api for members (#7372) ref tikv/pd#7519 Signed-off-by: husharp --- client/http/api.go | 7 ++ client/http/client.go | 14 +++ pkg/mcs/discovery/discover.go | 42 ++++++++ pkg/mcs/tso/server/apis/v1/api.go | 23 ++++ pkg/mcs/utils/util.go | 6 +- server/apiv2/handlers/micro_service.go | 57 ++++++++++ server/apiv2/router.go | 1 + tests/integrations/mcs/members/member_test.go | 100 ++++++++++++++++++ 8 files changed, 247 insertions(+), 3 deletions(-) create mode 100644 server/apiv2/handlers/micro_service.go create mode 100644 tests/integrations/mcs/members/member_test.go diff --git a/client/http/api.go b/client/http/api.go index 2153cd286e8..4cc169c6a33 100644 --- a/client/http/api.go +++ b/client/http/api.go @@ -75,6 +75,8 @@ const ( MinResolvedTSPrefix = "/pd/api/v1/min-resolved-ts" Status = "/pd/api/v1/status" Version = "/pd/api/v1/version" + // Micro Service + microServicePrefix = "/pd/api/v2/ms" ) // RegionByID returns the path of PD HTTP API to get region by ID. @@ -186,3 +188,8 @@ func PProfProfileAPIWithInterval(interval time.Duration) string { func PProfGoroutineWithDebugLevel(level int) string { return fmt.Sprintf("%s?debug=%d", PProfGoroutine, level) } + +// MicroServiceMembers returns the path of PD HTTP API to get the members of microservice. +func MicroServiceMembers(service string) string { + return fmt.Sprintf("%s/members/%s", microServicePrefix, service) +} diff --git a/client/http/client.go b/client/http/client.go index d74c77571d6..927450b74a2 100644 --- a/client/http/client.go +++ b/client/http/client.go @@ -89,6 +89,8 @@ type Client interface { AccelerateScheduleInBatch(context.Context, []*KeyRange) error /* Other interfaces */ GetMinResolvedTSByStoresIDs(context.Context, []uint64) (uint64, map[uint64]uint64, error) + /* Micro Service interfaces */ + GetMicroServiceMembers(context.Context, string) ([]string, error) /* Client-related methods */ // WithCallerID sets and returns a new client with the given caller ID. @@ -844,3 +846,15 @@ func (c *client) GetMinResolvedTSByStoresIDs(ctx context.Context, storeIDs []uin } return resp.MinResolvedTS, resp.StoresMinResolvedTS, nil } + +// GetMicroServiceMembers gets the members of the microservice. +func (c *client) GetMicroServiceMembers(ctx context.Context, service string) ([]string, error) { + var members []string + err := c.requestWithRetry(ctx, + "GetMicroServiceMembers", MicroServiceMembers(service), + http.MethodGet, nil, &members) + if err != nil { + return nil, err + } + return members, nil +} diff --git a/pkg/mcs/discovery/discover.go b/pkg/mcs/discovery/discover.go index 00e168114b0..89c45497a87 100644 --- a/pkg/mcs/discovery/discover.go +++ b/pkg/mcs/discovery/discover.go @@ -15,8 +15,16 @@ package discovery import ( + "strconv" + + "github.com/pingcap/errors" + "github.com/pingcap/log" + "github.com/tikv/pd/pkg/errs" + "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/storage/kv" "github.com/tikv/pd/pkg/utils/etcdutil" "go.etcd.io/etcd/clientv3" + "go.uber.org/zap" ) // Discover is used to get all the service instances of the specified service name. @@ -35,3 +43,37 @@ func Discover(cli *clientv3.Client, clusterID, serviceName string) ([]string, er } return values, nil } + +// GetMSMembers returns all the members of the specified service name. +func GetMSMembers(name string, client *clientv3.Client) ([]string, error) { + switch name { + case utils.TSOServiceName, utils.SchedulingServiceName, utils.ResourceManagerServiceName: + clusterID, err := etcdutil.GetClusterID(client, utils.ClusterIDPath) + if err != nil { + return nil, err + } + servicePath := ServicePath(strconv.FormatUint(clusterID, 10), name) + resps, err := kv.NewSlowLogTxn(client).Then(clientv3.OpGet(servicePath, clientv3.WithPrefix())).Commit() + if err != nil { + return nil, errs.ErrEtcdKVGet.Wrap(err).GenWithStackByCause() + } + if !resps.Succeeded { + return nil, errs.ErrEtcdTxnConflict.FastGenByArgs() + } + + var addrs []string + for _, resp := range resps.Responses { + for _, keyValue := range resp.GetResponseRange().GetKvs() { + var entry ServiceRegistryEntry + if err = entry.Deserialize(keyValue.Value); err != nil { + log.Error("try to deserialize service registry entry failed", zap.String("key", string(keyValue.Key)), zap.Error(err)) + continue + } + addrs = append(addrs, entry.ServiceAddr) + } + } + return addrs, nil + } + + return nil, errors.Errorf("unknown service name %s", name) +} diff --git a/pkg/mcs/tso/server/apis/v1/api.go b/pkg/mcs/tso/server/apis/v1/api.go index 33e1e0801aa..e5f0dfb5440 100644 --- a/pkg/mcs/tso/server/apis/v1/api.go +++ b/pkg/mcs/tso/server/apis/v1/api.go @@ -102,6 +102,7 @@ func NewService(srv *tsoserver.Service) *Service { } s.RegisterAdminRouter() s.RegisterKeyspaceGroupRouter() + s.RegisterHealth() return s } @@ -118,6 +119,12 @@ func (s *Service) RegisterKeyspaceGroupRouter() { router.GET("/members", GetKeyspaceGroupMembers) } +// RegisterHealth registers the router of the health handler. +func (s *Service) RegisterHealth() { + router := s.root.Group("health") + router.GET("", GetHealth) +} + func changeLogLevel(c *gin.Context) { svr := c.MustGet(multiservicesapi.ServiceContextKey).(*tsoserver.Service) var level string @@ -201,6 +208,22 @@ func ResetTS(c *gin.Context) { c.String(http.StatusOK, "Reset ts successfully.") } +// GetHealth returns the health status of the TSO service. +func GetHealth(c *gin.Context) { + svr := c.MustGet(multiservicesapi.ServiceContextKey).(*tsoserver.Service) + am, err := svr.GetKeyspaceGroupManager().GetAllocatorManager(utils.DefaultKeyspaceGroupID) + if err != nil { + c.String(http.StatusInternalServerError, err.Error()) + return + } + if am.GetMember().IsLeaderElected() { + c.IndentedJSON(http.StatusOK, "ok") + return + } + + c.String(http.StatusInternalServerError, "no leader elected") +} + // KeyspaceGroupMember contains the keyspace group and its member information. type KeyspaceGroupMember struct { Group *endpoint.KeyspaceGroup diff --git a/pkg/mcs/utils/util.go b/pkg/mcs/utils/util.go index 682e73f20ae..a0708f9bf88 100644 --- a/pkg/mcs/utils/util.go +++ b/pkg/mcs/utils/util.go @@ -45,8 +45,8 @@ import ( const ( // maxRetryTimes is the max retry times for initializing the cluster ID. maxRetryTimes = 5 - // clusterIDPath is the path to store cluster id - clusterIDPath = "/pd/cluster_id" + // ClusterIDPath is the path to store cluster id + ClusterIDPath = "/pd/cluster_id" // retryInterval is the interval to retry. retryInterval = time.Second ) @@ -56,7 +56,7 @@ func InitClusterID(ctx context.Context, client *clientv3.Client) (id uint64, err ticker := time.NewTicker(retryInterval) defer ticker.Stop() for i := 0; i < maxRetryTimes; i++ { - if clusterID, err := etcdutil.GetClusterID(client, clusterIDPath); err == nil && clusterID != 0 { + if clusterID, err := etcdutil.GetClusterID(client, ClusterIDPath); err == nil && clusterID != 0 { return clusterID, nil } select { diff --git a/server/apiv2/handlers/micro_service.go b/server/apiv2/handlers/micro_service.go new file mode 100644 index 00000000000..3c2be3748d4 --- /dev/null +++ b/server/apiv2/handlers/micro_service.go @@ -0,0 +1,57 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package handlers + +import ( + "net/http" + + "github.com/gin-gonic/gin" + "github.com/tikv/pd/pkg/mcs/discovery" + "github.com/tikv/pd/server" + "github.com/tikv/pd/server/apiv2/middlewares" +) + +// RegisterMicroService registers microservice handler to the router. +func RegisterMicroService(r *gin.RouterGroup) { + router := r.Group("ms") + router.Use(middlewares.BootstrapChecker()) + router.GET("members/:service", GetMembers) +} + +// GetMembers gets all members of the cluster for the specified service. +// @Tags members +// @Summary Get all members of the cluster for the specified service. +// @Produce json +// @Success 200 {object} []string +// @Router /ms/members/{service} [get] +func GetMembers(c *gin.Context) { + svr := c.MustGet(middlewares.ServerContextKey).(*server.Server) + if !svr.IsAPIServiceMode() { + c.AbortWithStatusJSON(http.StatusServiceUnavailable, "not support micro service") + return + } + + if service := c.Param("service"); len(service) > 0 { + addrs, err := discovery.GetMSMembers(service, svr.GetClient()) + if err != nil { + c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) + return + } + c.IndentedJSON(http.StatusOK, addrs) + return + } + + c.AbortWithStatusJSON(http.StatusInternalServerError, "please specify service") +} diff --git a/server/apiv2/router.go b/server/apiv2/router.go index 383d336caae..fd3ce38c0e4 100644 --- a/server/apiv2/router.go +++ b/server/apiv2/router.go @@ -64,5 +64,6 @@ func NewV2Handler(_ context.Context, svr *server.Server) (http.Handler, apiutil. root := router.Group(apiV2Prefix) handlers.RegisterKeyspace(root) handlers.RegisterTSOKeyspaceGroup(root) + handlers.RegisterMicroService(root) return router, group, nil } diff --git a/tests/integrations/mcs/members/member_test.go b/tests/integrations/mcs/members/member_test.go new file mode 100644 index 00000000000..d1ccb86a1c7 --- /dev/null +++ b/tests/integrations/mcs/members/member_test.go @@ -0,0 +1,100 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package members_test + +import ( + "context" + "testing" + + "github.com/stretchr/testify/suite" + pdClient "github.com/tikv/pd/client/http" + bs "github.com/tikv/pd/pkg/basicserver" + "github.com/tikv/pd/pkg/mcs/utils" + "github.com/tikv/pd/pkg/utils/tempurl" + "github.com/tikv/pd/pkg/utils/testutil" + "github.com/tikv/pd/tests" +) + +type memberTestSuite struct { + suite.Suite + ctx context.Context + cleanupFunc []testutil.CleanupFunc + cluster *tests.TestCluster + server *tests.TestServer + backendEndpoints string + dialClient pdClient.Client +} + +func TestMemberTestSuite(t *testing.T) { + suite.Run(t, new(memberTestSuite)) +} + +func (suite *memberTestSuite) SetupTest() { + ctx, cancel := context.WithCancel(context.Background()) + suite.ctx = ctx + cluster, err := tests.NewTestAPICluster(suite.ctx, 1) + suite.cluster = cluster + suite.NoError(err) + suite.NoError(cluster.RunInitialServers()) + suite.NotEmpty(cluster.WaitLeader()) + suite.server = cluster.GetLeaderServer() + suite.NoError(suite.server.BootstrapCluster()) + suite.backendEndpoints = suite.server.GetAddr() + suite.dialClient = pdClient.NewClient([]string{suite.server.GetAddr()}) + + // TSO + nodes := make(map[string]bs.Server) + for i := 0; i < utils.DefaultKeyspaceGroupReplicaCount; i++ { + s, cleanup := tests.StartSingleTSOTestServer(suite.ctx, suite.Require(), suite.backendEndpoints, tempurl.Alloc()) + nodes[s.GetAddr()] = s + suite.cleanupFunc = append(suite.cleanupFunc, func() { + cleanup() + }) + } + tests.WaitForPrimaryServing(suite.Require(), nodes) + + // Scheduling + nodes = make(map[string]bs.Server) + for i := 0; i < 3; i++ { + s, cleanup := tests.StartSingleSchedulingTestServer(suite.ctx, suite.Require(), suite.backendEndpoints, tempurl.Alloc()) + nodes[s.GetAddr()] = s + suite.cleanupFunc = append(suite.cleanupFunc, func() { + cleanup() + }) + } + tests.WaitForPrimaryServing(suite.Require(), nodes) + + suite.cleanupFunc = append(suite.cleanupFunc, func() { + cancel() + }) +} + +func (suite *memberTestSuite) TearDownTest() { + for _, cleanup := range suite.cleanupFunc { + cleanup() + } + suite.cluster.Destroy() +} + +func (suite *memberTestSuite) TestMembers() { + re := suite.Require() + members, err := suite.dialClient.GetMicroServiceMembers(suite.ctx, "tso") + re.NoError(err) + re.Len(members, utils.DefaultKeyspaceGroupReplicaCount) + + members, err = suite.dialClient.GetMicroServiceMembers(suite.ctx, "scheduling") + re.NoError(err) + re.Len(members, 3) +} From 5eae459c01a797cbd0c416054c6f0cad16b8740a Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Mon, 18 Dec 2023 16:06:22 +0800 Subject: [PATCH 099/137] *: add pre func in etcdutil and refactor for endpoint (#7555) ref tikv/pd#7418 Signed-off-by: lhy1024 Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- pkg/keyspace/tso_keyspace_group.go | 3 +- pkg/mcs/scheduling/server/cluster.go | 2 +- pkg/mcs/scheduling/server/config/watcher.go | 36 +++++++++---------- pkg/mcs/scheduling/server/meta/watcher.go | 7 ++-- pkg/mcs/scheduling/server/rule/watcher.go | 21 +++++------ pkg/mock/mockcluster/mockcluster.go | 2 +- pkg/schedule/checker/rule_checker.go | 8 ++--- pkg/schedule/placement/rule_manager.go | 5 ++- pkg/schedule/placement/rule_manager_test.go | 7 ++-- pkg/schedule/schedulers/shuffle_region.go | 7 ++-- .../schedulers/shuffle_region_config.go | 4 ++- pkg/statistics/region_collection_test.go | 5 +-- pkg/storage/endpoint/config.go | 8 ++--- pkg/storage/endpoint/gc_safe_point.go | 8 +---- pkg/storage/endpoint/key_path.go | 6 ++++ pkg/storage/endpoint/keyspace.go | 5 --- pkg/storage/endpoint/replication_status.go | 6 +--- pkg/storage/endpoint/rule.go | 25 ------------- pkg/storage/endpoint/safepoint_v2.go | 13 ++----- pkg/storage/endpoint/service_middleware.go | 6 +--- pkg/storage/endpoint/tso_keyspace_group.go | 7 +--- pkg/storage/endpoint/util.go | 32 ++++++++++++++++- pkg/tso/keyspace_group_manager.go | 6 ++-- pkg/utils/etcdutil/etcdutil.go | 35 ++++++++++++------ pkg/utils/etcdutil/etcdutil_test.go | 15 +++++--- server/cluster/cluster.go | 2 +- server/cluster/cluster_test.go | 10 +++--- server/keyspace_service.go | 3 +- server/server.go | 3 +- 29 files changed, 150 insertions(+), 147 deletions(-) diff --git a/pkg/keyspace/tso_keyspace_group.go b/pkg/keyspace/tso_keyspace_group.go index c8694c4a7c6..51a53f75cc2 100644 --- a/pkg/keyspace/tso_keyspace_group.go +++ b/pkg/keyspace/tso_keyspace_group.go @@ -245,9 +245,10 @@ func (m *GroupManager) initTSONodesWatcher(client *clientv3.Client, clusterID ui client, "tso-nodes-watcher", tsoServiceKey, + func([]*clientv3.Event) error { return nil }, putFn, deleteFn, - func() error { return nil }, + func([]*clientv3.Event) error { return nil }, clientv3.WithRange(tsoServiceEndKey), ) } diff --git a/pkg/mcs/scheduling/server/cluster.go b/pkg/mcs/scheduling/server/cluster.go index b24db7ac805..5dd1c9f7fce 100644 --- a/pkg/mcs/scheduling/server/cluster.go +++ b/pkg/mcs/scheduling/server/cluster.go @@ -65,7 +65,7 @@ func NewCluster(parentCtx context.Context, persistConfig *config.PersistConfig, cancel() return nil, err } - ruleManager := placement.NewRuleManager(storage, basicCluster, persistConfig) + ruleManager := placement.NewRuleManager(ctx, storage, basicCluster, persistConfig) c := &Cluster{ ctx: ctx, cancel: cancel, diff --git a/pkg/mcs/scheduling/server/config/watcher.go b/pkg/mcs/scheduling/server/config/watcher.go index 4ded93ceb1b..32028592504 100644 --- a/pkg/mcs/scheduling/server/config/watcher.go +++ b/pkg/mcs/scheduling/server/config/watcher.go @@ -139,14 +139,13 @@ func (cw *Watcher) initializeConfigWatcher() error { deleteFn := func(kv *mvccpb.KeyValue) error { return nil } - postEventFn := func() error { - return nil - } cw.configWatcher = etcdutil.NewLoopWatcher( cw.ctx, &cw.wg, cw.etcdClient, "scheduling-config-watcher", cw.configPath, - putFn, deleteFn, postEventFn, + func([]*clientv3.Event) error { return nil }, + putFn, deleteFn, + func([]*clientv3.Event) error { return nil }, ) cw.configWatcher.StartWatchLoop() return cw.configWatcher.WaitLoad() @@ -154,7 +153,7 @@ func (cw *Watcher) initializeConfigWatcher() error { func (cw *Watcher) initializeTTLConfigWatcher() error { putFn := func(kv *mvccpb.KeyValue) error { - key := string(kv.Key)[len(sc.TTLConfigPrefix)+1:] + key := strings.TrimPrefix(string(kv.Key), sc.TTLConfigPrefix+"/") value := string(kv.Value) leaseID := kv.Lease resp, err := cw.etcdClient.TimeToLive(cw.ctx, clientv3.LeaseID(leaseID)) @@ -166,18 +165,18 @@ func (cw *Watcher) initializeTTLConfigWatcher() error { return nil } deleteFn := func(kv *mvccpb.KeyValue) error { - key := string(kv.Key)[len(sc.TTLConfigPrefix)+1:] + key := strings.TrimPrefix(string(kv.Key), sc.TTLConfigPrefix+"/") cw.ttl.PutWithTTL(key, nil, 0) return nil } - postEventFn := func() error { - return nil - } cw.ttlConfigWatcher = etcdutil.NewLoopWatcher( cw.ctx, &cw.wg, cw.etcdClient, "scheduling-ttl-config-watcher", cw.ttlConfigPrefix, - putFn, deleteFn, postEventFn, clientv3.WithPrefix(), + func([]*clientv3.Event) error { return nil }, + putFn, deleteFn, + func([]*clientv3.Event) error { return nil }, + clientv3.WithPrefix(), ) cw.ttlConfigWatcher.StartWatchLoop() return cw.ttlConfigWatcher.WaitLoad() @@ -186,13 +185,14 @@ func (cw *Watcher) initializeTTLConfigWatcher() error { func (cw *Watcher) initializeSchedulerConfigWatcher() error { prefixToTrim := cw.schedulerConfigPathPrefix + "/" putFn := func(kv *mvccpb.KeyValue) error { - name := strings.TrimPrefix(string(kv.Key), prefixToTrim) + key := string(kv.Key) + name := strings.TrimPrefix(key, prefixToTrim) log.Info("update scheduler config", zap.String("name", name), zap.String("value", string(kv.Value))) err := cw.storage.SaveSchedulerConfig(name, kv.Value) if err != nil { log.Warn("failed to save scheduler config", - zap.String("event-kv-key", string(kv.Key)), + zap.String("event-kv-key", key), zap.String("trimmed-key", name), zap.Error(err)) return err @@ -204,19 +204,19 @@ func (cw *Watcher) initializeSchedulerConfigWatcher() error { return nil } deleteFn := func(kv *mvccpb.KeyValue) error { - log.Info("remove scheduler config", zap.String("key", string(kv.Key))) + key := string(kv.Key) + log.Info("remove scheduler config", zap.String("key", key)) return cw.storage.RemoveSchedulerConfig( - strings.TrimPrefix(string(kv.Key), prefixToTrim), + strings.TrimPrefix(key, prefixToTrim), ) } - postEventFn := func() error { - return nil - } cw.schedulerConfigWatcher = etcdutil.NewLoopWatcher( cw.ctx, &cw.wg, cw.etcdClient, "scheduling-scheduler-config-watcher", cw.schedulerConfigPathPrefix, - putFn, deleteFn, postEventFn, + func([]*clientv3.Event) error { return nil }, + putFn, deleteFn, + func([]*clientv3.Event) error { return nil }, clientv3.WithPrefix(), ) cw.schedulerConfigWatcher.StartWatchLoop() diff --git a/pkg/mcs/scheduling/server/meta/watcher.go b/pkg/mcs/scheduling/server/meta/watcher.go index 6fae537eab9..808e8fc565e 100644 --- a/pkg/mcs/scheduling/server/meta/watcher.go +++ b/pkg/mcs/scheduling/server/meta/watcher.go @@ -104,14 +104,13 @@ func (w *Watcher) initializeStoreWatcher() error { } return nil } - postEventFn := func() error { - return nil - } w.storeWatcher = etcdutil.NewLoopWatcher( w.ctx, &w.wg, w.etcdClient, "scheduling-store-watcher", w.storePathPrefix, - putFn, deleteFn, postEventFn, + func([]*clientv3.Event) error { return nil }, + putFn, deleteFn, + func([]*clientv3.Event) error { return nil }, clientv3.WithPrefix(), ) w.storeWatcher.StartWatchLoop() diff --git a/pkg/mcs/scheduling/server/rule/watcher.go b/pkg/mcs/scheduling/server/rule/watcher.go index 912fb9c01e5..96e19cf5002 100644 --- a/pkg/mcs/scheduling/server/rule/watcher.go +++ b/pkg/mcs/scheduling/server/rule/watcher.go @@ -131,14 +131,13 @@ func (rw *Watcher) initializeRuleWatcher() error { rw.checkerController.AddSuspectKeyRange(rule.StartKey, rule.EndKey) return rw.ruleManager.DeleteRule(rule.GroupID, rule.ID) } - postEventFn := func() error { - return nil - } rw.ruleWatcher = etcdutil.NewLoopWatcher( rw.ctx, &rw.wg, rw.etcdClient, "scheduling-rule-watcher", rw.rulesPathPrefix, - putFn, deleteFn, postEventFn, + func([]*clientv3.Event) error { return nil }, + putFn, deleteFn, + func([]*clientv3.Event) error { return nil }, clientv3.WithPrefix(), ) rw.ruleWatcher.StartWatchLoop() @@ -168,14 +167,13 @@ func (rw *Watcher) initializeGroupWatcher() error { } return rw.ruleManager.DeleteRuleGroup(trimmedKey) } - postEventFn := func() error { - return nil - } rw.groupWatcher = etcdutil.NewLoopWatcher( rw.ctx, &rw.wg, rw.etcdClient, "scheduling-rule-group-watcher", rw.ruleGroupPathPrefix, - putFn, deleteFn, postEventFn, + func([]*clientv3.Event) error { return nil }, + putFn, deleteFn, + func([]*clientv3.Event) error { return nil }, clientv3.WithPrefix(), ) rw.groupWatcher.StartWatchLoop() @@ -197,14 +195,13 @@ func (rw *Watcher) initializeRegionLabelWatcher() error { log.Info("delete region label rule", zap.String("key", key)) return rw.regionLabeler.DeleteLabelRule(strings.TrimPrefix(key, prefixToTrim)) } - postEventFn := func() error { - return nil - } rw.labelWatcher = etcdutil.NewLoopWatcher( rw.ctx, &rw.wg, rw.etcdClient, "scheduling-region-label-watcher", rw.regionLabelPathPrefix, - putFn, deleteFn, postEventFn, + func([]*clientv3.Event) error { return nil }, + putFn, deleteFn, + func([]*clientv3.Event) error { return nil }, clientv3.WithPrefix(), ) rw.labelWatcher.StartWatchLoop() diff --git a/pkg/mock/mockcluster/mockcluster.go b/pkg/mock/mockcluster/mockcluster.go index 01282b40534..6cf7ae143df 100644 --- a/pkg/mock/mockcluster/mockcluster.go +++ b/pkg/mock/mockcluster/mockcluster.go @@ -212,7 +212,7 @@ func (mc *Cluster) AllocPeer(storeID uint64) (*metapb.Peer, error) { func (mc *Cluster) initRuleManager() { if mc.RuleManager == nil { - mc.RuleManager = placement.NewRuleManager(mc.GetStorage(), mc, mc.GetSharedConfig()) + mc.RuleManager = placement.NewRuleManager(mc.ctx, mc.GetStorage(), mc, mc.GetSharedConfig()) mc.RuleManager.Initialize(int(mc.GetReplicationConfig().MaxReplicas), mc.GetReplicationConfig().LocationLabels, mc.GetReplicationConfig().IsolationLevel) } } diff --git a/pkg/schedule/checker/rule_checker.go b/pkg/schedule/checker/rule_checker.go index 553ece09e65..95cc77ade5d 100644 --- a/pkg/schedule/checker/rule_checker.go +++ b/pkg/schedule/checker/rule_checker.go @@ -199,7 +199,7 @@ func (c *RuleChecker) fixRulePeer(region *core.RegionInfo, fit *placement.Region if c.isDownPeer(region, peer) { if c.isStoreDownTimeHitMaxDownTime(peer.GetStoreId()) { ruleCheckerReplaceDownCounter.Inc() - return c.replaceUnexpectRulePeer(region, rf, fit, peer, downStatus) + return c.replaceUnexpectedRulePeer(region, rf, fit, peer, downStatus) } // When witness placement rule is enabled, promotes the witness to voter when region has down voter. if c.isWitnessEnabled() && core.IsVoter(peer) { @@ -211,7 +211,7 @@ func (c *RuleChecker) fixRulePeer(region *core.RegionInfo, fit *placement.Region } if c.isOfflinePeer(peer) { ruleCheckerReplaceOfflineCounter.Inc() - return c.replaceUnexpectRulePeer(region, rf, fit, peer, offlineStatus) + return c.replaceUnexpectedRulePeer(region, rf, fit, peer, offlineStatus) } } // fix loose matched peers. @@ -246,7 +246,7 @@ func (c *RuleChecker) addRulePeer(region *core.RegionInfo, fit *placement.Region continue } ruleCheckerNoStoreThenTryReplace.Inc() - op, err := c.replaceUnexpectRulePeer(region, oldPeerRuleFit, fit, p, "swap-fit") + op, err := c.replaceUnexpectedRulePeer(region, oldPeerRuleFit, fit, p, "swap-fit") if err != nil { return nil, err } @@ -267,7 +267,7 @@ func (c *RuleChecker) addRulePeer(region *core.RegionInfo, fit *placement.Region } // The peer's store may in Offline or Down, need to be replace. -func (c *RuleChecker) replaceUnexpectRulePeer(region *core.RegionInfo, rf *placement.RuleFit, fit *placement.RegionFit, peer *metapb.Peer, status string) (*operator.Operator, error) { +func (c *RuleChecker) replaceUnexpectedRulePeer(region *core.RegionInfo, rf *placement.RuleFit, fit *placement.RegionFit, peer *metapb.Peer, status string) (*operator.Operator, error) { var fastFailover bool // If the store to which the original peer belongs is TiFlash, the new peer cannot be set to witness, nor can it perform fast failover if c.isWitnessEnabled() && !c.cluster.GetStore(peer.StoreId).IsTiFlash() { diff --git a/pkg/schedule/placement/rule_manager.go b/pkg/schedule/placement/rule_manager.go index e25b8802b45..621c52d738e 100644 --- a/pkg/schedule/placement/rule_manager.go +++ b/pkg/schedule/placement/rule_manager.go @@ -16,6 +16,7 @@ package placement import ( "bytes" + "context" "encoding/hex" "encoding/json" "fmt" @@ -49,6 +50,7 @@ const ( // RuleManager is responsible for the lifecycle of all placement Rules. // It is thread safe. type RuleManager struct { + ctx context.Context storage endpoint.RuleStorage syncutil.RWMutex initialized bool @@ -63,8 +65,9 @@ type RuleManager struct { } // NewRuleManager creates a RuleManager instance. -func NewRuleManager(storage endpoint.RuleStorage, storeSetInformer core.StoreSetInformer, conf config.SharedConfigProvider) *RuleManager { +func NewRuleManager(ctx context.Context, storage endpoint.RuleStorage, storeSetInformer core.StoreSetInformer, conf config.SharedConfigProvider) *RuleManager { return &RuleManager{ + ctx: ctx, storage: storage, storeSetInformer: storeSetInformer, conf: conf, diff --git a/pkg/schedule/placement/rule_manager_test.go b/pkg/schedule/placement/rule_manager_test.go index 68a18b538d4..c0987f6dd33 100644 --- a/pkg/schedule/placement/rule_manager_test.go +++ b/pkg/schedule/placement/rule_manager_test.go @@ -15,6 +15,7 @@ package placement import ( + "context" "encoding/hex" "testing" @@ -32,7 +33,7 @@ func newTestManager(t *testing.T, enableWitness bool) (endpoint.RuleStorage, *Ru re := require.New(t) store := endpoint.NewStorageEndpoint(kv.NewMemoryKV(), nil) var err error - manager := NewRuleManager(store, nil, mockconfig.NewTestOptions()) + manager := NewRuleManager(context.Background(), store, nil, mockconfig.NewTestOptions()) manager.conf.SetEnableWitness(enableWitness) err = manager.Initialize(3, []string{"zone", "rack", "host"}, "") re.NoError(err) @@ -156,7 +157,7 @@ func TestSaveLoad(t *testing.T) { re.NoError(manager.SetRule(r.Clone())) } - m2 := NewRuleManager(store, nil, nil) + m2 := NewRuleManager(context.Background(), store, nil, nil) err := m2.Initialize(3, []string{"no", "labels"}, "") re.NoError(err) re.Len(m2.GetAllRules(), 3) @@ -174,7 +175,7 @@ func TestSetAfterGet(t *testing.T) { rule.Count = 1 manager.SetRule(rule) - m2 := NewRuleManager(store, nil, nil) + m2 := NewRuleManager(context.Background(), store, nil, nil) err := m2.Initialize(100, []string{}, "") re.NoError(err) rule = m2.GetRule(DefaultGroupID, DefaultRuleID) diff --git a/pkg/schedule/schedulers/shuffle_region.go b/pkg/schedule/schedulers/shuffle_region.go index f1d35e80925..f9bed18d3fa 100644 --- a/pkg/schedule/schedulers/shuffle_region.go +++ b/pkg/schedule/schedulers/shuffle_region.go @@ -139,18 +139,19 @@ func (s *shuffleRegionScheduler) scheduleRemovePeer(cluster sche.SchedulerCluste pendingFilter := filter.NewRegionPendingFilter() downFilter := filter.NewRegionDownFilter() replicaFilter := filter.NewRegionReplicatedFilter(cluster) + ranges := s.conf.GetRanges() for _, source := range candidates.Stores { var region *core.RegionInfo if s.conf.IsRoleAllow(roleFollower) { - region = filter.SelectOneRegion(cluster.RandFollowerRegions(source.GetID(), s.conf.Ranges), nil, + region = filter.SelectOneRegion(cluster.RandFollowerRegions(source.GetID(), ranges), nil, pendingFilter, downFilter, replicaFilter) } if region == nil && s.conf.IsRoleAllow(roleLeader) { - region = filter.SelectOneRegion(cluster.RandLeaderRegions(source.GetID(), s.conf.Ranges), nil, + region = filter.SelectOneRegion(cluster.RandLeaderRegions(source.GetID(), ranges), nil, pendingFilter, downFilter, replicaFilter) } if region == nil && s.conf.IsRoleAllow(roleLearner) { - region = filter.SelectOneRegion(cluster.RandLearnerRegions(source.GetID(), s.conf.Ranges), nil, + region = filter.SelectOneRegion(cluster.RandLearnerRegions(source.GetID(), ranges), nil, pendingFilter, downFilter, replicaFilter) } if region != nil { diff --git a/pkg/schedule/schedulers/shuffle_region_config.go b/pkg/schedule/schedulers/shuffle_region_config.go index 7d04879c992..552d7ea8bce 100644 --- a/pkg/schedule/schedulers/shuffle_region_config.go +++ b/pkg/schedule/schedulers/shuffle_region_config.go @@ -58,7 +58,9 @@ func (conf *shuffleRegionSchedulerConfig) GetRoles() []string { func (conf *shuffleRegionSchedulerConfig) GetRanges() []core.KeyRange { conf.RLock() defer conf.RUnlock() - return conf.Ranges + ranges := make([]core.KeyRange, len(conf.Ranges)) + copy(ranges, conf.Ranges) + return ranges } func (conf *shuffleRegionSchedulerConfig) IsRoleAllow(role string) bool { diff --git a/pkg/statistics/region_collection_test.go b/pkg/statistics/region_collection_test.go index f0df9ce6e07..cbbf7672bee 100644 --- a/pkg/statistics/region_collection_test.go +++ b/pkg/statistics/region_collection_test.go @@ -15,6 +15,7 @@ package statistics import ( + "context" "testing" "github.com/pingcap/kvproto/pkg/metapb" @@ -29,7 +30,7 @@ import ( func TestRegionStatistics(t *testing.T) { re := require.New(t) store := storage.NewStorageWithMemoryBackend() - manager := placement.NewRuleManager(store, nil, nil) + manager := placement.NewRuleManager(context.Background(), store, nil, nil) err := manager.Initialize(3, []string{"zone", "rack", "host"}, "") re.NoError(err) opt := mockconfig.NewTestOptions() @@ -118,7 +119,7 @@ func TestRegionStatistics(t *testing.T) { func TestRegionStatisticsWithPlacementRule(t *testing.T) { re := require.New(t) store := storage.NewStorageWithMemoryBackend() - manager := placement.NewRuleManager(store, nil, nil) + manager := placement.NewRuleManager(context.Background(), store, nil, nil) err := manager.Initialize(3, []string{"zone", "rack", "host"}, "") re.NoError(err) opt := mockconfig.NewTestOptions() diff --git a/pkg/storage/endpoint/config.go b/pkg/storage/endpoint/config.go index db5565a4b90..edfdcbca9a3 100644 --- a/pkg/storage/endpoint/config.go +++ b/pkg/storage/endpoint/config.go @@ -51,17 +51,13 @@ func (se *StorageEndpoint) LoadConfig(cfg interface{}) (bool, error) { // SaveConfig stores marshallable cfg to the configPath. func (se *StorageEndpoint) SaveConfig(cfg interface{}) error { - value, err := json.Marshal(cfg) - if err != nil { - return errs.ErrJSONMarshal.Wrap(err).GenWithStackByCause() - } - return se.Save(configPath, string(value)) + return se.saveJSON(configPath, cfg) } // LoadAllSchedulerConfigs loads all schedulers' config. func (se *StorageEndpoint) LoadAllSchedulerConfigs() ([]string, []string, error) { prefix := customSchedulerConfigPath + "/" - keys, values, err := se.LoadRange(prefix, clientv3.GetPrefixRangeEnd(prefix), 1000) + keys, values, err := se.LoadRange(prefix, clientv3.GetPrefixRangeEnd(prefix), MinKVRangeLimit) for i, key := range keys { keys[i] = strings.TrimPrefix(key, prefix) } diff --git a/pkg/storage/endpoint/gc_safe_point.go b/pkg/storage/endpoint/gc_safe_point.go index db5c58205c8..c2f09980651 100644 --- a/pkg/storage/endpoint/gc_safe_point.go +++ b/pkg/storage/endpoint/gc_safe_point.go @@ -169,13 +169,7 @@ func (se *StorageEndpoint) SaveServiceGCSafePoint(ssp *ServiceSafePoint) error { return errors.New("TTL of gc_worker's service safe point must be infinity") } - key := gcSafePointServicePath(ssp.ServiceID) - value, err := json.Marshal(ssp) - if err != nil { - return err - } - - return se.Save(key, string(value)) + return se.saveJSON(gcSafePointServicePath(ssp.ServiceID), ssp) } // RemoveServiceGCSafePoint removes a GC safepoint for the service diff --git a/pkg/storage/endpoint/key_path.go b/pkg/storage/endpoint/key_path.go index cac40db29c5..69b8d0f2f8e 100644 --- a/pkg/storage/endpoint/key_path.go +++ b/pkg/storage/endpoint/key_path.go @@ -31,6 +31,7 @@ const ( serviceMiddlewarePath = "service_middleware" schedulePath = "schedule" gcPath = "gc" + ruleCommonPath = "rule" rulesPath = "rules" ruleGroupPath = "rule_group" regionLabelPath = "region_label" @@ -102,6 +103,11 @@ func RulesPathPrefix(clusterID uint64) string { return path.Join(PDRootPath(clusterID), rulesPath) } +// RuleCommonPathPrefix returns the path prefix to save the placement rule common config. +func RuleCommonPathPrefix(clusterID uint64) string { + return path.Join(PDRootPath(clusterID), ruleCommonPath) +} + // RuleGroupPathPrefix returns the path prefix to save the placement rule groups. func RuleGroupPathPrefix(clusterID uint64) string { return path.Join(PDRootPath(clusterID), ruleGroupPath) diff --git a/pkg/storage/endpoint/keyspace.go b/pkg/storage/endpoint/keyspace.go index 09733ad59c1..77c81b2c8d6 100644 --- a/pkg/storage/endpoint/keyspace.go +++ b/pkg/storage/endpoint/keyspace.go @@ -97,11 +97,6 @@ func (se *StorageEndpoint) LoadKeyspaceID(txn kv.Txn, name string) (bool, uint32 return true, uint32(id64), nil } -// RunInTxn runs the given function in a transaction. -func (se *StorageEndpoint) RunInTxn(ctx context.Context, f func(txn kv.Txn) error) error { - return se.Base.RunInTxn(ctx, f) -} - // LoadRangeKeyspace loads keyspaces starting at startID. // limit specifies the limit of loaded keyspaces. func (se *StorageEndpoint) LoadRangeKeyspace(txn kv.Txn, startID uint32, limit int) ([]*keyspacepb.KeyspaceMeta, error) { diff --git a/pkg/storage/endpoint/replication_status.go b/pkg/storage/endpoint/replication_status.go index 4bac51071bc..0a14770ff47 100644 --- a/pkg/storage/endpoint/replication_status.go +++ b/pkg/storage/endpoint/replication_status.go @@ -43,9 +43,5 @@ func (se *StorageEndpoint) LoadReplicationStatus(mode string, status interface{} // SaveReplicationStatus stores replication status by mode. func (se *StorageEndpoint) SaveReplicationStatus(mode string, status interface{}) error { - value, err := json.Marshal(status) - if err != nil { - return errs.ErrJSONMarshal.Wrap(err).GenWithStackByArgs() - } - return se.Save(replicationModePath(mode), string(value)) + return se.saveJSON(replicationModePath(mode), status) } diff --git a/pkg/storage/endpoint/rule.go b/pkg/storage/endpoint/rule.go index 125c5bc31eb..80b6fc7c0ff 100644 --- a/pkg/storage/endpoint/rule.go +++ b/pkg/storage/endpoint/rule.go @@ -14,12 +14,6 @@ package endpoint -import ( - "strings" - - "go.etcd.io/etcd/clientv3" -) - // RuleStorage defines the storage operations on the rule. type RuleStorage interface { LoadRule(ruleKey string) (string, error) @@ -103,22 +97,3 @@ func (se *StorageEndpoint) LoadRule(ruleKey string) (string, error) { func (se *StorageEndpoint) LoadRules(f func(k, v string)) error { return se.loadRangeByPrefix(rulesPath+"/", f) } - -// loadRangeByPrefix iterates all key-value pairs in the storage that has the prefix. -func (se *StorageEndpoint) loadRangeByPrefix(prefix string, f func(k, v string)) error { - nextKey := prefix - endKey := clientv3.GetPrefixRangeEnd(prefix) - for { - keys, values, err := se.LoadRange(nextKey, endKey, MinKVRangeLimit) - if err != nil { - return err - } - for i := range keys { - f(strings.TrimPrefix(keys[i], prefix), values[i]) - } - if len(keys) < MinKVRangeLimit { - return nil - } - nextKey = keys[len(keys)-1] + "\x00" - } -} diff --git a/pkg/storage/endpoint/safepoint_v2.go b/pkg/storage/endpoint/safepoint_v2.go index cac2606a470..8d690d07261 100644 --- a/pkg/storage/endpoint/safepoint_v2.go +++ b/pkg/storage/endpoint/safepoint_v2.go @@ -79,12 +79,7 @@ func (se *StorageEndpoint) LoadGCSafePointV2(keyspaceID uint32) (*GCSafePointV2, // SaveGCSafePointV2 saves gc safe point for the given keyspace. func (se *StorageEndpoint) SaveGCSafePointV2(gcSafePoint *GCSafePointV2) error { - key := GCSafePointV2Path(gcSafePoint.KeyspaceID) - value, err := json.Marshal(gcSafePoint) - if err != nil { - return errs.ErrJSONMarshal.Wrap(err).GenWithStackByCause() - } - return se.Save(key, string(value)) + return se.saveJSON(GCSafePointV2Path(gcSafePoint.KeyspaceID), gcSafePoint) } // LoadAllGCSafePoints returns gc safe point for all keyspaces @@ -203,11 +198,7 @@ func (se *StorageEndpoint) SaveServiceSafePointV2(serviceSafePoint *ServiceSafeP } key := ServiceSafePointV2Path(serviceSafePoint.KeyspaceID, serviceSafePoint.ServiceID) - value, err := json.Marshal(serviceSafePoint) - if err != nil { - return errs.ErrJSONMarshal.Wrap(err).GenWithStackByCause() - } - return se.Save(key, string(value)) + return se.saveJSON(key, serviceSafePoint) } // RemoveServiceSafePointV2 removes a service safe point. diff --git a/pkg/storage/endpoint/service_middleware.go b/pkg/storage/endpoint/service_middleware.go index 62cf91c97bf..2becbf3686e 100644 --- a/pkg/storage/endpoint/service_middleware.go +++ b/pkg/storage/endpoint/service_middleware.go @@ -43,9 +43,5 @@ func (se *StorageEndpoint) LoadServiceMiddlewareConfig(cfg interface{}) (bool, e // SaveServiceMiddlewareConfig stores marshallable cfg to the serviceMiddlewarePath. func (se *StorageEndpoint) SaveServiceMiddlewareConfig(cfg interface{}) error { - value, err := json.Marshal(cfg) - if err != nil { - return errs.ErrJSONMarshal.Wrap(err).GenWithStackByCause() - } - return se.Save(serviceMiddlewarePath, string(value)) + return se.saveJSON(serviceMiddlewarePath, cfg) } diff --git a/pkg/storage/endpoint/tso_keyspace_group.go b/pkg/storage/endpoint/tso_keyspace_group.go index 498cd878887..39a08afe937 100644 --- a/pkg/storage/endpoint/tso_keyspace_group.go +++ b/pkg/storage/endpoint/tso_keyspace_group.go @@ -177,12 +177,7 @@ func (se *StorageEndpoint) LoadKeyspaceGroup(txn kv.Txn, id uint32) (*KeyspaceGr // SaveKeyspaceGroup saves the keyspace group. func (se *StorageEndpoint) SaveKeyspaceGroup(txn kv.Txn, kg *KeyspaceGroup) error { - key := KeyspaceGroupIDPath(kg.ID) - value, err := json.Marshal(kg) - if err != nil { - return err - } - return txn.Save(key, string(value)) + return saveJSONInTxn(txn, KeyspaceGroupIDPath(kg.ID), kg) } // DeleteKeyspaceGroup deletes the keyspace group. diff --git a/pkg/storage/endpoint/util.go b/pkg/storage/endpoint/util.go index 37f98a55709..3058c059628 100644 --- a/pkg/storage/endpoint/util.go +++ b/pkg/storage/endpoint/util.go @@ -16,9 +16,12 @@ package endpoint import ( "encoding/json" + "strings" "github.com/gogo/protobuf/proto" "github.com/tikv/pd/pkg/errs" + "github.com/tikv/pd/pkg/storage/kv" + "go.etcd.io/etcd/clientv3" ) func (se *StorageEndpoint) loadProto(key string, msg proto.Message) (bool, error) { @@ -42,9 +45,36 @@ func (se *StorageEndpoint) saveProto(key string, msg proto.Message) error { } func (se *StorageEndpoint) saveJSON(key string, data interface{}) error { + return saveJSONInTxn(se /* use the same interface */, key, data) +} + +func saveJSONInTxn(txn kv.Txn, key string, data interface{}) error { value, err := json.Marshal(data) if err != nil { return errs.ErrJSONMarshal.Wrap(err).GenWithStackByArgs() } - return se.Save(key, string(value)) + return txn.Save(key, string(value)) +} + +// loadRangeByPrefix iterates all key-value pairs in the storage that has the prefix. +func (se *StorageEndpoint) loadRangeByPrefix(prefix string, f func(k, v string)) error { + return loadRangeByPrefixInTxn(se /* use the same interface */, prefix, f) +} + +func loadRangeByPrefixInTxn(txn kv.Txn, prefix string, f func(k, v string)) error { + nextKey := prefix + endKey := clientv3.GetPrefixRangeEnd(prefix) + for { + keys, values, err := txn.LoadRange(nextKey, endKey, MinKVRangeLimit) + if err != nil { + return err + } + for i := range keys { + f(strings.TrimPrefix(keys[i], prefix), values[i]) + } + if len(keys) < MinKVRangeLimit { + return nil + } + nextKey = keys[len(keys)-1] + "\x00" + } } diff --git a/pkg/tso/keyspace_group_manager.go b/pkg/tso/keyspace_group_manager.go index 58534de1642..0e69986f255 100644 --- a/pkg/tso/keyspace_group_manager.go +++ b/pkg/tso/keyspace_group_manager.go @@ -514,9 +514,10 @@ func (kgm *KeyspaceGroupManager) InitializeTSOServerWatchLoop() error { kgm.etcdClient, "tso-nodes-watcher", kgm.tsoServiceKey, + func([]*clientv3.Event) error { return nil }, putFn, deleteFn, - func() error { return nil }, + func([]*clientv3.Event) error { return nil }, clientv3.WithRange(tsoServiceEndKey), ) kgm.tsoNodesWatcher.StartWatchLoop() @@ -558,7 +559,7 @@ func (kgm *KeyspaceGroupManager) InitializeGroupWatchLoop() error { kgm.deleteKeyspaceGroup(groupID) return nil } - postEventFn := func() error { + postEventFn := func([]*clientv3.Event) error { // Retry the groups that are not initialized successfully before. for id, group := range kgm.groupUpdateRetryList { delete(kgm.groupUpdateRetryList, id) @@ -572,6 +573,7 @@ func (kgm *KeyspaceGroupManager) InitializeGroupWatchLoop() error { kgm.etcdClient, "keyspace-watcher", startKey, + func([]*clientv3.Event) error { return nil }, putFn, deleteFn, postEventFn, diff --git a/pkg/utils/etcdutil/etcdutil.go b/pkg/utils/etcdutil/etcdutil.go index 03c2374efc6..0e1b2731474 100644 --- a/pkg/utils/etcdutil/etcdutil.go +++ b/pkg/utils/etcdutil/etcdutil.go @@ -587,8 +587,10 @@ type LoopWatcher struct { putFn func(*mvccpb.KeyValue) error // deleteFn is used to handle the delete event. deleteFn func(*mvccpb.KeyValue) error - // postEventFn is used to call after handling all events. - postEventFn func() error + // postEventsFn is used to call after handling all events. + postEventsFn func([]*clientv3.Event) error + // preEventsFn is used to call before handling all events. + preEventsFn func([]*clientv3.Event) error // forceLoadMu is used to ensure two force loads have minimal interval. forceLoadMu syncutil.RWMutex @@ -613,7 +615,9 @@ func NewLoopWatcher( ctx context.Context, wg *sync.WaitGroup, client *clientv3.Client, name, key string, - putFn, deleteFn func(*mvccpb.KeyValue) error, postEventFn func() error, + preEventsFn func([]*clientv3.Event) error, + putFn, deleteFn func(*mvccpb.KeyValue) error, + postEventsFn func([]*clientv3.Event) error, opts ...clientv3.OpOption, ) *LoopWatcher { return &LoopWatcher{ @@ -627,7 +631,8 @@ func NewLoopWatcher( updateClientCh: make(chan *clientv3.Client, 1), putFn: putFn, deleteFn: deleteFn, - postEventFn: postEventFn, + postEventsFn: postEventsFn, + preEventsFn: preEventsFn, opts: opts, lastTimeForceLoad: time.Now(), loadTimeout: defaultLoadDataFromEtcdTimeout, @@ -813,28 +818,34 @@ func (lw *LoopWatcher) watch(ctx context.Context, revision int64) (nextRevision zap.Int64("revision", revision), zap.String("name", lw.name), zap.String("key", lw.key)) goto watchChanLoop } + if err := lw.preEventsFn(wresp.Events); err != nil { + log.Error("run pre event failed in watch loop", zap.Error(err), + zap.Int64("revision", revision), zap.String("name", lw.name), zap.String("key", lw.key)) + } for _, event := range wresp.Events { switch event.Type { case clientv3.EventTypePut: if err := lw.putFn(event.Kv); err != nil { log.Error("put failed in watch loop", zap.Error(err), - zap.Int64("revision", revision), zap.String("name", lw.name), zap.String("key", lw.key)) + zap.Int64("revision", revision), zap.String("name", lw.name), + zap.String("watch-key", lw.key), zap.ByteString("event-kv-key", event.Kv.Key)) } else { - log.Debug("put in watch loop", zap.String("name", lw.name), + log.Debug("put successfully in watch loop", zap.String("name", lw.name), zap.ByteString("key", event.Kv.Key), zap.ByteString("value", event.Kv.Value)) } case clientv3.EventTypeDelete: if err := lw.deleteFn(event.Kv); err != nil { log.Error("delete failed in watch loop", zap.Error(err), - zap.Int64("revision", revision), zap.String("name", lw.name), zap.String("key", lw.key)) + zap.Int64("revision", revision), zap.String("name", lw.name), + zap.String("watch-key", lw.key), zap.ByteString("event-kv-key", event.Kv.Key)) } else { - log.Debug("delete in watch loop", zap.String("name", lw.name), + log.Debug("delete successfully in watch loop", zap.String("name", lw.name), zap.ByteString("key", event.Kv.Key)) } } } - if err := lw.postEventFn(); err != nil { + if err := lw.postEventsFn(wresp.Events); err != nil { log.Error("run post event failed in watch loop", zap.Error(err), zap.Int64("revision", revision), zap.String("name", lw.name), zap.String("key", lw.key)) } @@ -864,6 +875,10 @@ func (lw *LoopWatcher) load(ctx context.Context) (nextRevision int64, err error) zap.String("key", lw.key), zap.Error(err)) return 0, err } + if err := lw.preEventsFn([]*clientv3.Event{}); err != nil { + log.Error("run pre event failed in watch loop", zap.String("name", lw.name), + zap.String("key", lw.key), zap.Error(err)) + } for i, item := range resp.Kvs { if resp.More && i == len(resp.Kvs)-1 { // The last key is the start key of the next batch. @@ -878,7 +893,7 @@ func (lw *LoopWatcher) load(ctx context.Context) (nextRevision int64, err error) } // Note: if there are no keys in etcd, the resp.More is false. It also means the load is finished. if !resp.More { - if err := lw.postEventFn(); err != nil { + if err := lw.postEventsFn([]*clientv3.Event{}); err != nil { log.Error("run post event failed in watch loop", zap.String("name", lw.name), zap.String("key", lw.key), zap.Error(err)) } diff --git a/pkg/utils/etcdutil/etcdutil_test.go b/pkg/utils/etcdutil/etcdutil_test.go index f7fadd3bbf6..861a57cef13 100644 --- a/pkg/utils/etcdutil/etcdutil_test.go +++ b/pkg/utils/etcdutil/etcdutil_test.go @@ -410,6 +410,7 @@ func (suite *loopWatcherTestSuite) TestLoadWithoutKey() { suite.client, "test", "TestLoadWithoutKey", + func([]*clientv3.Event) error { return nil }, func(kv *mvccpb.KeyValue) error { cache.Lock() defer cache.Unlock() @@ -417,7 +418,7 @@ func (suite *loopWatcherTestSuite) TestLoadWithoutKey() { return nil }, func(kv *mvccpb.KeyValue) error { return nil }, - func() error { return nil }, + func([]*clientv3.Event) error { return nil }, ) watcher.StartWatchLoop() err := watcher.WaitLoad() @@ -441,6 +442,7 @@ func (suite *loopWatcherTestSuite) TestCallBack() { suite.client, "test", "TestCallBack", + func([]*clientv3.Event) error { return nil }, func(kv *mvccpb.KeyValue) error { result = append(result, string(kv.Key)) return nil @@ -451,7 +453,7 @@ func (suite *loopWatcherTestSuite) TestCallBack() { delete(cache.data, string(kv.Key)) return nil }, - func() error { + func([]*clientv3.Event) error { cache.Lock() defer cache.Unlock() for _, r := range result { @@ -506,6 +508,7 @@ func (suite *loopWatcherTestSuite) TestWatcherLoadLimit() { suite.client, "test", "TestWatcherLoadLimit", + func([]*clientv3.Event) error { return nil }, func(kv *mvccpb.KeyValue) error { cache.Lock() defer cache.Unlock() @@ -515,7 +518,7 @@ func (suite *loopWatcherTestSuite) TestWatcherLoadLimit() { func(kv *mvccpb.KeyValue) error { return nil }, - func() error { + func([]*clientv3.Event) error { return nil }, clientv3.WithPrefix(), @@ -550,6 +553,7 @@ func (suite *loopWatcherTestSuite) TestWatcherBreak() { suite.client, "test", "TestWatcherBreak", + func([]*clientv3.Event) error { return nil }, func(kv *mvccpb.KeyValue) error { if string(kv.Key) == "TestWatcherBreak" { cache.Lock() @@ -559,7 +563,7 @@ func (suite *loopWatcherTestSuite) TestWatcherBreak() { return nil }, func(kv *mvccpb.KeyValue) error { return nil }, - func() error { return nil }, + func([]*clientv3.Event) error { return nil }, ) watcher.watchChangeRetryInterval = 100 * time.Millisecond watcher.StartWatchLoop() @@ -633,9 +637,10 @@ func (suite *loopWatcherTestSuite) TestWatcherRequestProgress() { suite.client, "test", "TestWatcherChanBlock", + func([]*clientv3.Event) error { return nil }, func(kv *mvccpb.KeyValue) error { return nil }, func(kv *mvccpb.KeyValue) error { return nil }, - func() error { return nil }, + func([]*clientv3.Event) error { return nil }, ) suite.wg.Add(1) diff --git a/server/cluster/cluster.go b/server/cluster/cluster.go index 78f6ddd4364..ecbd40e2582 100644 --- a/server/cluster/cluster.go +++ b/server/cluster/cluster.go @@ -269,7 +269,7 @@ func (c *RaftCluster) InitCluster( c.unsafeRecoveryController = unsaferecovery.NewController(c) c.keyspaceGroupManager = keyspaceGroupManager c.hbstreams = hbstreams - c.ruleManager = placement.NewRuleManager(c.storage, c, c.GetOpts()) + c.ruleManager = placement.NewRuleManager(c.ctx, c.storage, c, c.GetOpts()) if c.opt.IsPlacementRulesEnabled() { err := c.ruleManager.Initialize(c.opt.GetMaxReplicas(), c.opt.GetLocationLabels(), c.opt.GetIsolationLevel()) if err != nil { diff --git a/server/cluster/cluster_test.go b/server/cluster/cluster_test.go index 85edf911779..7094fd6b673 100644 --- a/server/cluster/cluster_test.go +++ b/server/cluster/cluster_test.go @@ -241,7 +241,7 @@ func TestSetOfflineStore(t *testing.T) { re.NoError(err) cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend()) cluster.coordinator = schedule.NewCoordinator(ctx, cluster, nil) - cluster.ruleManager = placement.NewRuleManager(storage.NewStorageWithMemoryBackend(), cluster, cluster.GetOpts()) + cluster.ruleManager = placement.NewRuleManager(ctx, storage.NewStorageWithMemoryBackend(), cluster, cluster.GetOpts()) if opt.IsPlacementRulesEnabled() { err := cluster.ruleManager.Initialize(opt.GetMaxReplicas(), opt.GetLocationLabels(), opt.GetIsolationLevel()) if err != nil { @@ -438,7 +438,7 @@ func TestUpStore(t *testing.T) { re.NoError(err) cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend()) cluster.coordinator = schedule.NewCoordinator(ctx, cluster, nil) - cluster.ruleManager = placement.NewRuleManager(storage.NewStorageWithMemoryBackend(), cluster, cluster.GetOpts()) + cluster.ruleManager = placement.NewRuleManager(ctx, storage.NewStorageWithMemoryBackend(), cluster, cluster.GetOpts()) if opt.IsPlacementRulesEnabled() { err := cluster.ruleManager.Initialize(opt.GetMaxReplicas(), opt.GetLocationLabels(), opt.GetIsolationLevel()) if err != nil { @@ -541,7 +541,7 @@ func TestDeleteStoreUpdatesClusterVersion(t *testing.T) { re.NoError(err) cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend()) cluster.coordinator = schedule.NewCoordinator(ctx, cluster, nil) - cluster.ruleManager = placement.NewRuleManager(storage.NewStorageWithMemoryBackend(), cluster, cluster.GetOpts()) + cluster.ruleManager = placement.NewRuleManager(ctx, storage.NewStorageWithMemoryBackend(), cluster, cluster.GetOpts()) if opt.IsPlacementRulesEnabled() { err := cluster.ruleManager.Initialize(opt.GetMaxReplicas(), opt.GetLocationLabels(), opt.GetIsolationLevel()) if err != nil { @@ -1268,7 +1268,7 @@ func TestOfflineAndMerge(t *testing.T) { re.NoError(err) cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend()) cluster.coordinator = schedule.NewCoordinator(ctx, cluster, nil) - cluster.ruleManager = placement.NewRuleManager(storage.NewStorageWithMemoryBackend(), cluster, cluster.GetOpts()) + cluster.ruleManager = placement.NewRuleManager(ctx, storage.NewStorageWithMemoryBackend(), cluster, cluster.GetOpts()) if opt.IsPlacementRulesEnabled() { err := cluster.ruleManager.Initialize(opt.GetMaxReplicas(), opt.GetLocationLabels(), opt.GetIsolationLevel()) if err != nil { @@ -2130,7 +2130,7 @@ func newTestRaftCluster( ) *RaftCluster { rc := &RaftCluster{serverCtx: ctx, core: core.NewBasicCluster(), storage: s} rc.InitCluster(id, opt, nil, nil) - rc.ruleManager = placement.NewRuleManager(storage.NewStorageWithMemoryBackend(), rc, opt) + rc.ruleManager = placement.NewRuleManager(ctx, storage.NewStorageWithMemoryBackend(), rc, opt) if opt.IsPlacementRulesEnabled() { err := rc.ruleManager.Initialize(opt.GetMaxReplicas(), opt.GetLocationLabels(), opt.GetIsolationLevel()) if err != nil { diff --git a/server/keyspace_service.go b/server/keyspace_service.go index b17239ba0a4..1718108d73b 100644 --- a/server/keyspace_service.go +++ b/server/keyspace_service.go @@ -89,7 +89,7 @@ func (s *KeyspaceServer) WatchKeyspaces(request *keyspacepb.WatchKeyspacesReques deleteFn := func(kv *mvccpb.KeyValue) error { return nil } - postEventFn := func() error { + postEventFn := func([]*clientv3.Event) error { defer func() { keyspaces = keyspaces[:0] }() @@ -109,6 +109,7 @@ func (s *KeyspaceServer) WatchKeyspaces(request *keyspacepb.WatchKeyspacesReques s.client, "keyspace-server-watcher", startKey, + func([]*clientv3.Event) error { return nil }, putFn, deleteFn, postEventFn, diff --git a/server/server.go b/server/server.go index 187c30dbf7a..fcf71922a09 100644 --- a/server/server.go +++ b/server/server.go @@ -2017,9 +2017,10 @@ func (s *Server) initServicePrimaryWatcher(serviceName string, primaryKey string s.client, name, primaryKey, + func([]*clientv3.Event) error { return nil }, putFn, deleteFn, - func() error { return nil }, + func([]*clientv3.Event) error { return nil }, ) } From 40c252346cffa7a3d56a859856adddbec4a0a72c Mon Sep 17 00:00:00 2001 From: JmPotato Date: Mon, 18 Dec 2023 17:22:53 +0800 Subject: [PATCH 100/137] client/http, tests: implement retry mechanism based on the leader and follower (#7554) ref tikv/pd#7300 - Implement retry mechanism based on the leader and follower. - Move method definitions into a separate file. - Use a sturct `requestInfo` to gather the parameters. Signed-off-by: JmPotato Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- client/http/client.go | 910 +++++------------- client/http/client_test.go | 33 +- client/http/interface.go | 695 +++++++++++++ client/http/request_info.go | 123 +++ tests/integrations/client/http_client_test.go | 17 +- 5 files changed, 1092 insertions(+), 686 deletions(-) create mode 100644 client/http/interface.go create mode 100644 client/http/request_info.go diff --git a/client/http/client.go b/client/http/client.go index 927450b74a2..21a3727e00f 100644 --- a/client/http/client.go +++ b/client/http/client.go @@ -23,265 +23,172 @@ import ( "io" "net/http" "strings" + "sync" "time" "github.com/pingcap/errors" - "github.com/pingcap/kvproto/pkg/pdpb" "github.com/pingcap/log" "github.com/prometheus/client_golang/prometheus" "go.uber.org/zap" ) const ( - defaultCallerID = "pd-http-client" - httpScheme = "http" - httpsScheme = "https" - networkErrorStatus = "network error" - - defaultTimeout = 30 * time.Second + // defaultCallerID marks the default caller ID of the PD HTTP client. + defaultCallerID = "pd-http-client" + // defaultInnerCallerID marks the default caller ID of the inner PD HTTP client. + // It's used to distinguish the requests sent by the inner client via some internal logic. + defaultInnerCallerID = "pd-http-client-inner" + httpScheme = "http" + httpsScheme = "https" + networkErrorStatus = "network error" + + defaultMembersInfoUpdateInterval = time.Minute + defaultTimeout = 30 * time.Second ) -// Client is a PD (Placement Driver) HTTP client. -type Client interface { - /* Meta-related interfaces */ - GetRegionByID(context.Context, uint64) (*RegionInfo, error) - GetRegionByKey(context.Context, []byte) (*RegionInfo, error) - GetRegions(context.Context) (*RegionsInfo, error) - GetRegionsByKeyRange(context.Context, *KeyRange, int) (*RegionsInfo, error) - GetRegionsByStoreID(context.Context, uint64) (*RegionsInfo, error) - GetRegionsReplicatedStateByKeyRange(context.Context, *KeyRange) (string, error) - GetHotReadRegions(context.Context) (*StoreHotPeersInfos, error) - GetHotWriteRegions(context.Context) (*StoreHotPeersInfos, error) - GetHistoryHotRegions(context.Context, *HistoryHotRegionsRequest) (*HistoryHotRegions, error) - GetRegionStatusByKeyRange(context.Context, *KeyRange, bool) (*RegionStats, error) - GetStores(context.Context) (*StoresInfo, error) - GetStore(context.Context, uint64) (*StoreInfo, error) - SetStoreLabels(context.Context, int64, map[string]string) error - GetMembers(context.Context) (*MembersInfo, error) - GetLeader(context.Context) (*pdpb.Member, error) - TransferLeader(context.Context, string) error - /* Config-related interfaces */ - GetScheduleConfig(context.Context) (map[string]interface{}, error) - SetScheduleConfig(context.Context, map[string]interface{}) error - GetClusterVersion(context.Context) (string, error) - /* Scheduler-related interfaces */ - GetSchedulers(context.Context) ([]string, error) - CreateScheduler(ctx context.Context, name string, storeID uint64) error - SetSchedulerDelay(context.Context, string, int64) error - /* Rule-related interfaces */ - GetAllPlacementRuleBundles(context.Context) ([]*GroupBundle, error) - GetPlacementRuleBundleByGroup(context.Context, string) (*GroupBundle, error) - GetPlacementRulesByGroup(context.Context, string) ([]*Rule, error) - SetPlacementRule(context.Context, *Rule) error - SetPlacementRuleInBatch(context.Context, []*RuleOp) error - SetPlacementRuleBundles(context.Context, []*GroupBundle, bool) error - DeletePlacementRule(context.Context, string, string) error - GetAllPlacementRuleGroups(context.Context) ([]*RuleGroup, error) - GetPlacementRuleGroupByID(context.Context, string) (*RuleGroup, error) - SetPlacementRuleGroup(context.Context, *RuleGroup) error - DeletePlacementRuleGroupByID(context.Context, string) error - GetAllRegionLabelRules(context.Context) ([]*LabelRule, error) - GetRegionLabelRulesByIDs(context.Context, []string) ([]*LabelRule, error) - SetRegionLabelRule(context.Context, *LabelRule) error - PatchRegionLabelRules(context.Context, *LabelRulePatch) error - /* Scheduling-related interfaces */ - AccelerateSchedule(context.Context, *KeyRange) error - AccelerateScheduleInBatch(context.Context, []*KeyRange) error - /* Other interfaces */ - GetMinResolvedTSByStoresIDs(context.Context, []uint64) (uint64, map[uint64]uint64, error) - /* Micro Service interfaces */ - GetMicroServiceMembers(context.Context, string) ([]string, error) - - /* Client-related methods */ - // WithCallerID sets and returns a new client with the given caller ID. - WithCallerID(string) Client - // WithRespHandler sets and returns a new client with the given HTTP response handler. - // This allows the caller to customize how the response is handled, including error handling logic. - // Additionally, it is important for the caller to handle the content of the response body properly - // in order to ensure that it can be read and marshaled correctly into `res`. - WithRespHandler(func(resp *http.Response, res interface{}) error) Client - Close() -} - -var _ Client = (*client)(nil) +// respHandleFunc is the function to handle the HTTP response. +type respHandleFunc func(resp *http.Response, res interface{}) error -// clientInner is the inner implementation of the PD HTTP client, which will -// implement some internal logics, such as HTTP client, service discovery, etc. +// clientInner is the inner implementation of the PD HTTP client, which contains some fundamental fields. +// It is wrapped by the `client` struct to make sure the inner implementation won't be exposed and could +// be consistent during the copy. type clientInner struct { - pdAddrs []string - tlsConf *tls.Config - cli *http.Client -} + ctx context.Context + cancel context.CancelFunc -type client struct { - // Wrap this struct is to make sure the inner implementation - // won't be exposed and cloud be consistent during the copy. - inner *clientInner + sync.RWMutex + pdAddrs []string + leaderAddrIdx int - callerID string - respHandler func(resp *http.Response, res interface{}) error + tlsConf *tls.Config + cli *http.Client requestCounter *prometheus.CounterVec executionDuration *prometheus.HistogramVec } -// ClientOption configures the HTTP client. -type ClientOption func(c *client) - -// WithHTTPClient configures the client with the given initialized HTTP client. -func WithHTTPClient(cli *http.Client) ClientOption { - return func(c *client) { - c.inner.cli = cli - } -} - -// WithTLSConfig configures the client with the given TLS config. -// This option won't work if the client is configured with WithHTTPClient. -func WithTLSConfig(tlsConf *tls.Config) ClientOption { - return func(c *client) { - c.inner.tlsConf = tlsConf - } -} - -// WithMetrics configures the client with metrics. -func WithMetrics( - requestCounter *prometheus.CounterVec, - executionDuration *prometheus.HistogramVec, -) ClientOption { - return func(c *client) { - c.requestCounter = requestCounter - c.executionDuration = executionDuration - } +func newClientInner() *clientInner { + ctx, cancel := context.WithCancel(context.Background()) + return &clientInner{ctx: ctx, cancel: cancel, leaderAddrIdx: -1} } -// NewClient creates a PD HTTP client with the given PD addresses and TLS config. -func NewClient( - pdAddrs []string, - opts ...ClientOption, -) Client { - c := &client{inner: &clientInner{}, callerID: defaultCallerID} - // Apply the options first. - for _, opt := range opts { - opt(c) - } - // Normalize the addresses with correct scheme prefix. - for i, addr := range pdAddrs { - if !strings.HasPrefix(addr, httpScheme) { - var scheme string - if c.inner.tlsConf != nil { - scheme = httpsScheme - } else { - scheme = httpScheme - } - pdAddrs[i] = fmt.Sprintf("%s://%s", scheme, addr) - } - } - c.inner.pdAddrs = pdAddrs +func (ci *clientInner) init() { // Init the HTTP client if it's not configured. - if c.inner.cli == nil { - c.inner.cli = &http.Client{Timeout: defaultTimeout} - if c.inner.tlsConf != nil { + if ci.cli == nil { + ci.cli = &http.Client{Timeout: defaultTimeout} + if ci.tlsConf != nil { transport := http.DefaultTransport.(*http.Transport).Clone() - transport.TLSClientConfig = c.inner.tlsConf - c.inner.cli.Transport = transport + transport.TLSClientConfig = ci.tlsConf + ci.cli.Transport = transport } } - - return c + // Start the members info updater daemon. + go ci.membersInfoUpdater(ci.ctx) } -// Close closes the HTTP client. -func (c *client) Close() { - if c.inner == nil { - return - } - if c.inner.cli != nil { - c.inner.cli.CloseIdleConnections() +func (ci *clientInner) close() { + ci.cancel() + if ci.cli != nil { + ci.cli.CloseIdleConnections() } - log.Info("[pd] http client closed") } -// WithCallerID sets and returns a new client with the given caller ID. -func (c *client) WithCallerID(callerID string) Client { - newClient := *c - newClient.callerID = callerID - return &newClient -} - -// WithRespHandler sets and returns a new client with the given HTTP response handler. -func (c *client) WithRespHandler( - handler func(resp *http.Response, res interface{}) error, -) Client { - newClient := *c - newClient.respHandler = handler - return &newClient +// getPDAddrs returns the current PD addresses and the index of the leader address. +func (ci *clientInner) getPDAddrs() ([]string, int) { + ci.RLock() + defer ci.RUnlock() + return ci.pdAddrs, ci.leaderAddrIdx } -func (c *client) reqCounter(name, status string) { - if c.requestCounter == nil { - return +func (ci *clientInner) setPDAddrs(pdAddrs []string, leaderAddrIdx int) { + ci.Lock() + defer ci.Unlock() + // Normalize the addresses with correct scheme prefix. + var scheme string + if ci.tlsConf == nil { + scheme = httpScheme + } else { + scheme = httpsScheme + } + for i, addr := range pdAddrs { + if strings.HasPrefix(addr, httpScheme) { + continue + } + pdAddrs[i] = fmt.Sprintf("%s://%s", scheme, addr) } - c.requestCounter.WithLabelValues(name, status).Inc() + ci.pdAddrs = pdAddrs + ci.leaderAddrIdx = leaderAddrIdx } -func (c *client) execDuration(name string, duration time.Duration) { - if c.executionDuration == nil { +func (ci *clientInner) reqCounter(name, status string) { + if ci.requestCounter == nil { return } - c.executionDuration.WithLabelValues(name).Observe(duration.Seconds()) + ci.requestCounter.WithLabelValues(name, status).Inc() } -// Header key definition constants. -const ( - pdAllowFollowerHandleKey = "PD-Allow-Follower-Handle" - xCallerIDKey = "X-Caller-ID" -) - -// HeaderOption configures the HTTP header. -type HeaderOption func(header http.Header) - -// WithAllowFollowerHandle sets the header field to allow a PD follower to handle this request. -func WithAllowFollowerHandle() HeaderOption { - return func(header http.Header) { - header.Set(pdAllowFollowerHandleKey, "true") +func (ci *clientInner) execDuration(name string, duration time.Duration) { + if ci.executionDuration == nil { + return } + ci.executionDuration.WithLabelValues(name).Observe(duration.Seconds()) } -// At present, we will use the retry strategy of polling by default to keep -// it consistent with the current implementation of some clients (e.g. TiDB). -func (c *client) requestWithRetry( +// requestWithRetry will first try to send the request to the PD leader, if it fails, it will try to send +// the request to the other PD followers to gain a better availability. +// TODO: support custom retry logic, e.g. retry with customizable backoffer. +func (ci *clientInner) requestWithRetry( ctx context.Context, - name, uri, method string, - body []byte, res interface{}, + reqInfo *requestInfo, headerOpts ...HeaderOption, ) error { var ( - err error - addr string + err error + addr string + pdAddrs, leaderAddrIdx = ci.getPDAddrs() ) - for idx := 0; idx < len(c.inner.pdAddrs); idx++ { - addr = c.inner.pdAddrs[idx] - err = c.request(ctx, name, fmt.Sprintf("%s%s", addr, uri), method, body, res, headerOpts...) + // Try to send the request to the PD leader first. + if leaderAddrIdx != -1 { + addr = pdAddrs[leaderAddrIdx] + err = ci.doRequest(ctx, addr, reqInfo, headerOpts...) + if err == nil { + return nil + } + log.Debug("[pd] request leader addr failed", + zap.Int("leader-idx", leaderAddrIdx), zap.String("addr", addr), zap.Error(err)) + } + // Try to send the request to the other PD followers. + for idx := 0; idx < len(pdAddrs) && idx != leaderAddrIdx; idx++ { + addr = ci.pdAddrs[idx] + err = ci.doRequest(ctx, addr, reqInfo, headerOpts...) if err == nil { break } - log.Debug("[pd] request one addr failed", + log.Debug("[pd] request follower addr failed", zap.Int("idx", idx), zap.String("addr", addr), zap.Error(err)) } return err } -func (c *client) request( +func (ci *clientInner) doRequest( ctx context.Context, - name, url, method string, - body []byte, res interface{}, + addr string, reqInfo *requestInfo, headerOpts ...HeaderOption, ) error { + var ( + callerID = reqInfo.callerID + name = reqInfo.name + url = reqInfo.getURL(addr) + method = reqInfo.method + body = reqInfo.body + res = reqInfo.res + respHandler = reqInfo.respHandler + ) logFields := []zap.Field{ zap.String("name", name), zap.String("url", url), zap.String("method", method), - zap.String("caller-id", c.callerID), + zap.String("caller-id", callerID), } log.Debug("[pd] request the http url", logFields...) req, err := http.NewRequestWithContext(ctx, method, url, bytes.NewBuffer(body)) @@ -292,21 +199,21 @@ func (c *client) request( for _, opt := range headerOpts { opt(req.Header) } - req.Header.Set(xCallerIDKey, c.callerID) + req.Header.Set(xCallerIDKey, callerID) start := time.Now() - resp, err := c.inner.cli.Do(req) + resp, err := ci.cli.Do(req) if err != nil { - c.reqCounter(name, networkErrorStatus) + ci.reqCounter(name, networkErrorStatus) log.Error("[pd] do http request failed", append(logFields, zap.Error(err))...) return errors.Trace(err) } - c.execDuration(name, time.Since(start)) - c.reqCounter(name, resp.Status) + ci.execDuration(name, time.Since(start)) + ci.reqCounter(name, resp.Status) // Give away the response handling to the caller if the handler is set. - if c.respHandler != nil { - return c.respHandler(resp, res) + if respHandler != nil { + return respHandler(resp, res) } defer func() { @@ -341,520 +248,171 @@ func (c *client) request( return nil } -// GetRegionByID gets the region info by ID. -func (c *client) GetRegionByID(ctx context.Context, regionID uint64) (*RegionInfo, error) { - var region RegionInfo - err := c.requestWithRetry(ctx, - "GetRegionByID", RegionByID(regionID), - http.MethodGet, nil, ®ion) - if err != nil { - return nil, err - } - return ®ion, nil -} - -// GetRegionByKey gets the region info by key. -func (c *client) GetRegionByKey(ctx context.Context, key []byte) (*RegionInfo, error) { - var region RegionInfo - err := c.requestWithRetry(ctx, - "GetRegionByKey", RegionByKey(key), - http.MethodGet, nil, ®ion) - if err != nil { - return nil, err - } - return ®ion, nil -} - -// GetRegions gets the regions info. -func (c *client) GetRegions(ctx context.Context) (*RegionsInfo, error) { - var regions RegionsInfo - err := c.requestWithRetry(ctx, - "GetRegions", Regions, - http.MethodGet, nil, ®ions) - if err != nil { - return nil, err - } - return ®ions, nil -} - -// GetRegionsByKeyRange gets the regions info by key range. If the limit is -1, it will return all regions within the range. -// The keys in the key range should be encoded in the UTF-8 bytes format. -func (c *client) GetRegionsByKeyRange(ctx context.Context, keyRange *KeyRange, limit int) (*RegionsInfo, error) { - var regions RegionsInfo - err := c.requestWithRetry(ctx, - "GetRegionsByKeyRange", RegionsByKeyRange(keyRange, limit), - http.MethodGet, nil, ®ions) - if err != nil { - return nil, err - } - return ®ions, nil -} - -// GetRegionsByStoreID gets the regions info by store ID. -func (c *client) GetRegionsByStoreID(ctx context.Context, storeID uint64) (*RegionsInfo, error) { - var regions RegionsInfo - err := c.requestWithRetry(ctx, - "GetRegionsByStoreID", RegionsByStoreID(storeID), - http.MethodGet, nil, ®ions) - if err != nil { - return nil, err - } - return ®ions, nil -} - -// GetRegionsReplicatedStateByKeyRange gets the regions replicated state info by key range. -// The keys in the key range should be encoded in the hex bytes format (without encoding to the UTF-8 bytes). -func (c *client) GetRegionsReplicatedStateByKeyRange(ctx context.Context, keyRange *KeyRange) (string, error) { - var state string - err := c.requestWithRetry(ctx, - "GetRegionsReplicatedStateByKeyRange", RegionsReplicatedByKeyRange(keyRange), - http.MethodGet, nil, &state) - if err != nil { - return "", err - } - return state, nil -} - -// GetHotReadRegions gets the hot read region statistics info. -func (c *client) GetHotReadRegions(ctx context.Context) (*StoreHotPeersInfos, error) { - var hotReadRegions StoreHotPeersInfos - err := c.requestWithRetry(ctx, - "GetHotReadRegions", HotRead, - http.MethodGet, nil, &hotReadRegions) - if err != nil { - return nil, err - } - return &hotReadRegions, nil -} - -// GetHotWriteRegions gets the hot write region statistics info. -func (c *client) GetHotWriteRegions(ctx context.Context) (*StoreHotPeersInfos, error) { - var hotWriteRegions StoreHotPeersInfos - err := c.requestWithRetry(ctx, - "GetHotWriteRegions", HotWrite, - http.MethodGet, nil, &hotWriteRegions) - if err != nil { - return nil, err +func (ci *clientInner) membersInfoUpdater(ctx context.Context) { + ci.updateMembersInfo(ctx) + log.Info("[pd] http client member info updater started") + ticker := time.NewTicker(defaultMembersInfoUpdateInterval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + log.Info("[pd] http client member info updater stopped") + return + case <-ticker.C: + ci.updateMembersInfo(ctx) + } } - return &hotWriteRegions, nil } -// GetHistoryHotRegions gets the history hot region statistics info. -func (c *client) GetHistoryHotRegions(ctx context.Context, req *HistoryHotRegionsRequest) (*HistoryHotRegions, error) { - reqJSON, err := json.Marshal(req) +func (ci *clientInner) updateMembersInfo(ctx context.Context) { + var membersInfo MembersInfo + err := ci.requestWithRetry(ctx, newRequestInfo(). + WithCallerID(defaultInnerCallerID). + WithName(getMembersName). + WithURI(membersPrefix). + WithMethod(http.MethodGet). + WithResp(&membersInfo)) if err != nil { - return nil, errors.Trace(err) + log.Error("[pd] http client get members info failed", zap.Error(err)) + return } - var historyHotRegions HistoryHotRegions - err = c.requestWithRetry(ctx, - "GetHistoryHotRegions", HotHistory, - http.MethodGet, reqJSON, &historyHotRegions, - WithAllowFollowerHandle()) - if err != nil { - return nil, err + if len(membersInfo.Members) == 0 { + log.Error("[pd] http client get empty members info") + return } - return &historyHotRegions, nil -} - -// GetRegionStatusByKeyRange gets the region status by key range. -// If the `onlyCount` flag is true, the result will only include the count of regions. -// The keys in the key range should be encoded in the UTF-8 bytes format. -func (c *client) GetRegionStatusByKeyRange(ctx context.Context, keyRange *KeyRange, onlyCount bool) (*RegionStats, error) { - var regionStats RegionStats - err := c.requestWithRetry(ctx, - "GetRegionStatusByKeyRange", RegionStatsByKeyRange(keyRange, onlyCount), - http.MethodGet, nil, ®ionStats, + var ( + newPDAddrs []string + newLeaderAddrIdx int = -1 ) - if err != nil { - return nil, err - } - return ®ionStats, nil -} - -// SetStoreLabels sets the labels of a store. -func (c *client) SetStoreLabels(ctx context.Context, storeID int64, storeLabels map[string]string) error { - jsonInput, err := json.Marshal(storeLabels) - if err != nil { - return errors.Trace(err) - } - return c.requestWithRetry(ctx, "SetStoreLabel", LabelByStoreID(storeID), - http.MethodPost, jsonInput, nil) -} - -func (c *client) GetMembers(ctx context.Context) (*MembersInfo, error) { - var members MembersInfo - err := c.requestWithRetry(ctx, - "GetMembers", membersPrefix, - http.MethodGet, nil, &members) - if err != nil { - return nil, err - } - return &members, nil -} - -// GetLeader gets the leader of PD cluster. -func (c *client) GetLeader(ctx context.Context) (*pdpb.Member, error) { - var leader pdpb.Member - err := c.requestWithRetry(ctx, "GetLeader", leaderPrefix, - http.MethodGet, nil, &leader) - if err != nil { - return nil, err - } - return &leader, nil -} - -// TransferLeader transfers the PD leader. -func (c *client) TransferLeader(ctx context.Context, newLeader string) error { - return c.requestWithRetry(ctx, "TransferLeader", TransferLeaderByID(newLeader), - http.MethodPost, nil, nil) -} - -// GetScheduleConfig gets the schedule configurations. -func (c *client) GetScheduleConfig(ctx context.Context) (map[string]interface{}, error) { - var config map[string]interface{} - err := c.requestWithRetry(ctx, - "GetScheduleConfig", ScheduleConfig, - http.MethodGet, nil, &config) - if err != nil { - return nil, err - } - return config, nil -} - -// SetScheduleConfig sets the schedule configurations. -func (c *client) SetScheduleConfig(ctx context.Context, config map[string]interface{}) error { - configJSON, err := json.Marshal(config) - if err != nil { - return errors.Trace(err) - } - return c.requestWithRetry(ctx, - "SetScheduleConfig", ScheduleConfig, - http.MethodPost, configJSON, nil) -} - -// GetStores gets the stores info. -func (c *client) GetStores(ctx context.Context) (*StoresInfo, error) { - var stores StoresInfo - err := c.requestWithRetry(ctx, - "GetStores", Stores, - http.MethodGet, nil, &stores) - if err != nil { - return nil, err - } - return &stores, nil -} - -// GetStore gets the store info by ID. -func (c *client) GetStore(ctx context.Context, storeID uint64) (*StoreInfo, error) { - var store StoreInfo - err := c.requestWithRetry(ctx, - "GetStore", StoreByID(storeID), - http.MethodGet, nil, &store) - if err != nil { - return nil, err - } - return &store, nil -} - -// GetClusterVersion gets the cluster version. -func (c *client) GetClusterVersion(ctx context.Context) (string, error) { - var version string - err := c.requestWithRetry(ctx, - "GetClusterVersion", ClusterVersion, - http.MethodGet, nil, &version) - if err != nil { - return "", err - } - return version, nil -} - -// GetAllPlacementRuleBundles gets all placement rules bundles. -func (c *client) GetAllPlacementRuleBundles(ctx context.Context) ([]*GroupBundle, error) { - var bundles []*GroupBundle - err := c.requestWithRetry(ctx, - "GetPlacementRuleBundle", PlacementRuleBundle, - http.MethodGet, nil, &bundles) - if err != nil { - return nil, err - } - return bundles, nil -} - -// GetPlacementRuleBundleByGroup gets the placement rules bundle by group. -func (c *client) GetPlacementRuleBundleByGroup(ctx context.Context, group string) (*GroupBundle, error) { - var bundle GroupBundle - err := c.requestWithRetry(ctx, - "GetPlacementRuleBundleByGroup", PlacementRuleBundleByGroup(group), - http.MethodGet, nil, &bundle) - if err != nil { - return nil, err + for _, member := range membersInfo.Members { + if membersInfo.Leader != nil && member.GetMemberId() == membersInfo.Leader.GetMemberId() { + newLeaderAddrIdx = len(newPDAddrs) + } + newPDAddrs = append(newPDAddrs, member.GetClientUrls()...) } - return &bundle, nil -} - -// GetPlacementRulesByGroup gets the placement rules by group. -func (c *client) GetPlacementRulesByGroup(ctx context.Context, group string) ([]*Rule, error) { - var rules []*Rule - err := c.requestWithRetry(ctx, - "GetPlacementRulesByGroup", PlacementRulesByGroup(group), - http.MethodGet, nil, &rules) - if err != nil { - return nil, err + // Prevent setting empty addresses. + if len(newPDAddrs) == 0 { + log.Error("[pd] http client get empty member addresses") + return } - return rules, nil -} - -// SetPlacementRule sets the placement rule. -func (c *client) SetPlacementRule(ctx context.Context, rule *Rule) error { - ruleJSON, err := json.Marshal(rule) - if err != nil { - return errors.Trace(err) + oldPDAddrs, oldLeaderAddrIdx := ci.getPDAddrs() + ci.setPDAddrs(newPDAddrs, newLeaderAddrIdx) + // Log the member info change if it happens. + var oldPDLeaderAddr, newPDLeaderAddr string + if oldLeaderAddrIdx != -1 { + oldPDLeaderAddr = oldPDAddrs[oldLeaderAddrIdx] } - return c.requestWithRetry(ctx, - "SetPlacementRule", PlacementRule, - http.MethodPost, ruleJSON, nil) -} - -// SetPlacementRuleInBatch sets the placement rules in batch. -func (c *client) SetPlacementRuleInBatch(ctx context.Context, ruleOps []*RuleOp) error { - ruleOpsJSON, err := json.Marshal(ruleOps) - if err != nil { - return errors.Trace(err) + if newLeaderAddrIdx != -1 { + newPDLeaderAddr = newPDAddrs[newLeaderAddrIdx] } - return c.requestWithRetry(ctx, - "SetPlacementRuleInBatch", PlacementRulesInBatch, - http.MethodPost, ruleOpsJSON, nil) -} - -// SetPlacementRuleBundles sets the placement rule bundles. -// If `partial` is false, all old configurations will be over-written and dropped. -func (c *client) SetPlacementRuleBundles(ctx context.Context, bundles []*GroupBundle, partial bool) error { - bundlesJSON, err := json.Marshal(bundles) - if err != nil { - return errors.Trace(err) + oldMemberNum, newMemberNum := len(oldPDAddrs), len(newPDAddrs) + if oldPDLeaderAddr != newPDLeaderAddr || oldMemberNum != newMemberNum { + log.Info("[pd] http client members info changed", + zap.Int("old-member-num", oldMemberNum), zap.Int("new-member-num", newMemberNum), + zap.Strings("old-addrs", oldPDAddrs), zap.Strings("new-addrs", newPDAddrs), + zap.Int("old-leader-addr-idx", oldLeaderAddrIdx), zap.Int("new-leader-addr-idx", newLeaderAddrIdx), + zap.String("old-leader-addr", oldPDLeaderAddr), zap.String("new-leader-addr", newPDLeaderAddr)) } - return c.requestWithRetry(ctx, - "SetPlacementRuleBundles", PlacementRuleBundleWithPartialParameter(partial), - http.MethodPost, bundlesJSON, nil) } -// DeletePlacementRule deletes the placement rule. -func (c *client) DeletePlacementRule(ctx context.Context, group, id string) error { - return c.requestWithRetry(ctx, - "DeletePlacementRule", PlacementRuleByGroupAndID(group, id), - http.MethodDelete, nil, nil) -} +type client struct { + inner *clientInner -// GetAllPlacementRuleGroups gets all placement rule groups. -func (c *client) GetAllPlacementRuleGroups(ctx context.Context) ([]*RuleGroup, error) { - var ruleGroups []*RuleGroup - err := c.requestWithRetry(ctx, - "GetAllPlacementRuleGroups", placementRuleGroups, - http.MethodGet, nil, &ruleGroups) - if err != nil { - return nil, err - } - return ruleGroups, nil + callerID string + respHandler respHandleFunc } -// GetPlacementRuleGroupByID gets the placement rule group by ID. -func (c *client) GetPlacementRuleGroupByID(ctx context.Context, id string) (*RuleGroup, error) { - var ruleGroup RuleGroup - err := c.requestWithRetry(ctx, - "GetPlacementRuleGroupByID", PlacementRuleGroupByID(id), - http.MethodGet, nil, &ruleGroup) - if err != nil { - return nil, err - } - return &ruleGroup, nil -} +// ClientOption configures the HTTP client. +type ClientOption func(c *client) -// SetPlacementRuleGroup sets the placement rule group. -func (c *client) SetPlacementRuleGroup(ctx context.Context, ruleGroup *RuleGroup) error { - ruleGroupJSON, err := json.Marshal(ruleGroup) - if err != nil { - return errors.Trace(err) +// WithHTTPClient configures the client with the given initialized HTTP client. +func WithHTTPClient(cli *http.Client) ClientOption { + return func(c *client) { + c.inner.cli = cli } - return c.requestWithRetry(ctx, - "SetPlacementRuleGroup", placementRuleGroup, - http.MethodPost, ruleGroupJSON, nil) -} - -// DeletePlacementRuleGroupByID deletes the placement rule group by ID. -func (c *client) DeletePlacementRuleGroupByID(ctx context.Context, id string) error { - return c.requestWithRetry(ctx, - "DeletePlacementRuleGroupByID", PlacementRuleGroupByID(id), - http.MethodDelete, nil, nil) } -// GetAllRegionLabelRules gets all region label rules. -func (c *client) GetAllRegionLabelRules(ctx context.Context) ([]*LabelRule, error) { - var labelRules []*LabelRule - err := c.requestWithRetry(ctx, - "GetAllRegionLabelRules", RegionLabelRules, - http.MethodGet, nil, &labelRules) - if err != nil { - return nil, err +// WithTLSConfig configures the client with the given TLS config. +// This option won't work if the client is configured with WithHTTPClient. +func WithTLSConfig(tlsConf *tls.Config) ClientOption { + return func(c *client) { + c.inner.tlsConf = tlsConf } - return labelRules, nil } -// GetRegionLabelRulesByIDs gets the region label rules by IDs. -func (c *client) GetRegionLabelRulesByIDs(ctx context.Context, ruleIDs []string) ([]*LabelRule, error) { - idsJSON, err := json.Marshal(ruleIDs) - if err != nil { - return nil, errors.Trace(err) - } - var labelRules []*LabelRule - err = c.requestWithRetry(ctx, - "GetRegionLabelRulesByIDs", RegionLabelRulesByIDs, - http.MethodGet, idsJSON, &labelRules) - if err != nil { - return nil, err +// WithMetrics configures the client with metrics. +func WithMetrics( + requestCounter *prometheus.CounterVec, + executionDuration *prometheus.HistogramVec, +) ClientOption { + return func(c *client) { + c.inner.requestCounter = requestCounter + c.inner.executionDuration = executionDuration } - return labelRules, nil } -// SetRegionLabelRule sets the region label rule. -func (c *client) SetRegionLabelRule(ctx context.Context, labelRule *LabelRule) error { - labelRuleJSON, err := json.Marshal(labelRule) - if err != nil { - return errors.Trace(err) +// NewClient creates a PD HTTP client with the given PD addresses and TLS config. +func NewClient( + pdAddrs []string, + opts ...ClientOption, +) Client { + c := &client{inner: newClientInner(), callerID: defaultCallerID} + // Apply the options first. + for _, opt := range opts { + opt(c) } - return c.requestWithRetry(ctx, - "SetRegionLabelRule", RegionLabelRule, - http.MethodPost, labelRuleJSON, nil) + c.inner.setPDAddrs(pdAddrs, -1) + c.inner.init() + return c } -// PatchRegionLabelRules patches the region label rules. -func (c *client) PatchRegionLabelRules(ctx context.Context, labelRulePatch *LabelRulePatch) error { - labelRulePatchJSON, err := json.Marshal(labelRulePatch) - if err != nil { - return errors.Trace(err) - } - return c.requestWithRetry(ctx, - "PatchRegionLabelRules", RegionLabelRules, - http.MethodPatch, labelRulePatchJSON, nil) +// Close gracefully closes the HTTP client. +func (c *client) Close() { + c.inner.close() + log.Info("[pd] http client closed") } -// GetSchedulers gets the schedulers from PD cluster. -func (c *client) GetSchedulers(ctx context.Context) ([]string, error) { - var schedulers []string - err := c.requestWithRetry(ctx, "GetSchedulers", Schedulers, - http.MethodGet, nil, &schedulers) - if err != nil { - return nil, err - } - return schedulers, nil +// WithCallerID sets and returns a new client with the given caller ID. +func (c *client) WithCallerID(callerID string) Client { + newClient := *c + newClient.callerID = callerID + return &newClient } -// CreateScheduler creates a scheduler to PD cluster. -func (c *client) CreateScheduler(ctx context.Context, name string, storeID uint64) error { - inputJSON, err := json.Marshal(map[string]interface{}{ - "name": name, - "store_id": storeID, - }) - if err != nil { - return errors.Trace(err) - } - return c.requestWithRetry(ctx, - "CreateScheduler", Schedulers, - http.MethodPost, inputJSON, nil) +// WithRespHandler sets and returns a new client with the given HTTP response handler. +func (c *client) WithRespHandler( + handler func(resp *http.Response, res interface{}) error, +) Client { + newClient := *c + newClient.respHandler = handler + return &newClient } -// AccelerateSchedule accelerates the scheduling of the regions within the given key range. -// The keys in the key range should be encoded in the hex bytes format (without encoding to the UTF-8 bytes). -func (c *client) AccelerateSchedule(ctx context.Context, keyRange *KeyRange) error { - startKey, endKey := keyRange.EscapeAsHexStr() - inputJSON, err := json.Marshal(map[string]string{ - "start_key": startKey, - "end_key": endKey, - }) - if err != nil { - return errors.Trace(err) - } - return c.requestWithRetry(ctx, - "AccelerateSchedule", AccelerateSchedule, - http.MethodPost, inputJSON, nil) -} +// Header key definition constants. +const ( + pdAllowFollowerHandleKey = "PD-Allow-Follower-Handle" + xCallerIDKey = "X-Caller-ID" +) -// AccelerateScheduleInBatch accelerates the scheduling of the regions within the given key ranges in batch. -// The keys in the key ranges should be encoded in the hex bytes format (without encoding to the UTF-8 bytes). -func (c *client) AccelerateScheduleInBatch(ctx context.Context, keyRanges []*KeyRange) error { - input := make([]map[string]string, 0, len(keyRanges)) - for _, keyRange := range keyRanges { - startKey, endKey := keyRange.EscapeAsHexStr() - input = append(input, map[string]string{ - "start_key": startKey, - "end_key": endKey, - }) - } - inputJSON, err := json.Marshal(input) - if err != nil { - return errors.Trace(err) - } - return c.requestWithRetry(ctx, - "AccelerateScheduleInBatch", AccelerateScheduleInBatch, - http.MethodPost, inputJSON, nil) -} +// HeaderOption configures the HTTP header. +type HeaderOption func(header http.Header) -// SetSchedulerDelay sets the delay of given scheduler. -func (c *client) SetSchedulerDelay(ctx context.Context, scheduler string, delaySec int64) error { - m := map[string]int64{ - "delay": delaySec, - } - inputJSON, err := json.Marshal(m) - if err != nil { - return errors.Trace(err) +// WithAllowFollowerHandle sets the header field to allow a PD follower to handle this request. +func WithAllowFollowerHandle() HeaderOption { + return func(header http.Header) { + header.Set(pdAllowFollowerHandleKey, "true") } - return c.requestWithRetry(ctx, - "SetSchedulerDelay", SchedulerByName(scheduler), - http.MethodPost, inputJSON, nil) } -// GetMinResolvedTSByStoresIDs get min-resolved-ts by stores IDs. -// - When storeIDs has zero length, it will return (cluster-level's min_resolved_ts, nil, nil) when no error. -// - When storeIDs is {"cluster"}, it will return (cluster-level's min_resolved_ts, stores_min_resolved_ts, nil) when no error. -// - When storeID is specified to ID lists, it will return (min_resolved_ts of given stores, stores_min_resolved_ts, nil) when no error. -func (c *client) GetMinResolvedTSByStoresIDs(ctx context.Context, storeIDs []uint64) (uint64, map[uint64]uint64, error) { - uri := MinResolvedTSPrefix - // scope is an optional parameter, it can be `cluster` or specified store IDs. - // - When no scope is given, cluster-level's min_resolved_ts will be returned and storesMinResolvedTS will be nil. - // - When scope is `cluster`, cluster-level's min_resolved_ts will be returned and storesMinResolvedTS will be filled. - // - When scope given a list of stores, min_resolved_ts will be provided for each store - // and the scope-specific min_resolved_ts will be returned. - if len(storeIDs) != 0 { - storeIDStrs := make([]string, len(storeIDs)) - for idx, id := range storeIDs { - storeIDStrs[idx] = fmt.Sprintf("%d", id) - } - uri = fmt.Sprintf("%s?scope=%s", uri, strings.Join(storeIDStrs, ",")) - } - resp := struct { - MinResolvedTS uint64 `json:"min_resolved_ts"` - IsRealTime bool `json:"is_real_time,omitempty"` - StoresMinResolvedTS map[uint64]uint64 `json:"stores_min_resolved_ts"` - }{} - err := c.requestWithRetry(ctx, - "GetMinResolvedTSByStoresIDs", uri, - http.MethodGet, nil, &resp) - if err != nil { - return 0, nil, err - } - if !resp.IsRealTime { - return 0, nil, errors.Trace(errors.New("min resolved ts is not enabled")) - } - return resp.MinResolvedTS, resp.StoresMinResolvedTS, nil +func (c *client) request(ctx context.Context, reqInfo *requestInfo, headerOpts ...HeaderOption) error { + return c.inner.requestWithRetry(ctx, reqInfo. + WithCallerID(c.callerID). + WithRespHandler(c.respHandler), + headerOpts...) } -// GetMicroServiceMembers gets the members of the microservice. -func (c *client) GetMicroServiceMembers(ctx context.Context, service string) ([]string, error) { - var members []string - err := c.requestWithRetry(ctx, - "GetMicroServiceMembers", MicroServiceMembers(service), - http.MethodGet, nil, &members) - if err != nil { - return nil, err - } - return members, nil +// UpdateMembersInfo updates the members info of the PD cluster in the inner client. +// Exported for testing. +func (c *client) UpdateMembersInfo() { + c.inner.updateMembersInfo(c.inner.ctx) } diff --git a/client/http/client_test.go b/client/http/client_test.go index 70c2ddee08b..7c7da80827c 100644 --- a/client/http/client_test.go +++ b/client/http/client_test.go @@ -16,12 +16,28 @@ package http import ( "context" + "crypto/tls" "net/http" "testing" "github.com/stretchr/testify/require" + "go.uber.org/atomic" ) +func TestPDAddrNormalization(t *testing.T) { + re := require.New(t) + c := NewClient([]string{"127.0.0.1"}) + pdAddrs, leaderAddrIdx := c.(*client).inner.getPDAddrs() + re.Equal(1, len(pdAddrs)) + re.Equal(-1, leaderAddrIdx) + re.Contains(pdAddrs[0], httpScheme) + c = NewClient([]string{"127.0.0.1"}, WithTLSConfig(&tls.Config{})) + pdAddrs, leaderAddrIdx = c.(*client).inner.getPDAddrs() + re.Equal(1, len(pdAddrs)) + re.Equal(-1, leaderAddrIdx) + re.Contains(pdAddrs[0], httpsScheme) +} + // requestChecker is used to check the HTTP request sent by the client. type requestChecker struct { checker func(req *http.Request) error @@ -40,8 +56,11 @@ func newHTTPClientWithRequestChecker(checker func(req *http.Request) error) *htt func TestPDAllowFollowerHandleHeader(t *testing.T) { re := require.New(t) - var expectedVal string httpClient := newHTTPClientWithRequestChecker(func(req *http.Request) error { + var expectedVal string + if req.URL.Path == HotHistory { + expectedVal = "true" + } val := req.Header.Get(pdAllowFollowerHandleKey) if val != expectedVal { re.Failf("PD allow follower handler header check failed", @@ -51,16 +70,17 @@ func TestPDAllowFollowerHandleHeader(t *testing.T) { }) c := NewClient([]string{"http://127.0.0.1"}, WithHTTPClient(httpClient)) c.GetRegions(context.Background()) - expectedVal = "true" c.GetHistoryHotRegions(context.Background(), &HistoryHotRegionsRequest{}) + c.Close() } func TestCallerID(t *testing.T) { re := require.New(t) - expectedVal := defaultCallerID + expectedVal := atomic.NewString(defaultCallerID) httpClient := newHTTPClientWithRequestChecker(func(req *http.Request) error { val := req.Header.Get(xCallerIDKey) - if val != expectedVal { + // Exclude the request sent by the inner client. + if val != defaultInnerCallerID && val != expectedVal.Load() { re.Failf("Caller ID header check failed", "should be %s, but got %s", expectedVal, val) } @@ -68,6 +88,7 @@ func TestCallerID(t *testing.T) { }) c := NewClient([]string{"http://127.0.0.1"}, WithHTTPClient(httpClient)) c.GetRegions(context.Background()) - expectedVal = "test" - c.WithCallerID(expectedVal).GetRegions(context.Background()) + expectedVal.Store("test") + c.WithCallerID(expectedVal.Load()).GetRegions(context.Background()) + c.Close() } diff --git a/client/http/interface.go b/client/http/interface.go new file mode 100644 index 00000000000..25fa99fa0bd --- /dev/null +++ b/client/http/interface.go @@ -0,0 +1,695 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package http + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "strings" + + "github.com/pingcap/errors" + "github.com/pingcap/kvproto/pkg/pdpb" +) + +// Client is a PD (Placement Driver) HTTP client. +type Client interface { + /* Member-related interfaces */ + GetMembers(context.Context) (*MembersInfo, error) + GetLeader(context.Context) (*pdpb.Member, error) + TransferLeader(context.Context, string) error + /* Meta-related interfaces */ + GetRegionByID(context.Context, uint64) (*RegionInfo, error) + GetRegionByKey(context.Context, []byte) (*RegionInfo, error) + GetRegions(context.Context) (*RegionsInfo, error) + GetRegionsByKeyRange(context.Context, *KeyRange, int) (*RegionsInfo, error) + GetRegionsByStoreID(context.Context, uint64) (*RegionsInfo, error) + GetRegionsReplicatedStateByKeyRange(context.Context, *KeyRange) (string, error) + GetHotReadRegions(context.Context) (*StoreHotPeersInfos, error) + GetHotWriteRegions(context.Context) (*StoreHotPeersInfos, error) + GetHistoryHotRegions(context.Context, *HistoryHotRegionsRequest) (*HistoryHotRegions, error) + GetRegionStatusByKeyRange(context.Context, *KeyRange, bool) (*RegionStats, error) + GetStores(context.Context) (*StoresInfo, error) + GetStore(context.Context, uint64) (*StoreInfo, error) + SetStoreLabels(context.Context, int64, map[string]string) error + /* Config-related interfaces */ + GetScheduleConfig(context.Context) (map[string]interface{}, error) + SetScheduleConfig(context.Context, map[string]interface{}) error + GetClusterVersion(context.Context) (string, error) + /* Scheduler-related interfaces */ + GetSchedulers(context.Context) ([]string, error) + CreateScheduler(ctx context.Context, name string, storeID uint64) error + SetSchedulerDelay(context.Context, string, int64) error + /* Rule-related interfaces */ + GetAllPlacementRuleBundles(context.Context) ([]*GroupBundle, error) + GetPlacementRuleBundleByGroup(context.Context, string) (*GroupBundle, error) + GetPlacementRulesByGroup(context.Context, string) ([]*Rule, error) + SetPlacementRule(context.Context, *Rule) error + SetPlacementRuleInBatch(context.Context, []*RuleOp) error + SetPlacementRuleBundles(context.Context, []*GroupBundle, bool) error + DeletePlacementRule(context.Context, string, string) error + GetAllPlacementRuleGroups(context.Context) ([]*RuleGroup, error) + GetPlacementRuleGroupByID(context.Context, string) (*RuleGroup, error) + SetPlacementRuleGroup(context.Context, *RuleGroup) error + DeletePlacementRuleGroupByID(context.Context, string) error + GetAllRegionLabelRules(context.Context) ([]*LabelRule, error) + GetRegionLabelRulesByIDs(context.Context, []string) ([]*LabelRule, error) + SetRegionLabelRule(context.Context, *LabelRule) error + PatchRegionLabelRules(context.Context, *LabelRulePatch) error + /* Scheduling-related interfaces */ + AccelerateSchedule(context.Context, *KeyRange) error + AccelerateScheduleInBatch(context.Context, []*KeyRange) error + /* Other interfaces */ + GetMinResolvedTSByStoresIDs(context.Context, []uint64) (uint64, map[uint64]uint64, error) + /* Micro Service interfaces */ + GetMicroServiceMembers(context.Context, string) ([]string, error) + + /* Client-related methods */ + // WithCallerID sets and returns a new client with the given caller ID. + WithCallerID(string) Client + // WithRespHandler sets and returns a new client with the given HTTP response handler. + // This allows the caller to customize how the response is handled, including error handling logic. + // Additionally, it is important for the caller to handle the content of the response body properly + // in order to ensure that it can be read and marshaled correctly into `res`. + WithRespHandler(func(resp *http.Response, res interface{}) error) Client + // Close gracefully closes the HTTP client. + Close() +} + +var _ Client = (*client)(nil) + +// GetMembers gets the members info of PD cluster. +func (c *client) GetMembers(ctx context.Context) (*MembersInfo, error) { + var members MembersInfo + err := c.request(ctx, newRequestInfo(). + WithName(getMembersName). + WithURI(membersPrefix). + WithMethod(http.MethodGet). + WithResp(&members)) + if err != nil { + return nil, err + } + return &members, nil +} + +// GetLeader gets the leader of PD cluster. +func (c *client) GetLeader(ctx context.Context) (*pdpb.Member, error) { + var leader pdpb.Member + err := c.request(ctx, newRequestInfo(). + WithName(getLeaderName). + WithURI(leaderPrefix). + WithMethod(http.MethodGet). + WithResp(&leader)) + if err != nil { + return nil, err + } + return &leader, nil +} + +// TransferLeader transfers the PD leader. +func (c *client) TransferLeader(ctx context.Context, newLeader string) error { + return c.request(ctx, newRequestInfo(). + WithName(transferLeaderName). + WithURI(TransferLeaderByID(newLeader)). + WithMethod(http.MethodPost)) +} + +// GetRegionByID gets the region info by ID. +func (c *client) GetRegionByID(ctx context.Context, regionID uint64) (*RegionInfo, error) { + var region RegionInfo + err := c.request(ctx, newRequestInfo(). + WithName(getRegionByIDName). + WithURI(RegionByID(regionID)). + WithMethod(http.MethodGet). + WithResp(®ion)) + if err != nil { + return nil, err + } + return ®ion, nil +} + +// GetRegionByKey gets the region info by key. +func (c *client) GetRegionByKey(ctx context.Context, key []byte) (*RegionInfo, error) { + var region RegionInfo + err := c.request(ctx, newRequestInfo(). + WithName(getRegionByKeyName). + WithURI(RegionByKey(key)). + WithMethod(http.MethodGet). + WithResp(®ion)) + if err != nil { + return nil, err + } + return ®ion, nil +} + +// GetRegions gets the regions info. +func (c *client) GetRegions(ctx context.Context) (*RegionsInfo, error) { + var regions RegionsInfo + err := c.request(ctx, newRequestInfo(). + WithName(getRegionsName). + WithURI(Regions). + WithMethod(http.MethodGet). + WithResp(®ions)) + if err != nil { + return nil, err + } + return ®ions, nil +} + +// GetRegionsByKeyRange gets the regions info by key range. If the limit is -1, it will return all regions within the range. +// The keys in the key range should be encoded in the UTF-8 bytes format. +func (c *client) GetRegionsByKeyRange(ctx context.Context, keyRange *KeyRange, limit int) (*RegionsInfo, error) { + var regions RegionsInfo + err := c.request(ctx, newRequestInfo(). + WithName(getRegionsByKeyRangeName). + WithURI(RegionsByKeyRange(keyRange, limit)). + WithMethod(http.MethodGet). + WithResp(®ions)) + if err != nil { + return nil, err + } + return ®ions, nil +} + +// GetRegionsByStoreID gets the regions info by store ID. +func (c *client) GetRegionsByStoreID(ctx context.Context, storeID uint64) (*RegionsInfo, error) { + var regions RegionsInfo + err := c.request(ctx, newRequestInfo(). + WithName(getRegionsByStoreIDName). + WithURI(RegionsByStoreID(storeID)). + WithMethod(http.MethodGet). + WithResp(®ions)) + if err != nil { + return nil, err + } + return ®ions, nil +} + +// GetRegionsReplicatedStateByKeyRange gets the regions replicated state info by key range. +// The keys in the key range should be encoded in the hex bytes format (without encoding to the UTF-8 bytes). +func (c *client) GetRegionsReplicatedStateByKeyRange(ctx context.Context, keyRange *KeyRange) (string, error) { + var state string + err := c.request(ctx, newRequestInfo(). + WithName(getRegionsReplicatedStateByKeyRangeName). + WithURI(RegionsReplicatedByKeyRange(keyRange)). + WithMethod(http.MethodGet). + WithResp(&state)) + if err != nil { + return "", err + } + return state, nil +} + +// GetHotReadRegions gets the hot read region statistics info. +func (c *client) GetHotReadRegions(ctx context.Context) (*StoreHotPeersInfos, error) { + var hotReadRegions StoreHotPeersInfos + err := c.request(ctx, newRequestInfo(). + WithName(getHotReadRegionsName). + WithURI(HotRead). + WithMethod(http.MethodGet). + WithResp(&hotReadRegions)) + if err != nil { + return nil, err + } + return &hotReadRegions, nil +} + +// GetHotWriteRegions gets the hot write region statistics info. +func (c *client) GetHotWriteRegions(ctx context.Context) (*StoreHotPeersInfos, error) { + var hotWriteRegions StoreHotPeersInfos + err := c.request(ctx, newRequestInfo(). + WithName(getHotWriteRegionsName). + WithURI(HotWrite). + WithMethod(http.MethodGet). + WithResp(&hotWriteRegions)) + if err != nil { + return nil, err + } + return &hotWriteRegions, nil +} + +// GetHistoryHotRegions gets the history hot region statistics info. +func (c *client) GetHistoryHotRegions(ctx context.Context, req *HistoryHotRegionsRequest) (*HistoryHotRegions, error) { + reqJSON, err := json.Marshal(req) + if err != nil { + return nil, errors.Trace(err) + } + var historyHotRegions HistoryHotRegions + err = c.request(ctx, newRequestInfo(). + WithName(getHistoryHotRegionsName). + WithURI(HotHistory). + WithMethod(http.MethodGet). + WithBody(reqJSON). + WithResp(&historyHotRegions), + WithAllowFollowerHandle()) + if err != nil { + return nil, err + } + return &historyHotRegions, nil +} + +// GetRegionStatusByKeyRange gets the region status by key range. +// If the `onlyCount` flag is true, the result will only include the count of regions. +// The keys in the key range should be encoded in the UTF-8 bytes format. +func (c *client) GetRegionStatusByKeyRange(ctx context.Context, keyRange *KeyRange, onlyCount bool) (*RegionStats, error) { + var regionStats RegionStats + err := c.request(ctx, newRequestInfo(). + WithName(getRegionStatusByKeyRangeName). + WithURI(RegionStatsByKeyRange(keyRange, onlyCount)). + WithMethod(http.MethodGet). + WithResp(®ionStats)) + if err != nil { + return nil, err + } + return ®ionStats, nil +} + +// SetStoreLabels sets the labels of a store. +func (c *client) SetStoreLabels(ctx context.Context, storeID int64, storeLabels map[string]string) error { + jsonInput, err := json.Marshal(storeLabels) + if err != nil { + return errors.Trace(err) + } + return c.request(ctx, newRequestInfo(). + WithName(setStoreLabelsName). + WithURI(LabelByStoreID(storeID)). + WithMethod(http.MethodPost). + WithBody(jsonInput)) +} + +// GetScheduleConfig gets the schedule configurations. +func (c *client) GetScheduleConfig(ctx context.Context) (map[string]interface{}, error) { + var config map[string]interface{} + err := c.request(ctx, newRequestInfo(). + WithName(getScheduleConfigName). + WithURI(ScheduleConfig). + WithMethod(http.MethodGet). + WithResp(&config)) + if err != nil { + return nil, err + } + return config, nil +} + +// SetScheduleConfig sets the schedule configurations. +func (c *client) SetScheduleConfig(ctx context.Context, config map[string]interface{}) error { + configJSON, err := json.Marshal(config) + if err != nil { + return errors.Trace(err) + } + return c.request(ctx, newRequestInfo(). + WithName(setScheduleConfigName). + WithURI(ScheduleConfig). + WithMethod(http.MethodPost). + WithBody(configJSON)) +} + +// GetStores gets the stores info. +func (c *client) GetStores(ctx context.Context) (*StoresInfo, error) { + var stores StoresInfo + err := c.request(ctx, newRequestInfo(). + WithName(getStoresName). + WithURI(Stores). + WithMethod(http.MethodGet). + WithResp(&stores)) + if err != nil { + return nil, err + } + return &stores, nil +} + +// GetStore gets the store info by ID. +func (c *client) GetStore(ctx context.Context, storeID uint64) (*StoreInfo, error) { + var store StoreInfo + err := c.request(ctx, newRequestInfo(). + WithName(getStoreName). + WithURI(StoreByID(storeID)). + WithMethod(http.MethodGet). + WithResp(&store)) + if err != nil { + return nil, err + } + return &store, nil +} + +// GetClusterVersion gets the cluster version. +func (c *client) GetClusterVersion(ctx context.Context) (string, error) { + var version string + err := c.request(ctx, newRequestInfo(). + WithName(getClusterVersionName). + WithURI(ClusterVersion). + WithMethod(http.MethodGet). + WithResp(&version)) + if err != nil { + return "", err + } + return version, nil +} + +// GetAllPlacementRuleBundles gets all placement rules bundles. +func (c *client) GetAllPlacementRuleBundles(ctx context.Context) ([]*GroupBundle, error) { + var bundles []*GroupBundle + err := c.request(ctx, newRequestInfo(). + WithName(getAllPlacementRuleBundlesName). + WithURI(PlacementRuleBundle). + WithMethod(http.MethodGet). + WithResp(&bundles)) + if err != nil { + return nil, err + } + return bundles, nil +} + +// GetPlacementRuleBundleByGroup gets the placement rules bundle by group. +func (c *client) GetPlacementRuleBundleByGroup(ctx context.Context, group string) (*GroupBundle, error) { + var bundle GroupBundle + err := c.request(ctx, newRequestInfo(). + WithName(getPlacementRuleBundleByGroupName). + WithURI(PlacementRuleBundleByGroup(group)). + WithMethod(http.MethodGet). + WithResp(&bundle)) + if err != nil { + return nil, err + } + return &bundle, nil +} + +// GetPlacementRulesByGroup gets the placement rules by group. +func (c *client) GetPlacementRulesByGroup(ctx context.Context, group string) ([]*Rule, error) { + var rules []*Rule + err := c.request(ctx, newRequestInfo(). + WithName(getPlacementRulesByGroupName). + WithURI(PlacementRulesByGroup(group)). + WithMethod(http.MethodGet). + WithResp(&rules)) + if err != nil { + return nil, err + } + return rules, nil +} + +// SetPlacementRule sets the placement rule. +func (c *client) SetPlacementRule(ctx context.Context, rule *Rule) error { + ruleJSON, err := json.Marshal(rule) + if err != nil { + return errors.Trace(err) + } + return c.request(ctx, newRequestInfo(). + WithName(setPlacementRuleName). + WithURI(PlacementRule). + WithMethod(http.MethodPost). + WithBody(ruleJSON)) +} + +// SetPlacementRuleInBatch sets the placement rules in batch. +func (c *client) SetPlacementRuleInBatch(ctx context.Context, ruleOps []*RuleOp) error { + ruleOpsJSON, err := json.Marshal(ruleOps) + if err != nil { + return errors.Trace(err) + } + return c.request(ctx, newRequestInfo(). + WithName(setPlacementRuleInBatchName). + WithURI(PlacementRulesInBatch). + WithMethod(http.MethodPost). + WithBody(ruleOpsJSON)) +} + +// SetPlacementRuleBundles sets the placement rule bundles. +// If `partial` is false, all old configurations will be over-written and dropped. +func (c *client) SetPlacementRuleBundles(ctx context.Context, bundles []*GroupBundle, partial bool) error { + bundlesJSON, err := json.Marshal(bundles) + if err != nil { + return errors.Trace(err) + } + return c.request(ctx, newRequestInfo(). + WithName(setPlacementRuleBundlesName). + WithURI(PlacementRuleBundleWithPartialParameter(partial)). + WithMethod(http.MethodPost). + WithBody(bundlesJSON)) +} + +// DeletePlacementRule deletes the placement rule. +func (c *client) DeletePlacementRule(ctx context.Context, group, id string) error { + return c.request(ctx, newRequestInfo(). + WithName(deletePlacementRuleName). + WithURI(PlacementRuleByGroupAndID(group, id)). + WithMethod(http.MethodDelete)) +} + +// GetAllPlacementRuleGroups gets all placement rule groups. +func (c *client) GetAllPlacementRuleGroups(ctx context.Context) ([]*RuleGroup, error) { + var ruleGroups []*RuleGroup + err := c.request(ctx, newRequestInfo(). + WithName(getAllPlacementRuleGroupsName). + WithURI(placementRuleGroups). + WithMethod(http.MethodGet). + WithResp(&ruleGroups)) + if err != nil { + return nil, err + } + return ruleGroups, nil +} + +// GetPlacementRuleGroupByID gets the placement rule group by ID. +func (c *client) GetPlacementRuleGroupByID(ctx context.Context, id string) (*RuleGroup, error) { + var ruleGroup RuleGroup + err := c.request(ctx, newRequestInfo(). + WithName(getPlacementRuleGroupByIDName). + WithURI(PlacementRuleGroupByID(id)). + WithMethod(http.MethodGet). + WithResp(&ruleGroup)) + if err != nil { + return nil, err + } + return &ruleGroup, nil +} + +// SetPlacementRuleGroup sets the placement rule group. +func (c *client) SetPlacementRuleGroup(ctx context.Context, ruleGroup *RuleGroup) error { + ruleGroupJSON, err := json.Marshal(ruleGroup) + if err != nil { + return errors.Trace(err) + } + return c.request(ctx, newRequestInfo(). + WithName(setPlacementRuleGroupName). + WithURI(placementRuleGroup). + WithMethod(http.MethodPost). + WithBody(ruleGroupJSON)) +} + +// DeletePlacementRuleGroupByID deletes the placement rule group by ID. +func (c *client) DeletePlacementRuleGroupByID(ctx context.Context, id string) error { + return c.request(ctx, newRequestInfo(). + WithName(deletePlacementRuleGroupByIDName). + WithURI(PlacementRuleGroupByID(id)). + WithMethod(http.MethodDelete)) +} + +// GetAllRegionLabelRules gets all region label rules. +func (c *client) GetAllRegionLabelRules(ctx context.Context) ([]*LabelRule, error) { + var labelRules []*LabelRule + err := c.request(ctx, newRequestInfo(). + WithName(getAllRegionLabelRulesName). + WithURI(RegionLabelRules). + WithMethod(http.MethodGet). + WithResp(&labelRules)) + if err != nil { + return nil, err + } + return labelRules, nil +} + +// GetRegionLabelRulesByIDs gets the region label rules by IDs. +func (c *client) GetRegionLabelRulesByIDs(ctx context.Context, ruleIDs []string) ([]*LabelRule, error) { + idsJSON, err := json.Marshal(ruleIDs) + if err != nil { + return nil, errors.Trace(err) + } + var labelRules []*LabelRule + err = c.request(ctx, newRequestInfo(). + WithName(getRegionLabelRulesByIDsName). + WithURI(RegionLabelRules). + WithMethod(http.MethodGet). + WithBody(idsJSON). + WithResp(&labelRules)) + if err != nil { + return nil, err + } + return labelRules, nil +} + +// SetRegionLabelRule sets the region label rule. +func (c *client) SetRegionLabelRule(ctx context.Context, labelRule *LabelRule) error { + labelRuleJSON, err := json.Marshal(labelRule) + if err != nil { + return errors.Trace(err) + } + return c.request(ctx, newRequestInfo(). + WithName(setRegionLabelRuleName). + WithURI(RegionLabelRule). + WithMethod(http.MethodPost). + WithBody(labelRuleJSON)) +} + +// PatchRegionLabelRules patches the region label rules. +func (c *client) PatchRegionLabelRules(ctx context.Context, labelRulePatch *LabelRulePatch) error { + labelRulePatchJSON, err := json.Marshal(labelRulePatch) + if err != nil { + return errors.Trace(err) + } + return c.request(ctx, newRequestInfo(). + WithName(patchRegionLabelRulesName). + WithURI(RegionLabelRules). + WithMethod(http.MethodPatch). + WithBody(labelRulePatchJSON)) +} + +// GetSchedulers gets the schedulers from PD cluster. +func (c *client) GetSchedulers(ctx context.Context) ([]string, error) { + var schedulers []string + err := c.request(ctx, newRequestInfo(). + WithName(getSchedulersName). + WithURI(Schedulers). + WithMethod(http.MethodGet). + WithResp(&schedulers)) + if err != nil { + return nil, err + } + return schedulers, nil +} + +// CreateScheduler creates a scheduler to PD cluster. +func (c *client) CreateScheduler(ctx context.Context, name string, storeID uint64) error { + inputJSON, err := json.Marshal(map[string]interface{}{ + "name": name, + "store_id": storeID, + }) + if err != nil { + return errors.Trace(err) + } + return c.request(ctx, newRequestInfo(). + WithName(createSchedulerName). + WithURI(Schedulers). + WithMethod(http.MethodPost). + WithBody(inputJSON)) +} + +// AccelerateSchedule accelerates the scheduling of the regions within the given key range. +// The keys in the key range should be encoded in the hex bytes format (without encoding to the UTF-8 bytes). +func (c *client) AccelerateSchedule(ctx context.Context, keyRange *KeyRange) error { + startKey, endKey := keyRange.EscapeAsHexStr() + inputJSON, err := json.Marshal(map[string]string{ + "start_key": startKey, + "end_key": endKey, + }) + if err != nil { + return errors.Trace(err) + } + return c.request(ctx, newRequestInfo(). + WithName(accelerateScheduleName). + WithURI(AccelerateSchedule). + WithMethod(http.MethodPost). + WithBody(inputJSON)) +} + +// AccelerateScheduleInBatch accelerates the scheduling of the regions within the given key ranges in batch. +// The keys in the key ranges should be encoded in the hex bytes format (without encoding to the UTF-8 bytes). +func (c *client) AccelerateScheduleInBatch(ctx context.Context, keyRanges []*KeyRange) error { + input := make([]map[string]string, 0, len(keyRanges)) + for _, keyRange := range keyRanges { + startKey, endKey := keyRange.EscapeAsHexStr() + input = append(input, map[string]string{ + "start_key": startKey, + "end_key": endKey, + }) + } + inputJSON, err := json.Marshal(input) + if err != nil { + return errors.Trace(err) + } + return c.request(ctx, newRequestInfo(). + WithName(accelerateScheduleInBatchName). + WithURI(AccelerateScheduleInBatch). + WithMethod(http.MethodPost). + WithBody(inputJSON)) +} + +// SetSchedulerDelay sets the delay of given scheduler. +func (c *client) SetSchedulerDelay(ctx context.Context, scheduler string, delaySec int64) error { + m := map[string]int64{ + "delay": delaySec, + } + inputJSON, err := json.Marshal(m) + if err != nil { + return errors.Trace(err) + } + return c.request(ctx, newRequestInfo(). + WithName(setSchedulerDelayName). + WithURI(SchedulerByName(scheduler)). + WithMethod(http.MethodPost). + WithBody(inputJSON)) +} + +// GetMinResolvedTSByStoresIDs get min-resolved-ts by stores IDs. +// - When storeIDs has zero length, it will return (cluster-level's min_resolved_ts, nil, nil) when no error. +// - When storeIDs is {"cluster"}, it will return (cluster-level's min_resolved_ts, stores_min_resolved_ts, nil) when no error. +// - When storeID is specified to ID lists, it will return (min_resolved_ts of given stores, stores_min_resolved_ts, nil) when no error. +func (c *client) GetMinResolvedTSByStoresIDs(ctx context.Context, storeIDs []uint64) (uint64, map[uint64]uint64, error) { + uri := MinResolvedTSPrefix + // scope is an optional parameter, it can be `cluster` or specified store IDs. + // - When no scope is given, cluster-level's min_resolved_ts will be returned and storesMinResolvedTS will be nil. + // - When scope is `cluster`, cluster-level's min_resolved_ts will be returned and storesMinResolvedTS will be filled. + // - When scope given a list of stores, min_resolved_ts will be provided for each store + // and the scope-specific min_resolved_ts will be returned. + if len(storeIDs) != 0 { + storeIDStrs := make([]string, len(storeIDs)) + for idx, id := range storeIDs { + storeIDStrs[idx] = fmt.Sprintf("%d", id) + } + uri = fmt.Sprintf("%s?scope=%s", uri, strings.Join(storeIDStrs, ",")) + } + resp := struct { + MinResolvedTS uint64 `json:"min_resolved_ts"` + IsRealTime bool `json:"is_real_time,omitempty"` + StoresMinResolvedTS map[uint64]uint64 `json:"stores_min_resolved_ts"` + }{} + err := c.request(ctx, newRequestInfo(). + WithName(getMinResolvedTSByStoresIDsName). + WithURI(uri). + WithMethod(http.MethodGet). + WithResp(&resp)) + if err != nil { + return 0, nil, err + } + if !resp.IsRealTime { + return 0, nil, errors.Trace(errors.New("min resolved ts is not enabled")) + } + return resp.MinResolvedTS, resp.StoresMinResolvedTS, nil +} + +// GetMicroServiceMembers gets the members of the microservice. +func (c *client) GetMicroServiceMembers(ctx context.Context, service string) ([]string, error) { + var members []string + err := c.request(ctx, newRequestInfo(). + WithName(getMicroServiceMembersName). + WithURI(MicroServiceMembers(service)). + WithMethod(http.MethodGet). + WithResp(&members)) + if err != nil { + return nil, err + } + return members, nil +} diff --git a/client/http/request_info.go b/client/http/request_info.go new file mode 100644 index 00000000000..dc811618d9c --- /dev/null +++ b/client/http/request_info.go @@ -0,0 +1,123 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package http + +import "fmt" + +// The following constants are the names of the requests. +const ( + getMembersName = "GetMembers" + getLeaderName = "GetLeader" + transferLeaderName = "TransferLeader" + getRegionByIDName = "GetRegionByID" + getRegionByKeyName = "GetRegionByKey" + getRegionsName = "GetRegions" + getRegionsByKeyRangeName = "GetRegionsByKeyRange" + getRegionsByStoreIDName = "GetRegionsByStoreID" + getRegionsReplicatedStateByKeyRangeName = "GetRegionsReplicatedStateByKeyRange" + getHotReadRegionsName = "GetHotReadRegions" + getHotWriteRegionsName = "GetHotWriteRegions" + getHistoryHotRegionsName = "GetHistoryHotRegions" + getRegionStatusByKeyRangeName = "GetRegionStatusByKeyRange" + getStoresName = "GetStores" + getStoreName = "GetStore" + setStoreLabelsName = "SetStoreLabels" + getScheduleConfigName = "GetScheduleConfig" + setScheduleConfigName = "SetScheduleConfig" + getClusterVersionName = "GetClusterVersion" + getSchedulersName = "GetSchedulers" + createSchedulerName = "CreateScheduler" + setSchedulerDelayName = "SetSchedulerDelay" + getAllPlacementRuleBundlesName = "GetAllPlacementRuleBundles" + getPlacementRuleBundleByGroupName = "GetPlacementRuleBundleByGroup" + getPlacementRulesByGroupName = "GetPlacementRulesByGroup" + setPlacementRuleName = "SetPlacementRule" + setPlacementRuleInBatchName = "SetPlacementRuleInBatch" + setPlacementRuleBundlesName = "SetPlacementRuleBundles" + deletePlacementRuleName = "DeletePlacementRule" + getAllPlacementRuleGroupsName = "GetAllPlacementRuleGroups" + getPlacementRuleGroupByIDName = "GetPlacementRuleGroupByID" + setPlacementRuleGroupName = "SetPlacementRuleGroup" + deletePlacementRuleGroupByIDName = "DeletePlacementRuleGroupByID" + getAllRegionLabelRulesName = "GetAllRegionLabelRules" + getRegionLabelRulesByIDsName = "GetRegionLabelRulesByIDs" + setRegionLabelRuleName = "SetRegionLabelRule" + patchRegionLabelRulesName = "PatchRegionLabelRules" + accelerateScheduleName = "AccelerateSchedule" + accelerateScheduleInBatchName = "AccelerateScheduleInBatch" + getMinResolvedTSByStoresIDsName = "GetMinResolvedTSByStoresIDs" + getMicroServiceMembersName = "GetMicroServiceMembers" +) + +type requestInfo struct { + callerID string + name string + uri string + method string + body []byte + res interface{} + respHandler respHandleFunc +} + +// newRequestInfo creates a new request info. +func newRequestInfo() *requestInfo { + return &requestInfo{} +} + +// WithCallerID sets the caller ID of the request. +func (ri *requestInfo) WithCallerID(callerID string) *requestInfo { + ri.callerID = callerID + return ri +} + +// WithName sets the name of the request. +func (ri *requestInfo) WithName(name string) *requestInfo { + ri.name = name + return ri +} + +// WithURI sets the URI of the request. +func (ri *requestInfo) WithURI(uri string) *requestInfo { + ri.uri = uri + return ri +} + +// WithMethod sets the method of the request. +func (ri *requestInfo) WithMethod(method string) *requestInfo { + ri.method = method + return ri +} + +// WithBody sets the body of the request. +func (ri *requestInfo) WithBody(body []byte) *requestInfo { + ri.body = body + return ri +} + +// WithResp sets the response struct of the request. +func (ri *requestInfo) WithResp(res interface{}) *requestInfo { + ri.res = res + return ri +} + +// WithRespHandler sets the response handle function of the request. +func (ri *requestInfo) WithRespHandler(respHandler respHandleFunc) *requestInfo { + ri.respHandler = respHandler + return ri +} + +func (ri *requestInfo) getURL(addr string) string { + return fmt.Sprintf("%s%s", addr, ri.uri) +} diff --git a/tests/integrations/client/http_client_test.go b/tests/integrations/client/http_client_test.go index 7c8f66f4826..1a20ae2e784 100644 --- a/tests/integrations/client/http_client_test.go +++ b/tests/integrations/client/http_client_test.go @@ -438,13 +438,13 @@ func (suite *httpClientTestSuite) TestTransferLeader() { re.NoError(err) re.Len(members.Members, 2) - oldLeader, err := suite.client.GetLeader(suite.ctx) + leader, err := suite.client.GetLeader(suite.ctx) re.NoError(err) // Transfer leader to another pd for _, member := range members.Members { - if member.Name != oldLeader.Name { - err = suite.client.TransferLeader(suite.ctx, member.Name) + if member.GetName() != leader.GetName() { + err = suite.client.TransferLeader(suite.ctx, member.GetName()) re.NoError(err) break } @@ -453,5 +453,14 @@ func (suite *httpClientTestSuite) TestTransferLeader() { newLeader := suite.cluster.WaitLeader() re.NotEmpty(newLeader) re.NoError(err) - re.NotEqual(oldLeader.Name, newLeader) + re.NotEqual(leader.GetName(), newLeader) + // Force to update the members info. + suite.client.(interface{ UpdateMembersInfo() }).UpdateMembersInfo() + leader, err = suite.client.GetLeader(suite.ctx) + re.NoError(err) + re.Equal(newLeader, leader.GetName()) + members, err = suite.client.GetMembers(suite.ctx) + re.NoError(err) + re.Len(members.Members, 2) + re.Equal(leader.GetName(), members.Leader.GetName()) } From cfb1d8f94134b598f924aca231fee66326106ce8 Mon Sep 17 00:00:00 2001 From: lucasliang Date: Mon, 18 Dec 2023 18:27:24 +0800 Subject: [PATCH 101/137] scheduler: enable `evict-slow-store` by default. (#7505) close tikv/pd#7564, ref tikv/tikv#15909 Enable `evict-slow-store` scheduler by default. Signed-off-by: lucasliang --- pkg/schedule/config/config.go | 1 + server/cluster/cluster_test.go | 18 ++++++++++------- tests/integrations/mcs/scheduling/api_test.go | 5 +++-- .../mcs/scheduling/server_test.go | 4 ++-- tests/pdctl/scheduler/scheduler_test.go | 20 ++++++++++++++++++- tests/server/api/scheduler_test.go | 4 ++++ tests/server/cluster/cluster_test.go | 8 +++++--- 7 files changed, 45 insertions(+), 15 deletions(-) diff --git a/pkg/schedule/config/config.go b/pkg/schedule/config/config.go index 27b8917d1bf..90a37c93d91 100644 --- a/pkg/schedule/config/config.go +++ b/pkg/schedule/config/config.go @@ -556,6 +556,7 @@ var DefaultSchedulers = SchedulerConfigs{ {Type: "balance-witness"}, {Type: "hot-region"}, {Type: "transfer-witness-leader"}, + {Type: "evict-slow-store"}, } // IsDefaultScheduler checks whether the scheduler is enable by default. diff --git a/server/cluster/cluster_test.go b/server/cluster/cluster_test.go index 7094fd6b673..8c889923ea7 100644 --- a/server/cluster/cluster_test.go +++ b/server/cluster/cluster_test.go @@ -3017,6 +3017,7 @@ func TestAddScheduler(t *testing.T) { re.NoError(controller.RemoveScheduler(schedulers.HotRegionName)) re.NoError(controller.RemoveScheduler(schedulers.BalanceWitnessName)) re.NoError(controller.RemoveScheduler(schedulers.TransferWitnessLeaderName)) + re.NoError(controller.RemoveScheduler(schedulers.EvictSlowStoreName)) re.Empty(controller.GetSchedulerNames()) stream := mockhbstream.NewHeartbeatStream() @@ -3107,13 +3108,15 @@ func TestPersistScheduler(t *testing.T) { re.NoError(err) re.Len(sches, defaultCount+2) - // remove 5 schedulers + // remove all default schedulers re.NoError(controller.RemoveScheduler(schedulers.BalanceLeaderName)) re.NoError(controller.RemoveScheduler(schedulers.BalanceRegionName)) re.NoError(controller.RemoveScheduler(schedulers.HotRegionName)) re.NoError(controller.RemoveScheduler(schedulers.BalanceWitnessName)) re.NoError(controller.RemoveScheduler(schedulers.TransferWitnessLeaderName)) - re.Len(controller.GetSchedulerNames(), defaultCount-3) + re.NoError(controller.RemoveScheduler(schedulers.EvictSlowStoreName)) + // only remains 2 items with independent config. + re.Len(controller.GetSchedulerNames(), 2) re.NoError(co.GetCluster().GetSchedulerConfig().Persist(storage)) co.Stop() co.GetSchedulersController().Wait() @@ -3137,7 +3140,7 @@ func TestPersistScheduler(t *testing.T) { re.NoError(err) re.Len(sches, 3) - // option have 6 items because the default scheduler do not remove. + // option have 9 items because the default scheduler do not remove. re.Len(newOpt.GetSchedulers(), defaultCount+3) re.NoError(newOpt.Persist(storage)) tc.RaftCluster.SetScheduleConfig(newOpt.GetScheduleConfig()) @@ -3164,9 +3167,9 @@ func TestPersistScheduler(t *testing.T) { brs, err := schedulers.CreateScheduler(schedulers.BalanceRegionType, oc, storage, schedulers.ConfigSliceDecoder(schedulers.BalanceRegionType, []string{"", ""})) re.NoError(err) re.NoError(controller.AddScheduler(brs)) - re.Len(controller.GetSchedulerNames(), defaultCount) + re.Len(controller.GetSchedulerNames(), 5) - // the scheduler option should contain 6 items + // the scheduler option should contain 9 items // the `hot scheduler` are disabled re.Len(co.GetCluster().GetSchedulerConfig().(*config.PersistOptions).GetSchedulers(), defaultCount+3) re.NoError(controller.RemoveScheduler(schedulers.GrantLeaderName)) @@ -3185,9 +3188,9 @@ func TestPersistScheduler(t *testing.T) { co.Run() controller = co.GetSchedulersController() - re.Len(controller.GetSchedulerNames(), defaultCount-1) + re.Len(controller.GetSchedulerNames(), 4) re.NoError(controller.RemoveScheduler(schedulers.EvictLeaderName)) - re.Len(controller.GetSchedulerNames(), defaultCount-2) + re.Len(controller.GetSchedulerNames(), 3) } func TestRemoveScheduler(t *testing.T) { @@ -3225,6 +3228,7 @@ func TestRemoveScheduler(t *testing.T) { re.NoError(controller.RemoveScheduler(schedulers.GrantLeaderName)) re.NoError(controller.RemoveScheduler(schedulers.BalanceWitnessName)) re.NoError(controller.RemoveScheduler(schedulers.TransferWitnessLeaderName)) + re.NoError(controller.RemoveScheduler(schedulers.EvictSlowStoreName)) // all removed sches, _, err = storage.LoadAllSchedulerConfigs() re.NoError(err) diff --git a/tests/integrations/mcs/scheduling/api_test.go b/tests/integrations/mcs/scheduling/api_test.go index 4c71f8f14a3..4ad6680a7cd 100644 --- a/tests/integrations/mcs/scheduling/api_test.go +++ b/tests/integrations/mcs/scheduling/api_test.go @@ -382,13 +382,14 @@ func (suite *apiTestSuite) checkConfig(cluster *tests.TestCluster) { suite.Equal(cfg.DataDir, s.GetConfig().DataDir) testutil.Eventually(re, func() bool { // wait for all schedulers to be loaded in scheduling server. - return len(cfg.Schedule.SchedulersPayload) == 5 + return len(cfg.Schedule.SchedulersPayload) == 6 }) suite.Contains(cfg.Schedule.SchedulersPayload, "balance-leader-scheduler") suite.Contains(cfg.Schedule.SchedulersPayload, "balance-region-scheduler") suite.Contains(cfg.Schedule.SchedulersPayload, "balance-hot-region-scheduler") suite.Contains(cfg.Schedule.SchedulersPayload, "balance-witness-scheduler") suite.Contains(cfg.Schedule.SchedulersPayload, "transfer-witness-leader-scheduler") + suite.Contains(cfg.Schedule.SchedulersPayload, "evict-slow-store-scheduler") } func (suite *apiTestSuite) TestConfigForward() { @@ -412,7 +413,7 @@ func (suite *apiTestSuite) checkConfigForward(cluster *tests.TestCluster) { re.Equal(cfg["replication"].(map[string]interface{})["max-replicas"], float64(opts.GetReplicationConfig().MaxReplicas)) schedulers := cfg["schedule"].(map[string]interface{})["schedulers-payload"].(map[string]interface{}) - return len(schedulers) == 5 + return len(schedulers) == 6 }) // Test to change config in api server diff --git a/tests/integrations/mcs/scheduling/server_test.go b/tests/integrations/mcs/scheduling/server_test.go index c65352114df..0c0442300d9 100644 --- a/tests/integrations/mcs/scheduling/server_test.go +++ b/tests/integrations/mcs/scheduling/server_test.go @@ -136,7 +136,7 @@ func (suite *serverTestSuite) TestPrimaryChange() { testutil.Eventually(re, func() bool { watchedAddr, ok := suite.pdLeader.GetServicePrimaryAddr(suite.ctx, mcs.SchedulingServiceName) return ok && oldPrimaryAddr == watchedAddr && - len(primary.GetCluster().GetCoordinator().GetSchedulersController().GetSchedulerNames()) == 5 + len(primary.GetCluster().GetCoordinator().GetSchedulersController().GetSchedulerNames()) == 6 }) // change primary primary.Close() @@ -147,7 +147,7 @@ func (suite *serverTestSuite) TestPrimaryChange() { testutil.Eventually(re, func() bool { watchedAddr, ok := suite.pdLeader.GetServicePrimaryAddr(suite.ctx, mcs.SchedulingServiceName) return ok && newPrimaryAddr == watchedAddr && - len(primary.GetCluster().GetCoordinator().GetSchedulersController().GetSchedulerNames()) == 5 + len(primary.GetCluster().GetCoordinator().GetSchedulersController().GetSchedulerNames()) == 6 }) } diff --git a/tests/pdctl/scheduler/scheduler_test.go b/tests/pdctl/scheduler/scheduler_test.go index fb7c239b431..140ee7a7c44 100644 --- a/tests/pdctl/scheduler/scheduler_test.go +++ b/tests/pdctl/scheduler/scheduler_test.go @@ -56,6 +56,7 @@ func (suite *schedulerTestSuite) SetupSuite() { "balance-hot-region-scheduler", "balance-witness-scheduler", "transfer-witness-leader-scheduler", + "evict-slow-store-scheduler", } } @@ -173,6 +174,7 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) { "balance-hot-region-scheduler": true, "transfer-witness-leader-scheduler": true, "balance-witness-scheduler": true, + "evict-slow-store-scheduler": true, } checkSchedulerCommand(nil, expected) @@ -183,6 +185,7 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) { "balance-hot-region-scheduler": true, "transfer-witness-leader-scheduler": true, "balance-witness-scheduler": true, + "evict-slow-store-scheduler": true, } checkSchedulerCommand(args, expected) @@ -228,6 +231,7 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) { schedulers[idx]: true, "transfer-witness-leader-scheduler": true, "balance-witness-scheduler": true, + "evict-slow-store-scheduler": true, } checkSchedulerCommand(args, expected) @@ -245,6 +249,7 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) { schedulers[idx]: true, "transfer-witness-leader-scheduler": true, "balance-witness-scheduler": true, + "evict-slow-store-scheduler": true, } // check update success @@ -260,6 +265,7 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) { "balance-hot-region-scheduler": true, "transfer-witness-leader-scheduler": true, "balance-witness-scheduler": true, + "evict-slow-store-scheduler": true, } checkSchedulerCommand(args, expected) checkStorePause([]uint64{}, schedulers[idx]) @@ -272,6 +278,7 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) { schedulers[idx]: true, "transfer-witness-leader-scheduler": true, "balance-witness-scheduler": true, + "evict-slow-store-scheduler": true, } checkSchedulerCommand(args, expected) checkStorePause([]uint64{2}, schedulers[idx]) @@ -284,6 +291,7 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) { schedulers[idx]: true, "transfer-witness-leader-scheduler": true, "balance-witness-scheduler": true, + "evict-slow-store-scheduler": true, } checkSchedulerCommand(args, expected) @@ -300,6 +308,7 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) { schedulers[idx]: true, "transfer-witness-leader-scheduler": true, "balance-witness-scheduler": true, + "evict-slow-store-scheduler": true, } checkSchedulerCommand(args, expected) @@ -315,6 +324,7 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) { "balance-hot-region-scheduler": true, "transfer-witness-leader-scheduler": true, "balance-witness-scheduler": true, + "evict-slow-store-scheduler": true, } checkSchedulerCommand(args, expected) checkStorePause([]uint64{}, schedulers[idx]) @@ -327,6 +337,7 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) { "shuffle-region-scheduler": true, "transfer-witness-leader-scheduler": true, "balance-witness-scheduler": true, + "evict-slow-store-scheduler": true, }) var roles []string mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "shuffle-region-scheduler", "show-roles"}, &roles) @@ -348,6 +359,7 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) { "grant-hot-region-scheduler": true, "transfer-witness-leader-scheduler": true, "balance-witness-scheduler": true, + "evict-slow-store-scheduler": true, }) var conf3 map[string]interface{} expected3 := map[string]interface{}{ @@ -527,7 +539,11 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) { evictSlownessSchedulers := []string{"evict-slow-store-scheduler", "evict-slow-trend-scheduler"} for _, schedulerName := range evictSlownessSchedulers { echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "add", schedulerName}, nil) - re.Contains(echo, "Success!") + if strings.Contains(echo, "Success!") { + re.Contains(echo, "Success!") + } else { + re.Contains(echo, "scheduler existed") + } testutil.Eventually(re, func() bool { echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "show"}, nil) return strings.Contains(echo, schedulerName) @@ -546,6 +562,8 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) { return !strings.Contains(echo, schedulerName) }) } + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "add", "evict-slow-store-scheduler"}, nil) + re.Contains(echo, "Success!") // test shuffle hot region scheduler echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "add", "shuffle-hot-region-scheduler"}, nil) diff --git a/tests/server/api/scheduler_test.go b/tests/server/api/scheduler_test.go index b3810da154a..d0472795f93 100644 --- a/tests/server/api/scheduler_test.go +++ b/tests/server/api/scheduler_test.go @@ -477,6 +477,10 @@ func (suite *scheduleTestSuite) checkAPI(cluster *tests.TestCluster) { suite.NoError(err) }, }, + { + name: "evict-slow-store-scheduler", + createdName: "evict-slow-store-scheduler", + }, } for _, testCase := range testCases { input := make(map[string]interface{}) diff --git a/tests/server/cluster/cluster_test.go b/tests/server/cluster/cluster_test.go index 18a82bcf0fe..0b0779d9434 100644 --- a/tests/server/cluster/cluster_test.go +++ b/tests/server/cluster/cluster_test.go @@ -1310,6 +1310,7 @@ func TestTransferLeaderForScheduler(t *testing.T) { time.Sleep(time.Second) re.True(leaderServer.GetRaftCluster().IsPrepared()) + schedsNum := len(rc.GetCoordinator().GetSchedulersController().GetSchedulerNames()) // Add evict leader scheduler api.MustAddScheduler(re, leaderServer.GetAddr(), schedulers.EvictLeaderName, map[string]interface{}{ "store_id": 1, @@ -1318,8 +1319,9 @@ func TestTransferLeaderForScheduler(t *testing.T) { "store_id": 2, }) // Check scheduler updated. + schedsNum += 1 schedulersController := rc.GetCoordinator().GetSchedulersController() - re.Len(schedulersController.GetSchedulerNames(), 6) + re.Len(schedulersController.GetSchedulerNames(), schedsNum) checkEvictLeaderSchedulerExist(re, schedulersController, true) checkEvictLeaderStoreIDs(re, schedulersController, []uint64{1, 2}) @@ -1339,7 +1341,7 @@ func TestTransferLeaderForScheduler(t *testing.T) { re.True(leaderServer.GetRaftCluster().IsPrepared()) // Check scheduler updated. schedulersController = rc1.GetCoordinator().GetSchedulersController() - re.Len(schedulersController.GetSchedulerNames(), 6) + re.Len(schedulersController.GetSchedulerNames(), schedsNum) checkEvictLeaderSchedulerExist(re, schedulersController, true) checkEvictLeaderStoreIDs(re, schedulersController, []uint64{1, 2}) @@ -1358,7 +1360,7 @@ func TestTransferLeaderForScheduler(t *testing.T) { re.True(leaderServer.GetRaftCluster().IsPrepared()) // Check scheduler updated schedulersController = rc.GetCoordinator().GetSchedulersController() - re.Len(schedulersController.GetSchedulerNames(), 6) + re.Len(schedulersController.GetSchedulerNames(), schedsNum) checkEvictLeaderSchedulerExist(re, schedulersController, true) checkEvictLeaderStoreIDs(re, schedulersController, []uint64{1, 2}) From 0a332a95f603309f97176008779b39627acf99d3 Mon Sep 17 00:00:00 2001 From: JmPotato Date: Tue, 19 Dec 2023 08:27:22 +0800 Subject: [PATCH 102/137] Makefile: add test-real-cluster command to the root Makefile (#7567) ref tikv/pd#7298 Add `test-real-cluster` command to the root Makefile. Signed-off-by: JmPotato --- Makefile | 9 ++++++++- pd.code-workspace | 4 ++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 2a506eb576f..133f99cfac8 100644 --- a/Makefile +++ b/Makefile @@ -265,7 +265,13 @@ test-tso-consistency: install-tools CGO_ENABLED=1 go test -race -tags without_dashboard,tso_consistency_test,deadlock $(TSO_INTEGRATION_TEST_PKGS) || { $(FAILPOINT_DISABLE); exit 1; } @$(FAILPOINT_DISABLE) -.PHONY: test basic-test test-with-cover test-tso-function test-tso-consistency +REAL_CLUSTER_TEST_PATH := $(ROOT_PATH)/tests/integrations/realtiup + +test-real-cluster: + # testing with the real cluster... + cd $(REAL_CLUSTER_TEST_PATH) && $(MAKE) check + +.PHONY: test basic-test test-with-cover test-tso-function test-tso-consistency test-real-cluster #### Daily CI coverage analyze #### @@ -297,6 +303,7 @@ clean-test: rm -rf /tmp/test_pd* rm -rf /tmp/pd-tests* rm -rf /tmp/test_etcd* + rm -f $(REAL_CLUSTER_TEST_PATH)/playground.log go clean -testcache clean-build: diff --git a/pd.code-workspace b/pd.code-workspace index d6110b56a09..54a8ea324aa 100644 --- a/pd.code-workspace +++ b/pd.code-workspace @@ -20,6 +20,10 @@ "name": "tso-tests", "path": "tests/integrations/tso" }, + { + "name": "real-cluster-tests", + "path": "tests/integrations/realtiup" + }, { "name": "pd-tso-bench", "path": "tools/pd-tso-bench" From 25f48f0bdd27b0d454a41ff0132ed643c6dd51f9 Mon Sep 17 00:00:00 2001 From: JmPotato Date: Tue, 19 Dec 2023 11:19:51 +0800 Subject: [PATCH 103/137] client/http: require source mark when initializing the http client (#7565) ref tikv/pd#7300 When creating a client, it is required to pass in the `source` parameter to further distinguish the source of logs and requests. Signed-off-by: JmPotato Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- client/http/client.go | 32 +++++++++++-------- client/http/client_test.go | 11 ++++--- tests/integrations/client/http_client_test.go | 2 +- tests/integrations/mcs/members/member_test.go | 2 +- tests/integrations/realtiup/util.go | 2 +- 5 files changed, 28 insertions(+), 21 deletions(-) diff --git a/client/http/client.go b/client/http/client.go index 21a3727e00f..bf8e9af9bbe 100644 --- a/client/http/client.go +++ b/client/http/client.go @@ -60,6 +60,9 @@ type clientInner struct { pdAddrs []string leaderAddrIdx int + // source is used to mark the source of the client creation, + // it will also be used in the caller ID of the inner client. + source string tlsConf *tls.Config cli *http.Client @@ -67,9 +70,9 @@ type clientInner struct { executionDuration *prometheus.HistogramVec } -func newClientInner() *clientInner { +func newClientInner(source string) *clientInner { ctx, cancel := context.WithCancel(context.Background()) - return &clientInner{ctx: ctx, cancel: cancel, leaderAddrIdx: -1} + return &clientInner{ctx: ctx, cancel: cancel, leaderAddrIdx: -1, source: source} } func (ci *clientInner) init() { @@ -155,7 +158,7 @@ func (ci *clientInner) requestWithRetry( return nil } log.Debug("[pd] request leader addr failed", - zap.Int("leader-idx", leaderAddrIdx), zap.String("addr", addr), zap.Error(err)) + zap.String("source", ci.source), zap.Int("leader-idx", leaderAddrIdx), zap.String("addr", addr), zap.Error(err)) } // Try to send the request to the other PD followers. for idx := 0; idx < len(pdAddrs) && idx != leaderAddrIdx; idx++ { @@ -165,7 +168,7 @@ func (ci *clientInner) requestWithRetry( break } log.Debug("[pd] request follower addr failed", - zap.Int("idx", idx), zap.String("addr", addr), zap.Error(err)) + zap.String("source", ci.source), zap.Int("idx", idx), zap.String("addr", addr), zap.Error(err)) } return err } @@ -176,6 +179,7 @@ func (ci *clientInner) doRequest( headerOpts ...HeaderOption, ) error { var ( + source = ci.source callerID = reqInfo.callerID name = reqInfo.name url = reqInfo.getURL(addr) @@ -185,6 +189,7 @@ func (ci *clientInner) doRequest( respHandler = reqInfo.respHandler ) logFields := []zap.Field{ + zap.String("source", source), zap.String("name", name), zap.String("url", url), zap.String("method", method), @@ -250,13 +255,13 @@ func (ci *clientInner) doRequest( func (ci *clientInner) membersInfoUpdater(ctx context.Context) { ci.updateMembersInfo(ctx) - log.Info("[pd] http client member info updater started") + log.Info("[pd] http client member info updater started", zap.String("source", ci.source)) ticker := time.NewTicker(defaultMembersInfoUpdateInterval) defer ticker.Stop() for { select { case <-ctx.Done(): - log.Info("[pd] http client member info updater stopped") + log.Info("[pd] http client member info updater stopped", zap.String("source", ci.source)) return case <-ticker.C: ci.updateMembersInfo(ctx) @@ -267,17 +272,17 @@ func (ci *clientInner) membersInfoUpdater(ctx context.Context) { func (ci *clientInner) updateMembersInfo(ctx context.Context) { var membersInfo MembersInfo err := ci.requestWithRetry(ctx, newRequestInfo(). - WithCallerID(defaultInnerCallerID). + WithCallerID(fmt.Sprintf("%s-%s", ci.source, defaultInnerCallerID)). WithName(getMembersName). WithURI(membersPrefix). WithMethod(http.MethodGet). WithResp(&membersInfo)) if err != nil { - log.Error("[pd] http client get members info failed", zap.Error(err)) + log.Error("[pd] http client get members info failed", zap.String("source", ci.source), zap.Error(err)) return } if len(membersInfo.Members) == 0 { - log.Error("[pd] http client get empty members info") + log.Error("[pd] http client get empty members info", zap.String("source", ci.source)) return } var ( @@ -292,7 +297,7 @@ func (ci *clientInner) updateMembersInfo(ctx context.Context) { } // Prevent setting empty addresses. if len(newPDAddrs) == 0 { - log.Error("[pd] http client get empty member addresses") + log.Error("[pd] http client get empty member addresses", zap.String("source", ci.source)) return } oldPDAddrs, oldLeaderAddrIdx := ci.getPDAddrs() @@ -307,7 +312,7 @@ func (ci *clientInner) updateMembersInfo(ctx context.Context) { } oldMemberNum, newMemberNum := len(oldPDAddrs), len(newPDAddrs) if oldPDLeaderAddr != newPDLeaderAddr || oldMemberNum != newMemberNum { - log.Info("[pd] http client members info changed", + log.Info("[pd] http client members info changed", zap.String("source", ci.source), zap.Int("old-member-num", oldMemberNum), zap.Int("new-member-num", newMemberNum), zap.Strings("old-addrs", oldPDAddrs), zap.Strings("new-addrs", newPDAddrs), zap.Int("old-leader-addr-idx", oldLeaderAddrIdx), zap.Int("new-leader-addr-idx", newLeaderAddrIdx), @@ -353,10 +358,11 @@ func WithMetrics( // NewClient creates a PD HTTP client with the given PD addresses and TLS config. func NewClient( + source string, pdAddrs []string, opts ...ClientOption, ) Client { - c := &client{inner: newClientInner(), callerID: defaultCallerID} + c := &client{inner: newClientInner(source), callerID: defaultCallerID} // Apply the options first. for _, opt := range opts { opt(c) @@ -369,7 +375,7 @@ func NewClient( // Close gracefully closes the HTTP client. func (c *client) Close() { c.inner.close() - log.Info("[pd] http client closed") + log.Info("[pd] http client closed", zap.String("source", c.inner.source)) } // WithCallerID sets and returns a new client with the given caller ID. diff --git a/client/http/client_test.go b/client/http/client_test.go index 7c7da80827c..af16ac649b5 100644 --- a/client/http/client_test.go +++ b/client/http/client_test.go @@ -18,6 +18,7 @@ import ( "context" "crypto/tls" "net/http" + "strings" "testing" "github.com/stretchr/testify/require" @@ -26,12 +27,12 @@ import ( func TestPDAddrNormalization(t *testing.T) { re := require.New(t) - c := NewClient([]string{"127.0.0.1"}) + c := NewClient("test-http-pd-addr", []string{"127.0.0.1"}) pdAddrs, leaderAddrIdx := c.(*client).inner.getPDAddrs() re.Equal(1, len(pdAddrs)) re.Equal(-1, leaderAddrIdx) re.Contains(pdAddrs[0], httpScheme) - c = NewClient([]string{"127.0.0.1"}, WithTLSConfig(&tls.Config{})) + c = NewClient("test-https-pd-addr", []string{"127.0.0.1"}, WithTLSConfig(&tls.Config{})) pdAddrs, leaderAddrIdx = c.(*client).inner.getPDAddrs() re.Equal(1, len(pdAddrs)) re.Equal(-1, leaderAddrIdx) @@ -68,7 +69,7 @@ func TestPDAllowFollowerHandleHeader(t *testing.T) { } return nil }) - c := NewClient([]string{"http://127.0.0.1"}, WithHTTPClient(httpClient)) + c := NewClient("test-header", []string{"http://127.0.0.1"}, WithHTTPClient(httpClient)) c.GetRegions(context.Background()) c.GetHistoryHotRegions(context.Background(), &HistoryHotRegionsRequest{}) c.Close() @@ -80,13 +81,13 @@ func TestCallerID(t *testing.T) { httpClient := newHTTPClientWithRequestChecker(func(req *http.Request) error { val := req.Header.Get(xCallerIDKey) // Exclude the request sent by the inner client. - if val != defaultInnerCallerID && val != expectedVal.Load() { + if !strings.Contains(val, defaultInnerCallerID) && val != expectedVal.Load() { re.Failf("Caller ID header check failed", "should be %s, but got %s", expectedVal, val) } return nil }) - c := NewClient([]string{"http://127.0.0.1"}, WithHTTPClient(httpClient)) + c := NewClient("test-caller-id", []string{"http://127.0.0.1"}, WithHTTPClient(httpClient)) c.GetRegions(context.Background()) expectedVal.Store("test") c.WithCallerID(expectedVal.Load()).GetRegions(context.Background()) diff --git a/tests/integrations/client/http_client_test.go b/tests/integrations/client/http_client_test.go index 1a20ae2e784..12064c989a1 100644 --- a/tests/integrations/client/http_client_test.go +++ b/tests/integrations/client/http_client_test.go @@ -72,7 +72,7 @@ func (suite *httpClientTestSuite) SetupSuite() { for _, s := range testServers { endpoints = append(endpoints, s.GetConfig().AdvertiseClientUrls) } - suite.client = pd.NewClient(endpoints) + suite.client = pd.NewClient("pd-http-client-it", endpoints) } func (suite *httpClientTestSuite) TearDownSuite() { diff --git a/tests/integrations/mcs/members/member_test.go b/tests/integrations/mcs/members/member_test.go index d1ccb86a1c7..048bcf72154 100644 --- a/tests/integrations/mcs/members/member_test.go +++ b/tests/integrations/mcs/members/member_test.go @@ -52,7 +52,7 @@ func (suite *memberTestSuite) SetupTest() { suite.server = cluster.GetLeaderServer() suite.NoError(suite.server.BootstrapCluster()) suite.backendEndpoints = suite.server.GetAddr() - suite.dialClient = pdClient.NewClient([]string{suite.server.GetAddr()}) + suite.dialClient = pdClient.NewClient("mcs-member-test", []string{suite.server.GetAddr()}) // TSO nodes := make(map[string]bs.Server) diff --git a/tests/integrations/realtiup/util.go b/tests/integrations/realtiup/util.go index 66d6127b5c4..8f0c71038d6 100644 --- a/tests/integrations/realtiup/util.go +++ b/tests/integrations/realtiup/util.go @@ -24,7 +24,7 @@ const physicalShiftBits = 18 var ( pdAddrs = []string{"127.0.0.1:2379"} - pdHTTPCli = http.NewClient(pdAddrs) + pdHTTPCli = http.NewClient("pd-realtiup-test", pdAddrs) ) // GetTimeFromTS extracts time.Time from a timestamp. From 83da4c26b10e7f4e371ebc1a934fddd5981c10b0 Mon Sep 17 00:00:00 2001 From: Connor Date: Tue, 19 Dec 2023 16:10:52 +0800 Subject: [PATCH 104/137] dashboard: Pass TLS info to dashboard to fix TiKV heap profiling (#7563) close tikv/pd#7561 Pass TLS info to dashboard to fix TiKV heap profiling Signed-off-by: Connor1996 --- go.mod | 2 +- go.sum | 4 ++-- pkg/dashboard/adapter/config.go | 3 +++ pkg/utils/grpcutil/grpcutil.go | 31 +++++++++++++++++++++---------- tests/integrations/client/go.mod | 2 +- tests/integrations/client/go.sum | 4 ++-- tests/integrations/mcs/go.mod | 2 +- tests/integrations/mcs/go.sum | 4 ++-- tests/integrations/tso/go.mod | 2 +- tests/integrations/tso/go.sum | 4 ++-- 10 files changed, 36 insertions(+), 22 deletions(-) diff --git a/go.mod b/go.mod index d5cbc41f654..e60364d9c57 100644 --- a/go.mod +++ b/go.mod @@ -36,7 +36,7 @@ require ( github.com/pingcap/kvproto v0.0.0-20231018065736-c0689aded40c github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 - github.com/pingcap/tidb-dashboard v0.0.0-20231127105651-ce4097837c5e + github.com/pingcap/tidb-dashboard v0.0.0-20231218095437-aa621ed4de2c github.com/prometheus/client_golang v1.11.1 github.com/prometheus/common v0.26.0 github.com/sasha-s/go-deadlock v0.2.0 diff --git a/go.sum b/go.sum index bf35be7eb8c..b1136a6f2a2 100644 --- a/go.sum +++ b/go.sum @@ -466,8 +466,8 @@ github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8 github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 h1:QV6jqlfOkh8hqvEAgwBZa+4bSgO0EeKC7s5c6Luam2I= github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21/go.mod h1:QYnjfA95ZaMefyl1NO8oPtKeb8pYUdnDVhQgf+qdpjM= -github.com/pingcap/tidb-dashboard v0.0.0-20231127105651-ce4097837c5e h1:SJUSDejvKtj9vSh5ptRHh4iMrvPV3oKO8yp6/SYE8vc= -github.com/pingcap/tidb-dashboard v0.0.0-20231127105651-ce4097837c5e/go.mod h1:ucZBRz52icb23T/5Z4CsuUHmarYiin7p2MeiVBe+o8c= +github.com/pingcap/tidb-dashboard v0.0.0-20231218095437-aa621ed4de2c h1:iEZwsxxOxXaH0zEfzVAn6fjveOlPh3v3DsYlhWJAVi0= +github.com/pingcap/tidb-dashboard v0.0.0-20231218095437-aa621ed4de2c/go.mod h1:ucZBRz52icb23T/5Z4CsuUHmarYiin7p2MeiVBe+o8c= github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e h1:FBaTXU8C3xgt/drM58VHxojHo/QoG1oPsgWTGvaSpO4= github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= diff --git a/pkg/dashboard/adapter/config.go b/pkg/dashboard/adapter/config.go index 63c900acf77..a1661b84f2b 100644 --- a/pkg/dashboard/adapter/config.go +++ b/pkg/dashboard/adapter/config.go @@ -38,6 +38,9 @@ func GenDashboardConfig(srv *server.Server) (*config.Config, error) { if dashboardCfg.ClusterTLSConfig, err = cfg.Security.ToTLSConfig(); err != nil { return nil, err } + if dashboardCfg.ClusterTLSInfo, err = cfg.Security.ToTLSInfo(); err != nil { + return nil, err + } if dashboardCfg.TiDBTLSConfig, err = cfg.Dashboard.ToTiDBTLSConfig(); err != nil { return nil, err } diff --git a/pkg/utils/grpcutil/grpcutil.go b/pkg/utils/grpcutil/grpcutil.go index a001ec4bd03..0030551d0fc 100644 --- a/pkg/utils/grpcutil/grpcutil.go +++ b/pkg/utils/grpcutil/grpcutil.go @@ -56,6 +56,24 @@ type TLSConfig struct { SSLKEYBytes []byte } +// ToTLSInfo converts TLSConfig to transport.TLSInfo. +func (s TLSConfig) ToTLSInfo() (*transport.TLSInfo, error) { + if len(s.CertPath) == 0 && len(s.KeyPath) == 0 { + return nil, nil + } + allowedCN, err := s.GetOneAllowedCN() + if err != nil { + return nil, err + } + + return &transport.TLSInfo{ + CertFile: s.CertPath, + KeyFile: s.KeyPath, + TrustedCAFile: s.CAPath, + AllowedCN: allowedCN, + }, nil +} + // ToTLSConfig generates tls config. func (s TLSConfig) ToTLSConfig() (*tls.Config, error) { if len(s.SSLCABytes) != 0 || len(s.SSLCertBytes) != 0 || len(s.SSLKEYBytes) != 0 { @@ -77,19 +95,12 @@ func (s TLSConfig) ToTLSConfig() (*tls.Config, error) { }, nil } - if len(s.CertPath) == 0 && len(s.KeyPath) == 0 { + tlsInfo, err := s.ToTLSInfo() + if tlsInfo == nil { return nil, nil } - allowedCN, err := s.GetOneAllowedCN() if err != nil { - return nil, err - } - - tlsInfo := transport.TLSInfo{ - CertFile: s.CertPath, - KeyFile: s.KeyPath, - TrustedCAFile: s.CAPath, - AllowedCN: allowedCN, + return nil, errs.ErrEtcdTLSConfig.Wrap(err).GenWithStackByCause() } tlsConfig, err := tlsInfo.ClientConfig() diff --git a/tests/integrations/client/go.mod b/tests/integrations/client/go.mod index da130278ae0..5c73abfb8ed 100644 --- a/tests/integrations/client/go.mod +++ b/tests/integrations/client/go.mod @@ -123,7 +123,7 @@ require ( github.com/pingcap/errcode v0.3.0 // indirect github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c // indirect github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 // indirect - github.com/pingcap/tidb-dashboard v0.0.0-20231127105651-ce4097837c5e // indirect + github.com/pingcap/tidb-dashboard v0.0.0-20231218095437-aa621ed4de2c // indirect github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect diff --git a/tests/integrations/client/go.sum b/tests/integrations/client/go.sum index e13da5d8375..f70f08f366e 100644 --- a/tests/integrations/client/go.sum +++ b/tests/integrations/client/go.sum @@ -430,8 +430,8 @@ github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8 github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 h1:QV6jqlfOkh8hqvEAgwBZa+4bSgO0EeKC7s5c6Luam2I= github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21/go.mod h1:QYnjfA95ZaMefyl1NO8oPtKeb8pYUdnDVhQgf+qdpjM= -github.com/pingcap/tidb-dashboard v0.0.0-20231127105651-ce4097837c5e h1:SJUSDejvKtj9vSh5ptRHh4iMrvPV3oKO8yp6/SYE8vc= -github.com/pingcap/tidb-dashboard v0.0.0-20231127105651-ce4097837c5e/go.mod h1:ucZBRz52icb23T/5Z4CsuUHmarYiin7p2MeiVBe+o8c= +github.com/pingcap/tidb-dashboard v0.0.0-20231218095437-aa621ed4de2c h1:iEZwsxxOxXaH0zEfzVAn6fjveOlPh3v3DsYlhWJAVi0= +github.com/pingcap/tidb-dashboard v0.0.0-20231218095437-aa621ed4de2c/go.mod h1:ucZBRz52icb23T/5Z4CsuUHmarYiin7p2MeiVBe+o8c= github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e h1:FBaTXU8C3xgt/drM58VHxojHo/QoG1oPsgWTGvaSpO4= github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= diff --git a/tests/integrations/mcs/go.mod b/tests/integrations/mcs/go.mod index 1823a224fa1..2e41e87b746 100644 --- a/tests/integrations/mcs/go.mod +++ b/tests/integrations/mcs/go.mod @@ -123,7 +123,7 @@ require ( github.com/pingcap/errcode v0.3.0 // indirect github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c // indirect github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 // indirect - github.com/pingcap/tidb-dashboard v0.0.0-20231127105651-ce4097837c5e // indirect + github.com/pingcap/tidb-dashboard v0.0.0-20231218095437-aa621ed4de2c // indirect github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect diff --git a/tests/integrations/mcs/go.sum b/tests/integrations/mcs/go.sum index dfead54afe1..65e8cf72aab 100644 --- a/tests/integrations/mcs/go.sum +++ b/tests/integrations/mcs/go.sum @@ -434,8 +434,8 @@ github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8 github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 h1:QV6jqlfOkh8hqvEAgwBZa+4bSgO0EeKC7s5c6Luam2I= github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21/go.mod h1:QYnjfA95ZaMefyl1NO8oPtKeb8pYUdnDVhQgf+qdpjM= -github.com/pingcap/tidb-dashboard v0.0.0-20231127105651-ce4097837c5e h1:SJUSDejvKtj9vSh5ptRHh4iMrvPV3oKO8yp6/SYE8vc= -github.com/pingcap/tidb-dashboard v0.0.0-20231127105651-ce4097837c5e/go.mod h1:ucZBRz52icb23T/5Z4CsuUHmarYiin7p2MeiVBe+o8c= +github.com/pingcap/tidb-dashboard v0.0.0-20231218095437-aa621ed4de2c h1:iEZwsxxOxXaH0zEfzVAn6fjveOlPh3v3DsYlhWJAVi0= +github.com/pingcap/tidb-dashboard v0.0.0-20231218095437-aa621ed4de2c/go.mod h1:ucZBRz52icb23T/5Z4CsuUHmarYiin7p2MeiVBe+o8c= github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e h1:FBaTXU8C3xgt/drM58VHxojHo/QoG1oPsgWTGvaSpO4= github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= diff --git a/tests/integrations/tso/go.mod b/tests/integrations/tso/go.mod index 0d734716ed5..af38fcf4241 100644 --- a/tests/integrations/tso/go.mod +++ b/tests/integrations/tso/go.mod @@ -121,7 +121,7 @@ require ( github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c // indirect github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 // indirect github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 // indirect - github.com/pingcap/tidb-dashboard v0.0.0-20231127105651-ce4097837c5e // indirect + github.com/pingcap/tidb-dashboard v0.0.0-20231218095437-aa621ed4de2c // indirect github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect diff --git a/tests/integrations/tso/go.sum b/tests/integrations/tso/go.sum index 94fbde2ad57..c078eab2f6d 100644 --- a/tests/integrations/tso/go.sum +++ b/tests/integrations/tso/go.sum @@ -428,8 +428,8 @@ github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8 github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 h1:QV6jqlfOkh8hqvEAgwBZa+4bSgO0EeKC7s5c6Luam2I= github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21/go.mod h1:QYnjfA95ZaMefyl1NO8oPtKeb8pYUdnDVhQgf+qdpjM= -github.com/pingcap/tidb-dashboard v0.0.0-20231127105651-ce4097837c5e h1:SJUSDejvKtj9vSh5ptRHh4iMrvPV3oKO8yp6/SYE8vc= -github.com/pingcap/tidb-dashboard v0.0.0-20231127105651-ce4097837c5e/go.mod h1:ucZBRz52icb23T/5Z4CsuUHmarYiin7p2MeiVBe+o8c= +github.com/pingcap/tidb-dashboard v0.0.0-20231218095437-aa621ed4de2c h1:iEZwsxxOxXaH0zEfzVAn6fjveOlPh3v3DsYlhWJAVi0= +github.com/pingcap/tidb-dashboard v0.0.0-20231218095437-aa621ed4de2c/go.mod h1:ucZBRz52icb23T/5Z4CsuUHmarYiin7p2MeiVBe+o8c= github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e h1:FBaTXU8C3xgt/drM58VHxojHo/QoG1oPsgWTGvaSpO4= github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= From 59c9d0475fca695f441f525a1969d404715f752c Mon Sep 17 00:00:00 2001 From: JmPotato Date: Wed, 20 Dec 2023 10:42:22 +0800 Subject: [PATCH 105/137] tests: fix some errors detected by testifylint (#7580) ref tikv/pd#4813 Fix some errors detected by `testifylint`. Signed-off-by: JmPotato --- .golangci.yml | 19 ++- Makefile | 8 +- pkg/autoscaling/calculation_test.go | 2 +- pkg/autoscaling/prometheus_test.go | 2 +- pkg/balancer/balancer_test.go | 6 +- pkg/cache/cache_test.go | 32 ++-- pkg/core/storelimit/limit_test.go | 2 +- pkg/dashboard/adapter/redirector_test.go | 12 +- pkg/election/leadership_test.go | 2 +- pkg/encryption/config_test.go | 6 +- pkg/encryption/crypter_test.go | 10 +- pkg/encryption/key_manager_test.go | 2 +- pkg/keyspace/keyspace_test.go | 31 ++-- pkg/keyspace/tso_keyspace_group_test.go | 9 +- pkg/mcs/resourcemanager/server/config_test.go | 4 +- .../server/token_buckets_test.go | 14 +- pkg/ratelimit/controller_test.go | 38 ++--- pkg/ratelimit/limiter_test.go | 30 ++-- pkg/schedule/checker/rule_checker_test.go | 14 +- pkg/schedule/filter/counter_test.go | 10 +- pkg/schedule/labeler/rule_test.go | 4 +- pkg/schedule/operator/builder_test.go | 157 +++++++++--------- pkg/schedule/placement/rule_manager_test.go | 10 +- pkg/schedule/plan/balance_plan_test.go | 42 +++-- pkg/schedule/scatter/region_scatterer_test.go | 4 +- .../schedulers/balance_benchmark_test.go | 2 +- pkg/schedule/schedulers/balance_test.go | 38 +++-- .../schedulers/balance_witness_test.go | 2 +- pkg/schedule/schedulers/evict_leader_test.go | 2 +- .../schedulers/evict_slow_trend_test.go | 4 +- pkg/schedule/schedulers/hot_region_test.go | 24 +-- pkg/schedule/schedulers/hot_region_v2_test.go | 8 +- pkg/schedule/schedulers/scheduler_test.go | 2 +- pkg/storage/storage_gc_test.go | 2 +- pkg/tso/keyspace_group_manager_test.go | 2 +- .../unsafe_recovery_controller_test.go | 6 +- pkg/utils/etcdutil/etcdutil_test.go | 80 ++++----- pkg/utils/syncutil/lock_group_test.go | 4 +- pkg/utils/typeutil/duration_test.go | 2 +- pkg/window/policy_test.go | 6 +- pkg/window/window_test.go | 23 +-- scripts/check-test.sh | 37 ----- server/api/admin_test.go | 124 +++++++------- server/api/region_test.go | 2 +- server/cluster/cluster_test.go | 18 +- server/config/config_test.go | 4 +- tests/pdctl/hot/hot_test.go | 18 +- tests/pdctl/keyspace/keyspace_group_test.go | 16 +- tests/pdctl/keyspace/keyspace_test.go | 16 +- tests/pdctl/log/log_test.go | 7 +- tests/pdctl/operator/operator_test.go | 2 +- .../resource_manager_command_test.go | 22 +-- tests/pdctl/scheduler/scheduler_test.go | 6 +- tests/server/api/api_test.go | 42 ++--- tests/server/api/rule_test.go | 22 +-- tests/server/apiv2/handlers/keyspace_test.go | 16 +- .../apiv2/handlers/tso_keyspace_group_test.go | 9 +- tests/server/cluster/cluster_test.go | 2 +- tests/server/cluster/cluster_work_test.go | 5 +- tests/server/keyspace/keyspace_test.go | 2 +- tools/pd-backup/pdbackup/backup_test.go | 24 +-- .../simulator/simutil/key_test.go | 4 +- 62 files changed, 541 insertions(+), 534 deletions(-) delete mode 100755 scripts/check-test.sh diff --git a/.golangci.yml b/.golangci.yml index 079e25ec2b3..59954cecee3 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -11,15 +11,12 @@ linters: - makezero - gosec - bodyclose + # TODO: enable when all existing errors are fixed + # - testifylint disable: - errcheck linters-settings: gocritic: - # Which checks should be enabled; can't be combined with 'disabled-checks'; - # See https://go-critic.github.io/overview#checks-overview - # To check which checks are enabled run `GL_DEBUG=gocritic golangci-lint run` - # By default list of stable checks is used. - enabled-checks: # Which checks should be disabled; can't be combined with 'enabled-checks'; default is empty disabled-checks: - regexpMust @@ -33,3 +30,15 @@ linters-settings: - G402 - G404 - G601 + testifylint: + enable: + - bool-compare + - compares + - empty + - error-is-as + - error-nil + - expected-actual + - len + - require-error + - suite-dont-use-pkg + - suite-extra-assert-call diff --git a/Makefile b/Makefile index 133f99cfac8..bf25730c0d9 100644 --- a/Makefile +++ b/Makefile @@ -172,7 +172,7 @@ install-tools: #### Static checks #### -check: install-tools tidy static generate-errdoc check-test +check: install-tools tidy static generate-errdoc static: install-tools @ echo "gofmt ..." @@ -199,11 +199,7 @@ check-plugin: @echo "checking plugin..." cd ./plugin/scheduler_example && $(MAKE) evictLeaderPlugin.so && rm evictLeaderPlugin.so -check-test: - @echo "checking test..." - ./scripts/check-test.sh - -.PHONY: check static tidy generate-errdoc check-plugin check-test +.PHONY: check static tidy generate-errdoc check-plugin #### Test utils #### diff --git a/pkg/autoscaling/calculation_test.go b/pkg/autoscaling/calculation_test.go index de3be68d68c..85f723b562c 100644 --- a/pkg/autoscaling/calculation_test.go +++ b/pkg/autoscaling/calculation_test.go @@ -233,7 +233,7 @@ func TestGetTotalCPUUseTime(t *testing.T) { } totalCPUUseTime, _ := getTotalCPUUseTime(querier, TiDB, instances, time.Now(), 0) expected := mockResultValue * float64(len(instances)) - re.True(math.Abs(expected-totalCPUUseTime) < 1e-6) + re.Less(math.Abs(expected-totalCPUUseTime), 1e-6) } func TestGetTotalCPUQuota(t *testing.T) { diff --git a/pkg/autoscaling/prometheus_test.go b/pkg/autoscaling/prometheus_test.go index 6c30e3ead4c..2efdc348ead 100644 --- a/pkg/autoscaling/prometheus_test.go +++ b/pkg/autoscaling/prometheus_test.go @@ -196,7 +196,7 @@ func TestRetrieveCPUMetrics(t *testing.T) { for i := 0; i < len(addresses)-1; i++ { value, ok := result[addresses[i]] re.True(ok) - re.True(math.Abs(value-mockResultValue) < 1e-6) + re.Less(math.Abs(value-mockResultValue), 1e-6) } _, ok := result[addresses[len(addresses)-1]] diff --git a/pkg/balancer/balancer_test.go b/pkg/balancer/balancer_test.go index f95487a4cc7..996b4f1da35 100644 --- a/pkg/balancer/balancer_test.go +++ b/pkg/balancer/balancer_test.go @@ -62,7 +62,7 @@ func TestBalancerDuplicate(t *testing.T) { NewRoundRobin[uint32](), } for _, balancer := range balancers { - re.Len(balancer.GetAll(), 0) + re.Empty(balancer.GetAll()) // test duplicate put balancer.Put(1) re.Len(balancer.GetAll(), 1) @@ -70,9 +70,9 @@ func TestBalancerDuplicate(t *testing.T) { re.Len(balancer.GetAll(), 1) // test duplicate delete balancer.Delete(1) - re.Len(balancer.GetAll(), 0) + re.Empty(balancer.GetAll()) balancer.Delete(1) - re.Len(balancer.GetAll(), 0) + re.Empty(balancer.GetAll()) } } diff --git a/pkg/cache/cache_test.go b/pkg/cache/cache_test.go index b02e8823398..fe9f84223c1 100644 --- a/pkg/cache/cache_test.go +++ b/pkg/cache/cache_test.go @@ -77,7 +77,7 @@ func TestExpireRegionCache(t *testing.T) { re.Equal(3, cache.Len()) - re.Equal(sortIDs(cache.GetAllID()), []uint64{1, 2, 3}) + re.Equal([]uint64{1, 2, 3}, sortIDs(cache.GetAllID())) // after 20ms, the key 1 will be expired time.Sleep(20 * time.Millisecond) @@ -98,7 +98,7 @@ func TestExpireRegionCache(t *testing.T) { // we can't ensure whether gc is executed, so we check the length of cache in a loop. return cache.Len() == 2 }, testutil.WithWaitFor(50*time.Millisecond), testutil.WithTickInterval(time.Millisecond)) - re.Equal(sortIDs(cache.GetAllID()), []uint64{2, 3}) + re.Equal([]uint64{2, 3}, sortIDs(cache.GetAllID())) cache.Remove(2) @@ -111,7 +111,7 @@ func TestExpireRegionCache(t *testing.T) { re.Equal(3.0, value) re.Equal(1, cache.Len()) - re.Equal(sortIDs(cache.GetAllID()), []uint64{3}) + re.Equal([]uint64{3}, sortIDs(cache.GetAllID())) } func sortIDs(ids []uint64) []uint64 { @@ -131,15 +131,15 @@ func TestLRUCache(t *testing.T) { val, ok := cache.Get(3) re.True(ok) - re.Equal(val, "3") + re.Equal("3", val) val, ok = cache.Get(2) re.True(ok) - re.Equal(val, "2") + re.Equal("2", val) val, ok = cache.Get(1) re.True(ok) - re.Equal(val, "1") + re.Equal("1", val) re.Equal(3, cache.Len()) @@ -153,27 +153,27 @@ func TestLRUCache(t *testing.T) { val, ok = cache.Get(1) re.True(ok) - re.Equal(val, "1") + re.Equal("1", val) val, ok = cache.Get(2) re.True(ok) - re.Equal(val, "2") + re.Equal("2", val) val, ok = cache.Get(4) re.True(ok) - re.Equal(val, "4") + re.Equal("4", val) re.Equal(3, cache.Len()) val, ok = cache.Peek(1) re.True(ok) - re.Equal(val, "1") + re.Equal("1", val) elems := cache.Elems() re.Len(elems, 3) - re.Equal(elems[0].Value, "4") - re.Equal(elems[1].Value, "2") - re.Equal(elems[2].Value, "1") + re.Equal("4", elems[0].Value) + re.Equal("2", elems[1].Value) + re.Equal("1", elems[2].Value) cache.Remove(1) cache.Remove(2) @@ -247,15 +247,15 @@ func TestFifoFromLastSameElems(t *testing.T) { }) } items := fun() - re.Equal(1, len(items)) + re.Len(items, 1) cache.Put(1, &testStruct{value: "3"}) cache.Put(2, &testStruct{value: "3"}) items = fun() - re.Equal(3, len(items)) + re.Len(items, 3) re.Equal("3", items[0].Value.(*testStruct).value) cache.Put(1, &testStruct{value: "2"}) items = fun() - re.Equal(1, len(items)) + re.Len(items, 1) re.Equal("2", items[0].Value.(*testStruct).value) } diff --git a/pkg/core/storelimit/limit_test.go b/pkg/core/storelimit/limit_test.go index 6f57c01eccb..946729f8ce2 100644 --- a/pkg/core/storelimit/limit_test.go +++ b/pkg/core/storelimit/limit_test.go @@ -30,7 +30,7 @@ func TestStoreLimit(t *testing.T) { re := require.New(t) rate := int64(15) limit := NewStoreRateLimit(float64(rate)).(*StoreRateLimit) - re.Equal(limit.Rate(AddPeer), float64(15)) + re.Equal(float64(15), limit.Rate(AddPeer)) re.True(limit.Available(influence*rate, AddPeer, constant.Low)) re.True(limit.Take(influence*rate, AddPeer, constant.Low)) re.False(limit.Take(influence, AddPeer, constant.Low)) diff --git a/pkg/dashboard/adapter/redirector_test.go b/pkg/dashboard/adapter/redirector_test.go index c5d837507fc..5fc9ea5ea99 100644 --- a/pkg/dashboard/adapter/redirector_test.go +++ b/pkg/dashboard/adapter/redirector_test.go @@ -65,37 +65,39 @@ func (suite *redirectorTestSuite) TearDownSuite() { } func (suite *redirectorTestSuite) TestReverseProxy() { + re := suite.Require() redirectorServer := httptest.NewServer(http.HandlerFunc(suite.redirector.ReverseProxy)) defer redirectorServer.Close() suite.redirector.SetAddress(suite.tempServer.URL) // Test normal forwarding req, err := http.NewRequest(http.MethodGet, redirectorServer.URL, http.NoBody) - suite.NoError(err) + re.NoError(err) checkHTTPRequest(suite.Require(), suite.noRedirectHTTPClient, req, http.StatusOK, suite.tempText) // Test the requests that are forwarded by others req, err = http.NewRequest(http.MethodGet, redirectorServer.URL, http.NoBody) - suite.NoError(err) + re.NoError(err) req.Header.Set(proxyHeader, "other") checkHTTPRequest(suite.Require(), suite.noRedirectHTTPClient, req, http.StatusOK, suite.tempText) // Test LoopDetected suite.redirector.SetAddress(redirectorServer.URL) req, err = http.NewRequest(http.MethodGet, redirectorServer.URL, http.NoBody) - suite.NoError(err) + re.NoError(err) checkHTTPRequest(suite.Require(), suite.noRedirectHTTPClient, req, http.StatusLoopDetected, "") } func (suite *redirectorTestSuite) TestTemporaryRedirect() { + re := suite.Require() redirectorServer := httptest.NewServer(http.HandlerFunc(suite.redirector.TemporaryRedirect)) defer redirectorServer.Close() suite.redirector.SetAddress(suite.tempServer.URL) // Test TemporaryRedirect req, err := http.NewRequest(http.MethodGet, redirectorServer.URL, http.NoBody) - suite.NoError(err) + re.NoError(err) checkHTTPRequest(suite.Require(), suite.noRedirectHTTPClient, req, http.StatusTemporaryRedirect, "") // Test Response req, err = http.NewRequest(http.MethodGet, redirectorServer.URL, http.NoBody) - suite.NoError(err) + re.NoError(err) checkHTTPRequest(suite.Require(), http.DefaultClient, req, http.StatusOK, suite.tempText) } diff --git a/pkg/election/leadership_test.go b/pkg/election/leadership_test.go index c259476e44e..be1922fe381 100644 --- a/pkg/election/leadership_test.go +++ b/pkg/election/leadership_test.go @@ -175,7 +175,7 @@ func TestExitWatch(t *testing.T) { resp2, err := client.MemberList(context.Background()) re.NoError(err) - re.Equal(3, len(resp2.Members)) + re.Len(resp2.Members, 3) etcd2.Server.HardStop() etcd3.Server.HardStop() diff --git a/pkg/encryption/config_test.go b/pkg/encryption/config_test.go index 30c9c9dded8..6f7e4a41b03 100644 --- a/pkg/encryption/config_test.go +++ b/pkg/encryption/config_test.go @@ -38,19 +38,19 @@ func TestAdjustInvalidDataEncryptionMethod(t *testing.T) { t.Parallel() re := require.New(t) config := &Config{DataEncryptionMethod: "unknown"} - re.NotNil(config.Adjust()) + re.Error(config.Adjust()) } func TestAdjustNegativeRotationDuration(t *testing.T) { t.Parallel() re := require.New(t) config := &Config{DataKeyRotationPeriod: typeutil.NewDuration(time.Duration(int64(-1)))} - re.NotNil(config.Adjust()) + re.Error(config.Adjust()) } func TestAdjustInvalidMasterKeyType(t *testing.T) { t.Parallel() re := require.New(t) config := &Config{MasterKey: MasterKeyConfig{Type: "unknown"}} - re.NotNil(config.Adjust()) + re.Error(config.Adjust()) } diff --git a/pkg/encryption/crypter_test.go b/pkg/encryption/crypter_test.go index 2f952d5b729..12a851d1563 100644 --- a/pkg/encryption/crypter_test.go +++ b/pkg/encryption/crypter_test.go @@ -26,11 +26,11 @@ import ( func TestEncryptionMethodSupported(t *testing.T) { t.Parallel() re := require.New(t) - re.NotNil(CheckEncryptionMethodSupported(encryptionpb.EncryptionMethod_PLAINTEXT)) - re.NotNil(CheckEncryptionMethodSupported(encryptionpb.EncryptionMethod_UNKNOWN)) - re.Nil(CheckEncryptionMethodSupported(encryptionpb.EncryptionMethod_AES128_CTR)) - re.Nil(CheckEncryptionMethodSupported(encryptionpb.EncryptionMethod_AES192_CTR)) - re.Nil(CheckEncryptionMethodSupported(encryptionpb.EncryptionMethod_AES256_CTR)) + re.Error(CheckEncryptionMethodSupported(encryptionpb.EncryptionMethod_PLAINTEXT)) + re.Error(CheckEncryptionMethodSupported(encryptionpb.EncryptionMethod_UNKNOWN)) + re.NoError(CheckEncryptionMethodSupported(encryptionpb.EncryptionMethod_AES128_CTR)) + re.NoError(CheckEncryptionMethodSupported(encryptionpb.EncryptionMethod_AES192_CTR)) + re.NoError(CheckEncryptionMethodSupported(encryptionpb.EncryptionMethod_AES256_CTR)) } func TestKeyLength(t *testing.T) { diff --git a/pkg/encryption/key_manager_test.go b/pkg/encryption/key_manager_test.go index 3134e714543..96bdb3c0eb5 100644 --- a/pkg/encryption/key_manager_test.go +++ b/pkg/encryption/key_manager_test.go @@ -313,7 +313,7 @@ func TestLoadKeyEmpty(t *testing.T) { // Simulate keys get deleted. _, err = client.Delete(context.Background(), EncryptionKeysPath) re.NoError(err) - re.NotNil(m.loadKeys()) + re.Error(m.loadKeys()) } func TestWatcher(t *testing.T) { diff --git a/pkg/keyspace/keyspace_test.go b/pkg/keyspace/keyspace_test.go index 27e7de359ee..552adc8d83e 100644 --- a/pkg/keyspace/keyspace_test.go +++ b/pkg/keyspace/keyspace_test.go @@ -75,13 +75,14 @@ func (m *mockConfig) GetCheckRegionSplitInterval() time.Duration { } func (suite *keyspaceTestSuite) SetupTest() { + re := suite.Require() suite.ctx, suite.cancel = context.WithCancel(context.Background()) store := endpoint.NewStorageEndpoint(kv.NewMemoryKV(), nil) allocator := mockid.NewIDAllocator() kgm := NewKeyspaceGroupManager(suite.ctx, store, nil, 0) suite.manager = NewKeyspaceManager(suite.ctx, store, nil, allocator, &mockConfig{}, kgm) - suite.NoError(kgm.Bootstrap(suite.ctx)) - suite.NoError(suite.manager.Bootstrap()) + re.NoError(kgm.Bootstrap(suite.ctx)) + re.NoError(suite.manager.Bootstrap()) } func (suite *keyspaceTestSuite) TearDownTest() { @@ -89,11 +90,13 @@ func (suite *keyspaceTestSuite) TearDownTest() { } func (suite *keyspaceTestSuite) SetupSuite() { - suite.NoError(failpoint.Enable("github.com/tikv/pd/pkg/keyspace/skipSplitRegion", "return(true)")) + re := suite.Require() + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/keyspace/skipSplitRegion", "return(true)")) } func (suite *keyspaceTestSuite) TearDownSuite() { - suite.NoError(failpoint.Disable("github.com/tikv/pd/pkg/keyspace/skipSplitRegion")) + re := suite.Require() + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/keyspace/skipSplitRegion")) } func makeCreateKeyspaceRequests(count int) []*CreateKeyspaceRequest { @@ -205,20 +208,20 @@ func (suite *keyspaceTestSuite) TestUpdateKeyspaceState() { // Disabling an ENABLED keyspace is allowed. Should update StateChangedAt. updated, err := manager.UpdateKeyspaceState(createRequest.Name, keyspacepb.KeyspaceState_DISABLED, oldTime) re.NoError(err) - re.Equal(updated.State, keyspacepb.KeyspaceState_DISABLED) - re.Equal(updated.StateChangedAt, oldTime) + re.Equal(keyspacepb.KeyspaceState_DISABLED, updated.State) + re.Equal(oldTime, updated.StateChangedAt) newTime := time.Now().Unix() // Disabling an DISABLED keyspace is allowed. Should NOT update StateChangedAt. updated, err = manager.UpdateKeyspaceState(createRequest.Name, keyspacepb.KeyspaceState_DISABLED, newTime) re.NoError(err) - re.Equal(updated.State, keyspacepb.KeyspaceState_DISABLED) - re.Equal(updated.StateChangedAt, oldTime) + re.Equal(keyspacepb.KeyspaceState_DISABLED, updated.State) + re.Equal(oldTime, updated.StateChangedAt) // Archiving a DISABLED keyspace is allowed. Should update StateChangeAt. updated, err = manager.UpdateKeyspaceState(createRequest.Name, keyspacepb.KeyspaceState_ARCHIVED, newTime) re.NoError(err) - re.Equal(updated.State, keyspacepb.KeyspaceState_ARCHIVED) - re.Equal(updated.StateChangedAt, newTime) + re.Equal(keyspacepb.KeyspaceState_ARCHIVED, updated.State) + re.Equal(newTime, updated.StateChangedAt) // Changing state of an ARCHIVED keyspace is not allowed. _, err = manager.UpdateKeyspaceState(createRequest.Name, keyspacepb.KeyspaceState_ENABLED, newTime) re.Error(err) @@ -244,7 +247,7 @@ func (suite *keyspaceTestSuite) TestLoadRangeKeyspace() { // Load all keyspaces including the default keyspace. keyspaces, err := manager.LoadRangeKeyspace(0, 0) re.NoError(err) - re.Equal(total+1, len(keyspaces)) + re.Len(keyspaces, total+1) for i := range keyspaces { re.Equal(uint32(i), keyspaces[i].Id) if i != 0 { @@ -256,7 +259,7 @@ func (suite *keyspaceTestSuite) TestLoadRangeKeyspace() { // Result should be keyspaces with id 0 - 49. keyspaces, err = manager.LoadRangeKeyspace(0, 50) re.NoError(err) - re.Equal(50, len(keyspaces)) + re.Len(keyspaces, 50) for i := range keyspaces { re.Equal(uint32(i), keyspaces[i].Id) if i != 0 { @@ -269,7 +272,7 @@ func (suite *keyspaceTestSuite) TestLoadRangeKeyspace() { loadStart := 33 keyspaces, err = manager.LoadRangeKeyspace(uint32(loadStart), 20) re.NoError(err) - re.Equal(20, len(keyspaces)) + re.Len(keyspaces, 20) for i := range keyspaces { re.Equal(uint32(loadStart+i), keyspaces[i].Id) checkCreateRequest(re, requests[i+loadStart-1], keyspaces[i]) @@ -280,7 +283,7 @@ func (suite *keyspaceTestSuite) TestLoadRangeKeyspace() { loadStart = 90 keyspaces, err = manager.LoadRangeKeyspace(uint32(loadStart), 30) re.NoError(err) - re.Equal(11, len(keyspaces)) + re.Len(keyspaces, 11) for i := range keyspaces { re.Equal(uint32(loadStart+i), keyspaces[i].Id) checkCreateRequest(re, requests[i+loadStart-1], keyspaces[i]) diff --git a/pkg/keyspace/tso_keyspace_group_test.go b/pkg/keyspace/tso_keyspace_group_test.go index 993923d2fd7..2dec780c3c8 100644 --- a/pkg/keyspace/tso_keyspace_group_test.go +++ b/pkg/keyspace/tso_keyspace_group_test.go @@ -43,13 +43,14 @@ func TestKeyspaceGroupTestSuite(t *testing.T) { } func (suite *keyspaceGroupTestSuite) SetupTest() { + re := suite.Require() suite.ctx, suite.cancel = context.WithCancel(context.Background()) store := endpoint.NewStorageEndpoint(kv.NewMemoryKV(), nil) suite.kgm = NewKeyspaceGroupManager(suite.ctx, store, nil, 0) idAllocator := mockid.NewIDAllocator() cluster := mockcluster.NewCluster(suite.ctx, mockconfig.NewTestOptions()) suite.kg = NewKeyspaceManager(suite.ctx, store, cluster, idAllocator, &mockConfig{}, suite.kgm) - suite.NoError(suite.kgm.Bootstrap(suite.ctx)) + re.NoError(suite.kgm.Bootstrap(suite.ctx)) } func (suite *keyspaceGroupTestSuite) TearDownTest() { @@ -191,7 +192,7 @@ func (suite *keyspaceGroupTestSuite) TestUpdateKeyspace() { re.Len(kg2.Keyspaces, 1) kg3, err := suite.kgm.GetKeyspaceGroupByID(3) re.NoError(err) - re.Len(kg3.Keyspaces, 0) + re.Empty(kg3.Keyspaces) _, err = suite.kg.UpdateKeyspaceConfig("test", []*Mutation{ { @@ -211,7 +212,7 @@ func (suite *keyspaceGroupTestSuite) TestUpdateKeyspace() { re.Len(kg2.Keyspaces, 1) kg3, err = suite.kgm.GetKeyspaceGroupByID(3) re.NoError(err) - re.Len(kg3.Keyspaces, 0) + re.Empty(kg3.Keyspaces) _, err = suite.kg.UpdateKeyspaceConfig("test", []*Mutation{ { Op: OpPut, @@ -227,7 +228,7 @@ func (suite *keyspaceGroupTestSuite) TestUpdateKeyspace() { re.NoError(err) kg2, err = suite.kgm.GetKeyspaceGroupByID(2) re.NoError(err) - re.Len(kg2.Keyspaces, 0) + re.Empty(kg2.Keyspaces) kg3, err = suite.kgm.GetKeyspaceGroupByID(3) re.NoError(err) re.Len(kg3.Keyspaces, 1) diff --git a/pkg/mcs/resourcemanager/server/config_test.go b/pkg/mcs/resourcemanager/server/config_test.go index dd8dd2d2814..64fd133ea73 100644 --- a/pkg/mcs/resourcemanager/server/config_test.go +++ b/pkg/mcs/resourcemanager/server/config_test.go @@ -42,8 +42,8 @@ read-cpu-ms-cost = 5.0 err = cfg.Adjust(&meta, false) re.NoError(err) - re.Equal(cfg.Controller.DegradedModeWaitDuration.Duration, time.Second*2) - re.Equal(cfg.Controller.LTBMaxWaitDuration.Duration, time.Second*60) + re.Equal(time.Second*2, cfg.Controller.DegradedModeWaitDuration.Duration) + re.Equal(time.Second*60, cfg.Controller.LTBMaxWaitDuration.Duration) re.LessOrEqual(math.Abs(cfg.Controller.RequestUnit.CPUMsCost-5), 1e-7) re.LessOrEqual(math.Abs(cfg.Controller.RequestUnit.WriteCostPerByte-4), 1e-7) re.LessOrEqual(math.Abs(cfg.Controller.RequestUnit.WriteBaseCost-3), 1e-7) diff --git a/pkg/mcs/resourcemanager/server/token_buckets_test.go b/pkg/mcs/resourcemanager/server/token_buckets_test.go index a7d3b9e3bad..4138be5d66e 100644 --- a/pkg/mcs/resourcemanager/server/token_buckets_test.go +++ b/pkg/mcs/resourcemanager/server/token_buckets_test.go @@ -70,27 +70,27 @@ func TestGroupTokenBucketRequest(t *testing.T) { clientUniqueID := uint64(0) tb, trickle := gtb.request(time1, 190000, uint64(time.Second)*10/uint64(time.Millisecond), clientUniqueID) re.LessOrEqual(math.Abs(tb.Tokens-190000), 1e-7) - re.Equal(trickle, int64(0)) + re.Zero(trickle) // need to lend token tb, trickle = gtb.request(time1, 11000, uint64(time.Second)*10/uint64(time.Millisecond), clientUniqueID) re.LessOrEqual(math.Abs(tb.Tokens-11000), 1e-7) - re.Equal(trickle, int64(time.Second)*11000./4000./int64(time.Millisecond)) + re.Equal(int64(time.Second)*11000./4000./int64(time.Millisecond), trickle) tb, trickle = gtb.request(time1, 35000, uint64(time.Second)*10/uint64(time.Millisecond), clientUniqueID) re.LessOrEqual(math.Abs(tb.Tokens-35000), 1e-7) - re.Equal(trickle, int64(time.Second)*10/int64(time.Millisecond)) + re.Equal(int64(time.Second)*10/int64(time.Millisecond), trickle) tb, trickle = gtb.request(time1, 60000, uint64(time.Second)*10/uint64(time.Millisecond), clientUniqueID) re.LessOrEqual(math.Abs(tb.Tokens-22000), 1e-7) - re.Equal(trickle, int64(time.Second)*10/int64(time.Millisecond)) + re.Equal(int64(time.Second)*10/int64(time.Millisecond), trickle) // Get reserved 10000 tokens = fillrate(2000) * 10 * defaultReserveRatio(0.5) // Max loan tokens is 60000. tb, trickle = gtb.request(time1, 3000, uint64(time.Second)*10/uint64(time.Millisecond), clientUniqueID) re.LessOrEqual(math.Abs(tb.Tokens-3000), 1e-7) - re.Equal(trickle, int64(time.Second)*10/int64(time.Millisecond)) + re.Equal(int64(time.Second)*10/int64(time.Millisecond), trickle) tb, trickle = gtb.request(time1, 12000, uint64(time.Second)*10/uint64(time.Millisecond), clientUniqueID) re.LessOrEqual(math.Abs(tb.Tokens-10000), 1e-7) - re.Equal(trickle, int64(time.Second)*10/int64(time.Millisecond)) + re.Equal(int64(time.Second)*10/int64(time.Millisecond), trickle) time2 := time1.Add(20 * time.Second) tb, trickle = gtb.request(time2, 20000, uint64(time.Second)*10/uint64(time.Millisecond), clientUniqueID) re.LessOrEqual(math.Abs(tb.Tokens-20000), 1e-7) - re.Equal(trickle, int64(time.Second)*10/int64(time.Millisecond)) + re.Equal(int64(time.Second)*10/int64(time.Millisecond), trickle) } diff --git a/pkg/ratelimit/controller_test.go b/pkg/ratelimit/controller_test.go index a830217cb9f..59cc0c16445 100644 --- a/pkg/ratelimit/controller_test.go +++ b/pkg/ratelimit/controller_test.go @@ -88,7 +88,7 @@ func TestControllerWithConcurrencyLimiter(t *testing.T) { opt: UpdateConcurrencyLimiter(10), checkOptionStatus: func(label string, o Option) { status := limiter.Update(label, o) - re.True(status&ConcurrencyChanged != 0) + re.NotZero(status & ConcurrencyChanged) }, totalRequest: 15, fail: 5, @@ -105,7 +105,7 @@ func TestControllerWithConcurrencyLimiter(t *testing.T) { opt: UpdateConcurrencyLimiter(10), checkOptionStatus: func(label string, o Option) { status := limiter.Update(label, o) - re.True(status&ConcurrencyNoChange != 0) + re.NotZero(status & ConcurrencyNoChange) }, checkStatusFunc: func(label string) {}, }, @@ -113,7 +113,7 @@ func TestControllerWithConcurrencyLimiter(t *testing.T) { opt: UpdateConcurrencyLimiter(5), checkOptionStatus: func(label string, o Option) { status := limiter.Update(label, o) - re.True(status&ConcurrencyChanged != 0) + re.NotZero(status & ConcurrencyChanged) }, totalRequest: 15, fail: 10, @@ -130,7 +130,7 @@ func TestControllerWithConcurrencyLimiter(t *testing.T) { opt: UpdateConcurrencyLimiter(0), checkOptionStatus: func(label string, o Option) { status := limiter.Update(label, o) - re.True(status&ConcurrencyDeleted != 0) + re.NotZero(status & ConcurrencyDeleted) }, totalRequest: 15, fail: 0, @@ -152,7 +152,7 @@ func TestControllerWithConcurrencyLimiter(t *testing.T) { opt: UpdateConcurrencyLimiter(15), checkOptionStatus: func(label string, o Option) { status := limiter.Update(label, o) - re.True(status&ConcurrencyChanged != 0) + re.NotZero(status & ConcurrencyChanged) }, totalRequest: 10, fail: 0, @@ -169,7 +169,7 @@ func TestControllerWithConcurrencyLimiter(t *testing.T) { opt: UpdateConcurrencyLimiter(10), checkOptionStatus: func(label string, o Option) { status := limiter.Update(label, o) - re.True(status&ConcurrencyChanged != 0) + re.NotZero(status & ConcurrencyChanged) }, totalRequest: 10, fail: 10, @@ -202,7 +202,7 @@ func TestBlockList(t *testing.T) { re.True(limiter.IsInAllowList(label)) status := UpdateQPSLimiter(float64(rate.Every(time.Second)), 1)(label, limiter) - re.True(status&InAllowList != 0) + re.NotZero(status & InAllowList) for i := 0; i < 10; i++ { _, err := limiter.Allow(label) re.NoError(err) @@ -221,7 +221,7 @@ func TestControllerWithQPSLimiter(t *testing.T) { opt: UpdateQPSLimiter(float64(rate.Every(time.Second)), 1), checkOptionStatus: func(label string, o Option) { status := limiter.Update(label, o) - re.True(status&QPSChanged != 0) + re.NotZero(status & QPSChanged) }, totalRequest: 3, fail: 2, @@ -237,7 +237,7 @@ func TestControllerWithQPSLimiter(t *testing.T) { opt: UpdateQPSLimiter(float64(rate.Every(time.Second)), 1), checkOptionStatus: func(label string, o Option) { status := limiter.Update(label, o) - re.True(status&QPSNoChange != 0) + re.NotZero(status & QPSNoChange) }, checkStatusFunc: func(label string) {}, }, @@ -245,7 +245,7 @@ func TestControllerWithQPSLimiter(t *testing.T) { opt: UpdateQPSLimiter(5, 5), checkOptionStatus: func(label string, o Option) { status := limiter.Update(label, o) - re.True(status&QPSChanged != 0) + re.NotZero(status & QPSChanged) }, totalRequest: 10, fail: 5, @@ -261,7 +261,7 @@ func TestControllerWithQPSLimiter(t *testing.T) { opt: UpdateQPSLimiter(0, 0), checkOptionStatus: func(label string, o Option) { status := limiter.Update(label, o) - re.True(status&QPSDeleted != 0) + re.NotZero(status & QPSDeleted) }, totalRequest: 10, fail: 0, @@ -271,7 +271,7 @@ func TestControllerWithQPSLimiter(t *testing.T) { checkStatusFunc: func(label string) { limit, burst := limiter.GetQPSLimiterStatus(label) re.Equal(rate.Limit(0), limit) - re.Equal(0, burst) + re.Zero(burst) }, }, }, @@ -283,7 +283,7 @@ func TestControllerWithQPSLimiter(t *testing.T) { opt: UpdateQPSLimiter(50, 5), checkOptionStatus: func(label string, o Option) { status := limiter.Update(label, o) - re.True(status&QPSChanged != 0) + re.NotZero(status & QPSChanged) }, totalRequest: 10, fail: 5, @@ -299,7 +299,7 @@ func TestControllerWithQPSLimiter(t *testing.T) { opt: UpdateQPSLimiter(0, 0), checkOptionStatus: func(label string, o Option) { status := limiter.Update(label, o) - re.True(status&QPSDeleted != 0) + re.NotZero(status & QPSDeleted) }, totalRequest: 10, fail: 0, @@ -309,7 +309,7 @@ func TestControllerWithQPSLimiter(t *testing.T) { checkStatusFunc: func(label string) { limit, burst := limiter.GetQPSLimiterStatus(label) re.Equal(rate.Limit(0), limit) - re.Equal(0, burst) + re.Zero(burst) }, }, }, @@ -334,7 +334,7 @@ func TestControllerWithTwoLimiters(t *testing.T) { }), checkOptionStatus: func(label string, o Option) { status := limiter.Update(label, o) - re.True(status&QPSChanged != 0) + re.NotZero(status & QPSChanged) }, totalRequest: 200, fail: 100, @@ -354,7 +354,7 @@ func TestControllerWithTwoLimiters(t *testing.T) { opt: UpdateQPSLimiter(float64(rate.Every(time.Second)), 1), checkOptionStatus: func(label string, o Option) { status := limiter.Update(label, o) - re.True(status&QPSChanged != 0) + re.NotZero(status & QPSChanged) }, totalRequest: 200, fail: 199, @@ -376,7 +376,7 @@ func TestControllerWithTwoLimiters(t *testing.T) { opt: UpdateQPSLimiter(50, 5), checkOptionStatus: func(label string, o Option) { status := limiter.Update(label, o) - re.True(status&QPSChanged != 0) + re.NotZero(status & QPSChanged) }, totalRequest: 10, fail: 5, @@ -392,7 +392,7 @@ func TestControllerWithTwoLimiters(t *testing.T) { opt: UpdateQPSLimiter(0, 0), checkOptionStatus: func(label string, o Option) { status := limiter.Update(label, o) - re.True(status&QPSDeleted != 0) + re.NotZero(status & QPSDeleted) }, totalRequest: 10, fail: 0, diff --git a/pkg/ratelimit/limiter_test.go b/pkg/ratelimit/limiter_test.go index 8834495f3e9..88da865879b 100644 --- a/pkg/ratelimit/limiter_test.go +++ b/pkg/ratelimit/limiter_test.go @@ -45,7 +45,7 @@ func TestWithConcurrencyLimiter(t *testing.T) { limiter := newLimiter() status := limiter.updateConcurrencyConfig(10) - re.True(status&ConcurrencyChanged != 0) + re.NotZero(status & ConcurrencyChanged) var lock syncutil.Mutex successCount, failedCount := 0, 0 var wg sync.WaitGroup @@ -68,10 +68,10 @@ func TestWithConcurrencyLimiter(t *testing.T) { re.Equal(uint64(0), current) status = limiter.updateConcurrencyConfig(10) - re.True(status&ConcurrencyNoChange != 0) + re.NotZero(status & ConcurrencyNoChange) status = limiter.updateConcurrencyConfig(5) - re.True(status&ConcurrencyChanged != 0) + re.NotZero(status & ConcurrencyChanged) failedCount = 0 successCount = 0 for i := 0; i < 15; i++ { @@ -86,7 +86,7 @@ func TestWithConcurrencyLimiter(t *testing.T) { } status = limiter.updateConcurrencyConfig(0) - re.True(status&ConcurrencyDeleted != 0) + re.NotZero(status & ConcurrencyDeleted) failedCount = 0 successCount = 0 for i := 0; i < 15; i++ { @@ -107,7 +107,7 @@ func TestWithQPSLimiter(t *testing.T) { re := require.New(t) limiter := newLimiter() status := limiter.updateQPSConfig(float64(rate.Every(time.Second)), 1) - re.True(status&QPSChanged != 0) + re.NotZero(status & QPSChanged) var lock syncutil.Mutex successCount, failedCount := 0, 0 @@ -126,10 +126,10 @@ func TestWithQPSLimiter(t *testing.T) { re.Equal(1, burst) status = limiter.updateQPSConfig(float64(rate.Every(time.Second)), 1) - re.True(status&QPSNoChange != 0) + re.NotZero(status & QPSNoChange) status = limiter.updateQPSConfig(5, 5) - re.True(status&QPSChanged != 0) + re.NotZero(status & QPSChanged) limit, burst = limiter.getQPSLimiterStatus() re.Equal(rate.Limit(5), limit) re.Equal(5, burst) @@ -147,19 +147,19 @@ func TestWithQPSLimiter(t *testing.T) { time.Sleep(time.Second) status = limiter.updateQPSConfig(0, 0) - re.True(status&QPSDeleted != 0) + re.NotZero(status & QPSDeleted) for i := 0; i < 10; i++ { _, err := limiter.allow() re.NoError(err) } qLimit, qCurrent := limiter.getQPSLimiterStatus() re.Equal(rate.Limit(0), qLimit) - re.Equal(0, qCurrent) + re.Zero(qCurrent) successCount = 0 failedCount = 0 status = limiter.updateQPSConfig(float64(rate.Every(3*time.Second)), 100) - re.True(status&QPSChanged != 0) + re.NotZero(status & QPSChanged) wg.Add(200) for i := 0; i < 200; i++ { go countSingleLimiterHandleResult(limiter, &successCount, &failedCount, &lock, &wg, r) @@ -186,8 +186,8 @@ func TestWithTwoLimiters(t *testing.T) { } limiter := newLimiter() status := limiter.updateDimensionConfig(cfg) - re.True(status&QPSChanged != 0) - re.True(status&ConcurrencyChanged != 0) + re.NotZero(status & QPSChanged) + re.NotZero(status & ConcurrencyChanged) var lock syncutil.Mutex successCount, failedCount := 0, 0 @@ -214,7 +214,7 @@ func TestWithTwoLimiters(t *testing.T) { r.release() } status = limiter.updateQPSConfig(float64(rate.Every(10*time.Second)), 1) - re.True(status&QPSChanged != 0) + re.NotZero(status & QPSChanged) wg.Add(100) for i := 0; i < 100; i++ { go countSingleLimiterHandleResult(limiter, &successCount, &failedCount, &lock, &wg, r) @@ -228,8 +228,8 @@ func TestWithTwoLimiters(t *testing.T) { cfg = &DimensionConfig{} status = limiter.updateDimensionConfig(cfg) - re.True(status&ConcurrencyDeleted != 0) - re.True(status&QPSDeleted != 0) + re.NotZero(status & ConcurrencyDeleted) + re.NotZero(status & QPSDeleted) } func countSingleLimiterHandleResult(limiter *limiter, successCount *int, diff --git a/pkg/schedule/checker/rule_checker_test.go b/pkg/schedule/checker/rule_checker_test.go index e77830fac49..72d3e7e5ec4 100644 --- a/pkg/schedule/checker/rule_checker_test.go +++ b/pkg/schedule/checker/rule_checker_test.go @@ -426,6 +426,7 @@ func (suite *ruleCheckerTestSuite) TestFixRoleLeaderIssue3130() { } func (suite *ruleCheckerTestSuite) TestFixLeaderRoleWithUnhealthyRegion() { + re := suite.Require() suite.cluster.AddLabelsStore(1, 1, map[string]string{"rule": "follower"}) suite.cluster.AddLabelsStore(2, 1, map[string]string{"rule": "follower"}) suite.cluster.AddLabelsStore(3, 1, map[string]string{"rule": "leader"}) @@ -456,12 +457,12 @@ func (suite *ruleCheckerTestSuite) TestFixLeaderRoleWithUnhealthyRegion() { }, }, }) - suite.NoError(err) + re.NoError(err) // no Leader suite.cluster.AddNoLeaderRegion(1, 1, 2, 3) r := suite.cluster.GetRegion(1) op := suite.rc.Check(r) - suite.Nil(op) + re.Nil(op) } func (suite *ruleCheckerTestSuite) TestFixRuleWitness() { @@ -532,6 +533,7 @@ func (suite *ruleCheckerTestSuite) TestFixRuleWitness3() { } func (suite *ruleCheckerTestSuite) TestFixRuleWitness4() { + re := suite.Require() suite.cluster.AddLabelsStore(1, 1, map[string]string{"A": "leader"}) suite.cluster.AddLabelsStore(2, 1, map[string]string{"B": "voter"}) suite.cluster.AddLabelsStore(3, 1, map[string]string{"C": "learner"}) @@ -565,12 +567,12 @@ func (suite *ruleCheckerTestSuite) TestFixRuleWitness4() { }, }, }) - suite.NoError(err) + re.NoError(err) op := suite.rc.Check(r) - suite.NotNil(op) - suite.Equal("fix-non-witness-peer", op.Desc()) - suite.Equal(uint64(3), op.Step(0).(operator.BecomeNonWitness).StoreID) + re.NotNil(op) + re.Equal("fix-non-witness-peer", op.Desc()) + re.Equal(uint64(3), op.Step(0).(operator.BecomeNonWitness).StoreID) } func (suite *ruleCheckerTestSuite) TestFixRuleWitness5() { diff --git a/pkg/schedule/filter/counter_test.go b/pkg/schedule/filter/counter_test.go index 067a07f138b..78a1ef5395b 100644 --- a/pkg/schedule/filter/counter_test.go +++ b/pkg/schedule/filter/counter_test.go @@ -34,7 +34,7 @@ func TestString(t *testing.T) { for _, data := range testcases { re.Equal(data.expected, filterType(data.filterType).String()) } - re.Equal(int(filtersLen), len(filters)) + re.Len(filters, int(filtersLen)) } func TestCounter(t *testing.T) { @@ -42,9 +42,9 @@ func TestCounter(t *testing.T) { counter := NewCounter(BalanceLeader.String()) counter.inc(source, storeStateTombstone, 1, 2) counter.inc(target, storeStateTombstone, 1, 2) - re.Equal(counter.counter[source][storeStateTombstone][1][2], 1) - re.Equal(counter.counter[target][storeStateTombstone][1][2], 1) + re.Equal(1, counter.counter[source][storeStateTombstone][1][2]) + re.Equal(1, counter.counter[target][storeStateTombstone][1][2]) counter.Flush() - re.Equal(counter.counter[source][storeStateTombstone][1][2], 0) - re.Equal(counter.counter[target][storeStateTombstone][1][2], 0) + re.Zero(counter.counter[source][storeStateTombstone][1][2]) + re.Zero(counter.counter[target][storeStateTombstone][1][2]) } diff --git a/pkg/schedule/labeler/rule_test.go b/pkg/schedule/labeler/rule_test.go index 0b341754007..00c179b36b8 100644 --- a/pkg/schedule/labeler/rule_test.go +++ b/pkg/schedule/labeler/rule_test.go @@ -42,7 +42,7 @@ func TestRegionLabelTTL(t *testing.T) { label.TTL = "10h10m10s10ms" err = label.checkAndAdjustExpire() re.NoError(err) - re.Greater(len(label.StartAt), 0) + re.NotEmpty(label.StartAt) re.False(label.expireBefore(time.Now().Add(time.Hour))) re.True(label.expireBefore(time.Now().Add(24 * time.Hour))) @@ -56,5 +56,5 @@ func TestRegionLabelTTL(t *testing.T) { re.Equal(label.TTL, label2.TTL) label2.checkAndAdjustExpire() // The `expire` should be the same with minor inaccuracies. - re.True(math.Abs(label2.expire.Sub(*label.expire).Seconds()) < 1) + re.Less(math.Abs(label2.expire.Sub(*label.expire).Seconds()), 1.0) } diff --git a/pkg/schedule/operator/builder_test.go b/pkg/schedule/operator/builder_test.go index 864734eb5ff..b010dcf935b 100644 --- a/pkg/schedule/operator/builder_test.go +++ b/pkg/schedule/operator/builder_test.go @@ -62,21 +62,22 @@ func (suite *operatorBuilderTestSuite) TearDownTest() { } func (suite *operatorBuilderTestSuite) TestNewBuilder() { + re := suite.Require() peers := []*metapb.Peer{{Id: 11, StoreId: 1}, {Id: 12, StoreId: 2, Role: metapb.PeerRole_Learner}} region := core.NewRegionInfo(&metapb.Region{Id: 42, Peers: peers}, peers[0]) builder := NewBuilder("test", suite.cluster, region) - suite.NoError(builder.err) - suite.Len(builder.originPeers, 2) - suite.Equal(peers[0], builder.originPeers[1]) - suite.Equal(peers[1], builder.originPeers[2]) - suite.Equal(uint64(1), builder.originLeaderStoreID) - suite.Len(builder.targetPeers, 2) - suite.Equal(peers[0], builder.targetPeers[1]) - suite.Equal(peers[1], builder.targetPeers[2]) + re.NoError(builder.err) + re.Len(builder.originPeers, 2) + re.Equal(peers[0], builder.originPeers[1]) + re.Equal(peers[1], builder.originPeers[2]) + re.Equal(uint64(1), builder.originLeaderStoreID) + re.Len(builder.targetPeers, 2) + re.Equal(peers[0], builder.targetPeers[1]) + re.Equal(peers[1], builder.targetPeers[2]) region = region.Clone(core.WithLeader(nil)) builder = NewBuilder("test", suite.cluster, region) - suite.Error(builder.err) + re.Error(builder.err) } func (suite *operatorBuilderTestSuite) newBuilder() *Builder { @@ -90,18 +91,19 @@ func (suite *operatorBuilderTestSuite) newBuilder() *Builder { } func (suite *operatorBuilderTestSuite) TestRecord() { - suite.Error(suite.newBuilder().AddPeer(&metapb.Peer{StoreId: 1}).err) - suite.NoError(suite.newBuilder().AddPeer(&metapb.Peer{StoreId: 4}).err) - suite.Error(suite.newBuilder().PromoteLearner(1).err) - suite.NoError(suite.newBuilder().PromoteLearner(3).err) - suite.NoError(suite.newBuilder().SetLeader(1).SetLeader(2).err) - suite.Error(suite.newBuilder().SetLeader(3).err) - suite.Error(suite.newBuilder().RemovePeer(4).err) - suite.NoError(suite.newBuilder().AddPeer(&metapb.Peer{StoreId: 4, Role: metapb.PeerRole_Learner}).RemovePeer(4).err) - suite.Error(suite.newBuilder().SetLeader(2).RemovePeer(2).err) - suite.Error(suite.newBuilder().PromoteLearner(4).err) - suite.Error(suite.newBuilder().SetLeader(4).err) - suite.Error(suite.newBuilder().SetPeers(map[uint64]*metapb.Peer{2: {Id: 2}}).err) + re := suite.Require() + re.Error(suite.newBuilder().AddPeer(&metapb.Peer{StoreId: 1}).err) + re.NoError(suite.newBuilder().AddPeer(&metapb.Peer{StoreId: 4}).err) + re.Error(suite.newBuilder().PromoteLearner(1).err) + re.NoError(suite.newBuilder().PromoteLearner(3).err) + re.NoError(suite.newBuilder().SetLeader(1).SetLeader(2).err) + re.Error(suite.newBuilder().SetLeader(3).err) + re.Error(suite.newBuilder().RemovePeer(4).err) + re.NoError(suite.newBuilder().AddPeer(&metapb.Peer{StoreId: 4, Role: metapb.PeerRole_Learner}).RemovePeer(4).err) + re.Error(suite.newBuilder().SetLeader(2).RemovePeer(2).err) + re.Error(suite.newBuilder().PromoteLearner(4).err) + re.Error(suite.newBuilder().SetLeader(4).err) + re.Error(suite.newBuilder().SetPeers(map[uint64]*metapb.Peer{2: {Id: 2}}).err) m := map[uint64]*metapb.Peer{ 2: {StoreId: 2}, @@ -109,18 +111,19 @@ func (suite *operatorBuilderTestSuite) TestRecord() { 4: {StoreId: 4}, } builder := suite.newBuilder().SetPeers(m).SetAddLightPeer() - suite.Len(builder.targetPeers, 3) - suite.Equal(m[2], builder.targetPeers[2]) - suite.Equal(m[3], builder.targetPeers[3]) - suite.Equal(m[4], builder.targetPeers[4]) - suite.Equal(uint64(0), builder.targetLeaderStoreID) - suite.True(builder.addLightPeer) + re.Len(builder.targetPeers, 3) + re.Equal(m[2], builder.targetPeers[2]) + re.Equal(m[3], builder.targetPeers[3]) + re.Equal(m[4], builder.targetPeers[4]) + re.Equal(uint64(0), builder.targetLeaderStoreID) + re.True(builder.addLightPeer) } func (suite *operatorBuilderTestSuite) TestPrepareBuild() { + re := suite.Require() // no voter. _, err := suite.newBuilder().SetPeers(map[uint64]*metapb.Peer{4: {StoreId: 4, Role: metapb.PeerRole_Learner}}).prepareBuild() - suite.Error(err) + re.Error(err) // use joint consensus builder := suite.newBuilder().SetPeers(map[uint64]*metapb.Peer{ @@ -130,19 +133,19 @@ func (suite *operatorBuilderTestSuite) TestPrepareBuild() { 5: {StoreId: 5, Role: metapb.PeerRole_Learner}, }) _, err = builder.prepareBuild() - suite.NoError(err) - suite.Len(builder.toAdd, 2) - suite.NotEqual(metapb.PeerRole_Learner, builder.toAdd[4].GetRole()) - suite.Equal(uint64(14), builder.toAdd[4].GetId()) - suite.Equal(metapb.PeerRole_Learner, builder.toAdd[5].GetRole()) - suite.NotEqual(uint64(0), builder.toAdd[5].GetId()) - suite.Len(builder.toRemove, 1) - suite.NotNil(builder.toRemove[2]) - suite.Len(builder.toPromote, 1) - suite.NotNil(builder.toPromote[3]) - suite.Len(builder.toDemote, 1) - suite.NotNil(builder.toDemote[1]) - suite.Equal(uint64(1), builder.currentLeaderStoreID) + re.NoError(err) + re.Len(builder.toAdd, 2) + re.NotEqual(metapb.PeerRole_Learner, builder.toAdd[4].GetRole()) + re.Equal(uint64(14), builder.toAdd[4].GetId()) + re.Equal(metapb.PeerRole_Learner, builder.toAdd[5].GetRole()) + re.NotEqual(uint64(0), builder.toAdd[5].GetId()) + re.Len(builder.toRemove, 1) + re.NotNil(builder.toRemove[2]) + re.Len(builder.toPromote, 1) + re.NotNil(builder.toPromote[3]) + re.Len(builder.toDemote, 1) + re.NotNil(builder.toDemote[1]) + re.Equal(uint64(1), builder.currentLeaderStoreID) // do not use joint consensus builder = suite.newBuilder().SetPeers(map[uint64]*metapb.Peer{ @@ -154,22 +157,23 @@ func (suite *operatorBuilderTestSuite) TestPrepareBuild() { }) builder.useJointConsensus = false _, err = builder.prepareBuild() - suite.NoError(err) - suite.Len(builder.toAdd, 3) - suite.Equal(metapb.PeerRole_Learner, builder.toAdd[1].GetRole()) - suite.NotEqual(uint64(0), builder.toAdd[1].GetId()) - suite.NotEqual(metapb.PeerRole_Learner, builder.toAdd[4].GetRole()) - suite.Equal(uint64(14), builder.toAdd[4].GetId()) - suite.Equal(metapb.PeerRole_Learner, builder.toAdd[5].GetRole()) - suite.NotEqual(uint64(0), builder.toAdd[5].GetId()) - suite.Len(builder.toRemove, 1) - suite.NotNil(builder.toRemove[1]) - suite.Len(builder.toPromote, 1) - suite.NotNil(builder.toPromote[3]) - suite.Equal(uint64(1), builder.currentLeaderStoreID) + re.NoError(err) + re.Len(builder.toAdd, 3) + re.Equal(metapb.PeerRole_Learner, builder.toAdd[1].GetRole()) + re.NotEqual(uint64(0), builder.toAdd[1].GetId()) + re.NotEqual(metapb.PeerRole_Learner, builder.toAdd[4].GetRole()) + re.Equal(uint64(14), builder.toAdd[4].GetId()) + re.Equal(metapb.PeerRole_Learner, builder.toAdd[5].GetRole()) + re.NotEqual(uint64(0), builder.toAdd[5].GetId()) + re.Len(builder.toRemove, 1) + re.NotNil(builder.toRemove[1]) + re.Len(builder.toPromote, 1) + re.NotNil(builder.toPromote[3]) + re.Equal(uint64(1), builder.currentLeaderStoreID) } func (suite *operatorBuilderTestSuite) TestBuild() { + re := suite.Require() type testCase struct { name string useJointConsensus bool @@ -545,42 +549,42 @@ func (suite *operatorBuilderTestSuite) TestBuild() { builder.SetPeers(m).SetLeader(testCase.targetPeers[0].GetStoreId()) op, err := builder.Build(0) if len(testCase.steps) == 0 { - suite.Error(err) + re.Error(err) continue } - suite.NoError(err) - suite.Equal(testCase.kind, op.Kind()) - suite.Len(testCase.steps, op.Len()) + re.NoError(err) + re.Equal(testCase.kind, op.Kind()) + re.Len(testCase.steps, op.Len()) for i := 0; i < op.Len(); i++ { switch step := op.Step(i).(type) { case TransferLeader: - suite.Equal(testCase.steps[i].(TransferLeader).FromStore, step.FromStore) - suite.Equal(testCase.steps[i].(TransferLeader).ToStore, step.ToStore) + re.Equal(testCase.steps[i].(TransferLeader).FromStore, step.FromStore) + re.Equal(testCase.steps[i].(TransferLeader).ToStore, step.ToStore) case AddPeer: - suite.Equal(testCase.steps[i].(AddPeer).ToStore, step.ToStore) + re.Equal(testCase.steps[i].(AddPeer).ToStore, step.ToStore) case RemovePeer: - suite.Equal(testCase.steps[i].(RemovePeer).FromStore, step.FromStore) + re.Equal(testCase.steps[i].(RemovePeer).FromStore, step.FromStore) case AddLearner: - suite.Equal(testCase.steps[i].(AddLearner).ToStore, step.ToStore) + re.Equal(testCase.steps[i].(AddLearner).ToStore, step.ToStore) case PromoteLearner: - suite.Equal(testCase.steps[i].(PromoteLearner).ToStore, step.ToStore) + re.Equal(testCase.steps[i].(PromoteLearner).ToStore, step.ToStore) case ChangePeerV2Enter: - suite.Len(step.PromoteLearners, len(testCase.steps[i].(ChangePeerV2Enter).PromoteLearners)) - suite.Len(step.DemoteVoters, len(testCase.steps[i].(ChangePeerV2Enter).DemoteVoters)) + re.Len(step.PromoteLearners, len(testCase.steps[i].(ChangePeerV2Enter).PromoteLearners)) + re.Len(step.DemoteVoters, len(testCase.steps[i].(ChangePeerV2Enter).DemoteVoters)) for j, p := range testCase.steps[i].(ChangePeerV2Enter).PromoteLearners { - suite.Equal(p.ToStore, step.PromoteLearners[j].ToStore) + re.Equal(p.ToStore, step.PromoteLearners[j].ToStore) } for j, d := range testCase.steps[i].(ChangePeerV2Enter).DemoteVoters { - suite.Equal(d.ToStore, step.DemoteVoters[j].ToStore) + re.Equal(d.ToStore, step.DemoteVoters[j].ToStore) } case ChangePeerV2Leave: - suite.Len(step.PromoteLearners, len(testCase.steps[i].(ChangePeerV2Leave).PromoteLearners)) - suite.Len(step.DemoteVoters, len(testCase.steps[i].(ChangePeerV2Leave).DemoteVoters)) + re.Len(step.PromoteLearners, len(testCase.steps[i].(ChangePeerV2Leave).PromoteLearners)) + re.Len(step.DemoteVoters, len(testCase.steps[i].(ChangePeerV2Leave).DemoteVoters)) for j, p := range testCase.steps[i].(ChangePeerV2Leave).PromoteLearners { - suite.Equal(p.ToStore, step.PromoteLearners[j].ToStore) + re.Equal(p.ToStore, step.PromoteLearners[j].ToStore) } for j, d := range testCase.steps[i].(ChangePeerV2Leave).DemoteVoters { - suite.Equal(d.ToStore, step.DemoteVoters[j].ToStore) + re.Equal(d.ToStore, step.DemoteVoters[j].ToStore) } } } @@ -588,26 +592,27 @@ func (suite *operatorBuilderTestSuite) TestBuild() { } func (suite *operatorBuilderTestSuite) TestTargetUnhealthyPeer() { + re := suite.Require() p := &metapb.Peer{Id: 2, StoreId: 2, Role: metapb.PeerRole_Learner} region := core.NewRegionInfo(&metapb.Region{Id: 1, Peers: []*metapb.Peer{{Id: 1, StoreId: 1}, p}}, &metapb.Peer{Id: 1, StoreId: 1}, core.WithPendingPeers([]*metapb.Peer{p})) builder := NewBuilder("test", suite.cluster, region) builder.PromoteLearner(2) - suite.Error(builder.err) + re.Error(builder.err) region = core.NewRegionInfo(&metapb.Region{Id: 1, Peers: []*metapb.Peer{{Id: 1, StoreId: 1}, p}}, &metapb.Peer{Id: 1, StoreId: 1}, core.WithDownPeers([]*pdpb.PeerStats{{Peer: p}})) builder = NewBuilder("test", suite.cluster, region) builder.PromoteLearner(2) - suite.Error(builder.err) + re.Error(builder.err) p = &metapb.Peer{Id: 2, StoreId: 2, Role: metapb.PeerRole_Voter} region = core.NewRegionInfo(&metapb.Region{Id: 1, Peers: []*metapb.Peer{{Id: 1, StoreId: 1}, p}}, &metapb.Peer{Id: 1, StoreId: 1}, core.WithPendingPeers([]*metapb.Peer{p})) builder = NewBuilder("test", suite.cluster, region) builder.SetLeader(2) - suite.Error(builder.err) + re.Error(builder.err) region = core.NewRegionInfo(&metapb.Region{Id: 1, Peers: []*metapb.Peer{{Id: 1, StoreId: 1}, p}}, &metapb.Peer{Id: 1, StoreId: 1}, core.WithDownPeers([]*pdpb.PeerStats{{Peer: p}})) builder = NewBuilder("test", suite.cluster, region) builder.SetLeader(2) - suite.Error(builder.err) + re.Error(builder.err) } diff --git a/pkg/schedule/placement/rule_manager_test.go b/pkg/schedule/placement/rule_manager_test.go index c0987f6dd33..0539e935113 100644 --- a/pkg/schedule/placement/rule_manager_test.go +++ b/pkg/schedule/placement/rule_manager_test.go @@ -161,11 +161,11 @@ func TestSaveLoad(t *testing.T) { err := m2.Initialize(3, []string{"no", "labels"}, "") re.NoError(err) re.Len(m2.GetAllRules(), 3) - re.Equal(rules[0].String(), m2.GetRule(DefaultGroupID, DefaultRuleID).String()) - re.Equal(rules[1].String(), m2.GetRule("foo", "baz").String()) - re.Equal(rules[2].String(), m2.GetRule("foo", "bar").String()) - re.Equal(manager.GetRulesCount(), 3) - re.Equal(manager.GetGroupsCount(), 2) + re.Equal(m2.GetRule(DefaultGroupID, DefaultRuleID).String(), rules[0].String()) + re.Equal(m2.GetRule("foo", "baz").String(), rules[1].String()) + re.Equal(m2.GetRule("foo", "bar").String(), rules[2].String()) + re.Equal(3, manager.GetRulesCount()) + re.Equal(2, manager.GetGroupsCount()) } func TestSetAfterGet(t *testing.T) { diff --git a/pkg/schedule/plan/balance_plan_test.go b/pkg/schedule/plan/balance_plan_test.go index 59ad637d5c8..59f2acc689a 100644 --- a/pkg/schedule/plan/balance_plan_test.go +++ b/pkg/schedule/plan/balance_plan_test.go @@ -114,6 +114,7 @@ func (suite *balanceSchedulerPlanAnalyzeTestSuite) TearDownSuite() { } func (suite *balanceSchedulerPlanAnalyzeTestSuite) TestAnalyzerResult1() { + re := suite.Require() plans := make([]Plan, 0) plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[4], Step: 2, Target: suite.stores[0], Status: NewStatus(StatusStoreScoreDisallowed)}) plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[4], Step: 2, Target: suite.stores[1], Status: NewStatus(StatusStoreScoreDisallowed)}) @@ -141,9 +142,9 @@ func (suite *balanceSchedulerPlanAnalyzeTestSuite) TestAnalyzerResult1() { plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[0], Step: 2, Target: suite.stores[3], Status: NewStatus(StatusStoreNotMatchRule)}) plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[0], Step: 2, Target: suite.stores[4], Status: NewStatus(StatusStoreScoreDisallowed)}) statuses, isNormal, err := BalancePlanSummary(plans) - suite.NoError(err) - suite.True(isNormal) - suite.True(suite.check(statuses, + re.NoError(err) + re.True(isNormal) + re.True(suite.check(statuses, map[uint64]*Status{ 1: NewStatus(StatusStoreNotMatchRule), 2: NewStatus(StatusStoreNotMatchRule), @@ -154,6 +155,7 @@ func (suite *balanceSchedulerPlanAnalyzeTestSuite) TestAnalyzerResult1() { } func (suite *balanceSchedulerPlanAnalyzeTestSuite) TestAnalyzerResult2() { + re := suite.Require() plans := make([]Plan, 0) plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[4], Step: 0, Status: NewStatus(StatusStoreDown)}) plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[3], Step: 0, Status: NewStatus(StatusStoreDown)}) @@ -161,9 +163,9 @@ func (suite *balanceSchedulerPlanAnalyzeTestSuite) TestAnalyzerResult2() { plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[1], Step: 0, Status: NewStatus(StatusStoreDown)}) plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[0], Step: 0, Status: NewStatus(StatusStoreDown)}) statuses, isNormal, err := BalancePlanSummary(plans) - suite.NoError(err) - suite.False(isNormal) - suite.True(suite.check(statuses, + re.NoError(err) + re.False(isNormal) + re.True(suite.check(statuses, map[uint64]*Status{ 1: NewStatus(StatusStoreDown), 2: NewStatus(StatusStoreDown), @@ -174,6 +176,7 @@ func (suite *balanceSchedulerPlanAnalyzeTestSuite) TestAnalyzerResult2() { } func (suite *balanceSchedulerPlanAnalyzeTestSuite) TestAnalyzerResult3() { + re := suite.Require() plans := make([]Plan, 0) plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[4], Step: 0, Status: NewStatus(StatusStoreDown)}) plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[3], Region: suite.regions[0], Step: 1, Status: NewStatus(StatusRegionNotMatchRule)}) @@ -181,9 +184,9 @@ func (suite *balanceSchedulerPlanAnalyzeTestSuite) TestAnalyzerResult3() { plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[1], Region: suite.regions[1], Step: 1, Status: NewStatus(StatusRegionNotMatchRule)}) plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[0], Region: suite.regions[1], Step: 1, Status: NewStatus(StatusRegionNotMatchRule)}) statuses, isNormal, err := BalancePlanSummary(plans) - suite.NoError(err) - suite.False(isNormal) - suite.True(suite.check(statuses, + re.NoError(err) + re.False(isNormal) + re.True(suite.check(statuses, map[uint64]*Status{ 1: NewStatus(StatusRegionNotMatchRule), 2: NewStatus(StatusRegionNotMatchRule), @@ -193,6 +196,7 @@ func (suite *balanceSchedulerPlanAnalyzeTestSuite) TestAnalyzerResult3() { } func (suite *balanceSchedulerPlanAnalyzeTestSuite) TestAnalyzerResult4() { + re := suite.Require() plans := make([]Plan, 0) plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[4], Step: 0, Status: NewStatus(StatusStoreDown)}) plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[3], Region: suite.regions[0], Step: 1, Status: NewStatus(StatusRegionNotMatchRule)}) @@ -208,9 +212,9 @@ func (suite *balanceSchedulerPlanAnalyzeTestSuite) TestAnalyzerResult4() { plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[0], Target: suite.stores[3], Step: 2, Status: NewStatus(StatusStoreNotMatchRule)}) plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[0], Target: suite.stores[4], Step: 2, Status: NewStatus(StatusStoreDown)}) statuses, isNormal, err := BalancePlanSummary(plans) - suite.NoError(err) - suite.False(isNormal) - suite.True(suite.check(statuses, + re.NoError(err) + re.False(isNormal) + re.True(suite.check(statuses, map[uint64]*Status{ 1: NewStatus(StatusStoreAlreadyHasPeer), 2: NewStatus(StatusStoreAlreadyHasPeer), @@ -221,6 +225,7 @@ func (suite *balanceSchedulerPlanAnalyzeTestSuite) TestAnalyzerResult4() { } func (suite *balanceSchedulerPlanAnalyzeTestSuite) TestAnalyzerResult5() { + re := suite.Require() plans := make([]Plan, 0) plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[4], Step: 0, Status: NewStatus(StatusStoreRemoveLimitThrottled)}) plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[3], Region: suite.regions[0], Step: 1, Status: NewStatus(StatusRegionNotMatchRule)}) @@ -234,9 +239,9 @@ func (suite *balanceSchedulerPlanAnalyzeTestSuite) TestAnalyzerResult5() { plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[0], Target: suite.stores[2], Step: 2, Status: NewStatus(StatusStoreNotMatchRule)}) plans = append(plans, &BalanceSchedulerPlan{Source: suite.stores[0], Target: suite.stores[3], Step: 2, Status: NewStatus(StatusStoreNotMatchRule)}) statuses, isNormal, err := BalancePlanSummary(plans) - suite.NoError(err) - suite.False(isNormal) - suite.True(suite.check(statuses, + re.NoError(err) + re.False(isNormal) + re.True(suite.check(statuses, map[uint64]*Status{ 1: NewStatus(StatusStoreAlreadyHasPeer), 2: NewStatus(StatusStoreAlreadyHasPeer), @@ -247,6 +252,7 @@ func (suite *balanceSchedulerPlanAnalyzeTestSuite) TestAnalyzerResult5() { } func (suite *balanceSchedulerPlanAnalyzeTestSuite) TestAnalyzerResult6() { + re := suite.Require() basePlan := NewBalanceSchedulerPlan() collector := NewCollector(basePlan) collector.Collect(SetResourceWithStep(suite.stores[0], 2), SetStatus(NewStatus(StatusStoreDown))) @@ -258,9 +264,9 @@ func (suite *balanceSchedulerPlanAnalyzeTestSuite) TestAnalyzerResult6() { basePlan.Step++ collector.Collect(SetResource(suite.regions[0]), SetStatus(NewStatus(StatusRegionNoLeader))) statuses, isNormal, err := BalancePlanSummary(collector.GetPlans()) - suite.NoError(err) - suite.False(isNormal) - suite.True(suite.check(statuses, + re.NoError(err) + re.False(isNormal) + re.True(suite.check(statuses, map[uint64]*Status{ 1: NewStatus(StatusStoreDown), 2: NewStatus(StatusStoreDown), diff --git a/pkg/schedule/scatter/region_scatterer_test.go b/pkg/schedule/scatter/region_scatterer_test.go index 70517d23fee..af41ed04b76 100644 --- a/pkg/schedule/scatter/region_scatterer_test.go +++ b/pkg/schedule/scatter/region_scatterer_test.go @@ -350,7 +350,7 @@ func TestSomeStoresFilteredScatterGroupInConcurrency(t *testing.T) { // prevent store from being disconnected tc.SetStoreLastHeartbeatInterval(i, 40*time.Minute) } - re.Equal(tc.GetStore(uint64(6)).IsDisconnected(), true) + re.True(tc.GetStore(uint64(6)).IsDisconnected()) scatterer := NewRegionScatterer(ctx, tc, oc, tc.AddSuspectRegions) var wg sync.WaitGroup for j := 0; j < 10; j++ { @@ -466,7 +466,7 @@ func TestScatterForManyRegion(t *testing.T) { re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/schedule/scatter/scatterHbStreamsDrain", `return(true)`)) scatterer.scatterRegions(regions, failures, group, 3, false) re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/schedule/scatter/scatterHbStreamsDrain")) - re.Len(failures, 0) + re.Empty(failures) } func TestScattersGroup(t *testing.T) { diff --git a/pkg/schedule/schedulers/balance_benchmark_test.go b/pkg/schedule/schedulers/balance_benchmark_test.go index 694d5edb658..2d7befd27af 100644 --- a/pkg/schedule/schedulers/balance_benchmark_test.go +++ b/pkg/schedule/schedulers/balance_benchmark_test.go @@ -163,7 +163,7 @@ func BenchmarkPlacementRule(b *testing.B) { ops, plans = sc.Schedule(tc, false) } b.StopTimer() - re.Len(plans, 0) + re.Empty(plans) re.Len(ops, 1) re.Contains(ops[0].String(), "to [191]") } diff --git a/pkg/schedule/schedulers/balance_test.go b/pkg/schedule/schedulers/balance_test.go index 54fe8ff489b..dafe810b2b7 100644 --- a/pkg/schedule/schedulers/balance_test.go +++ b/pkg/schedule/schedulers/balance_test.go @@ -237,9 +237,10 @@ func TestBalanceLeaderSchedulerTestSuite(t *testing.T) { } func (suite *balanceLeaderSchedulerTestSuite) SetupTest() { + re := suite.Require() suite.cancel, suite.conf, suite.tc, suite.oc = prepareSchedulersTest() lb, err := CreateScheduler(BalanceLeaderType, suite.oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(BalanceLeaderType, []string{"", ""})) - suite.NoError(err) + re.NoError(err) suite.lb = lb } @@ -560,6 +561,7 @@ func (suite *balanceLeaderRangeSchedulerTestSuite) TearDownTest() { } func (suite *balanceLeaderRangeSchedulerTestSuite) TestSingleRangeBalance() { + re := suite.Require() // Stores: 1 2 3 4 // Leaders: 10 10 10 10 // Weight: 0.5 0.9 1 2 @@ -573,36 +575,36 @@ func (suite *balanceLeaderRangeSchedulerTestSuite) TestSingleRangeBalance() { suite.tc.UpdateStoreLeaderWeight(4, 2) suite.tc.AddLeaderRegionWithRange(1, "a", "g", 1, 2, 3, 4) lb, err := CreateScheduler(BalanceLeaderType, suite.oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(BalanceLeaderType, []string{"", ""})) - suite.NoError(err) + re.NoError(err) ops, _ := lb.Schedule(suite.tc, false) - suite.NotEmpty(ops) - suite.Len(ops, 1) - suite.Len(ops[0].Counters, 1) - suite.Len(ops[0].FinishedCounters, 1) + re.NotEmpty(ops) + re.Len(ops, 1) + re.Len(ops[0].Counters, 1) + re.Len(ops[0].FinishedCounters, 1) lb, err = CreateScheduler(BalanceLeaderType, suite.oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(BalanceLeaderType, []string{"h", "n"})) - suite.NoError(err) + re.NoError(err) ops, _ = lb.Schedule(suite.tc, false) - suite.Empty(ops) + re.Empty(ops) lb, err = CreateScheduler(BalanceLeaderType, suite.oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(BalanceLeaderType, []string{"b", "f"})) - suite.NoError(err) + re.NoError(err) ops, _ = lb.Schedule(suite.tc, false) - suite.Empty(ops) + re.Empty(ops) lb, err = CreateScheduler(BalanceLeaderType, suite.oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(BalanceLeaderType, []string{"", "a"})) - suite.NoError(err) + re.NoError(err) ops, _ = lb.Schedule(suite.tc, false) - suite.Empty(ops) + re.Empty(ops) lb, err = CreateScheduler(BalanceLeaderType, suite.oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(BalanceLeaderType, []string{"g", ""})) - suite.NoError(err) + re.NoError(err) ops, _ = lb.Schedule(suite.tc, false) - suite.Empty(ops) + re.Empty(ops) lb, err = CreateScheduler(BalanceLeaderType, suite.oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(BalanceLeaderType, []string{"", "f"})) - suite.NoError(err) + re.NoError(err) ops, _ = lb.Schedule(suite.tc, false) - suite.Empty(ops) + re.Empty(ops) lb, err = CreateScheduler(BalanceLeaderType, suite.oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(BalanceLeaderType, []string{"b", ""})) - suite.NoError(err) + re.NoError(err) ops, _ = lb.Schedule(suite.tc, false) - suite.Empty(ops) + re.Empty(ops) } func (suite *balanceLeaderRangeSchedulerTestSuite) TestMultiRangeBalance() { diff --git a/pkg/schedule/schedulers/balance_witness_test.go b/pkg/schedule/schedulers/balance_witness_test.go index 59bf04c2303..9bde7e33438 100644 --- a/pkg/schedule/schedulers/balance_witness_test.go +++ b/pkg/schedule/schedulers/balance_witness_test.go @@ -125,7 +125,7 @@ func (suite *balanceWitnessSchedulerTestSuite) TestTransferWitnessOut() { } } } - suite.Equal(3, len(regions)) + suite.Len(regions, 3) for _, count := range targets { suite.Zero(count) } diff --git a/pkg/schedule/schedulers/evict_leader_test.go b/pkg/schedule/schedulers/evict_leader_test.go index d804561f11c..a91b1c3c937 100644 --- a/pkg/schedule/schedulers/evict_leader_test.go +++ b/pkg/schedule/schedulers/evict_leader_test.go @@ -97,7 +97,7 @@ func TestConfigClone(t *testing.T) { con3 := con2.Clone() con3.StoreIDWithRanges[1], _ = getKeyRanges([]string{"a", "b", "c", "d"}) re.Empty(emptyConf.getKeyRangesByID(1)) - re.False(len(con3.getRanges(1)) == len(con2.getRanges(1))) + re.NotEqual(len(con3.getRanges(1)), len(con2.getRanges(1))) con4 := con3.Clone() re.True(bytes.Equal(con4.StoreIDWithRanges[1][0].StartKey, con3.StoreIDWithRanges[1][0].StartKey)) diff --git a/pkg/schedule/schedulers/evict_slow_trend_test.go b/pkg/schedule/schedulers/evict_slow_trend_test.go index 65a70962a20..aed41e83ecd 100644 --- a/pkg/schedule/schedulers/evict_slow_trend_test.go +++ b/pkg/schedule/schedulers/evict_slow_trend_test.go @@ -100,10 +100,10 @@ func (suite *evictSlowTrendTestSuite) TestEvictSlowTrendBasicFuncs() { // Pop captured store 1 and mark it has recovered. time.Sleep(50 * time.Millisecond) suite.Equal(es2.conf.popCandidate(true), store.GetID()) - suite.True(es2.conf.evictCandidate == (slowCandidate{})) + suite.Equal(slowCandidate{}, es2.conf.evictCandidate) es2.conf.markCandidateRecovered() lastCapturedCandidate = es2.conf.lastCapturedCandidate() - suite.True(lastCapturedCandidate.recoverTS.Compare(recoverTS) > 0) + suite.Greater(lastCapturedCandidate.recoverTS.Compare(recoverTS), 0) suite.Equal(lastCapturedCandidate.storeID, store.GetID()) // Test capture another store 2 diff --git a/pkg/schedule/schedulers/hot_region_test.go b/pkg/schedule/schedulers/hot_region_test.go index 6e7208e4251..5b1bc3db4b4 100644 --- a/pkg/schedule/schedulers/hot_region_test.go +++ b/pkg/schedule/schedulers/hot_region_test.go @@ -180,11 +180,11 @@ func checkGCPendingOpInfos(re *require.Assertions, enablePlacementRules bool) { kind := hb.regionPendings[regionID].op.Kind() switch typ { case transferLeader: - re.True(kind&operator.OpLeader != 0) - re.True(kind&operator.OpRegion == 0) + re.NotZero(kind & operator.OpLeader) + re.Zero(kind & operator.OpRegion) case movePeer: - re.True(kind&operator.OpLeader == 0) - re.True(kind&operator.OpRegion != 0) + re.Zero(kind & operator.OpLeader) + re.NotZero(kind & operator.OpRegion) } } } @@ -257,7 +257,7 @@ func TestSplitIfRegionTooHot(t *testing.T) { re.Equal(expectOp.Kind(), ops[0].Kind()) ops, _ = hb.Schedule(tc, false) - re.Len(ops, 0) + re.Empty(ops) tc.UpdateStorageWrittenBytes(1, 6*units.MiB*utils.StoreHeartBeatReportInterval) tc.UpdateStorageWrittenBytes(2, 1*units.MiB*utils.StoreHeartBeatReportInterval) @@ -276,7 +276,7 @@ func TestSplitIfRegionTooHot(t *testing.T) { re.Equal(operator.OpSplit, ops[0].Kind()) ops, _ = hb.Schedule(tc, false) - re.Len(ops, 0) + re.Empty(ops) } func TestSplitBucketsBySize(t *testing.T) { @@ -319,10 +319,10 @@ func TestSplitBucketsBySize(t *testing.T) { region.UpdateBuckets(b, region.GetBuckets()) ops := solve.createSplitOperator([]*core.RegionInfo{region}, bySize) if data.splitKeys == nil { - re.Equal(0, len(ops)) + re.Empty(ops) continue } - re.Equal(1, len(ops)) + re.Len(ops, 1) op := ops[0] re.Equal(splitHotReadBuckets, op.Desc()) @@ -380,10 +380,10 @@ func TestSplitBucketsByLoad(t *testing.T) { time.Sleep(time.Millisecond * 10) ops := solve.createSplitOperator([]*core.RegionInfo{region}, byLoad) if data.splitKeys == nil { - re.Equal(0, len(ops)) + re.Empty(ops) continue } - re.Equal(1, len(ops)) + re.Len(ops, 1) op := ops[0] re.Equal(splitHotReadBuckets, op.Desc()) @@ -731,7 +731,7 @@ func TestHotWriteRegionScheduleByteRateOnlyWithTiFlash(t *testing.T) { loadsEqual( hb.stLoadInfos[writeLeader][1].LoadPred.Expect.Loads, []float64{hotRegionBytesSum / allowLeaderTiKVCount, hotRegionKeysSum / allowLeaderTiKVCount, tikvQuerySum / allowLeaderTiKVCount})) - re.True(tikvQuerySum != hotRegionQuerySum) + re.NotEqual(tikvQuerySum, hotRegionQuerySum) re.True( loadsEqual( hb.stLoadInfos[writePeer][1].LoadPred.Expect.Loads, @@ -1574,7 +1574,7 @@ func TestHotReadWithEvictLeaderScheduler(t *testing.T) { // two dim are both enough uniform among three stores tc.SetStoreEvictLeader(4, true) ops, _ = hb.Schedule(tc, false) - re.Len(ops, 0) + re.Empty(ops) clearPendingInfluence(hb.(*hotScheduler)) } diff --git a/pkg/schedule/schedulers/hot_region_v2_test.go b/pkg/schedule/schedulers/hot_region_v2_test.go index d11ac44dde9..f5e21e02981 100644 --- a/pkg/schedule/schedulers/hot_region_v2_test.go +++ b/pkg/schedule/schedulers/hot_region_v2_test.go @@ -309,7 +309,7 @@ func TestSkipUniformStore(t *testing.T) { // when there is uniform store filter, not schedule stddevThreshold = 0.1 ops, _ = hb.Schedule(tc, false) - re.Len(ops, 0) + re.Empty(ops) clearPendingInfluence(hb.(*hotScheduler)) // Case2: the first dim is enough uniform, we should schedule the second dim @@ -380,7 +380,7 @@ func TestHotReadRegionScheduleWithSmallHotRegion(t *testing.T) { ops = checkHotReadRegionScheduleWithSmallHotRegion(re, highLoad, lowLoad, emptyFunc) re.Len(ops, 1) ops = checkHotReadRegionScheduleWithSmallHotRegion(re, lowLoad, highLoad, emptyFunc) - re.Len(ops, 0) + re.Empty(ops) // Case3: If there is larger hot region, we will schedule it. hotRegionID := uint64(100) @@ -418,7 +418,7 @@ func TestHotReadRegionScheduleWithSmallHotRegion(t *testing.T) { tc.AddRegionWithReadInfo(hotRegionID+1, 2, bigHotRegionByte, 0, bigHotRegionQuery, utils.StoreHeartBeatReportInterval, []uint64{1, 3}) tc.AddRegionWithReadInfo(hotRegionID+1, 1, bigHotRegionByte, 0, bigHotRegionQuery, utils.StoreHeartBeatReportInterval, []uint64{2, 3}) }) - re.Len(ops, 0) + re.Empty(ops) topnPosition = origin // Case7: If there are more than topnPosition hot regions, but them are pending, @@ -430,7 +430,7 @@ func TestHotReadRegionScheduleWithSmallHotRegion(t *testing.T) { tc.AddRegionWithReadInfo(hotRegionID+1, 1, bigHotRegionByte, 0, bigHotRegionQuery, utils.StoreHeartBeatReportInterval, []uint64{2, 3}) hb.regionPendings[hotRegionID+1] = &pendingInfluence{} }) - re.Len(ops, 0) + re.Empty(ops) topnPosition = origin } diff --git a/pkg/schedule/schedulers/scheduler_test.go b/pkg/schedule/schedulers/scheduler_test.go index 57f1fcf1e3f..77c190ad943 100644 --- a/pkg/schedule/schedulers/scheduler_test.go +++ b/pkg/schedule/schedulers/scheduler_test.go @@ -484,7 +484,7 @@ func TestBalanceLeaderWithConflictRule(t *testing.T) { } for _, testCase := range testCases { - re.Nil(tc.SetRule(testCase.rule)) + re.NoError(tc.SetRule(testCase.rule)) ops, _ := lb.Schedule(tc, false) if testCase.schedule { re.Len(ops, 1) diff --git a/pkg/storage/storage_gc_test.go b/pkg/storage/storage_gc_test.go index 141777d441e..77f7c7dbf65 100644 --- a/pkg/storage/storage_gc_test.go +++ b/pkg/storage/storage_gc_test.go @@ -93,7 +93,7 @@ func TestLoadMinServiceSafePoint(t *testing.T) { // gc_worker service safepoint will not be removed. ssp, err := storage.LoadMinServiceSafePointV2(testKeyspaceID, currentTime.Add(5000*time.Second)) re.NoError(err) - re.Equal(ssp.ServiceID, endpoint.GCWorkerServiceSafePointID) + re.Equal(endpoint.GCWorkerServiceSafePointID, ssp.ServiceID) re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/storage/endpoint/removeExpiredKeys")) } diff --git a/pkg/tso/keyspace_group_manager_test.go b/pkg/tso/keyspace_group_manager_test.go index 0c1b017d7aa..54a1adc6b34 100644 --- a/pkg/tso/keyspace_group_manager_test.go +++ b/pkg/tso/keyspace_group_manager_test.go @@ -988,7 +988,7 @@ func (suite *keyspaceGroupManagerTestSuite) TestUpdateKeyspaceGroupMembership() re.Equal(len(keyspaces), len(newGroup.Keyspaces)) for i := 0; i < len(newGroup.Keyspaces); i++ { if i > 0 { - re.True(newGroup.Keyspaces[i-1] < newGroup.Keyspaces[i]) + re.Less(newGroup.Keyspaces[i-1], newGroup.Keyspaces[i]) } } } diff --git a/pkg/unsaferecovery/unsafe_recovery_controller_test.go b/pkg/unsaferecovery/unsafe_recovery_controller_test.go index 44c4e4a7b4d..956b9b8729c 100644 --- a/pkg/unsaferecovery/unsafe_recovery_controller_test.go +++ b/pkg/unsaferecovery/unsafe_recovery_controller_test.go @@ -1158,7 +1158,7 @@ func TestExecutionTimeout(t *testing.T) { re.Equal(Failed, recoveryController.GetStage()) output := recoveryController.Show() - re.Equal(len(output), 3) + re.Len(output, 3) re.Contains(output[1].Details[0], "triggered by error: Exceeds timeout") } @@ -1768,7 +1768,7 @@ func TestEpochComparsion(t *testing.T) { cluster.PutStore(store) } recoveryController := NewController(cluster) - re.Nil(recoveryController.RemoveFailedStores(map[uint64]struct{}{ + re.NoError(recoveryController.RemoveFailedStores(map[uint64]struct{}{ 2: {}, 3: {}, }, 60, false)) @@ -1829,7 +1829,7 @@ func TestEpochComparsion(t *testing.T) { if expect, ok := expects[storeID]; ok { re.Equal(expect.PeerReports, report.PeerReports) } else { - re.Empty(len(report.PeerReports)) + re.Empty(report.PeerReports) } } } diff --git a/pkg/utils/etcdutil/etcdutil_test.go b/pkg/utils/etcdutil/etcdutil_test.go index 861a57cef13..d8b38e7b045 100644 --- a/pkg/utils/etcdutil/etcdutil_test.go +++ b/pkg/utils/etcdutil/etcdutil_test.go @@ -154,11 +154,11 @@ func TestInitClusterID(t *testing.T) { // Get any cluster key to parse the cluster ID. resp, err := EtcdKVGet(client, pdClusterIDPath) re.NoError(err) - re.Equal(0, len(resp.Kvs)) + re.Empty(resp.Kvs) clusterID, err := InitClusterID(client, pdClusterIDPath) re.NoError(err) - re.NotEqual(0, clusterID) + re.NotZero(clusterID) clusterID1, err := InitClusterID(client, pdClusterIDPath) re.NoError(err) @@ -375,15 +375,15 @@ func TestLoopWatcherTestSuite(t *testing.T) { } func (suite *loopWatcherTestSuite) SetupSuite() { + re := suite.Require() var err error - t := suite.T() suite.ctx, suite.cancel = context.WithCancel(context.Background()) suite.cleans = make([]func(), 0) // Start a etcd server and create a client with etcd1 as endpoint. - suite.config = newTestSingleConfig(t) - suite.startEtcd() + suite.config = newTestSingleConfig(suite.T()) + suite.startEtcd(re) suite.client, err = CreateEtcdClient(nil, suite.config.LCUrls) - suite.NoError(err) + re.NoError(err) suite.cleans = append(suite.cleans, func() { suite.client.Close() }) @@ -398,6 +398,7 @@ func (suite *loopWatcherTestSuite) TearDownSuite() { } func (suite *loopWatcherTestSuite) TestLoadWithoutKey() { + re := suite.Require() cache := struct { syncutil.RWMutex data map[string]struct{} @@ -422,13 +423,14 @@ func (suite *loopWatcherTestSuite) TestLoadWithoutKey() { ) watcher.StartWatchLoop() err := watcher.WaitLoad() - suite.NoError(err) // although no key, watcher returns no error + re.NoError(err) // although no key, watcher returns no error cache.RLock() defer cache.RUnlock() - suite.Len(cache.data, 0) + suite.Empty(cache.data) } func (suite *loopWatcherTestSuite) TestCallBack() { + re := suite.Require() cache := struct { syncutil.RWMutex data map[string]struct{} @@ -466,35 +468,36 @@ func (suite *loopWatcherTestSuite) TestCallBack() { ) watcher.StartWatchLoop() err := watcher.WaitLoad() - suite.NoError(err) + re.NoError(err) // put 10 keys for i := 0; i < 10; i++ { - suite.put(fmt.Sprintf("TestCallBack%d", i), "") + suite.put(re, fmt.Sprintf("TestCallBack%d", i), "") } time.Sleep(time.Second) cache.RLock() - suite.Len(cache.data, 10) + re.Len(cache.data, 10) cache.RUnlock() // delete 10 keys for i := 0; i < 10; i++ { key := fmt.Sprintf("TestCallBack%d", i) _, err = suite.client.Delete(suite.ctx, key) - suite.NoError(err) + re.NoError(err) } time.Sleep(time.Second) cache.RLock() - suite.Empty(cache.data) + re.Empty(cache.data) cache.RUnlock() } func (suite *loopWatcherTestSuite) TestWatcherLoadLimit() { + re := suite.Require() for count := 1; count < 10; count++ { for limit := 0; limit < 10; limit++ { ctx, cancel := context.WithCancel(suite.ctx) for i := 0; i < count; i++ { - suite.put(fmt.Sprintf("TestWatcherLoadLimit%d", i), "") + suite.put(re, fmt.Sprintf("TestWatcherLoadLimit%d", i), "") } cache := struct { syncutil.RWMutex @@ -525,9 +528,9 @@ func (suite *loopWatcherTestSuite) TestWatcherLoadLimit() { ) watcher.StartWatchLoop() err := watcher.WaitLoad() - suite.NoError(err) + re.NoError(err) cache.RLock() - suite.Len(cache.data, count) + re.Len(cache.data, count) cache.RUnlock() cancel() } @@ -535,6 +538,7 @@ func (suite *loopWatcherTestSuite) TestWatcherLoadLimit() { } func (suite *loopWatcherTestSuite) TestWatcherBreak() { + re := suite.Require() cache := struct { syncutil.RWMutex data string @@ -568,51 +572,51 @@ func (suite *loopWatcherTestSuite) TestWatcherBreak() { watcher.watchChangeRetryInterval = 100 * time.Millisecond watcher.StartWatchLoop() err := watcher.WaitLoad() - suite.NoError(err) + re.NoError(err) checkCache("") // we use close client and update client in failpoint to simulate the network error and recover - failpoint.Enable("github.com/tikv/pd/pkg/utils/etcdutil/updateClient", "return(true)") + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/utils/etcdutil/updateClient", "return(true)")) // Case1: restart the etcd server suite.etcd.Close() - suite.startEtcd() - suite.put("TestWatcherBreak", "0") + suite.startEtcd(re) + suite.put(re, "TestWatcherBreak", "0") checkCache("0") suite.etcd.Server.Stop() time.Sleep(DefaultRequestTimeout) suite.etcd.Close() - suite.startEtcd() - suite.put("TestWatcherBreak", "1") + suite.startEtcd(re) + suite.put(re, "TestWatcherBreak", "1") checkCache("1") // Case2: close the etcd client and put a new value after watcher restarts suite.client.Close() suite.client, err = CreateEtcdClient(nil, suite.config.LCUrls) - suite.NoError(err) + re.NoError(err) watcher.updateClientCh <- suite.client - suite.put("TestWatcherBreak", "2") + suite.put(re, "TestWatcherBreak", "2") checkCache("2") // Case3: close the etcd client and put a new value before watcher restarts suite.client.Close() suite.client, err = CreateEtcdClient(nil, suite.config.LCUrls) - suite.NoError(err) - suite.put("TestWatcherBreak", "3") + re.NoError(err) + suite.put(re, "TestWatcherBreak", "3") watcher.updateClientCh <- suite.client checkCache("3") // Case4: close the etcd client and put a new value with compact suite.client.Close() suite.client, err = CreateEtcdClient(nil, suite.config.LCUrls) - suite.NoError(err) - suite.put("TestWatcherBreak", "4") + re.NoError(err) + suite.put(re, "TestWatcherBreak", "4") resp, err := EtcdKVGet(suite.client, "TestWatcherBreak") - suite.NoError(err) + re.NoError(err) revision := resp.Header.Revision resp2, err := suite.etcd.Server.Compact(suite.ctx, &etcdserverpb.CompactionRequest{Revision: revision}) - suite.NoError(err) - suite.Equal(revision, resp2.Header.Revision) + re.NoError(err) + re.Equal(revision, resp2.Header.Revision) watcher.updateClientCh <- suite.client checkCache("4") @@ -623,7 +627,7 @@ func (suite *loopWatcherTestSuite) TestWatcherBreak() { watcher.ForceLoad() checkCache("4") - failpoint.Disable("github.com/tikv/pd/pkg/utils/etcdutil/updateClient") + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/utils/etcdutil/updateClient")) } func (suite *loopWatcherTestSuite) TestWatcherRequestProgress() { @@ -669,9 +673,9 @@ func (suite *loopWatcherTestSuite) TestWatcherRequestProgress() { checkWatcherRequestProgress(true) } -func (suite *loopWatcherTestSuite) startEtcd() { +func (suite *loopWatcherTestSuite) startEtcd(re *require.Assertions) { etcd1, err := embed.StartEtcd(suite.config) - suite.NoError(err) + re.NoError(err) suite.etcd = etcd1 <-etcd1.Server.ReadyNotify() suite.cleans = append(suite.cleans, func() { @@ -679,11 +683,11 @@ func (suite *loopWatcherTestSuite) startEtcd() { }) } -func (suite *loopWatcherTestSuite) put(key, value string) { +func (suite *loopWatcherTestSuite) put(re *require.Assertions, key, value string) { kv := clientv3.NewKV(suite.client) _, err := kv.Put(suite.ctx, key, value) - suite.NoError(err) + re.NoError(err) resp, err := kv.Get(suite.ctx, key) - suite.NoError(err) - suite.Equal(value, string(resp.Kvs[0].Value)) + re.NoError(err) + re.Equal(value, string(resp.Kvs[0].Value)) } diff --git a/pkg/utils/syncutil/lock_group_test.go b/pkg/utils/syncutil/lock_group_test.go index ff306983e05..897e6b777a6 100644 --- a/pkg/utils/syncutil/lock_group_test.go +++ b/pkg/utils/syncutil/lock_group_test.go @@ -60,14 +60,14 @@ func TestLockGroupWithRemoveEntryOnUnlock(t *testing.T) { for i := 0; i < maxID; i++ { group.Lock(uint32(i)) } - re.Equal(len(group.entries), maxID) + re.Len(group.entries, maxID) for i := 0; i < maxID; i++ { group.Unlock(uint32(i)) } wg.Wait() // Check that size of the lock group is limited. - re.Equal(len(group.entries), 0) + re.Empty(group.entries) } // mustSequentialUpdateSingle checks that for any given update, update is sequential. diff --git a/pkg/utils/typeutil/duration_test.go b/pkg/utils/typeutil/duration_test.go index 9a0beda7979..cff7c3cd66c 100644 --- a/pkg/utils/typeutil/duration_test.go +++ b/pkg/utils/typeutil/duration_test.go @@ -46,6 +46,6 @@ func TestDurationTOML(t *testing.T) { example := &example{} text := []byte(`interval = "1h1m1s"`) - re.Nil(toml.Unmarshal(text, example)) + re.NoError(toml.Unmarshal(text, example)) re.Equal(float64(60*60+60+1), example.Interval.Seconds()) } diff --git a/pkg/window/policy_test.go b/pkg/window/policy_test.go index 489c8428c9a..a81ef0ef82d 100644 --- a/pkg/window/policy_test.go +++ b/pkg/window/policy_test.go @@ -111,7 +111,7 @@ func TestRollingPolicy_AddWithTimespan(t *testing.T) { t.Logf("%+v", bkt) } - re.Equal(0, len(policy.window.buckets[0].Points)) + re.Empty(policy.window.buckets[0].Points) re.Equal(4, int(policy.window.buckets[1].Points[0])) re.Equal(2, int(policy.window.buckets[2].Points[0])) }) @@ -137,8 +137,8 @@ func TestRollingPolicy_AddWithTimespan(t *testing.T) { t.Logf("%+v", bkt) } - re.Equal(0, len(policy.window.buckets[0].Points)) + re.Zero(len(policy.window.buckets[0].Points)) re.Equal(4, int(policy.window.buckets[1].Points[0])) - re.Equal(0, len(policy.window.buckets[2].Points)) + re.Zero(len(policy.window.buckets[2].Points)) }) } diff --git a/pkg/window/window_test.go b/pkg/window/window_test.go index 0205aae47a3..f4df861fc2f 100644 --- a/pkg/window/window_test.go +++ b/pkg/window/window_test.go @@ -20,7 +20,6 @@ package window import ( "testing" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -33,7 +32,7 @@ func TestWindowResetWindow(t *testing.T) { } window.ResetWindow() for i := 0; i < opts.Size; i++ { - re.Equal(len(window.Bucket(i).Points), 0) + re.Empty(window.Bucket(i).Points) } } @@ -45,9 +44,9 @@ func TestWindowResetBucket(t *testing.T) { window.Append(i, 1.0) } window.ResetBucket(1) - re.Equal(len(window.Bucket(1).Points), 0) - re.Equal(window.Bucket(0).Points[0], float64(1.0)) - re.Equal(window.Bucket(2).Points[0], float64(1.0)) + re.Empty(window.Bucket(1).Points) + re.Equal(float64(1.0), window.Bucket(0).Points[0]) + re.Equal(float64(1.0), window.Bucket(2).Points[0]) } func TestWindowResetBuckets(t *testing.T) { @@ -59,7 +58,7 @@ func TestWindowResetBuckets(t *testing.T) { } window.ResetBuckets(0, 3) for i := 0; i < opts.Size; i++ { - re.Equal(len(window.Bucket(i).Points), 0) + re.Empty(window.Bucket(i).Points) } } @@ -74,28 +73,30 @@ func TestWindowAppend(t *testing.T) { window.Append(i, 2.0) } for i := 0; i < opts.Size; i++ { - re.Equal(window.Bucket(i).Points[0], float64(1.0)) + re.Equal(float64(1.0), window.Bucket(i).Points[0]) } for i := 1; i < opts.Size; i++ { - re.Equal(window.Bucket(i).Points[1], float64(2.0)) + re.Equal(float64(2.0), window.Bucket(i).Points[1]) } } func TestWindowAdd(t *testing.T) { + re := require.New(t) opts := Options{Size: 3} window := NewWindow(opts) window.Append(0, 1.0) window.Add(0, 1.0) - assert.Equal(t, window.Bucket(0).Points[0], float64(2.0)) + re.Equal(float64(2.0), window.Bucket(0).Points[0]) window = NewWindow(opts) window.Add(0, 1.0) window.Add(0, 1.0) - assert.Equal(t, window.Bucket(0).Points[0], float64(2.0)) + re.Equal(float64(2.0), window.Bucket(0).Points[0]) } func TestWindowSize(t *testing.T) { + re := require.New(t) opts := Options{Size: 3} window := NewWindow(opts) - assert.Equal(t, window.Size(), 3) + re.Equal(3, window.Size()) } diff --git a/scripts/check-test.sh b/scripts/check-test.sh deleted file mode 100755 index c3168066e3d..00000000000 --- a/scripts/check-test.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash - -# Check if there is any inefficient assert function usage in package. - -res=$(grep -rn --include=\*_test.go -E "(re|suite|require)\.(True|False)\((t, )?reflect\.DeepEqual\(" . | sort -u) \ - -if [ "$res" ]; then - echo "following packages use the inefficient assert function: please replace reflect.DeepEqual with require.Equal" - echo "$res" - exit 1 -fi - -res=$(grep -rn --include=\*_test.go -E "(re|suite|require)\.(True|False)\((t, )?strings\.Contains\(" . | sort -u) - -if [ "$res" ]; then - echo "following packages use the inefficient assert function: please replace strings.Contains with require.Contains" - echo "$res" - exit 1 -fi - -res=$(grep -rn --include=\*_test.go -E "(re|suite|require)\.(Nil|NotNil)\((t, )?(err|error)" . | sort -u) - -if [ "$res" ]; then - echo "following packages use the inefficient assert function: please replace require.Nil/NotNil with require.NoError/Error" - echo "$res" - exit 1 -fi - -res=$(grep -rn --include=\*_test.go -E "(re|suite|require)\.(Equal|NotEqual)\((t, )?(true|false)" . | sort -u) - -if [ "$res" ]; then - echo "following packages use the inefficient assert function: please replace require.Equal/NotEqual(true, xxx) with require.True/False" - echo "$res" - exit 1 -fi - -exit 0 diff --git a/server/api/admin_test.go b/server/api/admin_test.go index 76c5e729eb0..050aa9cfb32 100644 --- a/server/api/admin_test.go +++ b/server/api/admin_test.go @@ -60,6 +60,7 @@ func (suite *adminTestSuite) TearDownSuite() { } func (suite *adminTestSuite) TestDropRegion() { + re := suite.Require() cluster := suite.svr.GetRaftCluster() // Update region's epoch to (100, 100). @@ -73,7 +74,7 @@ func (suite *adminTestSuite) TestDropRegion() { }, })) err := cluster.HandleRegionHeartbeat(region) - suite.NoError(err) + re.NoError(err) // Region epoch cannot decrease. region = region.Clone( @@ -81,25 +82,26 @@ func (suite *adminTestSuite) TestDropRegion() { core.SetRegionVersion(50), ) err = cluster.HandleRegionHeartbeat(region) - suite.Error(err) + re.Error(err) // After drop region from cache, lower version is accepted. url := fmt.Sprintf("%s/admin/cache/region/%d", suite.urlPrefix, region.GetID()) req, err := http.NewRequest(http.MethodDelete, url, http.NoBody) - suite.NoError(err) + re.NoError(err) res, err := testDialClient.Do(req) - suite.NoError(err) - suite.Equal(http.StatusOK, res.StatusCode) + re.NoError(err) + re.Equal(http.StatusOK, res.StatusCode) res.Body.Close() err = cluster.HandleRegionHeartbeat(region) - suite.NoError(err) + re.NoError(err) region = cluster.GetRegionByKey([]byte("foo")) - suite.Equal(uint64(50), region.GetRegionEpoch().ConfVer) - suite.Equal(uint64(50), region.GetRegionEpoch().Version) + re.Equal(uint64(50), region.GetRegionEpoch().ConfVer) + re.Equal(uint64(50), region.GetRegionEpoch().Version) } func (suite *adminTestSuite) TestDropRegions() { + re := suite.Require() cluster := suite.svr.GetRaftCluster() n := uint64(10000) @@ -124,7 +126,7 @@ func (suite *adminTestSuite) TestDropRegions() { regions = append(regions, region) err := cluster.HandleRegionHeartbeat(region) - suite.NoError(err) + re.NoError(err) } // Region epoch cannot decrease. @@ -135,46 +137,46 @@ func (suite *adminTestSuite) TestDropRegions() { ) regions[i] = region err := cluster.HandleRegionHeartbeat(region) - suite.Error(err) + re.Error(err) } for i := uint64(0); i < n; i++ { region := cluster.GetRegionByKey([]byte(fmt.Sprintf("%d", i))) - suite.Equal(uint64(100), region.GetRegionEpoch().ConfVer) - suite.Equal(uint64(100), region.GetRegionEpoch().Version) + re.Equal(uint64(100), region.GetRegionEpoch().ConfVer) + re.Equal(uint64(100), region.GetRegionEpoch().Version) } // After drop all regions from cache, lower version is accepted. url := fmt.Sprintf("%s/admin/cache/regions", suite.urlPrefix) req, err := http.NewRequest(http.MethodDelete, url, http.NoBody) - suite.NoError(err) + re.NoError(err) res, err := testDialClient.Do(req) - suite.NoError(err) - suite.Equal(http.StatusOK, res.StatusCode) + re.NoError(err) + re.Equal(http.StatusOK, res.StatusCode) res.Body.Close() for _, region := range regions { err := cluster.HandleRegionHeartbeat(region) - suite.NoError(err) + re.NoError(err) } for i := uint64(0); i < n; i++ { region := cluster.GetRegionByKey([]byte(fmt.Sprintf("%d", i))) - suite.Equal(uint64(50), region.GetRegionEpoch().ConfVer) - suite.Equal(uint64(50), region.GetRegionEpoch().Version) + re.Equal(uint64(50), region.GetRegionEpoch().ConfVer) + re.Equal(uint64(50), region.GetRegionEpoch().Version) } } func (suite *adminTestSuite) TestPersistFile() { - data := []byte("#!/bin/sh\nrm -rf /") re := suite.Require() + data := []byte("#!/bin/sh\nrm -rf /") err := tu.CheckPostJSON(testDialClient, suite.urlPrefix+"/admin/persist-file/"+replication.DrStatusFile, data, tu.StatusNotOK(re)) - suite.NoError(err) + re.NoError(err) data = []byte(`{"foo":"bar"}`) err = tu.CheckPostJSON(testDialClient, suite.urlPrefix+"/admin/persist-file/"+replication.DrStatusFile, data, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) } func makeTS(offset time.Duration) uint64 { @@ -183,13 +185,13 @@ func makeTS(offset time.Duration) uint64 { } func (suite *adminTestSuite) TestResetTS() { + re := suite.Require() args := make(map[string]interface{}) t1 := makeTS(time.Hour) url := fmt.Sprintf("%s/admin/reset-ts", suite.urlPrefix) args["tso"] = fmt.Sprintf("%d", t1) values, err := json.Marshal(args) - suite.NoError(err) - re := suite.Require() + re.NoError(err) tu.Eventually(re, func() bool { resp, err := apiutil.PostJSON(testDialClient, url, values) re.NoError(err) @@ -208,128 +210,128 @@ func (suite *adminTestSuite) TestResetTS() { return false } }) - suite.NoError(err) + re.NoError(err) t2 := makeTS(32 * time.Hour) args["tso"] = fmt.Sprintf("%d", t2) values, err = json.Marshal(args) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, url, values, tu.Status(re, http.StatusForbidden), tu.StringContain(re, "too large")) - suite.NoError(err) + re.NoError(err) t3 := makeTS(-2 * time.Hour) args["tso"] = fmt.Sprintf("%d", t3) values, err = json.Marshal(args) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, url, values, tu.Status(re, http.StatusForbidden), tu.StringContain(re, "small")) - suite.NoError(err) + re.NoError(err) args["tso"] = "" values, err = json.Marshal(args) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, url, values, tu.Status(re, http.StatusBadRequest), tu.StringEqual(re, "\"invalid tso value\"\n")) - suite.NoError(err) + re.NoError(err) args["tso"] = "test" values, err = json.Marshal(args) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, url, values, tu.Status(re, http.StatusBadRequest), tu.StringEqual(re, "\"invalid tso value\"\n")) - suite.NoError(err) + re.NoError(err) t4 := makeTS(32 * time.Hour) args["tso"] = fmt.Sprintf("%d", t4) args["force-use-larger"] = "xxx" values, err = json.Marshal(args) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, url, values, tu.Status(re, http.StatusBadRequest), tu.StringContain(re, "invalid force-use-larger value")) - suite.NoError(err) + re.NoError(err) args["force-use-larger"] = false values, err = json.Marshal(args) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, url, values, tu.Status(re, http.StatusForbidden), tu.StringContain(re, "too large")) - suite.NoError(err) + re.NoError(err) args["force-use-larger"] = true values, err = json.Marshal(args) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, url, values, tu.StatusOK(re), tu.StringEqual(re, "\"Reset ts successfully.\"\n")) - suite.NoError(err) + re.NoError(err) } func (suite *adminTestSuite) TestMarkSnapshotRecovering() { re := suite.Require() url := fmt.Sprintf("%s/admin/cluster/markers/snapshot-recovering", suite.urlPrefix) // default to false - suite.NoError(tu.CheckGetJSON(testDialClient, url, nil, + re.NoError(tu.CheckGetJSON(testDialClient, url, nil, tu.StatusOK(re), tu.StringContain(re, "false"))) // mark - suite.NoError(tu.CheckPostJSON(testDialClient, url, nil, + re.NoError(tu.CheckPostJSON(testDialClient, url, nil, tu.StatusOK(re))) - suite.NoError(tu.CheckGetJSON(testDialClient, url, nil, + re.NoError(tu.CheckGetJSON(testDialClient, url, nil, tu.StatusOK(re), tu.StringContain(re, "true"))) // test using grpc call grpcServer := server.GrpcServer{Server: suite.svr} resp, err2 := grpcServer.IsSnapshotRecovering(context.Background(), &pdpb.IsSnapshotRecoveringRequest{}) - suite.NoError(err2) - suite.True(resp.Marked) + re.NoError(err2) + re.True(resp.Marked) // unmark err := tu.CheckDelete(testDialClient, url, tu.StatusOK(re)) - suite.NoError(err) - suite.NoError(tu.CheckGetJSON(testDialClient, url, nil, + re.NoError(err) + re.NoError(tu.CheckGetJSON(testDialClient, url, nil, tu.StatusOK(re), tu.StringContain(re, "false"))) } func (suite *adminTestSuite) TestRecoverAllocID() { re := suite.Require() url := fmt.Sprintf("%s/admin/base-alloc-id", suite.urlPrefix) - suite.NoError(tu.CheckPostJSON(testDialClient, url, []byte("invalid json"), tu.Status(re, http.StatusBadRequest))) + re.NoError(tu.CheckPostJSON(testDialClient, url, []byte("invalid json"), tu.Status(re, http.StatusBadRequest))) // no id or invalid id - suite.NoError(tu.CheckPostJSON(testDialClient, url, []byte(`{}`), + re.NoError(tu.CheckPostJSON(testDialClient, url, []byte(`{}`), tu.Status(re, http.StatusBadRequest), tu.StringContain(re, "invalid id value"))) - suite.NoError(tu.CheckPostJSON(testDialClient, url, []byte(`{"id": ""}`), + re.NoError(tu.CheckPostJSON(testDialClient, url, []byte(`{"id": ""}`), tu.Status(re, http.StatusBadRequest), tu.StringContain(re, "invalid id value"))) - suite.NoError(tu.CheckPostJSON(testDialClient, url, []byte(`{"id": 11}`), + re.NoError(tu.CheckPostJSON(testDialClient, url, []byte(`{"id": 11}`), tu.Status(re, http.StatusBadRequest), tu.StringContain(re, "invalid id value"))) - suite.NoError(tu.CheckPostJSON(testDialClient, url, []byte(`{"id": "aa"}`), + re.NoError(tu.CheckPostJSON(testDialClient, url, []byte(`{"id": "aa"}`), tu.Status(re, http.StatusBadRequest), tu.StringContain(re, "invalid syntax"))) // snapshot recovering=false - suite.NoError(tu.CheckPostJSON(testDialClient, url, []byte(`{"id": "100000"}`), + re.NoError(tu.CheckPostJSON(testDialClient, url, []byte(`{"id": "100000"}`), tu.Status(re, http.StatusForbidden), tu.StringContain(re, "can only recover alloc id when recovering"))) // mark and recover alloc id markRecoveringURL := fmt.Sprintf("%s/admin/cluster/markers/snapshot-recovering", suite.urlPrefix) - suite.NoError(tu.CheckPostJSON(testDialClient, markRecoveringURL, nil, + re.NoError(tu.CheckPostJSON(testDialClient, markRecoveringURL, nil, tu.StatusOK(re))) - suite.NoError(tu.CheckPostJSON(testDialClient, url, []byte(`{"id": "1000000"}`), + re.NoError(tu.CheckPostJSON(testDialClient, url, []byte(`{"id": "1000000"}`), tu.StatusOK(re))) id, err2 := suite.svr.GetAllocator().Alloc() - suite.NoError(err2) - suite.Equal(id, uint64(1000001)) + re.NoError(err2) + re.Equal(uint64(1000001), id) // recover alloc id again - suite.NoError(tu.CheckPostJSON(testDialClient, url, []byte(`{"id": "99000000"}`), + re.NoError(tu.CheckPostJSON(testDialClient, url, []byte(`{"id": "99000000"}`), tu.StatusOK(re))) id, err2 = suite.svr.GetAllocator().Alloc() - suite.NoError(err2) - suite.Equal(id, uint64(99000001)) + re.NoError(err2) + re.Equal(uint64(99000001), id) // unmark err := tu.CheckDelete(testDialClient, markRecoveringURL, tu.StatusOK(re)) - suite.NoError(err) - suite.NoError(tu.CheckGetJSON(testDialClient, markRecoveringURL, nil, + re.NoError(err) + re.NoError(tu.CheckGetJSON(testDialClient, markRecoveringURL, nil, tu.StatusOK(re), tu.StringContain(re, "false"))) - suite.NoError(tu.CheckPostJSON(testDialClient, url, []byte(`{"id": "100000"}`), + re.NoError(tu.CheckPostJSON(testDialClient, url, []byte(`{"id": "100000"}`), tu.Status(re, http.StatusForbidden), tu.StringContain(re, "can only recover alloc id when recovering"))) } diff --git a/server/api/region_test.go b/server/api/region_test.go index ea2f2871a95..7e48c80d7bc 100644 --- a/server/api/region_test.go +++ b/server/api/region_test.go @@ -224,7 +224,7 @@ func (suite *regionTestSuite) TestRegionCheck() { func (suite *regionTestSuite) TestRegions() { r := NewAPIRegionInfo(core.NewRegionInfo(&metapb.Region{Id: 1}, nil)) suite.Nil(r.Leader.Peer) - suite.Len(r.Leader.RoleName, 0) + suite.Empty(r.Leader.RoleName) rs := []*core.RegionInfo{ core.NewTestRegionInfo(2, 1, []byte("a"), []byte("b"), core.SetApproximateKeys(10), core.SetApproximateSize(10)), diff --git a/server/cluster/cluster_test.go b/server/cluster/cluster_test.go index 8c889923ea7..d5931394c1b 100644 --- a/server/cluster/cluster_test.go +++ b/server/cluster/cluster_test.go @@ -179,10 +179,10 @@ func TestStoreHeartbeat(t *testing.T) { time.Sleep(20 * time.Millisecond) storeStats = cluster.hotStat.RegionStats(utils.Read, 0) re.Empty(storeStats[1]) - re.Nil(cluster.HandleStoreHeartbeat(hotReq, hotResp)) + re.NoError(cluster.HandleStoreHeartbeat(hotReq, hotResp)) time.Sleep(20 * time.Millisecond) storeStats = cluster.hotStat.RegionStats(utils.Read, 1) - re.Len(storeStats[1], 0) + re.Empty(storeStats[1]) storeStats = cluster.hotStat.RegionStats(utils.Read, 3) re.Empty(storeStats[1]) // after 2 hot heartbeats, wo can find region 1 peer again @@ -2239,7 +2239,7 @@ func checkRegions(re *require.Assertions, cache *core.BasicCluster, regions []*c } } - re.Equal(len(regions), cache.GetTotalRegionCount()) + re.Len(regions, cache.GetTotalRegionCount()) for id, count := range regionCount { re.Equal(count, cache.GetStoreRegionCount(id)) } @@ -2744,7 +2744,7 @@ func TestMergeRegionCancelOneOperator(t *testing.T) { re.Len(ops, co.GetOperatorController().AddWaitingOperator(ops...)) // Cancel source operator. co.GetOperatorController().RemoveOperator(co.GetOperatorController().GetOperator(source.GetID())) - re.Len(co.GetOperatorController().GetOperators(), 0) + re.Empty(co.GetOperatorController().GetOperators()) // Cancel target region. ops, err = operator.CreateMergeRegionOperator("merge-region", tc, source, target, operator.OpMerge) @@ -2752,7 +2752,7 @@ func TestMergeRegionCancelOneOperator(t *testing.T) { re.Len(ops, co.GetOperatorController().AddWaitingOperator(ops...)) // Cancel target operator. co.GetOperatorController().RemoveOperator(co.GetOperatorController().GetOperator(target.GetID())) - re.Len(co.GetOperatorController().GetOperators(), 0) + re.Empty(co.GetOperatorController().GetOperators()) } func TestReplica(t *testing.T) { @@ -3047,8 +3047,8 @@ func TestAddScheduler(t *testing.T) { re.Equal(4, int(batch)) gls, err := schedulers.CreateScheduler(schedulers.GrantLeaderType, oc, storage.NewStorageWithMemoryBackend(), schedulers.ConfigSliceDecoder(schedulers.GrantLeaderType, []string{"0"}), controller.RemoveScheduler) re.NoError(err) - re.NotNil(controller.AddScheduler(gls)) - re.NotNil(controller.RemoveScheduler(gls.GetName())) + re.Error(controller.AddScheduler(gls)) + re.Error(controller.RemoveScheduler(gls.GetName())) gls, err = schedulers.CreateScheduler(schedulers.GrantLeaderType, oc, storage.NewStorageWithMemoryBackend(), schedulers.ConfigSliceDecoder(schedulers.GrantLeaderType, []string{"1"}), controller.RemoveScheduler) re.NoError(err) @@ -3445,7 +3445,7 @@ func TestStoreOverloaded(t *testing.T) { time.Sleep(time.Second) for i := 0; i < 100; i++ { ops, _ := lb.Schedule(tc, false /* dryRun */) - re.Greater(len(ops), 0) + re.NotEmpty(ops) } } @@ -3480,7 +3480,7 @@ func TestStoreOverloadedWithReplace(t *testing.T) { // sleep 2 seconds to make sure that token is filled up time.Sleep(2 * time.Second) ops, _ = lb.Schedule(tc, false /* dryRun */) - re.Greater(len(ops), 0) + re.NotEmpty(ops) } func TestDownStoreLimit(t *testing.T) { diff --git a/server/config/config_test.go b/server/config/config_test.go index 07cdc966409..69cfafd8d36 100644 --- a/server/config/config_test.go +++ b/server/config/config_test.go @@ -492,7 +492,7 @@ func TestRateLimitClone(t *testing.T) { ConcurrencyLimit: 200, } dc := cfg.LimiterConfig["test"] - re.Equal(dc.ConcurrencyLimit, uint64(0)) + re.Zero(dc.ConcurrencyLimit) gCfg := &GRPCRateLimitConfig{ EnableRateLimit: defaultEnableGRPCRateLimitMiddleware, @@ -503,5 +503,5 @@ func TestRateLimitClone(t *testing.T) { ConcurrencyLimit: 300, } gdc := gCfg.LimiterConfig["test"] - re.Equal(gdc.ConcurrencyLimit, uint64(0)) + re.Zero(gdc.ConcurrencyLimit) } diff --git a/tests/pdctl/hot/hot_test.go b/tests/pdctl/hot/hot_test.go index 03c26f40441..366887e19aa 100644 --- a/tests/pdctl/hot/hot_test.go +++ b/tests/pdctl/hot/hot_test.go @@ -368,11 +368,11 @@ func (suite *hotTestSuite) checkHotWithoutHotPeer(cluster *tests.TestCluster) { re.NoError(err) re.NoError(json.Unmarshal(output, &hotRegion)) re.NotNil(hotRegion.AsPeer[1]) - re.Equal(hotRegion.AsPeer[1].Count, 0) - re.Equal(0.0, hotRegion.AsPeer[1].TotalBytesRate) + re.Zero(hotRegion.AsPeer[1].Count) + re.Zero(hotRegion.AsPeer[1].TotalBytesRate) re.Equal(load, hotRegion.AsPeer[1].StoreByteRate) - re.Equal(hotRegion.AsLeader[1].Count, 0) - re.Equal(0.0, hotRegion.AsLeader[1].TotalBytesRate) + re.Zero(hotRegion.AsLeader[1].Count) + re.Zero(hotRegion.AsLeader[1].TotalBytesRate) re.Equal(load, hotRegion.AsLeader[1].StoreByteRate) } { @@ -381,12 +381,12 @@ func (suite *hotTestSuite) checkHotWithoutHotPeer(cluster *tests.TestCluster) { hotRegion := statistics.StoreHotPeersInfos{} re.NoError(err) re.NoError(json.Unmarshal(output, &hotRegion)) - re.Equal(0, hotRegion.AsPeer[1].Count) - re.Equal(0.0, hotRegion.AsPeer[1].TotalBytesRate) + re.Zero(hotRegion.AsPeer[1].Count) + re.Zero(hotRegion.AsPeer[1].TotalBytesRate) re.Equal(load, hotRegion.AsPeer[1].StoreByteRate) - re.Equal(0, hotRegion.AsLeader[1].Count) - re.Equal(0.0, hotRegion.AsLeader[1].TotalBytesRate) - re.Equal(0.0, hotRegion.AsLeader[1].StoreByteRate) // write leader sum + re.Zero(hotRegion.AsLeader[1].Count) + re.Zero(hotRegion.AsLeader[1].TotalBytesRate) + re.Zero(hotRegion.AsLeader[1].StoreByteRate) // write leader sum } } diff --git a/tests/pdctl/keyspace/keyspace_group_test.go b/tests/pdctl/keyspace/keyspace_group_test.go index cbfdf1d099a..0de48a85c64 100644 --- a/tests/pdctl/keyspace/keyspace_group_test.go +++ b/tests/pdctl/keyspace/keyspace_group_test.go @@ -78,14 +78,14 @@ func TestKeyspaceGroup(t *testing.T) { err = json.Unmarshal(output, &keyspaceGroup) re.NoError(err) re.Equal(uint32(1), keyspaceGroup.ID) - re.Equal(keyspaceGroup.Keyspaces, []uint32{111}) + re.Equal([]uint32{111}, keyspaceGroup.Keyspaces) output, err = pdctl.ExecuteCommand(cmd, append(args, "2")...) re.NoError(err) keyspaceGroup = endpoint.KeyspaceGroup{} err = json.Unmarshal(output, &keyspaceGroup) re.NoError(err) re.Equal(uint32(2), keyspaceGroup.ID) - re.Equal(keyspaceGroup.Keyspaces, []uint32{222, 333}) + re.Equal([]uint32{222, 333}, keyspaceGroup.Keyspaces) } func TestSplitKeyspaceGroup(t *testing.T) { @@ -133,8 +133,8 @@ func TestSplitKeyspaceGroup(t *testing.T) { err = json.Unmarshal(output, &keyspaceGroups) re.NoError(err) re.Len(keyspaceGroups, 2) - re.Equal(keyspaceGroups[0].ID, uint32(0)) - re.Equal(keyspaceGroups[1].ID, uint32(1)) + re.Zero(keyspaceGroups[0].ID) + re.Equal(uint32(1), keyspaceGroups[1].ID) re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/keyspace/acceleratedAllocNodes")) re.NoError(failpoint.Disable("github.com/tikv/pd/server/delayStartServerLoop")) @@ -448,7 +448,7 @@ func TestKeyspaceGroupState(t *testing.T) { var keyspaceGroups []*endpoint.KeyspaceGroup err = json.Unmarshal(output, &keyspaceGroups) re.NoError(err) - re.Len(keyspaceGroups, 0) + re.Empty(keyspaceGroups) testutil.Eventually(re, func() bool { args := []string{"-u", pdAddr, "keyspace-group", "split", "0", "2", "3"} output, err := pdctl.ExecuteCommand(cmd, args...) @@ -462,8 +462,8 @@ func TestKeyspaceGroupState(t *testing.T) { err = json.Unmarshal(output, &keyspaceGroups) re.NoError(err) re.Len(keyspaceGroups, 2) - re.Equal(keyspaceGroups[0].ID, uint32(0)) - re.Equal(keyspaceGroups[1].ID, uint32(2)) + re.Equal(uint32(0), keyspaceGroups[0].ID) + re.Equal(uint32(2), keyspaceGroups[1].ID) args = []string{"-u", pdAddr, "keyspace-group", "finish-split", "2"} output, err = pdctl.ExecuteCommand(cmd, args...) @@ -486,7 +486,7 @@ func TestKeyspaceGroupState(t *testing.T) { err = json.Unmarshal(output, &keyspaceGroups) re.NoError(err) re.Len(keyspaceGroups, 1) - re.Equal(keyspaceGroups[0].ID, uint32(0)) + re.Equal(uint32(0), keyspaceGroups[0].ID) re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/keyspace/acceleratedAllocNodes")) re.NoError(failpoint.Disable("github.com/tikv/pd/server/delayStartServerLoop")) diff --git a/tests/pdctl/keyspace/keyspace_test.go b/tests/pdctl/keyspace/keyspace_test.go index 3ff755fe601..f83d09760a9 100644 --- a/tests/pdctl/keyspace/keyspace_test.go +++ b/tests/pdctl/keyspace/keyspace_test.go @@ -147,22 +147,24 @@ func TestKeyspaceTestSuite(t *testing.T) { } func (suite *keyspaceTestSuite) SetupTest() { + re := suite.Require() suite.ctx, suite.cancel = context.WithCancel(context.Background()) - suite.NoError(failpoint.Enable("github.com/tikv/pd/server/delayStartServerLoop", `return(true)`)) - suite.NoError(failpoint.Enable("github.com/tikv/pd/pkg/keyspace/skipSplitRegion", "return(true)")) + re.NoError(failpoint.Enable("github.com/tikv/pd/server/delayStartServerLoop", `return(true)`)) + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/keyspace/skipSplitRegion", "return(true)")) tc, err := tests.NewTestAPICluster(suite.ctx, 1) - suite.NoError(err) - suite.NoError(tc.RunInitialServers()) + re.NoError(err) + re.NoError(tc.RunInitialServers()) tc.WaitLeader() leaderServer := tc.GetLeaderServer() - suite.NoError(leaderServer.BootstrapCluster()) + re.NoError(leaderServer.BootstrapCluster()) suite.cluster = tc suite.pdAddr = tc.GetConfig().GetClientURL() } func (suite *keyspaceTestSuite) TearDownTest() { - suite.NoError(failpoint.Disable("github.com/tikv/pd/server/delayStartServerLoop")) - suite.NoError(failpoint.Disable("github.com/tikv/pd/pkg/keyspace/skipSplitRegion")) + re := suite.Require() + re.NoError(failpoint.Disable("github.com/tikv/pd/server/delayStartServerLoop")) + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/keyspace/skipSplitRegion")) suite.cancel() } diff --git a/tests/pdctl/log/log_test.go b/tests/pdctl/log/log_test.go index e6995231329..08df4a78bea 100644 --- a/tests/pdctl/log/log_test.go +++ b/tests/pdctl/log/log_test.go @@ -39,11 +39,12 @@ func TestLogTestSuite(t *testing.T) { } func (suite *logTestSuite) SetupSuite() { + re := suite.Require() suite.ctx, suite.cancel = context.WithCancel(context.Background()) var err error suite.cluster, err = tests.NewTestCluster(suite.ctx, 3) - suite.NoError(err) - suite.NoError(suite.cluster.RunInitialServers()) + re.NoError(err) + re.NoError(suite.cluster.RunInitialServers()) suite.cluster.WaitLeader() suite.pdAddrs = suite.cluster.GetConfig().GetClientURLs() @@ -53,7 +54,7 @@ func (suite *logTestSuite) SetupSuite() { LastHeartbeat: time.Now().UnixNano(), } leaderServer := suite.cluster.GetLeaderServer() - suite.NoError(leaderServer.BootstrapCluster()) + re.NoError(leaderServer.BootstrapCluster()) tests.MustPutStore(suite.Require(), suite.cluster, store) } diff --git a/tests/pdctl/operator/operator_test.go b/tests/pdctl/operator/operator_test.go index aa2fe5d1304..1de61dca880 100644 --- a/tests/pdctl/operator/operator_test.go +++ b/tests/pdctl/operator/operator_test.go @@ -107,7 +107,7 @@ func (suite *operatorTestSuite) checkOperator(cluster *tests.TestCluster) { output, err := pdctl.ExecuteCommand(cmd, args...) re.NoError(err) re.NoError(json.Unmarshal(output, &slice)) - re.Len(slice, 0) + re.Empty(slice) args = []string{"-u", pdAddr, "operator", "check", "2"} output, err = pdctl.ExecuteCommand(cmd, args...) re.NoError(err) diff --git a/tests/pdctl/resourcemanager/resource_manager_command_test.go b/tests/pdctl/resourcemanager/resource_manager_command_test.go index ad43e0abca9..cbd9b481869 100644 --- a/tests/pdctl/resourcemanager/resource_manager_command_test.go +++ b/tests/pdctl/resourcemanager/resource_manager_command_test.go @@ -41,9 +41,10 @@ type testResourceManagerSuite struct { } func (s *testResourceManagerSuite) SetupSuite() { + re := s.Require() s.ctx, s.cancel = context.WithCancel(context.Background()) cluster, err := tests.NewTestCluster(s.ctx, 1) - s.Nil(err) + re.NoError(err) s.cluster = cluster s.cluster.RunInitialServers() cluster.WaitLeader() @@ -56,18 +57,19 @@ func (s *testResourceManagerSuite) TearDownSuite() { } func (s *testResourceManagerSuite) TestConfigController() { + re := s.Require() expectCfg := server.ControllerConfig{} expectCfg.Adjust(nil) // Show controller config checkShow := func() { args := []string{"-u", s.pdAddr, "resource-manager", "config", "controller", "show"} output, err := pdctl.ExecuteCommand(pdctlCmd.GetRootCmd(), args...) - s.Nil(err) + re.NoError(err) actualCfg := server.ControllerConfig{} err = json.Unmarshal(output, &actualCfg) - s.Nil(err) - s.Equal(expectCfg, actualCfg) + re.NoError(err) + re.Equal(expectCfg, actualCfg) } // Check default config @@ -76,22 +78,22 @@ func (s *testResourceManagerSuite) TestConfigController() { // Set controller config args := []string{"-u", s.pdAddr, "resource-manager", "config", "controller", "set", "ltb-max-wait-duration", "1h"} output, err := pdctl.ExecuteCommand(pdctlCmd.GetRootCmd(), args...) - s.Nil(err) - s.Contains(string(output), "Success!") + re.NoError(err) + re.Contains(string(output), "Success!") expectCfg.LTBMaxWaitDuration = typeutil.Duration{Duration: 1 * time.Hour} checkShow() args = []string{"-u", s.pdAddr, "resource-manager", "config", "controller", "set", "enable-controller-trace-log", "true"} output, err = pdctl.ExecuteCommand(pdctlCmd.GetRootCmd(), args...) - s.Nil(err) - s.Contains(string(output), "Success!") + re.NoError(err) + re.Contains(string(output), "Success!") expectCfg.EnableControllerTraceLog = true checkShow() args = []string{"-u", s.pdAddr, "resource-manager", "config", "controller", "set", "write-base-cost", "2"} output, err = pdctl.ExecuteCommand(pdctlCmd.GetRootCmd(), args...) - s.Nil(err) - s.Contains(string(output), "Success!") + re.NoError(err) + re.Contains(string(output), "Success!") expectCfg.RequestUnit.WriteBaseCost = 2 checkShow() } diff --git a/tests/pdctl/scheduler/scheduler_test.go b/tests/pdctl/scheduler/scheduler_test.go index 140ee7a7c44..585a5fd1199 100644 --- a/tests/pdctl/scheduler/scheduler_test.go +++ b/tests/pdctl/scheduler/scheduler_test.go @@ -48,7 +48,8 @@ func TestSchedulerTestSuite(t *testing.T) { } func (suite *schedulerTestSuite) SetupSuite() { - suite.NoError(failpoint.Enable("github.com/tikv/pd/server/cluster/skipStoreConfigSync", `return(true)`)) + re := suite.Require() + re.NoError(failpoint.Enable("github.com/tikv/pd/server/cluster/skipStoreConfigSync", `return(true)`)) suite.env = tests.NewSchedulingTestEnvironment(suite.T()) suite.defaultSchedulers = []string{ "balance-leader-scheduler", @@ -61,8 +62,9 @@ func (suite *schedulerTestSuite) SetupSuite() { } func (suite *schedulerTestSuite) TearDownSuite() { + re := suite.Require() suite.env.Cleanup() - suite.NoError(failpoint.Disable("github.com/tikv/pd/server/cluster/skipStoreConfigSync")) + re.NoError(failpoint.Disable("github.com/tikv/pd/server/cluster/skipStoreConfigSync")) } func (suite *schedulerTestSuite) TearDownTest() { diff --git a/tests/server/api/api_test.go b/tests/server/api/api_test.go index f5db6bb2513..946e65bc6e4 100644 --- a/tests/server/api/api_test.go +++ b/tests/server/api/api_test.go @@ -218,7 +218,7 @@ func (suite *middlewareTestSuite) TestRateLimitMiddleware() { resp, err := dialClient.Do(req) suite.NoError(err) resp.Body.Close() - suite.Equal(leader.GetServer().GetServiceMiddlewarePersistOptions().IsRateLimitEnabled(), true) + suite.True(leader.GetServer().GetServiceMiddlewarePersistOptions().IsRateLimitEnabled()) // returns StatusOK when no rate-limit config req, _ = http.NewRequest(http.MethodPost, leader.GetAddr()+"/pd/api/v1/admin/log", strings.NewReader("\"info\"")) @@ -227,7 +227,7 @@ func (suite *middlewareTestSuite) TestRateLimitMiddleware() { _, err = io.ReadAll(resp.Body) resp.Body.Close() suite.NoError(err) - suite.Equal(resp.StatusCode, http.StatusOK) + suite.Equal(http.StatusOK, resp.StatusCode) input = make(map[string]interface{}) input["type"] = "label" input["label"] = "SetLogLevel" @@ -241,7 +241,7 @@ func (suite *middlewareTestSuite) TestRateLimitMiddleware() { _, err = io.ReadAll(resp.Body) resp.Body.Close() suite.NoError(err) - suite.Equal(resp.StatusCode, http.StatusOK) + suite.Equal(http.StatusOK, resp.StatusCode) for i := 0; i < 3; i++ { req, _ = http.NewRequest(http.MethodPost, leader.GetAddr()+"/pd/api/v1/admin/log", strings.NewReader("\"info\"")) @@ -251,10 +251,10 @@ func (suite *middlewareTestSuite) TestRateLimitMiddleware() { resp.Body.Close() suite.NoError(err) if i > 0 { - suite.Equal(resp.StatusCode, http.StatusTooManyRequests) + suite.Equal(http.StatusTooManyRequests, resp.StatusCode) suite.Equal(string(data), fmt.Sprintf("%s\n", http.StatusText(http.StatusTooManyRequests))) } else { - suite.Equal(resp.StatusCode, http.StatusOK) + suite.Equal(http.StatusOK, resp.StatusCode) } } @@ -268,10 +268,10 @@ func (suite *middlewareTestSuite) TestRateLimitMiddleware() { resp.Body.Close() suite.NoError(err) if i > 0 { - suite.Equal(resp.StatusCode, http.StatusTooManyRequests) + suite.Equal(http.StatusTooManyRequests, resp.StatusCode) suite.Equal(string(data), fmt.Sprintf("%s\n", http.StatusText(http.StatusTooManyRequests))) } else { - suite.Equal(resp.StatusCode, http.StatusOK) + suite.Equal(http.StatusOK, resp.StatusCode) } } @@ -284,7 +284,7 @@ func (suite *middlewareTestSuite) TestRateLimitMiddleware() { data, err := io.ReadAll(resp.Body) resp.Body.Close() suite.NoError(err) - suite.Equal(resp.StatusCode, http.StatusTooManyRequests) + suite.Equal(http.StatusTooManyRequests, resp.StatusCode) suite.Equal(string(data), fmt.Sprintf("%s\n", http.StatusText(http.StatusTooManyRequests))) } @@ -297,12 +297,12 @@ func (suite *middlewareTestSuite) TestRateLimitMiddleware() { } server.MustWaitLeader(suite.Require(), servers) leader = suite.cluster.GetLeaderServer() - suite.Equal(leader.GetServer().GetServiceMiddlewarePersistOptions().IsRateLimitEnabled(), true) + suite.True(leader.GetServer().GetServiceMiddlewarePersistOptions().IsRateLimitEnabled()) cfg, ok := leader.GetServer().GetRateLimitConfig().LimiterConfig["SetLogLevel"] - suite.Equal(ok, true) - suite.Equal(cfg.ConcurrencyLimit, uint64(1)) - suite.Equal(cfg.QPS, 0.5) - suite.Equal(cfg.QPSBurst, 1) + suite.True(ok) + suite.Equal(uint64(1), cfg.ConcurrencyLimit) + suite.Equal(0.5, cfg.QPS) + suite.Equal(1, cfg.QPSBurst) for i := 0; i < 3; i++ { req, _ = http.NewRequest(http.MethodPost, leader.GetAddr()+"/pd/api/v1/admin/log", strings.NewReader("\"info\"")) @@ -312,10 +312,10 @@ func (suite *middlewareTestSuite) TestRateLimitMiddleware() { resp.Body.Close() suite.NoError(err) if i > 0 { - suite.Equal(resp.StatusCode, http.StatusTooManyRequests) + suite.Equal(http.StatusTooManyRequests, resp.StatusCode) suite.Equal(string(data), fmt.Sprintf("%s\n", http.StatusText(http.StatusTooManyRequests))) } else { - suite.Equal(resp.StatusCode, http.StatusOK) + suite.Equal(http.StatusOK, resp.StatusCode) } } @@ -329,10 +329,10 @@ func (suite *middlewareTestSuite) TestRateLimitMiddleware() { resp.Body.Close() suite.NoError(err) if i > 0 { - suite.Equal(resp.StatusCode, http.StatusTooManyRequests) + suite.Equal(http.StatusTooManyRequests, resp.StatusCode) suite.Equal(string(data), fmt.Sprintf("%s\n", http.StatusText(http.StatusTooManyRequests))) } else { - suite.Equal(resp.StatusCode, http.StatusOK) + suite.Equal(http.StatusOK, resp.StatusCode) } } @@ -345,7 +345,7 @@ func (suite *middlewareTestSuite) TestRateLimitMiddleware() { data, err := io.ReadAll(resp.Body) resp.Body.Close() suite.NoError(err) - suite.Equal(resp.StatusCode, http.StatusTooManyRequests) + suite.Equal(http.StatusTooManyRequests, resp.StatusCode) suite.Equal(string(data), fmt.Sprintf("%s\n", http.StatusText(http.StatusTooManyRequests))) } @@ -358,7 +358,7 @@ func (suite *middlewareTestSuite) TestRateLimitMiddleware() { resp, err = dialClient.Do(req) suite.NoError(err) resp.Body.Close() - suite.Equal(leader.GetServer().GetServiceMiddlewarePersistOptions().IsRateLimitEnabled(), false) + suite.False(leader.GetServer().GetServiceMiddlewarePersistOptions().IsRateLimitEnabled()) for i := 0; i < 3; i++ { req, _ = http.NewRequest(http.MethodPost, leader.GetAddr()+"/pd/api/v1/admin/log", strings.NewReader("\"info\"")) @@ -367,7 +367,7 @@ func (suite *middlewareTestSuite) TestRateLimitMiddleware() { _, err = io.ReadAll(resp.Body) resp.Body.Close() suite.NoError(err) - suite.Equal(resp.StatusCode, http.StatusOK) + suite.Equal(http.StatusOK, resp.StatusCode) } } @@ -377,7 +377,7 @@ func (suite *middlewareTestSuite) TestSwaggerUrl() { req, _ := http.NewRequest(http.MethodGet, leader.GetAddr()+"/swagger/ui/index", http.NoBody) resp, err := dialClient.Do(req) suite.NoError(err) - suite.True(resp.StatusCode == http.StatusNotFound) + suite.Equal(http.StatusNotFound, resp.StatusCode) resp.Body.Close() } diff --git a/tests/server/api/rule_test.go b/tests/server/api/rule_test.go index 0a0c3f2fb2e..eaa41cc11bc 100644 --- a/tests/server/api/rule_test.go +++ b/tests/server/api/rule_test.go @@ -256,7 +256,7 @@ func (suite *ruleTestSuite) checkGetAll(cluster *tests.TestCluster) { var resp2 []*placement.Rule err = tu.ReadGetJSON(re, testDialClient, urlPrefix+"/rules", &resp2) suite.NoError(err) - suite.GreaterOrEqual(len(resp2), 1) + suite.NotEmpty(resp2) } func (suite *ruleTestSuite) TestSetAll() { @@ -1039,40 +1039,40 @@ func (suite *regionRuleTestSuite) checkRegionPlacementRule(cluster *tests.TestCl u := fmt.Sprintf("%s/config/rules/region/%d/detail", urlPrefix, 1) err := tu.ReadGetJSON(re, testDialClient, u, fit) suite.NoError(err) - suite.Equal(len(fit.RuleFits), 1) - suite.Equal(len(fit.OrphanPeers), 1) + suite.Len(fit.RuleFits, 1) + suite.Len(fit.OrphanPeers, 1) u = fmt.Sprintf("%s/config/rules/region/%d/detail", urlPrefix, 2) fit = &placement.RegionFit{} err = tu.ReadGetJSON(re, testDialClient, u, fit) suite.NoError(err) - suite.Equal(len(fit.RuleFits), 2) - suite.Equal(len(fit.OrphanPeers), 0) + suite.Len(fit.RuleFits, 2) + suite.Empty(fit.OrphanPeers) u = fmt.Sprintf("%s/config/rules/region/%d/detail", urlPrefix, 3) fit = &placement.RegionFit{} err = tu.ReadGetJSON(re, testDialClient, u, fit) suite.NoError(err) - suite.Equal(len(fit.RuleFits), 0) - suite.Equal(len(fit.OrphanPeers), 2) + suite.Empty(fit.RuleFits) + suite.Len(fit.OrphanPeers, 2) var label labeler.LabelRule escapedID := url.PathEscape("keyspaces/0") u = fmt.Sprintf("%s/config/region-label/rule/%s", urlPrefix, escapedID) err = tu.ReadGetJSON(re, testDialClient, u, &label) suite.NoError(err) - suite.Equal(label.ID, "keyspaces/0") + suite.Equal("keyspaces/0", label.ID) var labels []labeler.LabelRule u = fmt.Sprintf("%s/config/region-label/rules", urlPrefix) err = tu.ReadGetJSON(re, testDialClient, u, &labels) suite.NoError(err) suite.Len(labels, 1) - suite.Equal(labels[0].ID, "keyspaces/0") + suite.Equal("keyspaces/0", labels[0].ID) u = fmt.Sprintf("%s/config/region-label/rules/ids", urlPrefix) err = tu.CheckGetJSON(testDialClient, u, []byte(`["rule1", "rule3"]`), func(resp []byte, statusCode int, _ http.Header) { err := json.Unmarshal(resp, &labels) suite.NoError(err) - suite.Len(labels, 0) + suite.Empty(labels) }) suite.NoError(err) @@ -1080,7 +1080,7 @@ func (suite *regionRuleTestSuite) checkRegionPlacementRule(cluster *tests.TestCl err := json.Unmarshal(resp, &labels) suite.NoError(err) suite.Len(labels, 1) - suite.Equal(labels[0].ID, "keyspaces/0") + suite.Equal("keyspaces/0", labels[0].ID) }) suite.NoError(err) diff --git a/tests/server/apiv2/handlers/keyspace_test.go b/tests/server/apiv2/handlers/keyspace_test.go index f7b43ab194d..535f01cc33e 100644 --- a/tests/server/apiv2/handlers/keyspace_test.go +++ b/tests/server/apiv2/handlers/keyspace_test.go @@ -46,22 +46,24 @@ func TestKeyspaceTestSuite(t *testing.T) { } func (suite *keyspaceTestSuite) SetupTest() { + re := suite.Require() ctx, cancel := context.WithCancel(context.Background()) suite.cleanup = cancel cluster, err := tests.NewTestCluster(ctx, 1) suite.cluster = cluster - suite.NoError(err) - suite.NoError(cluster.RunInitialServers()) - suite.NotEmpty(cluster.WaitLeader()) + re.NoError(err) + re.NoError(cluster.RunInitialServers()) + re.NotEmpty(cluster.WaitLeader()) suite.server = cluster.GetLeaderServer() - suite.NoError(suite.server.BootstrapCluster()) - suite.NoError(failpoint.Enable("github.com/tikv/pd/pkg/keyspace/skipSplitRegion", "return(true)")) + re.NoError(suite.server.BootstrapCluster()) + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/keyspace/skipSplitRegion", "return(true)")) } func (suite *keyspaceTestSuite) TearDownTest() { + re := suite.Require() suite.cleanup() suite.cluster.Destroy() - suite.NoError(failpoint.Disable("github.com/tikv/pd/pkg/keyspace/skipSplitRegion")) + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/keyspace/skipSplitRegion")) } func (suite *keyspaceTestSuite) TestCreateLoadKeyspace() { @@ -133,7 +135,7 @@ func (suite *keyspaceTestSuite) TestLoadRangeKeyspace() { loadResponse := sendLoadRangeRequest(re, suite.server, "", "") re.Empty(loadResponse.NextPageToken) // Load response should contain no more pages. // Load response should contain all created keyspace and a default. - re.Equal(len(keyspaces)+1, len(loadResponse.Keyspaces)) + re.Len(loadResponse.Keyspaces, len(keyspaces)+1) for i, created := range keyspaces { re.Equal(created, loadResponse.Keyspaces[i+1].KeyspaceMeta) } diff --git a/tests/server/apiv2/handlers/tso_keyspace_group_test.go b/tests/server/apiv2/handlers/tso_keyspace_group_test.go index 214de6e95ef..2bf2db715fa 100644 --- a/tests/server/apiv2/handlers/tso_keyspace_group_test.go +++ b/tests/server/apiv2/handlers/tso_keyspace_group_test.go @@ -39,14 +39,15 @@ func TestKeyspaceGroupTestSuite(t *testing.T) { } func (suite *keyspaceGroupTestSuite) SetupTest() { + re := suite.Require() suite.ctx, suite.cancel = context.WithCancel(context.Background()) cluster, err := tests.NewTestAPICluster(suite.ctx, 1) suite.cluster = cluster - suite.NoError(err) - suite.NoError(cluster.RunInitialServers()) - suite.NotEmpty(cluster.WaitLeader()) + re.NoError(err) + re.NoError(cluster.RunInitialServers()) + re.NotEmpty(cluster.WaitLeader()) suite.server = cluster.GetLeaderServer() - suite.NoError(suite.server.BootstrapCluster()) + re.NoError(suite.server.BootstrapCluster()) } func (suite *keyspaceGroupTestSuite) TearDownTest() { diff --git a/tests/server/cluster/cluster_test.go b/tests/server/cluster/cluster_test.go index 0b0779d9434..67c798d7f69 100644 --- a/tests/server/cluster/cluster_test.go +++ b/tests/server/cluster/cluster_test.go @@ -1485,7 +1485,7 @@ func TestMinResolvedTS(t *testing.T) { } // default run job - re.NotEqual(rc.GetPDServerConfig().MinResolvedTSPersistenceInterval.Duration, 0) + re.NotZero(rc.GetPDServerConfig().MinResolvedTSPersistenceInterval.Duration) setMinResolvedTSPersistenceInterval(re, rc, svr, 0) re.Equal(time.Duration(0), rc.GetPDServerConfig().MinResolvedTSPersistenceInterval.Duration) diff --git a/tests/server/cluster/cluster_work_test.go b/tests/server/cluster/cluster_work_test.go index eabecf8e29b..f503563dbb1 100644 --- a/tests/server/cluster/cluster_work_test.go +++ b/tests/server/cluster/cluster_work_test.go @@ -16,7 +16,6 @@ package cluster_test import ( "context" - "errors" "sort" "testing" "time" @@ -112,9 +111,9 @@ func TestAskSplit(t *testing.T) { re.NoError(leaderServer.GetServer().SaveTTLConfig(map[string]interface{}{"schedule.enable-tikv-split-region": 0}, time.Minute)) _, err = rc.HandleAskSplit(req) - re.True(errors.Is(err, errs.ErrSchedulerTiKVSplitDisabled)) + re.ErrorIs(err, errs.ErrSchedulerTiKVSplitDisabled) _, err = rc.HandleAskBatchSplit(req1) - re.True(errors.Is(err, errs.ErrSchedulerTiKVSplitDisabled)) + re.ErrorIs(err, errs.ErrSchedulerTiKVSplitDisabled) re.NoError(leaderServer.GetServer().SaveTTLConfig(map[string]interface{}{"schedule.enable-tikv-split-region": 0}, 0)) // wait ttl config takes effect time.Sleep(time.Second) diff --git a/tests/server/keyspace/keyspace_test.go b/tests/server/keyspace/keyspace_test.go index 86b8f6fd37c..3ee15e1edc1 100644 --- a/tests/server/keyspace/keyspace_test.go +++ b/tests/server/keyspace/keyspace_test.go @@ -100,7 +100,7 @@ func checkLabelRule(re *require.Assertions, id uint32, regionLabeler *labeler.Re rangeRule, ok := loadedLabel.Data.([]*labeler.KeyRangeRule) re.True(ok) - re.Equal(2, len(rangeRule)) + re.Len(rangeRule, 2) keyspaceIDBytes := make([]byte, 4) nextKeyspaceIDBytes := make([]byte, 4) diff --git a/tools/pd-backup/pdbackup/backup_test.go b/tools/pd-backup/pdbackup/backup_test.go index 40e4190f5d4..b35bf1e8a70 100644 --- a/tools/pd-backup/pdbackup/backup_test.go +++ b/tools/pd-backup/pdbackup/backup_test.go @@ -99,6 +99,7 @@ func setupServer() (*httptest.Server, *config.Config) { } func (s *backupTestSuite) BeforeTest(suiteName, testName string) { + re := s.Require() ctx, cancel := context.WithTimeout(context.Background(), time.Second*3) defer cancel() @@ -106,21 +107,21 @@ func (s *backupTestSuite) BeforeTest(suiteName, testName string) { ctx, pdClusterIDPath, string(typeutil.Uint64ToBytes(clusterID))) - s.NoError(err) + re.NoError(err) var ( rootPath = path.Join(pdRootPath, strconv.FormatUint(clusterID, 10)) allocTimestampMaxBytes = typeutil.Uint64ToBytes(allocTimestampMax) ) _, err = s.etcdClient.Put(ctx, endpoint.TimestampPath(rootPath), string(allocTimestampMaxBytes)) - s.NoError(err) + re.NoError(err) var ( allocIDPath = path.Join(rootPath, "alloc_id") allocIDMaxBytes = typeutil.Uint64ToBytes(allocIDMax) ) _, err = s.etcdClient.Put(ctx, allocIDPath, string(allocIDMaxBytes)) - s.NoError(err) + re.NoError(err) } func (s *backupTestSuite) AfterTest(suiteName, testName string) { @@ -128,8 +129,9 @@ func (s *backupTestSuite) AfterTest(suiteName, testName string) { } func (s *backupTestSuite) TestGetBackupInfo() { + re := s.Require() actual, err := GetBackupInfo(s.etcdClient, s.server.URL) - s.NoError(err) + re.NoError(err) expected := &BackupInfo{ ClusterID: clusterID, @@ -137,22 +139,22 @@ func (s *backupTestSuite) TestGetBackupInfo() { AllocTimestampMax: allocTimestampMax, Config: s.serverConfig, } - s.Equal(expected, actual) + re.Equal(expected, actual) tmpFile, err := os.CreateTemp(os.TempDir(), "pd_backup_info_test.json") - s.NoError(err) + re.NoError(err) defer os.RemoveAll(tmpFile.Name()) - s.NoError(OutputToFile(actual, tmpFile)) + re.NoError(OutputToFile(actual, tmpFile)) _, err = tmpFile.Seek(0, 0) - s.NoError(err) + re.NoError(err) b, err := io.ReadAll(tmpFile) - s.NoError(err) + re.NoError(err) var restored BackupInfo err = json.Unmarshal(b, &restored) - s.NoError(err) + re.NoError(err) - s.Equal(expected, &restored) + re.Equal(expected, &restored) } diff --git a/tools/pd-simulator/simulator/simutil/key_test.go b/tools/pd-simulator/simulator/simutil/key_test.go index b34f1bb3809..6f71bd12d14 100644 --- a/tools/pd-simulator/simulator/simutil/key_test.go +++ b/tools/pd-simulator/simulator/simutil/key_test.go @@ -102,13 +102,13 @@ func TestGenerateKeys(t *testing.T) { numKeys := 10 actual := GenerateKeys(numKeys) - re.Equal(len(actual), numKeys) + re.Len(actual, numKeys) // make sure every key: // i. has length `keyLen` // ii. has only characters from `keyChars` for _, key := range actual { - re.Equal(len(key), keyLen) + re.Len(key, keyLen) for _, char := range key { re.True(strings.ContainsRune(keyChars, char)) } From cfbc9b96cdc23d4f7c723b0b4130f6612f9ed00a Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Wed, 20 Dec 2023 12:01:52 +0800 Subject: [PATCH 106/137] mcs: watch rule change with txn (#7550) close tikv/pd#7418 Signed-off-by: lhy1024 Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- pkg/core/basic_cluster.go | 21 ++ pkg/mcs/scheduling/server/apis/v1/api.go | 2 +- pkg/mcs/scheduling/server/rule/watcher.go | 154 +++++---- pkg/schedule/placement/config.go | 32 +- pkg/schedule/placement/config_test.go | 40 +-- pkg/schedule/placement/rule_manager.go | 254 +++++++------- pkg/schedule/placement/rule_manager_test.go | 14 +- pkg/storage/endpoint/rule.go | 63 ++-- pkg/tso/keyspace_group_manager.go | 4 +- pkg/utils/etcdutil/etcdutil.go | 24 +- server/keyspace_service.go | 7 +- tests/server/api/region_test.go | 56 +++- tests/server/api/rule_test.go | 345 ++++++++++++++++++-- 13 files changed, 711 insertions(+), 305 deletions(-) diff --git a/pkg/core/basic_cluster.go b/pkg/core/basic_cluster.go index 2258a816324..d70b620db3b 100644 --- a/pkg/core/basic_cluster.go +++ b/pkg/core/basic_cluster.go @@ -309,3 +309,24 @@ func NewKeyRange(startKey, endKey string) KeyRange { EndKey: []byte(endKey), } } + +// KeyRanges is a slice of KeyRange. +type KeyRanges struct { + krs []*KeyRange +} + +// Append appends a KeyRange. +func (rs *KeyRanges) Append(startKey, endKey []byte) { + rs.krs = append(rs.krs, &KeyRange{ + StartKey: startKey, + EndKey: endKey, + }) +} + +// Ranges returns the slice of KeyRange. +func (rs *KeyRanges) Ranges() []*KeyRange { + if rs == nil { + return nil + } + return rs.krs +} diff --git a/pkg/mcs/scheduling/server/apis/v1/api.go b/pkg/mcs/scheduling/server/apis/v1/api.go index b59780b7a61..e6881f2f85c 100644 --- a/pkg/mcs/scheduling/server/apis/v1/api.go +++ b/pkg/mcs/scheduling/server/apis/v1/api.go @@ -1330,5 +1330,5 @@ func checkRegionsReplicated(c *gin.Context) { c.String(http.StatusBadRequest, err.Error()) return } - c.String(http.StatusOK, state) + c.IndentedJSON(http.StatusOK, state) } diff --git a/pkg/mcs/scheduling/server/rule/watcher.go b/pkg/mcs/scheduling/server/rule/watcher.go index 96e19cf5002..3e11cf9ff9d 100644 --- a/pkg/mcs/scheduling/server/rule/watcher.go +++ b/pkg/mcs/scheduling/server/rule/watcher.go @@ -20,6 +20,7 @@ import ( "sync" "github.com/pingcap/log" + "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/schedule/checker" "github.com/tikv/pd/pkg/schedule/labeler" "github.com/tikv/pd/pkg/schedule/placement" @@ -36,6 +37,10 @@ type Watcher struct { cancel context.CancelFunc wg sync.WaitGroup + // ruleCommonPathPrefix: + // - Key: /pd/{cluster_id}/rule + // - Value: placement.Rule or placement.RuleGroup + ruleCommonPathPrefix string // rulesPathPrefix: // - Key: /pd/{cluster_id}/rules/{group_id}-{rule_id} // - Value: placement.Rule @@ -60,8 +65,10 @@ type Watcher struct { regionLabeler *labeler.RegionLabeler ruleWatcher *etcdutil.LoopWatcher - groupWatcher *etcdutil.LoopWatcher labelWatcher *etcdutil.LoopWatcher + + // patch is used to cache the placement rule changes. + patch *placement.RuleConfigPatch } // NewWatcher creates a new watcher to watch the Placement Rule change from PD API server. @@ -79,6 +86,7 @@ func NewWatcher( ctx: ctx, cancel: cancel, rulesPathPrefix: endpoint.RulesPathPrefix(clusterID), + ruleCommonPathPrefix: endpoint.RuleCommonPathPrefix(clusterID), ruleGroupPathPrefix: endpoint.RuleGroupPathPrefix(clusterID), regionLabelPathPrefix: endpoint.RegionLabelPathPrefix(clusterID), etcdClient: etcdClient, @@ -91,10 +99,6 @@ func NewWatcher( if err != nil { return nil, err } - err = rw.initializeGroupWatcher() - if err != nil { - return nil, err - } err = rw.initializeRegionLabelWatcher() if err != nil { return nil, err @@ -103,83 +107,109 @@ func NewWatcher( } func (rw *Watcher) initializeRuleWatcher() error { - prefixToTrim := rw.rulesPathPrefix + "/" + var suspectKeyRanges *core.KeyRanges + + preEventsFn := func(events []*clientv3.Event) error { + // It will be locked until the postFn is finished. + rw.ruleManager.Lock() + rw.patch = rw.ruleManager.BeginPatch() + suspectKeyRanges = &core.KeyRanges{} + return nil + } + putFn := func(kv *mvccpb.KeyValue) error { - log.Info("update placement rule", zap.String("key", string(kv.Key)), zap.String("value", string(kv.Value))) - rule, err := placement.NewRuleFromJSON(kv.Value) - if err != nil { - return err - } - // Update the suspect key ranges in the checker. - rw.checkerController.AddSuspectKeyRange(rule.StartKey, rule.EndKey) - if oldRule := rw.ruleManager.GetRule(rule.GroupID, rule.ID); oldRule != nil { - rw.checkerController.AddSuspectKeyRange(oldRule.StartKey, oldRule.EndKey) + key := string(kv.Key) + if strings.HasPrefix(key, rw.rulesPathPrefix) { + log.Info("update placement rule", zap.String("key", key), zap.String("value", string(kv.Value))) + rule, err := placement.NewRuleFromJSON(kv.Value) + if err != nil { + return err + } + // Try to add the rule change to the patch. + if err := rw.ruleManager.AdjustRule(rule, ""); err != nil { + return err + } + rw.patch.SetRule(rule) + // Update the suspect key ranges in lock. + suspectKeyRanges.Append(rule.StartKey, rule.EndKey) + if oldRule := rw.ruleManager.GetRuleLocked(rule.GroupID, rule.ID); oldRule != nil { + suspectKeyRanges.Append(oldRule.StartKey, oldRule.EndKey) + } + return nil + } else if strings.HasPrefix(key, rw.ruleGroupPathPrefix) { + log.Info("update placement rule group", zap.String("key", key), zap.String("value", string(kv.Value))) + ruleGroup, err := placement.NewRuleGroupFromJSON(kv.Value) + if err != nil { + return err + } + // Try to add the rule group change to the patch. + rw.patch.SetGroup(ruleGroup) + // Update the suspect key ranges + for _, rule := range rw.ruleManager.GetRulesByGroupLocked(ruleGroup.ID) { + suspectKeyRanges.Append(rule.StartKey, rule.EndKey) + } + return nil + } else { + log.Warn("unknown key when updating placement rule", zap.String("key", key)) + return nil } - return rw.ruleManager.SetRule(rule) } deleteFn := func(kv *mvccpb.KeyValue) error { key := string(kv.Key) - log.Info("delete placement rule", zap.String("key", key)) - ruleJSON, err := rw.ruleStorage.LoadRule(strings.TrimPrefix(key, prefixToTrim)) - if err != nil { + if strings.HasPrefix(key, rw.rulesPathPrefix) { + log.Info("delete placement rule", zap.String("key", key)) + ruleJSON, err := rw.ruleStorage.LoadRule(strings.TrimPrefix(key, rw.rulesPathPrefix+"/")) + if err != nil { + return err + } + rule, err := placement.NewRuleFromJSON([]byte(ruleJSON)) + if err != nil { + return err + } + // Try to add the rule change to the patch. + rw.patch.DeleteRule(rule.GroupID, rule.ID) + // Update the suspect key ranges + suspectKeyRanges.Append(rule.StartKey, rule.EndKey) return err + } else if strings.HasPrefix(key, rw.ruleGroupPathPrefix) { + log.Info("delete placement rule group", zap.String("key", key)) + trimmedKey := strings.TrimPrefix(key, rw.ruleGroupPathPrefix+"/") + // Try to add the rule group change to the patch. + rw.patch.DeleteGroup(trimmedKey) + // Update the suspect key ranges + for _, rule := range rw.ruleManager.GetRulesByGroupLocked(trimmedKey) { + suspectKeyRanges.Append(rule.StartKey, rule.EndKey) + } + return nil + } else { + log.Warn("unknown key when deleting placement rule", zap.String("key", key)) + return nil } - rule, err := placement.NewRuleFromJSON([]byte(ruleJSON)) - if err != nil { + } + postEventsFn := func(events []*clientv3.Event) error { + defer rw.ruleManager.Unlock() + if err := rw.ruleManager.TryCommitPatch(rw.patch); err != nil { + log.Error("failed to commit patch", zap.Error(err)) return err } - rw.checkerController.AddSuspectKeyRange(rule.StartKey, rule.EndKey) - return rw.ruleManager.DeleteRule(rule.GroupID, rule.ID) + for _, kr := range suspectKeyRanges.Ranges() { + rw.checkerController.AddSuspectKeyRange(kr.StartKey, kr.EndKey) + } + return nil } rw.ruleWatcher = etcdutil.NewLoopWatcher( rw.ctx, &rw.wg, rw.etcdClient, - "scheduling-rule-watcher", rw.rulesPathPrefix, - func([]*clientv3.Event) error { return nil }, + "scheduling-rule-watcher", rw.ruleCommonPathPrefix, + preEventsFn, putFn, deleteFn, - func([]*clientv3.Event) error { return nil }, + postEventsFn, clientv3.WithPrefix(), ) rw.ruleWatcher.StartWatchLoop() return rw.ruleWatcher.WaitLoad() } -func (rw *Watcher) initializeGroupWatcher() error { - prefixToTrim := rw.ruleGroupPathPrefix + "/" - putFn := func(kv *mvccpb.KeyValue) error { - log.Info("update placement rule group", zap.String("key", string(kv.Key)), zap.String("value", string(kv.Value))) - ruleGroup, err := placement.NewRuleGroupFromJSON(kv.Value) - if err != nil { - return err - } - // Add all rule key ranges within the group to the suspect key ranges. - for _, rule := range rw.ruleManager.GetRulesByGroup(ruleGroup.ID) { - rw.checkerController.AddSuspectKeyRange(rule.StartKey, rule.EndKey) - } - return rw.ruleManager.SetRuleGroup(ruleGroup) - } - deleteFn := func(kv *mvccpb.KeyValue) error { - key := string(kv.Key) - log.Info("delete placement rule group", zap.String("key", key)) - trimmedKey := strings.TrimPrefix(key, prefixToTrim) - for _, rule := range rw.ruleManager.GetRulesByGroup(trimmedKey) { - rw.checkerController.AddSuspectKeyRange(rule.StartKey, rule.EndKey) - } - return rw.ruleManager.DeleteRuleGroup(trimmedKey) - } - rw.groupWatcher = etcdutil.NewLoopWatcher( - rw.ctx, &rw.wg, - rw.etcdClient, - "scheduling-rule-group-watcher", rw.ruleGroupPathPrefix, - func([]*clientv3.Event) error { return nil }, - putFn, deleteFn, - func([]*clientv3.Event) error { return nil }, - clientv3.WithPrefix(), - ) - rw.groupWatcher.StartWatchLoop() - return rw.groupWatcher.WaitLoad() -} - func (rw *Watcher) initializeRegionLabelWatcher() error { prefixToTrim := rw.regionLabelPathPrefix + "/" putFn := func(kv *mvccpb.KeyValue) error { diff --git a/pkg/schedule/placement/config.go b/pkg/schedule/placement/config.go index 878db4b2e0a..00c0f94b94e 100644 --- a/pkg/schedule/placement/config.go +++ b/pkg/schedule/placement/config.go @@ -79,28 +79,30 @@ func (c *ruleConfig) getGroup(id string) *RuleGroup { return &RuleGroup{ID: id} } -func (c *ruleConfig) beginPatch() *ruleConfigPatch { - return &ruleConfigPatch{ +func (c *ruleConfig) beginPatch() *RuleConfigPatch { + return &RuleConfigPatch{ c: c, mut: newRuleConfig(), } } -// A helper data structure to update ruleConfig. -type ruleConfigPatch struct { +// RuleConfigPatch is a helper data structure to update ruleConfig. +type RuleConfigPatch struct { c *ruleConfig // original configuration to be updated mut *ruleConfig // record all to-commit rules and groups } -func (p *ruleConfigPatch) setRule(r *Rule) { +// SetRule sets a rule to the patch. +func (p *RuleConfigPatch) SetRule(r *Rule) { p.mut.rules[r.Key()] = r } -func (p *ruleConfigPatch) deleteRule(group, id string) { +// DeleteRule deletes a rule from the patch. +func (p *RuleConfigPatch) DeleteRule(group, id string) { p.mut.rules[[2]string{group, id}] = nil } -func (p *ruleConfigPatch) getGroup(id string) *RuleGroup { +func (p *RuleConfigPatch) getGroup(id string) *RuleGroup { if g, ok := p.mut.groups[id]; ok { return g } @@ -110,15 +112,17 @@ func (p *ruleConfigPatch) getGroup(id string) *RuleGroup { return &RuleGroup{ID: id} } -func (p *ruleConfigPatch) setGroup(g *RuleGroup) { +// SetGroup sets a group to the patch. +func (p *RuleConfigPatch) SetGroup(g *RuleGroup) { p.mut.groups[g.ID] = g } -func (p *ruleConfigPatch) deleteGroup(id string) { - p.setGroup(&RuleGroup{ID: id}) +// DeleteGroup deletes a group from the patch. +func (p *RuleConfigPatch) DeleteGroup(id string) { + p.SetGroup(&RuleGroup{ID: id}) } -func (p *ruleConfigPatch) iterateRules(f func(*Rule)) { +func (p *RuleConfigPatch) iterateRules(f func(*Rule)) { for _, r := range p.mut.rules { if r != nil { // nil means delete. f(r) @@ -131,13 +135,13 @@ func (p *ruleConfigPatch) iterateRules(f func(*Rule)) { } } -func (p *ruleConfigPatch) adjust() { +func (p *RuleConfigPatch) adjust() { // setup rule.group for `buildRuleList` use. p.iterateRules(func(r *Rule) { r.group = p.getGroup(r.GroupID) }) } // trim unnecessary updates. For example, remove a rule then insert the same rule. -func (p *ruleConfigPatch) trim() { +func (p *RuleConfigPatch) trim() { for key, rule := range p.mut.rules { if jsonEquals(rule, p.c.getRule(key)) { delete(p.mut.rules, key) @@ -151,7 +155,7 @@ func (p *ruleConfigPatch) trim() { } // merge all mutations to ruleConfig. -func (p *ruleConfigPatch) commit() { +func (p *RuleConfigPatch) commit() { for key, rule := range p.mut.rules { if rule == nil { delete(p.c.rules, key) diff --git a/pkg/schedule/placement/config_test.go b/pkg/schedule/placement/config_test.go index 8f7161a56d7..ccee8837331 100644 --- a/pkg/schedule/placement/config_test.go +++ b/pkg/schedule/placement/config_test.go @@ -30,40 +30,40 @@ func TestTrim(t *testing.T) { rc.setGroup(&RuleGroup{ID: "g2", Index: 2}) testCases := []struct { - ops func(p *ruleConfigPatch) + ops func(p *RuleConfigPatch) mutRules map[[2]string]*Rule mutGroups map[string]*RuleGroup }{ { - func(p *ruleConfigPatch) { - p.setRule(&Rule{GroupID: "g1", ID: "id1", Index: 100}) - p.setRule(&Rule{GroupID: "g1", ID: "id2"}) - p.setGroup(&RuleGroup{ID: "g1", Index: 100}) - p.setGroup(&RuleGroup{ID: "g2", Index: 2}) + func(p *RuleConfigPatch) { + p.SetRule(&Rule{GroupID: "g1", ID: "id1", Index: 100}) + p.SetRule(&Rule{GroupID: "g1", ID: "id2"}) + p.SetGroup(&RuleGroup{ID: "g1", Index: 100}) + p.SetGroup(&RuleGroup{ID: "g2", Index: 2}) }, map[[2]string]*Rule{{"g1", "id1"}: {GroupID: "g1", ID: "id1", Index: 100}}, map[string]*RuleGroup{"g1": {ID: "g1", Index: 100}}, }, { - func(p *ruleConfigPatch) { - p.deleteRule("g1", "id1") - p.deleteGroup("g2") - p.deleteRule("g3", "id3") - p.deleteGroup("g3") + func(p *RuleConfigPatch) { + p.DeleteRule("g1", "id1") + p.DeleteGroup("g2") + p.DeleteRule("g3", "id3") + p.DeleteGroup("g3") }, map[[2]string]*Rule{{"g1", "id1"}: nil}, map[string]*RuleGroup{"g2": {ID: "g2"}}, }, { - func(p *ruleConfigPatch) { - p.setRule(&Rule{GroupID: "g1", ID: "id2", Index: 200}) - p.setRule(&Rule{GroupID: "g1", ID: "id2"}) - p.setRule(&Rule{GroupID: "g3", ID: "id3"}) - p.deleteRule("g3", "id3") - p.setGroup(&RuleGroup{ID: "g1", Index: 100}) - p.setGroup(&RuleGroup{ID: "g1", Index: 1}) - p.setGroup(&RuleGroup{ID: "g3", Index: 3}) - p.deleteGroup("g3") + func(p *RuleConfigPatch) { + p.SetRule(&Rule{GroupID: "g1", ID: "id2", Index: 200}) + p.SetRule(&Rule{GroupID: "g1", ID: "id2"}) + p.SetRule(&Rule{GroupID: "g3", ID: "id3"}) + p.DeleteRule("g3", "id3") + p.SetGroup(&RuleGroup{ID: "g1", Index: 100}) + p.SetGroup(&RuleGroup{ID: "g1", Index: 1}) + p.SetGroup(&RuleGroup{ID: "g3", Index: 3}) + p.DeleteGroup("g3") }, map[[2]string]*Rule{}, map[string]*RuleGroup{}, diff --git a/pkg/schedule/placement/rule_manager.go b/pkg/schedule/placement/rule_manager.go index 621c52d738e..ea85911462b 100644 --- a/pkg/schedule/placement/rule_manager.go +++ b/pkg/schedule/placement/rule_manager.go @@ -33,6 +33,7 @@ import ( "github.com/tikv/pd/pkg/schedule/config" "github.com/tikv/pd/pkg/slice" "github.com/tikv/pd/pkg/storage/endpoint" + "github.com/tikv/pd/pkg/storage/kv" "github.com/tikv/pd/pkg/utils/syncutil" "go.uber.org/zap" "golang.org/x/exp/slices" @@ -128,12 +129,17 @@ func (m *RuleManager) Initialize(maxReplica int, locationLabels []string, isolat IsolationLevel: isolationLevel, }) } - for _, defaultRule := range defaultRules { - if err := m.storage.SaveRule(defaultRule.StoreKey(), defaultRule); err != nil { - // TODO: Need to delete the previously successfully saved Rules? - return err + if err := m.storage.RunInTxn(m.ctx, func(txn kv.Txn) (err error) { + for _, defaultRule := range defaultRules { + if err := m.storage.SaveRule(txn, defaultRule.StoreKey(), defaultRule); err != nil { + // TODO: Need to delete the previously successfully saved Rules? + return err + } + m.ruleConfig.setRule(defaultRule) } - m.ruleConfig.setRule(defaultRule) + return nil + }); err != nil { + return err } } m.ruleConfig.adjust() @@ -151,61 +157,66 @@ func (m *RuleManager) loadRules() error { toSave []*Rule toDelete []string ) - err := m.storage.LoadRules(func(k, v string) { - r, err := NewRuleFromJSON([]byte(v)) - if err != nil { - log.Error("failed to unmarshal rule value", zap.String("rule-key", k), zap.String("rule-value", v), errs.ZapError(errs.ErrLoadRule)) - toDelete = append(toDelete, k) - return - } - err = m.adjustRule(r, "") + return m.storage.RunInTxn(m.ctx, func(txn kv.Txn) (err error) { + err = m.storage.LoadRules(txn, func(k, v string) { + r, err := NewRuleFromJSON([]byte(v)) + if err != nil { + log.Error("failed to unmarshal rule value", zap.String("rule-key", k), zap.String("rule-value", v), errs.ZapError(errs.ErrLoadRule)) + toDelete = append(toDelete, k) + return + } + err = m.AdjustRule(r, "") + if err != nil { + log.Error("rule is in bad format", zap.String("rule-key", k), zap.String("rule-value", v), errs.ZapError(errs.ErrLoadRule, err)) + toDelete = append(toDelete, k) + return + } + _, ok := m.ruleConfig.rules[r.Key()] + if ok { + log.Error("duplicated rule key", zap.String("rule-key", k), zap.String("rule-value", v), errs.ZapError(errs.ErrLoadRule)) + toDelete = append(toDelete, k) + return + } + if k != r.StoreKey() { + log.Error("mismatch data key, need to restore", zap.String("rule-key", k), zap.String("rule-value", v), errs.ZapError(errs.ErrLoadRule)) + toDelete = append(toDelete, k) + toSave = append(toSave, r) + } + m.ruleConfig.rules[r.Key()] = r + }) if err != nil { - log.Error("rule is in bad format", zap.String("rule-key", k), zap.String("rule-value", v), errs.ZapError(errs.ErrLoadRule, err)) - toDelete = append(toDelete, k) - return + return err } - _, ok := m.ruleConfig.rules[r.Key()] - if ok { - log.Error("duplicated rule key", zap.String("rule-key", k), zap.String("rule-value", v), errs.ZapError(errs.ErrLoadRule)) - toDelete = append(toDelete, k) - return + + for _, s := range toSave { + if err = m.storage.SaveRule(txn, s.StoreKey(), s); err != nil { + return err + } } - if k != r.StoreKey() { - log.Error("mismatch data key, need to restore", zap.String("rule-key", k), zap.String("rule-value", v), errs.ZapError(errs.ErrLoadRule)) - toDelete = append(toDelete, k) - toSave = append(toSave, r) + for _, d := range toDelete { + if err = m.storage.DeleteRule(txn, d); err != nil { + return err + } } - m.ruleConfig.rules[r.Key()] = r + return nil }) - if err != nil { - return err - } - for _, s := range toSave { - if err = m.storage.SaveRule(s.StoreKey(), s); err != nil { - return err - } - } - for _, d := range toDelete { - if err = m.storage.DeleteRule(d); err != nil { - return err - } - } - return nil } func (m *RuleManager) loadGroups() error { - return m.storage.LoadRuleGroups(func(k, v string) { - g, err := NewRuleGroupFromJSON([]byte(v)) - if err != nil { - log.Error("failed to unmarshal rule group", zap.String("group-id", k), errs.ZapError(errs.ErrLoadRuleGroup, err)) - return - } - m.ruleConfig.groups[g.ID] = g + return m.storage.RunInTxn(m.ctx, func(txn kv.Txn) (err error) { + return m.storage.LoadRuleGroups(txn, func(k, v string) { + g, err := NewRuleGroupFromJSON([]byte(v)) + if err != nil { + log.Error("failed to unmarshal rule group", zap.String("group-id", k), errs.ZapError(errs.ErrLoadRuleGroup, err)) + return + } + m.ruleConfig.groups[g.ID] = g + }) }) } -// check and adjust rule from client or storage. -func (m *RuleManager) adjustRule(r *Rule, groupID string) (err error) { +// AdjustRule check and adjust rule from client or storage. +func (m *RuleManager) AdjustRule(r *Rule, groupID string) (err error) { r.StartKey, err = hex.DecodeString(r.StartKeyHex) if err != nil { return errs.ErrHexDecodingString.FastGenByArgs(r.StartKeyHex) @@ -279,6 +290,11 @@ func (m *RuleManager) adjustRule(r *Rule, groupID string) (err error) { func (m *RuleManager) GetRule(group, id string) *Rule { m.RLock() defer m.RUnlock() + return m.GetRuleLocked(group, id) +} + +// GetRuleLocked returns the Rule with the same (group, id). +func (m *RuleManager) GetRuleLocked(group, id string) *Rule { if r := m.ruleConfig.getRule([2]string{group, id}); r != nil { return r.Clone() } @@ -287,14 +303,14 @@ func (m *RuleManager) GetRule(group, id string) *Rule { // SetRule inserts or updates a Rule. func (m *RuleManager) SetRule(rule *Rule) error { - if err := m.adjustRule(rule, ""); err != nil { + if err := m.AdjustRule(rule, ""); err != nil { return err } m.Lock() defer m.Unlock() - p := m.beginPatch() - p.setRule(rule) - if err := m.tryCommitPatch(p); err != nil { + p := m.BeginPatch() + p.SetRule(rule) + if err := m.TryCommitPatch(p); err != nil { return err } log.Info("placement rule updated", zap.String("rule", fmt.Sprint(rule))) @@ -305,9 +321,9 @@ func (m *RuleManager) SetRule(rule *Rule) error { func (m *RuleManager) DeleteRule(group, id string) error { m.Lock() defer m.Unlock() - p := m.beginPatch() - p.deleteRule(group, id) - if err := m.tryCommitPatch(p); err != nil { + p := m.BeginPatch() + p.DeleteRule(group, id) + if err := m.TryCommitPatch(p); err != nil { return err } log.Info("placement rule is removed", zap.String("group", group), zap.String("id", id)) @@ -351,6 +367,11 @@ func (m *RuleManager) GetGroupsCount() int { func (m *RuleManager) GetRulesByGroup(group string) []*Rule { m.RLock() defer m.RUnlock() + return m.GetRulesByGroupLocked(group) +} + +// GetRulesByGroupLocked returns sorted rules of a group. +func (m *RuleManager) GetRulesByGroupLocked(group string) []*Rule { var rules []*Rule for _, r := range m.ruleConfig.rules { if r.GroupID == group { @@ -442,11 +463,13 @@ func (m *RuleManager) CheckIsCachedDirectly(regionID uint64) bool { return ok } -func (m *RuleManager) beginPatch() *ruleConfigPatch { +// BeginPatch returns a patch for multiple changes. +func (m *RuleManager) BeginPatch() *RuleConfigPatch { return m.ruleConfig.beginPatch() } -func (m *RuleManager) tryCommitPatch(patch *ruleConfigPatch) error { +// TryCommitPatch tries to commit a patch. +func (m *RuleManager) TryCommitPatch(patch *RuleConfigPatch) error { patch.adjust() ruleList, err := buildRuleList(patch) @@ -469,49 +492,44 @@ func (m *RuleManager) tryCommitPatch(patch *ruleConfigPatch) error { } func (m *RuleManager) savePatch(p *ruleConfig) error { - // TODO: it is not completely safe - // 1. in case that half of rules applied, error.. we have to cancel persisted rules - // but that may fail too, causing memory/disk inconsistency - // either rely a transaction API, or clients to request again until success - // 2. in case that PD is suddenly down in the loop, inconsistency again - // now we can only rely clients to request again - var err error - for key, r := range p.rules { - if r == nil { - r = &Rule{GroupID: key[0], ID: key[1]} - err = m.storage.DeleteRule(r.StoreKey()) - } else { - err = m.storage.SaveRule(r.StoreKey(), r) - } - if err != nil { - return err - } - } - for id, g := range p.groups { - if g.isDefault() { - err = m.storage.DeleteRuleGroup(id) - } else { - err = m.storage.SaveRuleGroup(id, g) + return m.storage.RunInTxn(m.ctx, func(txn kv.Txn) (err error) { + for key, r := range p.rules { + if r == nil { + r = &Rule{GroupID: key[0], ID: key[1]} + err = m.storage.DeleteRule(txn, r.StoreKey()) + } else { + err = m.storage.SaveRule(txn, r.StoreKey(), r) + } + if err != nil { + return err + } } - if err != nil { - return err + for id, g := range p.groups { + if g.isDefault() { + err = m.storage.DeleteRuleGroup(txn, id) + } else { + err = m.storage.SaveRuleGroup(txn, id, g) + } + if err != nil { + return err + } } - } - return nil + return nil + }) } // SetRules inserts or updates lots of Rules at once. func (m *RuleManager) SetRules(rules []*Rule) error { m.Lock() defer m.Unlock() - p := m.beginPatch() + p := m.BeginPatch() for _, r := range rules { - if err := m.adjustRule(r, ""); err != nil { + if err := m.AdjustRule(r, ""); err != nil { return err } - p.setRule(r) + p.SetRule(r) } - if err := m.tryCommitPatch(p); err != nil { + if err := m.TryCommitPatch(p); err != nil { return err } @@ -546,7 +564,7 @@ func (r RuleOp) String() string { func (m *RuleManager) Batch(todo []RuleOp) error { for _, t := range todo { if t.Action == RuleOpAdd { - err := m.adjustRule(t.Rule, "") + err := m.AdjustRule(t.Rule, "") if err != nil { return err } @@ -556,25 +574,25 @@ func (m *RuleManager) Batch(todo []RuleOp) error { m.Lock() defer m.Unlock() - patch := m.beginPatch() + patch := m.BeginPatch() for _, t := range todo { switch t.Action { case RuleOpAdd: - patch.setRule(t.Rule) + patch.SetRule(t.Rule) case RuleOpDel: if !t.DeleteByIDPrefix { - patch.deleteRule(t.GroupID, t.ID) + patch.DeleteRule(t.GroupID, t.ID) } else { m.ruleConfig.iterateRules(func(r *Rule) { if r.GroupID == t.GroupID && strings.HasPrefix(r.ID, t.ID) { - patch.deleteRule(r.GroupID, r.ID) + patch.DeleteRule(r.GroupID, r.ID) } }) } } } - if err := m.tryCommitPatch(patch); err != nil { + if err := m.TryCommitPatch(patch); err != nil { return err } @@ -608,9 +626,9 @@ func (m *RuleManager) GetRuleGroups() []*RuleGroup { func (m *RuleManager) SetRuleGroup(group *RuleGroup) error { m.Lock() defer m.Unlock() - p := m.beginPatch() - p.setGroup(group) - if err := m.tryCommitPatch(p); err != nil { + p := m.BeginPatch() + p.SetGroup(group) + if err := m.TryCommitPatch(p); err != nil { return err } log.Info("group config updated", zap.String("group", fmt.Sprint(group))) @@ -621,9 +639,9 @@ func (m *RuleManager) SetRuleGroup(group *RuleGroup) error { func (m *RuleManager) DeleteRuleGroup(id string) error { m.Lock() defer m.Unlock() - p := m.beginPatch() - p.deleteGroup(id) - if err := m.tryCommitPatch(p); err != nil { + p := m.BeginPatch() + p.DeleteGroup(id) + if err := m.TryCommitPatch(p); err != nil { return err } log.Info("group config reset", zap.String("group", id)) @@ -681,7 +699,7 @@ func (m *RuleManager) GetGroupBundle(id string) (b GroupBundle) { func (m *RuleManager) SetAllGroupBundles(groups []GroupBundle, override bool) error { m.Lock() defer m.Unlock() - p := m.beginPatch() + p := m.BeginPatch() matchID := func(a string) bool { for _, g := range groups { if g.ID == a { @@ -692,28 +710,28 @@ func (m *RuleManager) SetAllGroupBundles(groups []GroupBundle, override bool) er } for k := range m.ruleConfig.rules { if override || matchID(k[0]) { - p.deleteRule(k[0], k[1]) + p.DeleteRule(k[0], k[1]) } } for id := range m.ruleConfig.groups { if override || matchID(id) { - p.deleteGroup(id) + p.DeleteGroup(id) } } for _, g := range groups { - p.setGroup(&RuleGroup{ + p.SetGroup(&RuleGroup{ ID: g.ID, Index: g.Index, Override: g.Override, }) for _, r := range g.Rules { - if err := m.adjustRule(r, g.ID); err != nil { + if err := m.AdjustRule(r, g.ID); err != nil { return err } - p.setRule(r) + p.SetRule(r) } } - if err := m.tryCommitPatch(p); err != nil { + if err := m.TryCommitPatch(p); err != nil { return err } log.Info("full config reset", zap.String("config", fmt.Sprint(groups))) @@ -725,26 +743,26 @@ func (m *RuleManager) SetAllGroupBundles(groups []GroupBundle, override bool) er func (m *RuleManager) SetGroupBundle(group GroupBundle) error { m.Lock() defer m.Unlock() - p := m.beginPatch() + p := m.BeginPatch() if _, ok := m.ruleConfig.groups[group.ID]; ok { for k := range m.ruleConfig.rules { if k[0] == group.ID { - p.deleteRule(k[0], k[1]) + p.DeleteRule(k[0], k[1]) } } } - p.setGroup(&RuleGroup{ + p.SetGroup(&RuleGroup{ ID: group.ID, Index: group.Index, Override: group.Override, }) for _, r := range group.Rules { - if err := m.adjustRule(r, group.ID); err != nil { + if err := m.AdjustRule(r, group.ID); err != nil { return err } - p.setRule(r) + p.SetRule(r) } - if err := m.tryCommitPatch(p); err != nil { + if err := m.TryCommitPatch(p); err != nil { return err } log.Info("group is reset", zap.String("group", fmt.Sprint(group))) @@ -765,18 +783,18 @@ func (m *RuleManager) DeleteGroupBundle(id string, regex bool) error { matchID = r.MatchString } - p := m.beginPatch() + p := m.BeginPatch() for k := range m.ruleConfig.rules { if matchID(k[0]) { - p.deleteRule(k[0], k[1]) + p.DeleteRule(k[0], k[1]) } } for _, g := range m.ruleConfig.groups { if matchID(g.ID) { - p.deleteGroup(g.ID) + p.DeleteGroup(g.ID) } } - if err := m.tryCommitPatch(p); err != nil { + if err := m.TryCommitPatch(p); err != nil { return err } log.Info("groups are removed", zap.String("id", id), zap.Bool("regexp", regex)) diff --git a/pkg/schedule/placement/rule_manager_test.go b/pkg/schedule/placement/rule_manager_test.go index 0539e935113..5494b3c5a9d 100644 --- a/pkg/schedule/placement/rule_manager_test.go +++ b/pkg/schedule/placement/rule_manager_test.go @@ -91,23 +91,23 @@ func TestAdjustRule(t *testing.T) { {GroupID: "group", ID: "id", StartKeyHex: "123abc", EndKeyHex: "123abf", Role: Voter, Count: -1}, {GroupID: "group", ID: "id", StartKeyHex: "123abc", EndKeyHex: "123abf", Role: Voter, Count: 3, LabelConstraints: []LabelConstraint{{Op: "foo"}}}, } - re.NoError(manager.adjustRule(&rules[0], "group")) + re.NoError(manager.AdjustRule(&rules[0], "group")) re.Equal([]byte{0x12, 0x3a, 0xbc}, rules[0].StartKey) re.Equal([]byte{0x12, 0x3a, 0xbf}, rules[0].EndKey) - re.Error(manager.adjustRule(&rules[1], "")) + re.Error(manager.AdjustRule(&rules[1], "")) for i := 2; i < len(rules); i++ { - re.Error(manager.adjustRule(&rules[i], "group")) + re.Error(manager.AdjustRule(&rules[i], "group")) } manager.SetKeyType(constant.Table.String()) - re.Error(manager.adjustRule(&Rule{GroupID: "group", ID: "id", StartKeyHex: "123abc", EndKeyHex: "123abf", Role: Voter, Count: 3}, "group")) + re.Error(manager.AdjustRule(&Rule{GroupID: "group", ID: "id", StartKeyHex: "123abc", EndKeyHex: "123abf", Role: Voter, Count: 3}, "group")) manager.SetKeyType(constant.Txn.String()) - re.Error(manager.adjustRule(&Rule{GroupID: "group", ID: "id", StartKeyHex: "123abc", EndKeyHex: "123abf", Role: Voter, Count: 3}, "group")) + re.Error(manager.AdjustRule(&Rule{GroupID: "group", ID: "id", StartKeyHex: "123abc", EndKeyHex: "123abf", Role: Voter, Count: 3}, "group")) - re.Error(manager.adjustRule(&Rule{ + re.Error(manager.AdjustRule(&Rule{ GroupID: "group", ID: "id", StartKeyHex: hex.EncodeToString(codec.EncodeBytes([]byte{0})), @@ -116,7 +116,7 @@ func TestAdjustRule(t *testing.T) { Count: 3, }, "group")) - re.Error(manager.adjustRule(&Rule{ + re.Error(manager.AdjustRule(&Rule{ GroupID: "tiflash", ID: "id", StartKeyHex: hex.EncodeToString(codec.EncodeBytes([]byte{0})), diff --git a/pkg/storage/endpoint/rule.go b/pkg/storage/endpoint/rule.go index 80b6fc7c0ff..b18360040ea 100644 --- a/pkg/storage/endpoint/rule.go +++ b/pkg/storage/endpoint/rule.go @@ -14,58 +14,54 @@ package endpoint +import ( + "context" + + "github.com/tikv/pd/pkg/storage/kv" +) + // RuleStorage defines the storage operations on the rule. type RuleStorage interface { + LoadRules(txn kv.Txn, f func(k, v string)) error + SaveRule(txn kv.Txn, ruleKey string, rule interface{}) error + DeleteRule(txn kv.Txn, ruleKey string) error + LoadRuleGroups(txn kv.Txn, f func(k, v string)) error + SaveRuleGroup(txn kv.Txn, groupID string, group interface{}) error + DeleteRuleGroup(txn kv.Txn, groupID string) error + // LoadRule is used only in rule watcher. LoadRule(ruleKey string) (string, error) - LoadRules(f func(k, v string)) error - SaveRule(ruleKey string, rule interface{}) error - SaveRuleJSON(ruleKey, rule string) error - DeleteRule(ruleKey string) error - LoadRuleGroups(f func(k, v string)) error - SaveRuleGroup(groupID string, group interface{}) error - SaveRuleGroupJSON(groupID, group string) error - DeleteRuleGroup(groupID string) error + LoadRegionRules(f func(k, v string)) error SaveRegionRule(ruleKey string, rule interface{}) error - SaveRegionRuleJSON(ruleKey, rule string) error DeleteRegionRule(ruleKey string) error + RunInTxn(ctx context.Context, f func(txn kv.Txn) error) error } var _ RuleStorage = (*StorageEndpoint)(nil) // SaveRule stores a rule cfg to the rulesPath. -func (se *StorageEndpoint) SaveRule(ruleKey string, rule interface{}) error { - return se.saveJSON(ruleKeyPath(ruleKey), rule) -} - -// SaveRuleJSON stores a rule cfg JSON to the rulesPath. -func (se *StorageEndpoint) SaveRuleJSON(ruleKey, rule string) error { - return se.Save(ruleKeyPath(ruleKey), rule) +func (se *StorageEndpoint) SaveRule(txn kv.Txn, ruleKey string, rule interface{}) error { + return saveJSONInTxn(txn, ruleKeyPath(ruleKey), rule) } // DeleteRule removes a rule from storage. -func (se *StorageEndpoint) DeleteRule(ruleKey string) error { - return se.Remove(ruleKeyPath(ruleKey)) +func (se *StorageEndpoint) DeleteRule(txn kv.Txn, ruleKey string) error { + return txn.Remove(ruleKeyPath(ruleKey)) } // LoadRuleGroups loads all rule groups from storage. -func (se *StorageEndpoint) LoadRuleGroups(f func(k, v string)) error { - return se.loadRangeByPrefix(ruleGroupPath+"/", f) +func (se *StorageEndpoint) LoadRuleGroups(txn kv.Txn, f func(k, v string)) error { + return loadRangeByPrefixInTxn(txn, ruleGroupPath+"/", f) } // SaveRuleGroup stores a rule group config to storage. -func (se *StorageEndpoint) SaveRuleGroup(groupID string, group interface{}) error { - return se.saveJSON(ruleGroupIDPath(groupID), group) -} - -// SaveRuleGroupJSON stores a rule group config JSON to storage. -func (se *StorageEndpoint) SaveRuleGroupJSON(groupID, group string) error { - return se.Save(ruleGroupIDPath(groupID), group) +func (se *StorageEndpoint) SaveRuleGroup(txn kv.Txn, groupID string, group interface{}) error { + return saveJSONInTxn(txn, ruleGroupIDPath(groupID), group) } // DeleteRuleGroup removes a rule group from storage. -func (se *StorageEndpoint) DeleteRuleGroup(groupID string) error { - return se.Remove(ruleGroupIDPath(groupID)) +func (se *StorageEndpoint) DeleteRuleGroup(txn kv.Txn, groupID string) error { + return txn.Remove(ruleGroupIDPath(groupID)) } // LoadRegionRules loads region rules from storage. @@ -78,11 +74,6 @@ func (se *StorageEndpoint) SaveRegionRule(ruleKey string, rule interface{}) erro return se.saveJSON(regionLabelKeyPath(ruleKey), rule) } -// SaveRegionRuleJSON saves a region rule JSON to the storage. -func (se *StorageEndpoint) SaveRegionRuleJSON(ruleKey, rule string) error { - return se.Save(regionLabelKeyPath(ruleKey), rule) -} - // DeleteRegionRule removes a region rule from storage. func (se *StorageEndpoint) DeleteRegionRule(ruleKey string) error { return se.Remove(regionLabelKeyPath(ruleKey)) @@ -94,6 +85,6 @@ func (se *StorageEndpoint) LoadRule(ruleKey string) (string, error) { } // LoadRules loads placement rules from storage. -func (se *StorageEndpoint) LoadRules(f func(k, v string)) error { - return se.loadRangeByPrefix(rulesPath+"/", f) +func (se *StorageEndpoint) LoadRules(txn kv.Txn, f func(k, v string)) error { + return loadRangeByPrefixInTxn(txn, rulesPath+"/", f) } diff --git a/pkg/tso/keyspace_group_manager.go b/pkg/tso/keyspace_group_manager.go index 0e69986f255..c48c066a2aa 100644 --- a/pkg/tso/keyspace_group_manager.go +++ b/pkg/tso/keyspace_group_manager.go @@ -559,7 +559,7 @@ func (kgm *KeyspaceGroupManager) InitializeGroupWatchLoop() error { kgm.deleteKeyspaceGroup(groupID) return nil } - postEventFn := func([]*clientv3.Event) error { + postEventsFn := func([]*clientv3.Event) error { // Retry the groups that are not initialized successfully before. for id, group := range kgm.groupUpdateRetryList { delete(kgm.groupUpdateRetryList, id) @@ -576,7 +576,7 @@ func (kgm *KeyspaceGroupManager) InitializeGroupWatchLoop() error { func([]*clientv3.Event) error { return nil }, putFn, deleteFn, - postEventFn, + postEventsFn, clientv3.WithRange(endKey), ) if kgm.loadKeyspaceGroupsTimeout > 0 { diff --git a/pkg/utils/etcdutil/etcdutil.go b/pkg/utils/etcdutil/etcdutil.go index 0e1b2731474..f6beafee511 100644 --- a/pkg/utils/etcdutil/etcdutil.go +++ b/pkg/utils/etcdutil/etcdutil.go @@ -865,6 +865,16 @@ func (lw *LoopWatcher) load(ctx context.Context) (nextRevision int64, err error) if limit != 0 { limit++ } + if err := lw.preEventsFn([]*clientv3.Event{}); err != nil { + log.Error("run pre event failed in watch loop", zap.String("name", lw.name), + zap.String("key", lw.key), zap.Error(err)) + } + defer func() { + if err := lw.postEventsFn([]*clientv3.Event{}); err != nil { + log.Error("run post event failed in watch loop", zap.String("name", lw.name), + zap.String("key", lw.key), zap.Error(err)) + } + }() for { // Sort by key to get the next key and we don't need to worry about the performance, // Because the default sort is just SortByKey and SortAscend @@ -875,10 +885,6 @@ func (lw *LoopWatcher) load(ctx context.Context) (nextRevision int64, err error) zap.String("key", lw.key), zap.Error(err)) return 0, err } - if err := lw.preEventsFn([]*clientv3.Event{}); err != nil { - log.Error("run pre event failed in watch loop", zap.String("name", lw.name), - zap.String("key", lw.key), zap.Error(err)) - } for i, item := range resp.Kvs { if resp.More && i == len(resp.Kvs)-1 { // The last key is the start key of the next batch. @@ -888,15 +894,15 @@ func (lw *LoopWatcher) load(ctx context.Context) (nextRevision int64, err error) } err = lw.putFn(item) if err != nil { - log.Error("put failed in watch loop when loading", zap.String("name", lw.name), zap.String("key", lw.key), zap.Error(err)) + log.Error("put failed in watch loop when loading", zap.String("name", lw.name), zap.String("watch-key", lw.key), + zap.ByteString("key", item.Key), zap.ByteString("value", item.Value), zap.Error(err)) + } else { + log.Debug("put successfully in watch loop when loading", zap.String("name", lw.name), zap.String("watch-key", lw.key), + zap.ByteString("key", item.Key), zap.ByteString("value", item.Value)) } } // Note: if there are no keys in etcd, the resp.More is false. It also means the load is finished. if !resp.More { - if err := lw.postEventsFn([]*clientv3.Event{}); err != nil { - log.Error("run post event failed in watch loop", zap.String("name", lw.name), - zap.String("key", lw.key), zap.Error(err)) - } return resp.Header.Revision + 1, err } } diff --git a/server/keyspace_service.go b/server/keyspace_service.go index 1718108d73b..11d912a5f54 100644 --- a/server/keyspace_service.go +++ b/server/keyspace_service.go @@ -89,7 +89,10 @@ func (s *KeyspaceServer) WatchKeyspaces(request *keyspacepb.WatchKeyspacesReques deleteFn := func(kv *mvccpb.KeyValue) error { return nil } - postEventFn := func([]*clientv3.Event) error { + postEventsFn := func([]*clientv3.Event) error { + if len(keyspaces) == 0 { + return nil + } defer func() { keyspaces = keyspaces[:0] }() @@ -112,7 +115,7 @@ func (s *KeyspaceServer) WatchKeyspaces(request *keyspacepb.WatchKeyspacesReques func([]*clientv3.Event) error { return nil }, putFn, deleteFn, - postEventFn, + postEventsFn, clientv3.WithRange(clientv3.GetPrefixRangeEnd(startKey)), ) watcher.StartWatchLoop() diff --git a/tests/server/api/region_test.go b/tests/server/api/region_test.go index 450995a6e5e..328c0fcd885 100644 --- a/tests/server/api/region_test.go +++ b/tests/server/api/region_test.go @@ -248,8 +248,7 @@ func (suite *regionTestSuite) checkScatterRegions(cluster *tests.TestCluster) { } func (suite *regionTestSuite) TestCheckRegionsReplicated() { - // Fixme: after delete+set rule, the key range will be empty, so the test will fail in api mode. - suite.env.RunTestInPDMode(suite.checkRegionsReplicated) + suite.env.RunTestInTwoModes(suite.checkRegionsReplicated) } func (suite *regionTestSuite) checkRegionsReplicated(cluster *tests.TestCluster) { @@ -304,6 +303,14 @@ func (suite *regionTestSuite) checkRegionsReplicated(cluster *tests.TestCluster) err = tu.CheckPostJSON(testDialClient, urlPrefix+"/config/placement-rule", data, tu.StatusOK(re)) suite.NoError(err) + tu.Eventually(re, func() bool { + respBundle := make([]placement.GroupBundle, 0) + err = tu.CheckGetJSON(testDialClient, urlPrefix+"/config/placement-rule", nil, + tu.StatusOK(re), tu.ExtractJSON(re, &respBundle)) + suite.NoError(err) + return len(respBundle) == 1 && respBundle[0].ID == "5" + }) + tu.Eventually(re, func() bool { err = tu.ReadGetJSON(re, testDialClient, url, &status) suite.NoError(err) @@ -328,9 +335,19 @@ func (suite *regionTestSuite) checkRegionsReplicated(cluster *tests.TestCluster) err = tu.CheckPostJSON(testDialClient, urlPrefix+"/config/placement-rule", data, tu.StatusOK(re)) suite.NoError(err) - err = tu.ReadGetJSON(re, testDialClient, url, &status) - suite.NoError(err) - suite.Equal("REPLICATED", status) + tu.Eventually(re, func() bool { + respBundle := make([]placement.GroupBundle, 0) + err = tu.CheckGetJSON(testDialClient, urlPrefix+"/config/placement-rule", nil, + tu.StatusOK(re), tu.ExtractJSON(re, &respBundle)) + suite.NoError(err) + return len(respBundle) == 1 && len(respBundle[0].Rules) == 2 + }) + + tu.Eventually(re, func() bool { + err = tu.ReadGetJSON(re, testDialClient, url, &status) + suite.NoError(err) + return status == "REPLICATED" + }) // test multiple bundles bundle = append(bundle, placement.GroupBundle{ @@ -347,17 +364,34 @@ func (suite *regionTestSuite) checkRegionsReplicated(cluster *tests.TestCluster) err = tu.CheckPostJSON(testDialClient, urlPrefix+"/config/placement-rule", data, tu.StatusOK(re)) suite.NoError(err) - err = tu.ReadGetJSON(re, testDialClient, url, &status) - suite.NoError(err) - suite.Equal("INPROGRESS", status) + tu.Eventually(re, func() bool { + respBundle := make([]placement.GroupBundle, 0) + err = tu.CheckGetJSON(testDialClient, urlPrefix+"/config/placement-rule", nil, + tu.StatusOK(re), tu.ExtractJSON(re, &respBundle)) + suite.NoError(err) + if len(respBundle) != 2 { + return false + } + s1 := respBundle[0].ID == "5" && respBundle[1].ID == "6" + s2 := respBundle[0].ID == "6" && respBundle[1].ID == "5" + return s1 || s2 + }) + + tu.Eventually(re, func() bool { + err = tu.ReadGetJSON(re, testDialClient, url, &status) + suite.NoError(err) + return status == "INPROGRESS" + }) r1 = core.NewTestRegionInfo(2, 1, []byte("a"), []byte("b")) r1.GetMeta().Peers = append(r1.GetMeta().Peers, &metapb.Peer{Id: 5, StoreId: 1}, &metapb.Peer{Id: 6, StoreId: 1}, &metapb.Peer{Id: 7, StoreId: 1}) tests.MustPutRegionInfo(re, cluster, r1) - err = tu.ReadGetJSON(re, testDialClient, url, &status) - suite.NoError(err) - suite.Equal("REPLICATED", status) + tu.Eventually(re, func() bool { + err = tu.ReadGetJSON(re, testDialClient, url, &status) + suite.NoError(err) + return status == "REPLICATED" + }) } func (suite *regionTestSuite) checkRegionCount(cluster *tests.TestCluster, count uint64) { diff --git a/tests/server/api/rule_test.go b/tests/server/api/rule_test.go index eaa41cc11bc..af70d5afed9 100644 --- a/tests/server/api/rule_test.go +++ b/tests/server/api/rule_test.go @@ -20,6 +20,9 @@ import ( "fmt" "net/http" "net/url" + "sort" + "strconv" + "sync" "testing" "github.com/pingcap/kvproto/pkg/metapb" @@ -28,6 +31,7 @@ import ( "github.com/tikv/pd/pkg/errs" "github.com/tikv/pd/pkg/schedule/labeler" "github.com/tikv/pd/pkg/schedule/placement" + "github.com/tikv/pd/pkg/utils/syncutil" tu "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/server/config" "github.com/tikv/pd/tests" @@ -777,7 +781,7 @@ func (suite *ruleTestSuite) checkBundle(cluster *tests.TestCluster) { err := tu.ReadGetJSON(re, testDialClient, urlPrefix+"/placement-rule", &bundles) suite.NoError(err) suite.Len(bundles, 1) - suite.compareBundle(bundles[0], b1) + suite.assertBundleEqual(bundles[0], b1) // Set b2 := placement.GroupBundle{ @@ -797,14 +801,14 @@ func (suite *ruleTestSuite) checkBundle(cluster *tests.TestCluster) { var bundle placement.GroupBundle err = tu.ReadGetJSON(re, testDialClient, urlPrefix+"/placement-rule/foo", &bundle) suite.NoError(err) - suite.compareBundle(bundle, b2) + suite.assertBundleEqual(bundle, b2) // GetAll again err = tu.ReadGetJSON(re, testDialClient, urlPrefix+"/placement-rule", &bundles) suite.NoError(err) suite.Len(bundles, 2) - suite.compareBundle(bundles[0], b1) - suite.compareBundle(bundles[1], b2) + suite.assertBundleEqual(bundles[0], b1) + suite.assertBundleEqual(bundles[1], b2) // Delete err = tu.CheckDelete(testDialClient, urlPrefix+"/placement-rule/pd", tu.StatusOK(suite.Require())) @@ -814,7 +818,7 @@ func (suite *ruleTestSuite) checkBundle(cluster *tests.TestCluster) { err = tu.ReadGetJSON(re, testDialClient, urlPrefix+"/placement-rule", &bundles) suite.NoError(err) suite.Len(bundles, 1) - suite.compareBundle(bundles[0], b2) + suite.assertBundleEqual(bundles[0], b2) // SetAll b2.Rules = append(b2.Rules, &placement.Rule{GroupID: "foo", ID: "baz", Index: 2, Role: placement.Follower, Count: 1}) @@ -829,9 +833,9 @@ func (suite *ruleTestSuite) checkBundle(cluster *tests.TestCluster) { err = tu.ReadGetJSON(re, testDialClient, urlPrefix+"/placement-rule", &bundles) suite.NoError(err) suite.Len(bundles, 3) - suite.compareBundle(bundles[0], b2) - suite.compareBundle(bundles[1], b1) - suite.compareBundle(bundles[2], b3) + suite.assertBundleEqual(bundles[0], b2) + suite.assertBundleEqual(bundles[1], b1) + suite.assertBundleEqual(bundles[2], b3) // Delete using regexp err = tu.CheckDelete(testDialClient, urlPrefix+"/placement-rule/"+url.PathEscape("foo.*")+"?regexp", tu.StatusOK(suite.Require())) @@ -841,7 +845,7 @@ func (suite *ruleTestSuite) checkBundle(cluster *tests.TestCluster) { err = tu.ReadGetJSON(re, testDialClient, urlPrefix+"/placement-rule", &bundles) suite.NoError(err) suite.Len(bundles, 1) - suite.compareBundle(bundles[0], b1) + suite.assertBundleEqual(bundles[0], b1) // Set id := "rule-without-group-id" @@ -862,14 +866,14 @@ func (suite *ruleTestSuite) checkBundle(cluster *tests.TestCluster) { // Get err = tu.ReadGetJSON(re, testDialClient, urlPrefix+"/placement-rule/"+id, &bundle) suite.NoError(err) - suite.compareBundle(bundle, b4) + suite.assertBundleEqual(bundle, b4) // GetAll again err = tu.ReadGetJSON(re, testDialClient, urlPrefix+"/placement-rule", &bundles) suite.NoError(err) suite.Len(bundles, 2) - suite.compareBundle(bundles[0], b1) - suite.compareBundle(bundles[1], b4) + suite.assertBundleEqual(bundles[0], b1) + suite.assertBundleEqual(bundles[1], b4) // SetAll b5 := placement.GroupBundle{ @@ -890,9 +894,9 @@ func (suite *ruleTestSuite) checkBundle(cluster *tests.TestCluster) { err = tu.ReadGetJSON(re, testDialClient, urlPrefix+"/placement-rule", &bundles) suite.NoError(err) suite.Len(bundles, 3) - suite.compareBundle(bundles[0], b1) - suite.compareBundle(bundles[1], b4) - suite.compareBundle(bundles[2], b5) + suite.assertBundleEqual(bundles[0], b1) + suite.assertBundleEqual(bundles[1], b4) + suite.assertBundleEqual(bundles[2], b5) } func (suite *ruleTestSuite) TestBundleBadRequest() { @@ -925,20 +929,315 @@ func (suite *ruleTestSuite) checkBundleBadRequest(cluster *tests.TestCluster) { } } -func (suite *ruleTestSuite) compareBundle(b1, b2 placement.GroupBundle) { - tu.Eventually(suite.Require(), func() bool { - if b2.ID != b1.ID || b2.Index != b1.Index || b2.Override != b1.Override || len(b2.Rules) != len(b1.Rules) { +func (suite *ruleTestSuite) TestLeaderAndVoter() { + suite.env.RunTestInTwoModes(suite.checkLeaderAndVoter) +} + +func (suite *ruleTestSuite) checkLeaderAndVoter(cluster *tests.TestCluster) { + re := suite.Require() + leaderServer := cluster.GetLeaderServer() + pdAddr := leaderServer.GetAddr() + urlPrefix := fmt.Sprintf("%s%s/api/v1", pdAddr, apiPrefix) + + stores := []*metapb.Store{ + { + Id: 1, + Address: "tikv1", + State: metapb.StoreState_Up, + NodeState: metapb.NodeState_Serving, + Version: "7.5.0", + Labels: []*metapb.StoreLabel{{Key: "zone", Value: "z1"}}, + }, + { + Id: 2, + Address: "tikv2", + State: metapb.StoreState_Up, + NodeState: metapb.NodeState_Serving, + Version: "7.5.0", + Labels: []*metapb.StoreLabel{{Key: "zone", Value: "z2"}}, + }, + } + + for _, store := range stores { + tests.MustPutStore(re, cluster, store) + } + + bundles := [][]placement.GroupBundle{ + { + { + ID: "1", + Index: 1, + Rules: []*placement.Rule{ + { + ID: "rule_1", Index: 1, Role: placement.Voter, Count: 1, GroupID: "1", + LabelConstraints: []placement.LabelConstraint{ + {Key: "zone", Op: "in", Values: []string{"z1"}}, + }, + }, + { + ID: "rule_2", Index: 2, Role: placement.Leader, Count: 1, GroupID: "1", + LabelConstraints: []placement.LabelConstraint{ + {Key: "zone", Op: "in", Values: []string{"z2"}}, + }, + }, + }, + }, + }, + { + { + ID: "1", + Index: 1, + Rules: []*placement.Rule{ + { + ID: "rule_1", Index: 1, Role: placement.Leader, Count: 1, GroupID: "1", + LabelConstraints: []placement.LabelConstraint{ + {Key: "zone", Op: "in", Values: []string{"z2"}}, + }, + }, + { + ID: "rule_2", Index: 2, Role: placement.Voter, Count: 1, GroupID: "1", + LabelConstraints: []placement.LabelConstraint{ + {Key: "zone", Op: "in", Values: []string{"z1"}}, + }, + }, + }, + }, + }} + for _, bundle := range bundles { + data, err := json.Marshal(bundle) + suite.NoError(err) + err = tu.CheckPostJSON(testDialClient, urlPrefix+"/config/placement-rule", data, tu.StatusOK(re)) + suite.NoError(err) + + tu.Eventually(re, func() bool { + respBundle := make([]placement.GroupBundle, 0) + err := tu.CheckGetJSON(testDialClient, urlPrefix+"/config/placement-rule", nil, + tu.StatusOK(re), tu.ExtractJSON(re, &respBundle)) + suite.NoError(err) + suite.Len(respBundle, 1) + if bundle[0].Rules[0].Role == placement.Leader { + return respBundle[0].Rules[0].Role == placement.Leader + } + if bundle[0].Rules[0].Role == placement.Voter { + return respBundle[0].Rules[0].Role == placement.Voter + } return false - } - for i := range b1.Rules { - if !suite.compareRule(b1.Rules[i], b2.Rules[i]) { + }) + } +} + +func (suite *ruleTestSuite) TestDeleteAndUpdate() { + suite.env.RunTestInTwoModes(suite.checkDeleteAndUpdate) +} + +func (suite *ruleTestSuite) checkDeleteAndUpdate(cluster *tests.TestCluster) { + re := suite.Require() + leaderServer := cluster.GetLeaderServer() + pdAddr := leaderServer.GetAddr() + urlPrefix := fmt.Sprintf("%s%s/api/v1", pdAddr, apiPrefix) + + bundles := [][]placement.GroupBundle{ + // 1 rule group with 1 rule + {{ + ID: "1", + Index: 1, + Rules: []*placement.Rule{ + { + ID: "foo", Index: 1, Role: placement.Voter, Count: 1, GroupID: "1", + }, + }, + }}, + // 2 rule groups with different range rules + {{ + ID: "1", + Index: 1, + Rules: []*placement.Rule{ + { + ID: "foo", Index: 1, Role: placement.Voter, Count: 1, GroupID: "1", + StartKey: []byte("a"), EndKey: []byte("b"), + }, + }, + }, { + ID: "2", + Index: 2, + Rules: []*placement.Rule{ + { + ID: "foo", Index: 2, Role: placement.Voter, Count: 1, GroupID: "2", + StartKey: []byte("b"), EndKey: []byte("c"), + }, + }, + }}, + // 2 rule groups with 1 rule and 2 rules + {{ + ID: "3", + Index: 3, + Rules: []*placement.Rule{ + { + ID: "foo", Index: 3, Role: placement.Voter, Count: 1, GroupID: "3", + }, + }, + }, { + ID: "4", + Index: 4, + Rules: []*placement.Rule{ + { + ID: "foo", Index: 4, Role: placement.Voter, Count: 1, GroupID: "4", + }, + { + ID: "bar", Index: 6, Role: placement.Voter, Count: 1, GroupID: "4", + }, + }, + }}, + // 1 rule group with 2 rules + {{ + ID: "5", + Index: 5, + Rules: []*placement.Rule{ + { + ID: "foo", Index: 5, Role: placement.Voter, Count: 1, GroupID: "5", + }, + { + ID: "bar", Index: 6, Role: placement.Voter, Count: 1, GroupID: "5", + }, + }, + }}, + } + + for _, bundle := range bundles { + data, err := json.Marshal(bundle) + suite.NoError(err) + err = tu.CheckPostJSON(testDialClient, urlPrefix+"/config/placement-rule", data, tu.StatusOK(re)) + suite.NoError(err) + + tu.Eventually(re, func() bool { + respBundle := make([]placement.GroupBundle, 0) + err = tu.CheckGetJSON(testDialClient, urlPrefix+"/config/placement-rule", nil, + tu.StatusOK(re), tu.ExtractJSON(re, &respBundle)) + suite.NoError(err) + if len(respBundle) != len(bundle) { return false } - } - return true + sort.Slice(respBundle, func(i, j int) bool { return respBundle[i].ID < respBundle[j].ID }) + sort.Slice(bundle, func(i, j int) bool { return bundle[i].ID < bundle[j].ID }) + for i := range respBundle { + if !suite.compareBundle(respBundle[i], bundle[i]) { + return false + } + } + return true + }) + } +} + +func (suite *ruleTestSuite) TestConcurrency() { + suite.env.RunTestInTwoModes(suite.checkConcurrency) +} + +func (suite *ruleTestSuite) checkConcurrency(cluster *tests.TestCluster) { + // test concurrency of set rule group with different group id + suite.checkConcurrencyWith(cluster, + func(i int) []placement.GroupBundle { + return []placement.GroupBundle{ + { + ID: strconv.Itoa(i), + Index: i, + Rules: []*placement.Rule{ + { + ID: "foo", Index: i, Role: placement.Voter, Count: 1, GroupID: strconv.Itoa(i), + }, + }, + }, + } + }, + func(resp []placement.GroupBundle, i int) bool { + return len(resp) == 1 && resp[0].ID == strconv.Itoa(i) + }, + ) + // test concurrency of set rule with different id + suite.checkConcurrencyWith(cluster, + func(i int) []placement.GroupBundle { + return []placement.GroupBundle{ + { + ID: "pd", + Index: 1, + Rules: []*placement.Rule{ + { + ID: strconv.Itoa(i), Index: i, Role: placement.Voter, Count: 1, GroupID: "pd", + }, + }, + }, + } + }, + func(resp []placement.GroupBundle, i int) bool { + return len(resp) == 1 && resp[0].ID == "pd" && resp[0].Rules[0].ID == strconv.Itoa(i) + }, + ) +} + +func (suite *ruleTestSuite) checkConcurrencyWith(cluster *tests.TestCluster, + genBundle func(int) []placement.GroupBundle, + checkBundle func([]placement.GroupBundle, int) bool) { + re := suite.Require() + leaderServer := cluster.GetLeaderServer() + pdAddr := leaderServer.GetAddr() + urlPrefix := fmt.Sprintf("%s%s/api/v1", pdAddr, apiPrefix) + expectResult := struct { + syncutil.RWMutex + val int + }{} + wg := sync.WaitGroup{} + + for i := 1; i <= 10; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + bundle := genBundle(i) + data, err := json.Marshal(bundle) + suite.NoError(err) + for j := 0; j < 10; j++ { + expectResult.Lock() + err = tu.CheckPostJSON(testDialClient, urlPrefix+"/config/placement-rule", data, tu.StatusOK(re)) + suite.NoError(err) + expectResult.val = i + expectResult.Unlock() + } + }(i) + } + + wg.Wait() + expectResult.RLock() + defer expectResult.RUnlock() + suite.NotZero(expectResult.val) + tu.Eventually(re, func() bool { + respBundle := make([]placement.GroupBundle, 0) + err := tu.CheckGetJSON(testDialClient, urlPrefix+"/config/placement-rule", nil, + tu.StatusOK(re), tu.ExtractJSON(re, &respBundle)) + suite.NoError(err) + suite.Len(respBundle, 1) + return checkBundle(respBundle, expectResult.val) + }) +} + +func (suite *ruleTestSuite) assertBundleEqual(b1, b2 placement.GroupBundle) { + tu.Eventually(suite.Require(), func() bool { + return suite.compareBundle(b1, b2) }) } +func (suite *ruleTestSuite) compareBundle(b1, b2 placement.GroupBundle) bool { + if b2.ID != b1.ID || b2.Index != b1.Index || b2.Override != b1.Override || len(b2.Rules) != len(b1.Rules) { + return false + } + sort.Slice(b1.Rules, func(i, j int) bool { return b1.Rules[i].ID < b1.Rules[j].ID }) + sort.Slice(b2.Rules, func(i, j int) bool { return b2.Rules[i].ID < b2.Rules[j].ID }) + for i := range b1.Rules { + if !suite.compareRule(b1.Rules[i], b2.Rules[i]) { + return false + } + } + return true +} + func (suite *ruleTestSuite) compareRule(r1 *placement.Rule, r2 *placement.Rule) bool { return r2.GroupID == r1.GroupID && r2.ID == r1.ID && From 48dbce1a7f4509c38b2920448a9f37c1963c8093 Mon Sep 17 00:00:00 2001 From: tongjian <1045931706@qq.com> Date: Wed, 20 Dec 2023 14:03:22 +0800 Subject: [PATCH 107/137] check: remove orphan peer only when the peers is greater than the rule count (#7581) close tikv/pd#7584 The healthy orphan peer should be the last one to be removed only if there are extra peers to keep the high availablility. Signed-off-by: bufferflies <1045931706@qq.com> Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- pkg/schedule/checker/rule_checker.go | 4 ++- pkg/schedule/checker/rule_checker_test.go | 36 +++++++++++++++++++++++ pkg/schedule/placement/fit.go | 9 ++++++ 3 files changed, 48 insertions(+), 1 deletion(-) diff --git a/pkg/schedule/checker/rule_checker.go b/pkg/schedule/checker/rule_checker.go index 95cc77ade5d..464f5e97be8 100644 --- a/pkg/schedule/checker/rule_checker.go +++ b/pkg/schedule/checker/rule_checker.go @@ -560,6 +560,7 @@ func (c *RuleChecker) fixOrphanPeers(region *core.RegionInfo, fit *placement.Reg } } + extra := fit.ExtraCount() // If hasUnhealthyFit is true, try to remove unhealthy orphan peers only if number of OrphanPeers is >= 2. // Ref https://github.com/tikv/pd/issues/4045 if len(fit.OrphanPeers) >= 2 { @@ -576,7 +577,8 @@ func (c *RuleChecker) fixOrphanPeers(region *core.RegionInfo, fit *placement.Reg ruleCheckerRemoveOrphanPeerCounter.Inc() return operator.CreateRemovePeerOperator("remove-unhealthy-orphan-peer", c.cluster, 0, region, orphanPeer.StoreId) } - if hasHealthPeer { + // The healthy orphan peer can be removed to keep the high availability only if the peer count is greater than the rule requirement. + if hasHealthPeer && extra > 0 { // there already exists a healthy orphan peer, so we can remove other orphan Peers. ruleCheckerRemoveOrphanPeerCounter.Inc() // if there exists a disconnected orphan peer, we will pick it to remove firstly. diff --git a/pkg/schedule/checker/rule_checker_test.go b/pkg/schedule/checker/rule_checker_test.go index 72d3e7e5ec4..6418483db7f 100644 --- a/pkg/schedule/checker/rule_checker_test.go +++ b/pkg/schedule/checker/rule_checker_test.go @@ -2029,3 +2029,39 @@ func (suite *ruleCheckerTestAdvancedSuite) TestReplaceAnExistingPeerCases() { suite.ruleManager.DeleteGroupBundle(groupName, false) } } + +func (suite *ruleCheckerTestSuite) TestRemoveOrphanPeer() { + suite.cluster.AddLabelsStore(1, 1, map[string]string{"zone": "z1", "host": "h1"}) + suite.cluster.AddLabelsStore(2, 1, map[string]string{"zone": "z1", "host": "h1"}) + suite.cluster.AddLabelsStore(3, 1, map[string]string{"zone": "z1", "host": "h1"}) + suite.cluster.AddLabelsStore(4, 1, map[string]string{"zone": "z2", "host": "h1"}) + suite.cluster.AddLabelsStore(5, 1, map[string]string{"zone": "z2", "host": "h2"}) + suite.cluster.AddLabelsStore(6, 1, map[string]string{"zone": "z2", "host": "h2"}) + rule := &placement.Rule{ + GroupID: "pd", + ID: "test2", + Role: placement.Voter, + Count: 3, + LabelConstraints: []placement.LabelConstraint{ + { + Key: "zone", + Op: placement.In, + Values: []string{"z2"}, + }, + }, + } + suite.ruleManager.SetRule(rule) + suite.ruleManager.DeleteRule("pd", "default") + + // case1: regionA has 3 peers but not extra peer can be removed, so it needs to add peer first + suite.cluster.AddLeaderRegionWithRange(1, "200", "300", 1, 2, 3) + op := suite.rc.Check(suite.cluster.GetRegion(1)) + suite.NotNil(op) + suite.Equal("add-rule-peer", op.Desc()) + + // case2: regionB has 4 peers and one extra peer can be removed, so it needs to remove extra peer first + suite.cluster.AddLeaderRegionWithRange(2, "300", "400", 1, 2, 3, 4) + op = suite.rc.Check(suite.cluster.GetRegion(2)) + suite.NotNil(op) + suite.Equal("remove-orphan-peer", op.Desc()) +} diff --git a/pkg/schedule/placement/fit.go b/pkg/schedule/placement/fit.go index 45afc5bcfa3..d907bcd011a 100644 --- a/pkg/schedule/placement/fit.go +++ b/pkg/schedule/placement/fit.go @@ -93,6 +93,15 @@ func (f *RegionFit) IsSatisfied() bool { return len(f.OrphanPeers) == 0 } +// ExtraCount return the extra count. +func (f *RegionFit) ExtraCount() int { + desired := 0 + for _, r := range f.RuleFits { + desired += r.Rule.Count + } + return len(f.regionStores) - desired +} + // GetRuleFit returns the RuleFit that contains the peer. func (f *RegionFit) GetRuleFit(peerID uint64) *RuleFit { for _, rf := range f.RuleFits { From 3b2e6115e288f1b53a4f0bffff3d8b2c1faf7b13 Mon Sep 17 00:00:00 2001 From: Hu# Date: Wed, 20 Dec 2023 14:42:22 +0800 Subject: [PATCH 108/137] makefile: make clean to remove dashboard temp (#7586) close tikv/pd#7585 Signed-off-by: husharp Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index bf25730c0d9..8e2353691b8 100644 --- a/Makefile +++ b/Makefile @@ -305,6 +305,7 @@ clean-test: clean-build: # Cleaning building files... rm -rf .dashboard_download_cache/ + rm -rf .dashboard_build_temp/ rm -rf $(BUILD_BIN_PATH) rm -rf $(GO_TOOLS_BIN_PATH) From d98daf3dd8f10a1315d9650a43e8bb4c27a04ebb Mon Sep 17 00:00:00 2001 From: JmPotato Date: Wed, 20 Dec 2023 16:00:23 +0800 Subject: [PATCH 109/137] tests, Makefile: rename the test from realtiup to realcluster (#7573) ref tikv/pd#7298 Rename the real-cluster test to a more general naming. Signed-off-by: JmPotato --- Makefile | 2 +- pd.code-workspace | 2 +- tests/integrations/{realtiup => realcluster}/Makefile | 0 tests/integrations/{realtiup => realcluster}/deploy.sh | 0 tests/integrations/{realtiup => realcluster}/go.mod | 2 +- tests/integrations/{realtiup => realcluster}/go.sum | 0 tests/integrations/{realtiup => realcluster}/mock_db.go | 2 +- .../integrations/{realtiup => realcluster}/reboot_pd_test.go | 2 +- .../{realtiup => realcluster}/transfer_leader_test.go | 2 +- tests/integrations/{realtiup => realcluster}/ts_test.go | 2 +- tests/integrations/{realtiup => realcluster}/util.go | 4 ++-- tests/integrations/{realtiup => realcluster}/wait_tiup.sh | 0 12 files changed, 9 insertions(+), 9 deletions(-) rename tests/integrations/{realtiup => realcluster}/Makefile (100%) rename tests/integrations/{realtiup => realcluster}/deploy.sh (100%) rename tests/integrations/{realtiup => realcluster}/go.mod (96%) rename tests/integrations/{realtiup => realcluster}/go.sum (100%) rename tests/integrations/{realtiup => realcluster}/mock_db.go (99%) rename tests/integrations/{realtiup => realcluster}/reboot_pd_test.go (98%) rename tests/integrations/{realtiup => realcluster}/transfer_leader_test.go (99%) rename tests/integrations/{realtiup => realcluster}/ts_test.go (98%) rename tests/integrations/{realtiup => realcluster}/util.go (92%) rename tests/integrations/{realtiup => realcluster}/wait_tiup.sh (100%) diff --git a/Makefile b/Makefile index 8e2353691b8..b94de374a81 100644 --- a/Makefile +++ b/Makefile @@ -261,7 +261,7 @@ test-tso-consistency: install-tools CGO_ENABLED=1 go test -race -tags without_dashboard,tso_consistency_test,deadlock $(TSO_INTEGRATION_TEST_PKGS) || { $(FAILPOINT_DISABLE); exit 1; } @$(FAILPOINT_DISABLE) -REAL_CLUSTER_TEST_PATH := $(ROOT_PATH)/tests/integrations/realtiup +REAL_CLUSTER_TEST_PATH := $(ROOT_PATH)/tests/integrations/realcluster test-real-cluster: # testing with the real cluster... diff --git a/pd.code-workspace b/pd.code-workspace index 54a8ea324aa..8cdb8d4d043 100644 --- a/pd.code-workspace +++ b/pd.code-workspace @@ -22,7 +22,7 @@ }, { "name": "real-cluster-tests", - "path": "tests/integrations/realtiup" + "path": "tests/integrations/realcluster" }, { "name": "pd-tso-bench", diff --git a/tests/integrations/realtiup/Makefile b/tests/integrations/realcluster/Makefile similarity index 100% rename from tests/integrations/realtiup/Makefile rename to tests/integrations/realcluster/Makefile diff --git a/tests/integrations/realtiup/deploy.sh b/tests/integrations/realcluster/deploy.sh similarity index 100% rename from tests/integrations/realtiup/deploy.sh rename to tests/integrations/realcluster/deploy.sh diff --git a/tests/integrations/realtiup/go.mod b/tests/integrations/realcluster/go.mod similarity index 96% rename from tests/integrations/realtiup/go.mod rename to tests/integrations/realcluster/go.mod index ccb23548f3e..66f40012863 100644 --- a/tests/integrations/realtiup/go.mod +++ b/tests/integrations/realcluster/go.mod @@ -1,4 +1,4 @@ -module github.com/tikv/pd/tests/integrations/realtiup +module github.com/tikv/pd/tests/integrations/realcluster go 1.21 diff --git a/tests/integrations/realtiup/go.sum b/tests/integrations/realcluster/go.sum similarity index 100% rename from tests/integrations/realtiup/go.sum rename to tests/integrations/realcluster/go.sum diff --git a/tests/integrations/realtiup/mock_db.go b/tests/integrations/realcluster/mock_db.go similarity index 99% rename from tests/integrations/realtiup/mock_db.go rename to tests/integrations/realcluster/mock_db.go index 95f3af8a06c..255ff6c0057 100644 --- a/tests/integrations/realtiup/mock_db.go +++ b/tests/integrations/realcluster/mock_db.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package realtiup +package realcluster import ( "testing" diff --git a/tests/integrations/realtiup/reboot_pd_test.go b/tests/integrations/realcluster/reboot_pd_test.go similarity index 98% rename from tests/integrations/realtiup/reboot_pd_test.go rename to tests/integrations/realcluster/reboot_pd_test.go index bccf465bde0..59747912897 100644 --- a/tests/integrations/realtiup/reboot_pd_test.go +++ b/tests/integrations/realcluster/reboot_pd_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package realtiup +package realcluster import ( "context" diff --git a/tests/integrations/realtiup/transfer_leader_test.go b/tests/integrations/realcluster/transfer_leader_test.go similarity index 99% rename from tests/integrations/realtiup/transfer_leader_test.go rename to tests/integrations/realcluster/transfer_leader_test.go index 51142be03f9..6a8f2e8a732 100644 --- a/tests/integrations/realtiup/transfer_leader_test.go +++ b/tests/integrations/realcluster/transfer_leader_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package realtiup +package realcluster import ( "context" diff --git a/tests/integrations/realtiup/ts_test.go b/tests/integrations/realcluster/ts_test.go similarity index 98% rename from tests/integrations/realtiup/ts_test.go rename to tests/integrations/realcluster/ts_test.go index 9bf8aee2d49..5d970556fbc 100644 --- a/tests/integrations/realtiup/ts_test.go +++ b/tests/integrations/realcluster/ts_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package realtiup +package realcluster import ( "testing" diff --git a/tests/integrations/realtiup/util.go b/tests/integrations/realcluster/util.go similarity index 92% rename from tests/integrations/realtiup/util.go rename to tests/integrations/realcluster/util.go index 8f0c71038d6..412df51894b 100644 --- a/tests/integrations/realtiup/util.go +++ b/tests/integrations/realcluster/util.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package realtiup +package realcluster import ( "time" @@ -24,7 +24,7 @@ const physicalShiftBits = 18 var ( pdAddrs = []string{"127.0.0.1:2379"} - pdHTTPCli = http.NewClient("pd-realtiup-test", pdAddrs) + pdHTTPCli = http.NewClient("pd-real-cluster-test", pdAddrs) ) // GetTimeFromTS extracts time.Time from a timestamp. diff --git a/tests/integrations/realtiup/wait_tiup.sh b/tests/integrations/realcluster/wait_tiup.sh similarity index 100% rename from tests/integrations/realtiup/wait_tiup.sh rename to tests/integrations/realcluster/wait_tiup.sh From ebd2fc09981a6a6cc58dee72e931744178ba6607 Mon Sep 17 00:00:00 2001 From: Hu# Date: Wed, 20 Dec 2023 16:12:22 +0800 Subject: [PATCH 110/137] dashboard-ui: make custom dashboard ui work (#7594) close tikv/pd#7593 Signed-off-by: husharp Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- scripts/embed-dashboard-ui.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/embed-dashboard-ui.sh b/scripts/embed-dashboard-ui.sh index 80c90bb1788..6ea9153d265 100755 --- a/scripts/embed-dashboard-ui.sh +++ b/scripts/embed-dashboard-ui.sh @@ -138,6 +138,8 @@ function compile_asset { echo '+ Build UI' cd "${BUILD_DIR}" + # ref https://github.com/tikv/pd/issues/7593 + git init make ui echo '+ Generating UI assets' From 84e60be69d811368220aac9ed39094a62b28c129 Mon Sep 17 00:00:00 2001 From: Yongbo Jiang Date: Wed, 20 Dec 2023 17:19:53 +0800 Subject: [PATCH 111/137] grafana: reassign the ID of the panel (#7588) ref tikv/pd#7583 Signed-off-by: Cabinfever_B Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- metrics/grafana/pd.json | 301 ++++++++++++++++++++-------------------- 1 file changed, 149 insertions(+), 152 deletions(-) diff --git a/metrics/grafana/pd.json b/metrics/grafana/pd.json index 809244771b7..a242235b204 100644 --- a/metrics/grafana/pd.json +++ b/metrics/grafana/pd.json @@ -95,7 +95,7 @@ "x": 0, "y": 0 }, - "id": 10, + "id": 100, "interval": null, "links": [], "mappingType": 1, @@ -181,7 +181,7 @@ "y": 0 }, "hideTimeOverride": false, - "id": 38, + "id": 101, "interval": null, "links": [], "mappingType": 1, @@ -266,7 +266,7 @@ "y": 0 }, "hideTimeOverride": false, - "id": 37, + "id": 102, "interval": null, "links": [], "mappingType": 1, @@ -348,7 +348,7 @@ "x": 12, "y": 0 }, - "id": 97, + "id": 103, "interval": null, "links": [], "mappingType": 1, @@ -434,7 +434,7 @@ "x": 16, "y": 0 }, - "id": 21, + "id": 104, "interval": null, "links": [], "mappingType": 1, @@ -518,7 +518,7 @@ "x": 20, "y": 0 }, - "id": 20, + "id": 105, "interval": null, "links": [], "mappingType": 1, @@ -594,7 +594,7 @@ "y": 6 }, "hideTimeOverride": true, - "id": 96, + "id": 106, "links": [], "pageSize": null, "scroll": false, @@ -744,7 +744,7 @@ "y": 6 }, "hiddenSeries": false, - "id": 72, + "id": 108, "legend": { "alignAsTable": true, "avg": false, @@ -845,7 +845,7 @@ "x": 16, "y": 6 }, - "id": 55, + "id": 109, "interval": null, "links": [], "maxDataPoints": 100, @@ -907,9 +907,6 @@ "indexByName": { "instance": 1, "service": 0 - }, - "renameByName": { - "instance": "" } } } @@ -944,7 +941,7 @@ "x": 0, "y": 13 }, - "id": 1449, + "id": 110, "options": { "orientation": "auto", "reduceOptions": { @@ -1002,7 +999,7 @@ "x": 4, "y": 13 }, - "id": 1453, + "id": 110, "options": { "reduceOptions": { "calcs": [ @@ -1039,7 +1036,7 @@ "x": 8, "y": 13 }, - "id": 1111, + "id": 111, "options": { "showHeader": false }, @@ -1100,7 +1097,7 @@ "x": 12, "y": 13 }, - "id": 1112, + "id": 112, "targets": [ { "expr": "pd_cluster_speed{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}", @@ -1159,7 +1156,7 @@ "y": 13 }, "hiddenSeries": false, - "id": 22, + "id": 113, "interval": null, "legend": { "alignAsTable": true, @@ -1252,7 +1249,7 @@ "x": 0, "y": 19 }, - "id": 118, + "id": 200, "panels": [ { "columns": [ @@ -1270,7 +1267,7 @@ "y": 20 }, "hideTimeOverride": true, - "id": 116, + "id": 201, "links": [], "pageSize": null, "scroll": true, @@ -1328,7 +1325,7 @@ "x": 4, "y": 20 }, - "id": 1433, + "id": 202, "links": [], "pageSize": null, "scroll": true, @@ -1453,7 +1450,7 @@ "y": 20 }, "hideTimeOverride": true, - "id": 139, + "id": 203, "links": [], "pageSize": null, "scroll": false, @@ -1522,7 +1519,7 @@ "y": 20 }, "hideTimeOverride": true, - "id": 103, + "id": 204, "links": [], "pageSize": null, "scroll": true, @@ -1602,7 +1599,7 @@ "y": 20 }, "hideTimeOverride": true, - "id": 117, + "id": 205, "links": [], "pageSize": null, "scroll": true, @@ -1704,7 +1701,7 @@ "y": 23 }, "hideTimeOverride": true, - "id": 115, + "id": 206, "interval": null, "links": [], "mappingType": 1, @@ -1784,7 +1781,7 @@ "y": 27 }, "hiddenSeries": false, - "id": 1427, + "id": 207, "legend": { "alignAsTable": true, "avg": true, @@ -1891,7 +1888,7 @@ "y": 27 }, "hiddenSeries": false, - "id": 1429, + "id": 208, "legend": { "alignAsTable": true, "avg": false, @@ -2034,7 +2031,7 @@ "y": 34 }, "hiddenSeries": false, - "id": 1430, + "id": 209, "legend": { "alignAsTable": true, "avg": false, @@ -2133,7 +2130,7 @@ "y": 34 }, "hiddenSeries": false, - "id": 1431, + "id": 210, "legend": { "alignAsTable": true, "avg": false, @@ -2243,7 +2240,7 @@ "y": 41 }, "hideTimeOverride": true, - "id": 137, + "id": 211, "interval": null, "links": [], "mappingType": 1, @@ -2329,7 +2326,7 @@ "y": 41 }, "hiddenSeries": false, - "id": 1443, + "id": 212, "legend": { "alignAsTable": true, "avg": false, @@ -2431,7 +2428,7 @@ "y": 41 }, "hiddenSeries": false, - "id": 1464, + "id": 213, "legend": { "alignAsTable": true, "avg": false, @@ -2547,7 +2544,7 @@ "y": 45 }, "hideTimeOverride": true, - "id": 1444, + "id": 214, "interval": null, "links": [], "mappingType": 1, @@ -2621,7 +2618,7 @@ "x": 0, "y": 20 }, - "id": 119, + "id": 300, "panels": [ { "aliasColors": {}, @@ -2638,7 +2635,7 @@ "x": 0, "y": 15 }, - "id": 45, + "id": 301, "legend": { "alignAsTable": true, "avg": true, @@ -2731,7 +2728,7 @@ "x": 12, "y": 15 }, - "id": 79, + "id": 302, "legend": { "alignAsTable": true, "avg": true, @@ -2824,7 +2821,7 @@ "x": 0, "y": 22 }, - "id": 77, + "id": 303, "legend": { "alignAsTable": true, "avg": true, @@ -2916,7 +2913,7 @@ "x": 12, "y": 22 }, - "id": 78, + "id": 304, "legend": { "alignAsTable": true, "avg": true, @@ -3009,7 +3006,7 @@ "x": 0, "y": 29 }, - "id": 80, + "id": 305, "legend": { "alignAsTable": true, "avg": true, @@ -3109,7 +3106,7 @@ "x": 12, "y": 29 }, - "id": 47, + "id": 306, "legend": { "alignAsTable": true, "avg": false, @@ -3203,7 +3200,7 @@ "x": 0, "y": 36 }, - "id": 67, + "id": 307, "legend": { "alignAsTable": true, "avg": false, @@ -3303,7 +3300,7 @@ "x": 12, "y": 36 }, - "id": 81, + "id": 308, "legend": { "alignAsTable": true, "avg": false, @@ -3403,7 +3400,7 @@ "x": 0, "y": 43 }, - "id": 1426, + "id": 309, "legend": { "alignAsTable": true, "avg": true, @@ -3501,7 +3498,7 @@ "x": 12, "y": 42 }, - "id": 1455, + "id": 310, "legend": { "alignAsTable": true, "avg": true, @@ -3590,7 +3587,7 @@ "x": 0, "y": 21 }, - "id": 120, + "id": 400, "panels": [ { "aliasColors": {}, @@ -3607,7 +3604,7 @@ "x": 0, "y": 23 }, - "id": 83, + "id": 401, "legend": { "alignAsTable": true, "avg": false, @@ -3705,7 +3702,7 @@ "x": 12, "y": 23 }, - "id": 91, + "id": 402, "legend": { "alignAsTable": true, "avg": false, @@ -3798,7 +3795,7 @@ "x": 0, "y": 29 }, - "id": 90, + "id": 403, "legend": { "alignAsTable": true, "avg": false, @@ -3892,7 +3889,7 @@ "x": 12, "y": 29 }, - "id": 84, + "id": 404, "legend": { "alignAsTable": true, "avg": false, @@ -3985,7 +3982,7 @@ "x": 0, "y": 35 }, - "id": 85, + "id": 405, "legend": { "alignAsTable": true, "avg": false, @@ -4081,7 +4078,7 @@ "x": 12, "y": 35 }, - "id": 41, + "id": 406, "legend": { "alignAsTable": true, "avg": false, @@ -4205,7 +4202,7 @@ "x": 0, "y": 41 }, - "id": 40, + "id": 407, "legend": { "alignAsTable": true, "avg": false, @@ -4309,7 +4306,7 @@ "x": 12, "y": 41 }, - "id": 57, + "id": 408, "legend": { "alignAsTable": true, "avg": false, @@ -4405,7 +4402,7 @@ "x": 0, "y": 47 }, - "id": 56, + "id": 409, "legend": { "alignAsTable": true, "avg": false, @@ -4500,7 +4497,7 @@ "x": 12, "y": 47 }, - "id": 59, + "id": 410, "legend": { "alignAsTable": true, "avg": false, @@ -4601,7 +4598,7 @@ "y": 111 }, "hiddenSeries": false, - "id": 1463, + "id": 411, "legend": { "alignAsTable": true, "avg": false, @@ -4701,7 +4698,7 @@ "x": 0, "y": 53 }, - "id": 58, + "id": 412, "legend": { "alignAsTable": true, "avg": false, @@ -4793,7 +4790,7 @@ "x": 0, "y": 22 }, - "id": 135, + "id": 500, "panels": [ { "aliasColors": {}, @@ -4810,7 +4807,7 @@ "x": 0, "y": 17 }, - "id": 50, + "id": 501, "legend": { "alignAsTable": true, "avg": false, @@ -4906,7 +4903,7 @@ "x": 12, "y": 17 }, - "id": 51, + "id": 502, "legend": { "alignAsTable": true, "avg": false, @@ -5002,7 +4999,7 @@ "x": 0, "y": 24 }, - "id": 61, + "id": 503, "legend": { "alignAsTable": true, "avg": false, @@ -5115,7 +5112,7 @@ "x": 12, "y": 24 }, - "id": 48, + "id": 504, "legend": { "alignAsTable": true, "avg": false, @@ -5220,7 +5217,7 @@ "x": 12, "y": 31 }, - "id": 1465, + "id": 505, "legend": { "alignAsTable": true, "avg": false, @@ -5324,7 +5321,7 @@ "x": 0, "y": 31 }, - "id": 143, + "id": 506, "legend": { "alignAsTable": true, "avg": false, @@ -5436,7 +5433,7 @@ "x": 12, "y": 38 }, - "id": 1445, + "id": 507, "legend": { "alignAsTable": true, "avg": false, @@ -5540,7 +5537,7 @@ "x": 0, "y": 38 }, - "id": 146, + "id": 508, "legend": { "alignAsTable": true, "avg": true, @@ -5636,7 +5633,7 @@ "x": 12, "y": 45 }, - "id": 106, + "id": 509, "legend": { "alignAsTable": true, "avg": true, @@ -5731,7 +5728,7 @@ "x": 0, "y": 45 }, - "id": 150, + "id": 510, "legend": { "alignAsTable": true, "avg": true, @@ -5833,7 +5830,7 @@ "x": 12, "y": 52 }, - "id": 148, + "id": 511, "legend": { "alignAsTable": true, "avg": true, @@ -5943,7 +5940,7 @@ "y": 52 }, "hiddenSeries": false, - "id": 1441, + "id": 512, "legend": { "alignAsTable": true, "avg": true, @@ -6039,7 +6036,7 @@ "x": 0, "y": 23 }, - "id": 121, + "id": 600, "panels": [ { "aliasColors": {}, @@ -6064,7 +6061,7 @@ "y": 53 }, "hiddenSeries": false, - "id": 60, + "id": 601, "legend": { "alignAsTable": true, "avg": false, @@ -6174,7 +6171,7 @@ "x": 12, "y": 53 }, - "id": 105, + "id": 602, "legend": { "alignAsTable": true, "avg": false, @@ -6285,7 +6282,7 @@ "x": 0, "y": 60 }, - "id": 107, + "id": 603, "legend": { "alignAsTable": true, "avg": false, @@ -6396,7 +6393,7 @@ "x": 12, "y": 60 }, - "id": 1446, + "id": 604, "legend": { "alignAsTable": true, "avg": false, @@ -6508,7 +6505,7 @@ "x": 0, "y": 67 }, - "id": 149, + "id": 605, "legend": { "alignAsTable": true, "avg": true, @@ -6610,7 +6607,7 @@ "x": 12, "y": 67 }, - "id": 144, + "id": 606, "legend": { "alignAsTable": true, "avg": true, @@ -6727,7 +6724,7 @@ "x": 0, "y": 74 }, - "id": 147, + "id": 607, "legend": { "alignAsTable": true, "avg": true, @@ -6829,7 +6826,7 @@ "x": 12, "y": 74 }, - "id": 1440, + "id": 608, "legend": { "alignAsTable": true, "avg": true, @@ -6928,7 +6925,7 @@ "x": 0, "y": 81 }, - "id": 1466, + "id": 609, "legend": { "alignAsTable": true, "avg": true, @@ -7022,7 +7019,7 @@ "x": 0, "y": 24 }, - "id": 122, + "id": 700, "panels": [ { "aliasColors": {}, @@ -7040,7 +7037,7 @@ "y": 25 }, "hiddenSeries": false, - "id": 46, + "id": 701, "legend": { "alignAsTable": true, "avg": false, @@ -7136,7 +7133,7 @@ "y": 33 }, "hiddenSeries": false, - "id": 87, + "id": 702, "legend": { "alignAsTable": true, "avg": false, @@ -7240,7 +7237,7 @@ }, "hiddenSeries": false, "hideTimeOverride": false, - "id": 86, + "id": 703, "legend": { "alignAsTable": true, "avg": false, @@ -7342,7 +7339,7 @@ "y": 41 }, "hiddenSeries": false, - "id": 52, + "id": 704, "legend": { "alignAsTable": true, "avg": false, @@ -7448,7 +7445,7 @@ "y": 41 }, "hiddenSeries": false, - "id": 53, + "id": 705, "legend": { "alignAsTable": true, "avg": false, @@ -7554,7 +7551,7 @@ "y": 49 }, "hiddenSeries": false, - "id": 1458, + "id": 706, "legend": { "alignAsTable": true, "avg": false, @@ -7651,7 +7648,7 @@ "y": 49 }, "hiddenSeries": false, - "id": 1459, + "id": 707, "legend": { "alignAsTable": true, "avg": false, @@ -7747,7 +7744,7 @@ "y": 57 }, "hiddenSeries": false, - "id": 108, + "id": 708, "legend": { "alignAsTable": true, "avg": false, @@ -7852,7 +7849,7 @@ "x": 0, "y": 65 }, - "id": 1424, + "id": 709, "interval": null, "links": [], "mappingType": 1, @@ -7927,7 +7924,7 @@ "y": 65 }, "hiddenSeries": false, - "id": 141, + "id": 710, "legend": { "alignAsTable": true, "avg": false, @@ -8019,7 +8016,7 @@ "x": 0, "y": 73 }, - "id": 70, + "id": 711, "legend": { "alignAsTable": true, "avg": false, @@ -8112,7 +8109,7 @@ "y": 73 }, "hiddenSeries": false, - "id": 71, + "id": 712, "legend": { "alignAsTable": true, "avg": false, @@ -8205,7 +8202,7 @@ "y": 81 }, "hiddenSeries": false, - "id": 109, + "id": 713, "legend": { "alignAsTable": true, "avg": false, @@ -8301,7 +8298,7 @@ "y": 81 }, "hiddenSeries": false, - "id": 110, + "id": 714, "legend": { "alignAsTable": true, "avg": false, @@ -8419,7 +8416,7 @@ "x": 0, "y": 25 }, - "id": 1437, + "id": 800, "panels": [ { "aliasColors": {}, @@ -8435,7 +8432,7 @@ "x": 0, "y": 20 }, - "id": 1439, + "id": 801, "legend": { "alignAsTable": true, "avg": true, @@ -8538,7 +8535,7 @@ "x": 12, "y": 20 }, - "id": 1435, + "id": 802, "legend": { "alignAsTable": true, "avg": false, @@ -8633,7 +8630,7 @@ "x": 0, "y": 26 }, - "id": 123, + "id": 900, "panels": [ { "aliasColors": {}, @@ -8653,7 +8650,7 @@ "x": 0, "y": 119 }, - "id": 1, + "id": 901, "legend": { "alignAsTable": true, "avg": false, @@ -8749,7 +8746,7 @@ "x": 12, "y": 119 }, - "id": 2, + "id": 902, "legend": { "alignAsTable": true, "avg": false, @@ -8843,7 +8840,7 @@ "x": 0, "y": 27 }, - "id": 124, + "id": 1000, "panels": [ { "aliasColors": {}, @@ -8863,7 +8860,7 @@ "x": 0, "y": 119 }, - "id": 1461, + "id": 1001, "legend": { "alignAsTable": true, "avg": false, @@ -8959,7 +8956,7 @@ "x": 12, "y": 119 }, - "id": 1462, + "id": 1002, "legend": { "alignAsTable": true, "avg": false, @@ -9053,7 +9050,7 @@ "x": 0, "y": 28 }, - "id": 125, + "id": 1100, "panels": [ { "aliasColors": {}, @@ -9068,7 +9065,7 @@ "x": 0, "y": 21 }, - "id": 1422, + "id": 1101, "legend": { "alignAsTable": true, "avg": false, @@ -9170,7 +9167,7 @@ "x": 12, "y": 21 }, - "id": 5, + "id": 1102, "legend": { "alignAsTable": true, "avg": false, @@ -9267,7 +9264,7 @@ "x": 0, "y": 29 }, - "id": 6, + "id": 1103, "legend": { "alignAsTable": true, "avg": false, @@ -9363,7 +9360,7 @@ "x": 12, "y": 29 }, - "id": 34, + "id": 1104, "legend": { "alignAsTable": true, "avg": false, @@ -9458,7 +9455,7 @@ "x": 0, "y": 37 }, - "id": 7, + "id": 1105, "legend": { "alignAsTable": true, "avg": false, @@ -9596,7 +9593,7 @@ "x": 12, "y": 37 }, - "id": 44, + "id": 1106, "legend": { "alignAsTable": true, "avg": false, @@ -9699,7 +9696,7 @@ "x": 0, "y": 45 }, - "id": 151, + "id": 1107, "legend": { "alignAsTable": true, "avg": false, @@ -9795,7 +9792,7 @@ "x": 12, "y": 45 }, - "id": 152, + "id": 1108, "legend": { "alignAsTable": true, "avg": false, @@ -9887,7 +9884,7 @@ "x": 0, "y": 53 }, - "id": 92, + "id": 1109, "legend": { "alignAsTable": true, "avg": false, @@ -9978,7 +9975,7 @@ "x": 8, "y": 53 }, - "id": 93, + "id": 1110, "legend": { "alignAsTable": true, "alignLevel": null, @@ -10070,7 +10067,7 @@ "x": 16, "y": 53 }, - "id": 94, + "id": 1111, "legend": { "alignAsTable": true, "avg": false, @@ -10160,7 +10157,7 @@ "x": 0, "y": 29 }, - "id": 126, + "id": 1200, "panels": [ { "aliasColors": {}, @@ -10179,7 +10176,7 @@ "x": 0, "y": 114 }, - "id": 29, + "id": 1201, "legend": { "alignAsTable": true, "avg": false, @@ -10304,7 +10301,7 @@ "x": 12, "y": 114 }, - "id": 142, + "id": 1202, "legend": { "alignAsTable": true, "avg": false, @@ -10410,7 +10407,7 @@ "x": 0, "y": 123 }, - "id": 28, + "id": 1203, "legend": { "alignAsTable": true, "avg": false, @@ -10516,7 +10513,7 @@ "x": 12, "y": 123 }, - "id": 1447, + "id": 1204, "legend": { "alignAsTable": true, "avg": false, @@ -10626,7 +10623,7 @@ "x": 0, "y": 30 }, - "id": 127, + "id": 1300, "panels": [ { "aliasColors": {}, @@ -10645,7 +10642,7 @@ "x": 0, "y": 23 }, - "id": 74, + "id": 1301, "legend": { "alignAsTable": true, "avg": false, @@ -10742,7 +10739,7 @@ "x": 12, "y": 23 }, - "id": 140, + "id": 1302, "legend": { "alignAsTable": true, "avg": false, @@ -10847,7 +10844,7 @@ "y": 31 }, "hiddenSeries": false, - "id": 1460, + "id": 1303, "legend": { "alignAsTable": true, "avg": false, @@ -10957,7 +10954,7 @@ "x": 12, "y": 31 }, - "id": 54, + "id": 1304, "legend": { "alignAsTable": true, "avg": true, @@ -11054,7 +11051,7 @@ "x": 0, "y": 39 }, - "id": 64, + "id": 1305, "legend": { "alignAsTable": true, "avg": true, @@ -11148,7 +11145,7 @@ "x": 12, "y": 39 }, - "id": 133, + "id": 1306, "legend": { "alignAsTable": true, "avg": true, @@ -11238,7 +11235,7 @@ "x": 0, "y": 47 }, - "id": 131, + "id": 1307, "legend": { "alignAsTable": true, "avg": true, @@ -11331,7 +11328,7 @@ "x": 12, "y": 47 }, - "id": 1400, + "id": 1308, "legend": { "alignAsTable": true, "avg": false, @@ -11428,7 +11425,7 @@ "x": 0, "y": 55 }, - "id": 1402, + "id": 1309, "legend": { "alignAsTable": true, "avg": true, @@ -11525,7 +11522,7 @@ "x": 12, "y": 55 }, - "id": 1403, + "id": 1310, "legend": { "alignAsTable": true, "avg": true, @@ -11622,7 +11619,7 @@ "x": 0, "y": 63 }, - "id": 1451, + "id": 1311, "legend": { "alignAsTable": true, "avg": true, @@ -11719,7 +11716,7 @@ "x": 12, "y": 63 }, - "id": 1452, + "id": 1312, "legend": { "alignAsTable": true, "avg": true, @@ -11812,7 +11809,7 @@ "x": 0, "y": 31 }, - "id": 1420, + "id": 1400, "panels": [ { "datasource": "${DS_TEST-CLUSTER}", @@ -11842,7 +11839,7 @@ "x": 0, "y": 24 }, - "id": 1407, + "id": 1401, "options": { "displayMode": "lcd", "orientation": "horizontal", @@ -11900,7 +11897,7 @@ "x": 12, "y": 24 }, - "id": 1411, + "id": 1402, "options": { "displayMode": "lcd", "orientation": "horizontal", @@ -11960,7 +11957,7 @@ "x": 0, "y": 32 }, - "id": 1406, + "id": 1403, "options": { "displayMode": "lcd", "orientation": "horizontal", @@ -12020,7 +12017,7 @@ "x": 12, "y": 32 }, - "id": 1412, + "id": 1404, "options": { "displayMode": "lcd", "orientation": "horizontal", @@ -12080,7 +12077,7 @@ "x": 0, "y": 40 }, - "id": 1408, + "id": 1405, "interval": "", "options": { "displayMode": "lcd", @@ -12142,7 +12139,7 @@ "x": 12, "y": 40 }, - "id": 1409, + "id": 1406, "interval": "", "options": { "displayMode": "lcd", @@ -12191,7 +12188,7 @@ "x": 0, "y": 48 }, - "id": 1454, + "id": 1407, "interval": "", "options": { "displayMode": "lcd", @@ -12240,7 +12237,7 @@ "x": 12, "y": 48 }, - "id": 1467, + "id": 1408, "interval": "", "options": { "displayMode": "lcd", @@ -12284,7 +12281,7 @@ "x": 0, "y": 32 }, - "id": 128, + "id": 1500, "panels": [ { "aliasColors": {}, @@ -12299,7 +12296,7 @@ "x": 0, "y": 213 }, - "id": 112, + "id": 1501, "legend": { "alignAsTable": true, "avg": false, @@ -12382,7 +12379,7 @@ "x": 12, "y": 213 }, - "id": 113, + "id": 1502, "legend": { "alignAsTable": true, "avg": false, @@ -12465,7 +12462,7 @@ "x": 0, "y": 33 }, - "id": 1471, + "id": 1600, "panels": [ { "datasource": "${DS_TEST-CLUSTER}", @@ -12530,7 +12527,7 @@ "x": 0, "y": 34 }, - "id": 1473, + "id": 1601, "options": { "colorMode": "value", "graphMode": "area", @@ -12584,7 +12581,7 @@ "y": 34 }, "hiddenSeries": false, - "id": 1478, + "id": 1602, "legend": { "alignAsTable": true, "avg": false, @@ -12686,7 +12683,7 @@ "y": 40 }, "hiddenSeries": false, - "id": 1474, + "id": 1603, "legend": { "alignAsTable": true, "avg": false, @@ -12788,7 +12785,7 @@ "y": 40 }, "hiddenSeries": false, - "id": 1477, + "id": 1604, "legend": { "alignAsTable": true, "avg": false, @@ -12890,7 +12887,7 @@ "y": 46 }, "hiddenSeries": false, - "id": 1475, + "id": 1605, "legend": { "alignAsTable": true, "avg": false, @@ -12992,7 +12989,7 @@ "y": 46 }, "hiddenSeries": false, - "id": 1476, + "id": 1606, "legend": { "alignAsTable": true, "avg": false, From a8aaae9b2b97cf9833501fa584e883bfbeb39c05 Mon Sep 17 00:00:00 2001 From: Hu# Date: Wed, 20 Dec 2023 17:33:23 +0800 Subject: [PATCH 112/137] member: return err when meet frequently campaign leader (#7566) close tikv/pd#7562 Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- errors.toml | 5 +++++ pkg/election/leadership.go | 11 +++++++++++ pkg/errs/errno.go | 17 +++++++++-------- pkg/member/member.go | 9 ++++++--- tests/server/member/member_test.go | 25 +++++++++++++++++++++++++ 5 files changed, 56 insertions(+), 11 deletions(-) diff --git a/errors.toml b/errors.toml index 69aa29d2dda..64101000478 100644 --- a/errors.toml +++ b/errors.toml @@ -766,6 +766,11 @@ error = ''' cannot set invalid configuration ''' +["PD:server:ErrLeaderFrequentlyChange"] +error = ''' +leader %s frequently changed, leader-key is [%s] +''' + ["PD:server:ErrLeaderNil"] error = ''' leader is nil diff --git a/pkg/election/leadership.go b/pkg/election/leadership.go index 132b3a8aad9..02f519dbc75 100644 --- a/pkg/election/leadership.go +++ b/pkg/election/leadership.go @@ -19,6 +19,7 @@ import ( "sync/atomic" "time" + "github.com/pingcap/errors" "github.com/pingcap/failpoint" "github.com/pingcap/kvproto/pkg/pdpb" "github.com/pingcap/log" @@ -156,6 +157,16 @@ func (ls *Leadership) Campaign(leaseTimeout int64, leaderData string, cmps ...cl lease: clientv3.NewLease(ls.client), } ls.setLease(newLease) + + failpoint.Inject("skipGrantLeader", func(val failpoint.Value) { + var member pdpb.Member + member.Unmarshal([]byte(leaderData)) + name, ok := val.(string) + if ok && member.Name == name { + failpoint.Return(errors.Errorf("failed to grant lease")) + } + }) + if err := newLease.Grant(leaseTimeout); err != nil { return err } diff --git a/pkg/errs/errno.go b/pkg/errs/errno.go index 03fa0f61158..8c3e914531b 100644 --- a/pkg/errs/errno.go +++ b/pkg/errs/errno.go @@ -208,14 +208,15 @@ var ( // server errors var ( - ErrServiceRegistered = errors.Normalize("service with path [%s] already registered", errors.RFCCodeText("PD:server:ErrServiceRegistered")) - ErrAPIInformationInvalid = errors.Normalize("invalid api information, group %s version %s", errors.RFCCodeText("PD:server:ErrAPIInformationInvalid")) - ErrClientURLEmpty = errors.Normalize("client url empty", errors.RFCCodeText("PD:server:ErrClientEmpty")) - ErrLeaderNil = errors.Normalize("leader is nil", errors.RFCCodeText("PD:server:ErrLeaderNil")) - ErrCancelStartEtcd = errors.Normalize("etcd start canceled", errors.RFCCodeText("PD:server:ErrCancelStartEtcd")) - ErrConfigItem = errors.Normalize("cannot set invalid configuration", errors.RFCCodeText("PD:server:ErrConfiguration")) - ErrServerNotStarted = errors.Normalize("server not started", errors.RFCCodeText("PD:server:ErrServerNotStarted")) - ErrRateLimitExceeded = errors.Normalize("rate limit exceeded", errors.RFCCodeText("PD:server:ErrRateLimitExceeded")) + ErrServiceRegistered = errors.Normalize("service with path [%s] already registered", errors.RFCCodeText("PD:server:ErrServiceRegistered")) + ErrAPIInformationInvalid = errors.Normalize("invalid api information, group %s version %s", errors.RFCCodeText("PD:server:ErrAPIInformationInvalid")) + ErrClientURLEmpty = errors.Normalize("client url empty", errors.RFCCodeText("PD:server:ErrClientEmpty")) + ErrLeaderNil = errors.Normalize("leader is nil", errors.RFCCodeText("PD:server:ErrLeaderNil")) + ErrCancelStartEtcd = errors.Normalize("etcd start canceled", errors.RFCCodeText("PD:server:ErrCancelStartEtcd")) + ErrConfigItem = errors.Normalize("cannot set invalid configuration", errors.RFCCodeText("PD:server:ErrConfiguration")) + ErrServerNotStarted = errors.Normalize("server not started", errors.RFCCodeText("PD:server:ErrServerNotStarted")) + ErrRateLimitExceeded = errors.Normalize("rate limit exceeded", errors.RFCCodeText("PD:server:ErrRateLimitExceeded")) + ErrLeaderFrequentlyChange = errors.Normalize("leader %s frequently changed, leader-key is [%s]", errors.RFCCodeText("PD:server:ErrLeaderFrequentlyChange")) ) // logutil errors diff --git a/pkg/member/member.go b/pkg/member/member.go index 1b901a1d04a..4e532270700 100644 --- a/pkg/member/member.go +++ b/pkg/member/member.go @@ -185,12 +185,15 @@ func (m *EmbeddedEtcdMember) CampaignLeader(ctx context.Context, leaseTimeout in failpoint.Inject("skipCampaignLeaderCheck", func() { failpoint.Return(m.leadership.Campaign(leaseTimeout, m.MemberValue())) }) + if m.leadership.GetCampaignTimesNum() >= campaignLeaderFrequencyTimes { - log.Warn("campaign times is too frequent, resign and campaign again", - zap.String("leader-name", m.Name()), zap.String("leader-key", m.GetLeaderPath())) m.leadership.ResetCampaignTimes() - return m.ResignEtcdLeader(ctx, m.Name(), "") + if err := m.ResignEtcdLeader(ctx, m.Name(), ""); err != nil { + return err + } + return errs.ErrLeaderFrequentlyChange.FastGenByArgs(m.Name(), m.GetLeaderPath()) } + return m.leadership.Campaign(leaseTimeout, m.MemberValue()) } diff --git a/tests/server/member/member_test.go b/tests/server/member/member_test.go index 68fbdb33bd9..d87e8a1a5c0 100644 --- a/tests/server/member/member_test.go +++ b/tests/server/member/member_test.go @@ -346,6 +346,31 @@ func TestCampaignLeaderFrequently(t *testing.T) { re.NotEqual(leader, cluster.GetLeader()) } +func TestGrantLeaseFailed(t *testing.T) { + re := require.New(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + cluster, err := tests.NewTestCluster(ctx, 5) + defer cluster.Destroy() + re.NoError(err) + + err = cluster.RunInitialServers() + re.NoError(err) + cluster.WaitLeader() + leader := cluster.GetLeader() + re.NotEmpty(cluster.GetLeader()) + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/election/skipGrantLeader", fmt.Sprintf("return(\"%s\")", leader))) + + for i := 0; i < 3; i++ { + cluster.GetLeaderServer().ResetPDLeader() + cluster.WaitLeader() + } + // PD leader should be different from before because etcd leader changed. + re.NotEmpty(cluster.GetLeader()) + re.NotEqual(leader, cluster.GetLeader()) + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/election/skipGrantLeader")) +} + func TestGetLeader(t *testing.T) { re := require.New(t) ctx, cancel := context.WithCancel(context.Background()) From 08296ebf968d08d64c8b9691380b5e752007bc50 Mon Sep 17 00:00:00 2001 From: ShuNing Date: Wed, 20 Dec 2023 17:52:53 +0800 Subject: [PATCH 113/137] resource_control: unify label name to group_name (#7547) close tikv/pd#7546 Signed-off-by: nolouch Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- .../resource_group/controller/controller.go | 16 ++++---- client/resource_group/controller/metrics.go | 16 ++++---- pkg/mcs/resourcemanager/server/manager.go | 40 +++++++++--------- pkg/mcs/resourcemanager/server/metrics.go | 41 ++++++++++--------- 4 files changed, 58 insertions(+), 55 deletions(-) diff --git a/client/resource_group/controller/controller.go b/client/resource_group/controller/controller.go index a07622e5f3b..241e4ab81bf 100755 --- a/client/resource_group/controller/controller.go +++ b/client/resource_group/controller/controller.go @@ -346,7 +346,7 @@ func (c *ResourceGroupsController) Start(ctx context.Context) { continue } if _, ok := c.groupsController.LoadAndDelete(group.Name); ok { - resourceGroupStatusGauge.DeleteLabelValues(group.Name) + resourceGroupStatusGauge.DeleteLabelValues(group.Name, group.Name) } } else { // Prev-kv is compacted means there must have been a delete event before this event, @@ -431,7 +431,7 @@ func (c *ResourceGroupsController) tryGetResourceGroup(ctx context.Context, name // Check again to prevent initializing the same resource group concurrently. tmp, loaded := c.groupsController.LoadOrStore(group.GetName(), gc) if !loaded { - resourceGroupStatusGauge.WithLabelValues(name).Set(1) + resourceGroupStatusGauge.WithLabelValues(name, group.Name).Set(1) log.Info("[resource group controller] create resource group cost controller", zap.String("name", group.GetName())) } return tmp.(*groupCostController), nil @@ -448,7 +448,7 @@ func (c *ResourceGroupsController) cleanUpResourceGroup() { if equalRU(latestConsumption, *gc.run.consumption) { if gc.tombstone { c.groupsController.Delete(resourceGroupName) - resourceGroupStatusGauge.DeleteLabelValues(resourceGroupName) + resourceGroupStatusGauge.DeleteLabelValues(resourceGroupName, resourceGroupName) return true } gc.tombstone = true @@ -713,11 +713,11 @@ func newGroupCostController( name: group.Name, mainCfg: mainCfg, mode: group.GetMode(), - successfulRequestDuration: successfulRequestDuration.WithLabelValues(group.Name), - failedLimitReserveDuration: failedLimitReserveDuration.WithLabelValues(group.Name), - failedRequestCounter: failedRequestCounter.WithLabelValues(group.Name), - requestRetryCounter: requestRetryCounter.WithLabelValues(group.Name), - tokenRequestCounter: resourceGroupTokenRequestCounter.WithLabelValues(group.Name), + successfulRequestDuration: successfulRequestDuration.WithLabelValues(group.Name, group.Name), + failedLimitReserveDuration: failedLimitReserveDuration.WithLabelValues(group.Name, group.Name), + failedRequestCounter: failedRequestCounter.WithLabelValues(group.Name, group.Name), + requestRetryCounter: requestRetryCounter.WithLabelValues(group.Name, group.Name), + tokenRequestCounter: resourceGroupTokenRequestCounter.WithLabelValues(group.Name, group.Name), calculators: []ResourceCalculator{ newKVCalculator(mainCfg), newSQLCalculator(mainCfg), diff --git a/client/resource_group/controller/metrics.go b/client/resource_group/controller/metrics.go index 7e6a559265b..4261705a6f6 100644 --- a/client/resource_group/controller/metrics.go +++ b/client/resource_group/controller/metrics.go @@ -21,7 +21,9 @@ const ( requestSubsystem = "request" tokenRequestSubsystem = "token_request" - resourceGroupNameLabel = "name" + // TODO: remove old label in 8.x + resourceGroupNameLabel = "name" + newResourceGroupNameLabel = "resource_group" ) var ( @@ -31,7 +33,7 @@ var ( Subsystem: "resource_group", Name: "status", Help: "Status of the resource group.", - }, []string{resourceGroupNameLabel}) + }, []string{resourceGroupNameLabel, newResourceGroupNameLabel}) successfulRequestDuration = prometheus.NewHistogramVec( prometheus.HistogramOpts{ @@ -40,7 +42,7 @@ var ( Name: "success", Buckets: []float64{.005, .01, .05, .1, .5, 1, 5, 10, 20, 25, 30}, // 0.005 ~ 30 Help: "Bucketed histogram of wait duration of successful request.", - }, []string{resourceGroupNameLabel}) + }, []string{resourceGroupNameLabel, newResourceGroupNameLabel}) failedLimitReserveDuration = prometheus.NewHistogramVec( prometheus.HistogramOpts{ @@ -49,7 +51,7 @@ var ( Name: "limit_reserve_time_failed", Buckets: []float64{.005, .01, .05, .1, .5, 1, 5, 10, 20, 25, 30}, // 0.005 ~ 30 Help: "Bucketed histogram of wait duration of failed request.", - }, []string{resourceGroupNameLabel}) + }, []string{resourceGroupNameLabel, newResourceGroupNameLabel}) failedRequestCounter = prometheus.NewCounterVec( prometheus.CounterOpts{ @@ -57,7 +59,7 @@ var ( Subsystem: requestSubsystem, Name: "fail", Help: "Counter of failed request.", - }, []string{resourceGroupNameLabel}) + }, []string{resourceGroupNameLabel, newResourceGroupNameLabel}) requestRetryCounter = prometheus.NewCounterVec( prometheus.CounterOpts{ @@ -65,7 +67,7 @@ var ( Subsystem: requestSubsystem, Name: "retry", Help: "Counter of retry time for request.", - }, []string{resourceGroupNameLabel}) + }, []string{resourceGroupNameLabel, newResourceGroupNameLabel}) tokenRequestDuration = prometheus.NewHistogramVec( prometheus.HistogramOpts{ @@ -81,7 +83,7 @@ var ( Subsystem: tokenRequestSubsystem, Name: "resource_group", Help: "Counter of token request by every resource group.", - }, []string{resourceGroupNameLabel}) + }, []string{resourceGroupNameLabel, newResourceGroupNameLabel}) ) var ( diff --git a/pkg/mcs/resourcemanager/server/manager.go b/pkg/mcs/resourcemanager/server/manager.go index 03db817cf12..6821938039b 100644 --- a/pkg/mcs/resourcemanager/server/manager.go +++ b/pkg/mcs/resourcemanager/server/manager.go @@ -373,15 +373,15 @@ func (m *Manager) backgroundMetricsFlush(ctx context.Context) { var ( name = consumptionInfo.resourceGroupName - rruMetrics = readRequestUnitCost.WithLabelValues(name, ruLabelType) - wruMetrics = writeRequestUnitCost.WithLabelValues(name, ruLabelType) - sqlLayerRuMetrics = sqlLayerRequestUnitCost.WithLabelValues(name) - readByteMetrics = readByteCost.WithLabelValues(name, ruLabelType) - writeByteMetrics = writeByteCost.WithLabelValues(name, ruLabelType) - kvCPUMetrics = kvCPUCost.WithLabelValues(name, ruLabelType) - sqlCPUMetrics = sqlCPUCost.WithLabelValues(name, ruLabelType) - readRequestCountMetrics = requestCount.WithLabelValues(name, readTypeLabel) - writeRequestCountMetrics = requestCount.WithLabelValues(name, writeTypeLabel) + rruMetrics = readRequestUnitCost.WithLabelValues(name, name, ruLabelType) + wruMetrics = writeRequestUnitCost.WithLabelValues(name, name, ruLabelType) + sqlLayerRuMetrics = sqlLayerRequestUnitCost.WithLabelValues(name, name) + readByteMetrics = readByteCost.WithLabelValues(name, name, ruLabelType) + writeByteMetrics = writeByteCost.WithLabelValues(name, name, ruLabelType) + kvCPUMetrics = kvCPUCost.WithLabelValues(name, name, ruLabelType) + sqlCPUMetrics = sqlCPUCost.WithLabelValues(name, name, ruLabelType) + readRequestCountMetrics = requestCount.WithLabelValues(name, name, readTypeLabel) + writeRequestCountMetrics = requestCount.WithLabelValues(name, name, writeTypeLabel) ) // RU info. if consumption.RRU > 0 { @@ -419,16 +419,16 @@ func (m *Manager) backgroundMetricsFlush(ctx context.Context) { // Clean up the metrics that have not been updated for a long time. for name, lastTime := range m.consumptionRecord { if time.Since(lastTime) > metricsCleanupTimeout { - readRequestUnitCost.DeleteLabelValues(name) - writeRequestUnitCost.DeleteLabelValues(name) - sqlLayerRequestUnitCost.DeleteLabelValues(name) - readByteCost.DeleteLabelValues(name) - writeByteCost.DeleteLabelValues(name) - kvCPUCost.DeleteLabelValues(name) - sqlCPUCost.DeleteLabelValues(name) - requestCount.DeleteLabelValues(name, readTypeLabel) - requestCount.DeleteLabelValues(name, writeTypeLabel) - availableRUCounter.DeleteLabelValues(name) + readRequestUnitCost.DeleteLabelValues(name, name) + writeRequestUnitCost.DeleteLabelValues(name, name) + sqlLayerRequestUnitCost.DeleteLabelValues(name, name) + readByteCost.DeleteLabelValues(name, name) + writeByteCost.DeleteLabelValues(name, name) + kvCPUCost.DeleteLabelValues(name, name) + sqlCPUCost.DeleteLabelValues(name, name) + requestCount.DeleteLabelValues(name, name, readTypeLabel) + requestCount.DeleteLabelValues(name, name, writeTypeLabel) + availableRUCounter.DeleteLabelValues(name, name) delete(m.consumptionRecord, name) } } @@ -442,7 +442,7 @@ func (m *Manager) backgroundMetricsFlush(ctx context.Context) { if ru < 0 { ru = 0 } - availableRUCounter.WithLabelValues(name).Set(ru) + availableRUCounter.WithLabelValues(name, name).Set(ru) } m.RUnlock() } diff --git a/pkg/mcs/resourcemanager/server/metrics.go b/pkg/mcs/resourcemanager/server/metrics.go index 25d0516d269..4322ed1a640 100644 --- a/pkg/mcs/resourcemanager/server/metrics.go +++ b/pkg/mcs/resourcemanager/server/metrics.go @@ -17,17 +17,18 @@ package server import "github.com/prometheus/client_golang/prometheus" const ( - namespace = "resource_manager" - serverSubsystem = "server" - ruSubsystem = "resource_unit" - resourceSubsystem = "resource" - resourceGroupNameLabel = "name" - typeLabel = "type" - readTypeLabel = "read" - writeTypeLabel = "write" - backgroundTypeLabel = "background" - tiflashTypeLabel = "ap" - defaultTypeLabel = "tp" + namespace = "resource_manager" + serverSubsystem = "server" + ruSubsystem = "resource_unit" + resourceSubsystem = "resource" + resourceGroupNameLabel = "name" + typeLabel = "type" + readTypeLabel = "read" + writeTypeLabel = "write" + backgroundTypeLabel = "background" + tiflashTypeLabel = "ap" + defaultTypeLabel = "tp" + newResourceGroupNameLabel = "resource_group" ) var ( @@ -47,21 +48,21 @@ var ( Subsystem: ruSubsystem, Name: "read_request_unit_sum", Help: "Counter of the read request unit cost for all resource groups.", - }, []string{resourceGroupNameLabel, typeLabel}) + }, []string{resourceGroupNameLabel, newResourceGroupNameLabel, typeLabel}) writeRequestUnitCost = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: namespace, Subsystem: ruSubsystem, Name: "write_request_unit_sum", Help: "Counter of the write request unit cost for all resource groups.", - }, []string{resourceGroupNameLabel, typeLabel}) + }, []string{resourceGroupNameLabel, newResourceGroupNameLabel, typeLabel}) sqlLayerRequestUnitCost = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: namespace, Subsystem: ruSubsystem, Name: "sql_layer_request_unit_sum", Help: "The number of the sql layer request unit cost for all resource groups.", - }, []string{resourceGroupNameLabel}) + }, []string{resourceGroupNameLabel, newResourceGroupNameLabel}) // Resource cost metrics. readByteCost = prometheus.NewCounterVec( @@ -70,35 +71,35 @@ var ( Subsystem: resourceSubsystem, Name: "read_byte_sum", Help: "Counter of the read byte cost for all resource groups.", - }, []string{resourceGroupNameLabel, typeLabel}) + }, []string{resourceGroupNameLabel, newResourceGroupNameLabel, typeLabel}) writeByteCost = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: namespace, Subsystem: resourceSubsystem, Name: "write_byte_sum", Help: "Counter of the write byte cost for all resource groups.", - }, []string{resourceGroupNameLabel, typeLabel}) + }, []string{resourceGroupNameLabel, newResourceGroupNameLabel, typeLabel}) kvCPUCost = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: namespace, Subsystem: resourceSubsystem, Name: "kv_cpu_time_ms_sum", Help: "Counter of the KV CPU time cost in milliseconds for all resource groups.", - }, []string{resourceGroupNameLabel, typeLabel}) + }, []string{resourceGroupNameLabel, newResourceGroupNameLabel, typeLabel}) sqlCPUCost = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: namespace, Subsystem: resourceSubsystem, Name: "sql_cpu_time_ms_sum", Help: "Counter of the SQL CPU time cost in milliseconds for all resource groups.", - }, []string{resourceGroupNameLabel, typeLabel}) + }, []string{resourceGroupNameLabel, newResourceGroupNameLabel, typeLabel}) requestCount = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: namespace, Subsystem: resourceSubsystem, Name: "request_count", Help: "The number of read/write requests for all resource groups.", - }, []string{resourceGroupNameLabel, typeLabel}) + }, []string{resourceGroupNameLabel, newResourceGroupNameLabel, typeLabel}) availableRUCounter = prometheus.NewGaugeVec( prometheus.GaugeOpts{ @@ -106,7 +107,7 @@ var ( Subsystem: ruSubsystem, Name: "available_ru", Help: "Counter of the available RU for all resource groups.", - }, []string{resourceGroupNameLabel}) + }, []string{resourceGroupNameLabel, newResourceGroupNameLabel}) ) func init() { From 66508f5a4943880adb7c6fa2a297988e5190af02 Mon Sep 17 00:00:00 2001 From: Yongbo Jiang Date: Wed, 20 Dec 2023 18:06:23 +0800 Subject: [PATCH 114/137] versioninfo: remove fips suffix (#7597) ref tikv/pd#7274 Signed-off-by: Cabinfever_B Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- pkg/versioninfo/fips.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pkg/versioninfo/fips.go b/pkg/versioninfo/fips.go index 02478b103fa..427988c853d 100644 --- a/pkg/versioninfo/fips.go +++ b/pkg/versioninfo/fips.go @@ -20,7 +20,3 @@ package versioninfo import ( _ "crypto/tls/fipsonly" ) - -func init() { - PDReleaseVersion += "-fips" -} From 7f156d909aeb4bf66f8dd0feafd7cb07e64757d5 Mon Sep 17 00:00:00 2001 From: JmPotato Date: Thu, 21 Dec 2023 12:03:23 +0800 Subject: [PATCH 115/137] tests: fix all errors detected by testifylint and enable it (#7589) ref tikv/pd#4813 Fix all errors detected by testifylint and enable it. Signed-off-by: JmPotato Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- .golangci.yml | 3 +- client/http/client_test.go | 4 +- client/http/types_test.go | 2 +- client/option_test.go | 2 +- .../controller/controller_test.go | 2 +- .../resource_group/controller/limiter_test.go | 2 +- client/resource_group/controller/util_test.go | 2 +- pkg/cache/cache_test.go | 28 +- pkg/cgroup/cgroup_cpu_test.go | 25 +- pkg/dashboard/adapter/redirector_test.go | 10 +- pkg/schedule/checker/merge_checker_test.go | 138 +++--- pkg/schedule/checker/replica_checker_test.go | 162 +++--- pkg/schedule/checker/rule_checker_test.go | 469 ++++++++++-------- pkg/schedule/operator/create_operator_test.go | 177 +++---- .../operator/operator_controller_test.go | 309 ++++++------ pkg/schedule/operator/operator_test.go | 203 ++++---- pkg/schedule/operator/step_test.go | 124 ++--- pkg/schedule/rangelist/range_list_test.go | 2 +- pkg/schedule/schedulers/balance_test.go | 154 +++--- .../schedulers/balance_witness_test.go | 17 +- .../schedulers/evict_slow_store_test.go | 58 ++- .../schedulers/evict_slow_trend_test.go | 133 ++--- pkg/schedule/splitter/region_splitter_test.go | 28 +- pkg/utils/etcdutil/etcdutil_test.go | 9 +- server/api/cluster_test.go | 32 +- server/api/diagnostic_test.go | 40 +- server/api/hot_status_test.go | 38 +- server/api/label_test.go | 26 +- server/api/log_test.go | 9 +- server/api/member_test.go | 54 +- server/api/min_resolved_ts_test.go | 35 +- server/api/pprof_test.go | 11 +- server/api/region_label_test.go | 31 +- server/api/region_test.go | 177 +++---- server/api/server_test.go | 53 +- server/api/service_gc_safepoint_test.go | 17 +- server/api/service_middleware_test.go | 216 ++++---- server/api/stats_test.go | 8 +- server/api/store_test.go | 201 ++++---- server/api/tso_test.go | 2 +- server/api/unsafe_operation_test.go | 14 +- server/server_test.go | 126 ++--- tests/dashboard/service_test.go | 53 +- tests/integrations/client/client_test.go | 252 +++++----- tests/integrations/client/gc_client_test.go | 67 +-- .../integrations/client/global_config_test.go | 151 +++--- tests/integrations/client/http_client_test.go | 10 +- .../mcs/discovery/register_test.go | 8 +- .../mcs/keyspace/tso_keyspace_group_test.go | 185 +++---- tests/integrations/mcs/members/member_test.go | 17 +- .../resourcemanager/resource_manager_test.go | 77 +-- tests/integrations/mcs/scheduling/api_test.go | 74 +-- .../integrations/mcs/scheduling/meta_test.go | 3 +- .../mcs/scheduling/server_test.go | 5 +- tests/integrations/mcs/tso/api_test.go | 4 +- tests/integrations/mcs/tso/proxy_test.go | 2 +- tests/integrations/mcs/tso/server_test.go | 89 ++-- .../realcluster/transfer_leader_test.go | 4 +- tests/integrations/tso/client_test.go | 14 +- tests/integrations/tso/consistency_test.go | 3 +- tests/pdctl/config/config_test.go | 78 +-- tests/pdctl/log/log_test.go | 12 +- tests/server/api/api_test.go | 263 +++++----- tests/server/api/checker_test.go | 81 ++- tests/server/api/operator_test.go | 64 +-- tests/server/api/region_test.go | 104 ++-- tests/server/api/rule_test.go | 306 ++++++------ tests/server/api/scheduler_test.go | 348 ++++++------- tests/server/config/config_test.go | 196 ++++---- tests/server/keyspace/keyspace_test.go | 9 +- tests/server/tso/consistency_test.go | 66 +-- 71 files changed, 2969 insertions(+), 2729 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 59954cecee3..9843142b4ab 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -11,8 +11,7 @@ linters: - makezero - gosec - bodyclose - # TODO: enable when all existing errors are fixed - # - testifylint + - testifylint disable: - errcheck linters-settings: diff --git a/client/http/client_test.go b/client/http/client_test.go index af16ac649b5..3965eb42068 100644 --- a/client/http/client_test.go +++ b/client/http/client_test.go @@ -29,12 +29,12 @@ func TestPDAddrNormalization(t *testing.T) { re := require.New(t) c := NewClient("test-http-pd-addr", []string{"127.0.0.1"}) pdAddrs, leaderAddrIdx := c.(*client).inner.getPDAddrs() - re.Equal(1, len(pdAddrs)) + re.Len(pdAddrs, 1) re.Equal(-1, leaderAddrIdx) re.Contains(pdAddrs[0], httpScheme) c = NewClient("test-https-pd-addr", []string{"127.0.0.1"}, WithTLSConfig(&tls.Config{})) pdAddrs, leaderAddrIdx = c.(*client).inner.getPDAddrs() - re.Equal(1, len(pdAddrs)) + re.Len(pdAddrs, 1) re.Equal(-1, leaderAddrIdx) re.Contains(pdAddrs[0], httpsScheme) } diff --git a/client/http/types_test.go b/client/http/types_test.go index 1dedbdc7d3b..39c53ae525d 100644 --- a/client/http/types_test.go +++ b/client/http/types_test.go @@ -45,7 +45,7 @@ func TestMergeRegionsInfo(t *testing.T) { } regionsInfo := regionsInfo1.Merge(regionsInfo2) re.Equal(int64(2), regionsInfo.Count) - re.Equal(2, len(regionsInfo.Regions)) + re.Len(regionsInfo.Regions, 2) re.Subset(regionsInfo.Regions, append(regionsInfo1.Regions, regionsInfo2.Regions...)) } diff --git a/client/option_test.go b/client/option_test.go index 1a8faf8fcd9..6dba0621078 100644 --- a/client/option_test.go +++ b/client/option_test.go @@ -31,7 +31,7 @@ func TestDynamicOptionChange(t *testing.T) { re.Equal(defaultEnableFollowerHandle, o.getEnableFollowerHandle()) // Check the invalid value setting. - re.NotNil(o.setMaxTSOBatchWaitInterval(time.Second)) + re.Error(o.setMaxTSOBatchWaitInterval(time.Second)) re.Equal(defaultMaxTSOBatchWaitInterval, o.getMaxTSOBatchWaitInterval()) expectInterval := time.Millisecond o.setMaxTSOBatchWaitInterval(expectInterval) diff --git a/client/resource_group/controller/controller_test.go b/client/resource_group/controller/controller_test.go index 4d09e338222..fea4a133ad0 100644 --- a/client/resource_group/controller/controller_test.go +++ b/client/resource_group/controller/controller_test.go @@ -63,7 +63,7 @@ func TestGroupControlBurstable(t *testing.T) { counter.limiter.Reconfigure(time.Now(), args) } gc.updateAvgRequestResourcePerSec() - re.Equal(gc.burstable.Load(), true) + re.True(gc.burstable.Load()) } func TestRequestAndResponseConsumption(t *testing.T) { diff --git a/client/resource_group/controller/limiter_test.go b/client/resource_group/controller/limiter_test.go index d963f830551..786e5c51cdf 100644 --- a/client/resource_group/controller/limiter_test.go +++ b/client/resource_group/controller/limiter_test.go @@ -161,7 +161,7 @@ func TestCancel(t *testing.T) { checkTokens(re, lim1, t2, 7) checkTokens(re, lim2, t2, 2) d, err := WaitReservations(ctx, t2, []*Reservation{r1, r2}) - re.Equal(d, 4*time.Second) + re.Equal(4*time.Second, d) re.Error(err) checkTokens(re, lim1, t3, 13) checkTokens(re, lim2, t3, 3) diff --git a/client/resource_group/controller/util_test.go b/client/resource_group/controller/util_test.go index b542e6713dc..a89ea08b955 100644 --- a/client/resource_group/controller/util_test.go +++ b/client/resource_group/controller/util_test.go @@ -46,6 +46,6 @@ func TestDurationTOML(t *testing.T) { example := &example{} text := []byte(`interval = "1h1m1s"`) - re.Nil(toml.Unmarshal(text, example)) + re.NoError(toml.Unmarshal(text, example)) re.Equal(float64(60*60+60+1), example.Interval.Seconds()) } diff --git a/pkg/cache/cache_test.go b/pkg/cache/cache_test.go index fe9f84223c1..904da1afb62 100644 --- a/pkg/cache/cache_test.go +++ b/pkg/cache/cache_test.go @@ -212,13 +212,13 @@ func TestFifoCache(t *testing.T) { elems := cache.Elems() re.Len(elems, 3) - re.Equal(elems[0].Value, "2") - re.Equal(elems[1].Value, "3") - re.Equal(elems[2].Value, "4") + re.Equal("2", elems[0].Value) + re.Equal("3", elems[1].Value) + re.Equal("4", elems[2].Value) elems = cache.FromElems(3) re.Len(elems, 1) - re.Equal(elems[0].Value, "4") + re.Equal("4", elems[0].Value) cache.Remove() cache.Remove() @@ -269,15 +269,15 @@ func TestTwoQueueCache(t *testing.T) { val, ok := cache.Get(3) re.True(ok) - re.Equal(val, "3") + re.Equal("3", val) val, ok = cache.Get(2) re.True(ok) - re.Equal(val, "2") + re.Equal("2", val) val, ok = cache.Get(1) re.True(ok) - re.Equal(val, "1") + re.Equal("1", val) re.Equal(3, cache.Len()) @@ -291,27 +291,27 @@ func TestTwoQueueCache(t *testing.T) { val, ok = cache.Get(1) re.True(ok) - re.Equal(val, "1") + re.Equal("1", val) val, ok = cache.Get(2) re.True(ok) - re.Equal(val, "2") + re.Equal("2", val) val, ok = cache.Get(4) re.True(ok) - re.Equal(val, "4") + re.Equal("4", val) re.Equal(3, cache.Len()) val, ok = cache.Peek(1) re.True(ok) - re.Equal(val, "1") + re.Equal("1", val) elems := cache.Elems() re.Len(elems, 3) - re.Equal(elems[0].Value, "4") - re.Equal(elems[1].Value, "2") - re.Equal(elems[2].Value, "1") + re.Equal("4", elems[0].Value) + re.Equal("2", elems[1].Value) + re.Equal("1", elems[2].Value) cache.Remove(1) cache.Remove(2) diff --git a/pkg/cgroup/cgroup_cpu_test.go b/pkg/cgroup/cgroup_cpu_test.go index c0dbdfcd2b9..f0b9239ecab 100644 --- a/pkg/cgroup/cgroup_cpu_test.go +++ b/pkg/cgroup/cgroup_cpu_test.go @@ -28,10 +28,10 @@ import ( "github.com/stretchr/testify/require" ) -func checkKernelVersionNewerThan(t *testing.T, major, minor int) bool { +func checkKernelVersionNewerThan(re *require.Assertions, t *testing.T, major, minor int) bool { u := syscall.Utsname{} err := syscall.Uname(&u) - require.NoError(t, err) + re.NoError(err) releaseBs := make([]byte, 0, len(u.Release)) for _, v := range u.Release { if v == 0 { @@ -42,16 +42,16 @@ func checkKernelVersionNewerThan(t *testing.T, major, minor int) bool { releaseStr := string(releaseBs) t.Log("kernel release string:", releaseStr) versionInfoRE := regexp.MustCompile(`[0-9]+\.[0-9]+\.[0-9]+`) - kernelVerion := versionInfoRE.FindAllString(releaseStr, 1) - require.Equal(t, 1, len(kernelVerion), fmt.Sprintf("release str is %s", releaseStr)) + kernelVersion := versionInfoRE.FindAllString(releaseStr, 1) + re.Len(kernelVersion, 1, fmt.Sprintf("release str is %s", releaseStr)) kernelVersionPartRE := regexp.MustCompile(`[0-9]+`) - kernelVersionParts := kernelVersionPartRE.FindAllString(kernelVerion[0], -1) - require.Equal(t, 3, len(kernelVersionParts), fmt.Sprintf("kernel verion str is %s", kernelVerion[0])) + kernelVersionParts := kernelVersionPartRE.FindAllString(kernelVersion[0], -1) + re.Len(kernelVersionParts, 3, fmt.Sprintf("kernel verion str is %s", kernelVersion[0])) t.Logf("parsed kernel version parts: major %s, minor %s, patch %s", kernelVersionParts[0], kernelVersionParts[1], kernelVersionParts[2]) mustConvInt := func(s string) int { i, err := strconv.Atoi(s) - require.NoError(t, err, s) + re.NoError(err, s) return i } versionNewerThanFlag := false @@ -64,6 +64,7 @@ func checkKernelVersionNewerThan(t *testing.T, major, minor int) bool { } func TestGetCgroupCPU(t *testing.T) { + re := require.New(t) exit := make(chan struct{}) var wg sync.WaitGroup for i := 0; i < 10; i++ { @@ -83,15 +84,15 @@ func TestGetCgroupCPU(t *testing.T) { cpu, err := GetCgroupCPU() if err == errNoCPUControllerDetected { // for more information, please refer https://github.com/pingcap/tidb/pull/41347 - if checkKernelVersionNewerThan(t, 4, 7) { - require.NoError(t, err, "linux version > v4.7 and err still happens") + if checkKernelVersionNewerThan(re, t, 4, 7) { + re.NoError(err, "linux version > v4.7 and err still happens") } else { t.Logf("the error '%s' is ignored because the kernel is too old", err) } } else { - require.NoError(t, err) - require.NotZero(t, cpu.Period) - require.Less(t, int64(1), cpu.Period) + re.NoError(err) + re.NotZero(cpu.Period) + re.Less(int64(1), cpu.Period) } close(exit) wg.Wait() diff --git a/pkg/dashboard/adapter/redirector_test.go b/pkg/dashboard/adapter/redirector_test.go index 5fc9ea5ea99..fff052f1d50 100644 --- a/pkg/dashboard/adapter/redirector_test.go +++ b/pkg/dashboard/adapter/redirector_test.go @@ -73,17 +73,17 @@ func (suite *redirectorTestSuite) TestReverseProxy() { // Test normal forwarding req, err := http.NewRequest(http.MethodGet, redirectorServer.URL, http.NoBody) re.NoError(err) - checkHTTPRequest(suite.Require(), suite.noRedirectHTTPClient, req, http.StatusOK, suite.tempText) + checkHTTPRequest(re, suite.noRedirectHTTPClient, req, http.StatusOK, suite.tempText) // Test the requests that are forwarded by others req, err = http.NewRequest(http.MethodGet, redirectorServer.URL, http.NoBody) re.NoError(err) req.Header.Set(proxyHeader, "other") - checkHTTPRequest(suite.Require(), suite.noRedirectHTTPClient, req, http.StatusOK, suite.tempText) + checkHTTPRequest(re, suite.noRedirectHTTPClient, req, http.StatusOK, suite.tempText) // Test LoopDetected suite.redirector.SetAddress(redirectorServer.URL) req, err = http.NewRequest(http.MethodGet, redirectorServer.URL, http.NoBody) re.NoError(err) - checkHTTPRequest(suite.Require(), suite.noRedirectHTTPClient, req, http.StatusLoopDetected, "") + checkHTTPRequest(re, suite.noRedirectHTTPClient, req, http.StatusLoopDetected, "") } func (suite *redirectorTestSuite) TestTemporaryRedirect() { @@ -94,11 +94,11 @@ func (suite *redirectorTestSuite) TestTemporaryRedirect() { // Test TemporaryRedirect req, err := http.NewRequest(http.MethodGet, redirectorServer.URL, http.NoBody) re.NoError(err) - checkHTTPRequest(suite.Require(), suite.noRedirectHTTPClient, req, http.StatusTemporaryRedirect, "") + checkHTTPRequest(re, suite.noRedirectHTTPClient, req, http.StatusTemporaryRedirect, "") // Test Response req, err = http.NewRequest(http.MethodGet, redirectorServer.URL, http.NoBody) re.NoError(err) - checkHTTPRequest(suite.Require(), http.DefaultClient, req, http.StatusOK, suite.tempText) + checkHTTPRequest(re, http.DefaultClient, req, http.StatusOK, suite.tempText) } func checkHTTPRequest(re *require.Assertions, client *http.Client, req *http.Request, expectedCode int, expectedText string) { diff --git a/pkg/schedule/checker/merge_checker_test.go b/pkg/schedule/checker/merge_checker_test.go index 5e9311c76cd..40466d33947 100644 --- a/pkg/schedule/checker/merge_checker_test.go +++ b/pkg/schedule/checker/merge_checker_test.go @@ -88,102 +88,103 @@ func (suite *mergeCheckerTestSuite) TearDownTest() { } func (suite *mergeCheckerTestSuite) TestBasic() { + re := suite.Require() suite.cluster.SetSplitMergeInterval(0) // should with same peer count ops := suite.mc.Check(suite.regions[0]) - suite.Nil(ops) + re.Nil(ops) // The size should be small enough. ops = suite.mc.Check(suite.regions[1]) - suite.Nil(ops) + re.Nil(ops) // target region size is too large suite.cluster.PutRegion(suite.regions[1].Clone(core.SetApproximateSize(600))) ops = suite.mc.Check(suite.regions[2]) - suite.Nil(ops) + re.Nil(ops) // it can merge if the max region size of the store is greater than the target region size. suite.cluster.SetRegionMaxSize("144MiB") suite.cluster.SetRegionSizeMB(1024) ops = suite.mc.Check(suite.regions[2]) - suite.NotNil(ops) + re.NotNil(ops) suite.cluster.SetRegionSizeMB(144) ops = suite.mc.Check(suite.regions[2]) - suite.Nil(ops) + re.Nil(ops) // change the size back suite.cluster.PutRegion(suite.regions[1].Clone(core.SetApproximateSize(200))) ops = suite.mc.Check(suite.regions[2]) - suite.NotNil(ops) + re.NotNil(ops) // Check merge with previous region. - suite.Equal(suite.regions[2].GetID(), ops[0].RegionID()) - suite.Equal(suite.regions[1].GetID(), ops[1].RegionID()) + re.Equal(suite.regions[2].GetID(), ops[0].RegionID()) + re.Equal(suite.regions[1].GetID(), ops[1].RegionID()) // Test the peer store check. store := suite.cluster.GetStore(1) - suite.NotNil(store) + re.NotNil(store) // Test the peer store is deleted. suite.cluster.DeleteStore(store) ops = suite.mc.Check(suite.regions[2]) - suite.Nil(ops) + re.Nil(ops) // Test the store is normal. suite.cluster.PutStore(store) ops = suite.mc.Check(suite.regions[2]) - suite.NotNil(ops) - suite.Equal(suite.regions[2].GetID(), ops[0].RegionID()) - suite.Equal(suite.regions[1].GetID(), ops[1].RegionID()) + re.NotNil(ops) + re.Equal(suite.regions[2].GetID(), ops[0].RegionID()) + re.Equal(suite.regions[1].GetID(), ops[1].RegionID()) // Test the store is offline. suite.cluster.SetStoreOffline(store.GetID()) ops = suite.mc.Check(suite.regions[2]) // Only target region have a peer on the offline store, // so it's not ok to merge. - suite.Nil(ops) + re.Nil(ops) // Test the store is up. suite.cluster.SetStoreUp(store.GetID()) ops = suite.mc.Check(suite.regions[2]) - suite.NotNil(ops) - suite.Equal(suite.regions[2].GetID(), ops[0].RegionID()) - suite.Equal(suite.regions[1].GetID(), ops[1].RegionID()) + re.NotNil(ops) + re.Equal(suite.regions[2].GetID(), ops[0].RegionID()) + re.Equal(suite.regions[1].GetID(), ops[1].RegionID()) store = suite.cluster.GetStore(5) - suite.NotNil(store) + re.NotNil(store) // Test the peer store is deleted. suite.cluster.DeleteStore(store) ops = suite.mc.Check(suite.regions[2]) - suite.Nil(ops) + re.Nil(ops) // Test the store is normal. suite.cluster.PutStore(store) ops = suite.mc.Check(suite.regions[2]) - suite.NotNil(ops) - suite.Equal(suite.regions[2].GetID(), ops[0].RegionID()) - suite.Equal(suite.regions[1].GetID(), ops[1].RegionID()) + re.NotNil(ops) + re.Equal(suite.regions[2].GetID(), ops[0].RegionID()) + re.Equal(suite.regions[1].GetID(), ops[1].RegionID()) // Test the store is offline. suite.cluster.SetStoreOffline(store.GetID()) ops = suite.mc.Check(suite.regions[2]) // Both regions have peers on the offline store, // so it's ok to merge. - suite.NotNil(ops) - suite.Equal(suite.regions[2].GetID(), ops[0].RegionID()) - suite.Equal(suite.regions[1].GetID(), ops[1].RegionID()) + re.NotNil(ops) + re.Equal(suite.regions[2].GetID(), ops[0].RegionID()) + re.Equal(suite.regions[1].GetID(), ops[1].RegionID()) // Test the store is up. suite.cluster.SetStoreUp(store.GetID()) ops = suite.mc.Check(suite.regions[2]) - suite.NotNil(ops) - suite.Equal(suite.regions[2].GetID(), ops[0].RegionID()) - suite.Equal(suite.regions[1].GetID(), ops[1].RegionID()) + re.NotNil(ops) + re.Equal(suite.regions[2].GetID(), ops[0].RegionID()) + re.Equal(suite.regions[1].GetID(), ops[1].RegionID()) // Enable one way merge suite.cluster.SetEnableOneWayMerge(true) ops = suite.mc.Check(suite.regions[2]) - suite.Nil(ops) + re.Nil(ops) suite.cluster.SetEnableOneWayMerge(false) // Make up peers for next region. suite.regions[3] = suite.regions[3].Clone(core.WithAddPeer(&metapb.Peer{Id: 110, StoreId: 1}), core.WithAddPeer(&metapb.Peer{Id: 111, StoreId: 2})) suite.cluster.PutRegion(suite.regions[3]) ops = suite.mc.Check(suite.regions[2]) - suite.NotNil(ops) + re.NotNil(ops) // Now it merges to next region. - suite.Equal(suite.regions[2].GetID(), ops[0].RegionID()) - suite.Equal(suite.regions[3].GetID(), ops[1].RegionID()) + re.Equal(suite.regions[2].GetID(), ops[0].RegionID()) + re.Equal(suite.regions[3].GetID(), ops[1].RegionID()) // merge cannot across rule key. suite.cluster.SetEnablePlacementRules(true) @@ -199,9 +200,9 @@ func (suite *mergeCheckerTestSuite) TestBasic() { }) // region 2 can only merge with previous region now. ops = suite.mc.Check(suite.regions[2]) - suite.NotNil(ops) - suite.Equal(suite.regions[2].GetID(), ops[0].RegionID()) - suite.Equal(suite.regions[1].GetID(), ops[1].RegionID()) + re.NotNil(ops) + re.Equal(suite.regions[2].GetID(), ops[0].RegionID()) + re.Equal(suite.regions[1].GetID(), ops[1].RegionID()) suite.cluster.RuleManager.DeleteRule(placement.DefaultGroupID, "test") // check 'merge_option' label @@ -212,46 +213,47 @@ func (suite *mergeCheckerTestSuite) TestBasic() { Data: makeKeyRanges("", "74"), }) ops = suite.mc.Check(suite.regions[0]) - suite.Empty(ops) + re.Empty(ops) ops = suite.mc.Check(suite.regions[1]) - suite.Empty(ops) + re.Empty(ops) // Skip recently split regions. suite.cluster.SetSplitMergeInterval(time.Hour) ops = suite.mc.Check(suite.regions[2]) - suite.Nil(ops) + re.Nil(ops) suite.mc.startTime = time.Now().Add(-2 * time.Hour) ops = suite.mc.Check(suite.regions[2]) - suite.NotNil(ops) + re.NotNil(ops) ops = suite.mc.Check(suite.regions[3]) - suite.NotNil(ops) + re.NotNil(ops) suite.mc.RecordRegionSplit([]uint64{suite.regions[2].GetID()}) ops = suite.mc.Check(suite.regions[2]) - suite.Nil(ops) + re.Nil(ops) ops = suite.mc.Check(suite.regions[3]) - suite.Nil(ops) + re.Nil(ops) suite.cluster.SetSplitMergeInterval(500 * time.Millisecond) ops = suite.mc.Check(suite.regions[2]) - suite.Nil(ops) + re.Nil(ops) ops = suite.mc.Check(suite.regions[3]) - suite.Nil(ops) + re.Nil(ops) time.Sleep(500 * time.Millisecond) ops = suite.mc.Check(suite.regions[2]) - suite.NotNil(ops) + re.NotNil(ops) ops = suite.mc.Check(suite.regions[3]) - suite.NotNil(ops) + re.NotNil(ops) } func (suite *mergeCheckerTestSuite) TestMatchPeers() { + re := suite.Require() suite.cluster.SetSplitMergeInterval(0) // partial store overlap not including leader ops := suite.mc.Check(suite.regions[2]) - suite.NotNil(ops) - operatorutil.CheckSteps(suite.Require(), ops[0], []operator.OpStep{ + re.NotNil(ops) + operatorutil.CheckSteps(re, ops[0], []operator.OpStep{ operator.AddLearner{ToStore: 1}, operator.PromoteLearner{ToStore: 1}, operator.RemovePeer{FromStore: 2}, @@ -265,7 +267,7 @@ func (suite *mergeCheckerTestSuite) TestMatchPeers() { IsPassive: false, }, }) - operatorutil.CheckSteps(suite.Require(), ops[1], []operator.OpStep{ + operatorutil.CheckSteps(re, ops[1], []operator.OpStep{ operator.MergeRegion{ FromRegion: suite.regions[2].GetMeta(), ToRegion: suite.regions[1].GetMeta(), @@ -285,7 +287,7 @@ func (suite *mergeCheckerTestSuite) TestMatchPeers() { suite.regions[2] = newRegion suite.cluster.PutRegion(suite.regions[2]) ops = suite.mc.Check(suite.regions[2]) - operatorutil.CheckSteps(suite.Require(), ops[0], []operator.OpStep{ + operatorutil.CheckSteps(re, ops[0], []operator.OpStep{ operator.AddLearner{ToStore: 4}, operator.PromoteLearner{ToStore: 4}, operator.RemovePeer{FromStore: 6}, @@ -295,7 +297,7 @@ func (suite *mergeCheckerTestSuite) TestMatchPeers() { IsPassive: false, }, }) - operatorutil.CheckSteps(suite.Require(), ops[1], []operator.OpStep{ + operatorutil.CheckSteps(re, ops[1], []operator.OpStep{ operator.MergeRegion{ FromRegion: suite.regions[2].GetMeta(), ToRegion: suite.regions[1].GetMeta(), @@ -311,14 +313,14 @@ func (suite *mergeCheckerTestSuite) TestMatchPeers() { })) suite.cluster.PutRegion(suite.regions[2]) ops = suite.mc.Check(suite.regions[2]) - operatorutil.CheckSteps(suite.Require(), ops[0], []operator.OpStep{ + operatorutil.CheckSteps(re, ops[0], []operator.OpStep{ operator.MergeRegion{ FromRegion: suite.regions[2].GetMeta(), ToRegion: suite.regions[1].GetMeta(), IsPassive: false, }, }) - operatorutil.CheckSteps(suite.Require(), ops[1], []operator.OpStep{ + operatorutil.CheckSteps(re, ops[1], []operator.OpStep{ operator.MergeRegion{ FromRegion: suite.regions[2].GetMeta(), ToRegion: suite.regions[1].GetMeta(), @@ -334,7 +336,7 @@ func (suite *mergeCheckerTestSuite) TestMatchPeers() { }), core.WithLeader(&metapb.Peer{Id: 109, StoreId: 2})) suite.cluster.PutRegion(suite.regions[2]) ops = suite.mc.Check(suite.regions[2]) - operatorutil.CheckSteps(suite.Require(), ops[0], []operator.OpStep{ + operatorutil.CheckSteps(re, ops[0], []operator.OpStep{ operator.AddLearner{ToStore: 1}, operator.PromoteLearner{ToStore: 1}, operator.RemovePeer{FromStore: 3}, @@ -351,7 +353,7 @@ func (suite *mergeCheckerTestSuite) TestMatchPeers() { IsPassive: false, }, }) - operatorutil.CheckSteps(suite.Require(), ops[1], []operator.OpStep{ + operatorutil.CheckSteps(re, ops[1], []operator.OpStep{ operator.MergeRegion{ FromRegion: suite.regions[2].GetMeta(), ToRegion: suite.regions[1].GetMeta(), @@ -370,7 +372,7 @@ func (suite *mergeCheckerTestSuite) TestMatchPeers() { ) suite.cluster.PutRegion(suite.regions[1]) ops = suite.mc.Check(suite.regions[2]) - operatorutil.CheckSteps(suite.Require(), ops[0], []operator.OpStep{ + operatorutil.CheckSteps(re, ops[0], []operator.OpStep{ operator.AddLearner{ToStore: 1}, operator.PromoteLearner{ToStore: 1}, operator.RemovePeer{FromStore: 3}, @@ -390,7 +392,7 @@ func (suite *mergeCheckerTestSuite) TestMatchPeers() { IsPassive: false, }, }) - operatorutil.CheckSteps(suite.Require(), ops[1], []operator.OpStep{ + operatorutil.CheckSteps(re, ops[1], []operator.OpStep{ operator.MergeRegion{ FromRegion: suite.regions[2].GetMeta(), ToRegion: suite.regions[1].GetMeta(), @@ -417,7 +419,7 @@ func (suite *mergeCheckerTestSuite) TestMatchPeers() { ) suite.cluster.PutRegion(suite.regions[1]) ops = suite.mc.Check(suite.regions[2]) - operatorutil.CheckSteps(suite.Require(), ops[0], []operator.OpStep{ + operatorutil.CheckSteps(re, ops[0], []operator.OpStep{ operator.AddLearner{ToStore: 1}, operator.PromoteLearner{ToStore: 1}, operator.RemovePeer{FromStore: 3}, @@ -431,7 +433,7 @@ func (suite *mergeCheckerTestSuite) TestMatchPeers() { IsPassive: false, }, }) - operatorutil.CheckSteps(suite.Require(), ops[1], []operator.OpStep{ + operatorutil.CheckSteps(re, ops[1], []operator.OpStep{ operator.MergeRegion{ FromRegion: suite.regions[2].GetMeta(), ToRegion: suite.regions[1].GetMeta(), @@ -441,6 +443,7 @@ func (suite *mergeCheckerTestSuite) TestMatchPeers() { } func (suite *mergeCheckerTestSuite) TestStoreLimitWithMerge() { + re := suite.Require() cfg := mockconfig.NewTestOptions() tc := mockcluster.NewCluster(suite.ctx, cfg) tc.SetMaxMergeRegionSize(2) @@ -481,8 +484,8 @@ func (suite *mergeCheckerTestSuite) TestStoreLimitWithMerge() { // The size of Region is less or equal than 1MB. for i := 0; i < 50; i++ { ops := mc.Check(regions[2]) - suite.NotNil(ops) - suite.True(oc.AddOperator(ops...)) + re.NotNil(ops) + re.True(oc.AddOperator(ops...)) for _, op := range ops { oc.RemoveOperator(op, operator.ExceedStoreLimit) } @@ -495,20 +498,21 @@ func (suite *mergeCheckerTestSuite) TestStoreLimitWithMerge() { // The size of Region is more than 1MB but no more than 20MB. for i := 0; i < 5; i++ { ops := mc.Check(regions[2]) - suite.NotNil(ops) - suite.True(oc.AddOperator(ops...)) + re.NotNil(ops) + re.True(oc.AddOperator(ops...)) for _, op := range ops { oc.RemoveOperator(op, operator.ExceedStoreLimit) } } { ops := mc.Check(regions[2]) - suite.NotNil(ops) - suite.False(oc.AddOperator(ops...)) + re.NotNil(ops) + re.False(oc.AddOperator(ops...)) } } func (suite *mergeCheckerTestSuite) TestCache() { + re := suite.Require() cfg := mockconfig.NewTestOptions() suite.cluster = mockcluster.NewCluster(suite.ctx, cfg) suite.cluster.SetMaxMergeRegionSize(2) @@ -533,11 +537,11 @@ func (suite *mergeCheckerTestSuite) TestCache() { suite.mc = NewMergeChecker(suite.ctx, suite.cluster, suite.cluster.GetCheckerConfig()) ops := suite.mc.Check(suite.regions[1]) - suite.Nil(ops) + re.Nil(ops) suite.cluster.SetSplitMergeInterval(0) time.Sleep(time.Second) ops = suite.mc.Check(suite.regions[1]) - suite.NotNil(ops) + re.NotNil(ops) } func makeKeyRanges(keys ...string) []interface{} { diff --git a/pkg/schedule/checker/replica_checker_test.go b/pkg/schedule/checker/replica_checker_test.go index a326d39d451..a9139ee9804 100644 --- a/pkg/schedule/checker/replica_checker_test.go +++ b/pkg/schedule/checker/replica_checker_test.go @@ -22,6 +22,7 @@ import ( "github.com/docker/go-units" "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/kvproto/pkg/pdpb" + "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "github.com/tikv/pd/pkg/cache" "github.com/tikv/pd/pkg/core" @@ -92,6 +93,7 @@ func (suite *replicaCheckerTestSuite) TearDownTest() { } func (suite *replicaCheckerTestSuite) TestReplacePendingPeer() { + re := suite.Require() peers := []*metapb.Peer{ { Id: 2, @@ -109,13 +111,14 @@ func (suite *replicaCheckerTestSuite) TestReplacePendingPeer() { r := core.NewRegionInfo(&metapb.Region{Id: 1, Peers: peers}, peers[1], core.WithPendingPeers(peers[0:1])) suite.cluster.PutRegion(r) op := suite.rc.Check(r) - suite.NotNil(op) - suite.Equal(uint64(4), op.Step(0).(operator.AddLearner).ToStore) - suite.Equal(uint64(4), op.Step(1).(operator.PromoteLearner).ToStore) - suite.Equal(uint64(1), op.Step(2).(operator.RemovePeer).FromStore) + re.NotNil(op) + re.Equal(uint64(4), op.Step(0).(operator.AddLearner).ToStore) + re.Equal(uint64(4), op.Step(1).(operator.PromoteLearner).ToStore) + re.Equal(uint64(1), op.Step(2).(operator.RemovePeer).FromStore) } func (suite *replicaCheckerTestSuite) TestReplaceOfflinePeer() { + re := suite.Require() suite.cluster.SetLabelProperty(config.RejectLeader, "noleader", "true") peers := []*metapb.Peer{ { @@ -134,14 +137,15 @@ func (suite *replicaCheckerTestSuite) TestReplaceOfflinePeer() { r := core.NewRegionInfo(&metapb.Region{Id: 2, Peers: peers}, peers[0]) suite.cluster.PutRegion(r) op := suite.rc.Check(r) - suite.NotNil(op) - suite.Equal(uint64(3), op.Step(0).(operator.TransferLeader).ToStore) - suite.Equal(uint64(4), op.Step(1).(operator.AddLearner).ToStore) - suite.Equal(uint64(4), op.Step(2).(operator.PromoteLearner).ToStore) - suite.Equal(uint64(1), op.Step(3).(operator.RemovePeer).FromStore) + re.NotNil(op) + re.Equal(uint64(3), op.Step(0).(operator.TransferLeader).ToStore) + re.Equal(uint64(4), op.Step(1).(operator.AddLearner).ToStore) + re.Equal(uint64(4), op.Step(2).(operator.PromoteLearner).ToStore) + re.Equal(uint64(1), op.Step(3).(operator.RemovePeer).FromStore) } func (suite *replicaCheckerTestSuite) TestOfflineWithOneReplica() { + re := suite.Require() suite.cluster.SetMaxReplicas(1) peers := []*metapb.Peer{ { @@ -152,23 +156,24 @@ func (suite *replicaCheckerTestSuite) TestOfflineWithOneReplica() { r := core.NewRegionInfo(&metapb.Region{Id: 2, Peers: peers}, peers[0]) suite.cluster.PutRegion(r) op := suite.rc.Check(r) - suite.NotNil(op) - suite.Equal("replace-offline-replica", op.Desc()) + re.NotNil(op) + re.Equal("replace-offline-replica", op.Desc()) } func (suite *replicaCheckerTestSuite) TestDownPeer() { + re := suite.Require() // down a peer, the number of normal peers(except learner) is enough. - op := suite.downPeerAndCheck(metapb.PeerRole_Voter) - suite.NotNil(op) - suite.Equal("remove-extra-down-replica", op.Desc()) + op := suite.downPeerAndCheck(re, metapb.PeerRole_Voter) + re.NotNil(op) + re.Equal("remove-extra-down-replica", op.Desc()) // down a peer, the number of peers(except learner) is not enough. - op = suite.downPeerAndCheck(metapb.PeerRole_Learner) - suite.NotNil(op) - suite.Equal("replace-down-replica", op.Desc()) + op = suite.downPeerAndCheck(re, metapb.PeerRole_Learner) + re.NotNil(op) + re.Equal("replace-down-replica", op.Desc()) } -func (suite *replicaCheckerTestSuite) downPeerAndCheck(aliveRole metapb.PeerRole) *operator.Operator { +func (suite *replicaCheckerTestSuite) downPeerAndCheck(re *require.Assertions, aliveRole metapb.PeerRole) *operator.Operator { suite.cluster.SetMaxReplicas(2) suite.cluster.SetStoreUp(1) downStoreID := uint64(3) @@ -198,11 +203,12 @@ func (suite *replicaCheckerTestSuite) downPeerAndCheck(aliveRole metapb.PeerRole DownSeconds: 24 * 60 * 60, } r = r.Clone(core.WithDownPeers(append(r.GetDownPeers(), downPeer))) - suite.Len(r.GetDownPeers(), 1) + re.Len(r.GetDownPeers(), 1) return suite.rc.Check(r) } func (suite *replicaCheckerTestSuite) TestBasic() { + re := suite.Require() opt := mockconfig.NewTestOptions() tc := mockcluster.NewCluster(suite.ctx, opt) tc.SetMaxSnapshotCount(2) @@ -219,41 +225,41 @@ func (suite *replicaCheckerTestSuite) TestBasic() { // Region has 2 peers, we need to add a new peer. region := tc.GetRegion(1) - operatorutil.CheckAddPeer(suite.Require(), rc.Check(region), operator.OpReplica, 4) + operatorutil.CheckAddPeer(re, rc.Check(region), operator.OpReplica, 4) // Disable make up replica feature. tc.SetEnableMakeUpReplica(false) - suite.Nil(rc.Check(region)) + re.Nil(rc.Check(region)) tc.SetEnableMakeUpReplica(true) // Test healthFilter. // If store 4 is down, we add to store 3. tc.SetStoreDown(4) - operatorutil.CheckAddPeer(suite.Require(), rc.Check(region), operator.OpReplica, 3) + operatorutil.CheckAddPeer(re, rc.Check(region), operator.OpReplica, 3) tc.SetStoreUp(4) - operatorutil.CheckAddPeer(suite.Require(), rc.Check(region), operator.OpReplica, 4) + operatorutil.CheckAddPeer(re, rc.Check(region), operator.OpReplica, 4) // Test snapshotCountFilter. // If snapshotCount > MaxSnapshotCount, we add to store 3. tc.UpdateSnapshotCount(4, 3) - operatorutil.CheckAddPeer(suite.Require(), rc.Check(region), operator.OpReplica, 3) + operatorutil.CheckAddPeer(re, rc.Check(region), operator.OpReplica, 3) // If snapshotCount < MaxSnapshotCount, we can add peer again. tc.UpdateSnapshotCount(4, 1) - operatorutil.CheckAddPeer(suite.Require(), rc.Check(region), operator.OpReplica, 4) + operatorutil.CheckAddPeer(re, rc.Check(region), operator.OpReplica, 4) // Add peer in store 4, and we have enough replicas. peer4, _ := tc.AllocPeer(4) region = region.Clone(core.WithAddPeer(peer4)) - suite.Nil(rc.Check(region)) + re.Nil(rc.Check(region)) // Add peer in store 3, and we have redundant replicas. peer3, _ := tc.AllocPeer(3) region = region.Clone(core.WithAddPeer(peer3)) - operatorutil.CheckRemovePeer(suite.Require(), rc.Check(region), 1) + operatorutil.CheckRemovePeer(re, rc.Check(region), 1) // Disable remove extra replica feature. tc.SetEnableRemoveExtraReplica(false) - suite.Nil(rc.Check(region)) + re.Nil(rc.Check(region)) tc.SetEnableRemoveExtraReplica(true) region = region.Clone(core.WithRemoveStorePeer(1), core.WithLeader(region.GetStorePeer(3))) @@ -266,16 +272,17 @@ func (suite *replicaCheckerTestSuite) TestBasic() { } region = region.Clone(core.WithDownPeers(append(region.GetDownPeers(), downPeer))) - operatorutil.CheckTransferPeer(suite.Require(), rc.Check(region), operator.OpReplica, 2, 1) + operatorutil.CheckTransferPeer(re, rc.Check(region), operator.OpReplica, 2, 1) region = region.Clone(core.WithDownPeers(nil)) - suite.Nil(rc.Check(region)) + re.Nil(rc.Check(region)) // Peer in store 3 is offline, transfer peer to store 1. tc.SetStoreOffline(3) - operatorutil.CheckTransferPeer(suite.Require(), rc.Check(region), operator.OpReplica, 3, 1) + operatorutil.CheckTransferPeer(re, rc.Check(region), operator.OpReplica, 3, 1) } func (suite *replicaCheckerTestSuite) TestLostStore() { + re := suite.Require() opt := mockconfig.NewTestOptions() tc := mockcluster.NewCluster(suite.ctx, opt) @@ -291,10 +298,11 @@ func (suite *replicaCheckerTestSuite) TestLostStore() { tc.AddLeaderRegion(1, 1, 2, 3) region := tc.GetRegion(1) op := rc.Check(region) - suite.Nil(op) + re.Nil(op) } func (suite *replicaCheckerTestSuite) TestOffline() { + re := suite.Require() opt := mockconfig.NewTestOptions() tc := mockcluster.NewCluster(suite.ctx, opt) tc.SetClusterVersion(versioninfo.MinSupportedVersion(versioninfo.Version4_0)) @@ -311,41 +319,42 @@ func (suite *replicaCheckerTestSuite) TestOffline() { region := tc.GetRegion(1) // Store 2 has different zone and smallest region score. - operatorutil.CheckAddPeer(suite.Require(), rc.Check(region), operator.OpReplica, 2) + operatorutil.CheckAddPeer(re, rc.Check(region), operator.OpReplica, 2) peer2, _ := tc.AllocPeer(2) region = region.Clone(core.WithAddPeer(peer2)) // Store 3 has different zone and smallest region score. - operatorutil.CheckAddPeer(suite.Require(), rc.Check(region), operator.OpReplica, 3) + operatorutil.CheckAddPeer(re, rc.Check(region), operator.OpReplica, 3) peer3, _ := tc.AllocPeer(3) region = region.Clone(core.WithAddPeer(peer3)) // Store 4 has the same zone with store 3 and larger region score. peer4, _ := tc.AllocPeer(4) region = region.Clone(core.WithAddPeer(peer4)) - operatorutil.CheckRemovePeer(suite.Require(), rc.Check(region), 4) + operatorutil.CheckRemovePeer(re, rc.Check(region), 4) // Test offline // the number of region peers more than the maxReplicas // remove the peer tc.SetStoreOffline(3) - operatorutil.CheckRemovePeer(suite.Require(), rc.Check(region), 3) + operatorutil.CheckRemovePeer(re, rc.Check(region), 3) region = region.Clone(core.WithRemoveStorePeer(4)) // the number of region peers equals the maxReplicas // Transfer peer to store 4. - operatorutil.CheckTransferPeer(suite.Require(), rc.Check(region), operator.OpReplica, 3, 4) + operatorutil.CheckTransferPeer(re, rc.Check(region), operator.OpReplica, 3, 4) // Store 5 has a same label score with store 4, but the region score smaller than store 4, we will choose store 5. tc.AddLabelsStore(5, 3, map[string]string{"zone": "z4", "rack": "r1", "host": "h1"}) - operatorutil.CheckTransferPeer(suite.Require(), rc.Check(region), operator.OpReplica, 3, 5) + operatorutil.CheckTransferPeer(re, rc.Check(region), operator.OpReplica, 3, 5) // Store 5 has too many snapshots, choose store 4 tc.UpdateSnapshotCount(5, 100) - operatorutil.CheckTransferPeer(suite.Require(), rc.Check(region), operator.OpReplica, 3, 4) + operatorutil.CheckTransferPeer(re, rc.Check(region), operator.OpReplica, 3, 4) tc.UpdatePendingPeerCount(4, 100) - suite.Nil(rc.Check(region)) + re.Nil(rc.Check(region)) } func (suite *replicaCheckerTestSuite) TestDistinctScore() { + re := suite.Require() opt := mockconfig.NewTestOptions() tc := mockcluster.NewCluster(suite.ctx, opt) tc.SetClusterVersion(versioninfo.MinSupportedVersion(versioninfo.Version4_0)) @@ -360,71 +369,72 @@ func (suite *replicaCheckerTestSuite) TestDistinctScore() { // We need 3 replicas. tc.AddLeaderRegion(1, 1) region := tc.GetRegion(1) - operatorutil.CheckAddPeer(suite.Require(), rc.Check(region), operator.OpReplica, 2) + operatorutil.CheckAddPeer(re, rc.Check(region), operator.OpReplica, 2) peer2, _ := tc.AllocPeer(2) region = region.Clone(core.WithAddPeer(peer2)) // Store 1,2,3 have the same zone, rack, and host. tc.AddLabelsStore(3, 5, map[string]string{"zone": "z1", "rack": "r1", "host": "h1"}) - operatorutil.CheckAddPeer(suite.Require(), rc.Check(region), operator.OpReplica, 3) + operatorutil.CheckAddPeer(re, rc.Check(region), operator.OpReplica, 3) // Store 4 has smaller region score. tc.AddLabelsStore(4, 4, map[string]string{"zone": "z1", "rack": "r1", "host": "h1"}) - operatorutil.CheckAddPeer(suite.Require(), rc.Check(region), operator.OpReplica, 4) + operatorutil.CheckAddPeer(re, rc.Check(region), operator.OpReplica, 4) // Store 5 has a different host. tc.AddLabelsStore(5, 5, map[string]string{"zone": "z1", "rack": "r1", "host": "h2"}) - operatorutil.CheckAddPeer(suite.Require(), rc.Check(region), operator.OpReplica, 5) + operatorutil.CheckAddPeer(re, rc.Check(region), operator.OpReplica, 5) // Store 6 has a different rack. tc.AddLabelsStore(6, 6, map[string]string{"zone": "z1", "rack": "r2", "host": "h1"}) - operatorutil.CheckAddPeer(suite.Require(), rc.Check(region), operator.OpReplica, 6) + operatorutil.CheckAddPeer(re, rc.Check(region), operator.OpReplica, 6) // Store 7 has a different zone. tc.AddLabelsStore(7, 7, map[string]string{"zone": "z2", "rack": "r1", "host": "h1"}) - operatorutil.CheckAddPeer(suite.Require(), rc.Check(region), operator.OpReplica, 7) + operatorutil.CheckAddPeer(re, rc.Check(region), operator.OpReplica, 7) // Test stateFilter. tc.SetStoreOffline(7) - operatorutil.CheckAddPeer(suite.Require(), rc.Check(region), operator.OpReplica, 6) + operatorutil.CheckAddPeer(re, rc.Check(region), operator.OpReplica, 6) tc.SetStoreUp(7) - operatorutil.CheckAddPeer(suite.Require(), rc.Check(region), operator.OpReplica, 7) + operatorutil.CheckAddPeer(re, rc.Check(region), operator.OpReplica, 7) // Add peer to store 7. peer7, _ := tc.AllocPeer(7) region = region.Clone(core.WithAddPeer(peer7)) // Replace peer in store 1 with store 6 because it has a different rack. - operatorutil.CheckTransferPeer(suite.Require(), rc.Check(region), operator.OpReplica, 1, 6) + operatorutil.CheckTransferPeer(re, rc.Check(region), operator.OpReplica, 1, 6) // Disable locationReplacement feature. tc.SetEnableLocationReplacement(false) - suite.Nil(rc.Check(region)) + re.Nil(rc.Check(region)) tc.SetEnableLocationReplacement(true) peer6, _ := tc.AllocPeer(6) region = region.Clone(core.WithAddPeer(peer6)) - operatorutil.CheckRemovePeer(suite.Require(), rc.Check(region), 1) + operatorutil.CheckRemovePeer(re, rc.Check(region), 1) region = region.Clone(core.WithRemoveStorePeer(1), core.WithLeader(region.GetStorePeer(2))) - suite.Nil(rc.Check(region)) + re.Nil(rc.Check(region)) // Store 8 has the same zone and different rack with store 7. // Store 1 has the same zone and different rack with store 6. // So store 8 and store 1 are equivalent. tc.AddLabelsStore(8, 1, map[string]string{"zone": "z2", "rack": "r2", "host": "h1"}) - suite.Nil(rc.Check(region)) + re.Nil(rc.Check(region)) // Store 10 has a different zone. // Store 2 and 6 have the same distinct score, but store 2 has larger region score. // So replace peer in store 2 with store 10. tc.AddLabelsStore(10, 1, map[string]string{"zone": "z3", "rack": "r1", "host": "h1"}) - operatorutil.CheckTransferPeer(suite.Require(), rc.Check(region), operator.OpReplica, 2, 10) + operatorutil.CheckTransferPeer(re, rc.Check(region), operator.OpReplica, 2, 10) peer10, _ := tc.AllocPeer(10) region = region.Clone(core.WithAddPeer(peer10)) - operatorutil.CheckRemovePeer(suite.Require(), rc.Check(region), 2) + operatorutil.CheckRemovePeer(re, rc.Check(region), 2) region = region.Clone(core.WithRemoveStorePeer(2)) - suite.Nil(rc.Check(region)) + re.Nil(rc.Check(region)) } func (suite *replicaCheckerTestSuite) TestDistinctScore2() { + re := suite.Require() opt := mockconfig.NewTestOptions() tc := mockcluster.NewCluster(suite.ctx, opt) tc.SetClusterVersion(versioninfo.MinSupportedVersion(versioninfo.Version4_0)) @@ -443,18 +453,19 @@ func (suite *replicaCheckerTestSuite) TestDistinctScore2() { tc.AddLeaderRegion(1, 1, 2, 4) region := tc.GetRegion(1) - operatorutil.CheckAddPeer(suite.Require(), rc.Check(region), operator.OpReplica, 6) + operatorutil.CheckAddPeer(re, rc.Check(region), operator.OpReplica, 6) peer6, _ := tc.AllocPeer(6) region = region.Clone(core.WithAddPeer(peer6)) - operatorutil.CheckAddPeer(suite.Require(), rc.Check(region), operator.OpReplica, 5) + operatorutil.CheckAddPeer(re, rc.Check(region), operator.OpReplica, 5) peer5, _ := tc.AllocPeer(5) region = region.Clone(core.WithAddPeer(peer5)) - suite.Nil(rc.Check(region)) + re.Nil(rc.Check(region)) } func (suite *replicaCheckerTestSuite) TestStorageThreshold() { + re := suite.Require() opt := mockconfig.NewTestOptions() tc := mockcluster.NewCluster(suite.ctx, opt) tc.SetLocationLabels([]string{"zone"}) @@ -475,22 +486,23 @@ func (suite *replicaCheckerTestSuite) TestStorageThreshold() { // Move peer to better location. tc.UpdateStorageRatio(4, 0, 1) - operatorutil.CheckTransferPeer(suite.Require(), rc.Check(region), operator.OpReplica, 1, 4) + operatorutil.CheckTransferPeer(re, rc.Check(region), operator.OpReplica, 1, 4) // If store4 is almost full, do not add peer on it. tc.UpdateStorageRatio(4, 0.9, 0.1) - suite.Nil(rc.Check(region)) + re.Nil(rc.Check(region)) tc.AddLeaderRegion(2, 1, 3) region = tc.GetRegion(2) // Add peer on store4. tc.UpdateStorageRatio(4, 0, 1) - operatorutil.CheckAddPeer(suite.Require(), rc.Check(region), operator.OpReplica, 4) + operatorutil.CheckAddPeer(re, rc.Check(region), operator.OpReplica, 4) // If store4 is almost full, do not add peer on it. tc.UpdateStorageRatio(4, 0.8, 0) - operatorutil.CheckAddPeer(suite.Require(), rc.Check(region), operator.OpReplica, 2) + operatorutil.CheckAddPeer(re, rc.Check(region), operator.OpReplica, 2) } func (suite *replicaCheckerTestSuite) TestOpts() { + re := suite.Require() opt := mockconfig.NewTestOptions() tc := mockcluster.NewCluster(suite.ctx, opt) tc.SetClusterVersion(versioninfo.MinSupportedVersion(versioninfo.Version4_0)) @@ -513,15 +525,16 @@ func (suite *replicaCheckerTestSuite) TestOpts() { })) tc.SetStoreOffline(2) // RemoveDownReplica has higher priority than replaceOfflineReplica. - operatorutil.CheckTransferPeer(suite.Require(), rc.Check(region), operator.OpReplica, 1, 4) + operatorutil.CheckTransferPeer(re, rc.Check(region), operator.OpReplica, 1, 4) tc.SetEnableRemoveDownReplica(false) - operatorutil.CheckTransferPeer(suite.Require(), rc.Check(region), operator.OpReplica, 2, 4) + operatorutil.CheckTransferPeer(re, rc.Check(region), operator.OpReplica, 2, 4) tc.SetEnableReplaceOfflineReplica(false) - suite.Nil(rc.Check(region)) + re.Nil(rc.Check(region)) } // See issue: https://github.com/tikv/pd/issues/3705 func (suite *replicaCheckerTestSuite) TestFixDownPeer() { + re := suite.Require() opt := mockconfig.NewTestOptions() tc := mockcluster.NewCluster(suite.ctx, opt) tc.SetClusterVersion(versioninfo.MinSupportedVersion(versioninfo.Version4_0)) @@ -536,23 +549,24 @@ func (suite *replicaCheckerTestSuite) TestFixDownPeer() { tc.AddLeaderRegion(1, 1, 3, 4) region := tc.GetRegion(1) - suite.Nil(rc.Check(region)) + re.Nil(rc.Check(region)) tc.SetStoreDown(4) region = region.Clone(core.WithDownPeers([]*pdpb.PeerStats{ {Peer: region.GetStorePeer(4), DownSeconds: 6000}, })) - operatorutil.CheckTransferPeer(suite.Require(), rc.Check(region), operator.OpRegion, 4, 5) + operatorutil.CheckTransferPeer(re, rc.Check(region), operator.OpRegion, 4, 5) tc.SetStoreDown(5) - operatorutil.CheckTransferPeer(suite.Require(), rc.Check(region), operator.OpRegion, 4, 2) + operatorutil.CheckTransferPeer(re, rc.Check(region), operator.OpRegion, 4, 2) tc.SetIsolationLevel("zone") - suite.Nil(rc.Check(region)) + re.Nil(rc.Check(region)) } // See issue: https://github.com/tikv/pd/issues/3705 func (suite *replicaCheckerTestSuite) TestFixOfflinePeer() { + re := suite.Require() opt := mockconfig.NewTestOptions() tc := mockcluster.NewCluster(suite.ctx, opt) tc.SetClusterVersion(versioninfo.MinSupportedVersion(versioninfo.Version4_0)) @@ -567,14 +581,14 @@ func (suite *replicaCheckerTestSuite) TestFixOfflinePeer() { tc.AddLeaderRegion(1, 1, 3, 4) region := tc.GetRegion(1) - suite.Nil(rc.Check(region)) + re.Nil(rc.Check(region)) tc.SetStoreOffline(4) - operatorutil.CheckTransferPeer(suite.Require(), rc.Check(region), operator.OpRegion, 4, 5) + operatorutil.CheckTransferPeer(re, rc.Check(region), operator.OpRegion, 4, 5) tc.SetStoreOffline(5) - operatorutil.CheckTransferPeer(suite.Require(), rc.Check(region), operator.OpRegion, 4, 2) + operatorutil.CheckTransferPeer(re, rc.Check(region), operator.OpRegion, 4, 2) tc.SetIsolationLevel("zone") - suite.Nil(rc.Check(region)) + re.Nil(rc.Check(region)) } diff --git a/pkg/schedule/checker/rule_checker_test.go b/pkg/schedule/checker/rule_checker_test.go index 6418483db7f..2668ac8cc43 100644 --- a/pkg/schedule/checker/rule_checker_test.go +++ b/pkg/schedule/checker/rule_checker_test.go @@ -70,18 +70,20 @@ func (suite *ruleCheckerTestSuite) TearDownTest() { } func (suite *ruleCheckerTestSuite) TestAddRulePeer() { + re := suite.Require() suite.cluster.AddLeaderStore(1, 1) suite.cluster.AddLeaderStore(2, 1) suite.cluster.AddLeaderStore(3, 1) suite.cluster.AddLeaderRegionWithRange(1, "", "", 1, 2) op := suite.rc.Check(suite.cluster.GetRegion(1)) - suite.NotNil(op) - suite.Equal("add-rule-peer", op.Desc()) - suite.Equal(constant.High, op.GetPriorityLevel()) - suite.Equal(uint64(3), op.Step(0).(operator.AddLearner).ToStore) + re.NotNil(op) + re.Equal("add-rule-peer", op.Desc()) + re.Equal(constant.High, op.GetPriorityLevel()) + re.Equal(uint64(3), op.Step(0).(operator.AddLearner).ToStore) } func (suite *ruleCheckerTestSuite) TestAddRulePeerWithIsolationLevel() { + re := suite.Require() suite.cluster.AddLabelsStore(1, 1, map[string]string{"zone": "z1", "rack": "r1", "host": "h1"}) suite.cluster.AddLabelsStore(2, 1, map[string]string{"zone": "z1", "rack": "r1", "host": "h2"}) suite.cluster.AddLabelsStore(3, 1, map[string]string{"zone": "z1", "rack": "r2", "host": "h1"}) @@ -98,7 +100,7 @@ func (suite *ruleCheckerTestSuite) TestAddRulePeerWithIsolationLevel() { IsolationLevel: "zone", }) op := suite.rc.Check(suite.cluster.GetRegion(1)) - suite.Nil(op) + re.Nil(op) suite.cluster.AddLeaderRegionWithRange(1, "", "", 1, 3) suite.ruleManager.SetRule(&placement.Rule{ GroupID: placement.DefaultGroupID, @@ -111,12 +113,13 @@ func (suite *ruleCheckerTestSuite) TestAddRulePeerWithIsolationLevel() { IsolationLevel: "rack", }) op = suite.rc.Check(suite.cluster.GetRegion(1)) - suite.NotNil(op) - suite.Equal("add-rule-peer", op.Desc()) - suite.Equal(uint64(4), op.Step(0).(operator.AddLearner).ToStore) + re.NotNil(op) + re.Equal("add-rule-peer", op.Desc()) + re.Equal(uint64(4), op.Step(0).(operator.AddLearner).ToStore) } func (suite *ruleCheckerTestSuite) TestReplaceDownPeerWithIsolationLevel() { + re := suite.Require() suite.cluster.SetMaxStoreDownTime(100 * time.Millisecond) suite.cluster.AddLabelsStore(1, 1, map[string]string{"zone": "z1", "host": "h1"}) suite.cluster.AddLabelsStore(2, 1, map[string]string{"zone": "z1", "host": "h2"}) @@ -137,7 +140,7 @@ func (suite *ruleCheckerTestSuite) TestReplaceDownPeerWithIsolationLevel() { IsolationLevel: "zone", }) op := suite.rc.Check(suite.cluster.GetRegion(1)) - suite.Nil(op) + re.Nil(op) region := suite.cluster.GetRegion(1) downPeer := []*pdpb.PeerStats{ {Peer: region.GetStorePeer(5), DownSeconds: 6000}, @@ -148,33 +151,34 @@ func (suite *ruleCheckerTestSuite) TestReplaceDownPeerWithIsolationLevel() { suite.cluster.SetStoreDown(6) time.Sleep(200 * time.Millisecond) op = suite.rc.Check(suite.cluster.GetRegion(1)) - suite.Nil(op) + re.Nil(op) } func (suite *ruleCheckerTestSuite) TestFixPeer() { + re := suite.Require() suite.cluster.AddLeaderStore(1, 1) suite.cluster.AddLeaderStore(2, 1) suite.cluster.AddLeaderStore(3, 1) suite.cluster.AddLeaderStore(4, 1) suite.cluster.AddLeaderRegionWithRange(1, "", "", 1, 2, 3) op := suite.rc.Check(suite.cluster.GetRegion(1)) - suite.Nil(op) + re.Nil(op) suite.cluster.SetStoreDown(2) r := suite.cluster.GetRegion(1) r = r.Clone(core.WithDownPeers([]*pdpb.PeerStats{{Peer: r.GetStorePeer(2), DownSeconds: 60000}})) op = suite.rc.Check(r) - suite.NotNil(op) - suite.Equal("fast-replace-rule-down-peer", op.Desc()) - suite.Equal(constant.Urgent, op.GetPriorityLevel()) + re.NotNil(op) + re.Equal("fast-replace-rule-down-peer", op.Desc()) + re.Equal(constant.Urgent, op.GetPriorityLevel()) var add operator.AddLearner - suite.IsType(add, op.Step(0)) + re.IsType(add, op.Step(0)) suite.cluster.SetStoreUp(2) suite.cluster.SetStoreOffline(2) op = suite.rc.Check(suite.cluster.GetRegion(1)) - suite.NotNil(op) - suite.Equal("replace-rule-offline-peer", op.Desc()) - suite.Equal(constant.High, op.GetPriorityLevel()) - suite.IsType(add, op.Step(0)) + re.NotNil(op) + re.Equal("replace-rule-offline-peer", op.Desc()) + re.Equal(constant.High, op.GetPriorityLevel()) + re.IsType(add, op.Step(0)) suite.cluster.SetStoreUp(2) // leader store offline @@ -185,29 +189,31 @@ func (suite *ruleCheckerTestSuite) TestFixPeer() { hasTransferLeader := false for i := 0; i < 100; i++ { op = suite.rc.Check(suite.cluster.GetRegion(1)) - suite.NotNil(op) + re.NotNil(op) if step, ok := op.Step(0).(operator.TransferLeader); ok { - suite.Equal(uint64(1), step.FromStore) - suite.NotEqual(uint64(3), step.ToStore) + re.Equal(uint64(1), step.FromStore) + re.NotEqual(uint64(3), step.ToStore) hasTransferLeader = true } } - suite.True(hasTransferLeader) + re.True(hasTransferLeader) } func (suite *ruleCheckerTestSuite) TestFixOrphanPeers() { + re := suite.Require() suite.cluster.AddLeaderStore(1, 1) suite.cluster.AddLeaderStore(2, 1) suite.cluster.AddLeaderStore(3, 1) suite.cluster.AddLeaderStore(4, 1) suite.cluster.AddLeaderRegionWithRange(1, "", "", 1, 2, 3, 4) op := suite.rc.Check(suite.cluster.GetRegion(1)) - suite.NotNil(op) - suite.Equal("remove-orphan-peer", op.Desc()) - suite.Equal(uint64(4), op.Step(0).(operator.RemovePeer).FromStore) + re.NotNil(op) + re.Equal("remove-orphan-peer", op.Desc()) + re.Equal(uint64(4), op.Step(0).(operator.RemovePeer).FromStore) } func (suite *ruleCheckerTestSuite) TestFixToManyOrphanPeers() { + re := suite.Require() suite.cluster.AddLeaderStore(1, 1) suite.cluster.AddLeaderStore(2, 1) suite.cluster.AddLeaderStore(3, 1) @@ -223,9 +229,9 @@ func (suite *ruleCheckerTestSuite) TestFixToManyOrphanPeers() { core.WithPendingPeers([]*metapb.Peer{region.GetStorePeer(3)})) suite.cluster.PutRegion(region) op := suite.rc.Check(suite.cluster.GetRegion(1)) - suite.NotNil(op) - suite.Equal("remove-orphan-peer", op.Desc()) - suite.Equal(uint64(5), op.Step(0).(operator.RemovePeer).FromStore) + re.NotNil(op) + re.Equal("remove-orphan-peer", op.Desc()) + re.Equal(uint64(5), op.Step(0).(operator.RemovePeer).FromStore) // Case2: // store 4, 5, 6 are orphan peers, and peer on store 3 is down peer. and peer on store 4, 5 are pending. region = suite.cluster.GetRegion(1) @@ -234,9 +240,9 @@ func (suite *ruleCheckerTestSuite) TestFixToManyOrphanPeers() { core.WithPendingPeers([]*metapb.Peer{region.GetStorePeer(4), region.GetStorePeer(5)})) suite.cluster.PutRegion(region) op = suite.rc.Check(suite.cluster.GetRegion(1)) - suite.NotNil(op) - suite.Equal("remove-unhealthy-orphan-peer", op.Desc()) - suite.Equal(uint64(4), op.Step(0).(operator.RemovePeer).FromStore) + re.NotNil(op) + re.Equal("remove-unhealthy-orphan-peer", op.Desc()) + re.Equal(uint64(4), op.Step(0).(operator.RemovePeer).FromStore) // Case3: // store 4, 5, 6 are orphan peers, and peer on one of stores is disconnect peer // we should remove disconnect peer first. @@ -248,9 +254,9 @@ func (suite *ruleCheckerTestSuite) TestFixToManyOrphanPeers() { core.WithPendingPeers([]*metapb.Peer{region.GetStorePeer(3)})) suite.cluster.PutRegion(region) op = suite.rc.Check(suite.cluster.GetRegion(1)) - suite.NotNil(op) - suite.Equal("remove-orphan-peer", op.Desc()) - suite.Equal(i, op.Step(0).(operator.RemovePeer).FromStore) + re.NotNil(op) + re.Equal("remove-orphan-peer", op.Desc()) + re.Equal(i, op.Step(0).(operator.RemovePeer).FromStore) suite.cluster.SetStoreUp(i) } // Case4: @@ -267,23 +273,24 @@ func (suite *ruleCheckerTestSuite) TestFixToManyOrphanPeers() { core.WithPendingPeers([]*metapb.Peer{region.GetStorePeer(3)})) suite.cluster.PutRegion(region) op = suite.rc.Check(suite.cluster.GetRegion(1)) - suite.NotNil(op) - suite.Equal("remove-orphan-peer", op.Desc()) + re.NotNil(op) + re.Equal("remove-orphan-peer", op.Desc()) removedPeerStoreID := op.Step(0).(operator.RemovePeer).FromStore - suite.NotEqual(i, removedPeerStoreID) + re.NotEqual(i, removedPeerStoreID) region = suite.cluster.GetRegion(1) newRegion := region.Clone(core.WithRemoveStorePeer(removedPeerStoreID)) suite.cluster.PutRegion(newRegion) op = suite.rc.Check(suite.cluster.GetRegion(1)) - suite.NotNil(op) - suite.Equal("remove-orphan-peer", op.Desc()) + re.NotNil(op) + re.Equal("remove-orphan-peer", op.Desc()) removedPeerStoreID = op.Step(0).(operator.RemovePeer).FromStore - suite.NotEqual(i, removedPeerStoreID) + re.NotEqual(i, removedPeerStoreID) suite.cluster.PutRegion(region) } } func (suite *ruleCheckerTestSuite) TestFixToManyOrphanPeers2() { + re := suite.Require() suite.cluster.AddLeaderStore(1, 1) suite.cluster.AddLeaderStore(2, 1) suite.cluster.AddLeaderStore(3, 1) @@ -302,9 +309,9 @@ func (suite *ruleCheckerTestSuite) TestFixToManyOrphanPeers2() { core.WithPendingPeers([]*metapb.Peer{region.GetStorePeer(3)})) suite.cluster.PutRegion(region) op := suite.rc.Check(suite.cluster.GetRegion(1)) - suite.NotNil(op) - suite.Equal("remove-orphan-peer", op.Desc()) - suite.Equal(i, op.Step(0).(operator.RemovePeer).FromStore) + re.NotNil(op) + re.Equal("remove-orphan-peer", op.Desc()) + re.Equal(i, op.Step(0).(operator.RemovePeer).FromStore) suite.cluster.SetStoreUp(i) } @@ -319,12 +326,13 @@ func (suite *ruleCheckerTestSuite) TestFixToManyOrphanPeers2() { core.WithPendingPeers([]*metapb.Peer{region.GetStorePeer(3)})) suite.cluster.PutRegion(region) op := suite.rc.Check(suite.cluster.GetRegion(1)) - suite.NotNil(op) - suite.Equal("remove-orphan-peer", op.Desc()) - suite.Equal(uint64(4), op.Step(0).(operator.RemovePeer).FromStore) + re.NotNil(op) + re.Equal("remove-orphan-peer", op.Desc()) + re.Equal(uint64(4), op.Step(0).(operator.RemovePeer).FromStore) } func (suite *ruleCheckerTestSuite) TestFixOrphanPeers2() { + re := suite.Require() // check orphan peers can only be handled when all rules are satisfied. suite.cluster.AddLabelsStore(1, 1, map[string]string{"foo": "bar"}) suite.cluster.AddLabelsStore(2, 1, map[string]string{"foo": "bar"}) @@ -343,10 +351,11 @@ func (suite *ruleCheckerTestSuite) TestFixOrphanPeers2() { }) suite.cluster.SetStoreDown(2) op := suite.rc.Check(suite.cluster.GetRegion(1)) - suite.Nil(op) + re.Nil(op) } func (suite *ruleCheckerTestSuite) TestFixRole() { + re := suite.Require() suite.cluster.AddLeaderStore(1, 1) suite.cluster.AddLeaderStore(2, 1) suite.cluster.AddLeaderStore(3, 1) @@ -356,12 +365,13 @@ func (suite *ruleCheckerTestSuite) TestFixRole() { p.Role = metapb.PeerRole_Learner r = r.Clone(core.WithLearners([]*metapb.Peer{p})) op := suite.rc.Check(r) - suite.NotNil(op) - suite.Equal("fix-peer-role", op.Desc()) - suite.Equal(uint64(1), op.Step(0).(operator.PromoteLearner).ToStore) + re.NotNil(op) + re.Equal("fix-peer-role", op.Desc()) + re.Equal(uint64(1), op.Step(0).(operator.PromoteLearner).ToStore) } func (suite *ruleCheckerTestSuite) TestFixRoleLeader() { + re := suite.Require() suite.cluster.AddLabelsStore(1, 1, map[string]string{"role": "follower"}) suite.cluster.AddLabelsStore(2, 1, map[string]string{"role": "follower"}) suite.cluster.AddLabelsStore(3, 1, map[string]string{"role": "voter"}) @@ -388,12 +398,13 @@ func (suite *ruleCheckerTestSuite) TestFixRoleLeader() { }, }) op := suite.rc.Check(suite.cluster.GetRegion(1)) - suite.NotNil(op) - suite.Equal("fix-follower-role", op.Desc()) - suite.Equal(uint64(3), op.Step(0).(operator.TransferLeader).ToStore) + re.NotNil(op) + re.Equal("fix-follower-role", op.Desc()) + re.Equal(uint64(3), op.Step(0).(operator.TransferLeader).ToStore) } func (suite *ruleCheckerTestSuite) TestFixRoleLeaderIssue3130() { + re := suite.Require() suite.cluster.AddLabelsStore(1, 1, map[string]string{"role": "follower"}) suite.cluster.AddLabelsStore(2, 1, map[string]string{"role": "leader"}) suite.cluster.AddLeaderRegion(1, 1, 2) @@ -409,20 +420,20 @@ func (suite *ruleCheckerTestSuite) TestFixRoleLeaderIssue3130() { }, }) op := suite.rc.Check(suite.cluster.GetRegion(1)) - suite.NotNil(op) - suite.Equal("fix-leader-role", op.Desc()) - suite.Equal(uint64(2), op.Step(0).(operator.TransferLeader).ToStore) + re.NotNil(op) + re.Equal("fix-leader-role", op.Desc()) + re.Equal(uint64(2), op.Step(0).(operator.TransferLeader).ToStore) suite.cluster.SetStoreBusy(2, true) op = suite.rc.Check(suite.cluster.GetRegion(1)) - suite.Nil(op) + re.Nil(op) suite.cluster.SetStoreBusy(2, false) suite.cluster.AddLeaderRegion(1, 2, 1) op = suite.rc.Check(suite.cluster.GetRegion(1)) - suite.NotNil(op) - suite.Equal("remove-orphan-peer", op.Desc()) - suite.Equal(uint64(1), op.Step(0).(operator.RemovePeer).FromStore) + re.NotNil(op) + re.Equal("remove-orphan-peer", op.Desc()) + re.Equal(uint64(1), op.Step(0).(operator.RemovePeer).FromStore) } func (suite *ruleCheckerTestSuite) TestFixLeaderRoleWithUnhealthyRegion() { @@ -466,6 +477,7 @@ func (suite *ruleCheckerTestSuite) TestFixLeaderRoleWithUnhealthyRegion() { } func (suite *ruleCheckerTestSuite) TestFixRuleWitness() { + re := suite.Require() suite.cluster.AddLabelsStore(1, 1, map[string]string{"A": "leader"}) suite.cluster.AddLabelsStore(2, 1, map[string]string{"B": "follower"}) suite.cluster.AddLabelsStore(3, 1, map[string]string{"C": "voter"}) @@ -484,13 +496,14 @@ func (suite *ruleCheckerTestSuite) TestFixRuleWitness() { }, }) op := suite.rc.Check(suite.cluster.GetRegion(1)) - suite.NotNil(op) - suite.Equal("add-rule-peer", op.Desc()) - suite.Equal(uint64(3), op.Step(0).(operator.AddLearner).ToStore) - suite.True(op.Step(0).(operator.AddLearner).IsWitness) + re.NotNil(op) + re.Equal("add-rule-peer", op.Desc()) + re.Equal(uint64(3), op.Step(0).(operator.AddLearner).ToStore) + re.True(op.Step(0).(operator.AddLearner).IsWitness) } func (suite *ruleCheckerTestSuite) TestFixRuleWitness2() { + re := suite.Require() suite.cluster.AddLabelsStore(1, 1, map[string]string{"A": "leader"}) suite.cluster.AddLabelsStore(2, 1, map[string]string{"B": "voter"}) suite.cluster.AddLabelsStore(3, 1, map[string]string{"C": "voter"}) @@ -510,12 +523,13 @@ func (suite *ruleCheckerTestSuite) TestFixRuleWitness2() { }, }) op := suite.rc.Check(suite.cluster.GetRegion(1)) - suite.NotNil(op) - suite.Equal("fix-witness-peer", op.Desc()) - suite.Equal(uint64(4), op.Step(0).(operator.BecomeWitness).StoreID) + re.NotNil(op) + re.Equal("fix-witness-peer", op.Desc()) + re.Equal(uint64(4), op.Step(0).(operator.BecomeWitness).StoreID) } func (suite *ruleCheckerTestSuite) TestFixRuleWitness3() { + re := suite.Require() suite.cluster.AddLabelsStore(1, 1, map[string]string{"A": "leader"}) suite.cluster.AddLabelsStore(2, 1, map[string]string{"B": "voter"}) suite.cluster.AddLabelsStore(3, 1, map[string]string{"C": "voter"}) @@ -526,10 +540,10 @@ func (suite *ruleCheckerTestSuite) TestFixRuleWitness3() { r = r.Clone(core.WithWitnesses([]*metapb.Peer{r.GetPeer(3)})) suite.cluster.PutRegion(r) op := suite.rc.Check(r) - suite.NotNil(op) - suite.Equal("fix-non-witness-peer", op.Desc()) - suite.Equal(uint64(3), op.Step(0).(operator.RemovePeer).FromStore) - suite.Equal(uint64(3), op.Step(1).(operator.AddLearner).ToStore) + re.NotNil(op) + re.Equal("fix-non-witness-peer", op.Desc()) + re.Equal(uint64(3), op.Step(0).(operator.RemovePeer).FromStore) + re.Equal(uint64(3), op.Step(1).(operator.AddLearner).ToStore) } func (suite *ruleCheckerTestSuite) TestFixRuleWitness4() { @@ -576,6 +590,7 @@ func (suite *ruleCheckerTestSuite) TestFixRuleWitness4() { } func (suite *ruleCheckerTestSuite) TestFixRuleWitness5() { + re := suite.Require() suite.cluster.AddLabelsStore(1, 1, map[string]string{"A": "leader"}) suite.cluster.AddLabelsStore(2, 1, map[string]string{"B": "voter"}) suite.cluster.AddLabelsStore(3, 1, map[string]string{"C": "voter"}) @@ -593,11 +608,12 @@ func (suite *ruleCheckerTestSuite) TestFixRuleWitness5() { {Key: "A", Op: "In", Values: []string{"leader"}}, }, }) - suite.Error(err) - suite.Equal(errs.ErrRuleContent.FastGenByArgs(fmt.Sprintf("define too many witness by count %d", 2)).Error(), err.Error()) + re.Error(err) + re.Equal(errs.ErrRuleContent.FastGenByArgs(fmt.Sprintf("define too many witness by count %d", 2)).Error(), err.Error()) } func (suite *ruleCheckerTestSuite) TestFixRuleWitness6() { + re := suite.Require() suite.cluster.AddLabelsStore(1, 1, map[string]string{"A": "leader"}) suite.cluster.AddLabelsStore(2, 1, map[string]string{"B": "voter"}) suite.cluster.AddLabelsStore(3, 1, map[string]string{"C": "voter"}) @@ -624,18 +640,19 @@ func (suite *ruleCheckerTestSuite) TestFixRuleWitness6() { }, }, }) - suite.NoError(err) + re.NoError(err) suite.rc.RecordRegionPromoteToNonWitness(1) op := suite.rc.Check(suite.cluster.GetRegion(1)) - suite.Nil(op) + re.Nil(op) suite.rc.switchWitnessCache.Remove(1) op = suite.rc.Check(suite.cluster.GetRegion(1)) - suite.NotNil(op) + re.NotNil(op) } func (suite *ruleCheckerTestSuite) TestDisableWitness() { + re := suite.Require() suite.cluster.AddLabelsStore(1, 1, map[string]string{"A": "leader"}) suite.cluster.AddLabelsStore(2, 1, map[string]string{"B": "voter"}) suite.cluster.AddLabelsStore(3, 1, map[string]string{"C": "voter"}) @@ -662,20 +679,21 @@ func (suite *ruleCheckerTestSuite) TestDisableWitness() { }, }, }) - suite.NoError(err) + re.NoError(err) r := suite.cluster.GetRegion(1) r = r.Clone(core.WithWitnesses([]*metapb.Peer{r.GetPeer(3)})) op := suite.rc.Check(r) - suite.Nil(op) + re.Nil(op) suite.cluster.SetEnableWitness(false) op = suite.rc.Check(r) - suite.NotNil(op) + re.NotNil(op) } func (suite *ruleCheckerTestSuite) TestBetterReplacement() { + re := suite.Require() suite.cluster.AddLabelsStore(1, 1, map[string]string{"host": "host1"}) suite.cluster.AddLabelsStore(2, 1, map[string]string{"host": "host1"}) suite.cluster.AddLabelsStore(3, 1, map[string]string{"host": "host2"}) @@ -691,15 +709,16 @@ func (suite *ruleCheckerTestSuite) TestBetterReplacement() { LocationLabels: []string{"host"}, }) op := suite.rc.Check(suite.cluster.GetRegion(1)) - suite.NotNil(op) - suite.Equal("move-to-better-location", op.Desc()) - suite.Equal(uint64(4), op.Step(0).(operator.AddLearner).ToStore) + re.NotNil(op) + re.Equal("move-to-better-location", op.Desc()) + re.Equal(uint64(4), op.Step(0).(operator.AddLearner).ToStore) suite.cluster.AddLeaderRegionWithRange(1, "", "", 1, 3, 4) op = suite.rc.Check(suite.cluster.GetRegion(1)) - suite.Nil(op) + re.Nil(op) } func (suite *ruleCheckerTestSuite) TestBetterReplacement2() { + re := suite.Require() suite.cluster.AddLabelsStore(1, 1, map[string]string{"zone": "z1", "host": "host1"}) suite.cluster.AddLabelsStore(2, 1, map[string]string{"zone": "z1", "host": "host2"}) suite.cluster.AddLabelsStore(3, 1, map[string]string{"zone": "z1", "host": "host3"}) @@ -715,15 +734,16 @@ func (suite *ruleCheckerTestSuite) TestBetterReplacement2() { LocationLabels: []string{"zone", "host"}, }) op := suite.rc.Check(suite.cluster.GetRegion(1)) - suite.NotNil(op) - suite.Equal("move-to-better-location", op.Desc()) - suite.Equal(uint64(4), op.Step(0).(operator.AddLearner).ToStore) + re.NotNil(op) + re.Equal("move-to-better-location", op.Desc()) + re.Equal(uint64(4), op.Step(0).(operator.AddLearner).ToStore) suite.cluster.AddLeaderRegionWithRange(1, "", "", 1, 3, 4) op = suite.rc.Check(suite.cluster.GetRegion(1)) - suite.Nil(op) + re.Nil(op) } func (suite *ruleCheckerTestSuite) TestNoBetterReplacement() { + re := suite.Require() suite.cluster.AddLabelsStore(1, 1, map[string]string{"host": "host1"}) suite.cluster.AddLabelsStore(2, 1, map[string]string{"host": "host1"}) suite.cluster.AddLabelsStore(3, 1, map[string]string{"host": "host2"}) @@ -738,10 +758,11 @@ func (suite *ruleCheckerTestSuite) TestNoBetterReplacement() { LocationLabels: []string{"host"}, }) op := suite.rc.Check(suite.cluster.GetRegion(1)) - suite.Nil(op) + re.Nil(op) } func (suite *ruleCheckerTestSuite) TestIssue2419() { + re := suite.Require() suite.cluster.AddLeaderStore(1, 1) suite.cluster.AddLeaderStore(2, 1) suite.cluster.AddLeaderStore(3, 1) @@ -751,23 +772,24 @@ func (suite *ruleCheckerTestSuite) TestIssue2419() { r := suite.cluster.GetRegion(1) r = r.Clone(core.WithAddPeer(&metapb.Peer{Id: 5, StoreId: 4, Role: metapb.PeerRole_Learner})) op := suite.rc.Check(r) - suite.NotNil(op) - suite.Equal("remove-orphan-peer", op.Desc()) - suite.Equal(uint64(4), op.Step(0).(operator.RemovePeer).FromStore) + re.NotNil(op) + re.Equal("remove-orphan-peer", op.Desc()) + re.Equal(uint64(4), op.Step(0).(operator.RemovePeer).FromStore) r = r.Clone(core.WithRemoveStorePeer(4)) op = suite.rc.Check(r) - suite.NotNil(op) - suite.Equal("replace-rule-offline-peer", op.Desc()) - suite.Equal(uint64(4), op.Step(0).(operator.AddLearner).ToStore) - suite.Equal(uint64(4), op.Step(1).(operator.PromoteLearner).ToStore) - suite.Equal(uint64(3), op.Step(2).(operator.RemovePeer).FromStore) + re.NotNil(op) + re.Equal("replace-rule-offline-peer", op.Desc()) + re.Equal(uint64(4), op.Step(0).(operator.AddLearner).ToStore) + re.Equal(uint64(4), op.Step(1).(operator.PromoteLearner).ToStore) + re.Equal(uint64(3), op.Step(2).(operator.RemovePeer).FromStore) } // Ref https://github.com/tikv/pd/issues/3521 https://github.com/tikv/pd/issues/5786 // The problem is when offline a store, we may add learner multiple times if // the operator is timeout. func (suite *ruleCheckerTestSuite) TestPriorityFixOrphanPeer() { + re := suite.Require() suite.cluster.AddLabelsStore(1, 1, map[string]string{"host": "host1"}) suite.cluster.AddLabelsStore(2, 1, map[string]string{"host": "host1"}) suite.cluster.AddLabelsStore(3, 1, map[string]string{"host": "host2"}) @@ -775,7 +797,7 @@ func (suite *ruleCheckerTestSuite) TestPriorityFixOrphanPeer() { suite.cluster.AddLabelsStore(5, 1, map[string]string{"host": "host5"}) suite.cluster.AddLeaderRegionWithRange(1, "", "", 1, 2, 3) op := suite.rc.Check(suite.cluster.GetRegion(1)) - suite.Nil(op) + re.Nil(op) var add operator.AddLearner var remove operator.RemovePeer // Ref 5786 @@ -788,16 +810,16 @@ func (suite *ruleCheckerTestSuite) TestPriorityFixOrphanPeer() { ) suite.cluster.PutRegion(testRegion) op = suite.rc.Check(suite.cluster.GetRegion(1)) - suite.NotNil(op) - suite.Equal("remove-unhealthy-orphan-peer", op.Desc()) - suite.IsType(remove, op.Step(0)) + re.NotNil(op) + re.Equal("remove-unhealthy-orphan-peer", op.Desc()) + re.IsType(remove, op.Step(0)) // Ref #3521 suite.cluster.SetStoreOffline(2) suite.cluster.PutRegion(originRegion) op = suite.rc.Check(suite.cluster.GetRegion(1)) - suite.NotNil(op) - suite.IsType(add, op.Step(0)) - suite.Equal("replace-rule-offline-peer", op.Desc()) + re.NotNil(op) + re.IsType(add, op.Step(0)) + re.Equal("replace-rule-offline-peer", op.Desc()) testRegion = suite.cluster.GetRegion(1).Clone(core.WithAddPeer( &metapb.Peer{ Id: 125, @@ -806,12 +828,13 @@ func (suite *ruleCheckerTestSuite) TestPriorityFixOrphanPeer() { })) suite.cluster.PutRegion(testRegion) op = suite.rc.Check(suite.cluster.GetRegion(1)) - suite.IsType(remove, op.Step(0)) - suite.Equal("remove-orphan-peer", op.Desc()) + re.IsType(remove, op.Step(0)) + re.Equal("remove-orphan-peer", op.Desc()) } // Ref https://github.com/tikv/pd/issues/7249 https://github.com/tikv/tikv/issues/15799 func (suite *ruleCheckerTestSuite) TestFixOrphanPeerWithDisconnectedStoreAndRuleChanged() { + re := suite.Require() // disconnect any two stores and change rule to 3 replicas stores := []uint64{1, 2, 3, 4, 5} testCases := [][]uint64{} @@ -845,9 +868,9 @@ func (suite *ruleCheckerTestSuite) TestFixOrphanPeerWithDisconnectedStoreAndRule EndKey: []byte{}, } err := suite.ruleManager.SetRule(rule) - suite.NoError(err) + re.NoError(err) op := suite.rc.Check(suite.cluster.GetRegion(1)) - suite.Nil(op) + re.Nil(op) // set two stores to disconnected suite.cluster.SetStoreDisconnect(testCase[0]) @@ -869,8 +892,8 @@ func (suite *ruleCheckerTestSuite) TestFixOrphanPeerWithDisconnectedStoreAndRule for j := 1; j <= 2; j++ { r1 := suite.cluster.GetRegion(1) op = suite.rc.Check(suite.cluster.GetRegion(1)) - suite.NotNil(op) - suite.Contains(op.Desc(), "orphan") + re.NotNil(op) + re.Contains(op.Desc(), "orphan") var removedPeerStoreID uint64 newLeaderStoreID := r1.GetLeader().GetStoreId() for i := 0; i < op.Len(); i++ { @@ -881,19 +904,19 @@ func (suite *ruleCheckerTestSuite) TestFixOrphanPeerWithDisconnectedStoreAndRule newLeaderStoreID = s.ToStore } } - suite.NotZero(removedPeerStoreID) + re.NotZero(removedPeerStoreID) r1 = r1.Clone( core.WithLeader(r1.GetStorePeer(newLeaderStoreID)), core.WithRemoveStorePeer(removedPeerStoreID)) suite.cluster.PutRegion(r1) r1 = suite.cluster.GetRegion(1) - suite.Len(r1.GetPeers(), 5-j) + re.Len(r1.GetPeers(), 5-j) } r1 := suite.cluster.GetRegion(1) for _, p := range r1.GetPeers() { - suite.NotEqual(p.GetStoreId(), testCase[0]) - suite.NotEqual(p.GetStoreId(), testCase[1]) + re.NotEqual(p.GetStoreId(), testCase[0]) + re.NotEqual(p.GetStoreId(), testCase[1]) } suite.TearDownTest() suite.SetupTest() @@ -903,6 +926,7 @@ func (suite *ruleCheckerTestSuite) TestFixOrphanPeerWithDisconnectedStoreAndRule // Ref https://github.com/tikv/pd/issues/7249 https://github.com/tikv/tikv/issues/15799 func (suite *ruleCheckerTestSuite) TestFixOrphanPeerWithDisconnectedStoreAndRuleChangedWithLearner() { + re := suite.Require() // disconnect any three stores and change rule to 3 replicas // and there is a learner in the disconnected store. stores := []uint64{1, 2, 3, 4, 5, 6} @@ -964,12 +988,12 @@ func (suite *ruleCheckerTestSuite) TestFixOrphanPeerWithDisconnectedStoreAndRule }, }, }) - suite.NoError(err) + re.NoError(err) r1 := suite.cluster.GetRegion(1) r1 = r1.Clone(core.WithAddPeer(&metapb.Peer{Id: 12, StoreId: learnerStore, Role: metapb.PeerRole_Learner})) suite.cluster.PutRegion(r1) op := suite.rc.Check(suite.cluster.GetRegion(1)) - suite.Nil(op) + re.Nil(op) // set three stores to disconnected suite.cluster.SetStoreDisconnect(testCase[0]) @@ -992,8 +1016,8 @@ func (suite *ruleCheckerTestSuite) TestFixOrphanPeerWithDisconnectedStoreAndRule for j := 1; j <= 3; j++ { r1 := suite.cluster.GetRegion(1) op = suite.rc.Check(suite.cluster.GetRegion(1)) - suite.NotNil(op) - suite.Contains(op.Desc(), "orphan") + re.NotNil(op) + re.Contains(op.Desc(), "orphan") var removedPeerStroeID uint64 newLeaderStoreID := r1.GetLeader().GetStoreId() for i := 0; i < op.Len(); i++ { @@ -1004,20 +1028,20 @@ func (suite *ruleCheckerTestSuite) TestFixOrphanPeerWithDisconnectedStoreAndRule newLeaderStoreID = s.ToStore } } - suite.NotZero(removedPeerStroeID) + re.NotZero(removedPeerStroeID) r1 = r1.Clone( core.WithLeader(r1.GetStorePeer(newLeaderStoreID)), core.WithRemoveStorePeer(removedPeerStroeID)) suite.cluster.PutRegion(r1) r1 = suite.cluster.GetRegion(1) - suite.Len(r1.GetPeers(), 6-j) + re.Len(r1.GetPeers(), 6-j) } r1 = suite.cluster.GetRegion(1) for _, p := range r1.GetPeers() { - suite.NotEqual(p.GetStoreId(), testCase[0]) - suite.NotEqual(p.GetStoreId(), testCase[1]) - suite.NotEqual(p.GetStoreId(), testCase[2]) + re.NotEqual(p.GetStoreId(), testCase[0]) + re.NotEqual(p.GetStoreId(), testCase[1]) + re.NotEqual(p.GetStoreId(), testCase[2]) } suite.TearDownTest() suite.SetupTest() @@ -1027,6 +1051,7 @@ func (suite *ruleCheckerTestSuite) TestFixOrphanPeerWithDisconnectedStoreAndRule } func (suite *ruleCheckerTestSuite) TestPriorityFitHealthWithDifferentRole1() { + re := suite.Require() suite.cluster.SetEnableUseJointConsensus(true) suite.cluster.AddLabelsStore(1, 1, map[string]string{"host": "host1"}) suite.cluster.AddLabelsStore(2, 1, map[string]string{"host": "host2"}) @@ -1047,21 +1072,22 @@ func (suite *ruleCheckerTestSuite) TestPriorityFitHealthWithDifferentRole1() { suite.cluster.PutRegion(r1) op := suite.rc.Check(suite.cluster.GetRegion(1)) - suite.Equal(uint64(3), op.Step(0).(operator.ChangePeerV2Enter).DemoteVoters[0].ToStore) - suite.Equal(uint64(4), op.Step(0).(operator.ChangePeerV2Enter).PromoteLearners[0].ToStore) - suite.Equal(uint64(3), op.Step(1).(operator.ChangePeerV2Leave).DemoteVoters[0].ToStore) - suite.Equal(uint64(4), op.Step(1).(operator.ChangePeerV2Leave).PromoteLearners[0].ToStore) - suite.Equal("replace-down-peer-with-orphan-peer", op.Desc()) + re.Equal(uint64(3), op.Step(0).(operator.ChangePeerV2Enter).DemoteVoters[0].ToStore) + re.Equal(uint64(4), op.Step(0).(operator.ChangePeerV2Enter).PromoteLearners[0].ToStore) + re.Equal(uint64(3), op.Step(1).(operator.ChangePeerV2Leave).DemoteVoters[0].ToStore) + re.Equal(uint64(4), op.Step(1).(operator.ChangePeerV2Leave).PromoteLearners[0].ToStore) + re.Equal("replace-down-peer-with-orphan-peer", op.Desc()) // set peer3 only pending suite.cluster.GetStore(3).GetMeta().LastHeartbeat = time.Now().UnixNano() r1 = r1.Clone(core.WithDownPeers(nil)) suite.cluster.PutRegion(r1) op = suite.rc.Check(suite.cluster.GetRegion(1)) - suite.Nil(op) + re.Nil(op) } func (suite *ruleCheckerTestSuite) TestPriorityFitHealthWithDifferentRole2() { + re := suite.Require() suite.cluster.SetEnableUseJointConsensus(true) suite.cluster.AddLabelsStore(1, 1, map[string]string{"host": "host1"}) suite.cluster.AddLabelsStore(2, 1, map[string]string{"host": "host2"}) @@ -1092,15 +1118,16 @@ func (suite *ruleCheckerTestSuite) TestPriorityFitHealthWithDifferentRole2() { Role: placement.Learner, Count: 1, }) - suite.NoError(err) + re.NoError(err) op := suite.rc.Check(suite.cluster.GetRegion(1)) - suite.Equal(uint64(5), op.Step(0).(operator.ChangePeerV2Enter).DemoteVoters[0].ToStore) - suite.Equal(uint64(3), op.Step(1).(operator.RemovePeer).FromStore) - suite.Equal("replace-down-peer-with-orphan-peer", op.Desc()) + re.Equal(uint64(5), op.Step(0).(operator.ChangePeerV2Enter).DemoteVoters[0].ToStore) + re.Equal(uint64(3), op.Step(1).(operator.RemovePeer).FromStore) + re.Equal("replace-down-peer-with-orphan-peer", op.Desc()) } func (suite *ruleCheckerTestSuite) TestPriorityFitHealthPeersAndTiFlash() { + re := suite.Require() suite.cluster.SetEnableUseJointConsensus(true) suite.cluster.AddLabelsStore(1, 1, map[string]string{"host": "host1"}) suite.cluster.AddLabelsStore(2, 1, map[string]string{"host": "host2"}) @@ -1144,16 +1171,17 @@ func (suite *ruleCheckerTestSuite) TestPriorityFitHealthPeersAndTiFlash() { op := suite.rc.Check(suite.cluster.GetRegion(1)) // should not promote tiflash peer - suite.Nil(op) + re.Nil(op) // scale a node, can replace the down peer suite.cluster.AddLabelsStore(5, 1, map[string]string{"host": "host5"}) op = suite.rc.Check(suite.cluster.GetRegion(1)) - suite.NotNil(op) - suite.Equal("fast-replace-rule-down-peer", op.Desc()) + re.NotNil(op) + re.Equal("fast-replace-rule-down-peer", op.Desc()) } func (suite *ruleCheckerTestSuite) TestIssue3293() { + re := suite.Require() suite.cluster.AddLabelsStore(1, 1, map[string]string{"host": "host1"}) suite.cluster.AddLabelsStore(2, 1, map[string]string{"host": "host1"}) suite.cluster.AddLabelsStore(3, 1, map[string]string{"host": "host2"}) @@ -1175,7 +1203,7 @@ func (suite *ruleCheckerTestSuite) TestIssue3293() { }, }, }) - suite.NoError(err) + re.NoError(err) suite.cluster.DeleteStore(suite.cluster.GetStore(5)) err = suite.ruleManager.SetRule(&placement.Rule{ GroupID: "TiDB_DDL_51", @@ -1183,15 +1211,16 @@ func (suite *ruleCheckerTestSuite) TestIssue3293() { Role: placement.Voter, Count: 3, }) - suite.NoError(err) + re.NoError(err) err = suite.ruleManager.DeleteRule(placement.DefaultGroupID, placement.DefaultRuleID) - suite.NoError(err) + re.NoError(err) op := suite.rc.Check(suite.cluster.GetRegion(1)) - suite.NotNil(op) - suite.Equal("add-rule-peer", op.Desc()) + re.NotNil(op) + re.Equal("add-rule-peer", op.Desc()) } func (suite *ruleCheckerTestSuite) TestIssue3299() { + re := suite.Require() suite.cluster.AddLabelsStore(1, 1, map[string]string{"host": "host1"}) suite.cluster.AddLabelsStore(2, 1, map[string]string{"dc": "sh"}) suite.cluster.AddLeaderRegionWithRange(1, "", "", 1, 2) @@ -1278,13 +1307,14 @@ func (suite *ruleCheckerTestSuite) TestIssue3299() { if testCase.err != "" { suite.Regexp(testCase.err, err.Error()) } else { - suite.NoError(err) + re.NoError(err) } } } // See issue: https://github.com/tikv/pd/issues/3705 func (suite *ruleCheckerTestSuite) TestFixDownPeer() { + re := suite.Require() suite.cluster.AddLabelsStore(1, 1, map[string]string{"zone": "z1"}) suite.cluster.AddLabelsStore(2, 1, map[string]string{"zone": "z1"}) suite.cluster.AddLabelsStore(3, 1, map[string]string{"zone": "z2"}) @@ -1303,23 +1333,24 @@ func (suite *ruleCheckerTestSuite) TestFixDownPeer() { suite.ruleManager.SetRule(rule) region := suite.cluster.GetRegion(1) - suite.Nil(suite.rc.Check(region)) + re.Nil(suite.rc.Check(region)) suite.cluster.SetStoreDown(4) region = region.Clone(core.WithDownPeers([]*pdpb.PeerStats{ {Peer: region.GetStorePeer(4), DownSeconds: 6000}, })) - operatorutil.CheckTransferPeer(suite.Require(), suite.rc.Check(region), operator.OpRegion, 4, 5) + operatorutil.CheckTransferPeer(re, suite.rc.Check(region), operator.OpRegion, 4, 5) suite.cluster.SetStoreDown(5) - operatorutil.CheckTransferPeer(suite.Require(), suite.rc.Check(region), operator.OpRegion, 4, 2) + operatorutil.CheckTransferPeer(re, suite.rc.Check(region), operator.OpRegion, 4, 2) rule.IsolationLevel = "zone" suite.ruleManager.SetRule(rule) - suite.Nil(suite.rc.Check(region)) + re.Nil(suite.rc.Check(region)) } func (suite *ruleCheckerTestSuite) TestFixDownPeerWithNoWitness() { + re := suite.Require() suite.cluster.AddLabelsStore(1, 1, map[string]string{"zone": "z1"}) suite.cluster.AddLabelsStore(2, 1, map[string]string{"zone": "z2"}) suite.cluster.AddLabelsStore(3, 1, map[string]string{"zone": "z3"}) @@ -1330,10 +1361,11 @@ func (suite *ruleCheckerTestSuite) TestFixDownPeerWithNoWitness() { r := suite.cluster.GetRegion(1) // set peer2 to down r = r.Clone(core.WithDownPeers([]*pdpb.PeerStats{{Peer: r.GetStorePeer(2), DownSeconds: 600}})) - suite.Nil(suite.rc.Check(r)) + re.Nil(suite.rc.Check(r)) } func (suite *ruleCheckerTestSuite) TestFixDownWitnessPeer() { + re := suite.Require() suite.cluster.AddLabelsStore(1, 1, map[string]string{"zone": "z1"}) suite.cluster.AddLabelsStore(2, 1, map[string]string{"zone": "z2"}) suite.cluster.AddLabelsStore(3, 1, map[string]string{"zone": "z3"}) @@ -1360,13 +1392,14 @@ func (suite *ruleCheckerTestSuite) TestFixDownWitnessPeer() { Count: 1, IsWitness: true, }) - suite.Nil(suite.rc.Check(r)) + re.Nil(suite.rc.Check(r)) suite.cluster.GetStore(2).GetMeta().LastHeartbeat = time.Now().Add(-31 * time.Minute).UnixNano() - suite.Nil(suite.rc.Check(r)) + re.Nil(suite.rc.Check(r)) } func (suite *ruleCheckerTestSuite) TestFixDownPeerWithAvailableWitness() { + re := suite.Require() suite.cluster.AddLabelsStore(1, 1, map[string]string{"zone": "z1"}) suite.cluster.AddLabelsStore(2, 1, map[string]string{"zone": "z2"}) suite.cluster.AddLabelsStore(3, 1, map[string]string{"zone": "z3"}) @@ -1396,15 +1429,16 @@ func (suite *ruleCheckerTestSuite) TestFixDownPeerWithAvailableWitness() { op := suite.rc.Check(r) - suite.NotNil(op) - suite.Equal("promote-witness-for-down", op.Desc()) - suite.Equal(uint64(3), op.Step(0).(operator.RemovePeer).FromStore) - suite.Equal(uint64(3), op.Step(1).(operator.AddLearner).ToStore) - suite.Equal(uint64(3), op.Step(2).(operator.BecomeNonWitness).StoreID) - suite.Equal(uint64(3), op.Step(3).(operator.PromoteLearner).ToStore) + re.NotNil(op) + re.Equal("promote-witness-for-down", op.Desc()) + re.Equal(uint64(3), op.Step(0).(operator.RemovePeer).FromStore) + re.Equal(uint64(3), op.Step(1).(operator.AddLearner).ToStore) + re.Equal(uint64(3), op.Step(2).(operator.BecomeNonWitness).StoreID) + re.Equal(uint64(3), op.Step(3).(operator.PromoteLearner).ToStore) } func (suite *ruleCheckerTestSuite) TestFixDownPeerWithAvailableWitness2() { + re := suite.Require() suite.cluster.AddLabelsStore(1, 1, map[string]string{"zone": "z1"}) suite.cluster.AddLabelsStore(2, 1, map[string]string{"zone": "z2"}) suite.cluster.AddLabelsStore(3, 1, map[string]string{"zone": "z3"}) @@ -1434,10 +1468,11 @@ func (suite *ruleCheckerTestSuite) TestFixDownPeerWithAvailableWitness2() { op := suite.rc.Check(r) - suite.Nil(op) + re.Nil(op) } func (suite *ruleCheckerTestSuite) TestFixDownPeerWithAvailableWitness3() { + re := suite.Require() suite.cluster.AddLabelsStore(1, 1, map[string]string{"zone": "z1"}) suite.cluster.AddLabelsStore(2, 1, map[string]string{"zone": "z2"}) suite.cluster.AddLabelsStore(3, 1, map[string]string{"zone": "z3"}) @@ -1468,16 +1503,17 @@ func (suite *ruleCheckerTestSuite) TestFixDownPeerWithAvailableWitness3() { op := suite.rc.Check(r) - suite.NotNil(op) - suite.Equal("fast-replace-rule-down-peer", op.Desc()) - suite.Equal(uint64(4), op.Step(0).(operator.AddLearner).ToStore) - suite.True(op.Step(0).(operator.AddLearner).IsWitness) - suite.Equal(uint64(4), op.Step(1).(operator.PromoteLearner).ToStore) - suite.True(op.Step(1).(operator.PromoteLearner).IsWitness) - suite.Equal(uint64(2), op.Step(2).(operator.RemovePeer).FromStore) + re.NotNil(op) + re.Equal("fast-replace-rule-down-peer", op.Desc()) + re.Equal(uint64(4), op.Step(0).(operator.AddLearner).ToStore) + re.True(op.Step(0).(operator.AddLearner).IsWitness) + re.Equal(uint64(4), op.Step(1).(operator.PromoteLearner).ToStore) + re.True(op.Step(1).(operator.PromoteLearner).IsWitness) + re.Equal(uint64(2), op.Step(2).(operator.RemovePeer).FromStore) } func (suite *ruleCheckerTestSuite) TestFixDownPeerWithAvailableWitness4() { + re := suite.Require() suite.cluster.AddLabelsStore(1, 1, map[string]string{"zone": "z1"}) suite.cluster.AddLabelsStore(2, 1, map[string]string{"zone": "z2"}) suite.cluster.AddLabelsStore(3, 1, map[string]string{"zone": "z3"}) @@ -1492,17 +1528,18 @@ func (suite *ruleCheckerTestSuite) TestFixDownPeerWithAvailableWitness4() { op := suite.rc.Check(r) - suite.NotNil(op) - suite.Equal("fast-replace-rule-down-peer", op.Desc()) - suite.Equal(uint64(4), op.Step(0).(operator.AddLearner).ToStore) - suite.True(op.Step(0).(operator.AddLearner).IsWitness) - suite.Equal(uint64(4), op.Step(1).(operator.PromoteLearner).ToStore) - suite.True(op.Step(1).(operator.PromoteLearner).IsWitness) - suite.Equal(uint64(2), op.Step(2).(operator.RemovePeer).FromStore) + re.NotNil(op) + re.Equal("fast-replace-rule-down-peer", op.Desc()) + re.Equal(uint64(4), op.Step(0).(operator.AddLearner).ToStore) + re.True(op.Step(0).(operator.AddLearner).IsWitness) + re.Equal(uint64(4), op.Step(1).(operator.PromoteLearner).ToStore) + re.True(op.Step(1).(operator.PromoteLearner).IsWitness) + re.Equal(uint64(2), op.Step(2).(operator.RemovePeer).FromStore) } // See issue: https://github.com/tikv/pd/issues/3705 func (suite *ruleCheckerTestSuite) TestFixOfflinePeer() { + re := suite.Require() suite.cluster.AddLabelsStore(1, 1, map[string]string{"zone": "z1"}) suite.cluster.AddLabelsStore(2, 1, map[string]string{"zone": "z1"}) suite.cluster.AddLabelsStore(3, 1, map[string]string{"zone": "z2"}) @@ -1521,20 +1558,21 @@ func (suite *ruleCheckerTestSuite) TestFixOfflinePeer() { suite.ruleManager.SetRule(rule) region := suite.cluster.GetRegion(1) - suite.Nil(suite.rc.Check(region)) + re.Nil(suite.rc.Check(region)) suite.cluster.SetStoreOffline(4) - operatorutil.CheckTransferPeer(suite.Require(), suite.rc.Check(region), operator.OpRegion, 4, 5) + operatorutil.CheckTransferPeer(re, suite.rc.Check(region), operator.OpRegion, 4, 5) suite.cluster.SetStoreOffline(5) - operatorutil.CheckTransferPeer(suite.Require(), suite.rc.Check(region), operator.OpRegion, 4, 2) + operatorutil.CheckTransferPeer(re, suite.rc.Check(region), operator.OpRegion, 4, 2) rule.IsolationLevel = "zone" suite.ruleManager.SetRule(rule) - suite.Nil(suite.rc.Check(region)) + re.Nil(suite.rc.Check(region)) } func (suite *ruleCheckerTestSuite) TestFixOfflinePeerWithAvaliableWitness() { + re := suite.Require() suite.cluster.AddLabelsStore(1, 1, map[string]string{"zone": "z1"}) suite.cluster.AddLabelsStore(2, 1, map[string]string{"zone": "z1"}) suite.cluster.AddLabelsStore(3, 1, map[string]string{"zone": "z2"}) @@ -1557,15 +1595,16 @@ func (suite *ruleCheckerTestSuite) TestFixOfflinePeerWithAvaliableWitness() { Count: 1, IsWitness: true, }) - suite.Nil(suite.rc.Check(r)) + re.Nil(suite.rc.Check(r)) suite.cluster.SetStoreOffline(4) op := suite.rc.Check(r) - suite.NotNil(op) - suite.Equal("replace-rule-offline-peer", op.Desc()) + re.NotNil(op) + re.Equal("replace-rule-offline-peer", op.Desc()) } func (suite *ruleCheckerTestSuite) TestRuleCache() { + re := suite.Require() suite.cluster.PersistOptions.SetPlacementRulesCacheEnabled(true) suite.cluster.AddLabelsStore(1, 1, map[string]string{"zone": "z1"}) suite.cluster.AddLabelsStore(2, 1, map[string]string{"zone": "z1"}) @@ -1586,7 +1625,7 @@ func (suite *ruleCheckerTestSuite) TestRuleCache() { suite.ruleManager.SetRule(rule) region := suite.cluster.GetRegion(1) region = region.Clone(core.WithIncConfVer(), core.WithIncVersion()) - suite.Nil(suite.rc.Check(region)) + re.Nil(suite.rc.Check(region)) testCases := []struct { name string @@ -1629,19 +1668,20 @@ func (suite *ruleCheckerTestSuite) TestRuleCache() { for _, testCase := range testCases { suite.T().Log(testCase.name) if testCase.stillCached { - suite.NoError(failpoint.Enable("github.com/tikv/pd/pkg/schedule/checker/assertShouldCache", "return(true)")) + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/schedule/checker/assertShouldCache", "return(true)")) suite.rc.Check(testCase.region) - suite.NoError(failpoint.Disable("github.com/tikv/pd/pkg/schedule/checker/assertShouldCache")) + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/schedule/checker/assertShouldCache")) } else { - suite.NoError(failpoint.Enable("github.com/tikv/pd/pkg/schedule/checker/assertShouldNotCache", "return(true)")) + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/schedule/checker/assertShouldNotCache", "return(true)")) suite.rc.Check(testCase.region) - suite.NoError(failpoint.Disable("github.com/tikv/pd/pkg/schedule/checker/assertShouldNotCache")) + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/schedule/checker/assertShouldNotCache")) } } } // Ref https://github.com/tikv/pd/issues/4045 -func (suite *ruleCheckerTestSuite) TestSkipFixOrphanPeerIfSelectedPeerisPendingOrDown() { +func (suite *ruleCheckerTestSuite) TestSkipFixOrphanPeerIfSelectedPeerIsPendingOrDown() { + re := suite.Require() suite.cluster.AddLabelsStore(1, 1, map[string]string{"host": "host1"}) suite.cluster.AddLabelsStore(2, 1, map[string]string{"host": "host1"}) suite.cluster.AddLabelsStore(3, 1, map[string]string{"host": "host2"}) @@ -1655,7 +1695,7 @@ func (suite *ruleCheckerTestSuite) TestSkipFixOrphanPeerIfSelectedPeerisPendingO // should not remove extra peer op := suite.rc.Check(suite.cluster.GetRegion(1)) - suite.Nil(op) + re.Nil(op) // set peer3 to down-peer r1 = r1.Clone(core.WithPendingPeers([]*metapb.Peer{r1.GetStorePeer(4)})) @@ -1669,7 +1709,7 @@ func (suite *ruleCheckerTestSuite) TestSkipFixOrphanPeerIfSelectedPeerisPendingO // should not remove extra peer op = suite.rc.Check(suite.cluster.GetRegion(1)) - suite.Nil(op) + re.Nil(op) // set peer3 to normal r1 = r1.Clone(core.WithDownPeers(nil)) @@ -1678,11 +1718,12 @@ func (suite *ruleCheckerTestSuite) TestSkipFixOrphanPeerIfSelectedPeerisPendingO // should remove extra peer now var remove operator.RemovePeer op = suite.rc.Check(suite.cluster.GetRegion(1)) - suite.IsType(remove, op.Step(0)) - suite.Equal("remove-orphan-peer", op.Desc()) + re.IsType(remove, op.Step(0)) + re.Equal("remove-orphan-peer", op.Desc()) } func (suite *ruleCheckerTestSuite) TestPriorityFitHealthPeers() { + re := suite.Require() suite.cluster.AddLabelsStore(1, 1, map[string]string{"host": "host1"}) suite.cluster.AddLabelsStore(2, 1, map[string]string{"host": "host2"}) suite.cluster.AddLabelsStore(3, 1, map[string]string{"host": "host3"}) @@ -1696,8 +1737,8 @@ func (suite *ruleCheckerTestSuite) TestPriorityFitHealthPeers() { var remove operator.RemovePeer op := suite.rc.Check(suite.cluster.GetRegion(1)) - suite.IsType(remove, op.Step(0)) - suite.Equal("remove-orphan-peer", op.Desc()) + re.IsType(remove, op.Step(0)) + re.Equal("remove-orphan-peer", op.Desc()) // set peer3 to down r1 = r1.Clone(core.WithDownPeers([]*pdpb.PeerStats{ @@ -1710,12 +1751,13 @@ func (suite *ruleCheckerTestSuite) TestPriorityFitHealthPeers() { suite.cluster.PutRegion(r1) op = suite.rc.Check(suite.cluster.GetRegion(1)) - suite.IsType(remove, op.Step(0)) - suite.Equal("remove-orphan-peer", op.Desc()) + re.IsType(remove, op.Step(0)) + re.Equal("remove-orphan-peer", op.Desc()) } // Ref https://github.com/tikv/pd/issues/4140 func (suite *ruleCheckerTestSuite) TestDemoteVoter() { + re := suite.Require() suite.cluster.AddLabelsStore(1, 1, map[string]string{"zone": "z1"}) suite.cluster.AddLabelsStore(4, 1, map[string]string{"zone": "z4"}) region := suite.cluster.AddLeaderRegion(1, 1, 4) @@ -1749,58 +1791,61 @@ func (suite *ruleCheckerTestSuite) TestDemoteVoter() { suite.ruleManager.SetRule(rule2) suite.ruleManager.DeleteRule(placement.DefaultGroupID, placement.DefaultRuleID) op := suite.rc.Check(region) - suite.NotNil(op) - suite.Equal("fix-demote-voter", op.Desc()) + re.NotNil(op) + re.Equal("fix-demote-voter", op.Desc()) } func (suite *ruleCheckerTestSuite) TestOfflineAndDownStore() { + re := suite.Require() suite.cluster.AddLabelsStore(1, 1, map[string]string{"zone": "z1"}) suite.cluster.AddLabelsStore(2, 1, map[string]string{"zone": "z4"}) suite.cluster.AddLabelsStore(3, 1, map[string]string{"zone": "z1"}) suite.cluster.AddLabelsStore(4, 1, map[string]string{"zone": "z4"}) region := suite.cluster.AddLeaderRegion(1, 1, 2, 3) op := suite.rc.Check(region) - suite.Nil(op) + re.Nil(op) // assert rule checker should generate replace offline peer operator after cached suite.cluster.SetStoreOffline(1) op = suite.rc.Check(region) - suite.NotNil(op) - suite.Equal("replace-rule-offline-peer", op.Desc()) + re.NotNil(op) + re.Equal("replace-rule-offline-peer", op.Desc()) // re-cache the regionFit suite.cluster.SetStoreUp(1) op = suite.rc.Check(region) - suite.Nil(op) + re.Nil(op) // assert rule checker should generate replace down peer operator after cached suite.cluster.SetStoreDown(2) region = region.Clone(core.WithDownPeers([]*pdpb.PeerStats{{Peer: region.GetStorePeer(2), DownSeconds: 60000}})) op = suite.rc.Check(region) - suite.NotNil(op) - suite.Equal("fast-replace-rule-down-peer", op.Desc()) + re.NotNil(op) + re.Equal("fast-replace-rule-down-peer", op.Desc()) } func (suite *ruleCheckerTestSuite) TestPendingList() { + re := suite.Require() // no enough store suite.cluster.AddLeaderStore(1, 1) suite.cluster.AddLeaderRegionWithRange(1, "", "", 1, 2) op := suite.rc.Check(suite.cluster.GetRegion(1)) - suite.Nil(op) + re.Nil(op) _, exist := suite.rc.pendingList.Get(1) - suite.True(exist) + re.True(exist) // add more stores suite.cluster.AddLeaderStore(2, 1) suite.cluster.AddLeaderStore(3, 1) op = suite.rc.Check(suite.cluster.GetRegion(1)) - suite.NotNil(op) - suite.Equal("add-rule-peer", op.Desc()) - suite.Equal(constant.High, op.GetPriorityLevel()) - suite.Equal(uint64(3), op.Step(0).(operator.AddLearner).ToStore) + re.NotNil(op) + re.Equal("add-rule-peer", op.Desc()) + re.Equal(constant.High, op.GetPriorityLevel()) + re.Equal(uint64(3), op.Step(0).(operator.AddLearner).ToStore) _, exist = suite.rc.pendingList.Get(1) - suite.False(exist) + re.False(exist) } func (suite *ruleCheckerTestSuite) TestLocationLabels() { + re := suite.Require() suite.cluster.AddLabelsStore(1, 1, map[string]string{"zone": "z1", "rack": "r1", "host": "h1"}) suite.cluster.AddLabelsStore(2, 1, map[string]string{"zone": "z1", "rack": "r1", "host": "h1"}) suite.cluster.AddLabelsStore(3, 1, map[string]string{"zone": "z1", "rack": "r2", "host": "h1"}) @@ -1855,11 +1900,12 @@ func (suite *ruleCheckerTestSuite) TestLocationLabels() { suite.ruleManager.SetRule(rule3) suite.ruleManager.DeleteRule(placement.DefaultGroupID, placement.DefaultRuleID) op := suite.rc.Check(suite.cluster.GetRegion(1)) - suite.NotNil(op) - suite.Equal("move-to-better-location", op.Desc()) + re.NotNil(op) + re.Equal("move-to-better-location", op.Desc()) } func (suite *ruleCheckerTestSuite) TestTiFlashLocationLabels() { + re := suite.Require() suite.cluster.SetEnableUseJointConsensus(true) suite.cluster.AddLabelsStore(1, 1, map[string]string{"zone": "z1", "rack": "r1", "host": "h1"}) suite.cluster.AddLabelsStore(2, 1, map[string]string{"zone": "z1", "rack": "r1", "host": "h1"}) @@ -1888,7 +1934,7 @@ func (suite *ruleCheckerTestSuite) TestTiFlashLocationLabels() { rule.LocationLabels = []string{"zone", "rack", "host"} suite.ruleManager.SetRule(rule) op := suite.rc.Check(suite.cluster.GetRegion(1)) - suite.Nil(op) + re.Nil(op) } type ruleCheckerTestAdvancedSuite struct { @@ -1987,6 +2033,7 @@ func makeRule(def string) *placement.Rule { // TestReplaceAnExistingPeerCases address issue: https://github.com/tikv/pd/issues/7185 func (suite *ruleCheckerTestAdvancedSuite) TestReplaceAnExistingPeerCases() { + re := suite.Require() stores := makeStores() for _, store := range stores.GetStores() { suite.cluster.PutStore(store) @@ -2019,12 +2066,12 @@ func (suite *ruleCheckerTestAdvancedSuite) TestReplaceAnExistingPeerCases() { bundle.Rules = append(bundle.Rules, rule) } err := suite.ruleManager.SetGroupBundle(bundle) - suite.NoError(err) + re.NoError(err) region := makeRegion(cas.region) suite.cluster.PutRegion(region) op := suite.rc.Check(region) if len(cas.opStr) > 0 { - suite.Contains(op.String(), cas.opStr, i, cas.opStr) + re.Contains(op.String(), cas.opStr, i, cas.opStr) } suite.ruleManager.DeleteGroupBundle(groupName, false) } diff --git a/pkg/schedule/operator/create_operator_test.go b/pkg/schedule/operator/create_operator_test.go index 2fcd45d11f2..80c6cac4a04 100644 --- a/pkg/schedule/operator/create_operator_test.go +++ b/pkg/schedule/operator/create_operator_test.go @@ -67,6 +67,7 @@ func (suite *createOperatorTestSuite) TearDownTest() { } func (suite *createOperatorTestSuite) TestCreateSplitRegionOperator() { + re := suite.Require() type testCase struct { startKey []byte endKey []byte @@ -143,19 +144,19 @@ func (suite *createOperatorTestSuite) TestCreateSplitRegionOperator() { }, testCase.originPeers[0]) op, err := CreateSplitRegionOperator("test", region, 0, testCase.policy, testCase.keys) if testCase.expectedError { - suite.Error(err) + re.Error(err) continue } - suite.NoError(err) - suite.Equal(OpSplit, op.Kind()) - suite.Len(op.steps, 1) + re.NoError(err) + re.Equal(OpSplit, op.Kind()) + re.Len(op.steps, 1) for i := 0; i < op.Len(); i++ { switch step := op.Step(i).(type) { case SplitRegion: - suite.Equal(testCase.startKey, step.StartKey) - suite.Equal(testCase.endKey, step.EndKey) - suite.Equal(testCase.policy, step.Policy) - suite.Equal(testCase.keys, step.SplitKeys) + re.Equal(testCase.startKey, step.StartKey) + re.Equal(testCase.endKey, step.EndKey) + re.Equal(testCase.policy, step.Policy) + re.Equal(testCase.keys, step.SplitKeys) default: suite.T().Errorf("unexpected type: %s", step.String()) } @@ -164,6 +165,7 @@ func (suite *createOperatorTestSuite) TestCreateSplitRegionOperator() { } func (suite *createOperatorTestSuite) TestCreateMergeRegionOperator() { + re := suite.Require() type testCase struct { sourcePeers []*metapb.Peer // first is leader targetPeers []*metapb.Peer // first is leader @@ -293,58 +295,59 @@ func (suite *createOperatorTestSuite) TestCreateMergeRegionOperator() { target := core.NewRegionInfo(&metapb.Region{Id: 86, Peers: testCase.targetPeers}, testCase.targetPeers[0]) ops, err := CreateMergeRegionOperator("test", suite.cluster, source, target, 0) if testCase.expectedError { - suite.Error(err) + re.Error(err) continue } - suite.NoError(err) - suite.Len(ops, 2) - suite.Equal(testCase.kind, ops[0].kind) - suite.Equal(len(testCase.prepareSteps)+1, ops[0].Len()) - suite.Equal(testCase.kind, ops[1].kind) - suite.Equal(1, ops[1].Len()) - suite.Equal(MergeRegion{source.GetMeta(), target.GetMeta(), true}, ops[1].Step(0).(MergeRegion)) + re.NoError(err) + re.Len(ops, 2) + re.Equal(testCase.kind, ops[0].kind) + re.Equal(len(testCase.prepareSteps)+1, ops[0].Len()) + re.Equal(testCase.kind, ops[1].kind) + re.Equal(1, ops[1].Len()) + re.Equal(MergeRegion{source.GetMeta(), target.GetMeta(), true}, ops[1].Step(0).(MergeRegion)) expectedSteps := append(testCase.prepareSteps, MergeRegion{source.GetMeta(), target.GetMeta(), false}) for i := 0; i < ops[0].Len(); i++ { switch step := ops[0].Step(i).(type) { case TransferLeader: - suite.Equal(expectedSteps[i].(TransferLeader).FromStore, step.FromStore) - suite.Equal(expectedSteps[i].(TransferLeader).ToStore, step.ToStore) + re.Equal(expectedSteps[i].(TransferLeader).FromStore, step.FromStore) + re.Equal(expectedSteps[i].(TransferLeader).ToStore, step.ToStore) case AddLearner: - suite.Equal(expectedSteps[i].(AddLearner).ToStore, step.ToStore) - suite.Equal(expectedSteps[i].(AddLearner).IsWitness, step.IsWitness) + re.Equal(expectedSteps[i].(AddLearner).ToStore, step.ToStore) + re.Equal(expectedSteps[i].(AddLearner).IsWitness, step.IsWitness) case RemovePeer: - suite.Equal(expectedSteps[i].(RemovePeer).FromStore, step.FromStore) + re.Equal(expectedSteps[i].(RemovePeer).FromStore, step.FromStore) case ChangePeerV2Enter: - suite.Len(step.PromoteLearners, len(expectedSteps[i].(ChangePeerV2Enter).PromoteLearners)) - suite.Len(step.DemoteVoters, len(expectedSteps[i].(ChangePeerV2Enter).DemoteVoters)) + re.Len(step.PromoteLearners, len(expectedSteps[i].(ChangePeerV2Enter).PromoteLearners)) + re.Len(step.DemoteVoters, len(expectedSteps[i].(ChangePeerV2Enter).DemoteVoters)) for j, p := range expectedSteps[i].(ChangePeerV2Enter).PromoteLearners { - suite.Equal(p.ToStore, step.PromoteLearners[j].ToStore) - suite.Equal(p.IsWitness, step.PromoteLearners[j].IsWitness) + re.Equal(p.ToStore, step.PromoteLearners[j].ToStore) + re.Equal(p.IsWitness, step.PromoteLearners[j].IsWitness) } for j, d := range expectedSteps[i].(ChangePeerV2Enter).DemoteVoters { - suite.Equal(d.ToStore, step.DemoteVoters[j].ToStore) - suite.Equal(d.IsWitness, step.DemoteVoters[j].IsWitness) + re.Equal(d.ToStore, step.DemoteVoters[j].ToStore) + re.Equal(d.IsWitness, step.DemoteVoters[j].IsWitness) } case ChangePeerV2Leave: - suite.Len(step.PromoteLearners, len(expectedSteps[i].(ChangePeerV2Leave).PromoteLearners)) - suite.Len(step.DemoteVoters, len(expectedSteps[i].(ChangePeerV2Leave).DemoteVoters)) + re.Len(step.PromoteLearners, len(expectedSteps[i].(ChangePeerV2Leave).PromoteLearners)) + re.Len(step.DemoteVoters, len(expectedSteps[i].(ChangePeerV2Leave).DemoteVoters)) for j, p := range expectedSteps[i].(ChangePeerV2Leave).PromoteLearners { - suite.Equal(p.ToStore, step.PromoteLearners[j].ToStore) - suite.Equal(p.IsWitness, step.PromoteLearners[j].IsWitness) + re.Equal(p.ToStore, step.PromoteLearners[j].ToStore) + re.Equal(p.IsWitness, step.PromoteLearners[j].IsWitness) } for j, d := range expectedSteps[i].(ChangePeerV2Leave).DemoteVoters { - suite.Equal(d.ToStore, step.DemoteVoters[j].ToStore) - suite.Equal(d.IsWitness, step.DemoteVoters[j].IsWitness) + re.Equal(d.ToStore, step.DemoteVoters[j].ToStore) + re.Equal(d.IsWitness, step.DemoteVoters[j].IsWitness) } case MergeRegion: - suite.Equal(expectedSteps[i].(MergeRegion), step) + re.Equal(expectedSteps[i].(MergeRegion), step) } } } } func (suite *createOperatorTestSuite) TestCreateTransferLeaderOperator() { + re := suite.Require() type testCase struct { originPeers []*metapb.Peer // first is leader targetLeaderStoreID uint64 @@ -423,17 +426,17 @@ func (suite *createOperatorTestSuite) TestCreateTransferLeaderOperator() { op, err := CreateTransferLeaderOperator("test", suite.cluster, region, testCase.originPeers[0].StoreId, testCase.targetLeaderStoreID, []uint64{}, 0) if testCase.isErr { - suite.Error(err) + re.Error(err) continue } - suite.NoError(err) - suite.Equal(OpLeader, op.Kind()) - suite.Len(op.steps, 1) + re.NoError(err) + re.Equal(OpLeader, op.Kind()) + re.Len(op.steps, 1) switch step := op.Step(0).(type) { case TransferLeader: - suite.Equal(testCase.originPeers[0].StoreId, step.FromStore) - suite.Equal(testCase.targetLeaderStoreID, step.ToStore) + re.Equal(testCase.originPeers[0].StoreId, step.FromStore) + re.Equal(testCase.targetLeaderStoreID, step.ToStore) default: suite.T().Errorf("unexpected type: %s", step.String()) } @@ -441,6 +444,7 @@ func (suite *createOperatorTestSuite) TestCreateTransferLeaderOperator() { } func (suite *createOperatorTestSuite) TestCreateLeaveJointStateOperator() { + re := suite.Require() type testCase struct { originPeers []*metapb.Peer // first is leader offlineStores []uint64 @@ -593,26 +597,26 @@ func (suite *createOperatorTestSuite) TestCreateLeaveJointStateOperator() { region := core.NewRegionInfo(&metapb.Region{Id: 1, Peers: testCase.originPeers}, testCase.originPeers[0]) op, err := CreateLeaveJointStateOperator("test", suite.cluster, region) if len(testCase.steps) == 0 { - suite.Error(err) + re.Error(err) revertOffline() continue } - suite.NoError(err) - suite.Equal(testCase.kind, op.Kind()) - suite.Len(op.steps, len(testCase.steps)) + re.NoError(err) + re.Equal(testCase.kind, op.Kind()) + re.Len(op.steps, len(testCase.steps)) for i := 0; i < op.Len(); i++ { switch step := op.Step(i).(type) { case TransferLeader: - suite.Equal(testCase.steps[i].(TransferLeader).FromStore, step.FromStore) - suite.Equal(testCase.steps[i].(TransferLeader).ToStore, step.ToStore) + re.Equal(testCase.steps[i].(TransferLeader).FromStore, step.FromStore) + re.Equal(testCase.steps[i].(TransferLeader).ToStore, step.ToStore) case ChangePeerV2Leave: - suite.Len(step.PromoteLearners, len(testCase.steps[i].(ChangePeerV2Leave).PromoteLearners)) - suite.Len(step.DemoteVoters, len(testCase.steps[i].(ChangePeerV2Leave).DemoteVoters)) + re.Len(step.PromoteLearners, len(testCase.steps[i].(ChangePeerV2Leave).PromoteLearners)) + re.Len(step.DemoteVoters, len(testCase.steps[i].(ChangePeerV2Leave).DemoteVoters)) for j, p := range testCase.steps[i].(ChangePeerV2Leave).PromoteLearners { - suite.Equal(p.ToStore, step.PromoteLearners[j].ToStore) + re.Equal(p.ToStore, step.PromoteLearners[j].ToStore) } for j, d := range testCase.steps[i].(ChangePeerV2Leave).DemoteVoters { - suite.Equal(d.ToStore, step.DemoteVoters[j].ToStore) + re.Equal(d.ToStore, step.DemoteVoters[j].ToStore) } default: suite.T().Errorf("unexpected type: %s", step.String()) @@ -624,6 +628,7 @@ func (suite *createOperatorTestSuite) TestCreateLeaveJointStateOperator() { } func (suite *createOperatorTestSuite) TestCreateMoveRegionOperator() { + re := suite.Require() type testCase struct { name string originPeers []*metapb.Peer // first is leader @@ -932,45 +937,45 @@ func (suite *createOperatorTestSuite) TestCreateMoveRegionOperator() { op, err := CreateMoveRegionOperator("test", suite.cluster, region, OpAdmin, testCase.targetPeerRoles) if testCase.expectedError == nil { - suite.NoError(err) + re.NoError(err) } else { - suite.Error(err) - suite.Contains(err.Error(), testCase.expectedError.Error()) + re.Error(err) + re.Contains(err.Error(), testCase.expectedError.Error()) continue } - suite.NotNil(op) + re.NotNil(op) - suite.Len(testCase.steps, op.Len()) + re.Len(testCase.steps, op.Len()) // Since the peer id may be generated by allocator in runtime, we only check store id. for i := 0; i < op.Len(); i++ { switch step := op.Step(i).(type) { case TransferLeader: - suite.Equal(testCase.steps[i].(TransferLeader).FromStore, step.FromStore) - suite.Equal(testCase.steps[i].(TransferLeader).ToStore, step.ToStore) + re.Equal(testCase.steps[i].(TransferLeader).FromStore, step.FromStore) + re.Equal(testCase.steps[i].(TransferLeader).ToStore, step.ToStore) case ChangePeerV2Leave: - suite.Len(step.PromoteLearners, len(testCase.steps[i].(ChangePeerV2Leave).PromoteLearners)) - suite.Len(step.DemoteVoters, len(testCase.steps[i].(ChangePeerV2Leave).DemoteVoters)) + re.Len(step.PromoteLearners, len(testCase.steps[i].(ChangePeerV2Leave).PromoteLearners)) + re.Len(step.DemoteVoters, len(testCase.steps[i].(ChangePeerV2Leave).DemoteVoters)) for j, p := range testCase.steps[i].(ChangePeerV2Leave).PromoteLearners { - suite.Equal(p.ToStore, step.PromoteLearners[j].ToStore) + re.Equal(p.ToStore, step.PromoteLearners[j].ToStore) } for j, d := range testCase.steps[i].(ChangePeerV2Leave).DemoteVoters { - suite.Equal(d.ToStore, step.DemoteVoters[j].ToStore) + re.Equal(d.ToStore, step.DemoteVoters[j].ToStore) } case ChangePeerV2Enter: - suite.Len(step.PromoteLearners, len(testCase.steps[i].(ChangePeerV2Enter).PromoteLearners)) - suite.Len(step.DemoteVoters, len(testCase.steps[i].(ChangePeerV2Enter).DemoteVoters)) + re.Len(step.PromoteLearners, len(testCase.steps[i].(ChangePeerV2Enter).PromoteLearners)) + re.Len(step.DemoteVoters, len(testCase.steps[i].(ChangePeerV2Enter).DemoteVoters)) for j, p := range testCase.steps[i].(ChangePeerV2Enter).PromoteLearners { - suite.Equal(p.ToStore, step.PromoteLearners[j].ToStore) + re.Equal(p.ToStore, step.PromoteLearners[j].ToStore) } for j, d := range testCase.steps[i].(ChangePeerV2Enter).DemoteVoters { - suite.Equal(d.ToStore, step.DemoteVoters[j].ToStore) + re.Equal(d.ToStore, step.DemoteVoters[j].ToStore) } case AddLearner: - suite.Equal(testCase.steps[i].(AddLearner).ToStore, step.ToStore) + re.Equal(testCase.steps[i].(AddLearner).ToStore, step.ToStore) case PromoteLearner: - suite.Equal(testCase.steps[i].(PromoteLearner).ToStore, step.ToStore) + re.Equal(testCase.steps[i].(PromoteLearner).ToStore, step.ToStore) case RemovePeer: - suite.Equal(testCase.steps[i].(RemovePeer).FromStore, step.FromStore) + re.Equal(testCase.steps[i].(RemovePeer).FromStore, step.FromStore) default: suite.T().Errorf("unexpected type: %s", step.String()) } @@ -979,6 +984,7 @@ func (suite *createOperatorTestSuite) TestCreateMoveRegionOperator() { } func (suite *createOperatorTestSuite) TestMoveRegionWithoutJointConsensus() { + re := suite.Require() type testCase struct { name string originPeers []*metapb.Peer // first is leader @@ -1107,27 +1113,27 @@ func (suite *createOperatorTestSuite) TestMoveRegionWithoutJointConsensus() { op, err := CreateMoveRegionOperator("test", suite.cluster, region, OpAdmin, testCase.targetPeerRoles) if testCase.expectedError == nil { - suite.NoError(err) + re.NoError(err) } else { - suite.Error(err) - suite.Contains(err.Error(), testCase.expectedError.Error()) + re.Error(err) + re.Contains(err.Error(), testCase.expectedError.Error()) continue } - suite.NotNil(op) + re.NotNil(op) - suite.Len(testCase.steps, op.Len()) + re.Len(testCase.steps, op.Len()) // Since the peer id may be generated by allocator in runtime, we only check store id. for i := 0; i < op.Len(); i++ { switch step := op.Step(i).(type) { case TransferLeader: - suite.Equal(testCase.steps[i].(TransferLeader).FromStore, step.FromStore) - suite.Equal(testCase.steps[i].(TransferLeader).ToStore, step.ToStore) + re.Equal(testCase.steps[i].(TransferLeader).FromStore, step.FromStore) + re.Equal(testCase.steps[i].(TransferLeader).ToStore, step.ToStore) case AddLearner: - suite.Equal(testCase.steps[i].(AddLearner).ToStore, step.ToStore) + re.Equal(testCase.steps[i].(AddLearner).ToStore, step.ToStore) case PromoteLearner: - suite.Equal(testCase.steps[i].(PromoteLearner).ToStore, step.ToStore) + re.Equal(testCase.steps[i].(PromoteLearner).ToStore, step.ToStore) case RemovePeer: - suite.Equal(testCase.steps[i].(RemovePeer).FromStore, step.FromStore) + re.Equal(testCase.steps[i].(RemovePeer).FromStore, step.FromStore) default: suite.T().Errorf("unexpected type: %s", step.String()) } @@ -1194,6 +1200,7 @@ func TestCreateLeaveJointStateOperatorWithoutFitRules(t *testing.T) { } func (suite *createOperatorTestSuite) TestCreateNonWitnessPeerOperator() { + re := suite.Require() type testCase struct { originPeers []*metapb.Peer // first is leader kind OpKind @@ -1234,21 +1241,21 @@ func (suite *createOperatorTestSuite) TestCreateNonWitnessPeerOperator() { for _, testCase := range testCases { region := core.NewRegionInfo(&metapb.Region{Id: 68, Peers: testCase.originPeers}, testCase.originPeers[0]) op, err := CreateNonWitnessPeerOperator("test", suite.cluster, region, testCase.originPeers[1]) - suite.NoError(err) - suite.NotNil(op) - suite.Equal(testCase.kind, op.kind) + re.NoError(err) + re.NotNil(op) + re.Equal(testCase.kind, op.kind) expectedSteps := testCase.prepareSteps for i := 0; i < op.Len(); i++ { switch step := op.Step(i).(type) { case ChangePeerV2Enter: - suite.Len(step.DemoteVoters, len(expectedSteps[i].(ChangePeerV2Enter).DemoteVoters)) + re.Len(step.DemoteVoters, len(expectedSteps[i].(ChangePeerV2Enter).DemoteVoters)) for j, d := range expectedSteps[i].(ChangePeerV2Enter).DemoteVoters { - suite.Equal(d.ToStore, step.DemoteVoters[j].ToStore) + re.Equal(d.ToStore, step.DemoteVoters[j].ToStore) } case BecomeNonWitness: - suite.Equal(step.StoreID, expectedSteps[i].(BecomeNonWitness).StoreID) - suite.Equal(step.PeerID, expectedSteps[i].(BecomeNonWitness).PeerID) + re.Equal(step.StoreID, expectedSteps[i].(BecomeNonWitness).StoreID) + re.Equal(step.PeerID, expectedSteps[i].(BecomeNonWitness).PeerID) } } } diff --git a/pkg/schedule/operator/operator_controller_test.go b/pkg/schedule/operator/operator_controller_test.go index 6d5f835ca68..e47281a2c68 100644 --- a/pkg/schedule/operator/operator_controller_test.go +++ b/pkg/schedule/operator/operator_controller_test.go @@ -26,6 +26,7 @@ import ( "github.com/pingcap/failpoint" "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/kvproto/pkg/pdpb" + "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/core/storelimit" @@ -47,8 +48,9 @@ func TestOperatorControllerTestSuite(t *testing.T) { } func (suite *operatorControllerTestSuite) SetupSuite() { + re := suite.Require() suite.ctx, suite.cancel = context.WithCancel(context.Background()) - suite.NoError(failpoint.Enable("github.com/tikv/pd/pkg/schedule/operator/unexpectedOperator", "return(true)")) + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/schedule/operator/unexpectedOperator", "return(true)")) } func (suite *operatorControllerTestSuite) TearDownSuite() { @@ -56,6 +58,7 @@ func (suite *operatorControllerTestSuite) TearDownSuite() { } func (suite *operatorControllerTestSuite) TestCacheInfluence() { + re := suite.Require() opt := mockconfig.NewTestOptions() tc := mockcluster.NewCluster(suite.ctx, opt) bc := tc.GetBasicCluster() @@ -68,27 +71,28 @@ func (suite *operatorControllerTestSuite) TestCacheInfluence() { } op := NewTestOperator(1, &metapb.RegionEpoch{}, OpRegion, steps...) oc.SetOperator(op) - suite.True(op.Start()) + re.True(op.Start()) influence := NewOpInfluence() AddOpInfluence(op, *influence, bc) - suite.Equal(int64(-96), influence.GetStoreInfluence(2).RegionSize) + re.Equal(int64(-96), influence.GetStoreInfluence(2).RegionSize) // case: influence is same even if the region size changed. region = region.Clone(core.SetApproximateSize(100)) tc.PutRegion(region) influence1 := NewOpInfluence() AddOpInfluence(op, *influence1, bc) - suite.Equal(int64(-96), influence1.GetStoreInfluence(2).RegionSize) + re.Equal(int64(-96), influence1.GetStoreInfluence(2).RegionSize) // case: influence is valid even if the region is removed. tc.RemoveRegion(region) influence2 := NewOpInfluence() AddOpInfluence(op, *influence2, bc) - suite.Equal(int64(-96), influence2.GetStoreInfluence(2).RegionSize) + re.Equal(int64(-96), influence2.GetStoreInfluence(2).RegionSize) } // issue #1338 func (suite *operatorControllerTestSuite) TestGetOpInfluence() { + re := suite.Require() opt := mockconfig.NewTestOptions() tc := mockcluster.NewCluster(suite.ctx, opt) oc := NewController(suite.ctx, tc.GetBasicCluster(), tc.GetSharedConfig(), nil) @@ -100,13 +104,12 @@ func (suite *operatorControllerTestSuite) TestGetOpInfluence() { } op1 := NewTestOperator(1, &metapb.RegionEpoch{}, OpRegion, steps...) op2 := NewTestOperator(2, &metapb.RegionEpoch{}, OpRegion, steps...) - suite.True(op1.Start()) + re.True(op1.Start()) oc.SetOperator(op1) - suite.True(op2.Start()) + re.True(op2.Start()) oc.SetOperator(op2) - re := suite.Require() go func(ctx context.Context) { - suite.checkRemoveOperatorSuccess(oc, op1) + suite.checkRemoveOperatorSuccess(re, oc, op1) for { select { case <-ctx.Done(): @@ -127,10 +130,11 @@ func (suite *operatorControllerTestSuite) TestGetOpInfluence() { } }(suite.ctx) time.Sleep(time.Second) - suite.NotNil(oc.GetOperator(2)) + re.NotNil(oc.GetOperator(2)) } func (suite *operatorControllerTestSuite) TestOperatorStatus() { + re := suite.Require() opt := mockconfig.NewTestOptions() tc := mockcluster.NewCluster(suite.ctx, opt) stream := hbstream.NewTestHeartbeatStreams(suite.ctx, tc.ID, tc, false /* no need to run */) @@ -147,25 +151,26 @@ func (suite *operatorControllerTestSuite) TestOperatorStatus() { region2 := tc.GetRegion(2) op1 := NewTestOperator(1, &metapb.RegionEpoch{}, OpRegion, steps...) op2 := NewTestOperator(2, &metapb.RegionEpoch{}, OpRegion, steps...) - suite.True(op1.Start()) + re.True(op1.Start()) oc.SetOperator(op1) - suite.True(op2.Start()) + re.True(op2.Start()) oc.SetOperator(op2) - suite.Equal(pdpb.OperatorStatus_RUNNING, oc.GetOperatorStatus(1).Status) - suite.Equal(pdpb.OperatorStatus_RUNNING, oc.GetOperatorStatus(2).Status) + re.Equal(pdpb.OperatorStatus_RUNNING, oc.GetOperatorStatus(1).Status) + re.Equal(pdpb.OperatorStatus_RUNNING, oc.GetOperatorStatus(2).Status) op1.SetStatusReachTime(STARTED, time.Now().Add(-SlowStepWaitTime-FastStepWaitTime)) region2 = ApplyOperatorStep(region2, op2) tc.PutRegion(region2) oc.Dispatch(region1, "test", nil) oc.Dispatch(region2, "test", nil) - suite.Equal(pdpb.OperatorStatus_TIMEOUT, oc.GetOperatorStatus(1).Status) - suite.Equal(pdpb.OperatorStatus_RUNNING, oc.GetOperatorStatus(2).Status) + re.Equal(pdpb.OperatorStatus_TIMEOUT, oc.GetOperatorStatus(1).Status) + re.Equal(pdpb.OperatorStatus_RUNNING, oc.GetOperatorStatus(2).Status) ApplyOperator(tc, op2) oc.Dispatch(region2, "test", nil) - suite.Equal(pdpb.OperatorStatus_SUCCESS, oc.GetOperatorStatus(2).Status) + re.Equal(pdpb.OperatorStatus_SUCCESS, oc.GetOperatorStatus(2).Status) } func (suite *operatorControllerTestSuite) TestFastFailOperator() { + re := suite.Require() opt := mockconfig.NewTestOptions() tc := mockcluster.NewCluster(suite.ctx, opt) stream := hbstream.NewTestHeartbeatStreams(suite.ctx, tc.ID, tc, false /* no need to run */) @@ -180,26 +185,27 @@ func (suite *operatorControllerTestSuite) TestFastFailOperator() { } region := tc.GetRegion(1) op := NewTestOperator(1, &metapb.RegionEpoch{}, OpRegion, steps...) - suite.True(op.Start()) + re.True(op.Start()) oc.SetOperator(op) oc.Dispatch(region, "test", nil) - suite.Equal(pdpb.OperatorStatus_RUNNING, oc.GetOperatorStatus(1).Status) + re.Equal(pdpb.OperatorStatus_RUNNING, oc.GetOperatorStatus(1).Status) // change the leader region = region.Clone(core.WithLeader(region.GetPeer(2))) oc.Dispatch(region, DispatchFromHeartBeat, nil) - suite.Equal(CANCELED, op.Status()) - suite.Nil(oc.GetOperator(region.GetID())) + re.Equal(CANCELED, op.Status()) + re.Nil(oc.GetOperator(region.GetID())) // transfer leader to an illegal store. op = NewTestOperator(1, &metapb.RegionEpoch{}, OpRegion, TransferLeader{ToStore: 5}) oc.SetOperator(op) oc.Dispatch(region, DispatchFromHeartBeat, nil) - suite.Equal(CANCELED, op.Status()) - suite.Nil(oc.GetOperator(region.GetID())) + re.Equal(CANCELED, op.Status()) + re.Nil(oc.GetOperator(region.GetID())) } // Issue 3353 func (suite *operatorControllerTestSuite) TestFastFailWithUnhealthyStore() { + re := suite.Require() opt := mockconfig.NewTestOptions() tc := mockcluster.NewCluster(suite.ctx, opt) stream := hbstream.NewTestHeartbeatStreams(suite.ctx, tc.ID, tc, false /* no need to run */) @@ -212,13 +218,14 @@ func (suite *operatorControllerTestSuite) TestFastFailWithUnhealthyStore() { steps := []OpStep{TransferLeader{ToStore: 2}} op := NewTestOperator(1, region.GetRegionEpoch(), OpLeader, steps...) oc.SetOperator(op) - suite.False(oc.checkStaleOperator(op, steps[0], region)) + re.False(oc.checkStaleOperator(op, steps[0], region)) tc.SetStoreDown(2) - suite.True(oc.checkStaleOperator(op, steps[0], region)) + re.True(oc.checkStaleOperator(op, steps[0], region)) } func (suite *operatorControllerTestSuite) TestCheckAddUnexpectedStatus() { - suite.NoError(failpoint.Disable("github.com/tikv/pd/pkg/schedule/operator/unexpectedOperator")) + re := suite.Require() + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/schedule/operator/unexpectedOperator")) opt := mockconfig.NewTestOptions() tc := mockcluster.NewCluster(suite.ctx, opt) @@ -236,55 +243,56 @@ func (suite *operatorControllerTestSuite) TestCheckAddUnexpectedStatus() { { // finished op op := NewTestOperator(1, &metapb.RegionEpoch{}, OpRegion, TransferLeader{ToStore: 2}) - suite.True(oc.checkAddOperator(false, op)) + re.True(oc.checkAddOperator(false, op)) op.Start() - suite.False(oc.checkAddOperator(false, op)) // started - suite.Nil(op.Check(region1)) + re.False(oc.checkAddOperator(false, op)) // started + re.Nil(op.Check(region1)) - suite.Equal(SUCCESS, op.Status()) - suite.False(oc.checkAddOperator(false, op)) // success + re.Equal(SUCCESS, op.Status()) + re.False(oc.checkAddOperator(false, op)) // success } { // finished op canceled op := NewTestOperator(1, &metapb.RegionEpoch{}, OpRegion, TransferLeader{ToStore: 2}) - suite.True(oc.checkAddOperator(false, op)) - suite.True(op.Cancel(AdminStop)) - suite.False(oc.checkAddOperator(false, op)) + re.True(oc.checkAddOperator(false, op)) + re.True(op.Cancel(AdminStop)) + re.False(oc.checkAddOperator(false, op)) } { // finished op replaced op := NewTestOperator(1, &metapb.RegionEpoch{}, OpRegion, TransferLeader{ToStore: 2}) - suite.True(oc.checkAddOperator(false, op)) - suite.True(op.Start()) - suite.True(op.Replace()) - suite.False(oc.checkAddOperator(false, op)) + re.True(oc.checkAddOperator(false, op)) + re.True(op.Start()) + re.True(op.Replace()) + re.False(oc.checkAddOperator(false, op)) } { // finished op expired op1 := NewTestOperator(1, &metapb.RegionEpoch{}, OpRegion, TransferLeader{ToStore: 2}) op2 := NewTestOperator(2, &metapb.RegionEpoch{}, OpRegion, TransferLeader{ToStore: 1}) - suite.True(oc.checkAddOperator(false, op1, op2)) + re.True(oc.checkAddOperator(false, op1, op2)) op1.SetStatusReachTime(CREATED, time.Now().Add(-OperatorExpireTime)) op2.SetStatusReachTime(CREATED, time.Now().Add(-OperatorExpireTime)) - suite.False(oc.checkAddOperator(false, op1, op2)) - suite.Equal(EXPIRED, op1.Status()) - suite.Equal(EXPIRED, op2.Status()) + re.False(oc.checkAddOperator(false, op1, op2)) + re.Equal(EXPIRED, op1.Status()) + re.Equal(EXPIRED, op2.Status()) } // finished op never timeout { // unfinished op timeout op := NewTestOperator(1, &metapb.RegionEpoch{}, OpRegion, steps...) - suite.True(oc.checkAddOperator(false, op)) + re.True(oc.checkAddOperator(false, op)) op.Start() op.SetStatusReachTime(STARTED, time.Now().Add(-SlowStepWaitTime-FastStepWaitTime)) - suite.True(op.CheckTimeout()) - suite.False(oc.checkAddOperator(false, op)) + re.True(op.CheckTimeout()) + re.False(oc.checkAddOperator(false, op)) } } // issue #1716 func (suite *operatorControllerTestSuite) TestConcurrentRemoveOperator() { + re := suite.Require() opt := mockconfig.NewTestOptions() tc := mockcluster.NewCluster(suite.ctx, opt) stream := hbstream.NewTestHeartbeatStreams(suite.ctx, tc.ID, tc, false /* no need to run */) @@ -302,10 +310,10 @@ func (suite *operatorControllerTestSuite) TestConcurrentRemoveOperator() { // unfinished op with high priority op2 := NewTestOperator(1, &metapb.RegionEpoch{}, OpRegion|OpAdmin, steps...) - suite.True(op1.Start()) + re.True(op1.Start()) oc.SetOperator(op1) - suite.NoError(failpoint.Enable("github.com/tikv/pd/pkg/schedule/operator/concurrentRemoveOperator", "return(true)")) + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/schedule/operator/concurrentRemoveOperator", "return(true)")) var wg sync.WaitGroup wg.Add(2) go func() { @@ -316,16 +324,17 @@ func (suite *operatorControllerTestSuite) TestConcurrentRemoveOperator() { time.Sleep(50 * time.Millisecond) success := oc.AddOperator(op2) // If the assert failed before wg.Done, the test will be blocked. - defer suite.True(success) + defer re.True(success) wg.Done() }() wg.Wait() - suite.Equal(op2, oc.GetOperator(1)) - suite.NoError(failpoint.Disable("github.com/tikv/pd/pkg/schedule/operator/concurrentRemoveOperator")) + re.Equal(op2, oc.GetOperator(1)) + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/schedule/operator/concurrentRemoveOperator")) } func (suite *operatorControllerTestSuite) TestPollDispatchRegion() { + re := suite.Require() opt := mockconfig.NewTestOptions() tc := mockcluster.NewCluster(suite.ctx, opt) stream := hbstream.NewTestHeartbeatStreams(suite.ctx, tc.ID, tc, false /* no need to run */) @@ -348,13 +357,13 @@ func (suite *operatorControllerTestSuite) TestPollDispatchRegion() { region4 := tc.GetRegion(4) // Adds operator and pushes to the notifier queue. { - suite.True(op1.Start()) + re.True(op1.Start()) oc.SetOperator(op1) - suite.True(op3.Start()) + re.True(op3.Start()) oc.SetOperator(op3) - suite.True(op4.Start()) + re.True(op4.Start()) oc.SetOperator(op4) - suite.True(op2.Start()) + re.True(op2.Start()) oc.SetOperator(op2) heap.Push(&oc.opNotifierQueue, &operatorWithTime{op: op1, time: time.Now().Add(100 * time.Millisecond)}) heap.Push(&oc.opNotifierQueue, &operatorWithTime{op: op3, time: time.Now().Add(300 * time.Millisecond)}) @@ -363,42 +372,43 @@ func (suite *operatorControllerTestSuite) TestPollDispatchRegion() { } // first poll got nil r, next := oc.pollNeedDispatchRegion() - suite.Nil(r) - suite.False(next) + re.Nil(r) + re.False(next) // after wait 100 millisecond, the region1 need to dispatch, but not region2. time.Sleep(100 * time.Millisecond) r, next = oc.pollNeedDispatchRegion() - suite.NotNil(r) - suite.True(next) - suite.Equal(region1.GetID(), r.GetID()) + re.NotNil(r) + re.True(next) + re.Equal(region1.GetID(), r.GetID()) // find op3 with nil region, remove it - suite.NotNil(oc.GetOperator(3)) + re.NotNil(oc.GetOperator(3)) r, next = oc.pollNeedDispatchRegion() - suite.Nil(r) - suite.True(next) - suite.Nil(oc.GetOperator(3)) + re.Nil(r) + re.True(next) + re.Nil(oc.GetOperator(3)) // find op4 finished r, next = oc.pollNeedDispatchRegion() - suite.NotNil(r) - suite.True(next) - suite.Equal(region4.GetID(), r.GetID()) + re.NotNil(r) + re.True(next) + re.Equal(region4.GetID(), r.GetID()) // after waiting 500 milliseconds, the region2 need to dispatch time.Sleep(400 * time.Millisecond) r, next = oc.pollNeedDispatchRegion() - suite.NotNil(r) - suite.True(next) - suite.Equal(region2.GetID(), r.GetID()) + re.NotNil(r) + re.True(next) + re.Equal(region2.GetID(), r.GetID()) r, next = oc.pollNeedDispatchRegion() - suite.Nil(r) - suite.False(next) + re.Nil(r) + re.False(next) } func (suite *operatorControllerTestSuite) TestStoreLimit() { + re := suite.Require() opt := mockconfig.NewTestOptions() tc := mockcluster.NewCluster(suite.ctx, opt) stream := hbstream.NewTestHeartbeatStreams(suite.ctx, tc.ID, tc, false /* no need to run */) @@ -415,58 +425,59 @@ func (suite *operatorControllerTestSuite) TestStoreLimit() { tc.SetStoreLimit(2, storelimit.AddPeer, 60) for i := uint64(1); i <= 5; i++ { op := NewTestOperator(1, &metapb.RegionEpoch{}, OpRegion, AddPeer{ToStore: 2, PeerID: i}) - suite.True(oc.AddOperator(op)) - suite.checkRemoveOperatorSuccess(oc, op) + re.True(oc.AddOperator(op)) + suite.checkRemoveOperatorSuccess(re, oc, op) } op := NewTestOperator(1, &metapb.RegionEpoch{}, OpRegion, AddPeer{ToStore: 2, PeerID: 1}) - suite.False(oc.AddOperator(op)) - suite.False(oc.RemoveOperator(op)) + re.False(oc.AddOperator(op)) + re.False(oc.RemoveOperator(op)) tc.SetStoreLimit(2, storelimit.AddPeer, 120) for i := uint64(1); i <= 10; i++ { op = NewTestOperator(i, &metapb.RegionEpoch{}, OpRegion, AddPeer{ToStore: 2, PeerID: i}) - suite.True(oc.AddOperator(op)) - suite.checkRemoveOperatorSuccess(oc, op) + re.True(oc.AddOperator(op)) + suite.checkRemoveOperatorSuccess(re, oc, op) } tc.SetAllStoresLimit(storelimit.AddPeer, 60) for i := uint64(1); i <= 5; i++ { op = NewTestOperator(i, &metapb.RegionEpoch{}, OpRegion, AddPeer{ToStore: 2, PeerID: i}) - suite.True(oc.AddOperator(op)) - suite.checkRemoveOperatorSuccess(oc, op) + re.True(oc.AddOperator(op)) + suite.checkRemoveOperatorSuccess(re, oc, op) } op = NewTestOperator(1, &metapb.RegionEpoch{}, OpRegion, AddPeer{ToStore: 2, PeerID: 1}) - suite.False(oc.AddOperator(op)) - suite.False(oc.RemoveOperator(op)) + re.False(oc.AddOperator(op)) + re.False(oc.RemoveOperator(op)) tc.SetStoreLimit(2, storelimit.RemovePeer, 60) for i := uint64(1); i <= 5; i++ { op := NewTestOperator(1, &metapb.RegionEpoch{}, OpRegion, RemovePeer{FromStore: 2}) - suite.True(oc.AddOperator(op)) - suite.checkRemoveOperatorSuccess(oc, op) + re.True(oc.AddOperator(op)) + suite.checkRemoveOperatorSuccess(re, oc, op) } op = NewTestOperator(1, &metapb.RegionEpoch{}, OpRegion, RemovePeer{FromStore: 2}) - suite.False(oc.AddOperator(op)) - suite.False(oc.RemoveOperator(op)) + re.False(oc.AddOperator(op)) + re.False(oc.RemoveOperator(op)) tc.SetStoreLimit(2, storelimit.RemovePeer, 120) for i := uint64(1); i <= 10; i++ { op = NewTestOperator(i, &metapb.RegionEpoch{}, OpRegion, RemovePeer{FromStore: 2}) - suite.True(oc.AddOperator(op)) - suite.checkRemoveOperatorSuccess(oc, op) + re.True(oc.AddOperator(op)) + suite.checkRemoveOperatorSuccess(re, oc, op) } tc.SetAllStoresLimit(storelimit.RemovePeer, 60) for i := uint64(1); i <= 5; i++ { op = NewTestOperator(i, &metapb.RegionEpoch{}, OpRegion, RemovePeer{FromStore: 2}) - suite.True(oc.AddOperator(op)) - suite.checkRemoveOperatorSuccess(oc, op) + re.True(oc.AddOperator(op)) + suite.checkRemoveOperatorSuccess(re, oc, op) } op = NewTestOperator(1, &metapb.RegionEpoch{}, OpRegion, RemovePeer{FromStore: 2}) - suite.False(oc.AddOperator(op)) - suite.False(oc.RemoveOperator(op)) + re.False(oc.AddOperator(op)) + re.False(oc.RemoveOperator(op)) } // #1652 func (suite *operatorControllerTestSuite) TestDispatchOutdatedRegion() { + re := suite.Require() cluster := mockcluster.NewCluster(suite.ctx, mockconfig.NewTestOptions()) stream := hbstream.NewTestHeartbeatStreams(suite.ctx, cluster.ID, cluster, false /* no need to run */) controller := NewController(suite.ctx, cluster.GetBasicCluster(), cluster.GetSharedConfig(), stream) @@ -481,42 +492,43 @@ func (suite *operatorControllerTestSuite) TestDispatchOutdatedRegion() { } op := NewTestOperator(1, &metapb.RegionEpoch{ConfVer: 0, Version: 0}, OpRegion, steps...) - suite.True(controller.AddOperator(op)) - suite.Equal(1, stream.MsgLength()) + re.True(controller.AddOperator(op)) + re.Equal(1, stream.MsgLength()) // report the result of transferring leader region := cluster.MockRegionInfo(1, 2, []uint64{1, 2}, []uint64{}, &metapb.RegionEpoch{ConfVer: 0, Version: 0}) controller.Dispatch(region, DispatchFromHeartBeat, nil) - suite.Equal(uint64(0), op.ConfVerChanged(region)) - suite.Equal(2, stream.MsgLength()) + re.Equal(uint64(0), op.ConfVerChanged(region)) + re.Equal(2, stream.MsgLength()) // report the result of removing peer region = cluster.MockRegionInfo(1, 2, []uint64{2}, []uint64{}, &metapb.RegionEpoch{ConfVer: 0, Version: 0}) controller.Dispatch(region, DispatchFromHeartBeat, nil) - suite.Equal(uint64(1), op.ConfVerChanged(region)) - suite.Equal(2, stream.MsgLength()) + re.Equal(uint64(1), op.ConfVerChanged(region)) + re.Equal(2, stream.MsgLength()) // add and dispatch op again, the op should be stale op = NewTestOperator(1, &metapb.RegionEpoch{ConfVer: 0, Version: 0}, OpRegion, steps...) - suite.True(controller.AddOperator(op)) - suite.Equal(uint64(0), op.ConfVerChanged(region)) - suite.Equal(3, stream.MsgLength()) + re.True(controller.AddOperator(op)) + re.Equal(uint64(0), op.ConfVerChanged(region)) + re.Equal(3, stream.MsgLength()) // report region with an abnormal confver region = cluster.MockRegionInfo(1, 1, []uint64{1, 2}, []uint64{}, &metapb.RegionEpoch{ConfVer: 1, Version: 0}) controller.Dispatch(region, DispatchFromHeartBeat, nil) - suite.Equal(uint64(0), op.ConfVerChanged(region)) + re.Equal(uint64(0), op.ConfVerChanged(region)) // no new step - suite.Equal(3, stream.MsgLength()) + re.Equal(3, stream.MsgLength()) } func (suite *operatorControllerTestSuite) TestCalcInfluence() { + re := suite.Require() cluster := mockcluster.NewCluster(suite.ctx, mockconfig.NewTestOptions()) stream := hbstream.NewTestHeartbeatStreams(suite.ctx, cluster.ID, cluster, false /* no need to run */) controller := NewController(suite.ctx, cluster.GetBasicCluster(), cluster.GetSharedConfig(), stream) @@ -535,16 +547,16 @@ func (suite *operatorControllerTestSuite) TestCalcInfluence() { RemovePeer{FromStore: 1}, } op := NewTestOperator(1, epoch, OpRegion, steps...) - suite.True(controller.AddOperator(op)) + re.True(controller.AddOperator(op)) check := func(influence OpInfluence, id uint64, expect *StoreInfluence) { si := influence.GetStoreInfluence(id) - suite.Equal(si.LeaderCount, expect.LeaderCount) - suite.Equal(si.LeaderSize, expect.LeaderSize) - suite.Equal(si.RegionCount, expect.RegionCount) - suite.Equal(si.RegionSize, expect.RegionSize) - suite.Equal(si.StepCost[storelimit.AddPeer], expect.StepCost[storelimit.AddPeer]) - suite.Equal(si.StepCost[storelimit.RemovePeer], expect.StepCost[storelimit.RemovePeer]) + re.Equal(si.LeaderCount, expect.LeaderCount) + re.Equal(si.LeaderSize, expect.LeaderSize) + re.Equal(si.RegionCount, expect.RegionCount) + re.Equal(si.RegionSize, expect.RegionSize) + re.Equal(si.StepCost[storelimit.AddPeer], expect.StepCost[storelimit.AddPeer]) + re.Equal(si.StepCost[storelimit.RemovePeer], expect.StepCost[storelimit.RemovePeer]) } influence := controller.GetOpInfluence(cluster.GetBasicCluster()) @@ -571,7 +583,7 @@ func (suite *operatorControllerTestSuite) TestCalcInfluence() { core.WithAddPeer(&metapb.Peer{Id: 3, StoreId: 3, Role: metapb.PeerRole_Learner}), core.WithIncConfVer(), ) - suite.True(steps[0].IsFinish(region2)) + re.True(steps[0].IsFinish(region2)) op.Check(region2) influence = controller.GetOpInfluence(cluster.GetBasicCluster()) @@ -594,6 +606,7 @@ func (suite *operatorControllerTestSuite) TestCalcInfluence() { } func (suite *operatorControllerTestSuite) TestDispatchUnfinishedStep() { + re := suite.Require() cluster := mockcluster.NewCluster(suite.ctx, mockconfig.NewTestOptions()) stream := hbstream.NewTestHeartbeatStreams(suite.ctx, cluster.ID, cluster, false /* no need to run */) controller := NewController(suite.ctx, cluster.GetBasicCluster(), cluster.GetSharedConfig(), stream) @@ -629,8 +642,8 @@ func (suite *operatorControllerTestSuite) TestDispatchUnfinishedStep() { for _, steps := range testSteps { // Create an operator op := NewTestOperator(1, epoch, OpRegion, steps...) - suite.True(controller.AddOperator(op)) - suite.Equal(1, stream.MsgLength()) + re.True(controller.AddOperator(op)) + re.Equal(1, stream.MsgLength()) // Create region2 which is cloned from the original region. // region2 has peer 2 in pending state, so the AddPeer step @@ -642,64 +655,64 @@ func (suite *operatorControllerTestSuite) TestDispatchUnfinishedStep() { }), core.WithIncConfVer(), ) - suite.NotNil(region2.GetPendingPeers()) + re.NotNil(region2.GetPendingPeers()) - suite.False(steps[0].IsFinish(region2)) + re.False(steps[0].IsFinish(region2)) controller.Dispatch(region2, DispatchFromHeartBeat, nil) // In this case, the conf version has been changed, but the // peer added is in pending state, the operator should not be // removed by the stale checker - suite.Equal(uint64(1), op.ConfVerChanged(region2)) - suite.NotNil(controller.GetOperator(1)) + re.Equal(uint64(1), op.ConfVerChanged(region2)) + re.NotNil(controller.GetOperator(1)) // The operator is valid yet, but the step should not be sent // again, because it is in pending state, so the message channel // should not be increased - suite.Equal(1, stream.MsgLength()) + re.Equal(1, stream.MsgLength()) // Finish the step by clearing the pending state region3 := region.Clone( core.WithAddPeer(&metapb.Peer{Id: 3, StoreId: 3, Role: metapb.PeerRole_Learner}), core.WithIncConfVer(), ) - suite.True(steps[0].IsFinish(region3)) + re.True(steps[0].IsFinish(region3)) controller.Dispatch(region3, DispatchFromHeartBeat, nil) - suite.Equal(uint64(1), op.ConfVerChanged(region3)) - suite.Equal(2, stream.MsgLength()) + re.Equal(uint64(1), op.ConfVerChanged(region3)) + re.Equal(2, stream.MsgLength()) region4 := region3.Clone( core.WithRole(3, metapb.PeerRole_Voter), core.WithIncConfVer(), ) - suite.True(steps[1].IsFinish(region4)) + re.True(steps[1].IsFinish(region4)) controller.Dispatch(region4, DispatchFromHeartBeat, nil) - suite.Equal(uint64(2), op.ConfVerChanged(region4)) - suite.Equal(3, stream.MsgLength()) + re.Equal(uint64(2), op.ConfVerChanged(region4)) + re.Equal(3, stream.MsgLength()) // Transfer leader region5 := region4.Clone( core.WithLeader(region4.GetStorePeer(3)), ) - suite.True(steps[2].IsFinish(region5)) + re.True(steps[2].IsFinish(region5)) controller.Dispatch(region5, DispatchFromHeartBeat, nil) - suite.Equal(uint64(2), op.ConfVerChanged(region5)) - suite.Equal(4, stream.MsgLength()) + re.Equal(uint64(2), op.ConfVerChanged(region5)) + re.Equal(4, stream.MsgLength()) // Remove peer region6 := region5.Clone( core.WithRemoveStorePeer(1), core.WithIncConfVer(), ) - suite.True(steps[3].IsFinish(region6)) + re.True(steps[3].IsFinish(region6)) controller.Dispatch(region6, DispatchFromHeartBeat, nil) - suite.Equal(uint64(3), op.ConfVerChanged(region6)) + re.Equal(uint64(3), op.ConfVerChanged(region6)) // The Operator has finished, so no message should be sent - suite.Equal(4, stream.MsgLength()) - suite.Nil(controller.GetOperator(1)) + re.Equal(4, stream.MsgLength()) + re.Nil(controller.GetOperator(1)) e := stream.Drain(4) - suite.NoError(e) + re.NoError(e) } } @@ -723,13 +736,14 @@ func newRegionInfo(id uint64, startKey, endKey string, size, keys int64, leader ) } -func (suite *operatorControllerTestSuite) checkRemoveOperatorSuccess(oc *Controller, op *Operator) { - suite.True(oc.RemoveOperator(op)) - suite.True(op.IsEnd()) - suite.Equal(op, oc.GetOperatorStatus(op.RegionID()).Operator) +func (suite *operatorControllerTestSuite) checkRemoveOperatorSuccess(re *require.Assertions, oc *Controller, op *Operator) { + re.True(oc.RemoveOperator(op)) + re.True(op.IsEnd()) + re.Equal(op, oc.GetOperatorStatus(op.RegionID()).Operator) } func (suite *operatorControllerTestSuite) TestAddWaitingOperator() { + re := suite.Require() opts := mockconfig.NewTestOptions() cluster := mockcluster.NewCluster(suite.ctx, opts) stream := hbstream.NewTestHeartbeatStreams(suite.ctx, cluster.ID, cluster, false /* no need to run */) @@ -746,8 +760,8 @@ func (suite *operatorControllerTestSuite) TestAddWaitingOperator() { StoreId: 2, } op, err := CreateAddPeerOperator("add-peer", cluster, region, peer, OpKind(0)) - suite.NoError(err) - suite.NotNil(op) + re.NoError(err) + re.NotNil(op) return op } @@ -758,21 +772,21 @@ func (suite *operatorControllerTestSuite) TestAddWaitingOperator() { batch = append(batch, addPeerOp(i)) } added := controller.AddWaitingOperator(batch...) - suite.Equal(int(cluster.GetSchedulerMaxWaitingOperator()), added) + re.Equal(int(cluster.GetSchedulerMaxWaitingOperator()), added) // test adding a batch of operators when some operators will get false in check // and remain operators can be added normally batch = append(batch, addPeerOp(cluster.GetSchedulerMaxWaitingOperator())) added = controller.AddWaitingOperator(batch...) - suite.Equal(1, added) + re.Equal(1, added) scheduleCfg := opts.GetScheduleConfig().Clone() scheduleCfg.SchedulerMaxWaitingOperator = 1 opts.SetScheduleConfig(scheduleCfg) batch = append(batch, addPeerOp(100)) added = controller.AddWaitingOperator(batch...) - suite.Equal(1, added) - suite.NotNil(controller.operators[uint64(100)]) + re.Equal(1, added) + re.NotNil(controller.operators[uint64(100)]) source := newRegionInfo(101, "1a", "1b", 1, 1, []uint64{101, 1}, []uint64{101, 1}) cluster.PutRegion(source) @@ -780,8 +794,8 @@ func (suite *operatorControllerTestSuite) TestAddWaitingOperator() { cluster.PutRegion(target) ops, err := CreateMergeRegionOperator("merge-region", cluster, source, target, OpMerge) - suite.NoError(err) - suite.Len(ops, 2) + re.NoError(err) + re.Len(ops, 2) // test with label schedule=deny labelerManager := cluster.GetRegionLabeler() @@ -792,13 +806,14 @@ func (suite *operatorControllerTestSuite) TestAddWaitingOperator() { Data: []interface{}{map[string]interface{}{"start_key": "1a", "end_key": "1b"}}, }) - suite.True(labelerManager.ScheduleDisabled(source)) + re.True(labelerManager.ScheduleDisabled(source)) // add operator should be success since it is not check in addWaitingOperator - suite.Equal(2, controller.AddWaitingOperator(ops...)) + re.Equal(2, controller.AddWaitingOperator(ops...)) } // issue #5279 func (suite *operatorControllerTestSuite) TestInvalidStoreId() { + re := suite.Require() opt := mockconfig.NewTestOptions() tc := mockcluster.NewCluster(suite.ctx, opt) stream := hbstream.NewTestHeartbeatStreams(suite.ctx, tc.ID, tc, false /* no need to run */) @@ -812,7 +827,7 @@ func (suite *operatorControllerTestSuite) TestInvalidStoreId() { RemovePeer{FromStore: 3, PeerID: 3, IsDownStore: false}, } op := NewTestOperator(1, &metapb.RegionEpoch{}, OpRegion, steps...) - suite.True(oc.addOperatorLocked(op)) + re.True(oc.addOperatorLocked(op)) // Although store 3 does not exist in PD, PD can also send op to TiKV. - suite.Equal(pdpb.OperatorStatus_RUNNING, oc.GetOperatorStatus(1).Status) + re.Equal(pdpb.OperatorStatus_RUNNING, oc.GetOperatorStatus(1).Status) } diff --git a/pkg/schedule/operator/operator_test.go b/pkg/schedule/operator/operator_test.go index c16d929f379..1b0ff8385bf 100644 --- a/pkg/schedule/operator/operator_test.go +++ b/pkg/schedule/operator/operator_test.go @@ -22,6 +22,7 @@ import ( "time" "github.com/pingcap/kvproto/pkg/metapb" + "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/core/constant" @@ -85,27 +86,29 @@ func (suite *operatorTestSuite) newTestRegion(regionID uint64, leaderPeer uint64 } func (suite *operatorTestSuite) TestOperatorStep() { + re := suite.Require() region := suite.newTestRegion(1, 1, [2]uint64{1, 1}, [2]uint64{2, 2}) - suite.False(TransferLeader{FromStore: 1, ToStore: 2}.IsFinish(region)) - suite.True(TransferLeader{FromStore: 2, ToStore: 1}.IsFinish(region)) - suite.False(AddPeer{ToStore: 3, PeerID: 3}.IsFinish(region)) - suite.True(AddPeer{ToStore: 1, PeerID: 1}.IsFinish(region)) - suite.False(RemovePeer{FromStore: 1}.IsFinish(region)) - suite.True(RemovePeer{FromStore: 3}.IsFinish(region)) + re.False(TransferLeader{FromStore: 1, ToStore: 2}.IsFinish(region)) + re.True(TransferLeader{FromStore: 2, ToStore: 1}.IsFinish(region)) + re.False(AddPeer{ToStore: 3, PeerID: 3}.IsFinish(region)) + re.True(AddPeer{ToStore: 1, PeerID: 1}.IsFinish(region)) + re.False(RemovePeer{FromStore: 1}.IsFinish(region)) + re.True(RemovePeer{FromStore: 3}.IsFinish(region)) } func (suite *operatorTestSuite) newTestOperator(regionID uint64, kind OpKind, steps ...OpStep) *Operator { return NewTestOperator(regionID, &metapb.RegionEpoch{}, kind, steps...) } -func (suite *operatorTestSuite) checkSteps(op *Operator, steps []OpStep) { - suite.Len(steps, op.Len()) +func (suite *operatorTestSuite) checkSteps(re *require.Assertions, op *Operator, steps []OpStep) { + re.Len(steps, op.Len()) for i := range steps { - suite.Equal(steps[i], op.Step(i)) + re.Equal(steps[i], op.Step(i)) } } func (suite *operatorTestSuite) TestOperator() { + re := suite.Require() region := suite.newTestRegion(1, 1, [2]uint64{1, 1}, [2]uint64{2, 2}) // addPeer1, transferLeader1, removePeer3 steps := []OpStep{ @@ -114,14 +117,14 @@ func (suite *operatorTestSuite) TestOperator() { RemovePeer{FromStore: 3}, } op := suite.newTestOperator(1, OpAdmin|OpLeader|OpRegion, steps...) - suite.Equal(constant.Urgent, op.GetPriorityLevel()) - suite.checkSteps(op, steps) + re.Equal(constant.Urgent, op.GetPriorityLevel()) + suite.checkSteps(re, op, steps) op.Start() - suite.Nil(op.Check(region)) + re.Nil(op.Check(region)) - suite.Equal(SUCCESS, op.Status()) + re.Equal(SUCCESS, op.Status()) op.SetStatusReachTime(STARTED, time.Now().Add(-SlowStepWaitTime-time.Second)) - suite.False(op.CheckTimeout()) + re.False(op.CheckTimeout()) // addPeer1, transferLeader1, removePeer2 steps = []OpStep{ @@ -130,38 +133,39 @@ func (suite *operatorTestSuite) TestOperator() { RemovePeer{FromStore: 2}, } op = suite.newTestOperator(1, OpLeader|OpRegion, steps...) - suite.Equal(constant.Medium, op.GetPriorityLevel()) - suite.checkSteps(op, steps) + re.Equal(constant.Medium, op.GetPriorityLevel()) + suite.checkSteps(re, op, steps) op.Start() - suite.Equal(RemovePeer{FromStore: 2}, op.Check(region)) - suite.Equal(int32(2), atomic.LoadInt32(&op.currentStep)) - suite.False(op.CheckTimeout()) + re.Equal(RemovePeer{FromStore: 2}, op.Check(region)) + re.Equal(int32(2), atomic.LoadInt32(&op.currentStep)) + re.False(op.CheckTimeout()) op.SetStatusReachTime(STARTED, op.GetStartTime().Add(-FastStepWaitTime-2*FastStepWaitTime+time.Second)) - suite.False(op.CheckTimeout()) + re.False(op.CheckTimeout()) op.SetStatusReachTime(STARTED, op.GetStartTime().Add(-SlowStepWaitTime-2*FastStepWaitTime-time.Second)) - suite.True(op.CheckTimeout()) + re.True(op.CheckTimeout()) res, err := json.Marshal(op) - suite.NoError(err) - suite.Len(res, len(op.String())+2) + re.NoError(err) + re.Len(res, len(op.String())+2) // check short timeout for transfer leader only operators. steps = []OpStep{TransferLeader{FromStore: 2, ToStore: 1}} op = suite.newTestOperator(1, OpLeader, steps...) op.Start() - suite.False(op.CheckTimeout()) + re.False(op.CheckTimeout()) op.SetStatusReachTime(STARTED, op.GetStartTime().Add(-FastStepWaitTime-time.Second)) - suite.True(op.CheckTimeout()) + re.True(op.CheckTimeout()) // case2: check timeout operator will return false not panic. op = NewTestOperator(1, &metapb.RegionEpoch{}, OpRegion, TransferLeader{ToStore: 1, FromStore: 4}) op.currentStep = 1 - suite.True(op.status.To(STARTED)) - suite.True(op.status.To(TIMEOUT)) - suite.False(op.CheckSuccess()) - suite.True(op.CheckTimeout()) + re.True(op.status.To(STARTED)) + re.True(op.status.To(TIMEOUT)) + re.False(op.CheckSuccess()) + re.True(op.CheckTimeout()) } func (suite *operatorTestSuite) TestInfluence() { + re := suite.Require() region := suite.newTestRegion(1, 1, [2]uint64{1, 1}, [2]uint64{2, 2}) opInfluence := OpInfluence{StoresInfluence: make(map[uint64]*StoreInfluence)} storeOpInfluence := opInfluence.StoresInfluence @@ -174,7 +178,7 @@ func (suite *operatorTestSuite) TestInfluence() { } AddLearner{ToStore: 2, PeerID: 2, SendStore: 1}.Influence(opInfluence, region) - suite.Equal(StoreInfluence{ + re.Equal(StoreInfluence{ LeaderSize: 0, LeaderCount: 0, RegionSize: 50, @@ -182,7 +186,7 @@ func (suite *operatorTestSuite) TestInfluence() { StepCost: map[storelimit.Type]int64{storelimit.AddPeer: 1000}, }, *storeOpInfluence[2]) - suite.Equal(StoreInfluence{ + re.Equal(StoreInfluence{ LeaderSize: 0, LeaderCount: 0, RegionSize: 0, @@ -192,7 +196,7 @@ func (suite *operatorTestSuite) TestInfluence() { resetInfluence() BecomeNonWitness{SendStore: 2, PeerID: 2, StoreID: 1}.Influence(opInfluence, region) - suite.Equal(StoreInfluence{ + re.Equal(StoreInfluence{ LeaderSize: 0, LeaderCount: 0, RegionSize: 50, @@ -201,7 +205,7 @@ func (suite *operatorTestSuite) TestInfluence() { StepCost: map[storelimit.Type]int64{storelimit.AddPeer: 1000}, }, *storeOpInfluence[1]) - suite.Equal(StoreInfluence{ + re.Equal(StoreInfluence{ LeaderSize: 0, LeaderCount: 0, RegionSize: 0, @@ -211,7 +215,7 @@ func (suite *operatorTestSuite) TestInfluence() { resetInfluence() AddPeer{ToStore: 2, PeerID: 2}.Influence(opInfluence, region) - suite.Equal(StoreInfluence{ + re.Equal(StoreInfluence{ LeaderSize: 0, LeaderCount: 0, RegionSize: 50, @@ -220,14 +224,14 @@ func (suite *operatorTestSuite) TestInfluence() { }, *storeOpInfluence[2]) TransferLeader{FromStore: 1, ToStore: 2}.Influence(opInfluence, region) - suite.Equal(StoreInfluence{ + re.Equal(StoreInfluence{ LeaderSize: -50, LeaderCount: -1, RegionSize: 0, RegionCount: 0, StepCost: nil, }, *storeOpInfluence[1]) - suite.Equal(StoreInfluence{ + re.Equal(StoreInfluence{ LeaderSize: 50, LeaderCount: 1, RegionSize: 50, @@ -236,14 +240,14 @@ func (suite *operatorTestSuite) TestInfluence() { }, *storeOpInfluence[2]) RemovePeer{FromStore: 1}.Influence(opInfluence, region) - suite.Equal(StoreInfluence{ + re.Equal(StoreInfluence{ LeaderSize: -50, LeaderCount: -1, RegionSize: -50, RegionCount: -1, StepCost: map[storelimit.Type]int64{storelimit.RemovePeer: 1000}, }, *storeOpInfluence[1]) - suite.Equal(StoreInfluence{ + re.Equal(StoreInfluence{ LeaderSize: 50, LeaderCount: 1, RegionSize: 50, @@ -252,14 +256,14 @@ func (suite *operatorTestSuite) TestInfluence() { }, *storeOpInfluence[2]) MergeRegion{IsPassive: false}.Influence(opInfluence, region) - suite.Equal(StoreInfluence{ + re.Equal(StoreInfluence{ LeaderSize: -50, LeaderCount: -1, RegionSize: -50, RegionCount: -1, StepCost: map[storelimit.Type]int64{storelimit.RemovePeer: 1000}, }, *storeOpInfluence[1]) - suite.Equal(StoreInfluence{ + re.Equal(StoreInfluence{ LeaderSize: 50, LeaderCount: 1, RegionSize: 50, @@ -268,14 +272,14 @@ func (suite *operatorTestSuite) TestInfluence() { }, *storeOpInfluence[2]) MergeRegion{IsPassive: true}.Influence(opInfluence, region) - suite.Equal(StoreInfluence{ + re.Equal(StoreInfluence{ LeaderSize: -50, LeaderCount: -2, RegionSize: -50, RegionCount: -2, StepCost: map[storelimit.Type]int64{storelimit.RemovePeer: 1000}, }, *storeOpInfluence[1]) - suite.Equal(StoreInfluence{ + re.Equal(StoreInfluence{ LeaderSize: 50, LeaderCount: 1, RegionSize: 50, @@ -285,18 +289,20 @@ func (suite *operatorTestSuite) TestInfluence() { } func (suite *operatorTestSuite) TestOperatorKind() { - suite.Equal("replica,leader", (OpLeader | OpReplica).String()) - suite.Equal("unknown", OpKind(0).String()) + re := suite.Require() + re.Equal("replica,leader", (OpLeader | OpReplica).String()) + re.Equal("unknown", OpKind(0).String()) k, err := ParseOperatorKind("region,leader") - suite.NoError(err) - suite.Equal(OpRegion|OpLeader, k) + re.NoError(err) + re.Equal(OpRegion|OpLeader, k) _, err = ParseOperatorKind("leader,region") - suite.NoError(err) + re.NoError(err) _, err = ParseOperatorKind("foobar") - suite.Error(err) + re.Error(err) } func (suite *operatorTestSuite) TestCheckSuccess() { + re := suite.Require() { steps := []OpStep{ AddPeer{ToStore: 1, PeerID: 1}, @@ -304,13 +310,13 @@ func (suite *operatorTestSuite) TestCheckSuccess() { RemovePeer{FromStore: 2}, } op := suite.newTestOperator(1, OpLeader|OpRegion, steps...) - suite.Equal(CREATED, op.Status()) - suite.False(op.CheckSuccess()) - suite.True(op.Start()) - suite.False(op.CheckSuccess()) + re.Equal(CREATED, op.Status()) + re.False(op.CheckSuccess()) + re.True(op.Start()) + re.False(op.CheckSuccess()) op.currentStep = int32(len(op.steps)) - suite.True(op.CheckSuccess()) - suite.True(op.CheckSuccess()) + re.True(op.CheckSuccess()) + re.True(op.CheckSuccess()) } { steps := []OpStep{ @@ -320,15 +326,16 @@ func (suite *operatorTestSuite) TestCheckSuccess() { } op := suite.newTestOperator(1, OpLeader|OpRegion, steps...) op.currentStep = int32(len(op.steps)) - suite.Equal(CREATED, op.Status()) - suite.False(op.CheckSuccess()) - suite.True(op.Start()) - suite.True(op.CheckSuccess()) - suite.True(op.CheckSuccess()) + re.Equal(CREATED, op.Status()) + re.False(op.CheckSuccess()) + re.True(op.Start()) + re.True(op.CheckSuccess()) + re.True(op.CheckSuccess()) } } func (suite *operatorTestSuite) TestCheckTimeout() { + re := suite.Require() { steps := []OpStep{ AddPeer{ToStore: 1, PeerID: 1}, @@ -336,11 +343,11 @@ func (suite *operatorTestSuite) TestCheckTimeout() { RemovePeer{FromStore: 2}, } op := suite.newTestOperator(1, OpLeader|OpRegion, steps...) - suite.Equal(CREATED, op.Status()) - suite.True(op.Start()) + re.Equal(CREATED, op.Status()) + re.True(op.Start()) op.currentStep = int32(len(op.steps)) - suite.False(op.CheckTimeout()) - suite.Equal(SUCCESS, op.Status()) + re.False(op.CheckTimeout()) + re.Equal(SUCCESS, op.Status()) } { steps := []OpStep{ @@ -349,44 +356,47 @@ func (suite *operatorTestSuite) TestCheckTimeout() { RemovePeer{FromStore: 2}, } op := suite.newTestOperator(1, OpLeader|OpRegion, steps...) - suite.Equal(CREATED, op.Status()) - suite.True(op.Start()) + re.Equal(CREATED, op.Status()) + re.True(op.Start()) op.currentStep = int32(len(op.steps)) op.SetStatusReachTime(STARTED, time.Now().Add(-SlowStepWaitTime)) - suite.False(op.CheckTimeout()) - suite.Equal(SUCCESS, op.Status()) + re.False(op.CheckTimeout()) + re.Equal(SUCCESS, op.Status()) } } func (suite *operatorTestSuite) TestStart() { + re := suite.Require() steps := []OpStep{ AddPeer{ToStore: 1, PeerID: 1}, TransferLeader{FromStore: 2, ToStore: 1}, RemovePeer{FromStore: 2}, } op := suite.newTestOperator(1, OpLeader|OpRegion, steps...) - suite.Equal(0, op.GetStartTime().Nanosecond()) - suite.Equal(CREATED, op.Status()) - suite.True(op.Start()) - suite.NotEqual(0, op.GetStartTime().Nanosecond()) - suite.Equal(STARTED, op.Status()) + re.Equal(0, op.GetStartTime().Nanosecond()) + re.Equal(CREATED, op.Status()) + re.True(op.Start()) + re.NotEqual(0, op.GetStartTime().Nanosecond()) + re.Equal(STARTED, op.Status()) } func (suite *operatorTestSuite) TestCheckExpired() { + re := suite.Require() steps := []OpStep{ AddPeer{ToStore: 1, PeerID: 1}, TransferLeader{FromStore: 2, ToStore: 1}, RemovePeer{FromStore: 2}, } op := suite.newTestOperator(1, OpLeader|OpRegion, steps...) - suite.False(op.CheckExpired()) - suite.Equal(CREATED, op.Status()) + re.False(op.CheckExpired()) + re.Equal(CREATED, op.Status()) op.SetStatusReachTime(CREATED, time.Now().Add(-OperatorExpireTime)) - suite.True(op.CheckExpired()) - suite.Equal(EXPIRED, op.Status()) + re.True(op.CheckExpired()) + re.Equal(EXPIRED, op.Status()) } func (suite *operatorTestSuite) TestCheck() { + re := suite.Require() { region := suite.newTestRegion(2, 2, [2]uint64{1, 1}, [2]uint64{2, 2}) steps := []OpStep{ @@ -395,14 +405,14 @@ func (suite *operatorTestSuite) TestCheck() { RemovePeer{FromStore: 2}, } op := suite.newTestOperator(2, OpLeader|OpRegion, steps...) - suite.True(op.Start()) - suite.NotNil(op.Check(region)) + re.True(op.Start()) + re.NotNil(op.Check(region)) - suite.Equal(STARTED, op.Status()) + re.Equal(STARTED, op.Status()) region = suite.newTestRegion(1, 1, [2]uint64{1, 1}) - suite.Nil(op.Check(region)) + re.Nil(op.Check(region)) - suite.Equal(SUCCESS, op.Status()) + re.Equal(SUCCESS, op.Status()) } { region := suite.newTestRegion(1, 1, [2]uint64{1, 1}, [2]uint64{2, 2}) @@ -412,12 +422,12 @@ func (suite *operatorTestSuite) TestCheck() { RemovePeer{FromStore: 2}, } op := suite.newTestOperator(1, OpLeader|OpRegion, steps...) - suite.True(op.Start()) - suite.NotNil(op.Check(region)) - suite.Equal(STARTED, op.Status()) + re.True(op.Start()) + re.NotNil(op.Check(region)) + re.Equal(STARTED, op.Status()) op.SetStatusReachTime(STARTED, time.Now().Add(-SlowStepWaitTime-2*FastStepWaitTime)) - suite.NotNil(op.Check(region)) - suite.Equal(TIMEOUT, op.Status()) + re.NotNil(op.Check(region)) + re.Equal(TIMEOUT, op.Status()) } { region := suite.newTestRegion(1, 1, [2]uint64{1, 1}, [2]uint64{2, 2}) @@ -427,17 +437,18 @@ func (suite *operatorTestSuite) TestCheck() { RemovePeer{FromStore: 2}, } op := suite.newTestOperator(1, OpLeader|OpRegion, steps...) - suite.True(op.Start()) - suite.NotNil(op.Check(region)) - suite.Equal(STARTED, op.Status()) + re.True(op.Start()) + re.NotNil(op.Check(region)) + re.Equal(STARTED, op.Status()) op.status.setTime(STARTED, time.Now().Add(-SlowStepWaitTime)) region = suite.newTestRegion(1, 1, [2]uint64{1, 1}) - suite.Nil(op.Check(region)) - suite.Equal(SUCCESS, op.Status()) + re.Nil(op.Check(region)) + re.Equal(SUCCESS, op.Status()) } } func (suite *operatorTestSuite) TestSchedulerKind() { + re := suite.Require() testData := []struct { op *Operator expect OpKind @@ -469,11 +480,12 @@ func (suite *operatorTestSuite) TestSchedulerKind() { }, } for _, v := range testData { - suite.Equal(v.expect, v.op.SchedulerKind()) + re.Equal(v.expect, v.op.SchedulerKind()) } } func (suite *operatorTestSuite) TestOpStepTimeout() { + re := suite.Require() testData := []struct { step []OpStep regionSize int64 @@ -515,16 +527,17 @@ func (suite *operatorTestSuite) TestOpStepTimeout() { for i, v := range testData { suite.T().Logf("case: %d", i) for _, step := range v.step { - suite.Equal(v.expect, step.Timeout(v.regionSize)) + re.Equal(v.expect, step.Timeout(v.regionSize)) } } } func (suite *operatorTestSuite) TestRecord() { + re := suite.Require() operator := suite.newTestOperator(1, OpLeader, AddLearner{ToStore: 1, PeerID: 1}, RemovePeer{FromStore: 1, PeerID: 1}) now := time.Now() time.Sleep(time.Second) ob := operator.Record(now) - suite.Equal(now, ob.FinishTime) - suite.Greater(ob.duration.Seconds(), time.Second.Seconds()) + re.Equal(now, ob.FinishTime) + re.Greater(ob.duration.Seconds(), time.Second.Seconds()) } diff --git a/pkg/schedule/operator/step_test.go b/pkg/schedule/operator/step_test.go index 4865180a8bb..f362d988f89 100644 --- a/pkg/schedule/operator/step_test.go +++ b/pkg/schedule/operator/step_test.go @@ -19,6 +19,7 @@ import ( "testing" "github.com/pingcap/kvproto/pkg/metapb" + "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/mock/mockcluster" @@ -39,7 +40,7 @@ type testCase struct { Peers []*metapb.Peer // first is leader ConfVerChanged uint64 IsFinish bool - CheckInProgress func(err error, msgAndArgs ...interface{}) bool + CheckInProgress func(err error, msgAndArgs ...interface{}) } func (suite *operatorStepTestSuite) SetupTest() { @@ -53,6 +54,7 @@ func (suite *operatorStepTestSuite) SetupTest() { } func (suite *operatorStepTestSuite) TestTransferLeader() { + re := suite.Require() step := TransferLeader{FromStore: 1, ToStore: 2} testCases := []testCase{ { @@ -63,7 +65,7 @@ func (suite *operatorStepTestSuite) TestTransferLeader() { }, 0, false, - suite.NoError, + re.NoError, }, { []*metapb.Peer{ @@ -73,7 +75,7 @@ func (suite *operatorStepTestSuite) TestTransferLeader() { }, 0, true, - suite.NoError, + re.NoError, }, { []*metapb.Peer{ @@ -83,10 +85,10 @@ func (suite *operatorStepTestSuite) TestTransferLeader() { }, 0, false, - suite.NoError, + re.NoError, }, } - suite.check(step, "transfer leader from store 1 to store 2", testCases) + suite.check(re, step, "transfer leader from store 1 to store 2", testCases) step = TransferLeader{FromStore: 1, ToStore: 9} // 9 is down testCases = []testCase{ @@ -98,13 +100,14 @@ func (suite *operatorStepTestSuite) TestTransferLeader() { }, 0, false, - suite.Error, + re.Error, }, } - suite.check(step, "transfer leader from store 1 to store 9", testCases) + suite.check(re, step, "transfer leader from store 1 to store 9", testCases) } func (suite *operatorStepTestSuite) TestAddPeer() { + re := suite.Require() step := AddPeer{ToStore: 2, PeerID: 2} testCases := []testCase{ { @@ -113,7 +116,7 @@ func (suite *operatorStepTestSuite) TestAddPeer() { }, 0, false, - suite.NoError, + re.NoError, }, { []*metapb.Peer{ @@ -122,10 +125,10 @@ func (suite *operatorStepTestSuite) TestAddPeer() { }, 1, true, - suite.NoError, + re.NoError, }, } - suite.check(step, "add peer 2 on store 2", testCases) + suite.check(re, step, "add peer 2 on store 2", testCases) step = AddPeer{ToStore: 9, PeerID: 9} testCases = []testCase{ @@ -135,13 +138,14 @@ func (suite *operatorStepTestSuite) TestAddPeer() { }, 0, false, - suite.Error, + re.Error, }, } - suite.check(step, "add peer 9 on store 9", testCases) + suite.check(re, step, "add peer 9 on store 9", testCases) } func (suite *operatorStepTestSuite) TestAddLearner() { + re := suite.Require() step := AddLearner{ToStore: 2, PeerID: 2} testCases := []testCase{ { @@ -150,7 +154,7 @@ func (suite *operatorStepTestSuite) TestAddLearner() { }, 0, false, - suite.NoError, + re.NoError, }, { []*metapb.Peer{ @@ -159,10 +163,10 @@ func (suite *operatorStepTestSuite) TestAddLearner() { }, 1, true, - suite.NoError, + re.NoError, }, } - suite.check(step, "add learner peer 2 on store 2", testCases) + suite.check(re, step, "add learner peer 2 on store 2", testCases) step = AddLearner{ToStore: 9, PeerID: 9} testCases = []testCase{ @@ -172,13 +176,14 @@ func (suite *operatorStepTestSuite) TestAddLearner() { }, 0, false, - suite.Error, + re.Error, }, } - suite.check(step, "add learner peer 9 on store 9", testCases) + suite.check(re, step, "add learner peer 9 on store 9", testCases) } func (suite *operatorStepTestSuite) TestChangePeerV2Enter() { + re := suite.Require() cpe := ChangePeerV2Enter{ PromoteLearners: []PromoteLearner{{PeerID: 3, ToStore: 3}, {PeerID: 4, ToStore: 4}}, DemoteVoters: []DemoteVoter{{PeerID: 1, ToStore: 1}, {PeerID: 2, ToStore: 2}}, @@ -193,7 +198,7 @@ func (suite *operatorStepTestSuite) TestChangePeerV2Enter() { }, 0, false, - suite.NoError, + re.NoError, }, { // after step []*metapb.Peer{ @@ -204,7 +209,7 @@ func (suite *operatorStepTestSuite) TestChangePeerV2Enter() { }, 4, true, - suite.NoError, + re.NoError, }, { // miss peer id []*metapb.Peer{ @@ -215,7 +220,7 @@ func (suite *operatorStepTestSuite) TestChangePeerV2Enter() { }, 0, false, - suite.Error, + re.Error, }, { // miss store id []*metapb.Peer{ @@ -226,7 +231,7 @@ func (suite *operatorStepTestSuite) TestChangePeerV2Enter() { }, 0, false, - suite.Error, + re.Error, }, { // miss peer id []*metapb.Peer{ @@ -237,7 +242,7 @@ func (suite *operatorStepTestSuite) TestChangePeerV2Enter() { }, 0, false, - suite.Error, + re.Error, }, { // change is not atomic []*metapb.Peer{ @@ -248,7 +253,7 @@ func (suite *operatorStepTestSuite) TestChangePeerV2Enter() { }, 0, false, - suite.Error, + re.Error, }, { // change is not atomic []*metapb.Peer{ @@ -259,7 +264,7 @@ func (suite *operatorStepTestSuite) TestChangePeerV2Enter() { }, 0, false, - suite.Error, + re.Error, }, { // there are other peers in the joint state []*metapb.Peer{ @@ -271,7 +276,7 @@ func (suite *operatorStepTestSuite) TestChangePeerV2Enter() { }, 4, true, - suite.Error, + re.Error, }, { // there are other peers in the joint state []*metapb.Peer{ @@ -284,16 +289,17 @@ func (suite *operatorStepTestSuite) TestChangePeerV2Enter() { }, 0, false, - suite.Error, + re.Error, }, } desc := "use joint consensus, " + "promote learner peer 3 on store 3 to voter, promote learner peer 4 on store 4 to voter, " + "demote voter peer 1 on store 1 to learner, demote voter peer 2 on store 2 to learner" - suite.check(cpe, desc, testCases) + suite.check(re, cpe, desc, testCases) } func (suite *operatorStepTestSuite) TestChangePeerV2EnterWithSingleChange() { + re := suite.Require() cpe := ChangePeerV2Enter{ PromoteLearners: []PromoteLearner{{PeerID: 3, ToStore: 3}}, } @@ -306,7 +312,7 @@ func (suite *operatorStepTestSuite) TestChangePeerV2EnterWithSingleChange() { }, 0, false, - suite.NoError, + re.NoError, }, { // after step []*metapb.Peer{ @@ -316,7 +322,7 @@ func (suite *operatorStepTestSuite) TestChangePeerV2EnterWithSingleChange() { }, 1, true, - suite.NoError, + re.NoError, }, { // after step (direct) []*metapb.Peer{ @@ -326,7 +332,7 @@ func (suite *operatorStepTestSuite) TestChangePeerV2EnterWithSingleChange() { }, 1, true, - suite.NoError, + re.NoError, }, { // error role []*metapb.Peer{ @@ -336,11 +342,11 @@ func (suite *operatorStepTestSuite) TestChangePeerV2EnterWithSingleChange() { }, 0, false, - suite.Error, + re.Error, }, } desc := "use joint consensus, promote learner peer 3 on store 3 to voter" - suite.check(cpe, desc, testCases) + suite.check(re, cpe, desc, testCases) cpe = ChangePeerV2Enter{ DemoteVoters: []DemoteVoter{{PeerID: 3, ToStore: 3}}, @@ -354,7 +360,7 @@ func (suite *operatorStepTestSuite) TestChangePeerV2EnterWithSingleChange() { }, 0, false, - suite.NoError, + re.NoError, }, { // after step []*metapb.Peer{ @@ -364,7 +370,7 @@ func (suite *operatorStepTestSuite) TestChangePeerV2EnterWithSingleChange() { }, 1, true, - suite.NoError, + re.NoError, }, { // after step (direct) []*metapb.Peer{ @@ -374,7 +380,7 @@ func (suite *operatorStepTestSuite) TestChangePeerV2EnterWithSingleChange() { }, 1, true, - suite.NoError, + re.NoError, }, { // demote and remove peer []*metapb.Peer{ @@ -383,7 +389,7 @@ func (suite *operatorStepTestSuite) TestChangePeerV2EnterWithSingleChange() { }, 1, // correct calculation is required false, - suite.Error, + re.Error, }, { // error role []*metapb.Peer{ @@ -393,14 +399,15 @@ func (suite *operatorStepTestSuite) TestChangePeerV2EnterWithSingleChange() { }, 0, false, - suite.Error, + re.Error, }, } desc = "use joint consensus, demote voter peer 3 on store 3 to learner" - suite.check(cpe, desc, testCases) + suite.check(re, cpe, desc, testCases) } func (suite *operatorStepTestSuite) TestChangePeerV2Leave() { + re := suite.Require() cpl := ChangePeerV2Leave{ PromoteLearners: []PromoteLearner{{PeerID: 3, ToStore: 3}, {PeerID: 4, ToStore: 4}}, DemoteVoters: []DemoteVoter{{PeerID: 1, ToStore: 1}, {PeerID: 2, ToStore: 2}}, @@ -415,7 +422,7 @@ func (suite *operatorStepTestSuite) TestChangePeerV2Leave() { }, 0, false, - suite.NoError, + re.NoError, }, { // after step []*metapb.Peer{ @@ -426,7 +433,7 @@ func (suite *operatorStepTestSuite) TestChangePeerV2Leave() { }, 4, true, - suite.NoError, + re.NoError, }, { // miss peer id []*metapb.Peer{ @@ -437,7 +444,7 @@ func (suite *operatorStepTestSuite) TestChangePeerV2Leave() { }, 0, false, - suite.Error, + re.Error, }, { // miss store id []*metapb.Peer{ @@ -448,7 +455,7 @@ func (suite *operatorStepTestSuite) TestChangePeerV2Leave() { }, 0, false, - suite.Error, + re.Error, }, { // miss peer id []*metapb.Peer{ @@ -459,7 +466,7 @@ func (suite *operatorStepTestSuite) TestChangePeerV2Leave() { }, 0, false, - suite.Error, + re.Error, }, { // change is not atomic []*metapb.Peer{ @@ -470,7 +477,7 @@ func (suite *operatorStepTestSuite) TestChangePeerV2Leave() { }, 0, false, - suite.Error, + re.Error, }, { // change is not atomic []*metapb.Peer{ @@ -481,7 +488,7 @@ func (suite *operatorStepTestSuite) TestChangePeerV2Leave() { }, 0, false, - suite.Error, + re.Error, }, { // there are other peers in the joint state []*metapb.Peer{ @@ -493,7 +500,7 @@ func (suite *operatorStepTestSuite) TestChangePeerV2Leave() { }, 0, false, - suite.Error, + re.Error, }, { // there are other peers in the joint state []*metapb.Peer{ @@ -506,7 +513,7 @@ func (suite *operatorStepTestSuite) TestChangePeerV2Leave() { }, 4, false, - suite.Error, + re.Error, }, { // demote leader []*metapb.Peer{ @@ -517,16 +524,17 @@ func (suite *operatorStepTestSuite) TestChangePeerV2Leave() { }, 0, false, - suite.Error, + re.Error, }, } desc := "leave joint state, " + "promote learner peer 3 on store 3 to voter, promote learner peer 4 on store 4 to voter, " + "demote voter peer 1 on store 1 to learner, demote voter peer 2 on store 2 to learner" - suite.check(cpl, desc, testCases) + suite.check(re, cpl, desc, testCases) } func (suite *operatorStepTestSuite) TestSwitchToWitness() { + re := suite.Require() step := BecomeWitness{StoreID: 2, PeerID: 2} testCases := []testCase{ { @@ -536,7 +544,7 @@ func (suite *operatorStepTestSuite) TestSwitchToWitness() { }, 0, false, - suite.NoError, + re.NoError, }, { []*metapb.Peer{ @@ -545,7 +553,7 @@ func (suite *operatorStepTestSuite) TestSwitchToWitness() { }, 0, false, - suite.NoError, + re.NoError, }, { []*metapb.Peer{ @@ -554,18 +562,18 @@ func (suite *operatorStepTestSuite) TestSwitchToWitness() { }, 1, true, - suite.NoError, + re.NoError, }, } - suite.check(step, "switch peer 2 on store 2 to witness", testCases) + suite.check(re, step, "switch peer 2 on store 2 to witness", testCases) } -func (suite *operatorStepTestSuite) check(step OpStep, desc string, testCases []testCase) { - suite.Equal(desc, step.String()) +func (suite *operatorStepTestSuite) check(re *require.Assertions, step OpStep, desc string, testCases []testCase) { + re.Equal(desc, step.String()) for _, testCase := range testCases { region := core.NewRegionInfo(&metapb.Region{Id: 1, Peers: testCase.Peers}, testCase.Peers[0]) - suite.Equal(testCase.ConfVerChanged, step.ConfVerChanged(region)) - suite.Equal(testCase.IsFinish, step.IsFinish(region)) + re.Equal(testCase.ConfVerChanged, step.ConfVerChanged(region)) + re.Equal(testCase.IsFinish, step.IsFinish(region)) err := step.CheckInProgress(suite.cluster.GetBasicCluster(), suite.cluster.GetSharedConfig(), region) testCase.CheckInProgress(err) _ = step.GetCmd(region, true) @@ -574,7 +582,7 @@ func (suite *operatorStepTestSuite) check(step OpStep, desc string, testCases [] // Ref https://github.com/tikv/pd/issues/5788 pendingPeers := region.GetLearners() region = region.Clone(core.WithPendingPeers(pendingPeers)) - suite.Equal(testCase.IsFinish, step.IsFinish(region)) + re.Equal(testCase.IsFinish, step.IsFinish(region)) } } } diff --git a/pkg/schedule/rangelist/range_list_test.go b/pkg/schedule/rangelist/range_list_test.go index 062c6a34d27..d8517705153 100644 --- a/pkg/schedule/rangelist/range_list_test.go +++ b/pkg/schedule/rangelist/range_list_test.go @@ -85,7 +85,7 @@ func TestRangeList2(t *testing.T) { } rl := b.Build() - re.Equal(len(expectKeys), rl.Len()) + re.Len(expectKeys, rl.Len()) for i := 0; i < rl.Len(); i++ { key, data := rl.Get(i) re.Equal(expectKeys[i], key) diff --git a/pkg/schedule/schedulers/balance_test.go b/pkg/schedule/schedulers/balance_test.go index dafe810b2b7..68332d7067e 100644 --- a/pkg/schedule/schedulers/balance_test.go +++ b/pkg/schedule/schedulers/balance_test.go @@ -259,6 +259,7 @@ func (suite *balanceLeaderSchedulerTestSuite) dryRun() []plan.Plan { } func (suite *balanceLeaderSchedulerTestSuite) TestBalanceLimit() { + re := suite.Require() suite.tc.SetTolerantSizeRatio(2.5) // Stores: 1 2 3 4 // Leaders: 1 0 0 0 @@ -268,13 +269,13 @@ func (suite *balanceLeaderSchedulerTestSuite) TestBalanceLimit() { suite.tc.AddLeaderStore(3, 0) suite.tc.AddLeaderStore(4, 0) suite.tc.AddLeaderRegion(1, 1, 2, 3, 4) - suite.Empty(suite.schedule()) + re.Empty(suite.schedule()) // Stores: 1 2 3 4 // Leaders: 16 0 0 0 // Region1: L F F F suite.tc.UpdateLeaderCount(1, 16) - suite.NotEmpty(suite.schedule()) + re.NotEmpty(suite.schedule()) // Stores: 1 2 3 4 // Leaders: 7 8 9 10 @@ -284,20 +285,21 @@ func (suite *balanceLeaderSchedulerTestSuite) TestBalanceLimit() { suite.tc.UpdateLeaderCount(3, 9) suite.tc.UpdateLeaderCount(4, 10) suite.tc.AddLeaderRegion(1, 4, 1, 2, 3) - suite.Empty(suite.schedule()) + re.Empty(suite.schedule()) plans := suite.dryRun() - suite.NotEmpty(plans) - suite.Equal(3, plans[0].GetStep()) - suite.Equal(plan.StatusStoreScoreDisallowed, int(plans[0].GetStatus().StatusCode)) + re.NotEmpty(plans) + re.Equal(3, plans[0].GetStep()) + re.Equal(plan.StatusStoreScoreDisallowed, int(plans[0].GetStatus().StatusCode)) // Stores: 1 2 3 4 // Leaders: 7 8 9 16 // Region1: F F F L suite.tc.UpdateLeaderCount(4, 16) - suite.NotEmpty(suite.schedule()) + re.NotEmpty(suite.schedule()) } func (suite *balanceLeaderSchedulerTestSuite) TestBalanceLeaderSchedulePolicy() { + re := suite.Require() // Stores: 1 2 3 4 // Leader Count: 10 10 10 10 // Leader Size : 10000 100 100 100 @@ -307,18 +309,19 @@ func (suite *balanceLeaderSchedulerTestSuite) TestBalanceLeaderSchedulePolicy() suite.tc.AddLeaderStore(3, 10, 100*units.MiB) suite.tc.AddLeaderStore(4, 10, 100*units.MiB) suite.tc.AddLeaderRegion(1, 1, 2, 3, 4) - suite.Equal(constant.ByCount.String(), suite.tc.GetScheduleConfig().LeaderSchedulePolicy) // default by count - suite.Empty(suite.schedule()) + re.Equal(constant.ByCount.String(), suite.tc.GetScheduleConfig().LeaderSchedulePolicy) // default by count + re.Empty(suite.schedule()) plans := suite.dryRun() - suite.NotEmpty(plans) - suite.Equal(3, plans[0].GetStep()) - suite.Equal(plan.StatusStoreScoreDisallowed, int(plans[0].GetStatus().StatusCode)) + re.NotEmpty(plans) + re.Equal(3, plans[0].GetStep()) + re.Equal(plan.StatusStoreScoreDisallowed, int(plans[0].GetStatus().StatusCode)) suite.tc.SetLeaderSchedulePolicy(constant.BySize.String()) - suite.NotEmpty(suite.schedule()) + re.NotEmpty(suite.schedule()) } func (suite *balanceLeaderSchedulerTestSuite) TestBalanceLeaderTolerantRatio() { + re := suite.Require() suite.tc.SetTolerantSizeRatio(2.5) // test schedule leader by count, with tolerantSizeRatio=2.5 // Stores: 1 2 3 4 @@ -330,17 +333,18 @@ func (suite *balanceLeaderSchedulerTestSuite) TestBalanceLeaderTolerantRatio() { suite.tc.AddLeaderStore(3, 10, 100) suite.tc.AddLeaderStore(4, 10, 100) suite.tc.AddLeaderRegion(1, 1, 2, 3, 4) - suite.Equal(constant.ByCount.String(), suite.tc.GetScheduleConfig().LeaderSchedulePolicy) // default by count - suite.Empty(suite.schedule()) - suite.Equal(14, suite.tc.GetStore(1).GetLeaderCount()) + re.Equal(constant.ByCount.String(), suite.tc.GetScheduleConfig().LeaderSchedulePolicy) // default by count + re.Empty(suite.schedule()) + re.Equal(14, suite.tc.GetStore(1).GetLeaderCount()) suite.tc.AddLeaderStore(1, 15, 100) - suite.Equal(15, suite.tc.GetStore(1).GetLeaderCount()) - suite.NotEmpty(suite.schedule()) + re.Equal(15, suite.tc.GetStore(1).GetLeaderCount()) + re.NotEmpty(suite.schedule()) suite.tc.SetTolerantSizeRatio(6) // (15-10)<6 - suite.Empty(suite.schedule()) + re.Empty(suite.schedule()) } func (suite *balanceLeaderSchedulerTestSuite) TestScheduleWithOpInfluence() { + re := suite.Require() suite.tc.SetTolerantSizeRatio(2.5) // Stores: 1 2 3 4 // Leaders: 7 8 9 14 @@ -351,15 +355,15 @@ func (suite *balanceLeaderSchedulerTestSuite) TestScheduleWithOpInfluence() { suite.tc.AddLeaderStore(4, 14) suite.tc.AddLeaderRegion(1, 4, 1, 2, 3) op := suite.schedule()[0] - suite.NotNil(op) + re.NotNil(op) suite.oc.SetOperator(op) // After considering the scheduled operator, leaders of store1 and store4 are 8 // and 13 respectively. As the `TolerantSizeRatio` is 2.5, `shouldBalance` // returns false when leader difference is not greater than 5. - suite.Equal(constant.ByCount.String(), suite.tc.GetScheduleConfig().LeaderSchedulePolicy) // default by count - suite.NotEmpty(suite.schedule()) + re.Equal(constant.ByCount.String(), suite.tc.GetScheduleConfig().LeaderSchedulePolicy) // default by count + re.NotEmpty(suite.schedule()) suite.tc.SetLeaderSchedulePolicy(constant.BySize.String()) - suite.Empty(suite.schedule()) + re.Empty(suite.schedule()) // Stores: 1 2 3 4 // Leaders: 8 8 9 13 @@ -369,10 +373,11 @@ func (suite *balanceLeaderSchedulerTestSuite) TestScheduleWithOpInfluence() { suite.tc.UpdateLeaderCount(3, 9) suite.tc.UpdateLeaderCount(4, 13) suite.tc.AddLeaderRegion(1, 4, 1, 2, 3) - suite.Empty(suite.schedule()) + re.Empty(suite.schedule()) } func (suite *balanceLeaderSchedulerTestSuite) TestTransferLeaderOut() { + re := suite.Require() // Stores: 1 2 3 4 // Leaders: 7 8 9 12 suite.tc.AddLeaderStore(1, 7) @@ -399,18 +404,19 @@ func (suite *balanceLeaderSchedulerTestSuite) TestTransferLeaderOut() { suite.oc.SetOperator(op) regions[op.RegionID()] = struct{}{} tr := op.Step(0).(operator.TransferLeader) - suite.Equal(uint64(4), tr.FromStore) + re.Equal(uint64(4), tr.FromStore) targets[tr.ToStore]-- } } } - suite.Len(regions, 3) + re.Len(regions, 3) for _, count := range targets { - suite.Zero(count) + re.Zero(count) } } func (suite *balanceLeaderSchedulerTestSuite) TestBalanceFilter() { + re := suite.Require() // Stores: 1 2 3 4 // Leaders: 1 2 3 16 // Region1: F F F L @@ -420,35 +426,36 @@ func (suite *balanceLeaderSchedulerTestSuite) TestBalanceFilter() { suite.tc.AddLeaderStore(4, 16) suite.tc.AddLeaderRegion(1, 4, 1, 2, 3) - operatorutil.CheckTransferLeader(suite.Require(), suite.schedule()[0], operator.OpKind(0), 4, 1) + operatorutil.CheckTransferLeader(re, suite.schedule()[0], operator.OpKind(0), 4, 1) // Test stateFilter. // if store 4 is offline, we should consider it // because it still provides services suite.tc.SetStoreOffline(4) - operatorutil.CheckTransferLeader(suite.Require(), suite.schedule()[0], operator.OpKind(0), 4, 1) + operatorutil.CheckTransferLeader(re, suite.schedule()[0], operator.OpKind(0), 4, 1) // If store 1 is down, it will be filtered, // store 2 becomes the store with least leaders. suite.tc.SetStoreDown(1) - operatorutil.CheckTransferLeader(suite.Require(), suite.schedule()[0], operator.OpKind(0), 4, 2) + operatorutil.CheckTransferLeader(re, suite.schedule()[0], operator.OpKind(0), 4, 2) plans := suite.dryRun() - suite.NotEmpty(plans) - suite.Equal(0, plans[0].GetStep()) - suite.Equal(plan.StatusStoreDown, int(plans[0].GetStatus().StatusCode)) - suite.Equal(uint64(1), plans[0].GetResource(0)) + re.NotEmpty(plans) + re.Equal(0, plans[0].GetStep()) + re.Equal(plan.StatusStoreDown, int(plans[0].GetStatus().StatusCode)) + re.Equal(uint64(1), plans[0].GetResource(0)) // Test healthFilter. // If store 2 is busy, it will be filtered, // store 3 becomes the store with least leaders. suite.tc.SetStoreBusy(2, true) - operatorutil.CheckTransferLeader(suite.Require(), suite.schedule()[0], operator.OpKind(0), 4, 3) + operatorutil.CheckTransferLeader(re, suite.schedule()[0], operator.OpKind(0), 4, 3) // Test disconnectFilter. // If store 3 is disconnected, no operator can be created. suite.tc.SetStoreDisconnect(3) - suite.Empty(suite.schedule()) + re.Empty(suite.schedule()) } func (suite *balanceLeaderSchedulerTestSuite) TestLeaderWeight() { + re := suite.Require() // Stores: 1 2 3 4 // Leaders: 10 10 10 10 // Weight: 0.5 0.9 1 2 @@ -462,12 +469,13 @@ func (suite *balanceLeaderSchedulerTestSuite) TestLeaderWeight() { suite.tc.UpdateStoreLeaderWeight(3, 1) suite.tc.UpdateStoreLeaderWeight(4, 2) suite.tc.AddLeaderRegion(1, 1, 2, 3, 4) - operatorutil.CheckTransferLeader(suite.Require(), suite.schedule()[0], operator.OpKind(0), 1, 4) + operatorutil.CheckTransferLeader(re, suite.schedule()[0], operator.OpKind(0), 1, 4) suite.tc.UpdateLeaderCount(4, 30) - operatorutil.CheckTransferLeader(suite.Require(), suite.schedule()[0], operator.OpKind(0), 1, 3) + operatorutil.CheckTransferLeader(re, suite.schedule()[0], operator.OpKind(0), 1, 3) } func (suite *balanceLeaderSchedulerTestSuite) TestBalancePolicy() { + re := suite.Require() // Stores: 1 2 3 4 // LeaderCount: 20 66 6 20 // LeaderSize: 66 20 20 6 @@ -478,12 +486,13 @@ func (suite *balanceLeaderSchedulerTestSuite) TestBalancePolicy() { suite.tc.AddLeaderRegion(1, 2, 1, 3, 4) suite.tc.AddLeaderRegion(2, 1, 2, 3, 4) suite.tc.SetLeaderSchedulePolicy("count") - operatorutil.CheckTransferLeader(suite.Require(), suite.schedule()[0], operator.OpKind(0), 2, 3) + operatorutil.CheckTransferLeader(re, suite.schedule()[0], operator.OpKind(0), 2, 3) suite.tc.SetLeaderSchedulePolicy("size") - operatorutil.CheckTransferLeader(suite.Require(), suite.schedule()[0], operator.OpKind(0), 1, 4) + operatorutil.CheckTransferLeader(re, suite.schedule()[0], operator.OpKind(0), 1, 4) } func (suite *balanceLeaderSchedulerTestSuite) TestBalanceSelector() { + re := suite.Require() // Stores: 1 2 3 4 // Leaders: 1 2 3 16 // Region1: - F F L @@ -496,7 +505,7 @@ func (suite *balanceLeaderSchedulerTestSuite) TestBalanceSelector() { suite.tc.AddLeaderRegion(2, 3, 1, 2) // store4 has max leader score, store1 has min leader score. // The scheduler try to move a leader out of 16 first. - operatorutil.CheckTransferLeader(suite.Require(), suite.schedule()[0], operator.OpKind(0), 4, 2) + operatorutil.CheckTransferLeader(re, suite.schedule()[0], operator.OpKind(0), 4, 2) // Stores: 1 2 3 4 // Leaders: 1 14 15 16 @@ -505,7 +514,7 @@ func (suite *balanceLeaderSchedulerTestSuite) TestBalanceSelector() { suite.tc.UpdateLeaderCount(2, 14) suite.tc.UpdateLeaderCount(3, 15) // Cannot move leader out of store4, move a leader into store1. - operatorutil.CheckTransferLeader(suite.Require(), suite.schedule()[0], operator.OpKind(0), 3, 1) + operatorutil.CheckTransferLeader(re, suite.schedule()[0], operator.OpKind(0), 3, 1) // Stores: 1 2 3 4 // Leaders: 1 2 15 16 @@ -515,7 +524,7 @@ func (suite *balanceLeaderSchedulerTestSuite) TestBalanceSelector() { suite.tc.AddLeaderRegion(1, 3, 2, 4) suite.tc.AddLeaderRegion(2, 1, 2, 3) // No leader in store16, no follower in store1. Now source and target are store3 and store2. - operatorutil.CheckTransferLeader(suite.Require(), suite.schedule()[0], operator.OpKind(0), 3, 2) + operatorutil.CheckTransferLeader(re, suite.schedule()[0], operator.OpKind(0), 3, 2) // Stores: 1 2 3 4 // Leaders: 9 10 10 11 @@ -527,7 +536,7 @@ func (suite *balanceLeaderSchedulerTestSuite) TestBalanceSelector() { suite.tc.AddLeaderRegion(1, 4, 2, 3) suite.tc.AddLeaderRegion(2, 1, 2, 3) // The cluster is balanced. - suite.Empty(suite.schedule()) + re.Empty(suite.schedule()) // store3's leader drops: // Stores: 1 2 3 4 @@ -538,7 +547,7 @@ func (suite *balanceLeaderSchedulerTestSuite) TestBalanceSelector() { suite.tc.AddLeaderStore(2, 13) suite.tc.AddLeaderStore(3, 0) suite.tc.AddLeaderStore(4, 16) - operatorutil.CheckTransferLeader(suite.Require(), suite.schedule()[0], operator.OpKind(0), 4, 3) + operatorutil.CheckTransferLeader(re, suite.schedule()[0], operator.OpKind(0), 4, 3) } type balanceLeaderRangeSchedulerTestSuite struct { @@ -608,6 +617,7 @@ func (suite *balanceLeaderRangeSchedulerTestSuite) TestSingleRangeBalance() { } func (suite *balanceLeaderRangeSchedulerTestSuite) TestMultiRangeBalance() { + re := suite.Require() // Stores: 1 2 3 4 // Leaders: 10 10 10 10 // Weight: 0.5 0.9 1 2 @@ -621,34 +631,35 @@ func (suite *balanceLeaderRangeSchedulerTestSuite) TestMultiRangeBalance() { suite.tc.UpdateStoreLeaderWeight(4, 2) suite.tc.AddLeaderRegionWithRange(1, "a", "g", 1, 2, 3, 4) lb, err := CreateScheduler(BalanceLeaderType, suite.oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(BalanceLeaderType, []string{"", "g", "o", "t"})) - suite.NoError(err) + re.NoError(err) ops, _ := lb.Schedule(suite.tc, false) - suite.Equal(uint64(1), ops[0].RegionID()) + re.Equal(uint64(1), ops[0].RegionID()) r := suite.tc.GetRegion(1) suite.tc.RemoveRegion(r) suite.tc.RemoveRegionFromSubTree(r) suite.tc.AddLeaderRegionWithRange(2, "p", "r", 1, 2, 3, 4) - suite.NoError(err) + re.NoError(err) ops, _ = lb.Schedule(suite.tc, false) - suite.Equal(uint64(2), ops[0].RegionID()) + re.Equal(uint64(2), ops[0].RegionID()) r = suite.tc.GetRegion(2) suite.tc.RemoveRegion(r) suite.tc.RemoveRegionFromSubTree(r) suite.tc.AddLeaderRegionWithRange(3, "u", "w", 1, 2, 3, 4) - suite.NoError(err) + re.NoError(err) ops, _ = lb.Schedule(suite.tc, false) - suite.Empty(ops) + re.Empty(ops) r = suite.tc.GetRegion(3) suite.tc.RemoveRegion(r) suite.tc.RemoveRegionFromSubTree(r) suite.tc.AddLeaderRegionWithRange(4, "", "", 1, 2, 3, 4) - suite.NoError(err) + re.NoError(err) ops, _ = lb.Schedule(suite.tc, false) - suite.Empty(ops) + re.Empty(ops) } func (suite *balanceLeaderRangeSchedulerTestSuite) TestBatchBalance() { + re := suite.Require() suite.tc.AddLeaderStore(1, 100) suite.tc.AddLeaderStore(2, 0) suite.tc.AddLeaderStore(3, 0) @@ -659,9 +670,9 @@ func (suite *balanceLeaderRangeSchedulerTestSuite) TestBatchBalance() { suite.tc.AddLeaderRegionWithRange(uint64(102), "102a", "102z", 1, 2, 3) suite.tc.AddLeaderRegionWithRange(uint64(103), "103a", "103z", 4, 5, 6) lb, err := CreateScheduler(BalanceLeaderType, suite.oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(BalanceLeaderType, []string{"", ""})) - suite.NoError(err) + re.NoError(err) ops, _ := lb.Schedule(suite.tc, false) - suite.Len(ops, 2) + re.Len(ops, 2) for i := 1; i <= 50; i++ { suite.tc.AddLeaderRegionWithRange(uint64(i), fmt.Sprintf("%da", i), fmt.Sprintf("%dz", i), 1, 2, 3) } @@ -670,15 +681,16 @@ func (suite *balanceLeaderRangeSchedulerTestSuite) TestBatchBalance() { } suite.tc.AddLeaderRegionWithRange(uint64(101), "101a", "101z", 5, 4, 3) ops, _ = lb.Schedule(suite.tc, false) - suite.Len(ops, 4) + re.Len(ops, 4) regions := make(map[uint64]struct{}) for _, op := range ops { regions[op.RegionID()] = struct{}{} } - suite.Len(regions, 4) + re.Len(regions, 4) } func (suite *balanceLeaderRangeSchedulerTestSuite) TestReSortStores() { + re := suite.Require() suite.tc.AddLeaderStore(1, 104) suite.tc.AddLeaderStore(2, 0) suite.tc.AddLeaderStore(3, 0) @@ -699,23 +711,23 @@ func (suite *balanceLeaderRangeSchedulerTestSuite) TestReSortStores() { cs := newCandidateStores(append(candidateStores, stores...), false, getScore) // in candidate,the order stores:1(104),5(100),4(100),6,3,2 // store 4 should in pos 2 - suite.Equal(2, cs.binarySearch(stores[3])) + re.Equal(2, cs.binarySearch(stores[3])) // store 1 should in pos 0 store1 := stores[0] - suite.Zero(cs.binarySearch(store1)) + re.Zero(cs.binarySearch(store1)) deltaMap[store1.GetID()] = -1 // store 1 cs.resortStoreWithPos(0) // store 1 should still in pos 0. - suite.Equal(uint64(1), cs.stores[0].GetID()) + re.Equal(uint64(1), cs.stores[0].GetID()) curIndex := cs.binarySearch(store1) - suite.Zero(curIndex) + re.Zero(curIndex) deltaMap[1] = -4 // store 1 update the scores to 104-4=100 // the order stores should be:5(100),4(100),1(100),6,3,2 cs.resortStoreWithPos(curIndex) - suite.Equal(uint64(1), cs.stores[2].GetID()) - suite.Equal(2, cs.binarySearch(store1)) + re.Equal(uint64(1), cs.stores[2].GetID()) + re.Equal(2, cs.binarySearch(store1)) // the top store is : 5(100) topStore := cs.stores[0] topStorePos := cs.binarySearch(topStore) @@ -723,18 +735,18 @@ func (suite *balanceLeaderRangeSchedulerTestSuite) TestReSortStores() { cs.resortStoreWithPos(topStorePos) // after recorder, the order stores should be: 4(100),1(100),5(99),6,3,2 - suite.Equal(uint64(1), cs.stores[1].GetID()) - suite.Equal(1, cs.binarySearch(store1)) - suite.Equal(topStore.GetID(), cs.stores[2].GetID()) - suite.Equal(2, cs.binarySearch(topStore)) + re.Equal(uint64(1), cs.stores[1].GetID()) + re.Equal(1, cs.binarySearch(store1)) + re.Equal(topStore.GetID(), cs.stores[2].GetID()) + re.Equal(2, cs.binarySearch(topStore)) bottomStore := cs.stores[5] deltaMap[bottomStore.GetID()] = 4 cs.resortStoreWithPos(5) // the order stores should be: 4(100),1(100),5(99),2(5),6,3 - suite.Equal(bottomStore.GetID(), cs.stores[3].GetID()) - suite.Equal(3, cs.binarySearch(bottomStore)) + re.Equal(bottomStore.GetID(), cs.stores[3].GetID()) + re.Equal(3, cs.binarySearch(bottomStore)) } func TestBalanceRegionSchedule1(t *testing.T) { @@ -778,9 +790,9 @@ func checkBalanceRegionSchedule1(re *require.Assertions, enablePlacementRules bo re.Len(plans, 101) re.Empty(ops) if enablePlacementRules { - re.Equal(int(plans[1].GetStatus().StatusCode), plan.StatusRegionNotMatchRule) + re.Equal(plan.StatusRegionNotMatchRule, int(plans[1].GetStatus().StatusCode)) } else { - re.Equal(int(plans[1].GetStatus().StatusCode), plan.StatusRegionNotReplicated) + re.Equal(plan.StatusRegionNotReplicated, int(plans[1].GetStatus().StatusCode)) } tc.SetStoreOffline(1) diff --git a/pkg/schedule/schedulers/balance_witness_test.go b/pkg/schedule/schedulers/balance_witness_test.go index 9bde7e33438..2b6723a5172 100644 --- a/pkg/schedule/schedulers/balance_witness_test.go +++ b/pkg/schedule/schedulers/balance_witness_test.go @@ -40,6 +40,7 @@ type balanceWitnessSchedulerTestSuite struct { } func (suite *balanceWitnessSchedulerTestSuite) SetupTest() { + re := suite.Require() suite.cancel, suite.conf, suite.tc, suite.oc = prepareSchedulersTest() suite.tc.RuleManager.SetRules([]*placement.Rule{ { @@ -50,7 +51,7 @@ func (suite *balanceWitnessSchedulerTestSuite) SetupTest() { }, }) lb, err := CreateScheduler(BalanceWitnessType, suite.oc, storage.NewStorageWithMemoryBackend(), ConfigSliceDecoder(BalanceWitnessType, []string{"", ""}), nil) - suite.NoError(err) + re.NoError(err) suite.lb = lb } @@ -64,6 +65,7 @@ func (suite *balanceWitnessSchedulerTestSuite) schedule() []*operator.Operator { } func (suite *balanceWitnessSchedulerTestSuite) TestScheduleWithOpInfluence() { + re := suite.Require() suite.tc.SetTolerantSizeRatio(2.5) // Stores: 1 2 3 4 // Witnesses: 7 8 9 14 @@ -74,12 +76,12 @@ func (suite *balanceWitnessSchedulerTestSuite) TestScheduleWithOpInfluence() { suite.tc.AddWitnessStore(4, 14) suite.tc.AddLeaderRegionWithWitness(1, 3, []uint64{1, 2, 4}, 4) op := suite.schedule()[0] - suite.NotNil(op) + re.NotNil(op) suite.oc.SetOperator(op) // After considering the scheduled operator, witnesses of store1 and store2 are 8 // and 13 respectively. As the `TolerantSizeRatio` is 2.5, `shouldBalance` // returns false when witness difference is not greater than 5. - suite.NotEmpty(suite.schedule()) + re.NotEmpty(suite.schedule()) // Stores: 1 2 3 4 // Witness: 8 8 9 13 @@ -89,10 +91,11 @@ func (suite *balanceWitnessSchedulerTestSuite) TestScheduleWithOpInfluence() { suite.tc.UpdateWitnessCount(3, 9) suite.tc.UpdateWitnessCount(4, 13) suite.tc.AddLeaderRegionWithWitness(1, 3, []uint64{1, 2, 4}, 4) - suite.Empty(suite.schedule()) + re.Empty(suite.schedule()) } func (suite *balanceWitnessSchedulerTestSuite) TestTransferWitnessOut() { + re := suite.Require() // Stores: 1 2 3 4 // Witnesses: 7 8 9 12 suite.tc.AddWitnessStore(1, 7) @@ -120,13 +123,13 @@ func (suite *balanceWitnessSchedulerTestSuite) TestTransferWitnessOut() { regions[op.RegionID()] = struct{}{} from := op.Step(0).(operator.ChangePeerV2Enter).DemoteVoters[0].ToStore to := op.Step(1).(operator.BatchSwitchWitness).ToWitnesses[0].StoreID - suite.Equal(from, uint64(4)) + re.Equal(uint64(4), from) targets[to]-- } } } - suite.Len(regions, 3) + re.Len(regions, 3) for _, count := range targets { - suite.Zero(count) + re.Zero(count) } } diff --git a/pkg/schedule/schedulers/evict_slow_store_test.go b/pkg/schedule/schedulers/evict_slow_store_test.go index 11cd69e60f7..6ed9764ba7c 100644 --- a/pkg/schedule/schedulers/evict_slow_store_test.go +++ b/pkg/schedule/schedulers/evict_slow_store_test.go @@ -43,6 +43,7 @@ func TestEvictSlowStoreTestSuite(t *testing.T) { } func (suite *evictSlowStoreTestSuite) SetupTest() { + re := suite.Require() suite.cancel, _, suite.tc, suite.oc = prepareSchedulersTest() // Add stores 1, 2 @@ -57,9 +58,9 @@ func (suite *evictSlowStoreTestSuite) SetupTest() { storage := storage.NewStorageWithMemoryBackend() var err error suite.es, err = CreateScheduler(EvictSlowStoreType, suite.oc, storage, ConfigSliceDecoder(EvictSlowStoreType, []string{}), nil) - suite.NoError(err) + re.NoError(err) suite.bs, err = CreateScheduler(BalanceLeaderType, suite.oc, storage, ConfigSliceDecoder(BalanceLeaderType, []string{}), nil) - suite.NoError(err) + re.NoError(err) } func (suite *evictSlowStoreTestSuite) TearDownTest() { @@ -67,41 +68,42 @@ func (suite *evictSlowStoreTestSuite) TearDownTest() { } func (suite *evictSlowStoreTestSuite) TestEvictSlowStore() { - suite.NoError(failpoint.Enable("github.com/tikv/pd/pkg/schedule/schedulers/transientRecoveryGap", "return(true)")) + re := suite.Require() + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/schedule/schedulers/transientRecoveryGap", "return(true)")) storeInfo := suite.tc.GetStore(1) newStoreInfo := storeInfo.Clone(func(store *core.StoreInfo) { store.GetStoreStats().SlowScore = 100 }) suite.tc.PutStore(newStoreInfo) - suite.True(suite.es.IsScheduleAllowed(suite.tc)) + re.True(suite.es.IsScheduleAllowed(suite.tc)) // Add evict leader scheduler to store 1 ops, _ := suite.es.Schedule(suite.tc, false) - operatorutil.CheckMultiTargetTransferLeader(suite.Require(), ops[0], operator.OpLeader, 1, []uint64{2}) - suite.Equal(EvictSlowStoreType, ops[0].Desc()) + operatorutil.CheckMultiTargetTransferLeader(re, ops[0], operator.OpLeader, 1, []uint64{2}) + re.Equal(EvictSlowStoreType, ops[0].Desc()) // Cannot balance leaders to store 1 ops, _ = suite.bs.Schedule(suite.tc, false) - suite.Empty(ops) + re.Empty(ops) newStoreInfo = storeInfo.Clone(func(store *core.StoreInfo) { store.GetStoreStats().SlowScore = 0 }) suite.tc.PutStore(newStoreInfo) // Evict leader scheduler of store 1 should be removed, then leader can be balanced to store 1 ops, _ = suite.es.Schedule(suite.tc, false) - suite.Empty(ops) + re.Empty(ops) ops, _ = suite.bs.Schedule(suite.tc, false) - operatorutil.CheckTransferLeader(suite.Require(), ops[0], operator.OpLeader, 2, 1) + operatorutil.CheckTransferLeader(re, ops[0], operator.OpLeader, 2, 1) // no slow store need to evict. ops, _ = suite.es.Schedule(suite.tc, false) - suite.Empty(ops) + re.Empty(ops) es2, ok := suite.es.(*evictSlowStoreScheduler) - suite.True(ok) - suite.Zero(es2.conf.evictStore()) + re.True(ok) + re.Zero(es2.conf.evictStore()) // check the value from storage. sches, vs, err := es2.conf.storage.LoadAllSchedulerConfigs() - suite.NoError(err) + re.NoError(err) valueStr := "" for id, sche := range sches { if strings.EqualFold(sche, EvictSlowStoreName) { @@ -111,41 +113,43 @@ func (suite *evictSlowStoreTestSuite) TestEvictSlowStore() { var persistValue evictSlowStoreSchedulerConfig err = json.Unmarshal([]byte(valueStr), &persistValue) - suite.NoError(err) - suite.Equal(es2.conf.EvictedStores, persistValue.EvictedStores) - suite.Zero(persistValue.evictStore()) - suite.True(persistValue.readyForRecovery()) - suite.NoError(failpoint.Disable("github.com/tikv/pd/pkg/schedule/schedulers/transientRecoveryGap")) + re.NoError(err) + re.Equal(es2.conf.EvictedStores, persistValue.EvictedStores) + re.Zero(persistValue.evictStore()) + re.True(persistValue.readyForRecovery()) + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/schedule/schedulers/transientRecoveryGap")) } func (suite *evictSlowStoreTestSuite) TestEvictSlowStorePrepare() { + re := suite.Require() es2, ok := suite.es.(*evictSlowStoreScheduler) - suite.True(ok) - suite.Zero(es2.conf.evictStore()) + re.True(ok) + re.Zero(es2.conf.evictStore()) // prepare with no evict store. suite.es.PrepareConfig(suite.tc) es2.conf.setStoreAndPersist(1) - suite.Equal(uint64(1), es2.conf.evictStore()) - suite.False(es2.conf.readyForRecovery()) + re.Equal(uint64(1), es2.conf.evictStore()) + re.False(es2.conf.readyForRecovery()) // prepare with evict store. suite.es.PrepareConfig(suite.tc) } func (suite *evictSlowStoreTestSuite) TestEvictSlowStorePersistFail() { + re := suite.Require() persisFail := "github.com/tikv/pd/pkg/schedule/schedulers/persistFail" - suite.NoError(failpoint.Enable(persisFail, "return(true)")) + re.NoError(failpoint.Enable(persisFail, "return(true)")) storeInfo := suite.tc.GetStore(1) newStoreInfo := storeInfo.Clone(func(store *core.StoreInfo) { store.GetStoreStats().SlowScore = 100 }) suite.tc.PutStore(newStoreInfo) - suite.True(suite.es.IsScheduleAllowed(suite.tc)) + re.True(suite.es.IsScheduleAllowed(suite.tc)) // Add evict leader scheduler to store 1 ops, _ := suite.es.Schedule(suite.tc, false) - suite.Empty(ops) - suite.NoError(failpoint.Disable(persisFail)) + re.Empty(ops) + re.NoError(failpoint.Disable(persisFail)) ops, _ = suite.es.Schedule(suite.tc, false) - suite.NotEmpty(ops) + re.NotEmpty(ops) } diff --git a/pkg/schedule/schedulers/evict_slow_trend_test.go b/pkg/schedule/schedulers/evict_slow_trend_test.go index aed41e83ecd..834ef337639 100644 --- a/pkg/schedule/schedulers/evict_slow_trend_test.go +++ b/pkg/schedule/schedulers/evict_slow_trend_test.go @@ -45,6 +45,7 @@ func TestEvictSlowTrendTestSuite(t *testing.T) { } func (suite *evictSlowTrendTestSuite) SetupTest() { + re := suite.Require() suite.cancel, _, suite.tc, suite.oc = prepareSchedulersTest() suite.tc.AddLeaderStore(1, 10) @@ -71,9 +72,9 @@ func (suite *evictSlowTrendTestSuite) SetupTest() { storage := storage.NewStorageWithMemoryBackend() var err error suite.es, err = CreateScheduler(EvictSlowTrendType, suite.oc, storage, ConfigSliceDecoder(EvictSlowTrendType, []string{})) - suite.NoError(err) + re.NoError(err) suite.bs, err = CreateScheduler(BalanceLeaderType, suite.oc, storage, ConfigSliceDecoder(BalanceLeaderType, []string{})) - suite.NoError(err) + re.NoError(err) } func (suite *evictSlowTrendTestSuite) TearDownTest() { @@ -81,51 +82,53 @@ func (suite *evictSlowTrendTestSuite) TearDownTest() { } func (suite *evictSlowTrendTestSuite) TestEvictSlowTrendBasicFuncs() { + re := suite.Require() es2, ok := suite.es.(*evictSlowTrendScheduler) - suite.True(ok) + re.True(ok) - suite.Equal(es2.conf.evictedStore(), uint64(0)) - suite.Equal(es2.conf.candidate(), uint64(0)) + re.Zero(es2.conf.evictedStore()) + re.Zero(es2.conf.candidate()) // Test capture store 1 store := suite.tc.GetStore(1) es2.conf.captureCandidate(store.GetID()) lastCapturedCandidate := es2.conf.lastCapturedCandidate() - suite.Equal(*lastCapturedCandidate, es2.conf.evictCandidate) - suite.Equal(es2.conf.candidateCapturedSecs(), uint64(0)) - suite.Equal(es2.conf.lastCandidateCapturedSecs(), uint64(0)) - suite.False(es2.conf.readyForRecovery()) + re.Equal(*lastCapturedCandidate, es2.conf.evictCandidate) + re.Zero(es2.conf.candidateCapturedSecs()) + re.Zero(es2.conf.lastCandidateCapturedSecs()) + re.False(es2.conf.readyForRecovery()) recoverTS := lastCapturedCandidate.recoverTS - suite.True(recoverTS.After(lastCapturedCandidate.captureTS)) + re.True(recoverTS.After(lastCapturedCandidate.captureTS)) // Pop captured store 1 and mark it has recovered. time.Sleep(50 * time.Millisecond) - suite.Equal(es2.conf.popCandidate(true), store.GetID()) - suite.Equal(slowCandidate{}, es2.conf.evictCandidate) + re.Equal(es2.conf.popCandidate(true), store.GetID()) + re.Equal(slowCandidate{}, es2.conf.evictCandidate) es2.conf.markCandidateRecovered() lastCapturedCandidate = es2.conf.lastCapturedCandidate() - suite.Greater(lastCapturedCandidate.recoverTS.Compare(recoverTS), 0) - suite.Equal(lastCapturedCandidate.storeID, store.GetID()) + re.Greater(lastCapturedCandidate.recoverTS.Compare(recoverTS), 0) + re.Equal(lastCapturedCandidate.storeID, store.GetID()) // Test capture another store 2 store = suite.tc.GetStore(2) es2.conf.captureCandidate(store.GetID()) lastCapturedCandidate = es2.conf.lastCapturedCandidate() - suite.Equal(lastCapturedCandidate.storeID, uint64(1)) - suite.Equal(es2.conf.candidate(), store.GetID()) - suite.Equal(es2.conf.candidateCapturedSecs(), uint64(0)) + re.Equal(uint64(1), lastCapturedCandidate.storeID) + re.Equal(es2.conf.candidate(), store.GetID()) + re.Zero(es2.conf.candidateCapturedSecs()) - suite.Equal(es2.conf.popCandidate(false), store.GetID()) - suite.Equal(lastCapturedCandidate.storeID, uint64(1)) + re.Equal(es2.conf.popCandidate(false), store.GetID()) + re.Equal(uint64(1), lastCapturedCandidate.storeID) } func (suite *evictSlowTrendTestSuite) TestEvictSlowTrend() { + re := suite.Require() es2, ok := suite.es.(*evictSlowTrendScheduler) - suite.True(ok) - suite.NoError(failpoint.Enable("github.com/tikv/pd/pkg/schedule/schedulers/transientRecoveryGap", "return(true)")) + re.True(ok) + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/schedule/schedulers/transientRecoveryGap", "return(true)")) // Set store-1 to slow status, generate evict candidate - suite.Equal(es2.conf.evictedStore(), uint64(0)) - suite.Equal(es2.conf.candidate(), uint64(0)) + re.Zero(es2.conf.evictedStore()) + re.Zero(es2.conf.candidate()) storeInfo := suite.tc.GetStore(1) newStoreInfo := storeInfo.Clone(func(store *core.StoreInfo) { store.GetStoreStats().SlowTrend = &pdpb.SlowTrend{ @@ -136,11 +139,11 @@ func (suite *evictSlowTrendTestSuite) TestEvictSlowTrend() { } }) suite.tc.PutStore(newStoreInfo) - suite.True(suite.es.IsScheduleAllowed(suite.tc)) + re.True(suite.es.IsScheduleAllowed(suite.tc)) ops, _ := suite.es.Schedule(suite.tc, false) - suite.Empty(ops) - suite.Equal(es2.conf.candidate(), uint64(1)) - suite.Equal(es2.conf.evictedStore(), uint64(0)) + re.Empty(ops) + re.Equal(uint64(1), es2.conf.candidate()) + re.Zero(es2.conf.evictedStore()) // Update other stores' heartbeat-ts, do evicting for storeID := uint64(2); storeID <= uint64(3); storeID++ { @@ -151,13 +154,13 @@ func (suite *evictSlowTrendTestSuite) TestEvictSlowTrend() { suite.tc.PutStore(newStoreInfo) } ops, _ = suite.es.Schedule(suite.tc, false) - operatorutil.CheckMultiTargetTransferLeader(suite.Require(), ops[0], operator.OpLeader, 1, []uint64{2, 3}) - suite.Equal(EvictSlowTrendType, ops[0].Desc()) - suite.Equal(es2.conf.candidate(), uint64(0)) - suite.Equal(es2.conf.evictedStore(), uint64(1)) + operatorutil.CheckMultiTargetTransferLeader(re, ops[0], operator.OpLeader, 1, []uint64{2, 3}) + re.Equal(EvictSlowTrendType, ops[0].Desc()) + re.Zero(es2.conf.candidate()) + re.Equal(uint64(1), es2.conf.evictedStore()) // Cannot balance leaders to store 1 ops, _ = suite.bs.Schedule(suite.tc, false) - suite.Empty(ops) + re.Empty(ops) // Set store-1 to normal status newStoreInfo = storeInfo.Clone(func(store *core.StoreInfo) { @@ -171,19 +174,19 @@ func (suite *evictSlowTrendTestSuite) TestEvictSlowTrend() { suite.tc.PutStore(newStoreInfo) // Evict leader scheduler of store 1 should be removed, then leaders should be balanced from store-3 to store-1 ops, _ = suite.es.Schedule(suite.tc, false) - suite.Empty(ops) - suite.Zero(es2.conf.evictedStore()) + re.Empty(ops) + re.Zero(es2.conf.evictedStore()) ops, _ = suite.bs.Schedule(suite.tc, false) - operatorutil.CheckTransferLeader(suite.Require(), ops[0], operator.OpLeader, 3, 1) + operatorutil.CheckTransferLeader(re, ops[0], operator.OpLeader, 3, 1) // no slow store need to evict. ops, _ = suite.es.Schedule(suite.tc, false) - suite.Empty(ops) - suite.Zero(es2.conf.evictedStore()) + re.Empty(ops) + re.Zero(es2.conf.evictedStore()) // check the value from storage. sches, vs, err := es2.conf.storage.LoadAllSchedulerConfigs() - suite.NoError(err) + re.NoError(err) valueStr := "" for id, sche := range sches { if strings.EqualFold(sche, EvictSlowTrendName) { @@ -193,20 +196,21 @@ func (suite *evictSlowTrendTestSuite) TestEvictSlowTrend() { var persistValue evictSlowTrendSchedulerConfig err = json.Unmarshal([]byte(valueStr), &persistValue) - suite.NoError(err) - suite.Equal(es2.conf.EvictedStores, persistValue.EvictedStores) - suite.Zero(persistValue.evictedStore()) - suite.NoError(failpoint.Disable("github.com/tikv/pd/pkg/schedule/schedulers/transientRecoveryGap")) + re.NoError(err) + re.Equal(es2.conf.EvictedStores, persistValue.EvictedStores) + re.Zero(persistValue.evictedStore()) + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/schedule/schedulers/transientRecoveryGap")) } func (suite *evictSlowTrendTestSuite) TestEvictSlowTrendV2() { + re := suite.Require() es2, ok := suite.es.(*evictSlowTrendScheduler) - suite.True(ok) - suite.NoError(failpoint.Enable("github.com/tikv/pd/pkg/schedule/schedulers/transientRecoveryGap", "return(true)")) - suite.NoError(failpoint.Enable("github.com/tikv/pd/pkg/schedule/schedulers/mockRaftKV2", "return(true)")) + re.True(ok) + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/schedule/schedulers/transientRecoveryGap", "return(true)")) + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/schedule/schedulers/mockRaftKV2", "return(true)")) - suite.Equal(es2.conf.evictedStore(), uint64(0)) - suite.Equal(es2.conf.candidate(), uint64(0)) + re.Zero(es2.conf.evictedStore()) + re.Zero(es2.conf.candidate()) // Set store-1 to slow status, generate slow candidate but under faster limit storeInfo := suite.tc.GetStore(1) newStoreInfo := storeInfo.Clone(func(store *core.StoreInfo) { @@ -218,17 +222,17 @@ func (suite *evictSlowTrendTestSuite) TestEvictSlowTrendV2() { } }) suite.tc.PutStore(newStoreInfo) - suite.True(suite.es.IsScheduleAllowed(suite.tc)) + re.True(suite.es.IsScheduleAllowed(suite.tc)) ops, _ := suite.es.Schedule(suite.tc, false) - suite.Empty(ops) - suite.Equal(es2.conf.evictedStore(), uint64(0)) - suite.Equal(es2.conf.candidate(), uint64(1)) - suite.Equal(es2.conf.lastCandidateCapturedSecs(), uint64(0)) + re.Empty(ops) + re.Zero(es2.conf.evictedStore()) + re.Equal(uint64(1), es2.conf.candidate()) + re.Zero(es2.conf.lastCandidateCapturedSecs()) // Rescheduling to make it filtered by the related faster judgement. ops, _ = suite.es.Schedule(suite.tc, false) - suite.Empty(ops) - suite.Equal(es2.conf.evictedStore(), uint64(0)) - suite.Equal(es2.conf.candidate(), uint64(0)) + re.Empty(ops) + re.Zero(es2.conf.evictedStore()) + re.Zero(es2.conf.candidate()) // Set store-1 to slow status as network-io delays storeInfo = suite.tc.GetStore(1) @@ -241,25 +245,26 @@ func (suite *evictSlowTrendTestSuite) TestEvictSlowTrendV2() { } }) suite.tc.PutStore(newStoreInfo) - suite.True(suite.es.IsScheduleAllowed(suite.tc)) + re.True(suite.es.IsScheduleAllowed(suite.tc)) ops, _ = suite.es.Schedule(suite.tc, false) - suite.Empty(ops) - suite.Equal(es2.conf.evictedStore(), uint64(0)) - suite.Equal(es2.conf.lastCandidateCapturedSecs(), uint64(0)) + re.Empty(ops) + re.Zero(es2.conf.evictedStore()) + re.Zero(es2.conf.lastCandidateCapturedSecs()) - suite.NoError(failpoint.Disable("github.com/tikv/pd/pkg/schedule/schedulers/mockRaftKV2")) - suite.NoError(failpoint.Disable("github.com/tikv/pd/pkg/schedule/schedulers/transientRecoveryGap")) + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/schedule/schedulers/mockRaftKV2")) + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/schedule/schedulers/transientRecoveryGap")) } func (suite *evictSlowTrendTestSuite) TestEvictSlowTrendPrepare() { + re := suite.Require() es2, ok := suite.es.(*evictSlowTrendScheduler) - suite.True(ok) - suite.Zero(es2.conf.evictedStore()) + re.True(ok) + re.Zero(es2.conf.evictedStore()) // prepare with no evict store. suite.es.PrepareConfig(suite.tc) es2.conf.setStoreAndPersist(1) - suite.Equal(uint64(1), es2.conf.evictedStore()) + re.Equal(uint64(1), es2.conf.evictedStore()) // prepare with evict store. suite.es.PrepareConfig(suite.tc) } diff --git a/pkg/schedule/splitter/region_splitter_test.go b/pkg/schedule/splitter/region_splitter_test.go index 8753d8bf2ec..ebb8b225a9b 100644 --- a/pkg/schedule/splitter/region_splitter_test.go +++ b/pkg/schedule/splitter/region_splitter_test.go @@ -81,6 +81,7 @@ func (suite *regionSplitterTestSuite) TearDownTest() { } func (suite *regionSplitterTestSuite) TestRegionSplitter() { + re := suite.Require() opt := mockconfig.NewTestOptions() opt.SetPlacementRuleEnabled(false) tc := mockcluster.NewCluster(suite.ctx, opt) @@ -90,24 +91,25 @@ func (suite *regionSplitterTestSuite) TestRegionSplitter() { newRegions := map[uint64]struct{}{} // assert success failureKeys := splitter.splitRegionsByKeys(suite.ctx, [][]byte{[]byte("fff"), []byte("ggg")}, newRegions) - suite.Empty(failureKeys) - suite.Len(newRegions, 2) + re.Empty(failureKeys) + re.Len(newRegions, 2) percentage, newRegionsID := splitter.SplitRegions(suite.ctx, [][]byte{[]byte("fff"), []byte("ggg")}, 1) - suite.Equal(100, percentage) - suite.Len(newRegionsID, 2) + re.Equal(100, percentage) + re.Len(newRegionsID, 2) // assert out of range newRegions = map[uint64]struct{}{} failureKeys = splitter.splitRegionsByKeys(suite.ctx, [][]byte{[]byte("aaa"), []byte("bbb")}, newRegions) - suite.Len(failureKeys, 2) - suite.Empty(newRegions) + re.Len(failureKeys, 2) + re.Empty(newRegions) percentage, newRegionsID = splitter.SplitRegions(suite.ctx, [][]byte{[]byte("aaa"), []byte("bbb")}, 1) - suite.Equal(0, percentage) - suite.Empty(newRegionsID) + re.Equal(0, percentage) + re.Empty(newRegionsID) } func (suite *regionSplitterTestSuite) TestGroupKeysByRegion() { + re := suite.Require() opt := mockconfig.NewTestOptions() opt.SetPlacementRuleEnabled(false) tc := mockcluster.NewCluster(suite.ctx, opt) @@ -122,15 +124,15 @@ func (suite *regionSplitterTestSuite) TestGroupKeysByRegion() { []byte("fff"), []byte("zzz"), }) - suite.Len(groupKeys, 2) + re.Len(groupKeys, 2) for k, v := range groupKeys { switch k { case uint64(1): - suite.Len(v.keys, 1) - suite.Equal([]byte("bbb"), v.keys[0]) + re.Len(v.keys, 1) + re.Equal([]byte("bbb"), v.keys[0]) case uint64(2): - suite.Len(v.keys, 1) - suite.Equal([]byte("ddd"), v.keys[0]) + re.Len(v.keys, 1) + re.Equal([]byte("ddd"), v.keys[0]) } } } diff --git a/pkg/utils/etcdutil/etcdutil_test.go b/pkg/utils/etcdutil/etcdutil_test.go index d8b38e7b045..d415d2d1873 100644 --- a/pkg/utils/etcdutil/etcdutil_test.go +++ b/pkg/utils/etcdutil/etcdutil_test.go @@ -426,7 +426,7 @@ func (suite *loopWatcherTestSuite) TestLoadWithoutKey() { re.NoError(err) // although no key, watcher returns no error cache.RLock() defer cache.RUnlock() - suite.Empty(cache.data) + re.Empty(cache.data) } func (suite *loopWatcherTestSuite) TestCallBack() { @@ -544,7 +544,7 @@ func (suite *loopWatcherTestSuite) TestWatcherBreak() { data string }{} checkCache := func(expect string) { - testutil.Eventually(suite.Require(), func() bool { + testutil.Eventually(re, func() bool { cache.RLock() defer cache.RUnlock() return cache.data == expect @@ -631,6 +631,7 @@ func (suite *loopWatcherTestSuite) TestWatcherBreak() { } func (suite *loopWatcherTestSuite) TestWatcherRequestProgress() { + re := suite.Require() checkWatcherRequestProgress := func(injectWatchChanBlock bool) { fname := testutil.InitTempFileLogger("debug") defer os.RemoveAll(fname) @@ -655,14 +656,14 @@ func (suite *loopWatcherTestSuite) TestWatcherRequestProgress() { if injectWatchChanBlock { failpoint.Enable("github.com/tikv/pd/pkg/utils/etcdutil/watchChanBlock", "return(true)") - testutil.Eventually(suite.Require(), func() bool { + testutil.Eventually(re, func() bool { b, _ := os.ReadFile(fname) l := string(b) return strings.Contains(l, "watch channel is blocked for a long time") }) failpoint.Disable("github.com/tikv/pd/pkg/utils/etcdutil/watchChanBlock") } else { - testutil.Eventually(suite.Require(), func() bool { + testutil.Eventually(re, func() bool { b, _ := os.ReadFile(fname) l := string(b) return strings.Contains(l, "watcher receives progress notify in watch loop") diff --git a/server/api/cluster_test.go b/server/api/cluster_test.go index d6d8effa365..0197489ecad 100644 --- a/server/api/cluster_test.go +++ b/server/api/cluster_test.go @@ -53,6 +53,7 @@ func (suite *clusterTestSuite) TearDownSuite() { } func (suite *clusterTestSuite) TestCluster() { + re := suite.Require() // Test get cluster status, and bootstrap cluster suite.testGetClusterStatus() suite.svr.GetPersistOptions().SetPlacementRuleEnabled(true) @@ -66,42 +67,41 @@ func (suite *clusterTestSuite) TestCluster() { // Test set the config url := fmt.Sprintf("%s/cluster", suite.urlPrefix) c1 := &metapb.Cluster{} - re := suite.Require() err := tu.ReadGetJSON(re, testDialClient, url, c1) - suite.NoError(err) + re.NoError(err) c2 := &metapb.Cluster{} r := sc.ReplicationConfig{ MaxReplicas: 6, EnablePlacementRules: true, } - suite.NoError(suite.svr.SetReplicationConfig(r)) + re.NoError(suite.svr.SetReplicationConfig(r)) err = tu.ReadGetJSON(re, testDialClient, url, c2) - suite.NoError(err) + re.NoError(err) c1.MaxPeerCount = 6 - suite.Equal(c2, c1) - suite.Equal(int(r.MaxReplicas), suite.svr.GetRaftCluster().GetRuleManager().GetRule(placement.DefaultGroupID, placement.DefaultRuleID).Count) + re.Equal(c2, c1) + re.Equal(int(r.MaxReplicas), suite.svr.GetRaftCluster().GetRuleManager().GetRule(placement.DefaultGroupID, placement.DefaultRuleID).Count) } func (suite *clusterTestSuite) testGetClusterStatus() { + re := suite.Require() url := fmt.Sprintf("%s/cluster/status", suite.urlPrefix) status := cluster.Status{} - re := suite.Require() err := tu.ReadGetJSON(re, testDialClient, url, &status) - suite.NoError(err) - suite.True(status.RaftBootstrapTime.IsZero()) - suite.False(status.IsInitialized) + re.NoError(err) + re.True(status.RaftBootstrapTime.IsZero()) + re.False(status.IsInitialized) now := time.Now() mustBootstrapCluster(re, suite.svr) err = tu.ReadGetJSON(re, testDialClient, url, &status) - suite.NoError(err) - suite.True(status.RaftBootstrapTime.After(now)) - suite.False(status.IsInitialized) + re.NoError(err) + re.True(status.RaftBootstrapTime.After(now)) + re.False(status.IsInitialized) suite.svr.SetReplicationConfig(sc.ReplicationConfig{MaxReplicas: 1}) err = tu.ReadGetJSON(re, testDialClient, url, &status) - suite.NoError(err) - suite.True(status.RaftBootstrapTime.After(now)) - suite.True(status.IsInitialized) + re.NoError(err) + re.True(status.RaftBootstrapTime.After(now)) + re.True(status.IsInitialized) } diff --git a/server/api/diagnostic_test.go b/server/api/diagnostic_test.go index 4e08426ea43..8ba77f1267d 100644 --- a/server/api/diagnostic_test.go +++ b/server/api/diagnostic_test.go @@ -65,11 +65,11 @@ func (suite *diagnosticTestSuite) TearDownSuite() { func (suite *diagnosticTestSuite) checkStatus(status string, url string) { re := suite.Require() err := tu.CheckGetUntilStatusCode(re, testDialClient, url, http.StatusOK) - suite.NoError(err) + re.NoError(err) suite.Eventually(func() bool { result := &schedulers.DiagnosticResult{} err := tu.ReadGetJSON(re, testDialClient, url, result) - suite.NoError(err) + re.NoError(err) return result.Status == status }, time.Second, time.Millisecond*50) } @@ -79,52 +79,52 @@ func (suite *diagnosticTestSuite) TestSchedulerDiagnosticAPI() { addr := suite.configPrefix cfg := &config.Config{} err := tu.ReadGetJSON(re, testDialClient, addr, cfg) - suite.NoError(err) + re.NoError(err) - suite.NoError(tu.ReadGetJSON(re, testDialClient, addr, cfg)) - suite.True(cfg.Schedule.EnableDiagnostic) + re.NoError(tu.ReadGetJSON(re, testDialClient, addr, cfg)) + re.True(cfg.Schedule.EnableDiagnostic) ms := map[string]interface{}{ "enable-diagnostic": "true", "max-replicas": 1, } postData, err := json.Marshal(ms) - suite.NoError(err) - suite.NoError(tu.CheckPostJSON(testDialClient, addr, postData, tu.StatusOK(re))) + re.NoError(err) + re.NoError(tu.CheckPostJSON(testDialClient, addr, postData, tu.StatusOK(re))) cfg = &config.Config{} - suite.NoError(tu.ReadGetJSON(re, testDialClient, addr, cfg)) - suite.True(cfg.Schedule.EnableDiagnostic) + re.NoError(tu.ReadGetJSON(re, testDialClient, addr, cfg)) + re.True(cfg.Schedule.EnableDiagnostic) balanceRegionURL := suite.urlPrefix + "/" + schedulers.BalanceRegionName result := &schedulers.DiagnosticResult{} err = tu.ReadGetJSON(re, testDialClient, balanceRegionURL, result) - suite.NoError(err) - suite.Equal("disabled", result.Status) + re.NoError(err) + re.Equal("disabled", result.Status) evictLeaderURL := suite.urlPrefix + "/" + schedulers.EvictLeaderName - suite.NoError(tu.CheckGetJSON(testDialClient, evictLeaderURL, nil, tu.StatusNotOK(re))) + re.NoError(tu.CheckGetJSON(testDialClient, evictLeaderURL, nil, tu.StatusNotOK(re))) input := make(map[string]interface{}) input["name"] = schedulers.BalanceRegionName body, err := json.Marshal(input) - suite.NoError(err) - err = tu.CheckPostJSON(testDialClient, suite.schedulerPrifex, body, tu.StatusOK(suite.Require())) - suite.NoError(err) + re.NoError(err) + err = tu.CheckPostJSON(testDialClient, suite.schedulerPrifex, body, tu.StatusOK(re)) + re.NoError(err) suite.checkStatus("pending", balanceRegionURL) input = make(map[string]interface{}) input["delay"] = 30 pauseArgs, err := json.Marshal(input) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, suite.schedulerPrifex+"/"+schedulers.BalanceRegionName, pauseArgs, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) suite.checkStatus("paused", balanceRegionURL) input["delay"] = 0 pauseArgs, err = json.Marshal(input) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, suite.schedulerPrifex+"/"+schedulers.BalanceRegionName, pauseArgs, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) suite.checkStatus("pending", balanceRegionURL) mustPutRegion(re, suite.svr, 1000, 1, []byte("a"), []byte("b"), core.SetApproximateSize(60)) @@ -132,6 +132,6 @@ func (suite *diagnosticTestSuite) TestSchedulerDiagnosticAPI() { deleteURL := fmt.Sprintf("%s/%s", suite.schedulerPrifex, schedulers.BalanceRegionName) err = tu.CheckDelete(testDialClient, deleteURL, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) suite.checkStatus("disabled", balanceRegionURL) } diff --git a/server/api/hot_status_test.go b/server/api/hot_status_test.go index 8dace1975da..1ffb9dd787f 100644 --- a/server/api/hot_status_test.go +++ b/server/api/hot_status_test.go @@ -57,27 +57,29 @@ func (suite *hotStatusTestSuite) TearDownSuite() { } func (suite *hotStatusTestSuite) TestGetHotStore() { + re := suite.Require() stat := handler.HotStoreStats{} - err := tu.ReadGetJSON(suite.Require(), testDialClient, suite.urlPrefix+"/stores", &stat) - suite.NoError(err) + err := tu.ReadGetJSON(re, testDialClient, suite.urlPrefix+"/stores", &stat) + re.NoError(err) } func (suite *hotStatusTestSuite) TestGetHistoryHotRegionsBasic() { + re := suite.Require() request := server.HistoryHotRegionsRequest{ StartTime: 0, EndTime: time.Now().AddDate(0, 2, 0).UnixNano() / int64(time.Millisecond), } data, err := json.Marshal(request) - suite.NoError(err) - re := suite.Require() + re.NoError(err) err = tu.CheckGetJSON(testDialClient, suite.urlPrefix+"/regions/history", data, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) errRequest := "{\"start_time\":\"err\"}" err = tu.CheckGetJSON(testDialClient, suite.urlPrefix+"/regions/history", []byte(errRequest), tu.StatusNotOK(re)) - suite.NoError(err) + re.NoError(err) } func (suite *hotStatusTestSuite) TestGetHistoryHotRegionsTimeRange() { + re := suite.Require() hotRegionStorage := suite.svr.GetHistoryHotRegionStorage() now := time.Now() hotRegions := []*storage.HistoryHotRegion{ @@ -95,23 +97,25 @@ func (suite *hotStatusTestSuite) TestGetHistoryHotRegionsTimeRange() { EndTime: now.Add(10*time.Second).UnixNano() / int64(time.Millisecond), } check := func(res []byte, statusCode int, _ http.Header) { - suite.Equal(200, statusCode) + re.Equal(200, statusCode) historyHotRegions := &storage.HistoryHotRegions{} - json.Unmarshal(res, historyHotRegions) + err := json.Unmarshal(res, historyHotRegions) + re.NoError(err) for _, region := range historyHotRegions.HistoryHotRegion { suite.GreaterOrEqual(region.UpdateTime, request.StartTime) suite.LessOrEqual(region.UpdateTime, request.EndTime) } } err := writeToDB(hotRegionStorage.LevelDBKV, hotRegions) - suite.NoError(err) + re.NoError(err) data, err := json.Marshal(request) - suite.NoError(err) + re.NoError(err) err = tu.CheckGetJSON(testDialClient, suite.urlPrefix+"/regions/history", data, check) - suite.NoError(err) + re.NoError(err) } func (suite *hotStatusTestSuite) TestGetHistoryHotRegionsIDAndTypes() { + re := suite.Require() hotRegionStorage := suite.svr.GetHistoryHotRegionStorage() now := time.Now() hotRegions := []*storage.HistoryHotRegion{ @@ -180,18 +184,18 @@ func (suite *hotStatusTestSuite) TestGetHistoryHotRegionsIDAndTypes() { EndTime: now.Add(10*time.Minute).UnixNano() / int64(time.Millisecond), } check := func(res []byte, statusCode int, _ http.Header) { - suite.Equal(200, statusCode) + re.Equal(200, statusCode) historyHotRegions := &storage.HistoryHotRegions{} json.Unmarshal(res, historyHotRegions) - suite.Len(historyHotRegions.HistoryHotRegion, 1) - suite.Equal(hotRegions[0], historyHotRegions.HistoryHotRegion[0]) + re.Len(historyHotRegions.HistoryHotRegion, 1) + re.Equal(hotRegions[0], historyHotRegions.HistoryHotRegion[0]) } err := writeToDB(hotRegionStorage.LevelDBKV, hotRegions) - suite.NoError(err) + re.NoError(err) data, err := json.Marshal(request) - suite.NoError(err) + re.NoError(err) err = tu.CheckGetJSON(testDialClient, suite.urlPrefix+"/regions/history", data, check) - suite.NoError(err) + re.NoError(err) } func writeToDB(kv *kv.LevelDBKV, hotRegions []*storage.HistoryHotRegion) error { diff --git a/server/api/label_test.go b/server/api/label_test.go index 9bcd40aac01..13474f89f22 100644 --- a/server/api/label_test.go +++ b/server/api/label_test.go @@ -135,12 +135,14 @@ func (suite *labelsStoreTestSuite) TearDownSuite() { } func (suite *labelsStoreTestSuite) TestLabelsGet() { + re := suite.Require() url := fmt.Sprintf("%s/labels", suite.urlPrefix) labels := make([]*metapb.StoreLabel, 0, len(suite.stores)) - suite.NoError(tu.ReadGetJSON(suite.Require(), testDialClient, url, &labels)) + re.NoError(tu.ReadGetJSON(re, testDialClient, url, &labels)) } func (suite *labelsStoreTestSuite) TestStoresLabelFilter() { + re := suite.Require() var testCases = []struct { name, value string want []*metapb.Store @@ -174,16 +176,15 @@ func (suite *labelsStoreTestSuite) TestStoresLabelFilter() { want: []*metapb.Store{}, }, } - re := suite.Require() for _, testCase := range testCases { url := fmt.Sprintf("%s/labels/stores?name=%s&value=%s", suite.urlPrefix, testCase.name, testCase.value) info := new(StoresInfo) err := tu.ReadGetJSON(re, testDialClient, url, info) - suite.NoError(err) + re.NoError(err) checkStoresInfo(re, info.Stores, testCase.want) } _, err := newStoresLabelFilter("test", ".[test") - suite.Error(err) + re.Error(err) } type strictlyLabelsStoreTestSuite struct { @@ -215,6 +216,7 @@ func (suite *strictlyLabelsStoreTestSuite) SetupSuite() { } func (suite *strictlyLabelsStoreTestSuite) TestStoreMatch() { + re := suite.Require() testCases := []struct { store *metapb.Store valid bool @@ -288,19 +290,19 @@ func (suite *strictlyLabelsStoreTestSuite) TestStoreMatch() { }, }) if testCase.valid { - suite.NoError(err) - suite.Nil(resp.GetHeader().GetError()) + re.NoError(err) + re.Nil(resp.GetHeader().GetError()) } else { - suite.Contains(resp.GetHeader().GetError().String(), testCase.expectError) + re.Contains(resp.GetHeader().GetError().String(), testCase.expectError) } } // enable placement rules. Report no error any more. - suite.NoError(tu.CheckPostJSON( + re.NoError(tu.CheckPostJSON( testDialClient, fmt.Sprintf("%s/config", suite.urlPrefix), []byte(`{"enable-placement-rules":"true"}`), - tu.StatusOK(suite.Require()))) + tu.StatusOK(re))) for _, testCase := range testCases { resp, err := suite.grpcSvr.PutStore(context.Background(), &pdpb.PutStoreRequest{ Header: &pdpb.RequestHeader{ClusterId: suite.svr.ClusterID()}, @@ -313,10 +315,10 @@ func (suite *strictlyLabelsStoreTestSuite) TestStoreMatch() { }, }) if testCase.valid { - suite.NoError(err) - suite.Nil(resp.GetHeader().GetError()) + re.NoError(err) + re.Nil(resp.GetHeader().GetError()) } else { - suite.Contains(resp.GetHeader().GetError().String(), testCase.expectError) + re.Contains(resp.GetHeader().GetError().String(), testCase.expectError) } } } diff --git a/server/api/log_test.go b/server/api/log_test.go index 27dceb52eca..2478f6f4e54 100644 --- a/server/api/log_test.go +++ b/server/api/log_test.go @@ -52,10 +52,11 @@ func (suite *logTestSuite) TearDownSuite() { } func (suite *logTestSuite) TestSetLogLevel() { + re := suite.Require() level := "error" data, err := json.Marshal(level) - suite.NoError(err) - err = tu.CheckPostJSON(testDialClient, suite.urlPrefix+"/log", data, tu.StatusOK(suite.Require())) - suite.NoError(err) - suite.Equal(level, log.GetLevel().String()) + re.NoError(err) + err = tu.CheckPostJSON(testDialClient, suite.urlPrefix+"/log", data, tu.StatusOK(re)) + re.NoError(err) + re.Equal(level, log.GetLevel().String()) } diff --git a/server/api/member_test.go b/server/api/member_test.go index eba2763704c..65c0ff67360 100644 --- a/server/api/member_test.go +++ b/server/api/member_test.go @@ -67,17 +67,16 @@ func relaxEqualStings(re *require.Assertions, a, b []string) { re.Equal(sortedStringB, sortedStringA) } -func (suite *memberTestSuite) checkListResponse(body []byte, cfgs []*config.Config) { +func (suite *memberTestSuite) checkListResponse(re *require.Assertions, body []byte, cfgs []*config.Config) { got := make(map[string][]*pdpb.Member) json.Unmarshal(body, &got) - suite.Len(cfgs, len(got["members"])) - re := suite.Require() + re.Len(cfgs, len(got["members"])) for _, member := range got["members"] { for _, cfg := range cfgs { if member.GetName() != cfg.Name { continue } - suite.Equal("dc-1", member.DcLocation) + re.Equal("dc-1", member.DcLocation) relaxEqualStings(re, member.ClientUrls, strings.Split(cfg.ClientUrls, ",")) relaxEqualStings(re, member.PeerUrls, strings.Split(cfg.PeerUrls, ",")) } @@ -85,43 +84,46 @@ func (suite *memberTestSuite) checkListResponse(body []byte, cfgs []*config.Conf } func (suite *memberTestSuite) TestMemberList() { + re := suite.Require() for _, cfg := range suite.cfgs { addr := cfg.ClientUrls + apiPrefix + "/api/v1/members" resp, err := testDialClient.Get(addr) - suite.NoError(err) + re.NoError(err) buf, err := io.ReadAll(resp.Body) - suite.NoError(err) + re.NoError(err) resp.Body.Close() - suite.checkListResponse(buf, suite.cfgs) + suite.checkListResponse(re, buf, suite.cfgs) } } func (suite *memberTestSuite) TestMemberLeader() { + re := suite.Require() leader := suite.servers[0].GetLeader() addr := suite.cfgs[rand.Intn(len(suite.cfgs))].ClientUrls + apiPrefix + "/api/v1/leader" resp, err := testDialClient.Get(addr) - suite.NoError(err) + re.NoError(err) defer resp.Body.Close() buf, err := io.ReadAll(resp.Body) - suite.NoError(err) + re.NoError(err) var got pdpb.Member - suite.NoError(json.Unmarshal(buf, &got)) - suite.Equal(leader.GetClientUrls(), got.GetClientUrls()) - suite.Equal(leader.GetMemberId(), got.GetMemberId()) + re.NoError(json.Unmarshal(buf, &got)) + re.Equal(leader.GetClientUrls(), got.GetClientUrls()) + re.Equal(leader.GetMemberId(), got.GetMemberId()) } func (suite *memberTestSuite) TestChangeLeaderPeerUrls() { + re := suite.Require() leader := suite.servers[0].GetLeader() addr := suite.cfgs[rand.Intn(len(suite.cfgs))].ClientUrls + apiPrefix + "/api/v1/leader" resp, err := testDialClient.Get(addr) - suite.NoError(err) + re.NoError(err) defer resp.Body.Close() buf, err := io.ReadAll(resp.Body) - suite.NoError(err) + re.NoError(err) var got pdpb.Member - suite.NoError(json.Unmarshal(buf, &got)) + re.NoError(json.Unmarshal(buf, &got)) id := got.GetMemberId() peerUrls := got.GetPeerUrls() @@ -129,29 +131,30 @@ func (suite *memberTestSuite) TestChangeLeaderPeerUrls() { suite.changeLeaderPeerUrls(leader, id, newPeerUrls) addr = suite.cfgs[rand.Intn(len(suite.cfgs))].ClientUrls + apiPrefix + "/api/v1/members" resp, err = testDialClient.Get(addr) - suite.NoError(err) + re.NoError(err) buf, err = io.ReadAll(resp.Body) - suite.NoError(err) + re.NoError(err) resp.Body.Close() got1 := make(map[string]*pdpb.Member) json.Unmarshal(buf, &got1) - suite.Equal(newPeerUrls, got1["leader"].GetPeerUrls()) - suite.Equal(newPeerUrls, got1["etcd_leader"].GetPeerUrls()) + re.Equal(newPeerUrls, got1["leader"].GetPeerUrls()) + re.Equal(newPeerUrls, got1["etcd_leader"].GetPeerUrls()) // reset suite.changeLeaderPeerUrls(leader, id, peerUrls) } func (suite *memberTestSuite) changeLeaderPeerUrls(leader *pdpb.Member, id uint64, urls []string) { + re := suite.Require() data := map[string][]string{"peerURLs": urls} postData, err := json.Marshal(data) - suite.NoError(err) + re.NoError(err) req, err := http.NewRequest(http.MethodPut, fmt.Sprintf("%s/v2/members/%s", leader.GetClientUrls()[0], fmt.Sprintf("%x", id)), bytes.NewBuffer(postData)) - suite.NoError(err) + re.NoError(err) req.Header.Set("Content-Type", "application/json") resp, err := testDialClient.Do(req) - suite.NoError(err) - suite.Equal(204, resp.StatusCode) + re.NoError(err) + re.Equal(204, resp.StatusCode) resp.Body.Close() } @@ -175,10 +178,11 @@ func (suite *resignTestSuite) TearDownSuite() { } func (suite *resignTestSuite) TestResignMyself() { + re := suite.Require() addr := suite.cfgs[0].ClientUrls + apiPrefix + "/api/v1/leader/resign" resp, err := testDialClient.Post(addr, "", nil) - suite.NoError(err) - suite.Equal(http.StatusOK, resp.StatusCode) + re.NoError(err) + re.Equal(http.StatusOK, resp.StatusCode) _, _ = io.Copy(io.Discard, resp.Body) resp.Body.Close() } diff --git a/server/api/min_resolved_ts_test.go b/server/api/min_resolved_ts_test.go index 3abc7555919..88c01602eab 100644 --- a/server/api/min_resolved_ts_test.go +++ b/server/api/min_resolved_ts_test.go @@ -23,6 +23,7 @@ import ( "time" "github.com/pingcap/kvproto/pkg/metapb" + "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/utils/apiutil" @@ -70,9 +71,10 @@ func (suite *minResolvedTSTestSuite) TearDownSuite() { } func (suite *minResolvedTSTestSuite) TestMinResolvedTS() { + re := suite.Require() // case1: default run job interval := suite.svr.GetRaftCluster().GetPDServerConfig().MinResolvedTSPersistenceInterval - suite.checkMinResolvedTS(&minResolvedTS{ + suite.checkMinResolvedTS(re, &minResolvedTS{ MinResolvedTS: 0, IsRealTime: true, PersistInterval: interval, @@ -80,7 +82,7 @@ func (suite *minResolvedTSTestSuite) TestMinResolvedTS() { // case2: stop run job zero := typeutil.Duration{Duration: 0} suite.setMinResolvedTSPersistenceInterval(zero) - suite.checkMinResolvedTS(&minResolvedTS{ + suite.checkMinResolvedTS(re, &minResolvedTS{ MinResolvedTS: 0, IsRealTime: false, PersistInterval: zero, @@ -91,7 +93,7 @@ func (suite *minResolvedTSTestSuite) TestMinResolvedTS() { suite.Eventually(func() bool { return interval == suite.svr.GetRaftCluster().GetPDServerConfig().MinResolvedTSPersistenceInterval }, time.Second*10, time.Millisecond*20) - suite.checkMinResolvedTS(&minResolvedTS{ + suite.checkMinResolvedTS(re, &minResolvedTS{ MinResolvedTS: 0, IsRealTime: true, PersistInterval: interval, @@ -99,7 +101,7 @@ func (suite *minResolvedTSTestSuite) TestMinResolvedTS() { // case4: set min resolved ts ts := uint64(233) suite.setAllStoresMinResolvedTS(ts) - suite.checkMinResolvedTS(&minResolvedTS{ + suite.checkMinResolvedTS(re, &minResolvedTS{ MinResolvedTS: ts, IsRealTime: true, PersistInterval: interval, @@ -107,13 +109,13 @@ func (suite *minResolvedTSTestSuite) TestMinResolvedTS() { // case5: stop persist and return last persist value when interval is 0 interval = typeutil.Duration{Duration: 0} suite.setMinResolvedTSPersistenceInterval(interval) - suite.checkMinResolvedTS(&minResolvedTS{ + suite.checkMinResolvedTS(re, &minResolvedTS{ MinResolvedTS: ts, IsRealTime: false, PersistInterval: interval, }) suite.setAllStoresMinResolvedTS(ts) - suite.checkMinResolvedTS(&minResolvedTS{ + suite.checkMinResolvedTS(re, &minResolvedTS{ MinResolvedTS: ts, // last persist value IsRealTime: false, PersistInterval: interval, @@ -121,6 +123,7 @@ func (suite *minResolvedTSTestSuite) TestMinResolvedTS() { } func (suite *minResolvedTSTestSuite) TestMinResolvedTSByStores() { + re := suite.Require() // run job. interval := typeutil.Duration{Duration: suite.defaultInterval} suite.setMinResolvedTSPersistenceInterval(interval) @@ -142,7 +145,7 @@ func (suite *minResolvedTSTestSuite) TestMinResolvedTSByStores() { testStoresID = append(testStoresID, strconv.Itoa(i)) } - suite.checkMinResolvedTSByStores(&minResolvedTS{ + suite.checkMinResolvedTSByStores(re, &minResolvedTS{ MinResolvedTS: 234, IsRealTime: true, PersistInterval: interval, @@ -151,7 +154,7 @@ func (suite *minResolvedTSTestSuite) TestMinResolvedTSByStores() { // set all stores min resolved ts. testStoresIDStr := strings.Join(testStoresID, ",") - suite.checkMinResolvedTSByStores(&minResolvedTS{ + suite.checkMinResolvedTSByStores(re, &minResolvedTS{ MinResolvedTS: 234, IsRealTime: true, PersistInterval: interval, @@ -162,7 +165,7 @@ func (suite *minResolvedTSTestSuite) TestMinResolvedTSByStores() { testStoresID = testStoresID[:len(testStoresID)-1] testStoresIDStr = strings.Join(testStoresID, ",") delete(testMap, uint64(suite.storesNum)) - suite.checkMinResolvedTSByStores(&minResolvedTS{ + suite.checkMinResolvedTSByStores(re, &minResolvedTS{ MinResolvedTS: 234, IsRealTime: true, PersistInterval: interval, @@ -183,28 +186,28 @@ func (suite *minResolvedTSTestSuite) setAllStoresMinResolvedTS(ts uint64) { } } -func (suite *minResolvedTSTestSuite) checkMinResolvedTS(expect *minResolvedTS) { +func (suite *minResolvedTSTestSuite) checkMinResolvedTS(re *require.Assertions, expect *minResolvedTS) { suite.Eventually(func() bool { res, err := testDialClient.Get(suite.url) - suite.NoError(err) + re.NoError(err) defer res.Body.Close() listResp := &minResolvedTS{} err = apiutil.ReadJSON(res.Body, listResp) - suite.NoError(err) - suite.Nil(listResp.StoresMinResolvedTS) + re.NoError(err) + re.Nil(listResp.StoresMinResolvedTS) return reflect.DeepEqual(expect, listResp) }, time.Second*10, time.Millisecond*20) } -func (suite *minResolvedTSTestSuite) checkMinResolvedTSByStores(expect *minResolvedTS, scope string) { +func (suite *minResolvedTSTestSuite) checkMinResolvedTSByStores(re *require.Assertions, expect *minResolvedTS, scope string) { suite.Eventually(func() bool { url := fmt.Sprintf("%s?scope=%s", suite.url, scope) res, err := testDialClient.Get(url) - suite.NoError(err) + re.NoError(err) defer res.Body.Close() listResp := &minResolvedTS{} err = apiutil.ReadJSON(res.Body, listResp) - suite.NoError(err) + re.NoError(err) return reflect.DeepEqual(expect, listResp) }, time.Second*10, time.Millisecond*20) } diff --git a/server/api/pprof_test.go b/server/api/pprof_test.go index 43ad7ea5978..a1acd84dcb6 100644 --- a/server/api/pprof_test.go +++ b/server/api/pprof_test.go @@ -52,13 +52,14 @@ func (suite *profTestSuite) TearDownSuite() { } func (suite *profTestSuite) TestGetZip() { + re := suite.Require() rsp, err := testDialClient.Get(suite.urlPrefix + "/pprof/zip?" + "seconds=5s") - suite.NoError(err) + re.NoError(err) defer rsp.Body.Close() body, err := io.ReadAll(rsp.Body) - suite.NoError(err) - suite.NotNil(body) + re.NoError(err) + re.NotNil(body) zipReader, err := zip.NewReader(bytes.NewReader(body), int64(len(body))) - suite.NoError(err) - suite.Len(zipReader.File, 7) + re.NoError(err) + re.Len(zipReader.File, 7) } diff --git a/server/api/region_label_test.go b/server/api/region_label_test.go index fd7401b83e0..811c83df678 100644 --- a/server/api/region_label_test.go +++ b/server/api/region_label_test.go @@ -46,21 +46,22 @@ func (suite *regionLabelTestSuite) SetupSuite() { addr := suite.svr.GetAddr() suite.urlPrefix = fmt.Sprintf("%s%s/api/v1/config/region-label/", addr, apiPrefix) - suite.NoError(failpoint.Enable("github.com/tikv/pd/pkg/keyspace/skipSplitRegion", "return(true)")) + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/keyspace/skipSplitRegion", "return(true)")) mustBootstrapCluster(re, suite.svr) } func (suite *regionLabelTestSuite) TearDownSuite() { + re := suite.Require() suite.cleanup() - suite.NoError(failpoint.Disable("github.com/tikv/pd/pkg/keyspace/skipSplitRegion")) + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/keyspace/skipSplitRegion")) } func (suite *regionLabelTestSuite) TestGetSet() { re := suite.Require() var resp []*labeler.LabelRule err := tu.ReadGetJSON(re, testDialClient, suite.urlPrefix+"rules", &resp) - suite.NoError(err) - suite.Empty(resp) + re.NoError(err) + re.Empty(resp) rules := []*labeler.LabelRule{ {ID: "rule1", Labels: []labeler.RegionLabel{{Key: "k1", Value: "v1"}}, RuleType: "key-range", Data: makeKeyRanges("1234", "5678")}, @@ -71,26 +72,26 @@ func (suite *regionLabelTestSuite) TestGetSet() { for _, rule := range rules { data, _ := json.Marshal(rule) err = tu.CheckPostJSON(testDialClient, suite.urlPrefix+"rule", data, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) } for i, id := range ruleIDs { var rule labeler.LabelRule err = tu.ReadGetJSON(re, testDialClient, suite.urlPrefix+"rule/"+url.QueryEscape(id), &rule) - suite.NoError(err) - suite.Equal(rules[i], &rule) + re.NoError(err) + re.Equal(rules[i], &rule) } err = tu.ReadGetJSONWithBody(re, testDialClient, suite.urlPrefix+"rules/ids", []byte(`["rule1", "rule3"]`), &resp) - suite.NoError(err) + re.NoError(err) expects := []*labeler.LabelRule{rules[0], rules[2]} - suite.Equal(expects, resp) + re.Equal(expects, resp) err = tu.CheckDelete(testDialClient, suite.urlPrefix+"rule/"+url.QueryEscape("rule2/a/b"), tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) err = tu.ReadGetJSON(re, testDialClient, suite.urlPrefix+"rules", &resp) - suite.NoError(err) + re.NoError(err) sort.Slice(resp, func(i, j int) bool { return resp[i].ID < resp[j].ID }) - suite.Equal([]*labeler.LabelRule{rules[0], rules[2]}, resp) + re.Equal([]*labeler.LabelRule{rules[0], rules[2]}, resp) patch := labeler.LabelRulePatch{ SetRules: []*labeler.LabelRule{ @@ -100,11 +101,11 @@ func (suite *regionLabelTestSuite) TestGetSet() { } data, _ := json.Marshal(patch) err = tu.CheckPatchJSON(testDialClient, suite.urlPrefix+"rules", data, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) err = tu.ReadGetJSON(re, testDialClient, suite.urlPrefix+"rules", &resp) - suite.NoError(err) + re.NoError(err) sort.Slice(resp, func(i, j int) bool { return resp[i].ID < resp[j].ID }) - suite.Equal([]*labeler.LabelRule{rules[1], rules[2]}, resp) + re.Equal([]*labeler.LabelRule{rules[1], rules[2]}, resp) } func makeKeyRanges(keys ...string) []interface{} { diff --git a/server/api/region_test.go b/server/api/region_test.go index 7e48c80d7bc..4838e24d632 100644 --- a/server/api/region_test.go +++ b/server/api/region_test.go @@ -126,30 +126,30 @@ func (suite *regionTestSuite) TestRegion() { url := fmt.Sprintf("%s/region/id/%d", suite.urlPrefix, r.GetID()) r1 := &RegionInfo{} r1m := make(map[string]interface{}) - suite.NoError(tu.ReadGetJSON(re, testDialClient, url, r1)) + re.NoError(tu.ReadGetJSON(re, testDialClient, url, r1)) r1.Adjust() - suite.Equal(NewAPIRegionInfo(r), r1) - suite.NoError(tu.ReadGetJSON(re, testDialClient, url, &r1m)) - suite.Equal(float64(r.GetBytesWritten()), r1m["written_bytes"].(float64)) - suite.Equal(float64(r.GetKeysWritten()), r1m["written_keys"].(float64)) - suite.Equal(float64(r.GetBytesRead()), r1m["read_bytes"].(float64)) - suite.Equal(float64(r.GetKeysRead()), r1m["read_keys"].(float64)) + re.Equal(NewAPIRegionInfo(r), r1) + re.NoError(tu.ReadGetJSON(re, testDialClient, url, &r1m)) + re.Equal(float64(r.GetBytesWritten()), r1m["written_bytes"].(float64)) + re.Equal(float64(r.GetKeysWritten()), r1m["written_keys"].(float64)) + re.Equal(float64(r.GetBytesRead()), r1m["read_bytes"].(float64)) + re.Equal(float64(r.GetKeysRead()), r1m["read_keys"].(float64)) keys := r1m["buckets"].([]interface{}) - suite.Len(keys, 2) - suite.Equal(core.HexRegionKeyStr([]byte("a")), keys[0].(string)) - suite.Equal(core.HexRegionKeyStr([]byte("b")), keys[1].(string)) + re.Len(keys, 2) + re.Equal(core.HexRegionKeyStr([]byte("a")), keys[0].(string)) + re.Equal(core.HexRegionKeyStr([]byte("b")), keys[1].(string)) url = fmt.Sprintf("%s/region/key/%s", suite.urlPrefix, "a") r2 := &RegionInfo{} - suite.NoError(tu.ReadGetJSON(re, testDialClient, url, r2)) + re.NoError(tu.ReadGetJSON(re, testDialClient, url, r2)) r2.Adjust() - suite.Equal(NewAPIRegionInfo(r), r2) + re.Equal(NewAPIRegionInfo(r), r2) url = fmt.Sprintf("%s/region/key/%s?format=hex", suite.urlPrefix, hex.EncodeToString([]byte("a"))) r2 = &RegionInfo{} - suite.NoError(tu.ReadGetJSON(re, testDialClient, url, r2)) + re.NoError(tu.ReadGetJSON(re, testDialClient, url, r2)) r2.Adjust() - suite.Equal(NewAPIRegionInfo(r), r2) + re.Equal(NewAPIRegionInfo(r), r2) } func (suite *regionTestSuite) TestRegionCheck() { @@ -165,66 +165,67 @@ func (suite *regionTestSuite) TestRegionCheck() { mustRegionHeartbeat(re, suite.svr, r) url := fmt.Sprintf("%s/region/id/%d", suite.urlPrefix, r.GetID()) r1 := &RegionInfo{} - suite.NoError(tu.ReadGetJSON(re, testDialClient, url, r1)) + re.NoError(tu.ReadGetJSON(re, testDialClient, url, r1)) r1.Adjust() - suite.Equal(NewAPIRegionInfo(r), r1) + re.Equal(NewAPIRegionInfo(r), r1) url = fmt.Sprintf("%s/regions/check/%s", suite.urlPrefix, "down-peer") r2 := &RegionsInfo{} - suite.NoError(tu.ReadGetJSON(re, testDialClient, url, r2)) + re.NoError(tu.ReadGetJSON(re, testDialClient, url, r2)) r2.Adjust() - suite.Equal(&RegionsInfo{Count: 1, Regions: []RegionInfo{*NewAPIRegionInfo(r)}}, r2) + re.Equal(&RegionsInfo{Count: 1, Regions: []RegionInfo{*NewAPIRegionInfo(r)}}, r2) url = fmt.Sprintf("%s/regions/check/%s", suite.urlPrefix, "pending-peer") r3 := &RegionsInfo{} - suite.NoError(tu.ReadGetJSON(re, testDialClient, url, r3)) + re.NoError(tu.ReadGetJSON(re, testDialClient, url, r3)) r3.Adjust() - suite.Equal(&RegionsInfo{Count: 1, Regions: []RegionInfo{*NewAPIRegionInfo(r)}}, r3) + re.Equal(&RegionsInfo{Count: 1, Regions: []RegionInfo{*NewAPIRegionInfo(r)}}, r3) url = fmt.Sprintf("%s/regions/check/%s", suite.urlPrefix, "offline-peer") r4 := &RegionsInfo{} - suite.NoError(tu.ReadGetJSON(re, testDialClient, url, r4)) + re.NoError(tu.ReadGetJSON(re, testDialClient, url, r4)) r4.Adjust() - suite.Equal(&RegionsInfo{Count: 0, Regions: []RegionInfo{}}, r4) + re.Equal(&RegionsInfo{Count: 0, Regions: []RegionInfo{}}, r4) r = r.Clone(core.SetApproximateSize(1)) mustRegionHeartbeat(re, suite.svr, r) url = fmt.Sprintf("%s/regions/check/%s", suite.urlPrefix, "empty-region") r5 := &RegionsInfo{} - suite.NoError(tu.ReadGetJSON(re, testDialClient, url, r5)) + re.NoError(tu.ReadGetJSON(re, testDialClient, url, r5)) r5.Adjust() - suite.Equal(&RegionsInfo{Count: 1, Regions: []RegionInfo{*NewAPIRegionInfo(r)}}, r5) + re.Equal(&RegionsInfo{Count: 1, Regions: []RegionInfo{*NewAPIRegionInfo(r)}}, r5) r = r.Clone(core.SetApproximateSize(1)) mustRegionHeartbeat(re, suite.svr, r) url = fmt.Sprintf("%s/regions/check/%s", suite.urlPrefix, "hist-size") r6 := make([]*histItem, 1) - suite.NoError(tu.ReadGetJSON(re, testDialClient, url, &r6)) + re.NoError(tu.ReadGetJSON(re, testDialClient, url, &r6)) histSizes := []*histItem{{Start: 1, End: 1, Count: 1}} - suite.Equal(histSizes, r6) + re.Equal(histSizes, r6) r = r.Clone(core.SetApproximateKeys(1000)) mustRegionHeartbeat(re, suite.svr, r) url = fmt.Sprintf("%s/regions/check/%s", suite.urlPrefix, "hist-keys") r7 := make([]*histItem, 1) - suite.NoError(tu.ReadGetJSON(re, testDialClient, url, &r7)) + re.NoError(tu.ReadGetJSON(re, testDialClient, url, &r7)) histKeys := []*histItem{{Start: 1000, End: 1999, Count: 1}} - suite.Equal(histKeys, r7) + re.Equal(histKeys, r7) mustPutStore(re, suite.svr, 2, metapb.StoreState_Offline, metapb.NodeState_Removing, []*metapb.StoreLabel{}) mustRegionHeartbeat(re, suite.svr, r) url = fmt.Sprintf("%s/regions/check/%s", suite.urlPrefix, "offline-peer") r8 := &RegionsInfo{} - suite.NoError(tu.ReadGetJSON(re, testDialClient, url, r8)) + re.NoError(tu.ReadGetJSON(re, testDialClient, url, r8)) r4.Adjust() - suite.Equal(1, r8.Count) - suite.Equal(r.GetID(), r8.Regions[0].ID) + re.Equal(1, r8.Count) + re.Equal(r.GetID(), r8.Regions[0].ID) } func (suite *regionTestSuite) TestRegions() { + re := suite.Require() r := NewAPIRegionInfo(core.NewRegionInfo(&metapb.Region{Id: 1}, nil)) - suite.Nil(r.Leader.Peer) - suite.Empty(r.Leader.RoleName) + re.Nil(r.Leader.Peer) + re.Empty(r.Leader.RoleName) rs := []*core.RegionInfo{ core.NewTestRegionInfo(2, 1, []byte("a"), []byte("b"), core.SetApproximateKeys(10), core.SetApproximateSize(10)), @@ -232,7 +233,6 @@ func (suite *regionTestSuite) TestRegions() { core.NewTestRegionInfo(4, 2, []byte("c"), []byte("d"), core.SetApproximateKeys(10), core.SetApproximateSize(10)), } regions := make([]RegionInfo, 0, len(rs)) - re := suite.Require() for _, r := range rs { regions = append(regions, *NewAPIRegionInfo(r)) mustRegionHeartbeat(re, suite.svr, r) @@ -240,15 +240,15 @@ func (suite *regionTestSuite) TestRegions() { url := fmt.Sprintf("%s/regions", suite.urlPrefix) regionsInfo := &RegionsInfo{} err := tu.ReadGetJSON(re, testDialClient, url, regionsInfo) - suite.NoError(err) - suite.Len(regions, regionsInfo.Count) + re.NoError(err) + re.Len(regions, regionsInfo.Count) sort.Slice(regionsInfo.Regions, func(i, j int) bool { return regionsInfo.Regions[i].ID < regionsInfo.Regions[j].ID }) for i, r := range regionsInfo.Regions { - suite.Equal(regions[i].ID, r.ID) - suite.Equal(regions[i].ApproximateSize, r.ApproximateSize) - suite.Equal(regions[i].ApproximateKeys, r.ApproximateKeys) + re.Equal(regions[i].ID, r.ID) + re.Equal(regions[i].ApproximateSize, r.ApproximateSize) + re.Equal(regions[i].ApproximateKeys, r.ApproximateKeys) } } @@ -265,29 +265,29 @@ func (suite *regionTestSuite) TestStoreRegions() { url := fmt.Sprintf("%s/regions/store/%d", suite.urlPrefix, 1) r4 := &RegionsInfo{} err := tu.ReadGetJSON(re, testDialClient, url, r4) - suite.NoError(err) - suite.Len(regionIDs, r4.Count) + re.NoError(err) + re.Len(regionIDs, r4.Count) sort.Slice(r4.Regions, func(i, j int) bool { return r4.Regions[i].ID < r4.Regions[j].ID }) for i, r := range r4.Regions { - suite.Equal(regionIDs[i], r.ID) + re.Equal(regionIDs[i], r.ID) } regionIDs = []uint64{4} url = fmt.Sprintf("%s/regions/store/%d", suite.urlPrefix, 2) r5 := &RegionsInfo{} err = tu.ReadGetJSON(re, testDialClient, url, r5) - suite.NoError(err) - suite.Len(regionIDs, r5.Count) + re.NoError(err) + re.Len(regionIDs, r5.Count) for i, r := range r5.Regions { - suite.Equal(regionIDs[i], r.ID) + re.Equal(regionIDs[i], r.ID) } regionIDs = []uint64{} url = fmt.Sprintf("%s/regions/store/%d", suite.urlPrefix, 3) r6 := &RegionsInfo{} err = tu.ReadGetJSON(re, testDialClient, url, r6) - suite.NoError(err) - suite.Len(regionIDs, r6.Count) + re.NoError(err) + re.Len(regionIDs, r6.Count) } func (suite *regionTestSuite) TestTop() { @@ -299,13 +299,13 @@ func (suite *regionTestSuite) TestTop() { mustRegionHeartbeat(re, suite.svr, r2) r3 := core.NewTestRegionInfo(3, 1, []byte("c"), []byte("d"), core.SetWrittenBytes(500), core.SetReadBytes(800), core.SetRegionConfVer(3), core.SetRegionVersion(2)) mustRegionHeartbeat(re, suite.svr, r3) - suite.checkTopRegions(fmt.Sprintf("%s/regions/writeflow", suite.urlPrefix), []uint64{2, 1, 3}) - suite.checkTopRegions(fmt.Sprintf("%s/regions/readflow", suite.urlPrefix), []uint64{1, 3, 2}) - suite.checkTopRegions(fmt.Sprintf("%s/regions/writeflow?limit=2", suite.urlPrefix), []uint64{2, 1}) - suite.checkTopRegions(fmt.Sprintf("%s/regions/confver", suite.urlPrefix), []uint64{3, 2, 1}) - suite.checkTopRegions(fmt.Sprintf("%s/regions/confver?limit=2", suite.urlPrefix), []uint64{3, 2}) - suite.checkTopRegions(fmt.Sprintf("%s/regions/version", suite.urlPrefix), []uint64{2, 3, 1}) - suite.checkTopRegions(fmt.Sprintf("%s/regions/version?limit=2", suite.urlPrefix), []uint64{2, 3}) + suite.checkTopRegions(re, fmt.Sprintf("%s/regions/writeflow", suite.urlPrefix), []uint64{2, 1, 3}) + suite.checkTopRegions(re, fmt.Sprintf("%s/regions/readflow", suite.urlPrefix), []uint64{1, 3, 2}) + suite.checkTopRegions(re, fmt.Sprintf("%s/regions/writeflow?limit=2", suite.urlPrefix), []uint64{2, 1}) + suite.checkTopRegions(re, fmt.Sprintf("%s/regions/confver", suite.urlPrefix), []uint64{3, 2, 1}) + suite.checkTopRegions(re, fmt.Sprintf("%s/regions/confver?limit=2", suite.urlPrefix), []uint64{3, 2}) + suite.checkTopRegions(re, fmt.Sprintf("%s/regions/version", suite.urlPrefix), []uint64{2, 3, 1}) + suite.checkTopRegions(re, fmt.Sprintf("%s/regions/version?limit=2", suite.urlPrefix), []uint64{2, 3}) // Top size. baseOpt := []core.RegionCreateOption{core.SetRegionConfVer(3), core.SetRegionVersion(3)} opt := core.SetApproximateSize(1000) @@ -317,8 +317,8 @@ func (suite *regionTestSuite) TestTop() { opt = core.SetApproximateSize(800) r3 = core.NewTestRegionInfo(3, 1, []byte("c"), []byte("d"), append(baseOpt, opt)...) mustRegionHeartbeat(re, suite.svr, r3) - suite.checkTopRegions(fmt.Sprintf("%s/regions/size?limit=2", suite.urlPrefix), []uint64{1, 2}) - suite.checkTopRegions(fmt.Sprintf("%s/regions/size", suite.urlPrefix), []uint64{1, 2, 3}) + suite.checkTopRegions(re, fmt.Sprintf("%s/regions/size?limit=2", suite.urlPrefix), []uint64{1, 2}) + suite.checkTopRegions(re, fmt.Sprintf("%s/regions/size", suite.urlPrefix), []uint64{1, 2, 3}) // Top CPU usage. baseOpt = []core.RegionCreateOption{core.SetRegionConfVer(4), core.SetRegionVersion(4)} opt = core.SetCPUUsage(100) @@ -330,21 +330,22 @@ func (suite *regionTestSuite) TestTop() { opt = core.SetCPUUsage(500) r3 = core.NewTestRegionInfo(3, 1, []byte("c"), []byte("d"), append(baseOpt, opt)...) mustRegionHeartbeat(re, suite.svr, r3) - suite.checkTopRegions(fmt.Sprintf("%s/regions/cpu?limit=2", suite.urlPrefix), []uint64{3, 2}) - suite.checkTopRegions(fmt.Sprintf("%s/regions/cpu", suite.urlPrefix), []uint64{3, 2, 1}) + suite.checkTopRegions(re, fmt.Sprintf("%s/regions/cpu?limit=2", suite.urlPrefix), []uint64{3, 2}) + suite.checkTopRegions(re, fmt.Sprintf("%s/regions/cpu", suite.urlPrefix), []uint64{3, 2, 1}) } -func (suite *regionTestSuite) checkTopRegions(url string, regionIDs []uint64) { +func (suite *regionTestSuite) checkTopRegions(re *require.Assertions, url string, regionIDs []uint64) { regions := &RegionsInfo{} - err := tu.ReadGetJSON(suite.Require(), testDialClient, url, regions) - suite.NoError(err) - suite.Len(regionIDs, regions.Count) + err := tu.ReadGetJSON(re, testDialClient, url, regions) + re.NoError(err) + re.Len(regionIDs, regions.Count) for i, r := range regions.Regions { - suite.Equal(regionIDs[i], r.ID) + re.Equal(regionIDs[i], r.ID) } } func (suite *regionTestSuite) TestTopN() { + re := suite.Require() writtenBytes := []uint64{10, 10, 9, 5, 3, 2, 2, 1, 0, 0} for n := 0; n <= len(writtenBytes)+1; n++ { regions := make([]*core.RegionInfo, 0, len(writtenBytes)) @@ -355,12 +356,12 @@ func (suite *regionTestSuite) TestTopN() { } topN := TopNRegions(regions, func(a, b *core.RegionInfo) bool { return a.GetBytesWritten() < b.GetBytesWritten() }, n) if n > len(writtenBytes) { - suite.Len(topN, len(writtenBytes)) + re.Len(topN, len(writtenBytes)) } else { - suite.Len(topN, n) + re.Len(topN, n) } for i := range topN { - suite.Equal(writtenBytes[i], topN[i].GetBytesWritten()) + re.Equal(writtenBytes[i], topN[i].GetBytesWritten()) } } } @@ -432,8 +433,8 @@ func (suite *getRegionTestSuite) TestRegionKey() { url := fmt.Sprintf("%s/region/key/%s", suite.urlPrefix, url.QueryEscape(string([]byte{0xFF, 0xFF, 0xBB}))) RegionInfo := &RegionInfo{} err := tu.ReadGetJSON(re, testDialClient, url, RegionInfo) - suite.NoError(err) - suite.Equal(RegionInfo.ID, r.GetID()) + re.NoError(err) + re.Equal(RegionInfo.ID, r.GetID()) } func (suite *getRegionTestSuite) TestScanRegionByKeys() { @@ -453,55 +454,55 @@ func (suite *getRegionTestSuite) TestScanRegionByKeys() { regionIDs := []uint64{3, 4, 5, 99} regions := &RegionsInfo{} err := tu.ReadGetJSON(re, testDialClient, url, regions) - suite.NoError(err) - suite.Len(regionIDs, regions.Count) + re.NoError(err) + re.Len(regionIDs, regions.Count) for i, v := range regionIDs { - suite.Equal(regions.Regions[i].ID, v) + re.Equal(regions.Regions[i].ID, v) } url = fmt.Sprintf("%s/regions/key?key=%s", suite.urlPrefix, "d") regionIDs = []uint64{4, 5, 99} regions = &RegionsInfo{} err = tu.ReadGetJSON(re, testDialClient, url, regions) - suite.NoError(err) - suite.Len(regionIDs, regions.Count) + re.NoError(err) + re.Len(regionIDs, regions.Count) for i, v := range regionIDs { - suite.Equal(regions.Regions[i].ID, v) + re.Equal(regions.Regions[i].ID, v) } url = fmt.Sprintf("%s/regions/key?key=%s", suite.urlPrefix, "g") regionIDs = []uint64{5, 99} regions = &RegionsInfo{} err = tu.ReadGetJSON(re, testDialClient, url, regions) - suite.NoError(err) - suite.Len(regionIDs, regions.Count) + re.NoError(err) + re.Len(regionIDs, regions.Count) for i, v := range regionIDs { - suite.Equal(regions.Regions[i].ID, v) + re.Equal(regions.Regions[i].ID, v) } url = fmt.Sprintf("%s/regions/key?end_key=%s", suite.urlPrefix, "e") regionIDs = []uint64{2, 3, 4} regions = &RegionsInfo{} err = tu.ReadGetJSON(re, testDialClient, url, regions) - suite.NoError(err) - suite.Len(regionIDs, regions.Count) + re.NoError(err) + re.Len(regionIDs, regions.Count) for i, v := range regionIDs { - suite.Equal(regions.Regions[i].ID, v) + re.Equal(regions.Regions[i].ID, v) } url = fmt.Sprintf("%s/regions/key?key=%s&end_key=%s", suite.urlPrefix, "b", "g") regionIDs = []uint64{3, 4} regions = &RegionsInfo{} err = tu.ReadGetJSON(re, testDialClient, url, regions) - suite.NoError(err) - suite.Len(regionIDs, regions.Count) + re.NoError(err) + re.Len(regionIDs, regions.Count) for i, v := range regionIDs { - suite.Equal(regions.Regions[i].ID, v) + re.Equal(regions.Regions[i].ID, v) } url = fmt.Sprintf("%s/regions/key?key=%s&end_key=%s", suite.urlPrefix, "b", []byte{0xFF, 0xFF, 0xCC}) regionIDs = []uint64{3, 4, 5, 99} regions = &RegionsInfo{} err = tu.ReadGetJSON(re, testDialClient, url, regions) - suite.NoError(err) - suite.Len(regionIDs, regions.Count) + re.NoError(err) + re.Len(regionIDs, regions.Count) for i, v := range regionIDs { - suite.Equal(regions.Regions[i].ID, v) + re.Equal(regions.Regions[i].ID, v) } } @@ -547,8 +548,8 @@ func (suite *getRegionRangeHolesTestSuite) TestRegionRangeHoles() { url := fmt.Sprintf("%s/regions/range-holes", suite.urlPrefix) rangeHoles := new([][]string) - suite.NoError(tu.ReadGetJSON(re, testDialClient, url, rangeHoles)) - suite.Equal([][]string{ + re.NoError(tu.ReadGetJSON(re, testDialClient, url, rangeHoles)) + re.Equal([][]string{ {"", core.HexRegionKeyStr(r1.GetStartKey())}, {core.HexRegionKeyStr(r1.GetEndKey()), core.HexRegionKeyStr(r3.GetStartKey())}, {core.HexRegionKeyStr(r4.GetEndKey()), core.HexRegionKeyStr(r6.GetStartKey())}, diff --git a/server/api/server_test.go b/server/api/server_test.go index d834941193b..38adab2de32 100644 --- a/server/api/server_test.go +++ b/server/api/server_test.go @@ -208,58 +208,59 @@ func (suite *serviceTestSuite) TearDownSuite() { } func (suite *serviceTestSuite) TestServiceLabels() { + re := suite.Require() accessPaths := suite.svr.GetServiceLabels("Profile") - suite.Len(accessPaths, 1) - suite.Equal("/pd/api/v1/debug/pprof/profile", accessPaths[0].Path) - suite.Equal("", accessPaths[0].Method) + re.Len(accessPaths, 1) + re.Equal("/pd/api/v1/debug/pprof/profile", accessPaths[0].Path) + re.Equal("", accessPaths[0].Method) serviceLabel := suite.svr.GetAPIAccessServiceLabel( apiutil.NewAccessPath("/pd/api/v1/debug/pprof/profile", "")) - suite.Equal("Profile", serviceLabel) + re.Equal("Profile", serviceLabel) serviceLabel = suite.svr.GetAPIAccessServiceLabel( apiutil.NewAccessPath("/pd/api/v1/debug/pprof/profile", http.MethodGet)) - suite.Equal("Profile", serviceLabel) + re.Equal("Profile", serviceLabel) accessPaths = suite.svr.GetServiceLabels("GetSchedulerConfig") - suite.Len(accessPaths, 1) - suite.Equal("/pd/api/v1/scheduler-config", accessPaths[0].Path) - suite.Equal("", accessPaths[0].Method) + re.Len(accessPaths, 1) + re.Equal("/pd/api/v1/scheduler-config", accessPaths[0].Path) + re.Equal("", accessPaths[0].Method) accessPaths = suite.svr.GetServiceLabels("ResignLeader") - suite.Len(accessPaths, 1) - suite.Equal("/pd/api/v1/leader/resign", accessPaths[0].Path) - suite.Equal(http.MethodPost, accessPaths[0].Method) + re.Len(accessPaths, 1) + re.Equal("/pd/api/v1/leader/resign", accessPaths[0].Path) + re.Equal(http.MethodPost, accessPaths[0].Method) serviceLabel = suite.svr.GetAPIAccessServiceLabel( apiutil.NewAccessPath("/pd/api/v1/leader/resign", http.MethodPost)) - suite.Equal("ResignLeader", serviceLabel) + re.Equal("ResignLeader", serviceLabel) serviceLabel = suite.svr.GetAPIAccessServiceLabel( apiutil.NewAccessPath("/pd/api/v1/leader/resign", http.MethodGet)) - suite.Equal("", serviceLabel) + re.Equal("", serviceLabel) serviceLabel = suite.svr.GetAPIAccessServiceLabel( apiutil.NewAccessPath("/pd/api/v1/leader/resign", "")) - suite.Equal("", serviceLabel) + re.Equal("", serviceLabel) accessPaths = suite.svr.GetServiceLabels("QueryMetric") - suite.Len(accessPaths, 4) + re.Len(accessPaths, 4) sort.Slice(accessPaths, func(i, j int) bool { if accessPaths[i].Path == accessPaths[j].Path { return accessPaths[i].Method < accessPaths[j].Method } return accessPaths[i].Path < accessPaths[j].Path }) - suite.Equal("/pd/api/v1/metric/query", accessPaths[0].Path) - suite.Equal(http.MethodGet, accessPaths[0].Method) - suite.Equal("/pd/api/v1/metric/query", accessPaths[1].Path) - suite.Equal(http.MethodPost, accessPaths[1].Method) - suite.Equal("/pd/api/v1/metric/query_range", accessPaths[2].Path) - suite.Equal(http.MethodGet, accessPaths[2].Method) - suite.Equal("/pd/api/v1/metric/query_range", accessPaths[3].Path) - suite.Equal(http.MethodPost, accessPaths[3].Method) + re.Equal("/pd/api/v1/metric/query", accessPaths[0].Path) + re.Equal(http.MethodGet, accessPaths[0].Method) + re.Equal("/pd/api/v1/metric/query", accessPaths[1].Path) + re.Equal(http.MethodPost, accessPaths[1].Method) + re.Equal("/pd/api/v1/metric/query_range", accessPaths[2].Path) + re.Equal(http.MethodGet, accessPaths[2].Method) + re.Equal("/pd/api/v1/metric/query_range", accessPaths[3].Path) + re.Equal(http.MethodPost, accessPaths[3].Method) serviceLabel = suite.svr.GetAPIAccessServiceLabel( apiutil.NewAccessPath("/pd/api/v1/metric/query", http.MethodPost)) - suite.Equal("QueryMetric", serviceLabel) + re.Equal("QueryMetric", serviceLabel) serviceLabel = suite.svr.GetAPIAccessServiceLabel( apiutil.NewAccessPath("/pd/api/v1/metric/query", http.MethodGet)) - suite.Equal("QueryMetric", serviceLabel) + re.Equal("QueryMetric", serviceLabel) } func (suite *adminTestSuite) TestCleanPath() { @@ -268,7 +269,7 @@ func (suite *adminTestSuite) TestCleanPath() { url := fmt.Sprintf("%s/admin/persist-file/../../config", suite.urlPrefix) cfg := &config.Config{} err := testutil.ReadGetJSON(re, testDialClient, url, cfg) - suite.NoError(err) + re.NoError(err) // handled by router response := httptest.NewRecorder() diff --git a/server/api/service_gc_safepoint_test.go b/server/api/service_gc_safepoint_test.go index 3df9102d116..c2d0603ac49 100644 --- a/server/api/service_gc_safepoint_test.go +++ b/server/api/service_gc_safepoint_test.go @@ -55,6 +55,7 @@ func (suite *serviceGCSafepointTestSuite) TearDownSuite() { } func (suite *serviceGCSafepointTestSuite) TestServiceGCSafepoint() { + re := suite.Require() sspURL := suite.urlPrefix + "/gc/safepoint" storage := suite.svr.GetStorage() @@ -80,22 +81,22 @@ func (suite *serviceGCSafepointTestSuite) TestServiceGCSafepoint() { } for _, ssp := range list.ServiceGCSafepoints { err := storage.SaveServiceGCSafePoint(ssp) - suite.NoError(err) + re.NoError(err) } storage.SaveGCSafePoint(1) res, err := testDialClient.Get(sspURL) - suite.NoError(err) + re.NoError(err) defer res.Body.Close() listResp := &ListServiceGCSafepoint{} err = apiutil.ReadJSON(res.Body, listResp) - suite.NoError(err) - suite.Equal(list, listResp) + re.NoError(err) + re.Equal(list, listResp) - err = testutil.CheckDelete(testDialClient, sspURL+"/a", testutil.StatusOK(suite.Require())) - suite.NoError(err) + err = testutil.CheckDelete(testDialClient, sspURL+"/a", testutil.StatusOK(re)) + re.NoError(err) left, err := storage.LoadAllServiceGCSafePoints() - suite.NoError(err) - suite.Equal(list.ServiceGCSafepoints[1:], left) + re.NoError(err) + re.Equal(list.ServiceGCSafepoints[1:], left) } diff --git a/server/api/service_middleware_test.go b/server/api/service_middleware_test.go index ec8094e670e..584e3600a4f 100644 --- a/server/api/service_middleware_test.go +++ b/server/api/service_middleware_test.go @@ -55,11 +55,11 @@ func (suite *auditMiddlewareTestSuite) TearDownSuite() { } func (suite *auditMiddlewareTestSuite) TestConfigAuditSwitch() { + re := suite.Require() addr := fmt.Sprintf("%s/service-middleware/config", suite.urlPrefix) sc := &config.ServiceMiddlewareConfig{} - re := suite.Require() - suite.NoError(tu.ReadGetJSON(re, testDialClient, addr, sc)) - suite.True(sc.EnableAudit) + re.NoError(tu.ReadGetJSON(re, testDialClient, addr, sc)) + re.True(sc.EnableAudit) ms := map[string]interface{}{ "enable-audit": "true", @@ -67,52 +67,52 @@ func (suite *auditMiddlewareTestSuite) TestConfigAuditSwitch() { "enable-grpc-rate-limit": "true", } postData, err := json.Marshal(ms) - suite.NoError(err) - suite.NoError(tu.CheckPostJSON(testDialClient, addr, postData, tu.StatusOK(re))) + re.NoError(err) + re.NoError(tu.CheckPostJSON(testDialClient, addr, postData, tu.StatusOK(re))) sc = &config.ServiceMiddlewareConfig{} - suite.NoError(tu.ReadGetJSON(re, testDialClient, addr, sc)) - suite.True(sc.EnableAudit) - suite.True(sc.RateLimitConfig.EnableRateLimit) - suite.True(sc.GRPCRateLimitConfig.EnableRateLimit) + re.NoError(tu.ReadGetJSON(re, testDialClient, addr, sc)) + re.True(sc.EnableAudit) + re.True(sc.RateLimitConfig.EnableRateLimit) + re.True(sc.GRPCRateLimitConfig.EnableRateLimit) ms = map[string]interface{}{ "audit.enable-audit": "false", "enable-rate-limit": "false", "enable-grpc-rate-limit": "false", } postData, err = json.Marshal(ms) - suite.NoError(err) - suite.NoError(tu.CheckPostJSON(testDialClient, addr, postData, tu.StatusOK(re))) + re.NoError(err) + re.NoError(tu.CheckPostJSON(testDialClient, addr, postData, tu.StatusOK(re))) sc = &config.ServiceMiddlewareConfig{} - suite.NoError(tu.ReadGetJSON(re, testDialClient, addr, sc)) - suite.False(sc.EnableAudit) - suite.False(sc.RateLimitConfig.EnableRateLimit) - suite.False(sc.GRPCRateLimitConfig.EnableRateLimit) + re.NoError(tu.ReadGetJSON(re, testDialClient, addr, sc)) + re.False(sc.EnableAudit) + re.False(sc.RateLimitConfig.EnableRateLimit) + re.False(sc.GRPCRateLimitConfig.EnableRateLimit) // test empty ms = map[string]interface{}{} postData, err = json.Marshal(ms) - suite.NoError(err) - suite.NoError(tu.CheckPostJSON(testDialClient, addr, postData, tu.StatusOK(re), tu.StringContain(re, "The input is empty."))) + re.NoError(err) + re.NoError(tu.CheckPostJSON(testDialClient, addr, postData, tu.StatusOK(re), tu.StringContain(re, "The input is empty."))) ms = map[string]interface{}{ "audit": "false", } postData, err = json.Marshal(ms) - suite.NoError(err) - suite.NoError(tu.CheckPostJSON(testDialClient, addr, postData, tu.Status(re, http.StatusBadRequest), tu.StringEqual(re, "config item audit not found"))) - suite.NoError(failpoint.Enable("github.com/tikv/pd/server/config/persistServiceMiddlewareFail", "return(true)")) + re.NoError(err) + re.NoError(tu.CheckPostJSON(testDialClient, addr, postData, tu.Status(re, http.StatusBadRequest), tu.StringEqual(re, "config item audit not found"))) + re.NoError(failpoint.Enable("github.com/tikv/pd/server/config/persistServiceMiddlewareFail", "return(true)")) ms = map[string]interface{}{ "audit.enable-audit": "true", } postData, err = json.Marshal(ms) - suite.NoError(err) - suite.NoError(tu.CheckPostJSON(testDialClient, addr, postData, tu.Status(re, http.StatusBadRequest))) - suite.NoError(failpoint.Disable("github.com/tikv/pd/server/config/persistServiceMiddlewareFail")) + re.NoError(err) + re.NoError(tu.CheckPostJSON(testDialClient, addr, postData, tu.Status(re, http.StatusBadRequest))) + re.NoError(failpoint.Disable("github.com/tikv/pd/server/config/persistServiceMiddlewareFail")) ms = map[string]interface{}{ "audit.audit": "false", } postData, err = json.Marshal(ms) - suite.NoError(err) - suite.NoError(tu.CheckPostJSON(testDialClient, addr, postData, tu.Status(re, http.StatusBadRequest), tu.StringEqual(re, "config item audit not found"))) + re.NoError(err) + re.NoError(tu.CheckPostJSON(testDialClient, addr, postData, tu.Status(re, http.StatusBadRequest), tu.StringEqual(re, "config item audit not found"))) } type rateLimitConfigTestSuite struct { @@ -139,74 +139,74 @@ func (suite *rateLimitConfigTestSuite) TearDownSuite() { } func (suite *rateLimitConfigTestSuite) TestUpdateRateLimitConfig() { + re := suite.Require() urlPrefix := fmt.Sprintf("%s%s/api/v1/service-middleware/config/rate-limit", suite.svr.GetAddr(), apiPrefix) // test empty type input := make(map[string]interface{}) input["type"] = 123 jsonBody, err := json.Marshal(input) - suite.NoError(err) - re := suite.Require() + re.NoError(err) err = tu.CheckPostJSON(testDialClient, urlPrefix, jsonBody, tu.Status(re, http.StatusBadRequest), tu.StringEqual(re, "\"The type is empty.\"\n")) - suite.NoError(err) + re.NoError(err) // test invalid type input = make(map[string]interface{}) input["type"] = "url" jsonBody, err = json.Marshal(input) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, urlPrefix, jsonBody, tu.Status(re, http.StatusBadRequest), tu.StringEqual(re, "\"The type is invalid.\"\n")) - suite.NoError(err) + re.NoError(err) // test empty label input = make(map[string]interface{}) input["type"] = "label" input["label"] = "" jsonBody, err = json.Marshal(input) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, urlPrefix, jsonBody, tu.Status(re, http.StatusBadRequest), tu.StringEqual(re, "\"The label is empty.\"\n")) - suite.NoError(err) + re.NoError(err) // test no label matched input = make(map[string]interface{}) input["type"] = "label" input["label"] = "TestLabel" jsonBody, err = json.Marshal(input) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, urlPrefix, jsonBody, tu.Status(re, http.StatusBadRequest), tu.StringEqual(re, "\"There is no label matched.\"\n")) - suite.NoError(err) + re.NoError(err) // test empty path input = make(map[string]interface{}) input["type"] = "path" input["path"] = "" jsonBody, err = json.Marshal(input) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, urlPrefix, jsonBody, tu.Status(re, http.StatusBadRequest), tu.StringEqual(re, "\"The path is empty.\"\n")) - suite.NoError(err) + re.NoError(err) // test path but no label matched input = make(map[string]interface{}) input["type"] = "path" input["path"] = "/pd/api/v1/test" jsonBody, err = json.Marshal(input) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, urlPrefix, jsonBody, tu.Status(re, http.StatusBadRequest), tu.StringEqual(re, "\"There is no label matched.\"\n")) - suite.NoError(err) + re.NoError(err) // no change input = make(map[string]interface{}) input["type"] = "label" input["label"] = "GetHealthStatus" jsonBody, err = json.Marshal(input) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, urlPrefix, jsonBody, tu.StatusOK(re), tu.StringEqual(re, "\"No changed.\"\n")) - suite.NoError(err) + re.NoError(err) // change concurrency input = make(map[string]interface{}) @@ -215,16 +215,16 @@ func (suite *rateLimitConfigTestSuite) TestUpdateRateLimitConfig() { input["method"] = http.MethodGet input["concurrency"] = 100 jsonBody, err = json.Marshal(input) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, urlPrefix, jsonBody, tu.StatusOK(re), tu.StringContain(re, "Concurrency limiter is changed.")) - suite.NoError(err) + re.NoError(err) input["concurrency"] = 0 jsonBody, err = json.Marshal(input) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, urlPrefix, jsonBody, tu.StatusOK(re), tu.StringContain(re, "Concurrency limiter is deleted.")) - suite.NoError(err) + re.NoError(err) // change qps input = make(map[string]interface{}) @@ -233,10 +233,10 @@ func (suite *rateLimitConfigTestSuite) TestUpdateRateLimitConfig() { input["method"] = http.MethodGet input["qps"] = 100 jsonBody, err = json.Marshal(input) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, urlPrefix, jsonBody, tu.StatusOK(re), tu.StringContain(re, "QPS rate limiter is changed.")) - suite.NoError(err) + re.NoError(err) input = make(map[string]interface{}) input["type"] = "path" @@ -244,18 +244,18 @@ func (suite *rateLimitConfigTestSuite) TestUpdateRateLimitConfig() { input["method"] = http.MethodGet input["qps"] = 0.3 jsonBody, err = json.Marshal(input) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, urlPrefix, jsonBody, tu.StatusOK(re), tu.StringContain(re, "QPS rate limiter is changed.")) - suite.NoError(err) - suite.Equal(1, suite.svr.GetRateLimitConfig().LimiterConfig["GetHealthStatus"].QPSBurst) + re.NoError(err) + re.Equal(1, suite.svr.GetRateLimitConfig().LimiterConfig["GetHealthStatus"].QPSBurst) input["qps"] = -1 jsonBody, err = json.Marshal(input) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, urlPrefix, jsonBody, tu.StatusOK(re), tu.StringContain(re, "QPS rate limiter is deleted.")) - suite.NoError(err) + re.NoError(err) // change both input = make(map[string]interface{}) @@ -264,17 +264,17 @@ func (suite *rateLimitConfigTestSuite) TestUpdateRateLimitConfig() { input["qps"] = 100 input["concurrency"] = 100 jsonBody, err = json.Marshal(input) - suite.NoError(err) + re.NoError(err) result := rateLimitResult{} err = tu.CheckPostJSON(testDialClient, urlPrefix, jsonBody, tu.StatusOK(re), tu.StringContain(re, "Concurrency limiter is changed."), tu.StringContain(re, "QPS rate limiter is changed."), tu.ExtractJSON(re, &result), ) - suite.Equal(100., result.LimiterConfig["Profile"].QPS) - suite.Equal(100, result.LimiterConfig["Profile"].QPSBurst) - suite.Equal(uint64(100), result.LimiterConfig["Profile"].ConcurrencyLimit) - suite.NoError(err) + re.Equal(100., result.LimiterConfig["Profile"].QPS) + re.Equal(100, result.LimiterConfig["Profile"].QPSBurst) + re.Equal(uint64(100), result.LimiterConfig["Profile"].ConcurrencyLimit) + re.NoError(err) limiter := suite.svr.GetServiceRateLimiter() limiter.Update("SetRateLimitConfig", ratelimit.AddLabelAllowList()) @@ -286,10 +286,10 @@ func (suite *rateLimitConfigTestSuite) TestUpdateRateLimitConfig() { input["qps"] = 100 input["concurrency"] = 100 jsonBody, err = json.Marshal(input) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, urlPrefix, jsonBody, tu.StatusNotOK(re), tu.StringEqual(re, "\"This service is in allow list whose config can not be changed.\"\n")) - suite.NoError(err) + re.NoError(err) } func (suite *rateLimitConfigTestSuite) TestUpdateGRPCRateLimitConfig() { @@ -300,70 +300,70 @@ func (suite *rateLimitConfigTestSuite) TestUpdateGRPCRateLimitConfig() { input := make(map[string]interface{}) input["label"] = "" jsonBody, err := json.Marshal(input) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, urlPrefix, jsonBody, tu.Status(re, http.StatusBadRequest), tu.StringEqual(re, "\"The label is empty.\"\n")) - suite.NoError(err) + re.NoError(err) // test no label matched input = make(map[string]interface{}) input["label"] = "TestLabel" jsonBody, err = json.Marshal(input) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, urlPrefix, jsonBody, tu.Status(re, http.StatusBadRequest), tu.StringEqual(re, "\"There is no label matched.\"\n")) - suite.NoError(err) + re.NoError(err) // no change input = make(map[string]interface{}) input["label"] = "StoreHeartbeat" jsonBody, err = json.Marshal(input) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, urlPrefix, jsonBody, tu.StatusOK(re), tu.StringEqual(re, "\"No changed.\"\n")) - suite.NoError(err) + re.NoError(err) // change concurrency input = make(map[string]interface{}) input["label"] = "StoreHeartbeat" input["concurrency"] = 100 jsonBody, err = json.Marshal(input) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, urlPrefix, jsonBody, tu.StatusOK(re), tu.StringContain(re, "Concurrency limiter is changed.")) - suite.NoError(err) + re.NoError(err) input["concurrency"] = 0 jsonBody, err = json.Marshal(input) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, urlPrefix, jsonBody, tu.StatusOK(re), tu.StringContain(re, "Concurrency limiter is deleted.")) - suite.NoError(err) + re.NoError(err) // change qps input = make(map[string]interface{}) input["label"] = "StoreHeartbeat" input["qps"] = 100 jsonBody, err = json.Marshal(input) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, urlPrefix, jsonBody, tu.StatusOK(re), tu.StringContain(re, "QPS rate limiter is changed.")) - suite.NoError(err) + re.NoError(err) input = make(map[string]interface{}) input["label"] = "StoreHeartbeat" input["qps"] = 0.3 jsonBody, err = json.Marshal(input) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, urlPrefix, jsonBody, tu.StatusOK(re), tu.StringContain(re, "QPS rate limiter is changed.")) - suite.NoError(err) - suite.Equal(1, suite.svr.GetGRPCRateLimitConfig().LimiterConfig["StoreHeartbeat"].QPSBurst) + re.NoError(err) + re.Equal(1, suite.svr.GetGRPCRateLimitConfig().LimiterConfig["StoreHeartbeat"].QPSBurst) input["qps"] = -1 jsonBody, err = json.Marshal(input) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, urlPrefix, jsonBody, tu.StatusOK(re), tu.StringContain(re, "QPS rate limiter is deleted.")) - suite.NoError(err) + re.NoError(err) // change both input = make(map[string]interface{}) @@ -371,79 +371,80 @@ func (suite *rateLimitConfigTestSuite) TestUpdateGRPCRateLimitConfig() { input["qps"] = 100 input["concurrency"] = 100 jsonBody, err = json.Marshal(input) - suite.NoError(err) + re.NoError(err) result := rateLimitResult{} err = tu.CheckPostJSON(testDialClient, urlPrefix, jsonBody, tu.StatusOK(re), tu.StringContain(re, "Concurrency limiter is changed."), tu.StringContain(re, "QPS rate limiter is changed."), tu.ExtractJSON(re, &result), ) - suite.Equal(100., result.LimiterConfig["GetStore"].QPS) - suite.Equal(100, result.LimiterConfig["GetStore"].QPSBurst) - suite.Equal(uint64(100), result.LimiterConfig["GetStore"].ConcurrencyLimit) - suite.NoError(err) + re.Equal(100., result.LimiterConfig["GetStore"].QPS) + re.Equal(100, result.LimiterConfig["GetStore"].QPSBurst) + re.Equal(uint64(100), result.LimiterConfig["GetStore"].ConcurrencyLimit) + re.NoError(err) } func (suite *rateLimitConfigTestSuite) TestConfigRateLimitSwitch() { addr := fmt.Sprintf("%s/service-middleware/config", suite.urlPrefix) sc := &config.ServiceMiddlewareConfig{} re := suite.Require() - suite.NoError(tu.ReadGetJSON(re, testDialClient, addr, sc)) - suite.False(sc.RateLimitConfig.EnableRateLimit) - suite.False(sc.GRPCRateLimitConfig.EnableRateLimit) + re.NoError(tu.ReadGetJSON(re, testDialClient, addr, sc)) + re.False(sc.RateLimitConfig.EnableRateLimit) + re.False(sc.GRPCRateLimitConfig.EnableRateLimit) ms := map[string]interface{}{ "enable-rate-limit": "true", "enable-grpc-rate-limit": "true", } postData, err := json.Marshal(ms) - suite.NoError(err) - suite.NoError(tu.CheckPostJSON(testDialClient, addr, postData, tu.StatusOK(re))) + re.NoError(err) + re.NoError(tu.CheckPostJSON(testDialClient, addr, postData, tu.StatusOK(re))) sc = &config.ServiceMiddlewareConfig{} - suite.NoError(tu.ReadGetJSON(re, testDialClient, addr, sc)) - suite.True(sc.RateLimitConfig.EnableRateLimit) - suite.True(sc.GRPCRateLimitConfig.EnableRateLimit) + re.NoError(tu.ReadGetJSON(re, testDialClient, addr, sc)) + re.True(sc.RateLimitConfig.EnableRateLimit) + re.True(sc.GRPCRateLimitConfig.EnableRateLimit) ms = map[string]interface{}{ "enable-rate-limit": "false", "enable-grpc-rate-limit": "false", } postData, err = json.Marshal(ms) - suite.NoError(err) - suite.NoError(tu.CheckPostJSON(testDialClient, addr, postData, tu.StatusOK(re))) + re.NoError(err) + re.NoError(tu.CheckPostJSON(testDialClient, addr, postData, tu.StatusOK(re))) sc = &config.ServiceMiddlewareConfig{} - suite.NoError(tu.ReadGetJSON(re, testDialClient, addr, sc)) - suite.False(sc.RateLimitConfig.EnableRateLimit) - suite.False(sc.GRPCRateLimitConfig.EnableRateLimit) + re.NoError(tu.ReadGetJSON(re, testDialClient, addr, sc)) + re.False(sc.RateLimitConfig.EnableRateLimit) + re.False(sc.GRPCRateLimitConfig.EnableRateLimit) // test empty ms = map[string]interface{}{} postData, err = json.Marshal(ms) - suite.NoError(err) - suite.NoError(tu.CheckPostJSON(testDialClient, addr, postData, tu.StatusOK(re), tu.StringContain(re, "The input is empty."))) + re.NoError(err) + re.NoError(tu.CheckPostJSON(testDialClient, addr, postData, tu.StatusOK(re), tu.StringContain(re, "The input is empty."))) ms = map[string]interface{}{ "rate-limit": "false", } postData, err = json.Marshal(ms) - suite.NoError(err) - suite.NoError(tu.CheckPostJSON(testDialClient, addr, postData, tu.Status(re, http.StatusBadRequest), tu.StringEqual(re, "config item rate-limit not found"))) - suite.NoError(failpoint.Enable("github.com/tikv/pd/server/config/persistServiceMiddlewareFail", "return(true)")) + re.NoError(err) + re.NoError(tu.CheckPostJSON(testDialClient, addr, postData, tu.Status(re, http.StatusBadRequest), tu.StringEqual(re, "config item rate-limit not found"))) + re.NoError(failpoint.Enable("github.com/tikv/pd/server/config/persistServiceMiddlewareFail", "return(true)")) ms = map[string]interface{}{ "rate-limit.enable-rate-limit": "true", "grpc-rate-limit.enable-grpc-rate-limit": "true", } postData, err = json.Marshal(ms) - suite.NoError(err) - suite.NoError(tu.CheckPostJSON(testDialClient, addr, postData, tu.Status(re, http.StatusBadRequest))) - suite.NoError(failpoint.Disable("github.com/tikv/pd/server/config/persistServiceMiddlewareFail")) + re.NoError(err) + re.NoError(tu.CheckPostJSON(testDialClient, addr, postData, tu.Status(re, http.StatusBadRequest))) + re.NoError(failpoint.Disable("github.com/tikv/pd/server/config/persistServiceMiddlewareFail")) ms = map[string]interface{}{ "rate-limit.rate-limit": "false", } postData, err = json.Marshal(ms) - suite.NoError(err) - suite.NoError(tu.CheckPostJSON(testDialClient, addr, postData, tu.Status(re, http.StatusBadRequest), tu.StringEqual(re, "config item rate-limit not found"))) + re.NoError(err) + re.NoError(tu.CheckPostJSON(testDialClient, addr, postData, tu.Status(re, http.StatusBadRequest), tu.StringEqual(re, "config item rate-limit not found"))) } func (suite *rateLimitConfigTestSuite) TestConfigLimiterConfigByOriginAPI() { + re := suite.Require() // this test case is used to test updating `limiter-config` by origin API simply addr := fmt.Sprintf("%s/service-middleware/config", suite.urlPrefix) dimensionConfig := ratelimit.DimensionConfig{QPS: 1} @@ -454,10 +455,9 @@ func (suite *rateLimitConfigTestSuite) TestConfigLimiterConfigByOriginAPI() { "limiter-config": limiterConfig, } postData, err := json.Marshal(ms) - suite.NoError(err) - re := suite.Require() - suite.NoError(tu.CheckPostJSON(testDialClient, addr, postData, tu.StatusOK(re))) + re.NoError(err) + re.NoError(tu.CheckPostJSON(testDialClient, addr, postData, tu.StatusOK(re))) sc := &config.ServiceMiddlewareConfig{} - suite.NoError(tu.ReadGetJSON(re, testDialClient, addr, sc)) - suite.Equal(1., sc.RateLimitConfig.LimiterConfig["CreateOperator"].QPS) + re.NoError(tu.ReadGetJSON(re, testDialClient, addr, sc)) + re.Equal(1., sc.RateLimitConfig.LimiterConfig["CreateOperator"].QPS) } diff --git a/server/api/stats_test.go b/server/api/stats_test.go index 4003f53ed9e..a4d8ef200a4 100644 --- a/server/api/stats_test.go +++ b/server/api/stats_test.go @@ -192,14 +192,14 @@ func (suite *statsTestSuite) TestRegionStats() { for _, query := range []string{"", "count"} { args := fmt.Sprintf("?start_key=%s&end_key=%s&%s", data.startKey, data.endKey, query) res, err := testDialClient.Get(statsURL + args) - suite.NoError(err) + re.NoError(err) defer res.Body.Close() stats := &statistics.RegionStats{} err = apiutil.ReadJSON(res.Body, stats) - suite.NoError(err) - suite.Equal(data.expect.Count, stats.Count) + re.NoError(err) + re.Equal(data.expect.Count, stats.Count) if query != "count" { - suite.Equal(data.expect, stats) + re.Equal(data.expect, stats) } } } diff --git a/server/api/store_test.go b/server/api/store_test.go index d0961d572f8..9a81244cfcb 100644 --- a/server/api/store_test.go +++ b/server/api/store_test.go @@ -50,15 +50,15 @@ func TestStoreTestSuite(t *testing.T) { suite.Run(t, new(storeTestSuite)) } -func (suite *storeTestSuite) requestStatusBody(client *http.Client, method string, url string) int { +func (suite *storeTestSuite) requestStatusBody(re *require.Assertions, client *http.Client, method string, url string) int { req, err := http.NewRequest(method, url, http.NoBody) - suite.NoError(err) + re.NoError(err) resp, err := client.Do(req) - suite.NoError(err) + re.NoError(err) _, err = io.ReadAll(resp.Body) - suite.NoError(err) + re.NoError(err) err = resp.Body.Close() - suite.NoError(err) + re.NoError(err) return resp.StatusCode } @@ -138,31 +138,31 @@ func (suite *storeTestSuite) TestStoresList() { info := new(StoresInfo) re := suite.Require() err := tu.ReadGetJSON(re, testDialClient, url, info) - suite.NoError(err) + re.NoError(err) checkStoresInfo(re, info.Stores, suite.stores[:3]) url = fmt.Sprintf("%s/stores/check?state=up", suite.urlPrefix) info = new(StoresInfo) err = tu.ReadGetJSON(re, testDialClient, url, info) - suite.NoError(err) + re.NoError(err) checkStoresInfo(re, info.Stores, suite.stores[:2]) url = fmt.Sprintf("%s/stores/check?state=offline", suite.urlPrefix) info = new(StoresInfo) err = tu.ReadGetJSON(re, testDialClient, url, info) - suite.NoError(err) + re.NoError(err) checkStoresInfo(re, info.Stores, suite.stores[2:3]) url = fmt.Sprintf("%s/stores/check?state=tombstone", suite.urlPrefix) info = new(StoresInfo) err = tu.ReadGetJSON(re, testDialClient, url, info) - suite.NoError(err) + re.NoError(err) checkStoresInfo(re, info.Stores, suite.stores[3:]) url = fmt.Sprintf("%s/stores/check?state=tombstone&state=offline", suite.urlPrefix) info = new(StoresInfo) err = tu.ReadGetJSON(re, testDialClient, url, info) - suite.NoError(err) + re.NoError(err) checkStoresInfo(re, info.Stores, suite.stores[2:]) // down store @@ -183,7 +183,7 @@ func (suite *storeTestSuite) TestStoresList() { url = fmt.Sprintf("%s/stores/check?state=down", suite.urlPrefix) info = new(StoresInfo) err = tu.ReadGetJSON(re, testDialClient, url, info) - suite.NoError(err) + re.NoError(err) checkStoresInfo(re, info.Stores, []*metapb.Store{store}) // disconnect store @@ -197,11 +197,12 @@ func (suite *storeTestSuite) TestStoresList() { url = fmt.Sprintf("%s/stores/check?state=disconnected", suite.urlPrefix) info = new(StoresInfo) err = tu.ReadGetJSON(re, testDialClient, url, info) - suite.NoError(err) + re.NoError(err) checkStoresInfo(re, info.Stores, []*metapb.Store{store}) } func (suite *storeTestSuite) TestStoreGet() { + re := suite.Require() url := fmt.Sprintf("%s/store/1", suite.urlPrefix) suite.grpcSvr.StoreHeartbeat( context.Background(), &pdpb.StoreHeartbeatRequest{ @@ -215,13 +216,13 @@ func (suite *storeTestSuite) TestStoreGet() { }, ) info := new(StoreInfo) - err := tu.ReadGetJSON(suite.Require(), testDialClient, url, info) - suite.NoError(err) + err := tu.ReadGetJSON(re, testDialClient, url, info) + re.NoError(err) capacity, _ := units.RAMInBytes("1.636TiB") available, _ := units.RAMInBytes("1.555TiB") - suite.Equal(capacity, int64(info.Status.Capacity)) - suite.Equal(available, int64(info.Status.Available)) - checkStoresInfo(suite.Require(), []*StoreInfo{info}, suite.stores[:1]) + re.Equal(capacity, int64(info.Status.Capacity)) + re.Equal(available, int64(info.Status.Available)) + checkStoresInfo(re, []*StoreInfo{info}, suite.stores[:1]) } func (suite *storeTestSuite) TestStoreLabel() { @@ -229,36 +230,36 @@ func (suite *storeTestSuite) TestStoreLabel() { re := suite.Require() var info StoreInfo err := tu.ReadGetJSON(re, testDialClient, url, &info) - suite.NoError(err) - suite.Empty(info.Store.Labels) + re.NoError(err) + re.Empty(info.Store.Labels) // Test merge. // enable label match check. labelCheck := map[string]string{"strictly-match-label": "true"} lc, _ := json.Marshal(labelCheck) err = tu.CheckPostJSON(testDialClient, suite.urlPrefix+"/config", lc, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) // Test set. labels := map[string]string{"zone": "cn", "host": "local"} b, err := json.Marshal(labels) - suite.NoError(err) + re.NoError(err) // TODO: supports strictly match check in placement rules err = tu.CheckPostJSON(testDialClient, url+"/label", b, tu.StatusNotOK(re), tu.StringContain(re, "key matching the label was not found")) - suite.NoError(err) + re.NoError(err) locationLabels := map[string]string{"location-labels": "zone,host"} ll, _ := json.Marshal(locationLabels) err = tu.CheckPostJSON(testDialClient, suite.urlPrefix+"/config", ll, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, url+"/label", b, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) err = tu.ReadGetJSON(re, testDialClient, url, &info) - suite.NoError(err) - suite.Len(info.Store.Labels, len(labels)) + re.NoError(err) + re.Len(info.Store.Labels, len(labels)) for _, l := range info.Store.Labels { - suite.Equal(l.Value, labels[l.Key]) + re.Equal(l.Value, labels[l.Key]) } // Test merge. @@ -266,33 +267,33 @@ func (suite *storeTestSuite) TestStoreLabel() { labelCheck = map[string]string{"strictly-match-label": "false"} lc, _ = json.Marshal(labelCheck) err = tu.CheckPostJSON(testDialClient, suite.urlPrefix+"/config", lc, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) labels = map[string]string{"zack": "zack1", "Host": "host1"} b, err = json.Marshal(labels) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, url+"/label", b, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) expectLabel := map[string]string{"zone": "cn", "zack": "zack1", "host": "host1"} err = tu.ReadGetJSON(re, testDialClient, url, &info) - suite.NoError(err) - suite.Len(info.Store.Labels, len(expectLabel)) + re.NoError(err) + re.Len(info.Store.Labels, len(expectLabel)) for _, l := range info.Store.Labels { - suite.Equal(expectLabel[l.Key], l.Value) + re.Equal(expectLabel[l.Key], l.Value) } // delete label b, err = json.Marshal(map[string]string{"host": ""}) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, url+"/label", b, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) err = tu.ReadGetJSON(re, testDialClient, url, &info) - suite.NoError(err) + re.NoError(err) delete(expectLabel, "host") - suite.Len(info.Store.Labels, len(expectLabel)) + re.Len(info.Store.Labels, len(expectLabel)) for _, l := range info.Store.Labels { - suite.Equal(expectLabel[l.Key], l.Value) + re.Equal(expectLabel[l.Key], l.Value) } suite.stores[0].Labels = info.Store.Labels @@ -319,40 +320,40 @@ func (suite *storeTestSuite) TestStoreDelete() { } for _, testCase := range testCases { url := fmt.Sprintf("%s/store/%d", suite.urlPrefix, testCase.id) - status := suite.requestStatusBody(testDialClient, http.MethodDelete, url) - suite.Equal(testCase.status, status) + status := suite.requestStatusBody(re, testDialClient, http.MethodDelete, url) + re.Equal(testCase.status, status) } // store 6 origin status:offline url := fmt.Sprintf("%s/store/6", suite.urlPrefix) store := new(StoreInfo) err := tu.ReadGetJSON(re, testDialClient, url, store) - suite.NoError(err) - suite.False(store.Store.PhysicallyDestroyed) - suite.Equal(metapb.StoreState_Offline, store.Store.State) + re.NoError(err) + re.False(store.Store.PhysicallyDestroyed) + re.Equal(metapb.StoreState_Offline, store.Store.State) // up store success because it is offline but not physically destroyed - status := suite.requestStatusBody(testDialClient, http.MethodPost, fmt.Sprintf("%s/state?state=Up", url)) - suite.Equal(http.StatusOK, status) + status := suite.requestStatusBody(re, testDialClient, http.MethodPost, fmt.Sprintf("%s/state?state=Up", url)) + re.Equal(http.StatusOK, status) - status = suite.requestStatusBody(testDialClient, http.MethodGet, url) - suite.Equal(http.StatusOK, status) + status = suite.requestStatusBody(re, testDialClient, http.MethodGet, url) + re.Equal(http.StatusOK, status) store = new(StoreInfo) err = tu.ReadGetJSON(re, testDialClient, url, store) - suite.NoError(err) - suite.Equal(metapb.StoreState_Up, store.Store.State) - suite.False(store.Store.PhysicallyDestroyed) + re.NoError(err) + re.Equal(metapb.StoreState_Up, store.Store.State) + re.False(store.Store.PhysicallyDestroyed) // offline store with physically destroyed - status = suite.requestStatusBody(testDialClient, http.MethodDelete, fmt.Sprintf("%s?force=true", url)) - suite.Equal(http.StatusOK, status) + status = suite.requestStatusBody(re, testDialClient, http.MethodDelete, fmt.Sprintf("%s?force=true", url)) + re.Equal(http.StatusOK, status) err = tu.ReadGetJSON(re, testDialClient, url, store) - suite.NoError(err) - suite.Equal(metapb.StoreState_Offline, store.Store.State) - suite.True(store.Store.PhysicallyDestroyed) + re.NoError(err) + re.Equal(metapb.StoreState_Offline, store.Store.State) + re.True(store.Store.PhysicallyDestroyed) // try to up store again failed because it is physically destroyed - status = suite.requestStatusBody(testDialClient, http.MethodPost, fmt.Sprintf("%s/state?state=Up", url)) - suite.Equal(http.StatusBadRequest, status) + status = suite.requestStatusBody(re, testDialClient, http.MethodPost, fmt.Sprintf("%s/state?state=Up", url)) + re.Equal(http.StatusBadRequest, status) // reset store 6 suite.cleanup() suite.SetupSuite() @@ -367,45 +368,46 @@ func (suite *storeTestSuite) TestStoreSetState() { url := fmt.Sprintf("%s/store/1", suite.urlPrefix) info := StoreInfo{} err := tu.ReadGetJSON(re, testDialClient, url, &info) - suite.NoError(err) - suite.Equal(metapb.StoreState_Up, info.Store.State) + re.NoError(err) + re.Equal(metapb.StoreState_Up, info.Store.State) // Set to Offline. info = StoreInfo{} err = tu.CheckPostJSON(testDialClient, url+"/state?state=Offline", nil, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) err = tu.ReadGetJSON(re, testDialClient, url, &info) - suite.NoError(err) - suite.Equal(metapb.StoreState_Offline, info.Store.State) + re.NoError(err) + re.Equal(metapb.StoreState_Offline, info.Store.State) // store not found info = StoreInfo{} err = tu.CheckPostJSON(testDialClient, suite.urlPrefix+"/store/10086/state?state=Offline", nil, tu.StatusNotOK(re)) - suite.NoError(err) + re.NoError(err) // Invalid state. invalidStates := []string{"Foo", "Tombstone"} for _, state := range invalidStates { info = StoreInfo{} err = tu.CheckPostJSON(testDialClient, url+"/state?state="+state, nil, tu.StatusNotOK(re)) - suite.NoError(err) + re.NoError(err) err := tu.ReadGetJSON(re, testDialClient, url, &info) - suite.NoError(err) - suite.Equal(metapb.StoreState_Offline, info.Store.State) + re.NoError(err) + re.Equal(metapb.StoreState_Offline, info.Store.State) } // Set back to Up. info = StoreInfo{} err = tu.CheckPostJSON(testDialClient, url+"/state?state=Up", nil, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) err = tu.ReadGetJSON(re, testDialClient, url, &info) - suite.NoError(err) - suite.Equal(metapb.StoreState_Up, info.Store.State) + re.NoError(err) + re.Equal(metapb.StoreState_Up, info.Store.State) suite.cleanup() suite.SetupSuite() } func (suite *storeTestSuite) TestUrlStoreFilter() { + re := suite.Require() testCases := []struct { u string want []*metapb.Store @@ -430,24 +432,25 @@ func (suite *storeTestSuite) TestUrlStoreFilter() { for _, testCase := range testCases { uu, err := url.Parse(testCase.u) - suite.NoError(err) + re.NoError(err) f, err := newStoreStateFilter(uu) - suite.NoError(err) - suite.Equal(testCase.want, f.filter(suite.stores)) + re.NoError(err) + re.Equal(testCase.want, f.filter(suite.stores)) } u, err := url.Parse("http://localhost:2379/pd/api/v1/stores?state=foo") - suite.NoError(err) + re.NoError(err) _, err = newStoreStateFilter(u) - suite.Error(err) + re.Error(err) u, err = url.Parse("http://localhost:2379/pd/api/v1/stores?state=999999") - suite.NoError(err) + re.NoError(err) _, err = newStoreStateFilter(u) - suite.Error(err) + re.Error(err) } func (suite *storeTestSuite) TestDownState() { + re := suite.Require() store := core.NewStoreInfo( &metapb.Store{ State: metapb.StoreState_Up, @@ -456,15 +459,15 @@ func (suite *storeTestSuite) TestDownState() { core.SetLastHeartbeatTS(time.Now()), ) storeInfo := newStoreInfo(suite.svr.GetScheduleConfig(), store) - suite.Equal(metapb.StoreState_Up.String(), storeInfo.Store.StateName) + re.Equal(metapb.StoreState_Up.String(), storeInfo.Store.StateName) newStore := store.Clone(core.SetLastHeartbeatTS(time.Now().Add(-time.Minute * 2))) storeInfo = newStoreInfo(suite.svr.GetScheduleConfig(), newStore) - suite.Equal(disconnectedName, storeInfo.Store.StateName) + re.Equal(disconnectedName, storeInfo.Store.StateName) newStore = store.Clone(core.SetLastHeartbeatTS(time.Now().Add(-time.Hour * 2))) storeInfo = newStoreInfo(suite.svr.GetScheduleConfig(), newStore) - suite.Equal(downStateName, storeInfo.Store.StateName) + re.Equal(downStateName, storeInfo.Store.StateName) } func (suite *storeTestSuite) TestGetAllLimit() { @@ -508,16 +511,17 @@ func (suite *storeTestSuite) TestGetAllLimit() { suite.T().Logf(testCase.name) info := make(map[uint64]interface{}, 4) err := tu.ReadGetJSON(re, testDialClient, testCase.url, &info) - suite.NoError(err) - suite.Len(info, len(testCase.expectedStores)) + re.NoError(err) + re.Len(info, len(testCase.expectedStores)) for id := range testCase.expectedStores { _, ok := info[id] - suite.True(ok) + re.True(ok) } } } func (suite *storeTestSuite) TestStoreLimitTTL() { + re := suite.Require() // add peer url := fmt.Sprintf("%s/store/1/limit?ttlSecond=%v", suite.urlPrefix, 5) data := map[string]interface{}{ @@ -525,19 +529,18 @@ func (suite *storeTestSuite) TestStoreLimitTTL() { "rate": 999, } postData, err := json.Marshal(data) - suite.NoError(err) - re := suite.Require() + re.NoError(err) err = tu.CheckPostJSON(testDialClient, url, postData, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) // remove peer data = map[string]interface{}{ "type": "remove-peer", "rate": 998, } postData, err = json.Marshal(data) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, url, postData, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) // all store limit add peer url = fmt.Sprintf("%s/stores/limit?ttlSecond=%v", suite.urlPrefix, 3) data = map[string]interface{}{ @@ -545,26 +548,26 @@ func (suite *storeTestSuite) TestStoreLimitTTL() { "rate": 997, } postData, err = json.Marshal(data) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, url, postData, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) // all store limit remove peer data = map[string]interface{}{ "type": "remove-peer", "rate": 996, } postData, err = json.Marshal(data) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, url, postData, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) - suite.Equal(float64(999), suite.svr.GetPersistOptions().GetStoreLimit(uint64(1)).AddPeer) - suite.Equal(float64(998), suite.svr.GetPersistOptions().GetStoreLimit(uint64(1)).RemovePeer) - suite.Equal(float64(997), suite.svr.GetPersistOptions().GetStoreLimit(uint64(2)).AddPeer) - suite.Equal(float64(996), suite.svr.GetPersistOptions().GetStoreLimit(uint64(2)).RemovePeer) + re.Equal(float64(999), suite.svr.GetPersistOptions().GetStoreLimit(uint64(1)).AddPeer) + re.Equal(float64(998), suite.svr.GetPersistOptions().GetStoreLimit(uint64(1)).RemovePeer) + re.Equal(float64(997), suite.svr.GetPersistOptions().GetStoreLimit(uint64(2)).AddPeer) + re.Equal(float64(996), suite.svr.GetPersistOptions().GetStoreLimit(uint64(2)).RemovePeer) time.Sleep(5 * time.Second) - suite.NotEqual(float64(999), suite.svr.GetPersistOptions().GetStoreLimit(uint64(1)).AddPeer) - suite.NotEqual(float64(998), suite.svr.GetPersistOptions().GetStoreLimit(uint64(1)).RemovePeer) - suite.NotEqual(float64(997), suite.svr.GetPersistOptions().GetStoreLimit(uint64(2)).AddPeer) - suite.NotEqual(float64(996), suite.svr.GetPersistOptions().GetStoreLimit(uint64(2)).RemovePeer) + re.NotEqual(float64(999), suite.svr.GetPersistOptions().GetStoreLimit(uint64(1)).AddPeer) + re.NotEqual(float64(998), suite.svr.GetPersistOptions().GetStoreLimit(uint64(1)).RemovePeer) + re.NotEqual(float64(997), suite.svr.GetPersistOptions().GetStoreLimit(uint64(2)).AddPeer) + re.NotEqual(float64(996), suite.svr.GetPersistOptions().GetStoreLimit(uint64(2)).RemovePeer) } diff --git a/server/api/tso_test.go b/server/api/tso_test.go index f61e460162e..c0fbe27321d 100644 --- a/server/api/tso_test.go +++ b/server/api/tso_test.go @@ -61,5 +61,5 @@ func (suite *tsoTestSuite) TestTransferAllocator() { }, tu.WithWaitFor(15*time.Second), tu.WithTickInterval(3*time.Second)) addr := suite.urlPrefix + "/tso/allocator/transfer/pd1?dcLocation=dc-1" err := tu.CheckPostJSON(testDialClient, addr, nil, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) } diff --git a/server/api/unsafe_operation_test.go b/server/api/unsafe_operation_test.go index 9a5b3887175..37c1506f6a7 100644 --- a/server/api/unsafe_operation_test.go +++ b/server/api/unsafe_operation_test.go @@ -60,29 +60,29 @@ func (suite *unsafeOperationTestSuite) TestRemoveFailedStores() { data, _ := json.Marshal(input) err := tu.CheckPostJSON(testDialClient, suite.urlPrefix+"/remove-failed-stores", data, tu.StatusNotOK(re), tu.StringEqual(re, "\"[PD:unsaferecovery:ErrUnsafeRecoveryInvalidInput]invalid input no store specified\"\n")) - suite.NoError(err) + re.NoError(err) input = map[string]interface{}{"stores": []string{"abc", "def"}} data, _ = json.Marshal(input) err = tu.CheckPostJSON(testDialClient, suite.urlPrefix+"/remove-failed-stores", data, tu.StatusNotOK(re), tu.StringEqual(re, "\"Store ids are invalid\"\n")) - suite.NoError(err) + re.NoError(err) input = map[string]interface{}{"stores": []uint64{1, 2}} data, _ = json.Marshal(input) err = tu.CheckPostJSON(testDialClient, suite.urlPrefix+"/remove-failed-stores", data, tu.StatusNotOK(re), tu.StringEqual(re, "\"[PD:unsaferecovery:ErrUnsafeRecoveryInvalidInput]invalid input store 2 doesn't exist\"\n")) - suite.NoError(err) + re.NoError(err) input = map[string]interface{}{"stores": []uint64{1}} data, _ = json.Marshal(input) err = tu.CheckPostJSON(testDialClient, suite.urlPrefix+"/remove-failed-stores", data, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) // Test show var output []unsaferecovery.StageOutput err = tu.ReadGetJSON(re, testDialClient, suite.urlPrefix+"/remove-failed-stores/show", &output) - suite.NoError(err) + re.NoError(err) } func (suite *unsafeOperationTestSuite) TestRemoveFailedStoresAutoDetect() { @@ -92,10 +92,10 @@ func (suite *unsafeOperationTestSuite) TestRemoveFailedStoresAutoDetect() { data, _ := json.Marshal(input) err := tu.CheckPostJSON(testDialClient, suite.urlPrefix+"/remove-failed-stores", data, tu.StatusNotOK(re), tu.StringEqual(re, "\"Store ids are invalid\"\n")) - suite.NoError(err) + re.NoError(err) input = map[string]interface{}{"auto-detect": true} data, _ = json.Marshal(input) err = tu.CheckPostJSON(testDialClient, suite.urlPrefix+"/remove-failed-stores", data, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) } diff --git a/server/server_test.go b/server/server_test.go index a0562879057..32f5d0646bc 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -53,21 +53,22 @@ func TestLeaderServerTestSuite(t *testing.T) { } func (suite *leaderServerTestSuite) SetupSuite() { + re := suite.Require() suite.ctx, suite.cancel = context.WithCancel(context.Background()) suite.svrs = make(map[string]*Server) - cfgs := NewTestMultiConfig(assertutil.CheckerWithNilAssert(suite.Require()), 3) + cfgs := NewTestMultiConfig(assertutil.CheckerWithNilAssert(re), 3) ch := make(chan *Server, 3) for i := 0; i < 3; i++ { cfg := cfgs[i] go func() { - mockHandler := CreateMockHandler(suite.Require(), "127.0.0.1") + mockHandler := CreateMockHandler(re, "127.0.0.1") svr, err := CreateServer(suite.ctx, cfg, nil, mockHandler) - suite.NoError(err) + re.NoError(err) err = svr.Run() - suite.NoError(err) + re.NoError(err) ch <- svr }() } @@ -87,13 +88,17 @@ func (suite *leaderServerTestSuite) TearDownSuite() { } } -func (suite *leaderServerTestSuite) newTestServersWithCfgs(ctx context.Context, cfgs []*config.Config) ([]*Server, testutil.CleanupFunc) { +func (suite *leaderServerTestSuite) newTestServersWithCfgs( + ctx context.Context, + cfgs []*config.Config, + re *require.Assertions, +) ([]*Server, testutil.CleanupFunc) { svrs := make([]*Server, 0, len(cfgs)) ch := make(chan *Server) for _, cfg := range cfgs { go func(cfg *config.Config) { - mockHandler := CreateMockHandler(suite.Require(), "127.0.0.1") + mockHandler := CreateMockHandler(re, "127.0.0.1") svr, err := CreateServer(ctx, cfg, nil, mockHandler) // prevent blocking if Asserts fails failed := true @@ -104,19 +109,19 @@ func (suite *leaderServerTestSuite) newTestServersWithCfgs(ctx context.Context, ch <- svr } }() - suite.NoError(err) + re.NoError(err) err = svr.Run() - suite.NoError(err) + re.NoError(err) failed = false }(cfg) } for i := 0; i < len(cfgs); i++ { svr := <-ch - suite.NotNil(svr) + re.NotNil(svr) svrs = append(svrs, svr) } - MustWaitLeader(suite.Require(), svrs) + MustWaitLeader(re, svrs) cleanup := func() { for _, svr := range svrs { @@ -131,9 +136,10 @@ func (suite *leaderServerTestSuite) newTestServersWithCfgs(ctx context.Context, } func (suite *leaderServerTestSuite) TestCheckClusterID() { + re := suite.Require() ctx, cancel := context.WithCancel(context.Background()) defer cancel() - cfgs := NewTestMultiConfig(assertutil.CheckerWithNilAssert(suite.Require()), 2) + cfgs := NewTestMultiConfig(assertutil.CheckerWithNilAssert(re), 2) for i, cfg := range cfgs { cfg.DataDir = fmt.Sprintf("/tmp/test_pd_check_clusterID_%d", i) // Clean up before testing. @@ -146,7 +152,7 @@ func (suite *leaderServerTestSuite) TestCheckClusterID() { cfgA, cfgB := cfgs[0], cfgs[1] // Start a standalone cluster. - svrsA, cleanA := suite.newTestServersWithCfgs(ctx, []*config.Config{cfgA}) + svrsA, cleanA := suite.newTestServersWithCfgs(ctx, []*config.Config{cfgA}, re) defer cleanA() // Close it. for _, svr := range svrsA { @@ -154,142 +160,146 @@ func (suite *leaderServerTestSuite) TestCheckClusterID() { } // Start another cluster. - _, cleanB := suite.newTestServersWithCfgs(ctx, []*config.Config{cfgB}) + _, cleanB := suite.newTestServersWithCfgs(ctx, []*config.Config{cfgB}, re) defer cleanB() // Start previous cluster, expect an error. cfgA.InitialCluster = originInitial - mockHandler := CreateMockHandler(suite.Require(), "127.0.0.1") + mockHandler := CreateMockHandler(re, "127.0.0.1") svr, err := CreateServer(ctx, cfgA, nil, mockHandler) - suite.NoError(err) + re.NoError(err) etcd, err := embed.StartEtcd(svr.etcdCfg) - suite.NoError(err) + re.NoError(err) urlsMap, err := types.NewURLsMap(svr.cfg.InitialCluster) - suite.NoError(err) + re.NoError(err) tlsConfig, err := svr.cfg.Security.ToTLSConfig() - suite.NoError(err) + re.NoError(err) err = etcdutil.CheckClusterID(etcd.Server.Cluster().ID(), urlsMap, tlsConfig) - suite.Error(err) + re.Error(err) etcd.Close() testutil.CleanServer(cfgA.DataDir) } func (suite *leaderServerTestSuite) TestRegisterServerHandler() { - cfg := NewTestSingleConfig(assertutil.CheckerWithNilAssert(suite.Require())) + re := suite.Require() + cfg := NewTestSingleConfig(assertutil.CheckerWithNilAssert(re)) ctx, cancel := context.WithCancel(context.Background()) - mockHandler := CreateMockHandler(suite.Require(), "127.0.0.1") + mockHandler := CreateMockHandler(re, "127.0.0.1") svr, err := CreateServer(ctx, cfg, nil, mockHandler) - suite.NoError(err) + re.NoError(err) _, err = CreateServer(ctx, cfg, nil, mockHandler, mockHandler) // Repeat register. - suite.Error(err) + re.Error(err) defer func() { cancel() svr.Close() testutil.CleanServer(svr.cfg.DataDir) }() err = svr.Run() - suite.NoError(err) + re.NoError(err) resp, err := http.Get(fmt.Sprintf("%s/pd/apis/mock/v1/hello", svr.GetAddr())) - suite.NoError(err) - suite.Equal(http.StatusOK, resp.StatusCode) + re.NoError(err) + re.Equal(http.StatusOK, resp.StatusCode) defer resp.Body.Close() bodyBytes, err := io.ReadAll(resp.Body) - suite.NoError(err) + re.NoError(err) bodyString := string(bodyBytes) - suite.Equal("Hello World\n", bodyString) + re.Equal("Hello World\n", bodyString) } func (suite *leaderServerTestSuite) TestSourceIpForHeaderForwarded() { - mockHandler := CreateMockHandler(suite.Require(), "127.0.0.2") - cfg := NewTestSingleConfig(assertutil.CheckerWithNilAssert(suite.Require())) + re := suite.Require() + mockHandler := CreateMockHandler(re, "127.0.0.2") + cfg := NewTestSingleConfig(assertutil.CheckerWithNilAssert(re)) ctx, cancel := context.WithCancel(context.Background()) svr, err := CreateServer(ctx, cfg, nil, mockHandler) - suite.NoError(err) + re.NoError(err) _, err = CreateServer(ctx, cfg, nil, mockHandler, mockHandler) // Repeat register. - suite.Error(err) + re.Error(err) defer func() { cancel() svr.Close() testutil.CleanServer(svr.cfg.DataDir) }() err = svr.Run() - suite.NoError(err) + re.NoError(err) req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("%s/pd/apis/mock/v1/hello", svr.GetAddr()), http.NoBody) - suite.NoError(err) + re.NoError(err) req.Header.Add(apiutil.XForwardedForHeader, "127.0.0.2") resp, err := http.DefaultClient.Do(req) - suite.NoError(err) - suite.Equal(http.StatusOK, resp.StatusCode) + re.NoError(err) + re.Equal(http.StatusOK, resp.StatusCode) defer resp.Body.Close() bodyBytes, err := io.ReadAll(resp.Body) - suite.NoError(err) + re.NoError(err) bodyString := string(bodyBytes) - suite.Equal("Hello World\n", bodyString) + re.Equal("Hello World\n", bodyString) } func (suite *leaderServerTestSuite) TestSourceIpForHeaderXReal() { - mockHandler := CreateMockHandler(suite.Require(), "127.0.0.2") - cfg := NewTestSingleConfig(assertutil.CheckerWithNilAssert(suite.Require())) + re := suite.Require() + mockHandler := CreateMockHandler(re, "127.0.0.2") + cfg := NewTestSingleConfig(assertutil.CheckerWithNilAssert(re)) ctx, cancel := context.WithCancel(context.Background()) svr, err := CreateServer(ctx, cfg, nil, mockHandler) - suite.NoError(err) + re.NoError(err) _, err = CreateServer(ctx, cfg, nil, mockHandler, mockHandler) // Repeat register. - suite.Error(err) + re.Error(err) defer func() { cancel() svr.Close() testutil.CleanServer(svr.cfg.DataDir) }() err = svr.Run() - suite.NoError(err) + re.NoError(err) req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("%s/pd/apis/mock/v1/hello", svr.GetAddr()), http.NoBody) - suite.NoError(err) + re.NoError(err) req.Header.Add(apiutil.XRealIPHeader, "127.0.0.2") resp, err := http.DefaultClient.Do(req) - suite.NoError(err) - suite.Equal(http.StatusOK, resp.StatusCode) + re.NoError(err) + re.Equal(http.StatusOK, resp.StatusCode) defer resp.Body.Close() bodyBytes, err := io.ReadAll(resp.Body) - suite.NoError(err) + re.NoError(err) bodyString := string(bodyBytes) - suite.Equal("Hello World\n", bodyString) + re.Equal("Hello World\n", bodyString) } func (suite *leaderServerTestSuite) TestSourceIpForHeaderBoth() { - mockHandler := CreateMockHandler(suite.Require(), "127.0.0.2") - cfg := NewTestSingleConfig(assertutil.CheckerWithNilAssert(suite.Require())) + re := suite.Require() + mockHandler := CreateMockHandler(re, "127.0.0.2") + cfg := NewTestSingleConfig(assertutil.CheckerWithNilAssert(re)) ctx, cancel := context.WithCancel(context.Background()) svr, err := CreateServer(ctx, cfg, nil, mockHandler) - suite.NoError(err) + re.NoError(err) _, err = CreateServer(ctx, cfg, nil, mockHandler, mockHandler) // Repeat register. - suite.Error(err) + re.Error(err) defer func() { cancel() svr.Close() testutil.CleanServer(svr.cfg.DataDir) }() err = svr.Run() - suite.NoError(err) + re.NoError(err) req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("%s/pd/apis/mock/v1/hello", svr.GetAddr()), http.NoBody) - suite.NoError(err) + re.NoError(err) req.Header.Add(apiutil.XForwardedForHeader, "127.0.0.2") req.Header.Add(apiutil.XRealIPHeader, "127.0.0.3") resp, err := http.DefaultClient.Do(req) - suite.NoError(err) - suite.Equal(http.StatusOK, resp.StatusCode) + re.NoError(err) + re.Equal(http.StatusOK, resp.StatusCode) defer resp.Body.Close() bodyBytes, err := io.ReadAll(resp.Body) - suite.NoError(err) + re.NoError(err) bodyString := string(bodyBytes) - suite.Equal("Hello World\n", bodyString) + re.Equal("Hello World\n", bodyString) } func TestAPIService(t *testing.T) { diff --git a/tests/dashboard/service_test.go b/tests/dashboard/service_test.go index 5f72efb2c36..90bcf796fbf 100644 --- a/tests/dashboard/service_test.go +++ b/tests/dashboard/service_test.go @@ -22,6 +22,7 @@ import ( "testing" "time" + "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "go.uber.org/goleak" @@ -69,20 +70,20 @@ func (suite *dashboardTestSuite) TearDownSuite() { } func (suite *dashboardTestSuite) TestDashboardRedirect() { - suite.testDashboard(false) + suite.testDashboard(suite.Require(), false) } func (suite *dashboardTestSuite) TestDashboardProxy() { - suite.testDashboard(true) + suite.testDashboard(suite.Require(), true) } -func (suite *dashboardTestSuite) checkRespCode(url string, code int) { +func (suite *dashboardTestSuite) checkRespCode(re *require.Assertions, url string, code int) { resp, err := suite.httpClient.Get(url) - suite.NoError(err) + re.NoError(err) _, err = io.ReadAll(resp.Body) - suite.NoError(err) + re.NoError(err) resp.Body.Close() - suite.Equal(code, resp.StatusCode) + re.Equal(code, resp.StatusCode) } func waitForConfigSync() { @@ -90,46 +91,46 @@ func waitForConfigSync() { time.Sleep(3 * time.Second) } -func (suite *dashboardTestSuite) checkServiceIsStarted(internalProxy bool, servers map[string]*tests.TestServer, leader *tests.TestServer) string { +func (suite *dashboardTestSuite) checkServiceIsStarted(re *require.Assertions, internalProxy bool, servers map[string]*tests.TestServer, leader *tests.TestServer) string { waitForConfigSync() dashboardAddress := leader.GetServer().GetPersistOptions().GetDashboardAddress() hasServiceNode := false for _, srv := range servers { - suite.Equal(dashboardAddress, srv.GetPersistOptions().GetDashboardAddress()) + re.Equal(dashboardAddress, srv.GetPersistOptions().GetDashboardAddress()) addr := srv.GetAddr() if addr == dashboardAddress || internalProxy { - suite.checkRespCode(fmt.Sprintf("%s/dashboard/", addr), http.StatusOK) - suite.checkRespCode(fmt.Sprintf("%s/dashboard/api/keyvisual/heatmaps", addr), http.StatusUnauthorized) + suite.checkRespCode(re, fmt.Sprintf("%s/dashboard/", addr), http.StatusOK) + suite.checkRespCode(re, fmt.Sprintf("%s/dashboard/api/keyvisual/heatmaps", addr), http.StatusUnauthorized) if addr == dashboardAddress { hasServiceNode = true } } else { - suite.checkRespCode(fmt.Sprintf("%s/dashboard/", addr), http.StatusTemporaryRedirect) - suite.checkRespCode(fmt.Sprintf("%s/dashboard/api/keyvisual/heatmaps", addr), http.StatusTemporaryRedirect) + suite.checkRespCode(re, fmt.Sprintf("%s/dashboard/", addr), http.StatusTemporaryRedirect) + suite.checkRespCode(re, fmt.Sprintf("%s/dashboard/api/keyvisual/heatmaps", addr), http.StatusTemporaryRedirect) } } - suite.True(hasServiceNode) + re.True(hasServiceNode) return dashboardAddress } -func (suite *dashboardTestSuite) checkServiceIsStopped(servers map[string]*tests.TestServer) { +func (suite *dashboardTestSuite) checkServiceIsStopped(re *require.Assertions, servers map[string]*tests.TestServer) { waitForConfigSync() for _, srv := range servers { - suite.Equal("none", srv.GetPersistOptions().GetDashboardAddress()) + re.Equal("none", srv.GetPersistOptions().GetDashboardAddress()) addr := srv.GetAddr() - suite.checkRespCode(fmt.Sprintf("%s/dashboard/", addr), http.StatusNotFound) - suite.checkRespCode(fmt.Sprintf("%s/dashboard/api/keyvisual/heatmaps", addr), http.StatusNotFound) + suite.checkRespCode(re, fmt.Sprintf("%s/dashboard/", addr), http.StatusNotFound) + suite.checkRespCode(re, fmt.Sprintf("%s/dashboard/api/keyvisual/heatmaps", addr), http.StatusNotFound) } } -func (suite *dashboardTestSuite) testDashboard(internalProxy bool) { +func (suite *dashboardTestSuite) testDashboard(re *require.Assertions, internalProxy bool) { cluster, err := tests.NewTestCluster(suite.ctx, 3, func(conf *config.Config, serverName string) { conf.Dashboard.InternalProxy = internalProxy }) - suite.NoError(err) + re.NoError(err) defer cluster.Destroy() err = cluster.RunInitialServers() - suite.NoError(err) + re.NoError(err) cmd := pdctlCmd.GetRootCmd() @@ -139,7 +140,7 @@ func (suite *dashboardTestSuite) testDashboard(internalProxy bool) { leaderAddr := leader.GetAddr() // auto select node - dashboardAddress1 := suite.checkServiceIsStarted(internalProxy, servers, leader) + dashboardAddress1 := suite.checkServiceIsStarted(re, internalProxy, servers, leader) // pd-ctl set another addr var dashboardAddress2 string @@ -151,13 +152,13 @@ func (suite *dashboardTestSuite) testDashboard(internalProxy bool) { } args := []string{"-u", leaderAddr, "config", "set", "dashboard-address", dashboardAddress2} _, err = pdctl.ExecuteCommand(cmd, args...) - suite.NoError(err) - suite.checkServiceIsStarted(internalProxy, servers, leader) - suite.Equal(dashboardAddress2, leader.GetServer().GetPersistOptions().GetDashboardAddress()) + re.NoError(err) + suite.checkServiceIsStarted(re, internalProxy, servers, leader) + re.Equal(dashboardAddress2, leader.GetServer().GetPersistOptions().GetDashboardAddress()) // pd-ctl set stop args = []string{"-u", leaderAddr, "config", "set", "dashboard-address", "none"} _, err = pdctl.ExecuteCommand(cmd, args...) - suite.NoError(err) - suite.checkServiceIsStopped(servers) + re.NoError(err) + suite.checkServiceIsStopped(re, servers) } diff --git a/tests/integrations/client/client_test.go b/tests/integrations/client/client_test.go index bb4d6851fd0..1fd8d75dec4 100644 --- a/tests/integrations/client/client_test.go +++ b/tests/integrations/client/client_test.go @@ -269,10 +269,10 @@ func TestTSOAllocatorLeader(t *testing.T) { } pdName, exist := allocatorLeaderMap[dcLocation] re.True(exist) - re.Greater(len(pdName), 0) + re.NotEmpty(pdName) pdURL, exist := endpointsMap[pdName] re.True(exist) - re.Greater(len(pdURL), 0) + re.NotEmpty(pdURL) re.Equal(pdURL, url) } } @@ -869,22 +869,22 @@ func (suite *clientTestSuite) SetupSuite() { var err error re := suite.Require() suite.srv, suite.cleanup, err = server.NewTestServer(re, assertutil.CheckerWithNilAssert(re)) - suite.NoError(err) + re.NoError(err) suite.grpcPDClient = testutil.MustNewGrpcClient(re, suite.srv.GetAddr()) suite.grpcSvr = &server.GrpcServer{Server: suite.srv} server.MustWaitLeader(re, []*server.Server{suite.srv}) - suite.bootstrapServer(newHeader(suite.srv), suite.grpcPDClient) + suite.bootstrapServer(re, newHeader(suite.srv), suite.grpcPDClient) suite.ctx, suite.clean = context.WithCancel(context.Background()) suite.client = setupCli(re, suite.ctx, suite.srv.GetEndpoints()) suite.regionHeartbeat, err = suite.grpcPDClient.RegionHeartbeat(suite.ctx) - suite.NoError(err) + re.NoError(err) suite.reportBucket, err = suite.grpcPDClient.ReportBuckets(suite.ctx) - suite.NoError(err) + re.NoError(err) cluster := suite.srv.GetRaftCluster() - suite.NotNil(cluster) + re.NotNil(cluster) now := time.Now().UnixNano() for _, store := range stores { suite.grpcSvr.PutStore(context.Background(), &pdpb.PutStoreRequest{ @@ -919,7 +919,7 @@ func newHeader(srv *server.Server) *pdpb.RequestHeader { } } -func (suite *clientTestSuite) bootstrapServer(header *pdpb.RequestHeader, client pdpb.PDClient) { +func (suite *clientTestSuite) bootstrapServer(re *require.Assertions, header *pdpb.RequestHeader, client pdpb.PDClient) { regionID := regionIDAllocator.alloc() region := &metapb.Region{ Id: regionID, @@ -935,11 +935,12 @@ func (suite *clientTestSuite) bootstrapServer(header *pdpb.RequestHeader, client Region: region, } resp, err := client.Bootstrap(context.Background(), req) - suite.NoError(err) - suite.Equal(pdpb.ErrorType_OK, resp.GetHeader().GetError().GetType()) + re.NoError(err) + re.Equal(pdpb.ErrorType_OK, resp.GetHeader().GetError().GetType()) } func (suite *clientTestSuite) TestGetRegion() { + re := suite.Require() regionID := regionIDAllocator.alloc() region := &metapb.Region{ Id: regionID, @@ -955,11 +956,10 @@ func (suite *clientTestSuite) TestGetRegion() { Leader: peers[0], } err := suite.regionHeartbeat.Send(req) - suite.NoError(err) - re := suite.Require() + re.NoError(err) testutil.Eventually(re, func() bool { r, err := suite.client.GetRegion(context.Background(), []byte("a")) - suite.NoError(err) + re.NoError(err) if r == nil { return false } @@ -984,10 +984,10 @@ func (suite *clientTestSuite) TestGetRegion() { }, }, } - suite.NoError(suite.reportBucket.Send(breq)) + re.NoError(suite.reportBucket.Send(breq)) testutil.Eventually(re, func() bool { r, err := suite.client.GetRegion(context.Background(), []byte("a"), pd.WithBuckets()) - suite.NoError(err) + re.NoError(err) if r == nil { return false } @@ -997,7 +997,7 @@ func (suite *clientTestSuite) TestGetRegion() { testutil.Eventually(re, func() bool { r, err := suite.client.GetRegion(context.Background(), []byte("a"), pd.WithBuckets()) - suite.NoError(err) + re.NoError(err) if r == nil { return false } @@ -1005,15 +1005,16 @@ func (suite *clientTestSuite) TestGetRegion() { }) suite.srv.GetRaftCluster().GetOpts().(*config.PersistOptions).SetRegionBucketEnabled(true) - suite.NoError(failpoint.Enable("github.com/tikv/pd/server/grpcClientClosed", `return(true)`)) - suite.NoError(failpoint.Enable("github.com/tikv/pd/server/useForwardRequest", `return(true)`)) - suite.NoError(suite.reportBucket.Send(breq)) - suite.Error(suite.reportBucket.RecvMsg(breq)) - suite.NoError(failpoint.Disable("github.com/tikv/pd/server/grpcClientClosed")) - suite.NoError(failpoint.Disable("github.com/tikv/pd/server/useForwardRequest")) + re.NoError(failpoint.Enable("github.com/tikv/pd/server/grpcClientClosed", `return(true)`)) + re.NoError(failpoint.Enable("github.com/tikv/pd/server/useForwardRequest", `return(true)`)) + re.NoError(suite.reportBucket.Send(breq)) + re.Error(suite.reportBucket.RecvMsg(breq)) + re.NoError(failpoint.Disable("github.com/tikv/pd/server/grpcClientClosed")) + re.NoError(failpoint.Disable("github.com/tikv/pd/server/useForwardRequest")) } func (suite *clientTestSuite) TestGetPrevRegion() { + re := suite.Require() regionLen := 10 regions := make([]*metapb.Region, 0, regionLen) for i := 0; i < regionLen; i++ { @@ -1035,13 +1036,13 @@ func (suite *clientTestSuite) TestGetPrevRegion() { Leader: peers[0], } err := suite.regionHeartbeat.Send(req) - suite.NoError(err) + re.NoError(err) } time.Sleep(500 * time.Millisecond) for i := 0; i < 20; i++ { - testutil.Eventually(suite.Require(), func() bool { + testutil.Eventually(re, func() bool { r, err := suite.client.GetPrevRegion(context.Background(), []byte{byte(i)}) - suite.NoError(err) + re.NoError(err) if i > 0 && i < regionLen { return reflect.DeepEqual(peers[0], r.Leader) && reflect.DeepEqual(regions[i-1], r.Meta) @@ -1052,6 +1053,7 @@ func (suite *clientTestSuite) TestGetPrevRegion() { } func (suite *clientTestSuite) TestScanRegions() { + re := suite.Require() regionLen := 10 regions := make([]*metapb.Region, 0, regionLen) for i := 0; i < regionLen; i++ { @@ -1073,11 +1075,11 @@ func (suite *clientTestSuite) TestScanRegions() { Leader: peers[0], } err := suite.regionHeartbeat.Send(req) - suite.NoError(err) + re.NoError(err) } // Wait for region heartbeats. - testutil.Eventually(suite.Require(), func() bool { + testutil.Eventually(re, func() bool { scanRegions, err := suite.client.ScanRegions(context.Background(), []byte{0}, nil, 10) return err == nil && len(scanRegions) == 10 }) @@ -1097,25 +1099,25 @@ func (suite *clientTestSuite) TestScanRegions() { t := suite.T() check := func(start, end []byte, limit int, expect []*metapb.Region) { scanRegions, err := suite.client.ScanRegions(context.Background(), start, end, limit) - suite.NoError(err) - suite.Len(scanRegions, len(expect)) + re.NoError(err) + re.Len(scanRegions, len(expect)) t.Log("scanRegions", scanRegions) t.Log("expect", expect) for i := range expect { - suite.Equal(expect[i], scanRegions[i].Meta) + re.Equal(expect[i], scanRegions[i].Meta) if scanRegions[i].Meta.GetId() == region3.GetID() { - suite.Equal(&metapb.Peer{}, scanRegions[i].Leader) + re.Equal(&metapb.Peer{}, scanRegions[i].Leader) } else { - suite.Equal(expect[i].Peers[0], scanRegions[i].Leader) + re.Equal(expect[i].Peers[0], scanRegions[i].Leader) } if scanRegions[i].Meta.GetId() == region4.GetID() { - suite.Equal([]*metapb.Peer{expect[i].Peers[1]}, scanRegions[i].DownPeers) + re.Equal([]*metapb.Peer{expect[i].Peers[1]}, scanRegions[i].DownPeers) } if scanRegions[i].Meta.GetId() == region5.GetID() { - suite.Equal([]*metapb.Peer{expect[i].Peers[1], expect[i].Peers[2]}, scanRegions[i].PendingPeers) + re.Equal([]*metapb.Peer{expect[i].Peers[1], expect[i].Peers[2]}, scanRegions[i].PendingPeers) } } } @@ -1128,6 +1130,7 @@ func (suite *clientTestSuite) TestScanRegions() { } func (suite *clientTestSuite) TestGetRegionByID() { + re := suite.Require() regionID := regionIDAllocator.alloc() region := &metapb.Region{ Id: regionID, @@ -1143,11 +1146,11 @@ func (suite *clientTestSuite) TestGetRegionByID() { Leader: peers[0], } err := suite.regionHeartbeat.Send(req) - suite.NoError(err) + re.NoError(err) - testutil.Eventually(suite.Require(), func() bool { + testutil.Eventually(re, func() bool { r, err := suite.client.GetRegionByID(context.Background(), regionID) - suite.NoError(err) + re.NoError(err) if r == nil { return false } @@ -1157,106 +1160,109 @@ func (suite *clientTestSuite) TestGetRegionByID() { } func (suite *clientTestSuite) TestGetStore() { + re := suite.Require() cluster := suite.srv.GetRaftCluster() - suite.NotNil(cluster) + re.NotNil(cluster) store := stores[0] // Get an up store should be OK. n, err := suite.client.GetStore(context.Background(), store.GetId()) - suite.NoError(err) - suite.Equal(store, n) + re.NoError(err) + re.Equal(store, n) actualStores, err := suite.client.GetAllStores(context.Background()) - suite.NoError(err) - suite.Len(actualStores, len(stores)) + re.NoError(err) + re.Len(actualStores, len(stores)) stores = actualStores // Mark the store as offline. err = cluster.RemoveStore(store.GetId(), false) - suite.NoError(err) + re.NoError(err) offlineStore := typeutil.DeepClone(store, core.StoreFactory) offlineStore.State = metapb.StoreState_Offline offlineStore.NodeState = metapb.NodeState_Removing // Get an offline store should be OK. n, err = suite.client.GetStore(context.Background(), store.GetId()) - suite.NoError(err) - suite.Equal(offlineStore, n) + re.NoError(err) + re.Equal(offlineStore, n) // Should return offline stores. contains := false stores, err = suite.client.GetAllStores(context.Background()) - suite.NoError(err) + re.NoError(err) for _, store := range stores { if store.GetId() == offlineStore.GetId() { contains = true - suite.Equal(offlineStore, store) + re.Equal(offlineStore, store) } } - suite.True(contains) + re.True(contains) // Mark the store as physically destroyed and offline. err = cluster.RemoveStore(store.GetId(), true) - suite.NoError(err) + re.NoError(err) physicallyDestroyedStoreID := store.GetId() // Get a physically destroyed and offline store // It should be Tombstone(become Tombstone automatically) or Offline n, err = suite.client.GetStore(context.Background(), physicallyDestroyedStoreID) - suite.NoError(err) + re.NoError(err) if n != nil { // store is still offline and physically destroyed - suite.Equal(metapb.NodeState_Removing, n.GetNodeState()) - suite.True(n.PhysicallyDestroyed) + re.Equal(metapb.NodeState_Removing, n.GetNodeState()) + re.True(n.PhysicallyDestroyed) } // Should return tombstone stores. contains = false stores, err = suite.client.GetAllStores(context.Background()) - suite.NoError(err) + re.NoError(err) for _, store := range stores { if store.GetId() == physicallyDestroyedStoreID { contains = true - suite.NotEqual(metapb.StoreState_Up, store.GetState()) - suite.True(store.PhysicallyDestroyed) + re.NotEqual(metapb.StoreState_Up, store.GetState()) + re.True(store.PhysicallyDestroyed) } } - suite.True(contains) + re.True(contains) // Should not return tombstone stores. stores, err = suite.client.GetAllStores(context.Background(), pd.WithExcludeTombstone()) - suite.NoError(err) + re.NoError(err) for _, store := range stores { if store.GetId() == physicallyDestroyedStoreID { - suite.Equal(metapb.StoreState_Offline, store.GetState()) - suite.True(store.PhysicallyDestroyed) + re.Equal(metapb.StoreState_Offline, store.GetState()) + re.True(store.PhysicallyDestroyed) } } } -func (suite *clientTestSuite) checkGCSafePoint(expectedSafePoint uint64) { +func (suite *clientTestSuite) checkGCSafePoint(re *require.Assertions, expectedSafePoint uint64) { req := &pdpb.GetGCSafePointRequest{ Header: newHeader(suite.srv), } resp, err := suite.grpcSvr.GetGCSafePoint(context.Background(), req) - suite.NoError(err) - suite.Equal(expectedSafePoint, resp.SafePoint) + re.NoError(err) + re.Equal(expectedSafePoint, resp.SafePoint) } func (suite *clientTestSuite) TestUpdateGCSafePoint() { - suite.checkGCSafePoint(0) + re := suite.Require() + suite.checkGCSafePoint(re, 0) for _, safePoint := range []uint64{0, 1, 2, 3, 233, 23333, 233333333333, math.MaxUint64} { newSafePoint, err := suite.client.UpdateGCSafePoint(context.Background(), safePoint) - suite.NoError(err) - suite.Equal(safePoint, newSafePoint) - suite.checkGCSafePoint(safePoint) + re.NoError(err) + re.Equal(safePoint, newSafePoint) + suite.checkGCSafePoint(re, safePoint) } // If the new safe point is less than the old one, it should not be updated. newSafePoint, err := suite.client.UpdateGCSafePoint(context.Background(), 1) - suite.Equal(uint64(math.MaxUint64), newSafePoint) - suite.NoError(err) - suite.checkGCSafePoint(math.MaxUint64) + re.Equal(uint64(math.MaxUint64), newSafePoint) + re.NoError(err) + suite.checkGCSafePoint(re, math.MaxUint64) } func (suite *clientTestSuite) TestUpdateServiceGCSafePoint() { + re := suite.Require() serviceSafePoints := []struct { ServiceID string TTL int64 @@ -1269,103 +1275,103 @@ func (suite *clientTestSuite) TestUpdateServiceGCSafePoint() { for _, ssp := range serviceSafePoints { min, err := suite.client.UpdateServiceGCSafePoint(context.Background(), ssp.ServiceID, 1000, ssp.SafePoint) - suite.NoError(err) + re.NoError(err) // An service safepoint of ID "gc_worker" is automatically initialized as 0 - suite.Equal(uint64(0), min) + re.Equal(uint64(0), min) } min, err := suite.client.UpdateServiceGCSafePoint(context.Background(), "gc_worker", math.MaxInt64, 10) - suite.NoError(err) - suite.Equal(uint64(1), min) + re.NoError(err) + re.Equal(uint64(1), min) min, err = suite.client.UpdateServiceGCSafePoint(context.Background(), "a", 1000, 4) - suite.NoError(err) - suite.Equal(uint64(2), min) + re.NoError(err) + re.Equal(uint64(2), min) min, err = suite.client.UpdateServiceGCSafePoint(context.Background(), "b", -100, 2) - suite.NoError(err) - suite.Equal(uint64(3), min) + re.NoError(err) + re.Equal(uint64(3), min) // Minimum safepoint does not regress min, err = suite.client.UpdateServiceGCSafePoint(context.Background(), "b", 1000, 2) - suite.NoError(err) - suite.Equal(uint64(3), min) + re.NoError(err) + re.Equal(uint64(3), min) // Update only the TTL of the minimum safepoint oldMinSsp, err := suite.srv.GetStorage().LoadMinServiceGCSafePoint(time.Now()) - suite.NoError(err) - suite.Equal("c", oldMinSsp.ServiceID) - suite.Equal(uint64(3), oldMinSsp.SafePoint) + re.NoError(err) + re.Equal("c", oldMinSsp.ServiceID) + re.Equal(uint64(3), oldMinSsp.SafePoint) min, err = suite.client.UpdateServiceGCSafePoint(context.Background(), "c", 2000, 3) - suite.NoError(err) - suite.Equal(uint64(3), min) + re.NoError(err) + re.Equal(uint64(3), min) minSsp, err := suite.srv.GetStorage().LoadMinServiceGCSafePoint(time.Now()) - suite.NoError(err) - suite.Equal("c", minSsp.ServiceID) - suite.Equal(uint64(3), oldMinSsp.SafePoint) + re.NoError(err) + re.Equal("c", minSsp.ServiceID) + re.Equal(uint64(3), oldMinSsp.SafePoint) suite.GreaterOrEqual(minSsp.ExpiredAt-oldMinSsp.ExpiredAt, int64(1000)) // Shrinking TTL is also allowed min, err = suite.client.UpdateServiceGCSafePoint(context.Background(), "c", 1, 3) - suite.NoError(err) - suite.Equal(uint64(3), min) + re.NoError(err) + re.Equal(uint64(3), min) minSsp, err = suite.srv.GetStorage().LoadMinServiceGCSafePoint(time.Now()) - suite.NoError(err) - suite.Equal("c", minSsp.ServiceID) - suite.Less(minSsp.ExpiredAt, oldMinSsp.ExpiredAt) + re.NoError(err) + re.Equal("c", minSsp.ServiceID) + re.Less(minSsp.ExpiredAt, oldMinSsp.ExpiredAt) // TTL can be infinite (represented by math.MaxInt64) min, err = suite.client.UpdateServiceGCSafePoint(context.Background(), "c", math.MaxInt64, 3) - suite.NoError(err) - suite.Equal(uint64(3), min) + re.NoError(err) + re.Equal(uint64(3), min) minSsp, err = suite.srv.GetStorage().LoadMinServiceGCSafePoint(time.Now()) - suite.NoError(err) - suite.Equal("c", minSsp.ServiceID) - suite.Equal(minSsp.ExpiredAt, int64(math.MaxInt64)) + re.NoError(err) + re.Equal("c", minSsp.ServiceID) + re.Equal(minSsp.ExpiredAt, int64(math.MaxInt64)) // Delete "a" and "c" min, err = suite.client.UpdateServiceGCSafePoint(context.Background(), "c", -1, 3) - suite.NoError(err) - suite.Equal(uint64(4), min) + re.NoError(err) + re.Equal(uint64(4), min) min, err = suite.client.UpdateServiceGCSafePoint(context.Background(), "a", -1, 4) - suite.NoError(err) + re.NoError(err) // Now gc_worker is the only remaining service safe point. - suite.Equal(uint64(10), min) + re.Equal(uint64(10), min) // gc_worker cannot be deleted. _, err = suite.client.UpdateServiceGCSafePoint(context.Background(), "gc_worker", -1, 10) - suite.Error(err) + re.Error(err) // Cannot set non-infinity TTL for gc_worker _, err = suite.client.UpdateServiceGCSafePoint(context.Background(), "gc_worker", 10000000, 10) - suite.Error(err) + re.Error(err) // Service safepoint must have a non-empty ID _, err = suite.client.UpdateServiceGCSafePoint(context.Background(), "", 1000, 15) - suite.Error(err) + re.Error(err) // Put some other safepoints to test fixing gc_worker's safepoint when there exists other safepoints. _, err = suite.client.UpdateServiceGCSafePoint(context.Background(), "a", 1000, 11) - suite.NoError(err) + re.NoError(err) _, err = suite.client.UpdateServiceGCSafePoint(context.Background(), "b", 1000, 12) - suite.NoError(err) + re.NoError(err) _, err = suite.client.UpdateServiceGCSafePoint(context.Background(), "c", 1000, 13) - suite.NoError(err) + re.NoError(err) // Force set invalid ttl to gc_worker gcWorkerKey := path.Join("gc", "safe_point", "service", "gc_worker") @@ -1376,38 +1382,39 @@ func (suite *clientTestSuite) TestUpdateServiceGCSafePoint() { SafePoint: 10, } value, err := json.Marshal(gcWorkerSsp) - suite.NoError(err) + re.NoError(err) err = suite.srv.GetStorage().Save(gcWorkerKey, string(value)) - suite.NoError(err) + re.NoError(err) } minSsp, err = suite.srv.GetStorage().LoadMinServiceGCSafePoint(time.Now()) - suite.NoError(err) - suite.Equal("gc_worker", minSsp.ServiceID) - suite.Equal(uint64(10), minSsp.SafePoint) - suite.Equal(int64(math.MaxInt64), minSsp.ExpiredAt) + re.NoError(err) + re.Equal("gc_worker", minSsp.ServiceID) + re.Equal(uint64(10), minSsp.SafePoint) + re.Equal(int64(math.MaxInt64), minSsp.ExpiredAt) // Force delete gc_worker, then the min service safepoint is 11 of "a". err = suite.srv.GetStorage().Remove(gcWorkerKey) - suite.NoError(err) + re.NoError(err) minSsp, err = suite.srv.GetStorage().LoadMinServiceGCSafePoint(time.Now()) - suite.NoError(err) - suite.Equal(uint64(11), minSsp.SafePoint) + re.NoError(err) + re.Equal(uint64(11), minSsp.SafePoint) // After calling LoadMinServiceGCS when "gc_worker"'s service safepoint is missing, "gc_worker"'s service safepoint // will be newly created. // Increase "a" so that "gc_worker" is the only minimum that will be returned by LoadMinServiceGCSafePoint. _, err = suite.client.UpdateServiceGCSafePoint(context.Background(), "a", 1000, 14) - suite.NoError(err) + re.NoError(err) minSsp, err = suite.srv.GetStorage().LoadMinServiceGCSafePoint(time.Now()) - suite.NoError(err) - suite.Equal("gc_worker", minSsp.ServiceID) - suite.Equal(uint64(11), minSsp.SafePoint) - suite.Equal(int64(math.MaxInt64), minSsp.ExpiredAt) + re.NoError(err) + re.Equal("gc_worker", minSsp.ServiceID) + re.Equal(uint64(11), minSsp.SafePoint) + re.Equal(int64(math.MaxInt64), minSsp.ExpiredAt) } func (suite *clientTestSuite) TestScatterRegion() { + re := suite.Require() CreateRegion := func() uint64 { regionID := regionIDAllocator.alloc() region := &metapb.Region{ @@ -1426,13 +1433,12 @@ func (suite *clientTestSuite) TestScatterRegion() { Leader: peers[0], } err := suite.regionHeartbeat.Send(req) - suite.NoError(err) + re.NoError(err) return regionID } var regionID = CreateRegion() regionsID := []uint64{regionID} // Test interface `ScatterRegions`. - re := suite.Require() testutil.Eventually(re, func() bool { scatterResp, err := suite.client.ScatterRegions(context.Background(), regionsID, pd.WithGroup("test"), pd.WithRetry(1)) if err != nil { diff --git a/tests/integrations/client/gc_client_test.go b/tests/integrations/client/gc_client_test.go index 24ee8506efd..a2c3c3263f7 100644 --- a/tests/integrations/client/gc_client_test.go +++ b/tests/integrations/client/gc_client_test.go @@ -66,24 +66,28 @@ func TestGcClientTestSuite(t *testing.T) { } func (suite *gcClientTestSuite) SetupSuite() { - var err error - var gsi *server.Server + re := suite.Require() + var ( + err error + gsi *server.Server + ) checker := assertutil.NewChecker() checker.FailNow = func() {} - gsi, suite.cleanup, err = server.NewTestServer(suite.Require(), checker) + gsi, suite.cleanup, err = server.NewTestServer(re, checker) suite.server = &server.GrpcServer{Server: gsi} - suite.NoError(err) + re.NoError(err) addr := suite.server.GetAddr() suite.client, err = pd.NewClientWithContext(suite.server.Context(), []string{addr}, pd.SecurityOption{}) - suite.NoError(err) + re.NoError(err) rootPath := path.Join("/pd", strconv.FormatUint(suite.server.ClusterID(), 10)) suite.gcSafePointV2Prefix = path.Join(rootPath, endpoint.GCSafePointV2Prefix()) // Enable the fail-point to skip checking keyspace validity. - suite.NoError(failpoint.Enable("github.com/tikv/pd/pkg/gc/checkKeyspace", "return(true)")) + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/gc/checkKeyspace", "return(true)")) } func (suite *gcClientTestSuite) TearDownSuite() { - suite.NoError(failpoint.Disable("github.com/tikv/pd/pkg/gc/checkKeyspace")) + re := suite.Require() + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/gc/checkKeyspace")) suite.cleanup() } @@ -92,11 +96,13 @@ func (suite *gcClientTestSuite) TearDownTest() { } func (suite *gcClientTestSuite) CleanupEtcdGCPath() { + re := suite.Require() _, err := suite.server.GetClient().Delete(suite.server.Context(), suite.gcSafePointV2Prefix, clientv3.WithPrefix()) - suite.NoError(err) + re.NoError(err) } func (suite *gcClientTestSuite) TestWatch1() { + re := suite.Require() receiver := gcClientTestReceiver{re: suite.Require()} go suite.server.WatchGCSafePointV2(&pdpb.WatchGCSafePointV2Request{ Revision: 0, @@ -104,22 +110,22 @@ func (suite *gcClientTestSuite) TestWatch1() { // Init gc safe points as index value of keyspace 0 ~ 5. for i := 0; i < 6; i++ { - suite.mustUpdateSafePoint(uint32(i), uint64(i)) + suite.mustUpdateSafePoint(re, uint32(i), uint64(i)) } // delete gc safe points of keyspace 3 ~ 5. for i := 3; i < 6; i++ { - suite.mustDeleteSafePoint(uint32(i)) + suite.mustDeleteSafePoint(re, uint32(i)) } // check gc safe point equal to keyspace id for keyspace 0 ~ 2 . for i := 0; i < 3; i++ { - suite.Equal(uint64(i), suite.mustLoadSafePoint(uint32(i))) + re.Equal(uint64(i), suite.mustLoadSafePoint(re, uint32(i))) } // check gc safe point is 0 for keyspace 3 ~ 5 after delete. for i := 3; i < 6; i++ { - suite.Equal(uint64(0), suite.mustLoadSafePoint(uint32(i))) + re.Equal(uint64(0), suite.mustLoadSafePoint(re, uint32(i))) } } @@ -129,21 +135,22 @@ func (suite *gcClientTestSuite) TestClientWatchWithRevision() { } func (suite *gcClientTestSuite) testClientWatchWithRevision(fromNewRevision bool) { + re := suite.Require() testKeyspaceID := uint32(100) initGCSafePoint := uint64(50) updatedGCSafePoint := uint64(100) // Init gc safe point. - suite.mustUpdateSafePoint(testKeyspaceID, initGCSafePoint) + suite.mustUpdateSafePoint(re, testKeyspaceID, initGCSafePoint) // Get the initial revision. - initRevision := suite.mustGetRevision(testKeyspaceID) + initRevision := suite.mustGetRevision(re, testKeyspaceID) // Update the gc safe point. - suite.mustUpdateSafePoint(testKeyspaceID, updatedGCSafePoint) + suite.mustUpdateSafePoint(re, testKeyspaceID, updatedGCSafePoint) // Get the revision of the updated gc safe point. - updatedRevision := suite.mustGetRevision(testKeyspaceID) + updatedRevision := suite.mustGetRevision(re, testKeyspaceID) // Set the start revision of the watch request based on fromNewRevision. startRevision := initRevision @@ -151,7 +158,7 @@ func (suite *gcClientTestSuite) testClientWatchWithRevision(fromNewRevision bool startRevision = updatedRevision } watchChan, err := suite.client.WatchGCSafePointV2(suite.server.Context(), startRevision) - suite.NoError(err) + re.NoError(err) timer := time.NewTimer(time.Second) defer timer.Stop() @@ -160,19 +167,19 @@ func (suite *gcClientTestSuite) testClientWatchWithRevision(fromNewRevision bool for { select { case <-timer.C: - suite.True(runTest) + re.True(runTest) return case res := <-watchChan: for _, r := range res { - suite.Equal(r.GetKeyspaceId(), testKeyspaceID) + re.Equal(r.GetKeyspaceId(), testKeyspaceID) if fromNewRevision { // If fromNewRevision, first response should be the updated gc safe point. - suite.Equal(r.GetSafePoint(), updatedGCSafePoint) + re.Equal(r.GetSafePoint(), updatedGCSafePoint) } else if isFirstUpdate { isFirstUpdate = false - suite.Equal(r.GetSafePoint(), initGCSafePoint) + re.Equal(r.GetSafePoint(), initGCSafePoint) } else { - suite.Equal(r.GetSafePoint(), updatedGCSafePoint) + re.Equal(r.GetSafePoint(), updatedGCSafePoint) continue } } @@ -182,30 +189,30 @@ func (suite *gcClientTestSuite) testClientWatchWithRevision(fromNewRevision bool } // mustUpdateSafePoint updates the gc safe point of the given keyspace id. -func (suite *gcClientTestSuite) mustUpdateSafePoint(keyspaceID uint32, safePoint uint64) { +func (suite *gcClientTestSuite) mustUpdateSafePoint(re *require.Assertions, keyspaceID uint32, safePoint uint64) { _, err := suite.client.UpdateGCSafePointV2(suite.server.Context(), keyspaceID, safePoint) - suite.NoError(err) + re.NoError(err) } // mustLoadSafePoint loads the gc safe point of the given keyspace id. -func (suite *gcClientTestSuite) mustLoadSafePoint(keyspaceID uint32) uint64 { +func (suite *gcClientTestSuite) mustLoadSafePoint(re *require.Assertions, keyspaceID uint32) uint64 { res, err := suite.server.GetSafePointV2Manager().LoadGCSafePoint(keyspaceID) - suite.NoError(err) + re.NoError(err) return res.SafePoint } // mustDeleteSafePoint deletes the gc safe point of the given keyspace id. -func (suite *gcClientTestSuite) mustDeleteSafePoint(keyspaceID uint32) { +func (suite *gcClientTestSuite) mustDeleteSafePoint(re *require.Assertions, keyspaceID uint32) { safePointPath := path.Join(suite.gcSafePointV2Prefix, endpoint.EncodeKeyspaceID(keyspaceID)) log.Info("test etcd path", zap.Any("path", safePointPath)) // TODO: Delete _, err := suite.server.GetClient().Delete(suite.server.Context(), safePointPath) - suite.NoError(err) + re.NoError(err) } // mustGetRevision gets the revision of the given keyspace's gc safe point. -func (suite *gcClientTestSuite) mustGetRevision(keyspaceID uint32) int64 { +func (suite *gcClientTestSuite) mustGetRevision(re *require.Assertions, keyspaceID uint32) int64 { safePointPath := path.Join(suite.gcSafePointV2Prefix, endpoint.EncodeKeyspaceID(keyspaceID)) res, err := suite.server.GetClient().Get(suite.server.Context(), safePointPath) - suite.NoError(err) + re.NoError(err) return res.Header.GetRevision() } diff --git a/tests/integrations/client/global_config_test.go b/tests/integrations/client/global_config_test.go index bfe432807ce..349b16579bd 100644 --- a/tests/integrations/client/global_config_test.go +++ b/tests/integrations/client/global_config_test.go @@ -62,16 +62,19 @@ func TestGlobalConfigTestSuite(t *testing.T) { } func (suite *globalConfigTestSuite) SetupSuite() { - var err error - var gsi *server.Server + re := suite.Require() + var ( + err error + gsi *server.Server + ) checker := assertutil.NewChecker() checker.FailNow = func() {} - gsi, suite.cleanup, err = server.NewTestServer(suite.Require(), checker) + gsi, suite.cleanup, err = server.NewTestServer(re, checker) suite.server = &server.GrpcServer{Server: gsi} - suite.NoError(err) + re.NoError(err) addr := suite.server.GetAddr() suite.client, err = pd.NewClientWithContext(suite.server.Context(), []string{addr}, pd.SecurityOption{}) - suite.NoError(err) + re.NoError(err) } func (suite *globalConfigTestSuite) TearDownSuite() { @@ -84,65 +87,69 @@ func (suite *globalConfigTestSuite) GetEtcdPath(configPath string) string { } func (suite *globalConfigTestSuite) TestLoadWithoutNames() { + re := suite.Require() defer func() { // clean up _, err := suite.server.GetClient().Delete(suite.server.Context(), suite.GetEtcdPath("test")) - suite.NoError(err) + re.NoError(err) }() r, err := suite.server.GetClient().Put(suite.server.Context(), suite.GetEtcdPath("test"), "test") - suite.NoError(err) + re.NoError(err) res, err := suite.server.LoadGlobalConfig(suite.server.Context(), &pdpb.LoadGlobalConfigRequest{ ConfigPath: globalConfigPath, }) - suite.NoError(err) - suite.Len(res.Items, 1) + re.NoError(err) + re.Len(res.Items, 1) suite.LessOrEqual(r.Header.GetRevision(), res.Revision) - suite.Equal("test", string(res.Items[0].Payload)) + re.Equal("test", string(res.Items[0].Payload)) } func (suite *globalConfigTestSuite) TestLoadWithoutConfigPath() { + re := suite.Require() defer func() { // clean up _, err := suite.server.GetClient().Delete(suite.server.Context(), suite.GetEtcdPath("source_id")) - suite.NoError(err) + re.NoError(err) }() _, err := suite.server.GetClient().Put(suite.server.Context(), suite.GetEtcdPath("source_id"), "1") - suite.NoError(err) + re.NoError(err) res, err := suite.server.LoadGlobalConfig(suite.server.Context(), &pdpb.LoadGlobalConfigRequest{ Names: []string{"source_id"}, }) - suite.NoError(err) - suite.Len(res.Items, 1) - suite.Equal([]byte("1"), res.Items[0].Payload) + re.NoError(err) + re.Len(res.Items, 1) + re.Equal([]byte("1"), res.Items[0].Payload) } func (suite *globalConfigTestSuite) TestLoadOtherConfigPath() { + re := suite.Require() defer func() { for i := 0; i < 3; i++ { _, err := suite.server.GetClient().Delete(suite.server.Context(), suite.GetEtcdPath(strconv.Itoa(i))) - suite.NoError(err) + re.NoError(err) } }() for i := 0; i < 3; i++ { _, err := suite.server.GetClient().Put(suite.server.Context(), path.Join("OtherConfigPath", strconv.Itoa(i)), strconv.Itoa(i)) - suite.NoError(err) + re.NoError(err) } res, err := suite.server.LoadGlobalConfig(suite.server.Context(), &pdpb.LoadGlobalConfigRequest{ Names: []string{"0", "1"}, ConfigPath: "OtherConfigPath", }) - suite.NoError(err) - suite.Len(res.Items, 2) + re.NoError(err) + re.Len(res.Items, 2) for i, item := range res.Items { - suite.Equal(&pdpb.GlobalConfigItem{Kind: pdpb.EventType_PUT, Name: strconv.Itoa(i), Payload: []byte(strconv.Itoa(i))}, item) + re.Equal(&pdpb.GlobalConfigItem{Kind: pdpb.EventType_PUT, Name: strconv.Itoa(i), Payload: []byte(strconv.Itoa(i))}, item) } } func (suite *globalConfigTestSuite) TestLoadAndStore() { + re := suite.Require() defer func() { for i := 0; i < 3; i++ { _, err := suite.server.GetClient().Delete(suite.server.Context(), suite.GetEtcdPath("test")) - suite.NoError(err) + re.NoError(err) } }() changes := []*pdpb.GlobalConfigItem{{Kind: pdpb.EventType_PUT, Name: "0", Payload: []byte("0")}, {Kind: pdpb.EventType_PUT, Name: "1", Payload: []byte("1")}, {Kind: pdpb.EventType_PUT, Name: "2", Payload: []byte("2")}} @@ -150,22 +157,23 @@ func (suite *globalConfigTestSuite) TestLoadAndStore() { ConfigPath: globalConfigPath, Changes: changes, }) - suite.NoError(err) + re.NoError(err) res, err := suite.server.LoadGlobalConfig(suite.server.Context(), &pdpb.LoadGlobalConfigRequest{ ConfigPath: globalConfigPath, }) - suite.Len(res.Items, 3) - suite.NoError(err) + re.Len(res.Items, 3) + re.NoError(err) for i, item := range res.Items { - suite.Equal(&pdpb.GlobalConfigItem{Kind: pdpb.EventType_PUT, Name: suite.GetEtcdPath(strconv.Itoa(i)), Payload: []byte(strconv.Itoa(i))}, item) + re.Equal(&pdpb.GlobalConfigItem{Kind: pdpb.EventType_PUT, Name: suite.GetEtcdPath(strconv.Itoa(i)), Payload: []byte(strconv.Itoa(i))}, item) } } func (suite *globalConfigTestSuite) TestStore() { + re := suite.Require() defer func() { for i := 0; i < 3; i++ { _, err := suite.server.GetClient().Delete(suite.server.Context(), suite.GetEtcdPath("test")) - suite.NoError(err) + re.NoError(err) } }() changes := []*pdpb.GlobalConfigItem{{Kind: pdpb.EventType_PUT, Name: "0", Payload: []byte("0")}, {Kind: pdpb.EventType_PUT, Name: "1", Payload: []byte("1")}, {Kind: pdpb.EventType_PUT, Name: "2", Payload: []byte("2")}} @@ -173,20 +181,21 @@ func (suite *globalConfigTestSuite) TestStore() { ConfigPath: globalConfigPath, Changes: changes, }) - suite.NoError(err) + re.NoError(err) for i := 0; i < 3; i++ { res, err := suite.server.GetClient().Get(suite.server.Context(), suite.GetEtcdPath(strconv.Itoa(i))) - suite.NoError(err) - suite.Equal(suite.GetEtcdPath(string(res.Kvs[0].Value)), string(res.Kvs[0].Key)) + re.NoError(err) + re.Equal(suite.GetEtcdPath(string(res.Kvs[0].Value)), string(res.Kvs[0].Key)) } } func (suite *globalConfigTestSuite) TestWatch() { + re := suite.Require() defer func() { for i := 0; i < 3; i++ { // clean up _, err := suite.server.GetClient().Delete(suite.server.Context(), suite.GetEtcdPath(strconv.Itoa(i))) - suite.NoError(err) + re.NoError(err) } }() server := testReceiver{re: suite.Require()} @@ -196,123 +205,128 @@ func (suite *globalConfigTestSuite) TestWatch() { }, server) for i := 0; i < 6; i++ { _, err := suite.server.GetClient().Put(suite.server.Context(), suite.GetEtcdPath(strconv.Itoa(i)), strconv.Itoa(i)) - suite.NoError(err) + re.NoError(err) } for i := 3; i < 6; i++ { _, err := suite.server.GetClient().Delete(suite.server.Context(), suite.GetEtcdPath(strconv.Itoa(i))) - suite.NoError(err) + re.NoError(err) } res, err := suite.server.LoadGlobalConfig(suite.server.Context(), &pdpb.LoadGlobalConfigRequest{ ConfigPath: globalConfigPath, }) - suite.Len(res.Items, 3) - suite.NoError(err) + re.Len(res.Items, 3) + re.NoError(err) } func (suite *globalConfigTestSuite) TestClientLoadWithoutNames() { + re := suite.Require() defer func() { for i := 0; i < 3; i++ { _, err := suite.server.GetClient().Delete(suite.server.Context(), suite.GetEtcdPath(strconv.Itoa(i))) - suite.NoError(err) + re.NoError(err) } }() for i := 0; i < 3; i++ { _, err := suite.server.GetClient().Put(suite.server.Context(), suite.GetEtcdPath(strconv.Itoa(i)), strconv.Itoa(i)) - suite.NoError(err) + re.NoError(err) } res, _, err := suite.client.LoadGlobalConfig(suite.server.Context(), nil, globalConfigPath) - suite.NoError(err) - suite.Len(res, 3) + re.NoError(err) + re.Len(res, 3) for i, item := range res { - suite.Equal(pd.GlobalConfigItem{EventType: pdpb.EventType_PUT, Name: suite.GetEtcdPath(strconv.Itoa(i)), PayLoad: []byte(strconv.Itoa(i)), Value: strconv.Itoa(i)}, item) + re.Equal(pd.GlobalConfigItem{EventType: pdpb.EventType_PUT, Name: suite.GetEtcdPath(strconv.Itoa(i)), PayLoad: []byte(strconv.Itoa(i)), Value: strconv.Itoa(i)}, item) } } func (suite *globalConfigTestSuite) TestClientLoadWithoutConfigPath() { + re := suite.Require() defer func() { _, err := suite.server.GetClient().Delete(suite.server.Context(), suite.GetEtcdPath("source_id")) - suite.NoError(err) + re.NoError(err) }() _, err := suite.server.GetClient().Put(suite.server.Context(), suite.GetEtcdPath("source_id"), "1") - suite.NoError(err) + re.NoError(err) res, _, err := suite.client.LoadGlobalConfig(suite.server.Context(), []string{"source_id"}, "") - suite.NoError(err) - suite.Len(res, 1) - suite.Equal(pd.GlobalConfigItem{EventType: pdpb.EventType_PUT, Name: "source_id", PayLoad: []byte("1"), Value: "1"}, res[0]) + re.NoError(err) + re.Len(res, 1) + re.Equal(pd.GlobalConfigItem{EventType: pdpb.EventType_PUT, Name: "source_id", PayLoad: []byte("1"), Value: "1"}, res[0]) } func (suite *globalConfigTestSuite) TestClientLoadOtherConfigPath() { + re := suite.Require() defer func() { for i := 0; i < 3; i++ { _, err := suite.server.GetClient().Delete(suite.server.Context(), suite.GetEtcdPath(strconv.Itoa(i))) - suite.NoError(err) + re.NoError(err) } }() for i := 0; i < 3; i++ { _, err := suite.server.GetClient().Put(suite.server.Context(), path.Join("OtherConfigPath", strconv.Itoa(i)), strconv.Itoa(i)) - suite.NoError(err) + re.NoError(err) } res, _, err := suite.client.LoadGlobalConfig(suite.server.Context(), []string{"0", "1"}, "OtherConfigPath") - suite.NoError(err) - suite.Len(res, 2) + re.NoError(err) + re.Len(res, 2) for i, item := range res { - suite.Equal(pd.GlobalConfigItem{EventType: pdpb.EventType_PUT, Name: strconv.Itoa(i), PayLoad: []byte(strconv.Itoa(i)), Value: strconv.Itoa(i)}, item) + re.Equal(pd.GlobalConfigItem{EventType: pdpb.EventType_PUT, Name: strconv.Itoa(i), PayLoad: []byte(strconv.Itoa(i)), Value: strconv.Itoa(i)}, item) } } func (suite *globalConfigTestSuite) TestClientStore() { + re := suite.Require() defer func() { for i := 0; i < 3; i++ { _, err := suite.server.GetClient().Delete(suite.server.Context(), suite.GetEtcdPath(strconv.Itoa(i))) - suite.NoError(err) + re.NoError(err) } }() err := suite.client.StoreGlobalConfig(suite.server.Context(), globalConfigPath, []pd.GlobalConfigItem{{Name: "0", Value: "0"}, {Name: "1", Value: "1"}, {Name: "2", Value: "2"}}) - suite.NoError(err) + re.NoError(err) for i := 0; i < 3; i++ { res, err := suite.server.GetClient().Get(suite.server.Context(), suite.GetEtcdPath(strconv.Itoa(i))) - suite.NoError(err) - suite.Equal(suite.GetEtcdPath(string(res.Kvs[0].Value)), string(res.Kvs[0].Key)) + re.NoError(err) + re.Equal(suite.GetEtcdPath(string(res.Kvs[0].Value)), string(res.Kvs[0].Key)) } } func (suite *globalConfigTestSuite) TestClientWatchWithRevision() { + re := suite.Require() ctx := suite.server.Context() defer func() { _, err := suite.server.GetClient().Delete(ctx, suite.GetEtcdPath("test")) - suite.NoError(err) + re.NoError(err) for i := 3; i < 9; i++ { _, err := suite.server.GetClient().Delete(ctx, suite.GetEtcdPath(strconv.Itoa(i))) - suite.NoError(err) + re.NoError(err) } }() // Mock get revision by loading r, err := suite.server.GetClient().Put(ctx, suite.GetEtcdPath("test"), "test") - suite.NoError(err) + re.NoError(err) res, revision, err := suite.client.LoadGlobalConfig(ctx, nil, globalConfigPath) - suite.NoError(err) - suite.Len(res, 1) + re.NoError(err) + re.Len(res, 1) suite.LessOrEqual(r.Header.GetRevision(), revision) - suite.Equal(pd.GlobalConfigItem{EventType: pdpb.EventType_PUT, Name: suite.GetEtcdPath("test"), PayLoad: []byte("test"), Value: "test"}, res[0]) + re.Equal(pd.GlobalConfigItem{EventType: pdpb.EventType_PUT, Name: suite.GetEtcdPath("test"), PayLoad: []byte("test"), Value: "test"}, res[0]) // Mock when start watcher there are existed some keys, will load firstly for i := 0; i < 6; i++ { _, err = suite.server.GetClient().Put(suite.server.Context(), suite.GetEtcdPath(strconv.Itoa(i)), strconv.Itoa(i)) - suite.NoError(err) + re.NoError(err) } // Start watcher at next revision configChan, err := suite.client.WatchGlobalConfig(suite.server.Context(), globalConfigPath, revision) - suite.NoError(err) + re.NoError(err) // Mock delete for i := 0; i < 3; i++ { _, err = suite.server.GetClient().Delete(suite.server.Context(), suite.GetEtcdPath(strconv.Itoa(i))) - suite.NoError(err) + re.NoError(err) } // Mock put for i := 6; i < 9; i++ { _, err = suite.server.GetClient().Put(suite.server.Context(), suite.GetEtcdPath(strconv.Itoa(i)), strconv.Itoa(i)) - suite.NoError(err) + re.NoError(err) } timer := time.NewTimer(time.Second) defer timer.Stop() @@ -320,11 +334,11 @@ func (suite *globalConfigTestSuite) TestClientWatchWithRevision() { for { select { case <-timer.C: - suite.True(runTest) + re.True(runTest) return case res := <-configChan: for _, r := range res { - suite.Equal(suite.GetEtcdPath(r.Value), r.Name) + re.Equal(suite.GetEtcdPath(r.Value), r.Name) } runTest = true } @@ -332,6 +346,7 @@ func (suite *globalConfigTestSuite) TestClientWatchWithRevision() { } func (suite *globalConfigTestSuite) TestEtcdNotStart() { + re := suite.Require() cli := suite.server.GetClient() defer func() { suite.mu.Lock() @@ -345,16 +360,16 @@ func (suite *globalConfigTestSuite) TestEtcdNotStart() { ConfigPath: globalConfigPath, Revision: 0, }, nil) - suite.Error(err) + re.Error(err) _, err = suite.server.StoreGlobalConfig(suite.server.Context(), &pdpb.StoreGlobalConfigRequest{ ConfigPath: globalConfigPath, Changes: []*pdpb.GlobalConfigItem{{Kind: pdpb.EventType_PUT, Name: "0", Payload: []byte("0")}}, }) - suite.Error(err) + re.Error(err) _, err = suite.server.LoadGlobalConfig(suite.server.Context(), &pdpb.LoadGlobalConfigRequest{ Names: []string{"test_etcd"}, }) - suite.Error(err) + re.Error(err) } diff --git a/tests/integrations/client/http_client_test.go b/tests/integrations/client/http_client_test.go index 12064c989a1..1ab8b0a6d48 100644 --- a/tests/integrations/client/http_client_test.go +++ b/tests/integrations/client/http_client_test.go @@ -129,7 +129,7 @@ func (suite *httpClientTestSuite) TestMeta() { EndTime: time.Now().AddDate(0, 0, 1).UnixNano() / int64(time.Millisecond), }) re.NoError(err) - re.Len(historyHorRegions.HistoryHotRegion, 0) + re.Empty(historyHorRegions.HistoryHotRegion) store, err := suite.client.GetStores(suite.ctx) re.NoError(err) re.Equal(1, store.Count) @@ -179,7 +179,7 @@ func (suite *httpClientTestSuite) TestRule() { bundles, err := suite.client.GetAllPlacementRuleBundles(suite.ctx) re.NoError(err) re.Len(bundles, 1) - re.Equal(bundles[0].ID, placement.DefaultGroupID) + re.Equal(placement.DefaultGroupID, bundles[0].ID) bundle, err := suite.client.GetPlacementRuleBundleByGroup(suite.ctx, placement.DefaultGroupID) re.NoError(err) re.Equal(bundles[0], bundle) @@ -360,14 +360,14 @@ func (suite *httpClientTestSuite) TestAccelerateSchedule() { re := suite.Require() raftCluster := suite.cluster.GetLeaderServer().GetRaftCluster() suspectRegions := raftCluster.GetSuspectRegions() - re.Len(suspectRegions, 0) + re.Empty(suspectRegions) err := suite.client.AccelerateSchedule(suite.ctx, pd.NewKeyRange([]byte("a1"), []byte("a2"))) re.NoError(err) suspectRegions = raftCluster.GetSuspectRegions() re.Len(suspectRegions, 1) raftCluster.ClearSuspectRegions() suspectRegions = raftCluster.GetSuspectRegions() - re.Len(suspectRegions, 0) + re.Empty(suspectRegions) err = suite.client.AccelerateScheduleInBatch(suite.ctx, []*pd.KeyRange{ pd.NewKeyRange([]byte("a1"), []byte("a2")), pd.NewKeyRange([]byte("a2"), []byte("a3")), @@ -396,7 +396,7 @@ func (suite *httpClientTestSuite) TestSchedulers() { re := suite.Require() schedulers, err := suite.client.GetSchedulers(suite.ctx) re.NoError(err) - re.Len(schedulers, 0) + re.Empty(schedulers) err = suite.client.CreateScheduler(suite.ctx, "evict-leader-scheduler", 1) re.NoError(err) diff --git a/tests/integrations/mcs/discovery/register_test.go b/tests/integrations/mcs/discovery/register_test.go index 2cdcccedddd..1b61a264232 100644 --- a/tests/integrations/mcs/discovery/register_test.go +++ b/tests/integrations/mcs/discovery/register_test.go @@ -96,10 +96,10 @@ func (suite *serverRegisterTestSuite) checkServerRegister(serviceName string) { re.Equal(addr, returnedEntry.ServiceAddr) // test primary when only one server - expectedPrimary := tests.WaitForPrimaryServing(suite.Require(), map[string]bs.Server{addr: s}) + expectedPrimary := tests.WaitForPrimaryServing(re, map[string]bs.Server{addr: s}) primary, exist := suite.pdLeader.GetServer().GetServicePrimaryAddr(suite.ctx, serviceName) re.True(exist) - re.Equal(primary, expectedPrimary) + re.Equal(expectedPrimary, primary) // test API server discovery after unregister cleanup() @@ -130,7 +130,7 @@ func (suite *serverRegisterTestSuite) checkServerPrimaryChange(serviceName strin serverMap[s.GetAddr()] = s } - expectedPrimary := tests.WaitForPrimaryServing(suite.Require(), serverMap) + expectedPrimary := tests.WaitForPrimaryServing(re, serverMap) primary, exist = suite.pdLeader.GetServer().GetServicePrimaryAddr(suite.ctx, serviceName) re.True(exist) re.Equal(expectedPrimary, primary) @@ -138,7 +138,7 @@ func (suite *serverRegisterTestSuite) checkServerPrimaryChange(serviceName strin serverMap[primary].Close() delete(serverMap, primary) - expectedPrimary = tests.WaitForPrimaryServing(suite.Require(), serverMap) + expectedPrimary = tests.WaitForPrimaryServing(re, serverMap) // test API server discovery client := suite.pdLeader.GetEtcdClient() endpoints, err := discovery.Discover(client, suite.clusterID, serviceName) diff --git a/tests/integrations/mcs/keyspace/tso_keyspace_group_test.go b/tests/integrations/mcs/keyspace/tso_keyspace_group_test.go index 7dcce498d56..ff9769cd1a5 100644 --- a/tests/integrations/mcs/keyspace/tso_keyspace_group_test.go +++ b/tests/integrations/mcs/keyspace/tso_keyspace_group_test.go @@ -25,6 +25,7 @@ import ( "time" "github.com/pingcap/failpoint" + "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" bs "github.com/tikv/pd/pkg/basicserver" "github.com/tikv/pd/pkg/mcs/utils" @@ -54,16 +55,17 @@ func TestKeyspaceGroupTestSuite(t *testing.T) { } func (suite *keyspaceGroupTestSuite) SetupTest() { - suite.NoError(failpoint.Enable("github.com/tikv/pd/pkg/keyspace/acceleratedAllocNodes", `return(true)`)) + re := suite.Require() + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/keyspace/acceleratedAllocNodes", `return(true)`)) ctx, cancel := context.WithCancel(context.Background()) suite.ctx = ctx cluster, err := tests.NewTestAPICluster(suite.ctx, 1) suite.cluster = cluster - suite.NoError(err) - suite.NoError(cluster.RunInitialServers()) - suite.NotEmpty(cluster.WaitLeader()) + re.NoError(err) + re.NoError(cluster.RunInitialServers()) + re.NotEmpty(cluster.WaitLeader()) suite.server = cluster.GetLeaderServer() - suite.NoError(suite.server.BootstrapCluster()) + re.NoError(suite.server.BootstrapCluster()) suite.backendEndpoints = suite.server.GetAddr() suite.dialClient = &http.Client{ Transport: &http.Transport{ @@ -76,20 +78,22 @@ func (suite *keyspaceGroupTestSuite) SetupTest() { } func (suite *keyspaceGroupTestSuite) TearDownTest() { + re := suite.Require() suite.cleanupFunc() suite.cluster.Destroy() - suite.NoError(failpoint.Disable("github.com/tikv/pd/pkg/keyspace/acceleratedAllocNodes")) + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/keyspace/acceleratedAllocNodes")) } func (suite *keyspaceGroupTestSuite) TestAllocNodesUpdate() { + re := suite.Require() // add three nodes. nodes := make(map[string]bs.Server) for i := 0; i < utils.DefaultKeyspaceGroupReplicaCount+1; i++ { - s, cleanup := tests.StartSingleTSOTestServer(suite.ctx, suite.Require(), suite.backendEndpoints, tempurl.Alloc()) + s, cleanup := tests.StartSingleTSOTestServer(suite.ctx, re, suite.backendEndpoints, tempurl.Alloc()) defer cleanup() nodes[s.GetAddr()] = s } - tests.WaitForPrimaryServing(suite.Require(), nodes) + tests.WaitForPrimaryServing(re, nodes) // create a keyspace group. kgs := &handlers.CreateKeyspaceGroupParams{KeyspaceGroups: []*endpoint.KeyspaceGroup{ @@ -98,68 +102,69 @@ func (suite *keyspaceGroupTestSuite) TestAllocNodesUpdate() { UserKind: endpoint.Standard.String(), }, }} - code := suite.tryCreateKeyspaceGroup(kgs) - suite.Equal(http.StatusOK, code) + code := suite.tryCreateKeyspaceGroup(re, kgs) + re.Equal(http.StatusOK, code) // alloc nodes for the keyspace group. id := 1 params := &handlers.AllocNodesForKeyspaceGroupParams{ Replica: utils.DefaultKeyspaceGroupReplicaCount, } - got, code := suite.tryAllocNodesForKeyspaceGroup(id, params) - suite.Equal(http.StatusOK, code) - suite.Equal(utils.DefaultKeyspaceGroupReplicaCount, len(got)) + got, code := suite.tryAllocNodesForKeyspaceGroup(re, id, params) + re.Equal(http.StatusOK, code) + re.Len(got, utils.DefaultKeyspaceGroupReplicaCount) oldMembers := make(map[string]struct{}) for _, member := range got { - suite.Contains(nodes, member.Address) + re.Contains(nodes, member.Address) oldMembers[member.Address] = struct{}{} } // alloc node update to 3. params.Replica = utils.DefaultKeyspaceGroupReplicaCount + 1 - got, code = suite.tryAllocNodesForKeyspaceGroup(id, params) - suite.Equal(http.StatusOK, code) - suite.Equal(params.Replica, len(got)) + got, code = suite.tryAllocNodesForKeyspaceGroup(re, id, params) + re.Equal(http.StatusOK, code) + re.Len(got, params.Replica) newMembers := make(map[string]struct{}) for _, member := range got { - suite.Contains(nodes, member.Address) + re.Contains(nodes, member.Address) newMembers[member.Address] = struct{}{} } for member := range oldMembers { // old members should be in new members. - suite.Contains(newMembers, member) + re.Contains(newMembers, member) } } func (suite *keyspaceGroupTestSuite) TestAllocReplica() { + re := suite.Require() nodes := make(map[string]bs.Server) for i := 0; i < utils.DefaultKeyspaceGroupReplicaCount; i++ { - s, cleanup := tests.StartSingleTSOTestServer(suite.ctx, suite.Require(), suite.backendEndpoints, tempurl.Alloc()) + s, cleanup := tests.StartSingleTSOTestServer(suite.ctx, re, suite.backendEndpoints, tempurl.Alloc()) defer cleanup() nodes[s.GetAddr()] = s } - tests.WaitForPrimaryServing(suite.Require(), nodes) + tests.WaitForPrimaryServing(re, nodes) // miss replica. id := 1 params := &handlers.AllocNodesForKeyspaceGroupParams{} - got, code := suite.tryAllocNodesForKeyspaceGroup(id, params) - suite.Equal(http.StatusBadRequest, code) - suite.Empty(got) + got, code := suite.tryAllocNodesForKeyspaceGroup(re, id, params) + re.Equal(http.StatusBadRequest, code) + re.Empty(got) // replica is less than default replica. params = &handlers.AllocNodesForKeyspaceGroupParams{ Replica: utils.DefaultKeyspaceGroupReplicaCount - 1, } - _, code = suite.tryAllocNodesForKeyspaceGroup(id, params) - suite.Equal(http.StatusBadRequest, code) + _, code = suite.tryAllocNodesForKeyspaceGroup(re, id, params) + re.Equal(http.StatusBadRequest, code) // there is no any keyspace group. params = &handlers.AllocNodesForKeyspaceGroupParams{ Replica: utils.DefaultKeyspaceGroupReplicaCount, } - _, code = suite.tryAllocNodesForKeyspaceGroup(id, params) - suite.Equal(http.StatusBadRequest, code) + _, code = suite.tryAllocNodesForKeyspaceGroup(re, id, params) + re.Equal(http.StatusBadRequest, code) // create a keyspace group. kgs := &handlers.CreateKeyspaceGroupParams{KeyspaceGroups: []*endpoint.KeyspaceGroup{ @@ -168,79 +173,80 @@ func (suite *keyspaceGroupTestSuite) TestAllocReplica() { UserKind: endpoint.Standard.String(), }, }} - code = suite.tryCreateKeyspaceGroup(kgs) - suite.Equal(http.StatusOK, code) + code = suite.tryCreateKeyspaceGroup(re, kgs) + re.Equal(http.StatusOK, code) params = &handlers.AllocNodesForKeyspaceGroupParams{ Replica: utils.DefaultKeyspaceGroupReplicaCount, } - got, code = suite.tryAllocNodesForKeyspaceGroup(id, params) - suite.Equal(http.StatusOK, code) + got, code = suite.tryAllocNodesForKeyspaceGroup(re, id, params) + re.Equal(http.StatusOK, code) for _, member := range got { - suite.Contains(nodes, member.Address) + re.Contains(nodes, member.Address) } // the keyspace group is exist, but the replica is more than the num of nodes. params = &handlers.AllocNodesForKeyspaceGroupParams{ Replica: utils.DefaultKeyspaceGroupReplicaCount + 1, } - _, code = suite.tryAllocNodesForKeyspaceGroup(id, params) - suite.Equal(http.StatusBadRequest, code) + _, code = suite.tryAllocNodesForKeyspaceGroup(re, id, params) + re.Equal(http.StatusBadRequest, code) // the keyspace group is exist, the new replica is more than the old replica. - s2, cleanup2 := tests.StartSingleTSOTestServer(suite.ctx, suite.Require(), suite.backendEndpoints, tempurl.Alloc()) + s2, cleanup2 := tests.StartSingleTSOTestServer(suite.ctx, re, suite.backendEndpoints, tempurl.Alloc()) defer cleanup2() nodes[s2.GetAddr()] = s2 - tests.WaitForPrimaryServing(suite.Require(), nodes) + tests.WaitForPrimaryServing(re, nodes) params = &handlers.AllocNodesForKeyspaceGroupParams{ Replica: utils.DefaultKeyspaceGroupReplicaCount + 1, } - got, code = suite.tryAllocNodesForKeyspaceGroup(id, params) - suite.Equal(http.StatusOK, code) + got, code = suite.tryAllocNodesForKeyspaceGroup(re, id, params) + re.Equal(http.StatusOK, code) for _, member := range got { - suite.Contains(nodes, member.Address) + re.Contains(nodes, member.Address) } // the keyspace group is exist, the new replica is equal to the old replica. params = &handlers.AllocNodesForKeyspaceGroupParams{ Replica: utils.DefaultKeyspaceGroupReplicaCount + 1, } - _, code = suite.tryAllocNodesForKeyspaceGroup(id, params) - suite.Equal(http.StatusBadRequest, code) + _, code = suite.tryAllocNodesForKeyspaceGroup(re, id, params) + re.Equal(http.StatusBadRequest, code) // the keyspace group is exist, the new replica is less than the old replica. params = &handlers.AllocNodesForKeyspaceGroupParams{ Replica: utils.DefaultKeyspaceGroupReplicaCount, } - _, code = suite.tryAllocNodesForKeyspaceGroup(id, params) - suite.Equal(http.StatusBadRequest, code) + _, code = suite.tryAllocNodesForKeyspaceGroup(re, id, params) + re.Equal(http.StatusBadRequest, code) // the keyspace group is not exist. id = 2 params = &handlers.AllocNodesForKeyspaceGroupParams{ Replica: utils.DefaultKeyspaceGroupReplicaCount, } - _, code = suite.tryAllocNodesForKeyspaceGroup(id, params) - suite.Equal(http.StatusBadRequest, code) + _, code = suite.tryAllocNodesForKeyspaceGroup(re, id, params) + re.Equal(http.StatusBadRequest, code) } func (suite *keyspaceGroupTestSuite) TestSetNodes() { + re := suite.Require() nodes := make(map[string]bs.Server) nodesList := []string{} for i := 0; i < utils.DefaultKeyspaceGroupReplicaCount; i++ { - s, cleanup := tests.StartSingleTSOTestServer(suite.ctx, suite.Require(), suite.backendEndpoints, tempurl.Alloc()) + s, cleanup := tests.StartSingleTSOTestServer(suite.ctx, re, suite.backendEndpoints, tempurl.Alloc()) defer cleanup() nodes[s.GetAddr()] = s nodesList = append(nodesList, s.GetAddr()) } - tests.WaitForPrimaryServing(suite.Require(), nodes) + tests.WaitForPrimaryServing(re, nodes) // the keyspace group is not exist. id := 1 params := &handlers.SetNodesForKeyspaceGroupParams{ Nodes: nodesList, } - _, code := suite.trySetNodesForKeyspaceGroup(id, params) - suite.Equal(http.StatusBadRequest, code) + _, code := suite.trySetNodesForKeyspaceGroup(re, id, params) + re.Equal(http.StatusBadRequest, code) // the keyspace group is exist. kgs := &handlers.CreateKeyspaceGroupParams{KeyspaceGroups: []*endpoint.KeyspaceGroup{ @@ -249,120 +255,121 @@ func (suite *keyspaceGroupTestSuite) TestSetNodes() { UserKind: endpoint.Standard.String(), }, }} - code = suite.tryCreateKeyspaceGroup(kgs) - suite.Equal(http.StatusOK, code) + code = suite.tryCreateKeyspaceGroup(re, kgs) + re.Equal(http.StatusOK, code) params = &handlers.SetNodesForKeyspaceGroupParams{ Nodes: nodesList, } - kg, code := suite.trySetNodesForKeyspaceGroup(id, params) - suite.Equal(http.StatusOK, code) - suite.Len(kg.Members, 2) + kg, code := suite.trySetNodesForKeyspaceGroup(re, id, params) + re.Equal(http.StatusOK, code) + re.Len(kg.Members, 2) for _, member := range kg.Members { - suite.Contains(nodes, member.Address) + re.Contains(nodes, member.Address) } // the keyspace group is exist, but the nodes is not exist. params = &handlers.SetNodesForKeyspaceGroupParams{ Nodes: append(nodesList, "pingcap.com:2379"), } - _, code = suite.trySetNodesForKeyspaceGroup(id, params) - suite.Equal(http.StatusBadRequest, code) + _, code = suite.trySetNodesForKeyspaceGroup(re, id, params) + re.Equal(http.StatusBadRequest, code) // the keyspace group is exist, but the count of nodes is less than the default replica. params = &handlers.SetNodesForKeyspaceGroupParams{ Nodes: []string{nodesList[0]}, } - _, code = suite.trySetNodesForKeyspaceGroup(id, params) - suite.Equal(http.StatusBadRequest, code) + _, code = suite.trySetNodesForKeyspaceGroup(re, id, params) + re.Equal(http.StatusBadRequest, code) // the keyspace group is not exist. id = 2 params = &handlers.SetNodesForKeyspaceGroupParams{ Nodes: nodesList, } - _, code = suite.trySetNodesForKeyspaceGroup(id, params) - suite.Equal(http.StatusBadRequest, code) + _, code = suite.trySetNodesForKeyspaceGroup(re, id, params) + re.Equal(http.StatusBadRequest, code) } func (suite *keyspaceGroupTestSuite) TestDefaultKeyspaceGroup() { + re := suite.Require() nodes := make(map[string]bs.Server) for i := 0; i < utils.DefaultKeyspaceGroupReplicaCount; i++ { - s, cleanup := tests.StartSingleTSOTestServer(suite.ctx, suite.Require(), suite.backendEndpoints, tempurl.Alloc()) + s, cleanup := tests.StartSingleTSOTestServer(suite.ctx, re, suite.backendEndpoints, tempurl.Alloc()) defer cleanup() nodes[s.GetAddr()] = s } - tests.WaitForPrimaryServing(suite.Require(), nodes) + tests.WaitForPrimaryServing(re, nodes) // the default keyspace group is exist. var kg *endpoint.KeyspaceGroup var code int - testutil.Eventually(suite.Require(), func() bool { - kg, code = suite.tryGetKeyspaceGroup(utils.DefaultKeyspaceGroupID) + testutil.Eventually(re, func() bool { + kg, code = suite.tryGetKeyspaceGroup(re, utils.DefaultKeyspaceGroupID) return code == http.StatusOK && kg != nil }, testutil.WithWaitFor(time.Second*1)) - suite.Equal(utils.DefaultKeyspaceGroupID, kg.ID) + re.Equal(utils.DefaultKeyspaceGroupID, kg.ID) // the allocNodesToAllKeyspaceGroups loop will run every 100ms. - testutil.Eventually(suite.Require(), func() bool { + testutil.Eventually(re, func() bool { return len(kg.Members) == utils.DefaultKeyspaceGroupReplicaCount }) for _, member := range kg.Members { - suite.Contains(nodes, member.Address) + re.Contains(nodes, member.Address) } } -func (suite *keyspaceGroupTestSuite) tryAllocNodesForKeyspaceGroup(id int, request *handlers.AllocNodesForKeyspaceGroupParams) ([]endpoint.KeyspaceGroupMember, int) { +func (suite *keyspaceGroupTestSuite) tryAllocNodesForKeyspaceGroup(re *require.Assertions, id int, request *handlers.AllocNodesForKeyspaceGroupParams) ([]endpoint.KeyspaceGroupMember, int) { data, err := json.Marshal(request) - suite.NoError(err) + re.NoError(err) httpReq, err := http.NewRequest(http.MethodPost, suite.server.GetAddr()+keyspaceGroupsPrefix+fmt.Sprintf("/%d/alloc", id), bytes.NewBuffer(data)) - suite.NoError(err) + re.NoError(err) resp, err := suite.dialClient.Do(httpReq) - suite.NoError(err) + re.NoError(err) defer resp.Body.Close() nodes := make([]endpoint.KeyspaceGroupMember, 0) if resp.StatusCode == http.StatusOK { bodyBytes, err := io.ReadAll(resp.Body) - suite.NoError(err) - suite.NoError(json.Unmarshal(bodyBytes, &nodes)) + re.NoError(err) + re.NoError(json.Unmarshal(bodyBytes, &nodes)) } return nodes, resp.StatusCode } -func (suite *keyspaceGroupTestSuite) tryCreateKeyspaceGroup(request *handlers.CreateKeyspaceGroupParams) int { +func (suite *keyspaceGroupTestSuite) tryCreateKeyspaceGroup(re *require.Assertions, request *handlers.CreateKeyspaceGroupParams) int { data, err := json.Marshal(request) - suite.NoError(err) + re.NoError(err) httpReq, err := http.NewRequest(http.MethodPost, suite.server.GetAddr()+keyspaceGroupsPrefix, bytes.NewBuffer(data)) - suite.NoError(err) + re.NoError(err) resp, err := suite.dialClient.Do(httpReq) - suite.NoError(err) + re.NoError(err) defer resp.Body.Close() return resp.StatusCode } -func (suite *keyspaceGroupTestSuite) tryGetKeyspaceGroup(id uint32) (*endpoint.KeyspaceGroup, int) { +func (suite *keyspaceGroupTestSuite) tryGetKeyspaceGroup(re *require.Assertions, id uint32) (*endpoint.KeyspaceGroup, int) { httpReq, err := http.NewRequest(http.MethodGet, suite.server.GetAddr()+keyspaceGroupsPrefix+fmt.Sprintf("/%d", id), http.NoBody) - suite.NoError(err) + re.NoError(err) resp, err := suite.dialClient.Do(httpReq) - suite.NoError(err) + re.NoError(err) defer resp.Body.Close() kg := &endpoint.KeyspaceGroup{} if resp.StatusCode == http.StatusOK { bodyBytes, err := io.ReadAll(resp.Body) - suite.NoError(err) - suite.NoError(json.Unmarshal(bodyBytes, kg)) + re.NoError(err) + re.NoError(json.Unmarshal(bodyBytes, kg)) } return kg, resp.StatusCode } -func (suite *keyspaceGroupTestSuite) trySetNodesForKeyspaceGroup(id int, request *handlers.SetNodesForKeyspaceGroupParams) (*endpoint.KeyspaceGroup, int) { +func (suite *keyspaceGroupTestSuite) trySetNodesForKeyspaceGroup(re *require.Assertions, id int, request *handlers.SetNodesForKeyspaceGroupParams) (*endpoint.KeyspaceGroup, int) { data, err := json.Marshal(request) - suite.NoError(err) + re.NoError(err) httpReq, err := http.NewRequest(http.MethodPatch, suite.server.GetAddr()+keyspaceGroupsPrefix+fmt.Sprintf("/%d", id), bytes.NewBuffer(data)) - suite.NoError(err) + re.NoError(err) resp, err := suite.dialClient.Do(httpReq) - suite.NoError(err) + re.NoError(err) defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return nil, resp.StatusCode } - return suite.tryGetKeyspaceGroup(uint32(id)) + return suite.tryGetKeyspaceGroup(re, uint32(id)) } diff --git a/tests/integrations/mcs/members/member_test.go b/tests/integrations/mcs/members/member_test.go index 048bcf72154..cadf6c9e03d 100644 --- a/tests/integrations/mcs/members/member_test.go +++ b/tests/integrations/mcs/members/member_test.go @@ -42,39 +42,40 @@ func TestMemberTestSuite(t *testing.T) { } func (suite *memberTestSuite) SetupTest() { + re := suite.Require() ctx, cancel := context.WithCancel(context.Background()) suite.ctx = ctx cluster, err := tests.NewTestAPICluster(suite.ctx, 1) suite.cluster = cluster - suite.NoError(err) - suite.NoError(cluster.RunInitialServers()) - suite.NotEmpty(cluster.WaitLeader()) + re.NoError(err) + re.NoError(cluster.RunInitialServers()) + re.NotEmpty(cluster.WaitLeader()) suite.server = cluster.GetLeaderServer() - suite.NoError(suite.server.BootstrapCluster()) + re.NoError(suite.server.BootstrapCluster()) suite.backendEndpoints = suite.server.GetAddr() suite.dialClient = pdClient.NewClient("mcs-member-test", []string{suite.server.GetAddr()}) // TSO nodes := make(map[string]bs.Server) for i := 0; i < utils.DefaultKeyspaceGroupReplicaCount; i++ { - s, cleanup := tests.StartSingleTSOTestServer(suite.ctx, suite.Require(), suite.backendEndpoints, tempurl.Alloc()) + s, cleanup := tests.StartSingleTSOTestServer(suite.ctx, re, suite.backendEndpoints, tempurl.Alloc()) nodes[s.GetAddr()] = s suite.cleanupFunc = append(suite.cleanupFunc, func() { cleanup() }) } - tests.WaitForPrimaryServing(suite.Require(), nodes) + tests.WaitForPrimaryServing(re, nodes) // Scheduling nodes = make(map[string]bs.Server) for i := 0; i < 3; i++ { - s, cleanup := tests.StartSingleSchedulingTestServer(suite.ctx, suite.Require(), suite.backendEndpoints, tempurl.Alloc()) + s, cleanup := tests.StartSingleSchedulingTestServer(suite.ctx, re, suite.backendEndpoints, tempurl.Alloc()) nodes[s.GetAddr()] = s suite.cleanupFunc = append(suite.cleanupFunc, func() { cleanup() }) } - tests.WaitForPrimaryServing(suite.Require(), nodes) + tests.WaitForPrimaryServing(re, nodes) suite.cleanupFunc = append(suite.cleanupFunc, func() { cancel() diff --git a/tests/integrations/mcs/resourcemanager/resource_manager_test.go b/tests/integrations/mcs/resourcemanager/resource_manager_test.go index 36eb87a83db..44477ca7693 100644 --- a/tests/integrations/mcs/resourcemanager/resource_manager_test.go +++ b/tests/integrations/mcs/resourcemanager/resource_manager_test.go @@ -28,6 +28,7 @@ import ( "github.com/pingcap/failpoint" rmpb "github.com/pingcap/kvproto/pkg/resource_manager" + "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" pd "github.com/tikv/pd/client" "github.com/tikv/pd/client/resource_group/controller" @@ -76,7 +77,7 @@ func (suite *resourceManagerClientTestSuite) SetupSuite() { suite.client, err = pd.NewClientWithContext(suite.ctx, suite.cluster.GetConfig().GetClientURLs(), pd.SecurityOption{}) re.NoError(err) leader := suite.cluster.GetServer(suite.cluster.WaitLeader()) - suite.waitLeader(suite.client, leader.GetAddr()) + suite.waitLeader(re, suite.client, leader.GetAddr()) suite.initGroups = []*rmpb.ResourceGroup{ { @@ -133,11 +134,11 @@ func (suite *resourceManagerClientTestSuite) SetupSuite() { } } -func (suite *resourceManagerClientTestSuite) waitLeader(cli pd.Client, leaderAddr string) { +func (suite *resourceManagerClientTestSuite) waitLeader(re *require.Assertions, cli pd.Client, leaderAddr string) { innerCli, ok := cli.(interface{ GetServiceDiscovery() pd.ServiceDiscovery }) - suite.True(ok) - suite.NotNil(innerCli) - testutil.Eventually(suite.Require(), func() bool { + re.True(ok) + re.NotNil(innerCli) + testutil.Eventually(re, func() bool { innerCli.GetServiceDiscovery().ScheduleCheckMemberChanged() return innerCli.GetServiceDiscovery().GetServingAddr() == leaderAddr }) @@ -153,29 +154,29 @@ func (suite *resourceManagerClientTestSuite) TearDownSuite() { } func (suite *resourceManagerClientTestSuite) TearDownTest() { - suite.cleanupResourceGroups() + suite.cleanupResourceGroups(suite.Require()) } -func (suite *resourceManagerClientTestSuite) cleanupResourceGroups() { +func (suite *resourceManagerClientTestSuite) cleanupResourceGroups(re *require.Assertions) { cli := suite.client groups, err := cli.ListResourceGroups(suite.ctx) - suite.NoError(err) + re.NoError(err) for _, group := range groups { deleteResp, err := cli.DeleteResourceGroup(suite.ctx, group.GetName()) if group.Name == "default" { - suite.Contains(err.Error(), "cannot delete reserved group") + re.Contains(err.Error(), "cannot delete reserved group") continue } - suite.NoError(err) - suite.Contains(deleteResp, "Success!") + re.NoError(err) + re.Contains(deleteResp, "Success!") } } -func (suite *resourceManagerClientTestSuite) resignAndWaitLeader() { - suite.NoError(suite.cluster.ResignLeader()) +func (suite *resourceManagerClientTestSuite) resignAndWaitLeader(re *require.Assertions) { + re.NoError(suite.cluster.ResignLeader()) newLeader := suite.cluster.GetServer(suite.cluster.WaitLeader()) - suite.NotNil(newLeader) - suite.waitLeader(suite.client, newLeader.GetAddr()) + re.NotNil(newLeader) + suite.waitLeader(re, suite.client, newLeader.GetAddr()) } func (suite *resourceManagerClientTestSuite) TestWatchResourceGroup() { @@ -262,7 +263,7 @@ func (suite *resourceManagerClientTestSuite) TestWatchResourceGroup() { re.NoError(failpoint.Disable("github.com/tikv/pd/client/resource_group/controller/watchStreamError")) // Mock delete resource groups - suite.cleanupResourceGroups() + suite.cleanupResourceGroups(re) for i := 0; i < groupsNum; i++ { testutil.Eventually(re, func() bool { name := groupNamePrefix + strconv.Itoa(i) @@ -335,7 +336,7 @@ func (suite *resourceManagerClientTestSuite) TestWatchWithSingleGroupByKeyspace( return meta.RUSettings.RU.Settings.FillRate == uint64(20000) }, testutil.WithTickInterval(100*time.Millisecond)) metaKeySpace = controllerKeySpace.GetActiveResourceGroup(group.Name) - re.Equal(metaKeySpace.RUSettings.RU.Settings.FillRate, uint64(10000)) + re.Equal(uint64(10000), metaKeySpace.RUSettings.RU.Settings.FillRate) } const buffDuration = time.Millisecond * 300 @@ -639,8 +640,8 @@ func (suite *resourceManagerClientTestSuite) TestResourcePenalty() { resp := controller.NewTestResponseInfo(0, time.Duration(30), true) _, penalty, _, _, err := c.OnRequestWait(suite.ctx, resourceGroupName, req) re.NoError(err) - re.Equal(penalty.WriteBytes, 0.0) - re.Equal(penalty.TotalCpuTimeMs, 0.0) + re.Zero(penalty.WriteBytes) + re.Zero(penalty.TotalCpuTimeMs) _, err = c.OnResponse(resourceGroupName, req, resp) re.NoError(err) @@ -648,8 +649,8 @@ func (suite *resourceManagerClientTestSuite) TestResourcePenalty() { resp = controller.NewTestResponseInfo(0, time.Duration(10), true) _, penalty, _, _, err = c.OnRequestWait(suite.ctx, resourceGroupName, req) re.NoError(err) - re.Equal(penalty.WriteBytes, 0.0) - re.Equal(penalty.TotalCpuTimeMs, 0.0) + re.Zero(penalty.WriteBytes) + re.Zero(penalty.TotalCpuTimeMs) _, err = c.OnResponse(resourceGroupName, req, resp) re.NoError(err) @@ -658,8 +659,8 @@ func (suite *resourceManagerClientTestSuite) TestResourcePenalty() { resp = controller.NewTestResponseInfo(0, time.Duration(0), false) _, penalty, _, _, err = c.OnRequestWait(suite.ctx, resourceGroupName, req) re.NoError(err) - re.Equal(penalty.WriteBytes, 0.0) - re.Equal(penalty.TotalCpuTimeMs, 0.0) + re.Zero(penalty.WriteBytes) + re.Zero(penalty.TotalCpuTimeMs) _, err = c.OnResponse(resourceGroupName, req, resp) re.NoError(err) @@ -668,7 +669,7 @@ func (suite *resourceManagerClientTestSuite) TestResourcePenalty() { resp1 := controller.NewTestResponseInfo(0, time.Duration(10), true) _, penalty, _, _, err = c.OnRequestWait(suite.ctx, resourceGroupName, req1) re.NoError(err) - re.Equal(penalty.WriteBytes, 0.0) + re.Zero(penalty.WriteBytes) _, err = c.OnResponse(resourceGroupName, req1, resp1) re.NoError(err) @@ -677,7 +678,7 @@ func (suite *resourceManagerClientTestSuite) TestResourcePenalty() { resp2 := controller.NewTestResponseInfo(0, time.Duration(10), true) _, penalty, _, _, err = c.OnRequestWait(suite.ctx, resourceGroupName, req2) re.NoError(err) - re.Equal(penalty.WriteBytes, 60.0) + re.Equal(60.0, penalty.WriteBytes) re.InEpsilon(penalty.TotalCpuTimeMs, 10.0/1000.0/1000.0, 1e-6) _, err = c.OnResponse(resourceGroupName, req2, resp2) re.NoError(err) @@ -687,7 +688,7 @@ func (suite *resourceManagerClientTestSuite) TestResourcePenalty() { resp3 := controller.NewTestResponseInfo(0, time.Duration(10), true) _, penalty, _, _, err = c.OnRequestWait(suite.ctx, resourceGroupName, req3) re.NoError(err) - re.Equal(penalty.WriteBytes, 0.0) + re.Zero(penalty.WriteBytes) _, err = c.OnResponse(resourceGroupName, req3, resp3) re.NoError(err) @@ -697,7 +698,7 @@ func (suite *resourceManagerClientTestSuite) TestResourcePenalty() { resp4 := controller.NewTestResponseInfo(0, time.Duration(10), true) _, penalty, _, _, err = c.OnRequestWait(suite.ctx, resourceGroupName, req4) re.NoError(err) - re.Equal(penalty.WriteBytes, 0.0) + re.Zero(penalty.WriteBytes) _, err = c.OnResponse(resourceGroupName, req4, resp4) re.NoError(err) @@ -720,7 +721,7 @@ func (suite *resourceManagerClientTestSuite) TestAcquireTokenBucket() { TargetRequestPeriodMs: uint64(time.Second * 10 / time.Millisecond), } re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/mcs/resourcemanager/server/fastPersist", `return(true)`)) - suite.resignAndWaitLeader() + suite.resignAndWaitLeader(re) for i := 0; i < 3; i++ { for _, group := range groups { requests := make([]*rmpb.RequestUnitItem, 0) @@ -742,7 +743,7 @@ func (suite *resourceManagerClientTestSuite) TestAcquireTokenBucket() { re.NoError(err) for _, resp := range aresp { re.Len(resp.GrantedRUTokens, 1) - re.Equal(resp.GrantedRUTokens[0].GrantedTokens.Tokens, 30000.) + re.Equal(30000., resp.GrantedRUTokens[0].GrantedTokens.Tokens) if resp.ResourceGroupName == "test2" { re.Equal(int64(-1), resp.GrantedRUTokens[0].GrantedTokens.GetSettings().GetBurstLimit()) } @@ -761,7 +762,7 @@ func (suite *resourceManagerClientTestSuite) TestAcquireTokenBucket() { } time.Sleep(250 * time.Millisecond) // to test persistent - suite.resignAndWaitLeader() + suite.resignAndWaitLeader(re) gresp, err = cli.GetResourceGroup(suite.ctx, groups[0].GetName()) re.NoError(err) checkFunc(gresp, groups[0]) @@ -776,7 +777,7 @@ func (suite *resourceManagerClientTestSuite) TestAcquireTokenBucket() { aresp, err := cli.AcquireTokenBuckets(suite.ctx, reqs) re.NoError(err) for _, resp := range aresp { - re.Len(resp.GrantedRUTokens, 0) + re.Empty(resp.GrantedRUTokens) } re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/mcs/resourcemanager/server/fastPersist")) @@ -915,7 +916,7 @@ func (suite *resourceManagerClientTestSuite) TestBasicResourceGroupCURD() { // List Resource Groups lresp, err := cli.ListResourceGroups(suite.ctx) re.NoError(err) - re.Equal(finalNum, len(lresp)) + re.Len(lresp, finalNum) for _, g := range lresp { // Delete Resource Group @@ -931,11 +932,11 @@ func (suite *resourceManagerClientTestSuite) TestBasicResourceGroupCURD() { } // to test the deletion of persistence - suite.resignAndWaitLeader() + suite.resignAndWaitLeader(re) // List Resource Group lresp, err = cli.ListResourceGroups(suite.ctx) re.NoError(err) - re.Equal(1, len(lresp)) + re.Len(lresp, 1) } } @@ -1002,7 +1003,7 @@ func (suite *resourceManagerClientTestSuite) TestBasicResourceGroupCURD() { re.NoError(err) groups := make([]*server.ResourceGroup, 0) json.Unmarshal(respString, &groups) - re.Equal(finalNum, len(groups)) + re.Len(groups, finalNum) // Delete all resource groups for _, g := range groups { @@ -1030,7 +1031,7 @@ func (suite *resourceManagerClientTestSuite) TestBasicResourceGroupCURD() { re.NoError(err) groups1 := make([]server.ResourceGroup, 0) json.Unmarshal(respString1, &groups1) - re.Equal(1, len(groups1)) + re.Len(groups1, 1) } } @@ -1051,7 +1052,7 @@ func (suite *resourceManagerClientTestSuite) TestBasicResourceGroupCURD() { re.NoError(err) cli = suite.client var newGroups []*rmpb.ResourceGroup - testutil.Eventually(suite.Require(), func() bool { + testutil.Eventually(re, func() bool { var err error newGroups, err = cli.ListResourceGroups(suite.ctx) return err == nil @@ -1084,7 +1085,7 @@ func (suite *resourceManagerClientTestSuite) TestResourceManagerClientFailover() modifyResp, err := cli.ModifyResourceGroup(suite.ctx, group) re.NoError(err) re.Contains(modifyResp, "Success!") - suite.resignAndWaitLeader() + suite.resignAndWaitLeader(re) getResp, err = cli.GetResourceGroup(suite.ctx, group.GetName()) re.NoError(err) re.NotNil(getResp) diff --git a/tests/integrations/mcs/scheduling/api_test.go b/tests/integrations/mcs/scheduling/api_test.go index 4ad6680a7cd..867a7bfb5df 100644 --- a/tests/integrations/mcs/scheduling/api_test.go +++ b/tests/integrations/mcs/scheduling/api_test.go @@ -41,12 +41,14 @@ func TestAPI(t *testing.T) { } func (suite *apiTestSuite) SetupSuite() { - suite.NoError(failpoint.Enable("github.com/tikv/pd/pkg/utils/apiutil/serverapi/checkHeader", "return(true)")) + re := suite.Require() + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/utils/apiutil/serverapi/checkHeader", "return(true)")) suite.env = tests.NewSchedulingTestEnvironment(suite.T()) } func (suite *apiTestSuite) TearDownSuite() { - suite.NoError(failpoint.Disable("github.com/tikv/pd/pkg/utils/apiutil/serverapi/checkHeader")) + re := suite.Require() + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/utils/apiutil/serverapi/checkHeader")) suite.env.Cleanup() } @@ -76,23 +78,23 @@ func (suite *apiTestSuite) checkGetCheckerByName(cluster *tests.TestCluster) { // normal run resp := make(map[string]interface{}) err := testutil.ReadGetJSON(re, testDialClient, fmt.Sprintf("%s/%s", urlPrefix, name), &resp) - suite.NoError(err) - suite.False(resp["paused"].(bool)) + re.NoError(err) + re.False(resp["paused"].(bool)) // paused err = co.PauseOrResumeChecker(name, 30) - suite.NoError(err) + re.NoError(err) resp = make(map[string]interface{}) err = testutil.ReadGetJSON(re, testDialClient, fmt.Sprintf("%s/%s", urlPrefix, name), &resp) - suite.NoError(err) - suite.True(resp["paused"].(bool)) + re.NoError(err) + re.True(resp["paused"].(bool)) // resumed err = co.PauseOrResumeChecker(name, 1) - suite.NoError(err) + re.NoError(err) time.Sleep(time.Second) resp = make(map[string]interface{}) err = testutil.ReadGetJSON(re, testDialClient, fmt.Sprintf("%s/%s", urlPrefix, name), &resp) - suite.NoError(err) - suite.False(resp["paused"].(bool)) + re.NoError(err) + re.False(resp["paused"].(bool)) } } @@ -115,11 +117,11 @@ func (suite *apiTestSuite) checkAPIForward(cluster *tests.TestCluster) { err := testutil.ReadGetJSON(re, testDialClient, fmt.Sprintf("%s/%s", urlPrefix, "operators"), &slice, testutil.WithHeader(re, apiutil.ForwardToMicroServiceHeader, "true")) re.NoError(err) - re.Len(slice, 0) + re.Empty(slice) err = testutil.CheckPostJSON(testDialClient, fmt.Sprintf("%s/%s", urlPrefix, "operators"), []byte(``), testutil.StatusNotOK(re), testutil.WithHeader(re, apiutil.ForwardToMicroServiceHeader, "true")) - suite.NoError(err) + re.NoError(err) err = testutil.CheckGetJSON(testDialClient, fmt.Sprintf("%s/%s", urlPrefix, "operators/2"), nil, testutil.StatusNotOK(re), testutil.WithHeader(re, apiutil.ForwardToMicroServiceHeader, "true")) @@ -137,17 +139,17 @@ func (suite *apiTestSuite) checkAPIForward(cluster *tests.TestCluster) { err = testutil.ReadGetJSON(re, testDialClient, fmt.Sprintf("%s/%s", urlPrefix, "checker/merge"), &resp, testutil.WithHeader(re, apiutil.ForwardToMicroServiceHeader, "true")) re.NoError(err) - suite.False(resp["paused"].(bool)) + re.False(resp["paused"].(bool)) // Test pause postChecker := func(delay int) { input := make(map[string]interface{}) input["delay"] = delay pauseArgs, err := json.Marshal(input) - suite.NoError(err) + re.NoError(err) err = testutil.CheckPostJSON(testDialClient, fmt.Sprintf("%s/%s", urlPrefix, "checker/merge"), pauseArgs, testutil.StatusOK(re), testutil.WithHeader(re, apiutil.ForwardToMicroServiceHeader, "true")) - suite.NoError(err) + re.NoError(err) } postChecker(30) postChecker(0) @@ -172,21 +174,21 @@ func (suite *apiTestSuite) checkAPIForward(cluster *tests.TestCluster) { input := make(map[string]interface{}) input["delay"] = delay pauseArgs, err := json.Marshal(input) - suite.NoError(err) + re.NoError(err) err = testutil.CheckPostJSON(testDialClient, fmt.Sprintf("%s/%s", urlPrefix, "schedulers/balance-leader-scheduler"), pauseArgs, testutil.WithHeader(re, apiutil.ForwardToMicroServiceHeader, "true")) - suite.NoError(err) + re.NoError(err) } postScheduler(30) postScheduler(0) err = testutil.ReadGetJSON(re, testDialClient, fmt.Sprintf("%s/%s", urlPrefix, "schedulers/diagnostic/balance-leader-scheduler"), &resp, testutil.WithHeader(re, apiutil.ForwardToMicroServiceHeader, "true")) - suite.NoError(err) + re.NoError(err) err = testutil.ReadGetJSON(re, testDialClient, fmt.Sprintf("%s/%s", urlPrefix, "scheduler-config"), &resp, testutil.WithHeader(re, apiutil.ForwardToMicroServiceHeader, "true")) - suite.NoError(err) + re.NoError(err) re.Contains(resp, "balance-leader-scheduler") re.Contains(resp, "balance-witness-scheduler") re.Contains(resp, "balance-hot-region-scheduler") @@ -199,7 +201,7 @@ func (suite *apiTestSuite) checkAPIForward(cluster *tests.TestCluster) { for _, schedulerName := range schedulers { err = testutil.ReadGetJSON(re, testDialClient, fmt.Sprintf("%s/%s/%s/%s", urlPrefix, "scheduler-config", schedulerName, "list"), &resp, testutil.WithHeader(re, apiutil.ForwardToMicroServiceHeader, "true")) - suite.NoError(err) + re.NoError(err) } err = testutil.CheckPostJSON(testDialClient, fmt.Sprintf("%s/%s", urlPrefix, "schedulers"), nil, @@ -273,17 +275,17 @@ func (suite *apiTestSuite) checkAPIForward(cluster *tests.TestCluster) { body = fmt.Sprintf(`{"start_key":"%s", "end_key": "%s"}`, hex.EncodeToString([]byte("b1")), hex.EncodeToString([]byte("b3"))) err = testutil.CheckPostJSON(testDialClient, fmt.Sprintf("%s/%s", urlPrefix, "regions/scatter"), []byte(body), testutil.WithHeader(re, apiutil.ForwardToMicroServiceHeader, "true")) - suite.NoError(err) + re.NoError(err) body = fmt.Sprintf(`{"retry_limit":%v, "split_keys": ["%s","%s","%s"]}`, 3, hex.EncodeToString([]byte("bbb")), hex.EncodeToString([]byte("ccc")), hex.EncodeToString([]byte("ddd"))) err = testutil.CheckPostJSON(testDialClient, fmt.Sprintf("%s/%s", urlPrefix, "regions/split"), []byte(body), testutil.StatusOK(re), testutil.WithHeader(re, apiutil.ForwardToMicroServiceHeader, "true")) - suite.NoError(err) + re.NoError(err) err = testutil.CheckGetJSON(testDialClient, fmt.Sprintf(`%s/regions/replicated?startKey=%s&endKey=%s`, urlPrefix, hex.EncodeToString([]byte("a1")), hex.EncodeToString([]byte("a2"))), nil, testutil.StatusOK(re), testutil.WithHeader(re, apiutil.ForwardToMicroServiceHeader, "true")) - suite.NoError(err) + re.NoError(err) // Test rules: only forward `GET` request var rules []*placement.Rule tests.MustPutRegion(re, cluster, 2, 1, []byte("a"), []byte("b"), core.SetApproximateSize(60)) @@ -297,7 +299,7 @@ func (suite *apiTestSuite) checkAPIForward(cluster *tests.TestCluster) { }, } rulesArgs, err := json.Marshal(rules) - suite.NoError(err) + re.NoError(err) err = testutil.ReadGetJSON(re, testDialClient, fmt.Sprintf("%s/%s", urlPrefix, "config/rules"), &rules, testutil.WithHeader(re, apiutil.ForwardToMicroServiceHeader, "true")) @@ -374,22 +376,22 @@ func (suite *apiTestSuite) checkConfig(cluster *tests.TestCluster) { var cfg config.Config testutil.ReadGetJSON(re, testDialClient, urlPrefix, &cfg) - suite.Equal(cfg.GetListenAddr(), s.GetConfig().GetListenAddr()) - suite.Equal(cfg.Schedule.LeaderScheduleLimit, s.GetConfig().Schedule.LeaderScheduleLimit) - suite.Equal(cfg.Schedule.EnableCrossTableMerge, s.GetConfig().Schedule.EnableCrossTableMerge) - suite.Equal(cfg.Replication.MaxReplicas, s.GetConfig().Replication.MaxReplicas) - suite.Equal(cfg.Replication.LocationLabels, s.GetConfig().Replication.LocationLabels) - suite.Equal(cfg.DataDir, s.GetConfig().DataDir) + re.Equal(cfg.GetListenAddr(), s.GetConfig().GetListenAddr()) + re.Equal(cfg.Schedule.LeaderScheduleLimit, s.GetConfig().Schedule.LeaderScheduleLimit) + re.Equal(cfg.Schedule.EnableCrossTableMerge, s.GetConfig().Schedule.EnableCrossTableMerge) + re.Equal(cfg.Replication.MaxReplicas, s.GetConfig().Replication.MaxReplicas) + re.Equal(cfg.Replication.LocationLabels, s.GetConfig().Replication.LocationLabels) + re.Equal(cfg.DataDir, s.GetConfig().DataDir) testutil.Eventually(re, func() bool { // wait for all schedulers to be loaded in scheduling server. return len(cfg.Schedule.SchedulersPayload) == 6 }) - suite.Contains(cfg.Schedule.SchedulersPayload, "balance-leader-scheduler") - suite.Contains(cfg.Schedule.SchedulersPayload, "balance-region-scheduler") - suite.Contains(cfg.Schedule.SchedulersPayload, "balance-hot-region-scheduler") - suite.Contains(cfg.Schedule.SchedulersPayload, "balance-witness-scheduler") - suite.Contains(cfg.Schedule.SchedulersPayload, "transfer-witness-leader-scheduler") - suite.Contains(cfg.Schedule.SchedulersPayload, "evict-slow-store-scheduler") + re.Contains(cfg.Schedule.SchedulersPayload, "balance-leader-scheduler") + re.Contains(cfg.Schedule.SchedulersPayload, "balance-region-scheduler") + re.Contains(cfg.Schedule.SchedulersPayload, "balance-hot-region-scheduler") + re.Contains(cfg.Schedule.SchedulersPayload, "balance-witness-scheduler") + re.Contains(cfg.Schedule.SchedulersPayload, "transfer-witness-leader-scheduler") + re.Contains(cfg.Schedule.SchedulersPayload, "evict-slow-store-scheduler") } func (suite *apiTestSuite) TestConfigForward() { diff --git a/tests/integrations/mcs/scheduling/meta_test.go b/tests/integrations/mcs/scheduling/meta_test.go index ce0dc620aef..abc1efd9021 100644 --- a/tests/integrations/mcs/scheduling/meta_test.go +++ b/tests/integrations/mcs/scheduling/meta_test.go @@ -60,7 +60,8 @@ func (suite *metaTestSuite) SetupSuite() { } func (suite *metaTestSuite) TearDownSuite() { - suite.NoError(failpoint.Disable("github.com/tikv/pd/server/cluster/highFrequencyClusterJobs")) + re := suite.Require() + re.NoError(failpoint.Disable("github.com/tikv/pd/server/cluster/highFrequencyClusterJobs")) suite.cancel() suite.cluster.Destroy() } diff --git a/tests/integrations/mcs/scheduling/server_test.go b/tests/integrations/mcs/scheduling/server_test.go index 0c0442300d9..164c6ffdc7d 100644 --- a/tests/integrations/mcs/scheduling/server_test.go +++ b/tests/integrations/mcs/scheduling/server_test.go @@ -70,13 +70,14 @@ func (suite *serverTestSuite) SetupSuite() { leaderName := suite.cluster.WaitLeader() suite.pdLeader = suite.cluster.GetServer(leaderName) suite.backendEndpoints = suite.pdLeader.GetAddr() - suite.NoError(suite.pdLeader.BootstrapCluster()) + re.NoError(suite.pdLeader.BootstrapCluster()) } func (suite *serverTestSuite) TearDownSuite() { + re := suite.Require() suite.cluster.Destroy() suite.cancel() - suite.NoError(failpoint.Disable("github.com/tikv/pd/server/cluster/highFrequencyClusterJobs")) + re.NoError(failpoint.Disable("github.com/tikv/pd/server/cluster/highFrequencyClusterJobs")) } func (suite *serverTestSuite) TestAllocID() { diff --git a/tests/integrations/mcs/tso/api_test.go b/tests/integrations/mcs/tso/api_test.go index 94b38dd93cb..59073aca1e1 100644 --- a/tests/integrations/mcs/tso/api_test.go +++ b/tests/integrations/mcs/tso/api_test.go @@ -115,7 +115,7 @@ func (suite *tsoAPITestSuite) TestForwardResetTS() { input := []byte(`{"tso":"121312", "force-use-larger":true}`) err := testutil.CheckPostJSON(dialClient, url, input, testutil.StatusOK(re), testutil.StringContain(re, "Reset ts successfully"), testutil.WithHeader(re, apiutil.ForwardToMicroServiceHeader, "true")) - suite.NoError(err) + re.NoError(err) // Test reset ts with invalid tso input = []byte(`{}`) @@ -230,7 +230,7 @@ func TestForwardOnlyTSONoScheduling(t *testing.T) { err = testutil.ReadGetJSON(re, dialClient, fmt.Sprintf("%s/%s", urlPrefix, "operators"), &slice, testutil.WithoutHeader(re, apiutil.ForwardToMicroServiceHeader)) re.NoError(err) - re.Len(slice, 0) + re.Empty(slice) // Test admin/reset-ts, it should forward to tso server. input := []byte(`{"tso":"121312", "force-use-larger":true}`) diff --git a/tests/integrations/mcs/tso/proxy_test.go b/tests/integrations/mcs/tso/proxy_test.go index 60280fa892d..a19776ffc48 100644 --- a/tests/integrations/mcs/tso/proxy_test.go +++ b/tests/integrations/mcs/tso/proxy_test.go @@ -66,7 +66,7 @@ func (s *tsoProxyTestSuite) SetupSuite() { leaderName := s.apiCluster.WaitLeader() s.apiLeader = s.apiCluster.GetServer(leaderName) s.backendEndpoints = s.apiLeader.GetAddr() - s.NoError(s.apiLeader.BootstrapCluster()) + re.NoError(s.apiLeader.BootstrapCluster()) // Create a TSO cluster with 2 servers s.tsoCluster, err = tests.NewTestTSOCluster(s.ctx, 2, s.backendEndpoints) diff --git a/tests/integrations/mcs/tso/server_test.go b/tests/integrations/mcs/tso/server_test.go index c81c39af094..9dc9aa32368 100644 --- a/tests/integrations/mcs/tso/server_test.go +++ b/tests/integrations/mcs/tso/server_test.go @@ -78,7 +78,7 @@ func (suite *tsoServerTestSuite) SetupSuite() { leaderName := suite.cluster.WaitLeader() suite.pdLeader = suite.cluster.GetServer(leaderName) suite.backendEndpoints = suite.pdLeader.GetAddr() - suite.NoError(suite.pdLeader.BootstrapCluster()) + re.NoError(suite.pdLeader.BootstrapCluster()) } func (suite *tsoServerTestSuite) TearDownSuite() { @@ -112,7 +112,7 @@ func (suite *tsoServerTestSuite) TestTSOServerStartAndStopNormally() { input := []byte(`{"tso":"121312", "force-use-larger":true}`) err = testutil.CheckPostJSON(dialClient, url, input, testutil.StatusOK(re), testutil.StringContain(re, "Reset ts successfully")) - suite.NoError(err) + re.NoError(err) // Test reset ts with invalid tso input = []byte(`{}`) @@ -285,40 +285,42 @@ func (suite *APIServerForwardTestSuite) SetupTest() { leaderName := suite.cluster.WaitLeader() suite.pdLeader = suite.cluster.GetServer(leaderName) suite.backendEndpoints = suite.pdLeader.GetAddr() - suite.NoError(suite.pdLeader.BootstrapCluster()) + re.NoError(suite.pdLeader.BootstrapCluster()) suite.addRegions() - suite.NoError(failpoint.Enable("github.com/tikv/pd/client/usePDServiceMode", "return(true)")) + re.NoError(failpoint.Enable("github.com/tikv/pd/client/usePDServiceMode", "return(true)")) suite.pdClient, err = pd.NewClientWithContext(context.Background(), []string{suite.backendEndpoints}, pd.SecurityOption{}, pd.WithMaxErrorRetry(1)) - suite.NoError(err) + re.NoError(err) } func (suite *APIServerForwardTestSuite) TearDownTest() { + re := suite.Require() suite.pdClient.Close() etcdClient := suite.pdLeader.GetEtcdClient() clusterID := strconv.FormatUint(suite.pdLeader.GetClusterID(), 10) endpoints, err := discovery.Discover(etcdClient, clusterID, utils.TSOServiceName) - suite.NoError(err) + re.NoError(err) if len(endpoints) != 0 { endpoints, err = discovery.Discover(etcdClient, clusterID, utils.TSOServiceName) - suite.NoError(err) - suite.Empty(endpoints) + re.NoError(err) + re.Empty(endpoints) } suite.cluster.Destroy() suite.cancel() - suite.NoError(failpoint.Disable("github.com/tikv/pd/client/usePDServiceMode")) + re.NoError(failpoint.Disable("github.com/tikv/pd/client/usePDServiceMode")) } func (suite *APIServerForwardTestSuite) TestForwardTSORelated() { + re := suite.Require() // Unable to use the tso-related interface without tso server - suite.checkUnavailableTSO() + suite.checkUnavailableTSO(re) tc, err := tests.NewTestTSOCluster(suite.ctx, 1, suite.backendEndpoints) - suite.NoError(err) + re.NoError(err) defer tc.Destroy() - tc.WaitForDefaultPrimaryServing(suite.Require()) - suite.checkAvailableTSO() + tc.WaitForDefaultPrimaryServing(re) + suite.checkAvailableTSO(re) } func (suite *APIServerForwardTestSuite) TestForwardTSOWhenPrimaryChanged() { @@ -332,7 +334,7 @@ func (suite *APIServerForwardTestSuite) TestForwardTSOWhenPrimaryChanged() { // can use the tso-related interface with old primary oldPrimary, exist := suite.pdLeader.GetServer().GetServicePrimaryAddr(suite.ctx, utils.TSOServiceName) re.True(exist) - suite.checkAvailableTSO() + suite.checkAvailableTSO(re) // can use the tso-related interface with new primary tc.DestroyServer(oldPrimary) @@ -341,11 +343,11 @@ func (suite *APIServerForwardTestSuite) TestForwardTSOWhenPrimaryChanged() { primary, exist := suite.pdLeader.GetServer().GetServicePrimaryAddr(suite.ctx, utils.TSOServiceName) re.True(exist) re.NotEqual(oldPrimary, primary) - suite.checkAvailableTSO() + suite.checkAvailableTSO(re) // can use the tso-related interface with old primary again tc.AddServer(oldPrimary) - suite.checkAvailableTSO() + suite.checkAvailableTSO(re) for addr := range tc.GetServers() { if addr != oldPrimary { tc.DestroyServer(addr) @@ -356,7 +358,7 @@ func (suite *APIServerForwardTestSuite) TestForwardTSOWhenPrimaryChanged() { primary, exist = suite.pdLeader.GetServer().GetServicePrimaryAddr(suite.ctx, utils.TSOServiceName) re.True(exist) re.Equal(oldPrimary, primary) - suite.checkAvailableTSO() + suite.checkAvailableTSO(re) } func (suite *APIServerForwardTestSuite) TestResignTSOPrimaryForward() { @@ -379,8 +381,8 @@ func (suite *APIServerForwardTestSuite) TestResignTSOPrimaryForward() { } time.Sleep(100 * time.Millisecond) } - suite.NoError(err) - suite.checkAvailableTSO() + re.NoError(err) + suite.checkAvailableTSO(re) } } @@ -402,35 +404,38 @@ func (suite *APIServerForwardTestSuite) TestResignAPIPrimaryForward() { suite.pdLeader = suite.cluster.GetServer(suite.cluster.WaitLeader()) suite.backendEndpoints = suite.pdLeader.GetAddr() _, _, err = suite.pdClient.GetTS(suite.ctx) - suite.NoError(err) + re.NoError(err) } } func (suite *APIServerForwardTestSuite) TestForwardTSOUnexpectedToFollower1() { + re := suite.Require() suite.checkForwardTSOUnexpectedToFollower(func() { // unary call will retry internally // try to update gc safe point min, err := suite.pdClient.UpdateServiceGCSafePoint(context.Background(), "a", 1000, 1) - suite.NoError(err) - suite.Equal(uint64(0), min) + re.NoError(err) + re.Equal(uint64(0), min) }) } func (suite *APIServerForwardTestSuite) TestForwardTSOUnexpectedToFollower2() { + re := suite.Require() suite.checkForwardTSOUnexpectedToFollower(func() { // unary call will retry internally // try to set external ts ts, err := suite.pdClient.GetExternalTimestamp(suite.ctx) - suite.NoError(err) + re.NoError(err) err = suite.pdClient.SetExternalTimestamp(suite.ctx, ts+1) - suite.NoError(err) + re.NoError(err) }) } func (suite *APIServerForwardTestSuite) TestForwardTSOUnexpectedToFollower3() { + re := suite.Require() suite.checkForwardTSOUnexpectedToFollower(func() { _, _, err := suite.pdClient.GetTS(suite.ctx) - suite.Error(err) + re.Error(err) }) } @@ -455,18 +460,18 @@ func (suite *APIServerForwardTestSuite) checkForwardTSOUnexpectedToFollower(chec // write follower's address to cache to simulate cache is not updated. suite.pdLeader.GetServer().SetServicePrimaryAddr(utils.TSOServiceName, follower) errorAddr, ok := suite.pdLeader.GetServer().GetServicePrimaryAddr(suite.ctx, utils.TSOServiceName) - suite.True(ok) - suite.Equal(follower, errorAddr) + re.True(ok) + re.Equal(follower, errorAddr) // test tso request checkTSO() // test tso request will success after cache is updated - suite.checkAvailableTSO() + suite.checkAvailableTSO(re) newPrimary, exist2 := suite.pdLeader.GetServer().GetServicePrimaryAddr(suite.ctx, utils.TSOServiceName) - suite.True(exist2) - suite.NotEqual(errorAddr, newPrimary) - suite.Equal(oldPrimary, newPrimary) + re.True(exist2) + re.NotEqual(errorAddr, newPrimary) + re.Equal(oldPrimary, newPrimary) tc.Destroy() } @@ -484,19 +489,18 @@ func (suite *APIServerForwardTestSuite) addRegions() { } } -func (suite *APIServerForwardTestSuite) checkUnavailableTSO() { +func (suite *APIServerForwardTestSuite) checkUnavailableTSO(re *require.Assertions) { _, _, err := suite.pdClient.GetTS(suite.ctx) - suite.Error(err) + re.Error(err) // try to update gc safe point _, err = suite.pdClient.UpdateServiceGCSafePoint(suite.ctx, "a", 1000, 1) - suite.Error(err) + re.Error(err) // try to set external ts err = suite.pdClient.SetExternalTimestamp(suite.ctx, 1000) - suite.Error(err) + re.Error(err) } -func (suite *APIServerForwardTestSuite) checkAvailableTSO() { - re := suite.Require() +func (suite *APIServerForwardTestSuite) checkAvailableTSO(re *require.Assertions) { mcs.WaitForTSOServiceAvailable(suite.ctx, re, suite.pdClient) // try to get ts _, _, err := suite.pdClient.GetTS(suite.ctx) @@ -541,24 +545,25 @@ func (suite *CommonTestSuite) SetupSuite() { leaderName := suite.cluster.WaitLeader() suite.pdLeader = suite.cluster.GetServer(leaderName) suite.backendEndpoints = suite.pdLeader.GetAddr() - suite.NoError(suite.pdLeader.BootstrapCluster()) + re.NoError(suite.pdLeader.BootstrapCluster()) suite.tsoCluster, err = tests.NewTestTSOCluster(suite.ctx, 1, suite.backendEndpoints) - suite.NoError(err) + re.NoError(err) suite.tsoCluster.WaitForDefaultPrimaryServing(re) suite.tsoDefaultPrimaryServer = suite.tsoCluster.GetPrimaryServer(utils.DefaultKeyspaceID, utils.DefaultKeyspaceGroupID) } func (suite *CommonTestSuite) TearDownSuite() { + re := suite.Require() suite.tsoCluster.Destroy() etcdClient := suite.pdLeader.GetEtcdClient() clusterID := strconv.FormatUint(suite.pdLeader.GetClusterID(), 10) endpoints, err := discovery.Discover(etcdClient, clusterID, utils.TSOServiceName) - suite.NoError(err) + re.NoError(err) if len(endpoints) != 0 { endpoints, err = discovery.Discover(etcdClient, clusterID, utils.TSOServiceName) - suite.NoError(err) - suite.Empty(endpoints) + re.NoError(err) + re.Empty(endpoints) } suite.cluster.Destroy() suite.cancel() diff --git a/tests/integrations/realcluster/transfer_leader_test.go b/tests/integrations/realcluster/transfer_leader_test.go index 6a8f2e8a732..0000f7e14a5 100644 --- a/tests/integrations/realcluster/transfer_leader_test.go +++ b/tests/integrations/realcluster/transfer_leader_test.go @@ -57,7 +57,7 @@ func TestTransferLeader(t *testing.T) { res, err = pdHTTPCli.GetSchedulers(ctx) re.NoError(err) - re.Equal(oldSchedulersLen, len(res)) + re.Len(res, oldSchedulersLen) // transfer leader to old leader re.NoError(pdHTTPCli.TransferLeader(ctx, oldLeader)) @@ -69,5 +69,5 @@ func TestTransferLeader(t *testing.T) { res, err = pdHTTPCli.GetSchedulers(ctx) re.NoError(err) - re.Equal(oldSchedulersLen, len(res)) + re.Len(res, oldSchedulersLen) } diff --git a/tests/integrations/tso/client_test.go b/tests/integrations/tso/client_test.go index b021e73a2f9..c8e8f5b2f52 100644 --- a/tests/integrations/tso/client_test.go +++ b/tests/integrations/tso/client_test.go @@ -186,6 +186,7 @@ func (suite *tsoClientTestSuite) TearDownSuite() { } func (suite *tsoClientTestSuite) TestGetTS() { + re := suite.Require() var wg sync.WaitGroup wg.Add(tsoRequestConcurrencyNumber * len(suite.clients)) for i := 0; i < tsoRequestConcurrencyNumber; i++ { @@ -195,9 +196,9 @@ func (suite *tsoClientTestSuite) TestGetTS() { var lastTS uint64 for j := 0; j < tsoRequestRound; j++ { physical, logical, err := client.GetTS(suite.ctx) - suite.NoError(err) + re.NoError(err) ts := tsoutil.ComposeTS(physical, logical) - suite.Less(lastTS, ts) + re.Less(lastTS, ts) lastTS = ts } }(client) @@ -207,6 +208,7 @@ func (suite *tsoClientTestSuite) TestGetTS() { } func (suite *tsoClientTestSuite) TestGetTSAsync() { + re := suite.Require() var wg sync.WaitGroup wg.Add(tsoRequestConcurrencyNumber * len(suite.clients)) for i := 0; i < tsoRequestConcurrencyNumber; i++ { @@ -220,9 +222,9 @@ func (suite *tsoClientTestSuite) TestGetTSAsync() { var lastTS uint64 = math.MaxUint64 for j := len(tsFutures) - 1; j >= 0; j-- { physical, logical, err := tsFutures[j].Wait() - suite.NoError(err) + re.NoError(err) ts := tsoutil.ComposeTS(physical, logical) - suite.Greater(lastTS, ts) + re.Greater(lastTS, ts) lastTS = ts } }(client) @@ -253,9 +255,9 @@ func (suite *tsoClientTestSuite) TestDiscoverTSOServiceWithLegacyPath() { var lastTS uint64 for j := 0; j < tsoRequestRound; j++ { physical, logical, err := client.GetTS(ctx) - suite.NoError(err) + re.NoError(err) ts := tsoutil.ComposeTS(physical, logical) - suite.Less(lastTS, ts) + re.Less(lastTS, ts) lastTS = ts } } diff --git a/tests/integrations/tso/consistency_test.go b/tests/integrations/tso/consistency_test.go index 74f5090bf3b..4d05c47225a 100644 --- a/tests/integrations/tso/consistency_test.go +++ b/tests/integrations/tso/consistency_test.go @@ -148,6 +148,7 @@ func (suite *tsoConsistencyTestSuite) TestRequestTSOConcurrently() { } func (suite *tsoConsistencyTestSuite) requestTSOConcurrently() { + re := suite.Require() ctx, cancel := context.WithCancel(suite.ctx) defer cancel() @@ -164,7 +165,7 @@ func (suite *tsoConsistencyTestSuite) requestTSOConcurrently() { for j := 0; j < tsoRequestRound; j++ { ts = suite.request(ctx, tsoCount) // Check whether the TSO fallbacks - suite.Equal(1, tsoutil.CompareTimestamp(ts, last)) + re.Equal(1, tsoutil.CompareTimestamp(ts, last)) last = ts time.Sleep(10 * time.Millisecond) } diff --git a/tests/pdctl/config/config_test.go b/tests/pdctl/config/config_test.go index c63160a32e5..1f14caf070b 100644 --- a/tests/pdctl/config/config_test.go +++ b/tests/pdctl/config/config_test.go @@ -79,6 +79,7 @@ func (suite *configTestSuite) TearDownSuite() { } func (suite *configTestSuite) TearDownTest() { + re := suite.Require() cleanFunc := func(cluster *tests.TestCluster) { def := placement.GroupBundle{ ID: "pd", @@ -87,20 +88,21 @@ func (suite *configTestSuite) TearDownTest() { }, } data, err := json.Marshal([]placement.GroupBundle{def}) - suite.NoError(err) + re.NoError(err) leader := cluster.GetLeaderServer() - suite.NotNil(leader) + re.NotNil(leader) urlPrefix := leader.GetAddr() - err = testutil.CheckPostJSON(testDialClient, urlPrefix+"/pd/api/v1/config/placement-rule", data, testutil.StatusOK(suite.Require())) - suite.NoError(err) + err = testutil.CheckPostJSON(testDialClient, urlPrefix+"/pd/api/v1/config/placement-rule", data, testutil.StatusOK(re)) + re.NoError(err) } suite.env.RunFuncInTwoModes(cleanFunc) } func (suite *configTestSuite) TestConfig() { - suite.NoError(failpoint.Enable("github.com/tikv/pd/pkg/dashboard/adapter/skipDashboardLoop", `return(true)`)) + re := suite.Require() + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/dashboard/adapter/skipDashboardLoop", `return(true)`)) suite.env.RunTestInTwoModes(suite.checkConfig) - suite.NoError(failpoint.Disable("github.com/tikv/pd/pkg/dashboard/adapter/skipDashboardLoop")) + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/dashboard/adapter/skipDashboardLoop")) } func (suite *configTestSuite) checkConfig(cluster *tests.TestCluster) { @@ -363,7 +365,7 @@ func (suite *configTestSuite) checkPlacementRules(cluster *tests.TestCluster) { re.Contains(string(output), "Success!") // test show - suite.checkShowRuleKey(pdAddr, [][2]string{{placement.DefaultGroupID, placement.DefaultRuleID}}) + suite.checkShowRuleKey(re, pdAddr, [][2]string{{placement.DefaultGroupID, placement.DefaultRuleID}}) f, _ := os.CreateTemp("/tmp", "pd_tests") fname := f.Name() @@ -371,7 +373,7 @@ func (suite *configTestSuite) checkPlacementRules(cluster *tests.TestCluster) { defer os.RemoveAll(fname) // test load - rules := suite.checkLoadRule(pdAddr, fname, [][2]string{{placement.DefaultGroupID, placement.DefaultRuleID}}) + rules := suite.checkLoadRule(re, pdAddr, fname, [][2]string{{placement.DefaultGroupID, placement.DefaultRuleID}}) // test save rules = append(rules, placement.Rule{ @@ -391,11 +393,11 @@ func (suite *configTestSuite) checkPlacementRules(cluster *tests.TestCluster) { re.NoError(err) // test show group - suite.checkShowRuleKey(pdAddr, [][2]string{{placement.DefaultGroupID, placement.DefaultRuleID}, {placement.DefaultGroupID, "test1"}}, "--group=pd") + suite.checkShowRuleKey(re, pdAddr, [][2]string{{placement.DefaultGroupID, placement.DefaultRuleID}, {placement.DefaultGroupID, "test1"}}, "--group=pd") // test rule region detail tests.MustPutRegion(re, cluster, 1, 1, []byte("a"), []byte("b")) - suite.checkShowRuleKey(pdAddr, [][2]string{{placement.DefaultGroupID, placement.DefaultRuleID}}, "--region=1", "--detail") + suite.checkShowRuleKey(re, pdAddr, [][2]string{{placement.DefaultGroupID, placement.DefaultRuleID}}, "--region=1", "--detail") // test delete // need clear up args, so create new a cobra.Command. Otherwise gourp still exists. @@ -404,7 +406,7 @@ func (suite *configTestSuite) checkPlacementRules(cluster *tests.TestCluster) { os.WriteFile(fname, b, 0600) _, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "save", "--in="+fname) re.NoError(err) - suite.checkShowRuleKey(pdAddr, [][2]string{{placement.DefaultGroupID, "test1"}}, "--group=pd") + suite.checkShowRuleKey(re, pdAddr, [][2]string{{placement.DefaultGroupID, "test1"}}, "--group=pd") } func (suite *configTestSuite) TestPlacementRuleGroups() { @@ -519,7 +521,7 @@ func (suite *configTestSuite) checkPlacementRuleBundle(cluster *tests.TestCluste defer os.RemoveAll(fname) // test load - suite.checkLoadRuleBundle(pdAddr, fname, []placement.GroupBundle{ + suite.checkLoadRuleBundle(re, pdAddr, fname, []placement.GroupBundle{ {ID: placement.DefaultGroupID, Index: 0, Override: false, Rules: []*placement.Rule{{GroupID: placement.DefaultGroupID, ID: placement.DefaultRuleID, Role: placement.Voter, Count: 3}}}, }) @@ -531,7 +533,7 @@ func (suite *configTestSuite) checkPlacementRuleBundle(cluster *tests.TestCluste re.NoError(os.WriteFile(fname, b, 0600)) _, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-bundle", "set", "--in="+fname) re.NoError(err) - suite.checkLoadRuleBundle(pdAddr, fname, []placement.GroupBundle{ + suite.checkLoadRuleBundle(re, pdAddr, fname, []placement.GroupBundle{ {ID: placement.DefaultGroupID, Index: 0, Override: false, Rules: []*placement.Rule{{GroupID: placement.DefaultGroupID, ID: placement.DefaultRuleID, Role: placement.Voter, Count: 3}}}, {ID: "pe", Index: 0, Override: false, Rules: []*placement.Rule{{GroupID: "pe", ID: placement.DefaultRuleID, Role: placement.Voter, Count: 3}}}, }) @@ -540,7 +542,7 @@ func (suite *configTestSuite) checkPlacementRuleBundle(cluster *tests.TestCluste _, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-bundle", "delete", placement.DefaultGroupID) re.NoError(err) - suite.checkLoadRuleBundle(pdAddr, fname, []placement.GroupBundle{ + suite.checkLoadRuleBundle(re, pdAddr, fname, []placement.GroupBundle{ {ID: "pe", Index: 0, Override: false, Rules: []*placement.Rule{{GroupID: "pe", ID: placement.DefaultRuleID, Role: placement.Voter, Count: 3}}}, }) @@ -552,7 +554,7 @@ func (suite *configTestSuite) checkPlacementRuleBundle(cluster *tests.TestCluste re.NoError(os.WriteFile(fname, b, 0600)) _, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-bundle", "set", "--in="+fname) re.NoError(err) - suite.checkLoadRuleBundle(pdAddr, fname, []placement.GroupBundle{ + suite.checkLoadRuleBundle(re, pdAddr, fname, []placement.GroupBundle{ {ID: "pe", Index: 0, Override: false, Rules: []*placement.Rule{{GroupID: "pe", ID: placement.DefaultRuleID, Role: placement.Voter, Count: 3}}}, {ID: "pf", Index: 0, Override: false, Rules: []*placement.Rule{{GroupID: "pf", ID: placement.DefaultRuleID, Role: placement.Voter, Count: 3}}}, }) @@ -563,7 +565,7 @@ func (suite *configTestSuite) checkPlacementRuleBundle(cluster *tests.TestCluste bundles := []placement.GroupBundle{ {ID: "pe", Index: 0, Override: false, Rules: []*placement.Rule{{GroupID: "pe", ID: placement.DefaultRuleID, Role: placement.Voter, Count: 3}}}, } - suite.checkLoadRuleBundle(pdAddr, fname, bundles) + suite.checkLoadRuleBundle(re, pdAddr, fname, bundles) // test save bundle.Rules = []*placement.Rule{{GroupID: "pf", ID: placement.DefaultRuleID, Role: placement.Voter, Count: 3}} @@ -573,7 +575,7 @@ func (suite *configTestSuite) checkPlacementRuleBundle(cluster *tests.TestCluste re.NoError(os.WriteFile(fname, b, 0600)) _, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-bundle", "save", "--in="+fname) re.NoError(err) - suite.checkLoadRuleBundle(pdAddr, fname, []placement.GroupBundle{ + suite.checkLoadRuleBundle(re, pdAddr, fname, []placement.GroupBundle{ {ID: "pe", Index: 0, Override: false, Rules: []*placement.Rule{{GroupID: "pe", ID: placement.DefaultRuleID, Role: placement.Voter, Count: 3}}}, {ID: "pf", Index: 0, Override: false, Rules: []*placement.Rule{{GroupID: "pf", ID: placement.DefaultRuleID, Role: placement.Voter, Count: 3}}}, }) @@ -586,7 +588,7 @@ func (suite *configTestSuite) checkPlacementRuleBundle(cluster *tests.TestCluste _, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-bundle", "save", "--in="+fname, "--partial") re.NoError(err) - suite.checkLoadRuleBundle(pdAddr, fname, []placement.GroupBundle{ + suite.checkLoadRuleBundle(re, pdAddr, fname, []placement.GroupBundle{ {ID: "pf", Index: 0, Override: false, Rules: []*placement.Rule{{GroupID: "pf", ID: placement.DefaultRuleID, Role: placement.Voter, Count: 3}}}, }) @@ -605,63 +607,65 @@ func (suite *configTestSuite) checkPlacementRuleBundle(cluster *tests.TestCluste _, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-bundle", "delete", "--regexp", ".*f") re.NoError(err) - suite.checkLoadRuleBundle(pdAddr, fname, []placement.GroupBundle{ + suite.checkLoadRuleBundle(re, pdAddr, fname, []placement.GroupBundle{ {ID: "pd", Index: 0, Override: false, Rules: []*placement.Rule{{GroupID: "pd", ID: placement.DefaultRuleID, Role: placement.Voter, Count: 3}}}, }) } -func (suite *configTestSuite) checkLoadRuleBundle(pdAddr string, fname string, expectValues []placement.GroupBundle) { +func (suite *configTestSuite) checkLoadRuleBundle(re *require.Assertions, pdAddr string, fname string, expectValues []placement.GroupBundle) { var bundles []placement.GroupBundle cmd := pdctlCmd.GetRootCmd() - testutil.Eventually(suite.Require(), func() bool { // wait for the config to be synced to the scheduling server + testutil.Eventually(re, func() bool { // wait for the config to be synced to the scheduling server _, err := pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-bundle", "load", "--out="+fname) - suite.NoError(err) + re.NoError(err) b, _ := os.ReadFile(fname) - suite.NoError(json.Unmarshal(b, &bundles)) + re.NoError(json.Unmarshal(b, &bundles)) return len(bundles) == len(expectValues) }) - assertBundles(suite.Require(), bundles, expectValues) + assertBundles(re, bundles, expectValues) } -func (suite *configTestSuite) checkLoadRule(pdAddr string, fname string, expectValues [][2]string) []placement.Rule { +func (suite *configTestSuite) checkLoadRule(re *require.Assertions, pdAddr string, fname string, expectValues [][2]string) []placement.Rule { var rules []placement.Rule cmd := pdctlCmd.GetRootCmd() - testutil.Eventually(suite.Require(), func() bool { // wait for the config to be synced to the scheduling server + testutil.Eventually(re, func() bool { // wait for the config to be synced to the scheduling server _, err := pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "load", "--out="+fname) - suite.NoError(err) + re.NoError(err) b, _ := os.ReadFile(fname) - suite.NoError(json.Unmarshal(b, &rules)) + re.NoError(json.Unmarshal(b, &rules)) return len(rules) == len(expectValues) }) for i, v := range expectValues { - suite.Equal(v, rules[i].Key()) + re.Equal(v, rules[i].Key()) } return rules } -func (suite *configTestSuite) checkShowRuleKey(pdAddr string, expectValues [][2]string, opts ...string) { - var rules []placement.Rule - var fit placement.RegionFit +func (suite *configTestSuite) checkShowRuleKey(re *require.Assertions, pdAddr string, expectValues [][2]string, opts ...string) { + var ( + rules []placement.Rule + fit placement.RegionFit + ) cmd := pdctlCmd.GetRootCmd() - testutil.Eventually(suite.Require(), func() bool { // wait for the config to be synced to the scheduling server + testutil.Eventually(re, func() bool { // wait for the config to be synced to the scheduling server args := []string{"-u", pdAddr, "config", "placement-rules", "show"} output, err := pdctl.ExecuteCommand(cmd, append(args, opts...)...) - suite.NoError(err) + re.NoError(err) err = json.Unmarshal(output, &rules) if err == nil { return len(rules) == len(expectValues) } - suite.NoError(json.Unmarshal(output, &fit)) + re.NoError(json.Unmarshal(output, &fit)) return len(fit.RuleFits) != 0 }) if len(rules) != 0 { for i, v := range expectValues { - suite.Equal(v, rules[i].Key()) + re.Equal(v, rules[i].Key()) } } if len(fit.RuleFits) != 0 { for i, v := range expectValues { - suite.Equal(v, fit.RuleFits[i].Rule.Key()) + re.Equal(v, fit.RuleFits[i].Rule.Key()) } } } diff --git a/tests/pdctl/log/log_test.go b/tests/pdctl/log/log_test.go index 08df4a78bea..c2299936265 100644 --- a/tests/pdctl/log/log_test.go +++ b/tests/pdctl/log/log_test.go @@ -55,7 +55,7 @@ func (suite *logTestSuite) SetupSuite() { } leaderServer := suite.cluster.GetLeaderServer() re.NoError(leaderServer.BootstrapCluster()) - tests.MustPutStore(suite.Require(), suite.cluster, store) + tests.MustPutStore(re, suite.cluster, store) } func (suite *logTestSuite) TearDownSuite() { @@ -64,6 +64,7 @@ func (suite *logTestSuite) TearDownSuite() { } func (suite *logTestSuite) TestLog() { + re := suite.Require() cmd := pdctlCmd.GetRootCmd() var testCases = []struct { cmd []string @@ -94,12 +95,13 @@ func (suite *logTestSuite) TestLog() { for _, testCase := range testCases { _, err := pdctl.ExecuteCommand(cmd, testCase.cmd...) - suite.NoError(err) - suite.Equal(testCase.expect, suite.cluster.GetLeaderServer().GetConfig().Log.Level) + re.NoError(err) + re.Equal(testCase.expect, suite.cluster.GetLeaderServer().GetConfig().Log.Level) } } func (suite *logTestSuite) TestInstanceLog() { + re := suite.Require() cmd := pdctlCmd.GetRootCmd() var testCases = []struct { cmd []string @@ -126,11 +128,11 @@ func (suite *logTestSuite) TestInstanceLog() { for _, testCase := range testCases { _, err := pdctl.ExecuteCommand(cmd, testCase.cmd...) - suite.NoError(err) + re.NoError(err) svrs := suite.cluster.GetServers() for _, svr := range svrs { if svr.GetAddr() == testCase.instance { - suite.Equal(testCase.expect, svr.GetConfig().Log.Level) + re.Equal(testCase.expect, svr.GetConfig().Log.Level) } } } diff --git a/tests/server/api/api_test.go b/tests/server/api/api_test.go index 946e65bc6e4..c80048b141f 100644 --- a/tests/server/api/api_test.go +++ b/tests/server/api/api_test.go @@ -118,70 +118,73 @@ func TestMiddlewareTestSuite(t *testing.T) { } func (suite *middlewareTestSuite) SetupSuite() { - suite.NoError(failpoint.Enable("github.com/tikv/pd/server/api/enableFailpointAPI", "return(true)")) + re := suite.Require() + re.NoError(failpoint.Enable("github.com/tikv/pd/server/api/enableFailpointAPI", "return(true)")) ctx, cancel := context.WithCancel(context.Background()) suite.cleanup = cancel cluster, err := tests.NewTestCluster(ctx, 3) - suite.NoError(err) - suite.NoError(cluster.RunInitialServers()) - suite.NotEmpty(cluster.WaitLeader()) + re.NoError(err) + re.NoError(cluster.RunInitialServers()) + re.NotEmpty(cluster.WaitLeader()) suite.cluster = cluster } func (suite *middlewareTestSuite) TearDownSuite() { - suite.NoError(failpoint.Disable("github.com/tikv/pd/server/api/enableFailpointAPI")) + re := suite.Require() + re.NoError(failpoint.Disable("github.com/tikv/pd/server/api/enableFailpointAPI")) suite.cleanup() suite.cluster.Destroy() } func (suite *middlewareTestSuite) TestRequestInfoMiddleware() { - suite.NoError(failpoint.Enable("github.com/tikv/pd/server/api/addRequestInfoMiddleware", "return(true)")) + re := suite.Require() + re.NoError(failpoint.Enable("github.com/tikv/pd/server/api/addRequestInfoMiddleware", "return(true)")) leader := suite.cluster.GetLeaderServer() - suite.NotNil(leader) + re.NotNil(leader) input := map[string]interface{}{ "enable-audit": "true", } data, err := json.Marshal(input) - suite.NoError(err) + re.NoError(err) req, _ := http.NewRequest(http.MethodPost, leader.GetAddr()+"/pd/api/v1/service-middleware/config", bytes.NewBuffer(data)) resp, err := dialClient.Do(req) - suite.NoError(err) + re.NoError(err) resp.Body.Close() - suite.True(leader.GetServer().GetServiceMiddlewarePersistOptions().IsAuditEnabled()) + re.True(leader.GetServer().GetServiceMiddlewarePersistOptions().IsAuditEnabled()) labels := make(map[string]interface{}) labels["testkey"] = "testvalue" data, _ = json.Marshal(labels) resp, err = dialClient.Post(leader.GetAddr()+"/pd/api/v1/debug/pprof/profile?force=true", "application/json", bytes.NewBuffer(data)) - suite.NoError(err) + re.NoError(err) _, err = io.ReadAll(resp.Body) resp.Body.Close() - suite.NoError(err) - suite.Equal(http.StatusOK, resp.StatusCode) + re.NoError(err) + re.Equal(http.StatusOK, resp.StatusCode) - suite.Equal("Profile", resp.Header.Get("service-label")) - suite.Equal("{\"force\":[\"true\"]}", resp.Header.Get("url-param")) - suite.Equal("{\"testkey\":\"testvalue\"}", resp.Header.Get("body-param")) - suite.Equal("HTTP/1.1/POST:/pd/api/v1/debug/pprof/profile", resp.Header.Get("method")) - suite.Equal("anonymous", resp.Header.Get("caller-id")) - suite.Equal("127.0.0.1", resp.Header.Get("ip")) + re.Equal("Profile", resp.Header.Get("service-label")) + re.Equal("{\"force\":[\"true\"]}", resp.Header.Get("url-param")) + re.Equal("{\"testkey\":\"testvalue\"}", resp.Header.Get("body-param")) + re.Equal("HTTP/1.1/POST:/pd/api/v1/debug/pprof/profile", resp.Header.Get("method")) + re.Equal("anonymous", resp.Header.Get("caller-id")) + re.Equal("127.0.0.1", resp.Header.Get("ip")) input = map[string]interface{}{ "enable-audit": "false", } data, err = json.Marshal(input) - suite.NoError(err) + re.NoError(err) req, _ = http.NewRequest(http.MethodPost, leader.GetAddr()+"/pd/api/v1/service-middleware/config", bytes.NewBuffer(data)) resp, err = dialClient.Do(req) - suite.NoError(err) + re.NoError(err) resp.Body.Close() - suite.False(leader.GetServer().GetServiceMiddlewarePersistOptions().IsAuditEnabled()) + re.False(leader.GetServer().GetServiceMiddlewarePersistOptions().IsAuditEnabled()) - header := mustRequestSuccess(suite.Require(), leader.GetServer()) - suite.Equal("", header.Get("service-label")) + header := mustRequestSuccess(re, leader.GetServer()) + re.Equal("", header.Get("service-label")) - suite.NoError(failpoint.Disable("github.com/tikv/pd/server/api/addRequestInfoMiddleware")) + re.NoError(failpoint.Disable("github.com/tikv/pd/server/api/addRequestInfoMiddleware")) } func BenchmarkDoRequestWithServiceMiddleware(b *testing.B) { @@ -207,54 +210,55 @@ func BenchmarkDoRequestWithServiceMiddleware(b *testing.B) { } func (suite *middlewareTestSuite) TestRateLimitMiddleware() { + re := suite.Require() leader := suite.cluster.GetLeaderServer() - suite.NotNil(leader) + re.NotNil(leader) input := map[string]interface{}{ "enable-rate-limit": "true", } data, err := json.Marshal(input) - suite.NoError(err) + re.NoError(err) req, _ := http.NewRequest(http.MethodPost, leader.GetAddr()+"/pd/api/v1/service-middleware/config", bytes.NewBuffer(data)) resp, err := dialClient.Do(req) - suite.NoError(err) + re.NoError(err) resp.Body.Close() - suite.True(leader.GetServer().GetServiceMiddlewarePersistOptions().IsRateLimitEnabled()) + re.True(leader.GetServer().GetServiceMiddlewarePersistOptions().IsRateLimitEnabled()) // returns StatusOK when no rate-limit config req, _ = http.NewRequest(http.MethodPost, leader.GetAddr()+"/pd/api/v1/admin/log", strings.NewReader("\"info\"")) resp, err = dialClient.Do(req) - suite.NoError(err) + re.NoError(err) _, err = io.ReadAll(resp.Body) resp.Body.Close() - suite.NoError(err) - suite.Equal(http.StatusOK, resp.StatusCode) + re.NoError(err) + re.Equal(http.StatusOK, resp.StatusCode) input = make(map[string]interface{}) input["type"] = "label" input["label"] = "SetLogLevel" input["qps"] = 0.5 input["concurrency"] = 1 jsonBody, err := json.Marshal(input) - suite.NoError(err) + re.NoError(err) req, _ = http.NewRequest(http.MethodPost, leader.GetAddr()+"/pd/api/v1/service-middleware/config/rate-limit", bytes.NewBuffer(jsonBody)) resp, err = dialClient.Do(req) - suite.NoError(err) + re.NoError(err) _, err = io.ReadAll(resp.Body) resp.Body.Close() - suite.NoError(err) - suite.Equal(http.StatusOK, resp.StatusCode) + re.NoError(err) + re.Equal(http.StatusOK, resp.StatusCode) for i := 0; i < 3; i++ { req, _ = http.NewRequest(http.MethodPost, leader.GetAddr()+"/pd/api/v1/admin/log", strings.NewReader("\"info\"")) resp, err = dialClient.Do(req) - suite.NoError(err) + re.NoError(err) data, err := io.ReadAll(resp.Body) resp.Body.Close() - suite.NoError(err) + re.NoError(err) if i > 0 { - suite.Equal(http.StatusTooManyRequests, resp.StatusCode) - suite.Equal(string(data), fmt.Sprintf("%s\n", http.StatusText(http.StatusTooManyRequests))) + re.Equal(http.StatusTooManyRequests, resp.StatusCode) + re.Equal(string(data), fmt.Sprintf("%s\n", http.StatusText(http.StatusTooManyRequests))) } else { - suite.Equal(http.StatusOK, resp.StatusCode) + re.Equal(http.StatusOK, resp.StatusCode) } } @@ -263,15 +267,15 @@ func (suite *middlewareTestSuite) TestRateLimitMiddleware() { for i := 0; i < 2; i++ { req, _ = http.NewRequest(http.MethodPost, leader.GetAddr()+"/pd/api/v1/admin/log", strings.NewReader("\"info\"")) resp, err = dialClient.Do(req) - suite.NoError(err) + re.NoError(err) data, err := io.ReadAll(resp.Body) resp.Body.Close() - suite.NoError(err) + re.NoError(err) if i > 0 { - suite.Equal(http.StatusTooManyRequests, resp.StatusCode) - suite.Equal(string(data), fmt.Sprintf("%s\n", http.StatusText(http.StatusTooManyRequests))) + re.Equal(http.StatusTooManyRequests, resp.StatusCode) + re.Equal(string(data), fmt.Sprintf("%s\n", http.StatusText(http.StatusTooManyRequests))) } else { - suite.Equal(http.StatusOK, resp.StatusCode) + re.Equal(http.StatusOK, resp.StatusCode) } } @@ -280,12 +284,12 @@ func (suite *middlewareTestSuite) TestRateLimitMiddleware() { for i := 0; i < 2; i++ { req, _ = http.NewRequest(http.MethodPost, leader.GetAddr()+"/pd/api/v1/admin/log", strings.NewReader("\"info\"")) resp, err = dialClient.Do(req) - suite.NoError(err) + re.NoError(err) data, err := io.ReadAll(resp.Body) resp.Body.Close() - suite.NoError(err) - suite.Equal(http.StatusTooManyRequests, resp.StatusCode) - suite.Equal(string(data), fmt.Sprintf("%s\n", http.StatusText(http.StatusTooManyRequests))) + re.NoError(err) + re.Equal(http.StatusTooManyRequests, resp.StatusCode) + re.Equal(string(data), fmt.Sprintf("%s\n", http.StatusText(http.StatusTooManyRequests))) } // resign leader @@ -295,27 +299,27 @@ func (suite *middlewareTestSuite) TestRateLimitMiddleware() { for _, s := range suite.cluster.GetServers() { servers = append(servers, s.GetServer()) } - server.MustWaitLeader(suite.Require(), servers) + server.MustWaitLeader(re, servers) leader = suite.cluster.GetLeaderServer() - suite.True(leader.GetServer().GetServiceMiddlewarePersistOptions().IsRateLimitEnabled()) + re.True(leader.GetServer().GetServiceMiddlewarePersistOptions().IsRateLimitEnabled()) cfg, ok := leader.GetServer().GetRateLimitConfig().LimiterConfig["SetLogLevel"] - suite.True(ok) - suite.Equal(uint64(1), cfg.ConcurrencyLimit) - suite.Equal(0.5, cfg.QPS) - suite.Equal(1, cfg.QPSBurst) + re.True(ok) + re.Equal(uint64(1), cfg.ConcurrencyLimit) + re.Equal(0.5, cfg.QPS) + re.Equal(1, cfg.QPSBurst) for i := 0; i < 3; i++ { req, _ = http.NewRequest(http.MethodPost, leader.GetAddr()+"/pd/api/v1/admin/log", strings.NewReader("\"info\"")) resp, err = dialClient.Do(req) - suite.NoError(err) + re.NoError(err) data, err := io.ReadAll(resp.Body) resp.Body.Close() - suite.NoError(err) + re.NoError(err) if i > 0 { - suite.Equal(http.StatusTooManyRequests, resp.StatusCode) - suite.Equal(string(data), fmt.Sprintf("%s\n", http.StatusText(http.StatusTooManyRequests))) + re.Equal(http.StatusTooManyRequests, resp.StatusCode) + re.Equal(string(data), fmt.Sprintf("%s\n", http.StatusText(http.StatusTooManyRequests))) } else { - suite.Equal(http.StatusOK, resp.StatusCode) + re.Equal(http.StatusOK, resp.StatusCode) } } @@ -324,15 +328,15 @@ func (suite *middlewareTestSuite) TestRateLimitMiddleware() { for i := 0; i < 2; i++ { req, _ = http.NewRequest(http.MethodPost, leader.GetAddr()+"/pd/api/v1/admin/log", strings.NewReader("\"info\"")) resp, err = dialClient.Do(req) - suite.NoError(err) + re.NoError(err) data, err := io.ReadAll(resp.Body) resp.Body.Close() - suite.NoError(err) + re.NoError(err) if i > 0 { - suite.Equal(http.StatusTooManyRequests, resp.StatusCode) - suite.Equal(string(data), fmt.Sprintf("%s\n", http.StatusText(http.StatusTooManyRequests))) + re.Equal(http.StatusTooManyRequests, resp.StatusCode) + re.Equal(string(data), fmt.Sprintf("%s\n", http.StatusText(http.StatusTooManyRequests))) } else { - suite.Equal(http.StatusOK, resp.StatusCode) + re.Equal(http.StatusOK, resp.StatusCode) } } @@ -341,74 +345,76 @@ func (suite *middlewareTestSuite) TestRateLimitMiddleware() { for i := 0; i < 2; i++ { req, _ = http.NewRequest(http.MethodPost, leader.GetAddr()+"/pd/api/v1/admin/log", strings.NewReader("\"info\"")) resp, err = dialClient.Do(req) - suite.NoError(err) + re.NoError(err) data, err := io.ReadAll(resp.Body) resp.Body.Close() - suite.NoError(err) - suite.Equal(http.StatusTooManyRequests, resp.StatusCode) - suite.Equal(string(data), fmt.Sprintf("%s\n", http.StatusText(http.StatusTooManyRequests))) + re.NoError(err) + re.Equal(http.StatusTooManyRequests, resp.StatusCode) + re.Equal(string(data), fmt.Sprintf("%s\n", http.StatusText(http.StatusTooManyRequests))) } input = map[string]interface{}{ "enable-rate-limit": "false", } data, err = json.Marshal(input) - suite.NoError(err) + re.NoError(err) req, _ = http.NewRequest(http.MethodPost, leader.GetAddr()+"/pd/api/v1/service-middleware/config", bytes.NewBuffer(data)) resp, err = dialClient.Do(req) - suite.NoError(err) + re.NoError(err) resp.Body.Close() - suite.False(leader.GetServer().GetServiceMiddlewarePersistOptions().IsRateLimitEnabled()) + re.False(leader.GetServer().GetServiceMiddlewarePersistOptions().IsRateLimitEnabled()) for i := 0; i < 3; i++ { req, _ = http.NewRequest(http.MethodPost, leader.GetAddr()+"/pd/api/v1/admin/log", strings.NewReader("\"info\"")) resp, err = dialClient.Do(req) - suite.NoError(err) + re.NoError(err) _, err = io.ReadAll(resp.Body) resp.Body.Close() - suite.NoError(err) - suite.Equal(http.StatusOK, resp.StatusCode) + re.NoError(err) + re.Equal(http.StatusOK, resp.StatusCode) } } func (suite *middlewareTestSuite) TestSwaggerUrl() { + re := suite.Require() leader := suite.cluster.GetLeaderServer() - suite.NotNil(leader) + re.NotNil(leader) req, _ := http.NewRequest(http.MethodGet, leader.GetAddr()+"/swagger/ui/index", http.NoBody) resp, err := dialClient.Do(req) - suite.NoError(err) - suite.Equal(http.StatusNotFound, resp.StatusCode) + re.NoError(err) + re.Equal(http.StatusNotFound, resp.StatusCode) resp.Body.Close() } func (suite *middlewareTestSuite) TestAuditPrometheusBackend() { + re := suite.Require() leader := suite.cluster.GetLeaderServer() - suite.NotNil(leader) + re.NotNil(leader) input := map[string]interface{}{ "enable-audit": "true", } data, err := json.Marshal(input) - suite.NoError(err) + re.NoError(err) req, _ := http.NewRequest(http.MethodPost, leader.GetAddr()+"/pd/api/v1/service-middleware/config", bytes.NewBuffer(data)) resp, err := dialClient.Do(req) - suite.NoError(err) + re.NoError(err) resp.Body.Close() - suite.True(leader.GetServer().GetServiceMiddlewarePersistOptions().IsAuditEnabled()) + re.True(leader.GetServer().GetServiceMiddlewarePersistOptions().IsAuditEnabled()) timeUnix := time.Now().Unix() - 20 req, _ = http.NewRequest(http.MethodGet, fmt.Sprintf("%s/pd/api/v1/trend?from=%d", leader.GetAddr(), timeUnix), http.NoBody) resp, err = dialClient.Do(req) - suite.NoError(err) + re.NoError(err) _, err = io.ReadAll(resp.Body) resp.Body.Close() - suite.NoError(err) + re.NoError(err) req, _ = http.NewRequest(http.MethodGet, leader.GetAddr()+"/metrics", http.NoBody) resp, err = dialClient.Do(req) - suite.NoError(err) + re.NoError(err) defer resp.Body.Close() content, _ := io.ReadAll(resp.Body) output := string(content) - suite.Contains(output, "pd_service_audit_handling_seconds_count{caller_id=\"anonymous\",ip=\"127.0.0.1\",method=\"HTTP\",service=\"GetTrend\"} 1") + re.Contains(output, "pd_service_audit_handling_seconds_count{caller_id=\"anonymous\",ip=\"127.0.0.1\",method=\"HTTP\",service=\"GetTrend\"} 1") // resign to test persist config oldLeaderName := leader.GetServer().Name() @@ -417,62 +423,63 @@ func (suite *middlewareTestSuite) TestAuditPrometheusBackend() { for _, s := range suite.cluster.GetServers() { servers = append(servers, s.GetServer()) } - server.MustWaitLeader(suite.Require(), servers) + server.MustWaitLeader(re, servers) leader = suite.cluster.GetLeaderServer() timeUnix = time.Now().Unix() - 20 req, _ = http.NewRequest(http.MethodGet, fmt.Sprintf("%s/pd/api/v1/trend?from=%d", leader.GetAddr(), timeUnix), http.NoBody) resp, err = dialClient.Do(req) - suite.NoError(err) + re.NoError(err) _, err = io.ReadAll(resp.Body) resp.Body.Close() - suite.NoError(err) + re.NoError(err) req, _ = http.NewRequest(http.MethodGet, leader.GetAddr()+"/metrics", http.NoBody) resp, err = dialClient.Do(req) - suite.NoError(err) + re.NoError(err) defer resp.Body.Close() content, _ = io.ReadAll(resp.Body) output = string(content) - suite.Contains(output, "pd_service_audit_handling_seconds_count{caller_id=\"anonymous\",ip=\"127.0.0.1\",method=\"HTTP\",service=\"GetTrend\"} 2") + re.Contains(output, "pd_service_audit_handling_seconds_count{caller_id=\"anonymous\",ip=\"127.0.0.1\",method=\"HTTP\",service=\"GetTrend\"} 2") input = map[string]interface{}{ "enable-audit": "false", } data, err = json.Marshal(input) - suite.NoError(err) + re.NoError(err) req, _ = http.NewRequest(http.MethodPost, leader.GetAddr()+"/pd/api/v1/service-middleware/config", bytes.NewBuffer(data)) resp, err = dialClient.Do(req) - suite.NoError(err) + re.NoError(err) resp.Body.Close() - suite.False(leader.GetServer().GetServiceMiddlewarePersistOptions().IsAuditEnabled()) + re.False(leader.GetServer().GetServiceMiddlewarePersistOptions().IsAuditEnabled()) } func (suite *middlewareTestSuite) TestAuditLocalLogBackend() { + re := suite.Require() fname := testutil.InitTempFileLogger("info") defer os.RemoveAll(fname) leader := suite.cluster.GetLeaderServer() - suite.NotNil(leader) + re.NotNil(leader) input := map[string]interface{}{ "enable-audit": "true", } data, err := json.Marshal(input) - suite.NoError(err) + re.NoError(err) req, _ := http.NewRequest(http.MethodPost, leader.GetAddr()+"/pd/api/v1/service-middleware/config", bytes.NewBuffer(data)) resp, err := dialClient.Do(req) - suite.NoError(err) + re.NoError(err) resp.Body.Close() - suite.True(leader.GetServer().GetServiceMiddlewarePersistOptions().IsAuditEnabled()) + re.True(leader.GetServer().GetServiceMiddlewarePersistOptions().IsAuditEnabled()) req, _ = http.NewRequest(http.MethodPost, leader.GetAddr()+"/pd/api/v1/admin/log", strings.NewReader("\"info\"")) resp, err = dialClient.Do(req) - suite.NoError(err) + re.NoError(err) _, err = io.ReadAll(resp.Body) resp.Body.Close() b, _ := os.ReadFile(fname) - suite.Contains(string(b), "audit log") - suite.NoError(err) - suite.Equal(http.StatusOK, resp.StatusCode) + re.Contains(string(b), "audit log") + re.NoError(err) + re.Equal(http.StatusOK, resp.StatusCode) } func BenchmarkDoRequestWithLocalLogAudit(b *testing.B) { @@ -567,15 +574,16 @@ func TestRedirectorTestSuite(t *testing.T) { } func (suite *redirectorTestSuite) SetupSuite() { + re := suite.Require() ctx, cancel := context.WithCancel(context.Background()) suite.cleanup = cancel cluster, err := tests.NewTestCluster(ctx, 3, func(conf *config.Config, serverName string) { conf.TickInterval = typeutil.Duration{Duration: 50 * time.Millisecond} conf.ElectionInterval = typeutil.Duration{Duration: 250 * time.Millisecond} }) - suite.NoError(err) - suite.NoError(cluster.RunInitialServers()) - suite.NotEmpty(cluster.WaitLeader(), 0) + re.NoError(err) + re.NoError(cluster.RunInitialServers()) + re.NotEmpty(cluster.WaitLeader(), 0) suite.cluster = cluster } @@ -587,19 +595,20 @@ func (suite *redirectorTestSuite) TearDownSuite() { func (suite *redirectorTestSuite) TestRedirect() { re := suite.Require() leader := suite.cluster.GetLeaderServer() - suite.NotNil(leader) + re.NotNil(leader) header := mustRequestSuccess(re, leader.GetServer()) header.Del("Date") for _, svr := range suite.cluster.GetServers() { if svr != leader { h := mustRequestSuccess(re, svr.GetServer()) h.Del("Date") - suite.Equal(h, header) + re.Equal(h, header) } } } func (suite *redirectorTestSuite) TestAllowFollowerHandle() { + re := suite.Require() // Find a follower. var follower *server.Server leader := suite.cluster.GetLeaderServer() @@ -612,18 +621,19 @@ func (suite *redirectorTestSuite) TestAllowFollowerHandle() { addr := follower.GetAddr() + "/pd/api/v1/version" request, err := http.NewRequest(http.MethodGet, addr, http.NoBody) - suite.NoError(err) + re.NoError(err) request.Header.Add(apiutil.PDAllowFollowerHandleHeader, "true") resp, err := dialClient.Do(request) - suite.NoError(err) - suite.Equal("", resp.Header.Get(apiutil.PDRedirectorHeader)) + re.NoError(err) + re.Equal("", resp.Header.Get(apiutil.PDRedirectorHeader)) defer resp.Body.Close() - suite.Equal(http.StatusOK, resp.StatusCode) + re.Equal(http.StatusOK, resp.StatusCode) _, err = io.ReadAll(resp.Body) - suite.NoError(err) + re.NoError(err) } func (suite *redirectorTestSuite) TestNotLeader() { + re := suite.Require() // Find a follower. var follower *server.Server leader := suite.cluster.GetLeaderServer() @@ -637,44 +647,45 @@ func (suite *redirectorTestSuite) TestNotLeader() { addr := follower.GetAddr() + "/pd/api/v1/version" // Request to follower without redirectorHeader is OK. request, err := http.NewRequest(http.MethodGet, addr, http.NoBody) - suite.NoError(err) + re.NoError(err) resp, err := dialClient.Do(request) - suite.NoError(err) + re.NoError(err) defer resp.Body.Close() - suite.Equal(http.StatusOK, resp.StatusCode) + re.Equal(http.StatusOK, resp.StatusCode) _, err = io.ReadAll(resp.Body) - suite.NoError(err) + re.NoError(err) // Request to follower with redirectorHeader will fail. request.RequestURI = "" request.Header.Set(apiutil.PDRedirectorHeader, "pd") resp1, err := dialClient.Do(request) - suite.NoError(err) + re.NoError(err) defer resp1.Body.Close() - suite.NotEqual(http.StatusOK, resp1.StatusCode) + re.NotEqual(http.StatusOK, resp1.StatusCode) _, err = io.ReadAll(resp1.Body) - suite.NoError(err) + re.NoError(err) } func (suite *redirectorTestSuite) TestXForwardedFor() { + re := suite.Require() leader := suite.cluster.GetLeaderServer() - suite.NoError(leader.BootstrapCluster()) + re.NoError(leader.BootstrapCluster()) fname := testutil.InitTempFileLogger("info") defer os.RemoveAll(fname) follower := suite.cluster.GetServer(suite.cluster.GetFollower()) addr := follower.GetAddr() + "/pd/api/v1/regions" request, err := http.NewRequest(http.MethodGet, addr, http.NoBody) - suite.NoError(err) + re.NoError(err) resp, err := dialClient.Do(request) - suite.NoError(err) + re.NoError(err) defer resp.Body.Close() - suite.Equal(http.StatusOK, resp.StatusCode) + re.Equal(http.StatusOK, resp.StatusCode) time.Sleep(1 * time.Second) b, _ := os.ReadFile(fname) l := string(b) - suite.Contains(l, "/pd/api/v1/regions") - suite.NotContains(l, suite.cluster.GetConfig().GetClientURLs()) + re.Contains(l, "/pd/api/v1/regions") + re.NotContains(l, suite.cluster.GetConfig().GetClientURLs()) } func mustRequestSuccess(re *require.Assertions, s *server.Server) http.Header { diff --git a/tests/server/api/checker_test.go b/tests/server/api/checker_test.go index 8037fcc3989..884772bba97 100644 --- a/tests/server/api/checker_test.go +++ b/tests/server/api/checker_test.go @@ -20,6 +20,7 @@ import ( "testing" "time" + "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" tu "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/tests" @@ -47,7 +48,8 @@ func (suite *checkerTestSuite) TestAPI() { } func (suite *checkerTestSuite) checkAPI(cluster *tests.TestCluster) { - suite.testErrCases(cluster) + re := suite.Require() + suite.testErrCases(re, cluster) testCases := []struct { name string @@ -60,75 +62,73 @@ func (suite *checkerTestSuite) checkAPI(cluster *tests.TestCluster) { {name: "joint-state"}, } for _, testCase := range testCases { - suite.testGetStatus(cluster, testCase.name) - suite.testPauseOrResume(cluster, testCase.name) + suite.testGetStatus(re, cluster, testCase.name) + suite.testPauseOrResume(re, cluster, testCase.name) } } -func (suite *checkerTestSuite) testErrCases(cluster *tests.TestCluster) { +func (suite *checkerTestSuite) testErrCases(re *require.Assertions, cluster *tests.TestCluster) { urlPrefix := fmt.Sprintf("%s/pd/api/v1/checker", cluster.GetLeaderServer().GetAddr()) // missing args input := make(map[string]interface{}) pauseArgs, err := json.Marshal(input) - suite.NoError(err) - re := suite.Require() + re.NoError(err) err = tu.CheckPostJSON(testDialClient, urlPrefix+"/merge", pauseArgs, tu.StatusNotOK(re)) - suite.NoError(err) + re.NoError(err) // negative delay input["delay"] = -10 pauseArgs, err = json.Marshal(input) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, urlPrefix+"/merge", pauseArgs, tu.StatusNotOK(re)) - suite.NoError(err) + re.NoError(err) // wrong name name := "dummy" input["delay"] = 30 pauseArgs, err = json.Marshal(input) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, urlPrefix+"/"+name, pauseArgs, tu.StatusNotOK(re)) - suite.NoError(err) + re.NoError(err) input["delay"] = 0 pauseArgs, err = json.Marshal(input) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, urlPrefix+"/"+name, pauseArgs, tu.StatusNotOK(re)) - suite.NoError(err) + re.NoError(err) } -func (suite *checkerTestSuite) testGetStatus(cluster *tests.TestCluster, name string) { +func (suite *checkerTestSuite) testGetStatus(re *require.Assertions, cluster *tests.TestCluster, name string) { input := make(map[string]interface{}) urlPrefix := fmt.Sprintf("%s/pd/api/v1/checker", cluster.GetLeaderServer().GetAddr()) // normal run resp := make(map[string]interface{}) - re := suite.Require() err := tu.ReadGetJSON(re, testDialClient, fmt.Sprintf("%s/%s", urlPrefix, name), &resp) - suite.NoError(err) - suite.False(resp["paused"].(bool)) + re.NoError(err) + re.False(resp["paused"].(bool)) // paused input["delay"] = 30 pauseArgs, err := json.Marshal(input) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, urlPrefix+"/"+name, pauseArgs, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) resp = make(map[string]interface{}) err = tu.ReadGetJSON(re, testDialClient, fmt.Sprintf("%s/%s", urlPrefix, name), &resp) - suite.NoError(err) - suite.True(resp["paused"].(bool)) + re.NoError(err) + re.True(resp["paused"].(bool)) // resumed input["delay"] = 0 pauseArgs, err = json.Marshal(input) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, urlPrefix+"/"+name, pauseArgs, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) time.Sleep(time.Second) resp = make(map[string]interface{}) err = tu.ReadGetJSON(re, testDialClient, fmt.Sprintf("%s/%s", urlPrefix, name), &resp) - suite.NoError(err) - suite.False(resp["paused"].(bool)) + re.NoError(err) + re.False(resp["paused"].(bool)) } -func (suite *checkerTestSuite) testPauseOrResume(cluster *tests.TestCluster, name string) { +func (suite *checkerTestSuite) testPauseOrResume(re *require.Assertions, cluster *tests.TestCluster, name string) { input := make(map[string]interface{}) urlPrefix := fmt.Sprintf("%s/pd/api/v1/checker", cluster.GetLeaderServer().GetAddr()) resp := make(map[string]interface{}) @@ -136,36 +136,35 @@ func (suite *checkerTestSuite) testPauseOrResume(cluster *tests.TestCluster, nam // test pause. input["delay"] = 30 pauseArgs, err := json.Marshal(input) - suite.NoError(err) - re := suite.Require() + re.NoError(err) err = tu.CheckPostJSON(testDialClient, urlPrefix+"/"+name, pauseArgs, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) err = tu.ReadGetJSON(re, testDialClient, fmt.Sprintf("%s/%s", urlPrefix, name), &resp) - suite.NoError(err) - suite.True(resp["paused"].(bool)) + re.NoError(err) + re.True(resp["paused"].(bool)) input["delay"] = 1 pauseArgs, err = json.Marshal(input) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, urlPrefix+"/"+name, pauseArgs, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) time.Sleep(time.Second) err = tu.ReadGetJSON(re, testDialClient, fmt.Sprintf("%s/%s", urlPrefix, name), &resp) - suite.NoError(err) - suite.False(resp["paused"].(bool)) + re.NoError(err) + re.False(resp["paused"].(bool)) // test resume. input = make(map[string]interface{}) input["delay"] = 30 pauseArgs, err = json.Marshal(input) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, urlPrefix+"/"+name, pauseArgs, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) input["delay"] = 0 pauseArgs, err = json.Marshal(input) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, urlPrefix+"/"+name, pauseArgs, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) err = tu.ReadGetJSON(re, testDialClient, fmt.Sprintf("%s/%s", urlPrefix, name), &resp) - suite.NoError(err) - suite.False(resp["paused"].(bool)) + re.NoError(err) + re.False(resp["paused"].(bool)) } diff --git a/tests/server/api/operator_test.go b/tests/server/api/operator_test.go index 41a687b1181..ab453e746a9 100644 --- a/tests/server/api/operator_test.go +++ b/tests/server/api/operator_test.go @@ -25,6 +25,7 @@ import ( "time" "github.com/pingcap/kvproto/pkg/metapb" + "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/schedule/operator" @@ -69,7 +70,7 @@ func (suite *operatorTestSuite) TestAddRemovePeer() { func (suite *operatorTestSuite) checkAddRemovePeer(cluster *tests.TestCluster) { re := suite.Require() - suite.pauseRuleChecker(cluster) + suite.pauseRuleChecker(re, cluster) stores := []*metapb.Store{ { Id: 1, @@ -113,35 +114,35 @@ func (suite *operatorTestSuite) checkAddRemovePeer(cluster *tests.TestCluster) { regionURL := fmt.Sprintf("%s/operators/%d", urlPrefix, region.GetId()) err := tu.CheckGetJSON(testDialClient, regionURL, nil, tu.StatusNotOK(re), tu.StringContain(re, "operator not found")) - suite.NoError(err) + re.NoError(err) recordURL := fmt.Sprintf("%s/operators/records?from=%s", urlPrefix, strconv.FormatInt(time.Now().Unix(), 10)) err = tu.CheckGetJSON(testDialClient, recordURL, nil, tu.StatusNotOK(re), tu.StringContain(re, "operator not found")) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, fmt.Sprintf("%s/operators", urlPrefix), []byte(`{"name":"add-peer", "region_id": 1, "store_id": 3}`), tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) err = tu.CheckGetJSON(testDialClient, regionURL, nil, tu.StatusOK(re), tu.StringContain(re, "add learner peer 1 on store 3"), tu.StringContain(re, "RUNNING")) - suite.NoError(err) + re.NoError(err) err = tu.CheckDelete(testDialClient, regionURL, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) err = tu.CheckGetJSON(testDialClient, recordURL, nil, tu.StatusOK(re), tu.StringContain(re, "admin-add-peer {add peer: store [3]}")) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, fmt.Sprintf("%s/operators", urlPrefix), []byte(`{"name":"remove-peer", "region_id": 1, "store_id": 2}`), tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) err = tu.CheckGetJSON(testDialClient, regionURL, nil, tu.StatusOK(re), tu.StringContain(re, "remove peer on store 2"), tu.StringContain(re, "RUNNING")) - suite.NoError(err) + re.NoError(err) err = tu.CheckDelete(testDialClient, regionURL, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) err = tu.CheckGetJSON(testDialClient, recordURL, nil, tu.StatusOK(re), tu.StringContain(re, "admin-remove-peer {rm peer: store [2]}")) - suite.NoError(err) + re.NoError(err) tests.MustPutStore(re, cluster, &metapb.Store{ Id: 4, @@ -150,27 +151,27 @@ func (suite *operatorTestSuite) checkAddRemovePeer(cluster *tests.TestCluster) { LastHeartbeat: time.Now().UnixNano(), }) err = tu.CheckPostJSON(testDialClient, fmt.Sprintf("%s/operators", urlPrefix), []byte(`{"name":"add-learner", "region_id": 1, "store_id": 4}`), tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) err = tu.CheckGetJSON(testDialClient, regionURL, nil, tu.StatusOK(re), tu.StringContain(re, "add learner peer 2 on store 4")) - suite.NoError(err) + re.NoError(err) // Fail to add peer to tombstone store. err = cluster.GetLeaderServer().GetRaftCluster().RemoveStore(3, true) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, fmt.Sprintf("%s/operators", urlPrefix), []byte(`{"name":"add-peer", "region_id": 1, "store_id": 3}`), tu.StatusNotOK(re)) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, fmt.Sprintf("%s/operators", urlPrefix), []byte(`{"name":"transfer-peer", "region_id": 1, "from_store_id": 1, "to_store_id": 3}`), tu.StatusNotOK(re)) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, fmt.Sprintf("%s/operators", urlPrefix), []byte(`{"name":"transfer-region", "region_id": 1, "to_store_ids": [1, 2, 3]}`), tu.StatusNotOK(re)) - suite.NoError(err) + re.NoError(err) // Fail to get operator if from is latest. time.Sleep(time.Second) url := fmt.Sprintf("%s/operators/records?from=%s", urlPrefix, strconv.FormatInt(time.Now().Unix(), 10)) err = tu.CheckGetJSON(testDialClient, url, nil, tu.StatusNotOK(re), tu.StringContain(re, "operator not found")) - suite.NoError(err) + re.NoError(err) } func (suite *operatorTestSuite) TestMergeRegionOperator() { @@ -204,7 +205,7 @@ func (suite *operatorTestSuite) checkMergeRegionOperator(cluster *tests.TestClus tests.MustPutStore(re, cluster, store) } - suite.pauseRuleChecker(cluster) + suite.pauseRuleChecker(re, cluster) r1 := core.NewTestRegionInfo(10, 1, []byte(""), []byte("b"), core.SetWrittenBytes(1000), core.SetReadBytes(1000), core.SetRegionConfVer(1), core.SetRegionVersion(1)) tests.MustPutRegionInfo(re, cluster, r1) r2 := core.NewTestRegionInfo(20, 1, []byte("b"), []byte("c"), core.SetWrittenBytes(2000), core.SetReadBytes(0), core.SetRegionConfVer(2), core.SetRegionVersion(3)) @@ -214,18 +215,18 @@ func (suite *operatorTestSuite) checkMergeRegionOperator(cluster *tests.TestClus urlPrefix := fmt.Sprintf("%s/pd/api/v1", cluster.GetLeaderServer().GetAddr()) err := tu.CheckPostJSON(testDialClient, fmt.Sprintf("%s/operators", urlPrefix), []byte(`{"name":"merge-region", "source_region_id": 10, "target_region_id": 20}`), tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) tu.CheckDelete(testDialClient, fmt.Sprintf("%s/operators/%d", urlPrefix, 10), tu.StatusOK(re)) err = tu.CheckPostJSON(testDialClient, fmt.Sprintf("%s/operators", urlPrefix), []byte(`{"name":"merge-region", "source_region_id": 20, "target_region_id": 10}`), tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) tu.CheckDelete(testDialClient, fmt.Sprintf("%s/operators/%d", urlPrefix, 10), tu.StatusOK(re)) err = tu.CheckPostJSON(testDialClient, fmt.Sprintf("%s/operators", urlPrefix), []byte(`{"name":"merge-region", "source_region_id": 10, "target_region_id": 30}`), tu.StatusNotOK(re), tu.StringContain(re, "not adjacent")) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, fmt.Sprintf("%s/operators", urlPrefix), []byte(`{"name":"merge-region", "source_region_id": 30, "target_region_id": 10}`), tu.StatusNotOK(re), tu.StringContain(re, "not adjacent")) - suite.NoError(err) + re.NoError(err) } func (suite *operatorTestSuite) TestTransferRegionWithPlacementRule() { @@ -240,7 +241,7 @@ func (suite *operatorTestSuite) TestTransferRegionWithPlacementRule() { func (suite *operatorTestSuite) checkTransferRegionWithPlacementRule(cluster *tests.TestCluster) { re := suite.Require() - suite.pauseRuleChecker(cluster) + suite.pauseRuleChecker(re, cluster) stores := []*metapb.Store{ { Id: 1, @@ -480,14 +481,14 @@ func (suite *operatorTestSuite) checkTransferRegionWithPlacementRule(cluster *te svr.GetRaftCluster().GetOpts().GetLocationLabels(), svr.GetRaftCluster().GetOpts().GetIsolationLevel(), ) - suite.NoError(err) + re.NoError(err) } if len(testCase.rules) > 0 { // add customized rule first and then remove default rule err := manager.SetRules(testCase.rules) - suite.NoError(err) + re.NoError(err) err = manager.DeleteRule(placement.DefaultGroupID, placement.DefaultRuleID) - suite.NoError(err) + re.NoError(err) } if testCase.expectedError == nil { err = tu.CheckPostJSON(testDialClient, fmt.Sprintf("%s/operators", urlPrefix), testCase.input, tu.StatusOK(re)) @@ -495,22 +496,21 @@ func (suite *operatorTestSuite) checkTransferRegionWithPlacementRule(cluster *te err = tu.CheckPostJSON(testDialClient, fmt.Sprintf("%s/operators", urlPrefix), testCase.input, tu.StatusNotOK(re), tu.StringContain(re, testCase.expectedError.Error())) } - suite.NoError(err) + re.NoError(err) if len(testCase.expectSteps) > 0 { err = tu.CheckGetJSON(testDialClient, regionURL, nil, tu.StatusOK(re), tu.StringContain(re, testCase.expectSteps)) - suite.NoError(err) + re.NoError(err) err = tu.CheckDelete(testDialClient, regionURL, tu.StatusOK(re)) } else { err = tu.CheckDelete(testDialClient, regionURL, tu.StatusNotOK(re)) } - suite.NoError(err) + re.NoError(err) } } // pauseRuleChecker will pause rule checker to avoid unexpected operator. -func (suite *operatorTestSuite) pauseRuleChecker(cluster *tests.TestCluster) { - re := suite.Require() +func (suite *operatorTestSuite) pauseRuleChecker(re *require.Assertions, cluster *tests.TestCluster) { checkerName := "rule" addr := cluster.GetLeaderServer().GetAddr() resp := make(map[string]interface{}) diff --git a/tests/server/api/region_test.go b/tests/server/api/region_test.go index 328c0fcd885..68325fb6c71 100644 --- a/tests/server/api/region_test.go +++ b/tests/server/api/region_test.go @@ -24,6 +24,7 @@ import ( "github.com/pingcap/failpoint" "github.com/pingcap/kvproto/pkg/metapb" + "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/schedule/placement" @@ -57,9 +58,9 @@ func (suite *regionTestSuite) TearDownTest() { for _, region := range leader.GetRegions() { url := fmt.Sprintf("%s/pd/api/v1/admin/cache/region/%d", pdAddr, region.GetID()) err := tu.CheckDelete(testDialClient, url, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) } - suite.Empty(leader.GetRegions()) + re.Empty(leader.GetRegions()) // clean rules def := placement.GroupBundle{ ID: "pd", @@ -68,18 +69,18 @@ func (suite *regionTestSuite) TearDownTest() { }, } data, err := json.Marshal([]placement.GroupBundle{def}) - suite.NoError(err) + re.NoError(err) urlPrefix := cluster.GetLeaderServer().GetAddr() - err = tu.CheckPostJSON(testDialClient, urlPrefix+"/pd/api/v1/config/placement-rule", data, tu.StatusOK(suite.Require())) - suite.NoError(err) + err = tu.CheckPostJSON(testDialClient, urlPrefix+"/pd/api/v1/config/placement-rule", data, tu.StatusOK(re)) + re.NoError(err) // clean stores // TODO: cannot sync to scheduling server? for _, store := range leader.GetStores() { - suite.NoError(cluster.GetLeaderServer().GetRaftCluster().RemoveStore(store.GetId(), true)) - suite.NoError(cluster.GetLeaderServer().GetRaftCluster().BuryStore(store.GetId(), true)) + re.NoError(cluster.GetLeaderServer().GetRaftCluster().RemoveStore(store.GetId(), true)) + re.NoError(cluster.GetLeaderServer().GetRaftCluster().BuryStore(store.GetId(), true)) } - suite.NoError(cluster.GetLeaderServer().GetRaftCluster().RemoveTombStoneRecords()) - suite.Empty(leader.GetStores()) + re.NoError(cluster.GetLeaderServer().GetRaftCluster().RemoveTombStoneRecords()) + re.Empty(leader.GetStores()) } suite.env.RunFuncInTwoModes(cleanFunc) } @@ -104,7 +105,7 @@ func (suite *regionTestSuite) checkSplitRegions(cluster *tests.TestCluster) { r1 := core.NewTestRegionInfo(601, 13, []byte("aaa"), []byte("ggg")) r1.GetMeta().Peers = append(r1.GetMeta().Peers, &metapb.Peer{Id: 5, StoreId: 14}, &metapb.Peer{Id: 6, StoreId: 15}) tests.MustPutRegionInfo(re, cluster, r1) - suite.checkRegionCount(cluster, 1) + suite.checkRegionCount(re, cluster, 1) newRegionID := uint64(11) body := fmt.Sprintf(`{"retry_limit":%v, "split_keys": ["%s","%s","%s"]}`, 3, @@ -117,14 +118,14 @@ func (suite *regionTestSuite) checkSplitRegions(cluster *tests.TestCluster) { NewRegionsID []uint64 `json:"regions-id"` }{} err := json.Unmarshal(res, s) - suite.NoError(err) - suite.Equal(100, s.ProcessedPercentage) - suite.Equal([]uint64{newRegionID}, s.NewRegionsID) + re.NoError(err) + re.Equal(100, s.ProcessedPercentage) + re.Equal([]uint64{newRegionID}, s.NewRegionsID) } - suite.NoError(failpoint.Enable("github.com/tikv/pd/pkg/schedule/handler/splitResponses", fmt.Sprintf("return(%v)", newRegionID))) + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/schedule/handler/splitResponses", fmt.Sprintf("return(%v)", newRegionID))) err := tu.CheckPostJSON(testDialClient, fmt.Sprintf("%s/regions/split", urlPrefix), []byte(body), checkOpt) - suite.NoError(failpoint.Disable("github.com/tikv/pd/pkg/schedule/handler/splitResponses")) - suite.NoError(err) + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/schedule/handler/splitResponses")) + re.NoError(err) } func (suite *regionTestSuite) TestAccelerateRegionsScheduleInRange() { @@ -149,12 +150,12 @@ func (suite *regionTestSuite) checkAccelerateRegionsScheduleInRange(cluster *tes r1.GetMeta().Peers = append(r1.GetMeta().Peers, &metapb.Peer{Id: 100 + i, StoreId: (i + 1) % regionCount}, &metapb.Peer{Id: 200 + i, StoreId: (i + 2) % regionCount}) tests.MustPutRegionInfo(re, cluster, r1) } - suite.checkRegionCount(cluster, regionCount) + suite.checkRegionCount(re, cluster, regionCount) body := fmt.Sprintf(`{"start_key":"%s", "end_key": "%s"}`, hex.EncodeToString([]byte("a1")), hex.EncodeToString([]byte("a3"))) err := tu.CheckPostJSON(testDialClient, fmt.Sprintf("%s/regions/accelerate-schedule", urlPrefix), []byte(body), tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) idList := leader.GetRaftCluster().GetSuspectRegions() if sche := cluster.GetSchedulingPrimaryServer(); sche != nil { idList = sche.GetCluster().GetCoordinator().GetCheckerController().GetSuspectRegions() @@ -184,13 +185,13 @@ func (suite *regionTestSuite) checkAccelerateRegionsScheduleInRanges(cluster *te r1.GetMeta().Peers = append(r1.GetMeta().Peers, &metapb.Peer{Id: 100 + i, StoreId: (i + 1) % regionCount}, &metapb.Peer{Id: 200 + i, StoreId: (i + 2) % regionCount}) tests.MustPutRegionInfo(re, cluster, r1) } - suite.checkRegionCount(cluster, regionCount) + suite.checkRegionCount(re, cluster, regionCount) body := fmt.Sprintf(`[{"start_key":"%s", "end_key": "%s"}, {"start_key":"%s", "end_key": "%s"}]`, hex.EncodeToString([]byte("a1")), hex.EncodeToString([]byte("a3")), hex.EncodeToString([]byte("a4")), hex.EncodeToString([]byte("a6"))) err := tu.CheckPostJSON(testDialClient, fmt.Sprintf("%s/regions/accelerate-schedule/batch", urlPrefix), []byte(body), tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) idList := leader.GetRaftCluster().GetSuspectRegions() if sche := cluster.GetSchedulingPrimaryServer(); sche != nil { idList = sche.GetCluster().GetCoordinator().GetCheckerController().GetSuspectRegions() @@ -226,11 +227,11 @@ func (suite *regionTestSuite) checkScatterRegions(cluster *tests.TestCluster) { tests.MustPutRegionInfo(re, cluster, r1) tests.MustPutRegionInfo(re, cluster, r2) tests.MustPutRegionInfo(re, cluster, r3) - suite.checkRegionCount(cluster, 3) + suite.checkRegionCount(re, cluster, 3) body := fmt.Sprintf(`{"start_key":"%s", "end_key": "%s"}`, hex.EncodeToString([]byte("b1")), hex.EncodeToString([]byte("b3"))) err := tu.CheckPostJSON(testDialClient, fmt.Sprintf("%s/regions/scatter", urlPrefix), []byte(body), tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) oc := leader.GetRaftCluster().GetOperatorController() if sche := cluster.GetSchedulingPrimaryServer(); sche != nil { oc = sche.GetCoordinator().GetOperatorController() @@ -240,11 +241,11 @@ func (suite *regionTestSuite) checkScatterRegions(cluster *tests.TestCluster) { op2 := oc.GetOperator(702) op3 := oc.GetOperator(703) // At least one operator used to scatter region - suite.True(op1 != nil || op2 != nil || op3 != nil) + re.True(op1 != nil || op2 != nil || op3 != nil) body = `{"regions_id": [701, 702, 703]}` err = tu.CheckPostJSON(testDialClient, fmt.Sprintf("%s/regions/scatter", urlPrefix), []byte(body), tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) } func (suite *regionTestSuite) TestCheckRegionsReplicated() { @@ -252,10 +253,10 @@ func (suite *regionTestSuite) TestCheckRegionsReplicated() { } func (suite *regionTestSuite) checkRegionsReplicated(cluster *tests.TestCluster) { - suite.pauseRuleChecker(cluster) + re := suite.Require() + suite.pauseRuleChecker(re, cluster) leader := cluster.GetLeaderServer() urlPrefix := leader.GetAddr() + "/pd/api/v1" - re := suite.Require() // add test region s1 := &metapb.Store{ @@ -266,7 +267,7 @@ func (suite *regionTestSuite) checkRegionsReplicated(cluster *tests.TestCluster) tests.MustPutStore(re, cluster, s1) r1 := core.NewTestRegionInfo(2, 1, []byte("a"), []byte("b")) tests.MustPutRegionInfo(re, cluster, r1) - suite.checkRegionCount(cluster, 1) + suite.checkRegionCount(re, cluster, 1) // set the bundle bundle := []placement.GroupBundle{ @@ -286,42 +287,42 @@ func (suite *regionTestSuite) checkRegionsReplicated(cluster *tests.TestCluster) // invalid url url := fmt.Sprintf(`%s/regions/replicated?startKey=%s&endKey=%s`, urlPrefix, "_", "t") err := tu.CheckGetJSON(testDialClient, url, nil, tu.Status(re, http.StatusBadRequest)) - suite.NoError(err) + re.NoError(err) url = fmt.Sprintf(`%s/regions/replicated?startKey=%s&endKey=%s`, urlPrefix, hex.EncodeToString(r1.GetStartKey()), "_") err = tu.CheckGetJSON(testDialClient, url, nil, tu.Status(re, http.StatusBadRequest)) - suite.NoError(err) + re.NoError(err) // correct test url = fmt.Sprintf(`%s/regions/replicated?startKey=%s&endKey=%s`, urlPrefix, hex.EncodeToString(r1.GetStartKey()), hex.EncodeToString(r1.GetEndKey())) err = tu.CheckGetJSON(testDialClient, url, nil, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) // test one rule data, err := json.Marshal(bundle) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, urlPrefix+"/config/placement-rule", data, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) tu.Eventually(re, func() bool { respBundle := make([]placement.GroupBundle, 0) err = tu.CheckGetJSON(testDialClient, urlPrefix+"/config/placement-rule", nil, tu.StatusOK(re), tu.ExtractJSON(re, &respBundle)) - suite.NoError(err) + re.NoError(err) return len(respBundle) == 1 && respBundle[0].ID == "5" }) tu.Eventually(re, func() bool { err = tu.ReadGetJSON(re, testDialClient, url, &status) - suite.NoError(err) + re.NoError(err) return status == "REPLICATED" }) - suite.NoError(failpoint.Enable("github.com/tikv/pd/pkg/schedule/handler/mockPending", "return(true)")) + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/schedule/handler/mockPending", "return(true)")) err = tu.ReadGetJSON(re, testDialClient, url, &status) - suite.NoError(err) - suite.Equal("PENDING", status) - suite.NoError(failpoint.Disable("github.com/tikv/pd/pkg/schedule/handler/mockPending")) + re.NoError(err) + re.Equal("PENDING", status) + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/schedule/handler/mockPending")) // test multiple rules r1 = core.NewTestRegionInfo(2, 1, []byte("a"), []byte("b")) r1.GetMeta().Peers = append(r1.GetMeta().Peers, &metapb.Peer{Id: 5, StoreId: 1}) @@ -331,21 +332,21 @@ func (suite *regionTestSuite) checkRegionsReplicated(cluster *tests.TestCluster) ID: "bar", Index: 1, Role: placement.Voter, Count: 1, }) data, err = json.Marshal(bundle) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, urlPrefix+"/config/placement-rule", data, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) tu.Eventually(re, func() bool { respBundle := make([]placement.GroupBundle, 0) err = tu.CheckGetJSON(testDialClient, urlPrefix+"/config/placement-rule", nil, tu.StatusOK(re), tu.ExtractJSON(re, &respBundle)) - suite.NoError(err) + re.NoError(err) return len(respBundle) == 1 && len(respBundle[0].Rules) == 2 }) tu.Eventually(re, func() bool { err = tu.ReadGetJSON(re, testDialClient, url, &status) - suite.NoError(err) + re.NoError(err) return status == "REPLICATED" }) @@ -360,15 +361,15 @@ func (suite *regionTestSuite) checkRegionsReplicated(cluster *tests.TestCluster) }, }) data, err = json.Marshal(bundle) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, urlPrefix+"/config/placement-rule", data, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) tu.Eventually(re, func() bool { respBundle := make([]placement.GroupBundle, 0) err = tu.CheckGetJSON(testDialClient, urlPrefix+"/config/placement-rule", nil, tu.StatusOK(re), tu.ExtractJSON(re, &respBundle)) - suite.NoError(err) + re.NoError(err) if len(respBundle) != 2 { return false } @@ -379,7 +380,7 @@ func (suite *regionTestSuite) checkRegionsReplicated(cluster *tests.TestCluster) tu.Eventually(re, func() bool { err = tu.ReadGetJSON(re, testDialClient, url, &status) - suite.NoError(err) + re.NoError(err) return status == "INPROGRESS" }) @@ -389,26 +390,25 @@ func (suite *regionTestSuite) checkRegionsReplicated(cluster *tests.TestCluster) tu.Eventually(re, func() bool { err = tu.ReadGetJSON(re, testDialClient, url, &status) - suite.NoError(err) + re.NoError(err) return status == "REPLICATED" }) } -func (suite *regionTestSuite) checkRegionCount(cluster *tests.TestCluster, count uint64) { +func (suite *regionTestSuite) checkRegionCount(re *require.Assertions, cluster *tests.TestCluster, count uint64) { leader := cluster.GetLeaderServer() - tu.Eventually(suite.Require(), func() bool { + tu.Eventually(re, func() bool { return leader.GetRaftCluster().GetRegionCount([]byte{}, []byte{}).Count == int(count) }) if sche := cluster.GetSchedulingPrimaryServer(); sche != nil { - tu.Eventually(suite.Require(), func() bool { + tu.Eventually(re, func() bool { return sche.GetCluster().GetRegionCount([]byte{}, []byte{}) == int(count) }) } } // pauseRuleChecker will pause rule checker to avoid unexpected operator. -func (suite *regionTestSuite) pauseRuleChecker(cluster *tests.TestCluster) { - re := suite.Require() +func (suite *regionTestSuite) pauseRuleChecker(re *require.Assertions, cluster *tests.TestCluster) { checkerName := "rule" addr := cluster.GetLeaderServer().GetAddr() resp := make(map[string]interface{}) diff --git a/tests/server/api/rule_test.go b/tests/server/api/rule_test.go index af70d5afed9..14bb23e383a 100644 --- a/tests/server/api/rule_test.go +++ b/tests/server/api/rule_test.go @@ -26,6 +26,7 @@ import ( "testing" "github.com/pingcap/kvproto/pkg/metapb" + "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/errs" @@ -58,6 +59,7 @@ func (suite *ruleTestSuite) TearDownSuite() { } func (suite *ruleTestSuite) TearDownTest() { + re := suite.Require() cleanFunc := func(cluster *tests.TestCluster) { def := placement.GroupBundle{ ID: "pd", @@ -66,10 +68,10 @@ func (suite *ruleTestSuite) TearDownTest() { }, } data, err := json.Marshal([]placement.GroupBundle{def}) - suite.NoError(err) + re.NoError(err) urlPrefix := cluster.GetLeaderServer().GetAddr() - err = tu.CheckPostJSON(testDialClient, urlPrefix+"/pd/api/v1/config/placement-rule", data, tu.StatusOK(suite.Require())) - suite.NoError(err) + err = tu.CheckPostJSON(testDialClient, urlPrefix+"/pd/api/v1/config/placement-rule", data, tu.StatusOK(re)) + re.NoError(err) } suite.env.RunFuncInTwoModes(cleanFunc) } @@ -79,31 +81,32 @@ func (suite *ruleTestSuite) TestSet() { } func (suite *ruleTestSuite) checkSet(cluster *tests.TestCluster) { + re := suite.Require() leaderServer := cluster.GetLeaderServer() pdAddr := leaderServer.GetAddr() urlPrefix := fmt.Sprintf("%s%s/api/v1/config", pdAddr, apiPrefix) rule := placement.Rule{GroupID: "a", ID: "10", StartKeyHex: "1111", EndKeyHex: "3333", Role: placement.Voter, Count: 1} successData, err := json.Marshal(rule) - suite.NoError(err) + re.NoError(err) oldStartKey, err := hex.DecodeString(rule.StartKeyHex) - suite.NoError(err) + re.NoError(err) oldEndKey, err := hex.DecodeString(rule.EndKeyHex) - suite.NoError(err) + re.NoError(err) parseErrData := []byte("foo") rule1 := placement.Rule{GroupID: "a", ID: "10", StartKeyHex: "XXXX", EndKeyHex: "3333", Role: placement.Voter, Count: 1} checkErrData, err := json.Marshal(rule1) - suite.NoError(err) + re.NoError(err) rule2 := placement.Rule{GroupID: "a", ID: "10", StartKeyHex: "1111", EndKeyHex: "3333", Role: placement.Voter, Count: -1} setErrData, err := json.Marshal(rule2) - suite.NoError(err) + re.NoError(err) rule3 := placement.Rule{GroupID: "a", ID: "10", StartKeyHex: "1111", EndKeyHex: "3333", Role: placement.Follower, Count: 3} updateData, err := json.Marshal(rule3) - suite.NoError(err) + re.NoError(err) newStartKey, err := hex.DecodeString(rule.StartKeyHex) - suite.NoError(err) + re.NoError(err) newEndKey, err := hex.DecodeString(rule.EndKeyHex) - suite.NoError(err) + re.NoError(err) testCases := []struct { name string @@ -162,7 +165,6 @@ func (suite *ruleTestSuite) checkSet(cluster *tests.TestCluster) { `, }, } - re := suite.Require() for _, testCase := range testCases { suite.T().Log(testCase.name) // clear suspect keyRanges to prevent test case from others @@ -172,21 +174,21 @@ func (suite *ruleTestSuite) checkSet(cluster *tests.TestCluster) { popKeyRangeMap := map[string]struct{}{} for i := 0; i < len(testCase.popKeyRange)/2; i++ { v, got := leaderServer.GetRaftCluster().PopOneSuspectKeyRange() - suite.True(got) + re.True(got) popKeyRangeMap[hex.EncodeToString(v[0])] = struct{}{} popKeyRangeMap[hex.EncodeToString(v[1])] = struct{}{} } - suite.Len(popKeyRangeMap, len(testCase.popKeyRange)) + re.Len(popKeyRangeMap, len(testCase.popKeyRange)) for k := range popKeyRangeMap { _, ok := testCase.popKeyRange[k] - suite.True(ok) + re.True(ok) } } else { err = tu.CheckPostJSON(testDialClient, urlPrefix+"/rule", testCase.rawData, tu.StatusNotOK(re), tu.StringEqual(re, testCase.response)) } - suite.NoError(err) + re.NoError(err) } } @@ -195,16 +197,16 @@ func (suite *ruleTestSuite) TestGet() { } func (suite *ruleTestSuite) checkGet(cluster *tests.TestCluster) { + re := suite.Require() leaderServer := cluster.GetLeaderServer() pdAddr := leaderServer.GetAddr() urlPrefix := fmt.Sprintf("%s%s/api/v1/config", pdAddr, apiPrefix) rule := placement.Rule{GroupID: "a", ID: "20", StartKeyHex: "1111", EndKeyHex: "3333", Role: placement.Voter, Count: 1} data, err := json.Marshal(rule) - suite.NoError(err) - re := suite.Require() + re.NoError(err) err = tu.CheckPostJSON(testDialClient, urlPrefix+"/rule", data, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) testCases := []struct { name string @@ -230,14 +232,14 @@ func (suite *ruleTestSuite) checkGet(cluster *tests.TestCluster) { var resp placement.Rule url := fmt.Sprintf("%s/rule/%s/%s", urlPrefix, testCase.rule.GroupID, testCase.rule.ID) if testCase.found { - tu.Eventually(suite.Require(), func() bool { + tu.Eventually(re, func() bool { err = tu.ReadGetJSON(re, testDialClient, url, &resp) return suite.compareRule(&resp, &testCase.rule) }) } else { err = tu.CheckGetJSON(testDialClient, url, nil, tu.Status(re, testCase.code)) } - suite.NoError(err) + re.NoError(err) } } @@ -246,21 +248,21 @@ func (suite *ruleTestSuite) TestGetAll() { } func (suite *ruleTestSuite) checkGetAll(cluster *tests.TestCluster) { + re := suite.Require() leaderServer := cluster.GetLeaderServer() pdAddr := leaderServer.GetAddr() urlPrefix := fmt.Sprintf("%s%s/api/v1/config", pdAddr, apiPrefix) rule := placement.Rule{GroupID: "b", ID: "20", StartKeyHex: "1111", EndKeyHex: "3333", Role: placement.Voter, Count: 1} data, err := json.Marshal(rule) - suite.NoError(err) - re := suite.Require() + re.NoError(err) err = tu.CheckPostJSON(testDialClient, urlPrefix+"/rule", data, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) var resp2 []*placement.Rule err = tu.ReadGetJSON(re, testDialClient, urlPrefix+"/rules", &resp2) - suite.NoError(err) - suite.NotEmpty(resp2) + re.NoError(err) + re.NotEmpty(resp2) } func (suite *ruleTestSuite) TestSetAll() { @@ -268,6 +270,7 @@ func (suite *ruleTestSuite) TestSetAll() { } func (suite *ruleTestSuite) checkSetAll(cluster *tests.TestCluster) { + re := suite.Require() leaderServer := cluster.GetLeaderServer() pdAddr := leaderServer.GetAddr() urlPrefix := fmt.Sprintf("%s%s/api/v1/config", pdAddr, apiPrefix) @@ -286,19 +289,19 @@ func (suite *ruleTestSuite) checkSetAll(cluster *tests.TestCluster) { leaderServer.GetRaftCluster().GetRuleManager().SetRule(defaultRule) successData, err := json.Marshal([]*placement.Rule{&rule1, &rule2}) - suite.NoError(err) + re.NoError(err) checkErrData, err := json.Marshal([]*placement.Rule{&rule1, &rule3}) - suite.NoError(err) + re.NoError(err) setErrData, err := json.Marshal([]*placement.Rule{&rule1, &rule4}) - suite.NoError(err) + re.NoError(err) defaultData, err := json.Marshal([]*placement.Rule{&rule1, &rule5}) - suite.NoError(err) + re.NoError(err) recoverData, err := json.Marshal([]*placement.Rule{&rule1, &rule6}) - suite.NoError(err) + re.NoError(err) testCases := []struct { name string @@ -362,19 +365,18 @@ func (suite *ruleTestSuite) checkSetAll(cluster *tests.TestCluster) { count: 3, }, } - re := suite.Require() for _, testCase := range testCases { suite.T().Log(testCase.name) if testCase.success { err := tu.CheckPostJSON(testDialClient, urlPrefix+"/rules", testCase.rawData, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) if testCase.isDefaultRule { - suite.Equal(int(leaderServer.GetPersistOptions().GetReplicationConfig().MaxReplicas), testCase.count) + re.Equal(int(leaderServer.GetPersistOptions().GetReplicationConfig().MaxReplicas), testCase.count) } } else { err := tu.CheckPostJSON(testDialClient, urlPrefix+"/rules", testCase.rawData, tu.StringEqual(re, testCase.response)) - suite.NoError(err) + re.NoError(err) } } } @@ -384,22 +386,22 @@ func (suite *ruleTestSuite) TestGetAllByGroup() { } func (suite *ruleTestSuite) checkGetAllByGroup(cluster *tests.TestCluster) { + re := suite.Require() leaderServer := cluster.GetLeaderServer() pdAddr := leaderServer.GetAddr() urlPrefix := fmt.Sprintf("%s%s/api/v1/config", pdAddr, apiPrefix) - re := suite.Require() rule := placement.Rule{GroupID: "c", ID: "20", StartKeyHex: "1111", EndKeyHex: "3333", Role: placement.Voter, Count: 1} data, err := json.Marshal(rule) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, urlPrefix+"/rule", data, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) rule1 := placement.Rule{GroupID: "c", ID: "30", StartKeyHex: "1111", EndKeyHex: "3333", Role: placement.Voter, Count: 1} data, err = json.Marshal(rule1) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, urlPrefix+"/rule", data, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) testCases := []struct { name string @@ -424,7 +426,7 @@ func (suite *ruleTestSuite) checkGetAllByGroup(cluster *tests.TestCluster) { url := fmt.Sprintf("%s/rules/group/%s", urlPrefix, testCase.groupID) tu.Eventually(re, func() bool { err = tu.ReadGetJSON(re, testDialClient, url, &resp) - suite.NoError(err) + re.NoError(err) if len(resp) != testCase.count { return false } @@ -441,16 +443,16 @@ func (suite *ruleTestSuite) TestGetAllByRegion() { } func (suite *ruleTestSuite) checkGetAllByRegion(cluster *tests.TestCluster) { + re := suite.Require() leaderServer := cluster.GetLeaderServer() pdAddr := leaderServer.GetAddr() urlPrefix := fmt.Sprintf("%s%s/api/v1/config", pdAddr, apiPrefix) rule := placement.Rule{GroupID: "e", ID: "20", StartKeyHex: "1111", EndKeyHex: "3333", Role: placement.Voter, Count: 1} data, err := json.Marshal(rule) - suite.NoError(err) - re := suite.Require() + re.NoError(err) err = tu.CheckPostJSON(testDialClient, urlPrefix+"/rule", data, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) r := core.NewTestRegionInfo(4, 1, []byte{0x22, 0x22}, []byte{0x33, 0x33}) tests.MustPutRegionInfo(re, cluster, r) @@ -485,7 +487,7 @@ func (suite *ruleTestSuite) checkGetAllByRegion(cluster *tests.TestCluster) { url := fmt.Sprintf("%s/rules/region/%s", urlPrefix, testCase.regionID) if testCase.success { - tu.Eventually(suite.Require(), func() bool { + tu.Eventually(re, func() bool { err = tu.ReadGetJSON(re, testDialClient, url, &resp) for _, r := range resp { if r.GroupID == "e" { @@ -497,7 +499,7 @@ func (suite *ruleTestSuite) checkGetAllByRegion(cluster *tests.TestCluster) { } else { err = tu.CheckGetJSON(testDialClient, url, nil, tu.Status(re, testCase.code)) } - suite.NoError(err) + re.NoError(err) } } @@ -507,16 +509,16 @@ func (suite *ruleTestSuite) TestGetAllByKey() { } func (suite *ruleTestSuite) checkGetAllByKey(cluster *tests.TestCluster) { + re := suite.Require() leaderServer := cluster.GetLeaderServer() pdAddr := leaderServer.GetAddr() urlPrefix := fmt.Sprintf("%s%s/api/v1/config", pdAddr, apiPrefix) rule := placement.Rule{GroupID: "f", ID: "40", StartKeyHex: "8888", EndKeyHex: "9111", Role: placement.Voter, Count: 1} data, err := json.Marshal(rule) - suite.NoError(err) - re := suite.Require() + re.NoError(err) err = tu.CheckPostJSON(testDialClient, urlPrefix+"/rule", data, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) testCases := []struct { name string @@ -557,7 +559,7 @@ func (suite *ruleTestSuite) checkGetAllByKey(cluster *tests.TestCluster) { } else { err = tu.CheckGetJSON(testDialClient, url, nil, tu.Status(re, testCase.code)) } - suite.NoError(err) + re.NoError(err) } } @@ -566,19 +568,20 @@ func (suite *ruleTestSuite) TestDelete() { } func (suite *ruleTestSuite) checkDelete(cluster *tests.TestCluster) { + re := suite.Require() leaderServer := cluster.GetLeaderServer() pdAddr := leaderServer.GetAddr() urlPrefix := fmt.Sprintf("%s%s/api/v1/config", pdAddr, apiPrefix) rule := placement.Rule{GroupID: "g", ID: "10", StartKeyHex: "8888", EndKeyHex: "9111", Role: placement.Voter, Count: 1} data, err := json.Marshal(rule) - suite.NoError(err) - err = tu.CheckPostJSON(testDialClient, urlPrefix+"/rule", data, tu.StatusOK(suite.Require())) - suite.NoError(err) + re.NoError(err) + err = tu.CheckPostJSON(testDialClient, urlPrefix+"/rule", data, tu.StatusOK(re)) + re.NoError(err) oldStartKey, err := hex.DecodeString(rule.StartKeyHex) - suite.NoError(err) + re.NoError(err) oldEndKey, err := hex.DecodeString(rule.EndKeyHex) - suite.NoError(err) + re.NoError(err) testCases := []struct { name string @@ -607,20 +610,20 @@ func (suite *ruleTestSuite) checkDelete(cluster *tests.TestCluster) { url := fmt.Sprintf("%s/rule/%s/%s", urlPrefix, testCase.groupID, testCase.id) // clear suspect keyRanges to prevent test case from others leaderServer.GetRaftCluster().ClearSuspectKeyRanges() - err = tu.CheckDelete(testDialClient, url, tu.StatusOK(suite.Require())) - suite.NoError(err) + err = tu.CheckDelete(testDialClient, url, tu.StatusOK(re)) + re.NoError(err) if len(testCase.popKeyRange) > 0 { popKeyRangeMap := map[string]struct{}{} for i := 0; i < len(testCase.popKeyRange)/2; i++ { v, got := leaderServer.GetRaftCluster().PopOneSuspectKeyRange() - suite.True(got) + re.True(got) popKeyRangeMap[hex.EncodeToString(v[0])] = struct{}{} popKeyRangeMap[hex.EncodeToString(v[1])] = struct{}{} } - suite.Len(popKeyRangeMap, len(testCase.popKeyRange)) + re.Len(popKeyRangeMap, len(testCase.popKeyRange)) for k := range popKeyRangeMap { _, ok := testCase.popKeyRange[k] - suite.True(ok) + re.True(ok) } } } @@ -631,6 +634,7 @@ func (suite *ruleTestSuite) TestBatch() { } func (suite *ruleTestSuite) checkBatch(cluster *tests.TestCluster) { + re := suite.Require() leaderServer := cluster.GetLeaderServer() pdAddr := leaderServer.GetAddr() urlPrefix := fmt.Sprintf("%s%s/api/v1/config", pdAddr, apiPrefix) @@ -674,19 +678,19 @@ func (suite *ruleTestSuite) checkBatch(cluster *tests.TestCluster) { } successData1, err := json.Marshal([]placement.RuleOp{opt1, opt2, opt3}) - suite.NoError(err) + re.NoError(err) successData2, err := json.Marshal([]placement.RuleOp{opt5, opt7}) - suite.NoError(err) + re.NoError(err) successData3, err := json.Marshal([]placement.RuleOp{opt4, opt6}) - suite.NoError(err) + re.NoError(err) checkErrData, err := json.Marshal([]placement.RuleOp{opt8}) - suite.NoError(err) + re.NoError(err) setErrData, err := json.Marshal([]placement.RuleOp{opt9}) - suite.NoError(err) + re.NoError(err) testCases := []struct { name string @@ -740,17 +744,16 @@ func (suite *ruleTestSuite) checkBatch(cluster *tests.TestCluster) { `, }, } - re := suite.Require() for _, testCase := range testCases { suite.T().Log(testCase.name) if testCase.success { err := tu.CheckPostJSON(testDialClient, urlPrefix+"/rules/batch", testCase.rawData, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) } else { err := tu.CheckPostJSON(testDialClient, urlPrefix+"/rules/batch", testCase.rawData, tu.StatusNotOK(re), tu.StringEqual(re, testCase.response)) - suite.NoError(err) + re.NoError(err) } } } @@ -779,9 +782,9 @@ func (suite *ruleTestSuite) checkBundle(cluster *tests.TestCluster) { } var bundles []placement.GroupBundle err := tu.ReadGetJSON(re, testDialClient, urlPrefix+"/placement-rule", &bundles) - suite.NoError(err) - suite.Len(bundles, 1) - suite.assertBundleEqual(bundles[0], b1) + re.NoError(err) + re.Len(bundles, 1) + suite.assertBundleEqual(re, bundles[0], b1) // Set b2 := placement.GroupBundle{ @@ -793,59 +796,59 @@ func (suite *ruleTestSuite) checkBundle(cluster *tests.TestCluster) { }, } data, err := json.Marshal(b2) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, urlPrefix+"/placement-rule/foo", data, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) // Get var bundle placement.GroupBundle err = tu.ReadGetJSON(re, testDialClient, urlPrefix+"/placement-rule/foo", &bundle) - suite.NoError(err) - suite.assertBundleEqual(bundle, b2) + re.NoError(err) + suite.assertBundleEqual(re, bundle, b2) // GetAll again err = tu.ReadGetJSON(re, testDialClient, urlPrefix+"/placement-rule", &bundles) - suite.NoError(err) - suite.Len(bundles, 2) - suite.assertBundleEqual(bundles[0], b1) - suite.assertBundleEqual(bundles[1], b2) + re.NoError(err) + re.Len(bundles, 2) + suite.assertBundleEqual(re, bundles[0], b1) + suite.assertBundleEqual(re, bundles[1], b2) // Delete - err = tu.CheckDelete(testDialClient, urlPrefix+"/placement-rule/pd", tu.StatusOK(suite.Require())) - suite.NoError(err) + err = tu.CheckDelete(testDialClient, urlPrefix+"/placement-rule/pd", tu.StatusOK(re)) + re.NoError(err) // GetAll again err = tu.ReadGetJSON(re, testDialClient, urlPrefix+"/placement-rule", &bundles) - suite.NoError(err) - suite.Len(bundles, 1) - suite.assertBundleEqual(bundles[0], b2) + re.NoError(err) + re.Len(bundles, 1) + suite.assertBundleEqual(re, bundles[0], b2) // SetAll b2.Rules = append(b2.Rules, &placement.Rule{GroupID: "foo", ID: "baz", Index: 2, Role: placement.Follower, Count: 1}) b2.Index, b2.Override = 0, false b3 := placement.GroupBundle{ID: "foobar", Index: 100} data, err = json.Marshal([]placement.GroupBundle{b1, b2, b3}) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, urlPrefix+"/placement-rule", data, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) // GetAll again err = tu.ReadGetJSON(re, testDialClient, urlPrefix+"/placement-rule", &bundles) - suite.NoError(err) - suite.Len(bundles, 3) - suite.assertBundleEqual(bundles[0], b2) - suite.assertBundleEqual(bundles[1], b1) - suite.assertBundleEqual(bundles[2], b3) + re.NoError(err) + re.Len(bundles, 3) + suite.assertBundleEqual(re, bundles[0], b2) + suite.assertBundleEqual(re, bundles[1], b1) + suite.assertBundleEqual(re, bundles[2], b3) // Delete using regexp - err = tu.CheckDelete(testDialClient, urlPrefix+"/placement-rule/"+url.PathEscape("foo.*")+"?regexp", tu.StatusOK(suite.Require())) - suite.NoError(err) + err = tu.CheckDelete(testDialClient, urlPrefix+"/placement-rule/"+url.PathEscape("foo.*")+"?regexp", tu.StatusOK(re)) + re.NoError(err) // GetAll again err = tu.ReadGetJSON(re, testDialClient, urlPrefix+"/placement-rule", &bundles) - suite.NoError(err) - suite.Len(bundles, 1) - suite.assertBundleEqual(bundles[0], b1) + re.NoError(err) + re.Len(bundles, 1) + suite.assertBundleEqual(re, bundles[0], b1) // Set id := "rule-without-group-id" @@ -856,24 +859,24 @@ func (suite *ruleTestSuite) checkBundle(cluster *tests.TestCluster) { }, } data, err = json.Marshal(b4) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, urlPrefix+"/placement-rule/"+id, data, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) b4.ID = id b4.Rules[0].GroupID = b4.ID // Get err = tu.ReadGetJSON(re, testDialClient, urlPrefix+"/placement-rule/"+id, &bundle) - suite.NoError(err) - suite.assertBundleEqual(bundle, b4) + re.NoError(err) + suite.assertBundleEqual(re, bundle, b4) // GetAll again err = tu.ReadGetJSON(re, testDialClient, urlPrefix+"/placement-rule", &bundles) - suite.NoError(err) - suite.Len(bundles, 2) - suite.assertBundleEqual(bundles[0], b1) - suite.assertBundleEqual(bundles[1], b4) + re.NoError(err) + re.Len(bundles, 2) + suite.assertBundleEqual(re, bundles[0], b1) + suite.assertBundleEqual(re, bundles[1], b4) // SetAll b5 := placement.GroupBundle{ @@ -884,19 +887,19 @@ func (suite *ruleTestSuite) checkBundle(cluster *tests.TestCluster) { }, } data, err = json.Marshal([]placement.GroupBundle{b1, b4, b5}) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, urlPrefix+"/placement-rule", data, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) b5.Rules[0].GroupID = b5.ID // GetAll again err = tu.ReadGetJSON(re, testDialClient, urlPrefix+"/placement-rule", &bundles) - suite.NoError(err) - suite.Len(bundles, 3) - suite.assertBundleEqual(bundles[0], b1) - suite.assertBundleEqual(bundles[1], b4) - suite.assertBundleEqual(bundles[2], b5) + re.NoError(err) + re.Len(bundles, 3) + suite.assertBundleEqual(re, bundles[0], b1) + suite.assertBundleEqual(re, bundles[1], b4) + suite.assertBundleEqual(re, bundles[2], b5) } func (suite *ruleTestSuite) TestBundleBadRequest() { @@ -904,6 +907,7 @@ func (suite *ruleTestSuite) TestBundleBadRequest() { } func (suite *ruleTestSuite) checkBundleBadRequest(cluster *tests.TestCluster) { + re := suite.Require() leaderServer := cluster.GetLeaderServer() pdAddr := leaderServer.GetAddr() urlPrefix := fmt.Sprintf("%s%s/api/v1/config", pdAddr, apiPrefix) @@ -923,9 +927,9 @@ func (suite *ruleTestSuite) checkBundleBadRequest(cluster *tests.TestCluster) { for _, testCase := range testCases { err := tu.CheckPostJSON(testDialClient, urlPrefix+testCase.uri, []byte(testCase.data), func(_ []byte, code int, _ http.Header) { - suite.Equal(testCase.ok, code == http.StatusOK) + re.Equal(testCase.ok, code == http.StatusOK) }) - suite.NoError(err) + re.NoError(err) } } @@ -1005,16 +1009,16 @@ func (suite *ruleTestSuite) checkLeaderAndVoter(cluster *tests.TestCluster) { }} for _, bundle := range bundles { data, err := json.Marshal(bundle) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, urlPrefix+"/config/placement-rule", data, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) tu.Eventually(re, func() bool { respBundle := make([]placement.GroupBundle, 0) err := tu.CheckGetJSON(testDialClient, urlPrefix+"/config/placement-rule", nil, tu.StatusOK(re), tu.ExtractJSON(re, &respBundle)) - suite.NoError(err) - suite.Len(respBundle, 1) + re.NoError(err) + re.Len(respBundle, 1) if bundle[0].Rules[0].Role == placement.Leader { return respBundle[0].Rules[0].Role == placement.Leader } @@ -1105,15 +1109,15 @@ func (suite *ruleTestSuite) checkDeleteAndUpdate(cluster *tests.TestCluster) { for _, bundle := range bundles { data, err := json.Marshal(bundle) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, urlPrefix+"/config/placement-rule", data, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) tu.Eventually(re, func() bool { respBundle := make([]placement.GroupBundle, 0) err = tu.CheckGetJSON(testDialClient, urlPrefix+"/config/placement-rule", nil, tu.StatusOK(re), tu.ExtractJSON(re, &respBundle)) - suite.NoError(err) + re.NoError(err) if len(respBundle) != len(bundle) { return false } @@ -1193,11 +1197,11 @@ func (suite *ruleTestSuite) checkConcurrencyWith(cluster *tests.TestCluster, defer wg.Done() bundle := genBundle(i) data, err := json.Marshal(bundle) - suite.NoError(err) + re.NoError(err) for j := 0; j < 10; j++ { expectResult.Lock() err = tu.CheckPostJSON(testDialClient, urlPrefix+"/config/placement-rule", data, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) expectResult.val = i expectResult.Unlock() } @@ -1207,19 +1211,19 @@ func (suite *ruleTestSuite) checkConcurrencyWith(cluster *tests.TestCluster, wg.Wait() expectResult.RLock() defer expectResult.RUnlock() - suite.NotZero(expectResult.val) + re.NotZero(expectResult.val) tu.Eventually(re, func() bool { respBundle := make([]placement.GroupBundle, 0) err := tu.CheckGetJSON(testDialClient, urlPrefix+"/config/placement-rule", nil, tu.StatusOK(re), tu.ExtractJSON(re, &respBundle)) - suite.NoError(err) - suite.Len(respBundle, 1) + re.NoError(err) + re.Len(respBundle, 1) return checkBundle(respBundle, expectResult.val) }) } -func (suite *ruleTestSuite) assertBundleEqual(b1, b2 placement.GroupBundle) { - tu.Eventually(suite.Require(), func() bool { +func (suite *ruleTestSuite) assertBundleEqual(re *require.Assertions, b1, b2 placement.GroupBundle) { + tu.Eventually(re, func() bool { return suite.compareBundle(b1, b2) }) } @@ -1337,61 +1341,61 @@ func (suite *regionRuleTestSuite) checkRegionPlacementRule(cluster *tests.TestCl u := fmt.Sprintf("%s/config/rules/region/%d/detail", urlPrefix, 1) err := tu.ReadGetJSON(re, testDialClient, u, fit) - suite.NoError(err) - suite.Len(fit.RuleFits, 1) - suite.Len(fit.OrphanPeers, 1) + re.NoError(err) + re.Len(fit.RuleFits, 1) + re.Len(fit.OrphanPeers, 1) u = fmt.Sprintf("%s/config/rules/region/%d/detail", urlPrefix, 2) fit = &placement.RegionFit{} err = tu.ReadGetJSON(re, testDialClient, u, fit) - suite.NoError(err) - suite.Len(fit.RuleFits, 2) - suite.Empty(fit.OrphanPeers) + re.NoError(err) + re.Len(fit.RuleFits, 2) + re.Empty(fit.OrphanPeers) u = fmt.Sprintf("%s/config/rules/region/%d/detail", urlPrefix, 3) fit = &placement.RegionFit{} err = tu.ReadGetJSON(re, testDialClient, u, fit) - suite.NoError(err) - suite.Empty(fit.RuleFits) - suite.Len(fit.OrphanPeers, 2) + re.NoError(err) + re.Empty(fit.RuleFits) + re.Len(fit.OrphanPeers, 2) var label labeler.LabelRule escapedID := url.PathEscape("keyspaces/0") u = fmt.Sprintf("%s/config/region-label/rule/%s", urlPrefix, escapedID) err = tu.ReadGetJSON(re, testDialClient, u, &label) - suite.NoError(err) - suite.Equal("keyspaces/0", label.ID) + re.NoError(err) + re.Equal("keyspaces/0", label.ID) var labels []labeler.LabelRule u = fmt.Sprintf("%s/config/region-label/rules", urlPrefix) err = tu.ReadGetJSON(re, testDialClient, u, &labels) - suite.NoError(err) - suite.Len(labels, 1) - suite.Equal("keyspaces/0", labels[0].ID) + re.NoError(err) + re.Len(labels, 1) + re.Equal("keyspaces/0", labels[0].ID) u = fmt.Sprintf("%s/config/region-label/rules/ids", urlPrefix) err = tu.CheckGetJSON(testDialClient, u, []byte(`["rule1", "rule3"]`), func(resp []byte, statusCode int, _ http.Header) { err := json.Unmarshal(resp, &labels) - suite.NoError(err) - suite.Empty(labels) + re.NoError(err) + re.Empty(labels) }) - suite.NoError(err) + re.NoError(err) err = tu.CheckGetJSON(testDialClient, u, []byte(`["keyspaces/0"]`), func(resp []byte, statusCode int, _ http.Header) { err := json.Unmarshal(resp, &labels) - suite.NoError(err) - suite.Len(labels, 1) - suite.Equal("keyspaces/0", labels[0].ID) + re.NoError(err) + re.Len(labels, 1) + re.Equal("keyspaces/0", labels[0].ID) }) - suite.NoError(err) + re.NoError(err) u = fmt.Sprintf("%s/config/rules/region/%d/detail", urlPrefix, 4) err = tu.CheckGetJSON(testDialClient, u, nil, tu.Status(re, http.StatusNotFound), tu.StringContain( re, "region 4 not found")) - suite.NoError(err) + re.NoError(err) u = fmt.Sprintf("%s/config/rules/region/%s/detail", urlPrefix, "id") err = tu.CheckGetJSON(testDialClient, u, nil, tu.Status(re, http.StatusBadRequest), tu.StringContain( re, errs.ErrRegionInvalidID.Error())) - suite.NoError(err) + re.NoError(err) data := make(map[string]interface{}) data["enable-placement-rules"] = "false" @@ -1409,5 +1413,5 @@ func (suite *regionRuleTestSuite) checkRegionPlacementRule(cluster *tests.TestCl u = fmt.Sprintf("%s/config/rules/region/%d/detail", urlPrefix, 1) err = tu.CheckGetJSON(testDialClient, u, nil, tu.Status(re, http.StatusPreconditionFailed), tu.StringContain( re, "placement rules feature is disabled")) - suite.NoError(err) + re.NoError(err) } diff --git a/tests/server/api/scheduler_test.go b/tests/server/api/scheduler_test.go index d0472795f93..49ec614d5a7 100644 --- a/tests/server/api/scheduler_test.go +++ b/tests/server/api/scheduler_test.go @@ -26,6 +26,7 @@ import ( "github.com/pingcap/failpoint" "github.com/pingcap/kvproto/pkg/metapb" + "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" sc "github.com/tikv/pd/pkg/schedule/config" "github.com/tikv/pd/pkg/slice" @@ -47,13 +48,15 @@ func TestScheduleTestSuite(t *testing.T) { } func (suite *scheduleTestSuite) SetupSuite() { - suite.NoError(failpoint.Enable("github.com/tikv/pd/server/cluster/skipStoreConfigSync", `return(true)`)) + re := suite.Require() + re.NoError(failpoint.Enable("github.com/tikv/pd/server/cluster/skipStoreConfigSync", `return(true)`)) suite.env = tests.NewSchedulingTestEnvironment(suite.T()) } func (suite *scheduleTestSuite) TearDownSuite() { + re := suite.Require() suite.env.Cleanup() - suite.NoError(failpoint.Disable("github.com/tikv/pd/server/cluster/skipStoreConfigSync")) + re.NoError(failpoint.Disable("github.com/tikv/pd/server/cluster/skipStoreConfigSync")) } func (suite *scheduleTestSuite) TestOriginAPI() { @@ -61,6 +64,7 @@ func (suite *scheduleTestSuite) TestOriginAPI() { } func (suite *scheduleTestSuite) checkOriginAPI(cluster *tests.TestCluster) { + re := suite.Require() leaderAddr := cluster.GetLeaderServer().GetAddr() urlPrefix := fmt.Sprintf("%s/pd/api/v1/schedulers", leaderAddr) for i := 1; i <= 4; i++ { @@ -70,58 +74,57 @@ func (suite *scheduleTestSuite) checkOriginAPI(cluster *tests.TestCluster) { NodeState: metapb.NodeState_Serving, LastHeartbeat: time.Now().UnixNano(), } - tests.MustPutStore(suite.Require(), cluster, store) + tests.MustPutStore(re, cluster, store) } input := make(map[string]interface{}) input["name"] = "evict-leader-scheduler" input["store_id"] = 1 body, err := json.Marshal(input) - suite.NoError(err) - re := suite.Require() - suite.NoError(tu.CheckPostJSON(testDialClient, urlPrefix, body, tu.StatusOK(re))) + re.NoError(err) + re.NoError(tu.CheckPostJSON(testDialClient, urlPrefix, body, tu.StatusOK(re))) suite.assertSchedulerExists(urlPrefix, "evict-leader-scheduler") resp := make(map[string]interface{}) listURL := fmt.Sprintf("%s%s%s/%s/list", leaderAddr, apiPrefix, server.SchedulerConfigHandlerPath, "evict-leader-scheduler") - suite.NoError(tu.ReadGetJSON(re, testDialClient, listURL, &resp)) - suite.Len(resp["store-id-ranges"], 1) + re.NoError(tu.ReadGetJSON(re, testDialClient, listURL, &resp)) + re.Len(resp["store-id-ranges"], 1) input1 := make(map[string]interface{}) input1["name"] = "evict-leader-scheduler" input1["store_id"] = 2 body, err = json.Marshal(input1) - suite.NoError(err) - suite.NoError(failpoint.Enable("github.com/tikv/pd/pkg/schedule/schedulers/persistFail", "return(true)")) - suite.NoError(tu.CheckPostJSON(testDialClient, urlPrefix, body, tu.StatusNotOK(re))) + re.NoError(err) + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/schedule/schedulers/persistFail", "return(true)")) + re.NoError(tu.CheckPostJSON(testDialClient, urlPrefix, body, tu.StatusNotOK(re))) suite.assertSchedulerExists(urlPrefix, "evict-leader-scheduler") resp = make(map[string]interface{}) - suite.NoError(tu.ReadGetJSON(re, testDialClient, listURL, &resp)) - suite.Len(resp["store-id-ranges"], 1) - suite.NoError(failpoint.Disable("github.com/tikv/pd/pkg/schedule/schedulers/persistFail")) - suite.NoError(tu.CheckPostJSON(testDialClient, urlPrefix, body, tu.StatusOK(re))) + re.NoError(tu.ReadGetJSON(re, testDialClient, listURL, &resp)) + re.Len(resp["store-id-ranges"], 1) + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/schedule/schedulers/persistFail")) + re.NoError(tu.CheckPostJSON(testDialClient, urlPrefix, body, tu.StatusOK(re))) suite.assertSchedulerExists(urlPrefix, "evict-leader-scheduler") resp = make(map[string]interface{}) - suite.NoError(tu.ReadGetJSON(re, testDialClient, listURL, &resp)) - suite.Len(resp["store-id-ranges"], 2) + re.NoError(tu.ReadGetJSON(re, testDialClient, listURL, &resp)) + re.Len(resp["store-id-ranges"], 2) deleteURL := fmt.Sprintf("%s/%s", urlPrefix, "evict-leader-scheduler-1") err = tu.CheckDelete(testDialClient, deleteURL, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) suite.assertSchedulerExists(urlPrefix, "evict-leader-scheduler") resp1 := make(map[string]interface{}) - suite.NoError(tu.ReadGetJSON(re, testDialClient, listURL, &resp1)) - suite.Len(resp1["store-id-ranges"], 1) + re.NoError(tu.ReadGetJSON(re, testDialClient, listURL, &resp1)) + re.Len(resp1["store-id-ranges"], 1) deleteURL = fmt.Sprintf("%s/%s", urlPrefix, "evict-leader-scheduler-2") - suite.NoError(failpoint.Enable("github.com/tikv/pd/server/config/persistFail", "return(true)")) + re.NoError(failpoint.Enable("github.com/tikv/pd/server/config/persistFail", "return(true)")) err = tu.CheckDelete(testDialClient, deleteURL, tu.Status(re, http.StatusInternalServerError)) - suite.NoError(err) + re.NoError(err) suite.assertSchedulerExists(urlPrefix, "evict-leader-scheduler") - suite.NoError(failpoint.Disable("github.com/tikv/pd/server/config/persistFail")) + re.NoError(failpoint.Disable("github.com/tikv/pd/server/config/persistFail")) err = tu.CheckDelete(testDialClient, deleteURL, tu.StatusOK(re)) - suite.NoError(err) - suite.assertNoScheduler(urlPrefix, "evict-leader-scheduler") - suite.NoError(tu.CheckGetJSON(testDialClient, listURL, nil, tu.Status(re, http.StatusNotFound))) + re.NoError(err) + suite.assertNoScheduler(re, urlPrefix, "evict-leader-scheduler") + re.NoError(tu.CheckGetJSON(testDialClient, listURL, nil, tu.Status(re, http.StatusNotFound))) err = tu.CheckDelete(testDialClient, deleteURL, tu.Status(re, http.StatusNotFound)) - suite.NoError(err) + re.NoError(err) } func (suite *scheduleTestSuite) TestAPI() { @@ -139,7 +142,7 @@ func (suite *scheduleTestSuite) checkAPI(cluster *tests.TestCluster) { NodeState: metapb.NodeState_Serving, LastHeartbeat: time.Now().UnixNano(), } - tests.MustPutStore(suite.Require(), cluster, store) + tests.MustPutStore(re, cluster, store) } type arg struct { @@ -158,17 +161,17 @@ func (suite *scheduleTestSuite) checkAPI(cluster *tests.TestCluster) { extraTestFunc: func(name string) { resp := make(map[string]interface{}) listURL := fmt.Sprintf("%s%s%s/%s/list", leaderAddr, apiPrefix, server.SchedulerConfigHandlerPath, name) - suite.NoError(tu.ReadGetJSON(re, testDialClient, listURL, &resp)) - suite.Equal(4.0, resp["batch"]) + re.NoError(tu.ReadGetJSON(re, testDialClient, listURL, &resp)) + re.Equal(4.0, resp["batch"]) dataMap := make(map[string]interface{}) dataMap["batch"] = 3 updateURL := fmt.Sprintf("%s%s%s/%s/config", leaderAddr, apiPrefix, server.SchedulerConfigHandlerPath, name) body, err := json.Marshal(dataMap) - suite.NoError(err) - suite.NoError(tu.CheckPostJSON(testDialClient, updateURL, body, tu.StatusOK(re))) + re.NoError(err) + re.NoError(tu.CheckPostJSON(testDialClient, updateURL, body, tu.StatusOK(re))) tu.Eventually(re, func() bool { // wait for scheduling server to be synced. resp = make(map[string]interface{}) - suite.NoError(tu.ReadGetJSON(re, testDialClient, listURL, &resp)) + re.NoError(tu.ReadGetJSON(re, testDialClient, listURL, &resp)) return resp["batch"] == 3.0 }) @@ -176,33 +179,33 @@ func (suite *scheduleTestSuite) checkAPI(cluster *tests.TestCluster) { err = tu.CheckPostJSON(testDialClient, updateURL, body, tu.StatusOK(re), tu.StringEqual(re, "\"Config is the same with origin, so do nothing.\"\n")) - suite.NoError(err) + re.NoError(err) // update invalidate batch dataMap = map[string]interface{}{} dataMap["batch"] = 100 body, err = json.Marshal(dataMap) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, updateURL, body, tu.Status(re, http.StatusBadRequest), tu.StringEqual(re, "\"invalid batch size which should be an integer between 1 and 10\"\n")) - suite.NoError(err) + re.NoError(err) resp = make(map[string]interface{}) - suite.NoError(tu.ReadGetJSON(re, testDialClient, listURL, &resp)) - suite.Equal(3.0, resp["batch"]) + re.NoError(tu.ReadGetJSON(re, testDialClient, listURL, &resp)) + re.Equal(3.0, resp["batch"]) // empty body err = tu.CheckPostJSON(testDialClient, updateURL, nil, tu.Status(re, http.StatusInternalServerError), tu.StringEqual(re, "\"unexpected end of JSON input\"\n")) - suite.NoError(err) + re.NoError(err) // config item not found dataMap = map[string]interface{}{} dataMap["error"] = 3 body, err = json.Marshal(dataMap) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, updateURL, body, tu.Status(re, http.StatusBadRequest), tu.StringEqual(re, "\"Config item is not found.\"\n")) - suite.NoError(err) + re.NoError(err) }, }, { @@ -211,7 +214,7 @@ func (suite *scheduleTestSuite) checkAPI(cluster *tests.TestCluster) { extraTestFunc: func(name string) { resp := make(map[string]interface{}) listURL := fmt.Sprintf("%s%s%s/%s/list", leaderAddr, apiPrefix, server.SchedulerConfigHandlerPath, name) - suite.NoError(tu.ReadGetJSON(re, testDialClient, listURL, &resp)) + re.NoError(tu.ReadGetJSON(re, testDialClient, listURL, &resp)) expectMap := map[string]interface{}{ "min-hot-byte-rate": 100.0, "min-hot-key-rate": 10.0, @@ -236,36 +239,36 @@ func (suite *scheduleTestSuite) checkAPI(cluster *tests.TestCluster) { } re.Equal(len(expectMap), len(resp), "expect %v, got %v", expectMap, resp) for key := range expectMap { - suite.Equal(expectMap[key], resp[key]) + re.Equal(expectMap[key], resp[key]) } dataMap := make(map[string]interface{}) dataMap["max-zombie-rounds"] = 5.0 expectMap["max-zombie-rounds"] = 5.0 updateURL := fmt.Sprintf("%s%s%s/%s/config", leaderAddr, apiPrefix, server.SchedulerConfigHandlerPath, name) body, err := json.Marshal(dataMap) - suite.NoError(err) - suite.NoError(tu.CheckPostJSON(testDialClient, updateURL, body, tu.StatusOK(re))) + re.NoError(err) + re.NoError(tu.CheckPostJSON(testDialClient, updateURL, body, tu.StatusOK(re))) resp = make(map[string]interface{}) - suite.NoError(tu.ReadGetJSON(re, testDialClient, listURL, &resp)) + re.NoError(tu.ReadGetJSON(re, testDialClient, listURL, &resp)) for key := range expectMap { - suite.Equal(expectMap[key], resp[key], "key %s", key) + re.Equal(expectMap[key], resp[key], "key %s", key) } // update again err = tu.CheckPostJSON(testDialClient, updateURL, body, tu.StatusOK(re), tu.StringEqual(re, "Config is the same with origin, so do nothing.")) - suite.NoError(err) + re.NoError(err) // config item not found dataMap = map[string]interface{}{} dataMap["error"] = 3 body, err = json.Marshal(dataMap) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, updateURL, body, tu.Status(re, http.StatusBadRequest), tu.StringEqual(re, "Config item is not found.")) - suite.NoError(err) + re.NoError(err) }, }, { @@ -274,37 +277,37 @@ func (suite *scheduleTestSuite) checkAPI(cluster *tests.TestCluster) { extraTestFunc: func(name string) { resp := make(map[string]interface{}) listURL := fmt.Sprintf("%s%s%s/%s/list", leaderAddr, apiPrefix, server.SchedulerConfigHandlerPath, name) - suite.NoError(tu.ReadGetJSON(re, testDialClient, listURL, &resp)) - suite.Equal(3.0, resp["degree"]) - suite.Equal(0.0, resp["split-limit"]) + re.NoError(tu.ReadGetJSON(re, testDialClient, listURL, &resp)) + re.Equal(3.0, resp["degree"]) + re.Equal(0.0, resp["split-limit"]) dataMap := make(map[string]interface{}) dataMap["degree"] = 4 updateURL := fmt.Sprintf("%s%s%s/%s/config", leaderAddr, apiPrefix, server.SchedulerConfigHandlerPath, name) body, err := json.Marshal(dataMap) - suite.NoError(err) - suite.NoError(tu.CheckPostJSON(testDialClient, updateURL, body, tu.StatusOK(re))) + re.NoError(err) + re.NoError(tu.CheckPostJSON(testDialClient, updateURL, body, tu.StatusOK(re))) resp = make(map[string]interface{}) - suite.NoError(tu.ReadGetJSON(re, testDialClient, listURL, &resp)) - suite.Equal(4.0, resp["degree"]) + re.NoError(tu.ReadGetJSON(re, testDialClient, listURL, &resp)) + re.Equal(4.0, resp["degree"]) // update again err = tu.CheckPostJSON(testDialClient, updateURL, body, tu.StatusOK(re), tu.StringEqual(re, "Config is the same with origin, so do nothing.")) - suite.NoError(err) + re.NoError(err) // empty body err = tu.CheckPostJSON(testDialClient, updateURL, nil, tu.Status(re, http.StatusInternalServerError), tu.StringEqual(re, "\"unexpected end of JSON input\"\n")) - suite.NoError(err) + re.NoError(err) // config item not found dataMap = map[string]interface{}{} dataMap["error"] = 3 body, err = json.Marshal(dataMap) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, updateURL, body, tu.Status(re, http.StatusBadRequest), tu.StringEqual(re, "Config item is not found.")) - suite.NoError(err) + re.NoError(err) }, }, { @@ -329,48 +332,48 @@ func (suite *scheduleTestSuite) checkAPI(cluster *tests.TestCluster) { extraTestFunc: func(name string) { resp := make(map[string]interface{}) listURL := fmt.Sprintf("%s%s%s/%s/list", leaderAddr, apiPrefix, server.SchedulerConfigHandlerPath, name) - suite.NoError(tu.ReadGetJSON(re, testDialClient, listURL, &resp)) - suite.Equal(4.0, resp["batch"]) + re.NoError(tu.ReadGetJSON(re, testDialClient, listURL, &resp)) + re.Equal(4.0, resp["batch"]) dataMap := make(map[string]interface{}) dataMap["batch"] = 3 updateURL := fmt.Sprintf("%s%s%s/%s/config", leaderAddr, apiPrefix, server.SchedulerConfigHandlerPath, name) body, err := json.Marshal(dataMap) - suite.NoError(err) - suite.NoError(tu.CheckPostJSON(testDialClient, updateURL, body, tu.StatusOK(re))) + re.NoError(err) + re.NoError(tu.CheckPostJSON(testDialClient, updateURL, body, tu.StatusOK(re))) resp = make(map[string]interface{}) - suite.NoError(tu.ReadGetJSON(re, testDialClient, listURL, &resp)) - suite.Equal(3.0, resp["batch"]) + re.NoError(tu.ReadGetJSON(re, testDialClient, listURL, &resp)) + re.Equal(3.0, resp["batch"]) // update again err = tu.CheckPostJSON(testDialClient, updateURL, body, tu.StatusOK(re), tu.StringEqual(re, "\"Config is the same with origin, so do nothing.\"\n")) - suite.NoError(err) + re.NoError(err) // update invalidate batch dataMap = map[string]interface{}{} dataMap["batch"] = 100 body, err = json.Marshal(dataMap) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, updateURL, body, tu.Status(re, http.StatusBadRequest), tu.StringEqual(re, "\"invalid batch size which should be an integer between 1 and 10\"\n")) - suite.NoError(err) + re.NoError(err) resp = make(map[string]interface{}) - suite.NoError(tu.ReadGetJSON(re, testDialClient, listURL, &resp)) - suite.Equal(3.0, resp["batch"]) + re.NoError(tu.ReadGetJSON(re, testDialClient, listURL, &resp)) + re.Equal(3.0, resp["batch"]) // empty body err = tu.CheckPostJSON(testDialClient, updateURL, nil, tu.Status(re, http.StatusInternalServerError), tu.StringEqual(re, "\"unexpected end of JSON input\"\n")) - suite.NoError(err) + re.NoError(err) // config item not found dataMap = map[string]interface{}{} dataMap["error"] = 3 body, err = json.Marshal(dataMap) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, updateURL, body, tu.Status(re, http.StatusBadRequest), tu.StringEqual(re, "\"Config item is not found.\"\n")) - suite.NoError(err) + re.NoError(err) }, }, { @@ -380,10 +383,10 @@ func (suite *scheduleTestSuite) checkAPI(cluster *tests.TestCluster) { extraTestFunc: func(name string) { resp := make(map[string]interface{}) listURL := fmt.Sprintf("%s%s%s/%s/list", leaderAddr, apiPrefix, server.SchedulerConfigHandlerPath, name) - suite.NoError(tu.ReadGetJSON(re, testDialClient, listURL, &resp)) + re.NoError(tu.ReadGetJSON(re, testDialClient, listURL, &resp)) exceptMap := make(map[string]interface{}) exceptMap["1"] = []interface{}{map[string]interface{}{"end-key": "", "start-key": ""}} - suite.Equal(exceptMap, resp["store-id-ranges"]) + re.Equal(exceptMap, resp["store-id-ranges"]) // using /pd/v1/schedule-config/grant-leader-scheduler/config to add new store to grant-leader-scheduler input := make(map[string]interface{}) @@ -391,23 +394,23 @@ func (suite *scheduleTestSuite) checkAPI(cluster *tests.TestCluster) { input["store_id"] = 2 updateURL := fmt.Sprintf("%s%s%s/%s/config", leaderAddr, apiPrefix, server.SchedulerConfigHandlerPath, name) body, err := json.Marshal(input) - suite.NoError(err) - suite.NoError(tu.CheckPostJSON(testDialClient, updateURL, body, tu.StatusOK(re))) + re.NoError(err) + re.NoError(tu.CheckPostJSON(testDialClient, updateURL, body, tu.StatusOK(re))) resp = make(map[string]interface{}) - suite.NoError(tu.ReadGetJSON(re, testDialClient, listURL, &resp)) + re.NoError(tu.ReadGetJSON(re, testDialClient, listURL, &resp)) exceptMap["2"] = []interface{}{map[string]interface{}{"end-key": "", "start-key": ""}} - suite.Equal(exceptMap, resp["store-id-ranges"]) + re.Equal(exceptMap, resp["store-id-ranges"]) // using /pd/v1/schedule-config/grant-leader-scheduler/config to delete exists store from grant-leader-scheduler deleteURL := fmt.Sprintf("%s%s%s/%s/delete/%s", leaderAddr, apiPrefix, server.SchedulerConfigHandlerPath, name, "2") err = tu.CheckDelete(testDialClient, deleteURL, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) resp = make(map[string]interface{}) - suite.NoError(tu.ReadGetJSON(re, testDialClient, listURL, &resp)) + re.NoError(tu.ReadGetJSON(re, testDialClient, listURL, &resp)) delete(exceptMap, "2") - suite.Equal(exceptMap, resp["store-id-ranges"]) + re.Equal(exceptMap, resp["store-id-ranges"]) err = tu.CheckDelete(testDialClient, deleteURL, tu.Status(re, http.StatusNotFound)) - suite.NoError(err) + re.NoError(err) }, }, { @@ -418,21 +421,21 @@ func (suite *scheduleTestSuite) checkAPI(cluster *tests.TestCluster) { extraTestFunc: func(name string) { resp := make(map[string]interface{}) listURL := fmt.Sprintf("%s%s%s/%s/list", leaderAddr, apiPrefix, server.SchedulerConfigHandlerPath, name) - suite.NoError(tu.ReadGetJSON(re, testDialClient, listURL, &resp)) - suite.Equal("", resp["start-key"]) - suite.Equal("", resp["end-key"]) - suite.Equal("test", resp["range-name"]) + re.NoError(tu.ReadGetJSON(re, testDialClient, listURL, &resp)) + re.Equal("", resp["start-key"]) + re.Equal("", resp["end-key"]) + re.Equal("test", resp["range-name"]) resp["start-key"] = "a_00" resp["end-key"] = "a_99" updateURL := fmt.Sprintf("%s%s%s/%s/config", leaderAddr, apiPrefix, server.SchedulerConfigHandlerPath, name) body, err := json.Marshal(resp) - suite.NoError(err) - suite.NoError(tu.CheckPostJSON(testDialClient, updateURL, body, tu.StatusOK(re))) + re.NoError(err) + re.NoError(tu.CheckPostJSON(testDialClient, updateURL, body, tu.StatusOK(re))) resp = make(map[string]interface{}) - suite.NoError(tu.ReadGetJSON(re, testDialClient, listURL, &resp)) - suite.Equal("a_00", resp["start-key"]) - suite.Equal("a_99", resp["end-key"]) - suite.Equal("test", resp["range-name"]) + re.NoError(tu.ReadGetJSON(re, testDialClient, listURL, &resp)) + re.Equal("a_00", resp["start-key"]) + re.Equal("a_99", resp["end-key"]) + re.Equal("test", resp["range-name"]) }, }, { @@ -443,10 +446,10 @@ func (suite *scheduleTestSuite) checkAPI(cluster *tests.TestCluster) { extraTestFunc: func(name string) { resp := make(map[string]interface{}) listURL := fmt.Sprintf("%s%s%s/%s/list", leaderAddr, apiPrefix, server.SchedulerConfigHandlerPath, name) - suite.NoError(tu.ReadGetJSON(re, testDialClient, listURL, &resp)) + re.NoError(tu.ReadGetJSON(re, testDialClient, listURL, &resp)) exceptMap := make(map[string]interface{}) exceptMap["3"] = []interface{}{map[string]interface{}{"end-key": "", "start-key": ""}} - suite.Equal(exceptMap, resp["store-id-ranges"]) + re.Equal(exceptMap, resp["store-id-ranges"]) // using /pd/v1/schedule-config/evict-leader-scheduler/config to add new store to evict-leader-scheduler input := make(map[string]interface{}) @@ -454,11 +457,11 @@ func (suite *scheduleTestSuite) checkAPI(cluster *tests.TestCluster) { input["store_id"] = 4 updateURL := fmt.Sprintf("%s%s%s/%s/config", leaderAddr, apiPrefix, server.SchedulerConfigHandlerPath, name) body, err := json.Marshal(input) - suite.NoError(err) - suite.NoError(tu.CheckPostJSON(testDialClient, updateURL, body, tu.StatusOK(re))) + re.NoError(err) + re.NoError(tu.CheckPostJSON(testDialClient, updateURL, body, tu.StatusOK(re))) resp = make(map[string]interface{}) tu.Eventually(re, func() bool { - suite.NoError(tu.ReadGetJSON(re, testDialClient, listURL, &resp)) + re.NoError(tu.ReadGetJSON(re, testDialClient, listURL, &resp)) exceptMap["4"] = []interface{}{map[string]interface{}{"end-key": "", "start-key": ""}} return reflect.DeepEqual(exceptMap, resp["store-id-ranges"]) }) @@ -466,15 +469,15 @@ func (suite *scheduleTestSuite) checkAPI(cluster *tests.TestCluster) { // using /pd/v1/schedule-config/evict-leader-scheduler/config to delete exist store from evict-leader-scheduler deleteURL := fmt.Sprintf("%s%s%s/%s/delete/%s", leaderAddr, apiPrefix, server.SchedulerConfigHandlerPath, name, "4") err = tu.CheckDelete(testDialClient, deleteURL, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) resp = make(map[string]interface{}) tu.Eventually(re, func() bool { - suite.NoError(tu.ReadGetJSON(re, testDialClient, listURL, &resp)) + re.NoError(tu.ReadGetJSON(re, testDialClient, listURL, &resp)) delete(exceptMap, "4") return reflect.DeepEqual(exceptMap, resp["store-id-ranges"]) }) err = tu.CheckDelete(testDialClient, deleteURL, tu.Status(re, http.StatusNotFound)) - suite.NoError(err) + re.NoError(err) }, }, { @@ -489,13 +492,13 @@ func (suite *scheduleTestSuite) checkAPI(cluster *tests.TestCluster) { input[a.opt] = a.value } body, err := json.Marshal(input) - suite.NoError(err) - suite.testPauseOrResume(urlPrefix, testCase.name, testCase.createdName, body) + re.NoError(err) + suite.testPauseOrResume(re, urlPrefix, testCase.name, testCase.createdName, body) if testCase.extraTestFunc != nil { testCase.extraTestFunc(testCase.createdName) } - suite.deleteScheduler(urlPrefix, testCase.createdName) - suite.assertNoScheduler(urlPrefix, testCase.createdName) + suite.deleteScheduler(re, urlPrefix, testCase.createdName) + suite.assertNoScheduler(re, urlPrefix, testCase.createdName) } // test pause and resume all schedulers. @@ -508,8 +511,8 @@ func (suite *scheduleTestSuite) checkAPI(cluster *tests.TestCluster) { input[a.opt] = a.value } body, err := json.Marshal(input) - suite.NoError(err) - suite.addScheduler(urlPrefix, body) + re.NoError(err) + suite.addScheduler(re, urlPrefix, body) suite.assertSchedulerExists(urlPrefix, testCase.createdName) // wait for scheduler to be synced. if testCase.extraTestFunc != nil { testCase.extraTestFunc(testCase.createdName) @@ -520,51 +523,51 @@ func (suite *scheduleTestSuite) checkAPI(cluster *tests.TestCluster) { input := make(map[string]interface{}) input["delay"] = 30 pauseArgs, err := json.Marshal(input) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, urlPrefix+"/all", pauseArgs, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) for _, testCase := range testCases { createdName := testCase.createdName if createdName == "" { createdName = testCase.name } - isPaused := suite.isSchedulerPaused(urlPrefix, createdName) - suite.True(isPaused) + isPaused := suite.isSchedulerPaused(re, urlPrefix, createdName) + re.True(isPaused) } input["delay"] = 1 pauseArgs, err = json.Marshal(input) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, urlPrefix+"/all", pauseArgs, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) time.Sleep(time.Second) for _, testCase := range testCases { createdName := testCase.createdName if createdName == "" { createdName = testCase.name } - isPaused := suite.isSchedulerPaused(urlPrefix, createdName) - suite.False(isPaused) + isPaused := suite.isSchedulerPaused(re, urlPrefix, createdName) + re.False(isPaused) } // test resume all schedulers. input["delay"] = 30 pauseArgs, err = json.Marshal(input) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, urlPrefix+"/all", pauseArgs, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) input["delay"] = 0 pauseArgs, err = json.Marshal(input) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, urlPrefix+"/all", pauseArgs, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) for _, testCase := range testCases { createdName := testCase.createdName if createdName == "" { createdName = testCase.name } - isPaused := suite.isSchedulerPaused(urlPrefix, createdName) - suite.False(isPaused) + isPaused := suite.isSchedulerPaused(re, urlPrefix, createdName) + re.False(isPaused) } // delete schedulers. @@ -573,8 +576,8 @@ func (suite *scheduleTestSuite) checkAPI(cluster *tests.TestCluster) { if createdName == "" { createdName = testCase.name } - suite.deleteScheduler(urlPrefix, createdName) - suite.assertNoScheduler(urlPrefix, createdName) + suite.deleteScheduler(re, urlPrefix, createdName) + suite.assertNoScheduler(re, urlPrefix, createdName) } } @@ -593,60 +596,59 @@ func (suite *scheduleTestSuite) checkDisable(cluster *tests.TestCluster) { NodeState: metapb.NodeState_Serving, LastHeartbeat: time.Now().UnixNano(), } - tests.MustPutStore(suite.Require(), cluster, store) + tests.MustPutStore(re, cluster, store) } name := "shuffle-leader-scheduler" input := make(map[string]interface{}) input["name"] = name body, err := json.Marshal(input) - suite.NoError(err) - suite.addScheduler(urlPrefix, body) + re.NoError(err) + suite.addScheduler(re, urlPrefix, body) u := fmt.Sprintf("%s%s/api/v1/config/schedule", leaderAddr, apiPrefix) var scheduleConfig sc.ScheduleConfig err = tu.ReadGetJSON(re, testDialClient, u, &scheduleConfig) - suite.NoError(err) + re.NoError(err) originSchedulers := scheduleConfig.Schedulers scheduleConfig.Schedulers = sc.SchedulerConfigs{sc.SchedulerConfig{Type: "shuffle-leader", Disable: true}} body, err = json.Marshal(scheduleConfig) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, u, body, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) - suite.assertNoScheduler(urlPrefix, name) + suite.assertNoScheduler(re, urlPrefix, name) suite.assertSchedulerExists(fmt.Sprintf("%s?status=disabled", urlPrefix), name) // reset schedule config scheduleConfig.Schedulers = originSchedulers body, err = json.Marshal(scheduleConfig) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, u, body, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) - suite.deleteScheduler(urlPrefix, name) - suite.assertNoScheduler(urlPrefix, name) + suite.deleteScheduler(re, urlPrefix, name) + suite.assertNoScheduler(re, urlPrefix, name) } -func (suite *scheduleTestSuite) addScheduler(urlPrefix string, body []byte) { - err := tu.CheckPostJSON(testDialClient, urlPrefix, body, tu.StatusOK(suite.Require())) - suite.NoError(err) +func (suite *scheduleTestSuite) addScheduler(re *require.Assertions, urlPrefix string, body []byte) { + err := tu.CheckPostJSON(testDialClient, urlPrefix, body, tu.StatusOK(re)) + re.NoError(err) } -func (suite *scheduleTestSuite) deleteScheduler(urlPrefix string, createdName string) { +func (suite *scheduleTestSuite) deleteScheduler(re *require.Assertions, urlPrefix string, createdName string) { deleteURL := fmt.Sprintf("%s/%s", urlPrefix, createdName) - err := tu.CheckDelete(testDialClient, deleteURL, tu.StatusOK(suite.Require())) - suite.NoError(err) + err := tu.CheckDelete(testDialClient, deleteURL, tu.StatusOK(re)) + re.NoError(err) } -func (suite *scheduleTestSuite) testPauseOrResume(urlPrefix string, name, createdName string, body []byte) { - re := suite.Require() +func (suite *scheduleTestSuite) testPauseOrResume(re *require.Assertions, urlPrefix string, name, createdName string, body []byte) { if createdName == "" { createdName = name } var schedulers []string - tu.ReadGetJSON(suite.Require(), testDialClient, urlPrefix, &schedulers) + tu.ReadGetJSON(re, testDialClient, urlPrefix, &schedulers) if !slice.Contains(schedulers, createdName) { err := tu.CheckPostJSON(testDialClient, urlPrefix, body, tu.StatusOK(re)) re.NoError(err) @@ -657,34 +659,34 @@ func (suite *scheduleTestSuite) testPauseOrResume(urlPrefix string, name, create input := make(map[string]interface{}) input["delay"] = 30 pauseArgs, err := json.Marshal(input) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, urlPrefix+"/"+createdName, pauseArgs, tu.StatusOK(re)) - suite.NoError(err) - isPaused := suite.isSchedulerPaused(urlPrefix, createdName) - suite.True(isPaused) + re.NoError(err) + isPaused := suite.isSchedulerPaused(re, urlPrefix, createdName) + re.True(isPaused) input["delay"] = 1 pauseArgs, err = json.Marshal(input) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, urlPrefix+"/"+createdName, pauseArgs, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) time.Sleep(time.Second * 2) - isPaused = suite.isSchedulerPaused(urlPrefix, createdName) - suite.False(isPaused) + isPaused = suite.isSchedulerPaused(re, urlPrefix, createdName) + re.False(isPaused) // test resume. input = make(map[string]interface{}) input["delay"] = 30 pauseArgs, err = json.Marshal(input) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, urlPrefix+"/"+createdName, pauseArgs, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) input["delay"] = 0 pauseArgs, err = json.Marshal(input) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, urlPrefix+"/"+createdName, pauseArgs, tu.StatusOK(re)) - suite.NoError(err) - isPaused = suite.isSchedulerPaused(urlPrefix, createdName) - suite.False(isPaused) + re.NoError(err) + isPaused = suite.isSchedulerPaused(re, urlPrefix, createdName) + re.False(isPaused) } func (suite *scheduleTestSuite) TestEmptySchedulers() { @@ -702,29 +704,29 @@ func (suite *scheduleTestSuite) checkEmptySchedulers(cluster *tests.TestCluster) NodeState: metapb.NodeState_Serving, LastHeartbeat: time.Now().UnixNano(), } - tests.MustPutStore(suite.Require(), cluster, store) + tests.MustPutStore(re, cluster, store) } for _, query := range []string{"", "?status=paused", "?status=disabled"} { schedulers := make([]string, 0) - suite.NoError(tu.ReadGetJSON(re, testDialClient, urlPrefix+query, &schedulers)) + re.NoError(tu.ReadGetJSON(re, testDialClient, urlPrefix+query, &schedulers)) for _, scheduler := range schedulers { if strings.Contains(query, "disable") { input := make(map[string]interface{}) input["name"] = scheduler body, err := json.Marshal(input) - suite.NoError(err) - suite.addScheduler(urlPrefix, body) + re.NoError(err) + suite.addScheduler(re, urlPrefix, body) } else { - suite.deleteScheduler(urlPrefix, scheduler) + suite.deleteScheduler(re, urlPrefix, scheduler) } } tu.Eventually(re, func() bool { resp, err := apiutil.GetJSON(testDialClient, urlPrefix+query, nil) - suite.NoError(err) + re.NoError(err) defer resp.Body.Close() - suite.Equal(http.StatusOK, resp.StatusCode) + re.Equal(http.StatusOK, resp.StatusCode) b, err := io.ReadAll(resp.Body) - suite.NoError(err) + re.NoError(err) return strings.Contains(string(b), "[]") && !strings.Contains(string(b), "null") }) } @@ -736,28 +738,26 @@ func (suite *scheduleTestSuite) assertSchedulerExists(urlPrefix string, schedule tu.Eventually(re, func() bool { err := tu.ReadGetJSON(re, testDialClient, urlPrefix, &schedulers, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) return slice.Contains(schedulers, scheduler) }) } -func (suite *scheduleTestSuite) assertNoScheduler(urlPrefix string, scheduler string) { +func (suite *scheduleTestSuite) assertNoScheduler(re *require.Assertions, urlPrefix string, scheduler string) { var schedulers []string - re := suite.Require() tu.Eventually(re, func() bool { err := tu.ReadGetJSON(re, testDialClient, urlPrefix, &schedulers, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) return !slice.Contains(schedulers, scheduler) }) } -func (suite *scheduleTestSuite) isSchedulerPaused(urlPrefix, name string) bool { +func (suite *scheduleTestSuite) isSchedulerPaused(re *require.Assertions, urlPrefix, name string) bool { var schedulers []string - re := suite.Require() err := tu.ReadGetJSON(re, testDialClient, fmt.Sprintf("%s?status=paused", urlPrefix), &schedulers, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) for _, scheduler := range schedulers { if scheduler == name { return true diff --git a/tests/server/config/config_test.go b/tests/server/config/config_test.go index faa03c15329..98754127e55 100644 --- a/tests/server/config/config_test.go +++ b/tests/server/config/config_test.go @@ -113,41 +113,41 @@ func (suite *configTestSuite) checkConfigAll(cluster *tests.TestCluster) { cfg := &config.Config{} tu.Eventually(re, func() bool { err := tu.ReadGetJSON(re, testDialClient, addr, cfg) - suite.NoError(err) + re.NoError(err) return cfg.PDServerCfg.DashboardAddress != "auto" }) // the original way r := map[string]int{"max-replicas": 5} postData, err := json.Marshal(r) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, addr, postData, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) l := map[string]interface{}{ "location-labels": "zone,rack", "region-schedule-limit": 10, } postData, err = json.Marshal(l) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, addr, postData, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) l = map[string]interface{}{ "metric-storage": "http://127.0.0.1:9090", } postData, err = json.Marshal(l) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, addr, postData, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) newCfg := &config.Config{} err = tu.ReadGetJSON(re, testDialClient, addr, newCfg) - suite.NoError(err) + re.NoError(err) cfg.Replication.MaxReplicas = 5 cfg.Replication.LocationLabels = []string{"zone", "rack"} cfg.Schedule.RegionScheduleLimit = 10 cfg.PDServerCfg.MetricStorage = "http://127.0.0.1:9090" - suite.Equal(newCfg, cfg) + re.Equal(newCfg, cfg) // the new way l = map[string]interface{}{ @@ -161,12 +161,12 @@ func (suite *configTestSuite) checkConfigAll(cluster *tests.TestCluster) { "replication-mode.dr-auto-sync.label-key": "foobar", } postData, err = json.Marshal(l) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, addr, postData, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) newCfg1 := &config.Config{} err = tu.ReadGetJSON(re, testDialClient, addr, newCfg1) - suite.NoError(err) + re.NoError(err) cfg.Schedule.EnableTiKVSplitRegion = false cfg.Schedule.TolerantSizeRatio = 2.5 cfg.Replication.LocationLabels = []string{"idc", "host"} @@ -175,47 +175,47 @@ func (suite *configTestSuite) checkConfigAll(cluster *tests.TestCluster) { cfg.ReplicationMode.DRAutoSync.LabelKey = "foobar" cfg.ReplicationMode.ReplicationMode = "dr-auto-sync" v, err := versioninfo.ParseVersion("v4.0.0-beta") - suite.NoError(err) + re.NoError(err) cfg.ClusterVersion = *v - suite.Equal(cfg, newCfg1) + re.Equal(cfg, newCfg1) // revert this to avoid it affects TestConfigTTL l["schedule.enable-tikv-split-region"] = "true" postData, err = json.Marshal(l) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, addr, postData, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) // illegal prefix l = map[string]interface{}{ "replicate.max-replicas": 1, } postData, err = json.Marshal(l) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, addr, postData, tu.StatusNotOK(re), tu.StringContain(re, "not found")) - suite.NoError(err) + re.NoError(err) // update prefix directly l = map[string]interface{}{ "replication-mode": nil, } postData, err = json.Marshal(l) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, addr, postData, tu.StatusNotOK(re), tu.StringContain(re, "cannot update config prefix")) - suite.NoError(err) + re.NoError(err) // config item not found l = map[string]interface{}{ "schedule.region-limit": 10, } postData, err = json.Marshal(l) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, addr, postData, tu.StatusNotOK(re), tu.StringContain(re, "not found")) - suite.NoError(err) + re.NoError(err) } func (suite *configTestSuite) TestConfigSchedule() { @@ -230,16 +230,16 @@ func (suite *configTestSuite) checkConfigSchedule(cluster *tests.TestCluster) { addr := fmt.Sprintf("%s/pd/api/v1/config/schedule", urlPrefix) scheduleConfig := &sc.ScheduleConfig{} - suite.NoError(tu.ReadGetJSON(re, testDialClient, addr, scheduleConfig)) + re.NoError(tu.ReadGetJSON(re, testDialClient, addr, scheduleConfig)) scheduleConfig.MaxStoreDownTime.Duration = time.Second postData, err := json.Marshal(scheduleConfig) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, addr, postData, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) tu.Eventually(re, func() bool { scheduleConfig1 := &sc.ScheduleConfig{} - suite.NoError(tu.ReadGetJSON(re, testDialClient, addr, scheduleConfig1)) + re.NoError(tu.ReadGetJSON(re, testDialClient, addr, scheduleConfig1)) return reflect.DeepEqual(*scheduleConfig1, *scheduleConfig) }) } @@ -256,33 +256,33 @@ func (suite *configTestSuite) checkConfigReplication(cluster *tests.TestCluster) addr := fmt.Sprintf("%s/pd/api/v1/config/replicate", urlPrefix) rc := &sc.ReplicationConfig{} err := tu.ReadGetJSON(re, testDialClient, addr, rc) - suite.NoError(err) + re.NoError(err) rc.MaxReplicas = 5 rc1 := map[string]int{"max-replicas": 5} postData, err := json.Marshal(rc1) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, addr, postData, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) rc.LocationLabels = []string{"zone", "rack"} rc2 := map[string]string{"location-labels": "zone,rack"} postData, err = json.Marshal(rc2) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, addr, postData, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) rc.IsolationLevel = "zone" rc3 := map[string]string{"isolation-level": "zone"} postData, err = json.Marshal(rc3) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, addr, postData, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) rc4 := &sc.ReplicationConfig{} tu.Eventually(re, func() bool { err = tu.ReadGetJSON(re, testDialClient, addr, rc4) - suite.NoError(err) + re.NoError(err) return reflect.DeepEqual(*rc4, *rc) }) } @@ -300,12 +300,12 @@ func (suite *configTestSuite) checkConfigLabelProperty(cluster *tests.TestCluste loadProperties := func() config.LabelPropertyConfig { var cfg config.LabelPropertyConfig err := tu.ReadGetJSON(re, testDialClient, addr, &cfg) - suite.NoError(err) + re.NoError(err) return cfg } cfg := loadProperties() - suite.Empty(cfg) + re.Empty(cfg) cmds := []string{ `{"type": "foo", "action": "set", "label-key": "zone", "label-value": "cn1"}`, @@ -314,16 +314,16 @@ func (suite *configTestSuite) checkConfigLabelProperty(cluster *tests.TestCluste } for _, cmd := range cmds { err := tu.CheckPostJSON(testDialClient, addr, []byte(cmd), tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) } cfg = loadProperties() - suite.Len(cfg, 2) - suite.Equal([]config.StoreLabel{ + re.Len(cfg, 2) + re.Equal([]config.StoreLabel{ {Key: "zone", Value: "cn1"}, {Key: "zone", Value: "cn2"}, }, cfg["foo"]) - suite.Equal([]config.StoreLabel{{Key: "host", Value: "h1"}}, cfg["bar"]) + re.Equal([]config.StoreLabel{{Key: "host", Value: "h1"}}, cfg["bar"]) cmds = []string{ `{"type": "foo", "action": "delete", "label-key": "zone", "label-value": "cn1"}`, @@ -331,12 +331,12 @@ func (suite *configTestSuite) checkConfigLabelProperty(cluster *tests.TestCluste } for _, cmd := range cmds { err := tu.CheckPostJSON(testDialClient, addr, []byte(cmd), tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) } cfg = loadProperties() - suite.Len(cfg, 1) - suite.Equal([]config.StoreLabel{{Key: "zone", Value: "cn2"}}, cfg["foo"]) + re.Len(cfg, 1) + re.Equal([]config.StoreLabel{{Key: "zone", Value: "cn2"}}, cfg["foo"]) } func (suite *configTestSuite) TestConfigDefault() { @@ -352,35 +352,35 @@ func (suite *configTestSuite) checkConfigDefault(cluster *tests.TestCluster) { r := map[string]int{"max-replicas": 5} postData, err := json.Marshal(r) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, addr, postData, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) l := map[string]interface{}{ "location-labels": "zone,rack", "region-schedule-limit": 10, } postData, err = json.Marshal(l) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, addr, postData, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) l = map[string]interface{}{ "metric-storage": "http://127.0.0.1:9090", } postData, err = json.Marshal(l) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, addr, postData, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) addr = fmt.Sprintf("%s/pd/api/v1/config/default", urlPrefix) defaultCfg := &config.Config{} err = tu.ReadGetJSON(re, testDialClient, addr, defaultCfg) - suite.NoError(err) + re.NoError(err) - suite.Equal(uint64(3), defaultCfg.Replication.MaxReplicas) - suite.Equal(typeutil.StringSlice([]string{}), defaultCfg.Replication.LocationLabels) - suite.Equal(uint64(2048), defaultCfg.Schedule.RegionScheduleLimit) - suite.Equal("", defaultCfg.PDServerCfg.MetricStorage) + re.Equal(uint64(3), defaultCfg.Replication.MaxReplicas) + re.Equal(typeutil.StringSlice([]string{}), defaultCfg.Replication.LocationLabels) + re.Equal(uint64(2048), defaultCfg.Schedule.RegionScheduleLimit) + re.Equal("", defaultCfg.PDServerCfg.MetricStorage) } func (suite *configTestSuite) TestConfigPDServer() { @@ -397,21 +397,21 @@ func (suite *configTestSuite) checkConfigPDServer(cluster *tests.TestCluster) { "metric-storage": "", } postData, err := json.Marshal(ms) - suite.NoError(err) - suite.NoError(tu.CheckPostJSON(testDialClient, addrPost, postData, tu.StatusOK(re))) + re.NoError(err) + re.NoError(tu.CheckPostJSON(testDialClient, addrPost, postData, tu.StatusOK(re))) addrGet := fmt.Sprintf("%s/pd/api/v1/config/pd-server", urlPrefix) sc := &config.PDServerConfig{} - suite.NoError(tu.ReadGetJSON(re, testDialClient, addrGet, sc)) - suite.Equal(bool(true), sc.UseRegionStorage) - suite.Equal("table", sc.KeyType) - suite.Equal(typeutil.StringSlice([]string{}), sc.RuntimeServices) - suite.Equal("", sc.MetricStorage) + re.NoError(tu.ReadGetJSON(re, testDialClient, addrGet, sc)) + re.Equal(bool(true), sc.UseRegionStorage) + re.Equal("table", sc.KeyType) + re.Equal(typeutil.StringSlice([]string{}), sc.RuntimeServices) + re.Equal("", sc.MetricStorage) if sc.DashboardAddress != "auto" { // dashboard has been assigned re.Equal(leaderServer.GetAddr(), sc.DashboardAddress) } - suite.Equal(int(3), sc.FlowRoundByDigit) - suite.Equal(typeutil.NewDuration(time.Second), sc.MinResolvedTSPersistenceInterval) - suite.Equal(24*time.Hour, sc.MaxResetTSGap.Duration) + re.Equal(int(3), sc.FlowRoundByDigit) + re.Equal(typeutil.NewDuration(time.Second), sc.MinResolvedTSPersistenceInterval) + re.Equal(24*time.Hour, sc.MaxResetTSGap.Duration) } var ttlConfig = map[string]interface{}{ @@ -447,12 +447,13 @@ type ttlConfigInterface interface { } func (suite *configTestSuite) assertTTLConfig( + re *require.Assertions, cluster *tests.TestCluster, expectedEqual bool, ) { - equality := suite.Equal + equality := re.Equal if !expectedEqual { - equality = suite.NotEqual + equality = re.NotEqual } checkFunc := func(options ttlConfigInterface) { equality(uint64(999), options.GetMaxSnapshotCount()) @@ -471,7 +472,7 @@ func (suite *configTestSuite) assertTTLConfig( if cluster.GetSchedulingPrimaryServer() != nil { // wait for the scheduling primary server to be synced options := cluster.GetSchedulingPrimaryServer().GetPersistConfig() - tu.Eventually(suite.Require(), func() bool { + tu.Eventually(re, func() bool { if expectedEqual { return uint64(999) == options.GetMaxSnapshotCount() } @@ -482,6 +483,7 @@ func (suite *configTestSuite) assertTTLConfig( } func (suite *configTestSuite) assertTTLConfigItemEqual( + re *require.Assertions, cluster *tests.TestCluster, item string, expectedValue interface{}, @@ -497,10 +499,10 @@ func (suite *configTestSuite) assertTTLConfigItemEqual( } return false } - suite.True(checkFunc(cluster.GetLeaderServer().GetServer().GetPersistOptions())) + re.True(checkFunc(cluster.GetLeaderServer().GetServer().GetPersistOptions())) if cluster.GetSchedulingPrimaryServer() != nil { // wait for the scheduling primary server to be synced - tu.Eventually(suite.Require(), func() bool { + tu.Eventually(re, func() bool { return checkFunc(cluster.GetSchedulingPrimaryServer().GetPersistConfig()) }) } @@ -519,56 +521,56 @@ func (suite *configTestSuite) checkConfigTTL(cluster *tests.TestCluster) { leaderServer := cluster.GetLeaderServer() urlPrefix := leaderServer.GetAddr() postData, err := json.Marshal(ttlConfig) - suite.NoError(err) + re.NoError(err) // test no config and cleaning up err = tu.CheckPostJSON(testDialClient, createTTLUrl(urlPrefix, 0), postData, tu.StatusOK(re)) - suite.NoError(err) - suite.assertTTLConfig(cluster, false) + re.NoError(err) + suite.assertTTLConfig(re, cluster, false) // test time goes by err = tu.CheckPostJSON(testDialClient, createTTLUrl(urlPrefix, 5), postData, tu.StatusOK(re)) - suite.NoError(err) - suite.assertTTLConfig(cluster, true) + re.NoError(err) + suite.assertTTLConfig(re, cluster, true) time.Sleep(5 * time.Second) - suite.assertTTLConfig(cluster, false) + suite.assertTTLConfig(re, cluster, false) // test cleaning up err = tu.CheckPostJSON(testDialClient, createTTLUrl(urlPrefix, 5), postData, tu.StatusOK(re)) - suite.NoError(err) - suite.assertTTLConfig(cluster, true) + re.NoError(err) + suite.assertTTLConfig(re, cluster, true) err = tu.CheckPostJSON(testDialClient, createTTLUrl(urlPrefix, 0), postData, tu.StatusOK(re)) - suite.NoError(err) - suite.assertTTLConfig(cluster, false) + re.NoError(err) + suite.assertTTLConfig(re, cluster, false) postData, err = json.Marshal(invalidTTLConfig) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, createTTLUrl(urlPrefix, 1), postData, tu.StatusNotOK(re), tu.StringEqual(re, "\"unsupported ttl config schedule.invalid-ttl-config\"\n")) - suite.NoError(err) + re.NoError(err) // only set max-merge-region-size mergeConfig := map[string]interface{}{ "schedule.max-merge-region-size": 999, } postData, err = json.Marshal(mergeConfig) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, createTTLUrl(urlPrefix, 1), postData, tu.StatusOK(re)) - suite.NoError(err) - suite.assertTTLConfigItemEqual(cluster, "max-merge-region-size", uint64(999)) + re.NoError(err) + suite.assertTTLConfigItemEqual(re, cluster, "max-merge-region-size", uint64(999)) // max-merge-region-keys should keep consistence with max-merge-region-size. - suite.assertTTLConfigItemEqual(cluster, "max-merge-region-keys", uint64(999*10000)) + suite.assertTTLConfigItemEqual(re, cluster, "max-merge-region-keys", uint64(999*10000)) // on invalid value, we use default config mergeConfig = map[string]interface{}{ "schedule.enable-tikv-split-region": "invalid", } postData, err = json.Marshal(mergeConfig) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, createTTLUrl(urlPrefix, 10), postData, tu.StatusOK(re)) - suite.NoError(err) - suite.assertTTLConfigItemEqual(cluster, "enable-tikv-split-region", true) + re.NoError(err) + suite.assertTTLConfigItemEqual(re, cluster, "enable-tikv-split-region", true) } func (suite *configTestSuite) TestTTLConflict() { @@ -581,25 +583,25 @@ func (suite *configTestSuite) checkTTLConflict(cluster *tests.TestCluster) { urlPrefix := leaderServer.GetAddr() addr := createTTLUrl(urlPrefix, 1) postData, err := json.Marshal(ttlConfig) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, addr, postData, tu.StatusOK(re)) - suite.NoError(err) - suite.assertTTLConfig(cluster, true) + re.NoError(err) + suite.assertTTLConfig(re, cluster, true) cfg := map[string]interface{}{"max-snapshot-count": 30} postData, err = json.Marshal(cfg) - suite.NoError(err) + re.NoError(err) addr = fmt.Sprintf("%s/pd/api/v1/config", urlPrefix) err = tu.CheckPostJSON(testDialClient, addr, postData, tu.StatusNotOK(re), tu.StringEqual(re, "\"need to clean up TTL first for schedule.max-snapshot-count\"\n")) - suite.NoError(err) + re.NoError(err) addr = fmt.Sprintf("%s/pd/api/v1/config/schedule", urlPrefix) err = tu.CheckPostJSON(testDialClient, addr, postData, tu.StatusNotOK(re), tu.StringEqual(re, "\"need to clean up TTL first for schedule.max-snapshot-count\"\n")) - suite.NoError(err) + re.NoError(err) cfg = map[string]interface{}{"schedule.max-snapshot-count": 30} postData, err = json.Marshal(cfg) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, createTTLUrl(urlPrefix, 0), postData, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) err = tu.CheckPostJSON(testDialClient, addr, postData, tu.StatusOK(re)) - suite.NoError(err) + re.NoError(err) } diff --git a/tests/server/keyspace/keyspace_test.go b/tests/server/keyspace/keyspace_test.go index 3ee15e1edc1..aa2e89296bb 100644 --- a/tests/server/keyspace/keyspace_test.go +++ b/tests/server/keyspace/keyspace_test.go @@ -50,18 +50,19 @@ func TestKeyspaceTestSuite(t *testing.T) { } func (suite *keyspaceTestSuite) SetupTest() { + re := suite.Require() ctx, cancel := context.WithCancel(context.Background()) suite.cancel = cancel cluster, err := tests.NewTestCluster(ctx, 3, func(conf *config.Config, serverName string) { conf.Keyspace.PreAlloc = preAllocKeyspace }) suite.cluster = cluster - suite.NoError(err) - suite.NoError(cluster.RunInitialServers()) - suite.NotEmpty(cluster.WaitLeader()) + re.NoError(err) + re.NoError(cluster.RunInitialServers()) + re.NotEmpty(cluster.WaitLeader()) suite.server = cluster.GetLeaderServer() suite.manager = suite.server.GetKeyspaceManager() - suite.NoError(suite.server.BootstrapCluster()) + re.NoError(suite.server.BootstrapCluster()) } func (suite *keyspaceTestSuite) TearDownTest() { diff --git a/tests/server/tso/consistency_test.go b/tests/server/tso/consistency_test.go index c50b791b47f..977b18beca9 100644 --- a/tests/server/tso/consistency_test.go +++ b/tests/server/tso/consistency_test.go @@ -74,14 +74,14 @@ func (suite *tsoConsistencyTestSuite) TestSynchronizedGlobalTSO() { conf.Labels[config.ZoneLabel] = dcLocationConfig[serverName] }) defer cluster.Destroy() - suite.NoError(err) - suite.NoError(cluster.RunInitialServers()) + re.NoError(err) + re.NoError(cluster.RunInitialServers()) re := suite.Require() cluster.WaitAllLeaders(re, dcLocationConfig) suite.leaderServer = cluster.GetLeaderServer() - suite.NotNil(suite.leaderServer) + re.NotNil(suite.leaderServer) suite.dcClientMap[tso.GlobalDCLocation] = testutil.MustNewGrpcClient(re, suite.leaderServer.GetAddr()) for _, dcLocation := range dcLocationConfig { pdName := suite.leaderServer.GetAllocatorLeader(dcLocation).GetName() @@ -97,12 +97,12 @@ func (suite *tsoConsistencyTestSuite) TestSynchronizedGlobalTSO() { for _, dcLocation := range dcLocationConfig { localTSO := suite.getTimestampByDC(ctx, cluster, dcLocation) oldLocalTSOs = append(oldLocalTSOs, localTSO) - suite.Equal(-1, tsoutil.CompareTimestamp(maxGlobalTSO, localTSO)) + re.Equal(-1, tsoutil.CompareTimestamp(maxGlobalTSO, localTSO)) } // Get a global TSO then globalTSO := suite.getTimestampByDC(ctx, cluster, tso.GlobalDCLocation) for _, oldLocalTSO := range oldLocalTSOs { - suite.Equal(1, tsoutil.CompareTimestamp(globalTSO, oldLocalTSO)) + re.Equal(1, tsoutil.CompareTimestamp(globalTSO, oldLocalTSO)) } if tsoutil.CompareTimestamp(maxGlobalTSO, globalTSO) < 0 { maxGlobalTSO = globalTSO @@ -113,7 +113,7 @@ func (suite *tsoConsistencyTestSuite) TestSynchronizedGlobalTSO() { newLocalTSOs = append(newLocalTSOs, suite.getTimestampByDC(ctx, cluster, dcLocation)) } for _, newLocalTSO := range newLocalTSOs { - suite.Equal(-1, tsoutil.CompareTimestamp(maxGlobalTSO, newLocalTSO)) + re.Equal(-1, tsoutil.CompareTimestamp(maxGlobalTSO, newLocalTSO)) } } } @@ -125,16 +125,16 @@ func (suite *tsoConsistencyTestSuite) getTimestampByDC(ctx context.Context, clus DcLocation: dcLocation, } pdClient, ok := suite.dcClientMap[dcLocation] - suite.True(ok) + re.True(ok) forwardedHost := cluster.GetServer(suite.leaderServer.GetAllocatorLeader(dcLocation).GetName()).GetAddr() ctx = grpcutil.BuildForwardContext(ctx, forwardedHost) tsoClient, err := pdClient.Tso(ctx) - suite.NoError(err) + re.NoError(err) defer tsoClient.CloseSend() - suite.NoError(tsoClient.Send(req)) + re.NoError(tsoClient.Send(req)) resp, err := tsoClient.Recv() - suite.NoError(err) - return checkAndReturnTimestampResponse(suite.Require(), req, resp) + re.NoError(err) + return checkAndReturnTimestampResponse(re, req, resp) } func (suite *tsoConsistencyTestSuite) TestSynchronizedGlobalTSOOverflow() { @@ -149,14 +149,14 @@ func (suite *tsoConsistencyTestSuite) TestSynchronizedGlobalTSOOverflow() { conf.Labels[config.ZoneLabel] = dcLocationConfig[serverName] }) defer cluster.Destroy() - suite.NoError(err) - suite.NoError(cluster.RunInitialServers()) + re.NoError(err) + re.NoError(cluster.RunInitialServers()) re := suite.Require() cluster.WaitAllLeaders(re, dcLocationConfig) suite.leaderServer = cluster.GetLeaderServer() - suite.NotNil(suite.leaderServer) + re.NotNil(suite.leaderServer) suite.dcClientMap[tso.GlobalDCLocation] = testutil.MustNewGrpcClient(re, suite.leaderServer.GetAddr()) for _, dcLocation := range dcLocationConfig { pdName := suite.leaderServer.GetAllocatorLeader(dcLocation).GetName() @@ -165,13 +165,13 @@ func (suite *tsoConsistencyTestSuite) TestSynchronizedGlobalTSOOverflow() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - suite.NoError(failpoint.Enable("github.com/tikv/pd/pkg/tso/globalTSOOverflow", `return(true)`)) + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/tso/globalTSOOverflow", `return(true)`)) suite.getTimestampByDC(ctx, cluster, tso.GlobalDCLocation) - suite.NoError(failpoint.Disable("github.com/tikv/pd/pkg/tso/globalTSOOverflow")) + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/tso/globalTSOOverflow")) } func (suite *tsoConsistencyTestSuite) TestLocalAllocatorLeaderChange() { - suite.NoError(failpoint.Enable("github.com/tikv/pd/server/mockLocalAllocatorLeaderChange", `return(true)`)) + re.NoError(failpoint.Enable("github.com/tikv/pd/server/mockLocalAllocatorLeaderChange", `return(true)`)) dcLocationConfig := map[string]string{ "pd1": "dc-1", } @@ -181,14 +181,14 @@ func (suite *tsoConsistencyTestSuite) TestLocalAllocatorLeaderChange() { conf.Labels[config.ZoneLabel] = dcLocationConfig[serverName] }) defer cluster.Destroy() - suite.NoError(err) - suite.NoError(cluster.RunInitialServers()) + re.NoError(err) + re.NoError(cluster.RunInitialServers()) re := suite.Require() cluster.WaitAllLeaders(re, dcLocationConfig) suite.leaderServer = cluster.GetLeaderServer() - suite.NotNil(suite.leaderServer) + re.NotNil(suite.leaderServer) suite.dcClientMap[tso.GlobalDCLocation] = testutil.MustNewGrpcClient(re, suite.leaderServer.GetAddr()) for _, dcLocation := range dcLocationConfig { pdName := suite.leaderServer.GetAllocatorLeader(dcLocation).GetName() @@ -198,7 +198,7 @@ func (suite *tsoConsistencyTestSuite) TestLocalAllocatorLeaderChange() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() suite.getTimestampByDC(ctx, cluster, tso.GlobalDCLocation) - suite.NoError(failpoint.Disable("github.com/tikv/pd/server/mockLocalAllocatorLeaderChange")) + re.NoError(failpoint.Disable("github.com/tikv/pd/server/mockLocalAllocatorLeaderChange")) } func (suite *tsoConsistencyTestSuite) TestLocalTSO() { @@ -213,10 +213,10 @@ func (suite *tsoConsistencyTestSuite) TestLocalTSO() { conf.Labels[config.ZoneLabel] = dcLocationConfig[serverName] }) defer cluster.Destroy() - suite.NoError(err) - suite.NoError(cluster.RunInitialServers()) + re.NoError(err) + re.NoError(cluster.RunInitialServers()) - cluster.WaitAllLeaders(suite.Require(), dcLocationConfig) + cluster.WaitAllLeaders(re, dcLocationConfig) suite.testTSO(cluster, dcLocationConfig, nil) } @@ -243,8 +243,8 @@ func (suite *tsoConsistencyTestSuite) TestLocalTSOAfterMemberChanged() { conf.Labels[config.ZoneLabel] = dcLocationConfig[serverName] }) defer cluster.Destroy() - suite.NoError(err) - suite.NoError(cluster.RunInitialServers()) + re.NoError(err) + re.NoError(cluster.RunInitialServers()) re := suite.Require() cluster.WaitAllLeaders(re, dcLocationConfig) @@ -265,15 +265,15 @@ func (suite *tsoConsistencyTestSuite) TestLocalTSOAfterMemberChanged() { time.Sleep(time.Second * 5) // Mock the situation that the system time of PD nodes in dc-4 is slower than others. - suite.NoError(failpoint.Enable("github.com/tikv/pd/pkg/tso/systemTimeSlow", `return(true)`)) + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/tso/systemTimeSlow", `return(true)`)) // Join a new dc-location pd4, err := cluster.Join(suite.ctx, func(conf *config.Config, serverName string) { conf.EnableLocalTSO = true conf.Labels[config.ZoneLabel] = "dc-4" }) - suite.NoError(err) - suite.NoError(pd4.Run()) + re.NoError(err) + re.NoError(pd4.Run()) dcLocationConfig["pd4"] = "dc-4" cluster.CheckClusterDCLocation() re.NotEqual("", cluster.WaitAllocatorLeader( @@ -282,7 +282,7 @@ func (suite *tsoConsistencyTestSuite) TestLocalTSOAfterMemberChanged() { )) suite.testTSO(cluster, dcLocationConfig, previousTS) - suite.NoError(failpoint.Disable("github.com/tikv/pd/pkg/tso/systemTimeSlow")) + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/tso/systemTimeSlow")) } func (suite *tsoConsistencyTestSuite) testTSO(cluster *tests.TestCluster, dcLocationConfig map[string]string, previousTS *pdpb.Timestamp) { @@ -319,15 +319,15 @@ func (suite *tsoConsistencyTestSuite) testTSO(cluster *tests.TestCluster, dcLoca cancel() lastTS := lastList[dcLocation] // Check whether the TSO fallbacks - suite.Equal(1, tsoutil.CompareTimestamp(ts, lastTS)) + re.Equal(1, tsoutil.CompareTimestamp(ts, lastTS)) if previousTS != nil { // Because we have a Global TSO synchronization, even though the system time // of the PD nodes in dc-4 is slower, its TSO will still be big enough. - suite.Equal(1, tsoutil.CompareTimestamp(ts, previousTS)) + re.Equal(1, tsoutil.CompareTimestamp(ts, previousTS)) } lastList[dcLocation] = ts // Check whether the TSO is not unique - suite.True(suite.checkTSOUnique(ts)) + re.True(suite.checkTSOUnique(ts)) } time.Sleep(10 * time.Millisecond) } From d4ebef8392bc3e4f74b7fb4028ed445cd556caed Mon Sep 17 00:00:00 2001 From: JmPotato Date: Thu, 21 Dec 2023 14:07:53 +0800 Subject: [PATCH 116/137] tests, Makefile: fix the TSO consistency test (#7603) ref tikv/pd#4813 Fix the TSO consistency test. Signed-off-by: JmPotato --- Makefile | 8 +++++++- tests/server/tso/consistency_test.go | 27 +++++++++++++++++---------- 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index b94de374a81..42c7c7660c9 100644 --- a/Makefile +++ b/Makefile @@ -249,6 +249,12 @@ ci-test-job: install-tools dashboard-ui TSO_INTEGRATION_TEST_PKGS := $(PD_PKG)/tests/server/tso +test-tso: install-tools + # testing TSO function & consistency... + @$(FAILPOINT_ENABLE) + CGO_ENABLED=1 go test -race -tags without_dashboard,tso_full_test,deadlock $(TSO_INTEGRATION_TEST_PKGS) || { $(FAILPOINT_DISABLE); exit 1; } + @$(FAILPOINT_DISABLE) + test-tso-function: install-tools # testing TSO function... @$(FAILPOINT_ENABLE) @@ -267,7 +273,7 @@ test-real-cluster: # testing with the real cluster... cd $(REAL_CLUSTER_TEST_PATH) && $(MAKE) check -.PHONY: test basic-test test-with-cover test-tso-function test-tso-consistency test-real-cluster +.PHONY: test basic-test test-with-cover test-tso test-tso-function test-tso-consistency test-real-cluster #### Daily CI coverage analyze #### diff --git a/tests/server/tso/consistency_test.go b/tests/server/tso/consistency_test.go index 977b18beca9..d1c45df7f17 100644 --- a/tests/server/tso/consistency_test.go +++ b/tests/server/tso/consistency_test.go @@ -25,6 +25,7 @@ import ( "github.com/pingcap/failpoint" "github.com/pingcap/kvproto/pkg/pdpb" + "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "github.com/tikv/pd/pkg/tso" "github.com/tikv/pd/pkg/utils/grpcutil" @@ -63,6 +64,7 @@ func (suite *tsoConsistencyTestSuite) TearDownSuite() { // TestSynchronizedGlobalTSO is used to test the synchronized way of global TSO generation. func (suite *tsoConsistencyTestSuite) TestSynchronizedGlobalTSO() { + re := suite.Require() dcLocationConfig := map[string]string{ "pd1": "dc-1", "pd2": "dc-2", @@ -77,7 +79,6 @@ func (suite *tsoConsistencyTestSuite) TestSynchronizedGlobalTSO() { re.NoError(err) re.NoError(cluster.RunInitialServers()) - re := suite.Require() cluster.WaitAllLeaders(re, dcLocationConfig) suite.leaderServer = cluster.GetLeaderServer() @@ -95,12 +96,12 @@ func (suite *tsoConsistencyTestSuite) TestSynchronizedGlobalTSO() { // Get some local TSOs first oldLocalTSOs := make([]*pdpb.Timestamp, 0, dcLocationNum) for _, dcLocation := range dcLocationConfig { - localTSO := suite.getTimestampByDC(ctx, cluster, dcLocation) + localTSO := suite.getTimestampByDC(ctx, re, cluster, dcLocation) oldLocalTSOs = append(oldLocalTSOs, localTSO) re.Equal(-1, tsoutil.CompareTimestamp(maxGlobalTSO, localTSO)) } // Get a global TSO then - globalTSO := suite.getTimestampByDC(ctx, cluster, tso.GlobalDCLocation) + globalTSO := suite.getTimestampByDC(ctx, re, cluster, tso.GlobalDCLocation) for _, oldLocalTSO := range oldLocalTSOs { re.Equal(1, tsoutil.CompareTimestamp(globalTSO, oldLocalTSO)) } @@ -110,7 +111,7 @@ func (suite *tsoConsistencyTestSuite) TestSynchronizedGlobalTSO() { // Get some local TSOs again newLocalTSOs := make([]*pdpb.Timestamp, 0, dcLocationNum) for _, dcLocation := range dcLocationConfig { - newLocalTSOs = append(newLocalTSOs, suite.getTimestampByDC(ctx, cluster, dcLocation)) + newLocalTSOs = append(newLocalTSOs, suite.getTimestampByDC(ctx, re, cluster, dcLocation)) } for _, newLocalTSO := range newLocalTSOs { re.Equal(-1, tsoutil.CompareTimestamp(maxGlobalTSO, newLocalTSO)) @@ -118,7 +119,12 @@ func (suite *tsoConsistencyTestSuite) TestSynchronizedGlobalTSO() { } } -func (suite *tsoConsistencyTestSuite) getTimestampByDC(ctx context.Context, cluster *tests.TestCluster, dcLocation string) *pdpb.Timestamp { +func (suite *tsoConsistencyTestSuite) getTimestampByDC( + ctx context.Context, + re *require.Assertions, + cluster *tests.TestCluster, + dcLocation string, +) *pdpb.Timestamp { req := &pdpb.TsoRequest{ Header: testutil.NewRequestHeader(suite.leaderServer.GetClusterID()), Count: tsoCount, @@ -138,6 +144,7 @@ func (suite *tsoConsistencyTestSuite) getTimestampByDC(ctx context.Context, clus } func (suite *tsoConsistencyTestSuite) TestSynchronizedGlobalTSOOverflow() { + re := suite.Require() dcLocationConfig := map[string]string{ "pd1": "dc-1", "pd2": "dc-2", @@ -152,7 +159,6 @@ func (suite *tsoConsistencyTestSuite) TestSynchronizedGlobalTSOOverflow() { re.NoError(err) re.NoError(cluster.RunInitialServers()) - re := suite.Require() cluster.WaitAllLeaders(re, dcLocationConfig) suite.leaderServer = cluster.GetLeaderServer() @@ -166,11 +172,12 @@ func (suite *tsoConsistencyTestSuite) TestSynchronizedGlobalTSOOverflow() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/tso/globalTSOOverflow", `return(true)`)) - suite.getTimestampByDC(ctx, cluster, tso.GlobalDCLocation) + suite.getTimestampByDC(ctx, re, cluster, tso.GlobalDCLocation) re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/tso/globalTSOOverflow")) } func (suite *tsoConsistencyTestSuite) TestLocalAllocatorLeaderChange() { + re := suite.Require() re.NoError(failpoint.Enable("github.com/tikv/pd/server/mockLocalAllocatorLeaderChange", `return(true)`)) dcLocationConfig := map[string]string{ "pd1": "dc-1", @@ -184,7 +191,6 @@ func (suite *tsoConsistencyTestSuite) TestLocalAllocatorLeaderChange() { re.NoError(err) re.NoError(cluster.RunInitialServers()) - re := suite.Require() cluster.WaitAllLeaders(re, dcLocationConfig) suite.leaderServer = cluster.GetLeaderServer() @@ -197,11 +203,12 @@ func (suite *tsoConsistencyTestSuite) TestLocalAllocatorLeaderChange() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - suite.getTimestampByDC(ctx, cluster, tso.GlobalDCLocation) + suite.getTimestampByDC(ctx, re, cluster, tso.GlobalDCLocation) re.NoError(failpoint.Disable("github.com/tikv/pd/server/mockLocalAllocatorLeaderChange")) } func (suite *tsoConsistencyTestSuite) TestLocalTSO() { + re := suite.Require() dcLocationConfig := map[string]string{ "pd1": "dc-1", "pd2": "dc-2", @@ -232,6 +239,7 @@ func (suite *tsoConsistencyTestSuite) checkTSOUnique(tso *pdpb.Timestamp) bool { } func (suite *tsoConsistencyTestSuite) TestLocalTSOAfterMemberChanged() { + re := suite.Require() dcLocationConfig := map[string]string{ "pd1": "dc-1", "pd2": "dc-2", @@ -246,7 +254,6 @@ func (suite *tsoConsistencyTestSuite) TestLocalTSOAfterMemberChanged() { re.NoError(err) re.NoError(cluster.RunInitialServers()) - re := suite.Require() cluster.WaitAllLeaders(re, dcLocationConfig) leaderServer := cluster.GetLeaderServer() From f15db4ba2b705c05a8a85d443a6109c6320447c5 Mon Sep 17 00:00:00 2001 From: lance6716 Date: Thu, 21 Dec 2023 15:04:54 +0800 Subject: [PATCH 117/137] client/http: add more interface for BR/lightning (#7602) ref tikv/pd#7300 Signed-off-by: lance6716 Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- client/http/interface.go | 30 +++++++++++++++++++ client/http/request_info.go | 2 ++ tests/integrations/client/http_client_test.go | 14 +++++++++ 3 files changed, 46 insertions(+) diff --git a/client/http/interface.go b/client/http/interface.go index 25fa99fa0bd..46db7cb6fd8 100644 --- a/client/http/interface.go +++ b/client/http/interface.go @@ -49,6 +49,7 @@ type Client interface { GetScheduleConfig(context.Context) (map[string]interface{}, error) SetScheduleConfig(context.Context, map[string]interface{}) error GetClusterVersion(context.Context) (string, error) + GetReplicateConfig(context.Context) (map[string]interface{}, error) /* Scheduler-related interfaces */ GetSchedulers(context.Context) ([]string, error) CreateScheduler(ctx context.Context, name string, storeID uint64) error @@ -57,6 +58,7 @@ type Client interface { GetAllPlacementRuleBundles(context.Context) ([]*GroupBundle, error) GetPlacementRuleBundleByGroup(context.Context, string) (*GroupBundle, error) GetPlacementRulesByGroup(context.Context, string) ([]*Rule, error) + GetPlacementRule(context.Context, string, string) (*Rule, error) SetPlacementRule(context.Context, *Rule) error SetPlacementRuleInBatch(context.Context, []*RuleOp) error SetPlacementRuleBundles(context.Context, []*GroupBundle, bool) error @@ -359,6 +361,20 @@ func (c *client) GetClusterVersion(ctx context.Context) (string, error) { return version, nil } +// GetReplicateConfig gets the replication configurations. +func (c *client) GetReplicateConfig(ctx context.Context) (map[string]interface{}, error) { + var config map[string]interface{} + err := c.request(ctx, newRequestInfo(). + WithName(getReplicateConfigName). + WithURI(ReplicateConfig). + WithMethod(http.MethodGet). + WithResp(&config)) + if err != nil { + return nil, err + } + return config, nil +} + // GetAllPlacementRuleBundles gets all placement rules bundles. func (c *client) GetAllPlacementRuleBundles(ctx context.Context) ([]*GroupBundle, error) { var bundles []*GroupBundle @@ -401,6 +417,20 @@ func (c *client) GetPlacementRulesByGroup(ctx context.Context, group string) ([] return rules, nil } +// GetPlacementRule gets the placement rule by group and ID. +func (c *client) GetPlacementRule(ctx context.Context, group, id string) (*Rule, error) { + var rule Rule + err := c.request(ctx, newRequestInfo(). + WithName(getPlacementRuleName). + WithURI(PlacementRuleByGroupAndID(group, id)). + WithMethod(http.MethodGet). + WithResp(&rule)) + if err != nil { + return nil, err + } + return &rule, nil +} + // SetPlacementRule sets the placement rule. func (c *client) SetPlacementRule(ctx context.Context, rule *Rule) error { ruleJSON, err := json.Marshal(rule) diff --git a/client/http/request_info.go b/client/http/request_info.go index dc811618d9c..1d64197b051 100644 --- a/client/http/request_info.go +++ b/client/http/request_info.go @@ -37,12 +37,14 @@ const ( getScheduleConfigName = "GetScheduleConfig" setScheduleConfigName = "SetScheduleConfig" getClusterVersionName = "GetClusterVersion" + getReplicateConfigName = "GetReplicateConfig" getSchedulersName = "GetSchedulers" createSchedulerName = "CreateScheduler" setSchedulerDelayName = "SetSchedulerDelay" getAllPlacementRuleBundlesName = "GetAllPlacementRuleBundles" getPlacementRuleBundleByGroupName = "GetPlacementRuleBundleByGroup" getPlacementRulesByGroupName = "GetPlacementRulesByGroup" + getPlacementRuleName = "GetPlacementRule" setPlacementRuleName = "SetPlacementRule" setPlacementRuleInBatchName = "SetPlacementRuleInBatch" setPlacementRuleBundlesName = "SetPlacementRuleBundles" diff --git a/tests/integrations/client/http_client_test.go b/tests/integrations/client/http_client_test.go index 1ab8b0a6d48..0d527198935 100644 --- a/tests/integrations/client/http_client_test.go +++ b/tests/integrations/client/http_client_test.go @@ -83,6 +83,9 @@ func (suite *httpClientTestSuite) TearDownSuite() { func (suite *httpClientTestSuite) TestMeta() { re := suite.Require() + replicateConfig, err := suite.client.GetReplicateConfig(suite.ctx) + re.NoError(err) + re.Equal(3.0, replicateConfig["max-replicas"]) region, err := suite.client.GetRegionByID(suite.ctx, 10) re.NoError(err) re.Equal(int64(10), region.ID) @@ -270,6 +273,17 @@ func (suite *httpClientTestSuite) checkRule( re *require.Assertions, rule *pd.Rule, totalRuleCount int, exist bool, ) { + if exist { + got, err := suite.client.GetPlacementRule(suite.ctx, rule.GroupID, rule.ID) + re.NoError(err) + // skip comparison of the generated field + got.StartKeyHex = rule.StartKeyHex + got.EndKeyHex = rule.EndKeyHex + re.Equal(rule, got) + } else { + _, err := suite.client.GetPlacementRule(suite.ctx, rule.GroupID, rule.ID) + re.ErrorContains(err, http.StatusText(http.StatusNotFound)) + } // Check through the `GetPlacementRulesByGroup` API. rules, err := suite.client.GetPlacementRulesByGroup(suite.ctx, rule.GroupID) re.NoError(err) From f1870c1f63b0378559bdd7f8c28bfbd3fea19ba8 Mon Sep 17 00:00:00 2001 From: Hu# Date: Thu, 21 Dec 2023 16:04:54 +0800 Subject: [PATCH 118/137] tests/realcluster: support local real cluster test (#7578) ref tikv/pd#7298 tests/realcluster: support local real cluster test Signed-off-by: husharp Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- tests/integrations/realcluster/Makefile | 16 ++++++++-------- tests/integrations/realcluster/deploy.sh | 15 +++++++++++---- tests/integrations/realcluster/reboot_pd_test.go | 5 ++++- 3 files changed, 23 insertions(+), 13 deletions(-) diff --git a/tests/integrations/realcluster/Makefile b/tests/integrations/realcluster/Makefile index c9ffd3d6599..e03007a4c31 100644 --- a/tests/integrations/realcluster/Makefile +++ b/tests/integrations/realcluster/Makefile @@ -30,24 +30,24 @@ tidy: git diff go.mod go.sum | cat git diff --quiet go.mod go.sum -check: deploy test kill_tiup +check: deploy test kill_cluster -deploy: kill_tiup +deploy: kill_cluster @ echo "deploying..." ./deploy.sh - @ echo "wait tiup cluster ready..." + @ echo "wait cluster ready..." ./wait_tiup.sh 15 20 @ echo "check cluster status..." - @ pid=$$(ps -ef | grep 'tiup' | grep -v grep | awk '{print $$2}' | head -n 1); \ + @ pid=$$(ps -ef | grep 'playground' | grep -v grep | awk '{print $$2}' | head -n 1); \ echo $$pid; -kill_tiup: - @ echo "kill tiup..." - @ pid=$$(ps -ef | grep 'tiup' | grep -v grep | awk '{print $$2}' | head -n 1); \ +kill_cluster: + @ echo "kill cluster..." + @ pid=$$(ps -ef | grep 'playground' | grep -v grep | awk '{print $$2}' | head -n 1); \ if [ ! -z "$$pid" ]; then \ echo $$pid; \ kill $$pid; \ - echo "waiting for tiup to exit..."; \ + echo "waiting for cluster to exit..."; \ sleep 10; \ fi diff --git a/tests/integrations/realcluster/deploy.sh b/tests/integrations/realcluster/deploy.sh index 18e6de7f0b9..d6cd0b27f72 100755 --- a/tests/integrations/realcluster/deploy.sh +++ b/tests/integrations/realcluster/deploy.sh @@ -15,9 +15,16 @@ curl --proto '=https' --tlsv1.2 -sSf https://tiup-mirrors.pingcap.com/install.sh $TIUP_BIN_DIR update playground cd ../../.. -# Run TiUP -$TIUP_BIN_DIR playground nightly --kv 3 --tiflash 1 --db 1 --pd 3 --without-monitor \ - --pd.binpath ./bin/pd-server --kv.binpath ./bin/tikv-server --db.binpath ./bin/tidb-server --tiflash.binpath ./bin/tiflash --tag pd_test \ - > $CUR_PATH/playground.log 2>&1 & +if [ ! -d "bin" ] || [ ! -e "bin/tikv-server" ] && [ ! -e "bin/tidb-server" ] && [ ! -e "bin/pd-server" ] && [ ! -e "bin/tiflash" ]; then + color-green "downloading binaries..." + color-green "this may take a few minutes, you can also download them manually and put them in the bin directory." + $TIUP_BIN_DIR playground nightly --kv 3 --tiflash 1 --db 1 --pd 3 --without-monitor --tag pd_test \ + > $CUR_PATH/playground.log 2>&1 & +else + color-green "using existing binaries..." + $TIUP_BIN_DIR playground nightly --kv 3 --tiflash 1 --db 1 --pd 3 --without-monitor \ + --pd.binpath ./bin/pd-server --kv.binpath ./bin/tikv-server --db.binpath ./bin/tidb-server --tiflash.binpath ./bin/tiflash --tag pd_test \ + > $CUR_PATH/playground.log 2>&1 & +fi cd $CUR_PATH diff --git a/tests/integrations/realcluster/reboot_pd_test.go b/tests/integrations/realcluster/reboot_pd_test.go index 59747912897..8e99b0822f0 100644 --- a/tests/integrations/realcluster/reboot_pd_test.go +++ b/tests/integrations/realcluster/reboot_pd_test.go @@ -40,10 +40,13 @@ func TestReloadLabel(t *testing.T) { resp, _ := pdHTTPCli.GetStores(ctx) setStore := resp.Stores[0] - re.Empty(setStore.Store.Labels, nil) + // TiFlash labels will be ["engine": "tiflash"] storeLabel := map[string]string{ "zone": "zone1", } + for _, label := range setStore.Store.Labels { + storeLabel[label.Key] = label.Value + } err := pdHTTPCli.SetStoreLabels(ctx, setStore.Store.ID, storeLabel) re.NoError(err) From 74ef91d1c8a28c1c7d45867bbcd34dcc9ae00cb5 Mon Sep 17 00:00:00 2001 From: Yongbo Jiang Date: Thu, 21 Dec 2023 17:52:55 +0800 Subject: [PATCH 119/137] client: Introduce `ServiceClient` (#7489) ref tikv/pd#7431, ref tikv/pd#7576 Signed-off-by: Cabinfever_B Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- client/client.go | 4 - client/go.mod | 12 +- client/go.sum | 24 +- client/grpcutil/grpcutil.go | 35 +++ client/pd_service_discovery.go | 237 +++++++++++++- client/pd_service_discovery_test.go | 295 ++++++++++++++++++ client/testutil/check_env_dummy.go | 21 ++ client/testutil/check_env_linux.go | 42 +++ client/testutil/leak.go | 4 + client/testutil/tempurl.go | 65 ++++ .../{check_env.go => check_env_dummy.go} | 0 tests/integrations/client/go.mod | 10 +- tests/integrations/client/go.sum | 20 +- tests/integrations/mcs/go.mod | 10 +- tests/integrations/mcs/go.sum | 20 +- tests/integrations/realcluster/go.mod | 8 +- tests/integrations/realcluster/go.sum | 16 +- tests/integrations/tso/go.mod | 10 +- tests/integrations/tso/go.sum | 20 +- tools/pd-api-bench/go.mod | 10 +- tools/pd-api-bench/go.sum | 20 +- tools/pd-tso-bench/go.mod | 8 +- tools/pd-tso-bench/go.sum | 20 +- 23 files changed, 806 insertions(+), 105 deletions(-) create mode 100644 client/pd_service_discovery_test.go create mode 100644 client/testutil/check_env_dummy.go create mode 100644 client/testutil/check_env_linux.go create mode 100644 client/testutil/tempurl.go rename pkg/utils/tempurl/{check_env.go => check_env_dummy.go} (100%) diff --git a/client/client.go b/client/client.go index 7053ed2be96..0ae362d06a8 100644 --- a/client/client.go +++ b/client/client.go @@ -930,10 +930,6 @@ func handleRegionResponse(res *pdpb.GetRegionResponse) *Region { return r } -func isNetworkError(code codes.Code) bool { - return code == codes.Unavailable || code == codes.DeadlineExceeded -} - func (c *client) GetRegionFromMember(ctx context.Context, key []byte, memberURLs []string, opts ...GetRegionOption) (*Region, error) { if span := opentracing.SpanFromContext(ctx); span != nil { span = opentracing.StartSpan("pdclient.GetRegionFromMember", opentracing.ChildOf(span.Context())) diff --git a/client/go.mod b/client/go.mod index 54be0c96765..a97a94a3e95 100644 --- a/client/go.mod +++ b/client/go.mod @@ -4,6 +4,7 @@ go 1.21 require ( github.com/BurntSushi/toml v0.3.1 + github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 github.com/cloudfoundry/gosigar v1.3.6 github.com/gogo/protobuf v1.3.2 github.com/opentracing/opentracing-go v1.2.0 @@ -33,11 +34,12 @@ require ( github.com/prometheus/common v0.26.0 // indirect github.com/prometheus/procfs v0.6.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/net v0.17.0 // indirect - golang.org/x/sys v0.13.0 // indirect - golang.org/x/text v0.13.0 // indirect - google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect - google.golang.org/protobuf v1.30.0 // indirect + golang.org/x/net v0.18.0 // indirect + golang.org/x/sys v0.14.0 // indirect + golang.org/x/text v0.14.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17 // indirect + google.golang.org/grpc/examples v0.0.0-20230419000256-16651f60ddc5 + google.golang.org/protobuf v1.31.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/client/go.sum b/client/go.sum index 921c4671581..4d4e9f396da 100644 --- a/client/go.sum +++ b/client/go.sum @@ -13,6 +13,8 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 h1:BjkPE3785EwPhhyuFkbINB+2a1xATwk8SNDWnJiD41g= +github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5/go.mod h1:jtAfVaU/2cu1+wdSRPWE2c1N2qeAA3K4RH9pYgqwets= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -176,8 +178,8 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= +golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -201,14 +203,14 @@ golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= +golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -225,10 +227,12 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= -google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17 h1:Jyp0Hsi0bmHXG6k9eATXoYtjd6e2UzZ1SCn/wIupY14= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17/go.mod h1:oQ5rr10WTTMvP4A36n8JpR1OrO1BEiV4f78CneXZxkA= google.golang.org/grpc v1.54.0 h1:EhTqbhiYeixwWQtAEZAxmV9MGqcjEU2mFx52xCzNyag= google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g= +google.golang.org/grpc/examples v0.0.0-20230419000256-16651f60ddc5 h1:FjbWL/mGfyRQNxjagfT1chiHL1569WEA/OGH0ZIzGcI= +google.golang.org/grpc/examples v0.0.0-20230419000256-16651f60ddc5/go.mod h1:5av8LiY5jU2KRcrX+SHtvLHnaOpPJ7gzWStBurgHlqY= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -237,8 +241,8 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/client/grpcutil/grpcutil.go b/client/grpcutil/grpcutil.go index fe149e76ecc..d37d937a09f 100644 --- a/client/grpcutil/grpcutil.go +++ b/client/grpcutil/grpcutil.go @@ -36,6 +36,8 @@ const ( dialTimeout = 3 * time.Second // ForwardMetadataKey is used to record the forwarded host of PD. ForwardMetadataKey = "pd-forwarded-host" + // FollowerHandleMetadataKey is used to mark the permit of follower handle. + FollowerHandleMetadataKey = "pd-allow-follower-handle" ) // GetClientConn returns a gRPC client connection. @@ -75,6 +77,39 @@ func BuildForwardContext(ctx context.Context, addr string) context.Context { return metadata.NewOutgoingContext(ctx, md) } +// GetForwardedHost returns the forwarded host in metadata. +// Only used for test. +func GetForwardedHost(ctx context.Context, f func(context.Context) (metadata.MD, bool)) string { + v, _ := getValueFromMetadata(ctx, ForwardMetadataKey, f) + return v +} + +// BuildFollowerHandleContext creates a context with follower handle metadata information. +// It is used in client side. +func BuildFollowerHandleContext(ctx context.Context) context.Context { + md := metadata.Pairs(FollowerHandleMetadataKey, "") + return metadata.NewOutgoingContext(ctx, md) +} + +// IsFollowerHandleEnabled returns the forwarded host in metadata. +// Only used for test. +func IsFollowerHandleEnabled(ctx context.Context, f func(context.Context) (metadata.MD, bool)) bool { + _, ok := getValueFromMetadata(ctx, FollowerHandleMetadataKey, f) + return ok +} + +func getValueFromMetadata(ctx context.Context, key string, f func(context.Context) (metadata.MD, bool)) (string, bool) { + md, ok := f(ctx) + if !ok { + return "", false + } + vs, ok := md[key] + if !ok { + return "", false + } + return vs[0], true +} + // GetOrCreateGRPCConn returns the corresponding grpc client connection of the given addr. // Returns the old one if's already existed in the clientConns; otherwise creates a new one and returns it. func GetOrCreateGRPCConn(ctx context.Context, clientConns *sync.Map, addr string, tlsCfg *tlsutil.TLSConfig, opt ...grpc.DialOption) (*grpc.ClientConn, error) { diff --git a/client/pd_service_discovery.go b/client/pd_service_discovery.go index b75276adbe9..33dda1ad282 100644 --- a/client/pd_service_discovery.go +++ b/client/pd_service_discovery.go @@ -33,6 +33,9 @@ import ( "github.com/tikv/pd/client/tlsutil" "go.uber.org/zap" "google.golang.org/grpc" + "google.golang.org/grpc/codes" + healthpb "google.golang.org/grpc/health/grpc_health_v1" + "google.golang.org/grpc/status" ) const ( @@ -74,8 +77,8 @@ type ServiceDiscovery interface { // GetServingAddr returns the serving endpoint which is the leader in a quorum-based cluster // or the primary in a primary/secondary configured cluster. GetServingAddr() string - // GetBackupAddrs gets the addresses of the current reachable and healthy backup service - // endpoints randomly. Backup service endpoints are followers in a quorum-based cluster or + // GetBackupAddrs gets the addresses of the current reachable backup service + // endpoints. Backup service endpoints are followers in a quorum-based cluster or // secondaries in a primary/secondary configured cluster. GetBackupAddrs() []string // GetOrCreateGRPCConn returns the corresponding grpc client connection of the given addr @@ -97,6 +100,236 @@ type ServiceDiscovery interface { AddServiceAddrsSwitchedCallback(callbacks ...func()) } +// ServiceClient is an interface that defines a set of operations for a raw PD gRPC client to specific PD server. +type ServiceClient interface { + // GetAddress returns the address information of the PD server. + GetAddress() string + // GetClientConn returns the gRPC connection of the service client + GetClientConn() *grpc.ClientConn + // BuildGRPCTargetContext builds a context object with a gRPC context. + // ctx: the original context object. + // mustLeader: whether must send to leader. + BuildGRPCTargetContext(ctx context.Context, mustLeader bool) context.Context + // IsConnectedToLeader returns whether the connected PD server is leader. + IsConnectedToLeader() bool + // Available returns if the network or other availability for the current service client is available. + Available() bool + // NeedRetry checks if client need to retry based on the PD server error response. + // And It will mark the client as unavailable if the pd error shows the follower can't handle request. + NeedRetry(*pdpb.Error, error) bool +} + +var ( + _ ServiceClient = (*pdServiceClient)(nil) + _ ServiceClient = (*pdServiceAPIClient)(nil) +) + +type pdServiceClient struct { + addr string + conn *grpc.ClientConn + isLeader bool + leaderAddr string + + networkFailure atomic.Bool +} + +func newPDServiceClient(addr, leaderAddr string, conn *grpc.ClientConn, isLeader bool) ServiceClient { + return &pdServiceClient{ + addr: addr, + conn: conn, + isLeader: isLeader, + leaderAddr: leaderAddr, + } +} + +// GetAddress implements ServiceClient. +func (c *pdServiceClient) GetAddress() string { + if c == nil { + return "" + } + return c.addr +} + +// BuildGRPCContext implements ServiceClient. +func (c *pdServiceClient) BuildGRPCTargetContext(ctx context.Context, toLeader bool) context.Context { + if c == nil || c.isLeader { + return ctx + } + if toLeader { + return grpcutil.BuildForwardContext(ctx, c.leaderAddr) + } + return grpcutil.BuildFollowerHandleContext(ctx) +} + +// IsConnectedToLeader implements ServiceClient. +func (c *pdServiceClient) IsConnectedToLeader() bool { + if c == nil { + return false + } + return c.isLeader +} + +// NetworkAvailable implements ServiceClient. +func (c *pdServiceClient) Available() bool { + if c == nil { + return false + } + return !c.networkFailure.Load() +} + +func (c *pdServiceClient) checkNetworkAvailable(ctx context.Context) { + if c == nil || c.conn == nil { + return + } + healthCli := healthpb.NewHealthClient(c.conn) + resp, err := healthCli.Check(ctx, &healthpb.HealthCheckRequest{Service: ""}) + failpoint.Inject("unreachableNetwork1", func() { + resp = nil + err = status.New(codes.Unavailable, "unavailable").Err() + }) + rpcErr, ok := status.FromError(err) + if (ok && isNetworkError(rpcErr.Code())) || resp.GetStatus() != healthpb.HealthCheckResponse_SERVING { + c.networkFailure.Store(true) + } else { + c.networkFailure.Store(false) + } +} + +func isNetworkError(code codes.Code) bool { + return code == codes.Unavailable || code == codes.DeadlineExceeded +} + +// GetClientConn implements ServiceClient. +func (c *pdServiceClient) GetClientConn() *grpc.ClientConn { + if c == nil { + return nil + } + return c.conn +} + +// NeedRetry implements ServiceClient. +func (c *pdServiceClient) NeedRetry(pdErr *pdpb.Error, err error) bool { + if c.IsConnectedToLeader() { + return false + } + return !(err == nil && pdErr == nil) +} + +type errFn func(pdErr *pdpb.Error) bool + +func regionAPIErrorFn(pdErr *pdpb.Error) bool { + return pdErr.GetType() == pdpb.ErrorType_REGION_NOT_FOUND +} + +// pdServiceAPIClient is a specific API client for PD service. +// It extends the pdServiceClient and adds additional fields for managing availability +type pdServiceAPIClient struct { + ServiceClient + fn errFn + + unavailable atomic.Bool + unavailableUntil atomic.Value +} + +func newPDServiceAPIClient(client ServiceClient, f errFn) ServiceClient { + return &pdServiceAPIClient{ + ServiceClient: client, + fn: f, + } +} + +// Available implements ServiceClient. +func (c *pdServiceAPIClient) Available() bool { + return c.ServiceClient.Available() && !c.unavailable.Load() +} + +func (c *pdServiceAPIClient) markAsAvailable() { + if !c.unavailable.Load() { + return + } + until := c.unavailableUntil.Load().(time.Time) + if time.Now().After(until) { + c.unavailable.Store(false) + } +} + +// NeedRetry implements ServiceClient. +func (c *pdServiceAPIClient) NeedRetry(pdErr *pdpb.Error, err error) bool { + if c.IsConnectedToLeader() { + return false + } + if err == nil && pdErr == nil { + return false + } + if c.fn(pdErr) && c.unavailable.CompareAndSwap(false, true) { + c.unavailableUntil.Store(time.Now().Add(time.Second * 10)) + failpoint.Inject("fastCheckAvailable", func() { + c.unavailableUntil.Store(time.Now().Add(time.Millisecond * 100)) + }) + } + return true +} + +// pdServiceBalancerNode is a balancer node for PD service. +// It extends the pdServiceClient and adds additional fields for the next polling client in the chain. +type pdServiceBalancerNode struct { + ServiceClient + next *pdServiceBalancerNode +} + +// pdServiceBalancer is a load balancer for PD service clients. +// It is used to balance the request to all servers and manage the connections to multiple PD service nodes. +type pdServiceBalancer struct { + mu sync.Mutex + now *pdServiceBalancerNode + totalNode int +} + +func (c *pdServiceBalancer) set(clients []ServiceClient) { + c.mu.Lock() + defer c.mu.Unlock() + if len(clients) == 0 { + return + } + c.totalNode = len(clients) + head := &pdServiceBalancerNode{ + ServiceClient: clients[0], + } + head.next = head + last := head + for i := 1; i < c.totalNode; i++ { + next := &pdServiceBalancerNode{ + ServiceClient: clients[i], + next: head, + } + head = next + last.next = head + } + c.now = head +} + +func (c *pdServiceBalancer) next() { + c.now = c.now.next +} + +func (c *pdServiceBalancer) get() (ret ServiceClient) { + c.mu.Lock() + defer c.mu.Unlock() + i := 0 + if c.now == nil { + return nil + } + for ; i < c.totalNode; i++ { + if c.now.Available() { + ret = c.now + c.next() + return + } + c.next() + } + return +} + type updateKeyspaceIDFunc func() error type tsoLocalServAddrsUpdatedFunc func(map[string]string) error type tsoGlobalServAddrUpdatedFunc func(string) error diff --git a/client/pd_service_discovery_test.go b/client/pd_service_discovery_test.go new file mode 100644 index 00000000000..98c5daed561 --- /dev/null +++ b/client/pd_service_discovery_test.go @@ -0,0 +1,295 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pd + +import ( + "context" + "errors" + "log" + "net" + "net/url" + "sync/atomic" + "testing" + "time" + + "github.com/pingcap/failpoint" + "github.com/pingcap/kvproto/pkg/pdpb" + "github.com/stretchr/testify/suite" + "github.com/tikv/pd/client/grpcutil" + "github.com/tikv/pd/client/testutil" + "google.golang.org/grpc" + pb "google.golang.org/grpc/examples/helloworld/helloworld" + "google.golang.org/grpc/health" + healthpb "google.golang.org/grpc/health/grpc_health_v1" + "google.golang.org/grpc/metadata" +) + +type testGRPCServer struct { + pb.UnimplementedGreeterServer + isLeader bool + leaderAddr string + leaderConn *grpc.ClientConn + handleCount atomic.Int32 + forwardCount atomic.Int32 +} + +// SayHello implements helloworld.GreeterServer +func (s *testGRPCServer) SayHello(ctx context.Context, in *pb.HelloRequest) (*pb.HelloReply, error) { + if !s.isLeader { + if !grpcutil.IsFollowerHandleEnabled(ctx, metadata.FromIncomingContext) { + if addr := grpcutil.GetForwardedHost(ctx, metadata.FromIncomingContext); addr == s.leaderAddr { + s.forwardCount.Add(1) + return pb.NewGreeterClient(s.leaderConn).SayHello(ctx, in) + } + return nil, errors.New("not leader") + } + } + s.handleCount.Add(1) + return &pb.HelloReply{Message: "Hello " + in.GetName()}, nil +} + +func (s *testGRPCServer) resetCount() { + s.handleCount.Store(0) + s.forwardCount.Store(0) +} + +func (s *testGRPCServer) getHandleCount() int32 { + return s.handleCount.Load() +} + +func (s *testGRPCServer) getForwardCount() int32 { + return s.forwardCount.Load() +} + +type testServer struct { + server *testGRPCServer + grpcServer *grpc.Server + addr string +} + +func newTestServer(isLeader bool) *testServer { + addr := testutil.Alloc() + u, err := url.Parse(addr) + if err != nil { + return nil + } + grpcServer := grpc.NewServer() + server := &testGRPCServer{ + isLeader: isLeader, + } + pb.RegisterGreeterServer(grpcServer, server) + hsrv := health.NewServer() + hsrv.SetServingStatus("", healthpb.HealthCheckResponse_SERVING) + healthpb.RegisterHealthServer(grpcServer, hsrv) + return &testServer{ + server: server, + grpcServer: grpcServer, + addr: u.Host, + } +} + +func (s *testServer) run() { + lis, err := net.Listen("tcp", s.addr) + if err != nil { + log.Fatalf("failed to serve: %v", err) + } + if err := s.grpcServer.Serve(lis); err != nil { + log.Fatalf("failed to serve: %v", err) + } +} + +type serviceClientTestSuite struct { + suite.Suite + ctx context.Context + clean context.CancelFunc + + leaderServer *testServer + followerServer *testServer + + leaderClient ServiceClient + followerClient ServiceClient +} + +func TestServiceClientClientTestSuite(t *testing.T) { + suite.Run(t, new(serviceClientTestSuite)) +} + +func (suite *serviceClientTestSuite) SetupSuite() { + suite.ctx, suite.clean = context.WithCancel(context.Background()) + + suite.leaderServer = newTestServer(true) + suite.followerServer = newTestServer(false) + go suite.leaderServer.run() + go suite.followerServer.run() + for i := 0; i < 10; i++ { + leaderConn, err1 := grpc.Dial(suite.leaderServer.addr, grpc.WithInsecure()) //nolint + followerConn, err2 := grpc.Dial(suite.followerServer.addr, grpc.WithInsecure()) //nolint + if err1 == nil && err2 == nil { + suite.followerClient = newPDServiceClient(suite.followerServer.addr, suite.leaderServer.addr, followerConn, false) + suite.leaderClient = newPDServiceClient(suite.leaderServer.addr, suite.leaderServer.addr, leaderConn, true) + suite.followerServer.server.leaderConn = suite.leaderClient.GetClientConn() + suite.followerServer.server.leaderAddr = suite.leaderClient.GetAddress() + return + } + time.Sleep(50 * time.Millisecond) + } + suite.NotNil(suite.leaderClient) +} + +func (suite *serviceClientTestSuite) TearDownTest() { + suite.leaderServer.server.resetCount() + suite.followerServer.server.resetCount() +} + +func (suite *serviceClientTestSuite) TearDownSuite() { + suite.leaderServer.grpcServer.GracefulStop() + suite.followerServer.grpcServer.GracefulStop() + suite.clean() +} + +func (suite *serviceClientTestSuite) TestServiceClient() { + re := suite.Require() + leaderAddress := suite.leaderServer.addr + followerAddress := suite.followerServer.addr + + follower := suite.followerClient + leader := suite.leaderClient + + re.Equal(follower.GetAddress(), followerAddress) + re.Equal(leader.GetAddress(), leaderAddress) + + re.True(follower.Available()) + re.True(leader.Available()) + + re.False(follower.IsConnectedToLeader()) + re.True(leader.IsConnectedToLeader()) + + re.NoError(failpoint.Enable("github.com/tikv/pd/client/unreachableNetwork1", "return(true)")) + follower.(*pdServiceClient).checkNetworkAvailable(suite.ctx) + leader.(*pdServiceClient).checkNetworkAvailable(suite.ctx) + re.False(follower.Available()) + re.False(leader.Available()) + re.NoError(failpoint.Disable("github.com/tikv/pd/client/unreachableNetwork1")) + + follower.(*pdServiceClient).checkNetworkAvailable(suite.ctx) + leader.(*pdServiceClient).checkNetworkAvailable(suite.ctx) + re.True(follower.Available()) + re.True(leader.Available()) + + followerConn := follower.GetClientConn() + leaderConn := leader.GetClientConn() + re.NotNil(followerConn) + re.NotNil(leaderConn) + + _, err := pb.NewGreeterClient(followerConn).SayHello(suite.ctx, &pb.HelloRequest{Name: "pd"}) + re.ErrorContains(err, "not leader") + resp, err := pb.NewGreeterClient(leaderConn).SayHello(suite.ctx, &pb.HelloRequest{Name: "pd"}) + re.NoError(err) + re.Equal("Hello pd", resp.GetMessage()) + + re.False(follower.NeedRetry(nil, nil)) + re.False(leader.NeedRetry(nil, nil)) + + ctx1 := context.WithoutCancel(suite.ctx) + ctx1 = follower.BuildGRPCTargetContext(ctx1, false) + re.True(grpcutil.IsFollowerHandleEnabled(ctx1, metadata.FromOutgoingContext)) + re.Empty(grpcutil.GetForwardedHost(ctx1, metadata.FromOutgoingContext)) + ctx2 := context.WithoutCancel(suite.ctx) + ctx2 = follower.BuildGRPCTargetContext(ctx2, true) + re.False(grpcutil.IsFollowerHandleEnabled(ctx2, metadata.FromOutgoingContext)) + re.Equal(grpcutil.GetForwardedHost(ctx2, metadata.FromOutgoingContext), leaderAddress) + + ctx3 := context.WithoutCancel(suite.ctx) + ctx3 = leader.BuildGRPCTargetContext(ctx3, false) + re.False(grpcutil.IsFollowerHandleEnabled(ctx3, metadata.FromOutgoingContext)) + re.Empty(grpcutil.GetForwardedHost(ctx3, metadata.FromOutgoingContext)) + ctx4 := context.WithoutCancel(suite.ctx) + ctx4 = leader.BuildGRPCTargetContext(ctx4, true) + re.False(grpcutil.IsFollowerHandleEnabled(ctx4, metadata.FromOutgoingContext)) + re.Empty(grpcutil.GetForwardedHost(ctx4, metadata.FromOutgoingContext)) + + followerAPIClient := newPDServiceAPIClient(follower, regionAPIErrorFn) + leaderAPIClient := newPDServiceAPIClient(leader, regionAPIErrorFn) + + re.NoError(failpoint.Enable("github.com/tikv/pd/client/fastCheckAvailable", "return(true)")) + + re.True(followerAPIClient.Available()) + re.True(leaderAPIClient.Available()) + pdErr1 := &pdpb.Error{ + Type: pdpb.ErrorType_UNKNOWN, + } + pdErr2 := &pdpb.Error{ + Type: pdpb.ErrorType_REGION_NOT_FOUND, + } + err = errors.New("error") + re.True(followerAPIClient.NeedRetry(pdErr1, nil)) + re.False(leaderAPIClient.NeedRetry(pdErr1, nil)) + re.True(followerAPIClient.Available()) + re.True(leaderAPIClient.Available()) + + re.True(followerAPIClient.NeedRetry(pdErr2, nil)) + re.False(leaderAPIClient.NeedRetry(pdErr2, nil)) + re.False(followerAPIClient.Available()) + re.True(leaderAPIClient.Available()) + followerAPIClient.(*pdServiceAPIClient).markAsAvailable() + leaderAPIClient.(*pdServiceAPIClient).markAsAvailable() + re.False(followerAPIClient.Available()) + time.Sleep(time.Millisecond * 100) + followerAPIClient.(*pdServiceAPIClient).markAsAvailable() + re.True(followerAPIClient.Available()) + + re.True(followerAPIClient.NeedRetry(nil, err)) + re.False(leaderAPIClient.NeedRetry(nil, err)) + re.True(followerAPIClient.Available()) + re.True(leaderAPIClient.Available()) + + re.NoError(failpoint.Disable("github.com/tikv/pd/client/fastCheckAvailable")) +} + +func (suite *serviceClientTestSuite) TestServiceClientBalancer() { + re := suite.Require() + follower := suite.followerClient + leader := suite.leaderClient + b := &pdServiceBalancer{} + b.set([]ServiceClient{leader, follower}) + re.Equal(2, b.totalNode) + + for i := 0; i < 10; i++ { + client := b.get() + ctx := client.BuildGRPCTargetContext(suite.ctx, false) + conn := client.GetClientConn() + re.NotNil(conn) + resp, err := pb.NewGreeterClient(conn).SayHello(ctx, &pb.HelloRequest{Name: "pd"}) + re.NoError(err) + re.Equal("Hello pd", resp.GetMessage()) + } + re.Equal(int32(5), suite.leaderServer.server.getHandleCount()) + re.Equal(int32(5), suite.followerServer.server.getHandleCount()) + suite.followerServer.server.resetCount() + suite.leaderServer.server.resetCount() + + for i := 0; i < 10; i++ { + client := b.get() + ctx := client.BuildGRPCTargetContext(suite.ctx, true) + conn := client.GetClientConn() + re.NotNil(conn) + resp, err := pb.NewGreeterClient(conn).SayHello(ctx, &pb.HelloRequest{Name: "pd"}) + re.NoError(err) + re.Equal("Hello pd", resp.GetMessage()) + } + re.Equal(int32(10), suite.leaderServer.server.getHandleCount()) + re.Equal(int32(0), suite.followerServer.server.getHandleCount()) + re.Equal(int32(5), suite.followerServer.server.getForwardCount()) +} diff --git a/client/testutil/check_env_dummy.go b/client/testutil/check_env_dummy.go new file mode 100644 index 00000000000..2fbcbd1a9e7 --- /dev/null +++ b/client/testutil/check_env_dummy.go @@ -0,0 +1,21 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//go:build !linux +// +build !linux + +package testutil + +func environmentCheck(addr string) bool { + return true +} diff --git a/client/testutil/check_env_linux.go b/client/testutil/check_env_linux.go new file mode 100644 index 00000000000..df45f359eb3 --- /dev/null +++ b/client/testutil/check_env_linux.go @@ -0,0 +1,42 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//go:build linux +// +build linux + +package testutil + +import ( + "github.com/cakturk/go-netstat/netstat" + "github.com/pingcap/log" + "go.uber.org/zap" +) + +func environmentCheck(addr string) bool { + valid, err := checkAddr(addr[len("http://"):]) + if err != nil { + log.Error("check port status failed", zap.Error(err)) + return false + } + return valid +} + +func checkAddr(addr string) (bool, error) { + tabs, err := netstat.TCPSocks(func(s *netstat.SockTabEntry) bool { + return s.RemoteAddr.String() == addr || s.LocalAddr.String() == addr + }) + if err != nil { + return false, err + } + return len(tabs) < 1, nil +} diff --git a/client/testutil/leak.go b/client/testutil/leak.go index 28b5baae60f..10345725171 100644 --- a/client/testutil/leak.go +++ b/client/testutil/leak.go @@ -23,4 +23,8 @@ var LeakOptions = []goleak.Option{ goleak.IgnoreTopFunction("google.golang.org/grpc.(*addrConn).createTransport"), goleak.IgnoreTopFunction("google.golang.org/grpc.(*addrConn).resetTransport"), goleak.IgnoreTopFunction("google.golang.org/grpc.(*Server).handleRawConn"), + // TODO: remove the below options once we fixed the http connection leak problems + goleak.IgnoreTopFunction("internal/poll.runtime_pollWait"), + goleak.IgnoreTopFunction("google.golang.org/grpc/internal/transport.(*controlBuffer).get"), + goleak.IgnoreTopFunction("google.golang.org/grpc/internal/transport.(*http2Server).keepalive"), } diff --git a/client/testutil/tempurl.go b/client/testutil/tempurl.go new file mode 100644 index 00000000000..ac8f29fa345 --- /dev/null +++ b/client/testutil/tempurl.go @@ -0,0 +1,65 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package testutil + +import ( + "fmt" + "net" + "sync" + "time" + + "github.com/pingcap/log" + "go.uber.org/zap" +) + +var ( + testAddrMutex sync.Mutex + testAddrMap = make(map[string]struct{}) +) + +// Alloc allocates a local URL for testing. +func Alloc() string { + for i := 0; i < 10; i++ { + if u := tryAllocTestURL(); u != "" { + return u + } + time.Sleep(time.Second) + } + log.Fatal("failed to alloc test URL") + return "" +} + +func tryAllocTestURL() string { + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + log.Fatal("listen failed", zap.Error(err)) + } + addr := fmt.Sprintf("http://%s", l.Addr()) + err = l.Close() + if err != nil { + log.Fatal("close failed", zap.Error(err)) + } + + testAddrMutex.Lock() + defer testAddrMutex.Unlock() + if _, ok := testAddrMap[addr]; ok { + return "" + } + if !environmentCheck(addr) { + return "" + } + testAddrMap[addr] = struct{}{} + return addr +} diff --git a/pkg/utils/tempurl/check_env.go b/pkg/utils/tempurl/check_env_dummy.go similarity index 100% rename from pkg/utils/tempurl/check_env.go rename to pkg/utils/tempurl/check_env_dummy.go diff --git a/tests/integrations/client/go.mod b/tests/integrations/client/go.mod index 5c73abfb8ed..d0d5a202ad4 100644 --- a/tests/integrations/client/go.mod +++ b/tests/integrations/client/go.mod @@ -165,19 +165,19 @@ require ( go.uber.org/fx v1.12.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/arch v0.3.0 // indirect - golang.org/x/crypto v0.14.0 // indirect + golang.org/x/crypto v0.15.0 // indirect golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4 // indirect golang.org/x/image v0.5.0 // indirect - golang.org/x/net v0.17.0 // indirect + golang.org/x/net v0.18.0 // indirect golang.org/x/oauth2 v0.4.0 // indirect golang.org/x/sync v0.1.0 // indirect - golang.org/x/sys v0.13.0 // indirect - golang.org/x/text v0.13.0 // indirect + golang.org/x/sys v0.14.0 // indirect + golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.1.0 // indirect golang.org/x/tools v0.6.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect - google.golang.org/protobuf v1.30.0 // indirect + google.golang.org/protobuf v1.31.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/tests/integrations/client/go.sum b/tests/integrations/client/go.sum index f70f08f366e..d07c79e1a92 100644 --- a/tests/integrations/client/go.sum +++ b/tests/integrations/client/go.sum @@ -637,8 +637,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= -golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA= +golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4 h1:QLureRX3moex6NVu/Lr4MGakp9FdA7sBHGBmvRW7NaM= golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= @@ -684,8 +684,8 @@ golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1 golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= +golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -740,8 +740,8 @@ golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= +golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -750,8 +750,8 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.1.0 h1:xYY+Bajn2a7VBmTM5GikTmnK8ZuX8YgnQCqZpbBNtmA= @@ -815,8 +815,8 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/tests/integrations/mcs/go.mod b/tests/integrations/mcs/go.mod index 2e41e87b746..6a7a255e895 100644 --- a/tests/integrations/mcs/go.mod +++ b/tests/integrations/mcs/go.mod @@ -165,19 +165,19 @@ require ( go.uber.org/fx v1.12.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/arch v0.3.0 // indirect - golang.org/x/crypto v0.14.0 // indirect + golang.org/x/crypto v0.15.0 // indirect golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4 // indirect golang.org/x/image v0.5.0 // indirect - golang.org/x/net v0.17.0 // indirect + golang.org/x/net v0.18.0 // indirect golang.org/x/oauth2 v0.4.0 // indirect golang.org/x/sync v0.1.0 // indirect - golang.org/x/sys v0.13.0 // indirect - golang.org/x/text v0.13.0 // indirect + golang.org/x/sys v0.14.0 // indirect + golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.1.0 // indirect golang.org/x/tools v0.6.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect - google.golang.org/protobuf v1.30.0 // indirect + google.golang.org/protobuf v1.31.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/tests/integrations/mcs/go.sum b/tests/integrations/mcs/go.sum index 65e8cf72aab..8392ed74f7b 100644 --- a/tests/integrations/mcs/go.sum +++ b/tests/integrations/mcs/go.sum @@ -641,8 +641,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= -golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA= +golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4 h1:QLureRX3moex6NVu/Lr4MGakp9FdA7sBHGBmvRW7NaM= golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= @@ -688,8 +688,8 @@ golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1 golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= +golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -744,8 +744,8 @@ golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= +golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -754,8 +754,8 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.1.0 h1:xYY+Bajn2a7VBmTM5GikTmnK8ZuX8YgnQCqZpbBNtmA= @@ -818,8 +818,8 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/tests/integrations/realcluster/go.mod b/tests/integrations/realcluster/go.mod index 66f40012863..0815fc23d8d 100644 --- a/tests/integrations/realcluster/go.mod +++ b/tests/integrations/realcluster/go.mod @@ -36,12 +36,12 @@ require ( go.uber.org/goleak v1.1.12 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.24.0 // indirect - golang.org/x/net v0.17.0 // indirect - golang.org/x/sys v0.13.0 // indirect - golang.org/x/text v0.13.0 // indirect + golang.org/x/net v0.18.0 // indirect + golang.org/x/sys v0.14.0 // indirect + golang.org/x/text v0.14.0 // indirect google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect google.golang.org/grpc v1.54.0 // indirect - google.golang.org/protobuf v1.30.0 // indirect + google.golang.org/protobuf v1.31.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/tests/integrations/realcluster/go.sum b/tests/integrations/realcluster/go.sum index fde38211174..4808a766575 100644 --- a/tests/integrations/realcluster/go.sum +++ b/tests/integrations/realcluster/go.sum @@ -164,8 +164,8 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= +golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -189,14 +189,14 @@ golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= +golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -222,8 +222,8 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/tests/integrations/tso/go.mod b/tests/integrations/tso/go.mod index af38fcf4241..84ac5a83f8e 100644 --- a/tests/integrations/tso/go.mod +++ b/tests/integrations/tso/go.mod @@ -166,19 +166,19 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.24.0 // indirect golang.org/x/arch v0.3.0 // indirect - golang.org/x/crypto v0.14.0 // indirect + golang.org/x/crypto v0.15.0 // indirect golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4 // indirect golang.org/x/image v0.5.0 // indirect - golang.org/x/net v0.17.0 // indirect + golang.org/x/net v0.18.0 // indirect golang.org/x/oauth2 v0.4.0 // indirect golang.org/x/sync v0.1.0 // indirect - golang.org/x/sys v0.13.0 // indirect - golang.org/x/text v0.13.0 // indirect + golang.org/x/sys v0.14.0 // indirect + golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.1.0 // indirect golang.org/x/tools v0.6.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect - google.golang.org/protobuf v1.30.0 // indirect + google.golang.org/protobuf v1.31.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/tests/integrations/tso/go.sum b/tests/integrations/tso/go.sum index c078eab2f6d..00860368c28 100644 --- a/tests/integrations/tso/go.sum +++ b/tests/integrations/tso/go.sum @@ -636,8 +636,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= -golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA= +golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4 h1:QLureRX3moex6NVu/Lr4MGakp9FdA7sBHGBmvRW7NaM= golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= @@ -683,8 +683,8 @@ golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1 golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= +golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -739,8 +739,8 @@ golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= +golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -749,8 +749,8 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.1.0 h1:xYY+Bajn2a7VBmTM5GikTmnK8ZuX8YgnQCqZpbBNtmA= @@ -813,8 +813,8 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/tools/pd-api-bench/go.mod b/tools/pd-api-bench/go.mod index 77b4891d3d8..5f834e0f565 100644 --- a/tools/pd-api-bench/go.mod +++ b/tools/pd-api-bench/go.mod @@ -98,14 +98,14 @@ require ( go.uber.org/goleak v1.1.12 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/arch v0.3.0 // indirect - golang.org/x/crypto v0.14.0 // indirect + golang.org/x/crypto v0.15.0 // indirect golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4 // indirect - golang.org/x/net v0.17.0 // indirect - golang.org/x/sys v0.13.0 // indirect - golang.org/x/text v0.13.0 // indirect + golang.org/x/net v0.18.0 // indirect + golang.org/x/sys v0.14.0 // indirect + golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.1.0 // indirect google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect - google.golang.org/protobuf v1.30.0 // indirect + google.golang.org/protobuf v1.31.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/tools/pd-api-bench/go.sum b/tools/pd-api-bench/go.sum index 1e40c511586..605b8cf3b5f 100644 --- a/tools/pd-api-bench/go.sum +++ b/tools/pd-api-bench/go.sum @@ -367,8 +367,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= -golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA= +golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4 h1:QLureRX3moex6NVu/Lr4MGakp9FdA7sBHGBmvRW7NaM= golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= @@ -397,8 +397,8 @@ golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= +golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -434,16 +434,16 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= +golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.1.0 h1:xYY+Bajn2a7VBmTM5GikTmnK8ZuX8YgnQCqZpbBNtmA= golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -487,8 +487,8 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/tools/pd-tso-bench/go.mod b/tools/pd-tso-bench/go.mod index 8d4b3d18a31..c8fe68aab44 100644 --- a/tools/pd-tso-bench/go.mod +++ b/tools/pd-tso-bench/go.mod @@ -27,11 +27,11 @@ require ( github.com/prometheus/procfs v0.6.0 // indirect go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/net v0.17.0 // indirect - golang.org/x/sys v0.13.0 // indirect - golang.org/x/text v0.13.0 // indirect + golang.org/x/net v0.18.0 // indirect + golang.org/x/sys v0.14.0 // indirect + golang.org/x/text v0.14.0 // indirect google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect - google.golang.org/protobuf v1.30.0 // indirect + google.golang.org/protobuf v1.31.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect ) diff --git a/tools/pd-tso-bench/go.sum b/tools/pd-tso-bench/go.sum index 15ba2923695..31875879c2d 100644 --- a/tools/pd-tso-bench/go.sum +++ b/tools/pd-tso-bench/go.sum @@ -12,6 +12,8 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 h1:BjkPE3785EwPhhyuFkbINB+2a1xATwk8SNDWnJiD41g= +github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5/go.mod h1:jtAfVaU/2cu1+wdSRPWE2c1N2qeAA3K4RH9pYgqwets= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -159,8 +161,8 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= +golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -180,13 +182,13 @@ golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= +golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -207,6 +209,8 @@ google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= google.golang.org/grpc v1.56.3 h1:8I4C0Yq1EjstUzUJzpcRVbuYA2mODtEmpWiQoN/b2nc= google.golang.org/grpc v1.56.3/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= +google.golang.org/grpc/examples v0.0.0-20230419000256-16651f60ddc5 h1:FjbWL/mGfyRQNxjagfT1chiHL1569WEA/OGH0ZIzGcI= +google.golang.org/grpc/examples v0.0.0-20230419000256-16651f60ddc5/go.mod h1:5av8LiY5jU2KRcrX+SHtvLHnaOpPJ7gzWStBurgHlqY= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -215,8 +219,8 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 5d00fafcd1589aa700d2de4124c86c4da046befd Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Fri, 22 Dec 2023 12:40:53 +0800 Subject: [PATCH 120/137] mcs: support status api (#7606) ref tikv/pd#5839 Signed-off-by: Ryan Leung Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- pkg/mcs/resourcemanager/server/apis/v1/api.go | 1 + pkg/mcs/scheduling/server/apis/v1/api.go | 1 + pkg/mcs/server/server.go | 5 +++ pkg/mcs/tso/server/apis/v1/api.go | 1 + pkg/mcs/utils/util.go | 16 +++++++ pkg/versioninfo/versioninfo.go | 9 ++++ server/api/status.go | 12 +---- server/api/status_test.go | 2 +- .../mcs/resourcemanager/server_test.go | 16 +++++++ tests/integrations/mcs/scheduling/api_test.go | 44 +++++++++++++++++++ tests/integrations/mcs/tso/api_test.go | 31 +++++++++++++ tests/integrations/mcs/tso/server_test.go | 12 ----- 12 files changed, 127 insertions(+), 23 deletions(-) diff --git a/pkg/mcs/resourcemanager/server/apis/v1/api.go b/pkg/mcs/resourcemanager/server/apis/v1/api.go index 7b5f2903484..93738664361 100644 --- a/pkg/mcs/resourcemanager/server/apis/v1/api.go +++ b/pkg/mcs/resourcemanager/server/apis/v1/api.go @@ -82,6 +82,7 @@ func NewService(srv *rmserver.Service) *Service { c.Next() }) apiHandlerEngine.GET("metrics", utils.PromHandler()) + apiHandlerEngine.GET("status", utils.StatusHandler) pprof.Register(apiHandlerEngine) endpoint := apiHandlerEngine.Group(APIPathPrefix) endpoint.Use(multiservicesapi.ServiceRedirector()) diff --git a/pkg/mcs/scheduling/server/apis/v1/api.go b/pkg/mcs/scheduling/server/apis/v1/api.go index e6881f2f85c..8c4ed015a5c 100644 --- a/pkg/mcs/scheduling/server/apis/v1/api.go +++ b/pkg/mcs/scheduling/server/apis/v1/api.go @@ -107,6 +107,7 @@ func NewService(srv *scheserver.Service) *Service { c.Next() }) apiHandlerEngine.GET("metrics", mcsutils.PromHandler()) + apiHandlerEngine.GET("status", mcsutils.StatusHandler) pprof.Register(apiHandlerEngine) root := apiHandlerEngine.Group(APIPathPrefix) root.Use(multiservicesapi.ServiceRedirector()) diff --git a/pkg/mcs/server/server.go b/pkg/mcs/server/server.go index 95b806e20ef..a8dedd8ad91 100644 --- a/pkg/mcs/server/server.go +++ b/pkg/mcs/server/server.go @@ -162,3 +162,8 @@ func (bs *BaseServer) GetListener() net.Listener { func (bs *BaseServer) IsSecure() bool { return bs.secure } + +// StartTimestamp returns the start timestamp of this server +func (bs *BaseServer) StartTimestamp() int64 { + return bs.startTimestamp +} diff --git a/pkg/mcs/tso/server/apis/v1/api.go b/pkg/mcs/tso/server/apis/v1/api.go index e5f0dfb5440..94282592151 100644 --- a/pkg/mcs/tso/server/apis/v1/api.go +++ b/pkg/mcs/tso/server/apis/v1/api.go @@ -91,6 +91,7 @@ func NewService(srv *tsoserver.Service) *Service { c.Next() }) apiHandlerEngine.GET("metrics", utils.PromHandler()) + apiHandlerEngine.GET("status", utils.StatusHandler) pprof.Register(apiHandlerEngine) root := apiHandlerEngine.Group(APIPathPrefix) root.Use(multiservicesapi.ServiceRedirector()) diff --git a/pkg/mcs/utils/util.go b/pkg/mcs/utils/util.go index a0708f9bf88..6fa6c2cb08e 100644 --- a/pkg/mcs/utils/util.go +++ b/pkg/mcs/utils/util.go @@ -33,9 +33,11 @@ import ( "github.com/soheilhy/cmux" "github.com/tikv/pd/pkg/errs" "github.com/tikv/pd/pkg/utils/apiutil" + "github.com/tikv/pd/pkg/utils/apiutil/multiservicesapi" "github.com/tikv/pd/pkg/utils/etcdutil" "github.com/tikv/pd/pkg/utils/grpcutil" "github.com/tikv/pd/pkg/utils/logutil" + "github.com/tikv/pd/pkg/versioninfo" "go.etcd.io/etcd/clientv3" "go.etcd.io/etcd/pkg/types" "go.uber.org/zap" @@ -78,6 +80,19 @@ func PromHandler() gin.HandlerFunc { } } +// StatusHandler is a handler to get status info. +func StatusHandler(c *gin.Context) { + svr := c.MustGet(multiservicesapi.ServiceContextKey).(server) + version := versioninfo.Status{ + BuildTS: versioninfo.PDBuildTS, + GitHash: versioninfo.PDGitHash, + Version: versioninfo.PDReleaseVersion, + StartTimestamp: svr.StartTimestamp(), + } + + c.IndentedJSON(http.StatusOK, version) +} + type server interface { GetBackendEndpoints() string Context() context.Context @@ -97,6 +112,7 @@ type server interface { RegisterGRPCService(*grpc.Server) SetUpRestHandler() (http.Handler, apiutil.APIServiceGroup) diagnosticspb.DiagnosticsServer + StartTimestamp() int64 } // WaitAPIServiceReady waits for the api service ready. diff --git a/pkg/versioninfo/versioninfo.go b/pkg/versioninfo/versioninfo.go index 7b666334002..d6f4738d786 100644 --- a/pkg/versioninfo/versioninfo.go +++ b/pkg/versioninfo/versioninfo.go @@ -24,6 +24,15 @@ import ( "go.uber.org/zap" ) +// Status is the status of PD server. +// NOTE: This type is exported by HTTP API. Please pay more attention when modifying it. +type Status struct { + BuildTS string `json:"build_ts"` + Version string `json:"version"` + GitHash string `json:"git_hash"` + StartTimestamp int64 `json:"start_timestamp"` +} + const ( // CommunityEdition is the default edition for building. CommunityEdition = "Community" diff --git a/server/api/status.go b/server/api/status.go index 8f837ec019d..ba8f3b484dd 100644 --- a/server/api/status.go +++ b/server/api/status.go @@ -27,14 +27,6 @@ type statusHandler struct { rd *render.Render } -// NOTE: This type is exported by HTTP API. Please pay more attention when modifying it. -type status struct { - BuildTS string `json:"build_ts"` - Version string `json:"version"` - GitHash string `json:"git_hash"` - StartTimestamp int64 `json:"start_timestamp"` -} - func newStatusHandler(svr *server.Server, rd *render.Render) *statusHandler { return &statusHandler{ svr: svr, @@ -44,10 +36,10 @@ func newStatusHandler(svr *server.Server, rd *render.Render) *statusHandler { // @Summary Get the build info of PD server. // @Produce json -// @Success 200 {object} status +// @Success 200 {object} versioninfo.Status // @Router /status [get] func (h *statusHandler) GetPDStatus(w http.ResponseWriter, r *http.Request) { - version := status{ + version := versioninfo.Status{ BuildTS: versioninfo.PDBuildTS, GitHash: versioninfo.PDGitHash, Version: versioninfo.PDReleaseVersion, diff --git a/server/api/status_test.go b/server/api/status_test.go index 7017c4271a1..065618efb6c 100644 --- a/server/api/status_test.go +++ b/server/api/status_test.go @@ -24,7 +24,7 @@ import ( ) func checkStatusResponse(re *require.Assertions, body []byte) { - got := status{} + got := versioninfo.Status{} re.NoError(json.Unmarshal(body, &got)) re.Equal(versioninfo.PDBuildTS, got.BuildTS) re.Equal(versioninfo.PDGitHash, got.GitHash) diff --git a/tests/integrations/mcs/resourcemanager/server_test.go b/tests/integrations/mcs/resourcemanager/server_test.go index b7a9b0be218..d3837ad15f3 100644 --- a/tests/integrations/mcs/resourcemanager/server_test.go +++ b/tests/integrations/mcs/resourcemanager/server_test.go @@ -26,6 +26,7 @@ import ( "github.com/stretchr/testify/require" "github.com/tikv/pd/client/grpcutil" "github.com/tikv/pd/pkg/utils/tempurl" + "github.com/tikv/pd/pkg/versioninfo" "github.com/tikv/pd/tests" ) @@ -102,4 +103,19 @@ func TestResourceManagerServer(t *testing.T) { re.NoError(err) re.Contains(string(respBytes), "resource_manager_server_info") } + + // Test status handler + { + resp, err := http.Get(addr + "/status") + re.NoError(err) + defer resp.Body.Close() + re.Equal(http.StatusOK, resp.StatusCode) + respBytes, err := io.ReadAll(resp.Body) + re.NoError(err) + var s versioninfo.Status + re.NoError(json.Unmarshal(respBytes, &s)) + re.Equal(versioninfo.PDBuildTS, s.BuildTS) + re.Equal(versioninfo.PDGitHash, s.GitHash) + re.Equal(versioninfo.PDReleaseVersion, s.Version) + } } diff --git a/tests/integrations/mcs/scheduling/api_test.go b/tests/integrations/mcs/scheduling/api_test.go index 867a7bfb5df..177f959b953 100644 --- a/tests/integrations/mcs/scheduling/api_test.go +++ b/tests/integrations/mcs/scheduling/api_test.go @@ -5,6 +5,7 @@ import ( "encoding/hex" "encoding/json" "fmt" + "io" "net/http" "testing" "time" @@ -22,6 +23,7 @@ import ( "github.com/tikv/pd/pkg/storage" "github.com/tikv/pd/pkg/utils/apiutil" "github.com/tikv/pd/pkg/utils/testutil" + "github.com/tikv/pd/pkg/versioninfo" "github.com/tikv/pd/tests" ) @@ -547,3 +549,45 @@ func (suite *apiTestSuite) checkFollowerForward(cluster *tests.TestCluster) { ) re.NoError(err) } + +func (suite *apiTestSuite) TestMetrics() { + suite.env.RunTestInAPIMode(suite.checkMetrics) +} + +func (suite *apiTestSuite) checkMetrics(cluster *tests.TestCluster) { + re := suite.Require() + s := cluster.GetSchedulingPrimaryServer() + testutil.Eventually(re, func() bool { + return s.IsServing() + }, testutil.WithWaitFor(5*time.Second), testutil.WithTickInterval(50*time.Millisecond)) + resp, err := http.Get(s.GetConfig().GetAdvertiseListenAddr() + "/metrics") + re.NoError(err) + defer resp.Body.Close() + re.Equal(http.StatusOK, resp.StatusCode) + respBytes, err := io.ReadAll(resp.Body) + re.NoError(err) + re.Contains(string(respBytes), "scheduling_server_info") +} + +func (suite *apiTestSuite) TestStatus() { + suite.env.RunTestInAPIMode(suite.checkStatus) +} + +func (suite *apiTestSuite) checkStatus(cluster *tests.TestCluster) { + re := suite.Require() + s := cluster.GetSchedulingPrimaryServer() + testutil.Eventually(re, func() bool { + return s.IsServing() + }, testutil.WithWaitFor(5*time.Second), testutil.WithTickInterval(50*time.Millisecond)) + resp, err := http.Get(s.GetConfig().GetAdvertiseListenAddr() + "/status") + re.NoError(err) + defer resp.Body.Close() + re.Equal(http.StatusOK, resp.StatusCode) + respBytes, err := io.ReadAll(resp.Body) + re.NoError(err) + var status versioninfo.Status + re.NoError(json.Unmarshal(respBytes, &status)) + re.Equal(versioninfo.PDBuildTS, status.BuildTS) + re.Equal(versioninfo.PDGitHash, status.GitHash) + re.Equal(versioninfo.PDReleaseVersion, status.Version) +} diff --git a/tests/integrations/mcs/tso/api_test.go b/tests/integrations/mcs/tso/api_test.go index 59073aca1e1..a5d68cfa90f 100644 --- a/tests/integrations/mcs/tso/api_test.go +++ b/tests/integrations/mcs/tso/api_test.go @@ -33,6 +33,7 @@ import ( "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/utils/apiutil" "github.com/tikv/pd/pkg/utils/testutil" + "github.com/tikv/pd/pkg/versioninfo" "github.com/tikv/pd/server/config" "github.com/tikv/pd/tests" ) @@ -244,3 +245,33 @@ func TestForwardOnlyTSONoScheduling(t *testing.T) { testutil.Status(re, http.StatusInternalServerError), testutil.StringContain(re, "[PD:apiutil:ErrRedirect]redirect failed")) re.NoError(err) } + +func (suite *tsoAPITestSuite) TestMetrics() { + re := suite.Require() + + primary := suite.tsoCluster.WaitForDefaultPrimaryServing(re) + resp, err := http.Get(primary.GetConfig().GetAdvertiseListenAddr() + "/metrics") + re.NoError(err) + defer resp.Body.Close() + re.Equal(http.StatusOK, resp.StatusCode) + respBytes, err := io.ReadAll(resp.Body) + re.NoError(err) + re.Contains(string(respBytes), "tso_server_info") +} + +func (suite *tsoAPITestSuite) TestStatus() { + re := suite.Require() + + primary := suite.tsoCluster.WaitForDefaultPrimaryServing(re) + resp, err := http.Get(primary.GetConfig().GetAdvertiseListenAddr() + "/status") + re.NoError(err) + defer resp.Body.Close() + re.Equal(http.StatusOK, resp.StatusCode) + respBytes, err := io.ReadAll(resp.Body) + re.NoError(err) + var s versioninfo.Status + re.NoError(json.Unmarshal(respBytes, &s)) + re.Equal(versioninfo.PDBuildTS, s.BuildTS) + re.Equal(versioninfo.PDGitHash, s.GitHash) + re.Equal(versioninfo.PDReleaseVersion, s.Version) +} diff --git a/tests/integrations/mcs/tso/server_test.go b/tests/integrations/mcs/tso/server_test.go index 9dc9aa32368..a6a2c42acc9 100644 --- a/tests/integrations/mcs/tso/server_test.go +++ b/tests/integrations/mcs/tso/server_test.go @@ -576,18 +576,6 @@ func (suite *CommonTestSuite) TestAdvertiseAddr() { re.Equal(conf.GetListenAddr(), conf.GetAdvertiseListenAddr()) } -func (suite *CommonTestSuite) TestMetrics() { - re := suite.Require() - - resp, err := http.Get(suite.tsoDefaultPrimaryServer.GetConfig().GetAdvertiseListenAddr() + "/metrics") - re.NoError(err) - defer resp.Body.Close() - re.Equal(http.StatusOK, resp.StatusCode) - respBytes, err := io.ReadAll(resp.Body) - re.NoError(err) - re.Contains(string(respBytes), "tso_server_info") -} - func (suite *CommonTestSuite) TestBootstrapDefaultKeyspaceGroup() { re := suite.Require() From 16e80d8770430b8b5a0866cbbeecff9bdab3f64b Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Fri, 22 Dec 2023 13:10:54 +0800 Subject: [PATCH 121/137] *: bump image and crypto (#7608) ref tikv/pd#4399 Signed-off-by: Ryan Leung Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 9 +++++++-- tests/integrations/client/go.mod | 6 +++--- tests/integrations/client/go.sum | 17 +++++++++++------ tests/integrations/mcs/go.mod | 6 +++--- tests/integrations/mcs/go.sum | 17 +++++++++++------ tests/integrations/tso/go.mod | 6 +++--- tests/integrations/tso/go.sum | 17 +++++++++++------ tools/pd-api-bench/go.mod | 4 ++-- tools/pd-api-bench/go.sum | 8 ++++---- 10 files changed, 56 insertions(+), 36 deletions(-) diff --git a/go.mod b/go.mod index e60364d9c57..157cdb4eab0 100644 --- a/go.mod +++ b/go.mod @@ -184,7 +184,7 @@ require ( go.uber.org/multierr v1.11.0 // indirect golang.org/x/arch v0.3.0 // indirect golang.org/x/crypto v0.14.0 // indirect - golang.org/x/image v0.5.0 // indirect + golang.org/x/image v0.10.0 // indirect golang.org/x/mod v0.11.0 // indirect golang.org/x/net v0.17.0 // indirect golang.org/x/oauth2 v0.4.0 // indirect diff --git a/go.sum b/go.sum index b1136a6f2a2..b244a791a9a 100644 --- a/go.sum +++ b/go.sum @@ -686,8 +686,8 @@ golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4 h1:QLureRX3moex6NVu/Lr4MGakp9FdA7sBHGBmvRW7NaM= golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/image v0.0.0-20200119044424-58c23975cae1/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/image v0.5.0 h1:5JMiNunQeQw++mMOz48/ISeNu3Iweh/JaZU8ZLqHRrI= -golang.org/x/image v0.5.0/go.mod h1:FVC7BI/5Ym8R25iw5OLsgshdUBbT1h5jZTpA+mvAdZ4= +golang.org/x/image v0.10.0 h1:gXjUUtwtx5yOE0VKWq1CH4IJAClq4UGgUA3i+rpON9M= +golang.org/x/image v0.10.0/go.mod h1:jtrku+n79PfroUbvDdeUWMAI+heR786BofxrbiSF+J0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -700,6 +700,7 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU= golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -727,6 +728,7 @@ golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1 golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -787,11 +789,13 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -800,6 +804,7 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/tests/integrations/client/go.mod b/tests/integrations/client/go.mod index d0d5a202ad4..5bbe8219bf0 100644 --- a/tests/integrations/client/go.mod +++ b/tests/integrations/client/go.mod @@ -165,13 +165,13 @@ require ( go.uber.org/fx v1.12.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/arch v0.3.0 // indirect - golang.org/x/crypto v0.15.0 // indirect + golang.org/x/crypto v0.17.0 // indirect golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4 // indirect - golang.org/x/image v0.5.0 // indirect + golang.org/x/image v0.10.0 // indirect golang.org/x/net v0.18.0 // indirect golang.org/x/oauth2 v0.4.0 // indirect golang.org/x/sync v0.1.0 // indirect - golang.org/x/sys v0.14.0 // indirect + golang.org/x/sys v0.15.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.1.0 // indirect golang.org/x/tools v0.6.0 // indirect diff --git a/tests/integrations/client/go.sum b/tests/integrations/client/go.sum index d07c79e1a92..2ed7bb415af 100644 --- a/tests/integrations/client/go.sum +++ b/tests/integrations/client/go.sum @@ -637,14 +637,14 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA= -golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g= +golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= +golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4 h1:QLureRX3moex6NVu/Lr4MGakp9FdA7sBHGBmvRW7NaM= golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/image v0.0.0-20200119044424-58c23975cae1/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/image v0.5.0 h1:5JMiNunQeQw++mMOz48/ISeNu3Iweh/JaZU8ZLqHRrI= -golang.org/x/image v0.5.0/go.mod h1:FVC7BI/5Ym8R25iw5OLsgshdUBbT1h5jZTpA+mvAdZ4= +golang.org/x/image v0.10.0 h1:gXjUUtwtx5yOE0VKWq1CH4IJAClq4UGgUA3i+rpON9M= +golang.org/x/image v0.10.0/go.mod h1:jtrku+n79PfroUbvDdeUWMAI+heR786BofxrbiSF+J0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -657,6 +657,7 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU= golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -684,6 +685,7 @@ golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1 golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -739,17 +741,20 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= -golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/tests/integrations/mcs/go.mod b/tests/integrations/mcs/go.mod index 6a7a255e895..5771535dd19 100644 --- a/tests/integrations/mcs/go.mod +++ b/tests/integrations/mcs/go.mod @@ -165,13 +165,13 @@ require ( go.uber.org/fx v1.12.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/arch v0.3.0 // indirect - golang.org/x/crypto v0.15.0 // indirect + golang.org/x/crypto v0.17.0 // indirect golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4 // indirect - golang.org/x/image v0.5.0 // indirect + golang.org/x/image v0.10.0 // indirect golang.org/x/net v0.18.0 // indirect golang.org/x/oauth2 v0.4.0 // indirect golang.org/x/sync v0.1.0 // indirect - golang.org/x/sys v0.14.0 // indirect + golang.org/x/sys v0.15.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.1.0 // indirect golang.org/x/tools v0.6.0 // indirect diff --git a/tests/integrations/mcs/go.sum b/tests/integrations/mcs/go.sum index 8392ed74f7b..36960bcb5a9 100644 --- a/tests/integrations/mcs/go.sum +++ b/tests/integrations/mcs/go.sum @@ -641,14 +641,14 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA= -golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g= +golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= +golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4 h1:QLureRX3moex6NVu/Lr4MGakp9FdA7sBHGBmvRW7NaM= golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/image v0.0.0-20200119044424-58c23975cae1/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/image v0.5.0 h1:5JMiNunQeQw++mMOz48/ISeNu3Iweh/JaZU8ZLqHRrI= -golang.org/x/image v0.5.0/go.mod h1:FVC7BI/5Ym8R25iw5OLsgshdUBbT1h5jZTpA+mvAdZ4= +golang.org/x/image v0.10.0 h1:gXjUUtwtx5yOE0VKWq1CH4IJAClq4UGgUA3i+rpON9M= +golang.org/x/image v0.10.0/go.mod h1:jtrku+n79PfroUbvDdeUWMAI+heR786BofxrbiSF+J0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -661,6 +661,7 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU= golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -688,6 +689,7 @@ golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1 golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -743,17 +745,20 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= -golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/tests/integrations/tso/go.mod b/tests/integrations/tso/go.mod index 84ac5a83f8e..3d95b558ebf 100644 --- a/tests/integrations/tso/go.mod +++ b/tests/integrations/tso/go.mod @@ -166,13 +166,13 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.24.0 // indirect golang.org/x/arch v0.3.0 // indirect - golang.org/x/crypto v0.15.0 // indirect + golang.org/x/crypto v0.17.0 // indirect golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4 // indirect - golang.org/x/image v0.5.0 // indirect + golang.org/x/image v0.10.0 // indirect golang.org/x/net v0.18.0 // indirect golang.org/x/oauth2 v0.4.0 // indirect golang.org/x/sync v0.1.0 // indirect - golang.org/x/sys v0.14.0 // indirect + golang.org/x/sys v0.15.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.1.0 // indirect golang.org/x/tools v0.6.0 // indirect diff --git a/tests/integrations/tso/go.sum b/tests/integrations/tso/go.sum index 00860368c28..668c1bce1a4 100644 --- a/tests/integrations/tso/go.sum +++ b/tests/integrations/tso/go.sum @@ -636,14 +636,14 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA= -golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g= +golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= +golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4 h1:QLureRX3moex6NVu/Lr4MGakp9FdA7sBHGBmvRW7NaM= golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/image v0.0.0-20200119044424-58c23975cae1/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/image v0.5.0 h1:5JMiNunQeQw++mMOz48/ISeNu3Iweh/JaZU8ZLqHRrI= -golang.org/x/image v0.5.0/go.mod h1:FVC7BI/5Ym8R25iw5OLsgshdUBbT1h5jZTpA+mvAdZ4= +golang.org/x/image v0.10.0 h1:gXjUUtwtx5yOE0VKWq1CH4IJAClq4UGgUA3i+rpON9M= +golang.org/x/image v0.10.0/go.mod h1:jtrku+n79PfroUbvDdeUWMAI+heR786BofxrbiSF+J0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -656,6 +656,7 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU= golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -683,6 +684,7 @@ golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1 golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -738,17 +740,20 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= -golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/tools/pd-api-bench/go.mod b/tools/pd-api-bench/go.mod index 5f834e0f565..5bbda93584d 100644 --- a/tools/pd-api-bench/go.mod +++ b/tools/pd-api-bench/go.mod @@ -98,10 +98,10 @@ require ( go.uber.org/goleak v1.1.12 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/arch v0.3.0 // indirect - golang.org/x/crypto v0.15.0 // indirect + golang.org/x/crypto v0.17.0 // indirect golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4 // indirect golang.org/x/net v0.18.0 // indirect - golang.org/x/sys v0.14.0 // indirect + golang.org/x/sys v0.15.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.1.0 // indirect google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect diff --git a/tools/pd-api-bench/go.sum b/tools/pd-api-bench/go.sum index 605b8cf3b5f..76ae3227d28 100644 --- a/tools/pd-api-bench/go.sum +++ b/tools/pd-api-bench/go.sum @@ -367,8 +367,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA= -golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g= +golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= +golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4 h1:QLureRX3moex6NVu/Lr4MGakp9FdA7sBHGBmvRW7NaM= golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= @@ -434,8 +434,8 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= -golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= From 07f7e78c54c78d61c1b7639859abaea871092b45 Mon Sep 17 00:00:00 2001 From: Hu# Date: Fri, 22 Dec 2023 17:01:55 +0800 Subject: [PATCH 122/137] tools/api: replace client to pd http client (#7574) ref tikv/pd#7300 change tools/api client to HTTP SDK Signed-off-by: husharp --- tools/pd-api-bench/README.md | 13 +- tools/pd-api-bench/cases/cases.go | 78 ++------- tools/pd-api-bench/cert_opt.sh | 0 tools/pd-api-bench/go.mod | 77 +-------- tools/pd-api-bench/go.sum | 262 +----------------------------- tools/pd-api-bench/main.go | 37 +---- 6 files changed, 21 insertions(+), 446 deletions(-) mode change 100644 => 100755 tools/pd-api-bench/cert_opt.sh diff --git a/tools/pd-api-bench/README.md b/tools/pd-api-bench/README.md index 0ab4ea6463b..49db200c0f4 100644 --- a/tools/pd-api-bench/README.md +++ b/tools/pd-api-bench/README.md @@ -63,18 +63,9 @@ The api bench cases we support are as follows: You can run shell as follows. ```shell -go run main.go -http-cases GetRegionStatus-1+1,GetMinResolvedTS-1+1 -client 1 -debug +go run main.go -http-cases GetRegionStatus-1+1,GetMinResolvedTS-1+1 -client 1 -debug true ``` -### HTTP params - -You can use the following command to set the params of HTTP request: -```shell -go run main.go -http-cases GetMinResolvedTS-1+1 -params 'scope=cluster' -client 1 -debug -``` -for more params, can use like `-params 'A=1&B=2&C=3'` - - ### TLS You can use the following command to generate a certificate for testing TLS: @@ -82,7 +73,7 @@ You can use the following command to generate a certificate for testing TLS: ```shell mkdir cert ./cert_opt.sh generate cert -go run main.go -http-cases GetRegionStatus-1+1,GetMinResolvedTS-1+1 -client 1 -debug -cacert ./cert/ca.pem -cert ./cert/pd-server.pem -key ./cert/pd-server-key.pem +go run main.go -http-cases GetRegionStatus-1+1,GetMinResolvedTS-1+1 -client 1 -debug true -cacert ./cert/ca.pem -cert ./cert/pd-server.pem -key ./cert/pd-server-key.pem ./cert_opt.sh cleanup cert rm -rf cert ``` diff --git a/tools/pd-api-bench/cases/cases.go b/tools/pd-api-bench/cases/cases.go index 2f93d2e9454..35b00d60c56 100644 --- a/tools/pd-api-bench/cases/cases.go +++ b/tools/pd-api-bench/cases/cases.go @@ -19,42 +19,26 @@ import ( "fmt" "log" "math/rand" - "net/http" - "net/url" pd "github.com/tikv/pd/client" - "github.com/tikv/pd/pkg/statistics" - "github.com/tikv/pd/pkg/utils/apiutil" - "github.com/tikv/pd/pkg/utils/typeutil" + pdHttp "github.com/tikv/pd/client/http" ) var ( - // PDAddress is the address of PD server. - PDAddress string // Debug is the flag to print the output of api response for debug. Debug bool -) -var ( totalRegion int totalStore int storesID []uint64 ) // InitCluster initializes the cluster. -func InitCluster(ctx context.Context, cli pd.Client, httpClit *http.Client) error { - req, _ := http.NewRequestWithContext(ctx, http.MethodGet, - PDAddress+"/pd/api/v1/stats/region?start_key=&end_key=&count", http.NoBody) - resp, err := httpClit.Do(req) - if err != nil { - return err - } - statsResp := &statistics.RegionStats{} - err = apiutil.ReadJSON(resp.Body, statsResp) +func InitCluster(ctx context.Context, cli pd.Client, httpCli pdHttp.Client) error { + statsResp, err := httpCli.GetRegionStatusByKeyRange(ctx, pdHttp.NewKeyRange([]byte(""), []byte("")), false) if err != nil { return err } - resp.Body.Close() totalRegion = statsResp.Count stores, err := cli.GetAllStores(ctx) @@ -122,8 +106,7 @@ var GRPCCaseMap = map[string]GRPCCase{ // HTTPCase is the interface for all HTTP cases. type HTTPCase interface { Case - Do(context.Context, *http.Client) error - Params(string) + Do(context.Context, pdHttp.Client) error } // HTTPCaseMap is the map for all HTTP cases. @@ -134,8 +117,6 @@ var HTTPCaseMap = map[string]HTTPCase{ type minResolvedTS struct { *baseCase - path string - params string } func newMinResolvedTS() *minResolvedTS { @@ -145,45 +126,23 @@ func newMinResolvedTS() *minResolvedTS { qps: 1000, burst: 1, }, - path: "/pd/api/v1/min-resolved-ts", } } -type minResolvedTSStruct struct { - IsRealTime bool `json:"is_real_time,omitempty"` - MinResolvedTS uint64 `json:"min_resolved_ts"` - PersistInterval typeutil.Duration `json:"persist_interval,omitempty"` - StoresMinResolvedTS map[uint64]uint64 `json:"stores_min_resolved_ts"` -} - -func (c *minResolvedTS) Do(ctx context.Context, cli *http.Client) error { - url := fmt.Sprintf("%s%s", PDAddress, c.path) - req, _ := http.NewRequestWithContext(ctx, http.MethodGet, url, http.NoBody) - res, err := cli.Do(req) - if err != nil { - return err - } - listResp := &minResolvedTSStruct{} - err = apiutil.ReadJSON(res.Body, listResp) +func (c *minResolvedTS) Do(ctx context.Context, cli pdHttp.Client) error { + minResolvedTS, storesMinResolvedTS, err := cli.GetMinResolvedTSByStoresIDs(ctx, storesID) if Debug { - log.Printf("Do %s: url: %s resp: %v err: %v", c.name, url, listResp, err) + log.Printf("Do %s: minResolvedTS: %d storesMinResolvedTS: %v err: %v", c.name, minResolvedTS, storesMinResolvedTS, err) } if err != nil { return err } - res.Body.Close() return nil } -func (c *minResolvedTS) Params(param string) { - c.params = param - c.path = fmt.Sprintf("%s?%s", c.path, c.params) -} - type regionsStats struct { *baseCase regionSample int - path string } func newRegionStats() *regionsStats { @@ -194,11 +153,10 @@ func newRegionStats() *regionsStats { burst: 1, }, regionSample: 1000, - path: "/pd/api/v1/stats/region", } } -func (c *regionsStats) Do(ctx context.Context, cli *http.Client) error { +func (c *regionsStats) Do(ctx context.Context, cli pdHttp.Client) error { upperBound := totalRegion / c.regionSample if upperBound < 1 { upperBound = 1 @@ -206,31 +164,17 @@ func (c *regionsStats) Do(ctx context.Context, cli *http.Client) error { random := rand.Intn(upperBound) startID := c.regionSample*random*4 + 1 endID := c.regionSample*(random+1)*4 + 1 - url := fmt.Sprintf("%s%s?start_key=%s&end_key=%s&%s", - PDAddress, - c.path, - url.QueryEscape(string(generateKeyForSimulator(startID, 56))), - url.QueryEscape(string(generateKeyForSimulator(endID, 56))), - "") - req, _ := http.NewRequestWithContext(ctx, http.MethodGet, url, http.NoBody) - res, err := cli.Do(req) - if err != nil { - return err - } - statsResp := &statistics.RegionStats{} - err = apiutil.ReadJSON(res.Body, statsResp) + regionStats, err := cli.GetRegionStatusByKeyRange(ctx, + pdHttp.NewKeyRange(generateKeyForSimulator(startID, 56), generateKeyForSimulator(endID, 56)), false) if Debug { - log.Printf("Do %s: url: %s resp: %v err: %v", c.name, url, statsResp, err) + log.Printf("Do %s: regionStats: %v err: %v", c.name, regionStats, err) } if err != nil { return err } - res.Body.Close() return nil } -func (c *regionsStats) Params(_ string) {} - type getRegion struct { *baseCase } diff --git a/tools/pd-api-bench/cert_opt.sh b/tools/pd-api-bench/cert_opt.sh old mode 100644 new mode 100755 diff --git a/tools/pd-api-bench/go.mod b/tools/pd-api-bench/go.mod index 5bbda93584d..cae22f335bf 100644 --- a/tools/pd-api-bench/go.mod +++ b/tools/pd-api-bench/go.mod @@ -3,113 +3,38 @@ module github.com/tools/pd-api-bench go 1.21 require ( - github.com/tikv/pd v0.0.0-00010101000000-000000000000 github.com/tikv/pd/client v0.0.0-20231101084237-a1a1eea8dafd go.uber.org/zap v1.24.0 google.golang.org/grpc v1.54.0 ) require ( - github.com/BurntSushi/toml v0.3.1 // indirect - github.com/aws/aws-sdk-go-v2 v1.17.7 // indirect - github.com/aws/aws-sdk-go-v2/config v1.18.19 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.13.18 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.1 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.31 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.25 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.3.32 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.25 // indirect - github.com/aws/aws-sdk-go-v2/service/kms v1.20.8 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.12.6 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.6 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.18.7 // indirect - github.com/aws/smithy-go v1.13.5 // indirect github.com/benbjohnson/clock v1.3.0 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/bytedance/sonic v1.9.1 // indirect - github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect - github.com/coreos/go-semver v0.3.0 // indirect - github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f // indirect - github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect - github.com/davecgh/go-spew v1.1.1 // indirect - github.com/docker/go-units v0.4.0 // indirect - github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4 // indirect - github.com/elliotchance/pie/v2 v2.1.0 // indirect - github.com/gabriel-vasile/mimetype v1.4.2 // indirect - github.com/gin-contrib/sse v0.1.0 // indirect - github.com/gin-gonic/gin v1.9.1 // indirect - github.com/go-playground/locales v0.14.1 // indirect - github.com/go-playground/universal-translator v0.18.1 // indirect - github.com/go-playground/validator/v10 v10.14.0 // indirect - github.com/goccy/go-json v0.10.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang-jwt/jwt v3.2.1+incompatible // indirect - github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef // indirect github.com/golang/protobuf v1.5.3 // indirect - github.com/golang/snappy v0.0.4 // indirect - github.com/google/btree v1.1.2 // indirect github.com/google/go-cmp v0.5.9 // indirect - github.com/google/uuid v1.3.0 // indirect - github.com/gorilla/mux v1.7.4 // indirect - github.com/gorilla/websocket v1.4.2 // indirect - github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 // indirect - github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect - github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect - github.com/jonboulle/clockwork v0.2.2 // indirect - github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/cpuid/v2 v2.2.4 // indirect - github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect - github.com/leodido/go-urn v1.2.4 // indirect - github.com/mattn/go-isatty v0.0.19 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect - github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.2 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect - github.com/pelletier/go-toml/v2 v2.0.8 // indirect - github.com/petermattis/goid v0.0.0-20211229010228-4d14c490ee36 // indirect - github.com/phf/go-queue v0.0.0-20170504031614-9abe38d0371d // indirect - github.com/pingcap/errcode v0.3.0 // indirect github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c // indirect github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 // indirect github.com/pingcap/kvproto v0.0.0-20231018065736-c0689aded40c // indirect github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_golang v1.11.1 // indirect github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/common v0.26.0 // indirect github.com/prometheus/procfs v0.6.0 // indirect - github.com/sasha-s/go-deadlock v0.2.0 // indirect - github.com/sirupsen/logrus v1.6.0 // indirect - github.com/smallnest/chanx v0.0.0-20221229104322-eb4c998d2072 // indirect - github.com/soheilhy/cmux v0.1.4 // indirect - github.com/spf13/pflag v1.0.5 // indirect github.com/stretchr/testify v1.8.3 // indirect - github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965 // indirect - github.com/tmc/grpc-websocket-proxy v0.0.0-20200427203606-3cfed13b9966 // indirect - github.com/twitchyliquid64/golang-asm v0.15.1 // indirect - github.com/ugorji/go/codec v1.2.11 // indirect - github.com/unrolled/render v1.0.1 // indirect - github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect - go.etcd.io/bbolt v1.3.6 // indirect - go.etcd.io/etcd v0.5.0-alpha.5.0.20220915004622-85b640cee793 // indirect go.uber.org/atomic v1.10.0 // indirect go.uber.org/goleak v1.1.12 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/arch v0.3.0 // indirect - golang.org/x/crypto v0.17.0 // indirect - golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4 // indirect golang.org/x/net v0.18.0 // indirect - golang.org/x/sys v0.15.0 // indirect + golang.org/x/sys v0.14.0 // indirect golang.org/x/text v0.14.0 // indirect - golang.org/x/time v0.1.0 // indirect google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect google.golang.org/protobuf v1.31.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect - sigs.k8s.io/yaml v1.2.0 // indirect ) replace google.golang.org/grpc v1.54.0 => google.golang.org/grpc v1.26.0 diff --git a/tools/pd-api-bench/go.sum b/tools/pd-api-bench/go.sum index 76ae3227d28..a59256c50d5 100644 --- a/tools/pd-api-bench/go.sum +++ b/tools/pd-api-bench/go.sum @@ -1,39 +1,11 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/aws/aws-sdk-go-v2 v1.17.7 h1:CLSjnhJSTSogvqUGhIC6LqFKATMRexcxLZ0i/Nzk9Eg= -github.com/aws/aws-sdk-go-v2 v1.17.7/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= -github.com/aws/aws-sdk-go-v2/config v1.18.19 h1:AqFK6zFNtq4i1EYu+eC7lcKHYnZagMn6SW171la0bGw= -github.com/aws/aws-sdk-go-v2/config v1.18.19/go.mod h1:XvTmGMY8d52ougvakOv1RpiTLPz9dlG/OQHsKU/cMmY= -github.com/aws/aws-sdk-go-v2/credentials v1.13.18 h1:EQMdtHwz0ILTW1hoP+EwuWhwCG1hD6l3+RWFQABET4c= -github.com/aws/aws-sdk-go-v2/credentials v1.13.18/go.mod h1:vnwlwjIe+3XJPBYKu1et30ZPABG3VaXJYr8ryohpIyM= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.1 h1:gt57MN3liKiyGopcqgNzJb2+d9MJaKT/q1OksHNXVE4= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.1/go.mod h1:lfUx8puBRdM5lVVMQlwt2v+ofiG/X6Ms+dy0UkG/kXw= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.31 h1:sJLYcS+eZn5EeNINGHSCRAwUJMFVqklwkH36Vbyai7M= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.31/go.mod h1:QT0BqUvX1Bh2ABdTGnjqEjvjzrCfIniM9Sc8zn9Yndo= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.25 h1:1mnRASEKnkqsntcxHaysxwgVoUUp5dkiB+l3llKnqyg= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.25/go.mod h1:zBHOPwhBc3FlQjQJE/D3IfPWiWaQmT06Vq9aNukDo0k= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.32 h1:p5luUImdIqywn6JpQsW3tq5GNOxKmOnEpybzPx+d1lk= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.32/go.mod h1:XGhIBZDEgfqmFIugclZ6FU7v75nHhBDtzuB4xB/tEi4= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.25 h1:5LHn8JQ0qvjD9L9JhMtylnkcw7j05GDZqM9Oin6hpr0= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.25/go.mod h1:/95IA+0lMnzW6XzqYJRpjjsAbKEORVeO0anQqjd2CNU= -github.com/aws/aws-sdk-go-v2/service/kms v1.20.8 h1:R5f4VOFi3ScTe7TtePyxLqEhNqTJIAxL57MzrXFNs6I= -github.com/aws/aws-sdk-go-v2/service/kms v1.20.8/go.mod h1:OtP3pBOgmJM+acQyQcQXtQHets3yJoVuanCx2T5M7v4= -github.com/aws/aws-sdk-go-v2/service/sso v1.12.6 h1:5V7DWLBd7wTELVz5bPpwzYy/sikk0gsgZfj40X+l5OI= -github.com/aws/aws-sdk-go-v2/service/sso v1.12.6/go.mod h1:Y1VOmit/Fn6Tz1uFAeCO6Q7M2fmfXSCLeL5INVYsLuY= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.6 h1:B8cauxOH1W1v7rd8RdI/MWnoR4Ze0wIHWrb90qczxj4= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.6/go.mod h1:Lh/bc9XUf8CfOY6Jp5aIkQtN+j1mc+nExc+KXj9jx2s= -github.com/aws/aws-sdk-go-v2/service/sts v1.18.7 h1:bWNgNdRko2x6gqa0blfATqAZKZokPIeM1vfmQt2pnvM= -github.com/aws/aws-sdk-go-v2/service/sts v1.18.7/go.mod h1:JuTnSoeePXmMVe9G8NcjjwgOKEfZ4cOjMuT2IBT/2eI= -github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8= -github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= @@ -41,93 +13,35 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= -github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s= -github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 h1:BjkPE3785EwPhhyuFkbINB+2a1xATwk8SNDWnJiD41g= github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5/go.mod h1:jtAfVaU/2cu1+wdSRPWE2c1N2qeAA3K4RH9pYgqwets= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= -github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams= -github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa h1:OaNxuTZr7kxeODyLWsRMC+OD03aFUH+mW6r2d+MWa5Y= -github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= -github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f h1:JOrtw2xFKzlg+cbHpyrpLDmnN1HqhBfnX7WDiW7eG2c= -github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= -github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= -github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4 h1:qk/FSDDxo05wdJH28W+p5yivv7LuLYLRXPPD8KQCtZs= -github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385 h1:clC1lXBpe2kTj2VHdaIu9ajZQe4kcEY9j0NsnDDBZ3o= -github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= -github.com/elliotchance/pie/v2 v2.1.0 h1:KEVAAzxYxTyFs4hvebFZVzBdEo3YeMzl2HYDWn+P3F4= -github.com/elliotchance/pie/v2 v2.1.0/go.mod h1:18t0dgGFH006g4eVdDtWfgFZPQEgl10IoEO8YWEq3Og= -github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= -github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= -github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= -github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg= -github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= -github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= -github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= -github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= -github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= -github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= -github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= -github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= -github.com/go-playground/validator/v10 v10.14.0 h1:vgvQWe3XCz3gIeFDm/HnTIbj6UGmg/+t63MyGU2n5js= -github.com/go-playground/validator/v10 v10.14.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= -github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang-jwt/jwt v3.2.1+incompatible h1:73Z+4BJcrTC+KczS6WvTPvRGOp1WmfEP4Q1lOd9Z/+c= -github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE= github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ= -github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef h1:veQD95Isof8w9/WXiA+pa3tz3fJXkt5B7QaRBrM62gk= -github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -138,107 +52,39 @@ github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= -github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= -github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc= -github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= -github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= -github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 h1:z53tR0945TRRQO/fLEVPI6SMv7ZflF0TEaTAoU7tOzg= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= -github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= -github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= -github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= -github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= -github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ= -github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= -github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= -github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= -github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= -github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a h1:N9zuLhTvBSRt0gWSiJswwQ2HqDmtX/ZCDJURnKUt1Ik= -github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a/go.mod h1:JKx41uQRwqlTZabZc+kILPrO/3jlKnQ2Z8b7YiVw5cE= -github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= -github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= -github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= -github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.20.1 h1:PA/3qinGoukvymdIDV8pii6tiZgC8kbmJO6Z5+b002Q= -github.com/onsi/gomega v1.20.1/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeREyVo= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= -github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= -github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4= -github.com/petermattis/goid v0.0.0-20211229010228-4d14c490ee36 h1:64bxqeTEN0/xoEqhKGowgihNuzISS9rEG6YUMU4bzJo= -github.com/petermattis/goid v0.0.0-20211229010228-4d14c490ee36/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4= -github.com/phf/go-queue v0.0.0-20170504031614-9abe38d0371d h1:U+PMnTlV2tu7RuMK5etusZG3Cf+rpow5hqQByeCzJ2g= -github.com/phf/go-queue v0.0.0-20170504031614-9abe38d0371d/go.mod h1:lXfE4PvvTW5xOjO6Mba8zDPyw8M93B6AQ7frTGnMlA8= -github.com/pingcap/errcode v0.3.0 h1:IF6LC/4+b1KNwrMlr2rBTUrojFPMexXBcDWZSpNwxjg= -github.com/pingcap/errcode v0.3.0/go.mod h1:4b2X8xSqxIroj/IZ9MX/VGZhAwc11wB9wRIzHvz6SeM= github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c h1:xpW9bvK+HuuTmyFqUwr+jcCvpVkK7sumiz+ko5H9eq4= @@ -249,16 +95,12 @@ github.com/pingcap/kvproto v0.0.0-20231018065736-c0689aded40c h1:5cpE29kMHjc8fv+ github.com/pingcap/kvproto v0.0.0-20231018065736-c0689aded40c/go.mod h1:r0q/CFcwvyeRhKtoqzmWMBebrtpIziQQ9vR+JKh1knc= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8IDP+SZrdhV1Kibl9KrHxJ9eciw= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= -github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 h1:QV6jqlfOkh8hqvEAgwBZa+4bSgO0EeKC7s5c6Luam2I= -github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21/go.mod h1:QYnjfA95ZaMefyl1NO8oPtKeb8pYUdnDVhQgf+qdpjM= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b h1:0LFwY6Q3gMACTjAbMZBjXAqTOzOwFaj2Ld6cjeQ7Rig= -github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= @@ -278,70 +120,21 @@ github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsT github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= -github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= -github.com/sasha-s/go-deadlock v0.2.0 h1:lMqc+fUb7RrFS3gQLtoQsJ7/6TV/pAIFvBsqX73DK8Y= -github.com/sasha-s/go-deadlock v0.2.0/go.mod h1:StQn567HiB1fF2yJ44N9au7wOhrPS3iZqiDbRupzT10= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= -github.com/shirou/gopsutil/v3 v3.23.3 h1:Syt5vVZXUDXPEXpIBt5ziWsJ4LdSAAxF4l/xZeQgSEE= -github.com/shirou/gopsutil/v3 v3.23.3/go.mod h1:lSBNN6t3+D6W5e5nXTxc8KIMMVxAcS+6IJlffjRRlMU= -github.com/shoenig/go-m1cpu v0.1.5 h1:LF57Z/Fpb/WdGLjt2HZilNnmZOxg/q2bSKTQhgbrLrQ= -github.com/shoenig/go-m1cpu v0.1.5/go.mod h1:Wwvst4LR89UxjeFtLRMrpgRiyY4xPsejnVZym39dbAQ= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/smallnest/chanx v0.0.0-20221229104322-eb4c998d2072 h1:Txo4SXVJq/OgEjwgkWoxkMoTjGlcrgsQE/XSghjmu0w= -github.com/smallnest/chanx v0.0.0-20221229104322-eb4c998d2072/go.mod h1:+4nWMF0+CqEcU74SnX2NxaGqZ8zX4pcQ8Jcs77DbX5A= -github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E= -github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= -github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/cobra v1.0.0 h1:6m/oheQuQ13N9ks4hubMG6BnvwOeaJrqSPLahSnczz8= -github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= -github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965 h1:1oFLiOyVl+W7bnBzGhf7BbIv9loSFQcieWWYIjLqcAw= -github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965/go.mod h1:9OrXJhf154huy1nPWmuSrkgjPUtUNhA+Zmy+6AESzuA= -github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM= -github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI= -github.com/tklauser/numcpus v0.6.0 h1:kebhY2Qt+3U6RNK7UqpYNA+tJ23IBEGKkB7JQBfDYms= -github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ5UVIcaL4= -github.com/tmc/grpc-websocket-proxy v0.0.0-20200427203606-3cfed13b9966 h1:j6JEOq5QWFker+d7mFQYOhjTZonQ7YkLTHm56dbn+yM= -github.com/tmc/grpc-websocket-proxy v0.0.0-20200427203606-3cfed13b9966/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= -github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= -github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= -github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= -github.com/unrolled/render v1.0.1 h1:VDDnQQVfBMsOsp3VaCJszSO0nkBIVEYoPWeRThk9spY= -github.com/unrolled/render v1.0.1/go.mod h1:gN9T0NhL4Bfbwu8ann7Ry/TGHYfosul+J0obPf6NBdM= -github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= -github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= -github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= -github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= -go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= -go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= -go.etcd.io/etcd v0.5.0-alpha.5.0.20220915004622-85b640cee793 h1:fqmtdYQlwZ/vKWSz5amW+a4cnjg23ojz5iL7rjf08Wg= -go.etcd.io/etcd v0.5.0-alpha.5.0.20220915004622-85b640cee793/go.mod h1:eBhtbxXP1qpW0F6+WxoJ64DM1Mrfx46PHtVxEdkLe0I= -go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= @@ -350,41 +143,28 @@ go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0 go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= -go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= -golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= -golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k= -golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= -golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4 h1:QLureRX3moex6NVu/Lr4MGakp9FdA7sBHGBmvRW7NaM= -golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -393,15 +173,12 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -412,8 +189,6 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -422,32 +197,20 @@ golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= +golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.1.0 h1:xYY+Bajn2a7VBmTM5GikTmnK8ZuX8YgnQCqZpbBNtmA= -golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -459,8 +222,6 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= -golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -469,16 +230,12 @@ google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9Ywl google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0 h1:2dTRdpdFEEhJYQD8EMLB61nnrzSCTbG38PhqdhvOltg= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -492,34 +249,19 @@ google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqw gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= -gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= -gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= -gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= -sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= -sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= -sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/tools/pd-api-bench/main.go b/tools/pd-api-bench/main.go index 722c7eb08fa..4a3e15b4cea 100644 --- a/tools/pd-api-bench/main.go +++ b/tools/pd-api-bench/main.go @@ -18,9 +18,7 @@ import ( "context" "crypto/tls" "flag" - "fmt" "log" - "net/http" "os" "os/signal" "strconv" @@ -29,6 +27,7 @@ import ( "time" pd "github.com/tikv/pd/client" + pdHttp "github.com/tikv/pd/client/http" "github.com/tikv/pd/client/tlsutil" "github.com/tools/pd-api-bench/cases" "go.uber.org/zap" @@ -48,9 +47,6 @@ var ( qps = flag.Int64("qps", 1000, "qps") burst = flag.Int64("burst", 1, "burst") - // http params - httpParams = flag.String("params", "", "http params") - // tls caPath = flag.String("cacert", "", "path of file that contains list of trusted SSL CAs") certPath = flag.String("cert", "", "path of file that contains X509 certificate in PEM format") @@ -76,12 +72,6 @@ func main() { cancel() }() - addr := trimHTTPPrefix(*pdAddr) - protocol := "http" - if len(*caPath) != 0 { - protocol = "https" - } - cases.PDAddress = fmt.Sprintf("%s://%s", protocol, addr) cases.Debug = *debugFlag hcases := make([]cases.HTTPCase, 0) @@ -158,9 +148,9 @@ func main() { for i := 0; i < *client; i++ { pdClis[i] = newPDClient(ctx) } - httpClis := make([]*http.Client, *client) + httpClis := make([]pdHttp.Client, *client) for i := 0; i < *client; i++ { - httpClis[i] = newHTTPClient() + httpClis[i] = pdHttp.NewClient("tools-api-bench", []string{*pdAddr}, pdHttp.WithTLSConfig(loadTLSConfig())) } err = cases.InitCluster(ctx, pdClis[0], httpClis[0]) if err != nil { @@ -214,16 +204,13 @@ func handleGRPCCase(ctx context.Context, gcase cases.GRPCCase, clients []pd.Clie } } -func handleHTTPCase(ctx context.Context, hcase cases.HTTPCase, httpClis []*http.Client) { +func handleHTTPCase(ctx context.Context, hcase cases.HTTPCase, httpClis []pdHttp.Client) { qps := hcase.GetQPS() burst := hcase.GetBurst() tt := time.Duration(base/qps*burst*int64(*client)) * time.Microsecond log.Printf("begin to run http case %s, with qps = %d and burst = %d, interval is %v", hcase.Name(), qps, burst, tt) - if *httpParams != "" { - hcase.Params(*httpParams) - } for _, hCli := range httpClis { - go func(hCli *http.Client) { + go func(hCli pdHttp.Client) { var ticker = time.NewTicker(tt) defer ticker.Stop() for { @@ -248,20 +235,6 @@ func exit(code int) { os.Exit(code) } -// newHTTPClient returns an HTTP(s) client. -func newHTTPClient() *http.Client { - // defaultTimeout for non-context requests. - const defaultTimeout = 30 * time.Second - cli := &http.Client{Timeout: defaultTimeout} - tlsConf := loadTLSConfig() - if tlsConf != nil { - transport := http.DefaultTransport.(*http.Transport).Clone() - transport.TLSClientConfig = tlsConf - cli.Transport = transport - } - return cli -} - func trimHTTPPrefix(str string) string { str = strings.TrimPrefix(str, "http://") str = strings.TrimPrefix(str, "https://") From b36b725c4b977a4345bd1c167e7ea6deaa730daf Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Fri, 22 Dec 2023 17:16:56 +0800 Subject: [PATCH 123/137] rule: split txn to multi batch (#7600) close tikv/pd#7599 Signed-off-by: lhy1024 Co-authored-by: Ryan Leung Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- pkg/keyspace/keyspace.go | 17 +- pkg/keyspace/keyspace_test.go | 19 +- pkg/keyspace/tso_keyspace_group.go | 4 +- pkg/keyspace/tso_keyspace_group_test.go | 3 +- pkg/schedule/placement/rule.go | 9 + pkg/schedule/placement/rule_manager.go | 166 ++++++++++-------- pkg/storage/endpoint/rule.go | 19 +- pkg/storage/endpoint/util.go | 6 +- pkg/utils/etcdutil/etcdutil.go | 10 +- .../mcs/tso/keyspace_group_manager_test.go | 6 +- tests/server/api/rule_test.go | 82 ++++++--- 11 files changed, 208 insertions(+), 133 deletions(-) diff --git a/pkg/keyspace/keyspace.go b/pkg/keyspace/keyspace.go index 1607676a37b..d84b3698f69 100644 --- a/pkg/keyspace/keyspace.go +++ b/pkg/keyspace/keyspace.go @@ -31,6 +31,7 @@ import ( "github.com/tikv/pd/pkg/slice" "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/storage/kv" + "github.com/tikv/pd/pkg/utils/etcdutil" "github.com/tikv/pd/pkg/utils/syncutil" "go.uber.org/zap" ) @@ -51,10 +52,6 @@ const ( // Note: Config[TSOKeyspaceGroupIDKey] is only used to judge whether there is keyspace group id. // It will not update the keyspace group id when merging or splitting. TSOKeyspaceGroupIDKey = "tso_keyspace_group_id" - // MaxEtcdTxnOps is the max value of operations in an etcd txn. The default limit of etcd txn op is 128. - // We use 120 here to leave some space for other operations. - // See: https://github.com/etcd-io/etcd/blob/d3e43d4de6f6d9575b489dd7850a85e37e0f6b6c/server/embed/config.go#L61 - MaxEtcdTxnOps = 120 ) // Config is the interface for keyspace config. @@ -710,7 +707,7 @@ func (manager *Manager) PatrolKeyspaceAssignment(startKeyspaceID, endKeyspaceID zap.Duration("cost", time.Since(start)), zap.Uint64("patrolled-keyspace-count", patrolledKeyspaceCount), zap.Uint64("assigned-keyspace-count", assignedKeyspaceCount), - zap.Int("batch-size", MaxEtcdTxnOps), + zap.Int("batch-size", etcdutil.MaxEtcdTxnOps), zap.Uint32("start-keyspace-id", startKeyspaceID), zap.Uint32("end-keyspace-id", endKeyspaceID), zap.Uint32("current-start-id", currentStartID), @@ -734,7 +731,7 @@ func (manager *Manager) PatrolKeyspaceAssignment(startKeyspaceID, endKeyspaceID if defaultKeyspaceGroup.IsMerging() { return ErrKeyspaceGroupInMerging(utils.DefaultKeyspaceGroupID) } - keyspaces, err := manager.store.LoadRangeKeyspace(txn, manager.nextPatrolStartID, MaxEtcdTxnOps) + keyspaces, err := manager.store.LoadRangeKeyspace(txn, manager.nextPatrolStartID, etcdutil.MaxEtcdTxnOps) if err != nil { return err } @@ -744,9 +741,9 @@ func (manager *Manager) PatrolKeyspaceAssignment(startKeyspaceID, endKeyspaceID currentStartID = keyspaces[0].GetId() nextStartID = keyspaces[keyspaceNum-1].GetId() + 1 } - // If there are less than `MaxEtcdTxnOps` keyspaces or the next start ID reaches the end, + // If there are less than ` etcdutil.MaxEtcdTxnOps` keyspaces or the next start ID reaches the end, // there is no need to patrol again. - moreToPatrol = keyspaceNum == MaxEtcdTxnOps + moreToPatrol = keyspaceNum == etcdutil.MaxEtcdTxnOps var ( assigned = false keyspaceIDsToUnlock = make([]uint32, 0, keyspaceNum) @@ -785,7 +782,7 @@ func (manager *Manager) PatrolKeyspaceAssignment(startKeyspaceID, endKeyspaceID err = manager.store.SaveKeyspaceMeta(txn, ks) if err != nil { log.Error("[keyspace] failed to save keyspace meta during patrol", - zap.Int("batch-size", MaxEtcdTxnOps), + zap.Int("batch-size", etcdutil.MaxEtcdTxnOps), zap.Uint32("start-keyspace-id", startKeyspaceID), zap.Uint32("end-keyspace-id", endKeyspaceID), zap.Uint32("current-start-id", currentStartID), @@ -799,7 +796,7 @@ func (manager *Manager) PatrolKeyspaceAssignment(startKeyspaceID, endKeyspaceID err = manager.kgm.store.SaveKeyspaceGroup(txn, defaultKeyspaceGroup) if err != nil { log.Error("[keyspace] failed to save default keyspace group meta during patrol", - zap.Int("batch-size", MaxEtcdTxnOps), + zap.Int("batch-size", etcdutil.MaxEtcdTxnOps), zap.Uint32("start-keyspace-id", startKeyspaceID), zap.Uint32("end-keyspace-id", endKeyspaceID), zap.Uint32("current-start-id", currentStartID), diff --git a/pkg/keyspace/keyspace_test.go b/pkg/keyspace/keyspace_test.go index 552adc8d83e..4bd8dfd5474 100644 --- a/pkg/keyspace/keyspace_test.go +++ b/pkg/keyspace/keyspace_test.go @@ -31,6 +31,7 @@ import ( "github.com/tikv/pd/pkg/mock/mockid" "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/storage/kv" + "github.com/tikv/pd/pkg/utils/etcdutil" "github.com/tikv/pd/pkg/utils/typeutil" ) @@ -408,7 +409,7 @@ func (suite *keyspaceTestSuite) TestPatrolKeyspaceAssignment() { func (suite *keyspaceTestSuite) TestPatrolKeyspaceAssignmentInBatch() { re := suite.Require() // Create some keyspaces without any keyspace group. - for i := 1; i < MaxEtcdTxnOps*2+1; i++ { + for i := 1; i < etcdutil.MaxEtcdTxnOps*2+1; i++ { now := time.Now().Unix() err := suite.manager.saveNewKeyspace(&keyspacepb.KeyspaceMeta{ Id: uint32(i), @@ -423,7 +424,7 @@ func (suite *keyspaceTestSuite) TestPatrolKeyspaceAssignmentInBatch() { defaultKeyspaceGroup, err := suite.manager.kgm.GetKeyspaceGroupByID(utils.DefaultKeyspaceGroupID) re.NoError(err) re.NotNil(defaultKeyspaceGroup) - for i := 1; i < MaxEtcdTxnOps*2+1; i++ { + for i := 1; i < etcdutil.MaxEtcdTxnOps*2+1; i++ { re.NotContains(defaultKeyspaceGroup.Keyspaces, uint32(i)) } // Patrol the keyspace assignment. @@ -433,7 +434,7 @@ func (suite *keyspaceTestSuite) TestPatrolKeyspaceAssignmentInBatch() { defaultKeyspaceGroup, err = suite.manager.kgm.GetKeyspaceGroupByID(utils.DefaultKeyspaceGroupID) re.NoError(err) re.NotNil(defaultKeyspaceGroup) - for i := 1; i < MaxEtcdTxnOps*2+1; i++ { + for i := 1; i < etcdutil.MaxEtcdTxnOps*2+1; i++ { re.Contains(defaultKeyspaceGroup.Keyspaces, uint32(i)) } } @@ -441,7 +442,7 @@ func (suite *keyspaceTestSuite) TestPatrolKeyspaceAssignmentInBatch() { func (suite *keyspaceTestSuite) TestPatrolKeyspaceAssignmentWithRange() { re := suite.Require() // Create some keyspaces without any keyspace group. - for i := 1; i < MaxEtcdTxnOps*2+1; i++ { + for i := 1; i < etcdutil.MaxEtcdTxnOps*2+1; i++ { now := time.Now().Unix() err := suite.manager.saveNewKeyspace(&keyspacepb.KeyspaceMeta{ Id: uint32(i), @@ -456,14 +457,14 @@ func (suite *keyspaceTestSuite) TestPatrolKeyspaceAssignmentWithRange() { defaultKeyspaceGroup, err := suite.manager.kgm.GetKeyspaceGroupByID(utils.DefaultKeyspaceGroupID) re.NoError(err) re.NotNil(defaultKeyspaceGroup) - for i := 1; i < MaxEtcdTxnOps*2+1; i++ { + for i := 1; i < etcdutil.MaxEtcdTxnOps*2+1; i++ { re.NotContains(defaultKeyspaceGroup.Keyspaces, uint32(i)) } - // Patrol the keyspace assignment with range [MaxEtcdTxnOps/2, MaxEtcdTxnOps/2+MaxEtcdTxnOps+1] + // Patrol the keyspace assignment with range [ etcdutil.MaxEtcdTxnOps/2, etcdutil.MaxEtcdTxnOps/2+ etcdutil.MaxEtcdTxnOps+1] // to make sure the range crossing the boundary of etcd transaction operation limit. var ( - startKeyspaceID = uint32(MaxEtcdTxnOps / 2) - endKeyspaceID = startKeyspaceID + MaxEtcdTxnOps + 1 + startKeyspaceID = uint32(etcdutil.MaxEtcdTxnOps / 2) + endKeyspaceID = startKeyspaceID + etcdutil.MaxEtcdTxnOps + 1 ) err = suite.manager.PatrolKeyspaceAssignment(startKeyspaceID, endKeyspaceID) re.NoError(err) @@ -471,7 +472,7 @@ func (suite *keyspaceTestSuite) TestPatrolKeyspaceAssignmentWithRange() { defaultKeyspaceGroup, err = suite.manager.kgm.GetKeyspaceGroupByID(utils.DefaultKeyspaceGroupID) re.NoError(err) re.NotNil(defaultKeyspaceGroup) - for i := 1; i < MaxEtcdTxnOps*2+1; i++ { + for i := 1; i < etcdutil.MaxEtcdTxnOps*2+1; i++ { keyspaceID := uint32(i) if keyspaceID >= startKeyspaceID && keyspaceID <= endKeyspaceID { re.Contains(defaultKeyspaceGroup.Keyspaces, keyspaceID) diff --git a/pkg/keyspace/tso_keyspace_group.go b/pkg/keyspace/tso_keyspace_group.go index 51a53f75cc2..55c9adf66d9 100644 --- a/pkg/keyspace/tso_keyspace_group.go +++ b/pkg/keyspace/tso_keyspace_group.go @@ -915,7 +915,7 @@ func (m *GroupManager) MergeKeyspaceGroups(mergeTargetID uint32, mergeList []uin // - Load and delete the keyspace groups in the merge list. // - Load and update the target keyspace group. // So we pre-check the number of operations to avoid exceeding the maximum number of etcd transaction. - if (mergeListNum+1)*2 > MaxEtcdTxnOps { + if (mergeListNum+1)*2 > etcdutil.MaxEtcdTxnOps { return ErrExceedMaxEtcdTxnOps } if slice.Contains(mergeList, utils.DefaultKeyspaceGroupID) { @@ -1062,7 +1062,7 @@ func (m *GroupManager) MergeAllIntoDefaultKeyspaceGroup() error { continue } var ( - maxBatchSize = MaxEtcdTxnOps/2 - 1 + maxBatchSize = etcdutil.MaxEtcdTxnOps/2 - 1 groupsToMerge = make([]uint32, 0, maxBatchSize) ) for idx, group := range groups.GetAll() { diff --git a/pkg/keyspace/tso_keyspace_group_test.go b/pkg/keyspace/tso_keyspace_group_test.go index 2dec780c3c8..b5df85c56a8 100644 --- a/pkg/keyspace/tso_keyspace_group_test.go +++ b/pkg/keyspace/tso_keyspace_group_test.go @@ -28,6 +28,7 @@ import ( "github.com/tikv/pd/pkg/mock/mockid" "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/storage/kv" + "github.com/tikv/pd/pkg/utils/etcdutil" ) type keyspaceGroupTestSuite struct { @@ -449,7 +450,7 @@ func (suite *keyspaceGroupTestSuite) TestKeyspaceGroupMerge() { err = suite.kgm.MergeKeyspaceGroups(4, []uint32{5}) re.ErrorContains(err, ErrKeyspaceGroupNotExists(5).Error()) // merge with the number of keyspace groups exceeds the limit - err = suite.kgm.MergeKeyspaceGroups(1, make([]uint32, MaxEtcdTxnOps/2)) + err = suite.kgm.MergeKeyspaceGroups(1, make([]uint32, etcdutil.MaxEtcdTxnOps/2)) re.ErrorIs(err, ErrExceedMaxEtcdTxnOps) // merge the default keyspace group err = suite.kgm.MergeKeyspaceGroups(1, []uint32{utils.DefaultKeyspaceGroupID}) diff --git a/pkg/schedule/placement/rule.go b/pkg/schedule/placement/rule.go index 7e57866512d..75ccd509ee8 100644 --- a/pkg/schedule/placement/rule.go +++ b/pkg/schedule/placement/rule.go @@ -139,6 +139,15 @@ func (g *RuleGroup) String() string { return string(b) } +// Clone returns a copy of RuleGroup. +func (g *RuleGroup) Clone() *RuleGroup { + return &RuleGroup{ + ID: g.ID, + Index: g.Index, + Override: g.Override, + } +} + // Rules are ordered by (GroupID, Index, ID). func compareRule(a, b *Rule) int { switch { diff --git a/pkg/schedule/placement/rule_manager.go b/pkg/schedule/placement/rule_manager.go index ea85911462b..0a66d82e865 100644 --- a/pkg/schedule/placement/rule_manager.go +++ b/pkg/schedule/placement/rule_manager.go @@ -34,6 +34,7 @@ import ( "github.com/tikv/pd/pkg/slice" "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/storage/kv" + "github.com/tikv/pd/pkg/utils/etcdutil" "github.com/tikv/pd/pkg/utils/syncutil" "go.uber.org/zap" "golang.org/x/exp/slices" @@ -157,61 +158,61 @@ func (m *RuleManager) loadRules() error { toSave []*Rule toDelete []string ) - return m.storage.RunInTxn(m.ctx, func(txn kv.Txn) (err error) { - err = m.storage.LoadRules(txn, func(k, v string) { - r, err := NewRuleFromJSON([]byte(v)) - if err != nil { - log.Error("failed to unmarshal rule value", zap.String("rule-key", k), zap.String("rule-value", v), errs.ZapError(errs.ErrLoadRule)) - toDelete = append(toDelete, k) - return - } - err = m.AdjustRule(r, "") - if err != nil { - log.Error("rule is in bad format", zap.String("rule-key", k), zap.String("rule-value", v), errs.ZapError(errs.ErrLoadRule, err)) - toDelete = append(toDelete, k) - return - } - _, ok := m.ruleConfig.rules[r.Key()] - if ok { - log.Error("duplicated rule key", zap.String("rule-key", k), zap.String("rule-value", v), errs.ZapError(errs.ErrLoadRule)) - toDelete = append(toDelete, k) - return - } - if k != r.StoreKey() { - log.Error("mismatch data key, need to restore", zap.String("rule-key", k), zap.String("rule-value", v), errs.ZapError(errs.ErrLoadRule)) - toDelete = append(toDelete, k) - toSave = append(toSave, r) - } - m.ruleConfig.rules[r.Key()] = r - }) + // load rules from storage + err := m.storage.LoadRules(func(k, v string) { + r, err := NewRuleFromJSON([]byte(v)) if err != nil { - return err + log.Error("failed to unmarshal rule value", zap.String("rule-key", k), zap.String("rule-value", v), errs.ZapError(errs.ErrLoadRule)) + toDelete = append(toDelete, k) + return } - - for _, s := range toSave { - if err = m.storage.SaveRule(txn, s.StoreKey(), s); err != nil { - return err - } + err = m.AdjustRule(r, "") + if err != nil { + log.Error("rule is in bad format", zap.String("rule-key", k), zap.String("rule-value", v), errs.ZapError(errs.ErrLoadRule, err)) + toDelete = append(toDelete, k) + return } - for _, d := range toDelete { - if err = m.storage.DeleteRule(txn, d); err != nil { - return err - } + _, ok := m.ruleConfig.rules[r.Key()] + if ok { + log.Error("duplicated rule key", zap.String("rule-key", k), zap.String("rule-value", v), errs.ZapError(errs.ErrLoadRule)) + toDelete = append(toDelete, k) + return } - return nil + if k != r.StoreKey() { + log.Error("mismatch data key, need to restore", zap.String("rule-key", k), zap.String("rule-value", v), errs.ZapError(errs.ErrLoadRule)) + toDelete = append(toDelete, k) + toSave = append(toSave, r) + } + m.ruleConfig.rules[r.Key()] = r }) + if err != nil { + return err + } + // save the rules with mismatch data key or bad format + var batch []func(kv.Txn) error + for _, s := range toSave { + localRule := s + batch = append(batch, func(txn kv.Txn) error { + return m.storage.SaveRule(txn, localRule.StoreKey(), localRule) + }) + } + for _, d := range toDelete { + localKey := d + batch = append(batch, func(txn kv.Txn) error { + return m.storage.DeleteRule(txn, localKey) + }) + } + return m.runBatchOpInTxn(batch) } func (m *RuleManager) loadGroups() error { - return m.storage.RunInTxn(m.ctx, func(txn kv.Txn) (err error) { - return m.storage.LoadRuleGroups(txn, func(k, v string) { - g, err := NewRuleGroupFromJSON([]byte(v)) - if err != nil { - log.Error("failed to unmarshal rule group", zap.String("group-id", k), errs.ZapError(errs.ErrLoadRuleGroup, err)) - return - } - m.ruleConfig.groups[g.ID] = g - }) + return m.storage.LoadRuleGroups(func(k, v string) { + g, err := NewRuleGroupFromJSON([]byte(v)) + if err != nil { + log.Error("failed to unmarshal rule group", zap.String("group-id", k), errs.ZapError(errs.ErrLoadRuleGroup, err)) + return + } + m.ruleConfig.groups[g.ID] = g }) } @@ -492,30 +493,35 @@ func (m *RuleManager) TryCommitPatch(patch *RuleConfigPatch) error { } func (m *RuleManager) savePatch(p *ruleConfig) error { - return m.storage.RunInTxn(m.ctx, func(txn kv.Txn) (err error) { - for key, r := range p.rules { - if r == nil { - r = &Rule{GroupID: key[0], ID: key[1]} - err = m.storage.DeleteRule(txn, r.StoreKey()) - } else { - err = m.storage.SaveRule(txn, r.StoreKey(), r) - } - if err != nil { - return err - } + var batch []func(kv.Txn) error + // add rules to batch + for key, r := range p.rules { + localKey, localRule := key, r + if r == nil { + rule := &Rule{GroupID: localKey[0], ID: localKey[1]} + batch = append(batch, func(txn kv.Txn) error { + return m.storage.DeleteRule(txn, rule.StoreKey()) + }) + } else { + batch = append(batch, func(txn kv.Txn) error { + return m.storage.SaveRule(txn, localRule.StoreKey(), localRule) + }) } - for id, g := range p.groups { - if g.isDefault() { - err = m.storage.DeleteRuleGroup(txn, id) - } else { - err = m.storage.SaveRuleGroup(txn, id, g) - } - if err != nil { - return err - } + } + // add groups to batch + for id, g := range p.groups { + localID, localGroup := id, g + if g.isDefault() { + batch = append(batch, func(txn kv.Txn) error { + return m.storage.DeleteRuleGroup(txn, localID) + }) + } else { + batch = append(batch, func(txn kv.Txn) error { + return m.storage.SaveRuleGroup(txn, localID, localGroup) + }) } - return nil - }) + } + return m.runBatchOpInTxn(batch) } // SetRules inserts or updates lots of Rules at once. @@ -808,6 +814,28 @@ func (m *RuleManager) IsInitialized() bool { return m.initialized } +func (m *RuleManager) runBatchOpInTxn(batch []func(kv.Txn) error) error { + // execute batch in transaction with limited operations per transaction + for start := 0; start < len(batch); start += etcdutil.MaxEtcdTxnOps { + end := start + etcdutil.MaxEtcdTxnOps + if end > len(batch) { + end = len(batch) + } + err := m.storage.RunInTxn(m.ctx, func(txn kv.Txn) (err error) { + for _, op := range batch[start:end] { + if err = op(txn); err != nil { + return err + } + } + return nil + }) + if err != nil { + return err + } + } + return nil +} + // checkRule check the rule whether will have RuleFit after FitRegion // in order to reduce the calculation. func checkRule(rule *Rule, stores []*core.StoreInfo) bool { diff --git a/pkg/storage/endpoint/rule.go b/pkg/storage/endpoint/rule.go index b18360040ea..ad245f527bb 100644 --- a/pkg/storage/endpoint/rule.go +++ b/pkg/storage/endpoint/rule.go @@ -22,14 +22,17 @@ import ( // RuleStorage defines the storage operations on the rule. type RuleStorage interface { - LoadRules(txn kv.Txn, f func(k, v string)) error + // Load in txn is unnecessary and may cause txn too large. + // because scheduling server will load rules from etcd rather than watching. + LoadRule(ruleKey string) (string, error) + LoadRules(f func(k, v string)) error + LoadRuleGroups(f func(k, v string)) error + // We need to use txn to avoid concurrent modification. + // And it is helpful for the scheduling server to watch the rule. SaveRule(txn kv.Txn, ruleKey string, rule interface{}) error DeleteRule(txn kv.Txn, ruleKey string) error - LoadRuleGroups(txn kv.Txn, f func(k, v string)) error SaveRuleGroup(txn kv.Txn, groupID string, group interface{}) error DeleteRuleGroup(txn kv.Txn, groupID string) error - // LoadRule is used only in rule watcher. - LoadRule(ruleKey string) (string, error) LoadRegionRules(f func(k, v string)) error SaveRegionRule(ruleKey string, rule interface{}) error @@ -50,8 +53,8 @@ func (se *StorageEndpoint) DeleteRule(txn kv.Txn, ruleKey string) error { } // LoadRuleGroups loads all rule groups from storage. -func (se *StorageEndpoint) LoadRuleGroups(txn kv.Txn, f func(k, v string)) error { - return loadRangeByPrefixInTxn(txn, ruleGroupPath+"/", f) +func (se *StorageEndpoint) LoadRuleGroups(f func(k, v string)) error { + return se.loadRangeByPrefix(ruleGroupPath+"/", f) } // SaveRuleGroup stores a rule group config to storage. @@ -85,6 +88,6 @@ func (se *StorageEndpoint) LoadRule(ruleKey string) (string, error) { } // LoadRules loads placement rules from storage. -func (se *StorageEndpoint) LoadRules(txn kv.Txn, f func(k, v string)) error { - return loadRangeByPrefixInTxn(txn, rulesPath+"/", f) +func (se *StorageEndpoint) LoadRules(f func(k, v string)) error { + return se.loadRangeByPrefix(rulesPath+"/", f) } diff --git a/pkg/storage/endpoint/util.go b/pkg/storage/endpoint/util.go index 3058c059628..62b170a1a8e 100644 --- a/pkg/storage/endpoint/util.go +++ b/pkg/storage/endpoint/util.go @@ -58,14 +58,10 @@ func saveJSONInTxn(txn kv.Txn, key string, data interface{}) error { // loadRangeByPrefix iterates all key-value pairs in the storage that has the prefix. func (se *StorageEndpoint) loadRangeByPrefix(prefix string, f func(k, v string)) error { - return loadRangeByPrefixInTxn(se /* use the same interface */, prefix, f) -} - -func loadRangeByPrefixInTxn(txn kv.Txn, prefix string, f func(k, v string)) error { nextKey := prefix endKey := clientv3.GetPrefixRangeEnd(prefix) for { - keys, values, err := txn.LoadRange(nextKey, endKey, MinKVRangeLimit) + keys, values, err := se.LoadRange(nextKey, endKey, MinKVRangeLimit) if err != nil { return err } diff --git a/pkg/utils/etcdutil/etcdutil.go b/pkg/utils/etcdutil/etcdutil.go index f6beafee511..11c640fe4ef 100644 --- a/pkg/utils/etcdutil/etcdutil.go +++ b/pkg/utils/etcdutil/etcdutil.go @@ -64,6 +64,11 @@ const ( DefaultSlowRequestTime = time.Second healthyPath = "health" + + // MaxEtcdTxnOps is the max value of operations in an etcd txn. The default limit of etcd txn op is 128. + // We use 120 here to leave some space for other operations. + // See: https://github.com/etcd-io/etcd/blob/d3e43d4de6f6d9575b489dd7850a85e37e0f6b6c/server/embed/config.go#L61 + MaxEtcdTxnOps = 120 ) // CheckClusterID checks etcd cluster ID, returns an error if mismatch. @@ -776,7 +781,10 @@ func (lw *LoopWatcher) watch(ctx context.Context, revision int64) (nextRevision revision, err = lw.load(ctx) if err != nil { log.Warn("force load key failed in watch loop", - zap.String("name", lw.name), zap.String("key", lw.key), zap.Error(err)) + zap.String("name", lw.name), zap.String("key", lw.key), zap.Int64("revision", revision), zap.Error(err)) + } else { + log.Info("force load key successfully in watch loop", + zap.String("name", lw.name), zap.String("key", lw.key), zap.Int64("revision", revision)) } continue case <-ticker.C: diff --git a/tests/integrations/mcs/tso/keyspace_group_manager_test.go b/tests/integrations/mcs/tso/keyspace_group_manager_test.go index d1a4cf35db4..52248086249 100644 --- a/tests/integrations/mcs/tso/keyspace_group_manager_test.go +++ b/tests/integrations/mcs/tso/keyspace_group_manager_test.go @@ -30,11 +30,11 @@ import ( pd "github.com/tikv/pd/client" "github.com/tikv/pd/pkg/election" "github.com/tikv/pd/pkg/errs" - "github.com/tikv/pd/pkg/keyspace" mcsutils "github.com/tikv/pd/pkg/mcs/utils" "github.com/tikv/pd/pkg/member" "github.com/tikv/pd/pkg/storage/endpoint" tsopkg "github.com/tikv/pd/pkg/tso" + "github.com/tikv/pd/pkg/utils/etcdutil" "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/pkg/utils/tsoutil" "github.com/tikv/pd/server/apiv2/handlers" @@ -761,7 +761,7 @@ func (suite *tsoKeyspaceGroupManagerTestSuite) TestKeyspaceGroupMergeIntoDefault re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/keyspace/acceleratedAllocNodes", `return(true)`)) var ( - keyspaceGroupNum = keyspace.MaxEtcdTxnOps + keyspaceGroupNum = etcdutil.MaxEtcdTxnOps keyspaceGroups = make([]*endpoint.KeyspaceGroup, 0, keyspaceGroupNum) keyspaces = make([]uint32, 0, keyspaceGroupNum) ) @@ -772,7 +772,7 @@ func (suite *tsoKeyspaceGroupManagerTestSuite) TestKeyspaceGroupMergeIntoDefault Keyspaces: []uint32{uint32(i)}, }) keyspaces = append(keyspaces, uint32(i)) - if len(keyspaceGroups) < keyspace.MaxEtcdTxnOps/2 && i != keyspaceGroupNum { + if len(keyspaceGroups) < etcdutil.MaxEtcdTxnOps/2 && i != keyspaceGroupNum { continue } handlersutil.MustCreateKeyspaceGroup(re, suite.pdLeaderServer, &handlers.CreateKeyspaceGroupParams{ diff --git a/tests/server/api/rule_test.go b/tests/server/api/rule_test.go index 14bb23e383a..9819e821d29 100644 --- a/tests/server/api/rule_test.go +++ b/tests/server/api/rule_test.go @@ -32,6 +32,7 @@ import ( "github.com/tikv/pd/pkg/errs" "github.com/tikv/pd/pkg/schedule/labeler" "github.com/tikv/pd/pkg/schedule/placement" + "github.com/tikv/pd/pkg/utils/etcdutil" "github.com/tikv/pd/pkg/utils/syncutil" tu "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/server/config" @@ -504,8 +505,7 @@ func (suite *ruleTestSuite) checkGetAllByRegion(cluster *tests.TestCluster) { } func (suite *ruleTestSuite) TestGetAllByKey() { - // Fixme: after delete+set rule, the key range will be empty, so the test will fail in api mode. - suite.env.RunTestInPDMode(suite.checkGetAllByKey) + suite.env.RunTestInTwoModes(suite.checkGetAllByKey) } func (suite *ruleTestSuite) checkGetAllByKey(cluster *tests.TestCluster) { @@ -1035,7 +1035,6 @@ func (suite *ruleTestSuite) TestDeleteAndUpdate() { } func (suite *ruleTestSuite) checkDeleteAndUpdate(cluster *tests.TestCluster) { - re := suite.Require() leaderServer := cluster.GetLeaderServer() pdAddr := leaderServer.GetAddr() urlPrefix := fmt.Sprintf("%s%s/api/v1", pdAddr, apiPrefix) @@ -1108,28 +1107,7 @@ func (suite *ruleTestSuite) checkDeleteAndUpdate(cluster *tests.TestCluster) { } for _, bundle := range bundles { - data, err := json.Marshal(bundle) - re.NoError(err) - err = tu.CheckPostJSON(testDialClient, urlPrefix+"/config/placement-rule", data, tu.StatusOK(re)) - re.NoError(err) - - tu.Eventually(re, func() bool { - respBundle := make([]placement.GroupBundle, 0) - err = tu.CheckGetJSON(testDialClient, urlPrefix+"/config/placement-rule", nil, - tu.StatusOK(re), tu.ExtractJSON(re, &respBundle)) - re.NoError(err) - if len(respBundle) != len(bundle) { - return false - } - sort.Slice(respBundle, func(i, j int) bool { return respBundle[i].ID < respBundle[j].ID }) - sort.Slice(bundle, func(i, j int) bool { return bundle[i].ID < bundle[j].ID }) - for i := range respBundle { - if !suite.compareBundle(respBundle[i], bundle[i]) { - return false - } - } - return true - }) + suite.postAndCheckRuleBundle(urlPrefix, bundle) } } @@ -1222,6 +1200,34 @@ func (suite *ruleTestSuite) checkConcurrencyWith(cluster *tests.TestCluster, }) } +func (suite *ruleTestSuite) TestLargeRules() { + suite.env.RunTestInTwoModes(suite.checkLargeRules) +} + +func (suite *ruleTestSuite) checkLargeRules(cluster *tests.TestCluster) { + leaderServer := cluster.GetLeaderServer() + pdAddr := leaderServer.GetAddr() + urlPrefix := fmt.Sprintf("%s%s/api/v1", pdAddr, apiPrefix) + genBundlesWithRulesNum := func(num int) []placement.GroupBundle { + bundle := []placement.GroupBundle{ + { + ID: "1", + Index: 1, + Rules: make([]*placement.Rule, 0), + }, + } + for i := 0; i < num; i++ { + bundle[0].Rules = append(bundle[0].Rules, &placement.Rule{ + ID: strconv.Itoa(i), Index: i, Role: placement.Voter, Count: 1, GroupID: "1", + StartKey: []byte(strconv.Itoa(i)), EndKey: []byte(strconv.Itoa(i + 1)), + }) + } + return bundle + } + suite.postAndCheckRuleBundle(urlPrefix, genBundlesWithRulesNum(etcdutil.MaxEtcdTxnOps/2)) + suite.postAndCheckRuleBundle(urlPrefix, genBundlesWithRulesNum(etcdutil.MaxEtcdTxnOps*2)) +} + func (suite *ruleTestSuite) assertBundleEqual(re *require.Assertions, b1, b2 placement.GroupBundle) { tu.Eventually(re, func() bool { return suite.compareBundle(b1, b2) @@ -1251,6 +1257,32 @@ func (suite *ruleTestSuite) compareRule(r1 *placement.Rule, r2 *placement.Rule) r2.Count == r1.Count } +func (suite *ruleTestSuite) postAndCheckRuleBundle(urlPrefix string, bundle []placement.GroupBundle) { + re := suite.Require() + data, err := json.Marshal(bundle) + re.NoError(err) + err = tu.CheckPostJSON(testDialClient, urlPrefix+"/config/placement-rule", data, tu.StatusOK(re)) + re.NoError(err) + + tu.Eventually(re, func() bool { + respBundle := make([]placement.GroupBundle, 0) + err = tu.CheckGetJSON(testDialClient, urlPrefix+"/config/placement-rule", nil, + tu.StatusOK(re), tu.ExtractJSON(re, &respBundle)) + re.NoError(err) + if len(respBundle) != len(bundle) { + return false + } + sort.Slice(respBundle, func(i, j int) bool { return respBundle[i].ID < respBundle[j].ID }) + sort.Slice(bundle, func(i, j int) bool { return bundle[i].ID < bundle[j].ID }) + for i := range respBundle { + if !suite.compareBundle(respBundle[i], bundle[i]) { + return false + } + } + return true + }) +} + type regionRuleTestSuite struct { suite.Suite env *tests.SchedulingTestEnvironment From 04131a85c9b4d10628bc498cd625aa75f8e7fcfb Mon Sep 17 00:00:00 2001 From: glorv Date: Mon, 25 Dec 2023 13:39:24 +0800 Subject: [PATCH 124/137] support returning ru statistics in Get/ListResourceGroup resp Signed-off-by: glorv --- client/go.mod | 6 +- client/go.sum | 12 +-- .../resource_group/controller/controller.go | 5 +- client/resource_manager_client.go | 33 +++++-- go.mod | 27 +++--- go.sum | 42 +++++---- pkg/mcs/resourcemanager/server/apis/v1/api.go | 15 ++-- .../resourcemanager/server/grpc_service.go | 88 ++++++++++--------- pkg/mcs/resourcemanager/server/manager.go | 27 +++--- .../resourcemanager/server/resource_group.go | 50 +++++++++-- .../server/resource_group_test.go | 2 +- tests/integrations/client/go.mod | 22 ++--- tests/integrations/client/go.sum | 36 +++++--- tests/integrations/mcs/go.mod | 22 ++--- tests/integrations/mcs/go.sum | 36 +++++--- .../resourcemanager/resource_manager_test.go | 80 +++++++++++++++++ 16 files changed, 343 insertions(+), 160 deletions(-) diff --git a/client/go.mod b/client/go.mod index a97a94a3e95..eb49eb674d8 100644 --- a/client/go.mod +++ b/client/go.mod @@ -10,7 +10,7 @@ require ( github.com/opentracing/opentracing-go v1.2.0 github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 - github.com/pingcap/kvproto v0.0.0-20230727073445-53e1f8730c30 + github.com/pingcap/kvproto v0.0.0-20231222062942-c0c73f41d0b2 github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 github.com/prometheus/client_golang v1.11.1 github.com/stretchr/testify v1.8.2 @@ -18,7 +18,8 @@ require ( go.uber.org/goleak v1.1.11 go.uber.org/zap v1.24.0 golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4 - google.golang.org/grpc v1.54.0 + google.golang.org/grpc v1.59.0 + google.golang.org/grpc/examples v0.0.0-20231221225426-4f03f3ff32c9 ) require ( @@ -38,7 +39,6 @@ require ( golang.org/x/sys v0.14.0 // indirect golang.org/x/text v0.14.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17 // indirect - google.golang.org/grpc/examples v0.0.0-20230419000256-16651f60ddc5 google.golang.org/protobuf v1.31.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/client/go.sum b/client/go.sum index 4d4e9f396da..c72e50e79ee 100644 --- a/client/go.sum +++ b/client/go.sum @@ -94,8 +94,8 @@ github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c h1:xpW9bvK+HuuTm github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg= github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 h1:C3N3itkduZXDZFh4N3vQ5HEtld3S+Y+StULhWVvumU0= github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00/go.mod h1:4qGtCB0QK0wBzKtFEGDhxXnSnbQApw1gc9siScUl8ew= -github.com/pingcap/kvproto v0.0.0-20230727073445-53e1f8730c30 h1:EvqKcDT7ceGLW0mXqM8Cp5Z8DfgQRnwj2YTnlCLj2QI= -github.com/pingcap/kvproto v0.0.0-20230727073445-53e1f8730c30/go.mod h1:r0q/CFcwvyeRhKtoqzmWMBebrtpIziQQ9vR+JKh1knc= +github.com/pingcap/kvproto v0.0.0-20231222062942-c0c73f41d0b2 h1:364A6VCS+l0oHBKZKotX9LzmfEtIO/NTccTIQcPp3Ug= +github.com/pingcap/kvproto v0.0.0-20231222062942-c0c73f41d0b2/go.mod h1:rXxWk2UnwfUhLXha1jxRWPADw9eMZGWEWCg92Tgmb/8= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8IDP+SZrdhV1Kibl9KrHxJ9eciw= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -229,10 +229,10 @@ golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8T google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17 h1:Jyp0Hsi0bmHXG6k9eATXoYtjd6e2UzZ1SCn/wIupY14= google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17/go.mod h1:oQ5rr10WTTMvP4A36n8JpR1OrO1BEiV4f78CneXZxkA= -google.golang.org/grpc v1.54.0 h1:EhTqbhiYeixwWQtAEZAxmV9MGqcjEU2mFx52xCzNyag= -google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g= -google.golang.org/grpc/examples v0.0.0-20230419000256-16651f60ddc5 h1:FjbWL/mGfyRQNxjagfT1chiHL1569WEA/OGH0ZIzGcI= -google.golang.org/grpc/examples v0.0.0-20230419000256-16651f60ddc5/go.mod h1:5av8LiY5jU2KRcrX+SHtvLHnaOpPJ7gzWStBurgHlqY= +google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= +google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= +google.golang.org/grpc/examples v0.0.0-20231221225426-4f03f3ff32c9 h1:ATnmU8nL2NfIyTSiBvJVDIDIr3qBmeW+c7z7XU21eWs= +google.golang.org/grpc/examples v0.0.0-20231221225426-4f03f3ff32c9/go.mod h1:j5uROIAAgi3YmtiETMt1LW0d/lHqQ7wwrIY4uGRXLQ4= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= diff --git a/client/resource_group/controller/controller.go b/client/resource_group/controller/controller.go index 241e4ab81bf..326f564b1df 100755 --- a/client/resource_group/controller/controller.go +++ b/client/resource_group/controller/controller.go @@ -75,14 +75,15 @@ type ResourceGroupKVInterceptor interface { // ResourceGroupProvider provides some api to interact with resource manager server. type ResourceGroupProvider interface { - GetResourceGroup(ctx context.Context, resourceGroupName string) (*rmpb.ResourceGroup, error) + GetResourceGroup(ctx context.Context, resourceGroupName string, opts ...pd.GetResourceGroupOption) (*rmpb.ResourceGroup, error) + ListResourceGroups(ctx context.Context, opts ...pd.GetResourceGroupOption) ([]*rmpb.ResourceGroup, error) AddResourceGroup(ctx context.Context, metaGroup *rmpb.ResourceGroup) (string, error) ModifyResourceGroup(ctx context.Context, metaGroup *rmpb.ResourceGroup) (string, error) DeleteResourceGroup(ctx context.Context, resourceGroupName string) (string, error) AcquireTokenBuckets(ctx context.Context, request *rmpb.TokenBucketsRequest) ([]*rmpb.TokenBucketResponse, error) - LoadResourceGroups(ctx context.Context) ([]*rmpb.ResourceGroup, int64, error) // meta storage client + LoadResourceGroups(ctx context.Context) ([]*rmpb.ResourceGroup, int64, error) Watch(ctx context.Context, key []byte, opts ...pd.OpOption) (chan []*meta_storagepb.Event, error) Get(ctx context.Context, key []byte, opts ...pd.OpOption) (*meta_storagepb.GetResponse, error) } diff --git a/client/resource_manager_client.go b/client/resource_manager_client.go index 30944308584..ce7d7f10393 100644 --- a/client/resource_manager_client.go +++ b/client/resource_manager_client.go @@ -49,8 +49,8 @@ var ControllerConfigPathPrefixBytes = []byte(controllerConfigPathPrefix) // ResourceManagerClient manages resource group info and token request. type ResourceManagerClient interface { - ListResourceGroups(ctx context.Context) ([]*rmpb.ResourceGroup, error) - GetResourceGroup(ctx context.Context, resourceGroupName string) (*rmpb.ResourceGroup, error) + ListResourceGroups(ctx context.Context, opts ...GetResourceGroupOption) ([]*rmpb.ResourceGroup, error) + GetResourceGroup(ctx context.Context, resourceGroupName string, opts ...GetResourceGroupOption) (*rmpb.ResourceGroup, error) AddResourceGroup(ctx context.Context, metaGroup *rmpb.ResourceGroup) (string, error) ModifyResourceGroup(ctx context.Context, metaGroup *rmpb.ResourceGroup) (string, error) DeleteResourceGroup(ctx context.Context, resourceGroupName string) (string, error) @@ -59,6 +59,18 @@ type ResourceManagerClient interface { Watch(ctx context.Context, key []byte, opts ...OpOption) (chan []*meta_storagepb.Event, error) } +// GetResourceGroupOp represents available options when getting resource group. +type GetResourceGroupOp struct { + withRUStats bool +} + +// GetResourceGroupOption configures GetResourceGroupOp. +type GetResourceGroupOption func(*GetResourceGroupOp) + +func WithRUStats(op *GetResourceGroupOp) { + op.withRUStats = true +} + // resourceManagerClient gets the ResourceManager client of current PD leader. func (c *client) resourceManagerClient() (rmpb.ResourceManagerClient, error) { cc, err := c.pdSvcDiscovery.GetOrCreateGRPCConn(c.GetLeaderAddr()) @@ -76,12 +88,18 @@ func (c *client) gRPCErrorHandler(err error) { } // ListResourceGroups loads and returns all metadata of resource groups. -func (c *client) ListResourceGroups(ctx context.Context) ([]*rmpb.ResourceGroup, error) { +func (c *client) ListResourceGroups(ctx context.Context, ops ...GetResourceGroupOption) ([]*rmpb.ResourceGroup, error) { cc, err := c.resourceManagerClient() if err != nil { return nil, err } - req := &rmpb.ListResourceGroupsRequest{} + getOp := &GetResourceGroupOp{} + for _, op := range ops { + op(getOp) + } + req := &rmpb.ListResourceGroupsRequest{ + WithRuStats: getOp.withRUStats, + } resp, err := cc.ListResourceGroups(ctx, req) if err != nil { c.gRPCErrorHandler(err) @@ -94,13 +112,18 @@ func (c *client) ListResourceGroups(ctx context.Context) ([]*rmpb.ResourceGroup, return resp.GetGroups(), nil } -func (c *client) GetResourceGroup(ctx context.Context, resourceGroupName string) (*rmpb.ResourceGroup, error) { +func (c *client) GetResourceGroup(ctx context.Context, resourceGroupName string, ops ...GetResourceGroupOption) (*rmpb.ResourceGroup, error) { cc, err := c.resourceManagerClient() if err != nil { return nil, err } + getOp := &GetResourceGroupOp{} + for _, op := range ops { + op(getOp) + } req := &rmpb.GetResourceGroupRequest{ ResourceGroupName: resourceGroupName, + WithRuStats: getOp.withRUStats, } resp, err := cc.GetResourceGroup(ctx, req) if err != nil { diff --git a/go.mod b/go.mod index 157cdb4eab0..ae3d92172e5 100644 --- a/go.mod +++ b/go.mod @@ -33,7 +33,7 @@ require ( github.com/pingcap/errcode v0.3.0 github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 - github.com/pingcap/kvproto v0.0.0-20231018065736-c0689aded40c + github.com/pingcap/kvproto v0.0.0-20231222062942-c0c73f41d0b2 github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 github.com/pingcap/tidb-dashboard v0.0.0-20231218095437-aa621ed4de2c @@ -57,12 +57,17 @@ require ( go.uber.org/zap v1.24.0 golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4 golang.org/x/text v0.13.0 - golang.org/x/time v0.1.0 - golang.org/x/tools v0.6.0 - google.golang.org/grpc v1.54.0 + golang.org/x/time v0.3.0 + golang.org/x/tools v0.10.0 + google.golang.org/grpc v1.59.0 gotest.tools/gotestsum v1.7.0 ) +require ( + google.golang.org/genproto/googleapis/api v0.0.0-20231002182017-d307bd883b97 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b // indirect +) + require ( github.com/KyleBanks/depth v1.2.1 // indirect github.com/Masterminds/semver v1.5.0 // indirect @@ -93,7 +98,7 @@ require ( github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dnephin/pflag v1.0.7 // indirect - github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/fatih/color v1.10.0 // indirect github.com/fatih/structtag v1.2.0 // indirect github.com/fogleman/gg v1.3.0 // indirect @@ -154,7 +159,7 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b // indirect - github.com/prometheus/client_model v0.2.0 // indirect + github.com/prometheus/client_model v0.4.0 // indirect github.com/prometheus/procfs v0.6.0 // indirect github.com/rs/cors v1.7.0 // indirect github.com/russross/blackfriday/v2 v2.0.1 // indirect @@ -187,13 +192,13 @@ require ( golang.org/x/image v0.10.0 // indirect golang.org/x/mod v0.11.0 // indirect golang.org/x/net v0.17.0 // indirect - golang.org/x/oauth2 v0.4.0 // indirect - golang.org/x/sync v0.1.0 // indirect + golang.org/x/oauth2 v0.11.0 // indirect + golang.org/x/sync v0.3.0 // indirect golang.org/x/sys v0.13.0 // indirect golang.org/x/term v0.13.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect - google.golang.org/protobuf v1.30.0 // indirect + google.golang.org/genproto v0.0.0-20231012201019-e917dd12ba7a // indirect + google.golang.org/protobuf v1.31.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect @@ -205,7 +210,7 @@ require ( sigs.k8s.io/yaml v1.2.0 // indirect ) -replace google.golang.org/grpc v1.54.0 => google.golang.org/grpc v1.26.0 +replace google.golang.org/grpc v1.59.0 => google.golang.org/grpc v1.26.0 // When you modify PD cooperatively with kvproto, this will be useful to submit the PR to PD and the PR to // kvproto at the same time. You can run `go mod tidy` to make it replaced with go-mod style specification. diff --git a/go.sum b/go.sum index b244a791a9a..6d7700a73fb 100644 --- a/go.sum +++ b/go.sum @@ -125,8 +125,9 @@ github.com/dnephin/pflag v1.0.7 h1:oxONGlWxhmUct0YzKTgrpQv9AUA1wtPBn7zuSjJqptk= github.com/dnephin/pflag v1.0.7/go.mod h1:uxE91IoWURlOiTUIA8Mq5ZZkAv3dPUfZNaT80Zm7OQE= github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4 h1:qk/FSDDxo05wdJH28W+p5yivv7LuLYLRXPPD8KQCtZs= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385 h1:clC1lXBpe2kTj2VHdaIu9ajZQe4kcEY9j0NsnDDBZ3o= github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= github.com/elliotchance/pie/v2 v2.1.0 h1:KEVAAzxYxTyFs4hvebFZVzBdEo3YeMzl2HYDWn+P3F4= @@ -225,8 +226,8 @@ github.com/golang-sql/sqlexp v0.1.0/go.mod h1:J4ad9Vo8ZCWQ2GMrC4UCQy1JpCbwU9m3EO github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE= -github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ= +github.com/golang/glog v1.1.2 h1:DVjP2PbBOzHyzA+dn3WhHIq4NdVu3Q+pvivFICf/7fo= +github.com/golang/glog v1.1.2/go.mod h1:zR+okUeTbrL6EL3xHUDxZuEtGv04p5shwip1+mL/rLQ= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef h1:veQD95Isof8w9/WXiA+pa3tz3fJXkt5B7QaRBrM62gk= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -459,8 +460,8 @@ github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ue github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 h1:C3N3itkduZXDZFh4N3vQ5HEtld3S+Y+StULhWVvumU0= github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00/go.mod h1:4qGtCB0QK0wBzKtFEGDhxXnSnbQApw1gc9siScUl8ew= github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= -github.com/pingcap/kvproto v0.0.0-20231018065736-c0689aded40c h1:5cpE29kMHjc8fv+mRiXbLTDfoZHiX5BTK6knVWlWvqk= -github.com/pingcap/kvproto v0.0.0-20231018065736-c0689aded40c/go.mod h1:r0q/CFcwvyeRhKtoqzmWMBebrtpIziQQ9vR+JKh1knc= +github.com/pingcap/kvproto v0.0.0-20231222062942-c0c73f41d0b2 h1:364A6VCS+l0oHBKZKotX9LzmfEtIO/NTccTIQcPp3Ug= +github.com/pingcap/kvproto v0.0.0-20231222062942-c0c73f41d0b2/go.mod h1:rXxWk2UnwfUhLXha1jxRWPADw9eMZGWEWCg92Tgmb/8= github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7/go.mod h1:8AanEdAHATuRurdGxZXBz0At+9avep+ub7U1AGYLIMM= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8IDP+SZrdhV1Kibl9KrHxJ9eciw= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= @@ -489,8 +490,9 @@ github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqr github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= +github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= @@ -734,8 +736,8 @@ golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.4.0 h1:NF0gk8LVPg1Ml7SSbGyySuoxdsXitj7TvgvuRxIMc/M= -golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec= +golang.org/x/oauth2 v0.11.0 h1:vPL4xzxBM4niKCW6g9whtaWVXTJf1U5e4aZxxFx/gbU= +golang.org/x/oauth2 v0.11.0/go.mod h1:LdF7O/8bLR/qWK9DrpXmbHLTouvRHK0SgJl0GmDBchk= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -745,8 +747,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -809,8 +811,8 @@ golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.1.0 h1:xYY+Bajn2a7VBmTM5GikTmnK8ZuX8YgnQCqZpbBNtmA= -golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -838,8 +840,8 @@ golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= -golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.10.0 h1:tvDr/iQoUqNdohiYm0LmmKcBk+q86lb9EprIUFhHHGg= +golang.org/x/tools v0.10.0/go.mod h1:UJwyiVBsOA2uwvK/e5OY3GTpDUJriEd+/YlqAwLPmyM= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -852,8 +854,12 @@ google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoA google.golang.org/genproto v0.0.0-20181004005441-af9cb2a35e7f/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= -google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= +google.golang.org/genproto v0.0.0-20231012201019-e917dd12ba7a h1:fwgW9j3vHirt4ObdHoYNwuO24BEZjSzbh+zPaNWoiY8= +google.golang.org/genproto v0.0.0-20231012201019-e917dd12ba7a/go.mod h1:EMfReVxb80Dq1hhioy0sOsY9jCE46YDgHlJ7fWVUWRE= +google.golang.org/genproto/googleapis/api v0.0.0-20231002182017-d307bd883b97 h1:W18sezcAYs+3tDZX4F80yctqa12jcP1PUS2gQu1zTPU= +google.golang.org/genproto/googleapis/api v0.0.0-20231002182017-d307bd883b97/go.mod h1:iargEX0SFPm3xcfMI0d1domjg0ZF4Aa0p2awqyxhvF0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b h1:ZlWIi1wSK56/8hn4QcBp/j9M7Gt3U/3hZw3mC7vDICo= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b/go.mod h1:swOH3j0KzcDDgGUWr+SNpyTen5YrXjS3eyPzFYKc6lc= google.golang.org/grpc v0.0.0-20180607172857-7a6a684ca69e/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -873,8 +879,8 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/pkg/mcs/resourcemanager/server/apis/v1/api.go b/pkg/mcs/resourcemanager/server/apis/v1/api.go index 93738664361..bf4e043732b 100644 --- a/pkg/mcs/resourcemanager/server/apis/v1/api.go +++ b/pkg/mcs/resourcemanager/server/apis/v1/api.go @@ -19,6 +19,7 @@ import ( "fmt" "net/http" "reflect" + "strings" "sync" "github.com/gin-contrib/cors" @@ -184,12 +185,14 @@ func (s *Service) putResourceGroup(c *gin.Context) { // // @Tags ResourceManager // @Summary Get resource group by name. -// @Success 200 {string} json format of rmserver.ResourceGroup -// @Failure 404 {string} error -// @Param name path string true "groupName" +// @Success 200 {string} json format of rmserver.ResourceGroup +// @Failure 404 {string} error +// @Param name path string true "groupName" +// @Param with_stats query bool false whether to return statistics data. // @Router /config/group/{name} [GET] func (s *Service) getResourceGroup(c *gin.Context) { - group := s.manager.GetResourceGroup(c.Param("name")) + withStats := strings.EqualFold(c.Query("with_stats"), "true") + group := s.manager.GetResourceGroup(c.Param("name"), withStats) if group == nil { c.String(http.StatusNotFound, errors.New("resource group not found").Error()) } @@ -202,9 +205,11 @@ func (s *Service) getResourceGroup(c *gin.Context) { // @Summary get all resource group with a list. // @Success 200 {string} json format of []rmserver.ResourceGroup // @Failure 404 {string} error +// @Param with_stats query bool false whether to return statistics data. // @Router /config/groups [GET] func (s *Service) getResourceGroupList(c *gin.Context) { - groups := s.manager.GetResourceGroupList() + withStats := strings.EqualFold(c.Query("with_stats"), "true") + groups := s.manager.GetResourceGroupList(withStats) c.IndentedJSON(http.StatusOK, groups) } diff --git a/pkg/mcs/resourcemanager/server/grpc_service.go b/pkg/mcs/resourcemanager/server/grpc_service.go index bd46c45fa63..2322cb5bff3 100644 --- a/pkg/mcs/resourcemanager/server/grpc_service.go +++ b/pkg/mcs/resourcemanager/server/grpc_service.go @@ -98,7 +98,7 @@ func (s *Service) GetResourceGroup(ctx context.Context, req *rmpb.GetResourceGro if err := s.checkServing(); err != nil { return nil, err } - rg := s.manager.GetResourceGroup(req.ResourceGroupName) + rg := s.manager.GetResourceGroup(req.ResourceGroupName, req.WithRuStats) if rg == nil { return nil, errors.New("resource group not found") } @@ -112,7 +112,7 @@ func (s *Service) ListResourceGroups(ctx context.Context, req *rmpb.ListResource if err := s.checkServing(); err != nil { return nil, err } - groups := s.manager.GetResourceGroupList() + groups := s.manager.GetResourceGroupList(req.WithRuStats) resp := &rmpb.ListResourceGroupsResponse{ Groups: make([]*rmpb.ResourceGroup, 0, len(groups)), } @@ -184,49 +184,55 @@ func (s *Service) AcquireTokenBuckets(stream rmpb.ResourceManager_AcquireTokenBu resps := &rmpb.TokenBucketsResponse{} for _, req := range request.Requests { resourceGroupName := req.GetResourceGroupName() + var err error // Get the resource group from manager to acquire token buckets. - rg := s.manager.GetMutableResourceGroup(resourceGroupName) - if rg == nil { - log.Warn("resource group not found", zap.String("resource-group", resourceGroupName)) - continue - } - // Send the consumption to update the metrics. - isBackground := req.GetIsBackground() - isTiFlash := req.GetIsTiflash() - if isBackground && isTiFlash { - return errors.New("background and tiflash cannot be true at the same time") - } - s.manager.consumptionDispatcher <- struct { - resourceGroupName string - *rmpb.Consumption - isBackground bool - isTiFlash bool - }{resourceGroupName, req.GetConsumptionSinceLastRequest(), isBackground, isTiFlash} - if isBackground { - continue - } - now := time.Now() - resp := &rmpb.TokenBucketResponse{ - ResourceGroupName: rg.Name, - } - switch rg.Mode { - case rmpb.GroupMode_RUMode: - var tokens *rmpb.GrantedRUTokenBucket - for _, re := range req.GetRuItems().GetRequestRU() { - if re.Type == rmpb.RequestUnitType_RU { - tokens = rg.RequestRU(now, re.Value, targetPeriodMs, clientUniqueID) - } - if tokens == nil { - continue + s.manager.WithMutableResourceGroup(resourceGroupName, func(rg *ResourceGroup) { + if rg == nil { + log.Warn("resource group not found", zap.String("resource-group", resourceGroupName)) + return + } + // Send the consumption to update the metrics. + isBackground := req.GetIsBackground() + isTiFlash := req.GetIsTiflash() + if isBackground && isTiFlash { + err = errors.New("background and tiflash cannot be true at the same time") + return + } + s.manager.consumptionDispatcher <- struct { + resourceGroupName string + *rmpb.Consumption + isBackground bool + isTiFlash bool + }{resourceGroupName, req.GetConsumptionSinceLastRequest(), isBackground, isTiFlash} + if isBackground { + return + } + now := time.Now() + resp := &rmpb.TokenBucketResponse{ + ResourceGroupName: rg.Name, + } + switch rg.Mode { + case rmpb.GroupMode_RUMode: + var tokens *rmpb.GrantedRUTokenBucket + for _, re := range req.GetRuItems().GetRequestRU() { + if re.Type == rmpb.RequestUnitType_RU { + tokens = rg.RequestRU(now, re.Value, targetPeriodMs, clientUniqueID) + } + if tokens == nil { + continue + } + resp.GrantedRUTokens = append(resp.GrantedRUTokens, tokens) } - resp.GrantedRUTokens = append(resp.GrantedRUTokens, tokens) + case rmpb.GroupMode_RawMode: + log.Warn("not supports the resource type", zap.String("resource-group", resourceGroupName), zap.String("mode", rmpb.GroupMode_name[int32(rmpb.GroupMode_RawMode)])) + return } - case rmpb.GroupMode_RawMode: - log.Warn("not supports the resource type", zap.String("resource-group", resourceGroupName), zap.String("mode", rmpb.GroupMode_name[int32(rmpb.GroupMode_RawMode)])) - continue + log.Debug("finish token request from", zap.String("resource-group", resourceGroupName)) + resps.Responses = append(resps.Responses, resp) + }) + if err != nil { + return err } - log.Debug("finish token request from", zap.String("resource-group", resourceGroupName)) - resps.Responses = append(resps.Responses, resp) } stream.Send(resps) } diff --git a/pkg/mcs/resourcemanager/server/manager.go b/pkg/mcs/resourcemanager/server/manager.go index 6821938039b..2e50fa79b60 100644 --- a/pkg/mcs/resourcemanager/server/manager.go +++ b/pkg/mcs/resourcemanager/server/manager.go @@ -278,31 +278,28 @@ func (m *Manager) DeleteResourceGroup(name string) error { } // GetResourceGroup returns a copy of a resource group. -func (m *Manager) GetResourceGroup(name string) *ResourceGroup { +func (m *Manager) GetResourceGroup(name string, withStats bool) *ResourceGroup { m.RLock() defer m.RUnlock() if group, ok := m.groups[name]; ok { - return group.Copy() + return group.Copy(withStats) } return nil } -// GetMutableResourceGroup returns a mutable resource group. -func (m *Manager) GetMutableResourceGroup(name string) *ResourceGroup { +// GetMutableResourceGroup apply target function on target resource group. +func (m *Manager) WithMutableResourceGroup(name string, f func(*ResourceGroup)) { m.RLock() defer m.RUnlock() - if group, ok := m.groups[name]; ok { - return group - } - return nil + f(m.groups[name]) } // GetResourceGroupList returns copies of resource group list. -func (m *Manager) GetResourceGroupList() []*ResourceGroup { +func (m *Manager) GetResourceGroupList(withStats bool) []*ResourceGroup { m.RLock() res := make([]*ResourceGroup, 0, len(m.groups)) for _, group := range m.groups { - res = append(res, group.Copy()) + res = append(res, group.Copy(withStats)) } m.RUnlock() sort.Slice(res, func(i, j int) bool { @@ -338,12 +335,10 @@ func (m *Manager) persistResourceGroupRunningState() { for idx := 0; idx < len(keys); idx++ { m.RLock() group, ok := m.groups[keys[idx]] - m.RUnlock() if ok { - m.Lock() group.persistStates(m.storage) - m.Unlock() } + m.RUnlock() } } @@ -415,6 +410,12 @@ func (m *Manager) backgroundMetricsFlush(ctx context.Context) { m.consumptionRecord[name] = time.Now() + // TODO: maybe we need to distinguish background ru. + m.WithMutableResourceGroup(name, func(rg *ResourceGroup) { + if rg != nil { + rg.UpdateRUConsumption(consumptionInfo.Consumption) + } + }) case <-cleanUpTicker.C: // Clean up the metrics that have not been updated for a long time. for name, lastTime := range m.consumptionRecord { diff --git a/pkg/mcs/resourcemanager/server/resource_group.go b/pkg/mcs/resourcemanager/server/resource_group.go index fc3a58cab51..d1449c11898 100644 --- a/pkg/mcs/resourcemanager/server/resource_group.go +++ b/pkg/mcs/resourcemanager/server/resource_group.go @@ -37,6 +37,8 @@ type ResourceGroup struct { Priority uint32 `json:"priority"` Runaway *rmpb.RunawaySettings `json:"runaway_settings,omitempty"` Background *rmpb.BackgroundSettings `json:"background_settings,omitempty"` + // total ru consumption + RUConsumption *rmpb.Consumption `json:"ru_consumption,omitempty"` } // RequestUnitSettings is the definition of the RU settings. @@ -61,7 +63,7 @@ func (rg *ResourceGroup) String() string { } // Copy copies the resource group. -func (rg *ResourceGroup) Copy() *ResourceGroup { +func (rg *ResourceGroup) Copy(withStats bool) *ResourceGroup { // TODO: use a better way to copy rg.RLock() defer rg.RUnlock() @@ -74,6 +76,9 @@ func (rg *ResourceGroup) Copy() *ResourceGroup { if err != nil { panic(err) } + if !withStats { + newRG.RUConsumption = nil + } return &newRG } @@ -116,11 +121,12 @@ func (rg *ResourceGroup) PatchSettings(metaGroup *rmpb.ResourceGroup) error { // FromProtoResourceGroup converts a rmpb.ResourceGroup to a ResourceGroup. func FromProtoResourceGroup(group *rmpb.ResourceGroup) *ResourceGroup { rg := &ResourceGroup{ - Name: group.Name, - Mode: group.Mode, - Priority: group.Priority, - Runaway: group.RunawaySettings, - Background: group.BackgroundSettings, + Name: group.Name, + Mode: group.Mode, + Priority: group.Priority, + Runaway: group.RunawaySettings, + Background: group.BackgroundSettings, + RUConsumption: &rmpb.Consumption{}, } switch group.GetMode() { case rmpb.GroupMode_RUMode: @@ -129,6 +135,9 @@ func FromProtoResourceGroup(group *rmpb.ResourceGroup) *ResourceGroup { } else { rg.RUSettings = NewRequestUnitSettings(group.GetRUSettings().GetRU()) } + if group.RUStats != nil { + rg.RUConsumption = group.RUStats + } case rmpb.GroupMode_RawMode: panic("no implementation") } @@ -168,6 +177,10 @@ func (rg *ResourceGroup) IntoProtoResourceGroup() *rmpb.ResourceGroup { RunawaySettings: rg.Runaway, BackgroundSettings: rg.Background, } + + if rg.RUConsumption != nil { + group.RUStats = &*rg.RUConsumption + } return group case rmpb.GroupMode_RawMode: // Raw mode panic("no implementation") @@ -186,6 +199,8 @@ func (rg *ResourceGroup) persistSettings(storage endpoint.ResourceGroupStorage) type GroupStates struct { // RU tokens RU *GroupTokenBucketState `json:"r_u,omitempty"` + // RU consumption + RUConsumption *rmpb.Consumption `json:"ru_consumption,omitempty"` // raw resource tokens CPU *GroupTokenBucketState `json:"cpu,omitempty"` IORead *GroupTokenBucketState `json:"io_read,omitempty"` @@ -200,7 +215,8 @@ func (rg *ResourceGroup) GetGroupStates() *GroupStates { switch rg.Mode { case rmpb.GroupMode_RUMode: // RU mode tokens := &GroupStates{ - RU: rg.RUSettings.RU.GroupTokenBucketState.Clone(), + RU: rg.RUSettings.RU.GroupTokenBucketState.Clone(), + RUConsumption: &*rg.RUConsumption, } return tokens case rmpb.GroupMode_RawMode: // Raw mode @@ -217,11 +233,31 @@ func (rg *ResourceGroup) SetStatesIntoResourceGroup(states *GroupStates) { rg.RUSettings.RU.setState(state) log.Debug("update group token bucket state", zap.String("name", rg.Name), zap.Any("state", state)) } + if states.RUConsumption != nil { + rg.UpdateRUConsumption(states.RUConsumption) + } case rmpb.GroupMode_RawMode: panic("no implementation") } } +// UpdateRUConsumption add delta consumption data to group ru statistics. +func (rg *ResourceGroup) UpdateRUConsumption(c *rmpb.Consumption) { + rg.Lock() + defer rg.Unlock() + + rc := rg.RUConsumption + rc.RRU += c.RRU + rc.WRU += c.WRU + rc.ReadBytes += c.ReadBytes + rc.WriteBytes += c.WriteBytes + rc.TotalCpuTimeMs += c.TotalCpuTimeMs + rc.SqlLayerCpuTimeMs += c.SqlLayerCpuTimeMs + rc.KvReadRpcCount += c.KvReadRpcCount + rc.KvWriteRpcCount += c.KvWriteRpcCount + +} + // persistStates persists the resource group tokens. func (rg *ResourceGroup) persistStates(storage endpoint.ResourceGroupStorage) error { states := rg.GetGroupStates() diff --git a/pkg/mcs/resourcemanager/server/resource_group_test.go b/pkg/mcs/resourcemanager/server/resource_group_test.go index ee775ee989d..8df1be6dccc 100644 --- a/pkg/mcs/resourcemanager/server/resource_group_test.go +++ b/pkg/mcs/resourcemanager/server/resource_group_test.go @@ -29,7 +29,7 @@ func TestPatchResourceGroup(t *testing.T) { re.NoError(err) err = rg.PatchSettings(patch) re.NoError(err) - res, err := json.Marshal(rg.Copy()) + res, err := json.Marshal(rg.Copy(false)) re.NoError(err) re.Equal(ca.expectJSONString, string(res)) } diff --git a/tests/integrations/client/go.mod b/tests/integrations/client/go.mod index 5bbe8219bf0..d20d1cacfe9 100644 --- a/tests/integrations/client/go.mod +++ b/tests/integrations/client/go.mod @@ -8,12 +8,12 @@ replace ( ) // reset grpc and protobuf deps in order to import client and server at the same time -replace google.golang.org/grpc v1.54.0 => google.golang.org/grpc v1.26.0 +replace google.golang.org/grpc v1.59.0 => google.golang.org/grpc v1.26.0 require ( github.com/docker/go-units v0.4.0 github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 - github.com/pingcap/kvproto v0.0.0-20231018065736-c0689aded40c + github.com/pingcap/kvproto v0.0.0-20231222062942-c0c73f41d0b2 github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 github.com/stretchr/testify v1.8.3 github.com/tikv/pd v0.0.0-00010101000000-000000000000 @@ -21,7 +21,7 @@ require ( go.etcd.io/etcd v0.5.0-alpha.5.0.20220915004622-85b640cee793 go.uber.org/goleak v1.1.12 go.uber.org/zap v1.24.0 - google.golang.org/grpc v1.54.0 + google.golang.org/grpc v1.59.0 ) require ( @@ -59,7 +59,7 @@ require ( github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f // indirect github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/elliotchance/pie/v2 v2.1.0 // indirect github.com/fogleman/gg v1.3.0 // indirect github.com/gabriel-vasile/mimetype v1.4.2 // indirect @@ -129,7 +129,7 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b // indirect github.com/prometheus/client_golang v1.11.1 // indirect - github.com/prometheus/client_model v0.2.0 // indirect + github.com/prometheus/client_model v0.4.0 // indirect github.com/prometheus/common v0.26.0 // indirect github.com/prometheus/procfs v0.6.0 // indirect github.com/rs/cors v1.7.0 // indirect @@ -169,14 +169,16 @@ require ( golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4 // indirect golang.org/x/image v0.10.0 // indirect golang.org/x/net v0.18.0 // indirect - golang.org/x/oauth2 v0.4.0 // indirect - golang.org/x/sync v0.1.0 // indirect + golang.org/x/oauth2 v0.11.0 // indirect + golang.org/x/sync v0.3.0 // indirect golang.org/x/sys v0.15.0 // indirect golang.org/x/text v0.14.0 // indirect - golang.org/x/time v0.1.0 // indirect - golang.org/x/tools v0.6.0 // indirect + golang.org/x/time v0.3.0 // indirect + golang.org/x/tools v0.10.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect + google.golang.org/genproto v0.0.0-20231030173426-d783a09b4405 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17 // indirect google.golang.org/protobuf v1.31.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/tests/integrations/client/go.sum b/tests/integrations/client/go.sum index 2ed7bb415af..9e42dfa900c 100644 --- a/tests/integrations/client/go.sum +++ b/tests/integrations/client/go.sum @@ -115,8 +115,9 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4 h1:qk/FSDDxo05wdJH28W+p5yivv7LuLYLRXPPD8KQCtZs= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385 h1:clC1lXBpe2kTj2VHdaIu9ajZQe4kcEY9j0NsnDDBZ3o= github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= github.com/elliotchance/pie/v2 v2.1.0 h1:KEVAAzxYxTyFs4hvebFZVzBdEo3YeMzl2HYDWn+P3F4= @@ -204,8 +205,8 @@ github.com/golang-sql/sqlexp v0.1.0/go.mod h1:J4ad9Vo8ZCWQ2GMrC4UCQy1JpCbwU9m3EO github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE= -github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ= +github.com/golang/glog v1.1.2 h1:DVjP2PbBOzHyzA+dn3WhHIq4NdVu3Q+pvivFICf/7fo= +github.com/golang/glog v1.1.2/go.mod h1:zR+okUeTbrL6EL3xHUDxZuEtGv04p5shwip1+mL/rLQ= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef h1:veQD95Isof8w9/WXiA+pa3tz3fJXkt5B7QaRBrM62gk= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -422,8 +423,8 @@ github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ue github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 h1:C3N3itkduZXDZFh4N3vQ5HEtld3S+Y+StULhWVvumU0= github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00/go.mod h1:4qGtCB0QK0wBzKtFEGDhxXnSnbQApw1gc9siScUl8ew= github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= -github.com/pingcap/kvproto v0.0.0-20231018065736-c0689aded40c h1:5cpE29kMHjc8fv+mRiXbLTDfoZHiX5BTK6knVWlWvqk= -github.com/pingcap/kvproto v0.0.0-20231018065736-c0689aded40c/go.mod h1:r0q/CFcwvyeRhKtoqzmWMBebrtpIziQQ9vR+JKh1knc= +github.com/pingcap/kvproto v0.0.0-20231222062942-c0c73f41d0b2 h1:364A6VCS+l0oHBKZKotX9LzmfEtIO/NTccTIQcPp3Ug= +github.com/pingcap/kvproto v0.0.0-20231222062942-c0c73f41d0b2/go.mod h1:rXxWk2UnwfUhLXha1jxRWPADw9eMZGWEWCg92Tgmb/8= github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7/go.mod h1:8AanEdAHATuRurdGxZXBz0At+9avep+ub7U1AGYLIMM= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8IDP+SZrdhV1Kibl9KrHxJ9eciw= @@ -453,8 +454,9 @@ github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqr github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= +github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= @@ -691,8 +693,8 @@ golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.4.0 h1:NF0gk8LVPg1Ml7SSbGyySuoxdsXitj7TvgvuRxIMc/M= -golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec= +golang.org/x/oauth2 v0.11.0 h1:vPL4xzxBM4niKCW6g9whtaWVXTJf1U5e4aZxxFx/gbU= +golang.org/x/oauth2 v0.11.0/go.mod h1:LdF7O/8bLR/qWK9DrpXmbHLTouvRHK0SgJl0GmDBchk= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -702,8 +704,9 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -759,8 +762,8 @@ golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.1.0 h1:xYY+Bajn2a7VBmTM5GikTmnK8ZuX8YgnQCqZpbBNtmA= -golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -785,8 +788,9 @@ golang.org/x/tools v0.0.0-20210112230658-8b4aab62c064/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.10.0 h1:tvDr/iQoUqNdohiYm0LmmKcBk+q86lb9EprIUFhHHGg= +golang.org/x/tools v0.10.0/go.mod h1:UJwyiVBsOA2uwvK/e5OY3GTpDUJriEd+/YlqAwLPmyM= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -799,8 +803,12 @@ google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoA google.golang.org/genproto v0.0.0-20181004005441-af9cb2a35e7f/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= -google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= +google.golang.org/genproto v0.0.0-20231030173426-d783a09b4405 h1:I6WNifs6pF9tNdSob2W24JtyxIYjzFB9qDlpUC76q+U= +google.golang.org/genproto v0.0.0-20231030173426-d783a09b4405/go.mod h1:3WDQMjmJk36UQhjQ89emUzb1mdaHcPeeAh4SCBKznB4= +google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b h1:CIC2YMXmIhYw6evmhPxBKJ4fmLbOFtXQN/GV3XOZR8k= +google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b/go.mod h1:IBQ646DjkDkvUIsVq/cc03FUFQ9wbZu7yE396YcL870= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17 h1:Jyp0Hsi0bmHXG6k9eATXoYtjd6e2UzZ1SCn/wIupY14= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17/go.mod h1:oQ5rr10WTTMvP4A36n8JpR1OrO1BEiV4f78CneXZxkA= google.golang.org/grpc v0.0.0-20180607172857-7a6a684ca69e/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= diff --git a/tests/integrations/mcs/go.mod b/tests/integrations/mcs/go.mod index 5771535dd19..233ba8b3af0 100644 --- a/tests/integrations/mcs/go.mod +++ b/tests/integrations/mcs/go.mod @@ -8,12 +8,12 @@ replace ( ) // reset grpc and protobuf deps in order to import client and server at the same time -replace google.golang.org/grpc v1.54.0 => google.golang.org/grpc v1.26.0 +replace google.golang.org/grpc v1.59.0 => google.golang.org/grpc v1.26.0 require ( github.com/docker/go-units v0.4.0 github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 - github.com/pingcap/kvproto v0.0.0-20231018065736-c0689aded40c + github.com/pingcap/kvproto v0.0.0-20231222062942-c0c73f41d0b2 github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 github.com/stretchr/testify v1.8.3 github.com/tikv/pd v0.0.0-00010101000000-000000000000 @@ -21,7 +21,7 @@ require ( go.etcd.io/etcd v0.5.0-alpha.5.0.20220915004622-85b640cee793 go.uber.org/goleak v1.1.12 go.uber.org/zap v1.24.0 - google.golang.org/grpc v1.54.0 + google.golang.org/grpc v1.59.0 ) require ( @@ -60,7 +60,7 @@ require ( github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f // indirect github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/elliotchance/pie/v2 v2.1.0 // indirect github.com/fogleman/gg v1.3.0 // indirect github.com/gabriel-vasile/mimetype v1.4.2 // indirect @@ -129,7 +129,7 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b // indirect github.com/prometheus/client_golang v1.11.1 // indirect - github.com/prometheus/client_model v0.2.0 // indirect + github.com/prometheus/client_model v0.4.0 // indirect github.com/prometheus/common v0.26.0 // indirect github.com/prometheus/procfs v0.6.0 // indirect github.com/rs/cors v1.7.0 // indirect @@ -169,14 +169,16 @@ require ( golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4 // indirect golang.org/x/image v0.10.0 // indirect golang.org/x/net v0.18.0 // indirect - golang.org/x/oauth2 v0.4.0 // indirect - golang.org/x/sync v0.1.0 // indirect + golang.org/x/oauth2 v0.11.0 // indirect + golang.org/x/sync v0.3.0 // indirect golang.org/x/sys v0.15.0 // indirect golang.org/x/text v0.14.0 // indirect - golang.org/x/time v0.1.0 // indirect - golang.org/x/tools v0.6.0 // indirect + golang.org/x/time v0.3.0 // indirect + golang.org/x/tools v0.10.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect + google.golang.org/genproto v0.0.0-20231030173426-d783a09b4405 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17 // indirect google.golang.org/protobuf v1.31.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/tests/integrations/mcs/go.sum b/tests/integrations/mcs/go.sum index 36960bcb5a9..6397ab5b017 100644 --- a/tests/integrations/mcs/go.sum +++ b/tests/integrations/mcs/go.sum @@ -117,8 +117,9 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4 h1:qk/FSDDxo05wdJH28W+p5yivv7LuLYLRXPPD8KQCtZs= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385 h1:clC1lXBpe2kTj2VHdaIu9ajZQe4kcEY9j0NsnDDBZ3o= github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= github.com/elliotchance/pie/v2 v2.1.0 h1:KEVAAzxYxTyFs4hvebFZVzBdEo3YeMzl2HYDWn+P3F4= @@ -208,8 +209,8 @@ github.com/golang-sql/sqlexp v0.1.0/go.mod h1:J4ad9Vo8ZCWQ2GMrC4UCQy1JpCbwU9m3EO github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE= -github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ= +github.com/golang/glog v1.1.2 h1:DVjP2PbBOzHyzA+dn3WhHIq4NdVu3Q+pvivFICf/7fo= +github.com/golang/glog v1.1.2/go.mod h1:zR+okUeTbrL6EL3xHUDxZuEtGv04p5shwip1+mL/rLQ= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef h1:veQD95Isof8w9/WXiA+pa3tz3fJXkt5B7QaRBrM62gk= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -427,8 +428,8 @@ github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ue github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 h1:C3N3itkduZXDZFh4N3vQ5HEtld3S+Y+StULhWVvumU0= github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00/go.mod h1:4qGtCB0QK0wBzKtFEGDhxXnSnbQApw1gc9siScUl8ew= github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= -github.com/pingcap/kvproto v0.0.0-20231018065736-c0689aded40c h1:5cpE29kMHjc8fv+mRiXbLTDfoZHiX5BTK6knVWlWvqk= -github.com/pingcap/kvproto v0.0.0-20231018065736-c0689aded40c/go.mod h1:r0q/CFcwvyeRhKtoqzmWMBebrtpIziQQ9vR+JKh1knc= +github.com/pingcap/kvproto v0.0.0-20231222062942-c0c73f41d0b2 h1:364A6VCS+l0oHBKZKotX9LzmfEtIO/NTccTIQcPp3Ug= +github.com/pingcap/kvproto v0.0.0-20231222062942-c0c73f41d0b2/go.mod h1:rXxWk2UnwfUhLXha1jxRWPADw9eMZGWEWCg92Tgmb/8= github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7/go.mod h1:8AanEdAHATuRurdGxZXBz0At+9avep+ub7U1AGYLIMM= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8IDP+SZrdhV1Kibl9KrHxJ9eciw= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= @@ -457,8 +458,9 @@ github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqr github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= +github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= @@ -695,8 +697,8 @@ golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.4.0 h1:NF0gk8LVPg1Ml7SSbGyySuoxdsXitj7TvgvuRxIMc/M= -golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec= +golang.org/x/oauth2 v0.11.0 h1:vPL4xzxBM4niKCW6g9whtaWVXTJf1U5e4aZxxFx/gbU= +golang.org/x/oauth2 v0.11.0/go.mod h1:LdF7O/8bLR/qWK9DrpXmbHLTouvRHK0SgJl0GmDBchk= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -706,8 +708,9 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -763,8 +766,8 @@ golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.1.0 h1:xYY+Bajn2a7VBmTM5GikTmnK8ZuX8YgnQCqZpbBNtmA= -golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -788,8 +791,9 @@ golang.org/x/tools v0.0.0-20210112230658-8b4aab62c064/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.10.0 h1:tvDr/iQoUqNdohiYm0LmmKcBk+q86lb9EprIUFhHHGg= +golang.org/x/tools v0.10.0/go.mod h1:UJwyiVBsOA2uwvK/e5OY3GTpDUJriEd+/YlqAwLPmyM= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -802,8 +806,12 @@ google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoA google.golang.org/genproto v0.0.0-20181004005441-af9cb2a35e7f/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= -google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= +google.golang.org/genproto v0.0.0-20231030173426-d783a09b4405 h1:I6WNifs6pF9tNdSob2W24JtyxIYjzFB9qDlpUC76q+U= +google.golang.org/genproto v0.0.0-20231030173426-d783a09b4405/go.mod h1:3WDQMjmJk36UQhjQ89emUzb1mdaHcPeeAh4SCBKznB4= +google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b h1:CIC2YMXmIhYw6evmhPxBKJ4fmLbOFtXQN/GV3XOZR8k= +google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b/go.mod h1:IBQ646DjkDkvUIsVq/cc03FUFQ9wbZu7yE396YcL870= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17 h1:Jyp0Hsi0bmHXG6k9eATXoYtjd6e2UzZ1SCn/wIupY14= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17/go.mod h1:oQ5rr10WTTMvP4A36n8JpR1OrO1BEiV4f78CneXZxkA= google.golang.org/grpc v0.0.0-20180607172857-7a6a684ca69e/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= diff --git a/tests/integrations/mcs/resourcemanager/resource_manager_test.go b/tests/integrations/mcs/resourcemanager/resource_manager_test.go index 44477ca7693..a9708598141 100644 --- a/tests/integrations/mcs/resourcemanager/resource_manager_test.go +++ b/tests/integrations/mcs/resourcemanager/resource_manager_test.go @@ -1060,6 +1060,86 @@ func (suite *resourceManagerClientTestSuite) TestBasicResourceGroupCURD() { re.Equal(groups, newGroups) } +func (suite *resourceManagerClientTestSuite) TestResourceGroupRUConsumption() { + re := suite.Require() + cli := suite.client + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/mcs/resourcemanager/server/fastPersist", `return(true)`)) + defer re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/mcs/resourcemanager/server/fastPersist")) + group := &rmpb.ResourceGroup{ + Name: "test_ru_consumption", + Mode: rmpb.GroupMode_RUMode, + RUSettings: &rmpb.GroupRequestUnitSettings{ + RU: &rmpb.TokenBucket{Settings: &rmpb.TokenLimitSettings{ + FillRate: 10000, + BurstLimit: 10000, + MaxTokens: 20000.0, + }}, + }, + } + _, err := cli.AddResourceGroup(suite.ctx, group) + re.NoError(err) + + g, err := cli.GetResourceGroup(suite.ctx, group.Name) + re.NoError(err) + re.Equal(group, g) + + // Test Resource Group Stats + testConsumption := &rmpb.Consumption{ + RRU: 200.0, + WRU: 100.0, + ReadBytes: 1024, + WriteBytes: 512, + TotalCpuTimeMs: 50.0, + SqlLayerCpuTimeMs: 40.0, + KvReadRpcCount: 5, + KvWriteRpcCount: 6, + } + _, err = cli.AcquireTokenBuckets(suite.ctx, &rmpb.TokenBucketsRequest{ + Requests: []*rmpb.TokenBucketRequest{ + { + ResourceGroupName: group.Name, + ConsumptionSinceLastRequest: testConsumption, + }, + }, + TargetRequestPeriodMs: 1000, + ClientUniqueId: 1, + }) + re.NoError(err) + time.Sleep(10 * time.Millisecond) + g, err = cli.GetResourceGroup(suite.ctx, group.Name, pd.WithRUStats) + re.NoError(err) + re.Equal(g.RUStats, testConsumption) + + // update resoruce group, ru stats not change + g.RUSettings.RU.Settings.FillRate = 12345 + _, err = cli.ModifyResourceGroup(suite.ctx, g) + re.NoError(err) + g1, err := cli.GetResourceGroup(suite.ctx, group.Name, pd.WithRUStats) + re.NoError(err) + re.Equal(g1, g) + + // test restart cluster + time.Sleep(250 * time.Millisecond) + servers := suite.cluster.GetServers() + re.NoError(suite.cluster.StopAll()) + cli.Close() + serverList := make([]*tests.TestServer, 0, len(servers)) + for _, s := range servers { + serverList = append(serverList, s) + } + re.NoError(suite.cluster.RunServers(serverList)) + suite.cluster.WaitLeader() + // re-connect client as well + suite.client, err = pd.NewClientWithContext(suite.ctx, suite.cluster.GetConfig().GetClientURLs(), pd.SecurityOption{}) + re.NoError(err) + cli = suite.client + + // check ru stats not loss after restart + g, err = cli.GetResourceGroup(suite.ctx, group.Name, pd.WithRUStats) + re.NoError(err) + re.Equal(g.RUStats, testConsumption) +} + func (suite *resourceManagerClientTestSuite) TestResourceManagerClientFailover() { re := suite.Require() cli := suite.client From 7f510a203a4b62753d2757847767b85e9ed753c7 Mon Sep 17 00:00:00 2001 From: glorv Date: Mon, 25 Dec 2023 14:07:40 +0800 Subject: [PATCH 125/137] fix swagger Signed-off-by: glorv --- pkg/mcs/resourcemanager/server/apis/v1/api.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/mcs/resourcemanager/server/apis/v1/api.go b/pkg/mcs/resourcemanager/server/apis/v1/api.go index bf4e043732b..fee648a5595 100644 --- a/pkg/mcs/resourcemanager/server/apis/v1/api.go +++ b/pkg/mcs/resourcemanager/server/apis/v1/api.go @@ -188,7 +188,7 @@ func (s *Service) putResourceGroup(c *gin.Context) { // @Success 200 {string} json format of rmserver.ResourceGroup // @Failure 404 {string} error // @Param name path string true "groupName" -// @Param with_stats query bool false whether to return statistics data. +// @Param with_stats query bool false "whether to return statistics data." // @Router /config/group/{name} [GET] func (s *Service) getResourceGroup(c *gin.Context) { withStats := strings.EqualFold(c.Query("with_stats"), "true") @@ -205,7 +205,7 @@ func (s *Service) getResourceGroup(c *gin.Context) { // @Summary get all resource group with a list. // @Success 200 {string} json format of []rmserver.ResourceGroup // @Failure 404 {string} error -// @Param with_stats query bool false whether to return statistics data. +// @Param with_stats query bool false "whether to return statistics data." // @Router /config/groups [GET] func (s *Service) getResourceGroupList(c *gin.Context) { withStats := strings.EqualFold(c.Query("with_stats"), "true") From 7774148f1bf9b5079f530eb5364060e07d59017d Mon Sep 17 00:00:00 2001 From: glorv Date: Mon, 25 Dec 2023 14:17:48 +0800 Subject: [PATCH 126/137] make tidy Signed-off-by: glorv --- tests/integrations/realcluster/go.mod | 6 +++--- tests/integrations/realcluster/go.sum | 12 ++++++------ tools/pd-tso-bench/go.mod | 6 +++--- tools/pd-tso-bench/go.sum | 16 ++++++++-------- 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/tests/integrations/realcluster/go.mod b/tests/integrations/realcluster/go.mod index 0815fc23d8d..dd5b2781e52 100644 --- a/tests/integrations/realcluster/go.mod +++ b/tests/integrations/realcluster/go.mod @@ -26,7 +26,7 @@ require ( github.com/jinzhu/now v1.1.5 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c // indirect - github.com/pingcap/kvproto v0.0.0-20230727073445-53e1f8730c30 // indirect + github.com/pingcap/kvproto v0.0.0-20231222062942-c0c73f41d0b2 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_golang v1.11.1 // indirect github.com/prometheus/client_model v0.2.0 // indirect @@ -39,8 +39,8 @@ require ( golang.org/x/net v0.18.0 // indirect golang.org/x/sys v0.14.0 // indirect golang.org/x/text v0.14.0 // indirect - google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect - google.golang.org/grpc v1.54.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17 // indirect + google.golang.org/grpc v1.59.0 // indirect google.golang.org/protobuf v1.31.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/tests/integrations/realcluster/go.sum b/tests/integrations/realcluster/go.sum index 4808a766575..b44d06ef103 100644 --- a/tests/integrations/realcluster/go.sum +++ b/tests/integrations/realcluster/go.sum @@ -86,8 +86,8 @@ github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRW github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c h1:xpW9bvK+HuuTmyFqUwr+jcCvpVkK7sumiz+ko5H9eq4= github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg= -github.com/pingcap/kvproto v0.0.0-20230727073445-53e1f8730c30 h1:EvqKcDT7ceGLW0mXqM8Cp5Z8DfgQRnwj2YTnlCLj2QI= -github.com/pingcap/kvproto v0.0.0-20230727073445-53e1f8730c30/go.mod h1:r0q/CFcwvyeRhKtoqzmWMBebrtpIziQQ9vR+JKh1knc= +github.com/pingcap/kvproto v0.0.0-20231222062942-c0c73f41d0b2 h1:364A6VCS+l0oHBKZKotX9LzmfEtIO/NTccTIQcPp3Ug= +github.com/pingcap/kvproto v0.0.0-20231222062942-c0c73f41d0b2/go.mod h1:rXxWk2UnwfUhLXha1jxRWPADw9eMZGWEWCg92Tgmb/8= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8IDP+SZrdhV1Kibl9KrHxJ9eciw= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -210,10 +210,10 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= -google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= -google.golang.org/grpc v1.54.0 h1:EhTqbhiYeixwWQtAEZAxmV9MGqcjEU2mFx52xCzNyag= -google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17 h1:Jyp0Hsi0bmHXG6k9eATXoYtjd6e2UzZ1SCn/wIupY14= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17/go.mod h1:oQ5rr10WTTMvP4A36n8JpR1OrO1BEiV4f78CneXZxkA= +google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= +google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= diff --git a/tools/pd-tso-bench/go.mod b/tools/pd-tso-bench/go.mod index c8fe68aab44..8e2db4a106c 100644 --- a/tools/pd-tso-bench/go.mod +++ b/tools/pd-tso-bench/go.mod @@ -9,7 +9,7 @@ require ( github.com/prometheus/client_golang v1.11.1 github.com/tikv/pd/client v0.0.0-00010101000000-000000000000 go.uber.org/zap v1.24.0 - google.golang.org/grpc v1.56.3 + google.golang.org/grpc v1.59.0 ) require ( @@ -21,7 +21,7 @@ require ( github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 // indirect - github.com/pingcap/kvproto v0.0.0-20230727073445-53e1f8730c30 // indirect + github.com/pingcap/kvproto v0.0.0-20231222062942-c0c73f41d0b2 // indirect github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/common v0.26.0 // indirect github.com/prometheus/procfs v0.6.0 // indirect @@ -30,7 +30,7 @@ require ( golang.org/x/net v0.18.0 // indirect golang.org/x/sys v0.14.0 // indirect golang.org/x/text v0.14.0 // indirect - google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17 // indirect google.golang.org/protobuf v1.31.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect ) diff --git a/tools/pd-tso-bench/go.sum b/tools/pd-tso-bench/go.sum index 31875879c2d..61917bc8f91 100644 --- a/tools/pd-tso-bench/go.sum +++ b/tools/pd-tso-bench/go.sum @@ -84,8 +84,8 @@ github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c h1:xpW9bvK+HuuTm github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg= github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 h1:C3N3itkduZXDZFh4N3vQ5HEtld3S+Y+StULhWVvumU0= github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00/go.mod h1:4qGtCB0QK0wBzKtFEGDhxXnSnbQApw1gc9siScUl8ew= -github.com/pingcap/kvproto v0.0.0-20230727073445-53e1f8730c30 h1:EvqKcDT7ceGLW0mXqM8Cp5Z8DfgQRnwj2YTnlCLj2QI= -github.com/pingcap/kvproto v0.0.0-20230727073445-53e1f8730c30/go.mod h1:r0q/CFcwvyeRhKtoqzmWMBebrtpIziQQ9vR+JKh1knc= +github.com/pingcap/kvproto v0.0.0-20231222062942-c0c73f41d0b2 h1:364A6VCS+l0oHBKZKotX9LzmfEtIO/NTccTIQcPp3Ug= +github.com/pingcap/kvproto v0.0.0-20231222062942-c0c73f41d0b2/go.mod h1:rXxWk2UnwfUhLXha1jxRWPADw9eMZGWEWCg92Tgmb/8= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8IDP+SZrdhV1Kibl9KrHxJ9eciw= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -205,12 +205,12 @@ gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca h1:PupagGYwj8+I4ubCxcmcBRk gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/netlib v0.0.0-20181029234149-ec6d1f5cefe6/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= -google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= -google.golang.org/grpc v1.56.3 h1:8I4C0Yq1EjstUzUJzpcRVbuYA2mODtEmpWiQoN/b2nc= -google.golang.org/grpc v1.56.3/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= -google.golang.org/grpc/examples v0.0.0-20230419000256-16651f60ddc5 h1:FjbWL/mGfyRQNxjagfT1chiHL1569WEA/OGH0ZIzGcI= -google.golang.org/grpc/examples v0.0.0-20230419000256-16651f60ddc5/go.mod h1:5av8LiY5jU2KRcrX+SHtvLHnaOpPJ7gzWStBurgHlqY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17 h1:Jyp0Hsi0bmHXG6k9eATXoYtjd6e2UzZ1SCn/wIupY14= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17/go.mod h1:oQ5rr10WTTMvP4A36n8JpR1OrO1BEiV4f78CneXZxkA= +google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= +google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= +google.golang.org/grpc/examples v0.0.0-20231221225426-4f03f3ff32c9 h1:ATnmU8nL2NfIyTSiBvJVDIDIr3qBmeW+c7z7XU21eWs= +google.golang.org/grpc/examples v0.0.0-20231221225426-4f03f3ff32c9/go.mod h1:j5uROIAAgi3YmtiETMt1LW0d/lHqQ7wwrIY4uGRXLQ4= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= From c13f21f2440dd4879bb1482c9dd3357b8e69b6a9 Mon Sep 17 00:00:00 2001 From: glorv Date: Mon, 25 Dec 2023 14:25:54 +0800 Subject: [PATCH 127/137] fix tidy again Signed-off-by: glorv --- go.sum | 2 ++ 1 file changed, 2 insertions(+) diff --git a/go.sum b/go.sum index 6d7700a73fb..77920541000 100644 --- a/go.sum +++ b/go.sum @@ -747,6 +747,7 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -840,6 +841,7 @@ golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.10.0 h1:tvDr/iQoUqNdohiYm0LmmKcBk+q86lb9EprIUFhHHGg= golang.org/x/tools v0.10.0/go.mod h1:UJwyiVBsOA2uwvK/e5OY3GTpDUJriEd+/YlqAwLPmyM= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= From 8950c3a9a874f03b52b3b90325bed2444d6e7059 Mon Sep 17 00:00:00 2001 From: Yongbo Jiang Date: Mon, 25 Dec 2023 14:49:56 +0800 Subject: [PATCH 128/137] *: add maxprocs metrics for mcs (#7604) ref tikv/pd#5839 Signed-off-by: Cabinfever_B --- .../server => basicserver}/metrics.go | 27 +++++++++++-------- pkg/mcs/resourcemanager/server/metrics.go | 9 ------- pkg/mcs/resourcemanager/server/server.go | 4 ++- pkg/mcs/scheduling/server/server.go | 5 ++-- pkg/mcs/tso/server/metrics.go | 9 ------- pkg/mcs/tso/server/server.go | 4 ++- server/metrics.go | 17 ------------ server/server.go | 5 ++-- .../mcs/resourcemanager/server_test.go | 2 +- tests/integrations/mcs/scheduling/api_test.go | 2 +- tests/integrations/mcs/tso/api_test.go | 2 +- 11 files changed, 31 insertions(+), 55 deletions(-) rename pkg/{mcs/scheduling/server => basicserver}/metrics.go (57%) diff --git a/pkg/mcs/scheduling/server/metrics.go b/pkg/basicserver/metrics.go similarity index 57% rename from pkg/mcs/scheduling/server/metrics.go rename to pkg/basicserver/metrics.go index b3f5b7b41de..8f26216d696 100644 --- a/pkg/mcs/scheduling/server/metrics.go +++ b/pkg/basicserver/metrics.go @@ -16,22 +16,27 @@ package server import "github.com/prometheus/client_golang/prometheus" -const ( - namespace = "scheduling" - serverSubsystem = "server" -) - var ( - // Meta & Server info. - serverInfo = prometheus.NewGaugeVec( + // ServerMaxProcsGauge record the maxprocs. + ServerMaxProcsGauge = prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: "pd", + Subsystem: "service", + Name: "maxprocs", + Help: "The value of GOMAXPROCS.", + }) + + // ServerInfoGauge indicates the pd server info including version and git hash. + ServerInfoGauge = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Namespace: namespace, - Subsystem: serverSubsystem, + Namespace: "pd", + Subsystem: "server", Name: "info", - Help: "Indicate the scheduling server info, and the value is the start timestamp (s).", + Help: "Indicate the pd server info, and the value is the start timestamp (s).", }, []string{"version", "hash"}) ) func init() { - prometheus.MustRegister(serverInfo) + prometheus.MustRegister(ServerMaxProcsGauge) + prometheus.MustRegister(ServerInfoGauge) } diff --git a/pkg/mcs/resourcemanager/server/metrics.go b/pkg/mcs/resourcemanager/server/metrics.go index 4322ed1a640..6bb90c45d12 100644 --- a/pkg/mcs/resourcemanager/server/metrics.go +++ b/pkg/mcs/resourcemanager/server/metrics.go @@ -32,14 +32,6 @@ const ( ) var ( - // Meta & Server info. - serverInfo = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: namespace, - Subsystem: serverSubsystem, - Name: "info", - Help: "Indicate the resource manager server info, and the value is the start timestamp (s).", - }, []string{"version", "hash"}) // RU cost metrics. // `sum` is added to the name to maintain compatibility with the previous use of histogram. readRequestUnitCost = prometheus.NewCounterVec( @@ -111,7 +103,6 @@ var ( ) func init() { - prometheus.MustRegister(serverInfo) prometheus.MustRegister(readRequestUnitCost) prometheus.MustRegister(writeRequestUnitCost) prometheus.MustRegister(sqlLayerRequestUnitCost) diff --git a/pkg/mcs/resourcemanager/server/server.go b/pkg/mcs/resourcemanager/server/server.go index 43d426bfc40..2d02fd00434 100644 --- a/pkg/mcs/resourcemanager/server/server.go +++ b/pkg/mcs/resourcemanager/server/server.go @@ -19,6 +19,7 @@ import ( "net/http" "os" "os/signal" + "runtime" "strconv" "sync" "sync/atomic" @@ -294,7 +295,8 @@ func (s *Server) startServer() (err error) { log.Info("init cluster id", zap.Uint64("cluster-id", s.clusterID)) // The independent Resource Manager service still reuses PD version info since PD and Resource Manager are just // different service modes provided by the same pd-server binary - serverInfo.WithLabelValues(versioninfo.PDReleaseVersion, versioninfo.PDGitHash).Set(float64(time.Now().Unix())) + bs.ServerInfoGauge.WithLabelValues(versioninfo.PDReleaseVersion, versioninfo.PDGitHash).Set(float64(time.Now().Unix())) + bs.ServerMaxProcsGauge.Set(float64(runtime.GOMAXPROCS(0))) uniqueName := s.cfg.GetAdvertiseListenAddr() uniqueID := memberutil.GenerateUniqueID(uniqueName) diff --git a/pkg/mcs/scheduling/server/server.go b/pkg/mcs/scheduling/server/server.go index 8ee8b81ae47..8013f1d0e7b 100644 --- a/pkg/mcs/scheduling/server/server.go +++ b/pkg/mcs/scheduling/server/server.go @@ -20,6 +20,7 @@ import ( "net/http" "os" "os/signal" + "runtime" "strconv" "sync" "sync/atomic" @@ -409,8 +410,8 @@ func (s *Server) startServer() (err error) { log.Info("init cluster id", zap.Uint64("cluster-id", s.clusterID)) // The independent Scheduling service still reuses PD version info since PD and Scheduling are just // different service modes provided by the same pd-server binary - serverInfo.WithLabelValues(versioninfo.PDReleaseVersion, versioninfo.PDGitHash).Set(float64(time.Now().Unix())) - + bs.ServerInfoGauge.WithLabelValues(versioninfo.PDReleaseVersion, versioninfo.PDGitHash).Set(float64(time.Now().Unix())) + bs.ServerMaxProcsGauge.Set(float64(runtime.GOMAXPROCS(0))) s.serviceID = &discovery.ServiceRegistryEntry{ServiceAddr: s.cfg.AdvertiseListenAddr} uniqueName := s.cfg.GetAdvertiseListenAddr() uniqueID := memberutil.GenerateUniqueID(uniqueName) diff --git a/pkg/mcs/tso/server/metrics.go b/pkg/mcs/tso/server/metrics.go index 288d650e1e7..4845fbc39a2 100644 --- a/pkg/mcs/tso/server/metrics.go +++ b/pkg/mcs/tso/server/metrics.go @@ -35,14 +35,6 @@ var ( Help: "Record critical metadata.", }, []string{"type"}) - serverInfo = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: namespace, - Subsystem: "server", - Name: "info", - Help: "Indicate the tso server info, and the value is the start timestamp (s).", - }, []string{"version", "hash"}) - tsoHandleDuration = prometheus.NewHistogramVec( prometheus.HistogramOpts{ Namespace: namespace, @@ -56,6 +48,5 @@ var ( func init() { prometheus.MustRegister(timeJumpBackCounter) prometheus.MustRegister(metaDataGauge) - prometheus.MustRegister(serverInfo) prometheus.MustRegister(tsoHandleDuration) } diff --git a/pkg/mcs/tso/server/server.go b/pkg/mcs/tso/server/server.go index 1a2430477d8..55473efc8bb 100644 --- a/pkg/mcs/tso/server/server.go +++ b/pkg/mcs/tso/server/server.go @@ -20,6 +20,7 @@ import ( "net/http" "os" "os/signal" + "runtime" "strconv" "sync" "sync/atomic" @@ -359,7 +360,8 @@ func (s *Server) startServer() (err error) { metaDataGauge.WithLabelValues(fmt.Sprintf("cluster%d", s.clusterID)).Set(0) // The independent TSO service still reuses PD version info since PD and TSO are just // different service modes provided by the same pd-server binary - serverInfo.WithLabelValues(versioninfo.PDReleaseVersion, versioninfo.PDGitHash).Set(float64(time.Now().Unix())) + bs.ServerInfoGauge.WithLabelValues(versioninfo.PDReleaseVersion, versioninfo.PDGitHash).Set(float64(time.Now().Unix())) + bs.ServerMaxProcsGauge.Set(float64(runtime.GOMAXPROCS(0))) // Initialize the TSO service. s.serverLoopCtx, s.serverLoopCancel = context.WithCancel(s.Context()) diff --git a/server/metrics.go b/server/metrics.go index 54c5830dc52..e06a0cc20dd 100644 --- a/server/metrics.go +++ b/server/metrics.go @@ -136,14 +136,6 @@ var ( Buckets: prometheus.ExponentialBuckets(0.0001, 2, 29), // 0.1ms ~ 7hours }, []string{"address", "store"}) - serverInfo = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: "pd", - Subsystem: "server", - Name: "info", - Help: "Indicate the pd server info, and the value is the start timestamp (s).", - }, []string{"version", "hash"}) - serviceAuditHistogram = prometheus.NewHistogramVec( prometheus.HistogramOpts{ Namespace: "pd", @@ -152,13 +144,6 @@ var ( Help: "PD server service handling audit", Buckets: prometheus.DefBuckets, }, []string{"service", "method", "caller_id", "ip"}) - serverMaxProcs = prometheus.NewGauge( - prometheus.GaugeOpts{ - Namespace: "pd", - Subsystem: "service", - Name: "maxprocs", - Help: "The value of GOMAXPROCS.", - }) forwardFailCounter = prometheus.NewCounterVec( prometheus.CounterOpts{ @@ -181,11 +166,9 @@ func init() { prometheus.MustRegister(tsoHandleDuration) prometheus.MustRegister(regionHeartbeatHandleDuration) prometheus.MustRegister(storeHeartbeatHandleDuration) - prometheus.MustRegister(serverInfo) prometheus.MustRegister(bucketReportCounter) prometheus.MustRegister(bucketReportLatency) prometheus.MustRegister(serviceAuditHistogram) prometheus.MustRegister(bucketReportInterval) - prometheus.MustRegister(serverMaxProcs) prometheus.MustRegister(forwardFailCounter) } diff --git a/server/server.go b/server/server.go index fcf71922a09..21ab983b3d1 100644 --- a/server/server.go +++ b/server/server.go @@ -43,6 +43,7 @@ import ( "github.com/pingcap/log" "github.com/pingcap/sysutil" "github.com/tikv/pd/pkg/audit" + bs "github.com/tikv/pd/pkg/basicserver" "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/encryption" "github.com/tikv/pd/pkg/errs" @@ -428,7 +429,7 @@ func (s *Server) startServer(ctx context.Context) error { log.Info("init cluster id", zap.Uint64("cluster-id", s.clusterID)) // It may lose accuracy if use float64 to store uint64. So we store the cluster id in label. metadataGauge.WithLabelValues(fmt.Sprintf("cluster%d", s.clusterID)).Set(0) - serverInfo.WithLabelValues(versioninfo.PDReleaseVersion, versioninfo.PDGitHash).Set(float64(time.Now().Unix())) + bs.ServerInfoGauge.WithLabelValues(versioninfo.PDReleaseVersion, versioninfo.PDGitHash).Set(float64(time.Now().Unix())) s.rootPath = endpoint.PDRootPath(s.clusterID) s.member.InitMemberInfo(s.cfg.AdvertiseClientUrls, s.cfg.AdvertisePeerUrls, s.Name(), s.rootPath) @@ -504,7 +505,7 @@ func (s *Server) startServer(ctx context.Context) error { // Server has started. atomic.StoreInt64(&s.isRunning, 1) - serverMaxProcs.Set(float64(runtime.GOMAXPROCS(0))) + bs.ServerMaxProcsGauge.Set(float64(runtime.GOMAXPROCS(0))) return nil } diff --git a/tests/integrations/mcs/resourcemanager/server_test.go b/tests/integrations/mcs/resourcemanager/server_test.go index d3837ad15f3..4e1fb018d56 100644 --- a/tests/integrations/mcs/resourcemanager/server_test.go +++ b/tests/integrations/mcs/resourcemanager/server_test.go @@ -101,7 +101,7 @@ func TestResourceManagerServer(t *testing.T) { re.Equal(http.StatusOK, resp.StatusCode) respBytes, err := io.ReadAll(resp.Body) re.NoError(err) - re.Contains(string(respBytes), "resource_manager_server_info") + re.Contains(string(respBytes), "pd_server_info") } // Test status handler diff --git a/tests/integrations/mcs/scheduling/api_test.go b/tests/integrations/mcs/scheduling/api_test.go index 177f959b953..2572ae459d5 100644 --- a/tests/integrations/mcs/scheduling/api_test.go +++ b/tests/integrations/mcs/scheduling/api_test.go @@ -566,7 +566,7 @@ func (suite *apiTestSuite) checkMetrics(cluster *tests.TestCluster) { re.Equal(http.StatusOK, resp.StatusCode) respBytes, err := io.ReadAll(resp.Body) re.NoError(err) - re.Contains(string(respBytes), "scheduling_server_info") + re.Contains(string(respBytes), "pd_server_info") } func (suite *apiTestSuite) TestStatus() { diff --git a/tests/integrations/mcs/tso/api_test.go b/tests/integrations/mcs/tso/api_test.go index a5d68cfa90f..b72da5ab7f6 100644 --- a/tests/integrations/mcs/tso/api_test.go +++ b/tests/integrations/mcs/tso/api_test.go @@ -256,7 +256,7 @@ func (suite *tsoAPITestSuite) TestMetrics() { re.Equal(http.StatusOK, resp.StatusCode) respBytes, err := io.ReadAll(resp.Body) re.NoError(err) - re.Contains(string(respBytes), "tso_server_info") + re.Contains(string(respBytes), "pd_server_info") } func (suite *tsoAPITestSuite) TestStatus() { From ad842dbc4f435ea1337b2d097d9a28ef7090a22c Mon Sep 17 00:00:00 2001 From: glorv Date: Mon, 25 Dec 2023 15:21:12 +0800 Subject: [PATCH 129/137] fix go.mod Signed-off-by: glorv --- tests/integrations/tso/go.mod | 22 +++++++++++---------- tests/integrations/tso/go.sum | 36 +++++++++++++++++++++-------------- tools/pd-api-bench/go.mod | 9 ++++----- tools/pd-api-bench/go.sum | 12 ++++++------ 4 files changed, 44 insertions(+), 35 deletions(-) diff --git a/tests/integrations/tso/go.mod b/tests/integrations/tso/go.mod index 3d95b558ebf..92921e6cd6c 100644 --- a/tests/integrations/tso/go.mod +++ b/tests/integrations/tso/go.mod @@ -9,16 +9,16 @@ replace ( ) // reset grpc and protobuf deps in order to import client and server at the same time -replace google.golang.org/grpc v1.54.0 => google.golang.org/grpc v1.26.0 +replace google.golang.org/grpc v1.59.0 => google.golang.org/grpc v1.26.0 require ( github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c - github.com/pingcap/kvproto v0.0.0-20231018065736-c0689aded40c + github.com/pingcap/kvproto v0.0.0-20231222062942-c0c73f41d0b2 github.com/stretchr/testify v1.8.4 github.com/tikv/pd v0.0.0-00010101000000-000000000000 github.com/tikv/pd/client v0.0.0-20231101084237-a1a1eea8dafd github.com/tikv/pd/tests/integrations/mcs v0.0.0-00010101000000-000000000000 - google.golang.org/grpc v1.54.0 + google.golang.org/grpc v1.59.0 ) require ( @@ -57,7 +57,7 @@ require ( github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/docker/go-units v0.4.0 // indirect - github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/elliotchance/pie/v2 v2.1.0 // indirect github.com/fogleman/gg v1.3.0 // indirect github.com/gabriel-vasile/mimetype v1.4.2 // indirect @@ -127,7 +127,7 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b // indirect github.com/prometheus/client_golang v1.11.1 // indirect - github.com/prometheus/client_model v0.2.0 // indirect + github.com/prometheus/client_model v0.4.0 // indirect github.com/prometheus/common v0.26.0 // indirect github.com/prometheus/procfs v0.6.0 // indirect github.com/rs/cors v1.7.0 // indirect @@ -170,14 +170,16 @@ require ( golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4 // indirect golang.org/x/image v0.10.0 // indirect golang.org/x/net v0.18.0 // indirect - golang.org/x/oauth2 v0.4.0 // indirect - golang.org/x/sync v0.1.0 // indirect + golang.org/x/oauth2 v0.11.0 // indirect + golang.org/x/sync v0.3.0 // indirect golang.org/x/sys v0.15.0 // indirect golang.org/x/text v0.14.0 // indirect - golang.org/x/time v0.1.0 // indirect - golang.org/x/tools v0.6.0 // indirect + golang.org/x/time v0.3.0 // indirect + golang.org/x/tools v0.10.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect + google.golang.org/genproto v0.0.0-20231030173426-d783a09b4405 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17 // indirect google.golang.org/protobuf v1.31.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/tests/integrations/tso/go.sum b/tests/integrations/tso/go.sum index 668c1bce1a4..a00d2941582 100644 --- a/tests/integrations/tso/go.sum +++ b/tests/integrations/tso/go.sum @@ -115,8 +115,9 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4 h1:qk/FSDDxo05wdJH28W+p5yivv7LuLYLRXPPD8KQCtZs= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385 h1:clC1lXBpe2kTj2VHdaIu9ajZQe4kcEY9j0NsnDDBZ3o= github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= github.com/elliotchance/pie/v2 v2.1.0 h1:KEVAAzxYxTyFs4hvebFZVzBdEo3YeMzl2HYDWn+P3F4= @@ -204,8 +205,8 @@ github.com/golang-sql/sqlexp v0.1.0/go.mod h1:J4ad9Vo8ZCWQ2GMrC4UCQy1JpCbwU9m3EO github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE= -github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ= +github.com/golang/glog v1.1.2 h1:DVjP2PbBOzHyzA+dn3WhHIq4NdVu3Q+pvivFICf/7fo= +github.com/golang/glog v1.1.2/go.mod h1:zR+okUeTbrL6EL3xHUDxZuEtGv04p5shwip1+mL/rLQ= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef h1:veQD95Isof8w9/WXiA+pa3tz3fJXkt5B7QaRBrM62gk= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -421,8 +422,8 @@ github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ue github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c h1:CgbKAHto5CQgWM9fSBIvaxsJHuGP0uM74HXtv3MyyGQ= github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c/go.mod h1:4qGtCB0QK0wBzKtFEGDhxXnSnbQApw1gc9siScUl8ew= github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= -github.com/pingcap/kvproto v0.0.0-20231018065736-c0689aded40c h1:5cpE29kMHjc8fv+mRiXbLTDfoZHiX5BTK6knVWlWvqk= -github.com/pingcap/kvproto v0.0.0-20231018065736-c0689aded40c/go.mod h1:r0q/CFcwvyeRhKtoqzmWMBebrtpIziQQ9vR+JKh1knc= +github.com/pingcap/kvproto v0.0.0-20231222062942-c0c73f41d0b2 h1:364A6VCS+l0oHBKZKotX9LzmfEtIO/NTccTIQcPp3Ug= +github.com/pingcap/kvproto v0.0.0-20231222062942-c0c73f41d0b2/go.mod h1:rXxWk2UnwfUhLXha1jxRWPADw9eMZGWEWCg92Tgmb/8= github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7/go.mod h1:8AanEdAHATuRurdGxZXBz0At+9avep+ub7U1AGYLIMM= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8IDP+SZrdhV1Kibl9KrHxJ9eciw= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= @@ -451,8 +452,9 @@ github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqr github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= +github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= @@ -690,8 +692,8 @@ golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.4.0 h1:NF0gk8LVPg1Ml7SSbGyySuoxdsXitj7TvgvuRxIMc/M= -golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec= +golang.org/x/oauth2 v0.11.0 h1:vPL4xzxBM4niKCW6g9whtaWVXTJf1U5e4aZxxFx/gbU= +golang.org/x/oauth2 v0.11.0/go.mod h1:LdF7O/8bLR/qWK9DrpXmbHLTouvRHK0SgJl0GmDBchk= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -701,8 +703,9 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -758,8 +761,8 @@ golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.1.0 h1:xYY+Bajn2a7VBmTM5GikTmnK8ZuX8YgnQCqZpbBNtmA= -golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -783,8 +786,9 @@ golang.org/x/tools v0.0.0-20210112230658-8b4aab62c064/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.10.0 h1:tvDr/iQoUqNdohiYm0LmmKcBk+q86lb9EprIUFhHHGg= +golang.org/x/tools v0.10.0/go.mod h1:UJwyiVBsOA2uwvK/e5OY3GTpDUJriEd+/YlqAwLPmyM= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -797,8 +801,12 @@ google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoA google.golang.org/genproto v0.0.0-20181004005441-af9cb2a35e7f/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= -google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= +google.golang.org/genproto v0.0.0-20231030173426-d783a09b4405 h1:I6WNifs6pF9tNdSob2W24JtyxIYjzFB9qDlpUC76q+U= +google.golang.org/genproto v0.0.0-20231030173426-d783a09b4405/go.mod h1:3WDQMjmJk36UQhjQ89emUzb1mdaHcPeeAh4SCBKznB4= +google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b h1:CIC2YMXmIhYw6evmhPxBKJ4fmLbOFtXQN/GV3XOZR8k= +google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b/go.mod h1:IBQ646DjkDkvUIsVq/cc03FUFQ9wbZu7yE396YcL870= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17 h1:Jyp0Hsi0bmHXG6k9eATXoYtjd6e2UzZ1SCn/wIupY14= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17/go.mod h1:oQ5rr10WTTMvP4A36n8JpR1OrO1BEiV4f78CneXZxkA= google.golang.org/grpc v0.0.0-20180607172857-7a6a684ca69e/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= diff --git a/tools/pd-api-bench/go.mod b/tools/pd-api-bench/go.mod index cae22f335bf..4a2c15be31e 100644 --- a/tools/pd-api-bench/go.mod +++ b/tools/pd-api-bench/go.mod @@ -5,7 +5,7 @@ go 1.21 require ( github.com/tikv/pd/client v0.0.0-20231101084237-a1a1eea8dafd go.uber.org/zap v1.24.0 - google.golang.org/grpc v1.54.0 + google.golang.org/grpc v1.59.0 ) require ( @@ -14,12 +14,11 @@ require ( github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.3 // indirect - github.com/google/go-cmp v0.5.9 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c // indirect github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 // indirect - github.com/pingcap/kvproto v0.0.0-20231018065736-c0689aded40c // indirect + github.com/pingcap/kvproto v0.0.0-20231222062942-c0c73f41d0b2 // indirect github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 // indirect github.com/prometheus/client_golang v1.11.1 // indirect github.com/prometheus/client_model v0.2.0 // indirect @@ -32,12 +31,12 @@ require ( golang.org/x/net v0.18.0 // indirect golang.org/x/sys v0.14.0 // indirect golang.org/x/text v0.14.0 // indirect - google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17 // indirect google.golang.org/protobuf v1.31.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect ) -replace google.golang.org/grpc v1.54.0 => google.golang.org/grpc v1.26.0 +replace google.golang.org/grpc v1.59.0 => google.golang.org/grpc v1.26.0 replace ( github.com/tikv/pd => ../.. diff --git a/tools/pd-api-bench/go.sum b/tools/pd-api-bench/go.sum index a59256c50d5..27cba94afce 100644 --- a/tools/pd-api-bench/go.sum +++ b/tools/pd-api-bench/go.sum @@ -36,8 +36,8 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE= -github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ= +github.com/golang/glog v1.1.2 h1:DVjP2PbBOzHyzA+dn3WhHIq4NdVu3Q+pvivFICf/7fo= +github.com/golang/glog v1.1.2/go.mod h1:zR+okUeTbrL6EL3xHUDxZuEtGv04p5shwip1+mL/rLQ= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -91,8 +91,8 @@ github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c h1:xpW9bvK+HuuTm github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg= github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 h1:C3N3itkduZXDZFh4N3vQ5HEtld3S+Y+StULhWVvumU0= github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00/go.mod h1:4qGtCB0QK0wBzKtFEGDhxXnSnbQApw1gc9siScUl8ew= -github.com/pingcap/kvproto v0.0.0-20231018065736-c0689aded40c h1:5cpE29kMHjc8fv+mRiXbLTDfoZHiX5BTK6knVWlWvqk= -github.com/pingcap/kvproto v0.0.0-20231018065736-c0689aded40c/go.mod h1:r0q/CFcwvyeRhKtoqzmWMBebrtpIziQQ9vR+JKh1knc= +github.com/pingcap/kvproto v0.0.0-20231222062942-c0c73f41d0b2 h1:364A6VCS+l0oHBKZKotX9LzmfEtIO/NTccTIQcPp3Ug= +github.com/pingcap/kvproto v0.0.0-20231222062942-c0c73f41d0b2/go.mod h1:rXxWk2UnwfUhLXha1jxRWPADw9eMZGWEWCg92Tgmb/8= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8IDP+SZrdhV1Kibl9KrHxJ9eciw= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -230,8 +230,8 @@ google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9Ywl google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= -google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17 h1:Jyp0Hsi0bmHXG6k9eATXoYtjd6e2UzZ1SCn/wIupY14= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17/go.mod h1:oQ5rr10WTTMvP4A36n8JpR1OrO1BEiV4f78CneXZxkA= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.26.0 h1:2dTRdpdFEEhJYQD8EMLB61nnrzSCTbG38PhqdhvOltg= From 32299aa510360ea94e6457ca3b2f7e55b1b07aee Mon Sep 17 00:00:00 2001 From: glorv Date: Mon, 25 Dec 2023 15:35:49 +0800 Subject: [PATCH 130/137] fix check Signed-off-by: glorv --- .../resourcemanager/server/grpc_service.go | 84 +++++++++---------- pkg/mcs/resourcemanager/server/manager.go | 17 ++-- .../resourcemanager/server/resource_group.go | 8 +- 3 files changed, 52 insertions(+), 57 deletions(-) diff --git a/pkg/mcs/resourcemanager/server/grpc_service.go b/pkg/mcs/resourcemanager/server/grpc_service.go index 2322cb5bff3..cf985a14764 100644 --- a/pkg/mcs/resourcemanager/server/grpc_service.go +++ b/pkg/mcs/resourcemanager/server/grpc_service.go @@ -184,55 +184,49 @@ func (s *Service) AcquireTokenBuckets(stream rmpb.ResourceManager_AcquireTokenBu resps := &rmpb.TokenBucketsResponse{} for _, req := range request.Requests { resourceGroupName := req.GetResourceGroupName() - var err error // Get the resource group from manager to acquire token buckets. - s.manager.WithMutableResourceGroup(resourceGroupName, func(rg *ResourceGroup) { - if rg == nil { - log.Warn("resource group not found", zap.String("resource-group", resourceGroupName)) - return - } - // Send the consumption to update the metrics. - isBackground := req.GetIsBackground() - isTiFlash := req.GetIsTiflash() - if isBackground && isTiFlash { - err = errors.New("background and tiflash cannot be true at the same time") - return - } - s.manager.consumptionDispatcher <- struct { - resourceGroupName string - *rmpb.Consumption - isBackground bool - isTiFlash bool - }{resourceGroupName, req.GetConsumptionSinceLastRequest(), isBackground, isTiFlash} - if isBackground { - return - } - now := time.Now() - resp := &rmpb.TokenBucketResponse{ - ResourceGroupName: rg.Name, - } - switch rg.Mode { - case rmpb.GroupMode_RUMode: - var tokens *rmpb.GrantedRUTokenBucket - for _, re := range req.GetRuItems().GetRequestRU() { - if re.Type == rmpb.RequestUnitType_RU { - tokens = rg.RequestRU(now, re.Value, targetPeriodMs, clientUniqueID) - } - if tokens == nil { - continue - } - resp.GrantedRUTokens = append(resp.GrantedRUTokens, tokens) + rg := s.manager.GetMutableResourceGroup(resourceGroupName) + if rg == nil { + log.Warn("resource group not found", zap.String("resource-group", resourceGroupName)) + continue + } + // Send the consumption to update the metrics. + isBackground := req.GetIsBackground() + isTiFlash := req.GetIsTiflash() + if isBackground && isTiFlash { + return errors.New("background and tiflash cannot be true at the same time") + } + s.manager.consumptionDispatcher <- struct { + resourceGroupName string + *rmpb.Consumption + isBackground bool + isTiFlash bool + }{resourceGroupName, req.GetConsumptionSinceLastRequest(), isBackground, isTiFlash} + if isBackground { + continue + } + now := time.Now() + resp := &rmpb.TokenBucketResponse{ + ResourceGroupName: rg.Name, + } + switch rg.Mode { + case rmpb.GroupMode_RUMode: + var tokens *rmpb.GrantedRUTokenBucket + for _, re := range req.GetRuItems().GetRequestRU() { + if re.Type == rmpb.RequestUnitType_RU { + tokens = rg.RequestRU(now, re.Value, targetPeriodMs, clientUniqueID) + } + if tokens == nil { + continue } - case rmpb.GroupMode_RawMode: - log.Warn("not supports the resource type", zap.String("resource-group", resourceGroupName), zap.String("mode", rmpb.GroupMode_name[int32(rmpb.GroupMode_RawMode)])) - return + resp.GrantedRUTokens = append(resp.GrantedRUTokens, tokens) } - log.Debug("finish token request from", zap.String("resource-group", resourceGroupName)) - resps.Responses = append(resps.Responses, resp) - }) - if err != nil { - return err + case rmpb.GroupMode_RawMode: + log.Warn("not supports the resource type", zap.String("resource-group", resourceGroupName), zap.String("mode", rmpb.GroupMode_name[int32(rmpb.GroupMode_RawMode)])) + continue } + log.Debug("finish token request from", zap.String("resource-group", resourceGroupName)) + resps.Responses = append(resps.Responses, resp) } stream.Send(resps) } diff --git a/pkg/mcs/resourcemanager/server/manager.go b/pkg/mcs/resourcemanager/server/manager.go index 2e50fa79b60..a4b49b38062 100644 --- a/pkg/mcs/resourcemanager/server/manager.go +++ b/pkg/mcs/resourcemanager/server/manager.go @@ -287,11 +287,14 @@ func (m *Manager) GetResourceGroup(name string, withStats bool) *ResourceGroup { return nil } -// GetMutableResourceGroup apply target function on target resource group. -func (m *Manager) WithMutableResourceGroup(name string, f func(*ResourceGroup)) { +// GetMutableResourceGroup returns a mutable resource group. +func (m *Manager) GetMutableResourceGroup(name string) *ResourceGroup { m.RLock() defer m.RUnlock() - f(m.groups[name]) + if group, ok := m.groups[name]; ok { + return group + } + return nil } // GetResourceGroupList returns copies of resource group list. @@ -411,11 +414,9 @@ func (m *Manager) backgroundMetricsFlush(ctx context.Context) { m.consumptionRecord[name] = time.Now() // TODO: maybe we need to distinguish background ru. - m.WithMutableResourceGroup(name, func(rg *ResourceGroup) { - if rg != nil { - rg.UpdateRUConsumption(consumptionInfo.Consumption) - } - }) + if rg := m.GetMutableResourceGroup(name); rg != nil { + rg.UpdateRUConsumption(consumptionInfo.Consumption) + } case <-cleanUpTicker.C: // Clean up the metrics that have not been updated for a long time. for name, lastTime := range m.consumptionRecord { diff --git a/pkg/mcs/resourcemanager/server/resource_group.go b/pkg/mcs/resourcemanager/server/resource_group.go index d1449c11898..09f8a33de9f 100644 --- a/pkg/mcs/resourcemanager/server/resource_group.go +++ b/pkg/mcs/resourcemanager/server/resource_group.go @@ -179,7 +179,8 @@ func (rg *ResourceGroup) IntoProtoResourceGroup() *rmpb.ResourceGroup { } if rg.RUConsumption != nil { - group.RUStats = &*rg.RUConsumption + consumption := *rg.RUConsumption + group.RUStats = &consumption } return group case rmpb.GroupMode_RawMode: // Raw mode @@ -214,9 +215,10 @@ func (rg *ResourceGroup) GetGroupStates() *GroupStates { switch rg.Mode { case rmpb.GroupMode_RUMode: // RU mode + consumption := *rg.RUConsumption tokens := &GroupStates{ RU: rg.RUSettings.RU.GroupTokenBucketState.Clone(), - RUConsumption: &*rg.RUConsumption, + RUConsumption: &consumption, } return tokens case rmpb.GroupMode_RawMode: // Raw mode @@ -245,7 +247,6 @@ func (rg *ResourceGroup) SetStatesIntoResourceGroup(states *GroupStates) { func (rg *ResourceGroup) UpdateRUConsumption(c *rmpb.Consumption) { rg.Lock() defer rg.Unlock() - rc := rg.RUConsumption rc.RRU += c.RRU rc.WRU += c.WRU @@ -255,7 +256,6 @@ func (rg *ResourceGroup) UpdateRUConsumption(c *rmpb.Consumption) { rc.SqlLayerCpuTimeMs += c.SqlLayerCpuTimeMs rc.KvReadRpcCount += c.KvReadRpcCount rc.KvWriteRpcCount += c.KvWriteRpcCount - } // persistStates persists the resource group tokens. From 22ff77459cb1ec48709c81674408250e9a437bfc Mon Sep 17 00:00:00 2001 From: glorv Date: Mon, 25 Dec 2023 15:46:39 +0800 Subject: [PATCH 131/137] exclude grpc CallbackSerializer Signed-off-by: glorv --- client/testutil/leak.go | 1 + 1 file changed, 1 insertion(+) diff --git a/client/testutil/leak.go b/client/testutil/leak.go index 10345725171..ec2a6543941 100644 --- a/client/testutil/leak.go +++ b/client/testutil/leak.go @@ -27,4 +27,5 @@ var LeakOptions = []goleak.Option{ goleak.IgnoreTopFunction("internal/poll.runtime_pollWait"), goleak.IgnoreTopFunction("google.golang.org/grpc/internal/transport.(*controlBuffer).get"), goleak.IgnoreTopFunction("google.golang.org/grpc/internal/transport.(*http2Server).keepalive"), + goleak.IgnoreTopFunction("google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run"), } From 5170aee4e53a498fb3911f10e15deeabb96af14e Mon Sep 17 00:00:00 2001 From: glorv Date: Mon, 25 Dec 2023 15:52:21 +0800 Subject: [PATCH 132/137] fix function comment Signed-off-by: glorv --- client/resource_manager_client.go | 1 + 1 file changed, 1 insertion(+) diff --git a/client/resource_manager_client.go b/client/resource_manager_client.go index ce7d7f10393..433d17ceeee 100644 --- a/client/resource_manager_client.go +++ b/client/resource_manager_client.go @@ -67,6 +67,7 @@ type GetResourceGroupOp struct { // GetResourceGroupOption configures GetResourceGroupOp. type GetResourceGroupOption func(*GetResourceGroupOp) +// WithRUStats specifies to return resource group with ru statistics data. func WithRUStats(op *GetResourceGroupOp) { op.withRUStats = true } From 4d985b2a24a1431900997db9b699d6031a26676a Mon Sep 17 00:00:00 2001 From: Yongbo Jiang Date: Tue, 26 Dec 2023 16:13:57 +0800 Subject: [PATCH 133/137] client: use ServiceClient for discovery (#7611) ref tikv/pd#7576 Signed-off-by: Cabinfever_B Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- client/client.go | 103 +--------- client/pd_service_discovery.go | 251 +++++++++++++++++++---- client/tso_service_discovery.go | 5 + tests/integrations/client/client_test.go | 150 +++++++------- 4 files changed, 305 insertions(+), 204 deletions(-) diff --git a/client/client.go b/client/client.go index 0ae362d06a8..4e03e5e3507 100644 --- a/client/client.go +++ b/client/client.go @@ -17,29 +17,22 @@ package pd import ( "context" "fmt" - "math/rand" "runtime/trace" "strings" "sync" - "sync/atomic" "time" "github.com/opentracing/opentracing-go" "github.com/pingcap/errors" - "github.com/pingcap/failpoint" "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/kvproto/pkg/pdpb" "github.com/pingcap/log" "github.com/prometheus/client_golang/prometheus" "github.com/tikv/pd/client/errs" - "github.com/tikv/pd/client/grpcutil" "github.com/tikv/pd/client/tlsutil" "github.com/tikv/pd/client/tsoutil" "go.uber.org/zap" "google.golang.org/grpc" - "google.golang.org/grpc/codes" - healthpb "google.golang.org/grpc/health/grpc_health_v1" - "google.golang.org/grpc/status" ) const ( @@ -217,9 +210,6 @@ func WithAllowFollowerHandle() GetRegionOption { return func(op *GetRegionOp) { op.allowFollowerHandle = true } } -// LeaderHealthCheckInterval might be changed in the unit to shorten the testing time. -var LeaderHealthCheckInterval = time.Second - var ( // errUnmatchedClusterID is returned when found a PD with a different cluster ID. errUnmatchedClusterID = errors.New("[pd] unmatched cluster id") @@ -316,7 +306,6 @@ type client struct { // For internal usage. updateTokenConnectionCh chan struct{} - leaderNetworkFailure int32 ctx context.Context cancel context.CancelFunc @@ -575,10 +564,6 @@ func (c *client) setup() error { // Create dispatchers c.createTokenDispatcher() - - // Start the daemons. - c.wg.Add(1) - go c.leaderCheckLoop() return nil } @@ -719,46 +704,6 @@ func (c *client) UpdateOption(option DynamicOption, value interface{}) error { return nil } -func (c *client) leaderCheckLoop() { - defer c.wg.Done() - - leaderCheckLoopCtx, leaderCheckLoopCancel := context.WithCancel(c.ctx) - defer leaderCheckLoopCancel() - - ticker := time.NewTicker(LeaderHealthCheckInterval) - defer ticker.Stop() - - for { - select { - case <-c.ctx.Done(): - return - case <-ticker.C: - c.checkLeaderHealth(leaderCheckLoopCtx) - } - } -} - -func (c *client) checkLeaderHealth(ctx context.Context) { - ctx, cancel := context.WithTimeout(ctx, c.option.timeout) - defer cancel() - if client := c.pdSvcDiscovery.GetServingEndpointClientConn(); client != nil { - healthCli := healthpb.NewHealthClient(client) - resp, err := healthCli.Check(ctx, &healthpb.HealthCheckRequest{Service: ""}) - failpoint.Inject("unreachableNetwork1", func() { - resp = nil - err = status.New(codes.Unavailable, "unavailable").Err() - }) - rpcErr, ok := status.FromError(err) - if (ok && isNetworkError(rpcErr.Code())) || resp.GetStatus() != healthpb.HealthCheckResponse_SERVING { - atomic.StoreInt32(&(c.leaderNetworkFailure), int32(1)) - } else { - atomic.StoreInt32(&(c.leaderNetworkFailure), int32(0)) - } - } else { - atomic.StoreInt32(&(c.leaderNetworkFailure), int32(1)) - } -} - func (c *client) GetAllMembers(ctx context.Context) ([]*pdpb.Member, error) { start := time.Now() defer func() { cmdDurationGetAllMembers.Observe(time.Since(start).Seconds()) }() @@ -778,50 +723,14 @@ func (c *client) GetAllMembers(ctx context.Context) ([]*pdpb.Member, error) { return resp.GetMembers(), nil } -// leaderClient gets the client of current PD leader. -func (c *client) leaderClient() pdpb.PDClient { - if client := c.pdSvcDiscovery.GetServingEndpointClientConn(); client != nil { - return pdpb.NewPDClient(client) - } - return nil -} - -// backupClientConn gets a grpc client connection of the current reachable and healthy -// backup service endpoints randomly. Backup service endpoints are followers in a -// quorum-based cluster or secondaries in a primary/secondary configured cluster. -func (c *client) backupClientConn() (*grpc.ClientConn, string) { - addrs := c.pdSvcDiscovery.GetBackupAddrs() - if len(addrs) < 1 { - return nil, "" - } - var ( - cc *grpc.ClientConn - err error - ) - for i := 0; i < len(addrs); i++ { - addr := addrs[rand.Intn(len(addrs))] - if cc, err = c.pdSvcDiscovery.GetOrCreateGRPCConn(addr); err != nil { - continue - } - healthCtx, healthCancel := context.WithTimeout(c.ctx, c.option.timeout) - resp, err := healthpb.NewHealthClient(cc).Check(healthCtx, &healthpb.HealthCheckRequest{Service: ""}) - healthCancel() - if err == nil && resp.GetStatus() == healthpb.HealthCheckResponse_SERVING { - return cc, addr - } - } - return nil, "" -} - +// getClientAndContext returns the leader pd client and the original context. If leader is unhealthy, it returns +// follower pd client and the context which holds forward information. func (c *client) getClientAndContext(ctx context.Context) (pdpb.PDClient, context.Context) { - if c.option.enableForwarding && atomic.LoadInt32(&c.leaderNetworkFailure) == 1 { - backupClientConn, addr := c.backupClientConn() - if backupClientConn != nil { - log.Debug("[pd] use follower client", zap.String("addr", addr)) - return pdpb.NewPDClient(backupClientConn), grpcutil.BuildForwardContext(ctx, c.GetLeaderAddr()) - } + serviceClient := c.pdSvcDiscovery.GetServiceClient() + if serviceClient == nil { + return nil, ctx } - return c.leaderClient(), ctx + return pdpb.NewPDClient(serviceClient.GetClientConn()), serviceClient.BuildGRPCTargetContext(ctx, true) } func (c *client) GetTSAsync(ctx context.Context) TSFuture { diff --git a/client/pd_service_discovery.go b/client/pd_service_discovery.go index 33dda1ad282..f9bdf888b9f 100644 --- a/client/pd_service_discovery.go +++ b/client/pd_service_discovery.go @@ -46,6 +46,16 @@ const ( updateMemberBackOffBaseTime = 100 * time.Millisecond ) +// MemberHealthCheckInterval might be changed in the unit to shorten the testing time. +var MemberHealthCheckInterval = time.Second + +type apiKind int + +const ( + forwardAPIKind apiKind = iota + apiKindCount +) + type serviceType int const ( @@ -81,6 +91,10 @@ type ServiceDiscovery interface { // endpoints. Backup service endpoints are followers in a quorum-based cluster or // secondaries in a primary/secondary configured cluster. GetBackupAddrs() []string + // GetServiceClient tries to get the leader/primary ServiceClient. + // If the leader ServiceClient meets network problem, + // it returns a follower/secondary ServiceClient which can forward the request to leader. + GetServiceClient() ServiceClient // GetOrCreateGRPCConn returns the corresponding grpc client connection of the given addr GetOrCreateGRPCConn(addr string) (*grpc.ClientConn, error) // ScheduleCheckMemberChanged is used to trigger a check to see if there is any membership change @@ -134,12 +148,16 @@ type pdServiceClient struct { } func newPDServiceClient(addr, leaderAddr string, conn *grpc.ClientConn, isLeader bool) ServiceClient { - return &pdServiceClient{ + cli := &pdServiceClient{ addr: addr, conn: conn, isLeader: isLeader, leaderAddr: leaderAddr, } + if conn == nil { + cli.networkFailure.Store(true) + } + return cli } // GetAddress implements ServiceClient. @@ -150,7 +168,7 @@ func (c *pdServiceClient) GetAddress() string { return c.addr } -// BuildGRPCContext implements ServiceClient. +// BuildGRPCTargetContext implements ServiceClient. func (c *pdServiceClient) BuildGRPCTargetContext(ctx context.Context, toLeader bool) context.Context { if c == nil || c.isLeader { return ctx @@ -169,7 +187,7 @@ func (c *pdServiceClient) IsConnectedToLeader() bool { return c.isLeader } -// NetworkAvailable implements ServiceClient. +// Available implements ServiceClient. func (c *pdServiceClient) Available() bool { if c == nil { return false @@ -183,9 +201,11 @@ func (c *pdServiceClient) checkNetworkAvailable(ctx context.Context) { } healthCli := healthpb.NewHealthClient(c.conn) resp, err := healthCli.Check(ctx, &healthpb.HealthCheckRequest{Service: ""}) - failpoint.Inject("unreachableNetwork1", func() { - resp = nil - err = status.New(codes.Unavailable, "unavailable").Err() + failpoint.Inject("unreachableNetwork1", func(val failpoint.Value) { + if val, ok := val.(string); (ok && val == c.GetAddress()) || !ok { + resp = nil + err = status.New(codes.Unavailable, "unavailable").Err() + } }) rpcErr, ok := status.FromError(err) if (ok && isNetworkError(rpcErr.Code())) || resp.GetStatus() != healthpb.HealthCheckResponse_SERVING { @@ -217,6 +237,10 @@ func (c *pdServiceClient) NeedRetry(pdErr *pdpb.Error, err error) bool { type errFn func(pdErr *pdpb.Error) bool +func emptyErrorFn(pdErr *pdpb.Error) bool { + return false +} + func regionAPIErrorFn(pdErr *pdpb.Error) bool { return pdErr.GetType() == pdpb.ErrorType_REGION_NOT_FOUND } @@ -243,6 +267,7 @@ func (c *pdServiceAPIClient) Available() bool { return c.ServiceClient.Available() && !c.unavailable.Load() } +// markAsAvailable is used to try to mark the client as available if unavailable status is expired. func (c *pdServiceAPIClient) markAsAvailable() { if !c.unavailable.Load() { return @@ -273,7 +298,7 @@ func (c *pdServiceAPIClient) NeedRetry(pdErr *pdpb.Error, err error) bool { // pdServiceBalancerNode is a balancer node for PD service. // It extends the pdServiceClient and adds additional fields for the next polling client in the chain. type pdServiceBalancerNode struct { - ServiceClient + *pdServiceAPIClient next *pdServiceBalancerNode } @@ -283,8 +308,14 @@ type pdServiceBalancer struct { mu sync.Mutex now *pdServiceBalancerNode totalNode int + errFn errFn } +func newPDServiceBalancer(fn errFn) *pdServiceBalancer { + return &pdServiceBalancer{ + errFn: fn, + } +} func (c *pdServiceBalancer) set(clients []ServiceClient) { c.mu.Lock() defer c.mu.Unlock() @@ -293,14 +324,14 @@ func (c *pdServiceBalancer) set(clients []ServiceClient) { } c.totalNode = len(clients) head := &pdServiceBalancerNode{ - ServiceClient: clients[0], + pdServiceAPIClient: newPDServiceAPIClient(clients[0], c.errFn).(*pdServiceAPIClient), } head.next = head last := head for i := 1; i < c.totalNode; i++ { next := &pdServiceBalancerNode{ - ServiceClient: clients[i], - next: head, + pdServiceAPIClient: newPDServiceAPIClient(clients[i], c.errFn).(*pdServiceAPIClient), + next: head, } head = next last.next = head @@ -308,6 +339,15 @@ func (c *pdServiceBalancer) set(clients []ServiceClient) { c.now = head } +func (c *pdServiceBalancer) check() { + c.mu.Lock() + defer c.mu.Unlock() + for i := 0; i < c.totalNode; i++ { + c.now.markAsAvailable() + c.next() + } +} + func (c *pdServiceBalancer) next() { c.now = c.now.next } @@ -352,9 +392,12 @@ type pdServiceDiscovery struct { urls atomic.Value // Store as []string // PD leader URL - leader atomic.Value // Store as string + leader atomic.Value // Store as pdServiceClient // PD follower URLs - followers atomic.Value // Store as []string + followers sync.Map // Store as map[string]pdServiceClient + apiCandidateNodes [apiKindCount]*pdServiceBalancer + // PD follower URLs. Only for tso. + followerAddresses atomic.Value // Store as []string clusterID uint64 // addr -> a gRPC connection @@ -402,6 +445,7 @@ func newPDServiceDiscovery( ctx: ctx, cancel: cancel, wg: wg, + apiCandidateNodes: [apiKindCount]*pdServiceBalancer{newPDServiceBalancer(emptyErrorFn)}, serviceModeUpdateCb: serviceModeUpdateCb, updateKeyspaceIDCb: updateKeyspaceIDCb, keyspaceID: keyspaceID, @@ -439,9 +483,10 @@ func (c *pdServiceDiscovery) Init() error { log.Warn("[pd] failed to check service mode and will check later", zap.Error(err)) } - c.wg.Add(2) + c.wg.Add(3) go c.updateMemberLoop() go c.updateServiceModeLoop() + go c.memberHealthCheckLoop() c.isInitialized = true return nil @@ -518,6 +563,46 @@ func (c *pdServiceDiscovery) updateServiceModeLoop() { } } } +func (c *pdServiceDiscovery) memberHealthCheckLoop() { + defer c.wg.Done() + + memberCheckLoopCtx, memberCheckLoopCancel := context.WithCancel(c.ctx) + defer memberCheckLoopCancel() + + ticker := time.NewTicker(MemberHealthCheckInterval) + defer ticker.Stop() + + for { + select { + case <-c.ctx.Done(): + return + case <-ticker.C: + c.checkLeaderHealth(memberCheckLoopCtx) + c.checkFollowerHealth(memberCheckLoopCtx) + } + } +} + +func (c *pdServiceDiscovery) checkLeaderHealth(ctx context.Context) { + ctx, cancel := context.WithTimeout(ctx, c.option.timeout) + defer cancel() + leader := c.getLeaderServiceClient() + leader.checkNetworkAvailable(ctx) +} + +func (c *pdServiceDiscovery) checkFollowerHealth(ctx context.Context) { + c.followers.Range(func(key, value any) bool { + // To ensure that the leader's healthy check is not delayed, shorten the duration. + ctx, cancel := context.WithTimeout(ctx, MemberHealthCheckInterval/3) + defer cancel() + serviceClient := value.(*pdServiceClient) + serviceClient.checkNetworkAvailable(ctx) + return true + }) + for _, balancer := range c.apiCandidateNodes { + balancer.check() + } +} // Close releases all resources. func (c *pdServiceDiscovery) Close() { @@ -606,12 +691,45 @@ func (c *pdServiceDiscovery) GetServingAddr() string { return c.getLeaderAddr() } -// GetBackupAddrs gets the addresses of the current reachable and healthy followers -// in a quorum-based cluster. +// GetBackupAddrs gets the addresses of the current reachable followers +// in a quorum-based cluster. Used for tso currently. func (c *pdServiceDiscovery) GetBackupAddrs() []string { return c.getFollowerAddrs() } +// getLeaderServiceClient returns the leader ServiceClient. +func (c *pdServiceDiscovery) getLeaderServiceClient() *pdServiceClient { + leader := c.leader.Load() + if leader == nil { + return nil + } + return leader.(*pdServiceClient) +} + +// getServiceClientByKind returns ServiceClient of the specific kind. +func (c *pdServiceDiscovery) getServiceClientByKind(kind apiKind) ServiceClient { + client := c.apiCandidateNodes[kind].get() + if client == nil { + return nil + } + return client +} + +// GetServiceClient returns the leader/primary ServiceClient if it is healthy. +func (c *pdServiceDiscovery) GetServiceClient() ServiceClient { + leaderClient := c.getLeaderServiceClient() + if c.option.enableForwarding && !leaderClient.Available() { + if followerClient := c.getServiceClientByKind(forwardAPIKind); followerClient != nil { + log.Debug("[pd] use follower client", zap.String("addr", followerClient.GetAddress())) + return followerClient + } + } + if leaderClient == nil { + return nil + } + return leaderClient +} + // ScheduleCheckMemberChanged is used to check if there is any membership // change among the leader and the followers. func (c *pdServiceDiscovery) ScheduleCheckMemberChanged() { @@ -657,16 +775,12 @@ func (c *pdServiceDiscovery) SetTSOGlobalServAddrUpdatedCallback(callback tsoGlo // getLeaderAddr returns the leader address. func (c *pdServiceDiscovery) getLeaderAddr() string { - leaderAddr := c.leader.Load() - if leaderAddr == nil { - return "" - } - return leaderAddr.(string) + return c.getLeaderServiceClient().GetAddress() } // getFollowerAddrs returns the follower address. func (c *pdServiceDiscovery) getFollowerAddrs() []string { - followerAddrs := c.followers.Load() + followerAddrs := c.followerAddresses.Load() if followerAddrs == nil { return []string{} } @@ -764,8 +878,7 @@ func (c *pdServiceDiscovery) updateMember() error { } c.updateURLs(members.GetMembers()) - c.updateFollowers(members.GetMembers(), members.GetLeader()) - if err := c.switchLeader(members.GetLeader().GetClientUrls()); err != nil { + if err := c.updateServiceClient(members.GetMembers(), members.GetLeader()); err != nil { return err } @@ -837,42 +950,104 @@ func (c *pdServiceDiscovery) updateURLs(members []*pdpb.Member) { log.Info("[pd] update member urls", zap.Strings("old-urls", oldURLs), zap.Strings("new-urls", urls)) } -func (c *pdServiceDiscovery) switchLeader(addrs []string) error { +func (c *pdServiceDiscovery) switchLeader(addrs []string) (bool, error) { // FIXME: How to safely compare leader urls? For now, only allows one client url. addr := addrs[0] - oldLeader := c.getLeaderAddr() - if addr == oldLeader { - return nil + oldLeader := c.getLeaderServiceClient() + if addr == oldLeader.GetAddress() && oldLeader.GetClientConn() != nil { + return false, nil } - if _, err := c.GetOrCreateGRPCConn(addr); err != nil { - log.Warn("[pd] failed to connect leader", zap.String("leader", addr), errs.ZapError(err)) + newConn, err := c.GetOrCreateGRPCConn(addr) + // If gRPC connect is created successfully or leader is new, still saves. + if addr != oldLeader.GetAddress() || newConn != nil { + // Set PD leader and Global TSO Allocator (which is also the PD leader) + leaderClient := newPDServiceClient(addr, addr, newConn, true) + c.leader.Store(leaderClient) } - // Set PD leader and Global TSO Allocator (which is also the PD leader) - c.leader.Store(addr) // Run callbacks if c.tsoGlobalAllocLeaderUpdatedCb != nil { if err := c.tsoGlobalAllocLeaderUpdatedCb(addr); err != nil { - return err + return true, err } } for _, cb := range c.leaderSwitchedCbs { cb() } - log.Info("[pd] switch leader", zap.String("new-leader", addr), zap.String("old-leader", oldLeader)) - return nil + log.Info("[pd] switch leader", zap.String("new-leader", addr), zap.String("old-leader", oldLeader.GetAddress())) + return true, err } -func (c *pdServiceDiscovery) updateFollowers(members []*pdpb.Member, leader *pdpb.Member) { - var addrs []string +func (c *pdServiceDiscovery) updateFollowers(members []*pdpb.Member, leader *pdpb.Member) (changed bool) { + followers := make(map[string]*pdServiceClient) + c.followers.Range(func(key, value any) bool { + followers[key.(string)] = value.(*pdServiceClient) + return true + }) + var followerAddrs []string for _, member := range members { if member.GetMemberId() != leader.GetMemberId() { if len(member.GetClientUrls()) > 0 { - addrs = append(addrs, member.GetClientUrls()...) + followerAddrs = append(followerAddrs, member.GetClientUrls()...) + + // FIXME: How to safely compare urls(also for leader)? For now, only allows one client url. + addr := member.GetClientUrls()[0] + if client, ok := c.followers.Load(addr); ok { + if client.(*pdServiceClient).GetClientConn() == nil { + conn, err := c.GetOrCreateGRPCConn(addr) + if err != nil || conn == nil { + log.Warn("[pd] failed to connect follower", zap.String("follower", addr), errs.ZapError(err)) + continue + } + follower := newPDServiceClient(addr, leader.GetClientUrls()[0], conn, false) + c.followers.Store(addr, follower) + changed = true + } + delete(followers, addr) + } else { + changed = true + conn, err := c.GetOrCreateGRPCConn(addr) + follower := newPDServiceClient(addr, leader.GetClientUrls()[0], conn, false) + if err != nil || conn == nil { + log.Warn("[pd] failed to connect follower", zap.String("follower", addr), errs.ZapError(err)) + } + c.followers.LoadOrStore(addr, follower) + } } } } - c.followers.Store(addrs) + if len(followers) > 0 { + changed = true + for key := range followers { + c.followers.Delete(key) + } + } + c.followerAddresses.Store(followerAddrs) + return +} + +func (c *pdServiceDiscovery) updateServiceClient(members []*pdpb.Member, leader *pdpb.Member) error { + leaderChanged, err := c.switchLeader(leader.GetClientUrls()) + followerChanged := c.updateFollowers(members, leader) + // don't need to recreate balancer if no changess. + if !followerChanged && !leaderChanged { + return err + } + // If error is not nil, still updates candidates. + clients := make([]ServiceClient, 0) + c.followers.Range(func(_, value any) bool { + clients = append(clients, value.(*pdServiceClient)) + return true + }) + leaderClient := c.getLeaderServiceClient() + if leaderClient != nil { + clients = append(clients, leaderClient) + } + // create candidate services for all kinds of request. + for i := 0; i < int(apiKindCount); i++ { + c.apiCandidateNodes[i].set(clients) + } + return err } func (c *pdServiceDiscovery) switchTSOAllocatorLeaders(allocatorMap map[string]*pdpb.Member) error { diff --git a/client/tso_service_discovery.go b/client/tso_service_discovery.go index 5f14c406797..7caaacd0dfe 100644 --- a/client/tso_service_discovery.go +++ b/client/tso_service_discovery.go @@ -373,6 +373,11 @@ func (c *tsoServiceDiscovery) SetTSOGlobalServAddrUpdatedCallback(callback tsoGl c.globalAllocPrimariesUpdatedCb = callback } +// GetServiceClient implements ServiceDiscovery +func (c *tsoServiceDiscovery) GetServiceClient() ServiceClient { + return c.apiSvcDiscovery.GetServiceClient() +} + // getPrimaryAddr returns the primary address. func (c *tsoServiceDiscovery) getPrimaryAddr() string { c.keyspaceGroupSD.RLock() diff --git a/tests/integrations/client/client_test.go b/tests/integrations/client/client_test.go index 1fd8d75dec4..e1e841342ad 100644 --- a/tests/integrations/client/client_test.go +++ b/tests/integrations/client/client_test.go @@ -518,18 +518,69 @@ func TestCustomTimeout(t *testing.T) { re.Less(time.Since(start), 2*time.Second) } -func TestGetRegionByFollowerForwarding(t *testing.T) { - re := require.New(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - pd.LeaderHealthCheckInterval = 100 * time.Millisecond - cluster, err := tests.NewTestCluster(ctx, 3) +type followerForwardAndHandleTestSuite struct { + suite.Suite + ctx context.Context + clean context.CancelFunc + + cluster *tests.TestCluster + endpoints []string + regionID uint64 +} + +func TestFollowerForwardAndHandleTestSuite(t *testing.T) { + suite.Run(t, new(followerForwardAndHandleTestSuite)) +} + +func (suite *followerForwardAndHandleTestSuite) SetupSuite() { + re := suite.Require() + suite.ctx, suite.clean = context.WithCancel(context.Background()) + pd.MemberHealthCheckInterval = 100 * time.Millisecond + cluster, err := tests.NewTestCluster(suite.ctx, 3) re.NoError(err) - defer cluster.Destroy() + suite.cluster = cluster + suite.endpoints = runServer(re, cluster) + cluster.WaitLeader() + leader := cluster.GetLeaderServer() + grpcPDClient := testutil.MustNewGrpcClient(re, leader.GetAddr()) + suite.regionID = regionIDAllocator.alloc() + testutil.Eventually(re, func() bool { + regionHeartbeat, err := grpcPDClient.RegionHeartbeat(suite.ctx) + re.NoError(err) + region := &metapb.Region{ + Id: suite.regionID, + RegionEpoch: &metapb.RegionEpoch{ + ConfVer: 1, + Version: 1, + }, + Peers: peers, + } + req := &pdpb.RegionHeartbeatRequest{ + Header: newHeader(leader.GetServer()), + Region: region, + Leader: peers[0], + } + err = regionHeartbeat.Send(req) + re.NoError(err) + _, err = regionHeartbeat.Recv() + return err == nil + }) +} - endpoints := runServer(re, cluster) - cli := setupCli(re, ctx, endpoints, pd.WithForwardingOption(true)) +func (suite *followerForwardAndHandleTestSuite) TearDownTest() { +} + +func (suite *followerForwardAndHandleTestSuite) TearDownSuite() { + suite.cluster.Destroy() + suite.clean() +} + +func (suite *followerForwardAndHandleTestSuite) TestGetRegionByFollowerForwarding() { + re := suite.Require() + ctx, cancel := context.WithCancel(suite.ctx) + defer cancel() + cli := setupCli(re, ctx, suite.endpoints, pd.WithForwardingOption(true)) re.NoError(failpoint.Enable("github.com/tikv/pd/client/unreachableNetwork1", "return(true)")) time.Sleep(200 * time.Millisecond) r, err := cli.GetRegion(context.Background(), []byte("a")) @@ -544,17 +595,11 @@ func TestGetRegionByFollowerForwarding(t *testing.T) { } // case 1: unreachable -> normal -func TestGetTsoByFollowerForwarding1(t *testing.T) { - re := require.New(t) - ctx, cancel := context.WithCancel(context.Background()) +func (suite *followerForwardAndHandleTestSuite) TestGetTsoByFollowerForwarding1() { + re := suite.Require() + ctx, cancel := context.WithCancel(suite.ctx) defer cancel() - pd.LeaderHealthCheckInterval = 100 * time.Millisecond - cluster, err := tests.NewTestCluster(ctx, 3) - re.NoError(err) - defer cluster.Destroy() - - endpoints := runServer(re, cluster) - cli := setupCli(re, ctx, endpoints, pd.WithForwardingOption(true)) + cli := setupCli(re, ctx, suite.endpoints, pd.WithForwardingOption(true)) re.NoError(failpoint.Enable("github.com/tikv/pd/client/unreachableNetwork", "return(true)")) var lastTS uint64 @@ -564,7 +609,7 @@ func TestGetTsoByFollowerForwarding1(t *testing.T) { lastTS = tsoutil.ComposeTS(physical, logical) return true } - t.Log(err) + suite.T().Log(err) return false }) @@ -575,17 +620,11 @@ func TestGetTsoByFollowerForwarding1(t *testing.T) { } // case 2: unreachable -> leader transfer -> normal -func TestGetTsoByFollowerForwarding2(t *testing.T) { - re := require.New(t) - ctx, cancel := context.WithCancel(context.Background()) +func (suite *followerForwardAndHandleTestSuite) TestGetTsoByFollowerForwarding2() { + re := suite.Require() + ctx, cancel := context.WithCancel(suite.ctx) defer cancel() - pd.LeaderHealthCheckInterval = 100 * time.Millisecond - cluster, err := tests.NewTestCluster(ctx, 3) - re.NoError(err) - defer cluster.Destroy() - - endpoints := runServer(re, cluster) - cli := setupCli(re, ctx, endpoints, pd.WithForwardingOption(true)) + cli := setupCli(re, ctx, suite.endpoints, pd.WithForwardingOption(true)) re.NoError(failpoint.Enable("github.com/tikv/pd/client/unreachableNetwork", "return(true)")) var lastTS uint64 @@ -595,13 +634,13 @@ func TestGetTsoByFollowerForwarding2(t *testing.T) { lastTS = tsoutil.ComposeTS(physical, logical) return true } - t.Log(err) + suite.T().Log(err) return false }) lastTS = checkTS(re, cli, lastTS) - re.NoError(cluster.GetLeaderServer().ResignLeader()) - re.NotEmpty(cluster.WaitLeader()) + re.NoError(suite.cluster.GetLeaderServer().ResignLeader()) + re.NotEmpty(suite.cluster.WaitLeader()) lastTS = checkTS(re, cli, lastTS) re.NoError(failpoint.Disable("github.com/tikv/pd/client/unreachableNetwork")) @@ -610,45 +649,18 @@ func TestGetTsoByFollowerForwarding2(t *testing.T) { } // case 3: network partition between client and follower A -> transfer leader to follower A -> normal -func TestGetTsoAndRegionByFollowerForwarding(t *testing.T) { - re := require.New(t) - ctx, cancel := context.WithCancel(context.Background()) +func (suite *followerForwardAndHandleTestSuite) TestGetTsoAndRegionByFollowerForwarding() { + re := suite.Require() + ctx, cancel := context.WithCancel(suite.ctx) defer cancel() - pd.LeaderHealthCheckInterval = 100 * time.Millisecond - cluster, err := tests.NewTestCluster(ctx, 3) - re.NoError(err) - defer cluster.Destroy() - endpoints := runServer(re, cluster) - re.NotEmpty(cluster.WaitLeader()) + cluster := suite.cluster leader := cluster.GetLeaderServer() - grpcPDClient := testutil.MustNewGrpcClient(re, leader.GetAddr()) - testutil.Eventually(re, func() bool { - regionHeartbeat, err := grpcPDClient.RegionHeartbeat(ctx) - re.NoError(err) - regionID := regionIDAllocator.alloc() - region := &metapb.Region{ - Id: regionID, - RegionEpoch: &metapb.RegionEpoch{ - ConfVer: 1, - Version: 1, - }, - Peers: peers, - } - req := &pdpb.RegionHeartbeatRequest{ - Header: newHeader(leader.GetServer()), - Region: region, - Leader: peers[0], - } - err = regionHeartbeat.Send(req) - re.NoError(err) - _, err = regionHeartbeat.Recv() - return err == nil - }) + follower := cluster.GetServer(cluster.GetFollower()) re.NoError(failpoint.Enable("github.com/tikv/pd/client/grpcutil/unreachableNetwork2", fmt.Sprintf("return(\"%s\")", follower.GetAddr()))) - cli := setupCli(re, ctx, endpoints, pd.WithForwardingOption(true)) + cli := setupCli(re, ctx, suite.endpoints, pd.WithForwardingOption(true)) var lastTS uint64 testutil.Eventually(re, func() bool { physical, logical, err := cli.GetTS(context.TODO()) @@ -656,7 +668,7 @@ func TestGetTsoAndRegionByFollowerForwarding(t *testing.T) { lastTS = tsoutil.ComposeTS(physical, logical) return true } - t.Log(err) + suite.T().Log(err) return false }) lastTS = checkTS(re, cli, lastTS) @@ -672,7 +684,7 @@ func TestGetTsoAndRegionByFollowerForwarding(t *testing.T) { lastTS = tsoutil.ComposeTS(physical, logical) return true } - t.Log(err) + suite.T().Log(err) return false }) lastTS = checkTS(re, cli, lastTS) @@ -691,7 +703,7 @@ func TestGetTsoAndRegionByFollowerForwarding(t *testing.T) { lastTS = tsoutil.ComposeTS(physical, logical) return true } - t.Log(err) + suite.T().Log(err) return false }) lastTS = checkTS(re, cli, lastTS) From ffa76e9d61140dab5b5b8e79dc015810f4862f1f Mon Sep 17 00:00:00 2001 From: glorv Date: Tue, 26 Dec 2023 17:01:30 +0800 Subject: [PATCH 134/137] fmt go.mod Signed-off-by: glorv --- go.mod | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index ae3d92172e5..b01ab4e1f81 100644 --- a/go.mod +++ b/go.mod @@ -63,11 +63,6 @@ require ( gotest.tools/gotestsum v1.7.0 ) -require ( - google.golang.org/genproto/googleapis/api v0.0.0-20231002182017-d307bd883b97 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b // indirect -) - require ( github.com/KyleBanks/depth v1.2.1 // indirect github.com/Masterminds/semver v1.5.0 // indirect @@ -198,6 +193,8 @@ require ( golang.org/x/term v0.13.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20231012201019-e917dd12ba7a // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20231002182017-d307bd883b97 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b // indirect google.golang.org/protobuf v1.31.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect From 13d79e54ba2c4763e4502dfaf230c299941868f1 Mon Sep 17 00:00:00 2001 From: glorv Date: Tue, 26 Dec 2023 17:51:22 +0800 Subject: [PATCH 135/137] do clean up Signed-off-by: glorv --- .../integrations/mcs/resourcemanager/resource_manager_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/integrations/mcs/resourcemanager/resource_manager_test.go b/tests/integrations/mcs/resourcemanager/resource_manager_test.go index a9708598141..723ecb2e39f 100644 --- a/tests/integrations/mcs/resourcemanager/resource_manager_test.go +++ b/tests/integrations/mcs/resourcemanager/resource_manager_test.go @@ -1138,6 +1138,9 @@ func (suite *resourceManagerClientTestSuite) TestResourceGroupRUConsumption() { g, err = cli.GetResourceGroup(suite.ctx, group.Name, pd.WithRUStats) re.NoError(err) re.Equal(g.RUStats, testConsumption) + + _, err = cli.DeleteResourceGroup(suite.ctx, group.Name) + re.NoError(err) } func (suite *resourceManagerClientTestSuite) TestResourceManagerClientFailover() { From 1089b1bae70c8fc6694eba0a55ccc93e503fc3d9 Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Tue, 26 Dec 2023 20:30:14 +0800 Subject: [PATCH 136/137] test Signed-off-by: Cabinfever_B --- .../resourcemanager/resource_manager_test.go | 83 ------------------- 1 file changed, 83 deletions(-) diff --git a/tests/integrations/mcs/resourcemanager/resource_manager_test.go b/tests/integrations/mcs/resourcemanager/resource_manager_test.go index 723ecb2e39f..44477ca7693 100644 --- a/tests/integrations/mcs/resourcemanager/resource_manager_test.go +++ b/tests/integrations/mcs/resourcemanager/resource_manager_test.go @@ -1060,89 +1060,6 @@ func (suite *resourceManagerClientTestSuite) TestBasicResourceGroupCURD() { re.Equal(groups, newGroups) } -func (suite *resourceManagerClientTestSuite) TestResourceGroupRUConsumption() { - re := suite.Require() - cli := suite.client - re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/mcs/resourcemanager/server/fastPersist", `return(true)`)) - defer re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/mcs/resourcemanager/server/fastPersist")) - group := &rmpb.ResourceGroup{ - Name: "test_ru_consumption", - Mode: rmpb.GroupMode_RUMode, - RUSettings: &rmpb.GroupRequestUnitSettings{ - RU: &rmpb.TokenBucket{Settings: &rmpb.TokenLimitSettings{ - FillRate: 10000, - BurstLimit: 10000, - MaxTokens: 20000.0, - }}, - }, - } - _, err := cli.AddResourceGroup(suite.ctx, group) - re.NoError(err) - - g, err := cli.GetResourceGroup(suite.ctx, group.Name) - re.NoError(err) - re.Equal(group, g) - - // Test Resource Group Stats - testConsumption := &rmpb.Consumption{ - RRU: 200.0, - WRU: 100.0, - ReadBytes: 1024, - WriteBytes: 512, - TotalCpuTimeMs: 50.0, - SqlLayerCpuTimeMs: 40.0, - KvReadRpcCount: 5, - KvWriteRpcCount: 6, - } - _, err = cli.AcquireTokenBuckets(suite.ctx, &rmpb.TokenBucketsRequest{ - Requests: []*rmpb.TokenBucketRequest{ - { - ResourceGroupName: group.Name, - ConsumptionSinceLastRequest: testConsumption, - }, - }, - TargetRequestPeriodMs: 1000, - ClientUniqueId: 1, - }) - re.NoError(err) - time.Sleep(10 * time.Millisecond) - g, err = cli.GetResourceGroup(suite.ctx, group.Name, pd.WithRUStats) - re.NoError(err) - re.Equal(g.RUStats, testConsumption) - - // update resoruce group, ru stats not change - g.RUSettings.RU.Settings.FillRate = 12345 - _, err = cli.ModifyResourceGroup(suite.ctx, g) - re.NoError(err) - g1, err := cli.GetResourceGroup(suite.ctx, group.Name, pd.WithRUStats) - re.NoError(err) - re.Equal(g1, g) - - // test restart cluster - time.Sleep(250 * time.Millisecond) - servers := suite.cluster.GetServers() - re.NoError(suite.cluster.StopAll()) - cli.Close() - serverList := make([]*tests.TestServer, 0, len(servers)) - for _, s := range servers { - serverList = append(serverList, s) - } - re.NoError(suite.cluster.RunServers(serverList)) - suite.cluster.WaitLeader() - // re-connect client as well - suite.client, err = pd.NewClientWithContext(suite.ctx, suite.cluster.GetConfig().GetClientURLs(), pd.SecurityOption{}) - re.NoError(err) - cli = suite.client - - // check ru stats not loss after restart - g, err = cli.GetResourceGroup(suite.ctx, group.Name, pd.WithRUStats) - re.NoError(err) - re.Equal(g.RUStats, testConsumption) - - _, err = cli.DeleteResourceGroup(suite.ctx, group.Name) - re.NoError(err) -} - func (suite *resourceManagerClientTestSuite) TestResourceManagerClientFailover() { re := suite.Require() cli := suite.client From feba19ea9f1130c863a25736a316577c77885d3a Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Wed, 27 Dec 2023 10:37:17 +0800 Subject: [PATCH 137/137] revert Signed-off-by: Cabinfever_B --- .../resourcemanager/resource_manager_test.go | 83 +++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/tests/integrations/mcs/resourcemanager/resource_manager_test.go b/tests/integrations/mcs/resourcemanager/resource_manager_test.go index 44477ca7693..723ecb2e39f 100644 --- a/tests/integrations/mcs/resourcemanager/resource_manager_test.go +++ b/tests/integrations/mcs/resourcemanager/resource_manager_test.go @@ -1060,6 +1060,89 @@ func (suite *resourceManagerClientTestSuite) TestBasicResourceGroupCURD() { re.Equal(groups, newGroups) } +func (suite *resourceManagerClientTestSuite) TestResourceGroupRUConsumption() { + re := suite.Require() + cli := suite.client + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/mcs/resourcemanager/server/fastPersist", `return(true)`)) + defer re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/mcs/resourcemanager/server/fastPersist")) + group := &rmpb.ResourceGroup{ + Name: "test_ru_consumption", + Mode: rmpb.GroupMode_RUMode, + RUSettings: &rmpb.GroupRequestUnitSettings{ + RU: &rmpb.TokenBucket{Settings: &rmpb.TokenLimitSettings{ + FillRate: 10000, + BurstLimit: 10000, + MaxTokens: 20000.0, + }}, + }, + } + _, err := cli.AddResourceGroup(suite.ctx, group) + re.NoError(err) + + g, err := cli.GetResourceGroup(suite.ctx, group.Name) + re.NoError(err) + re.Equal(group, g) + + // Test Resource Group Stats + testConsumption := &rmpb.Consumption{ + RRU: 200.0, + WRU: 100.0, + ReadBytes: 1024, + WriteBytes: 512, + TotalCpuTimeMs: 50.0, + SqlLayerCpuTimeMs: 40.0, + KvReadRpcCount: 5, + KvWriteRpcCount: 6, + } + _, err = cli.AcquireTokenBuckets(suite.ctx, &rmpb.TokenBucketsRequest{ + Requests: []*rmpb.TokenBucketRequest{ + { + ResourceGroupName: group.Name, + ConsumptionSinceLastRequest: testConsumption, + }, + }, + TargetRequestPeriodMs: 1000, + ClientUniqueId: 1, + }) + re.NoError(err) + time.Sleep(10 * time.Millisecond) + g, err = cli.GetResourceGroup(suite.ctx, group.Name, pd.WithRUStats) + re.NoError(err) + re.Equal(g.RUStats, testConsumption) + + // update resoruce group, ru stats not change + g.RUSettings.RU.Settings.FillRate = 12345 + _, err = cli.ModifyResourceGroup(suite.ctx, g) + re.NoError(err) + g1, err := cli.GetResourceGroup(suite.ctx, group.Name, pd.WithRUStats) + re.NoError(err) + re.Equal(g1, g) + + // test restart cluster + time.Sleep(250 * time.Millisecond) + servers := suite.cluster.GetServers() + re.NoError(suite.cluster.StopAll()) + cli.Close() + serverList := make([]*tests.TestServer, 0, len(servers)) + for _, s := range servers { + serverList = append(serverList, s) + } + re.NoError(suite.cluster.RunServers(serverList)) + suite.cluster.WaitLeader() + // re-connect client as well + suite.client, err = pd.NewClientWithContext(suite.ctx, suite.cluster.GetConfig().GetClientURLs(), pd.SecurityOption{}) + re.NoError(err) + cli = suite.client + + // check ru stats not loss after restart + g, err = cli.GetResourceGroup(suite.ctx, group.Name, pd.WithRUStats) + re.NoError(err) + re.Equal(g.RUStats, testConsumption) + + _, err = cli.DeleteResourceGroup(suite.ctx, group.Name) + re.NoError(err) +} + func (suite *resourceManagerClientTestSuite) TestResourceManagerClientFailover() { re := suite.Require() cli := suite.client