From 96d0fc40ad52c8f7596b753d57763e318703b1b6 Mon Sep 17 00:00:00 2001 From: ShuNing Date: Wed, 3 Jul 2024 14:56:57 +0800 Subject: [PATCH] This is an automated cherry-pick of #8352 close tikv/pd#8349 Signed-off-by: ti-chi-bot --- client/resource_group/controller/config.go | 76 ++++++++++++++++++- .../resource_group/controller/controller.go | 17 ++++- pkg/mcs/resource_manager/server/config.go | 18 +++++ .../resource_manager/server/config_test.go | 7 ++ .../resource_manager/resource_manager_test.go | 19 +++++ 5 files changed, 132 insertions(+), 5 deletions(-) diff --git a/client/resource_group/controller/config.go b/client/resource_group/controller/config.go index fe30cc445fd..887f6c718b7 100644 --- a/client/resource_group/controller/config.go +++ b/client/resource_group/controller/config.go @@ -53,6 +53,15 @@ const ( defaultTargetPeriod = 5 * time.Second // defaultMaxWaitDuration is the max duration to wait for the token before throwing error. defaultMaxWaitDuration = 30 * time.Second +<<<<<<< HEAD +======= + // defaultLTBTokenRPCMaxDelay is the upper bound of backoff delay for local token bucket RPC. + defaultLTBTokenRPCMaxDelay = 1 * time.Second + // defaultWaitRetryTimes is the times to retry when waiting for the token. + defaultWaitRetryTimes = 20 + // defaultWaitRetryInterval is the interval to retry when waiting for the token. + defaultWaitRetryInterval = 50 * time.Millisecond +>>>>>>> 6b25787af (resource_control: allow configuration of the maximum retry time for the local bucket (#8352)) ) const ( @@ -67,17 +76,43 @@ const ( // Because the resource manager has not been deployed in microservice mode, // do not enable this function. +<<<<<<< HEAD defaultDegradedModeWaitDuration = 0 +======= + defaultDegradedModeWaitDuration = time.Duration(0) + defaultAvgBatchProportion = 0.7 +>>>>>>> 6b25787af (resource_control: allow configuration of the maximum retry time for the local bucket (#8352)) ) -// Config is the configuration of the resource manager controller which includes some option for client needed. -type Config struct { +// TokenRPCParams is the parameters for local bucket RPC. +type TokenRPCParams struct { + // WaitRetryInterval is the interval to retry when waiting for the token. + WaitRetryInterval Duration `toml:"wait-retry-interval" json:"wait-retry-interval"` + + // WaitRetryTimes is the times to retry when waiting for the token. + WaitRetryTimes int `toml:"wait-retry-times" json:"wait-retry-times"` +} + +// LocalBucketConfig is the configuration for local bucket. not export to server side. +type LocalBucketConfig struct { + TokenRPCParams `toml:"token-rpc-params" json:"token-rpc-params"` +} + +// BaseConfig is the configuration of the resource manager controller which includes some option for client needed. +// TODO: unified the configuration for client and server, server side in pkg/mcs/resourcemanger/config.go. +type BaseConfig struct { // EnableDegradedMode is to control whether resource control client enable degraded mode when server is disconnect. DegradedModeWaitDuration Duration `toml:"degraded-mode-wait-duration" json:"degraded-mode-wait-duration"` // LTBMaxWaitDuration is the max wait time duration for local token bucket. LTBMaxWaitDuration Duration `toml:"ltb-max-wait-duration" json:"ltb-max-wait-duration"` +<<<<<<< HEAD +======= + // LTBTokenRPCMaxDelay is the upper bound of backoff delay for local token bucket RPC. + LTBTokenRPCMaxDelay Duration `toml:"ltb-token-rpc-max-delay" json:"ltb-token-rpc-max-delay"` + +>>>>>>> 6b25787af (resource_control: allow configuration of the maximum retry time for the local bucket (#8352)) // RequestUnit is the configuration determines the coefficients of the RRU and WRU cost. // This configuration should be modified carefully. RequestUnit RequestUnitConfig `toml:"request-unit" json:"request-unit"` @@ -86,13 +121,50 @@ type Config struct { EnableControllerTraceLog bool `toml:"enable-controller-trace-log" json:"enable-controller-trace-log,string"` } +// Config is the configuration of the resource manager controller. +type Config struct { + BaseConfig + LocalBucketConfig +} + +// Adjust adjusts the configuration. +func (c *Config) Adjust() { + // valid the configuration, TODO: separately add the valid function. + if c.BaseConfig.LTBMaxWaitDuration.Duration == 0 { + c.BaseConfig.LTBMaxWaitDuration = NewDuration(defaultMaxWaitDuration) + } + if c.LocalBucketConfig.WaitRetryInterval.Duration == 0 { + c.LocalBucketConfig.WaitRetryInterval = NewDuration(defaultWaitRetryInterval) + } + // adjust the client settings. calculate the retry times. + if int(c.BaseConfig.LTBTokenRPCMaxDelay.Duration) != int(c.LocalBucketConfig.WaitRetryInterval.Duration)*c.LocalBucketConfig.WaitRetryTimes { + c.LocalBucketConfig.WaitRetryTimes = int(c.BaseConfig.LTBTokenRPCMaxDelay.Duration / c.LocalBucketConfig.WaitRetryInterval.Duration) + } +} + // DefaultConfig returns the default resource manager controller configuration. func DefaultConfig() *Config { return &Config{ +<<<<<<< HEAD DegradedModeWaitDuration: NewDuration(defaultDegradedModeWaitDuration), LTBMaxWaitDuration: NewDuration(defaultMaxWaitDuration), RequestUnit: DefaultRequestUnitConfig(), EnableControllerTraceLog: false, +======= + BaseConfig: BaseConfig{ + DegradedModeWaitDuration: NewDuration(defaultDegradedModeWaitDuration), + RequestUnit: DefaultRequestUnitConfig(), + EnableControllerTraceLog: false, + LTBMaxWaitDuration: NewDuration(defaultMaxWaitDuration), + LTBTokenRPCMaxDelay: NewDuration(defaultLTBTokenRPCMaxDelay), + }, + LocalBucketConfig: LocalBucketConfig{ + TokenRPCParams: TokenRPCParams{ + WaitRetryInterval: NewDuration(defaultWaitRetryInterval), + WaitRetryTimes: defaultWaitRetryTimes, + }, + }, +>>>>>>> 6b25787af (resource_control: allow configuration of the maximum retry time for the local bucket (#8352)) } } diff --git a/client/resource_group/controller/controller.go b/client/resource_group/controller/controller.go index 1834e509696..1b39a9f338a 100755 --- a/client/resource_group/controller/controller.go +++ b/client/resource_group/controller/controller.go @@ -168,6 +168,7 @@ func NewResourceGroupController( log.Info("load resource controller config", zap.Reflect("config", config), zap.Reflect("ru-config", controller.ruConfig)) controller.calculators = []ResourceCalculator{newKVCalculator(controller.ruConfig), newSQLCalculator(controller.ruConfig)} controller.safeRuConfig.Store(controller.ruConfig) + enableControllerTraceLog.Store(config.EnableControllerTraceLog) return controller, nil } @@ -176,12 +177,23 @@ func loadServerConfig(ctx context.Context, provider ResourceGroupProvider) (*Con if err != nil { return nil, err } +<<<<<<< HEAD if len(resp.Kvs) == 0 { +======= + config := DefaultConfig() + defer config.Adjust() + kvs := resp.GetKvs() + if len(kvs) == 0 { +>>>>>>> 6b25787af (resource_control: allow configuration of the maximum retry time for the local bucket (#8352)) log.Warn("[resource group controller] server does not save config, load config failed") - return DefaultConfig(), nil + return config, nil } +<<<<<<< HEAD config := &Config{} err = json.Unmarshal(resp.Kvs[0].GetValue(), config) +======= + err = json.Unmarshal(kvs[0].GetValue(), config) +>>>>>>> 6b25787af (resource_control: allow configuration of the maximum retry time for the local bucket (#8352)) if err != nil { return nil, err } @@ -283,7 +295,6 @@ func (c *ResourceGroupsController) Start(ctx context.Context) { watchRetryTimer.Reset(watchRetryInterval) } } - case <-emergencyTokenAcquisitionTicker.C: c.executeOnAllGroups((*groupCostController).resetEmergencyTokenAcquisition) /* channels */ @@ -365,6 +376,7 @@ func (c *ResourceGroupsController) Start(ctx context.Context) { if err := json.Unmarshal(item.Kv.Value, config); err != nil { continue } + config.Adjust() c.ruConfig = GenerateRUConfig(config) // Stay compatible with serverless @@ -378,7 +390,6 @@ func (c *ResourceGroupsController) Start(ctx context.Context) { } log.Info("load resource controller config after config changed", zap.Reflect("config", config), zap.Reflect("ruConfig", c.ruConfig)) } - case gc := <-c.tokenBucketUpdateChan: now := gc.run.now go gc.handleTokenBucketUpdateEvent(c.loopCtx, now) diff --git a/pkg/mcs/resource_manager/server/config.go b/pkg/mcs/resource_manager/server/config.go index b3a29446b4f..49922ba48a2 100644 --- a/pkg/mcs/resource_manager/server/config.go +++ b/pkg/mcs/resource_manager/server/config.go @@ -53,6 +53,8 @@ const ( defaultDegradedModeWaitDuration = time.Second * 0 // defaultMaxWaitDuration is the max duration to wait for the token before throwing error. defaultMaxWaitDuration = 30 * time.Second + // defaultLTBTokenRPCMaxDelay is the upper bound of backoff delay for local token bucket RPC. + defaultLTBTokenRPCMaxDelay = 1 * time.Second ) // Config is the configuration for the resource manager. @@ -90,6 +92,9 @@ type ControllerConfig struct { // LTBMaxWaitDuration is the max wait time duration for local token bucket. LTBMaxWaitDuration typeutil.Duration `toml:"ltb-max-wait-duration" json:"ltb-max-wait-duration"` + // LTBTokenRPCMaxDelay is the upper bound of backoff delay for local token bucket RPC. + LTBTokenRPCMaxDelay typeutil.Duration `toml:"ltb-token-rpc-max-delay" json:"ltb-token-rpc-max-delay"` + // RequestUnit is the configuration determines the coefficients of the RRU and WRU cost. // This configuration should be modified carefully. RequestUnit RequestUnitConfig `toml:"request-unit" json:"request-unit"` @@ -103,10 +108,23 @@ func (rmc *ControllerConfig) Adjust(meta *configutil.ConfigMetaData) { if rmc == nil { return } +<<<<<<< HEAD:pkg/mcs/resource_manager/server/config.go rmc.RequestUnit.Adjust() configutil.AdjustDuration(&rmc.DegradedModeWaitDuration, defaultDegradedModeWaitDuration) configutil.AdjustDuration(&rmc.LTBMaxWaitDuration, defaultMaxWaitDuration) +======= + rmc.RequestUnit.Adjust(meta.Child("request-unit")) + if !meta.IsDefined("degraded-mode-wait-duration") { + configutil.AdjustDuration(&rmc.DegradedModeWaitDuration, defaultDegradedModeWaitDuration) + } + if !meta.IsDefined("ltb-max-wait-duration") { + configutil.AdjustDuration(&rmc.LTBMaxWaitDuration, defaultMaxWaitDuration) + } + if !meta.IsDefined("ltb-token-rpc-max-delay") { + configutil.AdjustDuration(&rmc.LTBTokenRPCMaxDelay, defaultLTBTokenRPCMaxDelay) + } +>>>>>>> 6b25787af (resource_control: allow configuration of the maximum retry time for the local bucket (#8352)):pkg/mcs/resourcemanager/server/config.go failpoint.Inject("enableDegradedMode", func() { configutil.AdjustDuration(&rmc.DegradedModeWaitDuration, time.Second) }) diff --git a/pkg/mcs/resource_manager/server/config_test.go b/pkg/mcs/resource_manager/server/config_test.go index dd8dd2d2814..d6761d1e628 100644 --- a/pkg/mcs/resource_manager/server/config_test.go +++ b/pkg/mcs/resource_manager/server/config_test.go @@ -28,6 +28,7 @@ func TestControllerConfig(t *testing.T) { cfgData := ` [controller] ltb-max-wait-duration = "60s" +ltb-token-rpc-max-delay = "500ms" degraded-mode-wait-duration = "2s" [controller.request-unit] read-base-cost = 1.0 @@ -42,8 +43,14 @@ read-cpu-ms-cost = 5.0 err = cfg.Adjust(&meta, false) re.NoError(err) +<<<<<<< HEAD:pkg/mcs/resource_manager/server/config_test.go re.Equal(cfg.Controller.DegradedModeWaitDuration.Duration, time.Second*2) re.Equal(cfg.Controller.LTBMaxWaitDuration.Duration, time.Second*60) +======= + re.Equal(2*time.Second, cfg.Controller.DegradedModeWaitDuration.Duration) + re.Equal(60*time.Second, cfg.Controller.LTBMaxWaitDuration.Duration) + re.Equal(500*time.Millisecond, cfg.Controller.LTBTokenRPCMaxDelay.Duration) +>>>>>>> 6b25787af (resource_control: allow configuration of the maximum retry time for the local bucket (#8352)):pkg/mcs/resourcemanager/server/config_test.go re.LessOrEqual(math.Abs(cfg.Controller.RequestUnit.CPUMsCost-5), 1e-7) re.LessOrEqual(math.Abs(cfg.Controller.RequestUnit.WriteCostPerByte-4), 1e-7) re.LessOrEqual(math.Abs(cfg.Controller.RequestUnit.WriteBaseCost-3), 1e-7) diff --git a/tests/integrations/mcs/resource_manager/resource_manager_test.go b/tests/integrations/mcs/resource_manager/resource_manager_test.go index 17d1a3090c4..502259462f7 100644 --- a/tests/integrations/mcs/resource_manager/resource_manager_test.go +++ b/tests/integrations/mcs/resource_manager/resource_manager_test.go @@ -1235,10 +1235,22 @@ func (suite *resourceManagerClientTestSuite) TestResourceGroupControllerConfigCh configURL := "/resource-manager/api/v1/config/controller" waitDuration := 10 * time.Second + tokenRPCMaxDelay := 2 * time.Second readBaseCost := 1.5 defaultCfg := controller.DefaultConfig() +<<<<<<< HEAD:tests/integrations/mcs/resource_manager/resource_manager_test.go // failpoint enableDegradedMode will setup and set it be 1s. defaultCfg.DegradedModeWaitDuration.Duration = time.Second +======= + expectCfg := server.ControllerConfig{ + // failpoint enableDegradedMode will setup and set it be 1s. + DegradedModeWaitDuration: typeutil.NewDuration(time.Second), + LTBMaxWaitDuration: typeutil.Duration(defaultCfg.LTBMaxWaitDuration), + LTBTokenRPCMaxDelay: typeutil.Duration(defaultCfg.LTBTokenRPCMaxDelay), + RequestUnit: server.RequestUnitConfig(defaultCfg.RequestUnit), + EnableControllerTraceLog: defaultCfg.EnableControllerTraceLog, + } +>>>>>>> 6b25787af (resource_control: allow configuration of the maximum retry time for the local bucket (#8352)):tests/integrations/mcs/resourcemanager/resource_manager_test.go expectRUCfg := controller.GenerateRUConfig(defaultCfg) // initial config verification respString := sendRequest("GET", getAddr()+configURL, nil) @@ -1257,6 +1269,13 @@ func (suite *resourceManagerClientTestSuite) TestResourceGroupControllerConfigCh value: waitDuration, expected: func(ruConfig *controller.RUConfig) { ruConfig.DegradedModeWaitDuration = waitDuration }, }, + { + configJSON: fmt.Sprintf(`{"ltb-token-rpc-max-delay": "%v"}`, tokenRPCMaxDelay), + value: waitDuration, + expected: func(ruConfig *controller.RUConfig) { + ruConfig.WaitRetryTimes = int(tokenRPCMaxDelay / ruConfig.WaitRetryInterval) + }, + }, { configJSON: fmt.Sprintf(`{"ltb-max-wait-duration": "%v"}`, waitDuration), value: waitDuration,