From 82bef5375680471135c2c3f6c919eb44aa635e92 Mon Sep 17 00:00:00 2001 From: Alex Robinson Date: Wed, 10 Oct 2018 17:39:30 -0500 Subject: [PATCH] storage: Jitter the StoreRebalancer loop's timing Release note: None --- pkg/storage/store_rebalancer.go | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/pkg/storage/store_rebalancer.go b/pkg/storage/store_rebalancer.go index 62a3c00b7bab..b5c444cd1d15 100644 --- a/pkg/storage/store_rebalancer.go +++ b/pkg/storage/store_rebalancer.go @@ -17,6 +17,7 @@ package storage import ( "context" "math" + "math/rand" "sort" "time" @@ -27,6 +28,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/cockroachdb/cockroach/pkg/util/metric" "github.com/cockroachdb/cockroach/pkg/util/stop" + "github.com/cockroachdb/cockroach/pkg/util/timeutil" "go.etcd.io/etcd/raft" ) @@ -173,8 +175,9 @@ func (sr *StoreRebalancer) Start(ctx context.Context, stopper *stop.Stopper) { // Start a goroutine that watches and proactively renews certain // expiration-based leases. stopper.RunWorker(ctx, func(ctx context.Context) { - ticker := time.NewTicker(storeRebalancerTimerDuration) - defer ticker.Stop() + timer := timeutil.NewTimer() + defer timer.Stop() + timer.Reset(jitteredInterval(storeRebalancerTimerDuration)) for { // Wait out the first tick before doing anything since the store is still // starting up and we might as well wait for some qps/wps stats to @@ -182,7 +185,9 @@ func (sr *StoreRebalancer) Start(ctx context.Context, stopper *stop.Stopper) { select { case <-stopper.ShouldQuiesce(): return - case <-ticker.C: + case <-timer.C: + timer.Read = true + timer.Reset(jitteredInterval(storeRebalancerTimerDuration)) } if !sr.st.Version.IsMinSupported(cluster.VersionLoadBasedRebalancing) { @@ -675,3 +680,9 @@ func storeListToMap(sl StoreList) map[roachpb.StoreID]*roachpb.StoreDescriptor { } return storeMap } + +// jitteredInterval returns a randomly jittered (+/-25%) duration +// from checkInterval. +func jitteredInterval(interval time.Duration) time.Duration { + return time.Duration(float64(interval) * (0.75 + 0.5*rand.Float64())) +}