Add MinIdleConnectionsHeadroom support to memcached client config

Signed-off-by: Marco Pracucci <marco@pracucci.com>

CHANGELOG.md

@@ -101,6 +101,7 @@
   * `<prefix>_cache_operations_total{backend="[memcached|redis]",...}`
   * `<prefix>_cache_requests_total{backend="[memcached|redis]",...}`
 * [ENHANCEMENT] Lifecycler: Added `HealthyInstancesInZoneCount` method returning the number of healthy instances in the ring that are registered in lifecycler's zone, updated during the last heartbeat period. #266
+* [ENHANCEMENT] Memcached: add `MinIdleConnectionsHeadroom` support. #269
 * [BUGFIX] spanlogger: Support multiple tenant IDs. #59
 * [BUGFIX] Memberlist: fixed corrupted packets when sending compound messages with more than 255 messages or messages bigger than 64KB. #85
 * [BUGFIX] Ring: `ring_member_ownership_percent` and `ring_tokens_owned` metrics are not updated on scale down. #109

cache/memcache_config.go

@@ -10,23 +10,26 @@ import (
 )
 
 var (
-	ErrNoMemcachedAddresses = errors.New("no memcached addresses configured")
+	ErrNoMemcachedAddresses              = errors.New("no memcached addresses configured")
+	ErrInvalidMinIdleConnectionsHeadroom = errors.New("memcached min idle connections headroom must be a number between 0 and 1")
 )
 
 type MemcachedConfig struct {
-	Addresses              string        `yaml:"addresses"`
-	Timeout                time.Duration `yaml:"timeout"`
-	MaxIdleConnections     int           `yaml:"max_idle_connections" category:"advanced"`
-	MaxAsyncConcurrency    int           `yaml:"max_async_concurrency" category:"advanced"`
-	MaxAsyncBufferSize     int           `yaml:"max_async_buffer_size" category:"advanced"`
-	MaxGetMultiConcurrency int           `yaml:"max_get_multi_concurrency" category:"advanced"`
-	MaxGetMultiBatchSize   int           `yaml:"max_get_multi_batch_size" category:"advanced"`
-	MaxItemSize            int           `yaml:"max_item_size" category:"advanced"`
+	Addresses                  string        `yaml:"addresses"`
+	Timeout                    time.Duration `yaml:"timeout"`
+	MinIdleConnectionsHeadroom float64       `yaml:"min_idle_connections_headroom" category:"advanced"`
+	MaxIdleConnections         int           `yaml:"max_idle_connections" category:"advanced"`
+	MaxAsyncConcurrency        int           `yaml:"max_async_concurrency" category:"advanced"`
+	MaxAsyncBufferSize         int           `yaml:"max_async_buffer_size" category:"advanced"`
+	MaxGetMultiConcurrency     int           `yaml:"max_get_multi_concurrency" category:"advanced"`
+	MaxGetMultiBatchSize       int           `yaml:"max_get_multi_batch_size" category:"advanced"`
+	MaxItemSize                int           `yaml:"max_item_size" category:"advanced"`
 }
 
 func (cfg *MemcachedConfig) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix string) {
 	f.StringVar(&cfg.Addresses, prefix+"addresses", "", "Comma-separated list of memcached addresses. Each address can be an IP address, hostname, or an entry specified in the DNS Service Discovery format.")
 	f.DurationVar(&cfg.Timeout, prefix+"timeout", 200*time.Millisecond, "The socket read/write timeout.")
+	f.Float64Var(&cfg.MinIdleConnectionsHeadroom, prefix+"min-idle-connections-headroom", 0, "The minimum number of idle connections to keep open as a percentage of the number of recently used idle connections. The percentage should be in the range [0, 1]. If zero, idle connections are kept open indefinitely.")
 	f.IntVar(&cfg.MaxIdleConnections, prefix+"max-idle-connections", 100, "The maximum number of idle connections that will be maintained per address.")
 	f.IntVar(&cfg.MaxAsyncConcurrency, prefix+"max-async-concurrency", 50, "The maximum number of concurrent asynchronous operations can occur.")
 	f.IntVar(&cfg.MaxAsyncBufferSize, prefix+"max-async-buffer-size", 25000, "The maximum number of enqueued asynchronous operations allowed.")
@@ -47,19 +50,23 @@ func (cfg *MemcachedConfig) Validate() error {
 	if len(cfg.GetAddresses()) == 0 {
 		return ErrNoMemcachedAddresses
 	}
+	if cfg.MinIdleConnectionsHeadroom < 0 || cfg.MinIdleConnectionsHeadroom > 1 {
+		return ErrInvalidMinIdleConnectionsHeadroom
+	}
 	return nil
 }
 
 func (cfg *MemcachedConfig) ToMemcachedClientConfig() MemcachedClientConfig {
 	return MemcachedClientConfig{
-		Addresses:                 cfg.GetAddresses(),
-		Timeout:                   cfg.Timeout,
-		MaxIdleConnections:        cfg.MaxIdleConnections,
-		MaxAsyncConcurrency:       cfg.MaxAsyncConcurrency,
-		MaxAsyncBufferSize:        cfg.MaxAsyncBufferSize,
-		MaxGetMultiConcurrency:    cfg.MaxGetMultiConcurrency,
-		MaxGetMultiBatchSize:      cfg.MaxGetMultiBatchSize,
-		MaxItemSize:               flagext.Bytes(cfg.MaxItemSize),
-		DNSProviderUpdateInterval: 30 * time.Second,
+		Addresses:                  cfg.GetAddresses(),
+		Timeout:                    cfg.Timeout,
+		MinIdleConnectionsHeadroom: cfg.MinIdleConnectionsHeadroom,
+		MaxIdleConnections:         cfg.MaxIdleConnections,
+		MaxAsyncConcurrency:        cfg.MaxAsyncConcurrency,
+		MaxAsyncBufferSize:         cfg.MaxAsyncBufferSize,
+		MaxGetMultiConcurrency:     cfg.MaxGetMultiConcurrency,
+		MaxGetMultiBatchSize:       cfg.MaxGetMultiBatchSize,
+		MaxItemSize:                flagext.Bytes(cfg.MaxItemSize),
+		DNSProviderUpdateInterval:  30 * time.Second,
 	}
 }

cache/memcached_client.go

@@ -62,6 +62,12 @@ type MemcachedClientConfig struct {
 	// Timeout specifies the socket read/write timeout.
 	Timeout time.Duration `yaml:"timeout"`
 
+	// MinIdleConnectionsHeadroom specifies the minimum number of idle connections
+	// to keep open as a percentage of the number of recently used idle connections.
+	// The percentage should be in the range [0, 1]. If zero, idle connections are
+	// kept open indefinitely.
+	MinIdleConnectionsHeadroom float64 `yaml:"min_idle_connections_headroom"`
+
 	// MaxIdleConnections specifies the maximum number of idle connections that
 	// will be maintained per address. For better performances, this should be
 	// set to a number higher than your peak parallel requests.
@@ -163,6 +169,7 @@ func NewMemcachedClientWithConfig(logger log.Logger, name string, config Memcach
 
 	client := memcache.NewFromSelector(selector)
 	client.Timeout = config.Timeout
+	client.MinIdleConnsHeadroom = config.MinIdleConnectionsHeadroom
 	client.MaxIdleConns = config.MaxIdleConnections
 
 	if reg != nil {
@@ -215,14 +222,15 @@ func newMemcachedClient(
 		Name: clientInfoMetricName,
 		Help: "A metric with a constant '1' value labeled by configuration options from which memcached client was configured.",
 		ConstLabels: prometheus.Labels{
-			"timeout":                      config.Timeout.String(),
-			"max_idle_connections":         strconv.Itoa(config.MaxIdleConnections),
-			"max_async_concurrency":        strconv.Itoa(config.MaxAsyncConcurrency),
-			"max_async_buffer_size":        strconv.Itoa(config.MaxAsyncBufferSize),
-			"max_item_size":                strconv.FormatUint(uint64(config.MaxItemSize), 10),
-			"max_get_multi_concurrency":    strconv.Itoa(config.MaxGetMultiConcurrency),
-			"max_get_multi_batch_size":     strconv.Itoa(config.MaxGetMultiBatchSize),
-			"dns_provider_update_interval": config.DNSProviderUpdateInterval.String(),
+			"timeout":                       config.Timeout.String(),
+			"min_idle_connections_headroom": fmt.Sprintf("%f.2", config.MinIdleConnectionsHeadroom),
+			"max_idle_connections":          strconv.Itoa(config.MaxIdleConnections),
+			"max_async_concurrency":         strconv.Itoa(config.MaxAsyncConcurrency),
+			"max_async_buffer_size":         strconv.Itoa(config.MaxAsyncBufferSize),
+			"max_item_size":                 strconv.FormatUint(uint64(config.MaxItemSize), 10),
+			"max_get_multi_concurrency":     strconv.Itoa(config.MaxGetMultiConcurrency),
+			"max_get_multi_batch_size":      strconv.Itoa(config.MaxGetMultiBatchSize),
+			"dns_provider_update_interval":  config.DNSProviderUpdateInterval.String(),
 		},
 	},
 		func() float64 { return 1 },

go.mod

@@ -14,7 +14,7 @@ require (
 	github.com/gogo/protobuf v1.3.2
 	github.com/gogo/status v1.1.0
 	github.com/golang/snappy v0.0.4
-	github.com/grafana/gomemcache v0.0.0-20230105173749-11f792309e1f
+	github.com/grafana/gomemcache v0.0.0-20230218080732-a3b7d09c9878
 	github.com/hashicorp/consul/api v1.15.3
 	github.com/hashicorp/go-cleanhttp v0.5.2
 	github.com/hashicorp/go-sockaddr v1.0.2

go.sum

@@ -262,8 +262,8 @@ github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+
 github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
 github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
 github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
-github.com/grafana/gomemcache v0.0.0-20230105173749-11f792309e1f h1:ANwIMe7kOiMNTK88tusoNDb840pWVskI4rCrdoMv5i0=
-github.com/grafana/gomemcache v0.0.0-20230105173749-11f792309e1f/go.mod h1:PGk3RjYHpxMM8HFPhKKo+vve3DdlPUELZLSDEFehPuU=
+github.com/grafana/gomemcache v0.0.0-20230218080732-a3b7d09c9878 h1:+C7fVSlAdZv4LxrubonZoR4AAKA+/xdkVk9CF0zWI4s=
+github.com/grafana/gomemcache v0.0.0-20230218080732-a3b7d09c9878/go.mod h1:PGk3RjYHpxMM8HFPhKKo+vve3DdlPUELZLSDEFehPuU=
 github.com/grafana/memberlist v0.3.1-0.20220708130638-bd88e10a3d91 h1:/NipyHnOmvRsVzj81j2qE0VxsvsqhOB0f4vJIhk2qCQ=
 github.com/grafana/memberlist v0.3.1-0.20220708130638-bd88e10a3d91/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=