gRPC: Support S2 compression

Support S2 and S2 in Snappy compatible mode gRPC compression.

Signed-off-by: Arve Knudsen <arve.knudsen@gmail.com>

CHANGELOG.md

@@ -50,6 +50,15 @@
 * [FEATURE] Query frontend: added new query pruning middleware to enable pruning dead code (eg. expressions that cannot produce any results) and simplifying expressions (eg. expressions that can be evaluated immediately) in queries. #9086
 * [FEATURE] Ruler: added experimental configuration, `-ruler.rule-evaluation-write-enabled`, to disable writing the result of rule evaluation to ingesters. This feature can be used for testing purposes. #9060
 * [FEATURE] Ingester: added experimental configuration `ingester.ignore-ooo-exemplars`. When set to `true` out of order exemplars are no longer reported to the remote write client. #9151
+* [FEATURE] gRPC: Add flags for using respectively S2 or Snappy compatible S2 gRPC compression. #9322
+ * `-alertmanager.alertmanager-client.grpc-compression`
+ * `-ingester.client.grpc-compression`
+ * `-querier.frontend-client.grpc-compression`
+ * `-querier.scheduler-client.grpc-compression`
+ * `-query-frontend.grpc-client-config.grpc-compression`
+ * `-query-scheduler.grpc-client-config.grpc-compression`
+ * `-ruler.client.grpc-compression`
+ * `-ruler.query-frontend.grpc-client-config.grpc-compression`
 * [ENHANCEMENT] Compactor: Add `cortex_compactor_compaction_job_duration_seconds` and `cortex_compactor_compaction_job_blocks` histogram metrics to track duration of individual compaction jobs and number of blocks per job. #8371
 * [ENHANCEMENT] Rules: Added per namespace max rules per rule group limit. The maximum number of rules per rule groups for all namespaces continues to be configured by `-ruler.max-rules-per-rule-group`, but now, this can be superseded by the new `-ruler.max-rules-per-rule-group-by-namespace` option on a per namespace basis. This new limit can be overridden using the overrides mechanism to be applied per-tenant. #8378
 * [ENHANCEMENT] Rules: Added per namespace max rule groups per tenant limit. The maximum number of rule groups per rule tenant for all namespaces continues to be configured by `-ruler.max-rule-groups-per-tenant`, but now, this can be superseded by the new `-ruler.max-rule-groups-per-tenant-by-namespace` option on a per namespace basis. This new limit can be overridden using the overrides mechanism to be applied per-tenant. #8425

cmd/mimir/config-descriptor.json

@@ -2101,7 +2101,7 @@
  "kind": "field",
  "name": "grpc_compression",
  "required": false,
- "desc": "Use compression when sending messages. Supported values are: 'gzip', 'snappy' and '' (disable compression)",
+ "desc": "Use compression when sending messages. Supported values are: 'gzip', 'snappy', 's2', 's2-snappy' and '' (disable compression)",
  "fieldValue": null,
  "fieldDefaultValue": "",
  "fieldFlag": "ingester.client.grpc-compression",
@@ -4838,7 +4838,7 @@
  "kind": "field",
  "name": "grpc_compression",
  "required": false,
- "desc": "Use compression when sending messages. Supported values are: 'gzip', 'snappy' and '' (disable compression)",
+ "desc": "Use compression when sending messages. Supported values are: 'gzip', 'snappy', 's2', 's2-snappy' and '' (disable compression)",
  "fieldValue": null,
  "fieldDefaultValue": "",
  "fieldFlag": "querier.frontend-client.grpc-compression",
@@ -5100,7 +5100,7 @@
  "kind": "field",
  "name": "grpc_compression",
  "required": false,
- "desc": "Use compression when sending messages. Supported values are: 'gzip', 'snappy' and '' (disable compression)",
+ "desc": "Use compression when sending messages. Supported values are: 'gzip', 'snappy', 's2', 's2-snappy' and '' (disable compression)",
  "fieldValue": null,
  "fieldDefaultValue": "",
  "fieldFlag": "querier.scheduler-client.grpc-compression",
@@ -5491,7 +5491,7 @@
  "kind": "field",
  "name": "grpc_compression",
  "required": false,
- "desc": "Use compression when sending messages. Supported values are: 'gzip', 'snappy' and '' (disable compression)",
+ "desc": "Use compression when sending messages. Supported values are: 'gzip', 'snappy', 's2', 's2-snappy' and '' (disable compression)",
  "fieldValue": null,
  "fieldDefaultValue": "",
  "fieldFlag": "query-frontend.grpc-client-config.grpc-compression",
@@ -11455,7 +11455,7 @@
  "kind": "field",
  "name": "grpc_compression",
  "required": false,
- "desc": "Use compression when sending messages. Supported values are: 'gzip', 'snappy' and '' (disable compression)",
+ "desc": "Use compression when sending messages. Supported values are: 'gzip', 'snappy', 's2', 's2-snappy' and '' (disable compression)",
  "fieldValue": null,
  "fieldDefaultValue": "",
  "fieldFlag": "ruler.client.grpc-compression",
@@ -12420,7 +12420,7 @@
  "kind": "field",
  "name": "grpc_compression",
  "required": false,
- "desc": "Use compression when sending messages. Supported values are: 'gzip', 'snappy' and '' (disable compression)",
+ "desc": "Use compression when sending messages. Supported values are: 'gzip', 'snappy', 's2', 's2-snappy' and '' (disable compression)",
  "fieldValue": null,
  "fieldDefaultValue": "",
  "fieldFlag": "ruler.query-frontend.grpc-client-config.grpc-compression",
@@ -14677,7 +14677,7 @@
  "kind": "field",
  "name": "grpc_compression",
  "required": false,
- "desc": "Use compression when sending messages. Supported values are: 'gzip', 'snappy' and '' (disable compression)",
+ "desc": "Use compression when sending messages. Supported values are: 'gzip', 'snappy', 's2', 's2-snappy' and '' (disable compression)",
  "fieldValue": null,
  "fieldDefaultValue": "",
  "fieldFlag": "alertmanager.alertmanager-client.grpc-compression",
@@ -16254,7 +16254,7 @@
  "kind": "field",
  "name": "grpc_compression",
  "required": false,
- "desc": "Use compression when sending messages. Supported values are: 'gzip', 'snappy' and '' (disable compression)",
+ "desc": "Use compression when sending messages. Supported values are: 'gzip', 'snappy', 's2', 's2-snappy' and '' (disable compression)",
  "fieldValue": null,
  "fieldDefaultValue": "",
  "fieldFlag": "query-scheduler.grpc-client-config.grpc-compression",

cmd/mimir/help-all.txt.tmpl

@@ -154,7 +154,7 @@ Usage of ./cmd/mimir/mimir:
  -alertmanager.alertmanager-client.grpc-client-rate-limit-burst int
  Rate limit burst for gRPC client.
  -alertmanager.alertmanager-client.grpc-compression string
- Use compression when sending messages. Supported values are: 'gzip', 'snappy' and '' (disable compression)
+ Use compression when sending messages. Supported values are: 'gzip', 'snappy', 's2', 's2-snappy' and '' (disable compression)
  -alertmanager.alertmanager-client.grpc-max-recv-msg-size int
  gRPC client max receive message size (bytes). (default 104857600)
  -alertmanager.alertmanager-client.grpc-max-send-msg-size int
@@ -1414,7 +1414,7 @@ Usage of ./cmd/mimir/mimir:
  -ingester.client.grpc-client-rate-limit-burst int
  Rate limit burst for gRPC client.
  -ingester.client.grpc-compression string
- Use compression when sending messages. Supported values are: 'gzip', 'snappy' and '' (disable compression)
+ Use compression when sending messages. Supported values are: 'gzip', 'snappy', 's2', 's2-snappy' and '' (disable compression)
  -ingester.client.grpc-max-recv-msg-size int
  gRPC client max receive message size (bytes). (default 104857600)
  -ingester.client.grpc-max-send-msg-size int
@@ -1868,7 +1868,7 @@ Usage of ./cmd/mimir/mimir:
  -querier.frontend-client.grpc-client-rate-limit-burst int
  Rate limit burst for gRPC client.
  -querier.frontend-client.grpc-compression string
- Use compression when sending messages. Supported values are: 'gzip', 'snappy' and '' (disable compression)
+ Use compression when sending messages. Supported values are: 'gzip', 'snappy', 's2', 's2-snappy' and '' (disable compression)
  -querier.frontend-client.grpc-max-recv-msg-size int
  gRPC client max receive message size (bytes). (default 104857600)
  -querier.frontend-client.grpc-max-send-msg-size int
@@ -1972,7 +1972,7 @@ Usage of ./cmd/mimir/mimir:
  -querier.scheduler-client.grpc-client-rate-limit-burst int
  Rate limit burst for gRPC client.
  -querier.scheduler-client.grpc-compression string
- Use compression when sending messages. Supported values are: 'gzip', 'snappy' and '' (disable compression)
+ Use compression when sending messages. Supported values are: 'gzip', 'snappy', 's2', 's2-snappy' and '' (disable compression)
  -querier.scheduler-client.grpc-max-recv-msg-size int
  gRPC client max receive message size (bytes). (default 104857600)
  -querier.scheduler-client.grpc-max-send-msg-size int
@@ -2050,7 +2050,7 @@ Usage of ./cmd/mimir/mimir:
  -query-frontend.grpc-client-config.grpc-client-rate-limit-burst int
  Rate limit burst for gRPC client.
  -query-frontend.grpc-client-config.grpc-compression string
- Use compression when sending messages. Supported values are: 'gzip', 'snappy' and '' (disable compression)
+ Use compression when sending messages. Supported values are: 'gzip', 'snappy', 's2', 's2-snappy' and '' (disable compression)
  -query-frontend.grpc-client-config.grpc-max-recv-msg-size int
  gRPC client max receive message size (bytes). (default 104857600)
  -query-frontend.grpc-client-config.grpc-max-send-msg-size int
@@ -2256,7 +2256,7 @@ Usage of ./cmd/mimir/mimir:
  -query-scheduler.grpc-client-config.grpc-client-rate-limit-burst int
  Rate limit burst for gRPC client.
  -query-scheduler.grpc-client-config.grpc-compression string
- Use compression when sending messages. Supported values are: 'gzip', 'snappy' and '' (disable compression)
+ Use compression when sending messages. Supported values are: 'gzip', 'snappy', 's2', 's2-snappy' and '' (disable compression)
  -query-scheduler.grpc-client-config.grpc-max-recv-msg-size int
  gRPC client max receive message size (bytes). (default 104857600)
  -query-scheduler.grpc-client-config.grpc-max-send-msg-size int
@@ -2628,7 +2628,7 @@ Usage of ./cmd/mimir/mimir:
  -ruler.client.grpc-client-rate-limit-burst int
  Rate limit burst for gRPC client.
  -ruler.client.grpc-compression string
- Use compression when sending messages. Supported values are: 'gzip', 'snappy' and '' (disable compression)
+ Use compression when sending messages. Supported values are: 'gzip', 'snappy', 's2', 's2-snappy' and '' (disable compression)
  -ruler.client.grpc-max-recv-msg-size int
  gRPC client max receive message size (bytes). (default 104857600)
  -ruler.client.grpc-max-send-msg-size int
@@ -2716,7 +2716,7 @@ Usage of ./cmd/mimir/mimir:
  -ruler.query-frontend.grpc-client-config.grpc-client-rate-limit-burst int
  Rate limit burst for gRPC client.
  -ruler.query-frontend.grpc-client-config.grpc-compression string
- Use compression when sending messages. Supported values are: 'gzip', 'snappy' and '' (disable compression)
+ Use compression when sending messages. Supported values are: 'gzip', 'snappy', 's2', 's2-snappy' and '' (disable compression)
  -ruler.query-frontend.grpc-client-config.grpc-max-recv-msg-size int
  gRPC client max receive message size (bytes). (default 104857600)
  -ruler.query-frontend.grpc-client-config.grpc-max-send-msg-size int

docs/sources/mimir/configure/configuration-parameters/index.md

@@ -2337,7 +2337,7 @@ alertmanager_client:
  [max_send_msg_size: <int> | default = 104857600]
 
  # (advanced) Use compression when sending messages. Supported values are:
- # 'gzip', 'snappy' and '' (disable compression)
+ # 'gzip', 'snappy', 's2', 's2-snappy' and '' (disable compression)
  # CLI flag: -alertmanager.alertmanager-client.grpc-compression
  [grpc_compression: <string> | default = ""]
 
@@ -2600,7 +2600,7 @@ The `grpc_client` block configures the gRPC client used to communicate between t
 [max_send_msg_size: <int> | default = 104857600]
 
 # (advanced) Use compression when sending messages. Supported values are:
-# 'gzip', 'snappy' and '' (disable compression)
+# 'gzip', 'snappy', 's2', 's2-snappy' and '' (disable compression)
 # CLI flag: -<prefix>.grpc-compression
 [grpc_compression: <string> | default = ""]
 

go.mod

@@ -200,7 +200,7 @@ require (
  github.com/google/go-querystring v1.1.0 // indirect
  github.com/google/pprof v0.0.0-20240711041743-f6c9dda6c6da // indirect
  github.com/google/s2a-go v0.1.8 // indirect
- github.com/googleapis/enterprise-certificate-proxy v0.3.3 // indirect
+ github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect
  github.com/googleapis/gax-go/v2 v2.13.0 // indirect
  github.com/gosimple/slug v1.1.1 // indirect
  github.com/grafana/gomemcache v0.0.0-20240229205252-cd6a66d6fb56 // indirect

go.sum

@@ -1218,8 +1218,8 @@ github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY
 github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg=
 github.com/googleapis/enterprise-certificate-proxy v0.2.1/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k=
 github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k=
-github.com/googleapis/enterprise-certificate-proxy v0.3.3 h1:QRje2j5GZimBzlbhGA2V2QlGNgL8G6e+wGo/+/2bWI0=
-github.com/googleapis/enterprise-certificate-proxy v0.3.3/go.mod h1:YKe7cfqYXjKGpGvmSg28/fFvhNzinZQm8DGnaburhGA=
+github.com/googleapis/enterprise-certificate-proxy v0.3.4 h1:XYIDZApgAnrN1c855gTgghdIA6Stxb52D5RnLI1SLyw=
+github.com/googleapis/enterprise-certificate-proxy v0.3.4/go.mod h1:YKe7cfqYXjKGpGvmSg28/fFvhNzinZQm8DGnaburhGA=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0=

pkg/alertmanager/alertmanager_client.go

@@ -20,6 +20,7 @@ import (
  "google.golang.org/grpc/health/grpc_health_v1"
 
  "github.com/grafana/mimir/pkg/alertmanager/alertmanagerpb"
+ "github.com/grafana/mimir/pkg/util/grpcencoding/s2"
 )
 
 // ClientsPool is the interface used to get the client from the pool for a specified address.
@@ -45,6 +46,7 @@ type ClientConfig struct {
 
 // RegisterFlagsWithPrefix registers flags with prefix.
 func (cfg *ClientConfig) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) {
+ cfg.GRPCClientConfig.CustomCompressors = []string{s2.Name, s2.SnappyCompatName}
  cfg.GRPCClientConfig.RegisterFlagsWithPrefix(prefix, f)
  f.DurationVar(&cfg.RemoteTimeout, prefix+".remote-timeout", 2*time.Second, "Timeout for downstream alertmanagers.")
 }

pkg/frontend/v2/frontend.go

@@ -38,6 +38,7 @@ import (
  "github.com/grafana/mimir/pkg/querier/stats"
  "github.com/grafana/mimir/pkg/scheduler/schedulerdiscovery"
  "github.com/grafana/mimir/pkg/util/globalerror"
+ "github.com/grafana/mimir/pkg/util/grpcencoding/s2"
  "github.com/grafana/mimir/pkg/util/httpgrpcutil"
  "github.com/grafana/mimir/pkg/util/spanlogger"
 )
@@ -76,6 +77,7 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet, logger log.Logger) {
  f.StringVar(&cfg.Addr, "query-frontend.instance-addr", "", "IP address to advertise to the querier (via scheduler) (default is auto-detected from network interfaces).")
  f.IntVar(&cfg.Port, "query-frontend.instance-port", 0, "Port to advertise to querier (via scheduler) (defaults to server.grpc-listen-port).")
 
+ cfg.GRPCClientConfig.CustomCompressors = []string{s2.Name, s2.SnappyCompatName}
  cfg.GRPCClientConfig.RegisterFlagsWithPrefix("query-frontend.grpc-client-config", f)
 }
 

pkg/ingester/client/client.go

@@ -16,6 +16,7 @@ import (
 
  "github.com/grafana/mimir/pkg/mimirpb"
  querierapi "github.com/grafana/mimir/pkg/querier/api"
+ "github.com/grafana/mimir/pkg/util/grpcencoding/s2"
 )
 
 // HealthAndIngesterClient is the union of IngesterClient and grpc_health_v1.HealthClient.
@@ -70,6 +71,7 @@ type Config struct {
 
 // RegisterFlags registers configuration settings used by the ingester client config.
 func (cfg *Config) RegisterFlags(f *flag.FlagSet) {
+ cfg.GRPCClientConfig.CustomCompressors = []string{s2.Name, s2.SnappyCompatName}
  cfg.GRPCClientConfig.RegisterFlagsWithPrefix("ingester.client", f)
 }
 

pkg/querier/worker/worker.go

@@ -24,6 +24,7 @@ import (
  "google.golang.org/grpc"
 
  "github.com/grafana/mimir/pkg/scheduler/schedulerdiscovery"
+ "github.com/grafana/mimir/pkg/util/grpcencoding/s2"
  "github.com/grafana/mimir/pkg/util/math"
 )
 
@@ -48,7 +49,9 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) {
  f.StringVar(&cfg.QuerierID, "querier.id", "", "Querier ID, sent to the query-frontend to identify requests from the same querier. Defaults to hostname.")
  f.BoolVar(&cfg.ResponseStreamingEnabled, "querier.response-streaming-enabled", false, "Enables streaming of responses from querier to query-frontend for response types that support it (currently only `active_series` responses do).")
 
+ cfg.QueryFrontendGRPCClientConfig.CustomCompressors = []string{s2.Name, s2.SnappyCompatName}
  cfg.QueryFrontendGRPCClientConfig.RegisterFlagsWithPrefix("querier.frontend-client", f)
+ cfg.QuerySchedulerGRPCClientConfig.CustomCompressors = []string{s2.Name, s2.SnappyCompatName}
  cfg.QuerySchedulerGRPCClientConfig.RegisterFlagsWithPrefix("querier.scheduler-client", f)
 }
 

pkg/ruler/remotequerier.go

@@ -35,6 +35,7 @@ import (
  "google.golang.org/grpc/codes"
 
  "github.com/grafana/mimir/pkg/querier/api"
+ "github.com/grafana/mimir/pkg/util/grpcencoding/s2"
  "github.com/grafana/mimir/pkg/util/spanlogger"
  "github.com/grafana/mimir/pkg/util/version"
 )
@@ -75,6 +76,7 @@ func (c *QueryFrontendConfig) RegisterFlags(f *flag.FlagSet) {
  "GRPC listen address of the query-frontend(s). Must be a DNS address (prefixed with dns:///) "+
  "to enable client side load balancing.")
 
+ c.GRPCClientConfig.CustomCompressors = []string{s2.Name, s2.SnappyCompatName}
  c.GRPCClientConfig.RegisterFlagsWithPrefix("ruler.query-frontend.grpc-client-config", f)
 
  f.StringVar(&c.QueryResultResponseFormat, "ruler.query-frontend.query-result-response-format", formatProtobuf, fmt.Sprintf("Format to use when retrieving query results from query-frontends. Supported values: %s", strings.Join(allFormats, ", ")))

pkg/ruler/ruler.go

@@ -40,6 +40,7 @@ import (
  "github.com/grafana/mimir/pkg/ruler/rulestore"
  "github.com/grafana/mimir/pkg/storage/tsdb/bucketcache"
  "github.com/grafana/mimir/pkg/util"
+ "github.com/grafana/mimir/pkg/util/grpcencoding/s2"
  util_log "github.com/grafana/mimir/pkg/util/log"
  "github.com/grafana/mimir/pkg/util/spanlogger"
  "github.com/grafana/mimir/pkg/util/validation"
@@ -167,6 +168,7 @@ func (cfg *Config) Validate(limits validation.Limits) error {
 
 // RegisterFlags adds the flags required to config this to the given FlagSet
 func (cfg *Config) RegisterFlags(f *flag.FlagSet, logger log.Logger) {
+ cfg.ClientTLSConfig.CustomCompressors = []string{s2.Name, s2.SnappyCompatName}
  cfg.ClientTLSConfig.RegisterFlagsWithPrefix("ruler.client", f)
  cfg.Ring.RegisterFlags(f, logger)
  cfg.Notifier.RegisterFlags(f)