gRPC: Support S2 compression

Support S2 and S2 in Snappy compatible mode gRPC compression.

Signed-off-by: Arve Knudsen <arve.knudsen@gmail.com>

CHANGELOG.md

@@ -49,6 +49,15 @@
 * [FEATURE] Query frontend: added new query pruning middleware to enable pruning dead code (eg. expressions that cannot produce any results) and simplifying expressions (eg. expressions that can be evaluated immediately) in queries. #9086
 * [FEATURE] Ruler: added experimental configuration, `-ruler.rule-evaluation-write-enabled`, to disable writing the result of rule evaluation to ingesters. This feature can be used for testing purposes. #9060
 * [FEATURE] Ingester: added experimental configuration `ingester.ignore-ooo-exemplars`. When set to `true` out of order exemplars are no longer reported to the remote write client. #9151
+* [FEATURE] gRPC: Add flags for using respectively S2 or Snappy compatible S2 gRPC compression. #9322
+ * `-alertmanager.alertmanager-client.grpc-compression`
+ * `-ingester.client.grpc-compression`
+ * `-querier.frontend-client.grpc-compression`
+ * `-querier.scheduler-client.grpc-compression`
+ * `-query-frontend.grpc-client-config.grpc-compression`
+ * `-query-scheduler.grpc-client-config.grpc-compression`
+ * `-ruler.client.grpc-compression`
+ * `-ruler.query-frontend.grpc-client-config.grpc-compression`
 * [ENHANCEMENT] Compactor: Add `cortex_compactor_compaction_job_duration_seconds` and `cortex_compactor_compaction_job_blocks` histogram metrics to track duration of individual compaction jobs and number of blocks per job. #8371
 * [ENHANCEMENT] Rules: Added per namespace max rules per rule group limit. The maximum number of rules per rule groups for all namespaces continues to be configured by `-ruler.max-rules-per-rule-group`, but now, this can be superseded by the new `-ruler.max-rules-per-rule-group-by-namespace` option on a per namespace basis. This new limit can be overridden using the overrides mechanism to be applied per-tenant. #8378
 * [ENHANCEMENT] Rules: Added per namespace max rule groups per tenant limit. The maximum number of rule groups per rule tenant for all namespaces continues to be configured by `-ruler.max-rule-groups-per-tenant`, but now, this can be superseded by the new `-ruler.max-rule-groups-per-tenant-by-namespace` option on a per namespace basis. This new limit can be overridden using the overrides mechanism to be applied per-tenant. #8425

cmd/mimir/config-descriptor.json

@@ -2101,7 +2101,7 @@
  "kind": "field",
  "name": "grpc_compression",
  "required": false,
- "desc": "Use compression when sending messages. Supported values are: 'gzip', 'snappy' and '' (disable compression)",
+ "desc": "Use compression when sending messages. Supported values are: 'gzip', 'snappy', 's2', 's2-snappy' and '' (disable compression)",
  "fieldValue": null,
  "fieldDefaultValue": "",
  "fieldFlag": "ingester.client.grpc-compression",
@@ -4838,7 +4838,7 @@
  "kind": "field",
  "name": "grpc_compression",
  "required": false,
- "desc": "Use compression when sending messages. Supported values are: 'gzip', 'snappy' and '' (disable compression)",
+ "desc": "Use compression when sending messages. Supported values are: 'gzip', 'snappy', 's2', 's2-snappy' and '' (disable compression)",
  "fieldValue": null,
  "fieldDefaultValue": "",
  "fieldFlag": "querier.frontend-client.grpc-compression",
@@ -5100,7 +5100,7 @@
  "kind": "field",
  "name": "grpc_compression",
  "required": false,
- "desc": "Use compression when sending messages. Supported values are: 'gzip', 'snappy' and '' (disable compression)",
+ "desc": "Use compression when sending messages. Supported values are: 'gzip', 'snappy', 's2', 's2-snappy' and '' (disable compression)",
  "fieldValue": null,
  "fieldDefaultValue": "",
  "fieldFlag": "querier.scheduler-client.grpc-compression",
@@ -5491,7 +5491,7 @@
  "kind": "field",
  "name": "grpc_compression",
  "required": false,
- "desc": "Use compression when sending messages. Supported values are: 'gzip', 'snappy' and '' (disable compression)",
+ "desc": "Use compression when sending messages. Supported values are: 'gzip', 'snappy', 's2', 's2-snappy' and '' (disable compression)",
  "fieldValue": null,
  "fieldDefaultValue": "",
  "fieldFlag": "query-frontend.grpc-client-config.grpc-compression",
@@ -11455,7 +11455,7 @@
  "kind": "field",
  "name": "grpc_compression",
  "required": false,
- "desc": "Use compression when sending messages. Supported values are: 'gzip', 'snappy' and '' (disable compression)",
+ "desc": "Use compression when sending messages. Supported values are: 'gzip', 'snappy', 's2', 's2-snappy' and '' (disable compression)",
  "fieldValue": null,
  "fieldDefaultValue": "",
  "fieldFlag": "ruler.client.grpc-compression",
@@ -12420,7 +12420,7 @@
  "kind": "field",
  "name": "grpc_compression",
  "required": false,
- "desc": "Use compression when sending messages. Supported values are: 'gzip', 'snappy' and '' (disable compression)",
+ "desc": "Use compression when sending messages. Supported values are: 'gzip', 'snappy', 's2', 's2-snappy' and '' (disable compression)",
  "fieldValue": null,
  "fieldDefaultValue": "",
  "fieldFlag": "ruler.query-frontend.grpc-client-config.grpc-compression",
@@ -14655,7 +14655,7 @@
  "kind": "field",
  "name": "grpc_compression",
  "required": false,
- "desc": "Use compression when sending messages. Supported values are: 'gzip', 'snappy' and '' (disable compression)",
+ "desc": "Use compression when sending messages. Supported values are: 'gzip', 'snappy', 's2', 's2-snappy' and '' (disable compression)",
  "fieldValue": null,
  "fieldDefaultValue": "",
  "fieldFlag": "alertmanager.alertmanager-client.grpc-compression",
@@ -16232,7 +16232,7 @@
  "kind": "field",
  "name": "grpc_compression",
  "required": false,
- "desc": "Use compression when sending messages. Supported values are: 'gzip', 'snappy' and '' (disable compression)",
+ "desc": "Use compression when sending messages. Supported values are: 'gzip', 'snappy', 's2', 's2-snappy' and '' (disable compression)",
  "fieldValue": null,
  "fieldDefaultValue": "",
  "fieldFlag": "query-scheduler.grpc-client-config.grpc-compression",

cmd/mimir/help-all.txt.tmpl

@@ -154,7 +154,7 @@ Usage of ./cmd/mimir/mimir:
  -alertmanager.alertmanager-client.grpc-client-rate-limit-burst int
  Rate limit burst for gRPC client.
  -alertmanager.alertmanager-client.grpc-compression string
- Use compression when sending messages. Supported values are: 'gzip', 'snappy' and '' (disable compression)
+ Use compression when sending messages. Supported values are: 'gzip', 'snappy', 's2', 's2-snappy' and '' (disable compression)
  -alertmanager.alertmanager-client.grpc-max-recv-msg-size int
  gRPC client max receive message size (bytes). (default 104857600)
  -alertmanager.alertmanager-client.grpc-max-send-msg-size int
@@ -1414,7 +1414,7 @@ Usage of ./cmd/mimir/mimir:
  -ingester.client.grpc-client-rate-limit-burst int
  Rate limit burst for gRPC client.
  -ingester.client.grpc-compression string
- Use compression when sending messages. Supported values are: 'gzip', 'snappy' and '' (disable compression)
+ Use compression when sending messages. Supported values are: 'gzip', 'snappy', 's2', 's2-snappy' and '' (disable compression)
  -ingester.client.grpc-max-recv-msg-size int
  gRPC client max receive message size (bytes). (default 104857600)
  -ingester.client.grpc-max-send-msg-size int
@@ -1868,7 +1868,7 @@ Usage of ./cmd/mimir/mimir:
  -querier.frontend-client.grpc-client-rate-limit-burst int
  Rate limit burst for gRPC client.
  -querier.frontend-client.grpc-compression string
- Use compression when sending messages. Supported values are: 'gzip', 'snappy' and '' (disable compression)
+ Use compression when sending messages. Supported values are: 'gzip', 'snappy', 's2', 's2-snappy' and '' (disable compression)
  -querier.frontend-client.grpc-max-recv-msg-size int
  gRPC client max receive message size (bytes). (default 104857600)
  -querier.frontend-client.grpc-max-send-msg-size int
@@ -1972,7 +1972,7 @@ Usage of ./cmd/mimir/mimir:
  -querier.scheduler-client.grpc-client-rate-limit-burst int
  Rate limit burst for gRPC client.
  -querier.scheduler-client.grpc-compression string
- Use compression when sending messages. Supported values are: 'gzip', 'snappy' and '' (disable compression)
+ Use compression when sending messages. Supported values are: 'gzip', 'snappy', 's2', 's2-snappy' and '' (disable compression)
  -querier.scheduler-client.grpc-max-recv-msg-size int
  gRPC client max receive message size (bytes). (default 104857600)
  -querier.scheduler-client.grpc-max-send-msg-size int
@@ -2050,7 +2050,7 @@ Usage of ./cmd/mimir/mimir:
  -query-frontend.grpc-client-config.grpc-client-rate-limit-burst int
  Rate limit burst for gRPC client.
  -query-frontend.grpc-client-config.grpc-compression string
- Use compression when sending messages. Supported values are: 'gzip', 'snappy' and '' (disable compression)
+ Use compression when sending messages. Supported values are: 'gzip', 'snappy', 's2', 's2-snappy' and '' (disable compression)
  -query-frontend.grpc-client-config.grpc-max-recv-msg-size int
  gRPC client max receive message size (bytes). (default 104857600)
  -query-frontend.grpc-client-config.grpc-max-send-msg-size int
@@ -2256,7 +2256,7 @@ Usage of ./cmd/mimir/mimir:
  -query-scheduler.grpc-client-config.grpc-client-rate-limit-burst int
  Rate limit burst for gRPC client.
  -query-scheduler.grpc-client-config.grpc-compression string
- Use compression when sending messages. Supported values are: 'gzip', 'snappy' and '' (disable compression)
+ Use compression when sending messages. Supported values are: 'gzip', 'snappy', 's2', 's2-snappy' and '' (disable compression)
  -query-scheduler.grpc-client-config.grpc-max-recv-msg-size int
  gRPC client max receive message size (bytes). (default 104857600)
  -query-scheduler.grpc-client-config.grpc-max-send-msg-size int
@@ -2628,7 +2628,7 @@ Usage of ./cmd/mimir/mimir:
  -ruler.client.grpc-client-rate-limit-burst int
  Rate limit burst for gRPC client.
  -ruler.client.grpc-compression string
- Use compression when sending messages. Supported values are: 'gzip', 'snappy' and '' (disable compression)
+ Use compression when sending messages. Supported values are: 'gzip', 'snappy', 's2', 's2-snappy' and '' (disable compression)
  -ruler.client.grpc-max-recv-msg-size int
  gRPC client max receive message size (bytes). (default 104857600)
  -ruler.client.grpc-max-send-msg-size int
@@ -2712,7 +2712,7 @@ Usage of ./cmd/mimir/mimir:
  -ruler.query-frontend.grpc-client-config.grpc-client-rate-limit-burst int
  Rate limit burst for gRPC client.
  -ruler.query-frontend.grpc-client-config.grpc-compression string
- Use compression when sending messages. Supported values are: 'gzip', 'snappy' and '' (disable compression)
+ Use compression when sending messages. Supported values are: 'gzip', 'snappy', 's2', 's2-snappy' and '' (disable compression)
  -ruler.query-frontend.grpc-client-config.grpc-max-recv-msg-size int
  gRPC client max receive message size (bytes). (default 104857600)
  -ruler.query-frontend.grpc-client-config.grpc-max-send-msg-size int

docs/sources/mimir/configure/configuration-parameters/index.md

@@ -2328,7 +2328,7 @@ alertmanager_client:
  [max_send_msg_size: <int> | default = 104857600]
 
  # (advanced) Use compression when sending messages. Supported values are:
- # 'gzip', 'snappy' and '' (disable compression)
+ # 'gzip', 'snappy', 's2', 's2-snappy' and '' (disable compression)
  # CLI flag: -alertmanager.alertmanager-client.grpc-compression
  [grpc_compression: <string> | default = ""]
 
@@ -2591,7 +2591,7 @@ The `grpc_client` block configures the gRPC client used to communicate between t
 [max_send_msg_size: <int> | default = 104857600]
 
 # (advanced) Use compression when sending messages. Supported values are:
-# 'gzip', 'snappy' and '' (disable compression)
+# 'gzip', 'snappy', 's2', 's2-snappy' and '' (disable compression)
 # CLI flag: -<prefix>.grpc-compression
 [grpc_compression: <string> | default = ""]
 

go.mod

@@ -22,7 +22,7 @@ require (
  github.com/golang/snappy v0.0.4
  github.com/google/gopacket v1.1.19
  github.com/gorilla/mux v1.8.1
- github.com/grafana/dskit v0.0.0-20240905221822-931a021fb06b
+ github.com/grafana/dskit v0.0.0-20240919080237-9102f24e6e9e
  github.com/grafana/e2e v0.1.2-0.20240118170847-db90b84177fc
  github.com/hashicorp/golang-lru v1.0.2 // indirect
  github.com/json-iterator/go v1.1.12
@@ -199,7 +199,7 @@ require (
  github.com/google/go-querystring v1.1.0 // indirect
  github.com/google/pprof v0.0.0-20240711041743-f6c9dda6c6da // indirect
  github.com/google/s2a-go v0.1.8 // indirect
- github.com/googleapis/enterprise-certificate-proxy v0.3.3 // indirect
+ github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect
  github.com/googleapis/gax-go/v2 v2.13.0 // indirect
  github.com/gosimple/slug v1.1.1 // indirect
  github.com/grafana/gomemcache v0.0.0-20240229205252-cd6a66d6fb56 // indirect

go.sum

@@ -1223,8 +1223,8 @@ github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY
 github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg=
 github.com/googleapis/enterprise-certificate-proxy v0.2.1/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k=
 github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k=
-github.com/googleapis/enterprise-certificate-proxy v0.3.3 h1:QRje2j5GZimBzlbhGA2V2QlGNgL8G6e+wGo/+/2bWI0=
-github.com/googleapis/enterprise-certificate-proxy v0.3.3/go.mod h1:YKe7cfqYXjKGpGvmSg28/fFvhNzinZQm8DGnaburhGA=
+github.com/googleapis/enterprise-certificate-proxy v0.3.4 h1:XYIDZApgAnrN1c855gTgghdIA6Stxb52D5RnLI1SLyw=
+github.com/googleapis/enterprise-certificate-proxy v0.3.4/go.mod h1:YKe7cfqYXjKGpGvmSg28/fFvhNzinZQm8DGnaburhGA=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0=
@@ -1255,8 +1255,8 @@ github.com/grafana-tools/sdk v0.0.0-20220919052116-6562121319fc h1:PXZQA2WCxe85T
 github.com/grafana-tools/sdk v0.0.0-20220919052116-6562121319fc/go.mod h1:AHHlOEv1+GGQ3ktHMlhuTUwo3zljV3QJbC0+8o2kn+4=
 github.com/grafana/alerting v0.0.0-20240906191856-cdc634f213ea h1:AGmVRk+9ZmzuiLJl6hzQE1vBlVz9wbEb2+J52Gui2ys=
 github.com/grafana/alerting v0.0.0-20240906191856-cdc634f213ea/go.mod h1:GMLi6d09Xqo96fCVUjNk//rcjP5NKEdjOzfWIffD5r4=
-github.com/grafana/dskit v0.0.0-20240905221822-931a021fb06b h1:x2HCzk29I0o5pRPfqWP/qwhXaPGlcz8pohq5kO1NZoE=
-github.com/grafana/dskit v0.0.0-20240905221822-931a021fb06b/go.mod h1:SPLNCARd4xdjCkue0O6hvuoveuS1dGJjDnfxYe405YQ=
+github.com/grafana/dskit v0.0.0-20240919080237-9102f24e6e9e h1:yGly8b1fa1FruasB9cCmbomRUk/EbfC8GGAs+0x6FpI=
+github.com/grafana/dskit v0.0.0-20240919080237-9102f24e6e9e/go.mod h1:SPLNCARd4xdjCkue0O6hvuoveuS1dGJjDnfxYe405YQ=
 github.com/grafana/e2e v0.1.2-0.20240118170847-db90b84177fc h1:BW+LjKJDz0So5LI8UZfW5neWeKpSkWqhmGjQFzcFfLM=
 github.com/grafana/e2e v0.1.2-0.20240118170847-db90b84177fc/go.mod h1:JVmqPBe8A/pZWwRoJW5ZjyALeY5OXMzPl7LrVXOdZAI=
 github.com/grafana/goautoneg v0.0.0-20240607115440-f335c04c58ce h1:WI1olbgS+sEl77qxEYbmt9TgRUz7iLqmjh8lYPpGlKQ=

pkg/alertmanager/alertmanager_client.go

@@ -20,6 +20,7 @@ import (
  "google.golang.org/grpc/health/grpc_health_v1"
 
  "github.com/grafana/mimir/pkg/alertmanager/alertmanagerpb"
+ "github.com/grafana/mimir/pkg/util/grpcencoding/s2"
 )
 
 // ClientsPool is the interface used to get the client from the pool for a specified address.
@@ -45,6 +46,7 @@ type ClientConfig struct {
 
 // RegisterFlagsWithPrefix registers flags with prefix.
 func (cfg *ClientConfig) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) {
+ cfg.GRPCClientConfig.CustomCompressors = []string{s2.Name, s2.SnappyCompatName}
  cfg.GRPCClientConfig.RegisterFlagsWithPrefix(prefix, f)
  f.DurationVar(&cfg.RemoteTimeout, prefix+".remote-timeout", 2*time.Second, "Timeout for downstream alertmanagers.")
 }

pkg/frontend/v2/frontend.go

@@ -38,6 +38,7 @@ import (
  "github.com/grafana/mimir/pkg/querier/stats"
  "github.com/grafana/mimir/pkg/scheduler/schedulerdiscovery"
  "github.com/grafana/mimir/pkg/util/globalerror"
+ "github.com/grafana/mimir/pkg/util/grpcencoding/s2"
  "github.com/grafana/mimir/pkg/util/httpgrpcutil"
  "github.com/grafana/mimir/pkg/util/spanlogger"
 )
@@ -76,6 +77,7 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet, logger log.Logger) {
  f.StringVar(&cfg.Addr, "query-frontend.instance-addr", "", "IP address to advertise to the querier (via scheduler) (default is auto-detected from network interfaces).")
  f.IntVar(&cfg.Port, "query-frontend.instance-port", 0, "Port to advertise to querier (via scheduler) (defaults to server.grpc-listen-port).")
 
+ cfg.GRPCClientConfig.CustomCompressors = []string{s2.Name, s2.SnappyCompatName}
  cfg.GRPCClientConfig.RegisterFlagsWithPrefix("query-frontend.grpc-client-config", f)
 }
 

pkg/ingester/client/client.go

@@ -16,6 +16,7 @@ import (
 
  "github.com/grafana/mimir/pkg/mimirpb"
  querierapi "github.com/grafana/mimir/pkg/querier/api"
+ "github.com/grafana/mimir/pkg/util/grpcencoding/s2"
 )
 
 // HealthAndIngesterClient is the union of IngesterClient and grpc_health_v1.HealthClient.
@@ -70,6 +71,7 @@ type Config struct {
 
 // RegisterFlags registers configuration settings used by the ingester client config.
 func (cfg *Config) RegisterFlags(f *flag.FlagSet) {
+ cfg.GRPCClientConfig.CustomCompressors = []string{s2.Name, s2.SnappyCompatName}
  cfg.GRPCClientConfig.RegisterFlagsWithPrefix("ingester.client", f)
 }
 

pkg/querier/worker/worker.go

@@ -24,6 +24,7 @@ import (
  "google.golang.org/grpc"
 
  "github.com/grafana/mimir/pkg/scheduler/schedulerdiscovery"
+ "github.com/grafana/mimir/pkg/util/grpcencoding/s2"
  "github.com/grafana/mimir/pkg/util/math"
 )
 
@@ -48,7 +49,9 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) {
  f.StringVar(&cfg.QuerierID, "querier.id", "", "Querier ID, sent to the query-frontend to identify requests from the same querier. Defaults to hostname.")
  f.BoolVar(&cfg.ResponseStreamingEnabled, "querier.response-streaming-enabled", false, "Enables streaming of responses from querier to query-frontend for response types that support it (currently only `active_series` responses do).")
 
+ cfg.QueryFrontendGRPCClientConfig.CustomCompressors = []string{s2.Name, s2.SnappyCompatName}
  cfg.QueryFrontendGRPCClientConfig.RegisterFlagsWithPrefix("querier.frontend-client", f)
+ cfg.QuerySchedulerGRPCClientConfig.CustomCompressors = []string{s2.Name, s2.SnappyCompatName}
  cfg.QuerySchedulerGRPCClientConfig.RegisterFlagsWithPrefix("querier.scheduler-client", f)
 }
 

pkg/ruler/remotequerier.go

@@ -35,6 +35,7 @@ import (
  "google.golang.org/grpc/codes"
 
  "github.com/grafana/mimir/pkg/querier/api"
+ "github.com/grafana/mimir/pkg/util/grpcencoding/s2"
  "github.com/grafana/mimir/pkg/util/spanlogger"
  "github.com/grafana/mimir/pkg/util/version"
 )
@@ -75,6 +76,7 @@ func (c *QueryFrontendConfig) RegisterFlags(f *flag.FlagSet) {
  "GRPC listen address of the query-frontend(s). Must be a DNS address (prefixed with dns:///) "+
  "to enable client side load balancing.")
 
+ c.GRPCClientConfig.CustomCompressors = []string{s2.Name, s2.SnappyCompatName}
  c.GRPCClientConfig.RegisterFlagsWithPrefix("ruler.query-frontend.grpc-client-config", f)
 
  f.StringVar(&c.QueryResultResponseFormat, "ruler.query-frontend.query-result-response-format", formatProtobuf, fmt.Sprintf("Format to use when retrieving query results from query-frontends. Supported values: %s", strings.Join(allFormats, ", ")))