Helm: prevent query-scheduler from terminating connections (#3262)

Helm: prevent query-scheduler from terminating connections Signed-off-by: Dimitar Dimitrov <dimitar.dimitrov@grafana.com> (cherry picked from commit 0107f45)
grafana · Oct 20, 2022 · dcce483 · dcce483
1 parent 7ef42f0
commit dcce483
Show file tree

Hide file tree

Showing 8 changed files with 15 additions and 0 deletions.
diff --git a/operations/helm/charts/mimir-distributed/CHANGELOG.md b/operations/helm/charts/mimir-distributed/CHANGELOG.md
@@ -78,6 +78,7 @@ Entries should include a reference to the Pull Request that introduced the chang
 * [ENHANCEMENT] Add podAntiAffinity to sizing plans (small.yaml, large.yaml, capped-small.yaml, capped-large.yaml). #2906
 * [ENHANCEMENT] Add ability to configure and run mimir-continuous-test. #3117
 * [BUGFIX] Fix wrong label selector in ingester anti affinity rules in the sizing plans. #2906
+* [BUGFIX] Query-scheduler no longer periodically terminates connections from query-frontends and queriers. This caused some queries to time out and EOF errors in the logs. #3262
 
 ## 3.1.0
 

diff --git a/operations/helm/charts/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml b/operations/helm/charts/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml
@@ -44,6 +44,8 @@ spec:
             - "-target=query-scheduler"
             - "-config.expand-env=true"
             - "-config.file=/etc/mimir/mimir.yaml"
+            - "-server.grpc.keepalive.max-connection-age=2562047h" # 100000 days, effectively infinity
+            - "-server.grpc.keepalive.max-connection-age-grace=2562047h" # 100000 days, effectively infinity
           {{- range $key, $value := .Values.query_scheduler.extraArgs }}
             - "-{{ $key }}={{ $value }}"
           {{- end }}

diff --git a/...led-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml b/...led-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml
@@ -45,6 +45,8 @@ spec:
             - "-target=query-scheduler"
             - "-config.expand-env=true"
             - "-config.file=/etc/mimir/mimir.yaml"
+            - "-server.grpc.keepalive.max-connection-age=2562047h" # 100000 days, effectively infinity
+            - "-server.grpc.keepalive.max-connection-age-grace=2562047h" # 100000 days, effectively infinity
           volumeMounts:
             - name: runtime-config
               mountPath: /var/mimir

diff --git a/...ing-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml b/...ing-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml
@@ -45,6 +45,8 @@ spec:
             - "-target=query-scheduler"
             - "-config.expand-env=true"
             - "-config.file=/etc/mimir/mimir.yaml"
+            - "-server.grpc.keepalive.max-connection-age=2562047h" # 100000 days, effectively infinity
+            - "-server.grpc.keepalive.max-connection-age-grace=2562047h" # 100000 days, effectively infinity
           volumeMounts:
             - name: runtime-config
               mountPath: /var/mimir

diff --git a/...map-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml b/...map-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml
@@ -46,6 +46,8 @@ spec:
             - "-target=query-scheduler"
             - "-config.expand-env=true"
             - "-config.file=/etc/mimir/mimir.yaml"
+            - "-server.grpc.keepalive.max-connection-age=2562047h" # 100000 days, effectively infinity
+            - "-server.grpc.keepalive.max-connection-age-grace=2562047h" # 100000 days, effectively infinity
           volumeMounts:
             - name: runtime-config
               mountPath: /var/mimir

diff --git a/...bel-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml b/...bel-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml
@@ -43,6 +43,8 @@ spec:
             - "-target=query-scheduler"
             - "-config.expand-env=true"
             - "-config.file=/etc/mimir/mimir.yaml"
+            - "-server.grpc.keepalive.max-connection-age=2562047h" # 100000 days, effectively infinity
+            - "-server.grpc.keepalive.max-connection-age-grace=2562047h" # 100000 days, effectively infinity
           volumeMounts:
             - name: runtime-config
               mountPath: /var/enterprise-metrics

diff --git a/...ise-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml b/...ise-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml
@@ -45,6 +45,8 @@ spec:
             - "-target=query-scheduler"
             - "-config.expand-env=true"
             - "-config.file=/etc/mimir/mimir.yaml"
+            - "-server.grpc.keepalive.max-connection-age=2562047h" # 100000 days, effectively infinity
+            - "-server.grpc.keepalive.max-connection-age-grace=2562047h" # 100000 days, effectively infinity
           volumeMounts:
             - name: runtime-config
               mountPath: /var/mimir

diff --git a/...oss-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml b/...oss-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml
@@ -46,6 +46,8 @@ spec:
             - "-target=query-scheduler"
             - "-config.expand-env=true"
             - "-config.file=/etc/mimir/mimir.yaml"
+            - "-server.grpc.keepalive.max-connection-age=2562047h" # 100000 days, effectively infinity
+            - "-server.grpc.keepalive.max-connection-age-grace=2562047h" # 100000 days, effectively infinity
           volumeMounts:
             - name: runtime-config
               mountPath: /var/mimir