From dcce483d2fda08bc74e96e0b0dc8dd6506c4d78f Mon Sep 17 00:00:00 2001 From: Dimitar Dimitrov Date: Thu, 20 Oct 2022 13:02:50 +0200 Subject: [PATCH] Helm: prevent query-scheduler from terminating connections (#3262) Helm: prevent query-scheduler from terminating connections Signed-off-by: Dimitar Dimitrov (cherry picked from commit 0107f45e0a370ef8ecc1828fd00b52344df12bb5) --- operations/helm/charts/mimir-distributed/CHANGELOG.md | 1 + .../templates/query-scheduler/query-scheduler-dep.yaml | 2 ++ .../templates/query-scheduler/query-scheduler-dep.yaml | 2 ++ .../templates/query-scheduler/query-scheduler-dep.yaml | 2 ++ .../templates/query-scheduler/query-scheduler-dep.yaml | 2 ++ .../templates/query-scheduler/query-scheduler-dep.yaml | 2 ++ .../templates/query-scheduler/query-scheduler-dep.yaml | 2 ++ .../templates/query-scheduler/query-scheduler-dep.yaml | 2 ++ 8 files changed, 15 insertions(+) diff --git a/operations/helm/charts/mimir-distributed/CHANGELOG.md b/operations/helm/charts/mimir-distributed/CHANGELOG.md index a7bba07da30..3b299a56bcd 100644 --- a/operations/helm/charts/mimir-distributed/CHANGELOG.md +++ b/operations/helm/charts/mimir-distributed/CHANGELOG.md @@ -78,6 +78,7 @@ Entries should include a reference to the Pull Request that introduced the chang * [ENHANCEMENT] Add podAntiAffinity to sizing plans (small.yaml, large.yaml, capped-small.yaml, capped-large.yaml). #2906 * [ENHANCEMENT] Add ability to configure and run mimir-continuous-test. #3117 * [BUGFIX] Fix wrong label selector in ingester anti affinity rules in the sizing plans. #2906 +* [BUGFIX] Query-scheduler no longer periodically terminates connections from query-frontends and queriers. This caused some queries to time out and EOF errors in the logs. #3262 ## 3.1.0 diff --git a/operations/helm/charts/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml b/operations/helm/charts/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml index 548bcb297ba..14187e39ac6 100644 --- a/operations/helm/charts/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml +++ b/operations/helm/charts/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml @@ -44,6 +44,8 @@ spec: - "-target=query-scheduler" - "-config.expand-env=true" - "-config.file=/etc/mimir/mimir.yaml" + - "-server.grpc.keepalive.max-connection-age=2562047h" # 100000 days, effectively infinity + - "-server.grpc.keepalive.max-connection-age-grace=2562047h" # 100000 days, effectively infinity {{- range $key, $value := .Values.query_scheduler.extraArgs }} - "-{{ $key }}={{ $value }}" {{- end }} diff --git a/operations/helm/tests/graphite-enabled-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml b/operations/helm/tests/graphite-enabled-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml index 96e8ba6307b..b28a3024d4b 100644 --- a/operations/helm/tests/graphite-enabled-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml +++ b/operations/helm/tests/graphite-enabled-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml @@ -45,6 +45,8 @@ spec: - "-target=query-scheduler" - "-config.expand-env=true" - "-config.file=/etc/mimir/mimir.yaml" + - "-server.grpc.keepalive.max-connection-age=2562047h" # 100000 days, effectively infinity + - "-server.grpc.keepalive.max-connection-age-grace=2562047h" # 100000 days, effectively infinity volumeMounts: - name: runtime-config mountPath: /var/mimir diff --git a/operations/helm/tests/metamonitoring-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml b/operations/helm/tests/metamonitoring-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml index bed61675071..410a087259f 100644 --- a/operations/helm/tests/metamonitoring-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml +++ b/operations/helm/tests/metamonitoring-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml @@ -45,6 +45,8 @@ spec: - "-target=query-scheduler" - "-config.expand-env=true" - "-config.file=/etc/mimir/mimir.yaml" + - "-server.grpc.keepalive.max-connection-age=2562047h" # 100000 days, effectively infinity + - "-server.grpc.keepalive.max-connection-age-grace=2562047h" # 100000 days, effectively infinity volumeMounts: - name: runtime-config mountPath: /var/mimir diff --git a/operations/helm/tests/test-enterprise-configmap-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml b/operations/helm/tests/test-enterprise-configmap-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml index 744106b0513..0521e9fd69f 100644 --- a/operations/helm/tests/test-enterprise-configmap-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml +++ b/operations/helm/tests/test-enterprise-configmap-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml @@ -46,6 +46,8 @@ spec: - "-target=query-scheduler" - "-config.expand-env=true" - "-config.file=/etc/mimir/mimir.yaml" + - "-server.grpc.keepalive.max-connection-age=2562047h" # 100000 days, effectively infinity + - "-server.grpc.keepalive.max-connection-age-grace=2562047h" # 100000 days, effectively infinity volumeMounts: - name: runtime-config mountPath: /var/mimir diff --git a/operations/helm/tests/test-enterprise-legacy-label-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml b/operations/helm/tests/test-enterprise-legacy-label-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml index 83d2a5aecbc..dab2fc6d2ae 100644 --- a/operations/helm/tests/test-enterprise-legacy-label-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml +++ b/operations/helm/tests/test-enterprise-legacy-label-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml @@ -43,6 +43,8 @@ spec: - "-target=query-scheduler" - "-config.expand-env=true" - "-config.file=/etc/mimir/mimir.yaml" + - "-server.grpc.keepalive.max-connection-age=2562047h" # 100000 days, effectively infinity + - "-server.grpc.keepalive.max-connection-age-grace=2562047h" # 100000 days, effectively infinity volumeMounts: - name: runtime-config mountPath: /var/enterprise-metrics diff --git a/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml index a5eb357d400..6ca5cf9c2dd 100644 --- a/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml +++ b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml @@ -45,6 +45,8 @@ spec: - "-target=query-scheduler" - "-config.expand-env=true" - "-config.file=/etc/mimir/mimir.yaml" + - "-server.grpc.keepalive.max-connection-age=2562047h" # 100000 days, effectively infinity + - "-server.grpc.keepalive.max-connection-age-grace=2562047h" # 100000 days, effectively infinity volumeMounts: - name: runtime-config mountPath: /var/mimir diff --git a/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml index f031862725b..de5fc092200 100644 --- a/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml +++ b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml @@ -46,6 +46,8 @@ spec: - "-target=query-scheduler" - "-config.expand-env=true" - "-config.file=/etc/mimir/mimir.yaml" + - "-server.grpc.keepalive.max-connection-age=2562047h" # 100000 days, effectively infinity + - "-server.grpc.keepalive.max-connection-age-grace=2562047h" # 100000 days, effectively infinity volumeMounts: - name: runtime-config mountPath: /var/mimir