From 8dd93af0f0d5080c7fab30db8365af9907b759a0 Mon Sep 17 00:00:00 2001 From: Dimitar Dimitrov Date: Thu, 20 Oct 2022 12:26:51 +0200 Subject: [PATCH 1/2] Helm: prevent query-scheduler from terminating connections Signed-off-by: Dimitar Dimitrov --- operations/helm/charts/mimir-distributed/CHANGELOG.md | 1 + .../templates/query-scheduler/query-scheduler-dep.yaml | 2 ++ .../templates/query-scheduler/query-scheduler-dep.yaml | 2 ++ .../templates/query-scheduler/query-scheduler-dep.yaml | 2 ++ .../templates/query-scheduler/query-scheduler-dep.yaml | 2 ++ .../templates/query-scheduler/query-scheduler-dep.yaml | 2 ++ .../templates/query-scheduler/query-scheduler-dep.yaml | 2 ++ .../templates/query-scheduler/query-scheduler-dep.yaml | 2 ++ .../templates/query-scheduler/query-scheduler-dep.yaml | 2 ++ .../templates/query-scheduler/query-scheduler-dep.yaml | 2 ++ 10 files changed, 19 insertions(+) diff --git a/operations/helm/charts/mimir-distributed/CHANGELOG.md b/operations/helm/charts/mimir-distributed/CHANGELOG.md index b4d45c3be14..ac0aa580310 100644 --- a/operations/helm/charts/mimir-distributed/CHANGELOG.md +++ b/operations/helm/charts/mimir-distributed/CHANGELOG.md @@ -104,6 +104,7 @@ Entries should include a reference to the Pull Request that introduced the chang * [ENHANCEMENT] Add podAntiAffinity to sizing plans (small.yaml, large.yaml, capped-small.yaml, capped-large.yaml). #2906 * [ENHANCEMENT] Add ability to configure and run mimir-continuous-test. #3117 * [BUGFIX] Fix wrong label selector in ingester anti affinity rules in the sizing plans. #2906 +* [BUGFIX] Query-scheduler no longer periodically terminates connections from query-frontends and queriers. This caused some queries to time out and EOF errors in the logs. ## 3.1.0 diff --git a/operations/helm/charts/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml b/operations/helm/charts/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml index 53ebc92d816..ba42b2036a5 100644 --- a/operations/helm/charts/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml +++ b/operations/helm/charts/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml @@ -44,6 +44,8 @@ spec: - "-target=query-scheduler" - "-config.expand-env=true" - "-config.file=/etc/mimir/mimir.yaml" + - "-server.grpc.keepalive.max-connection-age=2562047h" # 100000 days, effectively infinity + - "-server.grpc.keepalive.max-connection-age-grace=2562047h" # 100000 days, effectively infinity {{- range $key, $value := .Values.query_scheduler.extraArgs }} - "-{{ $key }}={{ $value }}" {{- end }} diff --git a/operations/helm/tests/graphite-enabled-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml b/operations/helm/tests/graphite-enabled-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml index 01fba723e1b..4d3079c1933 100644 --- a/operations/helm/tests/graphite-enabled-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml +++ b/operations/helm/tests/graphite-enabled-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml @@ -45,6 +45,8 @@ spec: - "-target=query-scheduler" - "-config.expand-env=true" - "-config.file=/etc/mimir/mimir.yaml" + - "-server.grpc.keepalive.max-connection-age=2562047h" # 100000 days, effectively infinity + - "-server.grpc.keepalive.max-connection-age-grace=2562047h" # 100000 days, effectively infinity volumeMounts: - name: runtime-config mountPath: /var/mimir diff --git a/operations/helm/tests/metamonitoring-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml b/operations/helm/tests/metamonitoring-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml index ee8754c831e..af006eaff8c 100644 --- a/operations/helm/tests/metamonitoring-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml +++ b/operations/helm/tests/metamonitoring-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml @@ -45,6 +45,8 @@ spec: - "-target=query-scheduler" - "-config.expand-env=true" - "-config.file=/etc/mimir/mimir.yaml" + - "-server.grpc.keepalive.max-connection-age=2562047h" # 100000 days, effectively infinity + - "-server.grpc.keepalive.max-connection-age-grace=2562047h" # 100000 days, effectively infinity volumeMounts: - name: runtime-config mountPath: /var/mimir diff --git a/operations/helm/tests/test-enterprise-configmap-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml b/operations/helm/tests/test-enterprise-configmap-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml index da9a19206f6..c0882691410 100644 --- a/operations/helm/tests/test-enterprise-configmap-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml +++ b/operations/helm/tests/test-enterprise-configmap-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml @@ -46,6 +46,8 @@ spec: - "-target=query-scheduler" - "-config.expand-env=true" - "-config.file=/etc/mimir/mimir.yaml" + - "-server.grpc.keepalive.max-connection-age=2562047h" # 100000 days, effectively infinity + - "-server.grpc.keepalive.max-connection-age-grace=2562047h" # 100000 days, effectively infinity volumeMounts: - name: runtime-config mountPath: /var/mimir diff --git a/operations/helm/tests/test-enterprise-legacy-label-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml b/operations/helm/tests/test-enterprise-legacy-label-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml index 6e963943648..8f3aa2dc3b4 100644 --- a/operations/helm/tests/test-enterprise-legacy-label-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml +++ b/operations/helm/tests/test-enterprise-legacy-label-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml @@ -43,6 +43,8 @@ spec: - "-target=query-scheduler" - "-config.expand-env=true" - "-config.file=/etc/mimir/mimir.yaml" + - "-server.grpc.keepalive.max-connection-age=2562047h" # 100000 days, effectively infinity + - "-server.grpc.keepalive.max-connection-age-grace=2562047h" # 100000 days, effectively infinity volumeMounts: - name: runtime-config mountPath: /var/enterprise-metrics diff --git a/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml index 237bd55383c..ebee4c4fccb 100644 --- a/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml +++ b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml @@ -45,6 +45,8 @@ spec: - "-target=query-scheduler" - "-config.expand-env=true" - "-config.file=/etc/mimir/mimir.yaml" + - "-server.grpc.keepalive.max-connection-age=2562047h" # 100000 days, effectively infinity + - "-server.grpc.keepalive.max-connection-age-grace=2562047h" # 100000 days, effectively infinity volumeMounts: - name: runtime-config mountPath: /var/mimir diff --git a/operations/helm/tests/test-oss-logical-multizone-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml b/operations/helm/tests/test-oss-logical-multizone-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml index 12f6921b22f..1ef1a7a7752 100644 --- a/operations/helm/tests/test-oss-logical-multizone-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml +++ b/operations/helm/tests/test-oss-logical-multizone-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml @@ -45,6 +45,8 @@ spec: - "-target=query-scheduler" - "-config.expand-env=true" - "-config.file=/etc/mimir/mimir.yaml" + - "-server.grpc.keepalive.max-connection-age=2562047h" # 100000 days, effectively infinity + - "-server.grpc.keepalive.max-connection-age-grace=2562047h" # 100000 days, effectively infinity volumeMounts: - name: runtime-config mountPath: /var/mimir diff --git a/operations/helm/tests/test-oss-multizone-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml b/operations/helm/tests/test-oss-multizone-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml index abcfe8864d4..f8ba2de9ab6 100644 --- a/operations/helm/tests/test-oss-multizone-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml +++ b/operations/helm/tests/test-oss-multizone-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml @@ -45,6 +45,8 @@ spec: - "-target=query-scheduler" - "-config.expand-env=true" - "-config.file=/etc/mimir/mimir.yaml" + - "-server.grpc.keepalive.max-connection-age=2562047h" # 100000 days, effectively infinity + - "-server.grpc.keepalive.max-connection-age-grace=2562047h" # 100000 days, effectively infinity volumeMounts: - name: runtime-config mountPath: /var/mimir diff --git a/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml index 5e0be404560..793144ca6aa 100644 --- a/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml +++ b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/query-scheduler/query-scheduler-dep.yaml @@ -46,6 +46,8 @@ spec: - "-target=query-scheduler" - "-config.expand-env=true" - "-config.file=/etc/mimir/mimir.yaml" + - "-server.grpc.keepalive.max-connection-age=2562047h" # 100000 days, effectively infinity + - "-server.grpc.keepalive.max-connection-age-grace=2562047h" # 100000 days, effectively infinity volumeMounts: - name: runtime-config mountPath: /var/mimir From 71b5abc0637acb4a25e3c4c01ce5fdd714cc0d63 Mon Sep 17 00:00:00 2001 From: Dimitar Dimitrov Date: Thu, 20 Oct 2022 12:30:50 +0200 Subject: [PATCH 2/2] Update CHANGELOG.md Signed-off-by: Dimitar Dimitrov --- operations/helm/charts/mimir-distributed/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/operations/helm/charts/mimir-distributed/CHANGELOG.md b/operations/helm/charts/mimir-distributed/CHANGELOG.md index ac0aa580310..1188b35aa07 100644 --- a/operations/helm/charts/mimir-distributed/CHANGELOG.md +++ b/operations/helm/charts/mimir-distributed/CHANGELOG.md @@ -104,7 +104,7 @@ Entries should include a reference to the Pull Request that introduced the chang * [ENHANCEMENT] Add podAntiAffinity to sizing plans (small.yaml, large.yaml, capped-small.yaml, capped-large.yaml). #2906 * [ENHANCEMENT] Add ability to configure and run mimir-continuous-test. #3117 * [BUGFIX] Fix wrong label selector in ingester anti affinity rules in the sizing plans. #2906 -* [BUGFIX] Query-scheduler no longer periodically terminates connections from query-frontends and queriers. This caused some queries to time out and EOF errors in the logs. +* [BUGFIX] Query-scheduler no longer periodically terminates connections from query-frontends and queriers. This caused some queries to time out and EOF errors in the logs. #3262 ## 3.1.0