From 864ad033e11ccd254c31f2b79da018fcf11b120b Mon Sep 17 00:00:00 2001 From: Matej Gera Date: Wed, 26 Oct 2022 10:55:10 +0200 Subject: [PATCH] Remove low ingestion rate warning Signed-off-by: Matej Gera --- examples/alerts/alerts.md | 14 -------------- examples/alerts/alerts.yaml | 14 -------------- mixin/alerts/receive.libsonnet | 18 ------------------ mixin/runbook.md | 1 - 4 files changed, 47 deletions(-) diff --git a/examples/alerts/alerts.md b/examples/alerts/alerts.md index 44c4a0529a..474b55b700 100644 --- a/examples/alerts/alerts.md +++ b/examples/alerts/alerts.md @@ -530,20 +530,6 @@ rules: for: 3h labels: severity: critical -- alert: ThanosReceiveTrafficBelowThreshold - annotations: - description: At Thanos Receive {{$labels.job}} in {{$labels.namespace}} , the average 1-hr avg. metrics ingestion rate is {{$value | humanize}}% of 12-hr avg. ingestion rate. - runbook_url: https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanosreceivetrafficbelowthreshold - summary: Thanos Receive is experiencing low avg. 1-hr ingestion rate relative to avg. 12-hr ingestion rate. - expr: | - ( - avg_over_time(rate(http_requests_total{job=~".*thanos-receive.*", code=~"2..", handler="receive"}[5m])[1h:5m]) - / - avg_over_time(rate(http_requests_total{job=~".*thanos-receive.*", code=~"2..", handler="receive"}[5m])[12h:5m]) - ) * 100 < 50 - for: 1h - labels: - severity: warning ``` ## Replicate diff --git a/examples/alerts/alerts.yaml b/examples/alerts/alerts.yaml index ed3a452123..b8ab03b65f 100644 --- a/examples/alerts/alerts.yaml +++ b/examples/alerts/alerts.yaml @@ -274,20 +274,6 @@ groups: for: 3h labels: severity: critical - - alert: ThanosReceiveTrafficBelowThreshold - annotations: - description: At Thanos Receive {{$labels.job}} in {{$labels.namespace}} , the average 1-hr avg. metrics ingestion rate is {{$value | humanize}}% of 12-hr avg. ingestion rate. - runbook_url: https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanosreceivetrafficbelowthreshold - summary: Thanos Receive is experiencing low avg. 1-hr ingestion rate relative to avg. 12-hr ingestion rate. - expr: | - ( - avg_over_time(rate(http_requests_total{job=~".*thanos-receive.*", code=~"2..", handler="receive"}[5m])[1h:5m]) - / - avg_over_time(rate(http_requests_total{job=~".*thanos-receive.*", code=~"2..", handler="receive"}[5m])[12h:5m]) - ) * 100 < 50 - for: 1h - labels: - severity: warning - name: thanos-sidecar rules: - alert: ThanosSidecarBucketOperationsFailed diff --git a/mixin/alerts/receive.libsonnet b/mixin/alerts/receive.libsonnet index d313b717cb..00da01211c 100644 --- a/mixin/alerts/receive.libsonnet +++ b/mixin/alerts/receive.libsonnet @@ -144,24 +144,6 @@ severity: 'critical', }, }, - { - alert: 'ThanosReceiveTrafficBelowThreshold', - annotations: { - description: 'At Thanos Receive {{$labels.job}} in {{$labels.namespace}} , the average 1-hr avg. metrics ingestion rate is {{$value | humanize}}% of 12-hr avg. ingestion rate.', - summary: 'Thanos Receive is experiencing low avg. 1-hr ingestion rate relative to avg. 12-hr ingestion rate.', - }, - expr: ||| - ( - avg_over_time(rate(http_requests_total{%(selector)s, code=~"2..", handler="receive"}[5m])[1h:5m]) - / - avg_over_time(rate(http_requests_total{%(selector)s, code=~"2..", handler="receive"}[5m])[12h:5m]) - ) * 100 < %(ingestionThreshold)s - ||| % thanos.receive, - 'for': '1h', - labels: { - severity: 'warning', - }, - }, ], }, ], diff --git a/mixin/runbook.md b/mixin/runbook.md index 6f251c1be6..e4b79fc3f0 100755 --- a/mixin/runbook.md +++ b/mixin/runbook.md @@ -63,7 +63,6 @@ |ThanosReceiveHighHashringFileRefreshFailures|Thanos Receive is failing to refresh hasring file.|Thanos Receive {{$labels.job}} is failing to refresh hashring file, {{$value humanize}} of attempts failed.|warning|[https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanosreceivehighhashringfilerefreshfailures](https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanosreceivehighhashringfilerefreshfailures)| |ThanosReceiveConfigReloadFailure|Thanos Receive has not been able to reload configuration.|Thanos Receive {{$labels.job}} has not been able to reload hashring configurations.|warning|[https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanosreceiveconfigreloadfailure](https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanosreceiveconfigreloadfailure)| |ThanosReceiveNoUpload|Thanos Receive has not uploaded latest data to object storage.|Thanos Receive {{$labels.instance}} has not uploaded latest data to object storage.|critical|[https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanosreceivenoupload](https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanosreceivenoupload)| -|ThanosReceiveTrafficBelowThreshold|Thanos Receive is experiencing low avg. 1-hr ingestion rate relative to avg. 12-hr ingestion rate.|At Thanos Receive {{$labels.job}} in {{$labels.namespace}} , the average 1-hr avg. metrics ingestion rate is {{$value humanize}}% of 12-hr avg. ingestion rate.|warning|[https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanosreceivetrafficbelowthreshold](https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanosreceivetrafficbelowthreshold)| ## thanos-rule