From 16b28700ab07071d4383399e816c371b7d2a5671 Mon Sep 17 00:00:00 2001 From: Robert Klotzner Date: Tue, 28 Sep 2021 15:51:20 +0200 Subject: [PATCH 1/2] Silence alert about large-statement-fetcher That task is not continuous and behaving exactly as desired. --- .maintain/monitoring/alerting-rules/alerting-rules.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.maintain/monitoring/alerting-rules/alerting-rules.yaml b/.maintain/monitoring/alerting-rules/alerting-rules.yaml index 7a69cba66c3f3..1e455657f9416 100644 --- a/.maintain/monitoring/alerting-rules/alerting-rules.yaml +++ b/.maintain/monitoring/alerting-rules/alerting-rules.yaml @@ -134,7 +134,7 @@ groups: ############################################################################## - alert: ContinuousTaskEnded - expr: '(polkadot_tasks_spawned_total{task_name != "basic-authorship-proposer", task_name != "substrate-rpc-subscription"} == 1) + expr: '(polkadot_tasks_spawned_total{task_name != "basic-authorship-proposer", task_name != "substrate-rpc-subscription", task_name != "large-statement-fetcher"} == 1) - on(instance, task_name) group_left() (polkadot_tasks_ended_total == 1)' for: 5m labels: From 6dd830b1aab84a5e24d5f817002da6e3fd49d7fd Mon Sep 17 00:00:00 2001 From: Robert Klotzner Date: Thu, 30 Sep 2021 14:57:42 +0200 Subject: [PATCH 2/2] Get rid of rule completely - Pierre thinks it does not add any value - Basti rightly complains that Polkadot specific exceptions make little sense in the Substrate repo. --- .../monitoring/alerting-rules/alerting-rules.yaml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.maintain/monitoring/alerting-rules/alerting-rules.yaml b/.maintain/monitoring/alerting-rules/alerting-rules.yaml index 1e455657f9416..2711610024330 100644 --- a/.maintain/monitoring/alerting-rules/alerting-rules.yaml +++ b/.maintain/monitoring/alerting-rules/alerting-rules.yaml @@ -133,16 +133,6 @@ groups: # Others ############################################################################## - - alert: ContinuousTaskEnded - expr: '(polkadot_tasks_spawned_total{task_name != "basic-authorship-proposer", task_name != "substrate-rpc-subscription", task_name != "large-statement-fetcher"} == 1) - - on(instance, task_name) group_left() (polkadot_tasks_ended_total == 1)' - for: 5m - labels: - severity: warning - annotations: - message: 'Continuous task {{ $labels.task_name }} on node - {{ $labels.instance }} ended unexpectedly.' - - alert: AuthorityDiscoveryDiscoveryFailureHigh expr: 'polkadot_authority_discovery_handle_value_found_event_failure / ignoring(name)