Trigger node high load warnings sooner

gitpod-io · Jan 21, 2022 · 71f5431 · 71f5431
1 parent c62ec66
commit 71f5431
Showing 1 changed file with 3 additions and 3 deletions.
diff --git a/operations/observability/mixins/workspace/rules/components/nodes/alerts.libsonnet b/operations/observability/mixins/workspace/rules/components/nodes/alerts.libsonnet
@@ -14,13 +14,13 @@
             labels: {
               severity: 'warning',
             },
-            'for': '5m',
+            'for': '2m',
             annotations: {
               runbook_url: 'https://github.com/gitpod-io/observability/blob/main/runbooks/GitpodWorkspaceNodeHighNormalizedLoadAverage.md',
-              summary: "Workspace node's normalized load average is higher than 5 for more than 5 minutes.",
+              summary: "Workspace node's normalized load average is higher than 3 for more than 2 minutes. Check for abuse.",
               description: 'Node {{ $labels.node }} is reporting {{ printf "%.2f" $value }}% normalized load average. Normalized load average is current load average divided by number of CPU cores of the node.',
             },
-            expr: 'nodepool:node_load1:normalized{nodepool=~".*workspace.*"} > 5',
+            expr: 'nodepool:node_load1:normalized{nodepool=~".*workspace.*"} > 3',
           },
         ],
       },