Cortex writes dashboard: add panels for KV activity

For HA-tracker, show which tenants are changing election. For ingester, show how many are active, leaving, etc.
grafana · Aug 18, 2021 · 193c647 · 193c647
1 parent c09fed3
commit 193c647
Showing 1 changed file with 18 additions and 1 deletion.
diff --git a/cortex-mixin/dashboards/writes.libsonnet b/cortex-mixin/dashboards/writes.libsonnet
@@ -110,6 +110,16 @@ local utils = import 'mixin-utils/utils.libsonnet';
         $.panel('Latency') +
         utils.latencyRecordingRulePanel('cortex_kv_request_duration_seconds', $.jobSelector($._config.job_names.distributor) + [utils.selector.eq('kv_name', 'distributor-hatracker')])
       )
+      .addPanel(
+        $.panel('Elected replica changes / min') +
+        $.queryPanel([
+          'max by(exported_cluster, user)(increase(cortex_ha_tracker_elected_replica_changes_total{%s}[1m])) >0' % $.jobMatcher($._config.job_names.distributor),
+        ], [
+          '{{user}}/{{exported_cluster}}',
+        ]) +
+        $.stack + {
+          yaxes: $.yaxes('cpm'),
+        },
       )
     )
     .addRow(
@@ -138,8 +148,15 @@ local utils = import 'mixin-utils/utils.libsonnet';
       )
       .addPanel(
         $.panel('Latency') +
-        utils.latencyRecordingRulePanel('cortex_kv_request_duration_seconds', $.jobSelector($._config.job_names.ingester)+ [utils.selector.eq('kv_name', 'ingester-lifecycler')])
+        utils.latencyRecordingRulePanel('cortex_kv_request_duration_seconds', $.jobSelector($._config.job_names.ingester) + [utils.selector.eq('kv_name', 'ingester-lifecycler')])
       )
+      .addPanel(
+        $.panel('Ingester status') +
+        $.queryPanel([
+          'max by (state)(cortex_ring_members{%s}) >0' % $.jobMatcher($._config.job_names.distributor),
+        ], [
+          '{{state}}',
+        ])
       )
     )
     .addRowIf(