From 193c647ae4ca6542bca5c16db0d8189171bc565c Mon Sep 17 00:00:00 2001
From: Bryan Boreham <bjboreham@gmail.com>
Date: Tue, 17 Aug 2021 16:59:05 +0000
Subject: [PATCH] Cortex writes dashboard: add panels for KV activity

For HA-tracker, show which tenants are changing election.
For ingester, show how many are active, leaving, etc.
---
 cortex-mixin/dashboards/writes.libsonnet | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/cortex-mixin/dashboards/writes.libsonnet b/cortex-mixin/dashboards/writes.libsonnet
index 999603be..bf383ab2 100644
--- a/cortex-mixin/dashboards/writes.libsonnet
+++ b/cortex-mixin/dashboards/writes.libsonnet
@@ -110,6 +110,16 @@ local utils = import 'mixin-utils/utils.libsonnet';
         $.panel('Latency') +
         utils.latencyRecordingRulePanel('cortex_kv_request_duration_seconds', $.jobSelector($._config.job_names.distributor) + [utils.selector.eq('kv_name', 'distributor-hatracker')])
       )
+      .addPanel(
+        $.panel('Elected replica changes / min') +
+        $.queryPanel([
+          'max by(exported_cluster, user)(increase(cortex_ha_tracker_elected_replica_changes_total{%s}[1m])) >0' % $.jobMatcher($._config.job_names.distributor),
+        ], [
+          '{{user}}/{{exported_cluster}}',
+        ]) +
+        $.stack + {
+          yaxes: $.yaxes('cpm'),
+        },
       )
     )
     .addRow(
@@ -138,8 +148,15 @@ local utils = import 'mixin-utils/utils.libsonnet';
       )
       .addPanel(
         $.panel('Latency') +
-        utils.latencyRecordingRulePanel('cortex_kv_request_duration_seconds', $.jobSelector($._config.job_names.ingester)+ [utils.selector.eq('kv_name', 'ingester-lifecycler')])
+        utils.latencyRecordingRulePanel('cortex_kv_request_duration_seconds', $.jobSelector($._config.job_names.ingester) + [utils.selector.eq('kv_name', 'ingester-lifecycler')])
       )
+      .addPanel(
+        $.panel('Ingester status') +
+        $.queryPanel([
+          'max by (state)(cortex_ring_members{%s}) >0' % $.jobMatcher($._config.job_names.distributor),
+        ], [
+          '{{state}}',
+        ])
       )
     )
     .addRowIf(