Cortex kv panels: narrow to one instance

Distributor uses multiple kv stores - for global limits and ha-tracker, as well as reading from the ingester ring - so we need to narrow the panel to just the one it says it is showing. For consistency, do the same on the ingester panel, although currently ingesters only have one kv store. Note that the renamed recording rule will mean that dashboards show no data for latency prior to the change.
grafana · Aug 17, 2021 · c09fed3 · c09fed3
1 parent 2250a8f
commit c09fed3
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 5 deletions.
diff --git a/cortex-mixin/dashboards/writes.libsonnet b/cortex-mixin/dashboards/writes.libsonnet
@@ -104,11 +104,12 @@ local utils = import 'mixin-utils/utils.libsonnet';
       $.row('Key-value store for high-availability (HA) deduplication')
       .addPanel(
         $.panel('Requests / sec') +
-        $.qpsPanel('cortex_kv_request_duration_seconds_count{%s}' % $.jobMatcher($._config.job_names.distributor))
+        $.qpsPanel('cortex_kv_request_duration_seconds_count{%s,kv_name="distributor-hatracker"}' % $.jobMatcher($._config.job_names.distributor))
       )
       .addPanel(
         $.panel('Latency') +
-        utils.latencyRecordingRulePanel('cortex_kv_request_duration_seconds', $.jobSelector($._config.job_names.distributor))
+        utils.latencyRecordingRulePanel('cortex_kv_request_duration_seconds', $.jobSelector($._config.job_names.distributor) + [utils.selector.eq('kv_name', 'distributor-hatracker')])
+      )
       )
     )
     .addRow(
@@ -133,11 +134,12 @@ local utils = import 'mixin-utils/utils.libsonnet';
       $.row('Key-value store for the ingesters ring')
       .addPanel(
         $.panel('Requests / sec') +
-        $.qpsPanel('cortex_kv_request_duration_seconds_count{%s}' % $.jobMatcher($._config.job_names.ingester))
+        $.qpsPanel('cortex_kv_request_duration_seconds_count{%s,kv_name="ingester-lifecycler"}' % $.jobMatcher($._config.job_names.ingester))
       )
       .addPanel(
         $.panel('Latency') +
-        utils.latencyRecordingRulePanel('cortex_kv_request_duration_seconds', $.jobSelector($._config.job_names.ingester))
+        utils.latencyRecordingRulePanel('cortex_kv_request_duration_seconds', $.jobSelector($._config.job_names.ingester)+ [utils.selector.eq('kv_name', 'ingester-lifecycler')])
+      )
       )
     )
     .addRowIf(

diff --git a/cortex-mixin/recording_rules.libsonnet b/cortex-mixin/recording_rules.libsonnet
@@ -42,7 +42,7 @@ local utils = import 'mixin-utils/utils.libsonnet';
           utils.histogramRules('cortex_chunk_store_chunks_per_query', ['cluster', 'job']) +
           utils.histogramRules('cortex_database_request_duration_seconds', ['cluster', 'job', 'method']) +
           utils.histogramRules('cortex_gcs_request_duration_seconds', ['cluster', 'job', 'operation']) +
-          utils.histogramRules('cortex_kv_request_duration_seconds', ['cluster', 'job']),
+          utils.histogramRules('cortex_kv_request_duration_seconds', ['cluster', 'job', 'kv_name']),
       },
       {
         name: 'cortex_queries',