Add row about Distributor's metric forwarding feature to "Mimir / Wri…

…tes" dashboard (#3182) * add row about distributor forwarding to mimir/writes dashboard Signed-off-by: Mauro Stettler <mauro.stettler@gmail.com> * changelog Signed-off-by: Mauro Stettler <mauro.stettler@gmail.com> * compile dashboards with forwarding disabled by default Signed-off-by: Mauro Stettler <mauro.stettler@gmail.com> * fix errors rate and latency Signed-off-by: Mauro Stettler <mauro.stettler@gmail.com> * move queries back into dashboard Signed-off-by: Mauro Stettler <mauro.stettler@gmail.com> * update changelog entry Signed-off-by: Mauro Stettler <mauro.stettler@gmail.com> * fix query to show only successes Signed-off-by: Mauro Stettler <mauro.stettler@gmail.com> * add legend to p99 latency panel Signed-off-by: Mauro Stettler <mauro.stettler@gmail.com> * jsonnetfmt 🤬 Signed-off-by: Mauro Stettler <mauro.stettler@gmail.com> * jsonnetfmt 🤯 Signed-off-by: Mauro Stettler <mauro.stettler@gmail.com> * simplify query Signed-off-by: Mauro Stettler <mauro.stettler@gmail.com> * preserve error status code Signed-off-by: Mauro Stettler <mauro.stettler@gmail.com> * use named placeholders Signed-off-by: Mauro Stettler <mauro.stettler@gmail.com> * use queryPanel and make it look like qpsPanel Signed-off-by: Mauro Stettler <mauro.stettler@gmail.com> Signed-off-by: Mauro Stettler <mauro.stettler@gmail.com>
grafana · Oct 12, 2022 · d857bf9 · d857bf9
1 parent d213161
commit d857bf9
Show file tree

Hide file tree

Showing 3 changed files with 72 additions and 0 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -12,6 +12,7 @@
 ### Mixin
 
 * [ENHANCEMENT] Alerts: Add MimirRingMembersMismatch firing when a component does not have the expected number of running jobs. #2404
+* [ENHANCEMENT] Dashboards: Add optional row about the Distributor's metric forwarding feature to the `Mimir / Writes` dashboard. #3182
 * [BUGFIX] Dashboards: Fix legend showing `persistentvolumeclaim` when using `deployment_type=baremetal` for `Disk space utilization` panels. #3173
 
 ### Jsonnet

diff --git a/operations/mimir-mixin/config.libsonnet b/operations/mimir-mixin/config.libsonnet
@@ -69,6 +69,9 @@
     // Whether grafana cloud alertmanager instance-mapper is enabled
     alertmanager_im_enabled: false,
 
+    // Whether the Distributor's forwarding feature is enabled.
+    forwarding_enabled: false,
+
     // The label used to differentiate between different application instances (i.e. 'pod' in a kubernetes install).
     per_instance_label: 'pod',
 

diff --git a/operations/mimir-mixin/dashboards/writes.libsonnet b/operations/mimir-mixin/dashboards/writes.libsonnet
@@ -137,6 +137,74 @@ local filename = 'mimir-writes.json';
         )
       )
     )
+    .addRowIf(
+      $._config.forwarding_enabled,
+      $.row('Distributor Forwarding')
+      .addPanel(
+        $.panel('Requests / sec') +
+        $.queryPanel(
+          [
+            |||
+              sum(
+                rate(
+                  cortex_distributor_forward_requests_total{%(distributorMatcher)s}[$__rate_interval]
+                )
+              )
+              -
+              sum(
+                rate(
+                  cortex_distributor_forward_errors_total{%(distributorMatcher)s}[$__rate_interval]
+                )
+              )
+            ||| % {
+              distributorMatcher: $.jobMatcher($._config.job_names.distributor),
+            },
+            |||
+              label_replace(
+                sum by (status_code) (
+                  rate(
+                    cortex_distributor_forward_errors_total{%(distributorMatcher)s}[$__rate_interval]
+                  )
+                ),
+                "status_code",
+                "error",
+                "status_code",
+                "failed"
+              )
+            ||| % {
+              distributorMatcher: $.jobMatcher($._config.job_names.distributor),
+            },
+          ], [
+            'success',
+            '{{ status_code }}',
+          ],
+        ) + $.stack + {
+          aliasColors: {
+            '1xx': '#EAB839',
+            '2xx': '#7EB26D',
+            '3xx': '#6ED0E0',
+            '4xx': '#EF843C',
+            '5xx': '#E24D42',
+            success: '#7EB26D',
+            'error': '#E24D42',
+          },
+        },
+      )
+      .addPanel(
+        $.panel('Latency') +
+        $.latencyPanel('cortex_distributor_forward_requests_latency_seconds', '{%s}' % $.jobMatcher($._config.job_names.distributor))
+      )
+      .addPanel(
+        $.timeseriesPanel('Per %s p99 latency' % $._config.per_instance_label) +
+        $.hiddenLegendQueryPanel(
+          'histogram_quantile(0.99, sum by(le, %s) (rate(cortex_distributor_forward_requests_latency_seconds_bucket{%s}[$__rate_interval])))' % [
+            $._config.per_instance_label,
+            $.jobMatcher($._config.job_names.distributor),
+          ],
+          '{{ %s }}' % $._config.per_instance_label
+        )
+      )
+    )
     .addRow(
       $.row('Ingester')
       .addPanel(