From e6c8f1812d839bd8e8fd707f69f35a6c4152597c Mon Sep 17 00:00:00 2001 From: Edward Welch Date: Tue, 6 Aug 2019 17:27:26 -0400 Subject: [PATCH 1/2] sum the read dashboard by route and make sure we are only showing query and label routes (not tail) --- production/loki-mixin/dashboards.libsonnet | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/production/loki-mixin/dashboards.libsonnet b/production/loki-mixin/dashboards.libsonnet index 8e00d55856853..4ee3e61850690 100644 --- a/production/loki-mixin/dashboards.libsonnet +++ b/production/loki-mixin/dashboards.libsonnet @@ -49,33 +49,33 @@ local utils = import "mixin-utils/utils.libsonnet"; g.row('Frontend (cortex_gw)') .addPanel( g.panel('QPS') + - g.qpsPanel('loki_request_duration_seconds_count{cluster="$cluster", job="$namespace/cortex-gw", route="api_prom_query"}') + g.qpsPanel('loki_request_duration_seconds_count{cluster="$cluster", job="$namespace/cortex-gw", route=~"api_prom_query|api_prom_label|api_prom_label_name_values"}') ) .addPanel( g.panel('Latency') + - utils.latencyRecordingRulePanel('loki_request_duration_seconds', [utils.selector.eq('job', '$namespace/cortex-gw'), utils.selector.eq('route', 'api_prom_query')], extra_selectors=[utils.selector.eq('cluster', '$cluster')]) + utils.latencyRecordingRulePanel('loki_request_duration_seconds', [utils.selector.eq('job', '$namespace/cortex-gw'), utils.selector.re('route', 'api_prom_query|api_prom_labels|api_prom_label_name_values')], extra_selectors=[utils.selector.eq('cluster', '$cluster')], sum_by=['route']) ) ) .addRow( g.row('Querier') .addPanel( g.panel('QPS') + - g.qpsPanel('loki_request_duration_seconds_count{cluster="$cluster", job="$namespace/querier"}') + g.qpsPanel('loki_request_duration_seconds_count{cluster="$cluster", job="$namespace/querier", route=~"api_prom_query|api_prom_label|api_prom_label_name_values"}') ) .addPanel( g.panel('Latency') + - utils.latencyRecordingRulePanel('loki_request_duration_seconds', [utils.selector.eq('job', '$namespace/querier')], extra_selectors=[utils.selector.eq('cluster', '$cluster')]) + utils.latencyRecordingRulePanel('loki_request_duration_seconds', [utils.selector.eq('job', '$namespace/querier'), utils.selector.re('route', 'api_prom_query|api_prom_labels|api_prom_label_name_values')], extra_selectors=[utils.selector.eq('cluster', '$cluster')], sum_by=['route']) ) ) .addRow( g.row('Ingester') .addPanel( g.panel('QPS') + - g.qpsPanel('loki_request_duration_seconds_count{cluster="$cluster", job="$namespace/ingester",route!~"/logproto.Pusher/Push|metrics|ready|traces"}') + g.qpsPanel('loki_request_duration_seconds_count{cluster="$cluster", job="$namespace/ingester",route=~"/logproto.Querier/Query|/logproto.Querier/Label"}') ) .addPanel( g.panel('Latency') + - utils.latencyRecordingRulePanel('loki_request_duration_seconds', [utils.selector.eq('job', '$namespace/ingester'), utils.selector.nre('route', '/logproto.Pusher/Push|metrics|ready')], extra_selectors=[utils.selector.eq('cluster', '$cluster')]) + utils.latencyRecordingRulePanel('loki_request_duration_seconds', [utils.selector.eq('job', '$namespace/ingester'), utils.selector.re('route', '/logproto.Querier/Query|/logproto.Querier/Label')], extra_selectors=[utils.selector.eq('cluster', '$cluster')], sum_by=['route']) ) ), From b50bddafeae4f613cbbd0f67729f4b48e13954d7 Mon Sep 17 00:00:00 2001 From: Edward Welch Date: Wed, 7 Aug 2019 13:59:59 -0400 Subject: [PATCH 2/2] Removing the loki-frontend dashboard as it only has one panel which is duplicated in the loki-reads panel, we can bring it back when add more to the loki frontend. --- production/loki-mixin/dashboards.libsonnet | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/production/loki-mixin/dashboards.libsonnet b/production/loki-mixin/dashboards.libsonnet index 4ee3e61850690..dce926923763f 100644 --- a/production/loki-mixin/dashboards.libsonnet +++ b/production/loki-mixin/dashboards.libsonnet @@ -131,22 +131,6 @@ local utils = import "mixin-utils/utils.libsonnet"; ), ), - 'loki-frontend.json': - g.dashboard('Loki / Frontend') - .addTemplate('cluster', 'kube_pod_container_info{image=~".*loki.*"}', 'cluster') - .addTemplate('namespace', 'kube_pod_container_info{image=~".*loki.*"}', 'namespace') - .addRow( - g.row('loki Reqs (cortex_gw)') - .addPanel( - g.panel('QPS') + - g.qpsPanel('loki_request_duration_seconds_count{cluster="$cluster", job="$namespace/cortex-gw"}') - ) - .addPanel( - g.panel('Latency') + - utils.latencyRecordingRulePanel('loki_request_duration_seconds', [utils.selector.eq('job', '$namespace/cortex-gw')], extra_selectors=[utils.selector.eq('cluster', '$cluster')]) - ) - ), - 'promtail.json': g.dashboard('Loki / Promtail') .addTemplate('cluster', 'kube_pod_container_info{image=~".*promtail.*"}', 'cluster')