Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

WIP: production/promtail-mixin: Make dashboard queries configurable #1978

Closed
wants to merge 1 commit into from
Closed

WIP: production/promtail-mixin: Make dashboard queries configurable #1978

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
83 changes: 73 additions & 10 deletions production/promtail-mixin/dashboards.libsonnet
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,24 +3,54 @@ local utils = import 'mixin-utils/utils.libsonnet';

{
grafanaDashboards+: {
local dashboards = self,

'promtail.json':
{
local cfg = self,

showMultiCluster:: true,
clusterLabel:: 'cluster',
clusterMatchers:: if cfg.showMultiCluster then [utils.selector.eq(cfg.clusterLabel, '$cluster')] else [],

matchers:: [utils.selector.eq('job', '$namespace/$name')],
Copy link
Contributor

@chancez chancez May 21, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What if our job label doesn't take this form? I've been using serviceMonitors with prometheus-operator and our jobLabels are unfortunately just the service name currently. I've been meaning to fix it but haven't quite figured out the best method to get job=$namespace/$name.

Copy link
Contributor

@chancez chancez May 21, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm a jsonnet newbie, but In hindsight, I'm guessing I would just override matchers to do that. I'm kind wondering still if we could avoid using job labels entirely with dashboards since it's a label that seems fairly inconsistent, while pod, container and namespace should be fairly universal (once #2070 is fixed).

selector:: std.join(',', ['%(label)s%(op)s"%(value)s"' % matcher for matcher in (cfg.clusterMatchers + dashboards['promtail.json'].matchers)]),

templateLabels:: (
if cfg.showMultiCluster then [
{
variable:: 'cluster',
label:: cfg.clusterLabel,
query:: 'kube_pod_container_info{image=~".*promtail.*"}',
},
] else []
) + [
{
variable:: 'namespace',
label:: 'namespace',
query:: 'kube_pod_container_info{image=~".*promtail.*"}',
},
{
variable:: 'name',
label:: 'created_by_name',
query:: 'kube_pod_info{namespace="$namespace",pod=~"promtail.*"}',
},
],
} +
g.dashboard('Loki / Promtail')
.addTemplate('cluster', 'kube_pod_container_info{image=~".*promtail.*"}', 'cluster')
.addTemplate('namespace', 'kube_pod_container_info{image=~".*promtail.*"}', 'namespace')
.addTemplate('name', 'kube_pod_info{namespace="$namespace",pod=~"promtail.*"}', 'created_by_name')
.addRow(
g.row('Targets & Files')
.addPanel(
g.panel('Active Targets') +
g.queryPanel(
'sum(promtail_targets_active_total{cluster="$cluster", job="$namespace/$name"})',
'sum(promtail_targets_active_total{%s})' % dashboards['promtail.json'].selector,
'Active Targets',
),
)
.addPanel(
g.panel('Active Files') +
g.queryPanel(
'sum(promtail_files_active_total{cluster="$cluster", job="$namespace/$name"})',
'sum(promtail_files_active_total{%s})' % dashboards['promtail.json'].selector,
'Active Targets',
),
)
Expand All @@ -30,15 +60,15 @@ local utils = import 'mixin-utils/utils.libsonnet';
.addPanel(
g.panel('Bps') +
g.queryPanel(
'sum(rate(promtail_read_bytes_total{cluster="$cluster", job="$namespace/$name"}[1m]))',
'sum(rate(promtail_read_bytes_total{%s}[1m]))' % dashboards['promtail.json'].selector,
'logs read',
) +
{ yaxes: g.yaxes('Bps') },
)
.addPanel(
g.panel('Lines') +
g.queryPanel(
'sum(rate(promtail_read_lines_total{cluster="$cluster", job="$namespace/$name"}[1m]))',
'sum(rate(promtail_read_lines_total{%s}[1m]))' % dashboards['promtail.json'].selector,
'lines read',
),
)
Expand All @@ -47,12 +77,45 @@ local utils = import 'mixin-utils/utils.libsonnet';
g.row('Requests')
.addPanel(
g.panel('QPS') +
g.qpsPanel('promtail_request_duration_seconds_count{cluster="$cluster", job="$namespace/$name"}')
g.qpsPanel('promtail_request_duration_seconds_count{%s}' % dashboards['promtail.json'].selector),
)
.addPanel(
g.panel('Latency') +
utils.latencyRecordingRulePanel('promtail_request_duration_seconds', [utils.selector.eq('job', '$namespace/$name')], extra_selectors=[utils.selector.eq('cluster', '$cluster')])
utils.latencyRecordingRulePanel('promtail_request_duration_seconds', dashboards['promtail.json'].matchers, extra_selectors=dashboards['promtail.json'].clusterMatchers)
)
),
) {
templating+: {
list+: [
{
allValue: null,
current: {
text: 'prod',
value: 'prod',
},
datasource: '$datasource',
hide: 0,
includeAll: false,
label: l.variable,
multi: false,
name: l.variable,
options: [

],
query: 'label_values(%s, %s)' % [l.query, l.label],
refresh: 1,
regex: '',
sort: 2,
tagValuesQuery: '',
tags: [

],
tagsQuery: '',
type: 'query',
useTags: false,
}
for l in dashboards['promtail.json'].templateLabels
],
},
},
},
}