From 893ba832481f03c01dee9b7742f3d6e53ca1bd00 Mon Sep 17 00:00:00 2001 From: Guillaume Berche Date: Wed, 17 Jan 2024 14:54:43 +0100 Subject: [PATCH 1/2] Add example grafana dashboard Signed-off-by: Guillaume Berche --- README.md | 1 + examples/k3s-kine-dashboard.json | 2308 ++++++++++++++++++++++++++++++ 2 files changed, 2309 insertions(+) create mode 100644 examples/k3s-kine-dashboard.json diff --git a/README.md b/README.md index 6eb9b120..ebd2ca2a 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,7 @@ Kine is an etcdshim that translates etcd API to: - Can be ran standalone so any k8s (not just K3s) can use Kine - Implements a subset of etcdAPI (not usable at all for general purpose etcd) - Translates etcdTX calls into the desired API (Create, Update, Delete) +- Exports metrics (see [definition](pkg/metrics/metrics.go) and [example grafana dashboard](examples/k3s-kine-dashboard.json)) See an [example](/examples/minimal.md). diff --git a/examples/k3s-kine-dashboard.json b/examples/k3s-kine-dashboard.json new file mode 100644 index 00000000..a04a3b5d --- /dev/null +++ b/examples/k3s-kine-dashboard.json @@ -0,0 +1,2308 @@ +{ + "__inputs": [ + { + "name": "DS_DS_PROMETHEUS_HA_DATASTORE_DIRECT_MANUAL", + "label": "DS_PROMETHEUS_HA_DATASTORE_DIRECT_manual", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + }, + { + "name": "DS_LOKI", + "label": "loki", + "description": "", + "type": "datasource", + "pluginId": "loki", + "pluginName": "Loki" + }, + { + "name": "DS_DS_PROMETHEUS", + "label": "DS_PROMETHEUS", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "9.3.2" + }, + { + "type": "panel", + "id": "logs", + "name": "Logs", + "version": "" + }, + { + "type": "datasource", + "id": "loki", + "name": "Loki", + "version": "1.0.0" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "text", + "name": "Text", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [ + { + "asDropdown": false, + "icon": "external link", + "includeVars": false, + "keepTime": false, + "tags": [], + "targetBlank": true, + "title": "RE2 syntax (regexp) https://github.com/google/re2/wiki/Syntax", + "tooltip": "", + "type": "link", + "url": "" + }, + { + "asDropdown": false, + "icon": "external link", + "includeVars": false, + "keepTime": false, + "tags": [], + "targetBlank": true, + "title": "RE2 regexp examples https://support.google.com/a/answer/1371417?hl=en", + "tooltip": "", + "type": "link", + "url": "" + }, + { + "asDropdown": false, + "icon": "external link", + "includeVars": false, + "keepTime": false, + "tags": [], + "targetBlank": true, + "title": "Interactive RE2 regexp builder https://regex101.com/r/WPDKO4/1", + "tooltip": "", + "type": "link", + "url": "" + }, + { + "asDropdown": false, + "icon": "external link", + "includeVars": false, + "keepTime": false, + "tags": [], + "targetBlank": true, + "title": "\t Interactive RE2 regexp builder https://regex101.com/r/WPDKO4/1", + "tooltip": "", + "type": "link", + "url": "" + } + ], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 35, + "panels": [], + "title": "Kine Pod bandwidth", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_DS_PROMETHEUS_HA_DATASTORE_DIRECT_MANUAL}" + }, + "description": "container_network_receive_bytes_total cumulative bytes received excluding eth0\nSee https://github.com/google/cadvisor/issues/2615", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 39, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max", + "stdDev" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_DS_PROMETHEUS_HA_DATASTORE_DIRECT_MANUAL}" + }, + "editorMode": "code", + "expr": "sum by(node) (\r\n sum by(pod) (rate(container_network_receive_bytes_total{namespace=\"$namespace\", interface!=\"eth0\"}[$__rate_interval]))\r\n + on(pod) group_left(node) sum by(pod, node) (kube_pod_info{namespace=\"$namespace\"})\r\n)", + "hide": false, + "interval": "", + "legendFormat": "receive by {{node}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_DS_PROMETHEUS_HA_DATASTORE_DIRECT_MANUAL}" + }, + "editorMode": "code", + "expr": "sum by (node) (\r\nsum by(pod) (rate(container_network_transmit_bytes_total{namespace=\"$namespace\", interface!=\"eth0\"}[$__rate_interval]))\r\n+ on(pod) group_left(node) sum by(pod,node) (kube_pod_info{namespace=\"$namespace\"})\r\n )", + "hide": false, + "interval": "", + "legendFormat": "transmit by {{node}}", + "range": true, + "refId": "B" + } + ], + "title": "Pod network bandwidth", + "type": "timeseries" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 12 + }, + "id": 51, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_DS_PROMETHEUS_HA_DATASTORE_DIRECT_MANUAL}" + }, + "description": "container_network_receive_bytes_total cumulative bytes received excluding eth0\nSee https://github.com/google/cadvisor/issues/2615", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 15, + "w": 24, + "x": 0, + "y": 13 + }, + "id": 49, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max", + "stdDev" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_DS_PROMETHEUS_HA_DATASTORE_DIRECT_MANUAL}" + }, + "editorMode": "code", + "expr": "sum by(node,interface) (\r\n sum by(pod,interface) (rate(container_network_receive_bytes_total{namespace=\"$namespace\"}[$__rate_interval]))\r\n + on(pod) group_left(node) sum by(pod, node) (kube_pod_info{namespace=\"$namespace\"})\r\n)", + "hide": false, + "interval": "", + "legendFormat": "receive by pod on node={{node}} on if={{interface}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_DS_PROMETHEUS_HA_DATASTORE_DIRECT_MANUAL}" + }, + "editorMode": "code", + "expr": "sum by (node,interface) (\r\n sum by(pod,interface) (rate(container_network_transmit_bytes_total{namespace=\"$namespace\"}[$__rate_interval]))\r\n + on(pod) group_left(node) sum by(pod,node) (kube_pod_info{namespace=\"$namespace\"})\r\n)", + "hide": false, + "interval": "", + "legendFormat": "transmit by pod on node={{node}} on if={{interface}}", + "range": true, + "refId": "B" + } + ], + "title": "Pod network bandwidth", + "type": "timeseries" + } + ], + "title": "Kine Pod bandwidth per node and interface", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 13 + }, + "id": 42, + "panels": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 46, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.3.2", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "editorMode": "code", + "expr": "sum by(pod) (count_over_time({namespace=~\"$namespace\"} |= \"$query\" |~ \"$query_regexp\" [$__interval]) ) ", + "hide": false, + "instant": false, + "legendFormat": "", + "queryType": "range", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "editorMode": "code", + "expr": "sum by (node) (\r\nsum by(pod) (count_over_time({namespace=~\"$namespace\"} |= \"$query\" |~ \"$query_regexp\" [$__interval])) \r\n+ on(pod) group_left(node) sum by(pod,node) (kube_pod_info{namespace=\"$namespace\"})\r\n)", + "hide": true, + "instant": false, + "legendFormat": "", + "queryType": "range", + "range": true, + "refId": "A" + } + ], + "type": "timeseries" + } + ], + "title": "Kine debug logs stats", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 44, + "panels": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "description": "Logs from services running in Kubernetes", + "gridPos": { + "h": 12, + "w": 23, + "x": 0, + "y": 15 + }, + "id": 48, + "options": { + "dedupStrategy": "none", + "enableLogDetails": false, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": false, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "editorMode": "code", + "expr": "{namespace=~\"$namespace\"} |= \"$query\"", + "hide": true, + "queryType": "range", + "refId": "A" + }, + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "editorMode": "code", + "expr": "{namespace=~\"$namespace\"} |= \"$query\" |~ \"$query_regexp\"", + "hide": false, + "queryType": "range", + "refId": "B" + } + ], + "type": "logs" + } + ], + "title": "Kine debug logs", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 15 + }, + "id": 27, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_DS_PROMETHEUS}" + }, + "description": "The number of connections currently in use.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "connections" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 42 + }, + "id": 29, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (node) (\r\n sum by(pod) (go_sql_in_use_connections ) \r\n + on(pod) group_left(node) sum by(pod,node) (kube_pod_info{namespace=\"94-shared-services-kine\"})\r\n)", + "format": "time_series", + "hide": false, + "legendFormat": "active on pod {{pod}} on node {{node}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (node) (\r\n sum by(pod) (go_sql_max_open_connections ) \r\n + on(pod) group_left(node) sum by(pod,node) (kube_pod_info{namespace=\"94-shared-services-kine\"})\r\n)", + "format": "time_series", + "hide": false, + "legendFormat": "max nb cnx for pod {{pod}} on node {{node}}", + "range": true, + "refId": "A" + } + ], + "title": "kine ", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_DS_PROMETHEUS}" + }, + "description": "The number of connections currently in use.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "connections" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 42 + }, + "id": 33, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (node) (\r\n sum by(pod) (go_sql_in_use_connections ) \r\n + on(pod) group_left(node) sum by(pod,node) (kube_pod_info{namespace=\"94-shared-services-kine\"})\r\n)", + "format": "time_series", + "hide": false, + "legendFormat": "active on pod {{pod}} on node {{node}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (node) (\r\n sum by(pod) (go_sql_max_open_connections ) \r\n + on(pod) group_left(node) sum by(pod,node) (kube_pod_info{namespace=\"94-shared-services-kine\"})\r\n)", + "format": "time_series", + "hide": false, + "legendFormat": "max nb cnx for pod {{pod}} on node {{node}}", + "range": true, + "refId": "A" + } + ], + "title": "kine sql active connections", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_DS_PROMETHEUS}" + }, + "description": "go_sql_max_idle_closed_total The total number of connections closed due to SetMaxIdleConns.\ngo_sql_max_lifetime_closed_total The total number of connections closed due to SetConnMaxLifetime.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "connections" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 53 + }, + "id": 30, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "", + "format": "time_series", + "hide": true, + "legendFormat": "pod {{pod}} on node {{node}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(go_sql_max_idle_closed_total [$__rate_interval])", + "format": "time_series", + "hide": false, + "interval": "", + "legendFormat": "max idle on pod {{pod}} ", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(go_sql_max_idle_time_closed_total [$__rate_interval])", + "format": "time_series", + "hide": false, + "interval": "", + "legendFormat": "max lifetime on pod {{pod}} ", + "range": true, + "refId": "B" + } + ], + "title": "kine sql closed connections per seconds ", + "type": "timeseries" + } + ], + "title": "SQL connections ", + "type": "row" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 24, + "panels": [], + "title": "Compactor", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_DS_PROMETHEUS}" + }, + "description": "each dot represents a change in the kine_compact_total metric\nThe metric is updated by kine after the compaction ends, see source at https://github.com/k3s-io/kine/blob/c1da9bfb3f95dc329f7bacdd6f46b6c824d29ec2/pkg/logstructured/sqllog/sql.go#L168", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisGridShow": true, + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "compactions par seconds" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 25, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (node) (\r\nsum by(pod) (rate(kine_compact_total[$__rate_interval]))\r\n* on(pod) group_left(node) sum by(pod,node) (kube_pod_info{namespace=\"$namespace\"})\r\n )\r\n", + "format": "time_series", + "hide": false, + "legendFormat": "pod on node={{node}}", + "range": true, + "refId": "C" + } + ], + "title": "compaction completion events per pod", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_DS_PROMETHEUS}" + }, + "description": "number of compactions", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "requests per hour" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 27 + }, + "id": 40, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by(pod, result) (rate(kine_compact_total{}[$__rate_interval])) * 3600", + "format": "time_series", + "hide": true, + "legendFormat": "pod {{pod}} on node {{node}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (node,result) (\r\n sum by(pod, result) (rate(kine_compact_total{}[$__rate_interval])) * 3600\r\n * on(pod) group_left(node) sum by(pod,node) (kube_pod_info{namespace=\"$namespace\"})\r\n)", + "format": "time_series", + "hide": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "rate of compactions per node and result", + "type": "timeseries" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 36 + }, + "id": 10, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_DS_PROMETHEUS}" + }, + "description": "number of SQL operations: kine_sql_total", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 73 + }, + "id": 14, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (node) (\r\n sum by(pod) (rate(kine_sql_total[$__rate_interval])) \r\n + on(pod) group_left(node) sum by(pod,node) (kube_pod_info{namespace=\"94-shared-services-kine\"})\r\n)", + "format": "time_series", + "hide": false, + "legendFormat": "pod {{pod}} on node {{node}}", + "range": true, + "refId": "C" + } + ], + "title": "kine sql queries per node", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_DS_PROMETHEUS}" + }, + "description": "number of SQL operations. Individual pods are listed, potentially multiple ones on a same node", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 73 + }, + "id": 18, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by(pod) (rate(kine_sql_total[$__rate_interval]))", + "hide": true, + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by(pod) (rate(kine_sql_total[$__rate_interval])) \r\n+ on(pod) group_left(node) sum by(pod,node) (kube_pod_info{namespace=\"94-shared-services-kine\"})", + "format": "time_series", + "hide": false, + "legendFormat": "pod {{pod}} on node {{node}}", + "range": true, + "refId": "C" + } + ], + "title": "kine sql queries per pod ", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_DS_PROMETHEUS}" + }, + "description": " Length of time per SQL operation (in seconds, metric kine_sql_time_seconds )", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 82 + }, + "id": 7, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (pod) (rate(kine_sql_time_seconds_sum{error_code=\"\"}[$__rate_interval]) / rate(kine_sql_time_seconds_count{error_code=\"\"}[$__rate_interval]))\r\n* on(pod) group_left(node) sum by(pod,node) (kube_pod_info{namespace=\"94-shared-services-kine\"})", + "hide": false, + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Successfull kine requests latency by pod", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_DS_PROMETHEUS}" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 82 + }, + "id": 20, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "Just a trick to insert a blank panel and align errors below", + "mode": "markdown" + }, + "pluginVersion": "9.3.2", + "title": "Layout tweak", + "type": "text" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_DS_PROMETHEUS}" + }, + "description": "number of SQL operations", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 90 + }, + "id": 6, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max", + "stdDev" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by(error_code) (rate(kine_sql_total{error_code!=\"\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Errored kine sql queries per error code", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_DS_PROMETHEUS}" + }, + "description": "number of SQL operations", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 91 + }, + "id": 11, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": " sum by(pod) (rate(kine_sql_total{error_code!=\"\"}[$__rate_interval])) \r\n \r\n", + "format": "time_series", + "hide": true, + "legendFormat": "__auto", + "range": true, + "refId": "raw sql queries by pod" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by(node) (\r\n sum by(pod) (rate(kine_sql_total{error_code!=\"\"}[$__rate_interval])) \r\n * on(pod) group_left(node) sum by(pod,node) (kube_pod_info{namespace=\"94-shared-services-kine\"})\r\n)", + "format": "time_series", + "hide": false, + "legendFormat": "pod {{pod}} on node {{node}}", + "range": true, + "refId": "sql queries joined with node name" + } + ], + "title": "Errored kine sql queries per node ", + "type": "timeseries" + } + ], + "title": "Sql request rate", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 37 + }, + "id": 13, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_DS_PROMETHEUS}" + }, + "description": " Length of time per SQL operation (in seconds, metric kine_sql_time_seconds )", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 74 + }, + "id": 17, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (node) (\r\n sum by (pod) (rate(kine_sql_time_seconds_sum{error_code=\"\"}[$__rate_interval]) / rate (kine_sql_time_seconds_count {error_code=\"\"}[$__rate_interval]))\r\n* on(pod) group_left(node) sum by(pod,node) (kube_pod_info{namespace=\"94-shared-services-kine\"})\r\n)", + "hide": false, + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Sucessfull kine requests latency by node", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_DS_PROMETHEUS}" + }, + "description": "number of SQL operations", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "log": 10, + "type": "log" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 74 + }, + "id": 5, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max", + "stdDev" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Min", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "(sum by(error_code) (rate(kine_sql_time_seconds_sum[$__rate_interval]))) / (sum by(error_code) (rate(kine_sql_time_seconds_count[$__rate_interval])))", + "interval": "", + "legendFormat": "{{error_code}}", + "range": true, + "refId": "A" + } + ], + "title": "latency of kine sql queries per error code", + "type": "timeseries" + } + ], + "title": "Sql request latency", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 38 + }, + "id": 32, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_DS_PROMETHEUS}" + }, + "description": "# HELP go_goroutines Number of goroutines that currently exist. \n# TYPE go_goroutines gauge \ngo_goroutines 29 ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "routines" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 50 + }, + "id": 28, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (node) (\r\n sum by(pod) (go_goroutines) \r\n + on(pod) group_left(node) sum by(pod,node) (kube_pod_info{namespace=\"94-shared-services-kine\"})\r\n)", + "format": "time_series", + "hide": false, + "interval": "", + "legendFormat": "routines on {{pod}} on node {{node}}", + "range": true, + "refId": "C" + } + ], + "title": "kine goroutines ", + "type": "timeseries" + } + ], + "title": "Go lang metrics", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 39 + }, + "id": 16, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_DS_PROMETHEUS}" + }, + "description": " Length of time per SQL operation", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 108 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(kine_sql_time_seconds_bucket[$__rate_interval])) by (le))", + "hide": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Kine requests latency per error ", + "type": "timeseries" + } + ], + "title": "To remove", + "type": "row" + } + ], + "refresh": false, + "schemaVersion": 37, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "DS_PROMETHEUS_HA_DATASTORE_DIRECT_manual", + "value": "DS_PROMETHEUS_HA_DATASTORE_DIRECT_manual" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_DS_PROMETHEUS_HA_DATASTORE_DIRECT_MANUAL}" + }, + "definition": "label_values(kube_pod_info, namespace)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "namespace", + "options": [], + "query": { + "query": "label_values(kube_pod_info, namespace)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "loki", + "value": "loki" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "loki_datasource", + "options": [], + "query": "loki", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": {}, + "description": "String to search for. Case sensitive. See external links for addiitional doc", + "hide": 0, + "label": "logs contains string", + "name": "query", + "options": [], + "query": "", + "skipUrlSync": false, + "type": "textbox" + }, + { + "current": {}, + "description": "Regexp to search for, ex: \"(?i)failed\" for matching FAILED or failed. By default the regexp is case sensitive. See additional doc links icons for further documentation.", + "hide": 0, + "label": "logs contains a match to the regular expression", + "name": "query_regexp", + "options": [], + "query": "", + "skipUrlSync": false, + "type": "textbox" + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "k3s kine", + "uid": "8498173060965e3cddc99c10671de88b6801acfd", + "version": 1, + "weekStart": "" +} \ No newline at end of file From 789f64341ea39337f34c62fcf6614834619efccb Mon Sep 17 00:00:00 2001 From: Guillaume Berche Date: Wed, 17 Jan 2024 15:07:41 +0100 Subject: [PATCH 2/2] Clean up duplicate panel Signed-off-by: Guillaume Berche --- examples/k3s-kine-dashboard.json | 722 +++++++++++++------------------ 1 file changed, 311 insertions(+), 411 deletions(-) diff --git a/examples/k3s-kine-dashboard.json b/examples/k3s-kine-dashboard.json index a04a3b5d..3715da65 100644 --- a/examples/k3s-kine-dashboard.json +++ b/examples/k3s-kine-dashboard.json @@ -143,7 +143,7 @@ "liveNow": false, "panels": [ { - "collapsed": false, + "collapsed": true, "gridPos": { "h": 1, "w": 24, @@ -151,125 +151,126 @@ "y": 0 }, "id": 35, - "panels": [], - "title": "Kine Pod bandwidth", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_DS_PROMETHEUS_HA_DATASTORE_DIRECT_MANUAL}" - }, - "description": "container_network_receive_bytes_total cumulative bytes received excluding eth0\nSee https://github.com/google/cadvisor/issues/2615", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 11, - "w": 24, - "x": 0, - "y": 1 - }, - "id": 39, - "options": { - "legend": { - "calcs": [ - "min", - "mean", - "max", - "stdDev" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ + "panels": [ { "datasource": { "type": "prometheus", "uid": "${DS_DS_PROMETHEUS_HA_DATASTORE_DIRECT_MANUAL}" }, - "editorMode": "code", - "expr": "sum by(node) (\r\n sum by(pod) (rate(container_network_receive_bytes_total{namespace=\"$namespace\", interface!=\"eth0\"}[$__rate_interval]))\r\n + on(pod) group_left(node) sum by(pod, node) (kube_pod_info{namespace=\"$namespace\"})\r\n)", - "hide": false, - "interval": "", - "legendFormat": "receive by {{node}}", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_DS_PROMETHEUS_HA_DATASTORE_DIRECT_MANUAL}" + "description": "container_network_receive_bytes_total cumulative bytes received excluding eth0\nSee https://github.com/google/cadvisor/issues/2615", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 39, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max", + "stdDev" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } }, - "editorMode": "code", - "expr": "sum by (node) (\r\nsum by(pod) (rate(container_network_transmit_bytes_total{namespace=\"$namespace\", interface!=\"eth0\"}[$__rate_interval]))\r\n+ on(pod) group_left(node) sum by(pod,node) (kube_pod_info{namespace=\"$namespace\"})\r\n )", - "hide": false, - "interval": "", - "legendFormat": "transmit by {{node}}", - "range": true, - "refId": "B" + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_DS_PROMETHEUS_HA_DATASTORE_DIRECT_MANUAL}" + }, + "editorMode": "code", + "expr": "sum by(node) (\r\n sum by(pod) (rate(container_network_receive_bytes_total{namespace=\"$namespace\", interface!=\"eth0\"}[$__rate_interval]))\r\n + on(pod) group_left(node) sum by(pod, node) (kube_pod_info{namespace=\"$namespace\"})\r\n)", + "hide": false, + "interval": "", + "legendFormat": "receive by {{node}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_DS_PROMETHEUS_HA_DATASTORE_DIRECT_MANUAL}" + }, + "editorMode": "code", + "expr": "sum by (node) (\r\nsum by(pod) (rate(container_network_transmit_bytes_total{namespace=\"$namespace\", interface!=\"eth0\"}[$__rate_interval]))\r\n+ on(pod) group_left(node) sum by(pod,node) (kube_pod_info{namespace=\"$namespace\"})\r\n )", + "hide": false, + "interval": "", + "legendFormat": "transmit by {{node}}", + "range": true, + "refId": "B" + } + ], + "title": "Pod network bandwidth", + "type": "timeseries" } ], - "title": "Pod network bandwidth", - "type": "timeseries" + "title": "Kine Pod bandwidth", + "type": "row" }, { "collapsed": true, @@ -277,7 +278,7 @@ "h": 1, "w": 24, "x": 0, - "y": 12 + "y": 1 }, "id": 51, "panels": [ @@ -327,7 +328,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -343,7 +345,7 @@ "h": 15, "w": 24, "x": 0, - "y": 13 + "y": 2 }, "id": 49, "options": { @@ -406,7 +408,7 @@ "h": 1, "w": 24, "x": 0, - "y": 13 + "y": 2 }, "id": 42, "panels": [ @@ -456,7 +458,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -471,7 +474,7 @@ "h": 9, "w": 24, "x": 0, - "y": 14 + "y": 3 }, "id": 46, "options": { @@ -535,7 +538,7 @@ "h": 1, "w": 24, "x": 0, - "y": 14 + "y": 3 }, "id": 44, "panels": [ @@ -549,7 +552,7 @@ "h": 12, "w": 23, "x": 0, - "y": 15 + "y": 4 }, "id": 48, "options": { @@ -598,7 +601,7 @@ "h": 1, "w": 24, "x": 0, - "y": 15 + "y": 4 }, "id": 27, "panels": [ @@ -607,7 +610,7 @@ "type": "prometheus", "uid": "${DS_DS_PROMETHEUS}" }, - "description": "The number of connections currently in use.", + "description": "The number of established connections both in use and idle. \ngo_sql_max_open_connections{db_name=\"kine\"}\n\nMaximum number of open connections to the database.\ngo_sql_max_open_connections ", "fieldConfig": { "defaults": { "color": { @@ -648,7 +651,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -664,7 +668,7 @@ "h": 11, "w": 12, "x": 0, - "y": 42 + "y": 5 }, "id": 29, "options": { @@ -713,7 +717,7 @@ "refId": "A" } ], - "title": "kine ", + "title": "kine active and max connections", "type": "timeseries" }, { @@ -721,7 +725,7 @@ "type": "prometheus", "uid": "${DS_DS_PROMETHEUS}" }, - "description": "The number of connections currently in use.", + "description": "go_sql_max_idle_closed_total The total number of connections closed due to SetMaxIdleConns.\ngo_sql_max_lifetime_closed_total The total number of connections closed due to SetConnMaxLifetime.", "fieldConfig": { "defaults": { "color": { @@ -762,7 +766,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -775,12 +780,12 @@ "overrides": [] }, "gridPos": { - "h": 11, + "h": 9, "w": 12, "x": 12, - "y": 42 + "y": 5 }, - "id": 33, + "id": 30, "options": { "legend": { "calcs": [ @@ -806,10 +811,10 @@ "uid": "${DS_DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "sum by (node) (\r\n sum by(pod) (go_sql_in_use_connections ) \r\n + on(pod) group_left(node) sum by(pod,node) (kube_pod_info{namespace=\"94-shared-services-kine\"})\r\n)", + "expr": "", "format": "time_series", - "hide": false, - "legendFormat": "active on pod {{pod}} on node {{node}}", + "hide": true, + "legendFormat": "pod {{pod}} on node {{node}}", "range": true, "refId": "C" }, @@ -819,23 +824,52 @@ "uid": "${DS_DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "sum by (node) (\r\n sum by(pod) (go_sql_max_open_connections ) \r\n + on(pod) group_left(node) sum by(pod,node) (kube_pod_info{namespace=\"94-shared-services-kine\"})\r\n)", + "expr": "rate(go_sql_max_idle_closed_total [$__rate_interval])", "format": "time_series", "hide": false, - "legendFormat": "max nb cnx for pod {{pod}} on node {{node}}", + "interval": "", + "legendFormat": "max idle on pod {{pod}} ", "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(go_sql_max_idle_time_closed_total [$__rate_interval])", + "format": "time_series", + "hide": false, + "interval": "", + "legendFormat": "max lifetime on pod {{pod}} ", + "range": true, + "refId": "B" } ], - "title": "kine sql active connections", + "title": "kine sql closed connections per seconds ", "type": "timeseries" - }, + } + ], + "title": "SQL connections ", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 24, + "panels": [ { "datasource": { "type": "prometheus", "uid": "${DS_DS_PROMETHEUS}" }, - "description": "go_sql_max_idle_closed_total The total number of connections closed due to SetMaxIdleConns.\ngo_sql_max_lifetime_closed_total The total number of connections closed due to SetConnMaxLifetime.", + "description": "each dot represents a change in the kine_compact_total metric\nThe metric is updated by kine after the compaction ends, see source at https://github.com/k3s-io/kine/blob/c1da9bfb3f95dc329f7bacdd6f46b6c824d29ec2/pkg/logstructured/sqllog/sql.go#L168", "fieldConfig": { "defaults": { "color": { @@ -844,10 +878,11 @@ "custom": { "axisCenteredZero": false, "axisColorMode": "text", + "axisGridShow": true, "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, - "drawStyle": "line", + "drawStyle": "points", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { @@ -876,7 +911,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -884,17 +920,17 @@ } ] }, - "unit": "connections" + "unit": "compactions par seconds" }, "overrides": [] }, "gridPos": { - "h": 9, - "w": 12, + "h": 10, + "w": 24, "x": 0, - "y": 53 + "y": 6 }, - "id": 30, + "id": 25, "options": { "legend": { "calcs": [ @@ -920,26 +956,114 @@ "uid": "${DS_DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "", + "expr": "sum by (node) (\r\nsum by(pod) (rate(kine_compact_total[$__rate_interval]))\r\n* on(pod) group_left(node) sum by(pod,node) (kube_pod_info{namespace=\"$namespace\"})\r\n )\r\n", "format": "time_series", - "hide": true, - "legendFormat": "pod {{pod}} on node {{node}}", + "hide": false, + "legendFormat": "pod on node={{node}}", "range": true, "refId": "C" + } + ], + "title": "compaction completion events per pod", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_DS_PROMETHEUS}" + }, + "description": "number of compactions", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "requests per hour" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 40, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "rate(go_sql_max_idle_closed_total [$__rate_interval])", + "expr": "sum by(pod, result) (rate(kine_compact_total{}[$__rate_interval])) * 3600", "format": "time_series", - "hide": false, - "interval": "", - "legendFormat": "max idle on pod {{pod}} ", + "hide": true, + "legendFormat": "pod {{pod}} on node {{node}}", "range": true, - "refId": "A" + "refId": "C" }, { "datasource": { @@ -947,260 +1071,28 @@ "uid": "${DS_DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "rate(go_sql_max_idle_time_closed_total [$__rate_interval])", + "expr": "sum by (node,result) (\r\n sum by(pod, result) (rate(kine_compact_total{}[$__rate_interval])) * 3600\r\n * on(pod) group_left(node) sum by(pod,node) (kube_pod_info{namespace=\"$namespace\"})\r\n)", "format": "time_series", "hide": false, - "interval": "", - "legendFormat": "max lifetime on pod {{pod}} ", + "legendFormat": "__auto", "range": true, - "refId": "B" + "refId": "A" } ], - "title": "kine sql closed connections per seconds ", + "title": "rate of compactions per node and result", "type": "timeseries" } ], - "title": "SQL connections ", - "type": "row" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 16 - }, - "id": 24, - "panels": [], "title": "Compactor", "type": "row" }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_DS_PROMETHEUS}" - }, - "description": "each dot represents a change in the kine_compact_total metric\nThe metric is updated by kine after the compaction ends, see source at https://github.com/k3s-io/kine/blob/c1da9bfb3f95dc329f7bacdd6f46b6c824d29ec2/pkg/logstructured/sqllog/sql.go#L168", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisGridShow": true, - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "compactions par seconds" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 17 - }, - "id": 25, - "options": { - "legend": { - "calcs": [ - "min", - "mean", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Max", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum by (node) (\r\nsum by(pod) (rate(kine_compact_total[$__rate_interval]))\r\n* on(pod) group_left(node) sum by(pod,node) (kube_pod_info{namespace=\"$namespace\"})\r\n )\r\n", - "format": "time_series", - "hide": false, - "legendFormat": "pod on node={{node}}", - "range": true, - "refId": "C" - } - ], - "title": "compaction completion events per pod", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_DS_PROMETHEUS}" - }, - "description": "number of compactions", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "requests per hour" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 24, - "x": 0, - "y": 27 - }, - "id": 40, - "options": { - "legend": { - "calcs": [ - "min", - "mean", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Max", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum by(pod, result) (rate(kine_compact_total{}[$__rate_interval])) * 3600", - "format": "time_series", - "hide": true, - "legendFormat": "pod {{pod}} on node {{node}}", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum by (node,result) (\r\n sum by(pod, result) (rate(kine_compact_total{}[$__rate_interval])) * 3600\r\n * on(pod) group_left(node) sum by(pod,node) (kube_pod_info{namespace=\"$namespace\"})\r\n)", - "format": "time_series", - "hide": false, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "rate of compactions per node and result", - "type": "timeseries" - }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 36 + "y": 6 }, "id": 10, "panels": [ @@ -1250,7 +1142,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1266,7 +1159,7 @@ "h": 9, "w": 12, "x": 0, - "y": 73 + "y": 7 }, "id": 14, "options": { @@ -1351,7 +1244,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1367,7 +1261,7 @@ "h": 9, "w": 12, "x": 12, - "y": 73 + "y": 7 }, "id": 18, "options": { @@ -1465,7 +1359,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1481,7 +1376,7 @@ "h": 9, "w": 12, "x": 0, - "y": 82 + "y": 16 }, "id": 7, "options": { @@ -1529,7 +1424,7 @@ "h": 8, "w": 12, "x": 12, - "y": 82 + "y": 16 }, "id": 20, "options": { @@ -1591,7 +1486,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1607,7 +1503,7 @@ "h": 9, "w": 12, "x": 12, - "y": 90 + "y": 24 }, "id": 6, "options": { @@ -1692,7 +1588,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1708,7 +1605,7 @@ "h": 9, "w": 12, "x": 0, - "y": 91 + "y": 25 }, "id": 11, "options": { @@ -1768,7 +1665,7 @@ "h": 1, "w": 24, "x": 0, - "y": 37 + "y": 7 }, "id": 13, "panels": [ @@ -1818,7 +1715,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1834,7 +1732,7 @@ "h": 9, "w": 12, "x": 0, - "y": 74 + "y": 8 }, "id": 17, "options": { @@ -1923,7 +1821,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1939,7 +1838,7 @@ "h": 9, "w": 12, "x": 12, - "y": 74 + "y": 8 }, "id": 5, "options": { @@ -1988,7 +1887,7 @@ "h": 1, "w": 24, "x": 0, - "y": 38 + "y": 8 }, "id": 32, "panels": [ @@ -2038,7 +1937,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2054,7 +1954,7 @@ "h": 9, "w": 12, "x": 0, - "y": 50 + "y": 9 }, "id": 28, "options": { @@ -2104,7 +2004,7 @@ "h": 1, "w": 24, "x": 0, - "y": 39 + "y": 9 }, "id": 16, "panels": [ @@ -2303,6 +2203,6 @@ "timezone": "", "title": "k3s kine", "uid": "8498173060965e3cddc99c10671de88b6801acfd", - "version": 1, + "version": 2, "weekStart": "" } \ No newline at end of file