diff --git a/production/loki-mixin-compiled-ssd/dashboards/loki-reads-resources.json b/production/loki-mixin-compiled-ssd/dashboards/loki-reads-resources.json deleted file mode 100644 index ed5fc3dd245d4..0000000000000 --- a/production/loki-mixin-compiled-ssd/dashboards/loki-reads-resources.json +++ /dev/null @@ -1,657 +0,0 @@ -{ - "annotations": { - "list": [ ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 0, - "hideControls": false, - "links": [ - { - "asDropdown": true, - "icon": "external link", - "includeVars": true, - "keepTime": true, - "tags": [ - "loki" - ], - "targetBlank": false, - "title": "Loki Dashboards", - "type": "dashboards" - } - ], - "refresh": "10s", - "rows": [ - { - "collapse": false, - "collapsed": false, - "panels": [ - { - "datasource": "$datasource", - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "pointSize": 5, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "request" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FFC000", - "mode": "fixed" - } - }, - { - "id": "custom.fillOpacity", - "value": 0 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "limit" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E02F44", - "mode": "fixed" - } - }, - { - "id": "custom.fillOpacity", - "value": 0 - } - ] - } - ] - }, - "gridPos": { }, - "id": 1, - "links": [ ], - "options": { - "legend": { - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki.*|enterprise-logs)-read.*\"}[$__rate_interval]))", - "format": "time_series", - "legendFormat": "{{pod}}", - "legendLink": null - }, - { - "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki.*|enterprise-logs)-read.*\", resource=\"cpu\"} > 0)", - "format": "time_series", - "legendFormat": "request", - "legendLink": null - }, - { - "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki.*|enterprise-logs)-read.*\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki.*|enterprise-logs)-read.*\"})", - "format": "time_series", - "legendFormat": "limit", - "legendLink": null - } - ], - "title": "CPU", - "tooltip": { - "sort": 2 - }, - "type": "timeseries" - }, - { - "datasource": "$datasource", - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "pointSize": 5, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "request" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FFC000", - "mode": "fixed" - } - }, - { - "id": "custom.fillOpacity", - "value": 0 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "limit" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E02F44", - "mode": "fixed" - } - }, - { - "id": "custom.fillOpacity", - "value": 0 - } - ] - } - ] - }, - "gridPos": { }, - "id": 2, - "links": [ ], - "options": { - "legend": { - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki.*|enterprise-logs)-read.*\"})", - "format": "time_series", - "legendFormat": "{{pod}}", - "legendLink": null - }, - { - "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki.*|enterprise-logs)-read.*\", resource=\"memory\"} > 0)", - "format": "time_series", - "legendFormat": "request", - "legendLink": null - }, - { - "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki.*|enterprise-logs)-read.*\"} > 0)", - "format": "time_series", - "legendFormat": "limit", - "legendLink": null - } - ], - "title": "Memory (workingset)", - "tooltip": { - "sort": 2 - }, - "type": "timeseries" - }, - { - "datasource": "$datasource", - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "pointSize": 5, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ ] - }, - "unit": "bytes" - }, - "overrides": [ ] - }, - "gridPos": { }, - "id": 3, - "links": [ ], - "options": { - "legend": { - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/(loki.*|enterprise-logs)-read\"})", - "format": "time_series", - "legendFormat": "{{pod}}", - "legendLink": null - } - ], - "title": "Memory (go heap inuse)", - "tooltip": { - "sort": 2 - }, - "type": "timeseries" - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Read path", - "titleSize": "h6", - "type": "row" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "datasource": "$datasource", - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "pointSize": 5, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "request" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FFC000", - "mode": "fixed" - } - }, - { - "id": "custom.fillOpacity", - "value": 0 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "limit" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E02F44", - "mode": "fixed" - } - }, - { - "id": "custom.fillOpacity", - "value": 0 - } - ] - } - ] - }, - "id": 4, - "links": [ ], - "options": { - "legend": { - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "span": 4, - "targets": [ - { - "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki.*|enterprise-logs)-write.*\"}[$__rate_interval]))", - "format": "time_series", - "legendFormat": "{{pod}}", - "legendLink": null - }, - { - "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki.*|enterprise-logs)-write.*\", resource=\"cpu\"} > 0)", - "format": "time_series", - "legendFormat": "request", - "legendLink": null - }, - { - "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki.*|enterprise-logs)-write.*\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki.*|enterprise-logs)-write.*\"})", - "format": "time_series", - "legendFormat": "limit", - "legendLink": null - } - ], - "title": "CPU", - "tooltip": { - "sort": 2 - }, - "type": "timeseries" - }, - { - "datasource": "$datasource", - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "pointSize": 5, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "request" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FFC000", - "mode": "fixed" - } - }, - { - "id": "custom.fillOpacity", - "value": 0 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "limit" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E02F44", - "mode": "fixed" - } - }, - { - "id": "custom.fillOpacity", - "value": 0 - } - ] - } - ] - }, - "id": 5, - "links": [ ], - "options": { - "legend": { - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "span": 4, - "targets": [ - { - "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki.*|enterprise-logs)-write.*\"})", - "format": "time_series", - "legendFormat": "{{pod}}", - "legendLink": null - }, - { - "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki.*|enterprise-logs)-write.*\", resource=\"memory\"} > 0)", - "format": "time_series", - "legendFormat": "request", - "legendLink": null - }, - { - "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki.*|enterprise-logs)-write.*\"} > 0)", - "format": "time_series", - "legendFormat": "limit", - "legendLink": null - } - ], - "title": "Memory (workingset)", - "tooltip": { - "sort": 2 - }, - "type": "timeseries" - }, - { - "datasource": "$datasource", - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "pointSize": 5, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ ] - }, - "unit": "bytes" - }, - "overrides": [ ] - }, - "id": 6, - "links": [ ], - "options": { - "legend": { - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "span": 4, - "targets": [ - { - "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/(loki.*|enterprise-logs)-write\"})", - "format": "time_series", - "legendFormat": "{{pod}}", - "legendLink": null - } - ], - "title": "Memory (go heap inuse)", - "tooltip": { - "sort": 2 - }, - "type": "timeseries" - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Ingester", - "titleSize": "h6" - } - ], - "schemaVersion": 14, - "style": "dark", - "tags": [ - "loki" - ], - "templating": { - "list": [ - { - "current": { - "text": "default", - "value": "default" - }, - "hide": 0, - "label": "Data source", - "name": "datasource", - "options": [ ], - "query": "prometheus", - "refresh": 1, - "regex": "", - "type": "datasource" - }, - { - "allValue": null, - "current": { - "text": "prod", - "value": "prod" - }, - "datasource": "$datasource", - "hide": 0, - "includeAll": false, - "label": "cluster", - "multi": false, - "name": "cluster", - "options": [ ], - "query": "label_values(loki_build_info, cluster)", - "refresh": 1, - "regex": "", - "sort": 2, - "tagValuesQuery": "", - "tags": [ ], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - "text": "prod", - "value": "prod" - }, - "datasource": "$datasource", - "hide": 0, - "includeAll": false, - "label": "namespace", - "multi": false, - "name": "namespace", - "options": [ ], - "query": "label_values(loki_build_info{cluster=~\"$cluster\"}, namespace)", - "refresh": 1, - "regex": "", - "sort": 2, - "tagValuesQuery": "", - "tags": [ ], - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "utc", - "title": "Loki / Reads Resources", - "uid": "reads-resources", - "version": 0 - } \ No newline at end of file diff --git a/production/loki-mixin-compiled-ssd/dashboards/loki-resources-overview.json b/production/loki-mixin-compiled-ssd/dashboards/loki-resources-overview.json new file mode 100644 index 0000000000000..5bf38f8e31b50 --- /dev/null +++ b/production/loki-mixin-compiled-ssd/dashboards/loki-resources-overview.json @@ -0,0 +1,1252 @@ +{ + "annotations": { + "list": [ ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "loki" + ], + "targetBlank": false, + "title": "Loki Dashboards", + "type": "dashboards" + } + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "request" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFC000", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E02F44", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "id": 1, + "links": [ ], + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "span": 4, + "targets": [ + { + "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki.*|enterprise-logs)-read.*\"}[$__rate_interval]))", + "format": "time_series", + "legendFormat": "{{pod}}", + "legendLink": null + }, + { + "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki.*|enterprise-logs)-read.*\", resource=\"cpu\"} > 0)", + "format": "time_series", + "legendFormat": "request", + "legendLink": null + }, + { + "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki.*|enterprise-logs)-read.*\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki.*|enterprise-logs)-read.*\"})", + "format": "time_series", + "legendFormat": "limit", + "legendLink": null + } + ], + "title": "CPU", + "tooltip": { + "sort": 2 + }, + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "request" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFC000", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E02F44", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "id": 2, + "links": [ ], + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "span": 4, + "targets": [ + { + "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki.*|enterprise-logs)-read.*\"})", + "format": "time_series", + "legendFormat": "{{pod}}", + "legendLink": null + }, + { + "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki.*|enterprise-logs)-read.*\", resource=\"memory\"} > 0)", + "format": "time_series", + "legendFormat": "request", + "legendLink": null + }, + { + "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki.*|enterprise-logs)-read.*\"} > 0)", + "format": "time_series", + "legendFormat": "limit", + "legendLink": null + } + ], + "title": "Memory (workingset)", + "tooltip": { + "sort": 2 + }, + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "bytes" + }, + "overrides": [ ] + }, + "id": 3, + "links": [ ], + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "span": 4, + "targets": [ + { + "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/(loki.*|enterprise-logs)-read\"})", + "format": "time_series", + "legendFormat": "{{pod}}", + "legendLink": null + } + ], + "title": "Memory (go heap inuse)", + "tooltip": { + "sort": 2 + }, + "type": "timeseries" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Read path", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, + "id": 4, + "links": [ ], + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "span": 1, + "targets": [ + { + "expr": "sum by(pod) (loki_write_memory_streams{cluster=~\"$cluster\", job=~\"($namespace)/(loki.*|enterprise-logs)-write\"})", + "format": "time_series", + "legendFormat": "{{pod}}", + "legendLink": null + } + ], + "title": "In-memory streams", + "tooltip": { + "sort": 2 + }, + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "request" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFC000", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E02F44", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "id": 5, + "links": [ ], + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "span": 1, + "targets": [ + { + "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki.*|enterprise-logs)-write.*\"}[$__rate_interval]))", + "format": "time_series", + "legendFormat": "{{pod}}", + "legendLink": null + }, + { + "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki.*|enterprise-logs)-write.*\", resource=\"cpu\"} > 0)", + "format": "time_series", + "legendFormat": "request", + "legendLink": null + }, + { + "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki.*|enterprise-logs)-write.*\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki.*|enterprise-logs)-write.*\"})", + "format": "time_series", + "legendFormat": "limit", + "legendLink": null + } + ], + "title": "CPU", + "tooltip": { + "sort": 2 + }, + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "request" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFC000", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E02F44", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "id": 6, + "links": [ ], + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "span": 1, + "targets": [ + { + "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki.*|enterprise-logs)-write.*\"})", + "format": "time_series", + "legendFormat": "{{pod}}", + "legendLink": null + }, + { + "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki.*|enterprise-logs)-write.*\", resource=\"memory\"} > 0)", + "format": "time_series", + "legendFormat": "request", + "legendLink": null + }, + { + "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki.*|enterprise-logs)-write.*\"} > 0)", + "format": "time_series", + "legendFormat": "limit", + "legendLink": null + } + ], + "title": "Memory (workingset)", + "tooltip": { + "sort": 2 + }, + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "bytes" + }, + "overrides": [ ] + }, + "id": 7, + "links": [ ], + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "span": 1, + "targets": [ + { + "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/(loki.*|enterprise-logs)-write\"})", + "format": "time_series", + "legendFormat": "{{pod}}", + "legendLink": null + } + ], + "title": "Memory (go heap inuse)", + "tooltip": { + "sort": 2 + }, + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 100, + "lineWidth": 0, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "Bps" + }, + "overrides": [ ] + }, + "id": 8, + "links": [ ], + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "span": 1, + "targets": [ + { + "expr": "sum by(instance, pod, device) (rate(node_disk_written_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki.*|enterprise-logs)-write.*\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n", + "format": "time_series", + "legendFormat": "{{pod}} - {{device}}", + "legendLink": null + } + ], + "title": "Disk Writes", + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 100, + "lineWidth": 0, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "Bps" + }, + "overrides": [ ] + }, + "id": 9, + "links": [ ], + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "span": 1, + "targets": [ + { + "expr": "sum by(instance, pod, device) (rate(node_disk_read_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki.*|enterprise-logs)-write.*\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n", + "format": "time_series", + "legendFormat": "{{pod}} - {{device}}", + "legendLink": null + } + ], + "title": "Disk Reads", + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "percentunit" + }, + "overrides": [ ] + }, + "id": 10, + "links": [ ], + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "span": 1, + "targets": [ + { + "expr": "max by(persistentvolumeclaim) (kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} / kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}) and count by(persistentvolumeclaim) (kube_persistentvolumeclaim_labels{cluster=~\"$cluster\", namespace=~\"$namespace\",label_name=~\"(loki.*|enterprise-logs)-write.*\"})", + "format": "time_series", + "legendFormat": "{{persistentvolumeclaim}}", + "legendLink": null + } + ], + "title": "Disk Space Utilization", + "type": "timeseries" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Write path", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "request" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFC000", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E02F44", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "id": 11, + "links": [ ], + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "span": 2, + "targets": [ + { + "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki.*|enterprise-logs)-backend.*\"}[$__rate_interval]))", + "format": "time_series", + "legendFormat": "{{pod}}", + "legendLink": null + }, + { + "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki.*|enterprise-logs)-backend.*\", resource=\"cpu\"} > 0)", + "format": "time_series", + "legendFormat": "request", + "legendLink": null + }, + { + "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki.*|enterprise-logs)-backend.*\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki.*|enterprise-logs)-backend.*\"})", + "format": "time_series", + "legendFormat": "limit", + "legendLink": null + } + ], + "title": "CPU", + "tooltip": { + "sort": 2 + }, + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "request" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFC000", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E02F44", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "id": 12, + "links": [ ], + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "span": 2, + "targets": [ + { + "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki.*|enterprise-logs)-backend.*\"})", + "format": "time_series", + "legendFormat": "{{pod}}", + "legendLink": null + }, + { + "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki.*|enterprise-logs)-backend.*\", resource=\"memory\"} > 0)", + "format": "time_series", + "legendFormat": "request", + "legendLink": null + }, + { + "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki.*|enterprise-logs)-backend.*\"} > 0)", + "format": "time_series", + "legendFormat": "limit", + "legendLink": null + } + ], + "title": "Memory (workingset)", + "tooltip": { + "sort": 2 + }, + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "bytes" + }, + "overrides": [ ] + }, + "id": 13, + "links": [ ], + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "span": 2, + "targets": [ + { + "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/(loki.*|enterprise-logs)-backend\"})", + "format": "time_series", + "legendFormat": "{{pod}}", + "legendLink": null + } + ], + "title": "Memory (go heap inuse)", + "tooltip": { + "sort": 2 + }, + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 100, + "lineWidth": 0, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "Bps" + }, + "overrides": [ ] + }, + "id": 14, + "links": [ ], + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "span": 2, + "targets": [ + { + "expr": "sum by(instance, pod, device) (rate(node_disk_written_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki.*|enterprise-logs)-backend.*\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n", + "format": "time_series", + "legendFormat": "{{pod}} - {{device}}", + "legendLink": null + } + ], + "title": "Disk Writes", + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 100, + "lineWidth": 0, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "Bps" + }, + "overrides": [ ] + }, + "id": 15, + "links": [ ], + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "span": 2, + "targets": [ + { + "expr": "sum by(instance, pod, device) (rate(node_disk_read_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki.*|enterprise-logs)-backend.*\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n", + "format": "time_series", + "legendFormat": "{{pod}} - {{device}}", + "legendLink": null + } + ], + "title": "Disk Reads", + "type": "timeseries" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "percentunit" + }, + "overrides": [ ] + }, + "id": 16, + "links": [ ], + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "span": 2, + "targets": [ + { + "expr": "max by(persistentvolumeclaim) (kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} / kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}) and count by(persistentvolumeclaim) (kube_persistentvolumeclaim_labels{cluster=~\"$cluster\", namespace=~\"$namespace\",label_name=~\"(loki.*|enterprise-logs)-backend.*\"})", + "format": "time_series", + "legendFormat": "{{persistentvolumeclaim}}", + "legendLink": null + } + ], + "title": "Disk Space Utilization", + "type": "timeseries" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Backend path", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "loki" + ], + "templating": { + "list": [ + { + "current": { + "text": "default", + "value": "default" + }, + "hide": 0, + "label": "Data source", + "name": "datasource", + "options": [ ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ ], + "query": "label_values(loki_build_info, cluster)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "namespace", + "multi": false, + "name": "namespace", + "options": [ ], + "query": "label_values(loki_build_info{cluster=~\"$cluster\"}, namespace)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "utc", + "title": "Loki / Resources Overview", + "uid": "resources-overview", + "version": 0 + } \ No newline at end of file diff --git a/production/loki-mixin-compiled-ssd/dashboards/loki-writes-resources.json b/production/loki-mixin-compiled-ssd/dashboards/loki-writes-resources.json deleted file mode 100644 index dd366ff201df2..0000000000000 --- a/production/loki-mixin-compiled-ssd/dashboards/loki-writes-resources.json +++ /dev/null @@ -1,584 +0,0 @@ -{ - "annotations": { - "list": [ ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 0, - "hideControls": false, - "links": [ - { - "asDropdown": true, - "icon": "external link", - "includeVars": true, - "keepTime": true, - "tags": [ - "loki" - ], - "targetBlank": false, - "title": "Loki Dashboards", - "type": "dashboards" - } - ], - "refresh": "10s", - "rows": [ - { - "collapse": false, - "collapsed": false, - "panels": [ - { - "datasource": "$datasource", - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "pointSize": 5, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ ] - }, - "unit": "short" - }, - "overrides": [ ] - }, - "gridPos": { }, - "id": 1, - "links": [ ], - "options": { - "legend": { - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "expr": "sum by(pod) (loki_ingester_memory_streams{cluster=~\"$cluster\", job=~\"($namespace)/(loki.*|enterprise-logs)-write\"})", - "format": "time_series", - "legendFormat": "{{pod}}", - "legendLink": null - } - ], - "title": "In-memory streams", - "tooltip": { - "sort": 2 - }, - "type": "timeseries" - }, - { - "datasource": "$datasource", - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "pointSize": 5, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "request" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FFC000", - "mode": "fixed" - } - }, - { - "id": "custom.fillOpacity", - "value": 0 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "limit" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E02F44", - "mode": "fixed" - } - }, - { - "id": "custom.fillOpacity", - "value": 0 - } - ] - } - ] - }, - "gridPos": { }, - "id": 2, - "links": [ ], - "options": { - "legend": { - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki.*|enterprise-logs)-write.*\"}[$__rate_interval]))", - "format": "time_series", - "legendFormat": "{{pod}}", - "legendLink": null - }, - { - "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki.*|enterprise-logs)-write.*\", resource=\"cpu\"} > 0)", - "format": "time_series", - "legendFormat": "request", - "legendLink": null - }, - { - "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki.*|enterprise-logs)-write.*\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki.*|enterprise-logs)-write.*\"})", - "format": "time_series", - "legendFormat": "limit", - "legendLink": null - } - ], - "title": "CPU", - "tooltip": { - "sort": 2 - }, - "type": "timeseries" - }, - { - "datasource": "$datasource", - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "pointSize": 5, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "request" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#FFC000", - "mode": "fixed" - } - }, - { - "id": "custom.fillOpacity", - "value": 0 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "limit" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#E02F44", - "mode": "fixed" - } - }, - { - "id": "custom.fillOpacity", - "value": 0 - } - ] - } - ] - }, - "gridPos": { }, - "id": 3, - "links": [ ], - "options": { - "legend": { - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki.*|enterprise-logs)-write.*\"})", - "format": "time_series", - "legendFormat": "{{pod}}", - "legendLink": null - }, - { - "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki.*|enterprise-logs)-write.*\", resource=\"memory\"} > 0)", - "format": "time_series", - "legendFormat": "request", - "legendLink": null - }, - { - "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki.*|enterprise-logs)-write.*\"} > 0)", - "format": "time_series", - "legendFormat": "limit", - "legendLink": null - } - ], - "title": "Memory (workingset)", - "tooltip": { - "sort": 2 - }, - "type": "timeseries" - }, - { - "datasource": "$datasource", - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "pointSize": 5, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ ] - }, - "unit": "bytes" - }, - "overrides": [ ] - }, - "gridPos": { }, - "id": 4, - "links": [ ], - "options": { - "legend": { - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/(loki.*|enterprise-logs)-write\"})", - "format": "time_series", - "legendFormat": "{{pod}}", - "legendLink": null - } - ], - "title": "Memory (go heap inuse)", - "tooltip": { - "sort": 2 - }, - "type": "timeseries" - }, - { - "datasource": "$datasource", - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 100, - "lineWidth": 0, - "pointSize": 5, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ ] - }, - "unit": "Bps" - }, - "overrides": [ ] - }, - "gridPos": { }, - "id": 5, - "links": [ ], - "options": { - "legend": { - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "expr": "sum by(instance, pod, device) (rate(node_disk_written_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki.*|enterprise-logs)-write.*\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n", - "format": "time_series", - "legendFormat": "{{pod}} - {{device}}", - "legendLink": null - } - ], - "title": "Disk Writes", - "type": "timeseries" - }, - { - "datasource": "$datasource", - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 100, - "lineWidth": 0, - "pointSize": 5, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ ] - }, - "unit": "Bps" - }, - "overrides": [ ] - }, - "gridPos": { }, - "id": 6, - "links": [ ], - "options": { - "legend": { - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "expr": "sum by(instance, pod, device) (rate(node_disk_read_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki.*|enterprise-logs)-write.*\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n", - "format": "time_series", - "legendFormat": "{{pod}} - {{device}}", - "legendLink": null - } - ], - "title": "Disk Reads", - "type": "timeseries" - }, - { - "datasource": "$datasource", - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "pointSize": 5, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ ] - }, - "unit": "percentunit" - }, - "overrides": [ ] - }, - "gridPos": { }, - "id": 7, - "links": [ ], - "options": { - "legend": { - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "expr": "max by(persistentvolumeclaim) (kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} / kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}) and count by(persistentvolumeclaim) (kube_persistentvolumeclaim_labels{cluster=~\"$cluster\", namespace=~\"$namespace\",label_name=~\"(loki.*|enterprise-logs)-write.*\"})", - "format": "time_series", - "legendFormat": "{{persistentvolumeclaim}}", - "legendLink": null - } - ], - "title": "Disk Space Utilization", - "type": "timeseries" - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Write path", - "titleSize": "h6", - "type": "row" - } - ], - "schemaVersion": 14, - "style": "dark", - "tags": [ - "loki" - ], - "templating": { - "list": [ - { - "current": { - "text": "default", - "value": "default" - }, - "hide": 0, - "label": "Data source", - "name": "datasource", - "options": [ ], - "query": "prometheus", - "refresh": 1, - "regex": "", - "type": "datasource" - }, - { - "allValue": null, - "current": { - "text": "prod", - "value": "prod" - }, - "datasource": "$datasource", - "hide": 0, - "includeAll": false, - "label": "cluster", - "multi": false, - "name": "cluster", - "options": [ ], - "query": "label_values(loki_build_info, cluster)", - "refresh": 1, - "regex": "", - "sort": 2, - "tagValuesQuery": "", - "tags": [ ], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - "text": "prod", - "value": "prod" - }, - "datasource": "$datasource", - "hide": 0, - "includeAll": false, - "label": "namespace", - "multi": false, - "name": "namespace", - "options": [ ], - "query": "label_values(loki_build_info{cluster=~\"$cluster\"}, namespace)", - "refresh": 1, - "regex": "", - "sort": 2, - "tagValuesQuery": "", - "tags": [ ], - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "utc", - "title": "Loki / Writes Resources", - "uid": "writes-resources", - "version": 0 - } \ No newline at end of file diff --git a/production/loki-mixin-compiled/dashboards/loki-reads-resources.json b/production/loki-mixin-compiled/dashboards/loki-reads-resources.json index ed3965e5d7a27..6774c9d2cfb95 100644 --- a/production/loki-mixin-compiled/dashboards/loki-reads-resources.json +++ b/production/loki-mixin-compiled/dashboards/loki-reads-resources.json @@ -552,7 +552,7 @@ }, { "collapse": false, - "collapsed": false, + "height": "250px", "panels": [ { "datasource": "$datasource", @@ -617,7 +617,6 @@ } ] }, - "gridPos": { }, "id": 7, "links": [ ], "options": { @@ -629,6 +628,7 @@ "sort": "none" } }, + "span": 2, "targets": [ { "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"querier\"}[$__rate_interval]))", @@ -718,7 +718,6 @@ } ] }, - "gridPos": { }, "id": 8, "links": [ ], "options": { @@ -730,6 +729,7 @@ "sort": "none" } }, + "span": 2, "targets": [ { "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"querier\"})", @@ -780,7 +780,6 @@ }, "overrides": [ ] }, - "gridPos": { }, "id": 9, "links": [ ], "options": { @@ -792,6 +791,7 @@ "sort": "none" } }, + "span": 2, "targets": [ { "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/querier\"})", @@ -830,7 +830,6 @@ }, "overrides": [ ] }, - "gridPos": { }, "id": 10, "links": [ ], "options": { @@ -842,6 +841,7 @@ "sort": "none" } }, + "span": 2, "targets": [ { "expr": "sum by(instance, pod, device) (rate(node_disk_written_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"querier\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n", @@ -877,7 +877,6 @@ }, "overrides": [ ] }, - "gridPos": { }, "id": 11, "links": [ ], "options": { @@ -889,6 +888,7 @@ "sort": "none" } }, + "span": 2, "targets": [ { "expr": "sum by(instance, pod, device) (rate(node_disk_read_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"querier\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n", @@ -924,7 +924,6 @@ }, "overrides": [ ] }, - "gridPos": { }, "id": 12, "links": [ ], "options": { @@ -936,6 +935,7 @@ "sort": "none" } }, + "span": 2, "targets": [ { "expr": "max by(persistentvolumeclaim) (kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} / kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}) and count by(persistentvolumeclaim) (kube_persistentvolumeclaim_labels{cluster=~\"$cluster\", namespace=~\"$namespace\",label_name=~\"querier.*\"})", @@ -953,12 +953,11 @@ "repeatRowId": null, "showTitle": true, "title": "Querier", - "titleSize": "h6", - "type": "row" + "titleSize": "h6" }, { "collapse": false, - "collapsed": false, + "height": "250px", "panels": [ { "datasource": "$datasource", @@ -1023,7 +1022,6 @@ } ] }, - "gridPos": { }, "id": 13, "links": [ ], "options": { @@ -1035,6 +1033,7 @@ "sort": "none" } }, + "span": 2, "targets": [ { "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"index-gateway\"}[$__rate_interval]))", @@ -1124,7 +1123,6 @@ } ] }, - "gridPos": { }, "id": 14, "links": [ ], "options": { @@ -1136,6 +1134,7 @@ "sort": "none" } }, + "span": 2, "targets": [ { "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"index-gateway\"})", @@ -1186,7 +1185,6 @@ }, "overrides": [ ] }, - "gridPos": { }, "id": 15, "links": [ ], "options": { @@ -1198,6 +1196,7 @@ "sort": "none" } }, + "span": 2, "targets": [ { "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/index-gateway\"})", @@ -1236,7 +1235,6 @@ }, "overrides": [ ] }, - "gridPos": { }, "id": 16, "links": [ ], "options": { @@ -1248,6 +1246,7 @@ "sort": "none" } }, + "span": 2, "targets": [ { "expr": "sum by(instance, pod, device) (rate(node_disk_written_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"index-gateway\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n", @@ -1283,7 +1282,6 @@ }, "overrides": [ ] }, - "gridPos": { }, "id": 17, "links": [ ], "options": { @@ -1295,6 +1293,7 @@ "sort": "none" } }, + "span": 2, "targets": [ { "expr": "sum by(instance, pod, device) (rate(node_disk_read_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"index-gateway\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n", @@ -1330,7 +1329,6 @@ }, "overrides": [ ] }, - "gridPos": { }, "id": 18, "links": [ ], "options": { @@ -1342,6 +1340,7 @@ "sort": "none" } }, + "span": 2, "targets": [ { "expr": "max by(persistentvolumeclaim) (kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} / kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}) and count by(persistentvolumeclaim) (kube_persistentvolumeclaim_labels{cluster=~\"$cluster\", namespace=~\"$namespace\",label_name=~\"index-gateway.*\"})", @@ -1359,12 +1358,11 @@ "repeatRowId": null, "showTitle": true, "title": "Index Gateway", - "titleSize": "h6", - "type": "row" + "titleSize": "h6" }, { "collapse": false, - "collapsed": false, + "height": "250px", "panels": [ { "datasource": "$datasource", @@ -1429,7 +1427,6 @@ } ] }, - "gridPos": { }, "id": 19, "links": [ ], "options": { @@ -1441,6 +1438,7 @@ "sort": "none" } }, + "span": 2, "targets": [ { "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-gateway\"}[$__rate_interval]))", @@ -1530,7 +1528,6 @@ } ] }, - "gridPos": { }, "id": 20, "links": [ ], "options": { @@ -1542,6 +1539,7 @@ "sort": "none" } }, + "span": 2, "targets": [ { "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-gateway\"})", @@ -1592,7 +1590,6 @@ }, "overrides": [ ] }, - "gridPos": { }, "id": 21, "links": [ ], "options": { @@ -1604,6 +1601,7 @@ "sort": "none" } }, + "span": 2, "targets": [ { "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/bloom-gateway\"})", @@ -1642,7 +1640,6 @@ }, "overrides": [ ] }, - "gridPos": { }, "id": 22, "links": [ ], "options": { @@ -1654,6 +1651,7 @@ "sort": "none" } }, + "span": 2, "targets": [ { "expr": "sum by(instance, pod, device) (rate(node_disk_written_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-gateway\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n", @@ -1689,7 +1687,6 @@ }, "overrides": [ ] }, - "gridPos": { }, "id": 23, "links": [ ], "options": { @@ -1701,6 +1698,7 @@ "sort": "none" } }, + "span": 2, "targets": [ { "expr": "sum by(instance, pod, device) (rate(node_disk_read_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-gateway\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n", @@ -1736,7 +1734,6 @@ }, "overrides": [ ] }, - "gridPos": { }, "id": 24, "links": [ ], "options": { @@ -1748,6 +1745,7 @@ "sort": "none" } }, + "span": 2, "targets": [ { "expr": "max by(persistentvolumeclaim) (kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} / kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}) and count by(persistentvolumeclaim) (kube_persistentvolumeclaim_labels{cluster=~\"$cluster\", namespace=~\"$namespace\",label_name=~\"bloom-gateway.*\"})", @@ -1765,8 +1763,7 @@ "repeatRowId": null, "showTitle": true, "title": "Bloom Gateway", - "titleSize": "h6", - "type": "row" + "titleSize": "h6" }, { "collapse": false, @@ -2034,7 +2031,7 @@ }, { "collapse": false, - "collapsed": false, + "height": "250px", "panels": [ { "datasource": "$datasource", @@ -2060,7 +2057,6 @@ }, "overrides": [ ] }, - "gridPos": { }, "id": 28, "links": [ ], "options": { @@ -2072,6 +2068,7 @@ "sort": "none" } }, + "span": 3, "targets": [ { "expr": "sum by(pod) (loki_prometheus_rule_group_rules{cluster=~\"$cluster\", job=~\"($namespace)/ruler\"}) or sum by(pod) (cortex_prometheus_rule_group_rules{cluster=~\"$cluster\", job=~\"($namespace)/ruler\"})", @@ -2146,7 +2143,6 @@ } ] }, - "gridPos": { }, "id": 29, "links": [ ], "options": { @@ -2158,6 +2154,7 @@ "sort": "none" } }, + "span": 3, "targets": [ { "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"ruler\"}[$__rate_interval]))", @@ -2247,7 +2244,6 @@ } ] }, - "gridPos": { }, "id": 30, "links": [ ], "options": { @@ -2259,6 +2255,7 @@ "sort": "none" } }, + "span": 3, "targets": [ { "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"ruler\"})", @@ -2309,7 +2306,6 @@ }, "overrides": [ ] }, - "gridPos": { }, "id": 31, "links": [ ], "options": { @@ -2321,6 +2317,7 @@ "sort": "none" } }, + "span": 3, "targets": [ { "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/ruler\"})", @@ -2341,8 +2338,7 @@ "repeatRowId": null, "showTitle": true, "title": "Ruler", - "titleSize": "h6", - "type": "row" + "titleSize": "h6" } ], "schemaVersion": 14, diff --git a/production/loki-mixin-compiled/dashboards/loki-writes-resources.json b/production/loki-mixin-compiled/dashboards/loki-writes-resources.json index 40e21484ee549..5252e7fc60f16 100644 --- a/production/loki-mixin-compiled/dashboards/loki-writes-resources.json +++ b/production/loki-mixin-compiled/dashboards/loki-writes-resources.json @@ -288,7 +288,7 @@ }, { "collapse": false, - "collapsed": false, + "height": "250px", "panels": [ { "datasource": "$datasource", @@ -314,7 +314,6 @@ }, "overrides": [ ] }, - "gridPos": { }, "id": 4, "links": [ ], "options": { @@ -326,6 +325,7 @@ "sort": "none" } }, + "span": 1, "targets": [ { "expr": "sum by(pod) (loki_ingester_memory_streams{cluster=~\"$cluster\", job=~\"($namespace)/ingester.*\"})", @@ -403,7 +403,6 @@ } ] }, - "gridPos": { }, "id": 5, "links": [ ], "options": { @@ -415,6 +414,7 @@ "sort": "none" } }, + "span": 1, "targets": [ { "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"ingester\"}[$__rate_interval]))", @@ -504,7 +504,6 @@ } ] }, - "gridPos": { }, "id": 6, "links": [ ], "options": { @@ -516,6 +515,7 @@ "sort": "none" } }, + "span": 1, "targets": [ { "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"ingester\"})", @@ -566,7 +566,6 @@ }, "overrides": [ ] }, - "gridPos": { }, "id": 7, "links": [ ], "options": { @@ -578,6 +577,7 @@ "sort": "none" } }, + "span": 1, "targets": [ { "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/ingester.*\"})", @@ -616,7 +616,6 @@ }, "overrides": [ ] }, - "gridPos": { }, "id": 8, "links": [ ], "options": { @@ -628,6 +627,7 @@ "sort": "none" } }, + "span": 1, "targets": [ { "expr": "sum by(instance, pod, device) (rate(node_disk_written_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"ingester\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n", @@ -663,7 +663,6 @@ }, "overrides": [ ] }, - "gridPos": { }, "id": 9, "links": [ ], "options": { @@ -675,6 +674,7 @@ "sort": "none" } }, + "span": 1, "targets": [ { "expr": "sum by(instance, pod, device) (rate(node_disk_read_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"ingester\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n", @@ -710,7 +710,6 @@ }, "overrides": [ ] }, - "gridPos": { }, "id": 10, "links": [ ], "options": { @@ -722,6 +721,7 @@ "sort": "none" } }, + "span": 1, "targets": [ { "expr": "max by(persistentvolumeclaim) (kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} / kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}) and count by(persistentvolumeclaim) (kube_persistentvolumeclaim_labels{cluster=~\"$cluster\", namespace=~\"$namespace\",label_name=~\"ingester.*.*\"})", @@ -739,8 +739,7 @@ "repeatRowId": null, "showTitle": true, "title": "Ingester", - "titleSize": "h6", - "type": "row" + "titleSize": "h6" } ], "schemaVersion": 14, diff --git a/production/loki-mixin/dashboards.libsonnet b/production/loki-mixin/dashboards.libsonnet index 33f3b136d5ad6..8b1ced08f3d15 100644 --- a/production/loki-mixin/dashboards.libsonnet +++ b/production/loki-mixin/dashboards.libsonnet @@ -3,10 +3,11 @@ (import 'dashboards/loki-chunks.libsonnet') + (import 'dashboards/loki-logs.libsonnet') + (import 'dashboards/loki-operational.libsonnet') + +(import 'dashboards/loki-resources-overview.libsonnet') + (import 'dashboards/loki-reads.libsonnet') + +(import 'dashboards/loki-reads-resources.libsonnet') + (import 'dashboards/loki-writes.libsonnet') + (import 'dashboards/loki-writes-resources.libsonnet') + -(import 'dashboards/loki-reads-resources.libsonnet') + (import 'dashboards/loki-deletion.libsonnet') + (import 'dashboards/loki-canary-dashboard.libsonnet') + (import 'dashboards/recording-rules.libsonnet') + diff --git a/production/loki-mixin/dashboards/loki-reads-resources.libsonnet b/production/loki-mixin/dashboards/loki-reads-resources.libsonnet index 0ec22e131edbc..2635596142d6c 100644 --- a/production/loki-mixin/dashboards/loki-reads-resources.libsonnet +++ b/production/loki-mixin/dashboards/loki-reads-resources.libsonnet @@ -1,209 +1,185 @@ -local grafana = import 'grafonnet/grafana.libsonnet'; -local utils = import 'mixin-utils/utils.libsonnet'; - (import 'dashboard-utils.libsonnet') { local index_gateway_pod_matcher = if $._config.meta_monitoring.enabled - then 'container=~"loki|index-gateway", pod=~"(index-gateway.*|%s-read.*|loki-single-binary)"' % $._config.ssd.pod_prefix_matcher - else if $._config.ssd.enabled then 'container="loki", pod=~"%s-read.*"' % $._config.ssd.pod_prefix_matcher else 'container="index-gateway"', + then 'container=~"loki|index-gateway", pod=~"(index-gateway.*|loki-single-binary)"' + else 'container="index-gateway"', local index_gateway_job_matcher = if $._config.meta_monitoring.enabled - then '(index-gateway.*|%s-read.*|loki-single-binary)' % $._config.ssd.pod_prefix_matcher - else if $._config.ssd.enabled then '%s-read' % $._config.ssd.pod_prefix_matcher else 'index-gateway', + then '(index-gateway.*|loki-single-binary)' + else 'index-gateway', local ingester_pod_matcher = if $._config.meta_monitoring.enabled - then 'container=~"loki|ingester", pod=~"(ingester.*|%s-write.*|loki-single-binary)"' % $._config.ssd.pod_prefix_matcher - else if $._config.ssd.enabled then 'container="loki", pod=~"%s-write.*"' % $._config.ssd.pod_prefix_matcher else 'container="ingester"', + then 'container=~"loki|ingester", pod=~"(ingester.*|loki-single-binary)"' + else 'container="ingester"', local ingester_job_matcher = if $._config.meta_monitoring.enabled - then '(ingester.+|%s-write|loki-single-binary)' % $._config.ssd.pod_prefix_matcher - else if $._config.ssd.enabled then '%s-write' % $._config.ssd.pod_prefix_matcher else 'ingester.+', + then '(ingester.+|loki-single-binary)' + else 'ingester.+', - grafanaDashboards+:: - { - 'loki-reads-resources.json': - ($.dashboard('Loki / Reads Resources', uid='reads-resources')) - .addCluster() - .addNamespace() - .addTag() - .addRowIf( - $._config.internal_components, - $.row('Gateway') - .addPanel( - $.containerCPUUsagePanel('CPU', 'cortex-gw(-internal)?'), - ) - .addPanel( - $.containerMemoryWorkingSetPanel('Memory (workingset)', 'cortex-gw(-internal)?'), - ) - .addPanel( - $.goHeapInUsePanel('Memory (go heap inuse)', 'cortex-gw(-internal)?'), - ) - ) - .addRowIf( - !$._config.ssd.enabled, - $.row('Query Frontend') - .addPanel( - $.containerCPUUsagePanel('CPU', 'query-frontend'), - ) - .addPanel( - $.containerMemoryWorkingSetPanel('Memory (workingset)', 'query-frontend'), - ) - .addPanel( - $.goHeapInUsePanel('Memory (go heap inuse)', 'query-frontend'), - ) - ) - .addRowIf( - !$._config.ssd.enabled, - $.row('Query Scheduler') - .addPanel( - $.containerCPUUsagePanel('CPU', 'query-scheduler'), - ) - .addPanel( - $.containerMemoryWorkingSetPanel('Memory (workingset)', 'query-scheduler'), - ) - .addPanel( - $.goHeapInUsePanel('Memory (go heap inuse)', 'query-scheduler'), - ) - ) - .addRowIf( - !$._config.ssd.enabled, - grafana.row.new('Querier') - .addPanel( - $.containerCPUUsagePanel('CPU', 'querier'), - ) - .addPanel( - $.containerMemoryWorkingSetPanel('Memory (workingset)', 'querier'), - ) - .addPanel( - $.goHeapInUsePanel('Memory (go heap inuse)', 'querier'), - ) - .addPanel( - $.newQueryPanel('Disk Writes', 'Bps') + - $.queryPanel( - 'sum by(%s, %s, device) (rate(node_disk_written_bytes_total[$__rate_interval])) + %s' % [$._config.per_node_label, $._config.per_instance_label, $.filterNodeDiskContainer('querier')], - '{{%s}} - {{device}}' % $._config.per_instance_label - ) + - $.withStacking, - ) - .addPanel( - $.newQueryPanel('Disk Reads', 'Bps') + - $.queryPanel( - 'sum by(%s, %s, device) (rate(node_disk_read_bytes_total[$__rate_interval])) + %s' % [$._config.per_node_label, $._config.per_instance_label, $.filterNodeDiskContainer('querier')], - '{{%s}} - {{device}}' % $._config.per_instance_label - ) + - $.withStacking, - ) - .addPanel( - $.containerDiskSpaceUtilizationPanel('Disk Space Utilization', 'querier'), - ) - ) - // Add the read path for single scalable deployment only. The read path should not display disk utilization as the index gateway is present in the backend pods. - .addRowIf( - $._config.ssd.enabled, - grafana.row.new('Read path') - .addPanel( - $.CPUUsagePanel('CPU', index_gateway_pod_matcher), - ) - .addPanel( - $.memoryWorkingSetPanel('Memory (workingset)', index_gateway_pod_matcher), - ) - .addPanel( - $.goHeapInUsePanel('Memory (go heap inuse)', index_gateway_job_matcher), - ) - ) - // Otherwise we add the index gateway information - .addRowIf( - !$._config.ssd.enabled, - grafana.row.new('Index Gateway') - .addPanel( - $.CPUUsagePanel('CPU', index_gateway_pod_matcher), - ) - .addPanel( - $.memoryWorkingSetPanel('Memory (workingset)', index_gateway_pod_matcher), - ) - .addPanel( - $.goHeapInUsePanel('Memory (go heap inuse)', index_gateway_job_matcher), - ) - .addPanel( - $.newQueryPanel('Disk Writes', 'Bps') + - $.queryPanel( - 'sum by(%s, %s, device) (rate(node_disk_written_bytes_total[$__rate_interval])) + %s' % [$._config.per_node_label, $._config.per_instance_label, $.filterNodeDisk(index_gateway_pod_matcher)], - '{{%s}} - {{device}}' % $._config.per_instance_label - ) + - $.withStacking, - ) - .addPanel( - $.newQueryPanel('Disk Reads', 'Bps') + - $.queryPanel( - 'sum by(%s, %s, device) (rate(node_disk_read_bytes_total[$__rate_interval])) + %s' % [$._config.per_node_label, $._config.per_instance_label, $.filterNodeDisk(index_gateway_pod_matcher)], - '{{%s}} - {{device}}' % $._config.per_instance_label - ) + - $.withStacking, - ) - .addPanel( - $.containerDiskSpaceUtilizationPanel('Disk Space Utilization', index_gateway_job_matcher), - ) - ) - .addRowIf( - !$._config.ssd.enabled, - grafana.row.new('Bloom Gateway') - .addPanel( - $.containerCPUUsagePanel('CPU', 'bloom-gateway'), - ) - .addPanel( - $.containerMemoryWorkingSetPanel('Memory (workingset)', 'bloom-gateway'), - ) - .addPanel( - $.goHeapInUsePanel('Memory (go heap inuse)', 'bloom-gateway'), - ) - .addPanel( - $.newQueryPanel('Disk Writes', 'Bps') + - $.queryPanel( - 'sum by(%s, %s, device) (rate(node_disk_written_bytes_total[$__rate_interval])) + %s' % [$._config.per_node_label, $._config.per_instance_label, $.filterNodeDiskContainer('bloom-gateway')], - '{{%s}} - {{device}}' % $._config.per_instance_label - ) + - $.withStacking, - ) - .addPanel( - $.newQueryPanel('Disk Reads', 'Bps') + - $.queryPanel( - 'sum by(%s, %s, device) (rate(node_disk_read_bytes_total[$__rate_interval])) + %s' % [$._config.per_node_label, $._config.per_instance_label, $.filterNodeDiskContainer('bloom-gateway')], - '{{%s}} - {{device}}' % $._config.per_instance_label - ) + - $.withStacking, - ) - .addPanel( - $.containerDiskSpaceUtilizationPanel('Disk Space Utilization', 'bloom-gateway'), - ) - ) - .addRow( - $.row('Ingester') - .addPanel( - $.CPUUsagePanel('CPU', ingester_pod_matcher), - ) - .addPanel( - $.memoryWorkingSetPanel('Memory (workingset)', ingester_pod_matcher), - ) - .addPanel( - $.goHeapInUsePanel('Memory (go heap inuse)', ingester_job_matcher), - ) - ) - .addRowIf( - !$._config.ssd.enabled, - grafana.row.new('Ruler') - .addPanel( - $.newQueryPanel('Rules') + - $.queryPanel( - 'sum by(%(label)s) (loki_prometheus_rule_group_rules{%(matcher)s}) or sum by(%(label)s) (cortex_prometheus_rule_group_rules{%(matcher)s})' % { label: $._config.per_instance_label, matcher: $.jobMatcher('ruler') }, - '{{%s}}' % $._config.per_instance_label - ), - ) - .addPanel( - $.containerCPUUsagePanel('CPU', 'ruler'), - ) - .addPanel( - $.containerMemoryWorkingSetPanel('Memory (workingset)', 'ruler'), - ) - .addPanel( - $.goHeapInUsePanel('Memory (go heap inuse)', 'ruler'), - ) - ), - }, + grafanaDashboards+:: if $._config.ssd.enabled then {} else { + 'loki-reads-resources.json': + ($.dashboard('Loki / Reads Resources', uid='reads-resources')) + .addCluster() + .addNamespace() + .addTag() + .addRowIf( + $._config.internal_components, + $.row('Gateway') + .addPanel( + $.containerCPUUsagePanel('CPU', 'cortex-gw(-internal)?'), + ) + .addPanel( + $.containerMemoryWorkingSetPanel('Memory (workingset)', 'cortex-gw(-internal)?'), + ) + .addPanel( + $.goHeapInUsePanel('Memory (go heap inuse)', 'cortex-gw(-internal)?'), + ) + ) + .addRow( + $.row('Query Frontend') + .addPanel( + $.containerCPUUsagePanel('CPU', 'query-frontend'), + ) + .addPanel( + $.containerMemoryWorkingSetPanel('Memory (workingset)', 'query-frontend'), + ) + .addPanel( + $.goHeapInUsePanel('Memory (go heap inuse)', 'query-frontend'), + ) + ) + .addRow( + $.row('Query Scheduler') + .addPanel( + $.containerCPUUsagePanel('CPU', 'query-scheduler'), + ) + .addPanel( + $.containerMemoryWorkingSetPanel('Memory (workingset)', 'query-scheduler'), + ) + .addPanel( + $.goHeapInUsePanel('Memory (go heap inuse)', 'query-scheduler'), + ) + ) + .addRow( + $.row('Querier') + .addPanel( + $.containerCPUUsagePanel('CPU', 'querier'), + ) + .addPanel( + $.containerMemoryWorkingSetPanel('Memory (workingset)', 'querier'), + ) + .addPanel( + $.goHeapInUsePanel('Memory (go heap inuse)', 'querier'), + ) + .addPanel( + $.newQueryPanel('Disk Writes', 'Bps') + + $.queryPanel( + 'sum by(%s, %s, device) (rate(node_disk_written_bytes_total[$__rate_interval])) + %s' % [$._config.per_node_label, $._config.per_instance_label, $.filterNodeDiskContainer('querier')], + '{{%s}} - {{device}}' % $._config.per_instance_label + ) + + $.withStacking, + ) + .addPanel( + $.newQueryPanel('Disk Reads', 'Bps') + + $.queryPanel( + 'sum by(%s, %s, device) (rate(node_disk_read_bytes_total[$__rate_interval])) + %s' % [$._config.per_node_label, $._config.per_instance_label, $.filterNodeDiskContainer('querier')], + '{{%s}} - {{device}}' % $._config.per_instance_label + ) + + $.withStacking, + ) + .addPanel( + $.containerDiskSpaceUtilizationPanel('Disk Space Utilization', 'querier'), + ) + ) + // Otherwise we add the index gateway information + .addRow( + $.row('Index Gateway') + .addPanel( + $.CPUUsagePanel('CPU', index_gateway_pod_matcher), + ) + .addPanel( + $.memoryWorkingSetPanel('Memory (workingset)', index_gateway_pod_matcher), + ) + .addPanel( + $.goHeapInUsePanel('Memory (go heap inuse)', index_gateway_job_matcher), + ) + .addPanel( + $.newQueryPanel('Disk Writes', 'Bps') + + $.queryPanel( + 'sum by(%s, %s, device) (rate(node_disk_written_bytes_total[$__rate_interval])) + %s' % [$._config.per_node_label, $._config.per_instance_label, $.filterNodeDisk(index_gateway_pod_matcher)], + '{{%s}} - {{device}}' % $._config.per_instance_label + ) + + $.withStacking, + ) + .addPanel( + $.newQueryPanel('Disk Reads', 'Bps') + + $.queryPanel( + 'sum by(%s, %s, device) (rate(node_disk_read_bytes_total[$__rate_interval])) + %s' % [$._config.per_node_label, $._config.per_instance_label, $.filterNodeDisk(index_gateway_pod_matcher)], + '{{%s}} - {{device}}' % $._config.per_instance_label + ) + + $.withStacking, + ) + .addPanel( + $.containerDiskSpaceUtilizationPanel('Disk Space Utilization', index_gateway_job_matcher), + ) + ) + .addRow( + $.row('Bloom Gateway') + .addPanel( + $.containerCPUUsagePanel('CPU', 'bloom-gateway'), + ) + .addPanel( + $.containerMemoryWorkingSetPanel('Memory (workingset)', 'bloom-gateway'), + ) + .addPanel( + $.goHeapInUsePanel('Memory (go heap inuse)', 'bloom-gateway'), + ) + .addPanel( + $.newQueryPanel('Disk Writes', 'Bps') + + $.queryPanel( + 'sum by(%s, %s, device) (rate(node_disk_written_bytes_total[$__rate_interval])) + %s' % [$._config.per_node_label, $._config.per_instance_label, $.filterNodeDiskContainer('bloom-gateway')], + '{{%s}} - {{device}}' % $._config.per_instance_label + ) + + $.withStacking, + ) + .addPanel( + $.newQueryPanel('Disk Reads', 'Bps') + + $.queryPanel( + 'sum by(%s, %s, device) (rate(node_disk_read_bytes_total[$__rate_interval])) + %s' % [$._config.per_node_label, $._config.per_instance_label, $.filterNodeDiskContainer('bloom-gateway')], + '{{%s}} - {{device}}' % $._config.per_instance_label + ) + + $.withStacking, + ) + .addPanel( + $.containerDiskSpaceUtilizationPanel('Disk Space Utilization', 'bloom-gateway'), + ) + ) + .addRow( + $.row('Ingester') + .addPanel( + $.CPUUsagePanel('CPU', ingester_pod_matcher), + ) + .addPanel( + $.memoryWorkingSetPanel('Memory (workingset)', ingester_pod_matcher), + ) + .addPanel( + $.goHeapInUsePanel('Memory (go heap inuse)', ingester_job_matcher), + ) + ) + .addRow( + $.row('Ruler') + .addPanel( + $.newQueryPanel('Rules') + + $.queryPanel( + 'sum by(%(label)s) (loki_prometheus_rule_group_rules{%(matcher)s}) or sum by(%(label)s) (cortex_prometheus_rule_group_rules{%(matcher)s})' % { label: $._config.per_instance_label, matcher: $.jobMatcher('ruler') }, + '{{%s}}' % $._config.per_instance_label + ), + ) + .addPanel( + $.containerCPUUsagePanel('CPU', 'ruler'), + ) + .addPanel( + $.containerMemoryWorkingSetPanel('Memory (workingset)', 'ruler'), + ) + .addPanel( + $.goHeapInUsePanel('Memory (go heap inuse)', 'ruler'), + ) + ), + }, } diff --git a/production/loki-mixin/dashboards/loki-resources-overview.libsonnet b/production/loki-mixin/dashboards/loki-resources-overview.libsonnet new file mode 100644 index 0000000000000..76c77a1453446 --- /dev/null +++ b/production/loki-mixin/dashboards/loki-resources-overview.libsonnet @@ -0,0 +1,107 @@ +(import 'dashboard-utils.libsonnet') { + local read_pod_matcher = 'container="loki", pod=~"%s-read.*"' % $._config.ssd.pod_prefix_matcher, + local read_job_matcher = '%s-read' % $._config.ssd.pod_prefix_matcher, + + local write_pod_matcher = 'container="loki", pod=~"%s-write.*"' % $._config.ssd.pod_prefix_matcher, + local write_job_matcher = '%s-write' % $._config.ssd.pod_prefix_matcher, + + local backend_pod_matcher = 'container="loki", pod=~"%s-backend.*"' % $._config.ssd.pod_prefix_matcher, + local backend_job_matcher = '%s-backend' % $._config.ssd.pod_prefix_matcher, + + // This dashboard is for the single scalable deployment only and it : + // - replaces the loki-reads-resources dashboards + // - replaces the loki-write-resources dashboards + // - adds backend pods resources + grafanaDashboards+:: if !$._config.ssd.enabled then {} else { + 'loki-resources-overview.json': + ($.dashboard('Loki / Resources Overview', uid='resources-overview')) + .addCluster() + .addNamespace() + .addTag() + .addRow( + // The read path does not display disk utilization as the index gateway is present in the backend pods. + $.row('Read path') + .addPanel( + $.CPUUsagePanel('CPU', read_pod_matcher), + ) + .addPanel( + $.memoryWorkingSetPanel('Memory (workingset)', read_pod_matcher), + ) + .addPanel( + $.goHeapInUsePanel('Memory (go heap inuse)', read_job_matcher), + ) + ) + .addRow( + $.row('Write path') + .addPanel( + $.newQueryPanel('In-memory streams') + + $.queryPanel( + 'sum by(%s) (loki_write_memory_streams{%s})' % [$._config.per_instance_label, $.jobMatcher(write_job_matcher)], + '{{%s}}' % $._config.per_instance_label + ) + + { + tooltip: { sort: 2 }, // Sort descending. + }, + ) + .addPanel( + $.CPUUsagePanel('CPU', write_pod_matcher), + ) + .addPanel( + $.memoryWorkingSetPanel('Memory (workingset)', write_pod_matcher), + ) + .addPanel( + $.goHeapInUsePanel('Memory (go heap inuse)', write_job_matcher), + ) + .addPanel( + $.newQueryPanel('Disk Writes', 'Bps') + + $.queryPanel( + 'sum by(%s, %s, device) (rate(node_disk_written_bytes_total[$__rate_interval])) + %s' % [$._config.per_node_label, $._config.per_instance_label, $.filterNodeDisk(write_pod_matcher)], + '{{%s}} - {{device}}' % $._config.per_instance_label + ) + + $.withStacking, + ) + .addPanel( + $.newQueryPanel('Disk Reads', 'Bps') + + $.queryPanel( + 'sum by(%s, %s, device) (rate(node_disk_read_bytes_total[$__rate_interval])) + %s' % [$._config.per_node_label, $._config.per_instance_label, $.filterNodeDisk(write_pod_matcher)], + '{{%s}} - {{device}}' % $._config.per_instance_label + ) + + $.withStacking, + ) + .addPanel( + $.containerDiskSpaceUtilizationPanel('Disk Space Utilization', write_job_matcher), + ) + ) + .addRow( + $.row('Backend path') + .addPanel( + $.CPUUsagePanel('CPU', backend_pod_matcher), + ) + .addPanel( + $.memoryWorkingSetPanel('Memory (workingset)', backend_pod_matcher), + ) + .addPanel( + $.goHeapInUsePanel('Memory (go heap inuse)', backend_job_matcher), + ) + .addPanel( + $.newQueryPanel('Disk Writes', 'Bps') + + $.queryPanel( + 'sum by(%s, %s, device) (rate(node_disk_written_bytes_total[$__rate_interval])) + %s' % [$._config.per_node_label, $._config.per_instance_label, $.filterNodeDisk(backend_pod_matcher)], + '{{%s}} - {{device}}' % $._config.per_instance_label + ) + + $.withStacking, + ) + .addPanel( + $.newQueryPanel('Disk Reads', 'Bps') + + $.queryPanel( + 'sum by(%s, %s, device) (rate(node_disk_read_bytes_total[$__rate_interval])) + %s' % [$._config.per_node_label, $._config.per_instance_label, $.filterNodeDisk(backend_pod_matcher)], + '{{%s}} - {{device}}' % $._config.per_instance_label + ) + + $.withStacking, + ) + .addPanel( + $.containerDiskSpaceUtilizationPanel('Disk Space Utilization', backend_job_matcher), + ) + ), + }, +} diff --git a/production/loki-mixin/dashboards/loki-retention.libsonnet b/production/loki-mixin/dashboards/loki-retention.libsonnet index c2e461c35f4c8..2a1c4777a2930 100644 --- a/production/loki-mixin/dashboards/loki-retention.libsonnet +++ b/production/loki-mixin/dashboards/loki-retention.libsonnet @@ -1,5 +1,3 @@ -local utils = import 'mixin-utils/utils.libsonnet'; - (import 'dashboard-utils.libsonnet') { local compactor_pod_matcher = if $._config.meta_monitoring.enabled then 'pod=~"(compactor.*|%s-backend.*|loki-single-binary)"' % $._config.ssd.pod_prefix_matcher diff --git a/production/loki-mixin/dashboards/loki-writes-resources.libsonnet b/production/loki-mixin/dashboards/loki-writes-resources.libsonnet index 1d4c693a9b9bd..c19b60606cd72 100644 --- a/production/loki-mixin/dashboards/loki-writes-resources.libsonnet +++ b/production/loki-mixin/dashboards/loki-writes-resources.libsonnet @@ -1,87 +1,82 @@ -local grafana = import 'grafonnet/grafana.libsonnet'; -local utils = import 'mixin-utils/utils.libsonnet'; - (import 'dashboard-utils.libsonnet') { local ingester_pod_matcher = if $._config.meta_monitoring.enabled - then 'container=~"loki|ingester", pod=~"(ingester.*|%s-write.*|loki-single-binary)"' % $._config.ssd.pod_prefix_matcher - else if $._config.ssd.enabled then 'container="loki", pod=~"%s-write.*"' % $._config.ssd.pod_prefix_matcher else 'container="ingester"', + then 'container=~"loki|ingester", pod=~"(ingester.*loki-single-binary)"' + else 'container="ingester"', local ingester_job_matcher = if $._config.meta_monitoring.enabled - then '(ingester.*|%s-write|loki-single-binary)' % $._config.ssd.pod_prefix_matcher - else if $._config.ssd.enabled then '%s-write' % $._config.ssd.pod_prefix_matcher else 'ingester.*', + then '(ingester.*|loki-single-binary)' + else 'ingester.*', - grafanaDashboards+:: - { - 'loki-writes-resources.json': - ($.dashboard('Loki / Writes Resources', uid='writes-resources')) - .addCluster() - .addNamespace() - .addTag() - .addRowIf( - $._config.internal_components, - $.row('Gateway') - .addPanel( - $.containerCPUUsagePanel('CPU', 'cortex-gw(-internal)?'), - ) - .addPanel( - $.containerMemoryWorkingSetPanel('Memory (workingset)', 'cortex-gw(-internal)?'), - ) - .addPanel( - $.goHeapInUsePanel('Memory (go heap inuse)', 'cortex-gw(-internal)?'), - ) + grafanaDashboards+:: if $._config.ssd.enabled then {} else { + 'loki-writes-resources.json': + ($.dashboard('Loki / Writes Resources', uid='writes-resources')) + .addCluster() + .addNamespace() + .addTag() + .addRowIf( + $._config.internal_components, + $.row('Gateway') + .addPanel( + $.containerCPUUsagePanel('CPU', 'cortex-gw(-internal)?'), + ) + .addPanel( + $.containerMemoryWorkingSetPanel('Memory (workingset)', 'cortex-gw(-internal)?'), + ) + .addPanel( + $.goHeapInUsePanel('Memory (go heap inuse)', 'cortex-gw(-internal)?'), + ) + ) + .addRow( + $.row('Distributor') + .addPanel( + $.containerCPUUsagePanel('CPU', 'distributor'), + ) + .addPanel( + $.containerMemoryWorkingSetPanel('Memory (workingset)', 'distributor'), + ) + .addPanel( + $.goHeapInUsePanel('Memory (go heap inuse)', 'distributor'), + ) + ) + .addRow( + $.row('Ingester') + .addPanel( + $.newQueryPanel('In-memory streams') + + $.queryPanel( + 'sum by(%s) (loki_ingester_memory_streams{%s})' % [$._config.per_instance_label, $.jobMatcher(ingester_job_matcher)], + '{{%s}}' % $._config.per_instance_label + ) + + { + tooltip: { sort: 2 }, // Sort descending. + }, + ) + .addPanel( + $.CPUUsagePanel('CPU', ingester_pod_matcher), + ) + .addPanel( + $.memoryWorkingSetPanel('Memory (workingset)', ingester_pod_matcher), + ) + .addPanel( + $.goHeapInUsePanel('Memory (go heap inuse)', ingester_job_matcher), + ) + .addPanel( + $.newQueryPanel('Disk Writes', 'Bps') + + $.queryPanel( + 'sum by(%s, %s, device) (rate(node_disk_written_bytes_total[$__rate_interval])) + %s' % [$._config.per_node_label, $._config.per_instance_label, $.filterNodeDisk(ingester_pod_matcher)], + '{{%s}} - {{device}}' % $._config.per_instance_label + ) + + $.withStacking, + ) + .addPanel( + $.newQueryPanel('Disk Reads', 'Bps') + + $.queryPanel( + 'sum by(%s, %s, device) (rate(node_disk_read_bytes_total[$__rate_interval])) + %s' % [$._config.per_node_label, $._config.per_instance_label, $.filterNodeDisk(ingester_pod_matcher)], + '{{%s}} - {{device}}' % $._config.per_instance_label + ) + + $.withStacking, ) - .addRowIf( - !$._config.ssd.enabled, - $.row('Distributor') - .addPanel( - $.containerCPUUsagePanel('CPU', 'distributor'), - ) - .addPanel( - $.containerMemoryWorkingSetPanel('Memory (workingset)', 'distributor'), - ) - .addPanel( - $.goHeapInUsePanel('Memory (go heap inuse)', 'distributor'), - ) + .addPanel( + $.containerDiskSpaceUtilizationPanel('Disk Space Utilization', ingester_job_matcher), ) - .addRow( - grafana.row.new(if $._config.ssd.enabled then 'Write path' else 'Ingester') - .addPanel( - $.newQueryPanel('In-memory streams') + - $.queryPanel( - 'sum by(%s) (loki_ingester_memory_streams{%s})' % [$._config.per_instance_label, $.jobMatcher(ingester_job_matcher)], - '{{%s}}' % $._config.per_instance_label - ) + - { - tooltip: { sort: 2 }, // Sort descending. - }, - ) - .addPanel( - $.CPUUsagePanel('CPU', ingester_pod_matcher), - ) - .addPanel( - $.memoryWorkingSetPanel('Memory (workingset)', ingester_pod_matcher), - ) - .addPanel( - $.goHeapInUsePanel('Memory (go heap inuse)', ingester_job_matcher), - ) - .addPanel( - $.newQueryPanel('Disk Writes', 'Bps') + - $.queryPanel( - 'sum by(%s, %s, device) (rate(node_disk_written_bytes_total[$__rate_interval])) + %s' % [$._config.per_node_label, $._config.per_instance_label, $.filterNodeDisk(ingester_pod_matcher)], - '{{%s}} - {{device}}' % $._config.per_instance_label - ) + - $.withStacking, - ) - .addPanel( - $.newQueryPanel('Disk Reads', 'Bps') + - $.queryPanel( - 'sum by(%s, %s, device) (rate(node_disk_read_bytes_total[$__rate_interval])) + %s' % [$._config.per_node_label, $._config.per_instance_label, $.filterNodeDisk(ingester_pod_matcher)], - '{{%s}} - {{device}}' % $._config.per_instance_label - ) + - $.withStacking, - ) - .addPanel( - $.containerDiskSpaceUtilizationPanel('Disk Space Utilization', ingester_job_matcher), - ) - ), - }, + ), + }, }