diff --git a/production/loki-mixin-compiled-ssd/dashboards/loki-resources-overview.json b/production/loki-mixin-compiled-ssd/dashboards/loki-resources-overview.json index 6e669361a057..72b6eaf785b4 100644 --- a/production/loki-mixin-compiled-ssd/dashboards/loki-resources-overview.json +++ b/production/loki-mixin-compiled-ssd/dashboards/loki-resources-overview.json @@ -325,7 +325,7 @@ "sort": "none" } }, - "span": 1, + "span": 3, "targets": [ { "expr": "sum by(pod) (loki_write_memory_streams{cluster=~\"$cluster\", job=~\"($namespace)/(loki.*|enterprise-logs)-write\"})", @@ -414,7 +414,7 @@ "sort": "none" } }, - "span": 1, + "span": 3, "targets": [ { "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki.*|enterprise-logs)-write.*\"}[$__rate_interval]))", @@ -515,7 +515,7 @@ "sort": "none" } }, - "span": 1, + "span": 3, "targets": [ { "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki.*|enterprise-logs)-write.*\"})", @@ -577,7 +577,7 @@ "sort": "none" } }, - "span": 1, + "span": 3, "targets": [ { "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/(loki.*|enterprise-logs)-write\"})", @@ -591,7 +591,19 @@ "sort": 2 }, "type": "timeseries" - }, + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Write path", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ { "datasource": "$datasource", "fieldConfig": { @@ -627,7 +639,7 @@ "sort": "none" } }, - "span": 1, + "span": 4, "targets": [ { "expr": "sum by(instance, pod, device) (rate(node_disk_written_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki.*|enterprise-logs)-write.*\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n", @@ -674,7 +686,7 @@ "sort": "none" } }, - "span": 1, + "span": 4, "targets": [ { "expr": "sum by(instance, pod, device) (rate(node_disk_read_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki.*|enterprise-logs)-write.*\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n", @@ -721,7 +733,7 @@ "sort": "none" } }, - "span": 1, + "span": 4, "targets": [ { "expr": "max by(persistentvolumeclaim) (kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", persistentvolumeclaim=~\".*(loki.*|enterprise-logs)-write.*\"} / kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", persistentvolumeclaim=~\".*(loki.*|enterprise-logs)-write.*\"})", @@ -738,7 +750,7 @@ "repeatIteration": null, "repeatRowId": null, "showTitle": true, - "title": "Write path", + "title": "", "titleSize": "h6" }, { @@ -819,7 +831,7 @@ "sort": "none" } }, - "span": 2, + "span": 4, "targets": [ { "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki.*|enterprise-logs)-backend.*\"}[$__rate_interval]))", @@ -920,7 +932,7 @@ "sort": "none" } }, - "span": 2, + "span": 4, "targets": [ { "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki.*|enterprise-logs)-backend.*\"})", @@ -982,7 +994,7 @@ "sort": "none" } }, - "span": 2, + "span": 4, "targets": [ { "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/(loki.*|enterprise-logs)-backend\"})", @@ -996,7 +1008,19 @@ "sort": 2 }, "type": "timeseries" - }, + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Backend path", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ { "datasource": "$datasource", "fieldConfig": { @@ -1032,7 +1056,7 @@ "sort": "none" } }, - "span": 2, + "span": 4, "targets": [ { "expr": "sum by(instance, pod, device) (rate(node_disk_written_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki.*|enterprise-logs)-backend.*\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n", @@ -1079,7 +1103,7 @@ "sort": "none" } }, - "span": 2, + "span": 4, "targets": [ { "expr": "sum by(instance, pod, device) (rate(node_disk_read_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki.*|enterprise-logs)-backend.*\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n", @@ -1126,7 +1150,7 @@ "sort": "none" } }, - "span": 2, + "span": 4, "targets": [ { "expr": "max by(persistentvolumeclaim) (kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", persistentvolumeclaim=~\".*(loki.*|enterprise-logs)-backend.*\"} / kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", persistentvolumeclaim=~\".*(loki.*|enterprise-logs)-backend.*\"})", @@ -1143,7 +1167,7 @@ "repeatIteration": null, "repeatRowId": null, "showTitle": true, - "title": "Backend path", + "title": "", "titleSize": "h6" } ], diff --git a/production/loki-mixin/dashboards/loki-resources-overview.libsonnet b/production/loki-mixin/dashboards/loki-resources-overview.libsonnet index 76c77a145344..a93df5d42e41 100644 --- a/production/loki-mixin/dashboards/loki-resources-overview.libsonnet +++ b/production/loki-mixin/dashboards/loki-resources-overview.libsonnet @@ -41,7 +41,7 @@ ) + { tooltip: { sort: 2 }, // Sort descending. - }, + } ) .addPanel( $.CPUUsagePanel('CPU', write_pod_matcher), @@ -52,6 +52,9 @@ .addPanel( $.goHeapInUsePanel('Memory (go heap inuse)', write_job_matcher), ) + ) + .addRow( + $.row('') .addPanel( $.newQueryPanel('Disk Writes', 'Bps') + $.queryPanel( @@ -83,6 +86,9 @@ .addPanel( $.goHeapInUsePanel('Memory (go heap inuse)', backend_job_matcher), ) + ) + .addRow( + $.row('') .addPanel( $.newQueryPanel('Disk Writes', 'Bps') + $.queryPanel(