From 1e1267ad5c18d7ece499d350d8d24bf11aa5bb8f Mon Sep 17 00:00:00 2001 From: eduardo aleixo Date: Wed, 25 Aug 2021 15:10:29 -0300 Subject: [PATCH] Benchmark dashboard (#349) * benchmark: generate new benchmark dashboard * benchmark: fix missing metrics * dashboard: update the grafana example to point to the jsonnet dash * clean up * benchmark: prepend metrics with _benchmark and use gauge for progress * benchmark: continue to display progress pane after execution is finished Co-authored-by: Dmitry Filimonov --- benchmark/README.md | 2 +- benchmark/docker-compose.yml | 1 + .../datasources/prometheus.yml | 2 +- benchmark/main.go | 40 +- benchmark/start.sh | 1 + .../grafana-integration/docker-compose.yml | 1 + monitoring/Makefile | 7 +- monitoring/README.md | 11 +- monitoring/benchmark.jsonnet | 8 + monitoring/config.libsonnet | 7 + monitoring/dashboard.jsonnet | 526 +--- monitoring/gen/benchmark-dashboard.json | 2431 +++++++++++++++++ monitoring/{ => gen}/dashboard.json | 12 +- monitoring/lib/dashboard.libsonnet | 598 ++++ 14 files changed, 3088 insertions(+), 559 deletions(-) create mode 100644 monitoring/benchmark.jsonnet create mode 100644 monitoring/config.libsonnet create mode 100644 monitoring/gen/benchmark-dashboard.json rename monitoring/{ => gen}/dashboard.json (99%) create mode 100644 monitoring/lib/dashboard.libsonnet diff --git a/benchmark/README.md b/benchmark/README.md index 2b56e915fc..725fbef520 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -32,6 +32,6 @@ Edit `run-parameters.env` file to change the parameters of the benchmark run. ## Browsing results -To view results open [http://localhost:8080/d/65gjqY3Mk/main?orgId=1](http://localhost:8080/d/65gjqY3Mk/main?orgId=1). +To view results open [http://localhost:8080/d/tsWRL6ReZQkirFirmyvnWX1akHXJeHT8I8emjGJo/main?orgId=1](http://localhost:8080/d/tsWRL6ReZQkirFirmyvnWX1akHXJeHT8I8emjGJo/main?orgId=1). You will also be able to see screenshots of the runs in `./runs` directory diff --git a/benchmark/docker-compose.yml b/benchmark/docker-compose.yml index c3c54e053f..16ddd153a7 100644 --- a/benchmark/docker-compose.yml +++ b/benchmark/docker-compose.yml @@ -40,6 +40,7 @@ services: volumes: - ./grafana-provisioning:/etc/grafana/provisioning - ./grafana/grafana.ini:/etc/grafana/grafana.ini + - ../monitoring/gen/benchmark-dashboard.json:/etc/grafana/provisioning/dashboards/benchmark-dashboard.json ports: - 8080:3000 diff --git a/benchmark/grafana-provisioning/datasources/prometheus.yml b/benchmark/grafana-provisioning/datasources/prometheus.yml index c856c6d715..90770e32c5 100644 --- a/benchmark/grafana-provisioning/datasources/prometheus.yml +++ b/benchmark/grafana-provisioning/datasources/prometheus.yml @@ -1,6 +1,6 @@ --- datasources: - - name: Prometheus + - name: prometheus # datasource type. Required type: prometheus # access mode. proxy or direct (Server or Browser in the UI). Required diff --git a/benchmark/main.go b/benchmark/main.go index 1eb7bedecf..05ffc321eb 100644 --- a/benchmark/main.go +++ b/benchmark/main.go @@ -51,7 +51,7 @@ func waitUntilEndpointReady(url string) { } } -func startClientThread(appName string, wg *sync.WaitGroup, appFixtures []*transporttrie.Trie, runProgress prometheus.Gauge) { +func startClientThread(appName string, wg *sync.WaitGroup, appFixtures []*transporttrie.Trie, runProgress prometheus.Gauge, successfulUploads prometheus.Counter, uploadErrors prometheus.Counter) { rc := remote.RemoteConfig{ UpstreamThreads: 1, UpstreamAddress: "http://pyroscope:4040", @@ -69,17 +69,6 @@ func startClientThread(appName string, wg *sync.WaitGroup, appFixtures []*transp st := threadStartTime - reg := prometheus.NewRegistry() - - uploadErrors := promauto.With(reg).NewCounter(prometheus.CounterOpts{ - Name: "pyroscope_upload_errors", - Help: "", - }) - successfulUploads := promauto.With(reg).NewCounter(prometheus.CounterOpts{ - Name: "pyroscope_successful_uploads", - Help: "", - }) - for i := 0; i < requestsCount; i++ { t := appFixtures[i%len(appFixtures)] @@ -205,16 +194,25 @@ func main() { logrus.Info("waiting for other services to load") - benchmark := promauto.NewGauge(prometheus.GaugeOpts{ - Name: "pyroscope_benchmark", - Help: "", - }) runProgress := promauto.NewGauge(prometheus.GaugeOpts{ - Name: "pyroscope_run_progress", - Help: "", + Namespace: "pyroscope", + Subsystem: "benchmark", + Name: "progress", + Help: "", }) - benchmark.Set(0) + uploadErrors := promauto.NewCounter(prometheus.CounterOpts{ + Namespace: "pyroscope", + Subsystem: "benchmark", + Name: "upload_errors", + Help: "", + }) + successfulUploads := promauto.NewCounter(prometheus.CounterOpts{ + Namespace: "pyroscope", + Subsystem: "benchmark", + Name: "successful_uploads", + Help: "", + }) waitUntilEndpointReady("pyroscope:4040") waitUntilEndpointReady("prometheus:9090") @@ -239,7 +237,6 @@ func main() { logrus.Info("done generating fixtures") logrus.Info("starting sending requests") - benchmark.Set(1) startTime := time.Now() reportSummaryMetric("start-time", startTime.Format(timeFmt)) wg := sync.WaitGroup{} @@ -249,12 +246,11 @@ func main() { for i := 0; i < appsCount; i++ { r.Read(appNameBuf) for j := 0; j < clientsCount; j++ { - go startClientThread(hex.EncodeToString(appNameBuf), &wg, fixtures[i], runProgress) + go startClientThread(hex.EncodeToString(appNameBuf), &wg, fixtures[i], runProgress, successfulUploads, uploadErrors) } } wg.Wait() logrus.Info("done sending requests") - benchmark.Set(0) reportSummaryMetric("stop-time", time.Now().Format(timeFmt)) reportSummaryMetric("duration", time.Since(startTime).String()) diff --git a/benchmark/start.sh b/benchmark/start.sh index 5724048515..ab3d873dc8 100755 --- a/benchmark/start.sh +++ b/benchmark/start.sh @@ -8,6 +8,7 @@ echo "building containers..." source ./run-parameters.env export PYROSCOPE_CPUS PYROSCOPE_MEMORY +export DOCKER_BUILDKIT=1 docker-compose build diff --git a/examples/grafana-integration/docker-compose.yml b/examples/grafana-integration/docker-compose.yml index b929b55129..27251ccf9c 100644 --- a/examples/grafana-integration/docker-compose.yml +++ b/examples/grafana-integration/docker-compose.yml @@ -10,6 +10,7 @@ services: image: grafana/grafana:8.1.1 volumes: - ./grafana-provisioning:/etc/grafana/provisioning + - ../../monitoring/gen/dashboard.json:/etc/grafana/provisioning/dashboards/dashboard.json - ./grafana/grafana.ini:/etc/grafana/grafana.ini - ./grafana/home.json:/default-dashboard.json environment: diff --git a/monitoring/Makefile b/monitoring/Makefile index 5c0e34d34b..f357bf8764 100644 --- a/monitoring/Makefile +++ b/monitoring/Makefile @@ -1,5 +1,10 @@ +all: dashboard benchmark-dashboard + dashboard: - jsonnet -J vendor dashboard.jsonnet | tee dashboard.json + jsonnet -J vendor dashboard.jsonnet | tee gen/dashboard.json + +benchmark-dashboard: + jsonnet -J vendor benchmark.jsonnet | tee gen/benchmark-dashboard.json .PHONY: init init: diff --git a/monitoring/README.md b/monitoring/README.md index 532b33e934..d087d5a8dc 100644 --- a/monitoring/README.md +++ b/monitoring/README.md @@ -7,9 +7,9 @@ make init ``` -3. (Re)Generate the dashboard +3. (Re)Generate the dashboards ``` -make dashboard +make ``` # Development @@ -18,11 +18,12 @@ make dashboard Run the [grafana-integration](../examples/grafana-integration) example docker-compose then copy the generated dashboard there: ``` -make dashboard && \ - cp dashboard.json ../examples/grafana-integration/grafana-provisioning/dashboards/ && \ - docker-compose -f ../examples/grafana-integration/docker-compose.yml up -d --force-recreate grafana +make && docker-compose -f ../examples/grafana-integration/docker-compose.yml up -d --force-recreate grafana ``` +# Warnings +* If you ever rename the dashboard path, don't forget to update the references (see all the docker-compose.yaml) + # References * https://grafana.github.io/grafonnet-lib/api-docs/ diff --git a/monitoring/benchmark.jsonnet b/monitoring/benchmark.jsonnet new file mode 100644 index 0000000000..5afb7f398c --- /dev/null +++ b/monitoring/benchmark.jsonnet @@ -0,0 +1,8 @@ +local config = import 'config.libsonnet'; +local dashboard = import './lib/dashboard.libsonnet'; + +(config + dashboard + { + _config+:: { + benchmark: true, + } +}).dashboard diff --git a/monitoring/config.libsonnet b/monitoring/config.libsonnet new file mode 100644 index 0000000000..1924345fa2 --- /dev/null +++ b/monitoring/config.libsonnet @@ -0,0 +1,7 @@ +{ + _config+:: { + selector: 'instance="$instance"', + // whether to add additional benchmark fields or not + benchmark: false, + } +} diff --git a/monitoring/dashboard.jsonnet b/monitoring/dashboard.jsonnet index 6c82e1e63e..4742cd4128 100644 --- a/monitoring/dashboard.jsonnet +++ b/monitoring/dashboard.jsonnet @@ -1,524 +1,4 @@ -local grafana = import 'grafonnet/grafana.libsonnet'; +local config = import 'config.libsonnet'; +local dashboard = import './lib/dashboard.libsonnet'; -grafana.dashboard.new( - 'Pyroscope Server', - tags=['pyroscope'], - time_from='now-1h', - uid='tsWRL6ReZQkirFirmyvnWX1akHXJeHT8I8emjGJo', -// editable='true', -) - -.addTemplate( - grafana.template.datasource( - 'PROMETHEUS_DS', - 'prometheus', - '', - hide='hidden', // anything other than '' and 'label works - ) -) -.addTemplate( - grafana.template.new( - 'instance', - '$PROMETHEUS_DS', - 'label_values(pyroscope_build_info, instance)', - label='instance', - ) -) - -.addRow( - grafana.row.new( - title='Meta', - ) - .addPanel( - grafana.tablePanel.new( - title='', - datasource='$PROMETHEUS_DS', - span=12, - height=10, - ) - // they don't provide any value - .hideColumn("__name__") - .hideColumn("Time") - .hideColumn("instance") - .hideColumn("Value") - .hideColumn("job") - - // somewhat useful but preferred to be hidden - // to make the table cleaner - .hideColumn("use_embedded_assets") - .addTarget( - grafana.prometheus.target( - 'pyroscope_build_info{instance="$instance"}', - instant=true, - format='table', - ) - ) - ) -) -.addRow( - grafana.row.new( - title='General', - ) - .addPanel( - grafana.graphPanel.new( - 'Request Latency P99', - datasource='$PROMETHEUS_DS', - format='seconds', - ) - .addTarget(grafana.prometheus.target(||| - histogram_quantile(0.99, - sum(rate(pyroscope_http_request_duration_seconds_bucket{ - instance="$instance", - handler!="/metrics", - handler!="/healthz" - }[$__rate_interval])) - by (le, handler) - ) - |||, - legendFormat='{{ handler }}', - )) - ) - - .addPanel( - grafana.graphPanel.new( - 'Error Rate', - datasource='$PROMETHEUS_DS', - ) - .addTarget(grafana.prometheus.target(||| - sum(rate(pyroscope_http_request_duration_seconds_count - {instance="$instance", code=~"5..", handler!="/metrics", handler!="/healthz"}[$__rate_interval])) by (handler) - / - sum(rate(pyroscope_http_request_duration_seconds_count{instance="$instance", handler!="/metrics", handler!="/healthz"}[$__rate_interval])) by (handler) - |||, - legendFormat='{{ handler }}', - )) - ) - - .addPanel( - grafana.graphPanel.new( - 'Throughput', - datasource='$PROMETHEUS_DS', - ) - .addTarget(grafana.prometheus.target('sum(rate(pyroscope_http_request_duration_seconds_count{instance="$instance", handler!="/metrics", handler!="/healthz"}[$__rate_interval])) by (handler)', - legendFormat='{{ handler }}', - )) - ) - - .addPanel( - grafana.graphPanel.new( - 'Response Size P99', - datasource='$PROMETHEUS_DS', - ) - .addTarget(grafana.prometheus.target('histogram_quantile(0.95, sum(rate(pyroscope_http_response_size_bytes_bucket{instance="$instance", handler!="/metrics", handler!="/healthz"}[$__rate_interval])) by (le, handler))', - legendFormat='{{ handler }}', - )) - ) - - .addPanel( - grafana.graphPanel.new( - 'CPU Utilization', - datasource='$PROMETHEUS_DS', - format='percent', - min='0', - max='100', - legend_show=false, - ) - .addTarget( - grafana.prometheus.target( - 'pyroscope_cpu_utilization{instance="$instance"}', - ) - ) - ) - -) - - -.addRow( - grafana.row.new( - title='Storage', - ) - .addPanel( - grafana.graphPanel.new( - 'Cache Hit/Misses', - datasource='$PROMETHEUS_DS', - legend_values='true', - legend_rightSide='true', - legend_alignAsTable='true', - legend_current='true', - legend_sort='current', - legend_sortDesc=true, - format='percentunit', - ) - .addTarget( - grafana.prometheus.target(||| - sum without(name) (rate(pyroscope_storage_cache_hits_total[$__rate_interval])) - / - sum without(name) (rate(pyroscope_storage_cache_reads_total[$__rate_interval])) - |||, - legendFormat='Hits', - ) - ) - .addTarget( - grafana.prometheus.target(||| - sum without(name) (rate(pyroscope_storage_cache_misses_total[$__rate_interval])) - / - sum without(name)(rate(pyroscope_storage_cache_reads_total[$__rate_interval])) - |||, - legendFormat='Misses', - ) - ) - ) - .addPanel( - grafana.graphPanel.new( - 'Cache Hit Ratio', - datasource='$PROMETHEUS_DS', - legend_values='true', - legend_rightSide='true', - legend_alignAsTable='true', - legend_current='true', - legend_sort='current', - legend_sortDesc=true, - format='percentunit', - ) - .addTarget( - grafana.prometheus.target(||| - rate(pyroscope_storage_cache_hits_total[$__rate_interval]) - / - rate(pyroscope_storage_cache_reads_total[$__rate_interval]) - |||, - legendFormat='{{ name }}', - ) - ) - ) - .addPanel( - grafana.graphPanel.new( - 'Rate of items persisted from cache to disk', - datasource='$PROMETHEUS_DS', - legend_values='true', - legend_rightSide='true', - legend_alignAsTable='true', - legend_current='true', - legend_sort='current', - legend_sortDesc=true, - ) - .addTarget( - grafana.prometheus.target( - // ignore the name - 'sum without(name) (rate(pyroscope_storage_cache_persisted_total[$__rate_interval]))', - legendFormat="Total", - ) - ) - .addTarget( - grafana.prometheus.target( - '(rate(pyroscope_storage_cache_persisted_total[$__rate_interval]))', - legendFormat="{{ name }}", - ) - ) - ) - - .addPanel( - grafana.graphPanel.new( - 'Storage Reads/Writes', - datasource='$PROMETHEUS_DS', - ) - .addTarget( - grafana.prometheus.target( - 'rate(pyroscope_storage_reads_total[$__rate_interval])', - legendFormat="Reads", - ) - ) - .addTarget( - grafana.prometheus.target( - 'rate(pyroscope_storage_writes_total[$__rate_interval])', - legendFormat="Writes", - ) - ) - ) - .addPanel( - grafana.graphPanel.new( - 'Periodic tasks', - datasource='$PROMETHEUS_DS', - legend_values='true', - format='seconds', - min='0.001', // as as the bucket minimum - max='10', // same as the maximum bucket - logBase1Y=2, - ) - .addTarget( - grafana.prometheus.target( - 'histogram_quantile(0.9, pyroscope_storage_evictions_duration_seconds_bucket)', - legendFormat='evictions', - ), - ) - .addTarget( - grafana.prometheus.target( - 'histogram_quantile(0.9, pyroscope_storage_writeback_duration_seconds_bucket)', - legendFormat='write-back', - ), - ) - .addTarget( - grafana.prometheus.target( - 'histogram_quantile(0.9, pyroscope_storage_retention_duration_seconds_bucket)', - legendFormat='retention', - ), - ) - ) - - .addPanel( - grafana.graphPanel.new( - 'Disk Usage', - datasource='$PROMETHEUS_DS', - format='bytes', - legend_values='true', - legend_rightSide='true', - legend_alignAsTable='true', - legend_current='true', - legend_sort='current', - legend_sortDesc=true, - ) - .addTarget( - grafana.prometheus.target( - 'pyroscope_storage_disk_bytes', - legendFormat='{{ name }}', - ), - ) - .addTarget( - grafana.prometheus.target( - 'sum without(name)(pyroscope_disk_bytes)', - legendFormat='total', - ), - ) - ) - - .addPanel( - grafana.graphPanel.new( - 'Cache Size (number of items)', - datasource='$PROMETHEUS_DS', - legend_values='true', - legend_rightSide='true', - legend_alignAsTable='true', - legend_current='true', - legend_sort='current', - legend_sortDesc=true, - ) - .addTarget( - grafana.prometheus.target( - 'pyroscope_storage_cache_size', - legendFormat='{{ name }}', - ), - ) - .addTarget( - grafana.prometheus.target( - 'sum without(name)(pyroscope_storage_cache_size)', - legendFormat='total', - ), - ) - ) - - .addPanel( - grafana.graphPanel.new( - 'Cache Size (Approximation)', - datasource='$PROMETHEUS_DS', - format='bytes', - legend_values='true', - legend_rightSide='true', - legend_alignAsTable='true', - legend_current=true, - legend_max=true, - legend_sort='current', - legend_sortDesc=true, - logBase1Y=2, - ) - .addTarget( - grafana.prometheus.target( - 'pyroscope_storage_evictions_alloc_bytes', - legendFormat='heap size', - ), - ) - .addTarget( - grafana.prometheus.target( - 'pyroscope_storage_evictions_total_mem_bytes', - legendFormat='total memory', - ), - ) - ) -) - -// inspired by -// https://github.com/aukhatov/grafana-dashboards/blob/master/Go%20Metrics-1567509764849.json -.addRow( - grafana.row.new( - title='Go', - collapse=true, - ) - .addPanel( - grafana.graphPanel.new( - 'Memory Off-heap', - datasource='$PROMETHEUS_DS', - format='bytes', - ) - .addTarget( - grafana.prometheus.target( - 'go_memstats_mspan_inuse_bytes{instance="$instance"}', - legendFormat='{{ __name__ }}', - ) - ) - .addTarget( - grafana.prometheus.target( - 'go_memstats_mspan_sys_bytes{instance="$instance"}', - legendFormat='{{ __name__ }}', - ) - ) - .addTarget(grafana.prometheus.target( - 'go_memstats_mcache_inuse_bytes{instance="$instance"}', - legendFormat='{{ __name__ }}', - )) - .addTarget(grafana.prometheus.target( - 'go_memstats_mcache_sys_bytes{instance="$instance"}', - legendFormat='{{ __name__ }}', - )) - .addTarget(grafana.prometheus.target( - 'go_memstats_buck_hash_sys_bytes{instance="$instance"}', - legendFormat='{{ __name__ }}', - )) - .addTarget(grafana.prometheus.target( - 'go_memstats_gc_sys_bytes{instance="$instance"}', - legendFormat='{{ __name__ }}', - )) - .addTarget(grafana.prometheus.target( - 'go_memstats_other_sys_bytes{instance="$instance"}', - legendFormat='{{ __name__ }}', - )) - .addTarget(grafana.prometheus.target( - 'go_memstats_next_gc_bytes{instance="$instance"}', - legendFormat='{{ __name__ }}', - )) - ) - - .addPanel( - grafana.graphPanel.new( - 'Memory In Heap', - datasource='$PROMETHEUS_DS', - format='bytes', - ) - .addTarget(grafana.prometheus.target( - 'go_memstats_heap_alloc_bytes{instance="$instance"}', - legendFormat='{{ __name__ }}', - )) - .addTarget(grafana.prometheus.target( - 'go_memstats_heap_sys_bytes{instance="$instance"}', - legendFormat='{{ __name__ }}', - )) - .addTarget(grafana.prometheus.target( - 'go_memstats_heap_idle_bytes{instance="$instance"}', - legendFormat='{{ __name__ }}', - )) - .addTarget(grafana.prometheus.target( - 'go_memstats_heap_inuse_bytes{instance="$instance"}', - legendFormat='{{ __name__ }}', - )) - .addTarget(grafana.prometheus.target( - 'go_memstats_heap_released_bytes{instance="$instance"}', - legendFormat='{{ __name__ }}', - )) - ) - - - .addPanel( - grafana.graphPanel.new( - 'Memory In Stack', - datasource='$PROMETHEUS_DS', - format='decbytes', - ) - .addTarget( - grafana.prometheus.target( - 'go_memstats_stack_inuse_bytes{instance="$instance"}', - legendFormat='{{ __name__ }}', - ) - ) - .addTarget( - grafana.prometheus.target( - 'go_memstats_stack_sys_bytes{instance="$instance"}', - legendFormat='{{ __name__ }}', - ) - ) - ) - - - - .addPanel( - grafana.graphPanel.new( - 'Total Used Memory', - datasource='$PROMETHEUS_DS', - format='decbytes', - ) - .addTarget(grafana.prometheus.target( - 'go_memstats_sys_bytes{instance="$instance"}', - legendFormat='{{ __name__ }}', - )) - ) - - - .addPanel( - grafana.graphPanel.new( - 'Number of Live Objects', - datasource='$PROMETHEUS_DS', - legend_show=false, - ) - .addTarget(grafana.prometheus.target( - 'go_memstats_mallocs_total{instance="$instance"} - go_memstats_frees_total{instance="$instance"}' - )) - ) - - .addPanel( - grafana.graphPanel.new( - 'Rate of Objects Allocated', - datasource='$PROMETHEUS_DS', - legend_show=false, - ) - .addTarget(grafana.prometheus.target('rate(go_memstats_mallocs_total{instance="$instance"}[$__rate_interval])')) - ) - - .addPanel( - grafana.graphPanel.new( - 'Rates of Allocation', - datasource='$PROMETHEUS_DS', - format="Bps", - legend_show=false, - ) - .addTarget(grafana.prometheus.target('rate(go_memstats_alloc_bytes_total{instance="$instance"}[$__rate_interval])')) - ) - - .addPanel( - grafana.graphPanel.new( - 'Goroutines', - datasource='$PROMETHEUS_DS', - legend_show=false, - ) - .addTarget(grafana.prometheus.target('go_goroutines{instance="$instance"}')) - ) - - .addPanel( - grafana.graphPanel.new( - 'GC duration quantile', - datasource='$PROMETHEUS_DS', - legend_show=false, - ) - .addTarget(grafana.prometheus.target('go_gc_duration_seconds{instance="$instance"}')) - ) - - .addPanel( - grafana.graphPanel.new( - 'File descriptors', - datasource='$PROMETHEUS_DS', - ) - .addTarget(grafana.prometheus.target( - 'process_open_fds{instance="$instance"}', - legendFormat='{{ __name__ }}', - )) - .addTarget(grafana.prometheus.target( - 'process_max_fds{instance="$instance"}', - legendFormat='{{ __name__ }}', - )) - ) -) +(config + dashboard).dashboard diff --git a/monitoring/gen/benchmark-dashboard.json b/monitoring/gen/benchmark-dashboard.json new file mode 100644 index 0000000000..7e2bb9f172 --- /dev/null +++ b/monitoring/gen/benchmark-dashboard.json @@ -0,0 +1,2431 @@ +{ + "__inputs": [ ], + "__requires": [ ], + "annotations": { + "list": [ ] + }, + "editable": "true", + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ ], + "refresh": "5s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "content": "