Fix/dashboard (#227)

* update dashboard workflow

* run on workflow_dispatch

* trigger on PR to dev for debugging

* update build push action

* update context && file

* remove trigger on PR

* add sample data provider

* Update dashboard, finalize data provider

* Add extractors for requests and responses as default instrumentors

* add 'get_last_request', 'get_last_response'; add charts for requests and responses; update test

* Update sections in superset_guide; update extractor functions tutor

* doc build fix: Change functions to func names in __all__

* Apply suggestions by RLKRO from code review

Co-authored-by: Roman Zlobin <trlkrot@gmail.com>

* intermediary update; update data provider tutor; update charts;

* Apply suggestions from code review by RLKRO

Co-authored-by: Roman Zlobin <trlkrot@gmail.com>

* Upload annotations.png via GUI

* Update requests & responses charts; update superset_guide

* expand the Superset guide with chart creation instructions;Add links to Superset documentation

* update graph plot; update dataset to get flow lag; update data provider

* update guide; move sample data collector to /utils

* update links to avoid sphinx warnings

* update dashboard description; update tutorials

* update superset guide

* update label extractor; add infile parameter to main; remove link to sample_data_provider

* update tests

* update format

* minor doc improvements

* update dashboard datasource config

* add dff_stats dataset; link charts to dff_stats dataset

* update tests and documentation to reflect config changes

* update formatting

* add the test configuration file

* move get_current_label to after_handler in sample_data_provider.py

* update sql with if statements

* specify healthcheck for clickhouse and wait for healthcheck with otlp; make a variable for superset metadata db; adjust dockerfile to use configuration overrides

* add dashboard-metadata docker image; adjust to run on port 5433

* Update .github/workflows/update_dashboard.yml

Apply suggestions from @RLKRo

Co-authored-by: Roman Zlobin <trlkrot@gmail.com>

* Update dff/stats/cli.py

Apply suggestions to label_lag/flow_lag

Co-authored-by: Roman Zlobin <trlkrot@gmail.com>

* remove query statements from dashboard files; remove filter values from charts; update docker-compose healthcheck; apply formatting

* wait for pg database; sleep to avoid process conflict

* Apply suggestions by @RLKRo

Apply suggestions by @RLKRo

Co-authored-by: Roman Zlobin <trlkrot@gmail.com>

* Update config files; Update sql expressions in CLI; update healthcheck expression; create volume for dashboard metadata

* convert request_id to int in final_nodes

* remove sql in final_nodes

* update history filter to numerical range type & change datasource to node_stats

* remove utils include from manifest

---------

Co-authored-by: Roman Zlobin <trlkrot@gmail.com>

.env_file

@@ -19,4 +19,5 @@ CLICKHOUSE_USER=username
 CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT=1
 CLICKHOUSE_PASSWORD=pass
 SUPERSET_USERNAME=superset
-SUPERSET_PASSWORD=superset
+SUPERSET_PASSWORD=superset
+SUPERSET_METADATA_PORT=5433

.github/workflows/update_dashboard.yml

@@ -5,8 +5,8 @@ on:
     branches:
       - 'master'
     paths:
-      - 'dff/utils/docker/dockerfile_stats'
-      - 'dff/utils/docker/entrypoint_stats.sh'
+      - 'dff/utils/docker/**'
+  workflow_dispatch:
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}

dff/config/superset_dashboard/charts/Current_topic_slot_bar_chart_4.yaml

@@ -0,0 +1,72 @@
+slice_name: Current topic slot [bar chart]
+description: null
+certified_by: null
+certification_details: null
+viz_type: dist_bar
+params:
+  datasource: 3__table
+  viz_type: dist_bar
+  slice_id: 6
+  granularity_sqla: start_time
+  time_range: No filter
+  metrics:
+  - count
+  adhoc_filters:
+  - clause: WHERE
+    comparator: get_slots
+    datasourceWarning: false
+    expressionType: SIMPLE
+    filterOptionName: filter_sz14lhn7d1d_c0zqn5dpgk
+    isExtra: false
+    isNew: false
+    operator: ==
+    operatorId: EQUALS
+    sqlExpression: null
+    subject: data_key
+  - clause: WHERE
+    comparator: null
+    datasourceWarning: false
+    expressionType: SQL
+    filterOptionName: filter_945dhn41x2m_sci7gkxy7o
+    isExtra: false
+    isNew: false
+    operator: null
+    sqlExpression: JSON_VALUE(data, '$.current_topic') <> ''
+    subject: null
+  groupby:
+  - request_id
+  columns:
+  - expressionType: SQL
+    label: My column
+    sqlExpression: JSON_VALUE(data, '$.current_topic')
+    datasourceWarning: false
+  row_limit: 10000
+  order_desc: true
+  color_scheme: echarts4Colors
+  show_legend: true
+  rich_tooltip: true
+  bar_stacked: true
+  order_bars: false
+  y_axis_format: SMART_NUMBER
+  y_axis_label: Topic counts
+  y_axis_bounds:
+  - null
+  - null
+  x_axis_label: Dialog turn
+  bottom_margin: auto
+  x_ticks_layout: auto
+  extra_form_data: {}
+  dashboards:
+  - 1
+query_context: '{"datasource":{"id":3,"type":"table"},"force":false,"queries":[{"time_range":"No
+  filter","granularity":"start_time","filters":[{"col":"data_key","op":"==","val":"get_slots"}],"extras":{"having":"","where":"(JSON_VALUE(data,
+  ''$.current_topic'') <> '''')"},"applied_time_extras":{},"columns":["request_id",{"expressionType":"SQL","label":"My
+  column","sqlExpression":"JSON_VALUE(data, ''$.current_topic'')","datasourceWarning":false}],"metrics":["count"],"annotation_layers":[],"row_limit":10000,"series_limit":0,"order_desc":true,"url_params":{},"custom_params":{},"custom_form_data":{}}],"form_data":{"datasource":"3__table","viz_type":"dist_bar","slice_id":6,"granularity_sqla":"start_time","time_range":"No
+  filter","metrics":["count"],"adhoc_filters":[{"clause":"WHERE","comparator":"get_slots","datasourceWarning":false,"expressionType":"SIMPLE","filterOptionName":"filter_sz14lhn7d1d_c0zqn5dpgk","isExtra":false,"isNew":false,"operator":"==","operatorId":"EQUALS","sqlExpression":null,"subject":"data_key"},{"clause":"WHERE","comparator":null,"datasourceWarning":false,"expressionType":"SQL","filterOptionName":"filter_945dhn41x2m_sci7gkxy7o","isExtra":false,"isNew":false,"operator":null,"sqlExpression":"JSON_VALUE(data,
+  ''$.current_topic'') <> ''''","subject":null}],"groupby":["request_id"],"columns":[{"expressionType":"SQL","label":"My
+  column","sqlExpression":"JSON_VALUE(data, ''$.current_topic'')","datasourceWarning":false}],"row_limit":10000,"order_desc":true,"color_scheme":"echarts4Colors","show_legend":true,"rich_tooltip":true,"bar_stacked":true,"order_bars":false,"y_axis_format":"SMART_NUMBER","y_axis_label":"Topic
+  counts","y_axis_bounds":[null,null],"x_axis_label":"Dialog turn","bottom_margin":"auto","x_ticks_layout":"auto","extra_form_data":{},"dashboards":[1],"force":false,"result_format":"json","result_type":"full"},"result_format":"json","result_type":"full"}'
+cache_timeout: null
+uuid: a70c05d0-770b-4068-a55d-934283f5b1bb
+version: 1.0.0
+dataset_uuid: 8ba2e188-2bf8-4809-a5ee-2477a539d493

dff/config/superset_dashboard/charts/Current_topic_time_series_bar_chart_2.yaml

@@ -0,0 +1,86 @@
+slice_name: Current topic [time series bar chart]
+description: null
+certified_by: null
+certification_details: null
+viz_type: echarts_timeseries_bar
+params:
+  datasource: 3__table
+  viz_type: echarts_timeseries_bar
+  slice_id: 10
+  granularity_sqla: start_time
+  time_grain_sqla: PT1M
+  time_range: No filter
+  metrics:
+  - count
+  groupby:
+  - datasourceWarning: false
+    expressionType: SQL
+    label: context_id
+    sqlExpression: JSON_VALUE(data, '$.current_topic')
+  adhoc_filters:
+  - clause: WHERE
+    comparator: get_slots
+    datasourceWarning: false
+    expressionType: SIMPLE
+    filterOptionName: filter_8tft5fr07ea_urtdezftgn
+    isExtra: false
+    isNew: false
+    operator: ==
+    operatorId: EQUALS
+    sqlExpression: null
+    subject: data_key
+  - clause: WHERE
+    comparator: null
+    datasourceWarning: false
+    expressionType: SQL
+    filterOptionName: filter_4ffdpny1zzm_vmlo11yw7i
+    isExtra: false
+    isNew: false
+    operator: null
+    sqlExpression: JSON_VALUE(data, '$.current_topic') <> ''
+    subject: null
+  order_desc: true
+  row_limit: 10000
+  truncate_metric: true
+  show_empty_columns: true
+  comparison_type: values
+  annotation_layers: []
+  forecastPeriods: 10
+  forecastInterval: 0.8
+  orientation: vertical
+  x_axis_title: Time axis
+  x_axis_title_margin: 30
+  y_axis_title: Value counts over time
+  y_axis_title_margin: 30
+  y_axis_title_position: Left
+  color_scheme: echarts4Colors
+  show_value: false
+  stack: true
+  only_total: true
+  show_legend: true
+  legendType: scroll
+  legendOrientation: top
+  x_axis_time_format: smart_date
+  y_axis_format: SMART_NUMBER
+  y_axis_bounds:
+  - null
+  - null
+  rich_tooltip: true
+  tooltipSortByMetric: true
+  tooltipTimeFormat: smart_date
+  extra_form_data: {}
+  dashboards:
+  - 1
+query_context: '{"datasource":{"id":3,"type":"table"},"force":false,"queries":[{"time_range":"No
+  filter","granularity":"start_time","filters":[{"col":"data_key","op":"==","val":"get_slots"}],"extras":{"time_grain_sqla":"PT1M","having":"","where":"(JSON_VALUE(data,
+  ''$.current_topic'') <> '''')"},"applied_time_extras":{},"columns":[{"datasourceWarning":false,"expressionType":"SQL","label":"context_id","sqlExpression":"JSON_VALUE(data,
+  ''$.current_topic'')"}],"metrics":["count"],"orderby":[["count",false]],"annotation_layers":[],"row_limit":10000,"series_columns":[{"datasourceWarning":false,"expressionType":"SQL","label":"context_id","sqlExpression":"JSON_VALUE(data,
+  ''$.current_topic'')"}],"series_limit":0,"order_desc":true,"url_params":{},"custom_params":{},"custom_form_data":{},"is_timeseries":true,"time_offsets":[],"post_processing":[{"operation":"pivot","options":{"index":["__timestamp"],"columns":["context_id"],"aggregates":{"count":{"operator":"mean"}},"drop_missing_columns":false}},{"operation":"rename","options":{"columns":{"count":null},"level":0,"inplace":true}},{"operation":"flatten"}]}],"form_data":{"datasource":"3__table","viz_type":"echarts_timeseries_bar","slice_id":10,"granularity_sqla":"start_time","time_grain_sqla":"PT1M","time_range":"No
+  filter","metrics":["count"],"groupby":[{"datasourceWarning":false,"expressionType":"SQL","label":"context_id","sqlExpression":"JSON_VALUE(data,
+  ''$.current_topic'')"}],"adhoc_filters":[{"clause":"WHERE","comparator":"get_slots","datasourceWarning":false,"expressionType":"SIMPLE","filterOptionName":"filter_8tft5fr07ea_urtdezftgn","isExtra":false,"isNew":false,"operator":"==","operatorId":"EQUALS","sqlExpression":null,"subject":"data_key"},{"clause":"WHERE","comparator":null,"datasourceWarning":false,"expressionType":"SQL","filterOptionName":"filter_4ffdpny1zzm_vmlo11yw7i","isExtra":false,"isNew":false,"operator":null,"sqlExpression":"JSON_VALUE(data,
+  ''$.current_topic'') <> ''''","subject":null}],"order_desc":true,"row_limit":10000,"truncate_metric":true,"show_empty_columns":true,"comparison_type":"values","annotation_layers":[],"forecastPeriods":10,"forecastInterval":0.8,"orientation":"vertical","x_axis_title":"Time
+  axis","x_axis_title_margin":30,"y_axis_title":"Value counts over time","y_axis_title_margin":30,"y_axis_title_position":"Left","color_scheme":"echarts4Colors","show_value":false,"stack":true,"only_total":true,"show_legend":true,"legendType":"scroll","legendOrientation":"top","x_axis_time_format":"smart_date","y_axis_format":"SMART_NUMBER","y_axis_bounds":[null,null],"rich_tooltip":true,"tooltipSortByMetric":true,"tooltipTimeFormat":"smart_date","extra_form_data":{},"dashboards":[1],"force":false,"result_format":"json","result_type":"full"},"result_format":"json","result_type":"full"}'
+cache_timeout: null
+uuid: f8215b4d-cdaf-489a-90b2-040da840ab35
+version: 1.0.0
+dataset_uuid: 8ba2e188-2bf8-4809-a5ee-2477a539d493