From 5a20bc866f1285118ac702db65f5642e1b4dd588 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C4=83lina=20Cenan?= Date: Wed, 1 May 2024 19:15:51 +0300 Subject: [PATCH] Change sim plots entrypoint, add tests (#979) * Adds new entrypoint for dash. * Add tests for dashboard. * Update md files. * fix multiple plots on the same row are not shrinking --------- Co-authored-by: Norbert --- READMEs/predictoor.md | 4 +- READMEs/trader.md | 4 +- pdr_backend/cli/cli_arguments.py | 35 +++++- pdr_backend/cli/cli_module.py | 7 ++ .../sim/dash_plots}/__init__.py | 0 .../sim/dash_plots}/callbacks.py | 8 +- .../sim/dash_plots}/util.py | 15 +-- .../sim/dash_plots}/view_elements.py | 25 ++-- pdr_backend/sim/sim_dash.py | 28 +++++ pdr_backend/sim/sim_plotter.py | 11 ++ pdr_backend/sim/test/test_dash_plots.py | 108 ++++++++++++++++++ sim_plots | 63 ---------- 12 files changed, 212 insertions(+), 96 deletions(-) rename {pdr_dash_plots => pdr_backend/sim/dash_plots}/__init__.py (100%) rename {pdr_dash_plots => pdr_backend/sim/dash_plots}/callbacks.py (93%) rename {pdr_dash_plots => pdr_backend/sim/dash_plots}/util.py (68%) rename {pdr_dash_plots => pdr_backend/sim/dash_plots}/view_elements.py (90%) create mode 100755 pdr_backend/sim/sim_dash.py create mode 100644 pdr_backend/sim/test/test_dash_plots.py delete mode 100755 sim_plots diff --git a/READMEs/predictoor.md b/READMEs/predictoor.md index 639d9225b..d6c93ecde 100644 --- a/READMEs/predictoor.md +++ b/READMEs/predictoor.md @@ -88,7 +88,7 @@ cd ~/code/pdr-backend # or wherever your pdr-backend dir is source venv/bin/activate # start the plots server -sim_plots +pdr sim_plots ``` The plots server will give a url, such as [http://127.0.0.1:8050](http://127.0.0.1:8050). Open that url in your browser to see plots update in real time. @@ -104,7 +104,7 @@ To see simulation CLI options: `pdr sim -h`. Simulation uses Python [logging](https://docs.python.org/3/howto/logging.html) framework. Configure it via [`logging.yaml`](../logging.yaml). [Here's](https://medium.com/@cyberdud3/a-step-by-step-guide-to-configuring-python-logging-with-yaml-files-914baea5a0e5) a tutorial on yaml settings. By default, Dash plots the latest sim (even if it is still running). To enable plotting for a specific run, e.g. if you used multisim or manually triggered different simulations, the sim engine assigns unique ids to each run. -Select that unique id from the `sim_state` folder, and run `sim_plots --run_id ` e.g. `sim_plots --run-id 97f9633c-a78c-4865-9cc6-b5152c9500a3` +Select that unique id from the `sim_state` folder, and run `pdr sim_plots --run_id ` e.g. `pdr sim_plots --run-id 97f9633c-a78c-4865-9cc6-b5152c9500a3` You can run many instances of Dash at once, with different URLs. To run on different ports, use the `--port` argument. diff --git a/READMEs/trader.md b/READMEs/trader.md index d0c46ccd0..395f88f32 100644 --- a/READMEs/trader.md +++ b/READMEs/trader.md @@ -80,7 +80,7 @@ cd ~/code/pdr-backend # or wherever your pdr-backend dir is source venv/bin/activate #start the plots server -sim_plots +pdr sim_plots ``` The plots server will give a url, such as [http://127.0.0.1:8050](http://127.0.0.1:8050). Open that url in your browser to see plots update in real time. @@ -96,7 +96,7 @@ To see simulation CLI options: `pdr sim -h`. Simulation uses Python [logging](https://docs.python.org/3/howto/logging.html) framework. Configure it via [`logging.yaml`](../logging.yaml). [Here's](https://medium.com/@cyberdud3/a-step-by-step-guide-to-configuring-python-logging-with-yaml-files-914baea5a0e5) a tutorial on yaml settings. By default, Dash plots the latest sim (even if it is still running). To enable plotting for a specific run, e.g. if you used multisim or manually triggered different simulations, the sim engine assigns unique ids to each run. -Select that unique id from the `sim_state` folder, and run `sim_plots --run_id ` e.g. `sim_plots --run_id 97f9633c-a78c-4865-9cc6-b5152c9500a3` +Select that unique id from the `sim_state` folder, and run `pdr sim_plots --run_id ` e.g. `pdr sim_plots --run_id 97f9633c-a78c-4865-9cc6-b5152c9500a3` You can run many instances of Dash at once, with different URLs. To run on different ports, use the `--port` argument. diff --git a/pdr_backend/cli/cli_arguments.py b/pdr_backend/cli/cli_arguments.py index 857323899..bc3e276d1 100644 --- a/pdr_backend/cli/cli_arguments.py +++ b/pdr_backend/cli/cli_arguments.py @@ -1,13 +1,14 @@ import argparse -from argparse import Namespace import logging import sys +from argparse import Namespace from typing import List from enforce_typing import enforce_types from eth_utils import to_checksum_address from pdr_backend.cli.nested_arg_parser import NestedArgParser +from pdr_backend.sim.sim_plotter import SimPlotter logger = logging.getLogger("cli") @@ -46,6 +47,7 @@ pdr deployer (for >1 predictoor bots) pdr lake PPSS_FILE NETWORK pdr analytics PPSS_FILE NETWORK + pdr sim_plots [--run_id RUN_ID] [--port PORT] Utilities: pdr get_predictoors_info ST END PQDIR PPSS_FILE NETWORK --PDRS @@ -153,6 +155,13 @@ def check_address(addr) -> str: return addr2 +def validate_run_id(run_id): + if run_id not in SimPlotter.get_all_run_names(): + raise ValueError(f"Invalid run_id: {run_id}") + + return run_id + + @enforce_types class PDRS_Mixin: def add_argument_PDRS(self): @@ -560,6 +569,29 @@ def network_choices(self): return ["sapphire-testnet", "sapphire-mainnet"] +class SimPlotsArgParser(CustomArgParser): + # pylint: disable=unused-argument + def __init__(self, description: str, command_name: str): + super().__init__(description=description) + + self.add_argument( + "--run_id", + help=( + "The run_id of the simulation to visualize. " + "If not provided, the latest run_id will be used." + ), + type=validate_run_id, + ) + + self.add_argument( + "--port", + nargs="?", + help="The port to run the server on. Default is 8050.", + type=int, + default=8050, + ) + + # below, list each entry in defined_parsers in same order as HELP_LONG defined_parsers = { # main tools @@ -604,6 +636,7 @@ def network_choices(self): "do_dfbuyer": DfbuyerArgParser("Run dfbuyer bot", "dfbuyer"), "do_publisher": PublisherArgParser("Publish feeds", "publisher"), "do_topup": TopupArgParser("Topup OCEAN and ROSE in dfbuyer, trueval, ..", "topup"), + "do_sim_plots": SimPlotsArgParser("Visualize simulation data", "sim_plots"), } diff --git a/pdr_backend/cli/cli_module.py b/pdr_backend/cli/cli_module.py index beb7e073e..164d9ed93 100644 --- a/pdr_backend/cli/cli_module.py +++ b/pdr_backend/cli/cli_module.py @@ -27,6 +27,7 @@ from pdr_backend.predictoor.predictoor_agent import PredictoorAgent from pdr_backend.publisher.publish_assets import publish_assets from pdr_backend.sim.multisim_engine import MultisimEngine +from pdr_backend.sim.sim_dash import sim_dash from pdr_backend.sim.sim_engine import SimEngine from pdr_backend.trader.approach1.trader_agent1 import TraderAgent1 from pdr_backend.trader.approach2.trader_agent2 import TraderAgent2 @@ -356,3 +357,9 @@ def do_deploy_pred_submitter_mgr(args, nested_args=None): logger.info( "Prediction Submitter Manager Contract deployed at %s", contract_address ) + + +@enforce_types +# pylint: disable=unused-argument +def do_sim_plots(args, nested_args=None): + sim_dash(args) diff --git a/pdr_dash_plots/__init__.py b/pdr_backend/sim/dash_plots/__init__.py similarity index 100% rename from pdr_dash_plots/__init__.py rename to pdr_backend/sim/dash_plots/__init__.py diff --git a/pdr_dash_plots/callbacks.py b/pdr_backend/sim/dash_plots/callbacks.py similarity index 93% rename from pdr_dash_plots/callbacks.py rename to pdr_backend/sim/dash_plots/callbacks.py index 0ce0cad08..e4afae4bd 100644 --- a/pdr_dash_plots/callbacks.py +++ b/pdr_backend/sim/dash_plots/callbacks.py @@ -2,9 +2,8 @@ from dash import Input, Output, State -from pdr_backend.sim.sim_plotter import SimPlotter -from pdr_dash_plots.util import get_figures_by_state, get_latest_run_id -from pdr_dash_plots.view_elements import ( +from pdr_backend.sim.dash_plots.util import get_figures_by_state +from pdr_backend.sim.dash_plots.view_elements import ( arrange_figures, get_header_elements, get_waiting_template, @@ -12,6 +11,7 @@ selected_var_checklist, snapshot_slider, ) +from pdr_backend.sim.sim_plotter import SimPlotter def wait_for_state(sim_plotter, run_id, set_ts): @@ -64,7 +64,7 @@ def update_selected_vars(clickData, selected_vars): ) # pylint: disable=unused-argument def update_graph_live(n, selected_vars, slider_value, selected_vars_old): - run_id = app.run_id if app.run_id else get_latest_run_id() + run_id = app.run_id if app.run_id else SimPlotter.get_latest_run_id() set_ts = None if slider_value is not None: diff --git a/pdr_dash_plots/util.py b/pdr_backend/sim/dash_plots/util.py similarity index 68% rename from pdr_dash_plots/util.py rename to pdr_backend/sim/dash_plots/util.py index c804fbbe2..e74e976cd 100644 --- a/pdr_dash_plots/util.py +++ b/pdr_backend/sim/dash_plots/util.py @@ -1,19 +1,6 @@ -import os -from pathlib import Path - from pdr_backend.aimodel import aimodel_plotter +from pdr_backend.sim.dash_plots.view_elements import figure_names from pdr_backend.sim.sim_plotter import SimPlotter -from pdr_dash_plots.view_elements import figure_names - - -def get_latest_run_id(): - path = sorted(Path("sim_state").iterdir(), key=os.path.getmtime)[-1] - return str(path).replace("sim_state/", "") - - -def get_all_run_names(): - path = Path("sim_state").iterdir() - return [str(p).replace("sim_state/", "") for p in path] def get_figures_by_state(sim_plotter: SimPlotter, selected_vars): diff --git a/pdr_dash_plots/view_elements.py b/pdr_backend/sim/dash_plots/view_elements.py similarity index 90% rename from pdr_dash_plots/view_elements.py rename to pdr_backend/sim/dash_plots/view_elements.py index b34a4116b..5d88fea38 100644 --- a/pdr_dash_plots/view_elements.py +++ b/pdr_backend/sim/dash_plots/view_elements.py @@ -1,4 +1,5 @@ from datetime import datetime + from dash import dcc, html from plotly.graph_objs import Figure @@ -62,11 +63,9 @@ def arrange_figures(figures): return [ html.Div( [ + dcc.Graph(figure=figures["pdr_profit_vs_time"], style={"width": "50%"}), dcc.Graph( - figure=figures["pdr_profit_vs_time"], style={"width": "100%"} - ), - dcc.Graph( - figure=figures["trader_profit_vs_time"], style={"width": "100%"} + figure=figures["trader_profit_vs_time"], style={"width": "50%"} ), ], style={"display": "flex", "justifyContent": "space-between"}, @@ -79,10 +78,10 @@ def arrange_figures(figures): html.Div( [ dcc.Graph( - figure=figures["pdr_profit_vs_ptrue"], style={"width": "100%"} + figure=figures["pdr_profit_vs_ptrue"], style={"width": "50%"} ), dcc.Graph( - figure=figures["trader_profit_vs_ptrue"], style={"width": "100%"} + figure=figures["trader_profit_vs_ptrue"], style={"width": "50%"} ), ], style={"display": "flex", "justifyContent": "space-between"}, @@ -92,9 +91,9 @@ def arrange_figures(figures): dcc.Graph( figure=figures["aimodel_varimps"], id="aimodel_varimps", - style={"width": "100%"}, + style={"width": "50%"}, ), - dcc.Graph(figure=figures["aimodel_response"], style={"width": "100%"}), + dcc.Graph(figure=figures["aimodel_response"], style={"width": "50%"}), ], style={"display": "flex", "justifyContent": "space-between"}, ), @@ -111,11 +110,17 @@ def format_ts(s): return datetime.strptime(base, "%Y%m%d%H%M%S").strftime("%H:%M:%S") -def snapshot_slider(run_id, set_ts, slider_value): +def prune_snapshots(run_id): snapshots = SimPlotter.available_snapshots(run_id)[:-1] max_states_ux = 50 if len(snapshots) > max_states_ux: - snapshots = snapshots[:: len(snapshots) // max_states_ux] + return snapshots[:: len(snapshots) // max_states_ux] + + return snapshots + + +def snapshot_slider(run_id, set_ts, slider_value): + snapshots = prune_snapshots(run_id) marks = { i: {"label": format_ts(s), "style": {"transform": "rotate(45deg)"}} diff --git a/pdr_backend/sim/sim_dash.py b/pdr_backend/sim/sim_dash.py new file mode 100755 index 000000000..d45028ce8 --- /dev/null +++ b/pdr_backend/sim/sim_dash.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python +from dash import Dash, dcc, html + +from pdr_backend.sim.dash_plots.callbacks import get_callbacks +from pdr_backend.sim.dash_plots.view_elements import empty_graphs_template + +app = Dash(__name__) +app.config["suppress_callback_exceptions"] = True +app.layout = html.Div( + html.Div( + [ + html.Div(empty_graphs_template, id="live-graphs"), + dcc.Interval( + id="interval-component", + interval=3 * 1000, # in milliseconds + n_intervals=0, + disabled=False, + ), + ] + ) +) + +get_callbacks(app) + + +def sim_dash(args): + app.run_id = args.run_id + app.run(debug=True, port=args.port) diff --git a/pdr_backend/sim/sim_plotter.py b/pdr_backend/sim/sim_plotter.py index 9840f6b17..d2e4fba7f 100644 --- a/pdr_backend/sim/sim_plotter.py +++ b/pdr_backend/sim/sim_plotter.py @@ -3,6 +3,7 @@ import pickle import time from datetime import datetime +from pathlib import Path from typing import Optional import numpy as np @@ -39,6 +40,16 @@ def available_snapshots(multi_id): return all_timestamps + ["final"] + @staticmethod + def get_latest_run_id(): + path = sorted(Path("sim_state").iterdir(), key=os.path.getmtime)[-1] + return str(path).replace("sim_state/", "") + + @staticmethod + def get_all_run_names(): + path = Path("sim_state").iterdir() + return [str(p).replace("sim_state/", "") for p in path] + def load_state(self, multi_id, timestamp: Optional[str] = None): root_path = f"sim_state/{multi_id}" diff --git a/pdr_backend/sim/test/test_dash_plots.py b/pdr_backend/sim/test/test_dash_plots.py new file mode 100644 index 000000000..6d92e9a30 --- /dev/null +++ b/pdr_backend/sim/test/test_dash_plots.py @@ -0,0 +1,108 @@ +from unittest.mock import Mock, patch + +from plotly.graph_objs import Figure + +from pdr_backend.sim.dash_plots.util import get_figures_by_state +from pdr_backend.sim.dash_plots.view_elements import ( + arrange_figures, + figure_names, + get_header_elements, + get_waiting_template, + prune_snapshots, + selected_var_checklist, + snapshot_slider, +) +from pdr_backend.sim.sim_plotter import SimPlotter + + +def test_get_waiting_template(): + result = get_waiting_template("custom message") + assert "custom message" in result.children[0].children + + +def test_get_header_elements(): + st = Mock() + st.iter_number = 5 + + result = get_header_elements("abcd", st, "ts") + assert "Simulation ID: abcd" in result[0].children + assert "Iter #5 (ts)" in result[1].children + assert result[1].className == "runningState" + + result = get_header_elements("abcd", st, "final") + assert "Simulation ID: abcd" in result[0].children + assert "Final sim state" in result[1].children + assert result[1].className == "finalState" + + +def test_arrange_figures(): + figures = {key: Figure() for key in figure_names} + result = arrange_figures(figures) + seen = set() + count = 0 + for div in result: + for graph in div.children: + count += 1 + if hasattr(graph, "id"): + seen.add(graph.id) + + assert count == len(figure_names) + assert seen == {"aimodel_varimps"} # only one with id present for now + set(figure_names) + + +def test_snapshot_slider(): + with patch( + "pdr_backend.sim.dash_plots.view_elements.prune_snapshots" + ) as mock_prune_snapshots: + mock_prune_snapshots.return_value = ["ts1", "ts2", "final"] + + result = snapshot_slider("abcd", "ts1", 1) + assert result + + +def test_prune_snapshots(): + with patch( + "pdr_backend.sim.dash_plots.view_elements.SimPlotter.available_snapshots" + ) as mock_snapshots: + mock_snapshots.return_value = ["ts1", "ts2", "final"] + assert prune_snapshots("abcd") == ["ts1", "ts2"] + + more_than_50 = [f"ts{i}" for i in range(100)] + ["final"] + with patch( + "pdr_backend.sim.dash_plots.view_elements.SimPlotter.available_snapshots" + ) as mock_snapshots: + mock_snapshots.return_value = more_than_50 + assert prune_snapshots("abcd") == [f"ts{i}" for i in range(100) if i % 2 == 0] + + +def test_selected_var_checklist(): + result = selected_var_checklist(["var1", "var2"], ["var1"]) + assert result.value == ["var1"] + assert result.options[0]["label"] == "var1" + assert result.options[1]["label"] == "var2" + + +def test_get_figures_by_state(): + mock_sim_plotter = Mock(spec=SimPlotter) + mock_sim_plotter.plot_pdr_profit_vs_time.return_value = Figure() + mock_sim_plotter.plot_trader_profit_vs_time.return_value = Figure() + mock_sim_plotter.plot_accuracy_vs_time.return_value = Figure() + mock_sim_plotter.plot_pdr_profit_vs_ptrue.return_value = Figure() + mock_sim_plotter.plot_trader_profit_vs_ptrue.return_value = Figure() + mock_sim_plotter.plot_f1_precision_recall_vs_time.return_value = Figure() + plotdata = Mock() + plotdata.colnames = ["var1", "var2"] + mock_sim_plotter.aimodel_plotdata = plotdata + + with patch( + "pdr_backend.sim.dash_plots.util.aimodel_plotter" + ) as mock_aimodel_plotter: + mock_aimodel_plotter.plot_aimodel_response.return_value = Figure() + mock_aimodel_plotter.plot_aimodel_varimps.return_value = Figure() + + result = get_figures_by_state(mock_sim_plotter, ["var1", "var2"]) + + for key in figure_names: + assert key in result + assert isinstance(result[key], Figure) diff --git a/sim_plots b/sim_plots deleted file mode 100755 index 94f5c0eb7..000000000 --- a/sim_plots +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env python -import argparse - -from dash import Dash, dcc, html - -from pdr_dash_plots.callbacks import get_callbacks -from pdr_dash_plots.view_elements import empty_graphs_template -from pdr_dash_plots.util import get_all_run_names - -app = Dash(__name__) -app.config["suppress_callback_exceptions"] = True -app.layout = html.Div( - html.Div( - [ - html.Div(empty_graphs_template, id="live-graphs"), - dcc.Interval( - id="interval-component", - interval=3 * 1000, # in milliseconds - n_intervals=0, - disabled=False, - ), - ] - ) -) - -get_callbacks(app) - - -def validate_run_id(run_id): - if run_id not in get_all_run_names(): - raise ValueError(f"Invalid run_id: {run_id}") - - return run_id - - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - prog='sim plots', - description='A script to visualize simulation data generated by pdr sim.', - epilog='powered by Dash, Plotly, and Flask.' - ) - - parser.add_argument( - '--run_id', - help=( - 'The run_id of the simulation to visualize. ' - 'If not provided, the latest run_id will be used.' - ), - type=validate_run_id - ) - - parser.add_argument( - '--port', - nargs='?', - help='The port to run the server on. Default is 8050.', - type=int, - default=8050 - ) - - args = parser.parse_args() - - app.run_id = args.run_id - app.run(debug=True, port=args.port)