Feat/strategy performance v 0.11

pages/strategy_performance/app.py

@@ -4,6 +4,7 @@
 import plotly.graph_objects as go
 import math
 import plotly.express as px
+from utils.os_utils import get_bots_data_paths
 from utils.database_manager import DatabaseManager
 from utils.graphs import CandlesGraph
 from utils.st_utils import initialize_st_page
@@ -28,10 +29,16 @@
 
 
 def get_databases():
-    sqlite_files = [db_name for db_name in os.listdir("data") if db_name.endswith(".sqlite")]
-    databases_list = [DatabaseManager(db) for db in sqlite_files]
-    if len(databases_list) > 0:
-        return {database.db_name: database for database in databases_list}
+    databases = {}
+    bots_data_paths = get_bots_data_paths()
+    for source_name, source_path in bots_data_paths.items():
+        sqlite_files = {}
+        for db_name in os.listdir(source_path):
+            if db_name.endswith(".sqlite"):
+                sqlite_files[db_name] = os.path.join(source_path, db_name)
+        databases[source_name] = sqlite_files
+    if len(databases) > 0:
+        return {key: value for key, value in databases.items() if value}
     else:
         return None
 
@@ -233,7 +240,6 @@ def hr_str(hr):
             yanchor="bottom"
         ),
     )
-
     return fig
 
 
@@ -265,7 +271,7 @@ def candles_graph(candles: pd.DataFrame, strat_data, show_volume=False, extra_ro
     cg = CandlesGraph(candles, show_volume=show_volume, extra_rows=extra_rows)
     cg.add_buy_trades(strat_data.buys)
     cg.add_sell_trades(strat_data.sells)
-    cg.add_pnl(strategy_data, row=2)
+    cg.add_pnl(strat_data, row=2)
     cg.add_quote_inventory_change(strat_data, row=3)
     return cg.figure()
 
@@ -282,9 +288,12 @@ def candles_graph(candles: pd.DataFrame, strat_data, show_volume=False, extra_ro
         selected_db = DatabaseManager(uploaded_db.name)
 dbs = get_databases()
 if dbs is not None:
-    db_names = [x.db_name for x in dbs.values()]
+    bot_source = st.selectbox("Choose your database source:", dbs.keys())
+    db_names = [x for x in dbs[bot_source]]
     selected_db_name = st.selectbox("Select a database to start:", db_names)
-    selected_db = dbs[selected_db_name]
+    executors_path = os.path.dirname(dbs[bot_source][selected_db_name])
+    selected_db = DatabaseManager(db_name=dbs[bot_source][selected_db_name],
+                                  executors_path=executors_path)
 else:
     st.warning("Ups! No databases were founded. Start uploading one")
     selected_db = None
@@ -334,8 +343,8 @@ def candles_graph(candles: pd.DataFrame, strat_data, show_volume=False, extra_ro
                             st.metric(label="Profit Factor",
                                       value=round(strategy_data_filtered.profit_factor, 2))
                         with col5:
-                            st.metric(label='Duration (Hours)',
-                                      value=round(strategy_data_filtered.duration_seconds / 3600, 2))
+                            st.metric(label='Duration (Days)',
+                                      value=round(strategy_data_filtered.duration_seconds / (60 * 60 * 24)))
                         with col6:
                             st.metric(label='Price change',
                                       value=f"{round(strategy_data_filtered.price_change * 100, 2)} %")

utils/data_manipulation.py

@@ -24,10 +24,8 @@ def full_series(series):
 
         strategy_data = self.trade_fill.copy()
         strategy_data["volume"] = strategy_data["amount"] * strategy_data["price"]
-        strategy_data["margin_volume"] = strategy_data["amount"] * strategy_data["price"] / strategy_data["leverage"]
         strategy_summary = strategy_data.groupby(["strategy", "market", "symbol"]).agg({"order_id": "count",
                                                                                         "volume": "sum",
-                                                                                        "margin_volume": "sum",
                                                                                         "net_realized_pnl": [full_series,
                                                                                                              "last"]}).reset_index()
         strategy_summary.columns = [f"{col[0]}_{col[1]}" if isinstance(col, tuple) and col[1] is not None else col for col in strategy_summary.columns]
@@ -36,7 +34,6 @@ def full_series(series):
                                          "symbol_": "Trading Pair",
                                          "order_id_count": "# Trades",
                                          "volume_sum": "Volume",
-                                         "margin_volume_sum": "Margin volume",
                                          "net_realized_pnl_full_series": "PnL Over Time",
                                          "net_realized_pnl_last": "Realized PnL"}, inplace=True)
         strategy_summary.sort_values(["Realized PnL"], ascending=True, inplace=True)
@@ -246,8 +243,8 @@ def inventory_change_base_asset(self):
 
     @property
     def accuracy(self):
-        total_wins = len(self.trade_fill["net_realized_pnl"] >= 0)
-        total_losses = len(self.trade_fill["net_realized_pnl"] < 0)
+        total_wins = (self.trade_fill["net_realized_pnl"] >= 0).sum()
+        total_losses = (self.trade_fill["net_realized_pnl"] < 0).sum()
         return total_wins / (total_wins + total_losses)
 
     @property

utils/database_manager.py

@@ -12,7 +12,7 @@ class DatabaseManager:
     def __init__(self, db_name: str, executors_path: str = "data"):
         self.db_name = db_name
         # TODO: Create db path for all types of db
-        self.db_path = f'sqlite:///{os.path.join("data", db_name)}'
+        self.db_path = f'sqlite:///{os.path.join(db_name)}'
         self.executors_path = executors_path
         self.engine = create_engine(self.db_path, connect_args={'check_same_thread': False})
         self.session_maker = sessionmaker(bind=self.engine)
@@ -157,9 +157,9 @@ def get_trade_fills(self, config_file_path=None, start_date=None, end_date=None)
             trade_fills["inventory_cost"] = trade_fills["cum_net_amount"] * trade_fills["price"]
             trade_fills["realized_trade_pnl"] = trade_fills["unrealized_trade_pnl"] + trade_fills["inventory_cost"]
             trade_fills["net_realized_pnl"] = trade_fills["realized_trade_pnl"] - trade_fills["cum_fees_in_quote"]
-            trade_fills["realized_pnl"] = trade_fills["net_realized_pnl"].diff()
-            trade_fills["gross_pnl"] = trade_fills["realized_trade_pnl"].diff()
-            trade_fills["trade_fee"] = trade_fills["cum_fees_in_quote"].diff()
+            trade_fills["realized_pnl"] = trade_fills.groupby(groupers)["net_realized_pnl"].diff()
+            trade_fills["gross_pnl"] = trade_fills.groupby(groupers)["realized_trade_pnl"].diff()
+            trade_fills["trade_fee"] = trade_fills.groupby(groupers)["cum_fees_in_quote"].diff()
             trade_fills["timestamp"] = pd.to_datetime(trade_fills["timestamp"], unit="ms")
             trade_fills["market"] = trade_fills["market"].apply(lambda x: x.lower().replace("_papertrade", ""))
             trade_fills["quote_volume"] = trade_fills["price"] * trade_fills["amount"]

utils/os_utils.py

@@ -104,6 +104,23 @@ def load_controllers(path):
     return controllers
 
 
+def get_bots_data_paths():
+    root_directory = "hummingbot_files/bots"
+    bots_data_paths = {"General / Uploaded data": "data"}
+    reserved_word = "hummingbot-"
+    # Walk through the directory tree
+    for dirpath, dirnames, filenames in os.walk(root_directory):
+        for dirname in dirnames:
+            if dirname == "data":
+                parent_folder = os.path.basename(dirpath)
+                if parent_folder.startswith(reserved_word):
+                    bots_data_paths[parent_folder] = os.path.join(dirpath, dirname)
+            if "dashboard" in bots_data_paths:
+                del bots_data_paths["dashboard"]
+    data_sources = {key: value for key, value in bots_data_paths.items() if value is not None}
+    return data_sources
+
+
 def get_function_from_file(file_path: str, function_name: str):
     # Create a module specification from the file path and load it
     spec = importlib.util.spec_from_file_location("module.name", file_path)