From 8d91949b157435ee8d76a36b0cbe7b03bd6784aa Mon Sep 17 00:00:00 2001
From: Sheng Lundquist <sheng@delv.tech>
Date: Tue, 16 Jan 2024 11:04:34 -0800
Subject: [PATCH 1/9] Interactive hyperdrive has optional log to rollbar

---
 .../interactive/interactive_hyperdrive.py           | 13 ++++++++++---
 .../agent0/interactive_fuzz/helpers/setup_fuzz.py   |  4 +++-
 lib/hyperlogs/hyperlogs/rollbar_utilities.py        |  2 +-
 3 files changed, 14 insertions(+), 5 deletions(-)

diff --git a/lib/agent0/agent0/hyperdrive/interactive/interactive_hyperdrive.py b/lib/agent0/agent0/hyperdrive/interactive/interactive_hyperdrive.py
index d9aa863be7..349f3aab11 100644
--- a/lib/agent0/agent0/hyperdrive/interactive/interactive_hyperdrive.py
+++ b/lib/agent0/agent0/hyperdrive/interactive/interactive_hyperdrive.py
@@ -15,8 +15,9 @@
 from chainsync import PostgresConfig
 from chainsync.dashboard.usernames import build_user_mapping
 from chainsync.db.base import add_addr_to_username, get_addr_to_username, get_username_to_user, initialize_session
+from chainsync.db.hyperdrive import get_checkpoint_info
+from chainsync.db.hyperdrive import get_current_wallet as chainsync_get_current_wallet
 from chainsync.db.hyperdrive import (
-    get_checkpoint_info,
     get_latest_block_number_from_analysis_table,
     get_pool_analysis,
     get_pool_config,
@@ -26,7 +27,6 @@
     get_wallet_deltas,
     get_wallet_pnl,
 )
-from chainsync.db.hyperdrive import get_current_wallet as chainsync_get_current_wallet
 from chainsync.exec import acquire_data, data_analysis
 from eth_account.account import Account
 from eth_typing import BlockNumber, ChecksumAddress
@@ -95,6 +95,8 @@ class Config:
             The timeout for the data pipeline. Defaults to 60 seconds.
         preview_before_trade: bool, optional
             Whether to preview the position before executing a trade. Defaults to False.
+        log_to_rollbar: bool, optional
+            Whether to log crash reports to rollbar. Defaults to False.
         rng_seed: int | None, optional
             The seed for the random number generator. Defaults to None.
         rng: Generator | None, optional
@@ -140,6 +142,7 @@ class Config:
         # Environment variables
         data_pipeline_timeout: int = 60
         preview_before_trade: bool = False
+        log_to_rollbar: bool = False
         # Random generators
         rng_seed: int | None = None
         rng: Generator | None = None
@@ -254,6 +257,7 @@ def __init__(self, chain: Chain, config: Config | None = None):
             self._run_blocking_data_pipeline()
 
         self.rng = config.rng
+        self.log_to_rollbar = config.log_to_rollbar
 
     def _launch_data_pipeline(self, start_block: int | None = None):
         """Launches the data pipeline in background threads.
@@ -948,7 +952,10 @@ def _handle_trade_result(self, trade_results: list[TradeResult] | TradeResult) -
             trade_result.anvil_state = get_anvil_state_dump(self.hyperdrive_interface.web3)
             # Defaults to CRITICAL
             log_hyperdrive_crash_report(
-                trade_result, crash_report_to_file=True, crash_report_file_prefix="interactive_hyperdrive"
+                trade_result,
+                crash_report_to_file=True,
+                crash_report_file_prefix="interactive_hyperdrive",
+                log_to_rollbar=self.log_to_rollbar,
             )
             raise trade_result.exception
 
diff --git a/lib/agent0/agent0/interactive_fuzz/helpers/setup_fuzz.py b/lib/agent0/agent0/interactive_fuzz/helpers/setup_fuzz.py
index 521c791a9a..e6cadef9f2 100644
--- a/lib/agent0/agent0/interactive_fuzz/helpers/setup_fuzz.py
+++ b/lib/agent0/agent0/interactive_fuzz/helpers/setup_fuzz.py
@@ -63,7 +63,9 @@ def setup_fuzz(
 
     # Parameters for pool initialization.
     # Using a day for checkpoint duration to speed things up
-    initial_pool_config = InteractiveHyperdrive.Config(preview_before_trade=True, checkpoint_duration=86400)
+    initial_pool_config = InteractiveHyperdrive.Config(
+        preview_before_trade=True, checkpoint_duration=86400, log_to_rollbar=True
+    )
     if not fees:
         initial_pool_config.curve_fee = FixedPoint(0)
         initial_pool_config.flat_fee = FixedPoint(0)
diff --git a/lib/hyperlogs/hyperlogs/rollbar_utilities.py b/lib/hyperlogs/hyperlogs/rollbar_utilities.py
index b8348ffb33..b191366a30 100644
--- a/lib/hyperlogs/hyperlogs/rollbar_utilities.py
+++ b/lib/hyperlogs/hyperlogs/rollbar_utilities.py
@@ -83,4 +83,4 @@ def log_rollbar_exception(exception: Exception, log_level: int, extra_data: dict
     try:
         raise exception
     except Exception:  # pylint: disable=broad-exception-caught
-        rollbar.report_exc_info(sys.exc_info(), log_level_name, extra_data=extra_data)
+        rollbar.report_exc_info(sys.exc_info(), level=log_level_name, extra_data=extra_data)

From bbd85ac2d6876a1d81fc9a9b5abadff110bf1f40 Mon Sep 17 00:00:00 2001
From: Sheng Lundquist <sheng@delv.tech>
Date: Tue, 16 Jan 2024 11:09:05 -0800
Subject: [PATCH 2/9] Adding flag for logging to rollbar in setup fuzz, and
 turning it off in path independence

---
 .../agent0/interactive_fuzz/fuzz_path_independence.py     | 8 +++++++-
 lib/agent0/agent0/interactive_fuzz/helpers/setup_fuzz.py  | 3 ++-
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/lib/agent0/agent0/interactive_fuzz/fuzz_path_independence.py b/lib/agent0/agent0/interactive_fuzz/fuzz_path_independence.py
index acfa469668..82606f557a 100644
--- a/lib/agent0/agent0/interactive_fuzz/fuzz_path_independence.py
+++ b/lib/agent0/agent0/interactive_fuzz/fuzz_path_independence.py
@@ -89,7 +89,13 @@ def fuzz_path_independence(
     # pylint: disable=too-many-statements
     # pylint: disable=too-many-arguments
     log_filename = ".logging/fuzz_path_independence.log"
-    chain, random_seed, rng, interactive_hyperdrive = setup_fuzz(log_filename, chain_config, log_to_stdout, fees=False)
+    chain, random_seed, rng, interactive_hyperdrive = setup_fuzz(
+        log_filename,
+        chain_config,
+        log_to_stdout,
+        log_to_rollbar=False,  # We don't log other crashes to rollbar since we expect failed paths here
+        fees=False,
+    )
 
     # Open some trades
     logging.info("Open random trades...")
diff --git a/lib/agent0/agent0/interactive_fuzz/helpers/setup_fuzz.py b/lib/agent0/agent0/interactive_fuzz/helpers/setup_fuzz.py
index e6cadef9f2..769a1afdfc 100644
--- a/lib/agent0/agent0/interactive_fuzz/helpers/setup_fuzz.py
+++ b/lib/agent0/agent0/interactive_fuzz/helpers/setup_fuzz.py
@@ -13,6 +13,7 @@ def setup_fuzz(
     log_filename: str,
     chain_config: LocalChain.Config | None = None,
     log_to_stdout: bool = False,
+    log_to_rollbar: bool = True,
     fees=True,
     var_interest=None,
 ) -> tuple[LocalChain, int, Generator, InteractiveHyperdrive]:
@@ -64,7 +65,7 @@ def setup_fuzz(
     # Parameters for pool initialization.
     # Using a day for checkpoint duration to speed things up
     initial_pool_config = InteractiveHyperdrive.Config(
-        preview_before_trade=True, checkpoint_duration=86400, log_to_rollbar=True
+        preview_before_trade=True, checkpoint_duration=86400, log_to_rollbar=log_to_rollbar
     )
     if not fees:
         initial_pool_config.curve_fee = FixedPoint(0)

From 17af0655dca981a29bca949fc9b068f0c9a7f177 Mon Sep 17 00:00:00 2001
From: Sheng Lundquist <sheng@delv.tech>
Date: Tue, 16 Jan 2024 11:56:53 -0800
Subject: [PATCH 3/9] Adding log level when crash reporting in interacative
 hyperdrive

---
 .../hyperdrive/interactive/interactive_hyperdrive.py  |  5 +++++
 .../agent0/interactive_fuzz/helpers/setup_fuzz.py     | 11 ++++++++++-
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/lib/agent0/agent0/hyperdrive/interactive/interactive_hyperdrive.py b/lib/agent0/agent0/hyperdrive/interactive/interactive_hyperdrive.py
index 349f3aab11..0fdc3a784c 100644
--- a/lib/agent0/agent0/hyperdrive/interactive/interactive_hyperdrive.py
+++ b/lib/agent0/agent0/hyperdrive/interactive/interactive_hyperdrive.py
@@ -97,6 +97,8 @@ class Config:
             Whether to preview the position before executing a trade. Defaults to False.
         log_to_rollbar: bool, optional
             Whether to log crash reports to rollbar. Defaults to False.
+        crash_log_level: int | None, optional
+            The log level to log crashes at. Defaults to critical.
         rng_seed: int | None, optional
             The seed for the random number generator. Defaults to None.
         rng: Generator | None, optional
@@ -143,6 +145,7 @@ class Config:
         data_pipeline_timeout: int = 60
         preview_before_trade: bool = False
         log_to_rollbar: bool = False
+        crash_log_level: int | None = None
         # Random generators
         rng_seed: int | None = None
         rng: Generator | None = None
@@ -258,6 +261,7 @@ def __init__(self, chain: Chain, config: Config | None = None):
 
         self.rng = config.rng
         self.log_to_rollbar = config.log_to_rollbar
+        self.crash_log_level = config.crash_log_level
 
     def _launch_data_pipeline(self, start_block: int | None = None):
         """Launches the data pipeline in background threads.
@@ -953,6 +957,7 @@ def _handle_trade_result(self, trade_results: list[TradeResult] | TradeResult) -
             # Defaults to CRITICAL
             log_hyperdrive_crash_report(
                 trade_result,
+                log_level=self.crash_log_level,
                 crash_report_to_file=True,
                 crash_report_file_prefix="interactive_hyperdrive",
                 log_to_rollbar=self.log_to_rollbar,
diff --git a/lib/agent0/agent0/interactive_fuzz/helpers/setup_fuzz.py b/lib/agent0/agent0/interactive_fuzz/helpers/setup_fuzz.py
index 769a1afdfc..d3eba4c2b0 100644
--- a/lib/agent0/agent0/interactive_fuzz/helpers/setup_fuzz.py
+++ b/lib/agent0/agent0/interactive_fuzz/helpers/setup_fuzz.py
@@ -14,6 +14,7 @@ def setup_fuzz(
     chain_config: LocalChain.Config | None = None,
     log_to_stdout: bool = False,
     log_to_rollbar: bool = True,
+    crash_log_level: int | None = None,
     fees=True,
     var_interest=None,
 ) -> tuple[LocalChain, int, Generator, InteractiveHyperdrive]:
@@ -29,6 +30,10 @@ def setup_fuzz(
     log_to_stdout: bool, optional
         If True, log to stdout in addition to a file.
         Defaults to False.
+    log_to_rollbar: bool, optional
+        If True, log errors rollbar. Defaults to True.
+    crash_log_level: int | None, optional
+        The log level to log crashes at. Defaults to critical.
     fees: bool, optional
         If False, will turn off fees when deploying hyperdrive. Defaults to True.
     var_interest: FixedPoint | None, optional
@@ -48,6 +53,7 @@ def setup_fuzz(
             interactive_hyperdrive: InteractiveHyperdrive
                 An instantiated InteractiveHyperdrive object.
     """
+    # pylint: disable=too-many-arguments
     setup_logging(
         log_filename=log_filename,
         delete_previous_logs=False,
@@ -65,7 +71,10 @@ def setup_fuzz(
     # Parameters for pool initialization.
     # Using a day for checkpoint duration to speed things up
     initial_pool_config = InteractiveHyperdrive.Config(
-        preview_before_trade=True, checkpoint_duration=86400, log_to_rollbar=log_to_rollbar
+        preview_before_trade=True,
+        checkpoint_duration=86400,
+        log_to_rollbar=log_to_rollbar,
+        crash_log_level=crash_log_level,
     )
     if not fees:
         initial_pool_config.curve_fee = FixedPoint(0)

From 55037d2f0c9866ddbc5d97ce2d317ce9aa9d368f Mon Sep 17 00:00:00 2001
From: Sheng Lundquist <sheng@delv.tech>
Date: Tue, 16 Jan 2024 11:58:42 -0800
Subject: [PATCH 4/9] Using report message instead of log exception in rollbar

---
 lib/hyperlogs/hyperlogs/rollbar_utilities.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/lib/hyperlogs/hyperlogs/rollbar_utilities.py b/lib/hyperlogs/hyperlogs/rollbar_utilities.py
index b191366a30..cae939c53e 100644
--- a/lib/hyperlogs/hyperlogs/rollbar_utilities.py
+++ b/lib/hyperlogs/hyperlogs/rollbar_utilities.py
@@ -6,10 +6,10 @@
 import logging
 import os
 import platform
-import sys
 
 import rollbar
 from dotenv import load_dotenv
+from ethpy.base.errors import ContractCallException
 
 load_dotenv("rollbar.env")
 ROLLBAR_API_KEY = os.getenv("ROLLBAR_API_KEY")
@@ -79,8 +79,8 @@ def log_rollbar_exception(exception: Exception, log_level: int, extra_data: dict
     extra_data: dict, optional.
         Extra data to send to rollbar. This usually contains the custom crash report json
     """
-    log_level_name = logging.getLevelName(log_level)
-    try:
-        raise exception
-    except Exception:  # pylint: disable=broad-exception-caught
-        rollbar.report_exc_info(sys.exc_info(), level=log_level_name, extra_data=extra_data)
+    log_level_name = logging.getLevelName(log_level).lower()
+    log_message = repr(exception)
+    if isinstance(exception, ContractCallException):
+        log_message += ": " + repr(exception.orig_exception)
+    rollbar.report_message(log_message, log_level_name, extra_data=extra_data)

From 6143706b9f9fcd6149ed9e42fa7d28fb4f9862e5 Mon Sep 17 00:00:00 2001
From: Sheng Lundquist <sheng@delv.tech>
Date: Tue, 16 Jan 2024 12:01:11 -0800
Subject: [PATCH 5/9] Path independence allows for failed paths, with logging
 on failed paths as info

---
 .../fuzz_path_independence.py                 | 32 ++++++++++++++++---
 1 file changed, 28 insertions(+), 4 deletions(-)

diff --git a/lib/agent0/agent0/interactive_fuzz/fuzz_path_independence.py b/lib/agent0/agent0/interactive_fuzz/fuzz_path_independence.py
index 82606f557a..3e920d673a 100644
--- a/lib/agent0/agent0/interactive_fuzz/fuzz_path_independence.py
+++ b/lib/agent0/agent0/interactive_fuzz/fuzz_path_independence.py
@@ -33,6 +33,8 @@
 from typing import Any, NamedTuple, Sequence
 
 import pandas as pd
+import rollbar
+from ethpy.base.errors import ContractCallException
 from fixedpointmath import FixedPoint
 
 from agent0.hyperdrive.crash_report import build_crash_trade_result, log_hyperdrive_crash_report
@@ -93,7 +95,9 @@ def fuzz_path_independence(
         log_filename,
         chain_config,
         log_to_stdout,
-        log_to_rollbar=False,  # We don't log other crashes to rollbar since we expect failed paths here
+        # Trade crashes in this file have expected failures, hence we log interactive
+        # hyperdrive crashes as info instead of critical.
+        crash_log_level=logging.INFO,
         fees=False,
     )
 
@@ -126,6 +130,8 @@ def fuzz_path_independence(
     check_data: dict[str, Any] | None = None
     first_run_state_dump_dir: str | None = None
     first_run_ticker: pd.DataFrame | None = None
+    invariance_checked: bool = False
+    latest_error: Exception | None = None
     for iteration in range(num_paths_checked):
         print(f"{iteration=}")
         logging.info("iteration %s out of %s", iteration, num_paths_checked - 1)
@@ -136,7 +142,13 @@ def fuzz_path_independence(
         # guarantee closing trades within the same checkpoint by getting the checkpoint id before
         # and after closing trades, then asserting they're the same
         starting_checkpoint_id = interactive_hyperdrive.hyperdrive_interface.calc_checkpoint_id()
-        close_random_trades(trade_events, rng)
+        try:
+            close_random_trades(trade_events, rng)
+        except ContractCallException:
+            # Trades can fail here due to invalid order, we ignore and move on
+            # These trades get logged as info
+            continue
+
         ending_checkpoint_id = interactive_hyperdrive.hyperdrive_interface.calc_checkpoint_id()
         assert starting_checkpoint_id == ending_checkpoint_id
 
@@ -160,6 +172,7 @@ def fuzz_path_independence(
 
         # On subsequent run, check against the saved final state
         else:
+            invariance_checked = True
             # Sanity check, ensure checkpoint id is the same after all runs
             assert ending_checkpoint_id == check_data["curr_checkpoint_id"]
 
@@ -203,8 +216,19 @@ def fuzz_path_independence(
                     log_to_rollbar=True,
                     rollbar_data=rollbar_data,
                 )
-                chain.cleanup()
-                raise error
+                latest_error = error
+
+    # If the number of successful paths < 2, we print and log a warning
+    if not invariance_checked:
+        warning_message = "No invariance checks were performed due to failed paths."
+        logging.warning(warning_message)
+        rollbar.report_message(warning_message, "warning")
+
+    # If any of the path checks broke, we throw an exception at the very end
+    if latest_error is not None:
+        chain.cleanup()
+        raise latest_error
+
     chain.cleanup()
     logging.info("Test passed!")
 

From bdf3ca12f2dae4c9daef0fa4d6d75807d9e6acac Mon Sep 17 00:00:00 2001
From: Sheng Lundquist <sheng@delv.tech>
Date: Tue, 16 Jan 2024 12:02:33 -0800
Subject: [PATCH 6/9] Using more paths checked

---
 lib/agent0/agent0/interactive_fuzz/run_all_fuzz_tests.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/agent0/agent0/interactive_fuzz/run_all_fuzz_tests.py b/lib/agent0/agent0/interactive_fuzz/run_all_fuzz_tests.py
index 4453594554..7441b2e094 100644
--- a/lib/agent0/agent0/interactive_fuzz/run_all_fuzz_tests.py
+++ b/lib/agent0/agent0/interactive_fuzz/run_all_fuzz_tests.py
@@ -30,7 +30,7 @@ def main(argv: Sequence[str] | None = None):
     initialize_rollbar("interactivefuzz")
 
     num_trades = 10
-    num_paths_checked = 10
+    num_paths_checked = 20
 
     num_checks = 0
     while True:

From 2ea3ac7189cfb6a6dad2a046d534666f4611fca4 Mon Sep 17 00:00:00 2001
From: Sheng Lundquist <sheng@delv.tech>
Date: Tue, 16 Jan 2024 12:46:00 -0800
Subject: [PATCH 7/9] Adding ticker and a message prefix to interactive
 hyperdrive rollbar logging

---
 .../hyperdrive/crash_report/crash_report.py       |  5 ++++-
 .../interactive/interactive_hyperdrive.py         | 15 +++++++++++++++
 .../fuzz_long_short_maturity_values.py            |  4 +++-
 .../interactive_fuzz/fuzz_path_independence.py    |  1 +
 .../agent0/interactive_fuzz/fuzz_present_value.py |  8 +++++++-
 .../agent0/interactive_fuzz/fuzz_profit_check.py  |  4 +++-
 .../agent0/interactive_fuzz/helpers/setup_fuzz.py |  5 +++++
 lib/hyperlogs/hyperlogs/rollbar_utilities.py      | 12 ++++++++++--
 8 files changed, 48 insertions(+), 6 deletions(-)

diff --git a/lib/agent0/agent0/hyperdrive/crash_report/crash_report.py b/lib/agent0/agent0/hyperdrive/crash_report/crash_report.py
index df51d8d63e..1e7e8976d4 100644
--- a/lib/agent0/agent0/hyperdrive/crash_report/crash_report.py
+++ b/lib/agent0/agent0/hyperdrive/crash_report/crash_report.py
@@ -186,6 +186,7 @@ def log_hyperdrive_crash_report(
     crash_report_to_file: bool = True,
     crash_report_file_prefix: str | None = None,
     log_to_rollbar: bool = False,
+    rollbar_log_prefix: str | None = None,
     rollbar_data: dict | None = None,
 ) -> None:
     # pylint: disable=too-many-arguments
@@ -207,6 +208,8 @@ def log_hyperdrive_crash_report(
     log_to_rollbar: bool, optional
         If enabled, logs errors to the rollbar service.
         Defaults to False.
+    rollbar_log_prefix: str | None, optional
+        The prefix to prepend to rollbar exception messages
     rollbar_data: dict | None, optional
         Optional dictionary of data to use for the the rollbar report.
         If not provided, will default to logging all of the crash report to rollbar.
@@ -306,7 +309,7 @@ def log_hyperdrive_crash_report(
 
         # Format data
         rollbar_data = json.loads(json.dumps(rollbar_data, indent=2, cls=ExtendedJSONEncoder))
-        log_rollbar_exception(trade_result.exception, log_level, rollbar_data)
+        log_rollbar_exception(trade_result.exception, log_level, rollbar_data, rollbar_log_prefix=rollbar_log_prefix)
 
 
 def _hyperdrive_wallet_to_dict(wallet: HyperdriveWallet | None) -> dict[str, Any]:
diff --git a/lib/agent0/agent0/hyperdrive/interactive/interactive_hyperdrive.py b/lib/agent0/agent0/hyperdrive/interactive/interactive_hyperdrive.py
index 0fdc3a784c..6851ec7bb9 100644
--- a/lib/agent0/agent0/hyperdrive/interactive/interactive_hyperdrive.py
+++ b/lib/agent0/agent0/hyperdrive/interactive/interactive_hyperdrive.py
@@ -97,8 +97,12 @@ class Config:
             Whether to preview the position before executing a trade. Defaults to False.
         log_to_rollbar: bool, optional
             Whether to log crash reports to rollbar. Defaults to False.
+        rollbar_log_prefix: str | None, optional
+            The prefix to prepend to rollbar exception messages
         crash_log_level: int | None, optional
             The log level to log crashes at. Defaults to critical.
+        crash_log_ticker: bool | None, optional
+            Whether to log the trade ticker in crash reports. Defaults to False.
         rng_seed: int | None, optional
             The seed for the random number generator. Defaults to None.
         rng: Generator | None, optional
@@ -145,7 +149,9 @@ class Config:
         data_pipeline_timeout: int = 60
         preview_before_trade: bool = False
         log_to_rollbar: bool = False
+        rollbar_log_prefix: str | None = None
         crash_log_level: int | None = None
+        crash_log_ticker: bool = False
         # Random generators
         rng_seed: int | None = None
         rng: Generator | None = None
@@ -261,7 +267,9 @@ def __init__(self, chain: Chain, config: Config | None = None):
 
         self.rng = config.rng
         self.log_to_rollbar = config.log_to_rollbar
+        self.rollbar_log_prefix = config.rollbar_log_prefix
         self.crash_log_level = config.crash_log_level
+        self.crash_log_ticker = config.crash_log_ticker
 
     def _launch_data_pipeline(self, start_block: int | None = None):
         """Launches the data pipeline in background threads.
@@ -954,6 +962,12 @@ def _handle_trade_result(self, trade_results: list[TradeResult] | TradeResult) -
             # We only get anvil state dump here, since it's an on chain call
             # and we don't want to do it when e.g., slippage happens
             trade_result.anvil_state = get_anvil_state_dump(self.hyperdrive_interface.web3)
+            if self.crash_log_ticker:
+                if trade_result.additional_info is None:
+                    trade_result.additional_info = {"ticker": self.get_ticker()}
+                else:
+                    trade_result.additional_info["ticker"] = self.get_ticker()
+
             # Defaults to CRITICAL
             log_hyperdrive_crash_report(
                 trade_result,
@@ -961,6 +975,7 @@ def _handle_trade_result(self, trade_results: list[TradeResult] | TradeResult) -
                 crash_report_to_file=True,
                 crash_report_file_prefix="interactive_hyperdrive",
                 log_to_rollbar=self.log_to_rollbar,
+                rollbar_log_prefix=self.rollbar_log_prefix,
             )
             raise trade_result.exception
 
diff --git a/lib/agent0/agent0/interactive_fuzz/fuzz_long_short_maturity_values.py b/lib/agent0/agent0/interactive_fuzz/fuzz_long_short_maturity_values.py
index 67136c084c..de2ee510df 100644
--- a/lib/agent0/agent0/interactive_fuzz/fuzz_long_short_maturity_values.py
+++ b/lib/agent0/agent0/interactive_fuzz/fuzz_long_short_maturity_values.py
@@ -70,7 +70,9 @@ def fuzz_long_short_maturity_values(
     # set a large block time so i can manually control when it ticks
     # TODO: set block time really high after contracts deployed:
     # chain_config = LocalChain.Config(block_time=1_000_000)
-    chain, random_seed, rng, interactive_hyperdrive = setup_fuzz(log_filename, chain_config, log_to_stdout)
+    chain, random_seed, rng, interactive_hyperdrive = setup_fuzz(
+        log_filename, chain_config, log_to_stdout, rollbar_log_prefix="fuzz_long_short_maturity_values"
+    )
     signer = interactive_hyperdrive.init_agent(eth=FixedPoint(100))
 
     # Advance time to ensure current time is in the middle of a checkpoint
diff --git a/lib/agent0/agent0/interactive_fuzz/fuzz_path_independence.py b/lib/agent0/agent0/interactive_fuzz/fuzz_path_independence.py
index 3e920d673a..e0756f782d 100644
--- a/lib/agent0/agent0/interactive_fuzz/fuzz_path_independence.py
+++ b/lib/agent0/agent0/interactive_fuzz/fuzz_path_independence.py
@@ -99,6 +99,7 @@ def fuzz_path_independence(
         # hyperdrive crashes as info instead of critical.
         crash_log_level=logging.INFO,
         fees=False,
+        rollbar_log_prefix="fuzz_path_independence",
     )
 
     # Open some trades
diff --git a/lib/agent0/agent0/interactive_fuzz/fuzz_present_value.py b/lib/agent0/agent0/interactive_fuzz/fuzz_present_value.py
index 9e5ba526dd..d16c3eb75b 100644
--- a/lib/agent0/agent0/interactive_fuzz/fuzz_present_value.py
+++ b/lib/agent0/agent0/interactive_fuzz/fuzz_present_value.py
@@ -65,7 +65,13 @@ def fuzz_present_value(
         Defaults to False.
     """
     log_filename = ".logging/fuzz_present_value.log"
-    chain, random_seed, rng, interactive_hyperdrive = setup_fuzz(log_filename, chain_config, log_to_stdout, fees=False)
+    chain, random_seed, rng, interactive_hyperdrive = setup_fuzz(
+        log_filename,
+        chain_config,
+        log_to_stdout,
+        fees=False,
+        rollbar_log_prefix="fuzz_present_value",
+    )
 
     initial_pool_state = interactive_hyperdrive.hyperdrive_interface.current_pool_state
     check_data: dict[str, Any] = {
diff --git a/lib/agent0/agent0/interactive_fuzz/fuzz_profit_check.py b/lib/agent0/agent0/interactive_fuzz/fuzz_profit_check.py
index eff9c7b431..478d25391a 100644
--- a/lib/agent0/agent0/interactive_fuzz/fuzz_profit_check.py
+++ b/lib/agent0/agent0/interactive_fuzz/fuzz_profit_check.py
@@ -61,7 +61,9 @@ def fuzz_profit_check(chain_config: LocalChain.Config | None = None, log_to_stdo
 
     # Setup the environment
     log_filename = ".logging/fuzz_profit_check.log"
-    chain, random_seed, rng, interactive_hyperdrive = setup_fuzz(log_filename, chain_config, log_to_stdout)
+    chain, random_seed, rng, interactive_hyperdrive = setup_fuzz(
+        log_filename, chain_config, log_to_stdout, rollbar_log_prefix="fuzz_profit_check"
+    )
 
     # Get a random trade amount
     long_trade_amount = FixedPoint(
diff --git a/lib/agent0/agent0/interactive_fuzz/helpers/setup_fuzz.py b/lib/agent0/agent0/interactive_fuzz/helpers/setup_fuzz.py
index d3eba4c2b0..e3d007eaa4 100644
--- a/lib/agent0/agent0/interactive_fuzz/helpers/setup_fuzz.py
+++ b/lib/agent0/agent0/interactive_fuzz/helpers/setup_fuzz.py
@@ -15,6 +15,7 @@ def setup_fuzz(
     log_to_stdout: bool = False,
     log_to_rollbar: bool = True,
     crash_log_level: int | None = None,
+    rollbar_log_prefix: str | None = None,
     fees=True,
     var_interest=None,
 ) -> tuple[LocalChain, int, Generator, InteractiveHyperdrive]:
@@ -34,6 +35,8 @@ def setup_fuzz(
         If True, log errors rollbar. Defaults to True.
     crash_log_level: int | None, optional
         The log level to log crashes at. Defaults to critical.
+    rollbar_log_prefix: str | None, optional
+        The prefix to prepend to rollbar exception messages
     fees: bool, optional
         If False, will turn off fees when deploying hyperdrive. Defaults to True.
     var_interest: FixedPoint | None, optional
@@ -74,7 +77,9 @@ def setup_fuzz(
         preview_before_trade=True,
         checkpoint_duration=86400,
         log_to_rollbar=log_to_rollbar,
+        rollbar_log_prefix=rollbar_log_prefix,
         crash_log_level=crash_log_level,
+        crash_log_ticker=True,
     )
     if not fees:
         initial_pool_config.curve_fee = FixedPoint(0)
diff --git a/lib/hyperlogs/hyperlogs/rollbar_utilities.py b/lib/hyperlogs/hyperlogs/rollbar_utilities.py
index cae939c53e..31989f0bae 100644
--- a/lib/hyperlogs/hyperlogs/rollbar_utilities.py
+++ b/lib/hyperlogs/hyperlogs/rollbar_utilities.py
@@ -67,7 +67,9 @@ def log_rollbar_message(message: str, log_level: int, extra_data: dict | None =
     rollbar.report_message(message, log_level_name, extra_data=extra_data)
 
 
-def log_rollbar_exception(exception: Exception, log_level: int, extra_data: dict | None = None):
+def log_rollbar_exception(
+    exception: Exception, log_level: int, extra_data: dict | None = None, rollbar_log_prefix: str | None = None
+):
     """Logs an exception to the rollbar service.
 
     Arguments
@@ -78,9 +80,15 @@ def log_rollbar_exception(exception: Exception, log_level: int, extra_data: dict
         The logging level enum value.
     extra_data: dict, optional.
         Extra data to send to rollbar. This usually contains the custom crash report json
+    rollbar_log_prefix: str, optional
+        The prefix to prepend to rollbar exception messages
     """
     log_level_name = logging.getLevelName(log_level).lower()
-    log_message = repr(exception)
+    if rollbar_log_prefix is None:
+        log_message = ""
+    else:
+        log_message = rollbar_log_prefix + ": "
+    log_message += repr(exception)
     if isinstance(exception, ContractCallException):
         log_message += ": " + repr(exception.orig_exception)
     rollbar.report_message(log_message, log_level_name, extra_data=extra_data)

From de043235f46882d03c70cf4081fd35a45101aed3 Mon Sep 17 00:00:00 2001
From: Sheng Lundquist <sheng@delv.tech>
Date: Tue, 16 Jan 2024 12:47:32 -0800
Subject: [PATCH 8/9] Catching and printing unexpected errors

---
 .../agent0/interactive_fuzz/run_all_fuzz_tests.py      | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/lib/agent0/agent0/interactive_fuzz/run_all_fuzz_tests.py b/lib/agent0/agent0/interactive_fuzz/run_all_fuzz_tests.py
index 7441b2e094..906c0603ef 100644
--- a/lib/agent0/agent0/interactive_fuzz/run_all_fuzz_tests.py
+++ b/lib/agent0/agent0/interactive_fuzz/run_all_fuzz_tests.py
@@ -40,6 +40,10 @@ def main(argv: Sequence[str] | None = None):
             fuzz_long_short_maturity_values(num_trades, chain_config)
         except FuzzAssertionException:
             pass
+        # We catch other exceptions here, for some reason rollbar needs to be continuously running in order
+        # to log.
+        except Exception:  # pylint: disable=broad-except
+            print("Unexpected error:", sys.exc_info()[0])
 
         try:
             print("Running path independence test")
@@ -51,6 +55,7 @@ def main(argv: Sequence[str] | None = None):
             )
         except FuzzAssertionException:
             pass
+        # No need to catch other exceptions here, the test itself catches them
 
         try:
             print("Running fuzz profit test")
@@ -58,6 +63,8 @@ def main(argv: Sequence[str] | None = None):
             fuzz_profit_check(chain_config)
         except FuzzAssertionException:
             pass
+        except Exception:  # pylint: disable=broad-except
+            print("Unexpected error:", sys.exc_info()[0])
 
         try:
             print("Running fuzz present value test")
@@ -66,6 +73,9 @@ def main(argv: Sequence[str] | None = None):
             fuzz_present_value(test_epsilon=present_value_epsilon, chain_config=chain_config)
         except FuzzAssertionException:
             pass
+        except Exception:  # pylint: disable=broad-except
+            print("Unexpected error:", sys.exc_info()[0])
+
         num_checks += 1
         if parsed_args.number_of_runs > 0 and num_checks >= parsed_args.number_of_runs:
             break

From 04cc04d12073bb662b00994062ad4a12fb048efa Mon Sep 17 00:00:00 2001
From: Sheng Lundquist <sheng@delv.tech>
Date: Tue, 16 Jan 2024 13:27:43 -0800
Subject: [PATCH 9/9] Explicitly setting crash log level instead of setting to
 none

---
 .../agent0/hyperdrive/interactive/interactive_hyperdrive.py  | 5 +++--
 lib/agent0/agent0/interactive_fuzz/helpers/setup_fuzz.py     | 5 +++++
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/lib/agent0/agent0/hyperdrive/interactive/interactive_hyperdrive.py b/lib/agent0/agent0/hyperdrive/interactive/interactive_hyperdrive.py
index 6851ec7bb9..334661e557 100644
--- a/lib/agent0/agent0/hyperdrive/interactive/interactive_hyperdrive.py
+++ b/lib/agent0/agent0/hyperdrive/interactive/interactive_hyperdrive.py
@@ -2,6 +2,7 @@
 from __future__ import annotations
 
 import asyncio
+import logging
 import os
 import time
 from dataclasses import asdict, dataclass
@@ -99,7 +100,7 @@ class Config:
             Whether to log crash reports to rollbar. Defaults to False.
         rollbar_log_prefix: str | None, optional
             The prefix to prepend to rollbar exception messages
-        crash_log_level: int | None, optional
+        crash_log_level: int, optional
             The log level to log crashes at. Defaults to critical.
         crash_log_ticker: bool | None, optional
             Whether to log the trade ticker in crash reports. Defaults to False.
@@ -150,7 +151,7 @@ class Config:
         preview_before_trade: bool = False
         log_to_rollbar: bool = False
         rollbar_log_prefix: str | None = None
-        crash_log_level: int | None = None
+        crash_log_level: int = logging.CRITICAL
         crash_log_ticker: bool = False
         # Random generators
         rng_seed: int | None = None
diff --git a/lib/agent0/agent0/interactive_fuzz/helpers/setup_fuzz.py b/lib/agent0/agent0/interactive_fuzz/helpers/setup_fuzz.py
index e3d007eaa4..8c7e4c9d8c 100644
--- a/lib/agent0/agent0/interactive_fuzz/helpers/setup_fuzz.py
+++ b/lib/agent0/agent0/interactive_fuzz/helpers/setup_fuzz.py
@@ -1,6 +1,8 @@
 """Setup an interactive enfironment for fuzz testing."""
 from __future__ import annotations
 
+import logging
+
 import numpy as np
 from fixedpointmath import FixedPoint
 from hyperlogs import setup_logging
@@ -73,6 +75,9 @@ def setup_fuzz(
 
     # Parameters for pool initialization.
     # Using a day for checkpoint duration to speed things up
+    if crash_log_level is None:
+        crash_log_level = logging.CRITICAL
+
     initial_pool_config = InteractiveHyperdrive.Config(
         preview_before_trade=True,
         checkpoint_duration=86400,