diff --git a/lib/agent0/agent0/interactive_fuzz/__init__.py b/lib/agent0/agent0/interactive_fuzz/__init__.py index d1f6420a2c..676f72d5c2 100644 --- a/lib/agent0/agent0/interactive_fuzz/__init__.py +++ b/lib/agent0/agent0/interactive_fuzz/__init__.py @@ -1,6 +1,5 @@ """Shared functions for interactive fuzz testing.""" -from .fuzz_hyperdrive_balance import fuzz_hyperdrive_balance from .fuzz_long_short_maturity_values import fuzz_long_short_maturity_values from .fuzz_path_independence import fuzz_path_independence from .fuzz_profit_check import fuzz_profit_check diff --git a/lib/agent0/agent0/interactive_fuzz/fuzz_hyperdrive_balance.py b/lib/agent0/agent0/interactive_fuzz/fuzz_hyperdrive_balance.py deleted file mode 100644 index e55fb28bf8..0000000000 --- a/lib/agent0/agent0/interactive_fuzz/fuzz_hyperdrive_balance.py +++ /dev/null @@ -1,232 +0,0 @@ -"""Fuzz test to verify that the Hyperdrive share reserves are valid after trades are opened and then closed. - -# Test procedure -- spin up local chain, deploy hyperdrive -- get initial_pool_state -- generate a list of random trades - - type in [open_short, open_long] - - amount in uniform[min_trade_amount, max_trade_amount) base -- open those trades in a random order & advance time randomly between - - total time advanced in uniform[0, position_duration) -- close the trades in a random order -- invariance checks - -# Invariance checks (these should be True): -- current effective share reserves calculated from hyperdrivepy == initial share reserves -- share resserves >= minimum share reserves -""" -from __future__ import annotations - -import argparse -import logging -import sys -from typing import Any, NamedTuple, Sequence - -from fixedpointmath import FixedPoint - -from agent0.hyperdrive.crash_report import build_crash_trade_result, log_hyperdrive_crash_report -from agent0.hyperdrive.interactive import InteractiveHyperdrive, LocalChain -from agent0.interactive_fuzz.helpers import ( - FuzzAssertionException, - close_random_trades, - execute_random_trades, - setup_fuzz, -) - -# main script has a lot of stuff going on -# pylint: disable=too-many-locals - - -def main(argv: Sequence[str] | None = None): - """Primary entrypoint. - - Arguments - --------- - argv: Sequence[str] - The argv values returned from argparser. - """ - # Setup the environment - parsed_args = parse_arguments(argv) - fuzz_hyperdrive_balance(*parsed_args) - - -def fuzz_hyperdrive_balance(num_trades: int, chain_config: LocalChain.Config, log_to_stdout: bool = False): - """Does fuzzy invariant checks on the hyperdrive contract's balances. - - Parameters - ---------- - num_trades: int - Number of trades to perform during the fuzz tests. - chain_config: LocalChain.Config, optional - Configuration options for the local chain. - log_to_stdout: bool, optional - If True, log to stdout in addition to a file. - Defaults to False. - """ - - log_filename = ".logging/fuzz_hyperdrive_balance.log" - chain, random_seed, rng, interactive_hyperdrive = setup_fuzz(log_filename, chain_config, log_to_stdout) - - # Get initial vault shares - pool_state = interactive_hyperdrive.hyperdrive_interface.get_hyperdrive_state() - initial_effective_share_reserves = interactive_hyperdrive.hyperdrive_interface.calc_effective_share_reserves( - pool_state - ) - - # Open some trades - trade_events = execute_random_trades(num_trades, chain, rng, interactive_hyperdrive, advance_time=True) - - # Close the trades - close_random_trades(trade_events, rng) - - assert len(trade_events) > 0 - agent = trade_events[0][0] - - # Check the reserve amounts; they should be unchanged now that all of the trades are closed - try: - invariant_check(initial_effective_share_reserves, interactive_hyperdrive) - except FuzzAssertionException as error: - dump_state_dir = chain.save_state(save_prefix="fuzz_long_short_maturity_values") - # The additional information going into the crash report - additional_info = { - "fuzz_random_seed": random_seed, - "dump_state_dir": dump_state_dir, - "trade_ticker": interactive_hyperdrive.get_ticker(), - } - additional_info.update(error.exception_data) - # The subset of information going into rollbar - rollbar_data = { - "fuzz_random_seed": random_seed, - "dump_state_dir": dump_state_dir, - } - rollbar_data.update(error.exception_data) - - report = build_crash_trade_result( - error, interactive_hyperdrive.hyperdrive_interface, agent.agent, additional_info=additional_info - ) - # Crash reporting already going to file in logging - log_hyperdrive_crash_report( - report, - crash_report_to_file=True, - crash_report_file_prefix="fuzz_hyperdrive_balance", - log_to_rollbar=True, - rollbar_data=rollbar_data, - ) - chain.cleanup() - raise error - chain.cleanup() - - -class Args(NamedTuple): - """Command line arguments for the invariant checker.""" - - num_trades: int - chain_config: LocalChain.Config - log_to_stdout: bool - - -def namespace_to_args(namespace: argparse.Namespace) -> Args: - """Converts argprase.Namespace to Args. - - Arguments - --------- - namespace: argparse.Namespace - Object for storing arg attributes. - - Returns - ------- - Args - Formatted arguments - """ - return Args( - num_trades=namespace.num_trades, - chain_config=LocalChain.Config(chain_port=namespace.chain_port), - log_to_stdout=namespace.log_to_stdout, - ) - - -def parse_arguments(argv: Sequence[str] | None = None) -> Args: - """Parses input arguments. - - Arguments - --------- - argv: Sequence[str] - The argv values returned from argparser. - - Returns - ------- - Args - Formatted arguments - """ - parser = argparse.ArgumentParser(description="Runs a loop to check Hyperdrive invariants at each block.") - parser.add_argument( - "--num_trades", - type=int, - default=5, - help="The number of random trades to open.", - ) - parser.add_argument( - "--chain_port", - type=int, - default=10000, - help="The port to use for the local chain.", - ) - parser.add_argument( - "--log_to_stdout", - type=bool, - default=False, - help="If True, log to stdout in addition to a file.", - ) - # Use system arguments if none were passed - if argv is None: - argv = sys.argv - return namespace_to_args(parser.parse_args()) - - -def invariant_check( - initial_effective_share_reserves: FixedPoint, - interactive_hyperdrive: InteractiveHyperdrive, -) -> None: - """Check the pool state invariants and throws an assertion exception if fails. - - Arguments - --------- - initial_effective_share_reserves: FixedPoint - The effective share reserves of the Hyperdrive pool when it was deployed. - interactive_hyperdrive: InteractiveHyperdrive - An instantiated InteractiveHyperdrive object. - """ - failed = False - exception_message: list[str] = ["Fuzz Hyperdrive Balance Invariant Check"] - exception_data: dict[str, Any] = {} - - pool_state = interactive_hyperdrive.hyperdrive_interface.get_hyperdrive_state() - - effective_share_reserves = interactive_hyperdrive.hyperdrive_interface.calc_effective_share_reserves(pool_state) - if effective_share_reserves != initial_effective_share_reserves: - difference_in_wei = abs(effective_share_reserves.scaled_value - initial_effective_share_reserves.scaled_value) - exception_message.append( - f"{effective_share_reserves=} != {initial_effective_share_reserves=}, {difference_in_wei=}" - ) - exception_data["invariance_check:effective_share_reserves"] = effective_share_reserves - exception_data["invariance_check:initial_effective_share_reserves"] = initial_effective_share_reserves - exception_data["invariance_check:effective_share_reserves_difference_in_wei"] = difference_in_wei - failed = True - - share_reserves = pool_state.pool_info.share_reserves - minimum_share_reserves = pool_state.pool_config.minimum_share_reserves - if share_reserves < minimum_share_reserves: - difference_in_wei = abs(share_reserves.scaled_value - minimum_share_reserves.scaled_value) - exception_message.append(f"{share_reserves=} < {minimum_share_reserves=}") - exception_data["invariance_check:share_reserves"] = share_reserves - exception_data["invariance_check:minimum_share_reserves"] = minimum_share_reserves - exception_data["invariance_check:share_reserves_difference_in_wei"] = difference_in_wei - failed = True - - if failed: - logging.critical("\n".join(exception_message)) - raise FuzzAssertionException(*exception_message, exception_data=exception_data) - - -if __name__ == "__main__": - main() diff --git a/lib/agent0/agent0/interactive_fuzz/fuzz_path_independence.py b/lib/agent0/agent0/interactive_fuzz/fuzz_path_independence.py index c5b77d02ef..acfa469668 100644 --- a/lib/agent0/agent0/interactive_fuzz/fuzz_path_independence.py +++ b/lib/agent0/agent0/interactive_fuzz/fuzz_path_independence.py @@ -63,6 +63,7 @@ def main(argv: Sequence[str] | None = None): def fuzz_path_independence( num_trades: int, num_paths_checked: int, + effective_share_reserves_epsilon: float, present_value_epsilon: float, chain_config: LocalChain.Config, log_to_stdout: bool = False, @@ -75,6 +76,8 @@ def fuzz_path_independence( Number of trades to perform during the fuzz tests. num_paths_checked: int Number of paths (order of operations for opening/closing) to perform. + effective_share_reserves_epsilon: float + The allowed error for effective share reserves equality tests. present_value_epsilon: float The allowed error for present value equality tests. chain_config: LocalChain.Config, optional @@ -84,10 +87,9 @@ def fuzz_path_independence( Defaults to False. """ # pylint: disable=too-many-statements + # pylint: disable=too-many-arguments log_filename = ".logging/fuzz_path_independence.log" - chain, random_seed, rng, interactive_hyperdrive = setup_fuzz( - log_filename, chain_config, log_to_stdout, fees=False, var_interest=FixedPoint(0) - ) + chain, random_seed, rng, interactive_hyperdrive = setup_fuzz(log_filename, chain_config, log_to_stdout, fees=False) # Open some trades logging.info("Open random trades...") @@ -95,6 +97,9 @@ def fuzz_path_independence( assert len(trade_events) > 0 agent = trade_events[0][0] + # All positions open, we set variable rate to 0 for closing all positions + interactive_hyperdrive.set_variable_rate(FixedPoint(0)) + # Snapshot the chain, so we can load the snapshot & close in different orders logging.info("Save chain snapshot...") chain.save_snapshot() @@ -157,7 +162,9 @@ def fuzz_path_independence( # Raise an error if it failed assert first_run_state_dump_dir is not None try: - invariant_check(check_data, present_value_epsilon, interactive_hyperdrive) + invariant_check( + check_data, effective_share_reserves_epsilon, present_value_epsilon, interactive_hyperdrive + ) except FuzzAssertionException as error: dump_state_dir = chain.save_state(save_prefix="fuzz_path_independence") @@ -201,6 +208,7 @@ class Args(NamedTuple): num_trades: int num_paths_checked: int + effective_share_reserves_epsilon: float present_value_epsilon: float chain_config: LocalChain.Config log_to_stdout: bool @@ -222,6 +230,7 @@ def namespace_to_args(namespace: argparse.Namespace) -> Args: return Args( num_trades=namespace.num_trades, num_paths_checked=namespace.num_paths_checked, + effective_share_reserves_epsilon=namespace.effective_share_reserves_epsilon, present_value_epsilon=namespace.present_value_epsilon, chain_config=LocalChain.Config(chain_port=namespace.chain_port), log_to_stdout=namespace.log_to_stdout, @@ -254,6 +263,12 @@ def parse_arguments(argv: Sequence[str] | None = None) -> Args: default=10, help="The port to use for the local chain.", ) + parser.add_argument( + "--effective_share_reserves_epsilon", + type=float, + default=1e-4, + help="The allowed error for effective share reserves equality tests.", + ) parser.add_argument( "--present_value_epsilon", type=float, @@ -280,6 +295,7 @@ def parse_arguments(argv: Sequence[str] | None = None) -> Args: def invariant_check( check_data: dict[str, Any], + effective_share_reserves_epsilon: float, present_value_epsilon: float, interactive_hyperdrive: InteractiveHyperdrive, ) -> None: @@ -289,6 +305,8 @@ def invariant_check( --------- check_data: dict[str, Any] The trade data to check. + effective_share_reserves_epsilon: float + The allowed error for effective share reserves equality tests. present_value_epsilon: float The allowed error for present value equality tests. interactive_hyperdrive: InteractiveHyperdrive @@ -305,7 +323,11 @@ def invariant_check( actual_effective_share_reserves = interactive_hyperdrive.hyperdrive_interface.calc_effective_share_reserves( pool_state ) - if expected_effective_share_reserves != actual_effective_share_reserves: + if not fp_isclose( + expected_effective_share_reserves, + actual_effective_share_reserves, + abs_tol=FixedPoint(str(effective_share_reserves_epsilon)), + ): difference_in_wei = abs( expected_effective_share_reserves.scaled_value - actual_effective_share_reserves.scaled_value ) diff --git a/lib/agent0/agent0/interactive_fuzz/helpers/execute_random_trades.py b/lib/agent0/agent0/interactive_fuzz/helpers/execute_random_trades.py index 5d8561ac30..54c1fa0568 100644 --- a/lib/agent0/agent0/interactive_fuzz/helpers/execute_random_trades.py +++ b/lib/agent0/agent0/interactive_fuzz/helpers/execute_random_trades.py @@ -81,6 +81,7 @@ def _get_open_trade_amount( rng: Generator, interactive_hyperdrive: InteractiveHyperdrive, max_budget: FixedPoint = FixedPoint("1e9"), + percent_max: FixedPoint = FixedPoint("0.75"), ) -> FixedPoint: """Get a trade amount for a given trade and Hyperdrive pool. @@ -94,6 +95,8 @@ def _get_open_trade_amount( An instantiated InteractiveHyperdrive object. max_budget: FixedPoint, optional An optional amount to set an upper bound for the trade, defaults to FixedPoint("1e9"). + percent_max: FixedPoint, optional + A percentage of the max trade to use for the upper bound for the trade, defaults to FixedPoint("0.75"). Returns ------- @@ -112,6 +115,8 @@ def _get_open_trade_amount( ) case _: raise ValueError(f"Invalid {trade_type=}\nOnly opening trades are allowed.") + + max_trade = max_trade * percent_max return FixedPoint(scaled_value=int(np.floor(rng.uniform(low=min_trade.scaled_value, high=max_trade.scaled_value)))) diff --git a/lib/agent0/agent0/interactive_fuzz/run_all_fuzz_tests.py b/lib/agent0/agent0/interactive_fuzz/run_all_fuzz_tests.py index e158553eba..a2b9c7d1ca 100644 --- a/lib/agent0/agent0/interactive_fuzz/run_all_fuzz_tests.py +++ b/lib/agent0/agent0/interactive_fuzz/run_all_fuzz_tests.py @@ -8,12 +8,7 @@ from hyperlogs.rollbar_utilities import initialize_rollbar from agent0.hyperdrive.interactive.chain import LocalChain -from agent0.interactive_fuzz import ( - fuzz_hyperdrive_balance, - fuzz_long_short_maturity_values, - fuzz_path_independence, - fuzz_profit_check, -) +from agent0.interactive_fuzz import fuzz_long_short_maturity_values, fuzz_path_independence, fuzz_profit_check from agent0.interactive_fuzz.helpers import FuzzAssertionException @@ -34,13 +29,6 @@ def main(argv: Sequence[str] | None = None): num_checks = 0 while True: - try: - print("Running hyperdrive balance test") - chain_config = LocalChain.Config(db_port=5433, chain_port=10000) - fuzz_hyperdrive_balance(num_trades, chain_config) - except FuzzAssertionException: - pass - try: print("Running long short maturity test") chain_config = LocalChain.Config(db_port=5434, chain_port=10001) @@ -51,8 +39,11 @@ def main(argv: Sequence[str] | None = None): try: print("Running path independence test") chain_config = LocalChain.Config(db_port=5435, chain_port=10002) + effective_share_reserves_epsilon = 1e-4 present_value_epsilon = 1e-4 - fuzz_path_independence(num_trades, num_paths_checked, present_value_epsilon, chain_config) + fuzz_path_independence( + num_trades, num_paths_checked, effective_share_reserves_epsilon, present_value_epsilon, chain_config + ) except FuzzAssertionException: pass