Lazy timestamp reader (#134)

- Create `extract_timestamps_json_rpc_lazy` that instead of reading block timestamps upfront for the given range,
  only calls JSON-RPC API when requested. It works on the cases where sparse event data is read over long block range
  and it is likely only few timestamps need to be fetched in this range.

CHANGELOG.md

@@ -1,3 +1,9 @@
+# 0.21.5
+
+- Create `extract_timestamps_json_rpc_lazy` that instead of reading block timestamps upfront for the given range,
+ only calls JSON-RPC API when requested. It works on the cases where sparse event data is read over long block range
+ and it is likely only few timestamps need to be fetched in this range.
+
 # 0.21.4
 
 - Added `eth_defi.enzyme.erc_20` helpers

eth_defi/enzyme/erc20.py

@@ -9,13 +9,7 @@
 from eth_defi.enzyme.vault import Vault
 
 
-def prepare_transfer(
- enzyme: EnzymeDeployment,
- vault: Vault,
- generic_adapter: Contract,
- token: Contract,
- receiver: HexAddress | str,
- amount: int) -> ContractFunction:
+def prepare_transfer(enzyme: EnzymeDeployment, vault: Vault, generic_adapter: Contract, token: Contract, receiver: HexAddress | str, amount: int) -> ContractFunction:
  """Prepare an ERC-20 transfer out from the Enzyme vault.
 
  - Tells the Enzyme vault to move away som etokes
@@ -53,9 +47,7 @@ def prepare_transfer(
 
  bound_call = execute_calls_for_generic_adapter(
  comptroller=vault.comptroller,
- external_calls=(
- (token, encoded_transfer),
- ),
+ external_calls=((token, encoded_transfer),),
  generic_adapter=generic_adapter,
  incoming_assets=incoming_assets,
  integration_manager=enzyme.contracts.integration_manager,

eth_defi/enzyme/uniswap_v2.py

@@ -99,5 +99,3 @@ def prepare_swap(enzyme: EnzymeDeployment, vault: Vault, uniswap_v2: UniswapV2De
  )
 
  return bound_call
-
-

eth_defi/event_reader/lazy_timestamp_reader.py

@@ -0,0 +1,127 @@
+"""Lazily load block timestamps and headers.
+
+See :py:func:`extract_timestamps_json_rpc_lazy`
+"""
+from hexbytes import HexBytes
+
+from eth_defi.event_reader.conversion import convert_jsonrpc_value_to_int
+from eth_typing import HexStr
+from web3 import Web3
+from web3.types import BlockIdentifier
+
+
+class OutOfSpecifiedRangeRead(Exception):
+ """We tried to read a block outside out original given range."""
+
+
+class LazyTimestampContainer:
+ """Dictionary-like object to get block timestamps on-demand.
+
+ Lazily load any block timestamp over JSON-RPC API if we have not
+ cached it yet.
+
+ See :py:func:`extract_timestamps_json_rpc_lazy`.
+ """
+
+ def __init__(self, web3: Web3, start_block: int, end_block: int):
+ """
+
+ :param web3:
+ Connection
+
+ :param start_block:
+ Start block range, inclusive
+
+ :param end_block:
+ End block range, inclusive
+ """
+ self.web3 = web3
+ self.start_block = start_block
+ self.end_block = end_block
+ assert start_block > 0
+ assert end_block >= start_block
+ self.cache_by_block_hash = {}
+ self.cache_by_block_number = {}
+
+ def update_block_hash(self, block_identifier: BlockIdentifier) -> int:
+ # Skip web3.py stack of slow result formatters
+ if type(block_identifier) == int:
+ assert block_identifier > 0
+ result = self.web3.manager.request_blocking("eth_getBlockByNumber", (block_identifier, False))
+ else:
+ if isinstance(block_identifier, HexBytes):
+ block_identifier = block_identifier.hex()
+ result = self.web3.manager.request_blocking("eth_getBlockByHash", (block_identifier, False))
+
+ # Note to self: block_number = 0 for the genesis block on Anvil
+ block_number = convert_jsonrpc_value_to_int(result["number"])
+ hash = result["hash"]
+
+ # Make sure we conform the spec
+ if not (self.start_block <= block_number <= self.end_block):
+ raise OutOfSpecifiedRangeRead(f"Read block number {block_number:,} {hash} out of bounds of range {self.start_block:,} - {self.end_block:,}")
+
+ timestamp = convert_jsonrpc_value_to_int(result["timestamp"])
+ self.cache_by_block_hash[hash] = timestamp
+ self.cache_by_block_number[block_number] = timestamp
+ return timestamp
+
+ def __getitem__(self, block_hash: HexStr | HexBytes | str):
+ """Get a timestamp of a block hash."""
+ assert not type(block_hash) == int, f"Use block hashes, block numbers not supported, passed {block_hash}"
+
+ assert type(block_hash) == str or isinstance(block_hash, HexBytes), f"Got: {block_hash} {block_hash.__class__}"
+
+ if type(block_hash) != str:
+ block_hash = block_hash.hex()
+
+ if block_hash not in self.cache_by_block_hash:
+ self.update_block_hash(block_hash)
+
+ return self.cache_by_block_hash[block_hash]
+
+
+def extract_timestamps_json_rpc_lazy(
+ web3: Web3,
+ start_block: int,
+ end_block: int,
+ fetch_boundaries=True,
+) -> LazyTimestampContainer:
+ """Create a cache container that instead of reading block timestamps upfront for the given range, only calls JSON-RPC API when requested
+
+ - Works on the cases where sparse event data is read over long block range
+ Use slow JSON-RPC block headers call to get this information.
+
+ - The reader is hash based. It is mainly meant to resolve `eth_getLogs` resulting block hashes to
+ corresponding event timestamps.
+
+ - This is a drop-in replacement for the dict returned by eager :py:func:`eth_defi.reader.extract_timestamps_json_rpc`
+
+ Example:
+
+ .. code-block:: python
+
+ # Allocate timestamp reader for blocks 1...100
+ timestamps = extract_timestamps_json_rpc_lazy(web3, 1, 100)
+
+ # Get a hash of some block
+ block_hash = web3.eth.get_block(5)["hash"]
+
+ # Read timestamp for block 5
+ unix_time = timestamps[block_hash]
+
+ For more information see
+
+ - :py:func:`eth_defi.reader.extract_timestamps_json_rpc`
+
+ - :py:class:`eth_defi.reorganisation_monitor.ReorganisationMonitor`
+
+ :return:
+ Wrapper object for block hash based timestamp access.
+
+ """
+ container = LazyTimestampContainer(web3, start_block, end_block)
+ if fetch_boundaries:
+ container.update_block_hash(start_block)
+ container.update_block_hash(end_block)
+ return container

eth_defi/revert_reason.py

@@ -145,11 +145,5 @@ def fetch_transaction_revert_reason(
  current_block_number = web3.eth.block_number
  # TODO: Convert to logger record
  pretty_result = pprint.pformat(result)
- logger.error(f"Transaction succeeded, when we tried to fetch its revert reason.\n"
- f"Hash: {tx_hash.hex()}, tx block num: {tx['blockNumber']}, current block number: {current_block_number}\n"
- f"Transaction result:\n"
- f"{pretty_result}\n"
- f"- Maybe the chain tip is unstable\n"
- f"- Maybe transaction failed due to slippage\n"
- f"- Maybe someone is frontrunning you and it does not happen with eth_call replay\n")
+ logger.error(f"Transaction succeeded, when we tried to fetch its revert reason.\n" f"Hash: {tx_hash.hex()}, tx block num: {tx['blockNumber']}, current block number: {current_block_number}\n" f"Transaction result:\n" f"{pretty_result}\n" f"- Maybe the chain tip is unstable\n" f"- Maybe transaction failed due to slippage\n" f"- Maybe someone is frontrunning you and it does not happen with eth_call replay\n")
  return unknown_error_message

eth_defi/tx.py

@@ -112,10 +112,7 @@ def __mul__(self, other: float | int) -> "AssetDelta":
  assert d2.raw_amount == int(10**6 * 0.99)
  """
  assert isinstance(other, (float, int))
- return AssetDelta(
- self.asset,
- int(self.raw_amount * other)
- )
+ return AssetDelta(self.asset, int(self.raw_amount * other))
 
  def is_incoming(self) -> bool:
  """This delta describes incoming assets."""

tests/enzyme/test_vault_controlled_wallet.py

@@ -95,7 +95,7 @@ def deployment(
 def test_asset_delta_mul(usdc: Contract):
  """Check that the asset delta multiplier works."""
 
- d = AssetDelta(usdc.address, 1*10**6)
+ d = AssetDelta(usdc.address, 1 * 10**6)
  d2 = d * 0.99
  assert d2.raw_amount == int(10**6 * 0.99)
 

tests/test_lazy_timestamp_reader.py

@@ -0,0 +1,75 @@
+"""Tests for lazy_timestamp_reader.py
+
+"""
+import pytest
+from eth_defi.anvil import launch_anvil, AnvilLaunch, mine
+from eth_defi.chain import install_chain_middleware
+from web3 import HTTPProvider, Web3
+
+from eth_defi.event_reader.lazy_timestamp_reader import extract_timestamps_json_rpc_lazy, LazyTimestampContainer, OutOfSpecifiedRangeRead
+
+
+@pytest.fixture()
+def anvil() -> AnvilLaunch:
+ """Launch Anvil for the test backend."""
+
+ anvil = launch_anvil()
+ try:
+ yield anvil
+ finally:
+ anvil.close()
+
+
+@pytest.fixture()
+def web3(anvil: AnvilLaunch) -> Web3:
+ """Set up the Anvil Web3 connection.
+
+ Also perform the Anvil state reset for each test.
+ """
+
+ provider = HTTPProvider(anvil.json_rpc_url)
+
+ # Web3 6.0 fixes - force no middlewares
+ provider.middlewares = (
+ # attrdict_middleware,
+ # default_transaction_fields_middleware,
+ # ethereum_tester_middleware,
+ )
+
+ web3 = Web3(provider)
+ # Get rid of attributeddict slow down
+ web3.middleware_onion.clear()
+ install_chain_middleware(web3)
+ return web3
+
+
+def test_lazy_timestamp_reader_block_range(web3: Web3):
+ """Read timestamps lazily."""
+
+ # Create some blocks
+ for i in range(1, 5 + 1):
+ mine(web3)
+
+ assert web3.eth.block_number == 5
+ timestamps = extract_timestamps_json_rpc_lazy(web3, 1, 5)
+ assert isinstance(timestamps, LazyTimestampContainer)
+
+ for i in range(1, 5 + 1):
+ block_hash = web3.eth.get_block(i)["hash"]
+ assert timestamps[block_hash] > 0
+
+
+def test_lazy_timestamp_reader_out_of_block_range(web3: Web3):
+ """Read timestamps lazily, but peek out of allowed range."""
+
+ # Create some blocks
+ for i in range(1, 5 + 1):
+ mine(web3)
+
+ assert web3.eth.block_number == 5
+ timestamps = extract_timestamps_json_rpc_lazy(web3, 1, 4)
+ assert isinstance(timestamps, LazyTimestampContainer)
+
+ with pytest.raises(OutOfSpecifiedRangeRead):
+ block_hash = web3.eth.get_block(5)["hash"]
+ timestamps[block_hash]