diff --git a/CHANGELOG.md b/CHANGELOG.md index 01330c26..cd745ea2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ # Current +- Adds support for extracting block data using Ankr +- Adds `AnkrReorganizationMonitor` - Add: Aave v3 reserve data queries # 0.22.7 diff --git a/eth_defi/chain.py b/eth_defi/chain.py index 66e3d328..85d8db94 100644 --- a/eth_defi/chain.py +++ b/eth_defi/chain.py @@ -235,3 +235,17 @@ def install_retry_middleware(web3: Web3): gracefully do exponential backoff retries. """ web3.middleware_onion.inject(http_retry_request_with_sleep_middleware, layer=0) + + +def has_ankr_support(provider: HTTPProvider) -> bool: + """Check if a node is an Ankr node. + """ + + assert isinstance(provider, HTTPProvider) + + base_url = provider.endpoint_uri + + if "https://rpc.ankr.com/" not in base_url: + return False + else: + return True \ No newline at end of file diff --git a/eth_defi/event_reader/ankr.py b/eth_defi/event_reader/ankr.py new file mode 100644 index 00000000..4b43ef1b --- /dev/null +++ b/eth_defi/event_reader/ankr.py @@ -0,0 +1,89 @@ +import requests +import json +from enum import Enum + +from eth_defi.event_reader.block_header import BlockHeader + + +class AnkrSupportedBlockchain(Enum): + eth = "eth" + bsc = "bsc" + polygon = "polygon" + fantom = "fantom" + arbitrum = "arbitrum" + avalanche = "avalanche" + syscoin = "syscoin" + + +def make_block_request_ankr(endpoint_url: str = "https://rpc.ankr.com/multichain/79258ce7f7ee046decc3b5292a24eb4bf7c910d7e39b691384c7ce0cfb839a01/", start_block: int | str = "latest", end_block: int | str = "latest", blockchain: AnkrSupportedBlockchain | None = None, request_id: int = 1) -> list[dict]: + """Fetch blocks from Ankr API + + :param endpoint_url: URL of Ankr API endpoint. Should be multichain endpoint. + + :param start_block: Block number to start fetching from. Can be an int or "latest". + + :param end_block: Block number to end fetching at. Can be an int or "latest". + + :param blockchain: Blockchain to fetch blocks from. Must be of type AnkrSupportedBlockchain. + + :return: List of blocks in JSON format. + """ + if start_block == "latest": + end_block = "latest" + + assert type(start_block) == int or start_block == "latest", "start_block must be an int or 'latest'" + assert type(end_block) == int or end_block == "latest", "end_block must be an int or None" + assert isinstance(blockchain, AnkrSupportedBlockchain), "blockchain must be of type AnkrSupportedBlockchain" + + headers = { + "accept": "application/json", + "content-type": "application/json", + } + + data = { + "jsonrpc": "2.0", + "method": "ankr_getBlocks", + "params": { + "blockchain": blockchain.value, + "fromBlock": start_block, + "toBlock": end_block, + "includeTxs": False, + "includeLogs": False, + }, + "id": request_id, + } + + result = requests.post(endpoint_url, headers=headers, json=data) + + j = result.json() + + blocks = j["result"]["blocks"] + + return blocks + + +def extract_timestamps_ankr_get_block( + endpoint_url: str, + start_block: int | None = None, + end_block: int | None = None, + max_blocks_at_once: int = 30, +) -> list[int]: + """Extract timestamps from Ankr API + + :param endpoint_url: URL of Ankr API endpoint. Should be multichain endpoint. + + :param start_block: Block number to start fetching from. Can be an int or None. + + :param end_block: Block number to end fetching at. Can be an int or None. + + :param max_blocks_at_once: Maximum number of blocks to fetch at once. Default is 30. + + :return: List of timestamps in int format. + """ + timestamps = [] + + for i in range(start_block, end_block + 1, max_blocks_at_once): + blocks = make_block_request_ankr(endpoint_url, i, min(i + max_blocks_at_once - 1, end_block)) + timestamps.extend([int(x["timestamp"], 16) for x in blocks]) + + return timestamps diff --git a/eth_defi/event_reader/reorganisation_monitor.py b/eth_defi/event_reader/reorganisation_monitor.py index 78df85c8..37aeeaf4 100644 --- a/eth_defi/event_reader/reorganisation_monitor.py +++ b/eth_defi/event_reader/reorganisation_monitor.py @@ -16,10 +16,12 @@ from tqdm import tqdm from web3 import Web3, HTTPProvider -from eth_defi.chain import has_graphql_support +from eth_defi.chain import has_graphql_support, has_ankr_support from eth_defi.event_reader.block_header import BlockHeader, Timestamp from eth_defi.provider.fallback import FallbackProvider from eth_defi.provider.mev_blocker import MEVBlockerProvider +from eth_defi.event_reader.ankr import AnkrSupportedBlockchain, make_block_request_ankr + logger = logging.getLogger(__name__) @@ -644,6 +646,80 @@ def fetch_block_data(self, start_block, end_block) -> Iterable[BlockHeader]: yield BlockHeader(block_number=number, block_hash=hash, timestamp=timestamp) +class AnkrReogranisationMonitor(ReorganisationMonitor): + """Watch blockchain for reorgs using eth_getBlockByNumber JSON-RPC API. + + - Use expensive eth_getBlockByNumber call to download + block hash and timestamp from Ethereum compatible node + """ + + def __init__(self, provider: HTTPProvider, ankr_url: str | None = "https://rpc.ankr.com/multichain/79258ce7f7ee046decc3b5292a24eb4bf7c910d7e39b691384c7ce0cfb839a01/", blockchain: AnkrSupportedBlockchain | None = None, **kwargs): + """ + :param provider: + HTTPProvider with ankr_url. Should not specify ankr_url if this is specified. + + :param ankr_url: + Should be multichain url. Should not specify provider if this is specified. + + :param blockchain: + Blockchain to use. AnkrSupportedBlockchain enum instance. + """ + + super().__init__(**kwargs) + + assert isinstance(blockchain, AnkrSupportedBlockchain), "Must provide blockchain as AnkrSupportedBlockchain enum" + self.blockchain = blockchain + + if provider: + self.ankr_url = provider.endpoint_uri + elif ankr_url: + assert not provider, "Give provider or ankr_url, not both" + self.ankr_url = ankr_url + + # starting id for jsonrpc requests + self.id = 1 + + self.max_blocks_at_once = 30 + + def get_last_block_live(self) -> int: + """Get the chain tip (last block) using Ankr. + + :return: + Last block number + """ + return int(make_block_request_ankr(self.ankr_url, blockchain=self.blockchain)[-1]["number"], 16) + + def create_block_header(self, block: dict) -> BlockHeader: + """Create a BlockHeader from a block dict. + + :param block: + Block dict from Ankr. + + :return: + BlockHeader instance + """ + return BlockHeader(int(block["number"], 16), block["hash"], int(block["timestamp"], 16)) + + def fetch_block_data(self, start_block: int, end_block: int) -> Iterable[BlockHeader]: + """Fetch block headers from Ankr. + + Note: can only fetch 30 blocks at a time. + + :param start_block: + Block number to start fetching from (inclusive) + + :param end_block: + Block number to end fetching at (inclusive) + + :return: + Iterable of BlockHeader instances + """ + blocks = make_block_request_ankr(self.ankr_url, start_block, end_block, self.blockchain, self.id) + self.id += 1 + block_headers = [self.create_block_header(block) for block in blocks] + yield from block_headers + + class MockChainAndReorganisationMonitor(ReorganisationMonitor): """A dummy reorganisation monitor for unit testing. @@ -738,6 +814,8 @@ def create_reorganisation_monitor(web3: Web3, check_depth=250) -> Reorganisation # 10x faster /graphql implementation, # not provided by public nodes reorg_mon = GraphQLReorganisationMonitor(graphql_url=urljoin(json_rpc_url, "/graphql"), check_depth=check_depth) + elif has_ankr_support(provider): + reorg_mon = AnkrReogranisationMonitor(web3, check_depth=check_depth) else: # Default slow implementation logger.warning("The node does not support /graphql interface. " "Downloading block headers and timestamps will be extremely slow." "Check documentation how to configure your node or choose a smaller timeframe for the buffer of trades.") diff --git a/tests/test_reorganisation_monitor_ankr.py b/tests/test_reorganisation_monitor_ankr.py new file mode 100644 index 00000000..8e4b15ea --- /dev/null +++ b/tests/test_reorganisation_monitor_ankr.py @@ -0,0 +1,41 @@ +"""Test chain reogranisation monitor and connect to live Ankr network.""" + +import os + +import pytest +from web3 import HTTPProvider + +from eth_defi.chain import has_ankr_support +from eth_defi.event_reader.reorganisation_monitor import AnkrReogranisationMonitor, AnkrSupportedBlockchain + +pytestmark = pytest.mark.skipif( + os.environ.get("JSON_RPC_ANKR_PRIVATE") is None, + reason="Set JSON_RPC_ANKR_PRIVATE environment variable to a privately configured Polygon node with GraphQL turned on", +) + + +def test_ankr_last_block(): + """Get last block num using Ankr API.""" + + provider = HTTPProvider(os.environ["JSON_RPC_ANKR_PRIVATE"]) + assert has_ankr_support(provider) + + reorg_mon = AnkrReogranisationMonitor(provider=provider, blockchain=AnkrSupportedBlockchain.arbitrum) + block_number = reorg_mon.get_last_block_live() + assert block_number > 30_000_000 + + +def test_ankr_block_headers(): + """Download block headers using Ankr API.""" + + provider = HTTPProvider(os.environ["JSON_RPC_ANKR_PRIVATE"]) + assert has_ankr_support(provider) + + reorg_mon = AnkrReogranisationMonitor(provider=provider, blockchain=AnkrSupportedBlockchain.arbitrum) + + start_block, end_block = reorg_mon.load_initial_block_headers(block_count=5) + + block = reorg_mon.get_block_by_number(start_block) + assert block.block_number > 0 + assert block.block_hash.startswith("0x") + assert block.timestamp > 0