From b20ffd7ff3179b0592140edd5e6e58674d0eaa65 Mon Sep 17 00:00:00 2001 From: barak manos <> Date: Thu, 8 Feb 2024 18:18:19 +0200 Subject: [PATCH 1/3] 1. Simplify the code for fetching events. 2. Avoid fetching duplicated events. --- run_blockchain_terraformer.py | 110 +++++----------------------------- 1 file changed, 16 insertions(+), 94 deletions(-) diff --git a/run_blockchain_terraformer.py b/run_blockchain_terraformer.py index a793a77ef..9b4d4e557 100644 --- a/run_blockchain_terraformer.py +++ b/run_blockchain_terraformer.py @@ -99,6 +99,14 @@ AERODROME_V2_NAME: 12, } +EXCHANGE_POOL_CREATION_EVENT_NAMES = { + UNISWAP_V2_NAME: "PairCreated", + UNISWAP_V3_NAME: "PoolCreated", + AERODROME_V2_NAME: "PoolCreated", + VELOCIMETER_V2_NAME: "PairCreated", + SCALE_V2_NAME: "PairCreated", +} + dataframe_key = [ "cid", "last_updated", @@ -737,91 +745,11 @@ def organize_pool_details_solidly_v2( return pool -def get_uni_pool_creation_events_v3( - factory_contract, block_number: int, web3: Web3, block_chunk_size=50000 -) -> List: - """ - This function retrieves Uniswap V3 pool generation events - - :param factory_contract: the initialized Factory contract - :param block_number: the block number from which to start - :param web3: the Web3 object - :param block_chunk_size: the number of blocks to fetch at a time - returns: a list of raw pool creation events - """ - current_block = web3.eth.block_number - events = [] - - for idx in range(int(math.ceil((current_block - block_number) / block_chunk_size))): - from_block = block_number + idx * block_chunk_size - to_block = ( - from_block + block_chunk_size - if from_block + block_chunk_size < current_block - else current_block - ) - events += factory_contract.events.PoolCreated.get_logs( - fromBlock=from_block, toBlock=to_block - ) - return events - - -def get_uni_pool_creation_events_v2( - factory_contract, block_number: int, web3: Web3, block_chunk_size=50000 -) -> List: - """ - This function retrieves Uniswap V2 pool generation events - - :param factory_contract: the initialized Factory contract - :param block_number: the block number from which to start - :param web3: the Web3 object - :param block_chunk_size: the number of blocks to fetch at a time - returns: a list of raw pool creation events - """ - current_block = web3.eth.block_number - events = [] - for idx in range(int(math.ceil((current_block - block_number) / block_chunk_size))): - from_block = block_number + idx * block_chunk_size - to_block = ( - from_block + block_chunk_size - if from_block + block_chunk_size < current_block - else current_block - ) - events += factory_contract.events.PairCreated.get_logs( - fromBlock=from_block, toBlock=to_block - ) - return events - - -def get_solidly_pool_creation_events_v2( - exchange, factory_contract, block_number: int, web3: Web3, block_chunk_size=50000 -) -> List: - """ - This function retrieves Solidly pool generation events - - :param factory_contract: the initialized Factory contract - :param block_number: the block number from which to start - :param web3: the Web3 object - :param block_chunk_size: the number of blocks to fetch at a time - returns: a list of raw pool creation events - """ - current_block = web3.eth.block_number - events = [] - for idx in range(int(math.ceil((current_block - block_number) / block_chunk_size))): - from_block = block_number + idx * block_chunk_size - to_block = ( - from_block + block_chunk_size - if from_block + block_chunk_size < current_block - else current_block - ) - if exchange in ["aerodrome_v2",]: - events += factory_contract.events.PoolCreated.get_logs( - fromBlock=from_block, toBlock=to_block - ) - elif exchange in ["velocimeter_v2", "scale_v2"]: - events += factory_contract.events.PairCreated.get_logs( - fromBlock=from_block, toBlock=to_block - ) - return events +def _get_events(contract, blockchain: str, exchange: str, start_block: int) -> list: + event_method = EXCHANGE_POOL_CREATION_EVENT_NAMES[exchange].get_logs + block_numbers = range(start_block, contract.w3.eth.block_number + 1, BLOCK_CHUNK_SIZE_MAP[blockchain]) + events_list = [event_method(fromBlock=block_numbers[n-1], toBlock=block_numbers[n]-1) for n in range(1, len(block_numbers))] + return [event for events in events_list for event in events] def get_uni_v3_pools( @@ -844,9 +772,7 @@ def get_uni_v3_pools( returns: a tuple containing a Dataframe of pool creation and a Dataframe of Uni V3 pool mappings """ - pool_data = get_uni_pool_creation_events_v3( - factory_contract=factory_contract, block_number=start_block, web3=web3, block_chunk_size=BLOCK_CHUNK_SIZE_MAP[blockchain] - ) + pool_data = _get_events(factory_contract, blockchain, exchange, start_block) with parallel_backend(n_jobs=-1, backend="threading"): pools = Parallel(n_jobs=-1)( @@ -891,9 +817,7 @@ def get_uni_v2_pools( :param blockchain: the blockchain name returns: a tuple containing a Dataframe of pool creation and a Dataframe of Uni V3 pool mappings """ - pool_data = get_uni_pool_creation_events_v2( - factory_contract=factory_contract, block_number=start_block, web3=web3, block_chunk_size=BLOCK_CHUNK_SIZE_MAP[blockchain] - ) + pool_data = _get_events(factory_contract, blockchain, exchange, start_block) with parallel_backend(n_jobs=-1, backend="threading"): pools = Parallel(n_jobs=-1)( @@ -938,9 +862,7 @@ def get_solidly_v2_pools( returns: a tuple containing a Dataframe of pool creation and a Dataframe of Uni V3 pool mappings """ - pool_data = get_solidly_pool_creation_events_v2( - exchange=exchange, factory_contract=factory_contract, block_number=start_block, web3=web3, block_chunk_size=BLOCK_CHUNK_SIZE_MAP[blockchain] - ) + pool_data = _get_events(factory_contract, blockchain, exchange, start_block) with parallel_backend(n_jobs=-1, backend="threading"): pools = Parallel(n_jobs=-1)( From 21801b034f137df212880069555c5aa826472998 Mon Sep 17 00:00:00 2001 From: barak manos <> Date: Thu, 8 Feb 2024 18:57:45 +0200 Subject: [PATCH 2/3] Fix internal method `_get_events` --- run_blockchain_terraformer.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/run_blockchain_terraformer.py b/run_blockchain_terraformer.py index 9b4d4e557..0e01e5e94 100644 --- a/run_blockchain_terraformer.py +++ b/run_blockchain_terraformer.py @@ -746,8 +746,10 @@ def organize_pool_details_solidly_v2( def _get_events(contract, blockchain: str, exchange: str, start_block: int) -> list: + end_block = contract.w3.eth.block_number + 1 + chunk_size = BLOCK_CHUNK_SIZE_MAP[blockchain] event_method = EXCHANGE_POOL_CREATION_EVENT_NAMES[exchange].get_logs - block_numbers = range(start_block, contract.w3.eth.block_number + 1, BLOCK_CHUNK_SIZE_MAP[blockchain]) + block_numbers = list(range(start_block, end_block, chunk_size)) + [end_block] events_list = [event_method(fromBlock=block_numbers[n-1], toBlock=block_numbers[n]-1) for n in range(1, len(block_numbers))] return [event for events in events_list for event in events] From 5f7d81f0f47e1a2b956258a487f8ef032f0e9033 Mon Sep 17 00:00:00 2001 From: barak manos <> Date: Thu, 8 Feb 2024 18:59:52 +0200 Subject: [PATCH 3/3] Fix internal method `_get_events` --- run_blockchain_terraformer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run_blockchain_terraformer.py b/run_blockchain_terraformer.py index 0e01e5e94..188724f7c 100644 --- a/run_blockchain_terraformer.py +++ b/run_blockchain_terraformer.py @@ -748,8 +748,8 @@ def organize_pool_details_solidly_v2( def _get_events(contract, blockchain: str, exchange: str, start_block: int) -> list: end_block = contract.w3.eth.block_number + 1 chunk_size = BLOCK_CHUNK_SIZE_MAP[blockchain] - event_method = EXCHANGE_POOL_CREATION_EVENT_NAMES[exchange].get_logs block_numbers = list(range(start_block, end_block, chunk_size)) + [end_block] + event_method = contract.events[EXCHANGE_POOL_CREATION_EVENT_NAMES[exchange]].get_logs events_list = [event_method(fromBlock=block_numbers[n-1], toBlock=block_numbers[n]-1) for n in range(1, len(block_numbers))] return [event for events in events_list for event in events]