From e3a5cf9494c84e5b4ddda12e29d0c8573d65d825 Mon Sep 17 00:00:00 2001 From: sslivkoff Date: Mon, 16 Oct 2023 11:34:16 -0700 Subject: [PATCH] Debug trace diffs (#75) * initial commit for debug trace diffs * add geth call traces --- crates/cli/src/parse/schemas.rs | 6 + crates/freeze/src/datasets/balance_diffs.rs | 16 +- crates/freeze/src/datasets/code_diffs.rs | 16 +- .../freeze/src/datasets/geth_balance_diffs.rs | 65 ++++ crates/freeze/src/datasets/geth_code_diffs.rs | 64 ++++ .../freeze/src/datasets/geth_nonce_diffs.rs | 64 ++++ .../freeze/src/datasets/geth_storage_diffs.rs | 65 ++++ crates/freeze/src/datasets/geth_traces.rs | 147 +++++++++ crates/freeze/src/datasets/mod.rs | 15 + crates/freeze/src/datasets/nonce_diffs.rs | 16 +- crates/freeze/src/datasets/storage_diffs.rs | 18 +- .../src/multi_datasets/geth_state_diffs.rs | 284 ++++++++++++++++++ crates/freeze/src/multi_datasets/mod.rs | 3 + .../src/types/datatypes/datatype_macros.rs | 6 + crates/freeze/src/types/datatypes/multi.rs | 16 +- crates/freeze/src/types/datatypes/scalar.rs | 5 + crates/freeze/src/types/sources.rs | 206 +++++++++++++ 17 files changed, 976 insertions(+), 36 deletions(-) create mode 100644 crates/freeze/src/datasets/geth_balance_diffs.rs create mode 100644 crates/freeze/src/datasets/geth_code_diffs.rs create mode 100644 crates/freeze/src/datasets/geth_nonce_diffs.rs create mode 100644 crates/freeze/src/datasets/geth_storage_diffs.rs create mode 100644 crates/freeze/src/datasets/geth_traces.rs create mode 100644 crates/freeze/src/multi_datasets/geth_state_diffs.rs diff --git a/crates/cli/src/parse/schemas.rs b/crates/cli/src/parse/schemas.rs index e8ded79f..d5bef998 100644 --- a/crates/cli/src/parse/schemas.rs +++ b/crates/cli/src/parse/schemas.rs @@ -18,6 +18,12 @@ fn parse_datatypes(raw_inputs: &Vec) -> Result, ParseError datatypes.push(Datatype::NonceDiffs); datatypes.push(Datatype::StorageDiffs); } + "geth_state_diffs" => { + datatypes.push(Datatype::GethBalanceDiffs); + datatypes.push(Datatype::GethCodeDiffs); + datatypes.push(Datatype::GethNonceDiffs); + datatypes.push(Datatype::GethStorageDiffs); + } datatype_str => datatypes.push(Datatype::from_str(datatype_str)?), } } diff --git a/crates/freeze/src/datasets/balance_diffs.rs b/crates/freeze/src/datasets/balance_diffs.rs index 76dfe7fa..0503a3cb 100644 --- a/crates/freeze/src/datasets/balance_diffs.rs +++ b/crates/freeze/src/datasets/balance_diffs.rs @@ -7,14 +7,14 @@ use std::collections::HashMap; #[cryo_to_df::to_df(Datatype::BalanceDiffs)] #[derive(Default)] pub struct BalanceDiffs { - n_rows: u64, - block_number: Vec>, - transaction_index: Vec>, - transaction_hash: Vec>>, - address: Vec>, - from_value: Vec, - to_value: Vec, - chain_id: Vec, + pub(crate) n_rows: u64, + pub(crate) block_number: Vec>, + pub(crate) transaction_index: Vec>, + pub(crate) transaction_hash: Vec>>, + pub(crate) address: Vec>, + pub(crate) from_value: Vec, + pub(crate) to_value: Vec, + pub(crate) chain_id: Vec, } type BlockTxsTraces = (Option, Vec>>, Vec); diff --git a/crates/freeze/src/datasets/code_diffs.rs b/crates/freeze/src/datasets/code_diffs.rs index 2206b5cd..713c5966 100644 --- a/crates/freeze/src/datasets/code_diffs.rs +++ b/crates/freeze/src/datasets/code_diffs.rs @@ -7,14 +7,14 @@ use std::collections::HashMap; #[cryo_to_df::to_df(Datatype::CodeDiffs)] #[derive(Default)] pub struct CodeDiffs { - n_rows: u64, - block_number: Vec>, - transaction_index: Vec>, - transaction_hash: Vec>>, - address: Vec>, - from_value: Vec>, - to_value: Vec>, - chain_id: Vec, + pub(crate) n_rows: u64, + pub(crate) block_number: Vec>, + pub(crate) transaction_index: Vec>, + pub(crate) transaction_hash: Vec>>, + pub(crate) address: Vec>, + pub(crate) from_value: Vec>, + pub(crate) to_value: Vec>, + pub(crate) chain_id: Vec, } #[async_trait::async_trait] diff --git a/crates/freeze/src/datasets/geth_balance_diffs.rs b/crates/freeze/src/datasets/geth_balance_diffs.rs new file mode 100644 index 00000000..4abde22f --- /dev/null +++ b/crates/freeze/src/datasets/geth_balance_diffs.rs @@ -0,0 +1,65 @@ +use crate::*; +use ethers::prelude::*; +use polars::prelude::*; +use std::collections::HashMap; + +/// columns for transactions +#[cryo_to_df::to_df(Datatype::GethBalanceDiffs)] +#[derive(Default)] +pub struct GethBalanceDiffs { + pub(crate) n_rows: u64, + pub(crate) block_number: Vec>, + pub(crate) transaction_index: Vec>, + pub(crate) transaction_hash: Vec>>, + pub(crate) address: Vec>, + pub(crate) from_value: Vec, + pub(crate) to_value: Vec, + pub(crate) chain_id: Vec, +} + +type Result = ::core::result::Result; + +#[async_trait::async_trait] +impl Dataset for GethBalanceDiffs { + fn name() -> &'static str { + "balance_diffs" + } + + fn default_sort() -> Vec { + vec!["block_number".to_string(), "transaction_index".to_string()] + } +} + +#[async_trait::async_trait] +impl CollectByBlock for GethBalanceDiffs { + type Response = ::Response; + + async fn extract( + request: Params, + source: Arc, + schemas: Schemas, + ) -> Result { + ::extract(request, source, schemas).await + } + + fn transform(response: Self::Response, columns: &mut Self, schemas: &Schemas) -> Result<()> { + geth_state_diffs::process_geth_diffs(&response, Some(columns), None, None, None, schemas) + } +} + +#[async_trait::async_trait] +impl CollectByTransaction for GethBalanceDiffs { + type Response = ::Response; + + async fn extract( + request: Params, + source: Arc, + schemas: Schemas, + ) -> Result { + ::extract(request, source, schemas).await + } + + fn transform(response: Self::Response, columns: &mut Self, schemas: &Schemas) -> Result<()> { + geth_state_diffs::process_geth_diffs(&response, Some(columns), None, None, None, schemas) + } +} diff --git a/crates/freeze/src/datasets/geth_code_diffs.rs b/crates/freeze/src/datasets/geth_code_diffs.rs new file mode 100644 index 00000000..a0f02637 --- /dev/null +++ b/crates/freeze/src/datasets/geth_code_diffs.rs @@ -0,0 +1,64 @@ +use crate::*; +use polars::prelude::*; +use std::collections::HashMap; + +/// columns for transactions +#[cryo_to_df::to_df(Datatype::GethCodeDiffs)] +#[derive(Default)] +pub struct GethCodeDiffs { + pub(crate) n_rows: u64, + pub(crate) block_number: Vec>, + pub(crate) transaction_index: Vec>, + pub(crate) transaction_hash: Vec>>, + pub(crate) address: Vec>, + pub(crate) from_value: Vec>, + pub(crate) to_value: Vec>, + pub(crate) chain_id: Vec, +} + +type Result = ::core::result::Result; + +#[async_trait::async_trait] +impl Dataset for GethCodeDiffs { + fn name() -> &'static str { + "code_diffs" + } + + fn default_sort() -> Vec { + vec!["block_number".to_string(), "transaction_index".to_string()] + } +} + +#[async_trait::async_trait] +impl CollectByBlock for GethCodeDiffs { + type Response = ::Response; + + async fn extract( + request: Params, + source: Arc, + schemas: Schemas, + ) -> Result { + ::extract(request, source, schemas).await + } + + fn transform(response: Self::Response, columns: &mut Self, schemas: &Schemas) -> Result<()> { + geth_state_diffs::process_geth_diffs(&response, None, Some(columns), None, None, schemas) + } +} + +#[async_trait::async_trait] +impl CollectByTransaction for GethCodeDiffs { + type Response = ::Response; + + async fn extract( + request: Params, + source: Arc, + schemas: Schemas, + ) -> Result { + ::extract(request, source, schemas).await + } + + fn transform(response: Self::Response, columns: &mut Self, schemas: &Schemas) -> Result<()> { + geth_state_diffs::process_geth_diffs(&response, None, Some(columns), None, None, schemas) + } +} diff --git a/crates/freeze/src/datasets/geth_nonce_diffs.rs b/crates/freeze/src/datasets/geth_nonce_diffs.rs new file mode 100644 index 00000000..0c794858 --- /dev/null +++ b/crates/freeze/src/datasets/geth_nonce_diffs.rs @@ -0,0 +1,64 @@ +use crate::*; +use ethers::prelude::*; +use polars::prelude::*; +use std::collections::HashMap; + +/// columns for transactions +#[cryo_to_df::to_df(Datatype::GethNonceDiffs)] +#[derive(Default)] +pub struct GethNonceDiffs { + pub(crate) n_rows: u64, + pub(crate) block_number: Vec>, + pub(crate) transaction_index: Vec>, + pub(crate) transaction_hash: Vec>>, + pub(crate) address: Vec>, + pub(crate) from_value: Vec, + pub(crate) to_value: Vec, + pub(crate) chain_id: Vec, +} +type Result = ::core::result::Result; + +#[async_trait::async_trait] +impl Dataset for GethNonceDiffs { + fn name() -> &'static str { + "nonce_diffs" + } + + fn default_sort() -> Vec { + vec!["block_number".to_string(), "transaction_index".to_string()] + } +} + +#[async_trait::async_trait] +impl CollectByBlock for GethNonceDiffs { + type Response = ::Response; + + async fn extract( + request: Params, + source: Arc, + schemas: Schemas, + ) -> Result { + ::extract(request, source, schemas).await + } + + fn transform(response: Self::Response, columns: &mut Self, schemas: &Schemas) -> Result<()> { + geth_state_diffs::process_geth_diffs(&response, None, None, Some(columns), None, schemas) + } +} + +#[async_trait::async_trait] +impl CollectByTransaction for GethNonceDiffs { + type Response = ::Response; + + async fn extract( + request: Params, + source: Arc, + schemas: Schemas, + ) -> Result { + ::extract(request, source, schemas).await + } + + fn transform(response: Self::Response, columns: &mut Self, schemas: &Schemas) -> Result<()> { + geth_state_diffs::process_geth_diffs(&response, None, None, Some(columns), None, schemas) + } +} diff --git a/crates/freeze/src/datasets/geth_storage_diffs.rs b/crates/freeze/src/datasets/geth_storage_diffs.rs new file mode 100644 index 00000000..e64d9781 --- /dev/null +++ b/crates/freeze/src/datasets/geth_storage_diffs.rs @@ -0,0 +1,65 @@ +use crate::*; +use polars::prelude::*; +use std::collections::HashMap; + +/// columns for transactions +#[cryo_to_df::to_df(Datatype::GethStorageDiffs)] +#[derive(Default)] +pub struct GethStorageDiffs { + pub(crate) n_rows: u64, + pub(crate) block_number: Vec>, + pub(crate) transaction_index: Vec>, + pub(crate) transaction_hash: Vec>>, + pub(crate) address: Vec>, + pub(crate) slot: Vec>, + pub(crate) from_value: Vec>, + pub(crate) to_value: Vec>, + pub(crate) chain_id: Vec, +} + +#[async_trait::async_trait] +impl Dataset for GethStorageDiffs { + fn name() -> &'static str { + "geth_storage_diffs" + } + + fn default_sort() -> Vec { + vec!["block_number".to_string(), "transaction_index".to_string()] + } +} + +type Result = ::core::result::Result; + +#[async_trait::async_trait] +impl CollectByBlock for GethStorageDiffs { + type Response = ::Response; + + async fn extract( + request: Params, + source: Arc, + schemas: Schemas, + ) -> Result { + ::extract(request, source, schemas).await + } + + fn transform(response: Self::Response, columns: &mut Self, schemas: &Schemas) -> Result<()> { + geth_state_diffs::process_geth_diffs(&response, None, None, None, Some(columns), schemas) + } +} + +#[async_trait::async_trait] +impl CollectByTransaction for GethStorageDiffs { + type Response = ::Response; + + async fn extract( + request: Params, + source: Arc, + schemas: Schemas, + ) -> Result { + ::extract(request, source, schemas).await + } + + fn transform(response: Self::Response, columns: &mut Self, schemas: &Schemas) -> Result<()> { + geth_state_diffs::process_geth_diffs(&response, None, None, None, Some(columns), schemas) + } +} diff --git a/crates/freeze/src/datasets/geth_traces.rs b/crates/freeze/src/datasets/geth_traces.rs new file mode 100644 index 00000000..0af132dc --- /dev/null +++ b/crates/freeze/src/datasets/geth_traces.rs @@ -0,0 +1,147 @@ +use crate::*; +use ethers::prelude::*; +use polars::prelude::*; +use std::collections::HashMap; + +/// columns for geth traces +#[cryo_to_df::to_df(Datatype::GethTraces)] +#[derive(Default)] +pub struct GethTraces { + n_rows: u64, + typ: Vec, + from_address: Vec>, + to_address: Vec>>, + value: Vec>, + gas: Vec, + gas_used: Vec, + input: Vec>, + output: Vec>>, + error: Vec>, + block_number: Vec>, + transaction_hash: Vec>>, + transaction_index: Vec, + trace_address: Vec, + chain_id: Vec, +} + +type Result = ::core::result::Result; + +#[async_trait::async_trait] +impl Dataset for GethTraces { + fn name() -> &'static str { + "geth_traces" + } + + fn default_sort() -> Vec { + vec!["block_number".to_string(), "transaction_index".to_string()] + } +} + +type BlockTxsTraces = (Option, Vec>>, Vec); + +#[async_trait::async_trait] +impl CollectByBlock for GethTraces { + type Response = BlockTxsTraces; + + async fn extract( + request: Params, + source: Arc, + schemas: Schemas, + ) -> Result<(Option, Vec>>, Vec)> { + let schema = + schemas.get(&Datatype::GethTraces).ok_or(err("schema for geth_traces missing"))?; + let include_transaction = schema.has_column("block_number"); + source + .fetcher + .geth_debug_trace_block_calls(request.block_number()? as u32, include_transaction) + .await + } + + fn transform(response: Self::Response, columns: &mut Self, schemas: &Schemas) -> Result<()> { + process_geth_traces(response, columns, schemas) + } +} + +#[async_trait::async_trait] +impl CollectByTransaction for GethTraces { + type Response = BlockTxsTraces; + + async fn extract( + request: Params, + source: Arc, + schemas: Schemas, + ) -> Result<(Option, Vec>>, Vec)> { + let schema = + schemas.get(&Datatype::GethTraces).ok_or(err("schema for geth_traces missing"))?; + let include_block_number = schema.has_column("block_number"); + source + .fetcher + .geth_debug_trace_transaction_calls(request.transaction_hash()?, include_block_number) + .await + } + + fn transform(response: Self::Response, columns: &mut Self, schemas: &Schemas) -> Result<()> { + process_geth_traces(response, columns, schemas) + } +} + +fn process_geth_traces( + traces: BlockTxsTraces, + columns: &mut GethTraces, + schemas: &Schemas, +) -> Result<()> { + let (block_number, txs, traces) = traces; + let schema = schemas.get(&Datatype::GethTraces).ok_or(err("schema for geth_traces missing"))?; + for (tx_index, (tx, trace)) in txs.into_iter().zip(traces).enumerate() { + process_trace(trace, columns, schema, &block_number, &tx, tx_index as u32, vec![])? + } + Ok(()) +} + +fn process_trace( + trace: CallFrame, + columns: &mut GethTraces, + schema: &Table, + block_number: &Option, + tx: &Option>, + tx_index: u32, + trace_address: Vec, +) -> Result<()> { + columns.n_rows += 1; + store!(schema, columns, typ, trace.typ); + store!(schema, columns, from_address, trace.from.as_bytes().to_vec()); + store!(schema, columns, to_address, noa_to_vec_u8(trace.to)?); + store!(schema, columns, value, trace.value); + store!(schema, columns, gas, trace.gas); + store!(schema, columns, gas_used, trace.gas_used); + store!(schema, columns, input, trace.input.0.to_vec()); + store!(schema, columns, output, trace.output.map(|x| x.0.to_vec())); + store!(schema, columns, error, trace.error); + store!(schema, columns, block_number, *block_number); + store!(schema, columns, transaction_hash, tx.clone()); + store!(schema, columns, transaction_index, tx_index); + store!( + schema, + columns, + trace_address, + trace_address.iter().map(|&n| n.to_string()).collect::>().join(" ") + ); + + if let Some(subcalls) = trace.calls { + for (s, subcall) in subcalls.into_iter().enumerate() { + let mut sub_trace_address = trace_address.clone(); + sub_trace_address.push(s as u32); + process_trace(subcall, columns, schema, block_number, tx, tx_index, sub_trace_address)? + } + } + + Ok(()) +} + +fn noa_to_vec_u8(value: Option) -> Result>> { + match value { + Some(NameOrAddress::Address(address)) => Ok(Some(address.as_bytes().to_vec())), + Some(NameOrAddress::Name(_)) => Err(err("block name string not allowed")), + None => Ok(None), + } +} diff --git a/crates/freeze/src/datasets/mod.rs b/crates/freeze/src/datasets/mod.rs index 8f15e4dc..b34d7244 100644 --- a/crates/freeze/src/datasets/mod.rs +++ b/crates/freeze/src/datasets/mod.rs @@ -24,6 +24,16 @@ pub mod erc721_metadata; pub mod erc721_transfers; /// eth calls pub mod eth_calls; +/// geth balance diffs +pub mod geth_balance_diffs; +/// geth code diffs +pub mod geth_code_diffs; +/// geth nonce diffs +pub mod geth_nonce_diffs; +/// geth storage diffs +pub mod geth_storage_diffs; +/// geth traces +pub mod geth_traces; /// logs pub mod logs; /// native transfers @@ -60,6 +70,11 @@ pub use erc20_transfers::*; pub use erc721_metadata::*; pub use erc721_transfers::*; pub use eth_calls::*; +pub use geth_balance_diffs::*; +pub use geth_code_diffs::*; +pub use geth_nonce_diffs::*; +pub use geth_storage_diffs::*; +pub use geth_traces::*; pub use logs::*; pub use native_transfers::*; pub use nonce_diffs::*; diff --git a/crates/freeze/src/datasets/nonce_diffs.rs b/crates/freeze/src/datasets/nonce_diffs.rs index c9c6d3fa..53e39942 100644 --- a/crates/freeze/src/datasets/nonce_diffs.rs +++ b/crates/freeze/src/datasets/nonce_diffs.rs @@ -7,14 +7,14 @@ use std::collections::HashMap; #[cryo_to_df::to_df(Datatype::NonceDiffs)] #[derive(Default)] pub struct NonceDiffs { - n_rows: u64, - block_number: Vec>, - transaction_index: Vec>, - transaction_hash: Vec>>, - address: Vec>, - from_value: Vec, - to_value: Vec, - chain_id: Vec, + pub(crate) n_rows: u64, + pub(crate) block_number: Vec>, + pub(crate) transaction_index: Vec>, + pub(crate) transaction_hash: Vec>>, + pub(crate) address: Vec>, + pub(crate) from_value: Vec, + pub(crate) to_value: Vec, + pub(crate) chain_id: Vec, } #[async_trait::async_trait] diff --git a/crates/freeze/src/datasets/storage_diffs.rs b/crates/freeze/src/datasets/storage_diffs.rs index bd2771dd..4de76e48 100644 --- a/crates/freeze/src/datasets/storage_diffs.rs +++ b/crates/freeze/src/datasets/storage_diffs.rs @@ -7,15 +7,15 @@ use std::collections::HashMap; #[cryo_to_df::to_df(Datatype::StorageDiffs)] #[derive(Default)] pub struct StorageDiffs { - n_rows: u64, - block_number: Vec>, - transaction_index: Vec>, - transaction_hash: Vec>>, - address: Vec>, - slot: Vec>, - from_value: Vec>, - to_value: Vec>, - chain_id: Vec, + pub(crate) n_rows: u64, + pub(crate) block_number: Vec>, + pub(crate) transaction_index: Vec>, + pub(crate) transaction_hash: Vec>>, + pub(crate) address: Vec>, + pub(crate) slot: Vec>, + pub(crate) from_value: Vec>, + pub(crate) to_value: Vec>, + pub(crate) chain_id: Vec, } #[async_trait::async_trait] diff --git a/crates/freeze/src/multi_datasets/geth_state_diffs.rs b/crates/freeze/src/multi_datasets/geth_state_diffs.rs new file mode 100644 index 00000000..af447b5c --- /dev/null +++ b/crates/freeze/src/multi_datasets/geth_state_diffs.rs @@ -0,0 +1,284 @@ +use crate::*; +use ethers::prelude::*; +use polars::prelude::*; +use std::collections::{BTreeMap, HashMap, HashSet}; + +/// state diffs from geth debug traces +pub struct GethStateDiffs( + pub Option, + pub Option, + pub Option, + pub Option, +); + +impl Default for GethStateDiffs { + fn default() -> GethStateDiffs { + GethStateDiffs( + Some(GethBalanceDiffs::default()), + Some(GethCodeDiffs::default()), + Some(GethNonceDiffs::default()), + Some(GethStorageDiffs::default()), + ) + } +} + +impl ToDataFrames for GethStateDiffs { + fn create_dfs( + self, + schemas: &HashMap, + chain_id: u64, + ) -> Result> { + let GethStateDiffs(balance_diffs, code_diffs, nonce_diffs, storage_diffs) = self; + let mut output = HashMap::new(); + if let Some(balance_diffs) = balance_diffs { + output.extend(balance_diffs.create_dfs(schemas, chain_id)?); + } + if let Some(code_diffs) = code_diffs { + output.extend(code_diffs.create_dfs(schemas, chain_id)?); + } + if let Some(nonce_diffs) = nonce_diffs { + output.extend(nonce_diffs.create_dfs(schemas, chain_id)?); + } + if let Some(storage_diffs) = storage_diffs { + output.extend(storage_diffs.create_dfs(schemas, chain_id)?); + } + Ok(output) + } +} + +type BlockTxsTraces = (Option, Vec>>, Vec); +type Result = ::core::result::Result; + +#[async_trait::async_trait] +impl CollectByBlock for GethStateDiffs { + type Response = BlockTxsTraces; + + async fn extract( + request: Params, + source: Arc, + schemas: Schemas, + ) -> Result { + let block_number = request.block_number()? as u32; + let include_txs = schemas.values().any(|x| x.has_column("transaction_hash")); + source.fetcher.geth_debug_trace_block_diffs(block_number, include_txs).await + } + + fn transform(response: Self::Response, columns: &mut Self, schemas: &Schemas) -> Result<()> { + let GethStateDiffs(ref mut balances, ref mut codes, ref mut nonces, ref mut storages) = + columns; + process_geth_diffs( + &response, + balances.as_mut(), + codes.as_mut(), + nonces.as_mut(), + storages.as_mut(), + schemas, + ) + } +} + +#[async_trait::async_trait] +impl CollectByTransaction for GethStateDiffs { + type Response = BlockTxsTraces; + + async fn extract( + request: Params, + source: Arc, + schemas: Schemas, + ) -> Result { + let include_block_number = schemas.values().any(|x| x.has_column("transaction_hash")); + source + .fetcher + .geth_debug_trace_transaction_diffs(request.transaction_hash()?, include_block_number) + .await + } + + fn transform(response: Self::Response, columns: &mut Self, schemas: &Schemas) -> Result<()> { + let GethStateDiffs(ref mut balances, ref mut codes, ref mut nonces, ref mut storages) = + columns; + process_geth_diffs( + &response, + balances.as_mut(), + codes.as_mut(), + nonces.as_mut(), + storages.as_mut(), + schemas, + ) + } +} + +pub(crate) fn process_geth_diffs( + response: &BlockTxsTraces, + mut balances: Option<&mut GethBalanceDiffs>, + mut codes: Option<&mut GethCodeDiffs>, + mut nonces: Option<&mut GethNonceDiffs>, + mut storages: Option<&mut GethStorageDiffs>, + schemas: &Schemas, +) -> Result<()> { + let (block_number, txs, traces) = response; + let balance_schema = schemas.get(&Datatype::GethBalanceDiffs); + let code_schema = schemas.get(&Datatype::GethCodeDiffs); + let nonce_schema = schemas.get(&Datatype::GethNonceDiffs); + let storage_schema = schemas.get(&Datatype::GethStorageDiffs); + + let blank = &AccountState::default(); + for (tx_index, (trace, tx)) in traces.iter().zip(txs).enumerate() { + let index = &(*block_number, tx_index as u32, tx.clone()); + let addresses: Vec<_> = trace + .pre + .keys() + .chain(trace.post.keys()) + .collect::>() + .into_iter() + .collect(); + for address in addresses.into_iter() { + let (pre, post) = match (trace.pre.get(address), trace.post.get(address)) { + (Some(pre), Some(post)) => (pre, post), + (Some(pre), None) => (pre, blank), + (None, Some(post)) => (blank, post), + (None, None) => (blank, blank), + }; + if let (Some(balances), Some(schema)) = (balances.as_mut(), balance_schema) { + add_balances(address, pre.balance, post.balance, balances, schema, index)?; + } + if let (Some(codes), Some(schema)) = (codes.as_mut(), code_schema) { + add_codes(address, &pre.code, &post.code, codes, schema, index)?; + } + if let (Some(nonces), Some(schema)) = (nonces.as_mut(), nonce_schema) { + add_nonces(address, pre.nonce, post.nonce, nonces, schema, index)?; + } + if let (Some(storages), Some(schema)) = (storages.as_mut(), storage_schema) { + add_storages(address, &pre.storage, &post.storage, storages, schema, index)?; + } + } + } + Ok(()) +} + +fn add_balances( + address: &H160, + pre: Option, + post: Option, + columns: &mut GethBalanceDiffs, + schema: &Table, + index: &(Option, u32, Option>), +) -> Result<()> { + let (from_value, to_value) = parse_pre_post(pre, post, U256::zero); + let (block_number, transaction_index, transaction_hash) = index; + columns.n_rows += 1; + store!(schema, columns, block_number, *block_number); + store!(schema, columns, transaction_index, Some(*transaction_index as u64)); + store!(schema, columns, transaction_hash, transaction_hash.clone()); + store!(schema, columns, address, address.as_bytes().to_vec()); + store!(schema, columns, from_value, from_value); + store!(schema, columns, to_value, to_value); + Ok(()) +} + +fn add_codes( + address: &H160, + pre: &Option, + post: &Option, + columns: &mut GethCodeDiffs, + schema: &Table, + index: &(Option, u32, Option>), +) -> Result<()> { + let blank = String::new(); + let (from_value, to_value) = match (pre, post) { + (Some(pre), Some(post)) => (pre, post), + (Some(pre), None) => (pre, &blank), + (None, Some(post)) => (&blank, post), + (None, None) => (&blank, &blank), + }; + let (block_number, transaction_index, transaction_hash) = index; + columns.n_rows += 1; + store!(schema, columns, block_number, *block_number); + store!(schema, columns, transaction_index, Some(*transaction_index as u64)); + store!(schema, columns, transaction_hash, transaction_hash.clone()); + store!(schema, columns, address, address.as_bytes().to_vec()); + let from_value = if !from_value.is_empty() { + prefix_hex::decode(from_value).map_err(|_| err("could not decode from code contents"))? + } else { + vec![] + }; + let to_value = if !to_value.is_empty() { + prefix_hex::decode(to_value).map_err(|_| err("could not decode to code contents"))? + } else { + vec![] + }; + store!(schema, columns, from_value, from_value); + store!(schema, columns, to_value, to_value); + Ok(()) +} + +fn add_nonces( + address: &H160, + pre: Option, + post: Option, + columns: &mut GethNonceDiffs, + schema: &Table, + index: &(Option, u32, Option>), +) -> Result<()> { + let (from_value, to_value) = parse_pre_post(pre, post, U256::zero); + let (block_number, transaction_index, transaction_hash) = index; + columns.n_rows += 1; + store!(schema, columns, block_number, *block_number); + store!(schema, columns, transaction_index, Some(*transaction_index as u64)); + store!(schema, columns, transaction_hash, transaction_hash.clone()); + store!(schema, columns, address, address.as_bytes().to_vec()); + store!(schema, columns, from_value, from_value); + store!(schema, columns, to_value, to_value); + Ok(()) +} + +fn add_storages( + address: &H160, + pre: &Option>, + post: &Option>, + columns: &mut GethStorageDiffs, + schema: &Table, + index: &(Option, u32, Option>), +) -> Result<()> { + let blank = BTreeMap::new(); + let (pre, post) = match (pre, post) { + (Some(pre), Some(post)) => (pre, post), + (Some(pre), None) => (pre, &blank), + (None, Some(post)) => (&blank, post), + (None, None) => (&blank, &blank), + }; + let (block_number, transaction_index, transaction_hash) = index; + let slots: Vec<_> = pre + .clone() + .into_keys() + .chain(post.clone().into_keys()) + .collect::>() + .into_iter() + .collect(); + let blank = H256::zero(); + for slot in slots.into_iter() { + let (from, to) = match (pre.get(&slot), post.get(&slot)) { + (Some(pre), Some(post)) => (pre, post), + (Some(pre), None) => (pre, &blank), + (None, Some(post)) => (&blank, post), + (None, None) => (&blank, &blank), + }; + columns.n_rows += 1; + store!(schema, columns, block_number, *block_number); + store!(schema, columns, transaction_index, Some(*transaction_index as u64)); + store!(schema, columns, transaction_hash, transaction_hash.clone()); + store!(schema, columns, address, address.as_bytes().to_vec()); + store!(schema, columns, slot, slot.as_bytes().to_vec()); + store!(schema, columns, from_value, from.as_bytes().to_vec()); + store!(schema, columns, to_value, to.as_bytes().to_vec()); + } + Ok(()) +} + +fn parse_pre_post(pre: Option, post: Option, new: fn() -> T) -> (T, T) { + match (pre, post) { + (Some(pre), Some(post)) => (pre, post), + (Some(pre), None) => (pre, new()), + (None, Some(post)) => (new(), post), + (None, None) => (new(), new()), + } +} diff --git a/crates/freeze/src/multi_datasets/mod.rs b/crates/freeze/src/multi_datasets/mod.rs index b15da8c9..30a6ff21 100644 --- a/crates/freeze/src/multi_datasets/mod.rs +++ b/crates/freeze/src/multi_datasets/mod.rs @@ -1,7 +1,10 @@ mod blocks_and_transactions; mod call_trace_derivatives; +/// geth state diffs +pub mod geth_state_diffs; mod state_diffs; pub use blocks_and_transactions::*; pub use call_trace_derivatives::*; +pub use geth_state_diffs::*; pub use state_diffs::*; diff --git a/crates/freeze/src/types/datatypes/datatype_macros.rs b/crates/freeze/src/types/datatypes/datatype_macros.rs index 243072a8..1a9495da 100644 --- a/crates/freeze/src/types/datatypes/datatype_macros.rs +++ b/crates/freeze/src/types/datatypes/datatype_macros.rs @@ -132,6 +132,9 @@ macro_rules! define_datatypes { MultiDatatype::CallTraceDerivatives => { CallTraceDerivatives::collect_by_block(partition, source, &schemas, None) } + MultiDatatype::GethStateDiffs => { + GethStateDiffs::collect_by_block(partition, source, &schemas, None) + }, MultiDatatype::StateDiffs => { StateDiffs::collect_by_block(partition, source, &schemas, None) }, @@ -169,6 +172,9 @@ macro_rules! define_datatypes { MultiDatatype::CallTraceDerivatives => { CallTraceDerivatives::collect_by_transaction(partition, source, &schemas, None) } + MultiDatatype::GethStateDiffs => { + GethStateDiffs::collect_by_transaction(partition, source, &schemas, None) + }, MultiDatatype::StateDiffs => { StateDiffs::collect_by_transaction(partition, source, &schemas, inner_request_size) } diff --git a/crates/freeze/src/types/datatypes/multi.rs b/crates/freeze/src/types/datatypes/multi.rs index 973f26cf..b28ce7d9 100644 --- a/crates/freeze/src/types/datatypes/multi.rs +++ b/crates/freeze/src/types/datatypes/multi.rs @@ -10,6 +10,9 @@ pub enum MultiDatatype { /// call trace derivatives CallTraceDerivatives, + /// geth debug versions of balance diffs, code diffs, nonce diffs, and storage diffs + GethStateDiffs, + /// balance diffs, code diffs, nonce diffs, and storage diffs StateDiffs, } @@ -19,15 +22,21 @@ impl MultiDatatype { pub fn datatypes(&self) -> Vec { match &self { MultiDatatype::BlocksAndTransactions => vec![Datatype::Blocks, Datatype::Transactions], + MultiDatatype::CallTraceDerivatives => { + vec![Datatype::Contracts, Datatype::NativeTransfers, Datatype::Traces] + } + MultiDatatype::GethStateDiffs => vec![ + Datatype::GethBalanceDiffs, + Datatype::GethCodeDiffs, + Datatype::GethNonceDiffs, + Datatype::GethStorageDiffs, + ], MultiDatatype::StateDiffs => vec![ Datatype::BalanceDiffs, Datatype::CodeDiffs, Datatype::NonceDiffs, Datatype::StorageDiffs, ], - MultiDatatype::CallTraceDerivatives => { - vec![Datatype::Contracts, Datatype::NativeTransfers, Datatype::Traces] - } } } @@ -36,6 +45,7 @@ impl MultiDatatype { vec![ MultiDatatype::BlocksAndTransactions, MultiDatatype::CallTraceDerivatives, + MultiDatatype::GethStateDiffs, MultiDatatype::StateDiffs, ] } diff --git a/crates/freeze/src/types/datatypes/scalar.rs b/crates/freeze/src/types/datatypes/scalar.rs index dc5be27e..65ac0d25 100644 --- a/crates/freeze/src/types/datatypes/scalar.rs +++ b/crates/freeze/src/types/datatypes/scalar.rs @@ -16,6 +16,11 @@ define_datatypes!( Erc721Metadata, Erc721Transfers, EthCalls, + GethCodeDiffs, + GethBalanceDiffs, + GethStorageDiffs, + GethNonceDiffs, + GethTraces, Logs, NativeTransfers, NonceDiffs, diff --git a/crates/freeze/src/types/sources.rs b/crates/freeze/src/types/sources.rs index cbe972f7..b5c282c6 100644 --- a/crates/freeze/src/types/sources.rs +++ b/crates/freeze/src/types/sources.rs @@ -328,6 +328,198 @@ impl Fetcher

{ .map_err(CollectError::ProviderError) } + /// get geth debug block traces + pub async fn geth_debug_trace_block( + &self, + block_number: u32, + options: GethDebugTracingOptions, + include_transaction_hashes: bool, + ) -> Result<(Option, Vec>>, Vec)> { + let traces = { + let _permit = self.permit_request().await; + self.provider + .debug_trace_block_by_number(Some(block_number.into()), options) + .await + .map_err(CollectError::ProviderError)? + }; + + let txs = if include_transaction_hashes { + match self.get_block(block_number as u64).await? { + Some(block) => { + block.transactions.iter().map(|x| Some(x.as_bytes().to_vec())).collect() + } + None => { + return Err(CollectError::CollectError( + "could not get block for txs".to_string(), + )) + } + } + } else { + vec![None; traces.len()] + }; + + Ok((Some(block_number), txs, traces)) + } + + /// get geth debug block call traces + pub async fn geth_debug_trace_block_calls( + &self, + block_number: u32, + include_transaction_hashes: bool, + ) -> Result<(Option, Vec>>, Vec)> { + let tracer = GethDebugTracerType::BuiltInTracer(GethDebugBuiltInTracerType::CallTracer); + let config = GethDebugTracerConfig::BuiltInTracer( + GethDebugBuiltInTracerConfig::CallTracer(CallConfig { ..Default::default() }), + ); + let options = GethDebugTracingOptions { + tracer: Some(tracer), + tracer_config: Some(config), + ..Default::default() + }; + let (block, txs, traces) = + self.geth_debug_trace_block(block_number, options, include_transaction_hashes).await?; + + let mut calls = Vec::new(); + for trace in traces.into_iter() { + match trace { + GethTrace::Known(GethTraceFrame::CallTracer(call_frame)) => calls.push(call_frame), + _ => return Err(CollectError::CollectError("invalid trace result".to_string())), + } + } + Ok((block, txs, calls)) + } + + /// get geth debug block diff traces + pub async fn geth_debug_trace_block_diffs( + &self, + block_number: u32, + include_transaction_hashes: bool, + ) -> Result<(Option, Vec>>, Vec)> { + let tracer = GethDebugTracerType::BuiltInTracer(GethDebugBuiltInTracerType::PreStateTracer); + let config = GethDebugTracerConfig::BuiltInTracer( + GethDebugBuiltInTracerConfig::PreStateTracer(PreStateConfig { diff_mode: Some(true) }), + ); + let options = GethDebugTracingOptions { + tracer: Some(tracer), + tracer_config: Some(config), + ..Default::default() + }; + let (block, txs, traces) = + self.geth_debug_trace_block(block_number, options, include_transaction_hashes).await?; + + let mut diffs = Vec::new(); + for trace in traces.into_iter() { + match trace { + GethTrace::Known(GethTraceFrame::PreStateTracer(PreStateFrame::Diff(diff))) => { + diffs.push(diff) + } + GethTrace::Unknown(ethers::utils::__serde_json::Value::Object(map)) => { + let diff = parse_geth_diff_object(map)?; + diffs.push(diff) + } + _ => { + println!("{:?}", trace); + return Err(CollectError::CollectError("invalid trace result".to_string())) + } + } + } + Ok((block, txs, diffs)) + } + + /// get geth debug transaction traces + pub async fn geth_debug_trace_transaction( + &self, + transaction_hash: Vec, + options: GethDebugTracingOptions, + include_block_number: bool, + ) -> Result<(Option, Vec>>, Vec)> { + let ethers_tx = H256::from_slice(&transaction_hash); + + let trace = { + let _permit = self.permit_request().await; + self.provider + .debug_trace_transaction(ethers_tx, options) + .await + .map_err(CollectError::ProviderError)? + }; + let traces = vec![trace]; + + let block_number = if include_block_number { + match self.get_transaction(ethers_tx).await? { + Some(tx) => tx.block_number.map(|x| x.as_u32()), + None => { + return Err(CollectError::CollectError( + "could not get block for txs".to_string(), + )) + } + } + } else { + None + }; + + Ok((block_number, vec![Some(transaction_hash)], traces)) + } + + /// get geth debug block call traces + pub async fn geth_debug_trace_transaction_calls( + &self, + transaction_hash: Vec, + include_block_number: bool, + ) -> Result<(Option, Vec>>, Vec)> { + let tracer = GethDebugTracerType::BuiltInTracer(GethDebugBuiltInTracerType::CallTracer); + let config = GethDebugTracerConfig::BuiltInTracer( + GethDebugBuiltInTracerConfig::CallTracer(CallConfig { ..Default::default() }), + ); + let options = GethDebugTracingOptions { + tracer: Some(tracer), + tracer_config: Some(config), + ..Default::default() + }; + let (block, txs, traces) = self + .geth_debug_trace_transaction(transaction_hash, options, include_block_number) + .await?; + + let mut calls = Vec::new(); + for trace in traces.into_iter() { + match trace { + GethTrace::Known(GethTraceFrame::CallTracer(call_frame)) => calls.push(call_frame), + _ => return Err(CollectError::CollectError("invalid trace result".to_string())), + } + } + Ok((block, txs, calls)) + } + + /// get geth debug block diff traces + pub async fn geth_debug_trace_transaction_diffs( + &self, + transaction_hash: Vec, + include_transaction_hashes: bool, + ) -> Result<(Option, Vec>>, Vec)> { + let tracer = GethDebugTracerType::BuiltInTracer(GethDebugBuiltInTracerType::PreStateTracer); + let config = GethDebugTracerConfig::BuiltInTracer( + GethDebugBuiltInTracerConfig::PreStateTracer(PreStateConfig { diff_mode: Some(true) }), + ); + let options = GethDebugTracingOptions { + tracer: Some(tracer), + tracer_config: Some(config), + ..Default::default() + }; + let (block, txs, traces) = self + .geth_debug_trace_transaction(transaction_hash, options, include_transaction_hashes) + .await?; + + let mut diffs = Vec::new(); + for trace in traces.into_iter() { + match trace { + GethTrace::Known(GethTraceFrame::PreStateTracer(PreStateFrame::Diff(diff))) => { + diffs.push(diff) + } + _ => return Err(CollectError::CollectError("invalid trace result".to_string())), + } + } + Ok((block, txs, diffs)) + } + async fn permit_request( &self, ) -> Option<::core::result::Result, AcquireError>> { @@ -346,8 +538,22 @@ impl Fetcher

{ } } +use crate::err; +use std::collections::BTreeMap; use tokio::task; +fn parse_geth_diff_object( + map: ethers::utils::__serde_json::Map, +) -> Result { + println!("HERE {:?}", map); + let pre: BTreeMap = serde_json::from_value(map["pre"].clone()) + .map_err(|_| err("cannot deserialize pre diff"))?; + let post: BTreeMap = serde_json::from_value(map["post"].clone()) + .map_err(|_| err("cannot deserialize pre diff"))?; + + Ok(DiffMode { pre, post }) +} + impl Source { /// get gas used by transactions in block pub async fn get_txs_gas_used(&self, block: &Block) -> Result> {