From 421109bf4bdb6f0107c6957ccd9d53f2275cc0d4 Mon Sep 17 00:00:00 2001 From: Dragan Pilipovic Date: Fri, 23 Jun 2023 02:51:09 +0200 Subject: [PATCH 01/13] work in progress --- bin/reth/src/db/get.rs | 51 +++++++++++++++++++++---- bin/reth/src/db/list.rs | 59 ++++++++++++++++++++++++++++- bin/reth/src/utils.rs | 23 ++++++++--- crates/storage/db/src/tables/mod.rs | 16 ++++++++ 4 files changed, 134 insertions(+), 15 deletions(-) diff --git a/bin/reth/src/db/get.rs b/bin/reth/src/db/get.rs index 4f4c2ba31ddf..536c28fc921e 100644 --- a/bin/reth/src/db/get.rs +++ b/bin/reth/src/db/get.rs @@ -1,7 +1,14 @@ +use std::error::Error; + use crate::utils::DbTool; use clap::Parser; -use reth_db::{database::Database, table::Table, TableType, TableViewer, Tables}; +use reth_db::{ + database::Database, + table::{DupSort, Table}, + TableType, TableViewer, Tables, +}; +use serde::Deserialize; use tracing::error; /// The arguments for the `reth db get` command @@ -12,20 +19,25 @@ pub struct Command { /// NOTE: The dupsort tables are not supported now. pub table: Tables, - /// The key to get content for + /// The key to get content for #[arg(value_parser = maybe_json_value_parser)] pub key: String, + + /// The subkey to get content for + #[arg(value_parser = maybe_json_value_parser)] + pub subkey: Option, } impl Command { /// Execute `db get` command pub fn execute(self, tool: &DbTool<'_, DB>) -> eyre::Result<()> { if self.table.table_type() == TableType::DupSort { + self.table.view_dupsort(&GetValueViewer { tool, args: &self })?; + error!(target: "reth::cli", "Unsupported table."); - return Ok(()) + return Ok(()); } - self.table.view(&GetValueViewer { tool, args: &self })?; Ok(()) @@ -37,6 +49,12 @@ impl Command { serde_json::from_str::(&self.key).map_err(|e| eyre::eyre!(e)) } + /// Get an instance of subkey for given dupsort table + fn table_subkey(&self) -> Result { + assert_eq!(T::NAME, self.table.name()); + serde_json::from_str::(&self.subkey.clone().unwrap_or_default()) + .map_err(|e| eyre::eyre!(e)) + } } struct GetValueViewer<'a, DB: Database> { @@ -48,22 +66,39 @@ impl TableViewer<()> for GetValueViewer<'_, DB> { type Error = eyre::Report; fn view(&self) -> Result<(), Self::Error> { + let key = self.args.table_key::()?; + assert_eq!(T::NAME, self.args.table.name()); + match self.tool.clone().get::(key)? { + Some(content) => { + println!("{}", serde_json::to_string_pretty(&content)?); + } + None => { + error!(target: "reth::cli", "No content for the given table key111."); + } + }; + + Ok(()) + } + + fn view_dupsort(&self) -> Result<(), Self::Error> { + // error here // get a key for given table let key = self.args.table_key::()?; - match self.tool.get::(key)? { + // process dupsort table + let subkey = self.args.table_subkey::()?; + + match self.tool.clone().get_dup::(key, subkey)? { Some(content) => { println!("{}", serde_json::to_string_pretty(&content)?); } None => { - error!(target: "reth::cli", "No content for the given table key."); + error!(target: "reth::cli", "No content for the given table subkey222s."); } }; - Ok(()) } } - /// Map the user input value to json fn maybe_json_value_parser(value: &str) -> Result { if serde_json::from_str::(value).is_ok() { diff --git a/bin/reth/src/db/list.rs b/bin/reth/src/db/list.rs index 274105dae84d..3403fc093ec7 100644 --- a/bin/reth/src/db/list.rs +++ b/bin/reth/src/db/list.rs @@ -2,8 +2,12 @@ use super::tui::DbListTUI; use crate::utils::{DbTool, ListFilter}; use clap::Parser; use eyre::WrapErr; -use reth_db::{database::Database, table::Table, DatabaseEnvRO, TableViewer, Tables}; use reth_primitives::hex; +use reth_db::{ + database::Database, + mdbx::{Env, WriteMap}, + table::{DupSort, Table}, + DatabaseEnvRO, TableType, TableViewer, Tables, use std::cell::RefCell; use tracing::error; @@ -39,7 +43,17 @@ pub struct Command { impl Command { /// Execute `db list` command pub fn execute(self, tool: &DbTool<'_, DatabaseEnvRO>) -> eyre::Result<()> { +<<<<<<< HEAD self.table.view(&ListTableViewer { tool, args: &self }) +======= + if self.table.table_type() == TableType::DupSort { + self.table.view_dupsort(&ListTableViewer { tool, args: &self })?; + } else { + self.table.view(&ListTableViewer { tool, args: &self })?; + } + + Ok(()) +>>>>>>> 5b1f48f2 (work in progress) } /// Generate [`ListFilter`] from command. @@ -49,7 +63,7 @@ impl Command { .as_ref() .map(|search| { if let Some(search) = search.strip_prefix("0x") { - return hex::decode(search).unwrap() + return hex::decode(search).unwrap(); } search.as_bytes().to_vec() }) @@ -113,4 +127,45 @@ impl TableViewer<()> for ListTableViewer<'_> { Ok(()) } + + fn view_dupsort(&self) -> Result<(), Self::Error> { + self.tool.db.view(|tx| { + let table_db = tx.inner.open_db(Some(self.args.table.name())).wrap_err("Could not open db.")?; + let stats = tx.inner.db_stat(&table_db).wrap_err(format!("Could not find table: {}", stringify!($table)))?; + let total_entries = stats.entries(); + if self.args.skip > total_entries - 1 { + error!( + target: "reth::cli", + "Start index {start} is greater than the final entry index ({final_entry_idx}) in the table {table}", + start = self.args.skip, + final_entry_idx = total_entries - 1, + table = self.args.table.name() + ); + return Ok(()); + } + + + let list_filter = self.args.list_filter(); + + if self.args.json || self.args.count { + let (list, count) = self.tool.list::(&list_filter)?; + + if self.args.count { + println!("{count} entries found.") + }else { + println!("{}", serde_json::to_string_pretty(&list)?); + } + Ok(()) + + } else { + let list_filter = RefCell::new(list_filter); + DbListTUI::<_, T>::new(|skip, len| { + list_filter.borrow_mut().update_page(skip, len); + self.tool.list::(&list_filter.borrow()).unwrap().0 + }, self.args.skip, self.args.len, total_entries).run() + } + })??; + + Ok(()) + } } diff --git a/bin/reth/src/utils.rs b/bin/reth/src/utils.rs index 57519da6428f..d562237ae5e1 100644 --- a/bin/reth/src/utils.rs +++ b/bin/reth/src/utils.rs @@ -4,9 +4,9 @@ use boyer_moore_magiclen::BMByte; use eyre::Result; use reth_consensus_common::validation::validate_block_standalone; use reth_db::{ - cursor::DbCursorRO, + cursor::{DbCursorRO, DbDupCursorRO}, database::Database, - table::{Decode, Decompress, Table, TableRow}, + table::{Decode, Decompress, DupSort, Table, TableRow}, transaction::{DbTx, DbTxMut}, DatabaseError, RawTable, TableRawRow, }; @@ -132,7 +132,7 @@ impl<'a, DB: Database> DbTool<'a, DB> { let result = || { if filter.only_count { - return None + return None; } Some(( ::Key::decode(&key).unwrap(), @@ -142,16 +142,21 @@ impl<'a, DB: Database> DbTool<'a, DB> { match &*bmb { Some(searcher) => { +<<<<<<< HEAD if searcher.find_first_in(&value).is_some() || searcher.find_first_in(&key).is_some() +======= + if searcher.find_first_in(v.raw_value()).is_some() + || searcher.find_first_in(k.raw_key()).is_some() +>>>>>>> 5b1f48f2 (work in progress) { hits += 1; - return result() + return result(); } } None => { hits += 1; - return result() + return result(); } } } @@ -183,6 +188,14 @@ impl<'a, DB: Database> DbTool<'a, DB> { self.db.view(|tx| tx.get::(key))?.map_err(|e| eyre::eyre!(e)) } + /// Grabs the content of the DupSort table for the given key and subkey + pub fn get_dup(&self, key: T::Key, subkey: T::SubKey) -> Result> { + self.db + .view(|tx| tx.cursor_dup_read::().unwrap().seek_by_key_subkey(key, subkey)) + .unwrap() + .map_err(|e| eyre::eyre!(e)) + } + /// Drops the database at the given path. pub fn drop(&mut self, path: impl AsRef) -> Result<()> { let path = path.as_ref(); diff --git a/crates/storage/db/src/tables/mod.rs b/crates/storage/db/src/tables/mod.rs index 7171f137c6b5..b2eae0d60ef2 100644 --- a/crates/storage/db/src/tables/mod.rs +++ b/crates/storage/db/src/tables/mod.rs @@ -87,6 +87,9 @@ pub trait TableViewer { /// operate on table in generic way fn view(&self) -> Result; + + /// operate on dupsort table in generic way + fn view_dupsort(&self) -> Result; } macro_rules! tables { @@ -133,6 +136,19 @@ macro_rules! tables { },)* } } + + /// Allows to operate on specific table type + pub fn view_dupsort(&self, visitor: &T) -> Result + where + T: TableViewer, + { + match self { + $(Tables::$table => { + visitor.view_dupsort::<$table>() + },)* + } + + } } impl Display for Tables { From b4c1bf940ecb41e2eeb2bbb51a6d5f7f671af340 Mon Sep 17 00:00:00 2001 From: Srdjan S Date: Sun, 10 Sep 2023 11:27:44 +0200 Subject: [PATCH 02/13] Pass TableViewer view fn into macro --- bin/reth/src/db/clear.rs | 8 ++++ bin/reth/src/db/get.rs | 12 +---- bin/reth/src/db/list.rs | 8 +++- crates/storage/db/src/tables/mod.rs | 71 ++++++++++++----------------- 4 files changed, 44 insertions(+), 55 deletions(-) diff --git a/bin/reth/src/db/clear.rs b/bin/reth/src/db/clear.rs index 626df4bae01d..6db4fa8717e3 100644 --- a/bin/reth/src/db/clear.rs +++ b/bin/reth/src/db/clear.rs @@ -36,4 +36,12 @@ impl TableViewer<()> for ClearViewer<'_, DB> { Ok(()) } + + fn view_dupsort(&self) -> Result<(), Self::Error> { + let tx = self.db.tx_mut()?; + tx.clear::()?; + tx.commit()?; + + Ok(()) + } } diff --git a/bin/reth/src/db/get.rs b/bin/reth/src/db/get.rs index 536c28fc921e..da1cb6bb40bb 100644 --- a/bin/reth/src/db/get.rs +++ b/bin/reth/src/db/get.rs @@ -1,14 +1,11 @@ -use std::error::Error; - use crate::utils::DbTool; use clap::Parser; use reth_db::{ database::Database, table::{DupSort, Table}, - TableType, TableViewer, Tables, + TableViewer, Tables, }; -use serde::Deserialize; use tracing::error; /// The arguments for the `reth db get` command @@ -31,13 +28,6 @@ pub struct Command { impl Command { /// Execute `db get` command pub fn execute(self, tool: &DbTool<'_, DB>) -> eyre::Result<()> { - if self.table.table_type() == TableType::DupSort { - self.table.view_dupsort(&GetValueViewer { tool, args: &self })?; - - error!(target: "reth::cli", "Unsupported table."); - - return Ok(()); - } self.table.view(&GetValueViewer { tool, args: &self })?; Ok(()) diff --git a/bin/reth/src/db/list.rs b/bin/reth/src/db/list.rs index 3403fc093ec7..6350a7054047 100644 --- a/bin/reth/src/db/list.rs +++ b/bin/reth/src/db/list.rs @@ -5,9 +5,9 @@ use eyre::WrapErr; use reth_primitives::hex; use reth_db::{ database::Database, - mdbx::{Env, WriteMap}, table::{DupSort, Table}, - DatabaseEnvRO, TableType, TableViewer, Tables, + DatabaseEnvRO, TableType, TableViewer, Tables +}; use std::cell::RefCell; use tracing::error; @@ -43,6 +43,7 @@ pub struct Command { impl Command { /// Execute `db list` command pub fn execute(self, tool: &DbTool<'_, DatabaseEnvRO>) -> eyre::Result<()> { +<<<<<<< HEAD <<<<<<< HEAD self.table.view(&ListTableViewer { tool, args: &self }) ======= @@ -51,6 +52,9 @@ impl Command { } else { self.table.view(&ListTableViewer { tool, args: &self })?; } +======= + self.table.view(&ListTableViewer { tool, args: &self })?; +>>>>>>> ad15a5c7 (Pass TableViewer view fn into macro) Ok(()) >>>>>>> 5b1f48f2 (work in progress) diff --git a/crates/storage/db/src/tables/mod.rs b/crates/storage/db/src/tables/mod.rs index b2eae0d60ef2..4f4653d79bae 100644 --- a/crates/storage/db/src/tables/mod.rs +++ b/crates/storage/db/src/tables/mod.rs @@ -89,11 +89,11 @@ pub trait TableViewer { fn view(&self) -> Result; /// operate on dupsort table in generic way - fn view_dupsort(&self) -> Result; + fn view_dupsort(&self) -> Result; } macro_rules! tables { - ([$(($table:ident, $type:expr)),*]) => { + ([$(($table:ident, $type:expr, $view_fn:ident)),*]) => { #[derive(Debug, PartialEq, Copy, Clone)] /// Default tables that should be present inside database. pub enum Tables { @@ -132,23 +132,10 @@ macro_rules! tables { { match self { $(Tables::$table => { - visitor.view::<$table>() + visitor.$view_fn::<$table>() },)* } } - - /// Allows to operate on specific table type - pub fn view_dupsort(&self, visitor: &T) -> Result - where - T: TableViewer, - { - match self { - $(Tables::$table => { - visitor.view_dupsort::<$table>() - },)* - } - - } } impl Display for Tables { @@ -175,32 +162,32 @@ macro_rules! tables { } tables!([ - (CanonicalHeaders, TableType::Table), - (HeaderTD, TableType::Table), - (HeaderNumbers, TableType::Table), - (Headers, TableType::Table), - (BlockBodyIndices, TableType::Table), - (BlockOmmers, TableType::Table), - (BlockWithdrawals, TableType::Table), - (TransactionBlock, TableType::Table), - (Transactions, TableType::Table), - (TxHashNumber, TableType::Table), - (Receipts, TableType::Table), - (PlainAccountState, TableType::Table), - (PlainStorageState, TableType::DupSort), - (Bytecodes, TableType::Table), - (AccountHistory, TableType::Table), - (StorageHistory, TableType::Table), - (AccountChangeSet, TableType::DupSort), - (StorageChangeSet, TableType::DupSort), - (HashedAccount, TableType::Table), - (HashedStorage, TableType::DupSort), - (AccountsTrie, TableType::Table), - (StoragesTrie, TableType::DupSort), - (TxSenders, TableType::Table), - (SyncStage, TableType::Table), - (SyncStageProgress, TableType::Table), - (PruneCheckpoints, TableType::Table) + (CanonicalHeaders, TableType::Table, view), + (HeaderTD, TableType::Table, view), + (HeaderNumbers, TableType::Table, view), + (Headers, TableType::Table, view), + (BlockBodyIndices, TableType::Table, view), + (BlockOmmers, TableType::Table, view), + (BlockWithdrawals, TableType::Table, view), + (TransactionBlock, TableType::Table, view), + (Transactions, TableType::Table, view), + (TxHashNumber, TableType::Table, view), + (Receipts, TableType::Table, view), + (PlainAccountState, TableType::Table, view), + (PlainStorageState, TableType::DupSort, view_dupsort), + (Bytecodes, TableType::Table, view), + (AccountHistory, TableType::Table, view), + (StorageHistory, TableType::Table, view), + (AccountChangeSet, TableType::DupSort, view_dupsort), + (StorageChangeSet, TableType::DupSort, view_dupsort), + (HashedAccount, TableType::Table, view), + (HashedStorage, TableType::DupSort, view_dupsort), + (AccountsTrie, TableType::Table, view), + (StoragesTrie, TableType::DupSort, view_dupsort), + (TxSenders, TableType::Table, view), + (SyncStage, TableType::Table, view), + (SyncStageProgress, TableType::Table, view), + (PruneCheckpoints, TableType::Table, view) ]); #[macro_export] From 36ff44d063b1a525b84a8d407701627b48fe8a5b Mon Sep 17 00:00:00 2001 From: Srdjan S Date: Sun, 10 Sep 2023 11:42:08 +0200 Subject: [PATCH 03/13] Remove redundant copy calls --- bin/reth/src/db/get.rs | 9 ++++----- bin/reth/src/db/list.rs | 2 +- bin/reth/src/utils.rs | 6 +++--- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/bin/reth/src/db/get.rs b/bin/reth/src/db/get.rs index da1cb6bb40bb..88386f1d317a 100644 --- a/bin/reth/src/db/get.rs +++ b/bin/reth/src/db/get.rs @@ -57,13 +57,12 @@ impl TableViewer<()> for GetValueViewer<'_, DB> { fn view(&self) -> Result<(), Self::Error> { let key = self.args.table_key::()?; - assert_eq!(T::NAME, self.args.table.name()); - match self.tool.clone().get::(key)? { + match self.tool.get::(key)? { Some(content) => { println!("{}", serde_json::to_string_pretty(&content)?); } None => { - error!(target: "reth::cli", "No content for the given table key111."); + error!(target: "reth::cli", "No content for the given table key."); } }; @@ -78,12 +77,12 @@ impl TableViewer<()> for GetValueViewer<'_, DB> { // process dupsort table let subkey = self.args.table_subkey::()?; - match self.tool.clone().get_dup::(key, subkey)? { + match self.tool.get_dup::(key, subkey)? { Some(content) => { println!("{}", serde_json::to_string_pretty(&content)?); } None => { - error!(target: "reth::cli", "No content for the given table subkey222s."); + error!(target: "reth::cli", "No content for the given table subkey."); } }; Ok(()) diff --git a/bin/reth/src/db/list.rs b/bin/reth/src/db/list.rs index 6350a7054047..75f36a41c76c 100644 --- a/bin/reth/src/db/list.rs +++ b/bin/reth/src/db/list.rs @@ -67,7 +67,7 @@ impl Command { .as_ref() .map(|search| { if let Some(search) = search.strip_prefix("0x") { - return hex::decode(search).unwrap(); + return hex::decode(search).unwrap() } search.as_bytes().to_vec() }) diff --git a/bin/reth/src/utils.rs b/bin/reth/src/utils.rs index d562237ae5e1..6bd7981dd8fd 100644 --- a/bin/reth/src/utils.rs +++ b/bin/reth/src/utils.rs @@ -132,7 +132,7 @@ impl<'a, DB: Database> DbTool<'a, DB> { let result = || { if filter.only_count { - return None; + return None } Some(( ::Key::decode(&key).unwrap(), @@ -151,12 +151,12 @@ impl<'a, DB: Database> DbTool<'a, DB> { >>>>>>> 5b1f48f2 (work in progress) { hits += 1; - return result(); + return result() } } None => { hits += 1; - return result(); + return result() } } } From 8e91e9d410dd37fa08d577480f97ac6ffd9ff7d4 Mon Sep 17 00:00:00 2001 From: Srdjan S Date: Tue, 26 Sep 2023 10:18:10 +0200 Subject: [PATCH 04/13] Use DupSort bound instead Table + DupSort --- bin/reth/src/db/clear.rs | 4 ++-- bin/reth/src/db/get.rs | 5 ++--- bin/reth/src/db/list.rs | 22 ++++------------------ bin/reth/src/utils.rs | 15 +++++---------- crates/storage/db/src/tables/mod.rs | 2 +- 5 files changed, 14 insertions(+), 34 deletions(-) diff --git a/bin/reth/src/db/clear.rs b/bin/reth/src/db/clear.rs index 6db4fa8717e3..ef49746d215d 100644 --- a/bin/reth/src/db/clear.rs +++ b/bin/reth/src/db/clear.rs @@ -1,7 +1,7 @@ use clap::Parser; use reth_db::{ database::Database, - table::Table, + table::{Table, DupSort}, transaction::{DbTx, DbTxMut}, TableViewer, Tables, }; @@ -37,7 +37,7 @@ impl TableViewer<()> for ClearViewer<'_, DB> { Ok(()) } - fn view_dupsort(&self) -> Result<(), Self::Error> { + fn view_dupsort(&self) -> Result<(), Self::Error> { let tx = self.db.tx_mut()?; tx.clear::()?; tx.commit()?; diff --git a/bin/reth/src/db/get.rs b/bin/reth/src/db/get.rs index 88386f1d317a..7fcdf37005cf 100644 --- a/bin/reth/src/db/get.rs +++ b/bin/reth/src/db/get.rs @@ -40,7 +40,7 @@ impl Command { serde_json::from_str::(&self.key).map_err(|e| eyre::eyre!(e)) } /// Get an instance of subkey for given dupsort table - fn table_subkey(&self) -> Result { + fn table_subkey(&self) -> Result { assert_eq!(T::NAME, self.table.name()); serde_json::from_str::(&self.subkey.clone().unwrap_or_default()) .map_err(|e| eyre::eyre!(e)) @@ -69,8 +69,7 @@ impl TableViewer<()> for GetValueViewer<'_, DB> { Ok(()) } - fn view_dupsort(&self) -> Result<(), Self::Error> { - // error here + fn view_dupsort(&self) -> Result<(), Self::Error> { // get a key for given table let key = self.args.table_key::()?; diff --git a/bin/reth/src/db/list.rs b/bin/reth/src/db/list.rs index 75f36a41c76c..6db7bbdeb661 100644 --- a/bin/reth/src/db/list.rs +++ b/bin/reth/src/db/list.rs @@ -2,12 +2,12 @@ use super::tui::DbListTUI; use crate::utils::{DbTool, ListFilter}; use clap::Parser; use eyre::WrapErr; -use reth_primitives::hex; use reth_db::{ database::Database, table::{DupSort, Table}, - DatabaseEnvRO, TableType, TableViewer, Tables + DatabaseEnvRO, TableViewer, Tables, }; +use reth_primitives::hex; use std::cell::RefCell; use tracing::error; @@ -43,21 +43,7 @@ pub struct Command { impl Command { /// Execute `db list` command pub fn execute(self, tool: &DbTool<'_, DatabaseEnvRO>) -> eyre::Result<()> { -<<<<<<< HEAD -<<<<<<< HEAD self.table.view(&ListTableViewer { tool, args: &self }) -======= - if self.table.table_type() == TableType::DupSort { - self.table.view_dupsort(&ListTableViewer { tool, args: &self })?; - } else { - self.table.view(&ListTableViewer { tool, args: &self })?; - } -======= - self.table.view(&ListTableViewer { tool, args: &self })?; ->>>>>>> ad15a5c7 (Pass TableViewer view fn into macro) - - Ok(()) ->>>>>>> 5b1f48f2 (work in progress) } /// Generate [`ListFilter`] from command. @@ -67,7 +53,7 @@ impl Command { .as_ref() .map(|search| { if let Some(search) = search.strip_prefix("0x") { - return hex::decode(search).unwrap() + return hex::decode(search).unwrap(); } search.as_bytes().to_vec() }) @@ -132,7 +118,7 @@ impl TableViewer<()> for ListTableViewer<'_> { Ok(()) } - fn view_dupsort(&self) -> Result<(), Self::Error> { + fn view_dupsort(&self) -> Result<(), Self::Error> { self.tool.db.view(|tx| { let table_db = tx.inner.open_db(Some(self.args.table.name())).wrap_err("Could not open db.")?; let stats = tx.inner.db_stat(&table_db).wrap_err(format!("Could not find table: {}", stringify!($table)))?; diff --git a/bin/reth/src/utils.rs b/bin/reth/src/utils.rs index 6bd7981dd8fd..8781e8ff80d0 100644 --- a/bin/reth/src/utils.rs +++ b/bin/reth/src/utils.rs @@ -132,7 +132,7 @@ impl<'a, DB: Database> DbTool<'a, DB> { let result = || { if filter.only_count { - return None + return None; } Some(( ::Key::decode(&key).unwrap(), @@ -142,21 +142,16 @@ impl<'a, DB: Database> DbTool<'a, DB> { match &*bmb { Some(searcher) => { -<<<<<<< HEAD - if searcher.find_first_in(&value).is_some() || - searcher.find_first_in(&key).is_some() -======= - if searcher.find_first_in(v.raw_value()).is_some() - || searcher.find_first_in(k.raw_key()).is_some() ->>>>>>> 5b1f48f2 (work in progress) + if searcher.find_first_in(&value).is_some() + || searcher.find_first_in(&key).is_some() { hits += 1; - return result() + return result(); } } None => { hits += 1; - return result() + return result(); } } } diff --git a/crates/storage/db/src/tables/mod.rs b/crates/storage/db/src/tables/mod.rs index 4f4653d79bae..061c32a93cfd 100644 --- a/crates/storage/db/src/tables/mod.rs +++ b/crates/storage/db/src/tables/mod.rs @@ -89,7 +89,7 @@ pub trait TableViewer { fn view(&self) -> Result; /// operate on dupsort table in generic way - fn view_dupsort(&self) -> Result; + fn view_dupsort(&self) -> Result; } macro_rules! tables { From 8ce381b988755c747660bf4dca421d76c95a5939 Mon Sep 17 00:00:00 2001 From: Srdjan S Date: Wed, 15 Nov 2023 14:49:25 +0100 Subject: [PATCH 05/13] Add raw arg to list --- bin/reth/src/db/get.rs | 4 +--- bin/reth/src/db/list.rs | 6 +++--- bin/reth/src/db/tui.rs | 6 +++--- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/bin/reth/src/db/get.rs b/bin/reth/src/db/get.rs index b093abde4c19..d0fe33098dc3 100644 --- a/bin/reth/src/db/get.rs +++ b/bin/reth/src/db/get.rs @@ -2,12 +2,10 @@ use crate::utils::DbTool; use clap::Parser; use reth_db::{ - RawKey, TableType, RawTable, database::Database, table::{DupSort, Table}, - TableViewer, Tables, + RawKey, RawTable, TableViewer, Tables, }; -use reth_db::{database::Database, table::Table, , RawTable, TableType, TableViewer, Tables}; use tracing::error; /// The arguments for the `reth db get` command diff --git a/bin/reth/src/db/list.rs b/bin/reth/src/db/list.rs index 80edcebd2459..cc6b97b3d386 100644 --- a/bin/reth/src/db/list.rs +++ b/bin/reth/src/db/list.rs @@ -4,8 +4,8 @@ use clap::Parser; use eyre::WrapErr; use reth_db::{ database::Database, - table::{DupSort, Table}, RawValue, - DatabaseEnvRO, TableViewer, Tables, + table::{DupSort, Table}, + DatabaseEnvRO, RawValue, TableViewer, Tables, }; use reth_primitives::hex; use std::cell::RefCell; @@ -169,7 +169,7 @@ impl TableViewer<()> for ListTableViewer<'_> { DbListTUI::<_, T>::new(|skip, len| { list_filter.borrow_mut().update_page(skip, len); self.tool.list::(&list_filter.borrow()).unwrap().0 - }, self.args.skip, self.args.len, total_entries).run() + }, self.args.skip, self.args.len, total_entries, self.args.raw).run() } })??; diff --git a/bin/reth/src/db/tui.rs b/bin/reth/src/db/tui.rs index 52c1e8a692ec..20986aac6f44 100644 --- a/bin/reth/src/db/tui.rs +++ b/bin/reth/src/db/tui.rs @@ -195,7 +195,7 @@ where /// Fetch the next page of items fn next_page(&mut self) { if self.skip + self.count >= self.total_entries { - return + return; } self.skip += self.count; @@ -205,7 +205,7 @@ where /// Fetch the previous page of items fn previous_page(&mut self) { if self.skip == 0 { - return + return; } self.skip = self.skip.saturating_sub(self.count); @@ -311,7 +311,7 @@ where } } - return Ok(false) + return Ok(false); } match event { From fe7fbabfde74868bb93fac6d300c573d5dff8e24 Mon Sep 17 00:00:00 2001 From: Roman Krasiuk Date: Mon, 18 Dec 2023 20:18:39 +0200 Subject: [PATCH 06/13] refactor tables macro to treat table & dupsort separately --- crates/storage/db/src/tables/mod.rs | 80 ++++++++++++++++++----------- 1 file changed, 50 insertions(+), 30 deletions(-) diff --git a/crates/storage/db/src/tables/mod.rs b/crates/storage/db/src/tables/mod.rs index 7088756a24f2..31275193393f 100644 --- a/crates/storage/db/src/tables/mod.rs +++ b/crates/storage/db/src/tables/mod.rs @@ -93,7 +93,10 @@ pub trait TableViewer { } macro_rules! tables { - ([$(($table:ident, $type:expr, $view_fn:ident)),*]) => { + ([ + (TableType::Table, [$($table:ident),*]), + (TableType::DupSort, [$($dupsort:ident),*]) + ]) => { #[derive(Debug, PartialEq, Copy, Clone)] /// Default tables that should be present inside database. pub enum Tables { @@ -101,11 +104,15 @@ macro_rules! tables { #[doc = concat!("Represents a ", stringify!($table), " table")] $table, )* + $( + #[doc = concat!("Represents a ", stringify!($dupsort), " dupsort table")] + $dupsort, + )* } impl Tables { /// Array of all tables in database - pub const ALL: [Tables; NUM_TABLES] = [$(Tables::$table,)*]; + pub const ALL: [Tables; NUM_TABLES] = [$(Tables::$table,)* $(Tables::$dupsort,)*]; /// The name of the given table in database pub const fn name(&self) -> &str { @@ -113,6 +120,9 @@ macro_rules! tables { $(Tables::$table => { $table::NAME },)* + $(Tables::$dupsort => { + $dupsort::NAME + },)* } } @@ -120,7 +130,10 @@ macro_rules! tables { pub const fn table_type(&self) -> TableType { match self { $(Tables::$table => { - $type + TableType::Table + },)* + $(Tables::$dupsort => { + TableType::DupSort },)* } } @@ -132,7 +145,10 @@ macro_rules! tables { { match self { $(Tables::$table => { - visitor.$view_fn::<$table>() + visitor.view::<$table>() + },)* + $(Tables::$dupsort => { + visitor.view_dupsort::<$dupsort>() },)* } } @@ -162,32 +178,36 @@ macro_rules! tables { } tables!([ - (CanonicalHeaders, TableType::Table, view), - (HeaderTD, TableType::Table, view), - (HeaderNumbers, TableType::Table, view), - (Headers, TableType::Table, view), - (BlockBodyIndices, TableType::Table, view), - (BlockOmmers, TableType::Table, view), - (BlockWithdrawals, TableType::Table, view), - (TransactionBlock, TableType::Table, view), - (Transactions, TableType::Table, view), - (TxHashNumber, TableType::Table, view), - (Receipts, TableType::Table, view), - (PlainAccountState, TableType::Table, view), - (PlainStorageState, TableType::DupSort, view_dupsort), - (Bytecodes, TableType::Table, view), - (AccountHistory, TableType::Table, view), - (StorageHistory, TableType::Table, view), - (AccountChangeSet, TableType::DupSort, view_dupsort), - (StorageChangeSet, TableType::DupSort, view_dupsort), - (HashedAccount, TableType::Table, view), - (HashedStorage, TableType::DupSort, view_dupsort), - (AccountsTrie, TableType::Table, view), - (StoragesTrie, TableType::DupSort, view_dupsort), - (TxSenders, TableType::Table, view), - (SyncStage, TableType::Table, view), - (SyncStageProgress, TableType::Table, view), - (PruneCheckpoints, TableType::Table, view) + ( + TableType::Table, + [ + CanonicalHeaders, + HeaderTD, + HeaderNumbers, + Headers, + BlockBodyIndices, + BlockOmmers, + BlockWithdrawals, + TransactionBlock, + Transactions, + TxHashNumber, + Receipts, + PlainAccountState, + Bytecodes, + AccountHistory, + StorageHistory, + HashedAccount, + AccountsTrie, + TxSenders, + SyncStage, + SyncStageProgress, + PruneCheckpoints + ] + ), + ( + TableType::DupSort, + [PlainStorageState, AccountChangeSet, StorageChangeSet, HashedStorage, StoragesTrie] + ) ]); /// Macro to declare key value table. From 555bb871d17e28f025b843e04fa2bc949d1ca5a4 Mon Sep 17 00:00:00 2001 From: Roman Krasiuk Date: Mon, 18 Dec 2023 10:39:12 -0800 Subject: [PATCH 07/13] Apply suggestions from code review --- bin/reth/src/db/list.rs | 4 ++-- bin/reth/src/db/tui.rs | 6 +++--- bin/reth/src/utils.rs | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/bin/reth/src/db/list.rs b/bin/reth/src/db/list.rs index d347e53bada3..28bddc956f38 100644 --- a/bin/reth/src/db/list.rs +++ b/bin/reth/src/db/list.rs @@ -65,7 +65,7 @@ impl Command { .as_ref() .map(|search| { if let Some(search) = search.strip_prefix("0x") { - return hex::decode(search).unwrap(); + return hex::decode(search).unwrap() } search.as_bytes().to_vec() }) @@ -148,7 +148,7 @@ impl TableViewer<()> for ListTableViewer<'_> { final_entry_idx = total_entries - 1, table = self.args.table.name() ); - return Ok(()); + return Ok(()) } diff --git a/bin/reth/src/db/tui.rs b/bin/reth/src/db/tui.rs index 20986aac6f44..52c1e8a692ec 100644 --- a/bin/reth/src/db/tui.rs +++ b/bin/reth/src/db/tui.rs @@ -195,7 +195,7 @@ where /// Fetch the next page of items fn next_page(&mut self) { if self.skip + self.count >= self.total_entries { - return; + return } self.skip += self.count; @@ -205,7 +205,7 @@ where /// Fetch the previous page of items fn previous_page(&mut self) { if self.skip == 0 { - return; + return } self.skip = self.skip.saturating_sub(self.count); @@ -311,7 +311,7 @@ where } } - return Ok(false); + return Ok(false) } match event { diff --git a/bin/reth/src/utils.rs b/bin/reth/src/utils.rs index 08a15658870b..d927ccb56b00 100644 --- a/bin/reth/src/utils.rs +++ b/bin/reth/src/utils.rs @@ -143,7 +143,7 @@ impl<'a, DB: Database> DbTool<'a, DB> { let result = || { if filter.only_count { - return None; + return None } Some(( ::Key::decode(&key).unwrap(), @@ -157,12 +157,12 @@ impl<'a, DB: Database> DbTool<'a, DB> { || searcher.find_first_in(&key).is_some() { hits += 1; - return result(); + return result() } } None => { hits += 1; - return result(); + return result() } } } From 5eb68a4e1c37f49a6182420e2856d633ce34acce Mon Sep 17 00:00:00 2001 From: Roman Krasiuk Date: Mon, 18 Dec 2023 20:42:02 +0200 Subject: [PATCH 08/13] remove unwraps --- bin/reth/src/utils.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/bin/reth/src/utils.rs b/bin/reth/src/utils.rs index d927ccb56b00..74af5288cebb 100644 --- a/bin/reth/src/utils.rs +++ b/bin/reth/src/utils.rs @@ -153,8 +153,8 @@ impl<'a, DB: Database> DbTool<'a, DB> { match &*bmb { Some(searcher) => { - if searcher.find_first_in(&value).is_some() - || searcher.find_first_in(&key).is_some() + if searcher.find_first_in(&value).is_some() || + searcher.find_first_in(&key).is_some() { hits += 1; return result() @@ -197,8 +197,7 @@ impl<'a, DB: Database> DbTool<'a, DB> { /// Grabs the content of the DupSort table for the given key and subkey pub fn get_dup(&self, key: T::Key, subkey: T::SubKey) -> Result> { self.db - .view(|tx| tx.cursor_dup_read::().unwrap().seek_by_key_subkey(key, subkey)) - .unwrap() + .view(|tx| tx.cursor_dup_read::()?.seek_by_key_subkey(key, subkey))? .map_err(|e| eyre::eyre!(e)) } From b676abc455c221e7fa3291a56704cd526493abd7 Mon Sep 17 00:00:00 2001 From: Roman Krasiuk Date: Thu, 21 Dec 2023 08:50:34 +0100 Subject: [PATCH 09/13] fix from str impl --- crates/storage/db/src/tables/mod.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/crates/storage/db/src/tables/mod.rs b/crates/storage/db/src/tables/mod.rs index 31275193393f..e160c67a8cec 100644 --- a/crates/storage/db/src/tables/mod.rs +++ b/crates/storage/db/src/tables/mod.rs @@ -166,10 +166,13 @@ macro_rules! tables { fn from_str(s: &str) -> Result { match s { $($table::NAME => { - return Ok(Tables::$table) + Ok(Tables::$table) + },)* + $($dupsort::NAME => { + Ok(Tables::$dupsort) },)* _ => { - return Err("Unknown table".to_string()) + Err("Unknown table".to_string()) } } } From bb183e9a71301fe9bd4bbb7a5e176a82cbfe15a7 Mon Sep 17 00:00:00 2001 From: Roman Krasiuk Date: Thu, 21 Dec 2023 08:52:17 +0100 Subject: [PATCH 10/13] fix fmt --- bin/reth/src/db/clear.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/reth/src/db/clear.rs b/bin/reth/src/db/clear.rs index ef49746d215d..6a3511788000 100644 --- a/bin/reth/src/db/clear.rs +++ b/bin/reth/src/db/clear.rs @@ -1,7 +1,7 @@ use clap::Parser; use reth_db::{ database::Database, - table::{Table, DupSort}, + table::{DupSort, Table}, transaction::{DbTx, DbTxMut}, TableViewer, Tables, }; From d1dfa054e408eb071c61ebfc2cd491bc464d5816 Mon Sep 17 00:00:00 2001 From: Roman Krasiuk Date: Thu, 21 Dec 2023 09:17:57 +0100 Subject: [PATCH 11/13] fix test and docs --- crates/storage/db/src/tables/mod.rs | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/crates/storage/db/src/tables/mod.rs b/crates/storage/db/src/tables/mod.rs index e160c67a8cec..6a4e73ae04ca 100644 --- a/crates/storage/db/src/tables/mod.rs +++ b/crates/storage/db/src/tables/mod.rs @@ -59,7 +59,10 @@ pub const NUM_TABLES: usize = 26; /// # Example /// /// ``` -/// use reth_db::{table::Table, TableViewer, Tables}; +/// use reth_db::{ +/// table::{DupSort, Table}, +/// TableViewer, Tables, +/// }; /// use std::str::FromStr; /// /// let headers = Tables::from_str("Headers").unwrap(); @@ -71,7 +74,12 @@ pub const NUM_TABLES: usize = 26; /// type Error = &'static str; /// /// fn view(&self) -> Result<(), Self::Error> { -/// // operate on table in generic way +/// // operate on table in a generic way +/// Ok(()) +/// } +/// +/// fn view_dupsort(&self) -> Result<(), Self::Error> { +/// // operate on a dupsort table in a generic way /// Ok(()) /// } /// } @@ -465,20 +473,20 @@ mod tests { (TableType::Table, TxHashNumber::NAME), (TableType::Table, Receipts::NAME), (TableType::Table, PlainAccountState::NAME), - (TableType::DupSort, PlainStorageState::NAME), (TableType::Table, Bytecodes::NAME), (TableType::Table, AccountHistory::NAME), (TableType::Table, StorageHistory::NAME), - (TableType::DupSort, AccountChangeSet::NAME), - (TableType::DupSort, StorageChangeSet::NAME), (TableType::Table, HashedAccount::NAME), - (TableType::DupSort, HashedStorage::NAME), (TableType::Table, AccountsTrie::NAME), - (TableType::DupSort, StoragesTrie::NAME), (TableType::Table, TxSenders::NAME), (TableType::Table, SyncStage::NAME), (TableType::Table, SyncStageProgress::NAME), (TableType::Table, PruneCheckpoints::NAME), + (TableType::DupSort, PlainStorageState::NAME), + (TableType::DupSort, AccountChangeSet::NAME), + (TableType::DupSort, StorageChangeSet::NAME), + (TableType::DupSort, HashedStorage::NAME), + (TableType::DupSort, StoragesTrie::NAME), ]; #[test] From 731be0362286215fab7e83f50ca0af2c511f3f8c Mon Sep 17 00:00:00 2001 From: Roman Krasiuk Date: Thu, 21 Dec 2023 09:42:00 +0100 Subject: [PATCH 12/13] delegate to view by default --- bin/reth/src/db/clear.rs | 11 +------ bin/reth/src/db/list.rs | 49 ++--------------------------- crates/storage/db/src/tables/mod.rs | 11 ++++--- 3 files changed, 10 insertions(+), 61 deletions(-) diff --git a/bin/reth/src/db/clear.rs b/bin/reth/src/db/clear.rs index 6a3511788000..e26f60631afe 100644 --- a/bin/reth/src/db/clear.rs +++ b/bin/reth/src/db/clear.rs @@ -1,7 +1,7 @@ use clap::Parser; use reth_db::{ database::Database, - table::{DupSort, Table}, + table::Table, transaction::{DbTx, DbTxMut}, TableViewer, Tables, }; @@ -33,15 +33,6 @@ impl TableViewer<()> for ClearViewer<'_, DB> { let tx = self.db.tx_mut()?; tx.clear::()?; tx.commit()?; - - Ok(()) - } - - fn view_dupsort(&self) -> Result<(), Self::Error> { - let tx = self.db.tx_mut()?; - tx.clear::()?; - tx.commit()?; - Ok(()) } } diff --git a/bin/reth/src/db/list.rs b/bin/reth/src/db/list.rs index 28bddc956f38..49cb4affe9fe 100644 --- a/bin/reth/src/db/list.rs +++ b/bin/reth/src/db/list.rs @@ -2,11 +2,7 @@ use super::tui::DbListTUI; use crate::utils::{DbTool, ListFilter}; use clap::Parser; use eyre::WrapErr; -use reth_db::{ - database::Database, - table::{DupSort, Table}, - DatabaseEnv, RawValue, TableViewer, Tables, -}; +use reth_db::{database::Database, table::Table, DatabaseEnv, RawValue, TableViewer, Tables}; use reth_primitives::hex; use std::cell::RefCell; use tracing::error; @@ -105,7 +101,7 @@ impl TableViewer<()> for ListTableViewer<'_> { final_entry_idx = total_entries - 1, table = self.args.table.name() ); - return Ok(()); + return Ok(()) } @@ -134,45 +130,4 @@ impl TableViewer<()> for ListTableViewer<'_> { Ok(()) } - - fn view_dupsort(&self) -> Result<(), Self::Error> { - self.tool.db.view(|tx| { - let table_db = tx.inner.open_db(Some(self.args.table.name())).wrap_err("Could not open db.")?; - let stats = tx.inner.db_stat(&table_db).wrap_err(format!("Could not find table: {}", stringify!($table)))?; - let total_entries = stats.entries(); - if self.args.skip > total_entries - 1 { - error!( - target: "reth::cli", - "Start index {start} is greater than the final entry index ({final_entry_idx}) in the table {table}", - start = self.args.skip, - final_entry_idx = total_entries - 1, - table = self.args.table.name() - ); - return Ok(()) - } - - - let list_filter = self.args.list_filter(); - - if self.args.json || self.args.count { - let (list, count) = self.tool.list::(&list_filter)?; - - if self.args.count { - println!("{count} entries found.") - }else { - println!("{}", serde_json::to_string_pretty(&list)?); - } - Ok(()) - - } else { - let list_filter = RefCell::new(list_filter); - DbListTUI::<_, T>::new(|skip, len| { - list_filter.borrow_mut().update_page(skip, len); - self.tool.list::(&list_filter.borrow()).unwrap().0 - }, self.args.skip, self.args.len, total_entries, self.args.raw).run() - } - })??; - - Ok(()) - } } diff --git a/crates/storage/db/src/tables/mod.rs b/crates/storage/db/src/tables/mod.rs index 6a4e73ae04ca..dc19a406a96b 100644 --- a/crates/storage/db/src/tables/mod.rs +++ b/crates/storage/db/src/tables/mod.rs @@ -90,14 +90,17 @@ pub const NUM_TABLES: usize = 26; /// let _ = transactions.view(&viewer); /// ``` pub trait TableViewer { - /// type of error to return + /// The error type returned by the viewer. type Error; - /// operate on table in generic way + /// Operate on the table in a generic way. fn view(&self) -> Result; - /// operate on dupsort table in generic way - fn view_dupsort(&self) -> Result; + /// Operate on the dupsort table in a generic way. + /// By default, the `view` function is invoked unless overridden. + fn view_dupsort(&self) -> Result { + self.view::() + } } macro_rules! tables { From 12d25fd23dff82bd8b3365d05d09eed2e341f09c Mon Sep 17 00:00:00 2001 From: Roman Krasiuk Date: Thu, 21 Dec 2023 15:35:58 +0100 Subject: [PATCH 13/13] fmt --- bin/reth/src/db/get.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/reth/src/db/get.rs b/bin/reth/src/db/get.rs index d0fe33098dc3..cb5527f3e64c 100644 --- a/bin/reth/src/db/get.rs +++ b/bin/reth/src/db/get.rs @@ -1,6 +1,5 @@ use crate::utils::DbTool; use clap::Parser; - use reth_db::{ database::Database, table::{DupSort, Table}, @@ -43,6 +42,7 @@ impl Command { serde_json::from_str::(&self.key).map_err(|e| eyre::eyre!(e)) } + /// Get an instance of subkey for given dupsort table fn table_subkey(&self) -> Result { assert_eq!(T::NAME, self.table.name()); @@ -100,6 +100,7 @@ impl TableViewer<()> for GetValueViewer<'_, DB> { Ok(()) } } + /// Map the user input value to json fn maybe_json_value_parser(value: &str) -> Result { if serde_json::from_str::(value).is_ok() {