From c4c6787ba701aba097fe2a470f45830813ad9155 Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Tue, 30 Jan 2024 14:36:32 +0000 Subject: [PATCH 1/5] feat(bin): show snapshots in `db stats` --- bin/reth/src/commands/db/mod.rs | 119 ++-------------- bin/reth/src/commands/db/stats.rs | 213 ++++++++++++++++++++++++++++ crates/storage/nippy-jar/src/lib.rs | 5 + 3 files changed, 229 insertions(+), 108 deletions(-) create mode 100644 bin/reth/src/commands/db/stats.rs diff --git a/bin/reth/src/commands/db/mod.rs b/bin/reth/src/commands/db/mod.rs index e6f199cd7a6e..afc101c4698d 100644 --- a/bin/reth/src/commands/db/mod.rs +++ b/bin/reth/src/commands/db/mod.rs @@ -9,16 +9,10 @@ use crate::{ utils::DbTool, }; use clap::{Parser, Subcommand}; -use comfy_table::{Cell, Row, Table as ComfyTable}; -use eyre::WrapErr; -use human_bytes::human_bytes; use reth_db::{ - database::Database, - mdbx, mdbx::DatabaseArguments, open_db, open_db_read_only, version::{get_db_version, DatabaseVersionError, DB_VERSION}, - Tables, }; use reth_primitives::ChainSpec; use std::{ @@ -31,6 +25,7 @@ mod diff; mod get; mod list; mod snapshots; +mod stats; /// DB List TUI mod tui; @@ -71,7 +66,7 @@ pub struct Command { /// `reth db` subcommands pub enum Subcommands { /// Lists all the tables, their entry count and their size - Stats, + Stats(stats::Command), /// Lists the contents of a table List(list::Command), /// Create a diff between two database tables or two entire databases. @@ -100,117 +95,27 @@ impl Command { // add network name to data dir let data_dir = self.datadir.unwrap_or_chain_default(self.chain.chain); let db_path = data_dir.db_path(); + let database_arguments = DatabaseArguments::default().log_level(self.db.log_level); match self.command { // TODO: We'll need to add this on the DB trait. - Subcommands::Stats { .. } => { - let db = open_db_read_only( - &db_path, - DatabaseArguments::default().log_level(self.db.log_level), - )?; + Subcommands::Stats(command) => { + let db = open_db_read_only(&db_path, database_arguments)?; let tool = DbTool::new(&db, self.chain.clone())?; - let mut stats_table = ComfyTable::new(); - stats_table.load_preset(comfy_table::presets::ASCII_MARKDOWN); - stats_table.set_header([ - "Table Name", - "# Entries", - "Branch Pages", - "Leaf Pages", - "Overflow Pages", - "Total Size", - ]); - - tool.db.view(|tx| { - let mut tables = - Tables::ALL.iter().map(|table| table.name()).collect::>(); - tables.sort(); - let mut total_size = 0; - for table in tables { - let table_db = - tx.inner.open_db(Some(table)).wrap_err("Could not open db.")?; - - let stats = tx - .inner - .db_stat(&table_db) - .wrap_err(format!("Could not find table: {table}"))?; - - // Defaults to 16KB right now but we should - // re-evaluate depending on the DB we end up using - // (e.g. REDB does not have these options as configurable intentionally) - let page_size = stats.page_size() as usize; - let leaf_pages = stats.leaf_pages(); - let branch_pages = stats.branch_pages(); - let overflow_pages = stats.overflow_pages(); - let num_pages = leaf_pages + branch_pages + overflow_pages; - let table_size = page_size * num_pages; - - total_size += table_size; - let mut row = Row::new(); - row.add_cell(Cell::new(table)) - .add_cell(Cell::new(stats.entries())) - .add_cell(Cell::new(branch_pages)) - .add_cell(Cell::new(leaf_pages)) - .add_cell(Cell::new(overflow_pages)) - .add_cell(Cell::new(human_bytes(table_size as f64))); - stats_table.add_row(row); - } - - let max_widths = stats_table.column_max_content_widths(); - - let mut seperator = Row::new(); - for width in max_widths { - seperator.add_cell(Cell::new("-".repeat(width as usize))); - } - stats_table.add_row(seperator); - - let mut row = Row::new(); - row.add_cell(Cell::new("Total DB size")) - .add_cell(Cell::new("")) - .add_cell(Cell::new("")) - .add_cell(Cell::new("")) - .add_cell(Cell::new("")) - .add_cell(Cell::new(human_bytes(total_size as f64))); - stats_table.add_row(row); - - let freelist = tx.inner.env().freelist()?; - let freelist_size = freelist * - tx.inner.db_stat(&mdbx::Database::freelist_db())?.page_size() as usize; - - let mut row = Row::new(); - row.add_cell(Cell::new("Freelist size")) - .add_cell(Cell::new(freelist)) - .add_cell(Cell::new("")) - .add_cell(Cell::new("")) - .add_cell(Cell::new("")) - .add_cell(Cell::new(human_bytes(freelist_size as f64))); - stats_table.add_row(row); - - Ok::<(), eyre::Report>(()) - })??; - - println!("{stats_table}"); + command.execute(data_dir, &tool)?; } Subcommands::List(command) => { - let db = open_db_read_only( - &db_path, - DatabaseArguments::default().log_level(self.db.log_level), - )?; + let db = open_db_read_only(&db_path, database_arguments)?; let tool = DbTool::new(&db, self.chain.clone())?; command.execute(&tool)?; } Subcommands::Diff(command) => { - let db = open_db_read_only( - &db_path, - DatabaseArguments::default().log_level(self.db.log_level), - )?; + let db = open_db_read_only(&db_path, database_arguments)?; let tool = DbTool::new(&db, self.chain.clone())?; command.execute(&tool)?; } Subcommands::Get(command) => { - let db = open_db_read_only( - &db_path, - DatabaseArguments::default().log_level(self.db.log_level), - )?; + let db = open_db_read_only(&db_path, database_arguments)?; let tool = DbTool::new(&db, self.chain.clone())?; command.execute(&tool)?; } @@ -230,14 +135,12 @@ impl Command { } } - let db = - open_db(&db_path, DatabaseArguments::default().log_level(self.db.log_level))?; + let db = open_db(&db_path, database_arguments)?; let mut tool = DbTool::new(&db, self.chain.clone())?; tool.drop(db_path)?; } Subcommands::Clear(command) => { - let db = - open_db(&db_path, DatabaseArguments::default().log_level(self.db.log_level))?; + let db = open_db(&db_path, database_arguments)?; command.execute(&db)?; } Subcommands::Snapshot(command) => { diff --git a/bin/reth/src/commands/db/stats.rs b/bin/reth/src/commands/db/stats.rs new file mode 100644 index 000000000000..d8f83202c5d2 --- /dev/null +++ b/bin/reth/src/commands/db/stats.rs @@ -0,0 +1,213 @@ +use crate::utils::DbTool; +use clap::Parser; +use comfy_table::{Cell, Row, Table as ComfyTable}; +use eyre::WrapErr; +use human_bytes::human_bytes; +use reth_db::{database::Database, mdbx, snapshot::iter_snapshots, DatabaseEnv, Tables}; +use reth_node_core::dirs::{ChainPath, DataDirPath}; +use reth_primitives::snapshot::find_fixed_range; +use reth_provider::providers::SnapshotProvider; +use std::fs::File; + +#[derive(Parser, Debug)] +/// The arguments for the `reth db stats` command +pub struct Command; + +impl Command { + /// Execute `db stats` command + pub fn execute( + self, + data_dir: ChainPath, + tool: &DbTool<'_, DatabaseEnv>, + ) -> eyre::Result<()> { + let db_stats_table = self.db_stats_table(tool)?; + println!("{db_stats_table}"); + + println!("\n"); + + let snapshots_stats_table = self.snapshots_stats_table(data_dir)?; + println!("{snapshots_stats_table}"); + + Ok(()) + } + + fn db_stats_table(&self, tool: &DbTool<'_, DatabaseEnv>) -> eyre::Result { + let mut table = ComfyTable::new(); + table.load_preset(comfy_table::presets::ASCII_MARKDOWN); + table.set_header([ + "Table Name", + "# Entries", + "Branch Pages", + "Leaf Pages", + "Overflow Pages", + "Total Size", + ]); + + tool.db.view(|tx| { + let mut db_tables = Tables::ALL.iter().map(|table| table.name()).collect::>(); + db_tables.sort(); + let mut total_size = 0; + for db_table in db_tables { + let table_db = tx.inner.open_db(Some(db_table)).wrap_err("Could not open db.")?; + + let stats = tx + .inner + .db_stat(&table_db) + .wrap_err(format!("Could not find table: {db_table}"))?; + + // Defaults to 16KB right now but we should + // re-evaluate depending on the DB we end up using + // (e.g. REDB does not have these options as configurable intentionally) + let page_size = stats.page_size() as usize; + let leaf_pages = stats.leaf_pages(); + let branch_pages = stats.branch_pages(); + let overflow_pages = stats.overflow_pages(); + let num_pages = leaf_pages + branch_pages + overflow_pages; + let table_size = page_size * num_pages; + + total_size += table_size; + let mut row = Row::new(); + row.add_cell(Cell::new(db_table)) + .add_cell(Cell::new(stats.entries())) + .add_cell(Cell::new(branch_pages)) + .add_cell(Cell::new(leaf_pages)) + .add_cell(Cell::new(overflow_pages)) + .add_cell(Cell::new(human_bytes(table_size as f64))); + table.add_row(row); + } + + let max_widths = table.column_max_content_widths(); + let mut seperator = Row::new(); + for width in max_widths { + seperator.add_cell(Cell::new("-".repeat(width as usize))); + } + table.add_row(seperator); + + let mut row = Row::new(); + row.add_cell(Cell::new("Tables")) + .add_cell(Cell::new("")) + .add_cell(Cell::new("")) + .add_cell(Cell::new("")) + .add_cell(Cell::new("")) + .add_cell(Cell::new(human_bytes(total_size as f64))); + table.add_row(row); + + let freelist = tx.inner.env().freelist()?; + let freelist_size = + freelist * tx.inner.db_stat(&mdbx::Database::freelist_db())?.page_size() as usize; + + let mut row = Row::new(); + row.add_cell(Cell::new("Freelist")) + .add_cell(Cell::new(freelist)) + .add_cell(Cell::new("")) + .add_cell(Cell::new("")) + .add_cell(Cell::new("")) + .add_cell(Cell::new(human_bytes(freelist_size as f64))); + table.add_row(row); + + Ok::<(), eyre::Report>(()) + })??; + + Ok(table) + } + + fn snapshots_stats_table(&self, data_dir: ChainPath) -> eyre::Result { + let mut table = ComfyTable::new(); + table.load_preset(comfy_table::presets::ASCII_MARKDOWN); + table.set_header([ + "Segment", + "Block Range", + "Transaction Range", + "Columns", + "Rows", + "Data Size", + "Index Size", + "Offsets Size", + "Config Size", + "Total Size", + ]); + + let snapshots = iter_snapshots(data_dir.snapshots_path())?; + let snapshot_provider = SnapshotProvider::new(data_dir.snapshots_path())?; + + let mut total_data_size = 0; + let mut total_index_size = 0; + let mut total_offsets_size = 0; + let mut total_config_size = 0; + + for (segment, ranges) in snapshots { + for (block_range, tx_range) in ranges { + let fixed_block_range = find_fixed_range(*block_range.start()); + let jar_provider = snapshot_provider + .get_segment_provider(segment, || Some(fixed_block_range.clone()), None)? + .expect("something went wrong"); + + let columns = jar_provider.columns(); + let rows = jar_provider.rows(); + let data_size = File::open(jar_provider.data_path()) + .and_then(|file| file.metadata()) + .map(|metadata| metadata.len()) + .unwrap_or_default(); + let index_size = File::open(jar_provider.index_path()) + .and_then(|file| file.metadata()) + .map(|metadata| metadata.len()) + .unwrap_or_default(); + let offsets_size = File::open(jar_provider.offsets_path()) + .and_then(|file| file.metadata()) + .map(|metadata| metadata.len()) + .unwrap_or_default(); + let config_size = File::open(jar_provider.config_path()) + .and_then(|file| file.metadata()) + .map(|metadata| metadata.len()) + .unwrap_or_default(); + + let mut row = Row::new(); + row.add_cell(Cell::new(segment)) + .add_cell(Cell::new(format!("{block_range:?}"))) + .add_cell(Cell::new( + tx_range.map_or("N/A".to_string(), |tx_range| format!("{tx_range:?}")), + )) + .add_cell(Cell::new(columns)) + .add_cell(Cell::new(rows)) + .add_cell(Cell::new(human_bytes(data_size as f64))) + .add_cell(Cell::new(human_bytes(index_size as f64))) + .add_cell(Cell::new(human_bytes(offsets_size as f64))) + .add_cell(Cell::new(human_bytes(config_size as f64))) + .add_cell(Cell::new(human_bytes( + (data_size + index_size + offsets_size + config_size) as f64, + ))); + table.add_row(row); + + total_data_size += data_size; + total_index_size += index_size; + total_offsets_size += offsets_size; + total_config_size += config_size; + } + } + + let max_widths = table.column_max_content_widths(); + let mut seperator = Row::new(); + for width in max_widths { + seperator.add_cell(Cell::new("-".repeat(width as usize))); + } + table.add_row(seperator); + + let mut row = Row::new(); + row.add_cell(Cell::new("Total")) + .add_cell(Cell::new("")) + .add_cell(Cell::new("")) + .add_cell(Cell::new("")) + .add_cell(Cell::new("")) + .add_cell(Cell::new(human_bytes(total_data_size as f64))) + .add_cell(Cell::new(human_bytes(total_index_size as f64))) + .add_cell(Cell::new(human_bytes(total_offsets_size as f64))) + .add_cell(Cell::new(human_bytes(total_config_size as f64))) + .add_cell(Cell::new(human_bytes( + (total_data_size + total_index_size + total_offsets_size + total_config_size) + as f64, + ))); + table.add_row(row); + + Ok(table) + } +} diff --git a/crates/storage/nippy-jar/src/lib.rs b/crates/storage/nippy-jar/src/lib.rs index 7aaf9b6eccdd..5e9259d5a9c7 100644 --- a/crates/storage/nippy-jar/src/lib.rs +++ b/crates/storage/nippy-jar/src/lib.rs @@ -207,6 +207,11 @@ impl NippyJar { &self.user_header } + /// Gets total columns in jar. + pub fn columns(&self) -> usize { + self.columns + } + /// Gets total rows in jar. pub fn rows(&self) -> usize { self.rows From c54aed4e9cd504d3cebc2462f74917c97f958095 Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Tue, 30 Jan 2024 14:48:19 +0000 Subject: [PATCH 2/5] --detailed-sizes --- bin/reth/src/commands/db/stats.rs | 81 ++++++++++++++++++------------- 1 file changed, 47 insertions(+), 34 deletions(-) diff --git a/bin/reth/src/commands/db/stats.rs b/bin/reth/src/commands/db/stats.rs index d8f83202c5d2..1cc41f353169 100644 --- a/bin/reth/src/commands/db/stats.rs +++ b/bin/reth/src/commands/db/stats.rs @@ -11,7 +11,11 @@ use std::fs::File; #[derive(Parser, Debug)] /// The arguments for the `reth db stats` command -pub struct Command; +pub struct Command { + /// Show detailed sizes for snapshot files. + #[arg(long, default_value_t = false)] + detailed_sizes: bool, +} impl Command { /// Execute `db stats` command @@ -25,7 +29,7 @@ impl Command { println!("\n"); - let snapshots_stats_table = self.snapshots_stats_table(data_dir)?; + let snapshots_stats_table = self.snapshots_stats_table(data_dir, self.detailed_sizes)?; println!("{snapshots_stats_table}"); Ok(()) @@ -111,21 +115,29 @@ impl Command { Ok(table) } - fn snapshots_stats_table(&self, data_dir: ChainPath) -> eyre::Result { + fn snapshots_stats_table( + &self, + data_dir: ChainPath, + detailed_sizes: bool, + ) -> eyre::Result { let mut table = ComfyTable::new(); table.load_preset(comfy_table::presets::ASCII_MARKDOWN); - table.set_header([ - "Segment", - "Block Range", - "Transaction Range", - "Columns", - "Rows", - "Data Size", - "Index Size", - "Offsets Size", - "Config Size", - "Total Size", - ]); + + if detailed_sizes { + table.set_header([ + "Segment", + "Block Range", + "Transaction Range", + "Shape", + "Data Size", + "Index Size", + "Offsets Size", + "Config Size", + "Total Size", + ]); + } else { + table.set_header(["Segment", "Block Range", "Transaction Range", "Shape", "Size"]); + } let snapshots = iter_snapshots(data_dir.snapshots_path())?; let snapshot_provider = SnapshotProvider::new(data_dir.snapshots_path())?; @@ -167,15 +179,16 @@ impl Command { .add_cell(Cell::new( tx_range.map_or("N/A".to_string(), |tx_range| format!("{tx_range:?}")), )) - .add_cell(Cell::new(columns)) - .add_cell(Cell::new(rows)) - .add_cell(Cell::new(human_bytes(data_size as f64))) - .add_cell(Cell::new(human_bytes(index_size as f64))) - .add_cell(Cell::new(human_bytes(offsets_size as f64))) - .add_cell(Cell::new(human_bytes(config_size as f64))) - .add_cell(Cell::new(human_bytes( - (data_size + index_size + offsets_size + config_size) as f64, - ))); + .add_cell(Cell::new(format!("{columns} x {rows}"))); + if detailed_sizes { + row.add_cell(Cell::new(human_bytes(data_size as f64))) + .add_cell(Cell::new(human_bytes(index_size as f64))) + .add_cell(Cell::new(human_bytes(offsets_size as f64))) + .add_cell(Cell::new(human_bytes(config_size as f64))); + } + row.add_cell(Cell::new(human_bytes( + (data_size + index_size + offsets_size + config_size) as f64, + ))); table.add_row(row); total_data_size += data_size; @@ -196,16 +209,16 @@ impl Command { row.add_cell(Cell::new("Total")) .add_cell(Cell::new("")) .add_cell(Cell::new("")) - .add_cell(Cell::new("")) - .add_cell(Cell::new("")) - .add_cell(Cell::new(human_bytes(total_data_size as f64))) - .add_cell(Cell::new(human_bytes(total_index_size as f64))) - .add_cell(Cell::new(human_bytes(total_offsets_size as f64))) - .add_cell(Cell::new(human_bytes(total_config_size as f64))) - .add_cell(Cell::new(human_bytes( - (total_data_size + total_index_size + total_offsets_size + total_config_size) - as f64, - ))); + .add_cell(Cell::new("")); + if detailed_sizes { + row.add_cell(Cell::new(human_bytes(total_data_size as f64))) + .add_cell(Cell::new(human_bytes(total_index_size as f64))) + .add_cell(Cell::new(human_bytes(total_offsets_size as f64))) + .add_cell(Cell::new(human_bytes(total_config_size as f64))); + } + row.add_cell(Cell::new(human_bytes( + (total_data_size + total_index_size + total_offsets_size + total_config_size) as f64, + ))); table.add_row(row); Ok(table) From 23b5e885bf02ce9259521a9149a91bf1f463b8e9 Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Tue, 30 Jan 2024 15:00:03 +0000 Subject: [PATCH 3/5] sort segments --- bin/reth/src/commands/db/stats.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/reth/src/commands/db/stats.rs b/bin/reth/src/commands/db/stats.rs index 1cc41f353169..6fbe88add6aa 100644 --- a/bin/reth/src/commands/db/stats.rs +++ b/bin/reth/src/commands/db/stats.rs @@ -3,6 +3,7 @@ use clap::Parser; use comfy_table::{Cell, Row, Table as ComfyTable}; use eyre::WrapErr; use human_bytes::human_bytes; +use itertools::Itertools; use reth_db::{database::Database, mdbx, snapshot::iter_snapshots, DatabaseEnv, Tables}; use reth_node_core::dirs::{ChainPath, DataDirPath}; use reth_primitives::snapshot::find_fixed_range; @@ -147,7 +148,7 @@ impl Command { let mut total_offsets_size = 0; let mut total_config_size = 0; - for (segment, ranges) in snapshots { + for (segment, ranges) in snapshots.into_iter().sorted_by_key(|(segment, _)| *segment) { for (block_range, tx_range) in ranges { let fixed_block_range = find_fixed_range(*block_range.start()); let jar_provider = snapshot_provider From b6767f687e3fbc6c00f34c826852c9ec2a983242 Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Wed, 31 Jan 2024 13:04:41 +0000 Subject: [PATCH 4/5] move snapshots table up --- bin/reth/src/commands/db/stats.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/reth/src/commands/db/stats.rs b/bin/reth/src/commands/db/stats.rs index 6fbe88add6aa..1078a2a77558 100644 --- a/bin/reth/src/commands/db/stats.rs +++ b/bin/reth/src/commands/db/stats.rs @@ -25,13 +25,13 @@ impl Command { data_dir: ChainPath, tool: &DbTool<'_, DatabaseEnv>, ) -> eyre::Result<()> { - let db_stats_table = self.db_stats_table(tool)?; - println!("{db_stats_table}"); + let snapshots_stats_table = self.snapshots_stats_table(data_dir, self.detailed_sizes)?; + println!("{snapshots_stats_table}"); println!("\n"); - let snapshots_stats_table = self.snapshots_stats_table(data_dir, self.detailed_sizes)?; - println!("{snapshots_stats_table}"); + let db_stats_table = self.db_stats_table(tool)?; + println!("{db_stats_table}"); Ok(()) } From fc925a043338963198c3c02c449e05f1e47429a0 Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Fri, 2 Feb 2024 12:40:35 +0000 Subject: [PATCH 5/5] display detailed sizes by default --- bin/reth/src/commands/db/stats.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/bin/reth/src/commands/db/stats.rs b/bin/reth/src/commands/db/stats.rs index 1078a2a77558..27a0acb3134e 100644 --- a/bin/reth/src/commands/db/stats.rs +++ b/bin/reth/src/commands/db/stats.rs @@ -13,9 +13,9 @@ use std::fs::File; #[derive(Parser, Debug)] /// The arguments for the `reth db stats` command pub struct Command { - /// Show detailed sizes for snapshot files. + /// Show only the total size for snapshot files. #[arg(long, default_value_t = false)] - detailed_sizes: bool, + only_total_size: bool, } impl Command { @@ -25,7 +25,7 @@ impl Command { data_dir: ChainPath, tool: &DbTool<'_, DatabaseEnv>, ) -> eyre::Result<()> { - let snapshots_stats_table = self.snapshots_stats_table(data_dir, self.detailed_sizes)?; + let snapshots_stats_table = self.snapshots_stats_table(data_dir, self.only_total_size)?; println!("{snapshots_stats_table}"); println!("\n"); @@ -119,12 +119,12 @@ impl Command { fn snapshots_stats_table( &self, data_dir: ChainPath, - detailed_sizes: bool, + only_total_size: bool, ) -> eyre::Result { let mut table = ComfyTable::new(); table.load_preset(comfy_table::presets::ASCII_MARKDOWN); - if detailed_sizes { + if !only_total_size { table.set_header([ "Segment", "Block Range", @@ -181,7 +181,7 @@ impl Command { tx_range.map_or("N/A".to_string(), |tx_range| format!("{tx_range:?}")), )) .add_cell(Cell::new(format!("{columns} x {rows}"))); - if detailed_sizes { + if !only_total_size { row.add_cell(Cell::new(human_bytes(data_size as f64))) .add_cell(Cell::new(human_bytes(index_size as f64))) .add_cell(Cell::new(human_bytes(offsets_size as f64))) @@ -211,7 +211,7 @@ impl Command { .add_cell(Cell::new("")) .add_cell(Cell::new("")) .add_cell(Cell::new("")); - if detailed_sizes { + if !only_total_size { row.add_cell(Cell::new(human_bytes(total_data_size as f64))) .add_cell(Cell::new(human_bytes(total_index_size as f64))) .add_cell(Cell::new(human_bytes(total_offsets_size as f64)))