Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat(bin): show snapshots in db stats #6285

Merged
merged 6 commits into from
Feb 2, 2024
Merged

feat(bin): show snapshots in db stats #6285

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
c4c6787
feat(bin): show snapshots in `db stats`
shekhirin Jan 30, 2024
c54aed4
--detailed-sizes
shekhirin Jan 30, 2024
23b5e88
sort segments
shekhirin Jan 30, 2024
b6767f6
move snapshots table up
shekhirin Jan 31, 2024
fc925a0
display detailed sizes by default
shekhirin Feb 2, 2024
18472f0
Merge remote-tracking branch 'origin/feat/static-files' into alexey/d…
shekhirin Feb 2, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
92 changes: 4 additions & 88 deletions bin/reth/src/commands/db/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,16 +9,10 @@ use crate::{
utils::DbTool,
};
use clap::{Parser, Subcommand};
use comfy_table::{Cell, Row, Table as ComfyTable};
use eyre::WrapErr;
use human_bytes::human_bytes;
use reth_db::{
database::Database,
mdbx,
mdbx::DatabaseArguments,
open_db, open_db_read_only,
version::{get_db_version, DatabaseVersionError, DB_VERSION},
Tables,
};
use reth_primitives::ChainSpec;
use reth_provider::ProviderFactory;
Expand All @@ -32,6 +26,7 @@ mod diff;
mod get;
mod list;
mod snapshots;
mod stats;
/// DB List TUI
mod tui;

Expand Down Expand Up @@ -72,7 +67,7 @@ pub struct Command {
/// `reth db` subcommands
pub enum Subcommands {
/// Lists all the tables, their entry count and their size
Stats,
Stats(stats::Command),
/// Lists the contents of a table
List(list::Command),
/// Create a diff between two database tables or two entire databases.
Expand Down Expand Up @@ -104,94 +99,15 @@ impl Command {

match self.command {
// TODO: We'll need to add this on the DB trait.
Subcommands::Stats { .. } => {
Subcommands::Stats(command) => {
let db = open_db_read_only(
&db_path,
DatabaseArguments::default().log_level(self.db.log_level),
)?;
let provider_factory = ProviderFactory::new(db, self.chain.clone())
.with_snapshots(data_dir.snapshots_path())?;
let tool = DbTool::new(provider_factory, self.chain.clone())?;
let mut stats_table = ComfyTable::new();
stats_table.load_preset(comfy_table::presets::ASCII_MARKDOWN);
stats_table.set_header([
"Table Name",
"# Entries",
"Branch Pages",
"Leaf Pages",
"Overflow Pages",
"Total Size",
]);

tool.provider_factory.db_ref().view(|tx| {
let mut tables =
Tables::ALL.iter().map(|table| table.name()).collect::<Vec<_>>();
tables.sort();
let mut total_size = 0;
for table in tables {
let table_db =
tx.inner.open_db(Some(table)).wrap_err("Could not open db.")?;

let stats = tx
.inner
.db_stat(&table_db)
.wrap_err(format!("Could not find table: {table}"))?;

// Defaults to 16KB right now but we should
// re-evaluate depending on the DB we end up using
// (e.g. REDB does not have these options as configurable intentionally)
let page_size = stats.page_size() as usize;
let leaf_pages = stats.leaf_pages();
let branch_pages = stats.branch_pages();
let overflow_pages = stats.overflow_pages();
let num_pages = leaf_pages + branch_pages + overflow_pages;
let table_size = page_size * num_pages;

total_size += table_size;
let mut row = Row::new();
row.add_cell(Cell::new(table))
.add_cell(Cell::new(stats.entries()))
.add_cell(Cell::new(branch_pages))
.add_cell(Cell::new(leaf_pages))
.add_cell(Cell::new(overflow_pages))
.add_cell(Cell::new(human_bytes(table_size as f64)));
stats_table.add_row(row);
}

let max_widths = stats_table.column_max_content_widths();

let mut seperator = Row::new();
for width in max_widths {
seperator.add_cell(Cell::new("-".repeat(width as usize)));
}
stats_table.add_row(seperator);

let mut row = Row::new();
row.add_cell(Cell::new("Total DB size"))
.add_cell(Cell::new(""))
.add_cell(Cell::new(""))
.add_cell(Cell::new(""))
.add_cell(Cell::new(""))
.add_cell(Cell::new(human_bytes(total_size as f64)));
stats_table.add_row(row);

let freelist = tx.inner.env().freelist()?;
let freelist_size = freelist *
tx.inner.db_stat(&mdbx::Database::freelist_db())?.page_size() as usize;

let mut row = Row::new();
row.add_cell(Cell::new("Freelist size"))
.add_cell(Cell::new(freelist))
.add_cell(Cell::new(""))
.add_cell(Cell::new(""))
.add_cell(Cell::new(""))
.add_cell(Cell::new(human_bytes(freelist_size as f64)));
stats_table.add_row(row);

Ok::<(), eyre::Report>(())
})??;

println!("{stats_table}");
command.execute(data_dir, &tool)?;
}
Subcommands::List(command) => {
let db = open_db_read_only(
Expand Down
227 changes: 227 additions & 0 deletions bin/reth/src/commands/db/stats.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,227 @@
use crate::utils::DbTool;
use clap::Parser;
use comfy_table::{Cell, Row, Table as ComfyTable};
use eyre::WrapErr;
use human_bytes::human_bytes;
use itertools::Itertools;
use reth_db::{database::Database, mdbx, snapshot::iter_snapshots, DatabaseEnv, Tables};
use reth_node_core::dirs::{ChainPath, DataDirPath};
use reth_primitives::snapshot::find_fixed_range;
use reth_provider::providers::SnapshotProvider;
use std::fs::File;

#[derive(Parser, Debug)]
/// The arguments for the `reth db stats` command
pub struct Command {
/// Show only the total size for snapshot files.
#[arg(long, default_value_t = false)]
only_total_size: bool,
}

impl Command {
/// Execute `db stats` command
pub fn execute(
self,
data_dir: ChainPath<DataDirPath>,
tool: &DbTool<DatabaseEnv>,
) -> eyre::Result<()> {
let snapshots_stats_table = self.snapshots_stats_table(data_dir, self.only_total_size)?;
println!("{snapshots_stats_table}");

println!("\n");

let db_stats_table = self.db_stats_table(tool)?;
println!("{db_stats_table}");

Ok(())
}

fn db_stats_table(&self, tool: &DbTool<DatabaseEnv>) -> eyre::Result<ComfyTable> {
let mut table = ComfyTable::new();
table.load_preset(comfy_table::presets::ASCII_MARKDOWN);
table.set_header([
"Table Name",
"# Entries",
"Branch Pages",
"Leaf Pages",
"Overflow Pages",
"Total Size",
]);

tool.provider_factory.db_ref().view(|tx| {
let mut db_tables = Tables::ALL.iter().map(|table| table.name()).collect::<Vec<_>>();
db_tables.sort();
let mut total_size = 0;
for db_table in db_tables {
let table_db = tx.inner.open_db(Some(db_table)).wrap_err("Could not open db.")?;

let stats = tx
.inner
.db_stat(&table_db)
.wrap_err(format!("Could not find table: {db_table}"))?;

// Defaults to 16KB right now but we should
// re-evaluate depending on the DB we end up using
// (e.g. REDB does not have these options as configurable intentionally)
let page_size = stats.page_size() as usize;
let leaf_pages = stats.leaf_pages();
let branch_pages = stats.branch_pages();
let overflow_pages = stats.overflow_pages();
let num_pages = leaf_pages + branch_pages + overflow_pages;
let table_size = page_size * num_pages;

total_size += table_size;
let mut row = Row::new();
row.add_cell(Cell::new(db_table))
.add_cell(Cell::new(stats.entries()))
.add_cell(Cell::new(branch_pages))
.add_cell(Cell::new(leaf_pages))
.add_cell(Cell::new(overflow_pages))
.add_cell(Cell::new(human_bytes(table_size as f64)));
table.add_row(row);
}

let max_widths = table.column_max_content_widths();
let mut seperator = Row::new();
for width in max_widths {
seperator.add_cell(Cell::new("-".repeat(width as usize)));
}
table.add_row(seperator);

let mut row = Row::new();
row.add_cell(Cell::new("Tables"))
.add_cell(Cell::new(""))
.add_cell(Cell::new(""))
.add_cell(Cell::new(""))
.add_cell(Cell::new(""))
.add_cell(Cell::new(human_bytes(total_size as f64)));
table.add_row(row);

let freelist = tx.inner.env().freelist()?;
let freelist_size =
freelist * tx.inner.db_stat(&mdbx::Database::freelist_db())?.page_size() as usize;

let mut row = Row::new();
row.add_cell(Cell::new("Freelist"))
.add_cell(Cell::new(freelist))
.add_cell(Cell::new(""))
.add_cell(Cell::new(""))
.add_cell(Cell::new(""))
.add_cell(Cell::new(human_bytes(freelist_size as f64)));
table.add_row(row);

Ok::<(), eyre::Report>(())
})??;

Ok(table)
}

fn snapshots_stats_table(
&self,
data_dir: ChainPath<DataDirPath>,
only_total_size: bool,
) -> eyre::Result<ComfyTable> {
let mut table = ComfyTable::new();
table.load_preset(comfy_table::presets::ASCII_MARKDOWN);

if !only_total_size {
table.set_header([
"Segment",
"Block Range",
"Transaction Range",
"Shape",
"Data Size",
"Index Size",
"Offsets Size",
"Config Size",
"Total Size",
]);
} else {
table.set_header(["Segment", "Block Range", "Transaction Range", "Shape", "Size"]);
}

let snapshots = iter_snapshots(data_dir.snapshots_path())?;
let snapshot_provider = SnapshotProvider::new(data_dir.snapshots_path())?;

let mut total_data_size = 0;
let mut total_index_size = 0;
let mut total_offsets_size = 0;
let mut total_config_size = 0;

for (segment, ranges) in snapshots.into_iter().sorted_by_key(|(segment, _)| *segment) {
for (block_range, tx_range) in ranges {
let fixed_block_range = find_fixed_range(*block_range.start());
let jar_provider = snapshot_provider
.get_segment_provider(segment, || Some(fixed_block_range.clone()), None)?
.expect("something went wrong");

let columns = jar_provider.columns();
let rows = jar_provider.rows();
let data_size = File::open(jar_provider.data_path())
.and_then(|file| file.metadata())
.map(|metadata| metadata.len())
.unwrap_or_default();
let index_size = File::open(jar_provider.index_path())
.and_then(|file| file.metadata())
.map(|metadata| metadata.len())
.unwrap_or_default();
let offsets_size = File::open(jar_provider.offsets_path())
.and_then(|file| file.metadata())
.map(|metadata| metadata.len())
.unwrap_or_default();
let config_size = File::open(jar_provider.config_path())
.and_then(|file| file.metadata())
.map(|metadata| metadata.len())
.unwrap_or_default();

let mut row = Row::new();
row.add_cell(Cell::new(segment))
.add_cell(Cell::new(format!("{block_range:?}")))
.add_cell(Cell::new(
tx_range.map_or("N/A".to_string(), |tx_range| format!("{tx_range:?}")),
))
.add_cell(Cell::new(format!("{columns} x {rows}")));
if !only_total_size {
row.add_cell(Cell::new(human_bytes(data_size as f64)))
.add_cell(Cell::new(human_bytes(index_size as f64)))
.add_cell(Cell::new(human_bytes(offsets_size as f64)))
.add_cell(Cell::new(human_bytes(config_size as f64)));
}
row.add_cell(Cell::new(human_bytes(
(data_size + index_size + offsets_size + config_size) as f64,
)));
table.add_row(row);

total_data_size += data_size;
total_index_size += index_size;
total_offsets_size += offsets_size;
total_config_size += config_size;
}
}

let max_widths = table.column_max_content_widths();
let mut seperator = Row::new();
for width in max_widths {
seperator.add_cell(Cell::new("-".repeat(width as usize)));
}
table.add_row(seperator);

let mut row = Row::new();
row.add_cell(Cell::new("Total"))
.add_cell(Cell::new(""))
.add_cell(Cell::new(""))
.add_cell(Cell::new(""));
if !only_total_size {
row.add_cell(Cell::new(human_bytes(total_data_size as f64)))
.add_cell(Cell::new(human_bytes(total_index_size as f64)))
.add_cell(Cell::new(human_bytes(total_offsets_size as f64)))
.add_cell(Cell::new(human_bytes(total_config_size as f64)));
}
row.add_cell(Cell::new(human_bytes(
(total_data_size + total_index_size + total_offsets_size + total_config_size) as f64,
)));
table.add_row(row);

Ok(table)
}
}
5 changes: 5 additions & 0 deletions crates/storage/nippy-jar/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -207,6 +207,11 @@ impl<H: NippyJarHeader> NippyJar<H> {
&self.user_header
}

/// Gets total columns in jar.
pub fn columns(&self) -> usize {
self.columns
}

/// Gets total rows in jar.
pub fn rows(&self) -> usize {
self.rows
Expand Down