Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(statedb/export): add time metrics to export process #2430

Merged
merged 10 commits into from
Aug 14, 2024
17 changes: 12 additions & 5 deletions crates/rooch/src/commands/statedb/commands/export.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
use std::fmt::Display;
use std::path::PathBuf;
use std::str::FromStr;
use std::time::Instant;

use anyhow::Result;
use clap::Parser;
Expand Down Expand Up @@ -305,6 +306,7 @@ impl ExportCommand {

Self::internal_export_indexer(moveos_store, root_state_root, writer, object_ids)?;
writer.flush()?;

Ok(())
}

Expand Down Expand Up @@ -346,6 +348,8 @@ impl ExportCommand {
object_name: Option<String>, // human-readable object name for debug
writer: &mut Writer<W>,
) -> Result<()> {
let start_time = Instant::now();

let starting_key = None;
let mut count: u64 = 0;

Expand All @@ -356,24 +360,27 @@ impl ExportCommand {
.get_state_store()
.iter(obj_state_root, starting_key)?;

let mut loop_time = Instant::now();
for item in iter {
let (k, v) = item?;
writer.write_record([k.to_string().as_str(), v.to_string().as_str()])?;
count += 1;
if count % 1_000_000 == 0 {
println!(
"exporting top_level_fields of object_id: {:?}({}), exported count: {}",
object_id, object_name, count
)
"exporting top_level_fields of object_id: {:?}({}), exported count: {}. cost: {:?}",
object_id, object_name, count, loop_time.elapsed()
);
loop_time = Instant::now();
}
}

println!(
"Done. export_top_level_fields of object_id: {:?}({}), state_root: {:?}, exported count: {}",
"Done. export_top_level_fields of object_id: {:?}({}), state_root: {:?}, exported count: {}. cost: {:?}",
object_id,
object_name,
obj_state_root,
count
count,
start_time.elapsed()
);
Ok(())
}
Expand Down
1 change: 0 additions & 1 deletion crates/rooch/src/commands/statedb/commands/genesis.rs
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,6 @@ impl GenesisCommand {
let apply_inscription_updates_thread = thread::spawn(move || {
apply_inscription_updates(ord_rx, moveos_store_clone, startup_update_set_clone);
});

// import utxo
let utxo_input_path = Arc::new(self.utxo_source.clone());
let utxo_input_path_clone1 = Arc::clone(&utxo_input_path);
Expand Down
36 changes: 36 additions & 0 deletions crates/rooch/src/commands/statedb/commands/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,10 @@ use std::path::PathBuf;
use std::str::FromStr;
use std::time::Instant;

use anyhow::Result;
use bitcoin::hashes::Hash;
use bitcoin::OutPoint;
use csv::Writer;
use xorf::{BinaryFuse8, Filter};
use xxhash_rust::xxh3::xxh3_64;

Expand All @@ -20,6 +22,7 @@ use moveos_types::move_std::option::MoveOption;
use moveos_types::move_std::string::MoveString;
use moveos_types::moveos_std::object::{ObjectID, ObjectMeta};
use moveos_types::moveos_std::simple_multimap::{Element, SimpleMultiMap};
use moveos_types::state::{FieldKey, ObjectState};
use rooch_common::fs::file_cache::FileCacheManager;
use rooch_config::RoochOpt;
use rooch_db::RoochDB;
Expand Down Expand Up @@ -387,6 +390,39 @@ where
None
}

#[allow(dead_code)]
struct ExportWriter {
writer: Option<Writer<File>>,
}

#[allow(dead_code)]
impl ExportWriter {
fn new(output: Option<PathBuf>) -> Self {
let writer = match output {
Some(output) => {
let file_name = output.display().to_string();
let mut writer_builder = csv::WriterBuilder::new();
let writer_builder = writer_builder
.delimiter(b',')
.double_quote(false)
.buffer_capacity(1 << 23);
let writer = writer_builder.from_path(file_name).unwrap();
Some(writer)
}
None => None,
};
ExportWriter { writer }
}
fn write(&mut self, k: &FieldKey, v: &ObjectState) -> anyhow::Result<()> {
if let Some(writer) = &mut self.writer {
writer.write_record([k.to_string().as_str(), v.to_string().as_str()])?;
Ok(())
} else {
Ok(())
}
}
}

#[cfg(test)]
mod tests {
use std::collections::HashSet;
Expand Down
Loading