diff --git a/Cargo.lock b/Cargo.lock index 03e002d39e..0e3aa20274 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -456,6 +456,12 @@ dependencies = [ "autocfg 1.1.0", ] +[[package]] +name = "atomic-counter" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62f447d68cfa5a9ab0c1c862a703da2a65b5ed1b7ce1153c9eb0169506d56019" + [[package]] name = "atomic-waker" version = "1.0.0" @@ -1839,11 +1845,14 @@ name = "db-exporter" version = "1.12.3" dependencies = [ "anyhow", + "atomic-counter", "bcs-ext", "clap 3.2.15", "csv", "hex", "indicatif", + "move-binary-format", + "move-bytecode-verifier", "pprof", "serde 1.0.140", "serde_json", @@ -1863,6 +1872,7 @@ dependencies = [ "starcoin-transaction-builder", "starcoin-types", "starcoin-vm-types", + "tokio 1.20.1", ] [[package]] diff --git a/cmd/db-exporter/Cargo.toml b/cmd/db-exporter/Cargo.toml index 3e21fa82b6..de9f5a20d8 100644 --- a/cmd/db-exporter/Cargo.toml +++ b/cmd/db-exporter/Cargo.toml @@ -10,6 +10,7 @@ version = "1.12.3" [dependencies] anyhow = "~1" +atomic-counter = "1.0.1" bcs-ext = {package = "bcs-ext", path = "../../commons/bcs_ext"} clap = {version = "3", features = ["derive"]} csv = "~1" @@ -18,6 +19,11 @@ indicatif = "0.16.2" logger = {path = "../../commons/logger", package = "starcoin-logger"} serde = "~1" serde_json = {version = "~1", features = ["arbitrary_precision"]} +tokio = {version = "^1", features = ["full"]} + +move-binary-format = {git = "https://github.com/starcoinorg/move", rev = "9d1e76b770d630ad646a540b325f88ef986af3a3"} +move-bytecode-verifier = {git = "https://github.com/starcoinorg/move", rev = "9d1e76b770d630ad646a540b325f88ef986af3a3"} + starcoin-account-api = {path = "../../account/api"} starcoin-accumulator = {path = "../../commons/accumulator", package = "starcoin-accumulator"} starcoin-chain = {path = "../../chain"} diff --git a/cmd/db-exporter/src/lib.rs b/cmd/db-exporter/src/lib.rs new file mode 100644 index 0000000000..e08288615e --- /dev/null +++ b/cmd/db-exporter/src/lib.rs @@ -0,0 +1,133 @@ +// Copyright (c) The Starcoin Core Contributors +// SPDX-License-Identifier: Apache-2.0 + +use anyhow::bail; +use atomic_counter::AtomicCounter; +use indicatif::{ProgressBar, ProgressStyle}; +use move_binary_format::errors::Location; +use starcoin_crypto::HashValue; +use starcoin_types::block::Block; +use starcoin_types::transaction::TransactionPayload; +use starcoin_vm_types::errors::VMError; +use starcoin_vm_types::file_format::CompiledModule; +use std::fs::File; +use std::io::{BufRead, BufReader}; +use std::path::PathBuf; +use std::sync::Arc; +use std::time::SystemTime; +use tokio::task; + +pub fn verify_modules_via_export_file(input_path: PathBuf) -> anyhow::Result<()> { + let start_time = SystemTime::now(); + let file_name = input_path.display().to_string(); + let reader = BufReader::new(File::open(input_path)?); + let mut blocks = vec![]; + for record in reader.lines() { + let record = record?; + let block: Block = serde_json::from_str(record.as_str())?; + + blocks.push(block); + } + if blocks.is_empty() { + println!("file {} has apply", file_name); + return Ok(()); + } + + if let Some(last_block) = blocks.last() { + let start = blocks.get(0).unwrap().header().number(); + let end = last_block.header().number(); + println!("verify [{},{}] block number", start, end); + } + let use_time = SystemTime::now().duration_since(start_time)?; + println!("load blocks from file use time: {:?}", use_time.as_millis()); + let start_time = SystemTime::now(); + let bar = ProgressBar::new(blocks.len() as u64); + bar.set_style( + ProgressStyle::default_bar() + .template("[{elapsed_precise}] {bar:100.cyan/blue} {percent}% {msg}"), + ); + + let success_counter = Arc::new(atomic_counter::RelaxedCounter::new(0)); + let error_counter = Arc::new(atomic_counter::RelaxedCounter::new(0)); + for block in blocks { + let total_modules = success_counter.get() + error_counter.get(); + let block_number = block.header().number(); + let success_counter = success_counter.clone(); + let error_counter = error_counter.clone(); + + task::spawn(async move { + let (success_count, errors) = verify_block_modules(block); + if !errors.is_empty() { + println!( + "verify block modules {} error modules: {:?}", + block_number, errors + ); + } + success_counter.add(success_count); + error_counter.add(errors.len()); + }); + bar.set_message(format!( + "verify block {} , total_modules: {}", + block_number, total_modules + )); + bar.inc(1); + } + bar.finish(); + let use_time = SystemTime::now().duration_since(start_time)?; + println!("verify block modules use time: {:?}, success modules: {}, error modules: {}, total modules: {}", use_time.as_secs(), success_counter.get(), error_counter.get(), success_counter.get() + error_counter.get()); + if error_counter.get() > 0 { + bail!("verify block modules error"); + } + Ok(()) +} + +#[derive(Debug)] +pub struct VerifyModuleError { + pub block_number: u64, + pub transaction_hash: HashValue, + pub error: VMError, +} + +fn verify_block_modules(block: Block) -> (usize, Vec) { + let mut errors = vec![]; + let mut success_modules = 0; + + for txn in block.transactions() { + match txn.payload() { + TransactionPayload::Package(package) => { + for module in package.modules() { + match CompiledModule::deserialize(module.code()) { + Ok(compiled_module) => { + match move_bytecode_verifier::verify_module(&compiled_module) { + Err(e) => { + errors.push(VerifyModuleError { + block_number: block.header().number(), + transaction_hash: txn.id(), + error: e, + }); + } + Ok(_) => { + success_modules += 1; + } + } + } + Err(e) => { + errors.push(VerifyModuleError { + block_number: block.header().number(), + transaction_hash: txn.id(), + error: e.finish(Location::Undefined), + }); + } + } + } + } + TransactionPayload::Script(_) => { + //TODO + } + TransactionPayload::ScriptFunction(_) => { + //continue + } + } + } + (success_modules, errors) +} diff --git a/cmd/db-exporter/src/main.rs b/cmd/db-exporter/src/main.rs index aeee6d92b7..65ad343c09 100644 --- a/cmd/db-exporter/src/main.rs +++ b/cmd/db-exporter/src/main.rs @@ -7,6 +7,7 @@ use bcs_ext::Sample; use clap::IntoApp; use clap::Parser; use csv::Writer; +use db_exporter::verify_modules_via_export_file; use indicatif::{MultiProgress, ProgressBar, ProgressStyle}; use serde::ser::SerializeMap; use serde::{Serialize, Serializer}; @@ -231,6 +232,7 @@ enum Cmd { ExportSnapshot(ExportSnapshotOptions), ApplySnapshot(ApplySnapshotOptions), ExportResource(ExportResourceOptions), + VerifyModules(VerifyModulesOptions), } #[derive(Debug, Clone, Parser)] @@ -412,7 +414,19 @@ pub struct ExportResourceOptions { pub fields: Vec, } -fn main() -> anyhow::Result<()> { +#[derive(Debug, Parser)] +#[clap( + name = "verify-modules", + about = "fast verify all modules, do not execute the transactions" +)] +pub struct VerifyModulesOptions { + #[clap(long, short = 'i', parse(from_os_str))] + /// input file, like accounts.csv + pub input_path: PathBuf, +} + +#[tokio::main(flavor = "multi_thread")] +async fn main() -> anyhow::Result<()> { let opt = Opt::parse(); let cmd = match opt.cmd { Some(cmd) => cmd, @@ -422,150 +436,147 @@ fn main() -> anyhow::Result<()> { } }; - if let Cmd::Exporter(option) = cmd { - let output = option.output.as_deref(); - let mut writer_builder = csv::WriterBuilder::new(); - let writer_builder = writer_builder.delimiter(b'\t').double_quote(false); - let result = match output { - Some(output) => { - let writer = writer_builder.from_path(output)?; - export( - // option.db_path.display().to_string().as_str(), - option.db_path.to_str().unwrap(), - writer, - option.schema, - ) - } - None => { - let writer = writer_builder.from_writer(std::io::stdout()); - export( - option.db_path.display().to_string().as_str(), - writer, - option.schema, - ) - } - }; - if let Err(err) = result { - let broken_pipe_err = err.downcast_ref::().and_then(|err| { - if let csv::ErrorKind::Io(io_err) = err.kind() { - if io_err.kind() == std::io::ErrorKind::BrokenPipe { - Some(io_err) + match cmd { + Cmd::Exporter(option) => { + let output = option.output.as_deref(); + let mut writer_builder = csv::WriterBuilder::new(); + let writer_builder = writer_builder.delimiter(b'\t').double_quote(false); + let result = match output { + Some(output) => { + let writer = writer_builder.from_path(output)?; + export( + // option.db_path.display().to_string().as_str(), + option.db_path.to_str().unwrap(), + writer, + option.schema, + ) + } + None => { + let writer = writer_builder.from_writer(std::io::stdout()); + export( + option.db_path.display().to_string().as_str(), + writer, + option.schema, + ) + } + }; + if let Err(err) = result { + let broken_pipe_err = err.downcast_ref::().and_then(|err| { + if let csv::ErrorKind::Io(io_err) = err.kind() { + if io_err.kind() == std::io::ErrorKind::BrokenPipe { + Some(io_err) + } else { + None + } } else { None } + }); + //ignore BrokenPipe + return if let Some(_broken_pipe_err) = broken_pipe_err { + Ok(()) } else { - None - } - }); - //ignore BrokenPipe - return if let Some(_broken_pipe_err) = broken_pipe_err { - Ok(()) - } else { - Err(err) - }; + Err(err) + }; + } + return result; } - return result; - } - - if let Cmd::Checkkey(option) = cmd { - let db = DBStorage::open_with_cfs( - option.db_path.display().to_string().as_str(), - StorageVersion::current_version() - .get_column_family_names() - .to_vec(), - true, - Default::default(), - None, - )?; + Cmd::Checkkey(option) => { + let db = DBStorage::open_with_cfs( + option.db_path.display().to_string().as_str(), + StorageVersion::current_version() + .get_column_family_names() + .to_vec(), + true, + Default::default(), + None, + )?; - let result = db.get(option.cf_name.as_str(), option.block_hash.to_vec())?; - if result.is_some() { - println!("{} block_hash {} exist", option.cf_name, option.block_hash); - } else { - println!( - "{} block_hash {} not exist", - option.cf_name, option.block_hash + let result = db.get(option.cf_name.as_str(), option.block_hash.to_vec())?; + if result.is_some() { + println!("{} block_hash {} exist", option.cf_name, option.block_hash); + } else { + println!( + "{} block_hash {} not exist", + option.cf_name, option.block_hash + ); + } + return Ok(()); + } + Cmd::ExportBlockRange(option) => { + let result = export_block_range( + option.db_path, + option.output, + option.net, + option.start, + option.end, ); + return result; } - return Ok(()); - } - - if let Cmd::ExportBlockRange(option) = cmd { - let result = export_block_range( - option.db_path, - option.output, - option.net, - option.start, - option.end, - ); - return result; - } - - if let Cmd::ApplyBlock(option) = cmd { - #[cfg(target_os = "linux")] - let guard = pprof::ProfilerGuard::new(100).unwrap(); - let verifier = option.verifier.unwrap_or(Verifier::Basic); - let result = apply_block(option.to_path, option.input_path, option.net, verifier); - #[cfg(target_os = "linux")] - if let Ok(report) = guard.report().build() { - let file = File::create("/tmp/flamegraph.svg").unwrap(); - report.flamegraph(file).unwrap(); + Cmd::ApplyBlock(option) => { + #[cfg(target_os = "linux")] + let guard = pprof::ProfilerGuard::new(100).unwrap(); + let verifier = option.verifier.unwrap_or(Verifier::Basic); + let result = apply_block(option.to_path, option.input_path, option.net, verifier); + #[cfg(target_os = "linux")] + if let Ok(report) = guard.report().build() { + let file = File::create("/tmp/flamegraph.svg").unwrap(); + report.flamegraph(file).unwrap(); + } + return result; } - return result; - } - - if let Cmd::StartupInfoBack(option) = cmd { - let result = startup_info_back(option.to_path, option.back_size, option.net); - return result; - } - - if let Cmd::GenBlockTransactions(option) = cmd { - let result = gen_block_transactions( - option.to_path, - option.block_num, - option.trans_num, - option.txn_type, - ); - return result; - } - - if let Cmd::ExportSnapshot(option) = cmd { - let result = export_snapshot( - option.db_path, - option.output, - option.net, - option.increment, - option.special_block_num, - ); - return result; - } - - if let Cmd::ApplySnapshot(option) = cmd { - let result = apply_snapshot(option.to_path, option.input_path, option.net); - return result; - } - - if let Cmd::ExportResource(option) = cmd { - #[cfg(target_os = "linux")] - let guard = pprof::ProfilerGuard::new(100).unwrap(); - let output = option.output.as_path(); - let block_hash = option.block_hash; - let resource = option.resource_type.clone(); - // let result = apply_block(option.to_path, option.input_path, option.net, verifier); - export_resource( - option.db_path.display().to_string().as_str(), - output, - block_hash, - resource, - option.fields.as_slice(), - )?; - #[cfg(target_os = "linux")] - if let Ok(report) = guard.report().build() { - let file = File::create("/tmp/flamegraph-db-export-resource-freq-100.svg").unwrap(); - report.flamegraph(file).unwrap(); + Cmd::StartupInfoBack(option) => { + let result = startup_info_back(option.to_path, option.back_size, option.net); + return result; + } + Cmd::GenBlockTransactions(option) => { + let result = gen_block_transactions( + option.to_path, + option.block_num, + option.trans_num, + option.txn_type, + ); + return result; + } + Cmd::ExportSnapshot(option) => { + let result = export_snapshot( + option.db_path, + option.output, + option.net, + option.increment, + option.special_block_num, + ); + return result; + } + Cmd::ApplySnapshot(option) => { + let result = apply_snapshot(option.to_path, option.input_path, option.net); + return result; + } + Cmd::ExportResource(option) => { + #[cfg(target_os = "linux")] + let guard = pprof::ProfilerGuard::new(100).unwrap(); + let output = option.output.as_path(); + let block_hash = option.block_hash; + let resource = option.resource_type.clone(); + // let result = apply_block(option.to_path, option.input_path, option.net, verifier); + export_resource( + option.db_path.display().to_string().as_str(), + output, + block_hash, + resource, + option.fields.as_slice(), + )?; + #[cfg(target_os = "linux")] + if let Ok(report) = guard.report().build() { + let file = File::create("/tmp/flamegraph-db-export-resource-freq-100.svg").unwrap(); + report.flamegraph(file).unwrap(); + } + } + Cmd::VerifyModules(option) => { + let result = verify_modules_via_export_file(option.input_path); + return result; } } - Ok(()) } diff --git a/scripts/verify_modules.sh b/scripts/verify_modules.sh new file mode 100755 index 0000000000..ead066b065 --- /dev/null +++ b/scripts/verify_modules.sh @@ -0,0 +1,76 @@ +#!/bin/bash +function download() { + net=$1 + name=$2 + if [ -f "$name" ]; then + echo -e "$net $name exists. use it" + return 0 + fi + compress_name=$name.tar.gz + url=https://s3.ap-northeast-1.amazonaws.com/main.starcoin.org/$net/$compress_name + for ((i = 0; i < 3; i++)); do + rm -f "$compress_name" + wget $url + case_status=$? + if [ $case_status -eq 0 ]; then + echo -e "download $net $name succ" + break + fi + done + case_status=$? + if [ $case_status -ne 0 ]; then + return $case_status + fi + tar xzvf "$compress_name" + case_status=$? + if [ $case_status -ne 0 ]; then + echo -e "tar $net $compress_name fail" + return $case_status + fi + return 0 +} + +function usage() { + echo -e "usage: verify_modules.sh net" + echo -e "net is main, barnard, proxima, halley" +} + +function verify_modules() { + net=$1 + block_list=$2 + + download "$net" "$block_list" + + while read line; do + name=$line + download "$net" "$name" + case_status=$? + if [ $case_status -ne 0 ]; then + echo -e "download $net $name fail" + exit $case_status + fi + + ./starcoin_db_exporter verify-modules -i "$name" + case_status=$? + if [ $case_status -ne 0 ]; then + echo -e "verify-modules $net $name fail" + exit $case_status + fi + done <"$block_list" + echo -e "$net verify-modules succ" +} + +if [ $# != 1 ]; then + usage + exit 1 +fi +net=$1 +case $net in +"main" | "barnard" | "proxima" |"halley") + verify_modules "$net" block_list.csv + ;; +*) + echo "$net not supported" + usage + ;; +esac