perf: optional SMT trie feature and migrate command

crates/benches/Cargo.toml

@@ -8,6 +8,7 @@ description = "Godwoken benchmarks."
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+anyhow = "1.0.66"
 
 [dev-dependencies]
 criterion = { version = "0.3", features = ["html_reports"] }

crates/benches/benches/benchmarks/smt.rs

@@ -1,5 +1,6 @@
 use std::sync::Arc;
 
+use anyhow::Result;
 use criterion::{criterion_group, BenchmarkId, Criterion, Throughput};
 use gw_common::{
     blake2b::new_blake2b,
@@ -114,7 +115,7 @@ impl Account {
 
 struct BenchChain;
 impl ChainView for BenchChain {
-    fn get_block_hash_by_number(&self, _: u64) -> Result<Option<H256>, gw_db::error::Error> {
+    fn get_block_hash_by_number(&self, _: u64) -> Result<Option<H256>> {
         unreachable!("bench chain store")
     }
 }
@@ -372,7 +373,7 @@ impl BenchExecutionEnvironment {
                 let block_proof = smt
                     .merkle_proof(vec![block_key.into()])
                     .unwrap()
-                    .compile(vec![(block_key.into(), genesis_hash.into())])
+                    .compile(vec![block_key.into()])
                     .unwrap();
                 let block_root = *smt.root();
                 (block_root, block_proof)

crates/benches/benches/benchmarks/sudt.rs

@@ -1,3 +1,4 @@
+use anyhow::{bail, Result};
 use criterion::*;
 use gw_common::{
     builtins::ETH_REGISTRY_ACCOUNT_ID, registry_address::RegistryAddress, smt::SMT, state::State,
@@ -68,8 +69,8 @@ fn build_backend_manage(rollup_config: &RollupConfig) -> BackendManage {
 struct DummyChainStore;
 
 impl ChainView for DummyChainStore {
-    fn get_block_hash_by_number(&self, _number: u64) -> Result<Option<H256>, gw_db::error::Error> {
-        Err("dummy chain store".to_string().into())
+    fn get_block_hash_by_number(&self, _number: u64) -> Result<Option<H256>> {
+        bail!("dummy chain store")
     }
 }
 

crates/block-producer/src/block_producer.rs

@@ -12,7 +12,7 @@ use crate::{
 use anyhow::{bail, ensure, Context, Result};
 use ckb_chain_spec::consensus::MAX_BLOCK_BYTES;
 use gw_chain::chain::Chain;
-use gw_common::{h256_ext::H256Ext, H256};
+use gw_common::H256;
 use gw_config::BlockProducerConfig;
 use gw_generator::Generator;
 use gw_jsonrpc_types::test_mode::TestModePayload;
@@ -270,13 +270,12 @@ impl BlockProducer {
                 assert_eq!(local_root, &global_revert_block_root);
 
                 let keys: Vec<H256> = collected_block_hashes.into_iter().collect();
-                let leaves = keys.iter().map(|hash| (*hash, H256::one()));
-                let proof = block_smt
-                    .merkle_proof(keys.clone())?
-                    .compile(leaves.collect())?;
                 for key in keys.iter() {
                     log::info!("submit revert block {:?}", hex::encode(key.as_slice()));
                 }
+                let proof = block_smt
+                    .merkle_proof(keys.clone())?
+                    .compile(keys.clone())?;
 
                 RollupSubmitBlock::new_builder()
                     .reverted_block_hashes(keys.pack())

crates/block-producer/src/block_sync_client.rs

@@ -74,8 +74,8 @@ impl BlockSyncClient {
         loop {
             if let Some(ref mut s) = p2p_stream {
                 if let Err(err) = run_with_p2p_stream(&mut self, s).await {
-                    if err.is::<gw_db::error::Error>() {
-                        // Cannot recover from db error.
+                    if err.is::<gw_db::transaction::CommitError>() {
+                        // Cannot recover from db commit error.
                         log::error!("db error, exiting: {:#}", err);
                         return;
                     }
@@ -93,7 +93,7 @@ impl BlockSyncClient {
                     continue;
                 }
                 if let Err(err) = run_once_without_p2p_stream(&mut self).await {
-                    if err.is::<gw_db::error::Error>() {
+                    if err.is::<gw_db::transaction::CommitError>() {
                         // Cannot recover from db error.
                         log::error!("db error, exiting: {:#}", err);
                         return;

crates/block-producer/src/produce_block.rs

@@ -137,7 +137,7 @@ pub fn produce_block(
     let block_proof = block_smt
         .merkle_proof(vec![H256::from_u64(number)])
         .map_err(|err| anyhow!("merkle proof error: {:?}", err))?
-        .compile(vec![(H256::from_u64(number), H256::zero())])?;
+        .compile(vec![H256::from_u64(number)])?;
     let packed_kv_state = kv_state.pack();
     let withdrawal_requests = withdrawals.iter().map(|w| w.request());
     let block = L2Block::new_builder()
@@ -228,10 +228,11 @@ pub fn generate_produce_block_param(
     } else {
         let state_smt = db.state_smt()?;
 
+        let keys: Vec<H256> = kv_state.iter().map(|(k, _v)| *k).collect();
         state_smt
-            .merkle_proof(kv_state.iter().map(|(k, _v)| *k).collect())
+            .merkle_proof(keys.clone())
             .map_err(|err| anyhow!("merkle proof error: {:?}", err))?
-            .compile(kv_state.clone())?
+            .compile(keys)?
             .0
     };
 

crates/block-producer/src/runner.rs

@@ -15,7 +15,7 @@ use gw_chain::chain::Chain;
 use gw_challenge::offchain::{OffChainMockContext, OffChainMockContextBuildArgs};
 use gw_common::{blake2b::new_blake2b, registry_address::RegistryAddress, H256};
 use gw_config::{BlockProducerConfig, Config, NodeMode};
-use gw_db::migrate::open_or_create_db;
+use gw_db::migrate::{init_migration_factory, open_or_create_db};
 use gw_dynamic_config::manager::DynamicConfigManager;
 use gw_generator::{
     account_lock_manage::{secp256k1::Secp256k1Eth, AccountLockManage},
@@ -329,7 +329,7 @@ impl BaseInitComponents {
             log::warn!("config.store.path is blank, using temporary store");
             Store::open_tmp().with_context(|| "init store")?
         } else {
-            Store::new(open_or_create_db(&config.store)?)
+            Store::new(open_or_create_db(&config.store, init_migration_factory())?)
         };
         let elapsed_ms = timer.elapsed().as_millis();
         log::debug!("Open rocksdb costs: {}ms.", elapsed_ms);

crates/block-producer/src/sync_l1.rs

@@ -35,7 +35,7 @@ pub async fn sync_l1(ctx: &(dyn SyncL1Context + Sync + Send)) -> Result<()> {
     let mut backoff = ExponentialBackoff::new(Duration::from_secs(1));
     loop {
         if let Err(err) = sync_l1_impl(ctx).await {
-            if err.is::<gw_db::error::Error>() {
+            if err.is::<gw_db::transaction::CommitError>() {
                 // We cannot recover from db commit error because Chain
                 // local_state would be wrong. Chain always assumes that commit
                 // will success.

crates/block-producer/src/test_mode_control.rs

@@ -207,7 +207,7 @@ impl TestModeControl {
             let bad_block_proof = db
                 .block_smt()?
                 .merkle_proof(vec![H256::from_u64(block_number)])?
-                .compile(vec![(H256::from_u64(block_number), H256::zero())])?;
+                .compile(vec![H256::from_u64(block_number)])?;
 
             // Generate new block smt for global state
             let bad_block_smt = {
@@ -262,10 +262,7 @@ impl TestModeControl {
         let block_proof = db
             .block_smt()?
             .merkle_proof(vec![raw_l2block.smt_key().into()])?
-            .compile(vec![(
-                raw_l2block.smt_key().into(),
-                raw_l2block.hash().into(),
-            )])?;
+            .compile(vec![raw_l2block.smt_key().into()])?;
 
         let target_type = match target_type {
             ChallengeType::TxExecution => ChallengeTargetType::TxExecution,

crates/challenge/src/context.rs

@@ -44,7 +44,7 @@ pub fn build_challenge_context(
     let block_smt = db.block_smt()?;
     let block_proof = block_smt
         .merkle_proof(vec![block.smt_key().into()])?
-        .compile(vec![(block.smt_key().into(), block.hash().into())])?;
+        .compile(vec![block.smt_key().into()])?;
 
     let witness = ChallengeWitness::new_builder()
         .raw_l2block(block.raw())
@@ -91,16 +91,14 @@ pub fn build_revert_context(
     let (post_reverted_block_root, reverted_block_proof) = {
         let mut smt = db.reverted_block_smt()?;
         let to_key = |b: &RawL2Block| H256::from(b.hash());
-        let to_leave = |b: &RawL2Block| (to_key(b), H256::one());
 
         let keys: Vec<H256> = reverted_raw_blocks.iter().map(to_key).collect();
         for key in keys.iter() {
             smt.update(key.to_owned(), H256::one())?;
         }
 
         let root = smt.root().to_owned();
-        let leaves = reverted_raw_blocks.iter().map(to_leave).collect();
-        let proof = smt.merkle_proof(keys)?.compile(leaves)?;
+        let proof = smt.merkle_proof(keys.clone())?.compile(keys.clone())?;
 
         (root, proof)
     };
@@ -528,12 +526,9 @@ fn build_block_proof(
 
     let block_proof = {
         let smt = db.block_smt()?;
-        let to_leave = |b: &RawL2Block| (b.smt_key().into(), b.hash().into());
 
-        let smt_keys = raw_blocks.iter().map(|rb| rb.smt_key().into());
-        let leaves = raw_blocks.iter().map(to_leave);
-        smt.merkle_proof(smt_keys.collect())?
-            .compile(leaves.collect())?
+        let smt_keys: Vec<H256> = raw_blocks.iter().map(|rb| rb.smt_key().into()).collect();
+        smt.merkle_proof(smt_keys.clone())?.compile(smt_keys)?
     };
 
     Ok((block_hashes, block_proof))

crates/challenge/src/offchain.rs

@@ -28,7 +28,6 @@ use std::{
 pub mod mock_block;
 pub mod mock_tx;
 pub mod verify_tx;
-pub use mock_block::RollBackSavePointError;
 pub use mock_tx::mock_cancel_challenge_tx;
 pub use verify_tx::dump_tx;
 

crates/challenge/src/offchain/mock_block.rs

@@ -29,10 +29,6 @@ use gw_utils::RollupContext;
 
 type MemTree = MemStateDB;
 
-#[derive(thiserror::Error, Debug)]
-#[error("{:?}", {0})]
-pub struct RollBackSavePointError(gw_db::error::Error);
-
 pub struct MockBlockParam {
     rollup_context: RollupContext,
     finality_blocks: u64,
@@ -307,7 +303,7 @@ impl MockBlockParam {
         let block_proof = block_smt
             .merkle_proof(vec![H256::from_u64(self.number)])
             .map_err(|err| anyhow!("merkle proof error: {:?}", err))?
-            .compile(vec![(H256::from_u64(self.number), H256::zero())])?;
+            .compile(vec![H256::from_u64(self.number)])?;
         let post_block = {
             let post_block_root = block_proof.compute_root::<Blake2bHasher>(vec![(
                 raw_block.smt_key().into(),
@@ -411,10 +407,11 @@ impl MockBlockParam {
             Unpack::<u32>::unpack(&tx.raw().nonce())
         );
 
-        let touched_keys = kv_state.iter().map(|(key, _)| key.to_owned()).collect();
+        let touched_keys: Vec<H256> = kv_state.iter().map(|(key, _)| key.to_owned()).collect();
         let kv_state_proof = {
             let smt = mem_tree.inner_smt_tree();
-            smt.merkle_proof(touched_keys)?.compile(kv_state.clone())?
+            smt.merkle_proof(touched_keys.clone())?
+                .compile(touched_keys)?
         };
 
         let account_count = mem_tree.get_account_count()?;

crates/common/Cargo.toml

@@ -8,7 +8,7 @@ edition = "2018"
 
 [dependencies]
 cfg-if = "0.1"
-sparse-merkle-tree = { version = "=0.5.2-rc1", default-features = false }
+sparse-merkle-tree = { version = "0.6.1", default-features = false }
 merkle-cbt = { version = "0.3.0", default-features = false }
 gw-hash = { path = "../hash" }
 thiserror = { version = "1.0", optional = true }
@@ -17,3 +17,4 @@ gw-types = { path = "../types", default-features = false }
 [features]
 default = ["std"]
 std = ["sparse-merkle-tree/std", "thiserror", "gw-types/std"]
+smt-trie = ["sparse-merkle-tree/trie"]

crates/common/src/smt.rs

@@ -2,9 +2,7 @@ use gw_hash::blake2b::{new_blake2b, Blake2b};
 use sparse_merkle_tree::{traits::Hasher, SparseMerkleTree};
 
 // re-exports
-pub use sparse_merkle_tree::{
-    default_store, error::Error, traits::Store, CompiledMerkleProof, MerkleProof, H256,
-};
+pub use sparse_merkle_tree::{default_store, error::Error, CompiledMerkleProof, MerkleProof, H256};
 
 pub type SMT<S> = SparseMerkleTree<Blake2bHasher, H256, S>;
 

crates/db/Cargo.toml

@@ -10,10 +10,13 @@ edition = "2018"
 rocksdb = { package = "ckb-rocksdb", version = "0.18", default-features = false, features = ["snappy", "march-native"] }
 gw-config = { path = "../config" }
 libc = "0.2"
-thiserror = "1.0"
 tempfile = "3.0"
 log = "0.4.14"
 serde = { version = "1.0", features = ["derive"] }
+anyhow = "1.0.66"
 
 [dev-dependencies]
 tempfile = "3.0"
+
+[features]
+smt-trie = []