Added boilerplate for Bonsai storage migration

CHANGELOG.md

@@ -2,6 +2,7 @@ git # Deoxys Changelog
 
 ## Next release
 
+- feat(storage): started migrating storage to the bonsai-lib
 - fix: l1HandlerTx computed for commit
 - refactor: optimise get_events RPC
 - fix(root): fixed state commitments broken due to genesis loader

Cargo.toml

@@ -5,7 +5,6 @@ members = [
   "crates/client/genesis-data-provider",
   "crates/client/mapping-sync",
   "crates/client/rpc",
-  "crates/client/storage",
   "crates/node",
   "crates/pallets/starknet",
   "crates/pallets/starknet/runtime_api/",
@@ -32,7 +31,6 @@ default-members = [
   "crates/client/genesis-data-provider",
   "crates/client/mapping-sync",
   "crates/client/rpc",
-  "crates/client/storage",
   "crates/node",
   "crates/pallets/starknet",
   "crates/pallets/starknet/runtime_api/",
@@ -56,14 +54,14 @@ panic = "abort"
 
 [profile.release]
 panic = "unwind"
-debug = true
 
 [profile.production]
 codegen-units = 1    # Setting this to 1 allows for more optimizations at the cost of slower compile time
 inherits = "release"
-lto = true           # Enables Link Time Optimization, enabling more aggressive optimizations across the entire codebase
+lto = "fat"          # Enables Link Time Optimization, enabling more aggressive optimizations across the entire codebase
 opt-level = 3        # Optimize for speed regardless of binary size or compile time
 rpath = false        # Disables adding rpath to the binary
+strip = "symbols"    # Removes debug info and symbold from final binary
 
 [workspace.package]
 authors = ["Antiyro <@antiyro>"]
@@ -206,7 +204,7 @@ pallet-grandpa = { default-features = true, git = "https://github.com/massalabs/
 pallet-timestamp = { default-features = true, git = "https://github.com/massalabs/polkadot-sdk", branch = "release-polkadot-v1.3.0-std" }
 
 # Bonsai trie dependencies
-bonsai-trie = { default-features = false, git = "https://github.com/keep-starknet-strange/bonsai-trie.git", branch = "oss", features = [
+bonsai-trie = { default-features = false, git = "https://github.com/trantorian1/bonsai-trie.git", branch = "snapshot", features = [
   "std",
 ] }
 
@@ -240,7 +238,6 @@ mc-db = { path = "crates/client/db" }
 mc-genesis-data-provider = { path = "crates/client/genesis-data-provider" }
 mc-mapping-sync = { path = "crates/client/mapping-sync" }
 mc-rpc = { path = "crates/client/rpc" }
-mc-storage = { path = "crates/client/storage" }
 mc-sync = { path = "crates/client/sync" }
 
 # Madara runtime
@@ -325,6 +322,7 @@ tokio = "1.34.0"
 url = "2.4.1"
 rayon = "1.10.0"
 arc-swap = "1.7.1"
+crossbeam-skiplist = "0.1"
 
 [patch."https://github.com/w3f/ring-vrf"]
 bandersnatch_vrfs = { git = "https://github.com/w3f/ring-vrf?rev=3ddc20", version = "0.0.4", rev = "3ddc20" }

crates/client/db/Cargo.toml

@@ -26,6 +26,9 @@ sp-database = { workspace = true, default-features = true }
 sp-runtime = { workspace = true, default-features = true }
 
 # Madara crates
+mp-block = { workspace = true }
+mp-contract = { workspace = true }
+mp-felt = { workspace = true }
 mp-hashers = { workspace = true }
 mp-types = { workspace = true }
 
@@ -47,10 +50,13 @@ starknet_api = { workspace = true, default-features = true }
 anyhow.workspace = true
 arc-swap = { workspace = true }
 bitvec = { workspace = true }
+crossbeam-skiplist = { workspace = true }
 ethers = { workspace = true }
 log = { workspace = true, default-features = true }
+rayon = { workspace = true }
 rocksdb = { version = "0.21", features = [
   # "multi-threaded-cf",
 ] }
 thiserror = { workspace = true }
+tokio = { workspace = true }
 uuid = "1.4.1"

crates/client/db/src/bonsai_db.rs

@@ -15,15 +15,15 @@ pub type RocksDBTransaction = WriteBatchWithTransaction<true>;
 pub(crate) struct DatabaseKeyMapping {
     pub(crate) flat: Column,
     pub(crate) trie: Column,
-    pub(crate) trie_log: Column,
+    pub(crate) log: Column,
 }
 
 impl DatabaseKeyMapping {
     pub(crate) fn map(&self, key: &DatabaseKey) -> Column {
         match key {
             DatabaseKey::Trie(_) => self.trie,
             DatabaseKey::Flat(_) => self.flat,
-            DatabaseKey::TrieLog(_) => self.trie_log,
+            DatabaseKey::TrieLog(_) => self.log,
         }
     }
 }

crates/client/db/src/lib.rs

@@ -27,17 +27,21 @@ use sc_client_db::DatabaseSource;
 
 mod error;
 mod mapping_db;
-use rocksdb::{BoundColumnFamily, ColumnFamilyDescriptor, MultiThreaded, OptimisticTransactionDB, Options};
+use rocksdb::{
+    BoundColumnFamily, ColumnFamilyDescriptor, DBCompressionType, MultiThreaded, OptimisticTransactionDB, Options,
+};
 mod da_db;
 use starknet_api::hash::StarkHash;
 use starknet_types_core::hash::{Pedersen, Poseidon};
 pub mod bonsai_db;
 mod l1_handler_tx_fee;
 mod meta_db;
-pub mod storage;
+pub mod storage_handler;
+pub mod storage_updates;
 
 pub use error::{BonsaiDbError, DbError};
 pub use mapping_db::MappingCommitment;
+use storage_handler::bonsai_identifier;
 
 const DB_HASH_LEN: usize = 32;
 /// Hash type that this backend uses for the database.
@@ -79,6 +83,7 @@ pub(crate) fn open_rocksdb(path: &Path, create: bool) -> Result<OptimisticTransa
     opts.create_missing_column_families(true);
     opts.set_bytes_per_sync(1024 * 1024);
     opts.set_keep_log_file_num(1);
+    opts.set_compression_type(DBCompressionType::Zstd);
     let cores = std::thread::available_parallelism().map(|e| e.get() as i32).unwrap_or(1);
     opts.increase_parallelism(i32::max(cores / 2, 1));
 
@@ -98,6 +103,11 @@ pub enum Column {
     TransactionMapping,
     SyncedMapping,
     Da,
+    BlockHashToNumber,
+    BlockNumberToHash,
+    ContractClassData,
+    ContractData,
+    ContractClassHashes,
 
     /// This column is used to map starknet block hashes to a list of transaction hashes that are
     /// contained in the block.
@@ -151,6 +161,11 @@ impl Column {
             StarknetTransactionHashesCache,
             StarknetBlockHashesCache,
             L1HandlerPaidFee,
+            BlockHashToNumber,
+            BlockNumberToHash,
+            ContractClassData,
+            ContractData,
+            ContractClassHashes,
             BonsaiContractsTrie,
             BonsaiContractsFlat,
             BonsaiContractsLog,
@@ -183,6 +198,11 @@ impl Column {
             Column::BonsaiClassesTrie => "bonsai_classes_trie",
             Column::BonsaiClassesFlat => "bonsai_classes_flat",
             Column::BonsaiClassesLog => "bonsai_classes_log",
+            Column::BlockHashToNumber => "block_hash_to_number_trie",
+            Column::BlockNumberToHash => "block_to_hash_trie",
+            Column::ContractClassData => "contract_class_data",
+            Column::ContractData => "contract_data",
+            Column::ContractClassHashes => "contract_class_hashes",
         }
     }
 
@@ -202,7 +222,11 @@ pub(crate) trait DatabaseExt {
 
 impl DatabaseExt for DB {
     fn get_column(&self, col: Column) -> Arc<BoundColumnFamily<'_>> {
-        self.cf_handle(col.rocksdb_name()).expect("column not inititalized")
+        let name = col.rocksdb_name();
+        match self.cf_handle(name) {
+            Some(column) => column,
+            None => panic!("column {name} not initialized"),
+        }
     }
 }
 
@@ -301,41 +325,40 @@ impl DeoxysBackend {
                 DatabaseKeyMapping {
                     flat: Column::BonsaiContractsFlat,
                     trie: Column::BonsaiContractsTrie,
-                    trie_log: Column::BonsaiContractsLog,
+                    log: Column::BonsaiContractsLog,
                 },
             ),
             bonsai_config.clone(),
         )
         .unwrap();
-        bonsai_contract.commit(BasicId::new(0)).unwrap();
+        bonsai_contract.init_tree(bonsai_identifier::CONTRACT).unwrap();
 
-        let mut bonsai_contract_storage = BonsaiStorage::new(
+        let bonsai_contract_storage = BonsaiStorage::new(
             BonsaiDb::new(
                 db,
                 DatabaseKeyMapping {
                     flat: Column::BonsaiContractsStorageFlat,
                     trie: Column::BonsaiContractsStorageTrie,
-                    trie_log: Column::BonsaiContractsStorageLog,
+                    log: Column::BonsaiContractsStorageLog,
                 },
             ),
             bonsai_config.clone(),
         )
         .unwrap();
-        bonsai_contract_storage.commit(BasicId::new(0)).unwrap();
 
         let mut bonsai_classes = BonsaiStorage::new(
             BonsaiDb::new(
                 db,
                 DatabaseKeyMapping {
                     flat: Column::BonsaiClassesFlat,
                     trie: Column::BonsaiClassesTrie,
-                    trie_log: Column::BonsaiClassesLog,
+                    log: Column::BonsaiClassesLog,
                 },
             ),
             bonsai_config.clone(),
         )
         .unwrap();
-        bonsai_classes.commit(BasicId::new(0)).unwrap();
+        bonsai_classes.init_tree(bonsai_identifier::CLASS).unwrap();
 
         Ok(Self {
             mapping: Arc::new(MappingDb::new(Arc::clone(db), cache_more_things)),
@@ -375,6 +398,10 @@ impl DeoxysBackend {
         BACKEND_SINGLETON.get().map(|backend| &backend.bonsai_class).expect("Backend not initialized")
     }
 
+    pub(crate) fn expose_db() -> &'static Arc<DB> {
+        DB_SINGLETON.get().expect("Databsae not initialized")
+    }
+
     /// Return l1 handler tx paid fee database manager
     pub fn l1_handler_paid_fee() -> &'static Arc<L1HandlerTxFeeDb> {
         BACKEND_SINGLETON.get().map(|backend| &backend.l1_handler_paid_fee).expect("Backend not initialized")