From e026ea68570d8bda613435fe9a414d86eb0cf029 Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 17 May 2021 18:03:51 +0200 Subject: [PATCH 001/127] starting --- Cargo.lock | 69 ++++-- Cargo.toml | 6 + .../state-machine/src/changes_trie/mod.rs | 10 +- .../state-machine/src/changes_trie/storage.rs | 12 +- .../state-machine/src/proving_backend.rs | 10 +- .../state-machine/src/trie_backend_essence.rs | 106 ++++++-- primitives/trie/src/lib.rs | 233 +++++++++++++----- primitives/trie/src/node_codec.rs | 60 +++-- 8 files changed, 381 insertions(+), 125 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f11953213d01e..1ccdd8f9ea5c7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2329,8 +2329,7 @@ dependencies = [ [[package]] name = "hash-db" version = "0.15.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d23bd4e7b5eda0d0f3a307e8b381fdc8ba9000f26fbe912250c0a4cc3956364a" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple2#594c900614841eea6ab66ac0d169352596ea3322" [[package]] name = "hash256-std-hasher" @@ -2341,6 +2340,14 @@ dependencies = [ "crunchy", ] +[[package]] +name = "hash256-std-hasher" +version = "0.15.2" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple2#594c900614841eea6ab66ac0d169352596ea3322" +dependencies = [ + "crunchy", +] + [[package]] name = "hashbrown" version = "0.9.1" @@ -2982,7 +2989,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "711adba9940a039f4374fc5724c0a5eaca84a2d558cce62256bfe26f0dbef05e" dependencies = [ "hash-db", - "hash256-std-hasher", + "hash256-std-hasher 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", + "tiny-keccak", +] + +[[package]] +name = "keccak-hasher" +version = "0.15.3" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple2#594c900614841eea6ab66ac0d169352596ea3322" +dependencies = [ + "hash-db", + "hash256-std-hasher 0.15.2 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple2)", "tiny-keccak", ] @@ -3771,8 +3788,7 @@ dependencies = [ [[package]] name = "memory-db" version = "0.26.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "814bbecfc0451fc314eeea34f05bbcd5b98a7ad7af37faee088b86a1e633f1d4" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple2#594c900614841eea6ab66ac0d169352596ea3322" dependencies = [ "hash-db", "hashbrown", @@ -4201,7 +4217,7 @@ dependencies = [ "sp-state-machine", "sp-trie", "substrate-test-client", - "trie-root", + "trie-root 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)", "wat", ] @@ -8776,7 +8792,7 @@ dependencies = [ "ed25519-dalek", "futures 0.3.13", "hash-db", - "hash256-std-hasher", + "hash256-std-hasher 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", "hex", "hex-literal", "impl-serde", @@ -9005,7 +9021,7 @@ name = "sp-runtime" version = "3.0.0" dependencies = [ "either", - "hash256-std-hasher", + "hash256-std-hasher 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", "impl-trait-for-tuples", "log", "parity-scale-codec", @@ -9161,7 +9177,7 @@ dependencies = [ "thiserror", "tracing", "trie-db", - "trie-root", + "trie-root 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -9267,8 +9283,8 @@ dependencies = [ "sp-std", "trie-bench", "trie-db", - "trie-root", - "trie-standardmap", + "trie-root 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)", + "trie-standardmap 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -10316,24 +10332,22 @@ checksum = "a7f741b240f1a48843f9b8e0444fb55fb2a4ff67293b50a9179dfd5ea67f8d41" [[package]] name = "trie-bench" version = "0.27.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "568257edb909a5c532b1f4ab38ee6b5dedfbf8775be6a55a29020513ebe3e072" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple2#594c900614841eea6ab66ac0d169352596ea3322" dependencies = [ "criterion", "hash-db", - "keccak-hasher", + "keccak-hasher 0.15.3 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple2)", "memory-db", "parity-scale-codec", "trie-db", - "trie-root", - "trie-standardmap", + "trie-root 0.16.0 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple2)", + "trie-standardmap 0.15.2 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple2)", ] [[package]] name = "trie-db" version = "0.22.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec051edf7f0fc9499a2cb0947652cab2148b9d7f61cee7605e312e9f970dacaf" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple2#594c900614841eea6ab66ac0d169352596ea3322" dependencies = [ "hash-db", "hashbrown", @@ -10351,6 +10365,14 @@ dependencies = [ "hash-db", ] +[[package]] +name = "trie-root" +version = "0.16.0" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple2#594c900614841eea6ab66ac0d169352596ea3322" +dependencies = [ + "hash-db", +] + [[package]] name = "trie-standardmap" version = "0.15.2" @@ -10358,7 +10380,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3161ba520ab28cd8e6b68e1126f1009f6e335339d1a73b978139011703264c8" dependencies = [ "hash-db", - "keccak-hasher", + "keccak-hasher 0.15.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "trie-standardmap" +version = "0.15.2" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple2#594c900614841eea6ab66ac0d169352596ea3322" +dependencies = [ + "hash-db", + "keccak-hasher 0.15.3 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple2)", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 1b35c7181d17d..1bda1bc4f5ec2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -269,3 +269,9 @@ zeroize = { opt-level = 3 } [profile.release] # Substrate runtime requires unwinding. panic = "unwind" + +[patch.crates-io] +hash-db = { git = "https://github.com/cheme/trie.git", branch = "hashed-value-simple2" } +memory-db = { git = "https://github.com/cheme/trie.git", branch = "hashed-value-simple2" } +trie-db = { git = "https://github.com/cheme/trie.git", branch = "hashed-value-simple2" } +trie-bench = { git = "https://github.com/cheme/trie.git", branch = "hashed-value-simple2" } diff --git a/primitives/state-machine/src/changes_trie/mod.rs b/primitives/state-machine/src/changes_trie/mod.rs index 105f3d7de6d39..89d869d38d999 100644 --- a/primitives/state-machine/src/changes_trie/mod.rs +++ b/primitives/state-machine/src/changes_trie/mod.rs @@ -167,9 +167,15 @@ pub struct TrieBackendStorageAdapter<'a, H: Hasher, Number: BlockNumber>(pub &'a impl<'a, H: Hasher, N: BlockNumber> crate::TrieBackendStorage for TrieBackendStorageAdapter<'a, H, N> { type Overlay = sp_trie::MemoryDB; - fn get(&self, key: &H::Out, prefix: Prefix) -> Result, String> { - self.0.get(key, prefix) + fn get(&self, key: &H::Out, prefix: Prefix, _parent: Option<&sp_trie::TrieMeta>) -> Result, String> { + match self.0.get(key, prefix) { + // change trie do not use meta. + Ok(Some(v)) => Ok(Some((v, Default::default()))), + Ok(None) => Ok(None), + Err(e) => Err(e), + } } + fn access_from(&self, _key: &H::Out) { } } /// Changes trie configuration. diff --git a/primitives/state-machine/src/changes_trie/storage.rs b/primitives/state-machine/src/changes_trie/storage.rs index e08fe36126c7b..b41b2e549e82a 100644 --- a/primitives/state-machine/src/changes_trie/storage.rs +++ b/primitives/state-machine/src/changes_trie/storage.rs @@ -190,7 +190,7 @@ impl Storage for InMemoryStorage Result, String> { - MemoryDB::::get(&self.data.read().mdb, key, prefix) + Ok( as hash_db::HashDBRef>::get(&self.data.read().mdb, key, prefix)) } } @@ -207,7 +207,13 @@ impl<'a, H, Number> TrieBackendStorage for TrieBackendAdapter<'a, H, Number> { type Overlay = MemoryDB; - fn get(&self, key: &H::Out, prefix: Prefix) -> Result, String> { - self.storage.get(key, prefix) + fn get(&self, key: &H::Out, prefix: Prefix, _parent: Option<&sp_trie::TrieMeta>) -> Result, String> { + match self.storage.get(key, prefix) { + // change trie do not use meta. + Ok(Some(v)) => Ok(Some((v, Default::default()))), + Ok(None) => Ok(None), + Err(e) => Err(e), + } } + fn access_from(&self, _key: &H::Out) { } } diff --git a/primitives/state-machine/src/proving_backend.rs b/primitives/state-machine/src/proving_backend.rs index 28672659fa10c..fc01cb58d8bf5 100644 --- a/primitives/state-machine/src/proving_backend.rs +++ b/primitives/state-machine/src/proving_backend.rs @@ -24,7 +24,7 @@ use log::debug; use hash_db::{Hasher, HashDB, EMPTY_PREFIX, Prefix}; use sp_trie::{ MemoryDB, empty_child_trie_root, read_trie_value_with, read_child_trie_value_with, - record_all_keys, StorageProof, + record_all_keys, StorageProof, TrieMeta, }; pub use sp_trie::{Recorder, trie_types::{Layout, TrieError}}; use crate::trie_backend::TrieBackend; @@ -219,15 +219,19 @@ impl<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> TrieBackendStorage { type Overlay = S::Overlay; - fn get(&self, key: &H::Out, prefix: Prefix) -> Result, String> { + fn get(&self, key: &H::Out, prefix: Prefix, parent: Option<&TrieMeta>) -> Result, String> { if let Some(v) = self.proof_recorder.get(key) { return Ok(v); } - let backend_value = self.backend.get(key, prefix)?; + let backend_value = self.backend.get(key, prefix, parent)?; self.proof_recorder.record(key.clone(), backend_value.clone()); Ok(backend_value) } + + fn access_from(&self, _key: &H::Out) { + // access_from is mainly for proof recorder, not forwarding it. + } } impl<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> std::fmt::Debug diff --git a/primitives/state-machine/src/trie_backend_essence.rs b/primitives/state-machine/src/trie_backend_essence.rs index c085099da77d8..9d57ff1cafef8 100644 --- a/primitives/state-machine/src/trie_backend_essence.rs +++ b/primitives/state-machine/src/trie_backend_essence.rs @@ -25,7 +25,7 @@ use crate::{warn, debug}; use hash_db::{self, Hasher, Prefix}; use sp_trie::{Trie, MemoryDB, PrefixedMemoryDB, DBValue, empty_child_trie_root, read_trie_value, read_child_trie_value, - for_keys_in_child_trie, KeySpacedDB, TrieDBIterator}; + for_keys_in_child_trie, KeySpacedDB, TrieDBIterator, TrieMeta}; use sp_trie::trie_types::{TrieDB, TrieError, Layout}; use crate::{backend::Consolidate, StorageKey, StorageValue}; use sp_core::storage::ChildInfo; @@ -281,11 +281,11 @@ pub(crate) struct Ephemeral<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> { overlay: &'a mut S::Overlay, } -impl<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> hash_db::AsHashDB +impl<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> hash_db::AsHashDB for Ephemeral<'a, S, H> { - fn as_hash_db<'b>(&'b self) -> &'b (dyn hash_db::HashDB + 'b) { self } - fn as_hash_db_mut<'b>(&'b mut self) -> &'b mut (dyn hash_db::HashDB + 'b) { self } + fn as_hash_db<'b>(&'b self) -> &'b (dyn hash_db::HashDB + 'b) { self } + fn as_hash_db_mut<'b>(&'b mut self) -> &'b mut (dyn hash_db::HashDB + 'b) { self } } impl<'a, S: TrieBackendStorage, H: Hasher> Ephemeral<'a, S, H> { @@ -297,14 +297,18 @@ impl<'a, S: TrieBackendStorage, H: Hasher> Ephemeral<'a, S, H> { } } -impl<'a, S: 'a + TrieBackendStorage, H: Hasher> hash_db::HashDB +impl<'a, S: 'a + TrieBackendStorage, H: Hasher> hash_db::HashDB for Ephemeral<'a, S, H> { fn get(&self, key: &H::Out, prefix: Prefix) -> Option { - if let Some(val) = hash_db::HashDB::get(self.overlay, key, prefix) { + Self::get_with_meta(self, key, prefix, None).map(|r| r.0) + } + + fn get_with_meta(&self, key: &H::Out, prefix: Prefix, parent: Option<&TrieMeta>) -> Option<(DBValue, TrieMeta)> { + if let Some(val) = hash_db::HashDB::get_with_meta(self.overlay, key, prefix, parent) { Some(val) } else { - match self.storage.get(&key, prefix) { + match self.storage.get(&key, prefix, parent) { Ok(x) => x, Err(e) => { warn!(target: "trie", "Failed to read from DB: {}", e); @@ -314,6 +318,12 @@ impl<'a, S: 'a + TrieBackendStorage, H: Hasher> hash_db::HashDB } } + fn access_from(&self, key: &H::Out, _at: Option<&H::Out>) -> Option { + // call back to storage even if the overlay was hit. + self.storage.access_from(key); + None + } + fn contains(&self, key: &H::Out, prefix: Prefix) -> bool { hash_db::HashDB::get(self, key, prefix).is_some() } @@ -322,6 +332,15 @@ impl<'a, S: 'a + TrieBackendStorage, H: Hasher> hash_db::HashDB hash_db::HashDB::insert(self.overlay, prefix, value) } + fn insert_with_meta( + &mut self, + prefix: Prefix, + value: &[u8], + meta: TrieMeta, + ) -> H::Out { + hash_db::HashDB::insert_with_meta(self.overlay, prefix, value, meta) + } + fn emplace(&mut self, key: H::Out, prefix: Prefix, value: DBValue) { hash_db::HashDB::emplace(self.overlay, key, prefix, value) } @@ -331,13 +350,21 @@ impl<'a, S: 'a + TrieBackendStorage, H: Hasher> hash_db::HashDB } } -impl<'a, S: 'a + TrieBackendStorage, H: Hasher> hash_db::HashDBRef +impl<'a, S: 'a + TrieBackendStorage, H: Hasher> hash_db::HashDBRef for Ephemeral<'a, S, H> { fn get(&self, key: &H::Out, prefix: Prefix) -> Option { hash_db::HashDB::get(self, key, prefix) } + fn get_with_meta(&self, key: &H::Out, prefix: Prefix, parent: Option<&TrieMeta>) -> Option<(DBValue, TrieMeta)> { + hash_db::HashDB::get_with_meta(self, key, prefix, parent) + } + + fn access_from(&self, key: &H::Out, at: Option<&H::Out>) -> Option { + hash_db::HashDB::access_from(self, key, at) + } + fn contains(&self, key: &H::Out, prefix: Prefix) -> bool { hash_db::HashDB::contains(self, key, prefix) } @@ -346,9 +373,11 @@ impl<'a, S: 'a + TrieBackendStorage, H: Hasher> hash_db::HashDBRef: Send + Sync { /// Type of in-memory overlay. - type Overlay: hash_db::HashDB + Default + Consolidate; + type Overlay: hash_db::HashDB + Default + Consolidate; /// Get the value stored at key. - fn get(&self, key: &H::Out, prefix: Prefix) -> Result>; + fn get(&self, key: &H::Out, prefix: Prefix, parent_meta: Option<&TrieMeta>) -> Result>; + /// Call back when value get accessed in trie. + fn access_from(&self, key: &H::Out); } // This implementation is used by normal storage trie clients. @@ -356,41 +385,58 @@ pub trait TrieBackendStorage: Send + Sync { impl TrieBackendStorage for Arc> { type Overlay = PrefixedMemoryDB; - fn get(&self, key: &H::Out, prefix: Prefix) -> Result> { + fn get(&self, key: &H::Out, prefix: Prefix, parent: Option<&TrieMeta>) -> Result> { + // TODO get impl from memorydb Storage::::get(self.deref(), key, prefix) } + + fn access_from(&self, key: &H::Out) { + Storage::::access_from(self.deref(), key) + } } // This implementation is used by test storage trie clients. impl TrieBackendStorage for PrefixedMemoryDB { type Overlay = PrefixedMemoryDB; - fn get(&self, key: &H::Out, prefix: Prefix) -> Result> { - Ok(hash_db::HashDB::get(self, key, prefix)) + fn get(&self, key: &H::Out, prefix: Prefix) -> Result> { + Ok(hash_db::HashDB::get_with_meta(self, key, prefix)) + } + + fn access_from(&self, key: &H::Out) { + hash_db::HashDB::access_from(self, key, None); } } impl TrieBackendStorage for MemoryDB { type Overlay = MemoryDB; - fn get(&self, key: &H::Out, prefix: Prefix) -> Result> { - Ok(hash_db::HashDB::get(self, key, prefix)) + fn get(&self, key: &H::Out, prefix: Prefix) -> Result> { + Ok(hash_db::HashDB::get_with_meta(self, key, prefix)) + } + + fn access_from(&self, key: &H::Out) { + hash_db::HashDB::access_from(self, key, None); } } -impl, H: Hasher> hash_db::AsHashDB +impl, H: Hasher> hash_db::AsHashDB for TrieBackendEssence { - fn as_hash_db<'b>(&'b self) -> &'b (dyn hash_db::HashDB + 'b) { self } - fn as_hash_db_mut<'b>(&'b mut self) -> &'b mut (dyn hash_db::HashDB + 'b) { self } + fn as_hash_db<'b>(&'b self) -> &'b (dyn hash_db::HashDB + 'b) { self } + fn as_hash_db_mut<'b>(&'b mut self) -> &'b mut (dyn hash_db::HashDB + 'b) { self } } -impl, H: Hasher> hash_db::HashDB +impl, H: Hasher> hash_db::HashDB for TrieBackendEssence { fn get(&self, key: &H::Out, prefix: Prefix) -> Option { + self.get_with_meta(key, prefix).map(|r| r.0) + } + + fn get_with_meta(&self, key: &H::Out, prefix: Prefix) -> Option<(DBValue, TrieMeta)> { if *key == self.empty { - return Some([0u8].to_vec()) + return Some(([0u8].to_vec(), ::meta_for_empty())) } match self.storage.get(&key, prefix) { Ok(x) => x, @@ -401,10 +447,20 @@ impl, H: Hasher> hash_db::HashDB } } + fn access_from(&self, key: &H::Out, _at: Option<&H::Out>) -> Option { + // access storage since this is only to register access for proof. + self.storage.access_from(key); + None + } + fn contains(&self, key: &H::Out, prefix: Prefix) -> bool { hash_db::HashDB::get(self, key, prefix).is_some() } + fn insert_with_meta(&mut self, _prefix: Prefix, _value: &[u8], _meta: TrieMeta) -> H::Out { + unimplemented!(); + } + fn insert(&mut self, _prefix: Prefix, _value: &[u8]) -> H::Out { unimplemented!(); } @@ -418,13 +474,21 @@ impl, H: Hasher> hash_db::HashDB } } -impl, H: Hasher> hash_db::HashDBRef +impl, H: Hasher> hash_db::HashDBRef for TrieBackendEssence { fn get(&self, key: &H::Out, prefix: Prefix) -> Option { hash_db::HashDB::get(self, key, prefix) } + fn get_with_meta(&self, key: &H::Out, prefix: Prefix, parent: Option<&TrieMeta>) -> Option<(DBValue, TrieMeta)> { + hash_db::HashDB::get_with_meta(self, key, prefix, parent) + } + + fn access_from(&self, key: &H::Out, at: Option<&H::Out>) -> Option { + hash_db::HashDB::access_from(self, key, at) + } + fn contains(&self, key: &H::Out, prefix: Prefix) -> bool { hash_db::HashDB::contains(self, key, prefix) } diff --git a/primitives/trie/src/lib.rs b/primitives/trie/src/lib.rs index 572283f1c027e..5892b26b91146 100644 --- a/primitives/trie/src/lib.rs +++ b/primitives/trie/src/lib.rs @@ -25,9 +25,9 @@ mod node_codec; mod storage_proof; mod trie_stream; -use sp_std::{boxed::Box, marker::PhantomData, vec::Vec, borrow::Borrow}; +use sp_std::{boxed::Box, marker::PhantomData, vec::Vec, borrow::Borrow, fmt}; use hash_db::{Hasher, Prefix}; -use trie_db::proof::{generate_proof, verify_proof}; +//use trie_db::proof::{generate_proof, verify_proof}; pub use trie_db::proof::VerifyError; /// Our `NodeCodec`-specific error. pub use error::Error; @@ -38,27 +38,90 @@ pub use node_codec::NodeCodec; pub use storage_proof::StorageProof; /// Various re-exports from the `trie-db` crate. pub use trie_db::{ - Trie, TrieMut, DBValue, Recorder, CError, Query, TrieLayout, TrieConfiguration, nibble_ops, TrieDBIterator, + Trie, TrieMut, DBValue, Recorder, CError, Query, TrieLayout, TrieConfiguration, + nibble_ops, TrieDBIterator, Meta, }; /// Various re-exports from the `memory-db` crate. pub use memory_db::KeyFunction; pub use memory_db::prefixed_key; /// Various re-exports from the `hash-db` crate. -pub use hash_db::{HashDB as HashDBT, EMPTY_PREFIX}; +pub use hash_db::{HashDB as HashDBT, EMPTY_PREFIX, MetaHasher}; +pub use hash_db::NoMeta; -#[derive(Default)] +/// Meta use by trie state. +pub type TrieMeta = (); /// substrate trie layout pub struct Layout(sp_std::marker::PhantomData); +impl fmt::Debug for Layout { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.debug_struct("Layout").finish() + } +} + +impl Default for Layout { + fn default() -> Self { + Layout(sp_std::marker::PhantomData) + } +} + +impl Clone for Layout { + fn clone(&self) -> Self { + Layout(sp_std::marker::PhantomData) + } +} + impl TrieLayout for Layout { const USE_EXTENSION: bool = false; const ALLOW_EMPTY: bool = true; type Hash = H; type Codec = NodeCodec; + type MetaHasher = StateHasher; + type Meta = TrieMeta; + + fn metainput_for_new_node(&self) -> ::MetaInput { + () + } + fn metainput_for_stored_inline_node(&self) -> ::MetaInput { + () + } +} + +/// Reimplement `NoMeta` `MetaHasher` with +/// additional constraint. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct StateHasher; + +impl MetaHasher for StateHasher + where + H: Hasher, + T: for<'a> From<&'a [u8]>, +{ + type Meta = TrieMeta; + + fn hash(value: &[u8], _meta: &Self::Meta) -> H::Out { + H::hash(value) + } + + fn stored_value(value: &[u8], _meta: Self::Meta) -> T { + value.into() + } + + fn stored_value_owned(value: T, _meta: Self::Meta) -> T { + value + } + + fn extract_value<'a>(stored: &'a [u8], _parent_meta: Option<&Self::Meta>) -> (&'a [u8], Self::Meta) { + (stored, ()) + } + + fn extract_value_owned(stored: T, _parent_meta: Option<&Self::Meta>) -> (T, Self::Meta) { + (stored, ()) + } } impl TrieConfiguration for Layout { - fn trie_root(input: I) -> ::Out where + fn trie_root(&self, input: I) -> ::Out where I: IntoIterator, A: AsRef<[u8]> + Ord, B: AsRef<[u8]>, @@ -66,7 +129,7 @@ impl TrieConfiguration for Layout { trie_root::trie_root_no_extension::(input) } - fn trie_root_unhashed(input: I) -> Vec where + fn trie_root_unhashed(&self, input: I) -> Vec where I: IntoIterator, A: AsRef<[u8]> + Ord, B: AsRef<[u8]>, @@ -87,25 +150,30 @@ type MemTracker = memory_db::MemCounter; /// TrieDB error over `TrieConfiguration` trait. pub type TrieError = trie_db::TrieError, CError>; /// Reexport from `hash_db`, with genericity set for `Hasher` trait. -pub trait AsHashDB: hash_db::AsHashDB {} -impl> AsHashDB for T {} +pub trait AsHashDB: hash_db::AsHashDB {} +impl> AsHashDB for T {} /// Reexport from `hash_db`, with genericity set for `Hasher` trait. -pub type HashDB<'a, H> = dyn hash_db::HashDB + 'a; +pub type HashDB<'a, H, M> = dyn hash_db::HashDB + 'a; /// Reexport from `hash_db`, with genericity set for `Hasher` trait. /// This uses a `KeyFunction` for prefixing keys internally (avoiding /// key conflict for non random keys). pub type PrefixedMemoryDB = memory_db::MemoryDB< - H, memory_db::PrefixedKey, trie_db::DBValue, MemTracker + H, memory_db::PrefixedKey, trie_db::DBValue, StateHasher, MemTracker >; /// Reexport from `hash_db`, with genericity set for `Hasher` trait. /// This uses a noops `KeyFunction` (key addressing must be hashed or using /// an encoding scheme that avoid key conflict). pub type MemoryDB = memory_db::MemoryDB< - H, memory_db::HashKey, trie_db::DBValue, MemTracker, + H, memory_db::HashKey, trie_db::DBValue, StateHasher, MemTracker, +>; +/// MemoryDB with specific meta hasher. +pub type MemoryDBMeta = memory_db::MemoryDB< + H, memory_db::HashKey, trie_db::DBValue, M, MemTracker, >; + /// Reexport from `hash_db`, with genericity set for `Hasher` trait. pub type GenericMemoryDB = memory_db::MemoryDB< - H, KF, trie_db::DBValue, MemTracker + H, KF, trie_db::DBValue, StateHasher, MemTracker >; /// Persistent trie database read-access interface for the a given hasher. @@ -116,7 +184,6 @@ pub type TrieDBMut<'a, L> = trie_db::TrieDBMut<'a, L>; pub type Lookup<'a, L, Q> = trie_db::Lookup<'a, L, Q>; /// Hash type for a trie layout. pub type TrieHash = <::Hash as Hasher>::Out; - /// This module is for non generic definition of trie type. /// Only the `Hasher` trait is generic in this case. pub mod trie_types { @@ -131,6 +198,7 @@ pub mod trie_types { pub type TrieError = trie_db::TrieError; } +/* /// Create a proof for a subset of keys in a trie. /// /// The `keys` may contain any set of keys regardless of each one of them is included @@ -146,7 +214,7 @@ pub fn generate_trie_proof<'a, L: TrieConfiguration, I, K, DB>( ) -> Result>, Box>> where I: IntoIterator, K: 'a + AsRef<[u8]>, - DB: hash_db::HashDBRef, + DB: hash_db::HashDBRef, { let trie = TrieDB::::new(db, &root)?; generate_proof(&trie, keys) @@ -171,6 +239,7 @@ pub fn verify_trie_proof<'a, L: TrieConfiguration, I, K, V>( { verify_proof::, _, _, _>(root, proof, items) } +*/ /// Determine a trie root given a hash DB and delta values. pub fn delta_trie_root( @@ -182,7 +251,7 @@ pub fn delta_trie_root( A: Borrow<[u8]>, B: Borrow>, V: Borrow<[u8]>, - DB: hash_db::HashDB, + DB: hash_db::HashDB, { { let mut trie = TrieDBMut::::from_existing(db, &mut root)?; @@ -202,7 +271,7 @@ pub fn delta_trie_root( } /// Read a value from the trie. -pub fn read_trie_value>( +pub fn read_trie_value>( db: &DB, root: &TrieHash, key: &[u8] @@ -214,7 +283,7 @@ pub fn read_trie_value, - DB: hash_db::HashDBRef + DB: hash_db::HashDBRef >( db: &DB, root: &TrieHash, @@ -226,17 +295,18 @@ pub fn read_trie_value_with< /// Determine the empty trie root. pub fn empty_trie_root() -> ::Out { - L::trie_root::<_, Vec, Vec>(core::iter::empty()) + L::default().trie_root::<_, Vec, Vec>(core::iter::empty()) } /// Determine the empty child trie root. pub fn empty_child_trie_root() -> ::Out { - L::trie_root::<_, Vec, Vec>(core::iter::empty()) + L::default().trie_root::<_, Vec, Vec>(core::iter::empty()) } /// Determine a child trie root given its ordered contents, closed form. H is the default hasher, /// but a generic implementation may ignore this type parameter and use other hashers. pub fn child_trie_root( + layout: &L, input: I, ) -> ::Out where @@ -244,7 +314,7 @@ pub fn child_trie_root( A: AsRef<[u8]> + Ord, B: AsRef<[u8]>, { - L::trie_root(input) + layout.trie_root(input) } /// Determine a child trie root given a hash DB and delta values. H is the default hasher, @@ -261,7 +331,7 @@ pub fn child_delta_trie_root( B: Borrow>, V: Borrow<[u8]>, RD: AsRef<[u8]>, - DB: hash_db::HashDB + DB: hash_db::HashDB { let mut root = TrieHash::::default(); // root is fetched from DB, not writable by runtime, so it's always valid. @@ -284,7 +354,7 @@ pub fn for_keys_in_child_trie bool, DB> mut f: F ) -> Result<(), Box>> where - DB: hash_db::HashDBRef + DB: hash_db::HashDBRef { let mut root = TrieHash::::default(); // root is fetched from DB, not writable by runtime, so it's always valid. @@ -310,7 +380,7 @@ pub fn record_all_keys( root: &TrieHash, recorder: &mut Recorder> ) -> Result<(), Box>> where - DB: hash_db::HashDBRef + DB: hash_db::HashDBRef { let trie = TrieDB::::new(&*db, root)?; let iter = trie.iter()?; @@ -335,7 +405,7 @@ pub fn read_child_trie_value( key: &[u8] ) -> Result>, Box>> where - DB: hash_db::HashDBRef + DB: hash_db::HashDBRef { let mut root = TrieHash::::default(); // root is fetched from DB, not writable by runtime, so it's always valid. @@ -354,7 +424,7 @@ pub fn read_child_trie_value_with Result>, Box>> where - DB: hash_db::HashDBRef + DB: hash_db::HashDBRef { let mut root = TrieHash::::default(); // root is fetched from DB, not writable by runtime, so it's always valid. @@ -401,8 +471,8 @@ impl<'a, DB, H> KeySpacedDBMut<'a, DB, H> where } } -impl<'a, DB, H, T> hash_db::HashDBRef for KeySpacedDB<'a, DB, H> where - DB: hash_db::HashDBRef, +impl<'a, DB, H, T, M> hash_db::HashDBRef for KeySpacedDB<'a, DB, H> where + DB: hash_db::HashDBRef, H: Hasher, T: From<&'static [u8]>, { @@ -411,14 +481,23 @@ impl<'a, DB, H, T> hash_db::HashDBRef for KeySpacedDB<'a, DB, H> where self.0.get(key, (&derived_prefix.0, derived_prefix.1)) } + fn access_from(&self, key: &H::Out, at: Option<&H::Out>) -> Option { + self.0.access_from(key, at) + } + + fn get_with_meta(&self, key: &H::Out, prefix: Prefix, parent: Option<&M>) -> Option<(T, M)> { + let derived_prefix = keyspace_as_prefix_alloc(self.1, prefix); + self.0.get_with_meta(key, (&derived_prefix.0, derived_prefix.1), parent) + } + fn contains(&self, key: &H::Out, prefix: Prefix) -> bool { let derived_prefix = keyspace_as_prefix_alloc(self.1, prefix); self.0.contains(key, (&derived_prefix.0, derived_prefix.1)) } } -impl<'a, DB, H, T> hash_db::HashDB for KeySpacedDBMut<'a, DB, H> where - DB: hash_db::HashDB, +impl<'a, DB, H, T, M> hash_db::HashDB for KeySpacedDBMut<'a, DB, H> where + DB: hash_db::HashDB, H: Hasher, T: Default + PartialEq + for<'b> From<&'b [u8]> + Clone + Send + Sync, { @@ -427,6 +506,15 @@ impl<'a, DB, H, T> hash_db::HashDB for KeySpacedDBMut<'a, DB, H> where self.0.get(key, (&derived_prefix.0, derived_prefix.1)) } + fn access_from(&self, key: &H::Out, at: Option<&H::Out>) -> Option { + self.0.access_from(key, at) + } + + fn get_with_meta(&self, key: &H::Out, prefix: Prefix, parent: Option<&M>) -> Option<(T, M)> { + let derived_prefix = keyspace_as_prefix_alloc(self.1, prefix); + self.0.get_with_meta(key, (&derived_prefix.0, derived_prefix.1), parent) + } + fn contains(&self, key: &H::Out, prefix: Prefix) -> bool { let derived_prefix = keyspace_as_prefix_alloc(self.1, prefix); self.0.contains(key, (&derived_prefix.0, derived_prefix.1)) @@ -437,6 +525,16 @@ impl<'a, DB, H, T> hash_db::HashDB for KeySpacedDBMut<'a, DB, H> where self.0.insert((&derived_prefix.0, derived_prefix.1), value) } + fn insert_with_meta( + &mut self, + prefix: Prefix, + value: &[u8], + meta: M, + ) -> H::Out { + let derived_prefix = keyspace_as_prefix_alloc(self.1, prefix); + self.0.insert_with_meta((&derived_prefix.0, derived_prefix.1), value, meta) + } + fn emplace(&mut self, key: H::Out, prefix: Prefix, value: T) { let derived_prefix = keyspace_as_prefix_alloc(self.1, prefix); self.0.emplace(key, (&derived_prefix.0, derived_prefix.1), value) @@ -448,14 +546,14 @@ impl<'a, DB, H, T> hash_db::HashDB for KeySpacedDBMut<'a, DB, H> where } } -impl<'a, DB, H, T> hash_db::AsHashDB for KeySpacedDBMut<'a, DB, H> where - DB: hash_db::HashDB, +impl<'a, DB, H, T, M> hash_db::AsHashDB for KeySpacedDBMut<'a, DB, H> where + DB: hash_db::HashDB, H: Hasher, T: Default + PartialEq + for<'b> From<&'b [u8]> + Clone + Send + Sync, { - fn as_hash_db(&self) -> &dyn hash_db::HashDB { &*self } + fn as_hash_db(&self) -> &dyn hash_db::HashDB { &*self } - fn as_hash_db_mut<'b>(&'b mut self) -> &'b mut (dyn hash_db::HashDB + 'b) { + fn as_hash_db_mut<'b>(&'b mut self) -> &'b mut (dyn hash_db::HashDB + 'b) { &mut *self } } @@ -482,18 +580,21 @@ mod tests { type Layout = super::Layout; fn hashed_null_node() -> TrieHash { - ::hashed_null_node() + >::hashed_null_node() } fn check_equivalent(input: &Vec<(&[u8], &[u8])>) { { - let closed_form = T::trie_root(input.clone()); - let d = T::trie_root_unhashed(input.clone()); + // TODO test flagged + let layout = T::default(); + let closed_form = layout.trie_root(input.clone()); + let d = layout.trie_root_unhashed(input.clone()); println!("Data: {:#x?}, {:#x?}", d, Blake2Hasher::hash(&d[..])); let persistent = { - let mut memdb = MemoryDB::default(); + let mut memdb = MemoryDBMeta::<_, T::MetaHasher>::default(); let mut root = Default::default(); - let mut t = TrieDBMut::::new(&mut memdb, &mut root); + // TODO test flagged + let mut t = TrieDBMut::::new(&mut memdb, &mut root, layout); for (x, y) in input.iter().rev() { t.insert(x, y).unwrap(); } @@ -504,16 +605,18 @@ mod tests { } fn check_iteration(input: &Vec<(&[u8], &[u8])>) { - let mut memdb = MemoryDB::default(); + // TODO test flagged + let layout = T::default(); + let mut memdb = MemoryDBMeta::<_, T::MetaHasher>::default(); let mut root = Default::default(); { - let mut t = TrieDBMut::::new(&mut memdb, &mut root); + let mut t = TrieDBMut::::new(&mut memdb, &mut root, layout.clone()); for (x, y) in input.clone() { t.insert(x, y).unwrap(); } } { - let t = TrieDB::::new(&mut memdb, &root).unwrap(); + let t = TrieDB::::new(&mut memdb, &root, layout).unwrap(); assert_eq!( input.iter().map(|(i, j)| (i.to_vec(), j.to_vec())).collect::>(), t.iter().unwrap() @@ -525,12 +628,14 @@ mod tests { #[test] fn default_trie_root() { + // TODO test flagged + let layout = Layout::default(); let mut db = MemoryDB::default(); let mut root = TrieHash::::default(); let mut empty = TrieDBMut::::new(&mut db, &mut root); empty.commit(); let root1 = empty.root().as_ref().to_vec(); - let root2: Vec = Layout::trie_root::<_, Vec, Vec>( + let root2: Vec = layout.trie_root::<_, Vec, Vec>( std::iter::empty(), ).as_ref().iter().cloned().collect(); @@ -633,11 +738,13 @@ mod tests { } fn populate_trie<'db, T: TrieConfiguration>( - db: &'db mut dyn HashDB, + db: &'db mut dyn HashDB, root: &'db mut TrieHash, + layout: T, v: &[(Vec, Vec)] ) -> TrieDBMut<'db, T> { - let mut t = TrieDBMut::::new(db, root); + // TODO test non default layout + let mut t = TrieDBMut::::new_with_layout(db, root, layout); for i in 0..v.len() { let key: &[u8]= &v[i].0; let val: &[u8] = &v[i].1; @@ -671,10 +778,12 @@ mod tests { count: 100, }.make_with(seed.as_fixed_bytes_mut()); - let real = Layout::trie_root(x.clone()); + // TODO test other layout states. + let layout = Layout::default(); + let real = layout.trie_root(x.clone()); let mut memdb = MemoryDB::default(); let mut root = Default::default(); - let mut memtrie = populate_trie::(&mut memdb, &mut root, &x); + let mut memtrie = populate_trie::(&mut memdb, &mut root, layout, &x); memtrie.commit(); if *memtrie.root() != real { @@ -707,18 +816,22 @@ mod tests { #[test] fn codec_trie_empty() { + // TODO test other layout states. + let layout = Layout::default(); let input: Vec<(&[u8], &[u8])> = vec![]; - let trie = Layout::trie_root_unhashed::<_, _, _>(input); + let trie = layout.trie_root_unhashed(input); println!("trie: {:#x?}", trie); assert_eq!(trie, vec![0x0]); } #[test] fn codec_trie_single_tuple() { + // TODO switch to old layout + let layout = Layout::default(); let input = vec![ (vec![0xaa], vec![0xbb]) ]; - let trie = Layout::trie_root_unhashed::<_, _, _>(input); + let trie = layout.trie_root_unhashed(input); println!("trie: {:#x?}", trie); assert_eq!(trie, vec![ 0x42, // leaf 0x40 (2^6) with (+) key of 2 nibbles (0x02) @@ -730,8 +843,10 @@ mod tests { #[test] fn codec_trie_two_tuples_disjoint_keys() { + // TODO switch to old layout + let layout = Layout::default(); let input = vec![(&[0x48, 0x19], &[0xfe]), (&[0x13, 0x14], &[0xff])]; - let trie = Layout::trie_root_unhashed::<_, _, _>(input); + let trie = layout.trie_root_unhashed(input); println!("trie: {:#x?}", trie); let mut ex = Vec::::new(); ex.push(0x80); // branch, no value (0b_10..) no nibble @@ -760,11 +875,13 @@ mod tests { (hex!("0103000000000000000469").to_vec(), hex!("0401000000").to_vec()), ]; + // TODO test non default layout + let layout = Layout::default(); let mut mdb = MemoryDB::default(); let mut root = Default::default(); - let _ = populate_trie::(&mut mdb, &mut root, &pairs); + let _ = populate_trie::(&mut mdb, &mut root, layout.clone(), &pairs); - let trie = TrieDB::::new(&mdb, &root).unwrap(); + let trie = TrieDB::::new_with_layout(&mdb, &root, layout).unwrap(); let iter = trie.iter().unwrap(); let mut iter_pairs = Vec::new(); @@ -775,7 +892,7 @@ mod tests { assert_eq!(pairs, iter_pairs); } - +/* #[test] fn proof_non_inclusion_works() { let pairs = vec![ @@ -860,9 +977,11 @@ mod tests { ).is_err() ); } - +*/ #[test] fn generate_storage_root_with_proof_works_independently_from_the_delta_order() { + // TODO use old format. + let layout = Layout::default(); let proof = StorageProof::decode(&mut &include_bytes!("../test-res/proof")[..]).unwrap(); let storage_root = sp_core::H256::decode( &mut &include_bytes!("../test-res/storage_root")[..], @@ -877,12 +996,14 @@ mod tests { ).unwrap(); let proof_db = proof.into_memory_db::(); - let first_storage_root = delta_trie_root::( + let first_storage_root = delta_trie_root( + layout.clone(), &mut proof_db.clone(), storage_root, valid_delta, ).unwrap(); - let second_storage_root = delta_trie_root::( + let second_storage_root = delta_trie_root( + layout, &mut proof_db.clone(), storage_root, invalid_delta, diff --git a/primitives/trie/src/node_codec.rs b/primitives/trie/src/node_codec.rs index 0c923ff024c55..11b379fce6f2d 100644 --- a/primitives/trie/src/node_codec.rs +++ b/primitives/trie/src/node_codec.rs @@ -23,8 +23,8 @@ use sp_std::vec::Vec; use sp_std::borrow::Borrow; use codec::{Encode, Decode, Input, Compact}; use hash_db::Hasher; -use trie_db::{self, node::{NibbleSlicePlan, NodePlan, NodeHandlePlan}, ChildReference, - nibble_ops, Partial, NodeCodec as NodeCodecT}; +use trie_db::{self, node::{NibbleSlicePlan, NodePlan, Value, ValuePlan, NodeHandlePlan}, + ChildReference, nibble_ops, Partial, NodeCodec as NodeCodecT, Meta}; use crate::error::Error; use crate::trie_constants; use super::{node_header::{NodeHeader, NodeKind}}; @@ -87,15 +87,15 @@ impl<'a> Input for ByteSliceInput<'a> { #[derive(Default, Clone)] pub struct NodeCodec(PhantomData); -impl NodeCodecT for NodeCodec { +impl NodeCodecT for NodeCodec { type Error = Error; type HashOut = H::Out; fn hashed_null_node() -> ::Out { - H::hash(::empty_node()) + H::hash(>::empty_node()) } - fn decode_plan(data: &[u8]) -> sp_std::result::Result { + fn decode_plan_inner(data: &[u8]) -> sp_std::result::Result { let mut input = ByteSliceInput::new(data); match NodeHeader::decode(&mut input)? { NodeHeader::Null => Ok(NodePlan::Empty), @@ -113,9 +113,9 @@ impl NodeCodecT for NodeCodec { let bitmap = Bitmap::decode(&data[bitmap_range])?; let value = if has_value { let count = >::decode(&mut input)?.0 as usize; - Some(input.take(count)?) + ValuePlan::Value(input.take(count)?) } else { - None + ValuePlan::NoValue }; let mut children = [ None, None, None, None, None, None, None, None, @@ -151,23 +151,31 @@ impl NodeCodecT for NodeCodec { let count = >::decode(&mut input)?.0 as usize; Ok(NodePlan::Leaf { partial: NibbleSlicePlan::new(partial, partial_padding), - value: input.take(count)?, + value: ValuePlan::Value(input.take(count)?), }) } } } fn is_empty_node(data: &[u8]) -> bool { - data == ::empty_node() + data == >::empty_node() } fn empty_node() -> &'static [u8] { &[trie_constants::EMPTY_TRIE] } - fn leaf_node(partial: Partial, value: &[u8]) -> Vec { + fn leaf_node(partial: Partial, value: Value, meta: &mut M) -> Vec { let mut output = partial_encode(partial, NodeKind::Leaf); - value.encode_to(&mut output); + if let Value::Value(value) = value { + Compact(value.len() as u32).encode_to(&mut output); + let start = output.len(); + output.extend_from_slice(value); + let end = output.len(); + meta.encoded_value_callback(ValuePlan::Value(start..end)); + } else { + unimplemented!("No support for incomplete nodes"); + } output } @@ -175,13 +183,15 @@ impl NodeCodecT for NodeCodec { _partial: impl Iterator, _nbnibble: usize, _child: ChildReference<::Out>, + _meta: &mut M, ) -> Vec { unreachable!() } fn branch_node( _children: impl Iterator::Out>>>>, - _maybe_value: Option<&[u8]>, + _maybe_value: Value, + _meta: &mut M, ) -> Vec { unreachable!() } @@ -190,19 +200,28 @@ impl NodeCodecT for NodeCodec { partial: impl Iterator, number_nibble: usize, children: impl Iterator::Out>>>>, - maybe_value: Option<&[u8]>, + maybe_value: Value, + meta: &mut M, ) -> Vec { - let mut output = if maybe_value.is_some() { - partial_from_iterator_encode(partial, number_nibble, NodeKind::BranchWithValue) - } else { - partial_from_iterator_encode(partial, number_nibble, NodeKind::BranchNoValue) + let mut output = match maybe_value { + Value::Value(..) => partial_from_iterator_encode(partial, number_nibble, NodeKind::BranchWithValue), + Value::NoValue => partial_from_iterator_encode(partial, number_nibble, NodeKind::BranchNoValue), + Value::HashedValue(..) => unimplemented!("No support for incomplete nodes"), }; let bitmap_index = output.len(); let mut bitmap: [u8; BITMAP_LENGTH] = [0; BITMAP_LENGTH]; (0..BITMAP_LENGTH).for_each(|_|output.push(0)); - if let Some(value) = maybe_value { - value.encode_to(&mut output); - }; + match maybe_value { + Value::Value(value) => { + Compact(value.len() as u32).encode_to(&mut output); + let start = output.len(); + output.extend_from_slice(value); + let end = output.len(); + meta.encoded_value_callback(ValuePlan::Value(start..end)); + }, + Value::NoValue => (), + Value::HashedValue(..) => unimplemented!("No support for incomplete nodes"), + } Bitmap::encode(children.map(|maybe_child| match maybe_child.borrow() { Some(ChildReference::Hash(h)) => { h.as_ref().encode_to(&mut output); @@ -218,7 +237,6 @@ impl NodeCodecT for NodeCodec { .copy_from_slice(&bitmap[..BITMAP_LENGTH]); output } - } // utils From 5b1c550c17b949714f0e3d7c33f097492d1f00e1 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 18 May 2021 10:53:29 +0200 Subject: [PATCH 002/127] Updated from other branch. --- bin/node/bench/src/simple_trie.rs | 22 ++++++++++--- bin/node/bench/src/trie.rs | 11 +++++-- bin/node/executor/tests/common.rs | 2 +- client/api/src/cht.rs | 2 +- client/db/src/bench.rs | 11 +++++-- client/db/src/lib.rs | 17 +++++++--- client/executor/src/integration_tests/mod.rs | 2 +- client/service/test/src/client/light.rs | 2 +- client/service/test/src/client/mod.rs | 2 +- primitives/io/src/lib.rs | 8 ++--- primitives/state-machine/src/basic.rs | 2 +- .../state-machine/src/proving_backend.rs | 17 +++++++--- .../state-machine/src/trie_backend_essence.rs | 23 ++++++------- primitives/trie/src/lib.rs | 33 +++++++------------ 14 files changed, 94 insertions(+), 60 deletions(-) diff --git a/bin/node/bench/src/simple_trie.rs b/bin/node/bench/src/simple_trie.rs index a29b51a38af58..6cc32e00e34d9 100644 --- a/bin/node/bench/src/simple_trie.rs +++ b/bin/node/bench/src/simple_trie.rs @@ -20,7 +20,7 @@ use std::{collections::HashMap, sync::Arc}; use kvdb::KeyValueDB; use node_primitives::Hash; -use sp_trie::DBValue; +use sp_trie::{DBValue, TrieMeta, StateHasher, MetaHasher}; use hash_db::{HashDB, AsHashDB, Prefix, Hasher as _}; pub type Hasher = sp_core::Blake2Hasher; @@ -31,15 +31,15 @@ pub struct SimpleTrie<'a> { pub overlay: &'a mut HashMap, Option>>, } -impl<'a> AsHashDB for SimpleTrie<'a> { - fn as_hash_db(&self) -> &dyn hash_db::HashDB { &*self } +impl<'a> AsHashDB for SimpleTrie<'a> { + fn as_hash_db(&self) -> &dyn hash_db::HashDB { &*self } - fn as_hash_db_mut<'b>(&'b mut self) -> &'b mut (dyn HashDB + 'b) { + fn as_hash_db_mut<'b>(&'b mut self) -> &'b mut (dyn HashDB + 'b) { &mut *self } } -impl<'a> HashDB for SimpleTrie<'a> { +impl<'a> HashDB for SimpleTrie<'a> { fn get(&self, key: &Hash, prefix: Prefix) -> Option { let key = sp_trie::prefixed_key::(key, prefix); if let Some(value) = self.overlay.get(&key) { @@ -48,10 +48,22 @@ impl<'a> HashDB for SimpleTrie<'a> { self.db.get(0, &key).expect("Database backend error") } + fn get_with_meta(&self, key: &Hash, prefix: Prefix, parent: Option<&TrieMeta>) -> Option<(DBValue, TrieMeta)> { + let result = self.get(key, prefix); + result.map(|value| >::extract_value_owned(value, parent)) + } + fn contains(&self, hash: &Hash, prefix: Prefix) -> bool { self.get(hash, prefix).is_some() } + fn insert_with_meta(&mut self, prefix: Prefix, value: &[u8], meta: TrieMeta) -> Hash { + let key = >::hash(value, &meta); + let stored_value = >::stored_value(value, meta); + self.emplace(key, prefix, stored_value); + key + } + fn insert(&mut self, prefix: Prefix, value: &[u8]) -> Hash { let key = Hasher::hash(value); self.emplace(key, prefix, value.to_vec()); diff --git a/bin/node/bench/src/trie.rs b/bin/node/bench/src/trie.rs index a3e7620473d98..267d0cc16c8c4 100644 --- a/bin/node/bench/src/trie.rs +++ b/bin/node/bench/src/trie.rs @@ -24,7 +24,7 @@ use lazy_static::lazy_static; use rand::Rng; use hash_db::Prefix; use sp_state_machine::Backend as _; -use sp_trie::{trie_types::TrieDBMut, TrieMut as _}; +use sp_trie::{trie_types::TrieDBMut, TrieMut as _, TrieMeta, MetaHasher, StateHasher}; use node_primitives::Hash; @@ -117,6 +117,7 @@ impl core::BenchmarkDescription for TrieReadBenchmarkDescription { let mut rng = rand::thread_rng(); let warmup_prefix = KUSAMA_STATE_DISTRIBUTION.key(&mut rng); + // TODO flag trie for hash of value. let mut key_values = KeyValues::new(); let mut warmup_keys = KeyValues::new(); let mut query_keys = KeyValues::new(); @@ -169,9 +170,15 @@ impl core::BenchmarkDescription for TrieReadBenchmarkDescription { struct Storage(Arc); impl sp_state_machine::Storage for Storage { - fn get(&self, key: &Hash, prefix: Prefix) -> Result>, String> { + fn get(&self, key: &Hash, prefix: Prefix, parent: Option<&TrieMeta>) -> Result, TrieMeta)>, String> { let key = sp_trie::prefixed_key::(key, prefix); self.0.get(0, &key).map_err(|e| format!("Database backend error: {:?}", e)) + .map(|result| result + .map(|value| >::extract_value_owned(value, parent)) + ) + } + + fn access_from(&self, _key: &Hash) { } } diff --git a/bin/node/executor/tests/common.rs b/bin/node/executor/tests/common.rs index 635155b5d00b2..7ccb4b511ae8a 100644 --- a/bin/node/executor/tests/common.rs +++ b/bin/node/executor/tests/common.rs @@ -157,7 +157,7 @@ pub fn construct_block( // calculate the header fields that we can. let extrinsics_root = - Layout::::ordered_trie_root(extrinsics.iter().map(Encode::encode)) + Layout::::default().ordered_trie_root(extrinsics.iter().map(Encode::encode)) .to_fixed_bytes() .into(); diff --git a/client/api/src/cht.rs b/client/api/src/cht.rs index 96a5a272916e5..1fdeedcde976f 100644 --- a/client/api/src/cht.rs +++ b/client/api/src/cht.rs @@ -93,7 +93,7 @@ pub fn compute_root( I: IntoIterator>>, { use sp_trie::TrieConfiguration; - Ok(sp_trie::trie_types::Layout::::trie_root( + Ok(sp_trie::trie_types::Layout::::default().trie_root( build_pairs::(cht_size, cht_num, hashes)? )) } diff --git a/client/db/src/bench.rs b/client/db/src/bench.rs index a2501891b31e3..c046c5e64545b 100644 --- a/client/db/src/bench.rs +++ b/client/db/src/bench.rs @@ -23,7 +23,7 @@ use std::cell::{Cell, RefCell}; use std::collections::HashMap; use hash_db::{Prefix, Hasher}; -use sp_trie::{MemoryDB, prefixed_key}; +use sp_trie::{MemoryDB, prefixed_key, StateHasher, TrieMeta, MetaHasher}; use sp_core::{ storage::{ChildInfo, TrackedStorageKey}, hexdisplay::HexDisplay @@ -49,21 +49,28 @@ struct StorageDb { } impl sp_state_machine::Storage> for StorageDb { - fn get(&self, key: &Block::Hash, prefix: Prefix) -> Result, String> { + fn get(&self, key: &Block::Hash, prefix: Prefix, parent: Option<&TrieMeta>) -> Result, String> { let prefixed_key = prefixed_key::>(key, prefix); if let Some(recorder) = &self.proof_recorder { if let Some(v) = recorder.get(&key) { return Ok(v.clone()); } let backend_value = self.db.get(0, &prefixed_key) + .map(|result| result.map(|value| , _>>::extract_value_owned(value, parent))) .map_err(|e| format!("Database backend error: {:?}", e))?; recorder.record(key.clone(), backend_value.clone()); Ok(backend_value) } else { self.db.get(0, &prefixed_key) + .map(|result| result.map(|value| , _>>::extract_value_owned(value, parent))) .map_err(|e| format!("Database backend error: {:?}", e)) } } + fn access_from(&self, key: &Block::Hash) { + if let Some(recorder) = &self.proof_recorder { + recorder.access_from(key); + } + } } /// Track whether a specific key has already been read or written to. diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index 94535cf28aea5..245fc3b84f04c 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -65,7 +65,8 @@ use sp_blockchain::{ }; use codec::{Decode, Encode}; use hash_db::Prefix; -use sp_trie::{MemoryDB, PrefixedMemoryDB, prefixed_key}; +use sp_trie::{MemoryDB, PrefixedMemoryDB, prefixed_key, StateHasher, TrieMeta, + MetaHasher}; use sp_database::Transaction; use sp_core::{Hasher, ChangesTrieConfiguration}; use sp_core::offchain::OffchainOverlayedChange; @@ -73,7 +74,8 @@ use sp_core::storage::{well_known_keys, ChildInfo}; use sp_arithmetic::traits::Saturating; use sp_runtime::{generic::{DigestItem, BlockId}, Justification, Justifications, Storage}; use sp_runtime::traits::{ - Block as BlockT, Header as HeaderT, NumberFor, Zero, One, SaturatedConversion, HashFor, + Block as BlockT, Header as HeaderT, NumberFor, Zero, One, SaturatedConversion, + HashFor, }; use sp_state_machine::{ DBValue, ChangesTrieTransaction, ChangesTrieCacheAction, UsageInfo as StateUsageInfo, @@ -858,15 +860,19 @@ struct StorageDb { } impl sp_state_machine::Storage> for StorageDb { - fn get(&self, key: &Block::Hash, prefix: Prefix) -> Result, String> { + fn get(&self, key: &Block::Hash, prefix: Prefix, parent: Option<&TrieMeta>) -> Result, String> { if self.prefix_keys { let key = prefixed_key::>(key, prefix); self.state_db.get(&key, self) } else { self.state_db.get(key.as_ref(), self) } + .map(|result| result.map(|value| , _>>::extract_value_owned(value, parent))) .map_err(|e| format!("Database backend error: {:?}", e)) } + + fn access_from(&self, _key: &Block::Hash) { + } } impl sc_state_db::NodeDb for StorageDb { @@ -890,9 +896,11 @@ impl DbGenesisStorage { } impl sp_state_machine::Storage> for DbGenesisStorage { - fn get(&self, _key: &Block::Hash, _prefix: Prefix) -> Result, String> { + fn get(&self, _key: &Block::Hash, _prefix: Prefix, _parent: Option<&TrieMeta>) -> Result, String> { Ok(None) } + fn access_from(&self, _key: &Block::Hash) { + } } /// Frozen `value` at time `at`. @@ -2110,6 +2118,7 @@ impl sc_client_api::backend::Backend for Backend { self.storage.as_ref(), &header.state_root, (&[], None), + None, ).unwrap_or(None).is_some() }, _ => false, diff --git a/client/executor/src/integration_tests/mod.rs b/client/executor/src/integration_tests/mod.rs index d08f830f40dae..ecb48da91490a 100644 --- a/client/executor/src/integration_tests/mod.rs +++ b/client/executor/src/integration_tests/mod.rs @@ -460,7 +460,7 @@ fn ordered_trie_root_should_work(wasm_method: WasmExecutionMethod) { wasm_method, &mut ext.ext(), ).unwrap(), - Layout::::ordered_trie_root(trie_input.iter()).as_bytes().encode(), + Layout::::default().ordered_trie_root(trie_input.iter()).as_bytes().encode(), ); } diff --git a/client/service/test/src/client/light.rs b/client/service/test/src/client/light.rs index a183cbce62bdb..33bcc96d1bb63 100644 --- a/client/service/test/src/client/light.rs +++ b/client/service/test/src/client/light.rs @@ -568,7 +568,7 @@ fn prepare_for_header_proof_check(insert_cht: bool) -> (TestChecker, Hash, Heade fn header_with_computed_extrinsics_root(extrinsics: Vec) -> Header { use sp_trie::{TrieConfiguration, trie_types::Layout}; let iter = extrinsics.iter().map(Encode::encode); - let extrinsics_root = Layout::::ordered_trie_root(iter); + let extrinsics_root = Layout::::default().ordered_trie_root(iter); // only care about `extrinsics_root` Header::new(0, extrinsics_root, H256::zero(), H256::zero(), Default::default()) diff --git a/client/service/test/src/client/mod.rs b/client/service/test/src/client/mod.rs index 0234f43513d56..0b4a9ed76c56d 100644 --- a/client/service/test/src/client/mod.rs +++ b/client/service/test/src/client/mod.rs @@ -156,7 +156,7 @@ fn construct_block( let transactions = txs.into_iter().map(|tx| tx.into_signed_tx()).collect::>(); let iter = transactions.iter().map(Encode::encode); - let extrinsics_root = Layout::::ordered_trie_root(iter).into(); + let extrinsics_root = Layout::::default().ordered_trie_root(iter).into(); let mut header = Header { parent_hash, diff --git a/primitives/io/src/lib.rs b/primitives/io/src/lib.rs index 35daaa3989907..af6f049f0c529 100644 --- a/primitives/io/src/lib.rs +++ b/primitives/io/src/lib.rs @@ -410,22 +410,22 @@ pub trait DefaultChildStorage { pub trait Trie { /// A trie root formed from the iterated items. fn blake2_256_root(input: Vec<(Vec, Vec)>) -> H256 { - Layout::::trie_root(input) + Layout::::default().trie_root(input) } /// A trie root formed from the enumerated items. fn blake2_256_ordered_root(input: Vec>) -> H256 { - Layout::::ordered_trie_root(input) + Layout::::default().ordered_trie_root(input) } /// A trie root formed from the iterated items. fn keccak_256_root(input: Vec<(Vec, Vec)>) -> H256 { - Layout::::trie_root(input) + Layout::::default().trie_root(input) } /// A trie root formed from the enumerated items. fn keccak_256_ordered_root(input: Vec>) -> H256 { - Layout::::ordered_trie_root(input) + Layout::::default().ordered_trie_root(input) } } diff --git a/primitives/state-machine/src/basic.rs b/primitives/state-machine/src/basic.rs index dda8f523b77f9..e4806c7878ed2 100644 --- a/primitives/state-machine/src/basic.rs +++ b/primitives/state-machine/src/basic.rs @@ -281,7 +281,7 @@ impl Externalities for BasicExternalities { } } - Layout::::trie_root(self.inner.top.clone()).as_ref().into() + Layout::::default().trie_root(self.inner.top.clone()).as_ref().into() } fn child_storage_root( diff --git a/primitives/state-machine/src/proving_backend.rs b/primitives/state-machine/src/proving_backend.rs index fc01cb58d8bf5..66d5b8b2acb2e 100644 --- a/primitives/state-machine/src/proving_backend.rs +++ b/primitives/state-machine/src/proving_backend.rs @@ -112,7 +112,7 @@ impl<'a, S, H> ProvingBackendRecorder<'a, S, H> #[derive(Default)] struct ProofRecorderInner { /// All the records that we have stored so far. - records: HashMap>, + records: HashMap>, /// The encoded size of all recorded values. encoded_size: usize, } @@ -125,11 +125,13 @@ pub struct ProofRecorder { impl ProofRecorder { /// Record the given `key` => `val` combination. - pub fn record(&self, key: Hash, val: Option) { + pub fn record(&self, key: Hash, val: Option<(DBValue, TrieMeta)>) { let mut inner = self.inner.write(); let encoded_size = if let Entry::Vacant(entry) = inner.records.entry(key) { let encoded_size = val.as_ref().map(Encode::encoded_size).unwrap_or(0); +// TODO with new meta +// val.as_mut().map(|val| val.1.set_accessed_value(false)); entry.insert(val); encoded_size } else { @@ -139,8 +141,15 @@ impl ProofRecorder { inner.encoded_size += encoded_size; } + /// Record actual trie level value access. + pub fn access_from(&self, _key: &Hash) { +// TODO with new meta +// self.inner.write().entry(key[..].to_vec()) +// .and_modify(|entry| entry.1.set_accessed_value(true)); + } + /// Returns the value at the given `key`. - pub fn get(&self, key: &Hash) -> Option> { + pub fn get(&self, key: &Hash) -> Option> { self.inner.read().records.get(key).cloned() } @@ -159,7 +168,7 @@ impl ProofRecorder { let trie_nodes = self.inner.read() .records .iter() - .filter_map(|(_k, v)| v.as_ref().map(|v| v.to_vec())) + .filter_map(|(_k, v)| v.as_ref().map(|v| v.0.to_vec())) .collect(); StorageProof::new(trie_nodes) diff --git a/primitives/state-machine/src/trie_backend_essence.rs b/primitives/state-machine/src/trie_backend_essence.rs index 9d57ff1cafef8..3b7d47d9b0c62 100644 --- a/primitives/state-machine/src/trie_backend_essence.rs +++ b/primitives/state-machine/src/trie_backend_essence.rs @@ -43,7 +43,9 @@ type Result = sp_std::result::Result; /// Patricia trie-based storage trait. pub trait Storage: Send + Sync { /// Get a trie node. - fn get(&self, key: &H::Out, prefix: Prefix) -> Result>; + fn get(&self, key: &H::Out, prefix: Prefix, parent: Option<&TrieMeta>) -> Result>; + /// Call back when value get accessed in trie. + fn access_from(&self, key: &H::Out); } /// Patricia trie-based pairs storage essence. @@ -129,7 +131,7 @@ impl, H: Hasher> TrieBackendEssence where H::Out: child_info: Option<&ChildInfo>, key: &[u8], ) -> Result> { - let dyn_eph: &dyn hash_db::HashDBRef<_, _>; + let dyn_eph: &dyn hash_db::HashDBRef<_, _, _>; let keyspace_eph; if let Some(child_info) = child_info.as_ref() { keyspace_eph = KeySpacedDB::new(self, child_info.keyspace()); @@ -386,8 +388,7 @@ impl TrieBackendStorage for Arc> { type Overlay = PrefixedMemoryDB; fn get(&self, key: &H::Out, prefix: Prefix, parent: Option<&TrieMeta>) -> Result> { - // TODO get impl from memorydb - Storage::::get(self.deref(), key, prefix) + Storage::::get(self.deref(), key, prefix, parent) } fn access_from(&self, key: &H::Out) { @@ -399,8 +400,8 @@ impl TrieBackendStorage for Arc> { impl TrieBackendStorage for PrefixedMemoryDB { type Overlay = PrefixedMemoryDB; - fn get(&self, key: &H::Out, prefix: Prefix) -> Result> { - Ok(hash_db::HashDB::get_with_meta(self, key, prefix)) + fn get(&self, key: &H::Out, prefix: Prefix, parent: Option<&TrieMeta>) -> Result> { + Ok(hash_db::HashDB::get_with_meta(self, key, prefix, parent)) } fn access_from(&self, key: &H::Out) { @@ -411,8 +412,8 @@ impl TrieBackendStorage for PrefixedMemoryDB { impl TrieBackendStorage for MemoryDB { type Overlay = MemoryDB; - fn get(&self, key: &H::Out, prefix: Prefix) -> Result> { - Ok(hash_db::HashDB::get_with_meta(self, key, prefix)) + fn get(&self, key: &H::Out, prefix: Prefix, parent: Option<&TrieMeta>) -> Result> { + Ok(hash_db::HashDB::get_with_meta(self, key, prefix, parent)) } fn access_from(&self, key: &H::Out) { @@ -431,14 +432,14 @@ impl, H: Hasher> hash_db::HashDB for TrieBackendEssence { fn get(&self, key: &H::Out, prefix: Prefix) -> Option { - self.get_with_meta(key, prefix).map(|r| r.0) + self.get_with_meta(key, prefix, None).map(|r| r.0) } - fn get_with_meta(&self, key: &H::Out, prefix: Prefix) -> Option<(DBValue, TrieMeta)> { + fn get_with_meta(&self, key: &H::Out, prefix: Prefix, parent: Option<&TrieMeta>) -> Option<(DBValue, TrieMeta)> { if *key == self.empty { return Some(([0u8].to_vec(), ::meta_for_empty())) } - match self.storage.get(&key, prefix) { + match self.storage.get(&key, prefix, parent) { Ok(x) => x, Err(e) => { warn!(target: "trie", "Failed to read from DB: {}", e); diff --git a/primitives/trie/src/lib.rs b/primitives/trie/src/lib.rs index 5892b26b91146..c1615b2fa5acc 100644 --- a/primitives/trie/src/lib.rs +++ b/primitives/trie/src/lib.rs @@ -594,7 +594,7 @@ mod tests { let mut memdb = MemoryDBMeta::<_, T::MetaHasher>::default(); let mut root = Default::default(); // TODO test flagged - let mut t = TrieDBMut::::new(&mut memdb, &mut root, layout); + let mut t = TrieDBMut::::new(&mut memdb, &mut root); for (x, y) in input.iter().rev() { t.insert(x, y).unwrap(); } @@ -605,18 +605,16 @@ mod tests { } fn check_iteration(input: &Vec<(&[u8], &[u8])>) { - // TODO test flagged - let layout = T::default(); let mut memdb = MemoryDBMeta::<_, T::MetaHasher>::default(); let mut root = Default::default(); { - let mut t = TrieDBMut::::new(&mut memdb, &mut root, layout.clone()); + let mut t = TrieDBMut::::new(&mut memdb, &mut root); for (x, y) in input.clone() { t.insert(x, y).unwrap(); } } { - let t = TrieDB::::new(&mut memdb, &root, layout).unwrap(); + let t = TrieDB::::new(&mut memdb, &root).unwrap(); assert_eq!( input.iter().map(|(i, j)| (i.to_vec(), j.to_vec())).collect::>(), t.iter().unwrap() @@ -628,14 +626,12 @@ mod tests { #[test] fn default_trie_root() { - // TODO test flagged - let layout = Layout::default(); let mut db = MemoryDB::default(); let mut root = TrieHash::::default(); let mut empty = TrieDBMut::::new(&mut db, &mut root); empty.commit(); let root1 = empty.root().as_ref().to_vec(); - let root2: Vec = layout.trie_root::<_, Vec, Vec>( + let root2: Vec = Layout::default().trie_root::<_, Vec, Vec>( std::iter::empty(), ).as_ref().iter().cloned().collect(); @@ -737,14 +733,13 @@ mod tests { check_iteration::(&input); } + // TODO add flag fn populate_trie<'db, T: TrieConfiguration>( db: &'db mut dyn HashDB, root: &'db mut TrieHash, - layout: T, v: &[(Vec, Vec)] ) -> TrieDBMut<'db, T> { - // TODO test non default layout - let mut t = TrieDBMut::::new_with_layout(db, root, layout); + let mut t = TrieDBMut::::new(db, root); for i in 0..v.len() { let key: &[u8]= &v[i].0; let val: &[u8] = &v[i].1; @@ -783,7 +778,7 @@ mod tests { let real = layout.trie_root(x.clone()); let mut memdb = MemoryDB::default(); let mut root = Default::default(); - let mut memtrie = populate_trie::(&mut memdb, &mut root, layout, &x); + let mut memtrie = populate_trie::(&mut memdb, &mut root, &x); memtrie.commit(); if *memtrie.root() != real { @@ -875,13 +870,11 @@ mod tests { (hex!("0103000000000000000469").to_vec(), hex!("0401000000").to_vec()), ]; - // TODO test non default layout - let layout = Layout::default(); let mut mdb = MemoryDB::default(); let mut root = Default::default(); - let _ = populate_trie::(&mut mdb, &mut root, layout.clone(), &pairs); + let _ = populate_trie::(&mut mdb, &mut root, &pairs); - let trie = TrieDB::::new_with_layout(&mdb, &root, layout).unwrap(); + let trie = TrieDB::::new(&mdb, &root).unwrap(); let iter = trie.iter().unwrap(); let mut iter_pairs = Vec::new(); @@ -980,8 +973,6 @@ mod tests { */ #[test] fn generate_storage_root_with_proof_works_independently_from_the_delta_order() { - // TODO use old format. - let layout = Layout::default(); let proof = StorageProof::decode(&mut &include_bytes!("../test-res/proof")[..]).unwrap(); let storage_root = sp_core::H256::decode( &mut &include_bytes!("../test-res/storage_root")[..], @@ -996,14 +987,12 @@ mod tests { ).unwrap(); let proof_db = proof.into_memory_db::(); - let first_storage_root = delta_trie_root( - layout.clone(), + let first_storage_root = delta_trie_root::( &mut proof_db.clone(), storage_root, valid_delta, ).unwrap(); - let second_storage_root = delta_trie_root( - layout, + let second_storage_root = delta_trie_root::( &mut proof_db.clone(), storage_root, invalid_delta, From 7e982917628e9d253a9541adaadee0b2dbe1641b Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 18 May 2021 16:07:17 +0200 Subject: [PATCH 003/127] setting flag --- client/api/src/cht.rs | 2 +- client/api/src/in_mem.rs | 1 + client/db/src/bench.rs | 9 +- client/db/src/lib.rs | 33 +- client/db/src/storage_cache.rs | 9 +- client/light/src/backend.rs | 6 +- client/service/test/src/client/light.rs | 6 +- primitives/externalities/src/lib.rs | 3 + primitives/io/src/lib.rs | 6 + primitives/state-machine/src/backend.rs | 9 +- primitives/state-machine/src/basic.rs | 6 + primitives/state-machine/src/ext.rs | 4 + .../state-machine/src/in_memory_backend.rs | 15 +- primitives/state-machine/src/lib.rs | 10 +- .../src/overlayed_changes/mod.rs | 14 +- .../state-machine/src/proving_backend.rs | 76 +++-- primitives/state-machine/src/read_only.rs | 4 + primitives/state-machine/src/testing.rs | 4 +- primitives/state-machine/src/trie_backend.rs | 28 +- primitives/tasks/src/async_externalities.rs | 4 + primitives/trie/src/lib.rs | 312 +++++++++++++++++- primitives/trie/src/node_codec.rs | 117 +++++-- 22 files changed, 567 insertions(+), 111 deletions(-) diff --git a/client/api/src/cht.rs b/client/api/src/cht.rs index 1fdeedcde976f..235976af09018 100644 --- a/client/api/src/cht.rs +++ b/client/api/src/cht.rs @@ -116,7 +116,7 @@ pub fn build_proof( .into_iter() .map(|(k, v)| (k, Some(v))) .collect::>(); - let mut storage = InMemoryBackend::::default().update(vec![(None, transaction)]); + let mut storage = InMemoryBackend::::default().update(vec![(None, transaction)], false); let trie_storage = storage.as_trie_backend() .expect("InMemoryState::as_trie_backend always returns Some; qed"); prove_read_on_trie_backend( diff --git a/client/api/src/in_mem.rs b/client/api/src/in_mem.rs index d756e1cc0bbc4..3f2a26a2c55c8 100644 --- a/client/api/src/in_mem.rs +++ b/client/api/src/in_mem.rs @@ -576,6 +576,7 @@ impl backend::BlockImportOperation for BlockImportOperatio let (root, transaction) = self.old_state.full_storage_root( storage.top.iter().map(|(k, v)| (k.as_ref(), Some(v.as_ref()))), child_delta, + false, // TODO push flag in storage ); self.new_state = Some(transaction); diff --git a/client/db/src/bench.rs b/client/db/src/bench.rs index c046c5e64545b..2b74dd139cf0a 100644 --- a/client/db/src/bench.rs +++ b/client/db/src/bench.rs @@ -58,7 +58,7 @@ impl sp_state_machine::Storage> for StorageDb, _>>::extract_value_owned(value, parent))) .map_err(|e| format!("Database backend error: {:?}", e))?; - recorder.record(key.clone(), backend_value.clone()); + recorder.record(key.clone(), backend_value.clone(), HashFor::::LENGTH); Ok(backend_value) } else { self.db.get(0, &prefixed_key) @@ -68,7 +68,7 @@ impl sp_state_machine::Storage> for StorageDb::LENGTH); } } } @@ -151,6 +151,7 @@ impl BenchmarkingState { state.add_whitelist_to_tracker(); state.reopen()?; + let flagged = false; // TODO from genesis Storage let child_delta = genesis.children_default.iter().map(|(_storage_key, child_content)| ( &child_content.child_info, child_content.data.iter().map(|(k, v)| (k.as_ref(), Some(v.as_ref()))), @@ -158,6 +159,7 @@ impl BenchmarkingState { let (root, transaction): (B::Hash, _) = state.state.borrow_mut().as_mut().unwrap().full_storage_root( genesis.top.iter().map(|(k, v)| (k.as_ref(), Some(v.as_ref()))), child_delta, + flagged, ); state.genesis = transaction.clone().drain(); state.genesis_root = root.clone(); @@ -399,8 +401,9 @@ impl StateBackend> for BenchmarkingState { fn storage_root<'a>( &self, delta: impl Iterator)>, + flag_hash_value: bool, ) -> (B::Hash, Self::Transaction) where B::Hash: Ord { - self.state.borrow().as_ref().map_or(Default::default(), |s| s.storage_root(delta)) + self.state.borrow().as_ref().map_or(Default::default(), |s| s.storage_root(delta, flag_hash_value)) } fn child_storage_root<'a>( diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index 245fc3b84f04c..ffeb9be4cd35c 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -227,8 +227,9 @@ impl StateBackend> for RefTrackingState { fn storage_root<'a>( &self, delta: impl Iterator)>, + flag_hash_value: bool, ) -> (B::Hash, Self::Transaction) where B::Hash: Ord { - self.state.storage_root(delta) + self.state.storage_root(delta, flag_hash_value) } fn child_storage_root<'a>( @@ -779,6 +780,7 @@ impl sc_client_api::backend::BlockImportOperation for Bloc )); let mut changes_trie_config: Option = None; + let flag = false; // TODO flag from storage!! let (root, transaction) = self.old_state.full_storage_root( storage.top.iter().map(|(k, v)| { if &k[..] == well_known_keys::CHANGES_TRIE_CONFIG { @@ -789,7 +791,8 @@ impl sc_client_api::backend::BlockImportOperation for Bloc } (&k[..], Some(&v[..])) }), - child_delta + child_delta, + flag, ); self.db_updates = transaction; @@ -2283,6 +2286,7 @@ pub(crate) mod tests { #[test] fn set_state_data() { + let flagged = false; // TODO test with flagged let db = Backend::::new_test(2, 0); let hash = { let mut op = db.begin_operation().unwrap(); @@ -2302,7 +2306,8 @@ pub(crate) mod tests { header.state_root = op.old_state.storage_root(storage .iter() - .map(|(x, y)| (&x[..], Some(&y[..]))) + .map(|(x, y)| (&x[..], Some(&y[..]))), + flagged, ).0.into(); let hash = header.hash(); @@ -2346,7 +2351,8 @@ pub(crate) mod tests { let (root, overlay) = op.old_state.storage_root( storage.iter() - .map(|(k, v)| (&k[..], v.as_ref().map(|v| &v[..]))) + .map(|(k, v)| (&k[..], v.as_ref().map(|v| &v[..]))), + flagged, ); op.update_db_storage(overlay).unwrap(); header.state_root = root.into(); @@ -2373,6 +2379,7 @@ pub(crate) mod tests { fn delete_only_when_negative_rc() { sp_tracing::try_init_simple(); let key; + let flagged = false; let backend = Backend::::new_test(1, 0); let hash = { @@ -2386,7 +2393,7 @@ pub(crate) mod tests { extrinsics_root: Default::default(), }; - header.state_root = op.old_state.storage_root(std::iter::empty()).0.into(); + header.state_root = op.old_state.storage_root(std::iter::empty(), flagged).0.into(); let hash = header.hash(); op.reset_storage(Storage { @@ -2426,7 +2433,8 @@ pub(crate) mod tests { header.state_root = op.old_state.storage_root(storage .iter() .cloned() - .map(|(x, y)| (x, Some(y))) + .map(|(x, y)| (x, Some(y))), + flagged, ).0.into(); let hash = header.hash(); @@ -2463,7 +2471,8 @@ pub(crate) mod tests { header.state_root = op.old_state.storage_root(storage .iter() .cloned() - .map(|(x, y)| (x, Some(y))) + .map(|(x, y)| (x, Some(y))), + flagged, ).0.into(); let hash = header.hash(); @@ -2500,7 +2509,8 @@ pub(crate) mod tests { header.state_root = op.old_state.storage_root(storage .iter() .cloned() - .map(|(x, y)| (x, Some(y))) + .map(|(x, y)| (x, Some(y))), + flagged, ).0.into(); op.set_block_data( @@ -2816,6 +2826,7 @@ pub(crate) mod tests { #[test] fn storage_hash_is_cached_correctly() { let backend = Backend::::new_test(10, 10); + let flagged = false; let hash0 = { let mut op = backend.begin_operation().unwrap(); @@ -2832,7 +2843,8 @@ pub(crate) mod tests { header.state_root = op.old_state.storage_root(storage .iter() - .map(|(x, y)| (&x[..], Some(&y[..]))) + .map(|(x, y)| (&x[..], Some(&y[..]))), + flagged, ).0.into(); let hash = header.hash(); @@ -2872,7 +2884,8 @@ pub(crate) mod tests { let (root, overlay) = op.old_state.storage_root( storage.iter() - .map(|(k, v)| (&k[..], v.as_ref().map(|v| &v[..]))) + .map(|(k, v)| (&k[..], v.as_ref().map(|v| &v[..]))), + flagged, ); op.update_db_storage(overlay).unwrap(); header.state_root = root.into(); diff --git a/client/db/src/storage_cache.rs b/client/db/src/storage_cache.rs index 8929972e26e66..eeb5092fa6c8f 100644 --- a/client/db/src/storage_cache.rs +++ b/client/db/src/storage_cache.rs @@ -645,8 +645,9 @@ impl>, B: BlockT> StateBackend> for Cachin fn storage_root<'a>( &self, delta: impl Iterator)>, + flag_hash_value: bool, ) -> (B::Hash, Self::Transaction) where B::Hash: Ord { - self.state.storage_root(delta) + self.state.storage_root(delta, flag_hash_value) } fn child_storage_root<'a>( @@ -827,8 +828,9 @@ impl>, B: BlockT> StateBackend> for Syncin fn storage_root<'a>( &self, delta: impl Iterator)>, + flag_hash_value: bool, ) -> (B::Hash, Self::Transaction) where B::Hash: Ord { - self.caching_state().storage_root(delta) + self.caching_state().storage_root(delta, flag_hash_value) } fn child_storage_root<'a>( @@ -1196,7 +1198,8 @@ mod tests { let shared = new_shared_cache::(256*1024, (0,1)); let mut backend = InMemoryBackend::::default(); - backend.insert(std::iter::once((None, vec![(key.clone(), Some(vec![1]))]))); + let flagged = false; + backend.insert(std::iter::once((None, vec![(key.clone(), Some(vec![1]))])), flagged); let mut s = CachingState::new( backend.clone(), diff --git a/client/light/src/backend.rs b/client/light/src/backend.rs index d6f86209afe9f..596a6fab61acc 100644 --- a/client/light/src/backend.rs +++ b/client/light/src/backend.rs @@ -346,7 +346,8 @@ impl BlockImportOperation for ImportOperation } let storage_update = InMemoryBackend::from(storage); - let (storage_root, _) = storage_update.full_storage_root(std::iter::empty(), child_delta); + let flag = false; // TODO flag_hash_value in Storage + let (storage_root, _) = storage_update.full_storage_root(std::iter::empty(), child_delta, flag); self.storage_update = Some(storage_update); Ok(storage_root) @@ -489,10 +490,11 @@ impl StateBackend for GenesisOrUnavailableState fn storage_root<'a>( &self, delta: impl Iterator)>, + flag_hash_value: bool, ) -> (H::Out, Self::Transaction) where H::Out: Ord { match *self { GenesisOrUnavailableState::Genesis(ref state) => - state.storage_root(delta), + state.storage_root(delta, flag_hash_value), GenesisOrUnavailableState::Unavailable => Default::default(), } } diff --git a/client/service/test/src/client/light.rs b/client/service/test/src/client/light.rs index 33bcc96d1bb63..f9daf460e8eb5 100644 --- a/client/service/test/src/client/light.rs +++ b/client/service/test/src/client/light.rs @@ -455,10 +455,11 @@ fn prepare_for_read_proof_check() -> (TestChecker, Header, StorageProof, u32) { // prepare remote client let remote_client = substrate_test_runtime_client::new(); let remote_block_id = BlockId::Number(0); + let flagged = false; let remote_block_hash = remote_client.block_hash(0).unwrap().unwrap(); let mut remote_block_header = remote_client.header(&remote_block_id).unwrap().unwrap(); remote_block_header.state_root = remote_client.state_at(&remote_block_id).unwrap() - .storage_root(::std::iter::empty()).0.into(); + .storage_root(std::iter::empty(), flagged).0.into(); // 'fetch' read proof from remote node let heap_pages = remote_client.storage(&remote_block_id, &StorageKey(well_known_keys::HEAP_PAGES.to_vec())) @@ -491,6 +492,7 @@ fn prepare_for_read_child_proof_check() -> (TestChecker, Header, StorageProof, V use substrate_test_runtime_client::TestClientBuilderExt; let child_info = ChildInfo::new_default(b"child1"); let child_info = &child_info; + let flagged = false; // prepare remote client let remote_client = substrate_test_runtime_client::TestClientBuilder::new() .add_extra_child_storage( @@ -502,7 +504,7 @@ fn prepare_for_read_child_proof_check() -> (TestChecker, Header, StorageProof, V let remote_block_hash = remote_client.block_hash(0).unwrap().unwrap(); let mut remote_block_header = remote_client.header(&remote_block_id).unwrap().unwrap(); remote_block_header.state_root = remote_client.state_at(&remote_block_id).unwrap() - .storage_root(::std::iter::empty()).0.into(); + .storage_root(std::iter::empty(), flagged).0.into(); // 'fetch' child read proof from remote node let child_value = remote_client.child_storage( diff --git a/primitives/externalities/src/lib.rs b/primitives/externalities/src/lib.rs index ce5a0990d738d..7d315812b2178 100644 --- a/primitives/externalities/src/lib.rs +++ b/primitives/externalities/src/lib.rs @@ -291,6 +291,9 @@ pub trait Externalities: ExtensionStore { fn proof_size(&self) -> Option { None } + + /// Set flag in inner state to activate hashing of values. + fn flag_hash_value(&mut self); } /// Extension for the [`Externalities`] trait. diff --git a/primitives/io/src/lib.rs b/primitives/io/src/lib.rs index af6f049f0c529..69e20347f8bf8 100644 --- a/primitives/io/src/lib.rs +++ b/primitives/io/src/lib.rs @@ -211,6 +211,12 @@ pub trait Storage { self.storage_commit_transaction() .expect("No open transaction that can be committed."); } + + /// Set flag to switch storage state + /// to internally hash its values. + fn flag_hash_value(&mut self) { + self.flag_hash_value(); + } } /// Interface for accessing the child storage for default child trie, diff --git a/primitives/state-machine/src/backend.rs b/primitives/state-machine/src/backend.rs index 1a8892f8dd141..e52fa999f149b 100644 --- a/primitives/state-machine/src/backend.rs +++ b/primitives/state-machine/src/backend.rs @@ -123,10 +123,12 @@ pub trait Backend: sp_std::fmt::Debug { /// Calculate the storage root, with given delta over what is already stored in /// the backend, and produce a "transaction" that can be used to commit. + /// A flag `flag_inner_hash_value` can be set, it switches inner trie implementation. /// Does not include child storage updates. fn storage_root<'a>( &self, delta: impl Iterator)>, + flag_inner_hash_value: bool, ) -> (H::Out, Self::Transaction) where H::Out: Ord; /// Calculate the child storage root, with given delta over what is already stored in @@ -174,6 +176,7 @@ pub trait Backend: sp_std::fmt::Debug { &'a ChildInfo, impl Iterator)>, )>, + flag_inner_hash_value: bool, ) -> (H::Out, Self::Transaction) where H::Out: Ord + Encode { let mut txs: Self::Transaction = Default::default(); let mut child_roots: Vec<_> = Default::default(); @@ -195,7 +198,8 @@ pub trait Backend: sp_std::fmt::Debug { child_roots .iter() .map(|(k, v)| (&k[..], v.as_ref().map(|v| &v[..]))) - ) + ), + flag_inner_hash_value, ); txs.consolidate(parent_txs); (root, txs) @@ -305,8 +309,9 @@ impl<'a, T: Backend, H: Hasher> Backend for &'a T { fn storage_root<'b>( &self, delta: impl Iterator)>, + flag_inner_hash_value: bool, ) -> (H::Out, Self::Transaction) where H::Out: Ord { - (*self).storage_root(delta) + (*self).storage_root(delta, flag_inner_hash_value) } fn child_storage_root<'b>( diff --git a/primitives/state-machine/src/basic.rs b/primitives/state-machine/src/basic.rs index e4806c7878ed2..5fcf349eb72fc 100644 --- a/primitives/state-machine/src/basic.rs +++ b/primitives/state-machine/src/basic.rs @@ -281,6 +281,7 @@ impl Externalities for BasicExternalities { } } + // TODO set flag on layout??? Layout::::default().trie_root(self.inner.top.clone()).as_ref().into() } @@ -332,6 +333,11 @@ impl Externalities for BasicExternalities { fn set_whitelist(&mut self, _: Vec) { unimplemented!("set_whitelist is not supported in Basic") } + + fn flag_hash_value(&mut self) { + unimplemented!("flag_hash_value is not supported in Basic") + // TODO consider flag in layout so doable by adding to storage. + } } impl sp_externalities::ExtensionStore for BasicExternalities { diff --git a/primitives/state-machine/src/ext.rs b/primitives/state-machine/src/ext.rs index 43793d3c815dc..abaa51939c914 100644 --- a/primitives/state-machine/src/ext.rs +++ b/primitives/state-machine/src/ext.rs @@ -737,6 +737,10 @@ where fn proof_size(&self) -> Option { self.backend.proof_size() } + + fn flag_hash_value(&mut self) { + self.overlay.set_flag_hash_value() + } } /// Implement `Encode` by forwarding the stored raw vec. diff --git a/primitives/state-machine/src/in_memory_backend.rs b/primitives/state-machine/src/in_memory_backend.rs index 4ee16dfd2f8a8..4e2bef05e50ab 100644 --- a/primitives/state-machine/src/in_memory_backend.rs +++ b/primitives/state-machine/src/in_memory_backend.rs @@ -45,9 +45,10 @@ where >( &self, changes: T, + flag_inner_hash_value: bool, ) -> Self { let mut clone = self.clone(); - clone.insert(changes); + clone.insert(changes, flag_inner_hash_value); clone } @@ -57,6 +58,7 @@ where >( &mut self, changes: T, + flag_inner_hash_value: bool, ) { let (top, child) = changes.into_iter().partition::, _>(|v| v.0.is_none()); let (root, transaction) = self.full_storage_root( @@ -65,6 +67,7 @@ where .filter_map(|v| v.0.as_ref().map(|c| (c, v.1.iter().map(|(k, v)| (&k[..], v.as_deref())))) ), + flag_inner_hash_value, ); self.apply_transaction(root, transaction); @@ -116,6 +119,7 @@ where let mut backend = new_in_mem(); backend.insert( inner.into_iter().map(|(k, m)| (k, m.into_iter().map(|(k, v)| (k, Some(v))).collect())), + false, ); backend } @@ -178,11 +182,13 @@ mod tests { let storage = new_in_mem::(); let child_info = ChildInfo::new_default(b"1"); let child_info = &child_info; + let flagged = false; let mut storage = storage.update( vec![( Some(child_info.clone()), vec![(b"2".to_vec(), Some(b"3".to_vec()))] - )] + )], + flagged, ); let trie_backend = storage.as_trie_backend().unwrap(); assert_eq!(trie_backend.child_storage(child_info, b"2").unwrap(), @@ -195,9 +201,10 @@ mod tests { fn insert_multiple_times_child_data_works() { let mut storage = new_in_mem::(); let child_info = ChildInfo::new_default(b"1"); + let flagged = false; - storage.insert(vec![(Some(child_info.clone()), vec![(b"2".to_vec(), Some(b"3".to_vec()))])]); - storage.insert(vec![(Some(child_info.clone()), vec![(b"1".to_vec(), Some(b"3".to_vec()))])]); + storage.insert(vec![(Some(child_info.clone()), vec![(b"2".to_vec(), Some(b"3".to_vec()))])], flagged); + storage.insert(vec![(Some(child_info.clone()), vec![(b"1".to_vec(), Some(b"3".to_vec()))])], flagged); assert_eq!(storage.child_storage(&child_info, &b"2"[..]), Ok(Some(b"3".to_vec()))); assert_eq!(storage.child_storage(&child_info, &b"1"[..]), Ok(Some(b"3".to_vec()))); diff --git a/primitives/state-machine/src/lib.rs b/primitives/state-machine/src/lib.rs index a6f1fb1f0e788..5c0074691b1c7 100644 --- a/primitives/state-machine/src/lib.rs +++ b/primitives/state-machine/src/lib.rs @@ -1060,7 +1060,8 @@ mod tests { // fetch execution proof from 'remote' full node let remote_backend = trie_backend::tests::test_trie(); - let remote_root = remote_backend.storage_root(std::iter::empty()).0; + let flagged = false; // TODO try with flagged and trie with test_trie of already flagged + let remote_root = remote_backend.storage_root(std::iter::empty(), flagged).0; let (remote_result, remote_proof) = prove_execution::<_, _, u64, _, _>( remote_backend, &mut Default::default(), @@ -1411,8 +1412,9 @@ mod tests { let child_info = ChildInfo::new_default(b"sub1"); let child_info = &child_info; // fetch read proof from 'remote' full node - let remote_backend = trie_backend::tests::test_trie(); - let remote_root = remote_backend.storage_root(::std::iter::empty()).0; + let remote_backend = trie_backend::tests::test_trie(); // TODO test with flagged and flagged. + let flagged = false; + let remote_root = remote_backend.storage_root(::std::iter::empty(), flagged).0; let remote_proof = prove_read(remote_backend, &[b"value2"]).unwrap(); // check proof locally let local_result1 = read_proof_check::( @@ -1433,7 +1435,7 @@ mod tests { assert_eq!(local_result2, false); // on child trie let remote_backend = trie_backend::tests::test_trie(); - let remote_root = remote_backend.storage_root(::std::iter::empty()).0; + let remote_root = remote_backend.storage_root(::std::iter::empty(), false).0; let remote_proof = prove_child_read( remote_backend, child_info, diff --git a/primitives/state-machine/src/overlayed_changes/mod.rs b/primitives/state-machine/src/overlayed_changes/mod.rs index 1d3cbb59ba0c1..5386192255ad3 100644 --- a/primitives/state-machine/src/overlayed_changes/mod.rs +++ b/primitives/state-machine/src/overlayed_changes/mod.rs @@ -107,6 +107,8 @@ pub struct OverlayedChanges { transaction_index_ops: Vec, /// True if extrinsics stats must be collected. collect_extrinsics: bool, + /// True if we flag inner state to store hash of values. + flag_hash_value: bool, /// Collect statistic on this execution. stats: StateMachineStats, } @@ -260,6 +262,16 @@ impl OverlayedChanges { self.collect_extrinsics = collect_extrinsics; } + /// Ask to switch state to use inner hash. + pub fn set_flag_hash_value(&mut self) { + self.flag_hash_value = true; + } + + /// Is `flag_hash_value` flag set. + pub fn flag_hash_value(&self) -> bool { + self.flag_hash_value + } + /// Returns a double-Option: None if the key is unknown (i.e. and the query should be referred /// to the backend); Some(None) if the key has been deleted. Some(Some(...)) for a key whose /// value has been set. @@ -631,7 +643,7 @@ impl OverlayedChanges { |(k, v)| (&k[..], v.value().map(|v| &v[..])) ))); - let (root, transaction) = backend.full_storage_root(delta, child_delta); + let (root, transaction) = backend.full_storage_root(delta, child_delta, self.flag_hash_value); cache.transaction = Some(transaction); cache.transaction_storage_root = Some(root); diff --git a/primitives/state-machine/src/proving_backend.rs b/primitives/state-machine/src/proving_backend.rs index 66d5b8b2acb2e..ef9e1f26c0a69 100644 --- a/primitives/state-machine/src/proving_backend.rs +++ b/primitives/state-machine/src/proving_backend.rs @@ -17,7 +17,7 @@ //! Proving state machine backend. -use std::{sync::Arc, collections::{HashMap, hash_map::Entry}}; +use std::{sync::Arc, collections::HashMap}; use parking_lot::RwLock; use codec::{Decode, Codec, Encode}; use log::debug; @@ -123,29 +123,38 @@ pub struct ProofRecorder { inner: Arc>>, } -impl ProofRecorder { +impl ProofRecorder { /// Record the given `key` => `val` combination. - pub fn record(&self, key: Hash, val: Option<(DBValue, TrieMeta)>) { + pub fn record(&self, key: Hash, mut val: Option<(DBValue, TrieMeta)>, hash_len: usize) { let mut inner = self.inner.write(); - let encoded_size = if let Entry::Vacant(entry) = inner.records.entry(key) { - let encoded_size = val.as_ref().map(Encode::encoded_size).unwrap_or(0); - -// TODO with new meta -// val.as_mut().map(|val| val.1.set_accessed_value(false)); - entry.insert(val); - encoded_size - } else { - 0 - }; - inner.encoded_size += encoded_size; + let ProofRecorderInner { encoded_size, records } = &mut *inner; + records.entry(key).or_insert_with(|| { + if let Some(val) = val.as_mut() { + val.1.set_accessed_value(false); + *encoded_size += sp_trie::estimate_entry_size(val, hash_len); + } + val + }); + } /// Record actual trie level value access. - pub fn access_from(&self, _key: &Hash) { -// TODO with new meta -// self.inner.write().entry(key[..].to_vec()) -// .and_modify(|entry| entry.1.set_accessed_value(true)); + pub fn access_from(&self, key: &Hash, hash_len: usize) { + let mut inner = self.inner.write(); + let ProofRecorderInner { encoded_size, records } = &mut *inner; + records.entry(key.clone()) + .and_modify(|entry| { + if let Some(entry) = entry.as_mut() { + if !entry.1.accessed_value() { + let old_size = sp_trie::estimate_entry_size(entry, hash_len); + entry.1.set_accessed_value(true); + let new_size = sp_trie::estimate_entry_size(entry, hash_len); + *encoded_size += new_size; + *encoded_size -= old_size; + } + } + }); } /// Returns the value at the given `key`. @@ -234,12 +243,12 @@ impl<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> TrieBackendStorage } let backend_value = self.backend.get(key, prefix, parent)?; - self.proof_recorder.record(key.clone(), backend_value.clone()); + self.proof_recorder.record(key.clone(), backend_value.clone(), H::LENGTH); Ok(backend_value) } - fn access_from(&self, _key: &H::Out) { - // access_from is mainly for proof recorder, not forwarding it. + fn access_from(&self, key: &H::Out) { + self.proof_recorder.access_from(key, H::LENGTH); } } @@ -329,8 +338,9 @@ impl<'a, S, H> Backend for ProvingBackend<'a, S, H> fn storage_root<'b>( &self, delta: impl Iterator)>, + flag_inner_hash_value: bool, ) -> (H::Out, Self::Transaction) where H::Out: Ord { - self.0.storage_root(delta) + self.0.storage_root(delta, flag_inner_hash_value) } fn child_storage_root<'b>( @@ -411,9 +421,10 @@ mod tests { let proving_backend = test_proving(&trie_backend); assert_eq!(trie_backend.storage(b"key").unwrap(), proving_backend.storage(b"key").unwrap()); assert_eq!(trie_backend.pairs(), proving_backend.pairs()); + let flagged = false; - let (trie_root, mut trie_mdb) = trie_backend.storage_root(std::iter::empty()); - let (proving_root, mut proving_mdb) = proving_backend.storage_root(std::iter::empty()); + let (trie_root, mut trie_mdb) = trie_backend.storage_root(std::iter::empty(), flagged); + let (proving_root, mut proving_mdb) = proving_backend.storage_root(std::iter::empty(), flagged); assert_eq!(trie_root, proving_root); assert_eq!(trie_mdb.drain(), proving_mdb.drain()); } @@ -422,12 +433,13 @@ mod tests { fn proof_recorded_and_checked() { let contents = (0..64).map(|i| (vec![i], Some(vec![i]))).collect::>(); let in_memory = InMemoryBackend::::default(); - let mut in_memory = in_memory.update(vec![(None, contents)]); - let in_memory_root = in_memory.storage_root(::std::iter::empty()).0; + let flagged = false; // TODO test with flag + let mut in_memory = in_memory.update(vec![(None, contents)], flagged); + let in_memory_root = in_memory.storage_root(::std::iter::empty(), flagged).0; (0..64).for_each(|i| assert_eq!(in_memory.storage(&[i]).unwrap().unwrap(), vec![i])); let trie = in_memory.as_trie_backend().unwrap(); - let trie_root = trie.storage_root(::std::iter::empty()).0; + let trie_root = trie.storage_root(::std::iter::empty(), flagged).0; assert_eq!(in_memory_root, trie_root); (0..64).for_each(|i| assert_eq!(trie.storage(&[i]).unwrap().unwrap(), vec![i])); @@ -454,11 +466,13 @@ mod tests { (10..15).map(|i| (vec![i], Some(vec![i]))).collect()), ]; let in_memory = InMemoryBackend::::default(); - let mut in_memory = in_memory.update(contents); + let flagged = false; // TODO test with flag + let mut in_memory = in_memory.update(contents, flagged); let child_storage_keys = vec![child_info_1.to_owned(), child_info_2.to_owned()]; let in_memory_root = in_memory.full_storage_root( std::iter::empty(), - child_storage_keys.iter().map(|k|(k, std::iter::empty())) + child_storage_keys.iter().map(|k|(k, std::iter::empty())), + flagged, ).0; (0..64).for_each(|i| assert_eq!( in_memory.storage(&[i]).unwrap().unwrap(), @@ -474,7 +488,7 @@ mod tests { )); let trie = in_memory.as_trie_backend().unwrap(); - let trie_root = trie.storage_root(std::iter::empty()).0; + let trie_root = trie.storage_root(std::iter::empty(), flagged).0; assert_eq!(in_memory_root, trie_root); (0..64).for_each(|i| assert_eq!( trie.storage(&[i]).unwrap().unwrap(), @@ -511,7 +525,7 @@ mod tests { } #[test] - fn storage_proof_encoded_size_estimation_works() { + fn storage_proof_encoded_size_estimation_works() { // TODO same with flag -> test_trie with flag let trie_backend = test_trie(); let backend = test_proving(&trie_backend); diff --git a/primitives/state-machine/src/read_only.rs b/primitives/state-machine/src/read_only.rs index 296520900c952..a54b6eaba6848 100644 --- a/primitives/state-machine/src/read_only.rs +++ b/primitives/state-machine/src/read_only.rs @@ -202,6 +202,10 @@ impl<'a, H: Hasher, B: 'a + Backend> Externalities for ReadOnlyExternalities< fn set_whitelist(&mut self, _: Vec) { unimplemented!("set_whitelist is not supported in ReadOnlyExternalities") } + + fn flag_hash_value(&mut self) { + unimplemented!("flag_hash_value is not supported by ReadOnlyExternalities"); + } } impl<'a, H: Hasher, B: 'a + Backend> sp_externalities::ExtensionStore for ReadOnlyExternalities<'a, H, B> { diff --git a/primitives/state-machine/src/testing.rs b/primitives/state-machine/src/testing.rs index f4b0cb6592ce2..359dfce8b77ea 100644 --- a/primitives/state-machine/src/testing.rs +++ b/primitives/state-machine/src/testing.rs @@ -139,7 +139,7 @@ where /// Insert key/value into backend pub fn insert(&mut self, k: StorageKey, v: StorageValue) { - self.backend.insert(vec![(None, vec![(k, Some(v))])]); + self.backend.insert(vec![(None, vec![(k, Some(v))])], false); } /// Registers the given extension for this instance. @@ -171,7 +171,7 @@ where )) } - self.backend.update(transaction) + self.backend.update(transaction, self.overlay.flag_hash_value()) } /// Commit all pending changes to the underlying backend. diff --git a/primitives/state-machine/src/trie_backend.rs b/primitives/state-machine/src/trie_backend.rs index 3e74f2d3df4b8..ae96d5bc97d2b 100644 --- a/primitives/state-machine/src/trie_backend.rs +++ b/primitives/state-machine/src/trie_backend.rs @@ -171,6 +171,7 @@ impl, H: Hasher> Backend for TrieBackend where fn storage_root<'a>( &self, delta: impl Iterator)>, + flag_inner_hash_value: bool, ) -> (H::Out, Self::Transaction) where H::Out: Ord { let mut write_overlay = S::Overlay::default(); let mut root = *self.essence.root(); @@ -180,7 +181,16 @@ impl, H: Hasher> Backend for TrieBackend where self.essence.backend_storage(), &mut write_overlay, ); - + if flag_inner_hash_value { + root = match sp_trie::flag_inner_meta_hasher::, _>(&mut eph, root) { + Ok(ret) => ret, + Err(e) => { + warn!(target: "trie", "Failed to flag trie: {}", e); + root + }, + } + } + match delta_trie_root::, _, _, _, _, _>(&mut eph, root, delta) { Ok(ret) => root = ret, Err(e) => warn!(target: "trie", "Failed to write to trie: {}", e), @@ -325,21 +335,31 @@ pub mod tests { #[test] fn storage_root_is_non_default() { - assert!(test_trie().storage_root(iter::empty()).0 != H256::repeat_byte(0)); + let flagged = false; + assert!(test_trie().storage_root(iter::empty(), flagged).0 != H256::repeat_byte(0)); } #[test] fn storage_root_transaction_is_empty() { - assert!(test_trie().storage_root(iter::empty()).1.drain().is_empty()); + let flagged = false; + assert!(test_trie().storage_root(iter::empty(), flagged).1.drain().is_empty()); + } + + #[test] + fn storage_root_flagged_is_not_empty() { + let flagged = true; + assert!(!test_trie().storage_root(iter::empty(), flagged).1.drain().is_empty()); } #[test] fn storage_root_transaction_is_non_empty() { + // TODO test with flagged `test_trie` (initially only). let (new_root, mut tx) = test_trie().storage_root( iter::once((&b"new-key"[..], Some(&b"new-value"[..]))), + false, ); assert!(!tx.drain().is_empty()); - assert!(new_root != test_trie().storage_root(iter::empty()).0); + assert!(new_root != test_trie().storage_root(iter::empty(), false).0); } #[test] diff --git a/primitives/tasks/src/async_externalities.rs b/primitives/tasks/src/async_externalities.rs index 5d99ca4368d0b..ff21e74889d06 100644 --- a/primitives/tasks/src/async_externalities.rs +++ b/primitives/tasks/src/async_externalities.rs @@ -189,6 +189,10 @@ impl Externalities for AsyncExternalities { fn set_whitelist(&mut self, _: Vec) { unimplemented!("set_whitelist is not supported in AsyncExternalities") } + + fn flag_hash_value(&mut self) { + unimplemented!("flag_hash_value is not supported in AsyncExternalities") + } } impl sp_externalities::ExtensionStore for AsyncExternalities { diff --git a/primitives/trie/src/lib.rs b/primitives/trie/src/lib.rs index c1615b2fa5acc..0d08ac22d2243 100644 --- a/primitives/trie/src/lib.rs +++ b/primitives/trie/src/lib.rs @@ -25,7 +25,7 @@ mod node_codec; mod storage_proof; mod trie_stream; -use sp_std::{boxed::Box, marker::PhantomData, vec::Vec, borrow::Borrow, fmt}; +use sp_std::{boxed::Box, marker::PhantomData, vec, vec::Vec, borrow::Borrow, fmt}; use hash_db::{Hasher, Prefix}; //use trie_db::proof::{generate_proof, verify_proof}; pub use trie_db::proof::VerifyError; @@ -39,7 +39,7 @@ pub use storage_proof::StorageProof; /// Various re-exports from the `trie-db` crate. pub use trie_db::{ Trie, TrieMut, DBValue, Recorder, CError, Query, TrieLayout, TrieConfiguration, - nibble_ops, TrieDBIterator, Meta, + nibble_ops, TrieDBIterator, Meta, NodeChange, node::{NodePlan, ValuePlan}, }; /// Various re-exports from the `memory-db` crate. pub use memory_db::KeyFunction; @@ -49,7 +49,146 @@ pub use hash_db::{HashDB as HashDBT, EMPTY_PREFIX, MetaHasher}; pub use hash_db::NoMeta; /// Meta use by trie state. -pub type TrieMeta = (); +#[derive(Default, Clone)] +pub struct TrieMeta { + // range of encoded value or hashed value. + pub range: Option>, + // When `do_value_hash` is true, try to + // store this behavior in top node + // encoded (need to be part of state). + pub recorded_do_value_hash: bool, + // Does current encoded contains a hash instead of + // a value (information stored in meta for proofs). + pub contain_hash: bool, + // Flag indicating if value hash can run. + // When defined for a node it gets active + // for all children node + pub do_value_hash: bool, + // Record if a value was accessed, this is + // set as accessed by defalult, but can be + // change on access explicitely: `HashDB::get_with_meta`. + // and reset on access explicitely: `HashDB::access_from`. + pub unused_value: bool, +} + +impl Meta for TrieMeta { + /// Layout do not have content. + type MetaInput = (); + + /// When true apply inner hashing of value. + type StateMeta = bool; + + fn set_state_meta(&mut self, state_meta: Self::StateMeta) { + self.recorded_do_value_hash = state_meta; + self.do_value_hash = state_meta; + } + + fn has_state_meta(&self) -> bool { + self.recorded_do_value_hash + } + + fn read_state_meta(&mut self, data: &[u8]) -> Result { + let offset = if data[0] == trie_constants::ENCODED_META_ALLOW_HASH { + self.recorded_do_value_hash = true; + self.do_value_hash = true; + 1 + } else { + 0 + }; + Ok(offset) + } + + fn write_state_meta(&self) -> Vec { + if self.do_value_hash { + // Note that this only works with sp_trie codec that + // cannot encode node starting by this byte. + [trie_constants::ENCODED_META_ALLOW_HASH].to_vec() + } else { + Vec::new() + } + } + + fn meta_for_new( + _input: Self::MetaInput, + parent: Option<&Self>, + ) -> Self { + let mut result = Self::default(); + result.do_value_hash = parent.map(|p| p.do_value_hash).unwrap_or_default(); + result + } + + fn meta_for_existing_inline_node( + input: Self::MetaInput, + parent: Option<&Self>, + ) -> Self { + Self::meta_for_new(input, parent) + } + + fn meta_for_empty( + ) -> Self { + Default::default() + } + + fn set_value_callback( + &mut self, + _new_value: Option<&[u8]>, + _is_branch: bool, + changed: NodeChange, + ) -> NodeChange { + changed + } + + fn encoded_value_callback( + &mut self, + value_plan: ValuePlan, + ) { + let (contain_hash, range) = match value_plan { + ValuePlan::Value(range) => (false, range), + ValuePlan::HashedValue(range, _size) => (true, range), + ValuePlan::NoValue => return, + }; + + self.range = Some(range); + self.contain_hash = contain_hash; + } + + fn set_child_callback( + &mut self, + _child: Option<&Self>, + changed: NodeChange, + _at: usize, + ) -> NodeChange { + changed + } + + fn decoded_callback( + &mut self, + _node_plan: &NodePlan, + ) { + } + + fn contains_hash_of_value(&self) -> bool { + self.contain_hash + } + + fn do_value_hash(&self) -> bool { + self.unused_value + } +} + +impl TrieMeta { + /// Was value accessed. + pub fn accessed_value(&mut self) -> bool { + !self.unused_value + } + + /// For proof, this allow setting node as unaccessed until + /// a call to `access_from`. + pub fn set_accessed_value(&mut self, accessed: bool) { + self.unused_value = !accessed; + } +} + /// substrate trie layout pub struct Layout(sp_std::marker::PhantomData); @@ -74,6 +213,7 @@ impl Clone for Layout { impl TrieLayout for Layout { const USE_EXTENSION: bool = false; const ALLOW_EMPTY: bool = true; + const USE_META: bool = true; type Hash = H; type Codec = NodeCodec; type MetaHasher = StateHasher; @@ -92,31 +232,90 @@ impl TrieLayout for Layout { #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub struct StateHasher; -impl MetaHasher for StateHasher +impl MetaHasher for StateHasher where H: Hasher, - T: for<'a> From<&'a [u8]>, { type Meta = TrieMeta; - fn hash(value: &[u8], _meta: &Self::Meta) -> H::Out { - H::hash(value) + fn hash(value: &[u8], meta: &Self::Meta) -> H::Out { + match &meta { + TrieMeta { range: Some(range), contain_hash: false, do_value_hash, .. } => { + if *do_value_hash { + let value = inner_hashed_value::(value, Some((range.start, range.end))); + H::hash(value.as_slice()) + } else { + H::hash(value) + } + }, + TrieMeta { range: Some(_range), contain_hash: true, .. } => { + // value contains a hash of data (already inner_hashed_value). + H::hash(value) + }, + _ => { + H::hash(value) + }, + } } - fn stored_value(value: &[u8], _meta: Self::Meta) -> T { - value.into() + fn stored_value(value: &[u8], mut meta: Self::Meta) -> DBValue { + let mut stored = Vec::with_capacity(value.len() + 1); + if meta.contain_hash { + // already contain hash, just flag it. + stored.push(trie_constants::DEAD_HEADER_META_HASHED_VALUE); + stored.extend_from_slice(value); + return stored; + } + if meta.unused_value { + if let Some(range) = meta.range.as_ref() { + if range.end - range.start >= trie_constants::INNER_HASH_TRESHOLD { + // Waring this assume that encoded value does not start by this, so it is tightly coupled + // with the header type of the codec: only for optimization. + stored.push(trie_constants::DEAD_HEADER_META_HASHED_VALUE); + let range = meta.range.as_ref().expect("Tested in condition"); + meta.contain_hash = true; // useless but could be with meta as &mut + // store hash instead of value. + let value = inner_hashed_value::(value, Some((range.start, range.end))); + stored.extend_from_slice(value.as_slice()); + return stored; + } + } + } + stored.extend_from_slice(value); + stored } - fn stored_value_owned(value: T, _meta: Self::Meta) -> T { - value + fn stored_value_owned(value: DBValue, meta: Self::Meta) -> DBValue { + >::stored_value(value.as_slice(), meta) } - fn extract_value<'a>(stored: &'a [u8], _parent_meta: Option<&Self::Meta>) -> (&'a [u8], Self::Meta) { - (stored, ()) + fn extract_value<'a>(mut stored: &'a [u8], parent_meta: Option<&Self::Meta>) -> (&'a [u8], Self::Meta) { + let input = &mut stored; + let mut contain_hash = false; + if input.get(0) == Some(&trie_constants::DEAD_HEADER_META_HASHED_VALUE) { + contain_hash = true; + *input = &input[1..]; + } + let mut meta = TrieMeta { + range: None, + unused_value: contain_hash, + contain_hash, + do_value_hash: false, + recorded_do_value_hash: false, + }; + // get recorded_do_value_hash + let _offset = meta.read_state_meta(stored) + .expect("State meta reading failure."); + //let stored = &stored[offset..]; + meta.do_value_hash = meta.recorded_do_value_hash || parent_meta.map(|m| m.do_value_hash).unwrap_or(false); + (stored, meta) } - fn extract_value_owned(stored: T, _parent_meta: Option<&Self::Meta>) -> (T, Self::Meta) { - (stored, ()) + fn extract_value_owned(mut stored: DBValue, parent_meta: Option<&Self::Meta>) -> (DBValue, Self::Meta) { + let len = stored.len(); + let (v, meta) = >::extract_value(stored.as_slice(), parent_meta); + let removed = len - v.len(); + (stored.split_off(removed), meta) } } @@ -270,6 +469,26 @@ pub fn delta_trie_root( Ok(root) } +/// Flag inner trie with state metadata to enable hash of value internally. +pub fn flag_inner_meta_hasher( + db: &mut DB, + mut root: TrieHash, +) -> Result, Box>> where + L: TrieConfiguration, + DB: hash_db::HashDB, +{ + { + let mut t = TrieDBMut::::from_existing(db, &mut root)?; + let flag = true; + let key: &[u8]= &[]; + if !t.flag(key, flag)? { + t.insert(key, b"")?; + assert!(t.flag(key, flag)?); + } + } + Ok(root) +} + /// Read a value from the trie. pub fn read_trie_value>( db: &DB, @@ -558,9 +777,70 @@ impl<'a, DB, H, T, M> hash_db::AsHashDB for KeySpacedDBMut<'a, DB, H> w } } +/// Representation of node with with inner hash instead of value. +pub fn inner_hashed_value(x: &[u8], range: Option<(usize, usize)>) -> Vec { + if let Some((start, end)) = range { + let len = x.len(); + if start < len && end == len { + // terminal inner hash + let hash_end = H::hash(&x[start..]); + let mut buff = vec![0; x.len() + hash_end.as_ref().len() - (end - start)]; + buff[..start].copy_from_slice(&x[..start]); + buff[start..].copy_from_slice(hash_end.as_ref()); + return buff; + } + if start == 0 && end < len { + // start inner hash + let hash_start = H::hash(&x[..start]); + let hash_len = hash_start.as_ref().len(); + let mut buff = vec![0; x.len() + hash_len - (end - start)]; + buff[..hash_len].copy_from_slice(hash_start.as_ref()); + buff[hash_len..].copy_from_slice(&x[end..]); + return buff; + } + if start < len && end < len { + // middle inner hash + let hash_middle = H::hash(&x[start..end]); + let hash_len = hash_middle.as_ref().len(); + let mut buff = vec![0; x.len() + hash_len - (end - start)]; + buff[..start].copy_from_slice(&x[..start]); + buff[start..start + hash_len].copy_from_slice(hash_middle.as_ref()); + buff[start + hash_len..].copy_from_slice(&x[end..]); + return buff; + } + } + // if anything wrong default to hash + x.to_vec() +} + +/// Estimate encoded size of node. +pub fn estimate_entry_size(entry: &(DBValue, TrieMeta), hash_len: usize) -> usize { + use codec::Encode; + let mut full_encoded = entry.0.encoded_size(); + if entry.1.unused_value { + if let Some(range) = entry.1.range.as_ref() { + let value_size = range.end - range.start; + if range.end - range.start >= trie_constants::INNER_HASH_TRESHOLD { + full_encoded -= value_size; + full_encoded += hash_len; + full_encoded += 1; + } + } + } + + full_encoded +} + /// Constants used into trie simplification codec. mod trie_constants { - pub const EMPTY_TRIE: u8 = 0; + /// Treshold for using hash of value instead of value + /// in encoded trie node when flagged. + pub const INNER_HASH_TRESHOLD: usize = 33; + const FIRST_PREFIX: u8 = 0b_00 << 6; + pub const EMPTY_TRIE: u8 = FIRST_PREFIX | 0b_00; + pub const ENCODED_META_ALLOW_HASH: u8 = FIRST_PREFIX | 0b_01; + /// In proof this header is used when only hashed value is stored. + pub const DEAD_HEADER_META_HASHED_VALUE: u8 = FIRST_PREFIX | 0b_00_10; pub const NIBBLE_SIZE_BOUND: usize = u16::max_value() as usize; pub const LEAF_PREFIX_MASK: u8 = 0b_01 << 6; pub const BRANCH_WITHOUT_MASK: u8 = 0b_10 << 6; diff --git a/primitives/trie/src/node_codec.rs b/primitives/trie/src/node_codec.rs index 11b379fce6f2d..b0b7c771c6b55 100644 --- a/primitives/trie/src/node_codec.rs +++ b/primitives/trie/src/node_codec.rs @@ -87,16 +87,24 @@ impl<'a> Input for ByteSliceInput<'a> { #[derive(Default, Clone)] pub struct NodeCodec(PhantomData); -impl NodeCodecT for NodeCodec { - type Error = Error; - type HashOut = H::Out; - - fn hashed_null_node() -> ::Out { - H::hash(>::empty_node()) - } - - fn decode_plan_inner(data: &[u8]) -> sp_std::result::Result { +impl NodeCodec { + fn decode_plan_inner_hashed( + data: &[u8], + meta: Option<&mut M>, + ) -> Result { + let contains_hash = meta.as_ref() + .map(|m| m.contains_hash_of_value()).unwrap_or_default(); + if data.len() < 1 { + return Err(Error::BadFormat); + } + let offset = if let Some(meta) = meta { + meta.read_state_meta(data).map_err(|_| Error::BadFormat)? + } else { + 0 + }; let mut input = ByteSliceInput::new(data); + let _ = input.take(offset)?; + match NodeHeader::decode(&mut input)? { NodeHeader::Null => Ok(NodePlan::Empty), NodeHeader::Branch(has_value, nibble_count) => { @@ -113,7 +121,11 @@ impl NodeCodecT for NodeCodec { let bitmap = Bitmap::decode(&data[bitmap_range])?; let value = if has_value { let count = >::decode(&mut input)?.0 as usize; - ValuePlan::Value(input.take(count)?) + if contains_hash { + ValuePlan::HashedValue(input.take(H::LENGTH)?, count) + } else { + ValuePlan::Value(input.take(count)?) + } } else { ValuePlan::NoValue }; @@ -149,13 +161,40 @@ impl NodeCodecT for NodeCodec { )?; let partial_padding = nibble_ops::number_padding(nibble_count); let count = >::decode(&mut input)?.0 as usize; + let value = if contains_hash { + ValuePlan::HashedValue(input.take(H::LENGTH)?, count) + } else { + ValuePlan::Value(input.take(count)?) + }; + Ok(NodePlan::Leaf { partial: NibbleSlicePlan::new(partial, partial_padding), - value: ValuePlan::Value(input.take(count)?), + value, }) } } } +} + +impl NodeCodecT for NodeCodec { + type Error = Error; + type HashOut = H::Out; + + fn hashed_null_node() -> ::Out { + H::hash(>::empty_node()) + } + + fn decode_plan(data: &[u8], meta: &mut M) -> Result { + Self::decode_plan_inner_hashed(data, Some(meta)).map(|plan| { + meta.decoded_callback(&plan); + plan + }) + } + + fn decode_plan_inner(data: &[u8]) -> Result { + let meta: Option<&mut M> = None; + Self::decode_plan_inner_hashed(data, meta) + } fn is_empty_node(data: &[u8]) -> bool { data == >::empty_node() @@ -166,15 +205,25 @@ impl NodeCodecT for NodeCodec { } fn leaf_node(partial: Partial, value: Value, meta: &mut M) -> Vec { - let mut output = partial_encode(partial, NodeKind::Leaf); - if let Value::Value(value) = value { - Compact(value.len() as u32).encode_to(&mut output); - let start = output.len(); - output.extend_from_slice(value); - let end = output.len(); - meta.encoded_value_callback(ValuePlan::Value(start..end)); - } else { - unimplemented!("No support for incomplete nodes"); + let mut output = meta.write_state_meta(); + output.append(&mut partial_encode(partial, NodeKind::Leaf)); + match value { + Value::Value(value) => { + Compact(value.len() as u32).encode_to(&mut output); + let start = output.len(); + output.extend_from_slice(value); + let end = output.len(); + meta.encoded_value_callback(ValuePlan::Value(start..end)); + }, + Value::HashedValue(hash, size) => { + debug_assert!(hash.len() == H::LENGTH); + Compact(size as u32).encode_to(&mut output); + let start = output.len(); + output.extend_from_slice(hash); + let end = output.len(); + meta.encoded_value_callback(ValuePlan::HashedValue(start..end, size)); + }, + Value::NoValue => unimplemented!("No support for incomplete nodes"), } output } @@ -203,11 +252,20 @@ impl NodeCodecT for NodeCodec { maybe_value: Value, meta: &mut M, ) -> Vec { - let mut output = match maybe_value { - Value::Value(..) => partial_from_iterator_encode(partial, number_nibble, NodeKind::BranchWithValue), - Value::NoValue => partial_from_iterator_encode(partial, number_nibble, NodeKind::BranchNoValue), - Value::HashedValue(..) => unimplemented!("No support for incomplete nodes"), - }; + let mut output = meta.write_state_meta(); + output.append(&mut if let Value::NoValue = &maybe_value { + partial_from_iterator_encode( + partial, + number_nibble, + NodeKind::BranchNoValue, + ) + } else { + partial_from_iterator_encode( + partial, + number_nibble, + NodeKind::BranchWithValue, + ) + }); let bitmap_index = output.len(); let mut bitmap: [u8; BITMAP_LENGTH] = [0; BITMAP_LENGTH]; (0..BITMAP_LENGTH).for_each(|_|output.push(0)); @@ -219,8 +277,15 @@ impl NodeCodecT for NodeCodec { let end = output.len(); meta.encoded_value_callback(ValuePlan::Value(start..end)); }, + Value::HashedValue(hash, size) => { + debug_assert!(hash.len() == H::LENGTH); + Compact(size as u32).encode_to(&mut output); + let start = output.len(); + output.extend_from_slice(hash); + let end = output.len(); + meta.encoded_value_callback(ValuePlan::HashedValue(start..end, size)); + }, Value::NoValue => (), - Value::HashedValue(..) => unimplemented!("No support for incomplete nodes"), } Bitmap::encode(children.map(|maybe_child| match maybe_child.borrow() { Some(ChildReference::Hash(h)) => { From 5d3a1b209626c5a7d27bcb89454f62f4311acb21 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 18 May 2021 17:05:34 +0200 Subject: [PATCH 004/127] flag in storage struct --- client/api/src/in_mem.rs | 2 +- client/chain-spec/src/chain_spec.rs | 11 ++++++++--- client/db/src/bench.rs | 2 +- client/db/src/lib.rs | 5 ++++- client/executor/src/integration_tests/mod.rs | 2 ++ client/light/src/backend.rs | 2 +- client/service/src/chain_ops/export_raw_state.rs | 3 ++- frame/support/test/tests/instance.rs | 3 ++- frame/system/src/lib.rs | 1 + primitives/io/src/lib.rs | 3 +++ primitives/state-machine/src/basic.rs | 12 +++++++----- primitives/state-machine/src/ext.rs | 6 +++++- primitives/storage/src/lib.rs | 2 ++ test-utils/runtime/src/genesismap.rs | 4 +++- test-utils/runtime/src/system.rs | 1 + 15 files changed, 43 insertions(+), 16 deletions(-) diff --git a/client/api/src/in_mem.rs b/client/api/src/in_mem.rs index 3f2a26a2c55c8..3dbcd4aa897de 100644 --- a/client/api/src/in_mem.rs +++ b/client/api/src/in_mem.rs @@ -576,7 +576,7 @@ impl backend::BlockImportOperation for BlockImportOperatio let (root, transaction) = self.old_state.full_storage_root( storage.top.iter().map(|(k, v)| (k.as_ref(), Some(v.as_ref()))), child_delta, - false, // TODO push flag in storage + storage.flag_hashed_value, ); self.new_state = Some(transaction); diff --git a/client/chain-spec/src/chain_spec.rs b/client/chain-spec/src/chain_spec.rs index 2faf95568290e..3838c5dbc0def 100644 --- a/client/chain-spec/src/chain_spec.rs +++ b/client/chain-spec/src/chain_spec.rs @@ -87,7 +87,9 @@ impl GenesisSource { ) .collect(); - Ok(Genesis::Raw(RawGenesis { top, children_default })) + let flag_hashed_value = storage.flag_hashed_value; + + Ok(Genesis::Raw(RawGenesis { top, children_default, flag_hashed_value })) }, } } @@ -97,7 +99,7 @@ impl BuildStorage for ChainSpec { fn build_storage(&self) -> Result { match self.genesis.resolve()? { Genesis::Runtime(gc) => gc.build_storage(), - Genesis::Raw(RawGenesis { top: map, children_default: children_map }) => Ok(Storage { + Genesis::Raw(RawGenesis { top: map, children_default: children_map, flag_hashed_value }) => Ok(Storage { top: map.into_iter().map(|(k, v)| (k.0, v.0)).collect(), children_default: children_map.into_iter().map(|(storage_key, child_content)| { let child_info = ChildInfo::new_default(storage_key.0.as_slice()); @@ -109,6 +111,7 @@ impl BuildStorage for ChainSpec { }, ) }).collect(), + flag_hashed_value, }), } } @@ -130,6 +133,7 @@ pub type GenesisStorage = HashMap; pub struct RawGenesis { pub top: GenesisStorage, pub children_default: HashMap, + pub flag_hashed_value: bool, } #[derive(Serialize, Deserialize)] @@ -316,8 +320,9 @@ impl ChainSpec { .collect(), )) .collect(); + let flag_hashed_value = storage.flag_hashed_value; - Genesis::Raw(RawGenesis { top, children_default }) + Genesis::Raw(RawGenesis { top, children_default, flag_hashed_value }) }, (_, genesis) => genesis, }; diff --git a/client/db/src/bench.rs b/client/db/src/bench.rs index 2b74dd139cf0a..bc5ab72dee66e 100644 --- a/client/db/src/bench.rs +++ b/client/db/src/bench.rs @@ -151,7 +151,7 @@ impl BenchmarkingState { state.add_whitelist_to_tracker(); state.reopen()?; - let flagged = false; // TODO from genesis Storage + let flagged = genesis.flag_hashed_value; let child_delta = genesis.children_default.iter().map(|(_storage_key, child_content)| ( &child_content.child_info, child_content.data.iter().map(|(k, v)| (k.as_ref(), Some(v.as_ref()))), diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index ffeb9be4cd35c..ce45c298e9de3 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -780,7 +780,7 @@ impl sc_client_api::backend::BlockImportOperation for Bloc )); let mut changes_trie_config: Option = None; - let flag = false; // TODO flag from storage!! + let flag = storage.flag_hashed_value; let (root, transaction) = self.old_state.full_storage_root( storage.top.iter().map(|(k, v)| { if &k[..] == well_known_keys::CHANGES_TRIE_CONFIG { @@ -2314,6 +2314,7 @@ pub(crate) mod tests { op.reset_storage(Storage { top: storage.into_iter().collect(), children_default: Default::default(), + flag_hashed_value: flagged, }).unwrap(); op.set_block_data( header.clone(), @@ -2399,6 +2400,7 @@ pub(crate) mod tests { op.reset_storage(Storage { top: Default::default(), children_default: Default::default(), + flag_hashed_value: flagged, }).unwrap(); key = op.db_updates.insert(EMPTY_PREFIX, b"hello"); @@ -2851,6 +2853,7 @@ pub(crate) mod tests { op.reset_storage(Storage { top: storage.into_iter().collect(), children_default: Default::default(), + flag_hashed_value: flagged, }).unwrap(); op.set_block_data( header.clone(), diff --git a/client/executor/src/integration_tests/mod.rs b/client/executor/src/integration_tests/mod.rs index ecb48da91490a..2b76825ea51bc 100644 --- a/client/executor/src/integration_tests/mod.rs +++ b/client/executor/src/integration_tests/mod.rs @@ -210,6 +210,7 @@ fn storage_should_work(wasm_method: WasmExecutionMethod) { b"baz".to_vec() => b"bar".to_vec() ], children_default: map![], + flag_hashed_value: false, }); assert_eq!(ext, expected); } @@ -243,6 +244,7 @@ fn clear_prefix_should_work(wasm_method: WasmExecutionMethod) { b"bbb".to_vec() => b"5".to_vec() ], children_default: map![], + flag_hashed_value: false, }); assert_eq!(expected, ext); } diff --git a/client/light/src/backend.rs b/client/light/src/backend.rs index 596a6fab61acc..49c861411bb45 100644 --- a/client/light/src/backend.rs +++ b/client/light/src/backend.rs @@ -346,7 +346,7 @@ impl BlockImportOperation for ImportOperation } let storage_update = InMemoryBackend::from(storage); - let flag = false; // TODO flag_hash_value in Storage + let flag = input.flag_hashed_value; let (storage_root, _) = storage_update.full_storage_root(std::iter::empty(), child_delta, flag); self.storage_update = Some(storage_update); diff --git a/client/service/src/chain_ops/export_raw_state.rs b/client/service/src/chain_ops/export_raw_state.rs index 71822cf6275f8..bdba4911f8db6 100644 --- a/client/service/src/chain_ops/export_raw_state.rs +++ b/client/service/src/chain_ops/export_raw_state.rs @@ -42,6 +42,7 @@ where let empty_key = StorageKey(Vec::new()); let mut top_storage = client.storage_pairs(&block, &empty_key)?; let mut children_default = HashMap::new(); + let flag_hashed_value = false; // TODO read from trie (through clinet api). // Remove all default child storage roots from the top storage and collect the child storage // pairs. @@ -69,5 +70,5 @@ where } let top = top_storage.into_iter().map(|(k, v)| (k.0, v.0)).collect(); - Ok(Storage { top, children_default }) + Ok(Storage { top, children_default, flag_hashed_value }) } diff --git a/frame/support/test/tests/instance.rs b/frame/support/test/tests/instance.rs index 077763ac9128d..e698f989de45f 100644 --- a/frame/support/test/tests/instance.rs +++ b/frame/support/test/tests/instance.rs @@ -326,7 +326,8 @@ fn new_test_ext() -> sp_io::TestExternalities { fn storage_instance_independence() { let mut storage = sp_core::storage::Storage { top: std::collections::BTreeMap::new(), - children_default: std::collections::HashMap::new() + children_default: std::collections::HashMap::new(), + flag_hashed_value: false, // TODO test with }; sp_state_machine::BasicExternalities::execute_with_storage(&mut storage, || { module2::Value::::put(0); diff --git a/frame/system/src/lib.rs b/frame/system/src/lib.rs index c3fe688420097..44e7a3d5752fe 100644 --- a/frame/system/src/lib.rs +++ b/frame/system/src/lib.rs @@ -1408,6 +1408,7 @@ impl Pallet { >::hashed_key().to_vec() => [69u8; 32].encode() ], children_default: map![], + flag_hashed_value: false, }) } diff --git a/primitives/io/src/lib.rs b/primitives/io/src/lib.rs index 69e20347f8bf8..03bfd73772724 100644 --- a/primitives/io/src/lib.rs +++ b/primitives/io/src/lib.rs @@ -1454,6 +1454,7 @@ mod tests { t = BasicExternalities::new(Storage { top: map![b"foo".to_vec() => b"bar".to_vec()], children_default: map![], + flag_hashed_value: false, }); t.execute_with(|| { @@ -1468,6 +1469,7 @@ mod tests { let mut t = BasicExternalities::new(Storage { top: map![b":test".to_vec() => value.clone()], children_default: map![], + flag_hashed_value: false, }); t.execute_with(|| { @@ -1490,6 +1492,7 @@ mod tests { b":abdd".to_vec() => b"\x0b\0\0\0Hello world".to_vec() ], children_default: map![], + flag_hashed_value: false, }); t.execute_with(|| { diff --git a/primitives/state-machine/src/basic.rs b/primitives/state-machine/src/basic.rs index 5fcf349eb72fc..950b6d715b18b 100644 --- a/primitives/state-machine/src/basic.rs +++ b/primitives/state-machine/src/basic.rs @@ -74,6 +74,7 @@ impl BasicExternalities { inner: Storage { top: std::mem::take(&mut storage.top), children_default: std::mem::take(&mut storage.children_default), + flag_hashed_value: storage.flag_hashed_value, }, extensions: Default::default(), }; @@ -128,6 +129,7 @@ impl From> for BasicExternalities { inner: Storage { top: hashmap, children_default: Default::default(), + flag_hashed_value: false, }, extensions: Default::default(), } @@ -335,8 +337,7 @@ impl Externalities for BasicExternalities { } fn flag_hash_value(&mut self) { - unimplemented!("flag_hash_value is not supported in Basic") - // TODO consider flag in layout so doable by adding to storage. + self.inner.flag_hashed_value = true; } } @@ -402,7 +403,8 @@ mod tests { data: map![ b"doe".to_vec() => b"reindeer".to_vec() ], child_info: child_info.to_owned(), } - ] + ], + flag_hashed_value: false, }); assert_eq!(ext.child_storage(child_info, b"doe"), Some(b"reindeer".to_vec())); @@ -432,10 +434,10 @@ mod tests { ], child_info: child_info.to_owned(), } - ] + ], + flag_hashed_value: false, }); - let res = ext.kill_child_storage(child_info, None); assert_eq!(res, (true, 3)); } diff --git a/primitives/state-machine/src/ext.rs b/primitives/state-machine/src/ext.rs index abaa51939c914..0c20a186a989e 100644 --- a/primitives/state-machine/src/ext.rs +++ b/primitives/state-machine/src/ext.rs @@ -950,7 +950,8 @@ mod tests { vec![20] => vec![20], vec![40] => vec![40] ], - children_default: map![] + children_default: map![], + flag_hashed_value: false, }.into(); let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); @@ -996,6 +997,7 @@ mod tests { child_info: child_info.to_owned(), } ], + flag_hashed_value: false, }.into(); let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); @@ -1040,6 +1042,7 @@ mod tests { child_info: child_info.to_owned(), } ], + flag_hashed_value: false, }.into(); let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); @@ -1079,6 +1082,7 @@ mod tests { child_info: child_info.to_owned(), } ], + flag_hashed_value: false, }.into(); let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); diff --git a/primitives/storage/src/lib.rs b/primitives/storage/src/lib.rs index ced8d8c02a80b..5c8028e40c2a8 100644 --- a/primitives/storage/src/lib.rs +++ b/primitives/storage/src/lib.rs @@ -140,6 +140,8 @@ pub struct Storage { /// trie kind, so this is exclusively for the `ChildType::ParentKeyId` /// tries. pub children_default: std::collections::HashMap, StorageChild>, + /// Flag state for using hash of values internally. + pub flag_hashed_value: bool, } /// Storage change set diff --git a/test-utils/runtime/src/genesismap.rs b/test-utils/runtime/src/genesismap.rs index 63c4bab55ec49..93b10e21e71d9 100644 --- a/test-utils/runtime/src/genesismap.rs +++ b/test-utils/runtime/src/genesismap.rs @@ -74,8 +74,10 @@ impl GenesisConfig { // Add the extra storage entries. map.extend(self.extra_storage.top.clone().into_iter()); + let flag_hashed_value = self.extra_storage.flag_hashed_value; + // Assimilate the system genesis config. - let mut storage = Storage { top: map, children_default: self.extra_storage.children_default.clone()}; + let mut storage = Storage { top: map, children_default: self.extra_storage.children_default.clone(), flag_hashed_value}; let mut config = system::GenesisConfig::default(); config.authorities = self.authorities.clone(); config.assimilate_storage(&mut storage).expect("Adding `system::GensisConfig` to the genesis"); diff --git a/test-utils/runtime/src/system.rs b/test-utils/runtime/src/system.rs index 33ef7b12d8db0..fbf6012d083e7 100644 --- a/test-utils/runtime/src/system.rs +++ b/test-utils/runtime/src/system.rs @@ -385,6 +385,7 @@ mod tests { } ], children_default: map![], + flag_hashed_value: false, // TODO test with true variant }, ) } From 4256a407a495f7e8f7590d87b91ebe398cf20f95 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 18 May 2021 18:17:47 +0200 Subject: [PATCH 005/127] fix flagging to access and insert. --- primitives/state-machine/src/lib.rs | 41 ++++++++--- .../state-machine/src/proving_backend.rs | 41 ++++++++--- primitives/state-machine/src/trie_backend.rs | 70 ++++++++++++++----- primitives/trie/src/lib.rs | 22 ++++-- 4 files changed, 130 insertions(+), 44 deletions(-) diff --git a/primitives/state-machine/src/lib.rs b/primitives/state-machine/src/lib.rs index 5c0074691b1c7..8b12c8aa030e6 100644 --- a/primitives/state-machine/src/lib.rs +++ b/primitives/state-machine/src/lib.rs @@ -959,7 +959,11 @@ mod tests { #[test] fn execute_works() { - let backend = trie_backend::tests::test_trie(); + execute_works_inner(false); + execute_works_inner(true); + } + fn execute_works_inner(hashed: bool) { + let backend = trie_backend::tests::test_trie(hashed); let mut overlayed_changes = Default::default(); let wasm_code = RuntimeCode::empty(); @@ -986,10 +990,13 @@ mod tests { ); } - #[test] fn execute_works_with_native_else_wasm() { - let backend = trie_backend::tests::test_trie(); + execute_works_with_native_else_wasm_inner(false); + execute_works_with_native_else_wasm_inner(true); + } + fn execute_works_with_native_else_wasm_inner(hashed: bool) { + let backend = trie_backend::tests::test_trie(hashed); let mut overlayed_changes = Default::default(); let wasm_code = RuntimeCode::empty(); @@ -1015,8 +1022,12 @@ mod tests { #[test] fn dual_execution_strategy_detects_consensus_failure() { + dual_execution_strategy_detects_consensus_failure_inner(false); + dual_execution_strategy_detects_consensus_failure_inner(true); + } + fn dual_execution_strategy_detects_consensus_failure_inner(hashed: bool) { let mut consensus_failed = false; - let backend = trie_backend::tests::test_trie(); + let backend = trie_backend::tests::test_trie(hashed); let mut overlayed_changes = Default::default(); let wasm_code = RuntimeCode::empty(); @@ -1051,6 +1062,10 @@ mod tests { #[test] fn prove_execution_and_proof_check_works() { + prove_execution_and_proof_check_works_inner(true); + prove_execution_and_proof_check_works_inner(false); + } + fn prove_execution_and_proof_check_works_inner(flagged: bool) { let executor = DummyCodeExecutor { change_changes_trie_config: false, native_available: true, @@ -1059,9 +1074,10 @@ mod tests { }; // fetch execution proof from 'remote' full node - let remote_backend = trie_backend::tests::test_trie(); - let flagged = false; // TODO try with flagged and trie with test_trie of already flagged + let remote_backend = trie_backend::tests::test_trie(flagged); let remote_root = remote_backend.storage_root(std::iter::empty(), flagged).0; + let remote_root_2 = remote_backend.storage_root(std::iter::empty(), false).0; + assert_eq!(remote_root, remote_root_2); let (remote_result, remote_proof) = prove_execution::<_, _, u64, _, _>( remote_backend, &mut Default::default(), @@ -1409,11 +1425,14 @@ mod tests { #[test] fn prove_read_and_proof_check_works() { + prove_read_and_proof_check_works_inner(false); + prove_read_and_proof_check_works_inner(true); + } + fn prove_read_and_proof_check_works_inner(flagged: bool) { let child_info = ChildInfo::new_default(b"sub1"); let child_info = &child_info; // fetch read proof from 'remote' full node - let remote_backend = trie_backend::tests::test_trie(); // TODO test with flagged and flagged. - let flagged = false; + let remote_backend = trie_backend::tests::test_trie(flagged); let remote_root = remote_backend.storage_root(::std::iter::empty(), flagged).0; let remote_proof = prove_read(remote_backend, &[b"value2"]).unwrap(); // check proof locally @@ -1434,7 +1453,7 @@ mod tests { ); assert_eq!(local_result2, false); // on child trie - let remote_backend = trie_backend::tests::test_trie(); + let remote_backend = trie_backend::tests::test_trie(flagged); let remote_root = remote_backend.storage_root(::std::iter::empty(), false).0; let remote_proof = prove_child_read( remote_backend, @@ -1473,7 +1492,7 @@ mod tests { let mut overlay = OverlayedChanges::default(); let mut transaction = { - let backend = test_trie(); + let backend = test_trie(false); let mut cache = StorageTransactionCache::default(); let mut ext = Ext::new( &mut overlay, @@ -1541,7 +1560,7 @@ mod tests { struct DummyExt(u32); } - let backend = trie_backend::tests::test_trie(); + let backend = trie_backend::tests::test_trie(false); let mut overlayed_changes = Default::default(); let wasm_code = RuntimeCode::empty(); diff --git a/primitives/state-machine/src/proving_backend.rs b/primitives/state-machine/src/proving_backend.rs index ef9e1f26c0a69..b6f1251a03dbf 100644 --- a/primitives/state-machine/src/proving_backend.rs +++ b/primitives/state-machine/src/proving_backend.rs @@ -393,13 +393,21 @@ mod tests { #[test] fn proof_is_empty_until_value_is_read() { - let trie_backend = test_trie(); + proof_is_empty_until_value_is_read_inner(false); + proof_is_empty_until_value_is_read_inner(true); + } + fn proof_is_empty_until_value_is_read_inner(flagged: bool) { + let trie_backend = test_trie(flagged); assert!(test_proving(&trie_backend).extract_proof().is_empty()); } #[test] fn proof_is_non_empty_after_value_is_read() { - let trie_backend = test_trie(); + proof_is_non_empty_after_value_is_read_inner(false); + proof_is_non_empty_after_value_is_read_inner(true); + } + fn proof_is_non_empty_after_value_is_read_inner(flagged: bool) { + let trie_backend = test_trie(flagged); let backend = test_proving(&trie_backend); assert_eq!(backend.storage(b"key").unwrap(), Some(b"value".to_vec())); assert!(!backend.extract_proof().is_empty()); @@ -417,11 +425,14 @@ mod tests { #[test] fn passes_through_backend_calls() { - let trie_backend = test_trie(); + passes_through_backend_calls_inner(false); + passes_through_backend_calls_inner(true); + } + fn passes_through_backend_calls_inner(flagged: bool) { + let trie_backend = test_trie(flagged); let proving_backend = test_proving(&trie_backend); assert_eq!(trie_backend.storage(b"key").unwrap(), proving_backend.storage(b"key").unwrap()); assert_eq!(trie_backend.pairs(), proving_backend.pairs()); - let flagged = false; let (trie_root, mut trie_mdb) = trie_backend.storage_root(std::iter::empty(), flagged); let (proving_root, mut proving_mdb) = proving_backend.storage_root(std::iter::empty(), flagged); @@ -431,15 +442,20 @@ mod tests { #[test] fn proof_recorded_and_checked() { + proof_recorded_and_checked_inner(false, false); + proof_recorded_and_checked_inner(false, true); + proof_recorded_and_checked_inner(true, false); + proof_recorded_and_checked_inner(true, true); + } + fn proof_recorded_and_checked_inner(flagged: bool, do_flag: bool) { let contents = (0..64).map(|i| (vec![i], Some(vec![i]))).collect::>(); let in_memory = InMemoryBackend::::default(); - let flagged = false; // TODO test with flag let mut in_memory = in_memory.update(vec![(None, contents)], flagged); let in_memory_root = in_memory.storage_root(::std::iter::empty(), flagged).0; (0..64).for_each(|i| assert_eq!(in_memory.storage(&[i]).unwrap().unwrap(), vec![i])); let trie = in_memory.as_trie_backend().unwrap(); - let trie_root = trie.storage_root(::std::iter::empty(), flagged).0; + let trie_root = trie.storage_root(::std::iter::empty(), do_flag).0; assert_eq!(in_memory_root, trie_root); (0..64).for_each(|i| assert_eq!(trie.storage(&[i]).unwrap().unwrap(), vec![i])); @@ -454,6 +470,10 @@ mod tests { #[test] fn proof_recorded_and_checked_with_child() { + proof_recorded_and_checked_with_child_inner(false); + proof_recorded_and_checked_with_child_inner(true); + } + fn proof_recorded_and_checked_with_child_inner(flagged: bool) { let child_info_1 = ChildInfo::new_default(b"sub1"); let child_info_2 = ChildInfo::new_default(b"sub2"); let child_info_1 = &child_info_1; @@ -466,7 +486,6 @@ mod tests { (10..15).map(|i| (vec![i], Some(vec![i]))).collect()), ]; let in_memory = InMemoryBackend::::default(); - let flagged = false; // TODO test with flag let mut in_memory = in_memory.update(contents, flagged); let child_storage_keys = vec![child_info_1.to_owned(), child_info_2.to_owned()]; let in_memory_root = in_memory.full_storage_root( @@ -525,8 +544,12 @@ mod tests { } #[test] - fn storage_proof_encoded_size_estimation_works() { // TODO same with flag -> test_trie with flag - let trie_backend = test_trie(); + fn storage_proof_encoded_size_estimation_works() { + storage_proof_encoded_size_estimation_works_inner(false); + storage_proof_encoded_size_estimation_works_inner(true); + } + fn storage_proof_encoded_size_estimation_works_inner(flagged: bool) { + let trie_backend = test_trie(flagged); let backend = test_proving(&trie_backend); let check_estimation = |backend: &ProvingBackend<'_, PrefixedMemoryDB, BlakeTwo256>| { diff --git a/primitives/state-machine/src/trie_backend.rs b/primitives/state-machine/src/trie_backend.rs index ae96d5bc97d2b..986f6db63670f 100644 --- a/primitives/state-machine/src/trie_backend.rs +++ b/primitives/state-machine/src/trie_backend.rs @@ -268,7 +268,7 @@ pub mod tests { const CHILD_KEY_1: &[u8] = b"sub1"; - fn test_db() -> (PrefixedMemoryDB, H256) { + fn test_db(hashed_value: bool) -> (PrefixedMemoryDB, H256) { let child_info = ChildInfo::new_default(CHILD_KEY_1); let mut root = H256::default(); let mut mdb = PrefixedMemoryDB::::default(); @@ -283,6 +283,10 @@ pub mod tests { let mut sub_root = Vec::new(); root.encode_to(&mut sub_root); let mut trie = TrieDBMut::new(&mut mdb, &mut root); + if hashed_value { + sp_trie::flag_meta_hasher(&mut trie).expect("flag failed"); + } + trie.insert(child_info.prefixed_storage_key().as_slice(), &sub_root[..]) .expect("insert failed"); trie.insert(b"key", b"value").expect("insert failed"); @@ -296,19 +300,27 @@ pub mod tests { (mdb, root) } - pub(crate) fn test_trie() -> TrieBackend, BlakeTwo256> { - let (mdb, root) = test_db(); + pub(crate) fn test_trie(hashed_value: bool) -> TrieBackend, BlakeTwo256> { + let (mdb, root) = test_db(hashed_value); TrieBackend::new(mdb, root) } #[test] fn read_from_storage_returns_some() { - assert_eq!(test_trie().storage(b"key").unwrap(), Some(b"value".to_vec())); + read_from_storage_returns_some_inner(false); + read_from_storage_returns_some_inner(true); + } + fn read_from_storage_returns_some_inner(flagged: bool) { + assert_eq!(test_trie(flagged).storage(b"key").unwrap(), Some(b"value".to_vec())); } #[test] fn read_from_child_storage_returns_some() { - let test_trie = test_trie(); + read_from_child_storage_returns_some_inner(false); + read_from_child_storage_returns_some_inner(true); + } + fn read_from_child_storage_returns_some_inner(flagged: bool) { + let test_trie = test_trie(flagged); assert_eq!( test_trie.child_storage(&ChildInfo::new_default(CHILD_KEY_1), b"value3").unwrap(), Some(vec![142u8]), @@ -317,12 +329,20 @@ pub mod tests { #[test] fn read_from_storage_returns_none() { - assert_eq!(test_trie().storage(b"non-existing-key").unwrap(), None); + read_from_storage_returns_none_inner(false); + read_from_storage_returns_none_inner(true); + } + fn read_from_storage_returns_none_inner(flagged: bool) { + assert_eq!(test_trie(flagged).storage(b"non-existing-key").unwrap(), None); } #[test] fn pairs_are_not_empty_on_non_empty_storage() { - assert!(!test_trie().pairs().is_empty()); + pairs_are_not_empty_on_non_empty_storage_inner(false); + pairs_are_not_empty_on_non_empty_storage_inner(true); + } + fn pairs_are_not_empty_on_non_empty_storage_inner(flagged: bool) { + assert!(!test_trie(flagged).pairs().is_empty()); } #[test] @@ -335,36 +355,50 @@ pub mod tests { #[test] fn storage_root_is_non_default() { - let flagged = false; - assert!(test_trie().storage_root(iter::empty(), flagged).0 != H256::repeat_byte(0)); + storage_root_is_non_default_inner(false); + storage_root_is_non_default_inner(true); + } + fn storage_root_is_non_default_inner(flagged: bool) { + assert!(test_trie(flagged).storage_root(iter::empty(), flagged).0 != H256::repeat_byte(0)); } #[test] fn storage_root_transaction_is_empty() { - let flagged = false; - assert!(test_trie().storage_root(iter::empty(), flagged).1.drain().is_empty()); + storage_root_transaction_is_empty_inner(false); + storage_root_transaction_is_empty_inner(true); + } + fn storage_root_transaction_is_empty_inner(flagged: bool) { + assert!(test_trie(flagged).storage_root(iter::empty(), false).1.drain().is_empty()); } #[test] fn storage_root_flagged_is_not_empty() { - let flagged = true; - assert!(!test_trie().storage_root(iter::empty(), flagged).1.drain().is_empty()); + assert!(!test_trie(false).storage_root(iter::empty(), true).1.drain().is_empty()); } #[test] fn storage_root_transaction_is_non_empty() { - // TODO test with flagged `test_trie` (initially only). - let (new_root, mut tx) = test_trie().storage_root( + storage_root_transaction_is_non_empty_inner(false, false); + storage_root_transaction_is_non_empty_inner(false, true); + storage_root_transaction_is_non_empty_inner(true, false); + storage_root_transaction_is_non_empty_inner(true, true); + } + fn storage_root_transaction_is_non_empty_inner(flagged: bool, do_flag: bool) { + let (new_root, mut tx) = test_trie(flagged).storage_root( iter::once((&b"new-key"[..], Some(&b"new-value"[..]))), - false, + do_flag, ); assert!(!tx.drain().is_empty()); - assert!(new_root != test_trie().storage_root(iter::empty(), false).0); + assert!(new_root != test_trie(false).storage_root(iter::empty(), false).0); } #[test] fn prefix_walking_works() { - let trie = test_trie(); + prefix_walking_works_inner(false); + prefix_walking_works_inner(true); + } + fn prefix_walking_works_inner(flagged: bool) { + let trie = test_trie(flagged); let mut seen = HashSet::new(); trie.for_keys_with_prefix(b"value", |key| { diff --git a/primitives/trie/src/lib.rs b/primitives/trie/src/lib.rs index 0d08ac22d2243..50ef70f121af6 100644 --- a/primitives/trie/src/lib.rs +++ b/primitives/trie/src/lib.rs @@ -479,16 +479,26 @@ pub fn flag_inner_meta_hasher( { { let mut t = TrieDBMut::::from_existing(db, &mut root)?; - let flag = true; - let key: &[u8]= &[]; - if !t.flag(key, flag)? { - t.insert(key, b"")?; - assert!(t.flag(key, flag)?); - } + flag_meta_hasher(&mut t)?; } Ok(root) } +/// Flag inner trie with state metadata to enable hash of value internally. +pub fn flag_meta_hasher( + t: &mut TrieDBMut +) -> Result<(), Box>> where + L: TrieConfiguration, +{ + let flag = true; + let key: &[u8]= &[]; + if !t.contains(key)? { + t.insert(key, b"")?; + } + assert!(t.flag(key, flag)?); + Ok(()) +} + /// Read a value from the trie. pub fn read_trie_value>( db: &DB, From 5bb0b6889bfc6b1994e98f7deb17326666484428 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 18 May 2021 18:36:07 +0200 Subject: [PATCH 006/127] added todo to fix --- primitives/state-machine/src/proving_backend.rs | 4 ++-- primitives/trie/src/storage_proof.rs | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/primitives/state-machine/src/proving_backend.rs b/primitives/state-machine/src/proving_backend.rs index b6f1251a03dbf..cbb136d017a83 100644 --- a/primitives/state-machine/src/proving_backend.rs +++ b/primitives/state-machine/src/proving_backend.rs @@ -451,11 +451,11 @@ mod tests { let contents = (0..64).map(|i| (vec![i], Some(vec![i]))).collect::>(); let in_memory = InMemoryBackend::::default(); let mut in_memory = in_memory.update(vec![(None, contents)], flagged); - let in_memory_root = in_memory.storage_root(::std::iter::empty(), flagged).0; + let in_memory_root = in_memory.storage_root(std::iter::empty(), do_flag).0; (0..64).for_each(|i| assert_eq!(in_memory.storage(&[i]).unwrap().unwrap(), vec![i])); let trie = in_memory.as_trie_backend().unwrap(); - let trie_root = trie.storage_root(::std::iter::empty(), do_flag).0; + let trie_root = trie.storage_root(std::iter::empty(), do_flag).0; assert_eq!(in_memory_root, trie_root); (0..64).for_each(|i| assert_eq!(trie.storage(&[i]).unwrap().unwrap(), vec![i])); diff --git a/primitives/trie/src/storage_proof.rs b/primitives/trie/src/storage_proof.rs index f0b2bfd4bc3d3..c6cfd34348ce8 100644 --- a/primitives/trie/src/storage_proof.rs +++ b/primitives/trie/src/storage_proof.rs @@ -103,6 +103,7 @@ impl From for crate::MemoryDB { fn from(proof: StorageProof) -> Self { let mut db = crate::MemoryDB::default(); for item in proof.iter_nodes() { + // TODO insert_with_meta here db.insert(crate::EMPTY_PREFIX, &item); } db From 708186489e4a4e26c90cd5de29a96b97110844d7 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 18 May 2021 18:39:05 +0200 Subject: [PATCH 007/127] also missing serialize meta to storage proof --- primitives/state-machine/src/proving_backend.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/primitives/state-machine/src/proving_backend.rs b/primitives/state-machine/src/proving_backend.rs index cbb136d017a83..1346bfc71c6fb 100644 --- a/primitives/state-machine/src/proving_backend.rs +++ b/primitives/state-machine/src/proving_backend.rs @@ -174,6 +174,7 @@ impl ProofRecorder { /// Convert into a [`StorageProof`]. pub fn to_storage_proof(&self) -> StorageProof { + // TODO serialize meta. let trie_nodes = self.inner.read() .records .iter() From 3f1ee8390d7b7cebd8fb929c05260c3006b2ce28 Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 19 May 2021 11:57:22 +0200 Subject: [PATCH 008/127] extract meta. --- Cargo.lock | 16 ++++----- .../state-machine/src/proving_backend.rs | 7 ++-- primitives/trie/src/lib.rs | 23 +++++++++++-- primitives/trie/src/storage_proof.rs | 34 +++++++++++++++++-- 4 files changed, 64 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1ccdd8f9ea5c7..b0acfa17a74c8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2329,7 +2329,7 @@ dependencies = [ [[package]] name = "hash-db" version = "0.15.2" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple2#594c900614841eea6ab66ac0d169352596ea3322" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple2#b3a6725a3d7f25185826b97f5efc1174a34c58f4" [[package]] name = "hash256-std-hasher" @@ -2343,7 +2343,7 @@ dependencies = [ [[package]] name = "hash256-std-hasher" version = "0.15.2" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple2#594c900614841eea6ab66ac0d169352596ea3322" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple2#b3a6725a3d7f25185826b97f5efc1174a34c58f4" dependencies = [ "crunchy", ] @@ -2996,7 +2996,7 @@ dependencies = [ [[package]] name = "keccak-hasher" version = "0.15.3" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple2#594c900614841eea6ab66ac0d169352596ea3322" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple2#b3a6725a3d7f25185826b97f5efc1174a34c58f4" dependencies = [ "hash-db", "hash256-std-hasher 0.15.2 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple2)", @@ -3788,7 +3788,7 @@ dependencies = [ [[package]] name = "memory-db" version = "0.26.0" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple2#594c900614841eea6ab66ac0d169352596ea3322" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple2#b3a6725a3d7f25185826b97f5efc1174a34c58f4" dependencies = [ "hash-db", "hashbrown", @@ -10332,7 +10332,7 @@ checksum = "a7f741b240f1a48843f9b8e0444fb55fb2a4ff67293b50a9179dfd5ea67f8d41" [[package]] name = "trie-bench" version = "0.27.0" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple2#594c900614841eea6ab66ac0d169352596ea3322" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple2#b3a6725a3d7f25185826b97f5efc1174a34c58f4" dependencies = [ "criterion", "hash-db", @@ -10347,7 +10347,7 @@ dependencies = [ [[package]] name = "trie-db" version = "0.22.3" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple2#594c900614841eea6ab66ac0d169352596ea3322" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple2#b3a6725a3d7f25185826b97f5efc1174a34c58f4" dependencies = [ "hash-db", "hashbrown", @@ -10368,7 +10368,7 @@ dependencies = [ [[package]] name = "trie-root" version = "0.16.0" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple2#594c900614841eea6ab66ac0d169352596ea3322" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple2#b3a6725a3d7f25185826b97f5efc1174a34c58f4" dependencies = [ "hash-db", ] @@ -10386,7 +10386,7 @@ dependencies = [ [[package]] name = "trie-standardmap" version = "0.15.2" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple2#594c900614841eea6ab66ac0d169352596ea3322" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple2#b3a6725a3d7f25185826b97f5efc1174a34c58f4" dependencies = [ "hash-db", "keccak-hasher 0.15.3 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple2)", diff --git a/primitives/state-machine/src/proving_backend.rs b/primitives/state-machine/src/proving_backend.rs index 1346bfc71c6fb..747467ee3b66d 100644 --- a/primitives/state-machine/src/proving_backend.rs +++ b/primitives/state-machine/src/proving_backend.rs @@ -125,14 +125,15 @@ pub struct ProofRecorder { impl ProofRecorder { /// Record the given `key` => `val` combination. - pub fn record(&self, key: Hash, mut val: Option<(DBValue, TrieMeta)>, hash_len: usize) { + pub fn record(&self, key: Hash, mut val: Option<(DBValue, TrieMeta)>) { let mut inner = self.inner.write(); let ProofRecorderInner { encoded_size, records } = &mut *inner; records.entry(key).or_insert_with(|| { if let Some(val) = val.as_mut() { val.1.set_accessed_value(false); - *encoded_size += sp_trie::estimate_entry_size(val, hash_len); + sp_trie::resolve_encoded_meta::(val); + *encoded_size += sp_trie::estimate_entry_size(val, H::LENGTH); } val }); @@ -244,7 +245,7 @@ impl<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> TrieBackendStorage } let backend_value = self.backend.get(key, prefix, parent)?; - self.proof_recorder.record(key.clone(), backend_value.clone(), H::LENGTH); + self.proof_recorder.record::(key.clone(), backend_value.clone()); Ok(backend_value) } diff --git a/primitives/trie/src/lib.rs b/primitives/trie/src/lib.rs index 50ef70f121af6..9456f1e2ddfcf 100644 --- a/primitives/trie/src/lib.rs +++ b/primitives/trie/src/lib.rs @@ -163,8 +163,17 @@ impl Meta for TrieMeta { fn decoded_callback( &mut self, - _node_plan: &NodePlan, + node_plan: &NodePlan, ) { + let (contain_hash, range) = match node_plan.value_plan() { + Some(ValuePlan::Value(range)) => (false, range.clone()), + Some(ValuePlan::HashedValue(range, _size)) => (true, range.clone()), + Some(ValuePlan::NoValue) => return, + None => return, + }; + + self.range = Some(range); + self.contain_hash = contain_hash; } fn contains_hash_of_value(&self) -> bool { @@ -241,7 +250,7 @@ impl MetaHasher for StateHasher fn hash(value: &[u8], meta: &Self::Meta) -> H::Out { match &meta { TrieMeta { range: Some(range), contain_hash: false, do_value_hash, .. } => { - if *do_value_hash { + if *do_value_hash && range.end - range.start >= trie_constants::INNER_HASH_TRESHOLD { let value = inner_hashed_value::(value, Some((range.start, range.end))); H::hash(value.as_slice()) } else { @@ -788,7 +797,7 @@ impl<'a, DB, H, T, M> hash_db::AsHashDB for KeySpacedDBMut<'a, DB, H> w } /// Representation of node with with inner hash instead of value. -pub fn inner_hashed_value(x: &[u8], range: Option<(usize, usize)>) -> Vec { +fn inner_hashed_value(x: &[u8], range: Option<(usize, usize)>) -> Vec { if let Some((start, end)) = range { let len = x.len(); if start < len && end == len { @@ -841,6 +850,14 @@ pub fn estimate_entry_size(entry: &(DBValue, TrieMeta), hash_len: usize) -> usiz full_encoded } +/// If needed, call to decode plan in order to record meta. +pub fn resolve_encoded_meta(entry: &mut (DBValue, TrieMeta)) { + use trie_db::NodeCodec; + if entry.1.do_value_hash { + let _ = as TrieLayout>::Codec::decode_plan(entry.0.as_slice(), &mut entry.1); + } +} + /// Constants used into trie simplification codec. mod trie_constants { /// Treshold for using hash of value instead of value diff --git a/primitives/trie/src/storage_proof.rs b/primitives/trie/src/storage_proof.rs index c6cfd34348ce8..ed755237c4be3 100644 --- a/primitives/trie/src/storage_proof.rs +++ b/primitives/trie/src/storage_proof.rs @@ -101,10 +101,40 @@ impl Iterator for StorageProofNodeIterator { impl From for crate::MemoryDB { fn from(proof: StorageProof) -> Self { + use hash_db::MetaHasher; + use trie_db::NodeCodec; + use crate::{Layout, TrieLayout}; let mut db = crate::MemoryDB::default(); - for item in proof.iter_nodes() { + // Needed because we do not read trie structure, so + // we do a heuristic related to the fact that host function + // only allow global definition. + // Using compact proof will work directly here (read trie structure and + // work directly. + let mut is_hashed_value = false; + let mut accum = Vec::new(); + for item in proof.trie_nodes.iter() { + // Note using `None` as parent meta does not impact `extract_value` of + // sp_trie meta hasher. + // But does not with `insert_with_meta`. + let (encoded_node, mut meta) = < + as TrieLayout>::MetaHasher as MetaHasher + >::extract_value(item.as_slice(), None); + if !is_hashed_value { + // read state meta. + let _ = as TrieLayout>::Codec::decode_plan(encoded_node, &mut meta); + if meta.recorded_do_value_hash { + is_hashed_value = true; + } + } // TODO insert_with_meta here - db.insert(crate::EMPTY_PREFIX, &item); + accum.push((encoded_node, meta)); + } + for mut item in accum.into_iter() { + if is_hashed_value { + // skipping hierarchy. + item.1.do_value_hash = true; + } + db.insert_with_meta(crate::EMPTY_PREFIX, item.0, item.1); } db } From 1bbdf35e6fa7e4734baca5a445d3e8f9311ea930 Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 19 May 2021 17:32:12 +0200 Subject: [PATCH 009/127] Isolate old trie layout. --- client/api/src/cht.rs | 7 +- client/db/src/bench.rs | 2 +- client/db/src/changes_tries_storage.rs | 2 +- client/db/src/lib.rs | 12 +-- client/light/src/fetcher.rs | 2 +- frame/session/src/historical/mod.rs | 2 +- .../api/proc-macro/src/impl_runtime_apis.rs | 2 +- primitives/state-machine/src/backend.rs | 32 ++++++- .../state-machine/src/changes_trie/mod.rs | 2 +- .../state-machine/src/changes_trie/prune.rs | 4 +- .../state-machine/src/changes_trie/storage.rs | 4 +- .../state-machine/src/in_memory_backend.rs | 2 +- primitives/state-machine/src/lib.rs | 19 +++- .../state-machine/src/proving_backend.rs | 39 +++++---- .../state-machine/src/trie_backend_essence.rs | 24 ++--- primitives/trie/src/lib.rs | 87 +++++++++++++++---- primitives/trie/src/storage_proof.rs | 22 +++-- 17 files changed, 184 insertions(+), 80 deletions(-) diff --git a/client/api/src/cht.rs b/client/api/src/cht.rs index 235976af09018..5c8dadcd6825f 100644 --- a/client/api/src/cht.rs +++ b/client/api/src/cht.rs @@ -32,8 +32,9 @@ use sp_trie; use sp_core::{H256, convert_hash}; use sp_runtime::traits::{Header as HeaderT, AtLeast32Bit, Zero, One}; use sp_state_machine::{ - MemoryDB, TrieBackend, Backend as StateBackend, StorageProof, InMemoryBackend, - prove_read_on_trie_backend, read_proof_check, read_proof_check_on_proving_backend + MemoryDBNoMeta as MemoryDB, TrieBackend, Backend as StateBackend, StorageProof, InMemoryBackend, + prove_read_on_trie_backend, read_proof_check, + read_proof_check_on_proving_backend_generic as read_proof_check_on_proving_backend, }; use sp_blockchain::{Error as ClientError, Result as ClientResult}; @@ -171,7 +172,7 @@ pub fn check_proof_on_proving_backend( local_number, remote_hash, |_, local_cht_key| - read_proof_check_on_proving_backend::( + read_proof_check_on_proving_backend::( proving_backend, local_cht_key, ).map_err(ClientError::from_state), diff --git a/client/db/src/bench.rs b/client/db/src/bench.rs index bc5ab72dee66e..fdc53ab224966 100644 --- a/client/db/src/bench.rs +++ b/client/db/src/bench.rs @@ -58,7 +58,7 @@ impl sp_state_machine::Storage> for StorageDb, _>>::extract_value_owned(value, parent))) .map_err(|e| format!("Database backend error: {:?}", e))?; - recorder.record(key.clone(), backend_value.clone(), HashFor::::LENGTH); + recorder.record::>(key.clone(), backend_value.clone()); Ok(backend_value) } else { self.db.get(0, &prefixed_key) diff --git a/client/db/src/changes_tries_storage.rs b/client/db/src/changes_tries_storage.rs index 860ca41730518..3f7e19cacfc24 100644 --- a/client/db/src/changes_tries_storage.rs +++ b/client/db/src/changes_tries_storage.rs @@ -24,7 +24,7 @@ use hash_db::Prefix; use codec::{Decode, Encode}; use parking_lot::RwLock; use sp_blockchain::{Error as ClientError, Result as ClientResult}; -use sp_trie::MemoryDB; +use sp_trie::MemoryDBNoMeta as MemoryDB; use sc_client_api::backend::PrunableStateChangesTrieStorage; use sp_blockchain::{well_known_cache_keys, Cache as BlockchainCache, HeaderMetadataCache}; use sp_core::{ChangesTrieConfiguration, ChangesTrieConfigurationRange, convert_hash}; diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index ce45c298e9de3..c465598632396 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -65,8 +65,8 @@ use sp_blockchain::{ }; use codec::{Decode, Encode}; use hash_db::Prefix; -use sp_trie::{MemoryDB, PrefixedMemoryDB, prefixed_key, StateHasher, TrieMeta, - MetaHasher}; +use sp_trie::{MemoryDB, MemoryDBNoMeta, PrefixedMemoryDB, prefixed_key, StateHasher, + TrieMeta, MetaHasher}; use sp_database::Transaction; use sp_core::{Hasher, ChangesTrieConfiguration}; use sp_core::offchain::OffchainOverlayedChange; @@ -690,7 +690,7 @@ pub struct BlockImportOperation { storage_updates: StorageCollection, child_storage_updates: ChildStorageCollection, offchain_storage_updates: OffchainChangesCollection, - changes_trie_updates: MemoryDB>, + changes_trie_updates: MemoryDBNoMeta>, changes_trie_build_cache_update: Option>>, changes_trie_config_update: Option>, pending_block: Option>, @@ -1731,7 +1731,7 @@ impl sc_client_api::backend::Backend for Backend { child_storage_updates: Default::default(), offchain_storage_updates: Default::default(), changes_trie_config_update: None, - changes_trie_updates: MemoryDB::default(), + changes_trie_updates: MemoryDBNoMeta::default(), changes_trie_build_cache_update: None, aux_ops: Vec::new(), finalized_blocks: Vec::new(), @@ -2158,9 +2158,9 @@ pub(crate) mod tests { pub(crate) type Block = RawBlock>; - pub fn prepare_changes(changes: Vec<(Vec, Vec)>) -> (H256, MemoryDB) { + pub fn prepare_changes(changes: Vec<(Vec, Vec)>) -> (H256, MemoryDBNoMeta) { let mut changes_root = H256::default(); - let mut changes_trie_update = MemoryDB::::default(); + let mut changes_trie_update = MemoryDBNoMeta::::default(); { let mut trie = TrieDBMut::::new( &mut changes_trie_update, diff --git a/client/light/src/fetcher.rs b/client/light/src/fetcher.rs index e39cfe07fbf5e..117c2d6970bd9 100644 --- a/client/light/src/fetcher.rs +++ b/client/light/src/fetcher.rs @@ -161,7 +161,7 @@ impl> LightDataChecker { H::Out: Ord + codec::Codec, { // all the checks are sharing the same storage - let storage = remote_roots_proof.into_memory_db(); + let storage: sp_state_machine::MemoryDBNoMeta = remote_roots_proof.into(); // remote_roots.keys() are sorted => we can use this to group changes tries roots // that are belongs to the same CHT diff --git a/frame/session/src/historical/mod.rs b/frame/session/src/historical/mod.rs index 8902ebe551f6c..0564d159e1e3f 100644 --- a/frame/session/src/historical/mod.rs +++ b/frame/session/src/historical/mod.rs @@ -35,7 +35,7 @@ use frame_support::{ decl_module, decl_storage, Parameter, print, traits::{ValidatorSet, ValidatorSetWithIdentification}, }; -use sp_trie::{MemoryDB, Trie, TrieMut, Recorder, EMPTY_PREFIX}; +use sp_trie::{MemoryDBNoMeta as MemoryDB, Trie, TrieMut, Recorder, EMPTY_PREFIX}; use sp_trie::trie_types::{TrieDBMut, TrieDB}; use super::{SessionIndex, Module as SessionModule}; diff --git a/primitives/api/proc-macro/src/impl_runtime_apis.rs b/primitives/api/proc-macro/src/impl_runtime_apis.rs index 85ba0788105d7..712987f8ac845 100644 --- a/primitives/api/proc-macro/src/impl_runtime_apis.rs +++ b/primitives/api/proc-macro/src/impl_runtime_apis.rs @@ -276,7 +276,7 @@ fn generate_runtime_api_base_structures() -> Result { fn extract_proof(&mut self) -> Option<#crate_::StorageProof> { self.recorder .take() - .map(|recorder| recorder.to_storage_proof()) + .map(|recorder| recorder.to_storage_proof::<#crate_::HashFor>()) } fn into_storage_changes( diff --git a/primitives/state-machine/src/backend.rs b/primitives/state-machine/src/backend.rs index e52fa999f149b..90923b19b21c9 100644 --- a/primitives/state-machine/src/backend.rs +++ b/primitives/state-machine/src/backend.rs @@ -358,13 +358,19 @@ impl Consolidate for Vec<( } } -impl> Consolidate for sp_trie::GenericMemoryDB { +impl Consolidate for sp_trie::GenericMemoryDB + where + H: Hasher, + MH: sp_trie::MetaHasher, + KF: sp_trie::KeyFunction, +{ fn consolidate(&mut self, other: Self) { sp_trie::GenericMemoryDB::consolidate(self, other) } } /// Insert input pairs into memory db. +/// TODO unused remove? #[cfg(test)] pub(crate) fn insert_into_memory_db(mdb: &mut sp_trie::MemoryDB, input: I) -> Option where @@ -387,6 +393,30 @@ pub(crate) fn insert_into_memory_db(mdb: &mut sp_trie::MemoryDB, input: Some(root) } +/// Insert input pairs into memory db. +#[cfg(test)] +pub(crate) fn insert_into_memory_db_no_meta(mdb: &mut sp_trie::MemoryDBNoMeta, input: I) -> Option + where + H: Hasher, + I: IntoIterator, +{ + use sp_trie::{TrieMut, trie_types::TrieDBMutNoMeta}; + + let mut root = ::Out::default(); + { + let mut trie = TrieDBMutNoMeta::::new(mdb, &mut root); + for (key, value) in input { + if let Err(e) = trie.insert(&key, &value) { + log::warn!(target: "trie", "Failed to write to trie: {}", e); + return None; + } + } + } + + Some(root) +} + + /// Wrapper to create a [`RuntimeCode`] from a type that implements [`Backend`]. #[cfg(feature = "std")] pub struct BackendRuntimeCode<'a, B, H> { diff --git a/primitives/state-machine/src/changes_trie/mod.rs b/primitives/state-machine/src/changes_trie/mod.rs index 89d869d38d999..dd6d4f0df3b88 100644 --- a/primitives/state-machine/src/changes_trie/mod.rs +++ b/primitives/state-machine/src/changes_trie/mod.rs @@ -73,7 +73,7 @@ use num_traits::{One, Zero}; use codec::{Decode, Encode}; use sp_core; use sp_core::storage::PrefixedStorageKey; -use sp_trie::{MemoryDB, DBValue, TrieMut}; +use sp_trie::{MemoryDBNoMeta as MemoryDB, DBValue, TrieMut}; use sp_trie::trie_types::TrieDBMut; use crate::{ StorageKey, diff --git a/primitives/state-machine/src/changes_trie/prune.rs b/primitives/state-machine/src/changes_trie/prune.rs index a741b814a5c70..4098eadb98b31 100644 --- a/primitives/state-machine/src/changes_trie/prune.rs +++ b/primitives/state-machine/src/changes_trie/prune.rs @@ -114,9 +114,9 @@ fn prune_trie( #[cfg(test)] mod tests { use std::collections::HashSet; - use sp_trie::MemoryDB; + use sp_trie::MemoryDBNoMeta as MemoryDB; use sp_core::H256; - use crate::backend::insert_into_memory_db; + use crate::backend::insert_into_memory_db_no_meta as insert_into_memory_db; use crate::changes_trie::storage::InMemoryStorage; use codec::Encode; use sp_runtime::traits::BlakeTwo256; diff --git a/primitives/state-machine/src/changes_trie/storage.rs b/primitives/state-machine/src/changes_trie/storage.rs index b41b2e549e82a..f81838f11f6e9 100644 --- a/primitives/state-machine/src/changes_trie/storage.rs +++ b/primitives/state-machine/src/changes_trie/storage.rs @@ -21,7 +21,7 @@ use std::collections::{BTreeMap, HashSet, HashMap}; use hash_db::{Hasher, Prefix, EMPTY_PREFIX}; use sp_core::storage::PrefixedStorageKey; use sp_trie::DBValue; -use sp_trie::MemoryDB; +use sp_trie::MemoryDBNoMeta as MemoryDB; use parking_lot::RwLock; use crate::{ StorageKey, @@ -30,7 +30,7 @@ use crate::{ }; #[cfg(test)] -use crate::backend::insert_into_memory_db; +use crate::backend::insert_into_memory_db_no_meta as insert_into_memory_db; #[cfg(test)] use crate::changes_trie::input::{InputPair, ChildIndex}; diff --git a/primitives/state-machine/src/in_memory_backend.rs b/primitives/state-machine/src/in_memory_backend.rs index 4e2bef05e50ab..d2a4bf6ae8e98 100644 --- a/primitives/state-machine/src/in_memory_backend.rs +++ b/primitives/state-machine/src/in_memory_backend.rs @@ -22,7 +22,7 @@ use crate::{ }; use std::collections::{BTreeMap, HashMap}; use hash_db::Hasher; -use sp_trie::{MemoryDB, empty_trie_root, Layout}; +use sp_trie::{MemoryDB, empty_trie_root, trie_types::Layout}; use codec::Codec; use sp_core::storage::{ChildInfo, Storage}; diff --git a/primitives/state-machine/src/lib.rs b/primitives/state-machine/src/lib.rs index 8b12c8aa030e6..d12686b43ab2b 100644 --- a/primitives/state-machine/src/lib.rs +++ b/primitives/state-machine/src/lib.rs @@ -143,7 +143,8 @@ mod changes_trie { #[cfg(feature = "std")] mod std_reexport { - pub use sp_trie::{trie_types::{Layout, TrieDBMut}, StorageProof, TrieMut, DBValue, MemoryDB}; + pub use sp_trie::{trie_types::{Layout, TrieDBMut}, StorageProof, TrieMut, + DBValue, MemoryDB, MemoryDBNoMeta}; pub use crate::testing::TestExternalities; pub use crate::basic::BasicExternalities; pub use crate::read_only::{ReadOnlyExternalities, InspectState}; @@ -194,7 +195,7 @@ mod execution { /// Type of changes trie transaction. pub type ChangesTrieTransaction = ( - MemoryDB, + MemoryDBNoMeta, ChangesTrieCacheAction<::Out, N>, ); @@ -842,6 +843,20 @@ mod execution { where H: Hasher, H::Out: Ord + Codec, + { + read_proof_check_on_proving_backend_generic(proving_backend, key) + } + + /// Check storage read proof on pre-created proving backend. + pub fn read_proof_check_on_proving_backend_generic( + proving_backend: &TrieBackend, H>, + key: &[u8], + ) -> Result>, Box> + where + H: Hasher, + H::Out: Ord + Codec, + MH: sp_trie::MetaHasher, + KF: sp_trie::KeyFunction + Send + Sync, { proving_backend.storage(key).map_err(|e| Box::new(e) as Box) } diff --git a/primitives/state-machine/src/proving_backend.rs b/primitives/state-machine/src/proving_backend.rs index 747467ee3b66d..d42d9ae51c3c9 100644 --- a/primitives/state-machine/src/proving_backend.rs +++ b/primitives/state-machine/src/proving_backend.rs @@ -174,12 +174,15 @@ impl ProofRecorder { } /// Convert into a [`StorageProof`]. - pub fn to_storage_proof(&self) -> StorageProof { - // TODO serialize meta. + pub fn to_storage_proof(&self) -> StorageProof { let trie_nodes = self.inner.read() .records .iter() - .filter_map(|(_k, v)| v.as_ref().map(|v| v.0.to_vec())) + .filter_map(|(_k, v)| v.as_ref().map(|v| { + < + as sp_trie::TrieLayout>::MetaHasher as hash_db::MetaHasher + >::stored_value(v.0.as_slice(), v.1.clone()) + })) .collect(); StorageProof::new(trie_nodes) @@ -230,7 +233,7 @@ impl<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> ProvingBackend<'a, S, H> /// Extracting the gathered unordered proof. pub fn extract_proof(&self) -> StorageProof { - self.0.essence().backend_storage().proof_recorder.to_storage_proof() + self.0.essence().backend_storage().proof_recorder.to_storage_proof::() } } @@ -443,31 +446,31 @@ mod tests { } #[test] - fn proof_recorded_and_checked() { - proof_recorded_and_checked_inner(false, false); - proof_recorded_and_checked_inner(false, true); - proof_recorded_and_checked_inner(true, false); - proof_recorded_and_checked_inner(true, true); - } - fn proof_recorded_and_checked_inner(flagged: bool, do_flag: bool) { - let contents = (0..64).map(|i| (vec![i], Some(vec![i]))).collect::>(); + fn proof_recorded_and_checked_top() { + proof_recorded_and_checked_inner(true); + proof_recorded_and_checked_inner(false); + } + fn proof_recorded_and_checked_inner(flagged: bool) { + let size_content = 33; // above hashable value treshold. + let value_range = 0..64; + let contents = value_range.clone().map(|i| (vec![i], Some(vec![i; size_content]))).collect::>(); let in_memory = InMemoryBackend::::default(); let mut in_memory = in_memory.update(vec![(None, contents)], flagged); - let in_memory_root = in_memory.storage_root(std::iter::empty(), do_flag).0; - (0..64).for_each(|i| assert_eq!(in_memory.storage(&[i]).unwrap().unwrap(), vec![i])); + let in_memory_root = in_memory.storage_root(std::iter::empty(), flagged).0; + value_range.clone().for_each(|i| assert_eq!(in_memory.storage(&[i]).unwrap().unwrap(), vec![i; size_content])); let trie = in_memory.as_trie_backend().unwrap(); - let trie_root = trie.storage_root(std::iter::empty(), do_flag).0; + let trie_root = trie.storage_root(std::iter::empty(), flagged).0; assert_eq!(in_memory_root, trie_root); - (0..64).for_each(|i| assert_eq!(trie.storage(&[i]).unwrap().unwrap(), vec![i])); + value_range.clone().for_each(|i| assert_eq!(trie.storage(&[i]).unwrap().unwrap(), vec![i; size_content])); let proving = ProvingBackend::new(trie); - assert_eq!(proving.storage(&[42]).unwrap().unwrap(), vec![42]); + assert_eq!(proving.storage(&[42]).unwrap().unwrap(), vec![42; size_content]); let proof = proving.extract_proof(); let proof_check = create_proof_check_backend::(in_memory_root.into(), proof).unwrap(); - assert_eq!(proof_check.storage(&[42]).unwrap().unwrap(), vec![42]); + assert_eq!(proof_check.storage(&[42]).unwrap().unwrap(), vec![42; size_content]); } #[test] diff --git a/primitives/state-machine/src/trie_backend_essence.rs b/primitives/state-machine/src/trie_backend_essence.rs index 3b7d47d9b0c62..594abf38fa698 100644 --- a/primitives/state-machine/src/trie_backend_essence.rs +++ b/primitives/state-machine/src/trie_backend_essence.rs @@ -23,7 +23,7 @@ use std::sync::Arc; use sp_std::{ops::Deref, boxed::Box, vec::Vec}; use crate::{warn, debug}; use hash_db::{self, Hasher, Prefix}; -use sp_trie::{Trie, MemoryDB, PrefixedMemoryDB, DBValue, +use sp_trie::{Trie, PrefixedMemoryDB, DBValue, empty_child_trie_root, read_trie_value, read_child_trie_value, for_keys_in_child_trie, KeySpacedDB, TrieDBIterator, TrieMeta}; use sp_trie::trie_types::{TrieDB, TrieError, Layout}; @@ -396,21 +396,13 @@ impl TrieBackendStorage for Arc> { } } -// This implementation is used by test storage trie clients. -impl TrieBackendStorage for PrefixedMemoryDB { - type Overlay = PrefixedMemoryDB; - - fn get(&self, key: &H::Out, prefix: Prefix, parent: Option<&TrieMeta>) -> Result> { - Ok(hash_db::HashDB::get_with_meta(self, key, prefix, parent)) - } - - fn access_from(&self, key: &H::Out) { - hash_db::HashDB::access_from(self, key, None); - } -} - -impl TrieBackendStorage for MemoryDB { - type Overlay = MemoryDB; +impl TrieBackendStorage for sp_trie::GenericMemoryDB + where + H: Hasher, + MH: sp_trie::MetaHasher, + KF: sp_trie::KeyFunction + Send + Sync, +{ + type Overlay = Self; fn get(&self, key: &H::Out, prefix: Prefix, parent: Option<&TrieMeta>) -> Result> { Ok(hash_db::HashDB::get_with_meta(self, key, prefix, parent)) diff --git a/primitives/trie/src/lib.rs b/primitives/trie/src/lib.rs index 9456f1e2ddfcf..d92a3d34f6110 100644 --- a/primitives/trie/src/lib.rs +++ b/primitives/trie/src/lib.rs @@ -99,7 +99,7 @@ impl Meta for TrieMeta { } fn write_state_meta(&self) -> Vec { - if self.do_value_hash { + if self.recorded_do_value_hash { // Note that this only works with sp_trie codec that // cannot encode node starting by this byte. [trie_constants::ENCODED_META_ALLOW_HASH].to_vec() @@ -199,34 +199,39 @@ impl TrieMeta { } /// substrate trie layout -pub struct Layout(sp_std::marker::PhantomData); +pub struct Layout(sp_std::marker::PhantomData<(H, M)>); -impl fmt::Debug for Layout { +impl fmt::Debug for Layout { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { f.debug_struct("Layout").finish() } } -impl Default for Layout { +impl Default for Layout { fn default() -> Self { Layout(sp_std::marker::PhantomData) } } -impl Clone for Layout { +impl Clone for Layout { fn clone(&self) -> Self { Layout(sp_std::marker::PhantomData) } } -impl TrieLayout for Layout { +impl TrieLayout for Layout + where + H: Hasher, + M: MetaHasher, + M::Meta: Meta, +{ const USE_EXTENSION: bool = false; const ALLOW_EMPTY: bool = true; const USE_META: bool = true; type Hash = H; type Codec = NodeCodec; - type MetaHasher = StateHasher; - type Meta = TrieMeta; + type MetaHasher = M; + type Meta = M::Meta; fn metainput_for_new_node(&self) -> ::MetaInput { () @@ -236,8 +241,7 @@ impl TrieLayout for Layout { } } -/// Reimplement `NoMeta` `MetaHasher` with -/// additional constraint. +/// Hasher with support to meta. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub struct StateHasher; @@ -328,7 +332,45 @@ impl MetaHasher for StateHasher } } -impl TrieConfiguration for Layout { +/// Reimplement `NoMeta` `MetaHasher` with +/// additional constraint. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct NoMetaHasher; + +impl MetaHasher for NoMetaHasher + where + H: Hasher, +{ + type Meta = TrieMeta; + + fn hash(value: &[u8], _meta: &Self::Meta) -> H::Out { + H::hash(value) + } + + fn stored_value(value: &[u8], _meta: Self::Meta) -> DBValue { + value.to_vec() + } + + fn stored_value_owned(value: DBValue, _meta: Self::Meta) -> DBValue { + value + } + + fn extract_value<'a>(stored: &'a [u8], _parent_meta: Option<&Self::Meta>) -> (&'a [u8], Self::Meta) { + (stored, Default::default()) + } + + fn extract_value_owned(stored: DBValue, _parent_meta: Option<&Self::Meta>) -> (DBValue, Self::Meta) { + (stored, Default::default()) + } +} + + +impl TrieConfiguration for Layout + where + H: Hasher, + M: MetaHasher, + M::Meta: Meta, +{ fn trie_root(&self, input: I) -> ::Out where I: IntoIterator, A: AsRef<[u8]> + Ord, @@ -374,14 +416,20 @@ pub type PrefixedMemoryDB = memory_db::MemoryDB< pub type MemoryDB = memory_db::MemoryDB< H, memory_db::HashKey, trie_db::DBValue, StateHasher, MemTracker, >; +/// Reexport from `hash_db`, with genericity set for `Hasher` trait. +/// This uses a noops `KeyFunction` (key addressing must be hashed or using +/// an encoding scheme that avoid key conflict). +pub type MemoryDBNoMeta = memory_db::MemoryDB< + H, memory_db::HashKey, trie_db::DBValue, NoMetaHasher, MemTracker, +>; /// MemoryDB with specific meta hasher. pub type MemoryDBMeta = memory_db::MemoryDB< H, memory_db::HashKey, trie_db::DBValue, M, MemTracker, >; /// Reexport from `hash_db`, with genericity set for `Hasher` trait. -pub type GenericMemoryDB = memory_db::MemoryDB< - H, KF, trie_db::DBValue, StateHasher, MemTracker +pub type GenericMemoryDB = memory_db::MemoryDB< + H, KF, trie_db::DBValue, MH, MemTracker >; /// Persistent trie database read-access interface for the a given hasher. @@ -395,11 +443,18 @@ pub type TrieHash = <::Hash as Hasher>::Out; /// This module is for non generic definition of trie type. /// Only the `Hasher` trait is generic in this case. pub mod trie_types { - pub type Layout = super::Layout; + /// State layout. + pub type Layout = super::Layout; + /// Old state layout definition, do not use meta, do not + /// do internal value hashing. + pub type LayoutNoMeta = super::Layout; /// Persistent trie database read-access interface for the a given hasher. pub type TrieDB<'a, H> = super::TrieDB<'a, Layout>; /// Persistent trie database write-access interface for the a given hasher. pub type TrieDBMut<'a, H> = super::TrieDBMut<'a, Layout>; + /// Persistent trie database write-access interface for the a given hasher, + /// old layout. + pub type TrieDBMutNoMeta<'a, H> = super::TrieDBMut<'a, LayoutNoMeta>; /// Querying interface, as in `trie_db` but less generic. pub type Lookup<'a, H, Q> = trie_db::Lookup<'a, Layout, Q>; /// As in `trie_db`, but less generic, error type for the crate. @@ -854,7 +909,7 @@ pub fn estimate_entry_size(entry: &(DBValue, TrieMeta), hash_len: usize) -> usiz pub fn resolve_encoded_meta(entry: &mut (DBValue, TrieMeta)) { use trie_db::NodeCodec; if entry.1.do_value_hash { - let _ = as TrieLayout>::Codec::decode_plan(entry.0.as_slice(), &mut entry.1); + let _ = as TrieLayout>::Codec::decode_plan(entry.0.as_slice(), &mut entry.1); } } @@ -884,7 +939,7 @@ mod tests { use trie_standardmap::{Alphabet, ValueMode, StandardMap}; use hex_literal::hex; - type Layout = super::Layout; + type Layout = super::trie_types::Layout; fn hashed_null_node() -> TrieHash { >::hashed_null_node() diff --git a/primitives/trie/src/storage_proof.rs b/primitives/trie/src/storage_proof.rs index ed755237c4be3..915dd92b1c16c 100644 --- a/primitives/trie/src/storage_proof.rs +++ b/primitives/trie/src/storage_proof.rs @@ -103,7 +103,7 @@ impl From for crate::MemoryDB { fn from(proof: StorageProof) -> Self { use hash_db::MetaHasher; use trie_db::NodeCodec; - use crate::{Layout, TrieLayout}; + use crate::{trie_types::Layout, TrieLayout}; let mut db = crate::MemoryDB::default(); // Needed because we do not read trie structure, so // we do a heuristic related to the fact that host function @@ -119,12 +119,10 @@ impl From for crate::MemoryDB { let (encoded_node, mut meta) = < as TrieLayout>::MetaHasher as MetaHasher >::extract_value(item.as_slice(), None); - if !is_hashed_value { - // read state meta. - let _ = as TrieLayout>::Codec::decode_plan(encoded_node, &mut meta); - if meta.recorded_do_value_hash { - is_hashed_value = true; - } + // read state meta. + let _ = as TrieLayout>::Codec::decode_plan(encoded_node, &mut meta); + if meta.recorded_do_value_hash { + is_hashed_value = true; } // TODO insert_with_meta here accum.push((encoded_node, meta)); @@ -139,3 +137,13 @@ impl From for crate::MemoryDB { db } } + +impl From for crate::MemoryDBNoMeta { + fn from(proof: StorageProof) -> Self { + let mut db = crate::MemoryDBNoMeta::default(); + for item in proof.iter_nodes() { + db.insert(crate::EMPTY_PREFIX, &item); + } + db + } +} From e5feabfc3bb1536e56f17cf81eefd7fbf22de3fb Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 19 May 2021 17:48:42 +0200 Subject: [PATCH 010/127] failing test that requires storing in meta when old hash scheme is used. --- Cargo.lock | 16 +++++------ .../state-machine/src/proving_backend.rs | 27 +++++++++++++++++++ 2 files changed, 35 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b0acfa17a74c8..79c33175dcfc3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2329,7 +2329,7 @@ dependencies = [ [[package]] name = "hash-db" version = "0.15.2" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple2#b3a6725a3d7f25185826b97f5efc1174a34c58f4" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple2#ab6d8bbc73d620af5357f350fa132142c2d57a33" [[package]] name = "hash256-std-hasher" @@ -2343,7 +2343,7 @@ dependencies = [ [[package]] name = "hash256-std-hasher" version = "0.15.2" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple2#b3a6725a3d7f25185826b97f5efc1174a34c58f4" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple2#ab6d8bbc73d620af5357f350fa132142c2d57a33" dependencies = [ "crunchy", ] @@ -2996,7 +2996,7 @@ dependencies = [ [[package]] name = "keccak-hasher" version = "0.15.3" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple2#b3a6725a3d7f25185826b97f5efc1174a34c58f4" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple2#ab6d8bbc73d620af5357f350fa132142c2d57a33" dependencies = [ "hash-db", "hash256-std-hasher 0.15.2 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple2)", @@ -3788,7 +3788,7 @@ dependencies = [ [[package]] name = "memory-db" version = "0.26.0" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple2#b3a6725a3d7f25185826b97f5efc1174a34c58f4" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple2#ab6d8bbc73d620af5357f350fa132142c2d57a33" dependencies = [ "hash-db", "hashbrown", @@ -10332,7 +10332,7 @@ checksum = "a7f741b240f1a48843f9b8e0444fb55fb2a4ff67293b50a9179dfd5ea67f8d41" [[package]] name = "trie-bench" version = "0.27.0" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple2#b3a6725a3d7f25185826b97f5efc1174a34c58f4" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple2#ab6d8bbc73d620af5357f350fa132142c2d57a33" dependencies = [ "criterion", "hash-db", @@ -10347,7 +10347,7 @@ dependencies = [ [[package]] name = "trie-db" version = "0.22.3" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple2#b3a6725a3d7f25185826b97f5efc1174a34c58f4" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple2#ab6d8bbc73d620af5357f350fa132142c2d57a33" dependencies = [ "hash-db", "hashbrown", @@ -10368,7 +10368,7 @@ dependencies = [ [[package]] name = "trie-root" version = "0.16.0" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple2#b3a6725a3d7f25185826b97f5efc1174a34c58f4" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple2#ab6d8bbc73d620af5357f350fa132142c2d57a33" dependencies = [ "hash-db", ] @@ -10386,7 +10386,7 @@ dependencies = [ [[package]] name = "trie-standardmap" version = "0.15.2" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple2#b3a6725a3d7f25185826b97f5efc1174a34c58f4" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple2#ab6d8bbc73d620af5357f350fa132142c2d57a33" dependencies = [ "hash-db", "keccak-hasher 0.15.3 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple2)", diff --git a/primitives/state-machine/src/proving_backend.rs b/primitives/state-machine/src/proving_backend.rs index d42d9ae51c3c9..47b478b1d2da0 100644 --- a/primitives/state-machine/src/proving_backend.rs +++ b/primitives/state-machine/src/proving_backend.rs @@ -473,6 +473,33 @@ mod tests { assert_eq!(proof_check.storage(&[42]).unwrap().unwrap(), vec![42; size_content]); } + #[test] + fn proof_recorded_and_checked_old_hash() { + // test proof starting with old hash content and flagging in between. + let size_content = 33; // above hashable value treshold. + let value_range = 0..64; + let contents = value_range.clone().map(|i| (vec![i], Some(vec![i; size_content]))).collect::>(); + let in_memory = InMemoryBackend::::default(); + let mut in_memory = in_memory.update(vec![(None, contents)], false); + let in_memory_root = in_memory.storage_root(std::iter::empty(), false).0; + value_range.clone().for_each(|i| assert_eq!(in_memory.storage(&[i]).unwrap().unwrap(), vec![i; size_content])); + + in_memory = in_memory.update(vec![], true); + let in_memory_root = in_memory.storage_root(std::iter::empty(), false).0; + let trie = in_memory.as_trie_backend().unwrap(); + let trie_root = trie.storage_root(std::iter::empty(), false).0; + assert_eq!(in_memory_root, trie_root); + value_range.clone().for_each(|i| assert_eq!(trie.storage(&[i]).unwrap().unwrap(), vec![i; size_content])); + + let proving = ProvingBackend::new(trie); + assert_eq!(proving.storage(&[42]).unwrap().unwrap(), vec![42; size_content]); + + let proof = proving.extract_proof(); + + let proof_check = create_proof_check_backend::(in_memory_root.into(), proof).unwrap(); + assert_eq!(proof_check.storage(&[42]).unwrap().unwrap(), vec![42; size_content]); + } + #[test] fn proof_recorded_and_checked_with_child() { proof_recorded_and_checked_with_child_inner(false); From aa78ed690e05048c66f6f3e86518a29fd825a7ec Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 19 May 2021 18:11:02 +0200 Subject: [PATCH 011/127] old hash compatibility --- primitives/trie/src/lib.rs | 38 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/primitives/trie/src/lib.rs b/primitives/trie/src/lib.rs index d92a3d34f6110..ade0c52ddcad5 100644 --- a/primitives/trie/src/lib.rs +++ b/primitives/trie/src/lib.rs @@ -69,6 +69,12 @@ pub struct TrieMeta { // change on access explicitely: `HashDB::get_with_meta`. // and reset on access explicitely: `HashDB::access_from`. pub unused_value: bool, + // Indicate that a node is using old hash scheme. + // Write with `do_value_hash` inactive will set this to + // true. + // In this case hash is not doing internal hashing, + // but next write with `do_value_hash` will remove switch scheme. + pub old_hash: bool, } impl Meta for TrieMeta { @@ -150,6 +156,10 @@ impl Meta for TrieMeta { self.range = Some(range); self.contain_hash = contain_hash; + if self.do_value_hash { + // Switch value hashing. + self.old_hash = false; + } } fn set_child_callback( @@ -253,7 +263,7 @@ impl MetaHasher for StateHasher fn hash(value: &[u8], meta: &Self::Meta) -> H::Out { match &meta { - TrieMeta { range: Some(range), contain_hash: false, do_value_hash, .. } => { + TrieMeta { range: Some(range), contain_hash: false, do_value_hash, old_hash: false, .. } => { if *do_value_hash && range.end - range.start >= trie_constants::INNER_HASH_TRESHOLD { let value = inner_hashed_value::(value, Some((range.start, range.end))); H::hash(value.as_slice()) @@ -273,6 +283,22 @@ impl MetaHasher for StateHasher fn stored_value(value: &[u8], mut meta: Self::Meta) -> DBValue { let mut stored = Vec::with_capacity(value.len() + 1); + if meta.old_hash { + // write as old hash. + stored.push(trie_constants::OLD_HASHING); + stored.extend_from_slice(value); + return stored; + } + if !meta.do_value_hash { + if let Some(range) = meta.range.as_ref() { + if range.end - range.start >= trie_constants::INNER_HASH_TRESHOLD { + // write as old hash. + stored.push(trie_constants::OLD_HASHING); + stored.extend_from_slice(value); + return stored; + } + } + } if meta.contain_hash { // already contain hash, just flag it. stored.push(trie_constants::DEAD_HEADER_META_HASHED_VALUE); @@ -305,16 +331,22 @@ impl MetaHasher for StateHasher fn extract_value<'a>(mut stored: &'a [u8], parent_meta: Option<&Self::Meta>) -> (&'a [u8], Self::Meta) { let input = &mut stored; let mut contain_hash = false; + let mut old_hash = false; if input.get(0) == Some(&trie_constants::DEAD_HEADER_META_HASHED_VALUE) { contain_hash = true; *input = &input[1..]; } + if input.get(0) == Some(&trie_constants::OLD_HASHING) { + old_hash = true; + *input = &input[1..]; + } let mut meta = TrieMeta { range: None, unused_value: contain_hash, contain_hash, do_value_hash: false, recorded_do_value_hash: false, + old_hash, }; // get recorded_do_value_hash let _offset = meta.read_state_meta(stored) @@ -364,7 +396,6 @@ impl MetaHasher for NoMetaHasher } } - impl TrieConfiguration for Layout where H: Hasher, @@ -923,6 +954,9 @@ mod trie_constants { pub const ENCODED_META_ALLOW_HASH: u8 = FIRST_PREFIX | 0b_01; /// In proof this header is used when only hashed value is stored. pub const DEAD_HEADER_META_HASHED_VALUE: u8 = FIRST_PREFIX | 0b_00_10; + /// If inner hashing should apply, but state is not flagged, then set + /// this meta to avoid checking both variant of hashes. + pub const OLD_HASHING: u8 = FIRST_PREFIX | 0b_00_11; pub const NIBBLE_SIZE_BOUND: usize = u16::max_value() as usize; pub const LEAF_PREFIX_MASK: u8 = 0b_01 << 6; pub const BRANCH_WITHOUT_MASK: u8 = 0b_10 << 6; From 045b0971ff1b281ee276e44959696cb40e0b8de3 Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 19 May 2021 19:10:10 +0200 Subject: [PATCH 012/127] Db migrate. --- client/db/src/upgrade.rs | 38 +++++++++++++++++++++++++++++++++++++- primitives/trie/src/lib.rs | 26 ++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 1 deletion(-) diff --git a/client/db/src/upgrade.rs b/client/db/src/upgrade.rs index ea91b8253e1d8..f5c0992ea231f 100644 --- a/client/db/src/upgrade.rs +++ b/client/db/src/upgrade.rs @@ -31,7 +31,7 @@ use codec::{Decode, Encode}; const VERSION_FILE_NAME: &'static str = "db_version"; /// Current db version. -const CURRENT_VERSION: u32 = 3; +const CURRENT_VERSION: u32 = 4; /// Number of columns in v1. const V1_NUM_COLUMNS: u32 = 11; @@ -49,6 +49,7 @@ pub fn upgrade_db(db_path: &Path, db_type: DatabaseType) -> sp_bl migrate_2_to_3::(db_path, db_type)? }, 2 => migrate_2_to_3::(db_path, db_type)?, + 3 => migrate_3_to_4::(db_path, db_type)?, CURRENT_VERSION => (), _ => Err(sp_blockchain::Error::Backend(format!("Future database version: {}", db_version)))?, } @@ -98,6 +99,41 @@ fn migrate_2_to_3(db_path: &Path, _db_type: DatabaseType) -> sp_b Ok(()) } +/// Migration from version3 to version4: +/// - Trie state meta for state that could be hashed internaly. +fn migrate_3_to_4(db_path: &Path, _db_type: DatabaseType) -> sp_blockchain::Result<()> { + let db_path = db_path.to_str() + .ok_or_else(|| sp_blockchain::Error::Backend("Invalid database path".into()))?; + let db_cfg = DatabaseConfig::with_columns(V2_NUM_COLUMNS); + let db = Database::open(&db_cfg, db_path).map_err(db_err)?; + + let batch_size = 10_000; // TODO use bigger size (need to iterate all each time). + loop { + let mut full_batch = false; + let mut size = 0; + let mut transaction = db.transaction(); + // Get all the keys we need to update. + // Note that every batch will restart full iter, could use + // a `iter_from` function. + for entry in db.iter(columns::STATE) { + if let Some(new_val) = sp_trie::tag_old_hashes::>(&entry.1) { + transaction.put_vec(columns::STATE, &entry.0, new_val); + size += 1; + if size == batch_size { + full_batch = true; + break; + } + } + } + db.write(transaction).map_err(db_err)?; + if !full_batch { + break; + } + } + Ok(()) +} + + /// Reads current database version from the file at given path. /// If the file does not exist returns 0. fn current_version(path: &Path) -> sp_blockchain::Result { diff --git a/primitives/trie/src/lib.rs b/primitives/trie/src/lib.rs index ade0c52ddcad5..33b3fd64e564c 100644 --- a/primitives/trie/src/lib.rs +++ b/primitives/trie/src/lib.rs @@ -963,6 +963,32 @@ mod trie_constants { pub const BRANCH_WITH_MASK: u8 = 0b_11 << 6; } +/// Utility to tag a state without meta with old_hash internal +/// hashing. +pub fn tag_old_hashes(existing: &[u8]) -> Option> { + use trie_db::NodeCodec; + let mut meta = TrieMeta::default(); + // allows restarting a migration. + if existing.len() > 0 && existing[0] == trie_constants::OLD_HASHING { + return None; // allow restarting a migration. + } + let _ = as TrieLayout>::Codec::decode_plan(existing, &mut meta) + .expect("Invalid db state entry found: {:?}, entry.0.as_slice()"); + match meta.range { + Some(range) => { + if range.end - range.start >= trie_constants::INNER_HASH_TRESHOLD { + let mut res = Vec::with_capacity(existing.len() + 1); + res.push(trie_constants::OLD_HASHING); + res.extend_from_slice(existing); + Some(res) + } else { + None + } + }, + None => None, + } +} + #[cfg(test)] mod tests { use super::*; From 79a43dd2d4518ab9dccd31a431f30d7e442469a1 Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 20 May 2021 12:02:29 +0200 Subject: [PATCH 013/127] runing tests with both states when interesting. --- bin/node/testing/src/client.rs | 2 - client/api/src/light.rs | 2 +- .../basic-authorship/src/basic_authorship.rs | 8 ++-- client/consensus/aura/src/lib.rs | 2 +- client/consensus/babe/src/aux_schema.rs | 2 +- client/consensus/slots/src/aux_schema.rs | 2 +- client/finality-grandpa/src/aux_schema.rs | 8 ++-- client/network/src/light_client_requests.rs | 26 ++++++++--- client/network/test/src/block_import.rs | 20 +++++---- client/network/test/src/lib.rs | 2 +- client/offchain/src/lib.rs | 2 +- client/rpc/src/chain/tests.rs | 18 +++++--- client/rpc/src/state/tests.rs | 27 +++++++++--- client/service/test/src/client/light.rs | 40 +++++++++++++---- client/service/test/src/client/mod.rs | 44 +++++++++++++------ client/transaction-pool/src/testing/pool.rs | 4 +- .../state-machine/src/proving_backend.rs | 2 - primitives/trie/src/lib.rs | 34 ++++++++++---- test-utils/client/src/lib.rs | 11 +++++ test-utils/runtime/client/src/lib.rs | 18 +++++--- utils/frame/rpc/system/src/lib.rs | 8 ++-- 21 files changed, 192 insertions(+), 90 deletions(-) diff --git a/bin/node/testing/src/client.rs b/bin/node/testing/src/client.rs index c4ace4ced9b42..e5bdd4e6f723c 100644 --- a/bin/node/testing/src/client.rs +++ b/bin/node/testing/src/client.rs @@ -75,5 +75,3 @@ impl TestClientBuilderExt for substrate_test_client::TestClientBuilder< self.build_with_native_executor(None).0 } } - - diff --git a/client/api/src/light.rs b/client/api/src/light.rs index a068e2d4a3417..ba6ee74a7caed 100644 --- a/client/api/src/light.rs +++ b/client/api/src/light.rs @@ -322,7 +322,7 @@ pub mod tests { fn into(self) -> ClientError { ClientError::Application(Box::new(self)) } - } + } pub type OkCallFetcher = Mutex>; diff --git a/client/basic-authorship/src/basic_authorship.rs b/client/basic-authorship/src/basic_authorship.rs index c8277d3b5d32c..084057bde29f1 100644 --- a/client/basic-authorship/src/basic_authorship.rs +++ b/client/basic-authorship/src/basic_authorship.rs @@ -504,7 +504,7 @@ mod tests { #[test] fn should_cease_building_block_when_deadline_is_reached() { // given - let client = Arc::new(substrate_test_runtime_client::new()); + let client = Arc::new(substrate_test_runtime_client::new(true)); let spawner = sp_core::testing::TaskExecutor::new(); let txpool = BasicPool::new_full( Default::default(), @@ -564,7 +564,7 @@ mod tests { #[test] fn should_not_panic_when_deadline_is_reached() { - let client = Arc::new(substrate_test_runtime_client::new()); + let client = Arc::new(substrate_test_runtime_client::new(true)); let spawner = sp_core::testing::TaskExecutor::new(); let txpool = BasicPool::new_full( Default::default(), @@ -675,7 +675,7 @@ mod tests { #[test] fn should_not_remove_invalid_transactions_when_skipping() { // given - let mut client = Arc::new(substrate_test_runtime_client::new()); + let mut client = Arc::new(substrate_test_runtime_client::new(true)); let spawner = sp_core::testing::TaskExecutor::new(); let txpool = BasicPool::new_full( Default::default(), @@ -766,7 +766,7 @@ mod tests { #[test] fn should_cease_building_block_when_block_limit_is_reached() { - let client = Arc::new(substrate_test_runtime_client::new()); + let client = Arc::new(substrate_test_runtime_client::new(true)); let spawner = sp_core::testing::TaskExecutor::new(); let txpool = BasicPool::new_full( Default::default(), diff --git a/client/consensus/aura/src/lib.rs b/client/consensus/aura/src/lib.rs index ce254799d61f7..5cb1079c3a98f 100644 --- a/client/consensus/aura/src/lib.rs +++ b/client/consensus/aura/src/lib.rs @@ -765,7 +765,7 @@ mod tests { #[test] fn authorities_call_works() { - let client = substrate_test_runtime_client::new(); + let client = substrate_test_runtime_client::new(true); assert_eq!(client.chain_info().best_number, 0); assert_eq!(authorities(&client, &BlockId::Number(0)).unwrap(), vec![ diff --git a/client/consensus/babe/src/aux_schema.rs b/client/consensus/babe/src/aux_schema.rs index 8b8804e3bfb02..cf8b3bbe0bd33 100644 --- a/client/consensus/babe/src/aux_schema.rs +++ b/client/consensus/babe/src/aux_schema.rs @@ -156,7 +156,7 @@ mod test { epoch_index: 1, duration: 100, }; - let client = substrate_test_runtime_client::new(); + let client = substrate_test_runtime_client::new(true); let mut v0_tree = ForkTree::, _>::new(); v0_tree.import::<_, ConsensusError>( Default::default(), diff --git a/client/consensus/slots/src/aux_schema.rs b/client/consensus/slots/src/aux_schema.rs index db94ec48855e4..13612613f47af 100644 --- a/client/consensus/slots/src/aux_schema.rs +++ b/client/consensus/slots/src/aux_schema.rs @@ -159,7 +159,7 @@ mod test { #[test] fn check_equivocation_works() { - let client = substrate_test_runtime_client::new(); + let client = substrate_test_runtime_client::new(true); let (pair, _seed) = sr25519::Pair::generate(); let public = pair.public(); diff --git a/client/finality-grandpa/src/aux_schema.rs b/client/finality-grandpa/src/aux_schema.rs index 296f7c13c5244..169366a9fa710 100644 --- a/client/finality-grandpa/src/aux_schema.rs +++ b/client/finality-grandpa/src/aux_schema.rs @@ -561,7 +561,7 @@ mod test { #[test] fn load_decode_from_v0_migrates_data_format() { - let client = substrate_test_runtime_client::new(); + let client = substrate_test_runtime_client::new(true); let authorities = vec![(AuthorityId::default(), 100)]; let set_id = 3; @@ -654,7 +654,7 @@ mod test { #[test] fn load_decode_from_v1_migrates_data_format() { - let client = substrate_test_runtime_client::new(); + let client = substrate_test_runtime_client::new(true); let authorities = vec![(AuthorityId::default(), 100)]; let set_id = 3; @@ -750,7 +750,7 @@ mod test { #[test] fn load_decode_from_v2_migrates_data_format() { - let client = substrate_test_runtime_client::new(); + let client = substrate_test_runtime_client::new(true); let authorities = vec![(AuthorityId::default(), 100)]; let set_id = 3; @@ -823,7 +823,7 @@ mod test { #[test] fn write_read_concluded_rounds() { - let client = substrate_test_runtime_client::new(); + let client = substrate_test_runtime_client::new(true); let hash = H256::random(); let round_state = RoundState::genesis((hash, 0)); diff --git a/client/network/src/light_client_requests.rs b/client/network/src/light_client_requests.rs index f859a35f45b24..14e95424ee70e 100644 --- a/client/network/src/light_client_requests.rs +++ b/client/network/src/light_client_requests.rs @@ -182,8 +182,8 @@ mod tests { type Block = sp_runtime::generic::Block, substrate_test_runtime::Extrinsic>; - fn send_receive(request: sender::Request, pool: &LocalPool) { - let client = Arc::new(substrate_test_runtime_client::new()); + fn send_receive(request: sender::Request, pool: &LocalPool, hashed_value: bool) { + let client = Arc::new(substrate_test_runtime_client::new(hashed_value)); let (handler, protocol_config) = handler::LightClientRequestHandler::new(&protocol_id(), client); pool.spawner().spawn_obj(handler.run().boxed().into()).unwrap(); @@ -215,6 +215,10 @@ mod tests { #[test] fn send_receive_call() { + send_receive_call_inner(true); + send_receive_call_inner(false); + } + fn send_receive_call_inner(hashed_value: bool) { let chan = oneshot::channel(); let request = light::RemoteCallRequest { block: Default::default(), @@ -228,13 +232,17 @@ mod tests { send_receive(sender::Request::Call { request, sender: chan.0, - }, &pool); + }, &pool, hashed_value); assert_eq!(vec![42], pool.run_until(chan.1).unwrap().unwrap()); // ^--- from `DummyFetchChecker::check_execution_proof` } #[test] fn send_receive_read() { + send_receive_read_inner(true); + send_receive_read_inner(false); + } + fn send_receive_read_inner(hashed_value: bool) { let chan = oneshot::channel(); let request = light::RemoteReadRequest { header: dummy_header(), @@ -246,7 +254,7 @@ mod tests { send_receive(sender::Request::Read { request, sender: chan.0, - }, &pool); + }, &pool, hashed_value); assert_eq!( Some(vec![42]), pool.run_until(chan.1) @@ -260,6 +268,10 @@ mod tests { #[test] fn send_receive_read_child() { + send_receive_read_child_inner(true); + send_receive_read_child_inner(false); + } + fn send_receive_read_child_inner(hashed_value: bool) { let chan = oneshot::channel(); let child_info = ChildInfo::new_default(&b":child_storage:default:sub"[..]); let request = light::RemoteReadChildRequest { @@ -273,7 +285,7 @@ mod tests { send_receive(sender::Request::ReadChild { request, sender: chan.0, - }, &pool); + }, &pool, hashed_value); assert_eq!( Some(vec![42]), pool.run_until(chan.1) @@ -298,7 +310,7 @@ mod tests { send_receive(sender::Request::Header { request, sender: chan.0, - }, &pool); + }, &pool, true); // The remote does not know block 1: assert_matches!( pool.run_until(chan.1).unwrap(), @@ -327,7 +339,7 @@ mod tests { send_receive(sender::Request::Changes { request, sender: chan.0, - }, &pool); + }, &pool, true); assert_eq!(vec![(100, 2)], pool.run_until(chan.1).unwrap().unwrap()); // ^--- from `DummyFetchChecker::check_changes_proof` } diff --git a/client/network/test/src/block_import.rs b/client/network/test/src/block_import.rs index b3641d4b41214..4119e009b195e 100644 --- a/client/network/test/src/block_import.rs +++ b/client/network/test/src/block_import.rs @@ -29,8 +29,8 @@ use sc_block_builder::BlockBuilderProvider; use futures::executor::block_on; use super::*; -fn prepare_good_block() -> (TestClient, Hash, u64, PeerId, IncomingBlock) { - let mut client = substrate_test_runtime_client::new(); +fn prepare_good_block(hashed_value: bool) -> (TestClient, Hash, u64, PeerId, IncomingBlock) { + let mut client = substrate_test_runtime_client::new(hashed_value); let block = client.new_block(Default::default()).unwrap().build().unwrap().block; block_on(client.import(BlockOrigin::File, block)).unwrap(); @@ -51,13 +51,17 @@ fn prepare_good_block() -> (TestClient, Hash, u64, PeerId, IncomingBlock) #[test] fn import_single_good_block_works() { - let (_, _hash, number, peer_id, block) = prepare_good_block(); + import_single_good_block_works_inner(true); + import_single_good_block_works_inner(false); +} +fn import_single_good_block_works_inner(hashed_value: bool) { + let (_, _hash, number, peer_id, block) = prepare_good_block(hashed_value); let mut expected_aux = ImportedAux::default(); expected_aux.is_new_best = true; match block_on(import_single_block( - &mut substrate_test_runtime_client::new(), + &mut substrate_test_runtime_client::new(hashed_value), BlockOrigin::File, block, &mut PassThroughVerifier::new(true) @@ -70,7 +74,7 @@ fn import_single_good_block_works() { #[test] fn import_single_good_known_block_is_ignored() { - let (mut client, _hash, number, _, block) = prepare_good_block(); + let (mut client, _hash, number, _, block) = prepare_good_block(true); match block_on(import_single_block( &mut client, BlockOrigin::File, @@ -84,10 +88,10 @@ fn import_single_good_known_block_is_ignored() { #[test] fn import_single_good_block_without_header_fails() { - let (_, _, _, peer_id, mut block) = prepare_good_block(); + let (_, _, _, peer_id, mut block) = prepare_good_block(true); block.header = None; match block_on(import_single_block( - &mut substrate_test_runtime_client::new(), + &mut substrate_test_runtime_client::new(true), BlockOrigin::File, block, &mut PassThroughVerifier::new(true) @@ -106,7 +110,7 @@ fn async_import_queue_drops() { let queue = BasicQueue::new( verifier, - Box::new(substrate_test_runtime_client::new()), + Box::new(substrate_test_runtime_client::new(true)), None, &executor, None, diff --git a/client/network/test/src/lib.rs b/client/network/test/src/lib.rs index 8e56005dad25d..47582607f8bb7 100644 --- a/client/network/test/src/lib.rs +++ b/client/network/test/src/lib.rs @@ -822,7 +822,7 @@ pub trait TestNetFactory: Sized where >: /// Add a light peer. fn add_light_peer(&mut self) { - let (c, backend) = substrate_test_runtime_client::new_light(); + let (c, backend) = substrate_test_runtime_client::new_light(true); let client = Arc::new(c); let ( block_import, diff --git a/client/offchain/src/lib.rs b/client/offchain/src/lib.rs index 26975edbd6b63..ae49f1d6384ff 100644 --- a/client/offchain/src/lib.rs +++ b/client/offchain/src/lib.rs @@ -285,7 +285,7 @@ mod tests { fn should_call_into_runtime_and_produce_extrinsic() { sp_tracing::try_init_simple(); - let client = Arc::new(substrate_test_runtime_client::new()); + let client = Arc::new(substrate_test_runtime_client::new(true)); let spawner = sp_core::testing::TaskExecutor::new(); let pool = TestPool(BasicPool::new_full( Default::default(), diff --git a/client/rpc/src/chain/tests.rs b/client/rpc/src/chain/tests.rs index bb673d65ea0f2..4c0ea8e8a43ae 100644 --- a/client/rpc/src/chain/tests.rs +++ b/client/rpc/src/chain/tests.rs @@ -30,7 +30,7 @@ use crate::testing::TaskExecutor; #[test] fn should_return_header() { - let client = Arc::new(substrate_test_runtime_client::new()); + let client = Arc::new(substrate_test_runtime_client::new(true)); let api = new_full(client.clone(), SubscriptionManager::new(Arc::new(TaskExecutor))); assert_matches!( @@ -62,7 +62,11 @@ fn should_return_header() { #[test] fn should_return_a_block() { - let mut client = Arc::new(substrate_test_runtime_client::new()); + should_return_a_block_inner(true); + should_return_a_block_inner(false); +} +fn should_return_a_block_inner(hashed_value: bool) { + let mut client = Arc::new(substrate_test_runtime_client::new(hashed_value)); let api = new_full(client.clone(), SubscriptionManager::new(Arc::new(TaskExecutor))); let block = client.new_block(Default::default()).unwrap().build().unwrap().block; @@ -113,7 +117,7 @@ fn should_return_a_block() { #[test] fn should_return_block_hash() { - let mut client = Arc::new(substrate_test_runtime_client::new()); + let mut client = Arc::new(substrate_test_runtime_client::new(true)); let api = new_full(client.clone(), SubscriptionManager::new(Arc::new(TaskExecutor))); assert_matches!( @@ -157,7 +161,7 @@ fn should_return_block_hash() { #[test] fn should_return_finalized_hash() { - let mut client = Arc::new(substrate_test_runtime_client::new()); + let mut client = Arc::new(substrate_test_runtime_client::new(true)); let api = new_full(client.clone(), SubscriptionManager::new(Arc::new(TaskExecutor))); assert_matches!( @@ -187,7 +191,7 @@ fn should_notify_about_latest_block() { let (subscriber, id, transport) = Subscriber::new_test("test"); { - let mut client = Arc::new(substrate_test_runtime_client::new()); + let mut client = Arc::new(substrate_test_runtime_client::new(true)); let api = new_full(client.clone(), SubscriptionManager::new(Arc::new(TaskExecutor))); api.subscribe_all_heads(Default::default(), subscriber); @@ -217,7 +221,7 @@ fn should_notify_about_best_block() { let (subscriber, id, transport) = Subscriber::new_test("test"); { - let mut client = Arc::new(substrate_test_runtime_client::new()); + let mut client = Arc::new(substrate_test_runtime_client::new(true)); let api = new_full(client.clone(), SubscriptionManager::new(Arc::new(TaskExecutor))); api.subscribe_new_heads(Default::default(), subscriber); @@ -247,7 +251,7 @@ fn should_notify_about_finalized_block() { let (subscriber, id, transport) = Subscriber::new_test("test"); { - let mut client = Arc::new(substrate_test_runtime_client::new()); + let mut client = Arc::new(substrate_test_runtime_client::new(true)); let api = new_full(client.clone(), SubscriptionManager::new(Arc::new(TaskExecutor))); api.subscribe_finalized_heads(Default::default(), subscriber); diff --git a/client/rpc/src/state/tests.rs b/client/rpc/src/state/tests.rs index cfc27c7bf525e..4529f515a9e20 100644 --- a/client/rpc/src/state/tests.rs +++ b/client/rpc/src/state/tests.rs @@ -138,7 +138,7 @@ fn should_return_child_storage() { #[test] fn should_call_contract() { - let client = Arc::new(substrate_test_runtime_client::new()); + let client = Arc::new(substrate_test_runtime_client::new(true)); let genesis_hash = client.genesis_hash(); let (client, _child) = new_full( client, @@ -157,7 +157,7 @@ fn should_notify_about_storage_changes() { let (subscriber, id, transport) = Subscriber::new_test("test"); { - let mut client = Arc::new(substrate_test_runtime_client::new()); + let mut client = Arc::new(substrate_test_runtime_client::new(true)); let (api, _child) = new_full( client.clone(), SubscriptionManager::new(Arc::new(TaskExecutor)), @@ -195,7 +195,7 @@ fn should_send_initial_storage_changes_and_notifications() { let (subscriber, id, transport) = Subscriber::new_test("test"); { - let mut client = Arc::new(substrate_test_runtime_client::new()); + let mut client = Arc::new(substrate_test_runtime_client::new(true)); let (api, _child) = new_full( client.clone(), SubscriptionManager::new(Arc::new(TaskExecutor)), @@ -435,7 +435,8 @@ fn should_query_storage() { ); } - run_tests(Arc::new(substrate_test_runtime_client::new()), false); + run_tests(Arc::new(substrate_test_runtime_client::new(true)), false); + run_tests(Arc::new(substrate_test_runtime_client::new(false)), false); run_tests( Arc::new( TestClientBuilder::new() @@ -444,6 +445,15 @@ fn should_query_storage() { ), true, ); + run_tests( + Arc::new( + TestClientBuilder::new() + .state_hashed_value() + .changes_trie_config(Some(ChangesTrieConfiguration::new(4, 2))) + .build(), + ), + true, + ); } #[test] @@ -455,10 +465,13 @@ fn should_split_ranges() { assert_eq!(split_range(100, Some(99)), (0..99, Some(99..100))); } - #[test] fn should_return_runtime_version() { - let client = Arc::new(substrate_test_runtime_client::new()); + should_return_runtime_version_inner(true); + should_return_runtime_version_inner(false); +} +fn should_return_runtime_version_inner(hashed_value: bool) { + let client = Arc::new(substrate_test_runtime_client::new(hashed_value)); let (api, _child) = new_full( client.clone(), SubscriptionManager::new(Arc::new(TaskExecutor)), @@ -485,7 +498,7 @@ fn should_notify_on_runtime_version_initially() { let (subscriber, id, transport) = Subscriber::new_test("test"); { - let client = Arc::new(substrate_test_runtime_client::new()); + let client = Arc::new(substrate_test_runtime_client::new(true)); let (api, _child) = new_full( client.clone(), SubscriptionManager::new(Arc::new(TaskExecutor)), diff --git a/client/service/test/src/client/light.rs b/client/service/test/src/client/light.rs index f9daf460e8eb5..d4e3220de73c6 100644 --- a/client/service/test/src/client/light.rs +++ b/client/service/test/src/client/light.rs @@ -305,6 +305,10 @@ fn light_aux_store_is_updated_via_non_importing_op() { #[test] fn execution_proof_is_generated_and_checked() { + execution_proof_is_generated_and_checked_inner(true); + execution_proof_is_generated_and_checked_inner(false); +} +fn execution_proof_is_generated_and_checked_inner(hashed_value: bool) { fn execute(remote_client: &TestClient, at: u64, method: &'static str) -> (Vec, Vec) { let remote_block_id = BlockId::Number(at); let remote_header = remote_client.header(&remote_block_id).unwrap().unwrap(); @@ -371,7 +375,7 @@ fn execution_proof_is_generated_and_checked() { } // prepare remote client - let mut remote_client = substrate_test_runtime_client::new(); + let mut remote_client = substrate_test_runtime_client::new(hashed_value); for i in 1u32..3u32 { let mut digest = Digest::default(); digest.push(sp_runtime::generic::DigestItem::Other::(i.to_le_bytes().to_vec())); @@ -451,9 +455,9 @@ type TestChecker = LightDataChecker< DummyStorage, >; -fn prepare_for_read_proof_check() -> (TestChecker, Header, StorageProof, u32) { +fn prepare_for_read_proof_check(hashed_value: bool) -> (TestChecker, Header, StorageProof, u32) { // prepare remote client - let remote_client = substrate_test_runtime_client::new(); + let remote_client = substrate_test_runtime_client::new(hashed_value); let remote_block_id = BlockId::Number(0); let flagged = false; let remote_block_hash = remote_client.block_hash(0).unwrap().unwrap(); @@ -536,9 +540,9 @@ fn prepare_for_read_child_proof_check() -> (TestChecker, Header, StorageProof, V (local_checker, remote_block_header, remote_read_proof, child_value) } -fn prepare_for_header_proof_check(insert_cht: bool) -> (TestChecker, Hash, Header, StorageProof) { +fn prepare_for_header_proof_check(insert_cht: bool, hashed_value: bool) -> (TestChecker, Hash, Header, StorageProof) { // prepare remote client - let mut remote_client = substrate_test_runtime_client::new(); + let mut remote_client = substrate_test_runtime_client::new(hashed_value); let mut local_headers_hashes = Vec::new(); for i in 0..4 { let block = remote_client.new_block(Default::default()).unwrap().build().unwrap().block; @@ -578,7 +582,16 @@ fn header_with_computed_extrinsics_root(extrinsics: Vec) -> Header { #[test] fn storage_read_proof_is_generated_and_checked() { - let (local_checker, remote_block_header, remote_read_proof, heap_pages) = prepare_for_read_proof_check(); + storage_read_proof_is_generated_and_checked_inner(true); + storage_read_proof_is_generated_and_checked_inner(false); +} +fn storage_read_proof_is_generated_and_checked_inner(hashed_value: bool) { + let ( + local_checker, + remote_block_header, + remote_read_proof, + heap_pages, + ) = prepare_for_read_proof_check(hashed_value); assert_eq!((&local_checker as &dyn FetchChecker).check_read_proof(&RemoteReadRequest::
{ block: remote_block_header.hash(), header: remote_block_header, @@ -610,7 +623,11 @@ fn storage_child_read_proof_is_generated_and_checked() { #[test] fn header_proof_is_generated_and_checked() { - let (local_checker, local_cht_root, remote_block_header, remote_header_proof) = prepare_for_header_proof_check(true); + header_proof_is_generated_and_checked_inner(true); + header_proof_is_generated_and_checked_inner(false); +} +fn header_proof_is_generated_and_checked_inner(hashed: bool) { + let (local_checker, local_cht_root, remote_block_header, remote_header_proof) = prepare_for_header_proof_check(true, hashed); assert_eq!((&local_checker as &dyn FetchChecker).check_header_proof(&RemoteHeaderRequest::
{ cht_root: local_cht_root, block: 1, @@ -620,7 +637,7 @@ fn header_proof_is_generated_and_checked() { #[test] fn check_header_proof_fails_if_cht_root_is_invalid() { - let (local_checker, _, mut remote_block_header, remote_header_proof) = prepare_for_header_proof_check(true); + let (local_checker, _, mut remote_block_header, remote_header_proof) = prepare_for_header_proof_check(true, true); remote_block_header.number = 100; assert!((&local_checker as &dyn FetchChecker).check_header_proof(&RemoteHeaderRequest::
{ cht_root: Default::default(), @@ -631,7 +648,12 @@ fn check_header_proof_fails_if_cht_root_is_invalid() { #[test] fn check_header_proof_fails_if_invalid_header_provided() { - let (local_checker, local_cht_root, mut remote_block_header, remote_header_proof) = prepare_for_header_proof_check(true); + let ( + local_checker, + local_cht_root, + mut remote_block_header, + remote_header_proof, + ) = prepare_for_header_proof_check(true, true); remote_block_header.number = 100; assert!((&local_checker as &dyn FetchChecker).check_header_proof(&RemoteHeaderRequest::
{ cht_root: local_cht_root, diff --git a/client/service/test/src/client/mod.rs b/client/service/test/src/client/mod.rs index 0b4a9ed76c56d..562a1aa830a91 100644 --- a/client/service/test/src/client/mod.rs +++ b/client/service/test/src/client/mod.rs @@ -340,7 +340,11 @@ fn construct_genesis_with_bad_transaction_should_panic() { #[test] fn client_initializes_from_genesis_ok() { - let client = substrate_test_runtime_client::new(); + client_initializes_from_genesis_ok_inner(false); + client_initializes_from_genesis_ok_inner(true); +} +fn client_initializes_from_genesis_ok_inner(hashed_value: bool) { + let client = substrate_test_runtime_client::new(hashed_value); assert_eq!( client.runtime_api().balance_of( @@ -360,7 +364,7 @@ fn client_initializes_from_genesis_ok() { #[test] fn block_builder_works_with_no_transactions() { - let mut client = substrate_test_runtime_client::new(); + let mut client = substrate_test_runtime_client::new(true); let block = client.new_block(Default::default()).unwrap().build().unwrap().block; @@ -371,7 +375,11 @@ fn block_builder_works_with_no_transactions() { #[test] fn block_builder_works_with_transactions() { - let mut client = substrate_test_runtime_client::new(); + block_builder_works_with_transactions_inner(true); + block_builder_works_with_transactions_inner(false); +} +fn block_builder_works_with_transactions_inner(hashed_value: bool) { + let mut client = substrate_test_runtime_client::new(hashed_value); let mut builder = client.new_block(Default::default()).unwrap(); @@ -408,7 +416,7 @@ fn block_builder_works_with_transactions() { #[test] fn block_builder_does_not_include_invalid() { - let mut client = substrate_test_runtime_client::new(); + let mut client = substrate_test_runtime_client::new(true); let mut builder = client.new_block(Default::default()).unwrap(); @@ -473,7 +481,7 @@ fn best_containing_with_hash_not_found() { fn uncles_with_only_ancestors() { // block tree: // G -> A1 -> A2 - let mut client = substrate_test_runtime_client::new(); + let mut client = substrate_test_runtime_client::new(true); // G -> A1 let a1 = client.new_block(Default::default()).unwrap().build().unwrap().block; @@ -493,7 +501,7 @@ fn uncles_with_multiple_forks() { // A1 -> B2 -> B3 -> B4 // B2 -> C3 // A1 -> D2 - let mut client = substrate_test_runtime_client::new(); + let mut client = substrate_test_runtime_client::new(true); // G -> A1 let a1 = client.new_block(Default::default()).unwrap().build().unwrap().block; @@ -1012,7 +1020,7 @@ fn key_changes_works() { #[test] fn import_with_justification() { - let mut client = substrate_test_runtime_client::new(); + let mut client = substrate_test_runtime_client::new(true); // G -> A1 let a1 = client.new_block(Default::default()).unwrap().build().unwrap().block; @@ -1058,7 +1066,7 @@ fn import_with_justification() { #[test] fn importing_diverged_finalized_block_should_trigger_reorg() { - let mut client = substrate_test_runtime_client::new(); + let mut client = substrate_test_runtime_client::new(true); // G -> A1 -> A2 // \ @@ -1115,7 +1123,9 @@ fn importing_diverged_finalized_block_should_trigger_reorg() { #[test] fn finalizing_diverged_block_should_trigger_reorg() { - let (mut client, select_chain) = TestClientBuilder::new().build_with_longest_chain(); + let (mut client, select_chain) = TestClientBuilder::new() + .state_hashed_value() + .build_with_longest_chain(); // G -> A1 -> A2 // \ @@ -1202,7 +1212,7 @@ fn finalizing_diverged_block_should_trigger_reorg() { #[test] fn get_header_by_block_number_doesnt_panic() { - let client = substrate_test_runtime_client::new(); + let client = substrate_test_runtime_client::new(true); // backend uses u32 for block numbers, make sure we don't panic when // trying to convert @@ -1213,7 +1223,7 @@ fn get_header_by_block_number_doesnt_panic() { #[test] fn state_reverted_on_reorg() { sp_tracing::try_init_simple(); - let mut client = substrate_test_runtime_client::new(); + let mut client = substrate_test_runtime_client::new(true); let current_balance = |client: &substrate_test_runtime_client::TestClient| client.runtime_api().balance_of( @@ -1741,7 +1751,11 @@ fn imports_blocks_with_changes_tries_config_change() { #[test] fn storage_keys_iter_prefix_and_start_key_works() { - let client = substrate_test_runtime_client::new(); + storage_keys_iter_prefix_and_start_key_works_inner(true); + storage_keys_iter_prefix_and_start_key_works_inner(false); +} +fn storage_keys_iter_prefix_and_start_key_works_inner(hashed_value: bool) { + let client = substrate_test_runtime_client::new(hashed_value); let prefix = StorageKey(hex!("3a").to_vec()); @@ -1766,7 +1780,11 @@ fn storage_keys_iter_prefix_and_start_key_works() { #[test] fn storage_keys_iter_works() { - let client = substrate_test_runtime_client::new(); + storage_keys_iter_works_inner(true); + storage_keys_iter_works_inner(false); +} +fn storage_keys_iter_works_inner(hashed_value: bool) { + let client = substrate_test_runtime_client::new(hashed_value); let prefix = StorageKey(hex!("").to_vec()); diff --git a/client/transaction-pool/src/testing/pool.rs b/client/transaction-pool/src/testing/pool.rs index 063947b383d03..325722ab02ede 100644 --- a/client/transaction-pool/src/testing/pool.rs +++ b/client/transaction-pool/src/testing/pool.rs @@ -932,7 +932,7 @@ fn ready_set_should_eventually_resolve_when_block_update_arrives() { fn should_not_accept_old_signatures() { use std::convert::TryFrom; - let client = Arc::new(substrate_test_runtime_client::new()); + let client = Arc::new(substrate_test_runtime_client::new(true)); let pool = Arc::new( BasicPool::new_test(Arc::new(FullChainApi::new(client, None))).0 @@ -968,7 +968,7 @@ fn should_not_accept_old_signatures() { #[test] fn import_notification_to_pool_maintain_works() { - let mut client = Arc::new(substrate_test_runtime_client::new()); + let mut client = Arc::new(substrate_test_runtime_client::new(true)); let pool = Arc::new( BasicPool::new_test(Arc::new(FullChainApi::new(client.clone(), None))).0 diff --git a/primitives/state-machine/src/proving_backend.rs b/primitives/state-machine/src/proving_backend.rs index 47b478b1d2da0..7c2d395f6069c 100644 --- a/primitives/state-machine/src/proving_backend.rs +++ b/primitives/state-machine/src/proving_backend.rs @@ -481,9 +481,7 @@ mod tests { let contents = value_range.clone().map(|i| (vec![i], Some(vec![i; size_content]))).collect::>(); let in_memory = InMemoryBackend::::default(); let mut in_memory = in_memory.update(vec![(None, contents)], false); - let in_memory_root = in_memory.storage_root(std::iter::empty(), false).0; value_range.clone().for_each(|i| assert_eq!(in_memory.storage(&[i]).unwrap().unwrap(), vec![i; size_content])); - in_memory = in_memory.update(vec![], true); let in_memory_root = in_memory.storage_root(std::iter::empty(), false).0; let trie = in_memory.as_trie_backend().unwrap(); diff --git a/primitives/trie/src/lib.rs b/primitives/trie/src/lib.rs index 33b3fd64e564c..c34522f64499e 100644 --- a/primitives/trie/src/lib.rs +++ b/primitives/trie/src/lib.rs @@ -1155,13 +1155,19 @@ mod tests { check_iteration::(&input); } - // TODO add flag - fn populate_trie<'db, T: TrieConfiguration>( + fn populate_trie<'db, T>( db: &'db mut dyn HashDB, root: &'db mut TrieHash, - v: &[(Vec, Vec)] - ) -> TrieDBMut<'db, T> { + v: &[(Vec, Vec)], + flag_hash: bool, + ) -> TrieDBMut<'db, T> + where + T: TrieConfiguration, + { let mut t = TrieDBMut::::new(db, root); + if flag_hash { + flag_meta_hasher(&mut t).unwrap(); + } for i in 0..v.len() { let key: &[u8]= &v[i].0; let val: &[u8] = &v[i].1; @@ -1182,6 +1188,10 @@ mod tests { #[test] fn random_should_work() { + random_should_work_inner(true); + random_should_work_inner(false); + } + fn random_should_work_inner(flag: bool) { let mut seed = ::Out::zero(); for test_i in 0..10000 { if test_i % 50 == 0 { @@ -1195,12 +1205,12 @@ mod tests { count: 100, }.make_with(seed.as_fixed_bytes_mut()); - // TODO test other layout states. let layout = Layout::default(); + // TODO implement variant for stream codec or use iter_build. let real = layout.trie_root(x.clone()); let mut memdb = MemoryDB::default(); let mut root = Default::default(); - let mut memtrie = populate_trie::(&mut memdb, &mut root, &x); + let mut memtrie = populate_trie::(&mut memdb, &mut root, &x, flag); memtrie.commit(); if *memtrie.root() != real { @@ -1260,7 +1270,6 @@ mod tests { #[test] fn codec_trie_two_tuples_disjoint_keys() { - // TODO switch to old layout let layout = Layout::default(); let input = vec![(&[0x48, 0x19], &[0xfe]), (&[0x13, 0x14], &[0xff])]; let trie = layout.trie_root_unhashed(input); @@ -1287,19 +1296,26 @@ mod tests { #[test] fn iterator_works() { - let pairs = vec![ + iterator_works_inner(true); + iterator_works_inner(false); + } + fn iterator_works_inner(flag: bool) { + let mut pairs = vec![ (hex!("0103000000000000000464").to_vec(), hex!("0400000000").to_vec()), (hex!("0103000000000000000469").to_vec(), hex!("0401000000").to_vec()), ]; let mut mdb = MemoryDB::default(); let mut root = Default::default(); - let _ = populate_trie::(&mut mdb, &mut root, &pairs); + let _ = populate_trie::(&mut mdb, &mut root, &pairs, flag); let trie = TrieDB::::new(&mdb, &root).unwrap(); let iter = trie.iter().unwrap(); let mut iter_pairs = Vec::new(); + if flag { + pairs.insert(0, (vec![], vec![])); + } for pair in iter { let (key, value) = pair.unwrap(); iter_pairs.push((key, value.to_vec())); diff --git a/test-utils/client/src/lib.rs b/test-utils/client/src/lib.rs index d8cc40d5561c1..c4cb965e7c834 100644 --- a/test-utils/client/src/lib.rs +++ b/test-utils/client/src/lib.rs @@ -80,6 +80,7 @@ pub struct TestClientBuilder { fork_blocks: ForkBlocks, bad_blocks: BadBlocks, enable_offchain_indexing_api: bool, + state_hashed_value: bool, } impl Default @@ -116,6 +117,7 @@ impl TestClientBuilder TestClientBuilder Self { + self.state_hashed_value = true; + self + } + /// Build the test client with the given native executor. pub fn build_with_executor( self, @@ -202,6 +210,9 @@ impl TestClientBuilder for LightFetcher { } /// Creates new client instance used for tests. -pub fn new() -> Client { - TestClientBuilder::new().build() +pub fn new(hashed_state: bool) -> Client { + let mut builder = TestClientBuilder::new(); + if hashed_state { + builder = builder.state_hashed_value(); + } + builder.build() } /// Creates new light client instance used for tests. -pub fn new_light() -> ( +pub fn new_light(hashed_state: bool) -> ( client::Client, Arc, ) { @@ -368,10 +372,12 @@ pub fn new_light() -> ( local_call_executor, ); + let mut builder = TestClientBuilder::with_backend(backend.clone()); + if hashed_state { + builder = builder.state_hashed_value(); + } ( - TestClientBuilder::with_backend(backend.clone()) - .build_with_executor(call_executor) - .0, + builder.build_with_executor(call_executor).0, backend, ) } diff --git a/utils/frame/rpc/system/src/lib.rs b/utils/frame/rpc/system/src/lib.rs index bbc51a28a59cd..2409afca66994 100644 --- a/utils/frame/rpc/system/src/lib.rs +++ b/utils/frame/rpc/system/src/lib.rs @@ -294,7 +294,7 @@ mod tests { sp_tracing::try_init_simple(); // given - let client = Arc::new(substrate_test_runtime_client::new()); + let client = Arc::new(substrate_test_runtime_client::new(true)); let spawner = sp_core::testing::TaskExecutor::new(); let pool = BasicPool::new_full( Default::default(), @@ -334,7 +334,7 @@ mod tests { sp_tracing::try_init_simple(); // given - let client = Arc::new(substrate_test_runtime_client::new()); + let client = Arc::new(substrate_test_runtime_client::new(true,)); let spawner = sp_core::testing::TaskExecutor::new(); let pool = BasicPool::new_full( Default::default(), @@ -358,7 +358,7 @@ mod tests { sp_tracing::try_init_simple(); // given - let client = Arc::new(substrate_test_runtime_client::new()); + let client = Arc::new(substrate_test_runtime_client::new(true)); let spawner = sp_core::testing::TaskExecutor::new(); let pool = BasicPool::new_full( Default::default(), @@ -391,7 +391,7 @@ mod tests { sp_tracing::try_init_simple(); // given - let client = Arc::new(substrate_test_runtime_client::new()); + let client = Arc::new(substrate_test_runtime_client::new(true)); let spawner = sp_core::testing::TaskExecutor::new(); let pool = BasicPool::new_full( Default::default(), From aa2e5c31ed25b8f973232ce5111893bb759bd957 Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 20 May 2021 13:11:35 +0200 Subject: [PATCH 014/127] fix chain spec test with serde default. --- client/basic-authorship/src/lib.rs | 2 +- client/chain-spec/src/chain_spec.rs | 1 + primitives/api/test/benches/bench.rs | 8 ++++---- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/client/basic-authorship/src/lib.rs b/client/basic-authorship/src/lib.rs index 133b833cdddc8..371307f8cdac2 100644 --- a/client/basic-authorship/src/lib.rs +++ b/client/basic-authorship/src/lib.rs @@ -30,7 +30,7 @@ //! # DefaultTestClientBuilderExt, TestClientBuilderExt, //! # }; //! # use sc_transaction_pool::{BasicPool, FullChainApi}; -//! # let client = Arc::new(substrate_test_runtime_client::new()); +//! # let client = Arc::new(substrate_test_runtime_client::new(true)); //! # let spawner = sp_core::testing::TaskExecutor::new(); //! # let txpool = BasicPool::new_full( //! # Default::default(), diff --git a/client/chain-spec/src/chain_spec.rs b/client/chain-spec/src/chain_spec.rs index 3838c5dbc0def..a18ad2949e52f 100644 --- a/client/chain-spec/src/chain_spec.rs +++ b/client/chain-spec/src/chain_spec.rs @@ -133,6 +133,7 @@ pub type GenesisStorage = HashMap; pub struct RawGenesis { pub top: GenesisStorage, pub children_default: HashMap, + #[serde(default)] pub flag_hashed_value: bool, } diff --git a/primitives/api/test/benches/bench.rs b/primitives/api/test/benches/bench.rs index 20ddbbe7116dc..a9fe79d1abcec 100644 --- a/primitives/api/test/benches/bench.rs +++ b/primitives/api/test/benches/bench.rs @@ -26,7 +26,7 @@ use sp_api::ProvideRuntimeApi; fn sp_api_benchmark(c: &mut Criterion) { c.bench_function("add one with same runtime api", |b| { - let client = substrate_test_runtime_client::new(); + let client = substrate_test_runtime_client::new(ture); let runtime_api = client.runtime_api(); let block_id = BlockId::Number(client.chain_info().best_number); @@ -34,14 +34,14 @@ fn sp_api_benchmark(c: &mut Criterion) { }); c.bench_function("add one with recreating runtime api", |b| { - let client = substrate_test_runtime_client::new(); + let client = substrate_test_runtime_client::new(true); let block_id = BlockId::Number(client.chain_info().best_number); b.iter(|| client.runtime_api().benchmark_add_one(&block_id, &1)) }); c.bench_function("vector add one with same runtime api", |b| { - let client = substrate_test_runtime_client::new(); + let client = substrate_test_runtime_client::new(true); let runtime_api = client.runtime_api(); let block_id = BlockId::Number(client.chain_info().best_number); let data = vec![0; 1000]; @@ -50,7 +50,7 @@ fn sp_api_benchmark(c: &mut Criterion) { }); c.bench_function("vector add one with recreating runtime api", |b| { - let client = substrate_test_runtime_client::new(); + let client = substrate_test_runtime_client::new(true); let block_id = BlockId::Number(client.chain_info().best_number); let data = vec![0; 1000]; From 91bd0aa861095c816702207bf714e519de716b9c Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 20 May 2021 14:01:35 +0200 Subject: [PATCH 015/127] export state (missing trie function). --- bin/node/bench/src/generator.rs | 4 ++++ bin/node/bench/src/trie.rs | 3 ++- client/api/src/backend.rs | 4 ++++ client/db/src/bench.rs | 4 ++++ client/db/src/lib.rs | 9 ++++++++- client/db/src/storage_cache.rs | 8 ++++++++ client/light/src/backend.rs | 7 +++++++ client/service/src/chain_ops/export_raw_state.rs | 2 +- client/service/src/client/client.rs | 4 ++++ primitives/state-machine/src/backend.rs | 7 +++++++ primitives/state-machine/src/proving_backend.rs | 4 ++++ primitives/state-machine/src/trie_backend.rs | 4 ++++ primitives/state-machine/src/trie_backend_essence.rs | 5 +++++ primitives/trie/src/lib.rs | 11 +++++++++++ 14 files changed, 73 insertions(+), 3 deletions(-) diff --git a/bin/node/bench/src/generator.rs b/bin/node/bench/src/generator.rs index c540ae147c9f0..a227c9081bb67 100644 --- a/bin/node/bench/src/generator.rs +++ b/bin/node/bench/src/generator.rs @@ -31,6 +31,7 @@ use crate::simple_trie::SimpleTrie; pub fn generate_trie( db: Arc, key_values: impl IntoIterator, Vec)>, + flag_hashed_value: bool, ) -> Hash { let mut root = Hash::default(); @@ -44,6 +45,9 @@ pub fn generate_trie( { let mut trie_db = TrieDBMut::new(&mut trie, &mut root); + if flag_hashed_value { + sp_trie::flag_meta_hasher(&mut trie_db).expect("flag trie failed"); + } for (key, value) in key_values { trie_db.insert(&key, &value).expect("trie insertion failed"); } diff --git a/bin/node/bench/src/trie.rs b/bin/node/bench/src/trie.rs index 267d0cc16c8c4..56bf7fe8ad604 100644 --- a/bin/node/bench/src/trie.rs +++ b/bin/node/bench/src/trie.rs @@ -117,7 +117,6 @@ impl core::BenchmarkDescription for TrieReadBenchmarkDescription { let mut rng = rand::thread_rng(); let warmup_prefix = KUSAMA_STATE_DISTRIBUTION.key(&mut rng); - // TODO flag trie for hash of value. let mut key_values = KeyValues::new(); let mut warmup_keys = KeyValues::new(); let mut query_keys = KeyValues::new(); @@ -146,6 +145,7 @@ impl core::BenchmarkDescription for TrieReadBenchmarkDescription { let root = generate_trie( database.open(self.database_type), key_values, + true, ); Box::new(TrieReadBenchmark { @@ -263,6 +263,7 @@ impl core::BenchmarkDescription for TrieWriteBenchmarkDescription { let root = generate_trie( database.open(self.database_type), key_values, + true, ); Box::new(TrieWriteBenchmark { diff --git a/client/api/src/backend.rs b/client/api/src/backend.rs index 09e9e0cb2e173..bd5e15c0222fd 100644 --- a/client/api/src/backend.rs +++ b/client/api/src/backend.rs @@ -385,6 +385,10 @@ pub trait StorageProvider> { storage_key: Option<&PrefixedStorageKey>, key: &StorageKey ) -> sp_blockchain::Result, u32)>>; + + /// Returns true when state allow hashing value and therefore + /// removing unaccess value from proofs. + fn state_hashed_value(&self, id: &BlockId) -> sp_blockchain::Result; } /// Client backend. diff --git a/client/db/src/bench.rs b/client/db/src/bench.rs index fdc53ab224966..21e20d1ff6211 100644 --- a/client/db/src/bench.rs +++ b/client/db/src/bench.rs @@ -528,6 +528,10 @@ impl StateBackend> for BenchmarkingState { fn proof_size(&self) -> Option { self.proof_recorder.as_ref().map(|recorder| recorder.estimate_encoded_size() as u32) } + + fn state_hashed_value(&self) -> bool { + self.state.borrow().as_ref().map_or(Default::default(), |s| s.state_hashed_value()) + } } impl std::fmt::Debug for BenchmarkingState { diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index c465598632396..2fa447b5e9b84 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -269,6 +269,10 @@ impl StateBackend> for RefTrackingState { fn usage_info(&self) -> StateUsageInfo { self.state.usage_info() } + + fn state_hashed_value(&self) -> bool { + self.state.state_hashed_value() + } } /// Database settings. @@ -2286,7 +2290,10 @@ pub(crate) mod tests { #[test] fn set_state_data() { - let flagged = false; // TODO test with flagged + set_state_data_inner(true); + set_state_data_inner(false); + } + fn set_state_data_inner(flagged: bool) { let db = Backend::::new_test(2, 0); let hash = { let mut op = db.begin_operation().unwrap(); diff --git a/client/db/src/storage_cache.rs b/client/db/src/storage_cache.rs index eeb5092fa6c8f..e6de7774f257f 100644 --- a/client/db/src/storage_cache.rs +++ b/client/db/src/storage_cache.rs @@ -687,6 +687,10 @@ impl>, B: BlockT> StateBackend> for Cachin info.include_state_machine_states(&self.overlay_stats); info } + + fn state_hashed_value(&self) -> bool { + self.state.state_hashed_value() + } } /// Extended [`CachingState`] that will sync the caches on drop. @@ -871,6 +875,10 @@ impl>, B: BlockT> StateBackend> for Syncin fn usage_info(&self) -> sp_state_machine::UsageInfo { self.caching_state().usage_info() } + + fn state_hashed_value(&self) -> bool { + self.caching_state().state_hashed_value() + } } impl Drop for SyncingCachingState { diff --git a/client/light/src/backend.rs b/client/light/src/backend.rs index 49c861411bb45..ea99273f8c869 100644 --- a/client/light/src/backend.rs +++ b/client/light/src/backend.rs @@ -540,4 +540,11 @@ impl StateBackend for GenesisOrUnavailableState GenesisOrUnavailableState::Unavailable => None, } } + + fn state_hashed_value(&self) -> bool { + match self { + GenesisOrUnavailableState::Genesis(state) => state.state_hashed_value(), + GenesisOrUnavailableState::Unavailable => false, + } + } } diff --git a/client/service/src/chain_ops/export_raw_state.rs b/client/service/src/chain_ops/export_raw_state.rs index bdba4911f8db6..fe96994ed2605 100644 --- a/client/service/src/chain_ops/export_raw_state.rs +++ b/client/service/src/chain_ops/export_raw_state.rs @@ -42,7 +42,7 @@ where let empty_key = StorageKey(Vec::new()); let mut top_storage = client.storage_pairs(&block, &empty_key)?; let mut children_default = HashMap::new(); - let flag_hashed_value = false; // TODO read from trie (through clinet api). + let flag_hashed_value = client.state_hashed_value(&block)?; // Remove all default child storage roots from the top storage and collect the child storage // pairs. diff --git a/client/service/src/client/client.rs b/client/service/src/client/client.rs index a958cb6865c79..661d1c9456dd6 100644 --- a/client/service/src/client/client.rs +++ b/client/service/src/client/client.rs @@ -1525,6 +1525,10 @@ impl StorageProvider for Client wher Ok(result) } + + fn state_hashed_value(&self, id: &BlockId) -> sp_blockchain::Result { + Ok(self.state_at(id)?.state_hashed_value()) + } } impl HeaderMetadata for Client where diff --git a/primitives/state-machine/src/backend.rs b/primitives/state-machine/src/backend.rs index 90923b19b21c9..771d51f274678 100644 --- a/primitives/state-machine/src/backend.rs +++ b/primitives/state-machine/src/backend.rs @@ -254,6 +254,9 @@ pub trait Backend: sp_std::fmt::Debug { fn proof_size(&self) -> Option { unimplemented!() } + + /// Does trie state allow hashing of value. + fn state_hashed_value(&self) -> bool; } impl<'a, T: Backend, H: Hasher> Backend for &'a T { @@ -335,6 +338,10 @@ impl<'a, T: Backend, H: Hasher> Backend for &'a T { fn usage_info(&self) -> UsageInfo { (*self).usage_info() } + + fn state_hashed_value(&self) -> bool { + (*self).state_hashed_value() + } } /// Trait that allows consolidate two transactions together. diff --git a/primitives/state-machine/src/proving_backend.rs b/primitives/state-machine/src/proving_backend.rs index 7c2d395f6069c..81ad50408e4c4 100644 --- a/primitives/state-machine/src/proving_backend.rs +++ b/primitives/state-machine/src/proving_backend.rs @@ -361,6 +361,10 @@ impl<'a, S, H> Backend for ProvingBackend<'a, S, H> fn usage_info(&self) -> crate::stats::UsageInfo { self.0.usage_info() } + + fn state_hashed_value(&self) -> bool { + self.0.state_hashed_value() + } } /// Create proof check backend. diff --git a/primitives/state-machine/src/trie_backend.rs b/primitives/state-machine/src/trie_backend.rs index 986f6db63670f..e110963a20817 100644 --- a/primitives/state-machine/src/trie_backend.rs +++ b/primitives/state-machine/src/trie_backend.rs @@ -255,6 +255,10 @@ impl, H: Hasher> Backend for TrieBackend where fn wipe(&self) -> Result<(), Self::Error> { Ok(()) } + + fn state_hashed_value(&self) -> bool { + self.essence.state_hashed_value() + } } #[cfg(test)] diff --git a/primitives/state-machine/src/trie_backend_essence.rs b/primitives/state-machine/src/trie_backend_essence.rs index 594abf38fa698..6d417204ebd8d 100644 --- a/primitives/state-machine/src/trie_backend_essence.rs +++ b/primitives/state-machine/src/trie_backend_essence.rs @@ -169,6 +169,11 @@ impl, H: Hasher> TrieBackendEssence where H::Out: Ok(next_key) } + /// Does current trie use inner hashed value. + pub fn state_hashed_value(&self) -> bool { + sp_trie::state_hashed_value::, _>(self, &self.root) + } + /// Get the value of storage at given key. pub fn storage(&self, key: &[u8]) -> Result> { let map_e = |e| format!("Trie lookup error: {}", e); diff --git a/primitives/trie/src/lib.rs b/primitives/trie/src/lib.rs index c34522f64499e..8b497a8a7a280 100644 --- a/primitives/trie/src/lib.rs +++ b/primitives/trie/src/lib.rs @@ -594,6 +594,17 @@ pub fn flag_meta_hasher( Ok(()) } +/// Resolve if inner hashing of value is active. +pub fn state_hashed_value>( + db: &DB, + root: &TrieHash, +) -> bool { + if let Ok(t) = TrieDB::::new(&*db, root) { + unimplemented!("TODO has_flag on triedb"); + } + false +} + /// Read a value from the trie. pub fn read_trie_value>( db: &DB, From 183b7b56b61541ad1e5c2dbc3b8ad9dd2bf386c1 Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Fri, 21 May 2021 18:51:09 +0200 Subject: [PATCH 016/127] Pending using new branch, lacking genericity on layout resolution. --- Cargo.lock | 41 ++++----- Cargo.toml | 9 +- primitives/trie/src/lib.rs | 148 ++++++++++++++---------------- primitives/trie/src/node_codec.rs | 10 +- 4 files changed, 99 insertions(+), 109 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 79c33175dcfc3..897eff025d517 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2329,7 +2329,7 @@ dependencies = [ [[package]] name = "hash-db" version = "0.15.2" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple2#ab6d8bbc73d620af5357f350fa132142c2d57a33" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#0b365f776e3716033f44f0fc9cbff7fd389a6c34" [[package]] name = "hash256-std-hasher" @@ -2343,7 +2343,7 @@ dependencies = [ [[package]] name = "hash256-std-hasher" version = "0.15.2" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple2#ab6d8bbc73d620af5357f350fa132142c2d57a33" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#0b365f776e3716033f44f0fc9cbff7fd389a6c34" dependencies = [ "crunchy", ] @@ -2996,10 +2996,10 @@ dependencies = [ [[package]] name = "keccak-hasher" version = "0.15.3" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple2#ab6d8bbc73d620af5357f350fa132142c2d57a33" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#0b365f776e3716033f44f0fc9cbff7fd389a6c34" dependencies = [ "hash-db", - "hash256-std-hasher 0.15.2 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple2)", + "hash256-std-hasher 0.15.2 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple4)", "tiny-keccak", ] @@ -3788,7 +3788,7 @@ dependencies = [ [[package]] name = "memory-db" version = "0.26.0" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple2#ab6d8bbc73d620af5357f350fa132142c2d57a33" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#0b365f776e3716033f44f0fc9cbff7fd389a6c34" dependencies = [ "hash-db", "hashbrown", @@ -4217,7 +4217,7 @@ dependencies = [ "sp-state-machine", "sp-trie", "substrate-test-client", - "trie-root 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)", + "trie-root", "wat", ] @@ -9177,7 +9177,7 @@ dependencies = [ "thiserror", "tracing", "trie-db", - "trie-root 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)", + "trie-root", ] [[package]] @@ -9283,7 +9283,7 @@ dependencies = [ "sp-std", "trie-bench", "trie-db", - "trie-root 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)", + "trie-root", "trie-standardmap 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -10332,22 +10332,22 @@ checksum = "a7f741b240f1a48843f9b8e0444fb55fb2a4ff67293b50a9179dfd5ea67f8d41" [[package]] name = "trie-bench" version = "0.27.0" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple2#ab6d8bbc73d620af5357f350fa132142c2d57a33" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#0b365f776e3716033f44f0fc9cbff7fd389a6c34" dependencies = [ "criterion", "hash-db", - "keccak-hasher 0.15.3 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple2)", + "keccak-hasher 0.15.3 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple4)", "memory-db", "parity-scale-codec", "trie-db", - "trie-root 0.16.0 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple2)", - "trie-standardmap 0.15.2 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple2)", + "trie-root", + "trie-standardmap 0.15.2 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple4)", ] [[package]] name = "trie-db" version = "0.22.3" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple2#ab6d8bbc73d620af5357f350fa132142c2d57a33" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#0b365f776e3716033f44f0fc9cbff7fd389a6c34" dependencies = [ "hash-db", "hashbrown", @@ -10359,16 +10359,7 @@ dependencies = [ [[package]] name = "trie-root" version = "0.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "652931506d2c1244d7217a70b99f56718a7b4161b37f04e7cd868072a99f68cd" -dependencies = [ - "hash-db", -] - -[[package]] -name = "trie-root" -version = "0.16.0" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple2#ab6d8bbc73d620af5357f350fa132142c2d57a33" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#0b365f776e3716033f44f0fc9cbff7fd389a6c34" dependencies = [ "hash-db", ] @@ -10386,10 +10377,10 @@ dependencies = [ [[package]] name = "trie-standardmap" version = "0.15.2" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple2#ab6d8bbc73d620af5357f350fa132142c2d57a33" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#0b365f776e3716033f44f0fc9cbff7fd389a6c34" dependencies = [ "hash-db", - "keccak-hasher 0.15.3 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple2)", + "keccak-hasher 0.15.3 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple4)", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 1bda1bc4f5ec2..3133235b02ad5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -271,7 +271,8 @@ zeroize = { opt-level = 3 } panic = "unwind" [patch.crates-io] -hash-db = { git = "https://github.com/cheme/trie.git", branch = "hashed-value-simple2" } -memory-db = { git = "https://github.com/cheme/trie.git", branch = "hashed-value-simple2" } -trie-db = { git = "https://github.com/cheme/trie.git", branch = "hashed-value-simple2" } -trie-bench = { git = "https://github.com/cheme/trie.git", branch = "hashed-value-simple2" } +hash-db = { git = "https://github.com/cheme/trie.git", branch = "hashed-value-simple4" } +memory-db = { git = "https://github.com/cheme/trie.git", branch = "hashed-value-simple4" } +trie-db = { git = "https://github.com/cheme/trie.git", branch = "hashed-value-simple4" } +trie-root = { git = "https://github.com/cheme/trie.git", branch = "hashed-value-simple4" } +trie-bench = { git = "https://github.com/cheme/trie.git", branch = "hashed-value-simple4" } diff --git a/primitives/trie/src/lib.rs b/primitives/trie/src/lib.rs index 8b497a8a7a280..34966333c23bc 100644 --- a/primitives/trie/src/lib.rs +++ b/primitives/trie/src/lib.rs @@ -39,7 +39,7 @@ pub use storage_proof::StorageProof; /// Various re-exports from the `trie-db` crate. pub use trie_db::{ Trie, TrieMut, DBValue, Recorder, CError, Query, TrieLayout, TrieConfiguration, - nibble_ops, TrieDBIterator, Meta, NodeChange, node::{NodePlan, ValuePlan}, + nibble_ops, TrieDBIterator, Meta, node::{NodePlan, ValuePlan}, GlobalMeta, }; /// Various re-exports from the `memory-db` crate. pub use memory_db::KeyFunction; @@ -79,7 +79,7 @@ pub struct TrieMeta { impl Meta for TrieMeta { /// Layout do not have content. - type MetaInput = (); + type GlobalMeta = bool; /// When true apply inner hashing of value. type StateMeta = bool; @@ -115,33 +115,23 @@ impl Meta for TrieMeta { } fn meta_for_new( - _input: Self::MetaInput, - parent: Option<&Self>, + global: Self::GlobalMeta, ) -> Self { let mut result = Self::default(); - result.do_value_hash = parent.map(|p| p.do_value_hash).unwrap_or_default(); + result.do_value_hash = global; result } fn meta_for_existing_inline_node( - input: Self::MetaInput, - parent: Option<&Self>, + global: Self::GlobalMeta, ) -> Self { - Self::meta_for_new(input, parent) + Self::meta_for_new(global) } fn meta_for_empty( + global: Self::GlobalMeta, ) -> Self { - Default::default() - } - - fn set_value_callback( - &mut self, - _new_value: Option<&[u8]>, - _is_branch: bool, - changed: NodeChange, - ) -> NodeChange { - changed + Self::meta_for_new(global) } fn encoded_value_callback( @@ -162,15 +152,6 @@ impl Meta for TrieMeta { } } - fn set_child_callback( - &mut self, - _child: Option<&Self>, - changed: NodeChange, - _at: usize, - ) -> NodeChange { - changed - } - fn decoded_callback( &mut self, node_plan: &NodePlan, @@ -209,7 +190,7 @@ impl TrieMeta { } /// substrate trie layout -pub struct Layout(sp_std::marker::PhantomData<(H, M)>); +pub struct Layout(bool, sp_std::marker::PhantomData<(H, M)>); impl fmt::Debug for Layout { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { @@ -217,37 +198,46 @@ impl fmt::Debug for Layout { } } -impl Default for Layout { - fn default() -> Self { - Layout(sp_std::marker::PhantomData) +impl Clone for Layout { + fn clone(&self) -> Self { + Layout(self.0, sp_std::marker::PhantomData) } } -impl Clone for Layout { - fn clone(&self) -> Self { - Layout(sp_std::marker::PhantomData) +impl Default for Layout { + fn default() -> Self { + Layout(false, sp_std::marker::PhantomData) } } impl TrieLayout for Layout where H: Hasher, - M: MetaHasher, - M::Meta: Meta, + M: MetaHasher, + M::Meta: Meta, { const USE_EXTENSION: bool = false; const ALLOW_EMPTY: bool = true; const USE_META: bool = true; + const READ_ROOT_STATE_META: bool = true; + type Hash = H; type Codec = NodeCodec; type MetaHasher = M; type Meta = M::Meta; - fn metainput_for_new_node(&self) -> ::MetaInput { - () + fn layout_meta(&self) -> GlobalMeta { + self.0 + } + fn initialize_from_root_meta(&mut self, root_meta: &Self::Meta) { + if root_meta.recorded_do_value_hash { + self.0 = true; + } } - fn metainput_for_stored_inline_node(&self) -> ::MetaInput { - () + fn set_root_meta(root_meta: &mut Self::Meta, global_meta: GlobalMeta) { + if global_meta { + root_meta.recorded_do_value_hash = true; + } } } @@ -260,6 +250,7 @@ impl MetaHasher for StateHasher H: Hasher, { type Meta = TrieMeta; + type GlobalMeta = bool; fn hash(value: &[u8], meta: &Self::Meta) -> H::Out { match &meta { @@ -328,7 +319,7 @@ impl MetaHasher for StateHasher >::stored_value(value.as_slice(), meta) } - fn extract_value<'a>(mut stored: &'a [u8], parent_meta: Option<&Self::Meta>) -> (&'a [u8], Self::Meta) { + fn extract_value(mut stored: &[u8], global_meta: Self::GlobalMeta) -> (&[u8], Self::Meta) { let input = &mut stored; let mut contain_hash = false; let mut old_hash = false; @@ -352,13 +343,13 @@ impl MetaHasher for StateHasher let _offset = meta.read_state_meta(stored) .expect("State meta reading failure."); //let stored = &stored[offset..]; - meta.do_value_hash = meta.recorded_do_value_hash || parent_meta.map(|m| m.do_value_hash).unwrap_or(false); + meta.do_value_hash = meta.recorded_do_value_hash || global_meta; (stored, meta) } - fn extract_value_owned(mut stored: DBValue, parent_meta: Option<&Self::Meta>) -> (DBValue, Self::Meta) { + fn extract_value_owned(mut stored: DBValue, global: Self::GlobalMeta) -> (DBValue, Self::Meta) { let len = stored.len(); - let (v, meta) = >::extract_value(stored.as_slice(), parent_meta); + let (v, meta) = >::extract_value(stored.as_slice(), global); let removed = len - v.len(); (stored.split_off(removed), meta) } @@ -374,6 +365,7 @@ impl MetaHasher for NoMetaHasher H: Hasher, { type Meta = TrieMeta; + type GlobalMeta = bool; fn hash(value: &[u8], _meta: &Self::Meta) -> H::Out { H::hash(value) @@ -387,11 +379,11 @@ impl MetaHasher for NoMetaHasher value } - fn extract_value<'a>(stored: &'a [u8], _parent_meta: Option<&Self::Meta>) -> (&'a [u8], Self::Meta) { + fn extract_value(stored: &[u8], _meta: Self::GlobalMeta) -> (&[u8], Self::Meta) { (stored, Default::default()) } - fn extract_value_owned(stored: DBValue, _parent_meta: Option<&Self::Meta>) -> (DBValue, Self::Meta) { + fn extract_value_owned(stored: DBValue, _meta: Self::GlobalMeta) -> (DBValue, Self::Meta) { (stored, Default::default()) } } @@ -399,15 +391,15 @@ impl MetaHasher for NoMetaHasher impl TrieConfiguration for Layout where H: Hasher, - M: MetaHasher, - M::Meta: Meta, + M: MetaHasher, + M::Meta: Meta, { fn trie_root(&self, input: I) -> ::Out where I: IntoIterator, A: AsRef<[u8]> + Ord, B: AsRef<[u8]>, { - trie_root::trie_root_no_extension::(input) + trie_root::trie_root_no_extension::(input, self.clone()) } fn trie_root_unhashed(&self, input: I) -> Vec where @@ -415,7 +407,7 @@ impl TrieConfiguration for Layout A: AsRef<[u8]> + Ord, B: AsRef<[u8]>, { - trie_root::unhashed_trie_no_extension::(input) + trie_root::unhashed_trie_no_extension::(input, self.clone()) } fn encode_index(input: u32) -> Vec { @@ -431,10 +423,10 @@ type MemTracker = memory_db::MemCounter; /// TrieDB error over `TrieConfiguration` trait. pub type TrieError = trie_db::TrieError, CError>; /// Reexport from `hash_db`, with genericity set for `Hasher` trait. -pub trait AsHashDB: hash_db::AsHashDB {} -impl> AsHashDB for T {} +pub trait AsHashDB: hash_db::AsHashDB {} +impl> AsHashDB for T {} /// Reexport from `hash_db`, with genericity set for `Hasher` trait. -pub type HashDB<'a, H, M> = dyn hash_db::HashDB + 'a; +pub type HashDB<'a, H, M, GM> = dyn hash_db::HashDB + 'a; /// Reexport from `hash_db`, with genericity set for `Hasher` trait. /// This uses a `KeyFunction` for prefixing keys internally (avoiding /// key conflict for non random keys). @@ -545,7 +537,7 @@ pub fn delta_trie_root( A: Borrow<[u8]>, B: Borrow>, V: Borrow<[u8]>, - DB: hash_db::HashDB, + DB: hash_db::HashDB>, { { let mut trie = TrieDBMut::::from_existing(db, &mut root)?; @@ -570,7 +562,7 @@ pub fn flag_inner_meta_hasher( mut root: TrieHash, ) -> Result, Box>> where L: TrieConfiguration, - DB: hash_db::HashDB, + DB: hash_db::HashDB>, { { let mut t = TrieDBMut::::from_existing(db, &mut root)?; @@ -595,7 +587,7 @@ pub fn flag_meta_hasher( } /// Resolve if inner hashing of value is active. -pub fn state_hashed_value>( +pub fn state_hashed_value>>( db: &DB, root: &TrieHash, ) -> bool { @@ -606,7 +598,7 @@ pub fn state_hashed_value>( +pub fn read_trie_value>>( db: &DB, root: &TrieHash, key: &[u8] @@ -617,8 +609,8 @@ pub fn read_trie_value, - DB: hash_db::HashDBRef + Q: Query, + DB: hash_db::HashDBRef> >( db: &DB, root: &TrieHash, @@ -666,7 +658,7 @@ pub fn child_delta_trie_root( B: Borrow>, V: Borrow<[u8]>, RD: AsRef<[u8]>, - DB: hash_db::HashDB + DB: hash_db::HashDB>, { let mut root = TrieHash::::default(); // root is fetched from DB, not writable by runtime, so it's always valid. @@ -689,7 +681,7 @@ pub fn for_keys_in_child_trie bool, DB> mut f: F ) -> Result<(), Box>> where - DB: hash_db::HashDBRef + DB: hash_db::HashDBRef>, { let mut root = TrieHash::::default(); // root is fetched from DB, not writable by runtime, so it's always valid. @@ -713,9 +705,9 @@ pub fn for_keys_in_child_trie bool, DB> pub fn record_all_keys( db: &DB, root: &TrieHash, - recorder: &mut Recorder> + recorder: &mut Recorder, L::Meta> ) -> Result<(), Box>> where - DB: hash_db::HashDBRef + DB: hash_db::HashDBRef>, { let trie = TrieDB::::new(&*db, root)?; let iter = trie.iter()?; @@ -740,7 +732,7 @@ pub fn read_child_trie_value( key: &[u8] ) -> Result>, Box>> where - DB: hash_db::HashDBRef + DB: hash_db::HashDBRef>, { let mut root = TrieHash::::default(); // root is fetched from DB, not writable by runtime, so it's always valid. @@ -751,7 +743,7 @@ pub fn read_child_trie_value( } /// Read a value from the child trie with given query. -pub fn read_child_trie_value_with, DB>( +pub fn read_child_trie_value_with, DB>( keyspace: &[u8], db: &DB, root_slice: &[u8], @@ -759,7 +751,7 @@ pub fn read_child_trie_value_with Result>, Box>> where - DB: hash_db::HashDBRef + DB: hash_db::HashDBRef>, { let mut root = TrieHash::::default(); // root is fetched from DB, not writable by runtime, so it's always valid. @@ -806,8 +798,8 @@ impl<'a, DB, H> KeySpacedDBMut<'a, DB, H> where } } -impl<'a, DB, H, T, M> hash_db::HashDBRef for KeySpacedDB<'a, DB, H> where - DB: hash_db::HashDBRef, +impl<'a, DB, H, T, M, GM> hash_db::HashDBRef for KeySpacedDB<'a, DB, H> where + DB: hash_db::HashDBRef, H: Hasher, T: From<&'static [u8]>, { @@ -820,9 +812,9 @@ impl<'a, DB, H, T, M> hash_db::HashDBRef for KeySpacedDB<'a, DB, H> whe self.0.access_from(key, at) } - fn get_with_meta(&self, key: &H::Out, prefix: Prefix, parent: Option<&M>) -> Option<(T, M)> { + fn get_with_meta(&self, key: &H::Out, prefix: Prefix, global: GM) -> Option<(T, M)> { let derived_prefix = keyspace_as_prefix_alloc(self.1, prefix); - self.0.get_with_meta(key, (&derived_prefix.0, derived_prefix.1), parent) + self.0.get_with_meta(key, (&derived_prefix.0, derived_prefix.1), global) } fn contains(&self, key: &H::Out, prefix: Prefix) -> bool { @@ -831,8 +823,8 @@ impl<'a, DB, H, T, M> hash_db::HashDBRef for KeySpacedDB<'a, DB, H> whe } } -impl<'a, DB, H, T, M> hash_db::HashDB for KeySpacedDBMut<'a, DB, H> where - DB: hash_db::HashDB, +impl<'a, DB, H, T, M, GM> hash_db::HashDB for KeySpacedDBMut<'a, DB, H> where + DB: hash_db::HashDB, H: Hasher, T: Default + PartialEq + for<'b> From<&'b [u8]> + Clone + Send + Sync, { @@ -845,9 +837,9 @@ impl<'a, DB, H, T, M> hash_db::HashDB for KeySpacedDBMut<'a, DB, H> whe self.0.access_from(key, at) } - fn get_with_meta(&self, key: &H::Out, prefix: Prefix, parent: Option<&M>) -> Option<(T, M)> { + fn get_with_meta(&self, key: &H::Out, prefix: Prefix, global: GM) -> Option<(T, M)> { let derived_prefix = keyspace_as_prefix_alloc(self.1, prefix); - self.0.get_with_meta(key, (&derived_prefix.0, derived_prefix.1), parent) + self.0.get_with_meta(key, (&derived_prefix.0, derived_prefix.1), global) } fn contains(&self, key: &H::Out, prefix: Prefix) -> bool { @@ -881,14 +873,14 @@ impl<'a, DB, H, T, M> hash_db::HashDB for KeySpacedDBMut<'a, DB, H> whe } } -impl<'a, DB, H, T, M> hash_db::AsHashDB for KeySpacedDBMut<'a, DB, H> where - DB: hash_db::HashDB, +impl<'a, DB, H, T, M, GM> hash_db::AsHashDB for KeySpacedDBMut<'a, DB, H> where + DB: hash_db::HashDB, H: Hasher, T: Default + PartialEq + for<'b> From<&'b [u8]> + Clone + Send + Sync, { - fn as_hash_db(&self) -> &dyn hash_db::HashDB { &*self } + fn as_hash_db(&self) -> &dyn hash_db::HashDB { &*self } - fn as_hash_db_mut<'b>(&'b mut self) -> &'b mut (dyn hash_db::HashDB + 'b) { + fn as_hash_db_mut<'b>(&'b mut self) -> &'b mut (dyn hash_db::HashDB + 'b) { &mut *self } } diff --git a/primitives/trie/src/node_codec.rs b/primitives/trie/src/node_codec.rs index b0b7c771c6b55..611e0fad0d18d 100644 --- a/primitives/trie/src/node_codec.rs +++ b/primitives/trie/src/node_codec.rs @@ -197,10 +197,16 @@ impl NodeCodecT for NodeCodec { } fn is_empty_node(data: &[u8]) -> bool { - data == >::empty_node() + data == >::empty_node_no_meta() } - fn empty_node() -> &'static [u8] { + fn empty_node(meta: &mut M) -> Vec { + let mut output = meta.write_state_meta(); + output.extend_from_slice(&[trie_constants::EMPTY_TRIE]); + output + } + + fn empty_node_no_meta() -> &'static [u8] { &[trie_constants::EMPTY_TRIE] } From 8511a40baea43e655227c26f1994d0054da8965d Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Fri, 21 May 2021 19:11:01 +0200 Subject: [PATCH 017/127] extract and set global meta --- Cargo.lock | 16 ++++++++-------- primitives/trie/src/lib.rs | 21 +++++++++++++++------ primitives/trie/src/node_codec.rs | 2 +- primitives/trie/src/trie_stream.rs | 1 + 4 files changed, 25 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 897eff025d517..83b0b359996c2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2329,7 +2329,7 @@ dependencies = [ [[package]] name = "hash-db" version = "0.15.2" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#0b365f776e3716033f44f0fc9cbff7fd389a6c34" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#e51687592a06ca1108c08c0ee8cb4dd00ff23c7d" [[package]] name = "hash256-std-hasher" @@ -2343,7 +2343,7 @@ dependencies = [ [[package]] name = "hash256-std-hasher" version = "0.15.2" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#0b365f776e3716033f44f0fc9cbff7fd389a6c34" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#e51687592a06ca1108c08c0ee8cb4dd00ff23c7d" dependencies = [ "crunchy", ] @@ -2996,7 +2996,7 @@ dependencies = [ [[package]] name = "keccak-hasher" version = "0.15.3" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#0b365f776e3716033f44f0fc9cbff7fd389a6c34" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#e51687592a06ca1108c08c0ee8cb4dd00ff23c7d" dependencies = [ "hash-db", "hash256-std-hasher 0.15.2 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple4)", @@ -3788,7 +3788,7 @@ dependencies = [ [[package]] name = "memory-db" version = "0.26.0" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#0b365f776e3716033f44f0fc9cbff7fd389a6c34" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#e51687592a06ca1108c08c0ee8cb4dd00ff23c7d" dependencies = [ "hash-db", "hashbrown", @@ -10332,7 +10332,7 @@ checksum = "a7f741b240f1a48843f9b8e0444fb55fb2a4ff67293b50a9179dfd5ea67f8d41" [[package]] name = "trie-bench" version = "0.27.0" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#0b365f776e3716033f44f0fc9cbff7fd389a6c34" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#e51687592a06ca1108c08c0ee8cb4dd00ff23c7d" dependencies = [ "criterion", "hash-db", @@ -10347,7 +10347,7 @@ dependencies = [ [[package]] name = "trie-db" version = "0.22.3" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#0b365f776e3716033f44f0fc9cbff7fd389a6c34" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#e51687592a06ca1108c08c0ee8cb4dd00ff23c7d" dependencies = [ "hash-db", "hashbrown", @@ -10359,7 +10359,7 @@ dependencies = [ [[package]] name = "trie-root" version = "0.16.0" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#0b365f776e3716033f44f0fc9cbff7fd389a6c34" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#e51687592a06ca1108c08c0ee8cb4dd00ff23c7d" dependencies = [ "hash-db", ] @@ -10377,7 +10377,7 @@ dependencies = [ [[package]] name = "trie-standardmap" version = "0.15.2" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#0b365f776e3716033f44f0fc9cbff7fd389a6c34" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#e51687592a06ca1108c08c0ee8cb4dd00ff23c7d" dependencies = [ "hash-db", "keccak-hasher 0.15.3 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple4)", diff --git a/primitives/trie/src/lib.rs b/primitives/trie/src/lib.rs index 34966333c23bc..b562c534dc183 100644 --- a/primitives/trie/src/lib.rs +++ b/primitives/trie/src/lib.rs @@ -89,6 +89,17 @@ impl Meta for TrieMeta { self.do_value_hash = state_meta; } + fn extract_global_meta(&self) -> Self::GlobalMeta { + self.recorded_do_value_hash + } + + fn set_global_meta(&mut self, global_meta: Self::GlobalMeta) { + if global_meta { + self.recorded_do_value_hash = true; + self.do_value_hash = true; + } + } + fn has_state_meta(&self) -> bool { self.recorded_do_value_hash } @@ -230,14 +241,12 @@ impl TrieLayout for Layout self.0 } fn initialize_from_root_meta(&mut self, root_meta: &Self::Meta) { - if root_meta.recorded_do_value_hash { + if root_meta.extract_global_meta() { self.0 = true; } } fn set_root_meta(root_meta: &mut Self::Meta, global_meta: GlobalMeta) { - if global_meta { - root_meta.recorded_do_value_hash = true; - } + root_meta.set_global_meta(true); } } @@ -399,7 +408,7 @@ impl TrieConfiguration for Layout A: AsRef<[u8]> + Ord, B: AsRef<[u8]>, { - trie_root::trie_root_no_extension::(input, self.clone()) + trie_root::trie_root_no_extension::(input, self.layout_meta()) } fn trie_root_unhashed(&self, input: I) -> Vec where @@ -407,7 +416,7 @@ impl TrieConfiguration for Layout A: AsRef<[u8]> + Ord, B: AsRef<[u8]>, { - trie_root::unhashed_trie_no_extension::(input, self.clone()) + trie_root::unhashed_trie_no_extension::(input, self.layout_meta()) } fn encode_index(input: u32) -> Vec { diff --git a/primitives/trie/src/node_codec.rs b/primitives/trie/src/node_codec.rs index 611e0fad0d18d..0dc647f17f047 100644 --- a/primitives/trie/src/node_codec.rs +++ b/primitives/trie/src/node_codec.rs @@ -181,7 +181,7 @@ impl NodeCodecT for NodeCodec { type HashOut = H::Out; fn hashed_null_node() -> ::Out { - H::hash(>::empty_node()) + H::hash(>::empty_node_no_meta()) } fn decode_plan(data: &[u8], meta: &mut M) -> Result { diff --git a/primitives/trie/src/trie_stream.rs b/primitives/trie/src/trie_stream.rs index 3a65c5a9190b4..e9f19cef5449e 100644 --- a/primitives/trie/src/trie_stream.rs +++ b/primitives/trie/src/trie_stream.rs @@ -66,6 +66,7 @@ fn fuse_nibbles_node<'a>(nibbles: &'a [u8], kind: NodeKind) -> impl Iterator Self { TrieStream { From 270ed28b0d999d48cfc14d296c7d6aac7b8694c2 Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Mon, 24 May 2021 17:29:20 +0200 Subject: [PATCH 018/127] Update to branch 4 --- Cargo.lock | 16 ++--- bin/node/bench/src/generator.rs | 12 ++-- bin/node/bench/src/simple_trie.rs | 12 ++-- bin/node/bench/src/trie.rs | 4 +- client/db/src/bench.rs | 6 +- client/db/src/lib.rs | 8 +-- .../src/changes_trie/changes_iterator.rs | 4 +- .../state-machine/src/changes_trie/mod.rs | 2 +- .../state-machine/src/changes_trie/prune.rs | 2 +- .../state-machine/src/changes_trie/storage.rs | 4 +- primitives/state-machine/src/lib.rs | 2 +- .../state-machine/src/proving_backend.rs | 6 +- primitives/state-machine/src/trie_backend.rs | 38 ++++++++---- .../state-machine/src/trie_backend_essence.rs | 62 +++++++++---------- primitives/trie/src/lib.rs | 61 +++++++----------- primitives/trie/src/storage_proof.rs | 10 ++- primitives/trie/src/trie_stream.rs | 55 +++++++++++++--- 17 files changed, 172 insertions(+), 132 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 83b0b359996c2..d1c2e7d69da81 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2329,7 +2329,7 @@ dependencies = [ [[package]] name = "hash-db" version = "0.15.2" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#e51687592a06ca1108c08c0ee8cb4dd00ff23c7d" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#09571017e3ad7ee3e04a40eb8d61bcff25e255ac" [[package]] name = "hash256-std-hasher" @@ -2343,7 +2343,7 @@ dependencies = [ [[package]] name = "hash256-std-hasher" version = "0.15.2" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#e51687592a06ca1108c08c0ee8cb4dd00ff23c7d" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#09571017e3ad7ee3e04a40eb8d61bcff25e255ac" dependencies = [ "crunchy", ] @@ -2996,7 +2996,7 @@ dependencies = [ [[package]] name = "keccak-hasher" version = "0.15.3" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#e51687592a06ca1108c08c0ee8cb4dd00ff23c7d" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#09571017e3ad7ee3e04a40eb8d61bcff25e255ac" dependencies = [ "hash-db", "hash256-std-hasher 0.15.2 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple4)", @@ -3788,7 +3788,7 @@ dependencies = [ [[package]] name = "memory-db" version = "0.26.0" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#e51687592a06ca1108c08c0ee8cb4dd00ff23c7d" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#09571017e3ad7ee3e04a40eb8d61bcff25e255ac" dependencies = [ "hash-db", "hashbrown", @@ -10332,7 +10332,7 @@ checksum = "a7f741b240f1a48843f9b8e0444fb55fb2a4ff67293b50a9179dfd5ea67f8d41" [[package]] name = "trie-bench" version = "0.27.0" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#e51687592a06ca1108c08c0ee8cb4dd00ff23c7d" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#09571017e3ad7ee3e04a40eb8d61bcff25e255ac" dependencies = [ "criterion", "hash-db", @@ -10347,7 +10347,7 @@ dependencies = [ [[package]] name = "trie-db" version = "0.22.3" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#e51687592a06ca1108c08c0ee8cb4dd00ff23c7d" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#09571017e3ad7ee3e04a40eb8d61bcff25e255ac" dependencies = [ "hash-db", "hashbrown", @@ -10359,7 +10359,7 @@ dependencies = [ [[package]] name = "trie-root" version = "0.16.0" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#e51687592a06ca1108c08c0ee8cb4dd00ff23c7d" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#09571017e3ad7ee3e04a40eb8d61bcff25e255ac" dependencies = [ "hash-db", ] @@ -10377,7 +10377,7 @@ dependencies = [ [[package]] name = "trie-standardmap" version = "0.15.2" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#e51687592a06ca1108c08c0ee8cb4dd00ff23c7d" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#09571017e3ad7ee3e04a40eb8d61bcff25e255ac" dependencies = [ "hash-db", "keccak-hasher 0.15.3 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple4)", diff --git a/bin/node/bench/src/generator.rs b/bin/node/bench/src/generator.rs index a227c9081bb67..9f65ddd77dafb 100644 --- a/bin/node/bench/src/generator.rs +++ b/bin/node/bench/src/generator.rs @@ -43,11 +43,15 @@ pub fn generate_trie( ); let mut trie = SimpleTrie { db, overlay: &mut overlay }; { - let mut trie_db = TrieDBMut::new(&mut trie, &mut root); - if flag_hashed_value { - sp_trie::flag_meta_hasher(&mut trie_db).expect("flag trie failed"); - } + let mut trie_db = if flag_hashed_value { + let layout = sp_trie::Layout::with_inner_hashing(); + let mut t = TrieDBMut::::new_with_layout(&mut trie, &mut root, layout); + t.force_layout_meta(); + t + } else { + TrieDBMut::new(&mut trie, &mut root) + }; for (key, value) in key_values { trie_db.insert(&key, &value).expect("trie insertion failed"); } diff --git a/bin/node/bench/src/simple_trie.rs b/bin/node/bench/src/simple_trie.rs index 6cc32e00e34d9..4d8e76ae3a7e1 100644 --- a/bin/node/bench/src/simple_trie.rs +++ b/bin/node/bench/src/simple_trie.rs @@ -31,15 +31,15 @@ pub struct SimpleTrie<'a> { pub overlay: &'a mut HashMap, Option>>, } -impl<'a> AsHashDB for SimpleTrie<'a> { - fn as_hash_db(&self) -> &dyn hash_db::HashDB { &*self } +impl<'a> AsHashDB for SimpleTrie<'a> { + fn as_hash_db(&self) -> &dyn hash_db::HashDB { &*self } - fn as_hash_db_mut<'b>(&'b mut self) -> &'b mut (dyn HashDB + 'b) { + fn as_hash_db_mut<'b>(&'b mut self) -> &'b mut (dyn HashDB + 'b) { &mut *self } } -impl<'a> HashDB for SimpleTrie<'a> { +impl<'a> HashDB for SimpleTrie<'a> { fn get(&self, key: &Hash, prefix: Prefix) -> Option { let key = sp_trie::prefixed_key::(key, prefix); if let Some(value) = self.overlay.get(&key) { @@ -48,9 +48,9 @@ impl<'a> HashDB for SimpleTrie<'a> { self.db.get(0, &key).expect("Database backend error") } - fn get_with_meta(&self, key: &Hash, prefix: Prefix, parent: Option<&TrieMeta>) -> Option<(DBValue, TrieMeta)> { + fn get_with_meta(&self, key: &Hash, prefix: Prefix, global: bool) -> Option<(DBValue, TrieMeta)> { let result = self.get(key, prefix); - result.map(|value| >::extract_value_owned(value, parent)) + result.map(|value| >::extract_value_owned(value, global)) } fn contains(&self, hash: &Hash, prefix: Prefix) -> bool { diff --git a/bin/node/bench/src/trie.rs b/bin/node/bench/src/trie.rs index 56bf7fe8ad604..735403f95c87c 100644 --- a/bin/node/bench/src/trie.rs +++ b/bin/node/bench/src/trie.rs @@ -170,11 +170,11 @@ impl core::BenchmarkDescription for TrieReadBenchmarkDescription { struct Storage(Arc); impl sp_state_machine::Storage for Storage { - fn get(&self, key: &Hash, prefix: Prefix, parent: Option<&TrieMeta>) -> Result, TrieMeta)>, String> { + fn get(&self, key: &Hash, prefix: Prefix, global: bool) -> Result, TrieMeta)>, String> { let key = sp_trie::prefixed_key::(key, prefix); self.0.get(0, &key).map_err(|e| format!("Database backend error: {:?}", e)) .map(|result| result - .map(|value| >::extract_value_owned(value, parent)) + .map(|value| >::extract_value_owned(value, global)) ) } diff --git a/client/db/src/bench.rs b/client/db/src/bench.rs index 21e20d1ff6211..8b73d0eb31622 100644 --- a/client/db/src/bench.rs +++ b/client/db/src/bench.rs @@ -49,20 +49,20 @@ struct StorageDb { } impl sp_state_machine::Storage> for StorageDb { - fn get(&self, key: &Block::Hash, prefix: Prefix, parent: Option<&TrieMeta>) -> Result, String> { + fn get(&self, key: &Block::Hash, prefix: Prefix, global: bool) -> Result, String> { let prefixed_key = prefixed_key::>(key, prefix); if let Some(recorder) = &self.proof_recorder { if let Some(v) = recorder.get(&key) { return Ok(v.clone()); } let backend_value = self.db.get(0, &prefixed_key) - .map(|result| result.map(|value| , _>>::extract_value_owned(value, parent))) + .map(|result| result.map(|value| , _>>::extract_value_owned(value, global))) .map_err(|e| format!("Database backend error: {:?}", e))?; recorder.record::>(key.clone(), backend_value.clone()); Ok(backend_value) } else { self.db.get(0, &prefixed_key) - .map(|result| result.map(|value| , _>>::extract_value_owned(value, parent))) + .map(|result| result.map(|value| , _>>::extract_value_owned(value, global))) .map_err(|e| format!("Database backend error: {:?}", e)) } } diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index 2fa447b5e9b84..ddfcd6d260222 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -867,14 +867,14 @@ struct StorageDb { } impl sp_state_machine::Storage> for StorageDb { - fn get(&self, key: &Block::Hash, prefix: Prefix, parent: Option<&TrieMeta>) -> Result, String> { + fn get(&self, key: &Block::Hash, prefix: Prefix, global: bool) -> Result, String> { if self.prefix_keys { let key = prefixed_key::>(key, prefix); self.state_db.get(&key, self) } else { self.state_db.get(key.as_ref(), self) } - .map(|result| result.map(|value| , _>>::extract_value_owned(value, parent))) + .map(|result| result.map(|value| , _>>::extract_value_owned(value, global))) .map_err(|e| format!("Database backend error: {:?}", e)) } @@ -903,7 +903,7 @@ impl DbGenesisStorage { } impl sp_state_machine::Storage> for DbGenesisStorage { - fn get(&self, _key: &Block::Hash, _prefix: Prefix, _parent: Option<&TrieMeta>) -> Result, String> { + fn get(&self, _key: &Block::Hash, _prefix: Prefix, _global: bool) -> Result, String> { Ok(None) } fn access_from(&self, _key: &Block::Hash) { @@ -2125,7 +2125,7 @@ impl sc_client_api::backend::Backend for Backend { self.storage.as_ref(), &header.state_root, (&[], None), - None, + Default::default(), ).unwrap_or(None).is_some() }, _ => false, diff --git a/primitives/state-machine/src/changes_trie/changes_iterator.rs b/primitives/state-machine/src/changes_trie/changes_iterator.rs index be35581e7514d..25eda86b4e66a 100644 --- a/primitives/state-machine/src/changes_trie/changes_iterator.rs +++ b/primitives/state-machine/src/changes_trie/changes_iterator.rs @@ -24,7 +24,7 @@ use codec::{Decode, Encode, Codec}; use hash_db::Hasher; use num_traits::Zero; use sp_core::storage::PrefixedStorageKey; -use sp_trie::Recorder; +use sp_trie::{Recorder, TrieMeta}; use crate::changes_trie::{AnchorBlockId, ConfigurationRange, RootsStorage, Storage, BlockNumber}; use crate::changes_trie::input::{DigestIndex, ExtrinsicIndex, DigestIndexValue, ExtrinsicIndexValue}; use crate::changes_trie::storage::{TrieBackendAdapter, InMemoryStorage}; @@ -337,7 +337,7 @@ struct ProvingDrilldownIterator<'a, H, Number> H::Out: 'a, { essence: DrilldownIteratorEssence<'a, H, Number>, - proof_recorder: RefCell>, + proof_recorder: RefCell>, } impl<'a, H, Number> ProvingDrilldownIterator<'a, H, Number> diff --git a/primitives/state-machine/src/changes_trie/mod.rs b/primitives/state-machine/src/changes_trie/mod.rs index dd6d4f0df3b88..404353fc308b4 100644 --- a/primitives/state-machine/src/changes_trie/mod.rs +++ b/primitives/state-machine/src/changes_trie/mod.rs @@ -167,7 +167,7 @@ pub struct TrieBackendStorageAdapter<'a, H: Hasher, Number: BlockNumber>(pub &'a impl<'a, H: Hasher, N: BlockNumber> crate::TrieBackendStorage for TrieBackendStorageAdapter<'a, H, N> { type Overlay = sp_trie::MemoryDB; - fn get(&self, key: &H::Out, prefix: Prefix, _parent: Option<&sp_trie::TrieMeta>) -> Result, String> { + fn get(&self, key: &H::Out, prefix: Prefix, _global: bool) -> Result, String> { match self.0.get(key, prefix) { // change trie do not use meta. Ok(Some(v)) => Ok(Some((v, Default::default()))), diff --git a/primitives/state-machine/src/changes_trie/prune.rs b/primitives/state-machine/src/changes_trie/prune.rs index 4098eadb98b31..987fb4ff63d08 100644 --- a/primitives/state-machine/src/changes_trie/prune.rs +++ b/primitives/state-machine/src/changes_trie/prune.rs @@ -95,7 +95,7 @@ fn prune_trie( // enumerate all changes trie' keys, recording all nodes that have been 'touched' // (effectively - all changes trie nodes) - let mut proof_recorder: Recorder = Default::default(); + let mut proof_recorder: Recorder = Default::default(); { let mut trie = ProvingBackendRecorder::<_, H> { backend: &TrieBackendEssence::new(TrieBackendAdapter::new(storage), root), diff --git a/primitives/state-machine/src/changes_trie/storage.rs b/primitives/state-machine/src/changes_trie/storage.rs index f81838f11f6e9..5414d3f8c2f6a 100644 --- a/primitives/state-machine/src/changes_trie/storage.rs +++ b/primitives/state-machine/src/changes_trie/storage.rs @@ -190,7 +190,7 @@ impl Storage for InMemoryStorage Result, String> { - Ok( as hash_db::HashDBRef>::get(&self.data.read().mdb, key, prefix)) + Ok( as hash_db::HashDBRef>::get(&self.data.read().mdb, key, prefix)) } } @@ -207,7 +207,7 @@ impl<'a, H, Number> TrieBackendStorage for TrieBackendAdapter<'a, H, Number> { type Overlay = MemoryDB; - fn get(&self, key: &H::Out, prefix: Prefix, _parent: Option<&sp_trie::TrieMeta>) -> Result, String> { + fn get(&self, key: &H::Out, prefix: Prefix, _global: bool) -> Result, String> { match self.storage.get(key, prefix) { // change trie do not use meta. Ok(Some(v)) => Ok(Some((v, Default::default()))), diff --git a/primitives/state-machine/src/lib.rs b/primitives/state-machine/src/lib.rs index d12686b43ab2b..6b6c661ea4f03 100644 --- a/primitives/state-machine/src/lib.rs +++ b/primitives/state-machine/src/lib.rs @@ -855,7 +855,7 @@ mod execution { where H: Hasher, H::Out: Ord + Codec, - MH: sp_trie::MetaHasher, + MH: sp_trie::MetaHasher, KF: sp_trie::KeyFunction + Send + Sync, { proving_backend.storage(key).map_err(|e| Box::new(e) as Box) diff --git a/primitives/state-machine/src/proving_backend.rs b/primitives/state-machine/src/proving_backend.rs index 81ad50408e4c4..b431fa0c0e04d 100644 --- a/primitives/state-machine/src/proving_backend.rs +++ b/primitives/state-machine/src/proving_backend.rs @@ -35,7 +35,7 @@ use sp_core::storage::ChildInfo; /// Patricia trie-based backend specialized in get value proofs. pub struct ProvingBackendRecorder<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> { pub(crate) backend: &'a TrieBackendEssence, - pub(crate) proof_recorder: &'a mut Recorder, + pub(crate) proof_recorder: &'a mut Recorder, } impl<'a, S, H> ProvingBackendRecorder<'a, S, H> @@ -242,12 +242,12 @@ impl<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> TrieBackendStorage { type Overlay = S::Overlay; - fn get(&self, key: &H::Out, prefix: Prefix, parent: Option<&TrieMeta>) -> Result, String> { + fn get(&self, key: &H::Out, prefix: Prefix, global: bool) -> Result, String> { if let Some(v) = self.proof_recorder.get(key) { return Ok(v); } - let backend_value = self.backend.get(key, prefix, parent)?; + let backend_value = self.backend.get(key, prefix, global)?; self.proof_recorder.record::(key.clone(), backend_value.clone()); Ok(backend_value) } diff --git a/primitives/state-machine/src/trie_backend.rs b/primitives/state-machine/src/trie_backend.rs index e110963a20817..788eb47f89588 100644 --- a/primitives/state-machine/src/trie_backend.rs +++ b/primitives/state-machine/src/trie_backend.rs @@ -181,17 +181,20 @@ impl, H: Hasher> Backend for TrieBackend where self.essence.backend_storage(), &mut write_overlay, ); - if flag_inner_hash_value { - root = match sp_trie::flag_inner_meta_hasher::, _>(&mut eph, root) { - Ok(ret) => ret, - Err(e) => { - warn!(target: "trie", "Failed to flag trie: {}", e); - root - }, + let res = || { + if flag_inner_hash_value { + let layout = sp_trie::Layout::with_inner_hashing(); + let mut t = sp_trie::trie_types::TrieDBMut::::from_existing_with_layout( + &mut eph, + &mut root, + layout, + )?; + t.force_layout_meta()?; } - } + delta_trie_root::, _, _, _, _, _>(&mut eph, root, delta) + }; - match delta_trie_root::, _, _, _, _, _>(&mut eph, root, delta) { + match res() { Ok(ret) => root = ret, Err(e) => warn!(target: "trie", "Failed to write to trie: {}", e), } @@ -286,10 +289,19 @@ pub mod tests { { let mut sub_root = Vec::new(); root.encode_to(&mut sub_root); - let mut trie = TrieDBMut::new(&mut mdb, &mut root); - if hashed_value { - sp_trie::flag_meta_hasher(&mut trie).expect("flag failed"); - } + let mut trie = if hashed_value { + let layout = Layout::with_inner_hashing(); + let mut t = TrieDBMut::new_with_layout( + &mut mdb, + &mut root, + layout, + ); + t.force_layout_meta() + .expect("failed forced layout change"); + t + } else { + TrieDBMut::new(&mut mdb, &mut root) + }; trie.insert(child_info.prefixed_storage_key().as_slice(), &sub_root[..]) .expect("insert failed"); diff --git a/primitives/state-machine/src/trie_backend_essence.rs b/primitives/state-machine/src/trie_backend_essence.rs index 6d417204ebd8d..157c8ff12097f 100644 --- a/primitives/state-machine/src/trie_backend_essence.rs +++ b/primitives/state-machine/src/trie_backend_essence.rs @@ -43,7 +43,7 @@ type Result = sp_std::result::Result; /// Patricia trie-based storage trait. pub trait Storage: Send + Sync { /// Get a trie node. - fn get(&self, key: &H::Out, prefix: Prefix, parent: Option<&TrieMeta>) -> Result>; + fn get(&self, key: &H::Out, prefix: Prefix, inner_hash: bool) -> Result>; /// Call back when value get accessed in trie. fn access_from(&self, key: &H::Out); } @@ -131,7 +131,7 @@ impl, H: Hasher> TrieBackendEssence where H::Out: child_info: Option<&ChildInfo>, key: &[u8], ) -> Result> { - let dyn_eph: &dyn hash_db::HashDBRef<_, _, _>; + let dyn_eph: &dyn hash_db::HashDBRef<_, _, _, _>; let keyspace_eph; if let Some(child_info) = child_info.as_ref() { keyspace_eph = KeySpacedDB::new(self, child_info.keyspace()); @@ -288,11 +288,11 @@ pub(crate) struct Ephemeral<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> { overlay: &'a mut S::Overlay, } -impl<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> hash_db::AsHashDB +impl<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> hash_db::AsHashDB for Ephemeral<'a, S, H> { - fn as_hash_db<'b>(&'b self) -> &'b (dyn hash_db::HashDB + 'b) { self } - fn as_hash_db_mut<'b>(&'b mut self) -> &'b mut (dyn hash_db::HashDB + 'b) { self } + fn as_hash_db<'b>(&'b self) -> &'b (dyn hash_db::HashDB + 'b) { self } + fn as_hash_db_mut<'b>(&'b mut self) -> &'b mut (dyn hash_db::HashDB + 'b) { self } } impl<'a, S: TrieBackendStorage, H: Hasher> Ephemeral<'a, S, H> { @@ -304,18 +304,18 @@ impl<'a, S: TrieBackendStorage, H: Hasher> Ephemeral<'a, S, H> { } } -impl<'a, S: 'a + TrieBackendStorage, H: Hasher> hash_db::HashDB +impl<'a, S: 'a + TrieBackendStorage, H: Hasher> hash_db::HashDB for Ephemeral<'a, S, H> { fn get(&self, key: &H::Out, prefix: Prefix) -> Option { - Self::get_with_meta(self, key, prefix, None).map(|r| r.0) + Self::get_with_meta(self, key, prefix, Default::default()).map(|r| r.0) } - fn get_with_meta(&self, key: &H::Out, prefix: Prefix, parent: Option<&TrieMeta>) -> Option<(DBValue, TrieMeta)> { - if let Some(val) = hash_db::HashDB::get_with_meta(self.overlay, key, prefix, parent) { + fn get_with_meta(&self, key: &H::Out, prefix: Prefix, global: bool) -> Option<(DBValue, TrieMeta)> { + if let Some(val) = hash_db::HashDB::get_with_meta(self.overlay, key, prefix, global) { Some(val) } else { - match self.storage.get(&key, prefix, parent) { + match self.storage.get(&key, prefix, global) { Ok(x) => x, Err(e) => { warn!(target: "trie", "Failed to read from DB: {}", e); @@ -357,15 +357,15 @@ impl<'a, S: 'a + TrieBackendStorage, H: Hasher> hash_db::HashDB, H: Hasher> hash_db::HashDBRef +impl<'a, S: 'a + TrieBackendStorage, H: Hasher> hash_db::HashDBRef for Ephemeral<'a, S, H> { fn get(&self, key: &H::Out, prefix: Prefix) -> Option { hash_db::HashDB::get(self, key, prefix) } - fn get_with_meta(&self, key: &H::Out, prefix: Prefix, parent: Option<&TrieMeta>) -> Option<(DBValue, TrieMeta)> { - hash_db::HashDB::get_with_meta(self, key, prefix, parent) + fn get_with_meta(&self, key: &H::Out, prefix: Prefix, global: bool) -> Option<(DBValue, TrieMeta)> { + hash_db::HashDB::get_with_meta(self, key, prefix, global) } fn access_from(&self, key: &H::Out, at: Option<&H::Out>) -> Option { @@ -380,9 +380,9 @@ impl<'a, S: 'a + TrieBackendStorage, H: Hasher> hash_db::HashDBRef: Send + Sync { /// Type of in-memory overlay. - type Overlay: hash_db::HashDB + Default + Consolidate; + type Overlay: hash_db::HashDB + Default + Consolidate; /// Get the value stored at key. - fn get(&self, key: &H::Out, prefix: Prefix, parent_meta: Option<&TrieMeta>) -> Result>; + fn get(&self, key: &H::Out, prefix: Prefix, global: bool) -> Result>; /// Call back when value get accessed in trie. fn access_from(&self, key: &H::Out); } @@ -392,8 +392,8 @@ pub trait TrieBackendStorage: Send + Sync { impl TrieBackendStorage for Arc> { type Overlay = PrefixedMemoryDB; - fn get(&self, key: &H::Out, prefix: Prefix, parent: Option<&TrieMeta>) -> Result> { - Storage::::get(self.deref(), key, prefix, parent) + fn get(&self, key: &H::Out, prefix: Prefix, global: bool) -> Result> { + Storage::::get(self.deref(), key, prefix, global) } fn access_from(&self, key: &H::Out) { @@ -404,13 +404,13 @@ impl TrieBackendStorage for Arc> { impl TrieBackendStorage for sp_trie::GenericMemoryDB where H: Hasher, - MH: sp_trie::MetaHasher, + MH: sp_trie::MetaHasher, KF: sp_trie::KeyFunction + Send + Sync, { type Overlay = Self; - fn get(&self, key: &H::Out, prefix: Prefix, parent: Option<&TrieMeta>) -> Result> { - Ok(hash_db::HashDB::get_with_meta(self, key, prefix, parent)) + fn get(&self, key: &H::Out, prefix: Prefix, global: bool) -> Result> { + Ok(hash_db::HashDB::get_with_meta(self, key, prefix, global)) } fn access_from(&self, key: &H::Out) { @@ -418,25 +418,25 @@ impl TrieBackendStorage for sp_trie::GenericMemoryDB } } -impl, H: Hasher> hash_db::AsHashDB +impl, H: Hasher> hash_db::AsHashDB for TrieBackendEssence { - fn as_hash_db<'b>(&'b self) -> &'b (dyn hash_db::HashDB + 'b) { self } - fn as_hash_db_mut<'b>(&'b mut self) -> &'b mut (dyn hash_db::HashDB + 'b) { self } + fn as_hash_db<'b>(&'b self) -> &'b (dyn hash_db::HashDB + 'b) { self } + fn as_hash_db_mut<'b>(&'b mut self) -> &'b mut (dyn hash_db::HashDB + 'b) { self } } -impl, H: Hasher> hash_db::HashDB +impl, H: Hasher> hash_db::HashDB for TrieBackendEssence { fn get(&self, key: &H::Out, prefix: Prefix) -> Option { - self.get_with_meta(key, prefix, None).map(|r| r.0) + self.get_with_meta(key, prefix, Default::default()).map(|r| r.0) } - fn get_with_meta(&self, key: &H::Out, prefix: Prefix, parent: Option<&TrieMeta>) -> Option<(DBValue, TrieMeta)> { + fn get_with_meta(&self, key: &H::Out, prefix: Prefix, global: bool) -> Option<(DBValue, TrieMeta)> { if *key == self.empty { - return Some(([0u8].to_vec(), ::meta_for_empty())) + return Some(([0u8].to_vec(), ::meta_for_empty(global))) } - match self.storage.get(&key, prefix, parent) { + match self.storage.get(&key, prefix, global) { Ok(x) => x, Err(e) => { warn!(target: "trie", "Failed to read from DB: {}", e); @@ -472,15 +472,15 @@ impl, H: Hasher> hash_db::HashDB } } -impl, H: Hasher> hash_db::HashDBRef +impl, H: Hasher> hash_db::HashDBRef for TrieBackendEssence { fn get(&self, key: &H::Out, prefix: Prefix) -> Option { hash_db::HashDB::get(self, key, prefix) } - fn get_with_meta(&self, key: &H::Out, prefix: Prefix, parent: Option<&TrieMeta>) -> Option<(DBValue, TrieMeta)> { - hash_db::HashDB::get_with_meta(self, key, prefix, parent) + fn get_with_meta(&self, key: &H::Out, prefix: Prefix, global: bool) -> Option<(DBValue, TrieMeta)> { + hash_db::HashDB::get_with_meta(self, key, prefix, global) } fn access_from(&self, key: &H::Out, at: Option<&H::Out>) -> Option { diff --git a/primitives/trie/src/lib.rs b/primitives/trie/src/lib.rs index b562c534dc183..2a5811b76d09a 100644 --- a/primitives/trie/src/lib.rs +++ b/primitives/trie/src/lib.rs @@ -220,6 +220,13 @@ impl Default for Layout { Layout(false, sp_std::marker::PhantomData) } } +impl Layout { + /// Layout with inner hashing active. + /// Will flag trie for hashing. + pub fn with_inner_hashing() -> Self { + Layout(true, sp_std::marker::PhantomData) + } +} impl TrieLayout for Layout where @@ -246,7 +253,7 @@ impl TrieLayout for Layout } } fn set_root_meta(root_meta: &mut Self::Meta, global_meta: GlobalMeta) { - root_meta.set_global_meta(true); + root_meta.set_global_meta(global_meta); } } @@ -461,7 +468,7 @@ pub type MemoryDBMeta = memory_db::MemoryDB< /// Reexport from `hash_db`, with genericity set for `Hasher` trait. pub type GenericMemoryDB = memory_db::MemoryDB< - H, KF, trie_db::DBValue, MH, MemTracker + H, KF, trie_db::DBValue, MH, MemTracker, >; /// Persistent trie database read-access interface for the a given hasher. @@ -565,36 +572,6 @@ pub fn delta_trie_root( Ok(root) } -/// Flag inner trie with state metadata to enable hash of value internally. -pub fn flag_inner_meta_hasher( - db: &mut DB, - mut root: TrieHash, -) -> Result, Box>> where - L: TrieConfiguration, - DB: hash_db::HashDB>, -{ - { - let mut t = TrieDBMut::::from_existing(db, &mut root)?; - flag_meta_hasher(&mut t)?; - } - Ok(root) -} - -/// Flag inner trie with state metadata to enable hash of value internally. -pub fn flag_meta_hasher( - t: &mut TrieDBMut -) -> Result<(), Box>> where - L: TrieConfiguration, -{ - let flag = true; - let key: &[u8]= &[]; - if !t.contains(key)? { - t.insert(key, b"")?; - } - assert!(t.flag(key, flag)?); - Ok(()) -} - /// Resolve if inner hashing of value is active. pub fn state_hashed_value>>( db: &DB, @@ -1168,7 +1145,7 @@ mod tests { } fn populate_trie<'db, T>( - db: &'db mut dyn HashDB, + db: &'db mut dyn HashDB>, root: &'db mut TrieHash, v: &[(Vec, Vec)], flag_hash: bool, @@ -1176,10 +1153,20 @@ mod tests { where T: TrieConfiguration, { - let mut t = TrieDBMut::::new(db, root); - if flag_hash { - flag_meta_hasher(&mut t).unwrap(); - } + let mut t = if flag_hash { + let mut root_meta = Default::default(); + T::set_root_meta(&mut root_meta, flag_hash); + + let mut layout = T::default(); + layout.initialize_from_root_meta(&root_meta); + + let mut t = TrieDBMut::::new_with_layout(db, root, layout); + t.force_layout_meta() + .expect("Could not force layout."); + t + } else { + TrieDBMut::::new(db, root) + }; for i in 0..v.len() { let key: &[u8]= &v[i].0; let val: &[u8] = &v[i].1; diff --git a/primitives/trie/src/storage_proof.rs b/primitives/trie/src/storage_proof.rs index 915dd92b1c16c..0f00a335ae2f1 100644 --- a/primitives/trie/src/storage_proof.rs +++ b/primitives/trie/src/storage_proof.rs @@ -113,23 +113,21 @@ impl From for crate::MemoryDB { let mut is_hashed_value = false; let mut accum = Vec::new(); for item in proof.trie_nodes.iter() { - // Note using `None` as parent meta does not impact `extract_value` of - // sp_trie meta hasher. - // But does not with `insert_with_meta`. + // Note using `default()` as global meta helps looking fro root node. + let layout_meta = Default::default(); let (encoded_node, mut meta) = < as TrieLayout>::MetaHasher as MetaHasher - >::extract_value(item.as_slice(), None); + >::extract_value(item.as_slice(), layout_meta); // read state meta. let _ = as TrieLayout>::Codec::decode_plan(encoded_node, &mut meta); if meta.recorded_do_value_hash { + debug_assert!(!is_hashed_value); is_hashed_value = true; } - // TODO insert_with_meta here accum.push((encoded_node, meta)); } for mut item in accum.into_iter() { if is_hashed_value { - // skipping hierarchy. item.1.do_value_hash = true; } db.insert_with_meta(crate::EMPTY_PREFIX, item.0, item.1); diff --git a/primitives/trie/src/trie_stream.rs b/primitives/trie/src/trie_stream.rs index e9f19cef5449e..dc754fbb903ec 100644 --- a/primitives/trie/src/trie_stream.rs +++ b/primitives/trie/src/trie_stream.rs @@ -17,13 +17,15 @@ //! `TrieStream` implementation for Substrate's trie format. -use hash_db::Hasher; +use hash_db::{MetaHasher, Hasher}; use trie_root; -use codec::Encode; +use codec::{Encode, Compact}; use sp_std::vec::Vec; -use crate::trie_constants; +use sp_std::ops::Range; +use crate::{trie_constants, TrieMeta, StateHasher}; use crate::node_header::{NodeKind, size_and_prefix_iterator}; use crate::node_codec::Bitmap; +use trie_db::Meta; const BRANCH_NODE_NO_VALUE: u8 = 254; const BRANCH_NODE_WITH_VALUE: u8 = 255; @@ -32,6 +34,8 @@ const BRANCH_NODE_WITH_VALUE: u8 = 255; /// Codec-flavored TrieStream. pub struct TrieStream { buffer: Vec, + inner_value_hashing: bool, + current_value_range: Option>, } impl TrieStream { @@ -68,9 +72,11 @@ fn fuse_nibbles_node<'a>(nibbles: &'a [u8], kind: NodeKind) -> impl Iterator Self { - TrieStream { - buffer: Vec::new() + fn new(meta: bool) -> Self { + Self { + buffer: Vec::new(), + inner_value_hashing: meta, + current_value_range: None, } } @@ -80,7 +86,9 @@ impl trie_root::TrieStream for TrieStream { fn append_leaf(&mut self, key: &[u8], value: &[u8]) { self.buffer.extend(fuse_nibbles_node(key, NodeKind::Leaf)); - value.encode_to(&mut self.buffer); + Compact(value.len() as u32).encode_to(&mut self.buffer); + self.current_value_range = Some(self.buffer.len()..self.buffer.len() + value.len()); + self.buffer.extend_from_slice(value); } fn begin_branch( @@ -102,7 +110,9 @@ impl trie_root::TrieStream for TrieStream { self.buffer.extend(&branch_node(maybe_value.is_some(), has_children)); } if let Some(value) = maybe_value { - value.encode_to(&mut self.buffer); + Compact(value.len() as u32).encode_to(&mut self.buffer); + self.current_value_range = Some(self.buffer.len()..self.buffer.len() + value.len()); + self.buffer.extend_from_slice(value); } } @@ -118,6 +128,35 @@ impl trie_root::TrieStream for TrieStream { } } + fn hash_root(self) -> H::Out { + let inner_value_hashing = self.inner_value_hashing; + let range = self.current_value_range; + let data = self.buffer; + if inner_value_hashing + && range.as_ref().map(|r| r.end - r.start >= trie_constants::INNER_HASH_TRESHOLD) + .unwrap_or_default() { + let meta = TrieMeta { + range: range, + unused_value: false, + contain_hash: false, + do_value_hash: true, + old_hash: false, + recorded_do_value_hash: true, + }; + // Add the recorded_do_value_hash to encoded + let mut encoded = meta.write_state_meta(); + let encoded = if encoded.len() > 0 { + encoded.extend(data); + encoded + } else { + data + }; + >>::hash(&encoded, &meta) + } else { + H::hash(&data) + } + } + fn out(self) -> Vec { self.buffer } } From b838e7e2aab7f0dd643cf40292e4e3bac2ab7411 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 25 May 2021 18:04:04 +0200 Subject: [PATCH 019/127] fix iterator with root flag (no longer insert node). --- bin/node/bench/src/generator.rs | 2 +- .../state-machine/src/trie_backend_essence.rs | 1 + primitives/trie/src/lib.rs | 25 +++++++++++++------ 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/bin/node/bench/src/generator.rs b/bin/node/bench/src/generator.rs index 9f65ddd77dafb..a27d6e529f277 100644 --- a/bin/node/bench/src/generator.rs +++ b/bin/node/bench/src/generator.rs @@ -47,7 +47,7 @@ pub fn generate_trie( let mut trie_db = if flag_hashed_value { let layout = sp_trie::Layout::with_inner_hashing(); let mut t = TrieDBMut::::new_with_layout(&mut trie, &mut root, layout); - t.force_layout_meta(); + t.force_layout_meta().expect("force layout meta failed"); t } else { TrieDBMut::new(&mut trie, &mut root) diff --git a/primitives/state-machine/src/trie_backend_essence.rs b/primitives/state-machine/src/trie_backend_essence.rs index 157c8ff12097f..5d1f50e9c5a5a 100644 --- a/primitives/state-machine/src/trie_backend_essence.rs +++ b/primitives/state-machine/src/trie_backend_essence.rs @@ -172,6 +172,7 @@ impl, H: Hasher> TrieBackendEssence where H::Out: /// Does current trie use inner hashed value. pub fn state_hashed_value(&self) -> bool { sp_trie::state_hashed_value::, _>(self, &self.root) + .unwrap_or_default() } /// Get the value of storage at given key. diff --git a/primitives/trie/src/lib.rs b/primitives/trie/src/lib.rs index 2a5811b76d09a..05a3aa701f99e 100644 --- a/primitives/trie/src/lib.rs +++ b/primitives/trie/src/lib.rs @@ -576,11 +576,25 @@ pub fn delta_trie_root( pub fn state_hashed_value>>( db: &DB, root: &TrieHash, -) -> bool { +) -> Option> { + struct ReadMeta { + hashed: Option>, + } + impl trie_db::Query for &mut ReadMeta { + type Item = DBValue; + fn decode(self, value: &[u8]) -> DBValue { value.to_vec() } + fn record(&mut self, _hash: &::Out, _data: &[u8], _depth: u32, meta: &L::Meta) { + debug_assert!(self.hashed.is_none()); + self.hashed = Some(meta.extract_global_meta()); + } + } + let mut read_meta: ReadMeta = ReadMeta { + hashed: None, + }; if let Ok(t) = TrieDB::::new(&*db, root) { - unimplemented!("TODO has_flag on triedb"); + let _ = t.get_with(&[], &mut read_meta); } - false + read_meta.hashed } /// Read a value from the trie. @@ -1299,7 +1313,7 @@ mod tests { iterator_works_inner(false); } fn iterator_works_inner(flag: bool) { - let mut pairs = vec![ + let pairs = vec![ (hex!("0103000000000000000464").to_vec(), hex!("0400000000").to_vec()), (hex!("0103000000000000000469").to_vec(), hex!("0401000000").to_vec()), ]; @@ -1312,9 +1326,6 @@ mod tests { let iter = trie.iter().unwrap(); let mut iter_pairs = Vec::new(); - if flag { - pairs.insert(0, (vec![], vec![])); - } for pair in iter { let (key, value) = pair.unwrap(); iter_pairs.push((key, value.to_vec())); From 30742b782e7c3906462e546c23ef5ed07d345b6d Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 25 May 2021 18:44:00 +0200 Subject: [PATCH 020/127] fix trie root hashing of root --- Cargo.lock | 16 ++++---- primitives/trie/src/lib.rs | 9 +++-- primitives/trie/src/trie_stream.rs | 59 ++++++++++++++++++++---------- 3 files changed, 54 insertions(+), 30 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d1c2e7d69da81..3a3a556fda9a0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2329,7 +2329,7 @@ dependencies = [ [[package]] name = "hash-db" version = "0.15.2" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#09571017e3ad7ee3e04a40eb8d61bcff25e255ac" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#f5b50501d4dcc7c9c3b8b31908f677291b0e2603" [[package]] name = "hash256-std-hasher" @@ -2343,7 +2343,7 @@ dependencies = [ [[package]] name = "hash256-std-hasher" version = "0.15.2" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#09571017e3ad7ee3e04a40eb8d61bcff25e255ac" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#f5b50501d4dcc7c9c3b8b31908f677291b0e2603" dependencies = [ "crunchy", ] @@ -2996,7 +2996,7 @@ dependencies = [ [[package]] name = "keccak-hasher" version = "0.15.3" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#09571017e3ad7ee3e04a40eb8d61bcff25e255ac" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#f5b50501d4dcc7c9c3b8b31908f677291b0e2603" dependencies = [ "hash-db", "hash256-std-hasher 0.15.2 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple4)", @@ -3788,7 +3788,7 @@ dependencies = [ [[package]] name = "memory-db" version = "0.26.0" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#09571017e3ad7ee3e04a40eb8d61bcff25e255ac" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#f5b50501d4dcc7c9c3b8b31908f677291b0e2603" dependencies = [ "hash-db", "hashbrown", @@ -10332,7 +10332,7 @@ checksum = "a7f741b240f1a48843f9b8e0444fb55fb2a4ff67293b50a9179dfd5ea67f8d41" [[package]] name = "trie-bench" version = "0.27.0" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#09571017e3ad7ee3e04a40eb8d61bcff25e255ac" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#f5b50501d4dcc7c9c3b8b31908f677291b0e2603" dependencies = [ "criterion", "hash-db", @@ -10347,7 +10347,7 @@ dependencies = [ [[package]] name = "trie-db" version = "0.22.3" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#09571017e3ad7ee3e04a40eb8d61bcff25e255ac" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#f5b50501d4dcc7c9c3b8b31908f677291b0e2603" dependencies = [ "hash-db", "hashbrown", @@ -10359,7 +10359,7 @@ dependencies = [ [[package]] name = "trie-root" version = "0.16.0" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#09571017e3ad7ee3e04a40eb8d61bcff25e255ac" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#f5b50501d4dcc7c9c3b8b31908f677291b0e2603" dependencies = [ "hash-db", ] @@ -10377,7 +10377,7 @@ dependencies = [ [[package]] name = "trie-standardmap" version = "0.15.2" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#09571017e3ad7ee3e04a40eb8d61bcff25e255ac" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#f5b50501d4dcc7c9c3b8b31908f677291b0e2603" dependencies = [ "hash-db", "keccak-hasher 0.15.3 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple4)", diff --git a/primitives/trie/src/lib.rs b/primitives/trie/src/lib.rs index 05a3aa701f99e..8e0eb24f95200 100644 --- a/primitives/trie/src/lib.rs +++ b/primitives/trie/src/lib.rs @@ -1206,7 +1206,7 @@ mod tests { } fn random_should_work_inner(flag: bool) { let mut seed = ::Out::zero(); - for test_i in 0..10000 { + for test_i in 0..10_000 { if test_i % 50 == 0 { println!("{:?} of 10000 stress tests done", test_i); } @@ -1218,8 +1218,11 @@ mod tests { count: 100, }.make_with(seed.as_fixed_bytes_mut()); - let layout = Layout::default(); - // TODO implement variant for stream codec or use iter_build. + let layout = if flag { + Layout::with_inner_hashing() + } else { + Layout::default() + }; let real = layout.trie_root(x.clone()); let mut memdb = MemoryDB::default(); let mut root = Default::default(); diff --git a/primitives/trie/src/trie_stream.rs b/primitives/trie/src/trie_stream.rs index dc754fbb903ec..f3c680f245c54 100644 --- a/primitives/trie/src/trie_stream.rs +++ b/primitives/trie/src/trie_stream.rs @@ -121,10 +121,28 @@ impl trie_root::TrieStream for TrieStream { } fn append_substream(&mut self, other: Self) { + let inner_value_hashing = other.inner_value_hashing; + let range = other.current_value_range.clone(); let data = other.out(); match data.len() { 0..=31 => data.encode_to(&mut self.buffer), - _ => H::hash(&data).as_ref().encode_to(&mut self.buffer), + _ => { + if inner_value_hashing + && range.as_ref().map(|r| r.end - r.start >= trie_constants::INNER_HASH_TRESHOLD) + .unwrap_or_default() { + let meta = TrieMeta { + range: range, + unused_value: false, + contain_hash: false, + do_value_hash: true, + old_hash: false, + recorded_do_value_hash: false, + }; + >>::hash(&data, &meta).as_ref().encode_to(&mut self.buffer); + } else { + H::hash(&data).as_ref().encode_to(&mut self.buffer); + } + }, } } @@ -132,28 +150,31 @@ impl trie_root::TrieStream for TrieStream { let inner_value_hashing = self.inner_value_hashing; let range = self.current_value_range; let data = self.buffer; + let meta = TrieMeta { + range: range, + unused_value: false, + contain_hash: false, + do_value_hash: inner_value_hashing, + old_hash: false, + recorded_do_value_hash: inner_value_hashing, + }; + + // Add the recorded_do_value_hash to encoded + let mut encoded = meta.write_state_meta(); + let encoded = if encoded.len() > 0 { + encoded.extend(data); + encoded + } else { + data + }; + if inner_value_hashing - && range.as_ref().map(|r| r.end - r.start >= trie_constants::INNER_HASH_TRESHOLD) + && meta.range.as_ref().map(|r| r.end - r.start >= trie_constants::INNER_HASH_TRESHOLD) .unwrap_or_default() { - let meta = TrieMeta { - range: range, - unused_value: false, - contain_hash: false, - do_value_hash: true, - old_hash: false, - recorded_do_value_hash: true, - }; - // Add the recorded_do_value_hash to encoded - let mut encoded = meta.write_state_meta(); - let encoded = if encoded.len() > 0 { - encoded.extend(data); - encoded - } else { - data - }; + >>::hash(&encoded, &meta) } else { - H::hash(&data) + H::hash(&encoded) } } From c620869ecb3634c0fb3650663f58e565c83f4e17 Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 26 May 2021 10:03:29 +0200 Subject: [PATCH 021/127] complete basic backend. --- frame/support/test/tests/instance.rs | 2 +- primitives/state-machine/src/basic.rs | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/frame/support/test/tests/instance.rs b/frame/support/test/tests/instance.rs index e698f989de45f..91363d0857e10 100644 --- a/frame/support/test/tests/instance.rs +++ b/frame/support/test/tests/instance.rs @@ -327,7 +327,7 @@ fn storage_instance_independence() { let mut storage = sp_core::storage::Storage { top: std::collections::BTreeMap::new(), children_default: std::collections::HashMap::new(), - flag_hashed_value: false, // TODO test with + flag_hashed_value: true, }; sp_state_machine::BasicExternalities::execute_with_storage(&mut storage, || { module2::Value::::put(0); diff --git a/primitives/state-machine/src/basic.rs b/primitives/state-machine/src/basic.rs index 950b6d715b18b..054c543e4245d 100644 --- a/primitives/state-machine/src/basic.rs +++ b/primitives/state-machine/src/basic.rs @@ -283,8 +283,12 @@ impl Externalities for BasicExternalities { } } - // TODO set flag on layout??? - Layout::::default().trie_root(self.inner.top.clone()).as_ref().into() + let layout = if self.inner.flag_hashed_value { + Layout::::with_inner_hashing() + } else { + Layout::::default() + }; + layout.trie_root(self.inner.top.clone()).as_ref().into() } fn child_storage_root( From a3b10398ac97c580fdf213f262c58685dd8ada40 Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 26 May 2021 11:04:04 +0200 Subject: [PATCH 022/127] Remove old_hash meta from proof that do not use inner_hashing. --- .../state-machine/src/proving_backend.rs | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/primitives/state-machine/src/proving_backend.rs b/primitives/state-machine/src/proving_backend.rs index b431fa0c0e04d..996c1570bb0cc 100644 --- a/primitives/state-machine/src/proving_backend.rs +++ b/primitives/state-machine/src/proving_backend.rs @@ -113,6 +113,8 @@ impl<'a, S, H> ProvingBackendRecorder<'a, S, H> struct ProofRecorderInner { /// All the records that we have stored so far. records: HashMap>, + /// Is inner hash in proof. + flagged_inner_hash: bool, /// The encoded size of all recorded values. encoded_size: usize, } @@ -128,22 +130,24 @@ impl ProofRecorder { pub fn record(&self, key: Hash, mut val: Option<(DBValue, TrieMeta)>) { let mut inner = self.inner.write(); - let ProofRecorderInner { encoded_size, records } = &mut *inner; + let ProofRecorderInner { encoded_size, records, flagged_inner_hash } = &mut *inner; records.entry(key).or_insert_with(|| { if let Some(val) = val.as_mut() { + if val.1.recorded_do_value_hash { + *flagged_inner_hash = true; + } val.1.set_accessed_value(false); sp_trie::resolve_encoded_meta::(val); *encoded_size += sp_trie::estimate_entry_size(val, H::LENGTH); } val }); - } /// Record actual trie level value access. pub fn access_from(&self, key: &Hash, hash_len: usize) { let mut inner = self.inner.write(); - let ProofRecorderInner { encoded_size, records } = &mut *inner; + let ProofRecorderInner { encoded_size, records, .. } = &mut *inner; records.entry(key.clone()) .and_modify(|entry| { if let Some(entry) = entry.as_mut() { @@ -175,13 +179,19 @@ impl ProofRecorder { /// Convert into a [`StorageProof`]. pub fn to_storage_proof(&self) -> StorageProof { - let trie_nodes = self.inner.read() + let inner = self.inner.read(); + let trie_nodes = inner .records .iter() .filter_map(|(_k, v)| v.as_ref().map(|v| { + let mut meta = v.1.clone(); + if !inner.flagged_inner_hash { + // Remove the old hash meta. + meta.old_hash = false; + } < as sp_trie::TrieLayout>::MetaHasher as hash_db::MetaHasher - >::stored_value(v.0.as_slice(), v.1.clone()) + >::stored_value(v.0.as_slice(), meta) })) .collect(); From 693e38b0287de5a978a21ab2883c796668f4f8a1 Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 26 May 2021 15:06:23 +0200 Subject: [PATCH 023/127] fix trie test for empty (force layout on empty deltas). --- frame/system/src/lib.rs | 2 +- primitives/io/src/lib.rs | 12 ++ primitives/state-machine/src/backend.rs | 24 --- primitives/state-machine/src/basic.rs | 2 +- primitives/trie/src/lib.rs | 189 +++++------------------- test-utils/runtime/src/system.rs | 2 +- 6 files changed, 49 insertions(+), 182 deletions(-) diff --git a/frame/system/src/lib.rs b/frame/system/src/lib.rs index 44e7a3d5752fe..30afc77cc60b4 100644 --- a/frame/system/src/lib.rs +++ b/frame/system/src/lib.rs @@ -1408,7 +1408,7 @@ impl Pallet { >::hashed_key().to_vec() => [69u8; 32].encode() ], children_default: map![], - flag_hashed_value: false, + flag_hashed_value: true, }) } diff --git a/primitives/io/src/lib.rs b/primitives/io/src/lib.rs index 03bfd73772724..31db22a98bef2 100644 --- a/primitives/io/src/lib.rs +++ b/primitives/io/src/lib.rs @@ -1461,6 +1461,18 @@ mod tests { assert_eq!(storage::get(b"hello"), None); assert_eq!(storage::get(b"foo"), Some(b"bar".to_vec())); }); + + let value = vec![7u8; 35]; + t = BasicExternalities::new(Storage { + top: map![b"foo00".to_vec() => value.clone()], + children_default: map![], + flag_hashed_value: true, + }); + + t.execute_with(|| { + assert_eq!(storage::get(b"hello"), None); + assert_eq!(storage::get(b"foo00"), Some(value.clone())); + }); } #[test] diff --git a/primitives/state-machine/src/backend.rs b/primitives/state-machine/src/backend.rs index 771d51f274678..728e4ef9ea21f 100644 --- a/primitives/state-machine/src/backend.rs +++ b/primitives/state-machine/src/backend.rs @@ -376,30 +376,6 @@ impl Consolidate for sp_trie::GenericMemoryDB } } -/// Insert input pairs into memory db. -/// TODO unused remove? -#[cfg(test)] -pub(crate) fn insert_into_memory_db(mdb: &mut sp_trie::MemoryDB, input: I) -> Option - where - H: Hasher, - I: IntoIterator, -{ - use sp_trie::{TrieMut, trie_types::TrieDBMut}; - - let mut root = ::Out::default(); - { - let mut trie = TrieDBMut::::new(mdb, &mut root); - for (key, value) in input { - if let Err(e) = trie.insert(&key, &value) { - log::warn!(target: "trie", "Failed to write to trie: {}", e); - return None; - } - } - } - - Some(root) -} - /// Insert input pairs into memory db. #[cfg(test)] pub(crate) fn insert_into_memory_db_no_meta(mdb: &mut sp_trie::MemoryDBNoMeta, input: I) -> Option diff --git a/primitives/state-machine/src/basic.rs b/primitives/state-machine/src/basic.rs index 054c543e4245d..c9794486b6acc 100644 --- a/primitives/state-machine/src/basic.rs +++ b/primitives/state-machine/src/basic.rs @@ -129,7 +129,7 @@ impl From> for BasicExternalities { inner: Storage { top: hashmap, children_default: Default::default(), - flag_hashed_value: false, + flag_hashed_value: true, }, extensions: Default::default(), } diff --git a/primitives/trie/src/lib.rs b/primitives/trie/src/lib.rs index 8e0eb24f95200..d867321ca088f 100644 --- a/primitives/trie/src/lib.rs +++ b/primitives/trie/src/lib.rs @@ -500,49 +500,6 @@ pub mod trie_types { pub type TrieError = trie_db::TrieError; } -/* -/// Create a proof for a subset of keys in a trie. -/// -/// The `keys` may contain any set of keys regardless of each one of them is included -/// in the `db`. -/// -/// For a key `K` that is included in the `db` a proof of inclusion is generated. -/// For a key `K` that is not included in the `db` a proof of non-inclusion is generated. -/// These can be later checked in `verify_trie_proof`. -pub fn generate_trie_proof<'a, L: TrieConfiguration, I, K, DB>( - db: &DB, - root: TrieHash, - keys: I, -) -> Result>, Box>> where - I: IntoIterator, - K: 'a + AsRef<[u8]>, - DB: hash_db::HashDBRef, -{ - let trie = TrieDB::::new(db, &root)?; - generate_proof(&trie, keys) -} - -/// Verify a set of key-value pairs against a trie root and a proof. -/// -/// Checks a set of keys with optional values for inclusion in the proof that was generated by -/// `generate_trie_proof`. -/// If the value in the pair is supplied (`(key, Some(value))`), this key-value pair will be -/// checked for inclusion in the proof. -/// If the value is omitted (`(key, None)`), this key will be checked for non-inclusion in the -/// proof. -pub fn verify_trie_proof<'a, L: TrieConfiguration, I, K, V>( - root: &TrieHash, - proof: &[Vec], - items: I, -) -> Result<(), VerifyError, error::Error>> where - I: IntoIterator)>, - K: 'a + AsRef<[u8]>, - V: 'a + AsRef<[u8]>, -{ - verify_proof::, _, _, _>(root, proof, items) -} -*/ - /// Determine a trie root given a hash DB and delta values. pub fn delta_trie_root( db: &mut DB, @@ -561,6 +518,9 @@ pub fn delta_trie_root( let mut delta = delta.into_iter().collect::>(); delta.sort_by(|l, r| l.0.borrow().cmp(r.0.borrow())); + if delta.len() == 0 { + trie.force_layout_meta()?; + } for (key, change) in delta { match change.borrow() { Some(val) => trie.insert(key.borrow(), val.borrow())?, @@ -1008,18 +968,18 @@ mod tests { >::hashed_null_node() } - fn check_equivalent(input: &Vec<(&[u8], &[u8])>) { + fn check_equivalent(input: &Vec<(&[u8], &[u8])>, layout: T) { { - // TODO test flagged - let layout = T::default(); let closed_form = layout.trie_root(input.clone()); let d = layout.trie_root_unhashed(input.clone()); println!("Data: {:#x?}, {:#x?}", d, Blake2Hasher::hash(&d[..])); let persistent = { let mut memdb = MemoryDBMeta::<_, T::MetaHasher>::default(); let mut root = Default::default(); - // TODO test flagged - let mut t = TrieDBMut::::new(&mut memdb, &mut root); + let mut t = TrieDBMut::::new_with_layout(&mut memdb, &mut root, layout); + if input.len() == 0 { + t.force_layout_meta().unwrap(); + } for (x, y) in input.iter().rev() { t.insert(x, y).unwrap(); } @@ -1029,16 +989,17 @@ mod tests { } } - fn check_iteration(input: &Vec<(&[u8], &[u8])>) { + fn check_iteration(input: &Vec<(&[u8], &[u8])>, layout: T) { let mut memdb = MemoryDBMeta::<_, T::MetaHasher>::default(); let mut root = Default::default(); { - let mut t = TrieDBMut::::new(&mut memdb, &mut root); + let mut t = TrieDBMut::::new_with_layout(&mut memdb, &mut root, layout); for (x, y) in input.clone() { t.insert(x, y).unwrap(); } } { + // Not using layout: it should be initialized from state root meta. let t = TrieDB::::new(&mut memdb, &root).unwrap(); assert_eq!( input.iter().map(|(i, j)| (i.to_vec(), j.to_vec())).collect::>(), @@ -1049,6 +1010,20 @@ mod tests { } } + fn check_input(input: &Vec<(&[u8], &[u8])>) { + + let layout = Layout::with_inner_hashing(); + check_equivalent::(input, layout.clone()); + + + let layout = Layout::default(); + check_equivalent::(input, layout.clone()); + check_iteration::(input, layout); + let layout = Layout::with_inner_hashing(); + check_equivalent::(input, layout.clone()); + check_iteration::(input, layout); + } + #[test] fn default_trie_root() { let mut db = MemoryDB::default(); @@ -1066,15 +1041,13 @@ mod tests { #[test] fn empty_is_equivalent() { let input: Vec<(&[u8], &[u8])> = vec![]; - check_equivalent::(&input); - check_iteration::(&input); + check_input(&input); } #[test] fn leaf_is_equivalent() { let input: Vec<(&[u8], &[u8])> = vec![(&[0xaa][..], &[0xbb][..])]; - check_equivalent::(&input); - check_iteration::(&input); + check_input(&input); } #[test] @@ -1083,8 +1056,7 @@ mod tests { (&[0xaa][..], &[0x10][..]), (&[0xba][..], &[0x11][..]), ]; - check_equivalent::(&input); - check_iteration::(&input); + check_input(&input); } #[test] @@ -1093,8 +1065,7 @@ mod tests { (&[0xaa][..], &[0x10][..]), (&[0xab][..], &[0x11][..]), ]; - check_equivalent::(&input); - check_iteration::(&input); + check_input(&input); } #[test] @@ -1109,8 +1080,7 @@ mod tests { let mut d = st.make(); d.sort_by(|&(ref a, _), &(ref b, _)| a.cmp(b)); let dr = d.iter().map(|v| (&v.0[..], &v.1[..])).collect(); - check_equivalent::(&dr); - check_iteration::(&dr); + check_input(&dr); } #[test] @@ -1120,8 +1090,7 @@ mod tests { (&[0xaa, 0xaa][..], &[0xaa][..]), (&[0xaa, 0xbb][..], &[0xab][..]) ]; - check_equivalent::(&input); - check_iteration::(&input); + check_input(&input); } #[test] @@ -1134,8 +1103,7 @@ mod tests { (&[0xbb, 0xbb][..], &[0xbb][..]), (&[0xbb, 0xcc][..], &[0xbc][..]), ]; - check_equivalent::(&input); - check_iteration::(&input); + check_input(&input); } #[test] @@ -1144,8 +1112,7 @@ mod tests { (&[0xaa][..], &b"ABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABC"[..]), (&[0xba][..], &[0x11][..]), ]; - check_equivalent::(&input); - check_iteration::(&input); + check_input(&input); } #[test] @@ -1154,8 +1121,7 @@ mod tests { (&[0xaa][..], &b"ABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABC"[..]), (&[0xba][..], &b"ABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABC"[..]) ]; - check_equivalent::(&input); - check_iteration::(&input); + check_input(&input); } fn populate_trie<'db, T>( @@ -1259,7 +1225,6 @@ mod tests { #[test] fn codec_trie_empty() { - // TODO test other layout states. let layout = Layout::default(); let input: Vec<(&[u8], &[u8])> = vec![]; let trie = layout.trie_root_unhashed(input); @@ -1269,7 +1234,6 @@ mod tests { #[test] fn codec_trie_single_tuple() { - // TODO switch to old layout let layout = Layout::default(); let input = vec![ (vec![0xaa], vec![0xbb]) @@ -1336,92 +1300,7 @@ mod tests { assert_eq!(pairs, iter_pairs); } -/* - #[test] - fn proof_non_inclusion_works() { - let pairs = vec![ - (hex!("0102").to_vec(), hex!("01").to_vec()), - (hex!("0203").to_vec(), hex!("0405").to_vec()), - ]; - - let mut memdb = MemoryDB::default(); - let mut root = Default::default(); - populate_trie::(&mut memdb, &mut root, &pairs); - - let non_included_key: Vec = hex!("0909").to_vec(); - let proof = generate_trie_proof::( - &memdb, - root, - &[non_included_key.clone()] - ).unwrap(); - - // Verifying that the K was not included into the trie should work. - assert!(verify_trie_proof::>( - &root, - &proof, - &[(non_included_key.clone(), None)], - ).is_ok() - ); - - // Verifying that the K was included into the trie should fail. - assert!(verify_trie_proof::>( - &root, - &proof, - &[(non_included_key, Some(hex!("1010").to_vec()))], - ).is_err() - ); - } - - #[test] - fn proof_inclusion_works() { - let pairs = vec![ - (hex!("0102").to_vec(), hex!("01").to_vec()), - (hex!("0203").to_vec(), hex!("0405").to_vec()), - ]; - - let mut memdb = MemoryDB::default(); - let mut root = Default::default(); - populate_trie::(&mut memdb, &mut root, &pairs); - - let proof = generate_trie_proof::( - &memdb, - root, - &[pairs[0].0.clone()] - ).unwrap(); - // Check that a K, V included into the proof are verified. - assert!(verify_trie_proof::( - &root, - &proof, - &[(pairs[0].0.clone(), Some(pairs[0].1.clone()))] - ).is_ok() - ); - - // Absence of the V is not verified with the proof that has K, V included. - assert!(verify_trie_proof::>( - &root, - &proof, - &[(pairs[0].0.clone(), None)] - ).is_err() - ); - - // K not included into the trie is not verified. - assert!(verify_trie_proof::( - &root, - &proof, - &[(hex!("4242").to_vec(), Some(pairs[0].1.clone()))] - ).is_err() - ); - - // K included into the trie but not included into the proof is not verified. - assert!(verify_trie_proof::( - &root, - &proof, - &[(pairs[1].0.clone(), Some(pairs[1].1.clone()))] - ).is_err() - ); - } -*/ #[test] fn generate_storage_root_with_proof_works_independently_from_the_delta_order() { let proof = StorageProof::decode(&mut &include_bytes!("../test-res/proof")[..]).unwrap(); diff --git a/test-utils/runtime/src/system.rs b/test-utils/runtime/src/system.rs index fbf6012d083e7..cfffb1f535028 100644 --- a/test-utils/runtime/src/system.rs +++ b/test-utils/runtime/src/system.rs @@ -385,7 +385,7 @@ mod tests { } ], children_default: map![], - flag_hashed_value: false, // TODO test with true variant + flag_hashed_value: true, }, ) } From 6e89b3afcdae80472864ebcf245bea16a582915d Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 26 May 2021 16:17:41 +0200 Subject: [PATCH 024/127] Root update fix. --- primitives/state-machine/src/trie_backend.rs | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/primitives/state-machine/src/trie_backend.rs b/primitives/state-machine/src/trie_backend.rs index 788eb47f89588..7848663135576 100644 --- a/primitives/state-machine/src/trie_backend.rs +++ b/primitives/state-machine/src/trie_backend.rs @@ -379,12 +379,16 @@ pub mod tests { } #[test] - fn storage_root_transaction_is_empty() { - storage_root_transaction_is_empty_inner(false); - storage_root_transaction_is_empty_inner(true); - } - fn storage_root_transaction_is_empty_inner(flagged: bool) { - assert!(test_trie(flagged).storage_root(iter::empty(), false).1.drain().is_empty()); + fn storage_root_transaction_state_root_update() { + // a drop a insert of same hash: rc is 0 + assert_eq!(test_trie(false).storage_root(iter::empty(), false).1.drain() + .into_iter().filter(|v| (v.1).1 != 0).count(), 0); + // a drop a insert + assert_eq!(test_trie(false).storage_root(iter::empty(), true).1.drain() + .into_iter().filter(|v| (v.1).1 != 0).count(), 2); + // a drop a insert of same hash: rc is 0 + assert_eq!(test_trie(true).storage_root(iter::empty(), true).1.drain() + .into_iter().filter(|v| (v.1).1 != 0).count(), 0); } #[test] From f56879466330a98d996142d87ab443e837cb5b26 Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 26 May 2021 17:28:10 +0200 Subject: [PATCH 025/127] debug on meta --- primitives/trie/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/primitives/trie/src/lib.rs b/primitives/trie/src/lib.rs index d867321ca088f..aebb5c6007141 100644 --- a/primitives/trie/src/lib.rs +++ b/primitives/trie/src/lib.rs @@ -49,7 +49,7 @@ pub use hash_db::{HashDB as HashDBT, EMPTY_PREFIX, MetaHasher}; pub use hash_db::NoMeta; /// Meta use by trie state. -#[derive(Default, Clone)] +#[derive(Default, Clone, Debug)] pub struct TrieMeta { // range of encoded value or hashed value. pub range: Option>, From fb0ea035129aa965c205387464317ff6a7cb483e Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 27 May 2021 10:13:52 +0200 Subject: [PATCH 026/127] Use trie key iteration that do not include value in proofs. --- Cargo.lock | 16 ++++---- .../state-machine/src/trie_backend_essence.rs | 39 +++++++++++++------ primitives/trie/src/lib.rs | 3 +- 3 files changed, 38 insertions(+), 20 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a9894d505cb74..5ba8be6d5afd4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2342,7 +2342,7 @@ dependencies = [ [[package]] name = "hash-db" version = "0.15.2" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#f5b50501d4dcc7c9c3b8b31908f677291b0e2603" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#24f114312feeb2f17510709cc04276b0843a7eb4" [[package]] name = "hash256-std-hasher" @@ -2356,7 +2356,7 @@ dependencies = [ [[package]] name = "hash256-std-hasher" version = "0.15.2" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#f5b50501d4dcc7c9c3b8b31908f677291b0e2603" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#24f114312feeb2f17510709cc04276b0843a7eb4" dependencies = [ "crunchy", ] @@ -3001,7 +3001,7 @@ dependencies = [ [[package]] name = "keccak-hasher" version = "0.15.3" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#f5b50501d4dcc7c9c3b8b31908f677291b0e2603" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#24f114312feeb2f17510709cc04276b0843a7eb4" dependencies = [ "hash-db", "hash256-std-hasher 0.15.2 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple4)", @@ -3793,7 +3793,7 @@ dependencies = [ [[package]] name = "memory-db" version = "0.26.0" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#f5b50501d4dcc7c9c3b8b31908f677291b0e2603" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#24f114312feeb2f17510709cc04276b0843a7eb4" dependencies = [ "hash-db", "hashbrown", @@ -10375,7 +10375,7 @@ checksum = "a7f741b240f1a48843f9b8e0444fb55fb2a4ff67293b50a9179dfd5ea67f8d41" [[package]] name = "trie-bench" version = "0.27.0" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#f5b50501d4dcc7c9c3b8b31908f677291b0e2603" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#24f114312feeb2f17510709cc04276b0843a7eb4" dependencies = [ "criterion", "hash-db", @@ -10390,7 +10390,7 @@ dependencies = [ [[package]] name = "trie-db" version = "0.22.3" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#f5b50501d4dcc7c9c3b8b31908f677291b0e2603" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#24f114312feeb2f17510709cc04276b0843a7eb4" dependencies = [ "hash-db", "hashbrown", @@ -10402,7 +10402,7 @@ dependencies = [ [[package]] name = "trie-root" version = "0.16.0" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#f5b50501d4dcc7c9c3b8b31908f677291b0e2603" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#24f114312feeb2f17510709cc04276b0843a7eb4" dependencies = [ "hash-db", ] @@ -10420,7 +10420,7 @@ dependencies = [ [[package]] name = "trie-standardmap" version = "0.15.2" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#f5b50501d4dcc7c9c3b8b31908f677291b0e2603" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#24f114312feeb2f17510709cc04276b0843a7eb4" dependencies = [ "hash-db", "keccak-hasher 0.15.3 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple4)", diff --git a/primitives/state-machine/src/trie_backend_essence.rs b/primitives/state-machine/src/trie_backend_essence.rs index 5d1f50e9c5a5a..7b2bd65c293d5 100644 --- a/primitives/state-machine/src/trie_backend_essence.rs +++ b/primitives/state-machine/src/trie_backend_essence.rs @@ -25,7 +25,8 @@ use crate::{warn, debug}; use hash_db::{self, Hasher, Prefix}; use sp_trie::{Trie, PrefixedMemoryDB, DBValue, empty_child_trie_root, read_trie_value, read_child_trie_value, - for_keys_in_child_trie, KeySpacedDB, TrieDBIterator, TrieMeta}; + for_keys_in_child_trie, KeySpacedDB, TrieDBIterator, TrieDBKeyIterator, + TrieMeta}; use sp_trie::trie_types::{TrieDB, TrieError, Layout}; use crate::{backend::Consolidate, StorageKey, StorageValue}; use sp_core::storage::ChildInfo; @@ -142,7 +143,7 @@ impl, H: Hasher> TrieBackendEssence where H::Out: let trie = TrieDB::::new(dyn_eph, root) .map_err(|e| format!("TrieDB creation error: {}", e))?; - let mut iter = trie.iter() + let mut iter = trie.key_iter() .map_err(|e| format!("TrieDB iteration error: {}", e))?; // The key just after the one given in input, basically `key++0`. @@ -159,7 +160,7 @@ impl, H: Hasher> TrieBackendEssence where H::Out: let next_element = iter.next(); let next_key = if let Some(next_element) = next_element { - let (next_key, _) = next_element + let next_key = next_element .map_err(|e| format!("TrieDB iterator next error: {}", e))?; Some(next_key) } else { @@ -238,15 +239,15 @@ impl, H: Hasher> TrieBackendEssence where H::Out: }; let mut root = H::Out::default(); root.as_mut().copy_from_slice(&root_vec); - self.keys_values_with_prefix_inner(&root, prefix, |k, _v| f(k), Some(child_info)) + self.keys_with_prefix_inner(&root, prefix, |k| f(k), Some(child_info)) } /// Execute given closure for all keys starting with prefix. pub fn for_keys_with_prefix(&self, prefix: &[u8], mut f: F) { - self.keys_values_with_prefix_inner(&self.root, prefix, |k, _v| f(k), None) + self.keys_with_prefix_inner(&self.root, prefix, |k| f(k), None) } - fn keys_values_with_prefix_inner( + fn keys_with_prefix_inner( &self, root: &H::Out, prefix: &[u8], @@ -256,12 +257,12 @@ impl, H: Hasher> TrieBackendEssence where H::Out: let mut iter = move |db| -> sp_std::result::Result<(), Box>> { let trie = TrieDB::::new(db, root)?; - for x in TrieDBIterator::new_prefixed(&trie, prefix)? { - let (key, value) = x?; + for x in TrieDBKeyIterator::new_prefixed(&trie, prefix)? { + let key = x?; debug_assert!(key.starts_with(prefix)); - f(&key, &value); + f(&key); } Ok(()) @@ -279,8 +280,24 @@ impl, H: Hasher> TrieBackendEssence where H::Out: } /// Execute given closure for all key and values starting with prefix. - pub fn for_key_values_with_prefix(&self, prefix: &[u8], f: F) { - self.keys_values_with_prefix_inner(&self.root, prefix, f, None) + pub fn for_key_values_with_prefix(&self, prefix: &[u8], mut f: F) { + let mut iter = move |db| -> sp_std::result::Result<(), Box>> { + let trie = TrieDB::::new(db, &self.root)?; + + for x in TrieDBIterator::new_prefixed(&trie, prefix)? { + let (key, value) = x?; + + debug_assert!(key.starts_with(prefix)); + + f(&key, &value); + } + + Ok(()) + }; + + if let Err(e) = iter(self) { + debug!(target: "trie", "Error while iterating by prefix: {}", e); + } } } diff --git a/primitives/trie/src/lib.rs b/primitives/trie/src/lib.rs index aebb5c6007141..8efc7398a4ac3 100644 --- a/primitives/trie/src/lib.rs +++ b/primitives/trie/src/lib.rs @@ -39,7 +39,8 @@ pub use storage_proof::StorageProof; /// Various re-exports from the `trie-db` crate. pub use trie_db::{ Trie, TrieMut, DBValue, Recorder, CError, Query, TrieLayout, TrieConfiguration, - nibble_ops, TrieDBIterator, Meta, node::{NodePlan, ValuePlan}, GlobalMeta, + nibble_ops, TrieDBIterator, TrieDBKeyIterator, Meta, node::{NodePlan, ValuePlan}, + GlobalMeta, }; /// Various re-exports from the `memory-db` crate. pub use memory_db::KeyFunction; From cd98390cb21729737ad7ca149fcf9eac82a461a0 Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 27 May 2021 10:39:29 +0200 Subject: [PATCH 027/127] switch default test ext to use inner hash. --- client/executor/src/integration_tests/mod.rs | 29 ++++++++++++++++++++ primitives/state-machine/src/testing.rs | 7 ++++- 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/client/executor/src/integration_tests/mod.rs b/client/executor/src/integration_tests/mod.rs index 0ea312f10a3e4..ca6ec42cb53b8 100644 --- a/client/executor/src/integration_tests/mod.rs +++ b/client/executor/src/integration_tests/mod.rs @@ -777,3 +777,32 @@ fn panic_in_spawned_instance_panics_on_joining_its_result(wasm_method: WasmExecu assert!(format!("{}", error_result).contains("Spawned task")); } + +test_wasm_execution!(state_hashing_update); +fn state_hashing_update(wasm_method: WasmExecutionMethod) { + // use externalities without storage flag. + let mut ext = TestExternalities::new(Default::default()); + + let mut ext = ext.ext(); + ext.set_storage(b"foo".to_vec(), vec![1u8; 1_000]); // big inner hash + ext.set_storage(b"foo2".to_vec(), vec![3u8; 16]); // no inner hash + ext.set_storage(b"foo222".to_vec(), vec![5u8; 100]); // inner hash + + let output = call_in_wasm( + "test_data_in", + &b"Hello world".to_vec().encode(), + wasm_method, + &mut ext, + ).unwrap(); + + assert_eq!(output, b"all ok!".to_vec().encode()); + + // TODO new call in wasm to flag state. + + // TODO Query check + + // TODO update values with same value (same root likely). + + + // TODO update values with same value (same root likely). +} diff --git a/primitives/state-machine/src/testing.rs b/primitives/state-machine/src/testing.rs index 033590b6d1052..b79d6866edcca 100644 --- a/primitives/state-machine/src/testing.rs +++ b/primitives/state-machine/src/testing.rs @@ -236,7 +236,12 @@ impl Default for TestExternalities where H::Out: Ord + 'static + codec::Codec, { - fn default() -> Self { Self::new(Default::default()) } + fn default() -> Self { + // default to inner hashed. + let mut storage = Storage::default(); + storage.flag_hashed_value = true; + Self::new(storage) + } } impl From for TestExternalities From 5ab0c012532ad9cce975bff699812c5244afe613 Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 27 May 2021 16:39:36 +0200 Subject: [PATCH 028/127] small integration test, and fix tx cache mgmt in ext. test failing --- client/executor/runtime-test/src/lib.rs | 7 +++ client/executor/src/integration_tests/mod.rs | 51 ++++++++++++++++++-- primitives/state-machine/src/ext.rs | 1 + 3 files changed, 55 insertions(+), 4 deletions(-) diff --git a/client/executor/runtime-test/src/lib.rs b/client/executor/runtime-test/src/lib.rs index bfba4ef039395..78b824140bcc7 100644 --- a/client/executor/runtime-test/src/lib.rs +++ b/client/executor/runtime-test/src/lib.rs @@ -62,6 +62,13 @@ sp_core::wasm_export_functions! { b"all ok!".to_vec() } + fn test_switch_state() { + print("switch_state"); + storage::flag_hash_value(); + print("switched!"); + } + + fn test_clear_prefix(input: Vec) -> Vec { storage::clear_prefix(&input); b"all ok!".to_vec() diff --git a/client/executor/src/integration_tests/mod.rs b/client/executor/src/integration_tests/mod.rs index ca6ec42cb53b8..75435b4c57041 100644 --- a/client/executor/src/integration_tests/mod.rs +++ b/client/executor/src/integration_tests/mod.rs @@ -797,12 +797,55 @@ fn state_hashing_update(wasm_method: WasmExecutionMethod) { assert_eq!(output, b"all ok!".to_vec().encode()); - // TODO new call in wasm to flag state. + let root1 = ext.storage_root(); + // flag state. + let _ = call_in_wasm( + "test_switch_state", + Default::default(), + wasm_method, + &mut ext, + ).unwrap(); + let root2 = ext.storage_root(); + + assert!(root1 != root2); + + // Same value update do not change root. + let output = call_in_wasm( + "test_data_in", + &b"Hello world".to_vec().encode(), + wasm_method, + &mut ext, + ).unwrap(); + + assert_eq!(output, b"all ok!".to_vec().encode()); + + let root3 = ext.storage_root(); + assert!(root2 == root3); - // TODO Query check + // change does + let output = call_in_wasm( + "test_data_in", + &b"Hello".to_vec().encode(), + wasm_method, + &mut ext, + ).unwrap(); + + assert_eq!(output, b"all ok!".to_vec().encode()); - // TODO update values with same value (same root likely). + let root3 = ext.storage_root(); + assert!(root2 != root3); + // restore is different from original + let output = call_in_wasm( + "test_data_in", + &b"Hello world".to_vec().encode(), + wasm_method, + &mut ext, + ).unwrap(); + + assert_eq!(output, b"all ok!".to_vec().encode()); - // TODO update values with same value (same root likely). + let root4 = ext.storage_root(); + assert!(root2 != root4); + assert!(root3 != root4); } diff --git a/primitives/state-machine/src/ext.rs b/primitives/state-machine/src/ext.rs index 972ea34fa687d..0b3e155c11594 100644 --- a/primitives/state-machine/src/ext.rs +++ b/primitives/state-machine/src/ext.rs @@ -739,6 +739,7 @@ where } fn flag_hash_value(&mut self) { + self.mark_dirty(); self.overlay.set_flag_hash_value() } } From 99ef85c50a80674bd18ecf5cf777ba630b37dd4e Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 27 May 2021 17:30:45 +0200 Subject: [PATCH 029/127] Proof scenario at state-machine level. --- client/executor/src/integration_tests/mod.rs | 41 +------- primitives/state-machine/src/lib.rs | 100 +++++++++++++++++++ primitives/state-machine/src/trie_backend.rs | 2 +- 3 files changed, 102 insertions(+), 41 deletions(-) diff --git a/client/executor/src/integration_tests/mod.rs b/client/executor/src/integration_tests/mod.rs index 75435b4c57041..5076352dff40a 100644 --- a/client/executor/src/integration_tests/mod.rs +++ b/client/executor/src/integration_tests/mod.rs @@ -807,45 +807,6 @@ fn state_hashing_update(wasm_method: WasmExecutionMethod) { ).unwrap(); let root2 = ext.storage_root(); + // Note that in this case all the value did switch (in memory changes). assert!(root1 != root2); - - // Same value update do not change root. - let output = call_in_wasm( - "test_data_in", - &b"Hello world".to_vec().encode(), - wasm_method, - &mut ext, - ).unwrap(); - - assert_eq!(output, b"all ok!".to_vec().encode()); - - let root3 = ext.storage_root(); - assert!(root2 == root3); - - // change does - let output = call_in_wasm( - "test_data_in", - &b"Hello".to_vec().encode(), - wasm_method, - &mut ext, - ).unwrap(); - - assert_eq!(output, b"all ok!".to_vec().encode()); - - let root3 = ext.storage_root(); - assert!(root2 != root3); - - // restore is different from original - let output = call_in_wasm( - "test_data_in", - &b"Hello world".to_vec().encode(), - wasm_method, - &mut ext, - ).unwrap(); - - assert_eq!(output, b"all ok!".to_vec().encode()); - - let root4 = ext.storage_root(); - assert!(root2 != root4); - assert!(root3 != root4); } diff --git a/primitives/state-machine/src/lib.rs b/primitives/state-machine/src/lib.rs index cb910552e814a..726d1e51b080b 100644 --- a/primitives/state-machine/src/lib.rs +++ b/primitives/state-machine/src/lib.rs @@ -1493,6 +1493,106 @@ mod tests { ); } + #[test] + fn inner_state_hashing_switch_proofs() { + + let (mut mdb, mut root) = trie_backend::tests::test_db(false); + { + let mut trie = TrieDBMut::from_existing_with_layout( + &mut mdb, + &mut root, + Layout::default(), + ).unwrap(); + trie.insert(b"foo", vec![1u8; 1_000].as_slice()) // big inner hash + .expect("insert failed"); + trie.insert(b"foo2", vec![3u8; 16].as_slice()) // no inner hash + .expect("insert failed"); + trie.insert(b"foo222", vec![5u8; 100].as_slice()) // inner hash + .expect("insert failed"); + } + + let check_proof = |mdb, root| -> StorageProof { + let remote_backend = TrieBackend::new(mdb, root); + let remote_root = remote_backend.storage_root(::std::iter::empty(), false).0; + let remote_proof = prove_read(remote_backend, &[b"foo222"]).unwrap(); + // check proof locally + let local_result1 = read_proof_check::( + remote_root, + remote_proof.clone(), + &[b"foo222"], + ).unwrap(); + // check that results are correct + assert_eq!( + local_result1.into_iter().collect::>(), + vec![(b"foo222".to_vec(), Some(vec![5u8; 100]))], + ); + remote_proof + }; + + let remote_proof = check_proof(mdb.clone(), root.clone()); + + // check full values in proof + assert!(remote_proof.encode().len() > 1_100); + assert!(remote_proof.encoded_size() > 1_100); + let root1 = root.clone(); + + + // trigger switch + { + let mut trie = TrieDBMut::from_existing_with_layout( + &mut mdb, + &mut root, + Layout::with_inner_hashing(), + ).unwrap(); + trie.force_layout_meta() + .expect("failed forced layout change"); + } + let root2 = root.clone(); + assert!(root1 != root2); + let remote_proof = check_proof(mdb.clone(), root.clone()); + // nodes are still with old hashing. + assert!(remote_proof.encode().len() > 1_100); + assert!(remote_proof.encoded_size() > 1_100); + assert_eq!(remote_proof.encode().len(), + remote_proof.encoded_size()); + + // update with same value do not change + { + let mut trie = TrieDBMut::from_existing_with_layout( + &mut mdb, + &mut root, + Layout::default(), + ).unwrap(); + trie.insert(b"foo222", vec![5u8; 100].as_slice()) // inner hash + .expect("insert failed"); + } + let root3 = root.clone(); + assert!(root2 == root3); + // different value then same is enough to update + // from triedbmut persipective (do not + // work with state machine as only changes do makes + // it to payload (would require a special host function). + { + let mut trie = TrieDBMut::from_existing_with_layout( + &mut mdb, + &mut root, + Layout::default(), + ).unwrap(); + trie.insert(b"foo222", vec![4u8].as_slice()) // inner hash + .expect("insert failed"); + trie.insert(b"foo222", vec![5u8; 100].as_slice()) // inner hash + .expect("insert failed"); + } + let root3 = root.clone(); + assert!(root2 != root3); + let remote_proof = check_proof(mdb.clone(), root.clone()); + // nodes foo is replaced by its hashed value form. + assert!(remote_proof.encode().len() < 1000); + assert!(remote_proof.encoded_size() < 1000); + assert_eq!(remote_proof.encode().len(), + remote_proof.encoded_size()); + } + #[test] fn child_storage_uuid() { diff --git a/primitives/state-machine/src/trie_backend.rs b/primitives/state-machine/src/trie_backend.rs index bb99f0dc331f3..afee980aedc6a 100644 --- a/primitives/state-machine/src/trie_backend.rs +++ b/primitives/state-machine/src/trie_backend.rs @@ -275,7 +275,7 @@ pub mod tests { const CHILD_KEY_1: &[u8] = b"sub1"; - fn test_db(hashed_value: bool) -> (PrefixedMemoryDB, H256) { + pub(crate) fn test_db(hashed_value: bool) -> (PrefixedMemoryDB, H256) { let child_info = ChildInfo::new_default(CHILD_KEY_1); let mut root = H256::default(); let mut mdb = PrefixedMemoryDB::::default(); From 4d6aee80f037c7e471564c4dc8421c8f893ad15e Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 27 May 2021 17:58:52 +0200 Subject: [PATCH 030/127] trace for db upgrade --- client/db/src/upgrade.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/client/db/src/upgrade.rs b/client/db/src/upgrade.rs index f5c0992ea231f..835ff1ba3aa42 100644 --- a/client/db/src/upgrade.rs +++ b/client/db/src/upgrade.rs @@ -26,6 +26,7 @@ use sp_runtime::traits::Block as BlockT; use crate::{columns, utils::DatabaseType}; use kvdb_rocksdb::{Database, DatabaseConfig}; use codec::{Decode, Encode}; +use log::info; /// Version file name. const VERSION_FILE_NAME: &'static str = "db_version"; @@ -102,11 +103,16 @@ fn migrate_2_to_3(db_path: &Path, _db_type: DatabaseType) -> sp_b /// Migration from version3 to version4: /// - Trie state meta for state that could be hashed internaly. fn migrate_3_to_4(db_path: &Path, _db_type: DatabaseType) -> sp_blockchain::Result<()> { + + info!("Starting trie node migration."); + let start_time = std::time::Instant::now(); let db_path = db_path.to_str() .ok_or_else(|| sp_blockchain::Error::Backend("Invalid database path".into()))?; let db_cfg = DatabaseConfig::with_columns(V2_NUM_COLUMNS); let db = Database::open(&db_cfg, db_path).map_err(db_err)?; + let mut nb_node_prefixed = 0; + let mut nb_node_seen = 0; let batch_size = 10_000; // TODO use bigger size (need to iterate all each time). loop { let mut full_batch = false; @@ -116,8 +122,10 @@ fn migrate_3_to_4(db_path: &Path, _db_type: DatabaseType) -> sp_b // Note that every batch will restart full iter, could use // a `iter_from` function. for entry in db.iter(columns::STATE) { + nb_node_seen += 1; if let Some(new_val) = sp_trie::tag_old_hashes::>(&entry.1) { transaction.put_vec(columns::STATE, &entry.0, new_val); + nb_node_prefixed += 1; size += 1; if size == batch_size { full_batch = true; @@ -125,11 +133,14 @@ fn migrate_3_to_4(db_path: &Path, _db_type: DatabaseType) -> sp_b } } } + info!("Committing batch, currently processed: {} of {} read nodes", nb_node_prefixed, nb_node_seen); db.write(transaction).map_err(db_err)?; if !full_batch { break; } } + info!("Trie node migration finished in {:?} ms.", start_time.elapsed().as_millis()); + info!("{:?} nodes prefixed for {:?} node.", nb_node_prefixed, nb_node_seen); Ok(()) } From 791acae981f706672acef4e11ce878e93e6cf97a Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 27 May 2021 20:04:02 +0200 Subject: [PATCH 031/127] try different param --- client/db/src/upgrade.rs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/client/db/src/upgrade.rs b/client/db/src/upgrade.rs index 835ff1ba3aa42..2197e5b11d82e 100644 --- a/client/db/src/upgrade.rs +++ b/client/db/src/upgrade.rs @@ -113,10 +113,11 @@ fn migrate_3_to_4(db_path: &Path, _db_type: DatabaseType) -> sp_b let mut nb_node_prefixed = 0; let mut nb_node_seen = 0; - let batch_size = 10_000; // TODO use bigger size (need to iterate all each time). + let batch_size = 50_000; // TODO use bigger size (need to iterate all each time). loop { let mut full_batch = false; let mut size = 0; + let mut last = vec![0u8, 0u8, 0u8, 0u8]; let mut transaction = db.transaction(); // Get all the keys we need to update. // Note that every batch will restart full iter, could use @@ -129,11 +130,19 @@ fn migrate_3_to_4(db_path: &Path, _db_type: DatabaseType) -> sp_b size += 1; if size == batch_size { full_batch = true; + if entry.0.len() > 3 { + last.copy_from_slice(&entry.0[..4]); + } break; } } } - info!("Committing batch, currently processed: {} of {} read nodes", nb_node_prefixed, nb_node_seen); + info!( + "Committing batch, currently processed: {} of {} read nodes at {:?}", + nb_node_prefixed, + nb_node_seen, + last, + ); db.write(transaction).map_err(db_err)?; if !full_batch { break; From b51eaa4c1e8e720f14bf126a101d15daa3c622d1 Mon Sep 17 00:00:00 2001 From: cheme Date: Fri, 28 May 2021 09:17:24 +0200 Subject: [PATCH 032/127] act more like iter_from. --- client/db/src/upgrade.rs | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/client/db/src/upgrade.rs b/client/db/src/upgrade.rs index 2197e5b11d82e..ad6ebfb2a70cc 100644 --- a/client/db/src/upgrade.rs +++ b/client/db/src/upgrade.rs @@ -117,31 +117,34 @@ fn migrate_3_to_4(db_path: &Path, _db_type: DatabaseType) -> sp_b loop { let mut full_batch = false; let mut size = 0; - let mut last = vec![0u8, 0u8, 0u8, 0u8]; + let mut last = Vec::new(); let mut transaction = db.transaction(); // Get all the keys we need to update. - // Note that every batch will restart full iter, could use - // a `iter_from` function. + // Note that every batch will restart full iter, + // if this prove to slow for archive node, this could be + // switch to a `iter_from` function but would require + // to upstream change to our rocksdb crate. for entry in db.iter(columns::STATE) { - nb_node_seen += 1; - if let Some(new_val) = sp_trie::tag_old_hashes::>(&entry.1) { - transaction.put_vec(columns::STATE, &entry.0, new_val); - nb_node_prefixed += 1; - size += 1; - if size == batch_size { - full_batch = true; - if entry.0.len() > 3 { - last.copy_from_slice(&entry.0[..4]); + if &entry.1[..] > last.as_slice() { + nb_node_seen += 1; + if let Some(new_val) = sp_trie::tag_old_hashes::>(&entry.1) { + transaction.put_vec(columns::STATE, &entry.0, new_val); + nb_node_prefixed += 1; + size += 1; + if size == batch_size { + full_batch = true; + last = entry.0.to_vec(); + break; } - break; } } } info!( - "Committing batch, currently processed: {} of {} read nodes at {:?}", + "Committing batch, currently processed: {} of {} read nodes at {:?}, {:?}", nb_node_prefixed, nb_node_seen, last, + start_time.elapsed().as_millis(), ); db.write(transaction).map_err(db_err)?; if !full_batch { From 715b1a536f2acd8e9143f593a20f482bc7bfc696 Mon Sep 17 00:00:00 2001 From: cheme Date: Fri, 28 May 2021 10:07:08 +0200 Subject: [PATCH 033/127] Bigger batches. --- client/db/src/upgrade.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/db/src/upgrade.rs b/client/db/src/upgrade.rs index ad6ebfb2a70cc..f0129251b691e 100644 --- a/client/db/src/upgrade.rs +++ b/client/db/src/upgrade.rs @@ -113,7 +113,7 @@ fn migrate_3_to_4(db_path: &Path, _db_type: DatabaseType) -> sp_b let mut nb_node_prefixed = 0; let mut nb_node_seen = 0; - let batch_size = 50_000; // TODO use bigger size (need to iterate all each time). + let batch_size = 250_000; loop { let mut full_batch = false; let mut size = 0; From 3cdb0ed5fdfdaa1f02b7a7cbb4c310e99941814f Mon Sep 17 00:00:00 2001 From: cheme Date: Fri, 28 May 2021 14:42:48 +0200 Subject: [PATCH 034/127] Update trie dependency. --- Cargo.lock | 18 +++++++++--------- primitives/state-machine/Cargo.toml | 2 +- primitives/trie/Cargo.toml | 2 +- test-utils/runtime/Cargo.toml | 2 +- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5ba8be6d5afd4..24c3ff64dee71 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2342,7 +2342,7 @@ dependencies = [ [[package]] name = "hash-db" version = "0.15.2" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#24f114312feeb2f17510709cc04276b0843a7eb4" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#da84037309542514f9eebc853909bcbfb2838805" [[package]] name = "hash256-std-hasher" @@ -2356,7 +2356,7 @@ dependencies = [ [[package]] name = "hash256-std-hasher" version = "0.15.2" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#24f114312feeb2f17510709cc04276b0843a7eb4" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#da84037309542514f9eebc853909bcbfb2838805" dependencies = [ "crunchy", ] @@ -3001,7 +3001,7 @@ dependencies = [ [[package]] name = "keccak-hasher" version = "0.15.3" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#24f114312feeb2f17510709cc04276b0843a7eb4" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#da84037309542514f9eebc853909bcbfb2838805" dependencies = [ "hash-db", "hash256-std-hasher 0.15.2 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple4)", @@ -3793,7 +3793,7 @@ dependencies = [ [[package]] name = "memory-db" version = "0.26.0" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#24f114312feeb2f17510709cc04276b0843a7eb4" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#da84037309542514f9eebc853909bcbfb2838805" dependencies = [ "hash-db", "hashbrown", @@ -10375,7 +10375,7 @@ checksum = "a7f741b240f1a48843f9b8e0444fb55fb2a4ff67293b50a9179dfd5ea67f8d41" [[package]] name = "trie-bench" version = "0.27.0" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#24f114312feeb2f17510709cc04276b0843a7eb4" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#da84037309542514f9eebc853909bcbfb2838805" dependencies = [ "criterion", "hash-db", @@ -10389,8 +10389,8 @@ dependencies = [ [[package]] name = "trie-db" -version = "0.22.3" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#24f114312feeb2f17510709cc04276b0843a7eb4" +version = "0.22.5" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#da84037309542514f9eebc853909bcbfb2838805" dependencies = [ "hash-db", "hashbrown", @@ -10402,7 +10402,7 @@ dependencies = [ [[package]] name = "trie-root" version = "0.16.0" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#24f114312feeb2f17510709cc04276b0843a7eb4" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#da84037309542514f9eebc853909bcbfb2838805" dependencies = [ "hash-db", ] @@ -10420,7 +10420,7 @@ dependencies = [ [[package]] name = "trie-standardmap" version = "0.15.2" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#24f114312feeb2f17510709cc04276b0843a7eb4" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#da84037309542514f9eebc853909bcbfb2838805" dependencies = [ "hash-db", "keccak-hasher 0.15.3 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple4)", diff --git a/primitives/state-machine/Cargo.toml b/primitives/state-machine/Cargo.toml index 79fccef08c199..e04dfeb6b1415 100644 --- a/primitives/state-machine/Cargo.toml +++ b/primitives/state-machine/Cargo.toml @@ -18,7 +18,7 @@ log = { version = "0.4.11", optional = true } thiserror = { version = "1.0.21", optional = true } parking_lot = { version = "0.11.1", optional = true } hash-db = { version = "0.15.2", default-features = false } -trie-db = { version = "0.22.2", default-features = false } +trie-db = { version = "0.22.5", default-features = false } trie-root = { version = "0.16.0", default-features = false } sp-trie = { version = "3.0.0", path = "../trie", default-features = false } sp-core = { version = "3.0.0", path = "../core", default-features = false } diff --git a/primitives/trie/Cargo.toml b/primitives/trie/Cargo.toml index 4396550a48a8f..bf91fff31b8b6 100644 --- a/primitives/trie/Cargo.toml +++ b/primitives/trie/Cargo.toml @@ -21,7 +21,7 @@ harness = false codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false } sp-std = { version = "3.0.0", default-features = false, path = "../std" } hash-db = { version = "0.15.2", default-features = false } -trie-db = { version = "0.22.2", default-features = false } +trie-db = { version = "0.22.5", default-features = false } trie-root = { version = "0.16.0", default-features = false } memory-db = { version = "0.26.0", default-features = false } sp-core = { version = "3.0.0", default-features = false, path = "../core" } diff --git a/test-utils/runtime/Cargo.toml b/test-utils/runtime/Cargo.toml index 96b7efff83380..0848d0506dfc4 100644 --- a/test-utils/runtime/Cargo.toml +++ b/test-utils/runtime/Cargo.toml @@ -38,7 +38,7 @@ pallet-timestamp = { version = "3.0.0", default-features = false, path = "../../ sp-finality-grandpa = { version = "3.0.0", default-features = false, path = "../../primitives/finality-grandpa" } sp-trie = { version = "3.0.0", default-features = false, path = "../../primitives/trie" } sp-transaction-pool = { version = "3.0.0", default-features = false, path = "../../primitives/transaction-pool" } -trie-db = { version = "0.22.2", default-features = false } +trie-db = { version = "0.22.5", default-features = false } parity-util-mem = { version = "0.9.0", default-features = false, features = ["primitive-types"] } sc-service = { version = "0.9.0", default-features = false, optional = true, features = ["test-helpers"], path = "../../client/service" } sp-state-machine = { version = "0.9.0", default-features = false, path = "../../primitives/state-machine" } From ae1454f2b6a6956e4157245abc42d05f9f45dcef Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 8 Jun 2021 12:22:48 +0200 Subject: [PATCH 035/127] drafting codec changes and refact --- primitives/trie/src/lib.rs | 193 +++++++++++++-------------- primitives/trie/src/node_codec.rs | 84 ++++++------ primitives/trie/src/node_header.rs | 63 +++++++-- primitives/trie/src/storage_proof.rs | 1 + 4 files changed, 187 insertions(+), 154 deletions(-) diff --git a/primitives/trie/src/lib.rs b/primitives/trie/src/lib.rs index 8efc7398a4ac3..57aa05adeed04 100644 --- a/primitives/trie/src/lib.rs +++ b/primitives/trie/src/lib.rs @@ -52,85 +52,104 @@ pub use hash_db::NoMeta; /// Meta use by trie state. #[derive(Default, Clone, Debug)] pub struct TrieMeta { - // range of encoded value or hashed value. + /// Range of encoded value or hashed value. pub range: Option>, - // When `do_value_hash` is true, try to - // store this behavior in top node - // encoded (need to be part of state). + /// Defined in the trie layout, when used with + /// `TrieDbMut` it switch nodes to alternative hashing + /// method by setting `do_value_hash` to true. + /// TODO may be useless (indicate that previous hash is + /// not using `do_value_hash`). + pub switch_to_value_hash: bool, + /// When `do_value_hash` is true, try to + /// store this behavior in top node + /// encoded (need to be part of state). + /// TODO remove pub recorded_do_value_hash: bool, - // Does current encoded contains a hash instead of - // a value (information stored in meta for proofs). + /// Does current encoded contains a hash instead of + /// a value (information stored in meta for proofs). pub contain_hash: bool, - // Flag indicating if value hash can run. - // When defined for a node it gets active - // for all children node + /// Flag indicating if alternative value hash can run. + /// This is read and written as a state meta of the node. + /// TODO replace by TrieDbMut node variant + /// TODO replace by Option being size treshold. pub do_value_hash: bool, - // Record if a value was accessed, this is - // set as accessed by defalult, but can be - // change on access explicitely: `HashDB::get_with_meta`. - // and reset on access explicitely: `HashDB::access_from`. + /// Record if a value was accessed, this is + /// set as accessed by defalult, but can be + /// change on access explicitely: `HashDB::get_with_meta`. + /// and reset on access explicitely: `HashDB::access_from`. + /// TODO!! remove from meta: only use in proof recorder context. pub unused_value: bool, - // Indicate that a node is using old hash scheme. - // Write with `do_value_hash` inactive will set this to - // true. - // In this case hash is not doing internal hashing, - // but next write with `do_value_hash` will remove switch scheme. + /// Indicate that a node is using old hash scheme. + /// TODO remove pub old_hash: bool, } impl Meta for TrieMeta { - /// Layout do not have content. + /// When true apply inner hashing of value. type GlobalMeta = bool; + // TODO remove upstraem /// When true apply inner hashing of value. type StateMeta = bool; - fn set_state_meta(&mut self, state_meta: Self::StateMeta) { - self.recorded_do_value_hash = state_meta; - self.do_value_hash = state_meta; + // TODO remove upstream + fn set_state_meta(&mut self, _state_meta: Self::StateMeta) { + /*if !self.do_value_hash && state_meta { + self.switch_to_value_hash = true; + self.do_value_hash = true; + }*/ } + // TODO remove upstream fn extract_global_meta(&self) -> Self::GlobalMeta { self.recorded_do_value_hash } fn set_global_meta(&mut self, global_meta: Self::GlobalMeta) { - if global_meta { - self.recorded_do_value_hash = true; + if !self.do_value_hash && state_meta { + self.switch_to_value_hash = true; self.do_value_hash = true; } } + // TODO remove upstream? fn has_state_meta(&self) -> bool { - self.recorded_do_value_hash + false + //self.do_value_hash } + // TODO consider removal upstream of this method (node type in codec) fn read_state_meta(&mut self, data: &[u8]) -> Result { - let offset = if data[0] == trie_constants::ENCODED_META_ALLOW_HASH { + unreachable!() + // TODO read directly from codec. +/* let offset = if data[0] == trie_constants::ENCODED_META_ALLOW_HASH { self.recorded_do_value_hash = true; self.do_value_hash = true; 1 } else { 0 }; - Ok(offset) + Ok(offset)*/ } + // TODO consider removal upstream of this method (node type in codec) + // `do_value_hash` method is enough function to write with codec. fn write_state_meta(&self) -> Vec { - if self.recorded_do_value_hash { - // Note that this only works with sp_trie codec that - // cannot encode node starting by this byte. + unreachable!() +/* if self.do_value_hash { + // Note that this only works with sp_trie codec. + // Acts as a boolean result. [trie_constants::ENCODED_META_ALLOW_HASH].to_vec() } else { Vec::new() - } + }*/ } fn meta_for_new( global: Self::GlobalMeta, ) -> Self { let mut result = Self::default(); - result.do_value_hash = global; + result.set_global_meta(global); result } @@ -140,12 +159,14 @@ impl Meta for TrieMeta { Self::meta_for_new(global) } + // TODO meta for empty is unused: can consider removal upstream. fn meta_for_empty( global: Self::GlobalMeta, ) -> Self { Self::meta_for_new(global) } + // TODO if removing all meta, the Option will replace it. fn encoded_value_callback( &mut self, value_plan: ValuePlan, @@ -158,9 +179,9 @@ impl Meta for TrieMeta { self.range = Some(range); self.contain_hash = contain_hash; - if self.do_value_hash { - // Switch value hashing. - self.old_hash = false; + if self.switch_to_value_hash { + // Switched value hashing. + self.switch_to_value_hash = false } } @@ -184,7 +205,7 @@ impl Meta for TrieMeta { } fn do_value_hash(&self) -> bool { - self.unused_value + self.do_value_hash } } @@ -224,6 +245,7 @@ impl Default for Layout { impl Layout { /// Layout with inner hashing active. /// Will flag trie for hashing. + /// TODO rename inner -> alt pub fn with_inner_hashing() -> Self { Layout(true, sp_std::marker::PhantomData) } @@ -248,13 +270,19 @@ impl TrieLayout for Layout fn layout_meta(&self) -> GlobalMeta { self.0 } - fn initialize_from_root_meta(&mut self, root_meta: &Self::Meta) { - if root_meta.extract_global_meta() { + + // TODO remove upstream + fn initialize_from_root_meta(&mut self, _root_meta: &Self::Meta) { + unreachable!() + /*if root_meta.extract_global_meta() { self.0 = true; - } + }*/ } - fn set_root_meta(root_meta: &mut Self::Meta, global_meta: GlobalMeta) { - root_meta.set_global_meta(global_meta); + + // TODO remove upstream + fn set_root_meta(_root_meta: &mut Self::Meta, _global_meta: GlobalMeta) { + unreachable!() +// root_meta.set_global_meta(global_meta); } } @@ -271,7 +299,7 @@ impl MetaHasher for StateHasher fn hash(value: &[u8], meta: &Self::Meta) -> H::Out { match &meta { - TrieMeta { range: Some(range), contain_hash: false, do_value_hash, old_hash: false, .. } => { + TrieMeta { range: Some(range), contain_hash: false, do_value_hash, switch_to_value_hash: false, .. } => { if *do_value_hash && range.end - range.start >= trie_constants::INNER_HASH_TRESHOLD { let value = inner_hashed_value::(value, Some((range.start, range.end))); H::hash(value.as_slice()) @@ -289,24 +317,10 @@ impl MetaHasher for StateHasher } } + // TODO if removing meta upstream, still need to get DEAD_HEADER_META_HASHED_VALUE + // from proof. fn stored_value(value: &[u8], mut meta: Self::Meta) -> DBValue { let mut stored = Vec::with_capacity(value.len() + 1); - if meta.old_hash { - // write as old hash. - stored.push(trie_constants::OLD_HASHING); - stored.extend_from_slice(value); - return stored; - } - if !meta.do_value_hash { - if let Some(range) = meta.range.as_ref() { - if range.end - range.start >= trie_constants::INNER_HASH_TRESHOLD { - // write as old hash. - stored.push(trie_constants::OLD_HASHING); - stored.extend_from_slice(value); - return stored; - } - } - } if meta.contain_hash { // already contain hash, just flag it. stored.push(trie_constants::DEAD_HEADER_META_HASHED_VALUE); @@ -336,6 +350,7 @@ impl MetaHasher for StateHasher >::stored_value(value.as_slice(), meta) } + // TODO remove upstream? fn extract_value(mut stored: &[u8], global_meta: Self::GlobalMeta) -> (&[u8], Self::Meta) { let input = &mut stored; let mut contain_hash = false; @@ -344,10 +359,6 @@ impl MetaHasher for StateHasher contain_hash = true; *input = &input[1..]; } - if input.get(0) == Some(&trie_constants::OLD_HASHING) { - old_hash = true; - *input = &input[1..]; - } let mut meta = TrieMeta { range: None, unused_value: contain_hash, @@ -356,14 +367,11 @@ impl MetaHasher for StateHasher recorded_do_value_hash: false, old_hash, }; - // get recorded_do_value_hash - let _offset = meta.read_state_meta(stored) - .expect("State meta reading failure."); - //let stored = &stored[offset..]; - meta.do_value_hash = meta.recorded_do_value_hash || global_meta; + meta.set_global_meta(global_meta); (stored, meta) } + // TODO remove upstream fn extract_value_owned(mut stored: DBValue, global: Self::GlobalMeta) -> (DBValue, Self::Meta) { let len = stored.len(); let (v, meta) = >::extract_value(stored.as_slice(), global); @@ -374,6 +382,8 @@ impl MetaHasher for StateHasher /// Reimplement `NoMeta` `MetaHasher` with /// additional constraint. +/// TODO remove the MetaHasher is ignored +/// when no node have do_value_hash or layout defines it. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub struct NoMetaHasher; @@ -900,7 +910,10 @@ pub fn estimate_entry_size(entry: &(DBValue, TrieMeta), hash_len: usize) -> usiz full_encoded } -/// If needed, call to decode plan in order to record meta. +/// If needed, call to decode plan in order to update meta earlier. +/// TODO if removing fully meta, this will still be needed but with +/// a less generic name: read variant of node from db value and indicate +/// if can hash value. pub fn resolve_encoded_meta(entry: &mut (DBValue, TrieMeta)) { use trie_db::NodeCodec; if entry.1.do_value_hash { @@ -912,45 +925,25 @@ pub fn resolve_encoded_meta(entry: &mut (DBValue, TrieMeta)) { mod trie_constants { /// Treshold for using hash of value instead of value /// in encoded trie node when flagged. + /// TODO design would be to make it the global meta, but then + /// when serializing proof we would need to attach it (no way to + /// hash the nodes otherwhise), which would + /// break proof format. + /// TODO attaching to storage proof in a compatible way could be + /// achieve by using a escaped header in first or last element of proof + /// and write it after. pub const INNER_HASH_TRESHOLD: usize = 33; const FIRST_PREFIX: u8 = 0b_00 << 6; - pub const EMPTY_TRIE: u8 = FIRST_PREFIX | 0b_00; - pub const ENCODED_META_ALLOW_HASH: u8 = FIRST_PREFIX | 0b_01; /// In proof this header is used when only hashed value is stored. - pub const DEAD_HEADER_META_HASHED_VALUE: u8 = FIRST_PREFIX | 0b_00_10; - /// If inner hashing should apply, but state is not flagged, then set - /// this meta to avoid checking both variant of hashes. - pub const OLD_HASHING: u8 = FIRST_PREFIX | 0b_00_11; + pub const DEAD_HEADER_META_HASHED_VALUE: u8 = EMPTY_TRIE | 0b_00_01; pub const NIBBLE_SIZE_BOUND: usize = u16::max_value() as usize; pub const LEAF_PREFIX_MASK: u8 = 0b_01 << 6; pub const BRANCH_WITHOUT_MASK: u8 = 0b_10 << 6; pub const BRANCH_WITH_MASK: u8 = 0b_11 << 6; -} - -/// Utility to tag a state without meta with old_hash internal -/// hashing. -pub fn tag_old_hashes(existing: &[u8]) -> Option> { - use trie_db::NodeCodec; - let mut meta = TrieMeta::default(); - // allows restarting a migration. - if existing.len() > 0 && existing[0] == trie_constants::OLD_HASHING { - return None; // allow restarting a migration. - } - let _ = as TrieLayout>::Codec::decode_plan(existing, &mut meta) - .expect("Invalid db state entry found: {:?}, entry.0.as_slice()"); - match meta.range { - Some(range) => { - if range.end - range.start >= trie_constants::INNER_HASH_TRESHOLD { - let mut res = Vec::with_capacity(existing.len() + 1); - res.push(trie_constants::OLD_HASHING); - res.extend_from_slice(existing); - Some(res) - } else { - None - } - }, - None => None, - } + pub const EMPTY_TRIE: u8 = FIRST_PREFIX | (0b_00 << 4); + pub const ALT_HASHING_LEAF_PREFIX_MASK: u8 = FIRST_PREFIX | (0b_01 << 4); + pub const ALT_HASHING_BRANCH_WITHOUT_MASK: u8 = FIRST_PREFIX | (0b_10 << 4); + pub const ALT_HASHING_BRANCH_WITH_MASK: u8 = FIRST_PREFIX | (0b_11 << 4); } #[cfg(test)] diff --git a/primitives/trie/src/node_codec.rs b/primitives/trie/src/node_codec.rs index 0dc647f17f047..af4adef01bdf5 100644 --- a/primitives/trie/src/node_codec.rs +++ b/primitives/trie/src/node_codec.rs @@ -90,23 +90,18 @@ pub struct NodeCodec(PhantomData); impl NodeCodec { fn decode_plan_inner_hashed( data: &[u8], - meta: Option<&mut M>, + meta: Option<&mut M>, // TODO when remove no meta, remove option ) -> Result { - let contains_hash = meta.as_ref() - .map(|m| m.contains_hash_of_value()).unwrap_or_default(); - if data.len() < 1 { - return Err(Error::BadFormat); - } - let offset = if let Some(meta) = meta { - meta.read_state_meta(data).map_err(|_| Error::BadFormat)? - } else { - 0 - }; let mut input = ByteSliceInput::new(data); let _ = input.take(offset)?; + let contains_hash = meta.as_ref() + .map(|m| m.contains_hash_of_value()).unwrap_or_default(); + let header = NodeHeader::decode(&mut input)?; + let alt_hashing = header.alt_hashing(); match NodeHeader::decode(&mut input)? { NodeHeader::Null => Ok(NodePlan::Empty), + NodeHeader::AltHashBranch(has_value, nibble_count) NodeHeader::Branch(has_value, nibble_count) => { let padding = nibble_count % nibble_ops::NIBBLE_PER_BYTE != 0; // check that the padding is valid (if any) @@ -120,10 +115,10 @@ impl NodeCodec { let bitmap_range = input.take(BITMAP_LENGTH)?; let bitmap = Bitmap::decode(&data[bitmap_range])?; let value = if has_value { - let count = >::decode(&mut input)?.0 as usize; - if contains_hash { + if alt_hashing && contains_hash { ValuePlan::HashedValue(input.take(H::LENGTH)?, count) } else { + let count = >::decode(&mut input)?.0 as usize; ValuePlan::Value(input.take(count)?) } } else { @@ -149,8 +144,9 @@ impl NodeCodec { value, children, }) - } - NodeHeader::Leaf(nibble_count) => { + }, + NodeHeader::AltHashLeaf(nibble_count) + | NodeHeader::Leaf(nibble_count) => { let padding = nibble_count % nibble_ops::NIBBLE_PER_BYTE != 0; // check that the padding is valid (if any) if padding && nibble_ops::pad_left(data[input.offset]) != 0 { @@ -160,10 +156,10 @@ impl NodeCodec { (nibble_count + (nibble_ops::NIBBLE_PER_BYTE - 1)) / nibble_ops::NIBBLE_PER_BYTE, )?; let partial_padding = nibble_ops::number_padding(nibble_count); - let count = >::decode(&mut input)?.0 as usize; - let value = if contains_hash { - ValuePlan::HashedValue(input.take(H::LENGTH)?, count) + if alt_hashing && contains_hash { + ValuePlan::HashedValue(input.take(H::LENGTH)?, 0) } else { + let count = >::decode(&mut input)?.0 as usize; ValuePlan::Value(input.take(count)?) }; @@ -201,9 +197,7 @@ impl NodeCodecT for NodeCodec { } fn empty_node(meta: &mut M) -> Vec { - let mut output = meta.write_state_meta(); - output.extend_from_slice(&[trie_constants::EMPTY_TRIE]); - output + empty_node_no_meta().to_vec() } fn empty_node_no_meta() -> &'static [u8] { @@ -211,8 +205,11 @@ impl NodeCodecT for NodeCodec { } fn leaf_node(partial: Partial, value: Value, meta: &mut M) -> Vec { - let mut output = meta.write_state_meta(); - output.append(&mut partial_encode(partial, NodeKind::Leaf)); + let mut output = if meta.do_value_hash() { + partial_encode(partial, NodeKind::AltHashLeaf) + } else { + partial_encode(partial, NodeKind::Leaf) + }; match value { Value::Value(value) => { Compact(value.len() as u32).encode_to(&mut output); @@ -221,9 +218,8 @@ impl NodeCodecT for NodeCodec { let end = output.len(); meta.encoded_value_callback(ValuePlan::Value(start..end)); }, - Value::HashedValue(hash, size) => { + Value::HashedValue(hash, _size) => { debug_assert!(hash.len() == H::LENGTH); - Compact(size as u32).encode_to(&mut output); let start = output.len(); output.extend_from_slice(hash); let end = output.len(); @@ -258,20 +254,21 @@ impl NodeCodecT for NodeCodec { maybe_value: Value, meta: &mut M, ) -> Vec { - let mut output = meta.write_state_meta(); - output.append(&mut if let Value::NoValue = &maybe_value { - partial_from_iterator_encode( - partial, - number_nibble, - NodeKind::BranchNoValue, - ) - } else { - partial_from_iterator_encode( - partial, - number_nibble, - NodeKind::BranchWithValue, - ) - }); + let mut output = match (&maybe_value, meta.do_value_hash()) { + (&Value::NoValue, false) => { + partial_from_iterator_encode(partial, number_nibble, NodeKind::BranchWithValue) + }, + (_, false) => { + partial_from_iterator_encode(partial, number_nibble, NodeKind::BranchNoValue) + }, + (&Value::NoValue, true) => { + partial_from_iterator_encode(partial, number_nibble, NodeKind::AltHashBranchWithValue) + }, + (_, true) => { + partial_from_iterator_encode(partial, number_nibble, NodeKind::AltHashBranchNoValue) + }, + }; + let bitmap_index = output.len(); let mut bitmap: [u8; BITMAP_LENGTH] = [0; BITMAP_LENGTH]; (0..BITMAP_LENGTH).for_each(|_|output.push(0)); @@ -283,9 +280,8 @@ impl NodeCodecT for NodeCodec { let end = output.len(); meta.encoded_value_callback(ValuePlan::Value(start..end)); }, - Value::HashedValue(hash, size) => { + Value::HashedValue(hash, _size) => { debug_assert!(hash.len() == H::LENGTH); - Compact(size as u32).encode_to(&mut output); let start = output.len(); output.extend_from_slice(hash); let end = output.len(); @@ -326,6 +322,9 @@ fn partial_from_iterator_encode>( NodeKind::Leaf => NodeHeader::Leaf(nibble_count).encode_to(&mut output), NodeKind::BranchWithValue => NodeHeader::Branch(true, nibble_count).encode_to(&mut output), NodeKind::BranchNoValue => NodeHeader::Branch(false, nibble_count).encode_to(&mut output), + NodeKind::AltHashLeaf => NodeHeader::AltHashLeaf(nibble_count).encode_to(&mut output), + NodeKind::AltHashBranchWithValue => NodeHeader::AltHashBranch(true, nibble_count).encode_to(&mut output), + NodeKind::AltHashBranchNoValue => NodeHeader::AltHashBranch(false, nibble_count).encode_to(&mut output), }; output.extend(partial); output @@ -344,6 +343,9 @@ fn partial_encode(partial: Partial, node_kind: NodeKind) -> Vec { NodeKind::Leaf => NodeHeader::Leaf(nibble_count).encode_to(&mut output), NodeKind::BranchWithValue => NodeHeader::Branch(true, nibble_count).encode_to(&mut output), NodeKind::BranchNoValue => NodeHeader::Branch(false, nibble_count).encode_to(&mut output), + NodeKind::AltHashLeaf => NodeHeader::AltHashLeaf(nibble_count).encode_to(&mut output), + NodeKind::AltHashBranchWithValue => NodeHeader::AltHashBranch(true, nibble_count).encode_to(&mut output), + NodeKind::AltHashBranchNoValue => NodeHeader::AltHashBranch(false, nibble_count).encode_to(&mut output), }; if number_nibble_encoded > 0 { output.push(nibble_ops::pad_right((partial.0).1)); diff --git a/primitives/trie/src/node_header.rs b/primitives/trie/src/node_header.rs index 0fdf6fefbd0bc..2e291ebd89203 100644 --- a/primitives/trie/src/node_header.rs +++ b/primitives/trie/src/node_header.rs @@ -28,6 +28,8 @@ pub(crate) enum NodeHeader { Null, Branch(bool, usize), Leaf(usize), + AltHashBranch(bool, usize), + AltHashLeaf(usize), } /// NodeHeader without content @@ -35,6 +37,9 @@ pub(crate) enum NodeKind { Leaf, BranchNoValue, BranchWithValue, + AltHashLeaf, + AltHashBranchNoValue, + AltHashBranchWithValue, } impl Encode for NodeHeader { @@ -47,10 +52,28 @@ impl Encode for NodeHeader { encode_size_and_prefix(*nibble_count, trie_constants::BRANCH_WITHOUT_MASK, output), NodeHeader::Leaf(nibble_count) => encode_size_and_prefix(*nibble_count, trie_constants::LEAF_PREFIX_MASK, output), + NodeHeader::AltHashBranch(true, nibble_count) => + encode_size_and_prefix_alt(*nibble_count, trie_constants::ALT_HASIHNG_BRANCH_WITH_MASK, output), + NodeHeader::AltHashBranch(false, nibble_count) => + encode_size_and_prefix_alt(*nibble_count, trie_constants::ALT_HASHING_BRANCH_WITHOUT_MASK, output), + NodeHeader::AltHashLeaf(nibble_count) => + encode_size_and_prefix_alt(*nibble_count, trie_constants::ALT_HASHING_LEAF_PREFIX_MASK, output), } } } +impl NodeHeader { + /// Is this header using alternate hashing scheme. + pub(crate) alt_hashing() -> bool { + match self { + NodeHeader::Null + | NodeHeader::Leaf(..) + | NodeHeader::Branch(..) => false, + NodeHeader::AltHashBranch(..) + | NodeHeader::AltHashLeaf(..) => true, + } + } +} impl codec::EncodeLike for NodeHeader {} impl Decode for NodeHeader { @@ -60,11 +83,16 @@ impl Decode for NodeHeader { return Ok(NodeHeader::Null); } match i & (0b11 << 6) { - trie_constants::LEAF_PREFIX_MASK => Ok(NodeHeader::Leaf(decode_size(i, input)?)), - trie_constants::BRANCH_WITHOUT_MASK => Ok(NodeHeader::Branch(false, decode_size(i, input)?)), - trie_constants::BRANCH_WITH_MASK => Ok(NodeHeader::Branch(true, decode_size(i, input)?)), - // do not allow any special encoding - _ => Err("Unallowed encoding".into()), + trie_constants::LEAF_PREFIX_MASK => Ok(NodeHeader::Leaf(decode_size(i, input, 2)?)), + trie_constants::BRANCH_WITH_MASK => Ok(NodeHeader::Branch(true, decode_size(i, input, 2)?)), + trie_constants::BRANCH_WITHOUT_MASK => Ok(NodeHeader::Branch(false, decode_size(i, input, 2)?)), + trie_constants::EMPTY_TRIE => match i & (0b1111 << 4) { + trie_constants::ALT_HASHING_LEAF_PREFIX_MASK => Ok(NodeHeader::AltHashLeaf(decode_size(i, input, 4)?)), + trie_constants::ALT_HASHING_BRANCH_WITH_MASK => Ok(NodeHeader::AltHashBranch(true, decode_size(i, input, 4)?)), + trie_constants::ALT_HASHING_BRANCH_WITHOUT_MASK => Ok(NodeHeader::AltHashBranch(false, decode_size(i, input, 4)?)), + // do not allow any special encoding + _ => Err("Unallowed encoding".into()), + }, } } } @@ -72,14 +100,15 @@ impl Decode for NodeHeader { /// Returns an iterator over encoded bytes for node header and size. /// Size encoding allows unlimited, length inefficient, representation, but /// is bounded to 16 bit maximum value to avoid possible DOS. -pub(crate) fn size_and_prefix_iterator(size: usize, prefix: u8) -> impl Iterator { +pub(crate) fn size_and_prefix_iterator(size: usize, prefix: u8, prefix_mask: usize) -> impl Iterator { let size = sp_std::cmp::min(trie_constants::NIBBLE_SIZE_BOUND, size); - let l1 = sp_std::cmp::min(62, size); + let max_value = 255 >> prefix_mask; + let l1 = sp_std::cmp::min(max_value - 1, size); let (first_byte, mut rem) = if size == l1 { (once(prefix + l1 as u8), 0) } else { - (once(prefix + 63), size - l1) + (once(prefix + max_value), size - l1) }; let next_bytes = move || { if rem > 0 { @@ -98,17 +127,25 @@ pub(crate) fn size_and_prefix_iterator(size: usize, prefix: u8) -> impl Iterator first_byte.chain(sp_std::iter::from_fn(next_bytes)) } -/// Encodes size and prefix to a stream output. +/// Encodes size and prefix to a stream output (prefix on 2 first bit only). fn encode_size_and_prefix(size: usize, prefix: u8, out: &mut W) { - for b in size_and_prefix_iterator(size, prefix) { + for b in size_and_prefix_iterator(size, prefix, 2) { + out.push_byte(b) + } +} + +/// Encodes size and prefix to a stream output with prefix (prefix on 4 first bit only). +fn encode_size_and_prefix_alt(size: usize, prefix: u8, out: &mut W) { + for b in size_and_prefix_iterator(size, prefix, 4) { out.push_byte(b) } } /// Decode size only from stream input and header byte. -fn decode_size(first: u8, input: &mut impl Input) -> Result { - let mut result = (first & 255u8 >> 2) as usize; - if result < 63 { +fn decode_size(first: u8, input: &mut impl Input, prefix_mask: usize) -> Result { + let max_value = 255u8 >> prefix_mask; + let mut result = (first & max_value) as usize; + if result < max_value { return Ok(result); } result -= 1; diff --git a/primitives/trie/src/storage_proof.rs b/primitives/trie/src/storage_proof.rs index 0f00a335ae2f1..dcb9cf08e74b7 100644 --- a/primitives/trie/src/storage_proof.rs +++ b/primitives/trie/src/storage_proof.rs @@ -113,6 +113,7 @@ impl From for crate::MemoryDB { let mut is_hashed_value = false; let mut accum = Vec::new(); for item in proof.trie_nodes.iter() { + // TODO remove this look up // Note using `default()` as global meta helps looking fro root node. let layout_meta = Default::default(); let (encoded_node, mut meta) = < From c807f44d40424ffe797f1f41b7f1a2e64b124a58 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 8 Jun 2021 16:21:55 +0200 Subject: [PATCH 036/127] before removing unused branch no value alt hashing. more work todo rename all flag var to alt_hash, and remove extrinsic replace by storage query at every storage_root call. --- bin/node/bench/src/generator.rs | 6 +- client/db/src/bench.rs | 10 ++- client/db/src/lib.rs | 7 +- client/db/src/storage_cache.rs | 14 ++-- client/db/src/upgrade.rs | 61 +------------- client/light/src/backend.rs | 7 +- primitives/state-machine/src/backend.rs | 12 +-- primitives/state-machine/src/basic.rs | 10 +-- primitives/state-machine/src/ext.rs | 2 +- primitives/state-machine/src/lib.rs | 28 ++----- .../state-machine/src/proving_backend.rs | 7 +- primitives/state-machine/src/testing.rs | 2 +- primitives/state-machine/src/trie_backend.rs | 34 ++++---- primitives/storage/src/lib.rs | 4 +- primitives/trie/src/lib.rs | 80 ++++++++----------- primitives/trie/src/node_codec.rs | 26 +++--- primitives/trie/src/node_header.rs | 14 ++-- primitives/trie/src/storage_proof.rs | 17 +--- primitives/trie/src/trie_stream.rs | 54 ++++++++----- 19 files changed, 159 insertions(+), 236 deletions(-) diff --git a/bin/node/bench/src/generator.rs b/bin/node/bench/src/generator.rs index a27d6e529f277..913c1ff8779f0 100644 --- a/bin/node/bench/src/generator.rs +++ b/bin/node/bench/src/generator.rs @@ -46,9 +46,7 @@ pub fn generate_trie( let mut trie_db = if flag_hashed_value { let layout = sp_trie::Layout::with_inner_hashing(); - let mut t = TrieDBMut::::new_with_layout(&mut trie, &mut root, layout); - t.force_layout_meta().expect("force layout meta failed"); - t + TrieDBMut::::new_with_layout(&mut trie, &mut root, layout) } else { TrieDBMut::new(&mut trie, &mut root) }; @@ -58,7 +56,7 @@ pub fn generate_trie( trie_db.commit(); } - ( trie.db, overlay ) + (trie.db, overlay) }; let mut transaction = db.transaction(); diff --git a/client/db/src/bench.rs b/client/db/src/bench.rs index ec53cecf1f1a4..0f111eed99189 100644 --- a/client/db/src/bench.rs +++ b/client/db/src/bench.rs @@ -401,17 +401,21 @@ impl StateBackend> for BenchmarkingState { fn storage_root<'a>( &self, delta: impl Iterator)>, - flag_hash_value: bool, + alt_hashing: bool, ) -> (B::Hash, Self::Transaction) where B::Hash: Ord { - self.state.borrow().as_ref().map_or(Default::default(), |s| s.storage_root(delta, flag_hash_value)) + self.state.borrow().as_ref().map_or(Default::default(), |s| s.storage_root(delta, alt_hashing)) } fn child_storage_root<'a>( &self, child_info: &ChildInfo, delta: impl Iterator)>, + alt_hashing: bool, ) -> (B::Hash, bool, Self::Transaction) where B::Hash: Ord { - self.state.borrow().as_ref().map_or(Default::default(), |s| s.child_storage_root(child_info, delta)) + self.state.borrow().as_ref().map_or( + Default::default(), + |s| s.child_storage_root(child_info, delta, alt_hashing), + ) } fn pairs(&self) -> Vec<(Vec, Vec)> { diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index f808df9c2675e..41a6853aebc71 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -227,17 +227,18 @@ impl StateBackend> for RefTrackingState { fn storage_root<'a>( &self, delta: impl Iterator)>, - flag_hash_value: bool, + alt_hashing: bool, ) -> (B::Hash, Self::Transaction) where B::Hash: Ord { - self.state.storage_root(delta, flag_hash_value) + self.state.storage_root(delta, alt_hashing) } fn child_storage_root<'a>( &self, child_info: &ChildInfo, delta: impl Iterator)>, + alt_hashing: bool, ) -> (B::Hash, bool, Self::Transaction) where B::Hash: Ord { - self.state.child_storage_root(child_info, delta) + self.state.child_storage_root(child_info, delta, alt_hashing) } fn pairs(&self) -> Vec<(Vec, Vec)> { diff --git a/client/db/src/storage_cache.rs b/client/db/src/storage_cache.rs index acf89c646a858..946f576142745 100644 --- a/client/db/src/storage_cache.rs +++ b/client/db/src/storage_cache.rs @@ -645,17 +645,18 @@ impl>, B: BlockT> StateBackend> for Cachin fn storage_root<'a>( &self, delta: impl Iterator)>, - flag_hash_value: bool, + alt_hashing: bool, ) -> (B::Hash, Self::Transaction) where B::Hash: Ord { - self.state.storage_root(delta, flag_hash_value) + self.state.storage_root(delta, alt_hashing) } fn child_storage_root<'a>( &self, child_info: &ChildInfo, delta: impl Iterator)>, + alt_hashing: bool, ) -> (B::Hash, bool, Self::Transaction) where B::Hash: Ord { - self.state.child_storage_root(child_info, delta) + self.state.child_storage_root(child_info, delta, alt_hashing) } fn pairs(&self) -> Vec<(Vec, Vec)> { @@ -832,17 +833,18 @@ impl>, B: BlockT> StateBackend> for Syncin fn storage_root<'a>( &self, delta: impl Iterator)>, - flag_hash_value: bool, + alt_hashing: bool, ) -> (B::Hash, Self::Transaction) where B::Hash: Ord { - self.caching_state().storage_root(delta, flag_hash_value) + self.caching_state().storage_root(delta, alt_hashing) } fn child_storage_root<'a>( &self, child_info: &ChildInfo, delta: impl Iterator)>, + alt_hashing: bool, ) -> (B::Hash, bool, Self::Transaction) where B::Hash: Ord { - self.caching_state().child_storage_root(child_info, delta) + self.caching_state().child_storage_root(child_info, delta, alt_hashing) } fn pairs(&self) -> Vec<(Vec, Vec)> { diff --git a/client/db/src/upgrade.rs b/client/db/src/upgrade.rs index f0129251b691e..ea91b8253e1d8 100644 --- a/client/db/src/upgrade.rs +++ b/client/db/src/upgrade.rs @@ -26,13 +26,12 @@ use sp_runtime::traits::Block as BlockT; use crate::{columns, utils::DatabaseType}; use kvdb_rocksdb::{Database, DatabaseConfig}; use codec::{Decode, Encode}; -use log::info; /// Version file name. const VERSION_FILE_NAME: &'static str = "db_version"; /// Current db version. -const CURRENT_VERSION: u32 = 4; +const CURRENT_VERSION: u32 = 3; /// Number of columns in v1. const V1_NUM_COLUMNS: u32 = 11; @@ -50,7 +49,6 @@ pub fn upgrade_db(db_path: &Path, db_type: DatabaseType) -> sp_bl migrate_2_to_3::(db_path, db_type)? }, 2 => migrate_2_to_3::(db_path, db_type)?, - 3 => migrate_3_to_4::(db_path, db_type)?, CURRENT_VERSION => (), _ => Err(sp_blockchain::Error::Backend(format!("Future database version: {}", db_version)))?, } @@ -100,63 +98,6 @@ fn migrate_2_to_3(db_path: &Path, _db_type: DatabaseType) -> sp_b Ok(()) } -/// Migration from version3 to version4: -/// - Trie state meta for state that could be hashed internaly. -fn migrate_3_to_4(db_path: &Path, _db_type: DatabaseType) -> sp_blockchain::Result<()> { - - info!("Starting trie node migration."); - let start_time = std::time::Instant::now(); - let db_path = db_path.to_str() - .ok_or_else(|| sp_blockchain::Error::Backend("Invalid database path".into()))?; - let db_cfg = DatabaseConfig::with_columns(V2_NUM_COLUMNS); - let db = Database::open(&db_cfg, db_path).map_err(db_err)?; - - let mut nb_node_prefixed = 0; - let mut nb_node_seen = 0; - let batch_size = 250_000; - loop { - let mut full_batch = false; - let mut size = 0; - let mut last = Vec::new(); - let mut transaction = db.transaction(); - // Get all the keys we need to update. - // Note that every batch will restart full iter, - // if this prove to slow for archive node, this could be - // switch to a `iter_from` function but would require - // to upstream change to our rocksdb crate. - for entry in db.iter(columns::STATE) { - if &entry.1[..] > last.as_slice() { - nb_node_seen += 1; - if let Some(new_val) = sp_trie::tag_old_hashes::>(&entry.1) { - transaction.put_vec(columns::STATE, &entry.0, new_val); - nb_node_prefixed += 1; - size += 1; - if size == batch_size { - full_batch = true; - last = entry.0.to_vec(); - break; - } - } - } - } - info!( - "Committing batch, currently processed: {} of {} read nodes at {:?}, {:?}", - nb_node_prefixed, - nb_node_seen, - last, - start_time.elapsed().as_millis(), - ); - db.write(transaction).map_err(db_err)?; - if !full_batch { - break; - } - } - info!("Trie node migration finished in {:?} ms.", start_time.elapsed().as_millis()); - info!("{:?} nodes prefixed for {:?} node.", nb_node_prefixed, nb_node_seen); - Ok(()) -} - - /// Reads current database version from the file at given path. /// If the file does not exist returns 0. fn current_version(path: &Path) -> sp_blockchain::Result { diff --git a/client/light/src/backend.rs b/client/light/src/backend.rs index b09b4d748b8ec..2d65c7b3347ff 100644 --- a/client/light/src/backend.rs +++ b/client/light/src/backend.rs @@ -490,11 +490,11 @@ impl StateBackend for GenesisOrUnavailableState fn storage_root<'a>( &self, delta: impl Iterator)>, - flag_hash_value: bool, + alt_hashing: bool, ) -> (H::Out, Self::Transaction) where H::Out: Ord { match *self { GenesisOrUnavailableState::Genesis(ref state) => - state.storage_root(delta, flag_hash_value), + state.storage_root(delta, alt_hashing), GenesisOrUnavailableState::Unavailable => Default::default(), } } @@ -503,10 +503,11 @@ impl StateBackend for GenesisOrUnavailableState &self, child_info: &ChildInfo, delta: impl Iterator)>, + alt_hashing: bool, ) -> (H::Out, bool, Self::Transaction) where H::Out: Ord { match *self { GenesisOrUnavailableState::Genesis(ref state) => { - let (root, is_equal, _) = state.child_storage_root(child_info, delta); + let (root, is_equal, _) = state.child_storage_root(child_info, delta, alt_hashing); (root, is_equal, Default::default()) }, GenesisOrUnavailableState::Unavailable => diff --git a/primitives/state-machine/src/backend.rs b/primitives/state-machine/src/backend.rs index 0f05d08858bb1..aaa6bbf16f8d7 100644 --- a/primitives/state-machine/src/backend.rs +++ b/primitives/state-machine/src/backend.rs @@ -123,12 +123,13 @@ pub trait Backend: sp_std::fmt::Debug { /// Calculate the storage root, with given delta over what is already stored in /// the backend, and produce a "transaction" that can be used to commit. - /// A flag `flag_inner_hash_value` can be set, it switches inner trie implementation. + /// `alt_hashing` indicate if trie state should apply alternate hashing + /// scheme (inner value hashed). /// Does not include child storage updates. fn storage_root<'a>( &self, delta: impl Iterator)>, - flag_inner_hash_value: bool, + alt_hashing: bool, ) -> (H::Out, Self::Transaction) where H::Out: Ord; /// Calculate the child storage root, with given delta over what is already stored in @@ -138,6 +139,7 @@ pub trait Backend: sp_std::fmt::Debug { &self, child_info: &ChildInfo, delta: impl Iterator)>, + alt_hashing: bool, ) -> (H::Out, bool, Self::Transaction) where H::Out: Ord; /// Get all key/value pairs into a Vec. @@ -176,14 +178,14 @@ pub trait Backend: sp_std::fmt::Debug { &'a ChildInfo, impl Iterator)>, )>, - flag_inner_hash_value: bool, + alt_hashing: bool, ) -> (H::Out, Self::Transaction) where H::Out: Ord + Encode { let mut txs: Self::Transaction = Default::default(); let mut child_roots: Vec<_> = Default::default(); // child first for (child_info, child_delta) in child_deltas { let (child_root, empty, child_txs) = - self.child_storage_root(&child_info, child_delta); + self.child_storage_root(&child_info, child_delta, alt_hashing); let prefixed_storage_key = child_info.prefixed_storage_key(); txs.consolidate(child_txs); if empty { @@ -199,7 +201,7 @@ pub trait Backend: sp_std::fmt::Debug { .iter() .map(|(k, v)| (&k[..], v.as_ref().map(|v| &v[..]))) ), - flag_inner_hash_value, + alt_hashing, ); txs.consolidate(parent_txs); (root, txs) diff --git a/primitives/state-machine/src/basic.rs b/primitives/state-machine/src/basic.rs index c9794486b6acc..2cc7314392bdf 100644 --- a/primitives/state-machine/src/basic.rs +++ b/primitives/state-machine/src/basic.rs @@ -74,7 +74,7 @@ impl BasicExternalities { inner: Storage { top: std::mem::take(&mut storage.top), children_default: std::mem::take(&mut storage.children_default), - flag_hashed_value: storage.flag_hashed_value, + alt_hashing: storage.alt_hashing, }, extensions: Default::default(), }; @@ -129,7 +129,7 @@ impl From> for BasicExternalities { inner: Storage { top: hashmap, children_default: Default::default(), - flag_hashed_value: true, + alt_hashing: true, }, extensions: Default::default(), } @@ -283,7 +283,7 @@ impl Externalities for BasicExternalities { } } - let layout = if self.inner.flag_hashed_value { + let layout = if self.inner.alt_hashing { Layout::::with_inner_hashing() } else { Layout::::default() @@ -298,7 +298,7 @@ impl Externalities for BasicExternalities { if let Some(child) = self.inner.children_default.get(child_info.storage_key()) { let delta = child.data.iter().map(|(k, v)| (k.as_ref(), Some(v.as_ref()))); crate::in_memory_backend::new_in_mem::() - .child_storage_root(&child.child_info, delta).0 + .child_storage_root(&child.child_info, delta, self.inner.alt_hashing).0 } else { empty_child_trie_root::>() }.encode() @@ -341,7 +341,7 @@ impl Externalities for BasicExternalities { } fn flag_hash_value(&mut self) { - self.inner.flag_hashed_value = true; + self.inner.alt_hashing = true; } } diff --git a/primitives/state-machine/src/ext.rs b/primitives/state-machine/src/ext.rs index 0b3e155c11594..e187c128345b8 100644 --- a/primitives/state-machine/src/ext.rs +++ b/primitives/state-machine/src/ext.rs @@ -547,7 +547,7 @@ where } else { let root = if let Some((changes, info)) = self.overlay.child_changes(storage_key) { let delta = changes.map(|(k, v)| (k.as_ref(), v.value().map(AsRef::as_ref))); - Some(self.backend.child_storage_root(info, delta)) + Some(self.backend.child_storage_root(info, delta, self.overlay.flag_hash_value())) } else { None }; diff --git a/primitives/state-machine/src/lib.rs b/primitives/state-machine/src/lib.rs index 726d1e51b080b..709357430a40c 100644 --- a/primitives/state-machine/src/lib.rs +++ b/primitives/state-machine/src/lib.rs @@ -1496,12 +1496,13 @@ mod tests { #[test] fn inner_state_hashing_switch_proofs() { + let mut layout = Layout::default(); let (mut mdb, mut root) = trie_backend::tests::test_db(false); { let mut trie = TrieDBMut::from_existing_with_layout( &mut mdb, &mut root, - Layout::default(), + layout.cloen(), ).unwrap(); trie.insert(b"foo", vec![1u8; 1_000].as_slice()) // big inner hash .expect("insert failed"); @@ -1537,31 +1538,14 @@ mod tests { let root1 = root.clone(); - // trigger switch - { - let mut trie = TrieDBMut::from_existing_with_layout( - &mut mdb, - &mut root, - Layout::with_inner_hashing(), - ).unwrap(); - trie.force_layout_meta() - .expect("failed forced layout change"); - } - let root2 = root.clone(); - assert!(root1 != root2); - let remote_proof = check_proof(mdb.clone(), root.clone()); - // nodes are still with old hashing. - assert!(remote_proof.encode().len() > 1_100); - assert!(remote_proof.encoded_size() > 1_100); - assert_eq!(remote_proof.encode().len(), - remote_proof.encoded_size()); - + // do switch + layout = Layout::with_inner_hashing(); // update with same value do not change { let mut trie = TrieDBMut::from_existing_with_layout( &mut mdb, &mut root, - Layout::default(), + layout.clone(), ).unwrap(); trie.insert(b"foo222", vec![5u8; 100].as_slice()) // inner hash .expect("insert failed"); @@ -1576,7 +1560,7 @@ mod tests { let mut trie = TrieDBMut::from_existing_with_layout( &mut mdb, &mut root, - Layout::default(), + layout.clone(), ).unwrap(); trie.insert(b"foo222", vec![4u8].as_slice()) // inner hash .expect("insert failed"); diff --git a/primitives/state-machine/src/proving_backend.rs b/primitives/state-machine/src/proving_backend.rs index 5e1a0de9ffc87..1ab9d09125648 100644 --- a/primitives/state-machine/src/proving_backend.rs +++ b/primitives/state-machine/src/proving_backend.rs @@ -353,17 +353,18 @@ impl<'a, S, H> Backend for ProvingBackend<'a, S, H> fn storage_root<'b>( &self, delta: impl Iterator)>, - flag_inner_hash_value: bool, + alt_hashing: bool, ) -> (H::Out, Self::Transaction) where H::Out: Ord { - self.0.storage_root(delta, flag_inner_hash_value) + self.0.storage_root(delta, alt_hashing) } fn child_storage_root<'b>( &self, child_info: &ChildInfo, delta: impl Iterator)>, + alt_hashing: bool, ) -> (H::Out, bool, Self::Transaction) where H::Out: Ord { - self.0.child_storage_root(child_info, delta) + self.0.child_storage_root(child_info, delta, alt_hashing) } fn register_overlay_stats(&self, _stats: &crate::stats::StateMachineStats) { } diff --git a/primitives/state-machine/src/testing.rs b/primitives/state-machine/src/testing.rs index b79d6866edcca..19006ccb52ac4 100644 --- a/primitives/state-machine/src/testing.rs +++ b/primitives/state-machine/src/testing.rs @@ -239,7 +239,7 @@ impl Default for TestExternalities fn default() -> Self { // default to inner hashed. let mut storage = Storage::default(); - storage.flag_hashed_value = true; + storage.alt_hashing = true; Self::new(storage) } } diff --git a/primitives/state-machine/src/trie_backend.rs b/primitives/state-machine/src/trie_backend.rs index afee980aedc6a..fbc1f12fd2ae1 100644 --- a/primitives/state-machine/src/trie_backend.rs +++ b/primitives/state-machine/src/trie_backend.rs @@ -171,7 +171,7 @@ impl, H: Hasher> Backend for TrieBackend where fn storage_root<'a>( &self, delta: impl Iterator)>, - flag_inner_hash_value: bool, + use_inner_hash_value: bool, ) -> (H::Out, Self::Transaction) where H::Out: Ord { let mut write_overlay = S::Overlay::default(); let mut root = *self.essence.root(); @@ -182,16 +182,12 @@ impl, H: Hasher> Backend for TrieBackend where &mut write_overlay, ); let res = || { - if flag_inner_hash_value { - let layout = sp_trie::Layout::with_inner_hashing(); - let mut t = sp_trie::trie_types::TrieDBMut::::from_existing_with_layout( - &mut eph, - &mut root, - layout, - )?; - t.force_layout_meta()?; - } - delta_trie_root::, _, _, _, _, _>(&mut eph, root, delta) + let layout = if use_inner_hash_value { + sp_trie::Layout::with_inner_hashing() + } else { + sp_trie::Layout::default() + }; + delta_trie_root::, _, _, _, _, _>(&mut eph, root, delta, layout) }; match res() { @@ -207,10 +203,16 @@ impl, H: Hasher> Backend for TrieBackend where &self, child_info: &ChildInfo, delta: impl Iterator)>, + use_inner_hash_value: bool, ) -> (H::Out, bool, Self::Transaction) where H::Out: Ord { let default_root = match child_info.child_type() { ChildType::ParentKeyId => empty_child_trie_root::>() }; + let layout = if use_inner_hash_value { + sp_trie::Layout::with_inner_hashing() + } else { + sp_trie::Layout::default() + }; let mut write_overlay = S::Overlay::default(); let prefixed_storage_key = child_info.prefixed_storage_key(); @@ -234,6 +236,7 @@ impl, H: Hasher> Backend for TrieBackend where &mut eph, root, delta, + layout, ) { Ok(ret) => root = ret, Err(e) => warn!(target: "trie", "Failed to write to trie: {}", e), @@ -291,14 +294,7 @@ pub mod tests { root.encode_to(&mut sub_root); let mut trie = if hashed_value { let layout = Layout::with_inner_hashing(); - let mut t = TrieDBMut::new_with_layout( - &mut mdb, - &mut root, - layout, - ); - t.force_layout_meta() - .expect("failed forced layout change"); - t + TrieDBMut::new_with_layout(&mut mdb, &mut root, layout) } else { TrieDBMut::new(&mut mdb, &mut root) }; diff --git a/primitives/storage/src/lib.rs b/primitives/storage/src/lib.rs index 5c8028e40c2a8..32645adecb535 100644 --- a/primitives/storage/src/lib.rs +++ b/primitives/storage/src/lib.rs @@ -140,8 +140,8 @@ pub struct Storage { /// trie kind, so this is exclusively for the `ChildType::ParentKeyId` /// tries. pub children_default: std::collections::HashMap, StorageChild>, - /// Flag state for using hash of values internally. - pub flag_hashed_value: bool, + /// `true` when state should hash values internally. + pub alt_hashing: bool, } /// Storage change set diff --git a/primitives/trie/src/lib.rs b/primitives/trie/src/lib.rs index 57aa05adeed04..fc7d16b71328f 100644 --- a/primitives/trie/src/lib.rs +++ b/primitives/trie/src/lib.rs @@ -88,25 +88,22 @@ impl Meta for TrieMeta { /// When true apply inner hashing of value. type GlobalMeta = bool; - // TODO remove upstraem /// When true apply inner hashing of value. type StateMeta = bool; - // TODO remove upstream - fn set_state_meta(&mut self, _state_meta: Self::StateMeta) { - /*if !self.do_value_hash && state_meta { - self.switch_to_value_hash = true; + fn set_state_meta(&mut self, state_meta: Self::StateMeta) { + if !self.do_value_hash && state_meta { self.do_value_hash = true; - }*/ + } } // TODO remove upstream fn extract_global_meta(&self) -> Self::GlobalMeta { - self.recorded_do_value_hash + self.switch_to_value_hash || self.do_value_hash } fn set_global_meta(&mut self, global_meta: Self::GlobalMeta) { - if !self.do_value_hash && state_meta { + if !self.do_value_hash && global_meta { self.switch_to_value_hash = true; self.do_value_hash = true; } @@ -114,12 +111,11 @@ impl Meta for TrieMeta { // TODO remove upstream? fn has_state_meta(&self) -> bool { - false - //self.do_value_hash + self.do_value_hash && !self.switch_to_value_hash } // TODO consider removal upstream of this method (node type in codec) - fn read_state_meta(&mut self, data: &[u8]) -> Result { + fn read_state_meta(&mut self, _data: &[u8]) -> Result { unreachable!() // TODO read directly from codec. /* let offset = if data[0] == trie_constants::ENCODED_META_ALLOW_HASH { @@ -204,6 +200,7 @@ impl Meta for TrieMeta { self.contain_hash } + // TODO could be rename to get_state_meta fn do_value_hash(&self) -> bool { self.do_value_hash } @@ -255,12 +252,12 @@ impl TrieLayout for Layout where H: Hasher, M: MetaHasher, - M::Meta: Meta, + M::Meta: Meta, { const USE_EXTENSION: bool = false; const ALLOW_EMPTY: bool = true; const USE_META: bool = true; - const READ_ROOT_STATE_META: bool = true; + const READ_ROOT_STATE_META: bool = false; // TODO rem type Hash = H; type Codec = NodeCodec; @@ -299,8 +296,8 @@ impl MetaHasher for StateHasher fn hash(value: &[u8], meta: &Self::Meta) -> H::Out { match &meta { - TrieMeta { range: Some(range), contain_hash: false, do_value_hash, switch_to_value_hash: false, .. } => { - if *do_value_hash && range.end - range.start >= trie_constants::INNER_HASH_TRESHOLD { + TrieMeta { range: Some(range), contain_hash: false, do_value_hash: true, switch_to_value_hash: false, .. } => { + if range.end - range.start >= trie_constants::INNER_HASH_TRESHOLD { let value = inner_hashed_value::(value, Some((range.start, range.end))); H::hash(value.as_slice()) } else { @@ -354,7 +351,6 @@ impl MetaHasher for StateHasher fn extract_value(mut stored: &[u8], global_meta: Self::GlobalMeta) -> (&[u8], Self::Meta) { let input = &mut stored; let mut contain_hash = false; - let mut old_hash = false; if input.get(0) == Some(&trie_constants::DEAD_HEADER_META_HASHED_VALUE) { contain_hash = true; *input = &input[1..]; @@ -365,7 +361,8 @@ impl MetaHasher for StateHasher contain_hash, do_value_hash: false, recorded_do_value_hash: false, - old_hash, + switch_to_value_hash: false, + old_hash: false, }; meta.set_global_meta(global_meta); (stored, meta) @@ -419,7 +416,7 @@ impl TrieConfiguration for Layout where H: Hasher, M: MetaHasher, - M::Meta: Meta, + M::Meta: Meta, { fn trie_root(&self, input: I) -> ::Out where I: IntoIterator, @@ -515,7 +512,8 @@ pub mod trie_types { pub fn delta_trie_root( db: &mut DB, mut root: TrieHash, - delta: I + delta: I, + layout: L, ) -> Result, Box>> where I: IntoIterator, A: Borrow<[u8]>, @@ -524,14 +522,11 @@ pub fn delta_trie_root( DB: hash_db::HashDB>, { { - let mut trie = TrieDBMut::::from_existing(db, &mut root)?; + let mut trie = TrieDBMut::::from_existing_with_layout(db, &mut root, layout)?; let mut delta = delta.into_iter().collect::>(); delta.sort_by(|l, r| l.0.borrow().cmp(r.0.borrow())); - if delta.len() == 0 { - trie.force_layout_meta()?; - } for (key, change) in delta { match change.borrow() { Some(val) => trie.insert(key.borrow(), val.borrow())?, @@ -622,6 +617,7 @@ pub fn child_delta_trie_root( db: &mut DB, root_data: RD, delta: I, + layout: L, ) -> Result<::Out, Box>> where I: IntoIterator, @@ -640,6 +636,7 @@ pub fn child_delta_trie_root( &mut db, root, delta, + layout, ) } @@ -971,9 +968,6 @@ mod tests { let mut memdb = MemoryDBMeta::<_, T::MetaHasher>::default(); let mut root = Default::default(); let mut t = TrieDBMut::::new_with_layout(&mut memdb, &mut root, layout); - if input.len() == 0 { - t.force_layout_meta().unwrap(); - } for (x, y) in input.iter().rev() { t.insert(x, y).unwrap(); } @@ -1005,7 +999,7 @@ mod tests { } fn check_input(input: &Vec<(&[u8], &[u8])>) { - +// TODO remove this iter let layout = Layout::with_inner_hashing(); check_equivalent::(input, layout.clone()); @@ -1122,25 +1116,12 @@ mod tests { db: &'db mut dyn HashDB>, root: &'db mut TrieHash, v: &[(Vec, Vec)], - flag_hash: bool, + layout: T, ) -> TrieDBMut<'db, T> where T: TrieConfiguration, { - let mut t = if flag_hash { - let mut root_meta = Default::default(); - T::set_root_meta(&mut root_meta, flag_hash); - - let mut layout = T::default(); - layout.initialize_from_root_meta(&root_meta); - - let mut t = TrieDBMut::::new_with_layout(db, root, layout); - t.force_layout_meta() - .expect("Could not force layout."); - t - } else { - TrieDBMut::::new(db, root) - }; + let mut t = TrieDBMut::::new_with_layout(db, root, layout); for i in 0..v.len() { let key: &[u8]= &v[i].0; let val: &[u8] = &v[i].1; @@ -1186,7 +1167,8 @@ mod tests { let real = layout.trie_root(x.clone()); let mut memdb = MemoryDB::default(); let mut root = Default::default(); - let mut memtrie = populate_trie::(&mut memdb, &mut root, &x, flag); + + let mut memtrie = populate_trie::(&mut memdb, &mut root, &x, layout.clone()); memtrie.commit(); if *memtrie.root() != real { @@ -1274,6 +1256,12 @@ mod tests { iterator_works_inner(false); } fn iterator_works_inner(flag: bool) { + let layout = if flag { + Layout::with_inner_hashing() + } else { + Layout::default() + }; + let pairs = vec![ (hex!("0103000000000000000464").to_vec(), hex!("0400000000").to_vec()), (hex!("0103000000000000000469").to_vec(), hex!("0401000000").to_vec()), @@ -1281,9 +1269,9 @@ mod tests { let mut mdb = MemoryDB::default(); let mut root = Default::default(); - let _ = populate_trie::(&mut mdb, &mut root, &pairs, flag); + let _ = populate_trie::(&mut mdb, &mut root, &pairs, layout.clone()); - let trie = TrieDB::::new(&mdb, &root).unwrap(); + let trie = TrieDB::::new_with_layout(&mdb, &root, layout).unwrap(); let iter = trie.iter().unwrap(); let mut iter_pairs = Vec::new(); @@ -1315,11 +1303,13 @@ mod tests { &mut proof_db.clone(), storage_root, valid_delta, + Default::default(), ).unwrap(); let second_storage_root = delta_trie_root::( &mut proof_db.clone(), storage_root, invalid_delta, + Default::default(), ).unwrap(); assert_eq!(first_storage_root, second_storage_root); diff --git a/primitives/trie/src/node_codec.rs b/primitives/trie/src/node_codec.rs index af4adef01bdf5..c239a5c07c97a 100644 --- a/primitives/trie/src/node_codec.rs +++ b/primitives/trie/src/node_codec.rs @@ -88,21 +88,23 @@ impl<'a> Input for ByteSliceInput<'a> { pub struct NodeCodec(PhantomData); impl NodeCodec { - fn decode_plan_inner_hashed( + fn decode_plan_inner_hashed>( data: &[u8], - meta: Option<&mut M>, // TODO when remove no meta, remove option + mut meta: Option<&mut M>, // TODO when remove no meta, remove option ) -> Result { let mut input = ByteSliceInput::new(data); - let _ = input.take(offset)?; let contains_hash = meta.as_ref() .map(|m| m.contains_hash_of_value()).unwrap_or_default(); let header = NodeHeader::decode(&mut input)?; let alt_hashing = header.alt_hashing(); - match NodeHeader::decode(&mut input)? { + meta.as_mut() + .map(|m| m.set_state_meta(alt_hashing)); + + match header { NodeHeader::Null => Ok(NodePlan::Empty), NodeHeader::AltHashBranch(has_value, nibble_count) - NodeHeader::Branch(has_value, nibble_count) => { + | NodeHeader::Branch(has_value, nibble_count) => { let padding = nibble_count % nibble_ops::NIBBLE_PER_BYTE != 0; // check that the padding is valid (if any) if padding && nibble_ops::pad_left(data[input.offset]) != 0 { @@ -116,7 +118,7 @@ impl NodeCodec { let bitmap = Bitmap::decode(&data[bitmap_range])?; let value = if has_value { if alt_hashing && contains_hash { - ValuePlan::HashedValue(input.take(H::LENGTH)?, count) + ValuePlan::HashedValue(input.take(H::LENGTH)?, 0) } else { let count = >::decode(&mut input)?.0 as usize; ValuePlan::Value(input.take(count)?) @@ -156,7 +158,7 @@ impl NodeCodec { (nibble_count + (nibble_ops::NIBBLE_PER_BYTE - 1)) / nibble_ops::NIBBLE_PER_BYTE, )?; let partial_padding = nibble_ops::number_padding(nibble_count); - if alt_hashing && contains_hash { + let value = if alt_hashing && contains_hash { ValuePlan::HashedValue(input.take(H::LENGTH)?, 0) } else { let count = >::decode(&mut input)?.0 as usize; @@ -172,7 +174,7 @@ impl NodeCodec { } } -impl NodeCodecT for NodeCodec { +impl> NodeCodecT for NodeCodec { type Error = Error; type HashOut = H::Out; @@ -196,8 +198,8 @@ impl NodeCodecT for NodeCodec { data == >::empty_node_no_meta() } - fn empty_node(meta: &mut M) -> Vec { - empty_node_no_meta().to_vec() + fn empty_node(_meta: &mut M) -> Vec { + sp_std::vec![trie_constants::EMPTY_TRIE] } fn empty_node_no_meta() -> &'static [u8] { @@ -223,7 +225,7 @@ impl NodeCodecT for NodeCodec { let start = output.len(); output.extend_from_slice(hash); let end = output.len(); - meta.encoded_value_callback(ValuePlan::HashedValue(start..end, size)); + meta.encoded_value_callback(ValuePlan::HashedValue(start..end, 0)); }, Value::NoValue => unimplemented!("No support for incomplete nodes"), } @@ -285,7 +287,7 @@ impl NodeCodecT for NodeCodec { let start = output.len(); output.extend_from_slice(hash); let end = output.len(); - meta.encoded_value_callback(ValuePlan::HashedValue(start..end, size)); + meta.encoded_value_callback(ValuePlan::HashedValue(start..end, 0)); }, Value::NoValue => (), } diff --git a/primitives/trie/src/node_header.rs b/primitives/trie/src/node_header.rs index 2e291ebd89203..a1f0090ecb8a2 100644 --- a/primitives/trie/src/node_header.rs +++ b/primitives/trie/src/node_header.rs @@ -53,7 +53,7 @@ impl Encode for NodeHeader { NodeHeader::Leaf(nibble_count) => encode_size_and_prefix(*nibble_count, trie_constants::LEAF_PREFIX_MASK, output), NodeHeader::AltHashBranch(true, nibble_count) => - encode_size_and_prefix_alt(*nibble_count, trie_constants::ALT_HASIHNG_BRANCH_WITH_MASK, output), + encode_size_and_prefix_alt(*nibble_count, trie_constants::ALT_HASHING_BRANCH_WITH_MASK, output), NodeHeader::AltHashBranch(false, nibble_count) => encode_size_and_prefix_alt(*nibble_count, trie_constants::ALT_HASHING_BRANCH_WITHOUT_MASK, output), NodeHeader::AltHashLeaf(nibble_count) => @@ -64,7 +64,7 @@ impl Encode for NodeHeader { impl NodeHeader { /// Is this header using alternate hashing scheme. - pub(crate) alt_hashing() -> bool { + pub(crate) fn alt_hashing(&self) -> bool { match self { NodeHeader::Null | NodeHeader::Leaf(..) @@ -74,6 +74,7 @@ impl NodeHeader { } } } + impl codec::EncodeLike for NodeHeader {} impl Decode for NodeHeader { @@ -93,6 +94,7 @@ impl Decode for NodeHeader { // do not allow any special encoding _ => Err("Unallowed encoding".into()), }, + _ => unreachable!(), } } } @@ -103,12 +105,12 @@ impl Decode for NodeHeader { pub(crate) fn size_and_prefix_iterator(size: usize, prefix: u8, prefix_mask: usize) -> impl Iterator { let size = sp_std::cmp::min(trie_constants::NIBBLE_SIZE_BOUND, size); - let max_value = 255 >> prefix_mask; - let l1 = sp_std::cmp::min(max_value - 1, size); + let max_value = 255u8 >> prefix_mask; + let l1 = sp_std::cmp::min(max_value as usize - 1, size); let (first_byte, mut rem) = if size == l1 { (once(prefix + l1 as u8), 0) } else { - (once(prefix + max_value), size - l1) + (once(prefix + max_value as u8), size - l1) }; let next_bytes = move || { if rem > 0 { @@ -145,7 +147,7 @@ fn encode_size_and_prefix_alt(size: usize, prefix: u8, out: fn decode_size(first: u8, input: &mut impl Input, prefix_mask: usize) -> Result { let max_value = 255u8 >> prefix_mask; let mut result = (first & max_value) as usize; - if result < max_value { + if result < max_value as usize { return Ok(result); } result -= 1; diff --git a/primitives/trie/src/storage_proof.rs b/primitives/trie/src/storage_proof.rs index dcb9cf08e74b7..77c1c0f3ad22a 100644 --- a/primitives/trie/src/storage_proof.rs +++ b/primitives/trie/src/storage_proof.rs @@ -110,28 +110,15 @@ impl From for crate::MemoryDB { // only allow global definition. // Using compact proof will work directly here (read trie structure and // work directly. - let mut is_hashed_value = false; - let mut accum = Vec::new(); for item in proof.trie_nodes.iter() { - // TODO remove this look up // Note using `default()` as global meta helps looking fro root node. let layout_meta = Default::default(); let (encoded_node, mut meta) = < as TrieLayout>::MetaHasher as MetaHasher >::extract_value(item.as_slice(), layout_meta); - // read state meta. + // read state meta (required for value layout and AltHash node. let _ = as TrieLayout>::Codec::decode_plan(encoded_node, &mut meta); - if meta.recorded_do_value_hash { - debug_assert!(!is_hashed_value); - is_hashed_value = true; - } - accum.push((encoded_node, meta)); - } - for mut item in accum.into_iter() { - if is_hashed_value { - item.1.do_value_hash = true; - } - db.insert_with_meta(crate::EMPTY_PREFIX, item.0, item.1); + db.insert_with_meta(crate::EMPTY_PREFIX, encoded_node, meta); } db } diff --git a/primitives/trie/src/trie_stream.rs b/primitives/trie/src/trie_stream.rs index f3c680f245c54..36ccb49175783 100644 --- a/primitives/trie/src/trie_stream.rs +++ b/primitives/trie/src/trie_stream.rs @@ -25,7 +25,6 @@ use sp_std::ops::Range; use crate::{trie_constants, TrieMeta, StateHasher}; use crate::node_header::{NodeKind, size_and_prefix_iterator}; use crate::node_codec::Bitmap; -use trie_db::Meta; const BRANCH_NODE_NO_VALUE: u8 = 254; const BRANCH_NODE_WITH_VALUE: u8 = 255; @@ -59,9 +58,12 @@ fn fuse_nibbles_node<'a>(nibbles: &'a [u8], kind: NodeKind) -> impl Iterator size_and_prefix_iterator(size, trie_constants::LEAF_PREFIX_MASK), - NodeKind::BranchNoValue => size_and_prefix_iterator(size, trie_constants::BRANCH_WITHOUT_MASK), - NodeKind::BranchWithValue => size_and_prefix_iterator(size, trie_constants::BRANCH_WITH_MASK), + NodeKind::Leaf => size_and_prefix_iterator(size, trie_constants::LEAF_PREFIX_MASK, 2), + NodeKind::BranchNoValue => size_and_prefix_iterator(size, trie_constants::BRANCH_WITHOUT_MASK, 2), + NodeKind::BranchWithValue => size_and_prefix_iterator(size, trie_constants::BRANCH_WITH_MASK, 2), + NodeKind::AltHashLeaf => size_and_prefix_iterator(size, trie_constants::ALT_HASHING_LEAF_PREFIX_MASK, 4), + NodeKind::AltHashBranchNoValue => size_and_prefix_iterator(size, trie_constants::ALT_HASHING_BRANCH_WITHOUT_MASK, 4), + NodeKind::AltHashBranchWithValue => size_and_prefix_iterator(size, trie_constants::ALT_HASHING_BRANCH_WITH_MASK, 4), }; iter_start .chain(if nibbles.len() % 2 == 1 { Some(nibbles[0]) } else { None }) @@ -85,7 +87,12 @@ impl trie_root::TrieStream for TrieStream { } fn append_leaf(&mut self, key: &[u8], value: &[u8]) { - self.buffer.extend(fuse_nibbles_node(key, NodeKind::Leaf)); + let kind = if self.inner_value_hashing { + NodeKind::AltHashLeaf + } else { + NodeKind::Leaf + }; + self.buffer.extend(fuse_nibbles_node(key, kind)); Compact(value.len() as u32).encode_to(&mut self.buffer); self.current_value_range = Some(self.buffer.len()..self.buffer.len() + value.len()); self.buffer.extend_from_slice(value); @@ -99,9 +106,19 @@ impl trie_root::TrieStream for TrieStream { ) { if let Some(partial) = maybe_partial { if maybe_value.is_some() { - self.buffer.extend(fuse_nibbles_node(partial, NodeKind::BranchWithValue)); + let kind = if self.inner_value_hashing { + NodeKind::AltHashBranchWithValue + } else { + NodeKind::BranchWithValue + }; + self.buffer.extend(fuse_nibbles_node(partial, kind)); } else { - self.buffer.extend(fuse_nibbles_node(partial, NodeKind::BranchNoValue)); + let kind = if self.inner_value_hashing { + NodeKind::AltHashBranchNoValue + } else { + NodeKind::BranchNoValue + }; + self.buffer.extend(fuse_nibbles_node(partial, kind)); } let bm = branch_node_bit_mask(has_children); self.buffer.extend([bm.0,bm.1].iter()); @@ -134,11 +151,14 @@ impl trie_root::TrieStream for TrieStream { range: range, unused_value: false, contain_hash: false, - do_value_hash: true, old_hash: false, recorded_do_value_hash: false, + // No existing state, no need to use switch_to_value_hash + switch_to_value_hash: false, + do_value_hash: true, }; - >>::hash(&data, &meta).as_ref().encode_to(&mut self.buffer); + let hash = >>::hash(&data, &meta); + self.buffer.extend_from_slice(hash.as_ref()); } else { H::hash(&data).as_ref().encode_to(&mut self.buffer); } @@ -154,27 +174,19 @@ impl trie_root::TrieStream for TrieStream { range: range, unused_value: false, contain_hash: false, - do_value_hash: inner_value_hashing, old_hash: false, recorded_do_value_hash: inner_value_hashing, - }; - - // Add the recorded_do_value_hash to encoded - let mut encoded = meta.write_state_meta(); - let encoded = if encoded.len() > 0 { - encoded.extend(data); - encoded - } else { - data + switch_to_value_hash: false, + do_value_hash: inner_value_hashing, }; if inner_value_hashing && meta.range.as_ref().map(|r| r.end - r.start >= trie_constants::INNER_HASH_TRESHOLD) .unwrap_or_default() { - >>::hash(&encoded, &meta) + >>::hash(&data, &meta) } else { - H::hash(&encoded) + H::hash(&data) } } From a70ce70363b1ac7cfadd555c93788ca6d0e10f5a Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 8 Jun 2021 16:49:05 +0200 Subject: [PATCH 037/127] alt hashing only for branch with value. --- frame/system/src/lib.rs | 2 +- primitives/trie/src/lib.rs | 5 ++- primitives/trie/src/node_codec.rs | 29 +++++++++--------- primitives/trie/src/node_header.rs | 49 ++++++++++++++---------------- primitives/trie/src/trie_stream.rs | 10 ++---- 5 files changed, 42 insertions(+), 53 deletions(-) diff --git a/frame/system/src/lib.rs b/frame/system/src/lib.rs index 55cacac640493..884a191b2e83b 100644 --- a/frame/system/src/lib.rs +++ b/frame/system/src/lib.rs @@ -1408,7 +1408,7 @@ impl Pallet { >::hashed_key().to_vec() => [69u8; 32].encode() ], children_default: map![], - flag_hashed_value: true, + alt_hashing: true, }) } diff --git a/primitives/trie/src/lib.rs b/primitives/trie/src/lib.rs index fc7d16b71328f..4cbe1777d3ee8 100644 --- a/primitives/trie/src/lib.rs +++ b/primitives/trie/src/lib.rs @@ -938,9 +938,8 @@ mod trie_constants { pub const BRANCH_WITHOUT_MASK: u8 = 0b_10 << 6; pub const BRANCH_WITH_MASK: u8 = 0b_11 << 6; pub const EMPTY_TRIE: u8 = FIRST_PREFIX | (0b_00 << 4); - pub const ALT_HASHING_LEAF_PREFIX_MASK: u8 = FIRST_PREFIX | (0b_01 << 4); - pub const ALT_HASHING_BRANCH_WITHOUT_MASK: u8 = FIRST_PREFIX | (0b_10 << 4); - pub const ALT_HASHING_BRANCH_WITH_MASK: u8 = FIRST_PREFIX | (0b_11 << 4); + pub const ALT_HASHING_LEAF_PREFIX_MASK: u8 = FIRST_PREFIX | (0b_1 << 5); + pub const ALT_HASHING_BRANCH_WITH_MASK: u8 = FIRST_PREFIX | (0b_01 << 4); } #[cfg(test)] diff --git a/primitives/trie/src/node_codec.rs b/primitives/trie/src/node_codec.rs index c239a5c07c97a..37d5510178d71 100644 --- a/primitives/trie/src/node_codec.rs +++ b/primitives/trie/src/node_codec.rs @@ -101,10 +101,16 @@ impl NodeCodec { meta.as_mut() .map(|m| m.set_state_meta(alt_hashing)); + let branch_has_value = if let NodeHeader::Branch(has_value, _) = &header { + *has_value + } else { + false + }; + match header { NodeHeader::Null => Ok(NodePlan::Empty), - NodeHeader::AltHashBranch(has_value, nibble_count) - | NodeHeader::Branch(has_value, nibble_count) => { + NodeHeader::AltHashBranch(nibble_count) + | NodeHeader::Branch(_, nibble_count) => { let padding = nibble_count % nibble_ops::NIBBLE_PER_BYTE != 0; // check that the padding is valid (if any) if padding && nibble_ops::pad_left(data[input.offset]) != 0 { @@ -116,7 +122,7 @@ impl NodeCodec { let partial_padding = nibble_ops::number_padding(nibble_count); let bitmap_range = input.take(BITMAP_LENGTH)?; let bitmap = Bitmap::decode(&data[bitmap_range])?; - let value = if has_value { + let value = if branch_has_value { if alt_hashing && contains_hash { ValuePlan::HashedValue(input.take(H::LENGTH)?, 0) } else { @@ -257,17 +263,14 @@ impl> NodeCodecT for NodeCodec { meta: &mut M, ) -> Vec { let mut output = match (&maybe_value, meta.do_value_hash()) { - (&Value::NoValue, false) => { - partial_from_iterator_encode(partial, number_nibble, NodeKind::BranchWithValue) - }, - (_, false) => { + (&Value::NoValue, _) => { partial_from_iterator_encode(partial, number_nibble, NodeKind::BranchNoValue) }, - (&Value::NoValue, true) => { - partial_from_iterator_encode(partial, number_nibble, NodeKind::AltHashBranchWithValue) + (_, false) => { + partial_from_iterator_encode(partial, number_nibble, NodeKind::BranchWithValue) }, (_, true) => { - partial_from_iterator_encode(partial, number_nibble, NodeKind::AltHashBranchNoValue) + partial_from_iterator_encode(partial, number_nibble, NodeKind::AltHashBranchWithValue) }, }; @@ -325,8 +328,7 @@ fn partial_from_iterator_encode>( NodeKind::BranchWithValue => NodeHeader::Branch(true, nibble_count).encode_to(&mut output), NodeKind::BranchNoValue => NodeHeader::Branch(false, nibble_count).encode_to(&mut output), NodeKind::AltHashLeaf => NodeHeader::AltHashLeaf(nibble_count).encode_to(&mut output), - NodeKind::AltHashBranchWithValue => NodeHeader::AltHashBranch(true, nibble_count).encode_to(&mut output), - NodeKind::AltHashBranchNoValue => NodeHeader::AltHashBranch(false, nibble_count).encode_to(&mut output), + NodeKind::AltHashBranchWithValue => NodeHeader::AltHashBranch(nibble_count).encode_to(&mut output), }; output.extend(partial); output @@ -346,8 +348,7 @@ fn partial_encode(partial: Partial, node_kind: NodeKind) -> Vec { NodeKind::BranchWithValue => NodeHeader::Branch(true, nibble_count).encode_to(&mut output), NodeKind::BranchNoValue => NodeHeader::Branch(false, nibble_count).encode_to(&mut output), NodeKind::AltHashLeaf => NodeHeader::AltHashLeaf(nibble_count).encode_to(&mut output), - NodeKind::AltHashBranchWithValue => NodeHeader::AltHashBranch(true, nibble_count).encode_to(&mut output), - NodeKind::AltHashBranchNoValue => NodeHeader::AltHashBranch(false, nibble_count).encode_to(&mut output), + NodeKind::AltHashBranchWithValue => NodeHeader::AltHashBranch(nibble_count).encode_to(&mut output), }; if number_nibble_encoded > 0 { output.push(nibble_ops::pad_right((partial.0).1)); diff --git a/primitives/trie/src/node_header.rs b/primitives/trie/src/node_header.rs index a1f0090ecb8a2..8d086d04d302e 100644 --- a/primitives/trie/src/node_header.rs +++ b/primitives/trie/src/node_header.rs @@ -28,7 +28,7 @@ pub(crate) enum NodeHeader { Null, Branch(bool, usize), Leaf(usize), - AltHashBranch(bool, usize), + AltHashBranch(usize), AltHashLeaf(usize), } @@ -38,7 +38,6 @@ pub(crate) enum NodeKind { BranchNoValue, BranchWithValue, AltHashLeaf, - AltHashBranchNoValue, AltHashBranchWithValue, } @@ -47,17 +46,15 @@ impl Encode for NodeHeader { match self { NodeHeader::Null => output.push_byte(trie_constants::EMPTY_TRIE), NodeHeader::Branch(true, nibble_count) => - encode_size_and_prefix(*nibble_count, trie_constants::BRANCH_WITH_MASK, output), + encode_size_and_prefix(*nibble_count, trie_constants::BRANCH_WITH_MASK, 2, output), NodeHeader::Branch(false, nibble_count) => - encode_size_and_prefix(*nibble_count, trie_constants::BRANCH_WITHOUT_MASK, output), + encode_size_and_prefix(*nibble_count, trie_constants::BRANCH_WITHOUT_MASK, 2, output), NodeHeader::Leaf(nibble_count) => - encode_size_and_prefix(*nibble_count, trie_constants::LEAF_PREFIX_MASK, output), - NodeHeader::AltHashBranch(true, nibble_count) => - encode_size_and_prefix_alt(*nibble_count, trie_constants::ALT_HASHING_BRANCH_WITH_MASK, output), - NodeHeader::AltHashBranch(false, nibble_count) => - encode_size_and_prefix_alt(*nibble_count, trie_constants::ALT_HASHING_BRANCH_WITHOUT_MASK, output), - NodeHeader::AltHashLeaf(nibble_count) => - encode_size_and_prefix_alt(*nibble_count, trie_constants::ALT_HASHING_LEAF_PREFIX_MASK, output), + encode_size_and_prefix(*nibble_count, trie_constants::LEAF_PREFIX_MASK, 2, output), + NodeHeader::AltHashBranch(nibble_count) => + encode_size_and_prefix(*nibble_count, trie_constants::ALT_HASHING_BRANCH_WITH_MASK, 4, output), + NodeHeader::AltHashLeaf(nibble_count) => + encode_size_and_prefix(*nibble_count, trie_constants::ALT_HASHING_LEAF_PREFIX_MASK, 3, output), } } } @@ -87,12 +84,15 @@ impl Decode for NodeHeader { trie_constants::LEAF_PREFIX_MASK => Ok(NodeHeader::Leaf(decode_size(i, input, 2)?)), trie_constants::BRANCH_WITH_MASK => Ok(NodeHeader::Branch(true, decode_size(i, input, 2)?)), trie_constants::BRANCH_WITHOUT_MASK => Ok(NodeHeader::Branch(false, decode_size(i, input, 2)?)), - trie_constants::EMPTY_TRIE => match i & (0b1111 << 4) { - trie_constants::ALT_HASHING_LEAF_PREFIX_MASK => Ok(NodeHeader::AltHashLeaf(decode_size(i, input, 4)?)), - trie_constants::ALT_HASHING_BRANCH_WITH_MASK => Ok(NodeHeader::AltHashBranch(true, decode_size(i, input, 4)?)), - trie_constants::ALT_HASHING_BRANCH_WITHOUT_MASK => Ok(NodeHeader::AltHashBranch(false, decode_size(i, input, 4)?)), - // do not allow any special encoding - _ => Err("Unallowed encoding".into()), + trie_constants::EMPTY_TRIE => { + if i & (0b111 << 5) == trie_constants::ALT_HASHING_LEAF_PREFIX_MASK { + Ok(NodeHeader::AltHashLeaf(decode_size(i, input, 3)?)) + } else if i & (0b1111 << 4) == trie_constants::ALT_HASHING_BRANCH_WITH_MASK { + Ok(NodeHeader::AltHashBranch(decode_size(i, input, 4)?)) + } else { + // do not allow any special encoding + Err("Unallowed encoding".into()) + } }, _ => unreachable!(), } @@ -105,7 +105,7 @@ impl Decode for NodeHeader { pub(crate) fn size_and_prefix_iterator(size: usize, prefix: u8, prefix_mask: usize) -> impl Iterator { let size = sp_std::cmp::min(trie_constants::NIBBLE_SIZE_BOUND, size); - let max_value = 255u8 >> prefix_mask; + let max_value = 255u8 >> (8 - prefix_mask); let l1 = sp_std::cmp::min(max_value as usize - 1, size); let (first_byte, mut rem) = if size == l1 { (once(prefix + l1 as u8), 0) @@ -130,15 +130,10 @@ pub(crate) fn size_and_prefix_iterator(size: usize, prefix: u8, prefix_mask: usi } /// Encodes size and prefix to a stream output (prefix on 2 first bit only). -fn encode_size_and_prefix(size: usize, prefix: u8, out: &mut W) { - for b in size_and_prefix_iterator(size, prefix, 2) { - out.push_byte(b) - } -} - -/// Encodes size and prefix to a stream output with prefix (prefix on 4 first bit only). -fn encode_size_and_prefix_alt(size: usize, prefix: u8, out: &mut W) { - for b in size_and_prefix_iterator(size, prefix, 4) { +fn encode_size_and_prefix(size: usize, prefix: u8, prefix_mask: usize, out: &mut W) + where W: Output + ?Sized, +{ + for b in size_and_prefix_iterator(size, prefix, prefix_mask) { out.push_byte(b) } } diff --git a/primitives/trie/src/trie_stream.rs b/primitives/trie/src/trie_stream.rs index 36ccb49175783..d5aa95421835e 100644 --- a/primitives/trie/src/trie_stream.rs +++ b/primitives/trie/src/trie_stream.rs @@ -61,8 +61,7 @@ fn fuse_nibbles_node<'a>(nibbles: &'a [u8], kind: NodeKind) -> impl Iterator size_and_prefix_iterator(size, trie_constants::LEAF_PREFIX_MASK, 2), NodeKind::BranchNoValue => size_and_prefix_iterator(size, trie_constants::BRANCH_WITHOUT_MASK, 2), NodeKind::BranchWithValue => size_and_prefix_iterator(size, trie_constants::BRANCH_WITH_MASK, 2), - NodeKind::AltHashLeaf => size_and_prefix_iterator(size, trie_constants::ALT_HASHING_LEAF_PREFIX_MASK, 4), - NodeKind::AltHashBranchNoValue => size_and_prefix_iterator(size, trie_constants::ALT_HASHING_BRANCH_WITHOUT_MASK, 4), + NodeKind::AltHashLeaf => size_and_prefix_iterator(size, trie_constants::ALT_HASHING_LEAF_PREFIX_MASK, 3), NodeKind::AltHashBranchWithValue => size_and_prefix_iterator(size, trie_constants::ALT_HASHING_BRANCH_WITH_MASK, 4), }; iter_start @@ -113,12 +112,7 @@ impl trie_root::TrieStream for TrieStream { }; self.buffer.extend(fuse_nibbles_node(partial, kind)); } else { - let kind = if self.inner_value_hashing { - NodeKind::AltHashBranchNoValue - } else { - NodeKind::BranchNoValue - }; - self.buffer.extend(fuse_nibbles_node(partial, kind)); + self.buffer.extend(fuse_nibbles_node(partial, NodeKind::BranchNoValue)); } let bm = branch_node_bit_mask(has_children); self.buffer.extend([bm.0,bm.1].iter()); From 11f0d5a66d4be0a6b3dfe840129ee37c9ada9f4f Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 8 Jun 2021 17:34:06 +0200 Subject: [PATCH 038/127] fix trie tests --- client/api/src/in_mem.rs | 2 +- primitives/trie/src/lib.rs | 6 +++--- primitives/trie/src/node_codec.rs | 3 ++- primitives/trie/src/node_header.rs | 2 +- primitives/trie/src/trie_stream.rs | 3 +-- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/client/api/src/in_mem.rs b/client/api/src/in_mem.rs index 3dbcd4aa897de..3030784b3c320 100644 --- a/client/api/src/in_mem.rs +++ b/client/api/src/in_mem.rs @@ -576,7 +576,7 @@ impl backend::BlockImportOperation for BlockImportOperatio let (root, transaction) = self.old_state.full_storage_root( storage.top.iter().map(|(k, v)| (k.as_ref(), Some(v.as_ref()))), child_delta, - storage.flag_hashed_value, + storage.alt_hashing, ); self.new_state = Some(transaction); diff --git a/primitives/trie/src/lib.rs b/primitives/trie/src/lib.rs index 4cbe1777d3ee8..07a398090a5ca 100644 --- a/primitives/trie/src/lib.rs +++ b/primitives/trie/src/lib.rs @@ -980,14 +980,13 @@ mod tests { let mut memdb = MemoryDBMeta::<_, T::MetaHasher>::default(); let mut root = Default::default(); { - let mut t = TrieDBMut::::new_with_layout(&mut memdb, &mut root, layout); + let mut t = TrieDBMut::::new_with_layout(&mut memdb, &mut root, layout.clone()); for (x, y) in input.clone() { t.insert(x, y).unwrap(); } } { - // Not using layout: it should be initialized from state root meta. - let t = TrieDB::::new(&mut memdb, &root).unwrap(); + let t = TrieDB::::new_with_layout(&mut memdb, &root, layout).unwrap(); assert_eq!( input.iter().map(|(i, j)| (i.to_vec(), j.to_vec())).collect::>(), t.iter().unwrap() @@ -1000,6 +999,7 @@ mod tests { fn check_input(input: &Vec<(&[u8], &[u8])>) { // TODO remove this iter let layout = Layout::with_inner_hashing(); + check_iteration::(input, layout.clone()); check_equivalent::(input, layout.clone()); diff --git a/primitives/trie/src/node_codec.rs b/primitives/trie/src/node_codec.rs index 37d5510178d71..d7849fcf52498 100644 --- a/primitives/trie/src/node_codec.rs +++ b/primitives/trie/src/node_codec.rs @@ -104,7 +104,8 @@ impl NodeCodec { let branch_has_value = if let NodeHeader::Branch(has_value, _) = &header { *has_value } else { - false + // alt_hash_branch + true }; match header { diff --git a/primitives/trie/src/node_header.rs b/primitives/trie/src/node_header.rs index 8d086d04d302e..6711c1a047127 100644 --- a/primitives/trie/src/node_header.rs +++ b/primitives/trie/src/node_header.rs @@ -105,7 +105,7 @@ impl Decode for NodeHeader { pub(crate) fn size_and_prefix_iterator(size: usize, prefix: u8, prefix_mask: usize) -> impl Iterator { let size = sp_std::cmp::min(trie_constants::NIBBLE_SIZE_BOUND, size); - let max_value = 255u8 >> (8 - prefix_mask); + let max_value = 255u8 >> prefix_mask; let l1 = sp_std::cmp::min(max_value as usize - 1, size); let (first_byte, mut rem) = if size == l1 { (once(prefix + l1 as u8), 0) diff --git a/primitives/trie/src/trie_stream.rs b/primitives/trie/src/trie_stream.rs index d5aa95421835e..3206ff5729870 100644 --- a/primitives/trie/src/trie_stream.rs +++ b/primitives/trie/src/trie_stream.rs @@ -151,8 +151,7 @@ impl trie_root::TrieStream for TrieStream { switch_to_value_hash: false, do_value_hash: true, }; - let hash = >>::hash(&data, &meta); - self.buffer.extend_from_slice(hash.as_ref()); + >>::hash(&data, &meta).as_ref().encode_to(&mut self.buffer); } else { H::hash(&data).as_ref().encode_to(&mut self.buffer); } From cf0f0dfc0eb371b65e2a4571ef955e92daae1cac Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 8 Jun 2021 19:28:37 +0200 Subject: [PATCH 039/127] Hash of value include the encoded size. --- Cargo.lock | 16 +++++----- bin/node/bench/src/generator.rs | 4 +-- client/chain-spec/src/chain_spec.rs | 14 ++++----- client/db/src/bench.rs | 2 +- client/db/src/lib.rs | 30 +++++++++---------- client/executor/runtime-test/src/lib.rs | 2 +- client/executor/src/integration_tests/mod.rs | 4 +-- client/light/src/backend.rs | 3 +- .../service/src/chain_ops/export_raw_state.rs | 4 +-- frame/support/test/tests/instance.rs | 2 +- primitives/externalities/src/lib.rs | 3 +- primitives/io/src/lib.rs | 15 +++++----- primitives/state-machine/src/basic.rs | 6 ++-- primitives/state-machine/src/ext.rs | 14 ++++----- primitives/state-machine/src/lib.rs | 9 +++--- .../src/overlayed_changes/mod.rs | 14 ++++----- .../state-machine/src/proving_backend.rs | 14 ++------- primitives/state-machine/src/read_only.rs | 4 +-- primitives/state-machine/src/testing.rs | 2 +- primitives/state-machine/src/trie_backend.rs | 8 ++--- primitives/tasks/src/async_externalities.rs | 4 +-- primitives/trie/src/lib.rs | 12 ++++---- primitives/trie/src/node_codec.rs | 12 +++++--- primitives/trie/src/storage_proof.rs | 3 +- test-utils/client/src/lib.rs | 2 +- test-utils/runtime/src/genesismap.rs | 4 +-- test-utils/runtime/src/system.rs | 2 +- 27 files changed, 101 insertions(+), 108 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 24c3ff64dee71..771af600156c7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2342,7 +2342,7 @@ dependencies = [ [[package]] name = "hash-db" version = "0.15.2" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#da84037309542514f9eebc853909bcbfb2838805" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#0f04db6e1d4781c41684df1539e3fd52e9aa85ae" [[package]] name = "hash256-std-hasher" @@ -2356,7 +2356,7 @@ dependencies = [ [[package]] name = "hash256-std-hasher" version = "0.15.2" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#da84037309542514f9eebc853909bcbfb2838805" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#0f04db6e1d4781c41684df1539e3fd52e9aa85ae" dependencies = [ "crunchy", ] @@ -3001,7 +3001,7 @@ dependencies = [ [[package]] name = "keccak-hasher" version = "0.15.3" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#da84037309542514f9eebc853909bcbfb2838805" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#0f04db6e1d4781c41684df1539e3fd52e9aa85ae" dependencies = [ "hash-db", "hash256-std-hasher 0.15.2 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple4)", @@ -3793,7 +3793,7 @@ dependencies = [ [[package]] name = "memory-db" version = "0.26.0" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#da84037309542514f9eebc853909bcbfb2838805" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#0f04db6e1d4781c41684df1539e3fd52e9aa85ae" dependencies = [ "hash-db", "hashbrown", @@ -10375,7 +10375,7 @@ checksum = "a7f741b240f1a48843f9b8e0444fb55fb2a4ff67293b50a9179dfd5ea67f8d41" [[package]] name = "trie-bench" version = "0.27.0" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#da84037309542514f9eebc853909bcbfb2838805" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#0f04db6e1d4781c41684df1539e3fd52e9aa85ae" dependencies = [ "criterion", "hash-db", @@ -10390,7 +10390,7 @@ dependencies = [ [[package]] name = "trie-db" version = "0.22.5" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#da84037309542514f9eebc853909bcbfb2838805" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#0f04db6e1d4781c41684df1539e3fd52e9aa85ae" dependencies = [ "hash-db", "hashbrown", @@ -10402,7 +10402,7 @@ dependencies = [ [[package]] name = "trie-root" version = "0.16.0" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#da84037309542514f9eebc853909bcbfb2838805" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#0f04db6e1d4781c41684df1539e3fd52e9aa85ae" dependencies = [ "hash-db", ] @@ -10420,7 +10420,7 @@ dependencies = [ [[package]] name = "trie-standardmap" version = "0.15.2" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#da84037309542514f9eebc853909bcbfb2838805" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#0f04db6e1d4781c41684df1539e3fd52e9aa85ae" dependencies = [ "hash-db", "keccak-hasher 0.15.3 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple4)", diff --git a/bin/node/bench/src/generator.rs b/bin/node/bench/src/generator.rs index 913c1ff8779f0..f811802d357ee 100644 --- a/bin/node/bench/src/generator.rs +++ b/bin/node/bench/src/generator.rs @@ -31,7 +31,7 @@ use crate::simple_trie::SimpleTrie; pub fn generate_trie( db: Arc, key_values: impl IntoIterator, Vec)>, - flag_hashed_value: bool, + alt_hashing: bool, ) -> Hash { let mut root = Hash::default(); @@ -44,7 +44,7 @@ pub fn generate_trie( let mut trie = SimpleTrie { db, overlay: &mut overlay }; { - let mut trie_db = if flag_hashed_value { + let mut trie_db = if alt_hashing { let layout = sp_trie::Layout::with_inner_hashing(); TrieDBMut::::new_with_layout(&mut trie, &mut root, layout) } else { diff --git a/client/chain-spec/src/chain_spec.rs b/client/chain-spec/src/chain_spec.rs index 7bf059ae8d0f4..51c3a8c677645 100644 --- a/client/chain-spec/src/chain_spec.rs +++ b/client/chain-spec/src/chain_spec.rs @@ -87,9 +87,9 @@ impl GenesisSource { ) .collect(); - let flag_hashed_value = storage.flag_hashed_value; + let alt_hashing = storage.alt_hashing; - Ok(Genesis::Raw(RawGenesis { top, children_default, flag_hashed_value })) + Ok(Genesis::Raw(RawGenesis { top, children_default, alt_hashing })) }, } } @@ -99,7 +99,7 @@ impl BuildStorage for ChainSpec { fn build_storage(&self) -> Result { match self.genesis.resolve()? { Genesis::Runtime(gc) => gc.build_storage(), - Genesis::Raw(RawGenesis { top: map, children_default: children_map, flag_hashed_value }) => Ok(Storage { + Genesis::Raw(RawGenesis { top: map, children_default: children_map, alt_hashing }) => Ok(Storage { top: map.into_iter().map(|(k, v)| (k.0, v.0)).collect(), children_default: children_map.into_iter().map(|(storage_key, child_content)| { let child_info = ChildInfo::new_default(storage_key.0.as_slice()); @@ -111,7 +111,7 @@ impl BuildStorage for ChainSpec { }, ) }).collect(), - flag_hashed_value, + alt_hashing, }), } } @@ -134,7 +134,7 @@ pub struct RawGenesis { pub top: GenesisStorage, pub children_default: HashMap, #[serde(default)] - pub flag_hashed_value: bool, + pub alt_hashing: bool, } #[derive(Serialize, Deserialize)] @@ -328,9 +328,9 @@ impl ChainSpec { .collect(), )) .collect(); - let flag_hashed_value = storage.flag_hashed_value; + let alt_hashing = storage.alt_hashing; - Genesis::Raw(RawGenesis { top, children_default, flag_hashed_value }) + Genesis::Raw(RawGenesis { top, children_default, alt_hashing }) }, (_, genesis) => genesis, }; diff --git a/client/db/src/bench.rs b/client/db/src/bench.rs index 0f111eed99189..409897b831440 100644 --- a/client/db/src/bench.rs +++ b/client/db/src/bench.rs @@ -151,7 +151,7 @@ impl BenchmarkingState { state.add_whitelist_to_tracker(); state.reopen()?; - let flagged = genesis.flag_hashed_value; + let flagged = genesis.alt_hashing; let child_delta = genesis.children_default.iter().map(|(_storage_key, child_content)| ( &child_content.child_info, child_content.data.iter().map(|(k, v)| (k.as_ref(), Some(v.as_ref()))), diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index 41a6853aebc71..1eefca0c74ca4 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -782,7 +782,7 @@ impl sc_client_api::backend::BlockImportOperation for Bloc )); let mut changes_trie_config: Option = None; - let flag = storage.flag_hashed_value; + let flag = storage.alt_hashing; let (root, transaction) = self.old_state.full_storage_root( storage.top.iter().map(|(k, v)| { if &k[..] == well_known_keys::CHANGES_TRIE_CONFIG { @@ -2291,7 +2291,7 @@ pub(crate) mod tests { set_state_data_inner(true); set_state_data_inner(false); } - fn set_state_data_inner(flagged: bool) { + fn set_state_data_inner(alt_hashing: bool) { let db = Backend::::new_test(2, 0); let hash = { let mut op = db.begin_operation().unwrap(); @@ -2312,14 +2312,14 @@ pub(crate) mod tests { header.state_root = op.old_state.storage_root(storage .iter() .map(|(x, y)| (&x[..], Some(&y[..]))), - flagged, + alt_hashing, ).0.into(); let hash = header.hash(); op.reset_storage(Storage { top: storage.into_iter().collect(), children_default: Default::default(), - flag_hashed_value: flagged, + alt_hashing: alt_hashing, }).unwrap(); op.set_block_data( header.clone(), @@ -2358,7 +2358,7 @@ pub(crate) mod tests { let (root, overlay) = op.old_state.storage_root( storage.iter() .map(|(k, v)| (&k[..], v.as_ref().map(|v| &v[..]))), - flagged, + alt_hashing, ); op.update_db_storage(overlay).unwrap(); header.state_root = root.into(); @@ -2385,7 +2385,7 @@ pub(crate) mod tests { fn delete_only_when_negative_rc() { sp_tracing::try_init_simple(); let key; - let flagged = false; + let alt_hashing = false; let backend = Backend::::new_test(1, 0); let hash = { @@ -2399,13 +2399,13 @@ pub(crate) mod tests { extrinsics_root: Default::default(), }; - header.state_root = op.old_state.storage_root(std::iter::empty(), flagged).0.into(); + header.state_root = op.old_state.storage_root(std::iter::empty(), alt_hashing).0.into(); let hash = header.hash(); op.reset_storage(Storage { top: Default::default(), children_default: Default::default(), - flag_hashed_value: flagged, + alt_hashing: alt_hashing, }).unwrap(); key = op.db_updates.insert(EMPTY_PREFIX, b"hello"); @@ -2441,7 +2441,7 @@ pub(crate) mod tests { .iter() .cloned() .map(|(x, y)| (x, Some(y))), - flagged, + alt_hashing, ).0.into(); let hash = header.hash(); @@ -2479,7 +2479,7 @@ pub(crate) mod tests { .iter() .cloned() .map(|(x, y)| (x, Some(y))), - flagged, + alt_hashing, ).0.into(); let hash = header.hash(); @@ -2517,7 +2517,7 @@ pub(crate) mod tests { .iter() .cloned() .map(|(x, y)| (x, Some(y))), - flagged, + alt_hashing, ).0.into(); op.set_block_data( @@ -2833,7 +2833,7 @@ pub(crate) mod tests { #[test] fn storage_hash_is_cached_correctly() { let backend = Backend::::new_test(10, 10); - let flagged = false; + let alt_hashing = false; let hash0 = { let mut op = backend.begin_operation().unwrap(); @@ -2851,14 +2851,14 @@ pub(crate) mod tests { header.state_root = op.old_state.storage_root(storage .iter() .map(|(x, y)| (&x[..], Some(&y[..]))), - flagged, + alt_hashing, ).0.into(); let hash = header.hash(); op.reset_storage(Storage { top: storage.into_iter().collect(), children_default: Default::default(), - flag_hashed_value: flagged, + alt_hashing: alt_hashing, }).unwrap(); op.set_block_data( header.clone(), @@ -2893,7 +2893,7 @@ pub(crate) mod tests { let (root, overlay) = op.old_state.storage_root( storage.iter() .map(|(k, v)| (&k[..], v.as_ref().map(|v| &v[..]))), - flagged, + alt_hashing, ); op.update_db_storage(overlay).unwrap(); header.state_root = root.into(); diff --git a/client/executor/runtime-test/src/lib.rs b/client/executor/runtime-test/src/lib.rs index 78b824140bcc7..fc452d135da07 100644 --- a/client/executor/runtime-test/src/lib.rs +++ b/client/executor/runtime-test/src/lib.rs @@ -64,7 +64,7 @@ sp_core::wasm_export_functions! { fn test_switch_state() { print("switch_state"); - storage::flag_hash_value(); + storage::alt_hashing(); print("switched!"); } diff --git a/client/executor/src/integration_tests/mod.rs b/client/executor/src/integration_tests/mod.rs index 5076352dff40a..7c702cb855d6e 100644 --- a/client/executor/src/integration_tests/mod.rs +++ b/client/executor/src/integration_tests/mod.rs @@ -211,7 +211,7 @@ fn storage_should_work(wasm_method: WasmExecutionMethod) { b"baz".to_vec() => b"bar".to_vec() ], children_default: map![], - flag_hashed_value: false, + alt_hashing: false, }); assert_eq!(ext, expected); } @@ -245,7 +245,7 @@ fn clear_prefix_should_work(wasm_method: WasmExecutionMethod) { b"bbb".to_vec() => b"5".to_vec() ], children_default: map![], - flag_hashed_value: false, + alt_hashing: false, }); assert_eq!(expected, ext); } diff --git a/client/light/src/backend.rs b/client/light/src/backend.rs index 2d65c7b3347ff..b593b8998a0ae 100644 --- a/client/light/src/backend.rs +++ b/client/light/src/backend.rs @@ -346,8 +346,7 @@ impl BlockImportOperation for ImportOperation } let storage_update = InMemoryBackend::from(storage); - let flag = input.flag_hashed_value; - let (storage_root, _) = storage_update.full_storage_root(std::iter::empty(), child_delta, flag); + let (storage_root, _) = storage_update.full_storage_root(std::iter::empty(), child_delta, input.alt_hashing); self.storage_update = Some(storage_update); Ok(storage_root) diff --git a/client/service/src/chain_ops/export_raw_state.rs b/client/service/src/chain_ops/export_raw_state.rs index fe96994ed2605..28fad7632fd80 100644 --- a/client/service/src/chain_ops/export_raw_state.rs +++ b/client/service/src/chain_ops/export_raw_state.rs @@ -42,7 +42,7 @@ where let empty_key = StorageKey(Vec::new()); let mut top_storage = client.storage_pairs(&block, &empty_key)?; let mut children_default = HashMap::new(); - let flag_hashed_value = client.state_hashed_value(&block)?; + let alt_hashing = client.state_hashed_value(&block)?; // Remove all default child storage roots from the top storage and collect the child storage // pairs. @@ -70,5 +70,5 @@ where } let top = top_storage.into_iter().map(|(k, v)| (k.0, v.0)).collect(); - Ok(Storage { top, children_default, flag_hashed_value }) + Ok(Storage { top, children_default, alt_hashing }) } diff --git a/frame/support/test/tests/instance.rs b/frame/support/test/tests/instance.rs index 91363d0857e10..1a07b7174e2ed 100644 --- a/frame/support/test/tests/instance.rs +++ b/frame/support/test/tests/instance.rs @@ -327,7 +327,7 @@ fn storage_instance_independence() { let mut storage = sp_core::storage::Storage { top: std::collections::BTreeMap::new(), children_default: std::collections::HashMap::new(), - flag_hashed_value: true, + alt_hashing: true, }; sp_state_machine::BasicExternalities::execute_with_storage(&mut storage, || { module2::Value::::put(0); diff --git a/primitives/externalities/src/lib.rs b/primitives/externalities/src/lib.rs index 7d315812b2178..54003d9886311 100644 --- a/primitives/externalities/src/lib.rs +++ b/primitives/externalities/src/lib.rs @@ -293,7 +293,8 @@ pub trait Externalities: ExtensionStore { } /// Set flag in inner state to activate hashing of values. - fn flag_hash_value(&mut self); + /// TODO remove + fn alt_hashing(&mut self); } /// Extension for the [`Externalities`] trait. diff --git a/primitives/io/src/lib.rs b/primitives/io/src/lib.rs index 22e55a9b78a68..a94e0d624be4a 100644 --- a/primitives/io/src/lib.rs +++ b/primitives/io/src/lib.rs @@ -214,10 +214,9 @@ pub trait Storage { .expect("No open transaction that can be committed."); } - /// Set flag to switch storage state - /// to internally hash its values. - fn flag_hash_value(&mut self) { - self.flag_hash_value(); + /// Swith state to alternate hashing. + fn alt_hashing(&mut self) { + self.alt_hashing(); } } @@ -1465,7 +1464,7 @@ mod tests { t = BasicExternalities::new(Storage { top: map![b"foo".to_vec() => b"bar".to_vec()], children_default: map![], - flag_hashed_value: false, + alt_hashing: false, }); t.execute_with(|| { @@ -1477,7 +1476,7 @@ mod tests { t = BasicExternalities::new(Storage { top: map![b"foo00".to_vec() => value.clone()], children_default: map![], - flag_hashed_value: true, + alt_hashing: true, }); t.execute_with(|| { @@ -1492,7 +1491,7 @@ mod tests { let mut t = BasicExternalities::new(Storage { top: map![b":test".to_vec() => value.clone()], children_default: map![], - flag_hashed_value: false, + alt_hashing: false, }); t.execute_with(|| { @@ -1515,7 +1514,7 @@ mod tests { b":abdd".to_vec() => b"\x0b\0\0\0Hello world".to_vec() ], children_default: map![], - flag_hashed_value: false, + alt_hashing: false, }); t.execute_with(|| { diff --git a/primitives/state-machine/src/basic.rs b/primitives/state-machine/src/basic.rs index 2cc7314392bdf..f4ddc6f36f80d 100644 --- a/primitives/state-machine/src/basic.rs +++ b/primitives/state-machine/src/basic.rs @@ -340,7 +340,7 @@ impl Externalities for BasicExternalities { unimplemented!("set_whitelist is not supported in Basic") } - fn flag_hash_value(&mut self) { + fn alt_hashing(&mut self) { self.inner.alt_hashing = true; } } @@ -408,7 +408,7 @@ mod tests { child_info: child_info.to_owned(), } ], - flag_hashed_value: false, + alt_hashing: false, }); assert_eq!(ext.child_storage(child_info, b"doe"), Some(b"reindeer".to_vec())); @@ -439,7 +439,7 @@ mod tests { child_info: child_info.to_owned(), } ], - flag_hashed_value: false, + alt_hashing: false, }); let res = ext.kill_child_storage(child_info, None); diff --git a/primitives/state-machine/src/ext.rs b/primitives/state-machine/src/ext.rs index e187c128345b8..dd6e0f519ef35 100644 --- a/primitives/state-machine/src/ext.rs +++ b/primitives/state-machine/src/ext.rs @@ -547,7 +547,7 @@ where } else { let root = if let Some((changes, info)) = self.overlay.child_changes(storage_key) { let delta = changes.map(|(k, v)| (k.as_ref(), v.value().map(AsRef::as_ref))); - Some(self.backend.child_storage_root(info, delta, self.overlay.flag_hash_value())) + Some(self.backend.child_storage_root(info, delta, self.overlay.alt_hashing())) } else { None }; @@ -738,9 +738,9 @@ where self.backend.proof_size() } - fn flag_hash_value(&mut self) { + fn alt_hashing(&mut self) { self.mark_dirty(); - self.overlay.set_flag_hash_value() + self.overlay.set_alt_hashing() } } @@ -952,7 +952,7 @@ mod tests { vec![40] => vec![40] ], children_default: map![], - flag_hashed_value: false, + alt_hashing: false, }.into(); let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); @@ -998,7 +998,7 @@ mod tests { child_info: child_info.to_owned(), } ], - flag_hashed_value: false, + alt_hashing: false, }.into(); let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); @@ -1043,7 +1043,7 @@ mod tests { child_info: child_info.to_owned(), } ], - flag_hashed_value: false, + alt_hashing: false, }.into(); let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); @@ -1083,7 +1083,7 @@ mod tests { child_info: child_info.to_owned(), } ], - flag_hashed_value: false, + alt_hashing: false, }.into(); let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); diff --git a/primitives/state-machine/src/lib.rs b/primitives/state-machine/src/lib.rs index 709357430a40c..19ab6b994d63c 100644 --- a/primitives/state-machine/src/lib.rs +++ b/primitives/state-machine/src/lib.rs @@ -1502,7 +1502,7 @@ mod tests { let mut trie = TrieDBMut::from_existing_with_layout( &mut mdb, &mut root, - layout.cloen(), + layout.clone(), ).unwrap(); trie.insert(b"foo", vec![1u8; 1_000].as_slice()) // big inner hash .expect("insert failed"); @@ -1527,11 +1527,12 @@ mod tests { local_result1.into_iter().collect::>(), vec![(b"foo222".to_vec(), Some(vec![5u8; 100]))], ); + println!("a{:?}", remote_proof.encode().len()); + println!("b{:?}", remote_proof.encoded_size()); remote_proof }; let remote_proof = check_proof(mdb.clone(), root.clone()); - // check full values in proof assert!(remote_proof.encode().len() > 1_100); assert!(remote_proof.encoded_size() > 1_100); @@ -1551,7 +1552,7 @@ mod tests { .expect("insert failed"); } let root3 = root.clone(); - assert!(root2 == root3); + assert!(root1 == root3); // different value then same is enough to update // from triedbmut persipective (do not // work with state machine as only changes do makes @@ -1568,7 +1569,7 @@ mod tests { .expect("insert failed"); } let root3 = root.clone(); - assert!(root2 != root3); + assert!(root1 != root3); let remote_proof = check_proof(mdb.clone(), root.clone()); // nodes foo is replaced by its hashed value form. assert!(remote_proof.encode().len() < 1000); diff --git a/primitives/state-machine/src/overlayed_changes/mod.rs b/primitives/state-machine/src/overlayed_changes/mod.rs index 5386192255ad3..5df20ba1f233b 100644 --- a/primitives/state-machine/src/overlayed_changes/mod.rs +++ b/primitives/state-machine/src/overlayed_changes/mod.rs @@ -108,7 +108,7 @@ pub struct OverlayedChanges { /// True if extrinsics stats must be collected. collect_extrinsics: bool, /// True if we flag inner state to store hash of values. - flag_hash_value: bool, + alt_hashing: bool, /// Collect statistic on this execution. stats: StateMachineStats, } @@ -263,13 +263,13 @@ impl OverlayedChanges { } /// Ask to switch state to use inner hash. - pub fn set_flag_hash_value(&mut self) { - self.flag_hash_value = true; + pub fn set_alt_hashing(&mut self) { + self.alt_hashing = true; } - /// Is `flag_hash_value` flag set. - pub fn flag_hash_value(&self) -> bool { - self.flag_hash_value + /// Is `alt_hashing` flag set. + pub fn alt_hashing(&self) -> bool { + self.alt_hashing } /// Returns a double-Option: None if the key is unknown (i.e. and the query should be referred @@ -643,7 +643,7 @@ impl OverlayedChanges { |(k, v)| (&k[..], v.value().map(|v| &v[..])) ))); - let (root, transaction) = backend.full_storage_root(delta, child_delta, self.flag_hash_value); + let (root, transaction) = backend.full_storage_root(delta, child_delta, self.alt_hashing); cache.transaction = Some(transaction); cache.transaction_storage_root = Some(root); diff --git a/primitives/state-machine/src/proving_backend.rs b/primitives/state-machine/src/proving_backend.rs index 1ab9d09125648..c1eaf7068c7ac 100644 --- a/primitives/state-machine/src/proving_backend.rs +++ b/primitives/state-machine/src/proving_backend.rs @@ -113,8 +113,6 @@ impl<'a, S, H> ProvingBackendRecorder<'a, S, H> struct ProofRecorderInner { /// All the records that we have stored so far. records: HashMap>, - /// Is inner hash in proof. - flagged_inner_hash: bool, /// The encoded size of all recorded values. encoded_size: usize, } @@ -130,12 +128,9 @@ impl ProofRecorder { pub fn record(&self, key: Hash, mut val: Option<(DBValue, TrieMeta)>) { let mut inner = self.inner.write(); - let ProofRecorderInner { encoded_size, records, flagged_inner_hash } = &mut *inner; + let ProofRecorderInner { encoded_size, records } = &mut *inner; records.entry(key).or_insert_with(|| { if let Some(val) = val.as_mut() { - if val.1.recorded_do_value_hash { - *flagged_inner_hash = true; - } val.1.set_accessed_value(false); sp_trie::resolve_encoded_meta::(val); *encoded_size += sp_trie::estimate_entry_size(val, H::LENGTH); @@ -184,14 +179,9 @@ impl ProofRecorder { .records .iter() .filter_map(|(_k, v)| v.as_ref().map(|v| { - let mut meta = v.1.clone(); - if !inner.flagged_inner_hash { - // Remove the old hash meta. - meta.old_hash = false; - } < as sp_trie::TrieLayout>::MetaHasher as hash_db::MetaHasher - >::stored_value(v.0.as_slice(), meta) + >::stored_value(v.0.as_slice(), v.1.clone()) })) .collect(); diff --git a/primitives/state-machine/src/read_only.rs b/primitives/state-machine/src/read_only.rs index a54b6eaba6848..b2e32726fc802 100644 --- a/primitives/state-machine/src/read_only.rs +++ b/primitives/state-machine/src/read_only.rs @@ -203,8 +203,8 @@ impl<'a, H: Hasher, B: 'a + Backend> Externalities for ReadOnlyExternalities< unimplemented!("set_whitelist is not supported in ReadOnlyExternalities") } - fn flag_hash_value(&mut self) { - unimplemented!("flag_hash_value is not supported by ReadOnlyExternalities"); + fn alt_hashing(&mut self) { + unimplemented!("alt_hashing is not supported by ReadOnlyExternalities"); } } diff --git a/primitives/state-machine/src/testing.rs b/primitives/state-machine/src/testing.rs index 19006ccb52ac4..49e963c7f4a99 100644 --- a/primitives/state-machine/src/testing.rs +++ b/primitives/state-machine/src/testing.rs @@ -171,7 +171,7 @@ where )) } - self.backend.update(transaction, self.overlay.flag_hash_value()) + self.backend.update(transaction, self.overlay.alt_hashing()) } /// Commit all pending changes to the underlying backend. diff --git a/primitives/state-machine/src/trie_backend.rs b/primitives/state-machine/src/trie_backend.rs index fbc1f12fd2ae1..62666134445d8 100644 --- a/primitives/state-machine/src/trie_backend.rs +++ b/primitives/state-machine/src/trie_backend.rs @@ -379,17 +379,17 @@ pub mod tests { // a drop a insert of same hash: rc is 0 assert_eq!(test_trie(false).storage_root(iter::empty(), false).1.drain() .into_iter().filter(|v| (v.1).1 != 0).count(), 0); - // a drop a insert + // Unchanged assert_eq!(test_trie(false).storage_root(iter::empty(), true).1.drain() - .into_iter().filter(|v| (v.1).1 != 0).count(), 2); + .into_iter().filter(|v| (v.1).1 != 0).count(), 0); // a drop a insert of same hash: rc is 0 assert_eq!(test_trie(true).storage_root(iter::empty(), true).1.drain() .into_iter().filter(|v| (v.1).1 != 0).count(), 0); } #[test] - fn storage_root_flagged_is_not_empty() { - assert!(!test_trie(false).storage_root(iter::empty(), true).1.drain().is_empty()); + fn storage_root_flagged_is_empty() { + assert!(test_trie(false).storage_root(iter::empty(), true).1.drain().is_empty()); } #[test] diff --git a/primitives/tasks/src/async_externalities.rs b/primitives/tasks/src/async_externalities.rs index ff21e74889d06..6451caf11b7eb 100644 --- a/primitives/tasks/src/async_externalities.rs +++ b/primitives/tasks/src/async_externalities.rs @@ -190,8 +190,8 @@ impl Externalities for AsyncExternalities { unimplemented!("set_whitelist is not supported in AsyncExternalities") } - fn flag_hash_value(&mut self) { - unimplemented!("flag_hash_value is not supported in AsyncExternalities") + fn alt_hashing(&mut self) { + unimplemented!("alt_hashing is not supported in AsyncExternalities") } } diff --git a/primitives/trie/src/lib.rs b/primitives/trie/src/lib.rs index 07a398090a5ca..e8bd4e783fc3e 100644 --- a/primitives/trie/src/lib.rs +++ b/primitives/trie/src/lib.rs @@ -168,7 +168,7 @@ impl Meta for TrieMeta { value_plan: ValuePlan, ) { let (contain_hash, range) = match value_plan { - ValuePlan::Value(range) => (false, range), + ValuePlan::Value(range, with_len) => (false, with_len..range.end), ValuePlan::HashedValue(range, _size) => (true, range), ValuePlan::NoValue => return, }; @@ -186,7 +186,7 @@ impl Meta for TrieMeta { node_plan: &NodePlan, ) { let (contain_hash, range) = match node_plan.value_plan() { - Some(ValuePlan::Value(range)) => (false, range.clone()), + Some(ValuePlan::Value(range, with_len)) => (false, *with_len..range.end), Some(ValuePlan::HashedValue(range, _size)) => (true, range.clone()), Some(ValuePlan::NoValue) => return, None => return, @@ -324,7 +324,7 @@ impl MetaHasher for StateHasher stored.extend_from_slice(value); return stored; } - if meta.unused_value { + if meta.unused_value && meta.do_value_hash && !meta.switch_to_value_hash { if let Some(range) = meta.range.as_ref() { if range.end - range.start >= trie_constants::INNER_HASH_TRESHOLD { // Waring this assume that encoded value does not start by this, so it is tightly coupled @@ -893,7 +893,7 @@ fn inner_hashed_value(x: &[u8], range: Option<(usize, usize)>) -> Vec pub fn estimate_entry_size(entry: &(DBValue, TrieMeta), hash_len: usize) -> usize { use codec::Encode; let mut full_encoded = entry.0.encoded_size(); - if entry.1.unused_value { + if entry.1.unused_value && entry.1.do_value_hash { if let Some(range) = entry.1.range.as_ref() { let value_size = range.end - range.start; if range.end - range.start >= trie_constants::INNER_HASH_TRESHOLD { @@ -913,9 +913,7 @@ pub fn estimate_entry_size(entry: &(DBValue, TrieMeta), hash_len: usize) -> usiz /// if can hash value. pub fn resolve_encoded_meta(entry: &mut (DBValue, TrieMeta)) { use trie_db::NodeCodec; - if entry.1.do_value_hash { - let _ = as TrieLayout>::Codec::decode_plan(entry.0.as_slice(), &mut entry.1); - } + let _ = as TrieLayout>::Codec::decode_plan(entry.0.as_slice(), &mut entry.1); } /// Constants used into trie simplification codec. diff --git a/primitives/trie/src/node_codec.rs b/primitives/trie/src/node_codec.rs index d7849fcf52498..f4f79f8394dfc 100644 --- a/primitives/trie/src/node_codec.rs +++ b/primitives/trie/src/node_codec.rs @@ -127,8 +127,9 @@ impl NodeCodec { if alt_hashing && contains_hash { ValuePlan::HashedValue(input.take(H::LENGTH)?, 0) } else { + let with_len = input.offset; let count = >::decode(&mut input)?.0 as usize; - ValuePlan::Value(input.take(count)?) + ValuePlan::Value(input.take(count)?, with_len) } } else { ValuePlan::NoValue @@ -168,8 +169,9 @@ impl NodeCodec { let value = if alt_hashing && contains_hash { ValuePlan::HashedValue(input.take(H::LENGTH)?, 0) } else { + let with_len = input.offset; let count = >::decode(&mut input)?.0 as usize; - ValuePlan::Value(input.take(count)?) + ValuePlan::Value(input.take(count)?, with_len) }; Ok(NodePlan::Leaf { @@ -221,11 +223,12 @@ impl> NodeCodecT for NodeCodec { }; match value { Value::Value(value) => { + let with_len = output.len(); Compact(value.len() as u32).encode_to(&mut output); let start = output.len(); output.extend_from_slice(value); let end = output.len(); - meta.encoded_value_callback(ValuePlan::Value(start..end)); + meta.encoded_value_callback(ValuePlan::Value(start..end, with_len)); }, Value::HashedValue(hash, _size) => { debug_assert!(hash.len() == H::LENGTH); @@ -280,11 +283,12 @@ impl> NodeCodecT for NodeCodec { (0..BITMAP_LENGTH).for_each(|_|output.push(0)); match maybe_value { Value::Value(value) => { + let with_len = output.len(); Compact(value.len() as u32).encode_to(&mut output); let start = output.len(); output.extend_from_slice(value); let end = output.len(); - meta.encoded_value_callback(ValuePlan::Value(start..end)); + meta.encoded_value_callback(ValuePlan::Value(start..end, with_len)); }, Value::HashedValue(hash, _size) => { debug_assert!(hash.len() == H::LENGTH); diff --git a/primitives/trie/src/storage_proof.rs b/primitives/trie/src/storage_proof.rs index 77c1c0f3ad22a..51522a92c756e 100644 --- a/primitives/trie/src/storage_proof.rs +++ b/primitives/trie/src/storage_proof.rs @@ -111,7 +111,8 @@ impl From for crate::MemoryDB { // Using compact proof will work directly here (read trie structure and // work directly. for item in proof.trie_nodes.iter() { - // Note using `default()` as global meta helps looking fro root node. + // Note using `default()` to build proof is fine, do_value being in header + // and no switch needed. let layout_meta = Default::default(); let (encoded_node, mut meta) = < as TrieLayout>::MetaHasher as MetaHasher diff --git a/test-utils/client/src/lib.rs b/test-utils/client/src/lib.rs index c087994e6c4b5..467b802f65cef 100644 --- a/test-utils/client/src/lib.rs +++ b/test-utils/client/src/lib.rs @@ -211,7 +211,7 @@ impl TestClientBuilder Date: Wed, 9 Jun 2021 09:34:20 +0200 Subject: [PATCH 040/127] removing fields(broken) --- primitives/storage/src/lib.rs | 5 ++++- primitives/trie/src/lib.rs | 19 ++++++------------- primitives/trie/src/trie_stream.rs | 4 ---- 3 files changed, 10 insertions(+), 18 deletions(-) diff --git a/primitives/storage/src/lib.rs b/primitives/storage/src/lib.rs index 32645adecb535..ef1ba64f256d9 100644 --- a/primitives/storage/src/lib.rs +++ b/primitives/storage/src/lib.rs @@ -170,6 +170,10 @@ pub mod well_known_keys { /// Current extrinsic index (u32) is stored under this key. pub const EXTRINSIC_INDEX: &'static [u8] = b":extrinsic_index"; + /// Configuration for trie internal hashing of value is stored + /// under this key. + pub const TRIE_HASHING_CONFIG: &'static [u8] = b":trie_hashing_conf"; + /// Changes trie configuration is stored under this key. pub const CHANGES_TRIE_CONFIG: &'static [u8] = b":changes_trie"; @@ -196,7 +200,6 @@ pub mod well_known_keys { CHILD_STORAGE_KEY_PREFIX.starts_with(key) } } - } /// Information related to a child state. diff --git a/primitives/trie/src/lib.rs b/primitives/trie/src/lib.rs index e8bd4e783fc3e..2a811461ac38a 100644 --- a/primitives/trie/src/lib.rs +++ b/primitives/trie/src/lib.rs @@ -57,14 +57,9 @@ pub struct TrieMeta { /// Defined in the trie layout, when used with /// `TrieDbMut` it switch nodes to alternative hashing /// method by setting `do_value_hash` to true. - /// TODO may be useless (indicate that previous hash is - /// not using `do_value_hash`). + /// TODO consider defining it without do_value + /// and set do_value on encoding only. pub switch_to_value_hash: bool, - /// When `do_value_hash` is true, try to - /// store this behavior in top node - /// encoded (need to be part of state). - /// TODO remove - pub recorded_do_value_hash: bool, /// Does current encoded contains a hash instead of /// a value (information stored in meta for proofs). pub contain_hash: bool, @@ -79,9 +74,6 @@ pub struct TrieMeta { /// and reset on access explicitely: `HashDB::access_from`. /// TODO!! remove from meta: only use in proof recorder context. pub unused_value: bool, - /// Indicate that a node is using old hash scheme. - /// TODO remove - pub old_hash: bool, } impl Meta for TrieMeta { @@ -201,8 +193,11 @@ impl Meta for TrieMeta { } // TODO could be rename to get_state_meta + // the type of node depend on it. fn do_value_hash(&self) -> bool { - self.do_value_hash + self.do_value_hash/* && self.range.as_ref().map(|range| + range.end - range.start >= trie_constants::INNER_HASH_TRESHOLD + ).unwrap_or(false)*/ } } @@ -360,9 +355,7 @@ impl MetaHasher for StateHasher unused_value: contain_hash, contain_hash, do_value_hash: false, - recorded_do_value_hash: false, switch_to_value_hash: false, - old_hash: false, }; meta.set_global_meta(global_meta); (stored, meta) diff --git a/primitives/trie/src/trie_stream.rs b/primitives/trie/src/trie_stream.rs index 3206ff5729870..b3cd7d00c00e7 100644 --- a/primitives/trie/src/trie_stream.rs +++ b/primitives/trie/src/trie_stream.rs @@ -145,8 +145,6 @@ impl trie_root::TrieStream for TrieStream { range: range, unused_value: false, contain_hash: false, - old_hash: false, - recorded_do_value_hash: false, // No existing state, no need to use switch_to_value_hash switch_to_value_hash: false, do_value_hash: true, @@ -167,8 +165,6 @@ impl trie_root::TrieStream for TrieStream { range: range, unused_value: false, contain_hash: false, - old_hash: false, - recorded_do_value_hash: inner_value_hashing, switch_to_value_hash: false, do_value_hash: inner_value_hashing, }; From 2f8360f5203914b2ec2a5c70da4ca21cd1cb5b99 Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 9 Jun 2021 09:58:11 +0200 Subject: [PATCH 041/127] fix trie_stream to also include value length in inner hash. --- primitives/trie/src/lib.rs | 2 +- primitives/trie/src/trie_stream.rs | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/primitives/trie/src/lib.rs b/primitives/trie/src/lib.rs index 2a811461ac38a..57c585c787dce 100644 --- a/primitives/trie/src/lib.rs +++ b/primitives/trie/src/lib.rs @@ -990,8 +990,8 @@ mod tests { fn check_input(input: &Vec<(&[u8], &[u8])>) { // TODO remove this iter let layout = Layout::with_inner_hashing(); - check_iteration::(input, layout.clone()); check_equivalent::(input, layout.clone()); + check_iteration::(input, layout.clone()); let layout = Layout::default(); diff --git a/primitives/trie/src/trie_stream.rs b/primitives/trie/src/trie_stream.rs index b3cd7d00c00e7..ffa9f6c6c8885 100644 --- a/primitives/trie/src/trie_stream.rs +++ b/primitives/trie/src/trie_stream.rs @@ -92,9 +92,10 @@ impl trie_root::TrieStream for TrieStream { NodeKind::Leaf }; self.buffer.extend(fuse_nibbles_node(key, kind)); + let start = self.buffer.len(); Compact(value.len() as u32).encode_to(&mut self.buffer); - self.current_value_range = Some(self.buffer.len()..self.buffer.len() + value.len()); self.buffer.extend_from_slice(value); + self.current_value_range = Some(start..self.buffer.len()); } fn begin_branch( @@ -121,9 +122,10 @@ impl trie_root::TrieStream for TrieStream { self.buffer.extend(&branch_node(maybe_value.is_some(), has_children)); } if let Some(value) = maybe_value { + let start = self.buffer.len(); Compact(value.len() as u32).encode_to(&mut self.buffer); - self.current_value_range = Some(self.buffer.len()..self.buffer.len() + value.len()); self.buffer.extend_from_slice(value); + self.current_value_range = Some(start..self.buffer.len()); } } From 76b6d7a96b656f358c425587efd11a232048818e Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 9 Jun 2021 10:18:36 +0200 Subject: [PATCH 042/127] triedbmut only using alt type if inner hashing. --- primitives/trie/src/lib.rs | 6 +++--- primitives/trie/src/node_codec.rs | 16 ++++++++++++---- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/primitives/trie/src/lib.rs b/primitives/trie/src/lib.rs index 57c585c787dce..edc72443cf692 100644 --- a/primitives/trie/src/lib.rs +++ b/primitives/trie/src/lib.rs @@ -53,6 +53,7 @@ pub use hash_db::NoMeta; #[derive(Default, Clone, Debug)] pub struct TrieMeta { /// Range of encoded value or hashed value. + /// When encoded value, it includes the length of the value. pub range: Option>, /// Defined in the trie layout, when used with /// `TrieDbMut` it switch nodes to alternative hashing @@ -194,10 +195,9 @@ impl Meta for TrieMeta { // TODO could be rename to get_state_meta // the type of node depend on it. + // Note that it is after encoding state meta here!! fn do_value_hash(&self) -> bool { - self.do_value_hash/* && self.range.as_ref().map(|range| - range.end - range.start >= trie_constants::INNER_HASH_TRESHOLD - ).unwrap_or(false)*/ + self.do_value_hash } } diff --git a/primitives/trie/src/node_codec.rs b/primitives/trie/src/node_codec.rs index f4f79f8394dfc..e125a1d50ee11 100644 --- a/primitives/trie/src/node_codec.rs +++ b/primitives/trie/src/node_codec.rs @@ -216,7 +216,7 @@ impl> NodeCodecT for NodeCodec { } fn leaf_node(partial: Partial, value: Value, meta: &mut M) -> Vec { - let mut output = if meta.do_value_hash() { + let mut output = if meta.do_value_hash() && value_do_hash(&value) { partial_encode(partial, NodeKind::AltHashLeaf) } else { partial_encode(partial, NodeKind::Leaf) @@ -263,10 +263,10 @@ impl> NodeCodecT for NodeCodec { partial: impl Iterator, number_nibble: usize, children: impl Iterator::Out>>>>, - maybe_value: Value, + value: Value, meta: &mut M, ) -> Vec { - let mut output = match (&maybe_value, meta.do_value_hash()) { + let mut output = match (&value, meta.do_value_hash() && value_do_hash(&value)) { (&Value::NoValue, _) => { partial_from_iterator_encode(partial, number_nibble, NodeKind::BranchNoValue) }, @@ -281,7 +281,7 @@ impl> NodeCodecT for NodeCodec { let bitmap_index = output.len(); let mut bitmap: [u8; BITMAP_LENGTH] = [0; BITMAP_LENGTH]; (0..BITMAP_LENGTH).for_each(|_|output.push(0)); - match maybe_value { + match value { Value::Value(value) => { let with_len = output.len(); Compact(value.len() as u32).encode_to(&mut output); @@ -318,6 +318,14 @@ impl> NodeCodecT for NodeCodec { // utils +fn value_do_hash(val: &Value) -> bool { + if let Value::Value(val) = val { + val.encoded_size() >= trie_constants::INNER_HASH_TRESHOLD + } else { + false + } +} + /// Encode and allocate node type header (type and size), and partial value. /// It uses an iterator over encoded partial bytes as input. fn partial_from_iterator_encode>( From cfa0ecd87f079288749de72f289a2246aa8c5a84 Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 9 Jun 2021 10:42:20 +0200 Subject: [PATCH 043/127] trie_stream to also only use alt hashing type when actually alt hashing. --- primitives/trie/src/lib.rs | 4 +++- primitives/trie/src/node_codec.rs | 5 +++++ primitives/trie/src/trie_stream.rs | 20 ++++++++++++++++---- 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/primitives/trie/src/lib.rs b/primitives/trie/src/lib.rs index edc72443cf692..33e25c5baafb2 100644 --- a/primitives/trie/src/lib.rs +++ b/primitives/trie/src/lib.rs @@ -59,7 +59,7 @@ pub struct TrieMeta { /// `TrieDbMut` it switch nodes to alternative hashing /// method by setting `do_value_hash` to true. /// TODO consider defining it without do_value - /// and set do_value on encoding only. + /// and set do_value on encoding only. TODO try_do_value_hash pub switch_to_value_hash: bool, /// Does current encoded contains a hash instead of /// a value (information stored in meta for proofs). @@ -68,6 +68,7 @@ pub struct TrieMeta { /// This is read and written as a state meta of the node. /// TODO replace by TrieDbMut node variant /// TODO replace by Option being size treshold. + /// TODO apply_do_value_hash (and remove size testing) pub do_value_hash: bool, /// Record if a value was accessed, this is /// set as accessed by defalult, but can be @@ -920,6 +921,7 @@ mod trie_constants { /// TODO attaching to storage proof in a compatible way could be /// achieve by using a escaped header in first or last element of proof /// and write it after. + /// TODO 33 is not good switch to 32 + 1 + 1: 34 (avoid hashing stored hash for a 1 byte gain). pub const INNER_HASH_TRESHOLD: usize = 33; const FIRST_PREFIX: u8 = 0b_00 << 6; /// In proof this header is used when only hashed value is stored. diff --git a/primitives/trie/src/node_codec.rs b/primitives/trie/src/node_codec.rs index e125a1d50ee11..318f621383185 100644 --- a/primitives/trie/src/node_codec.rs +++ b/primitives/trie/src/node_codec.rs @@ -216,6 +216,11 @@ impl> NodeCodecT for NodeCodec { } fn leaf_node(partial: Partial, value: Value, meta: &mut M) -> Vec { + // Note that we use AltHash type only if inner hashing will occur, + // this way we allow changing hash threshold. + // With fix inner hashing alt hash can be use with all node, but + // that is not better (encoding can use an additional nibble byte + // sometime). let mut output = if meta.do_value_hash() && value_do_hash(&value) { partial_encode(partial, NodeKind::AltHashLeaf) } else { diff --git a/primitives/trie/src/trie_stream.rs b/primitives/trie/src/trie_stream.rs index ffa9f6c6c8885..a6d9b08786886 100644 --- a/primitives/trie/src/trie_stream.rs +++ b/primitives/trie/src/trie_stream.rs @@ -32,8 +32,13 @@ const BRANCH_NODE_WITH_VALUE: u8 = 255; #[derive(Default, Clone)] /// Codec-flavored TrieStream. pub struct TrieStream { + /// Current node buffer. buffer: Vec, + /// Global trie alt hashing activation. inner_value_hashing: bool, + /// For current node, do we use alt hashing. + apply_inner_hashing: bool, + /// Keep trace of position of encoded value. current_value_range: Option>, } @@ -77,6 +82,7 @@ impl trie_root::TrieStream for TrieStream { Self { buffer: Vec::new(), inner_value_hashing: meta, + apply_inner_hashing: false, current_value_range: None, } } @@ -86,7 +92,8 @@ impl trie_root::TrieStream for TrieStream { } fn append_leaf(&mut self, key: &[u8], value: &[u8]) { - let kind = if self.inner_value_hashing { + self.apply_inner_hashing = self.inner_value_hashing && value_do_hash(value); + let kind = if self.apply_inner_hashing { NodeKind::AltHashLeaf } else { NodeKind::Leaf @@ -105,8 +112,9 @@ impl trie_root::TrieStream for TrieStream { has_children: impl Iterator, ) { if let Some(partial) = maybe_partial { - if maybe_value.is_some() { - let kind = if self.inner_value_hashing { + if let Some(value) = maybe_value { + self.apply_inner_hashing = self.inner_value_hashing && value_do_hash(value); + let kind = if self.apply_inner_hashing { NodeKind::AltHashBranchWithValue } else { NodeKind::BranchWithValue @@ -134,7 +142,7 @@ impl trie_root::TrieStream for TrieStream { } fn append_substream(&mut self, other: Self) { - let inner_value_hashing = other.inner_value_hashing; + let inner_value_hashing = other.apply_inner_hashing; let range = other.current_value_range.clone(); let data = other.out(); match data.len() { @@ -202,3 +210,7 @@ fn branch_node_buffered(has_value: bool, has_children: I, output: &mut[u8]) output[0] = first; Bitmap::encode(has_children, &mut output[1..]); } + +fn value_do_hash(val: &[u8]) -> bool { + val.encoded_size() >= trie_constants::INNER_HASH_TRESHOLD +} From 60f70125eb002dd9aa37273f33e0ed7ecd90ed57 Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 9 Jun 2021 11:16:06 +0200 Subject: [PATCH 044/127] Refactor meta state, logic should work with change of trie treshold. --- primitives/trie/src/lib.rs | 87 +++++++++++------------------- primitives/trie/src/node_codec.rs | 10 ++-- primitives/trie/src/trie_stream.rs | 24 ++++----- 3 files changed, 49 insertions(+), 72 deletions(-) diff --git a/primitives/trie/src/lib.rs b/primitives/trie/src/lib.rs index 33e25c5baafb2..59d51b9c84910 100644 --- a/primitives/trie/src/lib.rs +++ b/primitives/trie/src/lib.rs @@ -58,18 +58,12 @@ pub struct TrieMeta { /// Defined in the trie layout, when used with /// `TrieDbMut` it switch nodes to alternative hashing /// method by setting `do_value_hash` to true. - /// TODO consider defining it without do_value - /// and set do_value on encoding only. TODO try_do_value_hash - pub switch_to_value_hash: bool, + pub try_inner_hashing: bool, /// Does current encoded contains a hash instead of /// a value (information stored in meta for proofs). pub contain_hash: bool, - /// Flag indicating if alternative value hash can run. - /// This is read and written as a state meta of the node. - /// TODO replace by TrieDbMut node variant - /// TODO replace by Option being size treshold. - /// TODO apply_do_value_hash (and remove size testing) - pub do_value_hash: bool, + /// Flag indicating alternative value hash will be use. + pub apply_inner_hashing: bool, /// Record if a value was accessed, this is /// set as accessed by defalult, but can be /// change on access explicitely: `HashDB::get_with_meta`. @@ -86,26 +80,21 @@ impl Meta for TrieMeta { type StateMeta = bool; fn set_state_meta(&mut self, state_meta: Self::StateMeta) { - if !self.do_value_hash && state_meta { - self.do_value_hash = true; - } + self.apply_inner_hashing = state_meta; } - // TODO remove upstream + // TODO rename upstream as read_global_meta fn extract_global_meta(&self) -> Self::GlobalMeta { - self.switch_to_value_hash || self.do_value_hash + self.try_inner_hashing } fn set_global_meta(&mut self, global_meta: Self::GlobalMeta) { - if !self.do_value_hash && global_meta { - self.switch_to_value_hash = true; - self.do_value_hash = true; - } + self.try_inner_hashing = global_meta; } // TODO remove upstream? fn has_state_meta(&self) -> bool { - self.do_value_hash && !self.switch_to_value_hash + self.apply_inner_hashing } // TODO consider removal upstream of this method (node type in codec) @@ -167,12 +156,10 @@ impl Meta for TrieMeta { ValuePlan::NoValue => return, }; + self.apply_inner_hashing = self.try_inner_hashing + && range.end - range.start >= trie_constants::INNER_HASH_TRESHOLD; self.range = Some(range); self.contain_hash = contain_hash; - if self.switch_to_value_hash { - // Switched value hashing. - self.switch_to_value_hash = false - } } fn decoded_callback( @@ -194,11 +181,9 @@ impl Meta for TrieMeta { self.contain_hash } - // TODO could be rename to get_state_meta - // the type of node depend on it. - // Note that it is after encoding state meta here!! + // TODO remove upstream fn do_value_hash(&self) -> bool { - self.do_value_hash + self.apply_inner_hashing } } @@ -292,13 +277,9 @@ impl MetaHasher for StateHasher fn hash(value: &[u8], meta: &Self::Meta) -> H::Out { match &meta { - TrieMeta { range: Some(range), contain_hash: false, do_value_hash: true, switch_to_value_hash: false, .. } => { - if range.end - range.start >= trie_constants::INNER_HASH_TRESHOLD { - let value = inner_hashed_value::(value, Some((range.start, range.end))); - H::hash(value.as_slice()) - } else { - H::hash(value) - } + TrieMeta { range: Some(range), contain_hash: false, apply_inner_hashing: true, .. } => { + let value = inner_hashed_value::(value, Some((range.start, range.end))); + H::hash(value.as_slice()) }, TrieMeta { range: Some(_range), contain_hash: true, .. } => { // value contains a hash of data (already inner_hashed_value). @@ -320,19 +301,17 @@ impl MetaHasher for StateHasher stored.extend_from_slice(value); return stored; } - if meta.unused_value && meta.do_value_hash && !meta.switch_to_value_hash { - if let Some(range) = meta.range.as_ref() { - if range.end - range.start >= trie_constants::INNER_HASH_TRESHOLD { - // Waring this assume that encoded value does not start by this, so it is tightly coupled - // with the header type of the codec: only for optimization. - stored.push(trie_constants::DEAD_HEADER_META_HASHED_VALUE); - let range = meta.range.as_ref().expect("Tested in condition"); - meta.contain_hash = true; // useless but could be with meta as &mut - // store hash instead of value. - let value = inner_hashed_value::(value, Some((range.start, range.end))); - stored.extend_from_slice(value.as_slice()); - return stored; - } + if meta.unused_value && meta.apply_inner_hashing { + if meta.range.is_some() { + // Waring this assume that encoded value does not start by this, so it is tightly coupled + // with the header type of the codec: only for optimization. + stored.push(trie_constants::DEAD_HEADER_META_HASHED_VALUE); + let range = meta.range.as_ref().expect("Tested in condition"); + meta.contain_hash = true; // useless but could be with meta as &mut + // store hash instead of value. + let value = inner_hashed_value::(value, Some((range.start, range.end))); + stored.extend_from_slice(value.as_slice()); + return stored; } } stored.extend_from_slice(value); @@ -355,8 +334,8 @@ impl MetaHasher for StateHasher range: None, unused_value: contain_hash, contain_hash, - do_value_hash: false, - switch_to_value_hash: false, + apply_inner_hashing: false, + try_inner_hashing: false, }; meta.set_global_meta(global_meta); (stored, meta) @@ -887,14 +866,12 @@ fn inner_hashed_value(x: &[u8], range: Option<(usize, usize)>) -> Vec pub fn estimate_entry_size(entry: &(DBValue, TrieMeta), hash_len: usize) -> usize { use codec::Encode; let mut full_encoded = entry.0.encoded_size(); - if entry.1.unused_value && entry.1.do_value_hash { + if entry.1.unused_value && entry.1.apply_inner_hashing { if let Some(range) = entry.1.range.as_ref() { let value_size = range.end - range.start; - if range.end - range.start >= trie_constants::INNER_HASH_TRESHOLD { - full_encoded -= value_size; - full_encoded += hash_len; - full_encoded += 1; - } + full_encoded -= value_size; + full_encoded += hash_len; + full_encoded += 1; } } diff --git a/primitives/trie/src/node_codec.rs b/primitives/trie/src/node_codec.rs index 318f621383185..61bcf789d8817 100644 --- a/primitives/trie/src/node_codec.rs +++ b/primitives/trie/src/node_codec.rs @@ -183,7 +183,11 @@ impl NodeCodec { } } -impl> NodeCodecT for NodeCodec { +impl NodeCodecT for NodeCodec + where + H: Hasher, + M: Meta, +{ type Error = Error; type HashOut = H::Out; @@ -221,7 +225,7 @@ impl> NodeCodecT for NodeCodec { // With fix inner hashing alt hash can be use with all node, but // that is not better (encoding can use an additional nibble byte // sometime). - let mut output = if meta.do_value_hash() && value_do_hash(&value) { + let mut output = if meta.extract_global_meta() && value_do_hash(&value) { partial_encode(partial, NodeKind::AltHashLeaf) } else { partial_encode(partial, NodeKind::Leaf) @@ -271,7 +275,7 @@ impl> NodeCodecT for NodeCodec { value: Value, meta: &mut M, ) -> Vec { - let mut output = match (&value, meta.do_value_hash() && value_do_hash(&value)) { + let mut output = match (&value, meta.extract_global_meta() && value_do_hash(&value)) { (&Value::NoValue, _) => { partial_from_iterator_encode(partial, number_nibble, NodeKind::BranchNoValue) }, diff --git a/primitives/trie/src/trie_stream.rs b/primitives/trie/src/trie_stream.rs index a6d9b08786886..4e383c9d5763c 100644 --- a/primitives/trie/src/trie_stream.rs +++ b/primitives/trie/src/trie_stream.rs @@ -142,22 +142,21 @@ impl trie_root::TrieStream for TrieStream { } fn append_substream(&mut self, other: Self) { - let inner_value_hashing = other.apply_inner_hashing; + let apply_inner_hashing = other.apply_inner_hashing; let range = other.current_value_range.clone(); let data = other.out(); match data.len() { 0..=31 => data.encode_to(&mut self.buffer), _ => { - if inner_value_hashing - && range.as_ref().map(|r| r.end - r.start >= trie_constants::INNER_HASH_TRESHOLD) - .unwrap_or_default() { + if apply_inner_hashing { let meta = TrieMeta { range: range, unused_value: false, contain_hash: false, - // No existing state, no need to use switch_to_value_hash - switch_to_value_hash: false, - do_value_hash: true, + // Using `inner_value_hashing` instead to check this. + // And unused in hasher. + try_inner_hashing: false, + apply_inner_hashing: true, }; >>::hash(&data, &meta).as_ref().encode_to(&mut self.buffer); } else { @@ -168,21 +167,18 @@ impl trie_root::TrieStream for TrieStream { } fn hash_root(self) -> H::Out { - let inner_value_hashing = self.inner_value_hashing; + let apply_inner_hashing = self.apply_inner_hashing; let range = self.current_value_range; let data = self.buffer; let meta = TrieMeta { range: range, unused_value: false, contain_hash: false, - switch_to_value_hash: false, - do_value_hash: inner_value_hashing, + try_inner_hashing: false, + apply_inner_hashing: true, }; - if inner_value_hashing - && meta.range.as_ref().map(|r| r.end - r.start >= trie_constants::INNER_HASH_TRESHOLD) - .unwrap_or_default() { - + if apply_inner_hashing { >>::hash(&data, &meta) } else { H::hash(&data) From 40ec2f715c7d332485b2f25a8592190df5548843 Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 9 Jun 2021 12:05:13 +0200 Subject: [PATCH 045/127] Remove NoMeta variant. --- Cargo.lock | 16 +++--- client/api/src/cht.rs | 2 +- client/db/src/changes_tries_storage.rs | 2 +- client/db/src/lib.rs | 10 ++-- client/light/src/fetcher.rs | 2 +- frame/session/src/historical/mod.rs | 2 +- primitives/state-machine/src/backend.rs | 7 +-- .../state-machine/src/changes_trie/mod.rs | 2 +- .../state-machine/src/changes_trie/prune.rs | 4 +- .../state-machine/src/changes_trie/storage.rs | 4 +- primitives/state-machine/src/lib.rs | 4 +- primitives/trie/src/lib.rs | 56 ++----------------- primitives/trie/src/storage_proof.rs | 20 +++---- 13 files changed, 41 insertions(+), 90 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 771af600156c7..299383a5775b6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2342,7 +2342,7 @@ dependencies = [ [[package]] name = "hash-db" version = "0.15.2" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#0f04db6e1d4781c41684df1539e3fd52e9aa85ae" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#22910ad4af83e00b549cee918226efa940fdd3cf" [[package]] name = "hash256-std-hasher" @@ -2356,7 +2356,7 @@ dependencies = [ [[package]] name = "hash256-std-hasher" version = "0.15.2" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#0f04db6e1d4781c41684df1539e3fd52e9aa85ae" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#22910ad4af83e00b549cee918226efa940fdd3cf" dependencies = [ "crunchy", ] @@ -3001,7 +3001,7 @@ dependencies = [ [[package]] name = "keccak-hasher" version = "0.15.3" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#0f04db6e1d4781c41684df1539e3fd52e9aa85ae" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#22910ad4af83e00b549cee918226efa940fdd3cf" dependencies = [ "hash-db", "hash256-std-hasher 0.15.2 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple4)", @@ -3793,7 +3793,7 @@ dependencies = [ [[package]] name = "memory-db" version = "0.26.0" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#0f04db6e1d4781c41684df1539e3fd52e9aa85ae" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#22910ad4af83e00b549cee918226efa940fdd3cf" dependencies = [ "hash-db", "hashbrown", @@ -10375,7 +10375,7 @@ checksum = "a7f741b240f1a48843f9b8e0444fb55fb2a4ff67293b50a9179dfd5ea67f8d41" [[package]] name = "trie-bench" version = "0.27.0" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#0f04db6e1d4781c41684df1539e3fd52e9aa85ae" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#22910ad4af83e00b549cee918226efa940fdd3cf" dependencies = [ "criterion", "hash-db", @@ -10390,7 +10390,7 @@ dependencies = [ [[package]] name = "trie-db" version = "0.22.5" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#0f04db6e1d4781c41684df1539e3fd52e9aa85ae" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#22910ad4af83e00b549cee918226efa940fdd3cf" dependencies = [ "hash-db", "hashbrown", @@ -10402,7 +10402,7 @@ dependencies = [ [[package]] name = "trie-root" version = "0.16.0" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#0f04db6e1d4781c41684df1539e3fd52e9aa85ae" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#22910ad4af83e00b549cee918226efa940fdd3cf" dependencies = [ "hash-db", ] @@ -10420,7 +10420,7 @@ dependencies = [ [[package]] name = "trie-standardmap" version = "0.15.2" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#0f04db6e1d4781c41684df1539e3fd52e9aa85ae" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#22910ad4af83e00b549cee918226efa940fdd3cf" dependencies = [ "hash-db", "keccak-hasher 0.15.3 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple4)", diff --git a/client/api/src/cht.rs b/client/api/src/cht.rs index 5c8dadcd6825f..9739e71a9ab2c 100644 --- a/client/api/src/cht.rs +++ b/client/api/src/cht.rs @@ -32,7 +32,7 @@ use sp_trie; use sp_core::{H256, convert_hash}; use sp_runtime::traits::{Header as HeaderT, AtLeast32Bit, Zero, One}; use sp_state_machine::{ - MemoryDBNoMeta as MemoryDB, TrieBackend, Backend as StateBackend, StorageProof, InMemoryBackend, + MemoryDB, TrieBackend, Backend as StateBackend, StorageProof, InMemoryBackend, prove_read_on_trie_backend, read_proof_check, read_proof_check_on_proving_backend_generic as read_proof_check_on_proving_backend, }; diff --git a/client/db/src/changes_tries_storage.rs b/client/db/src/changes_tries_storage.rs index 3f7e19cacfc24..860ca41730518 100644 --- a/client/db/src/changes_tries_storage.rs +++ b/client/db/src/changes_tries_storage.rs @@ -24,7 +24,7 @@ use hash_db::Prefix; use codec::{Decode, Encode}; use parking_lot::RwLock; use sp_blockchain::{Error as ClientError, Result as ClientResult}; -use sp_trie::MemoryDBNoMeta as MemoryDB; +use sp_trie::MemoryDB; use sc_client_api::backend::PrunableStateChangesTrieStorage; use sp_blockchain::{well_known_cache_keys, Cache as BlockchainCache, HeaderMetadataCache}; use sp_core::{ChangesTrieConfiguration, ChangesTrieConfigurationRange, convert_hash}; diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index 1eefca0c74ca4..9e59513f6724a 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -65,7 +65,7 @@ use sp_blockchain::{ }; use codec::{Decode, Encode}; use hash_db::Prefix; -use sp_trie::{MemoryDB, MemoryDBNoMeta, PrefixedMemoryDB, prefixed_key, StateHasher, +use sp_trie::{MemoryDB, PrefixedMemoryDB, prefixed_key, StateHasher, TrieMeta, MetaHasher}; use sp_database::Transaction; use sp_core::{Hasher, ChangesTrieConfiguration}; @@ -692,7 +692,7 @@ pub struct BlockImportOperation { storage_updates: StorageCollection, child_storage_updates: ChildStorageCollection, offchain_storage_updates: OffchainChangesCollection, - changes_trie_updates: MemoryDBNoMeta>, + changes_trie_updates: MemoryDB>, changes_trie_build_cache_update: Option>>, changes_trie_config_update: Option>, pending_block: Option>, @@ -1733,7 +1733,7 @@ impl sc_client_api::backend::Backend for Backend { child_storage_updates: Default::default(), offchain_storage_updates: Default::default(), changes_trie_config_update: None, - changes_trie_updates: MemoryDBNoMeta::default(), + changes_trie_updates: MemoryDB::default(), changes_trie_build_cache_update: None, aux_ops: Vec::new(), finalized_blocks: Vec::new(), @@ -2160,9 +2160,9 @@ pub(crate) mod tests { pub(crate) type Block = RawBlock>; - pub fn prepare_changes(changes: Vec<(Vec, Vec)>) -> (H256, MemoryDBNoMeta) { + pub fn prepare_changes(changes: Vec<(Vec, Vec)>) -> (H256, MemoryDB) { let mut changes_root = H256::default(); - let mut changes_trie_update = MemoryDBNoMeta::::default(); + let mut changes_trie_update = MemoryDB::::default(); { let mut trie = TrieDBMut::::new( &mut changes_trie_update, diff --git a/client/light/src/fetcher.rs b/client/light/src/fetcher.rs index 117c2d6970bd9..ada0f6695038e 100644 --- a/client/light/src/fetcher.rs +++ b/client/light/src/fetcher.rs @@ -161,7 +161,7 @@ impl> LightDataChecker { H::Out: Ord + codec::Codec, { // all the checks are sharing the same storage - let storage: sp_state_machine::MemoryDBNoMeta = remote_roots_proof.into(); + let storage: sp_state_machine::MemoryDB = remote_roots_proof.into_memory_db_no_meta(); // remote_roots.keys() are sorted => we can use this to group changes tries roots // that are belongs to the same CHT diff --git a/frame/session/src/historical/mod.rs b/frame/session/src/historical/mod.rs index 0564d159e1e3f..8902ebe551f6c 100644 --- a/frame/session/src/historical/mod.rs +++ b/frame/session/src/historical/mod.rs @@ -35,7 +35,7 @@ use frame_support::{ decl_module, decl_storage, Parameter, print, traits::{ValidatorSet, ValidatorSetWithIdentification}, }; -use sp_trie::{MemoryDBNoMeta as MemoryDB, Trie, TrieMut, Recorder, EMPTY_PREFIX}; +use sp_trie::{MemoryDB, Trie, TrieMut, Recorder, EMPTY_PREFIX}; use sp_trie::trie_types::{TrieDBMut, TrieDB}; use super::{SessionIndex, Module as SessionModule}; diff --git a/primitives/state-machine/src/backend.rs b/primitives/state-machine/src/backend.rs index aaa6bbf16f8d7..fa44acf9dde8c 100644 --- a/primitives/state-machine/src/backend.rs +++ b/primitives/state-machine/src/backend.rs @@ -295,16 +295,16 @@ impl Consolidate for sp_trie::GenericMemoryDB /// Insert input pairs into memory db. #[cfg(test)] -pub(crate) fn insert_into_memory_db_no_meta(mdb: &mut sp_trie::MemoryDBNoMeta, input: I) -> Option +pub(crate) fn insert_into_memory_db(mdb: &mut sp_trie::MemoryDB, input: I) -> Option where H: Hasher, I: IntoIterator, { - use sp_trie::{TrieMut, trie_types::TrieDBMutNoMeta}; + use sp_trie::{TrieMut, trie_types::TrieDBMut}; let mut root = ::Out::default(); { - let mut trie = TrieDBMutNoMeta::::new(mdb, &mut root); + let mut trie = TrieDBMut::::new(mdb, &mut root); for (key, value) in input { if let Err(e) = trie.insert(&key, &value) { log::warn!(target: "trie", "Failed to write to trie: {}", e); @@ -316,7 +316,6 @@ pub(crate) fn insert_into_memory_db_no_meta(mdb: &mut sp_trie::MemoryDBNoM Some(root) } - /// Wrapper to create a [`RuntimeCode`] from a type that implements [`Backend`]. #[cfg(feature = "std")] pub struct BackendRuntimeCode<'a, B, H> { diff --git a/primitives/state-machine/src/changes_trie/mod.rs b/primitives/state-machine/src/changes_trie/mod.rs index 404353fc308b4..f2bbf371fb50e 100644 --- a/primitives/state-machine/src/changes_trie/mod.rs +++ b/primitives/state-machine/src/changes_trie/mod.rs @@ -73,7 +73,7 @@ use num_traits::{One, Zero}; use codec::{Decode, Encode}; use sp_core; use sp_core::storage::PrefixedStorageKey; -use sp_trie::{MemoryDBNoMeta as MemoryDB, DBValue, TrieMut}; +use sp_trie::{MemoryDB, DBValue, TrieMut}; use sp_trie::trie_types::TrieDBMut; use crate::{ StorageKey, diff --git a/primitives/state-machine/src/changes_trie/prune.rs b/primitives/state-machine/src/changes_trie/prune.rs index 987fb4ff63d08..6f00e9b6a8e0d 100644 --- a/primitives/state-machine/src/changes_trie/prune.rs +++ b/primitives/state-machine/src/changes_trie/prune.rs @@ -114,9 +114,9 @@ fn prune_trie( #[cfg(test)] mod tests { use std::collections::HashSet; - use sp_trie::MemoryDBNoMeta as MemoryDB; + use sp_trie::MemoryDB; use sp_core::H256; - use crate::backend::insert_into_memory_db_no_meta as insert_into_memory_db; + use crate::backend::insert_into_memory_db; use crate::changes_trie::storage::InMemoryStorage; use codec::Encode; use sp_runtime::traits::BlakeTwo256; diff --git a/primitives/state-machine/src/changes_trie/storage.rs b/primitives/state-machine/src/changes_trie/storage.rs index 5414d3f8c2f6a..115cc5461e1a9 100644 --- a/primitives/state-machine/src/changes_trie/storage.rs +++ b/primitives/state-machine/src/changes_trie/storage.rs @@ -21,7 +21,7 @@ use std::collections::{BTreeMap, HashSet, HashMap}; use hash_db::{Hasher, Prefix, EMPTY_PREFIX}; use sp_core::storage::PrefixedStorageKey; use sp_trie::DBValue; -use sp_trie::MemoryDBNoMeta as MemoryDB; +use sp_trie::MemoryDB; use parking_lot::RwLock; use crate::{ StorageKey, @@ -30,7 +30,7 @@ use crate::{ }; #[cfg(test)] -use crate::backend::insert_into_memory_db_no_meta as insert_into_memory_db; +use crate::backend::insert_into_memory_db; #[cfg(test)] use crate::changes_trie::input::{InputPair, ChildIndex}; diff --git a/primitives/state-machine/src/lib.rs b/primitives/state-machine/src/lib.rs index 19ab6b994d63c..d7d6cbc1da731 100644 --- a/primitives/state-machine/src/lib.rs +++ b/primitives/state-machine/src/lib.rs @@ -144,7 +144,7 @@ mod changes_trie { #[cfg(feature = "std")] mod std_reexport { pub use sp_trie::{trie_types::{Layout, TrieDBMut}, StorageProof, TrieMut, - DBValue, MemoryDB, MemoryDBNoMeta}; + DBValue, MemoryDB}; pub use crate::testing::TestExternalities; pub use crate::basic::BasicExternalities; pub use crate::read_only::{ReadOnlyExternalities, InspectState}; @@ -195,7 +195,7 @@ mod execution { /// Type of changes trie transaction. pub type ChangesTrieTransaction = ( - MemoryDBNoMeta, + MemoryDB, ChangesTrieCacheAction<::Out, N>, ); diff --git a/primitives/trie/src/lib.rs b/primitives/trie/src/lib.rs index 59d51b9c84910..de5d751aba038 100644 --- a/primitives/trie/src/lib.rs +++ b/primitives/trie/src/lib.rs @@ -350,41 +350,6 @@ impl MetaHasher for StateHasher } } -/// Reimplement `NoMeta` `MetaHasher` with -/// additional constraint. -/// TODO remove the MetaHasher is ignored -/// when no node have do_value_hash or layout defines it. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub struct NoMetaHasher; - -impl MetaHasher for NoMetaHasher - where - H: Hasher, -{ - type Meta = TrieMeta; - type GlobalMeta = bool; - - fn hash(value: &[u8], _meta: &Self::Meta) -> H::Out { - H::hash(value) - } - - fn stored_value(value: &[u8], _meta: Self::Meta) -> DBValue { - value.to_vec() - } - - fn stored_value_owned(value: DBValue, _meta: Self::Meta) -> DBValue { - value - } - - fn extract_value(stored: &[u8], _meta: Self::GlobalMeta) -> (&[u8], Self::Meta) { - (stored, Default::default()) - } - - fn extract_value_owned(stored: DBValue, _meta: Self::GlobalMeta) -> (DBValue, Self::Meta) { - (stored, Default::default()) - } -} - impl TrieConfiguration for Layout where H: Hasher, @@ -436,17 +401,6 @@ pub type PrefixedMemoryDB = memory_db::MemoryDB< pub type MemoryDB = memory_db::MemoryDB< H, memory_db::HashKey, trie_db::DBValue, StateHasher, MemTracker, >; -/// Reexport from `hash_db`, with genericity set for `Hasher` trait. -/// This uses a noops `KeyFunction` (key addressing must be hashed or using -/// an encoding scheme that avoid key conflict). -pub type MemoryDBNoMeta = memory_db::MemoryDB< - H, memory_db::HashKey, trie_db::DBValue, NoMetaHasher, MemTracker, ->; -/// MemoryDB with specific meta hasher. -pub type MemoryDBMeta = memory_db::MemoryDB< - H, memory_db::HashKey, trie_db::DBValue, M, MemTracker, ->; - /// Reexport from `hash_db`, with genericity set for `Hasher` trait. pub type GenericMemoryDB = memory_db::MemoryDB< H, KF, trie_db::DBValue, MH, MemTracker, @@ -465,16 +419,10 @@ pub type TrieHash = <::Hash as Hasher>::Out; pub mod trie_types { /// State layout. pub type Layout = super::Layout; - /// Old state layout definition, do not use meta, do not - /// do internal value hashing. - pub type LayoutNoMeta = super::Layout; /// Persistent trie database read-access interface for the a given hasher. pub type TrieDB<'a, H> = super::TrieDB<'a, Layout>; /// Persistent trie database write-access interface for the a given hasher. pub type TrieDBMut<'a, H> = super::TrieDBMut<'a, Layout>; - /// Persistent trie database write-access interface for the a given hasher, - /// old layout. - pub type TrieDBMutNoMeta<'a, H> = super::TrieDBMut<'a, LayoutNoMeta>; /// Querying interface, as in `trie_db` but less generic. pub type Lookup<'a, H, Q> = trie_db::Lookup<'a, Layout, Q>; /// As in `trie_db`, but less generic, error type for the crate. @@ -924,6 +872,10 @@ mod tests { type Layout = super::trie_types::Layout; + type MemoryDBMeta = memory_db::MemoryDB< + H, memory_db::HashKey, trie_db::DBValue, M, MemTracker, + >; + fn hashed_null_node() -> TrieHash { >::hashed_null_node() } diff --git a/primitives/trie/src/storage_proof.rs b/primitives/trie/src/storage_proof.rs index 51522a92c756e..a1b634df90b62 100644 --- a/primitives/trie/src/storage_proof.rs +++ b/primitives/trie/src/storage_proof.rs @@ -63,6 +63,16 @@ impl StorageProof { self.into() } + /// Creates a `MemoryDB` from `Self`. In case we do not need + /// to check meta (using alt hashing will always be disabled). + pub fn into_memory_db_no_meta(self) -> crate::MemoryDB { + let mut db = crate::MemoryDB::default(); + for item in self.iter_nodes() { + db.insert(crate::EMPTY_PREFIX, &item); + } + db + } + /// Merges multiple storage proofs covering potentially different sets of keys into one proof /// covering all keys. The merged proof output may be smaller than the aggregate size of the input /// proofs due to deduplication of trie nodes. @@ -124,13 +134,3 @@ impl From for crate::MemoryDB { db } } - -impl From for crate::MemoryDBNoMeta { - fn from(proof: StorageProof) -> Self { - let mut db = crate::MemoryDBNoMeta::default(); - for item in proof.iter_nodes() { - db.insert(crate::EMPTY_PREFIX, &item); - } - db - } -} From 204f51f5ca8877da052a40923d265c323597d50b Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 9 Jun 2021 22:06:11 +0200 Subject: [PATCH 046/127] Remove state_hashed trigger specific functions. --- Cargo.lock | 1 + client/api/src/backend.rs | 4 -- client/api/src/cht.rs | 2 +- client/api/src/in_mem.rs | 1 - client/block-builder/src/lib.rs | 2 +- client/chain-spec/src/chain_spec.rs | 12 ++--- client/db/src/bench.rs | 6 --- client/db/src/lib.rs | 20 ++++---- client/db/src/storage_cache.rs | 11 +--- client/executor/runtime-test/Cargo.toml | 1 + client/executor/runtime-test/src/lib.rs | 5 +- client/executor/src/integration_tests/mod.rs | 19 ++++--- client/light/src/backend.rs | 9 +--- client/network/test/src/lib.rs | 4 +- .../service/src/chain_ops/export_raw_state.rs | 3 +- client/service/src/client/client.rs | 4 -- client/service/test/src/client/mod.rs | 24 +++++---- frame/support/test/tests/instance.rs | 8 +-- frame/system/src/lib.rs | 9 ++-- primitives/externalities/src/lib.rs | 29 +++++++++-- primitives/io/src/lib.rs | 17 +++---- primitives/state-machine/src/backend.rs | 10 ++-- primitives/state-machine/src/basic.rs | 16 ++---- primitives/state-machine/src/ext.rs | 16 ++---- .../state-machine/src/in_memory_backend.rs | 16 ++---- .../src/overlayed_changes/mod.rs | 14 +----- .../state-machine/src/proving_backend.rs | 50 +++++++++++++------ primitives/state-machine/src/read_only.rs | 4 -- primitives/state-machine/src/testing.rs | 11 ++-- primitives/state-machine/src/trie_backend.rs | 12 +++-- .../state-machine/src/trie_backend_essence.rs | 6 --- primitives/storage/src/lib.rs | 40 ++++++++++++++- primitives/tasks/src/async_externalities.rs | 4 -- primitives/trie/src/lib.rs | 26 +--------- test-utils/client/src/lib.rs | 4 +- test-utils/runtime/src/genesismap.rs | 4 +- test-utils/runtime/src/system.rs | 8 ++- 37 files changed, 215 insertions(+), 217 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 299383a5775b6..a6721d91dd025 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7859,6 +7859,7 @@ dependencies = [ "sp-runtime", "sp-sandbox", "sp-std", + "sp-storage", "sp-tasks", "substrate-wasm-builder", ] diff --git a/client/api/src/backend.rs b/client/api/src/backend.rs index bd5e15c0222fd..09e9e0cb2e173 100644 --- a/client/api/src/backend.rs +++ b/client/api/src/backend.rs @@ -385,10 +385,6 @@ pub trait StorageProvider> { storage_key: Option<&PrefixedStorageKey>, key: &StorageKey ) -> sp_blockchain::Result, u32)>>; - - /// Returns true when state allow hashing value and therefore - /// removing unaccess value from proofs. - fn state_hashed_value(&self, id: &BlockId) -> sp_blockchain::Result; } /// Client backend. diff --git a/client/api/src/cht.rs b/client/api/src/cht.rs index 9739e71a9ab2c..d0ad9facd73c2 100644 --- a/client/api/src/cht.rs +++ b/client/api/src/cht.rs @@ -117,7 +117,7 @@ pub fn build_proof( .into_iter() .map(|(k, v)| (k, Some(v))) .collect::>(); - let mut storage = InMemoryBackend::::default().update(vec![(None, transaction)], false); + let mut storage = InMemoryBackend::::default().update(vec![(None, transaction)]); let trie_storage = storage.as_trie_backend() .expect("InMemoryState::as_trie_backend always returns Some; qed"); prove_read_on_trie_backend( diff --git a/client/api/src/in_mem.rs b/client/api/src/in_mem.rs index 3030784b3c320..d756e1cc0bbc4 100644 --- a/client/api/src/in_mem.rs +++ b/client/api/src/in_mem.rs @@ -576,7 +576,6 @@ impl backend::BlockImportOperation for BlockImportOperatio let (root, transaction) = self.old_state.full_storage_root( storage.top.iter().map(|(k, v)| (k.as_ref(), Some(v.as_ref()))), child_delta, - storage.alt_hashing, ); self.new_state = Some(transaction); diff --git a/client/block-builder/src/lib.rs b/client/block-builder/src/lib.rs index 7d391f8fb85b3..c264fc10e6a35 100644 --- a/client/block-builder/src/lib.rs +++ b/client/block-builder/src/lib.rs @@ -301,7 +301,7 @@ mod tests { #[test] fn block_building_storage_proof_does_not_include_runtime_by_default() { - let builder = substrate_test_runtime_client::TestClientBuilder::new(); + let builder = substrate_test_runtime_client::TestClientBuilder::new().state_hashed_value(); let backend = builder.backend(); let client = builder.build(); diff --git a/client/chain-spec/src/chain_spec.rs b/client/chain-spec/src/chain_spec.rs index 51c3a8c677645..59b55707e182b 100644 --- a/client/chain-spec/src/chain_spec.rs +++ b/client/chain-spec/src/chain_spec.rs @@ -87,9 +87,7 @@ impl GenesisSource { ) .collect(); - let alt_hashing = storage.alt_hashing; - - Ok(Genesis::Raw(RawGenesis { top, children_default, alt_hashing })) + Ok(Genesis::Raw(RawGenesis { top, children_default })) }, } } @@ -99,7 +97,7 @@ impl BuildStorage for ChainSpec { fn build_storage(&self) -> Result { match self.genesis.resolve()? { Genesis::Runtime(gc) => gc.build_storage(), - Genesis::Raw(RawGenesis { top: map, children_default: children_map, alt_hashing }) => Ok(Storage { + Genesis::Raw(RawGenesis { top: map, children_default: children_map }) => Ok(Storage { top: map.into_iter().map(|(k, v)| (k.0, v.0)).collect(), children_default: children_map.into_iter().map(|(storage_key, child_content)| { let child_info = ChildInfo::new_default(storage_key.0.as_slice()); @@ -111,7 +109,6 @@ impl BuildStorage for ChainSpec { }, ) }).collect(), - alt_hashing, }), } } @@ -133,8 +130,6 @@ pub type GenesisStorage = HashMap; pub struct RawGenesis { pub top: GenesisStorage, pub children_default: HashMap, - #[serde(default)] - pub alt_hashing: bool, } #[derive(Serialize, Deserialize)] @@ -328,9 +323,8 @@ impl ChainSpec { .collect(), )) .collect(); - let alt_hashing = storage.alt_hashing; - Genesis::Raw(RawGenesis { top, children_default, alt_hashing }) + Genesis::Raw(RawGenesis { top, children_default }) }, (_, genesis) => genesis, }; diff --git a/client/db/src/bench.rs b/client/db/src/bench.rs index 409897b831440..312f576052e87 100644 --- a/client/db/src/bench.rs +++ b/client/db/src/bench.rs @@ -151,7 +151,6 @@ impl BenchmarkingState { state.add_whitelist_to_tracker(); state.reopen()?; - let flagged = genesis.alt_hashing; let child_delta = genesis.children_default.iter().map(|(_storage_key, child_content)| ( &child_content.child_info, child_content.data.iter().map(|(k, v)| (k.as_ref(), Some(v.as_ref()))), @@ -159,7 +158,6 @@ impl BenchmarkingState { let (root, transaction): (B::Hash, _) = state.state.borrow_mut().as_mut().unwrap().full_storage_root( genesis.top.iter().map(|(k, v)| (k.as_ref(), Some(v.as_ref()))), child_delta, - flagged, ); state.genesis = transaction.clone().drain(); state.genesis_root = root.clone(); @@ -532,10 +530,6 @@ impl StateBackend> for BenchmarkingState { fn proof_size(&self) -> Option { self.proof_recorder.as_ref().map(|recorder| recorder.estimate_encoded_size() as u32) } - - fn state_hashed_value(&self) -> bool { - self.state.borrow().as_ref().map_or(Default::default(), |s| s.state_hashed_value()) - } } impl std::fmt::Debug for BenchmarkingState { diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index 9e59513f6724a..dc0d5465ecaf7 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -270,10 +270,6 @@ impl StateBackend> for RefTrackingState { fn usage_info(&self) -> StateUsageInfo { self.state.usage_info() } - - fn state_hashed_value(&self) -> bool { - self.state.state_hashed_value() - } } /// Database settings. @@ -782,7 +778,6 @@ impl sc_client_api::backend::BlockImportOperation for Bloc )); let mut changes_trie_config: Option = None; - let flag = storage.alt_hashing; let (root, transaction) = self.old_state.full_storage_root( storage.top.iter().map(|(k, v)| { if &k[..] == well_known_keys::CHANGES_TRIE_CONFIG { @@ -794,7 +789,6 @@ impl sc_client_api::backend::BlockImportOperation for Bloc (&k[..], Some(&v[..])) }), child_delta, - flag, ); self.db_updates = transaction; @@ -2304,11 +2298,20 @@ pub(crate) mod tests { extrinsics_root: Default::default(), }; - let storage = vec![ + let mut storage = vec![ (vec![1, 3, 5], vec![2, 4, 6]), (vec![1, 2, 3], vec![9, 9, 9]), ]; + if alt_hashing { + storage.push(( + sp_core::storage::well_known_keys::TRIE_HASHING_CONFIG.to_vec(), + sp_core::storage::trie_threshold_encode( + sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD, + ), + )); + } + header.state_root = op.old_state.storage_root(storage .iter() .map(|(x, y)| (&x[..], Some(&y[..]))), @@ -2319,7 +2322,6 @@ pub(crate) mod tests { op.reset_storage(Storage { top: storage.into_iter().collect(), children_default: Default::default(), - alt_hashing: alt_hashing, }).unwrap(); op.set_block_data( header.clone(), @@ -2405,7 +2407,6 @@ pub(crate) mod tests { op.reset_storage(Storage { top: Default::default(), children_default: Default::default(), - alt_hashing: alt_hashing, }).unwrap(); key = op.db_updates.insert(EMPTY_PREFIX, b"hello"); @@ -2858,7 +2859,6 @@ pub(crate) mod tests { op.reset_storage(Storage { top: storage.into_iter().collect(), children_default: Default::default(), - alt_hashing: alt_hashing, }).unwrap(); op.set_block_data( header.clone(), diff --git a/client/db/src/storage_cache.rs b/client/db/src/storage_cache.rs index 946f576142745..29dceb6b80828 100644 --- a/client/db/src/storage_cache.rs +++ b/client/db/src/storage_cache.rs @@ -688,10 +688,6 @@ impl>, B: BlockT> StateBackend> for Cachin info.include_state_machine_states(&self.overlay_stats); info } - - fn state_hashed_value(&self) -> bool { - self.state.state_hashed_value() - } } /// Extended [`CachingState`] that will sync the caches on drop. @@ -877,10 +873,6 @@ impl>, B: BlockT> StateBackend> for Syncin fn usage_info(&self) -> sp_state_machine::UsageInfo { self.caching_state().usage_info() } - - fn state_hashed_value(&self) -> bool { - self.caching_state().state_hashed_value() - } } impl Drop for SyncingCachingState { @@ -1208,8 +1200,7 @@ mod tests { let shared = new_shared_cache::(256*1024, (0,1)); let mut backend = InMemoryBackend::::default(); - let flagged = false; - backend.insert(std::iter::once((None, vec![(key.clone(), Some(vec![1]))])), flagged); + backend.insert(std::iter::once((None, vec![(key.clone(), Some(vec![1]))]))); let mut s = CachingState::new( backend.clone(), diff --git a/client/executor/runtime-test/Cargo.toml b/client/executor/runtime-test/Cargo.toml index 93ad463be16c3..a9be5c650625e 100644 --- a/client/executor/runtime-test/Cargo.toml +++ b/client/executor/runtime-test/Cargo.toml @@ -20,6 +20,7 @@ sp-runtime = { version = "3.0.0", default-features = false, path = "../../../pri sp-sandbox = { version = "0.9.0", default-features = false, path = "../../../primitives/sandbox" } sp-std = { version = "3.0.0", default-features = false, path = "../../../primitives/std" } sp-tasks = { version = "3.0.0", default-features = false, path = "../../../primitives/tasks" } +sp-storage = { version = "3.0.0", default-features = false, path = "../../../primitives/storage" } [build-dependencies] substrate-wasm-builder = { version = "4.0.0", path = "../../../utils/wasm-builder" } diff --git a/client/executor/runtime-test/src/lib.rs b/client/executor/runtime-test/src/lib.rs index fc452d135da07..2e595760815ab 100644 --- a/client/executor/runtime-test/src/lib.rs +++ b/client/executor/runtime-test/src/lib.rs @@ -64,7 +64,10 @@ sp_core::wasm_export_functions! { fn test_switch_state() { print("switch_state"); - storage::alt_hashing(); + storage::set( + sp_storage::well_known_keys::TRIE_HASHING_CONFIG, + sp_storage::trie_threshold_encode(sp_storage::TEST_DEFAULT_ALT_HASH_THRESHOLD).as_slice(), + ); print("switched!"); } diff --git a/client/executor/src/integration_tests/mod.rs b/client/executor/src/integration_tests/mod.rs index 7c702cb855d6e..164c576540809 100644 --- a/client/executor/src/integration_tests/mod.rs +++ b/client/executor/src/integration_tests/mod.rs @@ -204,15 +204,18 @@ fn storage_should_work(wasm_method: WasmExecutionMethod) { assert_eq!(output, b"all ok!".to_vec().encode()); } - let expected = TestExternalities::new(sp_core::storage::Storage { + let mut storage = sp_core::storage::Storage { top: map![ b"input".to_vec() => b"Hello world".to_vec(), b"foo".to_vec() => b"bar".to_vec(), b"baz".to_vec() => b"bar".to_vec() ], children_default: map![], - alt_hashing: false, - }); + }; + storage.modify_trie_alt_hashing_threshold(Some( + sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD + )); + let expected = TestExternalities::new(storage); assert_eq!(ext, expected); } @@ -238,15 +241,19 @@ fn clear_prefix_should_work(wasm_method: WasmExecutionMethod) { assert_eq!(output, b"all ok!".to_vec().encode()); } - let expected = TestExternalities::new(sp_core::storage::Storage { + let mut storage = sp_core::storage::Storage { top: map![ b"aaa".to_vec() => b"1".to_vec(), b"aab".to_vec() => b"2".to_vec(), b"bbb".to_vec() => b"5".to_vec() ], children_default: map![], - alt_hashing: false, - }); + }; + storage.modify_trie_alt_hashing_threshold(Some( + sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD + )); + + let expected = TestExternalities::new(storage); assert_eq!(expected, ext); } diff --git a/client/light/src/backend.rs b/client/light/src/backend.rs index b593b8998a0ae..d68507f549320 100644 --- a/client/light/src/backend.rs +++ b/client/light/src/backend.rs @@ -346,7 +346,7 @@ impl BlockImportOperation for ImportOperation } let storage_update = InMemoryBackend::from(storage); - let (storage_root, _) = storage_update.full_storage_root(std::iter::empty(), child_delta, input.alt_hashing); + let (storage_root, _) = storage_update.full_storage_root(std::iter::empty(), child_delta); self.storage_update = Some(storage_update); Ok(storage_root) @@ -540,11 +540,4 @@ impl StateBackend for GenesisOrUnavailableState GenesisOrUnavailableState::Unavailable => None, } } - - fn state_hashed_value(&self) -> bool { - match self { - GenesisOrUnavailableState::Genesis(state) => state.state_hashed_value(), - GenesisOrUnavailableState::Unavailable => false, - } - } } diff --git a/client/network/test/src/lib.rs b/client/network/test/src/lib.rs index 47582607f8bb7..e1e02a884f1cb 100644 --- a/client/network/test/src/lib.rs +++ b/client/network/test/src/lib.rs @@ -700,8 +700,8 @@ pub trait TestNetFactory: Sized where >: /// Add a full peer. fn add_full_peer_with_config(&mut self, config: FullPeerConfig) { let test_client_builder = match config.keep_blocks { - Some(keep_blocks) => TestClientBuilder::with_pruning_window(keep_blocks), - None => TestClientBuilder::with_default_backend(), + Some(keep_blocks) => TestClientBuilder::with_pruning_window(keep_blocks).state_hashed_value(), + None => TestClientBuilder::with_default_backend().state_hashed_value(), }; let backend = test_client_builder.backend(); let (c, longest_chain) = test_client_builder.build_with_longest_chain(); diff --git a/client/service/src/chain_ops/export_raw_state.rs b/client/service/src/chain_ops/export_raw_state.rs index 28fad7632fd80..71822cf6275f8 100644 --- a/client/service/src/chain_ops/export_raw_state.rs +++ b/client/service/src/chain_ops/export_raw_state.rs @@ -42,7 +42,6 @@ where let empty_key = StorageKey(Vec::new()); let mut top_storage = client.storage_pairs(&block, &empty_key)?; let mut children_default = HashMap::new(); - let alt_hashing = client.state_hashed_value(&block)?; // Remove all default child storage roots from the top storage and collect the child storage // pairs. @@ -70,5 +69,5 @@ where } let top = top_storage.into_iter().map(|(k, v)| (k.0, v.0)).collect(); - Ok(Storage { top, children_default, alt_hashing }) + Ok(Storage { top, children_default }) } diff --git a/client/service/src/client/client.rs b/client/service/src/client/client.rs index fd306c536fadd..b294be2268997 100644 --- a/client/service/src/client/client.rs +++ b/client/service/src/client/client.rs @@ -1539,10 +1539,6 @@ impl StorageProvider for Client wher Ok(result) } - - fn state_hashed_value(&self, id: &BlockId) -> sp_blockchain::Result { - Ok(self.state_at(id)?.state_hashed_value()) - } } impl HeaderMetadata for Client where diff --git a/client/service/test/src/client/mod.rs b/client/service/test/src/client/mod.rs index a4fb06a8d5e79..1fccd918be7c9 100644 --- a/client/service/test/src/client/mod.rs +++ b/client/service/test/src/client/mod.rs @@ -1751,11 +1751,7 @@ fn imports_blocks_with_changes_tries_config_change() { #[test] fn storage_keys_iter_prefix_and_start_key_works() { - storage_keys_iter_prefix_and_start_key_works_inner(true); - storage_keys_iter_prefix_and_start_key_works_inner(false); -} -fn storage_keys_iter_prefix_and_start_key_works_inner(hashed_value: bool) { - let client = substrate_test_runtime_client::new(hashed_value); + let client = substrate_test_runtime_client::new(false); let prefix = StorageKey(hex!("3a").to_vec()); @@ -1800,11 +1796,19 @@ fn storage_keys_iter_works_inner(hashed_value: bool) { .take(3) .map(|x| x.0) .collect(); - assert_eq!(res, [ - hex!("3a686561707061676573").to_vec(), - hex!("6644b9b8bc315888ac8e41a7968dc2b4141a5403c58acdf70b7e8f7e07bf5081").to_vec(), - hex!("79c07e2b1d2e2abfd4855b936617eeff5e0621c4869aa60c02be9adcc98a0d1d").to_vec(), - ]); + if hashed_value { + assert_eq!(res, [ + hex!("3a686561707061676573").to_vec(), + sp_core::storage::well_known_keys::TRIE_HASHING_CONFIG.to_vec(), + hex!("6644b9b8bc315888ac8e41a7968dc2b4141a5403c58acdf70b7e8f7e07bf5081").to_vec(), + ]); + } else { + assert_eq!(res, [ + hex!("3a686561707061676573").to_vec(), + hex!("6644b9b8bc315888ac8e41a7968dc2b4141a5403c58acdf70b7e8f7e07bf5081").to_vec(), + hex!("79c07e2b1d2e2abfd4855b936617eeff5e0621c4869aa60c02be9adcc98a0d1d").to_vec(), + ]); + } let res: Vec<_> = client.storage_keys_iter(&BlockId::Number(0), Some(&prefix), Some(&StorageKey(hex!("79c07e2b1d2e2abfd4855b936617eeff5e0621c4869aa60c02be9adcc98a0d1d").to_vec()))) .unwrap() diff --git a/frame/support/test/tests/instance.rs b/frame/support/test/tests/instance.rs index 1a07b7174e2ed..3171020182f33 100644 --- a/frame/support/test/tests/instance.rs +++ b/frame/support/test/tests/instance.rs @@ -327,8 +327,10 @@ fn storage_instance_independence() { let mut storage = sp_core::storage::Storage { top: std::collections::BTreeMap::new(), children_default: std::collections::HashMap::new(), - alt_hashing: true, }; + storage.modify_trie_alt_hashing_threshold(Some( + sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD, + )); sp_state_machine::BasicExternalities::execute_with_storage(&mut storage, || { module2::Value::::put(0); module2::Value::::put(0); @@ -343,8 +345,8 @@ fn storage_instance_independence() { module2::DoubleMap::::insert(&0, &0, &0); module2::DoubleMap::::insert(&0, &0, &0); }); - // 12 storage values. - assert_eq!(storage.top.len(), 12); + // 12 storage values and threshold. + assert_eq!(storage.top.len(), 13); } #[test] diff --git a/frame/system/src/lib.rs b/frame/system/src/lib.rs index 884a191b2e83b..98f6a2bfd6fd0 100644 --- a/frame/system/src/lib.rs +++ b/frame/system/src/lib.rs @@ -1401,15 +1401,18 @@ impl Pallet { /// Get the basic externalities for this pallet, useful for tests. #[cfg(any(feature = "std", test))] pub fn externalities() -> TestExternalities { - TestExternalities::new(sp_core::storage::Storage { + let mut storage = sp_core::storage::Storage { top: map![ >::hashed_key_for(T::BlockNumber::zero()) => [69u8; 32].encode(), >::hashed_key().to_vec() => T::BlockNumber::one().encode(), >::hashed_key().to_vec() => [69u8; 32].encode() ], children_default: map![], - alt_hashing: true, - }) + }; + storage.modify_trie_alt_hashing_threshold( + Some(sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD), + ); + TestExternalities::new(storage) } /// Set the block number to something in particular. Can be used as an alternative to diff --git a/primitives/externalities/src/lib.rs b/primitives/externalities/src/lib.rs index 54003d9886311..351d50bd90e60 100644 --- a/primitives/externalities/src/lib.rs +++ b/primitives/externalities/src/lib.rs @@ -291,10 +291,6 @@ pub trait Externalities: ExtensionStore { fn proof_size(&self) -> Option { None } - - /// Set flag in inner state to activate hashing of values. - /// TODO remove - fn alt_hashing(&mut self); } /// Extension for the [`Externalities`] trait. @@ -327,3 +323,28 @@ impl ExternalitiesExt for &mut dyn Externalities { self.deregister_extension_by_type_id(TypeId::of::()) } } + +/// Helpers method for the [`Externalities`] trait. +pub trait ExternalitiesHelpers: Externalities { + /// Utility function to get trie inner value hash threshold from + /// backend state or pending changes. + fn get_trie_alt_hashing_threshold(&self) -> Option { + self.storage(sp_storage::well_known_keys::TRIE_HASHING_CONFIG) + .and_then(|encoded| sp_storage::trie_threshold_decode(&mut encoded.as_slice())) + } + + /// Utility function to modify trie inner value hash threshold. + fn modify_trie_alt_hashing_threshold(&mut self, threshold: Option) { + match threshold { + Some(threshold) => { + let encoded = sp_storage::trie_threshold_encode(threshold); + self.set_storage(sp_storage::well_known_keys::TRIE_HASHING_CONFIG.to_vec(), encoded); + }, + None => { + self.clear_storage(sp_storage::well_known_keys::TRIE_HASHING_CONFIG); + }, + } + } +} + +impl ExternalitiesHelpers for E { } diff --git a/primitives/io/src/lib.rs b/primitives/io/src/lib.rs index a94e0d624be4a..8d93f3de774ee 100644 --- a/primitives/io/src/lib.rs +++ b/primitives/io/src/lib.rs @@ -213,11 +213,6 @@ pub trait Storage { self.storage_commit_transaction() .expect("No open transaction that can be committed."); } - - /// Swith state to alternate hashing. - fn alt_hashing(&mut self) { - self.alt_hashing(); - } } /// Interface for accessing the child storage for default child trie, @@ -1464,7 +1459,6 @@ mod tests { t = BasicExternalities::new(Storage { top: map![b"foo".to_vec() => b"bar".to_vec()], children_default: map![], - alt_hashing: false, }); t.execute_with(|| { @@ -1473,11 +1467,14 @@ mod tests { }); let value = vec![7u8; 35]; - t = BasicExternalities::new(Storage { + let mut storage = Storage { top: map![b"foo00".to_vec() => value.clone()], children_default: map![], - alt_hashing: true, - }); + }; + storage.modify_trie_alt_hashing_threshold(Some( + sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD + )); + t = BasicExternalities::new(storage); t.execute_with(|| { assert_eq!(storage::get(b"hello"), None); @@ -1491,7 +1488,6 @@ mod tests { let mut t = BasicExternalities::new(Storage { top: map![b":test".to_vec() => value.clone()], children_default: map![], - alt_hashing: false, }); t.execute_with(|| { @@ -1514,7 +1510,6 @@ mod tests { b":abdd".to_vec() => b"\x0b\0\0\0Hello world".to_vec() ], children_default: map![], - alt_hashing: false, }); t.execute_with(|| { diff --git a/primitives/state-machine/src/backend.rs b/primitives/state-machine/src/backend.rs index fa44acf9dde8c..65d0469100af1 100644 --- a/primitives/state-machine/src/backend.rs +++ b/primitives/state-machine/src/backend.rs @@ -178,8 +178,8 @@ pub trait Backend: sp_std::fmt::Debug { &'a ChildInfo, impl Iterator)>, )>, - alt_hashing: bool, ) -> (H::Out, Self::Transaction) where H::Out: Ord + Encode { + let alt_hashing = self.get_trie_alt_hashing_threshold().is_some(); let mut txs: Self::Transaction = Default::default(); let mut child_roots: Vec<_> = Default::default(); // child first @@ -257,8 +257,12 @@ pub trait Backend: sp_std::fmt::Debug { unimplemented!() } - /// Does trie state allow hashing of value. - fn state_hashed_value(&self) -> bool; + /// Read current trie hashing threshold. + /// Please do not reimplement. + fn get_trie_alt_hashing_threshold(&self) -> Option { + self.storage(sp_core::storage::well_known_keys::TRIE_HASHING_CONFIG).ok().flatten() + .and_then(|encoded| sp_core::storage::trie_threshold_decode(&mut encoded.as_slice())) + } } /// Trait that allows consolidate two transactions together. diff --git a/primitives/state-machine/src/basic.rs b/primitives/state-machine/src/basic.rs index f4ddc6f36f80d..57158d938ccd0 100644 --- a/primitives/state-machine/src/basic.rs +++ b/primitives/state-machine/src/basic.rs @@ -33,7 +33,7 @@ use sp_core::{ }; use log::warn; use codec::Encode; -use sp_externalities::{Extensions, Extension}; +use sp_externalities::{Extensions, Extension, ExternalitiesHelpers}; /// Simple Map-based Externalities impl. #[derive(Debug)] @@ -74,7 +74,6 @@ impl BasicExternalities { inner: Storage { top: std::mem::take(&mut storage.top), children_default: std::mem::take(&mut storage.children_default), - alt_hashing: storage.alt_hashing, }, extensions: Default::default(), }; @@ -129,7 +128,6 @@ impl From> for BasicExternalities { inner: Storage { top: hashmap, children_default: Default::default(), - alt_hashing: true, }, extensions: Default::default(), } @@ -283,7 +281,8 @@ impl Externalities for BasicExternalities { } } - let layout = if self.inner.alt_hashing { + let alt_hashing = self.get_trie_alt_hashing_threshold().is_some(); + let layout = if alt_hashing { Layout::::with_inner_hashing() } else { Layout::::default() @@ -297,8 +296,9 @@ impl Externalities for BasicExternalities { ) -> Vec { if let Some(child) = self.inner.children_default.get(child_info.storage_key()) { let delta = child.data.iter().map(|(k, v)| (k.as_ref(), Some(v.as_ref()))); + let alt_hashing = self.get_trie_alt_hashing_threshold().is_some(); crate::in_memory_backend::new_in_mem::() - .child_storage_root(&child.child_info, delta, self.inner.alt_hashing).0 + .child_storage_root(&child.child_info, delta, alt_hashing).0 } else { empty_child_trie_root::>() }.encode() @@ -339,10 +339,6 @@ impl Externalities for BasicExternalities { fn set_whitelist(&mut self, _: Vec) { unimplemented!("set_whitelist is not supported in Basic") } - - fn alt_hashing(&mut self) { - self.inner.alt_hashing = true; - } } impl sp_externalities::ExtensionStore for BasicExternalities { @@ -408,7 +404,6 @@ mod tests { child_info: child_info.to_owned(), } ], - alt_hashing: false, }); assert_eq!(ext.child_storage(child_info, b"doe"), Some(b"reindeer".to_vec())); @@ -439,7 +434,6 @@ mod tests { child_info: child_info.to_owned(), } ], - alt_hashing: false, }); let res = ext.kill_child_storage(child_info, None); diff --git a/primitives/state-machine/src/ext.rs b/primitives/state-machine/src/ext.rs index dd6e0f519ef35..4ecbe17fc3346 100644 --- a/primitives/state-machine/src/ext.rs +++ b/primitives/state-machine/src/ext.rs @@ -28,7 +28,7 @@ use sp_core::{ }; use sp_trie::{trie_types::Layout, empty_child_trie_root}; use sp_externalities::{ - Externalities, Extensions, Extension, ExtensionStore, + Externalities, Extensions, Extension, ExtensionStore, ExternalitiesHelpers, }; use codec::{Decode, Encode, EncodeAppend}; @@ -547,7 +547,8 @@ where } else { let root = if let Some((changes, info)) = self.overlay.child_changes(storage_key) { let delta = changes.map(|(k, v)| (k.as_ref(), v.value().map(AsRef::as_ref))); - Some(self.backend.child_storage_root(info, delta, self.overlay.alt_hashing())) + let alt_hashing = self.get_trie_alt_hashing_threshold().is_some(); + Some(self.backend.child_storage_root(info, delta, alt_hashing)) } else { None }; @@ -737,11 +738,6 @@ where fn proof_size(&self) -> Option { self.backend.proof_size() } - - fn alt_hashing(&mut self) { - self.mark_dirty(); - self.overlay.set_alt_hashing() - } } /// Implement `Encode` by forwarding the stored raw vec. @@ -951,8 +947,7 @@ mod tests { vec![20] => vec![20], vec![40] => vec![40] ], - children_default: map![], - alt_hashing: false, + children_default: map![] }.into(); let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); @@ -998,7 +993,6 @@ mod tests { child_info: child_info.to_owned(), } ], - alt_hashing: false, }.into(); let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); @@ -1043,7 +1037,6 @@ mod tests { child_info: child_info.to_owned(), } ], - alt_hashing: false, }.into(); let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); @@ -1083,7 +1076,6 @@ mod tests { child_info: child_info.to_owned(), } ], - alt_hashing: false, }.into(); let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); diff --git a/primitives/state-machine/src/in_memory_backend.rs b/primitives/state-machine/src/in_memory_backend.rs index d2a4bf6ae8e98..c5dc881caae91 100644 --- a/primitives/state-machine/src/in_memory_backend.rs +++ b/primitives/state-machine/src/in_memory_backend.rs @@ -45,10 +45,9 @@ where >( &self, changes: T, - flag_inner_hash_value: bool, ) -> Self { let mut clone = self.clone(); - clone.insert(changes, flag_inner_hash_value); + clone.insert(changes); clone } @@ -58,8 +57,8 @@ where >( &mut self, changes: T, - flag_inner_hash_value: bool, ) { + // Note that in case the threshold is changed, it will not be apply immediately. let (top, child) = changes.into_iter().partition::, _>(|v| v.0.is_none()); let (root, transaction) = self.full_storage_root( top.iter().map(|(_, v)| v).flatten().map(|(k, v)| (&k[..], v.as_deref())), @@ -67,7 +66,6 @@ where .filter_map(|v| v.0.as_ref().map(|c| (c, v.1.iter().map(|(k, v)| (&k[..], v.as_deref())))) ), - flag_inner_hash_value, ); self.apply_transaction(root, transaction); @@ -119,7 +117,6 @@ where let mut backend = new_in_mem(); backend.insert( inner.into_iter().map(|(k, m)| (k, m.into_iter().map(|(k, v)| (k, Some(v))).collect())), - false, ); backend } @@ -182,13 +179,11 @@ mod tests { let storage = new_in_mem::(); let child_info = ChildInfo::new_default(b"1"); let child_info = &child_info; - let flagged = false; let mut storage = storage.update( vec![( Some(child_info.clone()), vec![(b"2".to_vec(), Some(b"3".to_vec()))] - )], - flagged, + )] ); let trie_backend = storage.as_trie_backend().unwrap(); assert_eq!(trie_backend.child_storage(child_info, b"2").unwrap(), @@ -201,10 +196,9 @@ mod tests { fn insert_multiple_times_child_data_works() { let mut storage = new_in_mem::(); let child_info = ChildInfo::new_default(b"1"); - let flagged = false; - storage.insert(vec![(Some(child_info.clone()), vec![(b"2".to_vec(), Some(b"3".to_vec()))])], flagged); - storage.insert(vec![(Some(child_info.clone()), vec![(b"1".to_vec(), Some(b"3".to_vec()))])], flagged); + storage.insert(vec![(Some(child_info.clone()), vec![(b"2".to_vec(), Some(b"3".to_vec()))])]); + storage.insert(vec![(Some(child_info.clone()), vec![(b"1".to_vec(), Some(b"3".to_vec()))])]); assert_eq!(storage.child_storage(&child_info, &b"2"[..]), Ok(Some(b"3".to_vec()))); assert_eq!(storage.child_storage(&child_info, &b"1"[..]), Ok(Some(b"3".to_vec()))); diff --git a/primitives/state-machine/src/overlayed_changes/mod.rs b/primitives/state-machine/src/overlayed_changes/mod.rs index 5df20ba1f233b..1d3cbb59ba0c1 100644 --- a/primitives/state-machine/src/overlayed_changes/mod.rs +++ b/primitives/state-machine/src/overlayed_changes/mod.rs @@ -107,8 +107,6 @@ pub struct OverlayedChanges { transaction_index_ops: Vec, /// True if extrinsics stats must be collected. collect_extrinsics: bool, - /// True if we flag inner state to store hash of values. - alt_hashing: bool, /// Collect statistic on this execution. stats: StateMachineStats, } @@ -262,16 +260,6 @@ impl OverlayedChanges { self.collect_extrinsics = collect_extrinsics; } - /// Ask to switch state to use inner hash. - pub fn set_alt_hashing(&mut self) { - self.alt_hashing = true; - } - - /// Is `alt_hashing` flag set. - pub fn alt_hashing(&self) -> bool { - self.alt_hashing - } - /// Returns a double-Option: None if the key is unknown (i.e. and the query should be referred /// to the backend); Some(None) if the key has been deleted. Some(Some(...)) for a key whose /// value has been set. @@ -643,7 +631,7 @@ impl OverlayedChanges { |(k, v)| (&k[..], v.value().map(|v| &v[..])) ))); - let (root, transaction) = backend.full_storage_root(delta, child_delta, self.alt_hashing); + let (root, transaction) = backend.full_storage_root(delta, child_delta); cache.transaction = Some(transaction); cache.transaction_storage_root = Some(root); diff --git a/primitives/state-machine/src/proving_backend.rs b/primitives/state-machine/src/proving_backend.rs index c1eaf7068c7ac..d4a1ec66523dd 100644 --- a/primitives/state-machine/src/proving_backend.rs +++ b/primitives/state-machine/src/proving_backend.rs @@ -362,10 +362,6 @@ impl<'a, S, H> Backend for ProvingBackend<'a, S, H> fn usage_info(&self) -> crate::stats::UsageInfo { self.0.usage_info() } - - fn state_hashed_value(&self) -> bool { - self.0.state_hashed_value() - } } /// Create proof check backend. @@ -456,11 +452,19 @@ mod tests { proof_recorded_and_checked_inner(false); } fn proof_recorded_and_checked_inner(flagged: bool) { - let size_content = 33; // above hashable value treshold. + let size_content = 34; // above hashable value treshold. let value_range = 0..64; - let contents = value_range.clone().map(|i| (vec![i], Some(vec![i; size_content]))).collect::>(); + let mut contents = value_range.clone().map(|i| (vec![i], Some(vec![i; size_content]))).collect::>(); + if flagged { + contents.push(( + sp_core::storage::well_known_keys::TRIE_HASHING_CONFIG.to_vec(), + Some(sp_core::storage::trie_threshold_encode( + sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD, + )), + )); + } let in_memory = InMemoryBackend::::default(); - let mut in_memory = in_memory.update(vec![(None, contents)], flagged); + let mut in_memory = in_memory.update(vec![(None, contents)]); let in_memory_root = in_memory.storage_root(std::iter::empty(), flagged).0; value_range.clone().for_each(|i| assert_eq!(in_memory.storage(&[i]).unwrap().unwrap(), vec![i; size_content])); @@ -481,16 +485,23 @@ mod tests { #[test] fn proof_recorded_and_checked_old_hash() { // test proof starting with old hash content and flagging in between. - let size_content = 33; // above hashable value treshold. + // TODO not that usefull (we do run with direct update). -> replace by change of threshold + // test. + let size_content = 34; // above hashable value treshold. let value_range = 0..64; let contents = value_range.clone().map(|i| (vec![i], Some(vec![i; size_content]))).collect::>(); let in_memory = InMemoryBackend::::default(); - let mut in_memory = in_memory.update(vec![(None, contents)], false); + let mut in_memory = in_memory.update(vec![(None, contents)]); value_range.clone().for_each(|i| assert_eq!(in_memory.storage(&[i]).unwrap().unwrap(), vec![i; size_content])); - in_memory = in_memory.update(vec![], true); - let in_memory_root = in_memory.storage_root(std::iter::empty(), false).0; + in_memory = in_memory.update(vec![(None, vec![( + sp_core::storage::well_known_keys::TRIE_HASHING_CONFIG.to_vec(), + Some(sp_core::storage::trie_threshold_encode( + sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD, + )), + )])]); + let in_memory_root = in_memory.storage_root(std::iter::empty(), true).0; let trie = in_memory.as_trie_backend().unwrap(); - let trie_root = trie.storage_root(std::iter::empty(), false).0; + let trie_root = trie.storage_root(std::iter::empty(), true).0; assert_eq!(in_memory_root, trie_root); value_range.clone().for_each(|i| assert_eq!(trie.storage(&[i]).unwrap().unwrap(), vec![i; size_content])); @@ -513,20 +524,27 @@ mod tests { let child_info_2 = ChildInfo::new_default(b"sub2"); let child_info_1 = &child_info_1; let child_info_2 = &child_info_2; - let contents = vec![ - (None, (0..64).map(|i| (vec![i], Some(vec![i]))).collect()), + let mut contents = vec![ + (None, (0..64).map(|i| (vec![i], Some(vec![i]))).collect::>()), (Some(child_info_1.clone()), (28..65).map(|i| (vec![i], Some(vec![i]))).collect()), (Some(child_info_2.clone()), (10..15).map(|i| (vec![i], Some(vec![i]))).collect()), ]; + if flagged { + contents[0].1.push(( + sp_core::storage::well_known_keys::TRIE_HASHING_CONFIG.to_vec(), + Some(sp_core::storage::trie_threshold_encode( + sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD, + )), + )); + } let in_memory = InMemoryBackend::::default(); - let mut in_memory = in_memory.update(contents, flagged); + let mut in_memory = in_memory.update(contents); let child_storage_keys = vec![child_info_1.to_owned(), child_info_2.to_owned()]; let in_memory_root = in_memory.full_storage_root( std::iter::empty(), child_storage_keys.iter().map(|k|(k, std::iter::empty())), - flagged, ).0; (0..64).for_each(|i| assert_eq!( in_memory.storage(&[i]).unwrap().unwrap(), diff --git a/primitives/state-machine/src/read_only.rs b/primitives/state-machine/src/read_only.rs index b2e32726fc802..296520900c952 100644 --- a/primitives/state-machine/src/read_only.rs +++ b/primitives/state-machine/src/read_only.rs @@ -202,10 +202,6 @@ impl<'a, H: Hasher, B: 'a + Backend> Externalities for ReadOnlyExternalities< fn set_whitelist(&mut self, _: Vec) { unimplemented!("set_whitelist is not supported in ReadOnlyExternalities") } - - fn alt_hashing(&mut self) { - unimplemented!("alt_hashing is not supported by ReadOnlyExternalities"); - } } impl<'a, H: Hasher, B: 'a + Backend> sp_externalities::ExtensionStore for ReadOnlyExternalities<'a, H, B> { diff --git a/primitives/state-machine/src/testing.rs b/primitives/state-machine/src/testing.rs index 49e963c7f4a99..3327a78d016ea 100644 --- a/primitives/state-machine/src/testing.rs +++ b/primitives/state-machine/src/testing.rs @@ -139,7 +139,7 @@ where /// Insert key/value into backend pub fn insert(&mut self, k: StorageKey, v: StorageValue) { - self.backend.insert(vec![(None, vec![(k, Some(v))])], false); + self.backend.insert(vec![(None, vec![(k, Some(v))])]); } /// Registers the given extension for this instance. @@ -171,7 +171,7 @@ where )) } - self.backend.update(transaction, self.overlay.alt_hashing()) + self.backend.update(transaction) } /// Commit all pending changes to the underlying backend. @@ -239,7 +239,9 @@ impl Default for TestExternalities fn default() -> Self { // default to inner hashed. let mut storage = Storage::default(); - storage.alt_hashing = true; + storage.modify_trie_alt_hashing_threshold( + Some(sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD), + ); Self::new(storage) } } @@ -308,7 +310,8 @@ mod tests { #[test] fn commit_should_work() { - let mut ext = TestExternalities::::default(); + let storage = Storage::default(); // avoid adding the trie threshold. + let mut ext = TestExternalities::::from(storage); let mut ext = ext.ext(); ext.set_storage(b"doe".to_vec(), b"reindeer".to_vec()); ext.set_storage(b"dog".to_vec(), b"puppy".to_vec()); diff --git a/primitives/state-machine/src/trie_backend.rs b/primitives/state-machine/src/trie_backend.rs index 62666134445d8..52cadac871aa0 100644 --- a/primitives/state-machine/src/trie_backend.rs +++ b/primitives/state-machine/src/trie_backend.rs @@ -261,10 +261,6 @@ impl, H: Hasher> Backend for TrieBackend where fn wipe(&self) -> Result<(), Self::Error> { Ok(()) } - - fn state_hashed_value(&self) -> bool { - self.essence.state_hashed_value() - } } #[cfg(test)] @@ -305,6 +301,14 @@ pub mod tests { trie.insert(b"value1", &[42]).expect("insert failed"); trie.insert(b"value2", &[24]).expect("insert failed"); trie.insert(b":code", b"return 42").expect("insert failed"); + if hashed_value { + trie.insert( + sp_core::storage::well_known_keys::TRIE_HASHING_CONFIG, + sp_core::storage::trie_threshold_encode( + sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD, + ).as_slice(), + ).unwrap(); + } for i in 128u8..255u8 { trie.insert(&[i], &[i]).unwrap(); } diff --git a/primitives/state-machine/src/trie_backend_essence.rs b/primitives/state-machine/src/trie_backend_essence.rs index 7b2bd65c293d5..e7d41a11eb64b 100644 --- a/primitives/state-machine/src/trie_backend_essence.rs +++ b/primitives/state-machine/src/trie_backend_essence.rs @@ -170,12 +170,6 @@ impl, H: Hasher> TrieBackendEssence where H::Out: Ok(next_key) } - /// Does current trie use inner hashed value. - pub fn state_hashed_value(&self) -> bool { - sp_trie::state_hashed_value::, _>(self, &self.root) - .unwrap_or_default() - } - /// Get the value of storage at given key. pub fn storage(&self, key: &[u8]) -> Result> { let map_e = |e| format!("Trie lookup error: {}", e); diff --git a/primitives/storage/src/lib.rs b/primitives/storage/src/lib.rs index ef1ba64f256d9..f63323bbaa17f 100644 --- a/primitives/storage/src/lib.rs +++ b/primitives/storage/src/lib.rs @@ -140,8 +140,6 @@ pub struct Storage { /// trie kind, so this is exclusively for the `ChildType::ParentKeyId` /// tries. pub children_default: std::collections::HashMap, StorageChild>, - /// `true` when state should hash values internally. - pub alt_hashing: bool, } /// Storage change set @@ -202,6 +200,44 @@ pub mod well_known_keys { } } +/// Configuration value for a given threshold. +pub fn trie_threshold_encode(threshold: u32) -> Vec { + codec::Compact(threshold).encode() +} + +/// Configuration threshold from encoded, invalid encoded +/// is same as no threshold. +pub fn trie_threshold_decode(mut encoded: &[u8]) -> Option { + codec::Compact::::decode(&mut encoded).ok() + .map(|compact| compact.0) +} + +/// Default value to use as a threshold for testing. +pub const TEST_DEFAULT_ALT_HASH_THRESHOLD: u32 = 34; + +#[cfg(feature = "std")] +impl Storage { + /// Utility function to get trie inner value hash threshold from + /// backend state or pending changes. + pub fn get_trie_alt_hashing_threshold(&self) -> Option { + self.top.get(well_known_keys::TRIE_HASHING_CONFIG) + .and_then(|encoded| trie_threshold_decode(&mut encoded.as_slice())) + } + + /// Utility function to modify trie inner value hash threshold. + pub fn modify_trie_alt_hashing_threshold(&mut self, threshold: Option) { + match threshold { + Some(threshold) => { + let encoded = trie_threshold_encode(threshold); + self.top.insert(well_known_keys::TRIE_HASHING_CONFIG.to_vec(), encoded); + }, + None => { + self.top.remove(well_known_keys::TRIE_HASHING_CONFIG); + }, + } + } +} + /// Information related to a child state. #[derive(Debug, Clone)] #[cfg_attr(feature = "std", derive(PartialEq, Eq, Hash, PartialOrd, Ord))] diff --git a/primitives/tasks/src/async_externalities.rs b/primitives/tasks/src/async_externalities.rs index 6451caf11b7eb..5d99ca4368d0b 100644 --- a/primitives/tasks/src/async_externalities.rs +++ b/primitives/tasks/src/async_externalities.rs @@ -189,10 +189,6 @@ impl Externalities for AsyncExternalities { fn set_whitelist(&mut self, _: Vec) { unimplemented!("set_whitelist is not supported in AsyncExternalities") } - - fn alt_hashing(&mut self) { - unimplemented!("alt_hashing is not supported in AsyncExternalities") - } } impl sp_externalities::ExtensionStore for AsyncExternalities { diff --git a/primitives/trie/src/lib.rs b/primitives/trie/src/lib.rs index de5d751aba038..283147602e31e 100644 --- a/primitives/trie/src/lib.rs +++ b/primitives/trie/src/lib.rs @@ -459,31 +459,6 @@ pub fn delta_trie_root( Ok(root) } -/// Resolve if inner hashing of value is active. -pub fn state_hashed_value>>( - db: &DB, - root: &TrieHash, -) -> Option> { - struct ReadMeta { - hashed: Option>, - } - impl trie_db::Query for &mut ReadMeta { - type Item = DBValue; - fn decode(self, value: &[u8]) -> DBValue { value.to_vec() } - fn record(&mut self, _hash: &::Out, _data: &[u8], _depth: u32, meta: &L::Meta) { - debug_assert!(self.hashed.is_none()); - self.hashed = Some(meta.extract_global_meta()); - } - } - let mut read_meta: ReadMeta = ReadMeta { - hashed: None, - }; - if let Ok(t) = TrieDB::::new(&*db, root) { - let _ = t.get_with(&[], &mut read_meta); - } - read_meta.hashed -} - /// Read a value from the trie. pub fn read_trie_value>>( db: &DB, @@ -847,6 +822,7 @@ mod trie_constants { /// achieve by using a escaped header in first or last element of proof /// and write it after. /// TODO 33 is not good switch to 32 + 1 + 1: 34 (avoid hashing stored hash for a 1 byte gain). + /// TODO replace with sp_storage test one. pub const INNER_HASH_TRESHOLD: usize = 33; const FIRST_PREFIX: u8 = 0b_00 << 6; /// In proof this header is used when only hashed value is stored. diff --git a/test-utils/client/src/lib.rs b/test-utils/client/src/lib.rs index 467b802f65cef..0603f7dd7dfa8 100644 --- a/test-utils/client/src/lib.rs +++ b/test-utils/client/src/lib.rs @@ -211,7 +211,9 @@ impl TestClientBuilder authorities.encode(), blake2_256(&AccountKeyring::Alice.to_raw_public().to_keyed_vec(b"balance:")).to_vec() => { vec![111u8, 0, 0, 0, 0, 0, 0, 0] - } + }, + sp_core::storage::well_known_keys::TRIE_HASHING_CONFIG.to_vec() => + sp_core::storage::trie_threshold_encode( + sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD, + ), ], children_default: map![], - alt_hashing: true, }, ) } From b62cdf5948b81a432ad420993bb9f6f7dcea6516 Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 10 Jun 2021 13:22:55 +0200 Subject: [PATCH 047/127] pending switching to using threshold, new storage root api does not make much sense. --- bin/node/bench/src/generator.rs | 6 +- bin/node/bench/src/simple_trie.rs | 10 ++-- bin/node/bench/src/trie.rs | 11 +++- client/api/src/in_mem.rs | 4 +- client/db/src/bench.rs | 25 +++++--- client/db/src/lib.rs | 35 +++++++---- client/db/src/storage_cache.rs | 24 ++++---- client/light/src/backend.rs | 15 ++--- primitives/state-machine/src/backend.rs | 55 ++++++++++++++--- primitives/state-machine/src/basic.rs | 10 ++-- .../state-machine/src/changes_trie/mod.rs | 7 ++- .../state-machine/src/changes_trie/storage.rs | 7 ++- primitives/state-machine/src/ext.rs | 9 ++- .../state-machine/src/in_memory_backend.rs | 2 +- primitives/state-machine/src/lib.rs | 2 +- .../src/overlayed_changes/mod.rs | 18 +++++- .../state-machine/src/proving_backend.rs | 14 ++--- primitives/state-machine/src/trie_backend.rs | 16 ++--- .../state-machine/src/trie_backend_essence.rs | 60 ++++++++++++------- primitives/trie/src/lib.rs | 58 +++++++----------- primitives/trie/src/node_codec.rs | 14 +++-- primitives/trie/src/trie_stream.rs | 22 ++++--- 22 files changed, 268 insertions(+), 156 deletions(-) diff --git a/bin/node/bench/src/generator.rs b/bin/node/bench/src/generator.rs index f811802d357ee..6589bdcd0b277 100644 --- a/bin/node/bench/src/generator.rs +++ b/bin/node/bench/src/generator.rs @@ -31,7 +31,7 @@ use crate::simple_trie::SimpleTrie; pub fn generate_trie( db: Arc, key_values: impl IntoIterator, Vec)>, - alt_hashing: bool, + alt_hashing: Option, ) -> Hash { let mut root = Hash::default(); @@ -44,8 +44,8 @@ pub fn generate_trie( let mut trie = SimpleTrie { db, overlay: &mut overlay }; { - let mut trie_db = if alt_hashing { - let layout = sp_trie::Layout::with_inner_hashing(); + let mut trie_db = if let Some(threshold) = alt_hashing { + let layout = sp_trie::Layout::with_inner_hashing(threshold); TrieDBMut::::new_with_layout(&mut trie, &mut root, layout) } else { TrieDBMut::new(&mut trie, &mut root) diff --git a/bin/node/bench/src/simple_trie.rs b/bin/node/bench/src/simple_trie.rs index 4d8e76ae3a7e1..fea106ed196fe 100644 --- a/bin/node/bench/src/simple_trie.rs +++ b/bin/node/bench/src/simple_trie.rs @@ -31,15 +31,15 @@ pub struct SimpleTrie<'a> { pub overlay: &'a mut HashMap, Option>>, } -impl<'a> AsHashDB for SimpleTrie<'a> { - fn as_hash_db(&self) -> &dyn hash_db::HashDB { &*self } +impl<'a> AsHashDB> for SimpleTrie<'a> { + fn as_hash_db(&self) -> &dyn hash_db::HashDB> { &*self } - fn as_hash_db_mut<'b>(&'b mut self) -> &'b mut (dyn HashDB + 'b) { + fn as_hash_db_mut<'b>(&'b mut self) -> &'b mut (dyn HashDB> + 'b) { &mut *self } } -impl<'a> HashDB for SimpleTrie<'a> { +impl<'a> HashDB> for SimpleTrie<'a> { fn get(&self, key: &Hash, prefix: Prefix) -> Option { let key = sp_trie::prefixed_key::(key, prefix); if let Some(value) = self.overlay.get(&key) { @@ -48,7 +48,7 @@ impl<'a> HashDB for SimpleTrie<'a> { self.db.get(0, &key).expect("Database backend error") } - fn get_with_meta(&self, key: &Hash, prefix: Prefix, global: bool) -> Option<(DBValue, TrieMeta)> { + fn get_with_meta(&self, key: &Hash, prefix: Prefix, global: Option) -> Option<(DBValue, TrieMeta)> { let result = self.get(key, prefix); result.map(|value| >::extract_value_owned(value, global)) } diff --git a/bin/node/bench/src/trie.rs b/bin/node/bench/src/trie.rs index 735403f95c87c..f1bcd3b2239c6 100644 --- a/bin/node/bench/src/trie.rs +++ b/bin/node/bench/src/trie.rs @@ -145,7 +145,7 @@ impl core::BenchmarkDescription for TrieReadBenchmarkDescription { let root = generate_trie( database.open(self.database_type), key_values, - true, + Some(sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD), ); Box::new(TrieReadBenchmark { @@ -170,7 +170,12 @@ impl core::BenchmarkDescription for TrieReadBenchmarkDescription { struct Storage(Arc); impl sp_state_machine::Storage for Storage { - fn get(&self, key: &Hash, prefix: Prefix, global: bool) -> Result, TrieMeta)>, String> { + fn get( + &self, + key: &Hash, + prefix: Prefix, + global: Option, + ) -> Result, TrieMeta)>, String> { let key = sp_trie::prefixed_key::(key, prefix); self.0.get(0, &key).map_err(|e| format!("Database backend error: {:?}", e)) .map(|result| result @@ -263,7 +268,7 @@ impl core::BenchmarkDescription for TrieWriteBenchmarkDescription { let root = generate_trie( database.open(self.database_type), key_values, - true, + Some(sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD), ); Box::new(TrieWriteBenchmark { diff --git a/client/api/src/in_mem.rs b/client/api/src/in_mem.rs index d756e1cc0bbc4..93ac41b3c9dce 100644 --- a/client/api/src/in_mem.rs +++ b/client/api/src/in_mem.rs @@ -573,9 +573,11 @@ impl backend::BlockImportOperation for BlockImportOperatio ) ); - let (root, transaction) = self.old_state.full_storage_root( + let alt_hashing = storage.get_trie_alt_hashing_threshold(); + let (root, transaction) = self.old_state.full_storage_root_with_alt_hashing( storage.top.iter().map(|(k, v)| (k.as_ref(), Some(v.as_ref()))), child_delta, + alt_hashing, ); self.new_state = Some(transaction); diff --git a/client/db/src/bench.rs b/client/db/src/bench.rs index 312f576052e87..0201cd0b450a0 100644 --- a/client/db/src/bench.rs +++ b/client/db/src/bench.rs @@ -49,7 +49,12 @@ struct StorageDb { } impl sp_state_machine::Storage> for StorageDb { - fn get(&self, key: &Block::Hash, prefix: Prefix, global: bool) -> Result, String> { + fn get( + &self, + key: &Block::Hash, + prefix: Prefix, + global: Option, + ) -> Result, String> { let prefixed_key = prefixed_key::>(key, prefix); if let Some(recorder) = &self.proof_recorder { if let Some(v) = recorder.get(&key) { @@ -155,9 +160,12 @@ impl BenchmarkingState { &child_content.child_info, child_content.data.iter().map(|(k, v)| (k.as_ref(), Some(v.as_ref()))), )); - let (root, transaction): (B::Hash, _) = state.state.borrow_mut().as_mut().unwrap().full_storage_root( + let alt_hashing = genesis.get_trie_alt_hashing_threshold(); + let (root, transaction): (B::Hash, _) = state.state.borrow_mut().as_mut().unwrap() + .full_storage_root_with_alt_hashing( genesis.top.iter().map(|(k, v)| (k.as_ref(), Some(v.as_ref()))), child_delta, + alt_hashing, ); state.genesis = transaction.clone().drain(); state.genesis_root = root.clone(); @@ -396,23 +404,24 @@ impl StateBackend> for BenchmarkingState { } } - fn storage_root<'a>( + fn storage_root_with_alt_hashing<'a>( &self, delta: impl Iterator)>, - alt_hashing: bool, + alt_hashing: Option, ) -> (B::Hash, Self::Transaction) where B::Hash: Ord { - self.state.borrow().as_ref().map_or(Default::default(), |s| s.storage_root(delta, alt_hashing)) + self.state.borrow().as_ref() + .map_or(Default::default(), |s| s.storage_root_with_alt_hashing(delta, alt_hashing)) } - fn child_storage_root<'a>( + fn child_storage_root_with_alt_hashing<'a>( &self, child_info: &ChildInfo, delta: impl Iterator)>, - alt_hashing: bool, + alt_hashing: Option, ) -> (B::Hash, bool, Self::Transaction) where B::Hash: Ord { self.state.borrow().as_ref().map_or( Default::default(), - |s| s.child_storage_root(child_info, delta, alt_hashing), + |s| s.child_storage_root_with_alt_hashing(child_info, delta, alt_hashing), ) } diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index dc0d5465ecaf7..9b3d525ef9eba 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -224,21 +224,21 @@ impl StateBackend> for RefTrackingState { self.state.for_child_keys_with_prefix(child_info, prefix, f) } - fn storage_root<'a>( + fn storage_root_with_alt_hashing<'a>( &self, delta: impl Iterator)>, - alt_hashing: bool, + alt_hashing: Option, ) -> (B::Hash, Self::Transaction) where B::Hash: Ord { - self.state.storage_root(delta, alt_hashing) + self.state.storage_root_with_alt_hashing(delta, alt_hashing) } - fn child_storage_root<'a>( + fn child_storage_root_with_alt_hashing<'a>( &self, child_info: &ChildInfo, delta: impl Iterator)>, - alt_hashing: bool, + alt_hashing: Option, ) -> (B::Hash, bool, Self::Transaction) where B::Hash: Ord { - self.state.child_storage_root(child_info, delta, alt_hashing) + self.state.child_storage_root_with_alt_hashing(child_info, delta, alt_hashing) } fn pairs(&self) -> Vec<(Vec, Vec)> { @@ -778,7 +778,8 @@ impl sc_client_api::backend::BlockImportOperation for Bloc )); let mut changes_trie_config: Option = None; - let (root, transaction) = self.old_state.full_storage_root( + let alt_hashing = storage.get_trie_alt_hashing_threshold(); + let (root, transaction) = self.old_state.full_storage_root_with_alt_hashing( storage.top.iter().map(|(k, v)| { if &k[..] == well_known_keys::CHANGES_TRIE_CONFIG { changes_trie_config = Some( @@ -789,6 +790,7 @@ impl sc_client_api::backend::BlockImportOperation for Bloc (&k[..], Some(&v[..])) }), child_delta, + alt_hashing, ); self.db_updates = transaction; @@ -859,15 +861,21 @@ struct StorageDb { } impl sp_state_machine::Storage> for StorageDb { - fn get(&self, key: &Block::Hash, prefix: Prefix, global: bool) -> Result, String> { + fn get( + &self, + key: &Block::Hash, + prefix: Prefix, + global: Option, + ) -> Result, String> { if self.prefix_keys { let key = prefixed_key::>(key, prefix); self.state_db.get(&key, self) } else { self.state_db.get(key.as_ref(), self) } - .map(|result| result.map(|value| , _>>::extract_value_owned(value, global))) - .map_err(|e| format!("Database backend error: {:?}", e)) + .map(|result| result.map(|value| + , _>>::extract_value_owned(value, global) + )).map_err(|e| format!("Database backend error: {:?}", e)) } fn access_from(&self, _key: &Block::Hash) { @@ -895,7 +903,12 @@ impl DbGenesisStorage { } impl sp_state_machine::Storage> for DbGenesisStorage { - fn get(&self, _key: &Block::Hash, _prefix: Prefix, _global: bool) -> Result, String> { + fn get( + &self, + _key: &Block::Hash, + _prefix: Prefix, + _global: Option, + ) -> Result, String> { Ok(None) } fn access_from(&self, _key: &Block::Hash) { diff --git a/client/db/src/storage_cache.rs b/client/db/src/storage_cache.rs index 29dceb6b80828..bf929732caa2e 100644 --- a/client/db/src/storage_cache.rs +++ b/client/db/src/storage_cache.rs @@ -642,21 +642,21 @@ impl>, B: BlockT> StateBackend> for Cachin self.state.for_child_keys_with_prefix(child_info, prefix, f) } - fn storage_root<'a>( + fn storage_root_with_alt_hashing<'a>( &self, delta: impl Iterator)>, - alt_hashing: bool, + alt_hashing: Option, ) -> (B::Hash, Self::Transaction) where B::Hash: Ord { - self.state.storage_root(delta, alt_hashing) + self.state.storage_root_with_alt_hashing(delta, alt_hashing) } - fn child_storage_root<'a>( + fn child_storage_root_with_alt_hashing<'a>( &self, child_info: &ChildInfo, delta: impl Iterator)>, - alt_hashing: bool, + alt_hashing: Option, ) -> (B::Hash, bool, Self::Transaction) where B::Hash: Ord { - self.state.child_storage_root(child_info, delta, alt_hashing) + self.state.child_storage_root_with_alt_hashing(child_info, delta, alt_hashing) } fn pairs(&self) -> Vec<(Vec, Vec)> { @@ -826,21 +826,21 @@ impl>, B: BlockT> StateBackend> for Syncin self.caching_state().for_child_keys_with_prefix(child_info, prefix, f) } - fn storage_root<'a>( + fn storage_root_with_alt_hashing<'a>( &self, delta: impl Iterator)>, - alt_hashing: bool, + alt_hashing: Option, ) -> (B::Hash, Self::Transaction) where B::Hash: Ord { - self.caching_state().storage_root(delta, alt_hashing) + self.caching_state().storage_root_with_alt_hashing(delta, alt_hashing) } - fn child_storage_root<'a>( + fn child_storage_root_with_alt_hashing<'a>( &self, child_info: &ChildInfo, delta: impl Iterator)>, - alt_hashing: bool, + alt_hashing: Option, ) -> (B::Hash, bool, Self::Transaction) where B::Hash: Ord { - self.caching_state().child_storage_root(child_info, delta, alt_hashing) + self.caching_state().child_storage_root_with_alt_hashing(child_info, delta, alt_hashing) } fn pairs(&self) -> Vec<(Vec, Vec)> { diff --git a/client/light/src/backend.rs b/client/light/src/backend.rs index d68507f549320..d7ec2df029cf5 100644 --- a/client/light/src/backend.rs +++ b/client/light/src/backend.rs @@ -486,27 +486,28 @@ impl StateBackend for GenesisOrUnavailableState } } - fn storage_root<'a>( + fn storage_root_with_alt_hashing<'a>( &self, delta: impl Iterator)>, - alt_hashing: bool, + alt_hashing: Option, ) -> (H::Out, Self::Transaction) where H::Out: Ord { match *self { - GenesisOrUnavailableState::Genesis(ref state) => - state.storage_root(delta, alt_hashing), + GenesisOrUnavailableState::Genesis(ref state) => state + .storage_root_with_alt_hashing(delta, alt_hashing), GenesisOrUnavailableState::Unavailable => Default::default(), } } - fn child_storage_root<'a>( + fn child_storage_root_with_alt_hashing<'a>( &self, child_info: &ChildInfo, delta: impl Iterator)>, - alt_hashing: bool, + alt_hashing: Option, ) -> (H::Out, bool, Self::Transaction) where H::Out: Ord { match *self { GenesisOrUnavailableState::Genesis(ref state) => { - let (root, is_equal, _) = state.child_storage_root(child_info, delta, alt_hashing); + let (root, is_equal, _) = state + .child_storage_root_with_alt_hashing(child_info, delta, alt_hashing); (root, is_equal, Default::default()) }, GenesisOrUnavailableState::Unavailable => diff --git a/primitives/state-machine/src/backend.rs b/primitives/state-machine/src/backend.rs index 65d0469100af1..913ad4cd7431b 100644 --- a/primitives/state-machine/src/backend.rs +++ b/primitives/state-machine/src/backend.rs @@ -129,8 +129,10 @@ pub trait Backend: sp_std::fmt::Debug { fn storage_root<'a>( &self, delta: impl Iterator)>, - alt_hashing: bool, - ) -> (H::Out, Self::Transaction) where H::Out: Ord; + ) -> (H::Out, Self::Transaction) where H::Out: Ord { + let alt_hashing = self.get_trie_alt_hashing_threshold(); + self.storage_root_with_alt_hashing(delta, alt_hashing) + } /// Calculate the child storage root, with given delta over what is already stored in /// the backend, and produce a "transaction" that can be used to commit. The second argument @@ -139,9 +141,29 @@ pub trait Backend: sp_std::fmt::Debug { &self, child_info: &ChildInfo, delta: impl Iterator)>, - alt_hashing: bool, + ) -> (H::Out, bool, Self::Transaction) where H::Out: Ord { + let alt_hashing = self.get_trie_alt_hashing_threshold(); + self.child_storage_root_with_alt_hashing(child_info, delta, alt_hashing) + } + + /// Helpers function to avoid on query for full storage root. + /// See `storage_root`. + fn storage_root_with_alt_hashing<'a>( + &self, + delta: impl Iterator)>, + alt_hashing: Option, + ) -> (H::Out, Self::Transaction) where H::Out: Ord; + + /// Helpers function to avoid on query for full storage root. + /// See `child_storage_root`. + fn child_storage_root_with_alt_hashing<'a>( + &self, + child_info: &ChildInfo, + delta: impl Iterator)>, + alt_hashing: Option, ) -> (H::Out, bool, Self::Transaction) where H::Out: Ord; + /// Get all key/value pairs into a Vec. fn pairs(&self) -> Vec<(StorageKey, StorageValue)>; @@ -167,10 +189,10 @@ pub trait Backend: sp_std::fmt::Debug { fn as_trie_backend(&mut self) -> Option<&TrieBackend> { None } - /// Calculate the storage root, with given delta over what is already stored /// in the backend, and produce a "transaction" that can be used to commit. /// Does include child storage updates. + /// TODO remove (getting alt_hashing from delta is required) fn full_storage_root<'a>( &self, delta: impl Iterator)>, @@ -179,13 +201,30 @@ pub trait Backend: sp_std::fmt::Debug { impl Iterator)>, )>, ) -> (H::Out, Self::Transaction) where H::Out: Ord + Encode { - let alt_hashing = self.get_trie_alt_hashing_threshold().is_some(); + let alt_hashing = self.get_trie_alt_hashing_threshold(); + self.full_storage_root_with_alt_hashing(delta, child_deltas, alt_hashing) + } + + /// Helpers function to avoid on query for full storage root. + /// See `storage_root`. + fn full_storage_root_with_alt_hashing<'a>( + &self, + delta: impl Iterator)>, + child_deltas: impl Iterator)>, + )>, + alt_hashing: Option, + ) -> (H::Out, Self::Transaction) where H::Out: Ord + Encode { let mut txs: Self::Transaction = Default::default(); let mut child_roots: Vec<_> = Default::default(); // child first for (child_info, child_delta) in child_deltas { - let (child_root, empty, child_txs) = - self.child_storage_root(&child_info, child_delta, alt_hashing); + let (child_root, empty, child_txs) = self.child_storage_root_with_alt_hashing( + &child_info, + child_delta, + alt_hashing, + ); let prefixed_storage_key = child_info.prefixed_storage_key(); txs.consolidate(child_txs); if empty { @@ -194,7 +233,7 @@ pub trait Backend: sp_std::fmt::Debug { child_roots.push((prefixed_storage_key.into_inner(), Some(child_root.encode()))); } } - let (root, parent_txs) = self.storage_root(delta + let (root, parent_txs) = self.storage_root_with_alt_hashing(delta .map(|(k, v)| (&k[..], v.as_ref().map(|v| &v[..]))) .chain( child_roots diff --git a/primitives/state-machine/src/basic.rs b/primitives/state-machine/src/basic.rs index 57158d938ccd0..dcc4aa2cafdd9 100644 --- a/primitives/state-machine/src/basic.rs +++ b/primitives/state-machine/src/basic.rs @@ -281,9 +281,9 @@ impl Externalities for BasicExternalities { } } - let alt_hashing = self.get_trie_alt_hashing_threshold().is_some(); - let layout = if alt_hashing { - Layout::::with_inner_hashing() + let alt_hashing = self.get_trie_alt_hashing_threshold(); + let layout = if let Some(threshold) = alt_hashing { + Layout::::with_inner_hashing(threshold) } else { Layout::::default() }; @@ -296,9 +296,9 @@ impl Externalities for BasicExternalities { ) -> Vec { if let Some(child) = self.inner.children_default.get(child_info.storage_key()) { let delta = child.data.iter().map(|(k, v)| (k.as_ref(), Some(v.as_ref()))); - let alt_hashing = self.get_trie_alt_hashing_threshold().is_some(); + let alt_hashing = self.get_trie_alt_hashing_threshold(); crate::in_memory_backend::new_in_mem::() - .child_storage_root(&child.child_info, delta, alt_hashing).0 + .child_storage_root_with_alt_hashing(&child.child_info, delta, alt_hashing).0 } else { empty_child_trie_root::>() }.encode() diff --git a/primitives/state-machine/src/changes_trie/mod.rs b/primitives/state-machine/src/changes_trie/mod.rs index f2bbf371fb50e..5a2e206956ac9 100644 --- a/primitives/state-machine/src/changes_trie/mod.rs +++ b/primitives/state-machine/src/changes_trie/mod.rs @@ -167,7 +167,12 @@ pub struct TrieBackendStorageAdapter<'a, H: Hasher, Number: BlockNumber>(pub &'a impl<'a, H: Hasher, N: BlockNumber> crate::TrieBackendStorage for TrieBackendStorageAdapter<'a, H, N> { type Overlay = sp_trie::MemoryDB; - fn get(&self, key: &H::Out, prefix: Prefix, _global: bool) -> Result, String> { + fn get( + &self, + key: &H::Out, + prefix: Prefix, + _global: Option, + ) -> Result, String> { match self.0.get(key, prefix) { // change trie do not use meta. Ok(Some(v)) => Ok(Some((v, Default::default()))), diff --git a/primitives/state-machine/src/changes_trie/storage.rs b/primitives/state-machine/src/changes_trie/storage.rs index 115cc5461e1a9..0cea30f0d809c 100644 --- a/primitives/state-machine/src/changes_trie/storage.rs +++ b/primitives/state-machine/src/changes_trie/storage.rs @@ -207,7 +207,12 @@ impl<'a, H, Number> TrieBackendStorage for TrieBackendAdapter<'a, H, Number> { type Overlay = MemoryDB; - fn get(&self, key: &H::Out, prefix: Prefix, _global: bool) -> Result, String> { + fn get( + &self, + key: &H::Out, + prefix: Prefix, + _global: Option, + ) -> Result, String> { match self.storage.get(key, prefix) { // change trie do not use meta. Ok(Some(v)) => Ok(Some((v, Default::default()))), diff --git a/primitives/state-machine/src/ext.rs b/primitives/state-machine/src/ext.rs index 4ecbe17fc3346..01c3eb1e56c27 100644 --- a/primitives/state-machine/src/ext.rs +++ b/primitives/state-machine/src/ext.rs @@ -519,7 +519,10 @@ where return root.encode(); } - let root = self.overlay.storage_root(self.backend, self.storage_transaction_cache); + let root = self.overlay.storage_root( + self.backend, + self.storage_transaction_cache, + ); trace!(target: "state", "{:04x}: Root {}", self.id, HexDisplay::from(&root.as_ref())); root.encode() } @@ -547,8 +550,8 @@ where } else { let root = if let Some((changes, info)) = self.overlay.child_changes(storage_key) { let delta = changes.map(|(k, v)| (k.as_ref(), v.value().map(AsRef::as_ref))); - let alt_hashing = self.get_trie_alt_hashing_threshold().is_some(); - Some(self.backend.child_storage_root(info, delta, alt_hashing)) + let alt_hashing = self.get_trie_alt_hashing_threshold(); + Some(self.backend.child_storage_root_with_alt_hashing(info, delta, alt_hashing)) } else { None }; diff --git a/primitives/state-machine/src/in_memory_backend.rs b/primitives/state-machine/src/in_memory_backend.rs index c5dc881caae91..045558d9f8440 100644 --- a/primitives/state-machine/src/in_memory_backend.rs +++ b/primitives/state-machine/src/in_memory_backend.rs @@ -58,8 +58,8 @@ where &mut self, changes: T, ) { - // Note that in case the threshold is changed, it will not be apply immediately. let (top, child) = changes.into_iter().partition::, _>(|v| v.0.is_none()); + unimplemented!("get alt hashing changes here"); let (root, transaction) = self.full_storage_root( top.iter().map(|(_, v)| v).flatten().map(|(k, v)| (&k[..], v.as_deref())), child.iter() diff --git a/primitives/state-machine/src/lib.rs b/primitives/state-machine/src/lib.rs index d7d6cbc1da731..20efdb1236a1a 100644 --- a/primitives/state-machine/src/lib.rs +++ b/primitives/state-machine/src/lib.rs @@ -855,7 +855,7 @@ mod execution { where H: Hasher, H::Out: Ord + Codec, - MH: sp_trie::MetaHasher, + MH: sp_trie::MetaHasher>, KF: sp_trie::KeyFunction + Send + Sync, { proving_backend.storage(key).map_err(|e| Box::new(e) as Box) diff --git a/primitives/state-machine/src/overlayed_changes/mod.rs b/primitives/state-machine/src/overlayed_changes/mod.rs index 1d3cbb59ba0c1..6d28184501a5f 100644 --- a/primitives/state-machine/src/overlayed_changes/mod.rs +++ b/primitives/state-machine/src/overlayed_changes/mod.rs @@ -614,6 +614,16 @@ impl OverlayedChanges { } } + /// Utility function to get trie inner value hash threshold from + /// backend state or pending changes. + fn get_trie_alt_hashing_threshold>(&self, backend: &B) -> Option { + match self.storage(sp_core::storage::well_known_keys::TRIE_HASHING_CONFIG) { + Some(Some(mut encoded)) => sp_core::storage::trie_threshold_decode(&mut encoded), + Some(None) => None, + None => backend.get_trie_alt_hashing_threshold(), + } + } + /// Generate the storage root using `backend` and all changes /// as seen by the current transaction. /// @@ -625,13 +635,19 @@ impl OverlayedChanges { ) -> H::Out where H::Out: Ord + Encode, { + let alt_hashing = self.get_trie_alt_hashing_threshold(backend); + let delta = self.changes().map(|(k, v)| (&k[..], v.value().map(|v| &v[..]))); let child_delta = self.children() .map(|(changes, info)| (info, changes.map( |(k, v)| (&k[..], v.value().map(|v| &v[..])) ))); - let (root, transaction) = backend.full_storage_root(delta, child_delta); + let (root, transaction) = backend.full_storage_root_with_alt_hashing( + delta, + child_delta, + alt_hashing, + ); cache.transaction = Some(transaction); cache.transaction_storage_root = Some(root); diff --git a/primitives/state-machine/src/proving_backend.rs b/primitives/state-machine/src/proving_backend.rs index d4a1ec66523dd..9b3eb9cc5be58 100644 --- a/primitives/state-machine/src/proving_backend.rs +++ b/primitives/state-machine/src/proving_backend.rs @@ -242,7 +242,7 @@ impl<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> TrieBackendStorage { type Overlay = S::Overlay; - fn get(&self, key: &H::Out, prefix: Prefix, global: bool) -> Result, String> { + fn get(&self, key: &H::Out, prefix: Prefix, global: Option) -> Result, String> { if let Some(v) = self.proof_recorder.get(key) { return Ok(v); } @@ -340,21 +340,21 @@ impl<'a, S, H> Backend for ProvingBackend<'a, S, H> self.0.child_keys(child_info, prefix) } - fn storage_root<'b>( + fn storage_root_with_alt_hashing<'b>( &self, delta: impl Iterator)>, - alt_hashing: bool, + alt_hashing: Option, ) -> (H::Out, Self::Transaction) where H::Out: Ord { - self.0.storage_root(delta, alt_hashing) + self.0.storage_root_with_alt_hashing(delta, alt_hashing) } - fn child_storage_root<'b>( + fn child_storage_root_with_alt_hashing<'b>( &self, child_info: &ChildInfo, delta: impl Iterator)>, - alt_hashing: bool, + alt_hashing: Option, ) -> (H::Out, bool, Self::Transaction) where H::Out: Ord { - self.0.child_storage_root(child_info, delta, alt_hashing) + self.0.child_storage_root_with_alt_hashing(child_info, delta, alt_hashing) } fn register_overlay_stats(&self, _stats: &crate::stats::StateMachineStats) { } diff --git a/primitives/state-machine/src/trie_backend.rs b/primitives/state-machine/src/trie_backend.rs index 52cadac871aa0..4f7001800cbd0 100644 --- a/primitives/state-machine/src/trie_backend.rs +++ b/primitives/state-machine/src/trie_backend.rs @@ -168,10 +168,10 @@ impl, H: Hasher> Backend for TrieBackend where collect_all().map_err(|e| debug!(target: "trie", "Error extracting trie keys: {}", e)).unwrap_or_default() } - fn storage_root<'a>( + fn storage_root_with_alt_hashing<'a>( &self, delta: impl Iterator)>, - use_inner_hash_value: bool, + use_inner_hash_value: Option, ) -> (H::Out, Self::Transaction) where H::Out: Ord { let mut write_overlay = S::Overlay::default(); let mut root = *self.essence.root(); @@ -182,8 +182,8 @@ impl, H: Hasher> Backend for TrieBackend where &mut write_overlay, ); let res = || { - let layout = if use_inner_hash_value { - sp_trie::Layout::with_inner_hashing() + let layout = if let Some(threshold) = use_inner_hash_value { + sp_trie::Layout::with_inner_hashing(threshold) } else { sp_trie::Layout::default() }; @@ -199,17 +199,17 @@ impl, H: Hasher> Backend for TrieBackend where (root, write_overlay) } - fn child_storage_root<'a>( + fn child_storage_root_with_alt_hashing<'a>( &self, child_info: &ChildInfo, delta: impl Iterator)>, - use_inner_hash_value: bool, + use_inner_hash_value: Option, ) -> (H::Out, bool, Self::Transaction) where H::Out: Ord { let default_root = match child_info.child_type() { ChildType::ParentKeyId => empty_child_trie_root::>() }; - let layout = if use_inner_hash_value { - sp_trie::Layout::with_inner_hashing() + let layout = if let Some(threshold) = use_inner_hash_value { + sp_trie::Layout::with_inner_hashing(threshold) } else { sp_trie::Layout::default() }; diff --git a/primitives/state-machine/src/trie_backend_essence.rs b/primitives/state-machine/src/trie_backend_essence.rs index e7d41a11eb64b..63b1ccc2d9698 100644 --- a/primitives/state-machine/src/trie_backend_essence.rs +++ b/primitives/state-machine/src/trie_backend_essence.rs @@ -44,7 +44,12 @@ type Result = sp_std::result::Result; /// Patricia trie-based storage trait. pub trait Storage: Send + Sync { /// Get a trie node. - fn get(&self, key: &H::Out, prefix: Prefix, inner_hash: bool) -> Result>; + fn get( + &self, + key: &H::Out, + prefix: Prefix, + alt_hashing: Option, + ) -> Result>; /// Call back when value get accessed in trie. fn access_from(&self, key: &H::Out); } @@ -300,11 +305,11 @@ pub(crate) struct Ephemeral<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> { overlay: &'a mut S::Overlay, } -impl<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> hash_db::AsHashDB +impl<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> hash_db::AsHashDB> for Ephemeral<'a, S, H> { - fn as_hash_db<'b>(&'b self) -> &'b (dyn hash_db::HashDB + 'b) { self } - fn as_hash_db_mut<'b>(&'b mut self) -> &'b mut (dyn hash_db::HashDB + 'b) { self } + fn as_hash_db<'b>(&'b self) -> &'b (dyn hash_db::HashDB> + 'b) { self } + fn as_hash_db_mut<'b>(&'b mut self) -> &'b mut (dyn hash_db::HashDB> + 'b) { self } } impl<'a, S: TrieBackendStorage, H: Hasher> Ephemeral<'a, S, H> { @@ -316,14 +321,14 @@ impl<'a, S: TrieBackendStorage, H: Hasher> Ephemeral<'a, S, H> { } } -impl<'a, S: 'a + TrieBackendStorage, H: Hasher> hash_db::HashDB +impl<'a, S: 'a + TrieBackendStorage, H: Hasher> hash_db::HashDB> for Ephemeral<'a, S, H> { fn get(&self, key: &H::Out, prefix: Prefix) -> Option { Self::get_with_meta(self, key, prefix, Default::default()).map(|r| r.0) } - fn get_with_meta(&self, key: &H::Out, prefix: Prefix, global: bool) -> Option<(DBValue, TrieMeta)> { + fn get_with_meta(&self, key: &H::Out, prefix: Prefix, global: Option) -> Option<(DBValue, TrieMeta)> { if let Some(val) = hash_db::HashDB::get_with_meta(self.overlay, key, prefix, global) { Some(val) } else { @@ -369,14 +374,14 @@ impl<'a, S: 'a + TrieBackendStorage, H: Hasher> hash_db::HashDB, H: Hasher> hash_db::HashDBRef +impl<'a, S: 'a + TrieBackendStorage, H: Hasher> hash_db::HashDBRef> for Ephemeral<'a, S, H> { fn get(&self, key: &H::Out, prefix: Prefix) -> Option { hash_db::HashDB::get(self, key, prefix) } - fn get_with_meta(&self, key: &H::Out, prefix: Prefix, global: bool) -> Option<(DBValue, TrieMeta)> { + fn get_with_meta(&self, key: &H::Out, prefix: Prefix, global: Option) -> Option<(DBValue, TrieMeta)> { hash_db::HashDB::get_with_meta(self, key, prefix, global) } @@ -392,9 +397,9 @@ impl<'a, S: 'a + TrieBackendStorage, H: Hasher> hash_db::HashDBRef: Send + Sync { /// Type of in-memory overlay. - type Overlay: hash_db::HashDB + Default + Consolidate; + type Overlay: hash_db::HashDB> + Default + Consolidate; /// Get the value stored at key. - fn get(&self, key: &H::Out, prefix: Prefix, global: bool) -> Result>; + fn get(&self, key: &H::Out, prefix: Prefix, global: Option) -> Result>; /// Call back when value get accessed in trie. fn access_from(&self, key: &H::Out); } @@ -404,7 +409,12 @@ pub trait TrieBackendStorage: Send + Sync { impl TrieBackendStorage for Arc> { type Overlay = PrefixedMemoryDB; - fn get(&self, key: &H::Out, prefix: Prefix, global: bool) -> Result> { + fn get( + &self, + key: &H::Out, + prefix: Prefix, + global: Option, + ) -> Result> { Storage::::get(self.deref(), key, prefix, global) } @@ -416,12 +426,17 @@ impl TrieBackendStorage for Arc> { impl TrieBackendStorage for sp_trie::GenericMemoryDB where H: Hasher, - MH: sp_trie::MetaHasher, + MH: sp_trie::MetaHasher>, KF: sp_trie::KeyFunction + Send + Sync, { type Overlay = Self; - fn get(&self, key: &H::Out, prefix: Prefix, global: bool) -> Result> { + fn get( + &self, + key: &H::Out, + prefix: Prefix, + global: Option, + ) -> Result> { Ok(hash_db::HashDB::get_with_meta(self, key, prefix, global)) } @@ -430,21 +445,26 @@ impl TrieBackendStorage for sp_trie::GenericMemoryDB } } -impl, H: Hasher> hash_db::AsHashDB +impl, H: Hasher> hash_db::AsHashDB> for TrieBackendEssence { - fn as_hash_db<'b>(&'b self) -> &'b (dyn hash_db::HashDB + 'b) { self } - fn as_hash_db_mut<'b>(&'b mut self) -> &'b mut (dyn hash_db::HashDB + 'b) { self } + fn as_hash_db<'b>(&'b self) -> &'b (dyn hash_db::HashDB> + 'b) { self } + fn as_hash_db_mut<'b>(&'b mut self) -> &'b mut (dyn hash_db::HashDB> + 'b) { self } } -impl, H: Hasher> hash_db::HashDB +impl, H: Hasher> hash_db::HashDB> for TrieBackendEssence { fn get(&self, key: &H::Out, prefix: Prefix) -> Option { self.get_with_meta(key, prefix, Default::default()).map(|r| r.0) } - fn get_with_meta(&self, key: &H::Out, prefix: Prefix, global: bool) -> Option<(DBValue, TrieMeta)> { + fn get_with_meta( + &self, + key: &H::Out, + prefix: Prefix, + global: Option, + ) -> Option<(DBValue, TrieMeta)> { if *key == self.empty { return Some(([0u8].to_vec(), ::meta_for_empty(global))) } @@ -484,14 +504,14 @@ impl, H: Hasher> hash_db::HashDB, H: Hasher> hash_db::HashDBRef +impl, H: Hasher> hash_db::HashDBRef> for TrieBackendEssence { fn get(&self, key: &H::Out, prefix: Prefix) -> Option { hash_db::HashDB::get(self, key, prefix) } - fn get_with_meta(&self, key: &H::Out, prefix: Prefix, global: bool) -> Option<(DBValue, TrieMeta)> { + fn get_with_meta(&self, key: &H::Out, prefix: Prefix, global: Option) -> Option<(DBValue, TrieMeta)> { hash_db::HashDB::get_with_meta(self, key, prefix, global) } diff --git a/primitives/trie/src/lib.rs b/primitives/trie/src/lib.rs index 283147602e31e..1b8acc9a796a3 100644 --- a/primitives/trie/src/lib.rs +++ b/primitives/trie/src/lib.rs @@ -57,24 +57,27 @@ pub struct TrieMeta { pub range: Option>, /// Defined in the trie layout, when used with /// `TrieDbMut` it switch nodes to alternative hashing - /// method by setting `do_value_hash` to true. - pub try_inner_hashing: bool, + /// method by defining the threshold to use with alternative + /// hashing. + pub try_inner_hashing: Option, + /// Flag indicating alternative value hash is currently use + /// or will be use. + pub apply_inner_hashing: bool, /// Does current encoded contains a hash instead of /// a value (information stored in meta for proofs). pub contain_hash: bool, - /// Flag indicating alternative value hash will be use. - pub apply_inner_hashing: bool, /// Record if a value was accessed, this is /// set as accessed by defalult, but can be /// change on access explicitely: `HashDB::get_with_meta`. /// and reset on access explicitely: `HashDB::access_from`. - /// TODO!! remove from meta: only use in proof recorder context. + /// TODO!! could be remove from meta: only use in proof recorder context. + /// But does not add memory usage here. pub unused_value: bool, } impl Meta for TrieMeta { /// When true apply inner hashing of value. - type GlobalMeta = bool; + type GlobalMeta = Option; /// When true apply inner hashing of value. type StateMeta = bool; @@ -156,8 +159,9 @@ impl Meta for TrieMeta { ValuePlan::NoValue => return, }; - self.apply_inner_hashing = self.try_inner_hashing - && range.end - range.start >= trie_constants::INNER_HASH_TRESHOLD; + self.apply_inner_hashing = self.try_inner_hashing.as_ref().map(|threshold| + range.end - range.start >= *threshold as usize + ).unwrap_or(false); self.range = Some(range); self.contain_hash = contain_hash; } @@ -201,7 +205,7 @@ impl TrieMeta { } /// substrate trie layout -pub struct Layout(bool, sp_std::marker::PhantomData<(H, M)>); +pub struct Layout(Option, sp_std::marker::PhantomData<(H, M)>); impl fmt::Debug for Layout { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { @@ -217,23 +221,23 @@ impl Clone for Layout { impl Default for Layout { fn default() -> Self { - Layout(false, sp_std::marker::PhantomData) + Layout(None, sp_std::marker::PhantomData) } } impl Layout { /// Layout with inner hashing active. /// Will flag trie for hashing. /// TODO rename inner -> alt - pub fn with_inner_hashing() -> Self { - Layout(true, sp_std::marker::PhantomData) + pub fn with_inner_hashing(threshold: u32) -> Self { + Layout(Some(threshold), sp_std::marker::PhantomData) } } impl TrieLayout for Layout where H: Hasher, - M: MetaHasher, - M::Meta: Meta, + M: MetaHasher>, + M::Meta: Meta, StateMeta = bool>, { const USE_EXTENSION: bool = false; const ALLOW_EMPTY: bool = true; @@ -273,7 +277,7 @@ impl MetaHasher for StateHasher H: Hasher, { type Meta = TrieMeta; - type GlobalMeta = bool; + type GlobalMeta = Option; fn hash(value: &[u8], meta: &Self::Meta) -> H::Out { match &meta { @@ -335,7 +339,7 @@ impl MetaHasher for StateHasher unused_value: contain_hash, contain_hash, apply_inner_hashing: false, - try_inner_hashing: false, + try_inner_hashing: None, }; meta.set_global_meta(global_meta); (stored, meta) @@ -353,8 +357,8 @@ impl MetaHasher for StateHasher impl TrieConfiguration for Layout where H: Hasher, - M: MetaHasher, - M::Meta: Meta, + M: MetaHasher>, + M::Meta: Meta, StateMeta = bool>, { fn trie_root(&self, input: I) -> ::Out where I: IntoIterator, @@ -812,18 +816,6 @@ pub fn resolve_encoded_meta(entry: &mut (DBValue, TrieMeta)) { /// Constants used into trie simplification codec. mod trie_constants { - /// Treshold for using hash of value instead of value - /// in encoded trie node when flagged. - /// TODO design would be to make it the global meta, but then - /// when serializing proof we would need to attach it (no way to - /// hash the nodes otherwhise), which would - /// break proof format. - /// TODO attaching to storage proof in a compatible way could be - /// achieve by using a escaped header in first or last element of proof - /// and write it after. - /// TODO 33 is not good switch to 32 + 1 + 1: 34 (avoid hashing stored hash for a 1 byte gain). - /// TODO replace with sp_storage test one. - pub const INNER_HASH_TRESHOLD: usize = 33; const FIRST_PREFIX: u8 = 0b_00 << 6; /// In proof this header is used when only hashed value is stored. pub const DEAD_HEADER_META_HASHED_VALUE: u8 = EMPTY_TRIE | 0b_00_01; @@ -895,12 +887,6 @@ mod tests { } fn check_input(input: &Vec<(&[u8], &[u8])>) { -// TODO remove this iter - let layout = Layout::with_inner_hashing(); - check_equivalent::(input, layout.clone()); - check_iteration::(input, layout.clone()); - - let layout = Layout::default(); check_equivalent::(input, layout.clone()); check_iteration::(input, layout); diff --git a/primitives/trie/src/node_codec.rs b/primitives/trie/src/node_codec.rs index 61bcf789d8817..9e2931486cba0 100644 --- a/primitives/trie/src/node_codec.rs +++ b/primitives/trie/src/node_codec.rs @@ -186,7 +186,7 @@ impl NodeCodec { impl NodeCodecT for NodeCodec where H: Hasher, - M: Meta, + M: Meta>, { type Error = Error; type HashOut = H::Out; @@ -225,7 +225,9 @@ impl NodeCodecT for NodeCodec // With fix inner hashing alt hash can be use with all node, but // that is not better (encoding can use an additional nibble byte // sometime). - let mut output = if meta.extract_global_meta() && value_do_hash(&value) { + let mut output = if meta.extract_global_meta().as_ref().map(|threshold| + value_do_hash(&value, threshold) + ).unwrap_or(false) { partial_encode(partial, NodeKind::AltHashLeaf) } else { partial_encode(partial, NodeKind::Leaf) @@ -275,7 +277,9 @@ impl NodeCodecT for NodeCodec value: Value, meta: &mut M, ) -> Vec { - let mut output = match (&value, meta.extract_global_meta() && value_do_hash(&value)) { + let mut output = match (&value, meta.extract_global_meta().as_ref().map(|threshold| + value_do_hash(&value, threshold) + ).unwrap_or(false)) { (&Value::NoValue, _) => { partial_from_iterator_encode(partial, number_nibble, NodeKind::BranchNoValue) }, @@ -327,9 +331,9 @@ impl NodeCodecT for NodeCodec // utils -fn value_do_hash(val: &Value) -> bool { +fn value_do_hash(val: &Value, threshold: &u32) -> bool { if let Value::Value(val) = val { - val.encoded_size() >= trie_constants::INNER_HASH_TRESHOLD + val.encoded_size() >= *threshold as usize } else { false } diff --git a/primitives/trie/src/trie_stream.rs b/primitives/trie/src/trie_stream.rs index 4e383c9d5763c..2be98fad76ab6 100644 --- a/primitives/trie/src/trie_stream.rs +++ b/primitives/trie/src/trie_stream.rs @@ -35,7 +35,7 @@ pub struct TrieStream { /// Current node buffer. buffer: Vec, /// Global trie alt hashing activation. - inner_value_hashing: bool, + inner_value_hashing: Option, /// For current node, do we use alt hashing. apply_inner_hashing: bool, /// Keep trace of position of encoded value. @@ -76,9 +76,9 @@ fn fuse_nibbles_node<'a>(nibbles: &'a [u8], kind: NodeKind) -> impl Iterator; - fn new(meta: bool) -> Self { + fn new(meta: Option) -> Self { Self { buffer: Vec::new(), inner_value_hashing: meta, @@ -92,7 +92,9 @@ impl trie_root::TrieStream for TrieStream { } fn append_leaf(&mut self, key: &[u8], value: &[u8]) { - self.apply_inner_hashing = self.inner_value_hashing && value_do_hash(value); + self.apply_inner_hashing = self.inner_value_hashing.as_ref().map(|threshold| + value_do_hash(value, threshold) + ).unwrap_or(false); let kind = if self.apply_inner_hashing { NodeKind::AltHashLeaf } else { @@ -113,7 +115,9 @@ impl trie_root::TrieStream for TrieStream { ) { if let Some(partial) = maybe_partial { if let Some(value) = maybe_value { - self.apply_inner_hashing = self.inner_value_hashing && value_do_hash(value); + self.apply_inner_hashing = self.inner_value_hashing.as_ref().map(|threshold| + value_do_hash(value, threshold) + ).unwrap_or(false); let kind = if self.apply_inner_hashing { NodeKind::AltHashBranchWithValue } else { @@ -155,7 +159,7 @@ impl trie_root::TrieStream for TrieStream { contain_hash: false, // Using `inner_value_hashing` instead to check this. // And unused in hasher. - try_inner_hashing: false, + try_inner_hashing: None, apply_inner_hashing: true, }; >>::hash(&data, &meta).as_ref().encode_to(&mut self.buffer); @@ -174,7 +178,7 @@ impl trie_root::TrieStream for TrieStream { range: range, unused_value: false, contain_hash: false, - try_inner_hashing: false, + try_inner_hashing: None, apply_inner_hashing: true, }; @@ -207,6 +211,6 @@ fn branch_node_buffered(has_value: bool, has_children: I, output: &mut[u8]) Bitmap::encode(has_children, &mut output[1..]); } -fn value_do_hash(val: &[u8]) -> bool { - val.encoded_size() >= trie_constants::INNER_HASH_TRESHOLD +fn value_do_hash(val: &[u8], threshold: &u32) -> bool { + val.encoded_size() >= *threshold as usize } From 74fb9c8eab613fb21964a153218a07f8501cb979 Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 10 Jun 2021 14:51:17 +0200 Subject: [PATCH 048/127] refactoring to use state from backend (not possible payload changes). --- client/api/src/in_mem.rs | 2 +- client/db/src/bench.rs | 10 ++-- client/db/src/lib.rs | 10 ++-- client/db/src/storage_cache.rs | 16 +++---- client/light/src/backend.rs | 11 +++-- primitives/externalities/src/lib.rs | 2 + primitives/state-machine/src/backend.rs | 48 +++---------------- primitives/state-machine/src/basic.rs | 20 ++++---- primitives/state-machine/src/ext.rs | 6 +-- .../state-machine/src/in_memory_backend.rs | 15 ++++-- .../src/overlayed_changes/mod.rs | 14 +----- .../state-machine/src/proving_backend.rs | 8 ++-- primitives/state-machine/src/trie_backend.rs | 20 +++++++- primitives/storage/src/lib.rs | 16 ++++++- 14 files changed, 97 insertions(+), 101 deletions(-) diff --git a/client/api/src/in_mem.rs b/client/api/src/in_mem.rs index 93ac41b3c9dce..899fdd5e66690 100644 --- a/client/api/src/in_mem.rs +++ b/client/api/src/in_mem.rs @@ -574,7 +574,7 @@ impl backend::BlockImportOperation for BlockImportOperatio ); let alt_hashing = storage.get_trie_alt_hashing_threshold(); - let (root, transaction) = self.old_state.full_storage_root_with_alt_hashing( + let (root, transaction) = self.old_state.full_storage_root( storage.top.iter().map(|(k, v)| (k.as_ref(), Some(v.as_ref()))), child_delta, alt_hashing, diff --git a/client/db/src/bench.rs b/client/db/src/bench.rs index 0201cd0b450a0..af626d3af7c2f 100644 --- a/client/db/src/bench.rs +++ b/client/db/src/bench.rs @@ -162,7 +162,7 @@ impl BenchmarkingState { )); let alt_hashing = genesis.get_trie_alt_hashing_threshold(); let (root, transaction): (B::Hash, _) = state.state.borrow_mut().as_mut().unwrap() - .full_storage_root_with_alt_hashing( + .full_storage_root( genesis.top.iter().map(|(k, v)| (k.as_ref(), Some(v.as_ref()))), child_delta, alt_hashing, @@ -404,16 +404,16 @@ impl StateBackend> for BenchmarkingState { } } - fn storage_root_with_alt_hashing<'a>( + fn storage_root<'a>( &self, delta: impl Iterator)>, alt_hashing: Option, ) -> (B::Hash, Self::Transaction) where B::Hash: Ord { self.state.borrow().as_ref() - .map_or(Default::default(), |s| s.storage_root_with_alt_hashing(delta, alt_hashing)) + .map_or(Default::default(), |s| s.storage_root(delta, alt_hashing)) } - fn child_storage_root_with_alt_hashing<'a>( + fn child_storage_root<'a>( &self, child_info: &ChildInfo, delta: impl Iterator)>, @@ -421,7 +421,7 @@ impl StateBackend> for BenchmarkingState { ) -> (B::Hash, bool, Self::Transaction) where B::Hash: Ord { self.state.borrow().as_ref().map_or( Default::default(), - |s| s.child_storage_root_with_alt_hashing(child_info, delta, alt_hashing), + |s| s.child_storage_root(child_info, delta, alt_hashing), ) } diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index 9b3d525ef9eba..5fa44f8c2e1c3 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -224,21 +224,21 @@ impl StateBackend> for RefTrackingState { self.state.for_child_keys_with_prefix(child_info, prefix, f) } - fn storage_root_with_alt_hashing<'a>( + fn storage_root<'a>( &self, delta: impl Iterator)>, alt_hashing: Option, ) -> (B::Hash, Self::Transaction) where B::Hash: Ord { - self.state.storage_root_with_alt_hashing(delta, alt_hashing) + self.state.storage_root(delta, alt_hashing) } - fn child_storage_root_with_alt_hashing<'a>( + fn child_storage_root<'a>( &self, child_info: &ChildInfo, delta: impl Iterator)>, alt_hashing: Option, ) -> (B::Hash, bool, Self::Transaction) where B::Hash: Ord { - self.state.child_storage_root_with_alt_hashing(child_info, delta, alt_hashing) + self.state.child_storage_root(child_info, delta, alt_hashing) } fn pairs(&self) -> Vec<(Vec, Vec)> { @@ -779,7 +779,7 @@ impl sc_client_api::backend::BlockImportOperation for Bloc let mut changes_trie_config: Option = None; let alt_hashing = storage.get_trie_alt_hashing_threshold(); - let (root, transaction) = self.old_state.full_storage_root_with_alt_hashing( + let (root, transaction) = self.old_state.full_storage_root( storage.top.iter().map(|(k, v)| { if &k[..] == well_known_keys::CHANGES_TRIE_CONFIG { changes_trie_config = Some( diff --git a/client/db/src/storage_cache.rs b/client/db/src/storage_cache.rs index bf929732caa2e..b249ea14748ac 100644 --- a/client/db/src/storage_cache.rs +++ b/client/db/src/storage_cache.rs @@ -642,21 +642,21 @@ impl>, B: BlockT> StateBackend> for Cachin self.state.for_child_keys_with_prefix(child_info, prefix, f) } - fn storage_root_with_alt_hashing<'a>( + fn storage_root<'a>( &self, delta: impl Iterator)>, alt_hashing: Option, ) -> (B::Hash, Self::Transaction) where B::Hash: Ord { - self.state.storage_root_with_alt_hashing(delta, alt_hashing) + self.state.storage_root(delta, alt_hashing) } - fn child_storage_root_with_alt_hashing<'a>( + fn child_storage_root<'a>( &self, child_info: &ChildInfo, delta: impl Iterator)>, alt_hashing: Option, ) -> (B::Hash, bool, Self::Transaction) where B::Hash: Ord { - self.state.child_storage_root_with_alt_hashing(child_info, delta, alt_hashing) + self.state.child_storage_root(child_info, delta, alt_hashing) } fn pairs(&self) -> Vec<(Vec, Vec)> { @@ -826,21 +826,21 @@ impl>, B: BlockT> StateBackend> for Syncin self.caching_state().for_child_keys_with_prefix(child_info, prefix, f) } - fn storage_root_with_alt_hashing<'a>( + fn storage_root<'a>( &self, delta: impl Iterator)>, alt_hashing: Option, ) -> (B::Hash, Self::Transaction) where B::Hash: Ord { - self.caching_state().storage_root_with_alt_hashing(delta, alt_hashing) + self.caching_state().storage_root(delta, alt_hashing) } - fn child_storage_root_with_alt_hashing<'a>( + fn child_storage_root<'a>( &self, child_info: &ChildInfo, delta: impl Iterator)>, alt_hashing: Option, ) -> (B::Hash, bool, Self::Transaction) where B::Hash: Ord { - self.caching_state().child_storage_root_with_alt_hashing(child_info, delta, alt_hashing) + self.caching_state().child_storage_root(child_info, delta, alt_hashing) } fn pairs(&self) -> Vec<(Vec, Vec)> { diff --git a/client/light/src/backend.rs b/client/light/src/backend.rs index d7ec2df029cf5..7b07477012d72 100644 --- a/client/light/src/backend.rs +++ b/client/light/src/backend.rs @@ -324,6 +324,7 @@ impl BlockImportOperation for ImportOperation fn reset_storage(&mut self, input: Storage) -> ClientResult { check_genesis_storage(&input)?; + let alt_hashing = input.get_trie_alt_hashing_threshold(); // changes trie configuration let changes_trie_config = input.top.iter() .find(|(k, _)| &k[..] == well_known_keys::CHANGES_TRIE_CONFIG) @@ -346,7 +347,7 @@ impl BlockImportOperation for ImportOperation } let storage_update = InMemoryBackend::from(storage); - let (storage_root, _) = storage_update.full_storage_root(std::iter::empty(), child_delta); + let (storage_root, _) = storage_update.full_storage_root(std::iter::empty(), child_delta, alt_hashing); self.storage_update = Some(storage_update); Ok(storage_root) @@ -486,19 +487,19 @@ impl StateBackend for GenesisOrUnavailableState } } - fn storage_root_with_alt_hashing<'a>( + fn storage_root<'a>( &self, delta: impl Iterator)>, alt_hashing: Option, ) -> (H::Out, Self::Transaction) where H::Out: Ord { match *self { GenesisOrUnavailableState::Genesis(ref state) => state - .storage_root_with_alt_hashing(delta, alt_hashing), + .storage_root(delta, alt_hashing), GenesisOrUnavailableState::Unavailable => Default::default(), } } - fn child_storage_root_with_alt_hashing<'a>( + fn child_storage_root<'a>( &self, child_info: &ChildInfo, delta: impl Iterator)>, @@ -507,7 +508,7 @@ impl StateBackend for GenesisOrUnavailableState match *self { GenesisOrUnavailableState::Genesis(ref state) => { let (root, is_equal, _) = state - .child_storage_root_with_alt_hashing(child_info, delta, alt_hashing); + .child_storage_root(child_info, delta, alt_hashing); (root, is_equal, Default::default()) }, GenesisOrUnavailableState::Unavailable => diff --git a/primitives/externalities/src/lib.rs b/primitives/externalities/src/lib.rs index 351d50bd90e60..d902df436889f 100644 --- a/primitives/externalities/src/lib.rs +++ b/primitives/externalities/src/lib.rs @@ -326,12 +326,14 @@ impl ExternalitiesExt for &mut dyn Externalities { /// Helpers method for the [`Externalities`] trait. pub trait ExternalitiesHelpers: Externalities { + /* No we use backend in this case TODO remove function /// Utility function to get trie inner value hash threshold from /// backend state or pending changes. fn get_trie_alt_hashing_threshold(&self) -> Option { self.storage(sp_storage::well_known_keys::TRIE_HASHING_CONFIG) .and_then(|encoded| sp_storage::trie_threshold_decode(&mut encoded.as_slice())) } + */ /// Utility function to modify trie inner value hash threshold. fn modify_trie_alt_hashing_threshold(&mut self, threshold: Option) { diff --git a/primitives/state-machine/src/backend.rs b/primitives/state-machine/src/backend.rs index 913ad4cd7431b..4976d6a43d5db 100644 --- a/primitives/state-machine/src/backend.rs +++ b/primitives/state-machine/src/backend.rs @@ -126,44 +126,24 @@ pub trait Backend: sp_std::fmt::Debug { /// `alt_hashing` indicate if trie state should apply alternate hashing /// scheme (inner value hashed). /// Does not include child storage updates. + /// Alt hashing paremeter must contain possible changes from delta. fn storage_root<'a>( &self, delta: impl Iterator)>, - ) -> (H::Out, Self::Transaction) where H::Out: Ord { - let alt_hashing = self.get_trie_alt_hashing_threshold(); - self.storage_root_with_alt_hashing(delta, alt_hashing) - } + alt_hashing: Option, + ) -> (H::Out, Self::Transaction) where H::Out: Ord; /// Calculate the child storage root, with given delta over what is already stored in /// the backend, and produce a "transaction" that can be used to commit. The second argument /// is true if child storage root equals default storage root. + /// Alt hashing paremeter must contain possible changes from delta. fn child_storage_root<'a>( &self, child_info: &ChildInfo, delta: impl Iterator)>, - ) -> (H::Out, bool, Self::Transaction) where H::Out: Ord { - let alt_hashing = self.get_trie_alt_hashing_threshold(); - self.child_storage_root_with_alt_hashing(child_info, delta, alt_hashing) - } - - /// Helpers function to avoid on query for full storage root. - /// See `storage_root`. - fn storage_root_with_alt_hashing<'a>( - &self, - delta: impl Iterator)>, - alt_hashing: Option, - ) -> (H::Out, Self::Transaction) where H::Out: Ord; - - /// Helpers function to avoid on query for full storage root. - /// See `child_storage_root`. - fn child_storage_root_with_alt_hashing<'a>( - &self, - child_info: &ChildInfo, - delta: impl Iterator)>, alt_hashing: Option, ) -> (H::Out, bool, Self::Transaction) where H::Out: Ord; - /// Get all key/value pairs into a Vec. fn pairs(&self) -> Vec<(StorageKey, StorageValue)>; @@ -192,7 +172,7 @@ pub trait Backend: sp_std::fmt::Debug { /// Calculate the storage root, with given delta over what is already stored /// in the backend, and produce a "transaction" that can be used to commit. /// Does include child storage updates. - /// TODO remove (getting alt_hashing from delta is required) + /// Alt hashing paremeter must contain possible changes from delta. fn full_storage_root<'a>( &self, delta: impl Iterator)>, @@ -200,27 +180,13 @@ pub trait Backend: sp_std::fmt::Debug { &'a ChildInfo, impl Iterator)>, )>, - ) -> (H::Out, Self::Transaction) where H::Out: Ord + Encode { - let alt_hashing = self.get_trie_alt_hashing_threshold(); - self.full_storage_root_with_alt_hashing(delta, child_deltas, alt_hashing) - } - - /// Helpers function to avoid on query for full storage root. - /// See `storage_root`. - fn full_storage_root_with_alt_hashing<'a>( - &self, - delta: impl Iterator)>, - child_deltas: impl Iterator)>, - )>, alt_hashing: Option, ) -> (H::Out, Self::Transaction) where H::Out: Ord + Encode { let mut txs: Self::Transaction = Default::default(); let mut child_roots: Vec<_> = Default::default(); // child first for (child_info, child_delta) in child_deltas { - let (child_root, empty, child_txs) = self.child_storage_root_with_alt_hashing( + let (child_root, empty, child_txs) = self.child_storage_root( &child_info, child_delta, alt_hashing, @@ -233,7 +199,7 @@ pub trait Backend: sp_std::fmt::Debug { child_roots.push((prefixed_storage_key.into_inner(), Some(child_root.encode()))); } } - let (root, parent_txs) = self.storage_root_with_alt_hashing(delta + let (root, parent_txs) = self.storage_root(delta .map(|(k, v)| (&k[..], v.as_ref().map(|v| &v[..]))) .chain( child_roots diff --git a/primitives/state-machine/src/basic.rs b/primitives/state-machine/src/basic.rs index dcc4aa2cafdd9..4a691e4eb208b 100644 --- a/primitives/state-machine/src/basic.rs +++ b/primitives/state-machine/src/basic.rs @@ -33,19 +33,21 @@ use sp_core::{ }; use log::warn; use codec::Encode; -use sp_externalities::{Extensions, Extension, ExternalitiesHelpers}; +use sp_externalities::{Extensions, Extension}; /// Simple Map-based Externalities impl. #[derive(Debug)] pub struct BasicExternalities { inner: Storage, extensions: Extensions, + alt_hashing: Option, } impl BasicExternalities { /// Create a new instance of `BasicExternalities` pub fn new(inner: Storage) -> Self { - BasicExternalities { inner, extensions: Default::default() } + let alt_hashing = inner.get_trie_alt_hashing_threshold(); + BasicExternalities { inner, extensions: Default::default(), alt_hashing } } /// New basic externalities with empty storage. @@ -70,12 +72,14 @@ impl BasicExternalities { storage: &mut sp_core::storage::Storage, f: impl FnOnce() -> R, ) -> R { + let alt_hashing = storage.get_trie_alt_hashing_threshold(); let mut ext = Self { inner: Storage { top: std::mem::take(&mut storage.top), children_default: std::mem::take(&mut storage.children_default), }, extensions: Default::default(), + alt_hashing, }; let r = ext.execute_with(f); @@ -124,12 +128,14 @@ impl Default for BasicExternalities { impl From> for BasicExternalities { fn from(hashmap: BTreeMap) -> Self { + let alt_hashing = sp_core::storage::alt_hashing::get_trie_alt_hashing_threshold(&hashmap); BasicExternalities { inner: Storage { top: hashmap, children_default: Default::default(), }, extensions: Default::default(), + alt_hashing, } } } @@ -281,9 +287,8 @@ impl Externalities for BasicExternalities { } } - let alt_hashing = self.get_trie_alt_hashing_threshold(); - let layout = if let Some(threshold) = alt_hashing { - Layout::::with_inner_hashing(threshold) + let layout = if let Some(threshold) = self.alt_hashing.as_ref() { + Layout::::with_inner_hashing(*threshold) } else { Layout::::default() }; @@ -296,9 +301,8 @@ impl Externalities for BasicExternalities { ) -> Vec { if let Some(child) = self.inner.children_default.get(child_info.storage_key()) { let delta = child.data.iter().map(|(k, v)| (k.as_ref(), Some(v.as_ref()))); - let alt_hashing = self.get_trie_alt_hashing_threshold(); - crate::in_memory_backend::new_in_mem::() - .child_storage_root_with_alt_hashing(&child.child_info, delta, alt_hashing).0 + crate::in_memory_backend::new_in_mem::(self.alt_hashing.clone()) + .child_storage_root(&child.child_info, delta, self.alt_hashing.clone()).0 } else { empty_child_trie_root::>() }.encode() diff --git a/primitives/state-machine/src/ext.rs b/primitives/state-machine/src/ext.rs index 01c3eb1e56c27..3927432fbc94e 100644 --- a/primitives/state-machine/src/ext.rs +++ b/primitives/state-machine/src/ext.rs @@ -28,7 +28,7 @@ use sp_core::{ }; use sp_trie::{trie_types::Layout, empty_child_trie_root}; use sp_externalities::{ - Externalities, Extensions, Extension, ExtensionStore, ExternalitiesHelpers, + Externalities, Extensions, Extension, ExtensionStore, }; use codec::{Decode, Encode, EncodeAppend}; @@ -549,9 +549,9 @@ where root.encode() } else { let root = if let Some((changes, info)) = self.overlay.child_changes(storage_key) { + let alt_hashing = self.backend.get_trie_alt_hashing_threshold(); let delta = changes.map(|(k, v)| (k.as_ref(), v.value().map(AsRef::as_ref))); - let alt_hashing = self.get_trie_alt_hashing_threshold(); - Some(self.backend.child_storage_root_with_alt_hashing(info, delta, alt_hashing)) + Some(self.backend.child_storage_root(info, delta, alt_hashing)) } else { None }; diff --git a/primitives/state-machine/src/in_memory_backend.rs b/primitives/state-machine/src/in_memory_backend.rs index 045558d9f8440..f7a97f1bb2c26 100644 --- a/primitives/state-machine/src/in_memory_backend.rs +++ b/primitives/state-machine/src/in_memory_backend.rs @@ -27,12 +27,14 @@ use codec::Codec; use sp_core::storage::{ChildInfo, Storage}; /// Create a new empty instance of in-memory backend. -pub fn new_in_mem() -> TrieBackend, H> +pub fn new_in_mem(initial_alt_hashing: Option) -> TrieBackend, H> where H::Out: Codec + Ord, { let db = MemoryDB::default(); - TrieBackend::new(db, empty_trie_root::>()) + let mut trie = TrieBackend::new(db, empty_trie_root::>()); + trie.force_alt_hashing = Some(initial_alt_hashing); + trie } impl TrieBackend, H> @@ -59,13 +61,13 @@ where changes: T, ) { let (top, child) = changes.into_iter().partition::, _>(|v| v.0.is_none()); - unimplemented!("get alt hashing changes here"); let (root, transaction) = self.full_storage_root( top.iter().map(|(_, v)| v).flatten().map(|(k, v)| (&k[..], v.as_deref())), child.iter() .filter_map(|v| v.0.as_ref().map(|c| (c, v.1.iter().map(|(k, v)| (&k[..], v.as_deref())))) ), + self.force_alt_hashing.clone().flatten(), ); self.apply_transaction(root, transaction); @@ -104,7 +106,7 @@ where H::Out: Codec + Ord, { fn default() -> Self { - new_in_mem() + new_in_mem(None) } } @@ -114,7 +116,10 @@ where H::Out: Codec + Ord, { fn from(inner: HashMap, BTreeMap>) -> Self { - let mut backend = new_in_mem(); + let alt_hashing = inner.get(&None).map(|map| + sp_core::storage::alt_hashing::get_trie_alt_hashing_threshold(&map) + ).flatten(); + let mut backend = new_in_mem(alt_hashing); backend.insert( inner.into_iter().map(|(k, m)| (k, m.into_iter().map(|(k, v)| (k, Some(v))).collect())), ); diff --git a/primitives/state-machine/src/overlayed_changes/mod.rs b/primitives/state-machine/src/overlayed_changes/mod.rs index 6d28184501a5f..ea1c973ebbfa1 100644 --- a/primitives/state-machine/src/overlayed_changes/mod.rs +++ b/primitives/state-machine/src/overlayed_changes/mod.rs @@ -614,16 +614,6 @@ impl OverlayedChanges { } } - /// Utility function to get trie inner value hash threshold from - /// backend state or pending changes. - fn get_trie_alt_hashing_threshold>(&self, backend: &B) -> Option { - match self.storage(sp_core::storage::well_known_keys::TRIE_HASHING_CONFIG) { - Some(Some(mut encoded)) => sp_core::storage::trie_threshold_decode(&mut encoded), - Some(None) => None, - None => backend.get_trie_alt_hashing_threshold(), - } - } - /// Generate the storage root using `backend` and all changes /// as seen by the current transaction. /// @@ -635,7 +625,7 @@ impl OverlayedChanges { ) -> H::Out where H::Out: Ord + Encode, { - let alt_hashing = self.get_trie_alt_hashing_threshold(backend); + let alt_hashing = backend.get_trie_alt_hashing_threshold(); let delta = self.changes().map(|(k, v)| (&k[..], v.value().map(|v| &v[..]))); let child_delta = self.children() @@ -643,7 +633,7 @@ impl OverlayedChanges { |(k, v)| (&k[..], v.value().map(|v| &v[..])) ))); - let (root, transaction) = backend.full_storage_root_with_alt_hashing( + let (root, transaction) = backend.full_storage_root( delta, child_delta, alt_hashing, diff --git a/primitives/state-machine/src/proving_backend.rs b/primitives/state-machine/src/proving_backend.rs index 9b3eb9cc5be58..f48e225844ada 100644 --- a/primitives/state-machine/src/proving_backend.rs +++ b/primitives/state-machine/src/proving_backend.rs @@ -340,21 +340,21 @@ impl<'a, S, H> Backend for ProvingBackend<'a, S, H> self.0.child_keys(child_info, prefix) } - fn storage_root_with_alt_hashing<'b>( + fn storage_root<'b>( &self, delta: impl Iterator)>, alt_hashing: Option, ) -> (H::Out, Self::Transaction) where H::Out: Ord { - self.0.storage_root_with_alt_hashing(delta, alt_hashing) + self.0.storage_root(delta, alt_hashing) } - fn child_storage_root_with_alt_hashing<'b>( + fn child_storage_root<'b>( &self, child_info: &ChildInfo, delta: impl Iterator)>, alt_hashing: Option, ) -> (H::Out, bool, Self::Transaction) where H::Out: Ord { - self.0.child_storage_root_with_alt_hashing(child_info, delta, alt_hashing) + self.0.child_storage_root(child_info, delta, alt_hashing) } fn register_overlay_stats(&self, _stats: &crate::stats::StateMachineStats) { } diff --git a/primitives/state-machine/src/trie_backend.rs b/primitives/state-machine/src/trie_backend.rs index 4f7001800cbd0..1cb34ff669ec9 100644 --- a/primitives/state-machine/src/trie_backend.rs +++ b/primitives/state-machine/src/trie_backend.rs @@ -32,6 +32,10 @@ use sp_std::{boxed::Box, vec::Vec}; /// Patricia trie-based backend. Transaction type is an overlay of changes to commit. pub struct TrieBackend, H: Hasher> { pub (crate) essence: TrieBackendEssence, + // Allows setting alt hashing at start for testing only + // (see in_memory_backend that cannot read from state as + // it changes. + pub (crate) force_alt_hashing: Option>, } impl, H: Hasher> TrieBackend where H::Out: Codec { @@ -39,6 +43,7 @@ impl, H: Hasher> TrieBackend where H::Out: Codec pub fn new(storage: S, root: H::Out) -> Self { TrieBackend { essence: TrieBackendEssence::new(storage, root), + force_alt_hashing: None, } } @@ -168,11 +173,16 @@ impl, H: Hasher> Backend for TrieBackend where collect_all().map_err(|e| debug!(target: "trie", "Error extracting trie keys: {}", e)).unwrap_or_default() } - fn storage_root_with_alt_hashing<'a>( + fn storage_root<'a>( &self, delta: impl Iterator)>, use_inner_hash_value: Option, ) -> (H::Out, Self::Transaction) where H::Out: Ord { + let use_inner_hash_value = if let Some(force) = self.force_alt_hashing.as_ref() { + force.clone() + } else { + use_inner_hash_value + }; let mut write_overlay = S::Overlay::default(); let mut root = *self.essence.root(); @@ -199,12 +209,18 @@ impl, H: Hasher> Backend for TrieBackend where (root, write_overlay) } - fn child_storage_root_with_alt_hashing<'a>( + fn child_storage_root<'a>( &self, child_info: &ChildInfo, delta: impl Iterator)>, use_inner_hash_value: Option, ) -> (H::Out, bool, Self::Transaction) where H::Out: Ord { + let use_inner_hash_value = if let Some(force) = self.force_alt_hashing.as_ref() { + force.clone() + } else { + use_inner_hash_value + }; + let default_root = match child_info.child_type() { ChildType::ParentKeyId => empty_child_trie_root::>() }; diff --git a/primitives/storage/src/lib.rs b/primitives/storage/src/lib.rs index f63323bbaa17f..a38d3e41135f5 100644 --- a/primitives/storage/src/lib.rs +++ b/primitives/storage/src/lib.rs @@ -220,8 +220,7 @@ impl Storage { /// Utility function to get trie inner value hash threshold from /// backend state or pending changes. pub fn get_trie_alt_hashing_threshold(&self) -> Option { - self.top.get(well_known_keys::TRIE_HASHING_CONFIG) - .and_then(|encoded| trie_threshold_decode(&mut encoded.as_slice())) + alt_hashing::get_trie_alt_hashing_threshold(&self.top) } /// Utility function to modify trie inner value hash threshold. @@ -238,6 +237,19 @@ impl Storage { } } +/// alt hashing related utils. +#[cfg(feature = "std")] +pub mod alt_hashing { + use super::*; + + /// Utility function to get trie inner value hash threshold from + /// backend state or pending changes. + pub fn get_trie_alt_hashing_threshold(map: &StorageMap) -> Option { + map.get(well_known_keys::TRIE_HASHING_CONFIG) + .and_then(|encoded| trie_threshold_decode(&mut encoded.as_slice())) + } +} + /// Information related to a child state. #[derive(Debug, Clone)] #[cfg_attr(feature = "std", derive(PartialEq, Eq, Hash, PartialOrd, Ord))] From 97a1aa6721480396101e3d3c78a2ae10491554e1 Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 10 Jun 2021 18:15:02 +0200 Subject: [PATCH 049/127] Applying from previous state --- client/api/src/in_mem.rs | 2 - client/db/src/bench.rs | 8 +- client/db/src/lib.rs | 20 +---- client/db/src/storage_cache.rs | 12 +-- client/executor/src/integration_tests/mod.rs | 74 +++++++++++++------ client/light/src/backend.rs | 9 +-- client/service/test/src/client/light.rs | 6 +- primitives/state-machine/src/backend.rs | 7 -- primitives/state-machine/src/basic.rs | 2 +- primitives/state-machine/src/ext.rs | 3 +- .../state-machine/src/in_memory_backend.rs | 12 ++- primitives/state-machine/src/lib.rs | 20 +++-- .../src/overlayed_changes/mod.rs | 3 - .../state-machine/src/proving_backend.rs | 64 ++++------------ primitives/state-machine/src/trie_backend.rs | 45 +++-------- primitives/trie/src/lib.rs | 7 +- 16 files changed, 117 insertions(+), 177 deletions(-) diff --git a/client/api/src/in_mem.rs b/client/api/src/in_mem.rs index 899fdd5e66690..d756e1cc0bbc4 100644 --- a/client/api/src/in_mem.rs +++ b/client/api/src/in_mem.rs @@ -573,11 +573,9 @@ impl backend::BlockImportOperation for BlockImportOperatio ) ); - let alt_hashing = storage.get_trie_alt_hashing_threshold(); let (root, transaction) = self.old_state.full_storage_root( storage.top.iter().map(|(k, v)| (k.as_ref(), Some(v.as_ref()))), child_delta, - alt_hashing, ); self.new_state = Some(transaction); diff --git a/client/db/src/bench.rs b/client/db/src/bench.rs index af626d3af7c2f..0b297ad402871 100644 --- a/client/db/src/bench.rs +++ b/client/db/src/bench.rs @@ -160,12 +160,10 @@ impl BenchmarkingState { &child_content.child_info, child_content.data.iter().map(|(k, v)| (k.as_ref(), Some(v.as_ref()))), )); - let alt_hashing = genesis.get_trie_alt_hashing_threshold(); let (root, transaction): (B::Hash, _) = state.state.borrow_mut().as_mut().unwrap() .full_storage_root( genesis.top.iter().map(|(k, v)| (k.as_ref(), Some(v.as_ref()))), child_delta, - alt_hashing, ); state.genesis = transaction.clone().drain(); state.genesis_root = root.clone(); @@ -407,21 +405,19 @@ impl StateBackend> for BenchmarkingState { fn storage_root<'a>( &self, delta: impl Iterator)>, - alt_hashing: Option, ) -> (B::Hash, Self::Transaction) where B::Hash: Ord { self.state.borrow().as_ref() - .map_or(Default::default(), |s| s.storage_root(delta, alt_hashing)) + .map_or(Default::default(), |s| s.storage_root(delta)) } fn child_storage_root<'a>( &self, child_info: &ChildInfo, delta: impl Iterator)>, - alt_hashing: Option, ) -> (B::Hash, bool, Self::Transaction) where B::Hash: Ord { self.state.borrow().as_ref().map_or( Default::default(), - |s| s.child_storage_root(child_info, delta, alt_hashing), + |s| s.child_storage_root(child_info, delta), ) } diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index 5fa44f8c2e1c3..9f320d83839b0 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -227,18 +227,16 @@ impl StateBackend> for RefTrackingState { fn storage_root<'a>( &self, delta: impl Iterator)>, - alt_hashing: Option, ) -> (B::Hash, Self::Transaction) where B::Hash: Ord { - self.state.storage_root(delta, alt_hashing) + self.state.storage_root(delta) } fn child_storage_root<'a>( &self, child_info: &ChildInfo, delta: impl Iterator)>, - alt_hashing: Option, ) -> (B::Hash, bool, Self::Transaction) where B::Hash: Ord { - self.state.child_storage_root(child_info, delta, alt_hashing) + self.state.child_storage_root(child_info, delta) } fn pairs(&self) -> Vec<(Vec, Vec)> { @@ -778,7 +776,7 @@ impl sc_client_api::backend::BlockImportOperation for Bloc )); let mut changes_trie_config: Option = None; - let alt_hashing = storage.get_trie_alt_hashing_threshold(); + // TODO test genesis init with a threshold let (root, transaction) = self.old_state.full_storage_root( storage.top.iter().map(|(k, v)| { if &k[..] == well_known_keys::CHANGES_TRIE_CONFIG { @@ -790,7 +788,6 @@ impl sc_client_api::backend::BlockImportOperation for Bloc (&k[..], Some(&v[..])) }), child_delta, - alt_hashing, ); self.db_updates = transaction; @@ -2328,7 +2325,6 @@ pub(crate) mod tests { header.state_root = op.old_state.storage_root(storage .iter() .map(|(x, y)| (&x[..], Some(&y[..]))), - alt_hashing, ).0.into(); let hash = header.hash(); @@ -2373,7 +2369,6 @@ pub(crate) mod tests { let (root, overlay) = op.old_state.storage_root( storage.iter() .map(|(k, v)| (&k[..], v.as_ref().map(|v| &v[..]))), - alt_hashing, ); op.update_db_storage(overlay).unwrap(); header.state_root = root.into(); @@ -2400,7 +2395,6 @@ pub(crate) mod tests { fn delete_only_when_negative_rc() { sp_tracing::try_init_simple(); let key; - let alt_hashing = false; let backend = Backend::::new_test(1, 0); let hash = { @@ -2414,7 +2408,7 @@ pub(crate) mod tests { extrinsics_root: Default::default(), }; - header.state_root = op.old_state.storage_root(std::iter::empty(), alt_hashing).0.into(); + header.state_root = op.old_state.storage_root(std::iter::empty()).0.into(); let hash = header.hash(); op.reset_storage(Storage { @@ -2455,7 +2449,6 @@ pub(crate) mod tests { .iter() .cloned() .map(|(x, y)| (x, Some(y))), - alt_hashing, ).0.into(); let hash = header.hash(); @@ -2493,7 +2486,6 @@ pub(crate) mod tests { .iter() .cloned() .map(|(x, y)| (x, Some(y))), - alt_hashing, ).0.into(); let hash = header.hash(); @@ -2531,7 +2523,6 @@ pub(crate) mod tests { .iter() .cloned() .map(|(x, y)| (x, Some(y))), - alt_hashing, ).0.into(); op.set_block_data( @@ -2847,7 +2838,6 @@ pub(crate) mod tests { #[test] fn storage_hash_is_cached_correctly() { let backend = Backend::::new_test(10, 10); - let alt_hashing = false; let hash0 = { let mut op = backend.begin_operation().unwrap(); @@ -2865,7 +2855,6 @@ pub(crate) mod tests { header.state_root = op.old_state.storage_root(storage .iter() .map(|(x, y)| (&x[..], Some(&y[..]))), - alt_hashing, ).0.into(); let hash = header.hash(); @@ -2906,7 +2895,6 @@ pub(crate) mod tests { let (root, overlay) = op.old_state.storage_root( storage.iter() .map(|(k, v)| (&k[..], v.as_ref().map(|v| &v[..]))), - alt_hashing, ); op.update_db_storage(overlay).unwrap(); header.state_root = root.into(); diff --git a/client/db/src/storage_cache.rs b/client/db/src/storage_cache.rs index b249ea14748ac..cb2ab1de1b6c9 100644 --- a/client/db/src/storage_cache.rs +++ b/client/db/src/storage_cache.rs @@ -645,18 +645,16 @@ impl>, B: BlockT> StateBackend> for Cachin fn storage_root<'a>( &self, delta: impl Iterator)>, - alt_hashing: Option, ) -> (B::Hash, Self::Transaction) where B::Hash: Ord { - self.state.storage_root(delta, alt_hashing) + self.state.storage_root(delta) } fn child_storage_root<'a>( &self, child_info: &ChildInfo, delta: impl Iterator)>, - alt_hashing: Option, ) -> (B::Hash, bool, Self::Transaction) where B::Hash: Ord { - self.state.child_storage_root(child_info, delta, alt_hashing) + self.state.child_storage_root(child_info, delta) } fn pairs(&self) -> Vec<(Vec, Vec)> { @@ -829,18 +827,16 @@ impl>, B: BlockT> StateBackend> for Syncin fn storage_root<'a>( &self, delta: impl Iterator)>, - alt_hashing: Option, ) -> (B::Hash, Self::Transaction) where B::Hash: Ord { - self.caching_state().storage_root(delta, alt_hashing) + self.caching_state().storage_root(delta) } fn child_storage_root<'a>( &self, child_info: &ChildInfo, delta: impl Iterator)>, - alt_hashing: Option, ) -> (B::Hash, bool, Self::Transaction) where B::Hash: Ord { - self.caching_state().child_storage_root(child_info, delta, alt_hashing) + self.caching_state().child_storage_root(child_info, delta) } fn pairs(&self) -> Vec<(Vec, Vec)> { diff --git a/client/executor/src/integration_tests/mod.rs b/client/executor/src/integration_tests/mod.rs index 164c576540809..b06b5f4e8df76 100644 --- a/client/executor/src/integration_tests/mod.rs +++ b/client/executor/src/integration_tests/mod.rs @@ -790,30 +790,60 @@ fn state_hashing_update(wasm_method: WasmExecutionMethod) { // use externalities without storage flag. let mut ext = TestExternalities::new(Default::default()); - let mut ext = ext.ext(); - ext.set_storage(b"foo".to_vec(), vec![1u8; 1_000]); // big inner hash - ext.set_storage(b"foo2".to_vec(), vec![3u8; 16]); // no inner hash - ext.set_storage(b"foo222".to_vec(), vec![5u8; 100]); // inner hash - - let output = call_in_wasm( - "test_data_in", - &b"Hello world".to_vec().encode(), - wasm_method, - &mut ext, - ).unwrap(); + let root1 = { + let mut ext = ext.ext(); + ext.set_storage(b"foo".to_vec(), b"bar".to_vec()); + let output = call_in_wasm( + "test_data_in", + &vec![1u8; 100].encode(), + wasm_method, + &mut ext, + ).unwrap(); - assert_eq!(output, b"all ok!".to_vec().encode()); + assert_eq!(output, b"all ok!".to_vec().encode()); + ext.storage_root() + }; - let root1 = ext.storage_root(); - // flag state. - let _ = call_in_wasm( - "test_switch_state", - Default::default(), - wasm_method, - &mut ext, - ).unwrap(); - let root2 = ext.storage_root(); + ext.commit_all().unwrap(); + let root2 = { + let mut ext = ext.ext(); + // flag state. + let _ = call_in_wasm( + "test_switch_state", + Default::default(), + wasm_method, + &mut ext, + ).unwrap(); + ext.storage_root() + }; - // Note that in this case all the value did switch (in memory changes). assert!(root1 != root2); + + ext.commit_all().unwrap(); + let root3 = { + let mut ext = ext.ext(); + let _ = call_in_wasm( + "test_data_in", + &vec![2u8; 100].to_vec().encode(), + wasm_method, + &mut ext, + ).unwrap(); + ext.storage_root() + }; + assert!(root2 != root3); + + ext.commit_all().unwrap(); + let root3 = { + let mut ext = ext.ext(); + // revert to root 2 state, but this time + // inner hashing should apply + let _ = call_in_wasm( + "test_data_in", + &vec![1u8; 100].to_vec().encode(), + wasm_method, + &mut ext, + ).unwrap(); + ext.storage_root() + }; + assert!(root2 != root3); } diff --git a/client/light/src/backend.rs b/client/light/src/backend.rs index 7b07477012d72..d5707f4796b34 100644 --- a/client/light/src/backend.rs +++ b/client/light/src/backend.rs @@ -324,7 +324,6 @@ impl BlockImportOperation for ImportOperation fn reset_storage(&mut self, input: Storage) -> ClientResult { check_genesis_storage(&input)?; - let alt_hashing = input.get_trie_alt_hashing_threshold(); // changes trie configuration let changes_trie_config = input.top.iter() .find(|(k, _)| &k[..] == well_known_keys::CHANGES_TRIE_CONFIG) @@ -347,7 +346,7 @@ impl BlockImportOperation for ImportOperation } let storage_update = InMemoryBackend::from(storage); - let (storage_root, _) = storage_update.full_storage_root(std::iter::empty(), child_delta, alt_hashing); + let (storage_root, _) = storage_update.full_storage_root(std::iter::empty(), child_delta); self.storage_update = Some(storage_update); Ok(storage_root) @@ -490,11 +489,10 @@ impl StateBackend for GenesisOrUnavailableState fn storage_root<'a>( &self, delta: impl Iterator)>, - alt_hashing: Option, ) -> (H::Out, Self::Transaction) where H::Out: Ord { match *self { GenesisOrUnavailableState::Genesis(ref state) => state - .storage_root(delta, alt_hashing), + .storage_root(delta), GenesisOrUnavailableState::Unavailable => Default::default(), } } @@ -503,12 +501,11 @@ impl StateBackend for GenesisOrUnavailableState &self, child_info: &ChildInfo, delta: impl Iterator)>, - alt_hashing: Option, ) -> (H::Out, bool, Self::Transaction) where H::Out: Ord { match *self { GenesisOrUnavailableState::Genesis(ref state) => { let (root, is_equal, _) = state - .child_storage_root(child_info, delta, alt_hashing); + .child_storage_root(child_info, delta); (root, is_equal, Default::default()) }, GenesisOrUnavailableState::Unavailable => diff --git a/client/service/test/src/client/light.rs b/client/service/test/src/client/light.rs index d4e3220de73c6..e40538e08ebb9 100644 --- a/client/service/test/src/client/light.rs +++ b/client/service/test/src/client/light.rs @@ -459,11 +459,10 @@ fn prepare_for_read_proof_check(hashed_value: bool) -> (TestChecker, Header, Sto // prepare remote client let remote_client = substrate_test_runtime_client::new(hashed_value); let remote_block_id = BlockId::Number(0); - let flagged = false; let remote_block_hash = remote_client.block_hash(0).unwrap().unwrap(); let mut remote_block_header = remote_client.header(&remote_block_id).unwrap().unwrap(); remote_block_header.state_root = remote_client.state_at(&remote_block_id).unwrap() - .storage_root(std::iter::empty(), flagged).0.into(); + .storage_root(std::iter::empty()).0.into(); // 'fetch' read proof from remote node let heap_pages = remote_client.storage(&remote_block_id, &StorageKey(well_known_keys::HEAP_PAGES.to_vec())) @@ -496,7 +495,6 @@ fn prepare_for_read_child_proof_check() -> (TestChecker, Header, StorageProof, V use substrate_test_runtime_client::TestClientBuilderExt; let child_info = ChildInfo::new_default(b"child1"); let child_info = &child_info; - let flagged = false; // prepare remote client let remote_client = substrate_test_runtime_client::TestClientBuilder::new() .add_extra_child_storage( @@ -508,7 +506,7 @@ fn prepare_for_read_child_proof_check() -> (TestChecker, Header, StorageProof, V let remote_block_hash = remote_client.block_hash(0).unwrap().unwrap(); let mut remote_block_header = remote_client.header(&remote_block_id).unwrap().unwrap(); remote_block_header.state_root = remote_client.state_at(&remote_block_id).unwrap() - .storage_root(std::iter::empty(), flagged).0.into(); + .storage_root(std::iter::empty()).0.into(); // 'fetch' child read proof from remote node let child_value = remote_client.child_storage( diff --git a/primitives/state-machine/src/backend.rs b/primitives/state-machine/src/backend.rs index 4976d6a43d5db..946de1f6469c5 100644 --- a/primitives/state-machine/src/backend.rs +++ b/primitives/state-machine/src/backend.rs @@ -126,22 +126,18 @@ pub trait Backend: sp_std::fmt::Debug { /// `alt_hashing` indicate if trie state should apply alternate hashing /// scheme (inner value hashed). /// Does not include child storage updates. - /// Alt hashing paremeter must contain possible changes from delta. fn storage_root<'a>( &self, delta: impl Iterator)>, - alt_hashing: Option, ) -> (H::Out, Self::Transaction) where H::Out: Ord; /// Calculate the child storage root, with given delta over what is already stored in /// the backend, and produce a "transaction" that can be used to commit. The second argument /// is true if child storage root equals default storage root. - /// Alt hashing paremeter must contain possible changes from delta. fn child_storage_root<'a>( &self, child_info: &ChildInfo, delta: impl Iterator)>, - alt_hashing: Option, ) -> (H::Out, bool, Self::Transaction) where H::Out: Ord; /// Get all key/value pairs into a Vec. @@ -180,7 +176,6 @@ pub trait Backend: sp_std::fmt::Debug { &'a ChildInfo, impl Iterator)>, )>, - alt_hashing: Option, ) -> (H::Out, Self::Transaction) where H::Out: Ord + Encode { let mut txs: Self::Transaction = Default::default(); let mut child_roots: Vec<_> = Default::default(); @@ -189,7 +184,6 @@ pub trait Backend: sp_std::fmt::Debug { let (child_root, empty, child_txs) = self.child_storage_root( &child_info, child_delta, - alt_hashing, ); let prefixed_storage_key = child_info.prefixed_storage_key(); txs.consolidate(child_txs); @@ -206,7 +200,6 @@ pub trait Backend: sp_std::fmt::Debug { .iter() .map(|(k, v)| (&k[..], v.as_ref().map(|v| &v[..]))) ), - alt_hashing, ); txs.consolidate(parent_txs); (root, txs) diff --git a/primitives/state-machine/src/basic.rs b/primitives/state-machine/src/basic.rs index 4a691e4eb208b..550f6704d8cff 100644 --- a/primitives/state-machine/src/basic.rs +++ b/primitives/state-machine/src/basic.rs @@ -302,7 +302,7 @@ impl Externalities for BasicExternalities { if let Some(child) = self.inner.children_default.get(child_info.storage_key()) { let delta = child.data.iter().map(|(k, v)| (k.as_ref(), Some(v.as_ref()))); crate::in_memory_backend::new_in_mem::(self.alt_hashing.clone()) - .child_storage_root(&child.child_info, delta, self.alt_hashing.clone()).0 + .child_storage_root(&child.child_info, delta).0 } else { empty_child_trie_root::>() }.encode() diff --git a/primitives/state-machine/src/ext.rs b/primitives/state-machine/src/ext.rs index 3927432fbc94e..f788f6a4f3bf7 100644 --- a/primitives/state-machine/src/ext.rs +++ b/primitives/state-machine/src/ext.rs @@ -549,9 +549,8 @@ where root.encode() } else { let root = if let Some((changes, info)) = self.overlay.child_changes(storage_key) { - let alt_hashing = self.backend.get_trie_alt_hashing_threshold(); let delta = changes.map(|(k, v)| (k.as_ref(), v.value().map(AsRef::as_ref))); - Some(self.backend.child_storage_root(info, delta, alt_hashing)) + Some(self.backend.child_storage_root(info, delta)) } else { None }; diff --git a/primitives/state-machine/src/in_memory_backend.rs b/primitives/state-machine/src/in_memory_backend.rs index f7a97f1bb2c26..89ea70fd7106d 100644 --- a/primitives/state-machine/src/in_memory_backend.rs +++ b/primitives/state-machine/src/in_memory_backend.rs @@ -67,7 +67,6 @@ where .filter_map(|v| v.0.as_ref().map(|c| (c, v.1.iter().map(|(k, v)| (&k[..], v.as_deref())))) ), - self.force_alt_hashing.clone().flatten(), ); self.apply_transaction(root, transaction); @@ -90,6 +89,13 @@ where pub fn eq(&self, other: &Self) -> bool { self.root() == other.root() } + + + /// To reset with threshold for genesis storage, this function allows + /// setting a alt hashing threshold at start. + pub fn force_alt_hashing(&mut self, threshold: Option) { + self.force_alt_hashing = Some(threshold); + } } impl Clone for TrieBackend, H> @@ -181,7 +187,7 @@ mod tests { /// Assert in memory backend with only child trie keys works as trie backend. #[test] fn in_memory_with_child_trie_only() { - let storage = new_in_mem::(); + let storage = new_in_mem::(None); let child_info = ChildInfo::new_default(b"1"); let child_info = &child_info; let mut storage = storage.update( @@ -199,7 +205,7 @@ mod tests { #[test] fn insert_multiple_times_child_data_works() { - let mut storage = new_in_mem::(); + let mut storage = new_in_mem::(None); let child_info = ChildInfo::new_default(b"1"); storage.insert(vec![(Some(child_info.clone()), vec![(b"2".to_vec(), Some(b"3".to_vec()))])]); diff --git a/primitives/state-machine/src/lib.rs b/primitives/state-machine/src/lib.rs index 20efdb1236a1a..0b27ee7b47b3c 100644 --- a/primitives/state-machine/src/lib.rs +++ b/primitives/state-machine/src/lib.rs @@ -894,7 +894,7 @@ mod tests { traits::CodeExecutor, }; use crate::execution::CallResult; - + use sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD as TRESHOLD; #[derive(Clone)] struct DummyCodeExecutor { @@ -1086,9 +1086,7 @@ mod tests { // fetch execution proof from 'remote' full node let remote_backend = trie_backend::tests::test_trie(flagged); - let remote_root = remote_backend.storage_root(std::iter::empty(), flagged).0; - let remote_root_2 = remote_backend.storage_root(std::iter::empty(), false).0; - assert_eq!(remote_root, remote_root_2); + let remote_root = remote_backend.storage_root(std::iter::empty()).0; let (remote_result, remote_proof) = prove_execution::<_, _, u64, _, _>( remote_backend, &mut Default::default(), @@ -1244,7 +1242,7 @@ mod tests { fn set_child_storage_works() { let child_info = ChildInfo::new_default(b"sub1"); let child_info = &child_info; - let mut state = new_in_mem::(); + let mut state = new_in_mem::(None); let backend = state.as_trie_backend().unwrap(); let mut overlay = OverlayedChanges::default(); let mut cache = StorageTransactionCache::default(); @@ -1290,7 +1288,7 @@ mod tests { b"d4".to_vec(), ]; let key = b"key".to_vec(); - let mut state = new_in_mem::(); + let mut state = new_in_mem::(None); let backend = state.as_trie_backend().unwrap(); let mut overlay = OverlayedChanges::default(); let mut cache = StorageTransactionCache::default(); @@ -1351,7 +1349,7 @@ mod tests { let key = b"events".to_vec(); let mut cache = StorageTransactionCache::default(); - let mut state = new_in_mem::(); + let mut state = new_in_mem::(None); let backend = state.as_trie_backend().unwrap(); let mut overlay = OverlayedChanges::default(); @@ -1444,7 +1442,7 @@ mod tests { let child_info = &child_info; // fetch read proof from 'remote' full node let remote_backend = trie_backend::tests::test_trie(flagged); - let remote_root = remote_backend.storage_root(::std::iter::empty(), flagged).0; + let remote_root = remote_backend.storage_root(::std::iter::empty()).0; let remote_proof = prove_read(remote_backend, &[b"value2"]).unwrap(); // check proof locally let local_result1 = read_proof_check::( @@ -1465,7 +1463,7 @@ mod tests { assert_eq!(local_result2, false); // on child trie let remote_backend = trie_backend::tests::test_trie(flagged); - let remote_root = remote_backend.storage_root(::std::iter::empty(), false).0; + let remote_root = remote_backend.storage_root(::std::iter::empty()).0; let remote_proof = prove_child_read( remote_backend, child_info, @@ -1514,7 +1512,7 @@ mod tests { let check_proof = |mdb, root| -> StorageProof { let remote_backend = TrieBackend::new(mdb, root); - let remote_root = remote_backend.storage_root(::std::iter::empty(), false).0; + let remote_root = remote_backend.storage_root(::std::iter::empty()).0; let remote_proof = prove_read(remote_backend, &[b"foo222"]).unwrap(); // check proof locally let local_result1 = read_proof_check::( @@ -1540,7 +1538,7 @@ mod tests { // do switch - layout = Layout::with_inner_hashing(); + layout = Layout::with_inner_hashing(TRESHOLD); // update with same value do not change { let mut trie = TrieDBMut::from_existing_with_layout( diff --git a/primitives/state-machine/src/overlayed_changes/mod.rs b/primitives/state-machine/src/overlayed_changes/mod.rs index ea1c973ebbfa1..fc377b808f640 100644 --- a/primitives/state-machine/src/overlayed_changes/mod.rs +++ b/primitives/state-machine/src/overlayed_changes/mod.rs @@ -625,8 +625,6 @@ impl OverlayedChanges { ) -> H::Out where H::Out: Ord + Encode, { - let alt_hashing = backend.get_trie_alt_hashing_threshold(); - let delta = self.changes().map(|(k, v)| (&k[..], v.value().map(|v| &v[..]))); let child_delta = self.children() .map(|(changes, info)| (info, changes.map( @@ -636,7 +634,6 @@ impl OverlayedChanges { let (root, transaction) = backend.full_storage_root( delta, child_delta, - alt_hashing, ); cache.transaction = Some(transaction); diff --git a/primitives/state-machine/src/proving_backend.rs b/primitives/state-machine/src/proving_backend.rs index f48e225844ada..1e8b1498a4f6f 100644 --- a/primitives/state-machine/src/proving_backend.rs +++ b/primitives/state-machine/src/proving_backend.rs @@ -343,18 +343,16 @@ impl<'a, S, H> Backend for ProvingBackend<'a, S, H> fn storage_root<'b>( &self, delta: impl Iterator)>, - alt_hashing: Option, ) -> (H::Out, Self::Transaction) where H::Out: Ord { - self.0.storage_root(delta, alt_hashing) + self.0.storage_root(delta) } fn child_storage_root<'b>( &self, child_info: &ChildInfo, delta: impl Iterator)>, - alt_hashing: Option, ) -> (H::Out, bool, Self::Transaction) where H::Out: Ord { - self.0.child_storage_root(child_info, delta, alt_hashing) + self.0.child_storage_root(child_info, delta) } fn register_overlay_stats(&self, _stats: &crate::stats::StateMachineStats) { } @@ -440,8 +438,8 @@ mod tests { assert_eq!(trie_backend.storage(b"key").unwrap(), proving_backend.storage(b"key").unwrap()); assert_eq!(trie_backend.pairs(), proving_backend.pairs()); - let (trie_root, mut trie_mdb) = trie_backend.storage_root(std::iter::empty(), flagged); - let (proving_root, mut proving_mdb) = proving_backend.storage_root(std::iter::empty(), flagged); + let (trie_root, mut trie_mdb) = trie_backend.storage_root(std::iter::empty()); + let (proving_root, mut proving_mdb) = proving_backend.storage_root(std::iter::empty()); assert_eq!(trie_root, proving_root); assert_eq!(trie_mdb.drain(), proving_mdb.drain()); } @@ -454,54 +452,22 @@ mod tests { fn proof_recorded_and_checked_inner(flagged: bool) { let size_content = 34; // above hashable value treshold. let value_range = 0..64; - let mut contents = value_range.clone().map(|i| (vec![i], Some(vec![i; size_content]))).collect::>(); + let contents = value_range.clone().map(|i| (vec![i], Some(vec![i; size_content]))).collect::>(); + let mut in_memory = InMemoryBackend::::default(); if flagged { - contents.push(( + in_memory = in_memory.update(vec![(None, vec![( sp_core::storage::well_known_keys::TRIE_HASHING_CONFIG.to_vec(), Some(sp_core::storage::trie_threshold_encode( sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD, )), - )); + )])]); } - let in_memory = InMemoryBackend::::default(); let mut in_memory = in_memory.update(vec![(None, contents)]); - let in_memory_root = in_memory.storage_root(std::iter::empty(), flagged).0; + let in_memory_root = in_memory.storage_root(std::iter::empty()).0; value_range.clone().for_each(|i| assert_eq!(in_memory.storage(&[i]).unwrap().unwrap(), vec![i; size_content])); let trie = in_memory.as_trie_backend().unwrap(); - let trie_root = trie.storage_root(std::iter::empty(), flagged).0; - assert_eq!(in_memory_root, trie_root); - value_range.clone().for_each(|i| assert_eq!(trie.storage(&[i]).unwrap().unwrap(), vec![i; size_content])); - - let proving = ProvingBackend::new(trie); - assert_eq!(proving.storage(&[42]).unwrap().unwrap(), vec![42; size_content]); - - let proof = proving.extract_proof(); - - let proof_check = create_proof_check_backend::(in_memory_root.into(), proof).unwrap(); - assert_eq!(proof_check.storage(&[42]).unwrap().unwrap(), vec![42; size_content]); - } - - #[test] - fn proof_recorded_and_checked_old_hash() { - // test proof starting with old hash content and flagging in between. - // TODO not that usefull (we do run with direct update). -> replace by change of threshold - // test. - let size_content = 34; // above hashable value treshold. - let value_range = 0..64; - let contents = value_range.clone().map(|i| (vec![i], Some(vec![i; size_content]))).collect::>(); - let in_memory = InMemoryBackend::::default(); - let mut in_memory = in_memory.update(vec![(None, contents)]); - value_range.clone().for_each(|i| assert_eq!(in_memory.storage(&[i]).unwrap().unwrap(), vec![i; size_content])); - in_memory = in_memory.update(vec![(None, vec![( - sp_core::storage::well_known_keys::TRIE_HASHING_CONFIG.to_vec(), - Some(sp_core::storage::trie_threshold_encode( - sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD, - )), - )])]); - let in_memory_root = in_memory.storage_root(std::iter::empty(), true).0; - let trie = in_memory.as_trie_backend().unwrap(); - let trie_root = trie.storage_root(std::iter::empty(), true).0; + let trie_root = trie.storage_root(std::iter::empty()).0; assert_eq!(in_memory_root, trie_root); value_range.clone().for_each(|i| assert_eq!(trie.storage(&[i]).unwrap().unwrap(), vec![i; size_content])); @@ -531,16 +497,16 @@ mod tests { (Some(child_info_2.clone()), (10..15).map(|i| (vec![i], Some(vec![i]))).collect()), ]; + let mut in_memory = InMemoryBackend::::default(); if flagged { - contents[0].1.push(( + in_memory = in_memory.update(vec![(None, vec![( sp_core::storage::well_known_keys::TRIE_HASHING_CONFIG.to_vec(), Some(sp_core::storage::trie_threshold_encode( sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD, )), - )); + )])]); } - let in_memory = InMemoryBackend::::default(); - let mut in_memory = in_memory.update(contents); + in_memory = in_memory.update(contents); let child_storage_keys = vec![child_info_1.to_owned(), child_info_2.to_owned()]; let in_memory_root = in_memory.full_storage_root( std::iter::empty(), @@ -560,7 +526,7 @@ mod tests { )); let trie = in_memory.as_trie_backend().unwrap(); - let trie_root = trie.storage_root(std::iter::empty(), flagged).0; + let trie_root = trie.storage_root(std::iter::empty()).0; assert_eq!(in_memory_root, trie_root); (0..64).for_each(|i| assert_eq!( trie.storage(&[i]).unwrap().unwrap(), diff --git a/primitives/state-machine/src/trie_backend.rs b/primitives/state-machine/src/trie_backend.rs index 1cb34ff669ec9..e0e44dd264b89 100644 --- a/primitives/state-machine/src/trie_backend.rs +++ b/primitives/state-machine/src/trie_backend.rs @@ -176,12 +176,12 @@ impl, H: Hasher> Backend for TrieBackend where fn storage_root<'a>( &self, delta: impl Iterator)>, - use_inner_hash_value: Option, ) -> (H::Out, Self::Transaction) where H::Out: Ord { let use_inner_hash_value = if let Some(force) = self.force_alt_hashing.as_ref() { force.clone() } else { - use_inner_hash_value + // TODO try memoize in force + self.get_trie_alt_hashing_threshold() }; let mut write_overlay = S::Overlay::default(); let mut root = *self.essence.root(); @@ -213,12 +213,11 @@ impl, H: Hasher> Backend for TrieBackend where &self, child_info: &ChildInfo, delta: impl Iterator)>, - use_inner_hash_value: Option, ) -> (H::Out, bool, Self::Transaction) where H::Out: Ord { let use_inner_hash_value = if let Some(force) = self.force_alt_hashing.as_ref() { force.clone() } else { - use_inner_hash_value + self.get_trie_alt_hashing_threshold() }; let default_root = match child_info.child_type() { @@ -283,6 +282,7 @@ impl, H: Hasher> Backend for TrieBackend where pub mod tests { use std::{collections::HashSet, iter}; use sp_core::H256; + use sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD as TRESHOLD; use codec::Encode; use sp_trie::{TrieMut, PrefixedMemoryDB, trie_types::TrieDBMut, KeySpacedDBMut}; use sp_runtime::traits::BlakeTwo256; @@ -305,7 +305,7 @@ pub mod tests { let mut sub_root = Vec::new(); root.encode_to(&mut sub_root); let mut trie = if hashed_value { - let layout = Layout::with_inner_hashing(); + let layout = Layout::with_inner_hashing(TRESHOLD); TrieDBMut::new_with_layout(&mut mdb, &mut root, layout) } else { TrieDBMut::new(&mut mdb, &mut root) @@ -320,9 +320,7 @@ pub mod tests { if hashed_value { trie.insert( sp_core::storage::well_known_keys::TRIE_HASHING_CONFIG, - sp_core::storage::trie_threshold_encode( - sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD, - ).as_slice(), + sp_core::storage::trie_threshold_encode(TRESHOLD).as_slice(), ).unwrap(); } for i in 128u8..255u8 { @@ -391,41 +389,20 @@ pub mod tests { storage_root_is_non_default_inner(true); } fn storage_root_is_non_default_inner(flagged: bool) { - assert!(test_trie(flagged).storage_root(iter::empty(), flagged).0 != H256::repeat_byte(0)); - } - - #[test] - fn storage_root_transaction_state_root_update() { - // a drop a insert of same hash: rc is 0 - assert_eq!(test_trie(false).storage_root(iter::empty(), false).1.drain() - .into_iter().filter(|v| (v.1).1 != 0).count(), 0); - // Unchanged - assert_eq!(test_trie(false).storage_root(iter::empty(), true).1.drain() - .into_iter().filter(|v| (v.1).1 != 0).count(), 0); - // a drop a insert of same hash: rc is 0 - assert_eq!(test_trie(true).storage_root(iter::empty(), true).1.drain() - .into_iter().filter(|v| (v.1).1 != 0).count(), 0); - } - - #[test] - fn storage_root_flagged_is_empty() { - assert!(test_trie(false).storage_root(iter::empty(), true).1.drain().is_empty()); + assert!(test_trie(flagged).storage_root(iter::empty()).0 != H256::repeat_byte(0)); } #[test] fn storage_root_transaction_is_non_empty() { - storage_root_transaction_is_non_empty_inner(false, false); - storage_root_transaction_is_non_empty_inner(false, true); - storage_root_transaction_is_non_empty_inner(true, false); - storage_root_transaction_is_non_empty_inner(true, true); + storage_root_transaction_is_non_empty_inner(false); + storage_root_transaction_is_non_empty_inner(true); } - fn storage_root_transaction_is_non_empty_inner(flagged: bool, do_flag: bool) { + fn storage_root_transaction_is_non_empty_inner(flagged: bool) { let (new_root, mut tx) = test_trie(flagged).storage_root( iter::once((&b"new-key"[..], Some(&b"new-value"[..]))), - do_flag, ); assert!(!tx.drain().is_empty()); - assert!(new_root != test_trie(false).storage_root(iter::empty(), false).0); + assert!(new_root != test_trie(false).storage_root(iter::empty()).0); } #[test] diff --git a/primitives/trie/src/lib.rs b/primitives/trie/src/lib.rs index 1b8acc9a796a3..9eafffd312877 100644 --- a/primitives/trie/src/lib.rs +++ b/primitives/trie/src/lib.rs @@ -833,6 +833,7 @@ mod tests { use super::*; use codec::{Encode, Decode, Compact}; use sp_core::Blake2Hasher; + use sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD as TRESHOLD; use hash_db::{HashDB, Hasher}; use trie_db::{DBValue, TrieMut, Trie, NodeCodec as NodeCodecT}; use trie_standardmap::{Alphabet, ValueMode, StandardMap}; @@ -890,7 +891,7 @@ mod tests { let layout = Layout::default(); check_equivalent::(input, layout.clone()); check_iteration::(input, layout); - let layout = Layout::with_inner_hashing(); + let layout = Layout::with_inner_hashing(TRESHOLD); check_equivalent::(input, layout.clone()); check_iteration::(input, layout); } @@ -1043,7 +1044,7 @@ mod tests { }.make_with(seed.as_fixed_bytes_mut()); let layout = if flag { - Layout::with_inner_hashing() + Layout::with_inner_hashing(TRESHOLD) } else { Layout::default() }; @@ -1140,7 +1141,7 @@ mod tests { } fn iterator_works_inner(flag: bool) { let layout = if flag { - Layout::with_inner_hashing() + Layout::with_inner_hashing(TRESHOLD) } else { Layout::default() }; From f4849a2d8b5d7c5adef7a4223220bfddb4a6b807 Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 10 Jun 2021 18:28:59 +0200 Subject: [PATCH 050/127] Remove default from storage, genesis need a special build. --- primitives/state-machine/src/basic.rs | 5 +++-- .../state-machine/src/in_memory_backend.rs | 19 ++++++------------- primitives/state-machine/src/lib.rs | 6 +++--- .../state-machine/src/proving_backend.rs | 2 +- 4 files changed, 13 insertions(+), 19 deletions(-) diff --git a/primitives/state-machine/src/basic.rs b/primitives/state-machine/src/basic.rs index 550f6704d8cff..7d7be10f2b584 100644 --- a/primitives/state-machine/src/basic.rs +++ b/primitives/state-machine/src/basic.rs @@ -301,8 +301,9 @@ impl Externalities for BasicExternalities { ) -> Vec { if let Some(child) = self.inner.children_default.get(child_info.storage_key()) { let delta = child.data.iter().map(|(k, v)| (k.as_ref(), Some(v.as_ref()))); - crate::in_memory_backend::new_in_mem::(self.alt_hashing.clone()) - .child_storage_root(&child.child_info, delta).0 + let mut in_mem = crate::in_memory_backend::new_in_mem::(); + in_mem.force_alt_hashing(self.alt_hashing.clone()); + in_mem.child_storage_root(&child.child_info, delta).0 } else { empty_child_trie_root::>() }.encode() diff --git a/primitives/state-machine/src/in_memory_backend.rs b/primitives/state-machine/src/in_memory_backend.rs index 89ea70fd7106d..84c0b316387da 100644 --- a/primitives/state-machine/src/in_memory_backend.rs +++ b/primitives/state-machine/src/in_memory_backend.rs @@ -27,14 +27,12 @@ use codec::Codec; use sp_core::storage::{ChildInfo, Storage}; /// Create a new empty instance of in-memory backend. -pub fn new_in_mem(initial_alt_hashing: Option) -> TrieBackend, H> +pub fn new_in_mem() -> TrieBackend, H> where H::Out: Codec + Ord, { let db = MemoryDB::default(); - let mut trie = TrieBackend::new(db, empty_trie_root::>()); - trie.force_alt_hashing = Some(initial_alt_hashing); - trie + TrieBackend::new(db, empty_trie_root::>()) } impl TrieBackend, H> @@ -90,8 +88,6 @@ where self.root() == other.root() } - - /// To reset with threshold for genesis storage, this function allows /// setting a alt hashing threshold at start. pub fn force_alt_hashing(&mut self, threshold: Option) { self.force_alt_hashing = Some(threshold); @@ -112,7 +108,7 @@ where H::Out: Codec + Ord, { fn default() -> Self { - new_in_mem(None) + new_in_mem() } } @@ -122,10 +118,7 @@ where H::Out: Codec + Ord, { fn from(inner: HashMap, BTreeMap>) -> Self { - let alt_hashing = inner.get(&None).map(|map| - sp_core::storage::alt_hashing::get_trie_alt_hashing_threshold(&map) - ).flatten(); - let mut backend = new_in_mem(alt_hashing); + let mut backend = new_in_mem(); backend.insert( inner.into_iter().map(|(k, m)| (k, m.into_iter().map(|(k, v)| (k, Some(v))).collect())), ); @@ -187,7 +180,7 @@ mod tests { /// Assert in memory backend with only child trie keys works as trie backend. #[test] fn in_memory_with_child_trie_only() { - let storage = new_in_mem::(None); + let storage = new_in_mem::(); let child_info = ChildInfo::new_default(b"1"); let child_info = &child_info; let mut storage = storage.update( @@ -205,7 +198,7 @@ mod tests { #[test] fn insert_multiple_times_child_data_works() { - let mut storage = new_in_mem::(None); + let mut storage = new_in_mem::(); let child_info = ChildInfo::new_default(b"1"); storage.insert(vec![(Some(child_info.clone()), vec![(b"2".to_vec(), Some(b"3".to_vec()))])]); diff --git a/primitives/state-machine/src/lib.rs b/primitives/state-machine/src/lib.rs index 0b27ee7b47b3c..1b911e7696499 100644 --- a/primitives/state-machine/src/lib.rs +++ b/primitives/state-machine/src/lib.rs @@ -1242,7 +1242,7 @@ mod tests { fn set_child_storage_works() { let child_info = ChildInfo::new_default(b"sub1"); let child_info = &child_info; - let mut state = new_in_mem::(None); + let mut state = new_in_mem::(); let backend = state.as_trie_backend().unwrap(); let mut overlay = OverlayedChanges::default(); let mut cache = StorageTransactionCache::default(); @@ -1288,7 +1288,7 @@ mod tests { b"d4".to_vec(), ]; let key = b"key".to_vec(); - let mut state = new_in_mem::(None); + let mut state = new_in_mem::(); let backend = state.as_trie_backend().unwrap(); let mut overlay = OverlayedChanges::default(); let mut cache = StorageTransactionCache::default(); @@ -1349,7 +1349,7 @@ mod tests { let key = b"events".to_vec(); let mut cache = StorageTransactionCache::default(); - let mut state = new_in_mem::(None); + let mut state = new_in_mem::(); let backend = state.as_trie_backend().unwrap(); let mut overlay = OverlayedChanges::default(); diff --git a/primitives/state-machine/src/proving_backend.rs b/primitives/state-machine/src/proving_backend.rs index 1e8b1498a4f6f..558e22cdc7411 100644 --- a/primitives/state-machine/src/proving_backend.rs +++ b/primitives/state-machine/src/proving_backend.rs @@ -490,7 +490,7 @@ mod tests { let child_info_2 = ChildInfo::new_default(b"sub2"); let child_info_1 = &child_info_1; let child_info_2 = &child_info_2; - let mut contents = vec![ + let contents = vec![ (None, (0..64).map(|i| (vec![i], Some(vec![i]))).collect::>()), (Some(child_info_1.clone()), (28..65).map(|i| (vec![i], Some(vec![i]))).collect()), From 4eabbde29bc40141ca9867a5ca316b0a34e72540 Mon Sep 17 00:00:00 2001 From: cheme Date: Fri, 11 Jun 2021 09:23:38 +0200 Subject: [PATCH 051/127] rem empty space --- primitives/state-machine/src/trie_backend.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/primitives/state-machine/src/trie_backend.rs b/primitives/state-machine/src/trie_backend.rs index e0e44dd264b89..ff6067d44d0d8 100644 --- a/primitives/state-machine/src/trie_backend.rs +++ b/primitives/state-machine/src/trie_backend.rs @@ -199,7 +199,7 @@ impl, H: Hasher> Backend for TrieBackend where }; delta_trie_root::, _, _, _, _, _>(&mut eph, root, delta, layout) }; - + match res() { Ok(ret) => root = ret, Err(e) => warn!(target: "trie", "Failed to write to trie: {}", e), @@ -219,7 +219,7 @@ impl, H: Hasher> Backend for TrieBackend where } else { self.get_trie_alt_hashing_threshold() }; - + let default_root = match child_info.child_type() { ChildType::ParentKeyId => empty_child_trie_root::>() }; From ae627f6ca25187a201c8a193a9b628fe8ee855b3 Mon Sep 17 00:00:00 2001 From: cheme Date: Fri, 11 Jun 2021 15:00:21 +0200 Subject: [PATCH 052/127] Catch problem: when using triedb with default: we should not revert nodes: otherwhise thing as trie codec cannot decode-encode without changing state. --- Cargo.lock | 16 ++-- primitives/state-machine/src/lib.rs | 21 ++++- primitives/trie/src/lib.rs | 135 ++++++++++++++++++++++++++- primitives/trie/src/node_codec.rs | 12 ++- primitives/trie/src/node_header.rs | 2 +- primitives/trie/src/storage_proof.rs | 14 +-- primitives/trie/src/trie_codec.rs | 24 +++-- 7 files changed, 192 insertions(+), 32 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d2fe310d8bdbb..3871808cdef0b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2370,7 +2370,7 @@ dependencies = [ [[package]] name = "hash-db" version = "0.15.2" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#22910ad4af83e00b549cee918226efa940fdd3cf" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#3ba095cf3543b7b8140467741d90c8be0c69f5f3" [[package]] name = "hash256-std-hasher" @@ -2384,7 +2384,7 @@ dependencies = [ [[package]] name = "hash256-std-hasher" version = "0.15.2" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#22910ad4af83e00b549cee918226efa940fdd3cf" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#3ba095cf3543b7b8140467741d90c8be0c69f5f3" dependencies = [ "crunchy", ] @@ -3066,7 +3066,7 @@ dependencies = [ [[package]] name = "keccak-hasher" version = "0.15.3" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#22910ad4af83e00b549cee918226efa940fdd3cf" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#3ba095cf3543b7b8140467741d90c8be0c69f5f3" dependencies = [ "hash-db", "hash256-std-hasher 0.15.2 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple4)", @@ -3881,7 +3881,7 @@ dependencies = [ [[package]] name = "memory-db" version = "0.26.0" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#22910ad4af83e00b549cee918226efa940fdd3cf" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#3ba095cf3543b7b8140467741d90c8be0c69f5f3" dependencies = [ "hash-db", "hashbrown", @@ -10565,7 +10565,7 @@ checksum = "a7f741b240f1a48843f9b8e0444fb55fb2a4ff67293b50a9179dfd5ea67f8d41" [[package]] name = "trie-bench" version = "0.27.0" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#22910ad4af83e00b549cee918226efa940fdd3cf" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#3ba095cf3543b7b8140467741d90c8be0c69f5f3" dependencies = [ "criterion", "hash-db", @@ -10580,7 +10580,7 @@ dependencies = [ [[package]] name = "trie-db" version = "0.22.5" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#22910ad4af83e00b549cee918226efa940fdd3cf" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#3ba095cf3543b7b8140467741d90c8be0c69f5f3" dependencies = [ "hash-db", "hashbrown", @@ -10592,7 +10592,7 @@ dependencies = [ [[package]] name = "trie-root" version = "0.16.0" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#22910ad4af83e00b549cee918226efa940fdd3cf" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#3ba095cf3543b7b8140467741d90c8be0c69f5f3" dependencies = [ "hash-db", ] @@ -10610,7 +10610,7 @@ dependencies = [ [[package]] name = "trie-standardmap" version = "0.15.2" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#22910ad4af83e00b549cee918226efa940fdd3cf" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#3ba095cf3543b7b8140467741d90c8be0c69f5f3" dependencies = [ "hash-db", "keccak-hasher 0.15.3 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple4)", diff --git a/primitives/state-machine/src/lib.rs b/primitives/state-machine/src/lib.rs index ffb7152287f9b..420f8373fa43a 100644 --- a/primitives/state-machine/src/lib.rs +++ b/primitives/state-machine/src/lib.rs @@ -1585,19 +1585,31 @@ mod tests { remote_proof.encoded_size()); } + #[test] fn compact_multiple_child_trie() { + let size_inner_hash = compact_multiple_child_trie_inner(true); + let size_no_inner_hash = compact_multiple_child_trie_inner(false); + assert!(size_inner_hash < size_no_inner_hash); + } + fn compact_multiple_child_trie_inner(flagged: bool) -> usize { // this root will be queried let child_info1 = ChildInfo::new_default(b"sub1"); // this root will not be include in proof let child_info2 = ChildInfo::new_default(b"sub2"); // this root will be include in proof let child_info3 = ChildInfo::new_default(b"sub"); - let mut remote_backend = trie_backend::tests::test_trie(); + let mut remote_backend = trie_backend::tests::test_trie(flagged); + let long_vec: Vec = (0..1024usize).map(|_| 8u8).collect(); let (remote_root, transaction) = remote_backend.full_storage_root( std::iter::empty(), vec![ (&child_info1, vec![ - (&b"key1"[..], Some(&b"val2"[..])), + // a inner hashable node + (&b"k"[..], Some(&long_vec[..])), + // need to ensure this is not an inline node + // otherwhise we do not know what is accessed when + // storing proof. + (&b"key1"[..], Some(&vec![5u8; 32][..])), (&b"key2"[..], Some(&b"val3"[..])), ].into_iter()), (&child_info2, vec![ @@ -1612,11 +1624,13 @@ mod tests { ); remote_backend.backend_storage_mut().consolidate(transaction); remote_backend.essence.set_root(remote_root.clone()); + println!("{:?}", remote_root); let remote_proof = prove_child_read( remote_backend, &child_info1, &[b"key1"], ).unwrap(); + let size = remote_proof.encoded_size(); let remote_proof = test_compact(remote_proof, &remote_root); let local_result1 = read_child_proof_check::( remote_root, @@ -1625,7 +1639,8 @@ mod tests { &[b"key1"], ).unwrap(); assert_eq!(local_result1.len(), 1); - assert_eq!(local_result1.get(&b"key1"[..]), Some(&Some(b"val2".to_vec()))); + assert_eq!(local_result1.get(&b"key1"[..]), Some(&Some(vec![5u8; 32]))); + size } #[test] diff --git a/primitives/trie/src/lib.rs b/primitives/trie/src/lib.rs index 8acdcd1b8e85c..49e115c91a747 100644 --- a/primitives/trie/src/lib.rs +++ b/primitives/trie/src/lib.rs @@ -28,7 +28,7 @@ mod trie_stream; use sp_std::{boxed::Box, marker::PhantomData, vec, vec::Vec, borrow::Borrow, fmt}; use hash_db::{Hasher, Prefix}; -//use trie_db::proof::{generate_proof, verify_proof}; +use trie_db::proof::{generate_proof, verify_proof}; pub use trie_db::proof::VerifyError; /// Our `NodeCodec`-specific error. pub use error::Error; @@ -438,6 +438,52 @@ pub mod trie_types { pub type TrieError = trie_db::TrieError; } +/// Create a proof for a subset of keys in a trie. +/// +/// The `keys` may contain any set of keys regardless of each one of them is included +/// in the `db`. +/// +/// For a key `K` that is included in the `db` a proof of inclusion is generated. +/// For a key `K` that is not included in the `db` a proof of non-inclusion is generated. +/// These can be later checked in `verify_trie_proof`. +pub fn generate_trie_proof<'a, L, I, K, DB>( + db: &DB, + root: TrieHash, + keys: I, +) -> Result>, Box>> where + L: TrieConfiguration, + I: IntoIterator, + K: 'a + AsRef<[u8]>, + DB: hash_db::HashDBRef>, +{ + // Can use default layout (read only). + let trie = TrieDB::::new(db, &root)?; + generate_proof(&trie, keys) +} + +/// Verify a set of key-value pairs against a trie root and a proof. +/// +/// Checks a set of keys with optional values for inclusion in the proof that was generated by +/// `generate_trie_proof`. +/// If the value in the pair is supplied (`(key, Some(value))`), this key-value pair will be +/// checked for inclusion in the proof. +/// If the value is omitted (`(key, None)`), this key will be checked for non-inclusion in the +/// proof. +pub fn verify_trie_proof<'a, L, I, K, V>( + root: &TrieHash, + proof: &[Vec], + items: I, +) -> Result<(), VerifyError, error::Error>> where + L: TrieConfiguration, + I: IntoIterator)>, + K: 'a + AsRef<[u8]>, + V: 'a + AsRef<[u8]>, +{ + // No specific info to read from layout. + let layout = Default::default(); + verify_proof::, _, _, _>(root, proof, items, layout) +} + /// Determine a trie root given a hash DB and delta values. pub fn delta_trie_root( db: &mut DB, @@ -1172,6 +1218,93 @@ mod tests { assert_eq!(pairs, iter_pairs); } + #[test] + fn proof_non_inclusion_works() { + let pairs = vec![ + (hex!("0102").to_vec(), hex!("01").to_vec()), + (hex!("0203").to_vec(), hex!("0405").to_vec()), + ]; + + let mut memdb = MemoryDB::default(); + let mut root = Default::default(); + let layout = Layout::default(); + populate_trie::(&mut memdb, &mut root, &pairs, layout); + + let non_included_key: Vec = hex!("0909").to_vec(); + let proof = generate_trie_proof::( + &memdb, + root, + &[non_included_key.clone()] + ).unwrap(); + + // Verifying that the K was not included into the trie should work. + assert!(verify_trie_proof::>( + &root, + &proof, + &[(non_included_key.clone(), None)], + ).is_ok() + ); + + // Verifying that the K was included into the trie should fail. + assert!(verify_trie_proof::>( + &root, + &proof, + &[(non_included_key, Some(hex!("1010").to_vec()))], + ).is_err() + ); + } + + #[test] + fn proof_inclusion_works() { + let pairs = vec![ + (hex!("0102").to_vec(), hex!("01").to_vec()), + (hex!("0203").to_vec(), hex!("0405").to_vec()), + ]; + + let mut memdb = MemoryDB::default(); + let mut root = Default::default(); + let layout = Layout::default(); + populate_trie::(&mut memdb, &mut root, &pairs, layout); + + let proof = generate_trie_proof::( + &memdb, + root, + &[pairs[0].0.clone()] + ).unwrap(); + + // Check that a K, V included into the proof are verified. + assert!(verify_trie_proof::( + &root, + &proof, + &[(pairs[0].0.clone(), Some(pairs[0].1.clone()))] + ).is_ok() + ); + + // Absence of the V is not verified with the proof that has K, V included. + assert!(verify_trie_proof::>( + &root, + &proof, + &[(pairs[0].0.clone(), None)] + ).is_err() + ); + + // K not included into the trie is not verified. + assert!(verify_trie_proof::( + &root, + &proof, + &[(hex!("4242").to_vec(), Some(pairs[0].1.clone()))] + ).is_err() + ); + + // K included into the trie but not included into the proof is not verified. + assert!(verify_trie_proof::( + &root, + &proof, + &[(pairs[1].0.clone(), Some(pairs[1].1.clone()))] + ).is_err() + ); + } + #[test] fn generate_storage_root_with_proof_works_independently_from_the_delta_order() { let proof = StorageProof::decode(&mut &include_bytes!("../test-res/proof")[..]).unwrap(); diff --git a/primitives/trie/src/node_codec.rs b/primitives/trie/src/node_codec.rs index 9e2931486cba0..48186fa7f6b34 100644 --- a/primitives/trie/src/node_codec.rs +++ b/primitives/trie/src/node_codec.rs @@ -332,10 +332,14 @@ impl NodeCodecT for NodeCodec // utils fn value_do_hash(val: &Value, threshold: &u32) -> bool { - if let Value::Value(val) = val { - val.encoded_size() >= *threshold as usize - } else { - false + match val { + Value::Value(val) => { + val.encoded_size() >= *threshold as usize + }, + Value::HashedValue(..) => true, // can only keep hashed + Value::NoValue => { + false + }, } } diff --git a/primitives/trie/src/node_header.rs b/primitives/trie/src/node_header.rs index 6711c1a047127..7f5c68794e745 100644 --- a/primitives/trie/src/node_header.rs +++ b/primitives/trie/src/node_header.rs @@ -65,7 +65,7 @@ impl NodeHeader { match self { NodeHeader::Null | NodeHeader::Leaf(..) - | NodeHeader::Branch(..) => false, + | NodeHeader::Branch(..) => false, NodeHeader::AltHashBranch(..) | NodeHeader::AltHashLeaf(..) => true, } diff --git a/primitives/trie/src/storage_proof.rs b/primitives/trie/src/storage_proof.rs index b13cad34d6ccd..43a32b25ebfb7 100644 --- a/primitives/trie/src/storage_proof.rs +++ b/primitives/trie/src/storage_proof.rs @@ -18,6 +18,9 @@ use sp_std::vec::Vec; use codec::{Encode, Decode}; use hash_db::{Hasher, HashDB}; +use hash_db::MetaHasher; +use trie_db::NodeCodec; +use crate::{trie_types::Layout, TrieLayout}; /// A proof that some set of key-value pairs are included in the storage trie. The proof contains /// the storage values so that the partial storage backend can be reconstructed by a verifier that @@ -101,8 +104,8 @@ impl StorageProof { pub fn into_compact_proof( self, root: H::Out, - ) -> Result>> { - crate::encode_compact::>(self, root) + ) -> Result>> { + crate::encode_compact::>(self, root) } /// Returns the estimated encoded size of the compact proof. @@ -130,9 +133,9 @@ impl CompactProof { pub fn to_storage_proof( &self, expected_root: Option<&H::Out>, - ) -> Result<(StorageProof, H::Out), crate::CompactProofError>> { + ) -> Result<(StorageProof, H::Out), crate::CompactProofError>> { let mut db = crate::MemoryDB::::new(&[]); - let root = crate::decode_compact::, _, _>( + let root = crate::decode_compact::, _, _>( &mut db, self.iter_compact_encoded_nodes(), expected_root, @@ -171,9 +174,6 @@ impl Iterator for StorageProofNodeIterator { impl From for crate::MemoryDB { fn from(proof: StorageProof) -> Self { - use hash_db::MetaHasher; - use trie_db::NodeCodec; - use crate::{trie_types::Layout, TrieLayout}; let mut db = crate::MemoryDB::default(); // Needed because we do not read trie structure, so // we do a heuristic related to the fact that host function diff --git a/primitives/trie/src/trie_codec.rs b/primitives/trie/src/trie_codec.rs index efe3223580f3f..3ffc2a4197f95 100644 --- a/primitives/trie/src/trie_codec.rs +++ b/primitives/trie/src/trie_codec.rs @@ -22,7 +22,7 @@ use crate::{ EMPTY_PREFIX, HashDBT, TrieHash, TrieError, TrieConfiguration, - CompactProof, StorageProof, + CompactProof, StorageProof, GlobalMeta, TrieMeta, }; use sp_std::boxed::Box; use sp_std::vec::Vec; @@ -109,13 +109,17 @@ pub fn decode_compact<'a, L, DB, I>( ) -> Result, Error> where L: TrieConfiguration, - DB: HashDBT + hash_db::HashDBRef, + DB: HashDBT> + + hash_db::HashDBRef>, I: IntoIterator, { let mut nodes_iter = encoded.into_iter(); - let (top_root, _nb_used) = trie_db::decode_compact_from_iter::( + // Layout does not change trie reading. + let layout = L::default(); + let (top_root, _nb_used) = trie_db::decode_compact_from_iter::( db, &mut nodes_iter, + &layout, )?; // Only check root if expected root is passed as argument. @@ -128,7 +132,7 @@ pub fn decode_compact<'a, L, DB, I>( let mut child_tries = Vec::new(); { // fetch child trie roots - let trie = crate::TrieDB::::new(db, &top_root)?; + let trie = crate::TrieDB::::new_with_layout(db, &top_root, layout.clone())?; let mut iter = trie.iter()?; @@ -159,16 +163,17 @@ pub fn decode_compact<'a, L, DB, I>( } } - if !HashDBT::::contains(db, &top_root, EMPTY_PREFIX) { + if !HashDBT::::contains(db, &top_root, EMPTY_PREFIX) { return Err(Error::IncompleteProof); } let mut previous_extracted_child_trie = None; for child_root in child_tries.into_iter() { if previous_extracted_child_trie.is_none() { - let (top_root, _) = trie_db::decode_compact_from_iter::( + let (top_root, _) = trie_db::decode_compact_from_iter::( db, &mut nodes_iter, + &layout, )?; previous_extracted_child_trie = Some(top_root); } @@ -206,11 +211,14 @@ pub fn encode_compact( root: TrieHash, ) -> Result> where - L: TrieConfiguration, + L: TrieConfiguration, { let mut child_tries = Vec::new(); let partial_db = proof.into_memory_db(); let mut compact_proof = { + // Layout does not change trie reading. + // And meta for writing are read from state + // (no new node so using trie without threshold is safe here). let trie = crate::TrieDB::::new(&partial_db, &root)?; let mut iter = trie.iter()?; @@ -243,7 +251,7 @@ pub fn encode_compact( }; for child_root in child_tries { - if !HashDBT::::contains(&partial_db, &child_root, EMPTY_PREFIX) { + if !HashDBT::::contains(&partial_db, &child_root, EMPTY_PREFIX) { // child proof are allowed to be missing (unused root can be included // due to trie structure modification). continue; From 4706814001393d2deeec102c77ea2c22b2df6ecc Mon Sep 17 00:00:00 2001 From: cheme Date: Fri, 11 Jun 2021 15:19:56 +0200 Subject: [PATCH 053/127] fix compilation --- client/db/src/bench.rs | 2 +- primitives/transaction-storage-proof/src/lib.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/client/db/src/bench.rs b/client/db/src/bench.rs index 30f709bf9c9e8..de7e1b874c97e 100644 --- a/client/db/src/bench.rs +++ b/client/db/src/bench.rs @@ -540,7 +540,7 @@ impl StateBackend> for BenchmarkingState { fn proof_size(&self) -> Option { self.proof_recorder.as_ref().map(|recorder| { let proof_size = recorder.estimate_encoded_size() as u32; - let proof = recorder.to_storage_proof(); + let proof = recorder.to_storage_proof::>(); let proof_recorder_root = self.proof_recorder_root.get(); if proof_recorder_root == Default::default() || proof_size == 1 { // empty trie diff --git a/primitives/transaction-storage-proof/src/lib.rs b/primitives/transaction-storage-proof/src/lib.rs index 825de27b2a5a9..7891edb0b2dc5 100644 --- a/primitives/transaction-storage-proof/src/lib.rs +++ b/primitives/transaction-storage-proof/src/lib.rs @@ -141,7 +141,7 @@ pub mod registration { use super::*; type Hasher = sp_core::Blake2Hasher; - type TrieLayout = sp_trie::Layout::; + type TrieLayout = sp_trie::trie_types::Layout::; /// Create a new inherent data provider instance for a given parent block hash. pub fn new_data_provider( From 09d5cf6e8b252540e12abfe172193e7599f50a33 Mon Sep 17 00:00:00 2001 From: cheme Date: Fri, 11 Jun 2021 16:13:11 +0200 Subject: [PATCH 054/127] Right logic to avoid switch on reencode when default layout. --- primitives/trie/src/lib.rs | 11 +++++++---- primitives/trie/src/node_codec.rs | 4 ++-- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/primitives/trie/src/lib.rs b/primitives/trie/src/lib.rs index 49e115c91a747..7e4e77c8f8df1 100644 --- a/primitives/trie/src/lib.rs +++ b/primitives/trie/src/lib.rs @@ -63,6 +63,8 @@ pub struct TrieMeta { /// `TrieDbMut` it switch nodes to alternative hashing /// method by defining the threshold to use with alternative /// hashing. + /// Trie codec or other proof manipulation will always use + /// `None` in order to prevent state change on reencoding. pub try_inner_hashing: Option, /// Flag indicating alternative value hash is currently use /// or will be use. @@ -163,9 +165,10 @@ impl Meta for TrieMeta { ValuePlan::NoValue => return, }; - self.apply_inner_hashing = self.try_inner_hashing.as_ref().map(|threshold| - range.end - range.start >= *threshold as usize - ).unwrap_or(false); + if let Some(threshold) = self.try_inner_hashing.clone() { + self.apply_inner_hashing = range.end - range.start >= threshold as usize; + } + self.range = Some(range); self.contain_hash = contain_hash; } @@ -189,7 +192,7 @@ impl Meta for TrieMeta { self.contain_hash } - // TODO remove upstream + // TODO rename to get state meta fn do_value_hash(&self) -> bool { self.apply_inner_hashing } diff --git a/primitives/trie/src/node_codec.rs b/primitives/trie/src/node_codec.rs index 48186fa7f6b34..3528b5790d490 100644 --- a/primitives/trie/src/node_codec.rs +++ b/primitives/trie/src/node_codec.rs @@ -227,7 +227,7 @@ impl NodeCodecT for NodeCodec // sometime). let mut output = if meta.extract_global_meta().as_ref().map(|threshold| value_do_hash(&value, threshold) - ).unwrap_or(false) { + ).unwrap_or(meta.do_value_hash()) { partial_encode(partial, NodeKind::AltHashLeaf) } else { partial_encode(partial, NodeKind::Leaf) @@ -279,7 +279,7 @@ impl NodeCodecT for NodeCodec ) -> Vec { let mut output = match (&value, meta.extract_global_meta().as_ref().map(|threshold| value_do_hash(&value, threshold) - ).unwrap_or(false)) { + ).unwrap_or(meta.do_value_hash())) { (&Value::NoValue, _) => { partial_from_iterator_encode(partial, number_nibble, NodeKind::BranchNoValue) }, From 906cd48fc8061a27ed5de2b7f5477e8cfaa30976 Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 14 Jun 2021 17:13:45 +0200 Subject: [PATCH 055/127] Clean up some todos --- Cargo.lock | 24 ++++++++--------- Cargo.toml | 10 ++++---- client/db/src/lib.rs | 1 - primitives/externalities/src/lib.rs | 27 -------------------- primitives/state-machine/src/trie_backend.rs | 1 - primitives/trie/src/lib.rs | 8 ++---- primitives/trie/src/node_codec.rs | 15 +++++------ 7 files changed, 25 insertions(+), 61 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3871808cdef0b..ad845adf59f43 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2370,7 +2370,7 @@ dependencies = [ [[package]] name = "hash-db" version = "0.15.2" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#3ba095cf3543b7b8140467741d90c8be0c69f5f3" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-refact#3ba095cf3543b7b8140467741d90c8be0c69f5f3" [[package]] name = "hash256-std-hasher" @@ -2384,7 +2384,7 @@ dependencies = [ [[package]] name = "hash256-std-hasher" version = "0.15.2" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#3ba095cf3543b7b8140467741d90c8be0c69f5f3" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-refact#3ba095cf3543b7b8140467741d90c8be0c69f5f3" dependencies = [ "crunchy", ] @@ -3066,10 +3066,10 @@ dependencies = [ [[package]] name = "keccak-hasher" version = "0.15.3" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#3ba095cf3543b7b8140467741d90c8be0c69f5f3" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-refact#3ba095cf3543b7b8140467741d90c8be0c69f5f3" dependencies = [ "hash-db", - "hash256-std-hasher 0.15.2 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple4)", + "hash256-std-hasher 0.15.2 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-refact)", "tiny-keccak", ] @@ -3881,7 +3881,7 @@ dependencies = [ [[package]] name = "memory-db" version = "0.26.0" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#3ba095cf3543b7b8140467741d90c8be0c69f5f3" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-refact#3ba095cf3543b7b8140467741d90c8be0c69f5f3" dependencies = [ "hash-db", "hashbrown", @@ -10565,22 +10565,22 @@ checksum = "a7f741b240f1a48843f9b8e0444fb55fb2a4ff67293b50a9179dfd5ea67f8d41" [[package]] name = "trie-bench" version = "0.27.0" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#3ba095cf3543b7b8140467741d90c8be0c69f5f3" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-refact#3ba095cf3543b7b8140467741d90c8be0c69f5f3" dependencies = [ "criterion", "hash-db", - "keccak-hasher 0.15.3 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple4)", + "keccak-hasher 0.15.3 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-refact)", "memory-db", "parity-scale-codec", "trie-db", "trie-root", - "trie-standardmap 0.15.2 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple4)", + "trie-standardmap 0.15.2 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-refact)", ] [[package]] name = "trie-db" version = "0.22.5" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#3ba095cf3543b7b8140467741d90c8be0c69f5f3" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-refact#3ba095cf3543b7b8140467741d90c8be0c69f5f3" dependencies = [ "hash-db", "hashbrown", @@ -10592,7 +10592,7 @@ dependencies = [ [[package]] name = "trie-root" version = "0.16.0" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#3ba095cf3543b7b8140467741d90c8be0c69f5f3" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-refact#3ba095cf3543b7b8140467741d90c8be0c69f5f3" dependencies = [ "hash-db", ] @@ -10610,10 +10610,10 @@ dependencies = [ [[package]] name = "trie-standardmap" version = "0.15.2" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#3ba095cf3543b7b8140467741d90c8be0c69f5f3" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-refact#3ba095cf3543b7b8140467741d90c8be0c69f5f3" dependencies = [ "hash-db", - "keccak-hasher 0.15.3 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple4)", + "keccak-hasher 0.15.3 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-refact)", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 754e1ea9f9b7d..566401fbe28f5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -278,8 +278,8 @@ zeroize = { opt-level = 3 } panic = "unwind" [patch.crates-io] -hash-db = { git = "https://github.com/cheme/trie.git", branch = "hashed-value-simple4" } -memory-db = { git = "https://github.com/cheme/trie.git", branch = "hashed-value-simple4" } -trie-db = { git = "https://github.com/cheme/trie.git", branch = "hashed-value-simple4" } -trie-root = { git = "https://github.com/cheme/trie.git", branch = "hashed-value-simple4" } -trie-bench = { git = "https://github.com/cheme/trie.git", branch = "hashed-value-simple4" } +hash-db = { git = "https://github.com/cheme/trie.git", branch = "hashed-value-simple4-refact" } +memory-db = { git = "https://github.com/cheme/trie.git", branch = "hashed-value-simple4-refact" } +trie-db = { git = "https://github.com/cheme/trie.git", branch = "hashed-value-simple4-refact" } +trie-root = { git = "https://github.com/cheme/trie.git", branch = "hashed-value-simple4-refact" } +trie-bench = { git = "https://github.com/cheme/trie.git", branch = "hashed-value-simple4-refact" } diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index 66cedb6b54b7d..5c2ed5c607818 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -807,7 +807,6 @@ impl sc_client_api::backend::BlockImportOperation for Bloc )); let mut changes_trie_config: Option = None; - // TODO test genesis init with a threshold let (root, transaction) = self.old_state.full_storage_root( storage.top.iter().map(|(k, v)| { if &k[..] == well_known_keys::CHANGES_TRIE_CONFIG { diff --git a/primitives/externalities/src/lib.rs b/primitives/externalities/src/lib.rs index 3cf379131795a..14145e8798498 100644 --- a/primitives/externalities/src/lib.rs +++ b/primitives/externalities/src/lib.rs @@ -323,30 +323,3 @@ impl ExternalitiesExt for &mut dyn Externalities { self.deregister_extension_by_type_id(TypeId::of::()) } } - -/// Helpers method for the [`Externalities`] trait. -pub trait ExternalitiesHelpers: Externalities { - /* No we use backend in this case TODO remove function - /// Utility function to get trie inner value hash threshold from - /// backend state or pending changes. - fn get_trie_alt_hashing_threshold(&self) -> Option { - self.storage(sp_storage::well_known_keys::TRIE_HASHING_CONFIG) - .and_then(|encoded| sp_storage::trie_threshold_decode(&mut encoded.as_slice())) - } - */ - - /// Utility function to modify trie inner value hash threshold. - fn modify_trie_alt_hashing_threshold(&mut self, threshold: Option) { - match threshold { - Some(threshold) => { - let encoded = sp_storage::trie_threshold_encode(threshold); - self.set_storage(sp_storage::well_known_keys::TRIE_HASHING_CONFIG.to_vec(), encoded); - }, - None => { - self.clear_storage(sp_storage::well_known_keys::TRIE_HASHING_CONFIG); - }, - } - } -} - -impl ExternalitiesHelpers for E { } diff --git a/primitives/state-machine/src/trie_backend.rs b/primitives/state-machine/src/trie_backend.rs index ff6067d44d0d8..cfde64dbc119d 100644 --- a/primitives/state-machine/src/trie_backend.rs +++ b/primitives/state-machine/src/trie_backend.rs @@ -180,7 +180,6 @@ impl, H: Hasher> Backend for TrieBackend where let use_inner_hash_value = if let Some(force) = self.force_alt_hashing.as_ref() { force.clone() } else { - // TODO try memoize in force self.get_trie_alt_hashing_threshold() }; let mut write_overlay = S::Overlay::default(); diff --git a/primitives/trie/src/lib.rs b/primitives/trie/src/lib.rs index 7e4e77c8f8df1..8a3a0f60314cf 100644 --- a/primitives/trie/src/lib.rs +++ b/primitives/trie/src/lib.rs @@ -76,8 +76,7 @@ pub struct TrieMeta { /// set as accessed by defalult, but can be /// change on access explicitely: `HashDB::get_with_meta`. /// and reset on access explicitely: `HashDB::access_from`. - /// TODO!! could be remove from meta: only use in proof recorder context. - /// But does not add memory usage here. + /// Not strictly needed in this struct, but does not add memory usage here. pub unused_value: bool, } @@ -859,10 +858,7 @@ pub fn estimate_entry_size(entry: &(DBValue, TrieMeta), hash_len: usize) -> usiz full_encoded } -/// If needed, call to decode plan in order to update meta earlier. -/// TODO if removing fully meta, this will still be needed but with -/// a less generic name: read variant of node from db value and indicate -/// if can hash value. +/// Decode plan in order to update meta early (needed to register proofs). pub fn resolve_encoded_meta(entry: &mut (DBValue, TrieMeta)) { use trie_db::NodeCodec; let _ = as TrieLayout>::Codec::decode_plan(entry.0.as_slice(), &mut entry.1); diff --git a/primitives/trie/src/node_codec.rs b/primitives/trie/src/node_codec.rs index 3528b5790d490..1358089ea8a16 100644 --- a/primitives/trie/src/node_codec.rs +++ b/primitives/trie/src/node_codec.rs @@ -90,16 +90,14 @@ pub struct NodeCodec(PhantomData); impl NodeCodec { fn decode_plan_inner_hashed>( data: &[u8], - mut meta: Option<&mut M>, // TODO when remove no meta, remove option + meta: &mut M, ) -> Result { let mut input = ByteSliceInput::new(data); - let contains_hash = meta.as_ref() - .map(|m| m.contains_hash_of_value()).unwrap_or_default(); + let contains_hash = meta.contains_hash_of_value(); let header = NodeHeader::decode(&mut input)?; let alt_hashing = header.alt_hashing(); - meta.as_mut() - .map(|m| m.set_state_meta(alt_hashing)); + meta.set_state_meta(alt_hashing); let branch_has_value = if let NodeHeader::Branch(has_value, _) = &header { *has_value @@ -196,15 +194,14 @@ impl NodeCodecT for NodeCodec } fn decode_plan(data: &[u8], meta: &mut M) -> Result { - Self::decode_plan_inner_hashed(data, Some(meta)).map(|plan| { + Self::decode_plan_inner_hashed(data, meta).map(|plan| { meta.decoded_callback(&plan); plan }) } - fn decode_plan_inner(data: &[u8]) -> Result { - let meta: Option<&mut M> = None; - Self::decode_plan_inner_hashed(data, meta) + fn decode_plan_inner(_data: &[u8]) -> Result { + unreachable!("decode_plan is implemented") } fn is_empty_node(data: &[u8]) -> bool { From 662eabe858b1e58384b6c48dd22d95a37d7353f2 Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 14 Jun 2021 18:50:25 +0200 Subject: [PATCH 056/127] remove trie meta from root upstream --- Cargo.lock | 16 ++-- bin/node/bench/src/generator.rs | 2 +- primitives/state-machine/src/basic.rs | 2 +- primitives/state-machine/src/lib.rs | 2 +- primitives/state-machine/src/trie_backend.rs | 6 +- primitives/trie/src/lib.rs | 78 +++----------------- primitives/trie/src/node_codec.rs | 8 +- 7 files changed, 29 insertions(+), 85 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ad845adf59f43..e4eaab283dcc2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2370,7 +2370,7 @@ dependencies = [ [[package]] name = "hash-db" version = "0.15.2" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-refact#3ba095cf3543b7b8140467741d90c8be0c69f5f3" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-refact#b4b0d1919db7b100aba040c336d3c69ed11faa36" [[package]] name = "hash256-std-hasher" @@ -2384,7 +2384,7 @@ dependencies = [ [[package]] name = "hash256-std-hasher" version = "0.15.2" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-refact#3ba095cf3543b7b8140467741d90c8be0c69f5f3" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-refact#b4b0d1919db7b100aba040c336d3c69ed11faa36" dependencies = [ "crunchy", ] @@ -3066,7 +3066,7 @@ dependencies = [ [[package]] name = "keccak-hasher" version = "0.15.3" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-refact#3ba095cf3543b7b8140467741d90c8be0c69f5f3" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-refact#b4b0d1919db7b100aba040c336d3c69ed11faa36" dependencies = [ "hash-db", "hash256-std-hasher 0.15.2 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-refact)", @@ -3881,7 +3881,7 @@ dependencies = [ [[package]] name = "memory-db" version = "0.26.0" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-refact#3ba095cf3543b7b8140467741d90c8be0c69f5f3" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-refact#b4b0d1919db7b100aba040c336d3c69ed11faa36" dependencies = [ "hash-db", "hashbrown", @@ -10565,7 +10565,7 @@ checksum = "a7f741b240f1a48843f9b8e0444fb55fb2a4ff67293b50a9179dfd5ea67f8d41" [[package]] name = "trie-bench" version = "0.27.0" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-refact#3ba095cf3543b7b8140467741d90c8be0c69f5f3" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-refact#b4b0d1919db7b100aba040c336d3c69ed11faa36" dependencies = [ "criterion", "hash-db", @@ -10580,7 +10580,7 @@ dependencies = [ [[package]] name = "trie-db" version = "0.22.5" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-refact#3ba095cf3543b7b8140467741d90c8be0c69f5f3" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-refact#b4b0d1919db7b100aba040c336d3c69ed11faa36" dependencies = [ "hash-db", "hashbrown", @@ -10592,7 +10592,7 @@ dependencies = [ [[package]] name = "trie-root" version = "0.16.0" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-refact#3ba095cf3543b7b8140467741d90c8be0c69f5f3" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-refact#b4b0d1919db7b100aba040c336d3c69ed11faa36" dependencies = [ "hash-db", ] @@ -10610,7 +10610,7 @@ dependencies = [ [[package]] name = "trie-standardmap" version = "0.15.2" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-refact#3ba095cf3543b7b8140467741d90c8be0c69f5f3" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-refact#b4b0d1919db7b100aba040c336d3c69ed11faa36" dependencies = [ "hash-db", "keccak-hasher 0.15.3 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-refact)", diff --git a/bin/node/bench/src/generator.rs b/bin/node/bench/src/generator.rs index 6589bdcd0b277..876321a3a2756 100644 --- a/bin/node/bench/src/generator.rs +++ b/bin/node/bench/src/generator.rs @@ -45,7 +45,7 @@ pub fn generate_trie( { let mut trie_db = if let Some(threshold) = alt_hashing { - let layout = sp_trie::Layout::with_inner_hashing(threshold); + let layout = sp_trie::Layout::with_alt_hashing(threshold); TrieDBMut::::new_with_layout(&mut trie, &mut root, layout) } else { TrieDBMut::new(&mut trie, &mut root) diff --git a/primitives/state-machine/src/basic.rs b/primitives/state-machine/src/basic.rs index 7d7be10f2b584..60aa1918c6fd3 100644 --- a/primitives/state-machine/src/basic.rs +++ b/primitives/state-machine/src/basic.rs @@ -288,7 +288,7 @@ impl Externalities for BasicExternalities { } let layout = if let Some(threshold) = self.alt_hashing.as_ref() { - Layout::::with_inner_hashing(*threshold) + Layout::::with_alt_hashing(*threshold) } else { Layout::::default() }; diff --git a/primitives/state-machine/src/lib.rs b/primitives/state-machine/src/lib.rs index 420f8373fa43a..5130a432fece8 100644 --- a/primitives/state-machine/src/lib.rs +++ b/primitives/state-machine/src/lib.rs @@ -1547,7 +1547,7 @@ mod tests { // do switch - layout = Layout::with_inner_hashing(TRESHOLD); + layout = Layout::with_alt_hashing(TRESHOLD); // update with same value do not change { let mut trie = TrieDBMut::from_existing_with_layout( diff --git a/primitives/state-machine/src/trie_backend.rs b/primitives/state-machine/src/trie_backend.rs index cfde64dbc119d..e712fbc23f172 100644 --- a/primitives/state-machine/src/trie_backend.rs +++ b/primitives/state-machine/src/trie_backend.rs @@ -192,7 +192,7 @@ impl, H: Hasher> Backend for TrieBackend where ); let res = || { let layout = if let Some(threshold) = use_inner_hash_value { - sp_trie::Layout::with_inner_hashing(threshold) + sp_trie::Layout::with_alt_hashing(threshold) } else { sp_trie::Layout::default() }; @@ -223,7 +223,7 @@ impl, H: Hasher> Backend for TrieBackend where ChildType::ParentKeyId => empty_child_trie_root::>() }; let layout = if let Some(threshold) = use_inner_hash_value { - sp_trie::Layout::with_inner_hashing(threshold) + sp_trie::Layout::with_alt_hashing(threshold) } else { sp_trie::Layout::default() }; @@ -304,7 +304,7 @@ pub mod tests { let mut sub_root = Vec::new(); root.encode_to(&mut sub_root); let mut trie = if hashed_value { - let layout = Layout::with_inner_hashing(TRESHOLD); + let layout = Layout::with_alt_hashing(TRESHOLD); TrieDBMut::new_with_layout(&mut mdb, &mut root, layout) } else { TrieDBMut::new(&mut mdb, &mut root) diff --git a/primitives/trie/src/lib.rs b/primitives/trie/src/lib.rs index 8a3a0f60314cf..68d18a5621ce5 100644 --- a/primitives/trie/src/lib.rs +++ b/primitives/trie/src/lib.rs @@ -91,45 +91,16 @@ impl Meta for TrieMeta { self.apply_inner_hashing = state_meta; } - // TODO rename upstream as read_global_meta - fn extract_global_meta(&self) -> Self::GlobalMeta { - self.try_inner_hashing - } - - fn set_global_meta(&mut self, global_meta: Self::GlobalMeta) { - self.try_inner_hashing = global_meta; - } - - // TODO remove upstream? - fn has_state_meta(&self) -> bool { + fn read_state_meta(&self) -> Self::StateMeta { self.apply_inner_hashing } - // TODO consider removal upstream of this method (node type in codec) - fn read_state_meta(&mut self, _data: &[u8]) -> Result { - unreachable!() - // TODO read directly from codec. -/* let offset = if data[0] == trie_constants::ENCODED_META_ALLOW_HASH { - self.recorded_do_value_hash = true; - self.do_value_hash = true; - 1 - } else { - 0 - }; - Ok(offset)*/ + fn read_global_meta(&self) -> Self::GlobalMeta { + self.try_inner_hashing } - // TODO consider removal upstream of this method (node type in codec) - // `do_value_hash` method is enough function to write with codec. - fn write_state_meta(&self) -> Vec { - unreachable!() -/* if self.do_value_hash { - // Note that this only works with sp_trie codec. - // Acts as a boolean result. - [trie_constants::ENCODED_META_ALLOW_HASH].to_vec() - } else { - Vec::new() - }*/ + fn set_global_meta(&mut self, global_meta: Self::GlobalMeta) { + self.try_inner_hashing = global_meta; } fn meta_for_new( @@ -146,14 +117,12 @@ impl Meta for TrieMeta { Self::meta_for_new(global) } - // TODO meta for empty is unused: can consider removal upstream. fn meta_for_empty( global: Self::GlobalMeta, ) -> Self { Self::meta_for_new(global) } - // TODO if removing all meta, the Option will replace it. fn encoded_value_callback( &mut self, value_plan: ValuePlan, @@ -190,11 +159,6 @@ impl Meta for TrieMeta { fn contains_hash_of_value(&self) -> bool { self.contain_hash } - - // TODO rename to get state meta - fn do_value_hash(&self) -> bool { - self.apply_inner_hashing - } } impl TrieMeta { @@ -234,8 +198,7 @@ impl Default for Layout { impl Layout { /// Layout with inner hashing active. /// Will flag trie for hashing. - /// TODO rename inner -> alt - pub fn with_inner_hashing(threshold: u32) -> Self { + pub fn with_alt_hashing(threshold: u32) -> Self { Layout(Some(threshold), sp_std::marker::PhantomData) } } @@ -249,7 +212,6 @@ impl TrieLayout for Layout const USE_EXTENSION: bool = false; const ALLOW_EMPTY: bool = true; const USE_META: bool = true; - const READ_ROOT_STATE_META: bool = false; // TODO rem type Hash = H; type Codec = NodeCodec; @@ -259,20 +221,6 @@ impl TrieLayout for Layout fn layout_meta(&self) -> GlobalMeta { self.0 } - - // TODO remove upstream - fn initialize_from_root_meta(&mut self, _root_meta: &Self::Meta) { - unreachable!() - /*if root_meta.extract_global_meta() { - self.0 = true; - }*/ - } - - // TODO remove upstream - fn set_root_meta(_root_meta: &mut Self::Meta, _global_meta: GlobalMeta) { - unreachable!() -// root_meta.set_global_meta(global_meta); - } } /// Hasher with support to meta. @@ -302,8 +250,6 @@ impl MetaHasher for StateHasher } } - // TODO if removing meta upstream, still need to get DEAD_HEADER_META_HASHED_VALUE - // from proof. fn stored_value(value: &[u8], mut meta: Self::Meta) -> DBValue { let mut stored = Vec::with_capacity(value.len() + 1); if meta.contain_hash { @@ -314,8 +260,8 @@ impl MetaHasher for StateHasher } if meta.unused_value && meta.apply_inner_hashing { if meta.range.is_some() { - // Waring this assume that encoded value does not start by this, so it is tightly coupled - // with the header type of the codec: only for optimization. + // Warning this assumes that encoded value cannot start by this, + // so it is tightly coupled with the header type of the codec. stored.push(trie_constants::DEAD_HEADER_META_HASHED_VALUE); let range = meta.range.as_ref().expect("Tested in condition"); meta.contain_hash = true; // useless but could be with meta as &mut @@ -333,7 +279,6 @@ impl MetaHasher for StateHasher >::stored_value(value.as_slice(), meta) } - // TODO remove upstream? fn extract_value(mut stored: &[u8], global_meta: Self::GlobalMeta) -> (&[u8], Self::Meta) { let input = &mut stored; let mut contain_hash = false; @@ -352,7 +297,6 @@ impl MetaHasher for StateHasher (stored, meta) } - // TODO remove upstream fn extract_value_owned(mut stored: DBValue, global: Self::GlobalMeta) -> (DBValue, Self::Meta) { let len = stored.len(); let (v, meta) = >::extract_value(stored.as_slice(), global); @@ -941,7 +885,7 @@ mod tests { let layout = Layout::default(); check_equivalent::(input, layout.clone()); check_iteration::(input, layout); - let layout = Layout::with_inner_hashing(TRESHOLD); + let layout = Layout::with_alt_hashing(TRESHOLD); check_equivalent::(input, layout.clone()); check_iteration::(input, layout); } @@ -1094,7 +1038,7 @@ mod tests { }.make_with(seed.as_fixed_bytes_mut()); let layout = if flag { - Layout::with_inner_hashing(TRESHOLD) + Layout::with_alt_hashing(TRESHOLD) } else { Layout::default() }; @@ -1191,7 +1135,7 @@ mod tests { } fn iterator_works_inner(flag: bool) { let layout = if flag { - Layout::with_inner_hashing(TRESHOLD) + Layout::with_alt_hashing(TRESHOLD) } else { Layout::default() }; diff --git a/primitives/trie/src/node_codec.rs b/primitives/trie/src/node_codec.rs index 1358089ea8a16..bd6e6d476f3e8 100644 --- a/primitives/trie/src/node_codec.rs +++ b/primitives/trie/src/node_codec.rs @@ -222,9 +222,9 @@ impl NodeCodecT for NodeCodec // With fix inner hashing alt hash can be use with all node, but // that is not better (encoding can use an additional nibble byte // sometime). - let mut output = if meta.extract_global_meta().as_ref().map(|threshold| + let mut output = if meta.read_global_meta().as_ref().map(|threshold| value_do_hash(&value, threshold) - ).unwrap_or(meta.do_value_hash()) { + ).unwrap_or(meta.read_state_meta()) { partial_encode(partial, NodeKind::AltHashLeaf) } else { partial_encode(partial, NodeKind::Leaf) @@ -274,9 +274,9 @@ impl NodeCodecT for NodeCodec value: Value, meta: &mut M, ) -> Vec { - let mut output = match (&value, meta.extract_global_meta().as_ref().map(|threshold| + let mut output = match (&value, meta.read_global_meta().as_ref().map(|threshold| value_do_hash(&value, threshold) - ).unwrap_or(meta.do_value_hash())) { + ).unwrap_or(meta.read_state_meta())) { (&Value::NoValue, _) => { partial_from_iterator_encode(partial, number_nibble, NodeKind::BranchNoValue) }, From ac0e019f709abd084648f727f98fd06cb3c781ab Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 15 Jun 2021 11:30:02 +0200 Subject: [PATCH 057/127] update upstream and fix benches. --- Cargo.lock | 16 ++++++++-------- bin/node/executor/benches/bench.rs | 2 +- primitives/api/test/benches/bench.rs | 2 +- primitives/trie/benches/bench.rs | 4 ++-- primitives/trie/src/lib.rs | 6 +++--- 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index acfb520048c5d..e3ae3bdce9add 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2354,7 +2354,7 @@ dependencies = [ [[package]] name = "hash-db" version = "0.15.2" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-refact#b4b0d1919db7b100aba040c336d3c69ed11faa36" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-refact#54c61d7e3c32a4090d34468c38621d33b0f853b5" [[package]] name = "hash256-std-hasher" @@ -2368,7 +2368,7 @@ dependencies = [ [[package]] name = "hash256-std-hasher" version = "0.15.2" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-refact#b4b0d1919db7b100aba040c336d3c69ed11faa36" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-refact#54c61d7e3c32a4090d34468c38621d33b0f853b5" dependencies = [ "crunchy", ] @@ -3050,7 +3050,7 @@ dependencies = [ [[package]] name = "keccak-hasher" version = "0.15.3" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-refact#b4b0d1919db7b100aba040c336d3c69ed11faa36" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-refact#54c61d7e3c32a4090d34468c38621d33b0f853b5" dependencies = [ "hash-db", "hash256-std-hasher 0.15.2 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-refact)", @@ -3865,7 +3865,7 @@ dependencies = [ [[package]] name = "memory-db" version = "0.26.0" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-refact#b4b0d1919db7b100aba040c336d3c69ed11faa36" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-refact#54c61d7e3c32a4090d34468c38621d33b0f853b5" dependencies = [ "hash-db", "hashbrown", @@ -10547,7 +10547,7 @@ checksum = "a7f741b240f1a48843f9b8e0444fb55fb2a4ff67293b50a9179dfd5ea67f8d41" [[package]] name = "trie-bench" version = "0.27.0" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-refact#b4b0d1919db7b100aba040c336d3c69ed11faa36" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-refact#54c61d7e3c32a4090d34468c38621d33b0f853b5" dependencies = [ "criterion", "hash-db", @@ -10562,7 +10562,7 @@ dependencies = [ [[package]] name = "trie-db" version = "0.22.5" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-refact#b4b0d1919db7b100aba040c336d3c69ed11faa36" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-refact#54c61d7e3c32a4090d34468c38621d33b0f853b5" dependencies = [ "hash-db", "hashbrown", @@ -10574,7 +10574,7 @@ dependencies = [ [[package]] name = "trie-root" version = "0.16.0" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-refact#b4b0d1919db7b100aba040c336d3c69ed11faa36" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-refact#54c61d7e3c32a4090d34468c38621d33b0f853b5" dependencies = [ "hash-db", ] @@ -10592,7 +10592,7 @@ dependencies = [ [[package]] name = "trie-standardmap" version = "0.15.2" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-refact#b4b0d1919db7b100aba040c336d3c69ed11faa36" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-refact#54c61d7e3c32a4090d34468c38621d33b0f853b5" dependencies = [ "hash-db", "keccak-hasher 0.15.3 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-refact)", diff --git a/bin/node/executor/benches/bench.rs b/bin/node/executor/benches/bench.rs index d21aedd1d1849..9c947b84381cd 100644 --- a/bin/node/executor/benches/bench.rs +++ b/bin/node/executor/benches/bench.rs @@ -83,7 +83,7 @@ fn construct_block( let extrinsics = extrinsics.into_iter().map(sign).collect::>(); // calculate the header fields that we can. - let extrinsics_root = Layout::::ordered_trie_root( + let extrinsics_root = Layout::::default().ordered_trie_root( extrinsics.iter().map(Encode::encode) ).to_fixed_bytes() .into(); diff --git a/primitives/api/test/benches/bench.rs b/primitives/api/test/benches/bench.rs index a9fe79d1abcec..6180fc244a7cb 100644 --- a/primitives/api/test/benches/bench.rs +++ b/primitives/api/test/benches/bench.rs @@ -26,7 +26,7 @@ use sp_api::ProvideRuntimeApi; fn sp_api_benchmark(c: &mut Criterion) { c.bench_function("add one with same runtime api", |b| { - let client = substrate_test_runtime_client::new(ture); + let client = substrate_test_runtime_client::new(true); let runtime_api = client.runtime_api(); let block_id = BlockId::Number(client.chain_info().best_number); diff --git a/primitives/trie/benches/bench.rs b/primitives/trie/benches/bench.rs index c2ccb31328aae..c40907ac5cdf4 100644 --- a/primitives/trie/benches/bench.rs +++ b/primitives/trie/benches/bench.rs @@ -21,11 +21,11 @@ criterion_main!(benches); fn benchmark(c: &mut Criterion) { trie_bench::standard_benchmark::< - sp_trie::Layout, + sp_trie::trie_types::Layout, sp_trie::TrieStream, >(c, "substrate-blake2"); trie_bench::standard_benchmark::< - sp_trie::Layout, + sp_trie::trie_types::Layout, sp_trie::TrieStream, >(c, "substrate-keccak"); } diff --git a/primitives/trie/src/lib.rs b/primitives/trie/src/lib.rs index 68d18a5621ce5..90e47693a1410 100644 --- a/primitives/trie/src/lib.rs +++ b/primitives/trie/src/lib.rs @@ -218,7 +218,7 @@ impl TrieLayout for Layout type MetaHasher = M; type Meta = M::Meta; - fn layout_meta(&self) -> GlobalMeta { + fn global_meta(&self) -> GlobalMeta { self.0 } } @@ -316,7 +316,7 @@ impl TrieConfiguration for Layout A: AsRef<[u8]> + Ord, B: AsRef<[u8]>, { - trie_root::trie_root_no_extension::(input, self.layout_meta()) + trie_root::trie_root_no_extension::(input, self.global_meta()) } fn trie_root_unhashed(&self, input: I) -> Vec where @@ -324,7 +324,7 @@ impl TrieConfiguration for Layout A: AsRef<[u8]> + Ord, B: AsRef<[u8]>, { - trie_root::unhashed_trie_no_extension::(input, self.layout_meta()) + trie_root::unhashed_trie_no_extension::(input, self.global_meta()) } fn encode_index(input: u32) -> Vec { From 7e73a703e22c0431c944618b9f47e0c469684a44 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 15 Jun 2021 12:00:05 +0200 Subject: [PATCH 058/127] split some long lines. --- client/db/src/bench.rs | 10 ++- client/service/test/src/client/light.rs | 7 +- .../state-machine/src/proving_backend.rs | 16 +++- primitives/state-machine/src/trie_backend.rs | 4 +- .../state-machine/src/trie_backend_essence.rs | 89 +++++++++++-------- primitives/trie/src/lib.rs | 25 ++++-- primitives/trie/src/node_codec.rs | 6 +- primitives/trie/src/node_header.rs | 12 ++- primitives/trie/src/trie_stream.rs | 9 +- 9 files changed, 117 insertions(+), 61 deletions(-) diff --git a/client/db/src/bench.rs b/client/db/src/bench.rs index de7e1b874c97e..6d2d9e11b0098 100644 --- a/client/db/src/bench.rs +++ b/client/db/src/bench.rs @@ -61,14 +61,16 @@ impl sp_state_machine::Storage> for StorageDb, _>>::extract_value_owned(value, global))) - .map_err(|e| format!("Database backend error: {:?}", e))?; + .map(|result| result.map(|value| + , _>>::extract_value_owned(value, global) + )).map_err(|e| format!("Database backend error: {:?}", e))?; recorder.record::>(key.clone(), backend_value.clone()); Ok(backend_value) } else { self.db.get(0, &prefixed_key) - .map(|result| result.map(|value| , _>>::extract_value_owned(value, global))) - .map_err(|e| format!("Database backend error: {:?}", e)) + .map(|result| result.map(|value| + , _>>::extract_value_owned(value, global) + )).map_err(|e| format!("Database backend error: {:?}", e)) } } fn access_from(&self, key: &Block::Hash) { diff --git a/client/service/test/src/client/light.rs b/client/service/test/src/client/light.rs index e40538e08ebb9..fbbefbb5a5674 100644 --- a/client/service/test/src/client/light.rs +++ b/client/service/test/src/client/light.rs @@ -625,7 +625,12 @@ fn header_proof_is_generated_and_checked() { header_proof_is_generated_and_checked_inner(false); } fn header_proof_is_generated_and_checked_inner(hashed: bool) { - let (local_checker, local_cht_root, remote_block_header, remote_header_proof) = prepare_for_header_proof_check(true, hashed); + let ( + local_checker, + local_cht_root, + remote_block_header, + remote_header_proof, + ) = prepare_for_header_proof_check(true, hashed); assert_eq!((&local_checker as &dyn FetchChecker).check_header_proof(&RemoteHeaderRequest::
{ cht_root: local_cht_root, block: 1, diff --git a/primitives/state-machine/src/proving_backend.rs b/primitives/state-machine/src/proving_backend.rs index 558e22cdc7411..26c98d43703e3 100644 --- a/primitives/state-machine/src/proving_backend.rs +++ b/primitives/state-machine/src/proving_backend.rs @@ -242,7 +242,12 @@ impl<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> TrieBackendStorage { type Overlay = S::Overlay; - fn get(&self, key: &H::Out, prefix: Prefix, global: Option) -> Result, String> { + fn get( + &self, + key: &H::Out, + prefix: Prefix, + global: Option, + ) -> Result, String> { if let Some(v) = self.proof_recorder.get(key) { return Ok(v); } @@ -452,7 +457,8 @@ mod tests { fn proof_recorded_and_checked_inner(flagged: bool) { let size_content = 34; // above hashable value treshold. let value_range = 0..64; - let contents = value_range.clone().map(|i| (vec![i], Some(vec![i; size_content]))).collect::>(); + let contents = value_range.clone() + .map(|i| (vec![i], Some(vec![i; size_content]))).collect::>(); let mut in_memory = InMemoryBackend::::default(); if flagged { in_memory = in_memory.update(vec![(None, vec![( @@ -464,12 +470,14 @@ mod tests { } let mut in_memory = in_memory.update(vec![(None, contents)]); let in_memory_root = in_memory.storage_root(std::iter::empty()).0; - value_range.clone().for_each(|i| assert_eq!(in_memory.storage(&[i]).unwrap().unwrap(), vec![i; size_content])); + value_range.clone() + .for_each(|i| assert_eq!(in_memory.storage(&[i]).unwrap().unwrap(), vec![i; size_content])); let trie = in_memory.as_trie_backend().unwrap(); let trie_root = trie.storage_root(std::iter::empty()).0; assert_eq!(in_memory_root, trie_root); - value_range.clone().for_each(|i| assert_eq!(trie.storage(&[i]).unwrap().unwrap(), vec![i; size_content])); + value_range.clone() + .for_each(|i| assert_eq!(trie.storage(&[i]).unwrap().unwrap(), vec![i; size_content])); let proving = ProvingBackend::new(trie); assert_eq!(proving.storage(&[42]).unwrap().unwrap(), vec![42; size_content]); diff --git a/primitives/state-machine/src/trie_backend.rs b/primitives/state-machine/src/trie_backend.rs index e712fbc23f172..33b6c6da9d5ea 100644 --- a/primitives/state-machine/src/trie_backend.rs +++ b/primitives/state-machine/src/trie_backend.rs @@ -329,7 +329,9 @@ pub mod tests { (mdb, root) } - pub(crate) fn test_trie(hashed_value: bool) -> TrieBackend, BlakeTwo256> { + pub(crate) fn test_trie( + hashed_value: bool, + ) -> TrieBackend, BlakeTwo256> { let (mdb, root) = test_db(hashed_value); TrieBackend::new(mdb, root) } diff --git a/primitives/state-machine/src/trie_backend_essence.rs b/primitives/state-machine/src/trie_backend_essence.rs index 63b1ccc2d9698..8bc3b93fc7be5 100644 --- a/primitives/state-machine/src/trie_backend_essence.rs +++ b/primitives/state-machine/src/trie_backend_essence.rs @@ -22,7 +22,7 @@ use std::sync::Arc; use sp_std::{ops::Deref, boxed::Box, vec::Vec}; use crate::{warn, debug}; -use hash_db::{self, Hasher, Prefix}; +use hash_db::{self, Hasher, Prefix, AsHashDB, HashDB, HashDBRef}; use sp_trie::{Trie, PrefixedMemoryDB, DBValue, empty_child_trie_root, read_trie_value, read_child_trie_value, for_keys_in_child_trie, KeySpacedDB, TrieDBIterator, TrieDBKeyIterator, @@ -137,7 +137,7 @@ impl, H: Hasher> TrieBackendEssence where H::Out: child_info: Option<&ChildInfo>, key: &[u8], ) -> Result> { - let dyn_eph: &dyn hash_db::HashDBRef<_, _, _, _>; + let dyn_eph: &dyn HashDBRef<_, _, _, _>; let keyspace_eph; if let Some(child_info) = child_info.as_ref() { keyspace_eph = KeySpacedDB::new(self, child_info.keyspace()); @@ -305,11 +305,13 @@ pub(crate) struct Ephemeral<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> { overlay: &'a mut S::Overlay, } -impl<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> hash_db::AsHashDB> +impl<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> AsHashDB> for Ephemeral<'a, S, H> { - fn as_hash_db<'b>(&'b self) -> &'b (dyn hash_db::HashDB> + 'b) { self } - fn as_hash_db_mut<'b>(&'b mut self) -> &'b mut (dyn hash_db::HashDB> + 'b) { self } + fn as_hash_db<'b>(&'b self) -> &'b (dyn HashDB> + 'b) { self } + fn as_hash_db_mut<'b>( + &'b mut self, + ) -> &'b mut (dyn HashDB> + 'b) { self } } impl<'a, S: TrieBackendStorage, H: Hasher> Ephemeral<'a, S, H> { @@ -321,15 +323,20 @@ impl<'a, S: TrieBackendStorage, H: Hasher> Ephemeral<'a, S, H> { } } -impl<'a, S: 'a + TrieBackendStorage, H: Hasher> hash_db::HashDB> +impl<'a, S: 'a + TrieBackendStorage, H: Hasher> HashDB> for Ephemeral<'a, S, H> { fn get(&self, key: &H::Out, prefix: Prefix) -> Option { - Self::get_with_meta(self, key, prefix, Default::default()).map(|r| r.0) + HashDB::get_with_meta(self, key, prefix, Default::default()).map(|r| r.0) } - fn get_with_meta(&self, key: &H::Out, prefix: Prefix, global: Option) -> Option<(DBValue, TrieMeta)> { - if let Some(val) = hash_db::HashDB::get_with_meta(self.overlay, key, prefix, global) { + fn get_with_meta( + &self, + key: &H::Out, + prefix: Prefix, + global: Option, + ) -> Option<(DBValue, TrieMeta)> { + if let Some(val) = HashDB::get_with_meta(self.overlay, key, prefix, global) { Some(val) } else { match self.storage.get(&key, prefix, global) { @@ -349,11 +356,11 @@ impl<'a, S: 'a + TrieBackendStorage, H: Hasher> hash_db::HashDB bool { - hash_db::HashDB::get(self, key, prefix).is_some() + HashDB::get(self, key, prefix).is_some() } fn insert(&mut self, prefix: Prefix, value: &[u8]) -> H::Out { - hash_db::HashDB::insert(self.overlay, prefix, value) + HashDB::insert(self.overlay, prefix, value) } fn insert_with_meta( @@ -362,42 +369,47 @@ impl<'a, S: 'a + TrieBackendStorage, H: Hasher> hash_db::HashDB H::Out { - hash_db::HashDB::insert_with_meta(self.overlay, prefix, value, meta) + HashDB::insert_with_meta(self.overlay, prefix, value, meta) } fn emplace(&mut self, key: H::Out, prefix: Prefix, value: DBValue) { - hash_db::HashDB::emplace(self.overlay, key, prefix, value) + HashDB::emplace(self.overlay, key, prefix, value) } fn remove(&mut self, key: &H::Out, prefix: Prefix) { - hash_db::HashDB::remove(self.overlay, key, prefix) + HashDB::remove(self.overlay, key, prefix) } } -impl<'a, S: 'a + TrieBackendStorage, H: Hasher> hash_db::HashDBRef> +impl<'a, S: 'a + TrieBackendStorage, H: Hasher> HashDBRef> for Ephemeral<'a, S, H> { fn get(&self, key: &H::Out, prefix: Prefix) -> Option { - hash_db::HashDB::get(self, key, prefix) + HashDB::get(self, key, prefix) } - fn get_with_meta(&self, key: &H::Out, prefix: Prefix, global: Option) -> Option<(DBValue, TrieMeta)> { - hash_db::HashDB::get_with_meta(self, key, prefix, global) + fn get_with_meta( + &self, + key: &H::Out, + prefix: Prefix, + global: Option, + ) -> Option<(DBValue, TrieMeta)> { + HashDB::get_with_meta(self, key, prefix, global) } fn access_from(&self, key: &H::Out, at: Option<&H::Out>) -> Option { - hash_db::HashDB::access_from(self, key, at) + HashDB::access_from(self, key, at) } fn contains(&self, key: &H::Out, prefix: Prefix) -> bool { - hash_db::HashDB::contains(self, key, prefix) + HashDB::contains(self, key, prefix) } } /// Key-value pairs storage that is used by trie backend essence. pub trait TrieBackendStorage: Send + Sync { /// Type of in-memory overlay. - type Overlay: hash_db::HashDB> + Default + Consolidate; + type Overlay: HashDB> + Default + Consolidate; /// Get the value stored at key. fn get(&self, key: &H::Out, prefix: Prefix, global: Option) -> Result>; /// Call back when value get accessed in trie. @@ -437,26 +449,28 @@ impl TrieBackendStorage for sp_trie::GenericMemoryDB prefix: Prefix, global: Option, ) -> Result> { - Ok(hash_db::HashDB::get_with_meta(self, key, prefix, global)) + Ok(HashDB::get_with_meta(self, key, prefix, global)) } fn access_from(&self, key: &H::Out) { - hash_db::HashDB::access_from(self, key, None); + HashDB::access_from(self, key, None); } } -impl, H: Hasher> hash_db::AsHashDB> +impl, H: Hasher> AsHashDB> for TrieBackendEssence { - fn as_hash_db<'b>(&'b self) -> &'b (dyn hash_db::HashDB> + 'b) { self } - fn as_hash_db_mut<'b>(&'b mut self) -> &'b mut (dyn hash_db::HashDB> + 'b) { self } + fn as_hash_db<'b>(&'b self) -> &'b (dyn HashDB> + 'b) { self } + fn as_hash_db_mut<'b>( + &'b mut self, + ) -> &'b mut (dyn HashDB> + 'b) { self } } -impl, H: Hasher> hash_db::HashDB> +impl, H: Hasher> HashDB> for TrieBackendEssence { fn get(&self, key: &H::Out, prefix: Prefix) -> Option { - self.get_with_meta(key, prefix, Default::default()).map(|r| r.0) + HashDB::get_with_meta(self, key, prefix, Default::default()).map(|r| r.0) } fn get_with_meta( @@ -484,7 +498,7 @@ impl, H: Hasher> hash_db::HashDB bool { - hash_db::HashDB::get(self, key, prefix).is_some() + HashDB::get(self, key, prefix).is_some() } fn insert_with_meta(&mut self, _prefix: Prefix, _value: &[u8], _meta: TrieMeta) -> H::Out { @@ -504,23 +518,28 @@ impl, H: Hasher> hash_db::HashDB, H: Hasher> hash_db::HashDBRef> +impl, H: Hasher> HashDBRef> for TrieBackendEssence { fn get(&self, key: &H::Out, prefix: Prefix) -> Option { - hash_db::HashDB::get(self, key, prefix) + HashDB::get(self, key, prefix) } - fn get_with_meta(&self, key: &H::Out, prefix: Prefix, global: Option) -> Option<(DBValue, TrieMeta)> { - hash_db::HashDB::get_with_meta(self, key, prefix, global) + fn get_with_meta( + &self, + key: &H::Out, + prefix: Prefix, + global: Option, + ) -> Option<(DBValue, TrieMeta)> { + HashDB::get_with_meta(self, key, prefix, global) } fn access_from(&self, key: &H::Out, at: Option<&H::Out>) -> Option { - hash_db::HashDB::access_from(self, key, at) + HashDB::access_from(self, key, at) } fn contains(&self, key: &H::Out, prefix: Prefix) -> bool { - hash_db::HashDB::contains(self, key, prefix) + HashDB::contains(self, key, prefix) } } diff --git a/primitives/trie/src/lib.rs b/primitives/trie/src/lib.rs index 90e47693a1410..a102792950573 100644 --- a/primitives/trie/src/lib.rs +++ b/primitives/trie/src/lib.rs @@ -461,25 +461,30 @@ pub fn delta_trie_root( } /// Read a value from the trie. -pub fn read_trie_value>>( +pub fn read_trie_value( db: &DB, root: &TrieHash, key: &[u8] -) -> Result>, Box>> { +) -> Result>, Box>> + where + L: TrieConfiguration, + DB: hash_db::HashDBRef>, +{ Ok(TrieDB::::new(&*db, root)?.get(key).map(|x| x.map(|val| val.to_vec()))?) } /// Read a value from the trie with given Query. -pub fn read_trie_value_with< - L: TrieConfiguration, - Q: Query, - DB: hash_db::HashDBRef> ->( +pub fn read_trie_value_with ( db: &DB, root: &TrieHash, key: &[u8], query: Q -) -> Result>, Box>> { +) -> Result>, Box>> + where + L: TrieConfiguration, + Q: Query, + DB: hash_db::HashDBRef> +{ Ok(TrieDB::::new(&*db, root)?.get_with(key, query).map(|x| x.map(|val| val.to_vec()))?) } @@ -608,7 +613,7 @@ pub fn read_child_trie_value( } /// Read a value from the child trie with given query. -pub fn read_child_trie_value_with, DB>( +pub fn read_child_trie_value_with( keyspace: &[u8], db: &DB, root_slice: &[u8], @@ -616,6 +621,8 @@ pub fn read_child_trie_value_with Result>, Box>> where + L: TrieConfiguration, + Q: Query, DB: hash_db::HashDBRef>, { let mut root = TrieHash::::default(); diff --git a/primitives/trie/src/node_codec.rs b/primitives/trie/src/node_codec.rs index bd6e6d476f3e8..3f90bf8126352 100644 --- a/primitives/trie/src/node_codec.rs +++ b/primitives/trie/src/node_codec.rs @@ -355,7 +355,8 @@ fn partial_from_iterator_encode>( NodeKind::BranchWithValue => NodeHeader::Branch(true, nibble_count).encode_to(&mut output), NodeKind::BranchNoValue => NodeHeader::Branch(false, nibble_count).encode_to(&mut output), NodeKind::AltHashLeaf => NodeHeader::AltHashLeaf(nibble_count).encode_to(&mut output), - NodeKind::AltHashBranchWithValue => NodeHeader::AltHashBranch(nibble_count).encode_to(&mut output), + NodeKind::AltHashBranchWithValue => NodeHeader::AltHashBranch(nibble_count) + .encode_to(&mut output), }; output.extend(partial); output @@ -375,7 +376,8 @@ fn partial_encode(partial: Partial, node_kind: NodeKind) -> Vec { NodeKind::BranchWithValue => NodeHeader::Branch(true, nibble_count).encode_to(&mut output), NodeKind::BranchNoValue => NodeHeader::Branch(false, nibble_count).encode_to(&mut output), NodeKind::AltHashLeaf => NodeHeader::AltHashLeaf(nibble_count).encode_to(&mut output), - NodeKind::AltHashBranchWithValue => NodeHeader::AltHashBranch(nibble_count).encode_to(&mut output), + NodeKind::AltHashBranchWithValue => NodeHeader::AltHashBranch(nibble_count) + .encode_to(&mut output), }; if number_nibble_encoded > 0 { output.push(nibble_ops::pad_right((partial.0).1)); diff --git a/primitives/trie/src/node_header.rs b/primitives/trie/src/node_header.rs index 7f5c68794e745..43ac67cbabcaf 100644 --- a/primitives/trie/src/node_header.rs +++ b/primitives/trie/src/node_header.rs @@ -102,7 +102,11 @@ impl Decode for NodeHeader { /// Returns an iterator over encoded bytes for node header and size. /// Size encoding allows unlimited, length inefficient, representation, but /// is bounded to 16 bit maximum value to avoid possible DOS. -pub(crate) fn size_and_prefix_iterator(size: usize, prefix: u8, prefix_mask: usize) -> impl Iterator { +pub(crate) fn size_and_prefix_iterator( + size: usize, + prefix: u8, + prefix_mask: usize, +) -> impl Iterator { let size = sp_std::cmp::min(trie_constants::NIBBLE_SIZE_BOUND, size); let max_value = 255u8 >> prefix_mask; @@ -139,7 +143,11 @@ fn encode_size_and_prefix(size: usize, prefix: u8, prefix_mask: usize, out: & } /// Decode size only from stream input and header byte. -fn decode_size(first: u8, input: &mut impl Input, prefix_mask: usize) -> Result { +fn decode_size( + first: u8, + input: &mut impl Input, + prefix_mask: usize, +) -> Result { let max_value = 255u8 >> prefix_mask; let mut result = (first & max_value) as usize; if result < max_value as usize { diff --git a/primitives/trie/src/trie_stream.rs b/primitives/trie/src/trie_stream.rs index 2be98fad76ab6..bba41fe6d81c0 100644 --- a/primitives/trie/src/trie_stream.rs +++ b/primitives/trie/src/trie_stream.rs @@ -66,8 +66,10 @@ fn fuse_nibbles_node<'a>(nibbles: &'a [u8], kind: NodeKind) -> impl Iterator size_and_prefix_iterator(size, trie_constants::LEAF_PREFIX_MASK, 2), NodeKind::BranchNoValue => size_and_prefix_iterator(size, trie_constants::BRANCH_WITHOUT_MASK, 2), NodeKind::BranchWithValue => size_and_prefix_iterator(size, trie_constants::BRANCH_WITH_MASK, 2), - NodeKind::AltHashLeaf => size_and_prefix_iterator(size, trie_constants::ALT_HASHING_LEAF_PREFIX_MASK, 3), - NodeKind::AltHashBranchWithValue => size_and_prefix_iterator(size, trie_constants::ALT_HASHING_BRANCH_WITH_MASK, 4), + NodeKind::AltHashLeaf => + size_and_prefix_iterator(size, trie_constants::ALT_HASHING_LEAF_PREFIX_MASK, 3), + NodeKind::AltHashBranchWithValue => + size_and_prefix_iterator(size, trie_constants::ALT_HASHING_BRANCH_WITH_MASK, 4), }; iter_start .chain(if nibbles.len() % 2 == 1 { Some(nibbles[0]) } else { None }) @@ -162,7 +164,8 @@ impl trie_root::TrieStream for TrieStream { try_inner_hashing: None, apply_inner_hashing: true, }; - >>::hash(&data, &meta).as_ref().encode_to(&mut self.buffer); + >>::hash(&data, &meta).as_ref() + .encode_to(&mut self.buffer); } else { H::hash(&data).as_ref().encode_to(&mut self.buffer); } From 934021c1c05cdf8c6748aca513f93c3273c52c4b Mon Sep 17 00:00:00 2001 From: cheme Date: Fri, 18 Jun 2021 20:13:59 +0200 Subject: [PATCH 059/127] UPdate trie crate to work with new design. --- Cargo.lock | 24 +- Cargo.toml | 10 +- bin/node/bench/src/simple_trie.rs | 14 +- bin/node/bench/src/trie.rs | 4 +- client/db/src/bench.rs | 4 +- client/db/src/lib.rs | 6 +- primitives/io/src/lib.rs | 2 +- primitives/state-machine/src/backend.rs | 3 +- primitives/state-machine/src/basic.rs | 3 +- .../src/changes_trie/changes_iterator.rs | 4 +- .../state-machine/src/changes_trie/mod.rs | 15 +- .../state-machine/src/changes_trie/prune.rs | 2 +- .../state-machine/src/changes_trie/storage.rs | 17 +- primitives/state-machine/src/ext.rs | 2 +- .../state-machine/src/in_memory_backend.rs | 2 +- primitives/state-machine/src/lib.rs | 7 +- .../state-machine/src/proving_backend.rs | 50 +-- primitives/state-machine/src/trie_backend.rs | 5 +- .../state-machine/src/trie_backend_essence.rs | 119 ++---- primitives/trie/src/lib.rs | 358 ++++-------------- primitives/trie/src/node_codec.rs | 73 ++-- primitives/trie/src/node_header.rs | 39 +- primitives/trie/src/storage_proof.rs | 20 +- primitives/trie/src/trie_codec.rs | 11 +- primitives/trie/src/trie_stream.rs | 35 +- 25 files changed, 289 insertions(+), 540 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 94e601742d821..f97655a692d0b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2354,7 +2354,7 @@ dependencies = [ [[package]] name = "hash-db" version = "0.15.2" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-refact#54c61d7e3c32a4090d34468c38621d33b0f853b5" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple7#c7e0aeb8fd74c23845efe4b6daee43b7f4084faa" [[package]] name = "hash256-std-hasher" @@ -2368,7 +2368,7 @@ dependencies = [ [[package]] name = "hash256-std-hasher" version = "0.15.2" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-refact#54c61d7e3c32a4090d34468c38621d33b0f853b5" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple7#c7e0aeb8fd74c23845efe4b6daee43b7f4084faa" dependencies = [ "crunchy", ] @@ -3050,10 +3050,10 @@ dependencies = [ [[package]] name = "keccak-hasher" version = "0.15.3" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-refact#54c61d7e3c32a4090d34468c38621d33b0f853b5" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple7#c7e0aeb8fd74c23845efe4b6daee43b7f4084faa" dependencies = [ "hash-db", - "hash256-std-hasher 0.15.2 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-refact)", + "hash256-std-hasher 0.15.2 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple7)", "tiny-keccak", ] @@ -3865,7 +3865,7 @@ dependencies = [ [[package]] name = "memory-db" version = "0.26.0" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-refact#54c61d7e3c32a4090d34468c38621d33b0f853b5" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple7#c7e0aeb8fd74c23845efe4b6daee43b7f4084faa" dependencies = [ "hash-db", "hashbrown", @@ -10551,22 +10551,22 @@ checksum = "a7f741b240f1a48843f9b8e0444fb55fb2a4ff67293b50a9179dfd5ea67f8d41" [[package]] name = "trie-bench" version = "0.27.0" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-refact#54c61d7e3c32a4090d34468c38621d33b0f853b5" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple7#c7e0aeb8fd74c23845efe4b6daee43b7f4084faa" dependencies = [ "criterion", "hash-db", - "keccak-hasher 0.15.3 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-refact)", + "keccak-hasher 0.15.3 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple7)", "memory-db", "parity-scale-codec", "trie-db", "trie-root", - "trie-standardmap 0.15.2 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-refact)", + "trie-standardmap 0.15.2 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple7)", ] [[package]] name = "trie-db" version = "0.22.5" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-refact#54c61d7e3c32a4090d34468c38621d33b0f853b5" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple7#c7e0aeb8fd74c23845efe4b6daee43b7f4084faa" dependencies = [ "hash-db", "hashbrown", @@ -10578,7 +10578,7 @@ dependencies = [ [[package]] name = "trie-root" version = "0.16.0" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-refact#54c61d7e3c32a4090d34468c38621d33b0f853b5" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple7#c7e0aeb8fd74c23845efe4b6daee43b7f4084faa" dependencies = [ "hash-db", ] @@ -10596,10 +10596,10 @@ dependencies = [ [[package]] name = "trie-standardmap" version = "0.15.2" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-refact#54c61d7e3c32a4090d34468c38621d33b0f853b5" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple7#c7e0aeb8fd74c23845efe4b6daee43b7f4084faa" dependencies = [ "hash-db", - "keccak-hasher 0.15.3 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-refact)", + "keccak-hasher 0.15.3 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple7)", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 566401fbe28f5..fc427dd2f2e16 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -278,8 +278,8 @@ zeroize = { opt-level = 3 } panic = "unwind" [patch.crates-io] -hash-db = { git = "https://github.com/cheme/trie.git", branch = "hashed-value-simple4-refact" } -memory-db = { git = "https://github.com/cheme/trie.git", branch = "hashed-value-simple4-refact" } -trie-db = { git = "https://github.com/cheme/trie.git", branch = "hashed-value-simple4-refact" } -trie-root = { git = "https://github.com/cheme/trie.git", branch = "hashed-value-simple4-refact" } -trie-bench = { git = "https://github.com/cheme/trie.git", branch = "hashed-value-simple4-refact" } +hash-db = { git = "https://github.com/cheme/trie.git", branch = "hashed-value-simple7" } +memory-db = { git = "https://github.com/cheme/trie.git", branch = "hashed-value-simple7" } +trie-db = { git = "https://github.com/cheme/trie.git", branch = "hashed-value-simple7" } +trie-root = { git = "https://github.com/cheme/trie.git", branch = "hashed-value-simple7" } +trie-bench = { git = "https://github.com/cheme/trie.git", branch = "hashed-value-simple7" } diff --git a/bin/node/bench/src/simple_trie.rs b/bin/node/bench/src/simple_trie.rs index fea106ed196fe..f4a4576508f6c 100644 --- a/bin/node/bench/src/simple_trie.rs +++ b/bin/node/bench/src/simple_trie.rs @@ -20,7 +20,7 @@ use std::{collections::HashMap, sync::Arc}; use kvdb::KeyValueDB; use node_primitives::Hash; -use sp_trie::{DBValue, TrieMeta, StateHasher, MetaHasher}; +use sp_trie::{DBValue, Meta, StateHasher, MetaHasher}; use hash_db::{HashDB, AsHashDB, Prefix, Hasher as _}; pub type Hasher = sp_core::Blake2Hasher; @@ -31,15 +31,15 @@ pub struct SimpleTrie<'a> { pub overlay: &'a mut HashMap, Option>>, } -impl<'a> AsHashDB> for SimpleTrie<'a> { - fn as_hash_db(&self) -> &dyn hash_db::HashDB> { &*self } +impl<'a> AsHashDB> for SimpleTrie<'a> { + fn as_hash_db(&self) -> &dyn hash_db::HashDB> { &*self } - fn as_hash_db_mut<'b>(&'b mut self) -> &'b mut (dyn HashDB> + 'b) { + fn as_hash_db_mut<'b>(&'b mut self) -> &'b mut (dyn HashDB> + 'b) { &mut *self } } -impl<'a> HashDB> for SimpleTrie<'a> { +impl<'a> HashDB> for SimpleTrie<'a> { fn get(&self, key: &Hash, prefix: Prefix) -> Option { let key = sp_trie::prefixed_key::(key, prefix); if let Some(value) = self.overlay.get(&key) { @@ -48,7 +48,7 @@ impl<'a> HashDB> for SimpleTrie<'a> { self.db.get(0, &key).expect("Database backend error") } - fn get_with_meta(&self, key: &Hash, prefix: Prefix, global: Option) -> Option<(DBValue, TrieMeta)> { + fn get_with_meta(&self, key: &Hash, prefix: Prefix, global: Option) -> Option<(DBValue, Meta)> { let result = self.get(key, prefix); result.map(|value| >::extract_value_owned(value, global)) } @@ -57,7 +57,7 @@ impl<'a> HashDB> for SimpleTrie<'a> { self.get(hash, prefix).is_some() } - fn insert_with_meta(&mut self, prefix: Prefix, value: &[u8], meta: TrieMeta) -> Hash { + fn insert_with_meta(&mut self, prefix: Prefix, value: &[u8], meta: Meta) -> Hash { let key = >::hash(value, &meta); let stored_value = >::stored_value(value, meta); self.emplace(key, prefix, stored_value); diff --git a/bin/node/bench/src/trie.rs b/bin/node/bench/src/trie.rs index f1bcd3b2239c6..67d07a7c024b1 100644 --- a/bin/node/bench/src/trie.rs +++ b/bin/node/bench/src/trie.rs @@ -24,7 +24,7 @@ use lazy_static::lazy_static; use rand::Rng; use hash_db::Prefix; use sp_state_machine::Backend as _; -use sp_trie::{trie_types::TrieDBMut, TrieMut as _, TrieMeta, MetaHasher, StateHasher}; +use sp_trie::{trie_types::TrieDBMut, TrieMut as _, Meta, MetaHasher, StateHasher}; use node_primitives::Hash; @@ -175,7 +175,7 @@ impl sp_state_machine::Storage for Storage { key: &Hash, prefix: Prefix, global: Option, - ) -> Result, TrieMeta)>, String> { + ) -> Result, Meta)>, String> { let key = sp_trie::prefixed_key::(key, prefix); self.0.get(0, &key).map_err(|e| format!("Database backend error: {:?}", e)) .map(|result| result diff --git a/client/db/src/bench.rs b/client/db/src/bench.rs index 72be5dd5062d4..a43a733bdd47b 100644 --- a/client/db/src/bench.rs +++ b/client/db/src/bench.rs @@ -23,7 +23,7 @@ use std::cell::{Cell, RefCell}; use std::collections::HashMap; use hash_db::{Prefix, Hasher}; -use sp_trie::{MemoryDB, prefixed_key, StateHasher, TrieMeta, MetaHasher}; +use sp_trie::{MemoryDB, prefixed_key, StateHasher, Meta, MetaHasher}; use sp_core::{ storage::{ChildInfo, TrackedStorageKey}, hexdisplay::HexDisplay @@ -54,7 +54,7 @@ impl sp_state_machine::Storage> for StorageDb, - ) -> Result, String> { + ) -> Result, String> { let prefixed_key = prefixed_key::>(key, prefix); if let Some(recorder) = &self.proof_recorder { if let Some(v) = recorder.get(&key) { diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index c32a476a5d8da..fcbea6201e7bf 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -66,7 +66,7 @@ use sp_blockchain::{ use codec::{Decode, Encode}; use hash_db::Prefix; use sp_trie::{MemoryDB, PrefixedMemoryDB, prefixed_key, StateHasher, - TrieMeta, MetaHasher}; + Meta, MetaHasher}; use sp_database::Transaction; use sp_core::ChangesTrieConfiguration; use sp_core::offchain::OffchainOverlayedChange; @@ -894,7 +894,7 @@ impl sp_state_machine::Storage> for StorageDb, - ) -> Result, String> { + ) -> Result, String> { if self.prefix_keys { let key = prefixed_key::>(key, prefix); self.state_db.get(&key, self) @@ -936,7 +936,7 @@ impl sp_state_machine::Storage> for DbGenesisStora _key: &Block::Hash, _prefix: Prefix, _global: Option, - ) -> Result, String> { + ) -> Result, String> { Ok(None) } fn access_from(&self, _key: &Block::Hash) { diff --git a/primitives/io/src/lib.rs b/primitives/io/src/lib.rs index ffbfc4bdc54fb..21608c535c982 100644 --- a/primitives/io/src/lib.rs +++ b/primitives/io/src/lib.rs @@ -54,7 +54,7 @@ use sp_core::{ }; #[cfg(feature = "std")] -use sp_trie::{TrieConfiguration, trie_types::Layout}; +use sp_trie::{TrieConfiguration, Layout}; use sp_runtime_interface::{runtime_interface, Pointer}; use sp_runtime_interface::pass_by::{PassBy, PassByCodec}; diff --git a/primitives/state-machine/src/backend.rs b/primitives/state-machine/src/backend.rs index c90b5aa2013b2..c879053fce569 100644 --- a/primitives/state-machine/src/backend.rs +++ b/primitives/state-machine/src/backend.rs @@ -285,10 +285,9 @@ impl Consolidate for Vec<( } } -impl Consolidate for sp_trie::GenericMemoryDB +impl Consolidate for sp_trie::GenericMemoryDB where H: Hasher, - MH: sp_trie::MetaHasher, KF: sp_trie::KeyFunction, { fn consolidate(&mut self, other: Self) { diff --git a/primitives/state-machine/src/basic.rs b/primitives/state-machine/src/basic.rs index 288cb36993472..653bdc28a3c7d 100644 --- a/primitives/state-machine/src/basic.rs +++ b/primitives/state-machine/src/basic.rs @@ -22,8 +22,7 @@ use std::{ }; use crate::{Backend, StorageKey, StorageValue}; use hash_db::Hasher; -use sp_trie::{TrieConfiguration, empty_child_trie_root}; -use sp_trie::trie_types::Layout; +use sp_trie::{TrieConfiguration, empty_child_trie_root, Layout}; use sp_core::{ storage::{ well_known_keys::is_child_storage_key, Storage, diff --git a/primitives/state-machine/src/changes_trie/changes_iterator.rs b/primitives/state-machine/src/changes_trie/changes_iterator.rs index 25eda86b4e66a..be35581e7514d 100644 --- a/primitives/state-machine/src/changes_trie/changes_iterator.rs +++ b/primitives/state-machine/src/changes_trie/changes_iterator.rs @@ -24,7 +24,7 @@ use codec::{Decode, Encode, Codec}; use hash_db::Hasher; use num_traits::Zero; use sp_core::storage::PrefixedStorageKey; -use sp_trie::{Recorder, TrieMeta}; +use sp_trie::Recorder; use crate::changes_trie::{AnchorBlockId, ConfigurationRange, RootsStorage, Storage, BlockNumber}; use crate::changes_trie::input::{DigestIndex, ExtrinsicIndex, DigestIndexValue, ExtrinsicIndexValue}; use crate::changes_trie::storage::{TrieBackendAdapter, InMemoryStorage}; @@ -337,7 +337,7 @@ struct ProvingDrilldownIterator<'a, H, Number> H::Out: 'a, { essence: DrilldownIteratorEssence<'a, H, Number>, - proof_recorder: RefCell>, + proof_recorder: RefCell>, } impl<'a, H, Number> ProvingDrilldownIterator<'a, H, Number> diff --git a/primitives/state-machine/src/changes_trie/mod.rs b/primitives/state-machine/src/changes_trie/mod.rs index 5a2e206956ac9..f2e5ff461eecf 100644 --- a/primitives/state-machine/src/changes_trie/mod.rs +++ b/primitives/state-machine/src/changes_trie/mod.rs @@ -167,19 +167,10 @@ pub struct TrieBackendStorageAdapter<'a, H: Hasher, Number: BlockNumber>(pub &'a impl<'a, H: Hasher, N: BlockNumber> crate::TrieBackendStorage for TrieBackendStorageAdapter<'a, H, N> { type Overlay = sp_trie::MemoryDB; - fn get( - &self, - key: &H::Out, - prefix: Prefix, - _global: Option, - ) -> Result, String> { - match self.0.get(key, prefix) { - // change trie do not use meta. - Ok(Some(v)) => Ok(Some((v, Default::default()))), - Ok(None) => Ok(None), - Err(e) => Err(e), - } + fn get(&self, key: &H::Out, prefix: Prefix) -> Result, String> { + self.0.get(key, prefix) } + fn access_from(&self, _key: &H::Out) { } } diff --git a/primitives/state-machine/src/changes_trie/prune.rs b/primitives/state-machine/src/changes_trie/prune.rs index 6f00e9b6a8e0d..a741b814a5c70 100644 --- a/primitives/state-machine/src/changes_trie/prune.rs +++ b/primitives/state-machine/src/changes_trie/prune.rs @@ -95,7 +95,7 @@ fn prune_trie( // enumerate all changes trie' keys, recording all nodes that have been 'touched' // (effectively - all changes trie nodes) - let mut proof_recorder: Recorder = Default::default(); + let mut proof_recorder: Recorder = Default::default(); { let mut trie = ProvingBackendRecorder::<_, H> { backend: &TrieBackendEssence::new(TrieBackendAdapter::new(storage), root), diff --git a/primitives/state-machine/src/changes_trie/storage.rs b/primitives/state-machine/src/changes_trie/storage.rs index 0cea30f0d809c..d99d918012f94 100644 --- a/primitives/state-machine/src/changes_trie/storage.rs +++ b/primitives/state-machine/src/changes_trie/storage.rs @@ -190,7 +190,7 @@ impl Storage for InMemoryStorage Result, String> { - Ok( as hash_db::HashDBRef>::get(&self.data.read().mdb, key, prefix)) + Ok( as hash_db::HashDBRef>::get(&self.data.read().mdb, key, prefix)) } } @@ -207,18 +207,9 @@ impl<'a, H, Number> TrieBackendStorage for TrieBackendAdapter<'a, H, Number> { type Overlay = MemoryDB; - fn get( - &self, - key: &H::Out, - prefix: Prefix, - _global: Option, - ) -> Result, String> { - match self.storage.get(key, prefix) { - // change trie do not use meta. - Ok(Some(v)) => Ok(Some((v, Default::default()))), - Ok(None) => Ok(None), - Err(e) => Err(e), - } + fn get(&self, key: &H::Out, prefix: Prefix) -> Result, String> { + self.storage.get(key, prefix) } + fn access_from(&self, _key: &H::Out) { } } diff --git a/primitives/state-machine/src/ext.rs b/primitives/state-machine/src/ext.rs index 25e672b768652..2defaa49142af 100644 --- a/primitives/state-machine/src/ext.rs +++ b/primitives/state-machine/src/ext.rs @@ -26,7 +26,7 @@ use sp_core::{ storage::{well_known_keys::is_child_storage_key, ChildInfo, TrackedStorageKey}, hexdisplay::HexDisplay, }; -use sp_trie::{trie_types::Layout, empty_child_trie_root}; +use sp_trie::{Layout, empty_child_trie_root}; use sp_externalities::{ Externalities, Extensions, Extension, ExtensionStore, }; diff --git a/primitives/state-machine/src/in_memory_backend.rs b/primitives/state-machine/src/in_memory_backend.rs index 84c0b316387da..b122120b36e44 100644 --- a/primitives/state-machine/src/in_memory_backend.rs +++ b/primitives/state-machine/src/in_memory_backend.rs @@ -22,7 +22,7 @@ use crate::{ }; use std::collections::{BTreeMap, HashMap}; use hash_db::Hasher; -use sp_trie::{MemoryDB, empty_trie_root, trie_types::Layout}; +use sp_trie::{MemoryDB, empty_trie_root, Layout}; use codec::Codec; use sp_core::storage::{ChildInfo, Storage}; diff --git a/primitives/state-machine/src/lib.rs b/primitives/state-machine/src/lib.rs index 91bd575316fd7..34363dfa6fe3a 100644 --- a/primitives/state-machine/src/lib.rs +++ b/primitives/state-machine/src/lib.rs @@ -143,7 +143,7 @@ mod changes_trie { #[cfg(feature = "std")] mod std_reexport { - pub use sp_trie::{trie_types::{Layout, TrieDBMut}, StorageProof, TrieMut, + pub use sp_trie::{trie_types::TrieDBMut, Layout, StorageProof, TrieMut, DBValue, MemoryDB}; pub use crate::testing::TestExternalities; pub use crate::basic::BasicExternalities; @@ -848,14 +848,13 @@ mod execution { } /// Check storage read proof on pre-created proving backend. - pub fn read_proof_check_on_proving_backend_generic( - proving_backend: &TrieBackend, H>, + pub fn read_proof_check_on_proving_backend_generic( + proving_backend: &TrieBackend, H>, key: &[u8], ) -> Result>, Box> where H: Hasher, H::Out: Ord + Codec, - MH: sp_trie::MetaHasher>, KF: sp_trie::KeyFunction + Send + Sync, { proving_backend.storage(key).map_err(|e| Box::new(e) as Box) diff --git a/primitives/state-machine/src/proving_backend.rs b/primitives/state-machine/src/proving_backend.rs index ceeba59c83611..206024af44095 100644 --- a/primitives/state-machine/src/proving_backend.rs +++ b/primitives/state-machine/src/proving_backend.rs @@ -24,9 +24,9 @@ use log::debug; use hash_db::{Hasher, HashDB, EMPTY_PREFIX, Prefix}; use sp_trie::{ MemoryDB, empty_child_trie_root, read_trie_value_with, read_child_trie_value_with, - record_all_keys, StorageProof, TrieMeta, + record_all_keys, StorageProof, Meta, Layout, Recorder, }; -pub use sp_trie::{Recorder, trie_types::{Layout, TrieError}}; +pub use sp_trie::trie_types::TrieError; use crate::trie_backend::TrieBackend; use crate::trie_backend_essence::{Ephemeral, TrieBackendEssence, TrieBackendStorage}; use crate::{Error, ExecutionError, Backend, DBValue}; @@ -35,7 +35,7 @@ use sp_core::storage::ChildInfo; /// Patricia trie-based backend specialized in get value proofs. pub struct ProvingBackendRecorder<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> { pub(crate) backend: &'a TrieBackendEssence, - pub(crate) proof_recorder: &'a mut Recorder, + pub(crate) proof_recorder: &'a mut Recorder, } impl<'a, S, H> ProvingBackendRecorder<'a, S, H> @@ -112,7 +112,7 @@ impl<'a, S, H> ProvingBackendRecorder<'a, S, H> #[derive(Default)] struct ProofRecorderInner { /// All the records that we have stored so far. - records: HashMap>, + records: HashMap>, /// The encoded size of all recorded values. encoded_size: usize, } @@ -125,17 +125,17 @@ pub struct ProofRecorder { impl ProofRecorder { /// Record the given `key` => `val` combination. - pub fn record(&self, key: Hash, mut val: Option<(DBValue, TrieMeta)>) { + pub fn record(&self, key: Hash, val: Option) { let mut inner = self.inner.write(); let ProofRecorderInner { encoded_size, records } = &mut *inner; records.entry(key).or_insert_with(|| { - if let Some(val) = val.as_mut() { - val.1.set_accessed_value(false); - sp_trie::resolve_encoded_meta::(val); - *encoded_size += sp_trie::estimate_entry_size(val, H::LENGTH); - } - val + val.map(|val| { + let mut val = (val, Meta::default(), false); + sp_trie::resolve_encoded_meta::(&mut val); + *encoded_size += sp_trie::estimate_entry_size(&val, H::LENGTH); + val + }) }); } @@ -146,9 +146,9 @@ impl ProofRecorder { records.entry(key.clone()) .and_modify(|entry| { if let Some(entry) = entry.as_mut() { - if !entry.1.accessed_value() { + if !entry.2 { let old_size = sp_trie::estimate_entry_size(entry, hash_len); - entry.1.set_accessed_value(true); + entry.2 = true; let new_size = sp_trie::estimate_entry_size(entry, hash_len); *encoded_size += new_size; *encoded_size -= old_size; @@ -158,8 +158,9 @@ impl ProofRecorder { } /// Returns the value at the given `key`. - pub fn get(&self, key: &Hash) -> Option> { - self.inner.read().records.get(key).cloned() + pub fn get(&self, key: &Hash) -> Option> { + self.inner.read().records.get(key).as_ref() + .map(|v| v.as_ref().map(|v| v.0.clone())) } /// Returns the estimated encoded size of the proof. @@ -174,14 +175,18 @@ impl ProofRecorder { /// Convert into a [`StorageProof`]. pub fn to_storage_proof(&self) -> StorageProof { - let inner = self.inner.read(); - let trie_nodes = inner + let trie_nodes = self.inner.read() .records .iter() .filter_map(|(_k, v)| v.as_ref().map(|v| { - < - as sp_trie::TrieLayout>::MetaHasher as hash_db::MetaHasher - >::stored_value(v.0.as_slice(), v.1.clone()) + let mut meta = v.1.clone(); + if let Some(hashed) = sp_trie::to_hashed_variant::( + v.0.as_slice(), &mut meta, v.2, + ) { + hashed + } else { + v.0.clone() + } })) .collect(); @@ -246,13 +251,12 @@ impl<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> TrieBackendStorage &self, key: &H::Out, prefix: Prefix, - global: Option, - ) -> Result, String> { + ) -> Result, String> { if let Some(v) = self.proof_recorder.get(key) { return Ok(v); } - let backend_value = self.backend.get(key, prefix, global)?; + let backend_value = self.backend.get(key, prefix)?; self.proof_recorder.record::(key.clone(), backend_value.clone()); Ok(backend_value) } diff --git a/primitives/state-machine/src/trie_backend.rs b/primitives/state-machine/src/trie_backend.rs index 83643f645548c..c09a6e6e0899b 100644 --- a/primitives/state-machine/src/trie_backend.rs +++ b/primitives/state-machine/src/trie_backend.rs @@ -19,8 +19,9 @@ use crate::{warn, debug}; use hash_db::Hasher; -use sp_trie::{Trie, delta_trie_root, empty_child_trie_root, child_delta_trie_root}; -use sp_trie::trie_types::{TrieDB, TrieError, Layout}; +use sp_trie::{Trie, delta_trie_root, empty_child_trie_root, child_delta_trie_root, + Layout}; +use sp_trie::trie_types::{TrieDB, TrieError}; use sp_core::storage::{ChildInfo, ChildType}; use codec::{Codec, Decode}; use crate::{ diff --git a/primitives/state-machine/src/trie_backend_essence.rs b/primitives/state-machine/src/trie_backend_essence.rs index 0c45923e374e0..e22f4d96c680e 100644 --- a/primitives/state-machine/src/trie_backend_essence.rs +++ b/primitives/state-machine/src/trie_backend_essence.rs @@ -23,10 +23,10 @@ use std::sync::Arc; use sp_std::{ops::Deref, boxed::Box, vec::Vec}; use crate::{warn, debug}; use hash_db::{self, Hasher, Prefix, AsHashDB, HashDB, HashDBRef}; -use sp_trie::{Trie, PrefixedMemoryDB, DBValue, +use sp_trie::{Trie, PrefixedMemoryDB, DBValue, Layout, empty_child_trie_root, read_trie_value, read_child_trie_value, - KeySpacedDB, TrieDBIterator, TrieDBKeyIterator, TrieMeta}; -use sp_trie::trie_types::{TrieDB, TrieError, Layout}; + KeySpacedDB, TrieDBIterator, TrieDBKeyIterator}; +use sp_trie::trie_types::{TrieDB, TrieError}; use crate::{backend::Consolidate, StorageKey, StorageValue}; use sp_core::storage::ChildInfo; use codec::Encode; @@ -47,8 +47,7 @@ pub trait Storage: Send + Sync { &self, key: &H::Out, prefix: Prefix, - alt_hashing: Option, - ) -> Result>; + ) -> Result>; /// Call back when value get accessed in trie. fn access_from(&self, key: &H::Out); } @@ -136,7 +135,7 @@ impl, H: Hasher> TrieBackendEssence where H::Out: child_info: Option<&ChildInfo>, key: &[u8], ) -> Result> { - let dyn_eph: &dyn HashDBRef<_, _, _, _>; + let dyn_eph: &dyn HashDBRef<_, _>; let keyspace_eph; if let Some(child_info) = child_info.as_ref() { keyspace_eph = KeySpacedDB::new(self, child_info.keyspace()); @@ -336,13 +335,11 @@ pub(crate) struct Ephemeral<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> { overlay: &'a mut S::Overlay, } -impl<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> AsHashDB> +impl<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> AsHashDB for Ephemeral<'a, S, H> { - fn as_hash_db<'b>(&'b self) -> &'b (dyn HashDB> + 'b) { self } - fn as_hash_db_mut<'b>( - &'b mut self, - ) -> &'b mut (dyn HashDB> + 'b) { self } + fn as_hash_db<'b>(&'b self) -> &'b (dyn HashDB + 'b) { self } + fn as_hash_db_mut<'b>(&'b mut self) -> &'b mut (dyn HashDB + 'b) { self } } impl<'a, S: TrieBackendStorage, H: Hasher> Ephemeral<'a, S, H> { @@ -354,23 +351,14 @@ impl<'a, S: TrieBackendStorage, H: Hasher> Ephemeral<'a, S, H> { } } -impl<'a, S: 'a + TrieBackendStorage, H: Hasher> HashDB> +impl<'a, S: 'a + TrieBackendStorage, H: Hasher> hash_db::HashDB for Ephemeral<'a, S, H> { fn get(&self, key: &H::Out, prefix: Prefix) -> Option { - HashDB::get_with_meta(self, key, prefix, Default::default()).map(|r| r.0) - } - - fn get_with_meta( - &self, - key: &H::Out, - prefix: Prefix, - global: Option, - ) -> Option<(DBValue, TrieMeta)> { - if let Some(val) = HashDB::get_with_meta(self.overlay, key, prefix, global) { + if let Some(val) = HashDB::get(self.overlay, key, prefix) { Some(val) } else { - match self.storage.get(&key, prefix, global) { + match self.storage.get(&key, prefix) { Ok(x) => x, Err(e) => { warn!(target: "trie", "Failed to read from DB: {}", e); @@ -394,40 +382,26 @@ impl<'a, S: 'a + TrieBackendStorage, H: Hasher> HashDB H::Out { - HashDB::insert_with_meta(self.overlay, prefix, value, meta) - } - fn emplace(&mut self, key: H::Out, prefix: Prefix, value: DBValue) { HashDB::emplace(self.overlay, key, prefix, value) } + fn emplace_ref(&mut self, key: &H::Out, prefix: Prefix, value: &[u8]) { + HashDB::emplace_ref(self.overlay, key, prefix, value) + } + fn remove(&mut self, key: &H::Out, prefix: Prefix) { HashDB::remove(self.overlay, key, prefix) } } -impl<'a, S: 'a + TrieBackendStorage, H: Hasher> HashDBRef> +impl<'a, S: 'a + TrieBackendStorage, H: Hasher> HashDBRef for Ephemeral<'a, S, H> { fn get(&self, key: &H::Out, prefix: Prefix) -> Option { HashDB::get(self, key, prefix) } - fn get_with_meta( - &self, - key: &H::Out, - prefix: Prefix, - global: Option, - ) -> Option<(DBValue, TrieMeta)> { - HashDB::get_with_meta(self, key, prefix, global) - } - fn access_from(&self, key: &H::Out, at: Option<&H::Out>) -> Option { HashDB::access_from(self, key, at) } @@ -440,9 +414,9 @@ impl<'a, S: 'a + TrieBackendStorage, H: Hasher> HashDBRef: Send + Sync { /// Type of in-memory overlay. - type Overlay: HashDB> + Default + Consolidate; + type Overlay: HashDB + Default + Consolidate; /// Get the value stored at key. - fn get(&self, key: &H::Out, prefix: Prefix, global: Option) -> Result>; + fn get(&self, key: &H::Out, prefix: Prefix) -> Result>; /// Call back when value get accessed in trie. fn access_from(&self, key: &H::Out); } @@ -456,9 +430,8 @@ impl TrieBackendStorage for Arc> { &self, key: &H::Out, prefix: Prefix, - global: Option, - ) -> Result> { - Storage::::get(self.deref(), key, prefix, global) + ) -> Result> { + Storage::::get(self.deref(), key, prefix) } fn access_from(&self, key: &H::Out) { @@ -466,21 +439,15 @@ impl TrieBackendStorage for Arc> { } } -impl TrieBackendStorage for sp_trie::GenericMemoryDB +impl TrieBackendStorage for sp_trie::GenericMemoryDB where H: Hasher, - MH: sp_trie::MetaHasher>, KF: sp_trie::KeyFunction + Send + Sync, { type Overlay = Self; - fn get( - &self, - key: &H::Out, - prefix: Prefix, - global: Option, - ) -> Result> { - Ok(HashDB::get_with_meta(self, key, prefix, global)) + fn get(&self, key: &H::Out, prefix: Prefix) -> Result> { + Ok(hash_db::HashDB::get(self, key, prefix)) } fn access_from(&self, key: &H::Out) { @@ -488,32 +455,21 @@ impl TrieBackendStorage for sp_trie::GenericMemoryDB } } -impl, H: Hasher> AsHashDB> +impl, H: Hasher> AsHashDB for TrieBackendEssence { - fn as_hash_db<'b>(&'b self) -> &'b (dyn HashDB> + 'b) { self } - fn as_hash_db_mut<'b>( - &'b mut self, - ) -> &'b mut (dyn HashDB> + 'b) { self } + fn as_hash_db<'b>(&'b self) -> &'b (dyn HashDB + 'b) { self } + fn as_hash_db_mut<'b>(&'b mut self) -> &'b mut (dyn HashDB + 'b) { self } } -impl, H: Hasher> HashDB> +impl, H: Hasher> HashDB for TrieBackendEssence { fn get(&self, key: &H::Out, prefix: Prefix) -> Option { - HashDB::get_with_meta(self, key, prefix, Default::default()).map(|r| r.0) - } - - fn get_with_meta( - &self, - key: &H::Out, - prefix: Prefix, - global: Option, - ) -> Option<(DBValue, TrieMeta)> { if *key == self.empty { - return Some(([0u8].to_vec(), ::meta_for_empty(global))) + return Some([0u8].to_vec()) } - match self.storage.get(&key, prefix, global) { + match self.storage.get(&key, prefix) { Ok(x) => x, Err(e) => { warn!(target: "trie", "Failed to read from DB: {}", e); @@ -532,15 +488,15 @@ impl, H: Hasher> HashDB H::Out { + fn insert(&mut self, _prefix: Prefix, _value: &[u8]) -> H::Out { unimplemented!(); } - fn insert(&mut self, _prefix: Prefix, _value: &[u8]) -> H::Out { + fn emplace(&mut self, _key: H::Out, _prefix: Prefix, _value: DBValue) { unimplemented!(); } - fn emplace(&mut self, _key: H::Out, _prefix: Prefix, _value: DBValue) { + fn emplace_ref(&mut self, _key: &H::Out, _prefix: Prefix, _value: &[u8]) { unimplemented!(); } @@ -549,22 +505,13 @@ impl, H: Hasher> HashDB, H: Hasher> HashDBRef> +impl, H: Hasher> HashDBRef for TrieBackendEssence { fn get(&self, key: &H::Out, prefix: Prefix) -> Option { HashDB::get(self, key, prefix) } - fn get_with_meta( - &self, - key: &H::Out, - prefix: Prefix, - global: Option, - ) -> Option<(DBValue, TrieMeta)> { - HashDB::get_with_meta(self, key, prefix, global) - } - fn access_from(&self, key: &H::Out, at: Option<&H::Out>) -> Option { HashDB::access_from(self, key, at) } diff --git a/primitives/trie/src/lib.rs b/primitives/trie/src/lib.rs index 149e2166b68e9..79068c585eb6e 100644 --- a/primitives/trie/src/lib.rs +++ b/primitives/trie/src/lib.rs @@ -41,161 +41,38 @@ pub use storage_proof::{StorageProof, CompactProof}; pub use trie_db::{ Trie, TrieMut, DBValue, Recorder, CError, Query, TrieLayout, TrieConfiguration, nibble_ops, TrieDBIterator, TrieDBKeyIterator, Meta, node::{NodePlan, ValuePlan}, - GlobalMeta, }; /// Various re-exports from the `memory-db` crate. pub use memory_db::KeyFunction; pub use memory_db::prefixed_key; /// Various re-exports from the `hash-db` crate. -pub use hash_db::{HashDB as HashDBT, EMPTY_PREFIX, MetaHasher}; -pub use hash_db::NoMeta; +pub use hash_db::{HashDB as HashDBT, EMPTY_PREFIX}; /// Trie codec reexport, mainly child trie support /// for trie compact proof. pub use trie_codec::{decode_compact, encode_compact, Error as CompactProofError}; -/// Meta use by trie state. -#[derive(Default, Clone, Debug)] -pub struct TrieMeta { - /// Range of encoded value or hashed value. - /// When encoded value, it includes the length of the value. - pub range: Option>, - /// Defined in the trie layout, when used with - /// `TrieDbMut` it switch nodes to alternative hashing - /// method by defining the threshold to use with alternative - /// hashing. - /// Trie codec or other proof manipulation will always use - /// `None` in order to prevent state change on reencoding. - pub try_inner_hashing: Option, - /// Flag indicating alternative value hash is currently use - /// or will be use. - pub apply_inner_hashing: bool, - /// Does current encoded contains a hash instead of - /// a value (information stored in meta for proofs). - pub contain_hash: bool, - /// Record if a value was accessed, this is - /// set as accessed by defalult, but can be - /// change on access explicitely: `HashDB::get_with_meta`. - /// and reset on access explicitely: `HashDB::access_from`. - /// Not strictly needed in this struct, but does not add memory usage here. - pub unused_value: bool, -} - -impl Meta for TrieMeta { - /// When true apply inner hashing of value. - type GlobalMeta = Option; - - /// When true apply inner hashing of value. - type StateMeta = bool; - - fn set_state_meta(&mut self, state_meta: Self::StateMeta) { - self.apply_inner_hashing = state_meta; - } - - fn read_state_meta(&self) -> Self::StateMeta { - self.apply_inner_hashing - } - - fn read_global_meta(&self) -> Self::GlobalMeta { - self.try_inner_hashing - } - - fn set_global_meta(&mut self, global_meta: Self::GlobalMeta) { - self.try_inner_hashing = global_meta; - } - - fn meta_for_new( - global: Self::GlobalMeta, - ) -> Self { - let mut result = Self::default(); - result.set_global_meta(global); - result - } - - fn meta_for_existing_inline_node( - global: Self::GlobalMeta, - ) -> Self { - Self::meta_for_new(global) - } - - fn meta_for_empty( - global: Self::GlobalMeta, - ) -> Self { - Self::meta_for_new(global) - } - - fn encoded_value_callback( - &mut self, - value_plan: ValuePlan, - ) { - let (contain_hash, range) = match value_plan { - ValuePlan::Value(range, with_len) => (false, with_len..range.end), - ValuePlan::HashedValue(range, _size) => (true, range), - ValuePlan::NoValue => return, - }; - - if let Some(threshold) = self.try_inner_hashing.clone() { - self.apply_inner_hashing = range.end - range.start >= threshold as usize; - } - - self.range = Some(range); - self.contain_hash = contain_hash; - } - - fn decoded_callback( - &mut self, - node_plan: &NodePlan, - ) { - let (contain_hash, range) = match node_plan.value_plan() { - Some(ValuePlan::Value(range, with_len)) => (false, *with_len..range.end), - Some(ValuePlan::HashedValue(range, _size)) => (true, range.clone()), - Some(ValuePlan::NoValue) => return, - None => return, - }; - - self.range = Some(range); - self.contain_hash = contain_hash; - } - - fn contains_hash_of_value(&self) -> bool { - self.contain_hash - } -} - -impl TrieMeta { - /// Was value accessed. - pub fn accessed_value(&mut self) -> bool { - !self.unused_value - } - - /// For proof, this allow setting node as unaccessed until - /// a call to `access_from`. - pub fn set_accessed_value(&mut self, accessed: bool) { - self.unused_value = !accessed; - } -} - /// substrate trie layout -pub struct Layout(Option, sp_std::marker::PhantomData<(H, M)>); +pub struct Layout(Option, sp_std::marker::PhantomData); -impl fmt::Debug for Layout { +impl fmt::Debug for Layout { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { f.debug_struct("Layout").finish() } } -impl Clone for Layout { +impl Clone for Layout { fn clone(&self) -> Self { Layout(self.0, sp_std::marker::PhantomData) } } -impl Default for Layout { +impl Default for Layout { fn default() -> Self { Layout(None, sp_std::marker::PhantomData) } } -impl Layout { +impl Layout { /// Layout with inner hashing active. /// Will flag trie for hashing. pub fn with_alt_hashing(threshold: u32) -> Self { @@ -203,11 +80,9 @@ impl Layout { } } -impl TrieLayout for Layout +impl TrieLayout for Layout where H: Hasher, - M: MetaHasher>, - M::Meta: Meta, StateMeta = bool>, { const USE_EXTENSION: bool = false; const ALLOW_EMPTY: bool = true; @@ -215,108 +90,22 @@ impl TrieLayout for Layout type Hash = H; type Codec = NodeCodec; - type MetaHasher = M; - type Meta = M::Meta; - fn global_meta(&self) -> GlobalMeta { + fn alt_threshold(&self) -> Option { self.0 } } -/// Hasher with support to meta. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub struct StateHasher; - -impl MetaHasher for StateHasher - where - H: Hasher, -{ - type Meta = TrieMeta; - type GlobalMeta = Option; - - fn hash(value: &[u8], meta: &Self::Meta) -> H::Out { - match &meta { - TrieMeta { range: Some(range), contain_hash: false, apply_inner_hashing: true, .. } => { - let value = inner_hashed_value::(value, Some((range.start, range.end))); - H::hash(value.as_slice()) - }, - TrieMeta { range: Some(_range), contain_hash: true, .. } => { - // value contains a hash of data (already inner_hashed_value). - H::hash(value) - }, - _ => { - H::hash(value) - }, - } - } - - fn stored_value(value: &[u8], mut meta: Self::Meta) -> DBValue { - let mut stored = Vec::with_capacity(value.len() + 1); - if meta.contain_hash { - // already contain hash, just flag it. - stored.push(trie_constants::DEAD_HEADER_META_HASHED_VALUE); - stored.extend_from_slice(value); - return stored; - } - if meta.unused_value && meta.apply_inner_hashing { - if meta.range.is_some() { - // Warning this assumes that encoded value cannot start by this, - // so it is tightly coupled with the header type of the codec. - stored.push(trie_constants::DEAD_HEADER_META_HASHED_VALUE); - let range = meta.range.as_ref().expect("Tested in condition"); - meta.contain_hash = true; // useless but could be with meta as &mut - // store hash instead of value. - let value = inner_hashed_value::(value, Some((range.start, range.end))); - stored.extend_from_slice(value.as_slice()); - return stored; - } - } - stored.extend_from_slice(value); - stored - } - - fn stored_value_owned(value: DBValue, meta: Self::Meta) -> DBValue { - >::stored_value(value.as_slice(), meta) - } - - fn extract_value(mut stored: &[u8], global_meta: Self::GlobalMeta) -> (&[u8], Self::Meta) { - let input = &mut stored; - let mut contain_hash = false; - if input.get(0) == Some(&trie_constants::DEAD_HEADER_META_HASHED_VALUE) { - contain_hash = true; - *input = &input[1..]; - } - let mut meta = TrieMeta { - range: None, - unused_value: contain_hash, - contain_hash, - apply_inner_hashing: false, - try_inner_hashing: None, - }; - meta.set_global_meta(global_meta); - (stored, meta) - } - - fn extract_value_owned(mut stored: DBValue, global: Self::GlobalMeta) -> (DBValue, Self::Meta) { - let len = stored.len(); - let (v, meta) = >::extract_value(stored.as_slice(), global); - let removed = len - v.len(); - (stored.split_off(removed), meta) - } -} - -impl TrieConfiguration for Layout +impl TrieConfiguration for Layout where H: Hasher, - M: MetaHasher>, - M::Meta: Meta, StateMeta = bool>, { fn trie_root(&self, input: I) -> ::Out where I: IntoIterator, A: AsRef<[u8]> + Ord, B: AsRef<[u8]>, { - trie_root::trie_root_no_extension::(input, self.global_meta()) + trie_root::trie_root_no_extension::(input, self.alt_threshold()) } fn trie_root_unhashed(&self, input: I) -> Vec where @@ -324,7 +113,7 @@ impl TrieConfiguration for Layout A: AsRef<[u8]> + Ord, B: AsRef<[u8]>, { - trie_root::unhashed_trie_no_extension::(input, self.global_meta()) + trie_root::unhashed_trie_no_extension::(input, self.alt_threshold()) } fn encode_index(input: u32) -> Vec { @@ -340,25 +129,25 @@ type MemTracker = memory_db::MemCounter; /// TrieDB error over `TrieConfiguration` trait. pub type TrieError = trie_db::TrieError, CError>; /// Reexport from `hash_db`, with genericity set for `Hasher` trait. -pub trait AsHashDB: hash_db::AsHashDB {} -impl> AsHashDB for T {} +pub trait AsHashDB: hash_db::AsHashDB {} +impl> AsHashDB for T {} /// Reexport from `hash_db`, with genericity set for `Hasher` trait. -pub type HashDB<'a, H, M, GM> = dyn hash_db::HashDB + 'a; +pub type HashDB<'a, H> = dyn hash_db::HashDB + 'a; /// Reexport from `hash_db`, with genericity set for `Hasher` trait. /// This uses a `KeyFunction` for prefixing keys internally (avoiding /// key conflict for non random keys). pub type PrefixedMemoryDB = memory_db::MemoryDB< - H, memory_db::PrefixedKey, trie_db::DBValue, StateHasher, MemTracker + H, memory_db::PrefixedKey, trie_db::DBValue, MemTracker, >; /// Reexport from `hash_db`, with genericity set for `Hasher` trait. /// This uses a noops `KeyFunction` (key addressing must be hashed or using /// an encoding scheme that avoid key conflict). pub type MemoryDB = memory_db::MemoryDB< - H, memory_db::HashKey, trie_db::DBValue, StateHasher, MemTracker, + H, memory_db::HashKey, trie_db::DBValue, MemTracker, >; /// Reexport from `hash_db`, with genericity set for `Hasher` trait. -pub type GenericMemoryDB = memory_db::MemoryDB< - H, KF, trie_db::DBValue, MH, MemTracker, +pub type GenericMemoryDB = memory_db::MemoryDB< + H, KF, trie_db::DBValue, MemTracker, >; /// Persistent trie database read-access interface for the a given hasher. @@ -372,8 +161,7 @@ pub type TrieHash = <::Hash as Hasher>::Out; /// This module is for non generic definition of trie type. /// Only the `Hasher` trait is generic in this case. pub mod trie_types { - /// State layout. - pub type Layout = super::Layout; + use super::Layout; /// Persistent trie database read-access interface for the a given hasher. pub type TrieDB<'a, H> = super::TrieDB<'a, Layout>; /// Persistent trie database write-access interface for the a given hasher. @@ -400,7 +188,7 @@ pub fn generate_trie_proof<'a, L, I, K, DB>( L: TrieConfiguration, I: IntoIterator, K: 'a + AsRef<[u8]>, - DB: hash_db::HashDBRef>, + DB: hash_db::HashDBRef, { // Can use default layout (read only). let trie = TrieDB::::new(db, &root)?; @@ -427,7 +215,7 @@ pub fn verify_trie_proof<'a, L, I, K, V>( { // No specific info to read from layout. let layout = Default::default(); - verify_proof::, _, _, _>(root, proof, items, layout) + verify_proof::, _, _, _>(root, proof, items, layout) } /// Determine a trie root given a hash DB and delta values. @@ -441,7 +229,7 @@ pub fn delta_trie_root( A: Borrow<[u8]>, B: Borrow>, V: Borrow<[u8]>, - DB: hash_db::HashDB>, + DB: hash_db::HashDB, { { let mut trie = TrieDBMut::::from_existing_with_layout(db, &mut root, layout)?; @@ -468,7 +256,7 @@ pub fn read_trie_value( ) -> Result>, Box>> where L: TrieConfiguration, - DB: hash_db::HashDBRef>, + DB: hash_db::HashDBRef, { Ok(TrieDB::::new(&*db, root)?.get(key).map(|x| x.map(|val| val.to_vec()))?) } @@ -482,8 +270,8 @@ pub fn read_trie_value_with ( ) -> Result>, Box>> where L: TrieConfiguration, - Q: Query, - DB: hash_db::HashDBRef> + Q: Query, + DB: hash_db::HashDBRef { Ok(TrieDB::::new(&*db, root)?.get_with(key, query).map(|x| x.map(|val| val.to_vec()))?) } @@ -527,7 +315,7 @@ pub fn child_delta_trie_root( B: Borrow>, V: Borrow<[u8]>, RD: AsRef<[u8]>, - DB: hash_db::HashDB>, + DB: hash_db::HashDB, { let mut root = TrieHash::::default(); // root is fetched from DB, not writable by runtime, so it's always valid. @@ -546,9 +334,9 @@ pub fn child_delta_trie_root( pub fn record_all_keys( db: &DB, root: &TrieHash, - recorder: &mut Recorder, L::Meta> + recorder: &mut Recorder> ) -> Result<(), Box>> where - DB: hash_db::HashDBRef>, + DB: hash_db::HashDBRef, { let trie = TrieDB::::new(&*db, root)?; let iter = trie.iter()?; @@ -573,7 +361,7 @@ pub fn read_child_trie_value( key: &[u8] ) -> Result>, Box>> where - DB: hash_db::HashDBRef>, + DB: hash_db::HashDBRef, { let mut root = TrieHash::::default(); // root is fetched from DB, not writable by runtime, so it's always valid. @@ -593,8 +381,8 @@ pub fn read_child_trie_value_with( ) -> Result>, Box>> where L: TrieConfiguration, - Q: Query, - DB: hash_db::HashDBRef>, + Q: Query, + DB: hash_db::HashDBRef, { let mut root = TrieHash::::default(); // root is fetched from DB, not writable by runtime, so it's always valid. @@ -641,8 +429,8 @@ impl<'a, DB, H> KeySpacedDBMut<'a, DB, H> where } } -impl<'a, DB, H, T, M, GM> hash_db::HashDBRef for KeySpacedDB<'a, DB, H> where - DB: hash_db::HashDBRef, +impl<'a, DB, H, T> hash_db::HashDBRef for KeySpacedDB<'a, DB, H> where + DB: hash_db::HashDBRef, H: Hasher, T: From<&'static [u8]>, { @@ -655,19 +443,14 @@ impl<'a, DB, H, T, M, GM> hash_db::HashDBRef for KeySpacedDB<'a, DB self.0.access_from(key, at) } - fn get_with_meta(&self, key: &H::Out, prefix: Prefix, global: GM) -> Option<(T, M)> { - let derived_prefix = keyspace_as_prefix_alloc(self.1, prefix); - self.0.get_with_meta(key, (&derived_prefix.0, derived_prefix.1), global) - } - fn contains(&self, key: &H::Out, prefix: Prefix) -> bool { let derived_prefix = keyspace_as_prefix_alloc(self.1, prefix); self.0.contains(key, (&derived_prefix.0, derived_prefix.1)) } } -impl<'a, DB, H, T, M, GM> hash_db::HashDB for KeySpacedDBMut<'a, DB, H> where - DB: hash_db::HashDB, +impl<'a, DB, H, T> hash_db::HashDB for KeySpacedDBMut<'a, DB, H> where + DB: hash_db::HashDB, H: Hasher, T: Default + PartialEq + for<'b> From<&'b [u8]> + Clone + Send + Sync, { @@ -680,11 +463,6 @@ impl<'a, DB, H, T, M, GM> hash_db::HashDB for KeySpacedDBMut<'a, DB self.0.access_from(key, at) } - fn get_with_meta(&self, key: &H::Out, prefix: Prefix, global: GM) -> Option<(T, M)> { - let derived_prefix = keyspace_as_prefix_alloc(self.1, prefix); - self.0.get_with_meta(key, (&derived_prefix.0, derived_prefix.1), global) - } - fn contains(&self, key: &H::Out, prefix: Prefix) -> bool { let derived_prefix = keyspace_as_prefix_alloc(self.1, prefix); self.0.contains(key, (&derived_prefix.0, derived_prefix.1)) @@ -695,19 +473,14 @@ impl<'a, DB, H, T, M, GM> hash_db::HashDB for KeySpacedDBMut<'a, DB self.0.insert((&derived_prefix.0, derived_prefix.1), value) } - fn insert_with_meta( - &mut self, - prefix: Prefix, - value: &[u8], - meta: M, - ) -> H::Out { + fn emplace(&mut self, key: H::Out, prefix: Prefix, value: T) { let derived_prefix = keyspace_as_prefix_alloc(self.1, prefix); - self.0.insert_with_meta((&derived_prefix.0, derived_prefix.1), value, meta) + self.0.emplace(key, (&derived_prefix.0, derived_prefix.1), value) } - fn emplace(&mut self, key: H::Out, prefix: Prefix, value: T) { + fn emplace_ref(&mut self, key: &H::Out, prefix: Prefix, value: &[u8]) { let derived_prefix = keyspace_as_prefix_alloc(self.1, prefix); - self.0.emplace(key, (&derived_prefix.0, derived_prefix.1), value) + self.0.emplace_ref(key, (&derived_prefix.0, derived_prefix.1), value) } fn remove(&mut self, key: &H::Out, prefix: Prefix) { @@ -716,14 +489,14 @@ impl<'a, DB, H, T, M, GM> hash_db::HashDB for KeySpacedDBMut<'a, DB } } -impl<'a, DB, H, T, M, GM> hash_db::AsHashDB for KeySpacedDBMut<'a, DB, H> where - DB: hash_db::HashDB, +impl<'a, DB, H, T> hash_db::AsHashDB for KeySpacedDBMut<'a, DB, H> where + DB: hash_db::HashDB, H: Hasher, T: Default + PartialEq + for<'b> From<&'b [u8]> + Clone + Send + Sync, { - fn as_hash_db(&self) -> &dyn hash_db::HashDB { &*self } + fn as_hash_db(&self) -> &dyn hash_db::HashDB { &*self } - fn as_hash_db_mut<'b>(&'b mut self) -> &'b mut (dyn hash_db::HashDB + 'b) { + fn as_hash_db_mut<'b>(&'b mut self) -> &'b mut (dyn hash_db::HashDB + 'b) { &mut *self } } @@ -765,10 +538,10 @@ fn inner_hashed_value(x: &[u8], range: Option<(usize, usize)>) -> Vec } /// Estimate encoded size of node. -pub fn estimate_entry_size(entry: &(DBValue, TrieMeta), hash_len: usize) -> usize { +pub fn estimate_entry_size(entry: &(DBValue, Meta, bool), hash_len: usize) -> usize { use codec::Encode; let mut full_encoded = entry.0.encoded_size(); - if entry.1.unused_value && entry.1.apply_inner_hashing { + if !entry.2 && entry.1.apply_inner_hashing { if let Some(range) = entry.1.range.as_ref() { let value_size = range.end - range.start; full_encoded -= value_size; @@ -780,10 +553,31 @@ pub fn estimate_entry_size(entry: &(DBValue, TrieMeta), hash_len: usize) -> usiz full_encoded } +/// Switch to hashed value variant. +pub fn to_hashed_variant( + value: &[u8], + meta: &mut Meta, + used_value: bool, +) -> Option { + if !meta.contain_hash && meta.apply_inner_hashing && !used_value && meta.range.is_some() { + let mut stored = Vec::with_capacity(value.len() + 1); + // Warning this assumes that encoded value cannot start by this, + // so it is tightly coupled with the header type of the codec. + stored.push(trie_constants::DEAD_HEADER_META_HASHED_VALUE); + let range = meta.range.as_ref().expect("Tested in condition"); + // store hash instead of value. + let value = inner_hashed_value::(value, Some((range.start, range.end))); + stored.extend_from_slice(value.as_slice()); + meta.contain_hash = true; + return Some(stored); + } + None +} + /// Decode plan in order to update meta early (needed to register proofs). -pub fn resolve_encoded_meta(entry: &mut (DBValue, TrieMeta)) { +pub fn resolve_encoded_meta(entry: &mut (DBValue, Meta, bool)) { use trie_db::NodeCodec; - let _ = as TrieLayout>::Codec::decode_plan(entry.0.as_slice(), &mut entry.1); + let _ = as TrieLayout>::Codec::decode_plan(entry.0.as_slice(), &mut entry.1); } /// Constants used into trie simplification codec. @@ -811,14 +605,14 @@ mod tests { use trie_standardmap::{Alphabet, ValueMode, StandardMap}; use hex_literal::hex; - type Layout = super::trie_types::Layout; + type Layout = super::Layout; - type MemoryDBMeta = memory_db::MemoryDB< - H, memory_db::HashKey, trie_db::DBValue, M, MemTracker, + type MemoryDBMeta = memory_db::MemoryDB< + H, memory_db::HashKey, trie_db::DBValue, MemTracker, >; fn hashed_null_node() -> TrieHash { - >::hashed_null_node() + ::hashed_null_node() } fn check_equivalent(input: &Vec<(&[u8], &[u8])>, layout: T) { @@ -827,7 +621,7 @@ mod tests { let d = layout.trie_root_unhashed(input.clone()); println!("Data: {:#x?}, {:#x?}", d, Blake2Hasher::hash(&d[..])); let persistent = { - let mut memdb = MemoryDBMeta::<_, T::MetaHasher>::default(); + let mut memdb = MemoryDBMeta::default(); let mut root = Default::default(); let mut t = TrieDBMut::::new_with_layout(&mut memdb, &mut root, layout); for (x, y) in input.iter().rev() { @@ -840,7 +634,7 @@ mod tests { } fn check_iteration(input: &Vec<(&[u8], &[u8])>, layout: T) { - let mut memdb = MemoryDBMeta::<_, T::MetaHasher>::default(); + let mut memdb = MemoryDBMeta::default(); let mut root = Default::default(); { let mut t = TrieDBMut::::new_with_layout(&mut memdb, &mut root, layout.clone()); @@ -969,13 +763,13 @@ mod tests { } fn populate_trie<'db, T>( - db: &'db mut dyn HashDB>, + db: &'db mut dyn HashDB, root: &'db mut TrieHash, v: &[(Vec, Vec)], layout: T, ) -> TrieDBMut<'db, T> where - T: TrieConfiguration, + T: TrieConfiguration, { let mut t = TrieDBMut::::new_with_layout(db, root, layout); for i in 0..v.len() { diff --git a/primitives/trie/src/node_codec.rs b/primitives/trie/src/node_codec.rs index 3f90bf8126352..bed8ae73b5852 100644 --- a/primitives/trie/src/node_codec.rs +++ b/primitives/trie/src/node_codec.rs @@ -88,16 +88,16 @@ impl<'a> Input for ByteSliceInput<'a> { pub struct NodeCodec(PhantomData); impl NodeCodec { - fn decode_plan_inner_hashed>( + fn decode_plan_inner_hashed( data: &[u8], - meta: &mut M, + meta: &mut Meta, ) -> Result { let mut input = ByteSliceInput::new(data); - let contains_hash = meta.contains_hash_of_value(); let header = NodeHeader::decode(&mut input)?; + let contains_hash = header.contains_hash_of_value(); let alt_hashing = header.alt_hashing(); - meta.set_state_meta(alt_hashing); + meta.apply_inner_hashing = alt_hashing; let branch_has_value = if let NodeHeader::Branch(has_value, _) = &header { *has_value @@ -108,7 +108,7 @@ impl NodeCodec { match header { NodeHeader::Null => Ok(NodePlan::Empty), - NodeHeader::AltHashBranch(nibble_count) + NodeHeader::AltHashBranch(nibble_count, _) | NodeHeader::Branch(_, nibble_count) => { let padding = nibble_count % nibble_ops::NIBBLE_PER_BYTE != 0; // check that the padding is valid (if any) @@ -153,7 +153,7 @@ impl NodeCodec { children, }) }, - NodeHeader::AltHashLeaf(nibble_count) + NodeHeader::AltHashLeaf(nibble_count, _) | NodeHeader::Leaf(nibble_count) => { let padding = nibble_count % nibble_ops::NIBBLE_PER_BYTE != 0; // check that the padding is valid (if any) @@ -181,19 +181,19 @@ impl NodeCodec { } } -impl NodeCodecT for NodeCodec +impl NodeCodecT for NodeCodec where H: Hasher, - M: Meta>, { + const OFFSET_CONTAINS_HASH: usize = 1; type Error = Error; type HashOut = H::Out; fn hashed_null_node() -> ::Out { - H::hash(>::empty_node_no_meta()) + H::hash(::empty_node()) } - fn decode_plan(data: &[u8], meta: &mut M) -> Result { + fn decode_plan(data: &[u8], meta: &mut Meta) -> Result { Self::decode_plan_inner_hashed(data, meta).map(|plan| { meta.decoded_callback(&plan); plan @@ -205,27 +205,28 @@ impl NodeCodecT for NodeCodec } fn is_empty_node(data: &[u8]) -> bool { - data == >::empty_node_no_meta() + data == ::empty_node() } - fn empty_node(_meta: &mut M) -> Vec { - sp_std::vec![trie_constants::EMPTY_TRIE] - } - - fn empty_node_no_meta() -> &'static [u8] { + fn empty_node() -> &'static [u8] { &[trie_constants::EMPTY_TRIE] } - fn leaf_node(partial: Partial, value: Value, meta: &mut M) -> Vec { + fn leaf_node(partial: Partial, value: Value, meta: &mut Meta) -> Vec { + let contains_hash = matches!(&value, Value::HashedValue(..)); // Note that we use AltHash type only if inner hashing will occur, // this way we allow changing hash threshold. // With fix inner hashing alt hash can be use with all node, but // that is not better (encoding can use an additional nibble byte // sometime). - let mut output = if meta.read_global_meta().as_ref().map(|threshold| + let mut output = if meta.try_inner_hashing.as_ref().map(|threshold| value_do_hash(&value, threshold) - ).unwrap_or(meta.read_state_meta()) { - partial_encode(partial, NodeKind::AltHashLeaf) + ).unwrap_or(meta.apply_inner_hashing) { + if contains_hash { + partial_encode(partial, NodeKind::AltHashLeafHash) + } else { + partial_encode(partial, NodeKind::AltHashLeaf) + } } else { partial_encode(partial, NodeKind::Leaf) }; @@ -254,7 +255,7 @@ impl NodeCodecT for NodeCodec _partial: impl Iterator, _nbnibble: usize, _child: ChildReference<::Out>, - _meta: &mut M, + _meta: &mut Meta, ) -> Vec { unreachable!() } @@ -262,7 +263,7 @@ impl NodeCodecT for NodeCodec fn branch_node( _children: impl Iterator::Out>>>>, _maybe_value: Value, - _meta: &mut M, + _meta: &mut Meta, ) -> Vec { unreachable!() } @@ -272,11 +273,13 @@ impl NodeCodecT for NodeCodec number_nibble: usize, children: impl Iterator::Out>>>>, value: Value, - meta: &mut M, + meta: &mut Meta, ) -> Vec { - let mut output = match (&value, meta.read_global_meta().as_ref().map(|threshold| + + let contains_hash = matches!(&value, Value::HashedValue(..)); + let mut output = match (&value, meta.try_inner_hashing.as_ref().map(|threshold| value_do_hash(&value, threshold) - ).unwrap_or(meta.read_state_meta())) { + ).unwrap_or(meta.apply_inner_hashing)) { (&Value::NoValue, _) => { partial_from_iterator_encode(partial, number_nibble, NodeKind::BranchNoValue) }, @@ -284,7 +287,11 @@ impl NodeCodecT for NodeCodec partial_from_iterator_encode(partial, number_nibble, NodeKind::BranchWithValue) }, (_, true) => { - partial_from_iterator_encode(partial, number_nibble, NodeKind::AltHashBranchWithValue) + if contains_hash { + partial_from_iterator_encode(partial, number_nibble, NodeKind::AltHashBranchWithValueHash) + } else { + partial_from_iterator_encode(partial, number_nibble, NodeKind::AltHashBranchWithValue) + } }, }; @@ -354,8 +361,11 @@ fn partial_from_iterator_encode>( NodeKind::Leaf => NodeHeader::Leaf(nibble_count).encode_to(&mut output), NodeKind::BranchWithValue => NodeHeader::Branch(true, nibble_count).encode_to(&mut output), NodeKind::BranchNoValue => NodeHeader::Branch(false, nibble_count).encode_to(&mut output), - NodeKind::AltHashLeaf => NodeHeader::AltHashLeaf(nibble_count).encode_to(&mut output), - NodeKind::AltHashBranchWithValue => NodeHeader::AltHashBranch(nibble_count) + NodeKind::AltHashLeaf => NodeHeader::AltHashLeaf(nibble_count, false).encode_to(&mut output), + NodeKind::AltHashBranchWithValue => NodeHeader::AltHashBranch(nibble_count, false) + .encode_to(&mut output), + NodeKind::AltHashLeafHash => NodeHeader::AltHashLeaf(nibble_count, true).encode_to(&mut output), + NodeKind::AltHashBranchWithValueHash => NodeHeader::AltHashBranch(nibble_count, true) .encode_to(&mut output), }; output.extend(partial); @@ -375,8 +385,11 @@ fn partial_encode(partial: Partial, node_kind: NodeKind) -> Vec { NodeKind::Leaf => NodeHeader::Leaf(nibble_count).encode_to(&mut output), NodeKind::BranchWithValue => NodeHeader::Branch(true, nibble_count).encode_to(&mut output), NodeKind::BranchNoValue => NodeHeader::Branch(false, nibble_count).encode_to(&mut output), - NodeKind::AltHashLeaf => NodeHeader::AltHashLeaf(nibble_count).encode_to(&mut output), - NodeKind::AltHashBranchWithValue => NodeHeader::AltHashBranch(nibble_count) + NodeKind::AltHashLeaf => NodeHeader::AltHashLeaf(nibble_count, false).encode_to(&mut output), + NodeKind::AltHashBranchWithValue => NodeHeader::AltHashBranch(nibble_count, false) + .encode_to(&mut output), + NodeKind::AltHashLeafHash => NodeHeader::AltHashLeaf(nibble_count, true).encode_to(&mut output), + NodeKind::AltHashBranchWithValueHash => NodeHeader::AltHashBranch(nibble_count, true) .encode_to(&mut output), }; if number_nibble_encoded > 0 { diff --git a/primitives/trie/src/node_header.rs b/primitives/trie/src/node_header.rs index 43ac67cbabcaf..ca1f375fb8eff 100644 --- a/primitives/trie/src/node_header.rs +++ b/primitives/trie/src/node_header.rs @@ -26,10 +26,24 @@ use sp_std::iter::once; #[derive(sp_core::RuntimeDebug)] pub(crate) enum NodeHeader { Null, + // contains wether there is a value and nibble count Branch(bool, usize), + // contains nibble count Leaf(usize), - AltHashBranch(usize), - AltHashLeaf(usize), + // contains nibble count and wether the value is a hash. + AltHashBranch(usize, bool), + // contains nibble count and wether the value is a hash. + AltHashLeaf(usize, bool), +} + +impl NodeHeader { + pub(crate) fn contains_hash_of_value(&self) -> bool { + match self { + NodeHeader::AltHashBranch(_, true) + | NodeHeader::AltHashLeaf(_, true) => true, + _ => false, + } + } } /// NodeHeader without content @@ -39,10 +53,15 @@ pub(crate) enum NodeKind { BranchWithValue, AltHashLeaf, AltHashBranchWithValue, + AltHashLeafHash, + AltHashBranchWithValueHash, } impl Encode for NodeHeader { fn encode_to(&self, output: &mut T) { + if self.contains_hash_of_value() { + output.write(&[trie_constants::DEAD_HEADER_META_HASHED_VALUE]); + } match self { NodeHeader::Null => output.push_byte(trie_constants::EMPTY_TRIE), NodeHeader::Branch(true, nibble_count) => @@ -51,9 +70,9 @@ impl Encode for NodeHeader { encode_size_and_prefix(*nibble_count, trie_constants::BRANCH_WITHOUT_MASK, 2, output), NodeHeader::Leaf(nibble_count) => encode_size_and_prefix(*nibble_count, trie_constants::LEAF_PREFIX_MASK, 2, output), - NodeHeader::AltHashBranch(nibble_count) => + NodeHeader::AltHashBranch(nibble_count, _) => encode_size_and_prefix(*nibble_count, trie_constants::ALT_HASHING_BRANCH_WITH_MASK, 4, output), - NodeHeader::AltHashLeaf(nibble_count) => + NodeHeader::AltHashLeaf(nibble_count, _) => encode_size_and_prefix(*nibble_count, trie_constants::ALT_HASHING_LEAF_PREFIX_MASK, 3, output), } } @@ -76,19 +95,25 @@ impl codec::EncodeLike for NodeHeader {} impl Decode for NodeHeader { fn decode(input: &mut I) -> Result { - let i = input.read_byte()?; + let mut i = input.read_byte()?; if i == trie_constants::EMPTY_TRIE { return Ok(NodeHeader::Null); } + let contain_hash = if trie_constants::DEAD_HEADER_META_HASHED_VALUE == i { + i = input.read_byte()?; + true + } else { + false + }; match i & (0b11 << 6) { trie_constants::LEAF_PREFIX_MASK => Ok(NodeHeader::Leaf(decode_size(i, input, 2)?)), trie_constants::BRANCH_WITH_MASK => Ok(NodeHeader::Branch(true, decode_size(i, input, 2)?)), trie_constants::BRANCH_WITHOUT_MASK => Ok(NodeHeader::Branch(false, decode_size(i, input, 2)?)), trie_constants::EMPTY_TRIE => { if i & (0b111 << 5) == trie_constants::ALT_HASHING_LEAF_PREFIX_MASK { - Ok(NodeHeader::AltHashLeaf(decode_size(i, input, 3)?)) + Ok(NodeHeader::AltHashLeaf(decode_size(i, input, 3)?, contain_hash)) } else if i & (0b1111 << 4) == trie_constants::ALT_HASHING_BRANCH_WITH_MASK { - Ok(NodeHeader::AltHashBranch(decode_size(i, input, 4)?)) + Ok(NodeHeader::AltHashBranch(decode_size(i, input, 4)?, contain_hash)) } else { // do not allow any special encoding Err("Unallowed encoding".into()) diff --git a/primitives/trie/src/storage_proof.rs b/primitives/trie/src/storage_proof.rs index 43a32b25ebfb7..cebad1911315f 100644 --- a/primitives/trie/src/storage_proof.rs +++ b/primitives/trie/src/storage_proof.rs @@ -18,9 +18,8 @@ use sp_std::vec::Vec; use codec::{Encode, Decode}; use hash_db::{Hasher, HashDB}; -use hash_db::MetaHasher; use trie_db::NodeCodec; -use crate::{trie_types::Layout, TrieLayout}; +use crate::{Layout, TrieLayout}; /// A proof that some set of key-value pairs are included in the storage trie. The proof contains /// the storage values so that the partial storage backend can be reconstructed by a verifier that @@ -181,15 +180,14 @@ impl From for crate::MemoryDB { // Using compact proof will work directly here (read trie structure and // work directly. for item in proof.trie_nodes.iter() { - // Note using `default()` to build proof is fine, do_value being in header - // and no switch needed. - let layout_meta = Default::default(); - let (encoded_node, mut meta) = < - as TrieLayout>::MetaHasher as MetaHasher - >::extract_value(item.as_slice(), layout_meta); - // read state meta (required for value layout and AltHash node. - let _ = as TrieLayout>::Codec::decode_plan(encoded_node, &mut meta); - db.insert_with_meta(crate::EMPTY_PREFIX, encoded_node, meta); + let mut meta = Default::default(); + // read state meta (required for value layout). + let _ = as TrieLayout>::Codec::decode_plan(item.as_slice(), &mut meta); + db.alt_insert( + crate::EMPTY_PREFIX, + item, + meta.resolve_alt_hashing::< as TrieLayout>::Codec>(), + ); } db } diff --git a/primitives/trie/src/trie_codec.rs b/primitives/trie/src/trie_codec.rs index 3ffc2a4197f95..a04ffce939c67 100644 --- a/primitives/trie/src/trie_codec.rs +++ b/primitives/trie/src/trie_codec.rs @@ -22,7 +22,7 @@ use crate::{ EMPTY_PREFIX, HashDBT, TrieHash, TrieError, TrieConfiguration, - CompactProof, StorageProof, GlobalMeta, TrieMeta, + CompactProof, StorageProof, }; use sp_std::boxed::Box; use sp_std::vec::Vec; @@ -109,8 +109,7 @@ pub fn decode_compact<'a, L, DB, I>( ) -> Result, Error> where L: TrieConfiguration, - DB: HashDBT> - + hash_db::HashDBRef>, + DB: HashDBT + hash_db::HashDBRef, I: IntoIterator, { let mut nodes_iter = encoded.into_iter(); @@ -163,7 +162,7 @@ pub fn decode_compact<'a, L, DB, I>( } } - if !HashDBT::::contains(db, &top_root, EMPTY_PREFIX) { + if !HashDBT::::contains(db, &top_root, EMPTY_PREFIX) { return Err(Error::IncompleteProof); } @@ -211,7 +210,7 @@ pub fn encode_compact( root: TrieHash, ) -> Result> where - L: TrieConfiguration, + L: TrieConfiguration, { let mut child_tries = Vec::new(); let partial_db = proof.into_memory_db(); @@ -251,7 +250,7 @@ pub fn encode_compact( }; for child_root in child_tries { - if !HashDBT::::contains(&partial_db, &child_root, EMPTY_PREFIX) { + if !HashDBT::::contains(&partial_db, &child_root, EMPTY_PREFIX) { // child proof are allowed to be missing (unused root can be included // due to trie structure modification). continue; diff --git a/primitives/trie/src/trie_stream.rs b/primitives/trie/src/trie_stream.rs index bba41fe6d81c0..375db3de04cd0 100644 --- a/primitives/trie/src/trie_stream.rs +++ b/primitives/trie/src/trie_stream.rs @@ -17,12 +17,12 @@ //! `TrieStream` implementation for Substrate's trie format. -use hash_db::{MetaHasher, Hasher}; +use hash_db::Hasher; use trie_root; use codec::{Encode, Compact}; use sp_std::vec::Vec; use sp_std::ops::Range; -use crate::{trie_constants, TrieMeta, StateHasher}; +use crate::{trie_constants}; use crate::node_header::{NodeKind, size_and_prefix_iterator}; use crate::node_codec::Bitmap; @@ -70,6 +70,8 @@ fn fuse_nibbles_node<'a>(nibbles: &'a [u8], kind: NodeKind) -> impl Iterator size_and_prefix_iterator(size, trie_constants::ALT_HASHING_BRANCH_WITH_MASK, 4), + NodeKind::AltHashBranchWithValueHash + | NodeKind::AltHashLeafHash => unreachable!("only added value that do not contain hash"), }; iter_start .chain(if nibbles.len() % 2 == 1 { Some(nibbles[0]) } else { None }) @@ -78,8 +80,6 @@ fn fuse_nibbles_node<'a>(nibbles: &'a [u8], kind: NodeKind) -> impl Iterator; - fn new(meta: Option) -> Self { Self { buffer: Vec::new(), @@ -155,16 +155,10 @@ impl trie_root::TrieStream for TrieStream { 0..=31 => data.encode_to(&mut self.buffer), _ => { if apply_inner_hashing { - let meta = TrieMeta { - range: range, - unused_value: false, - contain_hash: false, - // Using `inner_value_hashing` instead to check this. - // And unused in hasher. - try_inner_hashing: None, - apply_inner_hashing: true, - }; - >>::hash(&data, &meta).as_ref() + hash_db::AltHashing { + encoded_offset: 0, + value_range: range.map(|r| (r.start, r.end)), + }.alt_hash::(&data).as_ref() .encode_to(&mut self.buffer); } else { H::hash(&data).as_ref().encode_to(&mut self.buffer); @@ -177,16 +171,11 @@ impl trie_root::TrieStream for TrieStream { let apply_inner_hashing = self.apply_inner_hashing; let range = self.current_value_range; let data = self.buffer; - let meta = TrieMeta { - range: range, - unused_value: false, - contain_hash: false, - try_inner_hashing: None, - apply_inner_hashing: true, - }; - if apply_inner_hashing { - >>::hash(&data, &meta) + hash_db::AltHashing { + encoded_offset: 0, + value_range: range.map(|r| (r.start, r.end)), + }.alt_hash::(&data) } else { H::hash(&data) } From 1e40e6d7da459de835387767f99664d5e17cb9b3 Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 21 Jun 2021 11:30:53 +0200 Subject: [PATCH 060/127] Finish update to refactored upstream. --- Cargo.lock | 24 ++++++++--------- Cargo.toml | 10 +++---- bin/node/bench/src/simple_trie.rs | 27 +++++++------------ bin/node/bench/src/trie.rs | 12 ++------- client/api/src/cht.rs | 4 +-- client/db/src/bench.rs | 18 ++++--------- client/db/src/lib.rs | 26 +++++------------- client/executor/src/integration_tests/mod.rs | 2 +- client/service/test/src/client/mod.rs | 2 +- .../state-machine/src/trie_backend_essence.rs | 1 + .../transaction-storage-proof/src/lib.rs | 2 +- primitives/trie/benches/bench.rs | 4 +-- primitives/trie/src/lib.rs | 2 +- 13 files changed, 49 insertions(+), 85 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f97655a692d0b..4fb09ef6c6d2c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2354,7 +2354,7 @@ dependencies = [ [[package]] name = "hash-db" version = "0.15.2" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple7#c7e0aeb8fd74c23845efe4b6daee43b7f4084faa" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#511e98bff38e2d618a5ec382eeca8661a5459df3" [[package]] name = "hash256-std-hasher" @@ -2368,7 +2368,7 @@ dependencies = [ [[package]] name = "hash256-std-hasher" version = "0.15.2" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple7#c7e0aeb8fd74c23845efe4b6daee43b7f4084faa" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#511e98bff38e2d618a5ec382eeca8661a5459df3" dependencies = [ "crunchy", ] @@ -3050,10 +3050,10 @@ dependencies = [ [[package]] name = "keccak-hasher" version = "0.15.3" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple7#c7e0aeb8fd74c23845efe4b6daee43b7f4084faa" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#511e98bff38e2d618a5ec382eeca8661a5459df3" dependencies = [ "hash-db", - "hash256-std-hasher 0.15.2 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple7)", + "hash256-std-hasher 0.15.2 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple4)", "tiny-keccak", ] @@ -3865,7 +3865,7 @@ dependencies = [ [[package]] name = "memory-db" version = "0.26.0" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple7#c7e0aeb8fd74c23845efe4b6daee43b7f4084faa" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#511e98bff38e2d618a5ec382eeca8661a5459df3" dependencies = [ "hash-db", "hashbrown", @@ -10551,22 +10551,22 @@ checksum = "a7f741b240f1a48843f9b8e0444fb55fb2a4ff67293b50a9179dfd5ea67f8d41" [[package]] name = "trie-bench" version = "0.27.0" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple7#c7e0aeb8fd74c23845efe4b6daee43b7f4084faa" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#511e98bff38e2d618a5ec382eeca8661a5459df3" dependencies = [ "criterion", "hash-db", - "keccak-hasher 0.15.3 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple7)", + "keccak-hasher 0.15.3 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple4)", "memory-db", "parity-scale-codec", "trie-db", "trie-root", - "trie-standardmap 0.15.2 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple7)", + "trie-standardmap 0.15.2 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple4)", ] [[package]] name = "trie-db" version = "0.22.5" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple7#c7e0aeb8fd74c23845efe4b6daee43b7f4084faa" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#511e98bff38e2d618a5ec382eeca8661a5459df3" dependencies = [ "hash-db", "hashbrown", @@ -10578,7 +10578,7 @@ dependencies = [ [[package]] name = "trie-root" version = "0.16.0" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple7#c7e0aeb8fd74c23845efe4b6daee43b7f4084faa" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#511e98bff38e2d618a5ec382eeca8661a5459df3" dependencies = [ "hash-db", ] @@ -10596,10 +10596,10 @@ dependencies = [ [[package]] name = "trie-standardmap" version = "0.15.2" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple7#c7e0aeb8fd74c23845efe4b6daee43b7f4084faa" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#511e98bff38e2d618a5ec382eeca8661a5459df3" dependencies = [ "hash-db", - "keccak-hasher 0.15.3 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple7)", + "keccak-hasher 0.15.3 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple4)", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index fc427dd2f2e16..754e1ea9f9b7d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -278,8 +278,8 @@ zeroize = { opt-level = 3 } panic = "unwind" [patch.crates-io] -hash-db = { git = "https://github.com/cheme/trie.git", branch = "hashed-value-simple7" } -memory-db = { git = "https://github.com/cheme/trie.git", branch = "hashed-value-simple7" } -trie-db = { git = "https://github.com/cheme/trie.git", branch = "hashed-value-simple7" } -trie-root = { git = "https://github.com/cheme/trie.git", branch = "hashed-value-simple7" } -trie-bench = { git = "https://github.com/cheme/trie.git", branch = "hashed-value-simple7" } +hash-db = { git = "https://github.com/cheme/trie.git", branch = "hashed-value-simple4" } +memory-db = { git = "https://github.com/cheme/trie.git", branch = "hashed-value-simple4" } +trie-db = { git = "https://github.com/cheme/trie.git", branch = "hashed-value-simple4" } +trie-root = { git = "https://github.com/cheme/trie.git", branch = "hashed-value-simple4" } +trie-bench = { git = "https://github.com/cheme/trie.git", branch = "hashed-value-simple4" } diff --git a/bin/node/bench/src/simple_trie.rs b/bin/node/bench/src/simple_trie.rs index f4a4576508f6c..ae3c1c6ce28df 100644 --- a/bin/node/bench/src/simple_trie.rs +++ b/bin/node/bench/src/simple_trie.rs @@ -20,7 +20,7 @@ use std::{collections::HashMap, sync::Arc}; use kvdb::KeyValueDB; use node_primitives::Hash; -use sp_trie::{DBValue, Meta, StateHasher, MetaHasher}; +use sp_trie::DBValue; use hash_db::{HashDB, AsHashDB, Prefix, Hasher as _}; pub type Hasher = sp_core::Blake2Hasher; @@ -31,15 +31,15 @@ pub struct SimpleTrie<'a> { pub overlay: &'a mut HashMap, Option>>, } -impl<'a> AsHashDB> for SimpleTrie<'a> { - fn as_hash_db(&self) -> &dyn hash_db::HashDB> { &*self } +impl<'a> AsHashDB for SimpleTrie<'a> { + fn as_hash_db(&self) -> &dyn hash_db::HashDB { &*self } - fn as_hash_db_mut<'b>(&'b mut self) -> &'b mut (dyn HashDB> + 'b) { + fn as_hash_db_mut<'b>(&'b mut self) -> &'b mut (dyn HashDB + 'b) { &mut *self } } -impl<'a> HashDB> for SimpleTrie<'a> { +impl<'a> HashDB for SimpleTrie<'a> { fn get(&self, key: &Hash, prefix: Prefix) -> Option { let key = sp_trie::prefixed_key::(key, prefix); if let Some(value) = self.overlay.get(&key) { @@ -48,22 +48,10 @@ impl<'a> HashDB> for SimpleTrie<'a> { self.db.get(0, &key).expect("Database backend error") } - fn get_with_meta(&self, key: &Hash, prefix: Prefix, global: Option) -> Option<(DBValue, Meta)> { - let result = self.get(key, prefix); - result.map(|value| >::extract_value_owned(value, global)) - } - fn contains(&self, hash: &Hash, prefix: Prefix) -> bool { self.get(hash, prefix).is_some() } - fn insert_with_meta(&mut self, prefix: Prefix, value: &[u8], meta: Meta) -> Hash { - let key = >::hash(value, &meta); - let stored_value = >::stored_value(value, meta); - self.emplace(key, prefix, stored_value); - key - } - fn insert(&mut self, prefix: Prefix, value: &[u8]) -> Hash { let key = Hasher::hash(value); self.emplace(key, prefix, value.to_vec()); @@ -75,6 +63,11 @@ impl<'a> HashDB> for SimpleTrie<'a> { self.overlay.insert(key, Some(value)); } + fn emplace_ref(&mut self, key: &Hash, prefix: Prefix, value: &[u8]) { + let key = sp_trie::prefixed_key::(key, prefix); + self.overlay.insert(key, Some(value.into())); + } + fn remove(&mut self, key: &Hash, prefix: Prefix) { let key = sp_trie::prefixed_key::(key, prefix); self.overlay.insert(key, None); diff --git a/bin/node/bench/src/trie.rs b/bin/node/bench/src/trie.rs index 67d07a7c024b1..7dcf2cd503e28 100644 --- a/bin/node/bench/src/trie.rs +++ b/bin/node/bench/src/trie.rs @@ -24,7 +24,7 @@ use lazy_static::lazy_static; use rand::Rng; use hash_db::Prefix; use sp_state_machine::Backend as _; -use sp_trie::{trie_types::TrieDBMut, TrieMut as _, Meta, MetaHasher, StateHasher}; +use sp_trie::{trie_types::TrieDBMut, TrieMut as _}; use node_primitives::Hash; @@ -170,17 +170,9 @@ impl core::BenchmarkDescription for TrieReadBenchmarkDescription { struct Storage(Arc); impl sp_state_machine::Storage for Storage { - fn get( - &self, - key: &Hash, - prefix: Prefix, - global: Option, - ) -> Result, Meta)>, String> { + fn get(&self, key: &Hash, prefix: Prefix) -> Result>, String> { let key = sp_trie::prefixed_key::(key, prefix); self.0.get(0, &key).map_err(|e| format!("Database backend error: {:?}", e)) - .map(|result| result - .map(|value| >::extract_value_owned(value, global)) - ) } fn access_from(&self, _key: &Hash) { diff --git a/client/api/src/cht.rs b/client/api/src/cht.rs index d0ad9facd73c2..498f7f8af5203 100644 --- a/client/api/src/cht.rs +++ b/client/api/src/cht.rs @@ -94,7 +94,7 @@ pub fn compute_root( I: IntoIterator>>, { use sp_trie::TrieConfiguration; - Ok(sp_trie::trie_types::Layout::::default().trie_root( + Ok(sp_trie::Layout::::default().trie_root( build_pairs::(cht_size, cht_num, hashes)? )) } @@ -172,7 +172,7 @@ pub fn check_proof_on_proving_backend( local_number, remote_hash, |_, local_cht_key| - read_proof_check_on_proving_backend::( + read_proof_check_on_proving_backend::( proving_backend, local_cht_key, ).map_err(ClientError::from_state), diff --git a/client/db/src/bench.rs b/client/db/src/bench.rs index a43a733bdd47b..a3db2142ca986 100644 --- a/client/db/src/bench.rs +++ b/client/db/src/bench.rs @@ -23,7 +23,7 @@ use std::cell::{Cell, RefCell}; use std::collections::HashMap; use hash_db::{Prefix, Hasher}; -use sp_trie::{MemoryDB, prefixed_key, StateHasher, Meta, MetaHasher}; +use sp_trie::{MemoryDB, prefixed_key}; use sp_core::{ storage::{ChildInfo, TrackedStorageKey}, hexdisplay::HexDisplay @@ -49,30 +49,22 @@ struct StorageDb { } impl sp_state_machine::Storage> for StorageDb { - fn get( - &self, - key: &Block::Hash, - prefix: Prefix, - global: Option, - ) -> Result, String> { + fn get(&self, key: &Block::Hash, prefix: Prefix) -> Result, String> { let prefixed_key = prefixed_key::>(key, prefix); if let Some(recorder) = &self.proof_recorder { if let Some(v) = recorder.get(&key) { return Ok(v.clone()); } let backend_value = self.db.get(0, &prefixed_key) - .map(|result| result.map(|value| - , _>>::extract_value_owned(value, global) - )).map_err(|e| format!("Database backend error: {:?}", e))?; + .map_err(|e| format!("Database backend error: {:?}", e))?; recorder.record::>(key.clone(), backend_value.clone()); Ok(backend_value) } else { self.db.get(0, &prefixed_key) - .map(|result| result.map(|value| - , _>>::extract_value_owned(value, global) - )).map_err(|e| format!("Database backend error: {:?}", e)) + .map_err(|e| format!("Database backend error: {:?}", e)) } } + fn access_from(&self, key: &Block::Hash) { if let Some(recorder) = &self.proof_recorder { recorder.access_from(key, HashFor::::LENGTH); diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index fcbea6201e7bf..89b02d3a0354b 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -65,8 +65,7 @@ use sp_blockchain::{ }; use codec::{Decode, Encode}; use hash_db::Prefix; -use sp_trie::{MemoryDB, PrefixedMemoryDB, prefixed_key, StateHasher, - Meta, MetaHasher}; +use sp_trie::{MemoryDB, PrefixedMemoryDB, prefixed_key}; use sp_database::Transaction; use sp_core::ChangesTrieConfiguration; use sp_core::offchain::OffchainOverlayedChange; @@ -889,25 +888,17 @@ struct StorageDb { } impl sp_state_machine::Storage> for StorageDb { - fn get( - &self, - key: &Block::Hash, - prefix: Prefix, - global: Option, - ) -> Result, String> { + fn get(&self, key: &Block::Hash, prefix: Prefix) -> Result, String> { if self.prefix_keys { let key = prefixed_key::>(key, prefix); self.state_db.get(&key, self) } else { self.state_db.get(key.as_ref(), self) } - .map(|result| result.map(|value| - , _>>::extract_value_owned(value, global) - )).map_err(|e| format!("Database backend error: {:?}", e)) + .map_err(|e| format!("Database backend error: {:?}", e)) } - fn access_from(&self, _key: &Block::Hash) { - } + fn access_from(&self, _key: &Block::Hash) { } } impl sc_state_db::NodeDb for StorageDb { @@ -931,14 +922,10 @@ impl DbGenesisStorage { } impl sp_state_machine::Storage> for DbGenesisStorage { - fn get( - &self, - _key: &Block::Hash, - _prefix: Prefix, - _global: Option, - ) -> Result, String> { + fn get(&self, _key: &Block::Hash, _prefix: Prefix) -> Result, String> { Ok(None) } + fn access_from(&self, _key: &Block::Hash) { } } @@ -2157,7 +2144,6 @@ impl sc_client_api::backend::Backend for Backend { self.storage.as_ref(), &header.state_root, (&[], None), - Default::default(), ).unwrap_or(None).is_some() }, _ => false, diff --git a/client/executor/src/integration_tests/mod.rs b/client/executor/src/integration_tests/mod.rs index 62767e28a31c3..f92e1b32c60c8 100644 --- a/client/executor/src/integration_tests/mod.rs +++ b/client/executor/src/integration_tests/mod.rs @@ -30,7 +30,7 @@ use sp_core::{ }; use sc_runtime_test::wasm_binary_unwrap; use sp_state_machine::TestExternalities as CoreTestExternalities; -use sp_trie::{TrieConfiguration, trie_types::Layout}; +use sp_trie::{TrieConfiguration, Layout}; use sp_wasm_interface::HostFunctions as _; use sp_runtime::traits::BlakeTwo256; use sc_executor_common::{wasm_runtime::WasmModule, runtime_blob::RuntimeBlob}; diff --git a/client/service/test/src/client/mod.rs b/client/service/test/src/client/mod.rs index 1fccd918be7c9..c045c767d4da2 100644 --- a/client/service/test/src/client/mod.rs +++ b/client/service/test/src/client/mod.rs @@ -51,7 +51,7 @@ use sp_consensus::{ BlockStatus, BlockImportParams, ForkChoiceStrategy, }; use sp_storage::StorageKey; -use sp_trie::{TrieConfiguration, trie_types::Layout}; +use sp_trie::{TrieConfiguration, Layout}; use sp_runtime::{generic::BlockId, DigestItem, Justifications}; use hex_literal::hex; use futures::executor::block_on; diff --git a/primitives/state-machine/src/trie_backend_essence.rs b/primitives/state-machine/src/trie_backend_essence.rs index e22f4d96c680e..dd7fe0f9b8d9f 100644 --- a/primitives/state-machine/src/trie_backend_essence.rs +++ b/primitives/state-machine/src/trie_backend_essence.rs @@ -48,6 +48,7 @@ pub trait Storage: Send + Sync { key: &H::Out, prefix: Prefix, ) -> Result>; + /// Call back when value get accessed in trie. fn access_from(&self, key: &H::Out); } diff --git a/primitives/transaction-storage-proof/src/lib.rs b/primitives/transaction-storage-proof/src/lib.rs index 7891edb0b2dc5..825de27b2a5a9 100644 --- a/primitives/transaction-storage-proof/src/lib.rs +++ b/primitives/transaction-storage-proof/src/lib.rs @@ -141,7 +141,7 @@ pub mod registration { use super::*; type Hasher = sp_core::Blake2Hasher; - type TrieLayout = sp_trie::trie_types::Layout::; + type TrieLayout = sp_trie::Layout::; /// Create a new inherent data provider instance for a given parent block hash. pub fn new_data_provider( diff --git a/primitives/trie/benches/bench.rs b/primitives/trie/benches/bench.rs index c40907ac5cdf4..c2ccb31328aae 100644 --- a/primitives/trie/benches/bench.rs +++ b/primitives/trie/benches/bench.rs @@ -21,11 +21,11 @@ criterion_main!(benches); fn benchmark(c: &mut Criterion) { trie_bench::standard_benchmark::< - sp_trie::trie_types::Layout, + sp_trie::Layout, sp_trie::TrieStream, >(c, "substrate-blake2"); trie_bench::standard_benchmark::< - sp_trie::trie_types::Layout, + sp_trie::Layout, sp_trie::TrieStream, >(c, "substrate-keccak"); } diff --git a/primitives/trie/src/lib.rs b/primitives/trie/src/lib.rs index 79068c585eb6e..714307e1347d5 100644 --- a/primitives/trie/src/lib.rs +++ b/primitives/trie/src/lib.rs @@ -161,7 +161,7 @@ pub type TrieHash = <::Hash as Hasher>::Out; /// This module is for non generic definition of trie type. /// Only the `Hasher` trait is generic in this case. pub mod trie_types { - use super::Layout; + pub type Layout = super::Layout; /// Persistent trie database read-access interface for the a given hasher. pub type TrieDB<'a, H> = super::TrieDB<'a, Layout>; /// Persistent trie database write-access interface for the a given hasher. From 602723e33fbedf6967f3d5ffdb16ab062c4559ed Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 21 Jun 2021 14:46:57 +0200 Subject: [PATCH 061/127] update to latest triedb changes. --- Cargo.lock | 16 ++++++++-------- primitives/trie/src/node_codec.rs | 12 ++++++------ 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4fb09ef6c6d2c..030a779c34e0d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2354,7 +2354,7 @@ dependencies = [ [[package]] name = "hash-db" version = "0.15.2" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#511e98bff38e2d618a5ec382eeca8661a5459df3" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#cec2aa252bd03779e57403610222fc818823fda4" [[package]] name = "hash256-std-hasher" @@ -2368,7 +2368,7 @@ dependencies = [ [[package]] name = "hash256-std-hasher" version = "0.15.2" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#511e98bff38e2d618a5ec382eeca8661a5459df3" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#cec2aa252bd03779e57403610222fc818823fda4" dependencies = [ "crunchy", ] @@ -3050,7 +3050,7 @@ dependencies = [ [[package]] name = "keccak-hasher" version = "0.15.3" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#511e98bff38e2d618a5ec382eeca8661a5459df3" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#cec2aa252bd03779e57403610222fc818823fda4" dependencies = [ "hash-db", "hash256-std-hasher 0.15.2 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple4)", @@ -3865,7 +3865,7 @@ dependencies = [ [[package]] name = "memory-db" version = "0.26.0" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#511e98bff38e2d618a5ec382eeca8661a5459df3" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#cec2aa252bd03779e57403610222fc818823fda4" dependencies = [ "hash-db", "hashbrown", @@ -10551,7 +10551,7 @@ checksum = "a7f741b240f1a48843f9b8e0444fb55fb2a4ff67293b50a9179dfd5ea67f8d41" [[package]] name = "trie-bench" version = "0.27.0" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#511e98bff38e2d618a5ec382eeca8661a5459df3" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#cec2aa252bd03779e57403610222fc818823fda4" dependencies = [ "criterion", "hash-db", @@ -10566,7 +10566,7 @@ dependencies = [ [[package]] name = "trie-db" version = "0.22.5" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#511e98bff38e2d618a5ec382eeca8661a5459df3" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#cec2aa252bd03779e57403610222fc818823fda4" dependencies = [ "hash-db", "hashbrown", @@ -10578,7 +10578,7 @@ dependencies = [ [[package]] name = "trie-root" version = "0.16.0" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#511e98bff38e2d618a5ec382eeca8661a5459df3" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#cec2aa252bd03779e57403610222fc818823fda4" dependencies = [ "hash-db", ] @@ -10596,7 +10596,7 @@ dependencies = [ [[package]] name = "trie-standardmap" version = "0.15.2" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#511e98bff38e2d618a5ec382eeca8661a5459df3" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#cec2aa252bd03779e57403610222fc818823fda4" dependencies = [ "hash-db", "keccak-hasher 0.15.3 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple4)", diff --git a/primitives/trie/src/node_codec.rs b/primitives/trie/src/node_codec.rs index bed8ae73b5852..58782e060f77d 100644 --- a/primitives/trie/src/node_codec.rs +++ b/primitives/trie/src/node_codec.rs @@ -123,7 +123,7 @@ impl NodeCodec { let bitmap = Bitmap::decode(&data[bitmap_range])?; let value = if branch_has_value { if alt_hashing && contains_hash { - ValuePlan::HashedValue(input.take(H::LENGTH)?, 0) + ValuePlan::HashedValue(input.take(H::LENGTH)?) } else { let with_len = input.offset; let count = >::decode(&mut input)?.0 as usize; @@ -165,7 +165,7 @@ impl NodeCodec { )?; let partial_padding = nibble_ops::number_padding(nibble_count); let value = if alt_hashing && contains_hash { - ValuePlan::HashedValue(input.take(H::LENGTH)?, 0) + ValuePlan::HashedValue(input.take(H::LENGTH)?) } else { let with_len = input.offset; let count = >::decode(&mut input)?.0 as usize; @@ -239,12 +239,12 @@ impl NodeCodecT for NodeCodec let end = output.len(); meta.encoded_value_callback(ValuePlan::Value(start..end, with_len)); }, - Value::HashedValue(hash, _size) => { + Value::HashedValue(hash) => { debug_assert!(hash.len() == H::LENGTH); let start = output.len(); output.extend_from_slice(hash); let end = output.len(); - meta.encoded_value_callback(ValuePlan::HashedValue(start..end, 0)); + meta.encoded_value_callback(ValuePlan::HashedValue(start..end)); }, Value::NoValue => unimplemented!("No support for incomplete nodes"), } @@ -307,12 +307,12 @@ impl NodeCodecT for NodeCodec let end = output.len(); meta.encoded_value_callback(ValuePlan::Value(start..end, with_len)); }, - Value::HashedValue(hash, _size) => { + Value::HashedValue(hash) => { debug_assert!(hash.len() == H::LENGTH); let start = output.len(); output.extend_from_slice(hash); let end = output.len(); - meta.encoded_value_callback(ValuePlan::HashedValue(start..end, 0)); + meta.encoded_value_callback(ValuePlan::HashedValue(start..end)); }, Value::NoValue => (), } From 66ee72db0895a7dc14e28383cfcaca243289372c Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 21 Jun 2021 15:22:50 +0200 Subject: [PATCH 062/127] Clean up. --- client/db/src/bench.rs | 11 +++-------- client/db/src/lib.rs | 3 +-- client/executor/runtime-test/src/lib.rs | 1 - client/light/src/backend.rs | 7 +++---- primitives/state-machine/src/backend.rs | 3 +-- primitives/state-machine/src/ext.rs | 7 ++----- primitives/state-machine/src/in_memory_backend.rs | 2 +- primitives/state-machine/src/lib.rs | 3 --- .../state-machine/src/overlayed_changes/mod.rs | 5 +---- primitives/state-machine/src/trie_backend.rs | 4 ++-- .../state-machine/src/trie_backend_essence.rs | 14 ++++---------- primitives/trie/src/storage_proof.rs | 7 +------ 12 files changed, 19 insertions(+), 48 deletions(-) diff --git a/client/db/src/bench.rs b/client/db/src/bench.rs index a3db2142ca986..c194e9672a1c0 100644 --- a/client/db/src/bench.rs +++ b/client/db/src/bench.rs @@ -156,8 +156,7 @@ impl BenchmarkingState { &child_content.child_info, child_content.data.iter().map(|(k, v)| (k.as_ref(), Some(v.as_ref()))), )); - let (root, transaction): (B::Hash, _) = state.state.borrow_mut().as_mut().unwrap() - .full_storage_root( + let (root, transaction): (B::Hash, _) = state.state.borrow_mut().as_mut().unwrap().full_storage_root( genesis.top.iter().map(|(k, v)| (k.as_ref(), Some(v.as_ref()))), child_delta, ); @@ -406,8 +405,7 @@ impl StateBackend> for BenchmarkingState { &self, delta: impl Iterator)>, ) -> (B::Hash, Self::Transaction) where B::Hash: Ord { - self.state.borrow().as_ref() - .map_or(Default::default(), |s| s.storage_root(delta)) + self.state.borrow().as_ref().map_or(Default::default(), |s| s.storage_root(delta)) } fn child_storage_root<'a>( @@ -415,10 +413,7 @@ impl StateBackend> for BenchmarkingState { child_info: &ChildInfo, delta: impl Iterator)>, ) -> (B::Hash, bool, Self::Transaction) where B::Hash: Ord { - self.state.borrow().as_ref().map_or( - Default::default(), - |s| s.child_storage_root(child_info, delta), - ) + self.state.borrow().as_ref().map_or(Default::default(), |s| s.child_storage_root(child_info, delta)) } fn pairs(&self) -> Vec<(Vec, Vec)> { diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index 89b02d3a0354b..64cfb975f79b9 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -73,8 +73,7 @@ use sp_core::storage::{well_known_keys, ChildInfo}; use sp_arithmetic::traits::Saturating; use sp_runtime::{generic::{DigestItem, BlockId}, Justification, Justifications, Storage}; use sp_runtime::traits::{ - Block as BlockT, Header as HeaderT, NumberFor, Zero, One, SaturatedConversion, - HashFor, + Block as BlockT, Header as HeaderT, NumberFor, Zero, One, SaturatedConversion, HashFor, }; use sp_state_machine::{ DBValue, ChangesTrieTransaction, ChangesTrieCacheAction, UsageInfo as StateUsageInfo, diff --git a/client/executor/runtime-test/src/lib.rs b/client/executor/runtime-test/src/lib.rs index c8963b7b87baa..926fddcf63073 100644 --- a/client/executor/runtime-test/src/lib.rs +++ b/client/executor/runtime-test/src/lib.rs @@ -79,7 +79,6 @@ sp_core::wasm_export_functions! { print("switched!"); } - fn test_clear_prefix(input: Vec) -> Vec { storage::clear_prefix(&input, None); b"all ok!".to_vec() diff --git a/client/light/src/backend.rs b/client/light/src/backend.rs index a63c7879de289..a7f1b8e0c1696 100644 --- a/client/light/src/backend.rs +++ b/client/light/src/backend.rs @@ -492,8 +492,8 @@ impl StateBackend for GenesisOrUnavailableState delta: impl Iterator)>, ) -> (H::Out, Self::Transaction) where H::Out: Ord { match *self { - GenesisOrUnavailableState::Genesis(ref state) => state - .storage_root(delta), + GenesisOrUnavailableState::Genesis(ref state) => + state.storage_root(delta), GenesisOrUnavailableState::Unavailable => Default::default(), } } @@ -505,8 +505,7 @@ impl StateBackend for GenesisOrUnavailableState ) -> (H::Out, bool, Self::Transaction) where H::Out: Ord { match *self { GenesisOrUnavailableState::Genesis(ref state) => { - let (root, is_equal, _) = state - .child_storage_root(child_info, delta); + let (root, is_equal, _) = state.child_storage_root(child_info, delta); (root, is_equal, Default::default()) }, GenesisOrUnavailableState::Unavailable => diff --git a/primitives/state-machine/src/backend.rs b/primitives/state-machine/src/backend.rs index c879053fce569..fb59458c0f9c1 100644 --- a/primitives/state-machine/src/backend.rs +++ b/primitives/state-machine/src/backend.rs @@ -169,7 +169,6 @@ pub trait Backend: sp_std::fmt::Debug { /// Calculate the storage root, with given delta over what is already stored /// in the backend, and produce a "transaction" that can be used to commit. /// Does include child storage updates. - /// Alt hashing paremeter must contain possible changes from delta. fn full_storage_root<'a>( &self, delta: impl Iterator)>, @@ -257,7 +256,7 @@ pub trait Backend: sp_std::fmt::Debug { } /// Read current trie hashing threshold. - /// Please do not reimplement. + /// Please do not change default implementation when implementing this trait. fn get_trie_alt_hashing_threshold(&self) -> Option { self.storage(sp_core::storage::well_known_keys::TRIE_HASHING_CONFIG).ok().flatten() .and_then(|encoded| sp_core::storage::trie_threshold_decode(&mut encoded.as_slice())) diff --git a/primitives/state-machine/src/ext.rs b/primitives/state-machine/src/ext.rs index 2defaa49142af..e66664647d9d8 100644 --- a/primitives/state-machine/src/ext.rs +++ b/primitives/state-machine/src/ext.rs @@ -26,7 +26,7 @@ use sp_core::{ storage::{well_known_keys::is_child_storage_key, ChildInfo, TrackedStorageKey}, hexdisplay::HexDisplay, }; -use sp_trie::{Layout, empty_child_trie_root}; +use sp_trie::{trie_types::Layout, empty_child_trie_root}; use sp_externalities::{ Externalities, Extensions, Extension, ExtensionStore, }; @@ -530,10 +530,7 @@ where return root.encode(); } - let root = self.overlay.storage_root( - self.backend, - self.storage_transaction_cache, - ); + let root = self.overlay.storage_root(self.backend, self.storage_transaction_cache); trace!(target: "state", "{:04x}: Root {}", self.id, HexDisplay::from(&root.as_ref())); root.encode() } diff --git a/primitives/state-machine/src/in_memory_backend.rs b/primitives/state-machine/src/in_memory_backend.rs index b122120b36e44..74338d5d77a52 100644 --- a/primitives/state-machine/src/in_memory_backend.rs +++ b/primitives/state-machine/src/in_memory_backend.rs @@ -88,7 +88,7 @@ where self.root() == other.root() } - /// setting a alt hashing threshold at start. + /// Setting a alternate hashing threshold at start. pub fn force_alt_hashing(&mut self, threshold: Option) { self.force_alt_hashing = Some(threshold); } diff --git a/primitives/state-machine/src/lib.rs b/primitives/state-machine/src/lib.rs index 34363dfa6fe3a..278126c7432e8 100644 --- a/primitives/state-machine/src/lib.rs +++ b/primitives/state-machine/src/lib.rs @@ -1561,8 +1561,6 @@ mod tests { local_result1.into_iter().collect::>(), vec![(b"foo222".to_vec(), Some(vec![5u8; 100]))], ); - println!("a{:?}", remote_proof.encode().len()); - println!("b{:?}", remote_proof.encoded_size()); remote_proof }; @@ -1651,7 +1649,6 @@ mod tests { ); remote_backend.backend_storage_mut().consolidate(transaction); remote_backend.essence.set_root(remote_root.clone()); - println!("{:?}", remote_root); let remote_proof = prove_child_read( remote_backend, &child_info1, diff --git a/primitives/state-machine/src/overlayed_changes/mod.rs b/primitives/state-machine/src/overlayed_changes/mod.rs index 1bca2da310c27..c01d56ab919a0 100644 --- a/primitives/state-machine/src/overlayed_changes/mod.rs +++ b/primitives/state-machine/src/overlayed_changes/mod.rs @@ -636,10 +636,7 @@ impl OverlayedChanges { |(k, v)| (&k[..], v.value().map(|v| &v[..])) ))); - let (root, transaction) = backend.full_storage_root( - delta, - child_delta, - ); + let (root, transaction) = backend.full_storage_root(delta, child_delta); cache.transaction = Some(transaction); cache.transaction_storage_root = Some(root); diff --git a/primitives/state-machine/src/trie_backend.rs b/primitives/state-machine/src/trie_backend.rs index c09a6e6e0899b..334f80f0dcf25 100644 --- a/primitives/state-machine/src/trie_backend.rs +++ b/primitives/state-machine/src/trie_backend.rs @@ -34,8 +34,8 @@ use sp_std::{boxed::Box, vec::Vec}; pub struct TrieBackend, H: Hasher> { pub (crate) essence: TrieBackendEssence, // Allows setting alt hashing at start for testing only - // (see in_memory_backend that cannot read from state as - // it changes. + // (mainly for in_memory_backend when it cannot read it from + // state). pub (crate) force_alt_hashing: Option>, } diff --git a/primitives/state-machine/src/trie_backend_essence.rs b/primitives/state-machine/src/trie_backend_essence.rs index dd7fe0f9b8d9f..f8945a88b5ec0 100644 --- a/primitives/state-machine/src/trie_backend_essence.rs +++ b/primitives/state-machine/src/trie_backend_essence.rs @@ -43,11 +43,7 @@ type Result = sp_std::result::Result; /// Patricia trie-based storage trait. pub trait Storage: Send + Sync { /// Get a trie node. - fn get( - &self, - key: &H::Out, - prefix: Prefix, - ) -> Result>; + fn get(&self, key: &H::Out, prefix: Prefix) -> Result>; /// Call back when value get accessed in trie. fn access_from(&self, key: &H::Out); @@ -416,8 +412,10 @@ impl<'a, S: 'a + TrieBackendStorage, H: Hasher> HashDBRef pub trait TrieBackendStorage: Send + Sync { /// Type of in-memory overlay. type Overlay: HashDB + Default + Consolidate; + /// Get the value stored at key. fn get(&self, key: &H::Out, prefix: Prefix) -> Result>; + /// Call back when value get accessed in trie. fn access_from(&self, key: &H::Out); } @@ -427,11 +425,7 @@ pub trait TrieBackendStorage: Send + Sync { impl TrieBackendStorage for Arc> { type Overlay = PrefixedMemoryDB; - fn get( - &self, - key: &H::Out, - prefix: Prefix, - ) -> Result> { + fn get(&self, key: &H::Out, prefix: Prefix) -> Result> { Storage::::get(self.deref(), key, prefix) } diff --git a/primitives/trie/src/storage_proof.rs b/primitives/trie/src/storage_proof.rs index cebad1911315f..78730ca13b7cc 100644 --- a/primitives/trie/src/storage_proof.rs +++ b/primitives/trie/src/storage_proof.rs @@ -174,14 +174,9 @@ impl Iterator for StorageProofNodeIterator { impl From for crate::MemoryDB { fn from(proof: StorageProof) -> Self { let mut db = crate::MemoryDB::default(); - // Needed because we do not read trie structure, so - // we do a heuristic related to the fact that host function - // only allow global definition. - // Using compact proof will work directly here (read trie structure and - // work directly. for item in proof.trie_nodes.iter() { let mut meta = Default::default(); - // read state meta (required for value layout). + // Read meta from state (required for value layout). let _ = as TrieLayout>::Codec::decode_plan(item.as_slice(), &mut meta); db.alt_insert( crate::EMPTY_PREFIX, From 23c5db3e5e1e4f72d897bf92d4a16e998bea393a Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 21 Jun 2021 18:24:04 +0200 Subject: [PATCH 063/127] fix executor test. --- client/executor/src/integration_tests/mod.rs | 18 ++++------ primitives/state-machine/src/testing.rs | 37 ++++++++++++++++++-- 2 files changed, 40 insertions(+), 15 deletions(-) diff --git a/client/executor/src/integration_tests/mod.rs b/client/executor/src/integration_tests/mod.rs index f92e1b32c60c8..622fac865683d 100644 --- a/client/executor/src/integration_tests/mod.rs +++ b/client/executor/src/integration_tests/mod.rs @@ -199,7 +199,7 @@ fn storage_should_work(wasm_method: WasmExecutionMethod) { let output = call_in_wasm( "test_data_in", - &b"Hello world".to_vec().encode(), + &b"Hello worldHello worldHello worldHello world".to_vec().encode(), wasm_method, &mut ext, ).unwrap(); @@ -207,18 +207,15 @@ fn storage_should_work(wasm_method: WasmExecutionMethod) { assert_eq!(output, b"all ok!".to_vec().encode()); } - let mut storage = sp_core::storage::Storage { + let storage = sp_core::storage::Storage { top: map![ - b"input".to_vec() => b"Hello world".to_vec(), + b"input".to_vec() => b"Hello worldHello worldHello worldHello world".to_vec(), b"foo".to_vec() => b"bar".to_vec(), b"baz".to_vec() => b"bar".to_vec() ], children_default: map![], }; - storage.modify_trie_alt_hashing_threshold(Some( - sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD - )); - let expected = TestExternalities::new(storage); + let expected = TestExternalities::new_with_alt_hashing(storage); assert_eq!(ext, expected); } @@ -244,7 +241,7 @@ fn clear_prefix_should_work(wasm_method: WasmExecutionMethod) { assert_eq!(output, b"all ok!".to_vec().encode()); } - let mut storage = sp_core::storage::Storage { + let storage = sp_core::storage::Storage { top: map![ b"aaa".to_vec() => b"1".to_vec(), b"aab".to_vec() => b"2".to_vec(), @@ -252,11 +249,8 @@ fn clear_prefix_should_work(wasm_method: WasmExecutionMethod) { ], children_default: map![], }; - storage.modify_trie_alt_hashing_threshold(Some( - sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD - )); - let expected = TestExternalities::new(storage); + let expected = TestExternalities::new_with_alt_hashing(storage); assert_eq!(expected, ext); } diff --git a/primitives/state-machine/src/testing.rs b/primitives/state-machine/src/testing.rs index e7f711eced39c..a8dccb7274571 100644 --- a/primitives/state-machine/src/testing.rs +++ b/primitives/state-machine/src/testing.rs @@ -18,6 +18,7 @@ //! Test implementation for Externalities. use std::{any::{Any, TypeId}, panic::{AssertUnwindSafe, UnwindSafe}}; +use std::collections::{HashMap, BTreeMap}; use crate::{ backend::Backend, OverlayedChanges, StorageTransactionCache, ext::Ext, InMemoryBackend, @@ -36,7 +37,7 @@ use sp_core::{ offchain::testing::TestPersistentOffchainDB, storage::{ well_known_keys::{CHANGES_TRIE_CONFIG, CODE, is_child_storage_key}, - Storage, + Storage, ChildInfo, }, traits::TaskExecutorExt, testing::TaskExecutor, @@ -88,13 +89,24 @@ where Self::new_with_code(&[], storage) } + /// Create a new instance of `TestExternalities` with storage + /// on a backend containing defined default alt hashing threshold. + pub fn new_with_alt_hashing(storage: Storage) -> Self { + Self::new_with_code_inner(&[], storage, true) + } + + /// New empty test externalities. pub fn new_empty() -> Self { Self::new_with_code(&[], Storage::default()) } /// Create a new instance of `TestExternalities` with code and storage. - pub fn new_with_code(code: &[u8], mut storage: Storage) -> Self { + pub fn new_with_code(code: &[u8], storage: Storage) -> Self { + Self::new_with_code_inner(code, storage, false) + } + + fn new_with_code_inner(code: &[u8], mut storage: Storage, force_alt_hashing: bool) -> Self { let mut overlay = OverlayedChanges::default(); let changes_trie_config = storage.top.get(CHANGES_TRIE_CONFIG) .and_then(|v| Decode::decode(&mut &v[..]).ok()); @@ -110,13 +122,32 @@ where let offchain_db = TestPersistentOffchainDB::new(); + let backend = if force_alt_hashing { + let mut backend: InMemoryBackend = { + let mut storage = Storage::default(); + storage.modify_trie_alt_hashing_threshold(Some( + sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD + )); + storage.into() + }; + let mut inner: HashMap, BTreeMap> + = storage.children_default.into_iter().map(|(_k, c)| (Some(c.child_info), c.data)).collect(); + inner.insert(None, storage.top); + backend.insert( + inner.into_iter().map(|(k, m)| (k, m.into_iter().map(|(k, v)| (k, Some(v))).collect())), + ); + backend + } else { + storage.into() + }; + TestExternalities { overlay, offchain_db, changes_trie_config, extensions, changes_trie_storage: ChangesTrieInMemoryStorage::new(), - backend: storage.into(), + backend, storage_transaction_cache: Default::default(), } } From 03d93b2f5aa9495119706072e51ea992a10da6a7 Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 22 Jul 2021 13:13:15 +0200 Subject: [PATCH 064/127] rust fmt from master. --- rustfmt.toml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 rustfmt.toml diff --git a/rustfmt.toml b/rustfmt.toml new file mode 100644 index 0000000000000..15e9bdcdf10f1 --- /dev/null +++ b/rustfmt.toml @@ -0,0 +1,20 @@ +# Basic +hard_tabs = true +max_width = 100 +use_small_heuristics = "Max" +# Imports +imports_granularity = "Crate" +reorder_imports = true +# Consistency +newline_style = "Unix" +# Misc +chain_width = 80 +spaces_around_ranges = false +binop_separator = "Back" +reorder_impl_items = false +match_arm_leading_pipes = "Preserve" +match_arm_blocks = false +match_block_trailing_comma = true +trailing_comma = "Vertical" +trailing_semicolon = false +use_field_init_shorthand = true From 93aaa4cdea3a2b8b219b9ef0b843ba6a70b4916c Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 22 Jul 2021 17:35:46 +0200 Subject: [PATCH 065/127] rust format. --- bin/node/bench/src/generator.rs | 7 +- bin/node/bench/src/trie.rs | 3 +- bin/node/executor/benches/bench.rs | 6 +- bin/node/executor/tests/common.rs | 8 +- client/api/src/cht.rs | 16 +- client/db/src/lib.rs | 93 ++++---- client/executor/src/integration_tests/mod.rs | 49 ++--- client/network/test/src/lib.rs | 6 +- client/service/test/src/client/light.rs | 90 +++++--- client/service/test/src/client/mod.rs | 31 +-- frame/support/test/tests/instance.rs | 5 +- frame/system/src/lib.rs | 6 +- primitives/io/src/lib.rs | 10 +- primitives/state-machine/src/backend.rs | 26 +-- primitives/state-machine/src/basic.rs | 2 +- .../state-machine/src/changes_trie/mod.rs | 2 +- .../state-machine/src/changes_trie/storage.rs | 4 +- primitives/state-machine/src/lib.rs | 131 +++++------ .../state-machine/src/proving_backend.rs | 154 ++++++------- primitives/state-machine/src/testing.rs | 24 +- primitives/state-machine/src/trie_backend.rs | 59 ++--- .../state-machine/src/trie_backend_essence.rs | 91 +++++--- primitives/storage/src/lib.rs | 3 +- primitives/trie/src/lib.rs | 207 +++++++++--------- primitives/trie/src/node_codec.rs | 139 ++++++------ primitives/trie/src/node_header.rs | 47 ++-- primitives/trie/src/storage_proof.rs | 6 +- primitives/trie/src/trie_codec.rs | 14 +- primitives/trie/src/trie_stream.rs | 54 ++--- test-utils/client/src/lib.rs | 6 +- test-utils/runtime/client/src/lib.rs | 9 +- utils/frame/rpc/system/src/lib.rs | 2 +- 32 files changed, 661 insertions(+), 649 deletions(-) diff --git a/bin/node/bench/src/generator.rs b/bin/node/bench/src/generator.rs index f349c4555a5ea..f95811c40ebee 100644 --- a/bin/node/bench/src/generator.rs +++ b/bin/node/bench/src/generator.rs @@ -30,7 +30,7 @@ use crate::simple_trie::SimpleTrie; /// return root. pub fn generate_trie( db: Arc, - key_values: impl IntoIterator, Vec)>, + key_values: impl IntoIterator, Vec)>, alt_hashing: Option, ) -> Hash { let mut root = Hash::default(); @@ -44,10 +44,11 @@ pub fn generate_trie( ); let mut trie = SimpleTrie { db, overlay: &mut overlay }; { - let mut trie_db = if let Some(threshold) = alt_hashing { let layout = sp_trie::Layout::with_alt_hashing(threshold); - TrieDBMut::::new_with_layout(&mut trie, &mut root, layout) + TrieDBMut::::new_with_layout( + &mut trie, &mut root, layout, + ) } else { TrieDBMut::new(&mut trie, &mut root) }; diff --git a/bin/node/bench/src/trie.rs b/bin/node/bench/src/trie.rs index ba8a42900bd2c..77ca3e85b8b05 100644 --- a/bin/node/bench/src/trie.rs +++ b/bin/node/bench/src/trie.rs @@ -176,8 +176,7 @@ impl sp_state_machine::Storage for Storage { self.0.get(0, &key).map_err(|e| format!("Database backend error: {:?}", e)) } - fn access_from(&self, _key: &Hash) { - } + fn access_from(&self, _key: &Hash) {} } impl core::Benchmark for TrieReadBenchmark { diff --git a/bin/node/executor/benches/bench.rs b/bin/node/executor/benches/bench.rs index 7645c43d25fc1..7539d14b31a4e 100644 --- a/bin/node/executor/benches/bench.rs +++ b/bin/node/executor/benches/bench.rs @@ -89,9 +89,9 @@ fn construct_block( let extrinsics = extrinsics.into_iter().map(sign).collect::>(); // calculate the header fields that we can. - let extrinsics_root = Layout::::default().ordered_trie_root( - extrinsics.iter().map(Encode::encode) - ).to_fixed_bytes() + let extrinsics_root = Layout::::default() + .ordered_trie_root(extrinsics.iter().map(Encode::encode)) + .to_fixed_bytes() .into(); let header = Header { diff --git a/bin/node/executor/tests/common.rs b/bin/node/executor/tests/common.rs index 08476cc428f65..853639bd50325 100644 --- a/bin/node/executor/tests/common.rs +++ b/bin/node/executor/tests/common.rs @@ -152,10 +152,10 @@ pub fn construct_block( let extrinsics = extrinsics.into_iter().map(sign).collect::>(); // calculate the header fields that we can. - let extrinsics_root = - Layout::::default().ordered_trie_root(extrinsics.iter().map(Encode::encode)) - .to_fixed_bytes() - .into(); + let extrinsics_root = Layout::::default() + .ordered_trie_root(extrinsics.iter().map(Encode::encode)) + .to_fixed_bytes() + .into(); let header = Header { parent_hash, diff --git a/client/api/src/cht.rs b/client/api/src/cht.rs index 75e45a50af1c0..996374314d3cb 100644 --- a/client/api/src/cht.rs +++ b/client/api/src/cht.rs @@ -32,9 +32,9 @@ use sp_trie; use sp_core::{convert_hash, H256}; use sp_runtime::traits::{AtLeast32Bit, Header as HeaderT, One, Zero}; use sp_state_machine::{ - MemoryDB, TrieBackend, Backend as StateBackend, StorageProof, InMemoryBackend, prove_read_on_trie_backend, read_proof_check, read_proof_check_on_proving_backend_generic as read_proof_check_on_proving_backend, + Backend as StateBackend, InMemoryBackend, MemoryDB, StorageProof, TrieBackend, }; use sp_blockchain::{Error as ClientError, Result as ClientResult}; @@ -94,9 +94,8 @@ where I: IntoIterator>>, { use sp_trie::TrieConfiguration; - Ok(sp_trie::Layout::::default().trie_root( - build_pairs::(cht_size, cht_num, hashes)? - )) + Ok(sp_trie::Layout::::default() + .trie_root(build_pairs::(cht_size, cht_num, hashes)?)) } /// Build CHT-based header proof. @@ -172,11 +171,10 @@ where local_root, local_number, remote_hash, - |_, local_cht_key| - read_proof_check_on_proving_backend::( - proving_backend, - local_cht_key, - ).map_err(ClientError::from_state), + |_, local_cht_key| { + read_proof_check_on_proving_backend::(proving_backend, local_cht_key) + .map_err(ClientError::from_state) + }, ) } diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index 2f2a7c35c4360..170862595bf03 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -882,18 +882,17 @@ impl sc_client_api::backend::BlockImportOperation Ok(()) } - fn reset_storage( - &mut self, - storage: Storage, - ) -> ClientResult { + fn reset_storage(&mut self, storage: Storage) -> ClientResult { if storage.top.keys().any(|k| well_known_keys::is_child_storage_key(&k)) { - return Err(sp_blockchain::Error::GenesisInvalid.into()); + return Err(sp_blockchain::Error::GenesisInvalid.into()) } - let child_delta = storage.children_default.iter().map(|(_storage_key, child_content)|( - &child_content.child_info, - child_content.data.iter().map(|(k, v)| (&k[..], Some(&v[..]))), - )); + let child_delta = storage.children_default.iter().map(|(_storage_key, child_content)| { + ( + &child_content.child_info, + child_content.data.iter().map(|(k, v)| (&k[..], Some(&v[..]))), + ) + }); let mut changes_trie_config: Option = None; let (root, transaction) = self.old_state.full_storage_root( @@ -901,7 +900,7 @@ impl sc_client_api::backend::BlockImportOperation if &k[..] == well_known_keys::CHANGES_TRIE_CONFIG { changes_trie_config = Some( Decode::decode(&mut &v[..]) - .expect("changes trie configuration is encoded properly at genesis") + .expect("changes trie configuration is encoded properly at genesis"), ); } (&k[..], Some(&v[..])) @@ -994,7 +993,7 @@ impl sp_state_machine::Storage> for StorageDb sc_state_db::NodeDb for StorageDb { @@ -1022,7 +1021,7 @@ impl sp_state_machine::Storage> for DbGenesisStora use hash_db::HashDB; Ok(self.storage.get(key, prefix)) } - fn access_from(&self, _key: &Block::Hash) { } + fn access_from(&self, _key: &Block::Hash) {} } struct EmptyStorage(pub Block::Hash); @@ -1041,8 +1040,7 @@ impl sp_state_machine::Storage> for EmptyStorage = vec![]; - header.state_root = op.old_state.storage_root(storage - .iter() - .cloned() - .map(|(x, y)| (x, Some(y))), - ).0.into(); + header.state_root = op + .old_state + .storage_root(storage.iter().cloned().map(|(x, y)| (x, Some(y)))) + .0 + .into(); let hash = header.hash(); op.db_updates.insert(EMPTY_PREFIX, b"hello"); @@ -2739,11 +2734,11 @@ pub(crate) mod tests { let storage: Vec<(_, _)> = vec![]; - header.state_root = op.old_state.storage_root(storage - .iter() - .cloned() - .map(|(x, y)| (x, Some(y))), - ).0.into(); + header.state_root = op + .old_state + .storage_root(storage.iter().cloned().map(|(x, y)| (x, Some(y)))) + .0 + .into(); let hash = header.hash(); op.db_updates.remove(&key, EMPTY_PREFIX); @@ -2773,11 +2768,11 @@ pub(crate) mod tests { let storage: Vec<(_, _)> = vec![]; - header.state_root = op.old_state.storage_root(storage - .iter() - .cloned() - .map(|(x, y)| (x, Some(y))), - ).0.into(); + header.state_root = op + .old_state + .storage_root(storage.iter().cloned().map(|(x, y)| (x, Some(y)))) + .0 + .into(); op.set_block_data(header, Some(vec![]), None, None, NewBlockState::Best) .unwrap(); @@ -3127,10 +3122,11 @@ pub(crate) mod tests { let storage = vec![(b"test".to_vec(), b"test".to_vec())]; - header.state_root = op.old_state.storage_root(storage - .iter() - .map(|(x, y)| (&x[..], Some(&y[..]))), - ).0.into(); + header.state_root = op + .old_state + .storage_root(storage.iter().map(|(x, y)| (&x[..], Some(&y[..])))) + .0 + .into(); let hash = header.hash(); op.reset_storage(Storage { @@ -3165,10 +3161,9 @@ pub(crate) mod tests { let storage = vec![(b"test".to_vec(), Some(b"test2".to_vec()))]; - let (root, overlay) = op.old_state.storage_root( - storage.iter() - .map(|(k, v)| (&k[..], v.as_ref().map(|v| &v[..]))), - ); + let (root, overlay) = op + .old_state + .storage_root(storage.iter().map(|(k, v)| (&k[..], v.as_ref().map(|v| &v[..])))); op.update_db_storage(overlay).unwrap(); header.state_root = root.into(); let hash = header.hash(); diff --git a/client/executor/src/integration_tests/mod.rs b/client/executor/src/integration_tests/mod.rs index 77eb5fd9e2704..357e3ea972a37 100644 --- a/client/executor/src/integration_tests/mod.rs +++ b/client/executor/src/integration_tests/mod.rs @@ -184,7 +184,8 @@ fn storage_should_work(wasm_method: WasmExecutionMethod) { &b"Hello worldHello worldHello worldHello world".to_vec().encode(), wasm_method, &mut ext, - ).unwrap(); + ) + .unwrap(); assert_eq!(output, b"all ok!".to_vec().encode()); } @@ -373,13 +374,11 @@ fn ordered_trie_root_should_work(wasm_method: WasmExecutionMethod) { let mut ext = TestExternalities::default(); let trie_input = vec![b"zero".to_vec(), b"one".to_vec(), b"two".to_vec()]; assert_eq!( - call_in_wasm( - "test_ordered_trie_root", - &[0], - wasm_method, - &mut ext.ext(), - ).unwrap(), - Layout::::default().ordered_trie_root(trie_input.iter()).as_bytes().encode(), + call_in_wasm("test_ordered_trie_root", &[0], wasm_method, &mut ext.ext(),).unwrap(), + Layout::::default() + .ordered_trie_root(trie_input.iter()) + .as_bytes() + .encode(), ); } @@ -672,12 +671,8 @@ fn state_hashing_update(wasm_method: WasmExecutionMethod) { let root1 = { let mut ext = ext.ext(); ext.set_storage(b"foo".to_vec(), b"bar".to_vec()); - let output = call_in_wasm( - "test_data_in", - &vec![1u8; 100].encode(), - wasm_method, - &mut ext, - ).unwrap(); + let output = + call_in_wasm("test_data_in", &vec![1u8; 100].encode(), wasm_method, &mut ext).unwrap(); assert_eq!(output, b"all ok!".to_vec().encode()); ext.storage_root() @@ -687,12 +682,8 @@ fn state_hashing_update(wasm_method: WasmExecutionMethod) { let root2 = { let mut ext = ext.ext(); // flag state. - let _ = call_in_wasm( - "test_switch_state", - Default::default(), - wasm_method, - &mut ext, - ).unwrap(); + let _ = + call_in_wasm("test_switch_state", Default::default(), wasm_method, &mut ext).unwrap(); ext.storage_root() }; @@ -701,12 +692,9 @@ fn state_hashing_update(wasm_method: WasmExecutionMethod) { ext.commit_all().unwrap(); let root3 = { let mut ext = ext.ext(); - let _ = call_in_wasm( - "test_data_in", - &vec![2u8; 100].to_vec().encode(), - wasm_method, - &mut ext, - ).unwrap(); + let _ = + call_in_wasm("test_data_in", &vec![2u8; 100].to_vec().encode(), wasm_method, &mut ext) + .unwrap(); ext.storage_root() }; assert!(root2 != root3); @@ -716,12 +704,9 @@ fn state_hashing_update(wasm_method: WasmExecutionMethod) { let mut ext = ext.ext(); // revert to root 2 state, but this time // inner hashing should apply - let _ = call_in_wasm( - "test_data_in", - &vec![1u8; 100].to_vec().encode(), - wasm_method, - &mut ext, - ).unwrap(); + let _ = + call_in_wasm("test_data_in", &vec![1u8; 100].to_vec().encode(), wasm_method, &mut ext) + .unwrap(); ext.storage_root() }; assert!(root2 != root3); diff --git a/client/network/test/src/lib.rs b/client/network/test/src/lib.rs index dc1b8d8dd4687..00293405c5aae 100644 --- a/client/network/test/src/lib.rs +++ b/client/network/test/src/lib.rs @@ -747,9 +747,11 @@ where /// Add a full peer. fn add_full_peer_with_config(&mut self, config: FullPeerConfig) { let mut test_client_builder = match (config.keep_blocks, config.storage_chain) { - (Some(keep_blocks), true) => TestClientBuilder::with_tx_storage(keep_blocks).state_hashed_value(), + (Some(keep_blocks), true) => + TestClientBuilder::with_tx_storage(keep_blocks).state_hashed_value(), (None, true) => TestClientBuilder::with_tx_storage(u32::MAX).state_hashed_value(), - (Some(keep_blocks), false) => TestClientBuilder::with_pruning_window(keep_blocks).state_hashed_value(), + (Some(keep_blocks), false) => + TestClientBuilder::with_pruning_window(keep_blocks).state_hashed_value(), (None, false) => TestClientBuilder::with_default_backend().state_hashed_value(), }; if matches!(config.sync_mode, SyncMode::Fast { .. }) { diff --git a/client/service/test/src/client/light.rs b/client/service/test/src/client/light.rs index 0c567b61db0a5..7a50f32d3ac24 100644 --- a/client/service/test/src/client/light.rs +++ b/client/service/test/src/client/light.rs @@ -467,8 +467,12 @@ fn prepare_for_read_proof_check(hashed_value: bool) -> (TestChecker, Header, Sto let remote_block_id = BlockId::Number(0); let remote_block_hash = remote_client.block_hash(0).unwrap().unwrap(); let mut remote_block_header = remote_client.header(&remote_block_id).unwrap().unwrap(); - remote_block_header.state_root = remote_client.state_at(&remote_block_id).unwrap() - .storage_root(std::iter::empty()).0.into(); + remote_block_header.state_root = remote_client + .state_at(&remote_block_id) + .unwrap() + .storage_root(std::iter::empty()) + .0 + .into(); // 'fetch' read proof from remote node let heap_pages = remote_client @@ -504,8 +508,12 @@ fn prepare_for_read_child_proof_check() -> (TestChecker, Header, StorageProof, V let remote_block_id = BlockId::Number(0); let remote_block_hash = remote_client.block_hash(0).unwrap().unwrap(); let mut remote_block_header = remote_client.header(&remote_block_id).unwrap().unwrap(); - remote_block_header.state_root = remote_client.state_at(&remote_block_id).unwrap() - .storage_root(std::iter::empty()).0.into(); + remote_block_header.state_root = remote_client + .state_at(&remote_block_id) + .unwrap() + .storage_root(std::iter::empty()) + .0 + .into(); // 'fetch' child read proof from remote node let child_value = remote_client @@ -531,7 +539,10 @@ fn prepare_for_read_child_proof_check() -> (TestChecker, Header, StorageProof, V (local_checker, remote_block_header, remote_read_proof, child_value) } -fn prepare_for_header_proof_check(insert_cht: bool, hashed_value: bool) -> (TestChecker, Hash, Header, StorageProof) { +fn prepare_for_header_proof_check( + insert_cht: bool, + hashed_value: bool, +) -> (TestChecker, Hash, Header, StorageProof) { // prepare remote client let mut remote_client = substrate_test_runtime_client::new(hashed_value); let mut local_headers_hashes = Vec::new(); @@ -580,18 +591,25 @@ fn storage_read_proof_is_generated_and_checked() { storage_read_proof_is_generated_and_checked_inner(false); } fn storage_read_proof_is_generated_and_checked_inner(hashed_value: bool) { - let ( - local_checker, - remote_block_header, - remote_read_proof, - heap_pages, - ) = prepare_for_read_proof_check(hashed_value); - assert_eq!((&local_checker as &dyn FetchChecker).check_read_proof(&RemoteReadRequest::
{ - block: remote_block_header.hash(), - header: remote_block_header, - keys: vec![well_known_keys::HEAP_PAGES.to_vec()], - retry_count: None, - }, remote_read_proof).unwrap().remove(well_known_keys::HEAP_PAGES).unwrap().unwrap()[0], heap_pages as u8); + let (local_checker, remote_block_header, remote_read_proof, heap_pages) = + prepare_for_read_proof_check(hashed_value); + assert_eq!( + (&local_checker as &dyn FetchChecker) + .check_read_proof( + &RemoteReadRequest::
{ + block: remote_block_header.hash(), + header: remote_block_header, + keys: vec![well_known_keys::HEAP_PAGES.to_vec()], + retry_count: None, + }, + remote_read_proof + ) + .unwrap() + .remove(well_known_keys::HEAP_PAGES) + .unwrap() + .unwrap()[0], + heap_pages as u8 + ); } #[test] @@ -625,22 +643,28 @@ fn header_proof_is_generated_and_checked() { header_proof_is_generated_and_checked_inner(false); } fn header_proof_is_generated_and_checked_inner(hashed: bool) { - let ( - local_checker, - local_cht_root, - remote_block_header, - remote_header_proof, - ) = prepare_for_header_proof_check(true, hashed); - assert_eq!((&local_checker as &dyn FetchChecker).check_header_proof(&RemoteHeaderRequest::
{ - cht_root: local_cht_root, - block: 1, - retry_count: None, - }, Some(remote_block_header.clone()), remote_header_proof).unwrap(), remote_block_header); + let (local_checker, local_cht_root, remote_block_header, remote_header_proof) = + prepare_for_header_proof_check(true, hashed); + assert_eq!( + (&local_checker as &dyn FetchChecker) + .check_header_proof( + &RemoteHeaderRequest::
{ + cht_root: local_cht_root, + block: 1, + retry_count: None, + }, + Some(remote_block_header.clone()), + remote_header_proof + ) + .unwrap(), + remote_block_header + ); } #[test] fn check_header_proof_fails_if_cht_root_is_invalid() { - let (local_checker, _, mut remote_block_header, remote_header_proof) = prepare_for_header_proof_check(true, true); + let (local_checker, _, mut remote_block_header, remote_header_proof) = + prepare_for_header_proof_check(true, true); remote_block_header.number = 100; assert!((&local_checker as &dyn FetchChecker) .check_header_proof( @@ -657,12 +681,8 @@ fn check_header_proof_fails_if_cht_root_is_invalid() { #[test] fn check_header_proof_fails_if_invalid_header_provided() { - let ( - local_checker, - local_cht_root, - mut remote_block_header, - remote_header_proof, - ) = prepare_for_header_proof_check(true, true); + let (local_checker, local_cht_root, mut remote_block_header, remote_header_proof) = + prepare_for_header_proof_check(true, true); remote_block_header.number = 100; assert!((&local_checker as &dyn FetchChecker) .check_header_proof( diff --git a/client/service/test/src/client/mod.rs b/client/service/test/src/client/mod.rs index 1b47efa1cc40c..ee169a007382a 100644 --- a/client/service/test/src/client/mod.rs +++ b/client/service/test/src/client/mod.rs @@ -1286,9 +1286,8 @@ fn importing_diverged_finalized_block_should_trigger_reorg() { #[test] fn finalizing_diverged_block_should_trigger_reorg() { - let (mut client, select_chain) = TestClientBuilder::new() - .state_hashed_value() - .build_with_longest_chain(); + let (mut client, select_chain) = + TestClientBuilder::new().state_hashed_value().build_with_longest_chain(); // G -> A1 -> A2 // \ @@ -2049,17 +2048,23 @@ fn storage_keys_iter_works_inner(hashed_value: bool) { .map(|x| x.0) .collect(); if hashed_value { - assert_eq!(res, [ - hex!("3a686561707061676573").to_vec(), - sp_core::storage::well_known_keys::TRIE_HASHING_CONFIG.to_vec(), - hex!("6644b9b8bc315888ac8e41a7968dc2b4141a5403c58acdf70b7e8f7e07bf5081").to_vec(), - ]); + assert_eq!( + res, + [ + hex!("3a686561707061676573").to_vec(), + sp_core::storage::well_known_keys::TRIE_HASHING_CONFIG.to_vec(), + hex!("6644b9b8bc315888ac8e41a7968dc2b4141a5403c58acdf70b7e8f7e07bf5081").to_vec(), + ] + ); } else { - assert_eq!(res, [ - hex!("3a686561707061676573").to_vec(), - hex!("6644b9b8bc315888ac8e41a7968dc2b4141a5403c58acdf70b7e8f7e07bf5081").to_vec(), - hex!("79c07e2b1d2e2abfd4855b936617eeff5e0621c4869aa60c02be9adcc98a0d1d").to_vec(), - ]); + assert_eq!( + res, + [ + hex!("3a686561707061676573").to_vec(), + hex!("6644b9b8bc315888ac8e41a7968dc2b4141a5403c58acdf70b7e8f7e07bf5081").to_vec(), + hex!("79c07e2b1d2e2abfd4855b936617eeff5e0621c4869aa60c02be9adcc98a0d1d").to_vec(), + ] + ); } let res: Vec<_> = client diff --git a/frame/support/test/tests/instance.rs b/frame/support/test/tests/instance.rs index 1fd8ebac218eb..d7dfe6316298a 100644 --- a/frame/support/test/tests/instance.rs +++ b/frame/support/test/tests/instance.rs @@ -345,9 +345,8 @@ fn storage_instance_independence() { top: std::collections::BTreeMap::new(), children_default: std::collections::HashMap::new(), }; - storage.modify_trie_alt_hashing_threshold(Some( - sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD, - )); + storage + .modify_trie_alt_hashing_threshold(Some(sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD)); sp_state_machine::BasicExternalities::execute_with_storage(&mut storage, || { module2::Value::::put(0); module2::Value::::put(0); diff --git a/frame/system/src/lib.rs b/frame/system/src/lib.rs index 28860a3784974..292594825ac0a 100644 --- a/frame/system/src/lib.rs +++ b/frame/system/src/lib.rs @@ -1445,9 +1445,9 @@ impl Pallet { ], children_default: map![], }; - storage.modify_trie_alt_hashing_threshold( - Some(sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD), - ); + storage.modify_trie_alt_hashing_threshold(Some( + sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD, + )); TestExternalities::new(storage) } diff --git a/primitives/io/src/lib.rs b/primitives/io/src/lib.rs index ba4f6ae520eff..58a79a1586f39 100644 --- a/primitives/io/src/lib.rs +++ b/primitives/io/src/lib.rs @@ -55,7 +55,7 @@ use sp_core::{ }; #[cfg(feature = "std")] -use sp_trie::{TrieConfiguration, Layout}; +use sp_trie::{Layout, TrieConfiguration}; use sp_runtime_interface::{ pass_by::{PassBy, PassByCodec}, @@ -1493,12 +1493,10 @@ mod tests { }); let value = vec![7u8; 35]; - let mut storage = Storage { - top: map![b"foo00".to_vec() => value.clone()], - children_default: map![], - }; + let mut storage = + Storage { top: map![b"foo00".to_vec() => value.clone()], children_default: map![] }; storage.modify_trie_alt_hashing_threshold(Some( - sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD + sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD, )); t = BasicExternalities::new(storage); diff --git a/primitives/state-machine/src/backend.rs b/primitives/state-machine/src/backend.rs index b6af5c02657c3..4d5de57a20631 100644 --- a/primitives/state-machine/src/backend.rs +++ b/primitives/state-machine/src/backend.rs @@ -195,10 +195,7 @@ pub trait Backend: sp_std::fmt::Debug { let mut child_roots: Vec<_> = Default::default(); // child first for (child_info, child_delta) in child_deltas { - let (child_root, empty, child_txs) = self.child_storage_root( - &child_info, - child_delta, - ); + let (child_root, empty, child_txs) = self.child_storage_root(&child_info, child_delta); let prefixed_storage_key = child_info.prefixed_storage_key(); txs.consolidate(child_txs); if empty { @@ -207,13 +204,10 @@ pub trait Backend: sp_std::fmt::Debug { child_roots.push((prefixed_storage_key.into_inner(), Some(child_root.encode()))); } } - let (root, parent_txs) = self.storage_root(delta - .map(|(k, v)| (&k[..], v.as_ref().map(|v| &v[..]))) - .chain( - child_roots - .iter() - .map(|(k, v)| (&k[..], v.as_ref().map(|v| &v[..]))) - ), + let (root, parent_txs) = self.storage_root( + delta + .map(|(k, v)| (&k[..], v.as_ref().map(|v| &v[..]))) + .chain(child_roots.iter().map(|(k, v)| (&k[..], v.as_ref().map(|v| &v[..])))), ); txs.consolidate(parent_txs); (root, txs) @@ -272,7 +266,9 @@ pub trait Backend: sp_std::fmt::Debug { /// Read current trie hashing threshold. /// Please do not change default implementation when implementing this trait. fn get_trie_alt_hashing_threshold(&self) -> Option { - self.storage(sp_core::storage::well_known_keys::TRIE_HASHING_CONFIG).ok().flatten() + self.storage(sp_core::storage::well_known_keys::TRIE_HASHING_CONFIG) + .ok() + .flatten() .and_then(|encoded| sp_core::storage::trie_threshold_decode(&mut encoded.as_slice())) } @@ -301,9 +297,9 @@ impl Consolidate for Vec<(Option, StorageCollection)> { } impl Consolidate for sp_trie::GenericMemoryDB - where - H: Hasher, - KF: sp_trie::KeyFunction, +where + H: Hasher, + KF: sp_trie::KeyFunction, { fn consolidate(&mut self, other: Self) { sp_trie::GenericMemoryDB::consolidate(self, other) diff --git a/primitives/state-machine/src/basic.rs b/primitives/state-machine/src/basic.rs index ed2e4f439c726..3d7ac00c18184 100644 --- a/primitives/state-machine/src/basic.rs +++ b/primitives/state-machine/src/basic.rs @@ -29,7 +29,7 @@ use sp_core::{ Blake2Hasher, }; use sp_externalities::{Extension, Extensions}; -use sp_trie::{TrieConfiguration, empty_child_trie_root, Layout}; +use sp_trie::{empty_child_trie_root, Layout, TrieConfiguration}; use std::{ any::{Any, TypeId}, collections::BTreeMap, diff --git a/primitives/state-machine/src/changes_trie/mod.rs b/primitives/state-machine/src/changes_trie/mod.rs index ab8c127f008eb..473360f4ffb26 100644 --- a/primitives/state-machine/src/changes_trie/mod.rs +++ b/primitives/state-machine/src/changes_trie/mod.rs @@ -202,7 +202,7 @@ impl<'a, H: Hasher, N: BlockNumber> crate::TrieBackendStorage self.0.get(key, prefix) } - fn access_from(&self, _key: &H::Out) { } + fn access_from(&self, _key: &H::Out) {} } /// Changes trie configuration. diff --git a/primitives/state-machine/src/changes_trie/storage.rs b/primitives/state-machine/src/changes_trie/storage.rs index b47a53f16f00d..b1fca1eb971a9 100644 --- a/primitives/state-machine/src/changes_trie/storage.rs +++ b/primitives/state-machine/src/changes_trie/storage.rs @@ -191,7 +191,7 @@ impl Storage for InMemoryStorage Result, String> { - Ok( as hash_db::HashDBRef>::get(&self.data.read().mdb, key, prefix)) + Ok( as hash_db::HashDBRef>::get(&self.data.read().mdb, key, prefix)) } } @@ -212,5 +212,5 @@ where self.storage.get(key, prefix) } - fn access_from(&self, _key: &H::Out) { } + fn access_from(&self, _key: &H::Out) {} } diff --git a/primitives/state-machine/src/lib.rs b/primitives/state-machine/src/lib.rs index 5fa07367c814d..d2c0ec0651502 100644 --- a/primitives/state-machine/src/lib.rs +++ b/primitives/state-machine/src/lib.rs @@ -119,6 +119,7 @@ impl sp_std::fmt::Display for DefaultError { pub use crate::{ backend::Backend, + error::{Error, ExecutionError}, ext::Ext, overlayed_changes::{ ChildStorageCollection, IndexOperation, OffchainChangesCollection, @@ -128,7 +129,6 @@ pub use crate::{ stats::{StateMachineStats, UsageInfo, UsageUnit}, trie_backend::TrieBackend, trie_backend_essence::{Storage, TrieBackendStorage}, - error::{Error, ExecutionError}, }; #[cfg(not(feature = "std"))] @@ -142,30 +142,25 @@ mod changes_trie { #[cfg(feature = "std")] mod std_reexport { - pub use sp_trie::{trie_types::TrieDBMut, Layout, StorageProof, TrieMut, - DBValue, MemoryDB}; - pub use crate::testing::TestExternalities; - pub use crate::basic::BasicExternalities; - pub use crate::read_only::{ReadOnlyExternalities, InspectState}; - pub use crate::proving_backend::{ - create_proof_check_backend, ProofRecorder, ProvingBackend, ProvingBackendRecorder, + pub use crate::{ + basic::BasicExternalities, + changes_trie::{ + disabled_state as disabled_changes_trie_state, key_changes, key_changes_proof, + key_changes_proof_check, key_changes_proof_check_with_db, prune as prune_changes_tries, + AnchorBlockId as ChangesTrieAnchorBlockId, BlockNumber as ChangesTrieBlockNumber, + BuildCache as ChangesTrieBuildCache, CacheAction as ChangesTrieCacheAction, + ConfigurationRange as ChangesTrieConfigurationRange, + InMemoryStorage as InMemoryChangesTrieStorage, RootsStorage as ChangesTrieRootsStorage, + State as ChangesTrieState, Storage as ChangesTrieStorage, + }, + in_memory_backend::new_in_mem, + proving_backend::{ + create_proof_check_backend, ProofRecorder, ProvingBackend, ProvingBackendRecorder, + }, + read_only::{InspectState, ReadOnlyExternalities}, + testing::TestExternalities, }; - pub use crate::changes_trie::{ - AnchorBlockId as ChangesTrieAnchorBlockId, - State as ChangesTrieState, - Storage as ChangesTrieStorage, - RootsStorage as ChangesTrieRootsStorage, - InMemoryStorage as InMemoryChangesTrieStorage, - BuildCache as ChangesTrieBuildCache, - CacheAction as ChangesTrieCacheAction, - ConfigurationRange as ChangesTrieConfigurationRange, - key_changes, key_changes_proof, - key_changes_proof_check, key_changes_proof_check_with_db, - prune as prune_changes_tries, - disabled_state as disabled_changes_trie_state, - BlockNumber as ChangesTrieBlockNumber, - }; - pub use crate::in_memory_backend::new_in_mem; + pub use sp_trie::{trie_types::TrieDBMut, DBValue, Layout, MemoryDB, StorageProof, TrieMut}; } #[cfg(feature = "std")] @@ -984,7 +979,7 @@ mod tests { use codec::{Decode, Encode}; use sp_core::{ map, - storage::ChildInfo, + storage::{ChildInfo, TEST_DEFAULT_ALT_HASH_THRESHOLD as TRESHOLD}, testing::TaskExecutor, traits::{CodeExecutor, Externalities, RuntimeCode}, NativeOrEncoded, NeverNativeValue, @@ -995,7 +990,6 @@ mod tests { panic::UnwindSafe, result, }; - use sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD as TRESHOLD; #[derive(Clone)] struct DummyCodeExecutor { @@ -1628,15 +1622,11 @@ mod tests { #[test] fn inner_state_hashing_switch_proofs() { - let mut layout = Layout::default(); let (mut mdb, mut root) = trie_backend::tests::test_db(false); { - let mut trie = TrieDBMut::from_existing_with_layout( - &mut mdb, - &mut root, - layout.clone(), - ).unwrap(); + let mut trie = + TrieDBMut::from_existing_with_layout(&mut mdb, &mut root, layout.clone()).unwrap(); trie.insert(b"foo", vec![1u8; 1_000].as_slice()) // big inner hash .expect("insert failed"); trie.insert(b"foo2", vec![3u8; 16].as_slice()) // no inner hash @@ -1644,17 +1634,15 @@ mod tests { trie.insert(b"foo222", vec![5u8; 100].as_slice()) // inner hash .expect("insert failed"); } - + let check_proof = |mdb, root| -> StorageProof { let remote_backend = TrieBackend::new(mdb, root); let remote_root = remote_backend.storage_root(::std::iter::empty()).0; let remote_proof = prove_read(remote_backend, &[b"foo222"]).unwrap(); // check proof locally - let local_result1 = read_proof_check::( - remote_root, - remote_proof.clone(), - &[b"foo222"], - ).unwrap(); + let local_result1 = + read_proof_check::(remote_root, remote_proof.clone(), &[b"foo222"]) + .unwrap(); // check that results are correct assert_eq!( local_result1.into_iter().collect::>(), @@ -1669,16 +1657,12 @@ mod tests { assert!(remote_proof.encoded_size() > 1_100); let root1 = root.clone(); - // do switch layout = Layout::with_alt_hashing(TRESHOLD); // update with same value do not change { - let mut trie = TrieDBMut::from_existing_with_layout( - &mut mdb, - &mut root, - layout.clone(), - ).unwrap(); + let mut trie = + TrieDBMut::from_existing_with_layout(&mut mdb, &mut root, layout.clone()).unwrap(); trie.insert(b"foo222", vec![5u8; 100].as_slice()) // inner hash .expect("insert failed"); } @@ -1689,11 +1673,8 @@ mod tests { // work with state machine as only changes do makes // it to payload (would require a special host function). { - let mut trie = TrieDBMut::from_existing_with_layout( - &mut mdb, - &mut root, - layout.clone(), - ).unwrap(); + let mut trie = + TrieDBMut::from_existing_with_layout(&mut mdb, &mut root, layout.clone()).unwrap(); trie.insert(b"foo222", vec![4u8].as_slice()) // inner hash .expect("insert failed"); trie.insert(b"foo222", vec![5u8; 100].as_slice()) // inner hash @@ -1705,8 +1686,7 @@ mod tests { // nodes foo is replaced by its hashed value form. assert!(remote_proof.encode().len() < 1000); assert!(remote_proof.encoded_size() < 1000); - assert_eq!(remote_proof.encode().len(), - remote_proof.encoded_size()); + assert_eq!(remote_proof.encode().len(), remote_proof.encoded_size()); } #[test] @@ -1727,32 +1707,35 @@ mod tests { let (remote_root, transaction) = remote_backend.full_storage_root( std::iter::empty(), vec![ - (&child_info1, vec![ - // a inner hashable node - (&b"k"[..], Some(&long_vec[..])), - // need to ensure this is not an inline node - // otherwhise we do not know what is accessed when - // storing proof. - (&b"key1"[..], Some(&vec![5u8; 32][..])), - (&b"key2"[..], Some(&b"val3"[..])), - ].into_iter()), - (&child_info2, vec![ - (&b"key3"[..], Some(&b"val4"[..])), - (&b"key4"[..], Some(&b"val5"[..])), - ].into_iter()), - (&child_info3, vec![ - (&b"key5"[..], Some(&b"val6"[..])), - (&b"key6"[..], Some(&b"val7"[..])), - ].into_iter()), - ].into_iter(), + ( + &child_info1, + vec![ + // a inner hashable node + (&b"k"[..], Some(&long_vec[..])), + // need to ensure this is not an inline node + // otherwhise we do not know what is accessed when + // storing proof. + (&b"key1"[..], Some(&vec![5u8; 32][..])), + (&b"key2"[..], Some(&b"val3"[..])), + ] + .into_iter(), + ), + ( + &child_info2, + vec![(&b"key3"[..], Some(&b"val4"[..])), (&b"key4"[..], Some(&b"val5"[..]))] + .into_iter(), + ), + ( + &child_info3, + vec![(&b"key5"[..], Some(&b"val6"[..])), (&b"key6"[..], Some(&b"val7"[..]))] + .into_iter(), + ), + ] + .into_iter(), ); remote_backend.backend_storage_mut().consolidate(transaction); remote_backend.essence.set_root(remote_root.clone()); - let remote_proof = prove_child_read( - remote_backend, - &child_info1, - &[b"key1"], - ).unwrap(); + let remote_proof = prove_child_read(remote_backend, &child_info1, &[b"key1"]).unwrap(); let size = remote_proof.encoded_size(); let remote_proof = test_compact(remote_proof, &remote_root); let local_result1 = read_child_proof_check::( diff --git a/primitives/state-machine/src/proving_backend.rs b/primitives/state-machine/src/proving_backend.rs index 8713b3c44ab2f..c09e0bff075ce 100644 --- a/primitives/state-machine/src/proving_backend.rs +++ b/primitives/state-machine/src/proving_backend.rs @@ -27,17 +27,12 @@ use hash_db::{HashDB, Hasher, Prefix, EMPTY_PREFIX}; use log::debug; use parking_lot::RwLock; use sp_core::storage::ChildInfo; +pub use sp_trie::trie_types::TrieError; use sp_trie::{ - MemoryDB, empty_child_trie_root, read_trie_value_with, read_child_trie_value_with, - record_all_keys, StorageProof, Meta, Layout, Recorder, -}; -pub use sp_trie::{ - trie_types::TrieError, -}; -use std::{ - collections::HashMap, - sync::Arc, + empty_child_trie_root, read_child_trie_value_with, read_trie_value_with, record_all_keys, + Layout, MemoryDB, Meta, Recorder, StorageProof, }; +use std::{collections::HashMap, sync::Arc}; /// Patricia trie-based backend specialized in get value proofs. pub struct ProvingBackendRecorder<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> { @@ -144,23 +139,26 @@ impl ProofRecorder { pub fn access_from(&self, key: &Hash, hash_len: usize) { let mut inner = self.inner.write(); let ProofRecorderInner { encoded_size, records, .. } = &mut *inner; - records.entry(key.clone()) - .and_modify(|entry| { - if let Some(entry) = entry.as_mut() { - if !entry.2 { - let old_size = sp_trie::estimate_entry_size(entry, hash_len); - entry.2 = true; - let new_size = sp_trie::estimate_entry_size(entry, hash_len); - *encoded_size += new_size; - *encoded_size -= old_size; - } + records.entry(key.clone()).and_modify(|entry| { + if let Some(entry) = entry.as_mut() { + if !entry.2 { + let old_size = sp_trie::estimate_entry_size(entry, hash_len); + entry.2 = true; + let new_size = sp_trie::estimate_entry_size(entry, hash_len); + *encoded_size += new_size; + *encoded_size -= old_size; } - }); + } + }); } /// Returns the value at the given `key`. pub fn get(&self, key: &Hash) -> Option> { - self.inner.read().records.get(key).as_ref() + self.inner + .read() + .records + .get(key) + .as_ref() .map(|v| v.as_ref().map(|v| v.0.clone())) } @@ -175,19 +173,23 @@ impl ProofRecorder { /// Convert into a [`StorageProof`]. pub fn to_storage_proof(&self) -> StorageProof { - let trie_nodes = self.inner.read() + let trie_nodes = self + .inner + .read() .records .iter() - .filter_map(|(_k, v)| v.as_ref().map(|v| { - let mut meta = v.1.clone(); - if let Some(hashed) = sp_trie::to_hashed_variant::( - v.0.as_slice(), &mut meta, v.2, - ) { - hashed - } else { - v.0.clone() - } - })) + .filter_map(|(_k, v)| { + v.as_ref().map(|v| { + let mut meta = v.1.clone(); + if let Some(hashed) = + sp_trie::to_hashed_variant::(v.0.as_slice(), &mut meta, v.2) + { + hashed + } else { + v.0.clone() + } + }) + }) .collect(); StorageProof::new(trie_nodes) @@ -253,11 +255,7 @@ impl<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> TrieBackendStorage { type Overlay = S::Overlay; - fn get( - &self, - key: &H::Out, - prefix: Prefix, - ) -> Result, String> { + fn get(&self, key: &H::Out, prefix: Prefix) -> Result, String> { if let Some(v) = self.proof_recorder.get(key) { return Ok(v) } @@ -482,26 +480,33 @@ mod tests { fn proof_recorded_and_checked_inner(flagged: bool) { let size_content = 34; // above hashable value treshold. let value_range = 0..64; - let contents = value_range.clone() - .map(|i| (vec![i], Some(vec![i; size_content]))).collect::>(); + let contents = value_range + .clone() + .map(|i| (vec![i], Some(vec![i; size_content]))) + .collect::>(); let mut in_memory = InMemoryBackend::::default(); if flagged { - in_memory = in_memory.update(vec![(None, vec![( - sp_core::storage::well_known_keys::TRIE_HASHING_CONFIG.to_vec(), - Some(sp_core::storage::trie_threshold_encode( - sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD, - )), - )])]); + in_memory = in_memory.update(vec![( + None, + vec![( + sp_core::storage::well_known_keys::TRIE_HASHING_CONFIG.to_vec(), + Some(sp_core::storage::trie_threshold_encode( + sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD, + )), + )], + )]); } let mut in_memory = in_memory.update(vec![(None, contents)]); let in_memory_root = in_memory.storage_root(std::iter::empty()).0; - value_range.clone() - .for_each(|i| assert_eq!(in_memory.storage(&[i]).unwrap().unwrap(), vec![i; size_content])); + value_range.clone().for_each(|i| { + assert_eq!(in_memory.storage(&[i]).unwrap().unwrap(), vec![i; size_content]) + }); let trie = in_memory.as_trie_backend().unwrap(); let trie_root = trie.storage_root(std::iter::empty()).0; assert_eq!(in_memory_root, trie_root); - value_range.clone() + value_range + .clone() .for_each(|i| assert_eq!(trie.storage(&[i]).unwrap().unwrap(), vec![i; size_content])); let proving = ProvingBackend::new(trie); @@ -509,7 +514,8 @@ mod tests { let proof = proving.extract_proof(); - let proof_check = create_proof_check_backend::(in_memory_root.into(), proof).unwrap(); + let proof_check = + create_proof_check_backend::(in_memory_root.into(), proof).unwrap(); assert_eq!(proof_check.storage(&[42]).unwrap().unwrap(), vec![42; size_content]); } @@ -525,38 +531,36 @@ mod tests { let child_info_2 = &child_info_2; let contents = vec![ (None, (0..64).map(|i| (vec![i], Some(vec![i]))).collect::>()), - (Some(child_info_1.clone()), - (28..65).map(|i| (vec![i], Some(vec![i]))).collect()), - (Some(child_info_2.clone()), - (10..15).map(|i| (vec![i], Some(vec![i]))).collect()), + (Some(child_info_1.clone()), (28..65).map(|i| (vec![i], Some(vec![i]))).collect()), + (Some(child_info_2.clone()), (10..15).map(|i| (vec![i], Some(vec![i]))).collect()), ]; let mut in_memory = InMemoryBackend::::default(); if flagged { - in_memory = in_memory.update(vec![(None, vec![( - sp_core::storage::well_known_keys::TRIE_HASHING_CONFIG.to_vec(), - Some(sp_core::storage::trie_threshold_encode( - sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD, - )), - )])]); + in_memory = in_memory.update(vec![( + None, + vec![( + sp_core::storage::well_known_keys::TRIE_HASHING_CONFIG.to_vec(), + Some(sp_core::storage::trie_threshold_encode( + sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD, + )), + )], + )]); } in_memory = in_memory.update(contents); let child_storage_keys = vec![child_info_1.to_owned(), child_info_2.to_owned()]; - let in_memory_root = in_memory.full_storage_root( - std::iter::empty(), - child_storage_keys.iter().map(|k|(k, std::iter::empty())), - ).0; - (0..64).for_each(|i| assert_eq!( - in_memory.storage(&[i]).unwrap().unwrap(), - vec![i] - )); - (28..65).for_each(|i| assert_eq!( - in_memory.child_storage(child_info_1, &[i]).unwrap().unwrap(), - vec![i] - )); - (10..15).for_each(|i| assert_eq!( - in_memory.child_storage(child_info_2, &[i]).unwrap().unwrap(), - vec![i] - )); + let in_memory_root = in_memory + .full_storage_root( + std::iter::empty(), + child_storage_keys.iter().map(|k| (k, std::iter::empty())), + ) + .0; + (0..64).for_each(|i| assert_eq!(in_memory.storage(&[i]).unwrap().unwrap(), vec![i])); + (28..65).for_each(|i| { + assert_eq!(in_memory.child_storage(child_info_1, &[i]).unwrap().unwrap(), vec![i]) + }); + (10..15).for_each(|i| { + assert_eq!(in_memory.child_storage(child_info_2, &[i]).unwrap().unwrap(), vec![i]) + }); let trie = in_memory.as_trie_backend().unwrap(); let trie_root = trie.storage_root(std::iter::empty()).0; diff --git a/primitives/state-machine/src/testing.rs b/primitives/state-machine/src/testing.rs index 42f447917f100..ce7aaee289c12 100644 --- a/primitives/state-machine/src/testing.rs +++ b/primitives/state-machine/src/testing.rs @@ -19,9 +19,9 @@ use std::{ any::{Any, TypeId}, + collections::{BTreeMap, HashMap}, panic::{AssertUnwindSafe, UnwindSafe}, }; -use std::collections::{HashMap, BTreeMap}; use crate::{ backend::Backend, @@ -39,7 +39,7 @@ use sp_core::{ offchain::testing::TestPersistentOffchainDB, storage::{ well_known_keys::{is_child_storage_key, CHANGES_TRIE_CONFIG, CODE}, - Storage, ChildInfo, + ChildInfo, Storage, }, testing::TaskExecutor, traits::TaskExecutorExt, @@ -97,7 +97,6 @@ where Self::new_with_code_inner(&[], storage, true) } - /// New empty test externalities. pub fn new_empty() -> Self { Self::new_with_code(&[], Storage::default()) @@ -130,15 +129,20 @@ where let mut backend: InMemoryBackend = { let mut storage = Storage::default(); storage.modify_trie_alt_hashing_threshold(Some( - sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD + sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD, )); storage.into() }; - let mut inner: HashMap, BTreeMap> - = storage.children_default.into_iter().map(|(_k, c)| (Some(c.child_info), c.data)).collect(); + let mut inner: HashMap, BTreeMap> = storage + .children_default + .into_iter() + .map(|(_k, c)| (Some(c.child_info), c.data)) + .collect(); inner.insert(None, storage.top); backend.insert( - inner.into_iter().map(|(k, m)| (k, m.into_iter().map(|(k, v)| (k, Some(v))).collect())), + inner + .into_iter() + .map(|(k, m)| (k, m.into_iter().map(|(k, v)| (k, Some(v))).collect())), ); backend } else { @@ -274,9 +278,9 @@ where fn default() -> Self { // default to inner hashed. let mut storage = Storage::default(); - storage.modify_trie_alt_hashing_threshold( - Some(sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD), - ); + storage.modify_trie_alt_hashing_threshold(Some( + sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD, + )); Self::new(storage) } } diff --git a/primitives/state-machine/src/trie_backend.rs b/primitives/state-machine/src/trie_backend.rs index 3e9360efc6d9a..704ae0424ba3b 100644 --- a/primitives/state-machine/src/trie_backend.rs +++ b/primitives/state-machine/src/trie_backend.rs @@ -17,26 +17,28 @@ //! Trie-based state machine backend. -use crate::{warn, debug}; -use hash_db::Hasher; -use sp_trie::{Trie, delta_trie_root, empty_child_trie_root, child_delta_trie_root, - Layout}; -use sp_trie::trie_types::{TrieDB, TrieError}; -use sp_core::storage::{ChildInfo, ChildType}; -use codec::{Codec, Decode}; use crate::{ + debug, trie_backend_essence::{Ephemeral, TrieBackendEssence, TrieBackendStorage}, - Backend, StorageKey, StorageValue, + warn, Backend, StorageKey, StorageValue, }; +use codec::{Codec, Decode}; +use hash_db::Hasher; +use sp_core::storage::{ChildInfo, ChildType}; use sp_std::{boxed::Box, vec::Vec}; +use sp_trie::{ + child_delta_trie_root, delta_trie_root, empty_child_trie_root, + trie_types::{TrieDB, TrieError}, + Layout, Trie, +}; /// Patricia trie-based backend. Transaction type is an overlay of changes to commit. pub struct TrieBackend, H: Hasher> { - pub (crate) essence: TrieBackendEssence, + pub(crate) essence: TrieBackendEssence, // Allows setting alt hashing at start for testing only // (mainly for in_memory_backend when it cannot read it from // state). - pub (crate) force_alt_hashing: Option>, + pub(crate) force_alt_hashing: Option>, } impl, H: Hasher> TrieBackend @@ -45,10 +47,7 @@ where { /// Create new trie-based backend. pub fn new(storage: S, root: H::Out) -> Self { - TrieBackend { - essence: TrieBackendEssence::new(storage, root), - force_alt_hashing: None, - } + TrieBackend { essence: TrieBackendEssence::new(storage, root), force_alt_hashing: None } } /// Get backend essence reference. @@ -195,8 +194,11 @@ where fn storage_root<'a>( &self, - delta: impl Iterator)>, - ) -> (H::Out, Self::Transaction) where H::Out: Ord { + delta: impl Iterator)>, + ) -> (H::Out, Self::Transaction) + where + H::Out: Ord, + { let use_inner_hash_value = if let Some(force) = self.force_alt_hashing.as_ref() { force.clone() } else { @@ -206,10 +208,7 @@ where let mut root = *self.essence.root(); { - let mut eph = Ephemeral::new( - self.essence.backend_storage(), - &mut write_overlay, - ); + let mut eph = Ephemeral::new(self.essence.backend_storage(), &mut write_overlay); let res = || { let layout = if let Some(threshold) = use_inner_hash_value { sp_trie::Layout::with_alt_hashing(threshold) @@ -231,8 +230,11 @@ where fn child_storage_root<'a>( &self, child_info: &ChildInfo, - delta: impl Iterator)>, - ) -> (H::Out, bool, Self::Transaction) where H::Out: Ord { + delta: impl Iterator)>, + ) -> (H::Out, bool, Self::Transaction) + where + H::Out: Ord, + { let use_inner_hash_value = if let Some(force) = self.force_alt_hashing.as_ref() { force.clone() } else { @@ -298,12 +300,11 @@ where #[cfg(test)] pub mod tests { use super::*; - use std::{collections::HashSet, iter}; - use sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD as TRESHOLD; use codec::Encode; - use sp_core::H256; + use sp_core::{storage::TEST_DEFAULT_ALT_HASH_THRESHOLD as TRESHOLD, H256}; use sp_runtime::traits::BlakeTwo256; use sp_trie::{trie_types::TrieDBMut, KeySpacedDBMut, PrefixedMemoryDB, TrieMut}; + use std::{collections::HashSet, iter}; const CHILD_KEY_1: &[u8] = b"sub1"; @@ -338,7 +339,8 @@ pub mod tests { trie.insert( sp_core::storage::well_known_keys::TRIE_HASHING_CONFIG, sp_core::storage::trie_threshold_encode(TRESHOLD).as_slice(), - ).unwrap(); + ) + .unwrap(); } for i in 128u8..255u8 { trie.insert(&[i], &[i]).unwrap(); @@ -421,9 +423,8 @@ pub mod tests { storage_root_transaction_is_non_empty_inner(true); } fn storage_root_transaction_is_non_empty_inner(flagged: bool) { - let (new_root, mut tx) = test_trie(flagged).storage_root( - iter::once((&b"new-key"[..], Some(&b"new-value"[..]))), - ); + let (new_root, mut tx) = + test_trie(flagged).storage_root(iter::once((&b"new-key"[..], Some(&b"new-value"[..])))); assert!(!tx.drain().is_empty()); assert!(new_root != test_trie(false).storage_root(iter::empty()).0); } diff --git a/primitives/state-machine/src/trie_backend_essence.rs b/primitives/state-machine/src/trie_backend_essence.rs index ac775e1aafec6..0e3db15761d43 100644 --- a/primitives/state-machine/src/trie_backend_essence.rs +++ b/primitives/state-machine/src/trie_backend_essence.rs @@ -18,15 +18,16 @@ //! Trie-based state machine backend essence used to read values //! from storage. -use sp_std::{ops::Deref, boxed::Box, vec::Vec}; -use hash_db::{self, Hasher, Prefix, AsHashDB, HashDB, HashDBRef}; -use sp_trie::{Trie, PrefixedMemoryDB, DBValue, Layout, - empty_child_trie_root, read_trie_value, read_child_trie_value, - KeySpacedDB, TrieDBIterator, TrieDBKeyIterator}; -use sp_trie::trie_types::{TrieDB, TrieError}; -use codec::Encode; use crate::{backend::Consolidate, debug, warn, StorageKey, StorageValue}; +use codec::Encode; +use hash_db::{self, AsHashDB, HashDB, HashDBRef, Hasher, Prefix}; use sp_core::storage::ChildInfo; +use sp_std::{boxed::Box, ops::Deref, vec::Vec}; +use sp_trie::{ + empty_child_trie_root, read_child_trie_value, read_trie_value, + trie_types::{TrieDB, TrieError}, + DBValue, KeySpacedDB, Layout, PrefixedMemoryDB, Trie, TrieDBIterator, TrieDBKeyIterator, +}; #[cfg(feature = "std")] use std::sync::Arc; @@ -139,10 +140,9 @@ where dyn_eph = self; } - let trie = TrieDB::::new(dyn_eph, root) - .map_err(|e| format!("TrieDB creation error: {}", e))?; - let mut iter = trie.key_iter() - .map_err(|e| format!("TrieDB iteration error: {}", e))?; + let trie = + TrieDB::::new(dyn_eph, root).map_err(|e| format!("TrieDB creation error: {}", e))?; + let mut iter = trie.key_iter().map_err(|e| format!("TrieDB iteration error: {}", e))?; // The key just after the one given in input, basically `key++0`. // Note: We are sure this is the next key if: @@ -158,8 +158,8 @@ where let next_element = iter.next(); let next_key = if let Some(next_element) = next_element { - let next_key = next_element - .map_err(|e| format!("TrieDB iterator next error: {}", e))?; + let next_key = + next_element.map_err(|e| format!("TrieDB iterator next error: {}", e))?; Some(next_key) } else { None @@ -263,12 +263,28 @@ where }; let mut root = H::Out::default(); root.as_mut().copy_from_slice(&root_vec); - self.trie_iter_key_inner(&root, Some(prefix), |k| { f(k); true }, Some(child_info)) + self.trie_iter_key_inner( + &root, + Some(prefix), + |k| { + f(k); + true + }, + Some(child_info), + ) } /// Execute given closure for all keys starting with prefix. pub fn for_keys_with_prefix(&self, prefix: &[u8], mut f: F) { - self.trie_iter_key_inner(&self.root, Some(prefix), |k| { f(k); true }, None) + self.trie_iter_key_inner( + &self.root, + Some(prefix), + |k| { + f(k); + true + }, + None, + ) } fn trie_iter_key_inner bool>( @@ -289,10 +305,13 @@ where for x in iter { let key = x?; - debug_assert!(prefix.as_ref().map(|prefix| key.starts_with(prefix)).unwrap_or(true)); + debug_assert!(prefix + .as_ref() + .map(|prefix| key.starts_with(prefix)) + .unwrap_or(true)); if !f(&key) { - break; + break } } @@ -379,8 +398,12 @@ pub(crate) struct Ephemeral<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> { impl<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> AsHashDB for Ephemeral<'a, S, H> { - fn as_hash_db<'b>(&'b self) -> &'b (dyn HashDB + 'b) { self } - fn as_hash_db_mut<'b>(&'b mut self) -> &'b mut (dyn HashDB + 'b) { self } + fn as_hash_db<'b>(&'b self) -> &'b (dyn HashDB + 'b) { + self + } + fn as_hash_db_mut<'b>(&'b mut self) -> &'b mut (dyn HashDB + 'b) { + self + } } impl<'a, S: TrieBackendStorage, H: Hasher> Ephemeral<'a, S, H> { @@ -433,9 +456,7 @@ impl<'a, S: 'a + TrieBackendStorage, H: Hasher> hash_db::HashDB } } -impl<'a, S: 'a + TrieBackendStorage, H: Hasher> HashDBRef - for Ephemeral<'a, S, H> -{ +impl<'a, S: 'a + TrieBackendStorage, H: Hasher> HashDBRef for Ephemeral<'a, S, H> { fn get(&self, key: &H::Out, prefix: Prefix) -> Option { HashDB::get(self, key, prefix) } @@ -476,9 +497,9 @@ impl TrieBackendStorage for Arc> { } impl TrieBackendStorage for sp_trie::GenericMemoryDB - where - H: Hasher, - KF: sp_trie::KeyFunction + Send + Sync, +where + H: Hasher, + KF: sp_trie::KeyFunction + Send + Sync, { type Overlay = Self; @@ -491,16 +512,16 @@ impl TrieBackendStorage for sp_trie::GenericMemoryDB } } -impl, H: Hasher> AsHashDB - for TrieBackendEssence -{ - fn as_hash_db<'b>(&'b self) -> &'b (dyn HashDB + 'b) { self } - fn as_hash_db_mut<'b>(&'b mut self) -> &'b mut (dyn HashDB + 'b) { self } +impl, H: Hasher> AsHashDB for TrieBackendEssence { + fn as_hash_db<'b>(&'b self) -> &'b (dyn HashDB + 'b) { + self + } + fn as_hash_db_mut<'b>(&'b mut self) -> &'b mut (dyn HashDB + 'b) { + self + } } -impl, H: Hasher> HashDB - for TrieBackendEssence -{ +impl, H: Hasher> HashDB for TrieBackendEssence { fn get(&self, key: &H::Out, prefix: Prefix) -> Option { if *key == self.empty { return Some([0u8].to_vec()) @@ -541,9 +562,7 @@ impl, H: Hasher> HashDB } } -impl, H: Hasher> HashDBRef - for TrieBackendEssence -{ +impl, H: Hasher> HashDBRef for TrieBackendEssence { fn get(&self, key: &H::Out, prefix: Prefix) -> Option { HashDB::get(self, key, prefix) } diff --git a/primitives/storage/src/lib.rs b/primitives/storage/src/lib.rs index ed0eac1f2d44b..b00a42d8aab18 100644 --- a/primitives/storage/src/lib.rs +++ b/primitives/storage/src/lib.rs @@ -244,8 +244,7 @@ pub fn trie_threshold_encode(threshold: u32) -> Vec { /// Configuration threshold from encoded, invalid encoded /// is same as no threshold. pub fn trie_threshold_decode(mut encoded: &[u8]) -> Option { - codec::Compact::::decode(&mut encoded).ok() - .map(|compact| compact.0) + codec::Compact::::decode(&mut encoded).ok().map(|compact| compact.0) } /// Default value to use as a threshold for testing. diff --git a/primitives/trie/src/lib.rs b/primitives/trie/src/lib.rs index 10a1e62eff32d..179f44db1e7a0 100644 --- a/primitives/trie/src/lib.rs +++ b/primitives/trie/src/lib.rs @@ -26,30 +26,32 @@ mod storage_proof; mod trie_codec; mod trie_stream; -use sp_std::{boxed::Box, marker::PhantomData, vec, vec::Vec, borrow::Borrow, fmt}; -use trie_db::proof::{generate_proof, verify_proof}; -pub use trie_db::proof::VerifyError; -/// The Substrate format implementation of `TrieStream`. -pub use trie_stream::TrieStream; -/// The Substrate format implementation of `NodeCodec`. -pub use node_codec::NodeCodec; -pub use storage_proof::{StorageProof, CompactProof}; -/// Various re-exports from the `trie-db` crate. -pub use trie_db::{ - Trie, TrieMut, DBValue, Recorder, CError, Query, TrieLayout, TrieConfiguration, - nibble_ops, TrieDBIterator, TrieDBKeyIterator, Meta, node::{NodePlan, ValuePlan}, -}; -/// Various re-exports from the `memory-db` crate. -pub use memory_db::KeyFunction; /// Our `NodeCodec`-specific error. pub use error::Error; /// Various re-exports from the `hash-db` crate. pub use hash_db::{HashDB as HashDBT, EMPTY_PREFIX}; use hash_db::{Hasher, Prefix}; pub use memory_db::prefixed_key; +/// Various re-exports from the `memory-db` crate. +pub use memory_db::KeyFunction; +/// The Substrate format implementation of `NodeCodec`. +pub use node_codec::NodeCodec; +use sp_std::{borrow::Borrow, boxed::Box, fmt, marker::PhantomData, vec, vec::Vec}; +pub use storage_proof::{CompactProof, StorageProof}; /// Trie codec reexport, mainly child trie support /// for trie compact proof. pub use trie_codec::{decode_compact, encode_compact, Error as CompactProofError}; +pub use trie_db::proof::VerifyError; +use trie_db::proof::{generate_proof, verify_proof}; +/// Various re-exports from the `trie-db` crate. +pub use trie_db::{ + nibble_ops, + node::{NodePlan, ValuePlan}, + CError, DBValue, Meta, Query, Recorder, Trie, TrieConfiguration, TrieDBIterator, + TrieDBKeyIterator, TrieLayout, TrieMut, +}; +/// The Substrate format implementation of `TrieStream`. +pub use trie_stream::TrieStream; /// substrate trie layout pub struct Layout(Option, sp_std::marker::PhantomData); @@ -81,8 +83,8 @@ impl Layout { } impl TrieLayout for Layout - where - H: Hasher, +where + H: Hasher, { const USE_EXTENSION: bool = false; const ALLOW_EMPTY: bool = true; @@ -97,10 +99,11 @@ impl TrieLayout for Layout } impl TrieConfiguration for Layout - where - H: Hasher, +where + H: Hasher, { - fn trie_root(&self, input: I) -> ::Out where + fn trie_root(&self, input: I) -> ::Out + where I: IntoIterator, A: AsRef<[u8]> + Ord, B: AsRef<[u8]>, @@ -108,7 +111,8 @@ impl TrieConfiguration for Layout trie_root::trie_root_no_extension::(input, self.alt_threshold()) } - fn trie_root_unhashed(&self, input: I) -> Vec where + fn trie_root_unhashed(&self, input: I) -> Vec + where I: IntoIterator, A: AsRef<[u8]> + Ord, B: AsRef<[u8]>, @@ -136,17 +140,14 @@ pub type HashDB<'a, H> = dyn hash_db::HashDB + 'a; /// Reexport from `hash_db`, with genericity set for `Hasher` trait. /// This uses a `KeyFunction` for prefixing keys internally (avoiding /// key conflict for non random keys). -pub type PrefixedMemoryDB = memory_db::MemoryDB< - H, memory_db::PrefixedKey, trie_db::DBValue, MemTracker, ->; +pub type PrefixedMemoryDB = + memory_db::MemoryDB, trie_db::DBValue, MemTracker>; /// Reexport from `hash_db`, with genericity set for `Hasher` trait. /// This uses a noops `KeyFunction` (key addressing must be hashed or using /// an encoding scheme that avoid key conflict). pub type MemoryDB = memory_db::MemoryDB, trie_db::DBValue, MemTracker>; /// Reexport from `hash_db`, with genericity set for `Hasher` trait. -pub type GenericMemoryDB = memory_db::MemoryDB< - H, KF, trie_db::DBValue, MemTracker, ->; +pub type GenericMemoryDB = memory_db::MemoryDB; /// Persistent trie database read-access interface for the a given hasher. pub type TrieDB<'a, L> = trie_db::TrieDB<'a, L>; @@ -182,9 +183,10 @@ pub fn generate_trie_proof<'a, L, I, K, DB>( db: &DB, root: TrieHash, keys: I, -) -> Result>, Box>> where +) -> Result>, Box>> +where L: TrieConfiguration, - I: IntoIterator, + I: IntoIterator, K: 'a + AsRef<[u8]>, DB: hash_db::HashDBRef, { @@ -205,9 +207,10 @@ pub fn verify_trie_proof<'a, L, I, K, V>( root: &TrieHash, proof: &[Vec], items: I, -) -> Result<(), VerifyError, error::Error>> where +) -> Result<(), VerifyError, error::Error>> +where L: TrieConfiguration, - I: IntoIterator)>, + I: IntoIterator)>, K: 'a + AsRef<[u8]>, V: 'a + AsRef<[u8]>, { @@ -222,7 +225,8 @@ pub fn delta_trie_root( mut root: TrieHash, delta: I, layout: L, -) -> Result, Box>> where +) -> Result, Box>> +where I: IntoIterator, A: Borrow<[u8]>, B: Borrow>, @@ -250,28 +254,30 @@ pub fn delta_trie_root( pub fn read_trie_value( db: &DB, root: &TrieHash, - key: &[u8] + key: &[u8], ) -> Result>, Box>> - where - L: TrieConfiguration, - DB: hash_db::HashDBRef, +where + L: TrieConfiguration, + DB: hash_db::HashDBRef, { Ok(TrieDB::::new(&*db, root)?.get(key).map(|x| x.map(|val| val.to_vec()))?) } /// Read a value from the trie with given Query. -pub fn read_trie_value_with ( +pub fn read_trie_value_with( db: &DB, root: &TrieHash, key: &[u8], - query: Q + query: Q, ) -> Result>, Box>> - where - L: TrieConfiguration, - Q: Query, - DB: hash_db::HashDBRef +where + L: TrieConfiguration, + Q: Query, + DB: hash_db::HashDBRef, { - Ok(TrieDB::::new(&*db, root)?.get_with(key, query).map(|x| x.map(|val| val.to_vec()))?) + Ok(TrieDB::::new(&*db, root)? + .get_with(key, query) + .map(|x| x.map(|val| val.to_vec()))?) } /// Determine the empty trie root. @@ -290,10 +296,10 @@ pub fn child_trie_root( layout: &L, input: I, ) -> ::Out - where - I: IntoIterator, - A: AsRef<[u8]> + Ord, - B: AsRef<[u8]>, +where + I: IntoIterator, + A: AsRef<[u8]> + Ord, + B: AsRef<[u8]>, { layout.trie_root(input) } @@ -307,33 +313,29 @@ pub fn child_delta_trie_root( delta: I, layout: L, ) -> Result<::Out, Box>> - where - I: IntoIterator, - A: Borrow<[u8]>, - B: Borrow>, - V: Borrow<[u8]>, - RD: AsRef<[u8]>, - DB: hash_db::HashDB, +where + I: IntoIterator, + A: Borrow<[u8]>, + B: Borrow>, + V: Borrow<[u8]>, + RD: AsRef<[u8]>, + DB: hash_db::HashDB, { let mut root = TrieHash::::default(); // root is fetched from DB, not writable by runtime, so it's always valid. root.as_mut().copy_from_slice(root_data.as_ref()); let mut db = KeySpacedDBMut::new(&mut *db, keyspace); - delta_trie_root::( - &mut db, - root, - delta, - layout, - ) + delta_trie_root::(&mut db, root, delta, layout) } /// Record all keys for a given root. pub fn record_all_keys( db: &DB, root: &TrieHash, - recorder: &mut Recorder> -) -> Result<(), Box>> where + recorder: &mut Recorder>, +) -> Result<(), Box>> +where DB: hash_db::HashDBRef, { let trie = TrieDB::::new(&*db, root)?; @@ -358,8 +360,8 @@ pub fn read_child_trie_value( root_slice: &[u8], key: &[u8], ) -> Result>, Box>> - where - DB: hash_db::HashDBRef, +where + DB: hash_db::HashDBRef, { let mut root = TrieHash::::default(); // root is fetched from DB, not writable by runtime, so it's always valid. @@ -377,10 +379,10 @@ pub fn read_child_trie_value_with( key: &[u8], query: Q, ) -> Result>, Box>> - where - L: TrieConfiguration, - Q: Query, - DB: hash_db::HashDBRef, +where + L: TrieConfiguration, + Q: Query, + DB: hash_db::HashDBRef, { let mut root = TrieHash::::default(); // root is fetched from DB, not writable by runtime, so it's always valid. @@ -518,7 +520,7 @@ fn inner_hashed_value(x: &[u8], range: Option<(usize, usize)>) -> Vec let mut buff = vec![0; x.len() + hash_end.as_ref().len() - (end - start)]; buff[..start].copy_from_slice(&x[..start]); buff[start..].copy_from_slice(hash_end.as_ref()); - return buff; + return buff } if start == 0 && end < len { // start inner hash @@ -527,7 +529,7 @@ fn inner_hashed_value(x: &[u8], range: Option<(usize, usize)>) -> Vec let mut buff = vec![0; x.len() + hash_len - (end - start)]; buff[..hash_len].copy_from_slice(hash_start.as_ref()); buff[hash_len..].copy_from_slice(&x[end..]); - return buff; + return buff } if start < len && end < len { // middle inner hash @@ -537,7 +539,7 @@ fn inner_hashed_value(x: &[u8], range: Option<(usize, usize)>) -> Vec buff[..start].copy_from_slice(&x[..start]); buff[start..start + hash_len].copy_from_slice(hash_middle.as_ref()); buff[start + hash_len..].copy_from_slice(&x[end..]); - return buff; + return buff } } // if anything wrong default to hash @@ -561,7 +563,7 @@ pub fn estimate_entry_size(entry: &(DBValue, Meta, bool), hash_len: usize) -> us } /// Switch to hashed value variant. -pub fn to_hashed_variant( +pub fn to_hashed_variant( value: &[u8], meta: &mut Meta, used_value: bool, @@ -576,7 +578,7 @@ pub fn to_hashed_variant( let value = inner_hashed_value::(value, Some((range.start, range.end))); stored.extend_from_slice(value.as_slice()); meta.contain_hash = true; - return Some(stored); + return Some(stored) } None } @@ -584,7 +586,7 @@ pub fn to_hashed_variant( /// Decode plan in order to update meta early (needed to register proofs). pub fn resolve_encoded_meta(entry: &mut (DBValue, Meta, bool)) { use trie_db::NodeCodec; - let _ = as TrieLayout>::Codec::decode_plan(entry.0.as_slice(), &mut entry.1); + let _ = as TrieLayout>::Codec::decode_plan(entry.0.as_slice(), &mut entry.1); } /// Constants used into trie simplification codec. @@ -604,19 +606,17 @@ mod trie_constants { #[cfg(test)] mod tests { use super::*; - use codec::{Encode, Decode, Compact}; - use sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD as TRESHOLD; + use codec::{Compact, Decode, Encode}; use hash_db::{HashDB, Hasher}; use hex_literal::hex; - use sp_core::Blake2Hasher; + use sp_core::{storage::TEST_DEFAULT_ALT_HASH_THRESHOLD as TRESHOLD, Blake2Hasher}; use trie_db::{DBValue, NodeCodec as NodeCodecT, Trie, TrieMut}; use trie_standardmap::{Alphabet, StandardMap, ValueMode}; type Layout = super::Layout; - type MemoryDBMeta = memory_db::MemoryDB< - H, memory_db::HashKey, trie_db::DBValue, MemTracker, - >; + type MemoryDBMeta = + memory_db::MemoryDB, trie_db::DBValue, MemTracker>; fn hashed_null_node() -> TrieHash { ::hashed_null_node() @@ -677,9 +677,12 @@ mod tests { let mut empty = TrieDBMut::::new(&mut db, &mut root); empty.commit(); let root1 = empty.root().as_ref().to_vec(); - let root2: Vec = Layout::default().trie_root::<_, Vec, Vec>( - std::iter::empty(), - ).as_ref().iter().cloned().collect(); + let root2: Vec = Layout::default() + .trie_root::<_, Vec, Vec>(std::iter::empty()) + .as_ref() + .iter() + .cloned() + .collect(); assert_eq!(root1, root2); } @@ -698,19 +701,15 @@ mod tests { #[test] fn branch_is_equivalent() { - let input: Vec<(&[u8], &[u8])> = vec![ - (&[0xaa][..], &[0x10][..]), - (&[0xba][..], &[0x11][..]), - ]; + let input: Vec<(&[u8], &[u8])> = + vec![(&[0xaa][..], &[0x10][..]), (&[0xba][..], &[0x11][..])]; check_input(&input); } #[test] fn extension_and_branch_is_equivalent() { - let input: Vec<(&[u8], &[u8])> = vec![ - (&[0xaa][..], &[0x10][..]), - (&[0xab][..], &[0x11][..]), - ]; + let input: Vec<(&[u8], &[u8])> = + vec![(&[0xaa][..], &[0x10][..]), (&[0xab][..], &[0x11][..])]; check_input(&input); } @@ -785,8 +784,8 @@ mod tests { v: &[(Vec, Vec)], layout: T, ) -> TrieDBMut<'db, T> - where - T: TrieConfiguration, + where + T: TrieConfiguration, { let mut t = TrieDBMut::::new_with_layout(db, root, layout); for i in 0..v.len() { @@ -827,11 +826,7 @@ mod tests { } .make_with(seed.as_fixed_bytes_mut()); - let layout = if flag { - Layout::with_alt_hashing(TRESHOLD) - } else { - Layout::default() - }; + let layout = if flag { Layout::with_alt_hashing(TRESHOLD) } else { Layout::default() }; let real = layout.trie_root(x.clone()); let mut memdb = MemoryDB::default(); let mut root = Default::default(); @@ -879,9 +874,7 @@ mod tests { #[test] fn codec_trie_single_tuple() { let layout = Layout::default(); - let input = vec![ - (vec![0xaa], vec![0xbb]) - ]; + let input = vec![(vec![0xaa], vec![0xbb])]; let trie = layout.trie_root_unhashed(input); println!("trie: {:#x?}", trie); assert_eq!( @@ -927,11 +920,7 @@ mod tests { iterator_works_inner(false); } fn iterator_works_inner(flag: bool) { - let layout = if flag { - Layout::with_alt_hashing(TRESHOLD) - } else { - Layout::default() - }; + let layout = if flag { Layout::with_alt_hashing(TRESHOLD) } else { Layout::default() }; let pairs = vec![ (hex!("0103000000000000000464").to_vec(), hex!("0400000000").to_vec()), @@ -1058,13 +1047,15 @@ mod tests { storage_root, valid_delta, Default::default(), - ).unwrap(); + ) + .unwrap(); let second_storage_root = delta_trie_root::( &mut proof_db.clone(), storage_root, invalid_delta, Default::default(), - ).unwrap(); + ) + .unwrap(); assert_eq!(first_storage_root, second_storage_root); } diff --git a/primitives/trie/src/node_codec.rs b/primitives/trie/src/node_codec.rs index 29108d76e7778..1c6a2bec72924 100644 --- a/primitives/trie/src/node_codec.rs +++ b/primitives/trie/src/node_codec.rs @@ -17,16 +17,16 @@ //! `NodeCodec` implementation for Substrate's trie format. -use sp_std::marker::PhantomData; -use sp_std::ops::Range; -use sp_std::vec::Vec; -use sp_std::borrow::Borrow; +use super::node_header::{NodeHeader, NodeKind}; use crate::{error::Error, trie_constants}; use codec::{Compact, Decode, Encode, Input}; use hash_db::Hasher; -use trie_db::{self, node::{NibbleSlicePlan, NodePlan, Value, ValuePlan, NodeHandlePlan}, - ChildReference, nibble_ops, Partial, NodeCodec as NodeCodecT, Meta}; -use super::{node_header::{NodeHeader, NodeKind}}; +use sp_std::{borrow::Borrow, marker::PhantomData, ops::Range, vec::Vec}; +use trie_db::{ + self, nibble_ops, + node::{NibbleSlicePlan, NodeHandlePlan, NodePlan, Value, ValuePlan}, + ChildReference, Meta, NodeCodec as NodeCodecT, Partial, +}; /// Helper struct for trie node decoder. This implements `codec::Input` on a byte slice, while /// tracking the absolute position. This is similar to `std::io::Cursor` but does not implement @@ -81,10 +81,7 @@ impl<'a> Input for ByteSliceInput<'a> { pub struct NodeCodec(PhantomData); impl NodeCodec { - fn decode_plan_inner_hashed( - data: &[u8], - meta: &mut Meta, - ) -> Result { + fn decode_plan_inner_hashed(data: &[u8], meta: &mut Meta) -> Result { let mut input = ByteSliceInput::new(data); let header = NodeHeader::decode(&mut input)?; @@ -101,8 +98,7 @@ impl NodeCodec { match header { NodeHeader::Null => Ok(NodePlan::Empty), - NodeHeader::AltHashBranch(nibble_count, _) - | NodeHeader::Branch(_, nibble_count) => { + NodeHeader::AltHashBranch(nibble_count, _) | NodeHeader::Branch(_, nibble_count) => { let padding = nibble_count % nibble_ops::NIBBLE_PER_BYTE != 0; // check that the padding is valid (if any) if padding && nibble_ops::pad_left(data[input.offset]) != 0 { @@ -147,8 +143,7 @@ impl NodeCodec { children, }) }, - NodeHeader::AltHashLeaf(nibble_count, _) - | NodeHeader::Leaf(nibble_count) => { + NodeHeader::AltHashLeaf(nibble_count, _) | NodeHeader::Leaf(nibble_count) => { let padding = nibble_count % nibble_ops::NIBBLE_PER_BYTE != 0; // check that the padding is valid (if any) if padding && nibble_ops::pad_left(data[input.offset]) != 0 { @@ -177,8 +172,8 @@ impl NodeCodec { } impl NodeCodecT for NodeCodec - where - H: Hasher, +where + H: Hasher, { const OFFSET_CONTAINS_HASH: usize = 1; type Error = Error; @@ -214,9 +209,12 @@ impl NodeCodecT for NodeCodec // With fix inner hashing alt hash can be use with all node, but // that is not better (encoding can use an additional nibble byte // sometime). - let mut output = if meta.try_inner_hashing.as_ref().map(|threshold| - value_do_hash(&value, threshold) - ).unwrap_or(meta.apply_inner_hashing) { + let mut output = if meta + .try_inner_hashing + .as_ref() + .map(|threshold| value_do_hash(&value, threshold)) + .unwrap_or(meta.apply_inner_hashing) + { if contains_hash { partial_encode(partial, NodeKind::AltHashLeafHash) } else { @@ -270,29 +268,37 @@ impl NodeCodecT for NodeCodec value: Value, meta: &mut Meta, ) -> Vec { - let contains_hash = matches!(&value, Value::HashedValue(..)); - let mut output = match (&value, meta.try_inner_hashing.as_ref().map(|threshold| - value_do_hash(&value, threshold) - ).unwrap_or(meta.apply_inner_hashing)) { - (&Value::NoValue, _) => { - partial_from_iterator_encode(partial, number_nibble, NodeKind::BranchNoValue) - }, - (_, false) => { - partial_from_iterator_encode(partial, number_nibble, NodeKind::BranchWithValue) - }, - (_, true) => { + let mut output = match ( + &value, + meta.try_inner_hashing + .as_ref() + .map(|threshold| value_do_hash(&value, threshold)) + .unwrap_or(meta.apply_inner_hashing), + ) { + (&Value::NoValue, _) => + partial_from_iterator_encode(partial, number_nibble, NodeKind::BranchNoValue), + (_, false) => + partial_from_iterator_encode(partial, number_nibble, NodeKind::BranchWithValue), + (_, true) => if contains_hash { - partial_from_iterator_encode(partial, number_nibble, NodeKind::AltHashBranchWithValueHash) + partial_from_iterator_encode( + partial, + number_nibble, + NodeKind::AltHashBranchWithValueHash, + ) } else { - partial_from_iterator_encode(partial, number_nibble, NodeKind::AltHashBranchWithValue) - } - }, + partial_from_iterator_encode( + partial, + number_nibble, + NodeKind::AltHashBranchWithValue, + ) + }, }; let bitmap_index = output.len(); let mut bitmap: [u8; BITMAP_LENGTH] = [0; BITMAP_LENGTH]; - (0..BITMAP_LENGTH).for_each(|_|output.push(0)); + (0..BITMAP_LENGTH).for_each(|_| output.push(0)); match value { Value::Value(value) => { let with_len = output.len(); @@ -311,17 +317,20 @@ impl NodeCodecT for NodeCodec }, Value::NoValue => (), } - Bitmap::encode(children.map(|maybe_child| match maybe_child.borrow() { - Some(ChildReference::Hash(h)) => { - h.as_ref().encode_to(&mut output); - true - } - &Some(ChildReference::Inline(inline_data, len)) => { - inline_data.as_ref()[..len].encode_to(&mut output); - true - } - None => false, - }), bitmap.as_mut()); + Bitmap::encode( + children.map(|maybe_child| match maybe_child.borrow() { + Some(ChildReference::Hash(h)) => { + h.as_ref().encode_to(&mut output); + true + }, + &Some(ChildReference::Inline(inline_data, len)) => { + inline_data.as_ref()[..len].encode_to(&mut output); + true + }, + None => false, + }), + bitmap.as_mut(), + ); output[bitmap_index..bitmap_index + BITMAP_LENGTH] .copy_from_slice(&bitmap[..BITMAP_LENGTH]); output @@ -332,13 +341,9 @@ impl NodeCodecT for NodeCodec fn value_do_hash(val: &Value, threshold: &u32) -> bool { match val { - Value::Value(val) => { - val.encoded_size() >= *threshold as usize - }, + Value::Value(val) => val.encoded_size() >= *threshold as usize, Value::HashedValue(..) => true, // can only keep hashed - Value::NoValue => { - false - }, + Value::NoValue => false, } } @@ -356,12 +361,14 @@ fn partial_from_iterator_encode>( NodeKind::Leaf => NodeHeader::Leaf(nibble_count).encode_to(&mut output), NodeKind::BranchWithValue => NodeHeader::Branch(true, nibble_count).encode_to(&mut output), NodeKind::BranchNoValue => NodeHeader::Branch(false, nibble_count).encode_to(&mut output), - NodeKind::AltHashLeaf => NodeHeader::AltHashLeaf(nibble_count, false).encode_to(&mut output), - NodeKind::AltHashBranchWithValue => NodeHeader::AltHashBranch(nibble_count, false) - .encode_to(&mut output), - NodeKind::AltHashLeafHash => NodeHeader::AltHashLeaf(nibble_count, true).encode_to(&mut output), - NodeKind::AltHashBranchWithValueHash => NodeHeader::AltHashBranch(nibble_count, true) - .encode_to(&mut output), + NodeKind::AltHashLeaf => + NodeHeader::AltHashLeaf(nibble_count, false).encode_to(&mut output), + NodeKind::AltHashBranchWithValue => + NodeHeader::AltHashBranch(nibble_count, false).encode_to(&mut output), + NodeKind::AltHashLeafHash => + NodeHeader::AltHashLeaf(nibble_count, true).encode_to(&mut output), + NodeKind::AltHashBranchWithValueHash => + NodeHeader::AltHashBranch(nibble_count, true).encode_to(&mut output), }; output.extend(partial); output @@ -380,12 +387,14 @@ fn partial_encode(partial: Partial, node_kind: NodeKind) -> Vec { NodeKind::Leaf => NodeHeader::Leaf(nibble_count).encode_to(&mut output), NodeKind::BranchWithValue => NodeHeader::Branch(true, nibble_count).encode_to(&mut output), NodeKind::BranchNoValue => NodeHeader::Branch(false, nibble_count).encode_to(&mut output), - NodeKind::AltHashLeaf => NodeHeader::AltHashLeaf(nibble_count, false).encode_to(&mut output), - NodeKind::AltHashBranchWithValue => NodeHeader::AltHashBranch(nibble_count, false) - .encode_to(&mut output), - NodeKind::AltHashLeafHash => NodeHeader::AltHashLeaf(nibble_count, true).encode_to(&mut output), - NodeKind::AltHashBranchWithValueHash => NodeHeader::AltHashBranch(nibble_count, true) - .encode_to(&mut output), + NodeKind::AltHashLeaf => + NodeHeader::AltHashLeaf(nibble_count, false).encode_to(&mut output), + NodeKind::AltHashBranchWithValue => + NodeHeader::AltHashBranch(nibble_count, false).encode_to(&mut output), + NodeKind::AltHashLeafHash => + NodeHeader::AltHashLeaf(nibble_count, true).encode_to(&mut output), + NodeKind::AltHashBranchWithValueHash => + NodeHeader::AltHashBranch(nibble_count, true).encode_to(&mut output), }; if number_nibble_encoded > 0 { output.push(nibble_ops::pad_right((partial.0).1)); diff --git a/primitives/trie/src/node_header.rs b/primitives/trie/src/node_header.rs index fd782981d4b5a..2443ad03dc53e 100644 --- a/primitives/trie/src/node_header.rs +++ b/primitives/trie/src/node_header.rs @@ -38,8 +38,7 @@ pub(crate) enum NodeHeader { impl NodeHeader { pub(crate) fn contains_hash_of_value(&self) -> bool { match self { - NodeHeader::AltHashBranch(_, true) - | NodeHeader::AltHashLeaf(_, true) => true, + NodeHeader::AltHashBranch(_, true) | NodeHeader::AltHashLeaf(_, true) => true, _ => false, } } @@ -63,16 +62,28 @@ impl Encode for NodeHeader { } match self { NodeHeader::Null => output.push_byte(trie_constants::EMPTY_TRIE), - NodeHeader::Branch(true, nibble_count) => + NodeHeader::Branch(true, nibble_count) => encode_size_and_prefix(*nibble_count, trie_constants::BRANCH_WITH_MASK, 2, output), - NodeHeader::Branch(false, nibble_count) => - encode_size_and_prefix(*nibble_count, trie_constants::BRANCH_WITHOUT_MASK, 2, output), + NodeHeader::Branch(false, nibble_count) => encode_size_and_prefix( + *nibble_count, + trie_constants::BRANCH_WITHOUT_MASK, + 2, + output, + ), NodeHeader::Leaf(nibble_count) => encode_size_and_prefix(*nibble_count, trie_constants::LEAF_PREFIX_MASK, 2, output), - NodeHeader::AltHashBranch(nibble_count, _) => - encode_size_and_prefix(*nibble_count, trie_constants::ALT_HASHING_BRANCH_WITH_MASK, 4, output), - NodeHeader::AltHashLeaf(nibble_count, _) => - encode_size_and_prefix(*nibble_count, trie_constants::ALT_HASHING_LEAF_PREFIX_MASK, 3, output), + NodeHeader::AltHashBranch(nibble_count, _) => encode_size_and_prefix( + *nibble_count, + trie_constants::ALT_HASHING_BRANCH_WITH_MASK, + 4, + output, + ), + NodeHeader::AltHashLeaf(nibble_count, _) => encode_size_and_prefix( + *nibble_count, + trie_constants::ALT_HASHING_LEAF_PREFIX_MASK, + 3, + output, + ), } } } @@ -81,11 +92,8 @@ impl NodeHeader { /// Is this header using alternate hashing scheme. pub(crate) fn alt_hashing(&self) -> bool { match self { - NodeHeader::Null - | NodeHeader::Leaf(..) - | NodeHeader::Branch(..) => false, - NodeHeader::AltHashBranch(..) - | NodeHeader::AltHashLeaf(..) => true, + NodeHeader::Null | NodeHeader::Leaf(..) | NodeHeader::Branch(..) => false, + NodeHeader::AltHashBranch(..) | NodeHeader::AltHashLeaf(..) => true, } } } @@ -106,8 +114,10 @@ impl Decode for NodeHeader { }; match i & (0b11 << 6) { trie_constants::LEAF_PREFIX_MASK => Ok(NodeHeader::Leaf(decode_size(i, input, 2)?)), - trie_constants::BRANCH_WITH_MASK => Ok(NodeHeader::Branch(true, decode_size(i, input, 2)?)), - trie_constants::BRANCH_WITHOUT_MASK => Ok(NodeHeader::Branch(false, decode_size(i, input, 2)?)), + trie_constants::BRANCH_WITH_MASK => + Ok(NodeHeader::Branch(true, decode_size(i, input, 2)?)), + trie_constants::BRANCH_WITHOUT_MASK => + Ok(NodeHeader::Branch(false, decode_size(i, input, 2)?)), trie_constants::EMPTY_TRIE => { if i & (0b111 << 5) == trie_constants::ALT_HASHING_LEAF_PREFIX_MASK { Ok(NodeHeader::AltHashLeaf(decode_size(i, input, 3)?, contain_hash)) @@ -159,7 +169,8 @@ pub(crate) fn size_and_prefix_iterator( /// Encodes size and prefix to a stream output (prefix on 2 first bit only). fn encode_size_and_prefix(size: usize, prefix: u8, prefix_mask: usize, out: &mut W) - where W: Output + ?Sized, +where + W: Output + ?Sized, { for b in size_and_prefix_iterator(size, prefix, prefix_mask) { out.push_byte(b) @@ -175,7 +186,7 @@ fn decode_size( let max_value = 255u8 >> prefix_mask; let mut result = (first & max_value) as usize; if result < max_value as usize { - return Ok(result); + return Ok(result) } result -= 1; while result <= trie_constants::NIBBLE_SIZE_BOUND { diff --git a/primitives/trie/src/storage_proof.rs b/primitives/trie/src/storage_proof.rs index e0344e9e02bdb..d9c57c7e8a95d 100644 --- a/primitives/trie/src/storage_proof.rs +++ b/primitives/trie/src/storage_proof.rs @@ -15,11 +15,11 @@ // See the License for the specific language governing permissions and // limitations under the License. +use crate::{Layout, TrieLayout}; use codec::{Decode, Encode}; use hash_db::{HashDB, Hasher}; use sp_std::vec::Vec; use trie_db::NodeCodec; -use crate::{Layout, TrieLayout}; /// A proof that some set of key-value pairs are included in the storage trie. The proof contains /// the storage values so that the partial storage backend can be reconstructed by a verifier that @@ -178,11 +178,11 @@ impl From for crate::MemoryDB { for item in proof.trie_nodes.iter() { let mut meta = Default::default(); // Read meta from state (required for value layout). - let _ = as TrieLayout>::Codec::decode_plan(item.as_slice(), &mut meta); + let _ = as TrieLayout>::Codec::decode_plan(item.as_slice(), &mut meta); db.alt_insert( crate::EMPTY_PREFIX, item, - meta.resolve_alt_hashing::< as TrieLayout>::Codec>(), + meta.resolve_alt_hashing::< as TrieLayout>::Codec>(), ); } db diff --git a/primitives/trie/src/trie_codec.rs b/primitives/trie/src/trie_codec.rs index 52a6c550da976..ad6228b30c511 100644 --- a/primitives/trie/src/trie_codec.rs +++ b/primitives/trie/src/trie_codec.rs @@ -113,11 +113,8 @@ where let mut nodes_iter = encoded.into_iter(); // Layout does not change trie reading. let layout = L::default(); - let (top_root, _nb_used) = trie_db::decode_compact_from_iter::( - db, - &mut nodes_iter, - &layout, - )?; + let (top_root, _nb_used) = + trie_db::decode_compact_from_iter::(db, &mut nodes_iter, &layout)?; // Only check root if expected root is passed as argument. if let Some(expected_root) = expected_root { @@ -167,11 +164,8 @@ where let mut previous_extracted_child_trie = None; for child_root in child_tries.into_iter() { if previous_extracted_child_trie.is_none() { - let (top_root, _) = trie_db::decode_compact_from_iter::( - db, - &mut nodes_iter, - &layout, - )?; + let (top_root, _) = + trie_db::decode_compact_from_iter::(db, &mut nodes_iter, &layout)?; previous_extracted_child_trie = Some(top_root); } diff --git a/primitives/trie/src/trie_stream.rs b/primitives/trie/src/trie_stream.rs index 52851bdba933c..aba5ea3d4aa14 100644 --- a/primitives/trie/src/trie_stream.rs +++ b/primitives/trie/src/trie_stream.rs @@ -22,11 +22,10 @@ use crate::{ node_header::{size_and_prefix_iterator, NodeKind}, trie_constants, }; -use codec::{Encode, Compact}; +use codec::{Compact, Encode}; use hash_db::Hasher; -use sp_std::vec::Vec; +use sp_std::{ops::Range, vec::Vec}; use trie_root; -use sp_std::ops::Range; const BRANCH_NODE_NO_VALUE: u8 = 254; const BRANCH_NODE_WITH_VALUE: u8 = 255; @@ -69,14 +68,16 @@ fn fuse_nibbles_node<'a>(nibbles: &'a [u8], kind: NodeKind) -> impl Iterator size_and_prefix_iterator(size, trie_constants::LEAF_PREFIX_MASK, 2), - NodeKind::BranchNoValue => size_and_prefix_iterator(size, trie_constants::BRANCH_WITHOUT_MASK, 2), - NodeKind::BranchWithValue => size_and_prefix_iterator(size, trie_constants::BRANCH_WITH_MASK, 2), + NodeKind::BranchNoValue => + size_and_prefix_iterator(size, trie_constants::BRANCH_WITHOUT_MASK, 2), + NodeKind::BranchWithValue => + size_and_prefix_iterator(size, trie_constants::BRANCH_WITH_MASK, 2), NodeKind::AltHashLeaf => size_and_prefix_iterator(size, trie_constants::ALT_HASHING_LEAF_PREFIX_MASK, 3), NodeKind::AltHashBranchWithValue => size_and_prefix_iterator(size, trie_constants::ALT_HASHING_BRANCH_WITH_MASK, 4), - NodeKind::AltHashBranchWithValueHash - | NodeKind::AltHashLeafHash => unreachable!("only added value that do not contain hash"), + NodeKind::AltHashBranchWithValueHash | NodeKind::AltHashLeafHash => + unreachable!("only added value that do not contain hash"), }; iter_start .chain(if nibbles.len() % 2 == 1 { Some(nibbles[0]) } else { None }) @@ -98,14 +99,12 @@ impl trie_root::TrieStream for TrieStream { } fn append_leaf(&mut self, key: &[u8], value: &[u8]) { - self.apply_inner_hashing = self.inner_value_hashing.as_ref().map(|threshold| - value_do_hash(value, threshold) - ).unwrap_or(false); - let kind = if self.apply_inner_hashing { - NodeKind::AltHashLeaf - } else { - NodeKind::Leaf - }; + self.apply_inner_hashing = self + .inner_value_hashing + .as_ref() + .map(|threshold| value_do_hash(value, threshold)) + .unwrap_or(false); + let kind = if self.apply_inner_hashing { NodeKind::AltHashLeaf } else { NodeKind::Leaf }; self.buffer.extend(fuse_nibbles_node(key, kind)); let start = self.buffer.len(); Compact(value.len() as u32).encode_to(&mut self.buffer); @@ -121,9 +120,11 @@ impl trie_root::TrieStream for TrieStream { ) { if let Some(partial) = maybe_partial { if let Some(value) = maybe_value { - self.apply_inner_hashing = self.inner_value_hashing.as_ref().map(|threshold| - value_do_hash(value, threshold) - ).unwrap_or(false); + self.apply_inner_hashing = self + .inner_value_hashing + .as_ref() + .map(|threshold| value_do_hash(value, threshold)) + .unwrap_or(false); let kind = if self.apply_inner_hashing { NodeKind::AltHashBranchWithValue } else { @@ -157,17 +158,18 @@ impl trie_root::TrieStream for TrieStream { let data = other.out(); match data.len() { 0..=31 => data.encode_to(&mut self.buffer), - _ => { + _ => if apply_inner_hashing { hash_db::AltHashing { encoded_offset: 0, value_range: range.map(|r| (r.start, r.end)), - }.alt_hash::(&data).as_ref() - .encode_to(&mut self.buffer); + } + .alt_hash::(&data) + .as_ref() + .encode_to(&mut self.buffer); } else { H::hash(&data).as_ref().encode_to(&mut self.buffer); - } - }, + }, } } @@ -176,10 +178,8 @@ impl trie_root::TrieStream for TrieStream { let range = self.current_value_range; let data = self.buffer; if apply_inner_hashing { - hash_db::AltHashing { - encoded_offset: 0, - value_range: range.map(|r| (r.start, r.end)), - }.alt_hash::(&data) + hash_db::AltHashing { encoded_offset: 0, value_range: range.map(|r| (r.start, r.end)) } + .alt_hash::(&data) } else { H::hash(&data) } diff --git a/test-utils/client/src/lib.rs b/test-utils/client/src/lib.rs index b1c0b9299ae0b..1c5135ff556ec 100644 --- a/test-utils/client/src/lib.rs +++ b/test-utils/client/src/lib.rs @@ -231,9 +231,9 @@ impl let storage = { let mut storage = self.genesis_init.genesis_storage(); if self.state_hashed_value { - storage.modify_trie_alt_hashing_threshold( - Some(sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD), - ); + storage.modify_trie_alt_hashing_threshold(Some( + sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD, + )); } // Add some child storage keys. diff --git a/test-utils/runtime/client/src/lib.rs b/test-utils/runtime/client/src/lib.rs index 29b7a423c350b..d5cd4d3ab1958 100644 --- a/test-utils/runtime/client/src/lib.rs +++ b/test-utils/runtime/client/src/lib.rs @@ -394,7 +394,9 @@ pub fn new(hashed_state: bool) -> Client { } /// Creates new light client instance used for tests. -pub fn new_light(hashed_state: bool) -> ( +pub fn new_light( + hashed_state: bool, +) -> ( client::Client< LightBackend, LightExecutor, @@ -420,10 +422,7 @@ pub fn new_light(hashed_state: bool) -> ( if hashed_state { builder = builder.state_hashed_value(); } - ( - builder.build_with_executor(call_executor).0, - backend, - ) + (builder.build_with_executor(call_executor).0, backend) } /// Creates new light client fetcher used for tests. diff --git a/utils/frame/rpc/system/src/lib.rs b/utils/frame/rpc/system/src/lib.rs index 2c8a9ba6f02d9..4126c1f4401fe 100644 --- a/utils/frame/rpc/system/src/lib.rs +++ b/utils/frame/rpc/system/src/lib.rs @@ -325,7 +325,7 @@ mod tests { sp_tracing::try_init_simple(); // given - let client = Arc::new(substrate_test_runtime_client::new(true,)); + let client = Arc::new(substrate_test_runtime_client::new(true)); let spawner = sp_core::testing::TaskExecutor::new(); let pool = BasicPool::new_full(Default::default(), true.into(), None, spawner, client.clone()); From 1e02c0166034df496167b1802eb25cef84fe1cc1 Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Mon, 16 Aug 2021 11:56:23 +0200 Subject: [PATCH 066/127] rustfmt --- client/light/src/fetcher.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/client/light/src/fetcher.rs b/client/light/src/fetcher.rs index 438d6b2fc114e..fba3d854b8d2b 100644 --- a/client/light/src/fetcher.rs +++ b/client/light/src/fetcher.rs @@ -169,7 +169,8 @@ impl> LightDataChecker { remote_roots_proof: StorageProof, ) -> ClientResult<()> { // all the checks are sharing the same storage - let storage: sp_state_machine::MemoryDB> = remote_roots_proof.into_memory_db_no_meta(); + let storage: sp_state_machine::MemoryDB> = + remote_roots_proof.into_memory_db_no_meta(); // remote_roots.keys() are sorted => we can use this to group changes tries roots // that are belongs to the same CHT From 50ca65208c36e0dcec9971a83c3b6a3b9e341e9d Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Tue, 17 Aug 2021 19:46:48 +0200 Subject: [PATCH 067/127] alt hashing as trie backend parameter. --- client/api/src/cht.rs | 3 +- primitives/state-machine/src/basic.rs | 5 +- .../state-machine/src/in_memory_backend.rs | 48 ++++++++++++------- .../state-machine/src/proving_backend.rs | 13 +++-- primitives/state-machine/src/testing.rs | 25 ++-------- primitives/state-machine/src/trie_backend.rs | 9 ++-- primitives/trie/src/storage_proof.rs | 23 ++++++--- primitives/trie/src/trie_codec.rs | 3 +- test-utils/runtime/src/lib.rs | 3 +- 9 files changed, 71 insertions(+), 61 deletions(-) diff --git a/client/api/src/cht.rs b/client/api/src/cht.rs index 6e754660b9306..48a882cfbdb1b 100644 --- a/client/api/src/cht.rs +++ b/client/api/src/cht.rs @@ -117,7 +117,8 @@ where .into_iter() .map(|(k, v)| (k, Some(v))) .collect::>(); - let mut storage = InMemoryBackend::::default().update(vec![(None, transaction)]); + let backend: InMemoryBackend = None.into(); // No inner hashing for cht. + let mut storage = backend.update(vec![(None, transaction)]); let trie_storage = storage .as_trie_backend() .expect("InMemoryState::as_trie_backend always returns Some; qed"); diff --git a/primitives/state-machine/src/basic.rs b/primitives/state-machine/src/basic.rs index 3d7ac00c18184..627194654795a 100644 --- a/primitives/state-machine/src/basic.rs +++ b/primitives/state-machine/src/basic.rs @@ -42,7 +42,7 @@ use std::{ pub struct BasicExternalities { inner: Storage, extensions: Extensions, - alt_hashing: Option, + alt_hashing: Option, // TODO same alt hashing type as all } impl BasicExternalities { @@ -311,8 +311,7 @@ impl Externalities for BasicExternalities { fn child_storage_root(&mut self, child_info: &ChildInfo) -> Vec { if let Some(child) = self.inner.children_default.get(child_info.storage_key()) { let delta = child.data.iter().map(|(k, v)| (k.as_ref(), Some(v.as_ref()))); - let mut in_mem = crate::in_memory_backend::new_in_mem::(); - in_mem.force_alt_hashing(self.alt_hashing.clone()); + let in_mem = crate::in_memory_backend::new_in_mem::(Some(self.alt_hashing.clone())); in_mem.child_storage_root(&child.child_info, delta).0 } else { empty_child_trie_root::>() diff --git a/primitives/state-machine/src/in_memory_backend.rs b/primitives/state-machine/src/in_memory_backend.rs index 62c6b1a59d008..ae7c88de67e46 100644 --- a/primitives/state-machine/src/in_memory_backend.rs +++ b/primitives/state-machine/src/in_memory_backend.rs @@ -27,12 +27,12 @@ use sp_trie::{empty_trie_root, Layout, MemoryDB}; use std::collections::{BTreeMap, HashMap}; /// Create a new empty instance of in-memory backend. -pub fn new_in_mem() -> TrieBackend, H> +pub fn new_in_mem(alt_hashing: Option>) -> TrieBackend, H> where H::Out: Codec + Ord, { let db = MemoryDB::default(); - TrieBackend::new(db, empty_trie_root::>()) + TrieBackend::new(db, empty_trie_root::>(), alt_hashing) } impl TrieBackend, H> @@ -69,7 +69,7 @@ where pub fn update_backend(&self, root: H::Out, changes: MemoryDB) -> Self { let mut clone = self.backend_storage().clone(); clone.consolidate(changes); - Self::new(clone, root) + Self::new(clone, root, self.force_alt_hashing.clone()) } /// Apply the given transaction to this backend and set the root to the given value. @@ -83,7 +83,7 @@ where self.root() == other.root() } - /// Setting a alternate hashing threshold at start. + /// Setting a alternate hashing threshold at start. TODO remove (on construct). pub fn force_alt_hashing(&mut self, threshold: Option) { self.force_alt_hashing = Some(threshold); } @@ -94,26 +94,38 @@ where H::Out: Codec + Ord, { fn clone(&self) -> Self { - TrieBackend::new(self.backend_storage().clone(), self.root().clone()) + TrieBackend::new(self.backend_storage().clone(), self.root().clone(), self.force_alt_hashing.clone()) } } +/* impl Default for TrieBackend, H> where H::Out: Codec + Ord, { fn default() -> Self { - new_in_mem() + new_in_mem() // TODO try removing this default } } +*/ -impl From, BTreeMap>> +impl From>> for TrieBackend, H> where H::Out: Codec + Ord, { - fn from(inner: HashMap, BTreeMap>) -> Self { - let mut backend = new_in_mem(); + fn from(alt_hashing: Option>) -> Self { + new_in_mem(alt_hashing) + } +} + +impl From<(HashMap, BTreeMap>, Option>)> + for TrieBackend, H> +where + H::Out: Codec + Ord, +{ + fn from((inner, alt_hashing): (HashMap, BTreeMap>, Option>)) -> Self { + let mut backend = new_in_mem(alt_hashing); backend.insert( inner .into_iter() @@ -123,37 +135,37 @@ where } } -impl From for TrieBackend, H> +impl From<(Storage, Option>)> for TrieBackend, H> where H::Out: Codec + Ord, { - fn from(inners: Storage) -> Self { + fn from((inners, alt_hashing): (Storage, Option>)) -> Self { let mut inner: HashMap, BTreeMap> = inners .children_default .into_iter() .map(|(_k, c)| (Some(c.child_info), c.data)) .collect(); inner.insert(None, inners.top); - inner.into() + (inner, alt_hashing).into() } } -impl From> for TrieBackend, H> +impl From<(BTreeMap, Option>)> for TrieBackend, H> where H::Out: Codec + Ord, { - fn from(inner: BTreeMap) -> Self { + fn from((inner, alt_hashing): (BTreeMap, Option>)) -> Self { let mut expanded = HashMap::new(); expanded.insert(None, inner); - expanded.into() + (expanded, alt_hashing).into() } } -impl From, StorageCollection)>> for TrieBackend, H> +impl From<(Vec<(Option, StorageCollection)>, Option>)> for TrieBackend, H> where H::Out: Codec + Ord, { - fn from(inner: Vec<(Option, StorageCollection)>) -> Self { + fn from((inner, alt_hashing): (Vec<(Option, StorageCollection)>, Option>)) -> Self { let mut expanded: HashMap, BTreeMap> = HashMap::new(); for (child_info, key_values) in inner { @@ -164,7 +176,7 @@ where } } } - expanded.into() + (expanded, alt_hashing).into() } } diff --git a/primitives/state-machine/src/proving_backend.rs b/primitives/state-machine/src/proving_backend.rs index c09e0bff075ce..d62f6ed6ef55c 100644 --- a/primitives/state-machine/src/proving_backend.rs +++ b/primitives/state-machine/src/proving_backend.rs @@ -172,7 +172,8 @@ impl ProofRecorder { } /// Convert into a [`StorageProof`]. - pub fn to_storage_proof(&self) -> StorageProof { + pub fn to_storage_proof(&self, alt_hashing: Option>) -> StorageProof { + // TODO consider alt_hashing as inner field let trie_nodes = self .inner .read() @@ -192,7 +193,7 @@ impl ProofRecorder { }) .collect(); - StorageProof::new(trie_nodes) + StorageProof::new(trie_nodes, alt_hashing) } /// Reset the internal state. @@ -233,12 +234,13 @@ where let essence = backend.essence(); let root = essence.root().clone(); let recorder = ProofRecorderBackend { backend: essence.backend_storage(), proof_recorder }; - ProvingBackend(TrieBackend::new(recorder, root)) + ProvingBackend(TrieBackend::new(recorder, root, backend.force_alt_hashing.clone())) } /// Extracting the gathered unordered proof. pub fn extract_proof(&self) -> StorageProof { - self.0.essence().backend_storage().proof_recorder.to_storage_proof::() + let alt_hashing = self.0.force_alt_hashing.clone(); + self.0.essence().backend_storage().proof_recorder.to_storage_proof::(alt_hashing) } /// Returns the estimated encoded size of the proof. @@ -398,10 +400,11 @@ where H: Hasher, H::Out: Codec, { + let alt_hashing = proof.alt_hashing.clone(); let db = proof.into_memory_db(); if db.contains(&root, EMPTY_PREFIX) { - Ok(TrieBackend::new(db, root)) + Ok(TrieBackend::new(db, root, alt_hashing)) } else { Err(Box::new(ExecutionError::InvalidProof)) } diff --git a/primitives/state-machine/src/testing.rs b/primitives/state-machine/src/testing.rs index ce7aaee289c12..1c2b7a3cad600 100644 --- a/primitives/state-machine/src/testing.rs +++ b/primitives/state-machine/src/testing.rs @@ -19,7 +19,6 @@ use std::{ any::{Any, TypeId}, - collections::{BTreeMap, HashMap}, panic::{AssertUnwindSafe, UnwindSafe}, }; @@ -39,7 +38,7 @@ use sp_core::{ offchain::testing::TestPersistentOffchainDB, storage::{ well_known_keys::{is_child_storage_key, CHANGES_TRIE_CONFIG, CODE}, - ChildInfo, Storage, + Storage, }, testing::TaskExecutor, traits::TaskExecutorExt, @@ -126,27 +125,9 @@ where let offchain_db = TestPersistentOffchainDB::new(); let backend = if force_alt_hashing { - let mut backend: InMemoryBackend = { - let mut storage = Storage::default(); - storage.modify_trie_alt_hashing_threshold(Some( - sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD, - )); - storage.into() - }; - let mut inner: HashMap, BTreeMap> = storage - .children_default - .into_iter() - .map(|(_k, c)| (Some(c.child_info), c.data)) - .collect(); - inner.insert(None, storage.top); - backend.insert( - inner - .into_iter() - .map(|(k, m)| (k, m.into_iter().map(|(k, v)| (k, Some(v))).collect())), - ); - backend + (storage, Some(Some(sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD))).into() } else { - storage.into() + (storage, None).into() }; TestExternalities { diff --git a/primitives/state-machine/src/trie_backend.rs b/primitives/state-machine/src/trie_backend.rs index 052752a7115de..d6f68077a467a 100644 --- a/primitives/state-machine/src/trie_backend.rs +++ b/primitives/state-machine/src/trie_backend.rs @@ -35,10 +35,11 @@ use sp_trie::{ /// Patricia trie-based backend. Transaction type is an overlay of changes to commit. pub struct TrieBackend, H: Hasher> { pub(crate) essence: TrieBackendEssence, + /// TODO remove pub // Allows setting alt hashing at start for testing only // (mainly for in_memory_backend when it cannot read it from - // state). - pub(crate) force_alt_hashing: Option>, + // state). TODO rename alt_hashing TODO accessor and no public. + pub force_alt_hashing: Option>, } impl, H: Hasher> TrieBackend @@ -46,8 +47,8 @@ where H::Out: Codec, { /// Create new trie-based backend. - pub fn new(storage: S, root: H::Out) -> Self { - TrieBackend { essence: TrieBackendEssence::new(storage, root), force_alt_hashing: None } + pub fn new(storage: S, root: H::Out, alt_hashing: Option>) -> Self { + TrieBackend { essence: TrieBackendEssence::new(storage, root), force_alt_hashing: alt_hashing } } /// Get backend essence reference. diff --git a/primitives/trie/src/storage_proof.rs b/primitives/trie/src/storage_proof.rs index 94dc9c1ea8a13..865a5bbca8d00 100644 --- a/primitives/trie/src/storage_proof.rs +++ b/primitives/trie/src/storage_proof.rs @@ -31,26 +31,29 @@ use trie_db::NodeCodec; #[derive(Debug, PartialEq, Eq, Clone, Encode, Decode)] pub struct StorageProof { trie_nodes: Vec>, + pub alt_hashing: Option>, // TODO non public } /// Storage proof in compact form. #[derive(Debug, PartialEq, Eq, Clone, Encode, Decode)] pub struct CompactProof { pub encoded_nodes: Vec>, + pub alt_hashing: Option>, } impl StorageProof { /// Constructs a storage proof from a subset of encoded trie nodes in a storage backend. - pub fn new(trie_nodes: Vec>) -> Self { - StorageProof { trie_nodes } + pub fn new(trie_nodes: Vec>, alt_hashing: Option>) -> Self { + StorageProof { trie_nodes, alt_hashing } } /// Returns a new empty proof. /// /// An empty proof is capable of only proving trivial statements (ie. that an empty set of /// key-value pairs exist in storage). - pub fn empty() -> Self { - StorageProof { trie_nodes: Vec::new() } + pub fn empty(alt_hashing: Option>) -> Self { + // TODO consider alt_hashing default value. + StorageProof { trie_nodes: Vec::new(), alt_hashing } } /// Returns whether this is an empty proof. @@ -90,14 +93,20 @@ impl StorageProof { where I: IntoIterator, { + let mut alt_hashing = None; + let alt_hashing = &mut alt_hashing; let trie_nodes = proofs .into_iter() - .flat_map(|proof| proof.iter_nodes()) + .flat_map(|proof| { + debug_assert!(alt_hashing == &None || alt_hashing == &proof.alt_hashing); + *alt_hashing = proof.alt_hashing.clone(); + proof.iter_nodes() + }) .collect::>() .into_iter() .collect(); - Self { trie_nodes } + Self { trie_nodes, alt_hashing: *alt_hashing } } /// Encode as a compact proof with default @@ -134,6 +143,7 @@ impl CompactProof { &self, expected_root: Option<&H::Out>, ) -> Result<(StorageProof, H::Out), crate::CompactProofError>> { + let alt_hashing = self.alt_hashing.clone(); let mut db = crate::MemoryDB::::new(&[]); let root = crate::decode_compact::, _, _>( &mut db, @@ -146,6 +156,7 @@ impl CompactProof { .into_iter() .filter_map(|kv| if (kv.1).1 > 0 { Some((kv.1).0) } else { None }) .collect(), + alt_hashing, ), root, )) diff --git a/primitives/trie/src/trie_codec.rs b/primitives/trie/src/trie_codec.rs index 4fe7750a552d2..a69500d446a9d 100644 --- a/primitives/trie/src/trie_codec.rs +++ b/primitives/trie/src/trie_codec.rs @@ -202,6 +202,7 @@ pub fn encode_compact(proof: StorageProof, root: TrieHash) -> Result = proof.into_memory_db(); - let backend = sp_state_machine::TrieBackend::<_, crate::Hashing>::new(db, root); + let backend = sp_state_machine::TrieBackend::<_, crate::Hashing>::new(db, root, alt_hashing); let mut overlay = sp_state_machine::OverlayedChanges::default(); let mut cache = sp_state_machine::StorageTransactionCache::<_, _, BlockNumber>::default(); let mut ext = sp_state_machine::Ext::new( From bd394d5b09c2ce724e110c3ccc52eebf9f17b19d Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Thu, 19 Aug 2021 12:35:47 +0200 Subject: [PATCH 068/127] initial config usage, needs global changes to use dedicated primitive types. --- bin/node/bench/src/trie.rs | 3 +- bin/node/testing/src/bench.rs | 5 +- client/api/src/cht.rs | 2 +- client/api/src/in_mem.rs | 34 +++++++++- client/block-builder/src/lib.rs | 7 ++- client/chain-spec/src/chain_spec.rs | 13 ++++ client/chain-spec/src/lib.rs | 3 + client/db/src/bench.rs | 12 +++- client/db/src/lib.rs | 62 ++++++++++++++++--- client/db/src/storage_cache.rs | 8 +++ client/light/src/backend.rs | 46 ++++++++++---- client/light/src/fetcher.rs | 2 +- client/light/src/lib.rs | 6 +- .../src/light_client_requests/handler.rs | 24 +++---- client/service/src/builder.rs | 34 +++++++--- client/service/src/client/light.rs | 10 +-- .../api/proc-macro/src/impl_runtime_apis.rs | 4 +- .../proc-macro/src/mock_impl_runtime_apis.rs | 2 +- primitives/api/src/lib.rs | 2 +- primitives/state-machine/src/backend.rs | 3 + .../state-machine/src/proving_backend.rs | 4 ++ primitives/state-machine/src/trie_backend.rs | 4 ++ primitives/trie/src/storage_proof.rs | 2 +- test-utils/client/src/lib.rs | 4 +- test-utils/runtime/client/src/lib.rs | 7 ++- 25 files changed, 235 insertions(+), 68 deletions(-) diff --git a/bin/node/bench/src/trie.rs b/bin/node/bench/src/trie.rs index 77ca3e85b8b05..9452203e704a4 100644 --- a/bin/node/bench/src/trie.rs +++ b/bin/node/bench/src/trie.rs @@ -186,7 +186,8 @@ impl core::Benchmark for TrieReadBenchmark { let storage: Arc> = Arc::new(Storage(db.open(self.database_type))); - let trie_backend = sp_state_machine::TrieBackend::new(storage, self.root); + let state_version = None; // TODO switch to default/latest version. + let trie_backend = sp_state_machine::TrieBackend::new(storage, self.root, state_version); for (warmup_key, warmup_value) in self.warmup_keys.iter() { let value = trie_backend .storage(&warmup_key[..]) diff --git a/bin/node/testing/src/bench.rs b/bin/node/testing/src/bench.rs index f6ed2418410ed..b4cf2e594aacb 100644 --- a/bin/node/testing/src/bench.rs +++ b/bin/node/testing/src/bench.rs @@ -34,7 +34,7 @@ use crate::{ }; use codec::{Decode, Encode}; use futures::executor; -use node_primitives::Block; +use node_primitives::{Block, BlockNumber}; use node_runtime::{ constants::currency::DOLLARS, AccountId, BalancesCall, Call, CheckedExtrinsic, MinimumPeriod, Signature, SystemCall, UncheckedExtrinsic, @@ -387,7 +387,8 @@ impl BenchDb { }; let task_executor = TaskExecutor::new(); - let backend = sc_service::new_db_backend(db_config).expect("Should not fail"); + let state_versions = vec![(BlockNumber::zero(), None)]; // TODO switch to default (more recent). + let backend = sc_service::new_db_backend(db_config, state_versions).expect("Should not fail"); let client = sc_service::new_client( backend.clone(), NativeExecutor::new(WasmExecutionMethod::Compiled, None, 8), diff --git a/client/api/src/cht.rs b/client/api/src/cht.rs index 48a882cfbdb1b..9739b4cfd5e46 100644 --- a/client/api/src/cht.rs +++ b/client/api/src/cht.rs @@ -117,7 +117,7 @@ where .into_iter() .map(|(k, v)| (k, Some(v))) .collect::>(); - let backend: InMemoryBackend = None.into(); // No inner hashing for cht. + let backend: InMemoryBackend = None.into(); // No inner hashing for cht. let mut storage = backend.update(vec![(None, transaction)]); let trie_storage = storage .as_trie_backend() diff --git a/client/api/src/in_mem.rs b/client/api/src/in_mem.rs index e8fce19f8124e..07bceafcd6287 100644 --- a/client/api/src/in_mem.rs +++ b/client/api/src/in_mem.rs @@ -112,6 +112,13 @@ struct BlockchainStorage { changes_trie_cht_roots: HashMap, Block::Hash>, leaves: LeafSet>, aux: HashMap, Vec>, + // TODO rename to state_versions and replace Option> to StateVersionEnum. + alt_hashing: Vec<(NumberFor, Option>)>, + /* + // TODO not sure if migration support. TODO replace () by StateMigration. + state_migration: Vec<(NumberFor, ())>, + * Probably not (client/api). + */ } /// In-memory blockchain. Supports concurrent reads. @@ -141,6 +148,29 @@ impl Blockchain { } } + /// Get version of state. + pub fn state_version(&self, id: BlockId) -> Option> { + let default = Some(Some(sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD)); + let number = match id { + BlockId::Hash(h) => { + if let Ok(Some(header)) = self.header(BlockId::Hash(h)) { + header.number().clone() + } else { + return default // Default to latest version TODO a latest version default with StateVersion. + } + }, + BlockId::Number(n) => n, + }; + let mut result = default; + for (change_state, state) in self.storage.read().alt_hashing.iter() { + if &number > change_state { + break + } + result = *state; + } + result + } + /// Create new in-memory blockchain storage. pub fn new() -> Blockchain { let storage = Arc::new(RwLock::new(BlockchainStorage { @@ -155,6 +185,7 @@ impl Blockchain { changes_trie_cht_roots: HashMap::new(), leaves: LeafSet::new(), aux: HashMap::new(), + alt_hashing: Vec::new(), })); Blockchain { storage } } @@ -855,7 +886,8 @@ where fn state_at(&self, block: BlockId) -> sp_blockchain::Result { match block { - BlockId::Hash(h) if h == Default::default() => return Ok(Self::State::default()), + BlockId::Hash(h) if h == Default::default() => + return Ok(self.blockchain.state_version(BlockId::Hash(h)).into()), _ => {}, } diff --git a/client/block-builder/src/lib.rs b/client/block-builder/src/lib.rs index ae8665ba4cfea..bc06b0eca329d 100644 --- a/client/block-builder/src/lib.rs +++ b/client/block-builder/src/lib.rs @@ -234,9 +234,12 @@ where ), ); - let proof = self.api.extract_proof(); - let state = self.backend.state_at(self.block_id)?; + let proof = { + use backend::StateBackend; + self.api.extract_proof(state.alt_hashing().clone()) + }; + let changes_trie_state = backend::changes_tries_state_at_block( &self.block_id, self.backend.changes_trie_storage(), diff --git a/client/chain-spec/src/chain_spec.rs b/client/chain-spec/src/chain_spec.rs index fcdb053c47c16..c7c4efd4a4890 100644 --- a/client/chain-spec/src/chain_spec.rs +++ b/client/chain-spec/src/chain_spec.rs @@ -170,6 +170,9 @@ struct ClientSpec { /// block hash onwards. #[serde(default)] code_substitutes: HashMap, + /// Ordered sequence of block number and their associated state version. + #[serde(default)] + state_versions: Vec<(u64, Option>)>, } /// A type denoting empty extensions. @@ -261,6 +264,7 @@ impl ChainSpec { consensus_engine: (), genesis: Default::default(), code_substitutes: HashMap::new(), + state_versions: vec![(0, Some(Some(sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD)))], }; ChainSpec { client_spec, genesis: GenesisSource::Factory(Arc::new(constructor)) } @@ -270,6 +274,11 @@ impl ChainSpec { fn chain_type(&self) -> ChainType { self.client_spec.chain_type.clone() } + + /// Defined state version for the chain. + fn state_versions(&self) -> Vec<(u64, Option>)> { + self.client_spec.state_versions.clone() + } } impl ChainSpec { @@ -401,6 +410,10 @@ where .map(|(h, c)| (h.clone(), c.0.clone())) .collect() } + + fn state_versions(&self) -> Vec<(u64, Option>)> { + ChainSpec::state_versions(self) + } } #[cfg(test)] diff --git a/client/chain-spec/src/lib.rs b/client/chain-spec/src/lib.rs index 334d8f8b3d7ac..fe815be0dbaae 100644 --- a/client/chain-spec/src/lib.rs +++ b/client/chain-spec/src/lib.rs @@ -187,6 +187,9 @@ pub trait ChainSpec: BuildStorage + Send + Sync { fn set_storage(&mut self, storage: Storage); /// Returns code substitutes that should be used for the on chain wasm. fn code_substitutes(&self) -> std::collections::HashMap>; + /// State versions for the chain. + /// Uses default state version when no definition from block 0. + fn state_versions(&self) -> Vec<(u64, Option>)>; } impl std::fmt::Debug for dyn ChainSpec { diff --git a/client/db/src/bench.rs b/client/db/src/bench.rs index 072989ae9ed1c..00af0ebdc3737 100644 --- a/client/db/src/bench.rs +++ b/client/db/src/bench.rs @@ -82,6 +82,7 @@ impl sp_state_machine::Storage> for StorageDb { root: Cell, + alt_hashing: Cell>>, genesis_root: B::Hash, state: RefCell>>, db: Cell>>, @@ -119,6 +120,9 @@ impl BenchmarkingState { state: RefCell::new(None), db: Cell::new(None), root: Cell::new(root.clone()), + // use old state for now to keep similar benches. + // will need update to heavier of both world before migrating. + alt_hashing: Cell::new(None), genesis: Default::default(), genesis_root: Default::default(), record: Default::default(), @@ -169,7 +173,7 @@ impl BenchmarkingState { _block: Default::default(), }); *self.state.borrow_mut() = Some(State::new( - DbState::::new(storage_db, self.root.get()), + DbState::::new(storage_db, self.root.get(), self.alt_hashing.get()), self.shared_cache.clone(), None, )); @@ -614,7 +618,7 @@ impl StateBackend> for BenchmarkingState { fn proof_size(&self) -> Option { self.proof_recorder.as_ref().map(|recorder| { let proof_size = recorder.estimate_encoded_size() as u32; - let proof = recorder.to_storage_proof::>(); + let proof = recorder.to_storage_proof::>(self.alt_hashing.get()); let proof_recorder_root = self.proof_recorder_root.get(); if proof_recorder_root == Default::default() || proof_size == 1 { // empty trie @@ -634,6 +638,10 @@ impl StateBackend> for BenchmarkingState { } }) } + + fn alt_hashing(&self) -> Option> { + self.alt_hashing.get() + } } impl std::fmt::Debug for BenchmarkingState { diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index 3e8c676bafe3a..8bef621f3ad0e 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -286,6 +286,10 @@ impl StateBackend> for RefTrackingState { fn usage_info(&self) -> StateUsageInfo { self.state.usage_info() } + + fn alt_hashing(&self) -> Option> { + self.state.alt_hashing() + } } /// Database settings. @@ -1114,15 +1118,30 @@ pub struct Backend { io_stats: FrozenForDuration<(kvdb::IoStats, StateUsageInfo)>, state_usage: Arc, genesis_state: RwLock>>>, + alt_hashing: Vec<(NumberFor, Option>)>, // TODO have specific type for history of versions + // Also TODO consider moving this alt_hashing into BlockChainBb } impl Backend { + // TODO util of StateVersionHistory + fn alt_hashing_at(&self, at: &NumberFor) -> Option> { + // Default + let mut result = Some(Some(sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD)); + for (start, alt) in self.alt_hashing.iter() { + if start > at { + break; + } + result = alt.clone(); + } + result + } /// Create a new instance of database backend. /// /// The pruning window is how old a block must be before the state is pruned. - pub fn new(config: DatabaseSettings, canonicalization_delay: u64) -> ClientResult { + pub fn new(config: DatabaseSettings, canonicalization_delay: u64, alt_hashing: Vec<(NumberFor, Option>)>) -> ClientResult { + // TODO alt_hashing could also be part of database settings let db = crate::utils::open_database::(&config, DatabaseType::Full)?; - Self::from_database(db as Arc<_>, canonicalization_delay, &config) + Self::from_database(db as Arc<_>, canonicalization_delay, &config, alt_hashing) } /// Create new memory-backed client backend for tests. @@ -1152,14 +1171,16 @@ impl Backend { keep_blocks: KeepBlocks::Some(keep_blocks), transaction_storage, }; + let alt_hashing = Vec::new(); - Self::new(db_setting, canonicalization_delay).expect("failed to create test-db") + Self::new(db_setting, canonicalization_delay, alt_hashing).expect("failed to create test-db") } fn from_database( db: Arc>, canonicalization_delay: u64, config: &DatabaseSettings, + alt_hashing: Vec<(NumberFor, Option>)>, // TODO could also be part of database settings ) -> ClientResult { let is_archive_pruning = config.state_pruning.is_archive(); let blockchain = BlockchainDb::new(db.clone(), config.transaction_storage.clone())?; @@ -1203,6 +1224,7 @@ impl Backend { keep_blocks: config.keep_blocks.clone(), transaction_storage: config.transaction_storage.clone(), genesis_state: RwLock::new(None), + alt_hashing, }; // Older DB versions have no last state key. Check if the state is available and set it. @@ -1870,7 +1892,18 @@ impl Backend { fn empty_state(&self) -> ClientResult, Block>> { let root = EmptyStorage::::new().0; // Empty trie - let db_state = DbState::::new(self.storage.clone(), root); + // alt_hashing for genesis in empty state. + let mut alt_hashing = Some(Some(sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD)); // Default + // TODO replace by inner search for number + if let Some((first, alt)) = self.alt_hashing.get(0) { + // chain should always define their state + // otherwhise uses latest state, but will need definition + // when default may change. + if first.is_zero() { + alt_hashing = alt.clone(); + } + } + let db_state = DbState::::new(self.storage.clone(), root, alt_hashing); let state = RefTrackingState::new(db_state, self.storage.clone(), None); let caching_state = CachingState::new(state, self.shared_cache.clone(), None); Ok(SyncingCachingState::new( @@ -2335,15 +2368,23 @@ impl sc_client_api::backend::Backend for Backend { fn state_at(&self, block: BlockId) -> ClientResult { use sc_client_api::blockchain::HeaderBackend as BcHeaderBackend; - let is_genesis = match &block { - BlockId::Number(n) if n.is_zero() => true, - BlockId::Hash(h) if h == &self.blockchain.meta.read().genesis_hash => true, - _ => false, + let (is_genesis, number) = match &block { + BlockId::Number(n) => (n.is_zero(), *n), + BlockId::Hash(h) if h == &self.blockchain.meta.read().genesis_hash => { + (true, NumberFor::::zero()) + }, + BlockId::Hash(h) => { + let n = self.blockchain.number(*h)?.ok_or_else(|| { + sp_blockchain::Error::UnknownBlock(format!("Unknown number for block hash {}", h)) + })?; + (false, n) + }, }; if is_genesis { if let Some(genesis_state) = &*self.genesis_state.read() { let root = genesis_state.root.clone(); - let db_state = DbState::::new(genesis_state.clone(), root); + let alt_hashing = self.alt_hashing_at(&number); + let db_state = DbState::::new(genesis_state.clone(), root, alt_hashing); let state = RefTrackingState::new(db_state, self.storage.clone(), None); let caching_state = CachingState::new(state, self.shared_cache.clone(), None); let mut state = SyncingCachingState::new( @@ -2374,7 +2415,8 @@ impl sc_client_api::backend::Backend for Backend { } if let Ok(()) = self.storage.state_db.pin(&hash) { let root = hdr.state_root; - let db_state = DbState::::new(self.storage.clone(), root); + let alt_hashing = self.alt_hashing_at(&number); + let db_state = DbState::::new(self.storage.clone(), root, alt_hashing); let state = RefTrackingState::new(db_state, self.storage.clone(), Some(hash.clone())); let caching_state = diff --git a/client/db/src/storage_cache.rs b/client/db/src/storage_cache.rs index 3193d34796196..b81e92218039f 100644 --- a/client/db/src/storage_cache.rs +++ b/client/db/src/storage_cache.rs @@ -716,6 +716,10 @@ impl>, B: BlockT> StateBackend> for Cachin info.include_state_machine_states(&self.overlay_stats); info } + + fn alt_hashing(&self) -> Option> { + self.state.alt_hashing() + } } /// Extended [`CachingState`] that will sync the caches on drop. @@ -915,6 +919,10 @@ impl>, B: BlockT> StateBackend> fn usage_info(&self) -> sp_state_machine::UsageInfo { self.caching_state().usage_info() } + + fn alt_hashing(&self) -> Option> { + self.caching_state().alt_hashing() + } } impl Drop for SyncingCachingState { diff --git a/client/light/src/backend.rs b/client/light/src/backend.rs index 87d7dba3ddfb1..356f0987a50bb 100644 --- a/client/light/src/backend.rs +++ b/client/light/src/backend.rs @@ -59,9 +59,12 @@ const IN_MEMORY_EXPECT_PROOF: &str = "InMemory state backend has Void error type and always succeeds; qed"; /// Light client backend. -pub struct Backend { +pub struct Backend { blockchain: Arc>, - genesis_state: RwLock>>, + genesis_state: RwLock>>>, + alt_hashing: Vec<(NumberFor, Option>)>, // TODO have specific type for history of versions + // Also TODO consider moving this alt_hashing into BlockChain (and add N to type) -> would make + // better api import_lock: RwLock<()>, } @@ -75,6 +78,7 @@ pub struct ImportOperation { set_head: Option>, storage_update: Option>>, changes_trie_config_update: Option>, + alt_hashing: Option>, _phantom: std::marker::PhantomData, } @@ -87,10 +91,10 @@ pub enum GenesisOrUnavailableState { Unavailable, } -impl Backend { +impl Backend { /// Create new light backend. - pub fn new(blockchain: Arc>) -> Self { - Self { blockchain, genesis_state: RwLock::new(None), import_lock: Default::default() } + pub fn new(blockchain: Arc>, alt_hashing: Vec<(NumberFor, Option>)>) -> Self { + Self { blockchain, genesis_state: RwLock::new(None), import_lock: Default::default(), alt_hashing } } /// Get shared blockchain reference. @@ -99,7 +103,7 @@ impl Backend { } } -impl AuxStore for Backend { +impl AuxStore for Backend { fn insert_aux< 'a, 'b: 'a, @@ -119,7 +123,7 @@ impl AuxStore for Backend { } } -impl ClientBackend for Backend> +impl ClientBackend for Backend where Block: BlockT, S: BlockchainStorage, @@ -140,15 +144,28 @@ where set_head: None, storage_update: None, changes_trie_config_update: None, + alt_hashing: None, // TODO this needs to be undefined with versioning (Option). _phantom: Default::default(), }) } fn begin_state_operation( &self, - _operation: &mut Self::BlockImportOperation, - _block: BlockId, + operation: &mut Self::BlockImportOperation, + block: BlockId, ) -> ClientResult<()> { + + if let Some(number) = self.blockchain.storage().block_number_from_id(&block)? { + let mut version = Some(Some(sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD)); + // TODO from utils of Version history + for (change_state, state) in self.alt_hashing.iter() { + if &number > change_state { + break + } + version = *state; + } + operation.alt_hashing = version; + } Ok(()) } @@ -261,7 +278,7 @@ where } } -impl RemoteBackend for Backend> +impl RemoteBackend for Backend where Block: BlockT, S: BlockchainStorage + 'static, @@ -355,7 +372,7 @@ where storage.insert(Some(storage_child.child_info), storage_child.data); } - let storage_update = InMemoryBackend::from(storage); + let storage_update = InMemoryBackend::from((storage, self.alt_hashing.clone())); let (storage_root, _) = storage_update.full_storage_root(std::iter::empty(), child_delta); if commit { self.storage_update = Some(storage_update); @@ -575,4 +592,11 @@ where GenesisOrUnavailableState::Unavailable => None, } } + + fn alt_hashing(&self) -> Option> { + match self { + GenesisOrUnavailableState::Genesis(state) => state.force_alt_hashing.clone(), + GenesisOrUnavailableState::Unavailable => Some(Some(sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD)), + } + } } diff --git a/client/light/src/fetcher.rs b/client/light/src/fetcher.rs index fba3d854b8d2b..b353a41fa7ae2 100644 --- a/client/light/src/fetcher.rs +++ b/client/light/src/fetcher.rs @@ -202,7 +202,7 @@ impl> LightDataChecker { } // check proof for single changes trie root - let proving_backend = TrieBackend::new(storage, local_cht_root); + let proving_backend = TrieBackend::new(storage, local_cht_root, None); let remote_changes_trie_root = remote_roots[&block]; cht::check_proof_on_proving_backend::>( local_cht_root, diff --git a/client/light/src/lib.rs b/client/light/src/lib.rs index 0c874326ef2e0..2a59be7e6b08f 100644 --- a/client/light/src/lib.rs +++ b/client/light/src/lib.rs @@ -19,7 +19,7 @@ //! Light client components. use sp_core::traits::{CodeExecutor, SpawnNamed}; -use sp_runtime::traits::{Block as BlockT, HashFor}; +use sp_runtime::traits::{NumberFor, Block as BlockT}; use std::sync::Arc; pub mod backend; @@ -50,10 +50,10 @@ pub fn new_light_blockchain>(storage: S) -> A } /// Create an instance of light client backend. -pub fn new_light_backend(blockchain: Arc>) -> Arc>> +pub fn new_light_backend(blockchain: Arc>, alt_hashing: Vec<(NumberFor, Option>)>) -> Arc> where B: BlockT, S: BlockchainStorage, { - Arc::new(Backend::new(blockchain)) + Arc::new(Backend::new(blockchain, alt_hashing)) } diff --git a/client/network/src/light_client_requests/handler.rs b/client/network/src/light_client_requests/handler.rs index 609ed35f4a9d1..d533bd05b433d 100644 --- a/client/network/src/light_client_requests/handler.rs +++ b/client/network/src/light_client_requests/handler.rs @@ -181,7 +181,7 @@ impl LightClientRequestHandler { request.block, e, ); - StorageProof::empty() + StorageProof::empty(Some(Some(sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD))) }, }; @@ -225,7 +225,7 @@ impl LightClientRequestHandler { request.block, error, ); - StorageProof::empty() + StorageProof::empty(Some(Some(sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD))) }, }; @@ -279,7 +279,7 @@ impl LightClientRequestHandler { request.block, error, ); - StorageProof::empty() + StorageProof::empty(Some(Some(sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD))) }, }; @@ -308,7 +308,7 @@ impl LightClientRequestHandler { request.block, error ); - (Default::default(), StorageProof::empty()) + (Default::default(), StorageProof::empty(Some(Some(sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD)))) }, }; @@ -357,19 +357,19 @@ impl LightClientRequestHandler { Ok(proof) => proof, Err(error) => { log::trace!( - "Remote changes proof request from {} for key {} ({:?}..{:?}) failed with: {}.", - peer, - format!("{} : {}", HexDisplay::from(&request.storage_key), HexDisplay::from(&key.0)), - request.first, - request.last, - error, - ); + "Remote changes proof request from {} for key {} ({:?}..{:?}) failed with: {}.", + peer, + format!("{} : {}", HexDisplay::from(&request.storage_key), HexDisplay::from(&key.0)), + request.first, + request.last, + error, + ); light::ChangesProof:: { max_block: Zero::zero(), proof: Vec::new(), roots: BTreeMap::new(), - roots_proof: StorageProof::empty(), + roots_proof: StorageProof::empty(Some(Some(sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD))), } }, }; diff --git a/client/service/src/builder.rs b/client/service/src/builder.rs index fc9fbad1ef475..93f6fe19cb348 100644 --- a/client/service/src/builder.rs +++ b/client/service/src/builder.rs @@ -58,7 +58,7 @@ use sp_core::traits::{CodeExecutor, SpawnNamed}; use sp_keystore::{CryptoStore, SyncCryptoStore, SyncCryptoStorePtr}; use sp_runtime::{ generic::BlockId, - traits::{Block as BlockT, BlockIdTo, HashFor, Zero}, + traits::{Block as BlockT, BlockIdTo, Zero, NumberFor}, BuildStorage, }; use sp_utils::mpsc::{tracing_unbounded, TracingUnboundedSender}; @@ -145,14 +145,14 @@ pub type TLightClient = /// Light client backend type. pub type TLightBackend = - sc_light::Backend, HashFor>; + sc_light::Backend, TBl>; /// Light call executor type. pub type TLightCallExecutor = sc_light::GenesisCallExecutor< - sc_light::Backend, HashFor>, + sc_light::Backend, TBl>, crate::client::LocalCallExecutor< TBl, - sc_light::Backend, HashFor>, + sc_light::Backend, TBl>, NativeExecutor, >, >; @@ -317,7 +317,16 @@ where transaction_storage: config.transaction_storage.clone(), }; - let backend = new_db_backend(db_config)?; + let mut state_versions = Vec::new(); + for (number, version) in config.chain_spec.state_versions().into_iter() { + let number = NumberFor::::from_str(&number.to_string()) + .map_err(|_| Error::Application(Box::from(format!( + "Failed to parse `{}` as block number for state versions.", + number, + ))))?; + state_versions.push((number.into(), version)); + } + let backend = new_db_backend(db_config, state_versions)?; let extensions = sc_client_api::execution_extensions::ExecutionExtensions::new( config.execution_strategies.clone(), @@ -407,7 +416,17 @@ where Box::new(task_manager.spawn_handle()), )); let on_demand = Arc::new(sc_network::config::OnDemand::new(fetch_checker)); - let backend = sc_light::new_light_backend(light_blockchain); + let mut state_versions = Vec::new(); + for (number, version) in config.chain_spec.state_versions().into_iter() { + let number = NumberFor::::from_str(&number.to_string()) + .map_err(|_| Error::Application(Box::from(format!( + "Failed to parse `{}` as block number for state versions.", + number, + ))))?; + state_versions.push((number.into(), version)); + } + + let backend = sc_light::new_light_backend(light_blockchain, state_versions); let client = Arc::new(light::new_light( backend.clone(), config.chain_spec.as_storage_builder(), @@ -423,13 +442,14 @@ where /// Create an instance of default DB-backend backend. pub fn new_db_backend( settings: DatabaseSettings, + state_versions: Vec<(NumberFor, Option>)>, ) -> Result>, sp_blockchain::Error> where Block: BlockT, { const CANONICALIZATION_DELAY: u64 = 4096; - Ok(Arc::new(Backend::new(settings, CANONICALIZATION_DELAY)?)) + Ok(Arc::new(Backend::new(settings, CANONICALIZATION_DELAY, state_versions)?)) } /// Create an instance of client backed by given backend. diff --git a/client/service/src/client/light.rs b/client/service/src/client/light.rs index 7c13b98843e05..909b7083d3a5e 100644 --- a/client/service/src/client/light.rs +++ b/client/service/src/client/light.rs @@ -26,7 +26,7 @@ use sc_telemetry::TelemetryHandle; use sp_blockchain::Result as ClientResult; use sp_core::traits::{CodeExecutor, SpawnNamed}; use sp_runtime::{ - traits::{Block as BlockT, HashFor}, + traits::Block as BlockT, BuildStorage, }; @@ -39,7 +39,7 @@ use sc_light::{Backend, GenesisCallExecutor}; /// Create an instance of light client. pub fn new_light( - backend: Arc>>, + backend: Arc>, genesis_storage: &dyn BuildStorage, code_executor: E, spawn_handle: Box, @@ -47,10 +47,10 @@ pub fn new_light( telemetry: Option, ) -> ClientResult< Client< - Backend>, + Backend, GenesisCallExecutor< - Backend>, - LocalCallExecutor>, E>, + Backend, + LocalCallExecutor, E>, >, B, RA, diff --git a/primitives/api/proc-macro/src/impl_runtime_apis.rs b/primitives/api/proc-macro/src/impl_runtime_apis.rs index 81287b1fac64b..73b86b33639e8 100644 --- a/primitives/api/proc-macro/src/impl_runtime_apis.rs +++ b/primitives/api/proc-macro/src/impl_runtime_apis.rs @@ -279,10 +279,10 @@ fn generate_runtime_api_base_structures() -> Result { self.recorder.clone() } - fn extract_proof(&mut self) -> Option<#crate_::StorageProof> { + fn extract_proof(&mut self, alt_hashing: Option>) -> Option<#crate_::StorageProof> { self.recorder .take() - .map(|recorder| recorder.to_storage_proof::<#crate_::HashFor>()) + .map(|recorder| recorder.to_storage_proof::<#crate_::HashFor>(alt_hashing)) } fn into_storage_changes( diff --git a/primitives/api/proc-macro/src/mock_impl_runtime_apis.rs b/primitives/api/proc-macro/src/mock_impl_runtime_apis.rs index 77f8a07f85c48..58c10525efbe1 100644 --- a/primitives/api/proc-macro/src/mock_impl_runtime_apis.rs +++ b/primitives/api/proc-macro/src/mock_impl_runtime_apis.rs @@ -105,7 +105,7 @@ fn implement_common_api_traits(block_type: TypePath, self_ty: Type) -> Result Option<#crate_::StorageProof> { + fn extract_proof(&mut self, alt_hashing: Option>) -> Option<#crate_::StorageProof> { unimplemented!("`extract_proof` not implemented for runtime api mocks") } diff --git a/primitives/api/src/lib.rs b/primitives/api/src/lib.rs index 82954d193e605..89da6085db8bc 100644 --- a/primitives/api/src/lib.rs +++ b/primitives/api/src/lib.rs @@ -502,7 +502,7 @@ pub trait ApiExt { /// This stops the proof recording. /// /// If `record_proof` was not called before, this will return `None`. - fn extract_proof(&mut self) -> Option; + fn extract_proof(&mut self, alt_hashing: Option>) -> Option; /// Returns the current active proof recorder. fn proof_recorder(&self) -> Option>; diff --git a/primitives/state-machine/src/backend.rs b/primitives/state-machine/src/backend.rs index 4d5de57a20631..a2747ae759641 100644 --- a/primitives/state-machine/src/backend.rs +++ b/primitives/state-machine/src/backend.rs @@ -276,6 +276,9 @@ pub trait Backend: sp_std::fmt::Debug { fn get_read_and_written_keys(&self) -> Vec<(Vec, u32, u32, bool)> { unimplemented!() } + + /// Get current state version in use. + fn alt_hashing(&self) -> Option>; } /// Trait that allows consolidate two transactions together. diff --git a/primitives/state-machine/src/proving_backend.rs b/primitives/state-machine/src/proving_backend.rs index d62f6ed6ef55c..2e9d8421acd7f 100644 --- a/primitives/state-machine/src/proving_backend.rs +++ b/primitives/state-machine/src/proving_backend.rs @@ -389,6 +389,10 @@ where fn usage_info(&self) -> crate::stats::UsageInfo { self.0.usage_info() } + + fn alt_hashing(&self) -> Option> { + self.0.force_alt_hashing.clone() + } } /// Create proof check backend. diff --git a/primitives/state-machine/src/trie_backend.rs b/primitives/state-machine/src/trie_backend.rs index d6f68077a467a..eb086a6ae2edc 100644 --- a/primitives/state-machine/src/trie_backend.rs +++ b/primitives/state-machine/src/trie_backend.rs @@ -296,6 +296,10 @@ where fn wipe(&self) -> Result<(), Self::Error> { Ok(()) } + + fn alt_hashing(&self) -> Option> { + self.force_alt_hashing.clone() + } } #[cfg(test)] diff --git a/primitives/trie/src/storage_proof.rs b/primitives/trie/src/storage_proof.rs index 865a5bbca8d00..636f28b66c79a 100644 --- a/primitives/trie/src/storage_proof.rs +++ b/primitives/trie/src/storage_proof.rs @@ -52,7 +52,7 @@ impl StorageProof { /// An empty proof is capable of only proving trivial statements (ie. that an empty set of /// key-value pairs exist in storage). pub fn empty(alt_hashing: Option>) -> Self { - // TODO consider alt_hashing default value. + // TODO consider alt_hashing default value. ++ StorageProof { trie_nodes: Vec::new(), alt_hashing } } diff --git a/test-utils/client/src/lib.rs b/test-utils/client/src/lib.rs index 8ac796e8ba67c..2f5e59ebde4e8 100644 --- a/test-utils/client/src/lib.rs +++ b/test-utils/client/src/lib.rs @@ -47,7 +47,7 @@ use serde::Deserialize; use sp_core::storage::ChildInfo; use sp_runtime::{ codec::Encode, - traits::{BlakeTwo256, Block as BlockT}, + traits::Block as BlockT, OpaqueExtrinsic, }; use std::{ @@ -58,7 +58,7 @@ use std::{ /// Test client light database backend. pub type LightBackend = - sc_light::Backend, BlakeTwo256>; + sc_light::Backend, Block>; /// A genesis storage initialization trait. pub trait GenesisInit: Default { diff --git a/test-utils/runtime/client/src/lib.rs b/test-utils/runtime/client/src/lib.rs index d5cd4d3ab1958..e2a021a2ecf1f 100644 --- a/test-utils/runtime/client/src/lib.rs +++ b/test-utils/runtime/client/src/lib.rs @@ -39,7 +39,7 @@ use sp_core::{ storage::{ChildInfo, Storage, StorageChild}, ChangesTrieConfiguration, }; -use sp_runtime::traits::{Block as BlockT, Hash as HashT, HashFor, Header as HeaderT, NumberFor}; +use sp_runtime::traits::{Block as BlockT, Hash as HashT, Header as HeaderT, NumberFor}; use substrate_test_runtime::genesismap::{additional_storage_with_genesis, GenesisConfig}; /// A prelude to import in tests. @@ -84,7 +84,7 @@ pub type LightExecutor = sc_light::GenesisCallExecutor< substrate_test_runtime::Block, sc_light::Backend< sc_client_db::light::LightStorage, - HashFor, + substrate_test_runtime::Block, >, NativeExecutor, >, @@ -407,7 +407,8 @@ pub fn new_light( ) { let storage = sc_client_db::light::LightStorage::new_test(); let blockchain = Arc::new(sc_light::Blockchain::new(storage)); - let backend = Arc::new(LightBackend::new(blockchain)); + let state_versions = vec![(0, Some(Some(sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD)))]; + let backend = Arc::new(LightBackend::new(blockchain, state_versions)); let executor = new_native_executor(); let local_call_executor = client::LocalCallExecutor::new( backend.clone(), From 250ad029bc5c78fc2fb119b663d121c5064b05f4 Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Thu, 19 Aug 2021 23:25:00 +0200 Subject: [PATCH 069/127] update tests and remove state threshold key. --- .../pallets/template/src/mock.rs | 3 +- bin/node/cli/src/chain_spec.rs | 13 +++- bin/node/executor/benches/bench.rs | 4 +- bin/node/executor/tests/common.rs | 4 +- bin/node/runtime/src/impls.rs | 7 +- bin/node/testing/src/client.rs | 3 +- client/chain-spec/src/chain_spec.rs | 20 +++-- client/db/src/lib.rs | 20 +++-- client/db/src/storage_cache.rs | 74 +++++++++---------- client/executor/runtime-test/src/lib.rs | 10 --- client/executor/src/integration_tests/mod.rs | 53 +------------ .../src/light_client_requests/sender.rs | 2 +- client/service/src/builder.rs | 11 +++ client/service/src/client/client.rs | 9 ++- client/service/src/client/wasm_override.rs | 2 +- client/service/src/client/wasm_substitutes.rs | 2 +- client/service/test/src/client/light.rs | 16 ++-- client/service/test/src/client/mod.rs | 25 +++---- frame/assets/src/mock.rs | 5 +- frame/atomic-swap/src/tests.rs | 7 +- frame/aura/src/mock.rs | 7 +- frame/authority-discovery/src/lib.rs | 6 +- frame/authorship/src/lib.rs | 5 +- frame/babe/src/mock.rs | 11 +-- frame/balances/src/lib.rs | 8 +- frame/balances/src/tests.rs | 10 ++- frame/balances/src/tests_composite.rs | 7 +- frame/balances/src/tests_local.rs | 7 +- frame/balances/src/tests_reentrancy.rs | 7 +- frame/benchmarking/src/tests.rs | 4 +- frame/benchmarking/src/tests_instance.rs | 4 +- frame/bounties/src/tests.rs | 25 ++++--- frame/collective/src/lib.rs | 11 +-- frame/contracts/src/tests.rs | 7 +- frame/democracy/src/tests.rs | 9 ++- .../election-provider-multi-phase/src/mock.rs | 7 +- frame/elections-phragmen/src/lib.rs | 7 +- frame/elections/src/mock.rs | 7 +- frame/example/src/tests.rs | 5 +- frame/executive/src/lib.rs | 14 ++-- frame/gilt/src/mock.rs | 9 ++- frame/grandpa/src/mock.rs | 11 +-- frame/identity/src/tests.rs | 7 +- frame/im-online/src/mock.rs | 5 +- frame/indices/src/mock.rs | 7 +- frame/lottery/src/mock.rs | 7 +- frame/membership/src/lib.rs | 10 ++- frame/merkle-mountain-range/src/tests.rs | 3 +- frame/multisig/src/tests.rs | 7 +- frame/nicks/src/lib.rs | 7 +- frame/node-authorization/src/mock.rs | 7 +- frame/offences/benchmarking/src/mock.rs | 5 +- frame/offences/src/mock.rs | 5 +- frame/proxy/src/tests.rs | 7 +- frame/randomness-collective-flip/src/lib.rs | 5 +- frame/recovery/src/mock.rs | 7 +- frame/scheduler/src/lib.rs | 5 +- frame/scored-pool/src/mock.rs | 9 ++- frame/session/benchmarking/src/mock.rs | 5 +- frame/session/src/historical/mod.rs | 9 ++- frame/session/src/historical/offchain.rs | 9 ++- frame/session/src/mock.rs | 9 ++- frame/society/src/mock.rs | 9 ++- frame/staking/src/mock.rs | 11 +-- frame/staking/src/pallet/mod.rs | 3 +- frame/sudo/src/mock.rs | 7 +- .../src/construct_runtime/expand/config.rs | 5 +- .../src/pallet/expand/genesis_build.rs | 3 +- .../src/storage/genesis_config/mod.rs | 10 ++- frame/support/src/lib.rs | 4 +- frame/support/src/storage/generator/mod.rs | 15 ++-- frame/support/src/storage/migration.rs | 12 +-- frame/support/src/traits/hooks.rs | 8 +- frame/support/test/tests/decl_storage.rs | 5 +- frame/support/test/tests/instance.rs | 14 ++-- frame/system/benches/bench.rs | 3 +- frame/system/benchmarking/src/mock.rs | 5 +- frame/system/src/lib.rs | 14 ++-- frame/system/src/mock.rs | 3 +- frame/timestamp/src/lib.rs | 5 +- frame/tips/src/tests.rs | 21 +++--- frame/transaction-payment/src/lib.rs | 9 ++- frame/transaction-storage/src/mock.rs | 5 +- frame/treasury/src/lib.rs | 3 +- frame/treasury/src/tests.rs | 25 ++++--- frame/uniques/src/mock.rs | 5 +- frame/utility/src/tests.rs | 7 +- frame/vesting/src/mock.rs | 9 ++- primitives/io/src/lib.rs | 23 +++--- primitives/runtime/src/lib.rs | 15 ++-- primitives/state-machine/src/backend.rs | 9 --- primitives/state-machine/src/basic.rs | 39 +++++----- .../state-machine/src/changes_trie/build.rs | 4 +- primitives/state-machine/src/ext.rs | 28 +++---- .../state-machine/src/in_memory_backend.rs | 4 +- primitives/state-machine/src/lib.rs | 23 +++--- .../src/overlayed_changes/mod.rs | 2 +- .../state-machine/src/proving_backend.rs | 43 ++++------- primitives/state-machine/src/testing.rs | 42 +++-------- primitives/state-machine/src/trie_backend.rs | 25 +------ primitives/storage/src/lib.rs | 39 ---------- test-utils/client/src/lib.rs | 8 +- test-utils/runtime/client/src/lib.rs | 1 + test-utils/runtime/src/genesismap.rs | 5 +- test-utils/runtime/src/lib.rs | 2 +- test-utils/runtime/src/system.rs | 6 +- test-utils/test-runner/src/utils.rs | 7 +- utils/frame/benchmarking-cli/src/command.rs | 6 +- utils/frame/remote-externalities/src/lib.rs | 2 +- utils/frame/try-runtime/cli/src/lib.rs | 9 ++- 110 files changed, 572 insertions(+), 603 deletions(-) diff --git a/bin/node-template/pallets/template/src/mock.rs b/bin/node-template/pallets/template/src/mock.rs index 4532d3d09b497..cdb4a3047e3ed 100644 --- a/bin/node-template/pallets/template/src/mock.rs +++ b/bin/node-template/pallets/template/src/mock.rs @@ -59,5 +59,6 @@ impl pallet_template::Config for Test { // Build genesis storage according to the mock runtime. pub fn new_test_ext() -> sp_io::TestExternalities { - system::GenesisConfig::default().build_storage::().unwrap().into() + let state_version = None; + (system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(), state_version).into() } diff --git a/bin/node/cli/src/chain_spec.rs b/bin/node/cli/src/chain_spec.rs index bbb2904beab3a..2a4b9455bd9b5 100644 --- a/bin/node/cli/src/chain_spec.rs +++ b/bin/node/cli/src/chain_spec.rs @@ -489,16 +489,23 @@ pub(crate) mod tests { #[test] fn test_create_development_chain_spec() { - development_config().build_storage().unwrap(); + let spec = development_config(); + let state_version = spec.genesis_state_version(); + spec.build_storage(state_version).unwrap(); + } #[test] fn test_create_local_testnet_chain_spec() { - local_testnet_config().build_storage().unwrap(); + let spec = local_testnet_config(); + let state_version = spec.genesis_state_version(); + spec.build_storage(state_version).unwrap(); } #[test] fn test_staging_test_net_chain_spec() { - staging_testnet_config().build_storage().unwrap(); + let spec = staging_testnet_config(); + let state_version = spec.genesis_state_version(); + spec.build_storage(state_version).unwrap(); } } diff --git a/bin/node/executor/benches/bench.rs b/bin/node/executor/benches/bench.rs index 18ab8d57059c7..c8cb7b2aa7aae 100644 --- a/bin/node/executor/benches/bench.rs +++ b/bin/node/executor/benches/bench.rs @@ -66,9 +66,11 @@ fn sign(xt: CheckedExtrinsic) -> UncheckedExtrinsic { } fn new_test_ext(genesis_config: &GenesisConfig) -> TestExternalities { + let state_version = None; let mut test_ext = TestExternalities::new_with_code( compact_code_unwrap(), - genesis_config.build_storage().unwrap(), + genesis_config.build_storage(state_version.clone()).unwrap(), + state_version, ); test_ext .ext() diff --git a/bin/node/executor/tests/common.rs b/bin/node/executor/tests/common.rs index b61126ba1b4a3..abd9a071d335a 100644 --- a/bin/node/executor/tests/common.rs +++ b/bin/node/executor/tests/common.rs @@ -125,11 +125,13 @@ pub fn executor_call< } pub fn new_test_ext(code: &[u8], support_changes_trie: bool) -> TestExternalities { + let state_version = None; let mut ext = TestExternalities::new_with_code( code, node_testing::genesis::config(support_changes_trie, Some(code)) - .build_storage() + .build_storage(state_version.clone()) .unwrap(), + state_version ); ext.changes_trie_storage().insert(0, GENESIS_HASH.into(), Default::default()); ext diff --git a/bin/node/runtime/src/impls.rs b/bin/node/runtime/src/impls.rs index e315a45e698ce..3250375fadf84 100644 --- a/bin/node/runtime/src/impls.rs +++ b/bin/node/runtime/src/impls.rs @@ -96,9 +96,10 @@ mod multiplier_tests { where F: Fn() -> (), { - let mut t: sp_io::TestExternalities = frame_system::GenesisConfig::default() - .build_storage::() - .unwrap() + let state_version = None; + let mut t: sp_io::TestExternalities = (frame_system::GenesisConfig::default() + .build_storage::(state_version.clone()) + .unwrap(), state_version) .into(); t.execute_with(|| { System::set_block_consumed_resources(w, 0); diff --git a/bin/node/testing/src/client.rs b/bin/node/testing/src/client.rs index 9538cd47d88a6..8bcd4c28ae46f 100644 --- a/bin/node/testing/src/client.rs +++ b/bin/node/testing/src/client.rs @@ -48,7 +48,8 @@ pub struct GenesisParameters { impl substrate_test_client::GenesisInit for GenesisParameters { fn genesis_storage(&self) -> Storage { - crate::genesis::config(self.support_changes_trie, None).build_storage().unwrap() + let state_version = Some(Some(sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD)); + crate::genesis::config(self.support_changes_trie, None).build_storage(state_version).unwrap() } } diff --git a/client/chain-spec/src/chain_spec.rs b/client/chain-spec/src/chain_spec.rs index c7c4efd4a4890..ec02cfbaf76d0 100644 --- a/client/chain-spec/src/chain_spec.rs +++ b/client/chain-spec/src/chain_spec.rs @@ -99,9 +99,9 @@ impl GenesisSource { } impl BuildStorage for ChainSpec { - fn build_storage(&self) -> Result { + fn build_storage(&self, alt_hashing: Option>) -> Result { match self.genesis.resolve()? { - Genesis::Runtime(gc) => gc.build_storage(), + Genesis::Runtime(gc) => gc.build_storage(alt_hashing), Genesis::Raw(RawGenesis { top: map, children_default: children_map }) => Ok(Storage { top: map.into_iter().map(|(k, v)| (k.0, v.0)).collect(), children_default: children_map @@ -121,7 +121,7 @@ impl BuildStorage for ChainSpec { } } - fn assimilate_storage(&self, _: &mut Storage) -> Result<(), String> { + fn assimilate_storage(&self, _: &mut Storage, _: Option>) -> Result<(), String> { Err("`assimilate_storage` not implemented for `ChainSpec`.".into()) } } @@ -279,6 +279,13 @@ impl ChainSpec { fn state_versions(&self) -> Vec<(u64, Option>)> { self.client_spec.state_versions.clone() } + + /// Defined state version for the chain. + pub fn genesis_state_version(&self) -> Option> { + self.client_spec.state_versions.get(0) + .and_then(|(n, s)| (n == &0).then(|| s.clone())) + .unwrap_or(Some(Some(sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD))) + } } impl ChainSpec { @@ -310,7 +317,10 @@ impl ChainSpec { fn json_container(&self, raw: bool) -> Result, String> { let genesis = match (raw, self.genesis.resolve()?) { (true, Genesis::Runtime(g)) => { - let storage = g.build_storage()?; + let state_version = self.state_versions().get(0) + .and_then(|(n, s)| (n == &0).then(|| s.clone())) + .unwrap_or(None); + let storage = g.build_storage(state_version)?; let top = storage.top.into_iter().map(|(k, v)| (StorageKey(k), StorageData(v))).collect(); let children_default = storage @@ -424,7 +434,7 @@ mod tests { struct Genesis(HashMap); impl BuildStorage for Genesis { - fn assimilate_storage(&self, storage: &mut Storage) -> Result<(), String> { + fn assimilate_storage(&self, storage: &mut Storage, _alt_hashing: Option>) -> Result<(), String> { storage.top.extend( self.0.iter().map(|(a, b)| (a.clone().into_bytes(), b.clone().into_bytes())), ); diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index 8bef621f3ad0e..6ccde328ca3fa 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -2603,6 +2603,7 @@ pub(crate) mod tests { transaction_storage: TransactionStorageMode::BlockBody, }, 0, + vec![], ) .unwrap(); assert_eq!(backend.blockchain().info().best_number, 9); @@ -2617,7 +2618,13 @@ pub(crate) mod tests { set_state_data_inner(false); } fn set_state_data_inner(alt_hashing: bool) { - let db = Backend::::new_test(2, 0); + let state_version = if alt_hashing { + Some(Some(sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD)) + } else { + None + }; + let mut db = Backend::::new_test(2, 0); + db.alt_hashing.push((0, state_version)); let hash = { let mut op = db.begin_operation().unwrap(); let mut header = Header { @@ -2628,16 +2635,7 @@ pub(crate) mod tests { extrinsics_root: Default::default(), }; - let mut storage = vec![(vec![1, 3, 5], vec![2, 4, 6]), (vec![1, 2, 3], vec![9, 9, 9])]; - - if alt_hashing { - storage.push(( - sp_core::storage::well_known_keys::TRIE_HASHING_CONFIG.to_vec(), - sp_core::storage::trie_threshold_encode( - sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD, - ), - )); - } + let storage = vec![(vec![1, 3, 5], vec![2, 4, 6]), (vec![1, 2, 3], vec![9, 9, 9])]; header.state_root = op .old_state diff --git a/client/db/src/storage_cache.rs b/client/db/src/storage_cache.rs index b81e92218039f..f1a23393a6bfa 100644 --- a/client/db/src/storage_cache.rs +++ b/client/db/src/storage_cache.rs @@ -972,7 +972,7 @@ mod tests { // blocks [ 3a(c) 2a(c) 2b 1b 1a(c) 0 ] // state [ 5 5 4 3 2 2 ] let mut s = CachingState::new( - InMemoryBackend::::default(), + InMemoryBackend::::from(None), // TODO replace all test new in mem by default (restoring implementation in state-machine/in_memory_backend.rs) shared.clone(), Some(root_parent), ); @@ -987,11 +987,11 @@ mod tests { ); let mut s = - CachingState::new(InMemoryBackend::::default(), shared.clone(), Some(h0)); + CachingState::new(InMemoryBackend::::from(None), shared.clone(), Some(h0)); s.cache.sync_cache(&[], &[], vec![], vec![], Some(h1a), Some(1), true); let mut s = - CachingState::new(InMemoryBackend::::default(), shared.clone(), Some(h0)); + CachingState::new(InMemoryBackend::::from(None), shared.clone(), Some(h0)); s.cache.sync_cache( &[], &[], @@ -1003,7 +1003,7 @@ mod tests { ); let mut s = - CachingState::new(InMemoryBackend::::default(), shared.clone(), Some(h1b)); + CachingState::new(InMemoryBackend::::from(None), shared.clone(), Some(h1b)); s.cache.sync_cache( &[], &[], @@ -1015,7 +1015,7 @@ mod tests { ); let mut s = - CachingState::new(InMemoryBackend::::default(), shared.clone(), Some(h1a)); + CachingState::new(InMemoryBackend::::from(None), shared.clone(), Some(h1a)); s.cache.sync_cache( &[], &[], @@ -1027,29 +1027,29 @@ mod tests { ); let mut s = - CachingState::new(InMemoryBackend::::default(), shared.clone(), Some(h2a)); + CachingState::new(InMemoryBackend::::from(None), shared.clone(), Some(h2a)); s.cache.sync_cache(&[], &[], vec![], vec![], Some(h3a), Some(3), true); let s = - CachingState::new(InMemoryBackend::::default(), shared.clone(), Some(h3a)); + CachingState::new(InMemoryBackend::::from(None), shared.clone(), Some(h3a)); assert_eq!(s.storage(&key).unwrap().unwrap(), vec![5]); let s = - CachingState::new(InMemoryBackend::::default(), shared.clone(), Some(h1a)); + CachingState::new(InMemoryBackend::::from(None), shared.clone(), Some(h1a)); assert!(s.storage(&key).unwrap().is_none()); let s = - CachingState::new(InMemoryBackend::::default(), shared.clone(), Some(h2b)); + CachingState::new(InMemoryBackend::::from(None), shared.clone(), Some(h2b)); assert!(s.storage(&key).unwrap().is_none()); let s = - CachingState::new(InMemoryBackend::::default(), shared.clone(), Some(h1b)); + CachingState::new(InMemoryBackend::::from(None), shared.clone(), Some(h1b)); assert!(s.storage(&key).unwrap().is_none()); // reorg to 3b // blocks [ 3b(c) 3a 2a 2b(c) 1b 1a 0 ] let mut s = - CachingState::new(InMemoryBackend::::default(), shared.clone(), Some(h2b)); + CachingState::new(InMemoryBackend::::from(None), shared.clone(), Some(h2b)); s.cache.sync_cache( &[h1b, h2b, h3b], &[h1a, h2a, h3a], @@ -1060,7 +1060,7 @@ mod tests { true, ); let s = - CachingState::new(InMemoryBackend::::default(), shared.clone(), Some(h3a)); + CachingState::new(InMemoryBackend::::from(None), shared.clone(), Some(h3a)); assert!(s.storage(&key).unwrap().is_none()); } @@ -1078,7 +1078,7 @@ mod tests { let shared = new_shared_cache::(256 * 1024, (0, 1)); let mut s = CachingState::new( - InMemoryBackend::::default(), + InMemoryBackend::::from(None), shared.clone(), Some(root_parent), ); @@ -1093,11 +1093,11 @@ mod tests { ); let mut s = - CachingState::new(InMemoryBackend::::default(), shared.clone(), Some(h1)); + CachingState::new(InMemoryBackend::::from(None), shared.clone(), Some(h1)); s.cache.sync_cache(&[], &[], vec![], vec![], Some(h2a), Some(2), true); let mut s = - CachingState::new(InMemoryBackend::::default(), shared.clone(), Some(h1)); + CachingState::new(InMemoryBackend::::from(None), shared.clone(), Some(h1)); s.cache.sync_cache( &[], &[], @@ -1109,7 +1109,7 @@ mod tests { ); let mut s = - CachingState::new(InMemoryBackend::::default(), shared.clone(), Some(h2b)); + CachingState::new(InMemoryBackend::::from(None), shared.clone(), Some(h2b)); s.cache.sync_cache( &[], &[], @@ -1121,7 +1121,7 @@ mod tests { ); let s = - CachingState::new(InMemoryBackend::::default(), shared.clone(), Some(h2a)); + CachingState::new(InMemoryBackend::::from(None), shared.clone(), Some(h2a)); assert_eq!(s.storage(&key).unwrap().unwrap(), vec![2]); } @@ -1138,18 +1138,18 @@ mod tests { let shared = new_shared_cache::(256 * 1024, (0, 1)); let mut s = CachingState::new( - InMemoryBackend::::default(), + InMemoryBackend::::from(None), shared.clone(), Some(root_parent), ); s.cache.sync_cache(&[], &[], vec![], vec![], Some(h1), Some(1), true); let mut s = - CachingState::new(InMemoryBackend::::default(), shared.clone(), Some(h1)); + CachingState::new(InMemoryBackend::::from(None), shared.clone(), Some(h1)); s.cache.sync_cache(&[], &[], vec![], vec![], Some(h2a), Some(2), true); let mut s = - CachingState::new(InMemoryBackend::::default(), shared.clone(), Some(h2a)); + CachingState::new(InMemoryBackend::::from(None), shared.clone(), Some(h2a)); s.cache.sync_cache( &[], &[], @@ -1161,11 +1161,11 @@ mod tests { ); let mut s = - CachingState::new(InMemoryBackend::::default(), shared.clone(), Some(h1)); + CachingState::new(InMemoryBackend::::from(None), shared.clone(), Some(h1)); s.cache.sync_cache(&[], &[], vec![], vec![], Some(h2b), Some(2), false); let mut s = - CachingState::new(InMemoryBackend::::default(), shared.clone(), Some(h2b)); + CachingState::new(InMemoryBackend::::from(None), shared.clone(), Some(h2b)); s.cache.sync_cache( &[], &[], @@ -1177,7 +1177,7 @@ mod tests { ); let s = - CachingState::new(InMemoryBackend::::default(), shared.clone(), Some(h3a)); + CachingState::new(InMemoryBackend::::from(None), shared.clone(), Some(h3a)); assert_eq!(s.storage(&key).unwrap().unwrap(), vec![2]); } @@ -1189,7 +1189,7 @@ mod tests { let h1b = H256::random(); let shared = new_shared_cache::(256 * 1024, (0, 1)); - let mut backend = InMemoryBackend::::default(); + let mut backend = InMemoryBackend::::from(None); backend.insert(std::iter::once((None, vec![(key.clone(), Some(vec![1]))]))); let mut s = CachingState::new(backend.clone(), shared.clone(), Some(root_parent)); @@ -1217,7 +1217,7 @@ mod tests { let h0 = H256::random(); let mut s = CachingState::new( - InMemoryBackend::::default(), + InMemoryBackend::::from(None), shared.clone(), Some(root_parent.clone()), ); @@ -1257,7 +1257,7 @@ mod tests { let h0 = H256::random(); let mut s = CachingState::new( - InMemoryBackend::::default(), + InMemoryBackend::::from(None), shared.clone(), Some(root_parent), ); @@ -1301,7 +1301,7 @@ mod tests { let shared = new_shared_cache::(256 * 1024, (0, 1)); let mut s = CachingState::new( - InMemoryBackend::::default(), + InMemoryBackend::::from(None), shared.clone(), Some(root_parent.clone()), ); @@ -1316,7 +1316,7 @@ mod tests { ); let mut s = - CachingState::new(InMemoryBackend::::default(), shared.clone(), Some(h0)); + CachingState::new(InMemoryBackend::::from(None), shared.clone(), Some(h0)); s.cache.sync_cache( &[], &[], @@ -1328,7 +1328,7 @@ mod tests { ); let mut s = - CachingState::new(InMemoryBackend::::default(), shared.clone(), Some(h1)); + CachingState::new(InMemoryBackend::::from(None), shared.clone(), Some(h1)); assert_eq!(s.storage(&key).unwrap(), Some(vec![3])); // Restart (or unknown block?), clear caches. @@ -1348,7 +1348,7 @@ mod tests { s.cache.sync_cache(&[], &[], vec![], vec![], None, None, true); let s = - CachingState::new(InMemoryBackend::::default(), shared.clone(), Some(h1)); + CachingState::new(InMemoryBackend::::from(None), shared.clone(), Some(h1)); assert_eq!(s.storage(&key).unwrap(), None); } @@ -1364,7 +1364,7 @@ mod tests { let shared = new_shared_cache::(256 * 1024, (0, 1)); let mut s = CachingState::new( - InMemoryBackend::::default(), + InMemoryBackend::::from(None), shared.clone(), Some(root_parent), ); @@ -1380,7 +1380,7 @@ mod tests { assert_eq!(shared.write().lru_storage.get(&key).unwrap(), &Some(vec![1])); let mut s = - CachingState::new(InMemoryBackend::::default(), shared.clone(), Some(h1)); + CachingState::new(InMemoryBackend::::from(None), shared.clone(), Some(h1)); // commit as non-best s.cache.sync_cache( @@ -1396,7 +1396,7 @@ mod tests { assert_eq!(shared.write().lru_storage.get(&key).unwrap(), &Some(vec![1])); let mut s = - CachingState::new(InMemoryBackend::::default(), shared.clone(), Some(h1)); + CachingState::new(InMemoryBackend::::from(None), shared.clone(), Some(h1)); // commit again as best with no changes s.cache.sync_cache(&[], &[], vec![], vec![], Some(h2), Some(2), true); @@ -1528,7 +1528,7 @@ mod qc { fn head_state(&self, hash: H256) -> CachingState, Block> { CachingState::new( - InMemoryBackend::::default(), + InMemoryBackend::::from(None), self.shared.clone(), Some(hash), ) @@ -1599,7 +1599,7 @@ mod qc { }; let mut state = CachingState::new( - InMemoryBackend::::default(), + InMemoryBackend::::from(None), self.shared.clone(), Some(parent), ); @@ -1636,7 +1636,7 @@ mod qc { } let mut state = CachingState::new( - InMemoryBackend::::default(), + InMemoryBackend::::from(None), self.shared.clone(), Some(parent_hash), ); @@ -1687,7 +1687,7 @@ mod qc { self.canon.push(node); let mut state = CachingState::new( - InMemoryBackend::::default(), + InMemoryBackend::::from(None), self.shared.clone(), Some(fork_at), ); diff --git a/client/executor/runtime-test/src/lib.rs b/client/executor/runtime-test/src/lib.rs index 436f87d88b1e0..c9f7d6b1e2970 100644 --- a/client/executor/runtime-test/src/lib.rs +++ b/client/executor/runtime-test/src/lib.rs @@ -54,20 +54,10 @@ static mut MUTABLE_STATIC: u64 = 32; static mut MUTABLE_STATIC_BSS: u64 = 0; sp_core::wasm_export_functions! { - fn test_switch_state() { - print("switch_state"); - storage::set( - sp_storage::well_known_keys::TRIE_HASHING_CONFIG, - sp_storage::trie_threshold_encode(sp_storage::TEST_DEFAULT_ALT_HASH_THRESHOLD).as_slice(), - ); - print("switched!"); - } - fn test_calling_missing_external() { unsafe { missing_external() } } - fn test_calling_yet_another_missing_external() { unsafe { yet_another_missing_external() } } diff --git a/client/executor/src/integration_tests/mod.rs b/client/executor/src/integration_tests/mod.rs index 357e3ea972a37..c4ded542d3946 100644 --- a/client/executor/src/integration_tests/mod.rs +++ b/client/executor/src/integration_tests/mod.rs @@ -198,7 +198,7 @@ fn storage_should_work(wasm_method: WasmExecutionMethod) { ], children_default: map![], }; - let expected = TestExternalities::new_with_alt_hashing(storage); + let expected = TestExternalities::new(storage, Some(Some(sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD))); assert_eq!(ext, expected); } @@ -230,7 +230,7 @@ fn clear_prefix_should_work(wasm_method: WasmExecutionMethod) { children_default: map![], }; - let expected = TestExternalities::new_with_alt_hashing(storage); + let expected = TestExternalities::new(storage, Some(Some(sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD))); assert_eq!(expected, ext); } @@ -662,52 +662,3 @@ fn panic_in_spawned_instance_panics_on_joining_its_result(wasm_method: WasmExecu assert!(format!("{}", error_result).contains("Spawned task")); } - -test_wasm_execution!(state_hashing_update); -fn state_hashing_update(wasm_method: WasmExecutionMethod) { - // use externalities without storage flag. - let mut ext = TestExternalities::new(Default::default()); - - let root1 = { - let mut ext = ext.ext(); - ext.set_storage(b"foo".to_vec(), b"bar".to_vec()); - let output = - call_in_wasm("test_data_in", &vec![1u8; 100].encode(), wasm_method, &mut ext).unwrap(); - - assert_eq!(output, b"all ok!".to_vec().encode()); - ext.storage_root() - }; - - ext.commit_all().unwrap(); - let root2 = { - let mut ext = ext.ext(); - // flag state. - let _ = - call_in_wasm("test_switch_state", Default::default(), wasm_method, &mut ext).unwrap(); - ext.storage_root() - }; - - assert!(root1 != root2); - - ext.commit_all().unwrap(); - let root3 = { - let mut ext = ext.ext(); - let _ = - call_in_wasm("test_data_in", &vec![2u8; 100].to_vec().encode(), wasm_method, &mut ext) - .unwrap(); - ext.storage_root() - }; - assert!(root2 != root3); - - ext.commit_all().unwrap(); - let root3 = { - let mut ext = ext.ext(); - // revert to root 2 state, but this time - // inner hashing should apply - let _ = - call_in_wasm("test_data_in", &vec![1u8; 100].to_vec().encode(), wasm_method, &mut ext) - .unwrap(); - ext.storage_root() - }; - assert!(root2 != root3); -} diff --git a/client/network/src/light_client_requests/sender.rs b/client/network/src/light_client_requests/sender.rs index 9e3185d94438a..3e6c069bcdc16 100644 --- a/client/network/src/light_client_requests/sender.rs +++ b/client/network/src/light_client_requests/sender.rs @@ -830,7 +830,7 @@ mod tests { use std::{collections::HashSet, iter::FromIterator}; fn empty_proof() -> Vec { - StorageProof::empty().encode() + StorageProof::empty(None).encode() } #[test] diff --git a/client/service/src/builder.rs b/client/service/src/builder.rs index 93f6fe19cb348..819f24e596097 100644 --- a/client/service/src/builder.rs +++ b/client/service/src/builder.rs @@ -349,6 +349,16 @@ where }) .collect::, Error>>()?; + let mut state_versions = Vec::new(); + for (number, version) in config.chain_spec.state_versions().into_iter() { + let number = NumberFor::::from_str(&number.to_string()) + .map_err(|_| Error::Application(Box::from(format!( + "Failed to parse `{}` as block number for state versions.", + number, + ))))?; + state_versions.push((number.into(), version)); + } + let client = new_client( backend.clone(), executor, @@ -368,6 +378,7 @@ where sc_network::config::SyncMode::Fast { .. } | sc_network::config::SyncMode::Warp ), wasm_runtime_substitutes, + state_versions, }, )?; diff --git a/client/service/src/client/client.rs b/client/service/src/client/client.rs index 9439a06a5af95..8fe999cd60426 100644 --- a/client/service/src/client/client.rs +++ b/client/service/src/client/client.rs @@ -199,6 +199,8 @@ pub struct ClientConfig { /// Map of WASM runtime substitute starting at the child of the given block until the runtime /// version doesn't match anymore. pub wasm_runtime_substitutes: HashMap>, + /// State version to use with chain. + pub state_versions: Vec<(NumberFor, Option>)>, } impl Default for ClientConfig { @@ -209,6 +211,7 @@ impl Default for ClientConfig { wasm_runtime_overrides: None, no_genesis: false, wasm_runtime_substitutes: HashMap::new(), + state_versions: Vec::new(), // TODO check usage or remove default impl } } } @@ -334,9 +337,13 @@ where config: ClientConfig, ) -> sp_blockchain::Result { let info = backend.blockchain().info(); + let state_version = config.state_versions.get(0) + .and_then(|(n, s)| n.is_zero().then(|| s.clone())) + .unwrap_or(None); + if info.finalized_state.is_none() { let genesis_storage = - build_genesis_storage.build_storage().map_err(sp_blockchain::Error::Storage)?; + build_genesis_storage.build_storage(state_version).map_err(sp_blockchain::Error::Storage)?; let mut op = backend.begin_operation()?; let state_root = op.set_genesis_state(genesis_storage, !config.no_genesis)?; let genesis_block = genesis::construct_genesis_block::(state_root.into()); diff --git a/client/service/src/client/wasm_override.rs b/client/service/src/client/wasm_override.rs index a04a48f9c4b49..7d9fd555ec7ca 100644 --- a/client/service/src/client/wasm_override.rs +++ b/client/service/src/client/wasm_override.rs @@ -181,7 +181,7 @@ where code: &WasmBlob, heap_pages: Option, ) -> Result { - let mut ext = BasicExternalities::default(); + let mut ext: BasicExternalities = None.into(); executor .runtime_version(&mut ext, &code.runtime_code(heap_pages)) .map_err(|e| WasmOverrideError::VersionInvalid(format!("{:?}", e)).into()) diff --git a/client/service/src/client/wasm_substitutes.rs b/client/service/src/client/wasm_substitutes.rs index 28975790e9b57..076d0bdc647b4 100644 --- a/client/service/src/client/wasm_substitutes.rs +++ b/client/service/src/client/wasm_substitutes.rs @@ -178,7 +178,7 @@ where executor: &Executor, code: &WasmSubstitute, ) -> Result { - let mut ext = BasicExternalities::default(); + let mut ext: BasicExternalities = None.into(); executor .runtime_version(&mut ext, &code.runtime_code(None)) .map_err(|e| WasmSubstituteError::VersionInvalid(format!("{:?}", e)).into()) diff --git a/client/service/test/src/client/light.rs b/client/service/test/src/client/light.rs index 963b29de73690..2871858157fe0 100644 --- a/client/service/test/src/client/light.rs +++ b/client/service/test/src/client/light.rs @@ -268,8 +268,8 @@ fn local_state_is_created_when_genesis_state_is_available() { let header0 = substrate_test_runtime_client::runtime::Header::new(0, def, def, def, Default::default()); - let backend: Backend<_, BlakeTwo256> = - Backend::new(Arc::new(DummyBlockchain::new(DummyStorage::new()))); + let backend: Backend<_, Block> = + Backend::new(Arc::new(DummyBlockchain::new(DummyStorage::new())), vec![]); let mut op = backend.begin_operation().unwrap(); op.set_block_data(header0, None, None, None, NewBlockState::Final).unwrap(); op.set_genesis_state(Default::default(), true).unwrap(); @@ -283,8 +283,8 @@ fn local_state_is_created_when_genesis_state_is_available() { #[test] fn unavailable_state_is_created_when_genesis_state_is_unavailable() { - let backend: Backend<_, BlakeTwo256> = - Backend::new(Arc::new(DummyBlockchain::new(DummyStorage::new()))); + let backend: Backend<_, Block> = + Backend::new(Arc::new(DummyBlockchain::new(DummyStorage::new())), vec![]); match backend.state_at(BlockId::Number(0)).unwrap() { GenesisOrUnavailableState::Unavailable => (), @@ -294,7 +294,7 @@ fn unavailable_state_is_created_when_genesis_state_is_unavailable() { #[test] fn light_aux_store_is_updated_via_non_importing_op() { - let backend = Backend::new(Arc::new(DummyBlockchain::new(DummyStorage::new()))); + let backend = Backend::new(Arc::new(DummyBlockchain::new(DummyStorage::new())), vec![]); let mut op = ClientBackend::::begin_operation(&backend).unwrap(); BlockImportOperation::::insert_aux(&mut op, vec![(vec![1], Some(vec![2]))]).unwrap(); ClientBackend::::commit_operation(&backend, op).unwrap(); @@ -891,7 +891,7 @@ fn check_changes_proof_fails_if_proof_is_wrong() { max_block: remote_proof.max_block, proof: remote_proof.proof.clone(), roots: vec![(begin - 1, Default::default())].into_iter().collect(), - roots_proof: StorageProof::empty(), + roots_proof: StorageProof::empty(None), } ) .is_err()); @@ -902,7 +902,7 @@ fn check_changes_proof_fails_if_proof_is_wrong() { max_block: remote_proof.max_block, proof: remote_proof.proof.clone(), roots: vec![(end + 1, Default::default())].into_iter().collect(), - roots_proof: StorageProof::empty(), + roots_proof: StorageProof::empty(None), } ) .is_err()); @@ -952,7 +952,7 @@ fn check_changes_tries_proof_fails_if_proof_is_wrong() { Box::new(TaskExecutor::new()), ); let result = - local_checker.check_changes_tries_proof(4, &remote_proof.roots, StorageProof::empty()); + local_checker.check_changes_tries_proof(4, &remote_proof.roots, StorageProof::empty(None)); assert!(result.is_err()); } diff --git a/client/service/test/src/client/mod.rs b/client/service/test/src/client/mod.rs index 87fd83fe90406..d8fffa5fefc2a 100644 --- a/client/service/test/src/client/mod.rs +++ b/client/service/test/src/client/mod.rs @@ -243,6 +243,7 @@ fn block1(genesis_hash: Hash, backend: &InMemoryBackend) -> (Vec = client .storage_keys_iter( diff --git a/frame/assets/src/mock.rs b/frame/assets/src/mock.rs index 1b2602792d844..338157851d882 100644 --- a/frame/assets/src/mock.rs +++ b/frame/assets/src/mock.rs @@ -144,9 +144,10 @@ pub(crate) fn hooks() -> Vec { } pub(crate) fn new_test_ext() -> sp_io::TestExternalities { - let t = frame_system::GenesisConfig::default().build_storage::().unwrap(); + let state_version = None; + let t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); - let mut ext = sp_io::TestExternalities::new(t); + let mut ext = sp_io::TestExternalities::new(t, state_version); ext.execute_with(|| System::set_block_number(1)); ext } diff --git a/frame/atomic-swap/src/tests.rs b/frame/atomic-swap/src/tests.rs index a76d0f20ffa3b..22bc399e5729b 100644 --- a/frame/atomic-swap/src/tests.rs +++ b/frame/atomic-swap/src/tests.rs @@ -83,10 +83,11 @@ const A: u64 = 1; const B: u64 = 2; pub fn new_test_ext() -> sp_io::TestExternalities { - let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); + let state_version = None; + let mut t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); let genesis = pallet_balances::GenesisConfig:: { balances: vec![(A, 100), (B, 200)] }; - genesis.assimilate_storage(&mut t).unwrap(); - t.into() + genesis.assimilate_storage(&mut t, state_version.clone()).unwrap(); + (t, state_version).into() } #[test] diff --git a/frame/aura/src/mock.rs b/frame/aura/src/mock.rs index 0e258fb9a67dc..3eb31893a5de2 100644 --- a/frame/aura/src/mock.rs +++ b/frame/aura/src/mock.rs @@ -116,11 +116,12 @@ impl pallet_aura::Config for Test { } pub fn new_test_ext(authorities: Vec) -> sp_io::TestExternalities { - let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); + let state_version = None; + let mut t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); pallet_aura::GenesisConfig:: { authorities: authorities.into_iter().map(|a| UintAuthorityId(a).to_public_key()).collect(), } - .assimilate_storage(&mut t) + .assimilate_storage(&mut t, state_version.clone()) .unwrap(); - t.into() + (t, state_version).into() } diff --git a/frame/authority-discovery/src/lib.rs b/frame/authority-discovery/src/lib.rs index 4577a9dd17226..5d8eb708ede95 100644 --- a/frame/authority-discovery/src/lib.rs +++ b/frame/authority-discovery/src/lib.rs @@ -245,6 +245,7 @@ mod tests { #[test] fn authorities_returns_current_and_next_authority_set() { + let state_version = None; // The whole authority discovery pallet ignores account ids, but we still need them for // `pallet_session::OneSessionHandler::on_new_session`, thus its safe to use the same value // everywhere. @@ -281,16 +282,17 @@ mod tests { .collect::>(); // Build genesis. - let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); + let mut t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); GenesisBuild::::assimilate_storage( &pallet_authority_discovery::GenesisConfig { keys: vec![] }, &mut t, + state_version.clone(), ) .unwrap(); // Create externalities. - let mut externalities = TestExternalities::new(t); + let mut externalities = TestExternalities::new(t, state_version); externalities.execute_with(|| { use frame_support::traits::OneSessionHandler; diff --git a/frame/authorship/src/lib.rs b/frame/authorship/src/lib.rs index 325f80c74aa1e..da1e3b80ae291 100644 --- a/frame/authorship/src/lib.rs +++ b/frame/authorship/src/lib.rs @@ -533,8 +533,9 @@ mod tests { } fn new_test_ext() -> sp_io::TestExternalities { - let t = frame_system::GenesisConfig::default().build_storage::().unwrap(); - t.into() + let state_version = None; + let t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); + (t, state_version).into() } #[test] diff --git a/frame/babe/src/mock.rs b/frame/babe/src/mock.rs index f872fb23b12aa..327c0779c2d2a 100644 --- a/frame/babe/src/mock.rs +++ b/frame/babe/src/mock.rs @@ -380,12 +380,13 @@ pub fn new_test_ext_with_pairs( } pub fn new_test_ext_raw_authorities(authorities: Vec) -> sp_io::TestExternalities { - let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); + let state_version = None; + let mut t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); let balances: Vec<_> = (0..authorities.len()).map(|i| (i as u64, 10_000_000)).collect(); pallet_balances::GenesisConfig:: { balances } - .assimilate_storage(&mut t) + .assimilate_storage(&mut t, state_version.clone()) .unwrap(); // stashes are the index. @@ -400,7 +401,7 @@ pub fn new_test_ext_raw_authorities(authorities: Vec) -> sp_io::Tes // NOTE: this will initialize the babe authorities // through OneSessionHandler::on_genesis_session pallet_session::GenesisConfig:: { keys: session_keys } - .assimilate_storage(&mut t) + .assimilate_storage(&mut t, state_version.clone()) .unwrap(); // controllers are the index + 1000 @@ -419,9 +420,9 @@ pub fn new_test_ext_raw_authorities(authorities: Vec) -> sp_io::Tes ..Default::default() }; - staking_config.assimilate_storage(&mut t).unwrap(); + staking_config.assimilate_storage(&mut t, state_version.clone()).unwrap(); - t.into() + (t, state_version).into() } /// Creates an equivocation at the current block, by generating two headers. diff --git a/frame/balances/src/lib.rs b/frame/balances/src/lib.rs index 7ab8a54de232d..b5fbc894ca413 100644 --- a/frame/balances/src/lib.rs +++ b/frame/balances/src/lib.rs @@ -586,15 +586,15 @@ impl, I: 'static> GenesisConfig { /// Direct implementation of `GenesisBuild::build_storage`. /// /// Kept in order not to break dependency. - pub fn build_storage(&self) -> Result { - >::build_storage(self) + pub fn build_storage(&self, alt_hashing: Option>) -> Result { + >::build_storage(self, alt_hashing) } /// Direct implementation of `GenesisBuild::assimilate_storage`. /// /// Kept in order not to break dependency. - pub fn assimilate_storage(&self, storage: &mut sp_runtime::Storage) -> Result<(), String> { - >::assimilate_storage(self, storage) + pub fn assimilate_storage(&self, storage: &mut sp_runtime::Storage, alt_hashing: Option>) -> Result<(), String> { + >::assimilate_storage(self, storage, alt_hashing) } } diff --git a/frame/balances/src/tests.rs b/frame/balances/src/tests.rs index fd57371b3a16b..f2bfc8bfd5339 100644 --- a/frame/balances/src/tests.rs +++ b/frame/balances/src/tests.rs @@ -629,20 +629,22 @@ macro_rules! decl_tests { #[test] #[should_panic = "the balance of any account should always be at least the existential deposit."] fn cannot_set_genesis_value_below_ed() { + let state_version = None; ($existential_deposit).with(|v| *v.borrow_mut() = 11); - let mut t = frame_system::GenesisConfig::default().build_storage::<$test>().unwrap(); + let mut t = frame_system::GenesisConfig::default().build_storage::<$test>(state_version.clone()).unwrap(); let _ = pallet_balances::GenesisConfig::<$test> { balances: vec![(1, 10)], - }.assimilate_storage(&mut t).unwrap(); + }.assimilate_storage(&mut t, state_version).unwrap(); } #[test] #[should_panic = "duplicate balances in genesis."] fn cannot_set_genesis_value_twice() { - let mut t = frame_system::GenesisConfig::default().build_storage::<$test>().unwrap(); + let state_version = None; + let mut t = frame_system::GenesisConfig::default().build_storage::<$test>(state_version.clone()).unwrap(); let _ = pallet_balances::GenesisConfig::<$test> { balances: vec![(1, 10), (2, 20), (1, 15)], - }.assimilate_storage(&mut t).unwrap(); + }.assimilate_storage(&mut t, state_version).unwrap(); } #[test] diff --git a/frame/balances/src/tests_composite.rs b/frame/balances/src/tests_composite.rs index f6faebed39316..720e0738a92a2 100644 --- a/frame/balances/src/tests_composite.rs +++ b/frame/balances/src/tests_composite.rs @@ -122,8 +122,9 @@ impl ExtBuilder { EXISTENTIAL_DEPOSIT.with(|v| *v.borrow_mut() = self.existential_deposit); } pub fn build(self) -> sp_io::TestExternalities { + let state_version = None; self.set_associated_consts(); - let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); + let mut t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); pallet_balances::GenesisConfig:: { balances: if self.monied { vec![ @@ -137,10 +138,10 @@ impl ExtBuilder { vec![] }, } - .assimilate_storage(&mut t) + .assimilate_storage(&mut t, state_version.clone()) .unwrap(); - let mut ext = sp_io::TestExternalities::new(t); + let mut ext = sp_io::TestExternalities::new(t, state_version); ext.execute_with(|| System::set_block_number(1)); ext } diff --git a/frame/balances/src/tests_local.rs b/frame/balances/src/tests_local.rs index d8c07aa9c42e5..9fd41d7ef9e1a 100644 --- a/frame/balances/src/tests_local.rs +++ b/frame/balances/src/tests_local.rs @@ -127,8 +127,9 @@ impl ExtBuilder { EXISTENTIAL_DEPOSIT.with(|v| *v.borrow_mut() = self.existential_deposit); } pub fn build(self) -> sp_io::TestExternalities { + let state_version = None; self.set_associated_consts(); - let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); + let mut t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); pallet_balances::GenesisConfig:: { balances: if self.monied { vec![ @@ -142,10 +143,10 @@ impl ExtBuilder { vec![] }, } - .assimilate_storage(&mut t) + .assimilate_storage(&mut t, state_version.clone()) .unwrap(); - let mut ext = sp_io::TestExternalities::new(t); + let mut ext = sp_io::TestExternalities::new(t, state_version); ext.execute_with(|| System::set_block_number(1)); ext } diff --git a/frame/balances/src/tests_reentrancy.rs b/frame/balances/src/tests_reentrancy.rs index 9c7ba3e1ec824..c5794a9d3ca16 100644 --- a/frame/balances/src/tests_reentrancy.rs +++ b/frame/balances/src/tests_reentrancy.rs @@ -130,12 +130,13 @@ impl ExtBuilder { } pub fn build(self) -> sp_io::TestExternalities { + let state_version = None; self.set_associated_consts(); - let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); + let mut t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); pallet_balances::GenesisConfig:: { balances: vec![] } - .assimilate_storage(&mut t) + .assimilate_storage(&mut t, state_version.clone()) .unwrap(); - let mut ext = sp_io::TestExternalities::new(t); + let mut ext = sp_io::TestExternalities::new(t, state_version); ext.execute_with(|| System::set_block_number(1)); ext } diff --git a/frame/benchmarking/src/tests.rs b/frame/benchmarking/src/tests.rs index 9cb5043a0dd7d..5ff8a6dbd48dd 100644 --- a/frame/benchmarking/src/tests.rs +++ b/frame/benchmarking/src/tests.rs @@ -128,7 +128,9 @@ impl pallet_test::OtherConfig for Test { } fn new_test_ext() -> sp_io::TestExternalities { - GenesisConfig::default().build_storage().unwrap().into() + let state_version = None; + let storage = GenesisConfig::default().build_storage(state_version.clone()).unwrap(); + (storage, state_version).into() } mod benchmarks { diff --git a/frame/benchmarking/src/tests_instance.rs b/frame/benchmarking/src/tests_instance.rs index caccebd39c70b..ab4cb6de3e23c 100644 --- a/frame/benchmarking/src/tests_instance.rs +++ b/frame/benchmarking/src/tests_instance.rs @@ -128,7 +128,9 @@ impl pallet_test::OtherConfig for Test { } fn new_test_ext() -> sp_io::TestExternalities { - GenesisConfig::default().build_storage().unwrap().into() + let state_version = None; + let storage = GenesisConfig::default().build_storage(state_version.clone()).unwrap(); + (storage, state_version).into() } mod benchmarks { diff --git a/frame/bounties/src/tests.rs b/frame/bounties/src/tests.rs index ff058a3601e07..94ee6b150ca70 100644 --- a/frame/bounties/src/tests.rs +++ b/frame/bounties/src/tests.rs @@ -149,15 +149,16 @@ impl Config for Test { type TreasuryError = pallet_treasury::Error; pub fn new_test_ext() -> sp_io::TestExternalities { - let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); + let state_version = None; + let mut t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); pallet_balances::GenesisConfig:: { // Total issuance will be 200 with treasury account initialized at ED. balances: vec![(0, 100), (1, 98), (2, 1)], } - .assimilate_storage(&mut t) + .assimilate_storage(&mut t, state_version.clone()) .unwrap(); - GenesisBuild::::assimilate_storage(&pallet_treasury::GenesisConfig, &mut t).unwrap(); - t.into() + GenesisBuild::::assimilate_storage(&pallet_treasury::GenesisConfig, &mut t, state_version.clone()).unwrap(); + (t, state_version).into() } fn last_event() -> RawEvent { @@ -356,12 +357,13 @@ fn treasury_account_doesnt_get_deleted() { // This is useful for chain that will just update runtime. #[test] fn inexistent_account_works() { - let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); + let state_version = None; + let mut t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); pallet_balances::GenesisConfig:: { balances: vec![(0, 100), (1, 99), (2, 1)] } - .assimilate_storage(&mut t) + .assimilate_storage(&mut t, state_version.clone()) .unwrap(); // Treasury genesis config is not build thus treasury account does not exist - let mut t: sp_io::TestExternalities = t.into(); + let mut t: sp_io::TestExternalities = (t, state_version).into(); t.execute_with(|| { assert_eq!(Balances::free_balance(Treasury::account_id()), 0); // Account does not exist @@ -936,16 +938,17 @@ fn extend_expiry() { #[test] fn genesis_funding_works() { - let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); + let state_version = None; + let mut t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); let initial_funding = 100; pallet_balances::GenesisConfig:: { // Total issuance will be 200 with treasury account initialized with 100. balances: vec![(0, 100), (Treasury::account_id(), initial_funding)], } - .assimilate_storage(&mut t) + .assimilate_storage(&mut t, state_version.clone()) .unwrap(); - GenesisBuild::::assimilate_storage(&pallet_treasury::GenesisConfig, &mut t).unwrap(); - let mut t: sp_io::TestExternalities = t.into(); + GenesisBuild::::assimilate_storage(&pallet_treasury::GenesisConfig, &mut t, state_version).unwrap(); + let mut t: sp_io::TestExternalities = (t, state_version).into(); t.execute_with(|| { assert_eq!(Balances::free_balance(Treasury::account_id()), initial_funding); diff --git a/frame/collective/src/lib.rs b/frame/collective/src/lib.rs index 39da8e2c45fb7..eff09338dc4c2 100644 --- a/frame/collective/src/lib.rs +++ b/frame/collective/src/lib.rs @@ -1083,7 +1083,8 @@ mod tests { ); pub fn new_test_ext() -> sp_io::TestExternalities { - let mut ext: sp_io::TestExternalities = GenesisConfig { + let state_version = None; + let mut ext: sp_io::TestExternalities = (GenesisConfig { collective: collective::GenesisConfig { members: vec![1, 2, 3], phantom: Default::default(), @@ -1094,9 +1095,8 @@ mod tests { }, default_collective: Default::default(), } - .build_storage() - .unwrap() - .into(); + .build_storage(state_version.clone()) + .unwrap(), state_version).into(); ext.execute_with(|| System::set_block_number(1)); ext } @@ -2158,11 +2158,12 @@ mod tests { #[test] #[should_panic(expected = "Members cannot contain duplicate accounts.")] fn genesis_build_panics_with_duplicate_members() { + let state_version = None; collective::GenesisConfig:: { members: vec![1, 2, 3, 1], phantom: Default::default(), } - .build_storage() + .build_storage(state_version) .unwrap(); } } diff --git a/frame/contracts/src/tests.rs b/frame/contracts/src/tests.rs index fc5a2cf5221e0..beace619ad4df 100644 --- a/frame/contracts/src/tests.rs +++ b/frame/contracts/src/tests.rs @@ -336,12 +336,13 @@ impl ExtBuilder { EXISTENTIAL_DEPOSIT.with(|v| *v.borrow_mut() = self.existential_deposit); } pub fn build(self) -> sp_io::TestExternalities { + let state_version = None; self.set_associated_consts(); - let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); + let mut t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); pallet_balances::GenesisConfig:: { balances: vec![] } - .assimilate_storage(&mut t) + .assimilate_storage(&mut t, state_version.clone()) .unwrap(); - let mut ext = sp_io::TestExternalities::new(t); + let mut ext = sp_io::TestExternalities::new(t, state_version); ext.execute_with(|| System::set_block_number(1)); ext } diff --git a/frame/democracy/src/tests.rs b/frame/democracy/src/tests.rs index 46d3cade36eb1..bde28cd2a67aa 100644 --- a/frame/democracy/src/tests.rs +++ b/frame/democracy/src/tests.rs @@ -194,16 +194,17 @@ impl Config for Test { } pub fn new_test_ext() -> sp_io::TestExternalities { - let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); + let state_version = None; + let mut t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); pallet_balances::GenesisConfig:: { balances: vec![(1, 10), (2, 20), (3, 30), (4, 40), (5, 50), (6, 60)], } - .assimilate_storage(&mut t) + .assimilate_storage(&mut t, state_version.clone()) .unwrap(); pallet_democracy::GenesisConfig::::default() - .assimilate_storage(&mut t) + .assimilate_storage(&mut t, state_version.clone()) .unwrap(); - let mut ext = sp_io::TestExternalities::new(t); + let mut ext = sp_io::TestExternalities::new(t, state_version); ext.execute_with(|| System::set_block_number(1)); ext } diff --git a/frame/election-provider-multi-phase/src/mock.rs b/frame/election-provider-multi-phase/src/mock.rs index f760676abf761..53dc66540db84 100644 --- a/frame/election-provider-multi-phase/src/mock.rs +++ b/frame/election-provider-multi-phase/src/mock.rs @@ -511,9 +511,10 @@ impl ExtBuilder { self } pub fn build(self) -> sp_io::TestExternalities { + let state_version = None; sp_tracing::try_init_simple(); let mut storage = - frame_system::GenesisConfig::default().build_storage::().unwrap(); + frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); let _ = pallet_balances::GenesisConfig:: { balances: vec![ @@ -523,9 +524,9 @@ impl ExtBuilder { (9999, 100), ], } - .assimilate_storage(&mut storage); + .assimilate_storage(&mut storage, state_version.clone()); - sp_io::TestExternalities::from(storage) + sp_io::TestExternalities::from((storage, state_version)) } pub fn build_offchainify( diff --git a/frame/elections-phragmen/src/lib.rs b/frame/elections-phragmen/src/lib.rs index 331d34180e98c..174121282cf42 100644 --- a/frame/elections-phragmen/src/lib.rs +++ b/frame/elections-phragmen/src/lib.rs @@ -1303,7 +1303,8 @@ mod tests { *m.borrow_mut() = self.genesis_members.iter().map(|(m, _)| m.clone()).collect::>() }); - let mut ext: sp_io::TestExternalities = GenesisConfig { + let state_version = None; + let mut ext: sp_io::TestExternalities = (GenesisConfig { balances: pallet_balances::GenesisConfig:: { balances: vec![ (1, 10 * self.balance_factor), @@ -1318,8 +1319,8 @@ mod tests { members: self.genesis_members, }, } - .build_storage() - .unwrap() + .build_storage(state_version) + .unwrap(), state_version) .into(); ext.execute_with(pre_conditions); ext.execute_with(test); diff --git a/frame/elections/src/mock.rs b/frame/elections/src/mock.rs index 91318e1e07bcc..bec7e56c5ecb0 100644 --- a/frame/elections/src/mock.rs +++ b/frame/elections/src/mock.rs @@ -193,11 +193,12 @@ impl ExtBuilder { self } pub fn build(self) -> sp_io::TestExternalities { + let state_version = None; VOTING_BOND.with(|v| *v.borrow_mut() = self.voting_bond); VOTING_FEE.with(|v| *v.borrow_mut() = self.voting_fee); PRESENT_SLASH_PER_VOTER.with(|v| *v.borrow_mut() = self.bad_presentation_punishment); DECAY_RATIO.with(|v| *v.borrow_mut() = self.decay_ratio); - let mut ext: sp_io::TestExternalities = GenesisConfig { + let mut ext: sp_io::TestExternalities = (GenesisConfig { balances: pallet_balances::GenesisConfig:: { balances: vec![ (1, 10 * self.balance_factor), @@ -215,8 +216,8 @@ impl ExtBuilder { term_duration: 5, }, } - .build_storage() - .unwrap() + .build_storage(state_version.clone()) + .unwrap(), state_version) .into(); ext.execute_with(|| System::set_block_number(1)); ext diff --git a/frame/example/src/tests.rs b/frame/example/src/tests.rs index 645b5c9bc13ab..a9f26eef53dae 100644 --- a/frame/example/src/tests.rs +++ b/frame/example/src/tests.rs @@ -107,6 +107,7 @@ impl Config for Test { // This function basically just builds a genesis storage key/value store according to // our desired mockup. pub fn new_test_ext() -> sp_io::TestExternalities { + let state_version = None; let t = GenesisConfig { // We use default for brevity, but you can configure as desired if needed. system: Default::default(), @@ -118,9 +119,9 @@ pub fn new_test_ext() -> sp_io::TestExternalities { foo: 24, }, } - .build_storage() + .build_storage(state_version.clone()) .unwrap(); - t.into() + (t, state_version).into() } #[test] diff --git a/frame/executive/src/lib.rs b/frame/executive/src/lib.rs index 5f1ae23c2f531..d4b2a34f62772 100644 --- a/frame/executive/src/lib.rs +++ b/frame/executive/src/lib.rs @@ -811,9 +811,10 @@ mod tests { #[test] fn balance_transfer_dispatch_works() { - let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); + let state_version = None; + let mut t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); pallet_balances::GenesisConfig:: { balances: vec![(1, 211)] } - .assimilate_storage(&mut t) + .assimilate_storage(&mut t, state_version.clone()) .unwrap(); let xt = TestXt::new(Call::Balances(BalancesCall::transfer(2, 69)), sign_extra(1, 0, 0)); let weight = xt.get_dispatch_info().weight + @@ -822,7 +823,7 @@ mod tests { .base_extrinsic; let fee: Balance = ::WeightToFee::calc(&weight); - let mut t = sp_io::TestExternalities::new(t); + let mut t = sp_io::TestExternalities::new(t, state_version); t.execute_with(|| { Executive::initialize_block(&Header::new( 1, @@ -839,11 +840,12 @@ mod tests { } fn new_test_ext(balance_factor: Balance) -> sp_io::TestExternalities { - let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); + let state_version = None; + let mut t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); pallet_balances::GenesisConfig:: { balances: vec![(1, 111 * balance_factor)] } - .assimilate_storage(&mut t) + .assimilate_storage(&mut t, state_version.clone()) .unwrap(); - t.into() + (t, state_version).into() } #[test] diff --git a/frame/gilt/src/mock.rs b/frame/gilt/src/mock.rs index ac3f4df1b71dd..71e47ac6de642 100644 --- a/frame/gilt/src/mock.rs +++ b/frame/gilt/src/mock.rs @@ -127,14 +127,15 @@ impl pallet_gilt::Config for Test { // This function basically just builds a genesis storage key/value store according to // our desired mockup. pub fn new_test_ext() -> sp_io::TestExternalities { - let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); + let state_version = None; + let mut t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); pallet_balances::GenesisConfig:: { balances: vec![(1, 100), (2, 100), (3, 100), (4, 100)], } - .assimilate_storage(&mut t) + .assimilate_storage(&mut t, state_version.clone()) .unwrap(); - GenesisBuild::::assimilate_storage(&crate::GenesisConfig, &mut t).unwrap(); - t.into() + GenesisBuild::::assimilate_storage(&crate::GenesisConfig, &mut t, state_version.clone()).unwrap(); + (t, state_version).into() } pub fn run_to_block(n: u64) { diff --git a/frame/grandpa/src/mock.rs b/frame/grandpa/src/mock.rs index b9c4858e353e8..a2b1a61ddac07 100644 --- a/frame/grandpa/src/mock.rs +++ b/frame/grandpa/src/mock.rs @@ -275,12 +275,13 @@ pub fn new_test_ext(vec: Vec<(u64, u64)>) -> sp_io::TestExternalities { } pub fn new_test_ext_raw_authorities(authorities: AuthorityList) -> sp_io::TestExternalities { - let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); + let state_version = None; + let mut t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); let balances: Vec<_> = (0..authorities.len()).map(|i| (i as u64, 10_000_000)).collect(); pallet_balances::GenesisConfig:: { balances } - .assimilate_storage(&mut t) + .assimilate_storage(&mut t, state_version.clone()) .unwrap(); // stashes are the index. @@ -299,7 +300,7 @@ pub fn new_test_ext_raw_authorities(authorities: AuthorityList) -> sp_io::TestEx // NOTE: this will initialize the grandpa authorities // through OneSessionHandler::on_genesis_session pallet_session::GenesisConfig:: { keys: session_keys } - .assimilate_storage(&mut t) + .assimilate_storage(&mut t, state_version.clone()) .unwrap(); // controllers are the index + 1000 @@ -318,9 +319,9 @@ pub fn new_test_ext_raw_authorities(authorities: AuthorityList) -> sp_io::TestEx ..Default::default() }; - staking_config.assimilate_storage(&mut t).unwrap(); + staking_config.assimilate_storage(&mut t, state_version.clone()).unwrap(); - t.into() + (t, state_version).into() } pub fn start_session(session_index: SessionIndex) { diff --git a/frame/identity/src/tests.rs b/frame/identity/src/tests.rs index c842b0e2f64be..ef14b7753edfa 100644 --- a/frame/identity/src/tests.rs +++ b/frame/identity/src/tests.rs @@ -118,13 +118,14 @@ impl pallet_identity::Config for Test { } pub fn new_test_ext() -> sp_io::TestExternalities { - let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); + let state_version = None; + let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); pallet_balances::GenesisConfig:: { balances: vec![(1, 10), (2, 10), (3, 10), (10, 100), (20, 100), (30, 100)], } - .assimilate_storage(&mut t) + .assimilate_storage(&mut t, None) .unwrap(); - t.into() + (t, state_version).into() } fn ten() -> IdentityInfo { diff --git a/frame/im-online/src/mock.rs b/frame/im-online/src/mock.rs index e4031b04271b9..7a4a50d0269f7 100644 --- a/frame/im-online/src/mock.rs +++ b/frame/im-online/src/mock.rs @@ -105,8 +105,9 @@ impl ReportOffence for OffenceHandler { } pub fn new_test_ext() -> sp_io::TestExternalities { - let t = frame_system::GenesisConfig::default().build_storage::().unwrap(); - t.into() + let state_version = None; + let t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); + (t, state_version).into() } parameter_types! { diff --git a/frame/indices/src/mock.rs b/frame/indices/src/mock.rs index f4c87016141b5..497c239008d5c 100644 --- a/frame/indices/src/mock.rs +++ b/frame/indices/src/mock.rs @@ -100,11 +100,12 @@ impl Config for Test { } pub fn new_test_ext() -> sp_io::TestExternalities { - let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); + let state_version = None; + let mut t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); pallet_balances::GenesisConfig:: { balances: vec![(1, 10), (2, 20), (3, 30), (4, 40), (5, 50), (6, 60)], } - .assimilate_storage(&mut t) + .assimilate_storage(&mut t, state_version.clone()) .unwrap(); - t.into() + (t, state_version).into() } diff --git a/frame/lottery/src/mock.rs b/frame/lottery/src/mock.rs index d1f090aa26dcb..0e8fc8dd60b94 100644 --- a/frame/lottery/src/mock.rs +++ b/frame/lottery/src/mock.rs @@ -120,13 +120,14 @@ pub type SystemCall = frame_system::Call; pub type BalancesCall = pallet_balances::Call; pub fn new_test_ext() -> sp_io::TestExternalities { - let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); + let state_version = None; + let mut t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); pallet_balances::GenesisConfig:: { balances: vec![(1, 100), (2, 100), (3, 100), (4, 100), (5, 100)], } - .assimilate_storage(&mut t) + .assimilate_storage(&mut t, state_version.clone()) .unwrap(); - t.into() + (t, state_version).into() } /// Run until a particular block. diff --git a/frame/membership/src/lib.rs b/frame/membership/src/lib.rs index a9bc59a361f06..7db4fa6a90e10 100644 --- a/frame/membership/src/lib.rs +++ b/frame/membership/src/lib.rs @@ -573,15 +573,16 @@ mod tests { } pub(crate) fn new_test_ext() -> sp_io::TestExternalities { - let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); + let state_version = None; + let mut t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); // We use default for brevity, but you can configure as desired if needed. pallet_membership::GenesisConfig:: { members: vec![10, 20, 30], ..Default::default() } - .assimilate_storage(&mut t) + .assimilate_storage(&mut t, state_version.clone()) .unwrap(); - t.into() + (t, state_version).into() } #[cfg(feature = "runtime-benchmarks")] @@ -738,11 +739,12 @@ mod tests { #[test] #[should_panic(expected = "Members cannot contain duplicate accounts.")] fn genesis_build_panics_with_duplicate_members() { + let state_version = None; pallet_membership::GenesisConfig:: { members: vec![1, 2, 3, 1], phantom: Default::default(), } - .build_storage() + .build_storage(state_version) .unwrap(); } } diff --git a/frame/merkle-mountain-range/src/tests.rs b/frame/merkle-mountain-range/src/tests.rs index 50512e9286951..01f374f52ef3c 100644 --- a/frame/merkle-mountain-range/src/tests.rs +++ b/frame/merkle-mountain-range/src/tests.rs @@ -25,7 +25,8 @@ use sp_core::{ }; pub(crate) fn new_test_ext() -> sp_io::TestExternalities { - frame_system::GenesisConfig::default().build_storage::().unwrap().into() + let state_version = None; + (frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(), state_version).into() } fn register_offchain_ext(ext: &mut sp_io::TestExternalities) { diff --git a/frame/multisig/src/tests.rs b/frame/multisig/src/tests.rs index 2809a106d66ea..ddccf9ac31bfa 100644 --- a/frame/multisig/src/tests.rs +++ b/frame/multisig/src/tests.rs @@ -117,13 +117,14 @@ impl Config for Test { use pallet_balances::{Call as BalancesCall, Error as BalancesError}; pub fn new_test_ext() -> sp_io::TestExternalities { - let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); + let state_version = None; + let mut t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); pallet_balances::GenesisConfig:: { balances: vec![(1, 10), (2, 10), (3, 10), (4, 10), (5, 2)], } - .assimilate_storage(&mut t) + .assimilate_storage(&mut t, state_version.clone()) .unwrap(); - let mut ext = sp_io::TestExternalities::new(t); + let mut ext = sp_io::TestExternalities::new(t, state_version); ext.execute_with(|| System::set_block_number(1)); ext } diff --git a/frame/nicks/src/lib.rs b/frame/nicks/src/lib.rs index a5c22b619a5eb..94e6cd9cb0f3b 100644 --- a/frame/nicks/src/lib.rs +++ b/frame/nicks/src/lib.rs @@ -339,11 +339,12 @@ mod tests { } fn new_test_ext() -> sp_io::TestExternalities { - let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); + let state_version = None; + let mut t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); pallet_balances::GenesisConfig:: { balances: vec![(1, 10), (2, 10)] } - .assimilate_storage(&mut t) + .assimilate_storage(&mut t, state_version.clone()) .unwrap(); - t.into() + (t, state_version).into() } #[test] diff --git a/frame/node-authorization/src/mock.rs b/frame/node-authorization/src/mock.rs index 6c79f601c197d..ee9dfe2a0c292 100644 --- a/frame/node-authorization/src/mock.rs +++ b/frame/node-authorization/src/mock.rs @@ -99,11 +99,12 @@ pub fn test_node(id: u8) -> PeerId { } pub fn new_test_ext() -> sp_io::TestExternalities { - let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); + let state_version = None; + let mut t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); pallet_node_authorization::GenesisConfig:: { nodes: vec![(test_node(10), 10), (test_node(20), 20), (test_node(30), 30)], } - .assimilate_storage(&mut t) + .assimilate_storage(&mut t, state_version.clone()) .unwrap(); - t.into() + (t, state_version).into() } diff --git a/frame/offences/benchmarking/src/mock.rs b/frame/offences/benchmarking/src/mock.rs index 3416dafa3547e..ea704b92c0dbf 100644 --- a/frame/offences/benchmarking/src/mock.rs +++ b/frame/offences/benchmarking/src/mock.rs @@ -226,6 +226,7 @@ frame_support::construct_runtime!( ); pub fn new_test_ext() -> sp_io::TestExternalities { - let t = frame_system::GenesisConfig::default().build_storage::().unwrap(); - sp_io::TestExternalities::new(t) + let state_version = None; + let t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); + sp_io::TestExternalities::new(t, state_version) } diff --git a/frame/offences/src/mock.rs b/frame/offences/src/mock.rs index 5e4c94944b6fd..57fcc9240b84d 100644 --- a/frame/offences/src/mock.rs +++ b/frame/offences/src/mock.rs @@ -120,8 +120,9 @@ impl Config for Runtime { } pub fn new_test_ext() -> sp_io::TestExternalities { - let t = frame_system::GenesisConfig::default().build_storage::().unwrap(); - let mut ext = sp_io::TestExternalities::new(t); + let state_version = None; + let t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); + let mut ext = sp_io::TestExternalities::new(t, state_version); ext.execute_with(|| System::set_block_number(1)); ext } diff --git a/frame/proxy/src/tests.rs b/frame/proxy/src/tests.rs index f3fe1d674a87d..f8e1085a1da77 100644 --- a/frame/proxy/src/tests.rs +++ b/frame/proxy/src/tests.rs @@ -165,13 +165,14 @@ use pallet_balances::{Call as BalancesCall, Error as BalancesError, Event as Bal use pallet_utility::{Call as UtilityCall, Event as UtilityEvent}; pub fn new_test_ext() -> sp_io::TestExternalities { - let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); + let state_version = None; + let mut t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); pallet_balances::GenesisConfig:: { balances: vec![(1, 10), (2, 10), (3, 10), (4, 10), (5, 2)], } - .assimilate_storage(&mut t) + .assimilate_storage(&mut t, state_version.clone()) .unwrap(); - let mut ext = sp_io::TestExternalities::new(t); + let mut ext = sp_io::TestExternalities::new(t, state_version); ext.execute_with(|| System::set_block_number(1)); ext } diff --git a/frame/randomness-collective-flip/src/lib.rs b/frame/randomness-collective-flip/src/lib.rs index 1b1d5cb5cd823..db7b1254b3434 100644 --- a/frame/randomness-collective-flip/src/lib.rs +++ b/frame/randomness-collective-flip/src/lib.rs @@ -224,8 +224,9 @@ mod tests { impl pallet_randomness_collective_flip::Config for Test {} fn new_test_ext() -> sp_io::TestExternalities { - let t = frame_system::GenesisConfig::default().build_storage::().unwrap(); - t.into() + let state_version = None; + let t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); + (t, state_version).into() } #[test] diff --git a/frame/recovery/src/mock.rs b/frame/recovery/src/mock.rs index f6d4a6b159431..731b98b602557 100644 --- a/frame/recovery/src/mock.rs +++ b/frame/recovery/src/mock.rs @@ -114,13 +114,14 @@ pub type BalancesCall = pallet_balances::Call; pub type RecoveryCall = super::Call; pub fn new_test_ext() -> sp_io::TestExternalities { - let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); + let state_version = None; + let mut t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); pallet_balances::GenesisConfig:: { balances: vec![(1, 100), (2, 100), (3, 100), (4, 100), (5, 100)], } - .assimilate_storage(&mut t) + .assimilate_storage(&mut t, state_version.clone()) .unwrap(); - t.into() + (t, state_version).into() } /// Run until a particular block. diff --git a/frame/scheduler/src/lib.rs b/frame/scheduler/src/lib.rs index ceb163a432e7b..adb369c753f61 100644 --- a/frame/scheduler/src/lib.rs +++ b/frame/scheduler/src/lib.rs @@ -985,8 +985,9 @@ mod tests { pub type LoggerCall = logger::Call; pub fn new_test_ext() -> sp_io::TestExternalities { - let t = system::GenesisConfig::default().build_storage::().unwrap(); - t.into() + let state_version = None; + let t = system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); + (t, state_version).into() } fn run_to_block(n: u64) { diff --git a/frame/scored-pool/src/mock.rs b/frame/scored-pool/src/mock.rs index 5c5425ae2bdd8..732568c936a55 100644 --- a/frame/scored-pool/src/mock.rs +++ b/frame/scored-pool/src/mock.rs @@ -135,7 +135,8 @@ impl Config for Test { } pub fn new_test_ext() -> sp_io::TestExternalities { - let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); + let state_version = None; + let mut t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); pallet_balances::GenesisConfig:: { balances: vec![ (5, 500_000), @@ -147,16 +148,16 @@ pub fn new_test_ext() -> sp_io::TestExternalities { (99, 1), ], } - .assimilate_storage(&mut t) + .assimilate_storage(&mut t, state_version.clone()) .unwrap(); pallet_scored_pool::GenesisConfig:: { pool: vec![(5, None), (10, Some(1)), (20, Some(2)), (31, Some(2)), (40, Some(3))], member_count: 2, ..Default::default() } - .assimilate_storage(&mut t) + .assimilate_storage(&mut t, state_version.clone()) .unwrap(); - t.into() + (t, state_version).into() } /// Fetch an entity from the pool, if existent. diff --git a/frame/session/benchmarking/src/mock.rs b/frame/session/benchmarking/src/mock.rs index 672862f5ed999..7ff1d95457ec9 100644 --- a/frame/session/benchmarking/src/mock.rs +++ b/frame/session/benchmarking/src/mock.rs @@ -191,6 +191,7 @@ impl pallet_staking::Config for Test { impl crate::Config for Test {} pub fn new_test_ext() -> sp_io::TestExternalities { - let t = frame_system::GenesisConfig::default().build_storage::().unwrap(); - sp_io::TestExternalities::new(t) + let state_version = None; + let t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); + sp_io::TestExternalities::new(t, state_version) } diff --git a/frame/session/src/historical/mod.rs b/frame/session/src/historical/mod.rs index c9b13e3c7f262..4f1cac7f111bf 100644 --- a/frame/session/src/historical/mod.rs +++ b/frame/session/src/historical/mod.rs @@ -375,17 +375,18 @@ pub(crate) mod tests { type Historical = Module; pub(crate) fn new_test_ext() -> sp_io::TestExternalities { - let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); + let state_version = None; + let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); let keys: Vec<_> = NEXT_VALIDATORS.with(|l| { l.borrow().iter().cloned().map(|i| (i, i, UintAuthorityId(i).into())).collect() }); - BasicExternalities::execute_with_storage(&mut t, || { + BasicExternalities::execute_with_storage(&mut t, state_version.clone(), || { for (ref k, ..) in &keys { frame_system::Pallet::::inc_providers(k); } }); - crate::GenesisConfig:: { keys }.assimilate_storage(&mut t).unwrap(); - sp_io::TestExternalities::new(t) + crate::GenesisConfig:: { keys }.assimilate_storage(&mut t, state_version.clone()).unwrap(); + sp_io::TestExternalities::new(t, state_version) } #[test] diff --git a/frame/session/src/historical/offchain.rs b/frame/session/src/historical/offchain.rs index 8583c2bb439be..d63a655bc2f7b 100644 --- a/frame/session/src/historical/offchain.rs +++ b/frame/session/src/historical/offchain.rs @@ -162,22 +162,23 @@ mod tests { type Historical = Module; pub fn new_test_ext() -> sp_io::TestExternalities { + let state_version = None; let mut t = frame_system::GenesisConfig::default() - .build_storage::() + .build_storage::(state_version.clone()) .expect("Failed to create test externalities."); let keys: Vec<_> = NEXT_VALIDATORS.with(|l| { l.borrow().iter().cloned().map(|i| (i, i, UintAuthorityId(i).into())).collect() }); - BasicExternalities::execute_with_storage(&mut t, || { + BasicExternalities::execute_with_storage(&mut t, state_version.clone(), || { for (ref k, ..) in &keys { frame_system::Pallet::::inc_providers(k); } }); - crate::GenesisConfig:: { keys }.assimilate_storage(&mut t).unwrap(); + crate::GenesisConfig:: { keys }.assimilate_storage(&mut t, state_version.clone()).unwrap(); - let mut ext = sp_io::TestExternalities::new(t); + let mut ext = sp_io::TestExternalities::new(t, state_version); let (offchain, offchain_state) = TestOffchainExt::with_offchain_db(ext.offchain_db()); diff --git a/frame/session/src/mock.rs b/frame/session/src/mock.rs index 449acaff5305d..29cfc72c4e4d3 100644 --- a/frame/session/src/mock.rs +++ b/frame/session/src/mock.rs @@ -209,10 +209,11 @@ pub fn reset_before_session_end_called() { } pub fn new_test_ext() -> sp_io::TestExternalities { - let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); + let state_version = None; + let mut t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); let keys: Vec<_> = NEXT_VALIDATORS .with(|l| l.borrow().iter().cloned().map(|i| (i, i, UintAuthorityId(i).into())).collect()); - BasicExternalities::execute_with_storage(&mut t, || { + BasicExternalities::execute_with_storage(&mut t, state_version.clone(), || { for (ref k, ..) in &keys { frame_system::Pallet::::inc_providers(k); } @@ -221,9 +222,9 @@ pub fn new_test_ext() -> sp_io::TestExternalities { frame_system::Pallet::::inc_providers(&69); }); pallet_session::GenesisConfig:: { keys } - .assimilate_storage(&mut t) + .assimilate_storage(&mut t, state_version.clone()) .unwrap(); - sp_io::TestExternalities::new(t) + sp_io::TestExternalities::new(t,state_version) } parameter_types! { diff --git a/frame/society/src/mock.rs b/frame/society/src/mock.rs index 38c2586323135..cd8db1f813091 100644 --- a/frame/society/src/mock.rs +++ b/frame/society/src/mock.rs @@ -154,19 +154,20 @@ impl EnvBuilder { } pub fn execute R>(mut self, f: F) -> R { - let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); + let state_version = None; + let mut t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); self.balances.push((Society::account_id(), self.balance.max(self.pot))); pallet_balances::GenesisConfig:: { balances: self.balances } - .assimilate_storage(&mut t) + .assimilate_storage(&mut t, state_version.clone()) .unwrap(); pallet_society::GenesisConfig:: { members: self.members, pot: self.pot, max_members: self.max_members, } - .assimilate_storage(&mut t) + .assimilate_storage(&mut t, state_version.clone()) .unwrap(); - let mut ext: sp_io::TestExternalities = t.into(); + let mut ext: sp_io::TestExternalities = (t, state_version).into(); ext.execute_with(f) } #[allow(dead_code)] diff --git a/frame/staking/src/mock.rs b/frame/staking/src/mock.rs index 776affde5d42d..59b059dbc962d 100644 --- a/frame/staking/src/mock.rs +++ b/frame/staking/src/mock.rs @@ -395,7 +395,8 @@ impl ExtBuilder { } fn build(self) -> sp_io::TestExternalities { sp_tracing::try_init_simple(); - let mut storage = frame_system::GenesisConfig::default().build_storage::().unwrap(); + let state_version = None; + let mut storage = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); let _ = pallet_balances::GenesisConfig:: { balances: vec![ @@ -429,7 +430,7 @@ impl ExtBuilder { (999, 1_000_000_000_000), ], } - .assimilate_storage(&mut storage); + .assimilate_storage(&mut storage, state_version.clone()); let mut stakers = vec![]; if self.has_stakers { @@ -482,7 +483,7 @@ impl ExtBuilder { min_validator_bond: self.min_validator_bond, ..Default::default() } - .assimilate_storage(&mut storage); + .assimilate_storage(&mut storage, state_version.clone()); let _ = pallet_session::GenesisConfig:: { keys: if self.has_stakers { @@ -495,9 +496,9 @@ impl ExtBuilder { .collect() }, } - .assimilate_storage(&mut storage); + .assimilate_storage(&mut storage, state_version.clone()); - let mut ext = sp_io::TestExternalities::from(storage); + let mut ext = sp_io::TestExternalities::from((storage, state_version)); ext.execute_with(|| { let validators = Session::validators(); SESSION.with(|x| *x.borrow_mut() = (validators.clone(), HashSet::new())); diff --git a/frame/staking/src/pallet/mod.rs b/frame/staking/src/pallet/mod.rs index ee09660d23d20..2f057b44e64c0 100644 --- a/frame/staking/src/pallet/mod.rs +++ b/frame/staking/src/pallet/mod.rs @@ -647,7 +647,8 @@ pub mod pallet { fn integrity_test() { sp_std::if_std! { - sp_io::TestExternalities::new_empty().execute_with(|| + let state_version = None; + sp_io::TestExternalities::new_empty(state_version).execute_with(|| assert!( T::SlashDeferDuration::get() < T::BondingDuration::get() || T::BondingDuration::get() == 0, "As per documentation, slash defer duration ({}) should be less than bonding duration ({}).", diff --git a/frame/sudo/src/mock.rs b/frame/sudo/src/mock.rs index 7fd55a618a6b5..e916bfe771273 100644 --- a/frame/sudo/src/mock.rs +++ b/frame/sudo/src/mock.rs @@ -164,9 +164,10 @@ pub type LoggerCall = logger::Call; // Build test environment by setting the root `key` for the Genesis. pub fn new_test_ext(root_key: u64) -> sp_io::TestExternalities { - let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); + let state_version = None; + let mut t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); sudo::GenesisConfig:: { key: root_key } - .assimilate_storage(&mut t) + .assimilate_storage(&mut t, state_version.clone()) .unwrap(); - t.into() + (t, state_version).into() } diff --git a/frame/support/procedural/src/construct_runtime/expand/config.rs b/frame/support/procedural/src/construct_runtime/expand/config.rs index 5e1b9d94700e6..4d291691709f7 100644 --- a/frame/support/procedural/src/construct_runtime/expand/config.rs +++ b/frame/support/procedural/src/construct_runtime/expand/config.rs @@ -78,10 +78,11 @@ pub fn expand_outer_config( fn assimilate_storage( &self, storage: &mut #scrate::sp_runtime::Storage, + alt_hashing: Option>, ) -> std::result::Result<(), String> { #build_storage_calls - #scrate::BasicExternalities::execute_with_storage(storage, || { + #scrate::BasicExternalities::execute_with_storage(storage, alt_hashing, || { ::on_genesis(); }); @@ -130,6 +131,6 @@ fn expand_config_build_storage_call( quote! { #scrate::sp_runtime::BuildModuleGenesisStorage:: - <#runtime, #instance>::build_module_genesis_storage(&self.#field_name, storage)?; + <#runtime, #instance>::build_module_genesis_storage(&self.#field_name, storage, alt_hashing)?; } } diff --git a/frame/support/procedural/src/pallet/expand/genesis_build.rs b/frame/support/procedural/src/pallet/expand/genesis_build.rs index 06acaf324254c..177a1206ca962 100644 --- a/frame/support/procedural/src/pallet/expand/genesis_build.rs +++ b/frame/support/procedural/src/pallet/expand/genesis_build.rs @@ -61,8 +61,9 @@ pub fn expand_genesis_build(def: &mut Def) -> proc_macro2::TokenStream { fn build_module_genesis_storage( &self, storage: &mut #frame_support::sp_runtime::Storage, + alt_hashing: Option>, ) -> std::result::Result<(), std::string::String> { - #frame_support::BasicExternalities::execute_with_storage(storage, || { + #frame_support::BasicExternalities::execute_with_storage(storage, alt_hashing, || { >::build(self); Ok(()) }) diff --git a/frame/support/procedural/src/storage/genesis_config/mod.rs b/frame/support/procedural/src/storage/genesis_config/mod.rs index d2d1afb017736..4d02bcf162bdc 100644 --- a/frame/support/procedural/src/storage/genesis_config/mod.rs +++ b/frame/support/procedural/src/storage/genesis_config/mod.rs @@ -139,12 +139,12 @@ fn impl_build_storage( #[cfg(feature = "std")] impl#genesis_impl GenesisConfig#genesis_struct #genesis_where_clause { /// Build the storage for this module. - pub fn build_storage #fn_generic (&self) -> std::result::Result< + pub fn build_storage #fn_generic (&self, alt_hashing: Option>) -> std::result::Result< #scrate::sp_runtime::Storage, String > #fn_where_clause { let mut storage = Default::default(); - self.assimilate_storage::<#fn_traitinstance>(&mut storage)?; + self.assimilate_storage::<#fn_traitinstance>(&mut storage, alt_hashing)?; Ok(storage) } @@ -152,8 +152,9 @@ fn impl_build_storage( pub fn assimilate_storage #fn_generic ( &self, storage: &mut #scrate::sp_runtime::Storage, + alt_hashing: Option>, ) -> std::result::Result<(), String> #fn_where_clause { - #scrate::BasicExternalities::execute_with_storage(storage, || { + #scrate::BasicExternalities::execute_with_storage(storage, alt_hashing, || { #( #builder_blocks )* Ok(()) }) @@ -167,8 +168,9 @@ fn impl_build_storage( fn build_module_genesis_storage( &self, storage: &mut #scrate::sp_runtime::Storage, + alt_hashing: Option>, ) -> std::result::Result<(), String> { - self.assimilate_storage::<#fn_traitinstance> (storage) + self.assimilate_storage::<#fn_traitinstance> (storage, alt_hashing) } } } diff --git a/frame/support/src/lib.rs b/frame/support/src/lib.rs index bf314161c7f87..4d99cc9215de4 100644 --- a/frame/support/src/lib.rs +++ b/frame/support/src/lib.rs @@ -847,7 +847,9 @@ pub mod tests { } fn new_test_ext() -> TestExternalities { - GenesisConfig::default().build_storage().unwrap().into() + let state_version = None; + let storage = GenesisConfig::default().build_storage(state_version.clone()).unwrap(); + (storage, state_version).into() } type Map = Data; diff --git a/frame/support/src/storage/generator/mod.rs b/frame/support/src/storage/generator/mod.rs index 576bada2e262c..8b3e239835616 100644 --- a/frame/support/src/storage/generator/mod.rs +++ b/frame/support/src/storage/generator/mod.rs @@ -74,8 +74,9 @@ mod tests { #[test] fn value_translate_works() { - let t = GenesisConfig::default().build_storage().unwrap(); - TestExternalities::new(t).execute_with(|| { + let state_version = None; + let t = GenesisConfig::default().build_storage(state_version.clone()).unwrap(); + TestExternalities::new(t, state_version).execute_with(|| { // put the old value `1111u32` in the storage. let key = Value::storage_value_final_key(); unhashed::put_raw(&key, &1111u32.encode()); @@ -94,8 +95,9 @@ mod tests { #[test] fn map_translate_works() { - let t = GenesisConfig::default().build_storage().unwrap(); - TestExternalities::new(t).execute_with(|| { + let state_version = None; + let t = GenesisConfig::default().build_storage(state_version.clone()).unwrap(); + TestExternalities::new(t, state_version).execute_with(|| { // start with a map of u32 -> u32. for i in 0u32..100u32 { unhashed::put(&NumberMap::hashed_key_for(&i), &(i as u64)); @@ -123,8 +125,9 @@ mod tests { #[test] fn try_mutate_works() { - let t = GenesisConfig::default().build_storage().unwrap(); - TestExternalities::new(t).execute_with(|| { + let state_version = None; + let t = GenesisConfig::default().build_storage(state_version.clone()).unwrap(); + TestExternalities::new(t, state_version).execute_with(|| { assert_eq!(Value::get(), (0, 0)); assert_eq!(NumberMap::get(0), 0); assert_eq!(DoubleMap::get(0, 0), 0); diff --git a/frame/support/src/storage/migration.rs b/frame/support/src/storage/migration.rs index 0f10c5cbb47d3..50a5e30efd1d9 100644 --- a/frame/support/src/storage/migration.rs +++ b/frame/support/src/storage/migration.rs @@ -283,7 +283,7 @@ pub fn take_storage_item /// "my_new_example_name", a migration can be: /// ``` /// # use frame_support::storage::migration::move_storage_from_pallet; -/// # sp_io::TestExternalities::new_empty().execute_with(|| { +/// # sp_io::TestExternalities::new_empty(None).execute_with(|| { /// move_storage_from_pallet(b"Foo", b"my_example", b"my_new_example_name"); /// move_storage_from_pallet(b"Bar", b"my_example", b"my_new_example_name"); /// # }) @@ -325,7 +325,7 @@ pub fn move_storage_from_pallet( /// "my_new_example_name", a migration can be: /// ``` /// # use frame_support::storage::migration::move_pallet; -/// # sp_io::TestExternalities::new_empty().execute_with(|| { +/// # sp_io::TestExternalities::new_empty(None).execute_with(|| { /// move_pallet(b"my_example", b"my_new_example_name"); /// # }) /// ``` @@ -406,7 +406,7 @@ mod tests { #[test] fn test_move_prefix() { - TestExternalities::new_empty().execute_with(|| { + TestExternalities::new_empty(None).execute_with(|| { OldStorageValue::put(3); OldStorageMap::insert(1, 2); OldStorageMap::insert(3, 4); @@ -422,7 +422,7 @@ mod tests { #[test] fn test_move_storage() { - TestExternalities::new_empty().execute_with(|| { + TestExternalities::new_empty(None).execute_with(|| { OldStorageValue::put(3); OldStorageMap::insert(1, 2); OldStorageMap::insert(3, 4); @@ -445,7 +445,7 @@ mod tests { #[test] fn test_move_pallet() { - TestExternalities::new_empty().execute_with(|| { + TestExternalities::new_empty(None).execute_with(|| { OldStorageValue::put(3); OldStorageMap::insert(1, 2); OldStorageMap::insert(3, 4); @@ -461,7 +461,7 @@ mod tests { #[test] fn test_storage_iter() { - TestExternalities::new_empty().execute_with(|| { + TestExternalities::new_empty(None).execute_with(|| { OldStorageValue::put(3); OldStorageMap::insert(1, 2); OldStorageMap::insert(3, 4); diff --git a/frame/support/src/traits/hooks.rs b/frame/support/src/traits/hooks.rs index 965cce234288f..61324295b36cf 100644 --- a/frame/support/src/traits/hooks.rs +++ b/frame/support/src/traits/hooks.rs @@ -307,15 +307,15 @@ pub trait GenesisBuild: Default + MaybeSerializeDeserialize { fn build(&self); /// Build the storage using `build` inside default storage. - fn build_storage(&self) -> Result { + fn build_storage(&self, alt_hashing: Option>) -> Result { let mut storage = Default::default(); - self.assimilate_storage(&mut storage)?; + self.assimilate_storage(&mut storage, alt_hashing)?; Ok(storage) } /// Assimilate the storage for this module into pre-existing overlays. - fn assimilate_storage(&self, storage: &mut sp_runtime::Storage) -> Result<(), String> { - sp_state_machine::BasicExternalities::execute_with_storage(storage, || { + fn assimilate_storage(&self, storage: &mut sp_runtime::Storage, alt_hashing: Option>) -> Result<(), String> { + sp_state_machine::BasicExternalities::execute_with_storage(storage, alt_hashing, || { self.build(); Ok(()) }) diff --git a/frame/support/test/tests/decl_storage.rs b/frame/support/test/tests/decl_storage.rs index 666dda49935ef..4853861738ad1 100644 --- a/frame/support/test/tests/decl_storage.rs +++ b/frame/support/test/tests/decl_storage.rs @@ -668,9 +668,10 @@ mod tests { #[test] fn check_builder_config() { + let state_version = None; let config = GenesisConfig::default(); - let storage = config.build_storage().unwrap(); - TestExternalities::from(storage).execute_with(|| { + let storage = config.build_storage(state_version.clone()).unwrap(); + TestExternalities::from((storage, state_version)).execute_with(|| { assert_eq!(Module::::u32_with_builder(), 1); assert_eq!(Module::::opt_u32_with_builder_some(), Some(1)); assert_eq!(Module::::opt_u32_with_builder_none(), None); diff --git a/frame/support/test/tests/instance.rs b/frame/support/test/tests/instance.rs index ba0569145df85..b79420c733a12 100644 --- a/frame/support/test/tests/instance.rs +++ b/frame/support/test/tests/instance.rs @@ -318,7 +318,9 @@ pub type Block = generic::Block; pub type UncheckedExtrinsic = generic::UncheckedExtrinsic; fn new_test_ext() -> sp_io::TestExternalities { - GenesisConfig { + let state_version = None; + + (GenesisConfig { module_1_1: module1::GenesisConfig { value: 3, test: 2 }, module_1_2: module1::GenesisConfig { value: 4, test: 5 }, module_2: module2::GenesisConfig { @@ -334,20 +336,18 @@ fn new_test_ext() -> sp_io::TestExternalities { module_2_2: Default::default(), module_2_3: Default::default(), } - .build_storage() - .unwrap() - .into() + .build_storage(state_version.clone()) + .unwrap(), state_version).into() } #[test] fn storage_instance_independence() { + let state_version = Some(Some(sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD)); let mut storage = sp_core::storage::Storage { top: std::collections::BTreeMap::new(), children_default: std::collections::HashMap::new(), }; - storage - .modify_trie_alt_hashing_threshold(Some(sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD)); - sp_state_machine::BasicExternalities::execute_with_storage(&mut storage, || { + sp_state_machine::BasicExternalities::execute_with_storage(&mut storage, state_version, || { module2::Value::::put(0); module2::Value::::put(0); module2::Value::::put(0); diff --git a/frame/system/benches/bench.rs b/frame/system/benches/bench.rs index 97c19c5e8159a..b1030a20acef4 100644 --- a/frame/system/benches/bench.rs +++ b/frame/system/benches/bench.rs @@ -101,7 +101,8 @@ impl module::Config for Runtime { } fn new_test_ext() -> sp_io::TestExternalities { - system::GenesisConfig::default().build_storage::().unwrap().into() + let state_version = None; + (system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(), state_version).into() } fn deposit_events(n: usize) { diff --git a/frame/system/benchmarking/src/mock.rs b/frame/system/benchmarking/src/mock.rs index d828fb22ff5ff..722bd84fdc93e 100644 --- a/frame/system/benchmarking/src/mock.rs +++ b/frame/system/benchmarking/src/mock.rs @@ -67,6 +67,7 @@ impl frame_system::Config for Test { impl crate::Config for Test {} pub fn new_test_ext() -> sp_io::TestExternalities { - let t = frame_system::GenesisConfig::default().build_storage::().unwrap(); - sp_io::TestExternalities::new(t) + let state_version = None; + let t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); + sp_io::TestExternalities::new(t, state_version) } diff --git a/frame/system/src/lib.rs b/frame/system/src/lib.rs index af4920d711e21..fffb83d93e6b1 100644 --- a/frame/system/src/lib.rs +++ b/frame/system/src/lib.rs @@ -733,8 +733,8 @@ impl GenesisConfig { /// Direct implementation of `GenesisBuild::build_storage`. /// /// Kept in order not to break dependency. - pub fn build_storage(&self) -> Result { - >::build_storage(self) + pub fn build_storage(&self, alt_hashing: Option>) -> Result { + >::build_storage(self, alt_hashing) } /// Direct implementation of `GenesisBuild::assimilate_storage`. @@ -743,8 +743,9 @@ impl GenesisConfig { pub fn assimilate_storage( &self, storage: &mut sp_runtime::Storage, + alt_hashing: Option>, ) -> Result<(), String> { - >::assimilate_storage(self, storage) + >::assimilate_storage(self, storage, alt_hashing) } } @@ -1438,7 +1439,7 @@ impl Pallet { /// Get the basic externalities for this pallet, useful for tests. #[cfg(any(feature = "std", test))] pub fn externalities() -> TestExternalities { - let mut storage = sp_core::storage::Storage { + let storage = sp_core::storage::Storage { top: map![ >::hashed_key_for(T::BlockNumber::zero()) => [69u8; 32].encode(), >::hashed_key().to_vec() => T::BlockNumber::one().encode(), @@ -1446,10 +1447,7 @@ impl Pallet { ], children_default: map![], }; - storage.modify_trie_alt_hashing_threshold(Some( - sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD, - )); - TestExternalities::new(storage) + TestExternalities::new(storage, Some(Some(sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD))) } /// Set the block number to something in particular. Can be used as an alternative to diff --git a/frame/system/src/mock.rs b/frame/system/src/mock.rs index 8039b73445ae6..68a40637abcdf 100644 --- a/frame/system/src/mock.rs +++ b/frame/system/src/mock.rs @@ -120,8 +120,9 @@ pub const CALL: &::Call = &Call::System(frame_system::Call::set_ /// Create new externalities for `System` module tests. pub fn new_test_ext() -> sp_io::TestExternalities { + let state_version = None; let mut ext: sp_io::TestExternalities = - GenesisConfig::default().build_storage().unwrap().into(); + (GenesisConfig::default().build_storage(state_version.clone()).unwrap(), state_version).into(); // Add to each test the initial weight of a block ext.execute_with(|| { System::register_extra_weight_unchecked( diff --git a/frame/timestamp/src/lib.rs b/frame/timestamp/src/lib.rs index a1ff8d37ff881..150ded70d7e17 100644 --- a/frame/timestamp/src/lib.rs +++ b/frame/timestamp/src/lib.rs @@ -316,8 +316,9 @@ mod tests { }; pub fn new_test_ext() -> TestExternalities { - let t = frame_system::GenesisConfig::default().build_storage::().unwrap(); - TestExternalities::new(t) + let state_version = None; + let t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); + TestExternalities::new(t, state_version) } type UncheckedExtrinsic = frame_system::mocking::MockUncheckedExtrinsic; diff --git a/frame/tips/src/tests.rs b/frame/tips/src/tests.rs index 8611320563c7d..320191a779afa 100644 --- a/frame/tips/src/tests.rs +++ b/frame/tips/src/tests.rs @@ -162,15 +162,16 @@ impl Config for Test { } pub fn new_test_ext() -> sp_io::TestExternalities { - let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); + let state_version = None; + let mut t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); pallet_balances::GenesisConfig:: { // Total issuance will be 200 with treasury account initialized at ED. balances: vec![(0, 100), (1, 98), (2, 1)], } - .assimilate_storage(&mut t) + .assimilate_storage(&mut t, state_version.clone()) .unwrap(); - GenesisBuild::::assimilate_storage(&pallet_treasury::GenesisConfig, &mut t).unwrap(); - t.into() + GenesisBuild::::assimilate_storage(&pallet_treasury::GenesisConfig, &mut t, state_version.clone()).unwrap(); + (t, state_version).into() } fn last_event() -> RawEvent { @@ -403,6 +404,7 @@ fn tip_changing_works() { fn test_last_reward_migration() { use sp_storage::Storage; + let state_version = None; let mut s = Storage::default(); #[derive(Clone, Eq, PartialEq, Encode, Decode, RuntimeDebug)] @@ -455,7 +457,7 @@ fn test_last_reward_migration() { s.top = data.into_iter().collect(); - sp_io::TestExternalities::new(s).execute_with(|| { + sp_io::TestExternalities::new(s, state_version).execute_with(|| { TipsModTestInst::migrate_retract_tip_for_tip_new(); // Test w/ finder @@ -490,16 +492,17 @@ fn test_last_reward_migration() { #[test] fn genesis_funding_works() { - let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); + let state_version = None; + let mut t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); let initial_funding = 100; pallet_balances::GenesisConfig:: { // Total issuance will be 200 with treasury account initialized with 100. balances: vec![(0, 100), (Treasury::account_id(), initial_funding)], } - .assimilate_storage(&mut t) + .assimilate_storage(&mut t, state_version.clone()) .unwrap(); - GenesisBuild::::assimilate_storage(&pallet_treasury::GenesisConfig, &mut t).unwrap(); - let mut t: sp_io::TestExternalities = t.into(); + GenesisBuild::::assimilate_storage(&pallet_treasury::GenesisConfig, &mut t, state_version.clone()).unwrap(); + let mut t: sp_io::TestExternalities = (t, state_version).into(); t.execute_with(|| { assert_eq!(Balances::free_balance(Treasury::account_id()), initial_funding); diff --git a/frame/transaction-payment/src/lib.rs b/frame/transaction-payment/src/lib.rs index 9e8dbf6cb5d1d..0c555830fbb69 100644 --- a/frame/transaction-payment/src/lib.rs +++ b/frame/transaction-payment/src/lib.rs @@ -350,7 +350,7 @@ pub mod pallet { target += addition; #[cfg(any(feature = "std", test))] - sp_io::TestExternalities::new_empty().execute_with(|| { + sp_io::TestExternalities::new_empty(None).execute_with(|| { >::set_block_consumed_resources(target, 0); let next = T::FeeMultiplierUpdate::convert(min_value); assert!( @@ -858,8 +858,9 @@ mod tests { WEIGHT_TO_FEE.with(|v| *v.borrow_mut() = self.weight_to_fee); } pub fn build(self) -> sp_io::TestExternalities { + let state_version = None; self.set_constants(); - let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); + let mut t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); pallet_balances::GenesisConfig:: { balances: if self.balance_factor > 0 { vec![ @@ -874,9 +875,9 @@ mod tests { vec![] }, } - .assimilate_storage(&mut t) + .assimilate_storage(&mut t, state_version.clone()) .unwrap(); - t.into() + (t, state_version).into() } } diff --git a/frame/transaction-storage/src/mock.rs b/frame/transaction-storage/src/mock.rs index 38d14129d76e2..c35dac4e76708 100644 --- a/frame/transaction-storage/src/mock.rs +++ b/frame/transaction-storage/src/mock.rs @@ -104,6 +104,7 @@ impl pallet_transaction_storage::Config for Test { } pub fn new_test_ext() -> sp_io::TestExternalities { + let state_version = None; let t = GenesisConfig { system: Default::default(), balances: pallet_balances::GenesisConfig:: { @@ -117,9 +118,9 @@ pub fn new_test_ext() -> sp_io::TestExternalities { max_transaction_size: crate::DEFAULT_MAX_TRANSACTION_SIZE, }, } - .build_storage() + .build_storage(state_version.clone()) .unwrap(); - t.into() + (t, state_version).into() } pub fn run_to_block(n: u64, f: impl Fn() -> Option) { diff --git a/frame/treasury/src/lib.rs b/frame/treasury/src/lib.rs index 965f06731c945..afca0dfb9f5cc 100644 --- a/frame/treasury/src/lib.rs +++ b/frame/treasury/src/lib.rs @@ -234,8 +234,9 @@ pub mod pallet { pub fn assimilate_storage, I: 'static>( &self, storage: &mut sp_runtime::Storage, + alt_hashing: Option>, ) -> Result<(), String> { - >::assimilate_storage(self, storage) + >::assimilate_storage(self, storage, alt_hashing) } } diff --git a/frame/treasury/src/tests.rs b/frame/treasury/src/tests.rs index 534661b2773bb..0c4ca3efff22a 100644 --- a/frame/treasury/src/tests.rs +++ b/frame/treasury/src/tests.rs @@ -126,15 +126,16 @@ impl Config for Test { } pub fn new_test_ext() -> sp_io::TestExternalities { - let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); + let state_version = None; + let mut t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); pallet_balances::GenesisConfig:: { // Total issuance will be 200 with treasury account initialized at ED. balances: vec![(0, 100), (1, 98), (2, 1)], } - .assimilate_storage(&mut t) + .assimilate_storage(&mut t, state_version.clone()) .unwrap(); - GenesisBuild::::assimilate_storage(&crate::GenesisConfig, &mut t).unwrap(); - t.into() + GenesisBuild::::assimilate_storage(&crate::GenesisConfig, &mut t, state_version.clone()).unwrap(); + (t, state_version).into() } #[test] @@ -321,12 +322,13 @@ fn treasury_account_doesnt_get_deleted() { // This is useful for chain that will just update runtime. #[test] fn inexistent_account_works() { - let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); + let state_version = None; + let mut t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); pallet_balances::GenesisConfig:: { balances: vec![(0, 100), (1, 99), (2, 1)] } - .assimilate_storage(&mut t) + .assimilate_storage(&mut t, state_version.clone()) .unwrap(); // Treasury genesis config is not build thus treasury account does not exist - let mut t: sp_io::TestExternalities = t.into(); + let mut t: sp_io::TestExternalities = (t, state_version).into(); t.execute_with(|| { assert_eq!(Balances::free_balance(Treasury::account_id()), 0); // Account does not exist @@ -353,16 +355,17 @@ fn inexistent_account_works() { #[test] fn genesis_funding_works() { - let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); + let state_version = None; + let mut t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); let initial_funding = 100; pallet_balances::GenesisConfig:: { // Total issuance will be 200 with treasury account initialized with 100. balances: vec![(0, 100), (Treasury::account_id(), initial_funding)], } - .assimilate_storage(&mut t) + .assimilate_storage(&mut t, state_version.clone()) .unwrap(); - GenesisBuild::::assimilate_storage(&crate::GenesisConfig, &mut t).unwrap(); - let mut t: sp_io::TestExternalities = t.into(); + GenesisBuild::::assimilate_storage(&crate::GenesisConfig, &mut t, state_version.clone()).unwrap(); + let mut t: sp_io::TestExternalities = (t, state_version).into(); t.execute_with(|| { assert_eq!(Balances::free_balance(Treasury::account_id()), initial_funding); diff --git a/frame/uniques/src/mock.rs b/frame/uniques/src/mock.rs index 658e82a5143e2..eeddbe8a79b92 100644 --- a/frame/uniques/src/mock.rs +++ b/frame/uniques/src/mock.rs @@ -117,9 +117,10 @@ impl Config for Test { } pub(crate) fn new_test_ext() -> sp_io::TestExternalities { - let t = frame_system::GenesisConfig::default().build_storage::().unwrap(); + let state_version = None; + let t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); - let mut ext = sp_io::TestExternalities::new(t); + let mut ext = sp_io::TestExternalities::new(t, state_version); ext.execute_with(|| System::set_block_number(1)); ext } diff --git a/frame/utility/src/tests.rs b/frame/utility/src/tests.rs index 2731b6ca0b8be..a90fed114c850 100644 --- a/frame/utility/src/tests.rs +++ b/frame/utility/src/tests.rs @@ -169,13 +169,14 @@ use frame_system::Call as SystemCall; use pallet_balances::{Call as BalancesCall, Error as BalancesError}; pub fn new_test_ext() -> sp_io::TestExternalities { - let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); + let state_version = None; + let mut t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); pallet_balances::GenesisConfig:: { balances: vec![(1, 10), (2, 10), (3, 10), (4, 10), (5, 2)], } - .assimilate_storage(&mut t) + .assimilate_storage(&mut t, state_version.clone()) .unwrap(); - let mut ext = sp_io::TestExternalities::new(t); + let mut ext = sp_io::TestExternalities::new(t, state_version); ext.execute_with(|| System::set_block_number(1)); ext } diff --git a/frame/vesting/src/mock.rs b/frame/vesting/src/mock.rs index 4efbabefe688f..ae812d9198c7d 100644 --- a/frame/vesting/src/mock.rs +++ b/frame/vesting/src/mock.rs @@ -111,8 +111,9 @@ impl ExtBuilder { } pub fn build(self) -> sp_io::TestExternalities { + let state_version = None; EXISTENTIAL_DEPOSIT.with(|v| *v.borrow_mut() = self.existential_deposit); - let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); + let mut t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); pallet_balances::GenesisConfig:: { balances: vec![ (1, 10 * self.existential_deposit), @@ -122,7 +123,7 @@ impl ExtBuilder { (12, 10 * self.existential_deposit), ], } - .assimilate_storage(&mut t) + .assimilate_storage(&mut t, state_version.clone()) .unwrap(); pallet_vesting::GenesisConfig:: { vesting: vec![ @@ -131,9 +132,9 @@ impl ExtBuilder { (12, 10, 20, 5 * self.existential_deposit), ], } - .assimilate_storage(&mut t) + .assimilate_storage(&mut t, state_version.clone()) .unwrap(); - let mut ext = sp_io::TestExternalities::new(t); + let mut ext = sp_io::TestExternalities::new(t, state_version); ext.execute_with(|| System::set_block_number(1)); ext } diff --git a/primitives/io/src/lib.rs b/primitives/io/src/lib.rs index 379a5a706813d..b885c08b0f348 100644 --- a/primitives/io/src/lib.rs +++ b/primitives/io/src/lib.rs @@ -477,7 +477,7 @@ pub trait Misc { fn runtime_version(&mut self, wasm: &[u8]) -> Option> { use sp_core::traits::ReadRuntimeVersionExt; - let mut ext = sp_state_machine::BasicExternalities::default(); + let mut ext: sp_state_machine::BasicExternalities = None.into(); match self .extension::() @@ -1475,7 +1475,7 @@ mod tests { #[test] fn storage_works() { - let mut t = BasicExternalities::default(); + let mut t: BasicExternalities = None.into(); t.execute_with(|| { assert_eq!(storage::get(b"hello"), None); storage::set(b"hello", b"world"); @@ -1487,7 +1487,7 @@ mod tests { t = BasicExternalities::new(Storage { top: map![b"foo".to_vec() => b"bar".to_vec()], children_default: map![], - }); + }, None); t.execute_with(|| { assert_eq!(storage::get(b"hello"), None); @@ -1495,12 +1495,9 @@ mod tests { }); let value = vec![7u8; 35]; - let mut storage = + let storage = Storage { top: map![b"foo00".to_vec() => value.clone()], children_default: map![] }; - storage.modify_trie_alt_hashing_threshold(Some( - sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD, - )); - t = BasicExternalities::new(storage); + t = BasicExternalities::new(storage, Some(Some(sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD))); t.execute_with(|| { assert_eq!(storage::get(b"hello"), None); @@ -1514,7 +1511,7 @@ mod tests { let mut t = BasicExternalities::new(Storage { top: map![b":test".to_vec() => value.clone()], children_default: map![], - }); + }, None); t.execute_with(|| { let mut v = [0u8; 4]; @@ -1536,7 +1533,7 @@ mod tests { b":abdd".to_vec() => b"\x0b\0\0\0Hello world".to_vec() ], children_default: map![], - }); + }, None); t.execute_with(|| { assert!(matches!( @@ -1553,7 +1550,7 @@ mod tests { #[test] fn batch_verify_start_finish_works() { - let mut ext = BasicExternalities::default(); + let mut ext: sp_state_machine::BasicExternalities = None.into(); ext.register_extension(TaskExecutorExt::new(TaskExecutor::new())); ext.execute_with(|| { @@ -1571,7 +1568,7 @@ mod tests { #[test] fn long_sr25519_batching() { - let mut ext = BasicExternalities::default(); + let mut ext: sp_state_machine::BasicExternalities = None.into(); ext.register_extension(TaskExecutorExt::new(TaskExecutor::new())); ext.execute_with(|| { let pair = sr25519::Pair::generate_with_phrase(None).0; @@ -1598,7 +1595,7 @@ mod tests { #[test] fn batching_works() { - let mut ext = BasicExternalities::default(); + let mut ext: sp_state_machine::BasicExternalities = None.into(); ext.register_extension(TaskExecutorExt::new(TaskExecutor::new())); ext.execute_with(|| { // invalid ed25519 signature diff --git a/primitives/runtime/src/lib.rs b/primitives/runtime/src/lib.rs index 4a9c6087fa5cc..f47f5cbe95f65 100644 --- a/primitives/runtime/src/lib.rs +++ b/primitives/runtime/src/lib.rs @@ -170,13 +170,13 @@ pub use serde::{de::DeserializeOwned, Deserialize, Serialize}; #[cfg(feature = "std")] pub trait BuildStorage { /// Build the storage out of this builder. - fn build_storage(&self) -> Result { + fn build_storage(&self, alt_hashing: Option>) -> Result { let mut storage = Default::default(); - self.assimilate_storage(&mut storage)?; + self.assimilate_storage(&mut storage, alt_hashing)?; Ok(storage) } /// Assimilate the storage for this module into pre-existing overlays. - fn assimilate_storage(&self, storage: &mut sp_core::storage::Storage) -> Result<(), String>; + fn assimilate_storage(&self, storage: &mut sp_core::storage::Storage, alt_hashing: Option>) -> Result<(), String>; } /// Something that can build the genesis storage of a module. @@ -186,12 +186,13 @@ pub trait BuildModuleGenesisStorage: Sized { fn build_module_genesis_storage( &self, storage: &mut sp_core::storage::Storage, + alt_hashing: Option>, ) -> Result<(), String>; } #[cfg(feature = "std")] impl BuildStorage for sp_core::storage::Storage { - fn assimilate_storage(&self, storage: &mut sp_core::storage::Storage) -> Result<(), String> { + fn assimilate_storage(&self, storage: &mut sp_core::storage::Storage, _alt_hashing: Option>) -> Result<(), String> { storage.top.extend(self.top.iter().map(|(k, v)| (k.clone(), v.clone()))); for (k, other_map) in self.children_default.iter() { let k = k.clone(); @@ -210,7 +211,7 @@ impl BuildStorage for sp_core::storage::Storage { #[cfg(feature = "std")] impl BuildStorage for () { - fn assimilate_storage(&self, _: &mut sp_core::storage::Storage) -> Result<(), String> { + fn assimilate_storage(&self, _: &mut sp_core::storage::Storage, _: Option>) -> Result<(), String> { Err("`assimilate_storage` not implemented for `()`".into()) } } @@ -992,7 +993,7 @@ mod tests { #[test] #[should_panic(expected = "Signature verification has not been called")] fn batching_still_finishes_when_not_called_directly() { - let mut ext = sp_state_machine::BasicExternalities::default(); + let mut ext = sp_state_machine::BasicExternalities::new_empty(None); ext.register_extension(sp_core::traits::TaskExecutorExt::new( sp_core::testing::TaskExecutor::new(), )); @@ -1006,7 +1007,7 @@ mod tests { #[test] #[should_panic(expected = "Hey, I'm an error")] fn batching_does_not_panic_while_thread_is_already_panicking() { - let mut ext = sp_state_machine::BasicExternalities::default(); + let mut ext = sp_state_machine::BasicExternalities::new_empty(None); ext.register_extension(sp_core::traits::TaskExecutorExt::new( sp_core::testing::TaskExecutor::new(), )); diff --git a/primitives/state-machine/src/backend.rs b/primitives/state-machine/src/backend.rs index a2747ae759641..36c400bd3312c 100644 --- a/primitives/state-machine/src/backend.rs +++ b/primitives/state-machine/src/backend.rs @@ -263,15 +263,6 @@ pub trait Backend: sp_std::fmt::Debug { unimplemented!() } - /// Read current trie hashing threshold. - /// Please do not change default implementation when implementing this trait. - fn get_trie_alt_hashing_threshold(&self) -> Option { - self.storage(sp_core::storage::well_known_keys::TRIE_HASHING_CONFIG) - .ok() - .flatten() - .and_then(|encoded| sp_core::storage::trie_threshold_decode(&mut encoded.as_slice())) - } - /// Extend storage info for benchmarking db fn get_read_and_written_keys(&self) -> Vec<(Vec, u32, u32, bool)> { unimplemented!() diff --git a/primitives/state-machine/src/basic.rs b/primitives/state-machine/src/basic.rs index 627194654795a..b829973765f1e 100644 --- a/primitives/state-machine/src/basic.rs +++ b/primitives/state-machine/src/basic.rs @@ -33,7 +33,6 @@ use sp_trie::{empty_child_trie_root, Layout, TrieConfiguration}; use std::{ any::{Any, TypeId}, collections::BTreeMap, - iter::FromIterator, ops::Bound, }; @@ -42,19 +41,18 @@ use std::{ pub struct BasicExternalities { inner: Storage, extensions: Extensions, - alt_hashing: Option, // TODO same alt hashing type as all + alt_hashing: Option>, // TODO same alt hashing type as all } impl BasicExternalities { /// Create a new instance of `BasicExternalities` - pub fn new(inner: Storage) -> Self { - let alt_hashing = inner.get_trie_alt_hashing_threshold(); + pub fn new(inner: Storage, alt_hashing: Option>) -> Self { BasicExternalities { inner, extensions: Default::default(), alt_hashing } } /// New basic externalities with empty storage. - pub fn new_empty() -> Self { - Self::new(Storage::default()) + pub fn new_empty(alt_hashing: Option>) -> Self { + Self::new(Storage::default(), alt_hashing) } /// Insert key/value @@ -72,9 +70,9 @@ impl BasicExternalities { /// Returns the result of the closure and updates `storage` with all changes. pub fn execute_with_storage( storage: &mut sp_core::storage::Storage, + alt_hashing: Option>, f: impl FnOnce() -> R, ) -> R { - let alt_hashing = storage.get_trie_alt_hashing_threshold(); let mut ext = Self { inner: Storage { top: std::mem::take(&mut storage.top), @@ -116,6 +114,7 @@ impl PartialEq for BasicExternalities { } } +/* impl FromIterator<(StorageKey, StorageValue)> for BasicExternalities { fn from_iter>(iter: I) -> Self { let mut t = Self::default(); @@ -123,16 +122,16 @@ impl FromIterator<(StorageKey, StorageValue)> for BasicExternalities { t } } +*/ -impl Default for BasicExternalities { - fn default() -> Self { - Self::new(Default::default()) +impl From>> for BasicExternalities { + fn from(alt_hashing: Option>) -> Self { + Self::new(Default::default(), alt_hashing) } } -impl From> for BasicExternalities { - fn from(hashmap: BTreeMap) -> Self { - let alt_hashing = sp_core::storage::alt_hashing::get_trie_alt_hashing_threshold(&hashmap); +impl From<(BTreeMap, Option>)> for BasicExternalities { + fn from((hashmap, alt_hashing): (BTreeMap, Option>)) -> Self { BasicExternalities { inner: Storage { top: hashmap, children_default: Default::default() }, extensions: Default::default(), @@ -300,7 +299,7 @@ impl Externalities for BasicExternalities { } } - let layout = if let Some(threshold) = self.alt_hashing.as_ref() { + let layout = if let Some(Some(threshold)) = self.alt_hashing.as_ref() { Layout::::with_alt_hashing(*threshold) } else { Layout::::default() @@ -311,7 +310,7 @@ impl Externalities for BasicExternalities { fn child_storage_root(&mut self, child_info: &ChildInfo) -> Vec { if let Some(child) = self.inner.children_default.get(child_info.storage_key()) { let delta = child.data.iter().map(|(k, v)| (k.as_ref(), Some(v.as_ref()))); - let in_mem = crate::in_memory_backend::new_in_mem::(Some(self.alt_hashing.clone())); + let in_mem = crate::in_memory_backend::new_in_mem::(self.alt_hashing.clone()); in_mem.child_storage_root(&child.child_info, delta).0 } else { empty_child_trie_root::>() @@ -396,7 +395,7 @@ mod tests { #[test] fn commit_should_work() { - let mut ext = BasicExternalities::default(); + let mut ext: BasicExternalities = None.into(); ext.set_storage(b"doe".to_vec(), b"reindeer".to_vec()); ext.set_storage(b"dog".to_vec(), b"puppy".to_vec()); ext.set_storage(b"dogglesworth".to_vec(), b"cat".to_vec()); @@ -408,7 +407,7 @@ mod tests { #[test] fn set_and_retrieve_code() { - let mut ext = BasicExternalities::default(); + let mut ext: BasicExternalities = None.into(); let code = vec![1, 2, 3]; ext.set_storage(CODE.to_vec(), code.clone()); @@ -428,7 +427,7 @@ mod tests { child_info: child_info.to_owned(), } ], - }); + }, None); assert_eq!(ext.child_storage(child_info, b"doe"), Some(b"reindeer".to_vec())); @@ -458,7 +457,7 @@ mod tests { child_info: child_info.to_owned(), } ], - }); + }, None); let res = ext.kill_child_storage(child_info, None); assert_eq!(res, (true, 3)); @@ -467,7 +466,7 @@ mod tests { #[test] fn basic_externalities_is_empty() { // Make sure no values are set by default in `BasicExternalities`. - let storage = BasicExternalities::new_empty().into_storages(); + let storage = BasicExternalities::new_empty(None).into_storages(); assert!(storage.top.is_empty()); assert!(storage.children_default.is_empty()); } diff --git a/primitives/state-machine/src/changes_trie/build.rs b/primitives/state-machine/src/changes_trie/build.rs index d3c6c12122c4f..732f2e6bfd5e9 100644 --- a/primitives/state-machine/src/changes_trie/build.rs +++ b/primitives/state-machine/src/changes_trie/build.rs @@ -382,7 +382,7 @@ mod test { ) { let child_info_1 = ChildInfo::new_default(b"storage_key1"); let child_info_2 = ChildInfo::new_default(b"storage_key2"); - let backend: InMemoryBackend<_> = vec![ + let backend: InMemoryBackend<_> = (vec![ (vec![100], vec![255]), (vec![101], vec![255]), (vec![102], vec![255]), @@ -391,7 +391,7 @@ mod test { (vec![105], vec![255]), ] .into_iter() - .collect::>() + .collect::>(), None) .into(); let prefixed_child_trie_key1 = child_info_1.prefixed_storage_key(); let storage = InMemoryStorage::with_inputs( diff --git a/primitives/state-machine/src/ext.rs b/primitives/state-machine/src/ext.rs index c9693ca6a88c1..a12efcdf64b5f 100644 --- a/primitives/state-machine/src/ext.rs +++ b/primitives/state-machine/src/ext.rs @@ -977,7 +977,7 @@ mod tests { fn storage_changes_root_is_none_when_storage_is_not_provided() { let mut overlay = prepare_overlay_with_changes(); let mut cache = StorageTransactionCache::default(); - let backend = TestBackend::default(); + let backend: TestBackend = None.into(); let mut ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); assert_eq!(ext.storage_changes_root(&H256::default().encode()).unwrap(), None); } @@ -986,7 +986,7 @@ mod tests { fn storage_changes_root_is_none_when_state_is_not_provided() { let mut overlay = prepare_overlay_with_changes(); let mut cache = StorageTransactionCache::default(); - let backend = TestBackend::default(); + let backend: TestBackend = None.into(); let mut ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); assert_eq!(ext.storage_changes_root(&H256::default().encode()).unwrap(), None); } @@ -997,7 +997,7 @@ mod tests { let mut cache = StorageTransactionCache::default(); let storage = TestChangesTrieStorage::with_blocks(vec![(99, Default::default())]); let state = Some(ChangesTrieState::new(changes_trie_config(), Zero::zero(), &storage)); - let backend = TestBackend::default(); + let backend: TestBackend = None.into(); let mut ext = TestExt::new(&mut overlay, &mut cache, &backend, state, None); assert_eq!( ext.storage_changes_root(&H256::default().encode()).unwrap(), @@ -1013,7 +1013,7 @@ mod tests { overlay.set_storage(vec![1], None); let storage = TestChangesTrieStorage::with_blocks(vec![(99, Default::default())]); let state = Some(ChangesTrieState::new(changes_trie_config(), Zero::zero(), &storage)); - let backend = TestBackend::default(); + let backend: TestBackend = None.into(); let mut ext = TestExt::new(&mut overlay, &mut cache, &backend, state, None); assert_eq!( ext.storage_changes_root(&H256::default().encode()).unwrap(), @@ -1027,14 +1027,14 @@ mod tests { let mut overlay = OverlayedChanges::default(); overlay.set_storage(vec![20], None); overlay.set_storage(vec![30], Some(vec![31])); - let backend = Storage { + let backend = (Storage { top: map![ vec![10] => vec![10], vec![20] => vec![20], vec![40] => vec![40] ], children_default: map![], - } + }, None) .into(); let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); @@ -1073,12 +1073,12 @@ mod tests { overlay.set_storage(vec![27], None); overlay.set_storage(vec![28], None); overlay.set_storage(vec![29], None); - let backend = Storage { + let backend = (Storage { top: map![ vec![30] => vec![30] ], children_default: map![], - } + }, None) .into(); let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); @@ -1097,7 +1097,7 @@ mod tests { let mut overlay = OverlayedChanges::default(); overlay.set_child_storage(child_info, vec![20], None); overlay.set_child_storage(child_info, vec![30], Some(vec![31])); - let backend = Storage { + let backend = (Storage { top: map![], children_default: map![ child_info.storage_key().to_vec() => StorageChild { @@ -1109,7 +1109,7 @@ mod tests { child_info: child_info.to_owned(), } ], - } + }, None) .into(); let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); @@ -1142,7 +1142,7 @@ mod tests { let mut overlay = OverlayedChanges::default(); overlay.set_child_storage(child_info, vec![20], None); overlay.set_child_storage(child_info, vec![30], Some(vec![31])); - let backend = Storage { + let backend = (Storage { top: map![], children_default: map![ child_info.storage_key().to_vec() => StorageChild { @@ -1154,7 +1154,7 @@ mod tests { child_info: child_info.to_owned(), } ], - } + }, None) .into(); let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); @@ -1181,7 +1181,7 @@ mod tests { let child_info = &child_info; let mut cache = StorageTransactionCache::default(); let mut overlay = OverlayedChanges::default(); - let backend = Storage { + let backend = (Storage { top: map![], children_default: map![ child_info.storage_key().to_vec() => StorageChild { @@ -1191,7 +1191,7 @@ mod tests { child_info: child_info.to_owned(), } ], - } + }, None) .into(); let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); diff --git a/primitives/state-machine/src/in_memory_backend.rs b/primitives/state-machine/src/in_memory_backend.rs index ae7c88de67e46..6ec0dbdf8f25c 100644 --- a/primitives/state-machine/src/in_memory_backend.rs +++ b/primitives/state-machine/src/in_memory_backend.rs @@ -189,7 +189,7 @@ mod tests { /// Assert in memory backend with only child trie keys works as trie backend. #[test] fn in_memory_with_child_trie_only() { - let storage = new_in_mem::(); + let storage = new_in_mem::(None); let child_info = ChildInfo::new_default(b"1"); let child_info = &child_info; let mut storage = storage @@ -202,7 +202,7 @@ mod tests { #[test] fn insert_multiple_times_child_data_works() { - let mut storage = new_in_mem::(); + let mut storage = new_in_mem::(None); let child_info = ChildInfo::new_default(b"1"); storage diff --git a/primitives/state-machine/src/lib.rs b/primitives/state-machine/src/lib.rs index ad9997a7ed3f4..62cf09ce397e4 100644 --- a/primitives/state-machine/src/lib.rs +++ b/primitives/state-machine/src/lib.rs @@ -1222,7 +1222,7 @@ mod tests { b"abc".to_vec() => b"2".to_vec(), b"bbb".to_vec() => b"3".to_vec() ]; - let mut state = InMemoryBackend::::from(initial); + let mut state = InMemoryBackend::::from((initial, None)); let backend = state.as_trie_backend().unwrap(); let mut overlay = OverlayedChanges::default(); @@ -1303,7 +1303,7 @@ mod tests { b"d".to_vec() => b"3".to_vec() ], ]; - let backend = InMemoryBackend::::from(initial); + let backend = InMemoryBackend::::from((initial, None)); let mut overlay = OverlayedChanges::default(); overlay.set_child_storage(&child_info, b"1".to_vec(), Some(b"1312".to_vec())); @@ -1351,7 +1351,7 @@ mod tests { b"d".to_vec() => b"3".to_vec() ], ]; - let backend = InMemoryBackend::::from(initial); + let backend = InMemoryBackend::::from((initial, None)); let mut overlay = OverlayedChanges::default(); let mut cache = StorageTransactionCache::default(); let mut ext = Ext::new( @@ -1375,7 +1375,7 @@ mod tests { fn set_child_storage_works() { let child_info = ChildInfo::new_default(b"sub1"); let child_info = &child_info; - let mut state = new_in_mem::(); + let mut state = new_in_mem::(None); let backend = state.as_trie_backend().unwrap(); let mut overlay = OverlayedChanges::default(); let mut cache = StorageTransactionCache::default(); @@ -1397,7 +1397,7 @@ mod tests { fn append_storage_works() { let reference_data = vec![b"data1".to_vec(), b"2".to_vec(), b"D3".to_vec(), b"d4".to_vec()]; let key = b"key".to_vec(); - let mut state = new_in_mem::(); + let mut state = new_in_mem::(None); let backend = state.as_trie_backend().unwrap(); let mut overlay = OverlayedChanges::default(); let mut cache = StorageTransactionCache::default(); @@ -1452,7 +1452,7 @@ mod tests { let key = b"events".to_vec(); let mut cache = StorageTransactionCache::default(); - let mut state = new_in_mem::(); + let mut state = new_in_mem::(None); let backend = state.as_trie_backend().unwrap(); let mut overlay = OverlayedChanges::default(); @@ -1641,6 +1641,7 @@ mod tests { assert_eq!(completed, true); } + // TODO test does not make lot of sense with migration. #[test] fn inner_state_hashing_switch_proofs() { let mut layout = Layout::default(); @@ -1656,8 +1657,8 @@ mod tests { .expect("insert failed"); } - let check_proof = |mdb, root| -> StorageProof { - let remote_backend = TrieBackend::new(mdb, root); + let check_proof = |mdb, root, state_version| -> StorageProof { + let remote_backend = TrieBackend::new(mdb, root, state_version); let remote_root = remote_backend.storage_root(::std::iter::empty()).0; let remote_proof = prove_read(remote_backend, &[b"foo222"]).unwrap(); // check proof locally @@ -1672,7 +1673,7 @@ mod tests { remote_proof }; - let remote_proof = check_proof(mdb.clone(), root.clone()); + let remote_proof = check_proof(mdb.clone(), root.clone(), None); // check full values in proof assert!(remote_proof.encode().len() > 1_100); assert!(remote_proof.encoded_size() > 1_100); @@ -1703,7 +1704,7 @@ mod tests { } let root3 = root.clone(); assert!(root1 != root3); - let remote_proof = check_proof(mdb.clone(), root.clone()); + let remote_proof = check_proof(mdb.clone(), root.clone(), Some(Some(TRESHOLD))); // nodes foo is replaced by its hashed value form. assert!(remote_proof.encode().len() < 1000); assert!(remote_proof.encoded_size() < 1000); @@ -1811,7 +1812,7 @@ mod tests { b"aaa".to_vec() => b"0".to_vec(), b"bbb".to_vec() => b"".to_vec() ]; - let mut state = InMemoryBackend::::from(initial); + let mut state = InMemoryBackend::::from((initial, None)); let backend = state.as_trie_backend().unwrap(); let mut overlay = OverlayedChanges::default(); diff --git a/primitives/state-machine/src/overlayed_changes/mod.rs b/primitives/state-machine/src/overlayed_changes/mod.rs index a0558e06a380e..f8c2bd8c659b3 100644 --- a/primitives/state-machine/src/overlayed_changes/mod.rs +++ b/primitives/state-machine/src/overlayed_changes/mod.rs @@ -931,7 +931,7 @@ mod tests { ] .into_iter() .collect(); - let backend = InMemoryBackend::::from(initial); + let backend = InMemoryBackend::::from((initial, None)); let mut overlay = OverlayedChanges::default(); overlay.set_collect_extrinsics(false); diff --git a/primitives/state-machine/src/proving_backend.rs b/primitives/state-machine/src/proving_backend.rs index 2e9d8421acd7f..5648fa2acf0b9 100644 --- a/primitives/state-machine/src/proving_backend.rs +++ b/primitives/state-machine/src/proving_backend.rs @@ -457,7 +457,7 @@ mod tests { use sp_core::H256; let result = create_proof_check_backend::( H256::from_low_u64_be(1), - StorageProof::empty(), + StorageProof::empty(None), ); assert!(result.is_err()); } @@ -487,23 +487,16 @@ mod tests { fn proof_recorded_and_checked_inner(flagged: bool) { let size_content = 34; // above hashable value treshold. let value_range = 0..64; - let contents = value_range + + let contents = vec![(None, value_range .clone() .map(|i| (vec![i], Some(vec![i; size_content]))) - .collect::>(); - let mut in_memory = InMemoryBackend::::default(); - if flagged { - in_memory = in_memory.update(vec![( - None, - vec![( - sp_core::storage::well_known_keys::TRIE_HASHING_CONFIG.to_vec(), - Some(sp_core::storage::trie_threshold_encode( - sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD, - )), - )], - )]); - } - let mut in_memory = in_memory.update(vec![(None, contents)]); + .collect::>())]; + let mut in_memory: InMemoryBackend = if flagged { + (contents, Some(Some(sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD))).into() + } else { + (contents, None).into() + }; let in_memory_root = in_memory.storage_root(std::iter::empty()).0; value_range.clone().for_each(|i| { assert_eq!(in_memory.storage(&[i]).unwrap().unwrap(), vec![i; size_content]) @@ -541,19 +534,11 @@ mod tests { (Some(child_info_1.clone()), (28..65).map(|i| (vec![i], Some(vec![i]))).collect()), (Some(child_info_2.clone()), (10..15).map(|i| (vec![i], Some(vec![i]))).collect()), ]; - let mut in_memory = InMemoryBackend::::default(); - if flagged { - in_memory = in_memory.update(vec![( - None, - vec![( - sp_core::storage::well_known_keys::TRIE_HASHING_CONFIG.to_vec(), - Some(sp_core::storage::trie_threshold_encode( - sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD, - )), - )], - )]); - } - in_memory = in_memory.update(contents); + let mut in_memory: InMemoryBackend = if flagged { + (contents, Some(Some(sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD))).into() + } else { + (contents, None).into() + }; let child_storage_keys = vec![child_info_1.to_owned(), child_info_2.to_owned()]; let in_memory_root = in_memory .full_storage_root( diff --git a/primitives/state-machine/src/testing.rs b/primitives/state-machine/src/testing.rs index 1c2b7a3cad600..2c11eec9341bb 100644 --- a/primitives/state-machine/src/testing.rs +++ b/primitives/state-machine/src/testing.rs @@ -86,27 +86,17 @@ where } /// Create a new instance of `TestExternalities` with storage. - pub fn new(storage: Storage) -> Self { - Self::new_with_code(&[], storage) - } - - /// Create a new instance of `TestExternalities` with storage - /// on a backend containing defined default alt hashing threshold. - pub fn new_with_alt_hashing(storage: Storage) -> Self { - Self::new_with_code_inner(&[], storage, true) + pub fn new(storage: Storage, alt_hashing: Option>) -> Self { + Self::new_with_code(&[], storage, alt_hashing) } /// New empty test externalities. - pub fn new_empty() -> Self { - Self::new_with_code(&[], Storage::default()) + pub fn new_empty(alt_hashing: Option>) -> Self { + Self::new_with_code(&[], Storage::default(), alt_hashing) } /// Create a new instance of `TestExternalities` with code and storage. - pub fn new_with_code(code: &[u8], storage: Storage) -> Self { - Self::new_with_code_inner(code, storage, false) - } - - fn new_with_code_inner(code: &[u8], mut storage: Storage, force_alt_hashing: bool) -> Self { + pub fn new_with_code(code: &[u8], mut storage: Storage, alt_hashing: Option>) -> Self { let mut overlay = OverlayedChanges::default(); let changes_trie_config = storage .top @@ -124,11 +114,7 @@ where let offchain_db = TestPersistentOffchainDB::new(); - let backend = if force_alt_hashing { - (storage, Some(Some(sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD))).into() - } else { - (storage, None).into() - }; + let backend = (storage, alt_hashing).into(); TestExternalities { overlay, @@ -257,21 +243,17 @@ where H::Out: Ord + 'static + codec::Codec, { fn default() -> Self { - // default to inner hashed. - let mut storage = Storage::default(); - storage.modify_trie_alt_hashing_threshold(Some( - sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD, - )); - Self::new(storage) + // default to default version. + Self::new(Storage::default(), Some(Some(sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD))) } } -impl From for TestExternalities +impl From<(Storage, Option>)> for TestExternalities where H::Out: Ord + 'static + codec::Codec, { - fn from(storage: Storage) -> Self { - Self::new(storage) + fn from((storage, alt_hashing): (Storage, Option>)) -> Self { + Self::new(storage, alt_hashing) } } @@ -334,7 +316,7 @@ mod tests { #[test] fn commit_should_work() { let storage = Storage::default(); // avoid adding the trie threshold. - let mut ext = TestExternalities::::from(storage); + let mut ext = TestExternalities::::from((storage, None)); let mut ext = ext.ext(); ext.set_storage(b"doe".to_vec(), b"reindeer".to_vec()); ext.set_storage(b"dog".to_vec(), b"puppy".to_vec()); diff --git a/primitives/state-machine/src/trie_backend.rs b/primitives/state-machine/src/trie_backend.rs index eb086a6ae2edc..ee5ca114ddc64 100644 --- a/primitives/state-machine/src/trie_backend.rs +++ b/primitives/state-machine/src/trie_backend.rs @@ -200,18 +200,13 @@ where where H::Out: Ord, { - let use_inner_hash_value = if let Some(force) = self.force_alt_hashing.as_ref() { - force.clone() - } else { - self.get_trie_alt_hashing_threshold() - }; let mut write_overlay = S::Overlay::default(); let mut root = *self.essence.root(); { let mut eph = Ephemeral::new(self.essence.backend_storage(), &mut write_overlay); let res = || { - let layout = if let Some(threshold) = use_inner_hash_value { + let layout = if let Some(Some(threshold)) = self.force_alt_hashing.clone() { sp_trie::Layout::with_alt_hashing(threshold) } else { sp_trie::Layout::default() @@ -236,16 +231,10 @@ where where H::Out: Ord, { - let use_inner_hash_value = if let Some(force) = self.force_alt_hashing.as_ref() { - force.clone() - } else { - self.get_trie_alt_hashing_threshold() - }; - let default_root = match child_info.child_type() { ChildType::ParentKeyId => empty_child_trie_root::>(), }; - let layout = if let Some(threshold) = use_inner_hash_value { + let layout = if let Some(Some(threshold)) = self.force_alt_hashing.clone() { sp_trie::Layout::with_alt_hashing(threshold) } else { sp_trie::Layout::default() @@ -340,13 +329,6 @@ pub mod tests { trie.insert(b"value1", &[42]).expect("insert failed"); trie.insert(b"value2", &[24]).expect("insert failed"); trie.insert(b":code", b"return 42").expect("insert failed"); - if hashed_value { - trie.insert( - sp_core::storage::well_known_keys::TRIE_HASHING_CONFIG, - sp_core::storage::trie_threshold_encode(TRESHOLD).as_slice(), - ) - .unwrap(); - } for i in 128u8..255u8 { trie.insert(&[i], &[i]).unwrap(); } @@ -358,7 +340,7 @@ pub mod tests { hashed_value: bool, ) -> TrieBackend, BlakeTwo256> { let (mdb, root) = test_db(hashed_value); - TrieBackend::new(mdb, root) + TrieBackend::new(mdb, root, None) } #[test] @@ -424,6 +406,7 @@ pub mod tests { assert!(TrieBackend::, BlakeTwo256>::new( PrefixedMemoryDB::default(), Default::default(), + None, ) .pairs() .is_empty()); diff --git a/primitives/storage/src/lib.rs b/primitives/storage/src/lib.rs index b00a42d8aab18..02fc1e1c41718 100644 --- a/primitives/storage/src/lib.rs +++ b/primitives/storage/src/lib.rs @@ -204,10 +204,6 @@ pub mod well_known_keys { /// Current extrinsic index (u32) is stored under this key. pub const EXTRINSIC_INDEX: &'static [u8] = b":extrinsic_index"; - /// Configuration for trie internal hashing of value is stored - /// under this key. - pub const TRIE_HASHING_CONFIG: &'static [u8] = b":trie_hashing_conf"; - /// Changes trie configuration is stored under this key. pub const CHANGES_TRIE_CONFIG: &'static [u8] = b":changes_trie"; @@ -250,41 +246,6 @@ pub fn trie_threshold_decode(mut encoded: &[u8]) -> Option { /// Default value to use as a threshold for testing. pub const TEST_DEFAULT_ALT_HASH_THRESHOLD: u32 = 34; -#[cfg(feature = "std")] -impl Storage { - /// Utility function to get trie inner value hash threshold from - /// backend state or pending changes. - pub fn get_trie_alt_hashing_threshold(&self) -> Option { - alt_hashing::get_trie_alt_hashing_threshold(&self.top) - } - - /// Utility function to modify trie inner value hash threshold. - pub fn modify_trie_alt_hashing_threshold(&mut self, threshold: Option) { - match threshold { - Some(threshold) => { - let encoded = trie_threshold_encode(threshold); - self.top.insert(well_known_keys::TRIE_HASHING_CONFIG.to_vec(), encoded); - }, - None => { - self.top.remove(well_known_keys::TRIE_HASHING_CONFIG); - }, - } - } -} - -/// alt hashing related utils. -#[cfg(feature = "std")] -pub mod alt_hashing { - use super::*; - - /// Utility function to get trie inner value hash threshold from - /// backend state or pending changes. - pub fn get_trie_alt_hashing_threshold(map: &StorageMap) -> Option { - map.get(well_known_keys::TRIE_HASHING_CONFIG) - .and_then(|encoded| trie_threshold_decode(&mut encoded.as_slice())) - } -} - /// Information related to a child state. #[derive(Debug, Clone)] #[cfg_attr(feature = "std", derive(PartialEq, Eq, Hash, PartialOrd, Ord))] diff --git a/test-utils/client/src/lib.rs b/test-utils/client/src/lib.rs index 2f5e59ebde4e8..8df3d9ea84788 100644 --- a/test-utils/client/src/lib.rs +++ b/test-utils/client/src/lib.rs @@ -85,7 +85,7 @@ pub struct TestClientBuilder { fork_blocks: ForkBlocks, bad_blocks: BadBlocks, enable_offchain_indexing_api: bool, - state_hashed_value: bool, + state_hashed_value: bool, // TODO useless ? no_genesis: bool, } @@ -230,12 +230,6 @@ impl { let storage = { let mut storage = self.genesis_init.genesis_storage(); - if self.state_hashed_value { - storage.modify_trie_alt_hashing_threshold(Some( - sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD, - )); - } - // Add some child storage keys. for (key, child_content) in self.child_storage_extension { storage.children_default.insert( diff --git a/test-utils/runtime/client/src/lib.rs b/test-utils/runtime/client/src/lib.rs index e2a021a2ecf1f..f45908b81fcff 100644 --- a/test-utils/runtime/client/src/lib.rs +++ b/test-utils/runtime/client/src/lib.rs @@ -116,6 +116,7 @@ impl GenesisParameters { 1000, self.heap_pages_override, self.extra_storage.clone(), + Some(Some(sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD)), ) } diff --git a/test-utils/runtime/src/genesismap.rs b/test-utils/runtime/src/genesismap.rs index a8801b8519dfe..5e576cfef08df 100644 --- a/test-utils/runtime/src/genesismap.rs +++ b/test-utils/runtime/src/genesismap.rs @@ -37,6 +37,7 @@ pub struct GenesisConfig { heap_pages_override: Option, /// Additional storage key pairs that will be added to the genesis map. extra_storage: Storage, + state_version: Option>, } impl GenesisConfig { @@ -47,6 +48,7 @@ impl GenesisConfig { balance: u64, heap_pages_override: Option, extra_storage: Storage, + state_version: Option>, ) -> Self { GenesisConfig { changes_trie_config, @@ -54,6 +56,7 @@ impl GenesisConfig { balances: endowed_accounts.into_iter().map(|a| (a, balance)).collect(), heap_pages_override, extra_storage, + state_version, } } @@ -90,7 +93,7 @@ impl GenesisConfig { let mut config = system::GenesisConfig::default(); config.authorities = self.authorities.clone(); config - .assimilate_storage(&mut storage) + .assimilate_storage(&mut storage, self.state_version.clone()) .expect("Adding `system::GensisConfig` to the genesis"); storage diff --git a/test-utils/runtime/src/lib.rs b/test-utils/runtime/src/lib.rs index 92dc4b8a9d806..ad7baea527a80 100644 --- a/test-utils/runtime/src/lib.rs +++ b/test-utils/runtime/src/lib.rs @@ -1316,7 +1316,7 @@ mod tests { #[test] fn witness_backend_works() { let (db, root) = witness_backend(); - let backend = sp_state_machine::TrieBackend::<_, crate::Hashing>::new(db, root); + let backend = sp_state_machine::TrieBackend::<_, crate::Hashing>::new(db, root, None); let proof = sp_state_machine::prove_read(backend, vec![b"value3"]).unwrap(); let client = TestClientBuilder::new().set_execution_strategy(ExecutionStrategy::Both).build(); diff --git a/test-utils/runtime/src/system.rs b/test-utils/runtime/src/system.rs index 67a6e8991e598..4ff9408bab809 100644 --- a/test-utils/runtime/src/system.rs +++ b/test-utils/runtime/src/system.rs @@ -366,6 +366,7 @@ mod tests { } fn new_test_ext() -> TestExternalities { + let state_version = None; let authorities = vec![ Sr25519Keyring::Alice.to_raw_public(), Sr25519Keyring::Bob.to_raw_public(), @@ -381,13 +382,10 @@ mod tests { blake2_256(&AccountKeyring::Alice.to_raw_public().to_keyed_vec(b"balance:")).to_vec() => { vec![111u8, 0, 0, 0, 0, 0, 0, 0] }, - sp_core::storage::well_known_keys::TRIE_HASHING_CONFIG.to_vec() => - sp_core::storage::trie_threshold_encode( - sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD, - ), ], children_default: map![], }, + state_version, ) } diff --git a/test-utils/test-runner/src/utils.rs b/test-utils/test-runner/src/utils.rs index 2fe3a98d44ad0..5ea6c23580fb6 100644 --- a/test-utils/test-runner/src/utils.rs +++ b/test-utils/test-runner/src/utils.rs @@ -50,9 +50,14 @@ pub fn default_config( let base_path = base_path(); let root_path = base_path.path().to_path_buf().join("chains").join(chain_spec.id()); + let state_version = chain_spec.state_versions() + .get(0) + .and_then(|(n, s)| (n == &0).then(|| s.clone())) + .unwrap_or(None); + let storage = chain_spec .as_storage_builder() - .build_storage() + .build_storage(state_version) .expect("could not build storage"); chain_spec.set_storage(storage); diff --git a/utils/frame/benchmarking-cli/src/command.rs b/utils/frame/benchmarking-cli/src/command.rs index 671386a721a02..fa697b167eb64 100644 --- a/utils/frame/benchmarking-cli/src/command.rs +++ b/utils/frame/benchmarking-cli/src/command.rs @@ -96,6 +96,10 @@ impl BenchmarkCmd { ::Hash: std::str::FromStr, ExecDispatch: NativeExecutionDispatch + 'static, { + let state_version = config.chain_spec.state_versions() + .get(0) + .and_then(|(n, s)| (n == &0).then(|| s.clone())) + .unwrap_or(None); if let Some(output_path) = &self.output { if !output_path.is_dir() && output_path.file_name().is_none() { return Err("Output file or path is invalid!".into()) @@ -122,7 +126,7 @@ impl BenchmarkCmd { let extrinsic = self.extrinsic.clone().unwrap_or_else(|| String::new()); let extrinsic = extrinsic.as_bytes(); - let genesis_storage = spec.build_storage()?; + let genesis_storage = spec.build_storage(state_version.clone())?; let mut changes = Default::default(); let cache_size = Some(self.database_cache_size as usize); let state_with_tracking = BenchmarkingState::::new( diff --git a/utils/frame/remote-externalities/src/lib.rs b/utils/frame/remote-externalities/src/lib.rs index 347cc8d66d917..2f250e4e5da08 100644 --- a/utils/frame/remote-externalities/src/lib.rs +++ b/utils/frame/remote-externalities/src/lib.rs @@ -485,7 +485,7 @@ impl Builder { /// Build the test externalities. pub async fn build(self) -> Result { let kv = self.pre_build().await?; - let mut ext = TestExternalities::new_empty(); + let mut ext = TestExternalities::new_empty(None); info!(target: LOG_TARGET, "injecting a total of {} keys", kv.len()); for (k, v) in kv { diff --git a/utils/frame/try-runtime/cli/src/lib.rs b/utils/frame/try-runtime/cli/src/lib.rs index 1f1eef70e1b93..8c949b06be56b 100644 --- a/utils/frame/try-runtime/cli/src/lib.rs +++ b/utils/frame/try-runtime/cli/src/lib.rs @@ -463,7 +463,14 @@ impl CliConfiguration for TryRuntimeCmd { /// Extract `:code` from the given chain spec and return as `StorageData` along with the /// corresponding `StorageKey`. fn extract_code(spec: Box) -> sc_cli::Result<(StorageKey, StorageData)> { - let genesis_storage = spec.build_storage()?; + let mut state_version = None; + if let Some((number, version)) = spec.state_versions().get(0) { + if number == &0 { + state_version = version.clone(); + } + } + + let genesis_storage = spec.build_storage(state_version)?; let code = StorageData( genesis_storage .top From b85b173e4e3eae805abfa876b0fd0a538b461c18 Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Fri, 20 Aug 2021 11:44:09 +0200 Subject: [PATCH 070/127] Base types for state versioning. --- primitives/core/src/lib.rs | 1 + primitives/core/src/state_version.rs | 50 ++++++++++++++++ primitives/runtime/src/lib.rs | 1 + primitives/runtime/src/state_version.rs | 78 +++++++++++++++++++++++++ 4 files changed, 130 insertions(+) create mode 100644 primitives/core/src/state_version.rs create mode 100644 primitives/runtime/src/state_version.rs diff --git a/primitives/core/src/lib.rs b/primitives/core/src/lib.rs index 0a61c90d71357..df58b972d80ba 100644 --- a/primitives/core/src/lib.rs +++ b/primitives/core/src/lib.rs @@ -70,6 +70,7 @@ pub mod testing; #[cfg(feature = "std")] pub mod traits; pub mod uint; +pub mod state_version; pub use self::{ hash::{convert_hash, H160, H256, H512}, diff --git a/primitives/core/src/state_version.rs b/primitives/core/src/state_version.rs new file mode 100644 index 0000000000000..70c59ffaa86b0 --- /dev/null +++ b/primitives/core/src/state_version.rs @@ -0,0 +1,50 @@ +// This file is part of Substrate. + +// Copyright (C) 2021-2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Substrate state versioning core types. + +/// Default state version to use with a new substrate chain. +/// +/// When this value change, old chain will require to force their +/// initial state versionning in their chainspec for block 0. +/// Therefore defining genesis version in chainspec is good practice +/// and this default should mostly be use when testing. +pub const DEFAULT_STATE_VERSION: StateVersion = StateVersion::V1 { threshold: 33 }; + +/// Supported version with substrate chain. +#[derive(Clone, Copy)] +#[cfg_attr(feature = "std", derive(serde::Serialize, serde::Deserialize))] +pub enum StateVersion { + /// Patricia trie Radix 16 without extension node. + V0, + /// Patricia trie Radix 16 without extension node, + /// with inner hashing applied on value of size. + V1 { + /// Inner hashing apply only when the value + /// is equal to `threshold`. + /// Threashold should ALWAYS be bigger than + /// the hasher output size due to inline node + /// (with most hasher at least 33). + threshold: u32, + }, +} + +impl Default for StateVersion { + fn default() -> Self { + DEFAULT_STATE_VERSION + } +} diff --git a/primitives/runtime/src/lib.rs b/primitives/runtime/src/lib.rs index f47f5cbe95f65..9918668763e5e 100644 --- a/primitives/runtime/src/lib.rs +++ b/primitives/runtime/src/lib.rs @@ -61,6 +61,7 @@ mod runtime_string; pub mod testing; pub mod traits; pub mod transaction_validity; +pub mod state_version; pub use crate::runtime_string::*; diff --git a/primitives/runtime/src/state_version.rs b/primitives/runtime/src/state_version.rs new file mode 100644 index 0000000000000..6c035df79acdc --- /dev/null +++ b/primitives/runtime/src/state_version.rs @@ -0,0 +1,78 @@ +// This file is part of Substrate. + +// Copyright (C) 2021-2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Substrate state versioning and migrations related types. + +pub use sp_core::state_version::{StateVersion, DEFAULT_STATE_VERSION}; +use crate::traits::{Block, NumberFor}; +use sp_std::str::FromStr; +use sp_std::vec::Vec; +use sp_arithmetic::traits::Zero; + +/// Multiple versions of state in use for a chain. +#[derive(Clone)] +pub struct StateVersions { + canonical_states: Vec<(NumberFor, StateVersion)>, +} + +impl StateVersions { + /// Access genesis state version. + /// This uses default state if undefined. + pub fn genesis_state_version(&self) -> StateVersion { + if let Some((number, version)) = self.canonical_states.get(0) { + if number.is_zero() { + return *version; + } + } + DEFAULT_STATE_VERSION + } + + /// Resolve state version for a given + /// block height. + pub fn state_version_at(&self, at: NumberFor) -> StateVersion { + let mut version = DEFAULT_STATE_VERSION; + for (number, state) in self.canonical_states.iter() { + if number > &at { + break; + } + version = *state; + } + version + } + + /// Convert from chainspec conf. + pub fn from_conf<'a, I>(conf: I) -> Option + where I: IntoIterator, + { + let iter = conf.into_iter(); + let mut canonical_states = match iter.size_hint() { + (s, None) => Vec::with_capacity(s), + (_, Some(s)) => Vec::with_capacity(s), + }; + + for (number, version) in iter { + if let Ok(number) = NumberFor::::from_str(number) { + canonical_states.push((number.into(), version)); + } else { + return None; + } + } + Some(StateVersions { + canonical_states, + }) + } +} From 08e1f4b0715b811d5647984fb29c90bba5c2a6a5 Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Fri, 20 Aug 2021 13:30:19 +0200 Subject: [PATCH 071/127] Partial changes. --- bin/node/bench/src/generator.rs | 20 +++--- client/api/src/in_mem.rs | 19 ++---- client/block-builder/src/lib.rs | 2 +- client/chain-spec/src/chain_spec.rs | 29 ++++----- client/chain-spec/src/lib.rs | 4 +- client/db/src/bench.rs | 12 ++-- client/db/src/lib.rs | 65 +++++++------------ client/db/src/storage_cache.rs | 10 +-- client/light/src/backend.rs | 32 ++++----- client/light/src/lib.rs | 4 +- client/service/src/builder.rs | 38 +++-------- frame/balances/src/lib.rs | 9 +-- .../src/construct_runtime/expand/config.rs | 6 +- .../src/pallet/expand/genesis_build.rs | 5 +- .../src/storage/genesis_config/mod.rs | 12 ++-- frame/support/src/traits/hooks.rs | 9 +-- frame/system/src/lib.rs | 8 +-- frame/treasury/src/lib.rs | 4 +- .../api/proc-macro/src/impl_runtime_apis.rs | 4 +- .../proc-macro/src/mock_impl_runtime_apis.rs | 2 +- primitives/api/src/lib.rs | 2 +- primitives/core/src/state_version.rs | 2 +- primitives/runtime/src/lib.rs | 14 ++-- primitives/runtime/src/state_version.rs | 23 ++++++- primitives/state-machine/src/backend.rs | 4 +- primitives/state-machine/src/basic.rs | 50 +++++++------- .../state-machine/src/in_memory_backend.rs | 44 ++++++------- .../state-machine/src/proving_backend.rs | 26 +++++--- primitives/state-machine/src/testing.rs | 19 +++--- primitives/state-machine/src/trie_backend.rs | 37 ++++++----- primitives/trie/src/storage_proof.rs | 32 +++++---- primitives/trie/src/trie_codec.rs | 4 +- test-utils/runtime/src/lib.rs | 4 +- test-utils/test-runner/src/utils.rs | 5 +- utils/frame/benchmarking-cli/src/command.rs | 6 +- utils/frame/try-runtime/cli/src/lib.rs | 9 +-- 36 files changed, 270 insertions(+), 305 deletions(-) diff --git a/bin/node/bench/src/generator.rs b/bin/node/bench/src/generator.rs index f95811c40ebee..f0422eed5a437 100644 --- a/bin/node/bench/src/generator.rs +++ b/bin/node/bench/src/generator.rs @@ -23,6 +23,7 @@ use node_primitives::Hash; use sp_trie::{trie_types::TrieDBMut, TrieMut}; use crate::simple_trie::SimpleTrie; +use sp_core::StateVersion; /// Generate trie from given `key_values`. /// @@ -31,7 +32,7 @@ use crate::simple_trie::SimpleTrie; pub fn generate_trie( db: Arc, key_values: impl IntoIterator, Vec)>, - alt_hashing: Option, + state_version: StateVersion, ) -> Hash { let mut root = Hash::default(); @@ -44,13 +45,16 @@ pub fn generate_trie( ); let mut trie = SimpleTrie { db, overlay: &mut overlay }; { - let mut trie_db = if let Some(threshold) = alt_hashing { - let layout = sp_trie::Layout::with_alt_hashing(threshold); - TrieDBMut::::new_with_layout( - &mut trie, &mut root, layout, - ) - } else { - TrieDBMut::new(&mut trie, &mut root) + let mut trie_db = match StateVersion { + V0 => { + TrieDBMut::new(&mut trie, &mut root) + }, + V1 { threshold } => { + let layout = sp_trie::Layout::with_alt_hashing(threshold); + TrieDBMut::::new_with_layout( + &mut trie, &mut root, layout, + ) + }, }; for (key, value) in key_values { trie_db.insert(&key, &value).expect("trie insertion failed"); diff --git a/client/api/src/in_mem.rs b/client/api/src/in_mem.rs index 07bceafcd6287..c704329cf4d8d 100644 --- a/client/api/src/in_mem.rs +++ b/client/api/src/in_mem.rs @@ -24,7 +24,7 @@ use sp_core::{ offchain::storage::InMemOffchainStorage as OffchainStorage, storage::well_known_keys, }; use sp_runtime::{ - generic::BlockId, + generic::BlockId, StateVersions, StateVersion, traits::{Block as BlockT, HashFor, Header as HeaderT, NumberFor, Zero}, Justification, Justifications, Storage, }; @@ -112,8 +112,7 @@ struct BlockchainStorage { changes_trie_cht_roots: HashMap, Block::Hash>, leaves: LeafSet>, aux: HashMap, Vec>, - // TODO rename to state_versions and replace Option> to StateVersionEnum. - alt_hashing: Vec<(NumberFor, Option>)>, + state_versions: StateVersions, /* // TODO not sure if migration support. TODO replace () by StateMigration. state_migration: Vec<(NumberFor, ())>, @@ -149,8 +148,7 @@ impl Blockchain { } /// Get version of state. - pub fn state_version(&self, id: BlockId) -> Option> { - let default = Some(Some(sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD)); + pub fn state_version(&self, id: BlockId) -> StateVersion { let number = match id { BlockId::Hash(h) => { if let Ok(Some(header)) = self.header(BlockId::Hash(h)) { @@ -161,14 +159,7 @@ impl Blockchain { }, BlockId::Number(n) => n, }; - let mut result = default; - for (change_state, state) in self.storage.read().alt_hashing.iter() { - if &number > change_state { - break - } - result = *state; - } - result + self.storage.read().state_versions.version_at(number) } /// Create new in-memory blockchain storage. @@ -185,7 +176,7 @@ impl Blockchain { changes_trie_cht_roots: HashMap::new(), leaves: LeafSet::new(), aux: HashMap::new(), - alt_hashing: Vec::new(), + state_versions: Default::default(), })); Blockchain { storage } } diff --git a/client/block-builder/src/lib.rs b/client/block-builder/src/lib.rs index bc06b0eca329d..d447ce5c2f5a4 100644 --- a/client/block-builder/src/lib.rs +++ b/client/block-builder/src/lib.rs @@ -237,7 +237,7 @@ where let state = self.backend.state_at(self.block_id)?; let proof = { use backend::StateBackend; - self.api.extract_proof(state.alt_hashing().clone()) + self.api.extract_proof(state.state_version()) }; let changes_trie_state = backend::changes_tries_state_at_block( diff --git a/client/chain-spec/src/chain_spec.rs b/client/chain-spec/src/chain_spec.rs index ec02cfbaf76d0..d990338a0f4fd 100644 --- a/client/chain-spec/src/chain_spec.rs +++ b/client/chain-spec/src/chain_spec.rs @@ -28,7 +28,7 @@ use sp_core::{ storage::{ChildInfo, Storage, StorageChild, StorageData, StorageKey}, Bytes, }; -use sp_runtime::BuildStorage; +use sp_runtime::{BuildStorage, StateVersions, StateVersion}; use std::{borrow::Cow, collections::HashMap, fs::File, path::PathBuf, sync::Arc}; enum GenesisSource { @@ -99,9 +99,9 @@ impl GenesisSource { } impl BuildStorage for ChainSpec { - fn build_storage(&self, alt_hashing: Option>) -> Result { + fn build_storage(&self, state_version: StateVersion) -> Result { match self.genesis.resolve()? { - Genesis::Runtime(gc) => gc.build_storage(alt_hashing), + Genesis::Runtime(gc) => gc.build_storage(state_version), Genesis::Raw(RawGenesis { top: map, children_default: children_map }) => Ok(Storage { top: map.into_iter().map(|(k, v)| (k.0, v.0)).collect(), children_default: children_map @@ -121,7 +121,7 @@ impl BuildStorage for ChainSpec { } } - fn assimilate_storage(&self, _: &mut Storage, _: Option>) -> Result<(), String> { + fn assimilate_storage(&self, _: &mut Storage, _: StateVersion) -> Result<(), String> { Err("`assimilate_storage` not implemented for `ChainSpec`.".into()) } } @@ -172,7 +172,7 @@ struct ClientSpec { code_substitutes: HashMap, /// Ordered sequence of block number and their associated state version. #[serde(default)] - state_versions: Vec<(u64, Option>)>, + state_versions: Vec<(String, StateVersion)>, } /// A type denoting empty extensions. @@ -264,7 +264,7 @@ impl ChainSpec { consensus_engine: (), genesis: Default::default(), code_substitutes: HashMap::new(), - state_versions: vec![(0, Some(Some(sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD)))], + state_versions: vec![("0".to_string(), Some(Some(sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD)))], }; ChainSpec { client_spec, genesis: GenesisSource::Factory(Arc::new(constructor)) } @@ -276,15 +276,10 @@ impl ChainSpec { } /// Defined state version for the chain. - fn state_versions(&self) -> Vec<(u64, Option>)> { - self.client_spec.state_versions.clone() - } - - /// Defined state version for the chain. - pub fn genesis_state_version(&self) -> Option> { - self.client_spec.state_versions.get(0) - .and_then(|(n, s)| (n == &0).then(|| s.clone())) - .unwrap_or(Some(Some(sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD))) + /// Return None on invalid definition. + fn state_versions(&self) -> Option { + StateVersions::from_conf(self.client_spec.state_versions + .iter().map(|(number, version)| (number.as_str(), *version))) } } @@ -421,7 +416,7 @@ where .collect() } - fn state_versions(&self) -> Vec<(u64, Option>)> { + fn state_versions(&self) -> Vec<(u64, StateVersion)> { ChainSpec::state_versions(self) } } @@ -434,7 +429,7 @@ mod tests { struct Genesis(HashMap); impl BuildStorage for Genesis { - fn assimilate_storage(&self, storage: &mut Storage, _alt_hashing: Option>) -> Result<(), String> { + fn assimilate_storage(&self, storage: &mut Storage, _state_version: StateVersion) -> Result<(), String> { storage.top.extend( self.0.iter().map(|(a, b)| (a.clone().into_bytes(), b.clone().into_bytes())), ); diff --git a/client/chain-spec/src/lib.rs b/client/chain-spec/src/lib.rs index fe815be0dbaae..aba5849fa82da 100644 --- a/client/chain-spec/src/lib.rs +++ b/client/chain-spec/src/lib.rs @@ -120,7 +120,7 @@ use sc_network::config::MultiaddrWithPeerId; use sc_telemetry::TelemetryEndpoints; use serde::{de::DeserializeOwned, Serialize}; use sp_core::storage::Storage; -use sp_runtime::BuildStorage; +use sp_runtime::{BuildStorage, StateVersions}; /// The type of a chain. /// @@ -189,7 +189,7 @@ pub trait ChainSpec: BuildStorage + Send + Sync { fn code_substitutes(&self) -> std::collections::HashMap>; /// State versions for the chain. /// Uses default state version when no definition from block 0. - fn state_versions(&self) -> Vec<(u64, Option>)>; + fn state_versions(&self) -> StateVersions; } impl std::fmt::Debug for dyn ChainSpec { diff --git a/client/db/src/bench.rs b/client/db/src/bench.rs index 00af0ebdc3737..707fcea035a0a 100644 --- a/client/db/src/bench.rs +++ b/client/db/src/bench.rs @@ -82,7 +82,7 @@ impl sp_state_machine::Storage> for StorageDb { root: Cell, - alt_hashing: Cell>>, + state_version: Cell, genesis_root: B::Hash, state: RefCell>>, db: Cell>>, @@ -122,7 +122,7 @@ impl BenchmarkingState { root: Cell::new(root.clone()), // use old state for now to keep similar benches. // will need update to heavier of both world before migrating. - alt_hashing: Cell::new(None), + state_version: Cell::new(None), genesis: Default::default(), genesis_root: Default::default(), record: Default::default(), @@ -173,7 +173,7 @@ impl BenchmarkingState { _block: Default::default(), }); *self.state.borrow_mut() = Some(State::new( - DbState::::new(storage_db, self.root.get(), self.alt_hashing.get()), + DbState::::new(storage_db, self.root.get(), self.state_version.get()), self.shared_cache.clone(), None, )); @@ -618,7 +618,7 @@ impl StateBackend> for BenchmarkingState { fn proof_size(&self) -> Option { self.proof_recorder.as_ref().map(|recorder| { let proof_size = recorder.estimate_encoded_size() as u32; - let proof = recorder.to_storage_proof::>(self.alt_hashing.get()); + let proof = recorder.to_storage_proof::>(self.state_version.get()); let proof_recorder_root = self.proof_recorder_root.get(); if proof_recorder_root == Default::default() || proof_size == 1 { // empty trie @@ -639,8 +639,8 @@ impl StateBackend> for BenchmarkingState { }) } - fn alt_hashing(&self) -> Option> { - self.alt_hashing.get() + fn state_version(&self) -> StateVersion { + self.state_version.get() } } diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index 6ccde328ca3fa..ccfbfa3f8cf21 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -287,8 +287,8 @@ impl StateBackend> for RefTrackingState { self.state.usage_info() } - fn alt_hashing(&self) -> Option> { - self.state.alt_hashing() + fn state_version(&self) -> StateVersion { + self.state.state_version() } } @@ -1118,30 +1118,18 @@ pub struct Backend { io_stats: FrozenForDuration<(kvdb::IoStats, StateUsageInfo)>, state_usage: Arc, genesis_state: RwLock>>>, - alt_hashing: Vec<(NumberFor, Option>)>, // TODO have specific type for history of versions - // Also TODO consider moving this alt_hashing into BlockChainBb + // TODO consider moving this state_version into BlockChainBb + state_versions: StateVersions, } impl Backend { - // TODO util of StateVersionHistory - fn alt_hashing_at(&self, at: &NumberFor) -> Option> { - // Default - let mut result = Some(Some(sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD)); - for (start, alt) in self.alt_hashing.iter() { - if start > at { - break; - } - result = alt.clone(); - } - result - } /// Create a new instance of database backend. /// /// The pruning window is how old a block must be before the state is pruned. - pub fn new(config: DatabaseSettings, canonicalization_delay: u64, alt_hashing: Vec<(NumberFor, Option>)>) -> ClientResult { - // TODO alt_hashing could also be part of database settings + pub fn new(config: DatabaseSettings, canonicalization_delay: u64, state_versions: StateVersions) -> ClientResult { + // TODO state_versions could also be part of database settings let db = crate::utils::open_database::(&config, DatabaseType::Full)?; - Self::from_database(db as Arc<_>, canonicalization_delay, &config, alt_hashing) + Self::from_database(db as Arc<_>, canonicalization_delay, &config, state_versions) } /// Create new memory-backed client backend for tests. @@ -1171,16 +1159,16 @@ impl Backend { keep_blocks: KeepBlocks::Some(keep_blocks), transaction_storage, }; - let alt_hashing = Vec::new(); + let state_versions = Default::default(); - Self::new(db_setting, canonicalization_delay, alt_hashing).expect("failed to create test-db") + Self::new(db_setting, canonicalization_delay, state_versions).expect("failed to create test-db") } fn from_database( db: Arc>, canonicalization_delay: u64, config: &DatabaseSettings, - alt_hashing: Vec<(NumberFor, Option>)>, // TODO could also be part of database settings + state_versions: StateVersions, // TODO could also be part of database settings ) -> ClientResult { let is_archive_pruning = config.state_pruning.is_archive(); let blockchain = BlockchainDb::new(db.clone(), config.transaction_storage.clone())?; @@ -1224,7 +1212,7 @@ impl Backend { keep_blocks: config.keep_blocks.clone(), transaction_storage: config.transaction_storage.clone(), genesis_state: RwLock::new(None), - alt_hashing, + state_versions, }; // Older DB versions have no last state key. Check if the state is available and set it. @@ -1892,18 +1880,9 @@ impl Backend { fn empty_state(&self) -> ClientResult, Block>> { let root = EmptyStorage::::new().0; // Empty trie - // alt_hashing for genesis in empty state. - let mut alt_hashing = Some(Some(sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD)); // Default - // TODO replace by inner search for number - if let Some((first, alt)) = self.alt_hashing.get(0) { - // chain should always define their state - // otherwhise uses latest state, but will need definition - // when default may change. - if first.is_zero() { - alt_hashing = alt.clone(); - } - } - let db_state = DbState::::new(self.storage.clone(), root, alt_hashing); + // state_version for genesis in empty state. + let state_version = self.state_versions.genesis_state_version(); + let db_state = DbState::::new(self.storage.clone(), root, state_version); let state = RefTrackingState::new(db_state, self.storage.clone(), None); let caching_state = CachingState::new(state, self.shared_cache.clone(), None); Ok(SyncingCachingState::new( @@ -2383,8 +2362,8 @@ impl sc_client_api::backend::Backend for Backend { if is_genesis { if let Some(genesis_state) = &*self.genesis_state.read() { let root = genesis_state.root.clone(); - let alt_hashing = self.alt_hashing_at(&number); - let db_state = DbState::::new(genesis_state.clone(), root, alt_hashing); + let state_version = self.state_versions.genesis_state_version(); + let db_state = DbState::::new(genesis_state.clone(), root, state_version); let state = RefTrackingState::new(db_state, self.storage.clone(), None); let caching_state = CachingState::new(state, self.shared_cache.clone(), None); let mut state = SyncingCachingState::new( @@ -2415,8 +2394,8 @@ impl sc_client_api::backend::Backend for Backend { } if let Ok(()) = self.storage.state_db.pin(&hash) { let root = hdr.state_root; - let alt_hashing = self.alt_hashing_at(&number); - let db_state = DbState::::new(self.storage.clone(), root, alt_hashing); + let state_version = self.state_versions.state_version_at(number); + let db_state = DbState::::new(self.storage.clone(), root, state_version); let state = RefTrackingState::new(db_state, self.storage.clone(), Some(hash.clone())); let caching_state = @@ -2619,12 +2598,14 @@ pub(crate) mod tests { } fn set_state_data_inner(alt_hashing: bool) { let state_version = if alt_hashing { - Some(Some(sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD)) + StateVersion::V1 { + threshold: 33, + } } else { - None + StateVersion::V0 }; let mut db = Backend::::new_test(2, 0); - db.alt_hashing.push((0, state_version)); + db.state_versions.add((0, state_version)); let hash = { let mut op = db.begin_operation().unwrap(); let mut header = Header { diff --git a/client/db/src/storage_cache.rs b/client/db/src/storage_cache.rs index f1a23393a6bfa..f4bfcf35ed960 100644 --- a/client/db/src/storage_cache.rs +++ b/client/db/src/storage_cache.rs @@ -25,7 +25,7 @@ use hash_db::Hasher; use linked_hash_map::{Entry, LinkedHashMap}; use log::trace; use parking_lot::{RwLock, RwLockUpgradableReadGuard}; -use sp_core::{hexdisplay::HexDisplay, storage::ChildInfo}; +use sp_core::{hexdisplay::HexDisplay, storage::ChildInfo, state_version::StateVersion}; use sp_runtime::traits::{Block as BlockT, HashFor, Header, NumberFor}; use sp_state_machine::{ backend::Backend as StateBackend, ChildStorageCollection, StorageCollection, StorageKey, @@ -717,8 +717,8 @@ impl>, B: BlockT> StateBackend> for Cachin info } - fn alt_hashing(&self) -> Option> { - self.state.alt_hashing() + fn state_version(&self) -> StateVersion { + self.state.state_version() } } @@ -920,8 +920,8 @@ impl>, B: BlockT> StateBackend> self.caching_state().usage_info() } - fn alt_hashing(&self) -> Option> { - self.caching_state().alt_hashing() + fn state_version(&self) -> StateVersion { + self.caching_state().state_version() } } diff --git a/client/light/src/backend.rs b/client/light/src/backend.rs index 356f0987a50bb..8048eb66f15b5 100644 --- a/client/light/src/backend.rs +++ b/client/light/src/backend.rs @@ -48,7 +48,7 @@ use sp_core::{ use sp_runtime::{ generic::BlockId, traits::{Block as BlockT, HashFor, Header, NumberFor, Zero}, - Justification, Justifications, Storage, + Justification, Justifications, Storage, StateVersions, StateVersion, }; use sp_state_machine::{ Backend as StateBackend, ChangesTrieTransaction, ChildStorageCollection, InMemoryBackend, @@ -62,8 +62,8 @@ const IN_MEMORY_EXPECT_PROOF: &str = pub struct Backend { blockchain: Arc>, genesis_state: RwLock>>>, - alt_hashing: Vec<(NumberFor, Option>)>, // TODO have specific type for history of versions - // Also TODO consider moving this alt_hashing into BlockChain (and add N to type) -> would make + state_versions: StateVersions, + // TODO consider moving this state_versions into BlockChain (and add N to type) -> would make // better api import_lock: RwLock<()>, } @@ -78,7 +78,7 @@ pub struct ImportOperation { set_head: Option>, storage_update: Option>>, changes_trie_config_update: Option>, - alt_hashing: Option>, + state_version: Option, _phantom: std::marker::PhantomData, } @@ -93,8 +93,8 @@ pub enum GenesisOrUnavailableState { impl Backend { /// Create new light backend. - pub fn new(blockchain: Arc>, alt_hashing: Vec<(NumberFor, Option>)>) -> Self { - Self { blockchain, genesis_state: RwLock::new(None), import_lock: Default::default(), alt_hashing } + pub fn new(blockchain: Arc>, state_versions: StateVersions) -> Self { + Self { blockchain, genesis_state: RwLock::new(None), import_lock: Default::default(), state_versions } } /// Get shared blockchain reference. @@ -144,7 +144,7 @@ where set_head: None, storage_update: None, changes_trie_config_update: None, - alt_hashing: None, // TODO this needs to be undefined with versioning (Option). + state_version: None, _phantom: Default::default(), }) } @@ -156,15 +156,7 @@ where ) -> ClientResult<()> { if let Some(number) = self.blockchain.storage().block_number_from_id(&block)? { - let mut version = Some(Some(sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD)); - // TODO from utils of Version history - for (change_state, state) in self.alt_hashing.iter() { - if &number > change_state { - break - } - version = *state; - } - operation.alt_hashing = version; + operation.state_version = self.state_versions.state_version_at(number); } Ok(()) } @@ -372,7 +364,7 @@ where storage.insert(Some(storage_child.child_info), storage_child.data); } - let storage_update = InMemoryBackend::from((storage, self.alt_hashing.clone())); + let storage_update = InMemoryBackend::from((storage, self.state_version)); let (storage_root, _) = storage_update.full_storage_root(std::iter::empty(), child_delta); if commit { self.storage_update = Some(storage_update); @@ -593,10 +585,10 @@ where } } - fn alt_hashing(&self) -> Option> { + fn state_version(&self) -> StateVersion { match self { - GenesisOrUnavailableState::Genesis(state) => state.force_alt_hashing.clone(), - GenesisOrUnavailableState::Unavailable => Some(Some(sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD)), + GenesisOrUnavailableState::Genesis(state) => state.state_version(), + GenesisOrUnavailableState::Unavailable => StateVersion::Default(), } } } diff --git a/client/light/src/lib.rs b/client/light/src/lib.rs index 2a59be7e6b08f..10a87494a9b9a 100644 --- a/client/light/src/lib.rs +++ b/client/light/src/lib.rs @@ -50,10 +50,10 @@ pub fn new_light_blockchain>(storage: S) -> A } /// Create an instance of light client backend. -pub fn new_light_backend(blockchain: Arc>, alt_hashing: Vec<(NumberFor, Option>)>) -> Arc> +pub fn new_light_backend(blockchain: Arc>, state_versions: sp_runtime::StateVersions -> Arc> where B: BlockT, S: BlockchainStorage, { - Arc::new(Backend::new(blockchain, alt_hashing)) + Arc::new(Backend::new(blockchain, state_versions)) } diff --git a/client/service/src/builder.rs b/client/service/src/builder.rs index 819f24e596097..2d9f0a204ec2b 100644 --- a/client/service/src/builder.rs +++ b/client/service/src/builder.rs @@ -60,6 +60,7 @@ use sp_runtime::{ generic::BlockId, traits::{Block as BlockT, BlockIdTo, Zero, NumberFor}, BuildStorage, + StateVersions, }; use sp_utils::mpsc::{tracing_unbounded, TracingUnboundedSender}; use std::{str::FromStr, sync::Arc}; @@ -317,16 +318,10 @@ where transaction_storage: config.transaction_storage.clone(), }; - let mut state_versions = Vec::new(); - for (number, version) in config.chain_spec.state_versions().into_iter() { - let number = NumberFor::::from_str(&number.to_string()) - .map_err(|_| Error::Application(Box::from(format!( - "Failed to parse `{}` as block number for state versions.", - number, - ))))?; - state_versions.push((number.into(), version)); - } - let backend = new_db_backend(db_config, state_versions)?; + let state_versions = config.chain_spec.state_versions() + .ok_or_else(|| Error::Application(Box::from("Invalid state versions for chain spec".to_string())))?; + + let backend = new_db_backend(db_config, state_versions.clone())?; let extensions = sc_client_api::execution_extensions::ExecutionExtensions::new( config.execution_strategies.clone(), @@ -349,16 +344,6 @@ where }) .collect::, Error>>()?; - let mut state_versions = Vec::new(); - for (number, version) in config.chain_spec.state_versions().into_iter() { - let number = NumberFor::::from_str(&number.to_string()) - .map_err(|_| Error::Application(Box::from(format!( - "Failed to parse `{}` as block number for state versions.", - number, - ))))?; - state_versions.push((number.into(), version)); - } - let client = new_client( backend.clone(), executor, @@ -427,15 +412,8 @@ where Box::new(task_manager.spawn_handle()), )); let on_demand = Arc::new(sc_network::config::OnDemand::new(fetch_checker)); - let mut state_versions = Vec::new(); - for (number, version) in config.chain_spec.state_versions().into_iter() { - let number = NumberFor::::from_str(&number.to_string()) - .map_err(|_| Error::Application(Box::from(format!( - "Failed to parse `{}` as block number for state versions.", - number, - ))))?; - state_versions.push((number.into(), version)); - } + let state_versions = config.chain_spec.state_versions() + .ok_or_else(|| Error::Application(Box::from("Invalid state versions for chain spec".to_string())))?; let backend = sc_light::new_light_backend(light_blockchain, state_versions); let client = Arc::new(light::new_light( @@ -453,7 +431,7 @@ where /// Create an instance of default DB-backend backend. pub fn new_db_backend( settings: DatabaseSettings, - state_versions: Vec<(NumberFor, Option>)>, + state_versions: StateVersions, ) -> Result>, sp_blockchain::Error> where Block: BlockT, diff --git a/frame/balances/src/lib.rs b/frame/balances/src/lib.rs index b5fbc894ca413..9f7148f343ef5 100644 --- a/frame/balances/src/lib.rs +++ b/frame/balances/src/lib.rs @@ -183,6 +183,7 @@ use sp_runtime::{ Saturating, StaticLookup, Zero, }, ArithmeticError, DispatchError, DispatchResult, RuntimeDebug, + StateVersion, }; use sp_std::{cmp, fmt::Debug, mem, ops::BitOr, prelude::*, result}; pub use weights::WeightInfo; @@ -586,15 +587,15 @@ impl, I: 'static> GenesisConfig { /// Direct implementation of `GenesisBuild::build_storage`. /// /// Kept in order not to break dependency. - pub fn build_storage(&self, alt_hashing: Option>) -> Result { - >::build_storage(self, alt_hashing) + pub fn build_storage(&self, state_version: StateVersion) -> Result { + >::build_storage(self, state_version) } /// Direct implementation of `GenesisBuild::assimilate_storage`. /// /// Kept in order not to break dependency. - pub fn assimilate_storage(&self, storage: &mut sp_runtime::Storage, alt_hashing: Option>) -> Result<(), String> { - >::assimilate_storage(self, storage, alt_hashing) + pub fn assimilate_storage(&self, storage: &mut sp_runtime::Storage, state_version: StateVersion) -> Result<(), String> { + >::assimilate_storage(self, storage, state_version) } } diff --git a/frame/support/procedural/src/construct_runtime/expand/config.rs b/frame/support/procedural/src/construct_runtime/expand/config.rs index 4d291691709f7..f57c7cee6f72c 100644 --- a/frame/support/procedural/src/construct_runtime/expand/config.rs +++ b/frame/support/procedural/src/construct_runtime/expand/config.rs @@ -78,11 +78,11 @@ pub fn expand_outer_config( fn assimilate_storage( &self, storage: &mut #scrate::sp_runtime::Storage, - alt_hashing: Option>, + state_version: #scrate::sp_runtime::StateVersion, ) -> std::result::Result<(), String> { #build_storage_calls - #scrate::BasicExternalities::execute_with_storage(storage, alt_hashing, || { + #scrate::BasicExternalities::execute_with_storage(storage, state_version, || { ::on_genesis(); }); @@ -131,6 +131,6 @@ fn expand_config_build_storage_call( quote! { #scrate::sp_runtime::BuildModuleGenesisStorage:: - <#runtime, #instance>::build_module_genesis_storage(&self.#field_name, storage, alt_hashing)?; + <#runtime, #instance>::build_module_genesis_storage(&self.#field_name, storage, state_version)?; } } diff --git a/frame/support/procedural/src/pallet/expand/genesis_build.rs b/frame/support/procedural/src/pallet/expand/genesis_build.rs index 177a1206ca962..d9d1d66537a2d 100644 --- a/frame/support/procedural/src/pallet/expand/genesis_build.rs +++ b/frame/support/procedural/src/pallet/expand/genesis_build.rs @@ -61,9 +61,10 @@ pub fn expand_genesis_build(def: &mut Def) -> proc_macro2::TokenStream { fn build_module_genesis_storage( &self, storage: &mut #frame_support::sp_runtime::Storage, - alt_hashing: Option>, + state_version: #frame_support::sp_runtime::StateVersion, + ) -> std::result::Result<(), std::string::String> { - #frame_support::BasicExternalities::execute_with_storage(storage, alt_hashing, || { + #frame_support::BasicExternalities::execute_with_storage(storage, state_version, || { >::build(self); Ok(()) }) diff --git a/frame/support/procedural/src/storage/genesis_config/mod.rs b/frame/support/procedural/src/storage/genesis_config/mod.rs index 4d02bcf162bdc..a78adfbc18639 100644 --- a/frame/support/procedural/src/storage/genesis_config/mod.rs +++ b/frame/support/procedural/src/storage/genesis_config/mod.rs @@ -139,12 +139,12 @@ fn impl_build_storage( #[cfg(feature = "std")] impl#genesis_impl GenesisConfig#genesis_struct #genesis_where_clause { /// Build the storage for this module. - pub fn build_storage #fn_generic (&self, alt_hashing: Option>) -> std::result::Result< + pub fn build_storage #fn_generic (&self, state_version: #scrate::sp_runtime::StateVersion) -> std::result::Result< #scrate::sp_runtime::Storage, String > #fn_where_clause { let mut storage = Default::default(); - self.assimilate_storage::<#fn_traitinstance>(&mut storage, alt_hashing)?; + self.assimilate_storage::<#fn_traitinstance>(&mut storage, state_version)?; Ok(storage) } @@ -152,9 +152,9 @@ fn impl_build_storage( pub fn assimilate_storage #fn_generic ( &self, storage: &mut #scrate::sp_runtime::Storage, - alt_hashing: Option>, + state_version: #scrate::sp_runtime::StateVersion, ) -> std::result::Result<(), String> #fn_where_clause { - #scrate::BasicExternalities::execute_with_storage(storage, alt_hashing, || { + #scrate::BasicExternalities::execute_with_storage(storage, state_version, || { #( #builder_blocks )* Ok(()) }) @@ -168,9 +168,9 @@ fn impl_build_storage( fn build_module_genesis_storage( &self, storage: &mut #scrate::sp_runtime::Storage, - alt_hashing: Option>, + state_version: #scrate::sp_runtime::StateVersion, ) -> std::result::Result<(), String> { - self.assimilate_storage::<#fn_traitinstance> (storage, alt_hashing) + self.assimilate_storage::<#fn_traitinstance> (storage, state_version) } } } diff --git a/frame/support/src/traits/hooks.rs b/frame/support/src/traits/hooks.rs index 61324295b36cf..d43463f18edd9 100644 --- a/frame/support/src/traits/hooks.rs +++ b/frame/support/src/traits/hooks.rs @@ -20,6 +20,7 @@ use impl_trait_for_tuples::impl_for_tuples; use sp_arithmetic::traits::Saturating; use sp_runtime::traits::{AtLeast32BitUnsigned, MaybeSerializeDeserialize}; +use sp_runtime::StateVersion; /// The block initialization trait. /// @@ -307,15 +308,15 @@ pub trait GenesisBuild: Default + MaybeSerializeDeserialize { fn build(&self); /// Build the storage using `build` inside default storage. - fn build_storage(&self, alt_hashing: Option>) -> Result { + fn build_storage(&self, state_version: StateVersion) -> Result { let mut storage = Default::default(); - self.assimilate_storage(&mut storage, alt_hashing)?; + self.assimilate_storage(&mut storage, state_version)?; Ok(storage) } /// Assimilate the storage for this module into pre-existing overlays. - fn assimilate_storage(&self, storage: &mut sp_runtime::Storage, alt_hashing: Option>) -> Result<(), String> { - sp_state_machine::BasicExternalities::execute_with_storage(storage, alt_hashing, || { + fn assimilate_storage(&self, storage: &mut sp_runtime::Storage, state_version: StateVersion) -> Result<(), String> { + sp_state_machine::BasicExternalities::execute_with_storage(storage, state_version, || { self.build(); Ok(()) }) diff --git a/frame/system/src/lib.rs b/frame/system/src/lib.rs index fffb83d93e6b1..1db3a43d03b5d 100644 --- a/frame/system/src/lib.rs +++ b/frame/system/src/lib.rs @@ -733,8 +733,8 @@ impl GenesisConfig { /// Direct implementation of `GenesisBuild::build_storage`. /// /// Kept in order not to break dependency. - pub fn build_storage(&self, alt_hashing: Option>) -> Result { - >::build_storage(self, alt_hashing) + pub fn build_storage(&self, state_version: sp_runtime::StateVersion) -> Result { + >::build_storage(self, state_version) } /// Direct implementation of `GenesisBuild::assimilate_storage`. @@ -743,9 +743,9 @@ impl GenesisConfig { pub fn assimilate_storage( &self, storage: &mut sp_runtime::Storage, - alt_hashing: Option>, + state_version: sp_runtime::StateVersion, ) -> Result<(), String> { - >::assimilate_storage(self, storage, alt_hashing) + >::assimilate_storage(self, storage, state_version) } } diff --git a/frame/treasury/src/lib.rs b/frame/treasury/src/lib.rs index afca0dfb9f5cc..88fb0a52a0db1 100644 --- a/frame/treasury/src/lib.rs +++ b/frame/treasury/src/lib.rs @@ -234,9 +234,9 @@ pub mod pallet { pub fn assimilate_storage, I: 'static>( &self, storage: &mut sp_runtime::Storage, - alt_hashing: Option>, + state_version: sp_runtime::StateVersion, ) -> Result<(), String> { - >::assimilate_storage(self, storage, alt_hashing) + >::assimilate_storage(self, storage, state_version) } } diff --git a/primitives/api/proc-macro/src/impl_runtime_apis.rs b/primitives/api/proc-macro/src/impl_runtime_apis.rs index 73b86b33639e8..6cbfd1458f8a6 100644 --- a/primitives/api/proc-macro/src/impl_runtime_apis.rs +++ b/primitives/api/proc-macro/src/impl_runtime_apis.rs @@ -279,10 +279,10 @@ fn generate_runtime_api_base_structures() -> Result { self.recorder.clone() } - fn extract_proof(&mut self, alt_hashing: Option>) -> Option<#crate_::StorageProof> { + fn extract_proof(&mut self, state_version: sp_runtime::StateVersion) -> Option<#crate_::StorageProof> { self.recorder .take() - .map(|recorder| recorder.to_storage_proof::<#crate_::HashFor>(alt_hashing)) + .map(|recorder| recorder.to_storage_proof::<#crate_::HashFor>(state_version)) } fn into_storage_changes( diff --git a/primitives/api/proc-macro/src/mock_impl_runtime_apis.rs b/primitives/api/proc-macro/src/mock_impl_runtime_apis.rs index 58c10525efbe1..140f0eb709af7 100644 --- a/primitives/api/proc-macro/src/mock_impl_runtime_apis.rs +++ b/primitives/api/proc-macro/src/mock_impl_runtime_apis.rs @@ -105,7 +105,7 @@ fn implement_common_api_traits(block_type: TypePath, self_ty: Type) -> Result>) -> Option<#crate_::StorageProof> { + fn extract_proof(&mut self, state_version: sp_runtime::StateVersion) -> Option<#crate_::StorageProof> { unimplemented!("`extract_proof` not implemented for runtime api mocks") } diff --git a/primitives/api/src/lib.rs b/primitives/api/src/lib.rs index 89da6085db8bc..7f164454b83c0 100644 --- a/primitives/api/src/lib.rs +++ b/primitives/api/src/lib.rs @@ -502,7 +502,7 @@ pub trait ApiExt { /// This stops the proof recording. /// /// If `record_proof` was not called before, this will return `None`. - fn extract_proof(&mut self, alt_hashing: Option>) -> Option; + fn extract_proof(&mut self, state_version: sp_runtime::StateVersion) -> Option; /// Returns the current active proof recorder. fn proof_recorder(&self) -> Option>; diff --git a/primitives/core/src/state_version.rs b/primitives/core/src/state_version.rs index 70c59ffaa86b0..b837eaf2704d9 100644 --- a/primitives/core/src/state_version.rs +++ b/primitives/core/src/state_version.rs @@ -26,7 +26,7 @@ pub const DEFAULT_STATE_VERSION: StateVersion = StateVersion::V1 { threshold: 33 }; /// Supported version with substrate chain. -#[derive(Clone, Copy)] +#[derive(Clone, Copy, Encode, Decode)] #[cfg_attr(feature = "std", derive(serde::Serialize, serde::Deserialize))] pub enum StateVersion { /// Patricia trie Radix 16 without extension node. diff --git a/primitives/runtime/src/lib.rs b/primitives/runtime/src/lib.rs index 9918668763e5e..cfd3aeb6bcd3e 100644 --- a/primitives/runtime/src/lib.rs +++ b/primitives/runtime/src/lib.rs @@ -94,6 +94,8 @@ pub use sp_arithmetic::{ pub use either::Either; +pub use state_version::{StateVersion, StateVersions}; + /// An abstraction over justification for a block's validity under a consensus algorithm. /// /// Essentially a finality proof. The exact formulation will vary between consensus @@ -171,13 +173,13 @@ pub use serde::{de::DeserializeOwned, Deserialize, Serialize}; #[cfg(feature = "std")] pub trait BuildStorage { /// Build the storage out of this builder. - fn build_storage(&self, alt_hashing: Option>) -> Result { + fn build_storage(&self, state_version: StateVersion) -> Result { let mut storage = Default::default(); - self.assimilate_storage(&mut storage, alt_hashing)?; + self.assimilate_storage(&mut storage, state_version)?; Ok(storage) } /// Assimilate the storage for this module into pre-existing overlays. - fn assimilate_storage(&self, storage: &mut sp_core::storage::Storage, alt_hashing: Option>) -> Result<(), String>; + fn assimilate_storage(&self, storage: &mut sp_core::storage::Storage, state_version: StateVersion) -> Result<(), String>; } /// Something that can build the genesis storage of a module. @@ -187,13 +189,13 @@ pub trait BuildModuleGenesisStorage: Sized { fn build_module_genesis_storage( &self, storage: &mut sp_core::storage::Storage, - alt_hashing: Option>, + state_version: StateVersion, ) -> Result<(), String>; } #[cfg(feature = "std")] impl BuildStorage for sp_core::storage::Storage { - fn assimilate_storage(&self, storage: &mut sp_core::storage::Storage, _alt_hashing: Option>) -> Result<(), String> { + fn assimilate_storage(&self, storage: &mut sp_core::storage::Storage, _state_version: StateVersion) -> Result<(), String> { storage.top.extend(self.top.iter().map(|(k, v)| (k.clone(), v.clone()))); for (k, other_map) in self.children_default.iter() { let k = k.clone(); @@ -212,7 +214,7 @@ impl BuildStorage for sp_core::storage::Storage { #[cfg(feature = "std")] impl BuildStorage for () { - fn assimilate_storage(&self, _: &mut sp_core::storage::Storage, _: Option>) -> Result<(), String> { + fn assimilate_storage(&self, _: &mut sp_core::storage::Storage, _: StateVersion) -> Result<(), String> { Err("`assimilate_storage` not implemented for `()`".into()) } } diff --git a/primitives/runtime/src/state_version.rs b/primitives/runtime/src/state_version.rs index 6c035df79acdc..8b3fd0d1453de 100644 --- a/primitives/runtime/src/state_version.rs +++ b/primitives/runtime/src/state_version.rs @@ -24,7 +24,7 @@ use sp_std::vec::Vec; use sp_arithmetic::traits::Zero; /// Multiple versions of state in use for a chain. -#[derive(Clone)] +#[derive(Clone, Default)] pub struct StateVersions { canonical_states: Vec<(NumberFor, StateVersion)>, } @@ -54,6 +54,27 @@ impl StateVersions { version } + /// Modify configuration, mostly for testing. + pub fn add(&mut self, (at, conf): (NumberFor, StateVersion)) -> StateVersion { + let mut insert = Some(0); + let mut replace = None; + for (i, (number, _)) in self.canonical_states.iter() { + if number == &at { + replace = Some(i); + break; + } + if number > &at { + break; + } + insert = i + 1; + } + if let Some(i) = replace { + self.canonical_states[i] = (at, conf); + } else if let Some(i) = insert { + self.canonical_states.insert(i, (at, conf)); + } + } + /// Convert from chainspec conf. pub fn from_conf<'a, I>(conf: I) -> Option where I: IntoIterator, diff --git a/primitives/state-machine/src/backend.rs b/primitives/state-machine/src/backend.rs index 36c400bd3312c..a7e618a46ea08 100644 --- a/primitives/state-machine/src/backend.rs +++ b/primitives/state-machine/src/backend.rs @@ -136,8 +136,6 @@ pub trait Backend: sp_std::fmt::Debug { /// Calculate the storage root, with given delta over what is already stored in /// the backend, and produce a "transaction" that can be used to commit. - /// `alt_hashing` indicate if trie state should apply alternate hashing - /// scheme (inner value hashed). /// Does not include child storage updates. fn storage_root<'a>( &self, @@ -269,7 +267,7 @@ pub trait Backend: sp_std::fmt::Debug { } /// Get current state version in use. - fn alt_hashing(&self) -> Option>; + fn state_version(&self) -> sp_core::state_version::StateVersion; } /// Trait that allows consolidate two transactions together. diff --git a/primitives/state-machine/src/basic.rs b/primitives/state-machine/src/basic.rs index b829973765f1e..3075bf67ba4d2 100644 --- a/primitives/state-machine/src/basic.rs +++ b/primitives/state-machine/src/basic.rs @@ -27,6 +27,7 @@ use sp_core::{ }, traits::Externalities, Blake2Hasher, + state_version::StateVersion, }; use sp_externalities::{Extension, Extensions}; use sp_trie::{empty_child_trie_root, Layout, TrieConfiguration}; @@ -41,18 +42,18 @@ use std::{ pub struct BasicExternalities { inner: Storage, extensions: Extensions, - alt_hashing: Option>, // TODO same alt hashing type as all + state_version: StateVersion, // TODO same alt hashing type as all } impl BasicExternalities { /// Create a new instance of `BasicExternalities` - pub fn new(inner: Storage, alt_hashing: Option>) -> Self { - BasicExternalities { inner, extensions: Default::default(), alt_hashing } + pub fn new(inner: Storage, state_version: StateVersion) -> Self { + BasicExternalities { inner, extensions: Default::default(), state_version } } /// New basic externalities with empty storage. - pub fn new_empty(alt_hashing: Option>) -> Self { - Self::new(Storage::default(), alt_hashing) + pub fn new_empty(state_version: StateVersion) -> Self { + Self::new(Storage::default(), state_version) } /// Insert key/value @@ -70,7 +71,7 @@ impl BasicExternalities { /// Returns the result of the closure and updates `storage` with all changes. pub fn execute_with_storage( storage: &mut sp_core::storage::Storage, - alt_hashing: Option>, + state_version: StateVersion, f: impl FnOnce() -> R, ) -> R { let mut ext = Self { @@ -79,7 +80,7 @@ impl BasicExternalities { children_default: std::mem::take(&mut storage.children_default), }, extensions: Default::default(), - alt_hashing, + state_version, }; let r = ext.execute_with(f); @@ -114,28 +115,18 @@ impl PartialEq for BasicExternalities { } } -/* -impl FromIterator<(StorageKey, StorageValue)> for BasicExternalities { - fn from_iter>(iter: I) -> Self { - let mut t = Self::default(); - t.inner.top.extend(iter); - t +impl From for BasicExternalities { + fn from(state_version: StateVersion) -> Self { + Self::new(Default::default(), state_version) } } -*/ -impl From>> for BasicExternalities { - fn from(alt_hashing: Option>) -> Self { - Self::new(Default::default(), alt_hashing) - } -} - -impl From<(BTreeMap, Option>)> for BasicExternalities { - fn from((hashmap, alt_hashing): (BTreeMap, Option>)) -> Self { +impl From<(BTreeMap, StateVersion)> for BasicExternalities { + fn from((hashmap, state_version): (BTreeMap, StateVersion)) -> Self { BasicExternalities { inner: Storage { top: hashmap, children_default: Default::default() }, extensions: Default::default(), - alt_hashing, + state_version, } } } @@ -299,10 +290,13 @@ impl Externalities for BasicExternalities { } } - let layout = if let Some(Some(threshold)) = self.alt_hashing.as_ref() { - Layout::::with_alt_hashing(*threshold) - } else { - Layout::::default() + let layout = match self.state_version { + StateVersion::V0 => { + Layout::::default() + }, + StateVersion::V1 { threshold } => { + Layout::::with_alt_hashing(*threshold) + }, }; layout.trie_root(self.inner.top.clone()).as_ref().into() } @@ -310,7 +304,7 @@ impl Externalities for BasicExternalities { fn child_storage_root(&mut self, child_info: &ChildInfo) -> Vec { if let Some(child) = self.inner.children_default.get(child_info.storage_key()) { let delta = child.data.iter().map(|(k, v)| (k.as_ref(), Some(v.as_ref()))); - let in_mem = crate::in_memory_backend::new_in_mem::(self.alt_hashing.clone()); + let in_mem = crate::in_memory_backend::new_in_mem::(self.state_version.clone()); in_mem.child_storage_root(&child.child_info, delta).0 } else { empty_child_trie_root::>() diff --git a/primitives/state-machine/src/in_memory_backend.rs b/primitives/state-machine/src/in_memory_backend.rs index 6ec0dbdf8f25c..eb0f04f137ad0 100644 --- a/primitives/state-machine/src/in_memory_backend.rs +++ b/primitives/state-machine/src/in_memory_backend.rs @@ -23,16 +23,17 @@ use crate::{ use codec::Codec; use hash_db::Hasher; use sp_core::storage::{ChildInfo, Storage}; +use sp_core::state_version::StateVersion; use sp_trie::{empty_trie_root, Layout, MemoryDB}; use std::collections::{BTreeMap, HashMap}; /// Create a new empty instance of in-memory backend. -pub fn new_in_mem(alt_hashing: Option>) -> TrieBackend, H> +pub fn new_in_mem(state_version: StateVersion) -> TrieBackend, H> where H::Out: Codec + Ord, { let db = MemoryDB::default(); - TrieBackend::new(db, empty_trie_root::>(), alt_hashing) + TrieBackend::new(db, empty_trie_root::>(), state_version) } impl TrieBackend, H> @@ -69,7 +70,7 @@ where pub fn update_backend(&self, root: H::Out, changes: MemoryDB) -> Self { let mut clone = self.backend_storage().clone(); clone.consolidate(changes); - Self::new(clone, root, self.force_alt_hashing.clone()) + Self::new(clone, root, self.state_version) } /// Apply the given transaction to this backend and set the root to the given value. @@ -82,11 +83,6 @@ where pub fn eq(&self, other: &Self) -> bool { self.root() == other.root() } - - /// Setting a alternate hashing threshold at start. TODO remove (on construct). - pub fn force_alt_hashing(&mut self, threshold: Option) { - self.force_alt_hashing = Some(threshold); - } } impl Clone for TrieBackend, H> @@ -94,7 +90,7 @@ where H::Out: Codec + Ord, { fn clone(&self) -> Self { - TrieBackend::new(self.backend_storage().clone(), self.root().clone(), self.force_alt_hashing.clone()) + TrieBackend::new(self.backend_storage().clone(), self.root().clone(), self.state_version) } } @@ -109,23 +105,23 @@ where } */ -impl From>> +impl From for TrieBackend, H> where H::Out: Codec + Ord, { - fn from(alt_hashing: Option>) -> Self { - new_in_mem(alt_hashing) + fn from(state_version: StateVersion) -> Self { + new_in_mem(state_version) } } -impl From<(HashMap, BTreeMap>, Option>)> +impl From<(HashMap, BTreeMap>, StateVersion)> for TrieBackend, H> where H::Out: Codec + Ord, { - fn from((inner, alt_hashing): (HashMap, BTreeMap>, Option>)) -> Self { - let mut backend = new_in_mem(alt_hashing); + fn from((inner, state_version): (HashMap, BTreeMap>, StateVersion)) -> Self { + let mut backend = new_in_mem(state_version); backend.insert( inner .into_iter() @@ -135,37 +131,37 @@ where } } -impl From<(Storage, Option>)> for TrieBackend, H> +impl From<(Storage, StateVersion)> for TrieBackend, H> where H::Out: Codec + Ord, { - fn from((inners, alt_hashing): (Storage, Option>)) -> Self { + fn from((inners, state_version): (Storage, StateVersion)) -> Self { let mut inner: HashMap, BTreeMap> = inners .children_default .into_iter() .map(|(_k, c)| (Some(c.child_info), c.data)) .collect(); inner.insert(None, inners.top); - (inner, alt_hashing).into() + (inner, state_version).into() } } -impl From<(BTreeMap, Option>)> for TrieBackend, H> +impl From<(BTreeMap, StateVersion)> for TrieBackend, H> where H::Out: Codec + Ord, { - fn from((inner, alt_hashing): (BTreeMap, Option>)) -> Self { + fn from((inner, state_version): (BTreeMap, StateVersion)) -> Self { let mut expanded = HashMap::new(); expanded.insert(None, inner); - (expanded, alt_hashing).into() + (expanded, state_version).into() } } -impl From<(Vec<(Option, StorageCollection)>, Option>)> for TrieBackend, H> +impl From<(Vec<(Option, StorageCollection)>, StateVersion)> for TrieBackend, H> where H::Out: Codec + Ord, { - fn from((inner, alt_hashing): (Vec<(Option, StorageCollection)>, Option>)) -> Self { + fn from((inner, state_version): (Vec<(Option, StorageCollection)>, StateVersion)) -> Self { let mut expanded: HashMap, BTreeMap> = HashMap::new(); for (child_info, key_values) in inner { @@ -176,7 +172,7 @@ where } } } - (expanded, alt_hashing).into() + (expanded, state_version).into() } } diff --git a/primitives/state-machine/src/proving_backend.rs b/primitives/state-machine/src/proving_backend.rs index 5648fa2acf0b9..0b8e46a875ebb 100644 --- a/primitives/state-machine/src/proving_backend.rs +++ b/primitives/state-machine/src/proving_backend.rs @@ -27,6 +27,7 @@ use hash_db::{HashDB, Hasher, Prefix, EMPTY_PREFIX}; use log::debug; use parking_lot::RwLock; use sp_core::storage::ChildInfo; +use sp_core::state_version::StateVersion; pub use sp_trie::trie_types::TrieError; use sp_trie::{ empty_child_trie_root, read_child_trie_value_with, read_trie_value_with, record_all_keys, @@ -172,8 +173,12 @@ impl ProofRecorder { } /// Convert into a [`StorageProof`]. - pub fn to_storage_proof(&self, alt_hashing: Option>) -> StorageProof { - // TODO consider alt_hashing as inner field + pub fn to_storage_proof(&self, state_version: StateVersion) -> StorageProof { + // TODO consider state_version as inner field + let try_inner_hashing = match proof.state_version { + StateVersion::V0 => None, + StateVersion::V1 { threshold } => Some(threshold), + }; let trie_nodes = self .inner .read() @@ -182,6 +187,7 @@ impl ProofRecorder { .filter_map(|(_k, v)| { v.as_ref().map(|v| { let mut meta = v.1.clone(); + meta.try_inner_hashing = try_inner_hashing.clone(); if let Some(hashed) = sp_trie::to_hashed_variant::(v.0.as_slice(), &mut meta, v.2) { @@ -193,7 +199,7 @@ impl ProofRecorder { }) .collect(); - StorageProof::new(trie_nodes, alt_hashing) + StorageProof::new(trie_nodes, state_version) } /// Reset the internal state. @@ -234,13 +240,13 @@ where let essence = backend.essence(); let root = essence.root().clone(); let recorder = ProofRecorderBackend { backend: essence.backend_storage(), proof_recorder }; - ProvingBackend(TrieBackend::new(recorder, root, backend.force_alt_hashing.clone())) + ProvingBackend(TrieBackend::new(recorder, root, backend.state_version)) } /// Extracting the gathered unordered proof. pub fn extract_proof(&self) -> StorageProof { - let alt_hashing = self.0.force_alt_hashing.clone(); - self.0.essence().backend_storage().proof_recorder.to_storage_proof::(alt_hashing) + let state_version = self.0.state_version; + self.0.essence().backend_storage().proof_recorder.to_storage_proof::(state_version) } /// Returns the estimated encoded size of the proof. @@ -390,8 +396,8 @@ where self.0.usage_info() } - fn alt_hashing(&self) -> Option> { - self.0.force_alt_hashing.clone() + fn state_version(&self) -> StateVersion { + self.0.state_version } } @@ -404,11 +410,11 @@ where H: Hasher, H::Out: Codec, { - let alt_hashing = proof.alt_hashing.clone(); + let state_version = proof.state_version.clone(); let db = proof.into_memory_db(); if db.contains(&root, EMPTY_PREFIX) { - Ok(TrieBackend::new(db, root, alt_hashing)) + Ok(TrieBackend::new(db, root, state_version)) } else { Err(Box::new(ExecutionError::InvalidProof)) } diff --git a/primitives/state-machine/src/testing.rs b/primitives/state-machine/src/testing.rs index 2c11eec9341bb..0ac1c8c1d9166 100644 --- a/primitives/state-machine/src/testing.rs +++ b/primitives/state-machine/src/testing.rs @@ -42,6 +42,7 @@ use sp_core::{ }, testing::TaskExecutor, traits::TaskExecutorExt, + state_version::StateVersion, }; use sp_externalities::{Extension, ExtensionStore, Extensions}; @@ -86,17 +87,17 @@ where } /// Create a new instance of `TestExternalities` with storage. - pub fn new(storage: Storage, alt_hashing: Option>) -> Self { - Self::new_with_code(&[], storage, alt_hashing) + pub fn new(storage: Storage, state_version: StateVersion) -> Self { + Self::new_with_code(&[], storage, state_version) } /// New empty test externalities. - pub fn new_empty(alt_hashing: Option>) -> Self { - Self::new_with_code(&[], Storage::default(), alt_hashing) + pub fn new_empty(state_version: StateVersion) -> Self { + Self::new_with_code(&[], Storage::default(), state_version) } /// Create a new instance of `TestExternalities` with code and storage. - pub fn new_with_code(code: &[u8], mut storage: Storage, alt_hashing: Option>) -> Self { + pub fn new_with_code(code: &[u8], mut storage: Storage, state_version: StateVersion) -> Self { let mut overlay = OverlayedChanges::default(); let changes_trie_config = storage .top @@ -114,7 +115,7 @@ where let offchain_db = TestPersistentOffchainDB::new(); - let backend = (storage, alt_hashing).into(); + let backend = (storage, state_version).into(); TestExternalities { overlay, @@ -248,12 +249,12 @@ where } } -impl From<(Storage, Option>)> for TestExternalities +impl From<(Storage, StateVersion)> for TestExternalities where H::Out: Ord + 'static + codec::Codec, { - fn from((storage, alt_hashing): (Storage, Option>)) -> Self { - Self::new(storage, alt_hashing) + fn from((storage, state_version): (Storage, StateVersion)) -> Self { + Self::new(storage, state_version) } } diff --git a/primitives/state-machine/src/trie_backend.rs b/primitives/state-machine/src/trie_backend.rs index ee5ca114ddc64..1777878e49d22 100644 --- a/primitives/state-machine/src/trie_backend.rs +++ b/primitives/state-machine/src/trie_backend.rs @@ -25,6 +25,7 @@ use crate::{ use codec::{Codec, Decode}; use hash_db::Hasher; use sp_core::storage::{ChildInfo, ChildType}; +use sp_core::state_version::StateVersion; use sp_std::{boxed::Box, vec::Vec}; use sp_trie::{ child_delta_trie_root, delta_trie_root, empty_child_trie_root, @@ -35,11 +36,7 @@ use sp_trie::{ /// Patricia trie-based backend. Transaction type is an overlay of changes to commit. pub struct TrieBackend, H: Hasher> { pub(crate) essence: TrieBackendEssence, - /// TODO remove pub - // Allows setting alt hashing at start for testing only - // (mainly for in_memory_backend when it cannot read it from - // state). TODO rename alt_hashing TODO accessor and no public. - pub force_alt_hashing: Option>, + state_version: StateVersion, } impl, H: Hasher> TrieBackend @@ -47,8 +44,8 @@ where H::Out: Codec, { /// Create new trie-based backend. - pub fn new(storage: S, root: H::Out, alt_hashing: Option>) -> Self { - TrieBackend { essence: TrieBackendEssence::new(storage, root), force_alt_hashing: alt_hashing } + pub fn new(storage: S, root: H::Out, state_version: StateVersion) -> Self { + TrieBackend { essence: TrieBackendEssence::new(storage, root), state_version: state_version } } /// Get backend essence reference. @@ -206,10 +203,13 @@ where { let mut eph = Ephemeral::new(self.essence.backend_storage(), &mut write_overlay); let res = || { - let layout = if let Some(Some(threshold)) = self.force_alt_hashing.clone() { - sp_trie::Layout::with_alt_hashing(threshold) - } else { - sp_trie::Layout::default() + let layout = match self.state_version { + StateVersion::V0 => { + sp_trie::Layout::default() + }, + StateVersion::V1{ threshold } => { + sp_trie::Layout::with_alt_hashing(threshold) + }, }; delta_trie_root::, _, _, _, _, _>(&mut eph, root, delta, layout) }; @@ -234,10 +234,13 @@ where let default_root = match child_info.child_type() { ChildType::ParentKeyId => empty_child_trie_root::>(), }; - let layout = if let Some(Some(threshold)) = self.force_alt_hashing.clone() { - sp_trie::Layout::with_alt_hashing(threshold) - } else { - sp_trie::Layout::default() + let layout = match self.state_version { + StateVersion::V0 => { + sp_trie::Layout::default() + }, + StateVersion::V1{ threshold } => { + sp_trie::Layout::with_alt_hashing(threshold) + }, }; let mut write_overlay = S::Overlay::default(); @@ -286,8 +289,8 @@ where Ok(()) } - fn alt_hashing(&self) -> Option> { - self.force_alt_hashing.clone() + fn state_version(&self) -> StateVersion { + self.state_version.clone() } } diff --git a/primitives/trie/src/storage_proof.rs b/primitives/trie/src/storage_proof.rs index 636f28b66c79a..6aba96c747974 100644 --- a/primitives/trie/src/storage_proof.rs +++ b/primitives/trie/src/storage_proof.rs @@ -20,6 +20,7 @@ use codec::{Decode, Encode}; use hash_db::{HashDB, Hasher}; use sp_std::vec::Vec; use trie_db::NodeCodec; +use sp_core::state_version::StateVersion; /// A proof that some set of key-value pairs are included in the storage trie. The proof contains /// the storage values so that the partial storage backend can be reconstructed by a verifier that @@ -31,19 +32,19 @@ use trie_db::NodeCodec; #[derive(Debug, PartialEq, Eq, Clone, Encode, Decode)] pub struct StorageProof { trie_nodes: Vec>, - pub alt_hashing: Option>, // TODO non public + state_version: StateVersion, } /// Storage proof in compact form. #[derive(Debug, PartialEq, Eq, Clone, Encode, Decode)] pub struct CompactProof { pub encoded_nodes: Vec>, - pub alt_hashing: Option>, + pub state_version: StateVersion, } impl StorageProof { /// Constructs a storage proof from a subset of encoded trie nodes in a storage backend. - pub fn new(trie_nodes: Vec>, alt_hashing: Option>) -> Self { + pub fn new(trie_nodes: Vec>, state_version: StateVersion) -> Self { StorageProof { trie_nodes, alt_hashing } } @@ -51,9 +52,8 @@ impl StorageProof { /// /// An empty proof is capable of only proving trivial statements (ie. that an empty set of /// key-value pairs exist in storage). - pub fn empty(alt_hashing: Option>) -> Self { - // TODO consider alt_hashing default value. ++ - StorageProof { trie_nodes: Vec::new(), alt_hashing } + pub fn empty() -> Self { + StorageProof { trie_nodes: Vec::new(), state_version: StateVersion::Default() } } /// Returns whether this is an empty proof. @@ -93,20 +93,20 @@ impl StorageProof { where I: IntoIterator, { - let mut alt_hashing = None; - let alt_hashing = &mut alt_hashing; + let mut state_version = StateVersion::default(); + let state_version = &mut state_version; let trie_nodes = proofs .into_iter() .flat_map(|proof| { - debug_assert!(alt_hashing == &None || alt_hashing == &proof.alt_hashing); - *alt_hashing = proof.alt_hashing.clone(); + debug_assert!(state_version == &StateVersion::default() || state_version == &proof.state_version); + *state_version = proof.state_version; proof.iter_nodes() }) .collect::>() .into_iter() .collect(); - Self { trie_nodes, alt_hashing: *alt_hashing } + Self { trie_nodes, state_version: *state_version } } /// Encode as a compact proof with default @@ -143,7 +143,7 @@ impl CompactProof { &self, expected_root: Option<&H::Out>, ) -> Result<(StorageProof, H::Out), crate::CompactProofError>> { - let alt_hashing = self.alt_hashing.clone(); + let state_version = self.state_version; let mut db = crate::MemoryDB::::new(&[]); let root = crate::decode_compact::, _, _>( &mut db, @@ -156,7 +156,7 @@ impl CompactProof { .into_iter() .filter_map(|kv| if (kv.1).1 > 0 { Some((kv.1).0) } else { None }) .collect(), - alt_hashing, + state_version, ), root, )) @@ -187,7 +187,11 @@ impl From for crate::MemoryDB { fn from(proof: StorageProof) -> Self { let mut db = crate::MemoryDB::default(); for item in proof.trie_nodes.iter() { - let mut meta = Default::default(); + let mut meta = Meta::default(); + meta.try_inner_hashing = match proof.state_version { + StateVersion::V0 => None, + StateVersion::V1 { threshold, .. } => Some(threshold), + }; // Read meta from state (required for value layout). let _ = as TrieLayout>::Codec::decode_plan(item.as_slice(), &mut meta); db.alt_insert( diff --git a/primitives/trie/src/trie_codec.rs b/primitives/trie/src/trie_codec.rs index a69500d446a9d..b3d00ab95776f 100644 --- a/primitives/trie/src/trie_codec.rs +++ b/primitives/trie/src/trie_codec.rs @@ -202,7 +202,7 @@ pub fn encode_compact(proof: StorageProof, root: TrieHash) -> Result = proof.into_memory_db(); - let backend = sp_state_machine::TrieBackend::<_, crate::Hashing>::new(db, root, alt_hashing); + let backend = sp_state_machine::TrieBackend::<_, crate::Hashing>::new(db, root, state_version); let mut overlay = sp_state_machine::OverlayedChanges::default(); let mut cache = sp_state_machine::StorageTransactionCache::<_, _, BlockNumber>::default(); let mut ext = sp_state_machine::Ext::new( diff --git a/test-utils/test-runner/src/utils.rs b/test-utils/test-runner/src/utils.rs index 5ea6c23580fb6..f1b1907270d3a 100644 --- a/test-utils/test-runner/src/utils.rs +++ b/test-utils/test-runner/src/utils.rs @@ -51,9 +51,8 @@ pub fn default_config( let root_path = base_path.path().to_path_buf().join("chains").join(chain_spec.id()); let state_version = chain_spec.state_versions() - .get(0) - .and_then(|(n, s)| (n == &0).then(|| s.clone())) - .unwrap_or(None); + .expect("Invalid state versions for chain spec") + .genesis_state_version(); let storage = chain_spec .as_storage_builder() diff --git a/utils/frame/benchmarking-cli/src/command.rs b/utils/frame/benchmarking-cli/src/command.rs index fa697b167eb64..72bcd31ac626c 100644 --- a/utils/frame/benchmarking-cli/src/command.rs +++ b/utils/frame/benchmarking-cli/src/command.rs @@ -97,9 +97,9 @@ impl BenchmarkCmd { ExecDispatch: NativeExecutionDispatch + 'static, { let state_version = config.chain_spec.state_versions() - .get(0) - .and_then(|(n, s)| (n == &0).then(|| s.clone())) - .unwrap_or(None); + .ok_or_else(|| Err("Invalid state versions for chain spec".into()))? + .genesis_state_version(); + if let Some(output_path) = &self.output { if !output_path.is_dir() && output_path.file_name().is_none() { return Err("Output file or path is invalid!".into()) diff --git a/utils/frame/try-runtime/cli/src/lib.rs b/utils/frame/try-runtime/cli/src/lib.rs index 8c949b06be56b..b2ecadd08fb30 100644 --- a/utils/frame/try-runtime/cli/src/lib.rs +++ b/utils/frame/try-runtime/cli/src/lib.rs @@ -463,12 +463,9 @@ impl CliConfiguration for TryRuntimeCmd { /// Extract `:code` from the given chain spec and return as `StorageData` along with the /// corresponding `StorageKey`. fn extract_code(spec: Box) -> sc_cli::Result<(StorageKey, StorageData)> { - let mut state_version = None; - if let Some((number, version)) = spec.state_versions().get(0) { - if number == &0 { - state_version = version.clone(); - } - } + let state_version = spec.state_versions() + .ok_or_else(|| "Invalid state versions for chain spec".into())? + .genesis_state_version(); let genesis_storage = spec.build_storage(state_version)?; let code = StorageData( From 8f2eaea26cea7bdca476de699e25d27c758654b2 Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Fri, 20 Aug 2021 16:09:29 +0200 Subject: [PATCH 072/127] fixes --- client/api/src/cht.rs | 3 ++- client/api/src/in_mem.rs | 8 +++--- client/light/src/lib.rs | 2 +- client/service/src/client/client.rs | 2 +- frame/authorship/src/lib.rs | 2 +- frame/collective/src/lib.rs | 6 ++--- frame/staking/src/pallet/mod.rs | 3 +-- frame/sudo/src/mock.rs | 6 ++--- frame/support/src/storage/migration.rs | 12 ++++----- frame/system/src/lib.rs | 2 +- frame/transaction-payment/src/lib.rs | 2 +- primitives/core/src/state_version.rs | 4 ++- primitives/io/src/lib.rs | 26 ++++++++++++------- primitives/runtime/src/state_version.rs | 16 +++++++++--- primitives/state-machine/src/basic.rs | 12 ++++----- .../state-machine/src/changes_trie/build.rs | 2 +- primitives/state-machine/src/ext.rs | 20 +++++++------- .../state-machine/src/in_memory_backend.rs | 12 ++++----- primitives/state-machine/src/lib.rs | 20 +++++++------- .../src/overlayed_changes/mod.rs | 2 +- .../state-machine/src/proving_backend.rs | 22 ++++++++-------- primitives/state-machine/src/testing.rs | 8 +++--- primitives/state-machine/src/trie_backend.rs | 4 +-- primitives/trie/src/storage_proof.rs | 11 +++++--- primitives/trie/src/trie_codec.rs | 2 +- test-utils/runtime/src/genesismap.rs | 6 ++--- test-utils/runtime/src/lib.rs | 2 +- utils/frame/remote-externalities/src/lib.rs | 2 +- 28 files changed, 119 insertions(+), 100 deletions(-) diff --git a/client/api/src/cht.rs b/client/api/src/cht.rs index 9739b4cfd5e46..ba464196f41ef 100644 --- a/client/api/src/cht.rs +++ b/client/api/src/cht.rs @@ -117,7 +117,8 @@ where .into_iter() .map(|(k, v)| (k, Some(v))) .collect::>(); - let backend: InMemoryBackend = None.into(); // No inner hashing for cht. + // No inner hashing for cht. + let backend: InMemoryBackend = sp_runtime::StateVersion::V0.into(); let mut storage = backend.update(vec![(None, transaction)]); let trie_storage = storage .as_trie_backend() diff --git a/client/api/src/in_mem.rs b/client/api/src/in_mem.rs index c704329cf4d8d..8a23853fee4ca 100644 --- a/client/api/src/in_mem.rs +++ b/client/api/src/in_mem.rs @@ -112,7 +112,7 @@ struct BlockchainStorage { changes_trie_cht_roots: HashMap, Block::Hash>, leaves: LeafSet>, aux: HashMap, Vec>, - state_versions: StateVersions, + state_versions: StateVersions, /* // TODO not sure if migration support. TODO replace () by StateMigration. state_migration: Vec<(NumberFor, ())>, @@ -154,12 +154,12 @@ impl Blockchain { if let Ok(Some(header)) = self.header(BlockId::Hash(h)) { header.number().clone() } else { - return default // Default to latest version TODO a latest version default with StateVersion. + 0u32.into() } }, BlockId::Number(n) => n, }; - self.storage.read().state_versions.version_at(number) + self.storage.read().state_versions.state_version_at(number) } /// Create new in-memory blockchain storage. @@ -176,7 +176,7 @@ impl Blockchain { changes_trie_cht_roots: HashMap::new(), leaves: LeafSet::new(), aux: HashMap::new(), - state_versions: Default::default(), + state_versions: StateVersions::default(), })); Blockchain { storage } } diff --git a/client/light/src/lib.rs b/client/light/src/lib.rs index 10a87494a9b9a..f06995fb9d38a 100644 --- a/client/light/src/lib.rs +++ b/client/light/src/lib.rs @@ -50,7 +50,7 @@ pub fn new_light_blockchain>(storage: S) -> A } /// Create an instance of light client backend. -pub fn new_light_backend(blockchain: Arc>, state_versions: sp_runtime::StateVersions -> Arc> +pub fn new_light_backend(blockchain: Arc>, state_versions: sp_runtime::StateVersions) -> Arc> where B: BlockT, S: BlockchainStorage, diff --git a/client/service/src/client/client.rs b/client/service/src/client/client.rs index 8fe999cd60426..eb4c197d66bb9 100644 --- a/client/service/src/client/client.rs +++ b/client/service/src/client/client.rs @@ -200,7 +200,7 @@ pub struct ClientConfig { /// version doesn't match anymore. pub wasm_runtime_substitutes: HashMap>, /// State version to use with chain. - pub state_versions: Vec<(NumberFor, Option>)>, + pub state_versions: sp_runtime::StateVersions, } impl Default for ClientConfig { diff --git a/frame/authorship/src/lib.rs b/frame/authorship/src/lib.rs index da1e3b80ae291..d5151d36c53ca 100644 --- a/frame/authorship/src/lib.rs +++ b/frame/authorship/src/lib.rs @@ -533,7 +533,7 @@ mod tests { } fn new_test_ext() -> sp_io::TestExternalities { - let state_version = None; + let state_version = sp_runtime::StateVersion::default(); let t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); (t, state_version).into() } diff --git a/frame/collective/src/lib.rs b/frame/collective/src/lib.rs index eff09338dc4c2..89d9542e8db84 100644 --- a/frame/collective/src/lib.rs +++ b/frame/collective/src/lib.rs @@ -1083,7 +1083,7 @@ mod tests { ); pub fn new_test_ext() -> sp_io::TestExternalities { - let state_version = None; + let state_version = sp_runtime::StateVersion::default(); let mut ext: sp_io::TestExternalities = (GenesisConfig { collective: collective::GenesisConfig { members: vec![1, 2, 3], @@ -1095,7 +1095,7 @@ mod tests { }, default_collective: Default::default(), } - .build_storage(state_version.clone()) + .build_storage(state_version) .unwrap(), state_version).into(); ext.execute_with(|| System::set_block_number(1)); ext @@ -2158,7 +2158,7 @@ mod tests { #[test] #[should_panic(expected = "Members cannot contain duplicate accounts.")] fn genesis_build_panics_with_duplicate_members() { - let state_version = None; + let state_version = sp_runtime::StateVersion::default(); collective::GenesisConfig:: { members: vec![1, 2, 3, 1], phantom: Default::default(), diff --git a/frame/staking/src/pallet/mod.rs b/frame/staking/src/pallet/mod.rs index 2f057b44e64c0..ee09660d23d20 100644 --- a/frame/staking/src/pallet/mod.rs +++ b/frame/staking/src/pallet/mod.rs @@ -647,8 +647,7 @@ pub mod pallet { fn integrity_test() { sp_std::if_std! { - let state_version = None; - sp_io::TestExternalities::new_empty(state_version).execute_with(|| + sp_io::TestExternalities::new_empty().execute_with(|| assert!( T::SlashDeferDuration::get() < T::BondingDuration::get() || T::BondingDuration::get() == 0, "As per documentation, slash defer duration ({}) should be less than bonding duration ({}).", diff --git a/frame/sudo/src/mock.rs b/frame/sudo/src/mock.rs index e916bfe771273..00b1ac7c3fd95 100644 --- a/frame/sudo/src/mock.rs +++ b/frame/sudo/src/mock.rs @@ -164,10 +164,10 @@ pub type LoggerCall = logger::Call; // Build test environment by setting the root `key` for the Genesis. pub fn new_test_ext(root_key: u64) -> sp_io::TestExternalities { - let state_version = None; - let mut t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); + let state_version = sp_runtime::StateVersion::default(); + let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); sudo::GenesisConfig:: { key: root_key } - .assimilate_storage(&mut t, state_version.clone()) + .assimilate_storage(&mut t, state_version) .unwrap(); (t, state_version).into() } diff --git a/frame/support/src/storage/migration.rs b/frame/support/src/storage/migration.rs index 50a5e30efd1d9..0f10c5cbb47d3 100644 --- a/frame/support/src/storage/migration.rs +++ b/frame/support/src/storage/migration.rs @@ -283,7 +283,7 @@ pub fn take_storage_item /// "my_new_example_name", a migration can be: /// ``` /// # use frame_support::storage::migration::move_storage_from_pallet; -/// # sp_io::TestExternalities::new_empty(None).execute_with(|| { +/// # sp_io::TestExternalities::new_empty().execute_with(|| { /// move_storage_from_pallet(b"Foo", b"my_example", b"my_new_example_name"); /// move_storage_from_pallet(b"Bar", b"my_example", b"my_new_example_name"); /// # }) @@ -325,7 +325,7 @@ pub fn move_storage_from_pallet( /// "my_new_example_name", a migration can be: /// ``` /// # use frame_support::storage::migration::move_pallet; -/// # sp_io::TestExternalities::new_empty(None).execute_with(|| { +/// # sp_io::TestExternalities::new_empty().execute_with(|| { /// move_pallet(b"my_example", b"my_new_example_name"); /// # }) /// ``` @@ -406,7 +406,7 @@ mod tests { #[test] fn test_move_prefix() { - TestExternalities::new_empty(None).execute_with(|| { + TestExternalities::new_empty().execute_with(|| { OldStorageValue::put(3); OldStorageMap::insert(1, 2); OldStorageMap::insert(3, 4); @@ -422,7 +422,7 @@ mod tests { #[test] fn test_move_storage() { - TestExternalities::new_empty(None).execute_with(|| { + TestExternalities::new_empty().execute_with(|| { OldStorageValue::put(3); OldStorageMap::insert(1, 2); OldStorageMap::insert(3, 4); @@ -445,7 +445,7 @@ mod tests { #[test] fn test_move_pallet() { - TestExternalities::new_empty(None).execute_with(|| { + TestExternalities::new_empty().execute_with(|| { OldStorageValue::put(3); OldStorageMap::insert(1, 2); OldStorageMap::insert(3, 4); @@ -461,7 +461,7 @@ mod tests { #[test] fn test_storage_iter() { - TestExternalities::new_empty(None).execute_with(|| { + TestExternalities::new_empty().execute_with(|| { OldStorageValue::put(3); OldStorageMap::insert(1, 2); OldStorageMap::insert(3, 4); diff --git a/frame/system/src/lib.rs b/frame/system/src/lib.rs index 1db3a43d03b5d..506e18c70554b 100644 --- a/frame/system/src/lib.rs +++ b/frame/system/src/lib.rs @@ -1447,7 +1447,7 @@ impl Pallet { ], children_default: map![], }; - TestExternalities::new(storage, Some(Some(sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD))) + TestExternalities::new(storage, Default::default()) } /// Set the block number to something in particular. Can be used as an alternative to diff --git a/frame/transaction-payment/src/lib.rs b/frame/transaction-payment/src/lib.rs index 0c555830fbb69..a600650924b0e 100644 --- a/frame/transaction-payment/src/lib.rs +++ b/frame/transaction-payment/src/lib.rs @@ -350,7 +350,7 @@ pub mod pallet { target += addition; #[cfg(any(feature = "std", test))] - sp_io::TestExternalities::new_empty(None).execute_with(|| { + sp_io::TestExternalities::new_empty().execute_with(|| { >::set_block_consumed_resources(target, 0); let next = T::FeeMultiplierUpdate::convert(min_value); assert!( diff --git a/primitives/core/src/state_version.rs b/primitives/core/src/state_version.rs index b837eaf2704d9..04586fe43bc4f 100644 --- a/primitives/core/src/state_version.rs +++ b/primitives/core/src/state_version.rs @@ -17,6 +17,8 @@ //! Substrate state versioning core types. +use codec::{Decode, Encode}; + /// Default state version to use with a new substrate chain. /// /// When this value change, old chain will require to force their @@ -26,7 +28,7 @@ pub const DEFAULT_STATE_VERSION: StateVersion = StateVersion::V1 { threshold: 33 }; /// Supported version with substrate chain. -#[derive(Clone, Copy, Encode, Decode)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Encode, Decode)] #[cfg_attr(feature = "std", derive(serde::Serialize, serde::Deserialize))] pub enum StateVersion { /// Patricia trie Radix 16 without extension node. diff --git a/primitives/io/src/lib.rs b/primitives/io/src/lib.rs index b885c08b0f348..cd4dfb5ddd9cc 100644 --- a/primitives/io/src/lib.rs +++ b/primitives/io/src/lib.rs @@ -51,7 +51,7 @@ use sp_core::{ offchain::{ HttpError, HttpRequestId, HttpRequestStatus, OpaqueNetworkState, StorageKind, Timestamp, }, - sr25519, LogLevel, LogLevelFilter, OpaquePeerId, H256, + sr25519, LogLevel, LogLevelFilter, OpaquePeerId, H256, state_version::StateVersion, }; #[cfg(feature = "std")] @@ -477,7 +477,7 @@ pub trait Misc { fn runtime_version(&mut self, wasm: &[u8]) -> Option> { use sp_core::traits::ReadRuntimeVersionExt; - let mut ext: sp_state_machine::BasicExternalities = None.into(); + let mut ext: sp_state_machine::BasicExternalities = StateVersion::default().into(); match self .extension::() @@ -1475,7 +1475,8 @@ mod tests { #[test] fn storage_works() { - let mut t: BasicExternalities = None.into(); + let state_version = StateVersion::default(); + let mut t: BasicExternalities = state_version.into(); t.execute_with(|| { assert_eq!(storage::get(b"hello"), None); storage::set(b"hello", b"world"); @@ -1487,7 +1488,7 @@ mod tests { t = BasicExternalities::new(Storage { top: map![b"foo".to_vec() => b"bar".to_vec()], children_default: map![], - }, None); + }, state_version); t.execute_with(|| { assert_eq!(storage::get(b"hello"), None); @@ -1497,7 +1498,7 @@ mod tests { let value = vec![7u8; 35]; let storage = Storage { top: map![b"foo00".to_vec() => value.clone()], children_default: map![] }; - t = BasicExternalities::new(storage, Some(Some(sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD))); + t = BasicExternalities::new(storage, state_version); t.execute_with(|| { assert_eq!(storage::get(b"hello"), None); @@ -1507,11 +1508,12 @@ mod tests { #[test] fn read_storage_works() { + let state_version = StateVersion::default(); let value = b"\x0b\0\0\0Hello world".to_vec(); let mut t = BasicExternalities::new(Storage { top: map![b":test".to_vec() => value.clone()], children_default: map![], - }, None); + }, state_version); t.execute_with(|| { let mut v = [0u8; 4]; @@ -1525,6 +1527,7 @@ mod tests { #[test] fn clear_prefix_works() { + let state_version = StateVersion::default(); let mut t = BasicExternalities::new(Storage { top: map![ b":a".to_vec() => b"\x0b\0\0\0Hello world".to_vec(), @@ -1533,7 +1536,7 @@ mod tests { b":abdd".to_vec() => b"\x0b\0\0\0Hello world".to_vec() ], children_default: map![], - }, None); + }, state_version); t.execute_with(|| { assert!(matches!( @@ -1550,7 +1553,8 @@ mod tests { #[test] fn batch_verify_start_finish_works() { - let mut ext: sp_state_machine::BasicExternalities = None.into(); + let state_version = StateVersion::default(); + let mut ext: sp_state_machine::BasicExternalities = state_version.into(); ext.register_extension(TaskExecutorExt::new(TaskExecutor::new())); ext.execute_with(|| { @@ -1568,7 +1572,8 @@ mod tests { #[test] fn long_sr25519_batching() { - let mut ext: sp_state_machine::BasicExternalities = None.into(); + let state_version = StateVersion::default(); + let mut ext: sp_state_machine::BasicExternalities = state_version.into(); ext.register_extension(TaskExecutorExt::new(TaskExecutor::new())); ext.execute_with(|| { let pair = sr25519::Pair::generate_with_phrase(None).0; @@ -1595,7 +1600,8 @@ mod tests { #[test] fn batching_works() { - let mut ext: sp_state_machine::BasicExternalities = None.into(); + let state_version = StateVersion::default(); + let mut ext: sp_state_machine::BasicExternalities = state_version.into(); ext.register_extension(TaskExecutorExt::new(TaskExecutor::new())); ext.execute_with(|| { // invalid ed25519 signature diff --git a/primitives/runtime/src/state_version.rs b/primitives/runtime/src/state_version.rs index 8b3fd0d1453de..4f90313cc9cd9 100644 --- a/primitives/runtime/src/state_version.rs +++ b/primitives/runtime/src/state_version.rs @@ -24,11 +24,19 @@ use sp_std::vec::Vec; use sp_arithmetic::traits::Zero; /// Multiple versions of state in use for a chain. -#[derive(Clone, Default)] +#[derive(Clone)] pub struct StateVersions { canonical_states: Vec<(NumberFor, StateVersion)>, } +impl Default for StateVersions { + fn default() -> Self { + StateVersions { + canonical_states: Vec::new(), + } + } +} + impl StateVersions { /// Access genesis state version. /// This uses default state if undefined. @@ -55,10 +63,10 @@ impl StateVersions { } /// Modify configuration, mostly for testing. - pub fn add(&mut self, (at, conf): (NumberFor, StateVersion)) -> StateVersion { + pub fn add(&mut self, (at, conf): (NumberFor, StateVersion)) { let mut insert = Some(0); let mut replace = None; - for (i, (number, _)) in self.canonical_states.iter() { + for (i, (number, _)) in self.canonical_states.iter().enumerate() { if number == &at { replace = Some(i); break; @@ -66,7 +74,7 @@ impl StateVersions { if number > &at { break; } - insert = i + 1; + insert = Some(i + 1); } if let Some(i) = replace { self.canonical_states[i] = (at, conf); diff --git a/primitives/state-machine/src/basic.rs b/primitives/state-machine/src/basic.rs index 3075bf67ba4d2..106e217850151 100644 --- a/primitives/state-machine/src/basic.rs +++ b/primitives/state-machine/src/basic.rs @@ -295,7 +295,7 @@ impl Externalities for BasicExternalities { Layout::::default() }, StateVersion::V1 { threshold } => { - Layout::::with_alt_hashing(*threshold) + Layout::::with_alt_hashing(threshold) }, }; layout.trie_root(self.inner.top.clone()).as_ref().into() @@ -389,7 +389,7 @@ mod tests { #[test] fn commit_should_work() { - let mut ext: BasicExternalities = None.into(); + let mut ext: BasicExternalities = StateVersion::default().into(); ext.set_storage(b"doe".to_vec(), b"reindeer".to_vec()); ext.set_storage(b"dog".to_vec(), b"puppy".to_vec()); ext.set_storage(b"dogglesworth".to_vec(), b"cat".to_vec()); @@ -401,7 +401,7 @@ mod tests { #[test] fn set_and_retrieve_code() { - let mut ext: BasicExternalities = None.into(); + let mut ext: BasicExternalities = StateVersion::default().into(); let code = vec![1, 2, 3]; ext.set_storage(CODE.to_vec(), code.clone()); @@ -421,7 +421,7 @@ mod tests { child_info: child_info.to_owned(), } ], - }, None); + }, StateVersion::default()); assert_eq!(ext.child_storage(child_info, b"doe"), Some(b"reindeer".to_vec())); @@ -451,7 +451,7 @@ mod tests { child_info: child_info.to_owned(), } ], - }, None); + }, StateVersion::default()); let res = ext.kill_child_storage(child_info, None); assert_eq!(res, (true, 3)); @@ -460,7 +460,7 @@ mod tests { #[test] fn basic_externalities_is_empty() { // Make sure no values are set by default in `BasicExternalities`. - let storage = BasicExternalities::new_empty(None).into_storages(); + let storage = BasicExternalities::new_empty(StateVersion::default()).into_storages(); assert!(storage.top.is_empty()); assert!(storage.children_default.is_empty()); } diff --git a/primitives/state-machine/src/changes_trie/build.rs b/primitives/state-machine/src/changes_trie/build.rs index 732f2e6bfd5e9..1377ab36084b4 100644 --- a/primitives/state-machine/src/changes_trie/build.rs +++ b/primitives/state-machine/src/changes_trie/build.rs @@ -391,7 +391,7 @@ mod test { (vec![105], vec![255]), ] .into_iter() - .collect::>(), None) + .collect::>(), sp_core::state_version::StateVersion::V0) .into(); let prefixed_child_trie_key1 = child_info_1.prefixed_storage_key(); let storage = InMemoryStorage::with_inputs( diff --git a/primitives/state-machine/src/ext.rs b/primitives/state-machine/src/ext.rs index a12efcdf64b5f..769d0968831f5 100644 --- a/primitives/state-machine/src/ext.rs +++ b/primitives/state-machine/src/ext.rs @@ -950,7 +950,7 @@ mod tests { use hex_literal::hex; use num_traits::Zero; use sp_core::{ - map, + map, state_version::StateVersion, storage::{well_known_keys::EXTRINSIC_INDEX, Storage, StorageChild}, Blake2Hasher, H256, }; @@ -977,7 +977,7 @@ mod tests { fn storage_changes_root_is_none_when_storage_is_not_provided() { let mut overlay = prepare_overlay_with_changes(); let mut cache = StorageTransactionCache::default(); - let backend: TestBackend = None.into(); + let backend: TestBackend = StateVersion::default().into(); let mut ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); assert_eq!(ext.storage_changes_root(&H256::default().encode()).unwrap(), None); } @@ -986,7 +986,7 @@ mod tests { fn storage_changes_root_is_none_when_state_is_not_provided() { let mut overlay = prepare_overlay_with_changes(); let mut cache = StorageTransactionCache::default(); - let backend: TestBackend = None.into(); + let backend: TestBackend = StateVersion::default().into(); let mut ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); assert_eq!(ext.storage_changes_root(&H256::default().encode()).unwrap(), None); } @@ -997,7 +997,7 @@ mod tests { let mut cache = StorageTransactionCache::default(); let storage = TestChangesTrieStorage::with_blocks(vec![(99, Default::default())]); let state = Some(ChangesTrieState::new(changes_trie_config(), Zero::zero(), &storage)); - let backend: TestBackend = None.into(); + let backend: TestBackend = StateVersion::default().into(); let mut ext = TestExt::new(&mut overlay, &mut cache, &backend, state, None); assert_eq!( ext.storage_changes_root(&H256::default().encode()).unwrap(), @@ -1013,7 +1013,7 @@ mod tests { overlay.set_storage(vec![1], None); let storage = TestChangesTrieStorage::with_blocks(vec![(99, Default::default())]); let state = Some(ChangesTrieState::new(changes_trie_config(), Zero::zero(), &storage)); - let backend: TestBackend = None.into(); + let backend: TestBackend = StateVersion::default().into(); let mut ext = TestExt::new(&mut overlay, &mut cache, &backend, state, None); assert_eq!( ext.storage_changes_root(&H256::default().encode()).unwrap(), @@ -1034,7 +1034,7 @@ mod tests { vec![40] => vec![40] ], children_default: map![], - }, None) + }, Default::default()) .into(); let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); @@ -1078,7 +1078,7 @@ mod tests { vec![30] => vec![30] ], children_default: map![], - }, None) + }, Default::default()) .into(); let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); @@ -1109,7 +1109,7 @@ mod tests { child_info: child_info.to_owned(), } ], - }, None) + }, Default::default()) .into(); let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); @@ -1154,7 +1154,7 @@ mod tests { child_info: child_info.to_owned(), } ], - }, None) + }, Default::default()) .into(); let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); @@ -1191,7 +1191,7 @@ mod tests { child_info: child_info.to_owned(), } ], - }, None) + }, Default::default()) .into(); let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); diff --git a/primitives/state-machine/src/in_memory_backend.rs b/primitives/state-machine/src/in_memory_backend.rs index eb0f04f137ad0..09ab023094b72 100644 --- a/primitives/state-machine/src/in_memory_backend.rs +++ b/primitives/state-machine/src/in_memory_backend.rs @@ -70,7 +70,7 @@ where pub fn update_backend(&self, root: H::Out, changes: MemoryDB) -> Self { let mut clone = self.backend_storage().clone(); clone.consolidate(changes); - Self::new(clone, root, self.state_version) + Self::new(clone, root, self.state_version()) } /// Apply the given transaction to this backend and set the root to the given value. @@ -90,20 +90,18 @@ where H::Out: Codec + Ord, { fn clone(&self) -> Self { - TrieBackend::new(self.backend_storage().clone(), self.root().clone(), self.state_version) + TrieBackend::new(self.backend_storage().clone(), self.root().clone(), self.state_version()) } } -/* impl Default for TrieBackend, H> where H::Out: Codec + Ord, { fn default() -> Self { - new_in_mem() // TODO try removing this default + new_in_mem(Default::default()) } } -*/ impl From for TrieBackend, H> @@ -185,7 +183,7 @@ mod tests { /// Assert in memory backend with only child trie keys works as trie backend. #[test] fn in_memory_with_child_trie_only() { - let storage = new_in_mem::(None); + let storage = new_in_mem::(StateVersion::default()); let child_info = ChildInfo::new_default(b"1"); let child_info = &child_info; let mut storage = storage @@ -198,7 +196,7 @@ mod tests { #[test] fn insert_multiple_times_child_data_works() { - let mut storage = new_in_mem::(None); + let mut storage = new_in_mem::(StateVersion::default()); let child_info = ChildInfo::new_default(b"1"); storage diff --git a/primitives/state-machine/src/lib.rs b/primitives/state-machine/src/lib.rs index 62cf09ce397e4..cb3a9308ab394 100644 --- a/primitives/state-machine/src/lib.rs +++ b/primitives/state-machine/src/lib.rs @@ -1003,7 +1003,7 @@ mod tests { storage::{ChildInfo, TEST_DEFAULT_ALT_HASH_THRESHOLD as TRESHOLD}, testing::TaskExecutor, traits::{CodeExecutor, Externalities, RuntimeCode}, - NativeOrEncoded, NeverNativeValue, + NativeOrEncoded, NeverNativeValue, state_version::StateVersion, }; use sp_runtime::traits::BlakeTwo256; use std::{ @@ -1222,7 +1222,7 @@ mod tests { b"abc".to_vec() => b"2".to_vec(), b"bbb".to_vec() => b"3".to_vec() ]; - let mut state = InMemoryBackend::::from((initial, None)); + let mut state = InMemoryBackend::::from((initial, Default::default())); let backend = state.as_trie_backend().unwrap(); let mut overlay = OverlayedChanges::default(); @@ -1303,7 +1303,7 @@ mod tests { b"d".to_vec() => b"3".to_vec() ], ]; - let backend = InMemoryBackend::::from((initial, None)); + let backend = InMemoryBackend::::from((initial, Default::default())); let mut overlay = OverlayedChanges::default(); overlay.set_child_storage(&child_info, b"1".to_vec(), Some(b"1312".to_vec())); @@ -1351,7 +1351,7 @@ mod tests { b"d".to_vec() => b"3".to_vec() ], ]; - let backend = InMemoryBackend::::from((initial, None)); + let backend = InMemoryBackend::::from((initial, Default::default())); let mut overlay = OverlayedChanges::default(); let mut cache = StorageTransactionCache::default(); let mut ext = Ext::new( @@ -1375,7 +1375,7 @@ mod tests { fn set_child_storage_works() { let child_info = ChildInfo::new_default(b"sub1"); let child_info = &child_info; - let mut state = new_in_mem::(None); + let mut state = new_in_mem::(Default::default()); let backend = state.as_trie_backend().unwrap(); let mut overlay = OverlayedChanges::default(); let mut cache = StorageTransactionCache::default(); @@ -1397,7 +1397,7 @@ mod tests { fn append_storage_works() { let reference_data = vec![b"data1".to_vec(), b"2".to_vec(), b"D3".to_vec(), b"d4".to_vec()]; let key = b"key".to_vec(); - let mut state = new_in_mem::(None); + let mut state = new_in_mem::(Default::default()); let backend = state.as_trie_backend().unwrap(); let mut overlay = OverlayedChanges::default(); let mut cache = StorageTransactionCache::default(); @@ -1452,7 +1452,7 @@ mod tests { let key = b"events".to_vec(); let mut cache = StorageTransactionCache::default(); - let mut state = new_in_mem::(None); + let mut state = new_in_mem::(Default::default()); let backend = state.as_trie_backend().unwrap(); let mut overlay = OverlayedChanges::default(); @@ -1673,7 +1673,7 @@ mod tests { remote_proof }; - let remote_proof = check_proof(mdb.clone(), root.clone(), None); + let remote_proof = check_proof(mdb.clone(), root.clone(), StateVersion::V0); // check full values in proof assert!(remote_proof.encode().len() > 1_100); assert!(remote_proof.encoded_size() > 1_100); @@ -1704,7 +1704,7 @@ mod tests { } let root3 = root.clone(); assert!(root1 != root3); - let remote_proof = check_proof(mdb.clone(), root.clone(), Some(Some(TRESHOLD))); + let remote_proof = check_proof(mdb.clone(), root.clone(), StateVersion::V1 { threshold: 33 }); // nodes foo is replaced by its hashed value form. assert!(remote_proof.encode().len() < 1000); assert!(remote_proof.encoded_size() < 1000); @@ -1812,7 +1812,7 @@ mod tests { b"aaa".to_vec() => b"0".to_vec(), b"bbb".to_vec() => b"".to_vec() ]; - let mut state = InMemoryBackend::::from((initial, None)); + let mut state = InMemoryBackend::::from((initial, Default::default())); let backend = state.as_trie_backend().unwrap(); let mut overlay = OverlayedChanges::default(); diff --git a/primitives/state-machine/src/overlayed_changes/mod.rs b/primitives/state-machine/src/overlayed_changes/mod.rs index f8c2bd8c659b3..36a394a99d05a 100644 --- a/primitives/state-machine/src/overlayed_changes/mod.rs +++ b/primitives/state-machine/src/overlayed_changes/mod.rs @@ -931,7 +931,7 @@ mod tests { ] .into_iter() .collect(); - let backend = InMemoryBackend::::from((initial, None)); + let backend = InMemoryBackend::::from((initial, Default::default())); let mut overlay = OverlayedChanges::default(); overlay.set_collect_extrinsics(false); diff --git a/primitives/state-machine/src/proving_backend.rs b/primitives/state-machine/src/proving_backend.rs index 0b8e46a875ebb..351b58af846c0 100644 --- a/primitives/state-machine/src/proving_backend.rs +++ b/primitives/state-machine/src/proving_backend.rs @@ -175,7 +175,7 @@ impl ProofRecorder { /// Convert into a [`StorageProof`]. pub fn to_storage_proof(&self, state_version: StateVersion) -> StorageProof { // TODO consider state_version as inner field - let try_inner_hashing = match proof.state_version { + let try_inner_hashing = match state_version { StateVersion::V0 => None, StateVersion::V1 { threshold } => Some(threshold), }; @@ -224,7 +224,7 @@ pub struct ProofRecorderBackend<'a, S: 'a + TrieBackendStorage, H: 'a + Hashe impl<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> ProvingBackend<'a, S, H> where - H::Out: Codec, + H::Out: Codec + Ord, { /// Create new proving backend. pub fn new(backend: &'a TrieBackend) -> Self { @@ -240,12 +240,12 @@ where let essence = backend.essence(); let root = essence.root().clone(); let recorder = ProofRecorderBackend { backend: essence.backend_storage(), proof_recorder }; - ProvingBackend(TrieBackend::new(recorder, root, backend.state_version)) + ProvingBackend(TrieBackend::new(recorder, root, backend.state_version())) } /// Extracting the gathered unordered proof. pub fn extract_proof(&self) -> StorageProof { - let state_version = self.0.state_version; + let state_version = self.0.state_version(); self.0.essence().backend_storage().proof_recorder.to_storage_proof::(state_version) } @@ -397,7 +397,7 @@ where } fn state_version(&self) -> StateVersion { - self.0.state_version + self.0.state_version() } } @@ -410,7 +410,7 @@ where H: Hasher, H::Out: Codec, { - let state_version = proof.state_version.clone(); + let state_version = proof.state_version(); let db = proof.into_memory_db(); if db.contains(&root, EMPTY_PREFIX) { @@ -463,7 +463,7 @@ mod tests { use sp_core::H256; let result = create_proof_check_backend::( H256::from_low_u64_be(1), - StorageProof::empty(None), + StorageProof::empty(), ); assert!(result.is_err()); } @@ -499,9 +499,9 @@ mod tests { .map(|i| (vec![i], Some(vec![i; size_content]))) .collect::>())]; let mut in_memory: InMemoryBackend = if flagged { - (contents, Some(Some(sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD))).into() + (contents, StateVersion::V1 { threshold: 33 }).into() } else { - (contents, None).into() + (contents, StateVersion::V0).into() }; let in_memory_root = in_memory.storage_root(std::iter::empty()).0; value_range.clone().for_each(|i| { @@ -541,9 +541,9 @@ mod tests { (Some(child_info_2.clone()), (10..15).map(|i| (vec![i], Some(vec![i]))).collect()), ]; let mut in_memory: InMemoryBackend = if flagged { - (contents, Some(Some(sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD))).into() + (contents, StateVersion::V1 { threshold: 33 }).into() } else { - (contents, None).into() + (contents, StateVersion::V0).into() }; let child_storage_keys = vec![child_info_1.to_owned(), child_info_2.to_owned()]; let in_memory_root = in_memory diff --git a/primitives/state-machine/src/testing.rs b/primitives/state-machine/src/testing.rs index 0ac1c8c1d9166..45729958ba453 100644 --- a/primitives/state-machine/src/testing.rs +++ b/primitives/state-machine/src/testing.rs @@ -92,8 +92,8 @@ where } /// New empty test externalities. - pub fn new_empty(state_version: StateVersion) -> Self { - Self::new_with_code(&[], Storage::default(), state_version) + pub fn new_empty() -> Self { + Self::new_with_code(&[], Storage::default(), Default::default()) } /// Create a new instance of `TestExternalities` with code and storage. @@ -245,7 +245,7 @@ where { fn default() -> Self { // default to default version. - Self::new(Storage::default(), Some(Some(sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD))) + Self::new(Storage::default(), Default::default()) } } @@ -317,7 +317,7 @@ mod tests { #[test] fn commit_should_work() { let storage = Storage::default(); // avoid adding the trie threshold. - let mut ext = TestExternalities::::from((storage, None)); + let mut ext = TestExternalities::::from((storage, Default::default())); let mut ext = ext.ext(); ext.set_storage(b"doe".to_vec(), b"reindeer".to_vec()); ext.set_storage(b"dog".to_vec(), b"puppy".to_vec()); diff --git a/primitives/state-machine/src/trie_backend.rs b/primitives/state-machine/src/trie_backend.rs index 1777878e49d22..76bef33af928a 100644 --- a/primitives/state-machine/src/trie_backend.rs +++ b/primitives/state-machine/src/trie_backend.rs @@ -343,7 +343,7 @@ pub mod tests { hashed_value: bool, ) -> TrieBackend, BlakeTwo256> { let (mdb, root) = test_db(hashed_value); - TrieBackend::new(mdb, root, None) + TrieBackend::new(mdb, root, Default::default()) } #[test] @@ -409,7 +409,7 @@ pub mod tests { assert!(TrieBackend::, BlakeTwo256>::new( PrefixedMemoryDB::default(), Default::default(), - None, + Default::default(), ) .pairs() .is_empty()); diff --git a/primitives/trie/src/storage_proof.rs b/primitives/trie/src/storage_proof.rs index 6aba96c747974..0eb13db4cd0fa 100644 --- a/primitives/trie/src/storage_proof.rs +++ b/primitives/trie/src/storage_proof.rs @@ -45,7 +45,7 @@ pub struct CompactProof { impl StorageProof { /// Constructs a storage proof from a subset of encoded trie nodes in a storage backend. pub fn new(trie_nodes: Vec>, state_version: StateVersion) -> Self { - StorageProof { trie_nodes, alt_hashing } + StorageProof { trie_nodes, state_version } } /// Returns a new empty proof. @@ -53,7 +53,7 @@ impl StorageProof { /// An empty proof is capable of only proving trivial statements (ie. that an empty set of /// key-value pairs exist in storage). pub fn empty() -> Self { - StorageProof { trie_nodes: Vec::new(), state_version: StateVersion::Default() } + StorageProof { trie_nodes: Vec::new(), state_version: StateVersion::default() } } /// Returns whether this is an empty proof. @@ -61,6 +61,11 @@ impl StorageProof { self.trie_nodes.is_empty() } + /// State version used in the proof. + pub fn state_version(&self) -> StateVersion { + self.state_version + } + /// Create an iterator over trie nodes constructed from the proof. The nodes are not guaranteed /// to be traversed in any particular order. pub fn iter_nodes(self) -> StorageProofNodeIterator { @@ -187,7 +192,7 @@ impl From for crate::MemoryDB { fn from(proof: StorageProof) -> Self { let mut db = crate::MemoryDB::default(); for item in proof.trie_nodes.iter() { - let mut meta = Meta::default(); + let mut meta = crate::Meta::default(); meta.try_inner_hashing = match proof.state_version { StateVersion::V0 => None, StateVersion::V1 { threshold, .. } => Some(threshold), diff --git a/primitives/trie/src/trie_codec.rs b/primitives/trie/src/trie_codec.rs index b3d00ab95776f..eb8bb3607a9fa 100644 --- a/primitives/trie/src/trie_codec.rs +++ b/primitives/trie/src/trie_codec.rs @@ -202,7 +202,7 @@ pub fn encode_compact(proof: StorageProof, root: TrieHash) -> Result, /// Additional storage key pairs that will be added to the genesis map. extra_storage: Storage, - state_version: Option>, + state_version: StateVersion, } impl GenesisConfig { @@ -48,7 +48,7 @@ impl GenesisConfig { balance: u64, heap_pages_override: Option, extra_storage: Storage, - state_version: Option>, + state_version: StateVersion, ) -> Self { GenesisConfig { changes_trie_config, diff --git a/test-utils/runtime/src/lib.rs b/test-utils/runtime/src/lib.rs index edb8ec866bc48..09b0e37eac104 100644 --- a/test-utils/runtime/src/lib.rs +++ b/test-utils/runtime/src/lib.rs @@ -1225,7 +1225,7 @@ fn test_read_child_storage() { fn test_witness(proof: StorageProof, root: crate::Hash) { use sp_externalities::Externalities; - let state_version = proof.state_version; + let state_version = proof.state_version(); let db: sp_trie::MemoryDB = proof.into_memory_db(); let backend = sp_state_machine::TrieBackend::<_, crate::Hashing>::new(db, root, state_version); let mut overlay = sp_state_machine::OverlayedChanges::default(); diff --git a/utils/frame/remote-externalities/src/lib.rs b/utils/frame/remote-externalities/src/lib.rs index 2f250e4e5da08..347cc8d66d917 100644 --- a/utils/frame/remote-externalities/src/lib.rs +++ b/utils/frame/remote-externalities/src/lib.rs @@ -485,7 +485,7 @@ impl Builder { /// Build the test externalities. pub async fn build(self) -> Result { let kv = self.pre_build().await?; - let mut ext = TestExternalities::new_empty(None); + let mut ext = TestExternalities::new_empty(); info!(target: LOG_TARGET, "injecting a total of {} keys", kv.len()); for (k, v) in kv { From 46d78d0bb5fc71b0aa8fba7497d3c9b95b4b3ce5 Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Sat, 21 Aug 2021 12:42:54 +0200 Subject: [PATCH 073/127] finish switching, chainspec new issue to get genesis without number. --- .../pallets/template/src/mock.rs | 4 +- bin/node/bench/src/generator.rs | 8 +- bin/node/bench/src/trie.rs | 6 +- bin/node/cli/src/chain_spec.rs | 12 +-- bin/node/executor/benches/bench.rs | 4 +- bin/node/executor/tests/common.rs | 4 +- bin/node/runtime/src/impls.rs | 4 +- bin/node/testing/src/bench.rs | 5 +- bin/node/testing/src/client.rs | 2 +- client/chain-spec/src/chain_spec.rs | 19 +++-- client/chain-spec/src/lib.rs | 5 +- client/db/src/bench.rs | 6 +- client/db/src/lib.rs | 10 +-- client/db/src/storage_cache.rs | 81 ++++++++++--------- client/executor/src/integration_tests/mod.rs | 4 +- client/light/src/backend.rs | 12 +-- client/light/src/fetcher.rs | 4 +- client/light/src/lib.rs | 4 +- .../src/light_client_requests/handler.rs | 10 +-- .../src/light_client_requests/sender.rs | 2 +- client/service/src/builder.rs | 9 ++- client/service/src/client/client.rs | 8 +- client/service/src/client/wasm_override.rs | 2 +- client/service/src/client/wasm_substitutes.rs | 2 +- client/service/test/src/client/light.rs | 12 +-- client/service/test/src/client/mod.rs | 14 ++-- frame/assets/src/mock.rs | 4 +- frame/atomic-swap/src/tests.rs | 6 +- frame/aura/src/mock.rs | 6 +- frame/authority-discovery/src/lib.rs | 6 +- frame/babe/src/mock.rs | 10 +-- frame/balances/src/tests.rs | 8 +- frame/balances/src/tests_composite.rs | 6 +- frame/balances/src/tests_local.rs | 6 +- frame/balances/src/tests_reentrancy.rs | 6 +- frame/benchmarking/src/tests.rs | 4 +- frame/benchmarking/src/tests_instance.rs | 4 +- frame/bounties/src/tests.rs | 20 ++--- frame/contracts/src/tests.rs | 6 +- frame/democracy/src/tests.rs | 8 +- .../election-provider-multi-phase/src/mock.rs | 6 +- frame/elections-phragmen/src/lib.rs | 2 +- frame/elections/src/mock.rs | 4 +- frame/example/src/tests.rs | 4 +- frame/executive/src/lib.rs | 12 +-- frame/gilt/src/mock.rs | 8 +- frame/grandpa/src/mock.rs | 10 +-- frame/identity/src/tests.rs | 4 +- frame/im-online/src/mock.rs | 4 +- frame/indices/src/mock.rs | 6 +- frame/lottery/src/mock.rs | 6 +- frame/membership/src/lib.rs | 8 +- frame/merkle-mountain-range/src/tests.rs | 4 +- frame/multisig/src/tests.rs | 6 +- frame/nicks/src/lib.rs | 6 +- frame/node-authorization/src/mock.rs | 6 +- frame/offences/benchmarking/src/mock.rs | 4 +- frame/offences/src/mock.rs | 4 +- frame/proxy/src/tests.rs | 6 +- frame/randomness-collective-flip/src/lib.rs | 4 +- frame/recovery/src/mock.rs | 6 +- frame/scheduler/src/lib.rs | 4 +- frame/scored-pool/src/mock.rs | 8 +- frame/session/benchmarking/src/mock.rs | 4 +- frame/session/src/historical/mod.rs | 6 +- frame/session/src/historical/offchain.rs | 8 +- frame/session/src/mock.rs | 8 +- frame/society/src/mock.rs | 8 +- frame/staking/src/mock.rs | 10 +-- frame/support/src/lib.rs | 4 +- frame/support/src/storage/generator/mod.rs | 12 +-- frame/support/test/tests/decl_storage.rs | 4 +- frame/support/test/tests/instance.rs | 6 +- frame/system/benches/bench.rs | 4 +- frame/system/benchmarking/src/mock.rs | 4 +- frame/system/src/mock.rs | 4 +- frame/timestamp/src/lib.rs | 4 +- frame/tips/src/tests.rs | 18 ++--- frame/transaction-payment/src/lib.rs | 6 +- frame/transaction-storage/src/mock.rs | 4 +- frame/treasury/src/tests.rs | 22 ++--- frame/uniques/src/mock.rs | 4 +- frame/utility/src/tests.rs | 6 +- frame/vesting/src/mock.rs | 8 +- primitives/runtime/src/lib.rs | 4 +- primitives/runtime/src/state_version.rs | 2 +- test-utils/runtime/client/src/lib.rs | 4 +- test-utils/runtime/src/lib.rs | 2 +- test-utils/runtime/src/system.rs | 2 +- test-utils/test-runner/src/utils.rs | 9 ++- utils/frame/benchmarking-cli/src/command.rs | 11 ++- utils/frame/try-runtime/cli/src/lib.rs | 13 +-- 92 files changed, 351 insertions(+), 335 deletions(-) diff --git a/bin/node-template/pallets/template/src/mock.rs b/bin/node-template/pallets/template/src/mock.rs index cdb4a3047e3ed..b772ceb2cde45 100644 --- a/bin/node-template/pallets/template/src/mock.rs +++ b/bin/node-template/pallets/template/src/mock.rs @@ -59,6 +59,6 @@ impl pallet_template::Config for Test { // Build genesis storage according to the mock runtime. pub fn new_test_ext() -> sp_io::TestExternalities { - let state_version = None; - (system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(), state_version).into() + let state_version = Default::default(); + (system::GenesisConfig::default().build_storage::(state_version).unwrap(), state_version).into() } diff --git a/bin/node/bench/src/generator.rs b/bin/node/bench/src/generator.rs index f0422eed5a437..36c93378559bb 100644 --- a/bin/node/bench/src/generator.rs +++ b/bin/node/bench/src/generator.rs @@ -23,7 +23,7 @@ use node_primitives::Hash; use sp_trie::{trie_types::TrieDBMut, TrieMut}; use crate::simple_trie::SimpleTrie; -use sp_core::StateVersion; +use sp_core::state_version::StateVersion; /// Generate trie from given `key_values`. /// @@ -45,11 +45,11 @@ pub fn generate_trie( ); let mut trie = SimpleTrie { db, overlay: &mut overlay }; { - let mut trie_db = match StateVersion { - V0 => { + let mut trie_db = match state_version { + StateVersion::V0 => { TrieDBMut::new(&mut trie, &mut root) }, - V1 { threshold } => { + StateVersion::V1 { threshold } => { let layout = sp_trie::Layout::with_alt_hashing(threshold); TrieDBMut::::new_with_layout( &mut trie, &mut root, layout, diff --git a/bin/node/bench/src/trie.rs b/bin/node/bench/src/trie.rs index 9452203e704a4..06796b0cd5b2c 100644 --- a/bin/node/bench/src/trie.rs +++ b/bin/node/bench/src/trie.rs @@ -145,7 +145,7 @@ impl core::BenchmarkDescription for TrieReadBenchmarkDescription { let root = generate_trie( database.open(self.database_type), key_values, - Some(sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD), + sp_core::state_version::StateVersion::V0, // TODO switch to v1 or default ); Box::new(TrieReadBenchmark { @@ -186,7 +186,7 @@ impl core::Benchmark for TrieReadBenchmark { let storage: Arc> = Arc::new(Storage(db.open(self.database_type))); - let state_version = None; // TODO switch to default/latest version. + let state_version = Default::default(); let trie_backend = sp_state_machine::TrieBackend::new(storage, self.root, state_version); for (warmup_key, warmup_value) in self.warmup_keys.iter() { let value = trie_backend @@ -258,7 +258,7 @@ impl core::BenchmarkDescription for TrieWriteBenchmarkDescription { let root = generate_trie( database.open(self.database_type), key_values, - Some(sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD), + sp_core::state_version::StateVersion::V0, // TODO switch to v1 or default ); Box::new(TrieWriteBenchmark { diff --git a/bin/node/cli/src/chain_spec.rs b/bin/node/cli/src/chain_spec.rs index 2a4b9455bd9b5..1baa8d6d1e472 100644 --- a/bin/node/cli/src/chain_spec.rs +++ b/bin/node/cli/src/chain_spec.rs @@ -489,23 +489,17 @@ pub(crate) mod tests { #[test] fn test_create_development_chain_spec() { - let spec = development_config(); - let state_version = spec.genesis_state_version(); - spec.build_storage(state_version).unwrap(); + development_config().build_storage(Default::default()).unwrap(); } #[test] fn test_create_local_testnet_chain_spec() { - let spec = local_testnet_config(); - let state_version = spec.genesis_state_version(); - spec.build_storage(state_version).unwrap(); + local_testnet_config().build_storage(Default::default()).unwrap(); } #[test] fn test_staging_test_net_chain_spec() { - let spec = staging_testnet_config(); - let state_version = spec.genesis_state_version(); - spec.build_storage(state_version).unwrap(); + staging_testnet_config().build_storage(Default::default()).unwrap(); } } diff --git a/bin/node/executor/benches/bench.rs b/bin/node/executor/benches/bench.rs index c8cb7b2aa7aae..2cb5c8ece4a74 100644 --- a/bin/node/executor/benches/bench.rs +++ b/bin/node/executor/benches/bench.rs @@ -66,10 +66,10 @@ fn sign(xt: CheckedExtrinsic) -> UncheckedExtrinsic { } fn new_test_ext(genesis_config: &GenesisConfig) -> TestExternalities { - let state_version = None; + let state_version = Default::default(); let mut test_ext = TestExternalities::new_with_code( compact_code_unwrap(), - genesis_config.build_storage(state_version.clone()).unwrap(), + genesis_config.build_storage(state_version).unwrap(), state_version, ); test_ext diff --git a/bin/node/executor/tests/common.rs b/bin/node/executor/tests/common.rs index abd9a071d335a..deadda1429484 100644 --- a/bin/node/executor/tests/common.rs +++ b/bin/node/executor/tests/common.rs @@ -125,11 +125,11 @@ pub fn executor_call< } pub fn new_test_ext(code: &[u8], support_changes_trie: bool) -> TestExternalities { - let state_version = None; + let state_version = Default::default(); let mut ext = TestExternalities::new_with_code( code, node_testing::genesis::config(support_changes_trie, Some(code)) - .build_storage(state_version.clone()) + .build_storage(state_version) .unwrap(), state_version ); diff --git a/bin/node/runtime/src/impls.rs b/bin/node/runtime/src/impls.rs index 3250375fadf84..f052f0c810218 100644 --- a/bin/node/runtime/src/impls.rs +++ b/bin/node/runtime/src/impls.rs @@ -96,9 +96,9 @@ mod multiplier_tests { where F: Fn() -> (), { - let state_version = None; + let state_version = Default::default(); let mut t: sp_io::TestExternalities = (frame_system::GenesisConfig::default() - .build_storage::(state_version.clone()) + .build_storage::(state_version) .unwrap(), state_version) .into(); t.execute_with(|| { diff --git a/bin/node/testing/src/bench.rs b/bin/node/testing/src/bench.rs index b4cf2e594aacb..6595336f4b731 100644 --- a/bin/node/testing/src/bench.rs +++ b/bin/node/testing/src/bench.rs @@ -55,7 +55,7 @@ use sp_inherents::InherentData; use sp_runtime::{ generic::BlockId, traits::{Block as BlockT, IdentifyAccount, Verify, Zero}, - OpaqueExtrinsic, + OpaqueExtrinsic, StateVersions, StateVersion, }; /// Keyring full of accounts for benching. @@ -387,7 +387,8 @@ impl BenchDb { }; let task_executor = TaskExecutor::new(); - let state_versions = vec![(BlockNumber::zero(), None)]; // TODO switch to default (more recent). + let mut state_versions = StateVersions::default(); + state_versions.add((BlockNumber::zero(), StateVersion::V0)); // TODO switch to default (more recent). let backend = sc_service::new_db_backend(db_config, state_versions).expect("Should not fail"); let client = sc_service::new_client( backend.clone(), diff --git a/bin/node/testing/src/client.rs b/bin/node/testing/src/client.rs index 8bcd4c28ae46f..df4c93874c388 100644 --- a/bin/node/testing/src/client.rs +++ b/bin/node/testing/src/client.rs @@ -48,7 +48,7 @@ pub struct GenesisParameters { impl substrate_test_client::GenesisInit for GenesisParameters { fn genesis_storage(&self) -> Storage { - let state_version = Some(Some(sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD)); + let state_version = Default::default(); crate::genesis::config(self.support_changes_trie, None).build_storage(state_version).unwrap() } } diff --git a/client/chain-spec/src/chain_spec.rs b/client/chain-spec/src/chain_spec.rs index d990338a0f4fd..14dd3c0d0a85f 100644 --- a/client/chain-spec/src/chain_spec.rs +++ b/client/chain-spec/src/chain_spec.rs @@ -28,7 +28,7 @@ use sp_core::{ storage::{ChildInfo, Storage, StorageChild, StorageData, StorageKey}, Bytes, }; -use sp_runtime::{BuildStorage, StateVersions, StateVersion}; +use sp_runtime::{BuildStorage, StateVersion}; use std::{borrow::Cow, collections::HashMap, fs::File, path::PathBuf, sync::Arc}; enum GenesisSource { @@ -264,7 +264,7 @@ impl ChainSpec { consensus_engine: (), genesis: Default::default(), code_substitutes: HashMap::new(), - state_versions: vec![("0".to_string(), Some(Some(sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD)))], + state_versions: vec![("0".to_string(), StateVersion::default())], }; ChainSpec { client_spec, genesis: GenesisSource::Factory(Arc::new(constructor)) } @@ -277,9 +277,8 @@ impl ChainSpec { /// Defined state version for the chain. /// Return None on invalid definition. - fn state_versions(&self) -> Option { - StateVersions::from_conf(self.client_spec.state_versions - .iter().map(|(number, version)| (number.as_str(), *version))) + fn state_versions(&self) -> &Vec<(String, StateVersion)> { + &self.client_spec.state_versions } } @@ -312,9 +311,13 @@ impl ChainSpec { fn json_container(&self, raw: bool) -> Result, String> { let genesis = match (raw, self.genesis.resolve()?) { (true, Genesis::Runtime(g)) => { + use std::str::FromStr; let state_version = self.state_versions().get(0) - .and_then(|(n, s)| (n == &0).then(|| s.clone())) - .unwrap_or(None); + // This is incorrect (can have number representation not compatible with u64 + .and_then(|(n, s)| u64::from_str(n).ok().map(|n| (n, s))) + .and_then(|(n, s)| (n == 0).then(|| s.clone())) + .unwrap_or_default(); + // TODO remove param from build storage and make it part of caller. let storage = g.build_storage(state_version)?; let top = storage.top.into_iter().map(|(k, v)| (StorageKey(k), StorageData(v))).collect(); @@ -416,7 +419,7 @@ where .collect() } - fn state_versions(&self) -> Vec<(u64, StateVersion)> { + fn state_versions(&self) -> &Vec<(String, StateVersion)> { ChainSpec::state_versions(self) } } diff --git a/client/chain-spec/src/lib.rs b/client/chain-spec/src/lib.rs index aba5849fa82da..c64a8909e0aa1 100644 --- a/client/chain-spec/src/lib.rs +++ b/client/chain-spec/src/lib.rs @@ -120,7 +120,8 @@ use sc_network::config::MultiaddrWithPeerId; use sc_telemetry::TelemetryEndpoints; use serde::{de::DeserializeOwned, Serialize}; use sp_core::storage::Storage; -use sp_runtime::{BuildStorage, StateVersions}; +use sp_core::state_version::StateVersion; +use sp_runtime::BuildStorage; /// The type of a chain. /// @@ -189,7 +190,7 @@ pub trait ChainSpec: BuildStorage + Send + Sync { fn code_substitutes(&self) -> std::collections::HashMap>; /// State versions for the chain. /// Uses default state version when no definition from block 0. - fn state_versions(&self) -> StateVersions; + fn state_versions(&self) -> &Vec<(String, StateVersion)>; } impl std::fmt::Debug for dyn ChainSpec { diff --git a/client/db/src/bench.rs b/client/db/src/bench.rs index 707fcea035a0a..6588fcd411905 100644 --- a/client/db/src/bench.rs +++ b/client/db/src/bench.rs @@ -34,7 +34,7 @@ use sp_core::{ }; use sp_runtime::{ traits::{Block as BlockT, HashFor}, - Storage, + Storage, StateVersion, }; use sp_state_machine::{ backend::Backend as StateBackend, ChildStorageCollection, DBValue, ProofRecorder, @@ -121,8 +121,8 @@ impl BenchmarkingState { db: Cell::new(None), root: Cell::new(root.clone()), // use old state for now to keep similar benches. - // will need update to heavier of both world before migrating. - state_version: Cell::new(None), + // TODO will need update to heavier of both world before migrating. + state_version: Cell::new(StateVersion::V0), genesis: Default::default(), genesis_root: Default::default(), record: Default::default(), diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index ccfbfa3f8cf21..379e00ea9f94e 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -88,7 +88,7 @@ use sp_runtime::{ Block as BlockT, Hash, HashFor, Header as HeaderT, NumberFor, One, SaturatedConversion, Zero, }, - Justification, Justifications, Storage, + Justification, Justifications, Storage, StateVersion, StateVersions, }; use sp_state_machine::{ backend::Backend as StateBackend, ChangesTrieCacheAction, ChangesTrieTransaction, @@ -1119,14 +1119,14 @@ pub struct Backend { state_usage: Arc, genesis_state: RwLock>>>, // TODO consider moving this state_version into BlockChainBb - state_versions: StateVersions, + state_versions: StateVersions, } impl Backend { /// Create a new instance of database backend. /// /// The pruning window is how old a block must be before the state is pruned. - pub fn new(config: DatabaseSettings, canonicalization_delay: u64, state_versions: StateVersions) -> ClientResult { + pub fn new(config: DatabaseSettings, canonicalization_delay: u64, state_versions: StateVersions) -> ClientResult { // TODO state_versions could also be part of database settings let db = crate::utils::open_database::(&config, DatabaseType::Full)?; Self::from_database(db as Arc<_>, canonicalization_delay, &config, state_versions) @@ -1168,7 +1168,7 @@ impl Backend { db: Arc>, canonicalization_delay: u64, config: &DatabaseSettings, - state_versions: StateVersions, // TODO could also be part of database settings + state_versions: StateVersions, // TODO could also be part of database settings ) -> ClientResult { let is_archive_pruning = config.state_pruning.is_archive(); let blockchain = BlockchainDb::new(db.clone(), config.transaction_storage.clone())?; @@ -2582,7 +2582,7 @@ pub(crate) mod tests { transaction_storage: TransactionStorageMode::BlockBody, }, 0, - vec![], + Default::default(), ) .unwrap(); assert_eq!(backend.blockchain().info().best_number, 9); diff --git a/client/db/src/storage_cache.rs b/client/db/src/storage_cache.rs index f4bfcf35ed960..94c2de74820d5 100644 --- a/client/db/src/storage_cache.rs +++ b/client/db/src/storage_cache.rs @@ -968,11 +968,12 @@ mod tests { let h3b = H256::random(); let shared = new_shared_cache::(256 * 1024, (0, 1)); + let state_version = StateVersion::default(); // blocks [ 3a(c) 2a(c) 2b 1b 1a(c) 0 ] // state [ 5 5 4 3 2 2 ] let mut s = CachingState::new( - InMemoryBackend::::from(None), // TODO replace all test new in mem by default (restoring implementation in state-machine/in_memory_backend.rs) + InMemoryBackend::::from(state_version), shared.clone(), Some(root_parent), ); @@ -987,11 +988,11 @@ mod tests { ); let mut s = - CachingState::new(InMemoryBackend::::from(None), shared.clone(), Some(h0)); + CachingState::new(InMemoryBackend::::from(state_version), shared.clone(), Some(h0)); s.cache.sync_cache(&[], &[], vec![], vec![], Some(h1a), Some(1), true); let mut s = - CachingState::new(InMemoryBackend::::from(None), shared.clone(), Some(h0)); + CachingState::new(InMemoryBackend::::from(state_version), shared.clone(), Some(h0)); s.cache.sync_cache( &[], &[], @@ -1003,7 +1004,7 @@ mod tests { ); let mut s = - CachingState::new(InMemoryBackend::::from(None), shared.clone(), Some(h1b)); + CachingState::new(InMemoryBackend::::from(state_version), shared.clone(), Some(h1b)); s.cache.sync_cache( &[], &[], @@ -1015,7 +1016,7 @@ mod tests { ); let mut s = - CachingState::new(InMemoryBackend::::from(None), shared.clone(), Some(h1a)); + CachingState::new(InMemoryBackend::::from(state_version), shared.clone(), Some(h1a)); s.cache.sync_cache( &[], &[], @@ -1027,29 +1028,29 @@ mod tests { ); let mut s = - CachingState::new(InMemoryBackend::::from(None), shared.clone(), Some(h2a)); + CachingState::new(InMemoryBackend::::from(state_version), shared.clone(), Some(h2a)); s.cache.sync_cache(&[], &[], vec![], vec![], Some(h3a), Some(3), true); let s = - CachingState::new(InMemoryBackend::::from(None), shared.clone(), Some(h3a)); + CachingState::new(InMemoryBackend::::from(state_version), shared.clone(), Some(h3a)); assert_eq!(s.storage(&key).unwrap().unwrap(), vec![5]); let s = - CachingState::new(InMemoryBackend::::from(None), shared.clone(), Some(h1a)); + CachingState::new(InMemoryBackend::::from(state_version), shared.clone(), Some(h1a)); assert!(s.storage(&key).unwrap().is_none()); let s = - CachingState::new(InMemoryBackend::::from(None), shared.clone(), Some(h2b)); + CachingState::new(InMemoryBackend::::from(state_version), shared.clone(), Some(h2b)); assert!(s.storage(&key).unwrap().is_none()); let s = - CachingState::new(InMemoryBackend::::from(None), shared.clone(), Some(h1b)); + CachingState::new(InMemoryBackend::::from(state_version), shared.clone(), Some(h1b)); assert!(s.storage(&key).unwrap().is_none()); // reorg to 3b // blocks [ 3b(c) 3a 2a 2b(c) 1b 1a 0 ] let mut s = - CachingState::new(InMemoryBackend::::from(None), shared.clone(), Some(h2b)); + CachingState::new(InMemoryBackend::::from(state_version), shared.clone(), Some(h2b)); s.cache.sync_cache( &[h1b, h2b, h3b], &[h1a, h2a, h3a], @@ -1060,7 +1061,7 @@ mod tests { true, ); let s = - CachingState::new(InMemoryBackend::::from(None), shared.clone(), Some(h3a)); + CachingState::new(InMemoryBackend::::from(state_version), shared.clone(), Some(h3a)); assert!(s.storage(&key).unwrap().is_none()); } @@ -1076,9 +1077,10 @@ mod tests { let h3b = H256::random(); let shared = new_shared_cache::(256 * 1024, (0, 1)); + let state_version = StateVersion::default(); let mut s = CachingState::new( - InMemoryBackend::::from(None), + InMemoryBackend::::from(state_version), shared.clone(), Some(root_parent), ); @@ -1093,11 +1095,11 @@ mod tests { ); let mut s = - CachingState::new(InMemoryBackend::::from(None), shared.clone(), Some(h1)); + CachingState::new(InMemoryBackend::::from(state_version), shared.clone(), Some(h1)); s.cache.sync_cache(&[], &[], vec![], vec![], Some(h2a), Some(2), true); let mut s = - CachingState::new(InMemoryBackend::::from(None), shared.clone(), Some(h1)); + CachingState::new(InMemoryBackend::::from(state_version), shared.clone(), Some(h1)); s.cache.sync_cache( &[], &[], @@ -1109,7 +1111,7 @@ mod tests { ); let mut s = - CachingState::new(InMemoryBackend::::from(None), shared.clone(), Some(h2b)); + CachingState::new(InMemoryBackend::::from(state_version), shared.clone(), Some(h2b)); s.cache.sync_cache( &[], &[], @@ -1121,7 +1123,7 @@ mod tests { ); let s = - CachingState::new(InMemoryBackend::::from(None), shared.clone(), Some(h2a)); + CachingState::new(InMemoryBackend::::from(state_version), shared.clone(), Some(h2a)); assert_eq!(s.storage(&key).unwrap().unwrap(), vec![2]); } @@ -1136,20 +1138,21 @@ mod tests { let h3b = H256::random(); let shared = new_shared_cache::(256 * 1024, (0, 1)); + let state_version = StateVersion::default(); let mut s = CachingState::new( - InMemoryBackend::::from(None), + InMemoryBackend::::from(state_version), shared.clone(), Some(root_parent), ); s.cache.sync_cache(&[], &[], vec![], vec![], Some(h1), Some(1), true); let mut s = - CachingState::new(InMemoryBackend::::from(None), shared.clone(), Some(h1)); + CachingState::new(InMemoryBackend::::from(state_version), shared.clone(), Some(h1)); s.cache.sync_cache(&[], &[], vec![], vec![], Some(h2a), Some(2), true); let mut s = - CachingState::new(InMemoryBackend::::from(None), shared.clone(), Some(h2a)); + CachingState::new(InMemoryBackend::::from(state_version), shared.clone(), Some(h2a)); s.cache.sync_cache( &[], &[], @@ -1161,11 +1164,11 @@ mod tests { ); let mut s = - CachingState::new(InMemoryBackend::::from(None), shared.clone(), Some(h1)); + CachingState::new(InMemoryBackend::::from(state_version), shared.clone(), Some(h1)); s.cache.sync_cache(&[], &[], vec![], vec![], Some(h2b), Some(2), false); let mut s = - CachingState::new(InMemoryBackend::::from(None), shared.clone(), Some(h2b)); + CachingState::new(InMemoryBackend::::from(state_version), shared.clone(), Some(h2b)); s.cache.sync_cache( &[], &[], @@ -1177,7 +1180,7 @@ mod tests { ); let s = - CachingState::new(InMemoryBackend::::from(None), shared.clone(), Some(h3a)); + CachingState::new(InMemoryBackend::::from(state_version), shared.clone(), Some(h3a)); assert_eq!(s.storage(&key).unwrap().unwrap(), vec![2]); } @@ -1189,7 +1192,7 @@ mod tests { let h1b = H256::random(); let shared = new_shared_cache::(256 * 1024, (0, 1)); - let mut backend = InMemoryBackend::::from(None); + let mut backend = InMemoryBackend::::from(StateVersion::default()); backend.insert(std::iter::once((None, vec![(key.clone(), Some(vec![1]))]))); let mut s = CachingState::new(backend.clone(), shared.clone(), Some(root_parent)); @@ -1217,7 +1220,7 @@ mod tests { let h0 = H256::random(); let mut s = CachingState::new( - InMemoryBackend::::from(None), + InMemoryBackend::::from(StateVersion::default()), shared.clone(), Some(root_parent.clone()), ); @@ -1257,7 +1260,7 @@ mod tests { let h0 = H256::random(); let mut s = CachingState::new( - InMemoryBackend::::from(None), + InMemoryBackend::::from(StateVersion::default()), shared.clone(), Some(root_parent), ); @@ -1300,8 +1303,9 @@ mod tests { let h1 = H256::random(); let shared = new_shared_cache::(256 * 1024, (0, 1)); + let state_version = StateVersion::default(); let mut s = CachingState::new( - InMemoryBackend::::from(None), + InMemoryBackend::::from(state_version), shared.clone(), Some(root_parent.clone()), ); @@ -1316,7 +1320,7 @@ mod tests { ); let mut s = - CachingState::new(InMemoryBackend::::from(None), shared.clone(), Some(h0)); + CachingState::new(InMemoryBackend::::from(state_version), shared.clone(), Some(h0)); s.cache.sync_cache( &[], &[], @@ -1328,7 +1332,7 @@ mod tests { ); let mut s = - CachingState::new(InMemoryBackend::::from(None), shared.clone(), Some(h1)); + CachingState::new(InMemoryBackend::::from(state_version), shared.clone(), Some(h1)); assert_eq!(s.storage(&key).unwrap(), Some(vec![3])); // Restart (or unknown block?), clear caches. @@ -1348,7 +1352,7 @@ mod tests { s.cache.sync_cache(&[], &[], vec![], vec![], None, None, true); let s = - CachingState::new(InMemoryBackend::::from(None), shared.clone(), Some(h1)); + CachingState::new(InMemoryBackend::::from(state_version), shared.clone(), Some(h1)); assert_eq!(s.storage(&key).unwrap(), None); } @@ -1362,9 +1366,10 @@ mod tests { let h2 = H256::random(); let shared = new_shared_cache::(256 * 1024, (0, 1)); + let state_version = StateVersion::default(); let mut s = CachingState::new( - InMemoryBackend::::from(None), + InMemoryBackend::::from(state_version), shared.clone(), Some(root_parent), ); @@ -1380,7 +1385,7 @@ mod tests { assert_eq!(shared.write().lru_storage.get(&key).unwrap(), &Some(vec![1])); let mut s = - CachingState::new(InMemoryBackend::::from(None), shared.clone(), Some(h1)); + CachingState::new(InMemoryBackend::::from(state_version), shared.clone(), Some(h1)); // commit as non-best s.cache.sync_cache( @@ -1396,7 +1401,7 @@ mod tests { assert_eq!(shared.write().lru_storage.get(&key).unwrap(), &Some(vec![1])); let mut s = - CachingState::new(InMemoryBackend::::from(None), shared.clone(), Some(h1)); + CachingState::new(InMemoryBackend::::from(state_version), shared.clone(), Some(h1)); // commit again as best with no changes s.cache.sync_cache(&[], &[], vec![], vec![], Some(h2), Some(2), true); @@ -1527,8 +1532,9 @@ mod qc { } fn head_state(&self, hash: H256) -> CachingState, Block> { + let state_version = StateVersion::default(); CachingState::new( - InMemoryBackend::::from(None), + InMemoryBackend::::from(state_version), self.shared.clone(), Some(hash), ) @@ -1562,6 +1568,7 @@ mod qc { &mut self, action: Action, ) -> Result, Block>, ()> { + let state_version = StateVersion::default(); let state = match action { Action::Fork { depth, hash, changes } => { let pos = self.canon.len() as isize - depth as isize; @@ -1599,7 +1606,7 @@ mod qc { }; let mut state = CachingState::new( - InMemoryBackend::::from(None), + InMemoryBackend::::from(state_version), self.shared.clone(), Some(parent), ); @@ -1636,7 +1643,7 @@ mod qc { } let mut state = CachingState::new( - InMemoryBackend::::from(None), + InMemoryBackend::::from(state_version), self.shared.clone(), Some(parent_hash), ); @@ -1687,7 +1694,7 @@ mod qc { self.canon.push(node); let mut state = CachingState::new( - InMemoryBackend::::from(None), + InMemoryBackend::::from(state_version), self.shared.clone(), Some(fork_at), ); diff --git a/client/executor/src/integration_tests/mod.rs b/client/executor/src/integration_tests/mod.rs index c4ded542d3946..454a5cfcfe6b5 100644 --- a/client/executor/src/integration_tests/mod.rs +++ b/client/executor/src/integration_tests/mod.rs @@ -198,7 +198,7 @@ fn storage_should_work(wasm_method: WasmExecutionMethod) { ], children_default: map![], }; - let expected = TestExternalities::new(storage, Some(Some(sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD))); + let expected = TestExternalities::new(storage, Default::default()); assert_eq!(ext, expected); } @@ -230,7 +230,7 @@ fn clear_prefix_should_work(wasm_method: WasmExecutionMethod) { children_default: map![], }; - let expected = TestExternalities::new(storage, Some(Some(sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD))); + let expected = TestExternalities::new(storage, Default::default()); assert_eq!(expected, ext); } diff --git a/client/light/src/backend.rs b/client/light/src/backend.rs index 8048eb66f15b5..69ea067fe116f 100644 --- a/client/light/src/backend.rs +++ b/client/light/src/backend.rs @@ -62,7 +62,7 @@ const IN_MEMORY_EXPECT_PROOF: &str = pub struct Backend { blockchain: Arc>, genesis_state: RwLock>>>, - state_versions: StateVersions, + state_versions: StateVersions, // TODO consider moving this state_versions into BlockChain (and add N to type) -> would make // better api import_lock: RwLock<()>, @@ -79,6 +79,7 @@ pub struct ImportOperation { storage_update: Option>>, changes_trie_config_update: Option>, state_version: Option, + genesis_state_version: StateVersion, _phantom: std::marker::PhantomData, } @@ -93,7 +94,7 @@ pub enum GenesisOrUnavailableState { impl Backend { /// Create new light backend. - pub fn new(blockchain: Arc>, state_versions: StateVersions) -> Self { + pub fn new(blockchain: Arc>, state_versions: StateVersions) -> Self { Self { blockchain, genesis_state: RwLock::new(None), import_lock: Default::default(), state_versions } } @@ -145,6 +146,7 @@ where storage_update: None, changes_trie_config_update: None, state_version: None, + genesis_state_version: self.state_versions.genesis_state_version(), _phantom: Default::default(), }) } @@ -156,7 +158,7 @@ where ) -> ClientResult<()> { if let Some(number) = self.blockchain.storage().block_number_from_id(&block)? { - operation.state_version = self.state_versions.state_version_at(number); + operation.state_version = Some(self.state_versions.state_version_at(number)); } Ok(()) } @@ -364,7 +366,7 @@ where storage.insert(Some(storage_child.child_info), storage_child.data); } - let storage_update = InMemoryBackend::from((storage, self.state_version)); + let storage_update = InMemoryBackend::from((storage, self.genesis_state_version)); let (storage_root, _) = storage_update.full_storage_root(std::iter::empty(), child_delta); if commit { self.storage_update = Some(storage_update); @@ -588,7 +590,7 @@ where fn state_version(&self) -> StateVersion { match self { GenesisOrUnavailableState::Genesis(state) => state.state_version(), - GenesisOrUnavailableState::Unavailable => StateVersion::Default(), + GenesisOrUnavailableState::Unavailable => StateVersion::default(), } } } diff --git a/client/light/src/fetcher.rs b/client/light/src/fetcher.rs index b353a41fa7ae2..be041b6c82989 100644 --- a/client/light/src/fetcher.rs +++ b/client/light/src/fetcher.rs @@ -28,7 +28,7 @@ use codec::{Decode, Encode}; use hash_db::{HashDB, Hasher, EMPTY_PREFIX}; use sp_blockchain::{Error as ClientError, Result as ClientResult}; use sp_core::{ - convert_hash, + convert_hash, state_version::StateVersion, storage::{ChildInfo, ChildType}, traits::{CodeExecutor, SpawnNamed}, }; @@ -202,7 +202,7 @@ impl> LightDataChecker { } // check proof for single changes trie root - let proving_backend = TrieBackend::new(storage, local_cht_root, None); + let proving_backend = TrieBackend::new(storage, local_cht_root, StateVersion::V0); let remote_changes_trie_root = remote_roots[&block]; cht::check_proof_on_proving_backend::>( local_cht_root, diff --git a/client/light/src/lib.rs b/client/light/src/lib.rs index f06995fb9d38a..83e1371b69a61 100644 --- a/client/light/src/lib.rs +++ b/client/light/src/lib.rs @@ -19,7 +19,7 @@ //! Light client components. use sp_core::traits::{CodeExecutor, SpawnNamed}; -use sp_runtime::traits::{NumberFor, Block as BlockT}; +use sp_runtime::traits::Block as BlockT; use std::sync::Arc; pub mod backend; @@ -50,7 +50,7 @@ pub fn new_light_blockchain>(storage: S) -> A } /// Create an instance of light client backend. -pub fn new_light_backend(blockchain: Arc>, state_versions: sp_runtime::StateVersions) -> Arc> +pub fn new_light_backend(blockchain: Arc>, state_versions: sp_runtime::StateVersions) -> Arc> where B: BlockT, S: BlockchainStorage, diff --git a/client/network/src/light_client_requests/handler.rs b/client/network/src/light_client_requests/handler.rs index d533bd05b433d..4359ca1a9aa46 100644 --- a/client/network/src/light_client_requests/handler.rs +++ b/client/network/src/light_client_requests/handler.rs @@ -181,7 +181,7 @@ impl LightClientRequestHandler { request.block, e, ); - StorageProof::empty(Some(Some(sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD))) + StorageProof::empty() }, }; @@ -225,7 +225,7 @@ impl LightClientRequestHandler { request.block, error, ); - StorageProof::empty(Some(Some(sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD))) + StorageProof::empty() }, }; @@ -279,7 +279,7 @@ impl LightClientRequestHandler { request.block, error, ); - StorageProof::empty(Some(Some(sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD))) + StorageProof::empty() }, }; @@ -308,7 +308,7 @@ impl LightClientRequestHandler { request.block, error ); - (Default::default(), StorageProof::empty(Some(Some(sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD)))) + (Default::default(), StorageProof::empty()) }, }; @@ -369,7 +369,7 @@ impl LightClientRequestHandler { max_block: Zero::zero(), proof: Vec::new(), roots: BTreeMap::new(), - roots_proof: StorageProof::empty(Some(Some(sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD))), + roots_proof: StorageProof::empty(), } }, }; diff --git a/client/network/src/light_client_requests/sender.rs b/client/network/src/light_client_requests/sender.rs index 3e6c069bcdc16..9e3185d94438a 100644 --- a/client/network/src/light_client_requests/sender.rs +++ b/client/network/src/light_client_requests/sender.rs @@ -830,7 +830,7 @@ mod tests { use std::{collections::HashSet, iter::FromIterator}; fn empty_proof() -> Vec { - StorageProof::empty(None).encode() + StorageProof::empty().encode() } #[test] diff --git a/client/service/src/builder.rs b/client/service/src/builder.rs index 2d9f0a204ec2b..5c0cd70be3429 100644 --- a/client/service/src/builder.rs +++ b/client/service/src/builder.rs @@ -58,7 +58,7 @@ use sp_core::traits::{CodeExecutor, SpawnNamed}; use sp_keystore::{CryptoStore, SyncCryptoStore, SyncCryptoStorePtr}; use sp_runtime::{ generic::BlockId, - traits::{Block as BlockT, BlockIdTo, Zero, NumberFor}, + traits::{Block as BlockT, BlockIdTo, Zero}, BuildStorage, StateVersions, }; @@ -318,7 +318,8 @@ where transaction_storage: config.transaction_storage.clone(), }; - let state_versions = config.chain_spec.state_versions() + let state_versions = StateVersions::from_conf(config.chain_spec.state_versions() + .iter().map(|(number, version)| (number.as_str(), *version))) .ok_or_else(|| Error::Application(Box::from("Invalid state versions for chain spec".to_string())))?; let backend = new_db_backend(db_config, state_versions.clone())?; @@ -412,7 +413,9 @@ where Box::new(task_manager.spawn_handle()), )); let on_demand = Arc::new(sc_network::config::OnDemand::new(fetch_checker)); - let state_versions = config.chain_spec.state_versions() + + let state_versions = StateVersions::from_conf(config.chain_spec.state_versions() + .iter().map(|(number, version)| (number.as_str(), *version))) .ok_or_else(|| Error::Application(Box::from("Invalid state versions for chain spec".to_string())))?; let backend = sc_light::new_light_backend(light_blockchain, state_versions); diff --git a/client/service/src/client/client.rs b/client/service/src/client/client.rs index eb4c197d66bb9..dba698aa81d63 100644 --- a/client/service/src/client/client.rs +++ b/client/service/src/client/client.rs @@ -200,7 +200,7 @@ pub struct ClientConfig { /// version doesn't match anymore. pub wasm_runtime_substitutes: HashMap>, /// State version to use with chain. - pub state_versions: sp_runtime::StateVersions, + pub state_versions: sp_runtime::StateVersions, } impl Default for ClientConfig { @@ -211,7 +211,7 @@ impl Default for ClientConfig { wasm_runtime_overrides: None, no_genesis: false, wasm_runtime_substitutes: HashMap::new(), - state_versions: Vec::new(), // TODO check usage or remove default impl + state_versions: Default::default(), // TODO check usage or remove default impl } } } @@ -337,9 +337,7 @@ where config: ClientConfig, ) -> sp_blockchain::Result { let info = backend.blockchain().info(); - let state_version = config.state_versions.get(0) - .and_then(|(n, s)| n.is_zero().then(|| s.clone())) - .unwrap_or(None); + let state_version = config.state_versions.genesis_state_version(); if info.finalized_state.is_none() { let genesis_storage = diff --git a/client/service/src/client/wasm_override.rs b/client/service/src/client/wasm_override.rs index 7d9fd555ec7ca..9939df56b5d5f 100644 --- a/client/service/src/client/wasm_override.rs +++ b/client/service/src/client/wasm_override.rs @@ -181,7 +181,7 @@ where code: &WasmBlob, heap_pages: Option, ) -> Result { - let mut ext: BasicExternalities = None.into(); + let mut ext: BasicExternalities = sp_runtime::StateVersion::default().into(); executor .runtime_version(&mut ext, &code.runtime_code(heap_pages)) .map_err(|e| WasmOverrideError::VersionInvalid(format!("{:?}", e)).into()) diff --git a/client/service/src/client/wasm_substitutes.rs b/client/service/src/client/wasm_substitutes.rs index 076d0bdc647b4..b3e980b63c675 100644 --- a/client/service/src/client/wasm_substitutes.rs +++ b/client/service/src/client/wasm_substitutes.rs @@ -178,7 +178,7 @@ where executor: &Executor, code: &WasmSubstitute, ) -> Result { - let mut ext: BasicExternalities = None.into(); + let mut ext: BasicExternalities = sp_runtime::StateVersion::default().into(); executor .runtime_version(&mut ext, &code.runtime_code(None)) .map_err(|e| WasmSubstituteError::VersionInvalid(format!("{:?}", e)).into()) diff --git a/client/service/test/src/client/light.rs b/client/service/test/src/client/light.rs index 2871858157fe0..d57daaaa59adf 100644 --- a/client/service/test/src/client/light.rs +++ b/client/service/test/src/client/light.rs @@ -269,7 +269,7 @@ fn local_state_is_created_when_genesis_state_is_available() { substrate_test_runtime_client::runtime::Header::new(0, def, def, def, Default::default()); let backend: Backend<_, Block> = - Backend::new(Arc::new(DummyBlockchain::new(DummyStorage::new())), vec![]); + Backend::new(Arc::new(DummyBlockchain::new(DummyStorage::new())), Default::default()); let mut op = backend.begin_operation().unwrap(); op.set_block_data(header0, None, None, None, NewBlockState::Final).unwrap(); op.set_genesis_state(Default::default(), true).unwrap(); @@ -284,7 +284,7 @@ fn local_state_is_created_when_genesis_state_is_available() { #[test] fn unavailable_state_is_created_when_genesis_state_is_unavailable() { let backend: Backend<_, Block> = - Backend::new(Arc::new(DummyBlockchain::new(DummyStorage::new())), vec![]); + Backend::new(Arc::new(DummyBlockchain::new(DummyStorage::new())), Default::default()); match backend.state_at(BlockId::Number(0)).unwrap() { GenesisOrUnavailableState::Unavailable => (), @@ -294,7 +294,7 @@ fn unavailable_state_is_created_when_genesis_state_is_unavailable() { #[test] fn light_aux_store_is_updated_via_non_importing_op() { - let backend = Backend::new(Arc::new(DummyBlockchain::new(DummyStorage::new())), vec![]); + let backend = Backend::new(Arc::new(DummyBlockchain::new(DummyStorage::new())), Default::default()); let mut op = ClientBackend::::begin_operation(&backend).unwrap(); BlockImportOperation::::insert_aux(&mut op, vec![(vec![1], Some(vec![2]))]).unwrap(); ClientBackend::::commit_operation(&backend, op).unwrap(); @@ -891,7 +891,7 @@ fn check_changes_proof_fails_if_proof_is_wrong() { max_block: remote_proof.max_block, proof: remote_proof.proof.clone(), roots: vec![(begin - 1, Default::default())].into_iter().collect(), - roots_proof: StorageProof::empty(None), + roots_proof: StorageProof::empty(), } ) .is_err()); @@ -902,7 +902,7 @@ fn check_changes_proof_fails_if_proof_is_wrong() { max_block: remote_proof.max_block, proof: remote_proof.proof.clone(), roots: vec![(end + 1, Default::default())].into_iter().collect(), - roots_proof: StorageProof::empty(None), + roots_proof: StorageProof::empty(), } ) .is_err()); @@ -952,7 +952,7 @@ fn check_changes_tries_proof_fails_if_proof_is_wrong() { Box::new(TaskExecutor::new()), ); let result = - local_checker.check_changes_tries_proof(4, &remote_proof.roots, StorageProof::empty(None)); + local_checker.check_changes_tries_proof(4, &remote_proof.roots, StorageProof::empty()); assert!(result.is_err()); } diff --git a/client/service/test/src/client/mod.rs b/client/service/test/src/client/mod.rs index d8fffa5fefc2a..7f72bda7235b6 100644 --- a/client/service/test/src/client/mod.rs +++ b/client/service/test/src/client/mod.rs @@ -243,7 +243,7 @@ fn block1(genesis_hash: Hash, backend: &InMemoryBackend) -> (Vec Vec { } pub(crate) fn new_test_ext() -> sp_io::TestExternalities { - let state_version = None; - let t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); + let state_version = Default::default(); + let t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); let mut ext = sp_io::TestExternalities::new(t, state_version); ext.execute_with(|| System::set_block_number(1)); diff --git a/frame/atomic-swap/src/tests.rs b/frame/atomic-swap/src/tests.rs index 22bc399e5729b..bf09b007fd588 100644 --- a/frame/atomic-swap/src/tests.rs +++ b/frame/atomic-swap/src/tests.rs @@ -83,10 +83,10 @@ const A: u64 = 1; const B: u64 = 2; pub fn new_test_ext() -> sp_io::TestExternalities { - let state_version = None; - let mut t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); + let state_version = Default::default(); + let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); let genesis = pallet_balances::GenesisConfig:: { balances: vec![(A, 100), (B, 200)] }; - genesis.assimilate_storage(&mut t, state_version.clone()).unwrap(); + genesis.assimilate_storage(&mut t, state_version).unwrap(); (t, state_version).into() } diff --git a/frame/aura/src/mock.rs b/frame/aura/src/mock.rs index 3eb31893a5de2..38439c1f29ae8 100644 --- a/frame/aura/src/mock.rs +++ b/frame/aura/src/mock.rs @@ -116,12 +116,12 @@ impl pallet_aura::Config for Test { } pub fn new_test_ext(authorities: Vec) -> sp_io::TestExternalities { - let state_version = None; - let mut t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); + let state_version = Default::default(); + let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); pallet_aura::GenesisConfig:: { authorities: authorities.into_iter().map(|a| UintAuthorityId(a).to_public_key()).collect(), } - .assimilate_storage(&mut t, state_version.clone()) + .assimilate_storage(&mut t, state_version) .unwrap(); (t, state_version).into() } diff --git a/frame/authority-discovery/src/lib.rs b/frame/authority-discovery/src/lib.rs index 5d8eb708ede95..04d844d353627 100644 --- a/frame/authority-discovery/src/lib.rs +++ b/frame/authority-discovery/src/lib.rs @@ -245,7 +245,7 @@ mod tests { #[test] fn authorities_returns_current_and_next_authority_set() { - let state_version = None; + let state_version = Default::default(); // The whole authority discovery pallet ignores account ids, but we still need them for // `pallet_session::OneSessionHandler::on_new_session`, thus its safe to use the same value // everywhere. @@ -282,12 +282,12 @@ mod tests { .collect::>(); // Build genesis. - let mut t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); + let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); GenesisBuild::::assimilate_storage( &pallet_authority_discovery::GenesisConfig { keys: vec![] }, &mut t, - state_version.clone(), + state_version, ) .unwrap(); diff --git a/frame/babe/src/mock.rs b/frame/babe/src/mock.rs index 327c0779c2d2a..99d94beefa69b 100644 --- a/frame/babe/src/mock.rs +++ b/frame/babe/src/mock.rs @@ -380,13 +380,13 @@ pub fn new_test_ext_with_pairs( } pub fn new_test_ext_raw_authorities(authorities: Vec) -> sp_io::TestExternalities { - let state_version = None; - let mut t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); + let state_version = Default::default(); + let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); let balances: Vec<_> = (0..authorities.len()).map(|i| (i as u64, 10_000_000)).collect(); pallet_balances::GenesisConfig:: { balances } - .assimilate_storage(&mut t, state_version.clone()) + .assimilate_storage(&mut t, state_version) .unwrap(); // stashes are the index. @@ -401,7 +401,7 @@ pub fn new_test_ext_raw_authorities(authorities: Vec) -> sp_io::Tes // NOTE: this will initialize the babe authorities // through OneSessionHandler::on_genesis_session pallet_session::GenesisConfig:: { keys: session_keys } - .assimilate_storage(&mut t, state_version.clone()) + .assimilate_storage(&mut t, state_version) .unwrap(); // controllers are the index + 1000 @@ -420,7 +420,7 @@ pub fn new_test_ext_raw_authorities(authorities: Vec) -> sp_io::Tes ..Default::default() }; - staking_config.assimilate_storage(&mut t, state_version.clone()).unwrap(); + staking_config.assimilate_storage(&mut t, state_version).unwrap(); (t, state_version).into() } diff --git a/frame/balances/src/tests.rs b/frame/balances/src/tests.rs index f2bfc8bfd5339..46d87df97a670 100644 --- a/frame/balances/src/tests.rs +++ b/frame/balances/src/tests.rs @@ -629,9 +629,9 @@ macro_rules! decl_tests { #[test] #[should_panic = "the balance of any account should always be at least the existential deposit."] fn cannot_set_genesis_value_below_ed() { - let state_version = None; + let state_version = Default::default(); ($existential_deposit).with(|v| *v.borrow_mut() = 11); - let mut t = frame_system::GenesisConfig::default().build_storage::<$test>(state_version.clone()).unwrap(); + let mut t = frame_system::GenesisConfig::default().build_storage::<$test>(state_version).unwrap(); let _ = pallet_balances::GenesisConfig::<$test> { balances: vec![(1, 10)], }.assimilate_storage(&mut t, state_version).unwrap(); @@ -640,8 +640,8 @@ macro_rules! decl_tests { #[test] #[should_panic = "duplicate balances in genesis."] fn cannot_set_genesis_value_twice() { - let state_version = None; - let mut t = frame_system::GenesisConfig::default().build_storage::<$test>(state_version.clone()).unwrap(); + let state_version = Default::default(); + let mut t = frame_system::GenesisConfig::default().build_storage::<$test>(state_version).unwrap(); let _ = pallet_balances::GenesisConfig::<$test> { balances: vec![(1, 10), (2, 20), (1, 15)], }.assimilate_storage(&mut t, state_version).unwrap(); diff --git a/frame/balances/src/tests_composite.rs b/frame/balances/src/tests_composite.rs index 720e0738a92a2..955f90d97766c 100644 --- a/frame/balances/src/tests_composite.rs +++ b/frame/balances/src/tests_composite.rs @@ -122,9 +122,9 @@ impl ExtBuilder { EXISTENTIAL_DEPOSIT.with(|v| *v.borrow_mut() = self.existential_deposit); } pub fn build(self) -> sp_io::TestExternalities { - let state_version = None; + let state_version = Default::default(); self.set_associated_consts(); - let mut t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); + let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); pallet_balances::GenesisConfig:: { balances: if self.monied { vec![ @@ -138,7 +138,7 @@ impl ExtBuilder { vec![] }, } - .assimilate_storage(&mut t, state_version.clone()) + .assimilate_storage(&mut t, state_version) .unwrap(); let mut ext = sp_io::TestExternalities::new(t, state_version); diff --git a/frame/balances/src/tests_local.rs b/frame/balances/src/tests_local.rs index 9fd41d7ef9e1a..a399a3f991055 100644 --- a/frame/balances/src/tests_local.rs +++ b/frame/balances/src/tests_local.rs @@ -127,9 +127,9 @@ impl ExtBuilder { EXISTENTIAL_DEPOSIT.with(|v| *v.borrow_mut() = self.existential_deposit); } pub fn build(self) -> sp_io::TestExternalities { - let state_version = None; + let state_version = Default::default(); self.set_associated_consts(); - let mut t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); + let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); pallet_balances::GenesisConfig:: { balances: if self.monied { vec![ @@ -143,7 +143,7 @@ impl ExtBuilder { vec![] }, } - .assimilate_storage(&mut t, state_version.clone()) + .assimilate_storage(&mut t, state_version) .unwrap(); let mut ext = sp_io::TestExternalities::new(t, state_version); diff --git a/frame/balances/src/tests_reentrancy.rs b/frame/balances/src/tests_reentrancy.rs index c5794a9d3ca16..42ed624875a26 100644 --- a/frame/balances/src/tests_reentrancy.rs +++ b/frame/balances/src/tests_reentrancy.rs @@ -130,11 +130,11 @@ impl ExtBuilder { } pub fn build(self) -> sp_io::TestExternalities { - let state_version = None; + let state_version = Default::default(); self.set_associated_consts(); - let mut t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); + let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); pallet_balances::GenesisConfig:: { balances: vec![] } - .assimilate_storage(&mut t, state_version.clone()) + .assimilate_storage(&mut t, state_version) .unwrap(); let mut ext = sp_io::TestExternalities::new(t, state_version); ext.execute_with(|| System::set_block_number(1)); diff --git a/frame/benchmarking/src/tests.rs b/frame/benchmarking/src/tests.rs index 5ff8a6dbd48dd..a7188c89df12f 100644 --- a/frame/benchmarking/src/tests.rs +++ b/frame/benchmarking/src/tests.rs @@ -128,8 +128,8 @@ impl pallet_test::OtherConfig for Test { } fn new_test_ext() -> sp_io::TestExternalities { - let state_version = None; - let storage = GenesisConfig::default().build_storage(state_version.clone()).unwrap(); + let state_version = Default::default(); + let storage = GenesisConfig::default().build_storage(state_version).unwrap(); (storage, state_version).into() } diff --git a/frame/benchmarking/src/tests_instance.rs b/frame/benchmarking/src/tests_instance.rs index ab4cb6de3e23c..02243da281e4e 100644 --- a/frame/benchmarking/src/tests_instance.rs +++ b/frame/benchmarking/src/tests_instance.rs @@ -128,8 +128,8 @@ impl pallet_test::OtherConfig for Test { } fn new_test_ext() -> sp_io::TestExternalities { - let state_version = None; - let storage = GenesisConfig::default().build_storage(state_version.clone()).unwrap(); + let state_version = Default::default(); + let storage = GenesisConfig::default().build_storage(state_version).unwrap(); (storage, state_version).into() } diff --git a/frame/bounties/src/tests.rs b/frame/bounties/src/tests.rs index 94ee6b150ca70..1d5b1ee5faa64 100644 --- a/frame/bounties/src/tests.rs +++ b/frame/bounties/src/tests.rs @@ -149,15 +149,15 @@ impl Config for Test { type TreasuryError = pallet_treasury::Error; pub fn new_test_ext() -> sp_io::TestExternalities { - let state_version = None; - let mut t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); + let state_version = Default::default(); + let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); pallet_balances::GenesisConfig:: { // Total issuance will be 200 with treasury account initialized at ED. balances: vec![(0, 100), (1, 98), (2, 1)], } - .assimilate_storage(&mut t, state_version.clone()) + .assimilate_storage(&mut t, state_version) .unwrap(); - GenesisBuild::::assimilate_storage(&pallet_treasury::GenesisConfig, &mut t, state_version.clone()).unwrap(); + GenesisBuild::::assimilate_storage(&pallet_treasury::GenesisConfig, &mut t, state_version).unwrap(); (t, state_version).into() } @@ -357,10 +357,10 @@ fn treasury_account_doesnt_get_deleted() { // This is useful for chain that will just update runtime. #[test] fn inexistent_account_works() { - let state_version = None; - let mut t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); + let state_version = Default::default(); + let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); pallet_balances::GenesisConfig:: { balances: vec![(0, 100), (1, 99), (2, 1)] } - .assimilate_storage(&mut t, state_version.clone()) + .assimilate_storage(&mut t, state_version) .unwrap(); // Treasury genesis config is not build thus treasury account does not exist let mut t: sp_io::TestExternalities = (t, state_version).into(); @@ -938,14 +938,14 @@ fn extend_expiry() { #[test] fn genesis_funding_works() { - let state_version = None; - let mut t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); + let state_version = Default::default(); + let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); let initial_funding = 100; pallet_balances::GenesisConfig:: { // Total issuance will be 200 with treasury account initialized with 100. balances: vec![(0, 100), (Treasury::account_id(), initial_funding)], } - .assimilate_storage(&mut t, state_version.clone()) + .assimilate_storage(&mut t, state_version) .unwrap(); GenesisBuild::::assimilate_storage(&pallet_treasury::GenesisConfig, &mut t, state_version).unwrap(); let mut t: sp_io::TestExternalities = (t, state_version).into(); diff --git a/frame/contracts/src/tests.rs b/frame/contracts/src/tests.rs index beace619ad4df..31742dc9d604e 100644 --- a/frame/contracts/src/tests.rs +++ b/frame/contracts/src/tests.rs @@ -336,11 +336,11 @@ impl ExtBuilder { EXISTENTIAL_DEPOSIT.with(|v| *v.borrow_mut() = self.existential_deposit); } pub fn build(self) -> sp_io::TestExternalities { - let state_version = None; + let state_version = Default::default(); self.set_associated_consts(); - let mut t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); + let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); pallet_balances::GenesisConfig:: { balances: vec![] } - .assimilate_storage(&mut t, state_version.clone()) + .assimilate_storage(&mut t, state_version) .unwrap(); let mut ext = sp_io::TestExternalities::new(t, state_version); ext.execute_with(|| System::set_block_number(1)); diff --git a/frame/democracy/src/tests.rs b/frame/democracy/src/tests.rs index bde28cd2a67aa..0f349a2ea5a8d 100644 --- a/frame/democracy/src/tests.rs +++ b/frame/democracy/src/tests.rs @@ -194,15 +194,15 @@ impl Config for Test { } pub fn new_test_ext() -> sp_io::TestExternalities { - let state_version = None; - let mut t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); + let state_version = Default::default(); + let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); pallet_balances::GenesisConfig:: { balances: vec![(1, 10), (2, 20), (3, 30), (4, 40), (5, 50), (6, 60)], } - .assimilate_storage(&mut t, state_version.clone()) + .assimilate_storage(&mut t, state_version) .unwrap(); pallet_democracy::GenesisConfig::::default() - .assimilate_storage(&mut t, state_version.clone()) + .assimilate_storage(&mut t, state_version) .unwrap(); let mut ext = sp_io::TestExternalities::new(t, state_version); ext.execute_with(|| System::set_block_number(1)); diff --git a/frame/election-provider-multi-phase/src/mock.rs b/frame/election-provider-multi-phase/src/mock.rs index 53dc66540db84..e2f022115f03a 100644 --- a/frame/election-provider-multi-phase/src/mock.rs +++ b/frame/election-provider-multi-phase/src/mock.rs @@ -511,10 +511,10 @@ impl ExtBuilder { self } pub fn build(self) -> sp_io::TestExternalities { - let state_version = None; + let state_version = Default::default(); sp_tracing::try_init_simple(); let mut storage = - frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); + frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); let _ = pallet_balances::GenesisConfig:: { balances: vec![ @@ -524,7 +524,7 @@ impl ExtBuilder { (9999, 100), ], } - .assimilate_storage(&mut storage, state_version.clone()); + .assimilate_storage(&mut storage, state_version); sp_io::TestExternalities::from((storage, state_version)) } diff --git a/frame/elections-phragmen/src/lib.rs b/frame/elections-phragmen/src/lib.rs index 174121282cf42..29f4827838655 100644 --- a/frame/elections-phragmen/src/lib.rs +++ b/frame/elections-phragmen/src/lib.rs @@ -1303,7 +1303,7 @@ mod tests { *m.borrow_mut() = self.genesis_members.iter().map(|(m, _)| m.clone()).collect::>() }); - let state_version = None; + let state_version = Default::default(); let mut ext: sp_io::TestExternalities = (GenesisConfig { balances: pallet_balances::GenesisConfig:: { balances: vec![ diff --git a/frame/elections/src/mock.rs b/frame/elections/src/mock.rs index bec7e56c5ecb0..09599e411be57 100644 --- a/frame/elections/src/mock.rs +++ b/frame/elections/src/mock.rs @@ -193,7 +193,7 @@ impl ExtBuilder { self } pub fn build(self) -> sp_io::TestExternalities { - let state_version = None; + let state_version = Default::default(); VOTING_BOND.with(|v| *v.borrow_mut() = self.voting_bond); VOTING_FEE.with(|v| *v.borrow_mut() = self.voting_fee); PRESENT_SLASH_PER_VOTER.with(|v| *v.borrow_mut() = self.bad_presentation_punishment); @@ -216,7 +216,7 @@ impl ExtBuilder { term_duration: 5, }, } - .build_storage(state_version.clone()) + .build_storage(state_version) .unwrap(), state_version) .into(); ext.execute_with(|| System::set_block_number(1)); diff --git a/frame/example/src/tests.rs b/frame/example/src/tests.rs index a9f26eef53dae..aa6988374ea8e 100644 --- a/frame/example/src/tests.rs +++ b/frame/example/src/tests.rs @@ -107,7 +107,7 @@ impl Config for Test { // This function basically just builds a genesis storage key/value store according to // our desired mockup. pub fn new_test_ext() -> sp_io::TestExternalities { - let state_version = None; + let state_version = Default::default(); let t = GenesisConfig { // We use default for brevity, but you can configure as desired if needed. system: Default::default(), @@ -119,7 +119,7 @@ pub fn new_test_ext() -> sp_io::TestExternalities { foo: 24, }, } - .build_storage(state_version.clone()) + .build_storage(state_version) .unwrap(); (t, state_version).into() } diff --git a/frame/executive/src/lib.rs b/frame/executive/src/lib.rs index d4b2a34f62772..d19dac3a20da5 100644 --- a/frame/executive/src/lib.rs +++ b/frame/executive/src/lib.rs @@ -811,10 +811,10 @@ mod tests { #[test] fn balance_transfer_dispatch_works() { - let state_version = None; - let mut t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); + let state_version = Default::default(); + let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); pallet_balances::GenesisConfig:: { balances: vec![(1, 211)] } - .assimilate_storage(&mut t, state_version.clone()) + .assimilate_storage(&mut t, state_version) .unwrap(); let xt = TestXt::new(Call::Balances(BalancesCall::transfer(2, 69)), sign_extra(1, 0, 0)); let weight = xt.get_dispatch_info().weight + @@ -840,10 +840,10 @@ mod tests { } fn new_test_ext(balance_factor: Balance) -> sp_io::TestExternalities { - let state_version = None; - let mut t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); + let state_version = Default::default(); + let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); pallet_balances::GenesisConfig:: { balances: vec![(1, 111 * balance_factor)] } - .assimilate_storage(&mut t, state_version.clone()) + .assimilate_storage(&mut t, state_version) .unwrap(); (t, state_version).into() } diff --git a/frame/gilt/src/mock.rs b/frame/gilt/src/mock.rs index 71e47ac6de642..f8bf3b972cafb 100644 --- a/frame/gilt/src/mock.rs +++ b/frame/gilt/src/mock.rs @@ -127,14 +127,14 @@ impl pallet_gilt::Config for Test { // This function basically just builds a genesis storage key/value store according to // our desired mockup. pub fn new_test_ext() -> sp_io::TestExternalities { - let state_version = None; - let mut t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); + let state_version = Default::default(); + let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); pallet_balances::GenesisConfig:: { balances: vec![(1, 100), (2, 100), (3, 100), (4, 100)], } - .assimilate_storage(&mut t, state_version.clone()) + .assimilate_storage(&mut t, state_version) .unwrap(); - GenesisBuild::::assimilate_storage(&crate::GenesisConfig, &mut t, state_version.clone()).unwrap(); + GenesisBuild::::assimilate_storage(&crate::GenesisConfig, &mut t, state_version).unwrap(); (t, state_version).into() } diff --git a/frame/grandpa/src/mock.rs b/frame/grandpa/src/mock.rs index a2b1a61ddac07..ee31eaffeb0bd 100644 --- a/frame/grandpa/src/mock.rs +++ b/frame/grandpa/src/mock.rs @@ -275,13 +275,13 @@ pub fn new_test_ext(vec: Vec<(u64, u64)>) -> sp_io::TestExternalities { } pub fn new_test_ext_raw_authorities(authorities: AuthorityList) -> sp_io::TestExternalities { - let state_version = None; - let mut t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); + let state_version = Default::default(); + let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); let balances: Vec<_> = (0..authorities.len()).map(|i| (i as u64, 10_000_000)).collect(); pallet_balances::GenesisConfig:: { balances } - .assimilate_storage(&mut t, state_version.clone()) + .assimilate_storage(&mut t, state_version) .unwrap(); // stashes are the index. @@ -300,7 +300,7 @@ pub fn new_test_ext_raw_authorities(authorities: AuthorityList) -> sp_io::TestEx // NOTE: this will initialize the grandpa authorities // through OneSessionHandler::on_genesis_session pallet_session::GenesisConfig:: { keys: session_keys } - .assimilate_storage(&mut t, state_version.clone()) + .assimilate_storage(&mut t, state_version) .unwrap(); // controllers are the index + 1000 @@ -319,7 +319,7 @@ pub fn new_test_ext_raw_authorities(authorities: AuthorityList) -> sp_io::TestEx ..Default::default() }; - staking_config.assimilate_storage(&mut t, state_version.clone()).unwrap(); + staking_config.assimilate_storage(&mut t, state_version).unwrap(); (t, state_version).into() } diff --git a/frame/identity/src/tests.rs b/frame/identity/src/tests.rs index ef14b7753edfa..4a7e85c6c1abb 100644 --- a/frame/identity/src/tests.rs +++ b/frame/identity/src/tests.rs @@ -118,12 +118,12 @@ impl pallet_identity::Config for Test { } pub fn new_test_ext() -> sp_io::TestExternalities { - let state_version = None; + let state_version = Default::default(); let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); pallet_balances::GenesisConfig:: { balances: vec![(1, 10), (2, 10), (3, 10), (10, 100), (20, 100), (30, 100)], } - .assimilate_storage(&mut t, None) + .assimilate_storage(&mut t, state_version) .unwrap(); (t, state_version).into() } diff --git a/frame/im-online/src/mock.rs b/frame/im-online/src/mock.rs index 7a4a50d0269f7..acade9ebd7588 100644 --- a/frame/im-online/src/mock.rs +++ b/frame/im-online/src/mock.rs @@ -105,8 +105,8 @@ impl ReportOffence for OffenceHandler { } pub fn new_test_ext() -> sp_io::TestExternalities { - let state_version = None; - let t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); + let state_version = Default::default(); + let t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); (t, state_version).into() } diff --git a/frame/indices/src/mock.rs b/frame/indices/src/mock.rs index 497c239008d5c..76d81d511edad 100644 --- a/frame/indices/src/mock.rs +++ b/frame/indices/src/mock.rs @@ -100,12 +100,12 @@ impl Config for Test { } pub fn new_test_ext() -> sp_io::TestExternalities { - let state_version = None; - let mut t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); + let state_version = Default::default(); + let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); pallet_balances::GenesisConfig:: { balances: vec![(1, 10), (2, 20), (3, 30), (4, 40), (5, 50), (6, 60)], } - .assimilate_storage(&mut t, state_version.clone()) + .assimilate_storage(&mut t, state_version) .unwrap(); (t, state_version).into() } diff --git a/frame/lottery/src/mock.rs b/frame/lottery/src/mock.rs index 0e8fc8dd60b94..c380fe390877c 100644 --- a/frame/lottery/src/mock.rs +++ b/frame/lottery/src/mock.rs @@ -120,12 +120,12 @@ pub type SystemCall = frame_system::Call; pub type BalancesCall = pallet_balances::Call; pub fn new_test_ext() -> sp_io::TestExternalities { - let state_version = None; - let mut t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); + let state_version = Default::default(); + let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); pallet_balances::GenesisConfig:: { balances: vec![(1, 100), (2, 100), (3, 100), (4, 100), (5, 100)], } - .assimilate_storage(&mut t, state_version.clone()) + .assimilate_storage(&mut t, state_version) .unwrap(); (t, state_version).into() } diff --git a/frame/membership/src/lib.rs b/frame/membership/src/lib.rs index 7db4fa6a90e10..250a1e08be341 100644 --- a/frame/membership/src/lib.rs +++ b/frame/membership/src/lib.rs @@ -573,14 +573,14 @@ mod tests { } pub(crate) fn new_test_ext() -> sp_io::TestExternalities { - let state_version = None; - let mut t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); + let state_version = Default::default(); + let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); // We use default for brevity, but you can configure as desired if needed. pallet_membership::GenesisConfig:: { members: vec![10, 20, 30], ..Default::default() } - .assimilate_storage(&mut t, state_version.clone()) + .assimilate_storage(&mut t, state_version) .unwrap(); (t, state_version).into() } @@ -739,7 +739,7 @@ mod tests { #[test] #[should_panic(expected = "Members cannot contain duplicate accounts.")] fn genesis_build_panics_with_duplicate_members() { - let state_version = None; + let state_version = Default::default(); pallet_membership::GenesisConfig:: { members: vec![1, 2, 3, 1], phantom: Default::default(), diff --git a/frame/merkle-mountain-range/src/tests.rs b/frame/merkle-mountain-range/src/tests.rs index 01f374f52ef3c..9c23a00f70daa 100644 --- a/frame/merkle-mountain-range/src/tests.rs +++ b/frame/merkle-mountain-range/src/tests.rs @@ -25,8 +25,8 @@ use sp_core::{ }; pub(crate) fn new_test_ext() -> sp_io::TestExternalities { - let state_version = None; - (frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(), state_version).into() + let state_version = Default::default(); + (frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(), state_version).into() } fn register_offchain_ext(ext: &mut sp_io::TestExternalities) { diff --git a/frame/multisig/src/tests.rs b/frame/multisig/src/tests.rs index ddccf9ac31bfa..a9972f1f9ce38 100644 --- a/frame/multisig/src/tests.rs +++ b/frame/multisig/src/tests.rs @@ -117,12 +117,12 @@ impl Config for Test { use pallet_balances::{Call as BalancesCall, Error as BalancesError}; pub fn new_test_ext() -> sp_io::TestExternalities { - let state_version = None; - let mut t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); + let state_version = Default::default(); + let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); pallet_balances::GenesisConfig:: { balances: vec![(1, 10), (2, 10), (3, 10), (4, 10), (5, 2)], } - .assimilate_storage(&mut t, state_version.clone()) + .assimilate_storage(&mut t, state_version) .unwrap(); let mut ext = sp_io::TestExternalities::new(t, state_version); ext.execute_with(|| System::set_block_number(1)); diff --git a/frame/nicks/src/lib.rs b/frame/nicks/src/lib.rs index 94e6cd9cb0f3b..22b063b96ee2e 100644 --- a/frame/nicks/src/lib.rs +++ b/frame/nicks/src/lib.rs @@ -339,10 +339,10 @@ mod tests { } fn new_test_ext() -> sp_io::TestExternalities { - let state_version = None; - let mut t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); + let state_version = Default::default(); + let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); pallet_balances::GenesisConfig:: { balances: vec![(1, 10), (2, 10)] } - .assimilate_storage(&mut t, state_version.clone()) + .assimilate_storage(&mut t, state_version) .unwrap(); (t, state_version).into() } diff --git a/frame/node-authorization/src/mock.rs b/frame/node-authorization/src/mock.rs index ee9dfe2a0c292..a5b243b542ced 100644 --- a/frame/node-authorization/src/mock.rs +++ b/frame/node-authorization/src/mock.rs @@ -99,12 +99,12 @@ pub fn test_node(id: u8) -> PeerId { } pub fn new_test_ext() -> sp_io::TestExternalities { - let state_version = None; - let mut t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); + let state_version = Default::default(); + let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); pallet_node_authorization::GenesisConfig:: { nodes: vec![(test_node(10), 10), (test_node(20), 20), (test_node(30), 30)], } - .assimilate_storage(&mut t, state_version.clone()) + .assimilate_storage(&mut t, state_version) .unwrap(); (t, state_version).into() } diff --git a/frame/offences/benchmarking/src/mock.rs b/frame/offences/benchmarking/src/mock.rs index ea704b92c0dbf..34be5a9361cf3 100644 --- a/frame/offences/benchmarking/src/mock.rs +++ b/frame/offences/benchmarking/src/mock.rs @@ -226,7 +226,7 @@ frame_support::construct_runtime!( ); pub fn new_test_ext() -> sp_io::TestExternalities { - let state_version = None; - let t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); + let state_version = Default::default(); + let t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); sp_io::TestExternalities::new(t, state_version) } diff --git a/frame/offences/src/mock.rs b/frame/offences/src/mock.rs index 57fcc9240b84d..fb103b2c3c26f 100644 --- a/frame/offences/src/mock.rs +++ b/frame/offences/src/mock.rs @@ -120,8 +120,8 @@ impl Config for Runtime { } pub fn new_test_ext() -> sp_io::TestExternalities { - let state_version = None; - let t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); + let state_version = Default::default(); + let t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); let mut ext = sp_io::TestExternalities::new(t, state_version); ext.execute_with(|| System::set_block_number(1)); ext diff --git a/frame/proxy/src/tests.rs b/frame/proxy/src/tests.rs index f8e1085a1da77..d41bb512b14d6 100644 --- a/frame/proxy/src/tests.rs +++ b/frame/proxy/src/tests.rs @@ -165,12 +165,12 @@ use pallet_balances::{Call as BalancesCall, Error as BalancesError, Event as Bal use pallet_utility::{Call as UtilityCall, Event as UtilityEvent}; pub fn new_test_ext() -> sp_io::TestExternalities { - let state_version = None; - let mut t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); + let state_version = Default::default(); + let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); pallet_balances::GenesisConfig:: { balances: vec![(1, 10), (2, 10), (3, 10), (4, 10), (5, 2)], } - .assimilate_storage(&mut t, state_version.clone()) + .assimilate_storage(&mut t, state_version) .unwrap(); let mut ext = sp_io::TestExternalities::new(t, state_version); ext.execute_with(|| System::set_block_number(1)); diff --git a/frame/randomness-collective-flip/src/lib.rs b/frame/randomness-collective-flip/src/lib.rs index db7b1254b3434..6f4c829c2488e 100644 --- a/frame/randomness-collective-flip/src/lib.rs +++ b/frame/randomness-collective-flip/src/lib.rs @@ -224,8 +224,8 @@ mod tests { impl pallet_randomness_collective_flip::Config for Test {} fn new_test_ext() -> sp_io::TestExternalities { - let state_version = None; - let t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); + let state_version = Default::default(); + let t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); (t, state_version).into() } diff --git a/frame/recovery/src/mock.rs b/frame/recovery/src/mock.rs index 731b98b602557..22be999741d3d 100644 --- a/frame/recovery/src/mock.rs +++ b/frame/recovery/src/mock.rs @@ -114,12 +114,12 @@ pub type BalancesCall = pallet_balances::Call; pub type RecoveryCall = super::Call; pub fn new_test_ext() -> sp_io::TestExternalities { - let state_version = None; - let mut t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); + let state_version = Default::default(); + let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); pallet_balances::GenesisConfig:: { balances: vec![(1, 100), (2, 100), (3, 100), (4, 100), (5, 100)], } - .assimilate_storage(&mut t, state_version.clone()) + .assimilate_storage(&mut t, state_version) .unwrap(); (t, state_version).into() } diff --git a/frame/scheduler/src/lib.rs b/frame/scheduler/src/lib.rs index adb369c753f61..4bcf94d45656b 100644 --- a/frame/scheduler/src/lib.rs +++ b/frame/scheduler/src/lib.rs @@ -985,8 +985,8 @@ mod tests { pub type LoggerCall = logger::Call; pub fn new_test_ext() -> sp_io::TestExternalities { - let state_version = None; - let t = system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); + let state_version = Default::default(); + let t = system::GenesisConfig::default().build_storage::(state_version).unwrap(); (t, state_version).into() } diff --git a/frame/scored-pool/src/mock.rs b/frame/scored-pool/src/mock.rs index 732568c936a55..c583a6dc54792 100644 --- a/frame/scored-pool/src/mock.rs +++ b/frame/scored-pool/src/mock.rs @@ -135,8 +135,8 @@ impl Config for Test { } pub fn new_test_ext() -> sp_io::TestExternalities { - let state_version = None; - let mut t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); + let state_version = Default::default(); + let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); pallet_balances::GenesisConfig:: { balances: vec![ (5, 500_000), @@ -148,14 +148,14 @@ pub fn new_test_ext() -> sp_io::TestExternalities { (99, 1), ], } - .assimilate_storage(&mut t, state_version.clone()) + .assimilate_storage(&mut t, state_version) .unwrap(); pallet_scored_pool::GenesisConfig:: { pool: vec![(5, None), (10, Some(1)), (20, Some(2)), (31, Some(2)), (40, Some(3))], member_count: 2, ..Default::default() } - .assimilate_storage(&mut t, state_version.clone()) + .assimilate_storage(&mut t, state_version) .unwrap(); (t, state_version).into() } diff --git a/frame/session/benchmarking/src/mock.rs b/frame/session/benchmarking/src/mock.rs index 7ff1d95457ec9..1482ef59e8a25 100644 --- a/frame/session/benchmarking/src/mock.rs +++ b/frame/session/benchmarking/src/mock.rs @@ -191,7 +191,7 @@ impl pallet_staking::Config for Test { impl crate::Config for Test {} pub fn new_test_ext() -> sp_io::TestExternalities { - let state_version = None; - let t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); + let state_version = Default::default(); + let t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); sp_io::TestExternalities::new(t, state_version) } diff --git a/frame/session/src/historical/mod.rs b/frame/session/src/historical/mod.rs index 4f1cac7f111bf..3a69aa2086fab 100644 --- a/frame/session/src/historical/mod.rs +++ b/frame/session/src/historical/mod.rs @@ -375,17 +375,17 @@ pub(crate) mod tests { type Historical = Module; pub(crate) fn new_test_ext() -> sp_io::TestExternalities { - let state_version = None; + let state_version = Default::default(); let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); let keys: Vec<_> = NEXT_VALIDATORS.with(|l| { l.borrow().iter().cloned().map(|i| (i, i, UintAuthorityId(i).into())).collect() }); - BasicExternalities::execute_with_storage(&mut t, state_version.clone(), || { + BasicExternalities::execute_with_storage(&mut t, state_version, || { for (ref k, ..) in &keys { frame_system::Pallet::::inc_providers(k); } }); - crate::GenesisConfig:: { keys }.assimilate_storage(&mut t, state_version.clone()).unwrap(); + crate::GenesisConfig:: { keys }.assimilate_storage(&mut t, state_version).unwrap(); sp_io::TestExternalities::new(t, state_version) } diff --git a/frame/session/src/historical/offchain.rs b/frame/session/src/historical/offchain.rs index d63a655bc2f7b..c969636da77de 100644 --- a/frame/session/src/historical/offchain.rs +++ b/frame/session/src/historical/offchain.rs @@ -162,21 +162,21 @@ mod tests { type Historical = Module; pub fn new_test_ext() -> sp_io::TestExternalities { - let state_version = None; + let state_version = Default::default(); let mut t = frame_system::GenesisConfig::default() - .build_storage::(state_version.clone()) + .build_storage::(state_version) .expect("Failed to create test externalities."); let keys: Vec<_> = NEXT_VALIDATORS.with(|l| { l.borrow().iter().cloned().map(|i| (i, i, UintAuthorityId(i).into())).collect() }); - BasicExternalities::execute_with_storage(&mut t, state_version.clone(), || { + BasicExternalities::execute_with_storage(&mut t, state_version, || { for (ref k, ..) in &keys { frame_system::Pallet::::inc_providers(k); } }); - crate::GenesisConfig:: { keys }.assimilate_storage(&mut t, state_version.clone()).unwrap(); + crate::GenesisConfig:: { keys }.assimilate_storage(&mut t, state_version).unwrap(); let mut ext = sp_io::TestExternalities::new(t, state_version); diff --git a/frame/session/src/mock.rs b/frame/session/src/mock.rs index 29cfc72c4e4d3..242b603f5abc9 100644 --- a/frame/session/src/mock.rs +++ b/frame/session/src/mock.rs @@ -209,11 +209,11 @@ pub fn reset_before_session_end_called() { } pub fn new_test_ext() -> sp_io::TestExternalities { - let state_version = None; - let mut t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); + let state_version = Default::default(); + let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); let keys: Vec<_> = NEXT_VALIDATORS .with(|l| l.borrow().iter().cloned().map(|i| (i, i, UintAuthorityId(i).into())).collect()); - BasicExternalities::execute_with_storage(&mut t, state_version.clone(), || { + BasicExternalities::execute_with_storage(&mut t, state_version, || { for (ref k, ..) in &keys { frame_system::Pallet::::inc_providers(k); } @@ -222,7 +222,7 @@ pub fn new_test_ext() -> sp_io::TestExternalities { frame_system::Pallet::::inc_providers(&69); }); pallet_session::GenesisConfig:: { keys } - .assimilate_storage(&mut t, state_version.clone()) + .assimilate_storage(&mut t, state_version) .unwrap(); sp_io::TestExternalities::new(t,state_version) } diff --git a/frame/society/src/mock.rs b/frame/society/src/mock.rs index cd8db1f813091..9763be59389da 100644 --- a/frame/society/src/mock.rs +++ b/frame/society/src/mock.rs @@ -154,18 +154,18 @@ impl EnvBuilder { } pub fn execute R>(mut self, f: F) -> R { - let state_version = None; - let mut t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); + let state_version = Default::default(); + let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); self.balances.push((Society::account_id(), self.balance.max(self.pot))); pallet_balances::GenesisConfig:: { balances: self.balances } - .assimilate_storage(&mut t, state_version.clone()) + .assimilate_storage(&mut t, state_version) .unwrap(); pallet_society::GenesisConfig:: { members: self.members, pot: self.pot, max_members: self.max_members, } - .assimilate_storage(&mut t, state_version.clone()) + .assimilate_storage(&mut t, state_version) .unwrap(); let mut ext: sp_io::TestExternalities = (t, state_version).into(); ext.execute_with(f) diff --git a/frame/staking/src/mock.rs b/frame/staking/src/mock.rs index 59b059dbc962d..8a2df2817007e 100644 --- a/frame/staking/src/mock.rs +++ b/frame/staking/src/mock.rs @@ -395,8 +395,8 @@ impl ExtBuilder { } fn build(self) -> sp_io::TestExternalities { sp_tracing::try_init_simple(); - let state_version = None; - let mut storage = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); + let state_version = Default::default(); + let mut storage = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); let _ = pallet_balances::GenesisConfig:: { balances: vec![ @@ -430,7 +430,7 @@ impl ExtBuilder { (999, 1_000_000_000_000), ], } - .assimilate_storage(&mut storage, state_version.clone()); + .assimilate_storage(&mut storage, state_version); let mut stakers = vec![]; if self.has_stakers { @@ -483,7 +483,7 @@ impl ExtBuilder { min_validator_bond: self.min_validator_bond, ..Default::default() } - .assimilate_storage(&mut storage, state_version.clone()); + .assimilate_storage(&mut storage, state_version); let _ = pallet_session::GenesisConfig:: { keys: if self.has_stakers { @@ -496,7 +496,7 @@ impl ExtBuilder { .collect() }, } - .assimilate_storage(&mut storage, state_version.clone()); + .assimilate_storage(&mut storage, state_version); let mut ext = sp_io::TestExternalities::from((storage, state_version)); ext.execute_with(|| { diff --git a/frame/support/src/lib.rs b/frame/support/src/lib.rs index 4d99cc9215de4..e5ca24c7c0112 100644 --- a/frame/support/src/lib.rs +++ b/frame/support/src/lib.rs @@ -847,8 +847,8 @@ pub mod tests { } fn new_test_ext() -> TestExternalities { - let state_version = None; - let storage = GenesisConfig::default().build_storage(state_version.clone()).unwrap(); + let state_version = Default::default(); + let storage = GenesisConfig::default().build_storage(state_version).unwrap(); (storage, state_version).into() } diff --git a/frame/support/src/storage/generator/mod.rs b/frame/support/src/storage/generator/mod.rs index 8b3e239835616..0b2755ff03bb9 100644 --- a/frame/support/src/storage/generator/mod.rs +++ b/frame/support/src/storage/generator/mod.rs @@ -74,8 +74,8 @@ mod tests { #[test] fn value_translate_works() { - let state_version = None; - let t = GenesisConfig::default().build_storage(state_version.clone()).unwrap(); + let state_version = Default::default(); + let t = GenesisConfig::default().build_storage(state_version).unwrap(); TestExternalities::new(t, state_version).execute_with(|| { // put the old value `1111u32` in the storage. let key = Value::storage_value_final_key(); @@ -95,8 +95,8 @@ mod tests { #[test] fn map_translate_works() { - let state_version = None; - let t = GenesisConfig::default().build_storage(state_version.clone()).unwrap(); + let state_version = Default::default(); + let t = GenesisConfig::default().build_storage(state_version).unwrap(); TestExternalities::new(t, state_version).execute_with(|| { // start with a map of u32 -> u32. for i in 0u32..100u32 { @@ -125,8 +125,8 @@ mod tests { #[test] fn try_mutate_works() { - let state_version = None; - let t = GenesisConfig::default().build_storage(state_version.clone()).unwrap(); + let state_version = Default::default(); + let t = GenesisConfig::default().build_storage(state_version).unwrap(); TestExternalities::new(t, state_version).execute_with(|| { assert_eq!(Value::get(), (0, 0)); assert_eq!(NumberMap::get(0), 0); diff --git a/frame/support/test/tests/decl_storage.rs b/frame/support/test/tests/decl_storage.rs index 4853861738ad1..72cebf900464a 100644 --- a/frame/support/test/tests/decl_storage.rs +++ b/frame/support/test/tests/decl_storage.rs @@ -668,9 +668,9 @@ mod tests { #[test] fn check_builder_config() { - let state_version = None; + let state_version = Default::default(); let config = GenesisConfig::default(); - let storage = config.build_storage(state_version.clone()).unwrap(); + let storage = config.build_storage(state_version).unwrap(); TestExternalities::from((storage, state_version)).execute_with(|| { assert_eq!(Module::::u32_with_builder(), 1); assert_eq!(Module::::opt_u32_with_builder_some(), Some(1)); diff --git a/frame/support/test/tests/instance.rs b/frame/support/test/tests/instance.rs index b79420c733a12..ee41b4b0f0309 100644 --- a/frame/support/test/tests/instance.rs +++ b/frame/support/test/tests/instance.rs @@ -318,7 +318,7 @@ pub type Block = generic::Block; pub type UncheckedExtrinsic = generic::UncheckedExtrinsic; fn new_test_ext() -> sp_io::TestExternalities { - let state_version = None; + let state_version = Default::default(); (GenesisConfig { module_1_1: module1::GenesisConfig { value: 3, test: 2 }, @@ -336,13 +336,13 @@ fn new_test_ext() -> sp_io::TestExternalities { module_2_2: Default::default(), module_2_3: Default::default(), } - .build_storage(state_version.clone()) + .build_storage(state_version) .unwrap(), state_version).into() } #[test] fn storage_instance_independence() { - let state_version = Some(Some(sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD)); + let state_version = Default::default(); let mut storage = sp_core::storage::Storage { top: std::collections::BTreeMap::new(), children_default: std::collections::HashMap::new(), diff --git a/frame/system/benches/bench.rs b/frame/system/benches/bench.rs index b1030a20acef4..eb6fbcd886ddb 100644 --- a/frame/system/benches/bench.rs +++ b/frame/system/benches/bench.rs @@ -101,8 +101,8 @@ impl module::Config for Runtime { } fn new_test_ext() -> sp_io::TestExternalities { - let state_version = None; - (system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(), state_version).into() + let state_version = Default::default(); + (system::GenesisConfig::default().build_storage::(state_version).unwrap(), state_version).into() } fn deposit_events(n: usize) { diff --git a/frame/system/benchmarking/src/mock.rs b/frame/system/benchmarking/src/mock.rs index 722bd84fdc93e..de6a806edf69c 100644 --- a/frame/system/benchmarking/src/mock.rs +++ b/frame/system/benchmarking/src/mock.rs @@ -67,7 +67,7 @@ impl frame_system::Config for Test { impl crate::Config for Test {} pub fn new_test_ext() -> sp_io::TestExternalities { - let state_version = None; - let t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); + let state_version = Default::default(); + let t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); sp_io::TestExternalities::new(t, state_version) } diff --git a/frame/system/src/mock.rs b/frame/system/src/mock.rs index 68a40637abcdf..cc0ff6b97c906 100644 --- a/frame/system/src/mock.rs +++ b/frame/system/src/mock.rs @@ -120,9 +120,9 @@ pub const CALL: &::Call = &Call::System(frame_system::Call::set_ /// Create new externalities for `System` module tests. pub fn new_test_ext() -> sp_io::TestExternalities { - let state_version = None; + let state_version = Default::default(); let mut ext: sp_io::TestExternalities = - (GenesisConfig::default().build_storage(state_version.clone()).unwrap(), state_version).into(); + (GenesisConfig::default().build_storage(state_version).unwrap(), state_version).into(); // Add to each test the initial weight of a block ext.execute_with(|| { System::register_extra_weight_unchecked( diff --git a/frame/timestamp/src/lib.rs b/frame/timestamp/src/lib.rs index 150ded70d7e17..f492290dc066b 100644 --- a/frame/timestamp/src/lib.rs +++ b/frame/timestamp/src/lib.rs @@ -316,8 +316,8 @@ mod tests { }; pub fn new_test_ext() -> TestExternalities { - let state_version = None; - let t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); + let state_version = Default::default(); + let t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); TestExternalities::new(t, state_version) } diff --git a/frame/tips/src/tests.rs b/frame/tips/src/tests.rs index 320191a779afa..b4155c0659dab 100644 --- a/frame/tips/src/tests.rs +++ b/frame/tips/src/tests.rs @@ -162,15 +162,15 @@ impl Config for Test { } pub fn new_test_ext() -> sp_io::TestExternalities { - let state_version = None; - let mut t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); + let state_version = Default::default(); + let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); pallet_balances::GenesisConfig:: { // Total issuance will be 200 with treasury account initialized at ED. balances: vec![(0, 100), (1, 98), (2, 1)], } - .assimilate_storage(&mut t, state_version.clone()) + .assimilate_storage(&mut t, state_version) .unwrap(); - GenesisBuild::::assimilate_storage(&pallet_treasury::GenesisConfig, &mut t, state_version.clone()).unwrap(); + GenesisBuild::::assimilate_storage(&pallet_treasury::GenesisConfig, &mut t, state_version).unwrap(); (t, state_version).into() } @@ -404,7 +404,7 @@ fn tip_changing_works() { fn test_last_reward_migration() { use sp_storage::Storage; - let state_version = None; + let state_version = Default::default(); let mut s = Storage::default(); #[derive(Clone, Eq, PartialEq, Encode, Decode, RuntimeDebug)] @@ -492,16 +492,16 @@ fn test_last_reward_migration() { #[test] fn genesis_funding_works() { - let state_version = None; - let mut t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); + let state_version = Default::default(); + let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); let initial_funding = 100; pallet_balances::GenesisConfig:: { // Total issuance will be 200 with treasury account initialized with 100. balances: vec![(0, 100), (Treasury::account_id(), initial_funding)], } - .assimilate_storage(&mut t, state_version.clone()) + .assimilate_storage(&mut t, state_version) .unwrap(); - GenesisBuild::::assimilate_storage(&pallet_treasury::GenesisConfig, &mut t, state_version.clone()).unwrap(); + GenesisBuild::::assimilate_storage(&pallet_treasury::GenesisConfig, &mut t, state_version).unwrap(); let mut t: sp_io::TestExternalities = (t, state_version).into(); t.execute_with(|| { diff --git a/frame/transaction-payment/src/lib.rs b/frame/transaction-payment/src/lib.rs index a600650924b0e..ae9ac087c07ee 100644 --- a/frame/transaction-payment/src/lib.rs +++ b/frame/transaction-payment/src/lib.rs @@ -858,9 +858,9 @@ mod tests { WEIGHT_TO_FEE.with(|v| *v.borrow_mut() = self.weight_to_fee); } pub fn build(self) -> sp_io::TestExternalities { - let state_version = None; + let state_version = Default::default(); self.set_constants(); - let mut t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); + let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); pallet_balances::GenesisConfig:: { balances: if self.balance_factor > 0 { vec![ @@ -875,7 +875,7 @@ mod tests { vec![] }, } - .assimilate_storage(&mut t, state_version.clone()) + .assimilate_storage(&mut t, state_version) .unwrap(); (t, state_version).into() } diff --git a/frame/transaction-storage/src/mock.rs b/frame/transaction-storage/src/mock.rs index c35dac4e76708..540601a6e3828 100644 --- a/frame/transaction-storage/src/mock.rs +++ b/frame/transaction-storage/src/mock.rs @@ -104,7 +104,7 @@ impl pallet_transaction_storage::Config for Test { } pub fn new_test_ext() -> sp_io::TestExternalities { - let state_version = None; + let state_version = Default::default(); let t = GenesisConfig { system: Default::default(), balances: pallet_balances::GenesisConfig:: { @@ -118,7 +118,7 @@ pub fn new_test_ext() -> sp_io::TestExternalities { max_transaction_size: crate::DEFAULT_MAX_TRANSACTION_SIZE, }, } - .build_storage(state_version.clone()) + .build_storage(state_version) .unwrap(); (t, state_version).into() } diff --git a/frame/treasury/src/tests.rs b/frame/treasury/src/tests.rs index 0c4ca3efff22a..0cc44c0aa836a 100644 --- a/frame/treasury/src/tests.rs +++ b/frame/treasury/src/tests.rs @@ -126,15 +126,15 @@ impl Config for Test { } pub fn new_test_ext() -> sp_io::TestExternalities { - let state_version = None; - let mut t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); + let state_version = Default::default(); + let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); pallet_balances::GenesisConfig:: { // Total issuance will be 200 with treasury account initialized at ED. balances: vec![(0, 100), (1, 98), (2, 1)], } - .assimilate_storage(&mut t, state_version.clone()) + .assimilate_storage(&mut t, state_version) .unwrap(); - GenesisBuild::::assimilate_storage(&crate::GenesisConfig, &mut t, state_version.clone()).unwrap(); + GenesisBuild::::assimilate_storage(&crate::GenesisConfig, &mut t, state_version).unwrap(); (t, state_version).into() } @@ -322,10 +322,10 @@ fn treasury_account_doesnt_get_deleted() { // This is useful for chain that will just update runtime. #[test] fn inexistent_account_works() { - let state_version = None; - let mut t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); + let state_version = Default::default(); + let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); pallet_balances::GenesisConfig:: { balances: vec![(0, 100), (1, 99), (2, 1)] } - .assimilate_storage(&mut t, state_version.clone()) + .assimilate_storage(&mut t, state_version) .unwrap(); // Treasury genesis config is not build thus treasury account does not exist let mut t: sp_io::TestExternalities = (t, state_version).into(); @@ -355,16 +355,16 @@ fn inexistent_account_works() { #[test] fn genesis_funding_works() { - let state_version = None; - let mut t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); + let state_version = Default::default(); + let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); let initial_funding = 100; pallet_balances::GenesisConfig:: { // Total issuance will be 200 with treasury account initialized with 100. balances: vec![(0, 100), (Treasury::account_id(), initial_funding)], } - .assimilate_storage(&mut t, state_version.clone()) + .assimilate_storage(&mut t, state_version) .unwrap(); - GenesisBuild::::assimilate_storage(&crate::GenesisConfig, &mut t, state_version.clone()).unwrap(); + GenesisBuild::::assimilate_storage(&crate::GenesisConfig, &mut t, state_version).unwrap(); let mut t: sp_io::TestExternalities = (t, state_version).into(); t.execute_with(|| { diff --git a/frame/uniques/src/mock.rs b/frame/uniques/src/mock.rs index eeddbe8a79b92..dbb090f4269ab 100644 --- a/frame/uniques/src/mock.rs +++ b/frame/uniques/src/mock.rs @@ -117,8 +117,8 @@ impl Config for Test { } pub(crate) fn new_test_ext() -> sp_io::TestExternalities { - let state_version = None; - let t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); + let state_version = Default::default(); + let t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); let mut ext = sp_io::TestExternalities::new(t, state_version); ext.execute_with(|| System::set_block_number(1)); diff --git a/frame/utility/src/tests.rs b/frame/utility/src/tests.rs index a90fed114c850..46345a670a479 100644 --- a/frame/utility/src/tests.rs +++ b/frame/utility/src/tests.rs @@ -169,12 +169,12 @@ use frame_system::Call as SystemCall; use pallet_balances::{Call as BalancesCall, Error as BalancesError}; pub fn new_test_ext() -> sp_io::TestExternalities { - let state_version = None; - let mut t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); + let state_version = Default::default(); + let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); pallet_balances::GenesisConfig:: { balances: vec![(1, 10), (2, 10), (3, 10), (4, 10), (5, 2)], } - .assimilate_storage(&mut t, state_version.clone()) + .assimilate_storage(&mut t, state_version) .unwrap(); let mut ext = sp_io::TestExternalities::new(t, state_version); ext.execute_with(|| System::set_block_number(1)); diff --git a/frame/vesting/src/mock.rs b/frame/vesting/src/mock.rs index ae812d9198c7d..5b7bc2606afe8 100644 --- a/frame/vesting/src/mock.rs +++ b/frame/vesting/src/mock.rs @@ -111,9 +111,9 @@ impl ExtBuilder { } pub fn build(self) -> sp_io::TestExternalities { - let state_version = None; + let state_version = Default::default(); EXISTENTIAL_DEPOSIT.with(|v| *v.borrow_mut() = self.existential_deposit); - let mut t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); + let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); pallet_balances::GenesisConfig:: { balances: vec![ (1, 10 * self.existential_deposit), @@ -123,7 +123,7 @@ impl ExtBuilder { (12, 10 * self.existential_deposit), ], } - .assimilate_storage(&mut t, state_version.clone()) + .assimilate_storage(&mut t, state_version) .unwrap(); pallet_vesting::GenesisConfig:: { vesting: vec![ @@ -132,7 +132,7 @@ impl ExtBuilder { (12, 10, 20, 5 * self.existential_deposit), ], } - .assimilate_storage(&mut t, state_version.clone()) + .assimilate_storage(&mut t, state_version) .unwrap(); let mut ext = sp_io::TestExternalities::new(t, state_version); ext.execute_with(|| System::set_block_number(1)); diff --git a/primitives/runtime/src/lib.rs b/primitives/runtime/src/lib.rs index cfd3aeb6bcd3e..7300949f67141 100644 --- a/primitives/runtime/src/lib.rs +++ b/primitives/runtime/src/lib.rs @@ -996,7 +996,7 @@ mod tests { #[test] #[should_panic(expected = "Signature verification has not been called")] fn batching_still_finishes_when_not_called_directly() { - let mut ext = sp_state_machine::BasicExternalities::new_empty(None); + let mut ext = sp_state_machine::BasicExternalities::new_empty(Default::default()); ext.register_extension(sp_core::traits::TaskExecutorExt::new( sp_core::testing::TaskExecutor::new(), )); @@ -1010,7 +1010,7 @@ mod tests { #[test] #[should_panic(expected = "Hey, I'm an error")] fn batching_does_not_panic_while_thread_is_already_panicking() { - let mut ext = sp_state_machine::BasicExternalities::new_empty(None); + let mut ext = sp_state_machine::BasicExternalities::new_empty(Default::default()); ext.register_extension(sp_core::traits::TaskExecutorExt::new( sp_core::testing::TaskExecutor::new(), )); diff --git a/primitives/runtime/src/state_version.rs b/primitives/runtime/src/state_version.rs index 4f90313cc9cd9..c60c125138860 100644 --- a/primitives/runtime/src/state_version.rs +++ b/primitives/runtime/src/state_version.rs @@ -24,7 +24,7 @@ use sp_std::vec::Vec; use sp_arithmetic::traits::Zero; /// Multiple versions of state in use for a chain. -#[derive(Clone)] +#[derive(Clone, crate::RuntimeDebug)] pub struct StateVersions { canonical_states: Vec<(NumberFor, StateVersion)>, } diff --git a/test-utils/runtime/client/src/lib.rs b/test-utils/runtime/client/src/lib.rs index f45908b81fcff..dd2893dc4981c 100644 --- a/test-utils/runtime/client/src/lib.rs +++ b/test-utils/runtime/client/src/lib.rs @@ -116,7 +116,7 @@ impl GenesisParameters { 1000, self.heap_pages_override, self.extra_storage.clone(), - Some(Some(sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD)), + Default::default(), ) } @@ -408,7 +408,7 @@ pub fn new_light( ) { let storage = sc_client_db::light::LightStorage::new_test(); let blockchain = Arc::new(sc_light::Blockchain::new(storage)); - let state_versions = vec![(0, Some(Some(sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD)))]; + let state_versions = Default::default(); let backend = Arc::new(LightBackend::new(blockchain, state_versions)); let executor = new_native_executor(); let local_call_executor = client::LocalCallExecutor::new( diff --git a/test-utils/runtime/src/lib.rs b/test-utils/runtime/src/lib.rs index 09b0e37eac104..371980c1ea375 100644 --- a/test-utils/runtime/src/lib.rs +++ b/test-utils/runtime/src/lib.rs @@ -1316,7 +1316,7 @@ mod tests { #[test] fn witness_backend_works() { let (db, root) = witness_backend(); - let backend = sp_state_machine::TrieBackend::<_, crate::Hashing>::new(db, root, None); + let backend = sp_state_machine::TrieBackend::<_, crate::Hashing>::new(db, root, Default::default()); let proof = sp_state_machine::prove_read(backend, vec![b"value3"]).unwrap(); let client = TestClientBuilder::new().set_execution_strategy(ExecutionStrategy::Both).build(); diff --git a/test-utils/runtime/src/system.rs b/test-utils/runtime/src/system.rs index 4ff9408bab809..47caf04e2e656 100644 --- a/test-utils/runtime/src/system.rs +++ b/test-utils/runtime/src/system.rs @@ -366,7 +366,7 @@ mod tests { } fn new_test_ext() -> TestExternalities { - let state_version = None; + let state_version = Default::default(); let authorities = vec![ Sr25519Keyring::Alice.to_raw_public(), Sr25519Keyring::Bob.to_raw_public(), diff --git a/test-utils/test-runner/src/utils.rs b/test-utils/test-runner/src/utils.rs index f1b1907270d3a..330ff4c8098a1 100644 --- a/test-utils/test-runner/src/utils.rs +++ b/test-utils/test-runner/src/utils.rs @@ -50,9 +50,12 @@ pub fn default_config( let base_path = base_path(); let root_path = base_path.path().to_path_buf().join("chains").join(chain_spec.id()); - let state_version = chain_spec.state_versions() - .expect("Invalid state versions for chain spec") - .genesis_state_version(); + use std::str::FromStr; + let state_version = chain_spec.state_versions().get(0) + // This is incorrect (can have genesis state version as different element in chain_spec). + .and_then(|(n, s)| u64::from_str(n).ok().map(|n| (n, s))) + .and_then(|(n, s)| (n == 0).then(|| s.clone())) + .expect("Invalid state versions for chain spec"); let storage = chain_spec .as_storage_builder() diff --git a/utils/frame/benchmarking-cli/src/command.rs b/utils/frame/benchmarking-cli/src/command.rs index 72bcd31ac626c..c64e4573f7bc3 100644 --- a/utils/frame/benchmarking-cli/src/command.rs +++ b/utils/frame/benchmarking-cli/src/command.rs @@ -96,9 +96,12 @@ impl BenchmarkCmd { ::Hash: std::str::FromStr, ExecDispatch: NativeExecutionDispatch + 'static, { - let state_version = config.chain_spec.state_versions() - .ok_or_else(|| Err("Invalid state versions for chain spec".into()))? - .genesis_state_version(); + let state_version = if let Some(versions) = sp_runtime::StateVersions::::from_conf(config.chain_spec.state_versions() + .iter().map(|(number, version)| (number.as_str(), *version))) { + versions.genesis_state_version() + } else { + return Err("Invalid state versions for chain spec".into()) + }; if let Some(output_path) = &self.output { if !output_path.is_dir() && output_path.file_name().is_none() { @@ -126,7 +129,7 @@ impl BenchmarkCmd { let extrinsic = self.extrinsic.clone().unwrap_or_else(|| String::new()); let extrinsic = extrinsic.as_bytes(); - let genesis_storage = spec.build_storage(state_version.clone())?; + let genesis_storage = spec.build_storage(state_version)?; let mut changes = Default::default(); let cache_size = Some(self.database_cache_size as usize); let state_with_tracking = BenchmarkingState::::new( diff --git a/utils/frame/try-runtime/cli/src/lib.rs b/utils/frame/try-runtime/cli/src/lib.rs index b2ecadd08fb30..c1f7dce9d34bd 100644 --- a/utils/frame/try-runtime/cli/src/lib.rs +++ b/utils/frame/try-runtime/cli/src/lib.rs @@ -211,7 +211,7 @@ where })), }; - let (code_key, code) = extract_code(config.chain_spec)?; + let (code_key, code) = extract_code::(config.chain_spec)?; builder .inject_key_value(&[(code_key, code)]) .inject_hashed_key(&[twox_128(b"System"), twox_128(b"LastRuntimeUpgrade")].concat()) @@ -292,7 +292,7 @@ where .mode(mode) .inject_hashed_key(&[twox_128(b"System"), twox_128(b"LastRuntimeUpgrade")].concat()); let mut ext = if shared.overwrite_code { - let (code_key, code) = extract_code(config.chain_spec)?; + let (code_key, code) = extract_code::(config.chain_spec)?; builder.inject_key_value(&[(code_key, code)]).build().await? } else { builder.inject_hashed_key(well_known_keys::CODE).build().await? @@ -379,7 +379,7 @@ where .mode(mode) .inject_hashed_key(&[twox_128(b"System"), twox_128(b"LastRuntimeUpgrade")].concat()); let mut ext = if shared.overwrite_code { - let (code_key, code) = extract_code(config.chain_spec)?; + let (code_key, code) = extract_code::(config.chain_spec)?; builder.inject_key_value(&[(code_key, code)]).build().await? } else { builder.inject_hashed_key(well_known_keys::CODE).build().await? @@ -462,9 +462,10 @@ impl CliConfiguration for TryRuntimeCmd { /// Extract `:code` from the given chain spec and return as `StorageData` along with the /// corresponding `StorageKey`. -fn extract_code(spec: Box) -> sc_cli::Result<(StorageKey, StorageData)> { - let state_version = spec.state_versions() - .ok_or_else(|| "Invalid state versions for chain spec".into())? +fn extract_code(spec: Box) -> sc_cli::Result<(StorageKey, StorageData)> { + let state_version = sp_runtime::StateVersions::::from_conf(spec.state_versions() + .iter().map(|(number, version)| (number.as_str(), *version))) + .ok_or_else(|| "Invalid state versions for chain spec".to_string())? .genesis_state_version(); let genesis_storage = spec.build_storage(state_version)?; From cb0cc704f735a49c90e75d3d5045de47d84769c3 Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Sun, 22 Aug 2021 12:28:34 +0200 Subject: [PATCH 074/127] partial change (if not removing param, keep chainspec util). --- bin/node/cli/src/chain_spec.rs | 1 - client/chain-spec/src/chain_spec.rs | 23 +++++++++++++---------- primitives/runtime/src/lib.rs | 18 ++++++++++-------- test-utils/test-runner/src/utils.rs | 7 +------ 4 files changed, 24 insertions(+), 25 deletions(-) diff --git a/bin/node/cli/src/chain_spec.rs b/bin/node/cli/src/chain_spec.rs index 1baa8d6d1e472..1a03998080636 100644 --- a/bin/node/cli/src/chain_spec.rs +++ b/bin/node/cli/src/chain_spec.rs @@ -490,7 +490,6 @@ pub(crate) mod tests { #[test] fn test_create_development_chain_spec() { development_config().build_storage(Default::default()).unwrap(); - } #[test] diff --git a/client/chain-spec/src/chain_spec.rs b/client/chain-spec/src/chain_spec.rs index 14dd3c0d0a85f..2842c19917fe6 100644 --- a/client/chain-spec/src/chain_spec.rs +++ b/client/chain-spec/src/chain_spec.rs @@ -99,7 +99,7 @@ impl GenesisSource { } impl BuildStorage for ChainSpec { - fn build_storage(&self, state_version: StateVersion) -> Result { + fn build_storage(&self) -> Result { match self.genesis.resolve()? { Genesis::Runtime(gc) => gc.build_storage(state_version), Genesis::Raw(RawGenesis { top: map, children_default: children_map }) => Ok(Storage { @@ -121,7 +121,7 @@ impl BuildStorage for ChainSpec { } } - fn assimilate_storage(&self, _: &mut Storage, _: StateVersion) -> Result<(), String> { + fn assimilate_storage(&self, _: &mut Storage) -> Result<(), String> { Err("`assimilate_storage` not implemented for `ChainSpec`.".into()) } } @@ -280,6 +280,16 @@ impl ChainSpec { fn state_versions(&self) -> &Vec<(String, StateVersion)> { &self.client_spec.state_versions } + + /// Return genesis state version. + fn genesis_state_version(&self) -> StateVersion { + use std::str::FromStr; + let state_version = self.state_versions().get(0) + // This is incorrect (can have number representation not compatible with u64 + .and_then(|(n, s)| u64::from_str(n).ok().map(|n| (n, s))) + .and_then(|(n, s)| (n == 0).then(|| s.clone())) + .unwrap_or_default(); + } } impl ChainSpec { @@ -311,14 +321,7 @@ impl ChainSpec { fn json_container(&self, raw: bool) -> Result, String> { let genesis = match (raw, self.genesis.resolve()?) { (true, Genesis::Runtime(g)) => { - use std::str::FromStr; - let state_version = self.state_versions().get(0) - // This is incorrect (can have number representation not compatible with u64 - .and_then(|(n, s)| u64::from_str(n).ok().map(|n| (n, s))) - .and_then(|(n, s)| (n == 0).then(|| s.clone())) - .unwrap_or_default(); - // TODO remove param from build storage and make it part of caller. - let storage = g.build_storage(state_version)?; + let storage = g.build_storage()?; let top = storage.top.into_iter().map(|(k, v)| (StorageKey(k), StorageData(v))).collect(); let children_default = storage diff --git a/primitives/runtime/src/lib.rs b/primitives/runtime/src/lib.rs index 7300949f67141..9cfb6478c0658 100644 --- a/primitives/runtime/src/lib.rs +++ b/primitives/runtime/src/lib.rs @@ -172,14 +172,16 @@ pub use serde::{de::DeserializeOwned, Deserialize, Serialize}; /// Complex storage builder stuff. #[cfg(feature = "std")] pub trait BuildStorage { + /// State version to use with storage. + fn state_version(&self) -> StateVersion; /// Build the storage out of this builder. - fn build_storage(&self, state_version: StateVersion) -> Result { + fn build_storage(&self) -> Result { let mut storage = Default::default(); - self.assimilate_storage(&mut storage, state_version)?; + self.assimilate_storage(&mut storage)?; Ok(storage) } /// Assimilate the storage for this module into pre-existing overlays. - fn assimilate_storage(&self, storage: &mut sp_core::storage::Storage, state_version: StateVersion) -> Result<(), String>; + fn assimilate_storage(&self, storage: &mut sp_core::storage::Storage) -> Result<(), String>; } /// Something that can build the genesis storage of a module. @@ -194,10 +196,10 @@ pub trait BuildModuleGenesisStorage: Sized { } #[cfg(feature = "std")] -impl BuildStorage for sp_core::storage::Storage { - fn assimilate_storage(&self, storage: &mut sp_core::storage::Storage, _state_version: StateVersion) -> Result<(), String> { - storage.top.extend(self.top.iter().map(|(k, v)| (k.clone(), v.clone()))); - for (k, other_map) in self.children_default.iter() { +impl BuildStorage for (sp_core::storage::Storage, StateVersion) { + fn assimilate_storage(&self, storage: &mut sp_core::storage::Storage) -> Result<(), String> { + storage.top.extend(self.0.top.iter().map(|(k, v)| (k.clone(), v.clone()))); + for (k, other_map) in self.0.children_default.iter() { let k = k.clone(); if let Some(map) = storage.children_default.get_mut(&k) { map.data.extend(other_map.data.iter().map(|(k, v)| (k.clone(), v.clone()))); @@ -214,7 +216,7 @@ impl BuildStorage for sp_core::storage::Storage { #[cfg(feature = "std")] impl BuildStorage for () { - fn assimilate_storage(&self, _: &mut sp_core::storage::Storage, _: StateVersion) -> Result<(), String> { + fn assimilate_storage(&self, _: &mut sp_core::storage::Storage) -> Result<(), String> { Err("`assimilate_storage` not implemented for `()`".into()) } } diff --git a/test-utils/test-runner/src/utils.rs b/test-utils/test-runner/src/utils.rs index 330ff4c8098a1..07afc283db4aa 100644 --- a/test-utils/test-runner/src/utils.rs +++ b/test-utils/test-runner/src/utils.rs @@ -50,12 +50,7 @@ pub fn default_config( let base_path = base_path(); let root_path = base_path.path().to_path_buf().join("chains").join(chain_spec.id()); - use std::str::FromStr; - let state_version = chain_spec.state_versions().get(0) - // This is incorrect (can have genesis state version as different element in chain_spec). - .and_then(|(n, s)| u64::from_str(n).ok().map(|n| (n, s))) - .and_then(|(n, s)| (n == 0).then(|| s.clone())) - .expect("Invalid state versions for chain spec"); + let state_version = chain_spec.genesis_state_versions(); let storage = chain_spec .as_storage_builder() From 7265bc2a10b86a387727d334ee6a495549bdfe5f Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Sun, 22 Aug 2021 18:24:04 +0200 Subject: [PATCH 075/127] pending --- client/chain-spec/src/chain_spec.rs | 12 ++++++++---- client/service/src/client/client.rs | 3 +-- frame/balances/src/lib.rs | 13 +++++++++---- frame/support/src/traits/hooks.rs | 10 +++++++--- frame/system/src/lib.rs | 12 ++++++++---- primitives/runtime/src/lib.rs | 9 +++++++++ 6 files changed, 42 insertions(+), 17 deletions(-) diff --git a/client/chain-spec/src/chain_spec.rs b/client/chain-spec/src/chain_spec.rs index 2842c19917fe6..56ad4efee663e 100644 --- a/client/chain-spec/src/chain_spec.rs +++ b/client/chain-spec/src/chain_spec.rs @@ -99,9 +99,13 @@ impl GenesisSource { } impl BuildStorage for ChainSpec { + fn state_version(&self) -> StateVersion { + self.genesis_state_version() + } + fn build_storage(&self) -> Result { match self.genesis.resolve()? { - Genesis::Runtime(gc) => gc.build_storage(state_version), + Genesis::Runtime(gc) => gc.build_storage(), Genesis::Raw(RawGenesis { top: map, children_default: children_map }) => Ok(Storage { top: map.into_iter().map(|(k, v)| (k.0, v.0)).collect(), children_default: children_map @@ -284,11 +288,11 @@ impl ChainSpec { /// Return genesis state version. fn genesis_state_version(&self) -> StateVersion { use std::str::FromStr; - let state_version = self.state_versions().get(0) + self.state_versions().get(0) // This is incorrect (can have number representation not compatible with u64 .and_then(|(n, s)| u64::from_str(n).ok().map(|n| (n, s))) .and_then(|(n, s)| (n == 0).then(|| s.clone())) - .unwrap_or_default(); + .unwrap_or_default() } } @@ -435,7 +439,7 @@ mod tests { struct Genesis(HashMap); impl BuildStorage for Genesis { - fn assimilate_storage(&self, storage: &mut Storage, _state_version: StateVersion) -> Result<(), String> { + fn assimilate_storage(&self, storage: &mut Storage) -> Result<(), String> { storage.top.extend( self.0.iter().map(|(a, b)| (a.clone().into_bytes(), b.clone().into_bytes())), ); diff --git a/client/service/src/client/client.rs b/client/service/src/client/client.rs index dba698aa81d63..ce53123e75198 100644 --- a/client/service/src/client/client.rs +++ b/client/service/src/client/client.rs @@ -337,11 +337,10 @@ where config: ClientConfig, ) -> sp_blockchain::Result { let info = backend.blockchain().info(); - let state_version = config.state_versions.genesis_state_version(); if info.finalized_state.is_none() { let genesis_storage = - build_genesis_storage.build_storage(state_version).map_err(sp_blockchain::Error::Storage)?; + build_genesis_storage.build_storage().map_err(sp_blockchain::Error::Storage)?; let mut op = backend.begin_operation()?; let state_root = op.set_genesis_state(genesis_storage, !config.no_genesis)?; let genesis_block = genesis::construct_genesis_block::(state_root.into()); diff --git a/frame/balances/src/lib.rs b/frame/balances/src/lib.rs index 9f7148f343ef5..ce9d42776b2b5 100644 --- a/frame/balances/src/lib.rs +++ b/frame/balances/src/lib.rs @@ -584,18 +584,23 @@ pub mod pallet { #[cfg(feature = "std")] impl, I: 'static> GenesisConfig { + /// State version for genesis. + pub fn genesis_state_version(&self) -> StateVersion { + >::genesis_state_version(self) + } + /// Direct implementation of `GenesisBuild::build_storage`. /// /// Kept in order not to break dependency. - pub fn build_storage(&self, state_version: StateVersion) -> Result { - >::build_storage(self, state_version) + pub fn build_storage(&self) -> Result { + >::build_storage(self) } /// Direct implementation of `GenesisBuild::assimilate_storage`. /// /// Kept in order not to break dependency. - pub fn assimilate_storage(&self, storage: &mut sp_runtime::Storage, state_version: StateVersion) -> Result<(), String> { - >::assimilate_storage(self, storage, state_version) + pub fn assimilate_storage(&self, storage: &mut sp_runtime::Storage) -> Result<(), String> { + >::assimilate_storage(self, storage) } } diff --git a/frame/support/src/traits/hooks.rs b/frame/support/src/traits/hooks.rs index d43463f18edd9..37c941d944e34 100644 --- a/frame/support/src/traits/hooks.rs +++ b/frame/support/src/traits/hooks.rs @@ -307,15 +307,19 @@ pub trait GenesisBuild: Default + MaybeSerializeDeserialize { /// Thus one can write to storage using regular pallet storages. fn build(&self); + /// State version for genesis. + fn genesis_state_version(&self) -> StateVersion; + /// Build the storage using `build` inside default storage. - fn build_storage(&self, state_version: StateVersion) -> Result { + fn build_storage(&self) -> Result { let mut storage = Default::default(); - self.assimilate_storage(&mut storage, state_version)?; + self.assimilate_storage(&mut storage)?; Ok(storage) } /// Assimilate the storage for this module into pre-existing overlays. - fn assimilate_storage(&self, storage: &mut sp_runtime::Storage, state_version: StateVersion) -> Result<(), String> { + fn assimilate_storage(&self, storage: &mut sp_runtime::Storage) -> Result<(), String> { + let state_version = self.genesis_state_version(); sp_state_machine::BasicExternalities::execute_with_storage(storage, state_version, || { self.build(); Ok(()) diff --git a/frame/system/src/lib.rs b/frame/system/src/lib.rs index 506e18c70554b..13bdf10aad942 100644 --- a/frame/system/src/lib.rs +++ b/frame/system/src/lib.rs @@ -730,11 +730,16 @@ pub mod migrations { #[cfg(feature = "std")] impl GenesisConfig { + /// State version to use for genesis build. + pub fn genesis_state_version(&self) -> StateVersion { + >::genesis_state_version(self) + } + /// Direct implementation of `GenesisBuild::build_storage`. /// /// Kept in order not to break dependency. - pub fn build_storage(&self, state_version: sp_runtime::StateVersion) -> Result { - >::build_storage(self, state_version) + pub fn build_storage(&self) -> Result { + >::build_storage(self) } /// Direct implementation of `GenesisBuild::assimilate_storage`. @@ -743,9 +748,8 @@ impl GenesisConfig { pub fn assimilate_storage( &self, storage: &mut sp_runtime::Storage, - state_version: sp_runtime::StateVersion, ) -> Result<(), String> { - >::assimilate_storage(self, storage, state_version) + >::assimilate_storage(self, storage) } } diff --git a/primitives/runtime/src/lib.rs b/primitives/runtime/src/lib.rs index 9cfb6478c0658..00ee44d2f2ff1 100644 --- a/primitives/runtime/src/lib.rs +++ b/primitives/runtime/src/lib.rs @@ -197,6 +197,10 @@ pub trait BuildModuleGenesisStorage: Sized { #[cfg(feature = "std")] impl BuildStorage for (sp_core::storage::Storage, StateVersion) { + fn state_version(&self) -> StateVersion { + self.1 + } + fn assimilate_storage(&self, storage: &mut sp_core::storage::Storage) -> Result<(), String> { storage.top.extend(self.0.top.iter().map(|(k, v)| (k.clone(), v.clone()))); for (k, other_map) in self.0.children_default.iter() { @@ -216,6 +220,11 @@ impl BuildStorage for (sp_core::storage::Storage, StateVersion) { #[cfg(feature = "std")] impl BuildStorage for () { + fn state_version(&self) -> StateVersion { + // Warning just a stub implementation, should not be use. + StateVersion::default() + } + fn assimilate_storage(&self, _: &mut sp_core::storage::Storage) -> Result<(), String> { Err("`assimilate_storage` not implemented for `()`".into()) } From 2e77b5a9467a1e7cf96243d7472132d0e2db2654 Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Sun, 22 Aug 2021 19:58:13 +0200 Subject: [PATCH 076/127] pending TODO check state_version needed for Genesisbuild runtime trait. --- frame/system/src/lib.rs | 11 ++++++++--- frame/transaction-payment/src/lib.rs | 1 + 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/frame/system/src/lib.rs b/frame/system/src/lib.rs index 13bdf10aad942..4b16b1277e2c7 100644 --- a/frame/system/src/lib.rs +++ b/frame/system/src/lib.rs @@ -73,7 +73,7 @@ use sp_runtime::{ CheckEqual, Dispatchable, Hash, Lookup, LookupError, MaybeDisplay, MaybeMallocSizeOf, MaybeSerializeDeserialize, Member, One, Saturating, SimpleBitOps, StaticLookup, Zero, }, - DispatchError, Either, Perbill, RuntimeDebug, + DispatchError, Either, Perbill, RuntimeDebug, StateVersion, }; #[cfg(any(feature = "std", test))] use sp_std::map; @@ -658,17 +658,22 @@ pub mod pallet { pub changes_trie_config: Option, #[serde(with = "sp_core::bytes")] pub code: Vec, + pub genesis_state_version: StateVersion, // TODO this requires to eventually rewrite to V0 for existing chain. } #[cfg(feature = "std")] impl Default for GenesisConfig { fn default() -> Self { - Self { changes_trie_config: Default::default(), code: Default::default() } + Self { changes_trie_config: Default::default(), code: Default::default(), genesis_state_version: Default::default() } } } #[pallet::genesis_build] impl GenesisBuild for GenesisConfig { + fn genesis_state_version(&self) -> sp_runtime::StateVersion { + self.genesis_state_version + } + fn build(&self) { >::insert::<_, T::Hash>(T::BlockNumber::zero(), hash69()); >::put::(hash69()); @@ -731,7 +736,7 @@ pub mod migrations { #[cfg(feature = "std")] impl GenesisConfig { /// State version to use for genesis build. - pub fn genesis_state_version(&self) -> StateVersion { + pub fn genesis_state_version(&self) -> sp_runtime::StateVersion { >::genesis_state_version(self) } diff --git a/frame/transaction-payment/src/lib.rs b/frame/transaction-payment/src/lib.rs index ae9ac087c07ee..3587bc84c620e 100644 --- a/frame/transaction-payment/src/lib.rs +++ b/frame/transaction-payment/src/lib.rs @@ -305,6 +305,7 @@ pub mod pallet { #[pallet::genesis_build] impl GenesisBuild for GenesisConfig { + fn genesis_config(&self) { fn build(&self) { StorageVersion::::put(Releases::V2); } From a036fdb448bd18c22cadc3a1cf7b60a7eb98c4a6 Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 23 Aug 2021 12:27:41 +0200 Subject: [PATCH 077/127] before restoring version param for build storage only --- frame/balances/src/lib.rs | 6 ---- frame/session/src/historical/mod.rs | 6 ++-- frame/session/src/historical/offchain.rs | 6 ++-- frame/session/src/mock.rs | 6 ++-- .../src/construct_runtime/expand/config.rs | 33 ++++++++++------- .../src/pallet/expand/genesis_build.rs | 4 +-- .../src/storage/genesis_config/mod.rs | 10 +++--- frame/support/src/traits/hooks.rs | 7 +--- frame/support/test/tests/instance.rs | 3 +- frame/system/src/lib.rs | 14 ++------ frame/transaction-payment/src/lib.rs | 1 - frame/treasury/src/lib.rs | 3 +- primitives/runtime/src/lib.rs | 1 - primitives/state-machine/src/basic.rs | 36 +++++++++++++++---- test-utils/client/src/lib.rs | 2 +- test-utils/runtime/src/genesismap.rs | 4 +-- test-utils/test-runner/src/utils.rs | 4 +-- utils/frame/benchmarking-cli/src/command.rs | 9 +---- utils/frame/try-runtime/cli/src/lib.rs | 7 +--- 19 files changed, 76 insertions(+), 86 deletions(-) diff --git a/frame/balances/src/lib.rs b/frame/balances/src/lib.rs index ce9d42776b2b5..7ab8a54de232d 100644 --- a/frame/balances/src/lib.rs +++ b/frame/balances/src/lib.rs @@ -183,7 +183,6 @@ use sp_runtime::{ Saturating, StaticLookup, Zero, }, ArithmeticError, DispatchError, DispatchResult, RuntimeDebug, - StateVersion, }; use sp_std::{cmp, fmt::Debug, mem, ops::BitOr, prelude::*, result}; pub use weights::WeightInfo; @@ -584,11 +583,6 @@ pub mod pallet { #[cfg(feature = "std")] impl, I: 'static> GenesisConfig { - /// State version for genesis. - pub fn genesis_state_version(&self) -> StateVersion { - >::genesis_state_version(self) - } - /// Direct implementation of `GenesisBuild::build_storage`. /// /// Kept in order not to break dependency. diff --git a/frame/session/src/historical/mod.rs b/frame/session/src/historical/mod.rs index 3a69aa2086fab..2600e3052ef0b 100644 --- a/frame/session/src/historical/mod.rs +++ b/frame/session/src/historical/mod.rs @@ -376,16 +376,16 @@ pub(crate) mod tests { pub(crate) fn new_test_ext() -> sp_io::TestExternalities { let state_version = Default::default(); - let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); + let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); let keys: Vec<_> = NEXT_VALIDATORS.with(|l| { l.borrow().iter().cloned().map(|i| (i, i, UintAuthorityId(i).into())).collect() }); - BasicExternalities::execute_with_storage(&mut t, state_version, || { + BasicExternalities::execute_with_storage(&mut t, || { for (ref k, ..) in &keys { frame_system::Pallet::::inc_providers(k); } }); - crate::GenesisConfig:: { keys }.assimilate_storage(&mut t, state_version).unwrap(); + crate::GenesisConfig:: { keys }.assimilate_storage(&mut t).unwrap(); sp_io::TestExternalities::new(t, state_version) } diff --git a/frame/session/src/historical/offchain.rs b/frame/session/src/historical/offchain.rs index c969636da77de..ed4dd2fbb2659 100644 --- a/frame/session/src/historical/offchain.rs +++ b/frame/session/src/historical/offchain.rs @@ -164,19 +164,19 @@ mod tests { pub fn new_test_ext() -> sp_io::TestExternalities { let state_version = Default::default(); let mut t = frame_system::GenesisConfig::default() - .build_storage::(state_version) + .build_storage::() .expect("Failed to create test externalities."); let keys: Vec<_> = NEXT_VALIDATORS.with(|l| { l.borrow().iter().cloned().map(|i| (i, i, UintAuthorityId(i).into())).collect() }); - BasicExternalities::execute_with_storage(&mut t, state_version, || { + BasicExternalities::execute_with_storage(&mut t, || { for (ref k, ..) in &keys { frame_system::Pallet::::inc_providers(k); } }); - crate::GenesisConfig:: { keys }.assimilate_storage(&mut t, state_version).unwrap(); + crate::GenesisConfig:: { keys }.assimilate_storage(&mut t).unwrap(); let mut ext = sp_io::TestExternalities::new(t, state_version); diff --git a/frame/session/src/mock.rs b/frame/session/src/mock.rs index 242b603f5abc9..2879a8e006192 100644 --- a/frame/session/src/mock.rs +++ b/frame/session/src/mock.rs @@ -210,10 +210,10 @@ pub fn reset_before_session_end_called() { pub fn new_test_ext() -> sp_io::TestExternalities { let state_version = Default::default(); - let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); + let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); let keys: Vec<_> = NEXT_VALIDATORS .with(|l| l.borrow().iter().cloned().map(|i| (i, i, UintAuthorityId(i).into())).collect()); - BasicExternalities::execute_with_storage(&mut t, state_version, || { + BasicExternalities::execute_with_storage(&mut t, || { for (ref k, ..) in &keys { frame_system::Pallet::::inc_providers(k); } @@ -222,7 +222,7 @@ pub fn new_test_ext() -> sp_io::TestExternalities { frame_system::Pallet::::inc_providers(&69); }); pallet_session::GenesisConfig:: { keys } - .assimilate_storage(&mut t, state_version) + .assimilate_storage(&mut t) .unwrap(); sp_io::TestExternalities::new(t,state_version) } diff --git a/frame/support/procedural/src/construct_runtime/expand/config.rs b/frame/support/procedural/src/construct_runtime/expand/config.rs index f57c7cee6f72c..20df765a04a1e 100644 --- a/frame/support/procedural/src/construct_runtime/expand/config.rs +++ b/frame/support/procedural/src/construct_runtime/expand/config.rs @@ -63,32 +63,41 @@ pub fn expand_outer_config( #types #[cfg(any(feature = "std", test))] - use #scrate::serde as __genesis_config_serde_import__; - #[cfg(any(feature = "std", test))] - #[derive(#scrate::serde::Serialize, #scrate::serde::Deserialize, Default)] - #[serde(rename_all = "camelCase")] - #[serde(deny_unknown_fields)] - #[serde(crate = "__genesis_config_serde_import__")] - pub struct GenesisConfig { - #fields + pub struct GenesisConfigWithStateVersion { + pub genesis_config: GenesisConfig, + pub genesis_state_version: #scrate::sp_runtime::StateVersion, } #[cfg(any(feature = "std", test))] - impl #scrate::sp_runtime::BuildStorage for GenesisConfig { + impl #scrate::sp_runtime::BuildStorage for GenesisConfigWithStateVersion { + fn state_version(&self) -> #scrate::sp_runtime::StateVersion { + self.genesis_state_version + } + fn assimilate_storage( &self, storage: &mut #scrate::sp_runtime::Storage, - state_version: #scrate::sp_runtime::StateVersion, ) -> std::result::Result<(), String> { #build_storage_calls - #scrate::BasicExternalities::execute_with_storage(storage, state_version, || { + #scrate::BasicExternalities::execute_with_storage_and_state(storage, self.genesis_state_version, || { ::on_genesis(); }); Ok(()) } } + + #[cfg(any(feature = "std", test))] + use #scrate::serde as __genesis_config_serde_import__; + #[cfg(any(feature = "std", test))] + #[derive(#scrate::serde::Serialize, #scrate::serde::Deserialize, Default)] + #[serde(rename_all = "camelCase")] + #[serde(deny_unknown_fields)] + #[serde(crate = "__genesis_config_serde_import__")] + pub struct GenesisConfig { + #fields + } } } @@ -131,6 +140,6 @@ fn expand_config_build_storage_call( quote! { #scrate::sp_runtime::BuildModuleGenesisStorage:: - <#runtime, #instance>::build_module_genesis_storage(&self.#field_name, storage, state_version)?; + <#runtime, #instance>::build_module_genesis_storage(&self.genesis_config.#field_name, storage)?; } } diff --git a/frame/support/procedural/src/pallet/expand/genesis_build.rs b/frame/support/procedural/src/pallet/expand/genesis_build.rs index d9d1d66537a2d..06acaf324254c 100644 --- a/frame/support/procedural/src/pallet/expand/genesis_build.rs +++ b/frame/support/procedural/src/pallet/expand/genesis_build.rs @@ -61,10 +61,8 @@ pub fn expand_genesis_build(def: &mut Def) -> proc_macro2::TokenStream { fn build_module_genesis_storage( &self, storage: &mut #frame_support::sp_runtime::Storage, - state_version: #frame_support::sp_runtime::StateVersion, - ) -> std::result::Result<(), std::string::String> { - #frame_support::BasicExternalities::execute_with_storage(storage, state_version, || { + #frame_support::BasicExternalities::execute_with_storage(storage, || { >::build(self); Ok(()) }) diff --git a/frame/support/procedural/src/storage/genesis_config/mod.rs b/frame/support/procedural/src/storage/genesis_config/mod.rs index a78adfbc18639..d2d1afb017736 100644 --- a/frame/support/procedural/src/storage/genesis_config/mod.rs +++ b/frame/support/procedural/src/storage/genesis_config/mod.rs @@ -139,12 +139,12 @@ fn impl_build_storage( #[cfg(feature = "std")] impl#genesis_impl GenesisConfig#genesis_struct #genesis_where_clause { /// Build the storage for this module. - pub fn build_storage #fn_generic (&self, state_version: #scrate::sp_runtime::StateVersion) -> std::result::Result< + pub fn build_storage #fn_generic (&self) -> std::result::Result< #scrate::sp_runtime::Storage, String > #fn_where_clause { let mut storage = Default::default(); - self.assimilate_storage::<#fn_traitinstance>(&mut storage, state_version)?; + self.assimilate_storage::<#fn_traitinstance>(&mut storage)?; Ok(storage) } @@ -152,9 +152,8 @@ fn impl_build_storage( pub fn assimilate_storage #fn_generic ( &self, storage: &mut #scrate::sp_runtime::Storage, - state_version: #scrate::sp_runtime::StateVersion, ) -> std::result::Result<(), String> #fn_where_clause { - #scrate::BasicExternalities::execute_with_storage(storage, state_version, || { + #scrate::BasicExternalities::execute_with_storage(storage, || { #( #builder_blocks )* Ok(()) }) @@ -168,9 +167,8 @@ fn impl_build_storage( fn build_module_genesis_storage( &self, storage: &mut #scrate::sp_runtime::Storage, - state_version: #scrate::sp_runtime::StateVersion, ) -> std::result::Result<(), String> { - self.assimilate_storage::<#fn_traitinstance> (storage, state_version) + self.assimilate_storage::<#fn_traitinstance> (storage) } } } diff --git a/frame/support/src/traits/hooks.rs b/frame/support/src/traits/hooks.rs index 37c941d944e34..965cce234288f 100644 --- a/frame/support/src/traits/hooks.rs +++ b/frame/support/src/traits/hooks.rs @@ -20,7 +20,6 @@ use impl_trait_for_tuples::impl_for_tuples; use sp_arithmetic::traits::Saturating; use sp_runtime::traits::{AtLeast32BitUnsigned, MaybeSerializeDeserialize}; -use sp_runtime::StateVersion; /// The block initialization trait. /// @@ -307,9 +306,6 @@ pub trait GenesisBuild: Default + MaybeSerializeDeserialize { /// Thus one can write to storage using regular pallet storages. fn build(&self); - /// State version for genesis. - fn genesis_state_version(&self) -> StateVersion; - /// Build the storage using `build` inside default storage. fn build_storage(&self) -> Result { let mut storage = Default::default(); @@ -319,8 +315,7 @@ pub trait GenesisBuild: Default + MaybeSerializeDeserialize { /// Assimilate the storage for this module into pre-existing overlays. fn assimilate_storage(&self, storage: &mut sp_runtime::Storage) -> Result<(), String> { - let state_version = self.genesis_state_version(); - sp_state_machine::BasicExternalities::execute_with_storage(storage, state_version, || { + sp_state_machine::BasicExternalities::execute_with_storage(storage, || { self.build(); Ok(()) }) diff --git a/frame/support/test/tests/instance.rs b/frame/support/test/tests/instance.rs index ee41b4b0f0309..5e18d964660a2 100644 --- a/frame/support/test/tests/instance.rs +++ b/frame/support/test/tests/instance.rs @@ -342,12 +342,11 @@ fn new_test_ext() -> sp_io::TestExternalities { #[test] fn storage_instance_independence() { - let state_version = Default::default(); let mut storage = sp_core::storage::Storage { top: std::collections::BTreeMap::new(), children_default: std::collections::HashMap::new(), }; - sp_state_machine::BasicExternalities::execute_with_storage(&mut storage, state_version, || { + sp_state_machine::BasicExternalities::execute_with_storage(&mut storage, || { module2::Value::::put(0); module2::Value::::put(0); module2::Value::::put(0); diff --git a/frame/system/src/lib.rs b/frame/system/src/lib.rs index 4b16b1277e2c7..590b87d1385df 100644 --- a/frame/system/src/lib.rs +++ b/frame/system/src/lib.rs @@ -73,7 +73,7 @@ use sp_runtime::{ CheckEqual, Dispatchable, Hash, Lookup, LookupError, MaybeDisplay, MaybeMallocSizeOf, MaybeSerializeDeserialize, Member, One, Saturating, SimpleBitOps, StaticLookup, Zero, }, - DispatchError, Either, Perbill, RuntimeDebug, StateVersion, + DispatchError, Either, Perbill, RuntimeDebug, }; #[cfg(any(feature = "std", test))] use sp_std::map; @@ -658,22 +658,17 @@ pub mod pallet { pub changes_trie_config: Option, #[serde(with = "sp_core::bytes")] pub code: Vec, - pub genesis_state_version: StateVersion, // TODO this requires to eventually rewrite to V0 for existing chain. } #[cfg(feature = "std")] impl Default for GenesisConfig { fn default() -> Self { - Self { changes_trie_config: Default::default(), code: Default::default(), genesis_state_version: Default::default() } + Self { changes_trie_config: Default::default(), code: Default::default() } } } #[pallet::genesis_build] impl GenesisBuild for GenesisConfig { - fn genesis_state_version(&self) -> sp_runtime::StateVersion { - self.genesis_state_version - } - fn build(&self) { >::insert::<_, T::Hash>(T::BlockNumber::zero(), hash69()); >::put::(hash69()); @@ -735,11 +730,6 @@ pub mod migrations { #[cfg(feature = "std")] impl GenesisConfig { - /// State version to use for genesis build. - pub fn genesis_state_version(&self) -> sp_runtime::StateVersion { - >::genesis_state_version(self) - } - /// Direct implementation of `GenesisBuild::build_storage`. /// /// Kept in order not to break dependency. diff --git a/frame/transaction-payment/src/lib.rs b/frame/transaction-payment/src/lib.rs index 3587bc84c620e..ae9ac087c07ee 100644 --- a/frame/transaction-payment/src/lib.rs +++ b/frame/transaction-payment/src/lib.rs @@ -305,7 +305,6 @@ pub mod pallet { #[pallet::genesis_build] impl GenesisBuild for GenesisConfig { - fn genesis_config(&self) { fn build(&self) { StorageVersion::::put(Releases::V2); } diff --git a/frame/treasury/src/lib.rs b/frame/treasury/src/lib.rs index 88fb0a52a0db1..965f06731c945 100644 --- a/frame/treasury/src/lib.rs +++ b/frame/treasury/src/lib.rs @@ -234,9 +234,8 @@ pub mod pallet { pub fn assimilate_storage, I: 'static>( &self, storage: &mut sp_runtime::Storage, - state_version: sp_runtime::StateVersion, ) -> Result<(), String> { - >::assimilate_storage(self, storage, state_version) + >::assimilate_storage(self, storage) } } diff --git a/primitives/runtime/src/lib.rs b/primitives/runtime/src/lib.rs index 00ee44d2f2ff1..264b839e59c59 100644 --- a/primitives/runtime/src/lib.rs +++ b/primitives/runtime/src/lib.rs @@ -191,7 +191,6 @@ pub trait BuildModuleGenesisStorage: Sized { fn build_module_genesis_storage( &self, storage: &mut sp_core::storage::Storage, - state_version: StateVersion, ) -> Result<(), String>; } diff --git a/primitives/state-machine/src/basic.rs b/primitives/state-machine/src/basic.rs index 106e217850151..3cc3d7d8aa83a 100644 --- a/primitives/state-machine/src/basic.rs +++ b/primitives/state-machine/src/basic.rs @@ -42,13 +42,13 @@ use std::{ pub struct BasicExternalities { inner: Storage, extensions: Extensions, - state_version: StateVersion, // TODO same alt hashing type as all + state_version: Option, } impl BasicExternalities { /// Create a new instance of `BasicExternalities` pub fn new(inner: Storage, state_version: StateVersion) -> Self { - BasicExternalities { inner, extensions: Default::default(), state_version } + BasicExternalities { inner, extensions: Default::default(), state_version: Some(state_version) } } /// New basic externalities with empty storage. @@ -69,10 +69,33 @@ impl BasicExternalities { /// Execute the given closure `f` with the externalities set and initialized with `storage`. /// /// Returns the result of the closure and updates `storage` with all changes. - pub fn execute_with_storage( + /// + /// Do not support runtime transaction. + pub fn execute_with_storage_and_state( storage: &mut sp_core::storage::Storage, state_version: StateVersion, f: impl FnOnce() -> R, + ) -> R { + Self::execute_with_storage_inner(storage, Some(state_version), f) + } + + /// Execute the given closure `f` with the externalities set and initialized with `storage`. + /// + /// Returns the result of the closure and updates `storage` with all changes. + /// + /// Do not support runtime transaction and root calculation. + /// This limitation is fine for most genesis runtime storage initialization. + pub fn execute_with_storage( + storage: &mut sp_core::storage::Storage, + f: impl FnOnce() -> R, + ) -> R { + Self::execute_with_storage_inner(storage, None, f) + } + + fn execute_with_storage_inner( + storage: &mut sp_core::storage::Storage, + state_version: Option, + f: impl FnOnce() -> R, ) -> R { let mut ext = Self { inner: Storage { @@ -126,7 +149,7 @@ impl From<(BTreeMap, StateVersion)> for BasicExternali BasicExternalities { inner: Storage { top: hashmap, children_default: Default::default() }, extensions: Default::default(), - state_version, + state_version: Some(state_version), } } } @@ -290,7 +313,7 @@ impl Externalities for BasicExternalities { } } - let layout = match self.state_version { + let layout = match self.state_version.expect("Unsupported state calculation for genesis storage build.") { StateVersion::V0 => { Layout::::default() }, @@ -302,9 +325,10 @@ impl Externalities for BasicExternalities { } fn child_storage_root(&mut self, child_info: &ChildInfo) -> Vec { + let state_version = self.state_version.expect("Unsupported state calculation for genesis storage build."); if let Some(child) = self.inner.children_default.get(child_info.storage_key()) { let delta = child.data.iter().map(|(k, v)| (k.as_ref(), Some(v.as_ref()))); - let in_mem = crate::in_memory_backend::new_in_mem::(self.state_version.clone()); + let in_mem = crate::in_memory_backend::new_in_mem::(state_version); in_mem.child_storage_root(&child.child_info, delta).0 } else { empty_child_trie_root::>() diff --git a/test-utils/client/src/lib.rs b/test-utils/client/src/lib.rs index 8df3d9ea84788..b70263a7d77d4 100644 --- a/test-utils/client/src/lib.rs +++ b/test-utils/client/src/lib.rs @@ -241,7 +241,7 @@ impl ); } - storage + (storage, sp_runtime::StateVersion::default()) }; let client = client::Client::new( diff --git a/test-utils/runtime/src/genesismap.rs b/test-utils/runtime/src/genesismap.rs index aab2fe5a9210b..55441b4e186e9 100644 --- a/test-utils/runtime/src/genesismap.rs +++ b/test-utils/runtime/src/genesismap.rs @@ -37,7 +37,7 @@ pub struct GenesisConfig { heap_pages_override: Option, /// Additional storage key pairs that will be added to the genesis map. extra_storage: Storage, - state_version: StateVersion, + state_version: StateVersion, // TODO check if in use } impl GenesisConfig { @@ -93,7 +93,7 @@ impl GenesisConfig { let mut config = system::GenesisConfig::default(); config.authorities = self.authorities.clone(); config - .assimilate_storage(&mut storage, self.state_version.clone()) + .assimilate_storage(&mut storage) .expect("Adding `system::GensisConfig` to the genesis"); storage diff --git a/test-utils/test-runner/src/utils.rs b/test-utils/test-runner/src/utils.rs index 07afc283db4aa..2fe3a98d44ad0 100644 --- a/test-utils/test-runner/src/utils.rs +++ b/test-utils/test-runner/src/utils.rs @@ -50,11 +50,9 @@ pub fn default_config( let base_path = base_path(); let root_path = base_path.path().to_path_buf().join("chains").join(chain_spec.id()); - let state_version = chain_spec.genesis_state_versions(); - let storage = chain_spec .as_storage_builder() - .build_storage(state_version) + .build_storage() .expect("could not build storage"); chain_spec.set_storage(storage); diff --git a/utils/frame/benchmarking-cli/src/command.rs b/utils/frame/benchmarking-cli/src/command.rs index c64e4573f7bc3..671386a721a02 100644 --- a/utils/frame/benchmarking-cli/src/command.rs +++ b/utils/frame/benchmarking-cli/src/command.rs @@ -96,13 +96,6 @@ impl BenchmarkCmd { ::Hash: std::str::FromStr, ExecDispatch: NativeExecutionDispatch + 'static, { - let state_version = if let Some(versions) = sp_runtime::StateVersions::::from_conf(config.chain_spec.state_versions() - .iter().map(|(number, version)| (number.as_str(), *version))) { - versions.genesis_state_version() - } else { - return Err("Invalid state versions for chain spec".into()) - }; - if let Some(output_path) = &self.output { if !output_path.is_dir() && output_path.file_name().is_none() { return Err("Output file or path is invalid!".into()) @@ -129,7 +122,7 @@ impl BenchmarkCmd { let extrinsic = self.extrinsic.clone().unwrap_or_else(|| String::new()); let extrinsic = extrinsic.as_bytes(); - let genesis_storage = spec.build_storage(state_version)?; + let genesis_storage = spec.build_storage()?; let mut changes = Default::default(); let cache_size = Some(self.database_cache_size as usize); let state_with_tracking = BenchmarkingState::::new( diff --git a/utils/frame/try-runtime/cli/src/lib.rs b/utils/frame/try-runtime/cli/src/lib.rs index c1f7dce9d34bd..46168b423dbff 100644 --- a/utils/frame/try-runtime/cli/src/lib.rs +++ b/utils/frame/try-runtime/cli/src/lib.rs @@ -463,12 +463,7 @@ impl CliConfiguration for TryRuntimeCmd { /// Extract `:code` from the given chain spec and return as `StorageData` along with the /// corresponding `StorageKey`. fn extract_code(spec: Box) -> sc_cli::Result<(StorageKey, StorageData)> { - let state_version = sp_runtime::StateVersions::::from_conf(spec.state_versions() - .iter().map(|(number, version)| (number.as_str(), *version))) - .ok_or_else(|| "Invalid state versions for chain spec".to_string())? - .genesis_state_version(); - - let genesis_storage = spec.build_storage(state_version)?; + let genesis_storage = spec.build_storage()?; let code = StorageData( genesis_storage .top From c38f36ea4f8b6e07482322f553b700f9140a2b5d Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 23 Aug 2021 15:38:19 +0200 Subject: [PATCH 078/127] No state for genesis storage building. --- bin/node-template/node/src/chain_spec.rs | 4 +++ bin/node/cli/src/chain_spec.rs | 3 ++ bin/node/testing/src/client.rs | 3 +- bin/utils/chain-spec-builder/src/main.rs | 1 + client/chain-spec/src/chain_spec.rs | 3 +- .../src/construct_runtime/expand/config.rs | 30 ++++++++----------- primitives/state-machine/src/basic.rs | 2 +- test-utils/runtime/client/src/lib.rs | 1 - test-utils/runtime/src/genesismap.rs | 5 +--- 9 files changed, 25 insertions(+), 27 deletions(-) diff --git a/bin/node-template/node/src/chain_spec.rs b/bin/node-template/node/src/chain_spec.rs index 7009b3be5c279..6556167c13363 100644 --- a/bin/node-template/node/src/chain_spec.rs +++ b/bin/node-template/node/src/chain_spec.rs @@ -72,6 +72,8 @@ pub fn development_config() -> Result { None, // Extensions None, + // StateVersion + Default::default(), )) } @@ -119,6 +121,8 @@ pub fn local_testnet_config() -> Result { None, // Extensions None, + // StateVersion + Default::default(), )) } diff --git a/bin/node/cli/src/chain_spec.rs b/bin/node/cli/src/chain_spec.rs index 1a03998080636..20752d27d9614 100644 --- a/bin/node/cli/src/chain_spec.rs +++ b/bin/node/cli/src/chain_spec.rs @@ -197,6 +197,7 @@ pub fn staging_testnet_config() -> ChainSpec { None, None, Default::default(), + Default::default(), ) } @@ -387,6 +388,7 @@ pub fn development_config() -> ChainSpec { None, None, Default::default(), + Default::default(), ) } @@ -411,6 +413,7 @@ pub fn local_testnet_config() -> ChainSpec { None, None, Default::default(), + Default::default(), ) } diff --git a/bin/node/testing/src/client.rs b/bin/node/testing/src/client.rs index df4c93874c388..9538cd47d88a6 100644 --- a/bin/node/testing/src/client.rs +++ b/bin/node/testing/src/client.rs @@ -48,8 +48,7 @@ pub struct GenesisParameters { impl substrate_test_client::GenesisInit for GenesisParameters { fn genesis_storage(&self) -> Storage { - let state_version = Default::default(); - crate::genesis::config(self.support_changes_trie, None).build_storage(state_version).unwrap() + crate::genesis::config(self.support_changes_trie, None).build_storage().unwrap() } } diff --git a/bin/utils/chain-spec-builder/src/main.rs b/bin/utils/chain-spec-builder/src/main.rs index bf5f1a149578e..1b74261638529 100644 --- a/bin/utils/chain-spec-builder/src/main.rs +++ b/bin/utils/chain-spec-builder/src/main.rs @@ -156,6 +156,7 @@ fn generate_chain_spec( None, None, Default::default(), + Default::default(), ); chain_spec.as_json(false).map_err(|err| err) diff --git a/client/chain-spec/src/chain_spec.rs b/client/chain-spec/src/chain_spec.rs index 56ad4efee663e..a5d9fe705d7fe 100644 --- a/client/chain-spec/src/chain_spec.rs +++ b/client/chain-spec/src/chain_spec.rs @@ -255,6 +255,7 @@ impl ChainSpec { protocol_id: Option<&str>, properties: Option, extensions: E, + state_version: StateVersion, ) -> Self { let client_spec = ClientSpec { name: name.to_owned(), @@ -268,7 +269,7 @@ impl ChainSpec { consensus_engine: (), genesis: Default::default(), code_substitutes: HashMap::new(), - state_versions: vec![("0".to_string(), StateVersion::default())], + state_versions: vec![("0".to_string(), state_version)], }; ChainSpec { client_spec, genesis: GenesisSource::Factory(Arc::new(constructor)) } diff --git a/frame/support/procedural/src/construct_runtime/expand/config.rs b/frame/support/procedural/src/construct_runtime/expand/config.rs index 20df765a04a1e..fd6bea304b1c9 100644 --- a/frame/support/procedural/src/construct_runtime/expand/config.rs +++ b/frame/support/procedural/src/construct_runtime/expand/config.rs @@ -63,15 +63,20 @@ pub fn expand_outer_config( #types #[cfg(any(feature = "std", test))] - pub struct GenesisConfigWithStateVersion { - pub genesis_config: GenesisConfig, - pub genesis_state_version: #scrate::sp_runtime::StateVersion, + use #scrate::serde as __genesis_config_serde_import__; + #[cfg(any(feature = "std", test))] + #[derive(#scrate::serde::Serialize, #scrate::serde::Deserialize, Default)] + #[serde(rename_all = "camelCase")] + #[serde(deny_unknown_fields)] + #[serde(crate = "__genesis_config_serde_import__")] + pub struct GenesisConfig { + #fields } #[cfg(any(feature = "std", test))] - impl #scrate::sp_runtime::BuildStorage for GenesisConfigWithStateVersion { + impl #scrate::sp_runtime::BuildStorage for GenesisConfig { fn state_version(&self) -> #scrate::sp_runtime::StateVersion { - self.genesis_state_version + unimplemented!("Genesis build storage do not support state version"); } fn assimilate_storage( @@ -80,24 +85,13 @@ pub fn expand_outer_config( ) -> std::result::Result<(), String> { #build_storage_calls - #scrate::BasicExternalities::execute_with_storage_and_state(storage, self.genesis_state_version, || { + #scrate::BasicExternalities::execute_with_storage(storage, || { ::on_genesis(); }); Ok(()) } } - - #[cfg(any(feature = "std", test))] - use #scrate::serde as __genesis_config_serde_import__; - #[cfg(any(feature = "std", test))] - #[derive(#scrate::serde::Serialize, #scrate::serde::Deserialize, Default)] - #[serde(rename_all = "camelCase")] - #[serde(deny_unknown_fields)] - #[serde(crate = "__genesis_config_serde_import__")] - pub struct GenesisConfig { - #fields - } } } @@ -140,6 +134,6 @@ fn expand_config_build_storage_call( quote! { #scrate::sp_runtime::BuildModuleGenesisStorage:: - <#runtime, #instance>::build_module_genesis_storage(&self.genesis_config.#field_name, storage)?; + <#runtime, #instance>::build_module_genesis_storage(&self.#field_name, storage)?; } } diff --git a/primitives/state-machine/src/basic.rs b/primitives/state-machine/src/basic.rs index 3cc3d7d8aa83a..3786879d1ed6c 100644 --- a/primitives/state-machine/src/basic.rs +++ b/primitives/state-machine/src/basic.rs @@ -70,7 +70,7 @@ impl BasicExternalities { /// /// Returns the result of the closure and updates `storage` with all changes. /// - /// Do not support runtime transaction. + /// Do not support runtime transaction. TODO useless?? pub fn execute_with_storage_and_state( storage: &mut sp_core::storage::Storage, state_version: StateVersion, diff --git a/test-utils/runtime/client/src/lib.rs b/test-utils/runtime/client/src/lib.rs index dd2893dc4981c..b72fed7d0db4f 100644 --- a/test-utils/runtime/client/src/lib.rs +++ b/test-utils/runtime/client/src/lib.rs @@ -116,7 +116,6 @@ impl GenesisParameters { 1000, self.heap_pages_override, self.extra_storage.clone(), - Default::default(), ) } diff --git a/test-utils/runtime/src/genesismap.rs b/test-utils/runtime/src/genesismap.rs index 55441b4e186e9..a8801b8519dfe 100644 --- a/test-utils/runtime/src/genesismap.rs +++ b/test-utils/runtime/src/genesismap.rs @@ -21,7 +21,7 @@ use super::{system, wasm_binary_unwrap, AccountId, AuthorityId}; use codec::{Encode, Joiner, KeyedVec}; use sc_service::client::genesis; use sp_core::{ - map, state_version::StateVersion, + map, storage::{well_known_keys, Storage}, ChangesTrieConfiguration, }; @@ -37,7 +37,6 @@ pub struct GenesisConfig { heap_pages_override: Option, /// Additional storage key pairs that will be added to the genesis map. extra_storage: Storage, - state_version: StateVersion, // TODO check if in use } impl GenesisConfig { @@ -48,7 +47,6 @@ impl GenesisConfig { balance: u64, heap_pages_override: Option, extra_storage: Storage, - state_version: StateVersion, ) -> Self { GenesisConfig { changes_trie_config, @@ -56,7 +54,6 @@ impl GenesisConfig { balances: endowed_accounts.into_iter().map(|a| (a, balance)).collect(), heap_pages_override, extra_storage, - state_version, } } From 472addd8657b1879a3774586c65987458d9329e7 Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 23 Aug 2021 16:54:57 +0200 Subject: [PATCH 079/127] Revert test to use old method (less frame breakage). --- .../pallets/template/src/mock.rs | 3 +- bin/node/cli/src/chain_spec.rs | 8 +++-- bin/node/executor/benches/bench.rs | 4 +-- bin/node/executor/tests/common.rs | 4 +-- bin/node/runtime/src/impls.rs | 7 ++-- client/chain-spec/src/chain_spec.rs | 4 +++ client/cli/src/commands/insert_key.rs | 1 + client/executor/src/integration_tests/mod.rs | 11 +++---- client/service/src/client/call_executor.rs | 5 ++- client/service/test/src/client/mod.rs | 8 ++--- frame/assets/src/mock.rs | 5 ++- frame/atomic-swap/src/tests.rs | 7 ++-- frame/aura/src/mock.rs | 7 ++-- frame/authority-discovery/src/lib.rs | 6 ++-- frame/authorship/src/lib.rs | 5 ++- frame/babe/src/mock.rs | 11 +++---- frame/balances/src/tests.rs | 10 +++--- frame/balances/src/tests_composite.rs | 7 ++-- frame/balances/src/tests_local.rs | 7 ++-- frame/balances/src/tests_reentrancy.rs | 7 ++-- frame/benchmarking/src/tests.rs | 4 +-- frame/benchmarking/src/tests_instance.rs | 4 +-- frame/bounties/src/tests.rs | 25 +++++++-------- frame/collective/src/lib.rs | 11 +++---- frame/contracts/src/tests.rs | 7 ++-- frame/democracy/src/tests.rs | 9 +++--- .../election-provider-multi-phase/src/mock.rs | 7 ++-- frame/elections-phragmen/src/lib.rs | 7 ++-- frame/elections/src/mock.rs | 7 ++-- frame/example/src/tests.rs | 5 ++- frame/executive/src/lib.rs | 14 ++++---- frame/gilt/src/mock.rs | 9 +++--- frame/grandpa/src/mock.rs | 11 +++---- frame/identity/src/tests.rs | 7 ++-- frame/im-online/src/mock.rs | 5 ++- frame/indices/src/mock.rs | 7 ++-- frame/lottery/src/mock.rs | 7 ++-- frame/membership/src/lib.rs | 10 +++--- frame/merkle-mountain-range/src/tests.rs | 3 +- frame/multisig/src/tests.rs | 7 ++-- frame/nicks/src/lib.rs | 7 ++-- frame/node-authorization/src/mock.rs | 7 ++-- frame/offences/benchmarking/src/mock.rs | 5 ++- frame/offences/src/mock.rs | 5 ++- frame/proxy/src/tests.rs | 7 ++-- frame/randomness-collective-flip/src/lib.rs | 5 ++- frame/recovery/src/mock.rs | 7 ++-- frame/scheduler/src/lib.rs | 5 ++- frame/scored-pool/src/mock.rs | 9 +++--- frame/session/benchmarking/src/mock.rs | 5 ++- frame/session/src/historical/mod.rs | 3 +- frame/session/src/historical/offchain.rs | 3 +- frame/session/src/mock.rs | 3 +- frame/society/src/mock.rs | 9 +++--- frame/staking/src/mock.rs | 11 +++---- frame/sudo/src/mock.rs | 7 ++-- frame/support/src/lib.rs | 4 +-- frame/support/src/storage/generator/mod.rs | 15 ++++----- frame/support/test/tests/decl_storage.rs | 5 ++- frame/support/test/tests/instance.rs | 9 +++--- frame/system/benches/bench.rs | 3 +- frame/system/benchmarking/src/mock.rs | 5 ++- frame/system/src/lib.rs | 5 ++- frame/system/src/mock.rs | 3 +- frame/timestamp/src/lib.rs | 5 ++- frame/tips/src/tests.rs | 21 ++++++------ frame/transaction-payment/src/lib.rs | 7 ++-- frame/transaction-storage/src/mock.rs | 5 ++- frame/treasury/src/tests.rs | 25 +++++++-------- frame/uniques/src/mock.rs | 5 ++- frame/utility/src/tests.rs | 7 ++-- frame/vesting/src/mock.rs | 9 +++--- primitives/state-machine/src/testing.rs | 32 +++++++++++++++---- test-utils/runtime/src/system.rs | 5 +-- 74 files changed, 251 insertions(+), 310 deletions(-) diff --git a/bin/node-template/pallets/template/src/mock.rs b/bin/node-template/pallets/template/src/mock.rs index b772ceb2cde45..4532d3d09b497 100644 --- a/bin/node-template/pallets/template/src/mock.rs +++ b/bin/node-template/pallets/template/src/mock.rs @@ -59,6 +59,5 @@ impl pallet_template::Config for Test { // Build genesis storage according to the mock runtime. pub fn new_test_ext() -> sp_io::TestExternalities { - let state_version = Default::default(); - (system::GenesisConfig::default().build_storage::(state_version).unwrap(), state_version).into() + system::GenesisConfig::default().build_storage::().unwrap().into() } diff --git a/bin/node/cli/src/chain_spec.rs b/bin/node/cli/src/chain_spec.rs index 20752d27d9614..073d058240682 100644 --- a/bin/node/cli/src/chain_spec.rs +++ b/bin/node/cli/src/chain_spec.rs @@ -445,6 +445,7 @@ pub(crate) mod tests { None, None, Default::default(), + Default::default(), ) } @@ -460,6 +461,7 @@ pub(crate) mod tests { None, None, Default::default(), + Default::default(), ) } @@ -492,16 +494,16 @@ pub(crate) mod tests { #[test] fn test_create_development_chain_spec() { - development_config().build_storage(Default::default()).unwrap(); + development_config().build_storage().unwrap(); } #[test] fn test_create_local_testnet_chain_spec() { - local_testnet_config().build_storage(Default::default()).unwrap(); + local_testnet_config().build_storage().unwrap(); } #[test] fn test_staging_test_net_chain_spec() { - staging_testnet_config().build_storage(Default::default()).unwrap(); + staging_testnet_config().build_storage().unwrap(); } } diff --git a/bin/node/executor/benches/bench.rs b/bin/node/executor/benches/bench.rs index 2cb5c8ece4a74..18ab8d57059c7 100644 --- a/bin/node/executor/benches/bench.rs +++ b/bin/node/executor/benches/bench.rs @@ -66,11 +66,9 @@ fn sign(xt: CheckedExtrinsic) -> UncheckedExtrinsic { } fn new_test_ext(genesis_config: &GenesisConfig) -> TestExternalities { - let state_version = Default::default(); let mut test_ext = TestExternalities::new_with_code( compact_code_unwrap(), - genesis_config.build_storage(state_version).unwrap(), - state_version, + genesis_config.build_storage().unwrap(), ); test_ext .ext() diff --git a/bin/node/executor/tests/common.rs b/bin/node/executor/tests/common.rs index deadda1429484..b61126ba1b4a3 100644 --- a/bin/node/executor/tests/common.rs +++ b/bin/node/executor/tests/common.rs @@ -125,13 +125,11 @@ pub fn executor_call< } pub fn new_test_ext(code: &[u8], support_changes_trie: bool) -> TestExternalities { - let state_version = Default::default(); let mut ext = TestExternalities::new_with_code( code, node_testing::genesis::config(support_changes_trie, Some(code)) - .build_storage(state_version) + .build_storage() .unwrap(), - state_version ); ext.changes_trie_storage().insert(0, GENESIS_HASH.into(), Default::default()); ext diff --git a/bin/node/runtime/src/impls.rs b/bin/node/runtime/src/impls.rs index f052f0c810218..e315a45e698ce 100644 --- a/bin/node/runtime/src/impls.rs +++ b/bin/node/runtime/src/impls.rs @@ -96,10 +96,9 @@ mod multiplier_tests { where F: Fn() -> (), { - let state_version = Default::default(); - let mut t: sp_io::TestExternalities = (frame_system::GenesisConfig::default() - .build_storage::(state_version) - .unwrap(), state_version) + let mut t: sp_io::TestExternalities = frame_system::GenesisConfig::default() + .build_storage::() + .unwrap() .into(); t.execute_with(|| { System::set_block_consumed_resources(w, 0); diff --git a/client/chain-spec/src/chain_spec.rs b/client/chain-spec/src/chain_spec.rs index a5d9fe705d7fe..1a26f80e8f1ba 100644 --- a/client/chain-spec/src/chain_spec.rs +++ b/client/chain-spec/src/chain_spec.rs @@ -440,6 +440,10 @@ mod tests { struct Genesis(HashMap); impl BuildStorage for Genesis { + fn state_version(&self) -> StateVersion { + Default::default() + } + fn assimilate_storage(&self, storage: &mut Storage) -> Result<(), String> { storage.top.extend( self.0.iter().map(|(a, b)| (a.clone().into_bytes(), b.clone().into_bytes())), diff --git a/client/cli/src/commands/insert_key.rs b/client/cli/src/commands/insert_key.rs index 05055dc53c1e2..3addb7889bc84 100644 --- a/client/cli/src/commands/insert_key.rs +++ b/client/cli/src/commands/insert_key.rs @@ -143,6 +143,7 @@ mod tests { None, None, NoExtension::None, + Default::default(), ))) } } diff --git a/client/executor/src/integration_tests/mod.rs b/client/executor/src/integration_tests/mod.rs index 454a5cfcfe6b5..cade4450de2c2 100644 --- a/client/executor/src/integration_tests/mod.rs +++ b/client/executor/src/integration_tests/mod.rs @@ -190,15 +190,14 @@ fn storage_should_work(wasm_method: WasmExecutionMethod) { assert_eq!(output, b"all ok!".to_vec().encode()); } - let storage = sp_core::storage::Storage { + let expected = TestExternalities::new(sp_core::storage::Storage { top: map![ b"input".to_vec() => b"Hello worldHello worldHello worldHello world".to_vec(), b"foo".to_vec() => b"bar".to_vec(), b"baz".to_vec() => b"bar".to_vec() ], children_default: map![], - }; - let expected = TestExternalities::new(storage, Default::default()); + }); assert_eq!(ext, expected); } @@ -221,16 +220,14 @@ fn clear_prefix_should_work(wasm_method: WasmExecutionMethod) { assert_eq!(output, b"all ok!".to_vec().encode()); } - let storage = sp_core::storage::Storage { + let expected = TestExternalities::new(sp_core::storage::Storage { top: map![ b"aaa".to_vec() => b"1".to_vec(), b"aab".to_vec() => b"2".to_vec(), b"bbb".to_vec() => b"5".to_vec() ], children_default: map![], - }; - - let expected = TestExternalities::new(storage, Default::default()); + }); assert_eq!(expected, ext); } diff --git a/client/service/src/client/call_executor.rs b/client/service/src/client/call_executor.rs index 23cc08b7e1887..efaab6dd1ed53 100644 --- a/client/service/src/client/call_executor.rs +++ b/client/service/src/client/call_executor.rs @@ -396,7 +396,10 @@ mod tests { >( backend.clone(), executor.clone(), - &substrate_test_runtime_client::GenesisParameters::default().genesis_storage(), + &( + substrate_test_runtime_client::GenesisParameters::default().genesis_storage(), + sp_runtime::StateVersion::default(), + ), None, Box::new(TaskExecutor::new()), None, diff --git a/client/service/test/src/client/mod.rs b/client/service/test/src/client/mod.rs index 7f72bda7235b6..a13f2bad8d3b6 100644 --- a/client/service/test/src/client/mod.rs +++ b/client/service/test/src/client/mod.rs @@ -251,7 +251,6 @@ fn construct_genesis_should_work_with_native() { 1000, None, Default::default(), - state_version, ) .genesis_map(); let genesis_hash = insert_genesis_block(&mut storage); @@ -288,7 +287,6 @@ fn construct_genesis_should_work_with_wasm() { 1000, None, Default::default(), - state_version, ) .genesis_map(); let genesis_hash = insert_genesis_block(&mut storage); @@ -325,7 +323,6 @@ fn construct_genesis_with_bad_transaction_should_panic() { 68, None, Default::default(), - state_version, ) .genesis_map(); let genesis_hash = insert_genesis_block(&mut storage); @@ -2106,7 +2103,10 @@ fn cleans_up_closed_notification_sinks_on_block_import() { substrate_test_runtime_client::runtime::RuntimeApi, >( substrate_test_runtime_client::new_native_executor(), - &substrate_test_runtime_client::GenesisParameters::default().genesis_storage(), + &( + substrate_test_runtime_client::GenesisParameters::default().genesis_storage(), + sp_runtime::StateVersion::default(), + ), None, None, None, diff --git a/frame/assets/src/mock.rs b/frame/assets/src/mock.rs index d141b4c49db4d..1b2602792d844 100644 --- a/frame/assets/src/mock.rs +++ b/frame/assets/src/mock.rs @@ -144,10 +144,9 @@ pub(crate) fn hooks() -> Vec { } pub(crate) fn new_test_ext() -> sp_io::TestExternalities { - let state_version = Default::default(); - let t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); + let t = frame_system::GenesisConfig::default().build_storage::().unwrap(); - let mut ext = sp_io::TestExternalities::new(t, state_version); + let mut ext = sp_io::TestExternalities::new(t); ext.execute_with(|| System::set_block_number(1)); ext } diff --git a/frame/atomic-swap/src/tests.rs b/frame/atomic-swap/src/tests.rs index bf09b007fd588..a76d0f20ffa3b 100644 --- a/frame/atomic-swap/src/tests.rs +++ b/frame/atomic-swap/src/tests.rs @@ -83,11 +83,10 @@ const A: u64 = 1; const B: u64 = 2; pub fn new_test_ext() -> sp_io::TestExternalities { - let state_version = Default::default(); - let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); + let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); let genesis = pallet_balances::GenesisConfig:: { balances: vec![(A, 100), (B, 200)] }; - genesis.assimilate_storage(&mut t, state_version).unwrap(); - (t, state_version).into() + genesis.assimilate_storage(&mut t).unwrap(); + t.into() } #[test] diff --git a/frame/aura/src/mock.rs b/frame/aura/src/mock.rs index 38439c1f29ae8..0e258fb9a67dc 100644 --- a/frame/aura/src/mock.rs +++ b/frame/aura/src/mock.rs @@ -116,12 +116,11 @@ impl pallet_aura::Config for Test { } pub fn new_test_ext(authorities: Vec) -> sp_io::TestExternalities { - let state_version = Default::default(); - let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); + let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); pallet_aura::GenesisConfig:: { authorities: authorities.into_iter().map(|a| UintAuthorityId(a).to_public_key()).collect(), } - .assimilate_storage(&mut t, state_version) + .assimilate_storage(&mut t) .unwrap(); - (t, state_version).into() + t.into() } diff --git a/frame/authority-discovery/src/lib.rs b/frame/authority-discovery/src/lib.rs index 04d844d353627..4577a9dd17226 100644 --- a/frame/authority-discovery/src/lib.rs +++ b/frame/authority-discovery/src/lib.rs @@ -245,7 +245,6 @@ mod tests { #[test] fn authorities_returns_current_and_next_authority_set() { - let state_version = Default::default(); // The whole authority discovery pallet ignores account ids, but we still need them for // `pallet_session::OneSessionHandler::on_new_session`, thus its safe to use the same value // everywhere. @@ -282,17 +281,16 @@ mod tests { .collect::>(); // Build genesis. - let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); + let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); GenesisBuild::::assimilate_storage( &pallet_authority_discovery::GenesisConfig { keys: vec![] }, &mut t, - state_version, ) .unwrap(); // Create externalities. - let mut externalities = TestExternalities::new(t, state_version); + let mut externalities = TestExternalities::new(t); externalities.execute_with(|| { use frame_support::traits::OneSessionHandler; diff --git a/frame/authorship/src/lib.rs b/frame/authorship/src/lib.rs index d5151d36c53ca..325f80c74aa1e 100644 --- a/frame/authorship/src/lib.rs +++ b/frame/authorship/src/lib.rs @@ -533,9 +533,8 @@ mod tests { } fn new_test_ext() -> sp_io::TestExternalities { - let state_version = sp_runtime::StateVersion::default(); - let t = frame_system::GenesisConfig::default().build_storage::(state_version.clone()).unwrap(); - (t, state_version).into() + let t = frame_system::GenesisConfig::default().build_storage::().unwrap(); + t.into() } #[test] diff --git a/frame/babe/src/mock.rs b/frame/babe/src/mock.rs index 99d94beefa69b..f872fb23b12aa 100644 --- a/frame/babe/src/mock.rs +++ b/frame/babe/src/mock.rs @@ -380,13 +380,12 @@ pub fn new_test_ext_with_pairs( } pub fn new_test_ext_raw_authorities(authorities: Vec) -> sp_io::TestExternalities { - let state_version = Default::default(); - let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); + let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); let balances: Vec<_> = (0..authorities.len()).map(|i| (i as u64, 10_000_000)).collect(); pallet_balances::GenesisConfig:: { balances } - .assimilate_storage(&mut t, state_version) + .assimilate_storage(&mut t) .unwrap(); // stashes are the index. @@ -401,7 +400,7 @@ pub fn new_test_ext_raw_authorities(authorities: Vec) -> sp_io::Tes // NOTE: this will initialize the babe authorities // through OneSessionHandler::on_genesis_session pallet_session::GenesisConfig:: { keys: session_keys } - .assimilate_storage(&mut t, state_version) + .assimilate_storage(&mut t) .unwrap(); // controllers are the index + 1000 @@ -420,9 +419,9 @@ pub fn new_test_ext_raw_authorities(authorities: Vec) -> sp_io::Tes ..Default::default() }; - staking_config.assimilate_storage(&mut t, state_version).unwrap(); + staking_config.assimilate_storage(&mut t).unwrap(); - (t, state_version).into() + t.into() } /// Creates an equivocation at the current block, by generating two headers. diff --git a/frame/balances/src/tests.rs b/frame/balances/src/tests.rs index 46d87df97a670..fd57371b3a16b 100644 --- a/frame/balances/src/tests.rs +++ b/frame/balances/src/tests.rs @@ -629,22 +629,20 @@ macro_rules! decl_tests { #[test] #[should_panic = "the balance of any account should always be at least the existential deposit."] fn cannot_set_genesis_value_below_ed() { - let state_version = Default::default(); ($existential_deposit).with(|v| *v.borrow_mut() = 11); - let mut t = frame_system::GenesisConfig::default().build_storage::<$test>(state_version).unwrap(); + let mut t = frame_system::GenesisConfig::default().build_storage::<$test>().unwrap(); let _ = pallet_balances::GenesisConfig::<$test> { balances: vec![(1, 10)], - }.assimilate_storage(&mut t, state_version).unwrap(); + }.assimilate_storage(&mut t).unwrap(); } #[test] #[should_panic = "duplicate balances in genesis."] fn cannot_set_genesis_value_twice() { - let state_version = Default::default(); - let mut t = frame_system::GenesisConfig::default().build_storage::<$test>(state_version).unwrap(); + let mut t = frame_system::GenesisConfig::default().build_storage::<$test>().unwrap(); let _ = pallet_balances::GenesisConfig::<$test> { balances: vec![(1, 10), (2, 20), (1, 15)], - }.assimilate_storage(&mut t, state_version).unwrap(); + }.assimilate_storage(&mut t).unwrap(); } #[test] diff --git a/frame/balances/src/tests_composite.rs b/frame/balances/src/tests_composite.rs index 955f90d97766c..f6faebed39316 100644 --- a/frame/balances/src/tests_composite.rs +++ b/frame/balances/src/tests_composite.rs @@ -122,9 +122,8 @@ impl ExtBuilder { EXISTENTIAL_DEPOSIT.with(|v| *v.borrow_mut() = self.existential_deposit); } pub fn build(self) -> sp_io::TestExternalities { - let state_version = Default::default(); self.set_associated_consts(); - let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); + let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); pallet_balances::GenesisConfig:: { balances: if self.monied { vec![ @@ -138,10 +137,10 @@ impl ExtBuilder { vec![] }, } - .assimilate_storage(&mut t, state_version) + .assimilate_storage(&mut t) .unwrap(); - let mut ext = sp_io::TestExternalities::new(t, state_version); + let mut ext = sp_io::TestExternalities::new(t); ext.execute_with(|| System::set_block_number(1)); ext } diff --git a/frame/balances/src/tests_local.rs b/frame/balances/src/tests_local.rs index a399a3f991055..d8c07aa9c42e5 100644 --- a/frame/balances/src/tests_local.rs +++ b/frame/balances/src/tests_local.rs @@ -127,9 +127,8 @@ impl ExtBuilder { EXISTENTIAL_DEPOSIT.with(|v| *v.borrow_mut() = self.existential_deposit); } pub fn build(self) -> sp_io::TestExternalities { - let state_version = Default::default(); self.set_associated_consts(); - let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); + let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); pallet_balances::GenesisConfig:: { balances: if self.monied { vec![ @@ -143,10 +142,10 @@ impl ExtBuilder { vec![] }, } - .assimilate_storage(&mut t, state_version) + .assimilate_storage(&mut t) .unwrap(); - let mut ext = sp_io::TestExternalities::new(t, state_version); + let mut ext = sp_io::TestExternalities::new(t); ext.execute_with(|| System::set_block_number(1)); ext } diff --git a/frame/balances/src/tests_reentrancy.rs b/frame/balances/src/tests_reentrancy.rs index 42ed624875a26..9c7ba3e1ec824 100644 --- a/frame/balances/src/tests_reentrancy.rs +++ b/frame/balances/src/tests_reentrancy.rs @@ -130,13 +130,12 @@ impl ExtBuilder { } pub fn build(self) -> sp_io::TestExternalities { - let state_version = Default::default(); self.set_associated_consts(); - let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); + let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); pallet_balances::GenesisConfig:: { balances: vec![] } - .assimilate_storage(&mut t, state_version) + .assimilate_storage(&mut t) .unwrap(); - let mut ext = sp_io::TestExternalities::new(t, state_version); + let mut ext = sp_io::TestExternalities::new(t); ext.execute_with(|| System::set_block_number(1)); ext } diff --git a/frame/benchmarking/src/tests.rs b/frame/benchmarking/src/tests.rs index a7188c89df12f..9cb5043a0dd7d 100644 --- a/frame/benchmarking/src/tests.rs +++ b/frame/benchmarking/src/tests.rs @@ -128,9 +128,7 @@ impl pallet_test::OtherConfig for Test { } fn new_test_ext() -> sp_io::TestExternalities { - let state_version = Default::default(); - let storage = GenesisConfig::default().build_storage(state_version).unwrap(); - (storage, state_version).into() + GenesisConfig::default().build_storage().unwrap().into() } mod benchmarks { diff --git a/frame/benchmarking/src/tests_instance.rs b/frame/benchmarking/src/tests_instance.rs index 02243da281e4e..caccebd39c70b 100644 --- a/frame/benchmarking/src/tests_instance.rs +++ b/frame/benchmarking/src/tests_instance.rs @@ -128,9 +128,7 @@ impl pallet_test::OtherConfig for Test { } fn new_test_ext() -> sp_io::TestExternalities { - let state_version = Default::default(); - let storage = GenesisConfig::default().build_storage(state_version).unwrap(); - (storage, state_version).into() + GenesisConfig::default().build_storage().unwrap().into() } mod benchmarks { diff --git a/frame/bounties/src/tests.rs b/frame/bounties/src/tests.rs index 1d5b1ee5faa64..ff058a3601e07 100644 --- a/frame/bounties/src/tests.rs +++ b/frame/bounties/src/tests.rs @@ -149,16 +149,15 @@ impl Config for Test { type TreasuryError = pallet_treasury::Error; pub fn new_test_ext() -> sp_io::TestExternalities { - let state_version = Default::default(); - let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); + let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); pallet_balances::GenesisConfig:: { // Total issuance will be 200 with treasury account initialized at ED. balances: vec![(0, 100), (1, 98), (2, 1)], } - .assimilate_storage(&mut t, state_version) + .assimilate_storage(&mut t) .unwrap(); - GenesisBuild::::assimilate_storage(&pallet_treasury::GenesisConfig, &mut t, state_version).unwrap(); - (t, state_version).into() + GenesisBuild::::assimilate_storage(&pallet_treasury::GenesisConfig, &mut t).unwrap(); + t.into() } fn last_event() -> RawEvent { @@ -357,13 +356,12 @@ fn treasury_account_doesnt_get_deleted() { // This is useful for chain that will just update runtime. #[test] fn inexistent_account_works() { - let state_version = Default::default(); - let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); + let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); pallet_balances::GenesisConfig:: { balances: vec![(0, 100), (1, 99), (2, 1)] } - .assimilate_storage(&mut t, state_version) + .assimilate_storage(&mut t) .unwrap(); // Treasury genesis config is not build thus treasury account does not exist - let mut t: sp_io::TestExternalities = (t, state_version).into(); + let mut t: sp_io::TestExternalities = t.into(); t.execute_with(|| { assert_eq!(Balances::free_balance(Treasury::account_id()), 0); // Account does not exist @@ -938,17 +936,16 @@ fn extend_expiry() { #[test] fn genesis_funding_works() { - let state_version = Default::default(); - let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); + let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); let initial_funding = 100; pallet_balances::GenesisConfig:: { // Total issuance will be 200 with treasury account initialized with 100. balances: vec![(0, 100), (Treasury::account_id(), initial_funding)], } - .assimilate_storage(&mut t, state_version) + .assimilate_storage(&mut t) .unwrap(); - GenesisBuild::::assimilate_storage(&pallet_treasury::GenesisConfig, &mut t, state_version).unwrap(); - let mut t: sp_io::TestExternalities = (t, state_version).into(); + GenesisBuild::::assimilate_storage(&pallet_treasury::GenesisConfig, &mut t).unwrap(); + let mut t: sp_io::TestExternalities = t.into(); t.execute_with(|| { assert_eq!(Balances::free_balance(Treasury::account_id()), initial_funding); diff --git a/frame/collective/src/lib.rs b/frame/collective/src/lib.rs index 89d9542e8db84..39da8e2c45fb7 100644 --- a/frame/collective/src/lib.rs +++ b/frame/collective/src/lib.rs @@ -1083,8 +1083,7 @@ mod tests { ); pub fn new_test_ext() -> sp_io::TestExternalities { - let state_version = sp_runtime::StateVersion::default(); - let mut ext: sp_io::TestExternalities = (GenesisConfig { + let mut ext: sp_io::TestExternalities = GenesisConfig { collective: collective::GenesisConfig { members: vec![1, 2, 3], phantom: Default::default(), @@ -1095,8 +1094,9 @@ mod tests { }, default_collective: Default::default(), } - .build_storage(state_version) - .unwrap(), state_version).into(); + .build_storage() + .unwrap() + .into(); ext.execute_with(|| System::set_block_number(1)); ext } @@ -2158,12 +2158,11 @@ mod tests { #[test] #[should_panic(expected = "Members cannot contain duplicate accounts.")] fn genesis_build_panics_with_duplicate_members() { - let state_version = sp_runtime::StateVersion::default(); collective::GenesisConfig:: { members: vec![1, 2, 3, 1], phantom: Default::default(), } - .build_storage(state_version) + .build_storage() .unwrap(); } } diff --git a/frame/contracts/src/tests.rs b/frame/contracts/src/tests.rs index 31742dc9d604e..fc5a2cf5221e0 100644 --- a/frame/contracts/src/tests.rs +++ b/frame/contracts/src/tests.rs @@ -336,13 +336,12 @@ impl ExtBuilder { EXISTENTIAL_DEPOSIT.with(|v| *v.borrow_mut() = self.existential_deposit); } pub fn build(self) -> sp_io::TestExternalities { - let state_version = Default::default(); self.set_associated_consts(); - let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); + let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); pallet_balances::GenesisConfig:: { balances: vec![] } - .assimilate_storage(&mut t, state_version) + .assimilate_storage(&mut t) .unwrap(); - let mut ext = sp_io::TestExternalities::new(t, state_version); + let mut ext = sp_io::TestExternalities::new(t); ext.execute_with(|| System::set_block_number(1)); ext } diff --git a/frame/democracy/src/tests.rs b/frame/democracy/src/tests.rs index 0f349a2ea5a8d..46d3cade36eb1 100644 --- a/frame/democracy/src/tests.rs +++ b/frame/democracy/src/tests.rs @@ -194,17 +194,16 @@ impl Config for Test { } pub fn new_test_ext() -> sp_io::TestExternalities { - let state_version = Default::default(); - let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); + let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); pallet_balances::GenesisConfig:: { balances: vec![(1, 10), (2, 20), (3, 30), (4, 40), (5, 50), (6, 60)], } - .assimilate_storage(&mut t, state_version) + .assimilate_storage(&mut t) .unwrap(); pallet_democracy::GenesisConfig::::default() - .assimilate_storage(&mut t, state_version) + .assimilate_storage(&mut t) .unwrap(); - let mut ext = sp_io::TestExternalities::new(t, state_version); + let mut ext = sp_io::TestExternalities::new(t); ext.execute_with(|| System::set_block_number(1)); ext } diff --git a/frame/election-provider-multi-phase/src/mock.rs b/frame/election-provider-multi-phase/src/mock.rs index e2f022115f03a..f760676abf761 100644 --- a/frame/election-provider-multi-phase/src/mock.rs +++ b/frame/election-provider-multi-phase/src/mock.rs @@ -511,10 +511,9 @@ impl ExtBuilder { self } pub fn build(self) -> sp_io::TestExternalities { - let state_version = Default::default(); sp_tracing::try_init_simple(); let mut storage = - frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); + frame_system::GenesisConfig::default().build_storage::().unwrap(); let _ = pallet_balances::GenesisConfig:: { balances: vec![ @@ -524,9 +523,9 @@ impl ExtBuilder { (9999, 100), ], } - .assimilate_storage(&mut storage, state_version); + .assimilate_storage(&mut storage); - sp_io::TestExternalities::from((storage, state_version)) + sp_io::TestExternalities::from(storage) } pub fn build_offchainify( diff --git a/frame/elections-phragmen/src/lib.rs b/frame/elections-phragmen/src/lib.rs index 29f4827838655..331d34180e98c 100644 --- a/frame/elections-phragmen/src/lib.rs +++ b/frame/elections-phragmen/src/lib.rs @@ -1303,8 +1303,7 @@ mod tests { *m.borrow_mut() = self.genesis_members.iter().map(|(m, _)| m.clone()).collect::>() }); - let state_version = Default::default(); - let mut ext: sp_io::TestExternalities = (GenesisConfig { + let mut ext: sp_io::TestExternalities = GenesisConfig { balances: pallet_balances::GenesisConfig:: { balances: vec![ (1, 10 * self.balance_factor), @@ -1319,8 +1318,8 @@ mod tests { members: self.genesis_members, }, } - .build_storage(state_version) - .unwrap(), state_version) + .build_storage() + .unwrap() .into(); ext.execute_with(pre_conditions); ext.execute_with(test); diff --git a/frame/elections/src/mock.rs b/frame/elections/src/mock.rs index 09599e411be57..91318e1e07bcc 100644 --- a/frame/elections/src/mock.rs +++ b/frame/elections/src/mock.rs @@ -193,12 +193,11 @@ impl ExtBuilder { self } pub fn build(self) -> sp_io::TestExternalities { - let state_version = Default::default(); VOTING_BOND.with(|v| *v.borrow_mut() = self.voting_bond); VOTING_FEE.with(|v| *v.borrow_mut() = self.voting_fee); PRESENT_SLASH_PER_VOTER.with(|v| *v.borrow_mut() = self.bad_presentation_punishment); DECAY_RATIO.with(|v| *v.borrow_mut() = self.decay_ratio); - let mut ext: sp_io::TestExternalities = (GenesisConfig { + let mut ext: sp_io::TestExternalities = GenesisConfig { balances: pallet_balances::GenesisConfig:: { balances: vec![ (1, 10 * self.balance_factor), @@ -216,8 +215,8 @@ impl ExtBuilder { term_duration: 5, }, } - .build_storage(state_version) - .unwrap(), state_version) + .build_storage() + .unwrap() .into(); ext.execute_with(|| System::set_block_number(1)); ext diff --git a/frame/example/src/tests.rs b/frame/example/src/tests.rs index aa6988374ea8e..645b5c9bc13ab 100644 --- a/frame/example/src/tests.rs +++ b/frame/example/src/tests.rs @@ -107,7 +107,6 @@ impl Config for Test { // This function basically just builds a genesis storage key/value store according to // our desired mockup. pub fn new_test_ext() -> sp_io::TestExternalities { - let state_version = Default::default(); let t = GenesisConfig { // We use default for brevity, but you can configure as desired if needed. system: Default::default(), @@ -119,9 +118,9 @@ pub fn new_test_ext() -> sp_io::TestExternalities { foo: 24, }, } - .build_storage(state_version) + .build_storage() .unwrap(); - (t, state_version).into() + t.into() } #[test] diff --git a/frame/executive/src/lib.rs b/frame/executive/src/lib.rs index d19dac3a20da5..5f1ae23c2f531 100644 --- a/frame/executive/src/lib.rs +++ b/frame/executive/src/lib.rs @@ -811,10 +811,9 @@ mod tests { #[test] fn balance_transfer_dispatch_works() { - let state_version = Default::default(); - let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); + let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); pallet_balances::GenesisConfig:: { balances: vec![(1, 211)] } - .assimilate_storage(&mut t, state_version) + .assimilate_storage(&mut t) .unwrap(); let xt = TestXt::new(Call::Balances(BalancesCall::transfer(2, 69)), sign_extra(1, 0, 0)); let weight = xt.get_dispatch_info().weight + @@ -823,7 +822,7 @@ mod tests { .base_extrinsic; let fee: Balance = ::WeightToFee::calc(&weight); - let mut t = sp_io::TestExternalities::new(t, state_version); + let mut t = sp_io::TestExternalities::new(t); t.execute_with(|| { Executive::initialize_block(&Header::new( 1, @@ -840,12 +839,11 @@ mod tests { } fn new_test_ext(balance_factor: Balance) -> sp_io::TestExternalities { - let state_version = Default::default(); - let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); + let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); pallet_balances::GenesisConfig:: { balances: vec![(1, 111 * balance_factor)] } - .assimilate_storage(&mut t, state_version) + .assimilate_storage(&mut t) .unwrap(); - (t, state_version).into() + t.into() } #[test] diff --git a/frame/gilt/src/mock.rs b/frame/gilt/src/mock.rs index f8bf3b972cafb..ac3f4df1b71dd 100644 --- a/frame/gilt/src/mock.rs +++ b/frame/gilt/src/mock.rs @@ -127,15 +127,14 @@ impl pallet_gilt::Config for Test { // This function basically just builds a genesis storage key/value store according to // our desired mockup. pub fn new_test_ext() -> sp_io::TestExternalities { - let state_version = Default::default(); - let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); + let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); pallet_balances::GenesisConfig:: { balances: vec![(1, 100), (2, 100), (3, 100), (4, 100)], } - .assimilate_storage(&mut t, state_version) + .assimilate_storage(&mut t) .unwrap(); - GenesisBuild::::assimilate_storage(&crate::GenesisConfig, &mut t, state_version).unwrap(); - (t, state_version).into() + GenesisBuild::::assimilate_storage(&crate::GenesisConfig, &mut t).unwrap(); + t.into() } pub fn run_to_block(n: u64) { diff --git a/frame/grandpa/src/mock.rs b/frame/grandpa/src/mock.rs index ee31eaffeb0bd..b9c4858e353e8 100644 --- a/frame/grandpa/src/mock.rs +++ b/frame/grandpa/src/mock.rs @@ -275,13 +275,12 @@ pub fn new_test_ext(vec: Vec<(u64, u64)>) -> sp_io::TestExternalities { } pub fn new_test_ext_raw_authorities(authorities: AuthorityList) -> sp_io::TestExternalities { - let state_version = Default::default(); - let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); + let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); let balances: Vec<_> = (0..authorities.len()).map(|i| (i as u64, 10_000_000)).collect(); pallet_balances::GenesisConfig:: { balances } - .assimilate_storage(&mut t, state_version) + .assimilate_storage(&mut t) .unwrap(); // stashes are the index. @@ -300,7 +299,7 @@ pub fn new_test_ext_raw_authorities(authorities: AuthorityList) -> sp_io::TestEx // NOTE: this will initialize the grandpa authorities // through OneSessionHandler::on_genesis_session pallet_session::GenesisConfig:: { keys: session_keys } - .assimilate_storage(&mut t, state_version) + .assimilate_storage(&mut t) .unwrap(); // controllers are the index + 1000 @@ -319,9 +318,9 @@ pub fn new_test_ext_raw_authorities(authorities: AuthorityList) -> sp_io::TestEx ..Default::default() }; - staking_config.assimilate_storage(&mut t, state_version).unwrap(); + staking_config.assimilate_storage(&mut t).unwrap(); - (t, state_version).into() + t.into() } pub fn start_session(session_index: SessionIndex) { diff --git a/frame/identity/src/tests.rs b/frame/identity/src/tests.rs index 4a7e85c6c1abb..c842b0e2f64be 100644 --- a/frame/identity/src/tests.rs +++ b/frame/identity/src/tests.rs @@ -118,14 +118,13 @@ impl pallet_identity::Config for Test { } pub fn new_test_ext() -> sp_io::TestExternalities { - let state_version = Default::default(); - let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); + let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); pallet_balances::GenesisConfig:: { balances: vec![(1, 10), (2, 10), (3, 10), (10, 100), (20, 100), (30, 100)], } - .assimilate_storage(&mut t, state_version) + .assimilate_storage(&mut t) .unwrap(); - (t, state_version).into() + t.into() } fn ten() -> IdentityInfo { diff --git a/frame/im-online/src/mock.rs b/frame/im-online/src/mock.rs index acade9ebd7588..e4031b04271b9 100644 --- a/frame/im-online/src/mock.rs +++ b/frame/im-online/src/mock.rs @@ -105,9 +105,8 @@ impl ReportOffence for OffenceHandler { } pub fn new_test_ext() -> sp_io::TestExternalities { - let state_version = Default::default(); - let t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); - (t, state_version).into() + let t = frame_system::GenesisConfig::default().build_storage::().unwrap(); + t.into() } parameter_types! { diff --git a/frame/indices/src/mock.rs b/frame/indices/src/mock.rs index 76d81d511edad..f4c87016141b5 100644 --- a/frame/indices/src/mock.rs +++ b/frame/indices/src/mock.rs @@ -100,12 +100,11 @@ impl Config for Test { } pub fn new_test_ext() -> sp_io::TestExternalities { - let state_version = Default::default(); - let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); + let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); pallet_balances::GenesisConfig:: { balances: vec![(1, 10), (2, 20), (3, 30), (4, 40), (5, 50), (6, 60)], } - .assimilate_storage(&mut t, state_version) + .assimilate_storage(&mut t) .unwrap(); - (t, state_version).into() + t.into() } diff --git a/frame/lottery/src/mock.rs b/frame/lottery/src/mock.rs index c380fe390877c..d1f090aa26dcb 100644 --- a/frame/lottery/src/mock.rs +++ b/frame/lottery/src/mock.rs @@ -120,14 +120,13 @@ pub type SystemCall = frame_system::Call; pub type BalancesCall = pallet_balances::Call; pub fn new_test_ext() -> sp_io::TestExternalities { - let state_version = Default::default(); - let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); + let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); pallet_balances::GenesisConfig:: { balances: vec![(1, 100), (2, 100), (3, 100), (4, 100), (5, 100)], } - .assimilate_storage(&mut t, state_version) + .assimilate_storage(&mut t) .unwrap(); - (t, state_version).into() + t.into() } /// Run until a particular block. diff --git a/frame/membership/src/lib.rs b/frame/membership/src/lib.rs index 250a1e08be341..a9bc59a361f06 100644 --- a/frame/membership/src/lib.rs +++ b/frame/membership/src/lib.rs @@ -573,16 +573,15 @@ mod tests { } pub(crate) fn new_test_ext() -> sp_io::TestExternalities { - let state_version = Default::default(); - let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); + let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); // We use default for brevity, but you can configure as desired if needed. pallet_membership::GenesisConfig:: { members: vec![10, 20, 30], ..Default::default() } - .assimilate_storage(&mut t, state_version) + .assimilate_storage(&mut t) .unwrap(); - (t, state_version).into() + t.into() } #[cfg(feature = "runtime-benchmarks")] @@ -739,12 +738,11 @@ mod tests { #[test] #[should_panic(expected = "Members cannot contain duplicate accounts.")] fn genesis_build_panics_with_duplicate_members() { - let state_version = Default::default(); pallet_membership::GenesisConfig:: { members: vec![1, 2, 3, 1], phantom: Default::default(), } - .build_storage(state_version) + .build_storage() .unwrap(); } } diff --git a/frame/merkle-mountain-range/src/tests.rs b/frame/merkle-mountain-range/src/tests.rs index 9c23a00f70daa..50512e9286951 100644 --- a/frame/merkle-mountain-range/src/tests.rs +++ b/frame/merkle-mountain-range/src/tests.rs @@ -25,8 +25,7 @@ use sp_core::{ }; pub(crate) fn new_test_ext() -> sp_io::TestExternalities { - let state_version = Default::default(); - (frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(), state_version).into() + frame_system::GenesisConfig::default().build_storage::().unwrap().into() } fn register_offchain_ext(ext: &mut sp_io::TestExternalities) { diff --git a/frame/multisig/src/tests.rs b/frame/multisig/src/tests.rs index a9972f1f9ce38..2809a106d66ea 100644 --- a/frame/multisig/src/tests.rs +++ b/frame/multisig/src/tests.rs @@ -117,14 +117,13 @@ impl Config for Test { use pallet_balances::{Call as BalancesCall, Error as BalancesError}; pub fn new_test_ext() -> sp_io::TestExternalities { - let state_version = Default::default(); - let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); + let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); pallet_balances::GenesisConfig:: { balances: vec![(1, 10), (2, 10), (3, 10), (4, 10), (5, 2)], } - .assimilate_storage(&mut t, state_version) + .assimilate_storage(&mut t) .unwrap(); - let mut ext = sp_io::TestExternalities::new(t, state_version); + let mut ext = sp_io::TestExternalities::new(t); ext.execute_with(|| System::set_block_number(1)); ext } diff --git a/frame/nicks/src/lib.rs b/frame/nicks/src/lib.rs index 22b063b96ee2e..a5c22b619a5eb 100644 --- a/frame/nicks/src/lib.rs +++ b/frame/nicks/src/lib.rs @@ -339,12 +339,11 @@ mod tests { } fn new_test_ext() -> sp_io::TestExternalities { - let state_version = Default::default(); - let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); + let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); pallet_balances::GenesisConfig:: { balances: vec![(1, 10), (2, 10)] } - .assimilate_storage(&mut t, state_version) + .assimilate_storage(&mut t) .unwrap(); - (t, state_version).into() + t.into() } #[test] diff --git a/frame/node-authorization/src/mock.rs b/frame/node-authorization/src/mock.rs index a5b243b542ced..6c79f601c197d 100644 --- a/frame/node-authorization/src/mock.rs +++ b/frame/node-authorization/src/mock.rs @@ -99,12 +99,11 @@ pub fn test_node(id: u8) -> PeerId { } pub fn new_test_ext() -> sp_io::TestExternalities { - let state_version = Default::default(); - let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); + let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); pallet_node_authorization::GenesisConfig:: { nodes: vec![(test_node(10), 10), (test_node(20), 20), (test_node(30), 30)], } - .assimilate_storage(&mut t, state_version) + .assimilate_storage(&mut t) .unwrap(); - (t, state_version).into() + t.into() } diff --git a/frame/offences/benchmarking/src/mock.rs b/frame/offences/benchmarking/src/mock.rs index 34be5a9361cf3..3416dafa3547e 100644 --- a/frame/offences/benchmarking/src/mock.rs +++ b/frame/offences/benchmarking/src/mock.rs @@ -226,7 +226,6 @@ frame_support::construct_runtime!( ); pub fn new_test_ext() -> sp_io::TestExternalities { - let state_version = Default::default(); - let t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); - sp_io::TestExternalities::new(t, state_version) + let t = frame_system::GenesisConfig::default().build_storage::().unwrap(); + sp_io::TestExternalities::new(t) } diff --git a/frame/offences/src/mock.rs b/frame/offences/src/mock.rs index fb103b2c3c26f..5e4c94944b6fd 100644 --- a/frame/offences/src/mock.rs +++ b/frame/offences/src/mock.rs @@ -120,9 +120,8 @@ impl Config for Runtime { } pub fn new_test_ext() -> sp_io::TestExternalities { - let state_version = Default::default(); - let t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); - let mut ext = sp_io::TestExternalities::new(t, state_version); + let t = frame_system::GenesisConfig::default().build_storage::().unwrap(); + let mut ext = sp_io::TestExternalities::new(t); ext.execute_with(|| System::set_block_number(1)); ext } diff --git a/frame/proxy/src/tests.rs b/frame/proxy/src/tests.rs index d41bb512b14d6..f3fe1d674a87d 100644 --- a/frame/proxy/src/tests.rs +++ b/frame/proxy/src/tests.rs @@ -165,14 +165,13 @@ use pallet_balances::{Call as BalancesCall, Error as BalancesError, Event as Bal use pallet_utility::{Call as UtilityCall, Event as UtilityEvent}; pub fn new_test_ext() -> sp_io::TestExternalities { - let state_version = Default::default(); - let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); + let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); pallet_balances::GenesisConfig:: { balances: vec![(1, 10), (2, 10), (3, 10), (4, 10), (5, 2)], } - .assimilate_storage(&mut t, state_version) + .assimilate_storage(&mut t) .unwrap(); - let mut ext = sp_io::TestExternalities::new(t, state_version); + let mut ext = sp_io::TestExternalities::new(t); ext.execute_with(|| System::set_block_number(1)); ext } diff --git a/frame/randomness-collective-flip/src/lib.rs b/frame/randomness-collective-flip/src/lib.rs index 6f4c829c2488e..1b1d5cb5cd823 100644 --- a/frame/randomness-collective-flip/src/lib.rs +++ b/frame/randomness-collective-flip/src/lib.rs @@ -224,9 +224,8 @@ mod tests { impl pallet_randomness_collective_flip::Config for Test {} fn new_test_ext() -> sp_io::TestExternalities { - let state_version = Default::default(); - let t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); - (t, state_version).into() + let t = frame_system::GenesisConfig::default().build_storage::().unwrap(); + t.into() } #[test] diff --git a/frame/recovery/src/mock.rs b/frame/recovery/src/mock.rs index 22be999741d3d..f6d4a6b159431 100644 --- a/frame/recovery/src/mock.rs +++ b/frame/recovery/src/mock.rs @@ -114,14 +114,13 @@ pub type BalancesCall = pallet_balances::Call; pub type RecoveryCall = super::Call; pub fn new_test_ext() -> sp_io::TestExternalities { - let state_version = Default::default(); - let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); + let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); pallet_balances::GenesisConfig:: { balances: vec![(1, 100), (2, 100), (3, 100), (4, 100), (5, 100)], } - .assimilate_storage(&mut t, state_version) + .assimilate_storage(&mut t) .unwrap(); - (t, state_version).into() + t.into() } /// Run until a particular block. diff --git a/frame/scheduler/src/lib.rs b/frame/scheduler/src/lib.rs index 4bcf94d45656b..ceb163a432e7b 100644 --- a/frame/scheduler/src/lib.rs +++ b/frame/scheduler/src/lib.rs @@ -985,9 +985,8 @@ mod tests { pub type LoggerCall = logger::Call; pub fn new_test_ext() -> sp_io::TestExternalities { - let state_version = Default::default(); - let t = system::GenesisConfig::default().build_storage::(state_version).unwrap(); - (t, state_version).into() + let t = system::GenesisConfig::default().build_storage::().unwrap(); + t.into() } fn run_to_block(n: u64) { diff --git a/frame/scored-pool/src/mock.rs b/frame/scored-pool/src/mock.rs index c583a6dc54792..5c5425ae2bdd8 100644 --- a/frame/scored-pool/src/mock.rs +++ b/frame/scored-pool/src/mock.rs @@ -135,8 +135,7 @@ impl Config for Test { } pub fn new_test_ext() -> sp_io::TestExternalities { - let state_version = Default::default(); - let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); + let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); pallet_balances::GenesisConfig:: { balances: vec![ (5, 500_000), @@ -148,16 +147,16 @@ pub fn new_test_ext() -> sp_io::TestExternalities { (99, 1), ], } - .assimilate_storage(&mut t, state_version) + .assimilate_storage(&mut t) .unwrap(); pallet_scored_pool::GenesisConfig:: { pool: vec![(5, None), (10, Some(1)), (20, Some(2)), (31, Some(2)), (40, Some(3))], member_count: 2, ..Default::default() } - .assimilate_storage(&mut t, state_version) + .assimilate_storage(&mut t) .unwrap(); - (t, state_version).into() + t.into() } /// Fetch an entity from the pool, if existent. diff --git a/frame/session/benchmarking/src/mock.rs b/frame/session/benchmarking/src/mock.rs index 1482ef59e8a25..672862f5ed999 100644 --- a/frame/session/benchmarking/src/mock.rs +++ b/frame/session/benchmarking/src/mock.rs @@ -191,7 +191,6 @@ impl pallet_staking::Config for Test { impl crate::Config for Test {} pub fn new_test_ext() -> sp_io::TestExternalities { - let state_version = Default::default(); - let t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); - sp_io::TestExternalities::new(t, state_version) + let t = frame_system::GenesisConfig::default().build_storage::().unwrap(); + sp_io::TestExternalities::new(t) } diff --git a/frame/session/src/historical/mod.rs b/frame/session/src/historical/mod.rs index 2600e3052ef0b..c9b13e3c7f262 100644 --- a/frame/session/src/historical/mod.rs +++ b/frame/session/src/historical/mod.rs @@ -375,7 +375,6 @@ pub(crate) mod tests { type Historical = Module; pub(crate) fn new_test_ext() -> sp_io::TestExternalities { - let state_version = Default::default(); let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); let keys: Vec<_> = NEXT_VALIDATORS.with(|l| { l.borrow().iter().cloned().map(|i| (i, i, UintAuthorityId(i).into())).collect() @@ -386,7 +385,7 @@ pub(crate) mod tests { } }); crate::GenesisConfig:: { keys }.assimilate_storage(&mut t).unwrap(); - sp_io::TestExternalities::new(t, state_version) + sp_io::TestExternalities::new(t) } #[test] diff --git a/frame/session/src/historical/offchain.rs b/frame/session/src/historical/offchain.rs index ed4dd2fbb2659..8583c2bb439be 100644 --- a/frame/session/src/historical/offchain.rs +++ b/frame/session/src/historical/offchain.rs @@ -162,7 +162,6 @@ mod tests { type Historical = Module; pub fn new_test_ext() -> sp_io::TestExternalities { - let state_version = Default::default(); let mut t = frame_system::GenesisConfig::default() .build_storage::() .expect("Failed to create test externalities."); @@ -178,7 +177,7 @@ mod tests { crate::GenesisConfig:: { keys }.assimilate_storage(&mut t).unwrap(); - let mut ext = sp_io::TestExternalities::new(t, state_version); + let mut ext = sp_io::TestExternalities::new(t); let (offchain, offchain_state) = TestOffchainExt::with_offchain_db(ext.offchain_db()); diff --git a/frame/session/src/mock.rs b/frame/session/src/mock.rs index 2879a8e006192..449acaff5305d 100644 --- a/frame/session/src/mock.rs +++ b/frame/session/src/mock.rs @@ -209,7 +209,6 @@ pub fn reset_before_session_end_called() { } pub fn new_test_ext() -> sp_io::TestExternalities { - let state_version = Default::default(); let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); let keys: Vec<_> = NEXT_VALIDATORS .with(|l| l.borrow().iter().cloned().map(|i| (i, i, UintAuthorityId(i).into())).collect()); @@ -224,7 +223,7 @@ pub fn new_test_ext() -> sp_io::TestExternalities { pallet_session::GenesisConfig:: { keys } .assimilate_storage(&mut t) .unwrap(); - sp_io::TestExternalities::new(t,state_version) + sp_io::TestExternalities::new(t) } parameter_types! { diff --git a/frame/society/src/mock.rs b/frame/society/src/mock.rs index 9763be59389da..38c2586323135 100644 --- a/frame/society/src/mock.rs +++ b/frame/society/src/mock.rs @@ -154,20 +154,19 @@ impl EnvBuilder { } pub fn execute R>(mut self, f: F) -> R { - let state_version = Default::default(); - let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); + let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); self.balances.push((Society::account_id(), self.balance.max(self.pot))); pallet_balances::GenesisConfig:: { balances: self.balances } - .assimilate_storage(&mut t, state_version) + .assimilate_storage(&mut t) .unwrap(); pallet_society::GenesisConfig:: { members: self.members, pot: self.pot, max_members: self.max_members, } - .assimilate_storage(&mut t, state_version) + .assimilate_storage(&mut t) .unwrap(); - let mut ext: sp_io::TestExternalities = (t, state_version).into(); + let mut ext: sp_io::TestExternalities = t.into(); ext.execute_with(f) } #[allow(dead_code)] diff --git a/frame/staking/src/mock.rs b/frame/staking/src/mock.rs index 8a2df2817007e..776affde5d42d 100644 --- a/frame/staking/src/mock.rs +++ b/frame/staking/src/mock.rs @@ -395,8 +395,7 @@ impl ExtBuilder { } fn build(self) -> sp_io::TestExternalities { sp_tracing::try_init_simple(); - let state_version = Default::default(); - let mut storage = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); + let mut storage = frame_system::GenesisConfig::default().build_storage::().unwrap(); let _ = pallet_balances::GenesisConfig:: { balances: vec![ @@ -430,7 +429,7 @@ impl ExtBuilder { (999, 1_000_000_000_000), ], } - .assimilate_storage(&mut storage, state_version); + .assimilate_storage(&mut storage); let mut stakers = vec![]; if self.has_stakers { @@ -483,7 +482,7 @@ impl ExtBuilder { min_validator_bond: self.min_validator_bond, ..Default::default() } - .assimilate_storage(&mut storage, state_version); + .assimilate_storage(&mut storage); let _ = pallet_session::GenesisConfig:: { keys: if self.has_stakers { @@ -496,9 +495,9 @@ impl ExtBuilder { .collect() }, } - .assimilate_storage(&mut storage, state_version); + .assimilate_storage(&mut storage); - let mut ext = sp_io::TestExternalities::from((storage, state_version)); + let mut ext = sp_io::TestExternalities::from(storage); ext.execute_with(|| { let validators = Session::validators(); SESSION.with(|x| *x.borrow_mut() = (validators.clone(), HashSet::new())); diff --git a/frame/sudo/src/mock.rs b/frame/sudo/src/mock.rs index 00b1ac7c3fd95..7fd55a618a6b5 100644 --- a/frame/sudo/src/mock.rs +++ b/frame/sudo/src/mock.rs @@ -164,10 +164,9 @@ pub type LoggerCall = logger::Call; // Build test environment by setting the root `key` for the Genesis. pub fn new_test_ext(root_key: u64) -> sp_io::TestExternalities { - let state_version = sp_runtime::StateVersion::default(); - let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); + let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); sudo::GenesisConfig:: { key: root_key } - .assimilate_storage(&mut t, state_version) + .assimilate_storage(&mut t) .unwrap(); - (t, state_version).into() + t.into() } diff --git a/frame/support/src/lib.rs b/frame/support/src/lib.rs index e5ca24c7c0112..bf314161c7f87 100644 --- a/frame/support/src/lib.rs +++ b/frame/support/src/lib.rs @@ -847,9 +847,7 @@ pub mod tests { } fn new_test_ext() -> TestExternalities { - let state_version = Default::default(); - let storage = GenesisConfig::default().build_storage(state_version).unwrap(); - (storage, state_version).into() + GenesisConfig::default().build_storage().unwrap().into() } type Map = Data; diff --git a/frame/support/src/storage/generator/mod.rs b/frame/support/src/storage/generator/mod.rs index 0b2755ff03bb9..576bada2e262c 100644 --- a/frame/support/src/storage/generator/mod.rs +++ b/frame/support/src/storage/generator/mod.rs @@ -74,9 +74,8 @@ mod tests { #[test] fn value_translate_works() { - let state_version = Default::default(); - let t = GenesisConfig::default().build_storage(state_version).unwrap(); - TestExternalities::new(t, state_version).execute_with(|| { + let t = GenesisConfig::default().build_storage().unwrap(); + TestExternalities::new(t).execute_with(|| { // put the old value `1111u32` in the storage. let key = Value::storage_value_final_key(); unhashed::put_raw(&key, &1111u32.encode()); @@ -95,9 +94,8 @@ mod tests { #[test] fn map_translate_works() { - let state_version = Default::default(); - let t = GenesisConfig::default().build_storage(state_version).unwrap(); - TestExternalities::new(t, state_version).execute_with(|| { + let t = GenesisConfig::default().build_storage().unwrap(); + TestExternalities::new(t).execute_with(|| { // start with a map of u32 -> u32. for i in 0u32..100u32 { unhashed::put(&NumberMap::hashed_key_for(&i), &(i as u64)); @@ -125,9 +123,8 @@ mod tests { #[test] fn try_mutate_works() { - let state_version = Default::default(); - let t = GenesisConfig::default().build_storage(state_version).unwrap(); - TestExternalities::new(t, state_version).execute_with(|| { + let t = GenesisConfig::default().build_storage().unwrap(); + TestExternalities::new(t).execute_with(|| { assert_eq!(Value::get(), (0, 0)); assert_eq!(NumberMap::get(0), 0); assert_eq!(DoubleMap::get(0, 0), 0); diff --git a/frame/support/test/tests/decl_storage.rs b/frame/support/test/tests/decl_storage.rs index 72cebf900464a..666dda49935ef 100644 --- a/frame/support/test/tests/decl_storage.rs +++ b/frame/support/test/tests/decl_storage.rs @@ -668,10 +668,9 @@ mod tests { #[test] fn check_builder_config() { - let state_version = Default::default(); let config = GenesisConfig::default(); - let storage = config.build_storage(state_version).unwrap(); - TestExternalities::from((storage, state_version)).execute_with(|| { + let storage = config.build_storage().unwrap(); + TestExternalities::from(storage).execute_with(|| { assert_eq!(Module::::u32_with_builder(), 1); assert_eq!(Module::::opt_u32_with_builder_some(), Some(1)); assert_eq!(Module::::opt_u32_with_builder_none(), None); diff --git a/frame/support/test/tests/instance.rs b/frame/support/test/tests/instance.rs index 5e18d964660a2..574c0dc671b42 100644 --- a/frame/support/test/tests/instance.rs +++ b/frame/support/test/tests/instance.rs @@ -318,9 +318,7 @@ pub type Block = generic::Block; pub type UncheckedExtrinsic = generic::UncheckedExtrinsic; fn new_test_ext() -> sp_io::TestExternalities { - let state_version = Default::default(); - - (GenesisConfig { + GenesisConfig { module_1_1: module1::GenesisConfig { value: 3, test: 2 }, module_1_2: module1::GenesisConfig { value: 4, test: 5 }, module_2: module2::GenesisConfig { @@ -336,8 +334,9 @@ fn new_test_ext() -> sp_io::TestExternalities { module_2_2: Default::default(), module_2_3: Default::default(), } - .build_storage(state_version) - .unwrap(), state_version).into() + .build_storage() + .unwrap() + .into() } #[test] diff --git a/frame/system/benches/bench.rs b/frame/system/benches/bench.rs index eb6fbcd886ddb..97c19c5e8159a 100644 --- a/frame/system/benches/bench.rs +++ b/frame/system/benches/bench.rs @@ -101,8 +101,7 @@ impl module::Config for Runtime { } fn new_test_ext() -> sp_io::TestExternalities { - let state_version = Default::default(); - (system::GenesisConfig::default().build_storage::(state_version).unwrap(), state_version).into() + system::GenesisConfig::default().build_storage::().unwrap().into() } fn deposit_events(n: usize) { diff --git a/frame/system/benchmarking/src/mock.rs b/frame/system/benchmarking/src/mock.rs index de6a806edf69c..d828fb22ff5ff 100644 --- a/frame/system/benchmarking/src/mock.rs +++ b/frame/system/benchmarking/src/mock.rs @@ -67,7 +67,6 @@ impl frame_system::Config for Test { impl crate::Config for Test {} pub fn new_test_ext() -> sp_io::TestExternalities { - let state_version = Default::default(); - let t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); - sp_io::TestExternalities::new(t, state_version) + let t = frame_system::GenesisConfig::default().build_storage::().unwrap(); + sp_io::TestExternalities::new(t) } diff --git a/frame/system/src/lib.rs b/frame/system/src/lib.rs index 590b87d1385df..a8bf253c392c4 100644 --- a/frame/system/src/lib.rs +++ b/frame/system/src/lib.rs @@ -1438,15 +1438,14 @@ impl Pallet { /// Get the basic externalities for this pallet, useful for tests. #[cfg(any(feature = "std", test))] pub fn externalities() -> TestExternalities { - let storage = sp_core::storage::Storage { + TestExternalities::new(sp_core::storage::Storage { top: map![ >::hashed_key_for(T::BlockNumber::zero()) => [69u8; 32].encode(), >::hashed_key().to_vec() => T::BlockNumber::one().encode(), >::hashed_key().to_vec() => [69u8; 32].encode() ], children_default: map![], - }; - TestExternalities::new(storage, Default::default()) + }) } /// Set the block number to something in particular. Can be used as an alternative to diff --git a/frame/system/src/mock.rs b/frame/system/src/mock.rs index cc0ff6b97c906..8039b73445ae6 100644 --- a/frame/system/src/mock.rs +++ b/frame/system/src/mock.rs @@ -120,9 +120,8 @@ pub const CALL: &::Call = &Call::System(frame_system::Call::set_ /// Create new externalities for `System` module tests. pub fn new_test_ext() -> sp_io::TestExternalities { - let state_version = Default::default(); let mut ext: sp_io::TestExternalities = - (GenesisConfig::default().build_storage(state_version).unwrap(), state_version).into(); + GenesisConfig::default().build_storage().unwrap().into(); // Add to each test the initial weight of a block ext.execute_with(|| { System::register_extra_weight_unchecked( diff --git a/frame/timestamp/src/lib.rs b/frame/timestamp/src/lib.rs index f492290dc066b..a1ff8d37ff881 100644 --- a/frame/timestamp/src/lib.rs +++ b/frame/timestamp/src/lib.rs @@ -316,9 +316,8 @@ mod tests { }; pub fn new_test_ext() -> TestExternalities { - let state_version = Default::default(); - let t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); - TestExternalities::new(t, state_version) + let t = frame_system::GenesisConfig::default().build_storage::().unwrap(); + TestExternalities::new(t) } type UncheckedExtrinsic = frame_system::mocking::MockUncheckedExtrinsic; diff --git a/frame/tips/src/tests.rs b/frame/tips/src/tests.rs index b4155c0659dab..8611320563c7d 100644 --- a/frame/tips/src/tests.rs +++ b/frame/tips/src/tests.rs @@ -162,16 +162,15 @@ impl Config for Test { } pub fn new_test_ext() -> sp_io::TestExternalities { - let state_version = Default::default(); - let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); + let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); pallet_balances::GenesisConfig:: { // Total issuance will be 200 with treasury account initialized at ED. balances: vec![(0, 100), (1, 98), (2, 1)], } - .assimilate_storage(&mut t, state_version) + .assimilate_storage(&mut t) .unwrap(); - GenesisBuild::::assimilate_storage(&pallet_treasury::GenesisConfig, &mut t, state_version).unwrap(); - (t, state_version).into() + GenesisBuild::::assimilate_storage(&pallet_treasury::GenesisConfig, &mut t).unwrap(); + t.into() } fn last_event() -> RawEvent { @@ -404,7 +403,6 @@ fn tip_changing_works() { fn test_last_reward_migration() { use sp_storage::Storage; - let state_version = Default::default(); let mut s = Storage::default(); #[derive(Clone, Eq, PartialEq, Encode, Decode, RuntimeDebug)] @@ -457,7 +455,7 @@ fn test_last_reward_migration() { s.top = data.into_iter().collect(); - sp_io::TestExternalities::new(s, state_version).execute_with(|| { + sp_io::TestExternalities::new(s).execute_with(|| { TipsModTestInst::migrate_retract_tip_for_tip_new(); // Test w/ finder @@ -492,17 +490,16 @@ fn test_last_reward_migration() { #[test] fn genesis_funding_works() { - let state_version = Default::default(); - let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); + let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); let initial_funding = 100; pallet_balances::GenesisConfig:: { // Total issuance will be 200 with treasury account initialized with 100. balances: vec![(0, 100), (Treasury::account_id(), initial_funding)], } - .assimilate_storage(&mut t, state_version) + .assimilate_storage(&mut t) .unwrap(); - GenesisBuild::::assimilate_storage(&pallet_treasury::GenesisConfig, &mut t, state_version).unwrap(); - let mut t: sp_io::TestExternalities = (t, state_version).into(); + GenesisBuild::::assimilate_storage(&pallet_treasury::GenesisConfig, &mut t).unwrap(); + let mut t: sp_io::TestExternalities = t.into(); t.execute_with(|| { assert_eq!(Balances::free_balance(Treasury::account_id()), initial_funding); diff --git a/frame/transaction-payment/src/lib.rs b/frame/transaction-payment/src/lib.rs index ae9ac087c07ee..9e8dbf6cb5d1d 100644 --- a/frame/transaction-payment/src/lib.rs +++ b/frame/transaction-payment/src/lib.rs @@ -858,9 +858,8 @@ mod tests { WEIGHT_TO_FEE.with(|v| *v.borrow_mut() = self.weight_to_fee); } pub fn build(self) -> sp_io::TestExternalities { - let state_version = Default::default(); self.set_constants(); - let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); + let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); pallet_balances::GenesisConfig:: { balances: if self.balance_factor > 0 { vec![ @@ -875,9 +874,9 @@ mod tests { vec![] }, } - .assimilate_storage(&mut t, state_version) + .assimilate_storage(&mut t) .unwrap(); - (t, state_version).into() + t.into() } } diff --git a/frame/transaction-storage/src/mock.rs b/frame/transaction-storage/src/mock.rs index 540601a6e3828..38d14129d76e2 100644 --- a/frame/transaction-storage/src/mock.rs +++ b/frame/transaction-storage/src/mock.rs @@ -104,7 +104,6 @@ impl pallet_transaction_storage::Config for Test { } pub fn new_test_ext() -> sp_io::TestExternalities { - let state_version = Default::default(); let t = GenesisConfig { system: Default::default(), balances: pallet_balances::GenesisConfig:: { @@ -118,9 +117,9 @@ pub fn new_test_ext() -> sp_io::TestExternalities { max_transaction_size: crate::DEFAULT_MAX_TRANSACTION_SIZE, }, } - .build_storage(state_version) + .build_storage() .unwrap(); - (t, state_version).into() + t.into() } pub fn run_to_block(n: u64, f: impl Fn() -> Option) { diff --git a/frame/treasury/src/tests.rs b/frame/treasury/src/tests.rs index 0cc44c0aa836a..534661b2773bb 100644 --- a/frame/treasury/src/tests.rs +++ b/frame/treasury/src/tests.rs @@ -126,16 +126,15 @@ impl Config for Test { } pub fn new_test_ext() -> sp_io::TestExternalities { - let state_version = Default::default(); - let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); + let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); pallet_balances::GenesisConfig:: { // Total issuance will be 200 with treasury account initialized at ED. balances: vec![(0, 100), (1, 98), (2, 1)], } - .assimilate_storage(&mut t, state_version) + .assimilate_storage(&mut t) .unwrap(); - GenesisBuild::::assimilate_storage(&crate::GenesisConfig, &mut t, state_version).unwrap(); - (t, state_version).into() + GenesisBuild::::assimilate_storage(&crate::GenesisConfig, &mut t).unwrap(); + t.into() } #[test] @@ -322,13 +321,12 @@ fn treasury_account_doesnt_get_deleted() { // This is useful for chain that will just update runtime. #[test] fn inexistent_account_works() { - let state_version = Default::default(); - let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); + let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); pallet_balances::GenesisConfig:: { balances: vec![(0, 100), (1, 99), (2, 1)] } - .assimilate_storage(&mut t, state_version) + .assimilate_storage(&mut t) .unwrap(); // Treasury genesis config is not build thus treasury account does not exist - let mut t: sp_io::TestExternalities = (t, state_version).into(); + let mut t: sp_io::TestExternalities = t.into(); t.execute_with(|| { assert_eq!(Balances::free_balance(Treasury::account_id()), 0); // Account does not exist @@ -355,17 +353,16 @@ fn inexistent_account_works() { #[test] fn genesis_funding_works() { - let state_version = Default::default(); - let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); + let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); let initial_funding = 100; pallet_balances::GenesisConfig:: { // Total issuance will be 200 with treasury account initialized with 100. balances: vec![(0, 100), (Treasury::account_id(), initial_funding)], } - .assimilate_storage(&mut t, state_version) + .assimilate_storage(&mut t) .unwrap(); - GenesisBuild::::assimilate_storage(&crate::GenesisConfig, &mut t, state_version).unwrap(); - let mut t: sp_io::TestExternalities = (t, state_version).into(); + GenesisBuild::::assimilate_storage(&crate::GenesisConfig, &mut t).unwrap(); + let mut t: sp_io::TestExternalities = t.into(); t.execute_with(|| { assert_eq!(Balances::free_balance(Treasury::account_id()), initial_funding); diff --git a/frame/uniques/src/mock.rs b/frame/uniques/src/mock.rs index dbb090f4269ab..658e82a5143e2 100644 --- a/frame/uniques/src/mock.rs +++ b/frame/uniques/src/mock.rs @@ -117,10 +117,9 @@ impl Config for Test { } pub(crate) fn new_test_ext() -> sp_io::TestExternalities { - let state_version = Default::default(); - let t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); + let t = frame_system::GenesisConfig::default().build_storage::().unwrap(); - let mut ext = sp_io::TestExternalities::new(t, state_version); + let mut ext = sp_io::TestExternalities::new(t); ext.execute_with(|| System::set_block_number(1)); ext } diff --git a/frame/utility/src/tests.rs b/frame/utility/src/tests.rs index 46345a670a479..2731b6ca0b8be 100644 --- a/frame/utility/src/tests.rs +++ b/frame/utility/src/tests.rs @@ -169,14 +169,13 @@ use frame_system::Call as SystemCall; use pallet_balances::{Call as BalancesCall, Error as BalancesError}; pub fn new_test_ext() -> sp_io::TestExternalities { - let state_version = Default::default(); - let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); + let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); pallet_balances::GenesisConfig:: { balances: vec![(1, 10), (2, 10), (3, 10), (4, 10), (5, 2)], } - .assimilate_storage(&mut t, state_version) + .assimilate_storage(&mut t) .unwrap(); - let mut ext = sp_io::TestExternalities::new(t, state_version); + let mut ext = sp_io::TestExternalities::new(t); ext.execute_with(|| System::set_block_number(1)); ext } diff --git a/frame/vesting/src/mock.rs b/frame/vesting/src/mock.rs index 5b7bc2606afe8..4efbabefe688f 100644 --- a/frame/vesting/src/mock.rs +++ b/frame/vesting/src/mock.rs @@ -111,9 +111,8 @@ impl ExtBuilder { } pub fn build(self) -> sp_io::TestExternalities { - let state_version = Default::default(); EXISTENTIAL_DEPOSIT.with(|v| *v.borrow_mut() = self.existential_deposit); - let mut t = frame_system::GenesisConfig::default().build_storage::(state_version).unwrap(); + let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); pallet_balances::GenesisConfig:: { balances: vec![ (1, 10 * self.existential_deposit), @@ -123,7 +122,7 @@ impl ExtBuilder { (12, 10 * self.existential_deposit), ], } - .assimilate_storage(&mut t, state_version) + .assimilate_storage(&mut t) .unwrap(); pallet_vesting::GenesisConfig:: { vesting: vec![ @@ -132,9 +131,9 @@ impl ExtBuilder { (12, 10, 20, 5 * self.existential_deposit), ], } - .assimilate_storage(&mut t, state_version) + .assimilate_storage(&mut t) .unwrap(); - let mut ext = sp_io::TestExternalities::new(t, state_version); + let mut ext = sp_io::TestExternalities::new(t); ext.execute_with(|| System::set_block_number(1)); ext } diff --git a/primitives/state-machine/src/testing.rs b/primitives/state-machine/src/testing.rs index 45729958ba453..eb12fddd6efbd 100644 --- a/primitives/state-machine/src/testing.rs +++ b/primitives/state-machine/src/testing.rs @@ -87,17 +87,28 @@ where } /// Create a new instance of `TestExternalities` with storage. - pub fn new(storage: Storage, state_version: StateVersion) -> Self { - Self::new_with_code(&[], storage, state_version) + pub fn new(storage: Storage) -> Self { + Self::new_with_code_and_state(&[], storage, Default::default()) } + /// Create a new instance of `TestExternalities` with storage for a given state version. + pub fn new_with_state_version(storage: Storage, state_version: StateVersion) -> Self { + Self::new_with_code_and_state(&[], storage, state_version) + } + + /// New empty test externalities. pub fn new_empty() -> Self { - Self::new_with_code(&[], Storage::default(), Default::default()) + Self::new_with_code_and_state(&[], Storage::default(), Default::default()) } /// Create a new instance of `TestExternalities` with code and storage. - pub fn new_with_code(code: &[u8], mut storage: Storage, state_version: StateVersion) -> Self { + pub fn new_with_code(code: &[u8], storage: Storage) -> Self { + Self::new_with_code_and_state(code, storage, Default::default()) + } + + /// Create a new instance of `TestExternalities` with code and storage for a given state version. + pub fn new_with_code_and_state(code: &[u8], mut storage: Storage, state_version: StateVersion) -> Self { let mut overlay = OverlayedChanges::default(); let changes_trie_config = storage .top @@ -245,7 +256,16 @@ where { fn default() -> Self { // default to default version. - Self::new(Storage::default(), Default::default()) + Self::new_with_state_version(Storage::default(), Default::default()) + } +} + +impl From for TestExternalities +where + H::Out: Ord + 'static + codec::Codec, +{ + fn from(storage: Storage) -> Self { + Self::new_with_state_version(storage, Default::default()) } } @@ -254,7 +274,7 @@ where H::Out: Ord + 'static + codec::Codec, { fn from((storage, state_version): (Storage, StateVersion)) -> Self { - Self::new(storage, state_version) + Self::new_with_state_version(storage, state_version) } } diff --git a/test-utils/runtime/src/system.rs b/test-utils/runtime/src/system.rs index 47caf04e2e656..316a553ed027d 100644 --- a/test-utils/runtime/src/system.rs +++ b/test-utils/runtime/src/system.rs @@ -366,13 +366,11 @@ mod tests { } fn new_test_ext() -> TestExternalities { - let state_version = Default::default(); let authorities = vec![ Sr25519Keyring::Alice.to_raw_public(), Sr25519Keyring::Bob.to_raw_public(), Sr25519Keyring::Charlie.to_raw_public(), ]; - TestExternalities::new_with_code( wasm_binary_unwrap(), sp_core::storage::Storage { @@ -381,11 +379,10 @@ mod tests { twox_128(b"sys:auth").to_vec() => authorities.encode(), blake2_256(&AccountKeyring::Alice.to_raw_public().to_keyed_vec(b"balance:")).to_vec() => { vec![111u8, 0, 0, 0, 0, 0, 0, 0] - }, + } ], children_default: map![], }, - state_version, ) } From a6359b417df886dbceda5e7b0e63df873cb71b0a Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 23 Aug 2021 19:25:16 +0200 Subject: [PATCH 080/127] fix some tests. --- frame/executive/src/lib.rs | 14 +++++++++++--- frame/support/test/tests/instance.rs | 2 +- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/frame/executive/src/lib.rs b/frame/executive/src/lib.rs index 5f1ae23c2f531..ebfd5bcb0ebab 100644 --- a/frame/executive/src/lib.rs +++ b/frame/executive/src/lib.rs @@ -846,9 +846,17 @@ mod tests { t.into() } + fn new_test_ext_v0(balance_factor: Balance) -> sp_io::TestExternalities { + let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); + pallet_balances::GenesisConfig:: { balances: vec![(1, 111 * balance_factor)] } + .assimilate_storage(&mut t) + .unwrap(); + (t, sp_runtime::StateVersion::V0).into() + } + #[test] fn block_import_works() { - new_test_ext(1).execute_with(|| { + new_test_ext_v0(1).execute_with(|| { Executive::execute_block(Block { header: Header { parent_hash: [69u8; 32].into(), @@ -871,7 +879,7 @@ mod tests { #[test] #[should_panic] fn block_import_of_bad_state_root_fails() { - new_test_ext(1).execute_with(|| { + new_test_ext_v0(1).execute_with(|| { Executive::execute_block(Block { header: Header { parent_hash: [69u8; 32].into(), @@ -891,7 +899,7 @@ mod tests { #[test] #[should_panic] fn block_import_of_bad_extrinsic_root_fails() { - new_test_ext(1).execute_with(|| { + new_test_ext_v0(1).execute_with(|| { Executive::execute_block(Block { header: Header { parent_hash: [69u8; 32].into(), diff --git a/frame/support/test/tests/instance.rs b/frame/support/test/tests/instance.rs index 574c0dc671b42..841b69dafed1a 100644 --- a/frame/support/test/tests/instance.rs +++ b/frame/support/test/tests/instance.rs @@ -360,7 +360,7 @@ fn storage_instance_independence() { module2::DoubleMap::::insert(&0, &0, &0); }); // 12 storage values and threshold. - assert_eq!(storage.top.len(), 13); + assert_eq!(storage.top.len(), 12); } #[test] From db0769ec23200458722fb03283d8bd0dce62307f Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 24 Aug 2021 09:31:46 +0200 Subject: [PATCH 081/127] default state version from proof to V0 --- primitives/trie/src/storage_proof.rs | 45 +++++++++++++++++++++++++--- 1 file changed, 41 insertions(+), 4 deletions(-) diff --git a/primitives/trie/src/storage_proof.rs b/primitives/trie/src/storage_proof.rs index 0eb13db4cd0fa..48548b490f133 100644 --- a/primitives/trie/src/storage_proof.rs +++ b/primitives/trie/src/storage_proof.rs @@ -29,19 +29,56 @@ use sp_core::state_version::StateVersion; /// The proof consists of the set of serialized nodes in the storage trie accessed when looking up /// the keys covered by the proof. Verifying the proof requires constructing the partial trie from /// the serialized nodes and performing the key lookups. -#[derive(Debug, PartialEq, Eq, Clone, Encode, Decode)] +#[derive(Debug, PartialEq, Eq, Clone, Encode)] pub struct StorageProof { - trie_nodes: Vec>, - state_version: StateVersion, + // TODO decode no more bytes to V0 for compatibility and remove pub(crate) + pub(crate) trie_nodes: Vec>, + pub(crate) state_version: StateVersion, } /// Storage proof in compact form. -#[derive(Debug, PartialEq, Eq, Clone, Encode, Decode)] +#[derive(Debug, PartialEq, Eq, Clone, Encode)] pub struct CompactProof { pub encoded_nodes: Vec>, pub state_version: StateVersion, } +mod decode_impl { + use super::*; + use codec::{Input, Error}; + impl Decode for StorageProof { + fn decode(input: &mut I) -> Result { + let trie_nodes: Vec> = Decode::decode(input)?; + let state_version = if let Ok(Some(0)) = input.remaining_len() { + // for compatibility with existing proof decoded from sized buffer + StateVersion::V0 + } else { + Decode::decode(input)? + }; + Ok(StorageProof { + trie_nodes, + state_version, + }) + } + } + + impl Decode for CompactProof { + fn decode(input: &mut I) -> Result { + let encoded_nodes: Vec> = Decode::decode(input)?; + let state_version = if let Ok(Some(0)) = input.remaining_len() { + // for compatibility with existing proof decoded from sized buffer + StateVersion::V0 + } else { + Decode::decode(input)? + }; + Ok(CompactProof { + encoded_nodes, + state_version, + }) + } + } +} + impl StorageProof { /// Constructs a storage proof from a subset of encoded trie nodes in a storage backend. pub fn new(trie_nodes: Vec>, state_version: StateVersion) -> Self { From dc47bfbf5a3e25b588fb4b8e8474f571a68e2440 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 24 Aug 2021 09:43:54 +0200 Subject: [PATCH 082/127] storage proof estimate size fix --- .../state-machine/src/proving_backend.rs | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/primitives/state-machine/src/proving_backend.rs b/primitives/state-machine/src/proving_backend.rs index 351b58af846c0..4bac7ee5b2e5b 100644 --- a/primitives/state-machine/src/proving_backend.rs +++ b/primitives/state-machine/src/proving_backend.rs @@ -112,6 +112,8 @@ struct ProofRecorderInner { records: HashMap>, /// The encoded size of all recorded values. encoded_size: usize, + /// State version in use. + state_version: StateVersion, } /// Global proof recorder, act as a layer over a hash db for recording queried data. @@ -125,7 +127,7 @@ impl ProofRecorder { pub fn record(&self, key: Hash, val: Option) { let mut inner = self.inner.write(); - let ProofRecorderInner { encoded_size, records } = &mut *inner; + let ProofRecorderInner { encoded_size, records, .. } = &mut *inner; records.entry(key).or_insert_with(|| { val.map(|val| { let mut val = (val, Meta::default(), false); @@ -170,18 +172,17 @@ impl ProofRecorder { pub fn estimate_encoded_size(&self) -> usize { let inner = self.inner.read(); inner.encoded_size + codec::Compact(inner.records.len() as u32).encoded_size() + + inner.state_version.encoded_size() } /// Convert into a [`StorageProof`]. - pub fn to_storage_proof(&self, state_version: StateVersion) -> StorageProof { - // TODO consider state_version as inner field - let try_inner_hashing = match state_version { + pub fn to_storage_proof(&self) -> StorageProof { + let inner = self.inner.read(); + let try_inner_hashing = match inner.state_version { StateVersion::V0 => None, StateVersion::V1 { threshold } => Some(threshold), }; - let trie_nodes = self - .inner - .read() + let trie_nodes = inner .records .iter() .filter_map(|(_k, v)| { @@ -199,7 +200,7 @@ impl ProofRecorder { }) .collect(); - StorageProof::new(trie_nodes, state_version) + StorageProof::new(trie_nodes, inner.state_version) } /// Reset the internal state. @@ -245,8 +246,7 @@ where /// Extracting the gathered unordered proof. pub fn extract_proof(&self) -> StorageProof { - let state_version = self.0.state_version(); - self.0.essence().backend_storage().proof_recorder.to_storage_proof::(state_version) + self.0.essence().backend_storage().proof_recorder.to_storage_proof::() } /// Returns the estimated encoded size of the proof. From bc95929d7b7b2567e87fc87300c56e078067d872 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 24 Aug 2021 09:58:41 +0200 Subject: [PATCH 083/127] fix test trie to use right hashing --- client/db/src/lib.rs | 4 ++-- primitives/core/src/state_version.rs | 2 +- primitives/state-machine/src/lib.rs | 6 +++--- primitives/state-machine/src/proving_backend.rs | 4 ++-- primitives/state-machine/src/trie_backend.rs | 8 +++++++- primitives/storage/src/lib.rs | 2 +- 6 files changed, 16 insertions(+), 10 deletions(-) diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index 379e00ea9f94e..960f580d32741 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -2590,7 +2590,7 @@ pub(crate) mod tests { assert!(backend.blockchain().hash(i).unwrap().is_some()) } } - +33 #[test] fn set_state_data() { set_state_data_inner(true); @@ -2599,7 +2599,7 @@ pub(crate) mod tests { fn set_state_data_inner(alt_hashing: bool) { let state_version = if alt_hashing { StateVersion::V1 { - threshold: 33, + threshold: sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD, } } else { StateVersion::V0 diff --git a/primitives/core/src/state_version.rs b/primitives/core/src/state_version.rs index 04586fe43bc4f..20eeb2565472a 100644 --- a/primitives/core/src/state_version.rs +++ b/primitives/core/src/state_version.rs @@ -25,7 +25,7 @@ use codec::{Decode, Encode}; /// initial state versionning in their chainspec for block 0. /// Therefore defining genesis version in chainspec is good practice /// and this default should mostly be use when testing. -pub const DEFAULT_STATE_VERSION: StateVersion = StateVersion::V1 { threshold: 33 }; +pub const DEFAULT_STATE_VERSION: StateVersion = StateVersion::V1 { threshold: crate::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD }; /// Supported version with substrate chain. #[derive(Debug, Clone, Copy, PartialEq, Eq, Encode, Decode)] diff --git a/primitives/state-machine/src/lib.rs b/primitives/state-machine/src/lib.rs index cb3a9308ab394..b64da7fcfa6fb 100644 --- a/primitives/state-machine/src/lib.rs +++ b/primitives/state-machine/src/lib.rs @@ -1704,7 +1704,7 @@ mod tests { } let root3 = root.clone(); assert!(root1 != root3); - let remote_proof = check_proof(mdb.clone(), root.clone(), StateVersion::V1 { threshold: 33 }); + let remote_proof = check_proof(mdb.clone(), root.clone(), StateVersion::V1 { threshold: TRESHOLD }); // nodes foo is replaced by its hashed value form. assert!(remote_proof.encode().len() < 1000); assert!(remote_proof.encoded_size() < 1000); @@ -1717,14 +1717,14 @@ mod tests { let size_no_inner_hash = compact_multiple_child_trie_inner(false); assert!(size_inner_hash < size_no_inner_hash); } - fn compact_multiple_child_trie_inner(flagged: bool) -> usize { + fn compact_multiple_child_trie_inner(inner_hashed: bool) -> usize { // this root will be queried let child_info1 = ChildInfo::new_default(b"sub1"); // this root will not be include in proof let child_info2 = ChildInfo::new_default(b"sub2"); // this root will be include in proof let child_info3 = ChildInfo::new_default(b"sub"); - let mut remote_backend = trie_backend::tests::test_trie(flagged); + let mut remote_backend = trie_backend::tests::test_trie(inner_hashed); let long_vec: Vec = (0..1024usize).map(|_| 8u8).collect(); let (remote_root, transaction) = remote_backend.full_storage_root( std::iter::empty(), diff --git a/primitives/state-machine/src/proving_backend.rs b/primitives/state-machine/src/proving_backend.rs index 4bac7ee5b2e5b..df02398a333cc 100644 --- a/primitives/state-machine/src/proving_backend.rs +++ b/primitives/state-machine/src/proving_backend.rs @@ -499,7 +499,7 @@ mod tests { .map(|i| (vec![i], Some(vec![i; size_content]))) .collect::>())]; let mut in_memory: InMemoryBackend = if flagged { - (contents, StateVersion::V1 { threshold: 33 }).into() + (contents, StateVersion::V1 { threshold: sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD }).into() } else { (contents, StateVersion::V0).into() }; @@ -541,7 +541,7 @@ mod tests { (Some(child_info_2.clone()), (10..15).map(|i| (vec![i], Some(vec![i]))).collect()), ]; let mut in_memory: InMemoryBackend = if flagged { - (contents, StateVersion::V1 { threshold: 33 }).into() + (contents, StateVersion::V1 { threshold: sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD }).into() } else { (contents, StateVersion::V0).into() }; diff --git a/primitives/state-machine/src/trie_backend.rs b/primitives/state-machine/src/trie_backend.rs index 76bef33af928a..cc198b6d21be4 100644 --- a/primitives/state-machine/src/trie_backend.rs +++ b/primitives/state-machine/src/trie_backend.rs @@ -343,7 +343,13 @@ pub mod tests { hashed_value: bool, ) -> TrieBackend, BlakeTwo256> { let (mdb, root) = test_db(hashed_value); - TrieBackend::new(mdb, root, Default::default()) + let state_version = if hashed_value { + StateVersion::V1 { threshold: sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD } + } else { + StateVersion::V0 + }; + + TrieBackend::new(mdb, root, state_version) } #[test] diff --git a/primitives/storage/src/lib.rs b/primitives/storage/src/lib.rs index 02fc1e1c41718..a02ad6c35ea38 100644 --- a/primitives/storage/src/lib.rs +++ b/primitives/storage/src/lib.rs @@ -244,7 +244,7 @@ pub fn trie_threshold_decode(mut encoded: &[u8]) -> Option { } /// Default value to use as a threshold for testing. -pub const TEST_DEFAULT_ALT_HASH_THRESHOLD: u32 = 34; +pub const TEST_DEFAULT_ALT_HASH_THRESHOLD: u32 = 33; /// Information related to a child state. #[derive(Debug, Clone)] From b8430c6c5e013403ab807771822cd19bcff6f71e Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 24 Aug 2021 10:18:04 +0200 Subject: [PATCH 084/127] fixes --- client/block-builder/src/lib.rs | 3 +-- client/db/src/bench.rs | 2 +- client/db/src/lib.rs | 2 +- primitives/api/proc-macro/src/impl_runtime_apis.rs | 4 ++-- primitives/api/proc-macro/src/mock_impl_runtime_apis.rs | 2 +- primitives/api/src/lib.rs | 2 +- 6 files changed, 7 insertions(+), 8 deletions(-) diff --git a/client/block-builder/src/lib.rs b/client/block-builder/src/lib.rs index d447ce5c2f5a4..8a0dbc45f81d7 100644 --- a/client/block-builder/src/lib.rs +++ b/client/block-builder/src/lib.rs @@ -236,8 +236,7 @@ where let state = self.backend.state_at(self.block_id)?; let proof = { - use backend::StateBackend; - self.api.extract_proof(state.state_version()) + self.api.extract_proof() }; let changes_trie_state = backend::changes_tries_state_at_block( diff --git a/client/db/src/bench.rs b/client/db/src/bench.rs index 6588fcd411905..56c73d62b077c 100644 --- a/client/db/src/bench.rs +++ b/client/db/src/bench.rs @@ -618,7 +618,7 @@ impl StateBackend> for BenchmarkingState { fn proof_size(&self) -> Option { self.proof_recorder.as_ref().map(|recorder| { let proof_size = recorder.estimate_encoded_size() as u32; - let proof = recorder.to_storage_proof::>(self.state_version.get()); + let proof = recorder.to_storage_proof::>(); let proof_recorder_root = self.proof_recorder_root.get(); if proof_recorder_root == Default::default() || proof_size == 1 { // empty trie diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index 960f580d32741..eb0bd02760433 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -2590,7 +2590,7 @@ pub(crate) mod tests { assert!(backend.blockchain().hash(i).unwrap().is_some()) } } -33 + #[test] fn set_state_data() { set_state_data_inner(true); diff --git a/primitives/api/proc-macro/src/impl_runtime_apis.rs b/primitives/api/proc-macro/src/impl_runtime_apis.rs index 6cbfd1458f8a6..81287b1fac64b 100644 --- a/primitives/api/proc-macro/src/impl_runtime_apis.rs +++ b/primitives/api/proc-macro/src/impl_runtime_apis.rs @@ -279,10 +279,10 @@ fn generate_runtime_api_base_structures() -> Result { self.recorder.clone() } - fn extract_proof(&mut self, state_version: sp_runtime::StateVersion) -> Option<#crate_::StorageProof> { + fn extract_proof(&mut self) -> Option<#crate_::StorageProof> { self.recorder .take() - .map(|recorder| recorder.to_storage_proof::<#crate_::HashFor>(state_version)) + .map(|recorder| recorder.to_storage_proof::<#crate_::HashFor>()) } fn into_storage_changes( diff --git a/primitives/api/proc-macro/src/mock_impl_runtime_apis.rs b/primitives/api/proc-macro/src/mock_impl_runtime_apis.rs index 140f0eb709af7..77f8a07f85c48 100644 --- a/primitives/api/proc-macro/src/mock_impl_runtime_apis.rs +++ b/primitives/api/proc-macro/src/mock_impl_runtime_apis.rs @@ -105,7 +105,7 @@ fn implement_common_api_traits(block_type: TypePath, self_ty: Type) -> Result Option<#crate_::StorageProof> { + fn extract_proof(&mut self) -> Option<#crate_::StorageProof> { unimplemented!("`extract_proof` not implemented for runtime api mocks") } diff --git a/primitives/api/src/lib.rs b/primitives/api/src/lib.rs index 7f164454b83c0..82954d193e605 100644 --- a/primitives/api/src/lib.rs +++ b/primitives/api/src/lib.rs @@ -502,7 +502,7 @@ pub trait ApiExt { /// This stops the proof recording. /// /// If `record_proof` was not called before, this will return `None`. - fn extract_proof(&mut self, state_version: sp_runtime::StateVersion) -> Option; + fn extract_proof(&mut self) -> Option; /// Returns the current active proof recorder. fn proof_recorder(&self) -> Option>; From 5b1fab8725b778e20c8b4ad7bbc113bbbb38ad67 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 24 Aug 2021 11:09:24 +0200 Subject: [PATCH 085/127] network test with migration. --- client/db/src/lib.rs | 13 ++++++++++- client/network/test/src/lib.rs | 39 ++++++++++++++++++++++++++++----- client/network/test/src/sync.rs | 15 +++++++++++++ test-utils/client/src/lib.rs | 26 ++++++++++++++++++---- 4 files changed, 82 insertions(+), 11 deletions(-) diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index eb0bd02760433..6fc9441b1c447 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -1148,6 +1148,18 @@ impl Backend { keep_blocks: u32, canonicalization_delay: u64, transaction_storage: TransactionStorageMode, + ) -> Self { + let state_versions = Default::default(); + Self::new_test_with_tx_storage_and_state_versions(keep_blocks, canonicalization_delay, transaction_storage, state_versions) + } + + /// Create new memory-backed client backend for tests. + #[cfg(any(test, feature = "test-helpers"))] + pub fn new_test_with_tx_storage_and_state_versions( + keep_blocks: u32, + canonicalization_delay: u64, + transaction_storage: TransactionStorageMode, + state_versions: StateVersions, ) -> Self { let db = kvdb_memorydb::create(crate::utils::NUM_COLUMNS); let db = sp_database::as_database(db); @@ -1159,7 +1171,6 @@ impl Backend { keep_blocks: KeepBlocks::Some(keep_blocks), transaction_storage, }; - let state_versions = Default::default(); Self::new(db_setting, canonicalization_delay, state_versions).expect("failed to create test-db") } diff --git a/client/network/test/src/lib.rs b/client/network/test/src/lib.rs index dc37d165222d5..7e7b47e5c0a7b 100644 --- a/client/network/test/src/lib.rs +++ b/client/network/test/src/lib.rs @@ -727,15 +727,19 @@ where self.add_full_peer_with_config(Default::default()) } + /// Get state versioning to use with test. + fn state_versions(&self) -> Option>; + /// Add a full peer. fn add_full_peer_with_config(&mut self, config: FullPeerConfig) { let mut test_client_builder = match (config.keep_blocks, config.storage_chain) { (Some(keep_blocks), true) => - TestClientBuilder::with_tx_storage(keep_blocks).state_hashed_value(), - (None, true) => TestClientBuilder::with_tx_storage(u32::MAX).state_hashed_value(), + TestClientBuilder::with_tx_storage(keep_blocks, self.state_versions()).state_hashed_value(), + (None, true) => TestClientBuilder::with_tx_storage(u32::MAX, self.state_versions()).state_hashed_value(), (Some(keep_blocks), false) => - TestClientBuilder::with_pruning_window(keep_blocks).state_hashed_value(), - (None, false) => TestClientBuilder::with_default_backend().state_hashed_value(), + TestClientBuilder::with_pruning_window(keep_blocks, self.state_versions()).state_hashed_value(), + (None, false) => + TestClientBuilder::with_default_backend_and_state_versions(self.state_versions()).state_hashed_value(), }; if matches!(config.sync_mode, SyncMode::Fast { .. }) { test_client_builder = test_client_builder.set_no_genesis(); @@ -1078,12 +1082,27 @@ where pub struct TestNet { peers: Vec>, fork_choice: ForkChoiceStrategy, + state_versions: Option>, } impl TestNet { /// Create a `TestNet` that used the given fork choice rule. pub fn with_fork_choice(fork_choice: ForkChoiceStrategy) -> Self { - Self { peers: Vec::new(), fork_choice } + Self { peers: Vec::new(), fork_choice, state_versions: None } + } + + /// Create new test network with this many peers. + pub fn new_with_state_versions(n: usize, state_versions: sp_runtime::StateVersions) -> Self { + trace!(target: "test_network", "Creating test network with peer config"); + let config = Self::default_config(); + let mut net = Self::from_config(&config); + net.state_versions = Some(state_versions); + + for i in 0..n { + trace!(target: "test_network", "Adding peer {}", i); + net.add_full_peer(); + } + net } } @@ -1094,7 +1113,7 @@ impl TestNetFactory for TestNet { /// Create new test network with peers and given config. fn from_config(_config: &ProtocolConfig) -> Self { - TestNet { peers: Vec::new(), fork_choice: ForkChoiceStrategy::LongestChain } + TestNet { peers: Vec::new(), fork_choice: ForkChoiceStrategy::LongestChain, state_versions: None } } fn make_verifier( @@ -1128,6 +1147,10 @@ impl TestNetFactory for TestNet { fn mut_peers>)>(&mut self, closure: F) { closure(&mut self.peers); } + + fn state_versions(&self) -> Option> { + self.state_versions.clone() + } } pub struct ForceFinalized(PeersClient); @@ -1197,4 +1220,8 @@ impl TestNetFactory for JustificationTestNet { ) { (client.as_block_import(), Some(Box::new(ForceFinalized(client))), Default::default()) } + + fn state_versions(&self) -> Option> { + self.0.state_versions.clone() + } } diff --git a/client/network/test/src/sync.rs b/client/network/test/src/sync.rs index 153a0f905bff7..1d07dfcbae1ea 100644 --- a/client/network/test/src/sync.rs +++ b/client/network/test/src/sync.rs @@ -157,6 +157,21 @@ fn sync_from_two_peers_works() { assert!(!net.peer(0).is_major_syncing()); } +#[test] +fn sync_from_two_peers_with_migrate_works() { + sp_tracing::try_init_simple(); + let mut state_versions = sp_runtime::StateVersions::::default(); + state_versions.add((0, sp_runtime::StateVersion::V0)); + state_versions.add((10, sp_runtime::StateVersion::default())); + let mut net = TestNet::new_with_state_versions(3, state_versions); + net.peer(1).push_blocks(100, false); + net.peer(2).push_blocks(100, false); + net.block_until_sync(); + let peer1 = &net.peers()[1]; + assert!(net.peers()[0].blockchain_canon_equals(peer1)); + assert!(!net.peer(0).is_major_syncing()); +} + #[test] fn sync_from_two_peers_with_ancestry_search_works() { sp_tracing::try_init_simple(); diff --git a/test-utils/client/src/lib.rs b/test-utils/client/src/lib.rs index b70263a7d77d4..018772a121255 100644 --- a/test-utils/client/src/lib.rs +++ b/test-utils/client/src/lib.rs @@ -106,18 +106,36 @@ impl Self::with_backend(backend) } + /// Create new `TestClientBuilder` with default backend and state versions. + pub fn with_default_backend_and_state_versions(state_versions: Option>) -> Self { + let backend = Arc::new(Backend::new_test_with_tx_storage_and_state_versions( + std::u32::MAX, + std::u64::MAX, + sc_client_db::TransactionStorageMode::BlockBody, + state_versions.unwrap_or_default(), + )); + Self::with_backend(backend) + } + + /// Create new `TestClientBuilder` with default backend and pruning window size - pub fn with_pruning_window(keep_blocks: u32) -> Self { - let backend = Arc::new(Backend::new_test(keep_blocks, 0)); + pub fn with_pruning_window(keep_blocks: u32, state_versions: Option>) -> Self { + let backend = Arc::new(Backend::new_test_with_tx_storage_and_state_versions( + keep_blocks, + 0, + sc_client_db::TransactionStorageMode::BlockBody, + state_versions.unwrap_or_default(), + )); Self::with_backend(backend) } /// Create new `TestClientBuilder` with default backend and storage chain mode - pub fn with_tx_storage(keep_blocks: u32) -> Self { - let backend = Arc::new(Backend::new_test_with_tx_storage( + pub fn with_tx_storage(keep_blocks: u32, state_versions: Option>) -> Self { + let backend = Arc::new(Backend::new_test_with_tx_storage_and_state_versions( keep_blocks, 0, sc_client_db::TransactionStorageMode::StorageChain, + state_versions.unwrap_or_default(), )); Self::with_backend(backend) } From 17703055af53c0054ac96ca908c804dd1ab209ea Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 24 Aug 2021 19:13:00 +0200 Subject: [PATCH 086/127] wrong approach: only work for block import but not production. really need to shortcut finalize_block_with_context: maybe inherent of migration that only accept no extrinsics. Or also implement built_block? --- client/db/src/lib.rs | 63 +++++- primitives/runtime/src/state_version.rs | 22 ++ primitives/state-machine/src/lib.rs | 2 +- primitives/state-machine/src/trie_backend.rs | 206 +++++++++++++++++++ 4 files changed, 290 insertions(+), 3 deletions(-) diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index 6fc9441b1c447..70beb09597428 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -46,7 +46,7 @@ mod upgrade; mod utils; use linked_hash_map::LinkedHashMap; -use log::{debug, trace, warn}; +use log::{debug, trace, warn, info}; use parking_lot::{Mutex, RwLock}; use std::{ collections::{HashMap, HashSet}, @@ -2057,10 +2057,69 @@ impl sc_client_api::backend::Backend for Backend { Ok(()) } - fn commit_operation(&self, operation: Self::BlockImportOperation) -> ClientResult<()> { + fn commit_operation(&self, mut operation: Self::BlockImportOperation) -> ClientResult<()> { let usage = operation.old_state.usage_info(); self.state_usage.merge_sm(usage); + + if let Some(pending_block) = operation.pending_block.as_mut() { + let number = pending_block.header.number().clone(); + + if let Some((migration_from, migration_to)) = self.state_versions.need_migrate(number) { + use std::time; + if let Some(prev_block_state) = operation.old_state.as_trie_backend() { + let limit_size: Option = None; + let limit_items: Option = None; + let init_root = pending_block.header.state_root().clone(); + let mut current_root = init_root.clone(); + let mut start_top = None; + let mut start_child = None; + + let timer = time::SystemTime::now(); + + let init_root = current_root.clone(); + info!("Starting migrating from state root {:x?} at block {:?}", ¤t_root, &number); + loop { + let sp_state_machine::MigrateProgress { + current_top, + current_child, + root, + } = prev_block_state.migrate( + migration_from, + migration_to, + limit_size, + limit_items, + &mut operation.db_updates, + init_root.clone(), + sp_state_machine::MigrateProgress { + current_top: start_top.take(), + current_child: start_child.take(), + root: Some(current_root), // TODO non optional rather? + } + ).map_err(|err| { + sp_blockchain::Error::Backend(format!( + "error migrating: {}", + err + )) + })?; + + info!("Finished migration iteration, new state root: {:x?}, elapsed time: {:?}.", ¤t_root, timer.elapsed()); + start_top = current_top; + start_child = current_child; + current_root = root.unwrap_or_else(|| pending_block.header.state_root().clone()); + if start_top.is_none() { + break; + } + } + + info!("Finished processing migration with final block state hash {:x?}, elapsed time: {:?}", ¤t_root, timer.elapsed()); + pending_block.header.set_state_root(current_root); + } else { + panic!("Could not migrate at block: {:?}", number); + } + } + } + match self.try_commit_operation(operation) { Ok(_) => { self.storage.state_db.apply_pending(); diff --git a/primitives/runtime/src/state_version.rs b/primitives/runtime/src/state_version.rs index c60c125138860..714f7c7390b2e 100644 --- a/primitives/runtime/src/state_version.rs +++ b/primitives/runtime/src/state_version.rs @@ -104,4 +104,26 @@ impl StateVersions { canonical_states, }) } + + + /// Indicate if block need migration. + /// Returns version to migrate from and to if + /// migration is needed. + pub fn need_migrate(&self, at: NumberFor) -> Option<(StateVersion, StateVersion)> { + // TODO switch to default when all V0 chains did migrate. + let mut from = StateVersion::V0; + let mut to = None; + let at = at + 1u32.into(); + for (number, version) in self.canonical_states.iter() { + if number == &at { + to = Some(*version); + break; + } + if number > &at { + break; + } + from = *version; + } + to.map(|to| (from, to)) + } } diff --git a/primitives/state-machine/src/lib.rs b/primitives/state-machine/src/lib.rs index b64da7fcfa6fb..3cf34e3c76ac3 100644 --- a/primitives/state-machine/src/lib.rs +++ b/primitives/state-machine/src/lib.rs @@ -127,7 +127,7 @@ pub use crate::{ StorageTransactionCache, StorageValue, }, stats::{StateMachineStats, UsageInfo, UsageUnit}, - trie_backend::TrieBackend, + trie_backend::{TrieBackend, MigrateProgress}, trie_backend_essence::{Storage, TrieBackendStorage}, }; diff --git a/primitives/state-machine/src/trie_backend.rs b/primitives/state-machine/src/trie_backend.rs index cc198b6d21be4..53f42a886ad8a 100644 --- a/primitives/state-machine/src/trie_backend.rs +++ b/primitives/state-machine/src/trie_backend.rs @@ -27,6 +27,7 @@ use hash_db::Hasher; use sp_core::storage::{ChildInfo, ChildType}; use sp_core::state_version::StateVersion; use sp_std::{boxed::Box, vec::Vec}; +use sp_std::{rc::Rc, cell::RefCell}; use sp_trie::{ child_delta_trie_root, delta_trie_root, empty_child_trie_root, trie_types::{TrieDB, TrieError}, @@ -294,6 +295,211 @@ where } } + +/// Migration progress. +pub struct MigrateProgress { + pub current_top: Option>, + pub current_child: Option<(Vec, H)>, + pub root: Option, +} + +/// Migration storage. +pub struct MigrateStorage<'a, S, H> +where + H: Hasher, +{ + overlay: Rc>>, + storage: &'a S, +} + +// type is neither send nor sync but only ever +// use in migrate method local to a single thread. +unsafe impl<'a, S, H: Hasher> Send for MigrateStorage<'a, S, H> { } +unsafe impl<'a, S, H: Hasher> Sync for MigrateStorage<'a, S, H> { } + +impl<'a, S, H: Hasher> Clone for MigrateStorage<'a, S, H> { + fn clone(&self) -> Self { + MigrateStorage { + overlay: self.overlay.clone(), + storage: self.storage + } + } +} + +use hash_db::{self, AsHashDB, HashDB, HashDBRef, Prefix}; +use sp_trie::DBValue; + +impl<'a, S: 'a + TrieBackendStorage, H: Hasher> hash_db::HashDB + for MigrateStorage<'a, S, H> +{ + fn get(&self, key: &H::Out, prefix: Prefix) -> Option { + if let Some(val) = HashDB::get(*self.overlay.borrow(), key, prefix) { + Some(val) + } else { + match self.storage.get(&key, prefix) { + Ok(x) => x, + Err(e) => { + warn!(target: "trie", "Failed to read from DB: {}", e); + None + }, + } + } + } + + fn access_from(&self, key: &H::Out, _at: Option<&H::Out>) -> Option { + // call back to storage even if the overlay was hit. + self.storage.access_from(key); + None + } + + fn contains(&self, key: &H::Out, prefix: Prefix) -> bool { + HashDB::get(self, key, prefix).is_some() + } + + fn insert(&mut self, prefix: Prefix, value: &[u8]) -> H::Out { + HashDB::insert(*self.overlay.borrow_mut(), prefix, value) + } + + fn emplace(&mut self, key: H::Out, prefix: Prefix, value: DBValue) { + HashDB::emplace(*self.overlay.borrow_mut(), key, prefix, value) + } + + fn emplace_ref(&mut self, key: &H::Out, prefix: Prefix, value: &[u8]) { + HashDB::emplace_ref(*self.overlay.borrow_mut(), key, prefix, value) + } + + fn remove(&mut self, key: &H::Out, prefix: Prefix) { + HashDB::remove(*self.overlay.borrow_mut(), key, prefix) + } +} + +impl<'a, S: 'a + TrieBackendStorage, H: Hasher> HashDBRef for MigrateStorage<'a, S, H> { + fn get(&self, key: &H::Out, prefix: Prefix) -> Option { + HashDB::get(self, key, prefix) + } + + fn access_from(&self, key: &H::Out, at: Option<&H::Out>) -> Option { + HashDB::access_from(self, key, at) + } + + fn contains(&self, key: &H::Out, prefix: Prefix) -> bool { + HashDB::contains(self, key, prefix) + } +} + +impl<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> AsHashDB + for MigrateStorage<'a, S, H> +{ + fn as_hash_db<'b>(&'b self) -> &'b (dyn HashDB + 'b) { + self + } + fn as_hash_db_mut<'b>(&'b mut self) -> &'b mut (dyn HashDB + 'b) { + self + } +} + +#[cfg(feature = "std")] +fn error_version() -> crate::DefaultError { + "Cannot migrate for given state versions".into() +} + +#[cfg(not(feature = "std"))] +fn error_version() -> crate::DefaultError { + crate::DefaultError +} + +#[cfg(feature = "std")] +fn trie_error(e: &sp_trie::TrieError>) -> crate::DefaultError { + format!("Migrate trieDB error: {}", e) +} + +#[cfg(not(feature = "std"))] +fn trie_error(_e: &sp_trie::TrieError>) -> crate::DefaultError { + crate::DefaultError +} + + +impl, H: Hasher> TrieBackend +where + H::Out: Ord + Codec, +{ + /// Execute a state migration, new and removed node are + /// directly written in the db update. + pub fn migrate( + &self, + migration_from: StateVersion, + migration_to: StateVersion, + mut limit_size: Option, + mut limit_items: Option, + changes: &mut sp_trie::PrefixedMemoryDB, + root: H::Out, + mut progress: MigrateProgress, + ) -> Result, crate::DefaultError> { + use sp_trie::{TrieDB, TrieDBMut, TrieMut, TrieDBIterator}; + let mut dest_threshold = 0; + match (migration_from, migration_to, self.state_version) { + (StateVersion::V0, StateVersion::V1 { threshold }, StateVersion::V0) => { + dest_threshold = threshold; + }, + _ => return Err(error_version()), + } + let empty_value = sp_std::vec![23u8; dest_threshold as usize]; + let empty_value2 = sp_std::vec![0u8; dest_threshold as usize]; + let dest_layout = Layout::with_alt_hashing(dest_threshold); + let overlay = Rc::new(RefCell::new(changes)); + let mut dest_db = MigrateStorage:: { + overlay, + storage: self.essence.backend_storage(), + }; + let ori_root = self.essence.root().clone(); + // Using db with remove node is ok as long as iteration + // start at uncommited key. + let ori_db = dest_db.clone(); + if progress.root.is_none() { + progress.root = Some(self.essence.root().clone()); + } + let dest_root = progress.root.as_mut().expect("Lazy init above."); + let ori = TrieDB::new_with_layout(&ori_db, &ori_root, Layout::default()) + .map_err(|e| trie_error::(&*e))?; + let mut dest = TrieDBMut::from_existing_with_layout(&mut dest_db, dest_root, dest_layout) + .map_err(|e| trie_error::(&*e))?; + + let mut iter = TrieDBIterator::new_prefixed_then_seek(&ori, &[], progress.current_top.as_ref().map(|s| s.as_slice()).unwrap_or(&[])).map_err(|e| trie_error::(&*e))?; + for elt in iter { + let (key, value) = elt + .map_err(|e| trie_error::(&*e))?; + if let Some(mut limit_size) = limit_size.as_mut() { + if *limit_size < value.len() as u64 { + progress.current_top = Some(key); + break; + } + *limit_size -= value.len() as u64; + } + if let Some(mut limit_items) = limit_items.as_mut() { + if *limit_items == 0 { + progress.current_top = Some(key); + break; + } + *limit_items -= 1; + } + if value.len() >= dest_threshold as usize { + // Force a change so triedbmut do not ignore + // setting the same value. + if value != empty_value { + dest.insert(key.as_slice(), empty_value.as_slice()); + } else { + dest.insert(key.as_slice(), empty_value2.as_slice()); + } + dest.insert(key.as_slice(), value.as_slice()); + } + } + + sp_std::mem::drop(ori); + sp_std::mem::drop(dest); + Ok(progress) + } +} + #[cfg(test)] pub mod tests { use super::*; From 5371a09bf852af5396c6cef6f6186f9e59e948f4 Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 25 Aug 2021 11:28:52 +0200 Subject: [PATCH 087/127] fix warnings --- client/db/src/lib.rs | 2 -- .../state-machine/src/proving_backend.rs | 2 +- primitives/state-machine/src/trie_backend.rs | 33 +++++++++++-------- 3 files changed, 20 insertions(+), 17 deletions(-) diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index 28281d9b07911..0ad78789203c0 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -2077,7 +2077,6 @@ impl sc_client_api::backend::Backend for Backend { let timer = time::SystemTime::now(); - let init_root = current_root.clone(); info!("Starting migrating from state root {:x?} at block {:?}", ¤t_root, &number); loop { let sp_state_machine::MigrateProgress { @@ -2090,7 +2089,6 @@ impl sc_client_api::backend::Backend for Backend { limit_size, limit_items, &mut operation.db_updates, - init_root.clone(), sp_state_machine::MigrateProgress { current_top: start_top.take(), current_child: start_child.take(), diff --git a/primitives/state-machine/src/proving_backend.rs b/primitives/state-machine/src/proving_backend.rs index b12bd1cd1a620..ab0b4a7c5b9e7 100644 --- a/primitives/state-machine/src/proving_backend.rs +++ b/primitives/state-machine/src/proving_backend.rs @@ -498,7 +498,7 @@ mod tests { .clone() .map(|i| (vec![i], Some(vec![i; size_content]))) .collect::>())]; - let mut in_memory: InMemoryBackend = if flagged { + let in_memory: InMemoryBackend = if flagged { (contents, StateVersion::V1 { threshold: sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD }).into() } else { (contents, StateVersion::V0).into() diff --git a/primitives/state-machine/src/trie_backend.rs b/primitives/state-machine/src/trie_backend.rs index 663d566f06612..7d3c7b0d5b8bc 100644 --- a/primitives/state-machine/src/trie_backend.rs +++ b/primitives/state-machine/src/trie_backend.rs @@ -298,9 +298,15 @@ where /// Migration progress. pub struct MigrateProgress { + /// Next key to migrate for top. pub current_top: Option>, - pub current_child: Option<(Vec, H)>, + /// Current migrated root of top trie. pub root: Option, + /// Next key to migrate with current migrated root for a child + /// trie. + /// When defined, `current_top` always point to the + /// child trie root top location. + pub current_child: Option<(Vec, H)>, } /// Migration storage. @@ -432,17 +438,13 @@ where mut limit_size: Option, mut limit_items: Option, changes: &mut sp_trie::PrefixedMemoryDB, - root: H::Out, mut progress: MigrateProgress, ) -> Result, crate::DefaultError> { use sp_trie::{TrieDB, TrieDBMut, TrieMut, TrieDBIterator}; - let mut dest_threshold = 0; - match (migration_from, migration_to, self.state_version) { - (StateVersion::V0, StateVersion::V1 { threshold }, StateVersion::V0) => { - dest_threshold = threshold; - }, + let dest_threshold = match (migration_from, migration_to, self.state_version) { + (StateVersion::V0, StateVersion::V1 { threshold }, StateVersion::V0) => threshold, _ => return Err(error_version()), - } + }; let empty_value = sp_std::vec![23u8; dest_threshold as usize]; let empty_value2 = sp_std::vec![0u8; dest_threshold as usize]; let dest_layout = Layout::with_alt_hashing(dest_threshold); @@ -464,18 +466,18 @@ where let mut dest = TrieDBMut::from_existing_with_layout(&mut dest_db, dest_root, dest_layout) .map_err(|e| trie_error::(&*e))?; - let mut iter = TrieDBIterator::new_prefixed_then_seek(&ori, &[], progress.current_top.as_ref().map(|s| s.as_slice()).unwrap_or(&[])).map_err(|e| trie_error::(&*e))?; + let iter = TrieDBIterator::new_prefixed_then_seek(&ori, &[], progress.current_top.as_ref().map(|s| s.as_slice()).unwrap_or(&[])).map_err(|e| trie_error::(&*e))?; for elt in iter { let (key, value) = elt .map_err(|e| trie_error::(&*e))?; - if let Some(mut limit_size) = limit_size.as_mut() { + if let Some(limit_size) = limit_size.as_mut() { if *limit_size < value.len() as u64 { progress.current_top = Some(key); break; } *limit_size -= value.len() as u64; } - if let Some(mut limit_items) = limit_items.as_mut() { + if let Some(limit_items) = limit_items.as_mut() { if *limit_items == 0 { progress.current_top = Some(key); break; @@ -486,11 +488,14 @@ where // Force a change so triedbmut do not ignore // setting the same value. if value != empty_value { - dest.insert(key.as_slice(), empty_value.as_slice()); + dest.insert(key.as_slice(), empty_value.as_slice()) + .map_err(|e| trie_error::(&*e))?; } else { - dest.insert(key.as_slice(), empty_value2.as_slice()); + dest.insert(key.as_slice(), empty_value2.as_slice()) + .map_err(|e| trie_error::(&*e))?; } - dest.insert(key.as_slice(), value.as_slice()); + dest.insert(key.as_slice(), value.as_slice()) + .map_err(|e| trie_error::(&*e))?; } } From edfcfd5714211760152a4a69a72b315df78bcd1b Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 25 Aug 2021 11:39:01 +0200 Subject: [PATCH 088/127] cargo format. --- bin/node/bench/src/generator.rs | 4 +- bin/node/testing/src/bench.rs | 5 +- client/api/src/in_mem.rs | 9 +- client/block-builder/src/lib.rs | 4 +- client/chain-spec/src/chain_spec.rs | 3 +- client/chain-spec/src/lib.rs | 3 +- client/db/src/bench.rs | 2 +- client/db/src/lib.rs | 88 +++++---- client/db/src/storage_cache.rs | 177 +++++++++++++----- client/light/src/backend.rs | 10 +- client/light/src/fetcher.rs | 6 +- client/light/src/lib.rs | 5 +- client/network/test/src/lib.rs | 23 ++- client/service/src/builder.rs | 32 +++- client/service/src/client/light.rs | 10 +- client/service/test/src/client/light.rs | 3 +- client/service/test/src/client/mod.rs | 16 +- primitives/core/src/lib.rs | 2 +- primitives/core/src/state_version.rs | 3 +- primitives/io/src/lib.rs | 41 ++-- primitives/runtime/src/lib.rs | 2 +- primitives/runtime/src/state_version.rs | 31 ++- primitives/state-machine/src/basic.rs | 77 ++++---- .../state-machine/src/changes_trie/build.rs | 25 +-- primitives/state-machine/src/ext.rs | 130 +++++++------ .../state-machine/src/in_memory_backend.rs | 28 ++- primitives/state-machine/src/lib.rs | 8 +- .../state-machine/src/proving_backend.rs | 31 ++- primitives/state-machine/src/testing.rs | 12 +- primitives/state-machine/src/trie_backend.rs | 64 +++---- primitives/trie/src/storage_proof.rs | 19 +- test-utils/client/src/lib.rs | 24 +-- test-utils/runtime/src/lib.rs | 3 +- utils/frame/try-runtime/cli/src/lib.rs | 4 +- 34 files changed, 533 insertions(+), 371 deletions(-) diff --git a/bin/node/bench/src/generator.rs b/bin/node/bench/src/generator.rs index 36c93378559bb..dfd2b549ce922 100644 --- a/bin/node/bench/src/generator.rs +++ b/bin/node/bench/src/generator.rs @@ -46,9 +46,7 @@ pub fn generate_trie( let mut trie = SimpleTrie { db, overlay: &mut overlay }; { let mut trie_db = match state_version { - StateVersion::V0 => { - TrieDBMut::new(&mut trie, &mut root) - }, + StateVersion::V0 => TrieDBMut::new(&mut trie, &mut root), StateVersion::V1 { threshold } => { let layout = sp_trie::Layout::with_alt_hashing(threshold); TrieDBMut::::new_with_layout( diff --git a/bin/node/testing/src/bench.rs b/bin/node/testing/src/bench.rs index 89cb3a3edfc09..e871604ed5560 100644 --- a/bin/node/testing/src/bench.rs +++ b/bin/node/testing/src/bench.rs @@ -55,7 +55,7 @@ use sp_inherents::InherentData; use sp_runtime::{ generic::BlockId, traits::{Block as BlockT, IdentifyAccount, Verify, Zero}, - OpaqueExtrinsic, StateVersions, StateVersion, + OpaqueExtrinsic, StateVersion, StateVersions, }; /// Keyring full of accounts for benching. @@ -389,7 +389,8 @@ impl BenchDb { let mut state_versions = StateVersions::default(); state_versions.add((BlockNumber::zero(), StateVersion::V0)); // TODO switch to default (more recent). - let backend = sc_service::new_db_backend(db_config, state_versions).expect("Should not fail"); + let backend = + sc_service::new_db_backend(db_config, state_versions).expect("Should not fail"); let client = sc_service::new_client( backend.clone(), NativeElseWasmExecutor::new(WasmExecutionMethod::Compiled, None, 8), diff --git a/client/api/src/in_mem.rs b/client/api/src/in_mem.rs index 8a23853fee4ca..1b170f99dc767 100644 --- a/client/api/src/in_mem.rs +++ b/client/api/src/in_mem.rs @@ -24,9 +24,9 @@ use sp_core::{ offchain::storage::InMemOffchainStorage as OffchainStorage, storage::well_known_keys, }; use sp_runtime::{ - generic::BlockId, StateVersions, StateVersion, + generic::BlockId, traits::{Block as BlockT, HashFor, Header as HeaderT, NumberFor, Zero}, - Justification, Justifications, Storage, + Justification, Justifications, StateVersion, StateVersions, Storage, }; use sp_state_machine::{ Backend as StateBackend, ChangesTrieTransaction, ChildStorageCollection, InMemoryBackend, @@ -150,13 +150,12 @@ impl Blockchain { /// Get version of state. pub fn state_version(&self, id: BlockId) -> StateVersion { let number = match id { - BlockId::Hash(h) => { + BlockId::Hash(h) => if let Ok(Some(header)) = self.header(BlockId::Hash(h)) { header.number().clone() } else { 0u32.into() - } - }, + }, BlockId::Number(n) => n, }; self.storage.read().state_versions.state_version_at(number) diff --git a/client/block-builder/src/lib.rs b/client/block-builder/src/lib.rs index 8a0dbc45f81d7..f381cfd236e7e 100644 --- a/client/block-builder/src/lib.rs +++ b/client/block-builder/src/lib.rs @@ -235,9 +235,7 @@ where ); let state = self.backend.state_at(self.block_id)?; - let proof = { - self.api.extract_proof() - }; + let proof = { self.api.extract_proof() }; let changes_trie_state = backend::changes_tries_state_at_block( &self.block_id, diff --git a/client/chain-spec/src/chain_spec.rs b/client/chain-spec/src/chain_spec.rs index 1a26f80e8f1ba..051d993a32946 100644 --- a/client/chain-spec/src/chain_spec.rs +++ b/client/chain-spec/src/chain_spec.rs @@ -289,7 +289,8 @@ impl ChainSpec { /// Return genesis state version. fn genesis_state_version(&self) -> StateVersion { use std::str::FromStr; - self.state_versions().get(0) + self.state_versions() + .get(0) // This is incorrect (can have number representation not compatible with u64 .and_then(|(n, s)| u64::from_str(n).ok().map(|n| (n, s))) .and_then(|(n, s)| (n == 0).then(|| s.clone())) diff --git a/client/chain-spec/src/lib.rs b/client/chain-spec/src/lib.rs index c64a8909e0aa1..45757e06cbb8e 100644 --- a/client/chain-spec/src/lib.rs +++ b/client/chain-spec/src/lib.rs @@ -119,8 +119,7 @@ pub use sc_chain_spec_derive::{ChainSpecExtension, ChainSpecGroup}; use sc_network::config::MultiaddrWithPeerId; use sc_telemetry::TelemetryEndpoints; use serde::{de::DeserializeOwned, Serialize}; -use sp_core::storage::Storage; -use sp_core::state_version::StateVersion; +use sp_core::{state_version::StateVersion, storage::Storage}; use sp_runtime::BuildStorage; /// The type of a chain. diff --git a/client/db/src/bench.rs b/client/db/src/bench.rs index a312d605b76a0..d605f29ef8bd2 100644 --- a/client/db/src/bench.rs +++ b/client/db/src/bench.rs @@ -34,7 +34,7 @@ use sp_core::{ }; use sp_runtime::{ traits::{Block as BlockT, HashFor}, - Storage, StateVersion, + StateVersion, Storage, }; use sp_state_machine::{ backend::Backend as StateBackend, ChildStorageCollection, DBValue, ProofRecorder, diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index 0ad78789203c0..807cdd0456af5 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -46,7 +46,7 @@ mod upgrade; mod utils; use linked_hash_map::LinkedHashMap; -use log::{debug, trace, warn, info}; +use log::{debug, info, trace, warn}; use parking_lot::{Mutex, RwLock}; use std::{ collections::{HashMap, HashSet}, @@ -88,7 +88,7 @@ use sp_runtime::{ Block as BlockT, Hash, HashFor, Header as HeaderT, NumberFor, One, SaturatedConversion, Zero, }, - Justification, Justifications, Storage, StateVersion, StateVersions, + Justification, Justifications, StateVersion, StateVersions, Storage, }; use sp_state_machine::{ backend::Backend as StateBackend, ChangesTrieCacheAction, ChangesTrieTransaction, @@ -1126,7 +1126,11 @@ impl Backend { /// Create a new instance of database backend. /// /// The pruning window is how old a block must be before the state is pruned. - pub fn new(config: DatabaseSettings, canonicalization_delay: u64, state_versions: StateVersions) -> ClientResult { + pub fn new( + config: DatabaseSettings, + canonicalization_delay: u64, + state_versions: StateVersions, + ) -> ClientResult { // TODO state_versions could also be part of database settings let db = crate::utils::open_database::(&config, DatabaseType::Full)?; Self::from_database(db as Arc<_>, canonicalization_delay, &config, state_versions) @@ -1150,7 +1154,12 @@ impl Backend { transaction_storage: TransactionStorageMode, ) -> Self { let state_versions = Default::default(); - Self::new_test_with_tx_storage_and_state_versions(keep_blocks, canonicalization_delay, transaction_storage, state_versions) + Self::new_test_with_tx_storage_and_state_versions( + keep_blocks, + canonicalization_delay, + transaction_storage, + state_versions, + ) } /// Create new memory-backed client backend for tests. @@ -1172,7 +1181,8 @@ impl Backend { transaction_storage, }; - Self::new(db_setting, canonicalization_delay, state_versions).expect("failed to create test-db") + Self::new(db_setting, canonicalization_delay, state_versions) + .expect("failed to create test-db") } fn from_database( @@ -1891,7 +1901,7 @@ impl Backend { fn empty_state(&self) -> ClientResult, Block>> { let root = EmptyStorage::::new().0; // Empty trie - // state_version for genesis in empty state. + // state_version for genesis in empty state. let state_version = self.state_versions.genesis_state_version(); let db_state = DbState::::new(self.storage.clone(), root, state_version); let state = RefTrackingState::new(db_state, self.storage.clone(), None); @@ -2061,7 +2071,6 @@ impl sc_client_api::backend::Backend for Backend { let usage = operation.old_state.usage_info(); self.state_usage.merge_sm(usage); - if let Some(pending_block) = operation.pending_block.as_mut() { let number = pending_block.header.number().clone(); @@ -2077,36 +2086,39 @@ impl sc_client_api::backend::Backend for Backend { let timer = time::SystemTime::now(); - info!("Starting migrating from state root {:x?} at block {:?}", ¤t_root, &number); + info!( + "Starting migrating from state root {:x?} at block {:?}", + ¤t_root, &number + ); loop { - let sp_state_machine::MigrateProgress { - current_top, - current_child, - root, - } = prev_block_state.migrate( - migration_from, - migration_to, - limit_size, - limit_items, - &mut operation.db_updates, - sp_state_machine::MigrateProgress { - current_top: start_top.take(), - current_child: start_child.take(), - root: Some(current_root), // TODO non optional rather? - } - ).map_err(|err| { - sp_blockchain::Error::Backend(format!( - "error migrating: {}", - err - )) - })?; + let sp_state_machine::MigrateProgress { current_top, current_child, root } = + prev_block_state + .migrate( + migration_from, + migration_to, + limit_size, + limit_items, + &mut operation.db_updates, + sp_state_machine::MigrateProgress { + current_top: start_top.take(), + current_child: start_child.take(), + root: Some(current_root), // TODO non optional rather? + }, + ) + .map_err(|err| { + sp_blockchain::Error::Backend(format!( + "error migrating: {}", + err + )) + })?; info!("Finished migration iteration, new state root: {:x?}, elapsed time: {:?}.", ¤t_root, timer.elapsed()); start_top = current_top; start_child = current_child; - current_root = root.unwrap_or_else(|| pending_block.header.state_root().clone()); + current_root = + root.unwrap_or_else(|| pending_block.header.state_root().clone()); if start_top.is_none() { - break; + break } } @@ -2417,12 +2429,14 @@ impl sc_client_api::backend::Backend for Backend { let (is_genesis, number) = match &block { BlockId::Number(n) => (n.is_zero(), *n), - BlockId::Hash(h) if h == &self.blockchain.meta.read().genesis_hash => { - (true, NumberFor::::zero()) - }, + BlockId::Hash(h) if h == &self.blockchain.meta.read().genesis_hash => + (true, NumberFor::::zero()), BlockId::Hash(h) => { let n = self.blockchain.number(*h)?.ok_or_else(|| { - sp_blockchain::Error::UnknownBlock(format!("Unknown number for block hash {}", h)) + sp_blockchain::Error::UnknownBlock(format!( + "Unknown number for block hash {}", + h + )) })?; (false, n) }, @@ -2666,9 +2680,7 @@ pub(crate) mod tests { } fn set_state_data_inner(alt_hashing: bool) { let state_version = if alt_hashing { - StateVersion::V1 { - threshold: sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD, - } + StateVersion::V1 { threshold: sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD } } else { StateVersion::V0 }; diff --git a/client/db/src/storage_cache.rs b/client/db/src/storage_cache.rs index 586cf465f3da3..e51c26f319b77 100644 --- a/client/db/src/storage_cache.rs +++ b/client/db/src/storage_cache.rs @@ -25,7 +25,7 @@ use hash_db::Hasher; use linked_hash_map::{Entry, LinkedHashMap}; use log::trace; use parking_lot::{RwLock, RwLockUpgradableReadGuard}; -use sp_core::{hexdisplay::HexDisplay, storage::ChildInfo, state_version::StateVersion}; +use sp_core::{hexdisplay::HexDisplay, state_version::StateVersion, storage::ChildInfo}; use sp_runtime::traits::{Block as BlockT, HashFor, Header, NumberFor}; use sp_state_machine::{ backend::Backend as StateBackend, ChildStorageCollection, StorageCollection, StorageKey, @@ -987,12 +987,18 @@ mod tests { true, ); - let mut s = - CachingState::new(InMemoryBackend::::from(state_version), shared.clone(), Some(h0)); + let mut s = CachingState::new( + InMemoryBackend::::from(state_version), + shared.clone(), + Some(h0), + ); s.cache.sync_cache(&[], &[], vec![], vec![], Some(h1a), Some(1), true); - let mut s = - CachingState::new(InMemoryBackend::::from(state_version), shared.clone(), Some(h0)); + let mut s = CachingState::new( + InMemoryBackend::::from(state_version), + shared.clone(), + Some(h0), + ); s.cache.sync_cache( &[], &[], @@ -1003,8 +1009,11 @@ mod tests { false, ); - let mut s = - CachingState::new(InMemoryBackend::::from(state_version), shared.clone(), Some(h1b)); + let mut s = CachingState::new( + InMemoryBackend::::from(state_version), + shared.clone(), + Some(h1b), + ); s.cache.sync_cache( &[], &[], @@ -1015,8 +1024,11 @@ mod tests { false, ); - let mut s = - CachingState::new(InMemoryBackend::::from(state_version), shared.clone(), Some(h1a)); + let mut s = CachingState::new( + InMemoryBackend::::from(state_version), + shared.clone(), + Some(h1a), + ); s.cache.sync_cache( &[], &[], @@ -1027,30 +1039,48 @@ mod tests { true, ); - let mut s = - CachingState::new(InMemoryBackend::::from(state_version), shared.clone(), Some(h2a)); + let mut s = CachingState::new( + InMemoryBackend::::from(state_version), + shared.clone(), + Some(h2a), + ); s.cache.sync_cache(&[], &[], vec![], vec![], Some(h3a), Some(3), true); - let s = - CachingState::new(InMemoryBackend::::from(state_version), shared.clone(), Some(h3a)); + let s = CachingState::new( + InMemoryBackend::::from(state_version), + shared.clone(), + Some(h3a), + ); assert_eq!(s.storage(&key).unwrap().unwrap(), vec![5]); - let s = - CachingState::new(InMemoryBackend::::from(state_version), shared.clone(), Some(h1a)); + let s = CachingState::new( + InMemoryBackend::::from(state_version), + shared.clone(), + Some(h1a), + ); assert!(s.storage(&key).unwrap().is_none()); - let s = - CachingState::new(InMemoryBackend::::from(state_version), shared.clone(), Some(h2b)); + let s = CachingState::new( + InMemoryBackend::::from(state_version), + shared.clone(), + Some(h2b), + ); assert!(s.storage(&key).unwrap().is_none()); - let s = - CachingState::new(InMemoryBackend::::from(state_version), shared.clone(), Some(h1b)); + let s = CachingState::new( + InMemoryBackend::::from(state_version), + shared.clone(), + Some(h1b), + ); assert!(s.storage(&key).unwrap().is_none()); // reorg to 3b // blocks [ 3b(c) 3a 2a 2b(c) 1b 1a 0 ] - let mut s = - CachingState::new(InMemoryBackend::::from(state_version), shared.clone(), Some(h2b)); + let mut s = CachingState::new( + InMemoryBackend::::from(state_version), + shared.clone(), + Some(h2b), + ); s.cache.sync_cache( &[h1b, h2b, h3b], &[h1a, h2a, h3a], @@ -1060,8 +1090,11 @@ mod tests { Some(3), true, ); - let s = - CachingState::new(InMemoryBackend::::from(state_version), shared.clone(), Some(h3a)); + let s = CachingState::new( + InMemoryBackend::::from(state_version), + shared.clone(), + Some(h3a), + ); assert!(s.storage(&key).unwrap().is_none()); } @@ -1094,12 +1127,18 @@ mod tests { true, ); - let mut s = - CachingState::new(InMemoryBackend::::from(state_version), shared.clone(), Some(h1)); + let mut s = CachingState::new( + InMemoryBackend::::from(state_version), + shared.clone(), + Some(h1), + ); s.cache.sync_cache(&[], &[], vec![], vec![], Some(h2a), Some(2), true); - let mut s = - CachingState::new(InMemoryBackend::::from(state_version), shared.clone(), Some(h1)); + let mut s = CachingState::new( + InMemoryBackend::::from(state_version), + shared.clone(), + Some(h1), + ); s.cache.sync_cache( &[], &[], @@ -1110,8 +1149,11 @@ mod tests { false, ); - let mut s = - CachingState::new(InMemoryBackend::::from(state_version), shared.clone(), Some(h2b)); + let mut s = CachingState::new( + InMemoryBackend::::from(state_version), + shared.clone(), + Some(h2b), + ); s.cache.sync_cache( &[], &[], @@ -1122,8 +1164,11 @@ mod tests { false, ); - let s = - CachingState::new(InMemoryBackend::::from(state_version), shared.clone(), Some(h2a)); + let s = CachingState::new( + InMemoryBackend::::from(state_version), + shared.clone(), + Some(h2a), + ); assert_eq!(s.storage(&key).unwrap().unwrap(), vec![2]); } @@ -1147,12 +1192,18 @@ mod tests { ); s.cache.sync_cache(&[], &[], vec![], vec![], Some(h1), Some(1), true); - let mut s = - CachingState::new(InMemoryBackend::::from(state_version), shared.clone(), Some(h1)); + let mut s = CachingState::new( + InMemoryBackend::::from(state_version), + shared.clone(), + Some(h1), + ); s.cache.sync_cache(&[], &[], vec![], vec![], Some(h2a), Some(2), true); - let mut s = - CachingState::new(InMemoryBackend::::from(state_version), shared.clone(), Some(h2a)); + let mut s = CachingState::new( + InMemoryBackend::::from(state_version), + shared.clone(), + Some(h2a), + ); s.cache.sync_cache( &[], &[], @@ -1163,12 +1214,18 @@ mod tests { true, ); - let mut s = - CachingState::new(InMemoryBackend::::from(state_version), shared.clone(), Some(h1)); + let mut s = CachingState::new( + InMemoryBackend::::from(state_version), + shared.clone(), + Some(h1), + ); s.cache.sync_cache(&[], &[], vec![], vec![], Some(h2b), Some(2), false); - let mut s = - CachingState::new(InMemoryBackend::::from(state_version), shared.clone(), Some(h2b)); + let mut s = CachingState::new( + InMemoryBackend::::from(state_version), + shared.clone(), + Some(h2b), + ); s.cache.sync_cache( &[], &[], @@ -1179,8 +1236,11 @@ mod tests { false, ); - let s = - CachingState::new(InMemoryBackend::::from(state_version), shared.clone(), Some(h3a)); + let s = CachingState::new( + InMemoryBackend::::from(state_version), + shared.clone(), + Some(h3a), + ); assert_eq!(s.storage(&key).unwrap().unwrap(), vec![2]); } @@ -1319,8 +1379,11 @@ mod tests { true, ); - let mut s = - CachingState::new(InMemoryBackend::::from(state_version), shared.clone(), Some(h0)); + let mut s = CachingState::new( + InMemoryBackend::::from(state_version), + shared.clone(), + Some(h0), + ); s.cache.sync_cache( &[], &[], @@ -1331,8 +1394,11 @@ mod tests { true, ); - let mut s = - CachingState::new(InMemoryBackend::::from(state_version), shared.clone(), Some(h1)); + let mut s = CachingState::new( + InMemoryBackend::::from(state_version), + shared.clone(), + Some(h1), + ); assert_eq!(s.storage(&key).unwrap(), Some(vec![3])); // Restart (or unknown block?), clear caches. @@ -1351,8 +1417,11 @@ mod tests { // New value is propagated. s.cache.sync_cache(&[], &[], vec![], vec![], None, None, true); - let s = - CachingState::new(InMemoryBackend::::from(state_version), shared.clone(), Some(h1)); + let s = CachingState::new( + InMemoryBackend::::from(state_version), + shared.clone(), + Some(h1), + ); assert_eq!(s.storage(&key).unwrap(), None); } @@ -1384,8 +1453,11 @@ mod tests { ); assert_eq!(shared.write().lru_storage.get(&key).unwrap(), &Some(vec![1])); - let mut s = - CachingState::new(InMemoryBackend::::from(state_version), shared.clone(), Some(h1)); + let mut s = CachingState::new( + InMemoryBackend::::from(state_version), + shared.clone(), + Some(h1), + ); // commit as non-best s.cache.sync_cache( @@ -1400,8 +1472,11 @@ mod tests { assert_eq!(shared.write().lru_storage.get(&key).unwrap(), &Some(vec![1])); - let mut s = - CachingState::new(InMemoryBackend::::from(state_version), shared.clone(), Some(h1)); + let mut s = CachingState::new( + InMemoryBackend::::from(state_version), + shared.clone(), + Some(h1), + ); // commit again as best with no changes s.cache.sync_cache(&[], &[], vec![], vec![], Some(h2), Some(2), true); diff --git a/client/light/src/backend.rs b/client/light/src/backend.rs index a859be87d2f26..481d41c65f026 100644 --- a/client/light/src/backend.rs +++ b/client/light/src/backend.rs @@ -48,7 +48,7 @@ use sp_core::{ use sp_runtime::{ generic::BlockId, traits::{Block as BlockT, HashFor, Header, NumberFor, Zero}, - Justification, Justifications, Storage, StateVersions, StateVersion, + Justification, Justifications, StateVersion, StateVersions, Storage, }; use sp_state_machine::{ Backend as StateBackend, ChangesTrieTransaction, ChildStorageCollection, InMemoryBackend, @@ -95,7 +95,12 @@ pub enum GenesisOrUnavailableState { impl Backend { /// Create new light backend. pub fn new(blockchain: Arc>, state_versions: StateVersions) -> Self { - Self { blockchain, genesis_state: RwLock::new(None), import_lock: Default::default(), state_versions } + Self { + blockchain, + genesis_state: RwLock::new(None), + import_lock: Default::default(), + state_versions, + } } /// Get shared blockchain reference. @@ -156,7 +161,6 @@ where operation: &mut Self::BlockImportOperation, block: BlockId, ) -> ClientResult<()> { - if let Some(number) = self.blockchain.storage().block_number_from_id(&block)? { operation.state_version = Some(self.state_versions.state_version_at(number)); } diff --git a/client/light/src/fetcher.rs b/client/light/src/fetcher.rs index be041b6c82989..ca346f019d2b2 100644 --- a/client/light/src/fetcher.rs +++ b/client/light/src/fetcher.rs @@ -28,7 +28,8 @@ use codec::{Decode, Encode}; use hash_db::{HashDB, Hasher, EMPTY_PREFIX}; use sp_blockchain::{Error as ClientError, Result as ClientResult}; use sp_core::{ - convert_hash, state_version::StateVersion, + convert_hash, + state_version::StateVersion, storage::{ChildInfo, ChildType}, traits::{CodeExecutor, SpawnNamed}, }; @@ -202,7 +203,8 @@ impl> LightDataChecker { } // check proof for single changes trie root - let proving_backend = TrieBackend::new(storage, local_cht_root, StateVersion::V0); + let proving_backend = + TrieBackend::new(storage, local_cht_root, StateVersion::V0); let remote_changes_trie_root = remote_roots[&block]; cht::check_proof_on_proving_backend::>( local_cht_root, diff --git a/client/light/src/lib.rs b/client/light/src/lib.rs index 83e1371b69a61..ca71a94496a44 100644 --- a/client/light/src/lib.rs +++ b/client/light/src/lib.rs @@ -50,7 +50,10 @@ pub fn new_light_blockchain>(storage: S) -> A } /// Create an instance of light client backend. -pub fn new_light_backend(blockchain: Arc>, state_versions: sp_runtime::StateVersions) -> Arc> +pub fn new_light_backend( + blockchain: Arc>, + state_versions: sp_runtime::StateVersions, +) -> Arc> where B: BlockT, S: BlockchainStorage, diff --git a/client/network/test/src/lib.rs b/client/network/test/src/lib.rs index b5d9f0e544a54..3bdee3a491ab2 100644 --- a/client/network/test/src/lib.rs +++ b/client/network/test/src/lib.rs @@ -734,12 +734,16 @@ where fn add_full_peer_with_config(&mut self, config: FullPeerConfig) { let mut test_client_builder = match (config.keep_blocks, config.storage_chain) { (Some(keep_blocks), true) => - TestClientBuilder::with_tx_storage(keep_blocks, self.state_versions()).state_hashed_value(), - (None, true) => TestClientBuilder::with_tx_storage(u32::MAX, self.state_versions()).state_hashed_value(), + TestClientBuilder::with_tx_storage(keep_blocks, self.state_versions()) + .state_hashed_value(), + (None, true) => TestClientBuilder::with_tx_storage(u32::MAX, self.state_versions()) + .state_hashed_value(), (Some(keep_blocks), false) => - TestClientBuilder::with_pruning_window(keep_blocks, self.state_versions()).state_hashed_value(), + TestClientBuilder::with_pruning_window(keep_blocks, self.state_versions()) + .state_hashed_value(), (None, false) => - TestClientBuilder::with_default_backend_and_state_versions(self.state_versions()).state_hashed_value(), + TestClientBuilder::with_default_backend_and_state_versions(self.state_versions()) + .state_hashed_value(), }; if matches!(config.sync_mode, SyncMode::Fast { .. }) { test_client_builder = test_client_builder.set_no_genesis(); @@ -1092,7 +1096,10 @@ impl TestNet { } /// Create new test network with this many peers. - pub fn new_with_state_versions(n: usize, state_versions: sp_runtime::StateVersions) -> Self { + pub fn new_with_state_versions( + n: usize, + state_versions: sp_runtime::StateVersions, + ) -> Self { trace!(target: "test_network", "Creating test network with peer config"); let config = Self::default_config(); let mut net = Self::from_config(&config); @@ -1113,7 +1120,11 @@ impl TestNetFactory for TestNet { /// Create new test network with peers and given config. fn from_config(_config: &ProtocolConfig) -> Self { - TestNet { peers: Vec::new(), fork_choice: ForkChoiceStrategy::LongestChain, state_versions: None } + TestNet { + peers: Vec::new(), + fork_choice: ForkChoiceStrategy::LongestChain, + state_versions: None, + } } fn make_verifier( diff --git a/client/service/src/builder.rs b/client/service/src/builder.rs index b6d0db1f01aea..b854709bf750e 100644 --- a/client/service/src/builder.rs +++ b/client/service/src/builder.rs @@ -59,8 +59,7 @@ use sp_keystore::{CryptoStore, SyncCryptoStore, SyncCryptoStorePtr}; use sp_runtime::{ generic::BlockId, traits::{Block as BlockT, BlockIdTo, Zero}, - BuildStorage, - StateVersions, + BuildStorage, StateVersions, }; use sp_utils::mpsc::{tracing_unbounded, TracingUnboundedSender}; use std::{str::FromStr, sync::Arc, time::SystemTime}; @@ -144,8 +143,7 @@ pub type TLightClient = TLightClientWithBackend>; /// Light client backend type. -pub type TLightBackend = - sc_light::Backend, TBl>; +pub type TLightBackend = sc_light::Backend, TBl>; /// Light call executor type. pub type TLightCallExecutor = sc_light::GenesisCallExecutor< @@ -310,9 +308,16 @@ where transaction_storage: config.transaction_storage.clone(), }; - let state_versions = StateVersions::from_conf(config.chain_spec.state_versions() - .iter().map(|(number, version)| (number.as_str(), *version))) - .ok_or_else(|| Error::Application(Box::from("Invalid state versions for chain spec".to_string())))?; + let state_versions = StateVersions::from_conf( + config + .chain_spec + .state_versions() + .iter() + .map(|(number, version)| (number.as_str(), *version)), + ) + .ok_or_else(|| { + Error::Application(Box::from("Invalid state versions for chain spec".to_string())) + })?; let backend = new_db_backend(db_config, state_versions.clone())?; @@ -401,9 +406,16 @@ where )); let on_demand = Arc::new(sc_network::config::OnDemand::new(fetch_checker)); - let state_versions = StateVersions::from_conf(config.chain_spec.state_versions() - .iter().map(|(number, version)| (number.as_str(), *version))) - .ok_or_else(|| Error::Application(Box::from("Invalid state versions for chain spec".to_string())))?; + let state_versions = StateVersions::from_conf( + config + .chain_spec + .state_versions() + .iter() + .map(|(number, version)| (number.as_str(), *version)), + ) + .ok_or_else(|| { + Error::Application(Box::from("Invalid state versions for chain spec".to_string())) + })?; let backend = sc_light::new_light_backend(light_blockchain, state_versions); let client = Arc::new(light::new_light( diff --git a/client/service/src/client/light.rs b/client/service/src/client/light.rs index 909b7083d3a5e..e50c41aa3acba 100644 --- a/client/service/src/client/light.rs +++ b/client/service/src/client/light.rs @@ -25,10 +25,7 @@ use sc_executor::RuntimeVersionOf; use sc_telemetry::TelemetryHandle; use sp_blockchain::Result as ClientResult; use sp_core::traits::{CodeExecutor, SpawnNamed}; -use sp_runtime::{ - traits::Block as BlockT, - BuildStorage, -}; +use sp_runtime::{traits::Block as BlockT, BuildStorage}; use super::{ call_executor::LocalCallExecutor, @@ -48,10 +45,7 @@ pub fn new_light( ) -> ClientResult< Client< Backend, - GenesisCallExecutor< - Backend, - LocalCallExecutor, E>, - >, + GenesisCallExecutor, LocalCallExecutor, E>>, B, RA, >, diff --git a/client/service/test/src/client/light.rs b/client/service/test/src/client/light.rs index 4cfc6ae20a6e2..6aebc83acbd3b 100644 --- a/client/service/test/src/client/light.rs +++ b/client/service/test/src/client/light.rs @@ -295,7 +295,8 @@ fn unavailable_state_is_created_when_genesis_state_is_unavailable() { #[test] fn light_aux_store_is_updated_via_non_importing_op() { - let backend = Backend::new(Arc::new(DummyBlockchain::new(DummyStorage::new())), Default::default()); + let backend = + Backend::new(Arc::new(DummyBlockchain::new(DummyStorage::new())), Default::default()); let mut op = ClientBackend::::begin_operation(&backend).unwrap(); BlockImportOperation::::insert_aux(&mut op, vec![(vec![1], Some(vec![2]))]).unwrap(); ClientBackend::::commit_operation(&backend, op).unwrap(); diff --git a/client/service/test/src/client/mod.rs b/client/service/test/src/client/mod.rs index 9060cb19ea918..55f7d8b5f5040 100644 --- a/client/service/test/src/client/mod.rs +++ b/client/service/test/src/client/mod.rs @@ -2069,14 +2069,14 @@ fn storage_keys_iter_works_inner(hashed_value: bool) { .take(3) .map(|x| x.0) .collect(); - assert_eq!( - res, - [ - hex!("3a686561707061676573").to_vec(), - hex!("6644b9b8bc315888ac8e41a7968dc2b4141a5403c58acdf70b7e8f7e07bf5081").to_vec(), - hex!("79c07e2b1d2e2abfd4855b936617eeff5e0621c4869aa60c02be9adcc98a0d1d").to_vec(), - ] - ); + assert_eq!( + res, + [ + hex!("3a686561707061676573").to_vec(), + hex!("6644b9b8bc315888ac8e41a7968dc2b4141a5403c58acdf70b7e8f7e07bf5081").to_vec(), + hex!("79c07e2b1d2e2abfd4855b936617eeff5e0621c4869aa60c02be9adcc98a0d1d").to_vec(), + ] + ); let res: Vec<_> = client .storage_keys_iter( diff --git a/primitives/core/src/lib.rs b/primitives/core/src/lib.rs index df58b972d80ba..20087c459a59c 100644 --- a/primitives/core/src/lib.rs +++ b/primitives/core/src/lib.rs @@ -66,11 +66,11 @@ mod hasher; pub mod offchain; pub mod sandbox; pub mod sr25519; +pub mod state_version; pub mod testing; #[cfg(feature = "std")] pub mod traits; pub mod uint; -pub mod state_version; pub use self::{ hash::{convert_hash, H160, H256, H512}, diff --git a/primitives/core/src/state_version.rs b/primitives/core/src/state_version.rs index 20eeb2565472a..ebb6db3e51d3f 100644 --- a/primitives/core/src/state_version.rs +++ b/primitives/core/src/state_version.rs @@ -25,7 +25,8 @@ use codec::{Decode, Encode}; /// initial state versionning in their chainspec for block 0. /// Therefore defining genesis version in chainspec is good practice /// and this default should mostly be use when testing. -pub const DEFAULT_STATE_VERSION: StateVersion = StateVersion::V1 { threshold: crate::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD }; +pub const DEFAULT_STATE_VERSION: StateVersion = + StateVersion::V1 { threshold: crate::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD }; /// Supported version with substrate chain. #[derive(Debug, Clone, Copy, PartialEq, Eq, Encode, Decode)] diff --git a/primitives/io/src/lib.rs b/primitives/io/src/lib.rs index 342823dccb855..bddebaa942610 100644 --- a/primitives/io/src/lib.rs +++ b/primitives/io/src/lib.rs @@ -51,7 +51,9 @@ use sp_core::{ offchain::{ HttpError, HttpRequestId, HttpRequestStatus, OpaqueNetworkState, StorageKind, Timestamp, }, - sr25519, LogLevel, LogLevelFilter, OpaquePeerId, H256, state_version::StateVersion, + sr25519, + state_version::StateVersion, + LogLevel, LogLevelFilter, OpaquePeerId, H256, }; #[cfg(feature = "std")] @@ -1542,10 +1544,10 @@ mod tests { storage::set(b"foo", &[1, 2, 3][..]); }); - t = BasicExternalities::new(Storage { - top: map![b"foo".to_vec() => b"bar".to_vec()], - children_default: map![], - }, state_version); + t = BasicExternalities::new( + Storage { top: map![b"foo".to_vec() => b"bar".to_vec()], children_default: map![] }, + state_version, + ); t.execute_with(|| { assert_eq!(storage::get(b"hello"), None); @@ -1567,10 +1569,10 @@ mod tests { fn read_storage_works() { let state_version = StateVersion::default(); let value = b"\x0b\0\0\0Hello world".to_vec(); - let mut t = BasicExternalities::new(Storage { - top: map![b":test".to_vec() => value.clone()], - children_default: map![], - }, state_version); + let mut t = BasicExternalities::new( + Storage { top: map![b":test".to_vec() => value.clone()], children_default: map![] }, + state_version, + ); t.execute_with(|| { let mut v = [0u8; 4]; @@ -1585,15 +1587,18 @@ mod tests { #[test] fn clear_prefix_works() { let state_version = StateVersion::default(); - let mut t = BasicExternalities::new(Storage { - top: map![ - b":a".to_vec() => b"\x0b\0\0\0Hello world".to_vec(), - b":abcd".to_vec() => b"\x0b\0\0\0Hello world".to_vec(), - b":abc".to_vec() => b"\x0b\0\0\0Hello world".to_vec(), - b":abdd".to_vec() => b"\x0b\0\0\0Hello world".to_vec() - ], - children_default: map![], - }, state_version); + let mut t = BasicExternalities::new( + Storage { + top: map![ + b":a".to_vec() => b"\x0b\0\0\0Hello world".to_vec(), + b":abcd".to_vec() => b"\x0b\0\0\0Hello world".to_vec(), + b":abc".to_vec() => b"\x0b\0\0\0Hello world".to_vec(), + b":abdd".to_vec() => b"\x0b\0\0\0Hello world".to_vec() + ], + children_default: map![], + }, + state_version, + ); t.execute_with(|| { assert!(matches!( diff --git a/primitives/runtime/src/lib.rs b/primitives/runtime/src/lib.rs index 264b839e59c59..ee7871342b975 100644 --- a/primitives/runtime/src/lib.rs +++ b/primitives/runtime/src/lib.rs @@ -57,11 +57,11 @@ mod multiaddress; pub mod offchain; pub mod runtime_logger; mod runtime_string; +pub mod state_version; #[cfg(feature = "std")] pub mod testing; pub mod traits; pub mod transaction_validity; -pub mod state_version; pub use crate::runtime_string::*; diff --git a/primitives/runtime/src/state_version.rs b/primitives/runtime/src/state_version.rs index 714f7c7390b2e..e6bf1ba56fecb 100644 --- a/primitives/runtime/src/state_version.rs +++ b/primitives/runtime/src/state_version.rs @@ -17,11 +17,10 @@ //! Substrate state versioning and migrations related types. -pub use sp_core::state_version::{StateVersion, DEFAULT_STATE_VERSION}; use crate::traits::{Block, NumberFor}; -use sp_std::str::FromStr; -use sp_std::vec::Vec; use sp_arithmetic::traits::Zero; +pub use sp_core::state_version::{StateVersion, DEFAULT_STATE_VERSION}; +use sp_std::{str::FromStr, vec::Vec}; /// Multiple versions of state in use for a chain. #[derive(Clone, crate::RuntimeDebug)] @@ -31,9 +30,7 @@ pub struct StateVersions { impl Default for StateVersions { fn default() -> Self { - StateVersions { - canonical_states: Vec::new(), - } + StateVersions { canonical_states: Vec::new() } } } @@ -43,7 +40,7 @@ impl StateVersions { pub fn genesis_state_version(&self) -> StateVersion { if let Some((number, version)) = self.canonical_states.get(0) { if number.is_zero() { - return *version; + return *version } } DEFAULT_STATE_VERSION @@ -55,7 +52,7 @@ impl StateVersions { let mut version = DEFAULT_STATE_VERSION; for (number, state) in self.canonical_states.iter() { if number > &at { - break; + break } version = *state; } @@ -69,10 +66,10 @@ impl StateVersions { for (i, (number, _)) in self.canonical_states.iter().enumerate() { if number == &at { replace = Some(i); - break; + break } if number > &at { - break; + break } insert = Some(i + 1); } @@ -85,7 +82,8 @@ impl StateVersions { /// Convert from chainspec conf. pub fn from_conf<'a, I>(conf: I) -> Option - where I: IntoIterator, + where + I: IntoIterator, { let iter = conf.into_iter(); let mut canonical_states = match iter.size_hint() { @@ -97,15 +95,12 @@ impl StateVersions { if let Ok(number) = NumberFor::::from_str(number) { canonical_states.push((number.into(), version)); } else { - return None; + return None } } - Some(StateVersions { - canonical_states, - }) + Some(StateVersions { canonical_states }) } - /// Indicate if block need migration. /// Returns version to migrate from and to if /// migration is needed. @@ -117,10 +112,10 @@ impl StateVersions { for (number, version) in self.canonical_states.iter() { if number == &at { to = Some(*version); - break; + break } if number > &at { - break; + break } from = *version; } diff --git a/primitives/state-machine/src/basic.rs b/primitives/state-machine/src/basic.rs index 3786879d1ed6c..7cbabd46e6387 100644 --- a/primitives/state-machine/src/basic.rs +++ b/primitives/state-machine/src/basic.rs @@ -22,12 +22,12 @@ use codec::Encode; use hash_db::Hasher; use log::warn; use sp_core::{ + state_version::StateVersion, storage::{ well_known_keys::is_child_storage_key, ChildInfo, Storage, StorageChild, TrackedStorageKey, }, traits::Externalities, Blake2Hasher, - state_version::StateVersion, }; use sp_externalities::{Extension, Extensions}; use sp_trie::{empty_child_trie_root, Layout, TrieConfiguration}; @@ -48,7 +48,11 @@ pub struct BasicExternalities { impl BasicExternalities { /// Create a new instance of `BasicExternalities` pub fn new(inner: Storage, state_version: StateVersion) -> Self { - BasicExternalities { inner, extensions: Default::default(), state_version: Some(state_version) } + BasicExternalities { + inner, + extensions: Default::default(), + state_version: Some(state_version), + } } /// New basic externalities with empty storage. @@ -78,7 +82,7 @@ impl BasicExternalities { ) -> R { Self::execute_with_storage_inner(storage, Some(state_version), f) } - + /// Execute the given closure `f` with the externalities set and initialized with `storage`. /// /// Returns the result of the closure and updates `storage` with all changes. @@ -313,19 +317,20 @@ impl Externalities for BasicExternalities { } } - let layout = match self.state_version.expect("Unsupported state calculation for genesis storage build.") { - StateVersion::V0 => { - Layout::::default() - }, - StateVersion::V1 { threshold } => { - Layout::::with_alt_hashing(threshold) - }, + let layout = match self + .state_version + .expect("Unsupported state calculation for genesis storage build.") + { + StateVersion::V0 => Layout::::default(), + StateVersion::V1 { threshold } => Layout::::with_alt_hashing(threshold), }; layout.trie_root(self.inner.top.clone()).as_ref().into() } fn child_storage_root(&mut self, child_info: &ChildInfo) -> Vec { - let state_version = self.state_version.expect("Unsupported state calculation for genesis storage build."); + let state_version = self + .state_version + .expect("Unsupported state calculation for genesis storage build."); if let Some(child) = self.inner.children_default.get(child_info.storage_key()) { let delta = child.data.iter().map(|(k, v)| (k.as_ref(), Some(v.as_ref()))); let in_mem = crate::in_memory_backend::new_in_mem::(state_version); @@ -437,15 +442,18 @@ mod tests { fn children_works() { let child_info = ChildInfo::new_default(b"storage_key"); let child_info = &child_info; - let mut ext = BasicExternalities::new(Storage { - top: Default::default(), - children_default: map![ - child_info.storage_key().to_vec() => StorageChild { - data: map![ b"doe".to_vec() => b"reindeer".to_vec() ], - child_info: child_info.to_owned(), - } - ], - }, StateVersion::default()); + let mut ext = BasicExternalities::new( + Storage { + top: Default::default(), + children_default: map![ + child_info.storage_key().to_vec() => StorageChild { + data: map![ b"doe".to_vec() => b"reindeer".to_vec() ], + child_info: child_info.to_owned(), + } + ], + }, + StateVersion::default(), + ); assert_eq!(ext.child_storage(child_info, b"doe"), Some(b"reindeer".to_vec())); @@ -463,19 +471,22 @@ mod tests { fn kill_child_storage_returns_num_elements_removed() { let child_info = ChildInfo::new_default(b"storage_key"); let child_info = &child_info; - let mut ext = BasicExternalities::new(Storage { - top: Default::default(), - children_default: map![ - child_info.storage_key().to_vec() => StorageChild { - data: map![ - b"doe".to_vec() => b"reindeer".to_vec(), - b"dog".to_vec() => b"puppy".to_vec(), - b"hello".to_vec() => b"world".to_vec(), - ], - child_info: child_info.to_owned(), - } - ], - }, StateVersion::default()); + let mut ext = BasicExternalities::new( + Storage { + top: Default::default(), + children_default: map![ + child_info.storage_key().to_vec() => StorageChild { + data: map![ + b"doe".to_vec() => b"reindeer".to_vec(), + b"dog".to_vec() => b"puppy".to_vec(), + b"hello".to_vec() => b"world".to_vec(), + ], + child_info: child_info.to_owned(), + } + ], + }, + StateVersion::default(), + ); let res = ext.kill_child_storage(child_info, None); assert_eq!(res, (true, 3)); diff --git a/primitives/state-machine/src/changes_trie/build.rs b/primitives/state-machine/src/changes_trie/build.rs index 1377ab36084b4..37f9d964a0ce3 100644 --- a/primitives/state-machine/src/changes_trie/build.rs +++ b/primitives/state-machine/src/changes_trie/build.rs @@ -382,17 +382,20 @@ mod test { ) { let child_info_1 = ChildInfo::new_default(b"storage_key1"); let child_info_2 = ChildInfo::new_default(b"storage_key2"); - let backend: InMemoryBackend<_> = (vec![ - (vec![100], vec![255]), - (vec![101], vec![255]), - (vec![102], vec![255]), - (vec![103], vec![255]), - (vec![104], vec![255]), - (vec![105], vec![255]), - ] - .into_iter() - .collect::>(), sp_core::state_version::StateVersion::V0) - .into(); + let backend: InMemoryBackend<_> = ( + vec![ + (vec![100], vec![255]), + (vec![101], vec![255]), + (vec![102], vec![255]), + (vec![103], vec![255]), + (vec![104], vec![255]), + (vec![105], vec![255]), + ] + .into_iter() + .collect::>(), + sp_core::state_version::StateVersion::V0, + ) + .into(); let prefixed_child_trie_key1 = child_info_1.prefixed_storage_key(); let storage = InMemoryStorage::with_inputs( vec![ diff --git a/primitives/state-machine/src/ext.rs b/primitives/state-machine/src/ext.rs index 769d0968831f5..c07a942f685d8 100644 --- a/primitives/state-machine/src/ext.rs +++ b/primitives/state-machine/src/ext.rs @@ -950,7 +950,8 @@ mod tests { use hex_literal::hex; use num_traits::Zero; use sp_core::{ - map, state_version::StateVersion, + map, + state_version::StateVersion, storage::{well_known_keys::EXTRINSIC_INDEX, Storage, StorageChild}, Blake2Hasher, H256, }; @@ -1027,15 +1028,18 @@ mod tests { let mut overlay = OverlayedChanges::default(); overlay.set_storage(vec![20], None); overlay.set_storage(vec![30], Some(vec![31])); - let backend = (Storage { - top: map![ - vec![10] => vec![10], - vec![20] => vec![20], - vec![40] => vec![40] - ], - children_default: map![], - }, Default::default()) - .into(); + let backend = ( + Storage { + top: map![ + vec![10] => vec![10], + vec![20] => vec![20], + vec![40] => vec![40] + ], + children_default: map![], + }, + Default::default(), + ) + .into(); let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); @@ -1073,13 +1077,16 @@ mod tests { overlay.set_storage(vec![27], None); overlay.set_storage(vec![28], None); overlay.set_storage(vec![29], None); - let backend = (Storage { - top: map![ - vec![30] => vec![30] - ], - children_default: map![], - }, Default::default()) - .into(); + let backend = ( + Storage { + top: map![ + vec![30] => vec![30] + ], + children_default: map![], + }, + Default::default(), + ) + .into(); let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); @@ -1097,20 +1104,23 @@ mod tests { let mut overlay = OverlayedChanges::default(); overlay.set_child_storage(child_info, vec![20], None); overlay.set_child_storage(child_info, vec![30], Some(vec![31])); - let backend = (Storage { - top: map![], - children_default: map![ - child_info.storage_key().to_vec() => StorageChild { - data: map![ - vec![10] => vec![10], - vec![20] => vec![20], - vec![40] => vec![40] - ], - child_info: child_info.to_owned(), - } - ], - }, Default::default()) - .into(); + let backend = ( + Storage { + top: map![], + children_default: map![ + child_info.storage_key().to_vec() => StorageChild { + data: map![ + vec![10] => vec![10], + vec![20] => vec![20], + vec![40] => vec![40] + ], + child_info: child_info.to_owned(), + } + ], + }, + Default::default(), + ) + .into(); let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); @@ -1142,20 +1152,23 @@ mod tests { let mut overlay = OverlayedChanges::default(); overlay.set_child_storage(child_info, vec![20], None); overlay.set_child_storage(child_info, vec![30], Some(vec![31])); - let backend = (Storage { - top: map![], - children_default: map![ - child_info.storage_key().to_vec() => StorageChild { - data: map![ - vec![10] => vec![10], - vec![20] => vec![20], - vec![30] => vec![40] - ], - child_info: child_info.to_owned(), - } - ], - }, Default::default()) - .into(); + let backend = ( + Storage { + top: map![], + children_default: map![ + child_info.storage_key().to_vec() => StorageChild { + data: map![ + vec![10] => vec![10], + vec![20] => vec![20], + vec![30] => vec![40] + ], + child_info: child_info.to_owned(), + } + ], + }, + Default::default(), + ) + .into(); let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); @@ -1181,18 +1194,21 @@ mod tests { let child_info = &child_info; let mut cache = StorageTransactionCache::default(); let mut overlay = OverlayedChanges::default(); - let backend = (Storage { - top: map![], - children_default: map![ - child_info.storage_key().to_vec() => StorageChild { - data: map![ - vec![30] => vec![40] - ], - child_info: child_info.to_owned(), - } - ], - }, Default::default()) - .into(); + let backend = ( + Storage { + top: map![], + children_default: map![ + child_info.storage_key().to_vec() => StorageChild { + data: map![ + vec![30] => vec![40] + ], + child_info: child_info.to_owned(), + } + ], + }, + Default::default(), + ) + .into(); let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); diff --git a/primitives/state-machine/src/in_memory_backend.rs b/primitives/state-machine/src/in_memory_backend.rs index cb18c2fa58e45..15161094eaf22 100644 --- a/primitives/state-machine/src/in_memory_backend.rs +++ b/primitives/state-machine/src/in_memory_backend.rs @@ -22,8 +22,10 @@ use crate::{ }; use codec::Codec; use hash_db::Hasher; -use sp_core::storage::{ChildInfo, Storage}; -use sp_core::state_version::StateVersion; +use sp_core::{ + state_version::StateVersion, + storage::{ChildInfo, Storage}, +}; use sp_trie::{empty_trie_root, Layout, MemoryDB}; use std::collections::{BTreeMap, HashMap}; @@ -103,8 +105,7 @@ where } } -impl From - for TrieBackend, H> +impl From for TrieBackend, H> where H::Out: Codec + Ord, { @@ -112,13 +113,18 @@ where new_in_mem(state_version) } } - + impl From<(HashMap, BTreeMap>, StateVersion)> for TrieBackend, H> where H::Out: Codec + Ord, { - fn from((inner, state_version): (HashMap, BTreeMap>, StateVersion)) -> Self { + fn from( + (inner, state_version): ( + HashMap, BTreeMap>, + StateVersion, + ), + ) -> Self { let mut backend = new_in_mem(state_version); backend.insert( inner @@ -144,7 +150,8 @@ where } } -impl From<(BTreeMap, StateVersion)> for TrieBackend, H> +impl From<(BTreeMap, StateVersion)> + for TrieBackend, H> where H::Out: Codec + Ord, { @@ -155,11 +162,14 @@ where } } -impl From<(Vec<(Option, StorageCollection)>, StateVersion)> for TrieBackend, H> +impl From<(Vec<(Option, StorageCollection)>, StateVersion)> + for TrieBackend, H> where H::Out: Codec + Ord, { - fn from((inner, state_version): (Vec<(Option, StorageCollection)>, StateVersion)) -> Self { + fn from( + (inner, state_version): (Vec<(Option, StorageCollection)>, StateVersion), + ) -> Self { let mut expanded: HashMap, BTreeMap> = HashMap::new(); for (child_info, key_values) in inner { diff --git a/primitives/state-machine/src/lib.rs b/primitives/state-machine/src/lib.rs index 0fcf4a771c365..d777be5cb8c36 100644 --- a/primitives/state-machine/src/lib.rs +++ b/primitives/state-machine/src/lib.rs @@ -127,7 +127,7 @@ pub use crate::{ StorageTransactionCache, StorageValue, }, stats::{StateMachineStats, UsageInfo, UsageUnit}, - trie_backend::{TrieBackend, MigrateProgress}, + trie_backend::{MigrateProgress, TrieBackend}, trie_backend_essence::{Storage, TrieBackendStorage}, }; @@ -1000,10 +1000,11 @@ mod tests { use codec::{Decode, Encode}; use sp_core::{ map, + state_version::StateVersion, storage::{ChildInfo, TEST_DEFAULT_ALT_HASH_THRESHOLD as TRESHOLD}, testing::TaskExecutor, traits::{CodeExecutor, Externalities, RuntimeCode}, - NativeOrEncoded, NeverNativeValue, state_version::StateVersion, + NativeOrEncoded, NeverNativeValue, }; use sp_runtime::traits::BlakeTwo256; use std::{ @@ -1704,7 +1705,8 @@ mod tests { } let root3 = root.clone(); assert!(root1 != root3); - let remote_proof = check_proof(mdb.clone(), root.clone(), StateVersion::V1 { threshold: TRESHOLD }); + let remote_proof = + check_proof(mdb.clone(), root.clone(), StateVersion::V1 { threshold: TRESHOLD }); // nodes foo is replaced by its hashed value form. assert!(remote_proof.encode().len() < 1000); assert!(remote_proof.encoded_size() < 1000); diff --git a/primitives/state-machine/src/proving_backend.rs b/primitives/state-machine/src/proving_backend.rs index ab0b4a7c5b9e7..52e8d3c527c44 100644 --- a/primitives/state-machine/src/proving_backend.rs +++ b/primitives/state-machine/src/proving_backend.rs @@ -26,8 +26,7 @@ use codec::{Codec, Decode, Encode}; use hash_db::{HashDB, Hasher, Prefix, EMPTY_PREFIX}; use log::debug; use parking_lot::RwLock; -use sp_core::storage::ChildInfo; -use sp_core::state_version::StateVersion; +use sp_core::{state_version::StateVersion, storage::ChildInfo}; pub use sp_trie::trie_types::TrieError; use sp_trie::{ empty_child_trie_root, read_child_trie_value_with, read_trie_value_with, record_all_keys, @@ -171,8 +170,9 @@ impl ProofRecorder { /// encoded proof. pub fn estimate_encoded_size(&self) -> usize { let inner = self.inner.read(); - inner.encoded_size + codec::Compact(inner.records.len() as u32).encoded_size() - + inner.state_version.encoded_size() + inner.encoded_size + + codec::Compact(inner.records.len() as u32).encoded_size() + + inner.state_version.encoded_size() } /// Convert into a [`StorageProof`]. @@ -494,12 +494,19 @@ mod tests { let size_content = 34; // above hashable value treshold. let value_range = 0..64; - let contents = vec![(None, value_range - .clone() - .map(|i| (vec![i], Some(vec![i; size_content]))) - .collect::>())]; + let contents = vec![( + None, + value_range + .clone() + .map(|i| (vec![i], Some(vec![i; size_content]))) + .collect::>(), + )]; let in_memory: InMemoryBackend = if flagged { - (contents, StateVersion::V1 { threshold: sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD }).into() + ( + contents, + StateVersion::V1 { threshold: sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD }, + ) + .into() } else { (contents, StateVersion::V0).into() }; @@ -541,7 +548,11 @@ mod tests { (Some(child_info_2.clone()), (10..15).map(|i| (vec![i], Some(vec![i]))).collect()), ]; let in_memory: InMemoryBackend = if flagged { - (contents, StateVersion::V1 { threshold: sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD }).into() + ( + contents, + StateVersion::V1 { threshold: sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD }, + ) + .into() } else { (contents, StateVersion::V0).into() }; diff --git a/primitives/state-machine/src/testing.rs b/primitives/state-machine/src/testing.rs index eb12fddd6efbd..01ff9a23dc1fc 100644 --- a/primitives/state-machine/src/testing.rs +++ b/primitives/state-machine/src/testing.rs @@ -36,13 +36,13 @@ use codec::Decode; use hash_db::Hasher; use sp_core::{ offchain::testing::TestPersistentOffchainDB, + state_version::StateVersion, storage::{ well_known_keys::{is_child_storage_key, CHANGES_TRIE_CONFIG, CODE}, Storage, }, testing::TaskExecutor, traits::TaskExecutorExt, - state_version::StateVersion, }; use sp_externalities::{Extension, ExtensionStore, Extensions}; @@ -96,7 +96,6 @@ where Self::new_with_code_and_state(&[], storage, state_version) } - /// New empty test externalities. pub fn new_empty() -> Self { Self::new_with_code_and_state(&[], Storage::default(), Default::default()) @@ -107,8 +106,13 @@ where Self::new_with_code_and_state(code, storage, Default::default()) } - /// Create a new instance of `TestExternalities` with code and storage for a given state version. - pub fn new_with_code_and_state(code: &[u8], mut storage: Storage, state_version: StateVersion) -> Self { + /// Create a new instance of `TestExternalities` with code and storage for a given state + /// version. + pub fn new_with_code_and_state( + code: &[u8], + mut storage: Storage, + state_version: StateVersion, + ) -> Self { let mut overlay = OverlayedChanges::default(); let changes_trie_config = storage .top diff --git a/primitives/state-machine/src/trie_backend.rs b/primitives/state-machine/src/trie_backend.rs index 7d3c7b0d5b8bc..1420aca0fb623 100644 --- a/primitives/state-machine/src/trie_backend.rs +++ b/primitives/state-machine/src/trie_backend.rs @@ -24,10 +24,11 @@ use crate::{ }; use codec::{Codec, Decode}; use hash_db::Hasher; -use sp_core::storage::{ChildInfo, ChildType}; -use sp_core::state_version::StateVersion; -use sp_std::{boxed::Box, vec::Vec}; -use sp_std::{rc::Rc, cell::RefCell}; +use sp_core::{ + state_version::StateVersion, + storage::{ChildInfo, ChildType}, +}; +use sp_std::{boxed::Box, cell::RefCell, rc::Rc, vec::Vec}; use sp_trie::{ child_delta_trie_root, delta_trie_root, empty_child_trie_root, trie_types::{TrieDB, TrieError}, @@ -46,7 +47,7 @@ where { /// Create new trie-based backend. pub fn new(storage: S, root: H::Out, state_version: StateVersion) -> Self { - TrieBackend { essence: TrieBackendEssence::new(storage, root), state_version: state_version } + TrieBackend { essence: TrieBackendEssence::new(storage, root), state_version } } /// Get backend essence reference. @@ -205,12 +206,8 @@ where let mut eph = Ephemeral::new(self.essence.backend_storage(), &mut write_overlay); let res = || { let layout = match self.state_version { - StateVersion::V0 => { - sp_trie::Layout::default() - }, - StateVersion::V1{ threshold } => { - sp_trie::Layout::with_alt_hashing(threshold) - }, + StateVersion::V0 => sp_trie::Layout::default(), + StateVersion::V1 { threshold } => sp_trie::Layout::with_alt_hashing(threshold), }; delta_trie_root::, _, _, _, _, _>(&mut eph, root, delta, layout) }; @@ -236,12 +233,8 @@ where ChildType::ParentKeyId => empty_child_trie_root::>(), }; let layout = match self.state_version { - StateVersion::V0 => { - sp_trie::Layout::default() - }, - StateVersion::V1{ threshold } => { - sp_trie::Layout::with_alt_hashing(threshold) - }, + StateVersion::V0 => sp_trie::Layout::default(), + StateVersion::V1 { threshold } => sp_trie::Layout::with_alt_hashing(threshold), }; let mut write_overlay = S::Overlay::default(); @@ -295,7 +288,6 @@ where } } - /// Migration progress. pub struct MigrateProgress { /// Next key to migrate for top. @@ -320,15 +312,12 @@ where // type is neither send nor sync but only ever // use in migrate method local to a single thread. -unsafe impl<'a, S, H: Hasher> Send for MigrateStorage<'a, S, H> { } -unsafe impl<'a, S, H: Hasher> Sync for MigrateStorage<'a, S, H> { } +unsafe impl<'a, S, H: Hasher> Send for MigrateStorage<'a, S, H> {} +unsafe impl<'a, S, H: Hasher> Sync for MigrateStorage<'a, S, H> {} impl<'a, S, H: Hasher> Clone for MigrateStorage<'a, S, H> { fn clone(&self) -> Self { - MigrateStorage { - overlay: self.overlay.clone(), - storage: self.storage - } + MigrateStorage { overlay: self.overlay.clone(), storage: self.storage } } } @@ -379,7 +368,9 @@ impl<'a, S: 'a + TrieBackendStorage, H: Hasher> hash_db::HashDB } } -impl<'a, S: 'a + TrieBackendStorage, H: Hasher> HashDBRef for MigrateStorage<'a, S, H> { +impl<'a, S: 'a + TrieBackendStorage, H: Hasher> HashDBRef + for MigrateStorage<'a, S, H> +{ fn get(&self, key: &H::Out, prefix: Prefix) -> Option { HashDB::get(self, key, prefix) } @@ -424,7 +415,6 @@ fn trie_error(_e: &sp_trie::TrieError>) -> crate::DefaultEr crate::DefaultError } - impl, H: Hasher> TrieBackend where H::Out: Ord + Codec, @@ -440,7 +430,7 @@ where changes: &mut sp_trie::PrefixedMemoryDB, mut progress: MigrateProgress, ) -> Result, crate::DefaultError> { - use sp_trie::{TrieDB, TrieDBMut, TrieMut, TrieDBIterator}; + use sp_trie::{TrieDB, TrieDBIterator, TrieDBMut, TrieMut}; let dest_threshold = match (migration_from, migration_to, self.state_version) { (StateVersion::V0, StateVersion::V1 { threshold }, StateVersion::V0) => threshold, _ => return Err(error_version()), @@ -449,10 +439,8 @@ where let empty_value2 = sp_std::vec![0u8; dest_threshold as usize]; let dest_layout = Layout::with_alt_hashing(dest_threshold); let overlay = Rc::new(RefCell::new(changes)); - let mut dest_db = MigrateStorage:: { - overlay, - storage: self.essence.backend_storage(), - }; + let mut dest_db = + MigrateStorage:: { overlay, storage: self.essence.backend_storage() }; let ori_root = self.essence.root().clone(); // Using db with remove node is ok as long as iteration // start at uncommited key. @@ -466,21 +454,25 @@ where let mut dest = TrieDBMut::from_existing_with_layout(&mut dest_db, dest_root, dest_layout) .map_err(|e| trie_error::(&*e))?; - let iter = TrieDBIterator::new_prefixed_then_seek(&ori, &[], progress.current_top.as_ref().map(|s| s.as_slice()).unwrap_or(&[])).map_err(|e| trie_error::(&*e))?; + let iter = TrieDBIterator::new_prefixed_then_seek( + &ori, + &[], + progress.current_top.as_ref().map(|s| s.as_slice()).unwrap_or(&[]), + ) + .map_err(|e| trie_error::(&*e))?; for elt in iter { - let (key, value) = elt - .map_err(|e| trie_error::(&*e))?; + let (key, value) = elt.map_err(|e| trie_error::(&*e))?; if let Some(limit_size) = limit_size.as_mut() { if *limit_size < value.len() as u64 { progress.current_top = Some(key); - break; + break } *limit_size -= value.len() as u64; } if let Some(limit_items) = limit_items.as_mut() { if *limit_items == 0 { progress.current_top = Some(key); - break; + break } *limit_items -= 1; } diff --git a/primitives/trie/src/storage_proof.rs b/primitives/trie/src/storage_proof.rs index 48548b490f133..1bdbc91475ed2 100644 --- a/primitives/trie/src/storage_proof.rs +++ b/primitives/trie/src/storage_proof.rs @@ -18,9 +18,9 @@ use crate::{Layout, TrieLayout}; use codec::{Decode, Encode}; use hash_db::{HashDB, Hasher}; +use sp_core::state_version::StateVersion; use sp_std::vec::Vec; use trie_db::NodeCodec; -use sp_core::state_version::StateVersion; /// A proof that some set of key-value pairs are included in the storage trie. The proof contains /// the storage values so that the partial storage backend can be reconstructed by a verifier that @@ -45,7 +45,7 @@ pub struct CompactProof { mod decode_impl { use super::*; - use codec::{Input, Error}; + use codec::{Error, Input}; impl Decode for StorageProof { fn decode(input: &mut I) -> Result { let trie_nodes: Vec> = Decode::decode(input)?; @@ -55,10 +55,7 @@ mod decode_impl { } else { Decode::decode(input)? }; - Ok(StorageProof { - trie_nodes, - state_version, - }) + Ok(StorageProof { trie_nodes, state_version }) } } @@ -71,10 +68,7 @@ mod decode_impl { } else { Decode::decode(input)? }; - Ok(CompactProof { - encoded_nodes, - state_version, - }) + Ok(CompactProof { encoded_nodes, state_version }) } } } @@ -140,7 +134,10 @@ impl StorageProof { let trie_nodes = proofs .into_iter() .flat_map(|proof| { - debug_assert!(state_version == &StateVersion::default() || state_version == &proof.state_version); + debug_assert!( + state_version == &StateVersion::default() || + state_version == &proof.state_version + ); *state_version = proof.state_version; proof.iter_nodes() }) diff --git a/test-utils/client/src/lib.rs b/test-utils/client/src/lib.rs index e5a84c2efb5ce..8fc50e4e6fc08 100644 --- a/test-utils/client/src/lib.rs +++ b/test-utils/client/src/lib.rs @@ -45,11 +45,7 @@ use sc_client_api::BlockchainEvents; use sc_service::client::{ClientConfig, LocalCallExecutor}; use serde::Deserialize; use sp_core::storage::ChildInfo; -use sp_runtime::{ - codec::Encode, - traits::Block as BlockT, - OpaqueExtrinsic, -}; +use sp_runtime::{codec::Encode, traits::Block as BlockT, OpaqueExtrinsic}; use std::{ collections::{HashMap, HashSet}, pin::Pin, @@ -57,8 +53,7 @@ use std::{ }; /// Test client light database backend. -pub type LightBackend = - sc_light::Backend, Block>; +pub type LightBackend = sc_light::Backend, Block>; /// A genesis storage initialization trait. pub trait GenesisInit: Default { @@ -107,7 +102,9 @@ impl } /// Create new `TestClientBuilder` with default backend and state versions. - pub fn with_default_backend_and_state_versions(state_versions: Option>) -> Self { + pub fn with_default_backend_and_state_versions( + state_versions: Option>, + ) -> Self { let backend = Arc::new(Backend::new_test_with_tx_storage_and_state_versions( std::u32::MAX, std::u64::MAX, @@ -117,9 +114,11 @@ impl Self::with_backend(backend) } - /// Create new `TestClientBuilder` with default backend and pruning window size - pub fn with_pruning_window(keep_blocks: u32, state_versions: Option>) -> Self { + pub fn with_pruning_window( + keep_blocks: u32, + state_versions: Option>, + ) -> Self { let backend = Arc::new(Backend::new_test_with_tx_storage_and_state_versions( keep_blocks, 0, @@ -130,7 +129,10 @@ impl } /// Create new `TestClientBuilder` with default backend and storage chain mode - pub fn with_tx_storage(keep_blocks: u32, state_versions: Option>) -> Self { + pub fn with_tx_storage( + keep_blocks: u32, + state_versions: Option>, + ) -> Self { let backend = Arc::new(Backend::new_test_with_tx_storage_and_state_versions( keep_blocks, 0, diff --git a/test-utils/runtime/src/lib.rs b/test-utils/runtime/src/lib.rs index 371980c1ea375..75cf7aac1870a 100644 --- a/test-utils/runtime/src/lib.rs +++ b/test-utils/runtime/src/lib.rs @@ -1316,7 +1316,8 @@ mod tests { #[test] fn witness_backend_works() { let (db, root) = witness_backend(); - let backend = sp_state_machine::TrieBackend::<_, crate::Hashing>::new(db, root, Default::default()); + let backend = + sp_state_machine::TrieBackend::<_, crate::Hashing>::new(db, root, Default::default()); let proof = sp_state_machine::prove_read(backend, vec![b"value3"]).unwrap(); let client = TestClientBuilder::new().set_execution_strategy(ExecutionStrategy::Both).build(); diff --git a/utils/frame/try-runtime/cli/src/lib.rs b/utils/frame/try-runtime/cli/src/lib.rs index 3063cbef57c2f..901dbd8484222 100644 --- a/utils/frame/try-runtime/cli/src/lib.rs +++ b/utils/frame/try-runtime/cli/src/lib.rs @@ -477,7 +477,9 @@ impl CliConfiguration for TryRuntimeCmd { /// Extract `:code` from the given chain spec and return as `StorageData` along with the /// corresponding `StorageKey`. -fn extract_code(spec: Box) -> sc_cli::Result<(StorageKey, StorageData)> { +fn extract_code( + spec: Box, +) -> sc_cli::Result<(StorageKey, StorageData)> { let genesis_storage = spec.build_storage()?; let code = StorageData( genesis_storage From 5828a5b256f09165908e551460e3002216b5fc08 Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 25 Aug 2021 18:40:31 +0200 Subject: [PATCH 089/127] migration digest definition --- client/block-builder/src/lib.rs | 57 +++++++++++++++++++- client/db/src/lib.rs | 3 +- client/service/src/client/client.rs | 2 + primitives/core/src/state_version.rs | 1 + primitives/runtime/src/generic/digest.rs | 17 ++++++ primitives/runtime/src/state_version.rs | 42 ++++++++++++--- primitives/state-machine/src/trie_backend.rs | 11 +++- test-utils/client/src/lib.rs | 29 +++++++--- 8 files changed, 142 insertions(+), 20 deletions(-) diff --git a/client/block-builder/src/lib.rs b/client/block-builder/src/lib.rs index f381cfd236e7e..f1cf3d1a27b96 100644 --- a/client/block-builder/src/lib.rs +++ b/client/block-builder/src/lib.rs @@ -34,7 +34,7 @@ use sp_api::{ use sp_blockchain::{ApplyExtrinsicFailed, Error}; use sp_core::ExecutionContext; use sp_runtime::{ - generic::BlockId, + generic::BlockId, StateVersions, traits::{Block as BlockT, DigestFor, Hash, HashFor, Header as HeaderT, NumberFor, One}, }; @@ -139,6 +139,7 @@ pub struct BlockBuilder<'a, Block: BlockT, A: ProvideRuntimeApi, B> { backend: &'a B, /// The estimated size of the block header. estimated_header_size: usize, + state_versions: &'a StateVersions, } impl<'a, Block, A, B> BlockBuilder<'a, Block, A, B> @@ -161,6 +162,7 @@ where record_proof: RecordProof, inherent_digests: DigestFor, backend: &'a B, + state_versions: &'a StateVersions, ) -> Result { let header = <::Header as HeaderT>::new( parent_number + One::one(), @@ -189,6 +191,7 @@ where block_id, backend, estimated_header_size, + state_versions, }) } @@ -223,7 +226,7 @@ where /// supplied by `self.api`, combined as [`BuiltBlock`]. /// The storage proof will be `Some(_)` when proof recording was enabled. pub fn build(mut self) -> Result>, Error> { - let header = self + let mut header = self .api .finalize_block_with_context(&self.block_id, ExecutionContext::BlockConstruction)?; @@ -248,6 +251,56 @@ where .into_storage_changes(&state, changes_trie_state.as_ref(), parent_hash) .map_err(|e| sp_blockchain::Error::StorageChanges(e))?; + let number = header.number().clone(); + if let Some((migration_from, migration_to)) = self.state_versions.need_migrate(number) { + // TODO remove code duplicate with client/db/src/lib import block + use sp_api::StateBackend; + if let Some(prev_block_state) = state.as_trie_backend() { + let limit_size: Option = None; + let limit_items: Option = None; + let init_root = header.state_root().clone(); + let mut current_root = init_root.clone(); + let mut start_top = None; + let mut start_child = None; + + loop { + let sp_state_machine::MigrateProgress { current_top, current_child, root } = + prev_block_state + .migrate( + migration_from, + migration_to, + limit_size, + limit_items, + //Some(&mut storage_changes.transaction), + None, + (&storage_changes.main_storage_changes, &storage_changes.child_storage_changes), + sp_state_machine::MigrateProgress { + current_top: start_top.take(), + current_child: start_child.take(), + root: Some(current_root), // TODO non optional rather? + }, + ) + .map_err(|err| { + sp_blockchain::Error::Backend(format!( + "error migrating: {}", + err + )) + })?; + + start_top = current_top; + start_child = current_child; + current_root = + root.unwrap_or_else(|| header.state_root().clone()); + if start_top.is_none() { + break + } + } + + header.set_state_root(current_root); + } else { + panic!("Could not migrate at block: {:?}", number); + } + } Ok(BuiltBlock { block: ::new(header, self.extrinsics), storage_changes, diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index 807cdd0456af5..2a47dff15b4db 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -2098,7 +2098,8 @@ impl sc_client_api::backend::Backend for Backend { migration_to, limit_size, limit_items, - &mut operation.db_updates, + Some(&mut operation.db_updates), + (&operation.storage_updates, &operation.child_storage_updates), sp_state_machine::MigrateProgress { current_top: start_top.take(), current_child: start_child.take(), diff --git a/client/service/src/client/client.rs b/client/service/src/client/client.rs index ce53123e75198..7839cbfca7897 100644 --- a/client/service/src/client/client.rs +++ b/client/service/src/client/client.rs @@ -1427,6 +1427,7 @@ where record_proof.into(), inherent_digests, &self.backend, + &self.config.state_versions, ) } @@ -1442,6 +1443,7 @@ where RecordProof::No, inherent_digests, &self.backend, + &self.config.state_versions, ) } } diff --git a/primitives/core/src/state_version.rs b/primitives/core/src/state_version.rs index ebb6db3e51d3f..06739b4e9ef88 100644 --- a/primitives/core/src/state_version.rs +++ b/primitives/core/src/state_version.rs @@ -31,6 +31,7 @@ pub const DEFAULT_STATE_VERSION: StateVersion = /// Supported version with substrate chain. #[derive(Debug, Clone, Copy, PartialEq, Eq, Encode, Decode)] #[cfg_attr(feature = "std", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "std", derive(parity_util_mem::MallocSizeOf))] pub enum StateVersion { /// Patricia trie Radix 16 without extension node. V0, diff --git a/primitives/runtime/src/generic/digest.rs b/primitives/runtime/src/generic/digest.rs index 390acb87f690d..8323dbf9b6713 100644 --- a/primitives/runtime/src/generic/digest.rs +++ b/primitives/runtime/src/generic/digest.rs @@ -116,6 +116,11 @@ pub enum DigestItem { /// native code. ChangesTrieSignal(ChangesTrieSignal), + /// System digest item that contains pending migration state. + /// This allows keeping trace of migration progress (root of + /// pending new state), and switching root to use on completion. + StateMigration(crate::state_version::StateMigrationDigest), + /// Some other thing. Unsupported and experimental. Other(Vec), } @@ -182,6 +187,10 @@ pub enum DigestItemRef<'a, Hash: 'a> { /// Digest item that contains signal from changes tries manager to the /// native code. ChangesTrieSignal(&'a ChangesTrieSignal), + /// System digest item that contains pending migration state. + /// This allows keeping trace of migration progress (root of + /// pending new state), and switching root to use on completion. + StateMigration(&'a crate::state_version::StateMigrationDigest), /// Any 'non-system' digest item, opaque to the native code. Other(&'a Vec), } @@ -199,6 +208,7 @@ pub enum DigestItemType { Seal = 5, PreRuntime = 6, ChangesTrieSignal = 7, + StateMigration = 8, } /// Type of a digest item that contains raw data; this also names the consensus engine ID where @@ -224,6 +234,7 @@ impl DigestItem { Self::Consensus(ref v, ref s) => DigestItemRef::Consensus(v, s), Self::Seal(ref v, ref s) => DigestItemRef::Seal(v, s), Self::ChangesTrieSignal(ref s) => DigestItemRef::ChangesTrieSignal(s), + Self::StateMigration(ref s) => DigestItemRef::StateMigration(s), Self::Other(ref v) => DigestItemRef::Other(v), } } @@ -321,6 +332,8 @@ impl Decode for DigestItem { }, DigestItemType::ChangesTrieSignal => Ok(Self::ChangesTrieSignal(Decode::decode(input)?)), + DigestItemType::StateMigration => + Ok(Self::StateMigration(Decode::decode(input)?)), DigestItemType::Other => Ok(Self::Other(Decode::decode(input)?)), } } @@ -453,6 +466,10 @@ impl<'a, Hash: Encode> Encode for DigestItemRef<'a, Hash> { DigestItemType::ChangesTrieSignal.encode_to(&mut v); changes_trie_signal.encode_to(&mut v); }, + Self::StateMigration(progress) => { + DigestItemType::StateMigration.encode_to(&mut v); + progress.encode_to(&mut v); + }, Self::Other(val) => { DigestItemType::Other.encode_to(&mut v); val.encode_to(&mut v); diff --git a/primitives/runtime/src/state_version.rs b/primitives/runtime/src/state_version.rs index e6bf1ba56fecb..3425d529ba092 100644 --- a/primitives/runtime/src/state_version.rs +++ b/primitives/runtime/src/state_version.rs @@ -21,6 +21,7 @@ use crate::traits::{Block, NumberFor}; use sp_arithmetic::traits::Zero; pub use sp_core::state_version::{StateVersion, DEFAULT_STATE_VERSION}; use sp_std::{str::FromStr, vec::Vec}; +use codec::{Encode, Decode}; /// Multiple versions of state in use for a chain. #[derive(Clone, crate::RuntimeDebug)] @@ -40,7 +41,7 @@ impl StateVersions { pub fn genesis_state_version(&self) -> StateVersion { if let Some((number, version)) = self.canonical_states.get(0) { if number.is_zero() { - return *version + return *version; } } DEFAULT_STATE_VERSION @@ -52,7 +53,7 @@ impl StateVersions { let mut version = DEFAULT_STATE_VERSION; for (number, state) in self.canonical_states.iter() { if number > &at { - break + break; } version = *state; } @@ -66,10 +67,10 @@ impl StateVersions { for (i, (number, _)) in self.canonical_states.iter().enumerate() { if number == &at { replace = Some(i); - break + break; } if number > &at { - break + break; } insert = Some(i + 1); } @@ -95,7 +96,7 @@ impl StateVersions { if let Ok(number) = NumberFor::::from_str(number) { canonical_states.push((number.into(), version)); } else { - return None + return None; } } Some(StateVersions { canonical_states }) @@ -112,13 +113,40 @@ impl StateVersions { for (number, version) in self.canonical_states.iter() { if number == &at { to = Some(*version); - break + break; } if number > &at { - break + break; } from = *version; } to.map(|to| (from, to)) } } + +/// Migration definition at a given block. +#[derive(PartialEq, Eq, Clone, crate::RuntimeDebug, Encode, Decode)] +#[cfg_attr(feature = "std", derive(parity_util_mem::MallocSizeOf))] +pub struct StateMigrationDigest { + /// State version to double check. + from: StateVersion, + /// State version to double check. + to: StateVersion, + /// State root of target state. + state_root: Hash, + /// Current state. + progress: StateMigrationProgress, +} + +/// State of migration for a given block. +#[derive(PartialEq, Eq, Clone, crate::RuntimeDebug, Encode, Decode)] +#[cfg_attr(feature = "std", derive(parity_util_mem::MallocSizeOf))] +pub enum StateMigrationProgress { + // Pending, read progress from destination at well known key + // (need to overwrite with origin value when finish). + /// Destination is equivalent to origin. + /// + /// Next block will use `to` state with the + /// state root of from the migration progress destination state. + Finished, +} diff --git a/primitives/state-machine/src/trie_backend.rs b/primitives/state-machine/src/trie_backend.rs index 1420aca0fb623..50887464e4cd8 100644 --- a/primitives/state-machine/src/trie_backend.rs +++ b/primitives/state-machine/src/trie_backend.rs @@ -427,7 +427,8 @@ where migration_to: StateVersion, mut limit_size: Option, mut limit_items: Option, - changes: &mut sp_trie::PrefixedMemoryDB, + do_update: Option<&mut sp_trie::PrefixedMemoryDB>, + previous_block_updates: (&crate::StorageCollection, &crate::ChildStorageCollection), mut progress: MigrateProgress, ) -> Result, crate::DefaultError> { use sp_trie::{TrieDB, TrieDBIterator, TrieDBMut, TrieMut}; @@ -438,7 +439,13 @@ where let empty_value = sp_std::vec![23u8; dest_threshold as usize]; let empty_value2 = sp_std::vec![0u8; dest_threshold as usize]; let dest_layout = Layout::with_alt_hashing(dest_threshold); - let overlay = Rc::new(RefCell::new(changes)); + let mut in_mem; + let overlay = if let Some(change) = do_update { + Rc::new(RefCell::new(change)) + } else { + in_mem = sp_trie::PrefixedMemoryDB::default(); + Rc::new(RefCell::new(&mut in_mem)) + }; let mut dest_db = MigrateStorage:: { overlay, storage: self.essence.backend_storage() }; let ori_root = self.essence.root().clone(); diff --git a/test-utils/client/src/lib.rs b/test-utils/client/src/lib.rs index 8fc50e4e6fc08..685967798d0d6 100644 --- a/test-utils/client/src/lib.rs +++ b/test-utils/client/src/lib.rs @@ -34,7 +34,7 @@ pub use sp_keyring::{ ed25519::Keyring as Ed25519Keyring, sr25519::Keyring as Sr25519Keyring, AccountKeyring, }; pub use sp_keystore::{SyncCryptoStore, SyncCryptoStorePtr}; -pub use sp_runtime::{Storage, StorageChild}; +pub use sp_runtime::{Storage, StorageChild, StateVersions}; pub use sp_state_machine::ExecutionStrategy; use futures::{ @@ -81,6 +81,7 @@ pub struct TestClientBuilder, enable_offchain_indexing_api: bool, state_hashed_value: bool, // TODO useless ? + state_versions: StateVersions, no_genesis: bool, } @@ -105,13 +106,14 @@ impl pub fn with_default_backend_and_state_versions( state_versions: Option>, ) -> Self { + let state_versions = state_versions.unwrap_or_default(); let backend = Arc::new(Backend::new_test_with_tx_storage_and_state_versions( std::u32::MAX, std::u64::MAX, sc_client_db::TransactionStorageMode::BlockBody, - state_versions.unwrap_or_default(), + state_versions.clone(), )); - Self::with_backend(backend) + Self::with_backend_and_state_versions(backend, state_versions) } /// Create new `TestClientBuilder` with default backend and pruning window size @@ -119,13 +121,14 @@ impl keep_blocks: u32, state_versions: Option>, ) -> Self { + let state_versions = state_versions.unwrap_or_default(); let backend = Arc::new(Backend::new_test_with_tx_storage_and_state_versions( keep_blocks, 0, sc_client_db::TransactionStorageMode::BlockBody, - state_versions.unwrap_or_default(), + state_versions.clone(), )); - Self::with_backend(backend) + Self::with_backend_and_state_versions(backend, state_versions) } /// Create new `TestClientBuilder` with default backend and storage chain mode @@ -133,13 +136,14 @@ impl keep_blocks: u32, state_versions: Option>, ) -> Self { + let state_versions = state_versions.unwrap_or_default(); let backend = Arc::new(Backend::new_test_with_tx_storage_and_state_versions( keep_blocks, 0, sc_client_db::TransactionStorageMode::StorageChain, - state_versions.unwrap_or_default(), + state_versions.clone(), )); - Self::with_backend(backend) + Self::with_backend_and_state_versions(backend, state_versions) } } @@ -148,6 +152,11 @@ impl { /// Create a new instance of the test client builder. pub fn with_backend(backend: Arc) -> Self { + Self::with_backend_and_state_versions(backend, Default::default()) + } + + /// Create a new instance of the test client builder with specific state versions. + pub fn with_backend_and_state_versions(backend: Arc, state_versions: StateVersions) -> Self { TestClientBuilder { backend, execution_strategies: ExecutionStrategies::default(), @@ -160,6 +169,7 @@ impl enable_offchain_indexing_api: false, state_hashed_value: false, no_genesis: false, + state_versions, } } @@ -280,6 +290,7 @@ impl ClientConfig { offchain_indexing_api: self.enable_offchain_indexing_api, no_genesis: self.no_genesis, + state_versions: self.state_versions.clone(), ..Default::default() }, ) @@ -320,11 +331,13 @@ impl let executor = executor.into().unwrap_or_else(|| { NativeElseWasmExecutor::new(WasmExecutionMethod::Interpreted, None, 8) }); + let mut client_config = ClientConfig::default(); + client_config.state_versions = self.state_versions.clone(); let executor = LocalCallExecutor::new( self.backend.clone(), executor, Box::new(sp_core::testing::TaskExecutor::new()), - Default::default(), + client_config, ) .expect("Creates LocalCallExecutor"); From 96223d7f8f71922e80b6e5a1e465eaa16ca8f83f Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 25 Aug 2021 20:39:02 +0200 Subject: [PATCH 090/127] check migration state and added todos --- client/block-builder/src/lib.rs | 14 ++++++++- client/db/src/lib.rs | 33 +++++++++++++++++++- primitives/blockchain/src/error.rs | 3 ++ primitives/blockchain/src/header_metadata.rs | 1 + primitives/runtime/src/lib.rs | 2 +- primitives/runtime/src/state_version.rs | 15 ++++++--- primitives/state-machine/src/trie_backend.rs | 16 ++++++++++ 7 files changed, 76 insertions(+), 8 deletions(-) diff --git a/client/block-builder/src/lib.rs b/client/block-builder/src/lib.rs index f1cf3d1a27b96..632b137d213c1 100644 --- a/client/block-builder/src/lib.rs +++ b/client/block-builder/src/lib.rs @@ -164,6 +164,10 @@ where backend: &'a B, state_versions: &'a StateVersions, ) -> Result { + // TODO call need_migration here and remove inner field + // then also compare (or init from) with parent header migration state. + // TODO pass parent header as param instead of hash and number + let header = <::Header as HeaderT>::new( parent_number + One::one(), Default::default(), @@ -296,7 +300,15 @@ where } } - header.set_state_root(current_root); + if start_top.is_some() { + unreachable!("TODO save progress in dest state in migrate fn and put a differnt progress digest"); + } + header.digest_mut().push(sp_runtime::DigestItem::StateMigration(sp_runtime::StateMigrationDigest { + from: migration_from, + to: migration_to, + state_root: current_root, + progress: sp_runtime::StateMigrationProgress::Finished, + })); } else { panic!("Could not migrate at block: {:?}", number); } diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index 2a47dff15b4db..97a789a966886 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -2059,6 +2059,8 @@ impl sc_client_api::backend::Backend for Backend { if block.is_pre_genesis() { operation.old_state = self.empty_state()?; } else { + // TODO migration from header cache of parent (block) + // if finished assert this is chainspec block 0 of new state operation.old_state = self.state_at(block)?; } operation.old_state.disable_syncing(); @@ -2123,8 +2125,37 @@ impl sc_client_api::backend::Backend for Backend { } } + if start_top.is_some() { + unreachable!("TODO save progress in dest state in migrate fn and put a differnt progress digest"); + } + + // TODO ensure no duplicate when setting. + let has_finished_digest = pending_block.header.digest().log(|digest| match digest { + sp_runtime::DigestItem::StateMigration(sp_runtime::StateMigrationDigest { + from, + to, + state_root, + progress, + }) => { + if from == &migration_from + && to == &migration_to + && state_root == ¤t_root + && progress == &sp_runtime::StateMigrationProgress::Finished { + return Some(&()); + } + // check for other state migration digest + // if there is. + None + }, + _ => None, + }); + + if has_finished_digest.is_none() { + info!("Migration state mismatch with final block state hash {:x?}, elapsed time: {:?}", ¤t_root, timer.elapsed()); + return Err(ClientError::InvalidMigrationState); + } + info!("Finished processing migration with final block state hash {:x?}, elapsed time: {:?}", ¤t_root, timer.elapsed()); - pending_block.header.set_state_root(current_root); } else { panic!("Could not migrate at block: {:?}", number); } diff --git a/primitives/blockchain/src/error.rs b/primitives/blockchain/src/error.rs index 6e60b2da6549f..0a2b5a83721f7 100644 --- a/primitives/blockchain/src/error.rs +++ b/primitives/blockchain/src/error.rs @@ -144,6 +144,9 @@ pub enum Error { #[error("Calculated state root does not match.")] InvalidStateRoot, + #[error("Calculated migration state does not match.")] + InvalidMigrationState, + #[error("Incomplete block import pipeline.")] IncompletePipeline, diff --git a/primitives/blockchain/src/header_metadata.rs b/primitives/blockchain/src/header_metadata.rs index 928409963bcd4..1bf95e2393fb2 100644 --- a/primitives/blockchain/src/header_metadata.rs +++ b/primitives/blockchain/src/header_metadata.rs @@ -258,6 +258,7 @@ pub struct CachedHeaderMetadata { pub state_root: Block::Hash, /// Hash of an ancestor header. Used to jump through the tree. ancestor: Block::Hash, + // TODO add migration progress!! to query this instead of full header. } impl From<&Block::Header> for CachedHeaderMetadata { diff --git a/primitives/runtime/src/lib.rs b/primitives/runtime/src/lib.rs index ee7871342b975..1ed5e18eb6470 100644 --- a/primitives/runtime/src/lib.rs +++ b/primitives/runtime/src/lib.rs @@ -94,7 +94,7 @@ pub use sp_arithmetic::{ pub use either::Either; -pub use state_version::{StateVersion, StateVersions}; +pub use state_version::{StateVersion, StateVersions, StateMigrationDigest, StateMigrationProgress}; /// An abstraction over justification for a block's validity under a consensus algorithm. /// diff --git a/primitives/runtime/src/state_version.rs b/primitives/runtime/src/state_version.rs index 3425d529ba092..185b00cb5f305 100644 --- a/primitives/runtime/src/state_version.rs +++ b/primitives/runtime/src/state_version.rs @@ -18,10 +18,10 @@ //! Substrate state versioning and migrations related types. use crate::traits::{Block, NumberFor}; +use codec::{Decode, Encode}; use sp_arithmetic::traits::Zero; pub use sp_core::state_version::{StateVersion, DEFAULT_STATE_VERSION}; use sp_std::{str::FromStr, vec::Vec}; -use codec::{Encode, Decode}; /// Multiple versions of state in use for a chain. #[derive(Clone, crate::RuntimeDebug)] @@ -129,21 +129,26 @@ impl StateVersions { #[cfg_attr(feature = "std", derive(parity_util_mem::MallocSizeOf))] pub struct StateMigrationDigest { /// State version to double check. - from: StateVersion, + pub from: StateVersion, /// State version to double check. - to: StateVersion, + pub to: StateVersion, /// State root of target state. - state_root: Hash, + pub state_root: Hash, /// Current state. - progress: StateMigrationProgress, + pub progress: StateMigrationProgress, } /// State of migration for a given block. #[derive(PartialEq, Eq, Clone, crate::RuntimeDebug, Encode, Decode)] #[cfg_attr(feature = "std", derive(parity_util_mem::MallocSizeOf))] pub enum StateMigrationProgress { + // Started, start requested from runtime (field `StateMigrationDigest` other than `to` should be ignored), + // then we switch the digest to `Pending` for empty state root and correct version. + // Pending, read progress from destination at well known key // (need to overwrite with origin value when finish). + + // /// Destination is equivalent to origin. /// /// Next block will use `to` state with the diff --git a/primitives/state-machine/src/trie_backend.rs b/primitives/state-machine/src/trie_backend.rs index 50887464e4cd8..88101fe315d93 100644 --- a/primitives/state-machine/src/trie_backend.rs +++ b/primitives/state-machine/src/trie_backend.rs @@ -498,6 +498,22 @@ where } } + // Set again changes from current block in dest (import may have overwrite them, + // or may have change a previous block migration change).c + for (key, value) in previous_block_updates.0.iter() { + if let Some(value) = value.as_ref() { + if value.len() >= dest_threshold as usize { + if progress.current_top.as_ref().map(|end| key < end).unwrap_or(true) { + dest.insert(key.as_slice(), value.as_slice()) + .map_err(|e| trie_error::(&*e))?; + } + } + } else { + dest.remove(key.as_slice()) + .map_err(|e| trie_error::(&*e))?; + } + } + // TODO same for child sp_std::mem::drop(ori); sp_std::mem::drop(dest); Ok(progress) From f3a861d2964542257b043f98c4be93082872ca25 Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 26 Aug 2021 10:48:22 +0200 Subject: [PATCH 091/127] tooling digest a bit. --- client/block-builder/src/lib.rs | 4 +- client/db/src/lib.rs | 25 ++--- primitives/runtime/src/state_version.rs | 139 +++++++++++++++++++++++- 3 files changed, 146 insertions(+), 22 deletions(-) diff --git a/client/block-builder/src/lib.rs b/client/block-builder/src/lib.rs index 632b137d213c1..a0e9479470dea 100644 --- a/client/block-builder/src/lib.rs +++ b/client/block-builder/src/lib.rs @@ -303,12 +303,12 @@ where if start_top.is_some() { unreachable!("TODO save progress in dest state in migrate fn and put a differnt progress digest"); } - header.digest_mut().push(sp_runtime::DigestItem::StateMigration(sp_runtime::StateMigrationDigest { + StateVersions::::set_migrate_digest(header.digest_mut(), sp_runtime::StateMigrationDigest { from: migration_from, to: migration_to, state_root: current_root, progress: sp_runtime::StateMigrationProgress::Finished, - })); + }); } else { panic!("Could not migrate at block: {:?}", number); } diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index 97a789a966886..5428cd9a2e0b8 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -2130,27 +2130,20 @@ impl sc_client_api::backend::Backend for Backend { } // TODO ensure no duplicate when setting. - let has_finished_digest = pending_block.header.digest().log(|digest| match digest { - sp_runtime::DigestItem::StateMigration(sp_runtime::StateMigrationDigest { + let has_finished_digest = match StateVersions::::migrate_digest(pending_block.header.digest()) { + Some(sp_runtime::StateMigrationDigest { from, to, state_root, progress, - }) => { - if from == &migration_from - && to == &migration_to - && state_root == ¤t_root - && progress == &sp_runtime::StateMigrationProgress::Finished { - return Some(&()); - } - // check for other state migration digest - // if there is. - None - }, - _ => None, - }); + }) => from == &migration_from + && to == &migration_to + && state_root == ¤t_root + && progress == &sp_runtime::StateMigrationProgress::Finished, + _ => false, + }; - if has_finished_digest.is_none() { + if !has_finished_digest { info!("Migration state mismatch with final block state hash {:x?}, elapsed time: {:?}", ¤t_root, timer.elapsed()); return Err(ClientError::InvalidMigrationState); } diff --git a/primitives/runtime/src/state_version.rs b/primitives/runtime/src/state_version.rs index 185b00cb5f305..e1422b95516f0 100644 --- a/primitives/runtime/src/state_version.rs +++ b/primitives/runtime/src/state_version.rs @@ -18,6 +18,7 @@ //! Substrate state versioning and migrations related types. use crate::traits::{Block, NumberFor}; +use crate::generic::{Digest, DigestItem}; use codec::{Decode, Encode}; use sp_arithmetic::traits::Zero; pub use sp_core::state_version::{StateVersion, DEFAULT_STATE_VERSION}; @@ -35,6 +36,35 @@ impl Default for StateVersions { } } +/// Different block specific migrate and state +/// behavior. This is read from chainspec and +/// header block digest. +pub enum MigrateState { + /// Use current define state version. + None(StateVersion), + /// Use state root from digest log. + Switch(StateVersion, H), + /// Start migrate between two version. + Start(StateVersion, StateVersion), + /// Pending migration between two version + /// with pending root of destination version. + Pending(StateVersion, StateVersion, H), +} + +/// Error related to migration digest from parent block header +/// and on import current block header. +pub enum InvalidErrorDigest { + /// Parent migrate do not match chainspec digest declaration. + InvalidParent, + /// Missing digest in parent header when expected. + MissingParentDigest, + /// Parent migrate switch to a state different than current one. + /// This is a bug. + InvalidParentSwitchState, + /// Imported digest do not match chainspec declaration. + InvalidImportedHeaderDigest, +} + impl StateVersions { /// Access genesis state version. /// This uses default state if undefined. @@ -106,21 +136,122 @@ impl StateVersions { /// Returns version to migrate from and to if /// migration is needed. pub fn need_migrate(&self, at: NumberFor) -> Option<(StateVersion, StateVersion)> { + let (from, to, _) = self.need_migrate_inner(at); + to.map(|to| (from, to)) + } + + fn need_migrate_inner(&self, at: NumberFor) -> (StateVersion, Option, bool) { // TODO switch to default when all V0 chains did migrate. let mut from = StateVersion::V0; let mut to = None; - let at = at + 1u32.into(); + let mut first = true; + let at_plus_one = at + 1u32.into(); for (number, version) in self.canonical_states.iter() { - if number == &at { + first = number == &at; + if number == &at_plus_one { to = Some(*version); break; } - if number > &at { + if number > &at_plus_one { break; } from = *version; } - to.map(|to| (from, to)) + (from, to, first) + } + + /// TODO remove pub and replace by resolve_migrate (and cache result). + pub fn migrate_digest(digests: &Digest) -> Option<&StateMigrationDigest> { + digests.log(|digest| if let DigestItem::StateMigration(digest) = digest { + Some(digest) + } else { + None + }) + } + + /// Push or update migration digest. + pub fn set_migrate_digest(digests: &mut Digest, migration_digest: StateMigrationDigest) { + for digest in digests.logs.iter_mut() { + if let DigestItem::StateMigration(digest) = digest { + *digest = migration_digest; + return + } + } + digests.logs.push(DigestItem::StateMigration(migration_digest)); + } + + /// Get migration specific block behavior. + pub fn resolve_migrate( + &self, + at: NumberFor, + parent_digests: &Digest, + imported_block_digests: Option<&Digest>, + ) -> Result, InvalidErrorDigest> { + if at.is_zero() { + // No migration at genesis (a bit useless since parent + // digest is undefiend, but one could use default). + // So ignoring input digest could also error. + return Ok(MigrateState::None(self.state_version_at(at))); + } + match self.need_migrate_inner(at) { + (from, Some(to), false) => { + // no migration digest allowed (only start at chainspec -1 and + // no consecutive mig support). and no from runtime trigger. + if Self::migrate_digest(parent_digests).is_some() { + return Err(InvalidErrorDigest::InvalidParent); + } + + + if let Some(digests) = imported_block_digests { + if Self::migrate_digest(digests).is_some() { + return Err(InvalidErrorDigest::InvalidImportedHeaderDigest); + } + } + Ok(MigrateState::Start(from, to)) + }, + (from, None, true) => { + // switch + let state_root = if let Some(digest) = Self::migrate_digest(parent_digests) { + match digest { + StateMigrationDigest { to, state_root, progress: StateMigrationProgress::Finished, .. } => { + if to != &from { + return Err(InvalidErrorDigest::InvalidParentSwitchState); + } + state_root + }, +/* _ => { + return Err(InvalidErrorDigest::InvalidParent); + },*/ + } + } else { + return Err(InvalidErrorDigest::MissingParentDigest); + }; + + if let Some(digests) = imported_block_digests { + // No support for consecutive migrations. + if Self::migrate_digest(digests).is_some() { + return Err(InvalidErrorDigest::InvalidImportedHeaderDigest); + } + } + + Ok(MigrateState::Switch(from, state_root.clone())) + }, + (from, None, false) => { + if Self::migrate_digest(parent_digests).is_some() { + return Err(InvalidErrorDigest::InvalidParent); + } + + if let Some(digests) = imported_block_digests { + // current digest need to be nothing TODO later could migrate from runtime trigger? + if Self::migrate_digest(digests).is_some() { + return Err(InvalidErrorDigest::InvalidImportedHeaderDigest); + } + } + Ok(MigrateState::None(from)) + }, + (_, Some(_), true) => unreachable!("need_migrate_inner exclude target block from migration range"), + } + } } From 503270a931c6c4929d434b9ffadcac9dd86784bf Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 26 Aug 2021 12:04:33 +0200 Subject: [PATCH 092/127] guard in_mem against migration (state_versions inner field is discutable). --- client/api/src/in_mem.rs | 61 ++++++++++++++++++------- client/db/src/lib.rs | 43 ++++++++++------- primitives/blockchain/src/error.rs | 2 +- primitives/runtime/src/lib.rs | 2 +- primitives/runtime/src/state_version.rs | 8 ++++ 5 files changed, 81 insertions(+), 35 deletions(-) diff --git a/client/api/src/in_mem.rs b/client/api/src/in_mem.rs index 1b170f99dc767..f7f9efe3e6d5a 100644 --- a/client/api/src/in_mem.rs +++ b/client/api/src/in_mem.rs @@ -26,7 +26,7 @@ use sp_core::{ use sp_runtime::{ generic::BlockId, traits::{Block as BlockT, HashFor, Header as HeaderT, NumberFor, Zero}, - Justification, Justifications, StateVersion, StateVersions, Storage, + Justification, Justifications, MigrateState, StateVersion, StateVersions, Storage, }; use sp_state_machine::{ Backend as StateBackend, ChangesTrieTransaction, ChildStorageCollection, InMemoryBackend, @@ -94,7 +94,7 @@ impl StoredBlock { StoredBlock::Full(block, just) => { let (header, body) = block.deconstruct(); (header, Some(body), just) - }, + } } } } @@ -150,12 +150,13 @@ impl Blockchain { /// Get version of state. pub fn state_version(&self, id: BlockId) -> StateVersion { let number = match id { - BlockId::Hash(h) => + BlockId::Hash(h) => { if let Ok(Some(header)) = self.header(BlockId::Hash(h)) { header.number().clone() } else { 0u32.into() - }, + } + } BlockId::Number(n) => n, }; self.storage.read().state_versions.state_version_at(number) @@ -225,7 +226,7 @@ impl Blockchain { pub fn equals_to(&self, other: &Self) -> bool { // Check ptr equality first to avoid double read locks. if ptr::eq(self, other) { - return true + return true; } self.canon_equals_to(other) && self.storage.read().blocks == other.storage.read().blocks } @@ -234,14 +235,14 @@ impl Blockchain { pub fn canon_equals_to(&self, other: &Self) -> bool { // Check ptr equality first to avoid double read locks. if ptr::eq(self, other) { - return true + return true; } let this = self.storage.read(); let other = other.storage.read(); - this.hashes == other.hashes && - this.best_hash == other.best_hash && - this.best_number == other.best_number && - this.genesis_hash == other.genesis_hash + this.hashes == other.hashes + && this.best_hash == other.best_hash + && this.best_number == other.best_number + && this.genesis_hash == other.genesis_hash } /// Insert header CHT root. @@ -346,7 +347,7 @@ impl Blockchain { if !stored_justifications.append(justification) { return Err(sp_blockchain::Error::BadJustification( "Duplicate consensus engine ID".into(), - )) + )); } } else { *block_justifications = Some(Justifications::from(justification)); @@ -826,6 +827,33 @@ where None => old_state.clone(), }; + if let Some(parent_header) = + self.blockchain.header(BlockId::Hash(header.parent_hash().clone()))? + { + match self + .blockchain + .storage + .read() + .state_versions + .resolve_migrate( + parent_header.number().clone() + 1u32.into(), + parent_header.digest(), + None, + ) + .map_err(|err| { + sp_blockchain::Error::InvalidMigrationState(format!("{}", err)) + })? { + MigrateState::None(..) => (), + MigrateState::Start(..) + | MigrateState::Pending(..) + | MigrateState::Switch(..) => { + return Err(sp_blockchain::Error::InvalidMigrationState( + "Migration unsupported for in memory client".to_string(), + )); + } + } + } + self.states.write().insert(hash, new_state); self.blockchain.insert(hash, header, justification, body, pending_block.state)?; @@ -876,9 +904,10 @@ where fn state_at(&self, block: BlockId) -> sp_blockchain::Result { match block { - BlockId::Hash(h) if h == Default::default() => - return Ok(self.blockchain.state_version(BlockId::Hash(h)).into()), - _ => {}, + BlockId::Hash(h) if h == Default::default() => { + return Ok(self.blockchain.state_version(BlockId::Hash(h)).into()) + } + _ => {} } self.blockchain @@ -925,7 +954,7 @@ where /// Check that genesis storage is valid. pub fn check_genesis_storage(storage: &Storage) -> sp_blockchain::Result<()> { if storage.top.iter().any(|(k, _)| well_known_keys::is_child_storage_key(k)) { - return Err(sp_blockchain::Error::InvalidState.into()) + return Err(sp_blockchain::Error::InvalidState.into()); } if storage @@ -933,7 +962,7 @@ pub fn check_genesis_storage(storage: &Storage) -> sp_blockchain::Result<()> { .keys() .any(|child_key| !well_known_keys::is_child_storage_key(&child_key)) { - return Err(sp_blockchain::Error::InvalidState.into()) + return Err(sp_blockchain::Error::InvalidState.into()); } Ok(()) diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index 5428cd9a2e0b8..d3ac42d169ce0 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -88,7 +88,7 @@ use sp_runtime::{ Block as BlockT, Hash, HashFor, Header as HeaderT, NumberFor, One, SaturatedConversion, Zero, }, - Justification, Justifications, StateVersion, StateVersions, Storage, + Justification, Justifications, StateVersion, StateVersions, Storage, MigrateState, }; use sp_state_machine::{ backend::Backend as StateBackend, ChangesTrieCacheAction, ChangesTrieTransaction, @@ -2145,7 +2145,7 @@ impl sc_client_api::backend::Backend for Backend { if !has_finished_digest { info!("Migration state mismatch with final block state hash {:x?}, elapsed time: {:?}", ¤t_root, timer.elapsed()); - return Err(ClientError::InvalidMigrationState); + return Err(ClientError::InvalidMigrationState("No finished digest found on chainspec switch".to_string())); } info!("Finished processing migration with final block state hash {:x?}, elapsed time: {:?}", ¤t_root, timer.elapsed()); @@ -2452,19 +2452,9 @@ impl sc_client_api::backend::Backend for Backend { fn state_at(&self, block: BlockId) -> ClientResult { use sc_client_api::blockchain::HeaderBackend as BcHeaderBackend; - let (is_genesis, number) = match &block { - BlockId::Number(n) => (n.is_zero(), *n), - BlockId::Hash(h) if h == &self.blockchain.meta.read().genesis_hash => - (true, NumberFor::::zero()), - BlockId::Hash(h) => { - let n = self.blockchain.number(*h)?.ok_or_else(|| { - sp_blockchain::Error::UnknownBlock(format!( - "Unknown number for block hash {}", - h - )) - })?; - (false, n) - }, + let is_genesis= match &block { + BlockId::Number(n) => n.is_zero(), + BlockId::Hash(h) => h == &self.blockchain.meta.read().genesis_hash, }; if is_genesis { if let Some(genesis_state) = &*self.genesis_state.read() { @@ -2491,6 +2481,26 @@ impl sc_client_api::backend::Backend for Backend { })?, }; + // TODO remove this to cache result and use from header_metadata instead. + let mut override_state_root = None; + let state_version = if let Some(parent_header) = self.blockchain.header(BlockId::Hash(hash))? { + match self.state_versions.resolve_migrate(parent_header.number().clone() + 1u32.into(), parent_header.digest(), None) + .map_err(|err| sp_blockchain::Error::InvalidMigrationState(format!("{}", err)))? { + MigrateState::None(v) => v, + MigrateState::Switch(v, dest_root) => { + override_state_root = Some(dest_root); + v + }, + MigrateState::Start(v, _) => v, + MigrateState::Pending(v, _, _) => v, + } + } else { + return Err(sp_blockchain::Error::UnknownBlock(format!( + "No header found for {:?}", + block + ))) + }; + match self.blockchain.header_metadata(hash) { Ok(ref hdr) => { if !self.have_state_at(&hash, hdr.number) { @@ -2500,8 +2510,7 @@ impl sc_client_api::backend::Backend for Backend { ))) } if let Ok(()) = self.storage.state_db.pin(&hash) { - let root = hdr.state_root; - let state_version = self.state_versions.state_version_at(number); + let root = override_state_root.unwrap_or_else(|| hdr.state_root); let db_state = DbState::::new(self.storage.clone(), root, state_version); let state = RefTrackingState::new(db_state, self.storage.clone(), Some(hash.clone())); diff --git a/primitives/blockchain/src/error.rs b/primitives/blockchain/src/error.rs index 0a2b5a83721f7..a7e7414b95604 100644 --- a/primitives/blockchain/src/error.rs +++ b/primitives/blockchain/src/error.rs @@ -145,7 +145,7 @@ pub enum Error { InvalidStateRoot, #[error("Calculated migration state does not match.")] - InvalidMigrationState, + InvalidMigrationState(String), #[error("Incomplete block import pipeline.")] IncompletePipeline, diff --git a/primitives/runtime/src/lib.rs b/primitives/runtime/src/lib.rs index 1ed5e18eb6470..2fec58d4f695a 100644 --- a/primitives/runtime/src/lib.rs +++ b/primitives/runtime/src/lib.rs @@ -94,7 +94,7 @@ pub use sp_arithmetic::{ pub use either::Either; -pub use state_version::{StateVersion, StateVersions, StateMigrationDigest, StateMigrationProgress}; +pub use state_version::{StateVersion, StateVersions, StateMigrationDigest, StateMigrationProgress, MigrateState}; /// An abstraction over justification for a block's validity under a consensus algorithm. /// diff --git a/primitives/runtime/src/state_version.rs b/primitives/runtime/src/state_version.rs index e1422b95516f0..24b32b03ac7b3 100644 --- a/primitives/runtime/src/state_version.rs +++ b/primitives/runtime/src/state_version.rs @@ -65,6 +65,14 @@ pub enum InvalidErrorDigest { InvalidImportedHeaderDigest, } +#[cfg(feature = "std")] +impl std::fmt::Display for InvalidErrorDigest { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + let msg = " TODO implement with comments"; + write!(f, "{}", msg) + } +} + impl StateVersions { /// Access genesis state version. /// This uses default state if undefined. From 96aa05361d522b1ee3d64bd39ecbc8634b0d67ff Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 26 Aug 2021 15:05:00 +0200 Subject: [PATCH 093/127] fix --- client/block-builder/src/lib.rs | 1 + client/finality-grandpa/rpc/src/lib.rs | 1 + client/service/src/client/client.rs | 7 ++++++- test-utils/client/src/lib.rs | 1 + 4 files changed, 9 insertions(+), 1 deletion(-) diff --git a/client/block-builder/src/lib.rs b/client/block-builder/src/lib.rs index a0e9479470dea..960cc4d3b5268 100644 --- a/client/block-builder/src/lib.rs +++ b/client/block-builder/src/lib.rs @@ -377,6 +377,7 @@ mod tests { RecordProof::Yes, Default::default(), &*backend, + &client.state_versions(), ) .unwrap() .build() diff --git a/client/finality-grandpa/rpc/src/lib.rs b/client/finality-grandpa/rpc/src/lib.rs index b8b8b2d956463..a0dc5449f2f8c 100644 --- a/client/finality-grandpa/rpc/src/lib.rs +++ b/client/finality-grandpa/rpc/src/lib.rs @@ -415,6 +415,7 @@ mod tests { RecordProof::No, Default::default(), &*backend, + &Default::default(), ) .unwrap() .build() diff --git a/client/service/src/client/client.rs b/client/service/src/client/client.rs index 7839cbfca7897..0edd775d1ca54 100644 --- a/client/service/src/client/client.rs +++ b/client/service/src/client/client.rs @@ -116,7 +116,7 @@ where importing_block: RwLock>, block_rules: BlockRules, execution_extensions: ExecutionExtensions, - config: ClientConfig, + config: ClientConfig, telemetry: Option, _phantom: PhantomData, } @@ -1266,6 +1266,11 @@ where trace!("Collected {} uncles", uncles.len()); Ok(uncles) } + + /// Access to configured state versions. + pub fn state_versions(&self) -> &sp_runtime::StateVersions { + &self.config.state_versions + } } impl UsageProvider for Client diff --git a/test-utils/client/src/lib.rs b/test-utils/client/src/lib.rs index 685967798d0d6..946f828bcc0ae 100644 --- a/test-utils/client/src/lib.rs +++ b/test-utils/client/src/lib.rs @@ -234,6 +234,7 @@ impl } /// Enable the internal value hash of state. + /// TODO remove and use state_versions instead pub fn state_hashed_value(mut self) -> Self { self.state_hashed_value = true; self From d33dbc73fcdf21650a0ed8b291a49605e5cdda6e Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 26 Aug 2021 15:38:13 +0200 Subject: [PATCH 094/127] fix for empty --- primitives/runtime/src/state_version.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/primitives/runtime/src/state_version.rs b/primitives/runtime/src/state_version.rs index 24b32b03ac7b3..9004b2bccf337 100644 --- a/primitives/runtime/src/state_version.rs +++ b/primitives/runtime/src/state_version.rs @@ -152,7 +152,7 @@ impl StateVersions { // TODO switch to default when all V0 chains did migrate. let mut from = StateVersion::V0; let mut to = None; - let mut first = true; + let mut first = false; let at_plus_one = at + 1u32.into(); for (number, version) in self.canonical_states.iter() { first = number == &at; From 85e72b9c90afa61cf7acaf0c2a2d6ec2c915992d Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 26 Aug 2021 17:05:33 +0200 Subject: [PATCH 095/127] update trie dep --- Cargo.lock | 16 ++++++++-------- primitives/trie/src/node_codec.rs | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3a8f62961afec..db2106996b877 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2441,7 +2441,7 @@ dependencies = [ [[package]] name = "hash-db" version = "0.15.2" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#618beec2d30931fb2aa7ee371012be5970ff20a6" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#20cac451adff73c736680b8e067cc11b43de4499" [[package]] name = "hash256-std-hasher" @@ -2455,7 +2455,7 @@ dependencies = [ [[package]] name = "hash256-std-hasher" version = "0.15.2" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#618beec2d30931fb2aa7ee371012be5970ff20a6" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#20cac451adff73c736680b8e067cc11b43de4499" dependencies = [ "crunchy", ] @@ -3108,7 +3108,7 @@ dependencies = [ [[package]] name = "keccak-hasher" version = "0.15.3" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#618beec2d30931fb2aa7ee371012be5970ff20a6" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#20cac451adff73c736680b8e067cc11b43de4499" dependencies = [ "hash-db", "hash256-std-hasher 0.15.2 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple4)", @@ -3969,7 +3969,7 @@ dependencies = [ [[package]] name = "memory-db" version = "0.27.0" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#618beec2d30931fb2aa7ee371012be5970ff20a6" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#20cac451adff73c736680b8e067cc11b43de4499" dependencies = [ "hash-db", "hashbrown 0.11.2", @@ -10480,7 +10480,7 @@ checksum = "a7f741b240f1a48843f9b8e0444fb55fb2a4ff67293b50a9179dfd5ea67f8d41" [[package]] name = "trie-bench" version = "0.28.0" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#618beec2d30931fb2aa7ee371012be5970ff20a6" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#20cac451adff73c736680b8e067cc11b43de4499" dependencies = [ "criterion", "hash-db", @@ -10495,7 +10495,7 @@ dependencies = [ [[package]] name = "trie-db" version = "0.22.6" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#618beec2d30931fb2aa7ee371012be5970ff20a6" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#20cac451adff73c736680b8e067cc11b43de4499" dependencies = [ "hash-db", "hashbrown 0.11.2", @@ -10507,7 +10507,7 @@ dependencies = [ [[package]] name = "trie-root" version = "0.16.0" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#618beec2d30931fb2aa7ee371012be5970ff20a6" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#20cac451adff73c736680b8e067cc11b43de4499" dependencies = [ "hash-db", ] @@ -10525,7 +10525,7 @@ dependencies = [ [[package]] name = "trie-standardmap" version = "0.15.2" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#618beec2d30931fb2aa7ee371012be5970ff20a6" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#20cac451adff73c736680b8e067cc11b43de4499" dependencies = [ "hash-db", "keccak-hasher 0.15.3 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple4)", diff --git a/primitives/trie/src/node_codec.rs b/primitives/trie/src/node_codec.rs index 1c6a2bec72924..d23ed0d6af801 100644 --- a/primitives/trie/src/node_codec.rs +++ b/primitives/trie/src/node_codec.rs @@ -175,7 +175,7 @@ impl NodeCodecT for NodeCodec where H: Hasher, { - const OFFSET_CONTAINS_HASH: usize = 1; + const OFFSET_IF_CONTAINS_HASH: usize = 1; type Error = Error; type HashOut = H::Out; From 8ffd9e28292e1fb4d1ddb0c9e811857da17ffe9b Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 26 Aug 2021 21:34:34 +0200 Subject: [PATCH 096/127] to revert --- primitives/state-machine/src/trie_backend.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/primitives/state-machine/src/trie_backend.rs b/primitives/state-machine/src/trie_backend.rs index 88101fe315d93..ef133a6334119 100644 --- a/primitives/state-machine/src/trie_backend.rs +++ b/primitives/state-machine/src/trie_backend.rs @@ -456,7 +456,8 @@ where progress.root = Some(self.essence.root().clone()); } let dest_root = progress.root.as_mut().expect("Lazy init above."); - let ori = TrieDB::new_with_layout(&ori_db, &ori_root, Layout::default()) + let in_mem2 = sp_trie::PrefixedMemoryDB::::default(); + let ori = TrieDB::new_with_layout(&in_mem2, &ori_root, Layout::default()) .map_err(|e| trie_error::(&*e))?; let mut dest = TrieDBMut::from_existing_with_layout(&mut dest_db, dest_root, dest_layout) .map_err(|e| trie_error::(&*e))?; From c57a82870f7faa06fbd7c6e7dab5601ccc7be527 Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 26 Aug 2021 22:14:34 +0200 Subject: [PATCH 097/127] simplify (migrate from backend and inject payload if needed. --- primitives/state-machine/src/trie_backend.rs | 45 +++++++++----------- 1 file changed, 21 insertions(+), 24 deletions(-) diff --git a/primitives/state-machine/src/trie_backend.rs b/primitives/state-machine/src/trie_backend.rs index ef133a6334119..9fc67ae35c15d 100644 --- a/primitives/state-machine/src/trie_backend.rs +++ b/primitives/state-machine/src/trie_backend.rs @@ -28,7 +28,7 @@ use sp_core::{ state_version::StateVersion, storage::{ChildInfo, ChildType}, }; -use sp_std::{boxed::Box, cell::RefCell, rc::Rc, vec::Vec}; +use sp_std::{boxed::Box, vec::Vec}; use sp_trie::{ child_delta_trie_root, delta_trie_root, empty_child_trie_root, trie_types::{TrieDB, TrieError}, @@ -306,7 +306,7 @@ pub struct MigrateStorage<'a, S, H> where H: Hasher, { - overlay: Rc>>, + overlay: &'a mut sp_trie::PrefixedMemoryDB, storage: &'a S, } @@ -315,12 +315,6 @@ where unsafe impl<'a, S, H: Hasher> Send for MigrateStorage<'a, S, H> {} unsafe impl<'a, S, H: Hasher> Sync for MigrateStorage<'a, S, H> {} -impl<'a, S, H: Hasher> Clone for MigrateStorage<'a, S, H> { - fn clone(&self) -> Self { - MigrateStorage { overlay: self.overlay.clone(), storage: self.storage } - } -} - use hash_db::{self, AsHashDB, HashDB, HashDBRef, Prefix}; use sp_trie::DBValue; @@ -328,7 +322,7 @@ impl<'a, S: 'a + TrieBackendStorage, H: Hasher> hash_db::HashDB for MigrateStorage<'a, S, H> { fn get(&self, key: &H::Out, prefix: Prefix) -> Option { - if let Some(val) = HashDB::get(*self.overlay.borrow(), key, prefix) { + if let Some(val) = HashDB::get(self.overlay, key, prefix) { Some(val) } else { match self.storage.get(&key, prefix) { @@ -352,19 +346,19 @@ impl<'a, S: 'a + TrieBackendStorage, H: Hasher> hash_db::HashDB } fn insert(&mut self, prefix: Prefix, value: &[u8]) -> H::Out { - HashDB::insert(*self.overlay.borrow_mut(), prefix, value) + HashDB::insert(self.overlay, prefix, value) } fn emplace(&mut self, key: H::Out, prefix: Prefix, value: DBValue) { - HashDB::emplace(*self.overlay.borrow_mut(), key, prefix, value) + HashDB::emplace(self.overlay, key, prefix, value) } fn emplace_ref(&mut self, key: &H::Out, prefix: Prefix, value: &[u8]) { - HashDB::emplace_ref(*self.overlay.borrow_mut(), key, prefix, value) + HashDB::emplace_ref(self.overlay, key, prefix, value) } fn remove(&mut self, key: &H::Out, prefix: Prefix) { - HashDB::remove(*self.overlay.borrow_mut(), key, prefix) + HashDB::remove(self.overlay, key, prefix) } } @@ -440,24 +434,24 @@ where let empty_value2 = sp_std::vec![0u8; dest_threshold as usize]; let dest_layout = Layout::with_alt_hashing(dest_threshold); let mut in_mem; + let without_payload; let overlay = if let Some(change) = do_update { - Rc::new(RefCell::new(change)) + without_payload = false; + change } else { + without_payload = true; in_mem = sp_trie::PrefixedMemoryDB::default(); - Rc::new(RefCell::new(&mut in_mem)) + &mut in_mem }; + // Using same db with remove node is ok, but require locking state pruning. let mut dest_db = MigrateStorage:: { overlay, storage: self.essence.backend_storage() }; - let ori_root = self.essence.root().clone(); - // Using db with remove node is ok as long as iteration - // start at uncommited key. - let ori_db = dest_db.clone(); if progress.root.is_none() { progress.root = Some(self.essence.root().clone()); } let dest_root = progress.root.as_mut().expect("Lazy init above."); - let in_mem2 = sp_trie::PrefixedMemoryDB::::default(); - let ori = TrieDB::new_with_layout(&in_mem2, &ori_root, Layout::default()) + let ori_root = self.essence.root().clone(); + let ori = TrieDB::new_with_layout(&self.essence, &ori_root, Layout::default()) .map_err(|e| trie_error::(&*e))?; let mut dest = TrieDBMut::from_existing_with_layout(&mut dest_db, dest_root, dest_layout) .map_err(|e| trie_error::(&*e))?; @@ -503,15 +497,18 @@ where // or may have change a previous block migration change).c for (key, value) in previous_block_updates.0.iter() { if let Some(value) = value.as_ref() { - if value.len() >= dest_threshold as usize { + if without_payload || value.len() >= dest_threshold as usize { if progress.current_top.as_ref().map(|end| key < end).unwrap_or(true) { dest.insert(key.as_slice(), value.as_slice()) .map_err(|e| trie_error::(&*e))?; } } } else { - dest.remove(key.as_slice()) - .map_err(|e| trie_error::(&*e))?; + if progress.current_top.as_ref().map(|end| key < end).unwrap_or(true) { + // rerun all remove in case we import old. . + dest.remove(key.as_slice()) + .map_err(|e| trie_error::(&*e))?; + } } } // TODO same for child From b69ea5f7a39580c24c3b9dd225852fc34ed105fc Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 26 Aug 2021 22:18:26 +0200 Subject: [PATCH 098/127] useless unsafe --- primitives/state-machine/src/trie_backend.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/primitives/state-machine/src/trie_backend.rs b/primitives/state-machine/src/trie_backend.rs index 9fc67ae35c15d..ec15fb036dfac 100644 --- a/primitives/state-machine/src/trie_backend.rs +++ b/primitives/state-machine/src/trie_backend.rs @@ -310,11 +310,6 @@ where storage: &'a S, } -// type is neither send nor sync but only ever -// use in migrate method local to a single thread. -unsafe impl<'a, S, H: Hasher> Send for MigrateStorage<'a, S, H> {} -unsafe impl<'a, S, H: Hasher> Sync for MigrateStorage<'a, S, H> {} - use hash_db::{self, AsHashDB, HashDB, HashDBRef, Prefix}; use sp_trie::DBValue; From c2d9fefb6a41b2b6d9b05888d7339d86ccb1758c Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 26 Aug 2021 22:46:41 +0200 Subject: [PATCH 099/127] to test --- primitives/state-machine/src/trie_backend.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/primitives/state-machine/src/trie_backend.rs b/primitives/state-machine/src/trie_backend.rs index ec15fb036dfac..2e3791f7d5038 100644 --- a/primitives/state-machine/src/trie_backend.rs +++ b/primitives/state-machine/src/trie_backend.rs @@ -451,10 +451,8 @@ where let mut dest = TrieDBMut::from_existing_with_layout(&mut dest_db, dest_root, dest_layout) .map_err(|e| trie_error::(&*e))?; - let iter = TrieDBIterator::new_prefixed_then_seek( + let iter = TrieDBIterator::new( &ori, - &[], - progress.current_top.as_ref().map(|s| s.as_slice()).unwrap_or(&[]), ) .map_err(|e| trie_error::(&*e))?; for elt in iter { From 94c1f6ce5869cc6ba5b34477c1d95ba901a1b6b1 Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 26 Aug 2021 23:28:53 +0200 Subject: [PATCH 100/127] Revert "to test" This reverts commit c2d9fefb6a41b2b6d9b05888d7339d86ccb1758c. --- primitives/state-machine/src/trie_backend.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/primitives/state-machine/src/trie_backend.rs b/primitives/state-machine/src/trie_backend.rs index 2e3791f7d5038..ec15fb036dfac 100644 --- a/primitives/state-machine/src/trie_backend.rs +++ b/primitives/state-machine/src/trie_backend.rs @@ -451,8 +451,10 @@ where let mut dest = TrieDBMut::from_existing_with_layout(&mut dest_db, dest_root, dest_layout) .map_err(|e| trie_error::(&*e))?; - let iter = TrieDBIterator::new( + let iter = TrieDBIterator::new_prefixed_then_seek( &ori, + &[], + progress.current_top.as_ref().map(|s| s.as_slice()).unwrap_or(&[]), ) .map_err(|e| trie_error::(&*e))?; for elt in iter { From 7805fa7bf3358872b1565a8bcd674d710a2e6246 Mon Sep 17 00:00:00 2001 From: cheme Date: Fri, 27 Aug 2021 00:09:43 +0200 Subject: [PATCH 101/127] fix state display --- client/db/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index d3ac42d169ce0..29632add74017 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -2115,11 +2115,11 @@ impl sc_client_api::backend::Backend for Backend { )) })?; - info!("Finished migration iteration, new state root: {:x?}, elapsed time: {:?}.", ¤t_root, timer.elapsed()); start_top = current_top; start_child = current_child; current_root = root.unwrap_or_else(|| pending_block.header.state_root().clone()); + info!("Finished migration iteration, new state root: {:x?}, elapsed time: {:?}.", ¤t_root, timer.elapsed()); if start_top.is_none() { break } From 1dd52abc036750d5836de2360d06a7cd8f963f9b Mon Sep 17 00:00:00 2001 From: cheme Date: Fri, 27 Aug 2021 14:52:49 +0200 Subject: [PATCH 102/127] quick additional logging --- primitives/state-machine/src/trie_backend.rs | 24 ++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/primitives/state-machine/src/trie_backend.rs b/primitives/state-machine/src/trie_backend.rs index ec15fb036dfac..ec7244b9f567c 100644 --- a/primitives/state-machine/src/trie_backend.rs +++ b/primitives/state-machine/src/trie_backend.rs @@ -34,6 +34,8 @@ use sp_trie::{ trie_types::{TrieDB, TrieError}, Layout, Trie, }; +#[cfg(feature = "std")] +use log::info; /// Patricia trie-based backend. Transaction type is an overlay of changes to commit. pub struct TrieBackend, H: Hasher> { @@ -457,6 +459,11 @@ where progress.current_top.as_ref().map(|s| s.as_slice()).unwrap_or(&[]), ) .map_err(|e| trie_error::(&*e))?; + + let mut total_top_size = 0u64; + let mut total_written_top_size = 0u64; + let mut total_top_items = 0u64; + let mut total_written_top_items = 0u64; for elt in iter { let (key, value) = elt.map_err(|e| trie_error::(&*e))?; if let Some(limit_size) = limit_size.as_mut() { @@ -473,7 +480,11 @@ where } *limit_items -= 1; } + total_top_size += value.len() as u64; + total_top_items += 1; if value.len() >= dest_threshold as usize { + total_written_top_size += value.len() as u64; + total_written_top_items += 1; // Force a change so triedbmut do not ignore // setting the same value. if value != empty_value { @@ -509,6 +520,19 @@ where // TODO same for child sp_std::mem::drop(ori); sp_std::mem::drop(dest); + #[cfg(feature = "std")] + info!("\n + total_top_size: {},\n + total_written_top_size: {},\n + total_top_items: {}, \n + total_written_top_items: {}, \n + ", + total_top_size, + total_written_top_size, + total_top_items, + total_written_top_items, + ); + Ok(progress) } } From a48970fd8731e964823e7596ea5a387b72662f1e Mon Sep 17 00:00:00 2001 From: cheme Date: Fri, 27 Aug 2021 15:14:27 +0200 Subject: [PATCH 103/127] fix --- primitives/state-machine/src/proving_backend.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/primitives/state-machine/src/proving_backend.rs b/primitives/state-machine/src/proving_backend.rs index ee097b8d89060..235f37b1c9c1e 100644 --- a/primitives/state-machine/src/proving_backend.rs +++ b/primitives/state-machine/src/proving_backend.rs @@ -484,7 +484,7 @@ mod tests { .clone() .map(|i| (vec![i], Some(vec![i; size_content]))) .collect::>(); - let in_memory = InMemoryBackend::::default(); + let mut in_memory = InMemoryBackend::::default(); if flagged { in_memory = in_memory.update(vec![( None, @@ -496,7 +496,7 @@ mod tests { )], )]); } - let mut in_memory = in_memory.update(vec![(None, contents)]); + let in_memory = in_memory.update(vec![(None, contents)]); let in_memory_root = in_memory.storage_root(std::iter::empty()).0; value_range.clone().for_each(|i| { assert_eq!(in_memory.storage(&[i]).unwrap().unwrap(), vec![i; size_content]) @@ -534,7 +534,7 @@ mod tests { (Some(child_info_1.clone()), (28..65).map(|i| (vec![i], Some(vec![i]))).collect()), (Some(child_info_2.clone()), (10..15).map(|i| (vec![i], Some(vec![i]))).collect()), ]; - let in_memory = InMemoryBackend::::default(); + let mut in_memory = InMemoryBackend::::default(); if flagged { in_memory = in_memory.update(vec![( None, From a5e68583b8f6b33a70a231c399524b95423ac822 Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 30 Aug 2021 17:51:59 +0200 Subject: [PATCH 104/127] TODO about fundamental design issue --- client/db/src/lib.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index 29632add74017..a3fe8980c9342 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -2056,6 +2056,12 @@ impl sc_client_api::backend::Backend for Backend { operation: &mut Self::BlockImportOperation, block: BlockId, ) -> ClientResult<()> { + // TODO with correct executive (not testnet), this would require removing form pending_block + // first the migration progress digest (so digest checks from executive point of view). + // + // TODO actually the parent block check from executive would not pass anyway: this would then + // fail anyway unless runing the migration BEFORE the block to change backend state writen + // by executive. if block.is_pre_genesis() { operation.old_state = self.empty_state()?; } else { From 2178698d719f982c668b32e2dc39a12a1d1dfed1 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 31 Aug 2021 13:15:29 +0200 Subject: [PATCH 105/127] Remove migration related code. --- client/api/src/in_mem.rs | 34 +---- client/block-builder/src/lib.rs | 66 +--------- client/db/src/lib.rs | 111 +--------------- client/finality-grandpa/rpc/src/lib.rs | 1 - client/service/src/client/client.rs | 2 - primitives/runtime/src/generic/digest.rs | 17 --- primitives/runtime/src/lib.rs | 2 +- primitives/runtime/src/state_version.rs | 156 ----------------------- 8 files changed, 9 insertions(+), 380 deletions(-) diff --git a/client/api/src/in_mem.rs b/client/api/src/in_mem.rs index f7f9efe3e6d5a..7eb2549723061 100644 --- a/client/api/src/in_mem.rs +++ b/client/api/src/in_mem.rs @@ -26,7 +26,7 @@ use sp_core::{ use sp_runtime::{ generic::BlockId, traits::{Block as BlockT, HashFor, Header as HeaderT, NumberFor, Zero}, - Justification, Justifications, MigrateState, StateVersion, StateVersions, Storage, + Justification, Justifications, StateVersion, StateVersions, Storage, }; use sp_state_machine::{ Backend as StateBackend, ChangesTrieTransaction, ChildStorageCollection, InMemoryBackend, @@ -113,11 +113,6 @@ struct BlockchainStorage { leaves: LeafSet>, aux: HashMap, Vec>, state_versions: StateVersions, - /* - // TODO not sure if migration support. TODO replace () by StateMigration. - state_migration: Vec<(NumberFor, ())>, - * Probably not (client/api). - */ } /// In-memory blockchain. Supports concurrent reads. @@ -827,33 +822,6 @@ where None => old_state.clone(), }; - if let Some(parent_header) = - self.blockchain.header(BlockId::Hash(header.parent_hash().clone()))? - { - match self - .blockchain - .storage - .read() - .state_versions - .resolve_migrate( - parent_header.number().clone() + 1u32.into(), - parent_header.digest(), - None, - ) - .map_err(|err| { - sp_blockchain::Error::InvalidMigrationState(format!("{}", err)) - })? { - MigrateState::None(..) => (), - MigrateState::Start(..) - | MigrateState::Pending(..) - | MigrateState::Switch(..) => { - return Err(sp_blockchain::Error::InvalidMigrationState( - "Migration unsupported for in memory client".to_string(), - )); - } - } - } - self.states.write().insert(hash, new_state); self.blockchain.insert(hash, header, justification, body, pending_block.state)?; diff --git a/client/block-builder/src/lib.rs b/client/block-builder/src/lib.rs index 960cc4d3b5268..f8bc48d427ad7 100644 --- a/client/block-builder/src/lib.rs +++ b/client/block-builder/src/lib.rs @@ -34,7 +34,7 @@ use sp_api::{ use sp_blockchain::{ApplyExtrinsicFailed, Error}; use sp_core::ExecutionContext; use sp_runtime::{ - generic::BlockId, StateVersions, + generic::BlockId, traits::{Block as BlockT, DigestFor, Hash, HashFor, Header as HeaderT, NumberFor, One}, }; @@ -139,7 +139,6 @@ pub struct BlockBuilder<'a, Block: BlockT, A: ProvideRuntimeApi, B> { backend: &'a B, /// The estimated size of the block header. estimated_header_size: usize, - state_versions: &'a StateVersions, } impl<'a, Block, A, B> BlockBuilder<'a, Block, A, B> @@ -162,7 +161,6 @@ where record_proof: RecordProof, inherent_digests: DigestFor, backend: &'a B, - state_versions: &'a StateVersions, ) -> Result { // TODO call need_migration here and remove inner field // then also compare (or init from) with parent header migration state. @@ -195,7 +193,6 @@ where block_id, backend, estimated_header_size, - state_versions, }) } @@ -230,7 +227,7 @@ where /// supplied by `self.api`, combined as [`BuiltBlock`]. /// The storage proof will be `Some(_)` when proof recording was enabled. pub fn build(mut self) -> Result>, Error> { - let mut header = self + let header = self .api .finalize_block_with_context(&self.block_id, ExecutionContext::BlockConstruction)?; @@ -255,64 +252,6 @@ where .into_storage_changes(&state, changes_trie_state.as_ref(), parent_hash) .map_err(|e| sp_blockchain::Error::StorageChanges(e))?; - let number = header.number().clone(); - if let Some((migration_from, migration_to)) = self.state_versions.need_migrate(number) { - // TODO remove code duplicate with client/db/src/lib import block - use sp_api::StateBackend; - if let Some(prev_block_state) = state.as_trie_backend() { - let limit_size: Option = None; - let limit_items: Option = None; - let init_root = header.state_root().clone(); - let mut current_root = init_root.clone(); - let mut start_top = None; - let mut start_child = None; - - loop { - let sp_state_machine::MigrateProgress { current_top, current_child, root } = - prev_block_state - .migrate( - migration_from, - migration_to, - limit_size, - limit_items, - //Some(&mut storage_changes.transaction), - None, - (&storage_changes.main_storage_changes, &storage_changes.child_storage_changes), - sp_state_machine::MigrateProgress { - current_top: start_top.take(), - current_child: start_child.take(), - root: Some(current_root), // TODO non optional rather? - }, - ) - .map_err(|err| { - sp_blockchain::Error::Backend(format!( - "error migrating: {}", - err - )) - })?; - - start_top = current_top; - start_child = current_child; - current_root = - root.unwrap_or_else(|| header.state_root().clone()); - if start_top.is_none() { - break - } - } - - if start_top.is_some() { - unreachable!("TODO save progress in dest state in migrate fn and put a differnt progress digest"); - } - StateVersions::::set_migrate_digest(header.digest_mut(), sp_runtime::StateMigrationDigest { - from: migration_from, - to: migration_to, - state_root: current_root, - progress: sp_runtime::StateMigrationProgress::Finished, - }); - } else { - panic!("Could not migrate at block: {:?}", number); - } - } Ok(BuiltBlock { block: ::new(header, self.extrinsics), storage_changes, @@ -377,7 +316,6 @@ mod tests { RecordProof::Yes, Default::default(), &*backend, - &client.state_versions(), ) .unwrap() .build() diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index a3fe8980c9342..fad486d47e5dd 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -46,7 +46,7 @@ mod upgrade; mod utils; use linked_hash_map::LinkedHashMap; -use log::{debug, info, trace, warn}; +use log::{debug, trace, warn}; use parking_lot::{Mutex, RwLock}; use std::{ collections::{HashMap, HashSet}, @@ -88,7 +88,7 @@ use sp_runtime::{ Block as BlockT, Hash, HashFor, Header as HeaderT, NumberFor, One, SaturatedConversion, Zero, }, - Justification, Justifications, StateVersion, StateVersions, Storage, MigrateState, + Justification, Justifications, StateVersion, StateVersions, Storage, }; use sp_state_machine::{ backend::Backend as StateBackend, ChangesTrieCacheAction, ChangesTrieTransaction, @@ -2075,92 +2075,10 @@ impl sc_client_api::backend::Backend for Backend { Ok(()) } - fn commit_operation(&self, mut operation: Self::BlockImportOperation) -> ClientResult<()> { + fn commit_operation(&self, operation: Self::BlockImportOperation) -> ClientResult<()> { let usage = operation.old_state.usage_info(); self.state_usage.merge_sm(usage); - if let Some(pending_block) = operation.pending_block.as_mut() { - let number = pending_block.header.number().clone(); - - if let Some((migration_from, migration_to)) = self.state_versions.need_migrate(number) { - use std::time; - if let Some(prev_block_state) = operation.old_state.as_trie_backend() { - let limit_size: Option = None; - let limit_items: Option = None; - let init_root = pending_block.header.state_root().clone(); - let mut current_root = init_root.clone(); - let mut start_top = None; - let mut start_child = None; - - let timer = time::SystemTime::now(); - - info!( - "Starting migrating from state root {:x?} at block {:?}", - ¤t_root, &number - ); - loop { - let sp_state_machine::MigrateProgress { current_top, current_child, root } = - prev_block_state - .migrate( - migration_from, - migration_to, - limit_size, - limit_items, - Some(&mut operation.db_updates), - (&operation.storage_updates, &operation.child_storage_updates), - sp_state_machine::MigrateProgress { - current_top: start_top.take(), - current_child: start_child.take(), - root: Some(current_root), // TODO non optional rather? - }, - ) - .map_err(|err| { - sp_blockchain::Error::Backend(format!( - "error migrating: {}", - err - )) - })?; - - start_top = current_top; - start_child = current_child; - current_root = - root.unwrap_or_else(|| pending_block.header.state_root().clone()); - info!("Finished migration iteration, new state root: {:x?}, elapsed time: {:?}.", ¤t_root, timer.elapsed()); - if start_top.is_none() { - break - } - } - - if start_top.is_some() { - unreachable!("TODO save progress in dest state in migrate fn and put a differnt progress digest"); - } - - // TODO ensure no duplicate when setting. - let has_finished_digest = match StateVersions::::migrate_digest(pending_block.header.digest()) { - Some(sp_runtime::StateMigrationDigest { - from, - to, - state_root, - progress, - }) => from == &migration_from - && to == &migration_to - && state_root == ¤t_root - && progress == &sp_runtime::StateMigrationProgress::Finished, - _ => false, - }; - - if !has_finished_digest { - info!("Migration state mismatch with final block state hash {:x?}, elapsed time: {:?}", ¤t_root, timer.elapsed()); - return Err(ClientError::InvalidMigrationState("No finished digest found on chainspec switch".to_string())); - } - - info!("Finished processing migration with final block state hash {:x?}, elapsed time: {:?}", ¤t_root, timer.elapsed()); - } else { - panic!("Could not migrate at block: {:?}", number); - } - } - } - match self.try_commit_operation(operation) { Ok(_) => { self.storage.state_db.apply_pending(); @@ -2487,26 +2405,6 @@ impl sc_client_api::backend::Backend for Backend { })?, }; - // TODO remove this to cache result and use from header_metadata instead. - let mut override_state_root = None; - let state_version = if let Some(parent_header) = self.blockchain.header(BlockId::Hash(hash))? { - match self.state_versions.resolve_migrate(parent_header.number().clone() + 1u32.into(), parent_header.digest(), None) - .map_err(|err| sp_blockchain::Error::InvalidMigrationState(format!("{}", err)))? { - MigrateState::None(v) => v, - MigrateState::Switch(v, dest_root) => { - override_state_root = Some(dest_root); - v - }, - MigrateState::Start(v, _) => v, - MigrateState::Pending(v, _, _) => v, - } - } else { - return Err(sp_blockchain::Error::UnknownBlock(format!( - "No header found for {:?}", - block - ))) - }; - match self.blockchain.header_metadata(hash) { Ok(ref hdr) => { if !self.have_state_at(&hash, hdr.number) { @@ -2516,7 +2414,8 @@ impl sc_client_api::backend::Backend for Backend { ))) } if let Ok(()) = self.storage.state_db.pin(&hash) { - let root = override_state_root.unwrap_or_else(|| hdr.state_root); + let root = hdr.state_root; + let state_version = self.state_versions.state_version_at(hdr.number); let db_state = DbState::::new(self.storage.clone(), root, state_version); let state = RefTrackingState::new(db_state, self.storage.clone(), Some(hash.clone())); diff --git a/client/finality-grandpa/rpc/src/lib.rs b/client/finality-grandpa/rpc/src/lib.rs index a0dc5449f2f8c..b8b8b2d956463 100644 --- a/client/finality-grandpa/rpc/src/lib.rs +++ b/client/finality-grandpa/rpc/src/lib.rs @@ -415,7 +415,6 @@ mod tests { RecordProof::No, Default::default(), &*backend, - &Default::default(), ) .unwrap() .build() diff --git a/client/service/src/client/client.rs b/client/service/src/client/client.rs index 0edd775d1ca54..0a5359be6095e 100644 --- a/client/service/src/client/client.rs +++ b/client/service/src/client/client.rs @@ -1432,7 +1432,6 @@ where record_proof.into(), inherent_digests, &self.backend, - &self.config.state_versions, ) } @@ -1448,7 +1447,6 @@ where RecordProof::No, inherent_digests, &self.backend, - &self.config.state_versions, ) } } diff --git a/primitives/runtime/src/generic/digest.rs b/primitives/runtime/src/generic/digest.rs index 8323dbf9b6713..390acb87f690d 100644 --- a/primitives/runtime/src/generic/digest.rs +++ b/primitives/runtime/src/generic/digest.rs @@ -116,11 +116,6 @@ pub enum DigestItem { /// native code. ChangesTrieSignal(ChangesTrieSignal), - /// System digest item that contains pending migration state. - /// This allows keeping trace of migration progress (root of - /// pending new state), and switching root to use on completion. - StateMigration(crate::state_version::StateMigrationDigest), - /// Some other thing. Unsupported and experimental. Other(Vec), } @@ -187,10 +182,6 @@ pub enum DigestItemRef<'a, Hash: 'a> { /// Digest item that contains signal from changes tries manager to the /// native code. ChangesTrieSignal(&'a ChangesTrieSignal), - /// System digest item that contains pending migration state. - /// This allows keeping trace of migration progress (root of - /// pending new state), and switching root to use on completion. - StateMigration(&'a crate::state_version::StateMigrationDigest), /// Any 'non-system' digest item, opaque to the native code. Other(&'a Vec), } @@ -208,7 +199,6 @@ pub enum DigestItemType { Seal = 5, PreRuntime = 6, ChangesTrieSignal = 7, - StateMigration = 8, } /// Type of a digest item that contains raw data; this also names the consensus engine ID where @@ -234,7 +224,6 @@ impl DigestItem { Self::Consensus(ref v, ref s) => DigestItemRef::Consensus(v, s), Self::Seal(ref v, ref s) => DigestItemRef::Seal(v, s), Self::ChangesTrieSignal(ref s) => DigestItemRef::ChangesTrieSignal(s), - Self::StateMigration(ref s) => DigestItemRef::StateMigration(s), Self::Other(ref v) => DigestItemRef::Other(v), } } @@ -332,8 +321,6 @@ impl Decode for DigestItem { }, DigestItemType::ChangesTrieSignal => Ok(Self::ChangesTrieSignal(Decode::decode(input)?)), - DigestItemType::StateMigration => - Ok(Self::StateMigration(Decode::decode(input)?)), DigestItemType::Other => Ok(Self::Other(Decode::decode(input)?)), } } @@ -466,10 +453,6 @@ impl<'a, Hash: Encode> Encode for DigestItemRef<'a, Hash> { DigestItemType::ChangesTrieSignal.encode_to(&mut v); changes_trie_signal.encode_to(&mut v); }, - Self::StateMigration(progress) => { - DigestItemType::StateMigration.encode_to(&mut v); - progress.encode_to(&mut v); - }, Self::Other(val) => { DigestItemType::Other.encode_to(&mut v); val.encode_to(&mut v); diff --git a/primitives/runtime/src/lib.rs b/primitives/runtime/src/lib.rs index 2fec58d4f695a..ee7871342b975 100644 --- a/primitives/runtime/src/lib.rs +++ b/primitives/runtime/src/lib.rs @@ -94,7 +94,7 @@ pub use sp_arithmetic::{ pub use either::Either; -pub use state_version::{StateVersion, StateVersions, StateMigrationDigest, StateMigrationProgress, MigrateState}; +pub use state_version::{StateVersion, StateVersions}; /// An abstraction over justification for a block's validity under a consensus algorithm. /// diff --git a/primitives/runtime/src/state_version.rs b/primitives/runtime/src/state_version.rs index 9004b2bccf337..79a80458a3cff 100644 --- a/primitives/runtime/src/state_version.rs +++ b/primitives/runtime/src/state_version.rs @@ -18,8 +18,6 @@ //! Substrate state versioning and migrations related types. use crate::traits::{Block, NumberFor}; -use crate::generic::{Digest, DigestItem}; -use codec::{Decode, Encode}; use sp_arithmetic::traits::Zero; pub use sp_core::state_version::{StateVersion, DEFAULT_STATE_VERSION}; use sp_std::{str::FromStr, vec::Vec}; @@ -139,158 +137,4 @@ impl StateVersions { } Some(StateVersions { canonical_states }) } - - /// Indicate if block need migration. - /// Returns version to migrate from and to if - /// migration is needed. - pub fn need_migrate(&self, at: NumberFor) -> Option<(StateVersion, StateVersion)> { - let (from, to, _) = self.need_migrate_inner(at); - to.map(|to| (from, to)) - } - - fn need_migrate_inner(&self, at: NumberFor) -> (StateVersion, Option, bool) { - // TODO switch to default when all V0 chains did migrate. - let mut from = StateVersion::V0; - let mut to = None; - let mut first = false; - let at_plus_one = at + 1u32.into(); - for (number, version) in self.canonical_states.iter() { - first = number == &at; - if number == &at_plus_one { - to = Some(*version); - break; - } - if number > &at_plus_one { - break; - } - from = *version; - } - (from, to, first) - } - - /// TODO remove pub and replace by resolve_migrate (and cache result). - pub fn migrate_digest(digests: &Digest) -> Option<&StateMigrationDigest> { - digests.log(|digest| if let DigestItem::StateMigration(digest) = digest { - Some(digest) - } else { - None - }) - } - - /// Push or update migration digest. - pub fn set_migrate_digest(digests: &mut Digest, migration_digest: StateMigrationDigest) { - for digest in digests.logs.iter_mut() { - if let DigestItem::StateMigration(digest) = digest { - *digest = migration_digest; - return - } - } - digests.logs.push(DigestItem::StateMigration(migration_digest)); - } - - /// Get migration specific block behavior. - pub fn resolve_migrate( - &self, - at: NumberFor, - parent_digests: &Digest, - imported_block_digests: Option<&Digest>, - ) -> Result, InvalidErrorDigest> { - if at.is_zero() { - // No migration at genesis (a bit useless since parent - // digest is undefiend, but one could use default). - // So ignoring input digest could also error. - return Ok(MigrateState::None(self.state_version_at(at))); - } - match self.need_migrate_inner(at) { - (from, Some(to), false) => { - // no migration digest allowed (only start at chainspec -1 and - // no consecutive mig support). and no from runtime trigger. - if Self::migrate_digest(parent_digests).is_some() { - return Err(InvalidErrorDigest::InvalidParent); - } - - - if let Some(digests) = imported_block_digests { - if Self::migrate_digest(digests).is_some() { - return Err(InvalidErrorDigest::InvalidImportedHeaderDigest); - } - } - Ok(MigrateState::Start(from, to)) - }, - (from, None, true) => { - // switch - let state_root = if let Some(digest) = Self::migrate_digest(parent_digests) { - match digest { - StateMigrationDigest { to, state_root, progress: StateMigrationProgress::Finished, .. } => { - if to != &from { - return Err(InvalidErrorDigest::InvalidParentSwitchState); - } - state_root - }, -/* _ => { - return Err(InvalidErrorDigest::InvalidParent); - },*/ - } - } else { - return Err(InvalidErrorDigest::MissingParentDigest); - }; - - if let Some(digests) = imported_block_digests { - // No support for consecutive migrations. - if Self::migrate_digest(digests).is_some() { - return Err(InvalidErrorDigest::InvalidImportedHeaderDigest); - } - } - - Ok(MigrateState::Switch(from, state_root.clone())) - }, - (from, None, false) => { - if Self::migrate_digest(parent_digests).is_some() { - return Err(InvalidErrorDigest::InvalidParent); - } - - if let Some(digests) = imported_block_digests { - // current digest need to be nothing TODO later could migrate from runtime trigger? - if Self::migrate_digest(digests).is_some() { - return Err(InvalidErrorDigest::InvalidImportedHeaderDigest); - } - } - Ok(MigrateState::None(from)) - }, - (_, Some(_), true) => unreachable!("need_migrate_inner exclude target block from migration range"), - } - - } -} - -/// Migration definition at a given block. -#[derive(PartialEq, Eq, Clone, crate::RuntimeDebug, Encode, Decode)] -#[cfg_attr(feature = "std", derive(parity_util_mem::MallocSizeOf))] -pub struct StateMigrationDigest { - /// State version to double check. - pub from: StateVersion, - /// State version to double check. - pub to: StateVersion, - /// State root of target state. - pub state_root: Hash, - /// Current state. - pub progress: StateMigrationProgress, -} - -/// State of migration for a given block. -#[derive(PartialEq, Eq, Clone, crate::RuntimeDebug, Encode, Decode)] -#[cfg_attr(feature = "std", derive(parity_util_mem::MallocSizeOf))] -pub enum StateMigrationProgress { - // Started, start requested from runtime (field `StateMigrationDigest` other than `to` should be ignored), - // then we switch the digest to `Pending` for empty state root and correct version. - - // Pending, read progress from destination at well known key - // (need to overwrite with origin value when finish). - - // - /// Destination is equivalent to origin. - /// - /// Next block will use `to` state with the - /// state root of from the migration progress destination state. - Finished, } From f04c0d9358e5db716115f239cbb2ad77c43c23bc Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 31 Aug 2021 13:23:42 +0200 Subject: [PATCH 106/127] bench to default trie and continue removing migration methods. --- bin/node/bench/src/trie.rs | 4 +- bin/node/testing/src/bench.rs | 4 +- client/block-builder/src/lib.rs | 4 - client/db/src/bench.rs | 4 +- client/db/src/lib.rs | 8 -- client/service/src/client/client.rs | 2 +- primitives/blockchain/src/header_metadata.rs | 1 - primitives/runtime/src/state_version.rs | 37 ------ primitives/state-machine/src/trie_backend.rs | 131 ------------------- 9 files changed, 5 insertions(+), 190 deletions(-) diff --git a/bin/node/bench/src/trie.rs b/bin/node/bench/src/trie.rs index 06796b0cd5b2c..7fce8300140bf 100644 --- a/bin/node/bench/src/trie.rs +++ b/bin/node/bench/src/trie.rs @@ -145,7 +145,7 @@ impl core::BenchmarkDescription for TrieReadBenchmarkDescription { let root = generate_trie( database.open(self.database_type), key_values, - sp_core::state_version::StateVersion::V0, // TODO switch to v1 or default + Default::default(), ); Box::new(TrieReadBenchmark { @@ -258,7 +258,7 @@ impl core::BenchmarkDescription for TrieWriteBenchmarkDescription { let root = generate_trie( database.open(self.database_type), key_values, - sp_core::state_version::StateVersion::V0, // TODO switch to v1 or default + Default::default(), ); Box::new(TrieWriteBenchmark { diff --git a/bin/node/testing/src/bench.rs b/bin/node/testing/src/bench.rs index e871604ed5560..60f31efc21e92 100644 --- a/bin/node/testing/src/bench.rs +++ b/bin/node/testing/src/bench.rs @@ -387,10 +387,8 @@ impl BenchDb { }; let task_executor = TaskExecutor::new(); - let mut state_versions = StateVersions::default(); - state_versions.add((BlockNumber::zero(), StateVersion::V0)); // TODO switch to default (more recent). let backend = - sc_service::new_db_backend(db_config, state_versions).expect("Should not fail"); + sc_service::new_db_backend(db_config, Default::default()).expect("Should not fail"); let client = sc_service::new_client( backend.clone(), NativeElseWasmExecutor::new(WasmExecutionMethod::Compiled, None, 8), diff --git a/client/block-builder/src/lib.rs b/client/block-builder/src/lib.rs index f8bc48d427ad7..f381cfd236e7e 100644 --- a/client/block-builder/src/lib.rs +++ b/client/block-builder/src/lib.rs @@ -162,10 +162,6 @@ where inherent_digests: DigestFor, backend: &'a B, ) -> Result { - // TODO call need_migration here and remove inner field - // then also compare (or init from) with parent header migration state. - // TODO pass parent header as param instead of hash and number - let header = <::Header as HeaderT>::new( parent_number + One::one(), Default::default(), diff --git a/client/db/src/bench.rs b/client/db/src/bench.rs index d605f29ef8bd2..46c8494e62769 100644 --- a/client/db/src/bench.rs +++ b/client/db/src/bench.rs @@ -120,9 +120,7 @@ impl BenchmarkingState { state: RefCell::new(None), db: Cell::new(None), root: Cell::new(root.clone()), - // use old state for now to keep similar benches. - // TODO will need update to heavier of both world before migrating. - state_version: Cell::new(StateVersion::V0), + state_version: Cell::new(Default::default()), genesis: Default::default(), genesis_root: Default::default(), record: Default::default(), diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index fad486d47e5dd..797018745a59b 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -2056,17 +2056,9 @@ impl sc_client_api::backend::Backend for Backend { operation: &mut Self::BlockImportOperation, block: BlockId, ) -> ClientResult<()> { - // TODO with correct executive (not testnet), this would require removing form pending_block - // first the migration progress digest (so digest checks from executive point of view). - // - // TODO actually the parent block check from executive would not pass anyway: this would then - // fail anyway unless runing the migration BEFORE the block to change backend state writen - // by executive. if block.is_pre_genesis() { operation.old_state = self.empty_state()?; } else { - // TODO migration from header cache of parent (block) - // if finished assert this is chainspec block 0 of new state operation.old_state = self.state_at(block)?; } operation.old_state.disable_syncing(); diff --git a/client/service/src/client/client.rs b/client/service/src/client/client.rs index 0a5359be6095e..55e08d2720381 100644 --- a/client/service/src/client/client.rs +++ b/client/service/src/client/client.rs @@ -211,7 +211,7 @@ impl Default for ClientConfig { wasm_runtime_overrides: None, no_genesis: false, wasm_runtime_substitutes: HashMap::new(), - state_versions: Default::default(), // TODO check usage or remove default impl + state_versions: Default::default(), } } } diff --git a/primitives/blockchain/src/header_metadata.rs b/primitives/blockchain/src/header_metadata.rs index 1bf95e2393fb2..928409963bcd4 100644 --- a/primitives/blockchain/src/header_metadata.rs +++ b/primitives/blockchain/src/header_metadata.rs @@ -258,7 +258,6 @@ pub struct CachedHeaderMetadata { pub state_root: Block::Hash, /// Hash of an ancestor header. Used to jump through the tree. ancestor: Block::Hash, - // TODO add migration progress!! to query this instead of full header. } impl From<&Block::Header> for CachedHeaderMetadata { diff --git a/primitives/runtime/src/state_version.rs b/primitives/runtime/src/state_version.rs index 79a80458a3cff..98fc7457c62cc 100644 --- a/primitives/runtime/src/state_version.rs +++ b/primitives/runtime/src/state_version.rs @@ -34,43 +34,6 @@ impl Default for StateVersions { } } -/// Different block specific migrate and state -/// behavior. This is read from chainspec and -/// header block digest. -pub enum MigrateState { - /// Use current define state version. - None(StateVersion), - /// Use state root from digest log. - Switch(StateVersion, H), - /// Start migrate between two version. - Start(StateVersion, StateVersion), - /// Pending migration between two version - /// with pending root of destination version. - Pending(StateVersion, StateVersion, H), -} - -/// Error related to migration digest from parent block header -/// and on import current block header. -pub enum InvalidErrorDigest { - /// Parent migrate do not match chainspec digest declaration. - InvalidParent, - /// Missing digest in parent header when expected. - MissingParentDigest, - /// Parent migrate switch to a state different than current one. - /// This is a bug. - InvalidParentSwitchState, - /// Imported digest do not match chainspec declaration. - InvalidImportedHeaderDigest, -} - -#[cfg(feature = "std")] -impl std::fmt::Display for InvalidErrorDigest { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - let msg = " TODO implement with comments"; - write!(f, "{}", msg) - } -} - impl StateVersions { /// Access genesis state version. /// This uses default state if undefined. diff --git a/primitives/state-machine/src/trie_backend.rs b/primitives/state-machine/src/trie_backend.rs index ec7244b9f567c..1fd87a95fd548 100644 --- a/primitives/state-machine/src/trie_backend.rs +++ b/primitives/state-machine/src/trie_backend.rs @@ -406,137 +406,6 @@ fn trie_error(_e: &sp_trie::TrieError>) -> crate::DefaultEr crate::DefaultError } -impl, H: Hasher> TrieBackend -where - H::Out: Ord + Codec, -{ - /// Execute a state migration, new and removed node are - /// directly written in the db update. - pub fn migrate( - &self, - migration_from: StateVersion, - migration_to: StateVersion, - mut limit_size: Option, - mut limit_items: Option, - do_update: Option<&mut sp_trie::PrefixedMemoryDB>, - previous_block_updates: (&crate::StorageCollection, &crate::ChildStorageCollection), - mut progress: MigrateProgress, - ) -> Result, crate::DefaultError> { - use sp_trie::{TrieDB, TrieDBIterator, TrieDBMut, TrieMut}; - let dest_threshold = match (migration_from, migration_to, self.state_version) { - (StateVersion::V0, StateVersion::V1 { threshold }, StateVersion::V0) => threshold, - _ => return Err(error_version()), - }; - let empty_value = sp_std::vec![23u8; dest_threshold as usize]; - let empty_value2 = sp_std::vec![0u8; dest_threshold as usize]; - let dest_layout = Layout::with_alt_hashing(dest_threshold); - let mut in_mem; - let without_payload; - let overlay = if let Some(change) = do_update { - without_payload = false; - change - } else { - without_payload = true; - in_mem = sp_trie::PrefixedMemoryDB::default(); - &mut in_mem - }; - // Using same db with remove node is ok, but require locking state pruning. - let mut dest_db = - MigrateStorage:: { overlay, storage: self.essence.backend_storage() }; - if progress.root.is_none() { - progress.root = Some(self.essence.root().clone()); - } - let dest_root = progress.root.as_mut().expect("Lazy init above."); - let ori_root = self.essence.root().clone(); - let ori = TrieDB::new_with_layout(&self.essence, &ori_root, Layout::default()) - .map_err(|e| trie_error::(&*e))?; - let mut dest = TrieDBMut::from_existing_with_layout(&mut dest_db, dest_root, dest_layout) - .map_err(|e| trie_error::(&*e))?; - - let iter = TrieDBIterator::new_prefixed_then_seek( - &ori, - &[], - progress.current_top.as_ref().map(|s| s.as_slice()).unwrap_or(&[]), - ) - .map_err(|e| trie_error::(&*e))?; - - let mut total_top_size = 0u64; - let mut total_written_top_size = 0u64; - let mut total_top_items = 0u64; - let mut total_written_top_items = 0u64; - for elt in iter { - let (key, value) = elt.map_err(|e| trie_error::(&*e))?; - if let Some(limit_size) = limit_size.as_mut() { - if *limit_size < value.len() as u64 { - progress.current_top = Some(key); - break - } - *limit_size -= value.len() as u64; - } - if let Some(limit_items) = limit_items.as_mut() { - if *limit_items == 0 { - progress.current_top = Some(key); - break - } - *limit_items -= 1; - } - total_top_size += value.len() as u64; - total_top_items += 1; - if value.len() >= dest_threshold as usize { - total_written_top_size += value.len() as u64; - total_written_top_items += 1; - // Force a change so triedbmut do not ignore - // setting the same value. - if value != empty_value { - dest.insert(key.as_slice(), empty_value.as_slice()) - .map_err(|e| trie_error::(&*e))?; - } else { - dest.insert(key.as_slice(), empty_value2.as_slice()) - .map_err(|e| trie_error::(&*e))?; - } - dest.insert(key.as_slice(), value.as_slice()) - .map_err(|e| trie_error::(&*e))?; - } - } - - // Set again changes from current block in dest (import may have overwrite them, - // or may have change a previous block migration change).c - for (key, value) in previous_block_updates.0.iter() { - if let Some(value) = value.as_ref() { - if without_payload || value.len() >= dest_threshold as usize { - if progress.current_top.as_ref().map(|end| key < end).unwrap_or(true) { - dest.insert(key.as_slice(), value.as_slice()) - .map_err(|e| trie_error::(&*e))?; - } - } - } else { - if progress.current_top.as_ref().map(|end| key < end).unwrap_or(true) { - // rerun all remove in case we import old. . - dest.remove(key.as_slice()) - .map_err(|e| trie_error::(&*e))?; - } - } - } - // TODO same for child - sp_std::mem::drop(ori); - sp_std::mem::drop(dest); - #[cfg(feature = "std")] - info!("\n - total_top_size: {},\n - total_written_top_size: {},\n - total_top_items: {}, \n - total_written_top_items: {}, \n - ", - total_top_size, - total_written_top_size, - total_top_items, - total_written_top_items, - ); - - Ok(progress) - } -} - #[cfg(test)] pub mod tests { use super::*; From bd03fdcbeff0e03f119056ecb45838b0f2c76f05 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 31 Aug 2021 14:17:34 +0200 Subject: [PATCH 107/127] clean test api --- bin/node/testing/src/bench.rs | 4 +-- .../basic-authorship/src/basic_authorship.rs | 8 ++--- client/block-builder/src/lib.rs | 2 +- client/consensus/aura/src/lib.rs | 2 +- client/consensus/babe/src/aux_schema.rs | 2 +- client/consensus/slots/src/aux_schema.rs | 2 +- client/finality-grandpa/src/aux_schema.rs | 8 ++--- client/network/src/light_client_requests.rs | 2 +- client/network/test/src/block_import.rs | 9 ++--- client/network/test/src/lib.rs | 14 +++----- client/offchain/src/lib.rs | 2 +- client/rpc/src/chain/tests.rs | 14 ++++---- client/rpc/src/state/tests.rs | 23 ++++--------- client/service/test/src/client/light.rs | 7 ++-- client/service/test/src/client/mod.rs | 24 +++++++------- client/transaction-pool/tests/pool.rs | 4 +-- primitives/state-machine/src/trie_backend.rs | 22 ------------- test-utils/client/src/lib.rs | 9 ----- test-utils/runtime/client/src/lib.rs | 33 +++++++++++-------- utils/frame/rpc/system/src/lib.rs | 8 ++--- 20 files changed, 82 insertions(+), 117 deletions(-) diff --git a/bin/node/testing/src/bench.rs b/bin/node/testing/src/bench.rs index 60f31efc21e92..04ca7288ae531 100644 --- a/bin/node/testing/src/bench.rs +++ b/bin/node/testing/src/bench.rs @@ -34,7 +34,7 @@ use crate::{ }; use codec::{Decode, Encode}; use futures::executor; -use node_primitives::{Block, BlockNumber}; +use node_primitives::Block; use node_runtime::{ constants::currency::DOLLARS, AccountId, BalancesCall, Call, CheckedExtrinsic, MinimumPeriod, Signature, SystemCall, UncheckedExtrinsic, @@ -55,7 +55,7 @@ use sp_inherents::InherentData; use sp_runtime::{ generic::BlockId, traits::{Block as BlockT, IdentifyAccount, Verify, Zero}, - OpaqueExtrinsic, StateVersion, StateVersions, + OpaqueExtrinsic, }; /// Keyring full of accounts for benching. diff --git a/client/basic-authorship/src/basic_authorship.rs b/client/basic-authorship/src/basic_authorship.rs index c4e617de9a76a..144a3ab6850ff 100644 --- a/client/basic-authorship/src/basic_authorship.rs +++ b/client/basic-authorship/src/basic_authorship.rs @@ -518,7 +518,7 @@ mod tests { #[test] fn should_cease_building_block_when_deadline_is_reached() { // given - let client = Arc::new(substrate_test_runtime_client::new(true)); + let client = Arc::new(substrate_test_runtime_client::new()); let spawner = sp_core::testing::TaskExecutor::new(); let txpool = BasicPool::new_full( Default::default(), @@ -574,7 +574,7 @@ mod tests { #[test] fn should_not_panic_when_deadline_is_reached() { - let client = Arc::new(substrate_test_runtime_client::new(true)); + let client = Arc::new(substrate_test_runtime_client::new()); let spawner = sp_core::testing::TaskExecutor::new(); let txpool = BasicPool::new_full( Default::default(), @@ -671,7 +671,7 @@ mod tests { #[test] fn should_not_remove_invalid_transactions_when_skipping() { // given - let mut client = Arc::new(substrate_test_runtime_client::new(true)); + let mut client = Arc::new(substrate_test_runtime_client::new()); let spawner = sp_core::testing::TaskExecutor::new(); let txpool = BasicPool::new_full( Default::default(), @@ -761,7 +761,7 @@ mod tests { #[test] fn should_cease_building_block_when_block_limit_is_reached() { - let client = Arc::new(substrate_test_runtime_client::new(true)); + let client = Arc::new(substrate_test_runtime_client::new()); let spawner = sp_core::testing::TaskExecutor::new(); let txpool = BasicPool::new_full( Default::default(), diff --git a/client/block-builder/src/lib.rs b/client/block-builder/src/lib.rs index f381cfd236e7e..954e91f4d1bab 100644 --- a/client/block-builder/src/lib.rs +++ b/client/block-builder/src/lib.rs @@ -301,7 +301,7 @@ mod tests { #[test] fn block_building_storage_proof_does_not_include_runtime_by_default() { - let builder = substrate_test_runtime_client::TestClientBuilder::new().state_hashed_value(); + let builder = substrate_test_runtime_client::TestClientBuilder::new(); let backend = builder.backend(); let client = builder.build(); diff --git a/client/consensus/aura/src/lib.rs b/client/consensus/aura/src/lib.rs index 07e4faf18adcf..95212ec38368c 100644 --- a/client/consensus/aura/src/lib.rs +++ b/client/consensus/aura/src/lib.rs @@ -792,7 +792,7 @@ mod tests { #[test] fn authorities_call_works() { - let client = substrate_test_runtime_client::new(true); + let client = substrate_test_runtime_client::new(); assert_eq!(client.chain_info().best_number, 0); assert_eq!( diff --git a/client/consensus/babe/src/aux_schema.rs b/client/consensus/babe/src/aux_schema.rs index fd62794f1021e..b18220c3e360a 100644 --- a/client/consensus/babe/src/aux_schema.rs +++ b/client/consensus/babe/src/aux_schema.rs @@ -149,7 +149,7 @@ mod test { epoch_index: 1, duration: 100, }; - let client = substrate_test_runtime_client::new(true); + let client = substrate_test_runtime_client::new(); let mut v0_tree = ForkTree::, _>::new(); v0_tree .import::<_, ConsensusError>( diff --git a/client/consensus/slots/src/aux_schema.rs b/client/consensus/slots/src/aux_schema.rs index 3a8dbe4f51049..c2fe3f6f4e6bb 100644 --- a/client/consensus/slots/src/aux_schema.rs +++ b/client/consensus/slots/src/aux_schema.rs @@ -157,7 +157,7 @@ mod test { #[test] fn check_equivocation_works() { - let client = substrate_test_runtime_client::new(true); + let client = substrate_test_runtime_client::new(); let (pair, _seed) = sr25519::Pair::generate(); let public = pair.public(); diff --git a/client/finality-grandpa/src/aux_schema.rs b/client/finality-grandpa/src/aux_schema.rs index 53167bd633897..bad01e6dfc62f 100644 --- a/client/finality-grandpa/src/aux_schema.rs +++ b/client/finality-grandpa/src/aux_schema.rs @@ -504,7 +504,7 @@ mod test { #[test] fn load_decode_from_v0_migrates_data_format() { - let client = substrate_test_runtime_client::new(true); + let client = substrate_test_runtime_client::new(); let authorities = vec![(AuthorityId::default(), 100)]; let set_id = 3; @@ -593,7 +593,7 @@ mod test { #[test] fn load_decode_from_v1_migrates_data_format() { - let client = substrate_test_runtime_client::new(true); + let client = substrate_test_runtime_client::new(); let authorities = vec![(AuthorityId::default(), 100)]; let set_id = 3; @@ -686,7 +686,7 @@ mod test { #[test] fn load_decode_from_v2_migrates_data_format() { - let client = substrate_test_runtime_client::new(true); + let client = substrate_test_runtime_client::new(); let authorities = vec![(AuthorityId::default(), 100)]; let set_id = 3; @@ -754,7 +754,7 @@ mod test { #[test] fn write_read_concluded_rounds() { - let client = substrate_test_runtime_client::new(true); + let client = substrate_test_runtime_client::new(); let hash = H256::random(); let round_state = RoundState::genesis((hash, 0)); diff --git a/client/network/src/light_client_requests.rs b/client/network/src/light_client_requests.rs index f3c3597378247..e782b543b9252 100644 --- a/client/network/src/light_client_requests.rs +++ b/client/network/src/light_client_requests.rs @@ -179,7 +179,7 @@ mod tests { sp_runtime::generic::Block, substrate_test_runtime::Extrinsic>; fn send_receive(request: sender::Request, pool: &LocalPool, hashed_value: bool) { - let client = Arc::new(substrate_test_runtime_client::new(hashed_value)); + let client = Arc::new(substrate_test_runtime_client::new_with_state(hashed_value)); let (handler, protocol_config) = handler::LightClientRequestHandler::new(&protocol_id(), client); pool.spawner().spawn_obj(handler.run().boxed().into()).unwrap(); diff --git a/client/network/test/src/block_import.rs b/client/network/test/src/block_import.rs index 53d7bf0c9c552..b2957d885569e 100644 --- a/client/network/test/src/block_import.rs +++ b/client/network/test/src/block_import.rs @@ -34,7 +34,7 @@ use substrate_test_runtime_client::{ }; fn prepare_good_block(hashed_value: bool) -> (TestClient, Hash, u64, PeerId, IncomingBlock) { - let mut client = substrate_test_runtime_client::new(hashed_value); + let mut client = substrate_test_runtime_client::new_with_state(hashed_value); let block = client.new_block(Default::default()).unwrap().build().unwrap().block; block_on(client.import(BlockOrigin::File, block)).unwrap(); @@ -73,8 +73,9 @@ fn import_single_good_block_works_inner(hashed_value: bool) { let mut expected_aux = ImportedAux::default(); expected_aux.is_new_best = true; + let mut client = substrate_test_runtime_client::new_with_state(hashed_value); match block_on(import_single_block( - &mut substrate_test_runtime_client::new(hashed_value), + &mut client, BlockOrigin::File, block, &mut PassThroughVerifier::new(true), @@ -104,7 +105,7 @@ fn import_single_good_block_without_header_fails() { let (_, _, _, peer_id, mut block) = prepare_good_block(true); block.header = None; match block_on(import_single_block( - &mut substrate_test_runtime_client::new(true), + &mut substrate_test_runtime_client::new(), BlockOrigin::File, block, &mut PassThroughVerifier::new(true), @@ -123,7 +124,7 @@ fn async_import_queue_drops() { let queue = BasicQueue::new( verifier, - Box::new(substrate_test_runtime_client::new(true)), + Box::new(substrate_test_runtime_client::new()), None, &executor, None, diff --git a/client/network/test/src/lib.rs b/client/network/test/src/lib.rs index 3bdee3a491ab2..00751b0554fdb 100644 --- a/client/network/test/src/lib.rs +++ b/client/network/test/src/lib.rs @@ -734,16 +734,12 @@ where fn add_full_peer_with_config(&mut self, config: FullPeerConfig) { let mut test_client_builder = match (config.keep_blocks, config.storage_chain) { (Some(keep_blocks), true) => - TestClientBuilder::with_tx_storage(keep_blocks, self.state_versions()) - .state_hashed_value(), - (None, true) => TestClientBuilder::with_tx_storage(u32::MAX, self.state_versions()) - .state_hashed_value(), + TestClientBuilder::with_tx_storage(keep_blocks, self.state_versions()), + (None, true) => TestClientBuilder::with_tx_storage(u32::MAX, self.state_versions()), (Some(keep_blocks), false) => - TestClientBuilder::with_pruning_window(keep_blocks, self.state_versions()) - .state_hashed_value(), + TestClientBuilder::with_pruning_window(keep_blocks, self.state_versions()), (None, false) => - TestClientBuilder::with_default_backend_and_state_versions(self.state_versions()) - .state_hashed_value(), + TestClientBuilder::with_default_backend_and_state_versions(self.state_versions()), }; if matches!(config.sync_mode, SyncMode::Fast { .. }) { test_client_builder = test_client_builder.set_no_genesis(); @@ -878,7 +874,7 @@ where /// Add a light peer. fn add_light_peer(&mut self) { - let (c, backend) = substrate_test_runtime_client::new_light(true); + let (c, backend) = substrate_test_runtime_client::new_light(); let client = Arc::new(c); let (block_import, justification_import, data) = self.make_block_import(PeersClient::Light(client.clone(), backend.clone())); diff --git a/client/offchain/src/lib.rs b/client/offchain/src/lib.rs index a859d4327daf2..be6e4238ca5f1 100644 --- a/client/offchain/src/lib.rs +++ b/client/offchain/src/lib.rs @@ -279,7 +279,7 @@ mod tests { fn should_call_into_runtime_and_produce_extrinsic() { sp_tracing::try_init_simple(); - let client = Arc::new(substrate_test_runtime_client::new(true)); + let client = Arc::new(substrate_test_runtime_client::new()); let spawner = sp_core::testing::TaskExecutor::new(); let pool = TestPool(BasicPool::new_full( Default::default(), diff --git a/client/rpc/src/chain/tests.rs b/client/rpc/src/chain/tests.rs index ec2fc82e1a3c1..7fb76e4bef5e9 100644 --- a/client/rpc/src/chain/tests.rs +++ b/client/rpc/src/chain/tests.rs @@ -30,7 +30,7 @@ use substrate_test_runtime_client::{ #[test] fn should_return_header() { - let client = Arc::new(substrate_test_runtime_client::new(true)); + let client = Arc::new(substrate_test_runtime_client::new()); let api = new_full(client.clone(), SubscriptionManager::new(Arc::new(TaskExecutor))); assert_matches!( @@ -69,7 +69,7 @@ fn should_return_a_block() { should_return_a_block_inner(false); } fn should_return_a_block_inner(hashed_value: bool) { - let mut client = Arc::new(substrate_test_runtime_client::new(hashed_value)); + let mut client = Arc::new(substrate_test_runtime_client::new_with_state(hashed_value)); let api = new_full(client.clone(), SubscriptionManager::new(Arc::new(TaskExecutor))); let block = client.new_block(Default::default()).unwrap().build().unwrap().block; @@ -117,7 +117,7 @@ fn should_return_a_block_inner(hashed_value: bool) { #[test] fn should_return_block_hash() { - let mut client = Arc::new(substrate_test_runtime_client::new(true)); + let mut client = Arc::new(substrate_test_runtime_client::new()); let api = new_full(client.clone(), SubscriptionManager::new(Arc::new(TaskExecutor))); assert_matches!( @@ -159,7 +159,7 @@ fn should_return_block_hash() { #[test] fn should_return_finalized_hash() { - let mut client = Arc::new(substrate_test_runtime_client::new(true)); + let mut client = Arc::new(substrate_test_runtime_client::new()); let api = new_full(client.clone(), SubscriptionManager::new(Arc::new(TaskExecutor))); assert_matches!( @@ -189,7 +189,7 @@ fn should_notify_about_latest_block() { let (subscriber, id, mut transport) = Subscriber::new_test("test"); { - let mut client = Arc::new(substrate_test_runtime_client::new(true)); + let mut client = Arc::new(substrate_test_runtime_client::new()); let api = new_full(client.clone(), SubscriptionManager::new(Arc::new(TaskExecutor))); api.subscribe_all_heads(Default::default(), subscriber); @@ -211,7 +211,7 @@ fn should_notify_about_best_block() { let (subscriber, id, mut transport) = Subscriber::new_test("test"); { - let mut client = Arc::new(substrate_test_runtime_client::new(true)); + let mut client = Arc::new(substrate_test_runtime_client::new()); let api = new_full(client.clone(), SubscriptionManager::new(Arc::new(TaskExecutor))); api.subscribe_new_heads(Default::default(), subscriber); @@ -233,7 +233,7 @@ fn should_notify_about_finalized_block() { let (subscriber, id, mut transport) = Subscriber::new_test("test"); { - let mut client = Arc::new(substrate_test_runtime_client::new(true)); + let mut client = Arc::new(substrate_test_runtime_client::new()); let api = new_full(client.clone(), SubscriptionManager::new(Arc::new(TaskExecutor))); api.subscribe_finalized_heads(Default::default(), subscriber); diff --git a/client/rpc/src/state/tests.rs b/client/rpc/src/state/tests.rs index 5cf81e4449c6b..c9fc837651466 100644 --- a/client/rpc/src/state/tests.rs +++ b/client/rpc/src/state/tests.rs @@ -132,7 +132,7 @@ fn should_return_child_storage() { #[test] fn should_call_contract() { - let client = Arc::new(substrate_test_runtime_client::new(true)); + let client = Arc::new(substrate_test_runtime_client::new()); let genesis_hash = client.genesis_hash(); let (client, _child) = new_full(client, SubscriptionManager::new(Arc::new(TaskExecutor)), DenyUnsafe::No, None); @@ -152,7 +152,7 @@ fn should_notify_about_storage_changes() { let (subscriber, id, mut transport) = Subscriber::new_test("test"); { - let mut client = Arc::new(substrate_test_runtime_client::new(true)); + let mut client = Arc::new(substrate_test_runtime_client::new()); let (api, _child) = new_full( client.clone(), SubscriptionManager::new(Arc::new(TaskExecutor)), @@ -188,7 +188,7 @@ fn should_send_initial_storage_changes_and_notifications() { let (subscriber, id, mut transport) = Subscriber::new_test("test"); { - let mut client = Arc::new(substrate_test_runtime_client::new(true)); + let mut client = Arc::new(substrate_test_runtime_client::new()); let (api, _child) = new_full( client.clone(), SubscriptionManager::new(Arc::new(TaskExecutor)), @@ -411,8 +411,8 @@ fn should_query_storage() { ); } - run_tests(Arc::new(substrate_test_runtime_client::new(true)), false); - run_tests(Arc::new(substrate_test_runtime_client::new(false)), false); + run_tests(Arc::new(substrate_test_runtime_client::new_with_state(false)), false); + run_tests(Arc::new(substrate_test_runtime_client::new_with_state(true)), false); run_tests( Arc::new( TestClientBuilder::new() @@ -421,15 +421,6 @@ fn should_query_storage() { ), true, ); - run_tests( - Arc::new( - TestClientBuilder::new() - .state_hashed_value() - .changes_trie_config(Some(ChangesTrieConfiguration::new(4, 2))) - .build(), - ), - true, - ); } #[test] @@ -447,7 +438,7 @@ fn should_return_runtime_version() { should_return_runtime_version_inner(false); } fn should_return_runtime_version_inner(hashed_value: bool) { - let client = Arc::new(substrate_test_runtime_client::new(hashed_value)); + let client = Arc::new(substrate_test_runtime_client::new_with_state(hashed_value)); let (api, _child) = new_full( client.clone(), SubscriptionManager::new(Arc::new(TaskExecutor)), @@ -475,7 +466,7 @@ fn should_notify_on_runtime_version_initially() { let (subscriber, id, mut transport) = Subscriber::new_test("test"); { - let client = Arc::new(substrate_test_runtime_client::new(true)); + let client = Arc::new(substrate_test_runtime_client::new()); let (api, _child) = new_full( client.clone(), SubscriptionManager::new(Arc::new(TaskExecutor)), diff --git a/client/service/test/src/client/light.rs b/client/service/test/src/client/light.rs index 6aebc83acbd3b..4f16f388a0ef0 100644 --- a/client/service/test/src/client/light.rs +++ b/client/service/test/src/client/light.rs @@ -383,7 +383,8 @@ fn execution_proof_is_generated_and_checked_inner(hashed_value: bool) { } // prepare remote client - let mut remote_client = substrate_test_runtime_client::new(hashed_value); + let mut remote_client = substrate_test_runtime_client::new_with_state(hashed_value); + for i in 1u32..3u32 { let mut digest = Digest::default(); digest.push(sp_runtime::generic::DigestItem::Other::(i.to_le_bytes().to_vec())); @@ -459,7 +460,7 @@ type TestChecker = LightDataChecker< fn prepare_for_read_proof_check(hashed_value: bool) -> (TestChecker, Header, StorageProof, u32) { // prepare remote client - let remote_client = substrate_test_runtime_client::new(hashed_value); + let remote_client = substrate_test_runtime_client::new_with_state(hashed_value); let remote_block_id = BlockId::Number(0); let remote_block_hash = remote_client.block_hash(0).unwrap().unwrap(); let mut remote_block_header = remote_client.header(&remote_block_id).unwrap().unwrap(); @@ -540,7 +541,7 @@ fn prepare_for_header_proof_check( hashed_value: bool, ) -> (TestChecker, Hash, Header, StorageProof) { // prepare remote client - let mut remote_client = substrate_test_runtime_client::new(hashed_value); + let mut remote_client = substrate_test_runtime_client::new_with_state(hashed_value); let mut local_headers_hashes = Vec::new(); for i in 0..4 { let block = remote_client.new_block(Default::default()).unwrap().build().unwrap().block; diff --git a/client/service/test/src/client/mod.rs b/client/service/test/src/client/mod.rs index 55f7d8b5f5040..99700c5d97f0f 100644 --- a/client/service/test/src/client/mod.rs +++ b/client/service/test/src/client/mod.rs @@ -362,7 +362,7 @@ fn client_initializes_from_genesis_ok() { client_initializes_from_genesis_ok_inner(true); } fn client_initializes_from_genesis_ok_inner(hashed_value: bool) { - let client = substrate_test_runtime_client::new(hashed_value); + let client = substrate_test_runtime_client::new_with_state(hashed_value); assert_eq!( client @@ -388,7 +388,7 @@ fn client_initializes_from_genesis_ok_inner(hashed_value: bool) { #[test] fn block_builder_works_with_no_transactions() { - let mut client = substrate_test_runtime_client::new(true); + let mut client = substrate_test_runtime_client::new(); let block = client.new_block(Default::default()).unwrap().build().unwrap().block; @@ -403,7 +403,7 @@ fn block_builder_works_with_transactions() { block_builder_works_with_transactions_inner(false); } fn block_builder_works_with_transactions_inner(hashed_value: bool) { - let mut client = substrate_test_runtime_client::new(hashed_value); + let mut client = substrate_test_runtime_client::new_with_state(hashed_value); let mut builder = client.new_block(Default::default()).unwrap(); @@ -448,7 +448,7 @@ fn block_builder_works_with_transactions_inner(hashed_value: bool) { #[test] fn block_builder_does_not_include_invalid() { - let mut client = substrate_test_runtime_client::new(true); + let mut client = substrate_test_runtime_client::new(); let mut builder = client.new_block(Default::default()).unwrap(); @@ -518,7 +518,7 @@ fn best_containing_with_hash_not_found() { fn uncles_with_only_ancestors() { // block tree: // G -> A1 -> A2 - let mut client = substrate_test_runtime_client::new(true); + let mut client = substrate_test_runtime_client::new(); // G -> A1 let a1 = client.new_block(Default::default()).unwrap().build().unwrap().block; @@ -538,7 +538,7 @@ fn uncles_with_multiple_forks() { // A1 -> B2 -> B3 -> B4 // B2 -> C3 // A1 -> D2 - let mut client = substrate_test_runtime_client::new(true); + let mut client = substrate_test_runtime_client::new(); // G -> A1 let a1 = client.new_block(Default::default()).unwrap().build().unwrap().block; @@ -1213,7 +1213,7 @@ fn key_changes_works() { #[test] fn import_with_justification() { - let mut client = substrate_test_runtime_client::new(true); + let mut client = substrate_test_runtime_client::new(); // G -> A1 let a1 = client.new_block(Default::default()).unwrap().build().unwrap().block; @@ -1250,7 +1250,7 @@ fn import_with_justification() { #[test] fn importing_diverged_finalized_block_should_trigger_reorg() { - let mut client = substrate_test_runtime_client::new(true); + let mut client = substrate_test_runtime_client::new(); // G -> A1 -> A2 // \ @@ -1298,7 +1298,7 @@ fn importing_diverged_finalized_block_should_trigger_reorg() { #[test] fn finalizing_diverged_block_should_trigger_reorg() { let (mut client, select_chain) = - TestClientBuilder::new().state_hashed_value().build_with_longest_chain(); + TestClientBuilder::new().build_with_longest_chain(); // G -> A1 -> A2 // \ @@ -1371,7 +1371,7 @@ fn finalizing_diverged_block_should_trigger_reorg() { #[test] fn get_header_by_block_number_doesnt_panic() { - let client = substrate_test_runtime_client::new(true); + let client = substrate_test_runtime_client::new(); // backend uses u32 for block numbers, make sure we don't panic when // trying to convert @@ -1382,7 +1382,7 @@ fn get_header_by_block_number_doesnt_panic() { #[test] fn state_reverted_on_reorg() { sp_tracing::try_init_simple(); - let mut client = substrate_test_runtime_client::new(true); + let mut client = substrate_test_runtime_client::new(); let current_balance = |client: &substrate_test_runtime_client::TestClient| { client @@ -2044,7 +2044,7 @@ fn storage_keys_iter_works() { storage_keys_iter_works_inner(false); } fn storage_keys_iter_works_inner(hashed_value: bool) { - let client = substrate_test_runtime_client::new(hashed_value); + let client = substrate_test_runtime_client::new_with_state(hashed_value); let prefix = StorageKey(hex!("").to_vec()); diff --git a/client/transaction-pool/tests/pool.rs b/client/transaction-pool/tests/pool.rs index 4d1edf5b04818..6c34d05cd5dcb 100644 --- a/client/transaction-pool/tests/pool.rs +++ b/client/transaction-pool/tests/pool.rs @@ -854,7 +854,7 @@ fn ready_set_should_eventually_resolve_when_block_update_arrives() { fn should_not_accept_old_signatures() { use std::convert::TryFrom; - let client = Arc::new(substrate_test_runtime_client::new(true)); + let client = Arc::new(substrate_test_runtime_client::new()); let pool = Arc::new( BasicPool::new_test(Arc::new(FullChainApi::new( @@ -895,7 +895,7 @@ fn should_not_accept_old_signatures() { #[test] fn import_notification_to_pool_maintain_works() { - let mut client = Arc::new(substrate_test_runtime_client::new(true)); + let mut client = Arc::new(substrate_test_runtime_client::new()); let pool = Arc::new( BasicPool::new_test(Arc::new(FullChainApi::new( diff --git a/primitives/state-machine/src/trie_backend.rs b/primitives/state-machine/src/trie_backend.rs index 1fd87a95fd548..80a695982ed21 100644 --- a/primitives/state-machine/src/trie_backend.rs +++ b/primitives/state-machine/src/trie_backend.rs @@ -34,8 +34,6 @@ use sp_trie::{ trie_types::{TrieDB, TrieError}, Layout, Trie, }; -#[cfg(feature = "std")] -use log::info; /// Patricia trie-based backend. Transaction type is an overlay of changes to commit. pub struct TrieBackend, H: Hasher> { @@ -386,26 +384,6 @@ impl<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> AsHashDB } } -#[cfg(feature = "std")] -fn error_version() -> crate::DefaultError { - "Cannot migrate for given state versions".into() -} - -#[cfg(not(feature = "std"))] -fn error_version() -> crate::DefaultError { - crate::DefaultError -} - -#[cfg(feature = "std")] -fn trie_error(e: &sp_trie::TrieError>) -> crate::DefaultError { - format!("Migrate trieDB error: {}", e) -} - -#[cfg(not(feature = "std"))] -fn trie_error(_e: &sp_trie::TrieError>) -> crate::DefaultError { - crate::DefaultError -} - #[cfg(test)] pub mod tests { use super::*; diff --git a/test-utils/client/src/lib.rs b/test-utils/client/src/lib.rs index 946f828bcc0ae..02e93d2265629 100644 --- a/test-utils/client/src/lib.rs +++ b/test-utils/client/src/lib.rs @@ -80,7 +80,6 @@ pub struct TestClientBuilder, bad_blocks: BadBlocks, enable_offchain_indexing_api: bool, - state_hashed_value: bool, // TODO useless ? state_versions: StateVersions, no_genesis: bool, } @@ -167,7 +166,6 @@ impl fork_blocks: None, bad_blocks: None, enable_offchain_indexing_api: false, - state_hashed_value: false, no_genesis: false, state_versions, } @@ -233,13 +231,6 @@ impl self } - /// Enable the internal value hash of state. - /// TODO remove and use state_versions instead - pub fn state_hashed_value(mut self) -> Self { - self.state_hashed_value = true; - self - } - /// Disable writing genesis. pub fn set_no_genesis(mut self) -> Self { self.no_genesis = true; diff --git a/test-utils/runtime/client/src/lib.rs b/test-utils/runtime/client/src/lib.rs index a91148a35cfda..c3bc84ae9cb6c 100644 --- a/test-utils/runtime/client/src/lib.rs +++ b/test-utils/runtime/client/src/lib.rs @@ -395,18 +395,24 @@ impl Fetcher for LightFetcher { } /// Creates new client instance used for tests. -pub fn new(hashed_state: bool) -> Client { - let mut builder = TestClientBuilder::new(); - if hashed_state { - builder = builder.state_hashed_value(); +pub fn new() -> Client { + TestClientBuilder::new().build() +} + +/// Creates new client instance used for tests, using +/// optionally inner state hashing. +pub fn new_with_state(inner_hashing: bool) -> Client { + let mut state_versions = StateVersions::default(); + if !inner_hashing { + state_versions.add((0, sp_core::state_version::StateVersion::V0)); } - builder.build() + TestClientBuilder::with_default_backend_and_state_versions( + Some(state_versions) + ).build() } /// Creates new light client instance used for tests. -pub fn new_light( - hashed_state: bool, -) -> ( +pub fn new_light() -> ( client::Client< LightBackend, LightExecutor, @@ -429,11 +435,12 @@ pub fn new_light( .expect("Creates LocalCallExecutor"); let call_executor = LightExecutor::new(backend.clone(), local_call_executor); - let mut builder = TestClientBuilder::with_backend(backend.clone()); - if hashed_state { - builder = builder.state_hashed_value(); - } - (builder.build_with_executor(call_executor).0, backend) + ( + TestClientBuilder::with_backend(backend.clone()) + .build_with_executor(call_executor) + .0, + backend, + ) } /// Creates new light client fetcher used for tests. diff --git a/utils/frame/rpc/system/src/lib.rs b/utils/frame/rpc/system/src/lib.rs index b668a720aa75f..f0f37f0b20675 100644 --- a/utils/frame/rpc/system/src/lib.rs +++ b/utils/frame/rpc/system/src/lib.rs @@ -291,7 +291,7 @@ mod tests { sp_tracing::try_init_simple(); // given - let client = Arc::new(substrate_test_runtime_client::new(true)); + let client = Arc::new(substrate_test_runtime_client::new()); let spawner = sp_core::testing::TaskExecutor::new(); let pool = BasicPool::new_full(Default::default(), true.into(), None, spawner, client.clone()); @@ -326,7 +326,7 @@ mod tests { sp_tracing::try_init_simple(); // given - let client = Arc::new(substrate_test_runtime_client::new(true)); + let client = Arc::new(substrate_test_runtime_client::new()); let spawner = sp_core::testing::TaskExecutor::new(); let pool = BasicPool::new_full(Default::default(), true.into(), None, spawner, client.clone()); @@ -345,7 +345,7 @@ mod tests { sp_tracing::try_init_simple(); // given - let client = Arc::new(substrate_test_runtime_client::new(true)); + let client = Arc::new(substrate_test_runtime_client::new()); let spawner = sp_core::testing::TaskExecutor::new(); let pool = BasicPool::new_full(Default::default(), true.into(), None, spawner, client.clone()); @@ -374,7 +374,7 @@ mod tests { sp_tracing::try_init_simple(); // given - let client = Arc::new(substrate_test_runtime_client::new(true)); + let client = Arc::new(substrate_test_runtime_client::new()); let spawner = sp_core::testing::TaskExecutor::new(); let pool = BasicPool::new_full(Default::default(), true.into(), None, spawner, client.clone()); From da5071117cc99a28df6d789847ad30f60cc07250 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 31 Aug 2021 17:05:26 +0200 Subject: [PATCH 108/127] fix tests. --- client/basic-authorship/src/lib.rs | 2 +- primitives/api/test/benches/bench.rs | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/client/basic-authorship/src/lib.rs b/client/basic-authorship/src/lib.rs index cb83719c119d2..2b2fe554efdff 100644 --- a/client/basic-authorship/src/lib.rs +++ b/client/basic-authorship/src/lib.rs @@ -30,7 +30,7 @@ //! # DefaultTestClientBuilderExt, TestClientBuilderExt, //! # }; //! # use sc_transaction_pool::{BasicPool, FullChainApi}; -//! # let client = Arc::new(substrate_test_runtime_client::new(true)); +//! # let client = Arc::new(substrate_test_runtime_client::new()); //! # let spawner = sp_core::testing::TaskExecutor::new(); //! # let txpool = BasicPool::new_full( //! # Default::default(), diff --git a/primitives/api/test/benches/bench.rs b/primitives/api/test/benches/bench.rs index 3232204b9b128..b3d96a2db6a56 100644 --- a/primitives/api/test/benches/bench.rs +++ b/primitives/api/test/benches/bench.rs @@ -25,7 +25,7 @@ use substrate_test_runtime_client::{ fn sp_api_benchmark(c: &mut Criterion) { c.bench_function("add one with same runtime api", |b| { - let client = substrate_test_runtime_client::new(true); + let client = substrate_test_runtime_client::new(); let runtime_api = client.runtime_api(); let block_id = BlockId::Number(client.chain_info().best_number); @@ -33,14 +33,14 @@ fn sp_api_benchmark(c: &mut Criterion) { }); c.bench_function("add one with recreating runtime api", |b| { - let client = substrate_test_runtime_client::new(true); + let client = substrate_test_runtime_client::new(); let block_id = BlockId::Number(client.chain_info().best_number); b.iter(|| client.runtime_api().benchmark_add_one(&block_id, &1)) }); c.bench_function("vector add one with same runtime api", |b| { - let client = substrate_test_runtime_client::new(true); + let client = substrate_test_runtime_client::new(); let runtime_api = client.runtime_api(); let block_id = BlockId::Number(client.chain_info().best_number); let data = vec![0; 1000]; @@ -49,7 +49,7 @@ fn sp_api_benchmark(c: &mut Criterion) { }); c.bench_function("vector add one with recreating runtime api", |b| { - let client = substrate_test_runtime_client::new(true); + let client = substrate_test_runtime_client::new(); let block_id = BlockId::Number(client.chain_info().best_number); let data = vec![0; 1000]; From e18c2d555cfe41c19df4dd7caadbcbaacd7603bd Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 1 Sep 2021 15:25:54 +0200 Subject: [PATCH 109/127] rust fmt --- bin/node/bench/src/trie.rs | 12 ++-------- client/api/src/in_mem.rs | 32 ++++++++++++------------- client/db/src/lib.rs | 2 +- client/service/src/client/client.rs | 2 +- client/service/test/src/client/mod.rs | 3 +-- primitives/runtime/src/state_version.rs | 10 ++++---- test-utils/client/src/lib.rs | 7 ++++-- test-utils/runtime/client/src/lib.rs | 4 +--- 8 files changed, 31 insertions(+), 41 deletions(-) diff --git a/bin/node/bench/src/trie.rs b/bin/node/bench/src/trie.rs index 7fce8300140bf..f4811703ed023 100644 --- a/bin/node/bench/src/trie.rs +++ b/bin/node/bench/src/trie.rs @@ -142,11 +142,7 @@ impl core::BenchmarkDescription for TrieReadBenchmarkDescription { assert_eq!(warmup_keys.len(), SAMPLE_SIZE); assert_eq!(query_keys.len(), SAMPLE_SIZE); - let root = generate_trie( - database.open(self.database_type), - key_values, - Default::default(), - ); + let root = generate_trie(database.open(self.database_type), key_values, Default::default()); Box::new(TrieReadBenchmark { database, @@ -255,11 +251,7 @@ impl core::BenchmarkDescription for TrieWriteBenchmarkDescription { assert_eq!(warmup_keys.len(), SAMPLE_SIZE); - let root = generate_trie( - database.open(self.database_type), - key_values, - Default::default(), - ); + let root = generate_trie(database.open(self.database_type), key_values, Default::default()); Box::new(TrieWriteBenchmark { database, diff --git a/client/api/src/in_mem.rs b/client/api/src/in_mem.rs index 7eb2549723061..66010e4846066 100644 --- a/client/api/src/in_mem.rs +++ b/client/api/src/in_mem.rs @@ -94,7 +94,7 @@ impl StoredBlock { StoredBlock::Full(block, just) => { let (header, body) = block.deconstruct(); (header, Some(body), just) - } + }, } } } @@ -145,13 +145,12 @@ impl Blockchain { /// Get version of state. pub fn state_version(&self, id: BlockId) -> StateVersion { let number = match id { - BlockId::Hash(h) => { + BlockId::Hash(h) => if let Ok(Some(header)) = self.header(BlockId::Hash(h)) { header.number().clone() } else { 0u32.into() - } - } + }, BlockId::Number(n) => n, }; self.storage.read().state_versions.state_version_at(number) @@ -221,7 +220,7 @@ impl Blockchain { pub fn equals_to(&self, other: &Self) -> bool { // Check ptr equality first to avoid double read locks. if ptr::eq(self, other) { - return true; + return true } self.canon_equals_to(other) && self.storage.read().blocks == other.storage.read().blocks } @@ -230,14 +229,14 @@ impl Blockchain { pub fn canon_equals_to(&self, other: &Self) -> bool { // Check ptr equality first to avoid double read locks. if ptr::eq(self, other) { - return true; + return true } let this = self.storage.read(); let other = other.storage.read(); - this.hashes == other.hashes - && this.best_hash == other.best_hash - && this.best_number == other.best_number - && this.genesis_hash == other.genesis_hash + this.hashes == other.hashes && + this.best_hash == other.best_hash && + this.best_number == other.best_number && + this.genesis_hash == other.genesis_hash } /// Insert header CHT root. @@ -342,7 +341,7 @@ impl Blockchain { if !stored_justifications.append(justification) { return Err(sp_blockchain::Error::BadJustification( "Duplicate consensus engine ID".into(), - )); + )) } } else { *block_justifications = Some(Justifications::from(justification)); @@ -872,10 +871,9 @@ where fn state_at(&self, block: BlockId) -> sp_blockchain::Result { match block { - BlockId::Hash(h) if h == Default::default() => { - return Ok(self.blockchain.state_version(BlockId::Hash(h)).into()) - } - _ => {} + BlockId::Hash(h) if h == Default::default() => + return Ok(self.blockchain.state_version(BlockId::Hash(h)).into()), + _ => {}, } self.blockchain @@ -922,7 +920,7 @@ where /// Check that genesis storage is valid. pub fn check_genesis_storage(storage: &Storage) -> sp_blockchain::Result<()> { if storage.top.iter().any(|(k, _)| well_known_keys::is_child_storage_key(k)) { - return Err(sp_blockchain::Error::InvalidState.into()); + return Err(sp_blockchain::Error::InvalidState.into()) } if storage @@ -930,7 +928,7 @@ pub fn check_genesis_storage(storage: &Storage) -> sp_blockchain::Result<()> { .keys() .any(|child_key| !well_known_keys::is_child_storage_key(&child_key)) { - return Err(sp_blockchain::Error::InvalidState.into()); + return Err(sp_blockchain::Error::InvalidState.into()) } Ok(()) diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index 797018745a59b..ba24a5047ec65 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -2368,7 +2368,7 @@ impl sc_client_api::backend::Backend for Backend { fn state_at(&self, block: BlockId) -> ClientResult { use sc_client_api::blockchain::HeaderBackend as BcHeaderBackend; - let is_genesis= match &block { + let is_genesis = match &block { BlockId::Number(n) => n.is_zero(), BlockId::Hash(h) => h == &self.blockchain.meta.read().genesis_hash, }; diff --git a/client/service/src/client/client.rs b/client/service/src/client/client.rs index 55e08d2720381..d72ad35a8f5af 100644 --- a/client/service/src/client/client.rs +++ b/client/service/src/client/client.rs @@ -116,7 +116,7 @@ where importing_block: RwLock>, block_rules: BlockRules, execution_extensions: ExecutionExtensions, - config: ClientConfig, + config: ClientConfig, telemetry: Option, _phantom: PhantomData, } diff --git a/client/service/test/src/client/mod.rs b/client/service/test/src/client/mod.rs index 99700c5d97f0f..afd0632d71cae 100644 --- a/client/service/test/src/client/mod.rs +++ b/client/service/test/src/client/mod.rs @@ -1297,8 +1297,7 @@ fn importing_diverged_finalized_block_should_trigger_reorg() { #[test] fn finalizing_diverged_block_should_trigger_reorg() { - let (mut client, select_chain) = - TestClientBuilder::new().build_with_longest_chain(); + let (mut client, select_chain) = TestClientBuilder::new().build_with_longest_chain(); // G -> A1 -> A2 // \ diff --git a/primitives/runtime/src/state_version.rs b/primitives/runtime/src/state_version.rs index 98fc7457c62cc..2a4cd28a23b3f 100644 --- a/primitives/runtime/src/state_version.rs +++ b/primitives/runtime/src/state_version.rs @@ -40,7 +40,7 @@ impl StateVersions { pub fn genesis_state_version(&self) -> StateVersion { if let Some((number, version)) = self.canonical_states.get(0) { if number.is_zero() { - return *version; + return *version } } DEFAULT_STATE_VERSION @@ -52,7 +52,7 @@ impl StateVersions { let mut version = DEFAULT_STATE_VERSION; for (number, state) in self.canonical_states.iter() { if number > &at { - break; + break } version = *state; } @@ -66,10 +66,10 @@ impl StateVersions { for (i, (number, _)) in self.canonical_states.iter().enumerate() { if number == &at { replace = Some(i); - break; + break } if number > &at { - break; + break } insert = Some(i + 1); } @@ -95,7 +95,7 @@ impl StateVersions { if let Ok(number) = NumberFor::::from_str(number) { canonical_states.push((number.into(), version)); } else { - return None; + return None } } Some(StateVersions { canonical_states }) diff --git a/test-utils/client/src/lib.rs b/test-utils/client/src/lib.rs index 02e93d2265629..704d29828ff22 100644 --- a/test-utils/client/src/lib.rs +++ b/test-utils/client/src/lib.rs @@ -34,7 +34,7 @@ pub use sp_keyring::{ ed25519::Keyring as Ed25519Keyring, sr25519::Keyring as Sr25519Keyring, AccountKeyring, }; pub use sp_keystore::{SyncCryptoStore, SyncCryptoStorePtr}; -pub use sp_runtime::{Storage, StorageChild, StateVersions}; +pub use sp_runtime::{StateVersions, Storage, StorageChild}; pub use sp_state_machine::ExecutionStrategy; use futures::{ @@ -155,7 +155,10 @@ impl } /// Create a new instance of the test client builder with specific state versions. - pub fn with_backend_and_state_versions(backend: Arc, state_versions: StateVersions) -> Self { + pub fn with_backend_and_state_versions( + backend: Arc, + state_versions: StateVersions, + ) -> Self { TestClientBuilder { backend, execution_strategies: ExecutionStrategies::default(), diff --git a/test-utils/runtime/client/src/lib.rs b/test-utils/runtime/client/src/lib.rs index c3bc84ae9cb6c..18eeb52203cf7 100644 --- a/test-utils/runtime/client/src/lib.rs +++ b/test-utils/runtime/client/src/lib.rs @@ -406,9 +406,7 @@ pub fn new_with_state(inner_hashing: bool) -> Client { if !inner_hashing { state_versions.add((0, sp_core::state_version::StateVersion::V0)); } - TestClientBuilder::with_default_backend_and_state_versions( - Some(state_versions) - ).build() + TestClientBuilder::with_default_backend_and_state_versions(Some(state_versions)).build() } /// Creates new light client instance used for tests. From 8847b904922cca7fe29d33a2774177da0137612e Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 1 Sep 2021 15:59:08 +0200 Subject: [PATCH 110/127] remove unused emplace_ref --- bin/node/bench/src/simple_trie.rs | 5 ----- primitives/state-machine/src/trie_backend.rs | 4 ---- primitives/state-machine/src/trie_backend_essence.rs | 8 -------- primitives/trie/src/lib.rs | 5 ----- 4 files changed, 22 deletions(-) diff --git a/bin/node/bench/src/simple_trie.rs b/bin/node/bench/src/simple_trie.rs index b33a588b463f7..651772c71575f 100644 --- a/bin/node/bench/src/simple_trie.rs +++ b/bin/node/bench/src/simple_trie.rs @@ -65,11 +65,6 @@ impl<'a> HashDB for SimpleTrie<'a> { self.overlay.insert(key, Some(value)); } - fn emplace_ref(&mut self, key: &Hash, prefix: Prefix, value: &[u8]) { - let key = sp_trie::prefixed_key::(key, prefix); - self.overlay.insert(key, Some(value.into())); - } - fn remove(&mut self, key: &Hash, prefix: Prefix) { let key = sp_trie::prefixed_key::(key, prefix); self.overlay.insert(key, None); diff --git a/primitives/state-machine/src/trie_backend.rs b/primitives/state-machine/src/trie_backend.rs index 80a695982ed21..7ea4399246711 100644 --- a/primitives/state-machine/src/trie_backend.rs +++ b/primitives/state-machine/src/trie_backend.rs @@ -348,10 +348,6 @@ impl<'a, S: 'a + TrieBackendStorage, H: Hasher> hash_db::HashDB HashDB::emplace(self.overlay, key, prefix, value) } - fn emplace_ref(&mut self, key: &H::Out, prefix: Prefix, value: &[u8]) { - HashDB::emplace_ref(self.overlay, key, prefix, value) - } - fn remove(&mut self, key: &H::Out, prefix: Prefix) { HashDB::remove(self.overlay, key, prefix) } diff --git a/primitives/state-machine/src/trie_backend_essence.rs b/primitives/state-machine/src/trie_backend_essence.rs index 663c17a787174..300bb9f156d11 100644 --- a/primitives/state-machine/src/trie_backend_essence.rs +++ b/primitives/state-machine/src/trie_backend_essence.rs @@ -499,10 +499,6 @@ impl<'a, S: 'a + TrieBackendStorage, H: Hasher> hash_db::HashDB HashDB::emplace(self.overlay, key, prefix, value) } - fn emplace_ref(&mut self, key: &H::Out, prefix: Prefix, value: &[u8]) { - HashDB::emplace_ref(self.overlay, key, prefix, value) - } - fn remove(&mut self, key: &H::Out, prefix: Prefix) { HashDB::remove(self.overlay, key, prefix) } @@ -605,10 +601,6 @@ impl, H: Hasher> HashDB for TrieBackendEsse unimplemented!(); } - fn emplace_ref(&mut self, _key: &H::Out, _prefix: Prefix, _value: &[u8]) { - unimplemented!(); - } - fn remove(&mut self, _key: &H::Out, _prefix: Prefix) { unimplemented!(); } diff --git a/primitives/trie/src/lib.rs b/primitives/trie/src/lib.rs index 179f44db1e7a0..0415858165e7a 100644 --- a/primitives/trie/src/lib.rs +++ b/primitives/trie/src/lib.rs @@ -484,11 +484,6 @@ where self.0.emplace(key, (&derived_prefix.0, derived_prefix.1), value) } - fn emplace_ref(&mut self, key: &H::Out, prefix: Prefix, value: &[u8]) { - let derived_prefix = keyspace_as_prefix_alloc(self.1, prefix); - self.0.emplace_ref(key, (&derived_prefix.0, derived_prefix.1), value) - } - fn remove(&mut self, key: &H::Out, prefix: Prefix) { let derived_prefix = keyspace_as_prefix_alloc(self.1, prefix); self.0.remove(key, (&derived_prefix.0, derived_prefix.1)) From 869e533e3e5bd1ab2ed5a8481333b903b11a1cf7 Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 1 Sep 2021 16:01:37 +0200 Subject: [PATCH 111/127] Revert "remove unused emplace_ref" This reverts commit 8847b904922cca7fe29d33a2774177da0137612e. --- bin/node/bench/src/simple_trie.rs | 5 +++++ primitives/state-machine/src/trie_backend.rs | 4 ++++ primitives/state-machine/src/trie_backend_essence.rs | 8 ++++++++ primitives/trie/src/lib.rs | 5 +++++ 4 files changed, 22 insertions(+) diff --git a/bin/node/bench/src/simple_trie.rs b/bin/node/bench/src/simple_trie.rs index 651772c71575f..b33a588b463f7 100644 --- a/bin/node/bench/src/simple_trie.rs +++ b/bin/node/bench/src/simple_trie.rs @@ -65,6 +65,11 @@ impl<'a> HashDB for SimpleTrie<'a> { self.overlay.insert(key, Some(value)); } + fn emplace_ref(&mut self, key: &Hash, prefix: Prefix, value: &[u8]) { + let key = sp_trie::prefixed_key::(key, prefix); + self.overlay.insert(key, Some(value.into())); + } + fn remove(&mut self, key: &Hash, prefix: Prefix) { let key = sp_trie::prefixed_key::(key, prefix); self.overlay.insert(key, None); diff --git a/primitives/state-machine/src/trie_backend.rs b/primitives/state-machine/src/trie_backend.rs index 7ea4399246711..80a695982ed21 100644 --- a/primitives/state-machine/src/trie_backend.rs +++ b/primitives/state-machine/src/trie_backend.rs @@ -348,6 +348,10 @@ impl<'a, S: 'a + TrieBackendStorage, H: Hasher> hash_db::HashDB HashDB::emplace(self.overlay, key, prefix, value) } + fn emplace_ref(&mut self, key: &H::Out, prefix: Prefix, value: &[u8]) { + HashDB::emplace_ref(self.overlay, key, prefix, value) + } + fn remove(&mut self, key: &H::Out, prefix: Prefix) { HashDB::remove(self.overlay, key, prefix) } diff --git a/primitives/state-machine/src/trie_backend_essence.rs b/primitives/state-machine/src/trie_backend_essence.rs index 300bb9f156d11..663c17a787174 100644 --- a/primitives/state-machine/src/trie_backend_essence.rs +++ b/primitives/state-machine/src/trie_backend_essence.rs @@ -499,6 +499,10 @@ impl<'a, S: 'a + TrieBackendStorage, H: Hasher> hash_db::HashDB HashDB::emplace(self.overlay, key, prefix, value) } + fn emplace_ref(&mut self, key: &H::Out, prefix: Prefix, value: &[u8]) { + HashDB::emplace_ref(self.overlay, key, prefix, value) + } + fn remove(&mut self, key: &H::Out, prefix: Prefix) { HashDB::remove(self.overlay, key, prefix) } @@ -601,6 +605,10 @@ impl, H: Hasher> HashDB for TrieBackendEsse unimplemented!(); } + fn emplace_ref(&mut self, _key: &H::Out, _prefix: Prefix, _value: &[u8]) { + unimplemented!(); + } + fn remove(&mut self, _key: &H::Out, _prefix: Prefix) { unimplemented!(); } diff --git a/primitives/trie/src/lib.rs b/primitives/trie/src/lib.rs index 0415858165e7a..179f44db1e7a0 100644 --- a/primitives/trie/src/lib.rs +++ b/primitives/trie/src/lib.rs @@ -484,6 +484,11 @@ where self.0.emplace(key, (&derived_prefix.0, derived_prefix.1), value) } + fn emplace_ref(&mut self, key: &H::Out, prefix: Prefix, value: &[u8]) { + let derived_prefix = keyspace_as_prefix_alloc(self.1, prefix); + self.0.emplace_ref(key, (&derived_prefix.0, derived_prefix.1), value) + } + fn remove(&mut self, key: &H::Out, prefix: Prefix) { let derived_prefix = keyspace_as_prefix_alloc(self.1, prefix); self.0.remove(key, (&derived_prefix.0, derived_prefix.1)) From b6590b2d78215d329f95673dea188f07470de2b1 Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 1 Sep 2021 16:55:16 +0200 Subject: [PATCH 112/127] minor change and remove unused code. --- Cargo.lock | 1 - client/api/src/cht.rs | 4 +- client/block-builder/src/lib.rs | 4 +- client/chain-spec/src/chain_spec.rs | 2 +- client/db/src/lib.rs | 2 +- client/executor/runtime-test/Cargo.toml | 1 - client/network/test/src/sync.rs | 2 +- frame/support/test/tests/instance.rs | 2 +- primitives/state-machine/src/backend.rs | 3 +- primitives/state-machine/src/lib.rs | 16 +--- primitives/state-machine/src/trie_backend.rs | 96 -------------------- 11 files changed, 11 insertions(+), 122 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0b83a7677651e..c3519f8d2a7cc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8155,7 +8155,6 @@ dependencies = [ "sp-runtime", "sp-sandbox", "sp-std", - "sp-storage", "sp-tasks", "substrate-wasm-builder", ] diff --git a/client/api/src/cht.rs b/client/api/src/cht.rs index be4eb8531b4c7..0b95f21fc322f 100644 --- a/client/api/src/cht.rs +++ b/client/api/src/cht.rs @@ -33,7 +33,7 @@ use sp_core::{convert_hash, H256}; use sp_runtime::traits::{AtLeast32Bit, Header as HeaderT, One, Zero}; use sp_state_machine::{ prove_read_on_trie_backend, read_proof_check, - read_proof_check_on_proving_backend_generic as read_proof_check_on_proving_backend, + read_proof_check_on_proving_backend, Backend as StateBackend, InMemoryBackend, MemoryDB, StorageProof, TrieBackend, }; @@ -175,7 +175,7 @@ where local_number, remote_hash, |_, local_cht_key| { - read_proof_check_on_proving_backend::(proving_backend, local_cht_key) + read_proof_check_on_proving_backend::(proving_backend, local_cht_key) .map_err(ClientError::from_state) }, ) diff --git a/client/block-builder/src/lib.rs b/client/block-builder/src/lib.rs index 954e91f4d1bab..e89421edfb168 100644 --- a/client/block-builder/src/lib.rs +++ b/client/block-builder/src/lib.rs @@ -234,9 +234,9 @@ where ), ); - let state = self.backend.state_at(self.block_id)?; - let proof = { self.api.extract_proof() }; + let proof = self.api.extract_proof(); + let state = self.backend.state_at(self.block_id)?; let changes_trie_state = backend::changes_tries_state_at_block( &self.block_id, self.backend.changes_trie_storage(), diff --git a/client/chain-spec/src/chain_spec.rs b/client/chain-spec/src/chain_spec.rs index 051d993a32946..5b683ba047d65 100644 --- a/client/chain-spec/src/chain_spec.rs +++ b/client/chain-spec/src/chain_spec.rs @@ -291,7 +291,7 @@ impl ChainSpec { use std::str::FromStr; self.state_versions() .get(0) - // This is incorrect (can have number representation not compatible with u64 + // This is can be incorrect (if number representation incompatible with u64) .and_then(|(n, s)| u64::from_str(n).ok().map(|n| (n, s))) .and_then(|(n, s)| (n == 0).then(|| s.clone())) .unwrap_or_default() diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index ba24a5047ec65..b0020d6c8447c 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -1901,7 +1901,7 @@ impl Backend { fn empty_state(&self) -> ClientResult, Block>> { let root = EmptyStorage::::new().0; // Empty trie - // state_version for genesis in empty state. + // Using genesis state_version in empty state. let state_version = self.state_versions.genesis_state_version(); let db_state = DbState::::new(self.storage.clone(), root, state_version); let state = RefTrackingState::new(db_state, self.storage.clone(), None); diff --git a/client/executor/runtime-test/Cargo.toml b/client/executor/runtime-test/Cargo.toml index 64cf1410495dc..a4fbc88cf5662 100644 --- a/client/executor/runtime-test/Cargo.toml +++ b/client/executor/runtime-test/Cargo.toml @@ -19,7 +19,6 @@ sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../.. sp-sandbox = { version = "0.10.0-dev", default-features = false, path = "../../../primitives/sandbox" } sp-std = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/std" } sp-tasks = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/tasks" } -sp-storage = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/storage" } [build-dependencies] substrate-wasm-builder = { version = "5.0.0-dev", path = "../../../utils/wasm-builder" } diff --git a/client/network/test/src/sync.rs b/client/network/test/src/sync.rs index 1d07dfcbae1ea..48a8688dde6a8 100644 --- a/client/network/test/src/sync.rs +++ b/client/network/test/src/sync.rs @@ -158,7 +158,7 @@ fn sync_from_two_peers_works() { } #[test] -fn sync_from_two_peers_with_migrate_works() { +fn sync_from_two_peers_with_versioning_switch_works() { sp_tracing::try_init_simple(); let mut state_versions = sp_runtime::StateVersions::::default(); state_versions.add((0, sp_runtime::StateVersion::V0)); diff --git a/frame/support/test/tests/instance.rs b/frame/support/test/tests/instance.rs index 841b69dafed1a..a948853ff2a44 100644 --- a/frame/support/test/tests/instance.rs +++ b/frame/support/test/tests/instance.rs @@ -359,7 +359,7 @@ fn storage_instance_independence() { module2::DoubleMap::::insert(&0, &0, &0); module2::DoubleMap::::insert(&0, &0, &0); }); - // 12 storage values and threshold. + // 12 storage values. assert_eq!(storage.top.len(), 12); } diff --git a/primitives/state-machine/src/backend.rs b/primitives/state-machine/src/backend.rs index ffae30323fd37..98fb52bcc25cc 100644 --- a/primitives/state-machine/src/backend.rs +++ b/primitives/state-machine/src/backend.rs @@ -176,6 +176,7 @@ pub trait Backend: sp_std::fmt::Debug { fn as_trie_backend(&self) -> Option<&TrieBackend> { None } + /// Calculate the storage root, with given delta over what is already stored /// in the backend, and produce a "transaction" that can be used to commit. /// Does include child storage updates. @@ -316,7 +317,7 @@ where for (key, value) in input { if let Err(e) = trie.insert(&key, &value) { log::warn!(target: "trie", "Failed to write to trie: {}", e); - return None + return None; } } } diff --git a/primitives/state-machine/src/lib.rs b/primitives/state-machine/src/lib.rs index d777be5cb8c36..8f3c68ede3472 100644 --- a/primitives/state-machine/src/lib.rs +++ b/primitives/state-machine/src/lib.rs @@ -127,7 +127,7 @@ pub use crate::{ StorageTransactionCache, StorageValue, }, stats::{StateMachineStats, UsageInfo, UsageUnit}, - trie_backend::{MigrateProgress, TrieBackend}, + trie_backend::TrieBackend, trie_backend_essence::{Storage, TrieBackendStorage}, }; @@ -928,19 +928,6 @@ mod execution { where H: Hasher, H::Out: Ord + Codec, - { - read_proof_check_on_proving_backend_generic(proving_backend, key) - } - - /// Check storage read proof on pre-created proving backend. - pub fn read_proof_check_on_proving_backend_generic( - proving_backend: &TrieBackend, H>, - key: &[u8], - ) -> Result>, Box> - where - H: Hasher, - H::Out: Ord + Codec, - KF: sp_trie::KeyFunction + Send + Sync, { proving_backend.storage(key).map_err(|e| Box::new(e) as Box) } @@ -1642,7 +1629,6 @@ mod tests { assert_eq!(completed, true); } - // TODO test does not make lot of sense with migration. #[test] fn inner_state_hashing_switch_proofs() { let mut layout = Layout::default(); diff --git a/primitives/state-machine/src/trie_backend.rs b/primitives/state-machine/src/trie_backend.rs index 80a695982ed21..f71d854a2f0f3 100644 --- a/primitives/state-machine/src/trie_backend.rs +++ b/primitives/state-machine/src/trie_backend.rs @@ -288,102 +288,6 @@ where } } -/// Migration progress. -pub struct MigrateProgress { - /// Next key to migrate for top. - pub current_top: Option>, - /// Current migrated root of top trie. - pub root: Option, - /// Next key to migrate with current migrated root for a child - /// trie. - /// When defined, `current_top` always point to the - /// child trie root top location. - pub current_child: Option<(Vec, H)>, -} - -/// Migration storage. -pub struct MigrateStorage<'a, S, H> -where - H: Hasher, -{ - overlay: &'a mut sp_trie::PrefixedMemoryDB, - storage: &'a S, -} - -use hash_db::{self, AsHashDB, HashDB, HashDBRef, Prefix}; -use sp_trie::DBValue; - -impl<'a, S: 'a + TrieBackendStorage, H: Hasher> hash_db::HashDB - for MigrateStorage<'a, S, H> -{ - fn get(&self, key: &H::Out, prefix: Prefix) -> Option { - if let Some(val) = HashDB::get(self.overlay, key, prefix) { - Some(val) - } else { - match self.storage.get(&key, prefix) { - Ok(x) => x, - Err(e) => { - warn!(target: "trie", "Failed to read from DB: {}", e); - None - }, - } - } - } - - fn access_from(&self, key: &H::Out, _at: Option<&H::Out>) -> Option { - // call back to storage even if the overlay was hit. - self.storage.access_from(key); - None - } - - fn contains(&self, key: &H::Out, prefix: Prefix) -> bool { - HashDB::get(self, key, prefix).is_some() - } - - fn insert(&mut self, prefix: Prefix, value: &[u8]) -> H::Out { - HashDB::insert(self.overlay, prefix, value) - } - - fn emplace(&mut self, key: H::Out, prefix: Prefix, value: DBValue) { - HashDB::emplace(self.overlay, key, prefix, value) - } - - fn emplace_ref(&mut self, key: &H::Out, prefix: Prefix, value: &[u8]) { - HashDB::emplace_ref(self.overlay, key, prefix, value) - } - - fn remove(&mut self, key: &H::Out, prefix: Prefix) { - HashDB::remove(self.overlay, key, prefix) - } -} - -impl<'a, S: 'a + TrieBackendStorage, H: Hasher> HashDBRef - for MigrateStorage<'a, S, H> -{ - fn get(&self, key: &H::Out, prefix: Prefix) -> Option { - HashDB::get(self, key, prefix) - } - - fn access_from(&self, key: &H::Out, at: Option<&H::Out>) -> Option { - HashDB::access_from(self, key, at) - } - - fn contains(&self, key: &H::Out, prefix: Prefix) -> bool { - HashDB::contains(self, key, prefix) - } -} - -impl<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> AsHashDB - for MigrateStorage<'a, S, H> -{ - fn as_hash_db<'b>(&'b self) -> &'b (dyn HashDB + 'b) { - self - } - fn as_hash_db_mut<'b>(&'b mut self) -> &'b mut (dyn HashDB + 'b) { - self - } -} - #[cfg(test)] pub mod tests { use super::*; From b695af56fedb00ef9e15ffe1376f181aecc5f367 Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 1 Sep 2021 17:40:33 +0200 Subject: [PATCH 113/127] move state version for light client to blockchain struct --- client/light/src/backend.rs | 20 +++++++----------- client/light/src/blockchain.rs | 28 +++++++++++++++---------- client/light/src/fetcher.rs | 4 ++-- client/light/src/lib.rs | 11 +++++----- client/service/src/builder.rs | 17 ++++++++------- client/service/test/src/client/light.rs | 28 ++++++++++++------------- test-utils/runtime/client/src/lib.rs | 6 +++--- 7 files changed, 58 insertions(+), 56 deletions(-) diff --git a/client/light/src/backend.rs b/client/light/src/backend.rs index 481d41c65f026..0d120038204d2 100644 --- a/client/light/src/backend.rs +++ b/client/light/src/backend.rs @@ -48,7 +48,7 @@ use sp_core::{ use sp_runtime::{ generic::BlockId, traits::{Block as BlockT, HashFor, Header, NumberFor, Zero}, - Justification, Justifications, StateVersion, StateVersions, Storage, + Justification, Justifications, StateVersion, Storage, }; use sp_state_machine::{ Backend as StateBackend, ChangesTrieTransaction, ChildStorageCollection, InMemoryBackend, @@ -60,11 +60,8 @@ const IN_MEMORY_EXPECT_PROOF: &str = /// Light client backend. pub struct Backend { - blockchain: Arc>, + blockchain: Arc>, genesis_state: RwLock>>>, - state_versions: StateVersions, - // TODO consider moving this state_versions into BlockChain (and add N to type) -> would make - // better api import_lock: RwLock<()>, } @@ -94,17 +91,16 @@ pub enum GenesisOrUnavailableState { impl Backend { /// Create new light backend. - pub fn new(blockchain: Arc>, state_versions: StateVersions) -> Self { + pub fn new(blockchain: Arc>) -> Self { Self { blockchain, genesis_state: RwLock::new(None), import_lock: Default::default(), - state_versions, } } /// Get shared blockchain reference. - pub fn blockchain(&self) -> &Arc> { + pub fn blockchain(&self) -> &Arc> { &self.blockchain } } @@ -136,7 +132,7 @@ where Block::Hash: Ord, { type BlockImportOperation = ImportOperation; - type Blockchain = Blockchain; + type Blockchain = Blockchain; type State = GenesisOrUnavailableState>; type OffchainStorage = InMemOffchainStorage; @@ -151,7 +147,7 @@ where storage_update: None, changes_trie_config_update: None, state_version: None, - genesis_state_version: self.state_versions.genesis_state_version(), + genesis_state_version: self.blockchain.state_versions().genesis_state_version(), _phantom: Default::default(), }) } @@ -162,7 +158,7 @@ where block: BlockId, ) -> ClientResult<()> { if let Some(number) = self.blockchain.storage().block_number_from_id(&block)? { - operation.state_version = Some(self.state_versions.state_version_at(number)); + operation.state_version = Some(self.blockchain.state_versions().state_version_at(number)); } Ok(()) } @@ -228,7 +224,7 @@ where Ok(()) } - fn blockchain(&self) -> &Blockchain { + fn blockchain(&self) -> &Blockchain { &self.blockchain } diff --git a/client/light/src/blockchain.rs b/client/light/src/blockchain.rs index e88c724193697..764b24720fd2c 100644 --- a/client/light/src/blockchain.rs +++ b/client/light/src/blockchain.rs @@ -24,7 +24,7 @@ use std::sync::Arc; use sp_runtime::{ generic::BlockId, traits::{Block as BlockT, Header as HeaderT, NumberFor, Zero}, - Justifications, + Justifications, StateVersions, }; use crate::fetcher::RemoteHeaderRequest; @@ -42,23 +42,29 @@ use sp_blockchain::{ }; /// Light client blockchain. -pub struct Blockchain { +pub struct Blockchain { storage: S, + state_versions: StateVersions, } -impl Blockchain { +impl Blockchain { /// Create new light blockchain backed with given storage. - pub fn new(storage: S) -> Self { - Self { storage } + pub fn new(storage: S, state_versions: StateVersions) -> Self { + Self { storage, state_versions } } /// Get storage reference. pub fn storage(&self) -> &S { &self.storage } + + /// Get state versions reference. + pub fn state_versions(&self) -> &StateVersions { + &self.state_versions + } } -impl BlockchainHeaderBackend for Blockchain +impl BlockchainHeaderBackend for Blockchain where Block: BlockT, S: Storage, @@ -91,7 +97,7 @@ where } } -impl HeaderMetadata for Blockchain +impl HeaderMetadata for Blockchain where Block: BlockT, S: Storage, @@ -114,7 +120,7 @@ where } } -impl BlockchainBackend for Blockchain +impl BlockchainBackend for Blockchain where Block: BlockT, S: Storage, @@ -155,13 +161,13 @@ where } } -impl, Block: BlockT> ProvideCache for Blockchain { +impl, Block: BlockT> ProvideCache for Blockchain { fn cache(&self) -> Option>> { self.storage.cache() } } -impl RemoteBlockchain for Blockchain +impl RemoteBlockchain for Blockchain where S: Storage, { @@ -200,7 +206,7 @@ where } } -impl, Block: BlockT> ProvideChtRoots for Blockchain { +impl, Block: BlockT> ProvideChtRoots for Blockchain { fn header_cht_root( &self, cht_size: NumberFor, diff --git a/client/light/src/fetcher.rs b/client/light/src/fetcher.rs index ca346f019d2b2..4785151270f4a 100644 --- a/client/light/src/fetcher.rs +++ b/client/light/src/fetcher.rs @@ -55,7 +55,7 @@ pub use sc_client_api::{ /// Remote data checker. pub struct LightDataChecker> { - blockchain: Arc>, + blockchain: Arc>, executor: E, spawn_handle: Box, _marker: PhantomData, @@ -64,7 +64,7 @@ pub struct LightDataChecker> { impl> LightDataChecker { /// Create new light data checker. pub fn new( - blockchain: Arc>, + blockchain: Arc>, executor: E, spawn_handle: Box, ) -> Self { diff --git a/client/light/src/lib.rs b/client/light/src/lib.rs index ca71a94496a44..7563377fd5e74 100644 --- a/client/light/src/lib.rs +++ b/client/light/src/lib.rs @@ -34,7 +34,7 @@ pub use fetcher::*; /// Create an instance of fetch data checker. pub fn new_fetch_checker>( - blockchain: Arc>, + blockchain: Arc>, executor: E, spawn_handle: Box, ) -> LightDataChecker @@ -45,18 +45,17 @@ where } /// Create an instance of light client blockchain backend. -pub fn new_light_blockchain>(storage: S) -> Arc> { - Arc::new(Blockchain::new(storage)) +pub fn new_light_blockchain>(storage: S, state_versions: sp_runtime::StateVersions) -> Arc> { + Arc::new(Blockchain::new(storage, state_versions)) } /// Create an instance of light client backend. pub fn new_light_backend( - blockchain: Arc>, - state_versions: sp_runtime::StateVersions, + blockchain: Arc>, ) -> Arc> where B: BlockT, S: BlockchainStorage, { - Arc::new(Backend::new(blockchain, state_versions)) + Arc::new(Backend::new(blockchain)) } diff --git a/client/service/src/builder.rs b/client/service/src/builder.rs index b854709bf750e..0fa7eba7df465 100644 --- a/client/service/src/builder.rs +++ b/client/service/src/builder.rs @@ -398,13 +398,6 @@ where }; sc_client_db::light::LightStorage::new(db_settings)? }; - let light_blockchain = sc_light::new_light_blockchain(db_storage); - let fetch_checker = Arc::new(sc_light::new_fetch_checker::<_, TBl, _>( - light_blockchain.clone(), - executor.clone(), - Box::new(task_manager.spawn_handle()), - )); - let on_demand = Arc::new(sc_network::config::OnDemand::new(fetch_checker)); let state_versions = StateVersions::from_conf( config @@ -417,7 +410,15 @@ where Error::Application(Box::from("Invalid state versions for chain spec".to_string())) })?; - let backend = sc_light::new_light_backend(light_blockchain, state_versions); + let light_blockchain = sc_light::new_light_blockchain(db_storage, state_versions); + let fetch_checker = Arc::new(sc_light::new_fetch_checker::<_, TBl, _>( + light_blockchain.clone(), + executor.clone(), + Box::new(task_manager.spawn_handle()), + )); + let on_demand = Arc::new(sc_network::config::OnDemand::new(fetch_checker)); + + let backend = sc_light::new_light_backend(light_blockchain); let client = Arc::new(light::new_light( backend.clone(), config.chain_spec.as_storage_builder(), diff --git a/client/service/test/src/client/light.rs b/client/service/test/src/client/light.rs index 4f16f388a0ef0..23253993cfcc7 100644 --- a/client/service/test/src/client/light.rs +++ b/client/service/test/src/client/light.rs @@ -64,7 +64,7 @@ use sp_core::{ }; use sp_state_machine::Backend as _; -pub type DummyBlockchain = Blockchain; +pub type DummyBlockchain = Blockchain; pub struct DummyStorage { pub changes_tries_cht_roots: HashMap, @@ -270,7 +270,7 @@ fn local_state_is_created_when_genesis_state_is_available() { substrate_test_runtime_client::runtime::Header::new(0, def, def, def, Default::default()); let backend: Backend<_, Block> = - Backend::new(Arc::new(DummyBlockchain::new(DummyStorage::new())), Default::default()); + Backend::new(Arc::new(DummyBlockchain::new(DummyStorage::new(), Default::default()))); let mut op = backend.begin_operation().unwrap(); op.set_block_data(header0, None, None, None, NewBlockState::Final).unwrap(); op.set_genesis_state(Default::default(), true).unwrap(); @@ -285,7 +285,7 @@ fn local_state_is_created_when_genesis_state_is_available() { #[test] fn unavailable_state_is_created_when_genesis_state_is_unavailable() { let backend: Backend<_, Block> = - Backend::new(Arc::new(DummyBlockchain::new(DummyStorage::new())), Default::default()); + Backend::new(Arc::new(DummyBlockchain::new(DummyStorage::new(), Default::default()))); match backend.state_at(BlockId::Number(0)).unwrap() { GenesisOrUnavailableState::Unavailable => (), @@ -296,7 +296,7 @@ fn unavailable_state_is_created_when_genesis_state_is_unavailable() { #[test] fn light_aux_store_is_updated_via_non_importing_op() { let backend = - Backend::new(Arc::new(DummyBlockchain::new(DummyStorage::new())), Default::default()); + Backend::new(Arc::new(DummyBlockchain::new(DummyStorage::new(), Default::default()))); let mut op = ClientBackend::::begin_operation(&backend).unwrap(); BlockImportOperation::::insert_aux(&mut op, vec![(vec![1], Some(vec![2]))]).unwrap(); ClientBackend::::commit_operation(&backend, op).unwrap(); @@ -487,7 +487,7 @@ fn prepare_for_read_proof_check(hashed_value: bool) -> (TestChecker, Header, Sto .insert(remote_block_hash, remote_block_header.clone(), None, None, NewBlockState::Final) .unwrap(); let local_checker = LightDataChecker::new( - Arc::new(DummyBlockchain::new(DummyStorage::new())), + Arc::new(DummyBlockchain::new(DummyStorage::new(), Default::default())), local_executor(), Box::new(TaskExecutor::new()), ); @@ -529,7 +529,7 @@ fn prepare_for_read_child_proof_check() -> (TestChecker, Header, StorageProof, V .insert(remote_block_hash, remote_block_header.clone(), None, None, NewBlockState::Final) .unwrap(); let local_checker = LightDataChecker::new( - Arc::new(DummyBlockchain::new(DummyStorage::new())), + Arc::new(DummyBlockchain::new(DummyStorage::new(), Default::default())), local_executor(), Box::new(TaskExecutor::new()), ); @@ -566,7 +566,7 @@ fn prepare_for_header_proof_check( local_storage.insert_cht_root(1, local_cht_root); } let local_checker = LightDataChecker::new( - Arc::new(DummyBlockchain::new(DummyStorage::new())), + Arc::new(DummyBlockchain::new(DummyStorage::new(), Default::default())), local_executor(), Box::new(TaskExecutor::new()), ); @@ -698,7 +698,7 @@ fn check_header_proof_fails_if_invalid_header_provided() { fn changes_proof_is_generated_and_checked_when_headers_are_not_pruned() { let (remote_client, local_roots, test_cases) = prepare_client_with_key_changes(); let local_checker = TestChecker::new( - Arc::new(DummyBlockchain::new(DummyStorage::new())), + Arc::new(DummyBlockchain::new(DummyStorage::new(), Default::default())), local_executor(), Box::new(TaskExecutor::new()), ); @@ -784,7 +784,7 @@ fn changes_proof_is_generated_and_checked_when_headers_are_pruned() { let mut local_storage = DummyStorage::new(); local_storage.changes_tries_cht_roots.insert(0, local_cht_root); let local_checker = TestChecker::new( - Arc::new(DummyBlockchain::new(local_storage)), + Arc::new(DummyBlockchain::new(local_storage, Default::default())), local_executor(), Box::new(TaskExecutor::new()), ); @@ -825,7 +825,7 @@ fn changes_proof_is_generated_and_checked_when_headers_are_pruned() { fn check_changes_proof_fails_if_proof_is_wrong() { let (remote_client, local_roots, test_cases) = prepare_client_with_key_changes(); let local_checker = TestChecker::new( - Arc::new(DummyBlockchain::new(DummyStorage::new())), + Arc::new(DummyBlockchain::new(DummyStorage::new(), Default::default())), local_executor(), Box::new(TaskExecutor::new()), ); @@ -938,7 +938,7 @@ fn check_changes_tries_proof_fails_if_proof_is_wrong() { // fails when changes trie CHT is missing from the local db let local_checker = TestChecker::new( - Arc::new(DummyBlockchain::new(DummyStorage::new())), + Arc::new(DummyBlockchain::new(DummyStorage::new(), Default::default())), local_executor(), Box::new(TaskExecutor::new()), ); @@ -950,7 +950,7 @@ fn check_changes_tries_proof_fails_if_proof_is_wrong() { let mut local_storage = DummyStorage::new(); local_storage.changes_tries_cht_roots.insert(0, local_cht_root); let local_checker = TestChecker::new( - Arc::new(DummyBlockchain::new(local_storage)), + Arc::new(DummyBlockchain::new(local_storage, Default::default())), local_executor(), Box::new(TaskExecutor::new()), ); @@ -966,7 +966,7 @@ fn check_body_proof_faulty() { let block = Block::new(header.clone(), Vec::new()); let local_checker = TestChecker::new( - Arc::new(DummyBlockchain::new(DummyStorage::new())), + Arc::new(DummyBlockchain::new(DummyStorage::new(), Default::default())), local_executor(), Box::new(TaskExecutor::new()), ); @@ -987,7 +987,7 @@ fn check_body_proof_of_same_data_should_succeed() { let block = Block::new(header.clone(), extrinsics); let local_checker = TestChecker::new( - Arc::new(DummyBlockchain::new(DummyStorage::new())), + Arc::new(DummyBlockchain::new(DummyStorage::new(), Default::default())), local_executor(), Box::new(TaskExecutor::new()), ); diff --git a/test-utils/runtime/client/src/lib.rs b/test-utils/runtime/client/src/lib.rs index 18eeb52203cf7..5dbc924c6bc00 100644 --- a/test-utils/runtime/client/src/lib.rs +++ b/test-utils/runtime/client/src/lib.rs @@ -419,10 +419,10 @@ pub fn new_light() -> ( >, Arc, ) { - let storage = sc_client_db::light::LightStorage::new_test(); - let blockchain = Arc::new(sc_light::Blockchain::new(storage)); let state_versions = Default::default(); - let backend = Arc::new(LightBackend::new(blockchain, state_versions)); + let storage = sc_client_db::light::LightStorage::new_test(); + let blockchain = Arc::new(sc_light::Blockchain::new(storage, state_versions)); + let backend = Arc::new(LightBackend::new(blockchain)); let executor = new_native_executor(); let local_call_executor = client::LocalCallExecutor::new( backend.clone(), From 7db873e674136627ef7e0455a3fbdbf16e6ac337 Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 1 Sep 2021 17:41:03 +0200 Subject: [PATCH 114/127] Revert "move state version for light client to blockchain struct" This reverts commit b695af56fedb00ef9e15ffe1376f181aecc5f367, the change was generally worse (more code, more generic parameter). --- client/light/src/backend.rs | 20 +++++++++++------- client/light/src/blockchain.rs | 28 ++++++++++--------------- client/light/src/fetcher.rs | 4 ++-- client/light/src/lib.rs | 11 +++++----- client/service/src/builder.rs | 17 +++++++-------- client/service/test/src/client/light.rs | 28 ++++++++++++------------- test-utils/runtime/client/src/lib.rs | 6 +++--- 7 files changed, 56 insertions(+), 58 deletions(-) diff --git a/client/light/src/backend.rs b/client/light/src/backend.rs index 0d120038204d2..481d41c65f026 100644 --- a/client/light/src/backend.rs +++ b/client/light/src/backend.rs @@ -48,7 +48,7 @@ use sp_core::{ use sp_runtime::{ generic::BlockId, traits::{Block as BlockT, HashFor, Header, NumberFor, Zero}, - Justification, Justifications, StateVersion, Storage, + Justification, Justifications, StateVersion, StateVersions, Storage, }; use sp_state_machine::{ Backend as StateBackend, ChangesTrieTransaction, ChildStorageCollection, InMemoryBackend, @@ -60,8 +60,11 @@ const IN_MEMORY_EXPECT_PROOF: &str = /// Light client backend. pub struct Backend { - blockchain: Arc>, + blockchain: Arc>, genesis_state: RwLock>>>, + state_versions: StateVersions, + // TODO consider moving this state_versions into BlockChain (and add N to type) -> would make + // better api import_lock: RwLock<()>, } @@ -91,16 +94,17 @@ pub enum GenesisOrUnavailableState { impl Backend { /// Create new light backend. - pub fn new(blockchain: Arc>) -> Self { + pub fn new(blockchain: Arc>, state_versions: StateVersions) -> Self { Self { blockchain, genesis_state: RwLock::new(None), import_lock: Default::default(), + state_versions, } } /// Get shared blockchain reference. - pub fn blockchain(&self) -> &Arc> { + pub fn blockchain(&self) -> &Arc> { &self.blockchain } } @@ -132,7 +136,7 @@ where Block::Hash: Ord, { type BlockImportOperation = ImportOperation; - type Blockchain = Blockchain; + type Blockchain = Blockchain; type State = GenesisOrUnavailableState>; type OffchainStorage = InMemOffchainStorage; @@ -147,7 +151,7 @@ where storage_update: None, changes_trie_config_update: None, state_version: None, - genesis_state_version: self.blockchain.state_versions().genesis_state_version(), + genesis_state_version: self.state_versions.genesis_state_version(), _phantom: Default::default(), }) } @@ -158,7 +162,7 @@ where block: BlockId, ) -> ClientResult<()> { if let Some(number) = self.blockchain.storage().block_number_from_id(&block)? { - operation.state_version = Some(self.blockchain.state_versions().state_version_at(number)); + operation.state_version = Some(self.state_versions.state_version_at(number)); } Ok(()) } @@ -224,7 +228,7 @@ where Ok(()) } - fn blockchain(&self) -> &Blockchain { + fn blockchain(&self) -> &Blockchain { &self.blockchain } diff --git a/client/light/src/blockchain.rs b/client/light/src/blockchain.rs index 764b24720fd2c..e88c724193697 100644 --- a/client/light/src/blockchain.rs +++ b/client/light/src/blockchain.rs @@ -24,7 +24,7 @@ use std::sync::Arc; use sp_runtime::{ generic::BlockId, traits::{Block as BlockT, Header as HeaderT, NumberFor, Zero}, - Justifications, StateVersions, + Justifications, }; use crate::fetcher::RemoteHeaderRequest; @@ -42,29 +42,23 @@ use sp_blockchain::{ }; /// Light client blockchain. -pub struct Blockchain { +pub struct Blockchain { storage: S, - state_versions: StateVersions, } -impl Blockchain { +impl Blockchain { /// Create new light blockchain backed with given storage. - pub fn new(storage: S, state_versions: StateVersions) -> Self { - Self { storage, state_versions } + pub fn new(storage: S) -> Self { + Self { storage } } /// Get storage reference. pub fn storage(&self) -> &S { &self.storage } - - /// Get state versions reference. - pub fn state_versions(&self) -> &StateVersions { - &self.state_versions - } } -impl BlockchainHeaderBackend for Blockchain +impl BlockchainHeaderBackend for Blockchain where Block: BlockT, S: Storage, @@ -97,7 +91,7 @@ where } } -impl HeaderMetadata for Blockchain +impl HeaderMetadata for Blockchain where Block: BlockT, S: Storage, @@ -120,7 +114,7 @@ where } } -impl BlockchainBackend for Blockchain +impl BlockchainBackend for Blockchain where Block: BlockT, S: Storage, @@ -161,13 +155,13 @@ where } } -impl, Block: BlockT> ProvideCache for Blockchain { +impl, Block: BlockT> ProvideCache for Blockchain { fn cache(&self) -> Option>> { self.storage.cache() } } -impl RemoteBlockchain for Blockchain +impl RemoteBlockchain for Blockchain where S: Storage, { @@ -206,7 +200,7 @@ where } } -impl, Block: BlockT> ProvideChtRoots for Blockchain { +impl, Block: BlockT> ProvideChtRoots for Blockchain { fn header_cht_root( &self, cht_size: NumberFor, diff --git a/client/light/src/fetcher.rs b/client/light/src/fetcher.rs index 4785151270f4a..ca346f019d2b2 100644 --- a/client/light/src/fetcher.rs +++ b/client/light/src/fetcher.rs @@ -55,7 +55,7 @@ pub use sc_client_api::{ /// Remote data checker. pub struct LightDataChecker> { - blockchain: Arc>, + blockchain: Arc>, executor: E, spawn_handle: Box, _marker: PhantomData, @@ -64,7 +64,7 @@ pub struct LightDataChecker> { impl> LightDataChecker { /// Create new light data checker. pub fn new( - blockchain: Arc>, + blockchain: Arc>, executor: E, spawn_handle: Box, ) -> Self { diff --git a/client/light/src/lib.rs b/client/light/src/lib.rs index 7563377fd5e74..ca71a94496a44 100644 --- a/client/light/src/lib.rs +++ b/client/light/src/lib.rs @@ -34,7 +34,7 @@ pub use fetcher::*; /// Create an instance of fetch data checker. pub fn new_fetch_checker>( - blockchain: Arc>, + blockchain: Arc>, executor: E, spawn_handle: Box, ) -> LightDataChecker @@ -45,17 +45,18 @@ where } /// Create an instance of light client blockchain backend. -pub fn new_light_blockchain>(storage: S, state_versions: sp_runtime::StateVersions) -> Arc> { - Arc::new(Blockchain::new(storage, state_versions)) +pub fn new_light_blockchain>(storage: S) -> Arc> { + Arc::new(Blockchain::new(storage)) } /// Create an instance of light client backend. pub fn new_light_backend( - blockchain: Arc>, + blockchain: Arc>, + state_versions: sp_runtime::StateVersions, ) -> Arc> where B: BlockT, S: BlockchainStorage, { - Arc::new(Backend::new(blockchain)) + Arc::new(Backend::new(blockchain, state_versions)) } diff --git a/client/service/src/builder.rs b/client/service/src/builder.rs index 0fa7eba7df465..b854709bf750e 100644 --- a/client/service/src/builder.rs +++ b/client/service/src/builder.rs @@ -398,6 +398,13 @@ where }; sc_client_db::light::LightStorage::new(db_settings)? }; + let light_blockchain = sc_light::new_light_blockchain(db_storage); + let fetch_checker = Arc::new(sc_light::new_fetch_checker::<_, TBl, _>( + light_blockchain.clone(), + executor.clone(), + Box::new(task_manager.spawn_handle()), + )); + let on_demand = Arc::new(sc_network::config::OnDemand::new(fetch_checker)); let state_versions = StateVersions::from_conf( config @@ -410,15 +417,7 @@ where Error::Application(Box::from("Invalid state versions for chain spec".to_string())) })?; - let light_blockchain = sc_light::new_light_blockchain(db_storage, state_versions); - let fetch_checker = Arc::new(sc_light::new_fetch_checker::<_, TBl, _>( - light_blockchain.clone(), - executor.clone(), - Box::new(task_manager.spawn_handle()), - )); - let on_demand = Arc::new(sc_network::config::OnDemand::new(fetch_checker)); - - let backend = sc_light::new_light_backend(light_blockchain); + let backend = sc_light::new_light_backend(light_blockchain, state_versions); let client = Arc::new(light::new_light( backend.clone(), config.chain_spec.as_storage_builder(), diff --git a/client/service/test/src/client/light.rs b/client/service/test/src/client/light.rs index 23253993cfcc7..4f16f388a0ef0 100644 --- a/client/service/test/src/client/light.rs +++ b/client/service/test/src/client/light.rs @@ -64,7 +64,7 @@ use sp_core::{ }; use sp_state_machine::Backend as _; -pub type DummyBlockchain = Blockchain; +pub type DummyBlockchain = Blockchain; pub struct DummyStorage { pub changes_tries_cht_roots: HashMap, @@ -270,7 +270,7 @@ fn local_state_is_created_when_genesis_state_is_available() { substrate_test_runtime_client::runtime::Header::new(0, def, def, def, Default::default()); let backend: Backend<_, Block> = - Backend::new(Arc::new(DummyBlockchain::new(DummyStorage::new(), Default::default()))); + Backend::new(Arc::new(DummyBlockchain::new(DummyStorage::new())), Default::default()); let mut op = backend.begin_operation().unwrap(); op.set_block_data(header0, None, None, None, NewBlockState::Final).unwrap(); op.set_genesis_state(Default::default(), true).unwrap(); @@ -285,7 +285,7 @@ fn local_state_is_created_when_genesis_state_is_available() { #[test] fn unavailable_state_is_created_when_genesis_state_is_unavailable() { let backend: Backend<_, Block> = - Backend::new(Arc::new(DummyBlockchain::new(DummyStorage::new(), Default::default()))); + Backend::new(Arc::new(DummyBlockchain::new(DummyStorage::new())), Default::default()); match backend.state_at(BlockId::Number(0)).unwrap() { GenesisOrUnavailableState::Unavailable => (), @@ -296,7 +296,7 @@ fn unavailable_state_is_created_when_genesis_state_is_unavailable() { #[test] fn light_aux_store_is_updated_via_non_importing_op() { let backend = - Backend::new(Arc::new(DummyBlockchain::new(DummyStorage::new(), Default::default()))); + Backend::new(Arc::new(DummyBlockchain::new(DummyStorage::new())), Default::default()); let mut op = ClientBackend::::begin_operation(&backend).unwrap(); BlockImportOperation::::insert_aux(&mut op, vec![(vec![1], Some(vec![2]))]).unwrap(); ClientBackend::::commit_operation(&backend, op).unwrap(); @@ -487,7 +487,7 @@ fn prepare_for_read_proof_check(hashed_value: bool) -> (TestChecker, Header, Sto .insert(remote_block_hash, remote_block_header.clone(), None, None, NewBlockState::Final) .unwrap(); let local_checker = LightDataChecker::new( - Arc::new(DummyBlockchain::new(DummyStorage::new(), Default::default())), + Arc::new(DummyBlockchain::new(DummyStorage::new())), local_executor(), Box::new(TaskExecutor::new()), ); @@ -529,7 +529,7 @@ fn prepare_for_read_child_proof_check() -> (TestChecker, Header, StorageProof, V .insert(remote_block_hash, remote_block_header.clone(), None, None, NewBlockState::Final) .unwrap(); let local_checker = LightDataChecker::new( - Arc::new(DummyBlockchain::new(DummyStorage::new(), Default::default())), + Arc::new(DummyBlockchain::new(DummyStorage::new())), local_executor(), Box::new(TaskExecutor::new()), ); @@ -566,7 +566,7 @@ fn prepare_for_header_proof_check( local_storage.insert_cht_root(1, local_cht_root); } let local_checker = LightDataChecker::new( - Arc::new(DummyBlockchain::new(DummyStorage::new(), Default::default())), + Arc::new(DummyBlockchain::new(DummyStorage::new())), local_executor(), Box::new(TaskExecutor::new()), ); @@ -698,7 +698,7 @@ fn check_header_proof_fails_if_invalid_header_provided() { fn changes_proof_is_generated_and_checked_when_headers_are_not_pruned() { let (remote_client, local_roots, test_cases) = prepare_client_with_key_changes(); let local_checker = TestChecker::new( - Arc::new(DummyBlockchain::new(DummyStorage::new(), Default::default())), + Arc::new(DummyBlockchain::new(DummyStorage::new())), local_executor(), Box::new(TaskExecutor::new()), ); @@ -784,7 +784,7 @@ fn changes_proof_is_generated_and_checked_when_headers_are_pruned() { let mut local_storage = DummyStorage::new(); local_storage.changes_tries_cht_roots.insert(0, local_cht_root); let local_checker = TestChecker::new( - Arc::new(DummyBlockchain::new(local_storage, Default::default())), + Arc::new(DummyBlockchain::new(local_storage)), local_executor(), Box::new(TaskExecutor::new()), ); @@ -825,7 +825,7 @@ fn changes_proof_is_generated_and_checked_when_headers_are_pruned() { fn check_changes_proof_fails_if_proof_is_wrong() { let (remote_client, local_roots, test_cases) = prepare_client_with_key_changes(); let local_checker = TestChecker::new( - Arc::new(DummyBlockchain::new(DummyStorage::new(), Default::default())), + Arc::new(DummyBlockchain::new(DummyStorage::new())), local_executor(), Box::new(TaskExecutor::new()), ); @@ -938,7 +938,7 @@ fn check_changes_tries_proof_fails_if_proof_is_wrong() { // fails when changes trie CHT is missing from the local db let local_checker = TestChecker::new( - Arc::new(DummyBlockchain::new(DummyStorage::new(), Default::default())), + Arc::new(DummyBlockchain::new(DummyStorage::new())), local_executor(), Box::new(TaskExecutor::new()), ); @@ -950,7 +950,7 @@ fn check_changes_tries_proof_fails_if_proof_is_wrong() { let mut local_storage = DummyStorage::new(); local_storage.changes_tries_cht_roots.insert(0, local_cht_root); let local_checker = TestChecker::new( - Arc::new(DummyBlockchain::new(local_storage, Default::default())), + Arc::new(DummyBlockchain::new(local_storage)), local_executor(), Box::new(TaskExecutor::new()), ); @@ -966,7 +966,7 @@ fn check_body_proof_faulty() { let block = Block::new(header.clone(), Vec::new()); let local_checker = TestChecker::new( - Arc::new(DummyBlockchain::new(DummyStorage::new(), Default::default())), + Arc::new(DummyBlockchain::new(DummyStorage::new())), local_executor(), Box::new(TaskExecutor::new()), ); @@ -987,7 +987,7 @@ fn check_body_proof_of_same_data_should_succeed() { let block = Block::new(header.clone(), extrinsics); let local_checker = TestChecker::new( - Arc::new(DummyBlockchain::new(DummyStorage::new(), Default::default())), + Arc::new(DummyBlockchain::new(DummyStorage::new())), local_executor(), Box::new(TaskExecutor::new()), ); diff --git a/test-utils/runtime/client/src/lib.rs b/test-utils/runtime/client/src/lib.rs index 5dbc924c6bc00..18eeb52203cf7 100644 --- a/test-utils/runtime/client/src/lib.rs +++ b/test-utils/runtime/client/src/lib.rs @@ -419,10 +419,10 @@ pub fn new_light() -> ( >, Arc, ) { - let state_versions = Default::default(); let storage = sc_client_db::light::LightStorage::new_test(); - let blockchain = Arc::new(sc_light::Blockchain::new(storage, state_versions)); - let backend = Arc::new(LightBackend::new(blockchain)); + let blockchain = Arc::new(sc_light::Blockchain::new(storage)); + let state_versions = Default::default(); + let backend = Arc::new(LightBackend::new(blockchain, state_versions)); let executor = new_native_executor(); let local_call_executor = client::LocalCallExecutor::new( backend.clone(), From 889b1a4f625d93eefdf4d514227ac9ce95c29b4e Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 1 Sep 2021 17:42:21 +0200 Subject: [PATCH 115/127] remove comment --- client/light/src/backend.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/client/light/src/backend.rs b/client/light/src/backend.rs index 481d41c65f026..4fa32a9b1ba95 100644 --- a/client/light/src/backend.rs +++ b/client/light/src/backend.rs @@ -63,8 +63,6 @@ pub struct Backend { blockchain: Arc>, genesis_state: RwLock>>>, state_versions: StateVersions, - // TODO consider moving this state_versions into BlockChain (and add N to type) -> would make - // better api import_lock: RwLock<()>, } From 0d9188285c127025e799ab2ef0b77ad271d868d3 Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 1 Sep 2021 18:23:45 +0200 Subject: [PATCH 116/127] rem TODO --- client/db/src/lib.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index b0020d6c8447c..04392eb608ee2 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -1118,7 +1118,6 @@ pub struct Backend { io_stats: FrozenForDuration<(kvdb::IoStats, StateUsageInfo)>, state_usage: Arc, genesis_state: RwLock>>>, - // TODO consider moving this state_version into BlockChainBb state_versions: StateVersions, } From ae4bdcb6b000e6da3ef1acc9f406487f1f50ec33 Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 1 Sep 2021 18:24:40 +0200 Subject: [PATCH 117/127] remove todos --- client/db/src/lib.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index 04392eb608ee2..fb9c0c3a7b993 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -1130,7 +1130,6 @@ impl Backend { canonicalization_delay: u64, state_versions: StateVersions, ) -> ClientResult { - // TODO state_versions could also be part of database settings let db = crate::utils::open_database::(&config, DatabaseType::Full)?; Self::from_database(db as Arc<_>, canonicalization_delay, &config, state_versions) } @@ -1188,7 +1187,7 @@ impl Backend { db: Arc>, canonicalization_delay: u64, config: &DatabaseSettings, - state_versions: StateVersions, // TODO could also be part of database settings + state_versions: StateVersions, ) -> ClientResult { let is_archive_pruning = config.state_pruning.is_archive(); let blockchain = BlockchainDb::new(db.clone(), config.transaction_storage.clone())?; From 2c7e248441458ad7a4c27653495f138cbb6922c3 Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 2 Sep 2021 09:26:46 +0200 Subject: [PATCH 118/127] New trie code does update value on change. --- Cargo.lock | 16 ++++++++-------- primitives/state-machine/src/lib.rs | 16 +--------------- 2 files changed, 9 insertions(+), 23 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c3519f8d2a7cc..8afd7dbcf943d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2441,7 +2441,7 @@ dependencies = [ [[package]] name = "hash-db" version = "0.15.2" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#20cac451adff73c736680b8e067cc11b43de4499" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#b995d5b0b1040e183dac1b1112657ad917003377" [[package]] name = "hash256-std-hasher" @@ -2455,7 +2455,7 @@ dependencies = [ [[package]] name = "hash256-std-hasher" version = "0.15.2" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#20cac451adff73c736680b8e067cc11b43de4499" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#b995d5b0b1040e183dac1b1112657ad917003377" dependencies = [ "crunchy", ] @@ -3108,7 +3108,7 @@ dependencies = [ [[package]] name = "keccak-hasher" version = "0.15.3" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#20cac451adff73c736680b8e067cc11b43de4499" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#b995d5b0b1040e183dac1b1112657ad917003377" dependencies = [ "hash-db", "hash256-std-hasher 0.15.2 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple4)", @@ -3969,7 +3969,7 @@ dependencies = [ [[package]] name = "memory-db" version = "0.27.0" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#20cac451adff73c736680b8e067cc11b43de4499" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#b995d5b0b1040e183dac1b1112657ad917003377" dependencies = [ "hash-db", "hashbrown 0.11.2", @@ -10479,7 +10479,7 @@ checksum = "a7f741b240f1a48843f9b8e0444fb55fb2a4ff67293b50a9179dfd5ea67f8d41" [[package]] name = "trie-bench" version = "0.28.0" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#20cac451adff73c736680b8e067cc11b43de4499" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#b995d5b0b1040e183dac1b1112657ad917003377" dependencies = [ "criterion", "hash-db", @@ -10494,7 +10494,7 @@ dependencies = [ [[package]] name = "trie-db" version = "0.22.6" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#20cac451adff73c736680b8e067cc11b43de4499" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#b995d5b0b1040e183dac1b1112657ad917003377" dependencies = [ "hash-db", "hashbrown 0.11.2", @@ -10506,7 +10506,7 @@ dependencies = [ [[package]] name = "trie-root" version = "0.16.0" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#20cac451adff73c736680b8e067cc11b43de4499" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#b995d5b0b1040e183dac1b1112657ad917003377" dependencies = [ "hash-db", ] @@ -10524,7 +10524,7 @@ dependencies = [ [[package]] name = "trie-standardmap" version = "0.15.2" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#20cac451adff73c736680b8e067cc11b43de4499" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#b995d5b0b1040e183dac1b1112657ad917003377" dependencies = [ "hash-db", "keccak-hasher 0.15.3 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple4)", diff --git a/primitives/state-machine/src/lib.rs b/primitives/state-machine/src/lib.rs index 8f3c68ede3472..b00f343f1580f 100644 --- a/primitives/state-machine/src/lib.rs +++ b/primitives/state-machine/src/lib.rs @@ -1668,7 +1668,7 @@ mod tests { // do switch layout = Layout::with_alt_hashing(TRESHOLD); - // update with same value do not change + // update with same value do change { let mut trie = TrieDBMut::from_existing_with_layout(&mut mdb, &mut root, layout.clone()).unwrap(); @@ -1676,20 +1676,6 @@ mod tests { .expect("insert failed"); } let root3 = root.clone(); - assert!(root1 == root3); - // different value then same is enough to update - // from triedbmut persipective (do not - // work with state machine as only changes do makes - // it to payload (would require a special host function). - { - let mut trie = - TrieDBMut::from_existing_with_layout(&mut mdb, &mut root, layout.clone()).unwrap(); - trie.insert(b"foo222", vec![4u8].as_slice()) // inner hash - .expect("insert failed"); - trie.insert(b"foo222", vec![5u8; 100].as_slice()) // inner hash - .expect("insert failed"); - } - let root3 = root.clone(); assert!(root1 != root3); let remote_proof = check_proof(mdb.clone(), root.clone(), StateVersion::V1 { threshold: TRESHOLD }); From 3c529da9f64e1266a08d17094f29bab66b57533e Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Sun, 5 Sep 2021 14:30:24 +0200 Subject: [PATCH 119/127] switch sp-trie code. --- Cargo.lock | 24 ++-- Cargo.toml | 10 +- bin/node/bench/src/trie.rs | 2 - client/db/src/bench.rs | 7 +- client/db/src/lib.rs | 5 - .../state-machine/src/changes_trie/mod.rs | 2 - .../state-machine/src/changes_trie/storage.rs | 2 - .../state-machine/src/proving_backend.rs | 68 ++------- .../state-machine/src/trie_backend_essence.rs | 42 ------ primitives/trie/src/lib.rs | 104 +------------- primitives/trie/src/node_codec.rs | 136 ++++-------------- primitives/trie/src/node_header.rs | 70 +++------ primitives/trie/src/storage_proof.rs | 18 +-- primitives/trie/src/trie_codec.rs | 4 +- primitives/trie/src/trie_stream.rs | 136 +++++------------- 15 files changed, 127 insertions(+), 503 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8afd7dbcf943d..4cf634525cae1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2441,7 +2441,7 @@ dependencies = [ [[package]] name = "hash-db" version = "0.15.2" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#b995d5b0b1040e183dac1b1112657ad917003377" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-new#8cfbde80decc0cd1840e6baa0b0e48c7c7403fc1" [[package]] name = "hash256-std-hasher" @@ -2455,7 +2455,7 @@ dependencies = [ [[package]] name = "hash256-std-hasher" version = "0.15.2" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#b995d5b0b1040e183dac1b1112657ad917003377" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-new#8cfbde80decc0cd1840e6baa0b0e48c7c7403fc1" dependencies = [ "crunchy", ] @@ -3108,10 +3108,10 @@ dependencies = [ [[package]] name = "keccak-hasher" version = "0.15.3" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#b995d5b0b1040e183dac1b1112657ad917003377" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-new#8cfbde80decc0cd1840e6baa0b0e48c7c7403fc1" dependencies = [ "hash-db", - "hash256-std-hasher 0.15.2 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple4)", + "hash256-std-hasher 0.15.2 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-new)", "tiny-keccak", ] @@ -3969,7 +3969,7 @@ dependencies = [ [[package]] name = "memory-db" version = "0.27.0" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#b995d5b0b1040e183dac1b1112657ad917003377" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-new#8cfbde80decc0cd1840e6baa0b0e48c7c7403fc1" dependencies = [ "hash-db", "hashbrown 0.11.2", @@ -10479,22 +10479,22 @@ checksum = "a7f741b240f1a48843f9b8e0444fb55fb2a4ff67293b50a9179dfd5ea67f8d41" [[package]] name = "trie-bench" version = "0.28.0" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#b995d5b0b1040e183dac1b1112657ad917003377" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-new#8cfbde80decc0cd1840e6baa0b0e48c7c7403fc1" dependencies = [ "criterion", "hash-db", - "keccak-hasher 0.15.3 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple4)", + "keccak-hasher 0.15.3 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-new)", "memory-db", "parity-scale-codec", "trie-db", "trie-root", - "trie-standardmap 0.15.2 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple4)", + "trie-standardmap 0.15.2 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-new)", ] [[package]] name = "trie-db" version = "0.22.6" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#b995d5b0b1040e183dac1b1112657ad917003377" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-new#8cfbde80decc0cd1840e6baa0b0e48c7c7403fc1" dependencies = [ "hash-db", "hashbrown 0.11.2", @@ -10506,7 +10506,7 @@ dependencies = [ [[package]] name = "trie-root" version = "0.16.0" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#b995d5b0b1040e183dac1b1112657ad917003377" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-new#8cfbde80decc0cd1840e6baa0b0e48c7c7403fc1" dependencies = [ "hash-db", ] @@ -10524,10 +10524,10 @@ dependencies = [ [[package]] name = "trie-standardmap" version = "0.15.2" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4#b995d5b0b1040e183dac1b1112657ad917003377" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-new#8cfbde80decc0cd1840e6baa0b0e48c7c7403fc1" dependencies = [ "hash-db", - "keccak-hasher 0.15.3 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple4)", + "keccak-hasher 0.15.3 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-new)", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index b6e704f0764d5..39c98ae5605ea 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -267,8 +267,8 @@ zeroize = { opt-level = 3 } panic = "unwind" [patch.crates-io] -hash-db = { git = "https://github.com/cheme/trie.git", branch = "hashed-value-simple4" } -memory-db = { git = "https://github.com/cheme/trie.git", branch = "hashed-value-simple4" } -trie-db = { git = "https://github.com/cheme/trie.git", branch = "hashed-value-simple4" } -trie-root = { git = "https://github.com/cheme/trie.git", branch = "hashed-value-simple4" } -trie-bench = { git = "https://github.com/cheme/trie.git", branch = "hashed-value-simple4" } +hash-db = { git = "https://github.com/cheme/trie.git", branch = "hashed-value-simple4-new" } +memory-db = { git = "https://github.com/cheme/trie.git", branch = "hashed-value-simple4-new" } +trie-db = { git = "https://github.com/cheme/trie.git", branch = "hashed-value-simple4-new" } +trie-root = { git = "https://github.com/cheme/trie.git", branch = "hashed-value-simple4-new" } +trie-bench = { git = "https://github.com/cheme/trie.git", branch = "hashed-value-simple4-new" } diff --git a/bin/node/bench/src/trie.rs b/bin/node/bench/src/trie.rs index f4811703ed023..b5df064645e1b 100644 --- a/bin/node/bench/src/trie.rs +++ b/bin/node/bench/src/trie.rs @@ -171,8 +171,6 @@ impl sp_state_machine::Storage for Storage { let key = sp_trie::prefixed_key::(key, prefix); self.0.get(0, &key).map_err(|e| format!("Database backend error: {:?}", e)) } - - fn access_from(&self, _key: &Hash) {} } impl core::Benchmark for TrieReadBenchmark { diff --git a/client/db/src/bench.rs b/client/db/src/bench.rs index 9160c486a6dc0..a111217650b34 100644 --- a/client/db/src/bench.rs +++ b/client/db/src/bench.rs @@ -72,13 +72,8 @@ impl sp_state_machine::Storage> for StorageDb::LENGTH); - } - } } + /// State that manages the backend database reference. Allows runtime to control the database. pub struct BenchmarkingState { root: Cell, diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index fb9c0c3a7b993..4b95d13286340 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -1010,8 +1010,6 @@ impl sp_state_machine::Storage> for StorageDb sc_state_db::NodeDb for StorageDb { @@ -1039,7 +1037,6 @@ impl sp_state_machine::Storage> for DbGenesisStora use hash_db::HashDB; Ok(self.storage.get(key, prefix)) } - fn access_from(&self, _key: &Block::Hash) {} } struct EmptyStorage(pub Block::Hash); @@ -1057,8 +1054,6 @@ impl sp_state_machine::Storage> for EmptyStorage Result, String> { Ok(None) } - - fn access_from(&self, _key: &Block::Hash) {} } /// Frozen `value` at time `at`. diff --git a/primitives/state-machine/src/changes_trie/mod.rs b/primitives/state-machine/src/changes_trie/mod.rs index 1f12cf19ae553..40148095247dd 100644 --- a/primitives/state-machine/src/changes_trie/mod.rs +++ b/primitives/state-machine/src/changes_trie/mod.rs @@ -201,8 +201,6 @@ impl<'a, H: Hasher, N: BlockNumber> crate::TrieBackendStorage fn get(&self, key: &H::Out, prefix: Prefix) -> Result, String> { self.0.get(key, prefix) } - - fn access_from(&self, _key: &H::Out) {} } /// Changes trie configuration. diff --git a/primitives/state-machine/src/changes_trie/storage.rs b/primitives/state-machine/src/changes_trie/storage.rs index b1fca1eb971a9..6a81edc90ff11 100644 --- a/primitives/state-machine/src/changes_trie/storage.rs +++ b/primitives/state-machine/src/changes_trie/storage.rs @@ -211,6 +211,4 @@ where fn get(&self, key: &H::Out, prefix: Prefix) -> Result, String> { self.storage.get(key, prefix) } - - fn access_from(&self, _key: &H::Out) {} } diff --git a/primitives/state-machine/src/proving_backend.rs b/primitives/state-machine/src/proving_backend.rs index 52e8d3c527c44..efdea2024959c 100644 --- a/primitives/state-machine/src/proving_backend.rs +++ b/primitives/state-machine/src/proving_backend.rs @@ -30,9 +30,9 @@ use sp_core::{state_version::StateVersion, storage::ChildInfo}; pub use sp_trie::trie_types::TrieError; use sp_trie::{ empty_child_trie_root, read_child_trie_value_with, read_trie_value_with, record_all_keys, - Layout, MemoryDB, Meta, Recorder, StorageProof, + Layout, MemoryDB, Recorder, StorageProof, }; -use std::{collections::HashMap, sync::Arc}; +use std::{collections::{hash_map::Entry, HashMap}, sync::Arc}; /// Patricia trie-based backend specialized in get value proofs. pub struct ProvingBackendRecorder<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> { @@ -108,7 +108,7 @@ where #[derive(Default)] struct ProofRecorderInner { /// All the records that we have stored so far. - records: HashMap>, + records: HashMap>, /// The encoded size of all recorded values. encoded_size: usize, /// State version in use. @@ -125,43 +125,21 @@ impl ProofRecorder { /// Record the given `key` => `val` combination. pub fn record(&self, key: Hash, val: Option) { let mut inner = self.inner.write(); + let encoded_size = if let Entry::Vacant(entry) = inner.records.entry(key) { + let encoded_size = val.as_ref().map(Encode::encoded_size).unwrap_or(0); - let ProofRecorderInner { encoded_size, records, .. } = &mut *inner; - records.entry(key).or_insert_with(|| { - val.map(|val| { - let mut val = (val, Meta::default(), false); - sp_trie::resolve_encoded_meta::(&mut val); - *encoded_size += sp_trie::estimate_entry_size(&val, H::LENGTH); - val - }) - }); - } + entry.insert(val); + encoded_size + } else { + 0 + }; - /// Record actual trie level value access. - pub fn access_from(&self, key: &Hash, hash_len: usize) { - let mut inner = self.inner.write(); - let ProofRecorderInner { encoded_size, records, .. } = &mut *inner; - records.entry(key.clone()).and_modify(|entry| { - if let Some(entry) = entry.as_mut() { - if !entry.2 { - let old_size = sp_trie::estimate_entry_size(entry, hash_len); - entry.2 = true; - let new_size = sp_trie::estimate_entry_size(entry, hash_len); - *encoded_size += new_size; - *encoded_size -= old_size; - } - } - }); + inner.encoded_size += encoded_size; } /// Returns the value at the given `key`. pub fn get(&self, key: &Hash) -> Option> { - self.inner - .read() - .records - .get(key) - .as_ref() - .map(|v| v.as_ref().map(|v| v.0.clone())) + self.inner.read().records.get(key).cloned() } /// Returns the estimated encoded size of the proof. @@ -178,26 +156,10 @@ impl ProofRecorder { /// Convert into a [`StorageProof`]. pub fn to_storage_proof(&self) -> StorageProof { let inner = self.inner.read(); - let try_inner_hashing = match inner.state_version { - StateVersion::V0 => None, - StateVersion::V1 { threshold } => Some(threshold), - }; let trie_nodes = inner .records .iter() - .filter_map(|(_k, v)| { - v.as_ref().map(|v| { - let mut meta = v.1.clone(); - meta.try_inner_hashing = try_inner_hashing.clone(); - if let Some(hashed) = - sp_trie::to_hashed_variant::(v.0.as_slice(), &mut meta, v.2) - { - hashed - } else { - v.0.clone() - } - }) - }) + .filter_map(|(_k, v)| v.as_ref().map(|v| v.to_vec())) .collect(); StorageProof::new(trie_nodes, inner.state_version) @@ -272,10 +234,6 @@ impl<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> TrieBackendStorage self.proof_recorder.record::(key.clone(), backend_value.clone()); Ok(backend_value) } - - fn access_from(&self, key: &H::Out) { - self.proof_recorder.access_from(key, H::LENGTH); - } } impl<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> std::fmt::Debug diff --git a/primitives/state-machine/src/trie_backend_essence.rs b/primitives/state-machine/src/trie_backend_essence.rs index 663c17a787174..4bf26ac60e569 100644 --- a/primitives/state-machine/src/trie_backend_essence.rs +++ b/primitives/state-machine/src/trie_backend_essence.rs @@ -48,9 +48,6 @@ type Result = sp_std::result::Result; pub trait Storage: Send + Sync { /// Get a trie node. fn get(&self, key: &H::Out, prefix: Prefix) -> Result>; - - /// Call back when value get accessed in trie. - fn access_from(&self, key: &H::Out); } /// Local cache for child root. @@ -481,12 +478,6 @@ impl<'a, S: 'a + TrieBackendStorage, H: Hasher> hash_db::HashDB } } - fn access_from(&self, key: &H::Out, _at: Option<&H::Out>) -> Option { - // call back to storage even if the overlay was hit. - self.storage.access_from(key); - None - } - fn contains(&self, key: &H::Out, prefix: Prefix) -> bool { HashDB::get(self, key, prefix).is_some() } @@ -499,10 +490,6 @@ impl<'a, S: 'a + TrieBackendStorage, H: Hasher> hash_db::HashDB HashDB::emplace(self.overlay, key, prefix, value) } - fn emplace_ref(&mut self, key: &H::Out, prefix: Prefix, value: &[u8]) { - HashDB::emplace_ref(self.overlay, key, prefix, value) - } - fn remove(&mut self, key: &H::Out, prefix: Prefix) { HashDB::remove(self.overlay, key, prefix) } @@ -513,10 +500,6 @@ impl<'a, S: 'a + TrieBackendStorage, H: Hasher> HashDBRef for Eph HashDB::get(self, key, prefix) } - fn access_from(&self, key: &H::Out, at: Option<&H::Out>) -> Option { - HashDB::access_from(self, key, at) - } - fn contains(&self, key: &H::Out, prefix: Prefix) -> bool { HashDB::contains(self, key, prefix) } @@ -529,9 +512,6 @@ pub trait TrieBackendStorage: Send + Sync { /// Get the value stored at key. fn get(&self, key: &H::Out, prefix: Prefix) -> Result>; - - /// Call back when value get accessed in trie. - fn access_from(&self, key: &H::Out); } // This implementation is used by normal storage trie clients. @@ -542,10 +522,6 @@ impl TrieBackendStorage for Arc> { fn get(&self, key: &H::Out, prefix: Prefix) -> Result> { Storage::::get(self.deref(), key, prefix) } - - fn access_from(&self, key: &H::Out) { - Storage::::access_from(self.deref(), key) - } } impl TrieBackendStorage for sp_trie::GenericMemoryDB @@ -558,10 +534,6 @@ where fn get(&self, key: &H::Out, prefix: Prefix) -> Result> { Ok(hash_db::HashDB::get(self, key, prefix)) } - - fn access_from(&self, key: &H::Out) { - HashDB::access_from(self, key, None); - } } impl, H: Hasher> AsHashDB for TrieBackendEssence { @@ -587,12 +559,6 @@ impl, H: Hasher> HashDB for TrieBackendEsse } } - fn access_from(&self, key: &H::Out, _at: Option<&H::Out>) -> Option { - // access storage since this is only to register access for proof. - self.storage.access_from(key); - None - } - fn contains(&self, key: &H::Out, prefix: Prefix) -> bool { HashDB::get(self, key, prefix).is_some() } @@ -605,10 +571,6 @@ impl, H: Hasher> HashDB for TrieBackendEsse unimplemented!(); } - fn emplace_ref(&mut self, _key: &H::Out, _prefix: Prefix, _value: &[u8]) { - unimplemented!(); - } - fn remove(&mut self, _key: &H::Out, _prefix: Prefix) { unimplemented!(); } @@ -619,10 +581,6 @@ impl, H: Hasher> HashDBRef for TrieBackendE HashDB::get(self, key, prefix) } - fn access_from(&self, key: &H::Out, at: Option<&H::Out>) -> Option { - HashDB::access_from(self, key, at) - } - fn contains(&self, key: &H::Out, prefix: Prefix) -> bool { HashDB::contains(self, key, prefix) } diff --git a/primitives/trie/src/lib.rs b/primitives/trie/src/lib.rs index 179f44db1e7a0..eeb85b8f11ae4 100644 --- a/primitives/trie/src/lib.rs +++ b/primitives/trie/src/lib.rs @@ -36,7 +36,7 @@ pub use memory_db::prefixed_key; pub use memory_db::KeyFunction; /// The Substrate format implementation of `NodeCodec`. pub use node_codec::NodeCodec; -use sp_std::{borrow::Borrow, boxed::Box, fmt, marker::PhantomData, vec, vec::Vec}; +use sp_std::{borrow::Borrow, boxed::Box, fmt, marker::PhantomData, vec::Vec}; pub use storage_proof::{CompactProof, StorageProof}; /// Trie codec reexport, mainly child trie support /// for trie compact proof. @@ -47,7 +47,7 @@ use trie_db::proof::{generate_proof, verify_proof}; pub use trie_db::{ nibble_ops, node::{NodePlan, ValuePlan}, - CError, DBValue, Meta, Query, Recorder, Trie, TrieConfiguration, TrieDBIterator, + CError, DBValue, Query, Recorder, Trie, TrieConfiguration, TrieDBIterator, TrieDBKeyIterator, TrieLayout, TrieMut, }; /// The Substrate format implementation of `TrieStream`. @@ -88,7 +88,6 @@ where { const USE_EXTENSION: bool = false; const ALLOW_EMPTY: bool = true; - const USE_META: bool = true; type Hash = H; type Codec = NodeCodec; @@ -214,9 +213,7 @@ where K: 'a + AsRef<[u8]>, V: 'a + AsRef<[u8]>, { - // No specific info to read from layout. - let layout = Default::default(); - verify_proof::, _, _, _>(root, proof, items, layout) + verify_proof::, _, _, _>(root, proof, items) } /// Determine a trie root given a hash DB and delta values. @@ -444,10 +441,6 @@ where self.0.get(key, (&derived_prefix.0, derived_prefix.1)) } - fn access_from(&self, key: &H::Out, at: Option<&H::Out>) -> Option { - self.0.access_from(key, at) - } - fn contains(&self, key: &H::Out, prefix: Prefix) -> bool { let derived_prefix = keyspace_as_prefix_alloc(self.1, prefix); self.0.contains(key, (&derived_prefix.0, derived_prefix.1)) @@ -465,10 +458,6 @@ where self.0.get(key, (&derived_prefix.0, derived_prefix.1)) } - fn access_from(&self, key: &H::Out, at: Option<&H::Out>) -> Option { - self.0.access_from(key, at) - } - fn contains(&self, key: &H::Out, prefix: Prefix) -> bool { let derived_prefix = keyspace_as_prefix_alloc(self.1, prefix); self.0.contains(key, (&derived_prefix.0, derived_prefix.1)) @@ -484,11 +473,6 @@ where self.0.emplace(key, (&derived_prefix.0, derived_prefix.1), value) } - fn emplace_ref(&mut self, key: &H::Out, prefix: Prefix, value: &[u8]) { - let derived_prefix = keyspace_as_prefix_alloc(self.1, prefix); - self.0.emplace_ref(key, (&derived_prefix.0, derived_prefix.1), value) - } - fn remove(&mut self, key: &H::Out, prefix: Prefix) { let derived_prefix = keyspace_as_prefix_alloc(self.1, prefix); self.0.remove(key, (&derived_prefix.0, derived_prefix.1)) @@ -510,90 +494,9 @@ where } } -/// Representation of node with with inner hash instead of value. -fn inner_hashed_value(x: &[u8], range: Option<(usize, usize)>) -> Vec { - if let Some((start, end)) = range { - let len = x.len(); - if start < len && end == len { - // terminal inner hash - let hash_end = H::hash(&x[start..]); - let mut buff = vec![0; x.len() + hash_end.as_ref().len() - (end - start)]; - buff[..start].copy_from_slice(&x[..start]); - buff[start..].copy_from_slice(hash_end.as_ref()); - return buff - } - if start == 0 && end < len { - // start inner hash - let hash_start = H::hash(&x[..start]); - let hash_len = hash_start.as_ref().len(); - let mut buff = vec![0; x.len() + hash_len - (end - start)]; - buff[..hash_len].copy_from_slice(hash_start.as_ref()); - buff[hash_len..].copy_from_slice(&x[end..]); - return buff - } - if start < len && end < len { - // middle inner hash - let hash_middle = H::hash(&x[start..end]); - let hash_len = hash_middle.as_ref().len(); - let mut buff = vec![0; x.len() + hash_len - (end - start)]; - buff[..start].copy_from_slice(&x[..start]); - buff[start..start + hash_len].copy_from_slice(hash_middle.as_ref()); - buff[start + hash_len..].copy_from_slice(&x[end..]); - return buff - } - } - // if anything wrong default to hash - x.to_vec() -} - -/// Estimate encoded size of node. -pub fn estimate_entry_size(entry: &(DBValue, Meta, bool), hash_len: usize) -> usize { - use codec::Encode; - let mut full_encoded = entry.0.encoded_size(); - if !entry.2 && entry.1.apply_inner_hashing { - if let Some(range) = entry.1.range.as_ref() { - let value_size = range.end - range.start; - full_encoded -= value_size; - full_encoded += hash_len; - full_encoded += 1; - } - } - - full_encoded -} - -/// Switch to hashed value variant. -pub fn to_hashed_variant( - value: &[u8], - meta: &mut Meta, - used_value: bool, -) -> Option { - if !meta.contain_hash && meta.apply_inner_hashing && !used_value && meta.range.is_some() { - let mut stored = Vec::with_capacity(value.len() + 1); - // Warning this assumes that encoded value cannot start by this, - // so it is tightly coupled with the header type of the codec. - stored.push(trie_constants::DEAD_HEADER_META_HASHED_VALUE); - let range = meta.range.as_ref().expect("Tested in condition"); - // store hash instead of value. - let value = inner_hashed_value::(value, Some((range.start, range.end))); - stored.extend_from_slice(value.as_slice()); - meta.contain_hash = true; - return Some(stored) - } - None -} - -/// Decode plan in order to update meta early (needed to register proofs). -pub fn resolve_encoded_meta(entry: &mut (DBValue, Meta, bool)) { - use trie_db::NodeCodec; - let _ = as TrieLayout>::Codec::decode_plan(entry.0.as_slice(), &mut entry.1); -} - /// Constants used into trie simplification codec. mod trie_constants { const FIRST_PREFIX: u8 = 0b_00 << 6; - /// In proof this header is used when only hashed value is stored. - pub const DEAD_HEADER_META_HASHED_VALUE: u8 = EMPTY_TRIE | 0b_00_01; pub const NIBBLE_SIZE_BOUND: usize = u16::max_value() as usize; pub const LEAF_PREFIX_MASK: u8 = 0b_01 << 6; pub const BRANCH_WITHOUT_MASK: u8 = 0b_10 << 6; @@ -601,6 +504,7 @@ mod trie_constants { pub const EMPTY_TRIE: u8 = FIRST_PREFIX | (0b_00 << 4); pub const ALT_HASHING_LEAF_PREFIX_MASK: u8 = FIRST_PREFIX | (0b_1 << 5); pub const ALT_HASHING_BRANCH_WITH_MASK: u8 = FIRST_PREFIX | (0b_01 << 4); + pub const ESCAPE_COMPACT_HEADER: u8 = EMPTY_TRIE | 0b_00_01; } #[cfg(test)] diff --git a/primitives/trie/src/node_codec.rs b/primitives/trie/src/node_codec.rs index d23ed0d6af801..ce5f84df3c718 100644 --- a/primitives/trie/src/node_codec.rs +++ b/primitives/trie/src/node_codec.rs @@ -25,7 +25,7 @@ use sp_std::{borrow::Borrow, marker::PhantomData, ops::Range, vec::Vec}; use trie_db::{ self, nibble_ops, node::{NibbleSlicePlan, NodeHandlePlan, NodePlan, Value, ValuePlan}, - ChildReference, Meta, NodeCodec as NodeCodecT, Partial, + ChildReference, NodeCodec as NodeCodecT, Partial, }; /// Helper struct for trie node decoder. This implements `codec::Input` on a byte slice, while @@ -81,13 +81,11 @@ impl<'a> Input for ByteSliceInput<'a> { pub struct NodeCodec(PhantomData); impl NodeCodec { - fn decode_plan_inner_hashed(data: &[u8], meta: &mut Meta) -> Result { + fn decode_plan_inner_hashed(data: &[u8]) -> Result { let mut input = ByteSliceInput::new(data); let header = NodeHeader::decode(&mut input)?; let contains_hash = header.contains_hash_of_value(); - let alt_hashing = header.alt_hashing(); - meta.apply_inner_hashing = alt_hashing; let branch_has_value = if let NodeHeader::Branch(has_value, _) = &header { *has_value @@ -98,7 +96,8 @@ impl NodeCodec { match header { NodeHeader::Null => Ok(NodePlan::Empty), - NodeHeader::AltHashBranch(nibble_count, _) | NodeHeader::Branch(_, nibble_count) => { + NodeHeader::AltHashBranch(nibble_count) + | NodeHeader::Branch(_, nibble_count) => { let padding = nibble_count % nibble_ops::NIBBLE_PER_BYTE != 0; // check that the padding is valid (if any) if padding && nibble_ops::pad_left(data[input.offset]) != 0 { @@ -112,19 +111,18 @@ impl NodeCodec { let bitmap_range = input.take(BITMAP_LENGTH)?; let bitmap = Bitmap::decode(&data[bitmap_range])?; let value = if branch_has_value { - if alt_hashing && contains_hash { + if contains_hash { ValuePlan::HashedValue(input.take(H::LENGTH)?) } else { - let with_len = input.offset; let count = >::decode(&mut input)?.0 as usize; - ValuePlan::Value(input.take(count)?, with_len) + ValuePlan::Value(input.take(count)?) } } else { ValuePlan::NoValue }; let mut children = [ - None, None, None, None, None, None, None, None, None, None, None, None, None, - None, None, None, + None, None, None, None, None, None, None, None, + None, None, None, None, None, None, None, None, ]; for i in 0..nibble_ops::NIBBLE_LENGTH { if bitmap.value_at(i) { @@ -143,7 +141,8 @@ impl NodeCodec { children, }) }, - NodeHeader::AltHashLeaf(nibble_count, _) | NodeHeader::Leaf(nibble_count) => { + NodeHeader::AltHashLeaf(nibble_count) + | NodeHeader::Leaf(nibble_count) => { let padding = nibble_count % nibble_ops::NIBBLE_PER_BYTE != 0; // check that the padding is valid (if any) if padding && nibble_ops::pad_left(data[input.offset]) != 0 { @@ -154,12 +153,11 @@ impl NodeCodec { nibble_ops::NIBBLE_PER_BYTE, )?; let partial_padding = nibble_ops::number_padding(nibble_count); - let value = if alt_hashing && contains_hash { + let value = if contains_hash { ValuePlan::HashedValue(input.take(H::LENGTH)?) } else { - let with_len = input.offset; let count = >::decode(&mut input)?.0 as usize; - ValuePlan::Value(input.take(count)?, with_len) + ValuePlan::Value(input.take(count)?) }; Ok(NodePlan::Leaf { @@ -175,7 +173,7 @@ impl NodeCodecT for NodeCodec where H: Hasher, { - const OFFSET_IF_CONTAINS_HASH: usize = 1; + const ESCAPE_HEADER: Option<&'static [u8]> = Some(&[trie_constants::ESCAPE_COMPACT_HEADER]); type Error = Error; type HashOut = H::Out; @@ -183,15 +181,8 @@ where H::hash(::empty_node()) } - fn decode_plan(data: &[u8], meta: &mut Meta) -> Result { - Self::decode_plan_inner_hashed(data, meta).map(|plan| { - meta.decoded_callback(&plan); - plan - }) - } - - fn decode_plan_inner(_data: &[u8]) -> Result { - unreachable!("decode_plan is implemented") + fn decode_plan(data: &[u8]) -> Result { + Self::decode_plan_inner_hashed(data) } fn is_empty_node(data: &[u8]) -> bool { @@ -202,42 +193,21 @@ where &[trie_constants::EMPTY_TRIE] } - fn leaf_node(partial: Partial, value: Value, meta: &mut Meta) -> Vec { + fn leaf_node(partial: Partial, value: Value) -> Vec { let contains_hash = matches!(&value, Value::HashedValue(..)); - // Note that we use AltHash type only if inner hashing will occur, - // this way we allow changing hash threshold. - // With fix inner hashing alt hash can be use with all node, but - // that is not better (encoding can use an additional nibble byte - // sometime). - let mut output = if meta - .try_inner_hashing - .as_ref() - .map(|threshold| value_do_hash(&value, threshold)) - .unwrap_or(meta.apply_inner_hashing) - { - if contains_hash { - partial_encode(partial, NodeKind::AltHashLeafHash) - } else { - partial_encode(partial, NodeKind::AltHashLeaf) - } + let mut output = if contains_hash { + partial_encode(partial, NodeKind::AltHashLeaf) } else { partial_encode(partial, NodeKind::Leaf) }; match value { Value::Value(value) => { - let with_len = output.len(); Compact(value.len() as u32).encode_to(&mut output); - let start = output.len(); output.extend_from_slice(value); - let end = output.len(); - meta.encoded_value_callback(ValuePlan::Value(start..end, with_len)); }, - Value::HashedValue(hash) => { + Value::HashedValue(hash, _) => { debug_assert!(hash.len() == H::LENGTH); - let start = output.len(); output.extend_from_slice(hash); - let end = output.len(); - meta.encoded_value_callback(ValuePlan::HashedValue(start..end)); }, Value::NoValue => unimplemented!("No support for incomplete nodes"), } @@ -248,7 +218,6 @@ where _partial: impl Iterator, _nbnibble: usize, _child: ChildReference<::Out>, - _meta: &mut Meta, ) -> Vec { unreachable!() } @@ -256,7 +225,6 @@ where fn branch_node( _children: impl Iterator::Out>>>>, _maybe_value: Value, - _meta: &mut Meta, ) -> Vec { unreachable!() } @@ -266,34 +234,15 @@ where number_nibble: usize, children: impl Iterator::Out>>>>, value: Value, - meta: &mut Meta, ) -> Vec { let contains_hash = matches!(&value, Value::HashedValue(..)); - let mut output = match ( - &value, - meta.try_inner_hashing - .as_ref() - .map(|threshold| value_do_hash(&value, threshold)) - .unwrap_or(meta.apply_inner_hashing), - ) { + let mut output = match (&value, contains_hash) { (&Value::NoValue, _) => partial_from_iterator_encode(partial, number_nibble, NodeKind::BranchNoValue), (_, false) => partial_from_iterator_encode(partial, number_nibble, NodeKind::BranchWithValue), (_, true) => - if contains_hash { - partial_from_iterator_encode( - partial, - number_nibble, - NodeKind::AltHashBranchWithValueHash, - ) - } else { - partial_from_iterator_encode( - partial, - number_nibble, - NodeKind::AltHashBranchWithValue, - ) - }, + partial_from_iterator_encode(partial, number_nibble, NodeKind::AltHashBranchWithValue), }; let bitmap_index = output.len(); @@ -301,19 +250,12 @@ where (0..BITMAP_LENGTH).for_each(|_| output.push(0)); match value { Value::Value(value) => { - let with_len = output.len(); Compact(value.len() as u32).encode_to(&mut output); - let start = output.len(); output.extend_from_slice(value); - let end = output.len(); - meta.encoded_value_callback(ValuePlan::Value(start..end, with_len)); }, - Value::HashedValue(hash) => { + Value::HashedValue(hash, _) => { debug_assert!(hash.len() == H::LENGTH); - let start = output.len(); output.extend_from_slice(hash); - let end = output.len(); - meta.encoded_value_callback(ValuePlan::HashedValue(start..end)); }, Value::NoValue => (), } @@ -339,14 +281,6 @@ where // utils -fn value_do_hash(val: &Value, threshold: &u32) -> bool { - match val { - Value::Value(val) => val.encoded_size() >= *threshold as usize, - Value::HashedValue(..) => true, // can only keep hashed - Value::NoValue => false, - } -} - /// Encode and allocate node type header (type and size), and partial value. /// It uses an iterator over encoded partial bytes as input. fn partial_from_iterator_encode>( @@ -356,19 +290,14 @@ fn partial_from_iterator_encode>( ) -> Vec { let nibble_count = sp_std::cmp::min(trie_constants::NIBBLE_SIZE_BOUND, nibble_count); - let mut output = Vec::with_capacity(3 + (nibble_count / nibble_ops::NIBBLE_PER_BYTE)); + let mut output = Vec::with_capacity(4 + (nibble_count / nibble_ops::NIBBLE_PER_BYTE)); match node_kind { NodeKind::Leaf => NodeHeader::Leaf(nibble_count).encode_to(&mut output), NodeKind::BranchWithValue => NodeHeader::Branch(true, nibble_count).encode_to(&mut output), NodeKind::BranchNoValue => NodeHeader::Branch(false, nibble_count).encode_to(&mut output), - NodeKind::AltHashLeaf => - NodeHeader::AltHashLeaf(nibble_count, false).encode_to(&mut output), - NodeKind::AltHashBranchWithValue => - NodeHeader::AltHashBranch(nibble_count, false).encode_to(&mut output), - NodeKind::AltHashLeafHash => - NodeHeader::AltHashLeaf(nibble_count, true).encode_to(&mut output), - NodeKind::AltHashBranchWithValueHash => - NodeHeader::AltHashBranch(nibble_count, true).encode_to(&mut output), + NodeKind::AltHashLeaf => NodeHeader::AltHashLeaf(nibble_count).encode_to(&mut output), + NodeKind::AltHashBranchWithValue => NodeHeader::AltHashBranch(nibble_count) + .encode_to(&mut output), }; output.extend(partial); output @@ -382,19 +311,14 @@ fn partial_encode(partial: Partial, node_kind: NodeKind) -> Vec { let nibble_count = sp_std::cmp::min(trie_constants::NIBBLE_SIZE_BOUND, nibble_count); - let mut output = Vec::with_capacity(3 + partial.1.len()); + let mut output = Vec::with_capacity(4 + partial.1.len()); match node_kind { NodeKind::Leaf => NodeHeader::Leaf(nibble_count).encode_to(&mut output), NodeKind::BranchWithValue => NodeHeader::Branch(true, nibble_count).encode_to(&mut output), NodeKind::BranchNoValue => NodeHeader::Branch(false, nibble_count).encode_to(&mut output), - NodeKind::AltHashLeaf => - NodeHeader::AltHashLeaf(nibble_count, false).encode_to(&mut output), - NodeKind::AltHashBranchWithValue => - NodeHeader::AltHashBranch(nibble_count, false).encode_to(&mut output), - NodeKind::AltHashLeafHash => - NodeHeader::AltHashLeaf(nibble_count, true).encode_to(&mut output), - NodeKind::AltHashBranchWithValueHash => - NodeHeader::AltHashBranch(nibble_count, true).encode_to(&mut output), + NodeKind::AltHashLeaf => NodeHeader::AltHashLeaf(nibble_count).encode_to(&mut output), + NodeKind::AltHashBranchWithValue => NodeHeader::AltHashBranch(nibble_count) + .encode_to(&mut output), }; if number_nibble_encoded > 0 { output.push(nibble_ops::pad_right((partial.0).1)); diff --git a/primitives/trie/src/node_header.rs b/primitives/trie/src/node_header.rs index 2443ad03dc53e..6130d118ed864 100644 --- a/primitives/trie/src/node_header.rs +++ b/primitives/trie/src/node_header.rs @@ -29,16 +29,17 @@ pub(crate) enum NodeHeader { Branch(bool, usize), // contains nibble count Leaf(usize), - // contains nibble count and wether the value is a hash. - AltHashBranch(usize, bool), - // contains nibble count and wether the value is a hash. - AltHashLeaf(usize, bool), + // contains nibble count. + AltHashBranch(usize), + // contains nibble count. + AltHashLeaf(usize), } impl NodeHeader { pub(crate) fn contains_hash_of_value(&self) -> bool { match self { - NodeHeader::AltHashBranch(_, true) | NodeHeader::AltHashLeaf(_, true) => true, + NodeHeader::AltHashBranch(_) + | NodeHeader::AltHashLeaf(_) => true, _ => false, } } @@ -51,49 +52,22 @@ pub(crate) enum NodeKind { BranchWithValue, AltHashLeaf, AltHashBranchWithValue, - AltHashLeafHash, - AltHashBranchWithValueHash, } impl Encode for NodeHeader { fn encode_to(&self, output: &mut T) { - if self.contains_hash_of_value() { - output.write(&[trie_constants::DEAD_HEADER_META_HASHED_VALUE]); - } match self { NodeHeader::Null => output.push_byte(trie_constants::EMPTY_TRIE), - NodeHeader::Branch(true, nibble_count) => + NodeHeader::Branch(true, nibble_count) => encode_size_and_prefix(*nibble_count, trie_constants::BRANCH_WITH_MASK, 2, output), - NodeHeader::Branch(false, nibble_count) => encode_size_and_prefix( - *nibble_count, - trie_constants::BRANCH_WITHOUT_MASK, - 2, - output, - ), + NodeHeader::Branch(false, nibble_count) => + encode_size_and_prefix(*nibble_count, trie_constants::BRANCH_WITHOUT_MASK, 2, output), NodeHeader::Leaf(nibble_count) => encode_size_and_prefix(*nibble_count, trie_constants::LEAF_PREFIX_MASK, 2, output), - NodeHeader::AltHashBranch(nibble_count, _) => encode_size_and_prefix( - *nibble_count, - trie_constants::ALT_HASHING_BRANCH_WITH_MASK, - 4, - output, - ), - NodeHeader::AltHashLeaf(nibble_count, _) => encode_size_and_prefix( - *nibble_count, - trie_constants::ALT_HASHING_LEAF_PREFIX_MASK, - 3, - output, - ), - } - } -} - -impl NodeHeader { - /// Is this header using alternate hashing scheme. - pub(crate) fn alt_hashing(&self) -> bool { - match self { - NodeHeader::Null | NodeHeader::Leaf(..) | NodeHeader::Branch(..) => false, - NodeHeader::AltHashBranch(..) | NodeHeader::AltHashLeaf(..) => true, + NodeHeader::AltHashBranch(nibble_count) => + encode_size_and_prefix(*nibble_count, trie_constants::ALT_HASHING_BRANCH_WITH_MASK, 4, output), + NodeHeader::AltHashLeaf(nibble_count) => + encode_size_and_prefix(*nibble_count, trie_constants::ALT_HASHING_LEAF_PREFIX_MASK, 3, output), } } } @@ -102,27 +76,19 @@ impl codec::EncodeLike for NodeHeader {} impl Decode for NodeHeader { fn decode(input: &mut I) -> Result { - let mut i = input.read_byte()?; + let i = input.read_byte()?; if i == trie_constants::EMPTY_TRIE { return Ok(NodeHeader::Null) } - let contain_hash = if trie_constants::DEAD_HEADER_META_HASHED_VALUE == i { - i = input.read_byte()?; - true - } else { - false - }; match i & (0b11 << 6) { trie_constants::LEAF_PREFIX_MASK => Ok(NodeHeader::Leaf(decode_size(i, input, 2)?)), - trie_constants::BRANCH_WITH_MASK => - Ok(NodeHeader::Branch(true, decode_size(i, input, 2)?)), - trie_constants::BRANCH_WITHOUT_MASK => - Ok(NodeHeader::Branch(false, decode_size(i, input, 2)?)), + trie_constants::BRANCH_WITH_MASK => Ok(NodeHeader::Branch(true, decode_size(i, input, 2)?)), + trie_constants::BRANCH_WITHOUT_MASK => Ok(NodeHeader::Branch(false, decode_size(i, input, 2)?)), trie_constants::EMPTY_TRIE => { if i & (0b111 << 5) == trie_constants::ALT_HASHING_LEAF_PREFIX_MASK { - Ok(NodeHeader::AltHashLeaf(decode_size(i, input, 3)?, contain_hash)) + Ok(NodeHeader::AltHashLeaf(decode_size(i, input, 3)?)) } else if i & (0b1111 << 4) == trie_constants::ALT_HASHING_BRANCH_WITH_MASK { - Ok(NodeHeader::AltHashBranch(decode_size(i, input, 4)?, contain_hash)) + Ok(NodeHeader::AltHashBranch(decode_size(i, input, 4)?)) } else { // do not allow any special encoding Err("Unallowed encoding".into()) diff --git a/primitives/trie/src/storage_proof.rs b/primitives/trie/src/storage_proof.rs index 1bdbc91475ed2..e0a89621c7bd3 100644 --- a/primitives/trie/src/storage_proof.rs +++ b/primitives/trie/src/storage_proof.rs @@ -15,12 +15,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::{Layout, TrieLayout}; +use crate::Layout; use codec::{Decode, Encode}; use hash_db::{HashDB, Hasher}; use sp_core::state_version::StateVersion; use sp_std::vec::Vec; -use trie_db::NodeCodec; /// A proof that some set of key-value pairs are included in the storage trie. The proof contains /// the storage values so that the partial storage backend can be reconstructed by a verifier that @@ -225,19 +224,8 @@ impl Iterator for StorageProofNodeIterator { impl From for crate::MemoryDB { fn from(proof: StorageProof) -> Self { let mut db = crate::MemoryDB::default(); - for item in proof.trie_nodes.iter() { - let mut meta = crate::Meta::default(); - meta.try_inner_hashing = match proof.state_version { - StateVersion::V0 => None, - StateVersion::V1 { threshold, .. } => Some(threshold), - }; - // Read meta from state (required for value layout). - let _ = as TrieLayout>::Codec::decode_plan(item.as_slice(), &mut meta); - db.alt_insert( - crate::EMPTY_PREFIX, - item, - meta.resolve_alt_hashing::< as TrieLayout>::Codec>(), - ); + for item in proof.iter_nodes() { + db.insert(crate::EMPTY_PREFIX, &item); } db } diff --git a/primitives/trie/src/trie_codec.rs b/primitives/trie/src/trie_codec.rs index eb8bb3607a9fa..299cf3cda5f4c 100644 --- a/primitives/trie/src/trie_codec.rs +++ b/primitives/trie/src/trie_codec.rs @@ -115,7 +115,7 @@ where // Layout does not change trie reading. let layout = L::default(); let (top_root, _nb_used) = - trie_db::decode_compact_from_iter::(db, &mut nodes_iter, &layout)?; + trie_db::decode_compact_from_iter::(db, &mut nodes_iter)?; // Only check root if expected root is passed as argument. if let Some(expected_root) = expected_root { @@ -166,7 +166,7 @@ where for child_root in child_tries.into_iter() { if previous_extracted_child_trie.is_none() { let (top_root, _) = - trie_db::decode_compact_from_iter::(db, &mut nodes_iter, &layout)?; + trie_db::decode_compact_from_iter::(db, &mut nodes_iter)?; previous_extracted_child_trie = Some(top_root); } diff --git a/primitives/trie/src/trie_stream.rs b/primitives/trie/src/trie_stream.rs index aba5ea3d4aa14..e9e35c3f12c3c 100644 --- a/primitives/trie/src/trie_stream.rs +++ b/primitives/trie/src/trie_stream.rs @@ -18,29 +18,19 @@ //! `TrieStream` implementation for Substrate's trie format. use crate::{ - node_codec::Bitmap, node_header::{size_and_prefix_iterator, NodeKind}, trie_constants, }; use codec::{Compact, Encode}; use hash_db::Hasher; -use sp_std::{ops::Range, vec::Vec}; +use sp_std::vec::Vec; use trie_root; -const BRANCH_NODE_NO_VALUE: u8 = 254; -const BRANCH_NODE_WITH_VALUE: u8 = 255; - #[derive(Default, Clone)] /// Codec-flavored TrieStream. pub struct TrieStream { /// Current node buffer. buffer: Vec, - /// Global trie alt hashing activation. - inner_value_hashing: Option, - /// For current node, do we use alt hashing. - apply_inner_hashing: bool, - /// Keep trace of position of encoded value. - current_value_range: Option>, } impl TrieStream { @@ -76,21 +66,17 @@ fn fuse_nibbles_node<'a>(nibbles: &'a [u8], kind: NodeKind) -> impl Iterator size_and_prefix_iterator(size, trie_constants::ALT_HASHING_BRANCH_WITH_MASK, 4), - NodeKind::AltHashBranchWithValueHash | NodeKind::AltHashLeafHash => - unreachable!("only added value that do not contain hash"), }; iter_start .chain(if nibbles.len() % 2 == 1 { Some(nibbles[0]) } else { None }) .chain(nibbles[nibbles.len() % 2..].chunks(2).map(|ch| ch[0] << 4 | ch[1])) } +use trie_root::Value as RValue; impl trie_root::TrieStream for TrieStream { - fn new(meta: Option) -> Self { + fn new() -> Self { Self { buffer: Vec::new(), - inner_value_hashing: meta, - apply_inner_hashing: false, - current_value_range: None, } } @@ -98,53 +84,53 @@ impl trie_root::TrieStream for TrieStream { self.buffer.push(trie_constants::EMPTY_TRIE); } - fn append_leaf(&mut self, key: &[u8], value: &[u8]) { - self.apply_inner_hashing = self - .inner_value_hashing - .as_ref() - .map(|threshold| value_do_hash(value, threshold)) - .unwrap_or(false); - let kind = if self.apply_inner_hashing { NodeKind::AltHashLeaf } else { NodeKind::Leaf }; + fn append_leaf(&mut self, key: &[u8], value: RValue) { + let kind = match &value { + RValue::NoValue => unreachable!(), + RValue::Value(..) => NodeKind::Leaf, + RValue::HashedValue(..) => NodeKind::AltHashLeaf, + }; self.buffer.extend(fuse_nibbles_node(key, kind)); - let start = self.buffer.len(); - Compact(value.len() as u32).encode_to(&mut self.buffer); - self.buffer.extend_from_slice(value); - self.current_value_range = Some(start..self.buffer.len()); + match &value { + RValue::NoValue => unreachable!(), + RValue::Value(value) => { + Compact(value.len() as u32).encode_to(&mut self.buffer); + self.buffer.extend_from_slice(value); + }, + RValue::HashedValue(hash) => { + self.buffer.extend_from_slice(hash.as_slice()); + }, + }; } fn begin_branch( &mut self, maybe_partial: Option<&[u8]>, - maybe_value: Option<&[u8]>, + maybe_value: RValue, has_children: impl Iterator, ) { if let Some(partial) = maybe_partial { - if let Some(value) = maybe_value { - self.apply_inner_hashing = self - .inner_value_hashing - .as_ref() - .map(|threshold| value_do_hash(value, threshold)) - .unwrap_or(false); - let kind = if self.apply_inner_hashing { - NodeKind::AltHashBranchWithValue - } else { - NodeKind::BranchWithValue - }; - self.buffer.extend(fuse_nibbles_node(partial, kind)); - } else { - self.buffer.extend(fuse_nibbles_node(partial, NodeKind::BranchNoValue)); - } + let kind = match &maybe_value { + RValue::NoValue => NodeKind::BranchNoValue, + RValue::Value(..) => NodeKind::BranchWithValue, + RValue::HashedValue(..) => NodeKind::AltHashBranchWithValue, + }; + + self.buffer.extend(fuse_nibbles_node(partial, kind)); let bm = branch_node_bit_mask(has_children); - self.buffer.extend([bm.0, bm.1].iter()); + self.buffer.extend([bm.0,bm.1].iter()); } else { - debug_assert!(false, "trie stream codec only for no extension trie"); - self.buffer.extend(&branch_node(maybe_value.is_some(), has_children)); + unreachable!("trie stream codec only for no extension trie"); } - if let Some(value) = maybe_value { - let start = self.buffer.len(); - Compact(value.len() as u32).encode_to(&mut self.buffer); - self.buffer.extend_from_slice(value); - self.current_value_range = Some(start..self.buffer.len()); + match maybe_value { + RValue::NoValue => (), + RValue::Value(value) => { + Compact(value.len() as u32).encode_to(&mut self.buffer); + self.buffer.extend_from_slice(value); + }, + RValue::HashedValue(hash) => { + self.buffer.extend_from_slice(hash.as_slice()); + }, } } @@ -153,35 +139,10 @@ impl trie_root::TrieStream for TrieStream { } fn append_substream(&mut self, other: Self) { - let apply_inner_hashing = other.apply_inner_hashing; - let range = other.current_value_range.clone(); let data = other.out(); match data.len() { 0..=31 => data.encode_to(&mut self.buffer), - _ => - if apply_inner_hashing { - hash_db::AltHashing { - encoded_offset: 0, - value_range: range.map(|r| (r.start, r.end)), - } - .alt_hash::(&data) - .as_ref() - .encode_to(&mut self.buffer); - } else { - H::hash(&data).as_ref().encode_to(&mut self.buffer); - }, - } - } - - fn hash_root(self) -> H::Out { - let apply_inner_hashing = self.apply_inner_hashing; - let range = self.current_value_range; - let data = self.buffer; - if apply_inner_hashing { - hash_db::AltHashing { encoded_offset: 0, value_range: range.map(|r| (r.start, r.end)) } - .alt_hash::(&data) - } else { - H::hash(&data) + _ => H::hash(&data).as_ref().encode_to(&mut self.buffer), } } @@ -189,22 +150,3 @@ impl trie_root::TrieStream for TrieStream { self.buffer } } - -fn branch_node(has_value: bool, has_children: impl Iterator) -> [u8; 3] { - let mut result = [0, 0, 0]; - branch_node_buffered(has_value, has_children, &mut result[..]); - result -} - -fn branch_node_buffered(has_value: bool, has_children: I, output: &mut [u8]) -where - I: Iterator, -{ - let first = if has_value { BRANCH_NODE_WITH_VALUE } else { BRANCH_NODE_NO_VALUE }; - output[0] = first; - Bitmap::encode(has_children, &mut output[1..]); -} - -fn value_do_hash(val: &[u8], threshold: &u32) -> bool { - val.encoded_size() >= *threshold as usize -} From 57324ae39eaafa2a25c654cbbbb7e658b4a55c0e Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 6 Sep 2021 10:59:36 +0200 Subject: [PATCH 120/127] clean old impl --- bin/node/bench/src/simple_trie.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/bin/node/bench/src/simple_trie.rs b/bin/node/bench/src/simple_trie.rs index b33a588b463f7..651772c71575f 100644 --- a/bin/node/bench/src/simple_trie.rs +++ b/bin/node/bench/src/simple_trie.rs @@ -65,11 +65,6 @@ impl<'a> HashDB for SimpleTrie<'a> { self.overlay.insert(key, Some(value)); } - fn emplace_ref(&mut self, key: &Hash, prefix: Prefix, value: &[u8]) { - let key = sp_trie::prefixed_key::(key, prefix); - self.overlay.insert(key, Some(value.into())); - } - fn remove(&mut self, key: &Hash, prefix: Prefix) { let key = sp_trie::prefixed_key::(key, prefix); self.overlay.insert(key, None); From 0cbb7f105d354920797c17584c8349db8eeff3a9 Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 6 Sep 2021 14:23:49 +0200 Subject: [PATCH 121/127] fix tests --- Cargo.lock | 16 ++++++++-------- primitives/state-machine/src/lib.rs | 4 ++++ .../state-machine/src/trie_backend_essence.rs | 3 +++ 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4cf634525cae1..d6b4feea7ab22 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2441,7 +2441,7 @@ dependencies = [ [[package]] name = "hash-db" version = "0.15.2" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-new#8cfbde80decc0cd1840e6baa0b0e48c7c7403fc1" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-new#91d81741dc45d66dcd3696c3b200fdb26bc6e682" [[package]] name = "hash256-std-hasher" @@ -2455,7 +2455,7 @@ dependencies = [ [[package]] name = "hash256-std-hasher" version = "0.15.2" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-new#8cfbde80decc0cd1840e6baa0b0e48c7c7403fc1" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-new#91d81741dc45d66dcd3696c3b200fdb26bc6e682" dependencies = [ "crunchy", ] @@ -3108,7 +3108,7 @@ dependencies = [ [[package]] name = "keccak-hasher" version = "0.15.3" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-new#8cfbde80decc0cd1840e6baa0b0e48c7c7403fc1" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-new#91d81741dc45d66dcd3696c3b200fdb26bc6e682" dependencies = [ "hash-db", "hash256-std-hasher 0.15.2 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-new)", @@ -3969,7 +3969,7 @@ dependencies = [ [[package]] name = "memory-db" version = "0.27.0" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-new#8cfbde80decc0cd1840e6baa0b0e48c7c7403fc1" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-new#91d81741dc45d66dcd3696c3b200fdb26bc6e682" dependencies = [ "hash-db", "hashbrown 0.11.2", @@ -10479,7 +10479,7 @@ checksum = "a7f741b240f1a48843f9b8e0444fb55fb2a4ff67293b50a9179dfd5ea67f8d41" [[package]] name = "trie-bench" version = "0.28.0" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-new#8cfbde80decc0cd1840e6baa0b0e48c7c7403fc1" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-new#91d81741dc45d66dcd3696c3b200fdb26bc6e682" dependencies = [ "criterion", "hash-db", @@ -10494,7 +10494,7 @@ dependencies = [ [[package]] name = "trie-db" version = "0.22.6" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-new#8cfbde80decc0cd1840e6baa0b0e48c7c7403fc1" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-new#91d81741dc45d66dcd3696c3b200fdb26bc6e682" dependencies = [ "hash-db", "hashbrown 0.11.2", @@ -10506,7 +10506,7 @@ dependencies = [ [[package]] name = "trie-root" version = "0.16.0" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-new#8cfbde80decc0cd1840e6baa0b0e48c7c7403fc1" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-new#91d81741dc45d66dcd3696c3b200fdb26bc6e682" dependencies = [ "hash-db", ] @@ -10524,7 +10524,7 @@ dependencies = [ [[package]] name = "trie-standardmap" version = "0.15.2" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-new#8cfbde80decc0cd1840e6baa0b0e48c7c7403fc1" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-new#91d81741dc45d66dcd3696c3b200fdb26bc6e682" dependencies = [ "hash-db", "keccak-hasher 0.15.3 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-new)", diff --git a/primitives/state-machine/src/lib.rs b/primitives/state-machine/src/lib.rs index b00f343f1580f..9aebd3d91b476 100644 --- a/primitives/state-machine/src/lib.rs +++ b/primitives/state-machine/src/lib.rs @@ -1672,8 +1672,12 @@ mod tests { { let mut trie = TrieDBMut::from_existing_with_layout(&mut mdb, &mut root, layout.clone()).unwrap(); + // changed trie.insert(b"foo222", vec![5u8; 100].as_slice()) // inner hash .expect("insert failed"); + // unchange but should also update node. + trie.insert(b"foo", vec![1u8; 1_000].as_slice()) // big inner hash + .expect("insert failed"); } let root3 = root.clone(); assert!(root1 != root3); diff --git a/primitives/state-machine/src/trie_backend_essence.rs b/primitives/state-machine/src/trie_backend_essence.rs index 4bf26ac60e569..8310bf14fbe63 100644 --- a/primitives/state-machine/src/trie_backend_essence.rs +++ b/primitives/state-machine/src/trie_backend_essence.rs @@ -94,6 +94,9 @@ where /// Get backend storage reference. pub fn backend_storage_mut(&mut self) -> &mut S { + #[cfg(std)] + self.cache.reset_cache(); + &mut self.storage } From 091f3487476bb70fc2c0aab0d3fbd790622ff87f Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 6 Sep 2021 16:33:42 +0200 Subject: [PATCH 122/127] update trie_db crate --- Cargo.lock | 16 ++++++------- client/light/src/fetcher.rs | 2 +- primitives/trie/src/lib.rs | 6 ++--- primitives/trie/src/node_codec.rs | 22 +++++++++--------- primitives/trie/src/node_header.rs | 20 ++++++++-------- primitives/trie/src/storage_proof.rs | 10 -------- primitives/trie/src/trie_stream.rs | 34 ++++++++++++++-------------- 7 files changed, 50 insertions(+), 60 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cc5420f475bd0..1ef365adf2cef 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2441,7 +2441,7 @@ dependencies = [ [[package]] name = "hash-db" version = "0.15.2" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-new#91d81741dc45d66dcd3696c3b200fdb26bc6e682" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-new#97caad53d595c3ecc604763a30b47dc634fd63cf" [[package]] name = "hash256-std-hasher" @@ -2455,7 +2455,7 @@ dependencies = [ [[package]] name = "hash256-std-hasher" version = "0.15.2" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-new#91d81741dc45d66dcd3696c3b200fdb26bc6e682" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-new#97caad53d595c3ecc604763a30b47dc634fd63cf" dependencies = [ "crunchy", ] @@ -3108,7 +3108,7 @@ dependencies = [ [[package]] name = "keccak-hasher" version = "0.15.3" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-new#91d81741dc45d66dcd3696c3b200fdb26bc6e682" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-new#97caad53d595c3ecc604763a30b47dc634fd63cf" dependencies = [ "hash-db", "hash256-std-hasher 0.15.2 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-new)", @@ -3969,7 +3969,7 @@ dependencies = [ [[package]] name = "memory-db" version = "0.27.0" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-new#91d81741dc45d66dcd3696c3b200fdb26bc6e682" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-new#97caad53d595c3ecc604763a30b47dc634fd63cf" dependencies = [ "hash-db", "hashbrown 0.11.2", @@ -10426,7 +10426,7 @@ checksum = "a7f741b240f1a48843f9b8e0444fb55fb2a4ff67293b50a9179dfd5ea67f8d41" [[package]] name = "trie-bench" version = "0.28.0" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-new#91d81741dc45d66dcd3696c3b200fdb26bc6e682" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-new#97caad53d595c3ecc604763a30b47dc634fd63cf" dependencies = [ "criterion", "hash-db", @@ -10441,7 +10441,7 @@ dependencies = [ [[package]] name = "trie-db" version = "0.22.6" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-new#91d81741dc45d66dcd3696c3b200fdb26bc6e682" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-new#97caad53d595c3ecc604763a30b47dc634fd63cf" dependencies = [ "hash-db", "hashbrown 0.11.2", @@ -10453,7 +10453,7 @@ dependencies = [ [[package]] name = "trie-root" version = "0.16.0" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-new#91d81741dc45d66dcd3696c3b200fdb26bc6e682" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-new#97caad53d595c3ecc604763a30b47dc634fd63cf" dependencies = [ "hash-db", ] @@ -10471,7 +10471,7 @@ dependencies = [ [[package]] name = "trie-standardmap" version = "0.15.2" -source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-new#91d81741dc45d66dcd3696c3b200fdb26bc6e682" +source = "git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-new#97caad53d595c3ecc604763a30b47dc634fd63cf" dependencies = [ "hash-db", "keccak-hasher 0.15.3 (git+https://github.com/cheme/trie.git?branch=hashed-value-simple4-new)", diff --git a/client/light/src/fetcher.rs b/client/light/src/fetcher.rs index ca346f019d2b2..f8346aa21d19a 100644 --- a/client/light/src/fetcher.rs +++ b/client/light/src/fetcher.rs @@ -171,7 +171,7 @@ impl> LightDataChecker { ) -> ClientResult<()> { // all the checks are sharing the same storage let storage: sp_state_machine::MemoryDB> = - remote_roots_proof.into_memory_db_no_meta(); + remote_roots_proof.into_memory_db(); // remote_roots.keys() are sorted => we can use this to group changes tries roots // that are belongs to the same CHT diff --git a/primitives/trie/src/lib.rs b/primitives/trie/src/lib.rs index eeb85b8f11ae4..6c3d8e23aa2d9 100644 --- a/primitives/trie/src/lib.rs +++ b/primitives/trie/src/lib.rs @@ -92,7 +92,7 @@ where type Hash = H; type Codec = NodeCodec; - fn alt_threshold(&self) -> Option { + fn max_inline_value(&self) -> Option { self.0 } } @@ -107,7 +107,7 @@ where A: AsRef<[u8]> + Ord, B: AsRef<[u8]>, { - trie_root::trie_root_no_extension::(input, self.alt_threshold()) + trie_root::trie_root_no_extension::(input, self.max_inline_value()) } fn trie_root_unhashed(&self, input: I) -> Vec @@ -116,7 +116,7 @@ where A: AsRef<[u8]> + Ord, B: AsRef<[u8]>, { - trie_root::unhashed_trie_no_extension::(input, self.alt_threshold()) + trie_root::unhashed_trie_no_extension::(input, self.max_inline_value()) } fn encode_index(input: u32) -> Vec { diff --git a/primitives/trie/src/node_codec.rs b/primitives/trie/src/node_codec.rs index ce5f84df3c718..5f47fe0e763a1 100644 --- a/primitives/trie/src/node_codec.rs +++ b/primitives/trie/src/node_codec.rs @@ -96,7 +96,7 @@ impl NodeCodec { match header { NodeHeader::Null => Ok(NodePlan::Empty), - NodeHeader::AltHashBranch(nibble_count) + NodeHeader::HashedValueBranch(nibble_count) | NodeHeader::Branch(_, nibble_count) => { let padding = nibble_count % nibble_ops::NIBBLE_PER_BYTE != 0; // check that the padding is valid (if any) @@ -141,7 +141,7 @@ impl NodeCodec { children, }) }, - NodeHeader::AltHashLeaf(nibble_count) + NodeHeader::HashedValueLeaf(nibble_count) | NodeHeader::Leaf(nibble_count) => { let padding = nibble_count % nibble_ops::NIBBLE_PER_BYTE != 0; // check that the padding is valid (if any) @@ -196,7 +196,7 @@ where fn leaf_node(partial: Partial, value: Value) -> Vec { let contains_hash = matches!(&value, Value::HashedValue(..)); let mut output = if contains_hash { - partial_encode(partial, NodeKind::AltHashLeaf) + partial_encode(partial, NodeKind::HashedValueLeaf) } else { partial_encode(partial, NodeKind::Leaf) }; @@ -209,7 +209,7 @@ where debug_assert!(hash.len() == H::LENGTH); output.extend_from_slice(hash); }, - Value::NoValue => unimplemented!("No support for incomplete nodes"), + Value::NoValue => unreachable!("Leaf node always with value."), } output } @@ -219,14 +219,14 @@ where _nbnibble: usize, _child: ChildReference<::Out>, ) -> Vec { - unreachable!() + unreachable!("No extension codec.") } fn branch_node( _children: impl Iterator::Out>>>>, _maybe_value: Value, ) -> Vec { - unreachable!() + unreachable!("No extension codec.") } fn branch_node_nibbled( @@ -242,7 +242,7 @@ where (_, false) => partial_from_iterator_encode(partial, number_nibble, NodeKind::BranchWithValue), (_, true) => - partial_from_iterator_encode(partial, number_nibble, NodeKind::AltHashBranchWithValue), + partial_from_iterator_encode(partial, number_nibble, NodeKind::HashedValueBranch), }; let bitmap_index = output.len(); @@ -295,8 +295,8 @@ fn partial_from_iterator_encode>( NodeKind::Leaf => NodeHeader::Leaf(nibble_count).encode_to(&mut output), NodeKind::BranchWithValue => NodeHeader::Branch(true, nibble_count).encode_to(&mut output), NodeKind::BranchNoValue => NodeHeader::Branch(false, nibble_count).encode_to(&mut output), - NodeKind::AltHashLeaf => NodeHeader::AltHashLeaf(nibble_count).encode_to(&mut output), - NodeKind::AltHashBranchWithValue => NodeHeader::AltHashBranch(nibble_count) + NodeKind::HashedValueLeaf => NodeHeader::HashedValueLeaf(nibble_count).encode_to(&mut output), + NodeKind::HashedValueBranch => NodeHeader::HashedValueBranch(nibble_count) .encode_to(&mut output), }; output.extend(partial); @@ -316,8 +316,8 @@ fn partial_encode(partial: Partial, node_kind: NodeKind) -> Vec { NodeKind::Leaf => NodeHeader::Leaf(nibble_count).encode_to(&mut output), NodeKind::BranchWithValue => NodeHeader::Branch(true, nibble_count).encode_to(&mut output), NodeKind::BranchNoValue => NodeHeader::Branch(false, nibble_count).encode_to(&mut output), - NodeKind::AltHashLeaf => NodeHeader::AltHashLeaf(nibble_count).encode_to(&mut output), - NodeKind::AltHashBranchWithValue => NodeHeader::AltHashBranch(nibble_count) + NodeKind::HashedValueLeaf => NodeHeader::HashedValueLeaf(nibble_count).encode_to(&mut output), + NodeKind::HashedValueBranch => NodeHeader::HashedValueBranch(nibble_count) .encode_to(&mut output), }; if number_nibble_encoded > 0 { diff --git a/primitives/trie/src/node_header.rs b/primitives/trie/src/node_header.rs index 6130d118ed864..154fbe15c29ad 100644 --- a/primitives/trie/src/node_header.rs +++ b/primitives/trie/src/node_header.rs @@ -30,16 +30,16 @@ pub(crate) enum NodeHeader { // contains nibble count Leaf(usize), // contains nibble count. - AltHashBranch(usize), + HashedValueBranch(usize), // contains nibble count. - AltHashLeaf(usize), + HashedValueLeaf(usize), } impl NodeHeader { pub(crate) fn contains_hash_of_value(&self) -> bool { match self { - NodeHeader::AltHashBranch(_) - | NodeHeader::AltHashLeaf(_) => true, + NodeHeader::HashedValueBranch(_) + | NodeHeader::HashedValueLeaf(_) => true, _ => false, } } @@ -50,8 +50,8 @@ pub(crate) enum NodeKind { Leaf, BranchNoValue, BranchWithValue, - AltHashLeaf, - AltHashBranchWithValue, + HashedValueLeaf, + HashedValueBranch, } impl Encode for NodeHeader { @@ -64,9 +64,9 @@ impl Encode for NodeHeader { encode_size_and_prefix(*nibble_count, trie_constants::BRANCH_WITHOUT_MASK, 2, output), NodeHeader::Leaf(nibble_count) => encode_size_and_prefix(*nibble_count, trie_constants::LEAF_PREFIX_MASK, 2, output), - NodeHeader::AltHashBranch(nibble_count) => + NodeHeader::HashedValueBranch(nibble_count) => encode_size_and_prefix(*nibble_count, trie_constants::ALT_HASHING_BRANCH_WITH_MASK, 4, output), - NodeHeader::AltHashLeaf(nibble_count) => + NodeHeader::HashedValueLeaf(nibble_count) => encode_size_and_prefix(*nibble_count, trie_constants::ALT_HASHING_LEAF_PREFIX_MASK, 3, output), } } @@ -86,9 +86,9 @@ impl Decode for NodeHeader { trie_constants::BRANCH_WITHOUT_MASK => Ok(NodeHeader::Branch(false, decode_size(i, input, 2)?)), trie_constants::EMPTY_TRIE => { if i & (0b111 << 5) == trie_constants::ALT_HASHING_LEAF_PREFIX_MASK { - Ok(NodeHeader::AltHashLeaf(decode_size(i, input, 3)?)) + Ok(NodeHeader::HashedValueLeaf(decode_size(i, input, 3)?)) } else if i & (0b1111 << 4) == trie_constants::ALT_HASHING_BRANCH_WITH_MASK { - Ok(NodeHeader::AltHashBranch(decode_size(i, input, 4)?)) + Ok(NodeHeader::HashedValueBranch(decode_size(i, input, 4)?)) } else { // do not allow any special encoding Err("Unallowed encoding".into()) diff --git a/primitives/trie/src/storage_proof.rs b/primitives/trie/src/storage_proof.rs index e0a89621c7bd3..2b8067a86c8f1 100644 --- a/primitives/trie/src/storage_proof.rs +++ b/primitives/trie/src/storage_proof.rs @@ -111,16 +111,6 @@ impl StorageProof { self.into() } - /// Creates a `MemoryDB` from `Self`. In case we do not need - /// to check meta (using alt hashing will always be disabled). - pub fn into_memory_db_no_meta(self) -> crate::MemoryDB { - let mut db = crate::MemoryDB::default(); - for item in self.iter_nodes() { - db.insert(crate::EMPTY_PREFIX, &item); - } - db - } - /// Merges multiple storage proofs covering potentially different sets of keys into one proof /// covering all keys. The merged proof output may be smaller than the aggregate size of the /// input proofs due to deduplication of trie nodes. diff --git a/primitives/trie/src/trie_stream.rs b/primitives/trie/src/trie_stream.rs index e9e35c3f12c3c..44d79770a5d33 100644 --- a/primitives/trie/src/trie_stream.rs +++ b/primitives/trie/src/trie_stream.rs @@ -62,9 +62,9 @@ fn fuse_nibbles_node<'a>(nibbles: &'a [u8], kind: NodeKind) -> impl Iterator size_and_prefix_iterator(size, trie_constants::BRANCH_WITH_MASK, 2), - NodeKind::AltHashLeaf => + NodeKind::HashedValueLeaf => size_and_prefix_iterator(size, trie_constants::ALT_HASHING_LEAF_PREFIX_MASK, 3), - NodeKind::AltHashBranchWithValue => + NodeKind::HashedValueBranch => size_and_prefix_iterator(size, trie_constants::ALT_HASHING_BRANCH_WITH_MASK, 4), }; iter_start @@ -72,7 +72,7 @@ fn fuse_nibbles_node<'a>(nibbles: &'a [u8], kind: NodeKind) -> impl Iterator Self { Self { @@ -84,20 +84,20 @@ impl trie_root::TrieStream for TrieStream { self.buffer.push(trie_constants::EMPTY_TRIE); } - fn append_leaf(&mut self, key: &[u8], value: RValue) { + fn append_leaf(&mut self, key: &[u8], value: TrieStreamValue) { let kind = match &value { - RValue::NoValue => unreachable!(), - RValue::Value(..) => NodeKind::Leaf, - RValue::HashedValue(..) => NodeKind::AltHashLeaf, + TrieStreamValue::NoValue => unreachable!(), + TrieStreamValue::Value(..) => NodeKind::Leaf, + TrieStreamValue::HashedValue(..) => NodeKind::HashedValueLeaf, }; self.buffer.extend(fuse_nibbles_node(key, kind)); match &value { - RValue::NoValue => unreachable!(), - RValue::Value(value) => { + TrieStreamValue::NoValue => unreachable!(), + TrieStreamValue::Value(value) => { Compact(value.len() as u32).encode_to(&mut self.buffer); self.buffer.extend_from_slice(value); }, - RValue::HashedValue(hash) => { + TrieStreamValue::HashedValue(hash) => { self.buffer.extend_from_slice(hash.as_slice()); }, }; @@ -106,14 +106,14 @@ impl trie_root::TrieStream for TrieStream { fn begin_branch( &mut self, maybe_partial: Option<&[u8]>, - maybe_value: RValue, + maybe_value: TrieStreamValue, has_children: impl Iterator, ) { if let Some(partial) = maybe_partial { let kind = match &maybe_value { - RValue::NoValue => NodeKind::BranchNoValue, - RValue::Value(..) => NodeKind::BranchWithValue, - RValue::HashedValue(..) => NodeKind::AltHashBranchWithValue, + TrieStreamValue::NoValue => NodeKind::BranchNoValue, + TrieStreamValue::Value(..) => NodeKind::BranchWithValue, + TrieStreamValue::HashedValue(..) => NodeKind::HashedValueBranch, }; self.buffer.extend(fuse_nibbles_node(partial, kind)); @@ -123,12 +123,12 @@ impl trie_root::TrieStream for TrieStream { unreachable!("trie stream codec only for no extension trie"); } match maybe_value { - RValue::NoValue => (), - RValue::Value(value) => { + TrieStreamValue::NoValue => (), + TrieStreamValue::Value(value) => { Compact(value.len() as u32).encode_to(&mut self.buffer); self.buffer.extend_from_slice(value); }, - RValue::HashedValue(hash) => { + TrieStreamValue::HashedValue(hash) => { self.buffer.extend_from_slice(hash.as_slice()); }, } From 07651511ad21c81ba05fe6ea6ce8ebc92cf6d2c4 Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 6 Sep 2021 16:46:36 +0200 Subject: [PATCH 123/127] clean --- bin/node/bench/src/generator.rs | 2 +- client/db/src/lib.rs | 6 +++--- primitives/blockchain/src/error.rs | 3 --- primitives/core/src/state_version.rs | 2 +- primitives/state-machine/src/basic.rs | 2 +- primitives/state-machine/src/lib.rs | 4 ++-- primitives/state-machine/src/proving_backend.rs | 4 ++-- primitives/state-machine/src/trie_backend.rs | 10 +++++----- primitives/storage/src/lib.rs | 2 +- primitives/trie/src/lib.rs | 17 ++++++++--------- primitives/trie/src/node_codec.rs | 2 +- 11 files changed, 25 insertions(+), 29 deletions(-) diff --git a/bin/node/bench/src/generator.rs b/bin/node/bench/src/generator.rs index dfd2b549ce922..3a4c806daf90a 100644 --- a/bin/node/bench/src/generator.rs +++ b/bin/node/bench/src/generator.rs @@ -48,7 +48,7 @@ pub fn generate_trie( let mut trie_db = match state_version { StateVersion::V0 => TrieDBMut::new(&mut trie, &mut root), StateVersion::V1 { threshold } => { - let layout = sp_trie::Layout::with_alt_hashing(threshold); + let layout = sp_trie::Layout::with_max_inline_value(threshold); TrieDBMut::::new_with_layout( &mut trie, &mut root, layout, ) diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index 4b95d13286340..2a5bf10741728 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -2602,9 +2602,9 @@ pub(crate) mod tests { set_state_data_inner(true); set_state_data_inner(false); } - fn set_state_data_inner(alt_hashing: bool) { - let state_version = if alt_hashing { - StateVersion::V1 { threshold: sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD } + fn set_state_data_inner(inner_hashing: bool) { + let state_version = if inner_hashing { + StateVersion::V1 { threshold: sp_core::storage::TEST_DEFAULT_INLINE_VALUE_THESHOLD } } else { StateVersion::V0 }; diff --git a/primitives/blockchain/src/error.rs b/primitives/blockchain/src/error.rs index a7e7414b95604..6e60b2da6549f 100644 --- a/primitives/blockchain/src/error.rs +++ b/primitives/blockchain/src/error.rs @@ -144,9 +144,6 @@ pub enum Error { #[error("Calculated state root does not match.")] InvalidStateRoot, - #[error("Calculated migration state does not match.")] - InvalidMigrationState(String), - #[error("Incomplete block import pipeline.")] IncompletePipeline, diff --git a/primitives/core/src/state_version.rs b/primitives/core/src/state_version.rs index 06739b4e9ef88..5afd404e16ccc 100644 --- a/primitives/core/src/state_version.rs +++ b/primitives/core/src/state_version.rs @@ -26,7 +26,7 @@ use codec::{Decode, Encode}; /// Therefore defining genesis version in chainspec is good practice /// and this default should mostly be use when testing. pub const DEFAULT_STATE_VERSION: StateVersion = - StateVersion::V1 { threshold: crate::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD }; + StateVersion::V1 { threshold: crate::storage::TEST_DEFAULT_INLINE_VALUE_THESHOLD }; /// Supported version with substrate chain. #[derive(Debug, Clone, Copy, PartialEq, Eq, Encode, Decode)] diff --git a/primitives/state-machine/src/basic.rs b/primitives/state-machine/src/basic.rs index 7cbabd46e6387..44a2fbd46d4d1 100644 --- a/primitives/state-machine/src/basic.rs +++ b/primitives/state-machine/src/basic.rs @@ -322,7 +322,7 @@ impl Externalities for BasicExternalities { .expect("Unsupported state calculation for genesis storage build.") { StateVersion::V0 => Layout::::default(), - StateVersion::V1 { threshold } => Layout::::with_alt_hashing(threshold), + StateVersion::V1 { threshold } => Layout::::with_max_inline_value(threshold), }; layout.trie_root(self.inner.top.clone()).as_ref().into() } diff --git a/primitives/state-machine/src/lib.rs b/primitives/state-machine/src/lib.rs index c57d352bd5e89..6bf1dde9d3d14 100644 --- a/primitives/state-machine/src/lib.rs +++ b/primitives/state-machine/src/lib.rs @@ -988,7 +988,7 @@ mod tests { use sp_core::{ map, state_version::StateVersion, - storage::{ChildInfo, TEST_DEFAULT_ALT_HASH_THRESHOLD as TRESHOLD}, + storage::{ChildInfo, TEST_DEFAULT_INLINE_VALUE_THESHOLD as TRESHOLD}, testing::TaskExecutor, traits::{CodeExecutor, Externalities, RuntimeCode}, NativeOrEncoded, NeverNativeValue, @@ -1667,7 +1667,7 @@ mod tests { let root1 = root.clone(); // do switch - layout = Layout::with_alt_hashing(TRESHOLD); + layout = Layout::with_max_inline_value(TRESHOLD); // update with same value do change { let mut trie = diff --git a/primitives/state-machine/src/proving_backend.rs b/primitives/state-machine/src/proving_backend.rs index efdea2024959c..d349d90247de1 100644 --- a/primitives/state-machine/src/proving_backend.rs +++ b/primitives/state-machine/src/proving_backend.rs @@ -462,7 +462,7 @@ mod tests { let in_memory: InMemoryBackend = if flagged { ( contents, - StateVersion::V1 { threshold: sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD }, + StateVersion::V1 { threshold: sp_core::storage::TEST_DEFAULT_INLINE_VALUE_THESHOLD }, ) .into() } else { @@ -508,7 +508,7 @@ mod tests { let in_memory: InMemoryBackend = if flagged { ( contents, - StateVersion::V1 { threshold: sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD }, + StateVersion::V1 { threshold: sp_core::storage::TEST_DEFAULT_INLINE_VALUE_THESHOLD }, ) .into() } else { diff --git a/primitives/state-machine/src/trie_backend.rs b/primitives/state-machine/src/trie_backend.rs index 62f6f2388a736..42d539d39aa43 100644 --- a/primitives/state-machine/src/trie_backend.rs +++ b/primitives/state-machine/src/trie_backend.rs @@ -207,7 +207,7 @@ where let res = || { let layout = match self.state_version { StateVersion::V0 => sp_trie::Layout::default(), - StateVersion::V1 { threshold } => sp_trie::Layout::with_alt_hashing(threshold), + StateVersion::V1 { threshold } => sp_trie::Layout::with_max_inline_value(threshold), }; delta_trie_root::, _, _, _, _, _>(&mut eph, root, delta, layout) }; @@ -234,7 +234,7 @@ where }; let layout = match self.state_version { StateVersion::V0 => sp_trie::Layout::default(), - StateVersion::V1 { threshold } => sp_trie::Layout::with_alt_hashing(threshold), + StateVersion::V1 { threshold } => sp_trie::Layout::with_max_inline_value(threshold), }; let mut write_overlay = S::Overlay::default(); @@ -292,7 +292,7 @@ where pub mod tests { use super::*; use codec::Encode; - use sp_core::{storage::TEST_DEFAULT_ALT_HASH_THRESHOLD as TRESHOLD, H256}; + use sp_core::{storage::TEST_DEFAULT_INLINE_VALUE_THESHOLD as TRESHOLD, H256}; use sp_runtime::traits::BlakeTwo256; use sp_trie::{trie_types::TrieDBMut, KeySpacedDBMut, PrefixedMemoryDB, TrieMut}; use std::{collections::HashSet, iter}; @@ -314,7 +314,7 @@ pub mod tests { let mut sub_root = Vec::new(); root.encode_to(&mut sub_root); let mut trie = if hashed_value { - let layout = Layout::with_alt_hashing(TRESHOLD); + let layout = Layout::with_max_inline_value(TRESHOLD); TrieDBMut::new_with_layout(&mut mdb, &mut root, layout) } else { TrieDBMut::new(&mut mdb, &mut root) @@ -338,7 +338,7 @@ pub mod tests { ) -> TrieBackend, BlakeTwo256> { let (mdb, root) = test_db(hashed_value); let state_version = if hashed_value { - StateVersion::V1 { threshold: sp_core::storage::TEST_DEFAULT_ALT_HASH_THRESHOLD } + StateVersion::V1 { threshold: sp_core::storage::TEST_DEFAULT_INLINE_VALUE_THESHOLD } } else { StateVersion::V0 }; diff --git a/primitives/storage/src/lib.rs b/primitives/storage/src/lib.rs index a02ad6c35ea38..3b20797d6dc0b 100644 --- a/primitives/storage/src/lib.rs +++ b/primitives/storage/src/lib.rs @@ -244,7 +244,7 @@ pub fn trie_threshold_decode(mut encoded: &[u8]) -> Option { } /// Default value to use as a threshold for testing. -pub const TEST_DEFAULT_ALT_HASH_THRESHOLD: u32 = 33; +pub const TEST_DEFAULT_INLINE_VALUE_THESHOLD: u32 = 33; /// Information related to a child state. #[derive(Debug, Clone)] diff --git a/primitives/trie/src/lib.rs b/primitives/trie/src/lib.rs index 6c3d8e23aa2d9..059aa188514f3 100644 --- a/primitives/trie/src/lib.rs +++ b/primitives/trie/src/lib.rs @@ -75,9 +75,8 @@ impl Default for Layout { } impl Layout { - /// Layout with inner hashing active. - /// Will flag trie for hashing. - pub fn with_alt_hashing(threshold: u32) -> Self { + /// Layout with inner hash value size limit active. + pub fn with_max_inline_value(threshold: u32) -> Self { Layout(Some(threshold), sp_std::marker::PhantomData) } } @@ -513,7 +512,7 @@ mod tests { use codec::{Compact, Decode, Encode}; use hash_db::{HashDB, Hasher}; use hex_literal::hex; - use sp_core::{storage::TEST_DEFAULT_ALT_HASH_THRESHOLD as TRESHOLD, Blake2Hasher}; + use sp_core::{storage::TEST_DEFAULT_INLINE_VALUE_THESHOLD as TRESHOLD, Blake2Hasher}; use trie_db::{DBValue, NodeCodec as NodeCodecT, Trie, TrieMut}; use trie_standardmap::{Alphabet, StandardMap, ValueMode}; @@ -569,7 +568,7 @@ mod tests { let layout = Layout::default(); check_equivalent::(input, layout.clone()); check_iteration::(input, layout); - let layout = Layout::with_alt_hashing(TRESHOLD); + let layout = Layout::with_max_inline_value(TRESHOLD); check_equivalent::(input, layout.clone()); check_iteration::(input, layout); } @@ -715,7 +714,7 @@ mod tests { random_should_work_inner(true); random_should_work_inner(false); } - fn random_should_work_inner(flag: bool) { + fn random_should_work_inner(limit_inline_value: bool) { let mut seed = ::Out::zero(); for test_i in 0..10_000 { if test_i % 50 == 0 { @@ -730,7 +729,7 @@ mod tests { } .make_with(seed.as_fixed_bytes_mut()); - let layout = if flag { Layout::with_alt_hashing(TRESHOLD) } else { Layout::default() }; + let layout = if limit_inline_value { Layout::with_max_inline_value(TRESHOLD) } else { Layout::default() }; let real = layout.trie_root(x.clone()); let mut memdb = MemoryDB::default(); let mut root = Default::default(); @@ -823,8 +822,8 @@ mod tests { iterator_works_inner(true); iterator_works_inner(false); } - fn iterator_works_inner(flag: bool) { - let layout = if flag { Layout::with_alt_hashing(TRESHOLD) } else { Layout::default() }; + fn iterator_works_inner(limit_inline_value: bool) { + let layout = if limit_inline_value { Layout::with_max_inline_value(TRESHOLD) } else { Layout::default() }; let pairs = vec![ (hex!("0103000000000000000464").to_vec(), hex!("0400000000").to_vec()), diff --git a/primitives/trie/src/node_codec.rs b/primitives/trie/src/node_codec.rs index 5f47fe0e763a1..e30da91fc2d92 100644 --- a/primitives/trie/src/node_codec.rs +++ b/primitives/trie/src/node_codec.rs @@ -90,7 +90,7 @@ impl NodeCodec { let branch_has_value = if let NodeHeader::Branch(has_value, _) = &header { *has_value } else { - // alt_hash_branch + // hashed_value_branch true }; From c5a354830247a38efc27006b0bf5ce4282655177 Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 6 Sep 2021 16:49:13 +0200 Subject: [PATCH 124/127] cargo fmt --- client/api/src/cht.rs | 3 +- client/db/src/lib.rs | 2 +- client/light/src/fetcher.rs | 3 +- primitives/state-machine/src/backend.rs | 2 +- primitives/state-machine/src/basic.rs | 3 +- .../state-machine/src/proving_backend.rs | 13 +++++-- primitives/state-machine/src/trie_backend.rs | 3 +- primitives/trie/src/lib.rs | 21 ++++++++--- primitives/trie/src/node_codec.rs | 24 ++++++------- primitives/trie/src/node_header.rs | 35 +++++++++++++------ primitives/trie/src/trie_codec.rs | 6 ++-- primitives/trie/src/trie_stream.rs | 8 ++--- 12 files changed, 75 insertions(+), 48 deletions(-) diff --git a/client/api/src/cht.rs b/client/api/src/cht.rs index 0b95f21fc322f..718974730ad8d 100644 --- a/client/api/src/cht.rs +++ b/client/api/src/cht.rs @@ -32,8 +32,7 @@ use sp_trie; use sp_core::{convert_hash, H256}; use sp_runtime::traits::{AtLeast32Bit, Header as HeaderT, One, Zero}; use sp_state_machine::{ - prove_read_on_trie_backend, read_proof_check, - read_proof_check_on_proving_backend, + prove_read_on_trie_backend, read_proof_check, read_proof_check_on_proving_backend, Backend as StateBackend, InMemoryBackend, MemoryDB, StorageProof, TrieBackend, }; diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index 2a5bf10741728..d001671112756 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -1894,7 +1894,7 @@ impl Backend { fn empty_state(&self) -> ClientResult, Block>> { let root = EmptyStorage::::new().0; // Empty trie - // Using genesis state_version in empty state. + // Using genesis state_version in empty state. let state_version = self.state_versions.genesis_state_version(); let db_state = DbState::::new(self.storage.clone(), root, state_version); let state = RefTrackingState::new(db_state, self.storage.clone(), None); diff --git a/client/light/src/fetcher.rs b/client/light/src/fetcher.rs index f8346aa21d19a..75c113744c8bb 100644 --- a/client/light/src/fetcher.rs +++ b/client/light/src/fetcher.rs @@ -170,8 +170,7 @@ impl> LightDataChecker { remote_roots_proof: StorageProof, ) -> ClientResult<()> { // all the checks are sharing the same storage - let storage: sp_state_machine::MemoryDB> = - remote_roots_proof.into_memory_db(); + let storage: sp_state_machine::MemoryDB> = remote_roots_proof.into_memory_db(); // remote_roots.keys() are sorted => we can use this to group changes tries roots // that are belongs to the same CHT diff --git a/primitives/state-machine/src/backend.rs b/primitives/state-machine/src/backend.rs index 98fb52bcc25cc..8aa1135722184 100644 --- a/primitives/state-machine/src/backend.rs +++ b/primitives/state-machine/src/backend.rs @@ -317,7 +317,7 @@ where for (key, value) in input { if let Err(e) = trie.insert(&key, &value) { log::warn!(target: "trie", "Failed to write to trie: {}", e); - return None; + return None } } } diff --git a/primitives/state-machine/src/basic.rs b/primitives/state-machine/src/basic.rs index 44a2fbd46d4d1..65f1539dd9ee7 100644 --- a/primitives/state-machine/src/basic.rs +++ b/primitives/state-machine/src/basic.rs @@ -322,7 +322,8 @@ impl Externalities for BasicExternalities { .expect("Unsupported state calculation for genesis storage build.") { StateVersion::V0 => Layout::::default(), - StateVersion::V1 { threshold } => Layout::::with_max_inline_value(threshold), + StateVersion::V1 { threshold } => + Layout::::with_max_inline_value(threshold), }; layout.trie_root(self.inner.top.clone()).as_ref().into() } diff --git a/primitives/state-machine/src/proving_backend.rs b/primitives/state-machine/src/proving_backend.rs index d349d90247de1..d870431d2ba36 100644 --- a/primitives/state-machine/src/proving_backend.rs +++ b/primitives/state-machine/src/proving_backend.rs @@ -32,7 +32,10 @@ use sp_trie::{ empty_child_trie_root, read_child_trie_value_with, read_trie_value_with, record_all_keys, Layout, MemoryDB, Recorder, StorageProof, }; -use std::{collections::{hash_map::Entry, HashMap}, sync::Arc}; +use std::{ + collections::{hash_map::Entry, HashMap}, + sync::Arc, +}; /// Patricia trie-based backend specialized in get value proofs. pub struct ProvingBackendRecorder<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> { @@ -462,7 +465,9 @@ mod tests { let in_memory: InMemoryBackend = if flagged { ( contents, - StateVersion::V1 { threshold: sp_core::storage::TEST_DEFAULT_INLINE_VALUE_THESHOLD }, + StateVersion::V1 { + threshold: sp_core::storage::TEST_DEFAULT_INLINE_VALUE_THESHOLD, + }, ) .into() } else { @@ -508,7 +513,9 @@ mod tests { let in_memory: InMemoryBackend = if flagged { ( contents, - StateVersion::V1 { threshold: sp_core::storage::TEST_DEFAULT_INLINE_VALUE_THESHOLD }, + StateVersion::V1 { + threshold: sp_core::storage::TEST_DEFAULT_INLINE_VALUE_THESHOLD, + }, ) .into() } else { diff --git a/primitives/state-machine/src/trie_backend.rs b/primitives/state-machine/src/trie_backend.rs index 42d539d39aa43..18808ff552f71 100644 --- a/primitives/state-machine/src/trie_backend.rs +++ b/primitives/state-machine/src/trie_backend.rs @@ -207,7 +207,8 @@ where let res = || { let layout = match self.state_version { StateVersion::V0 => sp_trie::Layout::default(), - StateVersion::V1 { threshold } => sp_trie::Layout::with_max_inline_value(threshold), + StateVersion::V1 { threshold } => + sp_trie::Layout::with_max_inline_value(threshold), }; delta_trie_root::, _, _, _, _, _>(&mut eph, root, delta, layout) }; diff --git a/primitives/trie/src/lib.rs b/primitives/trie/src/lib.rs index 059aa188514f3..d4b260ff5a1a7 100644 --- a/primitives/trie/src/lib.rs +++ b/primitives/trie/src/lib.rs @@ -47,8 +47,8 @@ use trie_db::proof::{generate_proof, verify_proof}; pub use trie_db::{ nibble_ops, node::{NodePlan, ValuePlan}, - CError, DBValue, Query, Recorder, Trie, TrieConfiguration, TrieDBIterator, - TrieDBKeyIterator, TrieLayout, TrieMut, + CError, DBValue, Query, Recorder, Trie, TrieConfiguration, TrieDBIterator, TrieDBKeyIterator, + TrieLayout, TrieMut, }; /// The Substrate format implementation of `TrieStream`. pub use trie_stream::TrieStream; @@ -115,7 +115,10 @@ where A: AsRef<[u8]> + Ord, B: AsRef<[u8]>, { - trie_root::unhashed_trie_no_extension::(input, self.max_inline_value()) + trie_root::unhashed_trie_no_extension::( + input, + self.max_inline_value(), + ) } fn encode_index(input: u32) -> Vec { @@ -729,7 +732,11 @@ mod tests { } .make_with(seed.as_fixed_bytes_mut()); - let layout = if limit_inline_value { Layout::with_max_inline_value(TRESHOLD) } else { Layout::default() }; + let layout = if limit_inline_value { + Layout::with_max_inline_value(TRESHOLD) + } else { + Layout::default() + }; let real = layout.trie_root(x.clone()); let mut memdb = MemoryDB::default(); let mut root = Default::default(); @@ -823,7 +830,11 @@ mod tests { iterator_works_inner(false); } fn iterator_works_inner(limit_inline_value: bool) { - let layout = if limit_inline_value { Layout::with_max_inline_value(TRESHOLD) } else { Layout::default() }; + let layout = if limit_inline_value { + Layout::with_max_inline_value(TRESHOLD) + } else { + Layout::default() + }; let pairs = vec![ (hex!("0103000000000000000464").to_vec(), hex!("0400000000").to_vec()), diff --git a/primitives/trie/src/node_codec.rs b/primitives/trie/src/node_codec.rs index e30da91fc2d92..e630f3222de1e 100644 --- a/primitives/trie/src/node_codec.rs +++ b/primitives/trie/src/node_codec.rs @@ -96,8 +96,7 @@ impl NodeCodec { match header { NodeHeader::Null => Ok(NodePlan::Empty), - NodeHeader::HashedValueBranch(nibble_count) - | NodeHeader::Branch(_, nibble_count) => { + NodeHeader::HashedValueBranch(nibble_count) | NodeHeader::Branch(_, nibble_count) => { let padding = nibble_count % nibble_ops::NIBBLE_PER_BYTE != 0; // check that the padding is valid (if any) if padding && nibble_ops::pad_left(data[input.offset]) != 0 { @@ -121,8 +120,8 @@ impl NodeCodec { ValuePlan::NoValue }; let mut children = [ - None, None, None, None, None, None, None, None, - None, None, None, None, None, None, None, None, + None, None, None, None, None, None, None, None, None, None, None, None, None, + None, None, None, ]; for i in 0..nibble_ops::NIBBLE_LENGTH { if bitmap.value_at(i) { @@ -141,8 +140,7 @@ impl NodeCodec { children, }) }, - NodeHeader::HashedValueLeaf(nibble_count) - | NodeHeader::Leaf(nibble_count) => { + NodeHeader::HashedValueLeaf(nibble_count) | NodeHeader::Leaf(nibble_count) => { let padding = nibble_count % nibble_ops::NIBBLE_PER_BYTE != 0; // check that the padding is valid (if any) if padding && nibble_ops::pad_left(data[input.offset]) != 0 { @@ -295,9 +293,10 @@ fn partial_from_iterator_encode>( NodeKind::Leaf => NodeHeader::Leaf(nibble_count).encode_to(&mut output), NodeKind::BranchWithValue => NodeHeader::Branch(true, nibble_count).encode_to(&mut output), NodeKind::BranchNoValue => NodeHeader::Branch(false, nibble_count).encode_to(&mut output), - NodeKind::HashedValueLeaf => NodeHeader::HashedValueLeaf(nibble_count).encode_to(&mut output), - NodeKind::HashedValueBranch => NodeHeader::HashedValueBranch(nibble_count) - .encode_to(&mut output), + NodeKind::HashedValueLeaf => + NodeHeader::HashedValueLeaf(nibble_count).encode_to(&mut output), + NodeKind::HashedValueBranch => + NodeHeader::HashedValueBranch(nibble_count).encode_to(&mut output), }; output.extend(partial); output @@ -316,9 +315,10 @@ fn partial_encode(partial: Partial, node_kind: NodeKind) -> Vec { NodeKind::Leaf => NodeHeader::Leaf(nibble_count).encode_to(&mut output), NodeKind::BranchWithValue => NodeHeader::Branch(true, nibble_count).encode_to(&mut output), NodeKind::BranchNoValue => NodeHeader::Branch(false, nibble_count).encode_to(&mut output), - NodeKind::HashedValueLeaf => NodeHeader::HashedValueLeaf(nibble_count).encode_to(&mut output), - NodeKind::HashedValueBranch => NodeHeader::HashedValueBranch(nibble_count) - .encode_to(&mut output), + NodeKind::HashedValueLeaf => + NodeHeader::HashedValueLeaf(nibble_count).encode_to(&mut output), + NodeKind::HashedValueBranch => + NodeHeader::HashedValueBranch(nibble_count).encode_to(&mut output), }; if number_nibble_encoded > 0 { output.push(nibble_ops::pad_right((partial.0).1)); diff --git a/primitives/trie/src/node_header.rs b/primitives/trie/src/node_header.rs index 154fbe15c29ad..839fffb87058f 100644 --- a/primitives/trie/src/node_header.rs +++ b/primitives/trie/src/node_header.rs @@ -38,8 +38,7 @@ pub(crate) enum NodeHeader { impl NodeHeader { pub(crate) fn contains_hash_of_value(&self) -> bool { match self { - NodeHeader::HashedValueBranch(_) - | NodeHeader::HashedValueLeaf(_) => true, + NodeHeader::HashedValueBranch(_) | NodeHeader::HashedValueLeaf(_) => true, _ => false, } } @@ -58,16 +57,28 @@ impl Encode for NodeHeader { fn encode_to(&self, output: &mut T) { match self { NodeHeader::Null => output.push_byte(trie_constants::EMPTY_TRIE), - NodeHeader::Branch(true, nibble_count) => + NodeHeader::Branch(true, nibble_count) => encode_size_and_prefix(*nibble_count, trie_constants::BRANCH_WITH_MASK, 2, output), - NodeHeader::Branch(false, nibble_count) => - encode_size_and_prefix(*nibble_count, trie_constants::BRANCH_WITHOUT_MASK, 2, output), + NodeHeader::Branch(false, nibble_count) => encode_size_and_prefix( + *nibble_count, + trie_constants::BRANCH_WITHOUT_MASK, + 2, + output, + ), NodeHeader::Leaf(nibble_count) => encode_size_and_prefix(*nibble_count, trie_constants::LEAF_PREFIX_MASK, 2, output), - NodeHeader::HashedValueBranch(nibble_count) => - encode_size_and_prefix(*nibble_count, trie_constants::ALT_HASHING_BRANCH_WITH_MASK, 4, output), - NodeHeader::HashedValueLeaf(nibble_count) => - encode_size_and_prefix(*nibble_count, trie_constants::ALT_HASHING_LEAF_PREFIX_MASK, 3, output), + NodeHeader::HashedValueBranch(nibble_count) => encode_size_and_prefix( + *nibble_count, + trie_constants::ALT_HASHING_BRANCH_WITH_MASK, + 4, + output, + ), + NodeHeader::HashedValueLeaf(nibble_count) => encode_size_and_prefix( + *nibble_count, + trie_constants::ALT_HASHING_LEAF_PREFIX_MASK, + 3, + output, + ), } } } @@ -82,8 +93,10 @@ impl Decode for NodeHeader { } match i & (0b11 << 6) { trie_constants::LEAF_PREFIX_MASK => Ok(NodeHeader::Leaf(decode_size(i, input, 2)?)), - trie_constants::BRANCH_WITH_MASK => Ok(NodeHeader::Branch(true, decode_size(i, input, 2)?)), - trie_constants::BRANCH_WITHOUT_MASK => Ok(NodeHeader::Branch(false, decode_size(i, input, 2)?)), + trie_constants::BRANCH_WITH_MASK => + Ok(NodeHeader::Branch(true, decode_size(i, input, 2)?)), + trie_constants::BRANCH_WITHOUT_MASK => + Ok(NodeHeader::Branch(false, decode_size(i, input, 2)?)), trie_constants::EMPTY_TRIE => { if i & (0b111 << 5) == trie_constants::ALT_HASHING_LEAF_PREFIX_MASK { Ok(NodeHeader::HashedValueLeaf(decode_size(i, input, 3)?)) diff --git a/primitives/trie/src/trie_codec.rs b/primitives/trie/src/trie_codec.rs index 299cf3cda5f4c..eb3a51f0713aa 100644 --- a/primitives/trie/src/trie_codec.rs +++ b/primitives/trie/src/trie_codec.rs @@ -114,8 +114,7 @@ where let mut nodes_iter = encoded.into_iter(); // Layout does not change trie reading. let layout = L::default(); - let (top_root, _nb_used) = - trie_db::decode_compact_from_iter::(db, &mut nodes_iter)?; + let (top_root, _nb_used) = trie_db::decode_compact_from_iter::(db, &mut nodes_iter)?; // Only check root if expected root is passed as argument. if let Some(expected_root) = expected_root { @@ -165,8 +164,7 @@ where let mut previous_extracted_child_trie = None; for child_root in child_tries.into_iter() { if previous_extracted_child_trie.is_none() { - let (top_root, _) = - trie_db::decode_compact_from_iter::(db, &mut nodes_iter)?; + let (top_root, _) = trie_db::decode_compact_from_iter::(db, &mut nodes_iter)?; previous_extracted_child_trie = Some(top_root); } diff --git a/primitives/trie/src/trie_stream.rs b/primitives/trie/src/trie_stream.rs index 44d79770a5d33..20cc35c6b8708 100644 --- a/primitives/trie/src/trie_stream.rs +++ b/primitives/trie/src/trie_stream.rs @@ -75,9 +75,7 @@ fn fuse_nibbles_node<'a>(nibbles: &'a [u8], kind: NodeKind) -> impl Iterator Self { - Self { - buffer: Vec::new(), - } + Self { buffer: Vec::new() } } fn append_empty_data(&mut self) { @@ -115,10 +113,10 @@ impl trie_root::TrieStream for TrieStream { TrieStreamValue::Value(..) => NodeKind::BranchWithValue, TrieStreamValue::HashedValue(..) => NodeKind::HashedValueBranch, }; - + self.buffer.extend(fuse_nibbles_node(partial, kind)); let bm = branch_node_bit_mask(has_children); - self.buffer.extend([bm.0,bm.1].iter()); + self.buffer.extend([bm.0, bm.1].iter()); } else { unreachable!("trie stream codec only for no extension trie"); } From 83e555346a5a9b60758364fed708e22c62ab70c0 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 7 Sep 2021 10:42:14 +0200 Subject: [PATCH 125/127] encoding of storage proof with default state version of 0 length --- client/network/src/protocol/message.rs | 7 +- .../state-machine/src/proving_backend.rs | 8 +- primitives/trie/src/lib.rs | 2 +- primitives/trie/src/storage_proof.rs | 144 +++++++++++++++--- 4 files changed, 130 insertions(+), 31 deletions(-) diff --git a/client/network/src/protocol/message.rs b/client/network/src/protocol/message.rs index 1ffc57de181cf..06f71db70e6f1 100644 --- a/client/network/src/protocol/message.rs +++ b/client/network/src/protocol/message.rs @@ -351,12 +351,13 @@ pub mod generic { let compact = CompactStatus::decode(value)?; let chain_status = match >::decode(value) { Ok(v) => v, - Err(e) => + Err(e) => { if compact.version <= LAST_CHAIN_STATUS_VERSION { - return Err(e) + return Err(e); } else { Vec::new() - }, + } + } }; let CompactStatus { diff --git a/primitives/state-machine/src/proving_backend.rs b/primitives/state-machine/src/proving_backend.rs index d870431d2ba36..e5a1865b4da62 100644 --- a/primitives/state-machine/src/proving_backend.rs +++ b/primitives/state-machine/src/proving_backend.rs @@ -151,9 +151,9 @@ impl ProofRecorder { /// encoded proof. pub fn estimate_encoded_size(&self) -> usize { let inner = self.inner.read(); - inner.encoded_size + - codec::Compact(inner.records.len() as u32).encoded_size() + - inner.state_version.encoded_size() + inner.encoded_size + + codec::Compact(inner.records.len() as u32).encoded_size() + + sp_trie::state_version_encoded_size(inner.state_version) } /// Convert into a [`StorageProof`]. @@ -230,7 +230,7 @@ impl<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> TrieBackendStorage fn get(&self, key: &H::Out, prefix: Prefix) -> Result, String> { if let Some(v) = self.proof_recorder.get(key) { - return Ok(v) + return Ok(v); } let backend_value = self.backend.get(key, prefix)?; diff --git a/primitives/trie/src/lib.rs b/primitives/trie/src/lib.rs index d4b260ff5a1a7..473cbc2337492 100644 --- a/primitives/trie/src/lib.rs +++ b/primitives/trie/src/lib.rs @@ -37,7 +37,7 @@ pub use memory_db::KeyFunction; /// The Substrate format implementation of `NodeCodec`. pub use node_codec::NodeCodec; use sp_std::{borrow::Borrow, boxed::Box, fmt, marker::PhantomData, vec::Vec}; -pub use storage_proof::{CompactProof, StorageProof}; +pub use storage_proof::{CompactProof, StorageProof, state_version_encoded_size}; /// Trie codec reexport, mainly child trie support /// for trie compact proof. pub use trie_codec::{decode_compact, encode_compact, Error as CompactProofError}; diff --git a/primitives/trie/src/storage_proof.rs b/primitives/trie/src/storage_proof.rs index 2b8067a86c8f1..b73581347de73 100644 --- a/primitives/trie/src/storage_proof.rs +++ b/primitives/trie/src/storage_proof.rs @@ -17,6 +17,7 @@ use crate::Layout; use codec::{Decode, Encode}; +pub use decode_encode_impl::state_version_encoded_size; use hash_db::{HashDB, Hasher}; use sp_core::state_version::StateVersion; use sp_std::vec::Vec; @@ -28,48 +29,145 @@ use sp_std::vec::Vec; /// The proof consists of the set of serialized nodes in the storage trie accessed when looking up /// the keys covered by the proof. Verifying the proof requires constructing the partial trie from /// the serialized nodes and performing the key lookups. -#[derive(Debug, PartialEq, Eq, Clone, Encode)] +#[derive(Debug, PartialEq, Eq, Clone)] pub struct StorageProof { - // TODO decode no more bytes to V0 for compatibility and remove pub(crate) - pub(crate) trie_nodes: Vec>, pub(crate) state_version: StateVersion, + pub(crate) trie_nodes: Vec>, } /// Storage proof in compact form. -#[derive(Debug, PartialEq, Eq, Clone, Encode)] +#[derive(Debug, PartialEq, Eq, Clone)] pub struct CompactProof { - pub encoded_nodes: Vec>, pub state_version: StateVersion, + pub encoded_nodes: Vec>, } -mod decode_impl { +mod decode_encode_impl { use super::*; - use codec::{Error, Input}; - impl Decode for StorageProof { - fn decode(input: &mut I) -> Result { - let trie_nodes: Vec> = Decode::decode(input)?; - let state_version = if let Ok(Some(0)) = input.remaining_len() { - // for compatibility with existing proof decoded from sized buffer - StateVersion::V0 + use codec::{Compact, Error, Input, Output}; + + const STATE_V0: &'static [u8] = &[3, 0]; // This is an invalid compact size encoding. + const STATE_V1: &'static [u8] = &[7, 1]; // or any correctly sized number of node. + + /// Prefix another input with a byte. + struct PrefixInput<'a, T> { + prefix1: Option, + prefix2: Option, + input: &'a mut T, + } + + struct StateVersionInProof(StateVersion); + + impl<'a, T: 'a + Input> Input for PrefixInput<'a, T> { + fn remaining_len(&mut self) -> Result, Error> { + let len = if let Some(len) = self.input.remaining_len()? { + Some(len.saturating_add( + self.prefix1.iter().count().saturating_add(self.prefix2.iter().count()), + )) } else { - Decode::decode(input)? + None }; + Ok(len) + } + + fn read(&mut self, buffer: &mut [u8]) -> Result<(), Error> { + let mut buff_i = 0; + if buffer.is_empty() { + return Ok(()); + } + match self.prefix1.take() { + Some(v) => { + buffer[buff_i] = v; + buff_i += 1; + if buffer.is_empty() { + return Ok(()); + } + } + _ => (), + } + + match self.prefix2.take() { + Some(v) => { + buffer[buff_i] = v; + buff_i += 1; + self.input.read(&mut buffer[buff_i..]) + } + _ => self.input.read(&mut buffer[buff_i..]), + } + } + } + + fn decode_inner(input: &mut I) -> Result<(StateVersion, Vec>), Error> { + let prefix1 = input.read_byte()?; + let (state_version, mut input) = if prefix1 == STATE_V0[0] || prefix1 == STATE_V1[0] { + let prefix2 = input.read_byte()?; + if prefix2 == STATE_V0[1] { + (StateVersion::V0, PrefixInput { prefix1: None, prefix2: None, input }) + } else if prefix2 == STATE_V1[1] { + let threshold = Compact::::decode(input)?.0; + ( + StateVersion::V1 { threshold }, + PrefixInput { prefix1: None, prefix2: None, input }, + ) + } else { + ( + Default::default(), + PrefixInput { prefix1: Some(prefix1), prefix2: Some(prefix2), input }, + ) + } + } else { + (Default::default(), PrefixInput { prefix1: None, prefix2: Some(prefix1), input }) + }; + let trie_nodes: Vec> = Decode::decode(&mut input)?; + Ok((state_version, trie_nodes)) + } + + impl Encode for StateVersionInProof { + fn encode_to(&self, dest: &mut T) { + if self.0 != Default::default() { + match self.0 { + StateVersion::V0 => dest.write(STATE_V0), + StateVersion::V1 { threshold } => { + dest.write(STATE_V1); + Compact(threshold).encode_to(dest); + } + } + } + } + } + + impl Decode for StorageProof { + fn decode(input: &mut I) -> Result { + let (state_version, trie_nodes) = decode_inner(input)?; Ok(StorageProof { trie_nodes, state_version }) } } impl Decode for CompactProof { fn decode(input: &mut I) -> Result { - let encoded_nodes: Vec> = Decode::decode(input)?; - let state_version = if let Ok(Some(0)) = input.remaining_len() { - // for compatibility with existing proof decoded from sized buffer - StateVersion::V0 - } else { - Decode::decode(input)? - }; + let (state_version, encoded_nodes) = decode_inner(input)?; Ok(CompactProof { encoded_nodes, state_version }) } } + + impl Encode for StorageProof { + fn encode_to(&self, dest: &mut T) { + StateVersionInProof(self.state_version).encode_to(dest); + self.trie_nodes.encode_to(dest); + } + } + + impl Encode for CompactProof { + fn encode_to(&self, dest: &mut T) { + StateVersionInProof(self.state_version).encode_to(dest); + self.encoded_nodes.encode_to(dest); + } + } + + /// Utility to get state version size encoded in proof. + pub fn state_version_encoded_size(state_version: StateVersion) -> usize { + StateVersionInProof(state_version).encoded_size() + } } impl StorageProof { @@ -124,8 +222,8 @@ impl StorageProof { .into_iter() .flat_map(|proof| { debug_assert!( - state_version == &StateVersion::default() || - state_version == &proof.state_version + state_version == &StateVersion::default() + || state_version == &proof.state_version ); *state_version = proof.state_version; proof.iter_nodes() From 4384ee632ac6ac5b419ea2039255bfbbc7d90006 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 7 Sep 2021 10:47:54 +0200 Subject: [PATCH 126/127] use a single byte. --- primitives/trie/src/storage_proof.rs | 61 ++++++++-------------------- 1 file changed, 17 insertions(+), 44 deletions(-) diff --git a/primitives/trie/src/storage_proof.rs b/primitives/trie/src/storage_proof.rs index b73581347de73..5348e4c25951c 100644 --- a/primitives/trie/src/storage_proof.rs +++ b/primitives/trie/src/storage_proof.rs @@ -46,13 +46,12 @@ mod decode_encode_impl { use super::*; use codec::{Compact, Error, Input, Output}; - const STATE_V0: &'static [u8] = &[3, 0]; // This is an invalid compact size encoding. - const STATE_V1: &'static [u8] = &[7, 1]; // or any correctly sized number of node. + const STATE_V0: u8 = 251; // This is an invalid compact size encoding. + const STATE_V1: u8 = 247; // or any correctly sized number of node. /// Prefix another input with a byte. struct PrefixInput<'a, T> { - prefix1: Option, - prefix2: Option, + prefix: Option, input: &'a mut T, } @@ -61,9 +60,7 @@ mod decode_encode_impl { impl<'a, T: 'a + Input> Input for PrefixInput<'a, T> { fn remaining_len(&mut self) -> Result, Error> { let len = if let Some(len) = self.input.remaining_len()? { - Some(len.saturating_add( - self.prefix1.iter().count().saturating_add(self.prefix2.iter().count()), - )) + Some(len.saturating_add(self.prefix.iter().count())) } else { None }; @@ -71,52 +68,28 @@ mod decode_encode_impl { } fn read(&mut self, buffer: &mut [u8]) -> Result<(), Error> { - let mut buff_i = 0; if buffer.is_empty() { return Ok(()); } - match self.prefix1.take() { + match self.prefix.take() { Some(v) => { - buffer[buff_i] = v; - buff_i += 1; - if buffer.is_empty() { - return Ok(()); - } + buffer[0] = v; + self.input.read(&mut buffer[1..]) } - _ => (), - } - - match self.prefix2.take() { - Some(v) => { - buffer[buff_i] = v; - buff_i += 1; - self.input.read(&mut buffer[buff_i..]) - } - _ => self.input.read(&mut buffer[buff_i..]), + _ => self.input.read(buffer), } } } fn decode_inner(input: &mut I) -> Result<(StateVersion, Vec>), Error> { - let prefix1 = input.read_byte()?; - let (state_version, mut input) = if prefix1 == STATE_V0[0] || prefix1 == STATE_V1[0] { - let prefix2 = input.read_byte()?; - if prefix2 == STATE_V0[1] { - (StateVersion::V0, PrefixInput { prefix1: None, prefix2: None, input }) - } else if prefix2 == STATE_V1[1] { - let threshold = Compact::::decode(input)?.0; - ( - StateVersion::V1 { threshold }, - PrefixInput { prefix1: None, prefix2: None, input }, - ) - } else { - ( - Default::default(), - PrefixInput { prefix1: Some(prefix1), prefix2: Some(prefix2), input }, - ) - } + let prefix = input.read_byte()?; + let (state_version, mut input) = if prefix == STATE_V0 { + (StateVersion::V0, PrefixInput { prefix: None, input }) + } else if prefix == STATE_V1 { + let threshold = Compact::::decode(input)?.0; + (StateVersion::V1 { threshold }, PrefixInput { prefix: None, input }) } else { - (Default::default(), PrefixInput { prefix1: None, prefix2: Some(prefix1), input }) + (Default::default(), PrefixInput { prefix: Some(prefix), input }) }; let trie_nodes: Vec> = Decode::decode(&mut input)?; Ok((state_version, trie_nodes)) @@ -126,9 +99,9 @@ mod decode_encode_impl { fn encode_to(&self, dest: &mut T) { if self.0 != Default::default() { match self.0 { - StateVersion::V0 => dest.write(STATE_V0), + StateVersion::V0 => dest.push_byte(STATE_V0), StateVersion::V1 { threshold } => { - dest.write(STATE_V1); + dest.push_byte(STATE_V1); Compact(threshold).encode_to(dest); } } From 1d6aa2fa8cde94c7f4bf4d2ebe22cecbd8391588 Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 8 Sep 2021 09:06:38 +0200 Subject: [PATCH 127/127] Fix comment on first byte (not a dead header). --- primitives/trie/src/storage_proof.rs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/primitives/trie/src/storage_proof.rs b/primitives/trie/src/storage_proof.rs index 5348e4c25951c..ea7af59aecb0e 100644 --- a/primitives/trie/src/storage_proof.rs +++ b/primitives/trie/src/storage_proof.rs @@ -45,9 +45,14 @@ pub struct CompactProof { mod decode_encode_impl { use super::*; use codec::{Compact, Error, Input, Output}; - - const STATE_V0: u8 = 251; // This is an invalid compact size encoding. - const STATE_V1: u8 = 247; // or any correctly sized number of node. + // This is compact size encoding first byte for + // length > u128::MAX. In case such big proof + // get needed state version could not be omitted. + // Use to indicate state V0. + const STATE_V0: u8 = 251; + + // First byte encoding for state V1. + const STATE_V1: u8 = 247; /// Prefix another input with a byte. struct PrefixInput<'a, T> {