Trie (non)-membership proofs

core/primitives/src/views.rs

@@ -193,9 +193,6 @@ impl From<AccessKeyView> for AccessKey {
     }
 }
 
-/// Set of serialized TrieNodes that are encoded in base64. Represent proof of inclusion of some TrieNode in the MerkleTrie.
-pub type TrieProofPath = Vec<String>;
-
 /// Item of the state, key and value are serialized in base64 and proof for inclusion of given state item.
 #[cfg_attr(feature = "deepsize_feature", derive(deepsize::DeepSizeOf))]
 #[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)]
@@ -204,14 +201,18 @@ pub struct StateItem {
     pub key: Vec<u8>,
     #[serde(with = "base64_format")]
     pub value: Vec<u8>,
-    pub proof: TrieProofPath,
+    /// Deprecated, always empty, eventually will be deleted.
+    // TODO(mina86): This was deprecated in 1.30.  Get rid of the field
+    // altogether at 1.33 or something.
+    #[serde(default)]
+    pub proof: Vec<()>,
 }
 
 #[cfg_attr(feature = "deepsize_feature", derive(deepsize::DeepSizeOf))]
 #[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)]
 pub struct ViewStateResult {
     pub values: Vec<StateItem>,
-    pub proof: TrieProofPath,
+    pub proof: Vec<Arc<[u8]>>,
 }
 
 #[cfg_attr(feature = "deepsize_feature", derive(deepsize::DeepSizeOf))]

core/store/src/lib.rs

@@ -31,9 +31,9 @@ use crate::db::{
 pub use crate::trie::iterator::TrieIterator;
 pub use crate::trie::update::{TrieUpdate, TrieUpdateIterator, TrieUpdateValuePtr};
 pub use crate::trie::{
-    estimator, split_state, ApplyStatePartResult, KeyForStateChanges, PartialStorage, ShardTries,
-    Trie, TrieAccess, TrieCache, TrieCacheFactory, TrieCachingStorage, TrieChanges, TrieStorage,
-    WrappedTrieChanges,
+    estimator, split_state, ApplyStatePartResult, KeyForStateChanges, NibbleSlice, PartialStorage,
+    RawTrieNode, RawTrieNodeWithSize, ShardTries, Trie, TrieAccess, TrieCache, TrieCacheFactory,
+    TrieCachingStorage, TrieChanges, TrieNodeWithSize, TrieStorage, WrappedTrieChanges,
 };
 
 mod columns;

core/store/src/trie/iterator.rs

@@ -36,6 +36,9 @@ pub struct TrieIterator<'a> {
     trie: &'a Trie,
     trail: Vec<Crumb>,
     pub(crate) key_nibbles: Vec<u8>,
+
+    /// If not `None`, a list of all nodes that the iterator has visited.
+    visited_nodes: Option<Vec<std::sync::Arc<[u8]>>>,
 }
 
 pub type TrieItem = (Vec<u8>, Vec<u8>);
@@ -56,6 +59,7 @@ impl<'a> TrieIterator<'a> {
             trie,
             trail: Vec::with_capacity(8),
             key_nibbles: Vec::with_capacity(64),
+            visited_nodes: None,
         };
         r.descend_into_node(&trie.root)?;
         Ok(r)
@@ -65,6 +69,23 @@ impl<'a> TrieIterator<'a> {
     pub fn seek<K: AsRef<[u8]>>(&mut self, key: K) -> Result<(), StorageError> {
         self.seek_nibble_slice(NibbleSlice::new(key.as_ref())).map(drop)
     }
+    /// Configures whether the iterator should remember all the nodes its
+    /// visiting.
+    ///
+    /// Use [`Self::into_visited_nodes`] to retrieve the list.
+    pub fn remember_visited_nodes(&mut self, remember: bool) {
+        self.visited_nodes = remember.then(|| Vec::new());
+    }
+
+    /// Consumes iterator and returns list of nodes it’s visited.
+    ///
+    /// By default the iterator *doesn’t* remember nodes it visits.  To enable
+    /// that feature use [`Self::remember_visited_nodes`] method.  If the
+    /// feature is disabled, this method returns an empty list.  Otherwise
+    /// it returns list of nodes visited since the feature was enabled.
+    pub fn into_visited_nodes(self) -> Vec<std::sync::Arc<[u8]>> {
+        self.visited_nodes.unwrap_or(Vec::new())
+    }
 
     /// Returns the hash of the last node
     pub(crate) fn seek_nibble_slice(
@@ -124,9 +145,15 @@ impl<'a> TrieIterator<'a> {
 
     /// Fetches block by its hash and adds it to the trail.
     ///
-    /// The node is stored as the last [`Crumb`] in the trail.
+    /// The node is stored as the last [`Crumb`] in the trail.  If iterator is
+    /// configured to remember all the nodes its visiting (which can be enabled
+    /// with [`Self::remember_visited_nodes`]), the node will be added to the
+    /// list.
     fn descend_into_node(&mut self, hash: &CryptoHash) -> Result<(), StorageError> {
-        let node = self.trie.retrieve_node(hash)?;
+        let (bytes, node) = self.trie.retrieve_node(hash)?;
+        if let (Some(bytes), Some(nodes)) = (bytes, &mut self.visited_nodes) {
+            nodes.push(bytes);
+        }
         self.trail.push(Crumb { status: CrumbStatus::Entering, node });
         Ok(())
     }

core/store/src/trie/mod.rs

@@ -17,7 +17,7 @@ use near_primitives::types::{StateRoot, StateRootNode};
 use crate::flat_state::FlatState;
 use crate::trie::insert_delete::NodesStorage;
 use crate::trie::iterator::TrieIterator;
-use crate::trie::nibble_slice::NibbleSlice;
+pub use crate::trie::nibble_slice::NibbleSlice;
 pub use crate::trie::shard_tries::{
     KeyForStateChanges, ShardTries, TrieCacheFactory, WrappedTrieChanges,
 };
@@ -257,7 +257,7 @@ impl TrieNode {
 
 #[derive(Debug, Eq, PartialEq)]
 #[allow(clippy::large_enum_variant)]
-enum RawTrieNode {
+pub enum RawTrieNode {
     Leaf(Vec<u8>, u32, CryptoHash),
     Branch([Option<CryptoHash>; 16], Option<(u32, CryptoHash)>),
     Extension(Vec<u8>, CryptoHash),
@@ -266,8 +266,8 @@ enum RawTrieNode {
 /// Trie node + memory cost of its subtree
 /// memory_usage is serialized, stored, and contributes to hash
 #[derive(Debug, Eq, PartialEq)]
-struct RawTrieNodeWithSize {
-    node: RawTrieNode,
+pub struct RawTrieNodeWithSize {
+    pub node: RawTrieNode,
     memory_usage: u64,
 }
 
@@ -344,7 +344,7 @@ impl RawTrieNode {
         out
     }
 
-    fn decode(bytes: &[u8]) -> Result<Self, std::io::Error> {
+    pub fn decode(bytes: &[u8]) -> Result<Self, std::io::Error> {
         let mut cursor = Cursor::new(bytes);
         match cursor.read_u8()? {
             LEAF_NODE => {
@@ -394,7 +394,7 @@ impl RawTrieNodeWithSize {
         out
     }
 
-    fn decode(bytes: &[u8]) -> Result<Self, std::io::Error> {
+    pub fn decode(bytes: &[u8]) -> Result<Self, std::io::Error> {
         if bytes.len() < 8 {
             return Err(std::io::Error::new(std::io::ErrorKind::Other, "Wrong type"));
         }
@@ -533,7 +533,7 @@ impl Trie {
         }
         let TrieNodeWithSize { node, memory_usage } = match handle {
             NodeHandle::InMemory(h) => memory.node_ref(h).clone(),
-            NodeHandle::Hash(h) => self.retrieve_node(&h).expect("storage failure"),
+            NodeHandle::Hash(h) => self.retrieve_node(&h).expect("storage failure").1,
         };
 
         let mut memory_usage_naive = node.memory_usage_direct(memory);
@@ -615,10 +615,19 @@ impl Trie {
         }
     }
 
-    fn retrieve_node(&self, hash: &CryptoHash) -> Result<TrieNodeWithSize, StorageError> {
+    /// Retrieves decoded node alongside with its raw bytes representation.
+    ///
+    /// Note that because Empty nodes (those which are referenced by
+    /// [`Self::EMPTY_ROOT`] hash) aren’t stored in the database, they don’t
+    /// have a bytes representation.  For those nodes the first return value
+    /// will be `None`.
+    fn retrieve_node(
+        &self,
+        hash: &CryptoHash,
+    ) -> Result<(Option<std::sync::Arc<[u8]>>, TrieNodeWithSize), StorageError> {
         match self.retrieve_raw_node(hash)? {
-            None => Ok(TrieNodeWithSize::empty()),
-            Some((_bytes, node)) => Ok(TrieNodeWithSize::from_raw(node)),
+            None => Ok((None, TrieNodeWithSize::empty())),
+            Some((bytes, node)) => Ok((Some(bytes), TrieNodeWithSize::from_raw(node))),
         }
     }
 

core/store/src/trie/state_parts.rs

@@ -71,7 +71,7 @@ impl Trie {
         if part_id == num_parts {
             return Ok(vec![16]);
         }
-        let root_node = self.retrieve_node(&self.root)?;
+        let (_bytes, root_node) = self.retrieve_node(&self.root)?;
         let total_size = root_node.memory_usage;
         let size_start = (total_size + num_parts - 1) / num_parts * part_id;
         self.find_path(&root_node, size_start)
@@ -106,7 +106,7 @@ impl Trie {
                         Some(NodeHandle::InMemory(_)) => {
                             unreachable!("only possible while mutating")
                         }
-                        Some(NodeHandle::Hash(h)) => self.retrieve_node(h)?,
+                        Some(NodeHandle::Hash(h)) => self.retrieve_node(h)?.1,
                     };
                     if *size_skipped + child.memory_usage <= size_start {
                         *size_skipped += child.memory_usage;
@@ -136,7 +136,7 @@ impl Trie {
             TrieNode::Extension(key, child_handle) => {
                 let child = match child_handle {
                     NodeHandle::InMemory(_) => unreachable!("only possible while mutating"),
-                    NodeHandle::Hash(h) => self.retrieve_node(h)?,
+                    NodeHandle::Hash(h) => self.retrieve_node(h)?.1,
                 };
                 let (slice, _is_leaf) = NibbleSlice::from_encoded(key);
                 key_nibbles.extend(slice.iter());
@@ -312,7 +312,7 @@ mod tests {
                 return Ok(());
             }
             let mut stack: Vec<(CryptoHash, TrieNodeWithSize, CrumbStatus)> = Vec::new();
-            let root_node = self.retrieve_node(&self.root)?;
+            let (_bytes, root_node) = self.retrieve_node(&self.root)?;
             stack.push((self.root.clone(), root_node, CrumbStatus::Entering));
             while let Some((hash, node, position)) = stack.pop() {
                 if let CrumbStatus::Entering = position {
@@ -353,7 +353,7 @@ mod tests {
                             }
                             if i < 16 {
                                 if let Some(NodeHandle::Hash(h)) = children[i].clone() {
-                                    let child = self.retrieve_node(&h)?;
+                                    let (_bytes, child) = self.retrieve_node(&h)?;
                                     stack.push((hash, node, CrumbStatus::AtChild(i + 1)));
                                     stack.push((h, child, CrumbStatus::Entering));
                                 } else {
@@ -377,7 +377,7 @@ mod tests {
                                     unreachable!("only possible while mutating")
                                 }
                                 NodeHandle::Hash(h) => {
-                                    let child = self.retrieve_node(&h)?;
+                                    let (_bytes, child) = self.retrieve_node(&h)?;
                                     stack.push((hash, node, CrumbStatus::Exiting));
                                     stack.push((h, child, CrumbStatus::Entering));
                                 }
@@ -394,7 +394,7 @@ mod tests {
             size_start: u64,
             size_end: u64,
         ) -> Result<(), StorageError> {
-            let root_node = self.retrieve_node(&self.root)?;
+            let (_bytes, root_node) = self.retrieve_node(&self.root)?;
             let path_begin = self.find_path(&root_node, size_start)?;
             let path_end = self.find_path(&root_node, size_end)?;
 
@@ -424,7 +424,7 @@ mod tests {
             part_id: PartId,
         ) -> Result<PartialState, StorageError> {
             assert!(self.storage.as_caching_storage().is_some());
-            let root_node = self.retrieve_node(&self.root)?;
+            let (_bytes, root_node) = self.retrieve_node(&self.root)?;
             let total_size = root_node.memory_usage;
             let size_start = (total_size + part_id.total - 1) / part_id.total * part_id.idx;
             let size_end = std::cmp::min(