diff --git a/crates/primitives/src/trie/hash_builder/mod.rs b/crates/primitives/src/trie/hash_builder/mod.rs index ed2c86de77ed2..f0daa9ca27d81 100644 --- a/crates/primitives/src/trie/hash_builder/mod.rs +++ b/crates/primitives/src/trie/hash_builder/mod.rs @@ -204,19 +204,22 @@ impl HashBuilder { trace!(target: "trie::hash_builder", ?current, ?succeeding, "updating merkle tree"); let mut i = 0; + let span = tracing::trace_span!( + target: "trie::hash_builder", + "loop", + i, + current = tracing::field::Empty, + build_extensions = tracing::field::Empty, + ) + .entered(); loop { - let span = tracing::span!( - target: "trie::hash_builder", - tracing::Level::TRACE, - "loop", - i, - ?current, - ?build_extensions - ); - let _enter = span.enter(); + if !span.is_disabled() { + span.record("current", &format!("{current:?}")); + span.record("build_extensions", &build_extensions); + } let preceding_exists = !self.groups.is_empty(); - let preceding_len: usize = self.groups.len().saturating_sub(1); + let preceding_len = self.groups.len().saturating_sub(1); let common_prefix_len = succeeding.common_prefix_length(current.as_slice()); let len = std::cmp::max(preceding_len, common_prefix_len); @@ -254,7 +257,7 @@ impl HashBuilder { if !succeeding.is_empty() || preceding_exists { len_from += 1; } - trace!(target: "trie::hash_builder", "skipping {} nibbles", len_from); + trace!(target: "trie::hash_builder", "skipping {len_from} nibbles"); // The key without the common prefix let short_node_key = current.slice(len_from..); diff --git a/crates/primitives/src/trie/nibbles.rs b/crates/primitives/src/trie/nibbles.rs index 7c6d92a2bfc82..e7bcd8b828559 100644 --- a/crates/primitives/src/trie/nibbles.rs +++ b/crates/primitives/src/trie/nibbles.rs @@ -67,7 +67,6 @@ impl Compact for StoredNibblesSubKey { /// The internal representation is a [`SmallVec`] that stores one nibble per byte. This means that /// each byte has its upper 4 bits set to zero and the lower 4 bits representing the nibble value. #[derive( - Clone, Default, PartialEq, Eq, @@ -83,6 +82,19 @@ impl Compact for StoredNibblesSubKey { #[cfg_attr(any(test, feature = "arbitrary"), derive(arbitrary::Arbitrary))] pub struct Nibbles(SmallVec<[u8; 64]>); +// Override `SmallVec::from` since it's not specialized for `Copy` types. +impl Clone for Nibbles { + #[inline] + fn clone(&self) -> Self { + Self(SmallVec::from_slice(&self.0)) + } + + #[inline] + fn clone_from(&mut self, source: &Self) { + self.0.clone_from(&source.0); + } +} + impl alloy_rlp::Encodable for Nibbles { #[inline] fn length(&self) -> usize { diff --git a/crates/primitives/src/trie/nodes/mod.rs b/crates/primitives/src/trie/nodes/mod.rs index 98e7d494af7ec..c0c661795d19b 100644 --- a/crates/primitives/src/trie/nodes/mod.rs +++ b/crates/primitives/src/trie/nodes/mod.rs @@ -28,5 +28,9 @@ fn rlp_node(rlp: &[u8]) -> Vec { // TODO: this could return [u8; 33] but Vec is needed everywhere this function is used #[inline] pub fn word_rlp(word: &B256) -> Vec { - [&[EMPTY_STRING_CODE + B256::len_bytes() as u8][..], &word[..]].concat() + // Gets optimized to alloc + write directly into it: https://godbolt.org/z/rfWGG6ebq + let mut arr = [0; 33]; + arr[0] = EMPTY_STRING_CODE + 32; + arr[1..].copy_from_slice(word.as_slice()); + arr.to_vec() } diff --git a/crates/storage/codecs/src/lib.rs b/crates/storage/codecs/src/lib.rs index 2e9a8d3cb522d..93d5e4ec22b78 100644 --- a/crates/storage/codecs/src/lib.rs +++ b/crates/storage/codecs/src/lib.rs @@ -53,6 +53,7 @@ pub trait Compact: Sized { fn from_compact(buf: &[u8], len: usize) -> (Self, &[u8]); /// "Optional": If there's no good reason to use it, don't. + #[inline] fn specialized_to_compact(self, buf: &mut B) -> usize where B: bytes::BufMut + AsMut<[u8]>, @@ -61,6 +62,7 @@ pub trait Compact: Sized { } /// "Optional": If there's no good reason to use it, don't. + #[inline] fn specialized_from_compact(buf: &[u8], len: usize) -> (Self, &[u8]) { Self::from_compact(buf, len) } @@ -70,6 +72,7 @@ macro_rules! impl_uint_compact { ($($name:tt),+) => { $( impl Compact for $name { + #[inline] fn to_compact(self, buf: &mut B) -> usize where B: bytes::BufMut + AsMut<[u8]> { @@ -78,16 +81,16 @@ macro_rules! impl_uint_compact { std::mem::size_of::<$name>() - leading } + #[inline] fn from_compact(mut buf: &[u8], len: usize) -> (Self, &[u8]) { - if len > 0 { - let mut arr = [0; std::mem::size_of::<$name>()]; - arr[std::mem::size_of::<$name>() - len..].copy_from_slice(&buf[..len]); - - buf.advance(len); - - return ($name::from_be_bytes(arr), buf) + if len == 0 { + return (0, buf); } - (0, buf) + + let mut arr = [0; std::mem::size_of::<$name>()]; + arr[std::mem::size_of::<$name>() - len..].copy_from_slice(&buf[..len]); + buf.advance(len); + ($name::from_be_bytes(arr), buf) } } )+ @@ -101,6 +104,7 @@ where T: Compact, { /// Returns 0 since we won't include it in the `StructFlags`. + #[inline] fn to_compact(self, buf: &mut B) -> usize where B: bytes::BufMut + AsMut<[u8]>, @@ -122,6 +126,7 @@ where 0 } + #[inline] fn from_compact(buf: &[u8], _: usize) -> (Self, &[u8]) { let (length, mut buf) = decode_varuint(buf); let mut list = Vec::with_capacity(length); @@ -139,6 +144,7 @@ where } /// To be used by fixed sized types like `Vec`. + #[inline] fn specialized_to_compact(self, buf: &mut B) -> usize where B: bytes::BufMut + AsMut<[u8]>, @@ -151,6 +157,7 @@ where } /// To be used by fixed sized types like `Vec`. + #[inline] fn specialized_from_compact(buf: &[u8], len: usize) -> (Self, &[u8]) { let (length, mut buf) = decode_varuint(buf); let mut list = Vec::with_capacity(length); @@ -170,25 +177,27 @@ where T: Compact, { /// Returns 0 for `None` and 1 for `Some(_)`. + #[inline] fn to_compact(self, buf: &mut B) -> usize where B: bytes::BufMut + AsMut<[u8]>, { - let mut tmp = Vec::with_capacity(64); + let Some(element) = self else { + return 0; + }; - if let Some(element) = self { - // We don't know the length until we compact it - let length = element.to_compact(&mut tmp); + // We don't know the length of the element until we compact it. + let mut tmp = Vec::with_capacity(64); + let length = element.to_compact(&mut tmp); - encode_varuint(length, buf); + encode_varuint(length, buf); - buf.put_slice(&tmp); + buf.put_slice(&tmp); - return 1 - } - 0 + 1 } + #[inline] fn from_compact(buf: &[u8], len: usize) -> (Self, &[u8]) { if len == 0 { return (None, buf) @@ -203,53 +212,58 @@ where } /// To be used by fixed sized types like `Option`. + #[inline] fn specialized_to_compact(self, buf: &mut B) -> usize where B: bytes::BufMut + AsMut<[u8]>, { if let Some(element) = self { element.to_compact(buf); - return 1 + 1 + } else { + 0 } - 0 } /// To be used by fixed sized types like `Option`. + #[inline] fn specialized_from_compact(buf: &[u8], len: usize) -> (Self, &[u8]) { if len == 0 { return (None, buf) } let (element, buf) = T::from_compact(buf, len); - (Some(element), buf) } } impl Compact for U256 { + #[inline] fn to_compact(self, buf: &mut B) -> usize where B: bytes::BufMut + AsMut<[u8]>, { - let inner: [u8; 32] = self.to_be_bytes(); + let inner = self.to_be_bytes::<32>(); let size = 32 - (self.leading_zeros() / 8); buf.put_slice(&inner[32 - size..]); size } + #[inline] fn from_compact(mut buf: &[u8], len: usize) -> (Self, &[u8]) { - if len > 0 { - let mut arr = [0; 32]; - arr[(32 - len)..].copy_from_slice(&buf[..len]); - buf.advance(len); - return (U256::from_be_bytes(arr), buf) + if len == 0 { + return (U256::ZERO, buf); } - (U256::ZERO, buf) + let mut arr = [0; 32]; + arr[(32 - len)..].copy_from_slice(&buf[..len]); + buf.advance(len); + (U256::from_be_bytes(arr), buf) } } impl Compact for Bytes { + #[inline] fn to_compact(self, buf: &mut B) -> usize where B: bytes::BufMut + AsMut<[u8]>, @@ -259,12 +273,14 @@ impl Compact for Bytes { len } + #[inline] fn from_compact(mut buf: &[u8], len: usize) -> (Self, &[u8]) { (buf.copy_to_bytes(len).into(), buf) } } impl Compact for [u8; N] { + #[inline] fn to_compact(self, buf: &mut B) -> usize where B: bytes::BufMut + AsMut<[u8]>, @@ -273,6 +289,7 @@ impl Compact for [u8; N] { N } + #[inline] fn from_compact(mut buf: &[u8], len: usize) -> (Self, &[u8]) { if len == 0 { return ([0; N], buf) @@ -290,6 +307,7 @@ macro_rules! impl_compact_for_bytes { ($($name:tt),+) => { $( impl Compact for $name { + #[inline] fn to_compact(self, buf: &mut B) -> usize where B: bytes::BufMut + AsMut<[u8]> @@ -297,6 +315,7 @@ macro_rules! impl_compact_for_bytes { self.0.to_compact(buf) } + #[inline] fn from_compact(buf: &[u8], len: usize) -> (Self, &[u8]) { let (v, buf) = <[u8; std::mem::size_of::<$name>()]>::from_compact(buf, len); (Self::from(v), buf) @@ -310,6 +329,7 @@ impl_compact_for_bytes!(Address, B256, B512, Bloom); impl Compact for bool { /// `bool` vars go directly to the `StructFlags` and are not written to the buffer. + #[inline] fn to_compact(self, _: &mut B) -> usize where B: bytes::BufMut + AsMut<[u8]>, @@ -318,6 +338,7 @@ impl Compact for bool { } /// `bool` expects the real value to come in `len`, and does not advance the cursor. + #[inline] fn from_compact(buf: &[u8], len: usize) -> (Self, &[u8]) { (len != 0, buf) } @@ -334,19 +355,24 @@ where buf.put_u8(n as u8); } -fn decode_varuint(mut buf: &[u8]) -> (usize, &[u8]) { - let mut value: usize = 0; +fn decode_varuint(buf: &[u8]) -> (usize, &[u8]) { + let mut value = 0; - for i in 0usize..33 { - let byte = buf.get_u8(); - if byte < 128 { - value |= usize::from(byte) << (i * 7); - return (value, buf) - } else { - value |= usize::from(byte & 0x7F) << (i * 7); + for i in 0..33 { + let byte = buf[i]; + value |= usize::from(byte & 0x7F) << (i * 7); + if byte < 0x80 { + return (value, &buf[i + 1..]); } } - panic!("Could not correctly decode value."); + + decode_varuint_panic(); +} + +#[inline(never)] +#[cold] +const fn decode_varuint_panic() -> ! { + panic!("could not decode varuint"); } #[cfg(test)]