diff --git a/ledger/src/shred.rs b/ledger/src/shred.rs index 15e29fee94756b..1c35ea6764515f 100644 --- a/ledger/src/shred.rs +++ b/ledger/src/shred.rs @@ -49,7 +49,6 @@ //! So, given a) - c), we must restrict data shred's payload length such that the entire coding //! payload can fit into one coding shred / packet. -pub(crate) use self::merkle::{MerkleRoot, SIZE_OF_MERKLE_ROOT}; #[cfg(test)] pub(crate) use self::shred_code::MAX_CODE_SHREDS_PER_SLOT; use { @@ -193,7 +192,7 @@ pub enum ShredType { enum ShredVariant { LegacyCode, // 0b0101_1010 LegacyData, // 0b1010_0101 - // proof_size is the number of proof entries in the merkle tree branch. + // proof_size is the number of merkle proof entries. MerkleCode(/*proof_size:*/ u8), // 0b0100_???? MerkleData(/*proof_size:*/ u8), // 0b1000_???? } @@ -233,14 +232,14 @@ pub enum Shred { pub(crate) enum SignedData<'a> { Chunk(&'a [u8]), // Chunk of payload past signature. - MerkleRoot(MerkleRoot), + MerkleRoot(Hash), } impl<'a> AsRef<[u8]> for SignedData<'a> { fn as_ref(&self) -> &[u8] { match self { Self::Chunk(chunk) => chunk, - Self::MerkleRoot(root) => root, + Self::MerkleRoot(root) => root.as_ref(), } } } @@ -660,7 +659,7 @@ pub mod layout { ShredVariant::LegacyCode | ShredVariant::LegacyData => { let offsets = self::legacy::SIGNED_MESSAGE_OFFSETS; (offsets.end <= shred.len()).then_some(offsets) - }, + } // Merkle shreds sign merkle tree root which can be recovered from // the merkle proof embedded in the payload but itself is not // stored the payload. @@ -681,7 +680,7 @@ pub mod layout { Ok(flags & ShredFlags::SHRED_TICK_REFERENCE_MASK.bits()) } - pub(crate) fn get_merkle_root(shred: &[u8]) -> Option { + pub(crate) fn get_merkle_root(shred: &[u8]) -> Option { match get_shred_variant(shred).ok()? { ShredVariant::LegacyCode | ShredVariant::LegacyData => None, ShredVariant::MerkleCode(proof_size) => { diff --git a/ledger/src/shred/merkle.rs b/ledger/src/shred/merkle.rs index 447b20ad399a0b..5da49428990c1c 100644 --- a/ledger/src/shred/merkle.rs +++ b/ledger/src/shred/merkle.rs @@ -34,8 +34,6 @@ use { }, }; -const_assert_eq!(SIZE_OF_MERKLE_ROOT, 20); -pub(crate) const SIZE_OF_MERKLE_ROOT: usize = std::mem::size_of::(); const_assert_eq!(SIZE_OF_MERKLE_PROOF_ENTRY, 20); const SIZE_OF_MERKLE_PROOF_ENTRY: usize = std::mem::size_of::(); const_assert_eq!(ShredData::SIZE_OF_PAYLOAD, 1203); @@ -45,11 +43,10 @@ const_assert_eq!(ShredData::SIZE_OF_PAYLOAD, 1203); const MERKLE_HASH_PREFIX_LEAF: &[u8] = &[0x00]; const MERKLE_HASH_PREFIX_NODE: &[u8] = &[0x01]; -pub(crate) type MerkleRoot = MerkleProofEntry; type MerkleProofEntry = [u8; 20]; -// Layout: {common, data} headers | data buffer | merkle branch -// The slice past signature and before merkle branch is erasure coded. +// Layout: {common, data} headers | data buffer | merkle proof +// The slice past signature and before the merkle proof is erasure coded. // Same slice is hashed to generate merkle tree. // The root of merkle tree is signed. #[derive(Clone, Debug, Eq, PartialEq)] @@ -59,8 +56,8 @@ pub struct ShredData { payload: Vec, } -// Layout: {common, coding} headers | erasure coded shard | merkle branch -// The slice past signature and before merkle branch is hashed to generate +// Layout: {common, coding} headers | erasure coded shard | merkle proof +// The slice past signature and before the merkle proof is hashed to generate // merkle tree. The root of merkle tree is signed. #[derive(Clone, Debug, Eq, PartialEq)] pub struct ShredCode { @@ -75,22 +72,23 @@ pub(super) enum Shred { ShredData(ShredData), } -struct MerkleBranch<'a> { - root: &'a MerkleRoot, - proof: Vec<&'a MerkleProofEntry>, -} - impl Shred { dispatch!(fn common_header(&self) -> &ShredCommonHeader); dispatch!(fn erasure_shard_as_slice(&self) -> Result<&[u8], Error>); dispatch!(fn erasure_shard_index(&self) -> Result); - dispatch!(fn merkle_root(&self) -> Result<&MerkleRoot, Error>); dispatch!(fn merkle_tree_node(&self) -> Result); dispatch!(fn payload(&self) -> &Vec); - dispatch!(fn sanitize(&self, verify_merkle_proof: bool) -> Result<(), Error>); - dispatch!(fn set_merkle_branch(&mut self, merkle_branch: &MerkleBranch) -> Result<(), Error>); + dispatch!(fn sanitize(&self) -> Result<(), Error>); + dispatch!(fn set_merkle_proof(&mut self, proof: &[&MerkleProofEntry]) -> Result<(), Error>); dispatch!(fn set_signature(&mut self, signature: Signature)); - dispatch!(fn signed_data(&self) -> Result); + dispatch!(fn signed_data(&self) -> Result); + + fn merkle_proof(&self) -> Result, Error> { + match self { + Self::ShredCode(shred) => shred.merkle_proof().map(Either::Left), + Self::ShredData(shred) => shred.merkle_proof().map(Either::Right), + } + } #[must_use] fn verify(&self, pubkey: &Pubkey) -> bool { @@ -115,6 +113,8 @@ impl Shred { #[cfg(test)] impl Shred { + dispatch!(fn merkle_root(&self) -> Result); + fn index(&self) -> u32 { self.common_header().index } @@ -125,7 +125,7 @@ impl Shred { } impl ShredData { - // proof_size is the number of proof entries in the merkle tree branch. + // proof_size is the number of merkle proof entries. fn proof_size(&self) -> Result { match self.common_header.shred_variant { ShredVariant::MerkleData(proof_size) => Ok(proof_size), @@ -141,36 +141,23 @@ impl ShredData { pub(super) fn capacity(proof_size: u8) -> Result { Self::SIZE_OF_PAYLOAD .checked_sub( - Self::SIZE_OF_HEADERS - + SIZE_OF_MERKLE_ROOT - + usize::from(proof_size) * SIZE_OF_MERKLE_PROOF_ENTRY, + Self::SIZE_OF_HEADERS + usize::from(proof_size) * SIZE_OF_MERKLE_PROOF_ENTRY, ) .ok_or(Error::InvalidProofSize(proof_size)) } - fn merkle_root(&self) -> Result<&MerkleRoot, Error> { + fn merkle_root(&self) -> Result { let proof_size = self.proof_size()?; let offset = Self::SIZE_OF_HEADERS + Self::capacity(proof_size)?; - let root = self - .payload - .get(offset..offset + SIZE_OF_MERKLE_ROOT) - .ok_or(Error::InvalidPayloadSize(self.payload.len()))?; - Ok(<&MerkleRoot>::try_from(root).unwrap()) + let index = self.erasure_shard_index()?; + let root = get_merkle_root(&self.payload, proof_size, index, offset); + root.ok_or(Error::InvalidMerkleProof) } - fn merkle_branch(&self) -> Result { + fn merkle_proof(&self) -> Result, Error> { let proof_size = self.proof_size()?; let offset = Self::SIZE_OF_HEADERS + Self::capacity(proof_size)?; - let size = SIZE_OF_MERKLE_ROOT + usize::from(proof_size) * SIZE_OF_MERKLE_PROOF_ENTRY; - let merkle_branch = MerkleBranch::try_from( - self.payload - .get(offset..offset + size) - .ok_or(Error::InvalidPayloadSize(self.payload.len()))?, - )?; - if merkle_branch.proof.len() != usize::from(proof_size) { - return Err(Error::InvalidMerkleProof); - } - Ok(merkle_branch) + get_merkle_proof(&self.payload, offset, proof_size) } fn merkle_tree_node(&self) -> Result { @@ -178,12 +165,6 @@ impl ShredData { Ok(hashv(&[MERKLE_HASH_PREFIX_LEAF, chunk])) } - fn verify_merkle_proof(&self) -> Result { - let node = self.merkle_tree_node()?; - let index = self.erasure_shard_index()?; - Ok(verify_merkle_proof(index, node, &self.merkle_branch()?)) - } - fn from_recovered_shard(signature: &Signature, mut shard: Vec) -> Result { let shard_size = shard.len(); if shard_size + SIZE_OF_SIGNATURE > Self::SIZE_OF_PAYLOAD { @@ -209,13 +190,13 @@ impl ShredData { payload: shard, }; // Merkle proof is not erasure coded and is not yet available here. - shred.sanitize(/*verify_merkle_proof:*/ false)?; + shred.sanitize()?; Ok(shred) } - fn set_merkle_branch(&mut self, merkle_branch: &MerkleBranch) -> Result<(), Error> { + fn set_merkle_proof(&mut self, proof: &[&MerkleProofEntry]) -> Result<(), Error> { let proof_size = self.proof_size()?; - if merkle_branch.proof.len() != usize::from(proof_size) { + if proof.len() != usize::from(proof_size) { return Err(Error::InvalidMerkleProof); } let offset = Self::SIZE_OF_HEADERS + Self::capacity(proof_size)?; @@ -224,25 +205,13 @@ impl ShredData { .get_mut(offset..) .ok_or(Error::InvalidProofSize(proof_size))?, ); - bincode::serialize_into(&mut cursor, &merkle_branch.root)?; - for entry in &merkle_branch.proof { + for entry in proof { bincode::serialize_into(&mut cursor, entry)?; } Ok(()) } - fn sanitize(&self, verify_merkle_proof: bool) -> Result<(), Error> { - let shred_variant = self.common_header.shred_variant; - if !matches!(shred_variant, ShredVariant::MerkleData(_)) { - return Err(Error::InvalidShredVariant); - } - if verify_merkle_proof && !self.verify_merkle_proof()? { - return Err(Error::InvalidMerkleProof); - } - shred_data::sanitize(self) - } - - pub(super) fn get_merkle_root(shred: &[u8], proof_size: u8) -> Option { + pub(super) fn get_merkle_root(shred: &[u8], proof_size: u8) -> Option { debug_assert_eq!( shred::layout::get_shred_variant(shred).unwrap(), ShredVariant::MerkleData(proof_size) @@ -257,14 +226,14 @@ impl ShredData { .map(usize::try_from)? .ok()? }; - // Where the merkle branch starts in the shred binary. + // Where the merkle proof starts in the shred binary. let offset = Self::SIZE_OF_HEADERS + Self::capacity(proof_size).ok()?; get_merkle_root(shred, proof_size, index, offset) } } impl ShredCode { - // proof_size is the number of proof entries in the merkle tree branch. + // proof_size is the number of merkle proof entries. fn proof_size(&self) -> Result { match self.common_header.shred_variant { ShredVariant::MerkleCode(proof_size) => Ok(proof_size), @@ -274,40 +243,27 @@ impl ShredCode { // Size of buffer embedding erasure codes. fn capacity(proof_size: u8) -> Result { - // Merkle branch is generated and signed after coding shreds are + // Merkle proof is generated and signed after coding shreds are // generated. Coding shred headers cannot be erasure coded either. Self::SIZE_OF_PAYLOAD .checked_sub( - Self::SIZE_OF_HEADERS - + SIZE_OF_MERKLE_ROOT - + SIZE_OF_MERKLE_PROOF_ENTRY * usize::from(proof_size), + Self::SIZE_OF_HEADERS + SIZE_OF_MERKLE_PROOF_ENTRY * usize::from(proof_size), ) .ok_or(Error::InvalidProofSize(proof_size)) } - fn merkle_root(&self) -> Result<&MerkleRoot, Error> { + fn merkle_root(&self) -> Result { let proof_size = self.proof_size()?; let offset = Self::SIZE_OF_HEADERS + Self::capacity(proof_size)?; - let root = self - .payload - .get(offset..offset + SIZE_OF_MERKLE_ROOT) - .ok_or(Error::InvalidPayloadSize(self.payload.len()))?; - Ok(<&MerkleRoot>::try_from(root).unwrap()) + let index = self.erasure_shard_index()?; + let root = get_merkle_root(&self.payload, proof_size, index, offset); + root.ok_or(Error::InvalidMerkleProof) } - fn merkle_branch(&self) -> Result { + fn merkle_proof(&self) -> Result, Error> { let proof_size = self.proof_size()?; let offset = Self::SIZE_OF_HEADERS + Self::capacity(proof_size)?; - let size = SIZE_OF_MERKLE_ROOT + usize::from(proof_size) * SIZE_OF_MERKLE_PROOF_ENTRY; - let merkle_branch = MerkleBranch::try_from( - self.payload - .get(offset..offset + size) - .ok_or(Error::InvalidPayloadSize(self.payload.len()))?, - )?; - if merkle_branch.proof.len() != usize::from(proof_size) { - return Err(Error::InvalidMerkleProof); - } - Ok(merkle_branch) + get_merkle_proof(&self.payload, offset, proof_size) } fn merkle_tree_node(&self) -> Result { @@ -320,12 +276,6 @@ impl ShredCode { Ok(hashv(&[MERKLE_HASH_PREFIX_LEAF, chunk])) } - fn verify_merkle_proof(&self) -> Result { - let node = self.merkle_tree_node()?; - let index = self.erasure_shard_index()?; - Ok(verify_merkle_proof(index, node, &self.merkle_branch()?)) - } - fn from_recovered_shard( common_header: ShredCommonHeader, coding_header: CodingShredHeader, @@ -353,13 +303,13 @@ impl ShredCode { payload: shard, }; // Merkle proof is not erasure coded and is not yet available here. - shred.sanitize(/*verify_merkle_proof:*/ false)?; + shred.sanitize()?; Ok(shred) } - fn set_merkle_branch(&mut self, merkle_branch: &MerkleBranch) -> Result<(), Error> { + fn set_merkle_proof(&mut self, proof: &[&MerkleProofEntry]) -> Result<(), Error> { let proof_size = self.proof_size()?; - if merkle_branch.proof.len() != usize::from(proof_size) { + if proof.len() != usize::from(proof_size) { return Err(Error::InvalidMerkleProof); } let offset = Self::SIZE_OF_HEADERS + Self::capacity(proof_size)?; @@ -368,25 +318,13 @@ impl ShredCode { .get_mut(offset..) .ok_or(Error::InvalidProofSize(proof_size))?, ); - bincode::serialize_into(&mut cursor, &merkle_branch.root)?; - for entry in &merkle_branch.proof { + for entry in proof { bincode::serialize_into(&mut cursor, entry)?; } Ok(()) } - fn sanitize(&self, verify_merkle_proof: bool) -> Result<(), Error> { - let shred_variant = self.common_header.shred_variant; - if !matches!(shred_variant, ShredVariant::MerkleCode(_)) { - return Err(Error::InvalidShredVariant); - } - if verify_merkle_proof && !self.verify_merkle_proof()? { - return Err(Error::InvalidMerkleProof); - } - shred_code::sanitize(self) - } - - pub(super) fn get_merkle_root(shred: &[u8], proof_size: u8) -> Option { + pub(super) fn get_merkle_root(shred: &[u8], proof_size: u8) -> Option { debug_assert_eq!( shred::layout::get_shred_variant(shred).unwrap(), ShredVariant::MerkleCode(proof_size) @@ -403,21 +341,20 @@ impl ShredCode { .ok()?; num_data_shreds.checked_add(position)? }; - // Where the merkle branch starts in the shred binary. + // Where the merkle proof starts in the shred binary. let offset = Self::SIZE_OF_HEADERS + Self::capacity(proof_size).ok()?; get_merkle_root(shred, proof_size, index, offset) } } impl<'a> ShredTrait<'a> for ShredData { - type SignedData = MerkleRoot; + type SignedData = Hash; impl_shred_common!(); // Also equal to: // ShredData::SIZE_OF_HEADERS // + ShredData::capacity(proof_size).unwrap() - // + SIZE_OF_MERKLE_ROOT // + usize::from(proof_size) * SIZE_OF_MERKLE_PROOF_ENTRY const SIZE_OF_PAYLOAD: usize = ShredCode::SIZE_OF_PAYLOAD - ShredCode::SIZE_OF_HEADERS + SIZE_OF_SIGNATURE; @@ -440,7 +377,7 @@ impl<'a> ShredTrait<'a> for ShredData { data_header, payload, }; - shred.sanitize(/*verify_merkle_proof:*/ true)?; + shred.sanitize()?; Ok(shred) } @@ -475,16 +412,20 @@ impl<'a> ShredTrait<'a> for ShredData { } fn sanitize(&self) -> Result<(), Error> { - self.sanitize(/*verify_merkle_proof:*/ true) + let shred_variant = self.common_header.shred_variant; + if !matches!(shred_variant, ShredVariant::MerkleData(_)) { + return Err(Error::InvalidShredVariant); + } + shred_data::sanitize(self) } fn signed_data(&'a self) -> Result { - self.merkle_root().copied() + self.merkle_root() } } impl<'a> ShredTrait<'a> for ShredCode { - type SignedData = MerkleRoot; + type SignedData = Hash; impl_shred_common!(); const SIZE_OF_PAYLOAD: usize = shred_code::ShredCode::SIZE_OF_PAYLOAD; @@ -507,7 +448,7 @@ impl<'a> ShredTrait<'a> for ShredCode { coding_header, payload, }; - shred.sanitize(/*verify_merkle_proof:*/ true)?; + shred.sanitize()?; Ok(shred) } @@ -542,11 +483,15 @@ impl<'a> ShredTrait<'a> for ShredCode { } fn sanitize(&self) -> Result<(), Error> { - self.sanitize(/*verify_merkle_proof:*/ true) + let shred_variant = self.common_header.shred_variant; + if !matches!(shred_variant, ShredVariant::MerkleCode(_)) { + return Err(Error::InvalidShredVariant); + } + shred_code::sanitize(self) } fn signed_data(&'a self) -> Result { - self.merkle_root().copied() + self.merkle_root() } } @@ -580,23 +525,6 @@ impl ShredCodeTrait for ShredCode { } } -impl<'a> TryFrom<&'a [u8]> for MerkleBranch<'a> { - type Error = Error; - fn try_from(merkle_branch: &'a [u8]) -> Result { - if merkle_branch.len() < SIZE_OF_MERKLE_ROOT { - return Err(Error::InvalidMerkleProof); - } - let (root, proof) = merkle_branch.split_at(SIZE_OF_MERKLE_ROOT); - let root = <&MerkleRoot>::try_from(root).unwrap(); - let proof = proof - .chunks(SIZE_OF_MERKLE_PROOF_ENTRY) - .map(<&MerkleProofEntry>::try_from) - .collect::>() - .map_err(|_| Error::InvalidMerkleProof)?; - Ok(Self { root, proof }) - } -} - // Obtains parent's hash by joining two sibiling nodes in merkle tree. fn join_nodes, T: AsRef<[u8]>>(node: S, other: T) -> Hash { let node = &node.as_ref()[..SIZE_OF_MERKLE_PROOF_ENTRY]; @@ -604,15 +532,9 @@ fn join_nodes, T: AsRef<[u8]>>(node: S, other: T) -> Hash { hashv(&[MERKLE_HASH_PREFIX_NODE, node, other]) } -fn verify_merkle_proof(index: usize, node: Hash, merkle_branch: &MerkleBranch) -> bool { - let proof = merkle_branch.proof.iter().copied(); - let root = fold_merkle_proof(index, node, proof); - root.as_ref() == Some(merkle_branch.root) -} - // Recovers root of the merkle tree from a leaf node // at the given index and the respective proof. -fn fold_merkle_proof<'a, I>(index: usize, node: Hash, proof: I) -> Option +fn fold_merkle_proof<'a, I>(index: usize, node: Hash, proof: I) -> Option where I: IntoIterator, { @@ -626,29 +548,33 @@ where }; (index >> 1, parent) }); - (index == 0).then(|| { - let root = &root.as_ref()[..SIZE_OF_MERKLE_ROOT]; - MerkleRoot::try_from(root).ok() - })? + (index == 0).then_some(root) } fn get_merkle_root( shred: &[u8], proof_size: u8, index: usize, // Shred index in the erasure batch. - offset: usize, // Where the merkle branch starts in the shred binary. -) -> Option { + offset: usize, // Where the merkle proof starts in the shred binary. +) -> Option { let node = shred.get(SIZE_OF_SIGNATURE..offset)?; let node = hashv(&[MERKLE_HASH_PREFIX_LEAF, node]); - // Merkle proof embedded in the payload. - let offset = offset + SIZE_OF_MERKLE_ROOT; - let size = usize::from(proof_size) * SIZE_OF_MERKLE_PROOF_ENTRY; - let proof = shred - .get(offset..offset + size)? + let proof = get_merkle_proof(shred, offset, proof_size).ok()?; + fold_merkle_proof(index, node, proof) +} + +fn get_merkle_proof( + shred: &[u8], + offset: usize, // Where the merkle proof starts. + proof_size: u8, // Number of proof entries. +) -> Result, Error> { + let proof_size = usize::from(proof_size) * SIZE_OF_MERKLE_PROOF_ENTRY; + Ok(shred + .get(offset..offset + proof_size) + .ok_or(Error::InvalidPayloadSize(shred.len()))? .chunks(SIZE_OF_MERKLE_PROOF_ENTRY) .map(<&MerkleProofEntry>::try_from) - .map(Result::unwrap); - fold_merkle_proof(index, node, proof) + .map(Result::unwrap)) } fn make_merkle_tree(mut nodes: Vec) -> Vec { @@ -666,11 +592,11 @@ fn make_merkle_tree(mut nodes: Vec) -> Vec { nodes } -fn make_merkle_branch( +fn make_merkle_proof( mut index: usize, // leaf index ~ shred's erasure shard index. mut size: usize, // number of leaves ~ erasure batch size. tree: &[Hash], -) -> Option { +) -> Option> { if index >= size { return None; } @@ -684,12 +610,7 @@ fn make_merkle_branch( size = (size + 1) >> 1; index >>= 1; } - if offset + 1 != tree.len() { - return None; - } - let root = &tree.last()?.as_ref()[..SIZE_OF_MERKLE_ROOT]; - let root = <&MerkleRoot>::try_from(root).unwrap(); - Some(MerkleBranch { root, proof }) + (offset + 1 == tree.len()).then_some(proof) } pub(super) fn recover( @@ -816,28 +737,25 @@ pub(super) fn recover( } }) .collect::>()?; - // Compute merkle tree and set the merkle branch on the recovered shreds. + // Compute merkle tree and set the merkle proof on the recovered shreds. let nodes: Vec<_> = shreds .iter() .map(Shred::merkle_tree_node) .collect::>()?; let tree = make_merkle_tree(nodes); - let merkle_root = &tree.last().unwrap().as_ref()[..SIZE_OF_MERKLE_ROOT]; - let merkle_root = MerkleRoot::try_from(merkle_root).unwrap(); for (index, (shred, mask)) in shreds.iter_mut().zip(&mask).enumerate() { + let proof = make_merkle_proof(index, num_shards, &tree).ok_or(Error::InvalidMerkleProof)?; + if proof.len() != usize::from(proof_size) { + return Err(Error::InvalidMerkleProof); + } if *mask { - if shred.merkle_root()? != &merkle_root { + if shred.merkle_proof()?.ne(proof) { return Err(Error::InvalidMerkleProof); } } else { - let merkle_branch = - make_merkle_branch(index, num_shards, &tree).ok_or(Error::InvalidMerkleProof)?; - if merkle_branch.proof.len() != usize::from(proof_size) { - return Err(Error::InvalidMerkleProof); - } - shred.set_merkle_branch(&merkle_branch)?; + shred.set_merkle_proof(&proof)?; // Already sanitized in Shred{Code,Data}::from_recovered_shard. - debug_assert_matches!(shred.sanitize(/*verify_merkle_proof:*/ true), Ok(())); + debug_assert_matches!(shred.sanitize(), Ok(())); // Assert that shred payload is fully populated. debug_assert_eq!(shred, { let shred = shred.payload().clone(); @@ -853,7 +771,7 @@ pub(super) fn recover( .collect()) } -// Maps number of (code + data) shreds to MerkleBranch.proof.len(). +// Maps number of (code + data) shreds to merkle_proof.len(). fn get_proof_size(num_shreds: usize) -> u8 { let bits = usize::BITS - num_shreds.leading_zeros(); let proof_size = if num_shreds.is_power_of_two() { @@ -1014,7 +932,7 @@ pub(super) fn make_shreds_from_data( out }) .collect(); - // Generate coding shreds, populate merkle branch + // Generate coding shreds, populate merkle proof // for all shreds and attach signature. let shreds: Result, Error> = if shreds.len() <= 1 { shreds @@ -1039,7 +957,7 @@ pub(super) fn make_shreds_from_data( shreds } -// Generates coding shreds from data shreds, populates merke branch for all +// Generates coding shreds from data shreds, populates merke proof for all // shreds and attaches signature. fn make_erasure_batch( keypair: &Keypair, @@ -1103,18 +1021,17 @@ fn make_erasure_batch( // Sign root of Merkle tree. let signature = { let root = tree.last().ok_or(Error::InvalidMerkleProof)?; - let root = &root.as_ref()[..SIZE_OF_MERKLE_ROOT]; - keypair.sign_message(root) + keypair.sign_message(root.as_ref()) }; - // Populate merkle branch for all shreds and attach signature. + // Populate merkle proof for all shreds and attach signature. for (index, shred) in shreds.iter_mut().enumerate() { - let merkle_branch = make_merkle_branch(index, erasure_batch_size, &tree) - .ok_or(Error::InvalidMerkleProof)?; - debug_assert_eq!(merkle_branch.proof.len(), usize::from(proof_size)); - shred.set_merkle_branch(&merkle_branch)?; + let proof = + make_merkle_proof(index, erasure_batch_size, &tree).ok_or(Error::InvalidMerkleProof)?; + debug_assert_eq!(proof.len(), usize::from(proof_size)); + shred.set_merkle_proof(&proof)?; shred.set_signature(signature); debug_assert!(shred.verify(&keypair.pubkey())); - debug_assert_matches!(shred.sanitize(/*verify_merkle_proof:*/ true), Ok(())); + debug_assert_matches!(shred.sanitize(), Ok(())); // Assert that shred payload is fully populated. debug_assert_eq!(shred, { let shred = shred.payload().clone(); @@ -1138,16 +1055,15 @@ mod test { test_case::test_case, }; - // Total size of a data shred including headers and merkle branch. + // Total size of a data shred including headers and merkle proof. fn shred_data_size_of_payload(proof_size: u8) -> usize { ShredData::SIZE_OF_HEADERS + ShredData::capacity(proof_size).unwrap() - + SIZE_OF_MERKLE_ROOT + usize::from(proof_size) * SIZE_OF_MERKLE_PROOF_ENTRY } - // Merkle branch is generated and signed after coding shreds are generated. - // All payload excluding merkle branch and the signature are erasure coded. + // Merkle proof is generated and signed after coding shreds are generated. + // All payload excluding merkle proof and the signature are erasure coded. // Therefore the data capacity is equal to erasure encoded shard size minus // size of erasure encoded header. fn shred_data_capacity(proof_size: u8) -> usize { @@ -1159,7 +1075,6 @@ mod test { fn shred_data_size_of_erasure_encoded_slice(proof_size: u8) -> usize { ShredData::SIZE_OF_PAYLOAD - SIZE_OF_SIGNATURE - - SIZE_OF_MERKLE_ROOT - usize::from(proof_size) * SIZE_OF_MERKLE_PROOF_ENTRY } @@ -1208,13 +1123,19 @@ mod test { let nodes = repeat_with(|| rng.gen::<[u8; 32]>()).map(Hash::from); let nodes: Vec<_> = nodes.take(size).collect(); let tree = make_merkle_tree(nodes.clone()); + let root = tree.last().copied().unwrap(); for index in 0..size { - let branch = make_merkle_branch(index, size, &tree).unwrap(); - let root = &tree.last().unwrap().as_ref()[..SIZE_OF_MERKLE_ROOT]; - assert_eq!(branch.root, root); - assert!(verify_merkle_proof(index, nodes[index], &branch)); + // XXX rm .iter().copied() + let proof = make_merkle_proof(index, size, &tree).unwrap(); + assert_eq!( + fold_merkle_proof(index, nodes[index], proof.iter().copied()), + Some(root) + ); for i in (0..size).filter(|&i| i != index) { - assert!(!verify_merkle_proof(i, nodes[i], &branch)); + assert_ne!( + fold_merkle_proof(i, nodes[i], proof.iter().copied()).unwrap(), + root + ); } } } @@ -1335,14 +1256,14 @@ mod test { .unwrap(); let tree = make_merkle_tree(nodes); for (index, shred) in shreds.iter_mut().enumerate() { - let merkle_branch = make_merkle_branch(index, num_shreds, &tree).unwrap(); - assert_eq!(merkle_branch.proof.len(), usize::from(proof_size)); - shred.set_merkle_branch(&merkle_branch).unwrap(); + let proof = make_merkle_proof(index, num_shreds, &tree).unwrap(); + assert_eq!(proof.len(), usize::from(proof_size)); + shred.set_merkle_proof(&proof).unwrap(); let data = shred.signed_data().unwrap(); let signature = keypair.sign_message(data.as_ref()); shred.set_signature(signature); assert!(shred.verify(&keypair.pubkey())); - assert_matches!(shred.sanitize(/*verify_merkle_proof:*/ true), Ok(())); + assert_matches!(shred.sanitize(), Ok(())); } assert_eq!(shreds.iter().map(Shred::signature).dedup().count(), 1); for size in num_data_shreds..num_shreds { @@ -1474,7 +1395,7 @@ mod test { let pubkey = keypair.pubkey(); for shred in &shreds { assert!(shred.verify(&pubkey)); - assert_matches!(shred.sanitize(/*verify_merkle_proof:*/ true), Ok(())); + assert_matches!(shred.sanitize(), Ok(())); let ShredCommonHeader { signature, shred_variant, @@ -1485,7 +1406,7 @@ mod test { } = *shred.common_header(); let shred_type = ShredType::from(shred_variant); let key = ShredId::new(slot, index, shred_type); - let merkle_root = shred.merkle_root().copied().ok(); + let merkle_root = shred.merkle_root().ok(); let shred = shred.payload(); assert_eq!(shred::layout::get_signature(shred), Some(signature)); assert_eq!( diff --git a/ledger/src/shred/shred_data.rs b/ledger/src/shred/shred_data.rs index 915ae29d4e3f2e..9bf2c0bf05f79e 100644 --- a/ledger/src/shred/shred_data.rs +++ b/ledger/src/shred/shred_data.rs @@ -102,8 +102,8 @@ impl ShredData { } // Maximum size of ledger data that can be embedded in a data-shred. - // merkle_proof_size is the number of proof entries in the merkle tree - // branch. None indicates a legacy data-shred. + // merkle_proof_size is the number of merkle proof entries. + // None indicates a legacy data-shred. pub fn capacity(merkle_proof_size: Option) -> Result { match merkle_proof_size { None => Ok(legacy::ShredData::CAPACITY), diff --git a/ledger/src/sigverify_shreds.rs b/ledger/src/sigverify_shreds.rs index 038673548e25e0..9d1a782ea13a80 100644 --- a/ledger/src/sigverify_shreds.rs +++ b/ledger/src/sigverify_shreds.rs @@ -1,6 +1,6 @@ #![allow(clippy::implicit_hasher)] use { - crate::shred::{self, MerkleRoot, SIZE_OF_MERKLE_ROOT}, + crate::shred, itertools::{izip, Itertools}, rayon::{prelude::*, ThreadPool}, sha2::{Digest, Sha512}, @@ -15,13 +15,17 @@ use { solana_rayon_threadlimit::get_thread_count, solana_sdk::{ clock::Slot, + hash::Hash, pubkey::Pubkey, signature::{Keypair, Signature, Signer}, }, + static_assertions::const_assert_eq, std::{collections::HashMap, fmt::Debug, iter::repeat, mem::size_of, ops::Range, sync::Arc}, }; const SIGN_SHRED_GPU_MIN: usize = 256; +const_assert_eq!(SIZE_OF_MERKLE_ROOT, 32); +const SIZE_OF_MERKLE_ROOT: usize = std::mem::size_of::(); lazy_static! { static ref SIGVERIFY_THREAD_POOL: ThreadPool = rayon::ThreadPoolBuilder::new() @@ -153,7 +157,7 @@ fn get_merkle_roots( PinnedVec, // Merkle roots Vec>, // Offsets ) { - let merkle_roots: Vec> = SIGVERIFY_THREAD_POOL.install(|| { + let merkle_roots: Vec> = SIGVERIFY_THREAD_POOL.install(|| { packets .par_iter() .flat_map(|packets| { @@ -179,7 +183,7 @@ fn get_merkle_roots( let root = root?; let offset = size; size += SIZE_OF_MERKLE_ROOT; - buffer[offset..size].copy_from_slice(&root); + buffer[offset..size].copy_from_slice(root.as_ref()); Some(offset) }) .collect()