diff --git a/core/benches/shredder.rs b/core/benches/shredder.rs index 93093b1920fb5d..3c1c3e204c269c 100644 --- a/core/benches/shredder.rs +++ b/core/benches/shredder.rs @@ -4,7 +4,7 @@ extern crate test; use { - rand::seq::SliceRandom, + rand::{seq::SliceRandom, Rng}, raptorq::{Decoder, Encoder}, solana_entry::entry::{create_ticks, Entry}, solana_ledger::shred::{ @@ -50,6 +50,7 @@ fn make_shreds(num_shreds: usize) -> Vec { &Keypair::new(), &entries, true, // is_last_in_slot + None, // chained_merkle_root 0, // next_shred_index 0, // next_code_index false, // merkle_variant @@ -80,12 +81,14 @@ fn bench_shredder_ticks(bencher: &mut Bencher) { let num_ticks = max_ticks_per_n_shreds(1, Some(LEGACY_SHRED_DATA_CAPACITY)) * num_shreds as u64; let entries = create_ticks(num_ticks, 0, Hash::default()); let reed_solomon_cache = ReedSolomonCache::default(); + let chained_merkle_root = Some(Hash::new_from_array(rand::thread_rng().gen())); bencher.iter(|| { let shredder = Shredder::new(1, 0, 0, 0).unwrap(); shredder.entries_to_shreds( &kp, &entries, true, + chained_merkle_root, 0, 0, true, // merkle_variant @@ -107,6 +110,7 @@ fn bench_shredder_large_entries(bencher: &mut Bencher) { Some(shred_size), ); let entries = make_large_unchained_entries(txs_per_entry, num_entries); + let chained_merkle_root = Some(Hash::new_from_array(rand::thread_rng().gen())); let reed_solomon_cache = ReedSolomonCache::default(); // 1Mb bencher.iter(|| { @@ -115,6 +119,7 @@ fn bench_shredder_large_entries(bencher: &mut Bencher) { &kp, &entries, true, + chained_merkle_root, 0, 0, true, // merkle_variant @@ -133,10 +138,12 @@ fn bench_deshredder(bencher: &mut Bencher) { let num_ticks = max_ticks_per_n_shreds(1, Some(shred_size)) * num_shreds as u64; let entries = create_ticks(num_ticks, 0, Hash::default()); let shredder = Shredder::new(1, 0, 0, 0).unwrap(); + let chained_merkle_root = Some(Hash::new_from_array(rand::thread_rng().gen())); let (data_shreds, _) = shredder.entries_to_shreds( &kp, &entries, true, + chained_merkle_root, 0, 0, true, // merkle_variant diff --git a/core/src/window_service.rs b/core/src/window_service.rs index aa801b7ebd37f2..504776db1e1a25 100644 --- a/core/src/window_service.rs +++ b/core/src/window_service.rs @@ -582,6 +582,7 @@ mod test { keypair, entries, true, // is_last_in_slot + None, // chained_merkle_root 0, // next_shred_index 0, // next_code_index true, // merkle_variant diff --git a/gossip/src/duplicate_shred.rs b/gossip/src/duplicate_shred.rs index 85f4f4fa0cf149..84c50ea602e8c8 100644 --- a/gossip/src/duplicate_shred.rs +++ b/gossip/src/duplicate_shred.rs @@ -409,6 +409,8 @@ pub(crate) mod tests { keypair, &entries, is_last_in_slot, + // chained_merkle_root + Some(Hash::new_from_array(rng.gen())), next_shred_index, next_code_index, // next_code_index merkle_variant, diff --git a/ledger/src/blockstore.rs b/ledger/src/blockstore.rs index 5b1dc475b9cf04..45c1cbf49bdb2c 100644 --- a/ledger/src/blockstore.rs +++ b/ledger/src/blockstore.rs @@ -29,6 +29,7 @@ use { crossbeam_channel::{bounded, Receiver, Sender, TrySendError}, dashmap::DashSet, log::*, + rand::Rng, rayon::{ iter::{IntoParallelIterator, IntoParallelRefIterator, ParallelIterator}, ThreadPool, @@ -1956,6 +1957,7 @@ impl Blockstore { let mut all_shreds = vec![]; let mut slot_entries = vec![]; let reed_solomon_cache = ReedSolomonCache::default(); + let mut chained_merkle_root = Some(Hash::new_from_array(rand::thread_rng().gen())); // Find all the entries for start_slot for entry in entries.into_iter() { if remaining_ticks_in_slot == 0 { @@ -1973,7 +1975,8 @@ impl Blockstore { let (mut data_shreds, mut coding_shreds) = shredder.entries_to_shreds( keypair, ¤t_entries, - true, // is_last_in_slot + true, // is_last_in_slot + chained_merkle_root, start_index, // next_shred_index start_index, // next_code_index true, // merkle_variant @@ -1982,6 +1985,7 @@ impl Blockstore { ); all_shreds.append(&mut data_shreds); all_shreds.append(&mut coding_shreds); + chained_merkle_root = Some(coding_shreds.last().unwrap().merkle_root().unwrap()); shredder = Shredder::new( current_slot, parent_slot, @@ -2002,6 +2006,7 @@ impl Blockstore { keypair, &slot_entries, is_full_slot, + chained_merkle_root, 0, // next_shred_index 0, // next_code_index true, // merkle_variant @@ -4285,6 +4290,8 @@ pub fn create_new_ledger( &Keypair::new(), &entries, true, // is_last_in_slot + // chained_merkle_root + Some(Hash::new_from_array(rand::thread_rng().gen())), 0, // next_shred_index 0, // next_code_index true, // merkle_variant @@ -4546,6 +4553,8 @@ pub fn entries_to_test_shreds( &Keypair::new(), entries, is_full_slot, + // chained_merkle_root + Some(Hash::new_from_array(rand::thread_rng().gen())), 0, // next_shred_index, 0, // next_code_index merkle_variant, @@ -4806,6 +4815,7 @@ pub mod tests { InnerInstruction, InnerInstructions, Reward, Rewards, TransactionTokenBalance, }, std::{cmp::Ordering, thread::Builder, time::Duration}, + test_case::test_case, }; // used for tests only @@ -7434,7 +7444,7 @@ pub mod tests { #[test] fn test_insert_multiple_is_last() { solana_logger::setup(); - let (shreds, _) = make_slot_entries(0, 0, 20, /*merkle_variant:*/ true); + let (shreds, _) = make_slot_entries(0, 0, 19, /*merkle_variant:*/ true); let num_shreds = shreds.len() as u64; let ledger_path = get_tmp_ledger_path_auto_delete!(); let blockstore = Blockstore::open(ledger_path.path()).unwrap(); @@ -7448,6 +7458,7 @@ pub mod tests { assert!(slot_meta.is_full()); let (shreds, _) = make_slot_entries(0, 0, 22, /*merkle_variant:*/ true); + assert!(shreds.len() > num_shreds as usize); blockstore.insert_shreds(shreds, None, false).unwrap(); let slot_meta = blockstore.meta(0).unwrap().unwrap(); @@ -9863,7 +9874,9 @@ pub mod tests { let (data_shreds, coding_shreds) = shredder.entries_to_shreds( &leader_keypair, &entries, - true, // is_last_in_slot + true, // is_last_in_slot + // chained_merkle_root + Some(Hash::new_from_array(rand::thread_rng().gen())), fec_set_index, // next_shred_index fec_set_index, // next_code_index true, // merkle_variant @@ -9916,18 +9929,21 @@ pub mod tests { assert_eq!(num_coding_in_index, num_coding); } - #[test] - fn test_duplicate_slot() { + #[test_case(false)] + #[test_case(true)] + fn test_duplicate_slot(chained: bool) { let slot = 0; let entries1 = make_slot_entries_with_transactions(1); let entries2 = make_slot_entries_with_transactions(1); let leader_keypair = Arc::new(Keypair::new()); let reed_solomon_cache = ReedSolomonCache::default(); let shredder = Shredder::new(slot, 0, 0, 0).unwrap(); + let chained_merkle_root = chained.then(|| Hash::new_from_array(rand::thread_rng().gen())); let (shreds, _) = shredder.entries_to_shreds( &leader_keypair, &entries1, true, // is_last_in_slot + chained_merkle_root, 0, // next_shred_index 0, // next_code_index, true, // merkle_variant @@ -9938,6 +9954,7 @@ pub mod tests { &leader_keypair, &entries2, true, // is_last_in_slot + chained_merkle_root, 0, // next_shred_index 0, // next_code_index true, // merkle_variant @@ -10323,7 +10340,11 @@ pub mod tests { let num_unique_entries = max_ticks_per_n_shreds(1, None) + 1; let (mut original_shreds, original_entries) = make_slot_entries(0, 0, num_unique_entries, /*merkle_variant:*/ true); - + let mut duplicate_shreds = original_shreds.clone(); + // Mutate signature so that payloads are not the same as the originals. + for shred in &mut duplicate_shreds { + shred.sign(&Keypair::new()); + } // Discard first shred, so that the slot is not full assert!(original_shreds.len() > 1); let last_index = original_shreds.last().unwrap().index() as u64; @@ -10345,14 +10366,6 @@ pub mod tests { assert!(!blockstore.is_full(0)); } - let duplicate_shreds = entries_to_test_shreds( - &original_entries, - 0, // slot - 0, // parent_slot - true, // is_full_slot - 0, // version - true, // merkle_variant - ); let num_shreds = duplicate_shreds.len() as u64; blockstore .insert_shreds(duplicate_shreds, None, false) diff --git a/ledger/src/shredder.rs b/ledger/src/shredder.rs index 07a0fe0ae5b41b..ba127ef009c7a5 100644 --- a/ledger/src/shredder.rs +++ b/ledger/src/shredder.rs @@ -13,7 +13,7 @@ use { solana_entry::entry::Entry, solana_measure::measure::Measure, solana_rayon_threadlimit::get_thread_count, - solana_sdk::{clock::Slot, signature::Keypair}, + solana_sdk::{clock::Slot, hash::Hash, signature::Keypair}, std::{ borrow::Borrow, fmt::Debug, @@ -69,11 +69,13 @@ impl Shredder { } } + #[allow(clippy::too_many_arguments)] pub fn entries_to_shreds( &self, keypair: &Keypair, entries: &[Entry], is_last_in_slot: bool, + chained_merkle_root: Option, next_shred_index: u32, next_code_index: u32, merkle_variant: bool, @@ -93,7 +95,7 @@ impl Shredder { self.version, self.reference_tick, is_last_in_slot, - None, // chained_merkle_root + chained_merkle_root, next_shred_index, next_code_index, reed_solomon_cache, @@ -500,6 +502,7 @@ mod tests { system_transaction, }, std::{collections::HashSet, convert::TryInto, iter::repeat_with, sync::Arc}, + test_case::test_case, }; fn verify_test_code_shred(shred: &Shred, index: u32, slot: Slot, pk: &Pubkey, verify: bool) { @@ -510,7 +513,7 @@ mod tests { assert_eq!(verify, shred.verify(pk)); } - fn run_test_data_shredder(slot: Slot) { + fn run_test_data_shredder(slot: Slot, chained: bool) { let keypair = Arc::new(Keypair::new()); // Test that parent cannot be > current slot @@ -548,6 +551,8 @@ mod tests { &keypair, &entries, is_last_in_slot, + // chained_merkle_root + chained.then(|| Hash::new_from_array(rand::thread_rng().gen())), start_index, // next_shred_index start_index, // next_code_index true, // merkle_variant @@ -602,13 +607,15 @@ mod tests { assert_eq!(entries, deshred_entries); } - #[test] - fn test_data_shredder() { - run_test_data_shredder(0x1234_5678_9abc_def0); + #[test_case(false)] + #[test_case(true)] + fn test_data_shredder(chained: bool) { + run_test_data_shredder(0x1234_5678_9abc_def0, chained); } - #[test] - fn test_deserialize_shred_payload() { + #[test_case(false)] + #[test_case(true)] + fn test_deserialize_shred_payload(chained: bool) { let keypair = Arc::new(Keypair::new()); let slot = 1; let parent_slot = 0; @@ -627,6 +634,8 @@ mod tests { &keypair, &entries, true, // is_last_in_slot + // chained_merkle_root + chained.then(|| Hash::new_from_array(rand::thread_rng().gen())), 0, // next_shred_index 0, // next_code_index true, // merkle_variant @@ -639,8 +648,9 @@ mod tests { assert_eq!(deserialized_shred, *data_shreds.last().unwrap()); } - #[test] - fn test_shred_reference_tick() { + #[test_case(false)] + #[test_case(true)] + fn test_shred_reference_tick(chained: bool) { let keypair = Arc::new(Keypair::new()); let slot = 1; let parent_slot = 0; @@ -659,6 +669,8 @@ mod tests { &keypair, &entries, true, // is_last_in_slot + // chained_merkle_root, + chained.then(|| Hash::new_from_array(rand::thread_rng().gen())), 0, // next_shred_index 0, // next_code_index true, // merkle_variant @@ -676,8 +688,9 @@ mod tests { assert_eq!(deserialized_shred.reference_tick(), 5); } - #[test] - fn test_shred_reference_tick_overflow() { + #[test_case(false)] + #[test_case(true)] + fn test_shred_reference_tick_overflow(chained: bool) { let keypair = Arc::new(Keypair::new()); let slot = 1; let parent_slot = 0; @@ -696,6 +709,8 @@ mod tests { &keypair, &entries, true, // is_last_in_slot + // chained_merkle_root + chained.then(|| Hash::new_from_array(rand::thread_rng().gen())), 0, // next_shred_index 0, // next_code_index true, // merkle_variant @@ -722,7 +737,7 @@ mod tests { ); } - fn run_test_data_and_code_shredder(slot: Slot) { + fn run_test_data_and_code_shredder(slot: Slot, chained: bool) { let keypair = Arc::new(Keypair::new()); let shredder = Shredder::new(slot, slot - 5, 0, 0).unwrap(); // Create enough entries to make > 1 shred @@ -742,6 +757,8 @@ mod tests { &keypair, &entries, true, // is_last_in_slot + // chained_merkle_root + chained.then(|| Hash::new_from_array(rand::thread_rng().gen())), 0, // next_shred_index 0, // next_code_index true, // merkle_variant @@ -766,9 +783,10 @@ mod tests { } } - #[test] - fn test_data_and_code_shredder() { - run_test_data_and_code_shredder(0x1234_5678_9abc_def0); + #[test_case(false)] + #[test_case(true)] + fn test_data_and_code_shredder(chained: bool) { + run_test_data_and_code_shredder(0x1234_5678_9abc_def0, chained); } fn run_test_recovery_and_reassembly(slot: Slot, is_last_in_slot: bool) { @@ -799,6 +817,7 @@ mod tests { &keypair, &entries, is_last_in_slot, + None, // chained_merkle_root 0, // next_shred_index 0, // next_code_index false, // merkle_variant @@ -936,6 +955,7 @@ mod tests { &keypair, &entries, true, // is_last_in_slot + None, // chained_merkle_root 25, // next_shred_index, 25, // next_code_index false, // merkle_variant @@ -1032,6 +1052,7 @@ mod tests { &keypair, &[entry], is_last_in_slot, + None, // chained_merkle_root next_shred_index, next_shred_index, // next_code_index false, // merkle_variant @@ -1073,8 +1094,9 @@ mod tests { } } - #[test] - fn test_shred_version() { + #[test_case(false)] + #[test_case(true)] + fn test_shred_version(chained: bool) { let keypair = Arc::new(Keypair::new()); let hash = hash(Hash::default().as_ref()); let version = shred_version::version_from_hash(&hash); @@ -1094,6 +1116,8 @@ mod tests { &keypair, &entries, true, // is_last_in_slot + // chained_merkle_root + chained.then(|| Hash::new_from_array(rand::thread_rng().gen())), 0, // next_shred_index 0, // next_code_index true, // merkle_variant @@ -1106,8 +1130,9 @@ mod tests { .any(|s| s.version() != version)); } - #[test] - fn test_shred_fec_set_index() { + #[test_case(false)] + #[test_case(true)] + fn test_shred_fec_set_index(chained: bool) { let keypair = Arc::new(Keypair::new()); let hash = hash(Hash::default().as_ref()); let version = shred_version::version_from_hash(&hash); @@ -1127,7 +1152,9 @@ mod tests { let (data_shreds, coding_shreds) = shredder.entries_to_shreds( &keypair, &entries, - true, // is_last_in_slot + true, // is_last_in_slot + // chained_merkle_root + chained.then(|| Hash::new_from_array(rand::thread_rng().gen())), start_index, // next_shred_index start_index, // next_code_index true, // merkle_variant diff --git a/ledger/src/sigverify_shreds.rs b/ledger/src/sigverify_shreds.rs index d52af07bf2cf46..f6d060d686757d 100644 --- a/ledger/src/sigverify_shreds.rs +++ b/ledger/src/sigverify_shreds.rs @@ -739,7 +739,9 @@ mod tests { .entries_to_shreds( keypair, &make_entries(rng, num_entries), - rng.gen(), // is_last_in_slot + rng.gen(), // is_last_in_slot + // chained_merkle_root + rng.gen::().then(|| Hash::new_from_array(rng.gen())), rng.gen_range(0..2671), // next_shred_index rng.gen_range(0..2781), // next_code_index rng.gen(), // merkle_variant, diff --git a/ledger/tests/shred.rs b/ledger/tests/shred.rs index 78cdb28d0b39ae..3c2a6771b635cd 100644 --- a/ledger/tests/shred.rs +++ b/ledger/tests/shred.rs @@ -53,6 +53,7 @@ fn test_multi_fec_block_coding() { &keypair, &entries, true, // is_last_in_slot + None, // chained_merkle_root 0, // next_shred_index 0, // next_code_index false, // merkle_variant @@ -226,6 +227,7 @@ fn setup_different_sized_fec_blocks( &keypair, &entries, is_last, + None, // chained_merkle_root next_shred_index, next_code_index, false, // merkle_variant diff --git a/local-cluster/tests/local_cluster.rs b/local-cluster/tests/local_cluster.rs index 1e15d6f79cc584..55c032ba881735 100644 --- a/local-cluster/tests/local_cluster.rs +++ b/local-cluster/tests/local_cluster.rs @@ -5602,6 +5602,7 @@ fn test_invalid_forks_persisted_on_restart() { &majority_keypair, &entries, true, // is_full_slot + None, // chained_merkle_root 0, // next_shred_index, 0, // next_code_index false, // merkle_variant diff --git a/turbine/benches/retransmit_stage.rs b/turbine/benches/retransmit_stage.rs index bfd68239feedab..c5490d5670e6c6 100644 --- a/turbine/benches/retransmit_stage.rs +++ b/turbine/benches/retransmit_stage.rs @@ -6,6 +6,7 @@ extern crate test; use { crossbeam_channel::unbounded, log::*, + rand::Rng, solana_entry::entry::Entry, solana_gossip::{ cluster_info::{ClusterInfo, Node}, @@ -105,6 +106,8 @@ fn bench_retransmitter(bencher: &mut Bencher) { &keypair, &entries, true, // is_last_in_slot + // chained_merkle_root + Some(Hash::new_from_array(rand::thread_rng().gen())), 0, // next_shred_index 0, // next_code_index true, // merkle_variant diff --git a/turbine/src/broadcast_stage.rs b/turbine/src/broadcast_stage.rs index 98566dfa24bc48..d799c0d9b62005 100644 --- a/turbine/src/broadcast_stage.rs +++ b/turbine/src/broadcast_stage.rs @@ -503,6 +503,7 @@ pub mod test { use { super::*, crossbeam_channel::unbounded, + rand::Rng, solana_entry::entry::create_ticks, solana_gossip::cluster_info::{ClusterInfo, Node}, solana_ledger::{ @@ -544,6 +545,8 @@ pub mod test { &Keypair::new(), &entries, true, // is_last_in_slot + // chained_merkle_root + Some(Hash::new_from_array(rand::thread_rng().gen())), 0, // next_shred_index, 0, // next_code_index true, // merkle_variant diff --git a/turbine/src/broadcast_stage/broadcast_duplicates_run.rs b/turbine/src/broadcast_stage/broadcast_duplicates_run.rs index bae5945aea0e13..8bee47068ac499 100644 --- a/turbine/src/broadcast_stage/broadcast_duplicates_run.rs +++ b/turbine/src/broadcast_stage/broadcast_duplicates_run.rs @@ -173,6 +173,7 @@ impl BroadcastRun for BroadcastDuplicatesRun { keypair, &receive_results.entries, last_tick_height == bank.max_tick_height() && last_entries.is_none(), + None, // chained_merkle_root self.next_shred_index, self.next_code_index, false, // merkle_variant @@ -190,6 +191,7 @@ impl BroadcastRun for BroadcastDuplicatesRun { keypair, &[original_last_entry], true, + None, // chained_merkle_root self.next_shred_index, self.next_code_index, false, // merkle_variant @@ -203,6 +205,7 @@ impl BroadcastRun for BroadcastDuplicatesRun { keypair, &duplicate_extra_last_entries, true, + None, // chained_merkle_root self.next_shred_index, self.next_code_index, false, // merkle_variant diff --git a/turbine/src/broadcast_stage/broadcast_fake_shreds_run.rs b/turbine/src/broadcast_stage/broadcast_fake_shreds_run.rs index 1464d46493d730..20d141dee01a73 100644 --- a/turbine/src/broadcast_stage/broadcast_fake_shreds_run.rs +++ b/turbine/src/broadcast_stage/broadcast_fake_shreds_run.rs @@ -60,6 +60,7 @@ impl BroadcastRun for BroadcastFakeShredsRun { keypair, &receive_results.entries, last_tick_height == bank.max_tick_height(), + None, // chained_merkle_root next_shred_index, self.next_code_index, true, // merkle_variant @@ -81,6 +82,7 @@ impl BroadcastRun for BroadcastFakeShredsRun { keypair, &fake_entries, last_tick_height == bank.max_tick_height(), + None, // chained_merkle_root next_shred_index, self.next_code_index, true, // merkle_variant diff --git a/turbine/src/broadcast_stage/fail_entry_verification_broadcast_run.rs b/turbine/src/broadcast_stage/fail_entry_verification_broadcast_run.rs index 1dda981e693218..b98972690c78a8 100644 --- a/turbine/src/broadcast_stage/fail_entry_verification_broadcast_run.rs +++ b/turbine/src/broadcast_stage/fail_entry_verification_broadcast_run.rs @@ -92,6 +92,7 @@ impl BroadcastRun for FailEntryVerificationBroadcastRun { keypair, &receive_results.entries, last_tick_height == bank.max_tick_height() && last_entries.is_none(), + None, // chained_merkle_root self.next_shred_index, self.next_code_index, true, // merkle_variant @@ -108,6 +109,7 @@ impl BroadcastRun for FailEntryVerificationBroadcastRun { keypair, &[good_last_entry], true, + None, // chained_merkle_root self.next_shred_index, self.next_code_index, true, // merkle_variant @@ -121,6 +123,7 @@ impl BroadcastRun for FailEntryVerificationBroadcastRun { keypair, &[bad_last_entry], false, + None, // chained_merkle_root self.next_shred_index, self.next_code_index, true, // merkle_variant diff --git a/turbine/src/broadcast_stage/standard_broadcast_run.rs b/turbine/src/broadcast_stage/standard_broadcast_run.rs index 82bd7f940c508d..e2b8871b4bc3c2 100644 --- a/turbine/src/broadcast_stage/standard_broadcast_run.rs +++ b/turbine/src/broadcast_stage/standard_broadcast_run.rs @@ -85,6 +85,7 @@ impl StandardBroadcastRun { keypair, &[], // entries true, // is_last_in_slot, + None, // chained_merkle_root state.next_shred_index, state.next_code_index, true, // merkle_variant @@ -143,6 +144,7 @@ impl StandardBroadcastRun { keypair, entries, is_slot_end, + None, // chained_merkle_root next_shred_index, next_code_index, true, // merkle_variant