From aa05693d4c2a51a17a2ce51a73ea8e992c5e097b Mon Sep 17 00:00:00 2001 From: Tyera Eulberg Date: Mon, 25 Sep 2023 17:13:24 -0600 Subject: [PATCH 01/57] Add helper trait for column key deprecation --- ledger/src/blockstore_db.rs | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/ledger/src/blockstore_db.rs b/ledger/src/blockstore_db.rs index 3fd33fa12acea3..6b8e3f175e1f1c 100644 --- a/ledger/src/blockstore_db.rs +++ b/ledger/src/blockstore_db.rs @@ -739,6 +739,36 @@ impl Column for T { } } +pub enum IndexError { + UnpackError, +} + +/// Helper trait to transition primary indexes out from the columns that are using them. +pub trait ColumnIndexDeprecation: Column { + const DEPRECATED_INDEX_LEN: usize; + const CURRENT_INDEX_LEN: usize; + type DeprecatedIndex; + + fn deprecated_key(index: Self::DeprecatedIndex) -> Vec; + fn try_deprecated_index(key: &[u8]) -> std::result::Result; + + fn try_current_index(key: &[u8]) -> std::result::Result; + fn convert_index(deprecated_index: Self::DeprecatedIndex) -> Self::Index; + + fn index(key: &[u8]) -> Self::Index { + if let Ok(index) = Self::try_current_index(key) { + index + } else if let Ok(index) = Self::try_deprecated_index(key) { + Self::convert_index(index) + } else { + // Way back in the day, we broke the TransactionStatus column key. This fallback + // preserves the existing logic for ancient keys, but realistically should never be + // executed. + Self::as_index(0) + } + } +} + impl Column for columns::TransactionStatus { type Index = (u64, Signature, Slot); From 65605a30a948be7e7613320b1ad3f12a58ffaf84 Mon Sep 17 00:00:00 2001 From: Tyera Eulberg Date: Mon, 25 Sep 2023 18:34:22 -0600 Subject: [PATCH 02/57] Add WriteBatch::delete_raw --- ledger/src/blockstore_db.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ledger/src/blockstore_db.rs b/ledger/src/blockstore_db.rs index 6b8e3f175e1f1c..caf528f7692420 100644 --- a/ledger/src/blockstore_db.rs +++ b/ledger/src/blockstore_db.rs @@ -1615,7 +1615,11 @@ impl<'a> WriteBatch<'a> { } pub fn delete(&mut self, key: C::Index) -> Result<()> { - self.write_batch.delete_cf(self.get_cf::(), C::key(key)); + self.delete_raw::(&C::key(key)) + } + + pub(crate) fn delete_raw(&mut self, key: &[u8]) -> Result<()> { + self.write_batch.delete_cf(self.get_cf::(), key); Ok(()) } From d7b9b77280a3596a0436dd9ac5c8be9adc8caac3 Mon Sep 17 00:00:00 2001 From: Tyera Eulberg Date: Mon, 25 Sep 2023 19:11:35 -0600 Subject: [PATCH 03/57] Add ProtobufColumn::get_raw_protobuf_or_bincode --- ledger/src/blockstore_db.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/ledger/src/blockstore_db.rs b/ledger/src/blockstore_db.rs index caf528f7692420..4e40c34182094c 100644 --- a/ledger/src/blockstore_db.rs +++ b/ledger/src/blockstore_db.rs @@ -1537,12 +1537,19 @@ where pub fn get_protobuf_or_bincode>( &self, key: C::Index, + ) -> Result> { + self.get_raw_protobuf_or_bincode::(&C::key(key)) + } + + pub(crate) fn get_raw_protobuf_or_bincode>( + &self, + key: &[u8], ) -> Result> { let is_perf_enabled = maybe_enable_rocksdb_perf( self.column_options.rocks_perf_sample_interval, &self.read_perf_status, ); - let result = self.backend.get_pinned_cf(self.handle(), &C::key(key)); + let result = self.backend.get_pinned_cf(self.handle(), key); if let Some(op_start_instant) = is_perf_enabled { report_rocksdb_read_perf( C::NAME, From cd622b94c62cc7b7f5b5ac0be0d7beeee77d1184 Mon Sep 17 00:00:00 2001 From: Tyera Eulberg Date: Mon, 25 Sep 2023 19:45:10 -0600 Subject: [PATCH 04/57] Add ColumnIndexDeprecation iterator methods --- ledger/src/blockstore_db.rs | 56 +++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/ledger/src/blockstore_db.rs b/ledger/src/blockstore_db.rs index 4e40c34182094c..89daabee5e055b 100644 --- a/ledger/src/blockstore_db.rs +++ b/ledger/src/blockstore_db.rs @@ -614,6 +614,23 @@ impl Rocks { self.db.iterator_cf(cf, iterator_mode) } + fn iterator_cf_raw_key( + &self, + cf: &ColumnFamily, + iterator_mode: IteratorMode>, + ) -> DBIterator { + let start_key; + let iterator_mode = match iterator_mode { + IteratorMode::From(start_from, direction) => { + start_key = start_from; + RocksIteratorMode::From(&start_key, direction) + } + IteratorMode::Start => RocksIteratorMode::Start, + IteratorMode::End => RocksIteratorMode::End, + }; + self.db.iterator_cf(cf, iterator_mode) + } + fn raw_iterator_cf(&self, cf: &ColumnFamily) -> DBRawIterator { self.db.raw_iterator_cf(cf) } @@ -1614,6 +1631,45 @@ where } } +impl LedgerColumn +where + C: ColumnIndexDeprecation + ColumnName, +{ + pub(crate) fn iter_filtered( + &self, + iterator_mode: IteratorMode, + ) -> Result)> + '_> { + let cf = self.handle(); + let iter = self.backend.iterator_cf::(cf, iterator_mode); + Ok(iter.filter_map(|pair| { + let (key, value) = pair.unwrap(); + C::try_current_index(&key).ok().map(|index| (index, value)) + })) + } + + pub(crate) fn iter_deprecated_index_filtered( + &self, + iterator_mode: IteratorMode, + ) -> Result)> + '_> { + let cf = self.handle(); + let iterator_mode_raw_key = match iterator_mode { + IteratorMode::Start => IteratorMode::Start, + IteratorMode::End => IteratorMode::End, + IteratorMode::From(start_from, direction) => { + let raw_key = C::deprecated_key(start_from); + IteratorMode::From(raw_key, direction) + } + }; + let iter = self.backend.iterator_cf_raw_key(cf, iterator_mode_raw_key); + Ok(iter.filter_map(|pair| { + let (key, value) = pair.unwrap(); + C::try_deprecated_index(&key) + .ok() + .map(|index| (index, value)) + })) + } +} + impl<'a> WriteBatch<'a> { pub fn put_bytes(&mut self, key: C::Index, bytes: &[u8]) -> Result<()> { self.write_batch From 0fcb05cd8851dca0466d14efd209bf2c3e8c82fd Mon Sep 17 00:00:00 2001 From: Tyera Eulberg Date: Mon, 25 Sep 2023 18:17:13 -0600 Subject: [PATCH 05/57] Impl ColumnIndexDeprecation for TransactionStatus (doesn't build) --- ledger/src/blockstore_db.rs | 71 +++++++++++++++++++++++++++---------- 1 file changed, 52 insertions(+), 19 deletions(-) diff --git a/ledger/src/blockstore_db.rs b/ledger/src/blockstore_db.rs index 89daabee5e055b..a67f5a4d98a6d8 100644 --- a/ledger/src/blockstore_db.rs +++ b/ledger/src/blockstore_db.rs @@ -270,7 +270,7 @@ pub mod columns { #[derive(Debug)] /// The transaction status column /// - /// * index type: `(u64, `[`Signature`]`, `[`Slot`])` + /// * index type: `(`[`Signature`]`, `[`Slot`])` /// * value type: [`generated::TransactionStatusMeta`] pub struct TransactionStatus; @@ -787,33 +787,28 @@ pub trait ColumnIndexDeprecation: Column { } impl Column for columns::TransactionStatus { - type Index = (u64, Signature, Slot); + type Index = (Signature, Slot); - fn key((index, signature, slot): (u64, Signature, Slot)) -> Vec { - let mut key = vec![0; 8 + 64 + 8]; // size_of u64 + size_of Signature + size_of Slot - BigEndian::write_u64(&mut key[0..8], index); - key[8..72].copy_from_slice(&signature.as_ref()[0..64]); - BigEndian::write_u64(&mut key[72..80], slot); + fn key((signature, slot): (Signature, Slot)) -> Vec { + let mut key = vec![0; Self::CURRENT_INDEX_LEN]; // size_of Signature + size_of Slot + key[0..64].copy_from_slice(&signature.as_ref()[0..64]); + BigEndian::write_u64(&mut key[64..72], slot); key } - fn index(key: &[u8]) -> (u64, Signature, Slot) { - if key.len() != 80 { - Self::as_index(0) - } else { - let index = BigEndian::read_u64(&key[0..8]); - let signature = Signature::try_from(&key[8..72]).unwrap(); - let slot = BigEndian::read_u64(&key[72..80]); - (index, signature, slot) - } + fn index(key: &[u8]) -> (Signature, Slot) { + ::index(key) } fn slot(index: Self::Index) -> Slot { - index.2 + index.1 } - fn as_index(index: u64) -> Self::Index { - (index, Signature::default(), 0) + // This trait method is primarily used by Database::delete_range_cf(). Because the + // TransactionStatus Column is not keyed by slot, it cannot be range deleted. Therefore this + // method is meaningless for this type, and simply returns a default Index. + fn as_index(_index: u64) -> Self::Index { + (Signature::default(), 0) } } impl ColumnName for columns::TransactionStatus { @@ -823,6 +818,44 @@ impl ProtobufColumn for columns::TransactionStatus { type Type = generated::TransactionStatusMeta; } +impl ColumnIndexDeprecation for columns::TransactionStatus { + const DEPRECATED_INDEX_LEN: usize = 80; + const CURRENT_INDEX_LEN: usize = 72; + type DeprecatedIndex = (u64, Signature, Slot); + + fn deprecated_key(index: Self::DeprecatedIndex) -> Vec { + let (index, signature, slot) = index; + let mut key = vec![0; 8]; + BigEndian::write_u64(&mut key[0..8], index); + key.extend_from_slice(&Self::key((signature, slot))); + key + } + + fn try_deprecated_index(key: &[u8]) -> std::result::Result { + if key.len() != Self::DEPRECATED_INDEX_LEN { + return Err(IndexError::UnpackError); + } + let primary_index = BigEndian::read_u64(&key[0..8]); + let signature = Signature::try_from(&key[8..72]).unwrap(); + let slot = BigEndian::read_u64(&key[72..80]); + Ok((primary_index, signature, slot)) + } + + fn try_current_index(key: &[u8]) -> std::result::Result { + if key.len() != Self::CURRENT_INDEX_LEN { + return Err(IndexError::UnpackError); + } + let signature = Signature::try_from(&key[0..64]).unwrap(); + let slot = BigEndian::read_u64(&key[64..72]); + Ok((signature, slot)) + } + + fn convert_index(deprecated_index: Self::DeprecatedIndex) -> Self::Index { + let (_primary_index, signature, slot) = deprecated_index; + (signature, slot) + } +} + impl Column for columns::AddressSignatures { type Index = (u64, Pubkey, Slot, Signature); From f163754c93eb2d8ea99e0da47b556e78e6ba54b3 Mon Sep 17 00:00:00 2001 From: Tyera Eulberg Date: Mon, 25 Sep 2023 18:20:21 -0600 Subject: [PATCH 06/57] Update TransactionStatus put --- ledger/src/blockstore.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ledger/src/blockstore.rs b/ledger/src/blockstore.rs index c23dc240d79f7f..abaab34e59fdd2 100644 --- a/ledger/src/blockstore.rs +++ b/ledger/src/blockstore.rs @@ -2260,7 +2260,7 @@ impl Blockstore { let primary_index = self.get_primary_index_to_write(slot, &w_active_transaction_status_index)?; self.transaction_status_cf - .put_protobuf((primary_index, signature, slot), &status)?; + .put_protobuf((signature, slot), &status)?; for address in writable_keys { self.address_signatures_cf.put( (primary_index, *address, slot, signature), From 2027cfb5d61addd52b361da1f17d6496c700884e Mon Sep 17 00:00:00 2001 From: Tyera Eulberg Date: Mon, 25 Sep 2023 18:52:42 -0600 Subject: [PATCH 07/57] Update TransactionStatus purge_exact --- ledger/src/blockstore/blockstore_purge.rs | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/ledger/src/blockstore/blockstore_purge.rs b/ledger/src/blockstore/blockstore_purge.rs index f7e8aab3db3ad7..3af178d4775e2a 100644 --- a/ledger/src/blockstore/blockstore_purge.rs +++ b/ledger/src/blockstore/blockstore_purge.rs @@ -1,4 +1,9 @@ -use {super::*, solana_sdk::message::AccountKeys, std::time::Instant}; +use { + super::*, + crate::blockstore_db::ColumnIndexDeprecation, + solana_sdk::message::AccountKeys, + std::{cmp, time::Instant}, +}; #[derive(Default)] pub struct PurgeStats { @@ -391,9 +396,8 @@ impl Blockstore { for slot in from_slot..=to_slot { let primary_indexes = slot_indexes(slot); - if primary_indexes.is_empty() { - continue; - } + let delete_new_column_key = + primary_indexes.is_empty() || (slot == cmp::max(index0.max_slot, index1.max_slot)); let slot_entries = self.get_any_valid_slot_entries(slot, 0); let transactions = slot_entries @@ -401,8 +405,17 @@ impl Blockstore { .flat_map(|entry| entry.transactions); for transaction in transactions { if let Some(&signature) = transaction.signatures.get(0) { + if delete_new_column_key { + batch.delete::((signature, slot))?; + } for primary_index in &primary_indexes { - batch.delete::((*primary_index, signature, slot))?; + batch.delete_raw::( + &cf::TransactionStatus::deprecated_key(( + *primary_index, + signature, + slot, + )), + )?; } let meta = self.read_transaction_status((signature, slot))?; From 4630f1674b4fdc127eeb3888f11728a954cc88e7 Mon Sep 17 00:00:00 2001 From: Tyera Eulberg Date: Mon, 25 Sep 2023 19:28:22 -0600 Subject: [PATCH 08/57] Fix read_transaction_status --- ledger/src/blockstore.rs | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/ledger/src/blockstore.rs b/ledger/src/blockstore.rs index abaab34e59fdd2..5d10f47726d3af 100644 --- a/ledger/src/blockstore.rs +++ b/ledger/src/blockstore.rs @@ -6,8 +6,8 @@ use { crate::{ ancestor_iterator::AncestorIterator, blockstore_db::{ - columns as cf, Column, Database, IteratorDirection, IteratorMode, LedgerColumn, Result, - WriteBatch, + columns as cf, Column, ColumnIndexDeprecation, Database, IteratorDirection, + IteratorMode, LedgerColumn, Result, WriteBatch, }, blockstore_meta::*, blockstore_options::{ @@ -2226,24 +2226,40 @@ impl Blockstore { Ok(i) } - pub fn read_transaction_status( + fn read_deprecated_transaction_status( &self, index: (Signature, Slot), ) -> Result> { let (signature, slot) = index; let result = self .transaction_status_cf - .get_protobuf_or_bincode::((0, signature, slot))?; + .get_raw_protobuf_or_bincode::( + &cf::TransactionStatus::deprecated_key((0, signature, slot)), + )?; if result.is_none() { Ok(self .transaction_status_cf - .get_protobuf_or_bincode::((1, signature, slot))? + .get_raw_protobuf_or_bincode::( + &cf::TransactionStatus::deprecated_key((1, signature, slot)), + )? .and_then(|meta| meta.try_into().ok())) } else { Ok(result.and_then(|meta| meta.try_into().ok())) } } + pub fn read_transaction_status( + &self, + index: (Signature, Slot), + ) -> Result> { + let result = self.transaction_status_cf.get_protobuf(index)?; + if result.is_none() { + self.read_deprecated_transaction_status(index) + } else { + Ok(result.and_then(|meta| meta.try_into().ok())) + } + } + pub fn write_transaction_status( &self, slot: Slot, From 97750ebaf9c98ae6024b28f347389f2ff02f4d42 Mon Sep 17 00:00:00 2001 From: Tyera Eulberg Date: Mon, 25 Sep 2023 20:20:34 -0600 Subject: [PATCH 09/57] Fix get_transaction_status_with_counter --- ledger/src/blockstore.rs | 43 ++++++++++++++++++++++++++++++++-------- 1 file changed, 35 insertions(+), 8 deletions(-) diff --git a/ledger/src/blockstore.rs b/ledger/src/blockstore.rs index 5d10f47726d3af..eabca8e5ad2dea 100644 --- a/ledger/src/blockstore.rs +++ b/ledger/src/blockstore.rs @@ -2344,15 +2344,40 @@ impl Blockstore { let (lock, _) = self.ensure_lowest_cleanup_slot(); let first_available_block = self.get_first_available_block()?; - for transaction_status_cf_primary_index in 0..=1 { - let index_iterator = self.transaction_status_cf.iter(IteratorMode::From( - ( - transaction_status_cf_primary_index, - signature, - first_available_block, - ), + let iterator = self + .transaction_status_cf + .iter_filtered(IteratorMode::From( + (signature, first_available_block), IteratorDirection::Forward, ))?; + + for ((sig, slot), _data) in iterator { + counter += 1; + if sig != signature { + break; + } + if !self.is_root(slot) && !confirmed_unrooted_slots.contains(&slot) { + continue; + } + let status = self + .transaction_status_cf + .get_protobuf((signature, slot))? + .and_then(|status| status.try_into().ok()) + .map(|status| (slot, status)); + return Ok((status, counter)); + } + + for transaction_status_cf_primary_index in 0..=1 { + let index_iterator = + self.transaction_status_cf + .iter_deprecated_index_filtered(IteratorMode::From( + ( + transaction_status_cf_primary_index, + signature, + first_available_block, + ), + IteratorDirection::Forward, + ))?; for ((i, sig, slot), _data) in index_iterator { counter += 1; if i != transaction_status_cf_primary_index || sig != signature { @@ -2363,7 +2388,9 @@ impl Blockstore { } let status = self .transaction_status_cf - .get_protobuf_or_bincode::((i, sig, slot))? + .get_raw_protobuf_or_bincode::( + &cf::TransactionStatus::deprecated_key((i, signature, slot)), + )? .and_then(|status| status.try_into().ok()) .map(|status| (slot, status)); return Ok((status, counter)); From 551302a411b0e8c07709aa1580abc4d9ec059769 Mon Sep 17 00:00:00 2001 From: Tyera Eulberg Date: Mon, 25 Sep 2023 20:21:36 -0600 Subject: [PATCH 10/57] Fix test_all_empty_or_min (builds except tests) --- ledger/src/blockstore.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ledger/src/blockstore.rs b/ledger/src/blockstore.rs index eabca8e5ad2dea..7e6515a330c66c 100644 --- a/ledger/src/blockstore.rs +++ b/ledger/src/blockstore.rs @@ -4522,9 +4522,7 @@ pub fn test_all_empty_or_min(blockstore: &Blockstore, min_slot: Slot) { .iter::(IteratorMode::Start) .unwrap() .next() - .map(|((primary_index, _, slot), _)| { - slot >= min_slot || (primary_index == 2 && slot == 0) - }) + .map(|((_, slot), _)| slot >= min_slot || slot == 0) .unwrap_or(true) & blockstore .db From 5b722d943dfee103d654cf84146afd4442808c9e Mon Sep 17 00:00:00 2001 From: Tyera Eulberg Date: Thu, 28 Sep 2023 11:39:25 -0600 Subject: [PATCH 11/57] Fix test_get_rooted_block --- ledger/src/blockstore.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ledger/src/blockstore.rs b/ledger/src/blockstore.rs index 7e6515a330c66c..5ab343e456fdc7 100644 --- a/ledger/src/blockstore.rs +++ b/ledger/src/blockstore.rs @@ -7245,7 +7245,7 @@ pub mod tests { .into(); blockstore .transaction_status_cf - .put_protobuf((0, signature, slot), &status) + .put_protobuf((signature, slot), &status) .unwrap(); let status = TransactionStatusMeta { status: Ok(()), @@ -7264,7 +7264,7 @@ pub mod tests { .into(); blockstore .transaction_status_cf - .put_protobuf((0, signature, slot + 1), &status) + .put_protobuf((signature, slot + 1), &status) .unwrap(); let status = TransactionStatusMeta { status: Ok(()), @@ -7283,7 +7283,7 @@ pub mod tests { .into(); blockstore .transaction_status_cf - .put_protobuf((0, signature, slot + 2), &status) + .put_protobuf((signature, slot + 2), &status) .unwrap(); VersionedTransactionWithStatusMeta { transaction, From beb1047823072e087e46af5e97af13e1d5710994 Mon Sep 17 00:00:00 2001 From: Tyera Eulberg Date: Thu, 28 Sep 2023 11:58:14 -0600 Subject: [PATCH 12/57] Fix test_persist_transaction_status --- ledger/src/blockstore.rs | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/ledger/src/blockstore.rs b/ledger/src/blockstore.rs index 5ab343e456fdc7..608092d4e4baaa 100644 --- a/ledger/src/blockstore.rs +++ b/ledger/src/blockstore.rs @@ -7424,7 +7424,7 @@ pub mod tests { // result not found assert!(transaction_status_cf - .get_protobuf_or_bincode::((0, Signature::default(), 0)) + .get_protobuf((Signature::default(), 0)) .unwrap() .is_none()); @@ -7445,7 +7445,7 @@ pub mod tests { } .into(); assert!(transaction_status_cf - .put_protobuf((0, Signature::default(), 0), &status,) + .put_protobuf((Signature::default(), 0), &status) .is_ok()); // result found @@ -7463,7 +7463,7 @@ pub mod tests { return_data, compute_units_consumed, } = transaction_status_cf - .get_protobuf_or_bincode::((0, Signature::default(), 0)) + .get_protobuf((Signature::default(), 0)) .unwrap() .unwrap() .try_into() @@ -7498,7 +7498,7 @@ pub mod tests { } .into(); assert!(transaction_status_cf - .put_protobuf((0, Signature::from([2u8; 64]), 9), &status,) + .put_protobuf((Signature::from([2u8; 64]), 9), &status,) .is_ok()); // result found @@ -7516,11 +7516,7 @@ pub mod tests { return_data, compute_units_consumed, } = transaction_status_cf - .get_protobuf_or_bincode::(( - 0, - Signature::from([2u8; 64]), - 9, - )) + .get_protobuf((Signature::from([2u8; 64]), 9)) .unwrap() .unwrap() .try_into() From d58776a0b447cc8014448dcb66cce69061426928 Mon Sep 17 00:00:00 2001 From: Tyera Eulberg Date: Thu, 28 Sep 2023 12:14:20 -0600 Subject: [PATCH 13/57] Fix test_get_transaction_status --- ledger/src/blockstore.rs | 75 ++++++++++++++++------------------------ 1 file changed, 30 insertions(+), 45 deletions(-) diff --git a/ledger/src/blockstore.rs b/ledger/src/blockstore.rs index 608092d4e4baaa..96376e84d3122f 100644 --- a/ledger/src/blockstore.rs +++ b/ledger/src/blockstore.rs @@ -7792,56 +7792,45 @@ pub mod tests { blockstore.set_roots([0, 2].iter()).unwrap(); - // Initialize index 0, including: - // signature2 in non-root and root, - // signature4 in non-root, + // Initialize statuses: + // signature2 in skipped slot and root, + // signature4 in skipped slot, // signature5 in skipped slot and non-root, // signature6 in skipped slot, + // signature5 extra entries transaction_status_cf - .put_protobuf((0, signature2, 1), &status) - .unwrap(); - - transaction_status_cf - .put_protobuf((0, signature2, 2), &status) - .unwrap(); - - transaction_status_cf - .put_protobuf((0, signature4, 1), &status) + .put_protobuf((signature2, 1), &status) .unwrap(); transaction_status_cf - .put_protobuf((0, signature5, 1), &status) + .put_protobuf((signature2, 2), &status) .unwrap(); transaction_status_cf - .put_protobuf((0, signature5, 3), &status) + .put_protobuf((signature4, 1), &status) .unwrap(); transaction_status_cf - .put_protobuf((0, signature6, 1), &status) + .put_protobuf((signature5, 1), &status) .unwrap(); - // Initialize index 1, including: - // signature4 in root, - // signature6 in non-root, - // signature5 extra entries transaction_status_cf - .put_protobuf((1, signature4, 2), &status) + .put_protobuf((signature5, 3), &status) .unwrap(); transaction_status_cf - .put_protobuf((1, signature5, 4), &status) + .put_protobuf((signature6, 1), &status) .unwrap(); transaction_status_cf - .put_protobuf((1, signature5, 5), &status) + .put_protobuf((signature5, 5), &status) .unwrap(); transaction_status_cf - .put_protobuf((1, signature6, 3), &status) + .put_protobuf((signature6, 3), &status) .unwrap(); - // Signature exists, root found in index 0 + // Signature exists, root found if let (Some((slot, _status)), counter) = blockstore .get_transaction_status_with_counter(signature2, &[].into()) .unwrap() @@ -7859,30 +7848,26 @@ pub mod tests { assert_eq!(counter, 2); } - // Signature exists, root found in index 1 - if let (Some((slot, _status)), counter) = blockstore + // Signature exists in skipped slot, no root found + let (status, counter) = blockstore .get_transaction_status_with_counter(signature4, &[].into()) - .unwrap() - { - assert_eq!(slot, 2); - assert_eq!(counter, 3); - } + .unwrap(); + assert_eq!(status, None); + assert_eq!(counter, 2); - // Signature exists, root found although not required, in index 1 - if let (Some((slot, _status)), counter) = blockstore + // Signature exists in skipped slot, no non-root found + let (status, counter) = blockstore .get_transaction_status_with_counter(signature4, &[3].into()) - .unwrap() - { - assert_eq!(slot, 2); - assert_eq!(counter, 3); - } + .unwrap(); + assert_eq!(status, None); + assert_eq!(counter, 2); // Signature exists, no root found let (status, counter) = blockstore .get_transaction_status_with_counter(signature5, &[].into()) .unwrap(); assert_eq!(status, None); - assert_eq!(counter, 6); + assert_eq!(counter, 4); // Signature exists, root not required if let (Some((slot, _status)), counter) = blockstore @@ -7898,39 +7883,39 @@ pub mod tests { .get_transaction_status_with_counter(signature1, &[].into()) .unwrap(); assert_eq!(status, None); - assert_eq!(counter, 2); + assert_eq!(counter, 1); let (status, counter) = blockstore .get_transaction_status_with_counter(signature1, &[3].into()) .unwrap(); assert_eq!(status, None); - assert_eq!(counter, 2); + assert_eq!(counter, 1); // Signature does not exist, between existing entries let (status, counter) = blockstore .get_transaction_status_with_counter(signature3, &[].into()) .unwrap(); assert_eq!(status, None); - assert_eq!(counter, 2); + assert_eq!(counter, 1); let (status, counter) = blockstore .get_transaction_status_with_counter(signature3, &[3].into()) .unwrap(); assert_eq!(status, None); - assert_eq!(counter, 2); + assert_eq!(counter, 1); // Signature does not exist, larger than existing entries let (status, counter) = blockstore .get_transaction_status_with_counter(signature7, &[].into()) .unwrap(); assert_eq!(status, None); - assert_eq!(counter, 1); + assert_eq!(counter, 0); let (status, counter) = blockstore .get_transaction_status_with_counter(signature7, &[3].into()) .unwrap(); assert_eq!(status, None); - assert_eq!(counter, 1); + assert_eq!(counter, 0); } fn do_test_lowest_cleanup_slot_and_special_cfs(simulate_ledger_cleanup_service: bool) { From 2e6c5061ac1893c129c5f0db5ea684e4fdb4ee0e Mon Sep 17 00:00:00 2001 From: Tyera Eulberg Date: Thu, 28 Sep 2023 12:23:20 -0600 Subject: [PATCH 14/57] Fix test_get_rooted_transaction --- ledger/src/blockstore.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/ledger/src/blockstore.rs b/ledger/src/blockstore.rs index 96376e84d3122f..b57891a4b69d6e 100644 --- a/ledger/src/blockstore.rs +++ b/ledger/src/blockstore.rs @@ -8121,7 +8121,7 @@ pub mod tests { .into(); blockstore .transaction_status_cf - .put_protobuf((0, signature, slot), &status) + .put_protobuf((signature, slot), &status) .unwrap(); VersionedTransactionWithStatusMeta { transaction, @@ -8165,11 +8165,13 @@ pub mod tests { ); } - blockstore.run_purge(0, 2, PurgeType::PrimaryIndex).unwrap(); + blockstore + .run_purge(0, slot, PurgeType::CompactionFilter) + .unwrap(); *blockstore.lowest_cleanup_slot.write().unwrap() = slot; for VersionedTransactionWithStatusMeta { transaction, .. } in expected_transactions { let signature = transaction.signatures[0]; - assert_eq!(blockstore.get_rooted_transaction(signature).unwrap(), None,); + assert_eq!(blockstore.get_rooted_transaction(signature).unwrap(), None); assert_eq!( blockstore .get_complete_transaction(signature, slot + 1) From 80c349f8862072355875236a5be0cbb3220e7019 Mon Sep 17 00:00:00 2001 From: Tyera Eulberg Date: Thu, 28 Sep 2023 12:26:52 -0600 Subject: [PATCH 15/57] Fix test_get_complete_transaction --- ledger/src/blockstore.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ledger/src/blockstore.rs b/ledger/src/blockstore.rs index b57891a4b69d6e..9c7116d09b2aff 100644 --- a/ledger/src/blockstore.rs +++ b/ledger/src/blockstore.rs @@ -8243,7 +8243,7 @@ pub mod tests { .into(); blockstore .transaction_status_cf - .put_protobuf((0, signature, slot), &status) + .put_protobuf((signature, slot), &status) .unwrap(); VersionedTransactionWithStatusMeta { transaction, @@ -8280,7 +8280,9 @@ pub mod tests { assert_eq!(blockstore.get_rooted_transaction(signature).unwrap(), None); } - blockstore.run_purge(0, 2, PurgeType::PrimaryIndex).unwrap(); + blockstore + .run_purge(0, slot, PurgeType::CompactionFilter) + .unwrap(); *blockstore.lowest_cleanup_slot.write().unwrap() = slot; for VersionedTransactionWithStatusMeta { transaction, .. } in expected_transactions { let signature = transaction.signatures[0]; From e9f8b946e6bf47eb9af5d55922951da7bb0404dd Mon Sep 17 00:00:00 2001 From: Tyera Eulberg Date: Thu, 28 Sep 2023 15:24:00 -0600 Subject: [PATCH 16/57] Fix test_lowest_cleanup_slot_and_special_cfs --- ledger/src/blockstore.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ledger/src/blockstore.rs b/ledger/src/blockstore.rs index 9c7116d09b2aff..9019c61427d360 100644 --- a/ledger/src/blockstore.rs +++ b/ledger/src/blockstore.rs @@ -7962,11 +7962,11 @@ pub mod tests { let lowest_available_slot = lowest_cleanup_slot + 1; transaction_status_cf - .put_protobuf((0, signature1, lowest_cleanup_slot), &status) + .put_protobuf((signature1, lowest_cleanup_slot), &status) .unwrap(); transaction_status_cf - .put_protobuf((0, signature2, lowest_available_slot), &status) + .put_protobuf((signature2, lowest_available_slot), &status) .unwrap(); let address0 = solana_sdk::pubkey::new_rand(); From 149bc28fec18de42815b463d6525ca0d6c8e0bee Mon Sep 17 00:00:00 2001 From: Tyera Eulberg Date: Thu, 28 Sep 2023 15:27:35 -0600 Subject: [PATCH 17/57] Fix test_map_transactions_to_statuses --- ledger/src/blockstore.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ledger/src/blockstore.rs b/ledger/src/blockstore.rs index 9019c61427d360..f108f6d3e430bd 100644 --- a/ledger/src/blockstore.rs +++ b/ledger/src/blockstore.rs @@ -9031,7 +9031,7 @@ pub mod tests { } .into(); transaction_status_cf - .put_protobuf((0, transaction.signatures[0], slot), &status) + .put_protobuf((transaction.signatures[0], slot), &status) .unwrap(); transactions.push(transaction.into()); } From 6f6c8b0f7cebc7da150393054162274bb59cbd00 Mon Sep 17 00:00:00 2001 From: Tyera Eulberg Date: Thu, 28 Sep 2023 15:35:25 -0600 Subject: [PATCH 18/57] Fix test_transaction_status_protobuf_backward_compatability --- ledger/src/blockstore.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/ledger/src/blockstore.rs b/ledger/src/blockstore.rs index f108f6d3e430bd..c86602e5d51b87 100644 --- a/ledger/src/blockstore.rs +++ b/ledger/src/blockstore.rs @@ -9724,6 +9724,10 @@ pub mod tests { } } + // This test is probably superfluous, since it is highly unlikely that bincode-format + // TransactionStatus entries exist in any current ledger. They certainly exist in historical + // ledger archives, but typically those require contemporaraneous software for other reasons. + // However, we are persisting the test since the apis still exist in `blockstore_db`. #[test] fn test_transaction_status_protobuf_backward_compatability() { let ledger_path = get_tmp_ledger_path_auto_delete!(); @@ -9781,13 +9785,13 @@ pub mod tests { let data = serialize(&deprecated_status).unwrap(); blockstore .transaction_status_cf - .put_bytes((0, Signature::default(), slot), &data) + .put_bytes((Signature::default(), slot), &data) .unwrap(); } for slot in 2..4 { blockstore .transaction_status_cf - .put_protobuf((0, Signature::default(), slot), &protobuf_status) + .put_protobuf((Signature::default(), slot), &protobuf_status) .unwrap(); } for slot in 0..4 { @@ -9795,7 +9799,6 @@ pub mod tests { blockstore .transaction_status_cf .get_protobuf_or_bincode::(( - 0, Signature::default(), slot )) From 02825d25694521ebf00071bc3904931937fd12ca Mon Sep 17 00:00:00 2001 From: Tyera Eulberg Date: Thu, 5 Oct 2023 12:53:45 -0600 Subject: [PATCH 19/57] Fix test_special_columns_empty --- ledger/src/blockstore/blockstore_purge.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/ledger/src/blockstore/blockstore_purge.rs b/ledger/src/blockstore/blockstore_purge.rs index 3af178d4775e2a..f882debc3ea3b7 100644 --- a/ledger/src/blockstore/blockstore_purge.rs +++ b/ledger/src/blockstore/blockstore_purge.rs @@ -927,6 +927,7 @@ pub mod tests { transaction.message.static_account_keys().iter().collect(), vec![], TransactionStatusMeta::default(), + 0, ) .unwrap(); } From 974a27edfb2eb397a8a274a517c20536caa39e65 Mon Sep 17 00:00:00 2001 From: Tyera Eulberg Date: Tue, 3 Oct 2023 14:37:49 -0600 Subject: [PATCH 20/57] Delete test_transaction_status_index --- ledger/src/blockstore.rs | 205 --------------------------------------- 1 file changed, 205 deletions(-) diff --git a/ledger/src/blockstore.rs b/ledger/src/blockstore.rs index c86602e5d51b87..19d7a22a9d9e93 100644 --- a/ledger/src/blockstore.rs +++ b/ledger/src/blockstore.rs @@ -7537,211 +7537,6 @@ pub mod tests { assert_eq!(compute_units_consumed, compute_units_consumed_2); } - #[test] - #[allow(clippy::cognitive_complexity)] - fn test_transaction_status_index() { - let ledger_path = get_tmp_ledger_path_auto_delete!(); - let blockstore = Blockstore::open(ledger_path.path()).unwrap(); - - let transaction_status_index_cf = &blockstore.transaction_status_index_cf; - let slot0 = 10; - - // Primary index column is initialized on Blockstore::open - assert!(transaction_status_index_cf.get(0).unwrap().is_some()); - assert!(transaction_status_index_cf.get(1).unwrap().is_some()); - - for _ in 0..5 { - let random_bytes: [u8; 64] = std::array::from_fn(|_| rand::random::()); - blockstore - .write_transaction_status( - slot0, - Signature::from(random_bytes), - vec![&Pubkey::try_from(&random_bytes[..32]).unwrap()], - vec![&Pubkey::try_from(&random_bytes[32..]).unwrap()], - TransactionStatusMeta::default(), - ) - .unwrap(); - } - - // New statuses bump index 0 max_slot - assert_eq!( - transaction_status_index_cf.get(0).unwrap().unwrap(), - TransactionStatusIndexMeta { - max_slot: slot0, - frozen: false, - } - ); - assert_eq!( - transaction_status_index_cf.get(1).unwrap().unwrap(), - TransactionStatusIndexMeta::default() - ); - - let first_status_entry = blockstore - .db - .iter::(IteratorMode::From( - cf::TransactionStatus::as_index(0), - IteratorDirection::Forward, - )) - .unwrap() - .next() - .unwrap() - .0; - assert_eq!(first_status_entry.0, 0); - assert_eq!(first_status_entry.2, slot0); - let first_address_entry = blockstore - .db - .iter::(IteratorMode::From( - cf::AddressSignatures::as_index(0), - IteratorDirection::Forward, - )) - .unwrap() - .next() - .unwrap() - .0; - assert_eq!(first_address_entry.0, 0); - assert_eq!(first_address_entry.2, slot0); - - blockstore.run_purge(0, 8, PurgeType::PrimaryIndex).unwrap(); - // First successful prune freezes index 0 - assert_eq!( - transaction_status_index_cf.get(0).unwrap().unwrap(), - TransactionStatusIndexMeta { - max_slot: slot0, - frozen: true, - } - ); - assert_eq!( - transaction_status_index_cf.get(1).unwrap().unwrap(), - TransactionStatusIndexMeta::default() - ); - - let slot1 = 20; - for _ in 0..5 { - let random_bytes: [u8; 64] = std::array::from_fn(|_| rand::random::()); - blockstore - .write_transaction_status( - slot1, - Signature::from(random_bytes), - vec![&Pubkey::try_from(&random_bytes[..32]).unwrap()], - vec![&Pubkey::try_from(&random_bytes[32..]).unwrap()], - TransactionStatusMeta::default(), - ) - .unwrap(); - } - - assert_eq!( - transaction_status_index_cf.get(0).unwrap().unwrap(), - TransactionStatusIndexMeta { - max_slot: slot0, - frozen: true, - } - ); - // Index 0 is frozen, so new statuses bump index 1 max_slot - assert_eq!( - transaction_status_index_cf.get(1).unwrap().unwrap(), - TransactionStatusIndexMeta { - max_slot: slot1, - frozen: false, - } - ); - - // Index 0 statuses and address records still exist - let first_status_entry = blockstore - .db - .iter::(IteratorMode::From( - cf::TransactionStatus::as_index(0), - IteratorDirection::Forward, - )) - .unwrap() - .next() - .unwrap() - .0; - assert_eq!(first_status_entry.0, 0); - assert_eq!(first_status_entry.2, 10); - let first_address_entry = blockstore - .db - .iter::(IteratorMode::From( - cf::AddressSignatures::as_index(0), - IteratorDirection::Forward, - )) - .unwrap() - .next() - .unwrap() - .0; - assert_eq!(first_address_entry.0, 0); - assert_eq!(first_address_entry.2, slot0); - // New statuses and address records are stored in index 1 - let index1_first_status_entry = blockstore - .db - .iter::(IteratorMode::From( - cf::TransactionStatus::as_index(1), - IteratorDirection::Forward, - )) - .unwrap() - .next() - .unwrap() - .0; - assert_eq!(index1_first_status_entry.0, 1); - assert_eq!(index1_first_status_entry.2, slot1); - let index1_first_address_entry = blockstore - .db - .iter::(IteratorMode::From( - cf::AddressSignatures::as_index(1), - IteratorDirection::Forward, - )) - .unwrap() - .next() - .unwrap() - .0; - assert_eq!(index1_first_address_entry.0, 1); - assert_eq!(index1_first_address_entry.2, slot1); - - blockstore - .run_purge(0, 18, PurgeType::PrimaryIndex) - .unwrap(); - // Successful prune toggles TransactionStatusIndex - assert_eq!( - transaction_status_index_cf.get(0).unwrap().unwrap(), - TransactionStatusIndexMeta { - max_slot: 0, - frozen: false, - } - ); - assert_eq!( - transaction_status_index_cf.get(1).unwrap().unwrap(), - TransactionStatusIndexMeta { - max_slot: slot1, - frozen: true, - } - ); - - // Index 0 has been pruned, so first status and address entries are now index 1 - let first_status_entry = blockstore - .db - .iter::(IteratorMode::From( - cf::TransactionStatus::as_index(0), - IteratorDirection::Forward, - )) - .unwrap() - .next() - .unwrap() - .0; - assert_eq!(first_status_entry.0, 1); - assert_eq!(first_status_entry.2, slot1); - let first_address_entry = blockstore - .db - .iter::(IteratorMode::From( - cf::AddressSignatures::as_index(0), - IteratorDirection::Forward, - )) - .unwrap() - .next() - .unwrap() - .0; - assert_eq!(first_address_entry.0, 1); - assert_eq!(first_address_entry.2, slot1); - } - #[test] fn test_get_transaction_status() { let ledger_path = get_tmp_ledger_path_auto_delete!(); From 7a20ed3ee5892fb5653838bb6c8ae24a52ddfd08 Mon Sep 17 00:00:00 2001 From: Tyera Eulberg Date: Tue, 3 Oct 2023 14:38:56 -0600 Subject: [PATCH 21/57] Delete test_purge_transaction_status --- ledger/src/blockstore/blockstore_purge.rs | 164 ---------------------- 1 file changed, 164 deletions(-) diff --git a/ledger/src/blockstore/blockstore_purge.rs b/ledger/src/blockstore/blockstore_purge.rs index f882debc3ea3b7..053efd27d9b3cf 100644 --- a/ledger/src/blockstore/blockstore_purge.rs +++ b/ledger/src/blockstore/blockstore_purge.rs @@ -571,170 +571,6 @@ pub mod tests { assert_eq!(entry.0, 0); } - #[test] - #[allow(clippy::cognitive_complexity)] - fn test_purge_transaction_status() { - let ledger_path = get_tmp_ledger_path_auto_delete!(); - let blockstore = Blockstore::open(ledger_path.path()).unwrap(); - - let transaction_status_index_cf = &blockstore.transaction_status_index_cf; - let slot = 10; - for _ in 0..5 { - let random_bytes: [u8; 64] = std::array::from_fn(|_| rand::random::()); - blockstore - .write_transaction_status( - slot, - Signature::from(random_bytes), - vec![&Pubkey::try_from(&random_bytes[..32]).unwrap()], - vec![&Pubkey::try_from(&random_bytes[32..]).unwrap()], - TransactionStatusMeta::default(), - ) - .unwrap(); - } - // Purge to freeze index 0 - blockstore.run_purge(0, 1, PurgeType::PrimaryIndex).unwrap(); - let mut status_entry_iterator = blockstore - .db - .iter::(IteratorMode::From( - cf::TransactionStatus::as_index(0), - IteratorDirection::Forward, - )) - .unwrap(); - for _ in 0..5 { - let entry = status_entry_iterator.next().unwrap().0; - assert_eq!(entry.0, 0); - assert_eq!(entry.2, slot); - } - let mut address_transactions_iterator = blockstore - .db - .iter::(IteratorMode::From( - (0, Pubkey::default(), 0, Signature::default()), - IteratorDirection::Forward, - )) - .unwrap(); - for _ in 0..10 { - let entry = address_transactions_iterator.next().unwrap().0; - assert_eq!(entry.0, 0); - assert_eq!(entry.2, slot); - } - assert_eq!( - transaction_status_index_cf.get(0).unwrap().unwrap(), - TransactionStatusIndexMeta { - max_slot: 10, - frozen: true, - } - ); - drop(status_entry_iterator); - drop(address_transactions_iterator); - - // Low purge should not affect state - blockstore.run_purge(0, 5, PurgeType::PrimaryIndex).unwrap(); - let mut status_entry_iterator = blockstore - .db - .iter::(IteratorMode::From( - cf::TransactionStatus::as_index(0), - IteratorDirection::Forward, - )) - .unwrap(); - for _ in 0..5 { - let entry = status_entry_iterator.next().unwrap().0; - assert_eq!(entry.0, 0); - assert_eq!(entry.2, slot); - } - let mut address_transactions_iterator = blockstore - .db - .iter::(IteratorMode::From( - cf::AddressSignatures::as_index(0), - IteratorDirection::Forward, - )) - .unwrap(); - for _ in 0..10 { - let entry = address_transactions_iterator.next().unwrap().0; - assert_eq!(entry.0, 0); - assert_eq!(entry.2, slot); - } - assert_eq!( - transaction_status_index_cf.get(0).unwrap().unwrap(), - TransactionStatusIndexMeta { - max_slot: 10, - frozen: true, - } - ); - drop(status_entry_iterator); - drop(address_transactions_iterator); - - // Test boundary conditions: < slot should not purge statuses; <= slot should - blockstore.run_purge(0, 9, PurgeType::PrimaryIndex).unwrap(); - let mut status_entry_iterator = blockstore - .db - .iter::(IteratorMode::From( - cf::TransactionStatus::as_index(0), - IteratorDirection::Forward, - )) - .unwrap(); - for _ in 0..5 { - let entry = status_entry_iterator.next().unwrap().0; - assert_eq!(entry.0, 0); - assert_eq!(entry.2, slot); - } - let mut address_transactions_iterator = blockstore - .db - .iter::(IteratorMode::From( - cf::AddressSignatures::as_index(0), - IteratorDirection::Forward, - )) - .unwrap(); - for _ in 0..10 { - let entry = address_transactions_iterator.next().unwrap().0; - assert_eq!(entry.0, 0); - assert_eq!(entry.2, slot); - } - assert_eq!( - transaction_status_index_cf.get(0).unwrap().unwrap(), - TransactionStatusIndexMeta { - max_slot: 10, - frozen: true, - } - ); - drop(status_entry_iterator); - drop(address_transactions_iterator); - - blockstore - .run_purge(0, 10, PurgeType::PrimaryIndex) - .unwrap(); - let mut status_entry_iterator = blockstore - .db - .iter::(IteratorMode::From( - cf::TransactionStatus::as_index(0), - IteratorDirection::Forward, - )) - .unwrap(); - assert!(status_entry_iterator.next().is_none()); - let mut address_transactions_iterator = blockstore - .db - .iter::(IteratorMode::From( - cf::AddressSignatures::as_index(0), - IteratorDirection::Forward, - )) - .unwrap(); - assert!(address_transactions_iterator.next().is_none()); - - assert_eq!( - transaction_status_index_cf.get(0).unwrap().unwrap(), - TransactionStatusIndexMeta { - max_slot: 0, - frozen: false, - } - ); - assert_eq!( - transaction_status_index_cf.get(1).unwrap().unwrap(), - TransactionStatusIndexMeta { - max_slot: 0, - frozen: true, - } - ); - } - fn clear_and_repopulate_transaction_statuses_for_test( blockstore: &Blockstore, index0_max_slot: u64, From 8f9bccb4b50dd3fc7167be68f212a7f5fcca7c55 Mon Sep 17 00:00:00 2001 From: Tyera Eulberg Date: Tue, 3 Oct 2023 17:09:30 -0600 Subject: [PATCH 22/57] Ignore some tests until both special columns are dealt with (all build) --- ledger/src/blockstore/blockstore_purge.rs | 1368 ++++++++++----------- 1 file changed, 684 insertions(+), 684 deletions(-) diff --git a/ledger/src/blockstore/blockstore_purge.rs b/ledger/src/blockstore/blockstore_purge.rs index 053efd27d9b3cf..2d52ff77d6449f 100644 --- a/ledger/src/blockstore/blockstore_purge.rs +++ b/ledger/src/blockstore/blockstore_purge.rs @@ -523,214 +523,214 @@ pub mod tests { }); } - #[test] - fn test_purge_front_of_ledger() { - let ledger_path = get_tmp_ledger_path_auto_delete!(); - let blockstore = Blockstore::open(ledger_path.path()).unwrap(); - - let max_slot = 10; - for x in 0..max_slot { - let random_bytes: [u8; 64] = std::array::from_fn(|_| rand::random::()); - blockstore - .write_transaction_status( - x, - Signature::from(random_bytes), - vec![&Pubkey::try_from(&random_bytes[..32]).unwrap()], - vec![&Pubkey::try_from(&random_bytes[32..]).unwrap()], - TransactionStatusMeta::default(), - ) - .unwrap(); - } - // Purge to freeze index 0 - blockstore.run_purge(0, 1, PurgeType::PrimaryIndex).unwrap(); - - for x in max_slot..2 * max_slot { - let random_bytes: [u8; 64] = std::array::from_fn(|_| rand::random::()); - blockstore - .write_transaction_status( - x, - Signature::from(random_bytes), - vec![&Pubkey::try_from(&random_bytes[..32]).unwrap()], - vec![&Pubkey::try_from(&random_bytes[32..]).unwrap()], - TransactionStatusMeta::default(), - ) - .unwrap(); - } - - // Purging range outside of TransactionStatus max slots should not affect TransactionStatus data - blockstore.run_purge(20, 30, PurgeType::Exact).unwrap(); - - let mut status_entry_iterator = blockstore - .db - .iter::(IteratorMode::From( - cf::TransactionStatus::as_index(0), - IteratorDirection::Forward, - )) - .unwrap(); - let entry = status_entry_iterator.next().unwrap().0; - assert_eq!(entry.0, 0); - } - - fn clear_and_repopulate_transaction_statuses_for_test( - blockstore: &Blockstore, - index0_max_slot: u64, - index1_max_slot: u64, - ) { - assert!(index1_max_slot > index0_max_slot); - let mut write_batch = blockstore.db.batch().unwrap(); - blockstore - .run_purge(0, index1_max_slot, PurgeType::PrimaryIndex) - .unwrap(); - blockstore - .db - .delete_range_cf::(&mut write_batch, 0, 2) - .unwrap(); - blockstore - .db - .delete_range_cf::(&mut write_batch, 0, 2) - .unwrap(); - blockstore.db.write(write_batch).unwrap(); - blockstore.initialize_transaction_status_index().unwrap(); - *blockstore.active_transaction_status_index.write().unwrap() = 0; - - for x in 0..index0_max_slot { - let entries = make_slot_entries_with_transactions(1); - let shreds = entries_to_test_shreds( - &entries, - x, // slot - x.saturating_sub(1), // parent_slot - true, // is_full_slot - 0, // version - true, // merkle_variant - ); - blockstore.insert_shreds(shreds, None, false).unwrap(); - let signature = entries - .iter() - .filter(|entry| !entry.is_tick()) - .cloned() - .flat_map(|entry| entry.transactions) - .map(|transaction| transaction.signatures[0]) - .collect::>()[0]; - let random_bytes: Vec = (0..64).map(|_| rand::random::()).collect(); - blockstore - .write_transaction_status( - x, - signature, - vec![&Pubkey::try_from(&random_bytes[..32]).unwrap()], - vec![&Pubkey::try_from(&random_bytes[32..]).unwrap()], - TransactionStatusMeta::default(), - ) - .unwrap(); - } - - // Add slot that crosses primary indexes - let entries = make_slot_entries_with_transactions(2); - let shreds = entries_to_test_shreds( - &entries, - index0_max_slot, // slot - index0_max_slot.saturating_sub(1), // parent_slot - true, // is_full_slot - 0, // version - true, // merkle_variant - ); - blockstore.insert_shreds(shreds, None, false).unwrap(); - let signatures = entries - .iter() - .filter(|entry| !entry.is_tick()) - .cloned() - .flat_map(|entry| entry.transactions) - .map(|transaction| transaction.signatures[0]) - .collect::>(); - let random_bytes: Vec = (0..64).map(|_| rand::random::()).collect(); - blockstore - .write_transaction_status( - index0_max_slot, - signatures[0], - vec![&Pubkey::try_from(&random_bytes[..32]).unwrap()], - vec![&Pubkey::try_from(&random_bytes[32..]).unwrap()], - TransactionStatusMeta::default(), - ) - .unwrap(); - - // Freeze index 0 - let mut write_batch = blockstore.db.batch().unwrap(); - let mut w_active_transaction_status_index = - blockstore.active_transaction_status_index.write().unwrap(); - blockstore - .toggle_transaction_status_index( - &mut write_batch, - &mut w_active_transaction_status_index, - index0_max_slot + 1, - ) - .unwrap(); - drop(w_active_transaction_status_index); - blockstore.db.write(write_batch).unwrap(); - - let random_bytes: Vec = (0..64).map(|_| rand::random::()).collect(); - blockstore - .write_transaction_status( - index0_max_slot, - signatures[1], - vec![&Pubkey::try_from(&random_bytes[..32]).unwrap()], - vec![&Pubkey::try_from(&random_bytes[32..]).unwrap()], - TransactionStatusMeta::default(), - ) - .unwrap(); - - // Note: index0_max_slot exists in both indexes - - for x in index0_max_slot + 1..index1_max_slot + 1 { - let entries = make_slot_entries_with_transactions(1); - let shreds = entries_to_test_shreds( - &entries, - x, // slot - x.saturating_sub(1), // parent_slot - true, // is_full_slot - 0, // version - true, // merkle_variant - ); - blockstore.insert_shreds(shreds, None, false).unwrap(); - let signature: Signature = entries - .iter() - .filter(|entry| !entry.is_tick()) - .cloned() - .flat_map(|entry| entry.transactions) - .map(|transaction| transaction.signatures[0]) - .collect::>()[0]; - let random_bytes: Vec = (0..64).map(|_| rand::random::()).collect(); - blockstore - .write_transaction_status( - x, - signature, - vec![&Pubkey::try_from(&random_bytes[..32]).unwrap()], - vec![&Pubkey::try_from(&random_bytes[32..]).unwrap()], - TransactionStatusMeta::default(), - ) - .unwrap(); - } - assert_eq!( - blockstore - .transaction_status_index_cf - .get(0) - .unwrap() - .unwrap(), - TransactionStatusIndexMeta { - max_slot: index0_max_slot, - frozen: true, - } - ); - assert_eq!( - blockstore - .transaction_status_index_cf - .get(1) - .unwrap() - .unwrap(), - TransactionStatusIndexMeta { - max_slot: index1_max_slot, - frozen: false, - } - ); - } + // #[test] + // fn test_purge_front_of_ledger() { + // let ledger_path = get_tmp_ledger_path_auto_delete!(); + // let blockstore = Blockstore::open(ledger_path.path()).unwrap(); + // + // let max_slot = 10; + // for x in 0..max_slot { + // let random_bytes: [u8; 64] = std::array::from_fn(|_| rand::random::()); + // blockstore + // .write_transaction_status( + // x, + // Signature::from(random_bytes), + // vec![&Pubkey::try_from(&random_bytes[..32]).unwrap()], + // vec![&Pubkey::try_from(&random_bytes[32..]).unwrap()], + // TransactionStatusMeta::default(), + // ) + // .unwrap(); + // } + // // Purge to freeze index 0 + // blockstore.run_purge(0, 1, PurgeType::PrimaryIndex).unwrap(); + // + // for x in max_slot..2 * max_slot { + // let random_bytes: [u8; 64] = std::array::from_fn(|_| rand::random::()); + // blockstore + // .write_transaction_status( + // x, + // Signature::from(random_bytes), + // vec![&Pubkey::try_from(&random_bytes[..32]).unwrap()], + // vec![&Pubkey::try_from(&random_bytes[32..]).unwrap()], + // TransactionStatusMeta::default(), + // ) + // .unwrap(); + // } + // + // // Purging range outside of TransactionStatus max slots should not affect TransactionStatus data + // blockstore.run_purge(20, 30, PurgeType::Exact).unwrap(); + // + // let mut status_entry_iterator = blockstore + // .db + // .iter::(IteratorMode::From( + // cf::TransactionStatus::as_index(0), + // IteratorDirection::Forward, + // )) + // .unwrap(); + // let entry = status_entry_iterator.next().unwrap().0; + // assert_eq!(entry.0, 0); + // } + // + // fn clear_and_repopulate_transaction_statuses_for_test( + // blockstore: &Blockstore, + // index0_max_slot: u64, + // index1_max_slot: u64, + // ) { + // assert!(index1_max_slot > index0_max_slot); + // let mut write_batch = blockstore.db.batch().unwrap(); + // blockstore + // .run_purge(0, index1_max_slot, PurgeType::PrimaryIndex) + // .unwrap(); + // blockstore + // .db + // .delete_range_cf::(&mut write_batch, 0, 2) + // .unwrap(); + // blockstore + // .db + // .delete_range_cf::(&mut write_batch, 0, 2) + // .unwrap(); + // blockstore.db.write(write_batch).unwrap(); + // blockstore.initialize_transaction_status_index().unwrap(); + // *blockstore.active_transaction_status_index.write().unwrap() = 0; + // + // for x in 0..index0_max_slot { + // let entries = make_slot_entries_with_transactions(1); + // let shreds = entries_to_test_shreds( + // &entries, + // x, // slot + // x.saturating_sub(1), // parent_slot + // true, // is_full_slot + // 0, // version + // true, // merkle_variant + // ); + // blockstore.insert_shreds(shreds, None, false).unwrap(); + // let signature = entries + // .iter() + // .filter(|entry| !entry.is_tick()) + // .cloned() + // .flat_map(|entry| entry.transactions) + // .map(|transaction| transaction.signatures[0]) + // .collect::>()[0]; + // let random_bytes: Vec = (0..64).map(|_| rand::random::()).collect(); + // blockstore + // .write_transaction_status( + // x, + // signature, + // vec![&Pubkey::try_from(&random_bytes[..32]).unwrap()], + // vec![&Pubkey::try_from(&random_bytes[32..]).unwrap()], + // TransactionStatusMeta::default(), + // ) + // .unwrap(); + // } + // + // // Add slot that crosses primary indexes + // let entries = make_slot_entries_with_transactions(2); + // let shreds = entries_to_test_shreds( + // &entries, + // index0_max_slot, // slot + // index0_max_slot.saturating_sub(1), // parent_slot + // true, // is_full_slot + // 0, // version + // true, // merkle_variant + // ); + // blockstore.insert_shreds(shreds, None, false).unwrap(); + // let signatures = entries + // .iter() + // .filter(|entry| !entry.is_tick()) + // .cloned() + // .flat_map(|entry| entry.transactions) + // .map(|transaction| transaction.signatures[0]) + // .collect::>(); + // let random_bytes: Vec = (0..64).map(|_| rand::random::()).collect(); + // blockstore + // .write_transaction_status( + // index0_max_slot, + // signatures[0], + // vec![&Pubkey::try_from(&random_bytes[..32]).unwrap()], + // vec![&Pubkey::try_from(&random_bytes[32..]).unwrap()], + // TransactionStatusMeta::default(), + // ) + // .unwrap(); + // + // // Freeze index 0 + // let mut write_batch = blockstore.db.batch().unwrap(); + // let mut w_active_transaction_status_index = + // blockstore.active_transaction_status_index.write().unwrap(); + // blockstore + // .toggle_transaction_status_index( + // &mut write_batch, + // &mut w_active_transaction_status_index, + // index0_max_slot + 1, + // ) + // .unwrap(); + // drop(w_active_transaction_status_index); + // blockstore.db.write(write_batch).unwrap(); + // + // let random_bytes: Vec = (0..64).map(|_| rand::random::()).collect(); + // blockstore + // .write_transaction_status( + // index0_max_slot, + // signatures[1], + // vec![&Pubkey::try_from(&random_bytes[..32]).unwrap()], + // vec![&Pubkey::try_from(&random_bytes[32..]).unwrap()], + // TransactionStatusMeta::default(), + // ) + // .unwrap(); + // + // // Note: index0_max_slot exists in both indexes + // + // for x in index0_max_slot + 1..index1_max_slot + 1 { + // let entries = make_slot_entries_with_transactions(1); + // let shreds = entries_to_test_shreds( + // &entries, + // x, // slot + // x.saturating_sub(1), // parent_slot + // true, // is_full_slot + // 0, // version + // true, // merkle_variant + // ); + // blockstore.insert_shreds(shreds, None, false).unwrap(); + // let signature: Signature = entries + // .iter() + // .filter(|entry| !entry.is_tick()) + // .cloned() + // .flat_map(|entry| entry.transactions) + // .map(|transaction| transaction.signatures[0]) + // .collect::>()[0]; + // let random_bytes: Vec = (0..64).map(|_| rand::random::()).collect(); + // blockstore + // .write_transaction_status( + // x, + // signature, + // vec![&Pubkey::try_from(&random_bytes[..32]).unwrap()], + // vec![&Pubkey::try_from(&random_bytes[32..]).unwrap()], + // TransactionStatusMeta::default(), + // ) + // .unwrap(); + // } + // assert_eq!( + // blockstore + // .transaction_status_index_cf + // .get(0) + // .unwrap() + // .unwrap(), + // TransactionStatusIndexMeta { + // max_slot: index0_max_slot, + // frozen: true, + // } + // ); + // assert_eq!( + // blockstore + // .transaction_status_index_cf + // .get(1) + // .unwrap() + // .unwrap(), + // TransactionStatusIndexMeta { + // max_slot: index1_max_slot, + // frozen: false, + // } + // ); + // } #[test] fn test_special_columns_empty() { @@ -781,480 +781,480 @@ pub mod tests { assert!(blockstore.special_columns_empty().unwrap()); } - #[test] - #[allow(clippy::cognitive_complexity)] - fn test_purge_transaction_status_exact() { - let ledger_path = get_tmp_ledger_path_auto_delete!(); - let blockstore = Blockstore::open(ledger_path.path()).unwrap(); - - let index0_max_slot = 9; - let index1_max_slot = 19; - - // Test purge outside bounds - clear_and_repopulate_transaction_statuses_for_test( - &blockstore, - index0_max_slot, - index1_max_slot, - ); - blockstore.run_purge(20, 22, PurgeType::Exact).unwrap(); - - let mut status_entry_iterator = blockstore - .db - .iter::(IteratorMode::From( - cf::TransactionStatus::as_index(0), - IteratorDirection::Forward, - )) - .unwrap(); - assert_eq!( - blockstore - .transaction_status_index_cf - .get(0) - .unwrap() - .unwrap(), - TransactionStatusIndexMeta { - max_slot: index0_max_slot, - frozen: true, - } - ); - for _ in 0..index0_max_slot + 1 { - let entry = status_entry_iterator.next().unwrap().0; - assert_eq!(entry.0, 0); - } - assert_eq!( - blockstore - .transaction_status_index_cf - .get(1) - .unwrap() - .unwrap(), - TransactionStatusIndexMeta { - max_slot: index1_max_slot, - frozen: false, - } - ); - for _ in index0_max_slot + 1..index1_max_slot + 1 { - let entry = status_entry_iterator.next().unwrap().0; - assert_eq!(entry.0, 1); - } - drop(status_entry_iterator); - - // Test purge inside index 0 - clear_and_repopulate_transaction_statuses_for_test( - &blockstore, - index0_max_slot, - index1_max_slot, - ); - blockstore.run_purge(2, 4, PurgeType::Exact).unwrap(); - - let mut status_entry_iterator = blockstore - .db - .iter::(IteratorMode::From( - cf::TransactionStatus::as_index(0), - IteratorDirection::Forward, - )) - .unwrap(); - assert_eq!( - blockstore - .transaction_status_index_cf - .get(0) - .unwrap() - .unwrap(), - TransactionStatusIndexMeta { - max_slot: index0_max_slot, - frozen: true, - } - ); - for _ in 0..7 { - // 7 entries remaining - let entry = status_entry_iterator.next().unwrap().0; - assert_eq!(entry.0, 0); - assert!(entry.2 < 2 || entry.2 > 4); - } - assert_eq!( - blockstore - .transaction_status_index_cf - .get(1) - .unwrap() - .unwrap(), - TransactionStatusIndexMeta { - max_slot: index1_max_slot, - frozen: false, - } - ); - for _ in index0_max_slot + 1..index1_max_slot + 1 { - let entry = status_entry_iterator.next().unwrap().0; - assert_eq!(entry.0, 1); - } - drop(status_entry_iterator); - - // Test purge inside index 0 at upper boundary - clear_and_repopulate_transaction_statuses_for_test( - &blockstore, - index0_max_slot, - index1_max_slot, - ); - blockstore - .run_purge(7, index0_max_slot, PurgeType::Exact) - .unwrap(); - - let mut status_entry_iterator = blockstore - .db - .iter::(IteratorMode::From( - cf::TransactionStatus::as_index(0), - IteratorDirection::Forward, - )) - .unwrap(); - assert_eq!( - blockstore - .transaction_status_index_cf - .get(0) - .unwrap() - .unwrap(), - TransactionStatusIndexMeta { - max_slot: 6, - frozen: true, - } - ); - for _ in 0..7 { - // 7 entries remaining - let entry = status_entry_iterator.next().unwrap().0; - assert_eq!(entry.0, 0); - assert!(entry.2 < 7); - } - assert_eq!( - blockstore - .transaction_status_index_cf - .get(1) - .unwrap() - .unwrap(), - TransactionStatusIndexMeta { - max_slot: index1_max_slot, - frozen: false, - } - ); - for _ in index0_max_slot + 1..index1_max_slot + 1 { - let entry = status_entry_iterator.next().unwrap().0; - assert_eq!(entry.0, 1); - } - drop(status_entry_iterator); - - // Test purge inside index 1 at lower boundary - clear_and_repopulate_transaction_statuses_for_test( - &blockstore, - index0_max_slot, - index1_max_slot, - ); - blockstore.run_purge(10, 12, PurgeType::Exact).unwrap(); - - let mut status_entry_iterator = blockstore - .db - .iter::(IteratorMode::From( - cf::TransactionStatus::as_index(0), - IteratorDirection::Forward, - )) - .unwrap(); - assert_eq!( - blockstore - .transaction_status_index_cf - .get(0) - .unwrap() - .unwrap(), - TransactionStatusIndexMeta { - max_slot: index0_max_slot, - frozen: true, - } - ); - for _ in 0..index0_max_slot + 1 { - let entry = status_entry_iterator.next().unwrap().0; - assert_eq!(entry.0, 0); - } - assert_eq!( - blockstore - .transaction_status_index_cf - .get(1) - .unwrap() - .unwrap(), - TransactionStatusIndexMeta { - max_slot: index1_max_slot, - frozen: false, - } - ); - for _ in 13..index1_max_slot + 1 { - let entry = status_entry_iterator.next().unwrap().0; - assert_eq!(entry.0, 1); - assert!(entry.2 == index0_max_slot || entry.2 > 12); - } - drop(status_entry_iterator); - - // Test purge across index boundaries - clear_and_repopulate_transaction_statuses_for_test( - &blockstore, - index0_max_slot, - index1_max_slot, - ); - blockstore.run_purge(7, 12, PurgeType::Exact).unwrap(); - - let mut status_entry_iterator = blockstore - .db - .iter::(IteratorMode::From( - cf::TransactionStatus::as_index(0), - IteratorDirection::Forward, - )) - .unwrap(); - assert_eq!( - blockstore - .transaction_status_index_cf - .get(0) - .unwrap() - .unwrap(), - TransactionStatusIndexMeta { - max_slot: 6, - frozen: true, - } - ); - for _ in 0..7 { - // 7 entries remaining - let entry = status_entry_iterator.next().unwrap().0; - assert_eq!(entry.0, 0); - assert!(entry.2 < 7); - } - assert_eq!( - blockstore - .transaction_status_index_cf - .get(1) - .unwrap() - .unwrap(), - TransactionStatusIndexMeta { - max_slot: index1_max_slot, - frozen: false, - } - ); - for _ in 13..index1_max_slot + 1 { - let entry = status_entry_iterator.next().unwrap().0; - assert_eq!(entry.0, 1); - assert!(entry.2 > 12); - } - drop(status_entry_iterator); - - // Test purge include complete index 1 - clear_and_repopulate_transaction_statuses_for_test( - &blockstore, - index0_max_slot, - index1_max_slot, - ); - blockstore.run_purge(7, 22, PurgeType::Exact).unwrap(); - - let mut status_entry_iterator = blockstore - .db - .iter::(IteratorMode::From( - cf::TransactionStatus::as_index(0), - IteratorDirection::Forward, - )) - .unwrap(); - assert_eq!( - blockstore - .transaction_status_index_cf - .get(0) - .unwrap() - .unwrap(), - TransactionStatusIndexMeta { - max_slot: 6, - frozen: true, - } - ); - for _ in 0..7 { - // 7 entries remaining - let entry = status_entry_iterator.next().unwrap().0; - assert_eq!(entry.0, 0); - assert!(entry.2 < 7); - } - assert_eq!( - blockstore - .transaction_status_index_cf - .get(1) - .unwrap() - .unwrap(), - TransactionStatusIndexMeta { - max_slot: 6, - frozen: false, - } - ); - drop(status_entry_iterator); - - // Purge up to but not including index0_max_slot - clear_and_repopulate_transaction_statuses_for_test( - &blockstore, - index0_max_slot, - index1_max_slot, - ); - blockstore - .run_purge(0, index0_max_slot - 1, PurgeType::Exact) - .unwrap(); - assert_eq!( - blockstore - .transaction_status_index_cf - .get(0) - .unwrap() - .unwrap(), - TransactionStatusIndexMeta { - max_slot: index0_max_slot, - frozen: true, - } - ); - - // Test purge all - clear_and_repopulate_transaction_statuses_for_test( - &blockstore, - index0_max_slot, - index1_max_slot, - ); - blockstore.run_purge(0, 22, PurgeType::Exact).unwrap(); - - let mut status_entry_iterator = blockstore - .db - .iter::(IteratorMode::From( - cf::TransactionStatus::as_index(0), - IteratorDirection::Forward, - )) - .unwrap(); - assert!(status_entry_iterator.next().is_none()); - - assert_eq!( - blockstore - .transaction_status_index_cf - .get(0) - .unwrap() - .unwrap(), - TransactionStatusIndexMeta { - max_slot: 0, - frozen: true, - } - ); - assert_eq!( - blockstore - .transaction_status_index_cf - .get(1) - .unwrap() - .unwrap(), - TransactionStatusIndexMeta { - max_slot: 0, - frozen: false, - } - ); - } - - #[test] - fn test_purge_special_columns_exact_no_sigs() { - let ledger_path = get_tmp_ledger_path_auto_delete!(); - let blockstore = Blockstore::open(ledger_path.path()).unwrap(); - - let slot = 1; - let mut entries: Vec = vec![]; - for x in 0..5 { - let mut tx = Transaction::new_unsigned(Message::default()); - tx.signatures = vec![]; - entries.push(next_entry_mut(&mut Hash::default(), 0, vec![tx])); - let mut tick = create_ticks(1, 0, hash(&serialize(&x).unwrap())); - entries.append(&mut tick); - } - let shreds = entries_to_test_shreds( - &entries, - slot, - slot - 1, // parent_slot - true, // is_full_slot - 0, // version - true, // merkle_variant - ); - blockstore.insert_shreds(shreds, None, false).unwrap(); - - let mut write_batch = blockstore.db.batch().unwrap(); - blockstore - .purge_special_columns_exact(&mut write_batch, slot, slot + 1) - .unwrap(); - } - - #[test] - fn test_purge_special_columns_compaction_filter() { - let ledger_path = get_tmp_ledger_path_auto_delete!(); - let blockstore = Blockstore::open(ledger_path.path()).unwrap(); - let index0_max_slot = 9; - let index1_max_slot = 19; - // includes slot 0, and slot 9 has 2 transactions - let num_total_transactions = index1_max_slot + 2; - - clear_and_repopulate_transaction_statuses_for_test( - &blockstore, - index0_max_slot, - index1_max_slot, - ); - let first_index = { - let mut status_entry_iterator = blockstore - .db - .iter::(IteratorMode::Start) - .unwrap(); - status_entry_iterator.next().unwrap().0 - }; - let last_index = { - let mut status_entry_iterator = blockstore - .db - .iter::(IteratorMode::End) - .unwrap(); - status_entry_iterator.next().unwrap().0 - }; - - let oldest_slot = 3; - blockstore.db.set_oldest_slot(oldest_slot); - blockstore.db.compact_range_cf::( - &cf::TransactionStatus::key(first_index), - &cf::TransactionStatus::key(last_index), - ); - - let status_entry_iterator = blockstore - .db - .iter::(IteratorMode::Start) - .unwrap(); - let mut count = 0; - for ((_primary_index, _signature, slot), _value) in status_entry_iterator { - assert!(slot >= oldest_slot); - count += 1; - } - assert_eq!(count, num_total_transactions - oldest_slot); - - clear_and_repopulate_transaction_statuses_for_test( - &blockstore, - index0_max_slot, - index1_max_slot, - ); - let first_index = { - let mut status_entry_iterator = blockstore - .db - .iter::(IteratorMode::Start) - .unwrap(); - status_entry_iterator.next().unwrap().0 - }; - let last_index = { - let mut status_entry_iterator = blockstore - .db - .iter::(IteratorMode::End) - .unwrap(); - status_entry_iterator.next().unwrap().0 - }; - - let oldest_slot = 12; - blockstore.db.set_oldest_slot(oldest_slot); - blockstore.db.compact_range_cf::( - &cf::TransactionStatus::key(first_index), - &cf::TransactionStatus::key(last_index), - ); - - let status_entry_iterator = blockstore - .db - .iter::(IteratorMode::Start) - .unwrap(); - let mut count = 0; - for ((_primary_index, _signature, slot), _value) in status_entry_iterator { - assert!(slot >= oldest_slot); - count += 1; - } - assert_eq!(count, num_total_transactions - oldest_slot - 1); // Extra transaction in slot 9 - } + // #[test] + // #[allow(clippy::cognitive_complexity)] + // fn test_purge_transaction_status_exact() { + // let ledger_path = get_tmp_ledger_path_auto_delete!(); + // let blockstore = Blockstore::open(ledger_path.path()).unwrap(); + // + // let index0_max_slot = 9; + // let index1_max_slot = 19; + // + // // Test purge outside bounds + // clear_and_repopulate_transaction_statuses_for_test( + // &blockstore, + // index0_max_slot, + // index1_max_slot, + // ); + // blockstore.run_purge(20, 22, PurgeType::Exact).unwrap(); + // + // let mut status_entry_iterator = blockstore + // .db + // .iter::(IteratorMode::From( + // cf::TransactionStatus::as_index(0), + // IteratorDirection::Forward, + // )) + // .unwrap(); + // assert_eq!( + // blockstore + // .transaction_status_index_cf + // .get(0) + // .unwrap() + // .unwrap(), + // TransactionStatusIndexMeta { + // max_slot: index0_max_slot, + // frozen: true, + // } + // ); + // for _ in 0..index0_max_slot + 1 { + // let entry = status_entry_iterator.next().unwrap().0; + // assert_eq!(entry.0, 0); + // } + // assert_eq!( + // blockstore + // .transaction_status_index_cf + // .get(1) + // .unwrap() + // .unwrap(), + // TransactionStatusIndexMeta { + // max_slot: index1_max_slot, + // frozen: false, + // } + // ); + // for _ in index0_max_slot + 1..index1_max_slot + 1 { + // let entry = status_entry_iterator.next().unwrap().0; + // assert_eq!(entry.0, 1); + // } + // drop(status_entry_iterator); + // + // // Test purge inside index 0 + // clear_and_repopulate_transaction_statuses_for_test( + // &blockstore, + // index0_max_slot, + // index1_max_slot, + // ); + // blockstore.run_purge(2, 4, PurgeType::Exact).unwrap(); + // + // let mut status_entry_iterator = blockstore + // .db + // .iter::(IteratorMode::From( + // cf::TransactionStatus::as_index(0), + // IteratorDirection::Forward, + // )) + // .unwrap(); + // assert_eq!( + // blockstore + // .transaction_status_index_cf + // .get(0) + // .unwrap() + // .unwrap(), + // TransactionStatusIndexMeta { + // max_slot: index0_max_slot, + // frozen: true, + // } + // ); + // for _ in 0..7 { + // // 7 entries remaining + // let entry = status_entry_iterator.next().unwrap().0; + // assert_eq!(entry.0, 0); + // assert!(entry.2 < 2 || entry.2 > 4); + // } + // assert_eq!( + // blockstore + // .transaction_status_index_cf + // .get(1) + // .unwrap() + // .unwrap(), + // TransactionStatusIndexMeta { + // max_slot: index1_max_slot, + // frozen: false, + // } + // ); + // for _ in index0_max_slot + 1..index1_max_slot + 1 { + // let entry = status_entry_iterator.next().unwrap().0; + // assert_eq!(entry.0, 1); + // } + // drop(status_entry_iterator); + // + // // Test purge inside index 0 at upper boundary + // clear_and_repopulate_transaction_statuses_for_test( + // &blockstore, + // index0_max_slot, + // index1_max_slot, + // ); + // blockstore + // .run_purge(7, index0_max_slot, PurgeType::Exact) + // .unwrap(); + // + // let mut status_entry_iterator = blockstore + // .db + // .iter::(IteratorMode::From( + // cf::TransactionStatus::as_index(0), + // IteratorDirection::Forward, + // )) + // .unwrap(); + // assert_eq!( + // blockstore + // .transaction_status_index_cf + // .get(0) + // .unwrap() + // .unwrap(), + // TransactionStatusIndexMeta { + // max_slot: 6, + // frozen: true, + // } + // ); + // for _ in 0..7 { + // // 7 entries remaining + // let entry = status_entry_iterator.next().unwrap().0; + // assert_eq!(entry.0, 0); + // assert!(entry.2 < 7); + // } + // assert_eq!( + // blockstore + // .transaction_status_index_cf + // .get(1) + // .unwrap() + // .unwrap(), + // TransactionStatusIndexMeta { + // max_slot: index1_max_slot, + // frozen: false, + // } + // ); + // for _ in index0_max_slot + 1..index1_max_slot + 1 { + // let entry = status_entry_iterator.next().unwrap().0; + // assert_eq!(entry.0, 1); + // } + // drop(status_entry_iterator); + // + // // Test purge inside index 1 at lower boundary + // clear_and_repopulate_transaction_statuses_for_test( + // &blockstore, + // index0_max_slot, + // index1_max_slot, + // ); + // blockstore.run_purge(10, 12, PurgeType::Exact).unwrap(); + // + // let mut status_entry_iterator = blockstore + // .db + // .iter::(IteratorMode::From( + // cf::TransactionStatus::as_index(0), + // IteratorDirection::Forward, + // )) + // .unwrap(); + // assert_eq!( + // blockstore + // .transaction_status_index_cf + // .get(0) + // .unwrap() + // .unwrap(), + // TransactionStatusIndexMeta { + // max_slot: index0_max_slot, + // frozen: true, + // } + // ); + // for _ in 0..index0_max_slot + 1 { + // let entry = status_entry_iterator.next().unwrap().0; + // assert_eq!(entry.0, 0); + // } + // assert_eq!( + // blockstore + // .transaction_status_index_cf + // .get(1) + // .unwrap() + // .unwrap(), + // TransactionStatusIndexMeta { + // max_slot: index1_max_slot, + // frozen: false, + // } + // ); + // for _ in 13..index1_max_slot + 1 { + // let entry = status_entry_iterator.next().unwrap().0; + // assert_eq!(entry.0, 1); + // assert!(entry.2 == index0_max_slot || entry.2 > 12); + // } + // drop(status_entry_iterator); + // + // // Test purge across index boundaries + // clear_and_repopulate_transaction_statuses_for_test( + // &blockstore, + // index0_max_slot, + // index1_max_slot, + // ); + // blockstore.run_purge(7, 12, PurgeType::Exact).unwrap(); + // + // let mut status_entry_iterator = blockstore + // .db + // .iter::(IteratorMode::From( + // cf::TransactionStatus::as_index(0), + // IteratorDirection::Forward, + // )) + // .unwrap(); + // assert_eq!( + // blockstore + // .transaction_status_index_cf + // .get(0) + // .unwrap() + // .unwrap(), + // TransactionStatusIndexMeta { + // max_slot: 6, + // frozen: true, + // } + // ); + // for _ in 0..7 { + // // 7 entries remaining + // let entry = status_entry_iterator.next().unwrap().0; + // assert_eq!(entry.0, 0); + // assert!(entry.2 < 7); + // } + // assert_eq!( + // blockstore + // .transaction_status_index_cf + // .get(1) + // .unwrap() + // .unwrap(), + // TransactionStatusIndexMeta { + // max_slot: index1_max_slot, + // frozen: false, + // } + // ); + // for _ in 13..index1_max_slot + 1 { + // let entry = status_entry_iterator.next().unwrap().0; + // assert_eq!(entry.0, 1); + // assert!(entry.2 > 12); + // } + // drop(status_entry_iterator); + // + // // Test purge include complete index 1 + // clear_and_repopulate_transaction_statuses_for_test( + // &blockstore, + // index0_max_slot, + // index1_max_slot, + // ); + // blockstore.run_purge(7, 22, PurgeType::Exact).unwrap(); + // + // let mut status_entry_iterator = blockstore + // .db + // .iter::(IteratorMode::From( + // cf::TransactionStatus::as_index(0), + // IteratorDirection::Forward, + // )) + // .unwrap(); + // assert_eq!( + // blockstore + // .transaction_status_index_cf + // .get(0) + // .unwrap() + // .unwrap(), + // TransactionStatusIndexMeta { + // max_slot: 6, + // frozen: true, + // } + // ); + // for _ in 0..7 { + // // 7 entries remaining + // let entry = status_entry_iterator.next().unwrap().0; + // assert_eq!(entry.0, 0); + // assert!(entry.2 < 7); + // } + // assert_eq!( + // blockstore + // .transaction_status_index_cf + // .get(1) + // .unwrap() + // .unwrap(), + // TransactionStatusIndexMeta { + // max_slot: 6, + // frozen: false, + // } + // ); + // drop(status_entry_iterator); + // + // // Purge up to but not including index0_max_slot + // clear_and_repopulate_transaction_statuses_for_test( + // &blockstore, + // index0_max_slot, + // index1_max_slot, + // ); + // blockstore + // .run_purge(0, index0_max_slot - 1, PurgeType::Exact) + // .unwrap(); + // assert_eq!( + // blockstore + // .transaction_status_index_cf + // .get(0) + // .unwrap() + // .unwrap(), + // TransactionStatusIndexMeta { + // max_slot: index0_max_slot, + // frozen: true, + // } + // ); + // + // // Test purge all + // clear_and_repopulate_transaction_statuses_for_test( + // &blockstore, + // index0_max_slot, + // index1_max_slot, + // ); + // blockstore.run_purge(0, 22, PurgeType::Exact).unwrap(); + // + // let mut status_entry_iterator = blockstore + // .db + // .iter::(IteratorMode::From( + // cf::TransactionStatus::as_index(0), + // IteratorDirection::Forward, + // )) + // .unwrap(); + // assert!(status_entry_iterator.next().is_none()); + // + // assert_eq!( + // blockstore + // .transaction_status_index_cf + // .get(0) + // .unwrap() + // .unwrap(), + // TransactionStatusIndexMeta { + // max_slot: 0, + // frozen: true, + // } + // ); + // assert_eq!( + // blockstore + // .transaction_status_index_cf + // .get(1) + // .unwrap() + // .unwrap(), + // TransactionStatusIndexMeta { + // max_slot: 0, + // frozen: false, + // } + // ); + // } + // + // #[test] + // fn test_purge_special_columns_exact_no_sigs() { + // let ledger_path = get_tmp_ledger_path_auto_delete!(); + // let blockstore = Blockstore::open(ledger_path.path()).unwrap(); + // + // let slot = 1; + // let mut entries: Vec = vec![]; + // for x in 0..5 { + // let mut tx = Transaction::new_unsigned(Message::default()); + // tx.signatures = vec![]; + // entries.push(next_entry_mut(&mut Hash::default(), 0, vec![tx])); + // let mut tick = create_ticks(1, 0, hash(&serialize(&x).unwrap())); + // entries.append(&mut tick); + // } + // let shreds = entries_to_test_shreds( + // &entries, + // slot, + // slot - 1, // parent_slot + // true, // is_full_slot + // 0, // version + // true, // merkle_variant + // ); + // blockstore.insert_shreds(shreds, None, false).unwrap(); + // + // let mut write_batch = blockstore.db.batch().unwrap(); + // blockstore + // .purge_special_columns_exact(&mut write_batch, slot, slot + 1) + // .unwrap(); + // } + // + // #[test] + // fn test_purge_special_columns_compaction_filter() { + // let ledger_path = get_tmp_ledger_path_auto_delete!(); + // let blockstore = Blockstore::open(ledger_path.path()).unwrap(); + // let index0_max_slot = 9; + // let index1_max_slot = 19; + // // includes slot 0, and slot 9 has 2 transactions + // let num_total_transactions = index1_max_slot + 2; + // + // clear_and_repopulate_transaction_statuses_for_test( + // &blockstore, + // index0_max_slot, + // index1_max_slot, + // ); + // let first_index = { + // let mut status_entry_iterator = blockstore + // .db + // .iter::(IteratorMode::Start) + // .unwrap(); + // status_entry_iterator.next().unwrap().0 + // }; + // let last_index = { + // let mut status_entry_iterator = blockstore + // .db + // .iter::(IteratorMode::End) + // .unwrap(); + // status_entry_iterator.next().unwrap().0 + // }; + // + // let oldest_slot = 3; + // blockstore.db.set_oldest_slot(oldest_slot); + // blockstore.db.compact_range_cf::( + // &cf::TransactionStatus::key(first_index), + // &cf::TransactionStatus::key(last_index), + // ); + // + // let status_entry_iterator = blockstore + // .db + // .iter::(IteratorMode::Start) + // .unwrap(); + // let mut count = 0; + // for ((_primary_index, _signature, slot), _value) in status_entry_iterator { + // assert!(slot >= oldest_slot); + // count += 1; + // } + // assert_eq!(count, num_total_transactions - oldest_slot); + // + // clear_and_repopulate_transaction_statuses_for_test( + // &blockstore, + // index0_max_slot, + // index1_max_slot, + // ); + // let first_index = { + // let mut status_entry_iterator = blockstore + // .db + // .iter::(IteratorMode::Start) + // .unwrap(); + // status_entry_iterator.next().unwrap().0 + // }; + // let last_index = { + // let mut status_entry_iterator = blockstore + // .db + // .iter::(IteratorMode::End) + // .unwrap(); + // status_entry_iterator.next().unwrap().0 + // }; + // + // let oldest_slot = 12; + // blockstore.db.set_oldest_slot(oldest_slot); + // blockstore.db.compact_range_cf::( + // &cf::TransactionStatus::key(first_index), + // &cf::TransactionStatus::key(last_index), + // ); + // + // let status_entry_iterator = blockstore + // .db + // .iter::(IteratorMode::Start) + // .unwrap(); + // let mut count = 0; + // for ((_primary_index, _signature, slot), _value) in status_entry_iterator { + // assert!(slot >= oldest_slot); + // count += 1; + // } + // assert_eq!(count, num_total_transactions - oldest_slot - 1); // Extra transaction in slot 9 + // } } From f0699e177b17ea2b612fa11d6a9c3f08bec27d61 Mon Sep 17 00:00:00 2001 From: Tyera Eulberg Date: Mon, 25 Sep 2023 20:51:39 -0600 Subject: [PATCH 23/57] Impl ColumnIndexDeprecation for AddressSignatures (doesn't build) --- ledger/src/blockstore_db.rs | 76 ++++++++++++++++++++++++++++--------- 1 file changed, 59 insertions(+), 17 deletions(-) diff --git a/ledger/src/blockstore_db.rs b/ledger/src/blockstore_db.rs index a67f5a4d98a6d8..1b8e0aa9ee3633 100644 --- a/ledger/src/blockstore_db.rs +++ b/ledger/src/blockstore_db.rs @@ -277,7 +277,7 @@ pub mod columns { #[derive(Debug)] /// The address signatures column /// - /// * index type: `(u64, `[`Pubkey`]`, `[`Slot`]`, `[`Signature`]`)` + /// * index type: `(`[`Pubkey`]`, `[`Slot`]`, u32, `[`Signature`]`)` /// * value type: [`blockstore_meta::AddressSignatureMeta`] pub struct AddressSignatures; @@ -857,37 +857,79 @@ impl ColumnIndexDeprecation for columns::TransactionStatus { } impl Column for columns::AddressSignatures { - type Index = (u64, Pubkey, Slot, Signature); + type Index = (Pubkey, Slot, u32, Signature); - fn key((index, pubkey, slot, signature): (u64, Pubkey, Slot, Signature)) -> Vec { - let mut key = vec![0; 8 + 32 + 8 + 64]; // size_of u64 + size_of Pubkey + size_of Slot + size_of Signature - BigEndian::write_u64(&mut key[0..8], index); - key[8..40].copy_from_slice(&pubkey.as_ref()[0..32]); - BigEndian::write_u64(&mut key[40..48], slot); - key[48..112].copy_from_slice(&signature.as_ref()[0..64]); + fn key((pubkey, slot, transaction_index, signature): Self::Index) -> Vec { + let mut key = vec![0; Self::CURRENT_INDEX_LEN]; + key[0..32].copy_from_slice(&pubkey.as_ref()[0..32]); + BigEndian::write_u64(&mut key[32..40], slot); + BigEndian::write_u32(&mut key[40..44], transaction_index); + key[44..108].copy_from_slice(&signature.as_ref()[0..64]); key } - fn index(key: &[u8]) -> (u64, Pubkey, Slot, Signature) { - let index = BigEndian::read_u64(&key[0..8]); - let pubkey = Pubkey::try_from(&key[8..40]).unwrap(); - let slot = BigEndian::read_u64(&key[40..48]); - let signature = Signature::try_from(&key[48..112]).unwrap(); - (index, pubkey, slot, signature) + fn index(key: &[u8]) -> Self::Index { + ::index(key) } fn slot(index: Self::Index) -> Slot { - index.2 + index.1 } - fn as_index(index: u64) -> Self::Index { - (index, Pubkey::default(), 0, Signature::default()) + // This trait method is primarily used by Database::delete_range_cf(). Because the + // AddressSignatures Column is not keyed by slot, it cannot be range deleted. Therefore this + // method is meaningless for this type, and simply returns a default Index. + fn as_index(_index: u64) -> Self::Index { + (Pubkey::default(), 0, 0, Signature::default()) } } impl ColumnName for columns::AddressSignatures { const NAME: &'static str = ADDRESS_SIGNATURES_CF; } +impl ColumnIndexDeprecation for columns::AddressSignatures { + const DEPRECATED_INDEX_LEN: usize = 112; + const CURRENT_INDEX_LEN: usize = 108; + type DeprecatedIndex = (u64, Pubkey, Slot, Signature); + + fn deprecated_key(index: Self::DeprecatedIndex) -> Vec { + let (primary_index, pubkey, slot, signature) = index; + let mut key = vec![0; Self::DEPRECATED_INDEX_LEN]; + BigEndian::write_u64(&mut key[0..8], primary_index); + key[8..40].clone_from_slice(&pubkey.as_ref()[0..32]); + BigEndian::write_u64(&mut key[40..48], slot); + key[48..112].clone_from_slice(&signature.as_ref()[0..64]); + key + } + + fn try_deprecated_index(key: &[u8]) -> std::result::Result { + if key.len() != Self::DEPRECATED_INDEX_LEN { + return Err(IndexError::UnpackError); + } + let primary_index = BigEndian::read_u64(&key[0..8]); + let pubkey = Pubkey::try_from(&key[8..40]).unwrap(); + let slot = BigEndian::read_u64(&key[40..48]); + let signature = Signature::try_from(&key[48..112]).unwrap(); + Ok((primary_index, pubkey, slot, signature)) + } + + fn try_current_index(key: &[u8]) -> std::result::Result { + if key.len() != Self::CURRENT_INDEX_LEN { + return Err(IndexError::UnpackError); + } + let pubkey = Pubkey::try_from(&key[0..32]).unwrap(); + let slot = BigEndian::read_u64(&key[32..40]); + let transaction_index = BigEndian::read_u32(&key[40..44]); + let signature = Signature::try_from(&key[44..108]).unwrap(); + Ok((pubkey, slot, transaction_index, signature)) + } + + fn convert_index(deprecated_index: Self::DeprecatedIndex) -> Self::Index { + let (_primary_index, pubkey, slot, signature) = deprecated_index; + (pubkey, slot, 0, signature) + } +} + impl Column for columns::TransactionMemos { type Index = Signature; From b448ba79d6888e23b1d0dbaaf58db86027c917a7 Mon Sep 17 00:00:00 2001 From: Tyera Eulberg Date: Tue, 26 Sep 2023 17:45:34 -0600 Subject: [PATCH 24/57] Add BlockstoreError variant --- ledger/src/blockstore_db.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ledger/src/blockstore_db.rs b/ledger/src/blockstore_db.rs index 1b8e0aa9ee3633..1a5010dadb9779 100644 --- a/ledger/src/blockstore_db.rs +++ b/ledger/src/blockstore_db.rs @@ -146,6 +146,8 @@ pub enum BlockstoreError { UnsupportedTransactionVersion, #[error("missing transaction metadata")] MissingTransactionMetadata, + #[error("transaction-index overflow")] + TransactionIndexOverflow, } pub type Result = std::result::Result; From 869207273dae0a71c0168b6ccd26f77f420e9b37 Mon Sep 17 00:00:00 2001 From: Tyera Eulberg Date: Mon, 25 Sep 2023 21:12:24 -0600 Subject: [PATCH 25/57] Update AddressSignatures put --- ledger/src/blockstore.rs | 7 +++++-- rpc/src/transaction_status_service.rs | 1 + 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/ledger/src/blockstore.rs b/ledger/src/blockstore.rs index 19d7a22a9d9e93..2331b013b02208 100644 --- a/ledger/src/blockstore.rs +++ b/ledger/src/blockstore.rs @@ -2267,8 +2267,11 @@ impl Blockstore { writable_keys: Vec<&Pubkey>, readonly_keys: Vec<&Pubkey>, status: TransactionStatusMeta, + transaction_index: usize, ) -> Result<()> { let status = status.into(); + let transaction_index = u32::try_from(transaction_index) + .map_err(|_| BlockstoreError::TransactionIndexOverflow)?; // This write lock prevents interleaving issues with the transaction_status_index_cf by gating // writes to that column let w_active_transaction_status_index = @@ -2279,13 +2282,13 @@ impl Blockstore { .put_protobuf((signature, slot), &status)?; for address in writable_keys { self.address_signatures_cf.put( - (primary_index, *address, slot, signature), + (*address, slot, transaction_index, signature), &AddressSignatureMeta { writeable: true }, )?; } for address in readonly_keys { self.address_signatures_cf.put( - (primary_index, *address, slot, signature), + (*address, slot, transaction_index, signature), &AddressSignatureMeta { writeable: false }, )?; } diff --git a/rpc/src/transaction_status_service.rs b/rpc/src/transaction_status_service.rs index c4c619a1a32568..aef1ab45f14260 100644 --- a/rpc/src/transaction_status_service.rs +++ b/rpc/src/transaction_status_service.rs @@ -199,6 +199,7 @@ impl TransactionStatusService { tx_account_locks.writable, tx_account_locks.readonly, transaction_status_meta, + transaction_index, ) .expect("Expect database write to succeed: TransactionStatus"); } From 2f7eeae8c5f8fe6d3d7c0889e492b6701a768eec Mon Sep 17 00:00:00 2001 From: Tyera Eulberg Date: Tue, 26 Sep 2023 17:47:45 -0600 Subject: [PATCH 26/57] Remove unneeded active_transaction_status_index column lock --- ledger/src/blockstore.rs | 24 +----------------------- 1 file changed, 1 insertion(+), 23 deletions(-) diff --git a/ledger/src/blockstore.rs b/ledger/src/blockstore.rs index 2331b013b02208..475daa0be37943 100644 --- a/ledger/src/blockstore.rs +++ b/ledger/src/blockstore.rs @@ -74,7 +74,7 @@ use { rc::Rc, sync::{ atomic::{AtomicBool, Ordering}, - Arc, Mutex, RwLock, RwLockWriteGuard, + Arc, Mutex, RwLock, }, }, tempfile::{Builder, TempDir}, @@ -2210,22 +2210,6 @@ impl Blockstore { } } - fn get_primary_index_to_write( - &self, - slot: Slot, - // take WriteGuard to require critical section semantics at call site - w_active_transaction_status_index: &RwLockWriteGuard, - ) -> Result { - let i = **w_active_transaction_status_index; - let mut index_meta = self.transaction_status_index_cf.get(i)?.unwrap(); - if slot > index_meta.max_slot { - assert!(!index_meta.frozen); - index_meta.max_slot = slot; - self.transaction_status_index_cf.put(i, &index_meta)?; - } - Ok(i) - } - fn read_deprecated_transaction_status( &self, index: (Signature, Slot), @@ -2272,12 +2256,6 @@ impl Blockstore { let status = status.into(); let transaction_index = u32::try_from(transaction_index) .map_err(|_| BlockstoreError::TransactionIndexOverflow)?; - // This write lock prevents interleaving issues with the transaction_status_index_cf by gating - // writes to that column - let w_active_transaction_status_index = - self.active_transaction_status_index.write().unwrap(); - let primary_index = - self.get_primary_index_to_write(slot, &w_active_transaction_status_index)?; self.transaction_status_cf .put_protobuf((signature, slot), &status)?; for address in writable_keys { From f049d4a6e4e3d0b1630edaf3c3122cbb8aca9474 Mon Sep 17 00:00:00 2001 From: Tyera Eulberg Date: Mon, 25 Sep 2023 21:08:39 -0600 Subject: [PATCH 27/57] Update AddressSignatures purge_exact --- ledger/src/blockstore/blockstore_purge.rs | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/ledger/src/blockstore/blockstore_purge.rs b/ledger/src/blockstore/blockstore_purge.rs index 2d52ff77d6449f..3ef9307d0fb6e2 100644 --- a/ledger/src/blockstore/blockstore_purge.rs +++ b/ledger/src/blockstore/blockstore_purge.rs @@ -403,7 +403,7 @@ impl Blockstore { let transactions = slot_entries .into_iter() .flat_map(|entry| entry.transactions); - for transaction in transactions { + for (i, transaction) in transactions.enumerate() { if let Some(&signature) = transaction.signatures.get(0) { if delete_new_column_key { batch.delete::((signature, slot))?; @@ -425,15 +425,27 @@ impl Blockstore { loaded_addresses.as_ref(), ); + let transaction_index = + u32::try_from(i).map_err(|_| BlockstoreError::TransactionIndexOverflow)?; for pubkey in account_keys.iter() { - for primary_index in &primary_indexes { + if delete_new_column_key { batch.delete::(( - *primary_index, *pubkey, slot, + transaction_index, signature, ))?; } + for primary_index in &primary_indexes { + batch.delete_raw::( + &cf::AddressSignatures::deprecated_key(( + *primary_index, + *pubkey, + slot, + signature, + )), + )?; + } } } } From 0af030da2f390c904cbe60c4b3bc3405de1d6655 Mon Sep 17 00:00:00 2001 From: Tyera Eulberg Date: Mon, 25 Sep 2023 21:39:37 -0600 Subject: [PATCH 28/57] Fix find_address_signatures_for_slot methods --- ledger/src/blockstore.rs | 75 ++++++++++++++++++++++++++++++---------- 1 file changed, 56 insertions(+), 19 deletions(-) diff --git a/ledger/src/blockstore.rs b/ledger/src/blockstore.rs index 475daa0be37943..def6b2d4905f37 100644 --- a/ledger/src/blockstore.rs +++ b/ledger/src/blockstore.rs @@ -2504,15 +2504,17 @@ impl Blockstore { return Ok(signatures); } for transaction_status_cf_primary_index in 0..=1 { - let index_iterator = self.address_signatures_cf.iter(IteratorMode::From( - ( - transaction_status_cf_primary_index, - pubkey, - start_slot.max(lowest_available_slot), - Signature::default(), - ), - IteratorDirection::Forward, - ))?; + let index_iterator = + self.address_signatures_cf + .iter_deprecated_index_filtered(IteratorMode::From( + ( + transaction_status_cf_primary_index, + pubkey, + start_slot.max(lowest_available_slot), + Signature::default(), + ), + IteratorDirection::Forward, + ))?; for ((i, address, slot, signature), _) in index_iterator { if i != transaction_status_cf_primary_index || slot > end_slot || address != pubkey { @@ -2531,7 +2533,7 @@ impl Blockstore { // Returns all signatures for an address in a particular slot, regardless of whether that slot // has been rooted. The transactions will be ordered by signature, and NOT by the order in // which the transactions exist in the block - fn find_address_signatures_for_slot( + fn find_address_signatures_for_slot_with_primary_index( &self, pubkey: Pubkey, slot: Slot, @@ -2542,15 +2544,17 @@ impl Blockstore { return Ok(signatures); } for transaction_status_cf_primary_index in 0..=1 { - let index_iterator = self.address_signatures_cf.iter(IteratorMode::From( - ( - transaction_status_cf_primary_index, - pubkey, - slot, - Signature::default(), - ), - IteratorDirection::Forward, - ))?; + let index_iterator = + self.address_signatures_cf + .iter_deprecated_index_filtered(IteratorMode::From( + ( + transaction_status_cf_primary_index, + pubkey, + slot, + Signature::default(), + ), + IteratorDirection::Forward, + ))?; for ((i, address, transaction_slot, signature), _) in index_iterator { if i != transaction_status_cf_primary_index || transaction_slot > slot @@ -2566,6 +2570,39 @@ impl Blockstore { Ok(signatures) } + // Returns all signatures for an address in a particular slot, regardless of whether that slot + // has been rooted. The transactions will be ordered by their occurrence in the block + fn find_address_signatures_for_slot( + &self, + pubkey: Pubkey, + slot: Slot, + ) -> Result> { + let (lock, lowest_available_slot) = self.ensure_lowest_cleanup_slot(); + let mut signatures: Vec<(Slot, Signature)> = vec![]; + if slot < lowest_available_slot { + return Ok(signatures); + } + let index_iterator = self + .address_signatures_cf + .iter_filtered(IteratorMode::From( + ( + pubkey, + slot.max(lowest_available_slot), + 0, + Signature::default(), + ), + IteratorDirection::Forward, + ))?; + for ((address, transaction_slot, _transaction_index, signature), _) in index_iterator { + if transaction_slot > slot || address != pubkey { + break; + } + signatures.push((slot, signature)); + } + drop(lock); + Ok(signatures) + } + // DEPRECATED pub fn get_confirmed_signatures_for_address( &self, From fbe20bf322d126d340dbeecee80048617c6f0da4 Mon Sep 17 00:00:00 2001 From: Tyera Eulberg Date: Mon, 25 Sep 2023 22:03:28 -0600 Subject: [PATCH 29/57] Fix get_block_signatures methods --- ledger/src/blockstore.rs | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/ledger/src/blockstore.rs b/ledger/src/blockstore.rs index def6b2d4905f37..b8ba96b4960ba2 100644 --- a/ledger/src/blockstore.rs +++ b/ledger/src/blockstore.rs @@ -2618,7 +2618,21 @@ impl Blockstore { .map(|signatures| signatures.iter().map(|(_, signature)| *signature).collect()) } + // Get all signatures in a block, sorted to match the deprecated ordering of the + // AddressSignatures column fn get_sorted_block_signatures(&self, slot: Slot) -> Result> { + // Load all signatures for the block + let mut slot_signatures: Vec<_> = self.get_block_signatures(slot)?; + + // Reverse sort signatures as a way to entire a stable ordering within a slot, as + // the AddressSignatures column is ordered by signatures within a slot, + // not by block ordering + slot_signatures.sort_unstable_by(|a, b| b.cmp(a)); + + Ok(slot_signatures) + } + + fn get_block_signatures(&self, slot: Slot) -> Result> { let block = self.get_complete_block(slot, false).map_err(|err| { BlockstoreError::Io(IoError::new( ErrorKind::Other, @@ -2626,8 +2640,7 @@ impl Blockstore { )) })?; - // Load all signatures for the block - let mut slot_signatures: Vec<_> = block + Ok(block .transactions .into_iter() .filter_map(|transaction_with_meta| { @@ -2637,14 +2650,7 @@ impl Blockstore { .into_iter() .next() }) - .collect(); - - // Reverse sort signatures as a way to entire a stable ordering within a slot, as - // the AddressSignatures column is ordered by signatures within a slot, - // not by block ordering - slot_signatures.sort_unstable_by(|a, b| b.cmp(a)); - - Ok(slot_signatures) + .collect()) } pub fn get_confirmed_signatures_for_address2( From 0961f2209e98a52b6f6ae92d7b949fc4eff8d785 Mon Sep 17 00:00:00 2001 From: Tyera Eulberg Date: Tue, 26 Sep 2023 18:40:08 -0600 Subject: [PATCH 30/57] Fix get_confirmed_signatures_for_address2 --- ledger/src/blockstore.rs | 126 ++++++--------------------------------- 1 file changed, 18 insertions(+), 108 deletions(-) diff --git a/ledger/src/blockstore.rs b/ledger/src/blockstore.rs index b8ba96b4960ba2..bf7ee5a2e3de8c 100644 --- a/ledger/src/blockstore.rs +++ b/ledger/src/blockstore.rs @@ -2530,46 +2530,6 @@ impl Blockstore { Ok(signatures) } - // Returns all signatures for an address in a particular slot, regardless of whether that slot - // has been rooted. The transactions will be ordered by signature, and NOT by the order in - // which the transactions exist in the block - fn find_address_signatures_for_slot_with_primary_index( - &self, - pubkey: Pubkey, - slot: Slot, - ) -> Result> { - let (lock, lowest_available_slot) = self.ensure_lowest_cleanup_slot(); - let mut signatures: Vec<(Slot, Signature)> = vec![]; - if slot < lowest_available_slot { - return Ok(signatures); - } - for transaction_status_cf_primary_index in 0..=1 { - let index_iterator = - self.address_signatures_cf - .iter_deprecated_index_filtered(IteratorMode::From( - ( - transaction_status_cf_primary_index, - pubkey, - slot, - Signature::default(), - ), - IteratorDirection::Forward, - ))?; - for ((i, address, transaction_slot, signature), _) in index_iterator { - if i != transaction_status_cf_primary_index - || transaction_slot > slot - || address != pubkey - { - break; - } - signatures.push((slot, signature)); - } - } - drop(lock); - signatures.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap().then(a.1.cmp(&b.1))); - Ok(signatures) - } - // Returns all signatures for an address in a particular slot, regardless of whether that slot // has been rooted. The transactions will be ordered by their occurrence in the block fn find_address_signatures_for_slot( @@ -2683,7 +2643,8 @@ impl Blockstore { match transaction_status { None => return Ok(SignatureInfosForAddress::default()), Some((slot, _)) => { - let mut slot_signatures = self.get_sorted_block_signatures(slot)?; + let mut slot_signatures = self.get_block_signatures(slot)?; + slot_signatures.reverse(); if let Some(pos) = slot_signatures.iter().position(|&x| x == before) { slot_signatures.truncate(pos + 1); } @@ -2710,7 +2671,8 @@ impl Blockstore { match transaction_status { None => (first_available_block, HashSet::new()), Some((slot, _)) => { - let mut slot_signatures = self.get_sorted_block_signatures(slot)?; + let mut slot_signatures = self.get_block_signatures(slot)?; + slot_signatures.reverse(); if let Some(pos) = slot_signatures.iter().position(|&x| x == until) { slot_signatures = slot_signatures.split_off(pos); } @@ -2740,65 +2702,21 @@ impl Blockstore { } get_initial_slot_timer.stop(); - // Check the active_transaction_status_index to see if it contains slot. If so, start with - // that index, as it will contain higher slots - let starting_primary_index = *self.active_transaction_status_index.read().unwrap(); - let next_primary_index = u64::from(starting_primary_index == 0); - let next_max_slot = self - .transaction_status_index_cf - .get(next_primary_index)? - .unwrap() - .max_slot; - - let mut starting_primary_index_iter_timer = Measure::start("starting_primary_index_iter"); - if slot > next_max_slot { - let mut starting_iterator = self.address_signatures_cf.iter(IteratorMode::From( - (starting_primary_index, address, slot, Signature::default()), + let mut address_signatures_iter_timer = Measure::start("iter_timer"); + let mut iterator = self + .address_signatures_cf + .iter_filtered(IteratorMode::From( + (address, slot, 0, Signature::default()), IteratorDirection::Reverse, ))?; - // Iterate through starting_iterator until limit is reached - while address_signatures.len() < limit { - if let Some(((i, key_address, slot, signature), _)) = starting_iterator.next() { - if slot == next_max_slot || slot < lowest_slot { - break; - } - if i == starting_primary_index && key_address == address { - if self.is_root(slot) || confirmed_unrooted_slots.contains(&slot) { - address_signatures.push((slot, signature)); - } - continue; - } - } - break; - } - - // Handle slots that cross primary indexes - if next_max_slot >= lowest_slot { - let mut signatures = - self.find_address_signatures_for_slot(address, next_max_slot)?; - signatures.reverse(); - address_signatures.append(&mut signatures); - } - } - starting_primary_index_iter_timer.stop(); - - // Iterate through next_iterator until limit is reached - let mut next_primary_index_iter_timer = Measure::start("next_primary_index_iter_timer"); - let mut next_iterator = self.address_signatures_cf.iter(IteratorMode::From( - (next_primary_index, address, slot, Signature::default()), - IteratorDirection::Reverse, - ))?; + // Iterate through starting_iterator until limit is reached while address_signatures.len() < limit { - if let Some(((i, key_address, slot, signature), _)) = next_iterator.next() { - // Skip next_max_slot, which is already included - if slot == next_max_slot { - continue; - } + if let Some(((key_address, slot, _transaction_index, signature), _)) = iterator.next() { if slot < lowest_slot { break; } - if i == next_primary_index && key_address == address { + if key_address == address { if self.is_root(slot) || confirmed_unrooted_slots.contains(&slot) { address_signatures.push((slot, signature)); } @@ -2807,7 +2725,8 @@ impl Blockstore { } break; } - next_primary_index_iter_timer.stop(); + address_signatures_iter_timer.stop(); + let mut address_signatures: Vec<(Slot, Signature)> = address_signatures .into_iter() .filter(|(_, signature)| !until_excluded_signatures.contains(signature)) @@ -2846,13 +2765,8 @@ impl Blockstore { i64 ), ( - "starting_primary_index_iter_us", - starting_primary_index_iter_timer.as_us() as i64, - i64 - ), - ( - "next_primary_index_iter_us", - next_primary_index_iter_timer.as_us() as i64, + "address_signatures_iter_us", + address_signatures_iter_timer.as_us() as i64, i64 ), ( @@ -8555,8 +8469,7 @@ pub mod tests { assert_eq!(results[2], all0[i + 2]); } - // Ensure that the signatures within a slot are reverse ordered by signature - // (current limitation of the .get_confirmed_signatures_for_address2()) + // Ensure that the signatures within a slot are reverse ordered by occurrence in block for i in (0..all1.len()).step_by(2) { let results = blockstore .get_confirmed_signatures_for_address2( @@ -8574,7 +8487,6 @@ pub mod tests { .infos; assert_eq!(results.len(), 2); assert_eq!(results[0].slot, results[1].slot); - assert!(results[0].signature >= results[1].signature); assert_eq!(results[0], all1[i]); assert_eq!(results[1], all1[i + 1]); } @@ -8730,8 +8642,7 @@ pub mod tests { assert_eq!(results[1], all0[i + 1]); } - // Ensure that the signatures within a slot are reverse ordered by signature - // (current limitation of the .get_confirmed_signatures_for_address2()) + // Ensure that the signatures within a slot are reverse ordered by occurrence in block for i in (0..all1.len()).step_by(2) { let results = blockstore .get_confirmed_signatures_for_address2( @@ -8749,7 +8660,6 @@ pub mod tests { .infos; assert_eq!(results.len(), 2); assert_eq!(results[0].slot, results[1].slot); - assert!(results[0].signature >= results[1].signature); assert_eq!(results[0], all1[i]); assert_eq!(results[1], all1[i + 1]); } From 74455d3d6b25a520434c7a274a1c2e0aa4ca78af Mon Sep 17 00:00:00 2001 From: Tyera Eulberg Date: Tue, 3 Oct 2023 21:40:53 -0600 Subject: [PATCH 31/57] Remove unused method --- ledger/src/blockstore.rs | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/ledger/src/blockstore.rs b/ledger/src/blockstore.rs index bf7ee5a2e3de8c..f8326bd1228edf 100644 --- a/ledger/src/blockstore.rs +++ b/ledger/src/blockstore.rs @@ -2578,20 +2578,6 @@ impl Blockstore { .map(|signatures| signatures.iter().map(|(_, signature)| *signature).collect()) } - // Get all signatures in a block, sorted to match the deprecated ordering of the - // AddressSignatures column - fn get_sorted_block_signatures(&self, slot: Slot) -> Result> { - // Load all signatures for the block - let mut slot_signatures: Vec<_> = self.get_block_signatures(slot)?; - - // Reverse sort signatures as a way to entire a stable ordering within a slot, as - // the AddressSignatures column is ordered by signatures within a slot, - // not by block ordering - slot_signatures.sort_unstable_by(|a, b| b.cmp(a)); - - Ok(slot_signatures) - } - fn get_block_signatures(&self, slot: Slot) -> Result> { let block = self.get_complete_block(slot, false).map_err(|err| { BlockstoreError::Io(IoError::new( From b5ae9c02ae455f6af41697fb81b50b5201db9180 Mon Sep 17 00:00:00 2001 From: Tyera Eulberg Date: Mon, 25 Sep 2023 21:25:55 -0600 Subject: [PATCH 32/57] Fix test_all_empty_or_min moar (builds except tests) --- ledger/src/blockstore.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ledger/src/blockstore.rs b/ledger/src/blockstore.rs index f8326bd1228edf..be245d7f633451 100644 --- a/ledger/src/blockstore.rs +++ b/ledger/src/blockstore.rs @@ -4453,9 +4453,7 @@ pub fn test_all_empty_or_min(blockstore: &Blockstore, min_slot: Slot) { .iter::(IteratorMode::Start) .unwrap() .next() - .map(|((primary_index, _, slot, _), _)| { - slot >= min_slot || (primary_index == 2 && slot == 0) - }) + .map(|((_, slot, _, _), _)| slot >= min_slot || slot == 0) .unwrap_or(true) & blockstore .db From 85be03677ff869d227b60d48d7b5fe1e49fe1731 Mon Sep 17 00:00:00 2001 From: Tyera Eulberg Date: Tue, 26 Sep 2023 18:58:48 -0600 Subject: [PATCH 33/57] Fix tests (all build) --- ledger/src/blockstore.rs | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/ledger/src/blockstore.rs b/ledger/src/blockstore.rs index be245d7f633451..fb2573f97eb8f4 100644 --- a/ledger/src/blockstore.rs +++ b/ledger/src/blockstore.rs @@ -8045,6 +8045,7 @@ pub mod tests { vec![&address0], vec![&address1], TransactionStatusMeta::default(), + x as usize, ) .unwrap(); } @@ -8058,6 +8059,7 @@ pub mod tests { vec![&address0], vec![&address1], TransactionStatusMeta::default(), + x as usize, ) .unwrap(); } @@ -8151,6 +8153,7 @@ pub mod tests { vec![&address0], vec![&address1], TransactionStatusMeta::default(), + 0, ) .unwrap(); } @@ -8180,6 +8183,7 @@ pub mod tests { vec![&address0], vec![&address1], TransactionStatusMeta::default(), + x as usize, ) .unwrap(); } @@ -8193,6 +8197,7 @@ pub mod tests { vec![&address0], vec![&address1], TransactionStatusMeta::default(), + x as usize, ) .unwrap(); } @@ -8205,6 +8210,7 @@ pub mod tests { vec![&address0], vec![&address1], TransactionStatusMeta::default(), + x as usize, ) .unwrap(); } @@ -8218,6 +8224,7 @@ pub mod tests { vec![&address0], vec![&address1], TransactionStatusMeta::default(), + x as usize, ) .unwrap(); } @@ -8290,6 +8297,7 @@ pub mod tests { ); blockstore.insert_shreds(shreds, None, false).unwrap(); + let mut counter = 0; for entry in entries.into_iter() { for transaction in entry.transactions { assert_eq!(transaction.signatures.len(), 1); @@ -8300,8 +8308,10 @@ pub mod tests { transaction.message.static_account_keys().iter().collect(), vec![], TransactionStatusMeta::default(), + counter, ) .unwrap(); + counter += 1; } } } @@ -8315,6 +8325,7 @@ pub mod tests { entries_to_test_shreds(&entries, slot, 8, true, 0, /*merkle_variant:*/ true); blockstore.insert_shreds(shreds, None, false).unwrap(); + let mut counter = 0; for entry in entries.into_iter() { for transaction in entry.transactions { assert_eq!(transaction.signatures.len(), 1); @@ -8325,8 +8336,10 @@ pub mod tests { transaction.message.static_account_keys().iter().collect(), vec![], TransactionStatusMeta::default(), + counter, ) .unwrap(); + counter += 1; } } } @@ -8678,7 +8691,7 @@ pub mod tests { // Remove signature blockstore .address_signatures_cf - .delete((0, address0, 2, all0[0].signature)) + .delete((address0, 2, 0, all0[0].signature)) .unwrap(); let sig_infos = blockstore .get_confirmed_signatures_for_address2( From 812ddc3375e771d03660b95cbe28da28c7b9a2db Mon Sep 17 00:00:00 2001 From: Tyera Eulberg Date: Thu, 28 Sep 2023 18:22:01 -0600 Subject: [PATCH 34/57] Fix test_get_confirmed_signatures_for_address --- ledger/src/blockstore.rs | 34 ++++++++++++++++++++++++++++++---- ledger/src/blockstore_db.rs | 27 +++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 4 deletions(-) diff --git a/ledger/src/blockstore.rs b/ledger/src/blockstore.rs index fb2573f97eb8f4..4ca532a5a416af 100644 --- a/ledger/src/blockstore.rs +++ b/ledger/src/blockstore.rs @@ -8029,6 +8029,34 @@ pub mod tests { #[test] fn test_get_confirmed_signatures_for_address() { + impl Blockstore { + fn write_deprecated_transaction_status( + &self, + slot: Slot, + signature: Signature, + writable_keys: Vec<&Pubkey>, + readonly_keys: Vec<&Pubkey>, + status: TransactionStatusMeta, + ) -> Result<()> { + let status = status.into(); + self.transaction_status_cf + .put_deprecated_protobuf((0, signature, slot), &status)?; + for address in writable_keys { + self.address_signatures_cf.put_deprecated( + (0, *address, slot, signature), + &AddressSignatureMeta { writeable: true }, + )?; + } + for address in readonly_keys { + self.address_signatures_cf.put_deprecated( + (0, *address, slot, signature), + &AddressSignatureMeta { writeable: false }, + )?; + } + Ok(()) + } + } + let ledger_path = get_tmp_ledger_path_auto_delete!(); let blockstore = Blockstore::open(ledger_path.path()).unwrap(); @@ -8039,13 +8067,12 @@ pub mod tests { for x in 1..5 { let signature = Signature::from([x; 64]); blockstore - .write_transaction_status( + .write_deprecated_transaction_status( slot0, signature, vec![&address0], vec![&address1], TransactionStatusMeta::default(), - x as usize, ) .unwrap(); } @@ -8053,13 +8080,12 @@ pub mod tests { for x in 5..9 { let signature = Signature::from([x; 64]); blockstore - .write_transaction_status( + .write_deprecated_transaction_status( slot1, signature, vec![&address0], vec![&address1], TransactionStatusMeta::default(), - x as usize, ) .unwrap(); } diff --git a/ledger/src/blockstore_db.rs b/ledger/src/blockstore_db.rs index 1a5010dadb9779..204ccf73c82d34 100644 --- a/ledger/src/blockstore_db.rs +++ b/ledger/src/blockstore_db.rs @@ -2150,4 +2150,31 @@ pub mod tests { }); assert!(!should_enable_cf_compaction("something else")); } + + impl LedgerColumn + where + C: ColumnIndexDeprecation + ProtobufColumn + ColumnName, + { + pub fn put_deprecated_protobuf( + &self, + key: C::DeprecatedIndex, + value: &C::Type, + ) -> Result<()> { + let mut buf = Vec::with_capacity(value.encoded_len()); + value.encode(&mut buf)?; + self.backend + .put_cf(self.handle(), &C::deprecated_key(key), &buf) + } + } + + impl LedgerColumn + where + C: ColumnIndexDeprecation + TypedColumn + ColumnName, + { + pub fn put_deprecated(&self, key: C::DeprecatedIndex, value: &C::Type) -> Result<()> { + let serialized_value = serialize(value)?; + self.backend + .put_cf(self.handle(), &C::deprecated_key(key), &serialized_value) + } + } } From 098853e4d5c8b3141310192c19886562d3d4c594 Mon Sep 17 00:00:00 2001 From: Tyera Eulberg Date: Thu, 28 Sep 2023 13:09:55 -0600 Subject: [PATCH 35/57] Fix test_lowest_cleanup_slot_and_special_cfs moar --- ledger/src/blockstore.rs | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/ledger/src/blockstore.rs b/ledger/src/blockstore.rs index 4ca532a5a416af..6c9ab44f3e8cdf 100644 --- a/ledger/src/blockstore.rs +++ b/ledger/src/blockstore.rs @@ -7695,6 +7695,7 @@ pub mod tests { vec![&address0], vec![], TransactionStatusMeta::default(), + 0, ) .unwrap(); blockstore @@ -7704,6 +7705,7 @@ pub mod tests { vec![&address1], vec![], TransactionStatusMeta::default(), + 0, ) .unwrap(); @@ -7718,10 +7720,6 @@ pub mod tests { .find_address_signatures_for_slot(address0, lowest_cleanup_slot) .unwrap() .is_empty(), - blockstore - .find_address_signatures(address0, lowest_cleanup_slot, lowest_cleanup_slot) - .unwrap() - .is_empty(), ) }; @@ -7736,17 +7734,13 @@ pub mod tests { .find_address_signatures_for_slot(address1, lowest_available_slot) .unwrap() .is_empty(), - !blockstore - .find_address_signatures(address1, lowest_available_slot, lowest_available_slot) - .unwrap() - .is_empty(), ); - assert_eq!(are_existing_always, (true, true, true)); + assert_eq!(are_existing_always, (true, true)); }; let are_missing = check_for_missing(); // should never be missing before the conditional compaction & simulation... - assert_eq!(are_missing, (false, false, false)); + assert_eq!(are_missing, (false, false)); assert_existing_always(); if simulate_ledger_cleanup_service { @@ -7758,10 +7752,10 @@ pub mod tests { if simulate_ledger_cleanup_service { // ... when either simulation (or both) is effective, we should observe to be missing // consistently - assert_eq!(are_missing, (true, true, true)); + assert_eq!(are_missing, (true, true)); } else { // ... otherwise, we should observe to be existing... - assert_eq!(are_missing, (false, false, false)); + assert_eq!(are_missing, (false, false)); } assert_existing_always(); } From 4c596ad60714bac7a1e0edd590acb2a35d89fcbd Mon Sep 17 00:00:00 2001 From: Tyera Eulberg Date: Tue, 3 Oct 2023 17:29:06 -0600 Subject: [PATCH 36/57] Unignore tests (builds except tests) --- ledger/src/blockstore/blockstore_purge.rs | 1368 ++++++++++----------- 1 file changed, 684 insertions(+), 684 deletions(-) diff --git a/ledger/src/blockstore/blockstore_purge.rs b/ledger/src/blockstore/blockstore_purge.rs index 3ef9307d0fb6e2..46d847e4c32628 100644 --- a/ledger/src/blockstore/blockstore_purge.rs +++ b/ledger/src/blockstore/blockstore_purge.rs @@ -535,214 +535,214 @@ pub mod tests { }); } - // #[test] - // fn test_purge_front_of_ledger() { - // let ledger_path = get_tmp_ledger_path_auto_delete!(); - // let blockstore = Blockstore::open(ledger_path.path()).unwrap(); - // - // let max_slot = 10; - // for x in 0..max_slot { - // let random_bytes: [u8; 64] = std::array::from_fn(|_| rand::random::()); - // blockstore - // .write_transaction_status( - // x, - // Signature::from(random_bytes), - // vec![&Pubkey::try_from(&random_bytes[..32]).unwrap()], - // vec![&Pubkey::try_from(&random_bytes[32..]).unwrap()], - // TransactionStatusMeta::default(), - // ) - // .unwrap(); - // } - // // Purge to freeze index 0 - // blockstore.run_purge(0, 1, PurgeType::PrimaryIndex).unwrap(); - // - // for x in max_slot..2 * max_slot { - // let random_bytes: [u8; 64] = std::array::from_fn(|_| rand::random::()); - // blockstore - // .write_transaction_status( - // x, - // Signature::from(random_bytes), - // vec![&Pubkey::try_from(&random_bytes[..32]).unwrap()], - // vec![&Pubkey::try_from(&random_bytes[32..]).unwrap()], - // TransactionStatusMeta::default(), - // ) - // .unwrap(); - // } - // - // // Purging range outside of TransactionStatus max slots should not affect TransactionStatus data - // blockstore.run_purge(20, 30, PurgeType::Exact).unwrap(); - // - // let mut status_entry_iterator = blockstore - // .db - // .iter::(IteratorMode::From( - // cf::TransactionStatus::as_index(0), - // IteratorDirection::Forward, - // )) - // .unwrap(); - // let entry = status_entry_iterator.next().unwrap().0; - // assert_eq!(entry.0, 0); - // } - // - // fn clear_and_repopulate_transaction_statuses_for_test( - // blockstore: &Blockstore, - // index0_max_slot: u64, - // index1_max_slot: u64, - // ) { - // assert!(index1_max_slot > index0_max_slot); - // let mut write_batch = blockstore.db.batch().unwrap(); - // blockstore - // .run_purge(0, index1_max_slot, PurgeType::PrimaryIndex) - // .unwrap(); - // blockstore - // .db - // .delete_range_cf::(&mut write_batch, 0, 2) - // .unwrap(); - // blockstore - // .db - // .delete_range_cf::(&mut write_batch, 0, 2) - // .unwrap(); - // blockstore.db.write(write_batch).unwrap(); - // blockstore.initialize_transaction_status_index().unwrap(); - // *blockstore.active_transaction_status_index.write().unwrap() = 0; - // - // for x in 0..index0_max_slot { - // let entries = make_slot_entries_with_transactions(1); - // let shreds = entries_to_test_shreds( - // &entries, - // x, // slot - // x.saturating_sub(1), // parent_slot - // true, // is_full_slot - // 0, // version - // true, // merkle_variant - // ); - // blockstore.insert_shreds(shreds, None, false).unwrap(); - // let signature = entries - // .iter() - // .filter(|entry| !entry.is_tick()) - // .cloned() - // .flat_map(|entry| entry.transactions) - // .map(|transaction| transaction.signatures[0]) - // .collect::>()[0]; - // let random_bytes: Vec = (0..64).map(|_| rand::random::()).collect(); - // blockstore - // .write_transaction_status( - // x, - // signature, - // vec![&Pubkey::try_from(&random_bytes[..32]).unwrap()], - // vec![&Pubkey::try_from(&random_bytes[32..]).unwrap()], - // TransactionStatusMeta::default(), - // ) - // .unwrap(); - // } - // - // // Add slot that crosses primary indexes - // let entries = make_slot_entries_with_transactions(2); - // let shreds = entries_to_test_shreds( - // &entries, - // index0_max_slot, // slot - // index0_max_slot.saturating_sub(1), // parent_slot - // true, // is_full_slot - // 0, // version - // true, // merkle_variant - // ); - // blockstore.insert_shreds(shreds, None, false).unwrap(); - // let signatures = entries - // .iter() - // .filter(|entry| !entry.is_tick()) - // .cloned() - // .flat_map(|entry| entry.transactions) - // .map(|transaction| transaction.signatures[0]) - // .collect::>(); - // let random_bytes: Vec = (0..64).map(|_| rand::random::()).collect(); - // blockstore - // .write_transaction_status( - // index0_max_slot, - // signatures[0], - // vec![&Pubkey::try_from(&random_bytes[..32]).unwrap()], - // vec![&Pubkey::try_from(&random_bytes[32..]).unwrap()], - // TransactionStatusMeta::default(), - // ) - // .unwrap(); - // - // // Freeze index 0 - // let mut write_batch = blockstore.db.batch().unwrap(); - // let mut w_active_transaction_status_index = - // blockstore.active_transaction_status_index.write().unwrap(); - // blockstore - // .toggle_transaction_status_index( - // &mut write_batch, - // &mut w_active_transaction_status_index, - // index0_max_slot + 1, - // ) - // .unwrap(); - // drop(w_active_transaction_status_index); - // blockstore.db.write(write_batch).unwrap(); - // - // let random_bytes: Vec = (0..64).map(|_| rand::random::()).collect(); - // blockstore - // .write_transaction_status( - // index0_max_slot, - // signatures[1], - // vec![&Pubkey::try_from(&random_bytes[..32]).unwrap()], - // vec![&Pubkey::try_from(&random_bytes[32..]).unwrap()], - // TransactionStatusMeta::default(), - // ) - // .unwrap(); - // - // // Note: index0_max_slot exists in both indexes - // - // for x in index0_max_slot + 1..index1_max_slot + 1 { - // let entries = make_slot_entries_with_transactions(1); - // let shreds = entries_to_test_shreds( - // &entries, - // x, // slot - // x.saturating_sub(1), // parent_slot - // true, // is_full_slot - // 0, // version - // true, // merkle_variant - // ); - // blockstore.insert_shreds(shreds, None, false).unwrap(); - // let signature: Signature = entries - // .iter() - // .filter(|entry| !entry.is_tick()) - // .cloned() - // .flat_map(|entry| entry.transactions) - // .map(|transaction| transaction.signatures[0]) - // .collect::>()[0]; - // let random_bytes: Vec = (0..64).map(|_| rand::random::()).collect(); - // blockstore - // .write_transaction_status( - // x, - // signature, - // vec![&Pubkey::try_from(&random_bytes[..32]).unwrap()], - // vec![&Pubkey::try_from(&random_bytes[32..]).unwrap()], - // TransactionStatusMeta::default(), - // ) - // .unwrap(); - // } - // assert_eq!( - // blockstore - // .transaction_status_index_cf - // .get(0) - // .unwrap() - // .unwrap(), - // TransactionStatusIndexMeta { - // max_slot: index0_max_slot, - // frozen: true, - // } - // ); - // assert_eq!( - // blockstore - // .transaction_status_index_cf - // .get(1) - // .unwrap() - // .unwrap(), - // TransactionStatusIndexMeta { - // max_slot: index1_max_slot, - // frozen: false, - // } - // ); - // } + #[test] + fn test_purge_front_of_ledger() { + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); + + let max_slot = 10; + for x in 0..max_slot { + let random_bytes: [u8; 64] = std::array::from_fn(|_| rand::random::()); + blockstore + .write_transaction_status( + x, + Signature::from(random_bytes), + vec![&Pubkey::try_from(&random_bytes[..32]).unwrap()], + vec![&Pubkey::try_from(&random_bytes[32..]).unwrap()], + TransactionStatusMeta::default(), + ) + .unwrap(); + } + // Purge to freeze index 0 + blockstore.run_purge(0, 1, PurgeType::PrimaryIndex).unwrap(); + + for x in max_slot..2 * max_slot { + let random_bytes: [u8; 64] = std::array::from_fn(|_| rand::random::()); + blockstore + .write_transaction_status( + x, + Signature::from(random_bytes), + vec![&Pubkey::try_from(&random_bytes[..32]).unwrap()], + vec![&Pubkey::try_from(&random_bytes[32..]).unwrap()], + TransactionStatusMeta::default(), + ) + .unwrap(); + } + + // Purging range outside of TransactionStatus max slots should not affect TransactionStatus data + blockstore.run_purge(20, 30, PurgeType::Exact).unwrap(); + + let mut status_entry_iterator = blockstore + .db + .iter::(IteratorMode::From( + cf::TransactionStatus::as_index(0), + IteratorDirection::Forward, + )) + .unwrap(); + let entry = status_entry_iterator.next().unwrap().0; + assert_eq!(entry.0, 0); + } + + fn clear_and_repopulate_transaction_statuses_for_test( + blockstore: &Blockstore, + index0_max_slot: u64, + index1_max_slot: u64, + ) { + assert!(index1_max_slot > index0_max_slot); + let mut write_batch = blockstore.db.batch().unwrap(); + blockstore + .run_purge(0, index1_max_slot, PurgeType::PrimaryIndex) + .unwrap(); + blockstore + .db + .delete_range_cf::(&mut write_batch, 0, 2) + .unwrap(); + blockstore + .db + .delete_range_cf::(&mut write_batch, 0, 2) + .unwrap(); + blockstore.db.write(write_batch).unwrap(); + blockstore.initialize_transaction_status_index().unwrap(); + *blockstore.active_transaction_status_index.write().unwrap() = 0; + + for x in 0..index0_max_slot { + let entries = make_slot_entries_with_transactions(1); + let shreds = entries_to_test_shreds( + &entries, + x, // slot + x.saturating_sub(1), // parent_slot + true, // is_full_slot + 0, // version + true, // merkle_variant + ); + blockstore.insert_shreds(shreds, None, false).unwrap(); + let signature = entries + .iter() + .filter(|entry| !entry.is_tick()) + .cloned() + .flat_map(|entry| entry.transactions) + .map(|transaction| transaction.signatures[0]) + .collect::>()[0]; + let random_bytes: Vec = (0..64).map(|_| rand::random::()).collect(); + blockstore + .write_transaction_status( + x, + signature, + vec![&Pubkey::try_from(&random_bytes[..32]).unwrap()], + vec![&Pubkey::try_from(&random_bytes[32..]).unwrap()], + TransactionStatusMeta::default(), + ) + .unwrap(); + } + + // Add slot that crosses primary indexes + let entries = make_slot_entries_with_transactions(2); + let shreds = entries_to_test_shreds( + &entries, + index0_max_slot, // slot + index0_max_slot.saturating_sub(1), // parent_slot + true, // is_full_slot + 0, // version + true, // merkle_variant + ); + blockstore.insert_shreds(shreds, None, false).unwrap(); + let signatures = entries + .iter() + .filter(|entry| !entry.is_tick()) + .cloned() + .flat_map(|entry| entry.transactions) + .map(|transaction| transaction.signatures[0]) + .collect::>(); + let random_bytes: Vec = (0..64).map(|_| rand::random::()).collect(); + blockstore + .write_transaction_status( + index0_max_slot, + signatures[0], + vec![&Pubkey::try_from(&random_bytes[..32]).unwrap()], + vec![&Pubkey::try_from(&random_bytes[32..]).unwrap()], + TransactionStatusMeta::default(), + ) + .unwrap(); + + // Freeze index 0 + let mut write_batch = blockstore.db.batch().unwrap(); + let mut w_active_transaction_status_index = + blockstore.active_transaction_status_index.write().unwrap(); + blockstore + .toggle_transaction_status_index( + &mut write_batch, + &mut w_active_transaction_status_index, + index0_max_slot + 1, + ) + .unwrap(); + drop(w_active_transaction_status_index); + blockstore.db.write(write_batch).unwrap(); + + let random_bytes: Vec = (0..64).map(|_| rand::random::()).collect(); + blockstore + .write_transaction_status( + index0_max_slot, + signatures[1], + vec![&Pubkey::try_from(&random_bytes[..32]).unwrap()], + vec![&Pubkey::try_from(&random_bytes[32..]).unwrap()], + TransactionStatusMeta::default(), + ) + .unwrap(); + + // Note: index0_max_slot exists in both indexes + + for x in index0_max_slot + 1..index1_max_slot + 1 { + let entries = make_slot_entries_with_transactions(1); + let shreds = entries_to_test_shreds( + &entries, + x, // slot + x.saturating_sub(1), // parent_slot + true, // is_full_slot + 0, // version + true, // merkle_variant + ); + blockstore.insert_shreds(shreds, None, false).unwrap(); + let signature: Signature = entries + .iter() + .filter(|entry| !entry.is_tick()) + .cloned() + .flat_map(|entry| entry.transactions) + .map(|transaction| transaction.signatures[0]) + .collect::>()[0]; + let random_bytes: Vec = (0..64).map(|_| rand::random::()).collect(); + blockstore + .write_transaction_status( + x, + signature, + vec![&Pubkey::try_from(&random_bytes[..32]).unwrap()], + vec![&Pubkey::try_from(&random_bytes[32..]).unwrap()], + TransactionStatusMeta::default(), + ) + .unwrap(); + } + assert_eq!( + blockstore + .transaction_status_index_cf + .get(0) + .unwrap() + .unwrap(), + TransactionStatusIndexMeta { + max_slot: index0_max_slot, + frozen: true, + } + ); + assert_eq!( + blockstore + .transaction_status_index_cf + .get(1) + .unwrap() + .unwrap(), + TransactionStatusIndexMeta { + max_slot: index1_max_slot, + frozen: false, + } + ); + } #[test] fn test_special_columns_empty() { @@ -793,480 +793,480 @@ pub mod tests { assert!(blockstore.special_columns_empty().unwrap()); } - // #[test] - // #[allow(clippy::cognitive_complexity)] - // fn test_purge_transaction_status_exact() { - // let ledger_path = get_tmp_ledger_path_auto_delete!(); - // let blockstore = Blockstore::open(ledger_path.path()).unwrap(); - // - // let index0_max_slot = 9; - // let index1_max_slot = 19; - // - // // Test purge outside bounds - // clear_and_repopulate_transaction_statuses_for_test( - // &blockstore, - // index0_max_slot, - // index1_max_slot, - // ); - // blockstore.run_purge(20, 22, PurgeType::Exact).unwrap(); - // - // let mut status_entry_iterator = blockstore - // .db - // .iter::(IteratorMode::From( - // cf::TransactionStatus::as_index(0), - // IteratorDirection::Forward, - // )) - // .unwrap(); - // assert_eq!( - // blockstore - // .transaction_status_index_cf - // .get(0) - // .unwrap() - // .unwrap(), - // TransactionStatusIndexMeta { - // max_slot: index0_max_slot, - // frozen: true, - // } - // ); - // for _ in 0..index0_max_slot + 1 { - // let entry = status_entry_iterator.next().unwrap().0; - // assert_eq!(entry.0, 0); - // } - // assert_eq!( - // blockstore - // .transaction_status_index_cf - // .get(1) - // .unwrap() - // .unwrap(), - // TransactionStatusIndexMeta { - // max_slot: index1_max_slot, - // frozen: false, - // } - // ); - // for _ in index0_max_slot + 1..index1_max_slot + 1 { - // let entry = status_entry_iterator.next().unwrap().0; - // assert_eq!(entry.0, 1); - // } - // drop(status_entry_iterator); - // - // // Test purge inside index 0 - // clear_and_repopulate_transaction_statuses_for_test( - // &blockstore, - // index0_max_slot, - // index1_max_slot, - // ); - // blockstore.run_purge(2, 4, PurgeType::Exact).unwrap(); - // - // let mut status_entry_iterator = blockstore - // .db - // .iter::(IteratorMode::From( - // cf::TransactionStatus::as_index(0), - // IteratorDirection::Forward, - // )) - // .unwrap(); - // assert_eq!( - // blockstore - // .transaction_status_index_cf - // .get(0) - // .unwrap() - // .unwrap(), - // TransactionStatusIndexMeta { - // max_slot: index0_max_slot, - // frozen: true, - // } - // ); - // for _ in 0..7 { - // // 7 entries remaining - // let entry = status_entry_iterator.next().unwrap().0; - // assert_eq!(entry.0, 0); - // assert!(entry.2 < 2 || entry.2 > 4); - // } - // assert_eq!( - // blockstore - // .transaction_status_index_cf - // .get(1) - // .unwrap() - // .unwrap(), - // TransactionStatusIndexMeta { - // max_slot: index1_max_slot, - // frozen: false, - // } - // ); - // for _ in index0_max_slot + 1..index1_max_slot + 1 { - // let entry = status_entry_iterator.next().unwrap().0; - // assert_eq!(entry.0, 1); - // } - // drop(status_entry_iterator); - // - // // Test purge inside index 0 at upper boundary - // clear_and_repopulate_transaction_statuses_for_test( - // &blockstore, - // index0_max_slot, - // index1_max_slot, - // ); - // blockstore - // .run_purge(7, index0_max_slot, PurgeType::Exact) - // .unwrap(); - // - // let mut status_entry_iterator = blockstore - // .db - // .iter::(IteratorMode::From( - // cf::TransactionStatus::as_index(0), - // IteratorDirection::Forward, - // )) - // .unwrap(); - // assert_eq!( - // blockstore - // .transaction_status_index_cf - // .get(0) - // .unwrap() - // .unwrap(), - // TransactionStatusIndexMeta { - // max_slot: 6, - // frozen: true, - // } - // ); - // for _ in 0..7 { - // // 7 entries remaining - // let entry = status_entry_iterator.next().unwrap().0; - // assert_eq!(entry.0, 0); - // assert!(entry.2 < 7); - // } - // assert_eq!( - // blockstore - // .transaction_status_index_cf - // .get(1) - // .unwrap() - // .unwrap(), - // TransactionStatusIndexMeta { - // max_slot: index1_max_slot, - // frozen: false, - // } - // ); - // for _ in index0_max_slot + 1..index1_max_slot + 1 { - // let entry = status_entry_iterator.next().unwrap().0; - // assert_eq!(entry.0, 1); - // } - // drop(status_entry_iterator); - // - // // Test purge inside index 1 at lower boundary - // clear_and_repopulate_transaction_statuses_for_test( - // &blockstore, - // index0_max_slot, - // index1_max_slot, - // ); - // blockstore.run_purge(10, 12, PurgeType::Exact).unwrap(); - // - // let mut status_entry_iterator = blockstore - // .db - // .iter::(IteratorMode::From( - // cf::TransactionStatus::as_index(0), - // IteratorDirection::Forward, - // )) - // .unwrap(); - // assert_eq!( - // blockstore - // .transaction_status_index_cf - // .get(0) - // .unwrap() - // .unwrap(), - // TransactionStatusIndexMeta { - // max_slot: index0_max_slot, - // frozen: true, - // } - // ); - // for _ in 0..index0_max_slot + 1 { - // let entry = status_entry_iterator.next().unwrap().0; - // assert_eq!(entry.0, 0); - // } - // assert_eq!( - // blockstore - // .transaction_status_index_cf - // .get(1) - // .unwrap() - // .unwrap(), - // TransactionStatusIndexMeta { - // max_slot: index1_max_slot, - // frozen: false, - // } - // ); - // for _ in 13..index1_max_slot + 1 { - // let entry = status_entry_iterator.next().unwrap().0; - // assert_eq!(entry.0, 1); - // assert!(entry.2 == index0_max_slot || entry.2 > 12); - // } - // drop(status_entry_iterator); - // - // // Test purge across index boundaries - // clear_and_repopulate_transaction_statuses_for_test( - // &blockstore, - // index0_max_slot, - // index1_max_slot, - // ); - // blockstore.run_purge(7, 12, PurgeType::Exact).unwrap(); - // - // let mut status_entry_iterator = blockstore - // .db - // .iter::(IteratorMode::From( - // cf::TransactionStatus::as_index(0), - // IteratorDirection::Forward, - // )) - // .unwrap(); - // assert_eq!( - // blockstore - // .transaction_status_index_cf - // .get(0) - // .unwrap() - // .unwrap(), - // TransactionStatusIndexMeta { - // max_slot: 6, - // frozen: true, - // } - // ); - // for _ in 0..7 { - // // 7 entries remaining - // let entry = status_entry_iterator.next().unwrap().0; - // assert_eq!(entry.0, 0); - // assert!(entry.2 < 7); - // } - // assert_eq!( - // blockstore - // .transaction_status_index_cf - // .get(1) - // .unwrap() - // .unwrap(), - // TransactionStatusIndexMeta { - // max_slot: index1_max_slot, - // frozen: false, - // } - // ); - // for _ in 13..index1_max_slot + 1 { - // let entry = status_entry_iterator.next().unwrap().0; - // assert_eq!(entry.0, 1); - // assert!(entry.2 > 12); - // } - // drop(status_entry_iterator); - // - // // Test purge include complete index 1 - // clear_and_repopulate_transaction_statuses_for_test( - // &blockstore, - // index0_max_slot, - // index1_max_slot, - // ); - // blockstore.run_purge(7, 22, PurgeType::Exact).unwrap(); - // - // let mut status_entry_iterator = blockstore - // .db - // .iter::(IteratorMode::From( - // cf::TransactionStatus::as_index(0), - // IteratorDirection::Forward, - // )) - // .unwrap(); - // assert_eq!( - // blockstore - // .transaction_status_index_cf - // .get(0) - // .unwrap() - // .unwrap(), - // TransactionStatusIndexMeta { - // max_slot: 6, - // frozen: true, - // } - // ); - // for _ in 0..7 { - // // 7 entries remaining - // let entry = status_entry_iterator.next().unwrap().0; - // assert_eq!(entry.0, 0); - // assert!(entry.2 < 7); - // } - // assert_eq!( - // blockstore - // .transaction_status_index_cf - // .get(1) - // .unwrap() - // .unwrap(), - // TransactionStatusIndexMeta { - // max_slot: 6, - // frozen: false, - // } - // ); - // drop(status_entry_iterator); - // - // // Purge up to but not including index0_max_slot - // clear_and_repopulate_transaction_statuses_for_test( - // &blockstore, - // index0_max_slot, - // index1_max_slot, - // ); - // blockstore - // .run_purge(0, index0_max_slot - 1, PurgeType::Exact) - // .unwrap(); - // assert_eq!( - // blockstore - // .transaction_status_index_cf - // .get(0) - // .unwrap() - // .unwrap(), - // TransactionStatusIndexMeta { - // max_slot: index0_max_slot, - // frozen: true, - // } - // ); - // - // // Test purge all - // clear_and_repopulate_transaction_statuses_for_test( - // &blockstore, - // index0_max_slot, - // index1_max_slot, - // ); - // blockstore.run_purge(0, 22, PurgeType::Exact).unwrap(); - // - // let mut status_entry_iterator = blockstore - // .db - // .iter::(IteratorMode::From( - // cf::TransactionStatus::as_index(0), - // IteratorDirection::Forward, - // )) - // .unwrap(); - // assert!(status_entry_iterator.next().is_none()); - // - // assert_eq!( - // blockstore - // .transaction_status_index_cf - // .get(0) - // .unwrap() - // .unwrap(), - // TransactionStatusIndexMeta { - // max_slot: 0, - // frozen: true, - // } - // ); - // assert_eq!( - // blockstore - // .transaction_status_index_cf - // .get(1) - // .unwrap() - // .unwrap(), - // TransactionStatusIndexMeta { - // max_slot: 0, - // frozen: false, - // } - // ); - // } - // - // #[test] - // fn test_purge_special_columns_exact_no_sigs() { - // let ledger_path = get_tmp_ledger_path_auto_delete!(); - // let blockstore = Blockstore::open(ledger_path.path()).unwrap(); - // - // let slot = 1; - // let mut entries: Vec = vec![]; - // for x in 0..5 { - // let mut tx = Transaction::new_unsigned(Message::default()); - // tx.signatures = vec![]; - // entries.push(next_entry_mut(&mut Hash::default(), 0, vec![tx])); - // let mut tick = create_ticks(1, 0, hash(&serialize(&x).unwrap())); - // entries.append(&mut tick); - // } - // let shreds = entries_to_test_shreds( - // &entries, - // slot, - // slot - 1, // parent_slot - // true, // is_full_slot - // 0, // version - // true, // merkle_variant - // ); - // blockstore.insert_shreds(shreds, None, false).unwrap(); - // - // let mut write_batch = blockstore.db.batch().unwrap(); - // blockstore - // .purge_special_columns_exact(&mut write_batch, slot, slot + 1) - // .unwrap(); - // } - // - // #[test] - // fn test_purge_special_columns_compaction_filter() { - // let ledger_path = get_tmp_ledger_path_auto_delete!(); - // let blockstore = Blockstore::open(ledger_path.path()).unwrap(); - // let index0_max_slot = 9; - // let index1_max_slot = 19; - // // includes slot 0, and slot 9 has 2 transactions - // let num_total_transactions = index1_max_slot + 2; - // - // clear_and_repopulate_transaction_statuses_for_test( - // &blockstore, - // index0_max_slot, - // index1_max_slot, - // ); - // let first_index = { - // let mut status_entry_iterator = blockstore - // .db - // .iter::(IteratorMode::Start) - // .unwrap(); - // status_entry_iterator.next().unwrap().0 - // }; - // let last_index = { - // let mut status_entry_iterator = blockstore - // .db - // .iter::(IteratorMode::End) - // .unwrap(); - // status_entry_iterator.next().unwrap().0 - // }; - // - // let oldest_slot = 3; - // blockstore.db.set_oldest_slot(oldest_slot); - // blockstore.db.compact_range_cf::( - // &cf::TransactionStatus::key(first_index), - // &cf::TransactionStatus::key(last_index), - // ); - // - // let status_entry_iterator = blockstore - // .db - // .iter::(IteratorMode::Start) - // .unwrap(); - // let mut count = 0; - // for ((_primary_index, _signature, slot), _value) in status_entry_iterator { - // assert!(slot >= oldest_slot); - // count += 1; - // } - // assert_eq!(count, num_total_transactions - oldest_slot); - // - // clear_and_repopulate_transaction_statuses_for_test( - // &blockstore, - // index0_max_slot, - // index1_max_slot, - // ); - // let first_index = { - // let mut status_entry_iterator = blockstore - // .db - // .iter::(IteratorMode::Start) - // .unwrap(); - // status_entry_iterator.next().unwrap().0 - // }; - // let last_index = { - // let mut status_entry_iterator = blockstore - // .db - // .iter::(IteratorMode::End) - // .unwrap(); - // status_entry_iterator.next().unwrap().0 - // }; - // - // let oldest_slot = 12; - // blockstore.db.set_oldest_slot(oldest_slot); - // blockstore.db.compact_range_cf::( - // &cf::TransactionStatus::key(first_index), - // &cf::TransactionStatus::key(last_index), - // ); - // - // let status_entry_iterator = blockstore - // .db - // .iter::(IteratorMode::Start) - // .unwrap(); - // let mut count = 0; - // for ((_primary_index, _signature, slot), _value) in status_entry_iterator { - // assert!(slot >= oldest_slot); - // count += 1; - // } - // assert_eq!(count, num_total_transactions - oldest_slot - 1); // Extra transaction in slot 9 - // } + #[test] + #[allow(clippy::cognitive_complexity)] + fn test_purge_transaction_status_exact() { + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); + + let index0_max_slot = 9; + let index1_max_slot = 19; + + // Test purge outside bounds + clear_and_repopulate_transaction_statuses_for_test( + &blockstore, + index0_max_slot, + index1_max_slot, + ); + blockstore.run_purge(20, 22, PurgeType::Exact).unwrap(); + + let mut status_entry_iterator = blockstore + .db + .iter::(IteratorMode::From( + cf::TransactionStatus::as_index(0), + IteratorDirection::Forward, + )) + .unwrap(); + assert_eq!( + blockstore + .transaction_status_index_cf + .get(0) + .unwrap() + .unwrap(), + TransactionStatusIndexMeta { + max_slot: index0_max_slot, + frozen: true, + } + ); + for _ in 0..index0_max_slot + 1 { + let entry = status_entry_iterator.next().unwrap().0; + assert_eq!(entry.0, 0); + } + assert_eq!( + blockstore + .transaction_status_index_cf + .get(1) + .unwrap() + .unwrap(), + TransactionStatusIndexMeta { + max_slot: index1_max_slot, + frozen: false, + } + ); + for _ in index0_max_slot + 1..index1_max_slot + 1 { + let entry = status_entry_iterator.next().unwrap().0; + assert_eq!(entry.0, 1); + } + drop(status_entry_iterator); + + // Test purge inside index 0 + clear_and_repopulate_transaction_statuses_for_test( + &blockstore, + index0_max_slot, + index1_max_slot, + ); + blockstore.run_purge(2, 4, PurgeType::Exact).unwrap(); + + let mut status_entry_iterator = blockstore + .db + .iter::(IteratorMode::From( + cf::TransactionStatus::as_index(0), + IteratorDirection::Forward, + )) + .unwrap(); + assert_eq!( + blockstore + .transaction_status_index_cf + .get(0) + .unwrap() + .unwrap(), + TransactionStatusIndexMeta { + max_slot: index0_max_slot, + frozen: true, + } + ); + for _ in 0..7 { + // 7 entries remaining + let entry = status_entry_iterator.next().unwrap().0; + assert_eq!(entry.0, 0); + assert!(entry.2 < 2 || entry.2 > 4); + } + assert_eq!( + blockstore + .transaction_status_index_cf + .get(1) + .unwrap() + .unwrap(), + TransactionStatusIndexMeta { + max_slot: index1_max_slot, + frozen: false, + } + ); + for _ in index0_max_slot + 1..index1_max_slot + 1 { + let entry = status_entry_iterator.next().unwrap().0; + assert_eq!(entry.0, 1); + } + drop(status_entry_iterator); + + // Test purge inside index 0 at upper boundary + clear_and_repopulate_transaction_statuses_for_test( + &blockstore, + index0_max_slot, + index1_max_slot, + ); + blockstore + .run_purge(7, index0_max_slot, PurgeType::Exact) + .unwrap(); + + let mut status_entry_iterator = blockstore + .db + .iter::(IteratorMode::From( + cf::TransactionStatus::as_index(0), + IteratorDirection::Forward, + )) + .unwrap(); + assert_eq!( + blockstore + .transaction_status_index_cf + .get(0) + .unwrap() + .unwrap(), + TransactionStatusIndexMeta { + max_slot: 6, + frozen: true, + } + ); + for _ in 0..7 { + // 7 entries remaining + let entry = status_entry_iterator.next().unwrap().0; + assert_eq!(entry.0, 0); + assert!(entry.2 < 7); + } + assert_eq!( + blockstore + .transaction_status_index_cf + .get(1) + .unwrap() + .unwrap(), + TransactionStatusIndexMeta { + max_slot: index1_max_slot, + frozen: false, + } + ); + for _ in index0_max_slot + 1..index1_max_slot + 1 { + let entry = status_entry_iterator.next().unwrap().0; + assert_eq!(entry.0, 1); + } + drop(status_entry_iterator); + + // Test purge inside index 1 at lower boundary + clear_and_repopulate_transaction_statuses_for_test( + &blockstore, + index0_max_slot, + index1_max_slot, + ); + blockstore.run_purge(10, 12, PurgeType::Exact).unwrap(); + + let mut status_entry_iterator = blockstore + .db + .iter::(IteratorMode::From( + cf::TransactionStatus::as_index(0), + IteratorDirection::Forward, + )) + .unwrap(); + assert_eq!( + blockstore + .transaction_status_index_cf + .get(0) + .unwrap() + .unwrap(), + TransactionStatusIndexMeta { + max_slot: index0_max_slot, + frozen: true, + } + ); + for _ in 0..index0_max_slot + 1 { + let entry = status_entry_iterator.next().unwrap().0; + assert_eq!(entry.0, 0); + } + assert_eq!( + blockstore + .transaction_status_index_cf + .get(1) + .unwrap() + .unwrap(), + TransactionStatusIndexMeta { + max_slot: index1_max_slot, + frozen: false, + } + ); + for _ in 13..index1_max_slot + 1 { + let entry = status_entry_iterator.next().unwrap().0; + assert_eq!(entry.0, 1); + assert!(entry.2 == index0_max_slot || entry.2 > 12); + } + drop(status_entry_iterator); + + // Test purge across index boundaries + clear_and_repopulate_transaction_statuses_for_test( + &blockstore, + index0_max_slot, + index1_max_slot, + ); + blockstore.run_purge(7, 12, PurgeType::Exact).unwrap(); + + let mut status_entry_iterator = blockstore + .db + .iter::(IteratorMode::From( + cf::TransactionStatus::as_index(0), + IteratorDirection::Forward, + )) + .unwrap(); + assert_eq!( + blockstore + .transaction_status_index_cf + .get(0) + .unwrap() + .unwrap(), + TransactionStatusIndexMeta { + max_slot: 6, + frozen: true, + } + ); + for _ in 0..7 { + // 7 entries remaining + let entry = status_entry_iterator.next().unwrap().0; + assert_eq!(entry.0, 0); + assert!(entry.2 < 7); + } + assert_eq!( + blockstore + .transaction_status_index_cf + .get(1) + .unwrap() + .unwrap(), + TransactionStatusIndexMeta { + max_slot: index1_max_slot, + frozen: false, + } + ); + for _ in 13..index1_max_slot + 1 { + let entry = status_entry_iterator.next().unwrap().0; + assert_eq!(entry.0, 1); + assert!(entry.2 > 12); + } + drop(status_entry_iterator); + + // Test purge include complete index 1 + clear_and_repopulate_transaction_statuses_for_test( + &blockstore, + index0_max_slot, + index1_max_slot, + ); + blockstore.run_purge(7, 22, PurgeType::Exact).unwrap(); + + let mut status_entry_iterator = blockstore + .db + .iter::(IteratorMode::From( + cf::TransactionStatus::as_index(0), + IteratorDirection::Forward, + )) + .unwrap(); + assert_eq!( + blockstore + .transaction_status_index_cf + .get(0) + .unwrap() + .unwrap(), + TransactionStatusIndexMeta { + max_slot: 6, + frozen: true, + } + ); + for _ in 0..7 { + // 7 entries remaining + let entry = status_entry_iterator.next().unwrap().0; + assert_eq!(entry.0, 0); + assert!(entry.2 < 7); + } + assert_eq!( + blockstore + .transaction_status_index_cf + .get(1) + .unwrap() + .unwrap(), + TransactionStatusIndexMeta { + max_slot: 6, + frozen: false, + } + ); + drop(status_entry_iterator); + + // Purge up to but not including index0_max_slot + clear_and_repopulate_transaction_statuses_for_test( + &blockstore, + index0_max_slot, + index1_max_slot, + ); + blockstore + .run_purge(0, index0_max_slot - 1, PurgeType::Exact) + .unwrap(); + assert_eq!( + blockstore + .transaction_status_index_cf + .get(0) + .unwrap() + .unwrap(), + TransactionStatusIndexMeta { + max_slot: index0_max_slot, + frozen: true, + } + ); + + // Test purge all + clear_and_repopulate_transaction_statuses_for_test( + &blockstore, + index0_max_slot, + index1_max_slot, + ); + blockstore.run_purge(0, 22, PurgeType::Exact).unwrap(); + + let mut status_entry_iterator = blockstore + .db + .iter::(IteratorMode::From( + cf::TransactionStatus::as_index(0), + IteratorDirection::Forward, + )) + .unwrap(); + assert!(status_entry_iterator.next().is_none()); + + assert_eq!( + blockstore + .transaction_status_index_cf + .get(0) + .unwrap() + .unwrap(), + TransactionStatusIndexMeta { + max_slot: 0, + frozen: true, + } + ); + assert_eq!( + blockstore + .transaction_status_index_cf + .get(1) + .unwrap() + .unwrap(), + TransactionStatusIndexMeta { + max_slot: 0, + frozen: false, + } + ); + } + + #[test] + fn test_purge_special_columns_exact_no_sigs() { + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); + + let slot = 1; + let mut entries: Vec = vec![]; + for x in 0..5 { + let mut tx = Transaction::new_unsigned(Message::default()); + tx.signatures = vec![]; + entries.push(next_entry_mut(&mut Hash::default(), 0, vec![tx])); + let mut tick = create_ticks(1, 0, hash(&serialize(&x).unwrap())); + entries.append(&mut tick); + } + let shreds = entries_to_test_shreds( + &entries, + slot, + slot - 1, // parent_slot + true, // is_full_slot + 0, // version + true, // merkle_variant + ); + blockstore.insert_shreds(shreds, None, false).unwrap(); + + let mut write_batch = blockstore.db.batch().unwrap(); + blockstore + .purge_special_columns_exact(&mut write_batch, slot, slot + 1) + .unwrap(); + } + + #[test] + fn test_purge_special_columns_compaction_filter() { + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); + let index0_max_slot = 9; + let index1_max_slot = 19; + // includes slot 0, and slot 9 has 2 transactions + let num_total_transactions = index1_max_slot + 2; + + clear_and_repopulate_transaction_statuses_for_test( + &blockstore, + index0_max_slot, + index1_max_slot, + ); + let first_index = { + let mut status_entry_iterator = blockstore + .db + .iter::(IteratorMode::Start) + .unwrap(); + status_entry_iterator.next().unwrap().0 + }; + let last_index = { + let mut status_entry_iterator = blockstore + .db + .iter::(IteratorMode::End) + .unwrap(); + status_entry_iterator.next().unwrap().0 + }; + + let oldest_slot = 3; + blockstore.db.set_oldest_slot(oldest_slot); + blockstore.db.compact_range_cf::( + &cf::TransactionStatus::key(first_index), + &cf::TransactionStatus::key(last_index), + ); + + let status_entry_iterator = blockstore + .db + .iter::(IteratorMode::Start) + .unwrap(); + let mut count = 0; + for ((_primary_index, _signature, slot), _value) in status_entry_iterator { + assert!(slot >= oldest_slot); + count += 1; + } + assert_eq!(count, num_total_transactions - oldest_slot); + + clear_and_repopulate_transaction_statuses_for_test( + &blockstore, + index0_max_slot, + index1_max_slot, + ); + let first_index = { + let mut status_entry_iterator = blockstore + .db + .iter::(IteratorMode::Start) + .unwrap(); + status_entry_iterator.next().unwrap().0 + }; + let last_index = { + let mut status_entry_iterator = blockstore + .db + .iter::(IteratorMode::End) + .unwrap(); + status_entry_iterator.next().unwrap().0 + }; + + let oldest_slot = 12; + blockstore.db.set_oldest_slot(oldest_slot); + blockstore.db.compact_range_cf::( + &cf::TransactionStatus::key(first_index), + &cf::TransactionStatus::key(last_index), + ); + + let status_entry_iterator = blockstore + .db + .iter::(IteratorMode::Start) + .unwrap(); + let mut count = 0; + for ((_primary_index, _signature, slot), _value) in status_entry_iterator { + assert!(slot >= oldest_slot); + count += 1; + } + assert_eq!(count, num_total_transactions - oldest_slot - 1); // Extra transaction in slot 9 + } } From 5b7e5553a8190f1501d11acd731826d82bc80eff Mon Sep 17 00:00:00 2001 From: Tyera Eulberg Date: Thu, 28 Sep 2023 16:41:38 -0600 Subject: [PATCH 37/57] Fix test_purge_transaction_status_exact --- ledger/src/blockstore/blockstore_purge.rs | 475 ++-------------------- 1 file changed, 29 insertions(+), 446 deletions(-) diff --git a/ledger/src/blockstore/blockstore_purge.rs b/ledger/src/blockstore/blockstore_purge.rs index 46d847e4c32628..db7d8bdca336a9 100644 --- a/ledger/src/blockstore/blockstore_purge.rs +++ b/ledger/src/blockstore/blockstore_purge.rs @@ -583,29 +583,15 @@ pub mod tests { assert_eq!(entry.0, 0); } - fn clear_and_repopulate_transaction_statuses_for_test( - blockstore: &Blockstore, - index0_max_slot: u64, - index1_max_slot: u64, - ) { - assert!(index1_max_slot > index0_max_slot); - let mut write_batch = blockstore.db.batch().unwrap(); - blockstore - .run_purge(0, index1_max_slot, PurgeType::PrimaryIndex) - .unwrap(); - blockstore - .db - .delete_range_cf::(&mut write_batch, 0, 2) - .unwrap(); - blockstore + fn clear_and_repopulate_transaction_statuses_for_test(blockstore: &Blockstore, max_slot: u64) { + blockstore.run_purge(0, max_slot, PurgeType::Exact).unwrap(); + let mut iter = blockstore .db - .delete_range_cf::(&mut write_batch, 0, 2) + .iter::(IteratorMode::Start) .unwrap(); - blockstore.db.write(write_batch).unwrap(); - blockstore.initialize_transaction_status_index().unwrap(); - *blockstore.active_transaction_status_index.write().unwrap() = 0; + assert_eq!(iter.next(), None); - for x in 0..index0_max_slot { + for x in 0..=max_slot { let entries = make_slot_entries_with_transactions(1); let shreds = entries_to_test_shreds( &entries, @@ -631,117 +617,10 @@ pub mod tests { vec![&Pubkey::try_from(&random_bytes[..32]).unwrap()], vec![&Pubkey::try_from(&random_bytes[32..]).unwrap()], TransactionStatusMeta::default(), + 0, ) .unwrap(); } - - // Add slot that crosses primary indexes - let entries = make_slot_entries_with_transactions(2); - let shreds = entries_to_test_shreds( - &entries, - index0_max_slot, // slot - index0_max_slot.saturating_sub(1), // parent_slot - true, // is_full_slot - 0, // version - true, // merkle_variant - ); - blockstore.insert_shreds(shreds, None, false).unwrap(); - let signatures = entries - .iter() - .filter(|entry| !entry.is_tick()) - .cloned() - .flat_map(|entry| entry.transactions) - .map(|transaction| transaction.signatures[0]) - .collect::>(); - let random_bytes: Vec = (0..64).map(|_| rand::random::()).collect(); - blockstore - .write_transaction_status( - index0_max_slot, - signatures[0], - vec![&Pubkey::try_from(&random_bytes[..32]).unwrap()], - vec![&Pubkey::try_from(&random_bytes[32..]).unwrap()], - TransactionStatusMeta::default(), - ) - .unwrap(); - - // Freeze index 0 - let mut write_batch = blockstore.db.batch().unwrap(); - let mut w_active_transaction_status_index = - blockstore.active_transaction_status_index.write().unwrap(); - blockstore - .toggle_transaction_status_index( - &mut write_batch, - &mut w_active_transaction_status_index, - index0_max_slot + 1, - ) - .unwrap(); - drop(w_active_transaction_status_index); - blockstore.db.write(write_batch).unwrap(); - - let random_bytes: Vec = (0..64).map(|_| rand::random::()).collect(); - blockstore - .write_transaction_status( - index0_max_slot, - signatures[1], - vec![&Pubkey::try_from(&random_bytes[..32]).unwrap()], - vec![&Pubkey::try_from(&random_bytes[32..]).unwrap()], - TransactionStatusMeta::default(), - ) - .unwrap(); - - // Note: index0_max_slot exists in both indexes - - for x in index0_max_slot + 1..index1_max_slot + 1 { - let entries = make_slot_entries_with_transactions(1); - let shreds = entries_to_test_shreds( - &entries, - x, // slot - x.saturating_sub(1), // parent_slot - true, // is_full_slot - 0, // version - true, // merkle_variant - ); - blockstore.insert_shreds(shreds, None, false).unwrap(); - let signature: Signature = entries - .iter() - .filter(|entry| !entry.is_tick()) - .cloned() - .flat_map(|entry| entry.transactions) - .map(|transaction| transaction.signatures[0]) - .collect::>()[0]; - let random_bytes: Vec = (0..64).map(|_| rand::random::()).collect(); - blockstore - .write_transaction_status( - x, - signature, - vec![&Pubkey::try_from(&random_bytes[..32]).unwrap()], - vec![&Pubkey::try_from(&random_bytes[32..]).unwrap()], - TransactionStatusMeta::default(), - ) - .unwrap(); - } - assert_eq!( - blockstore - .transaction_status_index_cf - .get(0) - .unwrap() - .unwrap(), - TransactionStatusIndexMeta { - max_slot: index0_max_slot, - frozen: true, - } - ); - assert_eq!( - blockstore - .transaction_status_index_cf - .get(1) - .unwrap() - .unwrap(), - TransactionStatusIndexMeta { - max_slot: index1_max_slot, - frozen: false, - } - ); } #[test] @@ -799,359 +678,63 @@ pub mod tests { let ledger_path = get_tmp_ledger_path_auto_delete!(); let blockstore = Blockstore::open(ledger_path.path()).unwrap(); - let index0_max_slot = 9; - let index1_max_slot = 19; + let max_slot = 9; // Test purge outside bounds - clear_and_repopulate_transaction_statuses_for_test( - &blockstore, - index0_max_slot, - index1_max_slot, - ); - blockstore.run_purge(20, 22, PurgeType::Exact).unwrap(); + clear_and_repopulate_transaction_statuses_for_test(&blockstore, max_slot); + blockstore.run_purge(10, 12, PurgeType::Exact).unwrap(); let mut status_entry_iterator = blockstore .db - .iter::(IteratorMode::From( - cf::TransactionStatus::as_index(0), - IteratorDirection::Forward, - )) + .iter::(IteratorMode::Start) .unwrap(); - assert_eq!( - blockstore - .transaction_status_index_cf - .get(0) - .unwrap() - .unwrap(), - TransactionStatusIndexMeta { - max_slot: index0_max_slot, - frozen: true, - } - ); - for _ in 0..index0_max_slot + 1 { + for _ in 0..max_slot + 1 { let entry = status_entry_iterator.next().unwrap().0; - assert_eq!(entry.0, 0); - } - assert_eq!( - blockstore - .transaction_status_index_cf - .get(1) - .unwrap() - .unwrap(), - TransactionStatusIndexMeta { - max_slot: index1_max_slot, - frozen: false, - } - ); - for _ in index0_max_slot + 1..index1_max_slot + 1 { - let entry = status_entry_iterator.next().unwrap().0; - assert_eq!(entry.0, 1); + assert!(entry.1 <= max_slot || entry.1 > 0); } + assert_eq!(status_entry_iterator.next(), None); drop(status_entry_iterator); - // Test purge inside index 0 - clear_and_repopulate_transaction_statuses_for_test( - &blockstore, - index0_max_slot, - index1_max_slot, - ); + // Test purge inside written range + clear_and_repopulate_transaction_statuses_for_test(&blockstore, max_slot); blockstore.run_purge(2, 4, PurgeType::Exact).unwrap(); let mut status_entry_iterator = blockstore .db - .iter::(IteratorMode::From( - cf::TransactionStatus::as_index(0), - IteratorDirection::Forward, - )) + .iter::(IteratorMode::Start) .unwrap(); - assert_eq!( - blockstore - .transaction_status_index_cf - .get(0) - .unwrap() - .unwrap(), - TransactionStatusIndexMeta { - max_slot: index0_max_slot, - frozen: true, - } - ); for _ in 0..7 { // 7 entries remaining let entry = status_entry_iterator.next().unwrap().0; - assert_eq!(entry.0, 0); - assert!(entry.2 < 2 || entry.2 > 4); - } - assert_eq!( - blockstore - .transaction_status_index_cf - .get(1) - .unwrap() - .unwrap(), - TransactionStatusIndexMeta { - max_slot: index1_max_slot, - frozen: false, - } - ); - for _ in index0_max_slot + 1..index1_max_slot + 1 { - let entry = status_entry_iterator.next().unwrap().0; - assert_eq!(entry.0, 1); + assert!(entry.1 < 2 || entry.1 > 4); } + assert_eq!(status_entry_iterator.next(), None); drop(status_entry_iterator); - // Test purge inside index 0 at upper boundary - clear_and_repopulate_transaction_statuses_for_test( - &blockstore, - index0_max_slot, - index1_max_slot, - ); + // Purge up to but not including max_slot + clear_and_repopulate_transaction_statuses_for_test(&blockstore, max_slot); blockstore - .run_purge(7, index0_max_slot, PurgeType::Exact) - .unwrap(); - - let mut status_entry_iterator = blockstore - .db - .iter::(IteratorMode::From( - cf::TransactionStatus::as_index(0), - IteratorDirection::Forward, - )) + .run_purge(0, max_slot - 1, PurgeType::Exact) .unwrap(); - assert_eq!( - blockstore - .transaction_status_index_cf - .get(0) - .unwrap() - .unwrap(), - TransactionStatusIndexMeta { - max_slot: 6, - frozen: true, - } - ); - for _ in 0..7 { - // 7 entries remaining - let entry = status_entry_iterator.next().unwrap().0; - assert_eq!(entry.0, 0); - assert!(entry.2 < 7); - } - assert_eq!( - blockstore - .transaction_status_index_cf - .get(1) - .unwrap() - .unwrap(), - TransactionStatusIndexMeta { - max_slot: index1_max_slot, - frozen: false, - } - ); - for _ in index0_max_slot + 1..index1_max_slot + 1 { - let entry = status_entry_iterator.next().unwrap().0; - assert_eq!(entry.0, 1); - } - drop(status_entry_iterator); - - // Test purge inside index 1 at lower boundary - clear_and_repopulate_transaction_statuses_for_test( - &blockstore, - index0_max_slot, - index1_max_slot, - ); - blockstore.run_purge(10, 12, PurgeType::Exact).unwrap(); - - let mut status_entry_iterator = blockstore - .db - .iter::(IteratorMode::From( - cf::TransactionStatus::as_index(0), - IteratorDirection::Forward, - )) - .unwrap(); - assert_eq!( - blockstore - .transaction_status_index_cf - .get(0) - .unwrap() - .unwrap(), - TransactionStatusIndexMeta { - max_slot: index0_max_slot, - frozen: true, - } - ); - for _ in 0..index0_max_slot + 1 { - let entry = status_entry_iterator.next().unwrap().0; - assert_eq!(entry.0, 0); - } - assert_eq!( - blockstore - .transaction_status_index_cf - .get(1) - .unwrap() - .unwrap(), - TransactionStatusIndexMeta { - max_slot: index1_max_slot, - frozen: false, - } - ); - for _ in 13..index1_max_slot + 1 { - let entry = status_entry_iterator.next().unwrap().0; - assert_eq!(entry.0, 1); - assert!(entry.2 == index0_max_slot || entry.2 > 12); - } - drop(status_entry_iterator); - - // Test purge across index boundaries - clear_and_repopulate_transaction_statuses_for_test( - &blockstore, - index0_max_slot, - index1_max_slot, - ); - blockstore.run_purge(7, 12, PurgeType::Exact).unwrap(); - - let mut status_entry_iterator = blockstore - .db - .iter::(IteratorMode::From( - cf::TransactionStatus::as_index(0), - IteratorDirection::Forward, - )) - .unwrap(); - assert_eq!( - blockstore - .transaction_status_index_cf - .get(0) - .unwrap() - .unwrap(), - TransactionStatusIndexMeta { - max_slot: 6, - frozen: true, - } - ); - for _ in 0..7 { - // 7 entries remaining - let entry = status_entry_iterator.next().unwrap().0; - assert_eq!(entry.0, 0); - assert!(entry.2 < 7); - } - assert_eq!( - blockstore - .transaction_status_index_cf - .get(1) - .unwrap() - .unwrap(), - TransactionStatusIndexMeta { - max_slot: index1_max_slot, - frozen: false, - } - ); - for _ in 13..index1_max_slot + 1 { - let entry = status_entry_iterator.next().unwrap().0; - assert_eq!(entry.0, 1); - assert!(entry.2 > 12); - } - drop(status_entry_iterator); - - // Test purge include complete index 1 - clear_and_repopulate_transaction_statuses_for_test( - &blockstore, - index0_max_slot, - index1_max_slot, - ); - blockstore.run_purge(7, 22, PurgeType::Exact).unwrap(); let mut status_entry_iterator = blockstore .db - .iter::(IteratorMode::From( - cf::TransactionStatus::as_index(0), - IteratorDirection::Forward, - )) + .iter::(IteratorMode::Start) .unwrap(); - assert_eq!( - blockstore - .transaction_status_index_cf - .get(0) - .unwrap() - .unwrap(), - TransactionStatusIndexMeta { - max_slot: 6, - frozen: true, - } - ); - for _ in 0..7 { - // 7 entries remaining - let entry = status_entry_iterator.next().unwrap().0; - assert_eq!(entry.0, 0); - assert!(entry.2 < 7); - } - assert_eq!( - blockstore - .transaction_status_index_cf - .get(1) - .unwrap() - .unwrap(), - TransactionStatusIndexMeta { - max_slot: 6, - frozen: false, - } - ); + let entry = status_entry_iterator.next().unwrap().0; + assert_eq!(entry.1, 9); + assert_eq!(status_entry_iterator.next(), None); drop(status_entry_iterator); - // Purge up to but not including index0_max_slot - clear_and_repopulate_transaction_statuses_for_test( - &blockstore, - index0_max_slot, - index1_max_slot, - ); - blockstore - .run_purge(0, index0_max_slot - 1, PurgeType::Exact) - .unwrap(); - assert_eq!( - blockstore - .transaction_status_index_cf - .get(0) - .unwrap() - .unwrap(), - TransactionStatusIndexMeta { - max_slot: index0_max_slot, - frozen: true, - } - ); - // Test purge all - clear_and_repopulate_transaction_statuses_for_test( - &blockstore, - index0_max_slot, - index1_max_slot, - ); + clear_and_repopulate_transaction_statuses_for_test(&blockstore, max_slot); blockstore.run_purge(0, 22, PurgeType::Exact).unwrap(); let mut status_entry_iterator = blockstore .db - .iter::(IteratorMode::From( - cf::TransactionStatus::as_index(0), - IteratorDirection::Forward, - )) + .iter::(IteratorMode::Start) .unwrap(); - assert!(status_entry_iterator.next().is_none()); - - assert_eq!( - blockstore - .transaction_status_index_cf - .get(0) - .unwrap() - .unwrap(), - TransactionStatusIndexMeta { - max_slot: 0, - frozen: true, - } - ); - assert_eq!( - blockstore - .transaction_status_index_cf - .get(1) - .unwrap() - .unwrap(), - TransactionStatusIndexMeta { - max_slot: 0, - frozen: false, - } - ); + assert_eq!(status_entry_iterator.next(), None); } #[test] From 077c4b0928b66e59fa66f05daf12061696fe11a8 Mon Sep 17 00:00:00 2001 From: Tyera Eulberg Date: Thu, 28 Sep 2023 18:30:22 -0600 Subject: [PATCH 38/57] Fix test_purge_front_of_ledger --- ledger/src/blockstore/blockstore_purge.rs | 31 +++++------------------ 1 file changed, 7 insertions(+), 24 deletions(-) diff --git a/ledger/src/blockstore/blockstore_purge.rs b/ledger/src/blockstore/blockstore_purge.rs index db7d8bdca336a9..29600c80a00e65 100644 --- a/ledger/src/blockstore/blockstore_purge.rs +++ b/ledger/src/blockstore/blockstore_purge.rs @@ -550,37 +550,20 @@ pub mod tests { vec![&Pubkey::try_from(&random_bytes[..32]).unwrap()], vec![&Pubkey::try_from(&random_bytes[32..]).unwrap()], TransactionStatusMeta::default(), - ) - .unwrap(); - } - // Purge to freeze index 0 - blockstore.run_purge(0, 1, PurgeType::PrimaryIndex).unwrap(); - - for x in max_slot..2 * max_slot { - let random_bytes: [u8; 64] = std::array::from_fn(|_| rand::random::()); - blockstore - .write_transaction_status( - x, - Signature::from(random_bytes), - vec![&Pubkey::try_from(&random_bytes[..32]).unwrap()], - vec![&Pubkey::try_from(&random_bytes[32..]).unwrap()], - TransactionStatusMeta::default(), + 0, ) .unwrap(); } // Purging range outside of TransactionStatus max slots should not affect TransactionStatus data - blockstore.run_purge(20, 30, PurgeType::Exact).unwrap(); + blockstore.run_purge(10, 20, PurgeType::Exact).unwrap(); - let mut status_entry_iterator = blockstore + let status_entries: Vec<_> = blockstore .db - .iter::(IteratorMode::From( - cf::TransactionStatus::as_index(0), - IteratorDirection::Forward, - )) - .unwrap(); - let entry = status_entry_iterator.next().unwrap().0; - assert_eq!(entry.0, 0); + .iter::(IteratorMode::Start) + .unwrap() + .collect(); + assert_eq!(status_entries.len(), 10); } fn clear_and_repopulate_transaction_statuses_for_test(blockstore: &Blockstore, max_slot: u64) { From eae2c299bc4b57cf2c715491959add9875733dc3 Mon Sep 17 00:00:00 2001 From: Tyera Eulberg Date: Mon, 2 Oct 2023 18:27:30 -0600 Subject: [PATCH 39/57] Fix test_purge_special_columns_compaction_filter (all build) --- ledger/src/blockstore/blockstore_purge.rs | 27 +++++++---------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/ledger/src/blockstore/blockstore_purge.rs b/ledger/src/blockstore/blockstore_purge.rs index 29600c80a00e65..80845394967240 100644 --- a/ledger/src/blockstore/blockstore_purge.rs +++ b/ledger/src/blockstore/blockstore_purge.rs @@ -754,16 +754,9 @@ pub mod tests { fn test_purge_special_columns_compaction_filter() { let ledger_path = get_tmp_ledger_path_auto_delete!(); let blockstore = Blockstore::open(ledger_path.path()).unwrap(); - let index0_max_slot = 9; - let index1_max_slot = 19; - // includes slot 0, and slot 9 has 2 transactions - let num_total_transactions = index1_max_slot + 2; - - clear_and_repopulate_transaction_statuses_for_test( - &blockstore, - index0_max_slot, - index1_max_slot, - ); + let max_slot = 19; + + clear_and_repopulate_transaction_statuses_for_test(&blockstore, max_slot); let first_index = { let mut status_entry_iterator = blockstore .db @@ -791,17 +784,13 @@ pub mod tests { .iter::(IteratorMode::Start) .unwrap(); let mut count = 0; - for ((_primary_index, _signature, slot), _value) in status_entry_iterator { + for ((_signature, slot), _value) in status_entry_iterator { assert!(slot >= oldest_slot); count += 1; } - assert_eq!(count, num_total_transactions - oldest_slot); + assert_eq!(count, max_slot - (oldest_slot - 1)); - clear_and_repopulate_transaction_statuses_for_test( - &blockstore, - index0_max_slot, - index1_max_slot, - ); + clear_and_repopulate_transaction_statuses_for_test(&blockstore, max_slot); let first_index = { let mut status_entry_iterator = blockstore .db @@ -829,10 +818,10 @@ pub mod tests { .iter::(IteratorMode::Start) .unwrap(); let mut count = 0; - for ((_primary_index, _signature, slot), _value) in status_entry_iterator { + for ((_signature, slot), _value) in status_entry_iterator { assert!(slot >= oldest_slot); count += 1; } - assert_eq!(count, num_total_transactions - oldest_slot - 1); // Extra transaction in slot 9 + assert_eq!(count, max_slot - (oldest_slot - 1)); } } From b23ad5aa357ff8eca257e7aa13460a2100e144c2 Mon Sep 17 00:00:00 2001 From: Tyera Eulberg Date: Mon, 2 Oct 2023 20:24:26 -0600 Subject: [PATCH 40/57] Move some test-harness stuff around --- ledger/src/blockstore.rs | 57 ++++++++++++----------- ledger/src/blockstore/blockstore_purge.rs | 48 ++++++++++++++++++- ledger/src/blockstore_db.rs | 13 ++++++ 3 files changed, 90 insertions(+), 28 deletions(-) diff --git a/ledger/src/blockstore.rs b/ledger/src/blockstore.rs index 6c9ab44f3e8cdf..17c3553a60a5ab 100644 --- a/ledger/src/blockstore.rs +++ b/ledger/src/blockstore.rs @@ -8021,36 +8021,37 @@ pub mod tests { ); } - #[test] - fn test_get_confirmed_signatures_for_address() { - impl Blockstore { - fn write_deprecated_transaction_status( - &self, - slot: Slot, - signature: Signature, - writable_keys: Vec<&Pubkey>, - readonly_keys: Vec<&Pubkey>, - status: TransactionStatusMeta, - ) -> Result<()> { - let status = status.into(); - self.transaction_status_cf - .put_deprecated_protobuf((0, signature, slot), &status)?; - for address in writable_keys { - self.address_signatures_cf.put_deprecated( - (0, *address, slot, signature), - &AddressSignatureMeta { writeable: true }, - )?; - } - for address in readonly_keys { - self.address_signatures_cf.put_deprecated( - (0, *address, slot, signature), - &AddressSignatureMeta { writeable: false }, - )?; - } - Ok(()) + impl Blockstore { + pub(crate) fn write_deprecated_transaction_status( + &self, + primary_index: u64, + slot: Slot, + signature: Signature, + writable_keys: Vec<&Pubkey>, + readonly_keys: Vec<&Pubkey>, + status: TransactionStatusMeta, + ) -> Result<()> { + let status = status.into(); + self.transaction_status_cf + .put_deprecated_protobuf((primary_index, signature, slot), &status)?; + for address in writable_keys { + self.address_signatures_cf.put_deprecated( + (primary_index, *address, slot, signature), + &AddressSignatureMeta { writeable: true }, + )?; } + for address in readonly_keys { + self.address_signatures_cf.put_deprecated( + (primary_index, *address, slot, signature), + &AddressSignatureMeta { writeable: false }, + )?; + } + Ok(()) } + } + #[test] + fn test_get_confirmed_signatures_for_address() { let ledger_path = get_tmp_ledger_path_auto_delete!(); let blockstore = Blockstore::open(ledger_path.path()).unwrap(); @@ -8062,6 +8063,7 @@ pub mod tests { let signature = Signature::from([x; 64]); blockstore .write_deprecated_transaction_status( + 0, slot0, signature, vec![&address0], @@ -8075,6 +8077,7 @@ pub mod tests { let signature = Signature::from([x; 64]); blockstore .write_deprecated_transaction_status( + 0, slot1, signature, vec![&address0], diff --git a/ledger/src/blockstore/blockstore_purge.rs b/ledger/src/blockstore/blockstore_purge.rs index 80845394967240..8ea31b41e5a64b 100644 --- a/ledger/src/blockstore/blockstore_purge.rs +++ b/ledger/src/blockstore/blockstore_purge.rs @@ -574,7 +574,15 @@ pub mod tests { .unwrap(); assert_eq!(iter.next(), None); - for x in 0..=max_slot { + populate_transaction_statuses_for_test(blockstore, 0, max_slot); + } + + fn populate_transaction_statuses_for_test( + blockstore: &Blockstore, + min_slot: u64, + max_slot: u64, + ) { + for x in min_slot..=max_slot { let entries = make_slot_entries_with_transactions(1); let shreds = entries_to_test_shreds( &entries, @@ -606,6 +614,44 @@ pub mod tests { } } + fn populate_deprecated_transaction_statuses_for_test( + blockstore: &Blockstore, + primary_index: u64, + min_slot: u64, + max_slot: u64, + ) { + for x in min_slot..=max_slot { + let entries = make_slot_entries_with_transactions(1); + let shreds = entries_to_test_shreds( + &entries, + x, // slot + x.saturating_sub(1), // parent_slot + true, // is_full_slot + 0, // version + true, // merkle_variant + ); + blockstore.insert_shreds(shreds, None, false).unwrap(); + let signature = entries + .iter() + .filter(|entry| !entry.is_tick()) + .cloned() + .flat_map(|entry| entry.transactions) + .map(|transaction| transaction.signatures[0]) + .collect::>()[0]; + let random_bytes: Vec = (0..64).map(|_| rand::random::()).collect(); + blockstore + .write_deprecated_transaction_status( + primary_index, + x, + signature, + vec![&Pubkey::try_from(&random_bytes[..32]).unwrap()], + vec![&Pubkey::try_from(&random_bytes[32..]).unwrap()], + TransactionStatusMeta::default(), + ) + .unwrap(); + } + } + #[test] fn test_special_columns_empty() { let ledger_path = get_tmp_ledger_path_auto_delete!(); diff --git a/ledger/src/blockstore_db.rs b/ledger/src/blockstore_db.rs index 204ccf73c82d34..bb2313c515cbd7 100644 --- a/ledger/src/blockstore_db.rs +++ b/ledger/src/blockstore_db.rs @@ -2177,4 +2177,17 @@ pub mod tests { .put_cf(self.handle(), &C::deprecated_key(key), &serialized_value) } } + + impl LedgerColumn + where + C: ColumnIndexDeprecation + ColumnName, + { + pub(crate) fn iterator_cf_raw_key( + &self, + iterator_mode: IteratorMode>, + ) -> DBIterator { + let cf = self.handle(); + self.backend.iterator_cf_raw_key(cf, iterator_mode) + } + } } From 645172204acdcf55851888e0bea676515eeba030 Mon Sep 17 00:00:00 2001 From: Tyera Eulberg Date: Mon, 2 Oct 2023 20:25:34 -0600 Subject: [PATCH 41/57] Add test cases for purge_special_columns_with_old_data --- ledger/src/blockstore/blockstore_purge.rs | 154 ++++++++++++++++++++++ 1 file changed, 154 insertions(+) diff --git a/ledger/src/blockstore/blockstore_purge.rs b/ledger/src/blockstore/blockstore_purge.rs index 8ea31b41e5a64b..56a45ce6adb89e 100644 --- a/ledger/src/blockstore/blockstore_purge.rs +++ b/ledger/src/blockstore/blockstore_purge.rs @@ -507,6 +507,7 @@ pub mod tests { message::Message, transaction::Transaction, }, + test_case::test_case, }; #[test] @@ -766,6 +767,159 @@ pub mod tests { assert_eq!(status_entry_iterator.next(), None); } + fn get_index_bounds(blockstore: &Blockstore) -> (Box<[u8]>, Box<[u8]>) { + let first_index = { + let mut status_entry_iterator = blockstore + .transaction_status_cf + .iterator_cf_raw_key(IteratorMode::Start); + status_entry_iterator.next().unwrap().unwrap().0 + }; + let last_index = { + let mut status_entry_iterator = blockstore + .transaction_status_cf + .iterator_cf_raw_key(IteratorMode::End); + status_entry_iterator.next().unwrap().unwrap().0 + }; + (first_index, last_index) + } + + fn purge_exact(blockstore: &Blockstore, oldest_slot: Slot) { + blockstore + .run_purge(0, oldest_slot - 1, PurgeType::Exact) + .unwrap(); + } + + fn purge_compaction_filter(blockstore: &Blockstore, oldest_slot: Slot) { + let (first_index, last_index) = get_index_bounds(blockstore); + blockstore.db.set_oldest_slot(oldest_slot); + blockstore + .db + .compact_range_cf::(&first_index, &last_index); + } + + #[test_case(purge_exact; "exact")] + #[test_case(purge_compaction_filter; "compaction_filter")] + fn test_purge_special_columns_with_old_data(purge: impl Fn(&Blockstore, Slot)) { + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); + + populate_deprecated_transaction_statuses_for_test(&blockstore, 0, 0, 4); + populate_deprecated_transaction_statuses_for_test(&blockstore, 1, 5, 9); + populate_transaction_statuses_for_test(&blockstore, 10, 14); + + let mut index0 = blockstore + .transaction_status_index_cf + .get(0) + .unwrap() + .unwrap(); + index0.frozen = true; + index0.max_slot = 4; + blockstore + .transaction_status_index_cf + .put(0, &index0) + .unwrap(); + let mut index1 = blockstore + .transaction_status_index_cf + .get(1) + .unwrap() + .unwrap(); + index1.frozen = false; + index1.max_slot = 9; + blockstore + .transaction_status_index_cf + .put(1, &index1) + .unwrap(); + + let statuses: Vec<_> = blockstore + .transaction_status_cf + .iterator_cf_raw_key(IteratorMode::Start) + .collect(); + assert_eq!(statuses.len(), 15); + + // Delete some of primary-index 0 + let oldest_slot = 3; + purge(&blockstore, oldest_slot); + let status_entry_iterator = blockstore + .transaction_status_cf + .iterator_cf_raw_key(IteratorMode::Start); + let mut count = 0; + for entry in status_entry_iterator { + let (key, _value) = entry.unwrap(); + let (_signature, slot) = ::index(&key); + assert!(slot >= oldest_slot); + count += 1; + } + assert_eq!(count, 12); + + // Delete the rest of primary-index 0 + let oldest_slot = 5; + purge(&blockstore, oldest_slot); + let status_entry_iterator = blockstore + .transaction_status_cf + .iterator_cf_raw_key(IteratorMode::Start); + let mut count = 0; + for entry in status_entry_iterator { + let (key, _value) = entry.unwrap(); + let (_signature, slot) = ::index(&key); + assert!(slot >= oldest_slot); + count += 1; + } + assert_eq!(count, 10); + + // Delete some of primary-index 1 + let oldest_slot = 8; + purge(&blockstore, oldest_slot); + let status_entry_iterator = blockstore + .transaction_status_cf + .iterator_cf_raw_key(IteratorMode::Start); + let mut count = 0; + for entry in status_entry_iterator { + let (key, _value) = entry.unwrap(); + let (_signature, slot) = ::index(&key); + assert!(slot >= oldest_slot); + count += 1; + } + assert_eq!(count, 7); + + // Delete the rest of primary-index 1 + let oldest_slot = 10; + purge(&blockstore, oldest_slot); + let status_entry_iterator = blockstore + .transaction_status_cf + .iterator_cf_raw_key(IteratorMode::Start); + let mut count = 0; + for entry in status_entry_iterator { + let (key, _value) = entry.unwrap(); + let (_signature, slot) = ::index(&key); + assert!(slot >= oldest_slot); + count += 1; + } + assert_eq!(count, 5); + + // Delete some of new-style entries + let oldest_slot = 13; + purge(&blockstore, oldest_slot); + let status_entry_iterator = blockstore + .transaction_status_cf + .iterator_cf_raw_key(IteratorMode::Start); + let mut count = 0; + for entry in status_entry_iterator { + let (key, _value) = entry.unwrap(); + let (_signature, slot) = ::index(&key); + assert!(slot >= oldest_slot); + count += 1; + } + assert_eq!(count, 2); + + // Delete the rest of the new-style entries + let oldest_slot = 20; + purge(&blockstore, oldest_slot); + let mut status_entry_iterator = blockstore + .transaction_status_cf + .iterator_cf_raw_key(IteratorMode::Start); + assert!(status_entry_iterator.next().is_none()); + } + #[test] fn test_purge_special_columns_exact_no_sigs() { let ledger_path = get_tmp_ledger_path_auto_delete!(); From 0e4e382577de27f3111272722862e6fd278b8309 Mon Sep 17 00:00:00 2001 From: Tyera Eulberg Date: Tue, 3 Oct 2023 18:51:04 -0600 Subject: [PATCH 42/57] Add test_read_transaction_status_with_old_data --- ledger/src/blockstore.rs | 70 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/ledger/src/blockstore.rs b/ledger/src/blockstore.rs index 17c3553a60a5ab..2a17838055ff6d 100644 --- a/ledger/src/blockstore.rs +++ b/ledger/src/blockstore.rs @@ -7459,6 +7459,76 @@ pub mod tests { assert_eq!(compute_units_consumed, compute_units_consumed_2); } + #[test] + fn test_read_transaction_status_with_old_data() { + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); + let signature = Signature::from([1; 64]); + + let index0_slot = 2; + blockstore + .write_deprecated_transaction_status( + 0, + index0_slot, + signature, + vec![&Pubkey::new_unique()], + vec![&Pubkey::new_unique()], + TransactionStatusMeta { + fee: index0_slot * 1_000, + ..TransactionStatusMeta::default() + }, + ) + .unwrap(); + + let index1_slot = 1; + blockstore + .write_deprecated_transaction_status( + 1, + index1_slot, + signature, + vec![&Pubkey::new_unique()], + vec![&Pubkey::new_unique()], + TransactionStatusMeta { + fee: index1_slot * 1_000, + ..TransactionStatusMeta::default() + }, + ) + .unwrap(); + + let slot = 3; + blockstore + .write_transaction_status( + slot, + signature, + vec![&Pubkey::new_unique()], + vec![&Pubkey::new_unique()], + TransactionStatusMeta { + fee: slot * 1_000, + ..TransactionStatusMeta::default() + }, + 0, + ) + .unwrap(); + + let meta = blockstore + .read_transaction_status((signature, slot)) + .unwrap() + .unwrap(); + assert_eq!(meta.fee, slot * 1000); + + let meta = blockstore + .read_transaction_status((signature, index0_slot)) + .unwrap() + .unwrap(); + assert_eq!(meta.fee, index0_slot * 1000); + + let meta = blockstore + .read_transaction_status((signature, index1_slot)) + .unwrap() + .unwrap(); + assert_eq!(meta.fee, index1_slot * 1000); + } + #[test] fn test_get_transaction_status() { let ledger_path = get_tmp_ledger_path_auto_delete!(); From 1794f1cd9a2ea1a05bc533dc89455b4c0d6ac188 Mon Sep 17 00:00:00 2001 From: Tyera Eulberg Date: Tue, 3 Oct 2023 19:21:39 -0600 Subject: [PATCH 43/57] Add test_get_transaction_status_with_old_data --- ledger/src/blockstore.rs | 125 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 125 insertions(+) diff --git a/ledger/src/blockstore.rs b/ledger/src/blockstore.rs index 2a17838055ff6d..d20121fefe883c 100644 --- a/ledger/src/blockstore.rs +++ b/ledger/src/blockstore.rs @@ -7705,6 +7705,131 @@ pub mod tests { assert_eq!(counter, 0); } + #[test] + fn test_get_transaction_status_with_old_data() { + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let blockstore = Blockstore::open(ledger_path.path()).unwrap(); + let transaction_status_cf = &blockstore.transaction_status_cf; + + let pre_balances_vec = vec![1, 2, 3]; + let post_balances_vec = vec![3, 2, 1]; + let status = TransactionStatusMeta { + status: solana_sdk::transaction::Result::<()>::Ok(()), + fee: 42u64, + pre_balances: pre_balances_vec, + post_balances: post_balances_vec, + inner_instructions: Some(vec![]), + log_messages: Some(vec![]), + pre_token_balances: Some(vec![]), + post_token_balances: Some(vec![]), + rewards: Some(vec![]), + loaded_addresses: LoadedAddresses::default(), + return_data: Some(TransactionReturnData::default()), + compute_units_consumed: Some(42u64), + } + .into(); + + let signature1 = Signature::from([1u8; 64]); + let signature2 = Signature::from([2u8; 64]); + let signature3 = Signature::from([3u8; 64]); + let signature4 = Signature::from([4u8; 64]); + let signature5 = Signature::from([5u8; 64]); + let signature6 = Signature::from([6u8; 64]); + + // Insert slots with fork + // 0 (root) + // / \ + // 1 | + // 2 (root) + // / | + // 3 | + // 4 (root) + // | + // 5 + let meta0 = SlotMeta::new(0, Some(0)); + blockstore.meta_cf.put(0, &meta0).unwrap(); + let meta1 = SlotMeta::new(1, Some(0)); + blockstore.meta_cf.put(1, &meta1).unwrap(); + let meta2 = SlotMeta::new(2, Some(0)); + blockstore.meta_cf.put(2, &meta2).unwrap(); + let meta3 = SlotMeta::new(3, Some(2)); + blockstore.meta_cf.put(3, &meta3).unwrap(); + let meta4 = SlotMeta::new(4, Some(2)); + blockstore.meta_cf.put(4, &meta4).unwrap(); + let meta5 = SlotMeta::new(5, Some(4)); + blockstore.meta_cf.put(5, &meta5).unwrap(); + + blockstore.set_roots([0, 2, 4].iter()).unwrap(); + + // Initialize statuses: + // signature1 in skipped slot and root (2), both index 1 + // signature2 in skipped slot and root (4), both index 0 + // signature3 in root + // signature4 in non-root, + // signature5 extra entries + transaction_status_cf + .put_deprecated_protobuf((1, signature1, 1), &status) + .unwrap(); + + transaction_status_cf + .put_deprecated_protobuf((1, signature1, 2), &status) + .unwrap(); + + transaction_status_cf + .put_deprecated_protobuf((0, signature2, 3), &status) + .unwrap(); + + transaction_status_cf + .put_deprecated_protobuf((0, signature2, 4), &status) + .unwrap(); + + transaction_status_cf + .put_protobuf((signature3, 4), &status) + .unwrap(); + + transaction_status_cf + .put_protobuf((signature4, 5), &status) + .unwrap(); + + transaction_status_cf + .put_protobuf((signature5, 5), &status) + .unwrap(); + + // Signature exists, root found in index 1 + if let (Some((slot, _status)), counter) = blockstore + .get_transaction_status_with_counter(signature1, &[].into()) + .unwrap() + { + assert_eq!(slot, 2); + assert_eq!(counter, 4); + } + + // Signature exists, root found in index 0 + if let (Some((slot, _status)), counter) = blockstore + .get_transaction_status_with_counter(signature2, &[].into()) + .unwrap() + { + assert_eq!(slot, 4); + assert_eq!(counter, 3); + } + + // Signature exists + if let (Some((slot, _status)), counter) = blockstore + .get_transaction_status_with_counter(signature3, &[].into()) + .unwrap() + { + assert_eq!(slot, 4); + assert_eq!(counter, 1); + } + + // Signature does not exist + let (status, counter) = blockstore + .get_transaction_status_with_counter(signature6, &[].into()) + .unwrap(); + assert_eq!(status, None); + assert_eq!(counter, 1); + } + fn do_test_lowest_cleanup_slot_and_special_cfs(simulate_ledger_cleanup_service: bool) { solana_logger::setup(); From ad70443a77010bb7a4a4ad5b5a4229e41ed9a968 Mon Sep 17 00:00:00 2001 From: Tyera Eulberg Date: Thu, 5 Oct 2023 17:50:08 -0600 Subject: [PATCH 44/57] Review comments --- ledger/src/blockstore.rs | 2 +- ledger/src/blockstore_db.rs | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/ledger/src/blockstore.rs b/ledger/src/blockstore.rs index d20121fefe883c..2fcb9e55b626a5 100644 --- a/ledger/src/blockstore.rs +++ b/ledger/src/blockstore.rs @@ -2696,7 +2696,7 @@ impl Blockstore { IteratorDirection::Reverse, ))?; - // Iterate through starting_iterator until limit is reached + // Iterate until limit is reached while address_signatures.len() < limit { if let Some(((key_address, slot, _transaction_index, signature), _)) = iterator.next() { if slot < lowest_slot { diff --git a/ledger/src/blockstore_db.rs b/ledger/src/blockstore_db.rs index bb2313c515cbd7..9d1ab69dc370df 100644 --- a/ledger/src/blockstore_db.rs +++ b/ledger/src/blockstore_db.rs @@ -696,6 +696,9 @@ pub trait Column { fn key(index: Self::Index) -> Vec; fn index(key: &[u8]) -> Self::Index; + // This trait method is primarily used by `Database::delete_range_cf()`, and is therefore only + // relevant for columns keyed by Slot: ie. SlotColumns and columns that feature a Slot as the + // first item in the key. fn as_index(slot: Slot) -> Self::Index; fn slot(index: Self::Index) -> Slot; } @@ -791,8 +794,8 @@ pub trait ColumnIndexDeprecation: Column { impl Column for columns::TransactionStatus { type Index = (Signature, Slot); - fn key((signature, slot): (Signature, Slot)) -> Vec { - let mut key = vec![0; Self::CURRENT_INDEX_LEN]; // size_of Signature + size_of Slot + fn key((signature, slot): Self::Index) -> Vec { + let mut key = vec![0; Self::CURRENT_INDEX_LEN]; key[0..64].copy_from_slice(&signature.as_ref()[0..64]); BigEndian::write_u64(&mut key[64..72], slot); key From 3a757da1da8a96c89a3f447984c728cd81f44ebc Mon Sep 17 00:00:00 2001 From: Tyera Eulberg Date: Thu, 5 Oct 2023 19:21:18 -0600 Subject: [PATCH 45/57] Move rev of block-signatures into helper --- ledger/src/blockstore.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/ledger/src/blockstore.rs b/ledger/src/blockstore.rs index 2fcb9e55b626a5..4e6e425baf8652 100644 --- a/ledger/src/blockstore.rs +++ b/ledger/src/blockstore.rs @@ -2578,7 +2578,7 @@ impl Blockstore { .map(|signatures| signatures.iter().map(|(_, signature)| *signature).collect()) } - fn get_block_signatures(&self, slot: Slot) -> Result> { + fn get_block_signatures_rev(&self, slot: Slot) -> Result> { let block = self.get_complete_block(slot, false).map_err(|err| { BlockstoreError::Io(IoError::new( ErrorKind::Other, @@ -2589,6 +2589,7 @@ impl Blockstore { Ok(block .transactions .into_iter() + .rev() .filter_map(|transaction_with_meta| { transaction_with_meta .transaction @@ -2629,8 +2630,7 @@ impl Blockstore { match transaction_status { None => return Ok(SignatureInfosForAddress::default()), Some((slot, _)) => { - let mut slot_signatures = self.get_block_signatures(slot)?; - slot_signatures.reverse(); + let mut slot_signatures = self.get_block_signatures_rev(slot)?; if let Some(pos) = slot_signatures.iter().position(|&x| x == before) { slot_signatures.truncate(pos + 1); } @@ -2657,8 +2657,7 @@ impl Blockstore { match transaction_status { None => (first_available_block, HashSet::new()), Some((slot, _)) => { - let mut slot_signatures = self.get_block_signatures(slot)?; - slot_signatures.reverse(); + let mut slot_signatures = self.get_block_signatures_rev(slot)?; if let Some(pos) = slot_signatures.iter().position(|&x| x == until) { slot_signatures = slot_signatures.split_off(pos); } From 55214b051fdc23fca75f70acfdaf654b04271830 Mon Sep 17 00:00:00 2001 From: Tyera Eulberg Date: Thu, 5 Oct 2023 19:32:32 -0600 Subject: [PATCH 46/57] Improve deprecated_key impls --- ledger/src/blockstore_db.rs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/ledger/src/blockstore_db.rs b/ledger/src/blockstore_db.rs index 9d1ab69dc370df..5010a0b3b777e9 100644 --- a/ledger/src/blockstore_db.rs +++ b/ledger/src/blockstore_db.rs @@ -828,11 +828,11 @@ impl ColumnIndexDeprecation for columns::TransactionStatus { const CURRENT_INDEX_LEN: usize = 72; type DeprecatedIndex = (u64, Signature, Slot); - fn deprecated_key(index: Self::DeprecatedIndex) -> Vec { - let (index, signature, slot) = index; - let mut key = vec![0; 8]; + fn deprecated_key((index, signature, slot): Self::DeprecatedIndex) -> Vec { + let mut key = vec![0; Self::DEPRECATED_INDEX_LEN]; BigEndian::write_u64(&mut key[0..8], index); - key.extend_from_slice(&Self::key((signature, slot))); + key[8..72].copy_from_slice(&signature.as_ref()[0..64]); + BigEndian::write_u64(&mut key[72..80], slot); key } @@ -897,8 +897,7 @@ impl ColumnIndexDeprecation for columns::AddressSignatures { const CURRENT_INDEX_LEN: usize = 108; type DeprecatedIndex = (u64, Pubkey, Slot, Signature); - fn deprecated_key(index: Self::DeprecatedIndex) -> Vec { - let (primary_index, pubkey, slot, signature) = index; + fn deprecated_key((primary_index, pubkey, slot, signature): Self::DeprecatedIndex) -> Vec { let mut key = vec![0; Self::DEPRECATED_INDEX_LEN]; BigEndian::write_u64(&mut key[0..8], primary_index); key[8..40].clone_from_slice(&pubkey.as_ref()[0..32]); From 08b2e4f3c5acc23a089ce9a93c603db1aacd47ea Mon Sep 17 00:00:00 2001 From: Tyera Eulberg Date: Thu, 5 Oct 2023 19:35:36 -0600 Subject: [PATCH 47/57] iter_filtered -> iter_current_index_filtered --- ledger/src/blockstore.rs | 46 ++++++++++++++++++------------------- ledger/src/blockstore_db.rs | 2 +- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/ledger/src/blockstore.rs b/ledger/src/blockstore.rs index 4e6e425baf8652..ebf39f037dc011 100644 --- a/ledger/src/blockstore.rs +++ b/ledger/src/blockstore.rs @@ -2325,12 +2325,12 @@ impl Blockstore { let (lock, _) = self.ensure_lowest_cleanup_slot(); let first_available_block = self.get_first_available_block()?; - let iterator = self - .transaction_status_cf - .iter_filtered(IteratorMode::From( - (signature, first_available_block), - IteratorDirection::Forward, - ))?; + let iterator = + self.transaction_status_cf + .iter_current_index_filtered(IteratorMode::From( + (signature, first_available_block), + IteratorDirection::Forward, + ))?; for ((sig, slot), _data) in iterator { counter += 1; @@ -2542,17 +2542,17 @@ impl Blockstore { if slot < lowest_available_slot { return Ok(signatures); } - let index_iterator = self - .address_signatures_cf - .iter_filtered(IteratorMode::From( - ( - pubkey, - slot.max(lowest_available_slot), - 0, - Signature::default(), - ), - IteratorDirection::Forward, - ))?; + let index_iterator = + self.address_signatures_cf + .iter_current_index_filtered(IteratorMode::From( + ( + pubkey, + slot.max(lowest_available_slot), + 0, + Signature::default(), + ), + IteratorDirection::Forward, + ))?; for ((address, transaction_slot, _transaction_index, signature), _) in index_iterator { if transaction_slot > slot || address != pubkey { break; @@ -2688,12 +2688,12 @@ impl Blockstore { get_initial_slot_timer.stop(); let mut address_signatures_iter_timer = Measure::start("iter_timer"); - let mut iterator = self - .address_signatures_cf - .iter_filtered(IteratorMode::From( - (address, slot, 0, Signature::default()), - IteratorDirection::Reverse, - ))?; + let mut iterator = + self.address_signatures_cf + .iter_current_index_filtered(IteratorMode::From( + (address, slot, 0, Signature::default()), + IteratorDirection::Reverse, + ))?; // Iterate until limit is reached while address_signatures.len() < limit { diff --git a/ledger/src/blockstore_db.rs b/ledger/src/blockstore_db.rs index 5010a0b3b777e9..af462e32fc8265 100644 --- a/ledger/src/blockstore_db.rs +++ b/ledger/src/blockstore_db.rs @@ -1714,7 +1714,7 @@ impl LedgerColumn where C: ColumnIndexDeprecation + ColumnName, { - pub(crate) fn iter_filtered( + pub(crate) fn iter_current_index_filtered( &self, iterator_mode: IteratorMode, ) -> Result)> + '_> { From d73872912a86e98c9ef43a41c10edd496810810f Mon Sep 17 00:00:00 2001 From: Tyera Eulberg Date: Thu, 5 Oct 2023 21:22:21 -0600 Subject: [PATCH 48/57] Add comment to explain why use the smallest (index, Signature) to seed the iterator --- ledger/src/blockstore.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ledger/src/blockstore.rs b/ledger/src/blockstore.rs index ebf39f037dc011..2cf320686aaf80 100644 --- a/ledger/src/blockstore.rs +++ b/ledger/src/blockstore.rs @@ -2691,6 +2691,11 @@ impl Blockstore { let mut iterator = self.address_signatures_cf .iter_current_index_filtered(IteratorMode::From( + // Ragardless of whether a `before` signature is provided, the latest relevant + // `slot` is queried directly with the `find_address_signatures_for_slot()` + // call above. Thus, this iterator starts at the lowest entry of `address, + // slot` and iterates backwards to continue reporting the next earliest + // signatures. (address, slot, 0, Signature::default()), IteratorDirection::Reverse, ))?; From dbaa70bf2bafd654ae33a01f7da42ebb2a43fd80 Mon Sep 17 00:00:00 2001 From: Tyera Eulberg Date: Fri, 6 Oct 2023 15:51:42 -0600 Subject: [PATCH 49/57] Impl ColumnIndexDeprecation for TransactionMemos (doesn't build) --- ledger/src/blockstore_db.rs | 48 ++++++++++++++++++++++++++++++------- 1 file changed, 39 insertions(+), 9 deletions(-) diff --git a/ledger/src/blockstore_db.rs b/ledger/src/blockstore_db.rs index af462e32fc8265..b81806b8bdbffc 100644 --- a/ledger/src/blockstore_db.rs +++ b/ledger/src/blockstore_db.rs @@ -935,30 +935,60 @@ impl ColumnIndexDeprecation for columns::AddressSignatures { } impl Column for columns::TransactionMemos { - type Index = Signature; + type Index = (Signature, Slot); - fn key(signature: Signature) -> Vec { - let mut key = vec![0; 64]; // size_of Signature + fn key((signature, slot): Self::Index) -> Vec { + let mut key = vec![0; Self::CURRENT_INDEX_LEN]; key[0..64].copy_from_slice(&signature.as_ref()[0..64]); + BigEndian::write_u64(&mut key[64..72], slot); key } - fn index(key: &[u8]) -> Signature { - Signature::try_from(&key[..64]).unwrap() + fn index(key: &[u8]) -> Self::Index { + ::index(key) } - fn slot(_index: Self::Index) -> Slot { - unimplemented!() + fn slot(index: Self::Index) -> Slot { + index.1 } - fn as_index(_index: u64) -> Self::Index { - Signature::default() + fn as_index(index: u64) -> Self::Index { + (Signature::default(), index) } } impl ColumnName for columns::TransactionMemos { const NAME: &'static str = TRANSACTION_MEMOS_CF; } +impl ColumnIndexDeprecation for columns::TransactionMemos { + const DEPRECATED_INDEX_LEN: usize = 64; + const CURRENT_INDEX_LEN: usize = 72; + type DeprecatedIndex = Signature; + + fn deprecated_key(signature: Self::DeprecatedIndex) -> Vec { + let mut key = vec![0; Self::DEPRECATED_INDEX_LEN]; + key[0..64].copy_from_slice(&signature.as_ref()[0..64]); + key + } + + fn try_deprecated_index(key: &[u8]) -> std::result::Result { + Signature::try_from(&key[..64]).map_err(|_| IndexError::UnpackError) + } + + fn try_current_index(key: &[u8]) -> std::result::Result { + if key.len() != Self::CURRENT_INDEX_LEN { + return Err(IndexError::UnpackError); + } + let signature = Signature::try_from(&key[0..64]).unwrap(); + let slot = BigEndian::read_u64(&key[64..72]); + Ok((signature, slot)) + } + + fn convert_index(deprecated_index: Self::DeprecatedIndex) -> Self::Index { + (deprecated_index, 0) + } +} + impl Column for columns::TransactionStatusIndex { type Index = u64; From a1609b5469f95f436ed1089b8535a87d6955e69b Mon Sep 17 00:00:00 2001 From: Tyera Eulberg Date: Fri, 6 Oct 2023 15:56:01 -0600 Subject: [PATCH 50/57] Update TransactionMemos put --- ledger/src/blockstore.rs | 9 +++++++-- rpc/src/transaction_status_service.rs | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/ledger/src/blockstore.rs b/ledger/src/blockstore.rs index 2cf320686aaf80..4f59ce2b7221e6 100644 --- a/ledger/src/blockstore.rs +++ b/ledger/src/blockstore.rs @@ -2277,8 +2277,13 @@ impl Blockstore { self.transaction_memos_cf.get(signature) } - pub fn write_transaction_memos(&self, signature: &Signature, memos: String) -> Result<()> { - self.transaction_memos_cf.put(*signature, &memos) + pub fn write_transaction_memos( + &self, + signature: &Signature, + slot: Slot, + memos: String, + ) -> Result<()> { + self.transaction_memos_cf.put((*signature, slot), &memos) } /// Acquires the `lowest_cleanup_slot` lock and returns a tuple of the held lock diff --git a/rpc/src/transaction_status_service.rs b/rpc/src/transaction_status_service.rs index aef1ab45f14260..ccc4364891dd75 100644 --- a/rpc/src/transaction_status_service.rs +++ b/rpc/src/transaction_status_service.rs @@ -188,7 +188,7 @@ impl TransactionStatusService { if enable_rpc_transaction_history { if let Some(memos) = extract_and_fmt_memos(transaction.message()) { blockstore - .write_transaction_memos(transaction.signature(), memos) + .write_transaction_memos(transaction.signature(), slot, memos) .expect("Expect database write to succeed: TransactionMemos"); } From 638868f34bf439c66cd371e7a0805d70acb47af8 Mon Sep 17 00:00:00 2001 From: Tyera Eulberg Date: Fri, 6 Oct 2023 16:31:58 -0600 Subject: [PATCH 51/57] Add LedgerColumn::get_raw --- ledger/src/blockstore_db.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ledger/src/blockstore_db.rs b/ledger/src/blockstore_db.rs index b81806b8bdbffc..316683996f5a52 100644 --- a/ledger/src/blockstore_db.rs +++ b/ledger/src/blockstore_db.rs @@ -1612,12 +1612,16 @@ where } pub fn get(&self, key: C::Index) -> Result> { + self.get_raw(&C::key(key)) + } + + pub fn get_raw(&self, key: &[u8]) -> Result> { let mut result = Ok(None); let is_perf_enabled = maybe_enable_rocksdb_perf( self.column_options.rocks_perf_sample_interval, &self.read_perf_status, ); - if let Some(pinnable_slice) = self.backend.get_pinned_cf(self.handle(), &C::key(key))? { + if let Some(pinnable_slice) = self.backend.get_pinned_cf(self.handle(), key)? { let value = deserialize(pinnable_slice.as_ref())?; result = Ok(Some(value)) } From c64f42c1c855a96ebadc4333ef38d9de5354ac18 Mon Sep 17 00:00:00 2001 From: Tyera Eulberg Date: Fri, 6 Oct 2023 16:32:37 -0600 Subject: [PATCH 52/57] Fix read_transaction_memos --- ledger/src/blockstore.rs | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/ledger/src/blockstore.rs b/ledger/src/blockstore.rs index 4f59ce2b7221e6..9b40c70e5a3e93 100644 --- a/ledger/src/blockstore.rs +++ b/ledger/src/blockstore.rs @@ -2273,8 +2273,18 @@ impl Blockstore { Ok(()) } - pub fn read_transaction_memos(&self, signature: Signature) -> Result> { - self.transaction_memos_cf.get(signature) + pub fn read_transaction_memos( + &self, + signature: Signature, + slot: Slot, + ) -> Result> { + let memos = self.transaction_memos_cf.get((signature, slot))?; + if memos.is_none() { + self.transaction_memos_cf + .get_raw(&cf::TransactionMemos::deprecated_key(signature)) + } else { + Ok(memos) + } } pub fn write_transaction_memos( @@ -2735,7 +2745,7 @@ impl Blockstore { let transaction_status = self.get_transaction_status(signature, &confirmed_unrooted_slots)?; let err = transaction_status.and_then(|(_slot, status)| status.status.err()); - let memo = self.read_transaction_memos(signature)?; + let memo = self.read_transaction_memos(signature, slot)?; let block_time = self.get_block_time(slot)?; infos.push(ConfirmedTransactionStatusWithSignature { signature, From aed03dd7ee3515d425060893177af1754f02ec0b Mon Sep 17 00:00:00 2001 From: Tyera Eulberg Date: Fri, 6 Oct 2023 16:41:48 -0600 Subject: [PATCH 53/57] Add TransactionMemos to purge_special_columns_exact --- ledger/src/blockstore/blockstore_purge.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ledger/src/blockstore/blockstore_purge.rs b/ledger/src/blockstore/blockstore_purge.rs index 56a45ce6adb89e..e2c6ef2fab78b6 100644 --- a/ledger/src/blockstore/blockstore_purge.rs +++ b/ledger/src/blockstore/blockstore_purge.rs @@ -407,6 +407,12 @@ impl Blockstore { if let Some(&signature) = transaction.signatures.get(0) { if delete_new_column_key { batch.delete::((signature, slot))?; + batch.delete::((signature, slot))?; + } + if !primary_indexes.is_empty() { + batch.delete_raw::( + &cf::TransactionMemos::deprecated_key(signature), + )?; } for primary_index in &primary_indexes { batch.delete_raw::( From 2aeacd29851c509bcf164154c5124a06530660eb Mon Sep 17 00:00:00 2001 From: Tyera Eulberg Date: Fri, 6 Oct 2023 16:43:17 -0600 Subject: [PATCH 54/57] Add TransactionMemos to compaction filter --- ledger/src/blockstore_db.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ledger/src/blockstore_db.rs b/ledger/src/blockstore_db.rs index 316683996f5a52..f1c73643b960d5 100644 --- a/ledger/src/blockstore_db.rs +++ b/ledger/src/blockstore_db.rs @@ -2092,7 +2092,9 @@ fn should_enable_cf_compaction(cf_name: &str) -> bool { // completed on a given range or file. matches!( cf_name, - columns::TransactionStatus::NAME | columns::AddressSignatures::NAME + columns::TransactionStatus::NAME + | columns::TransactionMemos::NAME + | columns::AddressSignatures::NAME ) } From 606835b8a792be717f2ba0ca188755201d438b99 Mon Sep 17 00:00:00 2001 From: Tyera Eulberg Date: Mon, 9 Oct 2023 10:46:46 -0600 Subject: [PATCH 55/57] Take find_address_signatures out of service --- ledger/src/blockstore.rs | 185 ++------------------------------------- 1 file changed, 8 insertions(+), 177 deletions(-) diff --git a/ledger/src/blockstore.rs b/ledger/src/blockstore.rs index 9b40c70e5a3e93..4ea608d3471c26 100644 --- a/ledger/src/blockstore.rs +++ b/ledger/src/blockstore.rs @@ -2502,47 +2502,15 @@ impl Blockstore { .find(|transaction| transaction.signatures[0] == signature)) } - // Returns all rooted signatures for an address, ordered by slot that the transaction was - // processed in. Within each slot the transactions will be ordered by signature, and NOT by - // the order in which the transactions exist in the block - // - // DEPRECATED + // DEPRECATED and decommissioned + // This method always returns an empty Vec fn find_address_signatures( &self, - pubkey: Pubkey, - start_slot: Slot, - end_slot: Slot, + _pubkey: Pubkey, + _start_slot: Slot, + _end_slot: Slot, ) -> Result> { - let (lock, lowest_available_slot) = self.ensure_lowest_cleanup_slot(); - let mut signatures: Vec<(Slot, Signature)> = vec![]; - if end_slot < lowest_available_slot { - return Ok(signatures); - } - for transaction_status_cf_primary_index in 0..=1 { - let index_iterator = - self.address_signatures_cf - .iter_deprecated_index_filtered(IteratorMode::From( - ( - transaction_status_cf_primary_index, - pubkey, - start_slot.max(lowest_available_slot), - Signature::default(), - ), - IteratorDirection::Forward, - ))?; - for ((i, address, slot, signature), _) in index_iterator { - if i != transaction_status_cf_primary_index || slot > end_slot || address != pubkey - { - break; - } - if self.is_root(slot) { - signatures.push((slot, signature)); - } - } - } - drop(lock); - signatures.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap().then(a.1.cmp(&b.1))); - Ok(signatures) + Ok(vec![]) } // Returns all signatures for an address in a particular slot, regardless of whether that slot @@ -2578,7 +2546,8 @@ impl Blockstore { Ok(signatures) } - // DEPRECATED + // DEPRECATED and decommissioned + // This method always returns an empty Vec pub fn get_confirmed_signatures_for_address( &self, pubkey: Pubkey, @@ -8264,144 +8233,6 @@ pub mod tests { } } - #[test] - fn test_get_confirmed_signatures_for_address() { - let ledger_path = get_tmp_ledger_path_auto_delete!(); - let blockstore = Blockstore::open(ledger_path.path()).unwrap(); - - let address0 = solana_sdk::pubkey::new_rand(); - let address1 = solana_sdk::pubkey::new_rand(); - - let slot0 = 10; - for x in 1..5 { - let signature = Signature::from([x; 64]); - blockstore - .write_deprecated_transaction_status( - 0, - slot0, - signature, - vec![&address0], - vec![&address1], - TransactionStatusMeta::default(), - ) - .unwrap(); - } - let slot1 = 20; - for x in 5..9 { - let signature = Signature::from([x; 64]); - blockstore - .write_deprecated_transaction_status( - 0, - slot1, - signature, - vec![&address0], - vec![&address1], - TransactionStatusMeta::default(), - ) - .unwrap(); - } - blockstore.set_roots([slot0, slot1].iter()).unwrap(); - - let all0 = blockstore - .get_confirmed_signatures_for_address(address0, 0, 50) - .unwrap(); - assert_eq!(all0.len(), 8); - for x in 1..9 { - let expected_signature = Signature::from([x; 64]); - assert_eq!(all0[x as usize - 1], expected_signature); - } - assert_eq!( - blockstore - .get_confirmed_signatures_for_address(address0, 20, 50) - .unwrap() - .len(), - 4 - ); - assert_eq!( - blockstore - .get_confirmed_signatures_for_address(address0, 0, 10) - .unwrap() - .len(), - 4 - ); - assert!(blockstore - .get_confirmed_signatures_for_address(address0, 1, 5) - .unwrap() - .is_empty()); - assert_eq!( - blockstore - .get_confirmed_signatures_for_address(address0, 1, 15) - .unwrap() - .len(), - 4 - ); - - let all1 = blockstore - .get_confirmed_signatures_for_address(address1, 0, 50) - .unwrap(); - assert_eq!(all1.len(), 8); - for x in 1..9 { - let expected_signature = Signature::from([x; 64]); - assert_eq!(all1[x as usize - 1], expected_signature); - } - - // Purge index 0 - blockstore - .run_purge(0, 10, PurgeType::PrimaryIndex) - .unwrap(); - assert_eq!( - blockstore - .get_confirmed_signatures_for_address(address0, 0, 50) - .unwrap() - .len(), - 4 - ); - assert_eq!( - blockstore - .get_confirmed_signatures_for_address(address0, 20, 50) - .unwrap() - .len(), - 4 - ); - assert!(blockstore - .get_confirmed_signatures_for_address(address0, 0, 10) - .unwrap() - .is_empty()); - assert!(blockstore - .get_confirmed_signatures_for_address(address0, 1, 5) - .unwrap() - .is_empty()); - assert_eq!( - blockstore - .get_confirmed_signatures_for_address(address0, 1, 25) - .unwrap() - .len(), - 4 - ); - - // Test sort, regardless of entry order or signature value - for slot in (21..25).rev() { - let random_bytes: [u8; 64] = std::array::from_fn(|_| rand::random::()); - let signature = Signature::from(random_bytes); - blockstore - .write_transaction_status( - slot, - signature, - vec![&address0], - vec![&address1], - TransactionStatusMeta::default(), - 0, - ) - .unwrap(); - } - blockstore.set_roots([21, 22, 23, 24].iter()).unwrap(); - let mut past_slot = 0; - for (slot, _) in blockstore.find_address_signatures(address0, 1, 25).unwrap() { - assert!(slot >= past_slot); - past_slot = slot; - } - } - #[test] fn test_find_address_signatures_for_slot() { let ledger_path = get_tmp_ledger_path_auto_delete!(); From 4b68c044f89b43460550e82031cbd49da6fd09e7 Mon Sep 17 00:00:00 2001 From: Tyera Eulberg Date: Mon, 9 Oct 2023 10:56:50 -0600 Subject: [PATCH 56/57] Remove faulty delete_new_column_key logic --- ledger/src/blockstore/blockstore_purge.rs | 28 ++++++++--------------- 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/ledger/src/blockstore/blockstore_purge.rs b/ledger/src/blockstore/blockstore_purge.rs index e2c6ef2fab78b6..5c1644aaa032a0 100644 --- a/ledger/src/blockstore/blockstore_purge.rs +++ b/ledger/src/blockstore/blockstore_purge.rs @@ -1,8 +1,6 @@ use { - super::*, - crate::blockstore_db::ColumnIndexDeprecation, - solana_sdk::message::AccountKeys, - std::{cmp, time::Instant}, + super::*, crate::blockstore_db::ColumnIndexDeprecation, solana_sdk::message::AccountKeys, + std::time::Instant, }; #[derive(Default)] @@ -396,8 +394,6 @@ impl Blockstore { for slot in from_slot..=to_slot { let primary_indexes = slot_indexes(slot); - let delete_new_column_key = - primary_indexes.is_empty() || (slot == cmp::max(index0.max_slot, index1.max_slot)); let slot_entries = self.get_any_valid_slot_entries(slot, 0); let transactions = slot_entries @@ -405,10 +401,8 @@ impl Blockstore { .flat_map(|entry| entry.transactions); for (i, transaction) in transactions.enumerate() { if let Some(&signature) = transaction.signatures.get(0) { - if delete_new_column_key { - batch.delete::((signature, slot))?; - batch.delete::((signature, slot))?; - } + batch.delete::((signature, slot))?; + batch.delete::((signature, slot))?; if !primary_indexes.is_empty() { batch.delete_raw::( &cf::TransactionMemos::deprecated_key(signature), @@ -434,14 +428,12 @@ impl Blockstore { let transaction_index = u32::try_from(i).map_err(|_| BlockstoreError::TransactionIndexOverflow)?; for pubkey in account_keys.iter() { - if delete_new_column_key { - batch.delete::(( - *pubkey, - slot, - transaction_index, - signature, - ))?; - } + batch.delete::(( + *pubkey, + slot, + transaction_index, + signature, + ))?; for primary_index in &primary_indexes { batch.delete_raw::( &cf::AddressSignatures::deprecated_key(( From 5923d840e18139e8faf057c82e06e5aee7f21d87 Mon Sep 17 00:00:00 2001 From: Tyera Eulberg Date: Mon, 9 Oct 2023 10:57:52 -0600 Subject: [PATCH 57/57] Simplify comments --- ledger/src/blockstore_db.rs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/ledger/src/blockstore_db.rs b/ledger/src/blockstore_db.rs index f1c73643b960d5..f9c87ce397d434 100644 --- a/ledger/src/blockstore_db.rs +++ b/ledger/src/blockstore_db.rs @@ -809,9 +809,8 @@ impl Column for columns::TransactionStatus { index.1 } - // This trait method is primarily used by Database::delete_range_cf(). Because the - // TransactionStatus Column is not keyed by slot, it cannot be range deleted. Therefore this - // method is meaningless for this type, and simply returns a default Index. + // The TransactionStatus column is not keyed by slot so this method is meaningless + // See Column::as_index() declaration for more details fn as_index(_index: u64) -> Self::Index { (Signature::default(), 0) } @@ -881,9 +880,8 @@ impl Column for columns::AddressSignatures { index.1 } - // This trait method is primarily used by Database::delete_range_cf(). Because the - // AddressSignatures Column is not keyed by slot, it cannot be range deleted. Therefore this - // method is meaningless for this type, and simply returns a default Index. + // The AddressSignatures column is not keyed by slot so this method is meaningless + // See Column::as_index() declaration for more details fn as_index(_index: u64) -> Self::Index { (Pubkey::default(), 0, 0, Signature::default()) }