Make account shrink configurable #17544 (backport #17778) (#18013)

* Make account shrink configurable #17544 (#17778)

1. Added both options for measuring space usage using total accounts usage and for individual store shrink ratio using an enum. Validator CLI options: --accounts-shrink-optimize-total-space and --accounts-shrink-ratio
2. Added code for selecting candidates based on total usage in a separate function select_candidates_by_total_usage
3. Added unit tests for the new functions added
4. The default implementations is kept at 0.8 shrink ratio with --accounts-shrink-optimize-total-space set to true

Fixes #17544

(cherry picked from commit 269d995)

# Conflicts:
#	core/tests/snapshots.rs
#	ledger/src/bank_forks_utils.rs
#	runtime/src/accounts_db.rs
#	runtime/src/snapshot_utils.rs

* fix some merge errors

Co-authored-by: Lijun Wang <83639177+lijunwangs@users.noreply.github.com>
Co-authored-by: Jeff Washington (jwash) <wash678@gmail.com>

accounts-bench/src/main.rs

@@ -6,6 +6,7 @@ use rayon::prelude::*;
 use solana_measure::measure::Measure;
 use solana_runtime::{
     accounts::{create_test_accounts, update_accounts_bench, Accounts},
+    accounts_db::AccountShrinkThreshold,
     accounts_index::AccountSecondaryIndexes,
     ancestors::Ancestors,
 };
@@ -64,6 +65,7 @@ fn main() {
         &ClusterType::Testnet,
         AccountSecondaryIndexes::default(),
         false,
+        AccountShrinkThreshold::default(),
     );
     println!("Creating {} accounts", num_accounts);
     let mut create_time = Measure::start("create accounts");

core/src/tvu.rs

@@ -37,6 +37,7 @@ use solana_runtime::{
     accounts_background_service::{
         AbsRequestHandler, AbsRequestSender, AccountsBackgroundService, SnapshotRequestHandler,
     },
+    accounts_db::AccountShrinkThreshold,
     bank_forks::{BankForks, SnapshotConfig},
     commitment::BlockCommitmentCache,
     vote_sender_types::ReplayVoteSender,
@@ -88,6 +89,7 @@ pub struct TvuConfig {
     pub rocksdb_compaction_interval: Option<u64>,
     pub rocksdb_max_compaction_jitter: Option<u64>,
     pub wait_for_vote_to_start_leader: bool,
+    pub accounts_shrink_ratio: AccountShrinkThreshold,
 }
 
 impl Tvu {

core/src/validator.rs

@@ -52,6 +52,7 @@ use solana_rpc::{
     transaction_status_service::TransactionStatusService,
 };
 use solana_runtime::{
+    accounts_db::AccountShrinkThreshold,
     accounts_index::AccountSecondaryIndexes,
     bank::Bank,
     bank_forks::{BankForks, SnapshotConfig},
@@ -138,6 +139,7 @@ pub struct ValidatorConfig {
     pub tpu_coalesce_ms: u64,
     pub validator_exit: Arc<RwLock<Exit>>,
     pub no_wait_for_vote_to_start_leader: bool,
+    pub accounts_shrink_ratio: AccountShrinkThreshold,
 }
 
 impl Default for ValidatorConfig {
@@ -194,6 +196,7 @@ impl Default for ValidatorConfig {
             tpu_coalesce_ms: DEFAULT_TPU_COALESCE_MS,
             validator_exit: Arc::new(RwLock::new(Exit::default())),
             no_wait_for_vote_to_start_leader: true,
+            accounts_shrink_ratio: AccountShrinkThreshold::default(),
         }
     }
 }
@@ -717,6 +720,7 @@ impl Validator {
                 rocksdb_compaction_interval: config.rocksdb_compaction_interval,
                 rocksdb_max_compaction_jitter: config.rocksdb_compaction_interval,
                 wait_for_vote_to_start_leader,
+                accounts_shrink_ratio: config.accounts_shrink_ratio,
             },
             &max_slots,
         );
@@ -1093,6 +1097,7 @@ fn new_banks_from_ledger(
         debug_keys: config.debug_keys.clone(),
         account_indexes: config.account_indexes.clone(),
         accounts_db_caching_enabled: config.accounts_db_caching_enabled,
+        shrink_ratio: config.accounts_shrink_ratio,
         ..blockstore_processor::ProcessOptions::default()
     };
 

core/tests/snapshots.rs

@@ -106,6 +106,7 @@ mod tests {
                 None,
                 AccountSecondaryIndexes::default(),
                 false,
+                accounts_db::AccountShrinkThreshold::default(),
                 false,
             );
             bank0.freeze();
@@ -167,6 +168,7 @@ mod tests {
             AccountSecondaryIndexes::default(),
             false,
             None,
+            accounts_db::AccountShrinkThreshold::default(),
             check_hash_calculation,
         )
         .unwrap();

ledger/src/bank_forks_utils.rs

@@ -142,6 +142,7 @@ fn load_from_snapshot(
         process_options.account_indexes.clone(),
         process_options.accounts_db_caching_enabled,
         process_options.limit_load_slot_count_from_snapshot,
+        process_options.shrink_ratio,
         process_options.accounts_db_test_hash_calculation,
     )
     .expect("Load from snapshot failed");

ledger/src/blockstore_processor.rs

@@ -16,6 +16,7 @@ use solana_measure::measure::Measure;
 use solana_metrics::{datapoint_error, inc_new_counter_debug};
 use solana_rayon_threadlimit::get_thread_count;
 use solana_runtime::{
+    accounts_db::AccountShrinkThreshold,
     accounts_index::AccountSecondaryIndexes,
     bank::{
         Bank, ExecuteTimings, InnerInstructionsList, RentDebits, TransactionBalancesSet,
@@ -374,6 +375,7 @@ pub struct ProcessOptions {
     pub limit_load_slot_count_from_snapshot: Option<usize>,
     pub allow_dead_slots: bool,
     pub accounts_db_test_hash_calculation: bool,
+    pub shrink_ratio: AccountShrinkThreshold,
 }
 
 pub fn process_blockstore(
@@ -401,6 +403,7 @@ pub fn process_blockstore(
         Some(&crate::builtins::get(opts.bpf_jit)),
         opts.account_indexes.clone(),
         opts.accounts_db_caching_enabled,
+        opts.shrink_ratio,
         false,
     );
     let bank0 = Arc::new(bank0);
@@ -3097,6 +3100,7 @@ pub mod tests {
             None,
             AccountSecondaryIndexes::default(),
             false,
+            AccountShrinkThreshold::default(),
             false,
         );
         *bank.epoch_schedule()

local-cluster/src/validator_configs.rs

@@ -56,6 +56,7 @@ pub fn safe_clone_config(config: &ValidatorConfig) -> ValidatorConfig {
         validator_exit: Arc::new(RwLock::new(Exit::default())),
         poh_hashes_per_batch: config.poh_hashes_per_batch,
         no_wait_for_vote_to_start_leader: config.no_wait_for_vote_to_start_leader,
+        accounts_shrink_ratio: config.accounts_shrink_ratio,
     }
 }
 

runtime/benches/accounts.rs

@@ -8,6 +8,7 @@ use rand::Rng;
 use rayon::iter::{IntoParallelRefIterator, ParallelIterator};
 use solana_runtime::{
     accounts::{create_test_accounts, AccountAddressFilter, Accounts},
+    accounts_db::AccountShrinkThreshold,
     accounts_index::AccountSecondaryIndexes,
     ancestors::Ancestors,
     bank::*,
@@ -59,6 +60,7 @@ fn test_accounts_create(bencher: &mut Bencher) {
         None,
         AccountSecondaryIndexes::default(),
         false,
+        AccountShrinkThreshold::default(),
         false,
     );
     bencher.iter(|| {
@@ -79,6 +81,7 @@ fn test_accounts_squash(bencher: &mut Bencher) {
         None,
         AccountSecondaryIndexes::default(),
         false,
+        AccountShrinkThreshold::default(),
         false,
     ));
     let mut pubkeys: Vec<Pubkey> = vec![];
@@ -105,6 +108,7 @@ fn test_accounts_hash_bank_hash(bencher: &mut Bencher) {
         &ClusterType::Development,
         AccountSecondaryIndexes::default(),
         false,
+        AccountShrinkThreshold::default(),
     );
     let mut pubkeys: Vec<Pubkey> = vec![];
     let num_accounts = 60_000;
@@ -131,6 +135,7 @@ fn test_update_accounts_hash(bencher: &mut Bencher) {
         &ClusterType::Development,
         AccountSecondaryIndexes::default(),
         false,
+        AccountShrinkThreshold::default(),
     );
     let mut pubkeys: Vec<Pubkey> = vec![];
     create_test_accounts(&accounts, &mut pubkeys, 50_000, 0);
@@ -148,6 +153,7 @@ fn test_accounts_delta_hash(bencher: &mut Bencher) {
         &ClusterType::Development,
         AccountSecondaryIndexes::default(),
         false,
+        AccountShrinkThreshold::default(),
     );
     let mut pubkeys: Vec<Pubkey> = vec![];
     create_test_accounts(&accounts, &mut pubkeys, 100_000, 0);
@@ -164,6 +170,7 @@ fn bench_delete_dependencies(bencher: &mut Bencher) {
         &ClusterType::Development,
         AccountSecondaryIndexes::default(),
         false,
+        AccountShrinkThreshold::default(),
     );
     let mut old_pubkey = Pubkey::default();
     let zero_account = AccountSharedData::new(0, 0, AccountSharedData::default().owner());
@@ -197,6 +204,7 @@ fn store_accounts_with_possible_contention<F: 'static>(
         &ClusterType::Development,
         AccountSecondaryIndexes::default(),
         false,
+        AccountShrinkThreshold::default(),
     ));
     let num_keys = 1000;
     let slot = 0;
@@ -326,6 +334,7 @@ fn setup_bench_dashmap_iter() -> (Arc<Accounts>, DashMap<Pubkey, (AccountSharedD
         &ClusterType::Development,
         AccountSecondaryIndexes::default(),
         false,
+        AccountShrinkThreshold::default(),
     ));
 
     let dashmap = DashMap::new();
@@ -380,6 +389,7 @@ fn bench_load_largest_accounts(b: &mut Bencher) {
         &ClusterType::Development,
         AccountSecondaryIndexes::default(),
         false,
+        AccountShrinkThreshold::default(),
     );
     let mut rng = rand::thread_rng();
     for _ in 0..10_000 {

runtime/src/accounts.rs

@@ -1,6 +1,7 @@
 use crate::{
     accounts_db::{
-        AccountsDb, BankHashInfo, ErrorCounters, LoadHint, LoadedAccount, ScanStorageResult,
+        AccountShrinkThreshold, AccountsDb, BankHashInfo, ErrorCounters, LoadHint, LoadedAccount,
+        ScanStorageResult,
     },
     accounts_index::{AccountSecondaryIndexes, IndexKey},
     ancestors::Ancestors,
@@ -117,12 +118,17 @@ pub enum AccountAddressFilter {
 }
 
 impl Accounts {
-    pub fn new(paths: Vec<PathBuf>, cluster_type: &ClusterType) -> Self {
+    pub fn new(
+        paths: Vec<PathBuf>,
+        cluster_type: &ClusterType,
+        shrink_ratio: AccountShrinkThreshold,
+    ) -> Self {
         Self::new_with_config(
             paths,
             cluster_type,
             AccountSecondaryIndexes::default(),
             false,
+            shrink_ratio,
         )
     }
 
@@ -131,13 +137,15 @@ impl Accounts {
         cluster_type: &ClusterType,
         account_indexes: AccountSecondaryIndexes,
         caching_enabled: bool,
+        shrink_ratio: AccountShrinkThreshold,
     ) -> Self {
         Self {
             accounts_db: Arc::new(AccountsDb::new_with_config(
                 paths,
                 cluster_type,
                 account_indexes,
                 caching_enabled,
+                shrink_ratio,
             )),
             account_locks: Mutex::new(AccountLocks::default()),
         }
@@ -1125,6 +1133,7 @@ mod tests {
             &ClusterType::Development,
             AccountSecondaryIndexes::default(),
             false,
+            AccountShrinkThreshold::default(),
         );
         for ka in ka.iter() {
             accounts.store_slow_uncached(0, &ka.0, &ka.1);
@@ -1662,6 +1671,7 @@ mod tests {
             &ClusterType::Development,
             AccountSecondaryIndexes::default(),
             false,
+            AccountShrinkThreshold::default(),
         );
 
         // Load accounts owned by various programs into AccountsDb
@@ -1690,6 +1700,7 @@ mod tests {
             &ClusterType::Development,
             AccountSecondaryIndexes::default(),
             false,
+            AccountShrinkThreshold::default(),
         );
         let mut error_counters = ErrorCounters::default();
         let ancestors = vec![(0, 0)].into_iter().collect();
@@ -1713,6 +1724,7 @@ mod tests {
             &ClusterType::Development,
             AccountSecondaryIndexes::default(),
             false,
+            AccountShrinkThreshold::default(),
         );
         accounts.bank_hash_at(1);
     }
@@ -1734,6 +1746,7 @@ mod tests {
             &ClusterType::Development,
             AccountSecondaryIndexes::default(),
             false,
+            AccountShrinkThreshold::default(),
         );
         accounts.store_slow_uncached(0, &keypair0.pubkey(), &account0);
         accounts.store_slow_uncached(0, &keypair1.pubkey(), &account1);
@@ -1860,6 +1873,7 @@ mod tests {
             &ClusterType::Development,
             AccountSecondaryIndexes::default(),
             false,
+            AccountShrinkThreshold::default(),
         );
         accounts.store_slow_uncached(0, &keypair0.pubkey(), &account0);
         accounts.store_slow_uncached(0, &keypair1.pubkey(), &account1);
@@ -2010,6 +2024,7 @@ mod tests {
             &ClusterType::Development,
             AccountSecondaryIndexes::default(),
             false,
+            AccountShrinkThreshold::default(),
         );
         {
             accounts
@@ -2062,6 +2077,7 @@ mod tests {
             &ClusterType::Development,
             AccountSecondaryIndexes::default(),
             false,
+            AccountShrinkThreshold::default(),
         );
         let mut old_pubkey = Pubkey::default();
         let zero_account = AccountSharedData::new(0, 0, AccountSharedData::default().owner());
@@ -2109,6 +2125,7 @@ mod tests {
             &ClusterType::Development,
             AccountSecondaryIndexes::default(),
             false,
+            AccountShrinkThreshold::default(),
         );
 
         let instructions_key = solana_sdk::sysvar::instructions::id();
@@ -2394,6 +2411,7 @@ mod tests {
             &ClusterType::Development,
             AccountSecondaryIndexes::default(),
             false,
+            AccountShrinkThreshold::default(),
         );
         let collected_accounts = accounts.collect_accounts_to_store(
             txs.iter(),
@@ -2513,6 +2531,7 @@ mod tests {
             &ClusterType::Development,
             AccountSecondaryIndexes::default(),
             false,
+            AccountShrinkThreshold::default(),
         );
         let collected_accounts = accounts.collect_accounts_to_store(
             txs.iter(),
@@ -2547,6 +2566,7 @@ mod tests {
             &ClusterType::Development,
             AccountSecondaryIndexes::default(),
             false,
+            AccountShrinkThreshold::default(),
         );
 
         let pubkey0 = Pubkey::new_unique();