fix: mitigate gc slowness by reducing step size (#2807)

A hot fix that mitigates #2806 by reducing garbage collection step to the absolute minimum. However, even if we just advance tail by 1 height at each step, `clear_data` still takes about 0.5s to execute, which is very suboptimal.

Test plan
----------
Deploy on betanet and observe that garbage collection speeds up from more than 30s to 0.5s at each step.

chain/chain/src/chain.rs

@@ -73,9 +73,6 @@ const NEAR_BASE: Balance = 1_000_000_000_000_000_000_000_000;
 /// Number of epochs for which we keep store data
 pub const NUM_EPOCHS_TO_KEEP_STORE_DATA: u64 = 5;
 
-/// Maximum number of heights to clear per one GC run
-pub const MAX_HEIGHTS_TO_CLEAR: u64 = 100;
-
 /// Block economics config taken from genesis config
 pub struct BlockEconomicsConfig {
     pub gas_price_adjustment_rate: Rational,
@@ -540,7 +537,11 @@ impl Chain {
     //    and the Trie is updated with having only Genesis data.
     // 4. State Sync Clearing happens in `reset_data_pre_state_sync()`.
     //
-    pub fn clear_data(&mut self, tries: ShardTries) -> Result<(), Error> {
+    pub fn clear_data(
+        &mut self,
+        tries: ShardTries,
+        gc_step_size: BlockHeightDelta,
+    ) -> Result<(), Error> {
         let head = self.store.head()?;
         let tail = self.store.tail()?;
         let mut gc_stop_height = self.runtime_adapter.get_gc_stop_height(&head.last_block_hash)?;
@@ -551,7 +552,7 @@ impl Chain {
             .into());
         }
         // To avoid network slowdown, we limit the number of heights to clear per GC execution
-        gc_stop_height = min(gc_stop_height, tail + MAX_HEIGHTS_TO_CLEAR);
+        gc_stop_height = min(gc_stop_height, tail + gc_step_size);
 
         // Forks Cleaning
         for height in tail..gc_stop_height {

chain/chain/src/store.rs

@@ -2479,12 +2479,14 @@ mod tests {
     use near_primitives::validator_signer::{InMemoryValidatorSigner, ValidatorSigner};
     use near_store::test_utils::create_test_store;
 
-    use crate::chain::{check_refcount_map, MAX_HEIGHTS_TO_CLEAR};
+    use crate::chain::check_refcount_map;
     use crate::store::{ChainStoreAccess, GCMode};
     use crate::store_validator::StoreValidator;
     use crate::test_utils::KeyValueRuntime;
     use crate::{Chain, ChainGenesis, DoomslugThresholdMode};
 
+    const MAX_HEIGHTS_TO_CLEAR: u64 = 100;
+
     fn get_chain() -> Chain {
         get_chain_with_epoch_length(10)
     }
@@ -2730,7 +2732,7 @@ mod tests {
         assert!(check_refcount_map(&mut chain).is_ok());
         chain.epoch_length = 1;
         let trie = chain.runtime_adapter.get_tries();
-        assert!(chain.clear_data(trie).is_ok());
+        assert!(chain.clear_data(trie, MAX_HEIGHTS_TO_CLEAR).is_ok());
 
         assert!(chain.get_block(&blocks[0].hash()).is_ok());
 
@@ -2848,7 +2850,7 @@ mod tests {
 
         for iter in 0..10 {
             println!("ITERATION #{:?}", iter);
-            assert!(chain.clear_data(trie.clone()).is_ok());
+            assert!(chain.clear_data(trie.clone(), MAX_HEIGHTS_TO_CLEAR).is_ok());
 
             assert!(chain.get_block(&blocks[0].hash()).is_ok());
 

chain/chain/tests/gc.rs

@@ -152,7 +152,7 @@ mod tests {
 
         assert!(check_refcount_map(&mut chain1).is_ok());
         // GC execution
-        let clear_data = chain1.clear_data(tries1.clone());
+        let clear_data = chain1.clear_data(tries1.clone(), 100);
         if clear_data.is_err() {
             println!("clear data failed = {:?}", clear_data);
             assert!(false);

chain/client/src/client.rs

@@ -784,7 +784,10 @@ impl Client {
         if status.is_new_head() {
             self.shards_mgr.update_largest_seen_height(block.header.inner_lite.height);
             if !self.config.archive {
-                if let Err(err) = self.chain.clear_data(self.runtime_adapter.get_tries()) {
+                if let Err(err) = self
+                    .chain
+                    .clear_data(self.runtime_adapter.get_tries(), self.config.gc_step_size)
+                {
                     error!(target: "client", "Can't clear old data, {:?}", err);
                     debug_assert!(false);
                 };

core/chain-configs/src/client_config.rs

@@ -66,6 +66,8 @@ pub struct ClientConfig {
     pub chunk_request_retry_period: Duration,
     /// Behind this horizon header fetch kicks in.
     pub block_header_fetch_horizon: BlockHeightDelta,
+    /// Number of heights to garbage collect at every gc call.
+    pub gc_step_size: BlockHeightDelta,
     /// Accounts that this client tracks
     pub tracked_accounts: Vec<AccountId>,
     /// Shards that this client tracks
@@ -118,6 +120,7 @@ impl ClientConfig {
                 Duration::from_millis(min_block_prod_time / 5),
             ),
             block_header_fetch_horizon: 50,
+            gc_step_size: 100,
             tracked_accounts: vec![],
             tracked_shards: vec![],
             archive,

neard/src/config.rs

@@ -258,6 +258,10 @@ fn default_sync_step_period() -> Duration {
     Duration::from_millis(10)
 }
 
+fn default_gc_step_size() -> BlockHeightDelta {
+    2
+}
+
 #[derive(Serialize, Deserialize, Clone, Debug)]
 pub struct Consensus {
     /// Minimum number of peers to start syncing.
@@ -345,6 +349,8 @@ pub struct Config {
     pub tracked_accounts: Vec<AccountId>,
     pub tracked_shards: Vec<ShardId>,
     pub archive: bool,
+    #[serde(default = "default_gc_step_size")]
+    pub gc_step_size: BlockHeightDelta,
 }
 
 impl Default for Config {
@@ -361,6 +367,7 @@ impl Default for Config {
             tracked_accounts: vec![],
             tracked_shards: vec![],
             archive: false,
+            gc_step_size: default_gc_step_size(),
         }
     }
 }
@@ -523,6 +530,7 @@ impl NearConfig {
                 tracked_accounts: config.tracked_accounts,
                 tracked_shards: config.tracked_shards,
                 archive: config.archive,
+                gc_step_size: config.gc_step_size,
             },
             network_config: NetworkConfig {
                 public_key: network_key_pair.public_key,