From b75cd3c8a582c34d4e109c7c3e4785f04e5e71a5 Mon Sep 17 00:00:00 2001 From: sydhds Date: Tue, 9 Jan 2024 18:05:46 +0100 Subject: [PATCH 1/8] Optimize deferred credits fetch --- massa-api/src/public.rs | 19 ++- .../src/controller_traits.rs | 6 +- massa-execution-worker/src/context.rs | 3 +- massa-execution-worker/src/controller.rs | 10 +- massa-execution-worker/src/execution.rs | 12 +- .../src/speculative_roll_state.rs | 121 +++++++++++++----- .../src/tests/scenarios_mandatories.rs | 4 +- massa-pos-exports/src/pos_final_state.rs | 20 ++- 8 files changed, 148 insertions(+), 47 deletions(-) diff --git a/massa-api/src/public.rs b/massa-api/src/public.rs index 67bda938266..b780519c0e9 100644 --- a/massa-api/src/public.rs +++ b/massa-api/src/public.rs @@ -832,8 +832,25 @@ impl MassaRpcServer for API { .collect() }; + // Compute a limit (as a slot) for deferred credits as it can be quite huge + let bound_ts = MassaTime::now() + .checked_add(MassaTime::from_millis( + 60 * 24 * 60 * 60 * 1000, // 60 Days, 24 hours, 60 minutes, 60 seconds, 1000 milliseconds + )) + .map_err(|e| ApiError::InternalServerError(e.to_string()))?; + + let deferred_credit_max_slot = timeslots::get_closest_slot_to_timestamp( + self.0.api_settings.thread_count, + self.0.api_settings.t0, + self.0.api_settings.genesis_timestamp, + bound_ts, + ); + // get execution info - let execution_infos = self.0.execution_controller.get_addresses_infos(&addresses); + let execution_infos = self + .0 + .execution_controller + .get_addresses_infos(&addresses, Some(deferred_credit_max_slot)); // get future draws from selector let selection_draws = { diff --git a/massa-execution-exports/src/controller_traits.rs b/massa-execution-exports/src/controller_traits.rs index a2f5a06cb7c..cf50f40a332 100644 --- a/massa-execution-exports/src/controller_traits.rs +++ b/massa-execution-exports/src/controller_traits.rs @@ -102,7 +102,11 @@ pub trait ExecutionController: Send + Sync { ) -> (bool, bool); /// Gets information about a batch of addresses - fn get_addresses_infos(&self, addresses: &[Address]) -> Vec; + fn get_addresses_infos( + &self, + addresses: &[Address], + deferred_credits_max_slot: Option, + ) -> Vec; /// Get execution statistics fn get_stats(&self) -> ExecutionStats; diff --git a/massa-execution-worker/src/context.rs b/massa-execution-worker/src/context.rs index 1c162d1b2ad..55b6f948c12 100644 --- a/massa-execution-worker/src/context.rs +++ b/massa-execution-worker/src/context.rs @@ -1032,13 +1032,14 @@ impl ExecutionContext { &self, address: &Address, thread_count: u8, + max_slot: Option, ) -> BTreeMap { let min_slot = self .slot .get_next_slot(thread_count) .expect("unexpected slot overflow in context.get_addresses_deferred_credits"); self.speculative_roll_state - .get_address_deferred_credits(address, min_slot) + .get_address_deferred_credits(address, min_slot, max_slot) } /// in case of diff --git a/massa-execution-worker/src/controller.rs b/massa-execution-worker/src/controller.rs index b968d58cdbb..aa0d5163104 100644 --- a/massa-execution-worker/src/controller.rs +++ b/massa-execution-worker/src/controller.rs @@ -431,7 +431,11 @@ impl ExecutionController for ExecutionControllerImpl { } /// Gets information about a batch of addresses - fn get_addresses_infos(&self, addresses: &[Address]) -> Vec { + fn get_addresses_infos( + &self, + addresses: &[Address], + deferred_credits_max_slot: Option, + ) -> Vec { let mut res = Vec::with_capacity(addresses.len()); let exec_state = self.execution_state.read(); for addr in addresses { @@ -441,6 +445,8 @@ impl ExecutionController for ExecutionControllerImpl { exec_state.get_final_and_candidate_balance(addr); let (final_roll_count, candidate_roll_count) = exec_state.get_final_and_candidate_rolls(addr); + let future_deferred_credits = + exec_state.get_address_future_deferred_credits(addr, deferred_credits_max_slot); res.push(ExecutionAddressInfo { final_datastore_keys: final_datastore_keys.unwrap_or_default(), candidate_datastore_keys: candidate_datastore_keys.unwrap_or_default(), @@ -448,7 +454,7 @@ impl ExecutionController for ExecutionControllerImpl { candidate_balance: candidate_balance.unwrap_or_default(), final_roll_count, candidate_roll_count, - future_deferred_credits: exec_state.get_address_future_deferred_credits(addr), + future_deferred_credits, cycle_infos: exec_state.get_address_cycle_infos(addr), }); } diff --git a/massa-execution-worker/src/execution.rs b/massa-execution-worker/src/execution.rs index b939c6edf1f..b59bb423411 100644 --- a/massa-execution-worker/src/execution.rs +++ b/massa-execution-worker/src/execution.rs @@ -1817,8 +1817,16 @@ impl ExecutionState { } /// Get future deferred credits of an address - pub fn get_address_future_deferred_credits(&self, address: &Address) -> BTreeMap { - context_guard!(self).get_address_future_deferred_credits(address, self.config.thread_count) + pub fn get_address_future_deferred_credits( + &self, + address: &Address, + max_slot: Option, + ) -> BTreeMap { + context_guard!(self).get_address_future_deferred_credits( + address, + self.config.thread_count, + max_slot, + ) } /// Get future deferred credits of an address diff --git a/massa-execution-worker/src/speculative_roll_state.rs b/massa-execution-worker/src/speculative_roll_state.rs index e0a49a6de9a..69bdb5f9caa 100644 --- a/massa-execution-worker/src/speculative_roll_state.rs +++ b/massa-execution-worker/src/speculative_roll_state.rs @@ -197,7 +197,7 @@ impl SpeculativeRollState { addr: &Address, amount: &Amount, ) -> Amount { - let credits = self.get_address_deferred_credits(addr, *slot); + let credits = self.get_address_deferred_credits(addr, *slot, None); let mut remaining_to_slash = *amount; for (credit_slot, credit_amount) in credits.iter() { @@ -298,51 +298,104 @@ impl SpeculativeRollState { &self, address: &Address, min_slot: Slot, + max_slot_: Option, ) -> BTreeMap { let mut res: HashMap = HashMap::default(); - // get added values - for (slot, addr_amount) in self - .added_changes - .deferred_credits - .credits - .range(min_slot..) - { - if let Some(amount) = addr_amount.get(address) { - res.entry(*slot).or_insert(*amount); - }; - } + if let Some(max_slot) = max_slot_ { + // get added values + for (slot, addr_amount) in self + .added_changes + .deferred_credits + .credits + .range(min_slot..max_slot) + { + if let Some(amount) = addr_amount.get(address) { + res.entry(*slot).or_insert(*amount); + }; + } - // get values from active history, backwards - { - let hist = self.active_history.read(); - for hist_item in hist.0.iter().rev() { - for (slot, addr_amount) in hist_item - .state_changes - .pos_changes - .deferred_credits - .credits - .range(min_slot..) - { + // get values from active history, backwards + { + let hist = self.active_history.read(); + for hist_item in hist.0.iter().rev() { + for (slot, addr_amount) in hist_item + .state_changes + .pos_changes + .deferred_credits + .credits + .range(min_slot..max_slot) + { + if let Some(amount) = addr_amount.get(address) { + res.entry(*slot).or_insert(*amount); + }; + } + } + } + + // get values from final state + { + let final_state = self.final_state.read(); + let deferred_credits = final_state + .get_pos_state() + .get_deferred_credits_range(min_slot..max_slot, Some(*address)); + println!("def creds len: {}", deferred_credits.credits.len()); + let mut inserted = 0; + for (slot, addr_amount) in deferred_credits.credits { if let Some(amount) = addr_amount.get(address) { - res.entry(*slot).or_insert(*amount); + res.entry(slot).or_insert(*amount); + inserted += 1; }; } + println!("inserted: {}", inserted); } - } - - // get values from final state - { - let final_state = self.final_state.read(); - for (slot, addr_amount) in final_state - .get_pos_state() - .get_deferred_credits_range(min_slot..) + } else { + // get added values + for (slot, addr_amount) in self + .added_changes + .deferred_credits .credits + .range(min_slot..) { if let Some(amount) = addr_amount.get(address) { - res.entry(slot).or_insert(*amount); + res.entry(*slot).or_insert(*amount); }; } + + // get values from active history, backwards + { + let hist = self.active_history.read(); + for hist_item in hist.0.iter().rev() { + for (slot, addr_amount) in hist_item + .state_changes + .pos_changes + .deferred_credits + .credits + .range(min_slot..) + { + if let Some(amount) = addr_amount.get(address) { + res.entry(*slot).or_insert(*amount); + }; + } + } + } + + // get values from final state + { + let final_state = self.final_state.read(); + let deferred_credits = final_state + .get_pos_state() + .get_deferred_credits_range(min_slot.., Some(*address)); + println!("def creds len: {}", deferred_credits.credits.len()); + let mut inserted = 0; + for (slot, addr_amount) in deferred_credits.credits { + if let Some(amount) = addr_amount.get(address) { + res.entry(slot).or_insert(*amount); + inserted += 1; + }; + } + println!("inserted: {}", inserted); + } } res.into_iter().filter(|(_s, v)| !v.is_zero()).collect() @@ -565,7 +618,7 @@ impl SpeculativeRollState { .final_state .read() .get_pos_state() - .get_deferred_credits_range(..=slot); + .get_deferred_credits_range(..=slot, None); // fetch active history deferred credits credits.extend( diff --git a/massa-execution-worker/src/tests/scenarios_mandatories.rs b/massa-execution-worker/src/tests/scenarios_mandatories.rs index 3dcad8c2bd3..623f5b5a177 100644 --- a/massa-execution-worker/src/tests/scenarios_mandatories.rs +++ b/massa-execution-worker/src/tests/scenarios_mandatories.rs @@ -2065,7 +2065,9 @@ fn datastore_manipulations() { ], }); // Just checking that is works no asserts for now - universe.module_controller.get_addresses_infos(&[addr]); + universe + .module_controller + .get_addresses_infos(&[addr], None); } /// This test checks causes a history rewrite in slot sequencing and ensures that emitted events match diff --git a/massa-pos-exports/src/pos_final_state.rs b/massa-pos-exports/src/pos_final_state.rs index 571513ba0ab..ed4a54e41ca 100644 --- a/massa-pos-exports/src/pos_final_state.rs +++ b/massa-pos-exports/src/pos_final_state.rs @@ -693,7 +693,11 @@ impl PoSFinalState { } /// Retrieves every deferred credit in a slot range - pub fn get_deferred_credits_range(&self, range: R) -> DeferredCredits + pub fn get_deferred_credits_range( + &self, + range: R, + filter_by: Option
, + ) -> DeferredCredits where R: RangeBounds, { @@ -743,6 +747,12 @@ impl PoSFinalState { .deserialize::(rest_key) .expect(DEFERRED_CREDITS_DESER_ERROR); + if let Some(addr) = filter_by { + if addr != address { + continue; + } + } + let (_, amount) = self .deferred_credits_deserializer .credit_deserializer @@ -1733,10 +1743,10 @@ mod tests { "deferred credits not loaded correctly" ); let credits_range_1 = - pos_state.get_deferred_credits_range(Slot::new(4, 0)..Slot::new(4, 1)); + pos_state.get_deferred_credits_range(Slot::new(4, 0)..Slot::new(4, 1), None); assert!(credits_range_1.is_empty()); let credits_range_2 = - pos_state.get_deferred_credits_range(Slot::new(2, 0)..Slot::new(3, 1)); + pos_state.get_deferred_credits_range(Slot::new(2, 0)..Slot::new(3, 1), None); let expected_credits_range_2 = vec![( Slot::new(3, 0), vec![(addr1, a_a1_s3), (addr2, a_a2_s3)] @@ -1747,10 +1757,10 @@ mod tests { .collect(); assert_eq!(credits_range_2.credits, expected_credits_range_2); let credits_range_3 = - pos_state.get_deferred_credits_range(Slot::new(7, 0)..Slot::new(9, 5)); + pos_state.get_deferred_credits_range(Slot::new(7, 0)..Slot::new(9, 5), None); assert!(credits_range_3.is_empty()); let credits_range_4 = - pos_state.get_deferred_credits_range(Slot::new(7, 0)..Slot::new(255, 1)); + pos_state.get_deferred_credits_range(Slot::new(7, 0)..Slot::new(255, 1), None); let a_a1_s255 = Amount::from_str("5.01").unwrap(); let expected_credits_range_4 = vec![( From fd9993e83b7faed1406d25224ecf9fcaca94b577 Mon Sep 17 00:00:00 2001 From: sydhds Date: Tue, 9 Jan 2024 18:09:42 +0100 Subject: [PATCH 2/8] Fix get_addresses test compilation --- massa-api/src/tests/public.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/massa-api/src/tests/public.rs b/massa-api/src/tests/public.rs index 19a0909c158..ae4bbfb3267 100644 --- a/massa-api/src/tests/public.rs +++ b/massa-api/src/tests/public.rs @@ -762,7 +762,7 @@ async fn get_addresses() { let (mut api_public, config) = start_public_api(addr); let mut exec_ctrl = MockExecutionController::new(); - exec_ctrl.expect_get_addresses_infos().returning(|a| { + exec_ctrl.expect_get_addresses_infos().returning(|a, _s| { a.iter() .map(|_addr| ExecutionAddressInfo { candidate_balance: Amount::from_str("100000").unwrap(), From 1c062174934e1f3fdc580959e25f5b57d00b38ff Mon Sep 17 00:00:00 2001 From: sydhds Date: Tue, 9 Jan 2024 18:11:24 +0100 Subject: [PATCH 3/8] Remove test print statements --- massa-execution-worker/src/speculative_roll_state.rs | 8 -------- 1 file changed, 8 deletions(-) diff --git a/massa-execution-worker/src/speculative_roll_state.rs b/massa-execution-worker/src/speculative_roll_state.rs index 69bdb5f9caa..490ff26d8ab 100644 --- a/massa-execution-worker/src/speculative_roll_state.rs +++ b/massa-execution-worker/src/speculative_roll_state.rs @@ -339,15 +339,11 @@ impl SpeculativeRollState { let deferred_credits = final_state .get_pos_state() .get_deferred_credits_range(min_slot..max_slot, Some(*address)); - println!("def creds len: {}", deferred_credits.credits.len()); - let mut inserted = 0; for (slot, addr_amount) in deferred_credits.credits { if let Some(amount) = addr_amount.get(address) { res.entry(slot).or_insert(*amount); - inserted += 1; }; } - println!("inserted: {}", inserted); } } else { // get added values @@ -386,15 +382,11 @@ impl SpeculativeRollState { let deferred_credits = final_state .get_pos_state() .get_deferred_credits_range(min_slot.., Some(*address)); - println!("def creds len: {}", deferred_credits.credits.len()); - let mut inserted = 0; for (slot, addr_amount) in deferred_credits.credits { if let Some(amount) = addr_amount.get(address) { res.entry(slot).or_insert(*amount); - inserted += 1; }; } - println!("inserted: {}", inserted); } } From d2e4e1ba3ea965aff4243001fa591ba9297750f8 Mon Sep 17 00:00:00 2001 From: sydhds Date: Tue, 9 Jan 2024 18:27:08 +0100 Subject: [PATCH 4/8] Use config for deferred credits delta --- massa-api-exports/src/config.rs | 2 ++ massa-api/src/public.rs | 4 +--- massa-api/src/tests/mock.rs | 4 +++- massa-models/src/config/constants.rs | 9 +++++++++ massa-node/src/main.rs | 3 ++- 5 files changed, 17 insertions(+), 5 deletions(-) diff --git a/massa-api-exports/src/config.rs b/massa-api-exports/src/config.rs index 0f530fbefd8..1073302e9db 100644 --- a/massa-api-exports/src/config.rs +++ b/massa-api-exports/src/config.rs @@ -79,4 +79,6 @@ pub struct APIConfig { pub last_start_period: u64, /// chain id pub chain_id: u64, + /// Delta to compute upper bounds when fetching deferred credits + pub deferred_credits_delta: MassaTime, } diff --git a/massa-api/src/public.rs b/massa-api/src/public.rs index b780519c0e9..ceed9bf3124 100644 --- a/massa-api/src/public.rs +++ b/massa-api/src/public.rs @@ -834,9 +834,7 @@ impl MassaRpcServer for API { // Compute a limit (as a slot) for deferred credits as it can be quite huge let bound_ts = MassaTime::now() - .checked_add(MassaTime::from_millis( - 60 * 24 * 60 * 60 * 1000, // 60 Days, 24 hours, 60 minutes, 60 seconds, 1000 milliseconds - )) + .checked_add(self.0.api_settings.deferred_credits_delta) .map_err(|e| ApiError::InternalServerError(e.to_string()))?; let deferred_credit_max_slot = timeslots::get_closest_slot_to_timestamp( diff --git a/massa-api/src/tests/mock.rs b/massa-api/src/tests/mock.rs index 4bbf8201bda..7cf0794d119 100644 --- a/massa-api/src/tests/mock.rs +++ b/massa-api/src/tests/mock.rs @@ -7,7 +7,7 @@ use std::{collections::HashMap, net::SocketAddr}; use massa_api_exports::config::APIConfig; use massa_consensus_exports::{ConsensusBroadcasts, MockConsensusController}; use massa_execution_exports::{GasCosts, MockExecutionController}; -use massa_models::config::CHAINID; +use massa_models::config::{CHAINID, DEFERRED_CREDITS_DELTA}; use massa_models::{ config::{ BASE_OPERATION_GAS_COST, ENDORSEMENT_COUNT, GENESIS_TIMESTAMP, MAX_DATASTORE_VALUE_LENGTH, @@ -67,6 +67,7 @@ pub(crate) fn get_apiv2_server(addr: &SocketAddr) -> (API, APIConfig) { periods_per_cycle: PERIODS_PER_CYCLE, last_start_period: 0, chain_id: *CHAINID, + deferred_credits_delta: DEFERRED_CREDITS_DELTA, }; // let shared_storage: massa_storage::Storage = massa_storage::Storage::create_root(); @@ -141,6 +142,7 @@ pub(crate) fn start_public_api(addr: SocketAddr) -> (API, APIConfig) { periods_per_cycle: PERIODS_PER_CYCLE, last_start_period: 0, chain_id: *CHAINID, + deferred_credits_delta: DEFERRED_CREDITS_DELTA, }; let shared_storage: massa_storage::Storage = massa_storage::Storage::create_root(); diff --git a/massa-models/src/config/constants.rs b/massa-models/src/config/constants.rs index 3b38b3e2429..fc22f0bc1d8 100644 --- a/massa-models/src/config/constants.rs +++ b/massa-models/src/config/constants.rs @@ -342,6 +342,15 @@ pub const ROLL_COUNT_TO_SLASH_ON_DENUNCIATION: u64 = 1; /// Maximum size of executed denunciations pub const MAX_DENUNCIATION_CHANGES_LENGTH: u64 = 1_000; +// +// Constants for Api +// + +/// Delta to compute upper bounds when fetching deferred credits +pub const DEFERRED_CREDITS_DELTA: MassaTime = MassaTime::from_millis( + 60 * 24 * 60 * 60 * 1000, // 60 Days, 24 hours, 60 minutes, 60 seconds, 1000 milliseconds +); + // Some checks at compile time that should not be ignored! #[allow(clippy::assertions_on_constants)] const _: () = { diff --git a/massa-node/src/main.rs b/massa-node/src/main.rs index 8deb6626cb3..3946efa31ca 100644 --- a/massa-node/src/main.rs +++ b/massa-node/src/main.rs @@ -74,7 +74,7 @@ use massa_models::config::constants::{ VERSION, }; use massa_models::config::{ - BASE_OPERATION_GAS_COST, CHAINID, KEEP_EXECUTED_HISTORY_EXTRA_PERIODS, + BASE_OPERATION_GAS_COST, CHAINID, DEFERRED_CREDITS_DELTA, KEEP_EXECUTED_HISTORY_EXTRA_PERIODS, MAX_BOOTSTRAP_FINAL_STATE_PARTS_SIZE, MAX_BOOTSTRAP_VERSIONING_ELEMENTS_SIZE, MAX_EVENT_DATA_SIZE, MAX_MESSAGE_SIZE, POOL_CONTROLLER_DENUNCIATIONS_CHANNEL_SIZE, POOL_CONTROLLER_ENDORSEMENTS_CHANNEL_SIZE, POOL_CONTROLLER_OPERATIONS_CHANNEL_SIZE, @@ -805,6 +805,7 @@ async fn launch( periods_per_cycle: PERIODS_PER_CYCLE, last_start_period: final_state.read().get_last_start_period(), chain_id: *CHAINID, + deferred_credits_delta: DEFERRED_CREDITS_DELTA, }; // spawn Massa API From 9c07e791be4aa5aaa65618a2c2ff68a0a1550fc2 Mon Sep 17 00:00:00 2001 From: sydhds Date: Tue, 9 Jan 2024 18:42:08 +0100 Subject: [PATCH 5/8] Move to Settings --- massa-api/src/tests/mock.rs | 2 +- massa-models/src/config/constants.rs | 9 --------- massa-node/base_config/config.toml | 2 ++ massa-node/src/main.rs | 4 ++-- massa-node/src/settings.rs | 1 + 5 files changed, 6 insertions(+), 12 deletions(-) diff --git a/massa-api/src/tests/mock.rs b/massa-api/src/tests/mock.rs index 7cf0794d119..199737cdbb3 100644 --- a/massa-api/src/tests/mock.rs +++ b/massa-api/src/tests/mock.rs @@ -142,7 +142,7 @@ pub(crate) fn start_public_api(addr: SocketAddr) -> (API, APIConfig) { periods_per_cycle: PERIODS_PER_CYCLE, last_start_period: 0, chain_id: *CHAINID, - deferred_credits_delta: DEFERRED_CREDITS_DELTA, + deferred_credits_delta: MassaTime::from_millis(5184000000), }; let shared_storage: massa_storage::Storage = massa_storage::Storage::create_root(); diff --git a/massa-models/src/config/constants.rs b/massa-models/src/config/constants.rs index fc22f0bc1d8..3b38b3e2429 100644 --- a/massa-models/src/config/constants.rs +++ b/massa-models/src/config/constants.rs @@ -342,15 +342,6 @@ pub const ROLL_COUNT_TO_SLASH_ON_DENUNCIATION: u64 = 1; /// Maximum size of executed denunciations pub const MAX_DENUNCIATION_CHANGES_LENGTH: u64 = 1_000; -// -// Constants for Api -// - -/// Delta to compute upper bounds when fetching deferred credits -pub const DEFERRED_CREDITS_DELTA: MassaTime = MassaTime::from_millis( - 60 * 24 * 60 * 60 * 1000, // 60 Days, 24 hours, 60 minutes, 60 seconds, 1000 milliseconds -); - // Some checks at compile time that should not be ignored! #[allow(clippy::assertions_on_constants)] const _: () = { diff --git a/massa-node/base_config/config.toml b/massa-node/base_config/config.toml index 786acc7ce12..32c58fd0689 100644 --- a/massa-node/base_config/config.toml +++ b/massa-node/base_config/config.toml @@ -37,6 +37,8 @@ enable_ws = false # whether to broadcast for blocks, endorsements and operations enable_broadcast = false + # deferred credits delta (in milliseconds) + deferred_credits_delta = 5184000000 # ~ 2 months [grpc] [grpc.public] diff --git a/massa-node/src/main.rs b/massa-node/src/main.rs index 3946efa31ca..562dbdbc23f 100644 --- a/massa-node/src/main.rs +++ b/massa-node/src/main.rs @@ -74,7 +74,7 @@ use massa_models::config::constants::{ VERSION, }; use massa_models::config::{ - BASE_OPERATION_GAS_COST, CHAINID, DEFERRED_CREDITS_DELTA, KEEP_EXECUTED_HISTORY_EXTRA_PERIODS, + BASE_OPERATION_GAS_COST, CHAINID, KEEP_EXECUTED_HISTORY_EXTRA_PERIODS, MAX_BOOTSTRAP_FINAL_STATE_PARTS_SIZE, MAX_BOOTSTRAP_VERSIONING_ELEMENTS_SIZE, MAX_EVENT_DATA_SIZE, MAX_MESSAGE_SIZE, POOL_CONTROLLER_DENUNCIATIONS_CHANNEL_SIZE, POOL_CONTROLLER_ENDORSEMENTS_CHANNEL_SIZE, POOL_CONTROLLER_OPERATIONS_CHANNEL_SIZE, @@ -805,7 +805,7 @@ async fn launch( periods_per_cycle: PERIODS_PER_CYCLE, last_start_period: final_state.read().get_last_start_period(), chain_id: *CHAINID, - deferred_credits_delta: DEFERRED_CREDITS_DELTA, + deferred_credits_delta: MassaTime::from_millis(SETTINGS.api.deferred_credits_delta), }; // spawn Massa API diff --git a/massa-node/src/settings.rs b/massa-node/src/settings.rs index 1ba0aa53c5e..9024f330ad0 100644 --- a/massa-node/src/settings.rs +++ b/massa-node/src/settings.rs @@ -121,6 +121,7 @@ pub struct APISettings { pub enable_ws: bool, // whether to broadcast for blocks, endorsement and operations pub enable_broadcast: bool, + pub deferred_credits_delta: u64, } #[derive(Debug, Deserialize, Clone)] From 50f66187ad4d738fcc2abaf8f94268ee6546b021 Mon Sep 17 00:00:00 2001 From: Damir Vodenicarevic Date: Tue, 9 Jan 2024 19:32:15 +0100 Subject: [PATCH 6/8] minor fixes --- massa-api/src/public.rs | 8 +- massa-api/src/tests/mock.rs | 4 +- .../src/controller_traits.rs | 2 +- massa-execution-worker/src/context.rs | 5 +- massa-execution-worker/src/controller.rs | 2 +- massa-execution-worker/src/execution.rs | 17 ++- .../src/speculative_roll_state.rs | 121 ++++++------------ .../src/tests/scenarios_mandatories.rs | 2 +- massa-pos-exports/src/pos_final_state.rs | 52 +------- 9 files changed, 68 insertions(+), 145 deletions(-) diff --git a/massa-api/src/public.rs b/massa-api/src/public.rs index ceed9bf3124..88b5e9926dd 100644 --- a/massa-api/src/public.rs +++ b/massa-api/src/public.rs @@ -845,10 +845,10 @@ impl MassaRpcServer for API { ); // get execution info - let execution_infos = self - .0 - .execution_controller - .get_addresses_infos(&addresses, Some(deferred_credit_max_slot)); + let execution_infos = self.0.execution_controller.get_addresses_infos( + &addresses, + std::ops::Bound::Excluded(deferred_credit_max_slot), + ); // get future draws from selector let selection_draws = { diff --git a/massa-api/src/tests/mock.rs b/massa-api/src/tests/mock.rs index 199737cdbb3..fe0bb1dbc80 100644 --- a/massa-api/src/tests/mock.rs +++ b/massa-api/src/tests/mock.rs @@ -7,7 +7,7 @@ use std::{collections::HashMap, net::SocketAddr}; use massa_api_exports::config::APIConfig; use massa_consensus_exports::{ConsensusBroadcasts, MockConsensusController}; use massa_execution_exports::{GasCosts, MockExecutionController}; -use massa_models::config::{CHAINID, DEFERRED_CREDITS_DELTA}; +use massa_models::config::CHAINID; use massa_models::{ config::{ BASE_OPERATION_GAS_COST, ENDORSEMENT_COUNT, GENESIS_TIMESTAMP, MAX_DATASTORE_VALUE_LENGTH, @@ -67,7 +67,7 @@ pub(crate) fn get_apiv2_server(addr: &SocketAddr) -> (API, APIConfig) { periods_per_cycle: PERIODS_PER_CYCLE, last_start_period: 0, chain_id: *CHAINID, - deferred_credits_delta: DEFERRED_CREDITS_DELTA, + deferred_credits_delta: MassaTime::from_millis(24 * 3600 * 2), }; // let shared_storage: massa_storage::Storage = massa_storage::Storage::create_root(); diff --git a/massa-execution-exports/src/controller_traits.rs b/massa-execution-exports/src/controller_traits.rs index cf50f40a332..d64f8274f8b 100644 --- a/massa-execution-exports/src/controller_traits.rs +++ b/massa-execution-exports/src/controller_traits.rs @@ -105,7 +105,7 @@ pub trait ExecutionController: Send + Sync { fn get_addresses_infos( &self, addresses: &[Address], - deferred_credits_max_slot: Option, + deferred_credits_max_slot: std::ops::Bound, ) -> Vec; /// Get execution statistics diff --git a/massa-execution-worker/src/context.rs b/massa-execution-worker/src/context.rs index 55b6f948c12..a3e84f8275d 100644 --- a/massa-execution-worker/src/context.rs +++ b/massa-execution-worker/src/context.rs @@ -1028,18 +1028,19 @@ impl ExecutionContext { } /// Get future deferred credits of an address + /// With optionally a limit slot (excluded) pub fn get_address_future_deferred_credits( &self, address: &Address, thread_count: u8, - max_slot: Option, + max_slot: std::ops::Bound, ) -> BTreeMap { let min_slot = self .slot .get_next_slot(thread_count) .expect("unexpected slot overflow in context.get_addresses_deferred_credits"); self.speculative_roll_state - .get_address_deferred_credits(address, min_slot, max_slot) + .get_address_deferred_credits(address, (std::ops::Bound::Included(min_slot), max_slot)) } /// in case of diff --git a/massa-execution-worker/src/controller.rs b/massa-execution-worker/src/controller.rs index aa0d5163104..f1381383366 100644 --- a/massa-execution-worker/src/controller.rs +++ b/massa-execution-worker/src/controller.rs @@ -434,7 +434,7 @@ impl ExecutionController for ExecutionControllerImpl { fn get_addresses_infos( &self, addresses: &[Address], - deferred_credits_max_slot: Option, + deferred_credits_max_slot: std::ops::Bound, ) -> Vec { let mut res = Vec::with_capacity(addresses.len()); let exec_state = self.execution_state.read(); diff --git a/massa-execution-worker/src/execution.rs b/massa-execution-worker/src/execution.rs index b59bb423411..830dd36fce5 100644 --- a/massa-execution-worker/src/execution.rs +++ b/massa-execution-worker/src/execution.rs @@ -1820,7 +1820,7 @@ impl ExecutionState { pub fn get_address_future_deferred_credits( &self, address: &Address, - max_slot: Option, + max_slot: std::ops::Bound, ) -> BTreeMap { context_guard!(self).get_address_future_deferred_credits( address, @@ -1840,7 +1840,14 @@ impl ExecutionState { .final_state .read() .get_pos_state() - .get_address_deferred_credits(address); + .get_deferred_credits_range(.., Some(address)) + .credits; + let res_final: BTreeMap = res_final + .into_iter() + .filter_map(|(slot, addr_amount)| { + addr_amount.get(address).map(|amount| (slot, *amount)) + }) + .collect(); // get values from active history, backwards let mut res_speculative: BTreeMap = BTreeMap::default(); @@ -1852,10 +1859,12 @@ impl ExecutionState { }; } } + // fill missing speculative entries with final entries - for (s, v) in res_final.iter() { - res_speculative.entry(*s).or_insert(*v); + for (slot, amount) in &res_final { + res_speculative.entry(*slot).or_insert(*amount); } + // remove zero entries from speculative res_speculative.retain(|_s, a| !a.is_zero()); diff --git a/massa-execution-worker/src/speculative_roll_state.rs b/massa-execution-worker/src/speculative_roll_state.rs index 490ff26d8ab..bd91be98a6c 100644 --- a/massa-execution-worker/src/speculative_roll_state.rs +++ b/massa-execution-worker/src/speculative_roll_state.rs @@ -197,7 +197,7 @@ impl SpeculativeRollState { addr: &Address, amount: &Amount, ) -> Amount { - let credits = self.get_address_deferred_credits(addr, *slot, None); + let credits = self.get_address_deferred_credits(addr, slot..); let mut remaining_to_slash = *amount; for (credit_slot, credit_amount) in credits.iter() { @@ -294,100 +294,59 @@ impl SpeculativeRollState { } /// Get deferred credits of an address starting from a given slot - pub fn get_address_deferred_credits( + /// Get deferred credits of an address starting from a given slot + pub fn get_address_deferred_credits( &self, address: &Address, - min_slot: Slot, - max_slot_: Option, - ) -> BTreeMap { + slot_range: R, + ) -> BTreeMap + where + R: std::ops::RangeBounds + Clone, + { let mut res: HashMap = HashMap::default(); - if let Some(max_slot) = max_slot_ { - // get added values - for (slot, addr_amount) in self - .added_changes - .deferred_credits - .credits - .range(min_slot..max_slot) - { - if let Some(amount) = addr_amount.get(address) { - res.entry(*slot).or_insert(*amount); - }; - } - - // get values from active history, backwards - { - let hist = self.active_history.read(); - for hist_item in hist.0.iter().rev() { - for (slot, addr_amount) in hist_item - .state_changes - .pos_changes - .deferred_credits - .credits - .range(min_slot..max_slot) - { - if let Some(amount) = addr_amount.get(address) { - res.entry(*slot).or_insert(*amount); - }; - } - } - } + // get added values + for (slot, addr_amount) in self + .added_changes + .deferred_credits + .credits + .range(slot_range.clone()) + { + if let Some(amount) = addr_amount.get(address) { + res.entry(*slot).or_insert(*amount); + }; + } - // get values from final state - { - let final_state = self.final_state.read(); - let deferred_credits = final_state - .get_pos_state() - .get_deferred_credits_range(min_slot..max_slot, Some(*address)); - for (slot, addr_amount) in deferred_credits.credits { + // get values from active history, backwards + { + let hist = self.active_history.read(); + for hist_item in hist.0.iter().rev() { + for (slot, addr_amount) in hist_item + .state_changes + .pos_changes + .deferred_credits + .credits + .range(slot_range.clone()) + { if let Some(amount) = addr_amount.get(address) { - res.entry(slot).or_insert(*amount); + res.entry(*slot).or_insert(*amount); }; } } - } else { - // get added values - for (slot, addr_amount) in self - .added_changes - .deferred_credits + } + + // get values from final state + { + let final_state = self.final_state.read(); + for (slot, addr_amount) in final_state + .get_pos_state() + .get_deferred_credits_range(slot_range, Some(address)) .credits - .range(min_slot..) { if let Some(amount) = addr_amount.get(address) { - res.entry(*slot).or_insert(*amount); + res.entry(slot).or_insert(*amount); }; } - - // get values from active history, backwards - { - let hist = self.active_history.read(); - for hist_item in hist.0.iter().rev() { - for (slot, addr_amount) in hist_item - .state_changes - .pos_changes - .deferred_credits - .credits - .range(min_slot..) - { - if let Some(amount) = addr_amount.get(address) { - res.entry(*slot).or_insert(*amount); - }; - } - } - } - - // get values from final state - { - let final_state = self.final_state.read(); - let deferred_credits = final_state - .get_pos_state() - .get_deferred_credits_range(min_slot.., Some(*address)); - for (slot, addr_amount) in deferred_credits.credits { - if let Some(amount) = addr_amount.get(address) { - res.entry(slot).or_insert(*amount); - }; - } - } } res.into_iter().filter(|(_s, v)| !v.is_zero()).collect() diff --git a/massa-execution-worker/src/tests/scenarios_mandatories.rs b/massa-execution-worker/src/tests/scenarios_mandatories.rs index 623f5b5a177..0bd0c4fac2f 100644 --- a/massa-execution-worker/src/tests/scenarios_mandatories.rs +++ b/massa-execution-worker/src/tests/scenarios_mandatories.rs @@ -2067,7 +2067,7 @@ fn datastore_manipulations() { // Just checking that is works no asserts for now universe .module_controller - .get_addresses_infos(&[addr], None); + .get_addresses_infos(&[addr], std::ops::Bound::Unbounded); } /// This test checks causes a history rewrite in slot sequencing and ensures that emitted events match diff --git a/massa-pos-exports/src/pos_final_state.rs b/massa-pos-exports/src/pos_final_state.rs index ed4a54e41ca..11f0f322f36 100644 --- a/massa-pos-exports/src/pos_final_state.rs +++ b/massa-pos-exports/src/pos_final_state.rs @@ -696,7 +696,7 @@ impl PoSFinalState { pub fn get_deferred_credits_range( &self, range: R, - filter_by: Option
, + addr_filter: Option<&Address>, ) -> DeferredCredits where R: RangeBounds, @@ -747,8 +747,8 @@ impl PoSFinalState { .deserialize::(rest_key) .expect(DEFERRED_CREDITS_DESER_ERROR); - if let Some(addr) = filter_by { - if addr != address { + if let Some(addr_filter_value) = &addr_filter { + if &&address != addr_filter_value { continue; } } @@ -766,52 +766,6 @@ impl PoSFinalState { deferred_credits } - /// Gets the deferred credits for an address - pub fn get_address_deferred_credits(&self, address: &Address) -> BTreeMap { - let db = self.db.read(); - - let mut deferred_credits = BTreeMap::new(); - - let start_key_buffer = DEFERRED_CREDITS_PREFIX.as_bytes().to_vec(); - - for (serialized_key, serialized_value) in db.iterator_cf( - STATE_CF, - MassaIteratorMode::From(&start_key_buffer, MassaDirection::Forward), - ) { - if !serialized_key.starts_with(DEFERRED_CREDITS_PREFIX.as_bytes()) { - break; - } - - // deserialize the slot - let rest_key = &serialized_key[DEFERRED_CREDITS_PREFIX.len()..]; - let (rest_key, slot) = buf_to_array_ctr(rest_key, Slot::from_bytes_key) - .expect(DEFERRED_CREDITS_DESER_ERROR); - - let (_, addr): (_, Address) = self - .deferred_credits_deserializer - .credit_deserializer - .address_deserializer - .deserialize::(rest_key) - .expect(DEFERRED_CREDITS_DESER_ERROR); - - if &addr != address { - // TODO improve performance - continue; - } - - let (_, amount) = self - .deferred_credits_deserializer - .credit_deserializer - .amount_deserializer - .deserialize::(&serialized_value) - .expect(DEFERRED_CREDITS_DESER_ERROR); - - deferred_credits.insert(slot, amount); - } - - deferred_credits - } - /// Gets the index of a cycle in history pub fn get_cycle_index(&self, cycle: u64) -> Option { let first_cycle = match self.cycle_history_cache.front() { From 7dbf75929c0a0da68698ad4608f3aa857cbff4fb Mon Sep 17 00:00:00 2001 From: Damir Vodenicarevic Date: Tue, 9 Jan 2024 19:42:02 +0100 Subject: [PATCH 7/8] minor fixes 2 --- massa-api/src/public.rs | 6 ++---- massa-api/src/tests/mock.rs | 2 +- massa-node/src/main.rs | 2 +- massa-node/src/settings.rs | 2 +- 4 files changed, 5 insertions(+), 7 deletions(-) diff --git a/massa-api/src/public.rs b/massa-api/src/public.rs index 88b5e9926dd..bc2166c0bdd 100644 --- a/massa-api/src/public.rs +++ b/massa-api/src/public.rs @@ -833,9 +833,7 @@ impl MassaRpcServer for API { }; // Compute a limit (as a slot) for deferred credits as it can be quite huge - let bound_ts = MassaTime::now() - .checked_add(self.0.api_settings.deferred_credits_delta) - .map_err(|e| ApiError::InternalServerError(e.to_string()))?; + let bound_ts = MassaTime::now().saturating_add(self.0.api_settings.deferred_credits_delta); let deferred_credit_max_slot = timeslots::get_closest_slot_to_timestamp( self.0.api_settings.thread_count, @@ -847,7 +845,7 @@ impl MassaRpcServer for API { // get execution info let execution_infos = self.0.execution_controller.get_addresses_infos( &addresses, - std::ops::Bound::Excluded(deferred_credit_max_slot), + std::ops::Bound::Included(deferred_credit_max_slot), ); // get future draws from selector diff --git a/massa-api/src/tests/mock.rs b/massa-api/src/tests/mock.rs index fe0bb1dbc80..efcb538ff55 100644 --- a/massa-api/src/tests/mock.rs +++ b/massa-api/src/tests/mock.rs @@ -142,7 +142,7 @@ pub(crate) fn start_public_api(addr: SocketAddr) -> (API, APIConfig) { periods_per_cycle: PERIODS_PER_CYCLE, last_start_period: 0, chain_id: *CHAINID, - deferred_credits_delta: MassaTime::from_millis(5184000000), + deferred_credits_delta: MassaTime::from_millis(24 * 3600 * 2), }; let shared_storage: massa_storage::Storage = massa_storage::Storage::create_root(); diff --git a/massa-node/src/main.rs b/massa-node/src/main.rs index 562dbdbc23f..b0b1f06da8d 100644 --- a/massa-node/src/main.rs +++ b/massa-node/src/main.rs @@ -805,7 +805,7 @@ async fn launch( periods_per_cycle: PERIODS_PER_CYCLE, last_start_period: final_state.read().get_last_start_period(), chain_id: *CHAINID, - deferred_credits_delta: MassaTime::from_millis(SETTINGS.api.deferred_credits_delta), + deferred_credits_delta: SETTINGS.api.deferred_credits_delta, }; // spawn Massa API diff --git a/massa-node/src/settings.rs b/massa-node/src/settings.rs index 9024f330ad0..e35cee44aa5 100644 --- a/massa-node/src/settings.rs +++ b/massa-node/src/settings.rs @@ -121,7 +121,7 @@ pub struct APISettings { pub enable_ws: bool, // whether to broadcast for blocks, endorsement and operations pub enable_broadcast: bool, - pub deferred_credits_delta: u64, + pub deferred_credits_delta: MassaTime, } #[derive(Debug, Deserialize, Clone)] From b3d0a0f34c3377ad9d735715f09aaf80e40b5895 Mon Sep 17 00:00:00 2001 From: Damir Vodenicarevic Date: Tue, 9 Jan 2024 19:42:52 +0100 Subject: [PATCH 8/8] double comment --- massa-execution-worker/src/speculative_roll_state.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/massa-execution-worker/src/speculative_roll_state.rs b/massa-execution-worker/src/speculative_roll_state.rs index bd91be98a6c..db825c0dd19 100644 --- a/massa-execution-worker/src/speculative_roll_state.rs +++ b/massa-execution-worker/src/speculative_roll_state.rs @@ -293,7 +293,6 @@ impl SpeculativeRollState { } } - /// Get deferred credits of an address starting from a given slot /// Get deferred credits of an address starting from a given slot pub fn get_address_deferred_credits( &self,