diff --git a/modules/auction-manager/src/lib.rs b/modules/auction-manager/src/lib.rs index 8ce562efa5..eb51d70bc6 100644 --- a/modules/auction-manager/src/lib.rs +++ b/modules/auction-manager/src/lib.rs @@ -352,7 +352,7 @@ impl Pallet { let max_iterations = StorageValueRef::persistent(OFFCHAIN_WORKER_MAX_ITERATIONS) .get::() .unwrap_or(Some(DEFAULT_MAX_ITERATIONS)) - .ok_or(OffchainErr::OffchainStore)?; + .unwrap_or(DEFAULT_MAX_ITERATIONS); log::debug!( target: "auction-manager", @@ -361,16 +361,16 @@ impl Pallet { ); // start iterations to cancel collateral auctions - let mut iterator = >::iter_from(start_key.ok_or(OffchainErr::OffchainStore)?); + let mut iterator = match start_key { + Some(key) => >::iter_from(key), + None => >::iter(), + }; + let mut iteration_count = 0; let mut finished = true; #[allow(clippy::while_let_on_iterator)] while let Some((collateral_auction_id, _)) = iterator.next() { - if iteration_count >= max_iterations { - finished = false; - break; - } iteration_count += 1; if let (Some(collateral_auction), Some((_, last_bid_price))) = ( @@ -380,10 +380,19 @@ impl Pallet { // if collateral auction has already been in reverse stage, // should skip it. if collateral_auction.in_reverse_stage(last_bid_price) { + if iteration_count == max_iterations { + finished = false; + break; + } continue; } } Self::submit_cancel_auction_tx(collateral_auction_id); + + if iteration_count == max_iterations { + finished = false; + break; + } guard.extend_lock().map_err(|_| OffchainErr::OffchainLock)?; } diff --git a/modules/auction-manager/src/mock.rs b/modules/auction-manager/src/mock.rs index 4a3b3adbae..ed978a8483 100644 --- a/modules/auction-manager/src/mock.rs +++ b/modules/auction-manager/src/mock.rs @@ -287,4 +287,13 @@ impl ExtBuilder { t.into() } + + pub fn lots_of_accounts() -> Self { + let mut balances = Vec::new(); + for i in 0..1001 { + let account_id: AccountId = i; + balances.push((account_id, BTC, 1000)); + } + Self { balances } + } } diff --git a/modules/auction-manager/src/tests.rs b/modules/auction-manager/src/tests.rs index c0a397ab4f..45b36eb9c8 100644 --- a/modules/auction-manager/src/tests.rs +++ b/modules/auction-manager/src/tests.rs @@ -22,9 +22,20 @@ use super::*; use frame_support::{assert_noop, assert_ok}; -use mock::{Event, *}; +use mock::{Call as MockCall, Event, *}; +use sp_core::offchain::{testing, DbExternalities, OffchainDbExt, OffchainWorkerExt, StorageKind, TransactionPoolExt}; +use sp_io::offchain; use sp_runtime::traits::One; +fn run_to_block_offchain(n: u64) { + while System::block_number() < n { + System::set_block_number(System::block_number() + 1); + AuctionManagerModule::offchain_worker(System::block_number()); + // this unlocks the concurrency storage lock so offchain_worker will fire next block + offchain::sleep_until(offchain::timestamp().add(Duration::from_millis(LOCK_DURATION + 200))); + } +} + #[test] fn get_auction_time_to_close_work() { ExtBuilder::default().build().execute_with(|| { @@ -471,3 +482,115 @@ fn cancel_collateral_auction_work() { assert_eq!(bob_ref_count_1, bob_ref_count_0 - 1); }); } + +#[test] +fn offchain_worker_cancels_auction_in_shutdown() { + let (offchain, _offchain_state) = testing::TestOffchainExt::new(); + let (pool, pool_state) = testing::TestTransactionPoolExt::new(); + let mut ext = ExtBuilder::default().build(); + ext.register_extension(OffchainWorkerExt::new(offchain.clone())); + ext.register_extension(TransactionPoolExt::new(pool)); + ext.register_extension(OffchainDbExt::new(offchain.clone())); + + ext.execute_with(|| { + System::set_block_number(1); + assert_ok!(AuctionManagerModule::new_collateral_auction(&ALICE, BTC, 10, 100)); + assert!(AuctionManagerModule::collateral_auctions(0).is_some()); + run_to_block_offchain(2); + // offchain worker does not have any tx because shutdown is false + assert!(!MockEmergencyShutdown::is_shutdown()); + assert!(pool_state.write().transactions.pop().is_none()); + mock_shutdown(); + assert!(MockEmergencyShutdown::is_shutdown()); + + // now offchain worker will cancel auction as shutdown is true + run_to_block_offchain(3); + let tx = pool_state.write().transactions.pop().unwrap(); + let tx = Extrinsic::decode(&mut &*tx).unwrap(); + if let MockCall::AuctionManagerModule(crate::Call::cancel(auction_id)) = tx.call { + assert_ok!(AuctionManagerModule::cancel(Origin::none(), auction_id)); + } + + // auction is canceled + assert!(AuctionManagerModule::collateral_auctions(0).is_none()); + assert!(pool_state.write().transactions.pop().is_none()); + }); +} + +#[test] +fn offchain_worker_max_iterations_check() { + let (mut offchain, _offchain_state) = testing::TestOffchainExt::new(); + let (pool, pool_state) = testing::TestTransactionPoolExt::new(); + let mut ext = ExtBuilder::default().build(); + ext.register_extension(OffchainWorkerExt::new(offchain.clone())); + ext.register_extension(TransactionPoolExt::new(pool)); + ext.register_extension(OffchainDbExt::new(offchain.clone())); + + ext.execute_with(|| { + System::set_block_number(1); + // sets max iterations value to 1 + offchain.local_storage_set(StorageKind::PERSISTENT, OFFCHAIN_WORKER_MAX_ITERATIONS, &1u32.encode()); + assert_ok!(AuctionManagerModule::new_collateral_auction(&ALICE, BTC, 10, 100)); + assert_ok!(AuctionManagerModule::new_collateral_auction(&BOB, BTC, 10, 100)); + assert!(AuctionManagerModule::collateral_auctions(1).is_some()); + assert!(AuctionManagerModule::collateral_auctions(0).is_some()); + mock_shutdown(); + assert!(MockEmergencyShutdown::is_shutdown()); + + run_to_block_offchain(2); + // now offchain worker will cancel one auction but the other one will cancel next block + let tx = pool_state.write().transactions.pop().unwrap(); + let tx = Extrinsic::decode(&mut &*tx).unwrap(); + if let MockCall::AuctionManagerModule(crate::Call::cancel(auction_id)) = tx.call { + assert_ok!(AuctionManagerModule::cancel(Origin::none(), auction_id)); + } + assert!( + AuctionManagerModule::collateral_auctions(1).is_some() + || AuctionManagerModule::collateral_auctions(0).is_some() + ); + // only one auction canceled so offchain tx pool is empty + assert!(pool_state.write().transactions.pop().is_none()); + + run_to_block_offchain(3); + // now offchain worker will cancel the next auction + let tx = pool_state.write().transactions.pop().unwrap(); + let tx = Extrinsic::decode(&mut &*tx).unwrap(); + if let MockCall::AuctionManagerModule(crate::Call::cancel(auction_id)) = tx.call { + assert_ok!(AuctionManagerModule::cancel(Origin::none(), auction_id)); + } + assert!(AuctionManagerModule::collateral_auctions(1).is_none()); + assert!(AuctionManagerModule::collateral_auctions(0).is_none()); + assert!(pool_state.write().transactions.pop().is_none()); + }); +} + +#[test] +fn offchain_default_max_iterator_works() { + let (mut offchain, _offchain_state) = testing::TestOffchainExt::new(); + let (pool, pool_state) = testing::TestTransactionPoolExt::new(); + let mut ext = ExtBuilder::lots_of_accounts().build(); + ext.register_extension(OffchainWorkerExt::new(offchain.clone())); + ext.register_extension(TransactionPoolExt::new(pool)); + ext.register_extension(OffchainDbExt::new(offchain.clone())); + + ext.execute_with(|| { + System::set_block_number(1); + // checks that max iterations is stored as none + assert!(offchain + .local_storage_get(StorageKind::PERSISTENT, OFFCHAIN_WORKER_MAX_ITERATIONS) + .is_none()); + + for i in 0..1001 { + let account_id: AccountId = i; + assert_ok!(AuctionManagerModule::new_collateral_auction(&account_id, BTC, 1, 10)); + } + + mock_shutdown(); + run_to_block_offchain(2); + // should only run 1000 iterations stopping due to DEFAULT_MAX_ITERATION + assert_eq!(pool_state.write().transactions.len(), 1000); + run_to_block_offchain(3); + // next block iterator starts where it left off and adds the final account to tx pool + assert_eq!(pool_state.write().transactions.len(), 1001); + }); +} diff --git a/modules/cdp-engine/src/lib.rs b/modules/cdp-engine/src/lib.rs index b51ea25fac..0ab51122d9 100644 --- a/modules/cdp-engine/src/lib.rs +++ b/modules/cdp-engine/src/lib.rs @@ -649,12 +649,17 @@ impl Pallet { let max_iterations = StorageValueRef::persistent(OFFCHAIN_WORKER_MAX_ITERATIONS) .get::() .unwrap_or(Some(DEFAULT_MAX_ITERATIONS)) - .ok_or(OffchainErr::OffchainStore)?; + .unwrap_or(DEFAULT_MAX_ITERATIONS); let currency_id = collateral_currency_ids[collateral_position as usize]; let is_shutdown = T::EmergencyShutdown::is_shutdown(); - let mut map_iterator = - >::iter_prefix_from(currency_id, start_key.clone().ok_or(OffchainErr::OffchainStore)?); + + // If start key is Some(value) continue iterating from that point in storage otherwise start + // iterating from the beginning of > + let mut map_iterator = match start_key.clone() { + Some(key) => >::iter_prefix_from(currency_id, key), + None => >::iter_prefix(currency_id), + }; let mut finished = true; let mut iteration_count = 0; @@ -662,11 +667,6 @@ impl Pallet { #[allow(clippy::while_let_on_iterator)] while let Some((who, Position { collateral, debit })) = map_iterator.next() { - if iteration_count >= max_iterations { - finished = false; - break; - } - if !is_shutdown && matches!( Self::check_cdp_status(currency_id, collateral, debit), @@ -680,7 +680,10 @@ impl Pallet { } iteration_count += 1; - + if iteration_count == max_iterations { + finished = false; + break; + } // extend offchain worker lock guard.extend_lock().map_err(|_| OffchainErr::OffchainLock)?; } diff --git a/modules/cdp-engine/src/mock.rs b/modules/cdp-engine/src/mock.rs index beda3b3271..24a29e51a5 100644 --- a/modules/cdp-engine/src/mock.rs +++ b/modules/cdp-engine/src/mock.rs @@ -380,4 +380,13 @@ impl ExtBuilder { t.into() } + + pub fn lots_of_accounts() -> Self { + let mut balances = Vec::new(); + for i in 0..1001 { + let account_id: AccountId = i; + balances.push((account_id, BTC, 1000)); + } + Self { balances } + } } diff --git a/modules/cdp-engine/src/tests.rs b/modules/cdp-engine/src/tests.rs index c31864c9ac..cf11dd6dba 100644 --- a/modules/cdp-engine/src/tests.rs +++ b/modules/cdp-engine/src/tests.rs @@ -22,11 +22,30 @@ use super::*; use frame_support::{assert_noop, assert_ok}; -use mock::{Event, *}; +use mock::{Call as MockCall, Event, *}; use orml_traits::MultiCurrency; -use sp_runtime::traits::BadOrigin; +use sp_core::offchain::{testing, OffchainDbExt, OffchainWorkerExt, TransactionPoolExt}; +use sp_io::offchain; +use sp_runtime::{ + offchain::{DbExternalities, StorageKind}, + traits::BadOrigin, +}; use support::DEXManager; +pub const INIT_TIMESTAMP: u64 = 30_000; +pub const BLOCK_TIME: u64 = 1000; + +fn run_to_block_offchain(n: u64) { + while System::block_number() < n { + System::set_block_number(System::block_number() + 1); + Timestamp::set_timestamp((System::block_number() as u64 * BLOCK_TIME) + INIT_TIMESTAMP); + CDPEngineModule::on_initialize(System::block_number()); + CDPEngineModule::offchain_worker(System::block_number()); + // this unlocks the concurrency storage lock so offchain_worker will fire next block + offchain::sleep_until(offchain::timestamp().add(Duration::from_millis(LOCK_DURATION + 200))); + } +} + #[test] fn check_cdp_status_work() { ExtBuilder::default().build().execute_with(|| { @@ -922,3 +941,192 @@ fn close_cdp_has_debit_by_swap_on_alternative_path() { assert_eq!(CDPTreasuryModule::get_debit_pool(), 50); }); } + +#[test] +fn offchain_worker_works_cdp() { + let (offchain, _offchain_state) = testing::TestOffchainExt::new(); + let (pool, pool_state) = testing::TestTransactionPoolExt::new(); + let mut ext = ExtBuilder::default().build(); + ext.register_extension(OffchainWorkerExt::new(offchain.clone())); + ext.register_extension(TransactionPoolExt::new(pool)); + ext.register_extension(OffchainDbExt::new(offchain.clone())); + + ext.execute_with(|| { + // number of currencies allowed as collateral (cycles through all of them) + let collateral_currencies_num = CollateralCurrencyIds::get().len() as u64; + System::set_block_number(1); + assert_ok!(CDPEngineModule::set_collateral_params( + Origin::signed(1), + BTC, + Change::NewValue(Some(Rate::saturating_from_rational(1, 100000))), + Change::NewValue(Some(Ratio::saturating_from_rational(3, 2))), + Change::NewValue(Some(Rate::saturating_from_rational(2, 10))), + Change::NewValue(Some(Ratio::saturating_from_rational(9, 5))), + Change::NewValue(10000), + )); + + // offchain worker will not liquidate alice + assert_ok!(CDPEngineModule::adjust_position(&ALICE, BTC, 100, 500)); + assert_ok!(CDPEngineModule::adjust_position(&BOB, BTC, 100, 100)); + assert_eq!(Currencies::free_balance(BTC, &ALICE), 900); + assert_eq!(Currencies::free_balance(AUSD, &ALICE), 50); + assert_eq!(LoansModule::positions(BTC, ALICE).debit, 500); + assert_eq!(LoansModule::positions(BTC, ALICE).collateral, 100); + // jump 2 blocks at a time because code rotates through the different T::CollateralCurrencyIds + run_to_block_offchain(System::block_number() + collateral_currencies_num); + + // checks that offchain worker tx pool is empty (therefore tx to liquidate alice is not present) + assert!(pool_state.write().transactions.pop().is_none()); + assert_eq!(LoansModule::positions(BTC, ALICE).debit, 500); + assert_eq!(LoansModule::positions(BTC, ALICE).collateral, 100); + + // changes alice into unsafe position + assert_ok!(CDPEngineModule::set_collateral_params( + Origin::signed(1), + BTC, + Change::NoChange, + Change::NewValue(Some(Ratio::saturating_from_rational(3, 1))), + Change::NoChange, + Change::NoChange, + Change::NoChange, + )); + run_to_block_offchain(System::block_number() + collateral_currencies_num); + + // offchain worker will liquidate alice + let tx = pool_state.write().transactions.pop().unwrap(); + let tx = Extrinsic::decode(&mut &*tx).unwrap(); + if let MockCall::CDPEngineModule(crate::Call::liquidate(currency_call, who_call)) = tx.call { + assert_ok!(CDPEngineModule::liquidate(Origin::none(), currency_call, who_call)); + } + // empty offchain tx pool (Bob was not liquidated) + assert!(pool_state.write().transactions.pop().is_none()); + // alice is liquidated but bob is not + assert_eq!(LoansModule::positions(BTC, ALICE).debit, 0); + assert_eq!(LoansModule::positions(BTC, ALICE).collateral, 0); + assert_eq!(LoansModule::positions(BTC, BOB).debit, 100); + assert_eq!(LoansModule::positions(BTC, BOB).collateral, 100); + + // emergency shutdown will settle Bobs debit position + mock_shutdown(); + assert!(MockEmergencyShutdown::is_shutdown()); + run_to_block_offchain(System::block_number() + collateral_currencies_num); + // offchain worker will settle bob's position + let tx = pool_state.write().transactions.pop().unwrap(); + let tx = Extrinsic::decode(&mut &*tx).unwrap(); + if let MockCall::CDPEngineModule(crate::Call::settle(currency_call, who_call)) = tx.call { + assert_ok!(CDPEngineModule::settle(Origin::none(), currency_call, who_call)); + } + // emergency shutdown settles bob's debit position + assert_eq!(LoansModule::positions(BTC, BOB).debit, 0); + assert_eq!(LoansModule::positions(BTC, BOB).collateral, 90); + }); +} + +#[test] +fn offchain_worker_iteration_limit_works() { + let (mut offchain, _offchain_state) = testing::TestOffchainExt::new(); + let (pool, pool_state) = testing::TestTransactionPoolExt::new(); + let mut ext = ExtBuilder::default().build(); + ext.register_extension(OffchainWorkerExt::new(offchain.clone())); + ext.register_extension(TransactionPoolExt::new(pool)); + ext.register_extension(OffchainDbExt::new(offchain.clone())); + + ext.execute_with(|| { + System::set_block_number(1); + // sets max iterations value to 1 + offchain.local_storage_set(StorageKind::PERSISTENT, OFFCHAIN_WORKER_MAX_ITERATIONS, &1u32.encode()); + assert_ok!(CDPEngineModule::set_collateral_params( + Origin::signed(1), + BTC, + Change::NewValue(Some(Rate::saturating_from_rational(1, 100000))), + Change::NewValue(Some(Ratio::saturating_from_rational(3, 2))), + Change::NewValue(Some(Rate::saturating_from_rational(2, 10))), + Change::NewValue(Some(Ratio::saturating_from_rational(9, 5))), + Change::NewValue(10000), + )); + + assert_ok!(CDPEngineModule::adjust_position(&ALICE, BTC, 100, 500)); + assert_ok!(CDPEngineModule::adjust_position(&BOB, BTC, 100, 500)); + // make both positions unsafe + assert_ok!(CDPEngineModule::set_collateral_params( + Origin::signed(1), + BTC, + Change::NoChange, + Change::NewValue(Some(Ratio::saturating_from_rational(3, 1))), + Change::NoChange, + Change::NoChange, + Change::NoChange, + )); + run_to_block_offchain(2); + let tx = pool_state.write().transactions.pop().unwrap(); + let tx = Extrinsic::decode(&mut &*tx).unwrap(); + if let MockCall::CDPEngineModule(crate::Call::liquidate(currency_call, who_call)) = tx.call { + assert_ok!(CDPEngineModule::liquidate(Origin::none(), currency_call, who_call)); + } + // alice is liquidated but not bob, he will get liquidated next block due to iteration limit + assert_eq!(LoansModule::positions(BTC, ALICE).debit, 0); + assert_eq!(LoansModule::positions(BTC, ALICE).collateral, 0); + // only one tx is submitted due to iteration limit + assert!(pool_state.write().transactions.pop().is_none()); + + // Iterator continues where it was from storage and now liquidates bob + run_to_block_offchain(3); + let tx = pool_state.write().transactions.pop().unwrap(); + let tx = Extrinsic::decode(&mut &*tx).unwrap(); + if let MockCall::CDPEngineModule(crate::Call::liquidate(currency_call, who_call)) = tx.call { + assert_ok!(CDPEngineModule::liquidate(Origin::none(), currency_call, who_call)); + } + assert_eq!(LoansModule::positions(BTC, BOB).debit, 0); + assert_eq!(LoansModule::positions(BTC, BOB).collateral, 0); + assert!(pool_state.write().transactions.pop().is_none()); + }); +} + +#[test] +fn offchain_default_max_iterator_works() { + let (mut offchain, _offchain_state) = testing::TestOffchainExt::new(); + let (pool, pool_state) = testing::TestTransactionPoolExt::new(); + let mut ext = ExtBuilder::lots_of_accounts().build(); + ext.register_extension(OffchainWorkerExt::new(offchain.clone())); + ext.register_extension(TransactionPoolExt::new(pool)); + ext.register_extension(OffchainDbExt::new(offchain.clone())); + + ext.execute_with(|| { + System::set_block_number(1); + assert_ok!(CDPEngineModule::set_collateral_params( + Origin::signed(1), + BTC, + Change::NewValue(Some(Rate::saturating_from_rational(1, 100000))), + Change::NewValue(Some(Ratio::saturating_from_rational(3, 2))), + Change::NewValue(Some(Rate::saturating_from_rational(2, 10))), + Change::NewValue(Some(Ratio::saturating_from_rational(9, 5))), + Change::NewValue(10000), + )); + // checks that max iterations is stored as none + assert!(offchain + .local_storage_get(StorageKind::PERSISTENT, OFFCHAIN_WORKER_MAX_ITERATIONS) + .is_none()); + + for i in 0..1001 { + let acount_id: AccountId = i; + assert_ok!(CDPEngineModule::adjust_position(&acount_id, BTC, 10, 50)); + } + + // make all positions unsafe + assert_ok!(CDPEngineModule::set_collateral_params( + Origin::signed(1), + BTC, + Change::NoChange, + Change::NewValue(Some(Ratio::saturating_from_rational(3, 1))), + Change::NoChange, + Change::NoChange, + Change::NoChange, + )); + run_to_block_offchain(2); + // should only run 1000 iterations stopping due to DEFAULT_MAX_ITERATIONS + assert_eq!(pool_state.write().transactions.len(), 1000); + // should only now run 1 iteration to finish off where it ended last block + run_to_block_offchain(3); + assert_eq!(pool_state.write().transactions.len(), 1001); + }); +}