Fix offchain workers and add tests

modules/auction-manager/src/lib.rs

@@ -352,7 +352,7 @@ impl<T: Config> Pallet<T> {
 		let max_iterations = StorageValueRef::persistent(OFFCHAIN_WORKER_MAX_ITERATIONS)
 			.get::<u32>()
 			.unwrap_or(Some(DEFAULT_MAX_ITERATIONS))
-			.ok_or(OffchainErr::OffchainStore)?;
+			.unwrap_or(DEFAULT_MAX_ITERATIONS);
 
 		log::debug!(
 			target: "auction-manager",
@@ -361,7 +361,11 @@ impl<T: Config> Pallet<T> {
 		);
 
 		// start iterations to cancel collateral auctions
-		let mut iterator = <CollateralAuctions<T>>::iter_from(start_key.ok_or(OffchainErr::OffchainStore)?);
+		let mut iterator = match start_key {
+			Some(key) => <CollateralAuctions<T>>::iter_from(key),
+			None => <CollateralAuctions<T>>::iter(),
+		};
+
 		let mut iteration_count = 0;
 		let mut finished = true;
 
@@ -390,7 +394,7 @@ impl<T: Config> Pallet<T> {
 		if finished {
 			to_be_continue.clear();
 		} else {
-			to_be_continue.set(&iterator.last_raw_key());
+			to_be_continue.set(&iterator.prefix());
 		}
 
 		// Consume the guard but **do not** unlock the underlying lock.

modules/auction-manager/src/tests.rs

@@ -22,9 +22,20 @@
 
 use super::*;
 use frame_support::{assert_noop, assert_ok};
-use mock::{Event, *};
+use mock::{Call as MockCall, Event, *};
+use sp_core::offchain::{testing, DbExternalities, OffchainDbExt, OffchainWorkerExt, StorageKind, TransactionPoolExt};
+use sp_io::offchain;
 use sp_runtime::traits::One;
 
+fn run_to_block_offchain(n: u64) {
+	while System::block_number() < n {
+		System::set_block_number(System::block_number() + 1);
+		AuctionManagerModule::offchain_worker(System::block_number());
+		// this unlocks the concurrency storage lock so offchain_worker will fire next block
+		offchain::sleep_until(offchain::timestamp().add(Duration::from_millis(LOCK_DURATION + 200)));
+	}
+}
+
 #[test]
 fn get_auction_time_to_close_work() {
 	ExtBuilder::default().build().execute_with(|| {
@@ -471,3 +482,84 @@ fn cancel_collateral_auction_work() {
 		assert_eq!(bob_ref_count_1, bob_ref_count_0 - 1);
 	});
 }
+
+#[test]
+fn offchain_worker_cancels_auction_in_shutdown() {
+	let (offchain, _offchain_state) = testing::TestOffchainExt::new();
+	let (pool, pool_state) = testing::TestTransactionPoolExt::new();
+	let mut ext = ExtBuilder::default().build();
+	ext.register_extension(OffchainWorkerExt::new(offchain.clone()));
+	ext.register_extension(TransactionPoolExt::new(pool));
+	ext.register_extension(OffchainDbExt::new(offchain.clone()));
+
+	ext.execute_with(|| {
+		System::set_block_number(1);
+		assert_ok!(AuctionManagerModule::new_collateral_auction(&ALICE, BTC, 10, 100));
+		assert!(AuctionManagerModule::collateral_auctions(0).is_some());
+		run_to_block_offchain(2);
+		// offchain worker does not have any tx because shutdown is false
+		assert!(!MockEmergencyShutdown::is_shutdown());
+		assert!(pool_state.write().transactions.pop().is_none());
+		mock_shutdown();
+		assert!(MockEmergencyShutdown::is_shutdown());
+
+		// now offchain worker will cancel auction as shutdown is true
+		run_to_block_offchain(3);
+		let tx = pool_state.write().transactions.pop().unwrap();
+		let tx = Extrinsic::decode(&mut &*tx).unwrap();
+		if let MockCall::AuctionManagerModule(crate::Call::cancel(auction_id)) = tx.call {
+			assert_ok!(AuctionManagerModule::cancel(Origin::none(), auction_id));
+		}
+
+		// auction is canceled
+		assert!(AuctionManagerModule::collateral_auctions(0).is_none());
+		assert!(pool_state.write().transactions.pop().is_none());
+	});
+}
+
+#[test]
+fn offchain_worker_max_iterations_check() {
+	let (mut offchain, _offchain_state) = testing::TestOffchainExt::new();
+	let (pool, pool_state) = testing::TestTransactionPoolExt::new();
+	let mut ext = ExtBuilder::default().build();
+	ext.register_extension(OffchainWorkerExt::new(offchain.clone()));
+	ext.register_extension(TransactionPoolExt::new(pool));
+	ext.register_extension(OffchainDbExt::new(offchain.clone()));
+
+	ext.execute_with(|| {
+		System::set_block_number(1);
+		// sets max iterations value to 1
+		offchain.local_storage_set(StorageKind::PERSISTENT, OFFCHAIN_WORKER_MAX_ITERATIONS, &1u32.encode());
+		assert_ok!(AuctionManagerModule::new_collateral_auction(&ALICE, BTC, 10, 100));
+		assert_ok!(AuctionManagerModule::new_collateral_auction(&BOB, BTC, 10, 100));
+		assert!(AuctionManagerModule::collateral_auctions(1).is_some());
+		assert!(AuctionManagerModule::collateral_auctions(0).is_some());
+		mock_shutdown();
+		assert!(MockEmergencyShutdown::is_shutdown());
+
+		run_to_block_offchain(2);
+		// now offchain worker will cancel one auction but the other one will cancel next block
+		let tx = pool_state.write().transactions.pop().unwrap();
+		let tx = Extrinsic::decode(&mut &*tx).unwrap();
+		if let MockCall::AuctionManagerModule(crate::Call::cancel(auction_id)) = tx.call {
+			assert_ok!(AuctionManagerModule::cancel(Origin::none(), auction_id));
+		}
+		// only one auction canceled so offchain tx pool is empty
+		assert!(
+			AuctionManagerModule::collateral_auctions(1).is_some()
+				|| AuctionManagerModule::collateral_auctions(0).is_some()
+		);
+		assert!(pool_state.write().transactions.pop().is_none());
+
+		run_to_block_offchain(3);
+		// now offchain worker will cancel one auction but the other one will cancel next block
+		let tx = pool_state.write().transactions.pop().unwrap();
+		let tx = Extrinsic::decode(&mut &*tx).unwrap();
+		if let MockCall::AuctionManagerModule(crate::Call::cancel(auction_id)) = tx.call {
+			assert_ok!(AuctionManagerModule::cancel(Origin::none(), auction_id));
+		}
+		assert!(AuctionManagerModule::collateral_auctions(1).is_none());
+		assert!(AuctionManagerModule::collateral_auctions(0).is_none());
+		assert!(pool_state.write().transactions.pop().is_none());
+	});
+}

modules/cdp-engine/src/lib.rs

@@ -649,12 +649,17 @@ impl<T: Config> Pallet<T> {
 		let max_iterations = StorageValueRef::persistent(OFFCHAIN_WORKER_MAX_ITERATIONS)
 			.get::<u32>()
 			.unwrap_or(Some(DEFAULT_MAX_ITERATIONS))
-			.ok_or(OffchainErr::OffchainStore)?;
+			.unwrap_or(DEFAULT_MAX_ITERATIONS);
 
 		let currency_id = collateral_currency_ids[collateral_position as usize];
 		let is_shutdown = T::EmergencyShutdown::is_shutdown();
-		let mut map_iterator =
-			<loans::Positions<T>>::iter_prefix_from(currency_id, start_key.clone().ok_or(OffchainErr::OffchainStore)?);
+
+		// If start key is Some(value) continue iterating from that point in storage otherwise start
+		// iterating from the beginning of <loans::Positons<T>>
+		let mut map_iterator = match start_key.clone() {
+			Some(key) => <loans::Positions<T>>::iter_prefix_from(currency_id, key),
+			None => <loans::Positions<T>>::iter_prefix(currency_id),
+		};
 
 		let mut finished = true;
 		let mut iteration_count = 0;
@@ -708,7 +713,7 @@ impl<T: Config> Pallet<T> {
 				};
 			to_be_continue.set(&(next_collateral_position, Option::<Vec<u8>>::None));
 		} else {
-			to_be_continue.set(&(collateral_position, Some(map_iterator.last_raw_key())));
+			to_be_continue.set(&(collateral_position, Some(map_iterator.prefix())));
 		}
 
 		// Consume the guard but **do not** unlock the underlying lock.

modules/cdp-engine/src/tests.rs

@@ -22,11 +22,30 @@
 
 use super::*;
 use frame_support::{assert_noop, assert_ok};
-use mock::{Event, *};
+use mock::{Call as MockCall, Event, *};
 use orml_traits::MultiCurrency;
-use sp_runtime::traits::BadOrigin;
+use sp_core::offchain::{testing, OffchainDbExt, OffchainWorkerExt, TransactionPoolExt};
+use sp_io::offchain;
+use sp_runtime::{
+	offchain::{DbExternalities, StorageKind},
+	traits::BadOrigin,
+};
 use support::DEXManager;
 
+pub const INIT_TIMESTAMP: u64 = 30_000;
+pub const BLOCK_TIME: u64 = 1000;
+
+fn run_to_block_offchain(n: u64) {
+	while System::block_number() < n {
+		System::set_block_number(System::block_number() + 1);
+		Timestamp::set_timestamp((System::block_number() as u64 * BLOCK_TIME) + INIT_TIMESTAMP);
+		CDPEngineModule::on_initialize(System::block_number());
+		CDPEngineModule::offchain_worker(System::block_number());
+		// this unlocks the concurrency storage lock so offchain_worker will fire next block
+		offchain::sleep_until(offchain::timestamp().add(Duration::from_millis(LOCK_DURATION + 200)));
+	}
+}
+
 #[test]
 fn check_cdp_status_work() {
 	ExtBuilder::default().build().execute_with(|| {
@@ -922,3 +941,143 @@ fn close_cdp_has_debit_by_swap_on_alternative_path() {
 		assert_eq!(CDPTreasuryModule::get_debit_pool(), 50);
 	});
 }
+
+#[test]
+fn offchain_worker_works_cdp() {
+	let (offchain, _offchain_state) = testing::TestOffchainExt::new();
+	let (pool, pool_state) = testing::TestTransactionPoolExt::new();
+	let mut ext = ExtBuilder::default().build();
+	ext.register_extension(OffchainWorkerExt::new(offchain.clone()));
+	ext.register_extension(TransactionPoolExt::new(pool));
+	ext.register_extension(OffchainDbExt::new(offchain.clone()));
+
+	ext.execute_with(|| {
+		// number of currencies allowed as collateral (cycles through all of them)
+		let collateral_currencies_num = CollateralCurrencyIds::get().len() as u64;
+		System::set_block_number(1);
+		assert_ok!(CDPEngineModule::set_collateral_params(
+			Origin::signed(1),
+			BTC,
+			Change::NewValue(Some(Rate::saturating_from_rational(1, 100000))),
+			Change::NewValue(Some(Ratio::saturating_from_rational(3, 2))),
+			Change::NewValue(Some(Rate::saturating_from_rational(2, 10))),
+			Change::NewValue(Some(Ratio::saturating_from_rational(9, 5))),
+			Change::NewValue(10000),
+		));
+
+		// offchain worker will not liquidate alice
+		assert_ok!(CDPEngineModule::adjust_position(&ALICE, BTC, 100, 500));
+		assert_ok!(CDPEngineModule::adjust_position(&BOB, BTC, 100, 100));
+		assert_eq!(Currencies::free_balance(BTC, &ALICE), 900);
+		assert_eq!(Currencies::free_balance(AUSD, &ALICE), 50);
+		assert_eq!(LoansModule::positions(BTC, ALICE).debit, 500);
+		assert_eq!(LoansModule::positions(BTC, ALICE).collateral, 100);
+		// jump 2 blocks at a time because code rotates through the different T::CollateralCurrencyIds
+		run_to_block_offchain(System::block_number() + collateral_currencies_num);
+
+		// checks that offchain worker tx pool is empty (therefore tx to liquidate alice is not present)
+		assert!(pool_state.write().transactions.pop().is_none());
+		assert_eq!(LoansModule::positions(BTC, ALICE).debit, 500);
+		assert_eq!(LoansModule::positions(BTC, ALICE).collateral, 100);
+
+		// changes alice into unsafe position
+		assert_ok!(CDPEngineModule::set_collateral_params(
+			Origin::signed(1),
+			BTC,
+			Change::NoChange,
+			Change::NewValue(Some(Ratio::saturating_from_rational(3, 1))),
+			Change::NoChange,
+			Change::NoChange,
+			Change::NoChange,
+		));
+		run_to_block_offchain(System::block_number() + collateral_currencies_num);
+
+		// offchain worker will liquidate alice
+		let tx = pool_state.write().transactions.pop().unwrap();
+		let tx = Extrinsic::decode(&mut &*tx).unwrap();
+		if let MockCall::CDPEngineModule(crate::Call::liquidate(currency_call, who_call)) = tx.call {
+			assert_ok!(CDPEngineModule::liquidate(Origin::none(), currency_call, who_call));
+		}
+		// empty offchain tx pool (Bob was not liquidated)
+		assert!(pool_state.write().transactions.pop().is_none());
+		// alice is liquidated but bob is not
+		assert_eq!(LoansModule::positions(BTC, ALICE).debit, 0);
+		assert_eq!(LoansModule::positions(BTC, ALICE).collateral, 0);
+		assert_eq!(LoansModule::positions(BTC, BOB).debit, 100);
+		assert_eq!(LoansModule::positions(BTC, BOB).collateral, 100);
+
+		// emergency shutdown will settle Bobs debit position
+		mock_shutdown();
+		assert!(MockEmergencyShutdown::is_shutdown());
+		run_to_block_offchain(System::block_number() + collateral_currencies_num);
+		// offchain worker will settle bob's position
+		let tx = pool_state.write().transactions.pop().unwrap();
+		let tx = Extrinsic::decode(&mut &*tx).unwrap();
+		if let MockCall::CDPEngineModule(crate::Call::settle(currency_call, who_call)) = tx.call {
+			assert_ok!(CDPEngineModule::settle(Origin::none(), currency_call, who_call));
+		}
+		// emergency shutdown settles bob's debit position
+		assert_eq!(LoansModule::positions(BTC, BOB).debit, 0);
+		assert_eq!(LoansModule::positions(BTC, BOB).collateral, 90);
+	});
+}
+
+#[test]
+fn offchain_worker_iteration_limit_works() {
+	let (mut offchain, _offchain_state) = testing::TestOffchainExt::new();
+	let (pool, pool_state) = testing::TestTransactionPoolExt::new();
+	let mut ext = ExtBuilder::default().build();
+	ext.register_extension(OffchainWorkerExt::new(offchain.clone()));
+	ext.register_extension(TransactionPoolExt::new(pool));
+	ext.register_extension(OffchainDbExt::new(offchain.clone()));
+
+	ext.execute_with(|| {
+		System::set_block_number(1);
+		// sets max iterations value to 1
+		offchain.local_storage_set(StorageKind::PERSISTENT, OFFCHAIN_WORKER_MAX_ITERATIONS, &1u32.encode());
+		assert_ok!(CDPEngineModule::set_collateral_params(
+			Origin::signed(1),
+			BTC,
+			Change::NewValue(Some(Rate::saturating_from_rational(1, 100000))),
+			Change::NewValue(Some(Ratio::saturating_from_rational(3, 2))),
+			Change::NewValue(Some(Rate::saturating_from_rational(2, 10))),
+			Change::NewValue(Some(Ratio::saturating_from_rational(9, 5))),
+			Change::NewValue(10000),
+		));
+
+		assert_ok!(CDPEngineModule::adjust_position(&ALICE, BTC, 100, 500));
+		assert_ok!(CDPEngineModule::adjust_position(&BOB, BTC, 100, 500));
+		// make both positions unsafe
+		assert_ok!(CDPEngineModule::set_collateral_params(
+			Origin::signed(1),
+			BTC,
+			Change::NoChange,
+			Change::NewValue(Some(Ratio::saturating_from_rational(3, 1))),
+			Change::NoChange,
+			Change::NoChange,
+			Change::NoChange,
+		));
+		run_to_block_offchain(2);
+		let tx = pool_state.write().transactions.pop().unwrap();
+		let tx = Extrinsic::decode(&mut &*tx).unwrap();
+		if let MockCall::CDPEngineModule(crate::Call::liquidate(currency_call, who_call)) = tx.call {
+			assert_ok!(CDPEngineModule::liquidate(Origin::none(), currency_call, who_call));
+		}
+		// alice is liquidated but not bob, he will get liquidated next block due to iteration limit
+		assert_eq!(LoansModule::positions(BTC, ALICE).debit, 0);
+		assert_eq!(LoansModule::positions(BTC, ALICE).collateral, 0);
+		// only one tx is submitted due to iteration limit
+		assert!(pool_state.write().transactions.pop().is_none());
+
+		// Iterator continues where it was from storage and now liquidates bob
+		run_to_block_offchain(3);
+		let tx = pool_state.write().transactions.pop().unwrap();
+		let tx = Extrinsic::decode(&mut &*tx).unwrap();
+		if let MockCall::CDPEngineModule(crate::Call::liquidate(currency_call, who_call)) = tx.call {
+			assert_ok!(CDPEngineModule::liquidate(Origin::none(), currency_call, who_call));
+		}
+		assert_eq!(LoansModule::positions(BTC, BOB).debit, 0);
+		assert_eq!(LoansModule::positions(BTC, BOB).collateral, 0);
+		assert!(pool_state.write().transactions.pop().is_none());
+	});
+}