Introduces account existence providers reference counting (#7363)

* Initial draft

* Latest changes

* Final bits.

* Fixes

* Fixes

* Test fixes

* Fix tests

* Fix babe tests

* Fix

* Fix

* Fix

* Fix

* Fix

* fix warnings in assets

* Fix UI tests

* fix line width

* Fix

* Update frame/system/src/lib.rs

Co-authored-by: Kian Paimani <5588131+kianenigma@users.noreply.github.com>

* Update frame/system/src/lib.rs

Co-authored-by: Kian Paimani <5588131+kianenigma@users.noreply.github.com>

* Fix

* fix unused warnings

* Fix

* Update frame/system/src/lib.rs

Co-authored-by: Guillaume Thiolliere <gui.thiolliere@gmail.com>

* Update frame/system/src/lib.rs

Co-authored-by: Guillaume Thiolliere <gui.thiolliere@gmail.com>

* Fix

* fix slash and comprehensive slash test

* fix reserved slash and comprehensive tests

* check slash on non-existent account

* Revert "Fix UI tests"

This reverts commit e818dc7.

* Fix

* Fix utility tests

* keep dispatch error backwards compatible

* Fix

* Fix

* fix ui test

* Companion checker shouldn't be so anal.

* Fix

* Fix

* Fix

* Apply suggestions from code review

Co-authored-by: Alexander Popiak <alexander.popiak@parity.io>

* Update frame/balances/src/lib.rs

Co-authored-by: Alexander Popiak <alexander.popiak@parity.io>

* return correct slash info when failing gracefully

* fix missing import

* Update frame/system/src/lib.rs

Co-authored-by: Guillaume Thiolliere <gui.thiolliere@gmail.com>

* Fix

* Update frame/balances/src/tests_local.rs

Co-authored-by: Guillaume Thiolliere <gui.thiolliere@gmail.com>

* Fixes

Co-authored-by: Shawn Tabrizi <shawntabrizi@gmail.com>
Co-authored-by: Kian Paimani <5588131+kianenigma@users.noreply.github.com>
Co-authored-by: Guillaume Thiolliere <gui.thiolliere@gmail.com>
Co-authored-by: Alexander Popiak <alexander.popiak@parity.io>

.maintain/gitlab/check_polkadot_companion_build.sh

@@ -67,8 +67,8 @@ then
   pr_body="$(sed -n -r 's/^[[:space:]]+"body": (".*")[^"]+$/\1/p' "${pr_data_file}")"
 
   pr_companion="$(echo "${pr_body}" | sed -n -r \
-      -e 's;^.*polkadot companion: paritytech/polkadot#([0-9]+).*$;\1;p' \
-      -e 's;^.*polkadot companion: https://github.com/paritytech/polkadot/pull/([0-9]+).*$;\1;p' \
+      -e 's;^.*[Cc]ompanion.*paritytech/polkadot#([0-9]+).*$;\1;p' \
+      -e 's;^.*[Cc]ompanion.*https://github.com/paritytech/polkadot/pull/([0-9]+).*$;\1;p' \
     | tail -n 1)"
 
   if [ "${pr_companion}" ]

.maintain/gitlab/check_polkadot_companion_status.sh

@@ -43,8 +43,8 @@ pr_body="$(curl -H "${github_header}" -s ${github_api_substrate_pull_url}/${CI_C
 
 # get companion if explicitly specified
 pr_companion="$(echo "${pr_body}" | sed -n -r \
-    -e 's;^.*polkadot companion: paritytech/polkadot#([0-9]+).*$;\1;p' \
-    -e 's;^.*polkadot companion: https://github.com/paritytech/polkadot/pull/([0-9]+).*$;\1;p' \
+    -e 's;^.*[Cc]ompanion.*paritytech/polkadot#([0-9]+).*$;\1;p' \
+    -e 's;^.*[Cc]ompanion.*https://github.com/paritytech/polkadot/pull/([0-9]+).*$;\1;p' \
   | tail -n 1)"
 
 if [ -z "${pr_companion}" ]

bin/node/executor/tests/basic.rs

@@ -192,7 +192,7 @@ fn bad_extrinsic_with_native_equivalent_code_gives_error() {
 	let mut t = new_test_ext(compact_code_unwrap(), false);
 	t.insert(
 		<frame_system::Account<Runtime>>::hashed_key_for(alice()),
-		(0u32, 0u32, 69u128, 0u128, 0u128, 0u128).encode()
+		(0u32, 0u32, 0u32, 69u128, 0u128, 0u128, 0u128).encode()
 	);
 	t.insert(<pallet_balances::TotalIssuance<Runtime>>::hashed_key().to_vec(), 69_u128.encode());
 	t.insert(<frame_system::BlockHash<Runtime>>::hashed_key_for(0), vec![0u8; 32]);
@@ -221,11 +221,11 @@ fn successful_execution_with_native_equivalent_code_gives_ok() {
 	let mut t = new_test_ext(compact_code_unwrap(), false);
 	t.insert(
 		<frame_system::Account<Runtime>>::hashed_key_for(alice()),
-		(0u32, 0u32, 111 * DOLLARS, 0u128, 0u128, 0u128).encode()
+		(0u32, 0u32, 0u32, 111 * DOLLARS, 0u128, 0u128, 0u128).encode()
 	);
 	t.insert(
 		<frame_system::Account<Runtime>>::hashed_key_for(bob()),
-		(0u32, 0u32, 0 * DOLLARS, 0u128, 0u128, 0u128).encode()
+		(0u32, 0u32, 0u32, 0 * DOLLARS, 0u128, 0u128, 0u128).encode()
 	);
 	t.insert(
 		<pallet_balances::TotalIssuance<Runtime>>::hashed_key().to_vec(),
@@ -264,11 +264,11 @@ fn successful_execution_with_foreign_code_gives_ok() {
 	let mut t = new_test_ext(bloaty_code_unwrap(), false);
 	t.insert(
 		<frame_system::Account<Runtime>>::hashed_key_for(alice()),
-		(0u32, 0u32, 111 * DOLLARS, 0u128, 0u128, 0u128).encode()
+		(0u32, 0u32, 0u32, 111 * DOLLARS, 0u128, 0u128, 0u128).encode()
 	);
 	t.insert(
 		<frame_system::Account<Runtime>>::hashed_key_for(bob()),
-		(0u32, 0u32, 0 * DOLLARS, 0u128, 0u128, 0u128).encode()
+		(0u32, 0u32, 0u32, 0 * DOLLARS, 0u128, 0u128, 0u128).encode()
 	);
 	t.insert(
 		<pallet_balances::TotalIssuance<Runtime>>::hashed_key().to_vec(),
@@ -702,7 +702,7 @@ fn panic_execution_gives_error() {
 	let mut t = new_test_ext(bloaty_code_unwrap(), false);
 	t.insert(
 		<frame_system::Account<Runtime>>::hashed_key_for(alice()),
-		(0u32, 0u32, 0 * DOLLARS, 0u128, 0u128, 0u128).encode()
+		(0u32, 0u32, 0u32, 0 * DOLLARS, 0u128, 0u128, 0u128).encode()
 	);
 	t.insert(<pallet_balances::TotalIssuance<Runtime>>::hashed_key().to_vec(), 0_u128.encode());
 	t.insert(<frame_system::BlockHash<Runtime>>::hashed_key_for(0), vec![0u8; 32]);
@@ -731,11 +731,11 @@ fn successful_execution_gives_ok() {
 	let mut t = new_test_ext(compact_code_unwrap(), false);
 	t.insert(
 		<frame_system::Account<Runtime>>::hashed_key_for(alice()),
-		(0u32, 0u32, 111 * DOLLARS, 0u128, 0u128, 0u128).encode()
+		(0u32, 0u32, 0u32, 111 * DOLLARS, 0u128, 0u128, 0u128).encode()
 	);
 	t.insert(
 		<frame_system::Account<Runtime>>::hashed_key_for(bob()),
-		(0u32, 0u32, 0 * DOLLARS, 0u128, 0u128, 0u128).encode()
+		(0u32, 0u32, 0u32, 0 * DOLLARS, 0u128, 0u128, 0u128).encode()
 	);
 	t.insert(
 		<pallet_balances::TotalIssuance<Runtime>>::hashed_key().to_vec(),

bin/node/executor/tests/fees.rs

@@ -121,6 +121,15 @@ fn fee_multiplier_increases_and_decreases_on_big_weight() {
 	});
 }
 
+fn new_account_info(free_dollars: u128) -> Vec<u8> {
+	frame_system::AccountInfo {
+		nonce: 0u32,
+		consumers: 0,
+		providers: 0,
+		data: (free_dollars * DOLLARS, 0 * DOLLARS, 0 * DOLLARS, 0 * DOLLARS),
+	}.encode()
+}
+
 #[test]
 fn transaction_fee_is_correct() {
 	// This uses the exact values of substrate-node.
@@ -131,14 +140,8 @@ fn transaction_fee_is_correct() {
 	//   - 1 milli-dot based on current polkadot runtime.
 	// (this baed on assigning 0.1 CENT to the cheapest tx with `weight = 100`)
 	let mut t = new_test_ext(compact_code_unwrap(), false);
-	t.insert(
-		<frame_system::Account<Runtime>>::hashed_key_for(alice()),
-		(0u32, 0u32, 100 * DOLLARS, 0 * DOLLARS, 0 * DOLLARS, 0 * DOLLARS).encode()
-	);
-	t.insert(
-		<frame_system::Account<Runtime>>::hashed_key_for(bob()),
-		(0u32, 0u32, 10 * DOLLARS, 0 * DOLLARS, 0 * DOLLARS, 0 * DOLLARS).encode()
-	);
+	t.insert(<frame_system::Account<Runtime>>::hashed_key_for(alice()), new_account_info(100));
+	t.insert(<frame_system::Account<Runtime>>::hashed_key_for(bob()), new_account_info(10));
 	t.insert(
 		<pallet_balances::TotalIssuance<Runtime>>::hashed_key().to_vec(),
 		(110 * DOLLARS).encode()

bin/node/executor/tests/submit_transaction.rs

@@ -253,7 +253,7 @@ fn submitted_transaction_should_be_valid() {
 		let author = extrinsic.signature.clone().unwrap().0;
 		let address = Indices::lookup(author).unwrap();
 		let data = pallet_balances::AccountData { free: 5_000_000_000_000, ..Default::default() };
-		let account = frame_system::AccountInfo { nonce: 0, refcount: 0, data };
+		let account = frame_system::AccountInfo { nonce: 0, consumers: 0, providers: 0, data };
 		<frame_system::Account<Runtime>>::insert(&address, account);
 
 		// check validity

frame/assets/src/lib.rs

@@ -274,6 +274,8 @@ decl_error! {
 		MinBalanceZero,
 		/// A mint operation lead to an overflow.
 		Overflow,
+		/// Some internal state is broken.
+		BadState,
 	}
 }
 
@@ -863,7 +865,7 @@ impl<T: Config> Module<T> {
 	) -> Result<bool, DispatchError> {
 		let accounts = d.accounts.checked_add(1).ok_or(Error::<T>::Overflow)?;
 		let r = Ok(if frame_system::Module::<T>::account_exists(who) {
-			frame_system::Module::<T>::inc_ref(who);
+			frame_system::Module::<T>::inc_consumers(who).map_err(|_| Error::<T>::BadState)?;
 			false
 		} else {
 			ensure!(d.zombies < d.max_zombies, Error::<T>::TooManyZombies);
@@ -881,7 +883,9 @@ impl<T: Config> Module<T> {
 		is_zombie: &mut bool,
 	) {
 		if *is_zombie && frame_system::Module::<T>::account_exists(who) {
-			frame_system::Module::<T>::inc_ref(who);
+			// If the account exists, then it should have at least one provider
+			// so this cannot fail... but being defensive anyway.
+			let _ = frame_system::Module::<T>::inc_consumers(who);
 			*is_zombie = false;
 			d.zombies = d.zombies.saturating_sub(1);
 		}
@@ -895,7 +899,7 @@ impl<T: Config> Module<T> {
 		if is_zombie {
 			d.zombies = d.zombies.saturating_sub(1);
 		} else {
-			frame_system::Module::<T>::dec_ref(who);
+			frame_system::Module::<T>::dec_consumers(who);
 		}
 		d.accounts = d.accounts.saturating_sub(1);
 	}

frame/babe/src/mock.rs

@@ -379,6 +379,14 @@ pub fn new_test_ext_raw_authorities(authorities: Vec<AuthorityId>) -> sp_io::Tes
 		.build_storage::<Test>()
 		.unwrap();
 
+	let balances: Vec<_> = (0..authorities.len())
+		.map(|i| (i as u64, 10_000_000))
+		.collect();
+
+	pallet_balances::GenesisConfig::<Test> { balances }
+		.assimilate_storage(&mut t)
+		.unwrap();
+
 	// stashes are the index.
 	let session_keys: Vec<_> = authorities
 		.iter()
@@ -394,6 +402,12 @@ pub fn new_test_ext_raw_authorities(authorities: Vec<AuthorityId>) -> sp_io::Tes
 		})
 		.collect();
 
+	// NOTE: this will initialize the babe authorities
+	// through OneSessionHandler::on_genesis_session
+	pallet_session::GenesisConfig::<Test> { keys: session_keys }
+		.assimilate_storage(&mut t)
+		.unwrap();
+
 	// controllers are the index + 1000
 	let stakers: Vec<_> = (0..authorities.len())
 		.map(|i| {
@@ -406,20 +420,6 @@ pub fn new_test_ext_raw_authorities(authorities: Vec<AuthorityId>) -> sp_io::Tes
 		})
 		.collect();
 
-	let balances: Vec<_> = (0..authorities.len())
-		.map(|i| (i as u64, 10_000_000))
-		.collect();
-
-	// NOTE: this will initialize the babe authorities
-	// through OneSessionHandler::on_genesis_session
-	pallet_session::GenesisConfig::<Test> { keys: session_keys }
-		.assimilate_storage(&mut t)
-		.unwrap();
-
-	pallet_balances::GenesisConfig::<Test> { balances }
-		.assimilate_storage(&mut t)
-		.unwrap();
-
 	let staking_config = pallet_staking::GenesisConfig::<Test> {
 		stakers,
 		validator_count: 8,